-4

arch/x86/Kconfig.assembler

reviewed

··· 15 15 def_bool $(as-instr,sha256msg1 %xmm0$(comma)%xmm1) 16 16 help 17 17 Supported by binutils >= 2.24 and LLVM integrated assembler 18 18 - config AS_TPAUSE 19 19 - def_bool $(as-instr,tpause %ecx) 20 20 - help 21 21 - Supported by binutils >= 2.31.1 and LLVM integrated assembler >= V7 22 18 23 19 config AS_GFNI 24 20 def_bool $(as-instr,vgf2p8mulb %xmm0$(comma)%xmm1$(comma)%xmm2)

+31 -41

arch/x86/include/asm/mwait.h

reviewed

··· 25 25 #define TPAUSE_C01_STATE 1 26 26 #define TPAUSE_C02_STATE 0 27 27 28 28 - static __always_inline void __monitor(const void *eax, unsigned long ecx, 29 29 - unsigned long edx) 28 28 + static __always_inline void __monitor(const void *eax, u32 ecx, u32 edx) 30 29 { 31 31 - /* "monitor %eax, %ecx, %edx;" */ 32 32 - asm volatile(".byte 0x0f, 0x01, 0xc8;" 30 30 + /* 31 31 + * Use the instruction mnemonic with implicit operands, as the LLVM 32 32 + * assembler fails to assemble the mnemonic with explicit operands: 33 33 + */ 34 34 + asm volatile("monitor" :: "a" (eax), "c" (ecx), "d" (edx)); 35 35 + } 36 36 + 37 37 + static __always_inline void __monitorx(const void *eax, u32 ecx, u32 edx) 38 38 + { 39 39 + /* "monitorx %eax, %ecx, %edx" */ 40 40 + asm volatile(".byte 0x0f, 0x01, 0xfa" 33 41 :: "a" (eax), "c" (ecx), "d"(edx)); 34 42 } 35 43 36 36 - static __always_inline void __monitorx(const void *eax, unsigned long ecx, 37 37 - unsigned long edx) 38 38 - { 39 39 - /* "monitorx %eax, %ecx, %edx;" */ 40 40 - asm volatile(".byte 0x0f, 0x01, 0xfa;" 41 41 - :: "a" (eax), "c" (ecx), "d"(edx)); 42 42 - } 43 43 - 44 44 - static __always_inline void __mwait(unsigned long eax, unsigned long ecx) 44 44 + static __always_inline void __mwait(u32 eax, u32 ecx) 45 45 { 46 46 mds_idle_clear_cpu_buffers(); 47 47 48 48 - /* "mwait %eax, %ecx;" */ 49 49 - asm volatile(".byte 0x0f, 0x01, 0xc9;" 50 50 - :: "a" (eax), "c" (ecx)); 48 48 + /* 49 49 + * Use the instruction mnemonic with implicit operands, as the LLVM 50 50 + * assembler fails to assemble the mnemonic with explicit operands: 51 51 + */ 52 52 + asm volatile("mwait" :: "a" (eax), "c" (ecx)); 51 53 } 52 54 53 55 /* ··· 78 76 * EAX (logical) address to monitor 79 77 * ECX #GP if not zero 80 78 */ 81 81 - static __always_inline void __mwaitx(unsigned long eax, unsigned long ebx, 82 82 - unsigned long ecx) 79 79 + static __always_inline void __mwaitx(u32 eax, u32 ebx, u32 ecx) 83 80 { 84 81 /* No MDS buffer clear as this is AMD/HYGON only */ 85 82 86 86 - /* "mwaitx %eax, %ebx, %ecx;" */ 87 87 - asm volatile(".byte 0x0f, 0x01, 0xfb;" 83 83 + /* "mwaitx %eax, %ebx, %ecx" */ 84 84 + asm volatile(".byte 0x0f, 0x01, 0xfb" 88 85 :: "a" (eax), "b" (ebx), "c" (ecx)); 89 86 } 90 87 ··· 96 95 * executing mwait, it would otherwise go unnoticed and the next tick 97 96 * would not be reprogrammed accordingly before mwait ever wakes up. 98 97 */ 99 99 - static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx) 98 98 + static __always_inline void __sti_mwait(u32 eax, u32 ecx) 100 99 { 101 100 mds_idle_clear_cpu_buffers(); 102 102 - /* "mwait %eax, %ecx;" */ 103 103 - asm volatile("sti; .byte 0x0f, 0x01, 0xc9;" 104 104 - :: "a" (eax), "c" (ecx)); 101 101 + 102 102 + asm volatile("sti; mwait" :: "a" (eax), "c" (ecx)); 105 103 } 106 104 107 105 /* ··· 113 113 * New with Core Duo processors, MWAIT can take some hints based on CPU 114 114 * capability. 115 115 */ 116 116 - static __always_inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx) 116 116 + static __always_inline void mwait_idle_with_hints(u32 eax, u32 ecx) 117 117 { 118 118 if (static_cpu_has_bug(X86_BUG_MONITOR) || !current_set_polling_and_test()) { 119 119 - if (static_cpu_has_bug(X86_BUG_CLFLUSH_MONITOR)) { 120 120 - mb(); 121 121 - clflush((void *)&current_thread_info()->flags); 122 122 - mb(); 123 123 - } 119 119 + const void *addr = &current_thread_info()->flags; 124 120 125 125 - __monitor((void *)&current_thread_info()->flags, 0, 0); 121 121 + alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr)); 122 122 + __monitor(addr, 0, 0); 126 123 127 124 if (!need_resched()) { 128 125 if (ecx & 1) { ··· 141 144 */ 142 145 static inline void __tpause(u32 ecx, u32 edx, u32 eax) 143 146 { 144 144 - /* "tpause %ecx, %edx, %eax;" */ 145 145 - #ifdef CONFIG_AS_TPAUSE 146 146 - asm volatile("tpause %%ecx\n" 147 147 - : 148 148 - : "c"(ecx), "d"(edx), "a"(eax)); 149 149 - #else 150 150 - asm volatile(".byte 0x66, 0x0f, 0xae, 0xf1\t\n" 151 151 - : 152 152 - : "c"(ecx), "d"(edx), "a"(eax)); 153 153 - #endif 147 147 + /* "tpause %ecx" */ 148 148 + asm volatile(".byte 0x66, 0x0f, 0xae, 0xf1" 149 149 + :: "c" (ecx), "d" (edx), "a" (eax)); 154 150 } 155 151 156 152 #endif /* _ASM_X86_MWAIT_H */

-1

arch/x86/include/asm/page_32_types.h

reviewed

··· 73 73 extern int sysctl_legacy_va_layout; 74 74 75 75 extern void find_low_pfn_range(void); 76 76 - extern void setup_bootmem_allocator(void); 77 76 78 77 #endif /* !__ASSEMBLER__ */ 79 78

-2

arch/x86/include/asm/pgtable-2level_types.h

reviewed

··· 18 18 } pte_t; 19 19 #endif /* !__ASSEMBLER__ */ 20 20 21 21 - #define SHARED_KERNEL_PMD 0 22 22 - 23 21 #define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED 24 22 25 23 /*

+1 -3

arch/x86/include/asm/pgtable-3level_types.h

reviewed

··· 27 27 } pmd_t; 28 28 #endif /* !__ASSEMBLER__ */ 29 29 30 30 - #define SHARED_KERNEL_PMD (!static_cpu_has(X86_FEATURE_PTI)) 31 31 - 32 32 - #define ARCH_PAGE_TABLE_SYNC_MASK (SHARED_KERNEL_PMD ? 0 : PGTBL_PMD_MODIFIED) 30 30 + #define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED 33 31 34 32 /* 35 33 * PGDIR_SHIFT determines what a top-level page table entry can map

-10

arch/x86/include/asm/pgtable.h

reviewed

··· 292 292 return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT; 293 293 } 294 294 295 295 - #define p4d_leaf p4d_leaf 296 296 - static inline bool p4d_leaf(p4d_t p4d) 297 297 - { 298 298 - /* No 512 GiB pages yet */ 299 299 - return 0; 300 300 - } 301 301 - 302 295 #define pte_page(pte) pfn_to_page(pte_pfn(pte)) 303 296 304 297 #define pmd_leaf pmd_leaf ··· 1464 1471 1465 1472 return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START); 1466 1473 } 1467 1467 - 1468 1468 - #define pgd_leaf pgd_leaf 1469 1469 - static inline bool pgd_leaf(pgd_t pgd) { return false; } 1470 1474 1471 1475 #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION 1472 1476 /*

-2

arch/x86/include/asm/pgtable_64_types.h

reviewed

··· 46 46 47 47 #endif /* !__ASSEMBLER__ */ 48 48 49 49 - #define SHARED_KERNEL_PMD 0 50 50 - 51 49 #ifdef CONFIG_X86_5LEVEL 52 50 53 51 /*

+3 -6

arch/x86/kernel/process.c

reviewed

··· 901 901 static __cpuidle void mwait_idle(void) 902 902 { 903 903 if (!current_set_polling_and_test()) { 904 904 - if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) { 905 905 - mb(); /* quirk */ 906 906 - clflush((void *)&current_thread_info()->flags); 907 907 - mb(); /* quirk */ 908 908 - } 904 904 + const void *addr = &current_thread_info()->flags; 909 905 910 910 - __monitor((void *)&current_thread_info()->flags, 0, 0); 906 906 + alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr)); 907 907 + __monitor(addr, 0, 0); 911 908 if (!need_resched()) { 912 909 __sti_mwait(0, 0); 913 910 raw_local_irq_disable();

+1 -1

arch/x86/mm/Makefile

reviewed

··· 47 47 mmiotrace-y := kmmio.o pf_in.o mmio-mod.o 48 48 obj-$(CONFIG_MMIOTRACE_TEST) += testmmiotrace.o 49 49 50 50 - obj-$(CONFIG_NUMA) += numa.o numa_$(BITS).o 50 50 + obj-$(CONFIG_NUMA) += numa.o 51 51 obj-$(CONFIG_AMD_NUMA) += amdtopology.o 52 52 obj-$(CONFIG_ACPI_NUMA) += srat.o 53 53

-3

arch/x86/mm/fault.c

reviewed

··· 13 13 #include <linux/mmiotrace.h> /* kmmio_handler, ... */ 14 14 #include <linux/perf_event.h> /* perf_sw_event */ 15 15 #include <linux/hugetlb.h> /* hstate_index_to_shift */ 16 16 - #include <linux/prefetch.h> /* prefetchw */ 17 16 #include <linux/context_tracking.h> /* exception_enter(), ... */ 18 17 #include <linux/uaccess.h> /* faulthandler_disabled() */ 19 18 #include <linux/efi.h> /* efi_crash_gracefully_on_page_fault()*/ ··· 1494 1495 unsigned long address; 1495 1496 1496 1497 address = cpu_feature_enabled(X86_FEATURE_FRED) ? fred_event_data(regs) : read_cr2(); 1497 1497 - 1498 1498 - prefetchw(&current->mm->mmap_lock); 1499 1498 1500 1499 /* 1501 1500 * KVM uses #PF vector to deliver 'page not present' events to guests

-7

arch/x86/mm/init_32.c

reviewed

··· 612 612 highmem_pfn_init(); 613 613 } 614 614 615 615 - #ifndef CONFIG_NUMA 616 615 void __init initmem_init(void) 617 616 { 618 617 #ifdef CONFIG_HIGHMEM ··· 632 633 printk(KERN_NOTICE "%ldMB LOWMEM available.\n", 633 634 pages_to_mb(max_low_pfn)); 634 635 635 635 - setup_bootmem_allocator(); 636 636 - } 637 637 - #endif /* !CONFIG_NUMA */ 638 638 - 639 639 - void __init setup_bootmem_allocator(void) 640 640 - { 641 636 printk(KERN_INFO " mapped low ram: 0 - %08lx\n", 642 637 max_pfn_mapped<<PAGE_SHIFT); 643 638 printk(KERN_INFO " low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT);

+6 -1

arch/x86/mm/init_64.c

reviewed

··· 805 805 } 806 806 807 807 #ifndef CONFIG_NUMA 808 808 - void __init initmem_init(void) 808 808 + static inline void x86_numa_init(void) 809 809 { 810 810 memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0); 811 811 } 812 812 #endif 813 813 + 814 814 + void __init initmem_init(void) 815 815 + { 816 816 + x86_numa_init(); 817 817 + } 813 818 814 819 void __init paging_init(void) 815 820 {

+4

arch/x86/mm/mm_internal.h

reviewed

··· 25 25 26 26 extern unsigned long tlb_single_page_flush_ceiling; 27 27 28 28 + #ifdef CONFIG_NUMA 29 29 + void __init x86_numa_init(void); 30 30 + #endif 31 31 + 28 32 #endif /* __X86_MM_INTERNAL_H */

+2 -1

arch/x86/mm/numa.c

reviewed

··· 18 18 #include <asm/e820/api.h> 19 19 #include <asm/proto.h> 20 20 #include <asm/dma.h> 21 21 + #include <asm/numa.h> 21 22 #include <asm/amd/nb.h> 22 23 23 23 - #include "numa_internal.h" 24 24 + #include "mm_internal.h" 24 25 25 26 int numa_off; 26 27

-61

arch/x86/mm/numa_32.c

reviewed

··· 1 1 - /* 2 2 - * Written by: Patricia Gaughen <gone@us.ibm.com>, IBM Corporation 3 3 - * August 2002: added remote node KVA remap - Martin J. Bligh 4 4 - * 5 5 - * Copyright (C) 2002, IBM Corp. 6 6 - * 7 7 - * All rights reserved. 8 8 - * 9 9 - * This program is free software; you can redistribute it and/or modify 10 10 - * it under the terms of the GNU General Public License as published by 11 11 - * the Free Software Foundation; either version 2 of the License, or 12 12 - * (at your option) any later version. 13 13 - * 14 14 - * This program is distributed in the hope that it will be useful, but 15 15 - * WITHOUT ANY WARRANTY; without even the implied warranty of 16 16 - * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 17 17 - * NON INFRINGEMENT. See the GNU General Public License for more 18 18 - * details. 19 19 - * 20 20 - * You should have received a copy of the GNU General Public License 21 21 - * along with this program; if not, write to the Free Software 22 22 - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 23 23 - */ 24 24 - 25 25 - #include <linux/memblock.h> 26 26 - #include <linux/init.h> 27 27 - #include <linux/vmalloc.h> 28 28 - #include <asm/pgtable_areas.h> 29 29 - 30 30 - #include "numa_internal.h" 31 31 - 32 32 - extern unsigned long highend_pfn, highstart_pfn; 33 33 - 34 34 - void __init initmem_init(void) 35 35 - { 36 36 - x86_numa_init(); 37 37 - 38 38 - #ifdef CONFIG_HIGHMEM 39 39 - highstart_pfn = highend_pfn = max_pfn; 40 40 - if (max_pfn > max_low_pfn) 41 41 - highstart_pfn = max_low_pfn; 42 42 - printk(KERN_NOTICE "%ldMB HIGHMEM available.\n", 43 43 - pages_to_mb(highend_pfn - highstart_pfn)); 44 44 - high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1; 45 45 - #else 46 46 - high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1; 47 47 - #endif 48 48 - printk(KERN_NOTICE "%ldMB LOWMEM available.\n", 49 49 - pages_to_mb(max_low_pfn)); 50 50 - printk(KERN_DEBUG "max_low_pfn = %lx, highstart_pfn = %lx\n", 51 51 - max_low_pfn, highstart_pfn); 52 52 - 53 53 - printk(KERN_DEBUG "Low memory ends at vaddr %08lx\n", 54 54 - (ulong) pfn_to_kaddr(max_low_pfn)); 55 55 - 56 56 - printk(KERN_DEBUG "High memory starts at vaddr %08lx\n", 57 57 - (ulong) pfn_to_kaddr(highstart_pfn)); 58 58 - 59 59 - __vmalloc_start_set = true; 60 60 - setup_bootmem_allocator(); 61 61 - }

-13

arch/x86/mm/numa_64.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0 2 2 - /* 3 3 - * Generic VM initialization for x86-64 NUMA setups. 4 4 - * Copyright 2002,2003 Andi Kleen, SuSE Labs. 5 5 - */ 6 6 - #include <linux/memblock.h> 7 7 - 8 8 - #include "numa_internal.h" 9 9 - 10 10 - void __init initmem_init(void) 11 11 - { 12 12 - x86_numa_init(); 13 13 - }

-10

arch/x86/mm/numa_internal.h

reviewed

··· 1 1 - /* SPDX-License-Identifier: GPL-2.0 */ 2 2 - #ifndef __X86_MM_NUMA_INTERNAL_H 3 3 - #define __X86_MM_NUMA_INTERNAL_H 4 4 - 5 5 - #include <linux/types.h> 6 6 - #include <asm/numa.h> 7 7 - 8 8 - void __init x86_numa_init(void); 9 9 - 10 10 - #endif /* __X86_MM_NUMA_INTERNAL_H */

+4 -27

arch/x86/mm/pat/memtype.c

reviewed

··· 38 38 #include <linux/kernel.h> 39 39 #include <linux/pfn_t.h> 40 40 #include <linux/slab.h> 41 41 + #include <linux/io.h> 41 42 #include <linux/mm.h> 42 43 #include <linux/highmem.h> 43 44 #include <linux/fs.h> ··· 774 773 return vma_prot; 775 774 } 776 775 777 777 - #ifdef CONFIG_STRICT_DEVMEM 778 778 - /* This check is done in drivers/char/mem.c in case of STRICT_DEVMEM */ 779 779 - static inline int range_is_allowed(unsigned long pfn, unsigned long size) 780 780 - { 781 781 - return 1; 782 782 - } 783 783 - #else 784 784 - /* This check is needed to avoid cache aliasing when PAT is enabled */ 785 785 - static inline int range_is_allowed(unsigned long pfn, unsigned long size) 786 786 - { 787 787 - u64 from = ((u64)pfn) << PAGE_SHIFT; 788 788 - u64 to = from + size; 789 789 - u64 cursor = from; 790 790 - 791 791 - if (!pat_enabled()) 792 792 - return 1; 793 793 - 794 794 - while (cursor < to) { 795 795 - if (!devmem_is_allowed(pfn)) 796 796 - return 0; 797 797 - cursor += PAGE_SIZE; 798 798 - pfn++; 799 799 - } 800 800 - return 1; 801 801 - } 802 802 - #endif /* CONFIG_STRICT_DEVMEM */ 803 803 - 804 776 int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, 805 777 unsigned long size, pgprot_t *vma_prot) 806 778 { 807 779 enum page_cache_mode pcm = _PAGE_CACHE_MODE_WB; 780 780 + 781 781 + if (!pat_enabled()) 782 782 + return 1; 808 783 809 784 if (!range_is_allowed(pfn, size)) 810 785 return 0;

+2 -2

arch/x86/mm/pat/set_memory.c

reviewed

··· 889 889 /* change init_mm */ 890 890 set_pte_atomic(kpte, pte); 891 891 #ifdef CONFIG_X86_32 892 892 - if (!SHARED_KERNEL_PMD) { 892 892 + { 893 893 struct page *page; 894 894 895 895 list_for_each_entry(page, &pgd_list, lru) { ··· 1293 1293 /* Queue the page table to be freed after TLB flush */ 1294 1294 list_add(&page_ptdesc(pmd_page(old_pmd))->pt_list, pgtables); 1295 1295 1296 1296 - if (IS_ENABLED(CONFIG_X86_32) && !SHARED_KERNEL_PMD) { 1296 1296 + if (IS_ENABLED(CONFIG_X86_32)) { 1297 1297 struct page *page; 1298 1298 1299 1299 /* Update all PGD tables to use the same large page */

+16 -87

arch/x86/mm/pgtable.c

reviewed

··· 69 69 list_del(&ptdesc->pt_list); 70 70 } 71 71 72 72 - #define UNSHARED_PTRS_PER_PGD \ 73 73 - (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD) 74 74 - #define MAX_UNSHARED_PTRS_PER_PGD \ 75 75 - MAX_T(size_t, KERNEL_PGD_BOUNDARY, PTRS_PER_PGD) 76 76 - 77 77 - 78 72 static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) 79 73 { 80 74 virt_to_ptdesc(pgd)->pt_mm = mm; ··· 81 87 82 88 static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) 83 89 { 84 84 - /* If the pgd points to a shared pagetable level (either the 85 85 - ptes in non-PAE, or shared PMD in PAE), then just copy the 86 86 - references from swapper_pg_dir. */ 87 87 - if (CONFIG_PGTABLE_LEVELS == 2 || 88 88 - (CONFIG_PGTABLE_LEVELS == 3 && SHARED_KERNEL_PMD) || 89 89 - CONFIG_PGTABLE_LEVELS >= 4) { 90 90 + /* PAE preallocates all its PMDs. No cloning needed. */ 91 91 + if (!IS_ENABLED(CONFIG_X86_PAE)) 90 92 clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY, 91 93 swapper_pg_dir + KERNEL_PGD_BOUNDARY, 92 94 KERNEL_PGD_PTRS); 93 93 - } 94 95 95 95 - /* list required to sync kernel mapping updates */ 96 96 - if (!SHARED_KERNEL_PMD) { 97 97 - pgd_set_mm(pgd, mm); 98 98 - pgd_list_add(pgd); 99 99 - } 96 96 + /* List used to sync kernel mapping updates */ 97 97 + pgd_set_mm(pgd, mm); 98 98 + pgd_list_add(pgd); 100 99 } 101 100 102 101 static void pgd_dtor(pgd_t *pgd) 103 102 { 104 104 - if (SHARED_KERNEL_PMD) 105 105 - return; 106 106 - 107 103 spin_lock(&pgd_lock); 108 104 pgd_list_del(pgd); 109 105 spin_unlock(&pgd_lock); ··· 117 133 * processor notices the update. Since this is expensive, and 118 134 * all 4 top-level entries are used almost immediately in a 119 135 * new process's life, we just pre-populate them here. 120 120 - * 121 121 - * Also, if we're in a paravirt environment where the kernel pmd is 122 122 - * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate 123 123 - * and initialize the kernel pmds here. 124 136 */ 125 125 - #define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD 126 126 - #define MAX_PREALLOCATED_PMDS MAX_UNSHARED_PTRS_PER_PGD 137 137 + #define PREALLOCATED_PMDS PTRS_PER_PGD 127 138 128 139 /* 140 140 + * "USER_PMDS" are the PMDs for the user copy of the page tables when 141 141 + * PTI is enabled. They do not exist when PTI is disabled. Note that 142 142 + * this is distinct from the user _portion_ of the kernel page tables 143 143 + * which always exists. 144 144 + * 129 145 * We allocate separate PMDs for the kernel part of the user page-table 130 146 * when PTI is enabled. We need them to map the per-process LDT into the 131 147 * user-space page-table. ··· 154 170 155 171 /* No need to prepopulate any pagetable entries in non-PAE modes. */ 156 172 #define PREALLOCATED_PMDS 0 157 157 - #define MAX_PREALLOCATED_PMDS 0 158 173 #define PREALLOCATED_USER_PMDS 0 159 174 #define MAX_PREALLOCATED_USER_PMDS 0 160 175 #endif /* CONFIG_X86_PAE */ ··· 302 319 { 303 320 } 304 321 #endif 305 305 - /* 306 306 - * Xen paravirt assumes pgd table should be in one page. 64 bit kernel also 307 307 - * assumes that pgd should be in one page. 308 308 - * 309 309 - * But kernel with PAE paging that is not running as a Xen domain 310 310 - * only needs to allocate 32 bytes for pgd instead of one page. 311 311 - */ 312 312 - #ifdef CONFIG_X86_PAE 313 313 - 314 314 - #include <linux/slab.h> 315 315 - 316 316 - #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) 317 317 - #define PGD_ALIGN 32 318 318 - 319 319 - static struct kmem_cache *pgd_cache; 320 320 - 321 321 - void __init pgtable_cache_init(void) 322 322 - { 323 323 - /* 324 324 - * When PAE kernel is running as a Xen domain, it does not use 325 325 - * shared kernel pmd. And this requires a whole page for pgd. 326 326 - */ 327 327 - if (!SHARED_KERNEL_PMD) 328 328 - return; 329 329 - 330 330 - /* 331 331 - * when PAE kernel is not running as a Xen domain, it uses 332 332 - * shared kernel pmd. Shared kernel pmd does not require a whole 333 333 - * page for pgd. We are able to just allocate a 32-byte for pgd. 334 334 - * During boot time, we create a 32-byte slab for pgd table allocation. 335 335 - */ 336 336 - pgd_cache = kmem_cache_create("pgd_cache", PGD_SIZE, PGD_ALIGN, 337 337 - SLAB_PANIC, NULL); 338 338 - } 339 322 340 323 static inline pgd_t *_pgd_alloc(struct mm_struct *mm) 341 324 { 342 325 /* 343 343 - * If no SHARED_KERNEL_PMD, PAE kernel is running as a Xen domain. 344 344 - * We allocate one page for pgd. 326 326 + * PTI and Xen need a whole page for the PAE PGD 327 327 + * even though the hardware only needs 32 bytes. 328 328 + * 329 329 + * For simplicity, allocate a page for all users. 345 330 */ 346 346 - if (!SHARED_KERNEL_PMD) 347 347 - return __pgd_alloc(mm, pgd_allocation_order()); 348 348 - 349 349 - /* 350 350 - * Now PAE kernel is not running as a Xen domain. We can allocate 351 351 - * a 32-byte slab for pgd to save memory space. 352 352 - */ 353 353 - return kmem_cache_alloc(pgd_cache, GFP_PGTABLE_USER); 354 354 - } 355 355 - 356 356 - static inline void _pgd_free(struct mm_struct *mm, pgd_t *pgd) 357 357 - { 358 358 - if (!SHARED_KERNEL_PMD) 359 359 - __pgd_free(mm, pgd); 360 360 - else 361 361 - kmem_cache_free(pgd_cache, pgd); 362 362 - } 363 363 - #else 364 364 - 365 365 - static inline pgd_t *_pgd_alloc(struct mm_struct *mm) 366 366 - { 367 331 return __pgd_alloc(mm, pgd_allocation_order()); 368 332 } 369 333 ··· 318 388 { 319 389 __pgd_free(mm, pgd); 320 390 } 321 321 - #endif /* CONFIG_X86_PAE */ 322 391 323 392 pgd_t *pgd_alloc(struct mm_struct *mm) 324 393 { 325 394 pgd_t *pgd; 326 395 pmd_t *u_pmds[MAX_PREALLOCATED_USER_PMDS]; 327 327 - pmd_t *pmds[MAX_PREALLOCATED_PMDS]; 396 396 + pmd_t *pmds[PREALLOCATED_PMDS]; 328 397 329 398 pgd = _pgd_alloc(mm); 330 399

+2 -2

arch/x86/mm/pti.c

reviewed

··· 185 185 186 186 set_pgd(pgd, __pgd(_KERNPG_TABLE | __pa(new_p4d_page))); 187 187 } 188 188 - BUILD_BUG_ON(pgd_leaf(*pgd) != 0); 188 188 + BUILD_BUG_ON(pgd_leaf(*pgd)); 189 189 190 190 return p4d_offset(pgd, address); 191 191 } ··· 206 206 if (!p4d) 207 207 return NULL; 208 208 209 209 - BUILD_BUG_ON(p4d_leaf(*p4d) != 0); 209 209 + BUILD_BUG_ON(p4d_leaf(*p4d)); 210 210 if (p4d_none(*p4d)) { 211 211 unsigned long new_pud_page = __get_free_page(gfp); 212 212 if (WARN_ON_ONCE(!new_pud_page))

+34 -29

arch/x86/mm/tlb.c

reviewed

··· 215 215 216 216 atomic64_t last_mm_ctx_id = ATOMIC64_INIT(1); 217 217 218 218 + struct new_asid { 219 219 + unsigned int asid : 16; 220 220 + unsigned int need_flush : 1; 221 221 + }; 218 222 219 219 - static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen, 220 220 - u16 *new_asid, bool *need_flush) 223 223 + static struct new_asid choose_new_asid(struct mm_struct *next, u64 next_tlb_gen) 221 224 { 225 225 + struct new_asid ns; 222 226 u16 asid; 223 227 224 228 if (!static_cpu_has(X86_FEATURE_PCID)) { 225 225 - *new_asid = 0; 226 226 - *need_flush = true; 227 227 - return; 229 229 + ns.asid = 0; 230 230 + ns.need_flush = 1; 231 231 + return ns; 228 232 } 229 233 230 234 /* ··· 239 235 u16 global_asid = mm_global_asid(next); 240 236 241 237 if (global_asid) { 242 242 - *new_asid = global_asid; 243 243 - *need_flush = false; 244 244 - return; 238 238 + ns.asid = global_asid; 239 239 + ns.need_flush = 0; 240 240 + return ns; 245 241 } 246 242 } 247 243 ··· 253 249 next->context.ctx_id) 254 250 continue; 255 251 256 256 - *new_asid = asid; 257 257 - *need_flush = (this_cpu_read(cpu_tlbstate.ctxs[asid].tlb_gen) < 258 258 - next_tlb_gen); 259 259 - return; 252 252 + ns.asid = asid; 253 253 + ns.need_flush = (this_cpu_read(cpu_tlbstate.ctxs[asid].tlb_gen) < next_tlb_gen); 254 254 + return ns; 260 255 } 261 256 262 257 /* 263 258 * We don't currently own an ASID slot on this CPU. 264 259 * Allocate a slot. 265 260 */ 266 266 - *new_asid = this_cpu_add_return(cpu_tlbstate.next_asid, 1) - 1; 267 267 - if (*new_asid >= TLB_NR_DYN_ASIDS) { 268 268 - *new_asid = 0; 261 261 + ns.asid = this_cpu_add_return(cpu_tlbstate.next_asid, 1) - 1; 262 262 + if (ns.asid >= TLB_NR_DYN_ASIDS) { 263 263 + ns.asid = 0; 269 264 this_cpu_write(cpu_tlbstate.next_asid, 1); 270 265 } 271 271 - *need_flush = true; 266 266 + ns.need_flush = true; 267 267 + 268 268 + return ns; 272 269 } 273 270 274 271 /* ··· 786 781 bool was_lazy = this_cpu_read(cpu_tlbstate_shared.is_lazy); 787 782 unsigned cpu = smp_processor_id(); 788 783 unsigned long new_lam; 784 784 + struct new_asid ns; 789 785 u64 next_tlb_gen; 790 790 - bool need_flush; 791 791 - u16 new_asid; 786 786 + 792 787 793 788 /* We don't want flush_tlb_func() to run concurrently with us. */ 794 789 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) ··· 860 855 /* Check if the current mm is transitioning to a global ASID */ 861 856 if (mm_needs_global_asid(next, prev_asid)) { 862 857 next_tlb_gen = atomic64_read(&next->context.tlb_gen); 863 863 - choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush); 858 858 + ns = choose_new_asid(next, next_tlb_gen); 864 859 goto reload_tlb; 865 860 } 866 861 ··· 895 890 * TLB contents went out of date while we were in lazy 896 891 * mode. Fall through to the TLB switching code below. 897 892 */ 898 898 - new_asid = prev_asid; 899 899 - need_flush = true; 893 893 + ns.asid = prev_asid; 894 894 + ns.need_flush = true; 900 895 } else { 901 896 /* 902 897 * Apply process to process speculation vulnerability ··· 917 912 cpumask_set_cpu(cpu, mm_cpumask(next)); 918 913 next_tlb_gen = atomic64_read(&next->context.tlb_gen); 919 914 920 920 - choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush); 915 915 + ns = choose_new_asid(next, next_tlb_gen); 921 916 } 922 917 923 918 reload_tlb: 924 919 new_lam = mm_lam_cr3_mask(next); 925 925 - if (need_flush) { 926 926 - VM_WARN_ON_ONCE(is_global_asid(new_asid)); 927 927 - this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id); 928 928 - this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen); 929 929 - load_new_mm_cr3(next->pgd, new_asid, new_lam, true); 920 920 + if (ns.need_flush) { 921 921 + VM_WARN_ON_ONCE(is_global_asid(ns.asid)); 922 922 + this_cpu_write(cpu_tlbstate.ctxs[ns.asid].ctx_id, next->context.ctx_id); 923 923 + this_cpu_write(cpu_tlbstate.ctxs[ns.asid].tlb_gen, next_tlb_gen); 924 924 + load_new_mm_cr3(next->pgd, ns.asid, new_lam, true); 930 925 931 926 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); 932 927 } else { 933 928 /* The new ASID is already up to date. */ 934 934 - load_new_mm_cr3(next->pgd, new_asid, new_lam, false); 929 929 + load_new_mm_cr3(next->pgd, ns.asid, new_lam, false); 935 930 936 931 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, 0); 937 932 } ··· 940 935 barrier(); 941 936 942 937 this_cpu_write(cpu_tlbstate.loaded_mm, next); 943 943 - this_cpu_write(cpu_tlbstate.loaded_mm_asid, new_asid); 938 938 + this_cpu_write(cpu_tlbstate.loaded_mm_asid, ns.asid); 944 939 cpu_tlbstate_update_lam(new_lam, mm_untag_mask(next)); 945 940 946 941 if (next != prev) {

-18

drivers/char/mem.c

reviewed

··· 61 61 { 62 62 return devmem_is_allowed(pfn); 63 63 } 64 64 - static inline int range_is_allowed(unsigned long pfn, unsigned long size) 65 65 - { 66 66 - u64 from = ((u64)pfn) << PAGE_SHIFT; 67 67 - u64 to = from + size; 68 68 - u64 cursor = from; 69 69 - 70 70 - while (cursor < to) { 71 71 - if (!devmem_is_allowed(pfn)) 72 72 - return 0; 73 73 - cursor += PAGE_SIZE; 74 74 - pfn++; 75 75 - } 76 76 - return 1; 77 77 - } 78 64 #else 79 65 static inline int page_is_allowed(unsigned long pfn) 80 80 - { 81 81 - return 1; 82 82 - } 83 83 - static inline int range_is_allowed(unsigned long pfn, unsigned long size) 84 66 { 85 67 return 1; 86 68 }

+21

include/linux/io.h

reviewed

··· 183 183 int devm_arch_io_reserve_memtype_wc(struct device *dev, resource_size_t start, 184 184 resource_size_t size); 185 185 186 186 + #ifdef CONFIG_STRICT_DEVMEM 187 187 + static inline int range_is_allowed(unsigned long pfn, unsigned long size) 188 188 + { 189 189 + u64 from = ((u64)pfn) << PAGE_SHIFT; 190 190 + u64 to = from + size; 191 191 + u64 cursor = from; 192 192 + 193 193 + while (cursor < to) { 194 194 + if (!devmem_is_allowed(pfn)) 195 195 + return 0; 196 196 + cursor += PAGE_SIZE; 197 197 + pfn++; 198 198 + } 199 199 + return 1; 200 200 + } 201 201 + #else 202 202 + static inline int range_is_allowed(unsigned long pfn, unsigned long size) 203 203 + { 204 204 + return 1; 205 205 + } 206 206 + #endif 186 207 #endif /* _LINUX_IO_H */

+6 -3

tools/testing/selftests/x86/lam.c

reviewed

··· 682 682 return 1; 683 683 684 684 if (fstat(file_fd, &st) < 0) 685 685 - return 1; 685 685 + goto cleanup; 686 686 687 687 off_t file_sz = st.st_size; 688 688 ··· 690 690 691 691 fi = malloc(sizeof(*fi) + sizeof(struct iovec) * blocks); 692 692 if (!fi) 693 693 - return 1; 693 693 + goto cleanup; 694 694 695 695 fi->file_sz = file_sz; 696 696 fi->file_fd = file_fd; ··· 698 698 ring = malloc(sizeof(*ring)); 699 699 if (!ring) { 700 700 free(fi); 701 701 - return 1; 701 701 + goto cleanup; 702 702 } 703 703 704 704 memset(ring, 0, sizeof(struct io_ring)); ··· 729 729 } 730 730 731 731 free(fi); 732 732 + cleanup: 733 733 + close(file_fd); 732 734 733 735 return ret; 734 736 } ··· 1191 1189 1192 1190 wq = mmap(NULL, 0x1000, PROT_WRITE, 1193 1191 MAP_SHARED | MAP_POPULATE, fd, 0); 1192 1192 + close(fd); 1194 1193 if (wq == MAP_FAILED) 1195 1194 perror("mmap"); 1196 1195