Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull more x86 pti fixes from Thomas Gleixner:
"Another small stash of fixes for fallout from the PTI work:
- Fix the modules vs. KASAN breakage which was caused by making
MODULES_END depend of the fixmap size. That was done when the cpu
entry area moved into the fixmap, but now that we have a separate
map space for that this is causing more issues than it solves.
- Use the proper cache flush methods for the debugstore buffers as
they are mapped/unmapped during runtime and not statically mapped
at boot time like the rest of the cpu entry area.
- Make the map layout of the cpu_entry_area consistent for 4 and 5
level paging and fix the KASLR vaddr_end wreckage.
- Use PER_CPU_EXPORT for per cpu variable and while at it unbreak
nvidia gfx drivers by dropping the GPL export. The subject line of
the commit tells it the other way around, but I noticed that too
late.
- Fix the ASM alternative macros so they can be used in the middle of
an inline asm block.
- Rename the BUG_CPU_INSECURE flag to BUG_CPU_MELTDOWN so the attack
vector is properly identified. The Spectre mitigations will come
with their own bug bits later"
* 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
x86/pti: Rename BUG_CPU_INSECURE to BUG_CPU_MELTDOWN
x86/alternatives: Add missing '\n' at end of ALTERNATIVE inline asm
x86/tlb: Drop the _GPL from the cpu_tlbstate export
x86/events/intel/ds: Use the proper cache flush method for mapping ds buffers
x86/kaslr: Fix the vaddr_end mess
x86/mm: Map cpu_entry_area at the same place on 4/5 level
x86/mm: Set MODULES_END to 0xffffffffff000000
+55
-43
+11
-7
Documentation/x86/x86_64/mm.txt
+11
-7
Documentation/x86/x86_64/mm.txt
···
12
12
... unused hole ...
13
13
ffffec0000000000 - fffffbffffffffff (=44 bits) kasan shadow memory (16TB)
14
14
... unused hole ...
15
-
fffffe0000000000 - fffffe7fffffffff (=39 bits) LDT remap for PTI
16
-
fffffe8000000000 - fffffeffffffffff (=39 bits) cpu_entry_area mapping
15
+
vaddr_end for KASLR
16
+
fffffe0000000000 - fffffe7fffffffff (=39 bits) cpu_entry_area mapping
17
+
fffffe8000000000 - fffffeffffffffff (=39 bits) LDT remap for PTI
17
18
ffffff0000000000 - ffffff7fffffffff (=39 bits) %esp fixup stacks
18
19
... unused hole ...
19
20
ffffffef00000000 - fffffffeffffffff (=64 GB) EFI region mapping space
···
38
37
... unused hole ...
39
38
ffdf000000000000 - fffffc0000000000 (=53 bits) kasan shadow memory (8PB)
40
39
... unused hole ...
41
-
fffffe8000000000 - fffffeffffffffff (=39 bits) cpu_entry_area mapping
40
+
vaddr_end for KASLR
41
+
fffffe0000000000 - fffffe7fffffffff (=39 bits) cpu_entry_area mapping
42
+
... unused hole ...
42
43
ffffff0000000000 - ffffff7fffffffff (=39 bits) %esp fixup stacks
43
44
... unused hole ...
44
45
ffffffef00000000 - fffffffeffffffff (=64 GB) EFI region mapping space
45
46
... unused hole ...
46
47
ffffffff80000000 - ffffffff9fffffff (=512 MB) kernel text mapping, from phys 0
47
-
ffffffffa0000000 - [fixmap start] (~1526 MB) module mapping space
48
+
ffffffffa0000000 - fffffffffeffffff (1520 MB) module mapping space
48
49
[fixmap start] - ffffffffff5fffff kernel-internal fixmap range
49
50
ffffffffff600000 - ffffffffff600fff (=4 kB) legacy vsyscall ABI
50
51
ffffffffffe00000 - ffffffffffffffff (=2 MB) unused hole
···
70
67
The mappings are not part of any other kernel PGD and are only available
71
68
during EFI runtime calls.
72
69
73
-
The module mapping space size changes based on the CONFIG requirements for the
74
-
following fixmap section.
75
-
76
70
Note that if CONFIG_RANDOMIZE_MEMORY is enabled, the direct mapping of all
77
71
physical memory, vmalloc/ioremap space and virtual memory map are randomized.
78
72
Their order is preserved but their base will be offset early at boot time.
73
+
74
+
Be very careful vs. KASLR when changing anything here. The KASLR address
75
+
range must not overlap with anything except the KASAN shadow area, which is
76
+
correct as KASAN disables KASLR.
+16
arch/x86/events/intel/ds.c
+16
arch/x86/events/intel/ds.c
···
5
5
6
6
#include <asm/cpu_entry_area.h>
7
7
#include <asm/perf_event.h>
8
+
#include <asm/tlbflush.h>
8
9
#include <asm/insn.h>
9
10
10
11
#include "../perf_event.h"
···
284
283
285
284
static void ds_update_cea(void *cea, void *addr, size_t size, pgprot_t prot)
286
285
{
286
+
unsigned long start = (unsigned long)cea;
287
287
phys_addr_t pa;
288
288
size_t msz = 0;
289
289
290
290
pa = virt_to_phys(addr);
291
+
292
+
preempt_disable();
291
293
for (; msz < size; msz += PAGE_SIZE, pa += PAGE_SIZE, cea += PAGE_SIZE)
292
294
cea_set_pte(cea, pa, prot);
295
+
296
+
/*
297
+
* This is a cross-CPU update of the cpu_entry_area, we must shoot down
298
+
* all TLB entries for it.
299
+
*/
300
+
flush_tlb_kernel_range(start, start + size);
301
+
preempt_enable();
293
302
}
294
303
295
304
static void ds_clear_cea(void *cea, size_t size)
296
305
{
306
+
unsigned long start = (unsigned long)cea;
297
307
size_t msz = 0;
298
308
309
+
preempt_disable();
299
310
for (; msz < size; msz += PAGE_SIZE, cea += PAGE_SIZE)
300
311
cea_set_pte(cea, 0, PAGE_NONE);
312
+
313
+
flush_tlb_kernel_range(start, start + size);
314
+
preempt_enable();
301
315
}
302
316
303
317
static void *dsalloc_pages(size_t size, gfp_t flags, int cpu)
+2
-2
arch/x86/include/asm/alternative.h
+2
-2
arch/x86/include/asm/alternative.h
···
140
140
".popsection\n" \
141
141
".pushsection .altinstr_replacement, \"ax\"\n" \
142
142
ALTINSTR_REPLACEMENT(newinstr, feature, 1) \
143
-
".popsection"
143
+
".popsection\n"
144
144
145
145
#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
146
146
OLDINSTR_2(oldinstr, 1, 2) \
···
151
151
".pushsection .altinstr_replacement, \"ax\"\n" \
152
152
ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \
153
153
ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \
154
-
".popsection"
154
+
".popsection\n"
155
155
156
156
/*
157
157
* Alternative instructions for different CPU types or capabilities.
+1
-1
arch/x86/include/asm/cpufeatures.h
+1
-1
arch/x86/include/asm/cpufeatures.h
···
341
341
#define X86_BUG_SWAPGS_FENCE X86_BUG(11) /* SWAPGS without input dep on GS */
342
342
#define X86_BUG_MONITOR X86_BUG(12) /* IPI required to wake up remote CPU */
343
343
#define X86_BUG_AMD_E400 X86_BUG(13) /* CPU is among the affected by Erratum 400 */
344
-
#define X86_BUG_CPU_INSECURE X86_BUG(14) /* CPU is insecure and needs kernel page table isolation */
344
+
#define X86_BUG_CPU_MELTDOWN X86_BUG(14) /* CPU is affected by meltdown attack and needs kernel page table isolation */
345
345
346
346
#endif /* _ASM_X86_CPUFEATURES_H */
+10
-4
arch/x86/include/asm/pgtable_64_types.h
+10
-4
arch/x86/include/asm/pgtable_64_types.h
···
75
75
#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)
76
76
#define PGDIR_MASK (~(PGDIR_SIZE - 1))
77
77
78
-
/* See Documentation/x86/x86_64/mm.txt for a description of the memory map. */
78
+
/*
79
+
* See Documentation/x86/x86_64/mm.txt for a description of the memory map.
80
+
*
81
+
* Be very careful vs. KASLR when changing anything here. The KASLR address
82
+
* range must not overlap with anything except the KASAN shadow area, which
83
+
* is correct as KASAN disables KASLR.
84
+
*/
79
85
#define MAXMEM _AC(__AC(1, UL) << MAX_PHYSMEM_BITS, UL)
80
86
81
87
#ifdef CONFIG_X86_5LEVEL
···
94
88
# define VMALLOC_SIZE_TB _AC(32, UL)
95
89
# define __VMALLOC_BASE _AC(0xffffc90000000000, UL)
96
90
# define __VMEMMAP_BASE _AC(0xffffea0000000000, UL)
97
-
# define LDT_PGD_ENTRY _AC(-4, UL)
91
+
# define LDT_PGD_ENTRY _AC(-3, UL)
98
92
# define LDT_BASE_ADDR (LDT_PGD_ENTRY << PGDIR_SHIFT)
99
93
#endif
100
94
···
110
104
111
105
#define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE)
112
106
/* The module sections ends with the start of the fixmap */
113
-
#define MODULES_END __fix_to_virt(__end_of_fixed_addresses + 1)
107
+
#define MODULES_END _AC(0xffffffffff000000, UL)
114
108
#define MODULES_LEN (MODULES_END - MODULES_VADDR)
115
109
116
110
#define ESPFIX_PGD_ENTRY _AC(-2, UL)
117
111
#define ESPFIX_BASE_ADDR (ESPFIX_PGD_ENTRY << P4D_SHIFT)
118
112
119
-
#define CPU_ENTRY_AREA_PGD _AC(-3, UL)
113
+
#define CPU_ENTRY_AREA_PGD _AC(-4, UL)
120
114
#define CPU_ENTRY_AREA_BASE (CPU_ENTRY_AREA_PGD << P4D_SHIFT)
121
115
122
116
#define EFI_VA_START ( -4 * (_AC(1, UL) << 30))
+1
-1
arch/x86/kernel/cpu/common.c
+1
-1
arch/x86/kernel/cpu/common.c
+1
-1
arch/x86/mm/dump_pagetables.c
+1
-1
arch/x86/mm/dump_pagetables.c
+1
-1
arch/x86/mm/init.c
+1
-1
arch/x86/mm/init.c
+9
-23
arch/x86/mm/kaslr.c
+9
-23
arch/x86/mm/kaslr.c
···
34
34
#define TB_SHIFT 40
35
35
36
36
/*
37
-
* Virtual address start and end range for randomization. The end changes base
38
-
* on configuration to have the highest amount of space for randomization.
39
-
* It increases the possible random position for each randomized region.
37
+
* Virtual address start and end range for randomization.
40
38
*
41
-
* You need to add an if/def entry if you introduce a new memory region
42
-
* compatible with KASLR. Your entry must be in logical order with memory
43
-
* layout. For example, ESPFIX is before EFI because its virtual address is
44
-
* before. You also need to add a BUILD_BUG_ON() in kernel_randomize_memory() to
45
-
* ensure that this order is correct and won't be changed.
39
+
* The end address could depend on more configuration options to make the
40
+
* highest amount of space for randomization available, but that's too hard
41
+
* to keep straight and caused issues already.
46
42
*/
47
43
static const unsigned long vaddr_start = __PAGE_OFFSET_BASE;
48
-
49
-
#if defined(CONFIG_X86_ESPFIX64)
50
-
static const unsigned long vaddr_end = ESPFIX_BASE_ADDR;
51
-
#elif defined(CONFIG_EFI)
52
-
static const unsigned long vaddr_end = EFI_VA_END;
53
-
#else
54
-
static const unsigned long vaddr_end = __START_KERNEL_map;
55
-
#endif
44
+
static const unsigned long vaddr_end = CPU_ENTRY_AREA_BASE;
56
45
57
46
/* Default values */
58
47
unsigned long page_offset_base = __PAGE_OFFSET_BASE;
···
90
101
unsigned long remain_entropy;
91
102
92
103
/*
93
-
* All these BUILD_BUG_ON checks ensures the memory layout is
94
-
* consistent with the vaddr_start/vaddr_end variables.
104
+
* These BUILD_BUG_ON checks ensure the memory layout is consistent
105
+
* with the vaddr_start/vaddr_end variables. These checks are very
106
+
* limited....
95
107
*/
96
108
BUILD_BUG_ON(vaddr_start >= vaddr_end);
97
-
BUILD_BUG_ON(IS_ENABLED(CONFIG_X86_ESPFIX64) &&
98
-
vaddr_end >= EFI_VA_END);
99
-
BUILD_BUG_ON((IS_ENABLED(CONFIG_X86_ESPFIX64) ||
100
-
IS_ENABLED(CONFIG_EFI)) &&
101
-
vaddr_end >= __START_KERNEL_map);
109
+
BUILD_BUG_ON(vaddr_end != CPU_ENTRY_AREA_BASE);
102
110
BUILD_BUG_ON(vaddr_end > __START_KERNEL_map);
103
111
104
112
if (!kaslr_memory_enabled())
+3
-3
arch/x86/mm/pti.c
+3
-3
arch/x86/mm/pti.c
···
56
56
57
57
static void __init pti_print_if_insecure(const char *reason)
58
58
{
59
-
if (boot_cpu_has_bug(X86_BUG_CPU_INSECURE))
59
+
if (boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
60
60
pr_info("%s\n", reason);
61
61
}
62
62
63
63
static void __init pti_print_if_secure(const char *reason)
64
64
{
65
-
if (!boot_cpu_has_bug(X86_BUG_CPU_INSECURE))
65
+
if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
66
66
pr_info("%s\n", reason);
67
67
}
68
68
···
96
96
}
97
97
98
98
autosel:
99
-
if (!boot_cpu_has_bug(X86_BUG_CPU_INSECURE))
99
+
if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
100
100
return;
101
101
enable:
102
102
setup_force_cpu_cap(X86_FEATURE_PTI);