Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'x86-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 fixes from Thomas Gleixner:
"A set of x86 specific fixes and updates:
- The CR2 corruption fixes which store CR2 early in the entry code
and hand the stored address to the fault handlers.
- Revert a forgotten leftover of the dropped FSGSBASE series.
- Plug a memory leak in the boot code.
- Make the Hyper-V assist functionality robust by zeroing the shadow
page.
- Remove a useless check for dead processes with LDT
- Update paravirt and VMware maintainers entries.
- A few cleanup patches addressing various compiler warnings"
* 'x86-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
x86/entry/64: Prevent clobbering of saved CR2 value
x86/hyper-v: Zero out the VP ASSIST PAGE on allocation
x86, boot: Remove multiple copy of static function sanitize_boot_params()
x86/boot/compressed/64: Remove unused variable
x86/boot/efi: Remove unused variables
x86/mm, tracing: Fix CR2 corruption
x86/entry/64: Update comments and sanity tests for create_gap
x86/entry/64: Simplify idtentry a little
x86/entry/32: Simplify common_exception
x86/paravirt: Make read_cr2() CALLEE_SAVE
MAINTAINERS: Update PARAVIRT_OPS_INTERFACE and VMWARE_HYPERVISOR_INTERFACE
x86/process: Delete useless check for dead process with LDT
x86: math-emu: Hide clang warnings for 16-bit overflow
x86/e820: Use proper booleans instead of 0/1
x86/apic: Silence -Wtype-limits compiler warnings
x86/mm: Free sme_early_buffer after init
x86/boot: Fix memory leak in default_get_smp_config()
Revert "x86/ptrace: Prevent ptrace from clearing the FS/GS selector" and fix the test
+226
-215
+4
-2
MAINTAINERS
+4
-2
MAINTAINERS
···
12133
12133
12134
12134
PARAVIRT_OPS INTERFACE
12135
12135
M: Juergen Gross <jgross@suse.com>
12136
-
M: Alok Kataria <akataria@vmware.com>
12136
+
M: Thomas Hellstrom <thellstrom@vmware.com>
12137
+
M: "VMware, Inc." <pv-drivers@vmware.com>
12137
12138
L: virtualization@lists.linux-foundation.org
12138
12139
S: Supported
12139
12140
F: Documentation/virtual/paravirt_ops.txt
···
17180
17179
F: drivers/misc/vmw_balloon.c
17181
17180
17182
17181
VMWARE HYPERVISOR INTERFACE
17183
-
M: Alok Kataria <akataria@vmware.com>
17182
+
M: Thomas Hellstrom <thellstrom@vmware.com>
17183
+
M: "VMware, Inc." <pv-drivers@vmware.com>
17184
17184
L: virtualization@lists.linux-foundation.org
17185
17185
S: Supported
17186
17186
F: arch/x86/kernel/cpu/vmware.c
+1
-9
arch/x86/boot/compressed/eboot.c
+1
-9
arch/x86/boot/compressed/eboot.c
···
384
384
struct apm_bios_info *bi;
385
385
struct setup_header *hdr;
386
386
efi_loaded_image_t *image;
387
-
void *options, *handle;
387
+
void *handle;
388
388
efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID;
389
389
int options_size = 0;
390
390
efi_status_t status;
391
391
char *cmdline_ptr;
392
-
u16 *s2;
393
-
u8 *s1;
394
-
int i;
395
392
unsigned long ramdisk_addr;
396
393
unsigned long ramdisk_size;
397
394
···
491
494
struct setup_data *e820ext, u32 nr_entries)
492
495
{
493
496
struct setup_data *data;
494
-
efi_status_t status;
495
-
unsigned long size;
496
497
497
498
e820ext->type = SETUP_E820_EXT;
498
499
e820ext->len = nr_entries * sizeof(struct boot_e820_entry);
···
672
677
void *priv)
673
678
{
674
679
const char *signature;
675
-
__u32 nr_desc;
676
-
efi_status_t status;
677
680
struct exit_boot_struct *p = priv;
678
681
679
682
signature = efi_is_64bit() ? EFI64_LOADER_SIGNATURE
···
740
747
efi_main(struct efi_config *c, struct boot_params *boot_params)
741
748
{
742
749
struct desc_ptr *gdt = NULL;
743
-
efi_loaded_image_t *image;
744
750
struct setup_header *hdr = &boot_params->hdr;
745
751
efi_status_t status;
746
752
struct desc_struct *desc;
+1
arch/x86/boot/compressed/misc.c
+1
arch/x86/boot/compressed/misc.c
-1
arch/x86/boot/compressed/misc.h
-1
arch/x86/boot/compressed/misc.h
-1
arch/x86/boot/compressed/pgtable_64.c
-1
arch/x86/boot/compressed/pgtable_64.c
+6
arch/x86/entry/calling.h
+6
arch/x86/entry/calling.h
+35
-26
arch/x86/entry/entry_32.S
+35
-26
arch/x86/entry/entry_32.S
···
294
294
.Lfinished_frame_\@:
295
295
.endm
296
296
297
-
.macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0
297
+
.macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0 skip_gs=0
298
298
cld
299
+
.if \skip_gs == 0
299
300
PUSH_GS
301
+
.endif
300
302
FIXUP_FRAME
301
303
pushl %fs
302
304
pushl %es
···
315
313
movl %edx, %es
316
314
movl $(__KERNEL_PERCPU), %edx
317
315
movl %edx, %fs
316
+
.if \skip_gs == 0
318
317
SET_KERNEL_GS %edx
319
-
318
+
.endif
320
319
/* Switch to kernel stack if necessary */
321
320
.if \switch_stacks > 0
322
321
SWITCH_TO_KERNEL_STACK
323
322
.endif
324
-
325
323
.endm
326
324
327
325
.macro SAVE_ALL_NMI cr3_reg:req
···
1443
1441
1444
1442
ENTRY(page_fault)
1445
1443
ASM_CLAC
1446
-
pushl $do_page_fault
1447
-
ALIGN
1448
-
jmp common_exception
1444
+
pushl $0; /* %gs's slot on the stack */
1445
+
1446
+
SAVE_ALL switch_stacks=1 skip_gs=1
1447
+
1448
+
ENCODE_FRAME_POINTER
1449
+
UNWIND_ESPFIX_STACK
1450
+
1451
+
/* fixup %gs */
1452
+
GS_TO_REG %ecx
1453
+
REG_TO_PTGS %ecx
1454
+
SET_KERNEL_GS %ecx
1455
+
1456
+
GET_CR2_INTO(%ecx) # might clobber %eax
1457
+
1458
+
/* fixup orig %eax */
1459
+
movl PT_ORIG_EAX(%esp), %edx # get the error code
1460
+
movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
1461
+
1462
+
TRACE_IRQS_OFF
1463
+
movl %esp, %eax # pt_regs pointer
1464
+
call do_page_fault
1465
+
jmp ret_from_exception
1449
1466
END(page_fault)
1450
1467
1451
1468
common_exception:
1452
1469
/* the function address is in %gs's slot on the stack */
1453
-
FIXUP_FRAME
1454
-
pushl %fs
1455
-
pushl %es
1456
-
pushl %ds
1457
-
pushl %eax
1458
-
movl $(__USER_DS), %eax
1459
-
movl %eax, %ds
1460
-
movl %eax, %es
1461
-
movl $(__KERNEL_PERCPU), %eax
1462
-
movl %eax, %fs
1463
-
pushl %ebp
1464
-
pushl %edi
1465
-
pushl %esi
1466
-
pushl %edx
1467
-
pushl %ecx
1468
-
pushl %ebx
1469
-
SWITCH_TO_KERNEL_STACK
1470
+
SAVE_ALL switch_stacks=1 skip_gs=1
1470
1471
ENCODE_FRAME_POINTER
1471
-
cld
1472
1472
UNWIND_ESPFIX_STACK
1473
+
1474
+
/* fixup %gs */
1473
1475
GS_TO_REG %ecx
1474
1476
movl PT_GS(%esp), %edi # get the function address
1475
-
movl PT_ORIG_EAX(%esp), %edx # get the error code
1476
-
movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
1477
1477
REG_TO_PTGS %ecx
1478
1478
SET_KERNEL_GS %ecx
1479
+
1480
+
/* fixup orig %eax */
1481
+
movl PT_ORIG_EAX(%esp), %edx # get the error code
1482
+
movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
1483
+
1479
1484
TRACE_IRQS_OFF
1480
1485
movl %esp, %eax # pt_regs pointer
1481
1486
CALL_NOSPEC %edi
+82
-73
arch/x86/entry/entry_64.S
+82
-73
arch/x86/entry/entry_64.S
···
864
864
*/
865
865
#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + (x) * 8)
866
866
867
+
.macro idtentry_part do_sym, has_error_code:req, read_cr2:req, paranoid:req, shift_ist=-1, ist_offset=0
868
+
869
+
.if \paranoid
870
+
call paranoid_entry
871
+
/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
872
+
.else
873
+
call error_entry
874
+
.endif
875
+
UNWIND_HINT_REGS
876
+
877
+
.if \read_cr2
878
+
/*
879
+
* Store CR2 early so subsequent faults cannot clobber it. Use R12 as
880
+
* intermediate storage as RDX can be clobbered in enter_from_user_mode().
881
+
* GET_CR2_INTO can clobber RAX.
882
+
*/
883
+
GET_CR2_INTO(%r12);
884
+
.endif
885
+
886
+
.if \shift_ist != -1
887
+
TRACE_IRQS_OFF_DEBUG /* reload IDT in case of recursion */
888
+
.else
889
+
TRACE_IRQS_OFF
890
+
.endif
891
+
892
+
.if \paranoid == 0
893
+
testb $3, CS(%rsp)
894
+
jz .Lfrom_kernel_no_context_tracking_\@
895
+
CALL_enter_from_user_mode
896
+
.Lfrom_kernel_no_context_tracking_\@:
897
+
.endif
898
+
899
+
movq %rsp, %rdi /* pt_regs pointer */
900
+
901
+
.if \has_error_code
902
+
movq ORIG_RAX(%rsp), %rsi /* get error code */
903
+
movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
904
+
.else
905
+
xorl %esi, %esi /* no error code */
906
+
.endif
907
+
908
+
.if \shift_ist != -1
909
+
subq $\ist_offset, CPU_TSS_IST(\shift_ist)
910
+
.endif
911
+
912
+
.if \read_cr2
913
+
movq %r12, %rdx /* Move CR2 into 3rd argument */
914
+
.endif
915
+
916
+
call \do_sym
917
+
918
+
.if \shift_ist != -1
919
+
addq $\ist_offset, CPU_TSS_IST(\shift_ist)
920
+
.endif
921
+
922
+
.if \paranoid
923
+
/* this procedure expect "no swapgs" flag in ebx */
924
+
jmp paranoid_exit
925
+
.else
926
+
jmp error_exit
927
+
.endif
928
+
929
+
.endm
930
+
867
931
/**
868
932
* idtentry - Generate an IDT entry stub
869
933
* @sym: Name of the generated entry point
870
-
* @do_sym: C function to be called
871
-
* @has_error_code: True if this IDT vector has an error code on the stack
872
-
* @paranoid: non-zero means that this vector may be invoked from
934
+
* @do_sym: C function to be called
935
+
* @has_error_code: True if this IDT vector has an error code on the stack
936
+
* @paranoid: non-zero means that this vector may be invoked from
873
937
* kernel mode with user GSBASE and/or user CR3.
874
938
* 2 is special -- see below.
875
939
* @shift_ist: Set to an IST index if entries from kernel mode should
876
-
* decrement the IST stack so that nested entries get a
940
+
* decrement the IST stack so that nested entries get a
877
941
* fresh stack. (This is for #DB, which has a nasty habit
878
-
* of recursing.)
942
+
* of recursing.)
943
+
* @create_gap: create a 6-word stack gap when coming from kernel mode.
944
+
* @read_cr2: load CR2 into the 3rd argument; done before calling any C code
879
945
*
880
946
* idtentry generates an IDT stub that sets up a usable kernel context,
881
947
* creates struct pt_regs, and calls @do_sym. The stub has the following
···
966
900
* @paranoid == 2 is special: the stub will never switch stacks. This is for
967
901
* #DF: if the thread stack is somehow unusable, we'll still get a useful OOPS.
968
902
*/
969
-
.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 ist_offset=0 create_gap=0
903
+
.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 ist_offset=0 create_gap=0 read_cr2=0
970
904
ENTRY(\sym)
971
905
UNWIND_HINT_IRET_REGS offset=\has_error_code*8
972
906
973
907
/* Sanity check */
974
-
.if \shift_ist != -1 && \paranoid == 0
908
+
.if \shift_ist != -1 && \paranoid != 1
975
909
.error "using shift_ist requires paranoid=1"
910
+
.endif
911
+
912
+
.if \create_gap && \paranoid
913
+
.error "using create_gap requires paranoid=0"
976
914
.endif
977
915
978
916
ASM_CLAC
···
1004
934
.Lfrom_usermode_no_gap_\@:
1005
935
.endif
1006
936
1007
-
.if \paranoid
1008
-
call paranoid_entry
1009
-
.else
1010
-
call error_entry
1011
-
.endif
1012
-
UNWIND_HINT_REGS
1013
-
/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
1014
-
1015
-
.if \paranoid
1016
-
.if \shift_ist != -1
1017
-
TRACE_IRQS_OFF_DEBUG /* reload IDT in case of recursion */
1018
-
.else
1019
-
TRACE_IRQS_OFF
1020
-
.endif
1021
-
.endif
1022
-
1023
-
movq %rsp, %rdi /* pt_regs pointer */
1024
-
1025
-
.if \has_error_code
1026
-
movq ORIG_RAX(%rsp), %rsi /* get error code */
1027
-
movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
1028
-
.else
1029
-
xorl %esi, %esi /* no error code */
1030
-
.endif
1031
-
1032
-
.if \shift_ist != -1
1033
-
subq $\ist_offset, CPU_TSS_IST(\shift_ist)
1034
-
.endif
1035
-
1036
-
call \do_sym
1037
-
1038
-
.if \shift_ist != -1
1039
-
addq $\ist_offset, CPU_TSS_IST(\shift_ist)
1040
-
.endif
1041
-
1042
-
/* these procedures expect "no swapgs" flag in ebx */
1043
-
.if \paranoid
1044
-
jmp paranoid_exit
1045
-
.else
1046
-
jmp error_exit
1047
-
.endif
937
+
idtentry_part \do_sym, \has_error_code, \read_cr2, \paranoid, \shift_ist, \ist_offset
1048
938
1049
939
.if \paranoid == 1
1050
940
/*
···
1013
983
* run in real process context if user_mode(regs).
1014
984
*/
1015
985
.Lfrom_usermode_switch_stack_\@:
1016
-
call error_entry
1017
-
1018
-
movq %rsp, %rdi /* pt_regs pointer */
1019
-
1020
-
.if \has_error_code
1021
-
movq ORIG_RAX(%rsp), %rsi /* get error code */
1022
-
movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
1023
-
.else
1024
-
xorl %esi, %esi /* no error code */
986
+
idtentry_part \do_sym, \has_error_code, \read_cr2, paranoid=0
1025
987
.endif
1026
988
1027
-
call \do_sym
1028
-
1029
-
jmp error_exit
1030
-
.endif
1031
989
_ASM_NOKPROBE(\sym)
1032
990
END(\sym)
1033
991
.endm
···
1025
1007
idtentry bounds do_bounds has_error_code=0
1026
1008
idtentry invalid_op do_invalid_op has_error_code=0
1027
1009
idtentry device_not_available do_device_not_available has_error_code=0
1028
-
idtentry double_fault do_double_fault has_error_code=1 paranoid=2
1010
+
idtentry double_fault do_double_fault has_error_code=1 paranoid=2 read_cr2=1
1029
1011
idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0
1030
1012
idtentry invalid_TSS do_invalid_TSS has_error_code=1
1031
1013
idtentry segment_not_present do_segment_not_present has_error_code=1
···
1197
1179
#endif
1198
1180
1199
1181
idtentry general_protection do_general_protection has_error_code=1
1200
-
idtentry page_fault do_page_fault has_error_code=1
1182
+
idtentry page_fault do_page_fault has_error_code=1 read_cr2=1
1201
1183
1202
1184
#ifdef CONFIG_KVM_GUEST
1203
-
idtentry async_page_fault do_async_page_fault has_error_code=1
1185
+
idtentry async_page_fault do_async_page_fault has_error_code=1 read_cr2=1
1204
1186
#endif
1205
1187
1206
1188
#ifdef CONFIG_X86_MCE
···
1299
1281
movq %rax, %rsp /* switch stack */
1300
1282
ENCODE_FRAME_POINTER
1301
1283
pushq %r12
1302
-
1303
-
/*
1304
-
* We need to tell lockdep that IRQs are off. We can't do this until
1305
-
* we fix gsbase, and we should do it before enter_from_user_mode
1306
-
* (which can take locks).
1307
-
*/
1308
-
TRACE_IRQS_OFF
1309
-
CALL_enter_from_user_mode
1310
1284
ret
1311
1285
1312
1286
.Lerror_entry_done:
1313
-
TRACE_IRQS_OFF
1314
1287
ret
1315
1288
1316
1289
/*
+11
-2
arch/x86/hyperv/hv_init.c
+11
-2
arch/x86/hyperv/hv_init.c
···
60
60
if (!hv_vp_assist_page)
61
61
return 0;
62
62
63
-
if (!*hvp)
64
-
*hvp = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL);
63
+
/*
64
+
* The VP ASSIST PAGE is an "overlay" page (see Hyper-V TLFS's Section
65
+
* 5.2.1 "GPA Overlay Pages"). Here it must be zeroed out to make sure
66
+
* we always write the EOI MSR in hv_apic_eoi_write() *after* the
67
+
* EOI optimization is disabled in hv_cpu_die(), otherwise a CPU may
68
+
* not be stopped in the case of CPU offlining and the VM will hang.
69
+
*/
70
+
if (!*hvp) {
71
+
*hvp = __vmalloc(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO,
72
+
PAGE_KERNEL);
73
+
}
65
74
66
75
if (*hvp) {
67
76
u64 val;
+1
-1
arch/x86/include/asm/apic.h
+1
-1
arch/x86/include/asm/apic.h
+1
-1
arch/x86/include/asm/kvm_para.h
+1
-1
arch/x86/include/asm/kvm_para.h
···
92
92
void kvm_async_pf_task_wake(u32 token);
93
93
u32 kvm_read_and_reset_pf_reason(void);
94
94
extern void kvm_disable_steal_time(void);
95
-
void do_async_page_fault(struct pt_regs *regs, unsigned long error_code);
95
+
void do_async_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address);
96
96
97
97
#ifdef CONFIG_PARAVIRT_SPINLOCKS
98
98
void __init kvm_spinlock_init(void);
+13
-9
arch/x86/include/asm/paravirt.h
+13
-9
arch/x86/include/asm/paravirt.h
···
116
116
117
117
static inline unsigned long read_cr2(void)
118
118
{
119
-
return PVOP_CALL0(unsigned long, mmu.read_cr2);
119
+
return PVOP_CALLEE0(unsigned long, mmu.read_cr2);
120
120
}
121
121
122
122
static inline void write_cr2(unsigned long x)
···
910
910
ANNOTATE_RETPOLINE_SAFE; \
911
911
call PARA_INDIRECT(pv_ops+PV_CPU_swapgs); \
912
912
)
913
-
#endif
914
913
915
-
#define GET_CR2_INTO_RAX \
916
-
ANNOTATE_RETPOLINE_SAFE; \
917
-
call PARA_INDIRECT(pv_ops+PV_MMU_read_cr2);
918
-
919
-
#ifdef CONFIG_PARAVIRT_XXL
920
914
#define USERGS_SYSRET64 \
921
915
PARA_SITE(PARA_PATCH(PV_CPU_usergs_sysret64), \
922
916
ANNOTATE_RETPOLINE_SAFE; \
···
924
930
call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl); \
925
931
PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
926
932
#endif
927
-
#endif
933
+
#endif /* CONFIG_PARAVIRT_XXL */
934
+
#endif /* CONFIG_X86_64 */
928
935
929
-
#endif /* CONFIG_X86_32 */
936
+
#ifdef CONFIG_PARAVIRT_XXL
937
+
938
+
#define GET_CR2_INTO_AX \
939
+
PARA_SITE(PARA_PATCH(PV_MMU_read_cr2), \
940
+
ANNOTATE_RETPOLINE_SAFE; \
941
+
call PARA_INDIRECT(pv_ops+PV_MMU_read_cr2); \
942
+
)
943
+
944
+
#endif /* CONFIG_PARAVIRT_XXL */
945
+
930
946
931
947
#endif /* __ASSEMBLY__ */
932
948
#else /* CONFIG_PARAVIRT */
+1
-1
arch/x86/include/asm/paravirt_types.h
+1
-1
arch/x86/include/asm/paravirt_types.h
+2
-2
arch/x86/include/asm/traps.h
+2
-2
arch/x86/include/asm/traps.h
···
74
74
dotraplinkage void do_segment_not_present(struct pt_regs *regs, long error_code);
75
75
dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code);
76
76
#ifdef CONFIG_X86_64
77
-
dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code);
77
+
dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long address);
78
78
asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs);
79
79
asmlinkage __visible notrace
80
80
struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s);
81
81
void __init trap_init(void);
82
82
#endif
83
83
dotraplinkage void do_general_protection(struct pt_regs *regs, long error_code);
84
-
dotraplinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code);
84
+
dotraplinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address);
85
85
dotraplinkage void do_spurious_interrupt_bug(struct pt_regs *regs, long error_code);
86
86
dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code);
87
87
dotraplinkage void do_alignment_check(struct pt_regs *regs, long error_code);
+1
-1
arch/x86/kernel/apic/apic.c
+1
-1
arch/x86/kernel/apic/apic.c
+1
arch/x86/kernel/asm-offsets.c
+1
arch/x86/kernel/asm-offsets.c
+2
-2
arch/x86/kernel/e820.c
+2
-2
arch/x86/kernel/e820.c
+1
-3
arch/x86/kernel/head_64.S
+1
-3
arch/x86/kernel/head_64.S
···
29
29
#ifdef CONFIG_PARAVIRT_XXL
30
30
#include <asm/asm-offsets.h>
31
31
#include <asm/paravirt.h>
32
-
#define GET_CR2_INTO(reg) GET_CR2_INTO_RAX ; movq %rax, reg
33
32
#else
34
-
#define GET_CR2_INTO(reg) movq %cr2, reg
35
33
#define INTERRUPT_RETURN iretq
36
34
#endif
37
35
···
321
323
322
324
cmpq $14,%rsi /* Page fault? */
323
325
jnz 10f
324
-
GET_CR2_INTO(%rdi) /* Can clobber any volatile register if pv */
326
+
GET_CR2_INTO(%rdi) /* can clobber %rax if pv */
325
327
call early_make_pgtable
326
328
andl %eax,%eax
327
329
jz 20f /* All good */
+4
-4
arch/x86/kernel/kvm.c
+4
-4
arch/x86/kernel/kvm.c
···
242
242
NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason);
243
243
244
244
dotraplinkage void
245
-
do_async_page_fault(struct pt_regs *regs, unsigned long error_code)
245
+
do_async_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address)
246
246
{
247
247
enum ctx_state prev_state;
248
248
249
249
switch (kvm_read_and_reset_pf_reason()) {
250
250
default:
251
-
do_page_fault(regs, error_code);
251
+
do_page_fault(regs, error_code, address);
252
252
break;
253
253
case KVM_PV_REASON_PAGE_NOT_PRESENT:
254
254
/* page is swapped out by the host. */
255
255
prev_state = exception_enter();
256
-
kvm_async_pf_task_wait((u32)read_cr2(), !user_mode(regs));
256
+
kvm_async_pf_task_wait((u32)address, !user_mode(regs));
257
257
exception_exit(prev_state);
258
258
break;
259
259
case KVM_PV_REASON_PAGE_READY:
260
260
rcu_irq_enter();
261
-
kvm_async_pf_task_wake((u32)read_cr2());
261
+
kvm_async_pf_task_wake((u32)address);
262
262
rcu_irq_exit();
263
263
break;
264
264
}
+4
-6
arch/x86/kernel/mpparse.c
+4
-6
arch/x86/kernel/mpparse.c
···
546
546
* local APIC has default address
547
547
*/
548
548
mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
549
-
return;
549
+
goto out;
550
550
}
551
551
552
552
pr_info("Default MP configuration #%d\n", mpf->feature1);
553
553
construct_default_ISA_mptable(mpf->feature1);
554
554
555
555
} else if (mpf->physptr) {
556
-
if (check_physptr(mpf, early)) {
557
-
early_memunmap(mpf, sizeof(*mpf));
558
-
return;
559
-
}
556
+
if (check_physptr(mpf, early))
557
+
goto out;
560
558
} else
561
559
BUG();
562
560
···
563
565
/*
564
566
* Only use the first configuration found.
565
567
*/
566
-
568
+
out:
567
569
early_memunmap(mpf, sizeof(*mpf));
568
570
}
569
571
+1
-1
arch/x86/kernel/paravirt.c
+1
-1
arch/x86/kernel/paravirt.c
···
370
370
.mmu.exit_mmap = paravirt_nop,
371
371
372
372
#ifdef CONFIG_PARAVIRT_XXL
373
-
.mmu.read_cr2 = native_read_cr2,
373
+
.mmu.read_cr2 = __PV_IS_CALLEE_SAVE(native_read_cr2),
374
374
.mmu.write_cr2 = native_write_cr2,
375
375
.mmu.read_cr3 = __native_read_cr3,
376
376
.mmu.write_cr3 = native_write_cr3,
+1
-11
arch/x86/kernel/process_64.c
+1
-11
arch/x86/kernel/process_64.c
···
143
143
144
144
void release_thread(struct task_struct *dead_task)
145
145
{
146
-
if (dead_task->mm) {
147
-
#ifdef CONFIG_MODIFY_LDT_SYSCALL
148
-
if (dead_task->mm->context.ldt) {
149
-
pr_warn("WARNING: dead process %s still has LDT? <%p/%d>\n",
150
-
dead_task->comm,
151
-
dead_task->mm->context.ldt->entries,
152
-
dead_task->mm->context.ldt->nr_entries);
153
-
BUG();
154
-
}
155
-
#endif
156
-
}
146
+
WARN_ON(dead_task->mm);
157
147
}
158
148
159
149
enum which_selector {
+12
-2
arch/x86/kernel/ptrace.c
+12
-2
arch/x86/kernel/ptrace.c
···
369
369
case offsetof(struct user_regs_struct,fs_base):
370
370
if (value >= TASK_SIZE_MAX)
371
371
return -EIO;
372
-
x86_fsbase_write_task(child, value);
372
+
/*
373
+
* When changing the FS base, use do_arch_prctl_64()
374
+
* to set the index to zero and to set the base
375
+
* as requested.
376
+
*/
377
+
if (child->thread.fsbase != value)
378
+
return do_arch_prctl_64(child, ARCH_SET_FS, value);
373
379
return 0;
374
380
case offsetof(struct user_regs_struct,gs_base):
381
+
/*
382
+
* Exactly the same here as the %fs handling above.
383
+
*/
375
384
if (value >= TASK_SIZE_MAX)
376
385
return -EIO;
377
-
x86_gsbase_write_task(child, value);
386
+
if (child->thread.gsbase != value)
387
+
return do_arch_prctl_64(child, ARCH_SET_GS, value);
378
388
return 0;
379
389
#endif
380
390
}
+1
-5
arch/x86/kernel/traps.c
+1
-5
arch/x86/kernel/traps.c
···
313
313
314
314
#ifdef CONFIG_X86_64
315
315
/* Runs on IST stack */
316
-
dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
316
+
dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long cr2)
317
317
{
318
318
static const char str[] = "double fault";
319
319
struct task_struct *tsk = current;
320
-
#ifdef CONFIG_VMAP_STACK
321
-
unsigned long cr2;
322
-
#endif
323
320
324
321
#ifdef CONFIG_X86_ESPFIX64
325
322
extern unsigned char native_irq_return_iret[];
···
412
415
* stack even if the actual trigger for the double fault was
413
416
* something else.
414
417
*/
415
-
cr2 = read_cr2();
416
418
if ((unsigned long)task_stack_page(tsk) - 1 - cr2 < PAGE_SIZE)
417
419
handle_stack_overflow("kernel stack overflow (double-fault)", regs, cr2);
418
420
#endif
+1
-1
arch/x86/math-emu/fpu_emu.h
+1
-1
arch/x86/math-emu/fpu_emu.h
···
177
177
#define setexponentpos(x,y) { (*(short *)&((x)->exp)) = \
178
178
((y) + EXTENDED_Ebias) & 0x7fff; }
179
179
#define exponent16(x) (*(short *)&((x)->exp))
180
-
#define setexponent16(x,y) { (*(short *)&((x)->exp)) = (y); }
180
+
#define setexponent16(x,y) { (*(short *)&((x)->exp)) = (u16)(y); }
181
181
#define addexponent(x,y) { (*(short *)&((x)->exp)) += (y); }
182
182
#define stdexp(x) { (*(short *)&((x)->exp)) += EXTENDED_Ebias; }
183
183
+1
-1
arch/x86/math-emu/reg_constant.c
+1
-1
arch/x86/math-emu/reg_constant.c
+11
-19
arch/x86/mm/fault.c
+11
-19
arch/x86/mm/fault.c
···
1490
1490
NOKPROBE_SYMBOL(do_user_addr_fault);
1491
1491
1492
1492
/*
1493
-
* This routine handles page faults. It determines the address,
1494
-
* and the problem, and then passes it off to one of the appropriate
1495
-
* routines.
1493
+
* Explicitly marked noinline such that the function tracer sees this as the
1494
+
* page_fault entry point.
1496
1495
*/
1497
1496
static noinline void
1498
1497
__do_page_fault(struct pt_regs *regs, unsigned long hw_error_code,
···
1510
1511
}
1511
1512
NOKPROBE_SYMBOL(__do_page_fault);
1512
1513
1513
-
static nokprobe_inline void
1514
-
trace_page_fault_entries(unsigned long address, struct pt_regs *regs,
1515
-
unsigned long error_code)
1514
+
static __always_inline void
1515
+
trace_page_fault_entries(struct pt_regs *regs, unsigned long error_code,
1516
+
unsigned long address)
1516
1517
{
1518
+
if (!trace_pagefault_enabled())
1519
+
return;
1520
+
1517
1521
if (user_mode(regs))
1518
1522
trace_page_fault_user(address, regs, error_code);
1519
1523
else
1520
1524
trace_page_fault_kernel(address, regs, error_code);
1521
1525
}
1522
1526
1523
-
/*
1524
-
* We must have this function blacklisted from kprobes, tagged with notrace
1525
-
* and call read_cr2() before calling anything else. To avoid calling any
1526
-
* kind of tracing machinery before we've observed the CR2 value.
1527
-
*
1528
-
* exception_{enter,exit}() contains all sorts of tracepoints.
1529
-
*/
1530
-
dotraplinkage void notrace
1531
-
do_page_fault(struct pt_regs *regs, unsigned long error_code)
1527
+
dotraplinkage void
1528
+
do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address)
1532
1529
{
1533
-
unsigned long address = read_cr2(); /* Get the faulting address */
1534
1530
enum ctx_state prev_state;
1535
1531
1536
1532
prev_state = exception_enter();
1537
-
if (trace_pagefault_enabled())
1538
-
trace_page_fault_entries(address, regs, error_code);
1539
-
1533
+
trace_page_fault_entries(regs, error_code, address);
1540
1534
__do_page_fault(regs, error_code, address);
1541
1535
exception_exit(prev_state);
1542
1536
}
+1
-1
arch/x86/mm/mem_encrypt.c
+1
-1
arch/x86/mm/mem_encrypt.c
···
41
41
bool sev_enabled __section(.data);
42
42
43
43
/* Buffer used for early in-place encryption by BSP, no locking needed */
44
-
static char sme_early_buffer[PAGE_SIZE] __aligned(PAGE_SIZE);
44
+
static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE);
45
45
46
46
/*
47
47
* This routine does not change the underlying encryption setting of the
+2
-1
arch/x86/xen/enlighten_pv.c
+2
-1
arch/x86/xen/enlighten_pv.c
···
998
998
__PV_IS_CALLEE_SAVE(xen_irq_disable_direct);
999
999
pv_ops.irq.irq_enable =
1000
1000
__PV_IS_CALLEE_SAVE(xen_irq_enable_direct);
1001
-
pv_ops.mmu.read_cr2 = xen_read_cr2_direct;
1001
+
pv_ops.mmu.read_cr2 =
1002
+
__PV_IS_CALLEE_SAVE(xen_read_cr2_direct);
1002
1003
}
1003
1004
}
1004
1005
+1
-11
arch/x86/xen/mmu_pv.c
+1
-11
arch/x86/xen/mmu_pv.c
···
1307
1307
this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
1308
1308
}
1309
1309
1310
-
static unsigned long xen_read_cr2(void)
1311
-
{
1312
-
return this_cpu_read(xen_vcpu)->arch.cr2;
1313
-
}
1314
-
1315
-
unsigned long xen_read_cr2_direct(void)
1316
-
{
1317
-
return this_cpu_read(xen_vcpu_info.arch.cr2);
1318
-
}
1319
-
1320
1310
static noinline void xen_flush_tlb(void)
1321
1311
{
1322
1312
struct mmuext_op *op;
···
2387
2397
}
2388
2398
2389
2399
static const struct pv_mmu_ops xen_mmu_ops __initconst = {
2390
-
.read_cr2 = xen_read_cr2,
2400
+
.read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2),
2391
2401
.write_cr2 = xen_write_cr2,
2392
2402
2393
2403
.read_cr3 = xen_read_cr3,
+16
arch/x86/xen/xen-asm.S
+16
arch/x86/xen/xen-asm.S
···
10
10
#include <asm/percpu.h>
11
11
#include <asm/processor-flags.h>
12
12
#include <asm/frame.h>
13
+
#include <asm/asm.h>
13
14
14
15
#include <linux/linkage.h>
15
16
···
136
135
FRAME_END
137
136
ret
138
137
ENDPROC(check_events)
138
+
139
+
ENTRY(xen_read_cr2)
140
+
FRAME_BEGIN
141
+
_ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
142
+
_ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
143
+
FRAME_END
144
+
ret
145
+
ENDPROC(xen_read_cr2);
146
+
147
+
ENTRY(xen_read_cr2_direct)
148
+
FRAME_BEGIN
149
+
_ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
150
+
FRAME_END
151
+
ret
152
+
ENDPROC(xen_read_cr2_direct);
+3
arch/x86/xen/xen-ops.h
+3
arch/x86/xen/xen-ops.h
···
134
134
__visible unsigned long xen_save_fl_direct(void);
135
135
__visible void xen_restore_fl_direct(unsigned long);
136
136
137
+
__visible unsigned long xen_read_cr2(void);
138
+
__visible unsigned long xen_read_cr2_direct(void);
139
+
137
140
/* These are not functions, and cannot be called normally */
138
141
__visible void xen_iret(void);
139
142
__visible void xen_sysret32(void);
+4
-18
tools/testing/selftests/x86/fsgsbase.c
+4
-18
tools/testing/selftests/x86/fsgsbase.c
···
489
489
* selector value is changed or not by the GSBASE write in
490
490
* a ptracer.
491
491
*/
492
-
if (gs != *shared_scratch) {
493
-
nerrs++;
494
-
printf("[FAIL]\tGS changed to %lx\n", gs);
495
-
496
-
/*
497
-
* On older kernels, poking a nonzero value into the
498
-
* base would zero the selector. On newer kernels,
499
-
* this behavior has changed -- poking the base
500
-
* changes only the base and, if FSGSBASE is not
501
-
* available, this may have no effect.
502
-
*/
503
-
if (gs == 0)
504
-
printf("\tNote: this is expected behavior on older kernels.\n");
505
-
} else if (have_fsgsbase && (base != 0xFF)) {
506
-
nerrs++;
507
-
printf("[FAIL]\tGSBASE changed to %lx\n", base);
492
+
if (gs == 0 && base == 0xFF) {
493
+
printf("[OK]\tGS was reset as expected\n");
508
494
} else {
509
-
printf("[OK]\tGS remained 0x%hx%s", *shared_scratch, have_fsgsbase ? " and GSBASE changed to 0xFF" : "");
510
-
printf("\n");
495
+
nerrs++;
496
+
printf("[FAIL]\tGS=0x%lx, GSBASE=0x%lx (should be 0, 0xFF)\n", gs, base);
511
497
}
512
498
}
513
499