Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'x86-entry-2024-01-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 entry updates from Ingo Molnar:
- Optimize common_interrupt_return()
- Harden the return-to-user code by making a CONFIG_DEBUG_ENTRY=y check
unconditional & moving it closer to the IRET.
* tag 'x86-entry-2024-01-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
x86/entry: Harden return-to-user
x86/entry: Optimize common_interrupt_return()
+31
-14
+9
-3
arch/x86/entry/calling.h
+9
-3
arch/x86/entry/calling.h
···
175
175
#define THIS_CPU_user_pcid_flush_mask \
176
176
PER_CPU_VAR(cpu_tlbstate) + TLB_STATE_user_pcid_flush_mask
177
177
178
-
.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req
179
-
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
178
+
.macro SWITCH_TO_USER_CR3 scratch_reg:req scratch_reg2:req
180
179
mov %cr3, \scratch_reg
181
180
182
181
ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
···
205
206
/* Flip the PGD to the user version */
206
207
orq $(PTI_USER_PGTABLE_MASK), \scratch_reg
207
208
mov \scratch_reg, %cr3
209
+
.endm
210
+
211
+
.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req
212
+
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
213
+
SWITCH_TO_USER_CR3 \scratch_reg \scratch_reg2
208
214
.Lend_\@:
209
215
.endm
210
216
211
217
.macro SWITCH_TO_USER_CR3_STACK scratch_reg:req
218
+
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
212
219
pushq %rax
213
-
SWITCH_TO_USER_CR3_NOSTACK scratch_reg=\scratch_reg scratch_reg2=%rax
220
+
SWITCH_TO_USER_CR3 scratch_reg=\scratch_reg scratch_reg2=%rax
214
221
popq %rax
222
+
.Lend_\@:
215
223
.endm
216
224
217
225
.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
+22
-11
arch/x86/entry/entry_64.S
+22
-11
arch/x86/entry/entry_64.S
···
559
559
SYM_CODE_START_LOCAL(common_interrupt_return)
560
560
SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
561
561
IBRS_EXIT
562
-
#ifdef CONFIG_DEBUG_ENTRY
563
-
/* Assert that pt_regs indicates user mode. */
564
-
testb $3, CS(%rsp)
565
-
jnz 1f
566
-
ud2
567
-
1:
568
-
#endif
569
562
#ifdef CONFIG_XEN_PV
570
563
ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
571
564
#endif
565
+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
566
+
ALTERNATIVE "", "jmp .Lpti_restore_regs_and_return_to_usermode", X86_FEATURE_PTI
567
+
#endif
572
568
569
+
STACKLEAK_ERASE
570
+
POP_REGS
571
+
add $8, %rsp /* orig_ax */
572
+
UNWIND_HINT_IRET_REGS
573
+
574
+
.Lswapgs_and_iret:
575
+
swapgs
576
+
/* Assert that the IRET frame indicates user mode. */
577
+
testb $3, 8(%rsp)
578
+
jnz .Lnative_iret
579
+
ud2
580
+
581
+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
582
+
.Lpti_restore_regs_and_return_to_usermode:
573
583
POP_REGS pop_rdi=0
574
584
575
585
/*
···
606
596
*/
607
597
STACKLEAK_ERASE_NOCLOBBER
608
598
609
-
SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
599
+
push %rax
600
+
SWITCH_TO_USER_CR3 scratch_reg=%rdi scratch_reg2=%rax
601
+
pop %rax
610
602
611
603
/* Restore RDI. */
612
604
popq %rdi
613
-
swapgs
614
-
jmp .Lnative_iret
615
-
605
+
jmp .Lswapgs_and_iret
606
+
#endif
616
607
617
608
SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
618
609
#ifdef CONFIG_DEBUG_ENTRY