Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 asm updates from Ingo Molnar:
"Most of the changes relate to Peter Zijlstra's cleanup of ptregs
handling, in particular the i386 part is now much simplified and
standardized - no more partial ptregs stack frames via the esp/ss
oddity. This simplifies ftrace, kprobes, the unwinder, ptrace, kdump
and kgdb.
There's also a CR4 hardening enhancements by Kees Cook, to make the
generic platform functions such as native_write_cr4() less useful as
ROP gadgets that disable SMEP/SMAP. Also protect the WP bit of CR0
against similar attacks.
The rest is smaller cleanups/fixes"
* 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
x86/alternatives: Add int3_emulate_call() selftest
x86/stackframe/32: Allow int3_emulate_push()
x86/stackframe/32: Provide consistent pt_regs
x86/stackframe, x86/ftrace: Add pt_regs frame annotations
x86/stackframe, x86/kprobes: Fix frame pointer annotations
x86/stackframe: Move ENCODE_FRAME_POINTER to asm/frame.h
x86/entry/32: Clean up return from interrupt preemption path
x86/asm: Pin sensitive CR0 bits
x86/asm: Pin sensitive CR4 bits
Documentation/x86: Fix path to entry_32.S
x86/asm: Remove unused TASK_TI_flags from asm-offsets.c
+394
-265
+1
-1
Documentation/x86/exception-tables.rst
+1
-1
Documentation/x86/exception-tables.rst
···
35
35
void do_page_fault(struct pt_regs *regs, unsigned long error_code)
36
36
37
37
in arch/x86/mm/fault.c. The parameters on the stack are set up by
38
-
the low level assembly glue in arch/x86/kernel/entry_32.S. The parameter
38
+
the low level assembly glue in arch/x86/entry/entry_32.S. The parameter
39
39
regs is a pointer to the saved registers on the stack, error_code
40
40
contains a reason code for the exception.
41
41
-15
arch/x86/entry/calling.h
-15
arch/x86/entry/calling.h
···
172
172
.endif
173
173
.endm
174
174
175
-
/*
176
-
* This is a sneaky trick to help the unwinder find pt_regs on the stack. The
177
-
* frame pointer is replaced with an encoded pointer to pt_regs. The encoding
178
-
* is just setting the LSB, which makes it an invalid stack address and is also
179
-
* a signal to the unwinder that it's a pt_regs pointer in disguise.
180
-
*
181
-
* NOTE: This macro must be used *after* PUSH_AND_CLEAR_REGS because it corrupts
182
-
* the original rbp.
183
-
*/
184
-
.macro ENCODE_FRAME_POINTER ptregs_offset=0
185
-
#ifdef CONFIG_FRAME_POINTER
186
-
leaq 1+\ptregs_offset(%rsp), %rbp
187
-
#endif
188
-
.endm
189
-
190
175
#ifdef CONFIG_PAGE_TABLE_ISOLATION
191
176
192
177
/*
+105
-40
arch/x86/entry/entry_32.S
+105
-40
arch/x86/entry/entry_32.S
···
67
67
# define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
68
68
#else
69
69
# define preempt_stop(clobbers)
70
-
# define resume_kernel restore_all_kernel
71
70
#endif
72
71
73
72
.macro TRACE_IRQS_IRET
···
202
203
.Lend_\@:
203
204
.endm
204
205
206
+
#define CS_FROM_ENTRY_STACK (1 << 31)
207
+
#define CS_FROM_USER_CR3 (1 << 30)
208
+
#define CS_FROM_KERNEL (1 << 29)
209
+
210
+
.macro FIXUP_FRAME
211
+
/*
212
+
* The high bits of the CS dword (__csh) are used for CS_FROM_*.
213
+
* Clear them in case hardware didn't do this for us.
214
+
*/
215
+
andl $0x0000ffff, 3*4(%esp)
216
+
217
+
#ifdef CONFIG_VM86
218
+
testl $X86_EFLAGS_VM, 4*4(%esp)
219
+
jnz .Lfrom_usermode_no_fixup_\@
220
+
#endif
221
+
testl $SEGMENT_RPL_MASK, 3*4(%esp)
222
+
jnz .Lfrom_usermode_no_fixup_\@
223
+
224
+
orl $CS_FROM_KERNEL, 3*4(%esp)
225
+
226
+
/*
227
+
* When we're here from kernel mode; the (exception) stack looks like:
228
+
*
229
+
* 5*4(%esp) - <previous context>
230
+
* 4*4(%esp) - flags
231
+
* 3*4(%esp) - cs
232
+
* 2*4(%esp) - ip
233
+
* 1*4(%esp) - orig_eax
234
+
* 0*4(%esp) - gs / function
235
+
*
236
+
* Lets build a 5 entry IRET frame after that, such that struct pt_regs
237
+
* is complete and in particular regs->sp is correct. This gives us
238
+
* the original 5 enties as gap:
239
+
*
240
+
* 12*4(%esp) - <previous context>
241
+
* 11*4(%esp) - gap / flags
242
+
* 10*4(%esp) - gap / cs
243
+
* 9*4(%esp) - gap / ip
244
+
* 8*4(%esp) - gap / orig_eax
245
+
* 7*4(%esp) - gap / gs / function
246
+
* 6*4(%esp) - ss
247
+
* 5*4(%esp) - sp
248
+
* 4*4(%esp) - flags
249
+
* 3*4(%esp) - cs
250
+
* 2*4(%esp) - ip
251
+
* 1*4(%esp) - orig_eax
252
+
* 0*4(%esp) - gs / function
253
+
*/
254
+
255
+
pushl %ss # ss
256
+
pushl %esp # sp (points at ss)
257
+
addl $6*4, (%esp) # point sp back at the previous context
258
+
pushl 6*4(%esp) # flags
259
+
pushl 6*4(%esp) # cs
260
+
pushl 6*4(%esp) # ip
261
+
pushl 6*4(%esp) # orig_eax
262
+
pushl 6*4(%esp) # gs / function
263
+
.Lfrom_usermode_no_fixup_\@:
264
+
.endm
265
+
266
+
.macro IRET_FRAME
267
+
testl $CS_FROM_KERNEL, 1*4(%esp)
268
+
jz .Lfinished_frame_\@
269
+
270
+
/*
271
+
* Reconstruct the 3 entry IRET frame right after the (modified)
272
+
* regs->sp without lowering %esp in between, such that an NMI in the
273
+
* middle doesn't scribble our stack.
274
+
*/
275
+
pushl %eax
276
+
pushl %ecx
277
+
movl 5*4(%esp), %eax # (modified) regs->sp
278
+
279
+
movl 4*4(%esp), %ecx # flags
280
+
movl %ecx, -4(%eax)
281
+
282
+
movl 3*4(%esp), %ecx # cs
283
+
andl $0x0000ffff, %ecx
284
+
movl %ecx, -8(%eax)
285
+
286
+
movl 2*4(%esp), %ecx # ip
287
+
movl %ecx, -12(%eax)
288
+
289
+
movl 1*4(%esp), %ecx # eax
290
+
movl %ecx, -16(%eax)
291
+
292
+
popl %ecx
293
+
lea -16(%eax), %esp
294
+
popl %eax
295
+
.Lfinished_frame_\@:
296
+
.endm
297
+
205
298
.macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0
206
299
cld
207
300
PUSH_GS
301
+
FIXUP_FRAME
208
302
pushl %fs
209
303
pushl %es
210
304
pushl %ds
···
337
245
SWITCH_TO_KERNEL_CR3 scratch_reg=\cr3_reg
338
246
339
247
.Lend_\@:
340
-
.endm
341
-
342
-
/*
343
-
* This is a sneaky trick to help the unwinder find pt_regs on the stack. The
344
-
* frame pointer is replaced with an encoded pointer to pt_regs. The encoding
345
-
* is just clearing the MSB, which makes it an invalid stack address and is also
346
-
* a signal to the unwinder that it's a pt_regs pointer in disguise.
347
-
*
348
-
* NOTE: This macro must be used *after* SAVE_ALL because it corrupts the
349
-
* original rbp.
350
-
*/
351
-
.macro ENCODE_FRAME_POINTER
352
-
#ifdef CONFIG_FRAME_POINTER
353
-
mov %esp, %ebp
354
-
andl $0x7fffffff, %ebp
355
-
#endif
356
248
.endm
357
249
358
250
.macro RESTORE_INT_REGS
···
451
375
* switch to it before we do any copying.
452
376
*/
453
377
454
-
#define CS_FROM_ENTRY_STACK (1 << 31)
455
-
#define CS_FROM_USER_CR3 (1 << 30)
456
-
457
378
.macro SWITCH_TO_KERNEL_STACK
458
379
459
380
ALTERNATIVE "", "jmp .Lend_\@", X86_FEATURE_XENPV
···
463
390
* %eax now contains the entry cr3 and we carry it forward in
464
391
* that register for the time this macro runs
465
392
*/
466
-
467
-
/*
468
-
* The high bits of the CS dword (__csh) are used for
469
-
* CS_FROM_ENTRY_STACK and CS_FROM_USER_CR3. Clear them in case
470
-
* hardware didn't do this for us.
471
-
*/
472
-
andl $(0x0000ffff), PT_CS(%esp)
473
393
474
394
/* Are we on the entry stack? Bail out if not! */
475
395
movl PER_CPU_VAR(cpu_entry_area), %ecx
···
821
755
andl $SEGMENT_RPL_MASK, %eax
822
756
#endif
823
757
cmpl $USER_RPL, %eax
824
-
jb resume_kernel # not returning to v8086 or userspace
758
+
jb restore_all_kernel # not returning to v8086 or userspace
825
759
826
760
ENTRY(resume_userspace)
827
761
DISABLE_INTERRUPTS(CLBR_ANY)
···
830
764
call prepare_exit_to_usermode
831
765
jmp restore_all
832
766
END(ret_from_exception)
833
-
834
-
#ifdef CONFIG_PREEMPT
835
-
ENTRY(resume_kernel)
836
-
DISABLE_INTERRUPTS(CLBR_ANY)
837
-
cmpl $0, PER_CPU_VAR(__preempt_count)
838
-
jnz restore_all_kernel
839
-
testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ?
840
-
jz restore_all_kernel
841
-
call preempt_schedule_irq
842
-
jmp restore_all_kernel
843
-
END(resume_kernel)
844
-
#endif
845
767
846
768
GLOBAL(__begin_SYSENTER_singlestep_region)
847
769
/*
···
1073
1019
/* Restore user state */
1074
1020
RESTORE_REGS pop=4 # skip orig_eax/error_code
1075
1021
.Lirq_return:
1022
+
IRET_FRAME
1076
1023
/*
1077
1024
* ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
1078
1025
* when returning from IPI handler and when returning from
···
1082
1027
INTERRUPT_RETURN
1083
1028
1084
1029
restore_all_kernel:
1030
+
#ifdef CONFIG_PREEMPT
1031
+
DISABLE_INTERRUPTS(CLBR_ANY)
1032
+
cmpl $0, PER_CPU_VAR(__preempt_count)
1033
+
jnz .Lno_preempt
1034
+
testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ?
1035
+
jz .Lno_preempt
1036
+
call preempt_schedule_irq
1037
+
.Lno_preempt:
1038
+
#endif
1085
1039
TRACE_IRQS_IRET
1086
1040
PARANOID_EXIT_TO_KERNEL_MODE
1087
1041
BUG_IF_WRONG_CR3
···
1448
1384
1449
1385
common_exception:
1450
1386
/* the function address is in %gs's slot on the stack */
1387
+
FIXUP_FRAME
1451
1388
pushl %fs
1452
1389
pushl %es
1453
1390
pushl %ds
+49
arch/x86/include/asm/frame.h
+49
arch/x86/include/asm/frame.h
···
22
22
pop %_ASM_BP
23
23
.endm
24
24
25
+
#ifdef CONFIG_X86_64
26
+
/*
27
+
* This is a sneaky trick to help the unwinder find pt_regs on the stack. The
28
+
* frame pointer is replaced with an encoded pointer to pt_regs. The encoding
29
+
* is just setting the LSB, which makes it an invalid stack address and is also
30
+
* a signal to the unwinder that it's a pt_regs pointer in disguise.
31
+
*
32
+
* NOTE: This macro must be used *after* PUSH_AND_CLEAR_REGS because it corrupts
33
+
* the original rbp.
34
+
*/
35
+
.macro ENCODE_FRAME_POINTER ptregs_offset=0
36
+
leaq 1+\ptregs_offset(%rsp), %rbp
37
+
.endm
38
+
#else /* !CONFIG_X86_64 */
39
+
/*
40
+
* This is a sneaky trick to help the unwinder find pt_regs on the stack. The
41
+
* frame pointer is replaced with an encoded pointer to pt_regs. The encoding
42
+
* is just clearing the MSB, which makes it an invalid stack address and is also
43
+
* a signal to the unwinder that it's a pt_regs pointer in disguise.
44
+
*
45
+
* NOTE: This macro must be used *after* SAVE_ALL because it corrupts the
46
+
* original ebp.
47
+
*/
48
+
.macro ENCODE_FRAME_POINTER
49
+
mov %esp, %ebp
50
+
andl $0x7fffffff, %ebp
51
+
.endm
52
+
#endif /* CONFIG_X86_64 */
53
+
25
54
#else /* !__ASSEMBLY__ */
26
55
27
56
#define FRAME_BEGIN \
···
59
30
60
31
#define FRAME_END "pop %" _ASM_BP "\n"
61
32
33
+
#ifdef CONFIG_X86_64
34
+
#define ENCODE_FRAME_POINTER \
35
+
"lea 1(%rsp), %rbp\n\t"
36
+
#else /* !CONFIG_X86_64 */
37
+
#define ENCODE_FRAME_POINTER \
38
+
"movl %esp, %ebp\n\t" \
39
+
"andl $0x7fffffff, %ebp\n\t"
40
+
#endif /* CONFIG_X86_64 */
41
+
62
42
#endif /* __ASSEMBLY__ */
63
43
64
44
#define FRAME_OFFSET __ASM_SEL(4, 8)
65
45
66
46
#else /* !CONFIG_FRAME_POINTER */
47
+
48
+
#ifdef __ASSEMBLY__
49
+
50
+
.macro ENCODE_FRAME_POINTER ptregs_offset=0
51
+
.endm
52
+
53
+
#else /* !__ASSEMBLY */
54
+
55
+
#define ENCODE_FRAME_POINTER
56
+
57
+
#endif
67
58
68
59
#define FRAME_BEGIN
69
60
#define FRAME_END
-17
arch/x86/include/asm/kexec.h
-17
arch/x86/include/asm/kexec.h
···
71
71
#define KEXEC_BACKUP_SRC_END (640 * 1024UL - 1) /* 640K */
72
72
73
73
/*
74
-
* CPU does not save ss and sp on stack if execution is already
75
-
* running in kernel mode at the time of NMI occurrence. This code
76
-
* fixes it.
77
-
*/
78
-
static inline void crash_fixup_ss_esp(struct pt_regs *newregs,
79
-
struct pt_regs *oldregs)
80
-
{
81
-
#ifdef CONFIG_X86_32
82
-
newregs->sp = (unsigned long)&(oldregs->sp);
83
-
asm volatile("xorl %%eax, %%eax\n\t"
84
-
"movw %%ss, %%ax\n\t"
85
-
:"=a"(newregs->ss));
86
-
#endif
87
-
}
88
-
89
-
/*
90
74
* This function is responsible for capturing register states if coming
91
75
* via panic otherwise just fix up the ss and sp if coming via kernel
92
76
* mode exception.
···
80
96
{
81
97
if (oldregs) {
82
98
memcpy(newregs, oldregs, sizeof(*newregs));
83
-
crash_fixup_ss_esp(newregs, oldregs);
84
99
} else {
85
100
#ifdef CONFIG_X86_32
86
101
asm volatile("movl %%ebx,%0" : "=m"(newregs->bx));
+2
-15
arch/x86/include/asm/ptrace.h
+2
-15
arch/x86/include/asm/ptrace.h
···
166
166
#define compat_user_stack_pointer() current_pt_regs()->sp
167
167
#endif
168
168
169
-
#ifdef CONFIG_X86_32
170
-
extern unsigned long kernel_stack_pointer(struct pt_regs *regs);
171
-
#else
172
169
static inline unsigned long kernel_stack_pointer(struct pt_regs *regs)
173
170
{
174
171
return regs->sp;
175
172
}
176
-
#endif
177
173
178
174
#define GET_IP(regs) ((regs)->ip)
179
175
#define GET_FP(regs) ((regs)->bp)
···
197
201
if (unlikely(offset > MAX_REG_OFFSET))
198
202
return 0;
199
203
#ifdef CONFIG_X86_32
200
-
/*
201
-
* Traps from the kernel do not save sp and ss.
202
-
* Use the helper function to retrieve sp.
203
-
*/
204
-
if (offset == offsetof(struct pt_regs, sp) &&
205
-
regs->cs == __KERNEL_CS)
206
-
return kernel_stack_pointer(regs);
207
-
208
204
/* The selector fields are 16-bit. */
209
205
if (offset == offsetof(struct pt_regs, cs) ||
210
206
offset == offsetof(struct pt_regs, ss) ||
···
222
234
static inline int regs_within_kernel_stack(struct pt_regs *regs,
223
235
unsigned long addr)
224
236
{
225
-
return ((addr & ~(THREAD_SIZE - 1)) ==
226
-
(kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1)));
237
+
return ((addr & ~(THREAD_SIZE - 1)) == (regs->sp & ~(THREAD_SIZE - 1)));
227
238
}
228
239
229
240
/**
···
236
249
*/
237
250
static inline unsigned long *regs_get_kernel_stack_nth_addr(struct pt_regs *regs, unsigned int n)
238
251
{
239
-
unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs);
252
+
unsigned long *addr = (unsigned long *)regs->sp;
240
253
241
254
addr += n;
242
255
if (regs_within_kernel_stack(regs, (unsigned long)addr))
+35
-2
arch/x86/include/asm/special_insns.h
+35
-2
arch/x86/include/asm/special_insns.h
···
6
6
#ifdef __KERNEL__
7
7
8
8
#include <asm/nops.h>
9
+
#include <asm/processor-flags.h>
10
+
#include <linux/jump_label.h>
9
11
10
12
/*
11
13
* Volatile isn't enough to prevent the compiler from reordering the
···
18
16
*/
19
17
extern unsigned long __force_order;
20
18
19
+
/* Starts false and gets enabled once CPU feature detection is done. */
20
+
DECLARE_STATIC_KEY_FALSE(cr_pinning);
21
+
extern unsigned long cr4_pinned_bits;
22
+
21
23
static inline unsigned long native_read_cr0(void)
22
24
{
23
25
unsigned long val;
···
31
25
32
26
static inline void native_write_cr0(unsigned long val)
33
27
{
34
-
asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order));
28
+
unsigned long bits_missing = 0;
29
+
30
+
set_register:
31
+
asm volatile("mov %0,%%cr0": "+r" (val), "+m" (__force_order));
32
+
33
+
if (static_branch_likely(&cr_pinning)) {
34
+
if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
35
+
bits_missing = X86_CR0_WP;
36
+
val |= bits_missing;
37
+
goto set_register;
38
+
}
39
+
/* Warn after we've set the missing bits. */
40
+
WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
41
+
}
35
42
}
36
43
37
44
static inline unsigned long native_read_cr2(void)
···
93
74
94
75
static inline void native_write_cr4(unsigned long val)
95
76
{
96
-
asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order));
77
+
unsigned long bits_missing = 0;
78
+
79
+
set_register:
80
+
asm volatile("mov %0,%%cr4": "+r" (val), "+m" (cr4_pinned_bits));
81
+
82
+
if (static_branch_likely(&cr_pinning)) {
83
+
if (unlikely((val & cr4_pinned_bits) != cr4_pinned_bits)) {
84
+
bits_missing = ~val & cr4_pinned_bits;
85
+
val |= bits_missing;
86
+
goto set_register;
87
+
}
88
+
/* Warn after we've set the missing bits. */
89
+
WARN_ONCE(bits_missing, "CR4 bits went missing: %lx!?\n",
90
+
bits_missing);
91
+
}
97
92
}
98
93
99
94
#ifdef CONFIG_X86_64
+1
-1
arch/x86/include/asm/stacktrace.h
+1
-1
arch/x86/include/asm/stacktrace.h
-2
arch/x86/include/asm/text-patching.h
-2
arch/x86/include/asm/text-patching.h
···
66
66
#define INT3_INSN_SIZE 1
67
67
#define CALL_INSN_SIZE 5
68
68
69
-
#ifdef CONFIG_X86_64
70
69
static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val)
71
70
{
72
71
/*
···
83
84
int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE);
84
85
int3_emulate_jmp(regs, func);
85
86
}
86
-
#endif /* CONFIG_X86_64 */
87
87
#endif /* !CONFIG_UML_X86 */
88
88
89
89
#endif /* _ASM_X86_TEXT_PATCHING_H */
+77
-4
arch/x86/kernel/alternative.c
+77
-4
arch/x86/kernel/alternative.c
···
616
616
__stop_parainstructions[];
617
617
#endif /* CONFIG_PARAVIRT */
618
618
619
+
/*
620
+
* Self-test for the INT3 based CALL emulation code.
621
+
*
622
+
* This exercises int3_emulate_call() to make sure INT3 pt_regs are set up
623
+
* properly and that there is a stack gap between the INT3 frame and the
624
+
* previous context. Without this gap doing a virtual PUSH on the interrupted
625
+
* stack would corrupt the INT3 IRET frame.
626
+
*
627
+
* See entry_{32,64}.S for more details.
628
+
*/
629
+
static void __init int3_magic(unsigned int *ptr)
630
+
{
631
+
*ptr = 1;
632
+
}
633
+
634
+
extern __initdata unsigned long int3_selftest_ip; /* defined in asm below */
635
+
636
+
static int __init
637
+
int3_exception_notify(struct notifier_block *self, unsigned long val, void *data)
638
+
{
639
+
struct die_args *args = data;
640
+
struct pt_regs *regs = args->regs;
641
+
642
+
if (!regs || user_mode(regs))
643
+
return NOTIFY_DONE;
644
+
645
+
if (val != DIE_INT3)
646
+
return NOTIFY_DONE;
647
+
648
+
if (regs->ip - INT3_INSN_SIZE != int3_selftest_ip)
649
+
return NOTIFY_DONE;
650
+
651
+
int3_emulate_call(regs, (unsigned long)&int3_magic);
652
+
return NOTIFY_STOP;
653
+
}
654
+
655
+
static void __init int3_selftest(void)
656
+
{
657
+
static __initdata struct notifier_block int3_exception_nb = {
658
+
.notifier_call = int3_exception_notify,
659
+
.priority = INT_MAX-1, /* last */
660
+
};
661
+
unsigned int val = 0;
662
+
663
+
BUG_ON(register_die_notifier(&int3_exception_nb));
664
+
665
+
/*
666
+
* Basically: int3_magic(&val); but really complicated :-)
667
+
*
668
+
* Stick the address of the INT3 instruction into int3_selftest_ip,
669
+
* then trigger the INT3, padded with NOPs to match a CALL instruction
670
+
* length.
671
+
*/
672
+
asm volatile ("1: int3; nop; nop; nop; nop\n\t"
673
+
".pushsection .init.data,\"aw\"\n\t"
674
+
".align " __ASM_SEL(4, 8) "\n\t"
675
+
".type int3_selftest_ip, @object\n\t"
676
+
".size int3_selftest_ip, " __ASM_SEL(4, 8) "\n\t"
677
+
"int3_selftest_ip:\n\t"
678
+
__ASM_SEL(.long, .quad) " 1b\n\t"
679
+
".popsection\n\t"
680
+
: : __ASM_SEL_RAW(a, D) (&val) : "memory");
681
+
682
+
BUG_ON(val != 1);
683
+
684
+
unregister_die_notifier(&int3_exception_nb);
685
+
}
686
+
619
687
void __init alternative_instructions(void)
620
688
{
621
-
/* The patching is not fully atomic, so try to avoid local interruptions
622
-
that might execute the to be patched code.
623
-
Other CPUs are not running. */
689
+
int3_selftest();
690
+
691
+
/*
692
+
* The patching is not fully atomic, so try to avoid local
693
+
* interruptions that might execute the to be patched code.
694
+
* Other CPUs are not running.
695
+
*/
624
696
stop_nmi();
625
697
626
698
/*
···
717
645
_text, _etext);
718
646
}
719
647
720
-
if (!uniproc_patched || num_possible_cpus() == 1)
648
+
if (!uniproc_patched || num_possible_cpus() == 1) {
721
649
free_init_pages("SMP alternatives",
722
650
(unsigned long)__smp_locks,
723
651
(unsigned long)__smp_locks_end);
652
+
}
724
653
#endif
725
654
726
655
apply_paravirt(__parainstructions, __parainstructions_end);
-1
arch/x86/kernel/asm-offsets.c
-1
arch/x86/kernel/asm-offsets.c
+20
arch/x86/kernel/cpu/common.c
+20
arch/x86/kernel/cpu/common.c
···
366
366
cr4_clear_bits(X86_CR4_UMIP);
367
367
}
368
368
369
+
DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
370
+
EXPORT_SYMBOL(cr_pinning);
371
+
unsigned long cr4_pinned_bits __ro_after_init;
372
+
EXPORT_SYMBOL(cr4_pinned_bits);
373
+
374
+
/*
375
+
* Once CPU feature detection is finished (and boot params have been
376
+
* parsed), record any of the sensitive CR bits that are set, and
377
+
* enable CR pinning.
378
+
*/
379
+
static void __init setup_cr_pinning(void)
380
+
{
381
+
unsigned long mask;
382
+
383
+
mask = (X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP);
384
+
cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & mask;
385
+
static_key_enable(&cr_pinning.key);
386
+
}
387
+
369
388
/*
370
389
* Protection Keys are not available in 32-bit mode.
371
390
*/
···
1487
1468
enable_sep_cpu();
1488
1469
#endif
1489
1470
cpu_detect_tlb(&boot_cpu_data);
1471
+
setup_cr_pinning();
1490
1472
}
1491
1473
1492
1474
void identify_secondary_cpu(struct cpuinfo_x86 *c)
-8
arch/x86/kernel/crash.c
-8
arch/x86/kernel/crash.c
···
73
73
74
74
static void kdump_nmi_callback(int cpu, struct pt_regs *regs)
75
75
{
76
-
#ifdef CONFIG_X86_32
77
-
struct pt_regs fixed_regs;
78
-
79
-
if (!user_mode(regs)) {
80
-
crash_fixup_ss_esp(&fixed_regs, regs);
81
-
regs = &fixed_regs;
82
-
}
83
-
#endif
84
76
crash_save_cpu(regs, cpu);
85
77
86
78
/*
-7
arch/x86/kernel/ftrace.c
-7
arch/x86/kernel/ftrace.c
···
310
310
311
311
ip = regs->ip - INT3_INSN_SIZE;
312
312
313
-
#ifdef CONFIG_X86_64
314
313
if (ftrace_location(ip)) {
315
314
int3_emulate_call(regs, (unsigned long)ftrace_regs_caller);
316
315
return 1;
···
321
322
int3_emulate_call(regs, ftrace_update_func_call);
322
323
return 1;
323
324
}
324
-
#else
325
-
if (ftrace_location(ip) || is_ftrace_caller(ip)) {
326
-
int3_emulate_jmp(regs, ip + CALL_INSN_SIZE);
327
-
return 1;
328
-
}
329
-
#endif
330
325
331
326
return 0;
332
327
}
+45
-33
arch/x86/kernel/ftrace_32.S
+45
-33
arch/x86/kernel/ftrace_32.S
···
9
9
#include <asm/export.h>
10
10
#include <asm/ftrace.h>
11
11
#include <asm/nospec-branch.h>
12
+
#include <asm/frame.h>
13
+
#include <asm/asm-offsets.h>
12
14
13
15
# define function_hook __fentry__
14
16
EXPORT_SYMBOL(__fentry__)
···
91
89
92
90
ENTRY(ftrace_regs_caller)
93
91
/*
94
-
* i386 does not save SS and ESP when coming from kernel.
95
-
* Instead, to get sp, ®s->sp is used (see ptrace.h).
96
-
* Unfortunately, that means eflags must be at the same location
97
-
* as the current return ip is. We move the return ip into the
98
-
* regs->ip location, and move flags into the return ip location.
92
+
* We're here from an mcount/fentry CALL, and the stack frame looks like:
93
+
*
94
+
* <previous context>
95
+
* RET-IP
96
+
*
97
+
* The purpose of this function is to call out in an emulated INT3
98
+
* environment with a stack frame like:
99
+
*
100
+
* <previous context>
101
+
* gap / RET-IP
102
+
* gap
103
+
* gap
104
+
* gap
105
+
* pt_regs
106
+
*
107
+
* We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds
99
108
*/
100
-
pushl $__KERNEL_CS
101
-
pushl 4(%esp) /* Save the return ip */
102
-
pushl $0 /* Load 0 into orig_ax */
109
+
subl $3*4, %esp # RET-IP + 3 gaps
110
+
pushl %ss # ss
111
+
pushl %esp # points at ss
112
+
addl $5*4, (%esp) # make it point at <previous context>
113
+
pushfl # flags
114
+
pushl $__KERNEL_CS # cs
115
+
pushl 7*4(%esp) # ip <- RET-IP
116
+
pushl $0 # orig_eax
117
+
103
118
pushl %gs
104
119
pushl %fs
105
120
pushl %es
106
121
pushl %ds
122
+
107
123
pushl %eax
108
-
109
-
/* Get flags and place them into the return ip slot */
110
-
pushf
111
-
popl %eax
112
-
movl %eax, 8*4(%esp)
113
-
114
124
pushl %ebp
115
125
pushl %edi
116
126
pushl %esi
···
130
116
pushl %ecx
131
117
pushl %ebx
132
118
133
-
movl 12*4(%esp), %eax /* Load ip (1st parameter) */
134
-
subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */
135
-
movl 15*4(%esp), %edx /* Load parent ip (2nd parameter) */
136
-
movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */
137
-
pushl %esp /* Save pt_regs as 4th parameter */
119
+
ENCODE_FRAME_POINTER
120
+
121
+
movl PT_EIP(%esp), %eax # 1st argument: IP
122
+
subl $MCOUNT_INSN_SIZE, %eax
123
+
movl 21*4(%esp), %edx # 2nd argument: parent ip
124
+
movl function_trace_op, %ecx # 3rd argument: ftrace_pos
125
+
pushl %esp # 4th argument: pt_regs
138
126
139
127
GLOBAL(ftrace_regs_call)
140
128
call ftrace_stub
141
129
142
-
addl $4, %esp /* Skip pt_regs */
130
+
addl $4, %esp # skip 4th argument
143
131
144
-
/* restore flags */
145
-
push 14*4(%esp)
146
-
popf
132
+
/* place IP below the new SP */
133
+
movl PT_OLDESP(%esp), %eax
134
+
movl PT_EIP(%esp), %ecx
135
+
movl %ecx, -4(%eax)
147
136
148
-
/* Move return ip back to its original location */
149
-
movl 12*4(%esp), %eax
150
-
movl %eax, 14*4(%esp)
137
+
/* place EAX below that */
138
+
movl PT_EAX(%esp), %ecx
139
+
movl %ecx, -8(%eax)
151
140
152
141
popl %ebx
153
142
popl %ecx
···
158
141
popl %esi
159
142
popl %edi
160
143
popl %ebp
161
-
popl %eax
162
-
popl %ds
163
-
popl %es
164
-
popl %fs
165
-
popl %gs
166
144
167
-
/* use lea to not affect flags */
168
-
lea 3*4(%esp), %esp /* Skip orig_ax, ip and cs */
145
+
lea -8(%eax), %esp
146
+
popl %eax
169
147
170
148
jmp .Lftrace_ret
171
149
+3
arch/x86/kernel/ftrace_64.S
+3
arch/x86/kernel/ftrace_64.S
···
9
9
#include <asm/export.h>
10
10
#include <asm/nospec-branch.h>
11
11
#include <asm/unwind_hints.h>
12
+
#include <asm/frame.h>
12
13
13
14
.code64
14
15
.section .entry.text, "ax"
···
203
202
/* Stack - skipping return address and flags */
204
203
leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx
205
204
movq %rcx, RSP(%rsp)
205
+
206
+
ENCODE_FRAME_POINTER
206
207
207
208
/* regs go into 4th parameter */
208
209
leaq (%rsp), %rcx
-8
arch/x86/kernel/kgdb.c
-8
arch/x86/kernel/kgdb.c
···
118
118
119
119
#ifdef CONFIG_X86_32
120
120
switch (regno) {
121
-
case GDB_SS:
122
-
if (!user_mode(regs))
123
-
*(unsigned long *)mem = __KERNEL_DS;
124
-
break;
125
-
case GDB_SP:
126
-
if (!user_mode(regs))
127
-
*(unsigned long *)mem = kernel_stack_pointer(regs);
128
-
break;
129
121
case GDB_GS:
130
122
case GDB_FS:
131
123
*(unsigned long *)mem = 0xFFFF;
+14
-14
arch/x86/kernel/kprobes/common.h
+14
-14
arch/x86/kernel/kprobes/common.h
···
5
5
/* Kprobes and Optprobes common header */
6
6
7
7
#include <asm/asm.h>
8
-
9
-
#ifdef CONFIG_FRAME_POINTER
10
-
# define SAVE_RBP_STRING " push %" _ASM_BP "\n" \
11
-
" mov %" _ASM_SP ", %" _ASM_BP "\n"
12
-
#else
13
-
# define SAVE_RBP_STRING " push %" _ASM_BP "\n"
14
-
#endif
8
+
#include <asm/frame.h>
15
9
16
10
#ifdef CONFIG_X86_64
11
+
17
12
#define SAVE_REGS_STRING \
18
13
/* Skip cs, ip, orig_ax. */ \
19
14
" subq $24, %rsp\n" \
···
22
27
" pushq %r10\n" \
23
28
" pushq %r11\n" \
24
29
" pushq %rbx\n" \
25
-
SAVE_RBP_STRING \
30
+
" pushq %rbp\n" \
26
31
" pushq %r12\n" \
27
32
" pushq %r13\n" \
28
33
" pushq %r14\n" \
29
-
" pushq %r15\n"
34
+
" pushq %r15\n" \
35
+
ENCODE_FRAME_POINTER
36
+
30
37
#define RESTORE_REGS_STRING \
31
38
" popq %r15\n" \
32
39
" popq %r14\n" \
···
48
51
/* Skip orig_ax, ip, cs */ \
49
52
" addq $24, %rsp\n"
50
53
#else
54
+
51
55
#define SAVE_REGS_STRING \
52
56
/* Skip cs, ip, orig_ax and gs. */ \
53
-
" subl $16, %esp\n" \
57
+
" subl $4*4, %esp\n" \
54
58
" pushl %fs\n" \
55
59
" pushl %es\n" \
56
60
" pushl %ds\n" \
57
61
" pushl %eax\n" \
58
-
SAVE_RBP_STRING \
62
+
" pushl %ebp\n" \
59
63
" pushl %edi\n" \
60
64
" pushl %esi\n" \
61
65
" pushl %edx\n" \
62
66
" pushl %ecx\n" \
63
-
" pushl %ebx\n"
67
+
" pushl %ebx\n" \
68
+
ENCODE_FRAME_POINTER
69
+
64
70
#define RESTORE_REGS_STRING \
65
71
" popl %ebx\n" \
66
72
" popl %ecx\n" \
···
72
72
" popl %edi\n" \
73
73
" popl %ebp\n" \
74
74
" popl %eax\n" \
75
-
/* Skip ds, es, fs, gs, orig_ax, and ip. Note: don't pop cs here*/\
76
-
" addl $24, %esp\n"
75
+
/* Skip ds, es, fs, gs, orig_ax, ip, and cs. */\
76
+
" addl $7*4, %esp\n"
77
77
#endif
78
78
79
79
/* Ensure if the instruction can be boostable */
+12
-17
arch/x86/kernel/kprobes/core.c
+12
-17
arch/x86/kernel/kprobes/core.c
···
56
56
DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
57
57
DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
58
58
59
-
#define stack_addr(regs) ((unsigned long *)kernel_stack_pointer(regs))
59
+
#define stack_addr(regs) ((unsigned long *)regs->sp)
60
60
61
61
#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
62
62
(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
···
718
718
".global kretprobe_trampoline\n"
719
719
".type kretprobe_trampoline, @function\n"
720
720
"kretprobe_trampoline:\n"
721
-
#ifdef CONFIG_X86_64
722
721
/* We don't bother saving the ss register */
722
+
#ifdef CONFIG_X86_64
723
723
" pushq %rsp\n"
724
724
" pushfq\n"
725
725
SAVE_REGS_STRING
726
726
" movq %rsp, %rdi\n"
727
727
" call trampoline_handler\n"
728
728
/* Replace saved sp with true return address. */
729
-
" movq %rax, 152(%rsp)\n"
729
+
" movq %rax, 19*8(%rsp)\n"
730
730
RESTORE_REGS_STRING
731
731
" popfq\n"
732
732
#else
733
-
" pushf\n"
733
+
" pushl %esp\n"
734
+
" pushfl\n"
734
735
SAVE_REGS_STRING
735
736
" movl %esp, %eax\n"
736
737
" call trampoline_handler\n"
737
-
/* Move flags to cs */
738
-
" movl 56(%esp), %edx\n"
739
-
" movl %edx, 52(%esp)\n"
740
-
/* Replace saved flags with true return address. */
741
-
" movl %eax, 56(%esp)\n"
738
+
/* Replace saved sp with true return address. */
739
+
" movl %eax, 15*4(%esp)\n"
742
740
RESTORE_REGS_STRING
743
-
" popf\n"
741
+
" popfl\n"
744
742
#endif
745
743
" ret\n"
746
744
".size kretprobe_trampoline, .-kretprobe_trampoline\n"
···
779
781
INIT_HLIST_HEAD(&empty_rp);
780
782
kretprobe_hash_lock(current, &head, &flags);
781
783
/* fixup registers */
782
-
#ifdef CONFIG_X86_64
783
784
regs->cs = __KERNEL_CS;
784
-
/* On x86-64, we use pt_regs->sp for return address holder. */
785
-
frame_pointer = ®s->sp;
786
-
#else
787
-
regs->cs = __KERNEL_CS | get_kernel_rpl();
785
+
#ifdef CONFIG_X86_32
786
+
regs->cs |= get_kernel_rpl();
788
787
regs->gs = 0;
789
-
/* On x86-32, we use pt_regs->flags for return address holder. */
790
-
frame_pointer = ®s->flags;
791
788
#endif
789
+
/* We use pt_regs->sp for return address holder. */
790
+
frame_pointer = ®s->sp;
792
791
regs->ip = trampoline_address;
793
792
regs->orig_ax = ~0UL;
794
793
+12
-8
arch/x86/kernel/kprobes/opt.c
+12
-8
arch/x86/kernel/kprobes/opt.c
···
102
102
"optprobe_template_call:\n"
103
103
ASM_NOP5
104
104
/* Move flags to rsp */
105
-
" movq 144(%rsp), %rdx\n"
106
-
" movq %rdx, 152(%rsp)\n"
105
+
" movq 18*8(%rsp), %rdx\n"
106
+
" movq %rdx, 19*8(%rsp)\n"
107
107
RESTORE_REGS_STRING
108
108
/* Skip flags entry */
109
109
" addq $8, %rsp\n"
110
110
" popfq\n"
111
111
#else /* CONFIG_X86_32 */
112
-
" pushf\n"
112
+
" pushl %esp\n"
113
+
" pushfl\n"
113
114
SAVE_REGS_STRING
114
115
" movl %esp, %edx\n"
115
116
".global optprobe_template_val\n"
···
119
118
".global optprobe_template_call\n"
120
119
"optprobe_template_call:\n"
121
120
ASM_NOP5
121
+
/* Move flags into esp */
122
+
" movl 14*4(%esp), %edx\n"
123
+
" movl %edx, 15*4(%esp)\n"
122
124
RESTORE_REGS_STRING
123
-
" addl $4, %esp\n" /* skip cs */
124
-
" popf\n"
125
+
/* Skip flags entry */
126
+
" addl $4, %esp\n"
127
+
" popfl\n"
125
128
#endif
126
129
".global optprobe_template_end\n"
127
130
"optprobe_template_end:\n"
···
157
152
} else {
158
153
struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
159
154
/* Save skipped registers */
160
-
#ifdef CONFIG_X86_64
161
155
regs->cs = __KERNEL_CS;
162
-
#else
163
-
regs->cs = __KERNEL_CS | get_kernel_rpl();
156
+
#ifdef CONFIG_X86_32
157
+
regs->cs |= get_kernel_rpl();
164
158
regs->gs = 0;
165
159
#endif
166
160
regs->ip = (unsigned long)op->kp.addr + INT3_SIZE;
+5
-11
arch/x86/kernel/process_32.c
+5
-11
arch/x86/kernel/process_32.c
···
62
62
{
63
63
unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L;
64
64
unsigned long d0, d1, d2, d3, d6, d7;
65
-
unsigned long sp;
66
-
unsigned short ss, gs;
65
+
unsigned short gs;
67
66
68
-
if (user_mode(regs)) {
69
-
sp = regs->sp;
70
-
ss = regs->ss;
67
+
if (user_mode(regs))
71
68
gs = get_user_gs(regs);
72
-
} else {
73
-
sp = kernel_stack_pointer(regs);
74
-
savesegment(ss, ss);
69
+
else
75
70
savesegment(gs, gs);
76
-
}
77
71
78
72
show_ip(regs, KERN_DEFAULT);
79
73
80
74
printk(KERN_DEFAULT "EAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n",
81
75
regs->ax, regs->bx, regs->cx, regs->dx);
82
76
printk(KERN_DEFAULT "ESI: %08lx EDI: %08lx EBP: %08lx ESP: %08lx\n",
83
-
regs->si, regs->di, regs->bp, sp);
77
+
regs->si, regs->di, regs->bp, regs->sp);
84
78
printk(KERN_DEFAULT "DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x EFLAGS: %08lx\n",
85
-
(u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, ss, regs->flags);
79
+
(u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, regs->ss, regs->flags);
86
80
87
81
if (mode != SHOW_REGS_ALL)
88
82
return;
-29
arch/x86/kernel/ptrace.c
-29
arch/x86/kernel/ptrace.c
···
155
155
156
156
#define FLAG_MASK FLAG_MASK_32
157
157
158
-
/*
159
-
* X86_32 CPUs don't save ss and esp if the CPU is already in kernel mode
160
-
* when it traps. The previous stack will be directly underneath the saved
161
-
* registers, and 'sp/ss' won't even have been saved. Thus the '®s->sp'.
162
-
*
163
-
* Now, if the stack is empty, '®s->sp' is out of range. In this
164
-
* case we try to take the previous stack. To always return a non-null
165
-
* stack pointer we fall back to regs as stack if no previous stack
166
-
* exists.
167
-
*
168
-
* This is valid only for kernel mode traps.
169
-
*/
170
-
unsigned long kernel_stack_pointer(struct pt_regs *regs)
171
-
{
172
-
unsigned long context = (unsigned long)regs & ~(THREAD_SIZE - 1);
173
-
unsigned long sp = (unsigned long)®s->sp;
174
-
u32 *prev_esp;
175
-
176
-
if (context == (sp & ~(THREAD_SIZE - 1)))
177
-
return sp;
178
-
179
-
prev_esp = (u32 *)(context);
180
-
if (*prev_esp)
181
-
return (unsigned long)*prev_esp;
182
-
183
-
return (unsigned long)regs;
184
-
}
185
-
EXPORT_SYMBOL_GPL(kernel_stack_pointer);
186
-
187
158
static unsigned long *pt_regs_access(struct pt_regs *regs, unsigned long regno)
188
159
{
189
160
BUILD_BUG_ON(offsetof(struct pt_regs, bx) != 0);
+7
-1
arch/x86/kernel/smpboot.c
+7
-1
arch/x86/kernel/smpboot.c
···
205
205
*/
206
206
static void notrace start_secondary(void *unused)
207
207
{
208
+
unsigned long cr4 = __read_cr4();
209
+
208
210
/*
209
211
* Don't put *anything* except direct CPU state initialization
210
212
* before cpu_init(), SMP booting is too fragile that we want to
211
213
* limit the things done here to the most necessary things.
212
214
*/
213
215
if (boot_cpu_has(X86_FEATURE_PCID))
214
-
__write_cr4(__read_cr4() | X86_CR4_PCIDE);
216
+
cr4 |= X86_CR4_PCIDE;
217
+
if (static_branch_likely(&cr_pinning))
218
+
cr4 |= cr4_pinned_bits;
219
+
220
+
__write_cr4(cr4);
215
221
216
222
#ifdef CONFIG_X86_32
217
223
/* switch away from the initial page table */
+1
-2
arch/x86/kernel/time.c
+1
-2
arch/x86/kernel/time.c
···
37
37
#ifdef CONFIG_FRAME_POINTER
38
38
return *(unsigned long *)(regs->bp + sizeof(long));
39
39
#else
40
-
unsigned long *sp =
41
-
(unsigned long *)kernel_stack_pointer(regs);
40
+
unsigned long *sp = (unsigned long *)regs->sp;
42
41
/*
43
42
* Return address is either directly at stack pointer
44
43
* or above a saved flags. Eflags has bits 22-31 zero,
+4
-28
arch/x86/kernel/unwind_frame.c
+4
-28
arch/x86/kernel/unwind_frame.c
···
70
70
}
71
71
}
72
72
73
-
static size_t regs_size(struct pt_regs *regs)
74
-
{
75
-
/* x86_32 regs from kernel mode are two words shorter: */
76
-
if (IS_ENABLED(CONFIG_X86_32) && !user_mode(regs))
77
-
return sizeof(*regs) - 2*sizeof(long);
78
-
79
-
return sizeof(*regs);
80
-
}
81
-
82
73
static bool in_entry_code(unsigned long ip)
83
74
{
84
75
char *addr = (char *)ip;
···
189
198
}
190
199
#endif
191
200
192
-
#ifdef CONFIG_X86_32
193
-
#define KERNEL_REGS_SIZE (sizeof(struct pt_regs) - 2*sizeof(long))
194
-
#else
195
-
#define KERNEL_REGS_SIZE (sizeof(struct pt_regs))
196
-
#endif
197
-
198
201
static bool update_stack_state(struct unwind_state *state,
199
202
unsigned long *next_bp)
200
203
{
···
199
214
size_t len;
200
215
201
216
if (state->regs)
202
-
prev_frame_end = (void *)state->regs + regs_size(state->regs);
217
+
prev_frame_end = (void *)state->regs + sizeof(*state->regs);
203
218
else
204
219
prev_frame_end = (void *)state->bp + FRAME_HEADER_SIZE;
205
220
···
207
222
regs = decode_frame_pointer(next_bp);
208
223
if (regs) {
209
224
frame = (unsigned long *)regs;
210
-
len = KERNEL_REGS_SIZE;
225
+
len = sizeof(*regs);
211
226
state->got_irq = true;
212
227
} else {
213
228
frame = next_bp;
···
229
244
/* Make sure it only unwinds up and doesn't overlap the prev frame: */
230
245
if (state->orig_sp && state->stack_info.type == prev_type &&
231
246
frame < prev_frame_end)
232
-
return false;
233
-
234
-
/*
235
-
* On 32-bit with user mode regs, make sure the last two regs are safe
236
-
* to access:
237
-
*/
238
-
if (IS_ENABLED(CONFIG_X86_32) && regs && user_mode(regs) &&
239
-
!on_stack(info, frame, len + 2*sizeof(long)))
240
247
return false;
241
248
242
249
/* Move state to the next frame: */
···
389
412
* Pretend that the frame is complete and that BP points to it, but save
390
413
* the real BP so that we can use it when looking for the next frame.
391
414
*/
392
-
if (regs && regs->ip == 0 &&
393
-
(unsigned long *)kernel_stack_pointer(regs) >= first_frame) {
415
+
if (regs && regs->ip == 0 && (unsigned long *)regs->sp >= first_frame) {
394
416
state->next_bp = bp;
395
-
bp = ((unsigned long *)kernel_stack_pointer(regs)) - 1;
417
+
bp = ((unsigned long *)regs->sp) - 1;
396
418
}
397
419
398
420
/* Initialize stack info and make sure the frame data is accessible: */