Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'x86-uaccess-cleanup': x86 uaccess header cleanups
Merge my x86 uaccess updates branch.
The LAM ("Linear Address Masking") updates in this release made me
unhappy about how "access_ok()" was done, and it actually turned out to
have a couple of small bugs in it too. This is my cleanup of the code:
- use the sign bit of the __user pointer rather than masking the
address and checking it against the TASK_SIZE range.
We already did this part for the get/put_user() side, but
'access_ok()' did the naïve "mask and range check" thing, which not
only generates nasty code, but also ended up meaning that __access_ok
itself didn't do a good job, and so copy_from_user_nmi() didn't get
the check right.
- move all the code that is 64-bit only into the 64-bit version of the
header file, so that we don't unnecessarily pollute the shared x86
code and make it look like LAM might work in 32-bit too.
- fix a bug in the address masking (that doesn't end up mattering: in
this case the fix was to just remove the buggy code entirely).
- a couple of trivial cleanups and added commentary about the
access_ok() rules.
* x86-uaccess-cleanup:
x86-64: mm: clarify the 'positive addresses' user address rules
x86: mm: remove 'sign' games from LAM untagged_addr*() macros
x86: uaccess: move 32-bit and 64-bit parts into proper <asm/uaccess_N.h> header
x86: mm: remove architecture-specific 'access_ok()' define
x86-64: make access_ok() independent of LAM
+125
-95
+3
-87
arch/x86/include/asm/uaccess.h
+3
-87
arch/x86/include/asm/uaccess.h
···
16
16
#include <asm/extable.h>
17
17
#include <asm/tlbflush.h>
18
18
19
-
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
20
-
static inline bool pagefault_disabled(void);
21
-
# define WARN_ON_IN_IRQ() \
22
-
WARN_ON_ONCE(!in_task() && !pagefault_disabled())
19
+
#ifdef CONFIG_X86_32
20
+
# include <asm/uaccess_32.h>
23
21
#else
24
-
# define WARN_ON_IN_IRQ()
22
+
# include <asm/uaccess_64.h>
25
23
#endif
26
-
27
-
#ifdef CONFIG_ADDRESS_MASKING
28
-
/*
29
-
* Mask out tag bits from the address.
30
-
*
31
-
* Magic with the 'sign' allows to untag userspace pointer without any branches
32
-
* while leaving kernel addresses intact.
33
-
*/
34
-
static inline unsigned long __untagged_addr(unsigned long addr)
35
-
{
36
-
long sign;
37
-
38
-
/*
39
-
* Refer tlbstate_untag_mask directly to avoid RIP-relative relocation
40
-
* in alternative instructions. The relocation gets wrong when gets
41
-
* copied to the target place.
42
-
*/
43
-
asm (ALTERNATIVE("",
44
-
"sar $63, %[sign]\n\t" /* user_ptr ? 0 : -1UL */
45
-
"or %%gs:tlbstate_untag_mask, %[sign]\n\t"
46
-
"and %[sign], %[addr]\n\t", X86_FEATURE_LAM)
47
-
: [addr] "+r" (addr), [sign] "=r" (sign)
48
-
: "m" (tlbstate_untag_mask), "[sign]" (addr));
49
-
50
-
return addr;
51
-
}
52
-
53
-
#define untagged_addr(addr) ({ \
54
-
unsigned long __addr = (__force unsigned long)(addr); \
55
-
(__force __typeof__(addr))__untagged_addr(__addr); \
56
-
})
57
-
58
-
static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,
59
-
unsigned long addr)
60
-
{
61
-
long sign = addr >> 63;
62
-
63
-
mmap_assert_locked(mm);
64
-
addr &= (mm)->context.untag_mask | sign;
65
-
66
-
return addr;
67
-
}
68
-
69
-
#define untagged_addr_remote(mm, addr) ({ \
70
-
unsigned long __addr = (__force unsigned long)(addr); \
71
-
(__force __typeof__(addr))__untagged_addr_remote(mm, __addr); \
72
-
})
73
-
74
-
#else
75
-
#define untagged_addr(addr) (addr)
76
-
#endif
77
-
78
-
/**
79
-
* access_ok - Checks if a user space pointer is valid
80
-
* @addr: User space pointer to start of block to check
81
-
* @size: Size of block to check
82
-
*
83
-
* Context: User context only. This function may sleep if pagefaults are
84
-
* enabled.
85
-
*
86
-
* Checks if a pointer to a block of memory in user space is valid.
87
-
*
88
-
* Note that, depending on architecture, this function probably just
89
-
* checks that the pointer is in the user space range - after calling
90
-
* this function, memory access functions may still return -EFAULT.
91
-
*
92
-
* Return: true (nonzero) if the memory block may be valid, false (zero)
93
-
* if it is definitely invalid.
94
-
*/
95
-
#define access_ok(addr, size) \
96
-
({ \
97
-
WARN_ON_IN_IRQ(); \
98
-
likely(__access_ok(untagged_addr(addr), size)); \
99
-
})
100
24
101
25
#include <asm-generic/access_ok.h>
102
26
···
509
585
#endif
510
586
511
587
#define ARCH_HAS_NOCACHE_UACCESS 1
512
-
513
-
#ifdef CONFIG_X86_32
514
-
unsigned long __must_check clear_user(void __user *mem, unsigned long len);
515
-
unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
516
-
# include <asm/uaccess_32.h>
517
-
#else
518
-
# include <asm/uaccess_64.h>
519
-
#endif
520
588
521
589
/*
522
590
* The "unsafe" user accesses aren't really "unsafe", but the naming
+3
arch/x86/include/asm/uaccess_32.h
+3
arch/x86/include/asm/uaccess_32.h
+82
-1
arch/x86/include/asm/uaccess_64.h
+82
-1
arch/x86/include/asm/uaccess_64.h
···
12
12
#include <asm/cpufeatures.h>
13
13
#include <asm/page.h>
14
14
15
+
#ifdef CONFIG_ADDRESS_MASKING
16
+
/*
17
+
* Mask out tag bits from the address.
18
+
*/
19
+
static inline unsigned long __untagged_addr(unsigned long addr)
20
+
{
21
+
/*
22
+
* Refer tlbstate_untag_mask directly to avoid RIP-relative relocation
23
+
* in alternative instructions. The relocation gets wrong when gets
24
+
* copied to the target place.
25
+
*/
26
+
asm (ALTERNATIVE("",
27
+
"and %%gs:tlbstate_untag_mask, %[addr]\n\t", X86_FEATURE_LAM)
28
+
: [addr] "+r" (addr) : "m" (tlbstate_untag_mask));
29
+
30
+
return addr;
31
+
}
32
+
33
+
#define untagged_addr(addr) ({ \
34
+
unsigned long __addr = (__force unsigned long)(addr); \
35
+
(__force __typeof__(addr))__untagged_addr(__addr); \
36
+
})
37
+
38
+
static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,
39
+
unsigned long addr)
40
+
{
41
+
mmap_assert_locked(mm);
42
+
return addr & (mm)->context.untag_mask;
43
+
}
44
+
45
+
#define untagged_addr_remote(mm, addr) ({ \
46
+
unsigned long __addr = (__force unsigned long)(addr); \
47
+
(__force __typeof__(addr))__untagged_addr_remote(mm, __addr); \
48
+
})
49
+
50
+
#endif
51
+
52
+
/*
53
+
* The virtual address space space is logically divided into a kernel
54
+
* half and a user half. When cast to a signed type, user pointers
55
+
* are positive and kernel pointers are negative.
56
+
*/
57
+
#define valid_user_address(x) ((long)(x) >= 0)
58
+
59
+
/*
60
+
* User pointers can have tag bits on x86-64. This scheme tolerates
61
+
* arbitrary values in those bits rather then masking them off.
62
+
*
63
+
* Enforce two rules:
64
+
* 1. 'ptr' must be in the user half of the address space
65
+
* 2. 'ptr+size' must not overflow into kernel addresses
66
+
*
67
+
* Note that addresses around the sign change are not valid addresses,
68
+
* and will GP-fault even with LAM enabled if the sign bit is set (see
69
+
* "CR3.LAM_SUP" that can narrow the canonicality check if we ever
70
+
* enable it, but not remove it entirely).
71
+
*
72
+
* So the "overflow into kernel addresses" does not imply some sudden
73
+
* exact boundary at the sign bit, and we can allow a lot of slop on the
74
+
* size check.
75
+
*
76
+
* In fact, we could probably remove the size check entirely, since
77
+
* any kernel accesses will be in increasing address order starting
78
+
* at 'ptr', and even if the end might be in kernel space, we'll
79
+
* hit the GP faults for non-canonical accesses before we ever get
80
+
* there.
81
+
*
82
+
* That's a separate optimization, for now just handle the small
83
+
* constant case.
84
+
*/
85
+
static inline bool __access_ok(const void __user *ptr, unsigned long size)
86
+
{
87
+
if (__builtin_constant_p(size <= PAGE_SIZE) && size <= PAGE_SIZE) {
88
+
return valid_user_address(ptr);
89
+
} else {
90
+
unsigned long sum = size + (unsigned long)ptr;
91
+
return valid_user_address(sum) && sum >= (unsigned long)ptr;
92
+
}
93
+
}
94
+
#define __access_ok __access_ok
95
+
15
96
/*
16
97
* Copy To/From Userspace
17
98
*/
···
187
106
188
107
static __always_inline unsigned long clear_user(void __user *to, unsigned long n)
189
108
{
190
-
if (access_ok(to, n))
109
+
if (__access_ok(to, n))
191
110
return __clear_user(to, n);
192
111
return n;
193
112
}
+35
-7
arch/x86/mm/extable.c
+35
-7
arch/x86/mm/extable.c
···
130
130
return true;
131
131
}
132
132
133
-
static bool ex_handler_uaccess(const struct exception_table_entry *fixup,
134
-
struct pt_regs *regs, int trapnr)
133
+
/*
134
+
* On x86-64, we end up being imprecise with 'access_ok()', and allow
135
+
* non-canonical user addresses to make the range comparisons simpler,
136
+
* and to not have to worry about LAM being enabled.
137
+
*
138
+
* In fact, we allow up to one page of "slop" at the sign boundary,
139
+
* which means that we can do access_ok() by just checking the sign
140
+
* of the pointer for the common case of having a small access size.
141
+
*/
142
+
static bool gp_fault_address_ok(unsigned long fault_address)
135
143
{
136
-
WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?");
144
+
#ifdef CONFIG_X86_64
145
+
/* Is it in the "user space" part of the non-canonical space? */
146
+
if (valid_user_address(fault_address))
147
+
return true;
148
+
149
+
/* .. or just above it? */
150
+
fault_address -= PAGE_SIZE;
151
+
if (valid_user_address(fault_address))
152
+
return true;
153
+
#endif
154
+
return false;
155
+
}
156
+
157
+
static bool ex_handler_uaccess(const struct exception_table_entry *fixup,
158
+
struct pt_regs *regs, int trapnr,
159
+
unsigned long fault_address)
160
+
{
161
+
WARN_ONCE(trapnr == X86_TRAP_GP && !gp_fault_address_ok(fault_address),
162
+
"General protection fault in user access. Non-canonical address?");
137
163
return ex_handler_default(fixup, regs);
138
164
}
139
165
···
215
189
}
216
190
217
191
static bool ex_handler_ucopy_len(const struct exception_table_entry *fixup,
218
-
struct pt_regs *regs, int trapnr, int reg, int imm)
192
+
struct pt_regs *regs, int trapnr,
193
+
unsigned long fault_address,
194
+
int reg, int imm)
219
195
{
220
196
regs->cx = imm * regs->cx + *pt_regs_nr(regs, reg);
221
-
return ex_handler_uaccess(fixup, regs, trapnr);
197
+
return ex_handler_uaccess(fixup, regs, trapnr, fault_address);
222
198
}
223
199
224
200
int ex_get_fixup_type(unsigned long ip)
···
266
238
case EX_TYPE_FAULT_MCE_SAFE:
267
239
return ex_handler_fault(e, regs, trapnr);
268
240
case EX_TYPE_UACCESS:
269
-
return ex_handler_uaccess(e, regs, trapnr);
241
+
return ex_handler_uaccess(e, regs, trapnr, fault_addr);
270
242
case EX_TYPE_COPY:
271
243
return ex_handler_copy(e, regs, trapnr);
272
244
case EX_TYPE_CLEAR_FS:
···
297
269
case EX_TYPE_FAULT_SGX:
298
270
return ex_handler_sgx(e, regs, trapnr);
299
271
case EX_TYPE_UCOPY_LEN:
300
-
return ex_handler_ucopy_len(e, regs, trapnr, reg, imm);
272
+
return ex_handler_ucopy_len(e, regs, trapnr, fault_addr, reg, imm);
301
273
case EX_TYPE_ZEROPAD:
302
274
return ex_handler_zeropad(e, regs, fault_addr);
303
275
}