Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge tag 'core-uaccess-2025-11-30' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scoped user access updates from Thomas Gleixner:
"Scoped user mode access and related changes:

- Implement the missing u64 user access function on ARM when
CONFIG_CPU_SPECTRE=n.

This makes it possible to access a 64bit value in generic code with
[unsafe_]get_user(). All other architectures and ARM variants
provide the relevant accessors already.

- Ensure that ASM GOTO jump label usage in the user mode access
helpers always goes through a local C scope label indirection
inside the helpers.

This is required because compilers are not supporting that a ASM
GOTO target leaves a auto cleanup scope. GCC silently fails to emit
the cleanup invocation and CLANG fails the build.

[ Editor's note: gcc-16 will have fixed the code generation issue
in commit f68fe3ddda4 ("eh: Invoke cleanups/destructors in asm
goto jumps [PR122835]"). But we obviously have to deal with clang
and older versions of gcc, so.. - Linus ]

This provides generic wrapper macros and the conversion of affected
architecture code to use them.

- Scoped user mode access with auto cleanup

Access to user mode memory can be required in hot code paths, but
if it has to be done with user controlled pointers, the access is
shielded with a speculation barrier, so that the CPU cannot
speculate around the address range check. Those speculation
barriers impact performance quite significantly.

This cost can be avoided by "masking" the provided pointer so it is
guaranteed to be in the valid user memory access range and
otherwise to point to a guaranteed unpopulated address space. This
has to be done without branches so it creates an address dependency
for the access, which the CPU cannot speculate ahead.

This results in repeating and error prone programming patterns:

if (can_do_masked_user_access())
from = masked_user_read_access_begin((from));
else if (!user_read_access_begin(from, sizeof(*from)))
return -EFAULT;
unsafe_get_user(val, from, Efault);
user_read_access_end();
return 0;
Efault:
user_read_access_end();
return -EFAULT;

which can be replaced with scopes and automatic cleanup:

scoped_user_read_access(from, Efault)
unsafe_get_user(val, from, Efault);
return 0;
Efault:
return -EFAULT;

- Convert code which implements the above pattern over to
scope_user.*.access(). This also corrects a couple of imbalanced
masked_*_begin() instances which are harmless on most
architectures, but prevent PowerPC from implementing the masking
optimization.

- Add a missing speculation barrier in copy_from_user_iter()"

* tag 'core-uaccess-2025-11-30' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
lib/strn*,uaccess: Use masked_user_{read/write}_access_begin when required
scm: Convert put_cmsg() to scoped user access
iov_iter: Add missing speculation barrier to copy_from_user_iter()
iov_iter: Convert copy_from_user_iter() to masked user access
select: Convert to scoped user access
x86/futex: Convert to scoped user access
futex: Convert to get/put_user_inline()
uaccess: Provide put/get_user_inline()
uaccess: Provide scoped user access regions
arm64: uaccess: Use unsafe wrappers for ASM GOTO
s390/uaccess: Use unsafe wrappers for ASM GOTO
riscv/uaccess: Use unsafe wrappers for ASM GOTO
powerpc/uaccess: Use unsafe wrappers for ASM GOTO
x86/uaccess: Use unsafe wrappers for ASM GOTO
uaccess: Provide ASM GOTO safe wrappers for unsafe_*_user()
ARM: uaccess: Implement missing __get_user_asm_dword()

+421 -150
+25 -1
arch/arm/include/asm/uaccess.h
··· 283 283 __gu_err; \ 284 284 }) 285 285 286 + /* 287 + * This is a type: either unsigned long, if the argument fits into 288 + * that type, or otherwise unsigned long long. 289 + */ 290 + #define __long_type(x) \ 291 + __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) 292 + 286 293 #define __get_user_err(x, ptr, err, __t) \ 287 294 do { \ 288 295 unsigned long __gu_addr = (unsigned long)(ptr); \ 289 - unsigned long __gu_val; \ 296 + __long_type(x) __gu_val; \ 290 297 unsigned int __ua_flags; \ 291 298 __chk_user_ptr(ptr); \ 292 299 might_fault(); \ ··· 302 295 case 1: __get_user_asm_byte(__gu_val, __gu_addr, err, __t); break; \ 303 296 case 2: __get_user_asm_half(__gu_val, __gu_addr, err, __t); break; \ 304 297 case 4: __get_user_asm_word(__gu_val, __gu_addr, err, __t); break; \ 298 + case 8: __get_user_asm_dword(__gu_val, __gu_addr, err, __t); break; \ 305 299 default: (__gu_val) = __get_user_bad(); \ 306 300 } \ 307 301 uaccess_restore(__ua_flags); \ ··· 360 352 361 353 #define __get_user_asm_word(x, addr, err, __t) \ 362 354 __get_user_asm(x, addr, err, "ldr" __t) 355 + 356 + #ifdef __ARMEB__ 357 + #define __WORD0_OFFS 4 358 + #define __WORD1_OFFS 0 359 + #else 360 + #define __WORD0_OFFS 0 361 + #define __WORD1_OFFS 4 362 + #endif 363 + 364 + #define __get_user_asm_dword(x, addr, err, __t) \ 365 + ({ \ 366 + unsigned long __w0, __w1; \ 367 + __get_user_asm(__w0, addr + __WORD0_OFFS, err, "ldr" __t); \ 368 + __get_user_asm(__w1, addr + __WORD1_OFFS, err, "ldr" __t); \ 369 + (x) = ((u64)__w1 << 32) | (u64) __w0; \ 370 + }) 363 371 364 372 #define __put_user_switch(x, ptr, __err, __fn) \ 365 373 do { \
+2 -2
arch/arm64/include/asm/uaccess.h
··· 422 422 } 423 423 #define user_access_begin(a,b) user_access_begin(a,b) 424 424 #define user_access_end() uaccess_ttbr0_disable() 425 - #define unsafe_put_user(x, ptr, label) \ 425 + #define arch_unsafe_put_user(x, ptr, label) \ 426 426 __raw_put_mem("sttr", x, uaccess_mask_ptr(ptr), label, U) 427 - #define unsafe_get_user(x, ptr, label) \ 427 + #define arch_unsafe_get_user(x, ptr, label) \ 428 428 __raw_get_mem("ldtr", x, uaccess_mask_ptr(ptr), label, U) 429 429 430 430 /*
+4 -4
arch/powerpc/include/asm/uaccess.h
··· 451 451 #define user_write_access_begin user_write_access_begin 452 452 #define user_write_access_end prevent_current_write_to_user 453 453 454 - #define unsafe_get_user(x, p, e) do { \ 454 + #define arch_unsafe_get_user(x, p, e) do { \ 455 455 __long_type(*(p)) __gu_val; \ 456 456 __typeof__(*(p)) __user *__gu_addr = (p); \ 457 457 \ ··· 459 459 (x) = (__typeof__(*(p)))__gu_val; \ 460 460 } while (0) 461 461 462 - #define unsafe_put_user(x, p, e) \ 462 + #define arch_unsafe_put_user(x, p, e) \ 463 463 __put_user_size_goto((__typeof__(*(p)))(x), (p), sizeof(*(p)), e) 464 464 465 465 #define unsafe_copy_from_user(d, s, l, e) \ ··· 504 504 unsafe_put_user(*(u8*)(_src + _i), (u8 __user *)(_dst + _i), e); \ 505 505 } while (0) 506 506 507 - #define __get_kernel_nofault(dst, src, type, err_label) \ 507 + #define arch_get_kernel_nofault(dst, src, type, err_label) \ 508 508 __get_user_size_goto(*((type *)(dst)), \ 509 509 (__force type __user *)(src), sizeof(type), err_label) 510 510 511 - #define __put_kernel_nofault(dst, src, type, err_label) \ 511 + #define arch_put_kernel_nofault(dst, src, type, err_label) \ 512 512 __put_user_size_goto(*((type *)(src)), \ 513 513 (__force type __user *)(dst), sizeof(type), err_label) 514 514
+4 -4
arch/riscv/include/asm/uaccess.h
··· 437 437 __clear_user(untagged_addr(to), n) : n; 438 438 } 439 439 440 - #define __get_kernel_nofault(dst, src, type, err_label) \ 440 + #define arch_get_kernel_nofault(dst, src, type, err_label) \ 441 441 __get_user_nocheck(*((type *)(dst)), (__force __user type *)(src), err_label) 442 442 443 - #define __put_kernel_nofault(dst, src, type, err_label) \ 443 + #define arch_put_kernel_nofault(dst, src, type, err_label) \ 444 444 __put_user_nocheck(*((type *)(src)), (__force __user type *)(dst), err_label) 445 445 446 446 static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len) ··· 460 460 * We want the unsafe accessors to always be inlined and use 461 461 * the error labels - thus the macro games. 462 462 */ 463 - #define unsafe_put_user(x, ptr, label) \ 463 + #define arch_unsafe_put_user(x, ptr, label) \ 464 464 __put_user_nocheck(x, (ptr), label) 465 465 466 - #define unsafe_get_user(x, ptr, label) do { \ 466 + #define arch_unsafe_get_user(x, ptr, label) do { \ 467 467 __inttype(*(ptr)) __gu_val; \ 468 468 __get_user_nocheck(__gu_val, (ptr), label); \ 469 469 (x) = (__force __typeof__(*(ptr)))__gu_val; \
+2 -2
arch/s390/include/asm/uaccess.h
··· 468 468 469 469 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT && CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */ 470 470 471 - #define __get_kernel_nofault __mvc_kernel_nofault 472 - #define __put_kernel_nofault __mvc_kernel_nofault 471 + #define arch_get_kernel_nofault __mvc_kernel_nofault 472 + #define arch_put_kernel_nofault __mvc_kernel_nofault 473 473 474 474 void __cmpxchg_user_key_called_with_bad_pointer(void); 475 475
+33 -42
arch/x86/include/asm/futex.h
··· 46 46 } while(0) 47 47 48 48 static __always_inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, 49 - u32 __user *uaddr) 49 + u32 __user *uaddr) 50 50 { 51 - if (can_do_masked_user_access()) 52 - uaddr = masked_user_access_begin(uaddr); 53 - else if (!user_access_begin(uaddr, sizeof(u32))) 54 - return -EFAULT; 55 - 56 - switch (op) { 57 - case FUTEX_OP_SET: 58 - unsafe_atomic_op1("xchgl %0, %2", oval, uaddr, oparg, Efault); 59 - break; 60 - case FUTEX_OP_ADD: 61 - unsafe_atomic_op1(LOCK_PREFIX "xaddl %0, %2", oval, 62 - uaddr, oparg, Efault); 63 - break; 64 - case FUTEX_OP_OR: 65 - unsafe_atomic_op2("orl %4, %3", oval, uaddr, oparg, Efault); 66 - break; 67 - case FUTEX_OP_ANDN: 68 - unsafe_atomic_op2("andl %4, %3", oval, uaddr, ~oparg, Efault); 69 - break; 70 - case FUTEX_OP_XOR: 71 - unsafe_atomic_op2("xorl %4, %3", oval, uaddr, oparg, Efault); 72 - break; 73 - default: 74 - user_access_end(); 75 - return -ENOSYS; 51 + scoped_user_rw_access(uaddr, Efault) { 52 + switch (op) { 53 + case FUTEX_OP_SET: 54 + unsafe_atomic_op1("xchgl %0, %2", oval, uaddr, oparg, Efault); 55 + break; 56 + case FUTEX_OP_ADD: 57 + unsafe_atomic_op1(LOCK_PREFIX "xaddl %0, %2", oval, uaddr, oparg, Efault); 58 + break; 59 + case FUTEX_OP_OR: 60 + unsafe_atomic_op2("orl %4, %3", oval, uaddr, oparg, Efault); 61 + break; 62 + case FUTEX_OP_ANDN: 63 + unsafe_atomic_op2("andl %4, %3", oval, uaddr, ~oparg, Efault); 64 + break; 65 + case FUTEX_OP_XOR: 66 + unsafe_atomic_op2("xorl %4, %3", oval, uaddr, oparg, Efault); 67 + break; 68 + default: 69 + return -ENOSYS; 70 + } 76 71 } 77 - user_access_end(); 78 72 return 0; 79 73 Efault: 80 - user_access_end(); 81 74 return -EFAULT; 82 75 } 83 76 ··· 79 86 { 80 87 int ret = 0; 81 88 82 - if (can_do_masked_user_access()) 83 - uaddr = masked_user_access_begin(uaddr); 84 - else if (!user_access_begin(uaddr, sizeof(u32))) 85 - return -EFAULT; 86 - asm volatile("\n" 87 - "1:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" 88 - "2:\n" 89 - _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %0) \ 90 - : "+r" (ret), "=a" (oldval), "+m" (*uaddr) 91 - : "r" (newval), "1" (oldval) 92 - : "memory" 93 - ); 94 - user_access_end(); 95 - *uval = oldval; 89 + scoped_user_rw_access(uaddr, Efault) { 90 + asm_inline volatile("\n" 91 + "1:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" 92 + "2:\n" 93 + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %0) 94 + : "+r" (ret), "=a" (oldval), "+m" (*uaddr) 95 + : "r" (newval), "1" (oldval) 96 + : "memory"); 97 + *uval = oldval; 98 + } 96 99 return ret; 100 + Efault: 101 + return -EFAULT; 97 102 } 98 103 99 104 #endif
+6 -6
arch/x86/include/asm/uaccess.h
··· 528 528 #define user_access_save() smap_save() 529 529 #define user_access_restore(x) smap_restore(x) 530 530 531 - #define unsafe_put_user(x, ptr, label) \ 531 + #define arch_unsafe_put_user(x, ptr, label) \ 532 532 __put_user_size((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), label) 533 533 534 534 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 535 - #define unsafe_get_user(x, ptr, err_label) \ 535 + #define arch_unsafe_get_user(x, ptr, err_label) \ 536 536 do { \ 537 537 __inttype(*(ptr)) __gu_val; \ 538 538 __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), err_label); \ 539 539 (x) = (__force __typeof__(*(ptr)))__gu_val; \ 540 540 } while (0) 541 541 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT 542 - #define unsafe_get_user(x, ptr, err_label) \ 542 + #define arch_unsafe_get_user(x, ptr, err_label) \ 543 543 do { \ 544 544 int __gu_err; \ 545 545 __inttype(*(ptr)) __gu_val; \ ··· 618 618 } while (0) 619 619 620 620 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 621 - #define __get_kernel_nofault(dst, src, type, err_label) \ 621 + #define arch_get_kernel_nofault(dst, src, type, err_label) \ 622 622 __get_user_size(*((type *)(dst)), (__force type __user *)(src), \ 623 623 sizeof(type), err_label) 624 624 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT 625 - #define __get_kernel_nofault(dst, src, type, err_label) \ 625 + #define arch_get_kernel_nofault(dst, src, type, err_label) \ 626 626 do { \ 627 627 int __kr_err; \ 628 628 \ ··· 633 633 } while (0) 634 634 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT 635 635 636 - #define __put_kernel_nofault(dst, src, type, err_label) \ 636 + #define arch_put_kernel_nofault(dst, src, type, err_label) \ 637 637 __put_user_size(*((type *)(src)), (__force type __user *)(dst), \ 638 638 sizeof(type), err_label) 639 639
+4 -8
fs/select.c
··· 776 776 { 777 777 // the path is hot enough for overhead of copy_from_user() to matter 778 778 if (from) { 779 - if (can_do_masked_user_access()) 780 - from = masked_user_access_begin(from); 781 - else if (!user_read_access_begin(from, sizeof(*from))) 782 - return -EFAULT; 783 - unsafe_get_user(to->p, &from->p, Efault); 784 - unsafe_get_user(to->size, &from->size, Efault); 785 - user_read_access_end(); 779 + scoped_user_read_access(from, Efault) { 780 + unsafe_get_user(to->p, &from->p, Efault); 781 + unsafe_get_user(to->size, &from->size, Efault); 782 + } 786 783 } 787 784 return 0; 788 785 Efault: 789 - user_read_access_end(); 790 786 return -EFAULT; 791 787 } 792 788
+310 -4
include/linux/uaccess.h
··· 2 2 #ifndef __LINUX_UACCESS_H__ 3 3 #define __LINUX_UACCESS_H__ 4 4 5 + #include <linux/cleanup.h> 5 6 #include <linux/fault-inject-usercopy.h> 6 7 #include <linux/instrumented.h> 7 8 #include <linux/minmax.h> ··· 36 35 37 36 #ifdef masked_user_access_begin 38 37 #define can_do_masked_user_access() 1 38 + # ifndef masked_user_write_access_begin 39 + # define masked_user_write_access_begin masked_user_access_begin 40 + # endif 41 + # ifndef masked_user_read_access_begin 42 + # define masked_user_read_access_begin masked_user_access_begin 43 + #endif 39 44 #else 40 45 #define can_do_masked_user_access() 0 41 46 #define masked_user_access_begin(src) NULL 47 + #define masked_user_read_access_begin(src) NULL 48 + #define masked_user_write_access_begin(src) NULL 42 49 #define mask_user_address(src) (src) 43 50 #endif 44 51 ··· 527 518 long count); 528 519 long strnlen_user_nofault(const void __user *unsafe_addr, long count); 529 520 530 - #ifndef __get_kernel_nofault 521 + #ifdef arch_get_kernel_nofault 522 + /* 523 + * Wrap the architecture implementation so that @label can be outside of a 524 + * cleanup() scope. A regular C goto works correctly, but ASM goto does 525 + * not. Clang rejects such an attempt, but GCC silently emits buggy code. 526 + */ 527 + #define __get_kernel_nofault(dst, src, type, label) \ 528 + do { \ 529 + __label__ local_label; \ 530 + arch_get_kernel_nofault(dst, src, type, local_label); \ 531 + if (0) { \ 532 + local_label: \ 533 + goto label; \ 534 + } \ 535 + } while (0) 536 + 537 + #define __put_kernel_nofault(dst, src, type, label) \ 538 + do { \ 539 + __label__ local_label; \ 540 + arch_put_kernel_nofault(dst, src, type, local_label); \ 541 + if (0) { \ 542 + local_label: \ 543 + goto label; \ 544 + } \ 545 + } while (0) 546 + 547 + #elif !defined(__get_kernel_nofault) /* arch_get_kernel_nofault */ 548 + 531 549 #define __get_kernel_nofault(dst, src, type, label) \ 532 550 do { \ 533 551 type __user *p = (type __force __user *)(src); \ ··· 571 535 if (__put_user(data, p)) \ 572 536 goto label; \ 573 537 } while (0) 574 - #endif 538 + 539 + #endif /* !__get_kernel_nofault */ 575 540 576 541 /** 577 542 * get_kernel_nofault(): safely attempt to read from a location ··· 586 549 copy_from_kernel_nofault(&(val), __gk_ptr, sizeof(val));\ 587 550 }) 588 551 589 - #ifndef user_access_begin 552 + #ifdef user_access_begin 553 + 554 + #ifdef arch_unsafe_get_user 555 + /* 556 + * Wrap the architecture implementation so that @label can be outside of a 557 + * cleanup() scope. A regular C goto works correctly, but ASM goto does 558 + * not. Clang rejects such an attempt, but GCC silently emits buggy code. 559 + * 560 + * Some architectures use internal local labels already, but this extra 561 + * indirection here is harmless because the compiler optimizes it out 562 + * completely in any case. This construct just ensures that the ASM GOTO 563 + * target is always in the local scope. The C goto 'label' works correctly 564 + * when leaving a cleanup() scope. 565 + */ 566 + #define unsafe_get_user(x, ptr, label) \ 567 + do { \ 568 + __label__ local_label; \ 569 + arch_unsafe_get_user(x, ptr, local_label); \ 570 + if (0) { \ 571 + local_label: \ 572 + goto label; \ 573 + } \ 574 + } while (0) 575 + 576 + #define unsafe_put_user(x, ptr, label) \ 577 + do { \ 578 + __label__ local_label; \ 579 + arch_unsafe_put_user(x, ptr, local_label); \ 580 + if (0) { \ 581 + local_label: \ 582 + goto label; \ 583 + } \ 584 + } while (0) 585 + #endif /* arch_unsafe_get_user */ 586 + 587 + #else /* user_access_begin */ 590 588 #define user_access_begin(ptr,len) access_ok(ptr, len) 591 589 #define user_access_end() do { } while (0) 592 590 #define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0) ··· 631 559 #define unsafe_copy_from_user(d,s,l,e) unsafe_op_wrap(__copy_from_user(d,s,l),e) 632 560 static inline unsigned long user_access_save(void) { return 0UL; } 633 561 static inline void user_access_restore(unsigned long flags) { } 634 - #endif 562 + #endif /* !user_access_begin */ 563 + 635 564 #ifndef user_write_access_begin 636 565 #define user_write_access_begin user_access_begin 637 566 #define user_write_access_end user_access_end ··· 641 568 #define user_read_access_begin user_access_begin 642 569 #define user_read_access_end user_access_end 643 570 #endif 571 + 572 + /* Define RW variant so the below _mode macro expansion works */ 573 + #define masked_user_rw_access_begin(u) masked_user_access_begin(u) 574 + #define user_rw_access_begin(u, s) user_access_begin(u, s) 575 + #define user_rw_access_end() user_access_end() 576 + 577 + /* Scoped user access */ 578 + #define USER_ACCESS_GUARD(_mode) \ 579 + static __always_inline void __user * \ 580 + class_user_##_mode##_begin(void __user *ptr) \ 581 + { \ 582 + return ptr; \ 583 + } \ 584 + \ 585 + static __always_inline void \ 586 + class_user_##_mode##_end(void __user *ptr) \ 587 + { \ 588 + user_##_mode##_access_end(); \ 589 + } \ 590 + \ 591 + DEFINE_CLASS(user_ ##_mode## _access, void __user *, \ 592 + class_user_##_mode##_end(_T), \ 593 + class_user_##_mode##_begin(ptr), void __user *ptr) \ 594 + \ 595 + static __always_inline class_user_##_mode##_access_t \ 596 + class_user_##_mode##_access_ptr(void __user *scope) \ 597 + { \ 598 + return scope; \ 599 + } 600 + 601 + USER_ACCESS_GUARD(read) 602 + USER_ACCESS_GUARD(write) 603 + USER_ACCESS_GUARD(rw) 604 + #undef USER_ACCESS_GUARD 605 + 606 + /** 607 + * __scoped_user_access_begin - Start a scoped user access 608 + * @mode: The mode of the access class (read, write, rw) 609 + * @uptr: The pointer to access user space memory 610 + * @size: Size of the access 611 + * @elbl: Error label to goto when the access region is rejected 612 + * 613 + * Internal helper for __scoped_user_access(). Don't use directly. 614 + */ 615 + #define __scoped_user_access_begin(mode, uptr, size, elbl) \ 616 + ({ \ 617 + typeof(uptr) __retptr; \ 618 + \ 619 + if (can_do_masked_user_access()) { \ 620 + __retptr = masked_user_##mode##_access_begin(uptr); \ 621 + } else { \ 622 + __retptr = uptr; \ 623 + if (!user_##mode##_access_begin(uptr, size)) \ 624 + goto elbl; \ 625 + } \ 626 + __retptr; \ 627 + }) 628 + 629 + /** 630 + * __scoped_user_access - Open a scope for user access 631 + * @mode: The mode of the access class (read, write, rw) 632 + * @uptr: The pointer to access user space memory 633 + * @size: Size of the access 634 + * @elbl: Error label to goto when the access region is rejected. It 635 + * must be placed outside the scope 636 + * 637 + * If the user access function inside the scope requires a fault label, it 638 + * can use @elbl or a different label outside the scope, which requires 639 + * that user access which is implemented with ASM GOTO has been properly 640 + * wrapped. See unsafe_get_user() for reference. 641 + * 642 + * scoped_user_rw_access(ptr, efault) { 643 + * unsafe_get_user(rval, &ptr->rval, efault); 644 + * unsafe_put_user(wval, &ptr->wval, efault); 645 + * } 646 + * return 0; 647 + * efault: 648 + * return -EFAULT; 649 + * 650 + * The scope is internally implemented as a autoterminating nested for() 651 + * loop, which can be left with 'return', 'break' and 'goto' at any 652 + * point. 653 + * 654 + * When the scope is left user_##@_mode##_access_end() is automatically 655 + * invoked. 656 + * 657 + * When the architecture supports masked user access and the access region 658 + * which is determined by @uptr and @size is not a valid user space 659 + * address, i.e. < TASK_SIZE, the scope sets the pointer to a faulting user 660 + * space address and does not terminate early. This optimizes for the good 661 + * case and lets the performance uncritical bad case go through the fault. 662 + * 663 + * The eventual modification of the pointer is limited to the scope. 664 + * Outside of the scope the original pointer value is unmodified, so that 665 + * the original pointer value is available for diagnostic purposes in an 666 + * out of scope fault path. 667 + * 668 + * Nesting scoped user access into a user access scope is invalid and fails 669 + * the build. Nesting into other guards, e.g. pagefault is safe. 670 + * 671 + * The masked variant does not check the size of the access and relies on a 672 + * mapping hole (e.g. guard page) to catch an out of range pointer, the 673 + * first access to user memory inside the scope has to be within 674 + * @uptr ... @uptr + PAGE_SIZE - 1 675 + * 676 + * Don't use directly. Use scoped_masked_user_$MODE_access() instead. 677 + */ 678 + #define __scoped_user_access(mode, uptr, size, elbl) \ 679 + for (bool done = false; !done; done = true) \ 680 + for (void __user *_tmpptr = __scoped_user_access_begin(mode, uptr, size, elbl); \ 681 + !done; done = true) \ 682 + for (CLASS(user_##mode##_access, scope)(_tmpptr); !done; done = true) \ 683 + /* Force modified pointer usage within the scope */ \ 684 + for (const typeof(uptr) uptr = _tmpptr; !done; done = true) 685 + 686 + /** 687 + * scoped_user_read_access_size - Start a scoped user read access with given size 688 + * @usrc: Pointer to the user space address to read from 689 + * @size: Size of the access starting from @usrc 690 + * @elbl: Error label to goto when the access region is rejected 691 + * 692 + * For further information see __scoped_user_access() above. 693 + */ 694 + #define scoped_user_read_access_size(usrc, size, elbl) \ 695 + __scoped_user_access(read, usrc, size, elbl) 696 + 697 + /** 698 + * scoped_user_read_access - Start a scoped user read access 699 + * @usrc: Pointer to the user space address to read from 700 + * @elbl: Error label to goto when the access region is rejected 701 + * 702 + * The size of the access starting from @usrc is determined via sizeof(*@usrc)). 703 + * 704 + * For further information see __scoped_user_access() above. 705 + */ 706 + #define scoped_user_read_access(usrc, elbl) \ 707 + scoped_user_read_access_size(usrc, sizeof(*(usrc)), elbl) 708 + 709 + /** 710 + * scoped_user_write_access_size - Start a scoped user write access with given size 711 + * @udst: Pointer to the user space address to write to 712 + * @size: Size of the access starting from @udst 713 + * @elbl: Error label to goto when the access region is rejected 714 + * 715 + * For further information see __scoped_user_access() above. 716 + */ 717 + #define scoped_user_write_access_size(udst, size, elbl) \ 718 + __scoped_user_access(write, udst, size, elbl) 719 + 720 + /** 721 + * scoped_user_write_access - Start a scoped user write access 722 + * @udst: Pointer to the user space address to write to 723 + * @elbl: Error label to goto when the access region is rejected 724 + * 725 + * The size of the access starting from @udst is determined via sizeof(*@udst)). 726 + * 727 + * For further information see __scoped_user_access() above. 728 + */ 729 + #define scoped_user_write_access(udst, elbl) \ 730 + scoped_user_write_access_size(udst, sizeof(*(udst)), elbl) 731 + 732 + /** 733 + * scoped_user_rw_access_size - Start a scoped user read/write access with given size 734 + * @uptr Pointer to the user space address to read from and write to 735 + * @size: Size of the access starting from @uptr 736 + * @elbl: Error label to goto when the access region is rejected 737 + * 738 + * For further information see __scoped_user_access() above. 739 + */ 740 + #define scoped_user_rw_access_size(uptr, size, elbl) \ 741 + __scoped_user_access(rw, uptr, size, elbl) 742 + 743 + /** 744 + * scoped_user_rw_access - Start a scoped user read/write access 745 + * @uptr Pointer to the user space address to read from and write to 746 + * @elbl: Error label to goto when the access region is rejected 747 + * 748 + * The size of the access starting from @uptr is determined via sizeof(*@uptr)). 749 + * 750 + * For further information see __scoped_user_access() above. 751 + */ 752 + #define scoped_user_rw_access(uptr, elbl) \ 753 + scoped_user_rw_access_size(uptr, sizeof(*(uptr)), elbl) 754 + 755 + /** 756 + * get_user_inline - Read user data inlined 757 + * @val: The variable to store the value read from user memory 758 + * @usrc: Pointer to the user space memory to read from 759 + * 760 + * Return: 0 if successful, -EFAULT when faulted 761 + * 762 + * Inlined variant of get_user(). Only use when there is a demonstrable 763 + * performance reason. 764 + */ 765 + #define get_user_inline(val, usrc) \ 766 + ({ \ 767 + __label__ efault; \ 768 + typeof(usrc) _tmpsrc = usrc; \ 769 + int _ret = 0; \ 770 + \ 771 + scoped_user_read_access(_tmpsrc, efault) \ 772 + unsafe_get_user(val, _tmpsrc, efault); \ 773 + if (0) { \ 774 + efault: \ 775 + _ret = -EFAULT; \ 776 + } \ 777 + _ret; \ 778 + }) 779 + 780 + /** 781 + * put_user_inline - Write to user memory inlined 782 + * @val: The value to write 783 + * @udst: Pointer to the user space memory to write to 784 + * 785 + * Return: 0 if successful, -EFAULT when faulted 786 + * 787 + * Inlined variant of put_user(). Only use when there is a demonstrable 788 + * performance reason. 789 + */ 790 + #define put_user_inline(val, udst) \ 791 + ({ \ 792 + __label__ efault; \ 793 + typeof(udst) _tmpdst = udst; \ 794 + int _ret = 0; \ 795 + \ 796 + scoped_user_write_access(_tmpdst, efault) \ 797 + unsafe_put_user(val, _tmpdst, efault); \ 798 + if (0) { \ 799 + efault: \ 800 + _ret = -EFAULT; \ 801 + } \ 802 + _ret; \ 803 + }) 644 804 645 805 #ifdef CONFIG_HARDENED_USERCOPY 646 806 void __noreturn usercopy_abort(const char *name, const char *detail,
+2 -2
kernel/futex/core.c
··· 581 581 if (flags & FLAGS_NUMA) { 582 582 u32 __user *naddr = (void *)uaddr + size / 2; 583 583 584 - if (futex_get_value(&node, naddr)) 584 + if (get_user_inline(node, naddr)) 585 585 return -EFAULT; 586 586 587 587 if ((node != FUTEX_NO_NODE) && ··· 601 601 node = numa_node_id(); 602 602 node_updated = true; 603 603 } 604 - if (node_updated && futex_put_value(node, naddr)) 604 + if (node_updated && put_user_inline(node, naddr)) 605 605 return -EFAULT; 606 606 } 607 607
+3 -55
kernel/futex/futex.h
··· 281 281 return ret; 282 282 } 283 283 284 - /* 285 - * This does a plain atomic user space read, and the user pointer has 286 - * already been verified earlier by get_futex_key() to be both aligned 287 - * and actually in user space, just like futex_atomic_cmpxchg_inatomic(). 288 - * 289 - * We still want to avoid any speculation, and while __get_user() is 290 - * the traditional model for this, it's actually slower than doing 291 - * this manually these days. 292 - * 293 - * We could just have a per-architecture special function for it, 294 - * the same way we do futex_atomic_cmpxchg_inatomic(), but rather 295 - * than force everybody to do that, write it out long-hand using 296 - * the low-level user-access infrastructure. 297 - * 298 - * This looks a bit overkill, but generally just results in a couple 299 - * of instructions. 300 - */ 301 - static __always_inline int futex_get_value(u32 *dest, u32 __user *from) 302 - { 303 - u32 val; 304 - 305 - if (can_do_masked_user_access()) 306 - from = masked_user_access_begin(from); 307 - else if (!user_read_access_begin(from, sizeof(*from))) 308 - return -EFAULT; 309 - unsafe_get_user(val, from, Efault); 310 - user_read_access_end(); 311 - *dest = val; 312 - return 0; 313 - Efault: 314 - user_read_access_end(); 315 - return -EFAULT; 316 - } 317 - 318 - static __always_inline int futex_put_value(u32 val, u32 __user *to) 319 - { 320 - if (can_do_masked_user_access()) 321 - to = masked_user_access_begin(to); 322 - else if (!user_write_access_begin(to, sizeof(*to))) 323 - return -EFAULT; 324 - unsafe_put_user(val, to, Efault); 325 - user_write_access_end(); 326 - return 0; 327 - Efault: 328 - user_write_access_end(); 329 - return -EFAULT; 330 - } 331 - 284 + /* Read from user memory with pagefaults disabled */ 332 285 static inline int futex_get_value_locked(u32 *dest, u32 __user *from) 333 286 { 334 - int ret; 335 - 336 - pagefault_disable(); 337 - ret = futex_get_value(dest, from); 338 - pagefault_enable(); 339 - 340 - return ret; 287 + guard(pagefault)(); 288 + return get_user_inline(*dest, from); 341 289 } 342 290 343 291 extern void __futex_unqueue(struct futex_q *q);
+17 -5
lib/iov_iter.c
··· 49 49 50 50 if (should_fail_usercopy()) 51 51 return len; 52 - if (access_ok(iter_from, len)) { 53 - to += progress; 54 - instrument_copy_from_user_before(to, iter_from, len); 55 - res = raw_copy_from_user(to, iter_from, len); 56 - instrument_copy_from_user_after(to, iter_from, len, res); 52 + if (can_do_masked_user_access()) { 53 + iter_from = mask_user_address(iter_from); 54 + } else { 55 + if (!access_ok(iter_from, len)) 56 + return res; 57 + 58 + /* 59 + * Ensure that bad access_ok() speculation will not 60 + * lead to nasty side effects *after* the copy is 61 + * finished: 62 + */ 63 + barrier_nospec(); 57 64 } 65 + to += progress; 66 + instrument_copy_from_user_before(to, iter_from, len); 67 + res = raw_copy_from_user(to, iter_from, len); 68 + instrument_copy_from_user_after(to, iter_from, len, res); 69 + 58 70 return res; 59 71 } 60 72
+1 -1
lib/strncpy_from_user.c
··· 126 126 if (can_do_masked_user_access()) { 127 127 long retval; 128 128 129 - src = masked_user_access_begin(src); 129 + src = masked_user_read_access_begin(src); 130 130 retval = do_strncpy_from_user(dst, src, count, count); 131 131 user_read_access_end(); 132 132 return retval;
+1 -1
lib/strnlen_user.c
··· 99 99 if (can_do_masked_user_access()) { 100 100 long retval; 101 101 102 - str = masked_user_access_begin(str); 102 + str = masked_user_read_access_begin(str); 103 103 retval = do_strnlen_user(str, count, count); 104 104 user_read_access_end(); 105 105 return retval;
+7 -13
net/core/scm.c
··· 273 273 274 274 check_object_size(data, cmlen - sizeof(*cm), true); 275 275 276 - if (can_do_masked_user_access()) 277 - cm = masked_user_access_begin(cm); 278 - else if (!user_write_access_begin(cm, cmlen)) 279 - goto efault; 280 - 281 - unsafe_put_user(cmlen, &cm->cmsg_len, efault_end); 282 - unsafe_put_user(level, &cm->cmsg_level, efault_end); 283 - unsafe_put_user(type, &cm->cmsg_type, efault_end); 284 - unsafe_copy_to_user(CMSG_USER_DATA(cm), data, 285 - cmlen - sizeof(*cm), efault_end); 286 - user_write_access_end(); 276 + scoped_user_write_access_size(cm, cmlen, efault) { 277 + unsafe_put_user(cmlen, &cm->cmsg_len, efault); 278 + unsafe_put_user(level, &cm->cmsg_level, efault); 279 + unsafe_put_user(type, &cm->cmsg_type, efault); 280 + unsafe_copy_to_user(CMSG_USER_DATA(cm), data, 281 + cmlen - sizeof(*cm), efault); 282 + } 287 283 } else { 288 284 struct cmsghdr *cm = msg->msg_control; 289 285 ··· 297 301 msg->msg_controllen -= cmlen; 298 302 return 0; 299 303 300 - efault_end: 301 - user_write_access_end(); 302 304 efault: 303 305 return -EFAULT; 304 306 }