tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge branch 'nocache-cleanup'

This series cleans up some of the special user copy functions naming and
semantics. In particular, get rid of the (very traditional) double
underscore names and behavior: the whole "optimize away the range check"
model has been largely excised from the other user accessors because
it's so subtle and can be unsafe, but also because it's just not a
relevant optimization any more.

To do that, a couple of drivers that misused the "user" copies as kernel
copies in order to get non-temporal stores had to be fixed up, but that
kind of code should never have been allowed anyway.

The x86-only "nocache" version was also renamed to more accurately
reflect what it actually does.

This was all done because I looked at this code due to a report by Jann
Horn, and I just couldn't stand the inconsistent naming, the horrible
semantics, and the random misuse of these functions. This code should
probably be cleaned up further, but it's at least slightly closer to
normal semantics.

I had a more intrusive series that went even further in trying to
normalize the semantics, but that ended up hitting so many other
inconsistencies between different architectures in this area (eg
'size_t' vs 'unsigned long' vs 'int' as size arguments, and various
iovec check differences that Vasily Gorbik pointed out) that I ended up
with this more limited version that fixed the worst of the issues.

Reported-by: Jann Horn <jannh@google.com>
Tested-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/all/CAHk-=wgg1QVWNWG-UCFo1hx0zqrPnB3qhPzUTrWNft+MtXQXig@mail.gmail.com/

* nocache-cleanup:
x86-64/arm64/powerpc: clean up and rename __copy_from_user_flushcache
x86: rename and clean up __copy_from_user_inatomic_nocache()
x86-64: rename misleadingly named '__copy_user_nocache()' function

Linus Torvalds fdcbb1bc 028ef9c9

+53 -52
+1 -1
arch/arm64/include/asm/uaccess.h
··· 478 478 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE 479 479 extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n); 480 480 481 - static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) 481 + static inline size_t copy_from_user_flushcache(void *dst, const void __user *src, size_t size) 482 482 { 483 483 kasan_check_write(dst, size); 484 484 return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size);
+1 -2
arch/powerpc/include/asm/uaccess.h
··· 434 434 } 435 435 #endif 436 436 437 - extern long __copy_from_user_flushcache(void *dst, const void __user *src, 438 - unsigned size); 437 + extern size_t copy_from_user_flushcache(void *dst, const void __user *src, size_t size); 439 438 440 439 static __must_check __always_inline bool __user_access_begin(const void __user *ptr, size_t len, 441 440 unsigned long dir)
+6 -5
arch/powerpc/lib/pmem.c
··· 66 66 /* 67 67 * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols 68 68 */ 69 - long __copy_from_user_flushcache(void *dest, const void __user *src, 70 - unsigned size) 69 + size_t copy_from_user_flushcache(void *dest, const void __user *src, 70 + size_t size) 71 71 { 72 - unsigned long copied, start = (unsigned long) dest; 72 + unsigned long not_copied, start = (unsigned long) dest; 73 73 74 - copied = __copy_from_user(dest, src, size); 74 + src = mask_user_address(src); 75 + not_copied = __copy_from_user(dest, src, size); 75 76 clean_pmem_range(start, start + size); 76 77 77 - return copied; 78 + return not_copied; 78 79 } 79 80 80 81 void memcpy_flushcache(void *dest, const void *src, size_t size)
+1 -1
arch/x86/include/asm/uaccess.h
··· 507 507 } ____cacheline_aligned_in_smp movsl_mask; 508 508 #endif 509 509 510 - #define ARCH_HAS_NOCACHE_UACCESS 1 510 + #define ARCH_HAS_NONTEMPORAL_UACCESS 1 511 511 512 512 /* 513 513 * The "unsafe" user accesses aren't really "unsafe", but the naming
+1 -7
arch/x86/include/asm/uaccess_32.h
··· 26 26 return __copy_user_ll(to, (__force const void *)from, n); 27 27 } 28 28 29 - static __always_inline unsigned long 30 - __copy_from_user_inatomic_nocache(void *to, const void __user *from, 31 - unsigned long n) 32 - { 33 - return __copy_from_user_ll_nocache_nozero(to, from, n); 34 - } 35 - 29 + unsigned long __must_check copy_from_user_inatomic_nontemporal(void *, const void __user *, unsigned long n); 36 30 unsigned long __must_check clear_user(void __user *mem, unsigned long len); 37 31 unsigned long __must_check __clear_user(void __user *mem, unsigned long len); 38 32
+9 -7
arch/x86/include/asm/uaccess_64.h
··· 147 147 return copy_user_generic((__force void *)dst, src, size); 148 148 } 149 149 150 - extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size); 151 - extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size); 150 + #define copy_to_nontemporal copy_to_nontemporal 151 + extern size_t copy_to_nontemporal(void *dst, const void *src, size_t size); 152 + extern size_t copy_user_flushcache(void *dst, const void __user *src, size_t size); 152 153 153 154 static inline int 154 - __copy_from_user_inatomic_nocache(void *dst, const void __user *src, 155 + copy_from_user_inatomic_nontemporal(void *dst, const void __user *src, 155 156 unsigned size) 156 157 { 157 158 long ret; 158 159 kasan_check_write(dst, size); 160 + src = mask_user_address(src); 159 161 stac(); 160 - ret = __copy_user_nocache(dst, src, size); 162 + ret = copy_to_nontemporal(dst, (__force const void *)src, size); 161 163 clac(); 162 164 return ret; 163 165 } 164 166 165 - static inline int 166 - __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) 167 + static inline size_t 168 + copy_from_user_flushcache(void *dst, const void __user *src, size_t size) 167 169 { 168 170 kasan_check_write(dst, size); 169 - return __copy_user_flushcache(dst, src, size); 171 + return copy_user_flushcache(dst, src, size); 170 172 } 171 173 172 174 /*
+3 -3
arch/x86/lib/copy_user_uncached_64.S
··· 27 27 * Output: 28 28 * rax uncopied bytes or 0 if successful. 29 29 */ 30 - SYM_FUNC_START(__copy_user_nocache) 30 + SYM_FUNC_START(copy_to_nontemporal) 31 31 ANNOTATE_NOENDBR 32 32 /* If destination is not 7-byte aligned, we'll have to align it */ 33 33 testb $7,%dil ··· 240 240 _ASM_EXTABLE_UA(52b, .Ldone0) 241 241 _ASM_EXTABLE_UA(53b, .Ldone0) 242 242 243 - SYM_FUNC_END(__copy_user_nocache) 244 - EXPORT_SYMBOL(__copy_user_nocache) 243 + SYM_FUNC_END(copy_to_nontemporal) 244 + EXPORT_SYMBOL(copy_to_nontemporal)
+5 -4
arch/x86/lib/usercopy_32.c
··· 322 322 } 323 323 EXPORT_SYMBOL(__copy_user_ll); 324 324 325 - unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, 325 + unsigned long copy_from_user_inatomic_nontemporal(void *to, const void __user *from, 326 326 unsigned long n) 327 327 { 328 - __uaccess_begin_nospec(); 328 + if (!user_access_begin(from, n)) 329 + return n; 329 330 #ifdef CONFIG_X86_INTEL_USERCOPY 330 331 if (n > 64 && static_cpu_has(X86_FEATURE_XMM2)) 331 332 n = __copy_user_intel_nocache(to, from, n); ··· 335 334 #else 336 335 __copy_user(to, from, n); 337 336 #endif 338 - __uaccess_end(); 337 + user_access_end(); 339 338 return n; 340 339 } 341 - EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero); 340 + EXPORT_SYMBOL(copy_from_user_inatomic_nontemporal);
+6 -6
arch/x86/lib/usercopy_64.c
··· 43 43 } 44 44 EXPORT_SYMBOL_GPL(arch_wb_cache_pmem); 45 45 46 - long __copy_user_flushcache(void *dst, const void __user *src, unsigned size) 46 + size_t copy_user_flushcache(void *dst, const void __user *src, size_t size) 47 47 { 48 48 unsigned long flushed, dest = (unsigned long) dst; 49 - long rc; 49 + unsigned long rc; 50 50 51 - stac(); 52 - rc = __copy_user_nocache(dst, src, size); 53 - clac(); 51 + src = masked_user_access_begin(src); 52 + rc = copy_to_nontemporal(dst, (__force const void *)src, size); 53 + user_access_end(); 54 54 55 55 /* 56 - * __copy_user_nocache() uses non-temporal stores for the bulk 56 + * copy_to_nontemporal() uses non-temporal stores for the bulk 57 57 * of the transfer, but we need to manually flush if the 58 58 * transfer is unaligned. A cached memory copy is used when 59 59 * destination or size is not naturally aligned. That is:
+1 -1
drivers/gpu/drm/i915/i915_gem.c
··· 520 520 521 521 /* We can use the cpu mem copy function because this is X86. */ 522 522 vaddr = io_mapping_map_atomic_wc(mapping, base); 523 - unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset, 523 + unwritten = copy_from_user_inatomic_nontemporal((void __force *)vaddr + offset, 524 524 user_data, length); 525 525 io_mapping_unmap_atomic(vaddr); 526 526 if (unwritten) {
+1 -1
drivers/gpu/drm/qxl/qxl_ioctl.c
··· 183 183 184 184 /* TODO copy slow path code from i915 */ 185 185 fb_cmd = qxl_bo_kmap_atomic_page(qdev, cmd_bo, (release->release_offset & PAGE_MASK)); 186 - unwritten = __copy_from_user_inatomic_nocache 186 + unwritten = copy_from_user_inatomic_nontemporal 187 187 (fb_cmd + sizeof(union qxl_release_info) + (release->release_offset & ~PAGE_MASK), 188 188 u64_to_user_ptr(cmd->command), cmd->command_size); 189 189
+3 -5
drivers/infiniband/sw/rdmavt/qp.c
··· 92 92 static void cacheless_memcpy(void *dst, void *src, size_t n) 93 93 { 94 94 /* 95 - * Use the only available X64 cacheless copy. Add a __user cast 96 - * to quiet sparse. The src agument is already in the kernel so 97 - * there are no security issues. The extra fault recovery machinery 98 - * is not invoked. 95 + * Use the only available X64 cacheless copy. 96 + * The extra fault recovery machinery is not invoked. 99 97 */ 100 - __copy_user_nocache(dst, (void __user *)src, n); 98 + copy_to_nontemporal(dst, src, n); 101 99 } 102 100 103 101 void rvt_wss_exit(struct rvt_dev_info *rdi)
+4 -3
drivers/ntb/ntb_transport.c
··· 1779 1779 1780 1780 static void ntb_memcpy_tx_on_stack(struct ntb_queue_entry *entry, void __iomem *offset) 1781 1781 { 1782 - #ifdef ARCH_HAS_NOCACHE_UACCESS 1782 + #ifdef copy_to_nontemporal 1783 1783 /* 1784 1784 * Using non-temporal mov to improve performance on non-cached 1785 - * writes, even though we aren't actually copying from user space. 1785 + * writes. This only works if __iomem is strictly memory-like, 1786 + * but that is the case on x86-64 1786 1787 */ 1787 - __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); 1788 + copy_to_nontemporal(offset, entry->buf, entry->len); 1788 1789 #else 1789 1790 memcpy_toio(offset, entry->buf, entry->len); 1790 1791 #endif
+8 -3
include/linux/uaccess.h
··· 331 331 332 332 #endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */ 333 333 334 - #ifndef ARCH_HAS_NOCACHE_UACCESS 334 + #ifndef ARCH_HAS_NONTEMPORAL_UACCESS 335 335 336 336 static inline __must_check unsigned long 337 - __copy_from_user_inatomic_nocache(void *to, const void __user *from, 337 + copy_from_user_inatomic_nontemporal(void *to, const void __user *from, 338 338 unsigned long n) 339 339 { 340 + if (can_do_masked_user_access()) 341 + from = mask_user_address(from); 342 + else 343 + if (!access_ok(from, n)) 344 + return n; 340 345 return __copy_from_user_inatomic(to, from, n); 341 346 } 342 347 343 - #endif /* ARCH_HAS_NOCACHE_UACCESS */ 348 + #endif /* ARCH_HAS_NONTEMPORAL_UACCESS */ 344 349 345 350 extern __must_check int check_zeroed_user(const void __user *from, size_t size); 346 351
+2 -2
lib/iov_iter.c
··· 277 277 size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress, 278 278 size_t len, void *to, void *priv2) 279 279 { 280 - return __copy_from_user_inatomic_nocache(to + progress, iter_from, len); 280 + return copy_from_user_inatomic_nontemporal(to + progress, iter_from, len); 281 281 } 282 282 283 283 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) ··· 296 296 size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress, 297 297 size_t len, void *to, void *priv2) 298 298 { 299 - return __copy_from_user_flushcache(to + progress, iter_from, len); 299 + return copy_from_user_flushcache(to + progress, iter_from, len); 300 300 } 301 301 302 302 static __always_inline
+1 -1
tools/objtool/check.c
··· 1301 1301 "copy_mc_enhanced_fast_string", 1302 1302 "rep_stos_alternative", 1303 1303 "rep_movs_alternative", 1304 - "__copy_user_nocache", 1304 + "copy_to_nontemporal", 1305 1305 NULL 1306 1306 }; 1307 1307