tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge tag 'x86_core_for_v5.17_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 core updates from Borislav Petkov:

- Get rid of all the .fixup sections because this generates
misleading/wrong stacktraces and confuse RELIABLE_STACKTRACE and
LIVEPATCH as the backtrace misses the function which is being fixed
up.

- Add Straight Line Speculation mitigation support which uses a new
compiler switch -mharden-sls= which sticks an INT3 after a RET or an
indirect branch in order to block speculation after them. Reportedly,
CPUs do speculate behind such insns.

- The usual set of cleanups and improvements

* tag 'x86_core_for_v5.17_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (32 commits)
x86/entry_32: Fix segment exceptions
objtool: Remove .fixup handling
x86: Remove .fixup section
x86/word-at-a-time: Remove .fixup usage
x86/usercopy: Remove .fixup usage
x86/usercopy_32: Simplify __copy_user_intel_nocache()
x86/sgx: Remove .fixup usage
x86/checksum_32: Remove .fixup usage
x86/vmx: Remove .fixup usage
x86/kvm: Remove .fixup usage
x86/segment: Remove .fixup usage
x86/fpu: Remove .fixup usage
x86/xen: Remove .fixup usage
x86/uaccess: Remove .fixup usage
x86/futex: Remove .fixup usage
x86/msr: Remove .fixup usage
x86/extable: Extend extable functionality
x86/entry_32: Remove .fixup usage
x86/entry_64: Remove .fixup usage
x86/copy_mc_64: Remove .fixup usage
...

Linus Torvalds 64ad9461 8e5b0ade

+1063 -1365
-11
Makefile
··· 688 688 CC_FLAGS_FTRACE := -pg 689 689 endif 690 690 691 - ifdef CONFIG_CC_IS_GCC 692 - RETPOLINE_CFLAGS := $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register) 693 - RETPOLINE_VDSO_CFLAGS := $(call cc-option,-mindirect-branch=thunk-inline -mindirect-branch-register) 694 - endif 695 - ifdef CONFIG_CC_IS_CLANG 696 - RETPOLINE_CFLAGS := -mretpoline-external-thunk 697 - RETPOLINE_VDSO_CFLAGS := -mretpoline 698 - endif 699 - export RETPOLINE_CFLAGS 700 - export RETPOLINE_VDSO_CFLAGS 701 - 702 691 include $(srctree)/arch/$(SRCARCH)/Makefile 703 692 704 693 ifdef need-config
+13 -1
arch/x86/Kconfig
··· 473 473 branches. Requires a compiler with -mindirect-branch=thunk-extern 474 474 support for full protection. The kernel may run slower. 475 475 476 + config CC_HAS_SLS 477 + def_bool $(cc-option,-mharden-sls=all) 478 + 479 + config SLS 480 + bool "Mitigate Straight-Line-Speculation" 481 + depends on CC_HAS_SLS && X86_64 482 + default n 483 + help 484 + Compile the kernel with straight-line-speculation options to guard 485 + against straight line speculation. The kernel image might be slightly 486 + larger. 487 + 476 488 config X86_CPU_RESCTRL 477 489 bool "x86 CPU resource control support" 478 490 depends on X86 && (CPU_SUP_INTEL || CPU_SUP_AMD) ··· 1964 1952 1965 1953 config EFI_STUB 1966 1954 bool "EFI stub support" 1967 - depends on EFI && !X86_USE_3DNOW 1955 + depends on EFI 1968 1956 depends on $(cc-option,-mabi=ms) || X86_32 1969 1957 select RELOCATABLE 1970 1958 help
-4
arch/x86/Kconfig.cpu
··· 342 342 def_bool y 343 343 depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM 344 344 345 - config X86_USE_3DNOW 346 - def_bool y 347 - depends on (MCYRIXIII || MK7 || MGEODE_LX) && !UML 348 - 349 345 # 350 346 # P6_NOPs are a relatively minor optimization that require a family >= 351 347 # 6 processor, except that it is broken on certain VIA chips.
+16
arch/x86/Makefile
··· 12 12 KBUILD_DEFCONFIG := $(ARCH)_defconfig 13 13 endif 14 14 15 + ifdef CONFIG_CC_IS_GCC 16 + RETPOLINE_CFLAGS := $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register) 17 + RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch-cs-prefix) 18 + RETPOLINE_VDSO_CFLAGS := $(call cc-option,-mindirect-branch=thunk-inline -mindirect-branch-register) 19 + endif 20 + ifdef CONFIG_CC_IS_CLANG 21 + RETPOLINE_CFLAGS := -mretpoline-external-thunk 22 + RETPOLINE_VDSO_CFLAGS := -mretpoline 23 + endif 24 + export RETPOLINE_CFLAGS 25 + export RETPOLINE_VDSO_CFLAGS 26 + 15 27 # For gcc stack alignment is specified with -mpreferred-stack-boundary, 16 28 # clang has the option -mstack-alignment for that purpose. 17 29 ifneq ($(call cc-option, -mpreferred-stack-boundary=4),) ··· 189 177 ifndef CONFIG_CC_IS_CLANG 190 178 KBUILD_CFLAGS += -fno-jump-tables 191 179 endif 180 + endif 181 + 182 + ifdef CONFIG_SLS 183 + KBUILD_CFLAGS += -mharden-sls=all 192 184 endif 193 185 194 186 KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE)
+1 -1
arch/x86/boot/compressed/efi_thunk_64.S
··· 101 101 102 102 pop %rbx 103 103 pop %rbp 104 - ret 104 + RET 105 105 SYM_FUNC_END(__efi64_thunk) 106 106 107 107 .code32
+4 -4
arch/x86/boot/compressed/head_64.S
··· 813 813 2: popl %edi // restore callee-save registers 814 814 popl %ebx 815 815 leave 816 - ret 816 + RET 817 817 SYM_FUNC_END(efi32_pe_entry) 818 818 819 819 .section ".rodata" ··· 868 868 869 869 pop %ecx 870 870 pop %ebx 871 - ret 871 + RET 872 872 SYM_FUNC_END(startup32_set_idt_entry) 873 873 #endif 874 874 ··· 884 884 movl %eax, rva(boot32_idt_desc+2)(%ebp) 885 885 lidt rva(boot32_idt_desc)(%ebp) 886 886 #endif 887 - ret 887 + RET 888 888 SYM_FUNC_END(startup32_load_idt) 889 889 890 890 /* ··· 954 954 popl %ebx 955 955 popl %eax 956 956 #endif 957 - ret 957 + RET 958 958 SYM_FUNC_END(startup32_check_sev_cbit) 959 959 960 960 /*
+3 -3
arch/x86/boot/compressed/mem_encrypt.S
··· 58 58 59 59 #endif /* CONFIG_AMD_MEM_ENCRYPT */ 60 60 61 - ret 61 + RET 62 62 SYM_FUNC_END(get_sev_encryption_bit) 63 63 64 64 /** ··· 92 92 /* All good - return success */ 93 93 xorl %eax, %eax 94 94 1: 95 - ret 95 + RET 96 96 2: 97 97 movl $-1, %eax 98 98 jmp 1b ··· 221 221 #endif 222 222 223 223 xor %rax, %rax 224 - ret 224 + RET 225 225 SYM_FUNC_END(set_sev_encryption_mask) 226 226 227 227 .data
+24 -24
arch/x86/crypto/aegis128-aesni-asm.S
··· 122 122 pxor T0, MSG 123 123 124 124 .Lld_partial_8: 125 - ret 125 + RET 126 126 SYM_FUNC_END(__load_partial) 127 127 128 128 /* ··· 180 180 mov %r10b, (%r9) 181 181 182 182 .Lst_partial_1: 183 - ret 183 + RET 184 184 SYM_FUNC_END(__store_partial) 185 185 186 186 /* ··· 225 225 movdqu STATE4, 0x40(STATEP) 226 226 227 227 FRAME_END 228 - ret 228 + RET 229 229 SYM_FUNC_END(crypto_aegis128_aesni_init) 230 230 231 231 /* ··· 337 337 movdqu STATE3, 0x30(STATEP) 338 338 movdqu STATE4, 0x40(STATEP) 339 339 FRAME_END 340 - ret 340 + RET 341 341 342 342 .Lad_out_1: 343 343 movdqu STATE4, 0x00(STATEP) ··· 346 346 movdqu STATE2, 0x30(STATEP) 347 347 movdqu STATE3, 0x40(STATEP) 348 348 FRAME_END 349 - ret 349 + RET 350 350 351 351 .Lad_out_2: 352 352 movdqu STATE3, 0x00(STATEP) ··· 355 355 movdqu STATE1, 0x30(STATEP) 356 356 movdqu STATE2, 0x40(STATEP) 357 357 FRAME_END 358 - ret 358 + RET 359 359 360 360 .Lad_out_3: 361 361 movdqu STATE2, 0x00(STATEP) ··· 364 364 movdqu STATE0, 0x30(STATEP) 365 365 movdqu STATE1, 0x40(STATEP) 366 366 FRAME_END 367 - ret 367 + RET 368 368 369 369 .Lad_out_4: 370 370 movdqu STATE1, 0x00(STATEP) ··· 373 373 movdqu STATE4, 0x30(STATEP) 374 374 movdqu STATE0, 0x40(STATEP) 375 375 FRAME_END 376 - ret 376 + RET 377 377 378 378 .Lad_out: 379 379 FRAME_END 380 - ret 380 + RET 381 381 SYM_FUNC_END(crypto_aegis128_aesni_ad) 382 382 383 383 .macro encrypt_block a s0 s1 s2 s3 s4 i ··· 452 452 movdqu STATE2, 0x30(STATEP) 453 453 movdqu STATE3, 0x40(STATEP) 454 454 FRAME_END 455 - ret 455 + RET 456 456 457 457 .Lenc_out_1: 458 458 movdqu STATE3, 0x00(STATEP) ··· 461 461 movdqu STATE1, 0x30(STATEP) 462 462 movdqu STATE2, 0x40(STATEP) 463 463 FRAME_END 464 - ret 464 + RET 465 465 466 466 .Lenc_out_2: 467 467 movdqu STATE2, 0x00(STATEP) ··· 470 470 movdqu STATE0, 0x30(STATEP) 471 471 movdqu STATE1, 0x40(STATEP) 472 472 FRAME_END 473 - ret 473 + RET 474 474 475 475 .Lenc_out_3: 476 476 movdqu STATE1, 0x00(STATEP) ··· 479 479 movdqu STATE4, 0x30(STATEP) 480 480 movdqu STATE0, 0x40(STATEP) 481 481 FRAME_END 482 - ret 482 + RET 483 483 484 484 .Lenc_out_4: 485 485 movdqu STATE0, 0x00(STATEP) ··· 488 488 movdqu STATE3, 0x30(STATEP) 489 489 movdqu STATE4, 0x40(STATEP) 490 490 FRAME_END 491 - ret 491 + RET 492 492 493 493 .Lenc_out: 494 494 FRAME_END 495 - ret 495 + RET 496 496 SYM_FUNC_END(crypto_aegis128_aesni_enc) 497 497 498 498 /* ··· 532 532 movdqu STATE3, 0x40(STATEP) 533 533 534 534 FRAME_END 535 - ret 535 + RET 536 536 SYM_FUNC_END(crypto_aegis128_aesni_enc_tail) 537 537 538 538 .macro decrypt_block a s0 s1 s2 s3 s4 i ··· 606 606 movdqu STATE2, 0x30(STATEP) 607 607 movdqu STATE3, 0x40(STATEP) 608 608 FRAME_END 609 - ret 609 + RET 610 610 611 611 .Ldec_out_1: 612 612 movdqu STATE3, 0x00(STATEP) ··· 615 615 movdqu STATE1, 0x30(STATEP) 616 616 movdqu STATE2, 0x40(STATEP) 617 617 FRAME_END 618 - ret 618 + RET 619 619 620 620 .Ldec_out_2: 621 621 movdqu STATE2, 0x00(STATEP) ··· 624 624 movdqu STATE0, 0x30(STATEP) 625 625 movdqu STATE1, 0x40(STATEP) 626 626 FRAME_END 627 - ret 627 + RET 628 628 629 629 .Ldec_out_3: 630 630 movdqu STATE1, 0x00(STATEP) ··· 633 633 movdqu STATE4, 0x30(STATEP) 634 634 movdqu STATE0, 0x40(STATEP) 635 635 FRAME_END 636 - ret 636 + RET 637 637 638 638 .Ldec_out_4: 639 639 movdqu STATE0, 0x00(STATEP) ··· 642 642 movdqu STATE3, 0x30(STATEP) 643 643 movdqu STATE4, 0x40(STATEP) 644 644 FRAME_END 645 - ret 645 + RET 646 646 647 647 .Ldec_out: 648 648 FRAME_END 649 - ret 649 + RET 650 650 SYM_FUNC_END(crypto_aegis128_aesni_dec) 651 651 652 652 /* ··· 696 696 movdqu STATE3, 0x40(STATEP) 697 697 698 698 FRAME_END 699 - ret 699 + RET 700 700 SYM_FUNC_END(crypto_aegis128_aesni_dec_tail) 701 701 702 702 /* ··· 743 743 movdqu MSG, (%rsi) 744 744 745 745 FRAME_END 746 - ret 746 + RET 747 747 SYM_FUNC_END(crypto_aegis128_aesni_final)
+1 -1
arch/x86/crypto/aes_ctrby8_avx-x86_64.S
··· 525 525 /* return updated IV */ 526 526 vpshufb xbyteswap, xcounter, xcounter 527 527 vmovdqu xcounter, (p_iv) 528 - ret 528 + RET 529 529 .endm 530 530 531 531 /*
+28 -28
arch/x86/crypto/aesni-intel_asm.S
··· 1594 1594 GCM_ENC_DEC dec 1595 1595 GCM_COMPLETE arg10, arg11 1596 1596 FUNC_RESTORE 1597 - ret 1597 + RET 1598 1598 SYM_FUNC_END(aesni_gcm_dec) 1599 1599 1600 1600 ··· 1683 1683 1684 1684 GCM_COMPLETE arg10, arg11 1685 1685 FUNC_RESTORE 1686 - ret 1686 + RET 1687 1687 SYM_FUNC_END(aesni_gcm_enc) 1688 1688 1689 1689 /***************************************************************************** ··· 1701 1701 FUNC_SAVE 1702 1702 GCM_INIT %arg3, %arg4,%arg5, %arg6 1703 1703 FUNC_RESTORE 1704 - ret 1704 + RET 1705 1705 SYM_FUNC_END(aesni_gcm_init) 1706 1706 1707 1707 /***************************************************************************** ··· 1716 1716 FUNC_SAVE 1717 1717 GCM_ENC_DEC enc 1718 1718 FUNC_RESTORE 1719 - ret 1719 + RET 1720 1720 SYM_FUNC_END(aesni_gcm_enc_update) 1721 1721 1722 1722 /***************************************************************************** ··· 1731 1731 FUNC_SAVE 1732 1732 GCM_ENC_DEC dec 1733 1733 FUNC_RESTORE 1734 - ret 1734 + RET 1735 1735 SYM_FUNC_END(aesni_gcm_dec_update) 1736 1736 1737 1737 /***************************************************************************** ··· 1746 1746 FUNC_SAVE 1747 1747 GCM_COMPLETE %arg3 %arg4 1748 1748 FUNC_RESTORE 1749 - ret 1749 + RET 1750 1750 SYM_FUNC_END(aesni_gcm_finalize) 1751 1751 1752 1752 #endif ··· 1762 1762 pxor %xmm1, %xmm0 1763 1763 movaps %xmm0, (TKEYP) 1764 1764 add $0x10, TKEYP 1765 - ret 1765 + RET 1766 1766 SYM_FUNC_END(_key_expansion_256a) 1767 1767 SYM_FUNC_END_ALIAS(_key_expansion_128) 1768 1768 ··· 1787 1787 shufps $0b01001110, %xmm2, %xmm1 1788 1788 movaps %xmm1, 0x10(TKEYP) 1789 1789 add $0x20, TKEYP 1790 - ret 1790 + RET 1791 1791 SYM_FUNC_END(_key_expansion_192a) 1792 1792 1793 1793 SYM_FUNC_START_LOCAL(_key_expansion_192b) ··· 1806 1806 1807 1807 movaps %xmm0, (TKEYP) 1808 1808 add $0x10, TKEYP 1809 - ret 1809 + RET 1810 1810 SYM_FUNC_END(_key_expansion_192b) 1811 1811 1812 1812 SYM_FUNC_START_LOCAL(_key_expansion_256b) ··· 1818 1818 pxor %xmm1, %xmm2 1819 1819 movaps %xmm2, (TKEYP) 1820 1820 add $0x10, TKEYP 1821 - ret 1821 + RET 1822 1822 SYM_FUNC_END(_key_expansion_256b) 1823 1823 1824 1824 /* ··· 1933 1933 popl KEYP 1934 1934 #endif 1935 1935 FRAME_END 1936 - ret 1936 + RET 1937 1937 SYM_FUNC_END(aesni_set_key) 1938 1938 1939 1939 /* ··· 1957 1957 popl KEYP 1958 1958 #endif 1959 1959 FRAME_END 1960 - ret 1960 + RET 1961 1961 SYM_FUNC_END(aesni_enc) 1962 1962 1963 1963 /* ··· 2014 2014 aesenc KEY, STATE 2015 2015 movaps 0x70(TKEYP), KEY 2016 2016 aesenclast KEY, STATE 2017 - ret 2017 + RET 2018 2018 SYM_FUNC_END(_aesni_enc1) 2019 2019 2020 2020 /* ··· 2122 2122 aesenclast KEY, STATE2 2123 2123 aesenclast KEY, STATE3 2124 2124 aesenclast KEY, STATE4 2125 - ret 2125 + RET 2126 2126 SYM_FUNC_END(_aesni_enc4) 2127 2127 2128 2128 /* ··· 2147 2147 popl KEYP 2148 2148 #endif 2149 2149 FRAME_END 2150 - ret 2150 + RET 2151 2151 SYM_FUNC_END(aesni_dec) 2152 2152 2153 2153 /* ··· 2204 2204 aesdec KEY, STATE 2205 2205 movaps 0x70(TKEYP), KEY 2206 2206 aesdeclast KEY, STATE 2207 - ret 2207 + RET 2208 2208 SYM_FUNC_END(_aesni_dec1) 2209 2209 2210 2210 /* ··· 2312 2312 aesdeclast KEY, STATE2 2313 2313 aesdeclast KEY, STATE3 2314 2314 aesdeclast KEY, STATE4 2315 - ret 2315 + RET 2316 2316 SYM_FUNC_END(_aesni_dec4) 2317 2317 2318 2318 /* ··· 2372 2372 popl LEN 2373 2373 #endif 2374 2374 FRAME_END 2375 - ret 2375 + RET 2376 2376 SYM_FUNC_END(aesni_ecb_enc) 2377 2377 2378 2378 /* ··· 2433 2433 popl LEN 2434 2434 #endif 2435 2435 FRAME_END 2436 - ret 2436 + RET 2437 2437 SYM_FUNC_END(aesni_ecb_dec) 2438 2438 2439 2439 /* ··· 2477 2477 popl IVP 2478 2478 #endif 2479 2479 FRAME_END 2480 - ret 2480 + RET 2481 2481 SYM_FUNC_END(aesni_cbc_enc) 2482 2482 2483 2483 /* ··· 2570 2570 popl IVP 2571 2571 #endif 2572 2572 FRAME_END 2573 - ret 2573 + RET 2574 2574 SYM_FUNC_END(aesni_cbc_dec) 2575 2575 2576 2576 /* ··· 2627 2627 popl IVP 2628 2628 #endif 2629 2629 FRAME_END 2630 - ret 2630 + RET 2631 2631 SYM_FUNC_END(aesni_cts_cbc_enc) 2632 2632 2633 2633 /* ··· 2688 2688 popl IVP 2689 2689 #endif 2690 2690 FRAME_END 2691 - ret 2691 + RET 2692 2692 SYM_FUNC_END(aesni_cts_cbc_dec) 2693 2693 2694 2694 .pushsection .rodata ··· 2725 2725 mov $1, TCTR_LOW 2726 2726 movq TCTR_LOW, INC 2727 2727 movq CTR, TCTR_LOW 2728 - ret 2728 + RET 2729 2729 SYM_FUNC_END(_aesni_inc_init) 2730 2730 2731 2731 /* ··· 2753 2753 .Linc_low: 2754 2754 movaps CTR, IV 2755 2755 pshufb BSWAP_MASK, IV 2756 - ret 2756 + RET 2757 2757 SYM_FUNC_END(_aesni_inc) 2758 2758 2759 2759 /* ··· 2816 2816 movups IV, (IVP) 2817 2817 .Lctr_enc_just_ret: 2818 2818 FRAME_END 2819 - ret 2819 + RET 2820 2820 SYM_FUNC_END(aesni_ctr_enc) 2821 2821 2822 2822 #endif ··· 2932 2932 popl IVP 2933 2933 #endif 2934 2934 FRAME_END 2935 - ret 2935 + RET 2936 2936 2937 2937 .Lxts_enc_1x: 2938 2938 add $64, LEN ··· 3092 3092 popl IVP 3093 3093 #endif 3094 3094 FRAME_END 3095 - ret 3095 + RET 3096 3096 3097 3097 .Lxts_dec_1x: 3098 3098 add $64, LEN
+20 -20
arch/x86/crypto/aesni-intel_avx-x86_64.S
··· 1767 1767 FUNC_SAVE 1768 1768 INIT GHASH_MUL_AVX, PRECOMPUTE_AVX 1769 1769 FUNC_RESTORE 1770 - ret 1770 + RET 1771 1771 SYM_FUNC_END(aesni_gcm_init_avx_gen2) 1772 1772 1773 1773 ############################################################################### ··· 1788 1788 # must be 192 1789 1789 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 11 1790 1790 FUNC_RESTORE 1791 - ret 1791 + RET 1792 1792 key_128_enc_update: 1793 1793 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 9 1794 1794 FUNC_RESTORE 1795 - ret 1795 + RET 1796 1796 key_256_enc_update: 1797 1797 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 13 1798 1798 FUNC_RESTORE 1799 - ret 1799 + RET 1800 1800 SYM_FUNC_END(aesni_gcm_enc_update_avx_gen2) 1801 1801 1802 1802 ############################################################################### ··· 1817 1817 # must be 192 1818 1818 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 11 1819 1819 FUNC_RESTORE 1820 - ret 1820 + RET 1821 1821 key_128_dec_update: 1822 1822 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 9 1823 1823 FUNC_RESTORE 1824 - ret 1824 + RET 1825 1825 key_256_dec_update: 1826 1826 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 13 1827 1827 FUNC_RESTORE 1828 - ret 1828 + RET 1829 1829 SYM_FUNC_END(aesni_gcm_dec_update_avx_gen2) 1830 1830 1831 1831 ############################################################################### ··· 1846 1846 # must be 192 1847 1847 GCM_COMPLETE GHASH_MUL_AVX, 11, arg3, arg4 1848 1848 FUNC_RESTORE 1849 - ret 1849 + RET 1850 1850 key_128_finalize: 1851 1851 GCM_COMPLETE GHASH_MUL_AVX, 9, arg3, arg4 1852 1852 FUNC_RESTORE 1853 - ret 1853 + RET 1854 1854 key_256_finalize: 1855 1855 GCM_COMPLETE GHASH_MUL_AVX, 13, arg3, arg4 1856 1856 FUNC_RESTORE 1857 - ret 1857 + RET 1858 1858 SYM_FUNC_END(aesni_gcm_finalize_avx_gen2) 1859 1859 1860 1860 ############################################################################### ··· 2735 2735 FUNC_SAVE 2736 2736 INIT GHASH_MUL_AVX2, PRECOMPUTE_AVX2 2737 2737 FUNC_RESTORE 2738 - ret 2738 + RET 2739 2739 SYM_FUNC_END(aesni_gcm_init_avx_gen4) 2740 2740 2741 2741 ############################################################################### ··· 2756 2756 # must be 192 2757 2757 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 11 2758 2758 FUNC_RESTORE 2759 - ret 2759 + RET 2760 2760 key_128_enc_update4: 2761 2761 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 9 2762 2762 FUNC_RESTORE 2763 - ret 2763 + RET 2764 2764 key_256_enc_update4: 2765 2765 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 13 2766 2766 FUNC_RESTORE 2767 - ret 2767 + RET 2768 2768 SYM_FUNC_END(aesni_gcm_enc_update_avx_gen4) 2769 2769 2770 2770 ############################################################################### ··· 2785 2785 # must be 192 2786 2786 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 11 2787 2787 FUNC_RESTORE 2788 - ret 2788 + RET 2789 2789 key_128_dec_update4: 2790 2790 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 9 2791 2791 FUNC_RESTORE 2792 - ret 2792 + RET 2793 2793 key_256_dec_update4: 2794 2794 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 13 2795 2795 FUNC_RESTORE 2796 - ret 2796 + RET 2797 2797 SYM_FUNC_END(aesni_gcm_dec_update_avx_gen4) 2798 2798 2799 2799 ############################################################################### ··· 2814 2814 # must be 192 2815 2815 GCM_COMPLETE GHASH_MUL_AVX2, 11, arg3, arg4 2816 2816 FUNC_RESTORE 2817 - ret 2817 + RET 2818 2818 key_128_finalize4: 2819 2819 GCM_COMPLETE GHASH_MUL_AVX2, 9, arg3, arg4 2820 2820 FUNC_RESTORE 2821 - ret 2821 + RET 2822 2822 key_256_finalize4: 2823 2823 GCM_COMPLETE GHASH_MUL_AVX2, 13, arg3, arg4 2824 2824 FUNC_RESTORE 2825 - ret 2825 + RET 2826 2826 SYM_FUNC_END(aesni_gcm_finalize_avx_gen4)
+2 -2
arch/x86/crypto/blake2s-core.S
··· 171 171 movdqu %xmm1,0x10(%rdi) 172 172 movdqu %xmm14,0x20(%rdi) 173 173 .Lendofloop: 174 - ret 174 + RET 175 175 SYM_FUNC_END(blake2s_compress_ssse3) 176 176 177 177 #ifdef CONFIG_AS_AVX512 ··· 251 251 vmovdqu %xmm1,0x10(%rdi) 252 252 vmovdqu %xmm4,0x20(%rdi) 253 253 vzeroupper 254 - retq 254 + RET 255 255 SYM_FUNC_END(blake2s_compress_avx512) 256 256 #endif /* CONFIG_AS_AVX512 */
+6 -6
arch/x86/crypto/blowfish-x86_64-asm_64.S
··· 135 135 jnz .L__enc_xor; 136 136 137 137 write_block(); 138 - ret; 138 + RET; 139 139 .L__enc_xor: 140 140 xor_block(); 141 - ret; 141 + RET; 142 142 SYM_FUNC_END(__blowfish_enc_blk) 143 143 144 144 SYM_FUNC_START(blowfish_dec_blk) ··· 170 170 171 171 movq %r11, %r12; 172 172 173 - ret; 173 + RET; 174 174 SYM_FUNC_END(blowfish_dec_blk) 175 175 176 176 /********************************************************************** ··· 322 322 323 323 popq %rbx; 324 324 popq %r12; 325 - ret; 325 + RET; 326 326 327 327 .L__enc_xor4: 328 328 xor_block4(); 329 329 330 330 popq %rbx; 331 331 popq %r12; 332 - ret; 332 + RET; 333 333 SYM_FUNC_END(__blowfish_enc_blk_4way) 334 334 335 335 SYM_FUNC_START(blowfish_dec_blk_4way) ··· 364 364 popq %rbx; 365 365 popq %r12; 366 366 367 - ret; 367 + RET; 368 368 SYM_FUNC_END(blowfish_dec_blk_4way)
+7 -7
arch/x86/crypto/camellia-aesni-avx-asm_64.S
··· 192 192 roundsm16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, 193 193 %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, 194 194 %rcx, (%r9)); 195 - ret; 195 + RET; 196 196 SYM_FUNC_END(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) 197 197 198 198 .align 8 ··· 200 200 roundsm16(%xmm4, %xmm5, %xmm6, %xmm7, %xmm0, %xmm1, %xmm2, %xmm3, 201 201 %xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11, 202 202 %rax, (%r9)); 203 - ret; 203 + RET; 204 204 SYM_FUNC_END(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) 205 205 206 206 /* ··· 778 778 %xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax)); 779 779 780 780 FRAME_END 781 - ret; 781 + RET; 782 782 783 783 .align 8 784 784 .Lenc_max32: ··· 865 865 %xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax)); 866 866 867 867 FRAME_END 868 - ret; 868 + RET; 869 869 870 870 .align 8 871 871 .Ldec_max32: ··· 906 906 %xmm8, %rsi); 907 907 908 908 FRAME_END 909 - ret; 909 + RET; 910 910 SYM_FUNC_END(camellia_ecb_enc_16way) 911 911 912 912 SYM_FUNC_START(camellia_ecb_dec_16way) ··· 936 936 %xmm8, %rsi); 937 937 938 938 FRAME_END 939 - ret; 939 + RET; 940 940 SYM_FUNC_END(camellia_ecb_dec_16way) 941 941 942 942 SYM_FUNC_START(camellia_cbc_dec_16way) ··· 987 987 %xmm8, %rsi); 988 988 989 989 FRAME_END 990 - ret; 990 + RET; 991 991 SYM_FUNC_END(camellia_cbc_dec_16way)
+7 -7
arch/x86/crypto/camellia-aesni-avx2-asm_64.S
··· 226 226 roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 227 227 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, 228 228 %rcx, (%r9)); 229 - ret; 229 + RET; 230 230 SYM_FUNC_END(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) 231 231 232 232 .align 8 ··· 234 234 roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3, 235 235 %ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11, 236 236 %rax, (%r9)); 237 - ret; 237 + RET; 238 238 SYM_FUNC_END(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) 239 239 240 240 /* ··· 814 814 %ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax)); 815 815 816 816 FRAME_END 817 - ret; 817 + RET; 818 818 819 819 .align 8 820 820 .Lenc_max32: ··· 901 901 %ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax)); 902 902 903 903 FRAME_END 904 - ret; 904 + RET; 905 905 906 906 .align 8 907 907 .Ldec_max32: ··· 946 946 vzeroupper; 947 947 948 948 FRAME_END 949 - ret; 949 + RET; 950 950 SYM_FUNC_END(camellia_ecb_enc_32way) 951 951 952 952 SYM_FUNC_START(camellia_ecb_dec_32way) ··· 980 980 vzeroupper; 981 981 982 982 FRAME_END 983 - ret; 983 + RET; 984 984 SYM_FUNC_END(camellia_ecb_dec_32way) 985 985 986 986 SYM_FUNC_START(camellia_cbc_dec_32way) ··· 1047 1047 1048 1048 addq $(16 * 32), %rsp; 1049 1049 FRAME_END 1050 - ret; 1050 + RET; 1051 1051 SYM_FUNC_END(camellia_cbc_dec_32way)
+6 -6
arch/x86/crypto/camellia-x86_64-asm_64.S
··· 213 213 enc_outunpack(mov, RT1); 214 214 215 215 movq RR12, %r12; 216 - ret; 216 + RET; 217 217 218 218 .L__enc_xor: 219 219 enc_outunpack(xor, RT1); 220 220 221 221 movq RR12, %r12; 222 - ret; 222 + RET; 223 223 SYM_FUNC_END(__camellia_enc_blk) 224 224 225 225 SYM_FUNC_START(camellia_dec_blk) ··· 257 257 dec_outunpack(); 258 258 259 259 movq RR12, %r12; 260 - ret; 260 + RET; 261 261 SYM_FUNC_END(camellia_dec_blk) 262 262 263 263 /********************************************************************** ··· 448 448 449 449 movq RR12, %r12; 450 450 popq %rbx; 451 - ret; 451 + RET; 452 452 453 453 .L__enc2_xor: 454 454 enc_outunpack2(xor, RT2); 455 455 456 456 movq RR12, %r12; 457 457 popq %rbx; 458 - ret; 458 + RET; 459 459 SYM_FUNC_END(__camellia_enc_blk_2way) 460 460 461 461 SYM_FUNC_START(camellia_dec_blk_2way) ··· 495 495 496 496 movq RR12, %r12; 497 497 movq RXOR, %rbx; 498 - ret; 498 + RET; 499 499 SYM_FUNC_END(camellia_dec_blk_2way)
+6 -6
arch/x86/crypto/cast5-avx-x86_64-asm_64.S
··· 279 279 outunpack_blocks(RR3, RL3, RTMP, RX, RKM); 280 280 outunpack_blocks(RR4, RL4, RTMP, RX, RKM); 281 281 282 - ret; 282 + RET; 283 283 SYM_FUNC_END(__cast5_enc_blk16) 284 284 285 285 .align 16 ··· 352 352 outunpack_blocks(RR3, RL3, RTMP, RX, RKM); 353 353 outunpack_blocks(RR4, RL4, RTMP, RX, RKM); 354 354 355 - ret; 355 + RET; 356 356 357 357 .L__skip_dec: 358 358 vpsrldq $4, RKR, RKR; ··· 393 393 394 394 popq %r15; 395 395 FRAME_END 396 - ret; 396 + RET; 397 397 SYM_FUNC_END(cast5_ecb_enc_16way) 398 398 399 399 SYM_FUNC_START(cast5_ecb_dec_16way) ··· 431 431 432 432 popq %r15; 433 433 FRAME_END 434 - ret; 434 + RET; 435 435 SYM_FUNC_END(cast5_ecb_dec_16way) 436 436 437 437 SYM_FUNC_START(cast5_cbc_dec_16way) ··· 483 483 popq %r15; 484 484 popq %r12; 485 485 FRAME_END 486 - ret; 486 + RET; 487 487 SYM_FUNC_END(cast5_cbc_dec_16way) 488 488 489 489 SYM_FUNC_START(cast5_ctr_16way) ··· 559 559 popq %r15; 560 560 popq %r12; 561 561 FRAME_END 562 - ret; 562 + RET; 563 563 SYM_FUNC_END(cast5_ctr_16way)
+5 -5
arch/x86/crypto/cast6-avx-x86_64-asm_64.S
··· 289 289 outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); 290 290 outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); 291 291 292 - ret; 292 + RET; 293 293 SYM_FUNC_END(__cast6_enc_blk8) 294 294 295 295 .align 8 ··· 336 336 outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); 337 337 outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); 338 338 339 - ret; 339 + RET; 340 340 SYM_FUNC_END(__cast6_dec_blk8) 341 341 342 342 SYM_FUNC_START(cast6_ecb_enc_8way) ··· 359 359 360 360 popq %r15; 361 361 FRAME_END 362 - ret; 362 + RET; 363 363 SYM_FUNC_END(cast6_ecb_enc_8way) 364 364 365 365 SYM_FUNC_START(cast6_ecb_dec_8way) ··· 382 382 383 383 popq %r15; 384 384 FRAME_END 385 - ret; 385 + RET; 386 386 SYM_FUNC_END(cast6_ecb_dec_8way) 387 387 388 388 SYM_FUNC_START(cast6_cbc_dec_8way) ··· 408 408 popq %r15; 409 409 popq %r12; 410 410 FRAME_END 411 - ret; 411 + RET; 412 412 SYM_FUNC_END(cast6_cbc_dec_8way)
+3 -3
arch/x86/crypto/chacha-avx2-x86_64.S
··· 193 193 194 194 .Ldone2: 195 195 vzeroupper 196 - ret 196 + RET 197 197 198 198 .Lxorpart2: 199 199 # xor remaining bytes from partial register into output ··· 498 498 499 499 .Ldone4: 500 500 vzeroupper 501 - ret 501 + RET 502 502 503 503 .Lxorpart4: 504 504 # xor remaining bytes from partial register into output ··· 992 992 .Ldone8: 993 993 vzeroupper 994 994 lea -8(%r10),%rsp 995 - ret 995 + RET 996 996 997 997 .Lxorpart8: 998 998 # xor remaining bytes from partial register into output
+3 -3
arch/x86/crypto/chacha-avx512vl-x86_64.S
··· 166 166 167 167 .Ldone2: 168 168 vzeroupper 169 - ret 169 + RET 170 170 171 171 .Lxorpart2: 172 172 # xor remaining bytes from partial register into output ··· 432 432 433 433 .Ldone4: 434 434 vzeroupper 435 - ret 435 + RET 436 436 437 437 .Lxorpart4: 438 438 # xor remaining bytes from partial register into output ··· 812 812 813 813 .Ldone8: 814 814 vzeroupper 815 - ret 815 + RET 816 816 817 817 .Lxorpart8: 818 818 # xor remaining bytes from partial register into output
+4 -4
arch/x86/crypto/chacha-ssse3-x86_64.S
··· 108 108 sub $2,%r8d 109 109 jnz .Ldoubleround 110 110 111 - ret 111 + RET 112 112 SYM_FUNC_END(chacha_permute) 113 113 114 114 SYM_FUNC_START(chacha_block_xor_ssse3) ··· 166 166 167 167 .Ldone: 168 168 FRAME_END 169 - ret 169 + RET 170 170 171 171 .Lxorpart: 172 172 # xor remaining bytes from partial register into output ··· 217 217 movdqu %xmm3,0x10(%rsi) 218 218 219 219 FRAME_END 220 - ret 220 + RET 221 221 SYM_FUNC_END(hchacha_block_ssse3) 222 222 223 223 SYM_FUNC_START(chacha_4block_xor_ssse3) ··· 762 762 763 763 .Ldone4: 764 764 lea -8(%r10),%rsp 765 - ret 765 + RET 766 766 767 767 .Lxorpart4: 768 768 # xor remaining bytes from partial register into output
+1 -1
arch/x86/crypto/crc32-pclmul_asm.S
··· 236 236 pxor %xmm2, %xmm1 237 237 pextrd $0x01, %xmm1, %eax 238 238 239 - ret 239 + RET 240 240 SYM_FUNC_END(crc32_pclmul_le_16)
+1 -1
arch/x86/crypto/crc32c-pcl-intel-asm_64.S
··· 306 306 popq %rsi 307 307 popq %rdi 308 308 popq %rbx 309 - ret 309 + RET 310 310 SYM_FUNC_END(crc_pcl) 311 311 312 312 .section .rodata, "a", @progbits
+1 -1
arch/x86/crypto/crct10dif-pcl-asm_64.S
··· 257 257 # Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of xmm0. 258 258 259 259 pextrw $0, %xmm0, %eax 260 - ret 260 + RET 261 261 262 262 .align 16 263 263 .Lless_than_256_bytes:
+2 -2
arch/x86/crypto/des3_ede-asm_64.S
··· 243 243 popq %r12; 244 244 popq %rbx; 245 245 246 - ret; 246 + RET; 247 247 SYM_FUNC_END(des3_ede_x86_64_crypt_blk) 248 248 249 249 /*********************************************************************** ··· 528 528 popq %r12; 529 529 popq %rbx; 530 530 531 - ret; 531 + RET; 532 532 SYM_FUNC_END(des3_ede_x86_64_crypt_blk_3way) 533 533 534 534 .section .rodata, "a", @progbits
+3 -3
arch/x86/crypto/ghash-clmulni-intel_asm.S
··· 85 85 psrlq $1, T2 86 86 pxor T2, T1 87 87 pxor T1, DATA 88 - ret 88 + RET 89 89 SYM_FUNC_END(__clmul_gf128mul_ble) 90 90 91 91 /* void clmul_ghash_mul(char *dst, const u128 *shash) */ ··· 99 99 pshufb BSWAP, DATA 100 100 movups DATA, (%rdi) 101 101 FRAME_END 102 - ret 102 + RET 103 103 SYM_FUNC_END(clmul_ghash_mul) 104 104 105 105 /* ··· 128 128 movups DATA, (%rdi) 129 129 .Lupdate_just_ret: 130 130 FRAME_END 131 - ret 131 + RET 132 132 SYM_FUNC_END(clmul_ghash_update)
+1 -1
arch/x86/crypto/nh-avx2-x86_64.S
··· 153 153 vpaddq T1, T0, T0 154 154 vpaddq T4, T0, T0 155 155 vmovdqu T0, (HASH) 156 - ret 156 + RET 157 157 SYM_FUNC_END(nh_avx2)
+1 -1
arch/x86/crypto/nh-sse2-x86_64.S
··· 119 119 paddq PASS2_SUMS, T1 120 120 movdqu T0, 0x00(HASH) 121 121 movdqu T1, 0x10(HASH) 122 - ret 122 + RET 123 123 SYM_FUNC_END(nh_sse2)
+5 -5
arch/x86/crypto/serpent-avx-x86_64-asm_64.S
··· 601 601 write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); 602 602 write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); 603 603 604 - ret; 604 + RET; 605 605 SYM_FUNC_END(__serpent_enc_blk8_avx) 606 606 607 607 .align 8 ··· 655 655 write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2); 656 656 write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2); 657 657 658 - ret; 658 + RET; 659 659 SYM_FUNC_END(__serpent_dec_blk8_avx) 660 660 661 661 SYM_FUNC_START(serpent_ecb_enc_8way_avx) ··· 673 673 store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 674 674 675 675 FRAME_END 676 - ret; 676 + RET; 677 677 SYM_FUNC_END(serpent_ecb_enc_8way_avx) 678 678 679 679 SYM_FUNC_START(serpent_ecb_dec_8way_avx) ··· 691 691 store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); 692 692 693 693 FRAME_END 694 - ret; 694 + RET; 695 695 SYM_FUNC_END(serpent_ecb_dec_8way_avx) 696 696 697 697 SYM_FUNC_START(serpent_cbc_dec_8way_avx) ··· 709 709 store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); 710 710 711 711 FRAME_END 712 - ret; 712 + RET; 713 713 SYM_FUNC_END(serpent_cbc_dec_8way_avx)
+5 -5
arch/x86/crypto/serpent-avx2-asm_64.S
··· 601 601 write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); 602 602 write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); 603 603 604 - ret; 604 + RET; 605 605 SYM_FUNC_END(__serpent_enc_blk16) 606 606 607 607 .align 8 ··· 655 655 write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2); 656 656 write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2); 657 657 658 - ret; 658 + RET; 659 659 SYM_FUNC_END(__serpent_dec_blk16) 660 660 661 661 SYM_FUNC_START(serpent_ecb_enc_16way) ··· 677 677 vzeroupper; 678 678 679 679 FRAME_END 680 - ret; 680 + RET; 681 681 SYM_FUNC_END(serpent_ecb_enc_16way) 682 682 683 683 SYM_FUNC_START(serpent_ecb_dec_16way) ··· 699 699 vzeroupper; 700 700 701 701 FRAME_END 702 - ret; 702 + RET; 703 703 SYM_FUNC_END(serpent_ecb_dec_16way) 704 704 705 705 SYM_FUNC_START(serpent_cbc_dec_16way) ··· 722 722 vzeroupper; 723 723 724 724 FRAME_END 725 - ret; 725 + RET; 726 726 SYM_FUNC_END(serpent_cbc_dec_16way)
+3 -3
arch/x86/crypto/serpent-sse2-i586-asm_32.S
··· 553 553 554 554 write_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE); 555 555 556 - ret; 556 + RET; 557 557 558 558 .L__enc_xor4: 559 559 xor_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE); 560 560 561 - ret; 561 + RET; 562 562 SYM_FUNC_END(__serpent_enc_blk_4way) 563 563 564 564 SYM_FUNC_START(serpent_dec_blk_4way) ··· 612 612 movl arg_dst(%esp), %eax; 613 613 write_blocks(%eax, RC, RD, RB, RE, RT0, RT1, RA); 614 614 615 - ret; 615 + RET; 616 616 SYM_FUNC_END(serpent_dec_blk_4way)
+3 -3
arch/x86/crypto/serpent-sse2-x86_64-asm_64.S
··· 675 675 write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); 676 676 write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); 677 677 678 - ret; 678 + RET; 679 679 680 680 .L__enc_xor8: 681 681 xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); 682 682 xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); 683 683 684 - ret; 684 + RET; 685 685 SYM_FUNC_END(__serpent_enc_blk_8way) 686 686 687 687 SYM_FUNC_START(serpent_dec_blk_8way) ··· 735 735 write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2); 736 736 write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2); 737 737 738 - ret; 738 + RET; 739 739 SYM_FUNC_END(serpent_dec_blk_8way)
+1 -1
arch/x86/crypto/sha1_avx2_x86_64_asm.S
··· 674 674 pop %r12 675 675 pop %rbx 676 676 677 - ret 677 + RET 678 678 679 679 SYM_FUNC_END(\name) 680 680 .endm
+1 -1
arch/x86/crypto/sha1_ni_asm.S
··· 290 290 mov %rbp, %rsp 291 291 pop %rbp 292 292 293 - ret 293 + RET 294 294 SYM_FUNC_END(sha1_ni_transform) 295 295 296 296 .section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
+1 -1
arch/x86/crypto/sha1_ssse3_asm.S
··· 99 99 pop %rbp 100 100 pop %r12 101 101 pop %rbx 102 - ret 102 + RET 103 103 104 104 SYM_FUNC_END(\name) 105 105 .endm
+1 -1
arch/x86/crypto/sha256-avx-asm.S
··· 458 458 popq %r13 459 459 popq %r12 460 460 popq %rbx 461 - ret 461 + RET 462 462 SYM_FUNC_END(sha256_transform_avx) 463 463 464 464 .section .rodata.cst256.K256, "aM", @progbits, 256
+1 -1
arch/x86/crypto/sha256-avx2-asm.S
··· 710 710 popq %r13 711 711 popq %r12 712 712 popq %rbx 713 - ret 713 + RET 714 714 SYM_FUNC_END(sha256_transform_rorx) 715 715 716 716 .section .rodata.cst512.K256, "aM", @progbits, 512
+1 -1
arch/x86/crypto/sha256-ssse3-asm.S
··· 472 472 popq %r12 473 473 popq %rbx 474 474 475 - ret 475 + RET 476 476 SYM_FUNC_END(sha256_transform_ssse3) 477 477 478 478 .section .rodata.cst256.K256, "aM", @progbits, 256
+1 -1
arch/x86/crypto/sha256_ni_asm.S
··· 326 326 327 327 .Ldone_hash: 328 328 329 - ret 329 + RET 330 330 SYM_FUNC_END(sha256_ni_transform) 331 331 332 332 .section .rodata.cst256.K256, "aM", @progbits, 256
+1 -1
arch/x86/crypto/sha512-avx-asm.S
··· 361 361 pop %rbx 362 362 363 363 nowork: 364 - ret 364 + RET 365 365 SYM_FUNC_END(sha512_transform_avx) 366 366 367 367 ########################################################################
+1 -1
arch/x86/crypto/sha512-avx2-asm.S
··· 679 679 pop %r12 680 680 pop %rbx 681 681 682 - ret 682 + RET 683 683 SYM_FUNC_END(sha512_transform_rorx) 684 684 685 685 ########################################################################
+1 -1
arch/x86/crypto/sha512-ssse3-asm.S
··· 363 363 pop %rbx 364 364 365 365 nowork: 366 - ret 366 + RET 367 367 SYM_FUNC_END(sha512_transform_ssse3) 368 368 369 369 ########################################################################
+6 -6
arch/x86/crypto/sm4-aesni-avx-asm_64.S
··· 246 246 .Lblk4_store_output_done: 247 247 vzeroall; 248 248 FRAME_END 249 - ret; 249 + RET; 250 250 SYM_FUNC_END(sm4_aesni_avx_crypt4) 251 251 252 252 .align 8 ··· 356 356 vpshufb RTMP2, RB3, RB3; 357 357 358 358 FRAME_END 359 - ret; 359 + RET; 360 360 SYM_FUNC_END(__sm4_crypt_blk8) 361 361 362 362 /* ··· 412 412 .Lblk8_store_output_done: 413 413 vzeroall; 414 414 FRAME_END 415 - ret; 415 + RET; 416 416 SYM_FUNC_END(sm4_aesni_avx_crypt8) 417 417 418 418 /* ··· 487 487 488 488 vzeroall; 489 489 FRAME_END 490 - ret; 490 + RET; 491 491 SYM_FUNC_END(sm4_aesni_avx_ctr_enc_blk8) 492 492 493 493 /* ··· 537 537 538 538 vzeroall; 539 539 FRAME_END 540 - ret; 540 + RET; 541 541 SYM_FUNC_END(sm4_aesni_avx_cbc_dec_blk8) 542 542 543 543 /* ··· 590 590 591 591 vzeroall; 592 592 FRAME_END 593 - ret; 593 + RET; 594 594 SYM_FUNC_END(sm4_aesni_avx_cfb_dec_blk8)
+4 -4
arch/x86/crypto/sm4-aesni-avx2-asm_64.S
··· 268 268 vpshufb RTMP2, RB3, RB3; 269 269 270 270 FRAME_END 271 - ret; 271 + RET; 272 272 SYM_FUNC_END(__sm4_crypt_blk16) 273 273 274 274 #define inc_le128(x, minus_one, tmp) \ ··· 387 387 388 388 vzeroall; 389 389 FRAME_END 390 - ret; 390 + RET; 391 391 SYM_FUNC_END(sm4_aesni_avx2_ctr_enc_blk16) 392 392 393 393 /* ··· 441 441 442 442 vzeroall; 443 443 FRAME_END 444 - ret; 444 + RET; 445 445 SYM_FUNC_END(sm4_aesni_avx2_cbc_dec_blk16) 446 446 447 447 /* ··· 497 497 498 498 vzeroall; 499 499 FRAME_END 500 - ret; 500 + RET; 501 501 SYM_FUNC_END(sm4_aesni_avx2_cfb_dec_blk16)
+5 -5
arch/x86/crypto/twofish-avx-x86_64-asm_64.S
··· 267 267 outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); 268 268 outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); 269 269 270 - ret; 270 + RET; 271 271 SYM_FUNC_END(__twofish_enc_blk8) 272 272 273 273 .align 8 ··· 307 307 outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2); 308 308 outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2); 309 309 310 - ret; 310 + RET; 311 311 SYM_FUNC_END(__twofish_dec_blk8) 312 312 313 313 SYM_FUNC_START(twofish_ecb_enc_8way) ··· 327 327 store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); 328 328 329 329 FRAME_END 330 - ret; 330 + RET; 331 331 SYM_FUNC_END(twofish_ecb_enc_8way) 332 332 333 333 SYM_FUNC_START(twofish_ecb_dec_8way) ··· 347 347 store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 348 348 349 349 FRAME_END 350 - ret; 350 + RET; 351 351 SYM_FUNC_END(twofish_ecb_dec_8way) 352 352 353 353 SYM_FUNC_START(twofish_cbc_dec_8way) ··· 372 372 popq %r12; 373 373 374 374 FRAME_END 375 - ret; 375 + RET; 376 376 SYM_FUNC_END(twofish_cbc_dec_8way)
+2 -2
arch/x86/crypto/twofish-i586-asm_32.S
··· 260 260 pop %ebx 261 261 pop %ebp 262 262 mov $1, %eax 263 - ret 263 + RET 264 264 SYM_FUNC_END(twofish_enc_blk) 265 265 266 266 SYM_FUNC_START(twofish_dec_blk) ··· 317 317 pop %ebx 318 318 pop %ebp 319 319 mov $1, %eax 320 - ret 320 + RET 321 321 SYM_FUNC_END(twofish_dec_blk)
+3 -3
arch/x86/crypto/twofish-x86_64-asm_64-3way.S
··· 258 258 popq %rbx; 259 259 popq %r12; 260 260 popq %r13; 261 - ret; 261 + RET; 262 262 263 263 .L__enc_xor3: 264 264 outunpack_enc3(xor); ··· 266 266 popq %rbx; 267 267 popq %r12; 268 268 popq %r13; 269 - ret; 269 + RET; 270 270 SYM_FUNC_END(__twofish_enc_blk_3way) 271 271 272 272 SYM_FUNC_START(twofish_dec_blk_3way) ··· 301 301 popq %rbx; 302 302 popq %r12; 303 303 popq %r13; 304 - ret; 304 + RET; 305 305 SYM_FUNC_END(twofish_dec_blk_3way)
+2 -2
arch/x86/crypto/twofish-x86_64-asm_64.S
··· 252 252 253 253 popq R1 254 254 movl $1,%eax 255 - ret 255 + RET 256 256 SYM_FUNC_END(twofish_enc_blk) 257 257 258 258 SYM_FUNC_START(twofish_dec_blk) ··· 304 304 305 305 popq R1 306 306 movl $1,%eax 307 - ret 307 + RET 308 308 SYM_FUNC_END(twofish_dec_blk)
+15 -22
arch/x86/entry/entry_32.S
··· 268 268 1: popl %ds 269 269 2: popl %es 270 270 3: popl %fs 271 - addl $(4 + \pop), %esp /* pop the unused "gs" slot */ 271 + 4: addl $(4 + \pop), %esp /* pop the unused "gs" slot */ 272 272 IRET_FRAME 273 - .pushsection .fixup, "ax" 274 - 4: movl $0, (%esp) 275 - jmp 1b 276 - 5: movl $0, (%esp) 277 - jmp 2b 278 - 6: movl $0, (%esp) 279 - jmp 3b 280 - .popsection 281 - _ASM_EXTABLE(1b, 4b) 282 - _ASM_EXTABLE(2b, 5b) 283 - _ASM_EXTABLE(3b, 6b) 273 + 274 + /* 275 + * There is no _ASM_EXTABLE_TYPE_REG() for ASM, however since this is 276 + * ASM the registers are known and we can trivially hard-code them. 277 + */ 278 + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_POP_ZERO|EX_REG_DS) 279 + _ASM_EXTABLE_TYPE(2b, 3b, EX_TYPE_POP_ZERO|EX_REG_ES) 280 + _ASM_EXTABLE_TYPE(3b, 4b, EX_TYPE_POP_ZERO|EX_REG_FS) 284 281 .endm 285 282 286 283 .macro RESTORE_ALL_NMI cr3_reg:req pop=0 ··· 737 740 popl %eax 738 741 739 742 FRAME_END 740 - ret 743 + RET 741 744 SYM_FUNC_END(schedule_tail_wrapper) 742 745 .popsection 743 746 ··· 922 925 sti 923 926 sysexit 924 927 925 - .pushsection .fixup, "ax" 926 - 2: movl $0, PT_FS(%esp) 927 - jmp 1b 928 - .popsection 928 + 2: movl $0, PT_FS(%esp) 929 + jmp 1b 929 930 _ASM_EXTABLE(1b, 2b) 930 931 931 932 .Lsysenter_fix_flags: ··· 991 996 */ 992 997 iret 993 998 994 - .section .fixup, "ax" 995 - SYM_CODE_START(asm_iret_error) 999 + .Lasm_iret_error: 996 1000 pushl $0 # no error code 997 1001 pushl $iret_error 998 1002 ··· 1008 1014 #endif 1009 1015 1010 1016 jmp handle_exception 1011 - SYM_CODE_END(asm_iret_error) 1012 - .previous 1013 - _ASM_EXTABLE(.Lirq_return, asm_iret_error) 1017 + 1018 + _ASM_EXTABLE(.Lirq_return, .Lasm_iret_error) 1014 1019 SYM_FUNC_END(entry_INT80_32) 1015 1020 1016 1021 .macro FIXUP_ESPFIX_STACK
+11 -12
arch/x86/entry/entry_64.S
··· 738 738 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE 739 739 swapgs 740 740 FRAME_END 741 - ret 742 - SYM_FUNC_END(asm_load_gs_index) 743 - EXPORT_SYMBOL(asm_load_gs_index) 741 + RET 744 742 745 - _ASM_EXTABLE(.Lgs_change, .Lbad_gs) 746 - .section .fixup, "ax" 747 743 /* running with kernelgs */ 748 - SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) 744 + .Lbad_gs: 749 745 swapgs /* switch back to user gs */ 750 746 .macro ZAP_GS 751 747 /* This can't be a string because the preprocessor needs to see it. */ ··· 752 756 xorl %eax, %eax 753 757 movl %eax, %gs 754 758 jmp 2b 755 - SYM_CODE_END(.Lbad_gs) 756 - .previous 759 + 760 + _ASM_EXTABLE(.Lgs_change, .Lbad_gs) 761 + 762 + SYM_FUNC_END(asm_load_gs_index) 763 + EXPORT_SYMBOL(asm_load_gs_index) 757 764 758 765 #ifdef CONFIG_XEN_PV 759 766 /* ··· 888 889 * is needed here. 889 890 */ 890 891 SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx 891 - ret 892 + RET 892 893 893 894 .Lparanoid_entry_checkgs: 894 895 /* EBX = 1 -> kernel GSBASE active, no restore required */ ··· 909 910 .Lparanoid_kernel_gsbase: 910 911 911 912 FENCE_SWAPGS_KERNEL_ENTRY 912 - ret 913 + RET 913 914 SYM_CODE_END(paranoid_entry) 914 915 915 916 /* ··· 988 989 movq %rax, %rsp /* switch stack */ 989 990 ENCODE_FRAME_POINTER 990 991 pushq %r12 991 - ret 992 + RET 992 993 993 994 /* 994 995 * There are two places in the kernel that can potentially fault with ··· 1019 1020 */ 1020 1021 .Lerror_entry_done_lfence: 1021 1022 FENCE_SWAPGS_KERNEL_ENTRY 1022 - ret 1023 + RET 1023 1024 1024 1025 .Lbstep_iret: 1025 1026 /* Fix truncated RIP */
+1 -1
arch/x86/entry/thunk_32.S
··· 24 24 popl %edx 25 25 popl %ecx 26 26 popl %eax 27 - ret 27 + RET 28 28 _ASM_NOKPROBE(\name) 29 29 SYM_CODE_END(\name) 30 30 .endm
+1 -1
arch/x86/entry/thunk_64.S
··· 50 50 popq %rsi 51 51 popq %rdi 52 52 popq %rbp 53 - ret 53 + RET 54 54 _ASM_NOKPROBE(__thunk_restore) 55 55 SYM_CODE_END(__thunk_restore) 56 56 #endif
-1
arch/x86/entry/vdso/vdso-layout.lds.S
··· 77 77 78 78 .text : { 79 79 *(.text*) 80 - *(.fixup) 81 80 } :text =0x90909090, 82 81 83 82
+1 -1
arch/x86/entry/vdso/vdso32/system_call.S
··· 78 78 popl %ecx 79 79 CFI_RESTORE ecx 80 80 CFI_ADJUST_CFA_OFFSET -4 81 - ret 81 + RET 82 82 CFI_ENDPROC 83 83 84 84 .size __kernel_vsyscall,.-__kernel_vsyscall
+1 -1
arch/x86/entry/vdso/vsgx.S
··· 81 81 pop %rbx 82 82 leave 83 83 .cfi_def_cfa %rsp, 8 84 - ret 84 + RET 85 85 86 86 /* The out-of-line code runs with the pre-leave stack frame. */ 87 87 .cfi_def_cfa %rbp, 16
+3 -3
arch/x86/entry/vsyscall/vsyscall_emu_64.S
··· 19 19 20 20 mov $__NR_gettimeofday, %rax 21 21 syscall 22 - ret 22 + RET 23 23 24 24 .balign 1024, 0xcc 25 25 mov $__NR_time, %rax 26 26 syscall 27 - ret 27 + RET 28 28 29 29 .balign 1024, 0xcc 30 30 mov $__NR_getcpu, %rax 31 31 syscall 32 - ret 32 + RET 33 33 34 34 .balign 4096, 0xcc 35 35
+37
arch/x86/include/asm/asm.h
··· 152 152 153 153 #else /* ! __ASSEMBLY__ */ 154 154 155 + # define DEFINE_EXTABLE_TYPE_REG \ 156 + ".macro extable_type_reg type:req reg:req\n" \ 157 + ".set found, 0\n" \ 158 + ".set regnr, 0\n" \ 159 + ".irp rs,rax,rcx,rdx,rbx,rsp,rbp,rsi,rdi,r8,r9,r10,r11,r12,r13,r14,r15\n" \ 160 + ".ifc \\reg, %%\\rs\n" \ 161 + ".set found, found+1\n" \ 162 + ".long \\type + (regnr << 8)\n" \ 163 + ".endif\n" \ 164 + ".set regnr, regnr+1\n" \ 165 + ".endr\n" \ 166 + ".set regnr, 0\n" \ 167 + ".irp rs,eax,ecx,edx,ebx,esp,ebp,esi,edi,r8d,r9d,r10d,r11d,r12d,r13d,r14d,r15d\n" \ 168 + ".ifc \\reg, %%\\rs\n" \ 169 + ".set found, found+1\n" \ 170 + ".long \\type + (regnr << 8)\n" \ 171 + ".endif\n" \ 172 + ".set regnr, regnr+1\n" \ 173 + ".endr\n" \ 174 + ".if (found != 1)\n" \ 175 + ".error \"extable_type_reg: bad register argument\"\n" \ 176 + ".endif\n" \ 177 + ".endm\n" 178 + 179 + # define UNDEFINE_EXTABLE_TYPE_REG \ 180 + ".purgem extable_type_reg\n" 181 + 155 182 # define _ASM_EXTABLE_TYPE(from, to, type) \ 156 183 " .pushsection \"__ex_table\",\"a\"\n" \ 157 184 " .balign 4\n" \ 158 185 " .long (" #from ") - .\n" \ 159 186 " .long (" #to ") - .\n" \ 160 187 " .long " __stringify(type) " \n" \ 188 + " .popsection\n" 189 + 190 + # define _ASM_EXTABLE_TYPE_REG(from, to, type, reg) \ 191 + " .pushsection \"__ex_table\",\"a\"\n" \ 192 + " .balign 4\n" \ 193 + " .long (" #from ") - .\n" \ 194 + " .long (" #to ") - .\n" \ 195 + DEFINE_EXTABLE_TYPE_REG \ 196 + "extable_type_reg reg=" __stringify(reg) ", type=" __stringify(type) " \n"\ 197 + UNDEFINE_EXTABLE_TYPE_REG \ 161 198 " .popsection\n" 162 199 163 200 /* For C file, we already have NOKPROBE_SYMBOL macro */
+3 -3
arch/x86/include/asm/extable.h
··· 21 21 */ 22 22 23 23 struct exception_table_entry { 24 - int insn, fixup, type; 24 + int insn, fixup, data; 25 25 }; 26 26 struct pt_regs; 27 27 ··· 31 31 do { \ 32 32 (a)->fixup = (b)->fixup + (delta); \ 33 33 (b)->fixup = (tmp).fixup - (delta); \ 34 - (a)->type = (b)->type; \ 35 - (b)->type = (tmp).type; \ 34 + (a)->data = (b)->data; \ 35 + (b)->data = (tmp).data; \ 36 36 } while (0) 37 37 38 38 extern int fixup_exception(struct pt_regs *regs, int trapnr,
+52 -7
arch/x86/include/asm/extable_fixup_types.h
··· 2 2 #ifndef _ASM_X86_EXTABLE_FIXUP_TYPES_H 3 3 #define _ASM_X86_EXTABLE_FIXUP_TYPES_H 4 4 5 + /* 6 + * Our IMM is signed, as such it must live at the top end of the word. Also, 7 + * since C99 hex constants are of ambigious type, force cast the mask to 'int' 8 + * so that FIELD_GET() will DTRT and sign extend the value when it extracts it. 9 + */ 10 + #define EX_DATA_TYPE_MASK ((int)0x000000FF) 11 + #define EX_DATA_REG_MASK ((int)0x00000F00) 12 + #define EX_DATA_FLAG_MASK ((int)0x0000F000) 13 + #define EX_DATA_IMM_MASK ((int)0xFFFF0000) 14 + 15 + #define EX_DATA_REG_SHIFT 8 16 + #define EX_DATA_FLAG_SHIFT 12 17 + #define EX_DATA_IMM_SHIFT 16 18 + 19 + #define EX_DATA_REG(reg) ((reg) << EX_DATA_REG_SHIFT) 20 + #define EX_DATA_FLAG(flag) ((flag) << EX_DATA_FLAG_SHIFT) 21 + #define EX_DATA_IMM(imm) ((imm) << EX_DATA_IMM_SHIFT) 22 + 23 + /* segment regs */ 24 + #define EX_REG_DS EX_DATA_REG(8) 25 + #define EX_REG_ES EX_DATA_REG(9) 26 + #define EX_REG_FS EX_DATA_REG(10) 27 + #define EX_REG_GS EX_DATA_REG(11) 28 + 29 + /* flags */ 30 + #define EX_FLAG_CLEAR_AX EX_DATA_FLAG(1) 31 + #define EX_FLAG_CLEAR_DX EX_DATA_FLAG(2) 32 + #define EX_FLAG_CLEAR_AX_DX EX_DATA_FLAG(3) 33 + 34 + /* types */ 5 35 #define EX_TYPE_NONE 0 6 36 #define EX_TYPE_DEFAULT 1 7 37 #define EX_TYPE_FAULT 2 ··· 39 9 #define EX_TYPE_COPY 4 40 10 #define EX_TYPE_CLEAR_FS 5 41 11 #define EX_TYPE_FPU_RESTORE 6 42 - #define EX_TYPE_WRMSR 7 43 - #define EX_TYPE_RDMSR 8 44 - #define EX_TYPE_BPF 9 12 + #define EX_TYPE_BPF 7 13 + #define EX_TYPE_WRMSR 8 14 + #define EX_TYPE_RDMSR 9 15 + #define EX_TYPE_WRMSR_SAFE 10 /* reg := -EIO */ 16 + #define EX_TYPE_RDMSR_SAFE 11 /* reg := -EIO */ 17 + #define EX_TYPE_WRMSR_IN_MCE 12 18 + #define EX_TYPE_RDMSR_IN_MCE 13 19 + #define EX_TYPE_DEFAULT_MCE_SAFE 14 20 + #define EX_TYPE_FAULT_MCE_SAFE 15 45 21 46 - #define EX_TYPE_WRMSR_IN_MCE 10 47 - #define EX_TYPE_RDMSR_IN_MCE 11 22 + #define EX_TYPE_POP_REG 16 /* sp += sizeof(long) */ 23 + #define EX_TYPE_POP_ZERO (EX_TYPE_POP_REG | EX_DATA_IMM(0)) 48 24 49 - #define EX_TYPE_DEFAULT_MCE_SAFE 12 50 - #define EX_TYPE_FAULT_MCE_SAFE 13 25 + #define EX_TYPE_IMM_REG 17 /* reg := (long)imm */ 26 + #define EX_TYPE_EFAULT_REG (EX_TYPE_IMM_REG | EX_DATA_IMM(-EFAULT)) 27 + #define EX_TYPE_ZERO_REG (EX_TYPE_IMM_REG | EX_DATA_IMM(0)) 28 + #define EX_TYPE_ONE_REG (EX_TYPE_IMM_REG | EX_DATA_IMM(1)) 29 + 30 + #define EX_TYPE_FAULT_SGX 18 31 + 32 + #define EX_TYPE_UCOPY_LEN 19 /* cx := reg + imm*cx */ 33 + #define EX_TYPE_UCOPY_LEN1 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(1)) 34 + #define EX_TYPE_UCOPY_LEN4 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(4)) 35 + #define EX_TYPE_UCOPY_LEN8 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(8)) 51 36 52 37 #endif
+8 -20
arch/x86/include/asm/futex.h
··· 17 17 int oldval = 0, ret; \ 18 18 asm volatile("1:\t" insn "\n" \ 19 19 "2:\n" \ 20 - "\t.section .fixup,\"ax\"\n" \ 21 - "3:\tmov\t%3, %1\n" \ 22 - "\tjmp\t2b\n" \ 23 - "\t.previous\n" \ 24 - _ASM_EXTABLE_UA(1b, 3b) \ 20 + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %1) \ 25 21 : "=r" (oldval), "=r" (ret), "+m" (*uaddr) \ 26 - : "i" (-EFAULT), "0" (oparg), "1" (0)); \ 22 + : "0" (oparg), "1" (0)); \ 27 23 if (ret) \ 28 24 goto label; \ 29 25 *oval = oldval; \ ··· 35 39 "3:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \ 36 40 "\tjnz\t2b\n" \ 37 41 "4:\n" \ 38 - "\t.section .fixup,\"ax\"\n" \ 39 - "5:\tmov\t%5, %1\n" \ 40 - "\tjmp\t4b\n" \ 41 - "\t.previous\n" \ 42 - _ASM_EXTABLE_UA(1b, 5b) \ 43 - _ASM_EXTABLE_UA(3b, 5b) \ 42 + _ASM_EXTABLE_TYPE_REG(1b, 4b, EX_TYPE_EFAULT_REG, %1) \ 43 + _ASM_EXTABLE_TYPE_REG(3b, 4b, EX_TYPE_EFAULT_REG, %1) \ 44 44 : "=&a" (oldval), "=&r" (ret), \ 45 45 "+m" (*uaddr), "=&r" (tem) \ 46 - : "r" (oparg), "i" (-EFAULT), "1" (0)); \ 46 + : "r" (oparg), "1" (0)); \ 47 47 if (ret) \ 48 48 goto label; \ 49 49 *oval = oldval; \ ··· 87 95 if (!user_access_begin(uaddr, sizeof(u32))) 88 96 return -EFAULT; 89 97 asm volatile("\n" 90 - "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" 98 + "1:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" 91 99 "2:\n" 92 - "\t.section .fixup, \"ax\"\n" 93 - "3:\tmov %3, %0\n" 94 - "\tjmp 2b\n" 95 - "\t.previous\n" 96 - _ASM_EXTABLE_UA(1b, 3b) 100 + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %0) \ 97 101 : "+r" (ret), "=a" (oldval), "+m" (*uaddr) 98 - : "i" (-EFAULT), "r" (newval), "1" (oldval) 102 + : "r" (newval), "1" (oldval) 99 103 : "memory" 100 104 ); 101 105 user_access_end();
+2
arch/x86/include/asm/insn-eval.h
··· 15 15 #define INSN_CODE_SEG_OPND_SZ(params) (params & 0xf) 16 16 #define INSN_CODE_SEG_PARAMS(oper_sz, addr_sz) (oper_sz | (addr_sz << 4)) 17 17 18 + int pt_regs_offset(struct pt_regs *regs, int regno); 19 + 18 20 bool insn_has_rep_prefix(struct insn *insn); 19 21 void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs); 20 22 int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs);
+14
arch/x86/include/asm/linkage.h
··· 18 18 #define __ALIGN_STR __stringify(__ALIGN) 19 19 #endif 20 20 21 + #ifdef CONFIG_SLS 22 + #define RET ret; int3 23 + #else 24 + #define RET ret 25 + #endif 26 + 27 + #else /* __ASSEMBLY__ */ 28 + 29 + #ifdef CONFIG_SLS 30 + #define ASM_RET "ret; int3\n\t" 31 + #else 32 + #define ASM_RET "ret\n\t" 33 + #endif 34 + 21 35 #endif /* __ASSEMBLY__ */ 22 36 23 37 #endif /* _ASM_X86_LINKAGE_H */
-15
arch/x86/include/asm/mmx.h
··· 1 - /* SPDX-License-Identifier: GPL-2.0 */ 2 - #ifndef _ASM_X86_MMX_H 3 - #define _ASM_X86_MMX_H 4 - 5 - /* 6 - * MMX 3Dnow! helper operations 7 - */ 8 - 9 - #include <linux/types.h> 10 - 11 - extern void *_mmx_memcpy(void *to, const void *from, size_t size); 12 - extern void mmx_clear_page(void *page); 13 - extern void mmx_copy_page(void *to, void *from); 14 - 15 - #endif /* _ASM_X86_MMX_H */
+8 -18
arch/x86/include/asm/msr.h
··· 137 137 { 138 138 DECLARE_ARGS(val, low, high); 139 139 140 - asm volatile("2: rdmsr ; xor %[err],%[err]\n" 141 - "1:\n\t" 142 - ".section .fixup,\"ax\"\n\t" 143 - "3: mov %[fault],%[err]\n\t" 144 - "xorl %%eax, %%eax\n\t" 145 - "xorl %%edx, %%edx\n\t" 146 - "jmp 1b\n\t" 147 - ".previous\n\t" 148 - _ASM_EXTABLE(2b, 3b) 140 + asm volatile("1: rdmsr ; xor %[err],%[err]\n" 141 + "2:\n\t" 142 + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_RDMSR_SAFE, %[err]) 149 143 : [err] "=r" (*err), EAX_EDX_RET(val, low, high) 150 - : "c" (msr), [fault] "i" (-EIO)); 144 + : "c" (msr)); 151 145 if (tracepoint_enabled(read_msr)) 152 146 do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err); 153 147 return EAX_EDX_VAL(val, low, high); ··· 163 169 { 164 170 int err; 165 171 166 - asm volatile("2: wrmsr ; xor %[err],%[err]\n" 167 - "1:\n\t" 168 - ".section .fixup,\"ax\"\n\t" 169 - "3: mov %[fault],%[err] ; jmp 1b\n\t" 170 - ".previous\n\t" 171 - _ASM_EXTABLE(2b, 3b) 172 + asm volatile("1: wrmsr ; xor %[err],%[err]\n" 173 + "2:\n\t" 174 + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_WRMSR_SAFE, %[err]) 172 175 : [err] "=a" (err) 173 - : "c" (msr), "0" (low), "d" (high), 174 - [fault] "i" (-EIO) 176 + : "c" (msr), "0" (low), "d" (high) 175 177 : "memory"); 176 178 if (tracepoint_enabled(write_msr)) 177 179 do_trace_write_msr(msr, ((u64)high << 32 | low), err);
-14
arch/x86/include/asm/page_32.h
··· 19 19 #define pfn_valid(pfn) ((pfn) < max_mapnr) 20 20 #endif /* CONFIG_FLATMEM */ 21 21 22 - #ifdef CONFIG_X86_USE_3DNOW 23 - #include <asm/mmx.h> 24 - 25 - static inline void clear_page(void *page) 26 - { 27 - mmx_clear_page(page); 28 - } 29 - 30 - static inline void copy_page(void *to, void *from) 31 - { 32 - mmx_copy_page(to, from); 33 - } 34 - #else /* !CONFIG_X86_USE_3DNOW */ 35 22 #include <linux/string.h> 36 23 37 24 static inline void clear_page(void *page) ··· 30 43 { 31 44 memcpy(to, from, PAGE_SIZE); 32 45 } 33 - #endif /* CONFIG_X86_USE_3DNOW */ 34 46 #endif /* !__ASSEMBLY__ */ 35 47 36 48 #endif /* _ASM_X86_PAGE_32_H */
+1 -1
arch/x86/include/asm/paravirt.h
··· 671 671 "call " #func ";" \ 672 672 PV_RESTORE_ALL_CALLER_REGS \ 673 673 FRAME_END \ 674 - "ret;" \ 674 + ASM_RET \ 675 675 ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \ 676 676 ".popsection") 677 677
+2 -2
arch/x86/include/asm/qspinlock_paravirt.h
··· 48 48 "jne .slowpath;" 49 49 "pop %rdx;" 50 50 FRAME_END 51 - "ret;" 51 + ASM_RET 52 52 ".slowpath: " 53 53 "push %rsi;" 54 54 "movzbl %al,%esi;" ··· 56 56 "pop %rsi;" 57 57 "pop %rdx;" 58 58 FRAME_END 59 - "ret;" 59 + ASM_RET 60 60 ".size " PV_UNLOCK ", .-" PV_UNLOCK ";" 61 61 ".popsection"); 62 62
-4
arch/x86/include/asm/required-features.h
··· 35 35 # define NEED_CMOV 0 36 36 #endif 37 37 38 - #ifdef CONFIG_X86_USE_3DNOW 39 - # define NEED_3DNOW (1<<(X86_FEATURE_3DNOW & 31)) 40 - #else 41 38 # define NEED_3DNOW 0 42 - #endif 43 39 44 40 #if defined(CONFIG_X86_P6_NOP) || defined(CONFIG_X86_64) 45 41 # define NEED_NOPL (1<<(X86_FEATURE_NOPL & 31))
+1 -8
arch/x86/include/asm/segment.h
··· 307 307 \ 308 308 asm volatile(" \n" \ 309 309 "1: movl %k0,%%" #seg " \n" \ 310 - \ 311 - ".section .fixup,\"ax\" \n" \ 312 - "2: xorl %k0,%k0 \n" \ 313 - " jmp 1b \n" \ 314 - ".previous \n" \ 315 - \ 316 - _ASM_EXTABLE(1b, 2b) \ 317 - \ 310 + _ASM_EXTABLE_TYPE_REG(1b, 1b, EX_TYPE_ZERO_REG, %k0)\ 318 311 : "+r" (__val) : : "memory"); \ 319 312 } while (0) 320 313
+18
arch/x86/include/asm/sgx.h
··· 46 46 }; 47 47 48 48 /** 49 + * SGX_ENCLS_FAULT_FLAG - flag signifying an ENCLS return code is a trapnr 50 + * 51 + * ENCLS has its own (positive value) error codes and also generates 52 + * ENCLS specific #GP and #PF faults. And the ENCLS values get munged 53 + * with system error codes as everything percolates back up the stack. 54 + * Unfortunately (for us), we need to precisely identify each unique 55 + * error code, e.g. the action taken if EWB fails varies based on the 56 + * type of fault and on the exact SGX error code, i.e. we can't simply 57 + * convert all faults to -EFAULT. 58 + * 59 + * To make all three error types coexist, we set bit 30 to identify an 60 + * ENCLS fault. Bit 31 (technically bits N:31) is used to differentiate 61 + * between positive (faults and SGX error codes) and negative (system 62 + * error codes) values. 63 + */ 64 + #define SGX_ENCLS_FAULT_FLAG 0x40000000 65 + 66 + /** 49 67 * enum sgx_return_code - The return code type for ENCLS, ENCLU and ENCLV 50 68 * %SGX_NOT_TRACKED: Previous ETRACK's shootdown sequence has not 51 69 * been completed yet.
+1 -1
arch/x86/include/asm/static_call.h
··· 36 36 __ARCH_DEFINE_STATIC_CALL_TRAMP(name, ".byte 0xe9; .long " #func " - (. + 4)") 37 37 38 38 #define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) \ 39 - __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; nop; nop; nop; nop") 39 + __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; int3; nop; nop; nop") 40 40 41 41 42 42 #define ARCH_ADD_TRAMP_KEY(name) \
-33
arch/x86/include/asm/string_32.h
··· 146 146 extern void *memcpy(void *, const void *, size_t); 147 147 148 148 #ifndef CONFIG_FORTIFY_SOURCE 149 - #ifdef CONFIG_X86_USE_3DNOW 150 - 151 - #include <asm/mmx.h> 152 - 153 - /* 154 - * This CPU favours 3DNow strongly (eg AMD Athlon) 155 - */ 156 - 157 - static inline void *__constant_memcpy3d(void *to, const void *from, size_t len) 158 - { 159 - if (len < 512) 160 - return __constant_memcpy(to, from, len); 161 - return _mmx_memcpy(to, from, len); 162 - } 163 - 164 - static inline void *__memcpy3d(void *to, const void *from, size_t len) 165 - { 166 - if (len < 512) 167 - return __memcpy(to, from, len); 168 - return _mmx_memcpy(to, from, len); 169 - } 170 - 171 - #define memcpy(t, f, n) \ 172 - (__builtin_constant_p((n)) \ 173 - ? __constant_memcpy3d((t), (f), (n)) \ 174 - : __memcpy3d((t), (f), (n))) 175 - 176 - #else 177 - 178 - /* 179 - * No 3D Now! 180 - */ 181 149 182 150 #define memcpy(t, f, n) __builtin_memcpy(t, f, n) 183 151 184 - #endif 185 152 #endif /* !CONFIG_FORTIFY_SOURCE */ 186 153 187 154 #define __HAVE_ARCH_MEMMOVE
+17 -22
arch/x86/include/asm/uaccess.h
··· 352 352 "1: movl %[lowbits],%%eax\n" \ 353 353 "2: movl %[highbits],%%edx\n" \ 354 354 "3:\n" \ 355 - ".section .fixup,\"ax\"\n" \ 356 - "4: mov %[efault],%[errout]\n" \ 357 - " xorl %%eax,%%eax\n" \ 358 - " xorl %%edx,%%edx\n" \ 359 - " jmp 3b\n" \ 360 - ".previous\n" \ 361 - _ASM_EXTABLE_UA(1b, 4b) \ 362 - _ASM_EXTABLE_UA(2b, 4b) \ 355 + _ASM_EXTABLE_TYPE_REG(1b, 3b, EX_TYPE_EFAULT_REG | \ 356 + EX_FLAG_CLEAR_AX_DX, \ 357 + %[errout]) \ 358 + _ASM_EXTABLE_TYPE_REG(2b, 3b, EX_TYPE_EFAULT_REG | \ 359 + EX_FLAG_CLEAR_AX_DX, \ 360 + %[errout]) \ 363 361 : [errout] "=r" (retval), \ 364 362 [output] "=&A"(x) \ 365 363 : [lowbits] "m" (__m(__ptr)), \ 366 364 [highbits] "m" __m(((u32 __user *)(__ptr)) + 1), \ 367 - [efault] "i" (-EFAULT), "0" (retval)); \ 365 + "0" (retval)); \ 368 366 }) 369 367 370 368 #else 371 369 #define __get_user_asm_u64(x, ptr, retval) \ 372 - __get_user_asm(x, ptr, retval, "q", "=r") 370 + __get_user_asm(x, ptr, retval, "q") 373 371 #endif 374 372 375 373 #define __get_user_size(x, ptr, size, retval) \ ··· 378 380 __chk_user_ptr(ptr); \ 379 381 switch (size) { \ 380 382 case 1: \ 381 - __get_user_asm(x_u8__, ptr, retval, "b", "=q"); \ 383 + __get_user_asm(x_u8__, ptr, retval, "b"); \ 382 384 (x) = x_u8__; \ 383 385 break; \ 384 386 case 2: \ 385 - __get_user_asm(x, ptr, retval, "w", "=r"); \ 387 + __get_user_asm(x, ptr, retval, "w"); \ 386 388 break; \ 387 389 case 4: \ 388 - __get_user_asm(x, ptr, retval, "l", "=r"); \ 390 + __get_user_asm(x, ptr, retval, "l"); \ 389 391 break; \ 390 392 case 8: \ 391 393 __get_user_asm_u64(x, ptr, retval); \ ··· 395 397 } \ 396 398 } while (0) 397 399 398 - #define __get_user_asm(x, addr, err, itype, ltype) \ 400 + #define __get_user_asm(x, addr, err, itype) \ 399 401 asm volatile("\n" \ 400 402 "1: mov"itype" %[umem],%[output]\n" \ 401 403 "2:\n" \ 402 - ".section .fixup,\"ax\"\n" \ 403 - "3: mov %[efault],%[errout]\n" \ 404 - " xorl %k[output],%k[output]\n" \ 405 - " jmp 2b\n" \ 406 - ".previous\n" \ 407 - _ASM_EXTABLE_UA(1b, 3b) \ 404 + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG | \ 405 + EX_FLAG_CLEAR_AX, \ 406 + %[errout]) \ 408 407 : [errout] "=r" (err), \ 409 - [output] ltype(x) \ 408 + [output] "=a" (x) \ 410 409 : [umem] "m" (__m(addr)), \ 411 - [efault] "i" (-EFAULT), "0" (err)) 410 + "0" (err)) 412 411 413 412 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT 414 413
+48 -20
arch/x86/include/asm/word-at-a-time.h
··· 77 77 * and the next page not being mapped, take the exception and 78 78 * return zeroes in the non-existing part. 79 79 */ 80 + #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 81 + 80 82 static inline unsigned long load_unaligned_zeropad(const void *addr) 81 83 { 82 - unsigned long ret, dummy; 84 + unsigned long offset, data; 85 + unsigned long ret; 83 86 84 - asm( 85 - "1:\tmov %2,%0\n" 86 - "2:\n" 87 - ".section .fixup,\"ax\"\n" 88 - "3:\t" 89 - "lea %2,%1\n\t" 90 - "and %3,%1\n\t" 91 - "mov (%1),%0\n\t" 92 - "leal %2,%%ecx\n\t" 93 - "andl %4,%%ecx\n\t" 94 - "shll $3,%%ecx\n\t" 95 - "shr %%cl,%0\n\t" 96 - "jmp 2b\n" 97 - ".previous\n" 98 - _ASM_EXTABLE(1b, 3b) 99 - :"=&r" (ret),"=&c" (dummy) 100 - :"m" (*(unsigned long *)addr), 101 - "i" (-sizeof(unsigned long)), 102 - "i" (sizeof(unsigned long)-1)); 87 + asm_volatile_goto( 88 + "1: mov %[mem], %[ret]\n" 89 + 90 + _ASM_EXTABLE(1b, %l[do_exception]) 91 + 92 + : [ret] "=r" (ret) 93 + : [mem] "m" (*(unsigned long *)addr) 94 + : : do_exception); 95 + 96 + return ret; 97 + 98 + do_exception: 99 + offset = (unsigned long)addr & (sizeof(long) - 1); 100 + addr = (void *)((unsigned long)addr & ~(sizeof(long) - 1)); 101 + data = *(unsigned long *)addr; 102 + ret = data >> offset * 8; 103 + 103 104 return ret; 104 105 } 106 + 107 + #else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 108 + 109 + static inline unsigned long load_unaligned_zeropad(const void *addr) 110 + { 111 + unsigned long offset, data; 112 + unsigned long ret, err = 0; 113 + 114 + asm( "1: mov %[mem], %[ret]\n" 115 + "2:\n" 116 + 117 + _ASM_EXTABLE_FAULT(1b, 2b) 118 + 119 + : [ret] "=&r" (ret), "+a" (err) 120 + : [mem] "m" (*(unsigned long *)addr)); 121 + 122 + if (unlikely(err)) { 123 + offset = (unsigned long)addr & (sizeof(long) - 1); 124 + addr = (void *)((unsigned long)addr & ~(sizeof(long) - 1)); 125 + data = *(unsigned long *)addr; 126 + ret = data >> offset * 8; 127 + } 128 + 129 + return ret; 130 + } 131 + 132 + #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 105 133 106 134 #endif /* _ASM_WORD_AT_A_TIME_H */
+3 -11
arch/x86/include/asm/xen/page.h
··· 96 96 97 97 asm volatile("1: mov %[val], %[ptr]\n" 98 98 "2:\n" 99 - ".section .fixup, \"ax\"\n" 100 - "3: sub $1, %[ret]\n" 101 - " jmp 2b\n" 102 - ".previous\n" 103 - _ASM_EXTABLE(1b, 3b) 99 + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %[ret]) 104 100 : [ret] "+r" (ret), [ptr] "=m" (*addr) 105 101 : [val] "r" (val)); 106 102 ··· 106 110 static inline int xen_safe_read_ulong(const unsigned long *addr, 107 111 unsigned long *val) 108 112 { 109 - int ret = 0; 110 113 unsigned long rval = ~0ul; 114 + int ret = 0; 111 115 112 116 asm volatile("1: mov %[ptr], %[rval]\n" 113 117 "2:\n" 114 - ".section .fixup, \"ax\"\n" 115 - "3: sub $1, %[ret]\n" 116 - " jmp 2b\n" 117 - ".previous\n" 118 - _ASM_EXTABLE(1b, 3b) 118 + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %[ret]) 119 119 : [ret] "+r" (ret), [rval] "+r" (rval) 120 120 : [ptr] "m" (*addr)); 121 121 *val = rval;
+3 -3
arch/x86/kernel/acpi/wakeup_32.S
··· 60 60 popl saved_context_eflags 61 61 62 62 movl $ret_point, saved_eip 63 - ret 63 + RET 64 64 65 65 66 66 restore_registers: ··· 70 70 movl saved_context_edi, %edi 71 71 pushl saved_context_eflags 72 72 popfl 73 - ret 73 + RET 74 74 75 75 SYM_CODE_START(do_suspend_lowlevel) 76 76 call save_processor_state ··· 86 86 ret_point: 87 87 call restore_registers 88 88 call restore_processor_state 89 - ret 89 + RET 90 90 SYM_CODE_END(do_suspend_lowlevel) 91 91 92 92 .data
+35 -16
arch/x86/kernel/alternative.c
··· 714 714 " .type int3_magic, @function\n" 715 715 "int3_magic:\n" 716 716 " movl $1, (%" _ASM_ARG1 ")\n" 717 - " ret\n" 717 + ASM_RET 718 718 " .size int3_magic, .-int3_magic\n" 719 719 " .popsection\n" 720 720 ); ··· 1113 1113 } 1114 1114 1115 1115 struct text_poke_loc { 1116 - s32 rel_addr; /* addr := _stext + rel_addr */ 1117 - s32 rel32; 1116 + /* addr := _stext + rel_addr */ 1117 + s32 rel_addr; 1118 + s32 disp; 1119 + u8 len; 1118 1120 u8 opcode; 1119 1121 const u8 text[POKE_MAX_OPCODE_SIZE]; 1122 + /* see text_poke_bp_batch() */ 1120 1123 u8 old; 1121 1124 }; 1122 1125 ··· 1134 1131 static __always_inline 1135 1132 struct bp_patching_desc *try_get_desc(struct bp_patching_desc **descp) 1136 1133 { 1137 - struct bp_patching_desc *desc = __READ_ONCE(*descp); /* rcu_dereference */ 1134 + /* rcu_dereference */ 1135 + struct bp_patching_desc *desc = __READ_ONCE(*descp); 1138 1136 1139 1137 if (!desc || !arch_atomic_inc_not_zero(&desc->refs)) 1140 1138 return NULL; ··· 1169 1165 { 1170 1166 struct bp_patching_desc *desc; 1171 1167 struct text_poke_loc *tp; 1172 - int len, ret = 0; 1168 + int ret = 0; 1173 1169 void *ip; 1174 1170 1175 1171 if (user_mode(regs)) ··· 1209 1205 goto out_put; 1210 1206 } 1211 1207 1212 - len = text_opcode_size(tp->opcode); 1213 - ip += len; 1208 + ip += tp->len; 1214 1209 1215 1210 switch (tp->opcode) { 1216 1211 case INT3_INSN_OPCODE: ··· 1224 1221 break; 1225 1222 1226 1223 case CALL_INSN_OPCODE: 1227 - int3_emulate_call(regs, (long)ip + tp->rel32); 1224 + int3_emulate_call(regs, (long)ip + tp->disp); 1228 1225 break; 1229 1226 1230 1227 case JMP32_INSN_OPCODE: 1231 1228 case JMP8_INSN_OPCODE: 1232 - int3_emulate_jmp(regs, (long)ip + tp->rel32); 1229 + int3_emulate_jmp(regs, (long)ip + tp->disp); 1233 1230 break; 1234 1231 1235 1232 default: ··· 1304 1301 */ 1305 1302 for (do_sync = 0, i = 0; i < nr_entries; i++) { 1306 1303 u8 old[POKE_MAX_OPCODE_SIZE] = { tp[i].old, }; 1307 - int len = text_opcode_size(tp[i].opcode); 1304 + int len = tp[i].len; 1308 1305 1309 1306 if (len - INT3_INSN_SIZE > 0) { 1310 1307 memcpy(old + INT3_INSN_SIZE, ··· 1381 1378 const void *opcode, size_t len, const void *emulate) 1382 1379 { 1383 1380 struct insn insn; 1384 - int ret; 1381 + int ret, i; 1385 1382 1386 1383 memcpy((void *)tp->text, opcode, len); 1387 1384 if (!emulate) 1388 1385 emulate = opcode; 1389 1386 1390 1387 ret = insn_decode_kernel(&insn, emulate); 1391 - 1392 1388 BUG_ON(ret < 0); 1393 - BUG_ON(len != insn.length); 1394 1389 1395 1390 tp->rel_addr = addr - (void *)_stext; 1391 + tp->len = len; 1396 1392 tp->opcode = insn.opcode.bytes[0]; 1393 + 1394 + switch (tp->opcode) { 1395 + case RET_INSN_OPCODE: 1396 + case JMP32_INSN_OPCODE: 1397 + case JMP8_INSN_OPCODE: 1398 + /* 1399 + * Control flow instructions without implied execution of the 1400 + * next instruction can be padded with INT3. 1401 + */ 1402 + for (i = insn.length; i < len; i++) 1403 + BUG_ON(tp->text[i] != INT3_INSN_OPCODE); 1404 + break; 1405 + 1406 + default: 1407 + BUG_ON(len != insn.length); 1408 + }; 1409 + 1397 1410 1398 1411 switch (tp->opcode) { 1399 1412 case INT3_INSN_OPCODE: ··· 1419 1400 case CALL_INSN_OPCODE: 1420 1401 case JMP32_INSN_OPCODE: 1421 1402 case JMP8_INSN_OPCODE: 1422 - tp->rel32 = insn.immediate.value; 1403 + tp->disp = insn.immediate.value; 1423 1404 break; 1424 1405 1425 1406 default: /* assume NOP */ ··· 1427 1408 case 2: /* NOP2 -- emulate as JMP8+0 */ 1428 1409 BUG_ON(memcmp(emulate, x86_nops[len], len)); 1429 1410 tp->opcode = JMP8_INSN_OPCODE; 1430 - tp->rel32 = 0; 1411 + tp->disp = 0; 1431 1412 break; 1432 1413 1433 1414 case 5: /* NOP5 -- emulate as JMP32+0 */ 1434 1415 BUG_ON(memcmp(emulate, x86_nops[len], len)); 1435 1416 tp->opcode = JMP32_INSN_OPCODE; 1436 - tp->rel32 = 0; 1417 + tp->disp = 0; 1437 1418 break; 1438 1419 1439 1420 default: /* unknown instruction */
+5 -31
arch/x86/kernel/cpu/sgx/encls.h
··· 11 11 #include <asm/traps.h> 12 12 #include "sgx.h" 13 13 14 - /** 15 - * ENCLS_FAULT_FLAG - flag signifying an ENCLS return code is a trapnr 16 - * 17 - * ENCLS has its own (positive value) error codes and also generates 18 - * ENCLS specific #GP and #PF faults. And the ENCLS values get munged 19 - * with system error codes as everything percolates back up the stack. 20 - * Unfortunately (for us), we need to precisely identify each unique 21 - * error code, e.g. the action taken if EWB fails varies based on the 22 - * type of fault and on the exact SGX error code, i.e. we can't simply 23 - * convert all faults to -EFAULT. 24 - * 25 - * To make all three error types coexist, we set bit 30 to identify an 26 - * ENCLS fault. Bit 31 (technically bits N:31) is used to differentiate 27 - * between positive (faults and SGX error codes) and negative (system 28 - * error codes) values. 29 - */ 30 - #define ENCLS_FAULT_FLAG 0x40000000 31 - 32 14 /* Retrieve the encoded trapnr from the specified return code. */ 33 - #define ENCLS_TRAPNR(r) ((r) & ~ENCLS_FAULT_FLAG) 15 + #define ENCLS_TRAPNR(r) ((r) & ~SGX_ENCLS_FAULT_FLAG) 34 16 35 17 /* Issue a WARN() about an ENCLS function. */ 36 18 #define ENCLS_WARN(r, name) { \ ··· 32 50 */ 33 51 static inline bool encls_faulted(int ret) 34 52 { 35 - return ret & ENCLS_FAULT_FLAG; 53 + return ret & SGX_ENCLS_FAULT_FLAG; 36 54 } 37 55 38 56 /** ··· 70 88 asm volatile( \ 71 89 "1: .byte 0x0f, 0x01, 0xcf;\n\t" \ 72 90 "2:\n" \ 73 - ".section .fixup,\"ax\"\n" \ 74 - "3: orl $"__stringify(ENCLS_FAULT_FLAG)",%%eax\n" \ 75 - " jmp 2b\n" \ 76 - ".previous\n" \ 77 - _ASM_EXTABLE_FAULT(1b, 3b) \ 91 + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_SGX) \ 78 92 : "=a"(ret) \ 79 93 : "a"(rax), inputs \ 80 94 : "memory", "cc"); \ ··· 105 127 * 106 128 * Return: 107 129 * 0 on success, 108 - * trapnr with ENCLS_FAULT_FLAG set on fault 130 + * trapnr with SGX_ENCLS_FAULT_FLAG set on fault 109 131 */ 110 132 #define __encls_N(rax, rbx_out, inputs...) \ 111 133 ({ \ ··· 114 136 "1: .byte 0x0f, 0x01, 0xcf;\n\t" \ 115 137 " xor %%eax,%%eax;\n" \ 116 138 "2:\n" \ 117 - ".section .fixup,\"ax\"\n" \ 118 - "3: orl $"__stringify(ENCLS_FAULT_FLAG)",%%eax\n" \ 119 - " jmp 2b\n" \ 120 - ".previous\n" \ 121 - _ASM_EXTABLE_FAULT(1b, 3b) \ 139 + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_SGX) \ 122 140 : "=a"(ret), "=b"(rbx_out) \ 123 141 : "a"(rax), inputs \ 124 142 : "memory"); \
+1 -5
arch/x86/kernel/fpu/legacy.h
··· 35 35 int err; \ 36 36 asm volatile("1:" #insn "\n\t" \ 37 37 "2:\n" \ 38 - ".section .fixup,\"ax\"\n" \ 39 - "3: movl $-1,%[err]\n" \ 40 - " jmp 2b\n" \ 41 - ".previous\n" \ 42 - _ASM_EXTABLE(1b, 3b) \ 38 + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %[err]) \ 43 39 : [err] "=r" (err), output \ 44 40 : "0"(0), input); \ 45 41 err; \
+1 -5
arch/x86/kernel/fpu/xstate.h
··· 108 108 "\n" \ 109 109 "xor %[err], %[err]\n" \ 110 110 "3:\n" \ 111 - ".pushsection .fixup,\"ax\"\n" \ 112 - "4: movl $-2, %[err]\n" \ 113 - "jmp 3b\n" \ 114 - ".popsection\n" \ 115 - _ASM_EXTABLE(661b, 4b) \ 111 + _ASM_EXTABLE_TYPE_REG(661b, 3b, EX_TYPE_EFAULT_REG, %[err]) \ 116 112 : [err] "=r" (err) \ 117 113 : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ 118 114 : "memory")
+1 -1
arch/x86/kernel/ftrace.c
··· 303 303 } __attribute__((packed)); 304 304 }; 305 305 306 - #define RET_SIZE 1 306 + #define RET_SIZE 1 + IS_ENABLED(CONFIG_SLS) 307 307 308 308 static unsigned long 309 309 create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
+3 -3
arch/x86/kernel/ftrace_32.S
··· 19 19 #endif 20 20 21 21 SYM_FUNC_START(__fentry__) 22 - ret 22 + RET 23 23 SYM_FUNC_END(__fentry__) 24 24 EXPORT_SYMBOL(__fentry__) 25 25 ··· 84 84 85 85 /* This is weak to keep gas from relaxing the jumps */ 86 86 SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) 87 - ret 87 + RET 88 88 SYM_CODE_END(ftrace_caller) 89 89 90 90 SYM_CODE_START(ftrace_regs_caller) ··· 177 177 popl %edx 178 178 popl %ecx 179 179 popl %eax 180 - ret 180 + RET 181 181 SYM_CODE_END(ftrace_graph_caller) 182 182 183 183 .globl return_to_handler
+4 -4
arch/x86/kernel/ftrace_64.S
··· 132 132 #ifdef CONFIG_DYNAMIC_FTRACE 133 133 134 134 SYM_FUNC_START(__fentry__) 135 - retq 135 + RET 136 136 SYM_FUNC_END(__fentry__) 137 137 EXPORT_SYMBOL(__fentry__) 138 138 ··· 176 176 SYM_FUNC_START(ftrace_epilogue) 177 177 /* 178 178 * This is weak to keep gas from relaxing the jumps. 179 - * It is also used to copy the retq for trampolines. 179 + * It is also used to copy the RET for trampolines. 180 180 */ 181 181 SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) 182 182 UNWIND_HINT_FUNC 183 - retq 183 + RET 184 184 SYM_FUNC_END(ftrace_epilogue) 185 185 186 186 SYM_FUNC_START(ftrace_regs_caller) ··· 284 284 jnz trace 285 285 286 286 SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL) 287 - retq 287 + RET 288 288 289 289 trace: 290 290 /* save_mcount_regs fills in first two parameters */
+1 -1
arch/x86/kernel/head_32.S
··· 340 340 __INIT 341 341 setup_once: 342 342 andl $0,setup_once_ref /* Once is enough, thanks */ 343 - ret 343 + RET 344 344 345 345 SYM_FUNC_START(early_idt_handler_array) 346 346 # 36(%esp) %eflags
+1 -1
arch/x86/kernel/irqflags.S
··· 11 11 SYM_FUNC_START(native_save_fl) 12 12 pushf 13 13 pop %_ASM_AX 14 - ret 14 + RET 15 15 SYM_FUNC_END(native_save_fl) 16 16 .popsection 17 17 EXPORT_SYMBOL(native_save_fl)
+1 -1
arch/x86/kernel/kprobes/core.c
··· 1051 1051 " addl $4, %esp\n" 1052 1052 " popfl\n" 1053 1053 #endif 1054 - " ret\n" 1054 + ASM_RET 1055 1055 ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n" 1056 1056 ); 1057 1057 NOKPROBE_SYMBOL(__kretprobe_trampoline);
+2 -2
arch/x86/kernel/paravirt.c
··· 41 41 asm (".pushsection .entry.text, \"ax\"\n" 42 42 ".global _paravirt_nop\n" 43 43 "_paravirt_nop:\n\t" 44 - "ret\n\t" 44 + ASM_RET 45 45 ".size _paravirt_nop, . - _paravirt_nop\n\t" 46 46 ".type _paravirt_nop, @function\n\t" 47 47 ".popsection"); ··· 51 51 ".global paravirt_ret0\n" 52 52 "paravirt_ret0:\n\t" 53 53 "xor %" _ASM_AX ", %" _ASM_AX ";\n\t" 54 - "ret\n\t" 54 + ASM_RET 55 55 ".size paravirt_ret0, . - paravirt_ret0\n\t" 56 56 ".type paravirt_ret0, @function\n\t" 57 57 ".popsection");
+5 -5
arch/x86/kernel/relocate_kernel_32.S
··· 91 91 movl %edi, %eax 92 92 addl $(identity_mapped - relocate_kernel), %eax 93 93 pushl %eax 94 - ret 94 + RET 95 95 SYM_CODE_END(relocate_kernel) 96 96 97 97 SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) ··· 159 159 xorl %edx, %edx 160 160 xorl %esi, %esi 161 161 xorl %ebp, %ebp 162 - ret 162 + RET 163 163 1: 164 164 popl %edx 165 165 movl CP_PA_SWAP_PAGE(%edi), %esp ··· 190 190 movl %edi, %eax 191 191 addl $(virtual_mapped - relocate_kernel), %eax 192 192 pushl %eax 193 - ret 193 + RET 194 194 SYM_CODE_END(identity_mapped) 195 195 196 196 SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) ··· 208 208 popl %edi 209 209 popl %esi 210 210 popl %ebx 211 - ret 211 + RET 212 212 SYM_CODE_END(virtual_mapped) 213 213 214 214 /* Do the copies */ ··· 271 271 popl %edi 272 272 popl %ebx 273 273 popl %ebp 274 - ret 274 + RET 275 275 SYM_CODE_END(swap_pages) 276 276 277 277 .globl kexec_control_code_size
+5 -5
arch/x86/kernel/relocate_kernel_64.S
··· 104 104 /* jump to identity mapped page */ 105 105 addq $(identity_mapped - relocate_kernel), %r8 106 106 pushq %r8 107 - ret 107 + RET 108 108 SYM_CODE_END(relocate_kernel) 109 109 110 110 SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) ··· 191 191 xorl %r14d, %r14d 192 192 xorl %r15d, %r15d 193 193 194 - ret 194 + RET 195 195 196 196 1: 197 197 popq %rdx ··· 210 210 call swap_pages 211 211 movq $virtual_mapped, %rax 212 212 pushq %rax 213 - ret 213 + RET 214 214 SYM_CODE_END(identity_mapped) 215 215 216 216 SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) ··· 231 231 popq %r12 232 232 popq %rbp 233 233 popq %rbx 234 - ret 234 + RET 235 235 SYM_CODE_END(virtual_mapped) 236 236 237 237 /* Do the copies */ ··· 288 288 lea PAGE_SIZE(%rax), %rsi 289 289 jmp 0b 290 290 3: 291 - ret 291 + RET 292 292 SYM_CODE_END(swap_pages) 293 293 294 294 .globl kexec_control_code_size
+1 -1
arch/x86/kernel/sev_verify_cbit.S
··· 85 85 #endif 86 86 /* Return page-table pointer */ 87 87 movq %rdi, %rax 88 - ret 88 + RET 89 89 SYM_FUNC_END(sev_verify_cbit)
+3 -2
arch/x86/kernel/static_call.c
··· 17 17 */ 18 18 static const u8 xor5rax[] = { 0x66, 0x66, 0x48, 0x31, 0xc0 }; 19 19 20 + static const u8 retinsn[] = { RET_INSN_OPCODE, 0xcc, 0xcc, 0xcc, 0xcc }; 21 + 20 22 static void __ref __static_call_transform(void *insn, enum insn_type type, void *func) 21 23 { 22 24 const void *emulate = NULL; ··· 44 42 break; 45 43 46 44 case RET: 47 - code = text_gen_insn(RET_INSN_OPCODE, insn, func); 48 - size = RET_INSN_SIZE; 45 + code = &retinsn; 49 46 break; 50 47 } 51 48
+2 -2
arch/x86/kernel/verify_cpu.S
··· 132 132 .Lverify_cpu_no_longmode: 133 133 popf # Restore caller passed flags 134 134 movl $1,%eax 135 - ret 135 + RET 136 136 .Lverify_cpu_sse_ok: 137 137 popf # Restore caller passed flags 138 138 xorl %eax, %eax 139 - ret 139 + RET 140 140 SYM_FUNC_END(verify_cpu)
-1
arch/x86/kernel/vmlinux.lds.S
··· 137 137 ALIGN_ENTRY_TEXT_END 138 138 SOFTIRQENTRY_TEXT 139 139 STATIC_CALL_TEXT 140 - *(.fixup) 141 140 *(.gnu.warning) 142 141 143 142 #ifdef CONFIG_RETPOLINE
+4 -12
arch/x86/kvm/emulate.c
··· 315 315 __FOP_FUNC(#name) 316 316 317 317 #define __FOP_RET(name) \ 318 - "ret \n\t" \ 318 + "11: " ASM_RET \ 319 319 ".size " name ", .-" name "\n\t" 320 320 321 321 #define FOP_RET(name) \ ··· 344 344 __FOP_RET(#op "_" #dst) 345 345 346 346 #define FOP1EEX(op, dst) \ 347 - FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception) 347 + FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi) 348 348 349 349 #define FASTOP1(op) \ 350 350 FOP_START(op) \ ··· 434 434 #op " %al \n\t" \ 435 435 __FOP_RET(#op) 436 436 437 - asm(".pushsection .fixup, \"ax\"\n" 438 - "kvm_fastop_exception: xor %esi, %esi; ret\n" 439 - ".popsection"); 440 - 441 437 FOP_START(setcc) 442 438 FOP_SETCC(seto) 443 439 FOP_SETCC(setno) ··· 469 473 \ 470 474 asm volatile("1:" insn "\n" \ 471 475 "2:\n" \ 472 - ".pushsection .fixup, \"ax\"\n" \ 473 - "3: movl $1, %[_fault]\n" \ 474 - " jmp 2b\n" \ 475 - ".popsection\n" \ 476 - _ASM_EXTABLE(1b, 3b) \ 477 - : [_fault] "+qm"(_fault) inoutclob ); \ 476 + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %[_fault]) \ 477 + : [_fault] "+r"(_fault) inoutclob ); \ 478 478 \ 479 479 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \ 480 480 })
+2 -2
arch/x86/kvm/svm/vmenter.S
··· 148 148 pop %edi 149 149 #endif 150 150 pop %_ASM_BP 151 - ret 151 + RET 152 152 153 153 3: cmpb $0, kvm_rebooting 154 154 jne 2b ··· 202 202 pop %edi 203 203 #endif 204 204 pop %_ASM_BP 205 - ret 205 + RET 206 206 207 207 3: cmpb $0, kvm_rebooting 208 208 jne 2b
+7 -7
arch/x86/kvm/vmx/vmenter.S
··· 49 49 je 2f 50 50 51 51 1: vmresume 52 - ret 52 + RET 53 53 54 54 2: vmlaunch 55 - ret 55 + RET 56 56 57 57 3: cmpb $0, kvm_rebooting 58 58 je 4f 59 - ret 59 + RET 60 60 4: ud2 61 61 62 62 _ASM_EXTABLE(1b, 3b) ··· 89 89 pop %_ASM_AX 90 90 .Lvmexit_skip_rsb: 91 91 #endif 92 - ret 92 + RET 93 93 SYM_FUNC_END(vmx_vmexit) 94 94 95 95 /** ··· 228 228 pop %edi 229 229 #endif 230 230 pop %_ASM_BP 231 - ret 231 + RET 232 232 233 233 /* VM-Fail. Out-of-line to avoid a taken Jcc after VM-Exit. */ 234 234 2: mov $1, %eax ··· 293 293 pop %_ASM_AX 294 294 pop %_ASM_BP 295 295 296 - ret 296 + RET 297 297 SYM_FUNC_END(vmread_error_trampoline) 298 298 299 299 SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff) ··· 326 326 */ 327 327 mov %_ASM_BP, %_ASM_SP 328 328 pop %_ASM_BP 329 - ret 329 + RET 330 330 SYM_FUNC_END(vmx_do_interrupt_nmi_irqoff)
+7 -9
arch/x86/kvm/vmx/vmx_ops.h
··· 80 80 * @field, and bounce through the trampoline to preserve 81 81 * volatile registers. 82 82 */ 83 - "push $0\n\t" 83 + "xorl %k1, %k1\n\t" 84 + "2:\n\t" 85 + "push %1\n\t" 84 86 "push %2\n\t" 85 - "2:call vmread_error_trampoline\n\t" 87 + "call vmread_error_trampoline\n\t" 86 88 87 89 /* 88 90 * Unwind the stack. Note, the trampoline zeros out the ··· 95 93 "3:\n\t" 96 94 97 95 /* VMREAD faulted. As above, except push '1' for @fault. */ 98 - ".pushsection .fixup, \"ax\"\n\t" 99 - "4: push $1\n\t" 100 - "push %2\n\t" 101 - "jmp 2b\n\t" 102 - ".popsection\n\t" 103 - _ASM_EXTABLE(1b, 4b) 104 - : ASM_CALL_CONSTRAINT, "=r"(value) : "r"(field) : "cc"); 96 + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %1) 97 + 98 + : ASM_CALL_CONSTRAINT, "=&r"(value) : "r"(field) : "cc"); 105 99 return value; 106 100 } 107 101
-1
arch/x86/lib/Makefile
··· 63 63 ifneq ($(CONFIG_X86_CMPXCHG64),y) 64 64 lib-y += cmpxchg8b_emu.o atomic64_386_32.o 65 65 endif 66 - lib-$(CONFIG_X86_USE_3DNOW) += mmx_32.o 67 66 else 68 67 obj-y += iomap_copy_64.o 69 68 lib-y += csum-partial_64.o csum-copy_64.o csum-wrappers_64.o
+47 -39
arch/x86/lib/atomic64_386_32.S
··· 9 9 #include <asm/alternative.h> 10 10 11 11 /* if you want SMP support, implement these with real spinlocks */ 12 - .macro LOCK reg 12 + .macro IRQ_SAVE reg 13 13 pushfl 14 14 cli 15 15 .endm 16 16 17 - .macro UNLOCK reg 17 + .macro IRQ_RESTORE reg 18 18 popfl 19 19 .endm 20 20 21 - #define BEGIN(op) \ 21 + #define BEGIN_IRQ_SAVE(op) \ 22 22 .macro endp; \ 23 23 SYM_FUNC_END(atomic64_##op##_386); \ 24 24 .purgem endp; \ 25 25 .endm; \ 26 26 SYM_FUNC_START(atomic64_##op##_386); \ 27 - LOCK v; 27 + IRQ_SAVE v; 28 28 29 29 #define ENDP endp 30 30 31 - #define RET \ 32 - UNLOCK v; \ 33 - ret 34 - 35 - #define RET_ENDP \ 36 - RET; \ 37 - ENDP 31 + #define RET_IRQ_RESTORE \ 32 + IRQ_RESTORE v; \ 33 + RET 38 34 39 35 #define v %ecx 40 - BEGIN(read) 36 + BEGIN_IRQ_SAVE(read) 41 37 movl (v), %eax 42 38 movl 4(v), %edx 43 - RET_ENDP 39 + RET_IRQ_RESTORE 40 + ENDP 44 41 #undef v 45 42 46 43 #define v %esi 47 - BEGIN(set) 44 + BEGIN_IRQ_SAVE(set) 48 45 movl %ebx, (v) 49 46 movl %ecx, 4(v) 50 - RET_ENDP 47 + RET_IRQ_RESTORE 48 + ENDP 51 49 #undef v 52 50 53 51 #define v %esi 54 - BEGIN(xchg) 52 + BEGIN_IRQ_SAVE(xchg) 55 53 movl (v), %eax 56 54 movl 4(v), %edx 57 55 movl %ebx, (v) 58 56 movl %ecx, 4(v) 59 - RET_ENDP 57 + RET_IRQ_RESTORE 58 + ENDP 60 59 #undef v 61 60 62 61 #define v %ecx 63 - BEGIN(add) 62 + BEGIN_IRQ_SAVE(add) 64 63 addl %eax, (v) 65 64 adcl %edx, 4(v) 66 - RET_ENDP 65 + RET_IRQ_RESTORE 66 + ENDP 67 67 #undef v 68 68 69 69 #define v %ecx 70 - BEGIN(add_return) 70 + BEGIN_IRQ_SAVE(add_return) 71 71 addl (v), %eax 72 72 adcl 4(v), %edx 73 73 movl %eax, (v) 74 74 movl %edx, 4(v) 75 - RET_ENDP 75 + RET_IRQ_RESTORE 76 + ENDP 76 77 #undef v 77 78 78 79 #define v %ecx 79 - BEGIN(sub) 80 + BEGIN_IRQ_SAVE(sub) 80 81 subl %eax, (v) 81 82 sbbl %edx, 4(v) 82 - RET_ENDP 83 + RET_IRQ_RESTORE 84 + ENDP 83 85 #undef v 84 86 85 87 #define v %ecx 86 - BEGIN(sub_return) 88 + BEGIN_IRQ_SAVE(sub_return) 87 89 negl %edx 88 90 negl %eax 89 91 sbbl $0, %edx ··· 93 91 adcl 4(v), %edx 94 92 movl %eax, (v) 95 93 movl %edx, 4(v) 96 - RET_ENDP 94 + RET_IRQ_RESTORE 95 + ENDP 97 96 #undef v 98 97 99 98 #define v %esi 100 - BEGIN(inc) 99 + BEGIN_IRQ_SAVE(inc) 101 100 addl $1, (v) 102 101 adcl $0, 4(v) 103 - RET_ENDP 102 + RET_IRQ_RESTORE 103 + ENDP 104 104 #undef v 105 105 106 106 #define v %esi 107 - BEGIN(inc_return) 107 + BEGIN_IRQ_SAVE(inc_return) 108 108 movl (v), %eax 109 109 movl 4(v), %edx 110 110 addl $1, %eax 111 111 adcl $0, %edx 112 112 movl %eax, (v) 113 113 movl %edx, 4(v) 114 - RET_ENDP 114 + RET_IRQ_RESTORE 115 + ENDP 115 116 #undef v 116 117 117 118 #define v %esi 118 - BEGIN(dec) 119 + BEGIN_IRQ_SAVE(dec) 119 120 subl $1, (v) 120 121 sbbl $0, 4(v) 121 - RET_ENDP 122 + RET_IRQ_RESTORE 123 + ENDP 122 124 #undef v 123 125 124 126 #define v %esi 125 - BEGIN(dec_return) 127 + BEGIN_IRQ_SAVE(dec_return) 126 128 movl (v), %eax 127 129 movl 4(v), %edx 128 130 subl $1, %eax 129 131 sbbl $0, %edx 130 132 movl %eax, (v) 131 133 movl %edx, 4(v) 132 - RET_ENDP 134 + RET_IRQ_RESTORE 135 + ENDP 133 136 #undef v 134 137 135 138 #define v %esi 136 - BEGIN(add_unless) 139 + BEGIN_IRQ_SAVE(add_unless) 137 140 addl %eax, %ecx 138 141 adcl %edx, %edi 139 142 addl (v), %eax ··· 150 143 movl %edx, 4(v) 151 144 movl $1, %eax 152 145 2: 153 - RET 146 + RET_IRQ_RESTORE 154 147 3: 155 148 cmpl %edx, %edi 156 149 jne 1b ··· 160 153 #undef v 161 154 162 155 #define v %esi 163 - BEGIN(inc_not_zero) 156 + BEGIN_IRQ_SAVE(inc_not_zero) 164 157 movl (v), %eax 165 158 movl 4(v), %edx 166 159 testl %eax, %eax ··· 172 165 movl %edx, 4(v) 173 166 movl $1, %eax 174 167 2: 175 - RET 168 + RET_IRQ_RESTORE 176 169 3: 177 170 testl %edx, %edx 178 171 jne 1b ··· 181 174 #undef v 182 175 183 176 #define v %esi 184 - BEGIN(dec_if_positive) 177 + BEGIN_IRQ_SAVE(dec_if_positive) 185 178 movl (v), %eax 186 179 movl 4(v), %edx 187 180 subl $1, %eax ··· 190 183 movl %eax, (v) 191 184 movl %edx, 4(v) 192 185 1: 193 - RET_ENDP 186 + RET_IRQ_RESTORE 187 + ENDP 194 188 #undef v
+8 -8
arch/x86/lib/atomic64_cx8_32.S
··· 18 18 19 19 SYM_FUNC_START(atomic64_read_cx8) 20 20 read64 %ecx 21 - ret 21 + RET 22 22 SYM_FUNC_END(atomic64_read_cx8) 23 23 24 24 SYM_FUNC_START(atomic64_set_cx8) ··· 28 28 cmpxchg8b (%esi) 29 29 jne 1b 30 30 31 - ret 31 + RET 32 32 SYM_FUNC_END(atomic64_set_cx8) 33 33 34 34 SYM_FUNC_START(atomic64_xchg_cx8) ··· 37 37 cmpxchg8b (%esi) 38 38 jne 1b 39 39 40 - ret 40 + RET 41 41 SYM_FUNC_END(atomic64_xchg_cx8) 42 42 43 43 .macro addsub_return func ins insc ··· 68 68 popl %esi 69 69 popl %ebx 70 70 popl %ebp 71 - ret 71 + RET 72 72 SYM_FUNC_END(atomic64_\func\()_return_cx8) 73 73 .endm 74 74 ··· 93 93 movl %ebx, %eax 94 94 movl %ecx, %edx 95 95 popl %ebx 96 - ret 96 + RET 97 97 SYM_FUNC_END(atomic64_\func\()_return_cx8) 98 98 .endm 99 99 ··· 118 118 movl %ebx, %eax 119 119 movl %ecx, %edx 120 120 popl %ebx 121 - ret 121 + RET 122 122 SYM_FUNC_END(atomic64_dec_if_positive_cx8) 123 123 124 124 SYM_FUNC_START(atomic64_add_unless_cx8) ··· 149 149 addl $8, %esp 150 150 popl %ebx 151 151 popl %ebp 152 - ret 152 + RET 153 153 4: 154 154 cmpl %edx, 4(%esp) 155 155 jne 2b ··· 176 176 movl $1, %eax 177 177 3: 178 178 popl %ebx 179 - ret 179 + RET 180 180 SYM_FUNC_END(atomic64_inc_not_zero_cx8)
+7 -20
arch/x86/lib/checksum_32.S
··· 127 127 8: 128 128 popl %ebx 129 129 popl %esi 130 - ret 130 + RET 131 131 SYM_FUNC_END(csum_partial) 132 132 133 133 #else ··· 245 245 90: 246 246 popl %ebx 247 247 popl %esi 248 - ret 248 + RET 249 249 SYM_FUNC_END(csum_partial) 250 250 251 251 #endif ··· 260 260 * Copy from ds while checksumming, otherwise like csum_partial 261 261 */ 262 262 263 - #define EXC(y...) \ 264 - 9999: y; \ 265 - _ASM_EXTABLE_UA(9999b, 6001f) 263 + #define EXC(y...) \ 264 + 9999: y; \ 265 + _ASM_EXTABLE_TYPE(9999b, 7f, EX_TYPE_UACCESS | EX_FLAG_CLEAR_AX) 266 266 267 267 #ifndef CONFIG_X86_USE_PPRO_CHECKSUM 268 268 ··· 358 358 adcl $0, %eax 359 359 7: 360 360 361 - # Exception handler: 362 - .section .fixup, "ax" 363 - 364 - 6001: 365 - xorl %eax, %eax 366 - jmp 7b 367 - 368 - .previous 369 - 370 361 popl %ebx 371 362 popl %esi 372 363 popl %edi 373 364 popl %ecx # equivalent to addl $4,%esp 374 - ret 365 + RET 375 366 SYM_FUNC_END(csum_partial_copy_generic) 376 367 377 368 #else ··· 430 439 6: addl %edx, %eax 431 440 adcl $0, %eax 432 441 7: 433 - .section .fixup, "ax" 434 - 6001: xorl %eax, %eax 435 - jmp 7b 436 - .previous 437 442 438 443 popl %esi 439 444 popl %edi 440 445 popl %ebx 441 - ret 446 + RET 442 447 SYM_FUNC_END(csum_partial_copy_generic) 443 448 444 449 #undef ROUND
+3 -3
arch/x86/lib/clear_page_64.S
··· 17 17 movl $4096/8,%ecx 18 18 xorl %eax,%eax 19 19 rep stosq 20 - ret 20 + RET 21 21 SYM_FUNC_END(clear_page_rep) 22 22 EXPORT_SYMBOL_GPL(clear_page_rep) 23 23 ··· 39 39 leaq 64(%rdi),%rdi 40 40 jnz .Lloop 41 41 nop 42 - ret 42 + RET 43 43 SYM_FUNC_END(clear_page_orig) 44 44 EXPORT_SYMBOL_GPL(clear_page_orig) 45 45 ··· 47 47 movl $4096,%ecx 48 48 xorl %eax,%eax 49 49 rep stosb 50 - ret 50 + RET 51 51 SYM_FUNC_END(clear_page_erms) 52 52 EXPORT_SYMBOL_GPL(clear_page_erms)
+2 -2
arch/x86/lib/cmpxchg16b_emu.S
··· 37 37 38 38 popfq 39 39 mov $1, %al 40 - ret 40 + RET 41 41 42 42 .Lnot_same: 43 43 popfq 44 44 xor %al,%al 45 - ret 45 + RET 46 46 47 47 SYM_FUNC_END(this_cpu_cmpxchg16b_emu)
+2 -2
arch/x86/lib/cmpxchg8b_emu.S
··· 32 32 movl %ecx, 4(%esi) 33 33 34 34 popfl 35 - ret 35 + RET 36 36 37 37 .Lnot_same: 38 38 movl (%esi), %eax ··· 40 40 movl 4(%esi), %edx 41 41 42 42 popfl 43 - ret 43 + RET 44 44 45 45 SYM_FUNC_END(cmpxchg8b_emu) 46 46 EXPORT_SYMBOL(cmpxchg8b_emu)
+7 -11
arch/x86/lib/copy_mc_64.S
··· 77 77 .L_done_memcpy_trap: 78 78 xorl %eax, %eax 79 79 .L_done: 80 - ret 81 - SYM_FUNC_END(copy_mc_fragile) 80 + RET 82 81 83 - .section .fixup, "ax" 84 82 /* 85 83 * Return number of bytes not copied for any failure. Note that 86 84 * there is no "tail" handling since the source buffer is 8-byte ··· 103 105 movl %ecx, %edx 104 106 jmp copy_mc_fragile_handle_tail 105 107 106 - .previous 107 - 108 108 _ASM_EXTABLE_TYPE(.L_read_leading_bytes, .E_leading_bytes, EX_TYPE_DEFAULT_MCE_SAFE) 109 109 _ASM_EXTABLE_TYPE(.L_read_words, .E_read_words, EX_TYPE_DEFAULT_MCE_SAFE) 110 110 _ASM_EXTABLE_TYPE(.L_read_trailing_bytes, .E_trailing_bytes, EX_TYPE_DEFAULT_MCE_SAFE) 111 111 _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes) 112 112 _ASM_EXTABLE(.L_write_words, .E_write_words) 113 113 _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes) 114 + 115 + SYM_FUNC_END(copy_mc_fragile) 114 116 #endif /* CONFIG_X86_MCE */ 115 117 116 118 /* ··· 130 132 rep movsb 131 133 /* Copy successful. Return zero */ 132 134 xorl %eax, %eax 133 - ret 134 - SYM_FUNC_END(copy_mc_enhanced_fast_string) 135 + RET 135 136 136 - .section .fixup, "ax" 137 137 .E_copy: 138 138 /* 139 139 * On fault %rcx is updated such that the copy instruction could ··· 141 145 * user-copy routines. 142 146 */ 143 147 movq %rcx, %rax 144 - ret 145 - 146 - .previous 148 + RET 147 149 148 150 _ASM_EXTABLE_TYPE(.L_copy, .E_copy, EX_TYPE_DEFAULT_MCE_SAFE) 151 + 152 + SYM_FUNC_END(copy_mc_enhanced_fast_string) 149 153 #endif /* !CONFIG_UML */
+2 -2
arch/x86/lib/copy_page_64.S
··· 17 17 ALTERNATIVE "jmp copy_page_regs", "", X86_FEATURE_REP_GOOD 18 18 movl $4096/8, %ecx 19 19 rep movsq 20 - ret 20 + RET 21 21 SYM_FUNC_END(copy_page) 22 22 EXPORT_SYMBOL(copy_page) 23 23 ··· 85 85 movq (%rsp), %rbx 86 86 movq 1*8(%rsp), %r12 87 87 addq $2*8, %rsp 88 - ret 88 + RET 89 89 SYM_FUNC_END(copy_page_regs)
+16 -26
arch/x86/lib/copy_user_64.S
··· 32 32 decl %ecx 33 33 jnz 100b 34 34 102: 35 - .section .fixup,"ax" 36 - 103: addl %ecx,%edx /* ecx is zerorest also */ 37 - jmp .Lcopy_user_handle_tail 38 - .previous 39 35 40 - _ASM_EXTABLE_CPY(100b, 103b) 41 - _ASM_EXTABLE_CPY(101b, 103b) 42 - .endm 36 + _ASM_EXTABLE_CPY(100b, .Lcopy_user_handle_align) 37 + _ASM_EXTABLE_CPY(101b, .Lcopy_user_handle_align) 38 + .endm 43 39 44 40 /* 45 41 * copy_user_generic_unrolled - memory copy with exception handling. ··· 101 105 jnz 21b 102 106 23: xor %eax,%eax 103 107 ASM_CLAC 104 - ret 108 + RET 105 109 106 - .section .fixup,"ax" 107 110 30: shll $6,%ecx 108 111 addl %ecx,%edx 109 112 jmp 60f ··· 110 115 jmp 60f 111 116 50: movl %ecx,%edx 112 117 60: jmp .Lcopy_user_handle_tail /* ecx is zerorest also */ 113 - .previous 114 118 115 119 _ASM_EXTABLE_CPY(1b, 30b) 116 120 _ASM_EXTABLE_CPY(2b, 30b) ··· 160 166 movl %edx,%ecx 161 167 shrl $3,%ecx 162 168 andl $7,%edx 163 - 1: rep 164 - movsq 169 + 1: rep movsq 165 170 2: movl %edx,%ecx 166 - 3: rep 167 - movsb 171 + 3: rep movsb 168 172 xorl %eax,%eax 169 173 ASM_CLAC 170 - ret 174 + RET 171 175 172 - .section .fixup,"ax" 173 176 11: leal (%rdx,%rcx,8),%ecx 174 177 12: movl %ecx,%edx /* ecx is zerorest also */ 175 178 jmp .Lcopy_user_handle_tail 176 - .previous 177 179 178 180 _ASM_EXTABLE_CPY(1b, 11b) 179 181 _ASM_EXTABLE_CPY(3b, 12b) ··· 193 203 /* CPUs without FSRM should avoid rep movsb for short copies */ 194 204 ALTERNATIVE "cmpl $64, %edx; jb .L_copy_short_string", "", X86_FEATURE_FSRM 195 205 movl %edx,%ecx 196 - 1: rep 197 - movsb 206 + 1: rep movsb 198 207 xorl %eax,%eax 199 208 ASM_CLAC 200 - ret 209 + RET 201 210 202 - .section .fixup,"ax" 203 211 12: movl %ecx,%edx /* ecx is zerorest also */ 204 212 jmp .Lcopy_user_handle_tail 205 - .previous 206 213 207 214 _ASM_EXTABLE_CPY(1b, 12b) 208 215 SYM_FUNC_END(copy_user_enhanced_fast_string) ··· 228 241 1: rep movsb 229 242 2: mov %ecx,%eax 230 243 ASM_CLAC 231 - ret 244 + RET 232 245 233 246 3: 234 247 movl %edx,%eax ··· 236 249 RET 237 250 238 251 _ASM_EXTABLE_CPY(1b, 2b) 252 + 253 + .Lcopy_user_handle_align: 254 + addl %ecx,%edx /* ecx is zerorest also */ 255 + jmp .Lcopy_user_handle_tail 256 + 239 257 SYM_CODE_END(.Lcopy_user_handle_tail) 240 258 241 259 /* ··· 349 357 xorl %eax,%eax 350 358 ASM_CLAC 351 359 sfence 352 - ret 360 + RET 353 361 354 - .section .fixup,"ax" 355 362 .L_fixup_4x8b_copy: 356 363 shll $6,%ecx 357 364 addl %ecx,%edx ··· 366 375 .L_fixup_handle_tail: 367 376 sfence 368 377 jmp .Lcopy_user_handle_tail 369 - .previous 370 378 371 379 _ASM_EXTABLE_CPY(1b, .L_fixup_4x8b_copy) 372 380 _ASM_EXTABLE_CPY(2b, .L_fixup_4x8b_copy)
+1 -1
arch/x86/lib/csum-copy_64.S
··· 201 201 movq 3*8(%rsp), %r13 202 202 movq 4*8(%rsp), %r15 203 203 addq $5*8, %rsp 204 - ret 204 + RET 205 205 .Lshort: 206 206 movl %ecx, %r10d 207 207 jmp .L1
+105 -106
arch/x86/lib/csum-partial_64.c
··· 9 9 #include <linux/compiler.h> 10 10 #include <linux/export.h> 11 11 #include <asm/checksum.h> 12 + #include <asm/word-at-a-time.h> 12 13 13 14 static inline unsigned short from32to16(unsigned a) 14 15 { ··· 22 21 } 23 22 24 23 /* 25 - * Do a 64-bit checksum on an arbitrary memory area. 24 + * Do a checksum on an arbitrary memory area. 26 25 * Returns a 32bit checksum. 27 26 * 28 27 * This isn't as time critical as it used to be because many NICs 29 28 * do hardware checksumming these days. 30 - * 31 - * Things tried and found to not make it faster: 32 - * Manual Prefetching 33 - * Unrolling to an 128 bytes inner loop. 34 - * Using interleaving with more registers to break the carry chains. 35 - */ 36 - static unsigned do_csum(const unsigned char *buff, unsigned len) 37 - { 38 - unsigned odd, count; 39 - unsigned long result = 0; 40 - 41 - if (unlikely(len == 0)) 42 - return result; 43 - odd = 1 & (unsigned long) buff; 44 - if (unlikely(odd)) { 45 - result = *buff << 8; 46 - len--; 47 - buff++; 48 - } 49 - count = len >> 1; /* nr of 16-bit words.. */ 50 - if (count) { 51 - if (2 & (unsigned long) buff) { 52 - result += *(unsigned short *)buff; 53 - count--; 54 - len -= 2; 55 - buff += 2; 56 - } 57 - count >>= 1; /* nr of 32-bit words.. */ 58 - if (count) { 59 - unsigned long zero; 60 - unsigned count64; 61 - if (4 & (unsigned long) buff) { 62 - result += *(unsigned int *) buff; 63 - count--; 64 - len -= 4; 65 - buff += 4; 66 - } 67 - count >>= 1; /* nr of 64-bit words.. */ 68 - 69 - /* main loop using 64byte blocks */ 70 - zero = 0; 71 - count64 = count >> 3; 72 - while (count64) { 73 - asm("addq 0*8(%[src]),%[res]\n\t" 74 - "adcq 1*8(%[src]),%[res]\n\t" 75 - "adcq 2*8(%[src]),%[res]\n\t" 76 - "adcq 3*8(%[src]),%[res]\n\t" 77 - "adcq 4*8(%[src]),%[res]\n\t" 78 - "adcq 5*8(%[src]),%[res]\n\t" 79 - "adcq 6*8(%[src]),%[res]\n\t" 80 - "adcq 7*8(%[src]),%[res]\n\t" 81 - "adcq %[zero],%[res]" 82 - : [res] "=r" (result) 83 - : [src] "r" (buff), [zero] "r" (zero), 84 - "[res]" (result)); 85 - buff += 64; 86 - count64--; 87 - } 88 - 89 - /* last up to 7 8byte blocks */ 90 - count %= 8; 91 - while (count) { 92 - asm("addq %1,%0\n\t" 93 - "adcq %2,%0\n" 94 - : "=r" (result) 95 - : "m" (*(unsigned long *)buff), 96 - "r" (zero), "0" (result)); 97 - --count; 98 - buff += 8; 99 - } 100 - result = add32_with_carry(result>>32, 101 - result&0xffffffff); 102 - 103 - if (len & 4) { 104 - result += *(unsigned int *) buff; 105 - buff += 4; 106 - } 107 - } 108 - if (len & 2) { 109 - result += *(unsigned short *) buff; 110 - buff += 2; 111 - } 112 - } 113 - if (len & 1) 114 - result += *buff; 115 - result = add32_with_carry(result>>32, result & 0xffffffff); 116 - if (unlikely(odd)) { 117 - result = from32to16(result); 118 - result = ((result >> 8) & 0xff) | ((result & 0xff) << 8); 119 - } 120 - return result; 121 - } 122 - 123 - /* 124 - * computes the checksum of a memory block at buff, length len, 125 - * and adds in "sum" (32-bit) 126 29 * 127 - * returns a 32-bit number suitable for feeding into itself 128 - * or csum_tcpudp_magic 129 - * 130 - * this function must be called with even lengths, except 131 - * for the last fragment, which may be odd 132 - * 30 + * Still, with CHECKSUM_COMPLETE this is called to compute 31 + * checksums on IPv6 headers (40 bytes) and other small parts. 133 32 * it's best to have buff aligned on a 64-bit boundary 134 33 */ 135 34 __wsum csum_partial(const void *buff, int len, __wsum sum) 136 35 { 137 - return (__force __wsum)add32_with_carry(do_csum(buff, len), 138 - (__force u32)sum); 36 + u64 temp64 = (__force u64)sum; 37 + unsigned odd, result; 38 + 39 + odd = 1 & (unsigned long) buff; 40 + if (unlikely(odd)) { 41 + if (unlikely(len == 0)) 42 + return sum; 43 + temp64 = ror32((__force u32)sum, 8); 44 + temp64 += (*(unsigned char *)buff << 8); 45 + len--; 46 + buff++; 47 + } 48 + 49 + while (unlikely(len >= 64)) { 50 + asm("addq 0*8(%[src]),%[res]\n\t" 51 + "adcq 1*8(%[src]),%[res]\n\t" 52 + "adcq 2*8(%[src]),%[res]\n\t" 53 + "adcq 3*8(%[src]),%[res]\n\t" 54 + "adcq 4*8(%[src]),%[res]\n\t" 55 + "adcq 5*8(%[src]),%[res]\n\t" 56 + "adcq 6*8(%[src]),%[res]\n\t" 57 + "adcq 7*8(%[src]),%[res]\n\t" 58 + "adcq $0,%[res]" 59 + : [res] "+r" (temp64) 60 + : [src] "r" (buff) 61 + : "memory"); 62 + buff += 64; 63 + len -= 64; 64 + } 65 + 66 + if (len & 32) { 67 + asm("addq 0*8(%[src]),%[res]\n\t" 68 + "adcq 1*8(%[src]),%[res]\n\t" 69 + "adcq 2*8(%[src]),%[res]\n\t" 70 + "adcq 3*8(%[src]),%[res]\n\t" 71 + "adcq $0,%[res]" 72 + : [res] "+r" (temp64) 73 + : [src] "r" (buff) 74 + : "memory"); 75 + buff += 32; 76 + } 77 + if (len & 16) { 78 + asm("addq 0*8(%[src]),%[res]\n\t" 79 + "adcq 1*8(%[src]),%[res]\n\t" 80 + "adcq $0,%[res]" 81 + : [res] "+r" (temp64) 82 + : [src] "r" (buff) 83 + : "memory"); 84 + buff += 16; 85 + } 86 + if (len & 8) { 87 + asm("addq 0*8(%[src]),%[res]\n\t" 88 + "adcq $0,%[res]" 89 + : [res] "+r" (temp64) 90 + : [src] "r" (buff) 91 + : "memory"); 92 + buff += 8; 93 + } 94 + if (len & 7) { 95 + #ifdef CONFIG_DCACHE_WORD_ACCESS 96 + unsigned int shift = (8 - (len & 7)) * 8; 97 + unsigned long trail; 98 + 99 + trail = (load_unaligned_zeropad(buff) << shift) >> shift; 100 + 101 + asm("addq %[trail],%[res]\n\t" 102 + "adcq $0,%[res]" 103 + : [res] "+r" (temp64) 104 + : [trail] "r" (trail)); 105 + #else 106 + if (len & 4) { 107 + asm("addq %[val],%[res]\n\t" 108 + "adcq $0,%[res]" 109 + : [res] "+r" (temp64) 110 + : [val] "r" ((u64)*(u32 *)buff) 111 + : "memory"); 112 + buff += 4; 113 + } 114 + if (len & 2) { 115 + asm("addq %[val],%[res]\n\t" 116 + "adcq $0,%[res]" 117 + : [res] "+r" (temp64) 118 + : [val] "r" ((u64)*(u16 *)buff) 119 + : "memory"); 120 + buff += 2; 121 + } 122 + if (len & 1) { 123 + asm("addq %[val],%[res]\n\t" 124 + "adcq $0,%[res]" 125 + : [res] "+r" (temp64) 126 + : [val] "r" ((u64)*(u8 *)buff) 127 + : "memory"); 128 + } 129 + #endif 130 + } 131 + result = add32_with_carry(temp64 >> 32, temp64 & 0xffffffff); 132 + if (unlikely(odd)) { 133 + result = from32to16(result); 134 + result = ((result >> 8) & 0xff) | ((result & 0xff) << 8); 135 + } 136 + return (__force __wsum)result; 139 137 } 140 138 EXPORT_SYMBOL(csum_partial); 141 139 ··· 147 147 return csum_fold(csum_partial(buff,len,0)); 148 148 } 149 149 EXPORT_SYMBOL(ip_compute_csum); 150 -
+2 -1
arch/x86/lib/error-inject.c
··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 3 + #include <linux/linkage.h> 3 4 #include <linux/error-injection.h> 4 5 #include <linux/kprobes.h> 5 6 ··· 11 10 ".type just_return_func, @function\n" 12 11 ".globl just_return_func\n" 13 12 "just_return_func:\n" 14 - " ret\n" 13 + ASM_RET 15 14 ".size just_return_func, .-just_return_func\n" 16 15 ); 17 16
+11 -11
arch/x86/lib/getuser.S
··· 57 57 1: movzbl (%_ASM_AX),%edx 58 58 xor %eax,%eax 59 59 ASM_CLAC 60 - ret 60 + RET 61 61 SYM_FUNC_END(__get_user_1) 62 62 EXPORT_SYMBOL(__get_user_1) 63 63 ··· 71 71 2: movzwl (%_ASM_AX),%edx 72 72 xor %eax,%eax 73 73 ASM_CLAC 74 - ret 74 + RET 75 75 SYM_FUNC_END(__get_user_2) 76 76 EXPORT_SYMBOL(__get_user_2) 77 77 ··· 85 85 3: movl (%_ASM_AX),%edx 86 86 xor %eax,%eax 87 87 ASM_CLAC 88 - ret 88 + RET 89 89 SYM_FUNC_END(__get_user_4) 90 90 EXPORT_SYMBOL(__get_user_4) 91 91 ··· 100 100 4: movq (%_ASM_AX),%rdx 101 101 xor %eax,%eax 102 102 ASM_CLAC 103 - ret 103 + RET 104 104 #else 105 105 LOAD_TASK_SIZE_MINUS_N(7) 106 106 cmp %_ASM_DX,%_ASM_AX ··· 112 112 5: movl 4(%_ASM_AX),%ecx 113 113 xor %eax,%eax 114 114 ASM_CLAC 115 - ret 115 + RET 116 116 #endif 117 117 SYM_FUNC_END(__get_user_8) 118 118 EXPORT_SYMBOL(__get_user_8) ··· 124 124 6: movzbl (%_ASM_AX),%edx 125 125 xor %eax,%eax 126 126 ASM_CLAC 127 - ret 127 + RET 128 128 SYM_FUNC_END(__get_user_nocheck_1) 129 129 EXPORT_SYMBOL(__get_user_nocheck_1) 130 130 ··· 134 134 7: movzwl (%_ASM_AX),%edx 135 135 xor %eax,%eax 136 136 ASM_CLAC 137 - ret 137 + RET 138 138 SYM_FUNC_END(__get_user_nocheck_2) 139 139 EXPORT_SYMBOL(__get_user_nocheck_2) 140 140 ··· 144 144 8: movl (%_ASM_AX),%edx 145 145 xor %eax,%eax 146 146 ASM_CLAC 147 - ret 147 + RET 148 148 SYM_FUNC_END(__get_user_nocheck_4) 149 149 EXPORT_SYMBOL(__get_user_nocheck_4) 150 150 ··· 159 159 #endif 160 160 xor %eax,%eax 161 161 ASM_CLAC 162 - ret 162 + RET 163 163 SYM_FUNC_END(__get_user_nocheck_8) 164 164 EXPORT_SYMBOL(__get_user_nocheck_8) 165 165 ··· 169 169 bad_get_user: 170 170 xor %edx,%edx 171 171 mov $(-EFAULT),%_ASM_AX 172 - ret 172 + RET 173 173 SYM_CODE_END(.Lbad_get_user_clac) 174 174 175 175 #ifdef CONFIG_X86_32 ··· 179 179 xor %edx,%edx 180 180 xor %ecx,%ecx 181 181 mov $(-EFAULT),%_ASM_AX 182 - ret 182 + RET 183 183 SYM_CODE_END(.Lbad_get_user_8_clac) 184 184 #endif 185 185
+3 -3
arch/x86/lib/hweight.S
··· 32 32 imull $0x01010101, %eax, %eax # w_tmp *= 0x01010101 33 33 shrl $24, %eax # w = w_tmp >> 24 34 34 __ASM_SIZE(pop,) %__ASM_REG(dx) 35 - ret 35 + RET 36 36 SYM_FUNC_END(__sw_hweight32) 37 37 EXPORT_SYMBOL(__sw_hweight32) 38 38 ··· 65 65 66 66 popq %rdx 67 67 popq %rdi 68 - ret 68 + RET 69 69 #else /* CONFIG_X86_32 */ 70 70 /* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */ 71 71 pushl %ecx ··· 77 77 addl %ecx, %eax # result 78 78 79 79 popl %ecx 80 - ret 80 + RET 81 81 #endif 82 82 SYM_FUNC_END(__sw_hweight64) 83 83 EXPORT_SYMBOL(__sw_hweight64)
+47 -24
arch/x86/lib/insn-eval.c
··· 410 410 #endif /* CONFIG_X86_64 */ 411 411 } 412 412 413 - static int get_reg_offset(struct insn *insn, struct pt_regs *regs, 414 - enum reg_type type) 413 + static const int pt_regoff[] = { 414 + offsetof(struct pt_regs, ax), 415 + offsetof(struct pt_regs, cx), 416 + offsetof(struct pt_regs, dx), 417 + offsetof(struct pt_regs, bx), 418 + offsetof(struct pt_regs, sp), 419 + offsetof(struct pt_regs, bp), 420 + offsetof(struct pt_regs, si), 421 + offsetof(struct pt_regs, di), 422 + #ifdef CONFIG_X86_64 423 + offsetof(struct pt_regs, r8), 424 + offsetof(struct pt_regs, r9), 425 + offsetof(struct pt_regs, r10), 426 + offsetof(struct pt_regs, r11), 427 + offsetof(struct pt_regs, r12), 428 + offsetof(struct pt_regs, r13), 429 + offsetof(struct pt_regs, r14), 430 + offsetof(struct pt_regs, r15), 431 + #else 432 + offsetof(struct pt_regs, ds), 433 + offsetof(struct pt_regs, es), 434 + offsetof(struct pt_regs, fs), 435 + offsetof(struct pt_regs, gs), 436 + #endif 437 + }; 438 + 439 + int pt_regs_offset(struct pt_regs *regs, int regno) 415 440 { 441 + if ((unsigned)regno < ARRAY_SIZE(pt_regoff)) 442 + return pt_regoff[regno]; 443 + return -EDOM; 444 + } 445 + 446 + static int get_regno(struct insn *insn, enum reg_type type) 447 + { 448 + int nr_registers = ARRAY_SIZE(pt_regoff); 416 449 int regno = 0; 417 450 418 - static const int regoff[] = { 419 - offsetof(struct pt_regs, ax), 420 - offsetof(struct pt_regs, cx), 421 - offsetof(struct pt_regs, dx), 422 - offsetof(struct pt_regs, bx), 423 - offsetof(struct pt_regs, sp), 424 - offsetof(struct pt_regs, bp), 425 - offsetof(struct pt_regs, si), 426 - offsetof(struct pt_regs, di), 427 - #ifdef CONFIG_X86_64 428 - offsetof(struct pt_regs, r8), 429 - offsetof(struct pt_regs, r9), 430 - offsetof(struct pt_regs, r10), 431 - offsetof(struct pt_regs, r11), 432 - offsetof(struct pt_regs, r12), 433 - offsetof(struct pt_regs, r13), 434 - offsetof(struct pt_regs, r14), 435 - offsetof(struct pt_regs, r15), 436 - #endif 437 - }; 438 - int nr_registers = ARRAY_SIZE(regoff); 439 451 /* 440 452 * Don't possibly decode a 32-bit instructions as 441 453 * reading a 64-bit-only register. ··· 515 503 WARN_ONCE(1, "decoded an instruction with an invalid register"); 516 504 return -EINVAL; 517 505 } 518 - return regoff[regno]; 506 + return regno; 507 + } 508 + 509 + static int get_reg_offset(struct insn *insn, struct pt_regs *regs, 510 + enum reg_type type) 511 + { 512 + int regno = get_regno(insn, type); 513 + 514 + if (regno < 0) 515 + return regno; 516 + 517 + return pt_regs_offset(regs, regno); 519 518 } 520 519 521 520 /**
+1 -1
arch/x86/lib/iomap_copy_64.S
··· 11 11 SYM_FUNC_START(__iowrite32_copy) 12 12 movl %edx,%ecx 13 13 rep movsd 14 - ret 14 + RET 15 15 SYM_FUNC_END(__iowrite32_copy)
-4
arch/x86/lib/memcpy_32.c
··· 7 7 8 8 __visible void *memcpy(void *to, const void *from, size_t n) 9 9 { 10 - #if defined(CONFIG_X86_USE_3DNOW) && !defined(CONFIG_FORTIFY_SOURCE) 11 - return __memcpy3d(to, from, n); 12 - #else 13 10 return __memcpy(to, from, n); 14 - #endif 15 11 } 16 12 EXPORT_SYMBOL(memcpy); 17 13
+6 -6
arch/x86/lib/memcpy_64.S
··· 39 39 rep movsq 40 40 movl %edx, %ecx 41 41 rep movsb 42 - ret 42 + RET 43 43 SYM_FUNC_END(memcpy) 44 44 SYM_FUNC_END_ALIAS(__memcpy) 45 45 EXPORT_SYMBOL(memcpy) ··· 53 53 movq %rdi, %rax 54 54 movq %rdx, %rcx 55 55 rep movsb 56 - ret 56 + RET 57 57 SYM_FUNC_END(memcpy_erms) 58 58 59 59 SYM_FUNC_START_LOCAL(memcpy_orig) ··· 137 137 movq %r9, 1*8(%rdi) 138 138 movq %r10, -2*8(%rdi, %rdx) 139 139 movq %r11, -1*8(%rdi, %rdx) 140 - retq 140 + RET 141 141 .p2align 4 142 142 .Lless_16bytes: 143 143 cmpl $8, %edx ··· 149 149 movq -1*8(%rsi, %rdx), %r9 150 150 movq %r8, 0*8(%rdi) 151 151 movq %r9, -1*8(%rdi, %rdx) 152 - retq 152 + RET 153 153 .p2align 4 154 154 .Lless_8bytes: 155 155 cmpl $4, %edx ··· 162 162 movl -4(%rsi, %rdx), %r8d 163 163 movl %ecx, (%rdi) 164 164 movl %r8d, -4(%rdi, %rdx) 165 - retq 165 + RET 166 166 .p2align 4 167 167 .Lless_3bytes: 168 168 subl $1, %edx ··· 180 180 movb %cl, (%rdi) 181 181 182 182 .Lend: 183 - retq 183 + RET 184 184 SYM_FUNC_END(memcpy_orig) 185 185 186 186 .popsection
+2 -2
arch/x86/lib/memmove_64.S
··· 40 40 /* FSRM implies ERMS => no length checks, do the copy directly */ 41 41 .Lmemmove_begin_forward: 42 42 ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM 43 - ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; retq", X86_FEATURE_ERMS 43 + ALTERNATIVE "", __stringify(movq %rdx, %rcx; rep movsb; RET), X86_FEATURE_ERMS 44 44 45 45 /* 46 46 * movsq instruction have many startup latency ··· 205 205 movb (%rsi), %r11b 206 206 movb %r11b, (%rdi) 207 207 13: 208 - retq 208 + RET 209 209 SYM_FUNC_END(__memmove) 210 210 SYM_FUNC_END_ALIAS(memmove) 211 211 EXPORT_SYMBOL(__memmove)
+3 -3
arch/x86/lib/memset_64.S
··· 40 40 movl %edx,%ecx 41 41 rep stosb 42 42 movq %r9,%rax 43 - ret 43 + RET 44 44 SYM_FUNC_END(__memset) 45 45 SYM_FUNC_END_ALIAS(memset) 46 46 EXPORT_SYMBOL(memset) ··· 63 63 movq %rdx,%rcx 64 64 rep stosb 65 65 movq %r9,%rax 66 - ret 66 + RET 67 67 SYM_FUNC_END(memset_erms) 68 68 69 69 SYM_FUNC_START_LOCAL(memset_orig) ··· 125 125 126 126 .Lende: 127 127 movq %r10,%rax 128 - ret 128 + RET 129 129 130 130 .Lbad_alignment: 131 131 cmpq $7,%rdx
-388
arch/x86/lib/mmx_32.c
··· 1 - // SPDX-License-Identifier: GPL-2.0 2 - /* 3 - * MMX 3DNow! library helper functions 4 - * 5 - * To do: 6 - * We can use MMX just for prefetch in IRQ's. This may be a win. 7 - * (reported so on K6-III) 8 - * We should use a better code neutral filler for the short jump 9 - * leal ebx. [ebx] is apparently best for K6-2, but Cyrix ?? 10 - * We also want to clobber the filler register so we don't get any 11 - * register forwarding stalls on the filler. 12 - * 13 - * Add *user handling. Checksums are not a win with MMX on any CPU 14 - * tested so far for any MMX solution figured. 15 - * 16 - * 22/09/2000 - Arjan van de Ven 17 - * Improved for non-engineering-sample Athlons 18 - * 19 - */ 20 - #include <linux/hardirq.h> 21 - #include <linux/string.h> 22 - #include <linux/export.h> 23 - #include <linux/sched.h> 24 - #include <linux/types.h> 25 - 26 - #include <asm/fpu/api.h> 27 - #include <asm/asm.h> 28 - 29 - /* 30 - * Use KFPU_387. MMX instructions are not affected by MXCSR, 31 - * but both AMD and Intel documentation states that even integer MMX 32 - * operations will result in #MF if an exception is pending in FCW. 33 - * 34 - * EMMS is not needed afterwards because, after calling kernel_fpu_end(), 35 - * any subsequent user of the 387 stack will reinitialize it using 36 - * KFPU_387. 37 - */ 38 - 39 - void *_mmx_memcpy(void *to, const void *from, size_t len) 40 - { 41 - void *p; 42 - int i; 43 - 44 - if (unlikely(in_interrupt())) 45 - return __memcpy(to, from, len); 46 - 47 - p = to; 48 - i = len >> 6; /* len/64 */ 49 - 50 - kernel_fpu_begin_mask(KFPU_387); 51 - 52 - __asm__ __volatile__ ( 53 - "1: prefetch (%0)\n" /* This set is 28 bytes */ 54 - " prefetch 64(%0)\n" 55 - " prefetch 128(%0)\n" 56 - " prefetch 192(%0)\n" 57 - " prefetch 256(%0)\n" 58 - "2: \n" 59 - ".section .fixup, \"ax\"\n" 60 - "3: movw $0x1AEB, 1b\n" /* jmp on 26 bytes */ 61 - " jmp 2b\n" 62 - ".previous\n" 63 - _ASM_EXTABLE(1b, 3b) 64 - : : "r" (from)); 65 - 66 - for ( ; i > 5; i--) { 67 - __asm__ __volatile__ ( 68 - "1: prefetch 320(%0)\n" 69 - "2: movq (%0), %%mm0\n" 70 - " movq 8(%0), %%mm1\n" 71 - " movq 16(%0), %%mm2\n" 72 - " movq 24(%0), %%mm3\n" 73 - " movq %%mm0, (%1)\n" 74 - " movq %%mm1, 8(%1)\n" 75 - " movq %%mm2, 16(%1)\n" 76 - " movq %%mm3, 24(%1)\n" 77 - " movq 32(%0), %%mm0\n" 78 - " movq 40(%0), %%mm1\n" 79 - " movq 48(%0), %%mm2\n" 80 - " movq 56(%0), %%mm3\n" 81 - " movq %%mm0, 32(%1)\n" 82 - " movq %%mm1, 40(%1)\n" 83 - " movq %%mm2, 48(%1)\n" 84 - " movq %%mm3, 56(%1)\n" 85 - ".section .fixup, \"ax\"\n" 86 - "3: movw $0x05EB, 1b\n" /* jmp on 5 bytes */ 87 - " jmp 2b\n" 88 - ".previous\n" 89 - _ASM_EXTABLE(1b, 3b) 90 - : : "r" (from), "r" (to) : "memory"); 91 - 92 - from += 64; 93 - to += 64; 94 - } 95 - 96 - for ( ; i > 0; i--) { 97 - __asm__ __volatile__ ( 98 - " movq (%0), %%mm0\n" 99 - " movq 8(%0), %%mm1\n" 100 - " movq 16(%0), %%mm2\n" 101 - " movq 24(%0), %%mm3\n" 102 - " movq %%mm0, (%1)\n" 103 - " movq %%mm1, 8(%1)\n" 104 - " movq %%mm2, 16(%1)\n" 105 - " movq %%mm3, 24(%1)\n" 106 - " movq 32(%0), %%mm0\n" 107 - " movq 40(%0), %%mm1\n" 108 - " movq 48(%0), %%mm2\n" 109 - " movq 56(%0), %%mm3\n" 110 - " movq %%mm0, 32(%1)\n" 111 - " movq %%mm1, 40(%1)\n" 112 - " movq %%mm2, 48(%1)\n" 113 - " movq %%mm3, 56(%1)\n" 114 - : : "r" (from), "r" (to) : "memory"); 115 - 116 - from += 64; 117 - to += 64; 118 - } 119 - /* 120 - * Now do the tail of the block: 121 - */ 122 - __memcpy(to, from, len & 63); 123 - kernel_fpu_end(); 124 - 125 - return p; 126 - } 127 - EXPORT_SYMBOL(_mmx_memcpy); 128 - 129 - #ifdef CONFIG_MK7 130 - 131 - /* 132 - * The K7 has streaming cache bypass load/store. The Cyrix III, K6 and 133 - * other MMX using processors do not. 134 - */ 135 - 136 - static void fast_clear_page(void *page) 137 - { 138 - int i; 139 - 140 - kernel_fpu_begin_mask(KFPU_387); 141 - 142 - __asm__ __volatile__ ( 143 - " pxor %%mm0, %%mm0\n" : : 144 - ); 145 - 146 - for (i = 0; i < 4096/64; i++) { 147 - __asm__ __volatile__ ( 148 - " movntq %%mm0, (%0)\n" 149 - " movntq %%mm0, 8(%0)\n" 150 - " movntq %%mm0, 16(%0)\n" 151 - " movntq %%mm0, 24(%0)\n" 152 - " movntq %%mm0, 32(%0)\n" 153 - " movntq %%mm0, 40(%0)\n" 154 - " movntq %%mm0, 48(%0)\n" 155 - " movntq %%mm0, 56(%0)\n" 156 - : : "r" (page) : "memory"); 157 - page += 64; 158 - } 159 - 160 - /* 161 - * Since movntq is weakly-ordered, a "sfence" is needed to become 162 - * ordered again: 163 - */ 164 - __asm__ __volatile__("sfence\n"::); 165 - 166 - kernel_fpu_end(); 167 - } 168 - 169 - static void fast_copy_page(void *to, void *from) 170 - { 171 - int i; 172 - 173 - kernel_fpu_begin_mask(KFPU_387); 174 - 175 - /* 176 - * maybe the prefetch stuff can go before the expensive fnsave... 177 - * but that is for later. -AV 178 - */ 179 - __asm__ __volatile__( 180 - "1: prefetch (%0)\n" 181 - " prefetch 64(%0)\n" 182 - " prefetch 128(%0)\n" 183 - " prefetch 192(%0)\n" 184 - " prefetch 256(%0)\n" 185 - "2: \n" 186 - ".section .fixup, \"ax\"\n" 187 - "3: movw $0x1AEB, 1b\n" /* jmp on 26 bytes */ 188 - " jmp 2b\n" 189 - ".previous\n" 190 - _ASM_EXTABLE(1b, 3b) : : "r" (from)); 191 - 192 - for (i = 0; i < (4096-320)/64; i++) { 193 - __asm__ __volatile__ ( 194 - "1: prefetch 320(%0)\n" 195 - "2: movq (%0), %%mm0\n" 196 - " movntq %%mm0, (%1)\n" 197 - " movq 8(%0), %%mm1\n" 198 - " movntq %%mm1, 8(%1)\n" 199 - " movq 16(%0), %%mm2\n" 200 - " movntq %%mm2, 16(%1)\n" 201 - " movq 24(%0), %%mm3\n" 202 - " movntq %%mm3, 24(%1)\n" 203 - " movq 32(%0), %%mm4\n" 204 - " movntq %%mm4, 32(%1)\n" 205 - " movq 40(%0), %%mm5\n" 206 - " movntq %%mm5, 40(%1)\n" 207 - " movq 48(%0), %%mm6\n" 208 - " movntq %%mm6, 48(%1)\n" 209 - " movq 56(%0), %%mm7\n" 210 - " movntq %%mm7, 56(%1)\n" 211 - ".section .fixup, \"ax\"\n" 212 - "3: movw $0x05EB, 1b\n" /* jmp on 5 bytes */ 213 - " jmp 2b\n" 214 - ".previous\n" 215 - _ASM_EXTABLE(1b, 3b) : : "r" (from), "r" (to) : "memory"); 216 - 217 - from += 64; 218 - to += 64; 219 - } 220 - 221 - for (i = (4096-320)/64; i < 4096/64; i++) { 222 - __asm__ __volatile__ ( 223 - "2: movq (%0), %%mm0\n" 224 - " movntq %%mm0, (%1)\n" 225 - " movq 8(%0), %%mm1\n" 226 - " movntq %%mm1, 8(%1)\n" 227 - " movq 16(%0), %%mm2\n" 228 - " movntq %%mm2, 16(%1)\n" 229 - " movq 24(%0), %%mm3\n" 230 - " movntq %%mm3, 24(%1)\n" 231 - " movq 32(%0), %%mm4\n" 232 - " movntq %%mm4, 32(%1)\n" 233 - " movq 40(%0), %%mm5\n" 234 - " movntq %%mm5, 40(%1)\n" 235 - " movq 48(%0), %%mm6\n" 236 - " movntq %%mm6, 48(%1)\n" 237 - " movq 56(%0), %%mm7\n" 238 - " movntq %%mm7, 56(%1)\n" 239 - : : "r" (from), "r" (to) : "memory"); 240 - from += 64; 241 - to += 64; 242 - } 243 - /* 244 - * Since movntq is weakly-ordered, a "sfence" is needed to become 245 - * ordered again: 246 - */ 247 - __asm__ __volatile__("sfence \n"::); 248 - kernel_fpu_end(); 249 - } 250 - 251 - #else /* CONFIG_MK7 */ 252 - 253 - /* 254 - * Generic MMX implementation without K7 specific streaming 255 - */ 256 - static void fast_clear_page(void *page) 257 - { 258 - int i; 259 - 260 - kernel_fpu_begin_mask(KFPU_387); 261 - 262 - __asm__ __volatile__ ( 263 - " pxor %%mm0, %%mm0\n" : : 264 - ); 265 - 266 - for (i = 0; i < 4096/128; i++) { 267 - __asm__ __volatile__ ( 268 - " movq %%mm0, (%0)\n" 269 - " movq %%mm0, 8(%0)\n" 270 - " movq %%mm0, 16(%0)\n" 271 - " movq %%mm0, 24(%0)\n" 272 - " movq %%mm0, 32(%0)\n" 273 - " movq %%mm0, 40(%0)\n" 274 - " movq %%mm0, 48(%0)\n" 275 - " movq %%mm0, 56(%0)\n" 276 - " movq %%mm0, 64(%0)\n" 277 - " movq %%mm0, 72(%0)\n" 278 - " movq %%mm0, 80(%0)\n" 279 - " movq %%mm0, 88(%0)\n" 280 - " movq %%mm0, 96(%0)\n" 281 - " movq %%mm0, 104(%0)\n" 282 - " movq %%mm0, 112(%0)\n" 283 - " movq %%mm0, 120(%0)\n" 284 - : : "r" (page) : "memory"); 285 - page += 128; 286 - } 287 - 288 - kernel_fpu_end(); 289 - } 290 - 291 - static void fast_copy_page(void *to, void *from) 292 - { 293 - int i; 294 - 295 - kernel_fpu_begin_mask(KFPU_387); 296 - 297 - __asm__ __volatile__ ( 298 - "1: prefetch (%0)\n" 299 - " prefetch 64(%0)\n" 300 - " prefetch 128(%0)\n" 301 - " prefetch 192(%0)\n" 302 - " prefetch 256(%0)\n" 303 - "2: \n" 304 - ".section .fixup, \"ax\"\n" 305 - "3: movw $0x1AEB, 1b\n" /* jmp on 26 bytes */ 306 - " jmp 2b\n" 307 - ".previous\n" 308 - _ASM_EXTABLE(1b, 3b) : : "r" (from)); 309 - 310 - for (i = 0; i < 4096/64; i++) { 311 - __asm__ __volatile__ ( 312 - "1: prefetch 320(%0)\n" 313 - "2: movq (%0), %%mm0\n" 314 - " movq 8(%0), %%mm1\n" 315 - " movq 16(%0), %%mm2\n" 316 - " movq 24(%0), %%mm3\n" 317 - " movq %%mm0, (%1)\n" 318 - " movq %%mm1, 8(%1)\n" 319 - " movq %%mm2, 16(%1)\n" 320 - " movq %%mm3, 24(%1)\n" 321 - " movq 32(%0), %%mm0\n" 322 - " movq 40(%0), %%mm1\n" 323 - " movq 48(%0), %%mm2\n" 324 - " movq 56(%0), %%mm3\n" 325 - " movq %%mm0, 32(%1)\n" 326 - " movq %%mm1, 40(%1)\n" 327 - " movq %%mm2, 48(%1)\n" 328 - " movq %%mm3, 56(%1)\n" 329 - ".section .fixup, \"ax\"\n" 330 - "3: movw $0x05EB, 1b\n" /* jmp on 5 bytes */ 331 - " jmp 2b\n" 332 - ".previous\n" 333 - _ASM_EXTABLE(1b, 3b) 334 - : : "r" (from), "r" (to) : "memory"); 335 - 336 - from += 64; 337 - to += 64; 338 - } 339 - kernel_fpu_end(); 340 - } 341 - 342 - #endif /* !CONFIG_MK7 */ 343 - 344 - /* 345 - * Favour MMX for page clear and copy: 346 - */ 347 - static void slow_zero_page(void *page) 348 - { 349 - int d0, d1; 350 - 351 - __asm__ __volatile__( 352 - "cld\n\t" 353 - "rep ; stosl" 354 - 355 - : "=&c" (d0), "=&D" (d1) 356 - :"a" (0), "1" (page), "0" (1024) 357 - :"memory"); 358 - } 359 - 360 - void mmx_clear_page(void *page) 361 - { 362 - if (unlikely(in_interrupt())) 363 - slow_zero_page(page); 364 - else 365 - fast_clear_page(page); 366 - } 367 - EXPORT_SYMBOL(mmx_clear_page); 368 - 369 - static void slow_copy_page(void *to, void *from) 370 - { 371 - int d0, d1, d2; 372 - 373 - __asm__ __volatile__( 374 - "cld\n\t" 375 - "rep ; movsl" 376 - : "=&c" (d0), "=&D" (d1), "=&S" (d2) 377 - : "0" (1024), "1" ((long) to), "2" ((long) from) 378 - : "memory"); 379 - } 380 - 381 - void mmx_copy_page(void *to, void *from) 382 - { 383 - if (unlikely(in_interrupt())) 384 - slow_copy_page(to, from); 385 - else 386 - fast_copy_page(to, from); 387 - } 388 - EXPORT_SYMBOL(mmx_copy_page);
+2 -2
arch/x86/lib/msr-reg.S
··· 35 35 movl %edi, 28(%r10) 36 36 popq %r12 37 37 popq %rbx 38 - ret 38 + RET 39 39 3: 40 40 movl $-EIO, %r11d 41 41 jmp 2b ··· 77 77 popl %esi 78 78 popl %ebp 79 79 popl %ebx 80 - ret 80 + RET 81 81 3: 82 82 movl $-EIO, 4(%esp) 83 83 jmp 2b
+3 -3
arch/x86/lib/putuser.S
··· 52 52 1: movb %al,(%_ASM_CX) 53 53 xor %ecx,%ecx 54 54 ASM_CLAC 55 - ret 55 + RET 56 56 SYM_FUNC_END(__put_user_1) 57 57 EXPORT_SYMBOL(__put_user_1) 58 58 EXPORT_SYMBOL(__put_user_nocheck_1) ··· 66 66 2: movw %ax,(%_ASM_CX) 67 67 xor %ecx,%ecx 68 68 ASM_CLAC 69 - ret 69 + RET 70 70 SYM_FUNC_END(__put_user_2) 71 71 EXPORT_SYMBOL(__put_user_2) 72 72 EXPORT_SYMBOL(__put_user_nocheck_2) ··· 80 80 3: movl %eax,(%_ASM_CX) 81 81 xor %ecx,%ecx 82 82 ASM_CLAC 83 - ret 83 + RET 84 84 SYM_FUNC_END(__put_user_4) 85 85 EXPORT_SYMBOL(__put_user_4) 86 86 EXPORT_SYMBOL(__put_user_nocheck_4)
+2 -2
arch/x86/lib/retpoline.S
··· 23 23 .Ldo_rop_\@: 24 24 mov %\reg, (%_ASM_SP) 25 25 UNWIND_HINT_FUNC 26 - ret 26 + RET 27 27 .endm 28 28 29 29 .macro THUNK reg ··· 34 34 35 35 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ 36 36 __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \ 37 - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD 37 + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_AMD 38 38 39 39 .endm 40 40
+24 -43
arch/x86/lib/usercopy_32.c
··· 8 8 */ 9 9 #include <linux/export.h> 10 10 #include <linux/uaccess.h> 11 - #include <asm/mmx.h> 12 11 #include <asm/asm.h> 13 12 14 13 #ifdef CONFIG_X86_INTEL_USERCOPY ··· 42 43 " movl %2,%0\n" \ 43 44 "1: rep; stosb\n" \ 44 45 "2: " ASM_CLAC "\n" \ 45 - ".section .fixup,\"ax\"\n" \ 46 - "3: lea 0(%2,%0,4),%0\n" \ 47 - " jmp 2b\n" \ 48 - ".previous\n" \ 49 - _ASM_EXTABLE_UA(0b, 3b) \ 46 + _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %2) \ 50 47 _ASM_EXTABLE_UA(1b, 2b) \ 51 48 : "=&c"(size), "=&D" (__d0) \ 52 49 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \ ··· 144 149 "36: movl %%eax, %0\n" 145 150 "37: rep; movsb\n" 146 151 "100:\n" 147 - ".section .fixup,\"ax\"\n" 148 - "101: lea 0(%%eax,%0,4),%0\n" 149 - " jmp 100b\n" 150 - ".previous\n" 151 152 _ASM_EXTABLE_UA(1b, 100b) 152 153 _ASM_EXTABLE_UA(2b, 100b) 153 154 _ASM_EXTABLE_UA(3b, 100b) ··· 181 190 _ASM_EXTABLE_UA(35b, 100b) 182 191 _ASM_EXTABLE_UA(36b, 100b) 183 192 _ASM_EXTABLE_UA(37b, 100b) 184 - _ASM_EXTABLE_UA(99b, 101b) 193 + _ASM_EXTABLE_TYPE_REG(99b, 100b, EX_TYPE_UCOPY_LEN4, %%eax) 185 194 : "=&c"(size), "=&D" (d0), "=&S" (d1) 186 195 : "1"(to), "2"(from), "0"(size) 187 196 : "eax", "edx", "memory"); ··· 246 255 " movl %%eax,%0\n" 247 256 "7: rep; movsb\n" 248 257 "8:\n" 249 - ".section .fixup,\"ax\"\n" 250 - "9: lea 0(%%eax,%0,4),%0\n" 251 - "16: jmp 8b\n" 252 - ".previous\n" 253 - _ASM_EXTABLE_UA(0b, 16b) 254 - _ASM_EXTABLE_UA(1b, 16b) 255 - _ASM_EXTABLE_UA(2b, 16b) 256 - _ASM_EXTABLE_UA(21b, 16b) 257 - _ASM_EXTABLE_UA(3b, 16b) 258 - _ASM_EXTABLE_UA(31b, 16b) 259 - _ASM_EXTABLE_UA(4b, 16b) 260 - _ASM_EXTABLE_UA(41b, 16b) 261 - _ASM_EXTABLE_UA(10b, 16b) 262 - _ASM_EXTABLE_UA(51b, 16b) 263 - _ASM_EXTABLE_UA(11b, 16b) 264 - _ASM_EXTABLE_UA(61b, 16b) 265 - _ASM_EXTABLE_UA(12b, 16b) 266 - _ASM_EXTABLE_UA(71b, 16b) 267 - _ASM_EXTABLE_UA(13b, 16b) 268 - _ASM_EXTABLE_UA(81b, 16b) 269 - _ASM_EXTABLE_UA(14b, 16b) 270 - _ASM_EXTABLE_UA(91b, 16b) 271 - _ASM_EXTABLE_UA(6b, 9b) 272 - _ASM_EXTABLE_UA(7b, 16b) 258 + _ASM_EXTABLE_UA(0b, 8b) 259 + _ASM_EXTABLE_UA(1b, 8b) 260 + _ASM_EXTABLE_UA(2b, 8b) 261 + _ASM_EXTABLE_UA(21b, 8b) 262 + _ASM_EXTABLE_UA(3b, 8b) 263 + _ASM_EXTABLE_UA(31b, 8b) 264 + _ASM_EXTABLE_UA(4b, 8b) 265 + _ASM_EXTABLE_UA(41b, 8b) 266 + _ASM_EXTABLE_UA(10b, 8b) 267 + _ASM_EXTABLE_UA(51b, 8b) 268 + _ASM_EXTABLE_UA(11b, 8b) 269 + _ASM_EXTABLE_UA(61b, 8b) 270 + _ASM_EXTABLE_UA(12b, 8b) 271 + _ASM_EXTABLE_UA(71b, 8b) 272 + _ASM_EXTABLE_UA(13b, 8b) 273 + _ASM_EXTABLE_UA(81b, 8b) 274 + _ASM_EXTABLE_UA(14b, 8b) 275 + _ASM_EXTABLE_UA(91b, 8b) 276 + _ASM_EXTABLE_TYPE_REG(6b, 8b, EX_TYPE_UCOPY_LEN4, %%eax) 277 + _ASM_EXTABLE_UA(7b, 8b) 273 278 : "=&c"(size), "=&D" (d0), "=&S" (d1) 274 279 : "1"(to), "2"(from), "0"(size) 275 280 : "eax", "edx", "memory"); ··· 302 315 " movl %3,%0\n" \ 303 316 "1: rep; movsb\n" \ 304 317 "2:\n" \ 305 - ".section .fixup,\"ax\"\n" \ 306 - "5: addl %3,%0\n" \ 307 - " jmp 2b\n" \ 308 - "3: lea 0(%3,%0,4),%0\n" \ 309 - " jmp 2b\n" \ 310 - ".previous\n" \ 311 - _ASM_EXTABLE_UA(4b, 5b) \ 312 - _ASM_EXTABLE_UA(0b, 3b) \ 318 + _ASM_EXTABLE_TYPE_REG(4b, 2b, EX_TYPE_UCOPY_LEN1, %3) \ 319 + _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %3) \ 313 320 _ASM_EXTABLE_UA(1b, 2b) \ 314 321 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ 315 322 : "3"(size), "0"(size), "1"(to), "2"(from) \
+3 -5
arch/x86/lib/usercopy_64.c
··· 35 35 " incq %[dst]\n" 36 36 " decl %%ecx ; jnz 1b\n" 37 37 "2:\n" 38 - ".section .fixup,\"ax\"\n" 39 - "3: lea 0(%[size1],%[size8],8),%[size8]\n" 40 - " jmp 2b\n" 41 - ".previous\n" 42 - _ASM_EXTABLE_UA(0b, 3b) 38 + 39 + _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN8, %[size1]) 43 40 _ASM_EXTABLE_UA(1b, 2b) 41 + 44 42 : [size8] "=&c"(size), [dst] "=&D" (__d0) 45 43 : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr)); 46 44 clac();
+1 -1
arch/x86/math-emu/div_Xsig.S
··· 341 341 popl %esi 342 342 343 343 leave 344 - ret 344 + RET 345 345 346 346 347 347 #ifdef PARANOID
+1 -1
arch/x86/math-emu/div_small.S
··· 44 44 popl %esi 45 45 46 46 leave 47 - ret 47 + RET 48 48 SYM_FUNC_END(FPU_div_small)
+3 -3
arch/x86/math-emu/mul_Xsig.S
··· 62 62 63 63 popl %esi 64 64 leave 65 - ret 65 + RET 66 66 SYM_FUNC_END(mul32_Xsig) 67 67 68 68 ··· 115 115 116 116 popl %esi 117 117 leave 118 - ret 118 + RET 119 119 SYM_FUNC_END(mul64_Xsig) 120 120 121 121 ··· 175 175 176 176 popl %esi 177 177 leave 178 - ret 178 + RET 179 179 SYM_FUNC_END(mul_Xsig_Xsig)
+1 -1
arch/x86/math-emu/polynom_Xsig.S
··· 133 133 popl %edi 134 134 popl %esi 135 135 leave 136 - ret 136 + RET 137 137 SYM_FUNC_END(polynomial_Xsig)
+3 -3
arch/x86/math-emu/reg_norm.S
··· 72 72 L_exit: 73 73 popl %ebx 74 74 leave 75 - ret 75 + RET 76 76 77 77 78 78 L_zero: ··· 138 138 139 139 popl %ebx 140 140 leave 141 - ret 141 + RET 142 142 143 143 L_exit_nuo_zero: 144 144 movl TAG_Zero,%eax ··· 146 146 147 147 popl %ebx 148 148 leave 149 - ret 149 + RET 150 150 SYM_FUNC_END(FPU_normalize_nuo)
+1 -1
arch/x86/math-emu/reg_round.S
··· 437 437 popl %edi 438 438 popl %esi 439 439 leave 440 - ret 440 + RET 441 441 442 442 443 443 /*
+1 -1
arch/x86/math-emu/reg_u_add.S
··· 164 164 popl %edi 165 165 popl %esi 166 166 leave 167 - ret 167 + RET 168 168 #endif /* PARANOID */ 169 169 SYM_FUNC_END(FPU_u_add)
+1 -1
arch/x86/math-emu/reg_u_div.S
··· 468 468 popl %esi 469 469 470 470 leave 471 - ret 471 + RET 472 472 #endif /* PARANOID */ 473 473 474 474 SYM_FUNC_END(FPU_u_div)
+1 -1
arch/x86/math-emu/reg_u_mul.S
··· 144 144 popl %edi 145 145 popl %esi 146 146 leave 147 - ret 147 + RET 148 148 #endif /* PARANOID */ 149 149 150 150 SYM_FUNC_END(FPU_u_mul)
+1 -1
arch/x86/math-emu/reg_u_sub.S
··· 270 270 popl %edi 271 271 popl %esi 272 272 leave 273 - ret 273 + RET 274 274 SYM_FUNC_END(FPU_u_sub)
+2 -2
arch/x86/math-emu/round_Xsig.S
··· 78 78 popl %esi 79 79 popl %ebx 80 80 leave 81 - ret 81 + RET 82 82 SYM_FUNC_END(round_Xsig) 83 83 84 84 ··· 138 138 popl %esi 139 139 popl %ebx 140 140 leave 141 - ret 141 + RET 142 142 SYM_FUNC_END(norm_Xsig)
+4 -4
arch/x86/math-emu/shr_Xsig.S
··· 45 45 popl %ebx 46 46 popl %esi 47 47 leave 48 - ret 48 + RET 49 49 50 50 L_more_than_31: 51 51 cmpl $64,%ecx ··· 61 61 movl $0,8(%esi) 62 62 popl %esi 63 63 leave 64 - ret 64 + RET 65 65 66 66 L_more_than_63: 67 67 cmpl $96,%ecx ··· 76 76 movl %edx,8(%esi) 77 77 popl %esi 78 78 leave 79 - ret 79 + RET 80 80 81 81 L_more_than_95: 82 82 xorl %eax,%eax ··· 85 85 movl %eax,8(%esi) 86 86 popl %esi 87 87 leave 88 - ret 88 + RET 89 89 SYM_FUNC_END(shr_Xsig)
+8 -8
arch/x86/math-emu/wm_shrx.S
··· 55 55 popl %ebx 56 56 popl %esi 57 57 leave 58 - ret 58 + RET 59 59 60 60 L_more_than_31: 61 61 cmpl $64,%ecx ··· 70 70 movl $0,4(%esi) 71 71 popl %esi 72 72 leave 73 - ret 73 + RET 74 74 75 75 L_more_than_63: 76 76 cmpl $96,%ecx ··· 84 84 movl %edx,4(%esi) 85 85 popl %esi 86 86 leave 87 - ret 87 + RET 88 88 89 89 L_more_than_95: 90 90 xorl %eax,%eax ··· 92 92 movl %eax,4(%esi) 93 93 popl %esi 94 94 leave 95 - ret 95 + RET 96 96 SYM_FUNC_END(FPU_shrx) 97 97 98 98 ··· 146 146 popl %ebx 147 147 popl %esi 148 148 leave 149 - ret 149 + RET 150 150 151 151 /* Shift by [0..31] bits */ 152 152 Ls_less_than_32: ··· 163 163 popl %ebx 164 164 popl %esi 165 165 leave 166 - ret 166 + RET 167 167 168 168 /* Shift by [64..95] bits */ 169 169 Ls_more_than_63: ··· 189 189 popl %ebx 190 190 popl %esi 191 191 leave 192 - ret 192 + RET 193 193 194 194 Ls_more_than_95: 195 195 /* Shift by [96..inf) bits */ ··· 203 203 popl %ebx 204 204 popl %esi 205 205 leave 206 - ret 206 + RET 207 207 SYM_FUNC_END(FPU_shrxs)
+87 -28
arch/x86/mm/extable.c
··· 2 2 #include <linux/extable.h> 3 3 #include <linux/uaccess.h> 4 4 #include <linux/sched/debug.h> 5 + #include <linux/bitfield.h> 5 6 #include <xen/xen.h> 6 7 7 8 #include <asm/fpu/api.h> 8 9 #include <asm/sev.h> 9 10 #include <asm/traps.h> 10 11 #include <asm/kdebug.h> 12 + #include <asm/insn-eval.h> 13 + #include <asm/sgx.h> 14 + 15 + static inline unsigned long *pt_regs_nr(struct pt_regs *regs, int nr) 16 + { 17 + int reg_offset = pt_regs_offset(regs, nr); 18 + static unsigned long __dummy; 19 + 20 + if (WARN_ON_ONCE(reg_offset < 0)) 21 + return &__dummy; 22 + 23 + return (unsigned long *)((unsigned long)regs + reg_offset); 24 + } 11 25 12 26 static inline unsigned long 13 27 ex_fixup_addr(const struct exception_table_entry *x) ··· 29 15 return (unsigned long)&x->fixup + x->fixup; 30 16 } 31 17 32 - static bool ex_handler_default(const struct exception_table_entry *fixup, 18 + static bool ex_handler_default(const struct exception_table_entry *e, 33 19 struct pt_regs *regs) 34 20 { 35 - regs->ip = ex_fixup_addr(fixup); 21 + if (e->data & EX_FLAG_CLEAR_AX) 22 + regs->ax = 0; 23 + if (e->data & EX_FLAG_CLEAR_DX) 24 + regs->dx = 0; 25 + 26 + regs->ip = ex_fixup_addr(e); 36 27 return true; 37 28 } 38 29 ··· 45 26 struct pt_regs *regs, int trapnr) 46 27 { 47 28 regs->ax = trapnr; 29 + return ex_handler_default(fixup, regs); 30 + } 31 + 32 + static bool ex_handler_sgx(const struct exception_table_entry *fixup, 33 + struct pt_regs *regs, int trapnr) 34 + { 35 + regs->ax = trapnr | SGX_ENCLS_FAULT_FLAG; 48 36 return ex_handler_default(fixup, regs); 49 37 } 50 38 ··· 91 65 return ex_handler_fault(fixup, regs, trapnr); 92 66 } 93 67 94 - static bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup, 95 - struct pt_regs *regs) 68 + static bool ex_handler_msr(const struct exception_table_entry *fixup, 69 + struct pt_regs *regs, bool wrmsr, bool safe, int reg) 96 70 { 97 - if (pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", 98 - (unsigned int)regs->cx, regs->ip, (void *)regs->ip)) 99 - show_stack_regs(regs); 100 - 101 - /* Pretend that the read succeeded and returned 0. */ 102 - regs->ax = 0; 103 - regs->dx = 0; 104 - return ex_handler_default(fixup, regs); 105 - } 106 - 107 - static bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup, 108 - struct pt_regs *regs) 109 - { 110 - if (pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", 71 + if (!safe && wrmsr && 72 + pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", 111 73 (unsigned int)regs->cx, (unsigned int)regs->dx, 112 74 (unsigned int)regs->ax, regs->ip, (void *)regs->ip)) 113 75 show_stack_regs(regs); 114 76 115 - /* Pretend that the write succeeded. */ 77 + if (!safe && !wrmsr && 78 + pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", 79 + (unsigned int)regs->cx, regs->ip, (void *)regs->ip)) 80 + show_stack_regs(regs); 81 + 82 + if (!wrmsr) { 83 + /* Pretend that the read succeeded and returned 0. */ 84 + regs->ax = 0; 85 + regs->dx = 0; 86 + } 87 + 88 + if (safe) 89 + *pt_regs_nr(regs, reg) = -EIO; 90 + 116 91 return ex_handler_default(fixup, regs); 117 92 } 118 93 ··· 126 99 return ex_handler_default(fixup, regs); 127 100 } 128 101 102 + static bool ex_handler_imm_reg(const struct exception_table_entry *fixup, 103 + struct pt_regs *regs, int reg, int imm) 104 + { 105 + *pt_regs_nr(regs, reg) = (long)imm; 106 + return ex_handler_default(fixup, regs); 107 + } 108 + 109 + static bool ex_handler_ucopy_len(const struct exception_table_entry *fixup, 110 + struct pt_regs *regs, int trapnr, int reg, int imm) 111 + { 112 + regs->cx = imm * regs->cx + *pt_regs_nr(regs, reg); 113 + return ex_handler_uaccess(fixup, regs, trapnr); 114 + } 115 + 129 116 int ex_get_fixup_type(unsigned long ip) 130 117 { 131 118 const struct exception_table_entry *e = search_exception_tables(ip); 132 119 133 - return e ? e->type : EX_TYPE_NONE; 120 + return e ? FIELD_GET(EX_DATA_TYPE_MASK, e->data) : EX_TYPE_NONE; 134 121 } 135 122 136 123 int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, 137 124 unsigned long fault_addr) 138 125 { 139 126 const struct exception_table_entry *e; 127 + int type, reg, imm; 140 128 141 129 #ifdef CONFIG_PNPBIOS 142 130 if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) { ··· 171 129 if (!e) 172 130 return 0; 173 131 174 - switch (e->type) { 132 + type = FIELD_GET(EX_DATA_TYPE_MASK, e->data); 133 + reg = FIELD_GET(EX_DATA_REG_MASK, e->data); 134 + imm = FIELD_GET(EX_DATA_IMM_MASK, e->data); 135 + 136 + switch (type) { 175 137 case EX_TYPE_DEFAULT: 176 138 case EX_TYPE_DEFAULT_MCE_SAFE: 177 139 return ex_handler_default(e, regs); ··· 190 144 return ex_handler_clear_fs(e, regs); 191 145 case EX_TYPE_FPU_RESTORE: 192 146 return ex_handler_fprestore(e, regs); 193 - case EX_TYPE_RDMSR: 194 - return ex_handler_rdmsr_unsafe(e, regs); 195 - case EX_TYPE_WRMSR: 196 - return ex_handler_wrmsr_unsafe(e, regs); 197 147 case EX_TYPE_BPF: 198 148 return ex_handler_bpf(e, regs); 199 - case EX_TYPE_RDMSR_IN_MCE: 200 - ex_handler_msr_mce(regs, false); 201 - break; 149 + case EX_TYPE_WRMSR: 150 + return ex_handler_msr(e, regs, true, false, reg); 151 + case EX_TYPE_RDMSR: 152 + return ex_handler_msr(e, regs, false, false, reg); 153 + case EX_TYPE_WRMSR_SAFE: 154 + return ex_handler_msr(e, regs, true, true, reg); 155 + case EX_TYPE_RDMSR_SAFE: 156 + return ex_handler_msr(e, regs, false, true, reg); 202 157 case EX_TYPE_WRMSR_IN_MCE: 203 158 ex_handler_msr_mce(regs, true); 204 159 break; 160 + case EX_TYPE_RDMSR_IN_MCE: 161 + ex_handler_msr_mce(regs, false); 162 + break; 163 + case EX_TYPE_POP_REG: 164 + regs->sp += sizeof(long); 165 + fallthrough; 166 + case EX_TYPE_IMM_REG: 167 + return ex_handler_imm_reg(e, regs, reg, imm); 168 + case EX_TYPE_FAULT_SGX: 169 + return ex_handler_sgx(e, regs, trapnr); 170 + case EX_TYPE_UCOPY_LEN: 171 + return ex_handler_ucopy_len(e, regs, trapnr, reg, imm); 205 172 } 206 173 BUG(); 207 174 }
+2 -2
arch/x86/mm/mem_encrypt_boot.S
··· 65 65 movq %rbp, %rsp /* Restore original stack pointer */ 66 66 pop %rbp 67 67 68 - ret 68 + RET 69 69 SYM_FUNC_END(sme_encrypt_execute) 70 70 71 71 SYM_FUNC_START(__enc_copy) ··· 151 151 pop %r12 152 152 pop %r15 153 153 154 - ret 154 + RET 155 155 .L__enc_copy_end: 156 156 SYM_FUNC_END(__enc_copy)
+1 -1
arch/x86/net/bpf_jit_comp.c
··· 1326 1326 } 1327 1327 ex->insn = delta; 1328 1328 1329 - ex->type = EX_TYPE_BPF; 1329 + ex->data = EX_TYPE_BPF; 1330 1330 1331 1331 if (dst_reg > BPF_REG_9) { 1332 1332 pr_err("verifier error\n");
+1 -1
arch/x86/platform/efi/efi_stub_32.S
··· 56 56 57 57 movl 16(%esp), %ebx 58 58 leave 59 - ret 59 + RET 60 60 SYM_FUNC_END(efi_call_svam)
+1 -1
arch/x86/platform/efi/efi_stub_64.S
··· 23 23 mov %rsi, %rcx 24 24 CALL_NOSPEC rdi 25 25 leave 26 - ret 26 + RET 27 27 SYM_FUNC_END(__efi_call)
+1 -1
arch/x86/platform/efi/efi_thunk_64.S
··· 73 73 1: movq 0x20(%rsp), %rsp 74 74 pop %rbx 75 75 pop %rbp 76 - retq 76 + RET 77 77 78 78 .code32 79 79 2: pushl $__KERNEL_CS
+3 -3
arch/x86/platform/olpc/xo1-wakeup.S
··· 77 77 pushfl 78 78 popl saved_context_eflags 79 79 80 - ret 80 + RET 81 81 82 82 restore_registers: 83 83 movl saved_context_ebp, %ebp ··· 88 88 pushl saved_context_eflags 89 89 popfl 90 90 91 - ret 91 + RET 92 92 93 93 SYM_CODE_START(do_olpc_suspend_lowlevel) 94 94 call save_processor_state ··· 109 109 110 110 call restore_registers 111 111 call restore_processor_state 112 - ret 112 + RET 113 113 SYM_CODE_END(do_olpc_suspend_lowlevel) 114 114 115 115 .data
+2 -2
arch/x86/power/hibernate_asm_32.S
··· 32 32 FRAME_BEGIN 33 33 call swsusp_save 34 34 FRAME_END 35 - ret 35 + RET 36 36 SYM_FUNC_END(swsusp_arch_suspend) 37 37 38 38 SYM_CODE_START(restore_image) ··· 108 108 /* tell the hibernation core that we've just restored the memory */ 109 109 movl %eax, in_suspend 110 110 111 - ret 111 + RET 112 112 SYM_FUNC_END(restore_registers)
+2 -2
arch/x86/power/hibernate_asm_64.S
··· 66 66 /* tell the hibernation core that we've just restored the memory */ 67 67 movq %rax, in_suspend(%rip) 68 68 69 - ret 69 + RET 70 70 SYM_FUNC_END(restore_registers) 71 71 72 72 SYM_FUNC_START(swsusp_arch_suspend) ··· 96 96 FRAME_BEGIN 97 97 call swsusp_save 98 98 FRAME_END 99 - ret 99 + RET 100 100 SYM_FUNC_END(swsusp_arch_suspend) 101 101 102 102 SYM_FUNC_START(restore_image)
+2 -2
arch/x86/um/checksum_32.S
··· 110 110 7: 111 111 popl %ebx 112 112 popl %esi 113 - ret 113 + RET 114 114 115 115 #else 116 116 ··· 208 208 80: 209 209 popl %ebx 210 210 popl %esi 211 - ret 211 + RET 212 212 213 213 #endif 214 214 EXPORT_SYMBOL(csum_partial)
+1 -1
arch/x86/um/setjmp_32.S
··· 34 34 movl %esi,12(%edx) 35 35 movl %edi,16(%edx) 36 36 movl %ecx,20(%edx) # Return address 37 - ret 37 + RET 38 38 39 39 .size kernel_setjmp,.-kernel_setjmp 40 40
+1 -1
arch/x86/um/setjmp_64.S
··· 33 33 movq %r14,40(%rdi) 34 34 movq %r15,48(%rdi) 35 35 movq %rsi,56(%rdi) # Return address 36 - ret 36 + RET 37 37 38 38 .size kernel_setjmp,.-kernel_setjmp 39 39
+6 -6
arch/x86/xen/xen-asm.S
··· 29 29 */ 30 30 SYM_FUNC_START(xen_irq_disable_direct) 31 31 movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 32 - ret 32 + RET 33 33 SYM_FUNC_END(xen_irq_disable_direct) 34 34 35 35 /* ··· 58 58 pop %rcx 59 59 pop %rax 60 60 FRAME_END 61 - ret 61 + RET 62 62 SYM_FUNC_END(check_events) 63 63 64 64 /* ··· 84 84 call check_events 85 85 1: 86 86 FRAME_END 87 - ret 87 + RET 88 88 SYM_FUNC_END(xen_irq_enable_direct) 89 89 90 90 /* ··· 100 100 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 101 101 setz %ah 102 102 addb %ah, %ah 103 - ret 103 + RET 104 104 SYM_FUNC_END(xen_save_fl_direct) 105 105 106 106 SYM_FUNC_START(xen_read_cr2) ··· 108 108 _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX 109 109 _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX 110 110 FRAME_END 111 - ret 111 + RET 112 112 SYM_FUNC_END(xen_read_cr2); 113 113 114 114 SYM_FUNC_START(xen_read_cr2_direct) 115 115 FRAME_BEGIN 116 116 _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX 117 117 FRAME_END 118 - ret 118 + RET 119 119 SYM_FUNC_END(xen_read_cr2_direct); 120 120 .popsection 121 121
+1 -1
arch/x86/xen/xen-head.S
··· 26 26 .rept (PAGE_SIZE / 32) 27 27 UNWIND_HINT_FUNC 28 28 .skip 31, 0x90 29 - ret 29 + RET 30 30 .endr 31 31 32 32 #define HYPERCALL(n) \
+18 -1
include/linux/bitfield.h
··· 41 41 42 42 #define __bf_shf(x) (__builtin_ffsll(x) - 1) 43 43 44 + #define __scalar_type_to_unsigned_cases(type) \ 45 + unsigned type: (unsigned type)0, \ 46 + signed type: (unsigned type)0 47 + 48 + #define __unsigned_scalar_typeof(x) typeof( \ 49 + _Generic((x), \ 50 + char: (unsigned char)0, \ 51 + __scalar_type_to_unsigned_cases(char), \ 52 + __scalar_type_to_unsigned_cases(short), \ 53 + __scalar_type_to_unsigned_cases(int), \ 54 + __scalar_type_to_unsigned_cases(long), \ 55 + __scalar_type_to_unsigned_cases(long long), \ 56 + default: (x))) 57 + 58 + #define __bf_cast_unsigned(type, x) ((__unsigned_scalar_typeof(type))(x)) 59 + 44 60 #define __BF_FIELD_CHECK(_mask, _reg, _val, _pfx) \ 45 61 ({ \ 46 62 BUILD_BUG_ON_MSG(!__builtin_constant_p(_mask), \ ··· 65 49 BUILD_BUG_ON_MSG(__builtin_constant_p(_val) ? \ 66 50 ~((_mask) >> __bf_shf(_mask)) & (_val) : 0, \ 67 51 _pfx "value too large for the field"); \ 68 - BUILD_BUG_ON_MSG((_mask) > (typeof(_reg))~0ull, \ 52 + BUILD_BUG_ON_MSG(__bf_cast_unsigned(_mask, _mask) > \ 53 + __bf_cast_unsigned(_reg, ~0ull), \ 69 54 _pfx "type of reg too small for mask"); \ 70 55 __BUILD_BUG_ON_NOT_POWER_OF_2((_mask) + \ 71 56 (1ULL << __bf_shf(_mask))); \
+2 -2
samples/ftrace/ftrace-direct-modify.c
··· 34 34 " call my_direct_func1\n" 35 35 " leave\n" 36 36 " .size my_tramp1, .-my_tramp1\n" 37 - " ret\n" 37 + ASM_RET 38 38 " .type my_tramp2, @function\n" 39 39 " .globl my_tramp2\n" 40 40 " my_tramp2:" ··· 42 42 " movq %rsp, %rbp\n" 43 43 " call my_direct_func2\n" 44 44 " leave\n" 45 - " ret\n" 45 + ASM_RET 46 46 " .size my_tramp2, .-my_tramp2\n" 47 47 " .popsection\n" 48 48 );
+1 -1
samples/ftrace/ftrace-direct-too.c
··· 34 34 " popq %rsi\n" 35 35 " popq %rdi\n" 36 36 " leave\n" 37 - " ret\n" 37 + ASM_RET 38 38 " .size my_tramp, .-my_tramp\n" 39 39 " .popsection\n" 40 40 );
+1 -1
samples/ftrace/ftrace-direct.c
··· 27 27 " call my_direct_func\n" 28 28 " popq %rdi\n" 29 29 " leave\n" 30 - " ret\n" 30 + ASM_RET 31 31 " .size my_tramp, .-my_tramp\n" 32 32 " .popsection\n" 33 33 );
+2 -1
scripts/Makefile.build
··· 234 234 $(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\ 235 235 $(if $(CONFIG_RETPOLINE), --retpoline) \ 236 236 $(if $(CONFIG_X86_SMAP), --uaccess) \ 237 - $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) 237 + $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) \ 238 + $(if $(CONFIG_SLS), --sls) 238 239 239 240 cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool_args) $@) 240 241 cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(objtool))' ; } >> $(dot-target).cmd)
-4
tools/arch/x86/include/asm/required-features.h
··· 35 35 # define NEED_CMOV 0 36 36 #endif 37 37 38 - #ifdef CONFIG_X86_USE_3DNOW 39 - # define NEED_3DNOW (1<<(X86_FEATURE_3DNOW & 31)) 40 - #else 41 38 # define NEED_3DNOW 0 42 - #endif 43 39 44 40 #if defined(CONFIG_X86_P6_NOP) || defined(CONFIG_X86_64) 45 41 # define NEED_NOPL (1<<(X86_FEATURE_NOPL & 31))
+9 -4
tools/objtool/arch/x86/decode.c
··· 531 531 } 532 532 break; 533 533 534 + case 0xcc: 535 + /* int3 */ 536 + *type = INSN_TRAP; 537 + break; 538 + 534 539 case 0xe3: 535 540 /* jecxz/jrcxz */ 536 541 *type = INSN_JUMP_CONDITIONAL; ··· 702 697 { 703 698 static const char ret[5][5] = { 704 699 { BYTE_RET }, 705 - { BYTE_RET, BYTES_NOP1 }, 706 - { BYTE_RET, BYTES_NOP2 }, 707 - { BYTE_RET, BYTES_NOP3 }, 708 - { BYTE_RET, BYTES_NOP4 }, 700 + { BYTE_RET, 0xcc }, 701 + { BYTE_RET, 0xcc, BYTES_NOP1 }, 702 + { BYTE_RET, 0xcc, BYTES_NOP2 }, 703 + { BYTE_RET, 0xcc, BYTES_NOP3 }, 709 704 }; 710 705 711 706 if (len < 1 || len > 5) {
+2 -1
tools/objtool/builtin-check.c
··· 20 20 #include <objtool/objtool.h> 21 21 22 22 bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, 23 - validate_dup, vmlinux, mcount, noinstr, backup; 23 + validate_dup, vmlinux, mcount, noinstr, backup, sls; 24 24 25 25 static const char * const check_usage[] = { 26 26 "objtool check [<options>] file.o", ··· 45 45 OPT_BOOLEAN('l', "vmlinux", &vmlinux, "vmlinux.o validation"), 46 46 OPT_BOOLEAN('M', "mcount", &mcount, "generate __mcount_loc"), 47 47 OPT_BOOLEAN('B', "backup", &backup, "create .orig files before modification"), 48 + OPT_BOOLEAN('S', "sls", &sls, "validate straight-line-speculation"), 48 49 OPT_END(), 49 50 }; 50 51
+16 -6
tools/objtool/check.c
··· 3113 3113 switch (insn->type) { 3114 3114 3115 3115 case INSN_RETURN: 3116 + if (next_insn && next_insn->type == INSN_TRAP) { 3117 + next_insn->ignore = true; 3118 + } else if (sls && !insn->retpoline_safe) { 3119 + WARN_FUNC("missing int3 after ret", 3120 + insn->sec, insn->offset); 3121 + } 3116 3122 return validate_return(func, insn, &state); 3117 3123 3118 3124 case INSN_CALL: ··· 3162 3156 break; 3163 3157 3164 3158 case INSN_JUMP_DYNAMIC: 3159 + if (next_insn && next_insn->type == INSN_TRAP) { 3160 + next_insn->ignore = true; 3161 + } else if (sls && !insn->retpoline_safe) { 3162 + WARN_FUNC("missing int3 after indirect jump", 3163 + insn->sec, insn->offset); 3164 + } 3165 + 3166 + /* fallthrough */ 3165 3167 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3166 3168 if (is_sibling_call(insn)) { 3167 3169 ret = validate_sibling_call(file, insn, &state); ··· 3339 3325 return true; 3340 3326 3341 3327 /* 3342 - * Ignore any unused exceptions. This can happen when a whitelisted 3343 - * function has an exception table entry. 3344 - * 3345 - * Also ignore alternative replacement instructions. This can happen 3328 + * Ignore alternative replacement instructions. This can happen 3346 3329 * when a whitelisted function uses one of the ALTERNATIVE macros. 3347 3330 */ 3348 - if (!strcmp(insn->sec->name, ".fixup") || 3349 - !strcmp(insn->sec->name, ".altinstr_replacement") || 3331 + if (!strcmp(insn->sec->name, ".altinstr_replacement") || 3350 3332 !strcmp(insn->sec->name, ".altinstr_aux")) 3351 3333 return true; 3352 3334
+1
tools/objtool/include/objtool/arch.h
··· 26 26 INSN_CLAC, 27 27 INSN_STD, 28 28 INSN_CLD, 29 + INSN_TRAP, 29 30 INSN_OTHER, 30 31 }; 31 32
+1 -1
tools/objtool/include/objtool/builtin.h
··· 9 9 10 10 extern const struct option check_options[]; 11 11 extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, 12 - validate_dup, vmlinux, mcount, noinstr, backup; 12 + validate_dup, vmlinux, mcount, noinstr, backup, sls; 13 13 14 14 extern int cmd_parse_options(int argc, const char **argv, const char * const usage[]); 15 15