tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge branch 'bpf-riscv64-support-load-acquire-and-store-release-instructions'

Peilin Ye says:

====================
bpf, riscv64: Support load-acquire and store-release instructions

Hi all!

Patchset [1] introduced BPF load-acquire (BPF_LOAD_ACQ) and
store-release (BPF_STORE_REL) instructions, and added x86-64 and arm64
JIT compiler support. As a follow-up, this v2 patchset supports
load-acquire and store-release instructions for the riscv64 JIT
compiler, and introduces some related selftests/ changes.

Specifically:

* PATCH 1 makes insn_def_regno() handle load-acquires properly for
bpf_jit_needs_zext() (true for riscv64) architectures
* PATCH 2, 3 from Andrea Parri add the actual support to the riscv64
JIT compiler
* PATCH 4 optimizes code emission by skipping redundant zext
instructions inserted by the verifier
* PATCH 5, 6 and 7 are minor selftest/ improvements
* PATCH 8 enables (non-arena) load-acquire/store-release selftests for
riscv64

v1: https://lore.kernel.org/bpf/cover.1745970908.git.yepeilin@google.com/
Changes since v1:

* add Acked-by:, Reviewed-by: and Tested-by: tags from Lehui and Björn
* simplify code logic in PATCH 1 (Lehui)
* in PATCH 3, avoid changing 'return 0;' to 'return ret;' at the end of
bpf_jit_emit_insn() (Lehui)

Please refer to individual patches for details. Thanks!

[1] https://lore.kernel.org/all/cover.1741049567.git.yepeilin@google.com/
====================

Link: https://patch.msgid.link/cover.1746588351.git.yepeilin@google.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

Alexei Starovoitov 32c563d1 b69d4413

+313 -146
+15
arch/riscv/net/bpf_jit.h
··· 608 608 return rv_i_insn(imm11_0, 0, 0, 0, 0xf); 609 609 } 610 610 611 + static inline void emit_fence_r_rw(struct rv_jit_context *ctx) 612 + { 613 + emit(rv_fence(0x2, 0x3), ctx); 614 + } 615 + 616 + static inline void emit_fence_rw_w(struct rv_jit_context *ctx) 617 + { 618 + emit(rv_fence(0x3, 0x1), ctx); 619 + } 620 + 621 + static inline void emit_fence_rw_rw(struct rv_jit_context *ctx) 622 + { 623 + emit(rv_fence(0x3, 0x3), ctx); 624 + } 625 + 611 626 static inline u32 rv_nop(void) 612 627 { 613 628 return rv_i_insn(0, 0, 0, 0, 0x13);
+227 -105
arch/riscv/net/bpf_jit_comp64.c
··· 473 473 emit(hash, ctx); 474 474 } 475 475 476 - static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, 477 - struct rv_jit_context *ctx) 476 + static int emit_load_8(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx) 478 477 { 479 - u8 r0; 478 + int insns_start; 479 + 480 + if (is_12b_int(off)) { 481 + insns_start = ctx->ninsns; 482 + if (sign_ext) 483 + emit(rv_lb(rd, off, rs), ctx); 484 + else 485 + emit(rv_lbu(rd, off, rs), ctx); 486 + return ctx->ninsns - insns_start; 487 + } 488 + 489 + emit_imm(RV_REG_T1, off, ctx); 490 + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 491 + insns_start = ctx->ninsns; 492 + if (sign_ext) 493 + emit(rv_lb(rd, 0, RV_REG_T1), ctx); 494 + else 495 + emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 496 + return ctx->ninsns - insns_start; 497 + } 498 + 499 + static int emit_load_16(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx) 500 + { 501 + int insns_start; 502 + 503 + if (is_12b_int(off)) { 504 + insns_start = ctx->ninsns; 505 + if (sign_ext) 506 + emit(rv_lh(rd, off, rs), ctx); 507 + else 508 + emit(rv_lhu(rd, off, rs), ctx); 509 + return ctx->ninsns - insns_start; 510 + } 511 + 512 + emit_imm(RV_REG_T1, off, ctx); 513 + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 514 + insns_start = ctx->ninsns; 515 + if (sign_ext) 516 + emit(rv_lh(rd, 0, RV_REG_T1), ctx); 517 + else 518 + emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 519 + return ctx->ninsns - insns_start; 520 + } 521 + 522 + static int emit_load_32(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx) 523 + { 524 + int insns_start; 525 + 526 + if (is_12b_int(off)) { 527 + insns_start = ctx->ninsns; 528 + if (sign_ext) 529 + emit(rv_lw(rd, off, rs), ctx); 530 + else 531 + emit(rv_lwu(rd, off, rs), ctx); 532 + return ctx->ninsns - insns_start; 533 + } 534 + 535 + emit_imm(RV_REG_T1, off, ctx); 536 + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 537 + insns_start = ctx->ninsns; 538 + if (sign_ext) 539 + emit(rv_lw(rd, 0, RV_REG_T1), ctx); 540 + else 541 + emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 542 + return ctx->ninsns - insns_start; 543 + } 544 + 545 + static int emit_load_64(bool sign_ext, u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx) 546 + { 547 + int insns_start; 548 + 549 + if (is_12b_int(off)) { 550 + insns_start = ctx->ninsns; 551 + emit_ld(rd, off, rs, ctx); 552 + return ctx->ninsns - insns_start; 553 + } 554 + 555 + emit_imm(RV_REG_T1, off, ctx); 556 + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 557 + insns_start = ctx->ninsns; 558 + emit_ld(rd, 0, RV_REG_T1, ctx); 559 + return ctx->ninsns - insns_start; 560 + } 561 + 562 + static void emit_store_8(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx) 563 + { 564 + if (is_12b_int(off)) { 565 + emit(rv_sb(rd, off, rs), ctx); 566 + return; 567 + } 568 + 569 + emit_imm(RV_REG_T1, off, ctx); 570 + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 571 + emit(rv_sb(RV_REG_T1, 0, rs), ctx); 572 + } 573 + 574 + static void emit_store_16(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx) 575 + { 576 + if (is_12b_int(off)) { 577 + emit(rv_sh(rd, off, rs), ctx); 578 + return; 579 + } 580 + 581 + emit_imm(RV_REG_T1, off, ctx); 582 + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 583 + emit(rv_sh(RV_REG_T1, 0, rs), ctx); 584 + } 585 + 586 + static void emit_store_32(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx) 587 + { 588 + if (is_12b_int(off)) { 589 + emit_sw(rd, off, rs, ctx); 590 + return; 591 + } 592 + 593 + emit_imm(RV_REG_T1, off, ctx); 594 + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 595 + emit_sw(RV_REG_T1, 0, rs, ctx); 596 + } 597 + 598 + static void emit_store_64(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx) 599 + { 600 + if (is_12b_int(off)) { 601 + emit_sd(rd, off, rs, ctx); 602 + return; 603 + } 604 + 605 + emit_imm(RV_REG_T1, off, ctx); 606 + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 607 + emit_sd(RV_REG_T1, 0, rs, ctx); 608 + } 609 + 610 + static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn, 611 + struct rv_jit_context *ctx) 612 + { 613 + u8 code = insn->code; 614 + s32 imm = insn->imm; 615 + s16 off = insn->off; 616 + 617 + switch (imm) { 618 + /* dst_reg = load_acquire(src_reg + off16) */ 619 + case BPF_LOAD_ACQ: 620 + switch (BPF_SIZE(code)) { 621 + case BPF_B: 622 + emit_load_8(false, rd, off, rs, ctx); 623 + break; 624 + case BPF_H: 625 + emit_load_16(false, rd, off, rs, ctx); 626 + break; 627 + case BPF_W: 628 + emit_load_32(false, rd, off, rs, ctx); 629 + break; 630 + case BPF_DW: 631 + emit_load_64(false, rd, off, rs, ctx); 632 + break; 633 + } 634 + emit_fence_r_rw(ctx); 635 + 636 + /* If our next insn is a redundant zext, return 1 to tell 637 + * build_body() to skip it. 638 + */ 639 + if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1])) 640 + return 1; 641 + break; 642 + /* store_release(dst_reg + off16, src_reg) */ 643 + case BPF_STORE_REL: 644 + emit_fence_rw_w(ctx); 645 + switch (BPF_SIZE(code)) { 646 + case BPF_B: 647 + emit_store_8(rd, off, rs, ctx); 648 + break; 649 + case BPF_H: 650 + emit_store_16(rd, off, rs, ctx); 651 + break; 652 + case BPF_W: 653 + emit_store_32(rd, off, rs, ctx); 654 + break; 655 + case BPF_DW: 656 + emit_store_64(rd, off, rs, ctx); 657 + break; 658 + } 659 + break; 660 + default: 661 + pr_err_once("bpf-jit: invalid atomic load/store opcode %02x\n", imm); 662 + return -EINVAL; 663 + } 664 + 665 + return 0; 666 + } 667 + 668 + static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn, 669 + struct rv_jit_context *ctx) 670 + { 671 + u8 r0, code = insn->code; 672 + s16 off = insn->off; 673 + s32 imm = insn->imm; 480 674 int jmp_offset; 675 + bool is64; 676 + 677 + if (BPF_SIZE(code) != BPF_W && BPF_SIZE(code) != BPF_DW) { 678 + pr_err_once("bpf-jit: 1- and 2-byte RMW atomics are not supported\n"); 679 + return -EINVAL; 680 + } 681 + is64 = BPF_SIZE(code) == BPF_DW; 481 682 482 683 if (off) { 483 684 if (is_12b_int(off)) { ··· 755 554 rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx); 756 555 jmp_offset = ninsns_rvoff(-6); 757 556 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); 758 - emit(rv_fence(0x3, 0x3), ctx); 557 + emit_fence_rw_rw(ctx); 759 558 break; 559 + default: 560 + pr_err_once("bpf-jit: invalid atomic RMW opcode %02x\n", imm); 561 + return -EINVAL; 760 562 } 563 + 564 + return 0; 761 565 } 762 566 763 567 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) ··· 1856 1650 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1857 1651 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1858 1652 { 1859 - int insn_len, insns_start; 1860 1653 bool sign_ext; 1654 + int insn_len; 1861 1655 1862 1656 sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || 1863 1657 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; ··· 1869 1663 1870 1664 switch (BPF_SIZE(code)) { 1871 1665 case BPF_B: 1872 - if (is_12b_int(off)) { 1873 - insns_start = ctx->ninsns; 1874 - if (sign_ext) 1875 - emit(rv_lb(rd, off, rs), ctx); 1876 - else 1877 - emit(rv_lbu(rd, off, rs), ctx); 1878 - insn_len = ctx->ninsns - insns_start; 1879 - break; 1880 - } 1881 - 1882 - emit_imm(RV_REG_T1, off, ctx); 1883 - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1884 - insns_start = ctx->ninsns; 1885 - if (sign_ext) 1886 - emit(rv_lb(rd, 0, RV_REG_T1), ctx); 1887 - else 1888 - emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 1889 - insn_len = ctx->ninsns - insns_start; 1666 + insn_len = emit_load_8(sign_ext, rd, off, rs, ctx); 1890 1667 break; 1891 1668 case BPF_H: 1892 - if (is_12b_int(off)) { 1893 - insns_start = ctx->ninsns; 1894 - if (sign_ext) 1895 - emit(rv_lh(rd, off, rs), ctx); 1896 - else 1897 - emit(rv_lhu(rd, off, rs), ctx); 1898 - insn_len = ctx->ninsns - insns_start; 1899 - break; 1900 - } 1901 - 1902 - emit_imm(RV_REG_T1, off, ctx); 1903 - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1904 - insns_start = ctx->ninsns; 1905 - if (sign_ext) 1906 - emit(rv_lh(rd, 0, RV_REG_T1), ctx); 1907 - else 1908 - emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 1909 - insn_len = ctx->ninsns - insns_start; 1669 + insn_len = emit_load_16(sign_ext, rd, off, rs, ctx); 1910 1670 break; 1911 1671 case BPF_W: 1912 - if (is_12b_int(off)) { 1913 - insns_start = ctx->ninsns; 1914 - if (sign_ext) 1915 - emit(rv_lw(rd, off, rs), ctx); 1916 - else 1917 - emit(rv_lwu(rd, off, rs), ctx); 1918 - insn_len = ctx->ninsns - insns_start; 1919 - break; 1920 - } 1921 - 1922 - emit_imm(RV_REG_T1, off, ctx); 1923 - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1924 - insns_start = ctx->ninsns; 1925 - if (sign_ext) 1926 - emit(rv_lw(rd, 0, RV_REG_T1), ctx); 1927 - else 1928 - emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 1929 - insn_len = ctx->ninsns - insns_start; 1672 + insn_len = emit_load_32(sign_ext, rd, off, rs, ctx); 1930 1673 break; 1931 1674 case BPF_DW: 1932 - if (is_12b_int(off)) { 1933 - insns_start = ctx->ninsns; 1934 - emit_ld(rd, off, rs, ctx); 1935 - insn_len = ctx->ninsns - insns_start; 1936 - break; 1937 - } 1938 - 1939 - emit_imm(RV_REG_T1, off, ctx); 1940 - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1941 - insns_start = ctx->ninsns; 1942 - emit_ld(rd, 0, RV_REG_T1, ctx); 1943 - insn_len = ctx->ninsns - insns_start; 1675 + insn_len = emit_load_64(sign_ext, rd, off, rs, ctx); 1944 1676 break; 1945 1677 } 1946 1678 ··· 2023 1879 2024 1880 /* STX: *(size *)(dst + off) = src */ 2025 1881 case BPF_STX | BPF_MEM | BPF_B: 2026 - if (is_12b_int(off)) { 2027 - emit(rv_sb(rd, off, rs), ctx); 2028 - break; 2029 - } 2030 - 2031 - emit_imm(RV_REG_T1, off, ctx); 2032 - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 2033 - emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1882 + emit_store_8(rd, off, rs, ctx); 2034 1883 break; 2035 1884 case BPF_STX | BPF_MEM | BPF_H: 2036 - if (is_12b_int(off)) { 2037 - emit(rv_sh(rd, off, rs), ctx); 2038 - break; 2039 - } 2040 - 2041 - emit_imm(RV_REG_T1, off, ctx); 2042 - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 2043 - emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1885 + emit_store_16(rd, off, rs, ctx); 2044 1886 break; 2045 1887 case BPF_STX | BPF_MEM | BPF_W: 2046 - if (is_12b_int(off)) { 2047 - emit_sw(rd, off, rs, ctx); 2048 - break; 2049 - } 2050 - 2051 - emit_imm(RV_REG_T1, off, ctx); 2052 - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 2053 - emit_sw(RV_REG_T1, 0, rs, ctx); 1888 + emit_store_32(rd, off, rs, ctx); 2054 1889 break; 2055 1890 case BPF_STX | BPF_MEM | BPF_DW: 2056 - if (is_12b_int(off)) { 2057 - emit_sd(rd, off, rs, ctx); 2058 - break; 2059 - } 2060 - 2061 - emit_imm(RV_REG_T1, off, ctx); 2062 - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 2063 - emit_sd(RV_REG_T1, 0, rs, ctx); 1891 + emit_store_64(rd, off, rs, ctx); 2064 1892 break; 1893 + case BPF_STX | BPF_ATOMIC | BPF_B: 1894 + case BPF_STX | BPF_ATOMIC | BPF_H: 2065 1895 case BPF_STX | BPF_ATOMIC | BPF_W: 2066 1896 case BPF_STX | BPF_ATOMIC | BPF_DW: 2067 - emit_atomic(rd, rs, off, imm, 2068 - BPF_SIZE(code) == BPF_DW, ctx); 1897 + if (bpf_atomic_is_load_store(insn)) 1898 + ret = emit_atomic_ld_st(rd, rs, insn, ctx); 1899 + else 1900 + ret = emit_atomic_rmw(rd, rs, insn, ctx); 1901 + if (ret) 1902 + return ret; 2069 1903 break; 2070 1904 2071 1905 case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
+1 -2
arch/riscv/net/bpf_jit_core.c
··· 26 26 int ret; 27 27 28 28 ret = bpf_jit_emit_insn(insn, ctx, extra_pass); 29 - /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */ 30 29 if (ret > 0) 31 - i++; 30 + i++; /* skip the next instruction */ 32 31 if (offset) 33 32 offset[i] = ctx->ninsns; 34 33 if (ret < 0)
+6 -6
kernel/bpf/verifier.c
··· 3649 3649 case BPF_ST: 3650 3650 return -1; 3651 3651 case BPF_STX: 3652 - if ((BPF_MODE(insn->code) == BPF_ATOMIC || 3653 - BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && 3654 - (insn->imm & BPF_FETCH)) { 3652 + if (BPF_MODE(insn->code) == BPF_ATOMIC || 3653 + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { 3655 3654 if (insn->imm == BPF_CMPXCHG) 3656 3655 return BPF_REG_0; 3657 - else 3656 + else if (insn->imm == BPF_LOAD_ACQ) 3657 + return insn->dst_reg; 3658 + else if (insn->imm & BPF_FETCH) 3658 3659 return insn->src_reg; 3659 - } else { 3660 - return -1; 3661 3660 } 3661 + return -1; 3662 3662 default: 3663 3663 return insn->dst_reg; 3664 3664 }
+3 -2
tools/testing/selftests/bpf/progs/bpf_misc.h
··· 225 225 #define CAN_USE_BPF_ST 226 226 #endif 227 227 228 - #if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \ 229 - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) 228 + #if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \ 229 + (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ 230 + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) 230 231 #define CAN_USE_LOAD_ACQ_STORE_REL 231 232 #endif 232 233
+32 -16
tools/testing/selftests/bpf/progs/verifier_load_acquire.c
··· 10 10 11 11 SEC("socket") 12 12 __description("load-acquire, 8-bit") 13 - __success __success_unpriv __retval(0x12) 13 + __success __success_unpriv __retval(0) 14 14 __naked void load_acquire_8(void) 15 15 { 16 16 asm volatile ( 17 - "w1 = 0x12;" 17 + "r0 = 0;" 18 + "w1 = 0xfe;" 18 19 "*(u8 *)(r10 - 1) = w1;" 19 - ".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r10 - 1)); 20 + ".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1)); 21 + "if r2 == r1 goto 1f;" 22 + "r0 = 1;" 23 + "1:" 20 24 "exit;" 21 25 : 22 26 : __imm_insn(load_acquire_insn, 23 - BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -1)) 27 + BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -1)) 24 28 : __clobber_all); 25 29 } 26 30 27 31 SEC("socket") 28 32 __description("load-acquire, 16-bit") 29 - __success __success_unpriv __retval(0x1234) 33 + __success __success_unpriv __retval(0) 30 34 __naked void load_acquire_16(void) 31 35 { 32 36 asm volatile ( 33 - "w1 = 0x1234;" 37 + "r0 = 0;" 38 + "w1 = 0xfedc;" 34 39 "*(u16 *)(r10 - 2) = w1;" 35 - ".8byte %[load_acquire_insn];" // w0 = load_acquire((u16 *)(r10 - 2)); 40 + ".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2)); 41 + "if r2 == r1 goto 1f;" 42 + "r0 = 1;" 43 + "1:" 36 44 "exit;" 37 45 : 38 46 : __imm_insn(load_acquire_insn, 39 - BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -2)) 47 + BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -2)) 40 48 : __clobber_all); 41 49 } 42 50 43 51 SEC("socket") 44 52 __description("load-acquire, 32-bit") 45 - __success __success_unpriv __retval(0x12345678) 53 + __success __success_unpriv __retval(0) 46 54 __naked void load_acquire_32(void) 47 55 { 48 56 asm volatile ( 49 - "w1 = 0x12345678;" 57 + "r0 = 0;" 58 + "w1 = 0xfedcba09;" 50 59 "*(u32 *)(r10 - 4) = w1;" 51 - ".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 4)); 60 + ".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4)); 61 + "if r2 == r1 goto 1f;" 62 + "r0 = 1;" 63 + "1:" 52 64 "exit;" 53 65 : 54 66 : __imm_insn(load_acquire_insn, 55 - BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -4)) 67 + BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -4)) 56 68 : __clobber_all); 57 69 } 58 70 59 71 SEC("socket") 60 72 __description("load-acquire, 64-bit") 61 - __success __success_unpriv __retval(0x1234567890abcdef) 73 + __success __success_unpriv __retval(0) 62 74 __naked void load_acquire_64(void) 63 75 { 64 76 asm volatile ( 65 - "r1 = 0x1234567890abcdef ll;" 77 + "r0 = 0;" 78 + "r1 = 0xfedcba0987654321 ll;" 66 79 "*(u64 *)(r10 - 8) = r1;" 67 - ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r10 - 8)); 80 + ".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8)); 81 + "if r2 == r1 goto 1f;" 82 + "r0 = 1;" 83 + "1:" 68 84 "exit;" 69 85 : 70 86 : __imm_insn(load_acquire_insn, 71 - BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8)) 87 + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8)) 72 88 : __clobber_all); 73 89 } 74 90
+2 -3
tools/testing/selftests/bpf/progs/verifier_precision.c
··· 91 91 ::: __clobber_all); 92 92 } 93 93 94 - #if defined(ENABLE_ATOMICS_TESTS) && \ 95 - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) 94 + #ifdef CAN_USE_LOAD_ACQ_STORE_REL 96 95 97 96 SEC("?raw_tp") 98 97 __success __log_level(2) ··· 137 138 : __clobber_all); 138 139 } 139 140 140 - #endif /* load-acquire, store-release */ 141 + #endif /* CAN_USE_LOAD_ACQ_STORE_REL */ 141 142 #endif /* v4 instruction */ 142 143 143 144 SEC("?raw_tp")
+27 -12
tools/testing/selftests/bpf/progs/verifier_store_release.c
··· 6 6 #include "../../../include/linux/filter.h" 7 7 #include "bpf_misc.h" 8 8 9 - #if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \ 10 - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) 9 + #ifdef CAN_USE_LOAD_ACQ_STORE_REL 11 10 12 11 SEC("socket") 13 12 __description("store-release, 8-bit") 14 - __success __success_unpriv __retval(0x12) 13 + __success __success_unpriv __retval(0) 15 14 __naked void store_release_8(void) 16 15 { 17 16 asm volatile ( 17 + "r0 = 0;" 18 18 "w1 = 0x12;" 19 19 ".8byte %[store_release_insn];" // store_release((u8 *)(r10 - 1), w1); 20 - "w0 = *(u8 *)(r10 - 1);" 20 + "w2 = *(u8 *)(r10 - 1);" 21 + "if r2 == r1 goto 1f;" 22 + "r0 = 1;" 23 + "1:" 21 24 "exit;" 22 25 : 23 26 : __imm_insn(store_release_insn, ··· 30 27 31 28 SEC("socket") 32 29 __description("store-release, 16-bit") 33 - __success __success_unpriv __retval(0x1234) 30 + __success __success_unpriv __retval(0) 34 31 __naked void store_release_16(void) 35 32 { 36 33 asm volatile ( 34 + "r0 = 0;" 37 35 "w1 = 0x1234;" 38 36 ".8byte %[store_release_insn];" // store_release((u16 *)(r10 - 2), w1); 39 - "w0 = *(u16 *)(r10 - 2);" 37 + "w2 = *(u16 *)(r10 - 2);" 38 + "if r2 == r1 goto 1f;" 39 + "r0 = 1;" 40 + "1:" 40 41 "exit;" 41 42 : 42 43 : __imm_insn(store_release_insn, ··· 50 43 51 44 SEC("socket") 52 45 __description("store-release, 32-bit") 53 - __success __success_unpriv __retval(0x12345678) 46 + __success __success_unpriv __retval(0) 54 47 __naked void store_release_32(void) 55 48 { 56 49 asm volatile ( 50 + "r0 = 0;" 57 51 "w1 = 0x12345678;" 58 52 ".8byte %[store_release_insn];" // store_release((u32 *)(r10 - 4), w1); 59 - "w0 = *(u32 *)(r10 - 4);" 53 + "w2 = *(u32 *)(r10 - 4);" 54 + "if r2 == r1 goto 1f;" 55 + "r0 = 1;" 56 + "1:" 60 57 "exit;" 61 58 : 62 59 : __imm_insn(store_release_insn, ··· 70 59 71 60 SEC("socket") 72 61 __description("store-release, 64-bit") 73 - __success __success_unpriv __retval(0x1234567890abcdef) 62 + __success __success_unpriv __retval(0) 74 63 __naked void store_release_64(void) 75 64 { 76 65 asm volatile ( 66 + "r0 = 0;" 77 67 "r1 = 0x1234567890abcdef ll;" 78 68 ".8byte %[store_release_insn];" // store_release((u64 *)(r10 - 8), r1); 79 - "r0 = *(u64 *)(r10 - 8);" 69 + "r2 = *(u64 *)(r10 - 8);" 70 + "if r2 == r1 goto 1f;" 71 + "r0 = 1;" 72 + "1:" 80 73 "exit;" 81 74 : 82 75 : __imm_insn(store_release_insn, ··· 286 271 : __clobber_all); 287 272 } 288 273 289 - #else 274 + #else /* CAN_USE_LOAD_ACQ_STORE_REL */ 290 275 291 276 SEC("socket") 292 277 __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test") ··· 296 281 return 0; 297 282 } 298 283 299 - #endif 284 + #endif /* CAN_USE_LOAD_ACQ_STORE_REL */ 300 285 301 286 char _license[] SEC("license") = "GPL";