riscv, bpf: Add support arena atomics for RV64

+59 -3

2 changed files

expand all

arch

riscv

net

bpf_jit.h

bpf_jit_comp64.c

arch/riscv/net/bpf_jit.h

··· 1301 1301 int jmp_offset; 1302 1302 1303 1303 if (rv_ext_enabled(ZACAS)) { 1304 + ctx->ex_insn_off = ctx->ninsns; 1304 1305 emit(is64 ? rvzacas_amocas_d(r0, rs, rd, 1, 1) : 1305 1306 rvzacas_amocas_w(r0, rs, rd, 1, 1), ctx); 1307 + ctx->ex_jmp_off = ctx->ninsns; 1306 1308 if (!is64) 1307 1309 emit_zextw(r0, r0, ctx); 1308 1310 return;

+57 -3

arch/riscv/net/bpf_jit_comp64.c

··· 571 571 switch (imm) { 572 572 /* dst_reg = load_acquire(src_reg + off16) */ 573 573 case BPF_LOAD_ACQ: 574 + if (BPF_MODE(code) == BPF_PROBE_ATOMIC) { 575 + emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx); 576 + rs = RV_REG_T2; 577 + } 578 + 574 579 emit_ldx(rd, off, rs, BPF_SIZE(code), false, ctx); 575 580 emit_fence_r_rw(ctx); 576 581 ··· 587 582 break; 588 583 /* store_release(dst_reg + off16, src_reg) */ 589 584 case BPF_STORE_REL: 585 + if (BPF_MODE(code) == BPF_PROBE_ATOMIC) { 586 + emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx); 587 + rd = RV_REG_T2; 588 + } 589 + 590 590 emit_fence_rw_w(ctx); 591 591 emit_stx(rd, off, rs, BPF_SIZE(code), ctx); 592 592 break; ··· 609 599 u8 code = insn->code; 610 600 s16 off = insn->off; 611 601 s32 imm = insn->imm; 612 - bool is64; 602 + bool is64 = BPF_SIZE(code) == BPF_DW; 613 603 614 604 if (BPF_SIZE(code) != BPF_W && BPF_SIZE(code) != BPF_DW) { 615 605 pr_err_once("bpf-jit: 1- and 2-byte RMW atomics are not supported\n"); 616 606 return -EINVAL; 617 607 } 618 - is64 = BPF_SIZE(code) == BPF_DW; 619 608 620 609 if (off) { 621 610 if (is_12b_int(off)) { ··· 626 617 rd = RV_REG_T1; 627 618 } 628 619 620 + if (BPF_MODE(code) == BPF_PROBE_ATOMIC) { 621 + emit_add(RV_REG_T1, rd, RV_REG_ARENA, ctx); 622 + rd = RV_REG_T1; 623 + } 624 + 629 625 switch (imm) { 630 626 /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */ 631 627 case BPF_ADD: 628 + ctx->ex_insn_off = ctx->ninsns; 632 629 emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) : 633 630 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 631 + ctx->ex_jmp_off = ctx->ninsns; 634 632 break; 635 633 case BPF_AND: 634 + ctx->ex_insn_off = ctx->ninsns; 636 635 emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) : 637 636 rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 637 + ctx->ex_jmp_off = ctx->ninsns; 638 638 break; 639 639 case BPF_OR: 640 + ctx->ex_insn_off = ctx->ninsns; 640 641 emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) : 641 642 rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 643 + ctx->ex_jmp_off = ctx->ninsns; 642 644 break; 643 645 case BPF_XOR: 646 + ctx->ex_insn_off = ctx->ninsns; 644 647 emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) : 645 648 rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 649 + ctx->ex_jmp_off = ctx->ninsns; 646 650 break; 647 651 /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */ 648 652 case BPF_ADD | BPF_FETCH: 653 + ctx->ex_insn_off = ctx->ninsns; 649 654 emit(is64 ? rv_amoadd_d(rs, rs, rd, 1, 1) : 650 655 rv_amoadd_w(rs, rs, rd, 1, 1), ctx); 656 + ctx->ex_jmp_off = ctx->ninsns; 651 657 if (!is64) 652 658 emit_zextw(rs, rs, ctx); 653 659 break; 654 660 case BPF_AND | BPF_FETCH: 661 + ctx->ex_insn_off = ctx->ninsns; 655 662 emit(is64 ? rv_amoand_d(rs, rs, rd, 1, 1) : 656 663 rv_amoand_w(rs, rs, rd, 1, 1), ctx); 664 + ctx->ex_jmp_off = ctx->ninsns; 657 665 if (!is64) 658 666 emit_zextw(rs, rs, ctx); 659 667 break; 660 668 case BPF_OR | BPF_FETCH: 669 + ctx->ex_insn_off = ctx->ninsns; 661 670 emit(is64 ? rv_amoor_d(rs, rs, rd, 1, 1) : 662 671 rv_amoor_w(rs, rs, rd, 1, 1), ctx); 672 + ctx->ex_jmp_off = ctx->ninsns; 663 673 if (!is64) 664 674 emit_zextw(rs, rs, ctx); 665 675 break; 666 676 case BPF_XOR | BPF_FETCH: 677 + ctx->ex_insn_off = ctx->ninsns; 667 678 emit(is64 ? rv_amoxor_d(rs, rs, rd, 1, 1) : 668 679 rv_amoxor_w(rs, rs, rd, 1, 1), ctx); 680 + ctx->ex_jmp_off = ctx->ninsns; 669 681 if (!is64) 670 682 emit_zextw(rs, rs, ctx); 671 683 break; 672 684 /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */ 673 685 case BPF_XCHG: 686 + ctx->ex_insn_off = ctx->ninsns; 674 687 emit(is64 ? rv_amoswap_d(rs, rs, rd, 1, 1) : 675 688 rv_amoswap_w(rs, rs, rd, 1, 1), ctx); 689 + ctx->ex_jmp_off = ctx->ninsns; 676 690 if (!is64) 677 691 emit_zextw(rs, rs, ctx); 678 692 break; ··· 743 711 744 712 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 745 713 BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 746 - BPF_MODE(insn->code) != BPF_PROBE_MEM32) 714 + BPF_MODE(insn->code) != BPF_PROBE_MEM32 && 715 + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 747 716 return 0; 748 717 749 718 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) ··· 1874 1841 return ret; 1875 1842 break; 1876 1843 1844 + /* Atomics */ 1877 1845 case BPF_STX | BPF_ATOMIC | BPF_B: 1878 1846 case BPF_STX | BPF_ATOMIC | BPF_H: 1879 1847 case BPF_STX | BPF_ATOMIC | BPF_W: 1880 1848 case BPF_STX | BPF_ATOMIC | BPF_DW: 1849 + case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: 1850 + case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: 1851 + case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: 1852 + case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: 1881 1853 if (bpf_atomic_is_load_store(insn)) 1882 1854 ret = emit_atomic_ld_st(rd, rs, insn, ctx); 1883 1855 else 1884 1856 ret = emit_atomic_rmw(rd, rs, insn, ctx); 1857 + 1858 + ret = ret ?: add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); 1885 1859 if (ret) 1886 1860 return ret; 1887 1861 break; ··· 2016 1976 2017 1977 bool bpf_jit_supports_arena(void) 2018 1978 { 1979 + return true; 1980 + } 1981 + 1982 + bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 1983 + { 1984 + if (in_arena) { 1985 + switch (insn->code) { 1986 + case BPF_STX | BPF_ATOMIC | BPF_W: 1987 + case BPF_STX | BPF_ATOMIC | BPF_DW: 1988 + if (insn->imm == BPF_CMPXCHG) 1989 + return rv_ext_enabled(ZACAS); 1990 + } 1991 + } 1992 + 2019 1993 return true; 2020 1994 } 2021 1995

Configure Feed

Configure Feed