bpf, arm64: Add support for lse atomics in bpf_arena

When LSE atomics are available, BPF atomic instructions are implemented
as single ARM64 atomic instructions, therefore it is easy to enable
these in bpf_arena using the currently available exception handling
setup.

LL_SC atomics use loops and therefore would need more work to enable in
bpf_arena.

Enable LSE atomics based instructions in bpf_arena and use the
bpf_jit_supports_insn() callback to reject atomics in bpf_arena if LSE
atomics are not available.

All atomics and arena_atomics selftests are passing:

[root@ip-172-31-2-216 bpf]# ./test_progs -a atomics,arena_atomics
#3/1 arena_atomics/add:OK
#3/2 arena_atomics/sub:OK
#3/3 arena_atomics/and:OK
#3/4 arena_atomics/or:OK
#3/5 arena_atomics/xor:OK
#3/6 arena_atomics/cmpxchg:OK
#3/7 arena_atomics/xchg:OK
#3 arena_atomics:OK
#10/1 atomics/add:OK
#10/2 atomics/sub:OK
#10/3 atomics/and:OK
#10/4 atomics/or:OK
#10/5 atomics/xor:OK
#10/6 atomics/cmpxchg:OK
#10/7 atomics/xchg:OK
#10 atomics:OK
Summary: 2/14 PASSED, 0 SKIPPED, 0 FAILED

Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
Link: https://lore.kernel.org/r/20240426161116.441-1-puranjay@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

authored by

Puranjay Mohan and committed by

Alexei Starovoitov 2 years ago e612b5c1 7e2c7a3f

+40 -9

2 changed files

expand all

arch

arm64

net

bpf_jit_comp.c

tools

testing

selftests

bpf

DENYLIST.aarch64

+40 -8

arch/arm64/net/bpf_jit_comp.c

··· 494 494 static int emit_lse_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) 495 495 { 496 496 const u8 code = insn->code; 497 + const u8 arena_vm_base = bpf2a64[ARENA_VM_START]; 497 498 const u8 dst = bpf2a64[insn->dst_reg]; 498 499 const u8 src = bpf2a64[insn->src_reg]; 499 500 const u8 tmp = bpf2a64[TMP_REG_1]; 500 501 const u8 tmp2 = bpf2a64[TMP_REG_2]; 501 502 const bool isdw = BPF_SIZE(code) == BPF_DW; 503 + const bool arena = BPF_MODE(code) == BPF_PROBE_ATOMIC; 502 504 const s16 off = insn->off; 503 - u8 reg; 505 + u8 reg = dst; 504 506 505 - if (!off) { 506 - reg = dst; 507 - } else { 508 - emit_a64_mov_i(1, tmp, off, ctx); 509 - emit(A64_ADD(1, tmp, tmp, dst), ctx); 510 - reg = tmp; 507 + if (off || arena) { 508 + if (off) { 509 + emit_a64_mov_i(1, tmp, off, ctx); 510 + emit(A64_ADD(1, tmp, tmp, dst), ctx); 511 + reg = tmp; 512 + } 513 + if (arena) { 514 + emit(A64_ADD(1, tmp, reg, arena_vm_base), ctx); 515 + reg = tmp; 516 + } 511 517 } 512 518 513 519 switch (insn->imm) { ··· 581 575 const bool isdw = BPF_SIZE(code) == BPF_DW; 582 576 u8 reg; 583 577 s32 jmp_offset; 578 + 579 + if (BPF_MODE(code) == BPF_PROBE_ATOMIC) { 580 + /* ll_sc based atomics don't support unsafe pointers yet. */ 581 + pr_err_once("unknown atomic opcode %02x\n", code); 582 + return -EINVAL; 583 + } 584 584 585 585 if (!off) { 586 586 reg = dst; ··· 789 777 790 778 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 791 779 BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 792 - BPF_MODE(insn->code) != BPF_PROBE_MEM32) 780 + BPF_MODE(insn->code) != BPF_PROBE_MEM32 && 781 + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 793 782 return 0; 794 783 795 784 if (!ctx->prog->aux->extable || ··· 1487 1474 1488 1475 case BPF_STX | BPF_ATOMIC | BPF_W: 1489 1476 case BPF_STX | BPF_ATOMIC | BPF_DW: 1477 + case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: 1478 + case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: 1490 1479 if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS)) 1491 1480 ret = emit_lse_atomic(insn, ctx); 1492 1481 else 1493 1482 ret = emit_ll_sc_atomic(insn, ctx); 1483 + if (ret) 1484 + return ret; 1485 + 1486 + ret = add_exception_handler(insn, ctx, dst); 1494 1487 if (ret) 1495 1488 return ret; 1496 1489 break; ··· 2543 2524 2544 2525 bool bpf_jit_supports_arena(void) 2545 2526 { 2527 + return true; 2528 + } 2529 + 2530 + bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 2531 + { 2532 + if (!in_arena) 2533 + return true; 2534 + switch (insn->code) { 2535 + case BPF_STX | BPF_ATOMIC | BPF_W: 2536 + case BPF_STX | BPF_ATOMIC | BPF_DW: 2537 + if (!cpus_have_cap(ARM64_HAS_LSE_ATOMICS)) 2538 + return false; 2539 + } 2546 2540 return true; 2547 2541 } 2548 2542

-1

tools/testing/selftests/bpf/DENYLIST.aarch64

··· 10 10 fill_link_info/kretprobe_multi_link_info # bpf_program__attach_kprobe_multi_opts unexpected error: -95 11 11 fill_link_info/kprobe_multi_invalid_ubuff # bpf_program__attach_kprobe_multi_opts unexpected error: -95 12 12 missed/kprobe_recursion # missed_kprobe_recursion__attach unexpected error: -95 (errno 95) 13 - arena_atomics

Configure Feed

Configure Feed