Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
riscv, bpf: Optimize cmpxchg insn with Zacas support
Optimize cmpxchg instruction with amocas.w and amocas.d
Zacas instructions.
Signed-off-by: Pu Lehui <pulehui@huawei.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Tested-by: Björn Töpel <bjorn@rivosinc.com>
Reviewed-by: Björn Töpel <bjorn@rivosinc.com>
Acked-by: Björn Töpel <bjorn@kernel.org>
Link: https://lore.kernel.org/bpf/20250719091730.2660197-8-pulehui@huaweicloud.com
authored by
Pu Lehui
and committed by
Daniel Borkmann
1c0196b8
de39d2c4
+29
-16
+27
arch/riscv/net/bpf_jit.h
+27
arch/riscv/net/bpf_jit.h
···
1294
1294
emit_mv(rd, RV_REG_T2, ctx);
1295
1295
}
1296
1296
1297
+
static inline void emit_cmpxchg(u8 rd, u8 rs, u8 r0, bool is64, struct rv_jit_context *ctx)
1298
+
{
1299
+
int jmp_offset;
1300
+
1301
+
if (rv_ext_enabled(ZACAS)) {
1302
+
emit(is64 ? rvzacas_amocas_d(r0, rs, rd, 1, 1) :
1303
+
rvzacas_amocas_w(r0, rs, rd, 1, 1), ctx);
1304
+
if (!is64)
1305
+
emit_zextw(r0, r0, ctx);
1306
+
return;
1307
+
}
1308
+
1309
+
if (is64)
1310
+
emit_mv(RV_REG_T2, r0, ctx);
1311
+
else
1312
+
emit_addiw(RV_REG_T2, r0, 0, ctx);
1313
+
emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
1314
+
rv_lr_w(r0, 0, rd, 0, 0), ctx);
1315
+
jmp_offset = ninsns_rvoff(8);
1316
+
emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);
1317
+
emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) :
1318
+
rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx);
1319
+
jmp_offset = ninsns_rvoff(-6);
1320
+
emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
1321
+
emit_fence_rw_rw(ctx);
1322
+
}
1323
+
1297
1324
#endif /* __riscv_xlen == 64 */
1298
1325
1299
1326
void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog);
+2
-16
arch/riscv/net/bpf_jit_comp64.c
+2
-16
arch/riscv/net/bpf_jit_comp64.c
···
599
599
static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn,
600
600
struct rv_jit_context *ctx)
601
601
{
602
-
u8 r0, code = insn->code;
602
+
u8 code = insn->code;
603
603
s16 off = insn->off;
604
604
s32 imm = insn->imm;
605
-
int jmp_offset;
606
605
bool is64;
607
606
608
607
if (BPF_SIZE(code) != BPF_W && BPF_SIZE(code) != BPF_DW) {
···
672
673
break;
673
674
/* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */
674
675
case BPF_CMPXCHG:
675
-
r0 = bpf_to_rv_reg(BPF_REG_0, ctx);
676
-
if (is64)
677
-
emit_mv(RV_REG_T2, r0, ctx);
678
-
else
679
-
emit_addiw(RV_REG_T2, r0, 0, ctx);
680
-
emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
681
-
rv_lr_w(r0, 0, rd, 0, 0), ctx);
682
-
jmp_offset = ninsns_rvoff(8);
683
-
emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);
684
-
emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) :
685
-
rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx);
686
-
jmp_offset = ninsns_rvoff(-6);
687
-
emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
688
-
emit_fence_rw_rw(ctx);
676
+
emit_cmpxchg(rd, rs, regmap[BPF_REG_0], is64, ctx);
689
677
break;
690
678
default:
691
679
pr_err_once("bpf-jit: invalid atomic RMW opcode %02x\n", imm);