LoongArch: BPF: Implement PROBE_MEM32 pseudo instructions

Add support for `{LDX,STX,ST} | PROBE_MEM32 | {B,H,W,DW}` instructions.
They are similar to PROBE_MEM instructions with the following differences:
* PROBE_MEM32 supports store.
* PROBE_MEM32 relies on the verifier to clear upper 32-bit of the
src/dst register
* PROBE_MEM32 adds 64-bit kern_vm_start address (which is stored in S6
in the prologue). Due to bpf_arena constructions such S6 + reg +
off16 access is guaranteed to be within arena virtual range, so no
address check at run-time.
* S6 is a free callee-saved register, so it is used to store arena_vm_start
* PROBE_MEM32 allows ST and STX. If they fault the store is a nop. When
LDX faults the destination register is zeroed.

To support these on LoongArch, we employ the t2/t3 registers to store the
intermediate results of reg_arena + src/dst reg and use the t2/t3 registers
as the new src/dst reg. This allows us to reuse most of the existing code.

Acked-by: Tiezhu Yang <yangtiezhu@loongson.cn>
Tested-by: Vincent Li <vincent.mc.li@gmail.com>
Signed-off-by: Hengqi Chen <hengqi.chen@gmail.com>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>

authored by

Hengqi Chen and committed by

Huacai Chen 2 months ago ef54c517 4ab17e76

+68 -6

2 changed files

expand all

arch

loongarch

net

bpf_jit.c

bpf_jit.h

+67 -6

arch/loongarch/net/bpf_jit.c

··· 17 17 #define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) 18 18 19 19 #define REG_TCC LOONGARCH_GPR_A6 20 + #define REG_ARENA LOONGARCH_GPR_S6 /* For storing arena_vm_start */ 20 21 #define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80) 21 22 22 23 static const int regmap[] = { ··· 137 136 /* To store tcc and tcc_ptr */ 138 137 stack_adjust += sizeof(long) * 2; 139 138 139 + if (ctx->arena_vm_start) 140 + stack_adjust += 8; 141 + 140 142 stack_adjust = round_up(stack_adjust, 16); 141 143 stack_adjust += bpf_stack_adjust; 142 144 ··· 182 178 store_offset -= sizeof(long); 183 179 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); 184 180 181 + if (ctx->arena_vm_start) { 182 + store_offset -= sizeof(long); 183 + emit_insn(ctx, std, REG_ARENA, LOONGARCH_GPR_SP, store_offset); 184 + } 185 + 185 186 prepare_bpf_tail_call_cnt(ctx, &store_offset); 186 187 187 188 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); ··· 195 186 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); 196 187 197 188 ctx->stack_size = stack_adjust; 189 + 190 + if (ctx->arena_vm_start) 191 + move_imm(ctx, REG_ARENA, ctx->arena_vm_start, false); 198 192 } 199 193 200 194 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) ··· 228 216 229 217 load_offset -= sizeof(long); 230 218 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); 219 + 220 + if (ctx->arena_vm_start) { 221 + load_offset -= sizeof(long); 222 + emit_insn(ctx, ldd, REG_ARENA, LOONGARCH_GPR_SP, load_offset); 223 + } 231 224 232 225 /* 233 226 * When push into the stack, follow the order of tcc then tcc_ptr. ··· 459 442 460 443 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 461 444 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 445 + #define REG_DONT_CLEAR_MARKER 0 462 446 463 447 bool ex_handler_bpf(const struct exception_table_entry *ex, 464 448 struct pt_regs *regs) ··· 467 449 int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 468 450 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 469 451 470 - regs->regs[dst_reg] = 0; 452 + if (dst_reg != REG_DONT_CLEAR_MARKER) 453 + regs->regs[dst_reg] = 0; 471 454 regs->csr_era = (unsigned long)&ex->fixup - offset; 472 455 473 456 return true; ··· 487 468 return 0; 488 469 489 470 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 490 - BPF_MODE(insn->code) != BPF_PROBE_MEMSX) 471 + BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 472 + BPF_MODE(insn->code) != BPF_PROBE_MEM32) 491 473 return 0; 492 474 493 475 if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries)) ··· 548 528 const u8 cond = BPF_OP(code); 549 529 const u8 t1 = LOONGARCH_GPR_T1; 550 530 const u8 t2 = LOONGARCH_GPR_T2; 551 - const u8 src = regmap[insn->src_reg]; 552 - const u8 dst = regmap[insn->dst_reg]; 531 + const u8 t3 = LOONGARCH_GPR_T3; 532 + u8 src = regmap[insn->src_reg]; 533 + u8 dst = regmap[insn->dst_reg]; 553 534 const s16 off = insn->off; 554 535 const s32 imm = insn->imm; 555 536 const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32; ··· 1056 1035 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1057 1036 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1058 1037 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1059 - sign_extend = BPF_MODE(insn->code) == BPF_MEMSX || 1060 - BPF_MODE(insn->code) == BPF_PROBE_MEMSX; 1038 + /* LDX | PROBE_MEM32: dst = *(unsigned size *)(src + REG_ARENA + off) */ 1039 + case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1040 + case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1041 + case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1042 + case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1043 + sign_extend = BPF_MODE(code) == BPF_MEMSX || 1044 + BPF_MODE(code) == BPF_PROBE_MEMSX; 1045 + 1046 + if (BPF_MODE(code) == BPF_PROBE_MEM32) { 1047 + emit_insn(ctx, addd, t2, src, REG_ARENA); 1048 + src = t2; 1049 + } 1050 + 1061 1051 switch (BPF_SIZE(code)) { 1062 1052 case BPF_B: 1063 1053 if (is_signed_imm12(off)) { ··· 1128 1096 case BPF_ST | BPF_MEM | BPF_H: 1129 1097 case BPF_ST | BPF_MEM | BPF_W: 1130 1098 case BPF_ST | BPF_MEM | BPF_DW: 1099 + /* ST | PROBE_MEM32: *(size *)(dst + REG_ARENA + off) = imm */ 1100 + case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1101 + case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1102 + case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1103 + case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1104 + if (BPF_MODE(code) == BPF_PROBE_MEM32) { 1105 + emit_insn(ctx, addd, t3, dst, REG_ARENA); 1106 + dst = t3; 1107 + } 1108 + 1131 1109 switch (BPF_SIZE(code)) { 1132 1110 case BPF_B: 1133 1111 move_imm(ctx, t1, imm, is32); ··· 1180 1138 } 1181 1139 break; 1182 1140 } 1141 + 1142 + ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER); 1143 + if (ret) 1144 + return ret; 1183 1145 break; 1184 1146 1185 1147 /* *(size *)(dst + off) = src */ ··· 1191 1145 case BPF_STX | BPF_MEM | BPF_H: 1192 1146 case BPF_STX | BPF_MEM | BPF_W: 1193 1147 case BPF_STX | BPF_MEM | BPF_DW: 1148 + /* STX | PROBE_MEM32: *(size *)(dst + REG_ARENA + off) = src */ 1149 + case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1150 + case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1151 + case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1152 + case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1153 + if (BPF_MODE(code) == BPF_PROBE_MEM32) { 1154 + emit_insn(ctx, addd, t2, dst, REG_ARENA); 1155 + dst = t2; 1156 + } 1157 + 1194 1158 switch (BPF_SIZE(code)) { 1195 1159 case BPF_B: 1196 1160 if (is_signed_imm12(off)) { ··· 1239 1183 } 1240 1184 break; 1241 1185 } 1186 + 1187 + ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER); 1188 + if (ret) 1189 + return ret; 1242 1190 break; 1243 1191 1244 1192 case BPF_STX | BPF_ATOMIC | BPF_W: ··· 1954 1894 1955 1895 memset(&ctx, 0, sizeof(ctx)); 1956 1896 ctx.prog = prog; 1897 + ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); 1957 1898 1958 1899 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); 1959 1900 if (ctx.offset == NULL) {

arch/loongarch/net/bpf_jit.h

··· 20 20 union loongarch_instruction *image; 21 21 union loongarch_instruction *ro_image; 22 22 u32 stack_size; 23 + u64 arena_vm_start; 23 24 }; 24 25 25 26 struct jit_data {

Configure Feed

Configure Feed