Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
powerpc64/bpf: Introduce bpf_jit_emit_atomic_ops() to emit atomic instructions
The existing code for emitting bpf atomic instruction sequences for
atomic operations such as XCHG, CMPXCHG, ADD, AND, OR, and XOR has been
refactored into a reusable function, bpf_jit_emit_ppc_atomic_op().
It also computes the jump offset and tracks the instruction index for jited
LDARX/LWARX to be used in case it causes a fault.
Reviewed-by: Hari Bathini <hbathini@linux.ibm.com>
Tested-by: Venkat Rao Bagalkote <venkat88@linux.ibm.com>
Signed-off-by: Saket Kumar Bhaskar <skb99@linux.ibm.com>
Signed-off-by: Madhavan Srinivasan <maddy@linux.ibm.com>
Link: https://patch.msgid.link/20250904100835.1100423-4-skb99@linux.ibm.com
authored by
Saket Kumar Bhaskar
and committed by
Madhavan Srinivasan
45ed2e8b
a2485d06
+115
-88
+115
-88
arch/powerpc/net/bpf_jit_comp64.c
+115
-88
arch/powerpc/net/bpf_jit_comp64.c
···
423
423
" blr ;"
424
424
);
425
425
426
+
static int bpf_jit_emit_atomic_ops(u32 *image, struct codegen_context *ctx,
427
+
const struct bpf_insn *insn, u32 *jmp_off,
428
+
u32 *tmp_idx, u32 *addrp)
429
+
{
430
+
u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
431
+
u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
432
+
u32 size = BPF_SIZE(insn->code);
433
+
u32 src_reg = bpf_to_ppc(insn->src_reg);
434
+
u32 dst_reg = bpf_to_ppc(insn->dst_reg);
435
+
s32 imm = insn->imm;
436
+
437
+
u32 save_reg = tmp2_reg;
438
+
u32 ret_reg = src_reg;
439
+
u32 fixup_idx;
440
+
441
+
/* Get offset into TMP_REG_1 */
442
+
EMIT(PPC_RAW_LI(tmp1_reg, insn->off));
443
+
/*
444
+
* Enforce full ordering for operations with BPF_FETCH by emitting a 'sync'
445
+
* before and after the operation.
446
+
*
447
+
* This is a requirement in the Linux Kernel Memory Model.
448
+
* See __cmpxchg_u64() in asm/cmpxchg.h as an example.
449
+
*/
450
+
if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))
451
+
EMIT(PPC_RAW_SYNC());
452
+
453
+
*tmp_idx = ctx->idx;
454
+
455
+
/* load value from memory into TMP_REG_2 */
456
+
if (size == BPF_DW)
457
+
EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0));
458
+
else
459
+
EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0));
460
+
/* Save old value in _R0 */
461
+
if (imm & BPF_FETCH)
462
+
EMIT(PPC_RAW_MR(_R0, tmp2_reg));
463
+
464
+
switch (imm) {
465
+
case BPF_ADD:
466
+
case BPF_ADD | BPF_FETCH:
467
+
EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
468
+
break;
469
+
case BPF_AND:
470
+
case BPF_AND | BPF_FETCH:
471
+
EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));
472
+
break;
473
+
case BPF_OR:
474
+
case BPF_OR | BPF_FETCH:
475
+
EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));
476
+
break;
477
+
case BPF_XOR:
478
+
case BPF_XOR | BPF_FETCH:
479
+
EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));
480
+
break;
481
+
case BPF_CMPXCHG:
482
+
/*
483
+
* Return old value in BPF_REG_0 for BPF_CMPXCHG &
484
+
* in src_reg for other cases.
485
+
*/
486
+
ret_reg = bpf_to_ppc(BPF_REG_0);
487
+
488
+
/* Compare with old value in BPF_R0 */
489
+
if (size == BPF_DW)
490
+
EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));
491
+
else
492
+
EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));
493
+
/* Don't set if different from old value */
494
+
PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
495
+
fallthrough;
496
+
case BPF_XCHG:
497
+
save_reg = src_reg;
498
+
break;
499
+
default:
500
+
return -EOPNOTSUPP;
501
+
}
502
+
503
+
/* store new value */
504
+
if (size == BPF_DW)
505
+
EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg));
506
+
else
507
+
EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg));
508
+
/* we're done if this succeeded */
509
+
PPC_BCC_SHORT(COND_NE, *tmp_idx * 4);
510
+
fixup_idx = ctx->idx;
511
+
512
+
if (imm & BPF_FETCH) {
513
+
/* Emit 'sync' to enforce full ordering */
514
+
if (IS_ENABLED(CONFIG_SMP))
515
+
EMIT(PPC_RAW_SYNC());
516
+
EMIT(PPC_RAW_MR(ret_reg, _R0));
517
+
/*
518
+
* Skip unnecessary zero-extension for 32-bit cmpxchg.
519
+
* For context, see commit 39491867ace5.
520
+
*/
521
+
if (size != BPF_DW && imm == BPF_CMPXCHG &&
522
+
insn_is_zext(insn + 1))
523
+
*addrp = ctx->idx * 4;
524
+
}
525
+
526
+
*jmp_off = (fixup_idx - *tmp_idx) * 4;
527
+
528
+
return 0;
529
+
}
530
+
426
531
static int bpf_jit_emit_probe_mem_store(struct codegen_context *ctx, u32 src_reg, s16 off,
427
532
u32 code, u32 *image)
428
533
{
···
643
538
u32 size = BPF_SIZE(code);
644
539
u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
645
540
u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
646
-
u32 save_reg, ret_reg;
647
541
s16 off = insn[i].off;
648
542
s32 imm = insn[i].imm;
649
543
bool func_addr_fixed;
···
650
546
u64 imm64;
651
547
u32 true_cond;
652
548
u32 tmp_idx;
549
+
u32 jmp_off;
653
550
654
551
/*
655
552
* addrs[] maps a BPF bytecode address into a real offset from
···
1185
1080
return -EOPNOTSUPP;
1186
1081
}
1187
1082
1188
-
save_reg = tmp2_reg;
1189
-
ret_reg = src_reg;
1190
-
1191
-
/* Get offset into TMP_REG_1 */
1192
-
EMIT(PPC_RAW_LI(tmp1_reg, off));
1193
-
/*
1194
-
* Enforce full ordering for operations with BPF_FETCH by emitting a 'sync'
1195
-
* before and after the operation.
1196
-
*
1197
-
* This is a requirement in the Linux Kernel Memory Model.
1198
-
* See __cmpxchg_u64() in asm/cmpxchg.h as an example.
1199
-
*/
1200
-
if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))
1201
-
EMIT(PPC_RAW_SYNC());
1202
-
tmp_idx = ctx->idx * 4;
1203
-
/* load value from memory into TMP_REG_2 */
1204
-
if (size == BPF_DW)
1205
-
EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0));
1206
-
else
1207
-
EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0));
1208
-
1209
-
/* Save old value in _R0 */
1210
-
if (imm & BPF_FETCH)
1211
-
EMIT(PPC_RAW_MR(_R0, tmp2_reg));
1212
-
1213
-
switch (imm) {
1214
-
case BPF_ADD:
1215
-
case BPF_ADD | BPF_FETCH:
1216
-
EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
1217
-
break;
1218
-
case BPF_AND:
1219
-
case BPF_AND | BPF_FETCH:
1220
-
EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));
1221
-
break;
1222
-
case BPF_OR:
1223
-
case BPF_OR | BPF_FETCH:
1224
-
EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));
1225
-
break;
1226
-
case BPF_XOR:
1227
-
case BPF_XOR | BPF_FETCH:
1228
-
EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));
1229
-
break;
1230
-
case BPF_CMPXCHG:
1231
-
/*
1232
-
* Return old value in BPF_REG_0 for BPF_CMPXCHG &
1233
-
* in src_reg for other cases.
1234
-
*/
1235
-
ret_reg = bpf_to_ppc(BPF_REG_0);
1236
-
1237
-
/* Compare with old value in BPF_R0 */
1238
-
if (size == BPF_DW)
1239
-
EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));
1240
-
else
1241
-
EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));
1242
-
/* Don't set if different from old value */
1243
-
PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
1244
-
fallthrough;
1245
-
case BPF_XCHG:
1246
-
save_reg = src_reg;
1247
-
break;
1248
-
default:
1249
-
pr_err_ratelimited(
1250
-
"eBPF filter atomic op code %02x (@%d) unsupported\n",
1251
-
code, i);
1252
-
return -EOPNOTSUPP;
1253
-
}
1254
-
1255
-
/* store new value */
1256
-
if (size == BPF_DW)
1257
-
EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg));
1258
-
else
1259
-
EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg));
1260
-
/* we're done if this succeeded */
1261
-
PPC_BCC_SHORT(COND_NE, tmp_idx);
1262
-
1263
-
if (imm & BPF_FETCH) {
1264
-
/* Emit 'sync' to enforce full ordering */
1265
-
if (IS_ENABLED(CONFIG_SMP))
1266
-
EMIT(PPC_RAW_SYNC());
1267
-
EMIT(PPC_RAW_MR(ret_reg, _R0));
1268
-
/*
1269
-
* Skip unnecessary zero-extension for 32-bit cmpxchg.
1270
-
* For context, see commit 39491867ace5.
1271
-
*/
1272
-
if (size != BPF_DW && imm == BPF_CMPXCHG &&
1273
-
insn_is_zext(&insn[i + 1]))
1274
-
addrs[++i] = ctx->idx * 4;
1083
+
ret = bpf_jit_emit_atomic_ops(image, ctx, &insn[i],
1084
+
&jmp_off, &tmp_idx, &addrs[i + 1]);
1085
+
if (ret) {
1086
+
if (ret == -EOPNOTSUPP) {
1087
+
pr_err_ratelimited(
1088
+
"eBPF filter atomic op code %02x (@%d) unsupported\n",
1089
+
code, i);
1090
+
}
1091
+
return ret;
1275
1092
}
1276
1093
break;
1277
1094