Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
bpf: make bpf_insn_successors to return a pointer
The bpf_insn_successors() function is used to return successors
to a BPF instruction. So far, an instruction could have 0, 1 or 2
successors. Prepare the verifier code to introduction of instructions
with more than 2 successors (namely, indirect jumps).
To do this, introduce a new struct, struct bpf_iarray, containing
an array of bpf instruction indexes and make bpf_insn_successors
to return a pointer of that type. The storage for all instructions
is allocated in the env->succ, which holds an array of size 2,
to be used for all instructions.
Signed-off-by: Anton Protopopov <a.s.protopopov@gmail.com>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20251019202145.3944697-10-a.s.protopopov@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
authored by
Anton Protopopov
and committed by
Alexei Starovoitov
2f69c568
44481e49
+75
-33
+11
-1
include/linux/bpf_verifier.h
+11
-1
include/linux/bpf_verifier.h
···
509
509
#define BPF_ALU_SANITIZE (BPF_ALU_SANITIZE_SRC | \
510
510
BPF_ALU_SANITIZE_DST)
511
511
512
+
/*
513
+
* An array of BPF instructions.
514
+
* Primary usage: return value of bpf_insn_successors.
515
+
*/
516
+
struct bpf_iarray {
517
+
int cnt;
518
+
u32 items[];
519
+
};
520
+
512
521
struct bpf_insn_aux_data {
513
522
union {
514
523
enum bpf_reg_type ptr_type; /* pointer type for load/store insns */
···
837
828
/* array of pointers to bpf_scc_info indexed by SCC id */
838
829
struct bpf_scc_info **scc_info;
839
830
u32 scc_cnt;
831
+
struct bpf_iarray *succ;
840
832
};
841
833
842
834
static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)
···
1060
1050
1061
1051
struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off);
1062
1052
int bpf_jmp_offset(struct bpf_insn *insn);
1063
-
int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2]);
1053
+
struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);
1064
1054
void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
1065
1055
bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx);
1066
1056
+24
-12
kernel/bpf/liveness.c
+24
-12
kernel/bpf/liveness.c
···
34
34
* - read and write marks propagation.
35
35
* - The propagation phase is a textbook live variable data flow analysis:
36
36
*
37
-
* state[cc, i].live_after = U [state[cc, s].live_before for s in insn_successors(i)]
37
+
* state[cc, i].live_after = U [state[cc, s].live_before for s in bpf_insn_successors(i)]
38
38
* state[cc, i].live_before =
39
39
* (state[cc, i].live_after / state[cc, i].must_write) U state[i].may_read
40
40
*
···
54
54
* The equation for "must_write_acc" propagation looks as follows:
55
55
*
56
56
* state[cc, i].must_write_acc =
57
-
* ∩ [state[cc, s].must_write_acc for s in insn_successors(i)]
57
+
* ∩ [state[cc, s].must_write_acc for s in bpf_insn_successors(i)]
58
58
* U state[cc, i].must_write
59
59
*
60
60
* (An intersection of all "must_write_acc" for instruction successors
···
447
447
__diag_push();
448
448
__diag_ignore_all("-Woverride-init", "Allow field initialization overrides for opcode_info_tbl");
449
449
450
-
inline int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2])
450
+
/*
451
+
* Returns an array of instructions succ, with succ->items[0], ...,
452
+
* succ->items[n-1] with successor instructions, where n=succ->cnt
453
+
*/
454
+
inline struct bpf_iarray *
455
+
bpf_insn_successors(struct bpf_verifier_env *env, u32 idx)
451
456
{
452
457
static const struct opcode_info {
453
458
bool can_jump;
···
479
474
_J(BPF_JSET, {.can_jump = true, .can_fallthrough = true}),
480
475
#undef _J
481
476
};
477
+
struct bpf_prog *prog = env->prog;
482
478
struct bpf_insn *insn = &prog->insnsi[idx];
483
479
const struct opcode_info *opcode_info;
484
-
int i = 0, insn_sz;
480
+
struct bpf_iarray *succ;
481
+
int insn_sz;
482
+
483
+
/* pre-allocated array of size up to 2; reset cnt, as it may have been used already */
484
+
succ = env->succ;
485
+
succ->cnt = 0;
485
486
486
487
opcode_info = &opcode_info_tbl[BPF_CLASS(insn->code) | BPF_OP(insn->code)];
487
488
insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
488
489
if (opcode_info->can_fallthrough)
489
-
succ[i++] = idx + insn_sz;
490
+
succ->items[succ->cnt++] = idx + insn_sz;
490
491
491
492
if (opcode_info->can_jump)
492
-
succ[i++] = idx + bpf_jmp_offset(insn) + 1;
493
+
succ->items[succ->cnt++] = idx + bpf_jmp_offset(insn) + 1;
493
494
494
-
return i;
495
+
return succ;
495
496
}
496
497
497
498
__diag_pop();
···
559
548
struct bpf_insn_aux_data *aux = env->insn_aux_data;
560
549
u64 new_before, new_after, must_write_acc;
561
550
struct per_frame_masks *insn, *succ_insn;
562
-
u32 succ_num, s, succ[2];
551
+
struct bpf_iarray *succ;
552
+
u32 s;
563
553
bool changed;
564
554
565
-
succ_num = bpf_insn_successors(env->prog, insn_idx, succ);
566
-
if (unlikely(succ_num == 0))
555
+
succ = bpf_insn_successors(env, insn_idx);
556
+
if (succ->cnt == 0)
567
557
return false;
568
558
569
559
changed = false;
···
576
564
* of successors plus all "must_write" slots of instruction itself.
577
565
*/
578
566
must_write_acc = U64_MAX;
579
-
for (s = 0; s < succ_num; ++s) {
580
-
succ_insn = get_frame_masks(instance, frame, succ[s]);
567
+
for (s = 0; s < succ->cnt; ++s) {
568
+
succ_insn = get_frame_masks(instance, frame, succ->items[s]);
581
569
new_after |= succ_insn->live_before;
582
570
must_write_acc &= succ_insn->must_write_acc;
583
571
}
+40
-20
kernel/bpf/verifier.c
+40
-20
kernel/bpf/verifier.c
···
17805
17805
return 0;
17806
17806
}
17807
17807
17808
+
static struct bpf_iarray *iarray_realloc(struct bpf_iarray *old, size_t n_elem)
17809
+
{
17810
+
size_t new_size = sizeof(struct bpf_iarray) + n_elem * sizeof(old->items[0]);
17811
+
struct bpf_iarray *new;
17812
+
17813
+
new = kvrealloc(old, new_size, GFP_KERNEL_ACCOUNT);
17814
+
if (!new) {
17815
+
/* this is what callers always want, so simplify the call site */
17816
+
kvfree(old);
17817
+
return NULL;
17818
+
}
17819
+
17820
+
new->cnt = n_elem;
17821
+
return new;
17822
+
}
17823
+
17808
17824
/* Visits the instruction at index t and returns one of the following:
17809
17825
* < 0 - an error occurred
17810
17826
* DONE_EXPLORING - the instruction was fully explored
···
18041
18025
*/
18042
18026
static int compute_postorder(struct bpf_verifier_env *env)
18043
18027
{
18044
-
u32 cur_postorder, i, top, stack_sz, s, succ_cnt, succ[2];
18028
+
u32 cur_postorder, i, top, stack_sz, s;
18045
18029
int *stack = NULL, *postorder = NULL, *state = NULL;
18030
+
struct bpf_iarray *succ;
18046
18031
18047
18032
postorder = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
18048
18033
state = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
···
18067
18050
stack_sz--;
18068
18051
continue;
18069
18052
}
18070
-
succ_cnt = bpf_insn_successors(env->prog, top, succ);
18071
-
for (s = 0; s < succ_cnt; ++s) {
18072
-
if (!state[succ[s]]) {
18073
-
stack[stack_sz++] = succ[s];
18074
-
state[succ[s]] |= DISCOVERED;
18053
+
succ = bpf_insn_successors(env, top);
18054
+
for (s = 0; s < succ->cnt; ++s) {
18055
+
if (!state[succ->items[s]]) {
18056
+
stack[stack_sz++] = succ->items[s];
18057
+
state[succ->items[s]] |= DISCOVERED;
18075
18058
}
18076
18059
}
18077
18060
state[top] |= EXPLORED;
···
24330
24313
for (i = 0; i < env->cfg.cur_postorder; ++i) {
24331
24314
int insn_idx = env->cfg.insn_postorder[i];
24332
24315
struct insn_live_regs *live = &state[insn_idx];
24333
-
int succ_num;
24334
-
u32 succ[2];
24316
+
struct bpf_iarray *succ;
24335
24317
u16 new_out = 0;
24336
24318
u16 new_in = 0;
24337
24319
24338
-
succ_num = bpf_insn_successors(env->prog, insn_idx, succ);
24339
-
for (int s = 0; s < succ_num; ++s)
24340
-
new_out |= state[succ[s]].in;
24320
+
succ = bpf_insn_successors(env, insn_idx);
24321
+
for (int s = 0; s < succ->cnt; ++s)
24322
+
new_out |= state[succ->items[s]].in;
24341
24323
new_in = (new_out & ~live->def) | live->use;
24342
24324
if (new_out != live->out || new_in != live->in) {
24343
24325
live->in = new_in;
···
24389
24373
const u32 insn_cnt = env->prog->len;
24390
24374
int stack_sz, dfs_sz, err = 0;
24391
24375
u32 *stack, *pre, *low, *dfs;
24392
-
u32 succ_cnt, i, j, t, w;
24376
+
u32 i, j, t, w;
24393
24377
u32 next_preorder_num;
24394
24378
u32 next_scc_id;
24395
24379
bool assign_scc;
24396
-
u32 succ[2];
24380
+
struct bpf_iarray *succ;
24397
24381
24398
24382
next_preorder_num = 1;
24399
24383
next_scc_id = 1;
···
24500
24484
stack[stack_sz++] = w;
24501
24485
}
24502
24486
/* Visit 'w' successors */
24503
-
succ_cnt = bpf_insn_successors(env->prog, w, succ);
24504
-
for (j = 0; j < succ_cnt; ++j) {
24505
-
if (pre[succ[j]]) {
24506
-
low[w] = min(low[w], low[succ[j]]);
24487
+
succ = bpf_insn_successors(env, w);
24488
+
for (j = 0; j < succ->cnt; ++j) {
24489
+
if (pre[succ->items[j]]) {
24490
+
low[w] = min(low[w], low[succ->items[j]]);
24507
24491
} else {
24508
-
dfs[dfs_sz++] = succ[j];
24492
+
dfs[dfs_sz++] = succ->items[j];
24509
24493
goto dfs_continue;
24510
24494
}
24511
24495
}
···
24522
24506
* or if component has a self reference.
24523
24507
*/
24524
24508
assign_scc = stack[stack_sz - 1] != w;
24525
-
for (j = 0; j < succ_cnt; ++j) {
24526
-
if (succ[j] == w) {
24509
+
for (j = 0; j < succ->cnt; ++j) {
24510
+
if (succ->items[j] == w) {
24527
24511
assign_scc = true;
24528
24512
break;
24529
24513
}
···
24585
24569
goto err_free_env;
24586
24570
for (i = 0; i < len; i++)
24587
24571
env->insn_aux_data[i].orig_idx = i;
24572
+
env->succ = iarray_realloc(NULL, 2);
24573
+
if (!env->succ)
24574
+
goto err_free_env;
24588
24575
env->prog = *prog;
24589
24576
env->ops = bpf_verifier_ops[env->prog->type];
24590
24577
···
24836
24817
bpf_stack_liveness_free(env);
24837
24818
kvfree(env->cfg.insn_postorder);
24838
24819
kvfree(env->scc_info);
24820
+
kvfree(env->succ);
24839
24821
kvfree(env);
24840
24822
return ret;
24841
24823
}