Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'bpf-prep-patches-for-static-stack-liveness'
Alexei Starovoitov says:
====================
bpf: Prep patches for static stack liveness.
v4->v5:
- minor test fixup
v3->v4:
- fixed invalid recursion detection when calback is called multiple times
v3: https://lore.kernel.org/bpf/20260402212856.86606-1-alexei.starovoitov@gmail.com/
v2->v3:
- added recursive call detection
- fixed ubsan warning
- removed double declaration in the header
- added Acks
v2: https://lore.kernel.org/bpf/20260402061744.10885-1-alexei.starovoitov@gmail.com/
v1->v2:
. fixed bugs spotted by Eduard, Mykyta, claude and gemini
. fixed selftests that were failing in unpriv
. gemini(sashiko) found several precision improvements in patch 6,
but they made no difference in real programs.
v1: https://lore.kernel.org/bpf/20260401021635.34636-1-alexei.starovoitov@gmail.com/
First 6 prep patches for static stack liveness.
. do src/dst_reg validation early and remove defensive checks
. sort subprog in topo order. We wanted to do this long ago
to process global subprogs this way and in other cases.
. Add constant folding pass that computes map_ptr, subprog_idx,
loads from readonly maps, and other constants that fit into 32-bit
. Use these constants to eliminate dead code. Replace predicted
conditional branches with "jmp always". That reduces JIT prog size.
. Add two helpers that return access size from their arguments.
====================
Link: https://patch.msgid.link/20260403024422.87231-1-alexei.starovoitov@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+1057
-98
+59
include/linux/bpf_verifier.h
+59
include/linux/bpf_verifier.h
···
595
595
u32 scc;
596
596
/* registers alive before this instruction. */
597
597
u16 live_regs_before;
598
+
/*
599
+
* Bitmask of R0-R9 that hold known values at this instruction.
600
+
* const_reg_mask: scalar constants that fit in 32 bits.
601
+
* const_reg_map_mask: map pointers, val is map_index into used_maps[].
602
+
* const_reg_subprog_mask: subprog pointers, val is subprog number.
603
+
* const_reg_vals[i] holds the 32-bit value for register i.
604
+
* Populated by compute_const_regs() pre-pass.
605
+
*/
606
+
u16 const_reg_mask;
607
+
u16 const_reg_map_mask;
608
+
u16 const_reg_subprog_mask;
609
+
u32 const_reg_vals[10];
598
610
};
599
611
600
612
#define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */
···
799
787
const struct bpf_line_info *prev_linfo;
800
788
struct bpf_verifier_log log;
801
789
struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */
790
+
/* subprog indices sorted in topological order: leaves first, callers last */
791
+
int subprog_topo_order[BPF_MAX_SUBPROGS + 2];
802
792
union {
803
793
struct bpf_idmap idmap_scratch;
804
794
struct bpf_idset idset_scratch;
···
877
863
static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env, int subprog)
878
864
{
879
865
return &env->subprog_info[subprog];
866
+
}
867
+
868
+
struct bpf_call_summary {
869
+
u8 num_params;
870
+
bool is_void;
871
+
bool fastcall;
872
+
};
873
+
874
+
static inline bool bpf_helper_call(const struct bpf_insn *insn)
875
+
{
876
+
return insn->code == (BPF_JMP | BPF_CALL) &&
877
+
insn->src_reg == 0;
878
+
}
879
+
880
+
static inline bool bpf_pseudo_call(const struct bpf_insn *insn)
881
+
{
882
+
return insn->code == (BPF_JMP | BPF_CALL) &&
883
+
insn->src_reg == BPF_PSEUDO_CALL;
884
+
}
885
+
886
+
static inline bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
887
+
{
888
+
return insn->code == (BPF_JMP | BPF_CALL) &&
889
+
insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
880
890
}
881
891
882
892
__printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
···
980
942
void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab);
981
943
982
944
int mark_chain_precision(struct bpf_verifier_env *env, int regno);
945
+
946
+
bool bpf_map_is_rdonly(const struct bpf_map *map);
947
+
int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
948
+
bool is_ldsx);
983
949
984
950
#define BPF_BASE_TYPE_MASK GENMASK(BPF_BASE_TYPE_BITS - 1, 0)
985
951
···
1127
1085
struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);
1128
1086
void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
1129
1087
bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx);
1088
+
1089
+
int bpf_find_subprog(struct bpf_verifier_env *env, int off);
1090
+
int bpf_compute_const_regs(struct bpf_verifier_env *env);
1091
+
int bpf_prune_dead_branches(struct bpf_verifier_env *env);
1092
+
int bpf_compute_postorder(struct bpf_verifier_env *env);
1093
+
bool bpf_insn_is_cond_jump(u8 code);
1094
+
bool bpf_is_may_goto_insn(struct bpf_insn *insn);
1095
+
1096
+
void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn);
1097
+
bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
1098
+
struct bpf_call_summary *cs);
1099
+
s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env,
1100
+
struct bpf_insn *insn, int arg,
1101
+
int insn_idx);
1102
+
s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env,
1103
+
struct bpf_insn *insn, int arg,
1104
+
int insn_idx);
1130
1105
1131
1106
int bpf_stack_liveness_init(struct bpf_verifier_env *env);
1132
1107
void bpf_stack_liveness_free(struct bpf_verifier_env *env);
+1
-1
kernel/bpf/Makefile
+1
-1
kernel/bpf/Makefile
···
6
6
endif
7
7
CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy)
8
8
9
-
obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o liveness.o
9
+
obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o liveness.o const_fold.o
10
10
obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o
11
11
obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o
12
12
obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o
+396
kernel/bpf/const_fold.c
+396
kernel/bpf/const_fold.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
3
+
4
+
#include <linux/bpf_verifier.h>
5
+
6
+
/*
7
+
* Forward dataflow analysis to determine constant register values at every
8
+
* instruction. Tracks 64-bit constant values in R0-R9 through the program,
9
+
* using a fixed-point iteration in reverse postorder. Records which registers
10
+
* hold known constants and their values in
11
+
* env->insn_aux_data[].{const_reg_mask, const_reg_vals}.
12
+
*/
13
+
14
+
enum const_arg_state {
15
+
CONST_ARG_UNVISITED, /* instruction not yet reached */
16
+
CONST_ARG_UNKNOWN, /* register value not a known constant */
17
+
CONST_ARG_CONST, /* register holds a known 64-bit constant */
18
+
CONST_ARG_MAP_PTR, /* register holds a map pointer, map_index is set */
19
+
CONST_ARG_MAP_VALUE, /* register points to map value data, val is offset */
20
+
CONST_ARG_SUBPROG, /* register holds a subprog pointer, val is subprog number */
21
+
};
22
+
23
+
struct const_arg_info {
24
+
enum const_arg_state state;
25
+
u32 map_index;
26
+
u64 val;
27
+
};
28
+
29
+
static bool ci_is_unvisited(const struct const_arg_info *ci)
30
+
{
31
+
return ci->state == CONST_ARG_UNVISITED;
32
+
}
33
+
34
+
static bool ci_is_unknown(const struct const_arg_info *ci)
35
+
{
36
+
return ci->state == CONST_ARG_UNKNOWN;
37
+
}
38
+
39
+
static bool ci_is_const(const struct const_arg_info *ci)
40
+
{
41
+
return ci->state == CONST_ARG_CONST;
42
+
}
43
+
44
+
static bool ci_is_map_value(const struct const_arg_info *ci)
45
+
{
46
+
return ci->state == CONST_ARG_MAP_VALUE;
47
+
}
48
+
49
+
/* Transfer function: compute output register state from instruction. */
50
+
static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info *ci_out,
51
+
struct bpf_insn *insn, struct bpf_insn *insns, int idx)
52
+
{
53
+
struct const_arg_info unknown = { .state = CONST_ARG_UNKNOWN, .val = 0 };
54
+
struct const_arg_info *dst = &ci_out[insn->dst_reg];
55
+
struct const_arg_info *src = &ci_out[insn->src_reg];
56
+
u8 class = BPF_CLASS(insn->code);
57
+
u8 mode = BPF_MODE(insn->code);
58
+
u8 opcode = BPF_OP(insn->code) | BPF_SRC(insn->code);
59
+
int r;
60
+
61
+
switch (class) {
62
+
case BPF_ALU:
63
+
case BPF_ALU64:
64
+
switch (opcode) {
65
+
case BPF_MOV | BPF_K:
66
+
dst->state = CONST_ARG_CONST;
67
+
dst->val = (s64)insn->imm;
68
+
break;
69
+
case BPF_MOV | BPF_X:
70
+
*dst = *src;
71
+
if (!insn->off)
72
+
break;
73
+
if (!ci_is_const(dst)) {
74
+
*dst = unknown;
75
+
break;
76
+
}
77
+
switch (insn->off) {
78
+
case 8: dst->val = (s8)dst->val; break;
79
+
case 16: dst->val = (s16)dst->val; break;
80
+
case 32: dst->val = (s32)dst->val; break;
81
+
default: *dst = unknown; break;
82
+
}
83
+
break;
84
+
case BPF_ADD | BPF_K:
85
+
if (!ci_is_const(dst) && !ci_is_map_value(dst)) {
86
+
*dst = unknown;
87
+
break;
88
+
}
89
+
dst->val += insn->imm;
90
+
break;
91
+
case BPF_SUB | BPF_K:
92
+
if (!ci_is_const(dst) && !ci_is_map_value(dst)) {
93
+
*dst = unknown;
94
+
break;
95
+
}
96
+
dst->val -= insn->imm;
97
+
break;
98
+
case BPF_AND | BPF_K:
99
+
if (!ci_is_const(dst)) {
100
+
if (!insn->imm) {
101
+
dst->state = CONST_ARG_CONST;
102
+
dst->val = 0;
103
+
} else {
104
+
*dst = unknown;
105
+
}
106
+
break;
107
+
}
108
+
dst->val &= (s64)insn->imm;
109
+
break;
110
+
case BPF_AND | BPF_X:
111
+
if (ci_is_const(dst) && dst->val == 0)
112
+
break; /* 0 & x == 0 */
113
+
if (ci_is_const(src) && src->val == 0) {
114
+
dst->state = CONST_ARG_CONST;
115
+
dst->val = 0;
116
+
break;
117
+
}
118
+
if (!ci_is_const(dst) || !ci_is_const(src)) {
119
+
*dst = unknown;
120
+
break;
121
+
}
122
+
dst->val &= src->val;
123
+
break;
124
+
default:
125
+
*dst = unknown;
126
+
break;
127
+
}
128
+
if (class == BPF_ALU) {
129
+
if (ci_is_const(dst))
130
+
dst->val = (u32)dst->val;
131
+
else if (!ci_is_unknown(dst))
132
+
*dst = unknown;
133
+
}
134
+
break;
135
+
case BPF_LD:
136
+
if (mode == BPF_ABS || mode == BPF_IND)
137
+
goto process_call;
138
+
if (mode != BPF_IMM || BPF_SIZE(insn->code) != BPF_DW)
139
+
break;
140
+
if (insn->src_reg == BPF_PSEUDO_FUNC) {
141
+
int subprog = bpf_find_subprog(env, idx + insn->imm + 1);
142
+
143
+
if (subprog >= 0) {
144
+
dst->state = CONST_ARG_SUBPROG;
145
+
dst->val = subprog;
146
+
} else {
147
+
*dst = unknown;
148
+
}
149
+
} else if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
150
+
insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
151
+
dst->state = CONST_ARG_MAP_VALUE;
152
+
dst->map_index = env->insn_aux_data[idx].map_index;
153
+
dst->val = env->insn_aux_data[idx].map_off;
154
+
} else if (insn->src_reg == BPF_PSEUDO_MAP_FD ||
155
+
insn->src_reg == BPF_PSEUDO_MAP_IDX) {
156
+
dst->state = CONST_ARG_MAP_PTR;
157
+
dst->map_index = env->insn_aux_data[idx].map_index;
158
+
} else if (insn->src_reg == 0) {
159
+
dst->state = CONST_ARG_CONST;
160
+
dst->val = (u64)(u32)insn->imm | ((u64)(u32)insns[idx + 1].imm << 32);
161
+
} else {
162
+
*dst = unknown;
163
+
}
164
+
break;
165
+
case BPF_LDX:
166
+
if (!ci_is_map_value(src)) {
167
+
*dst = unknown;
168
+
break;
169
+
}
170
+
struct bpf_map *map = env->used_maps[src->map_index];
171
+
int size = bpf_size_to_bytes(BPF_SIZE(insn->code));
172
+
bool is_ldsx = mode == BPF_MEMSX;
173
+
int off = src->val + insn->off;
174
+
u64 val = 0;
175
+
176
+
if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr ||
177
+
map->map_type == BPF_MAP_TYPE_INSN_ARRAY ||
178
+
off < 0 || off + size > map->value_size ||
179
+
bpf_map_direct_read(map, off, size, &val, is_ldsx)) {
180
+
*dst = unknown;
181
+
break;
182
+
}
183
+
dst->state = CONST_ARG_CONST;
184
+
dst->val = val;
185
+
break;
186
+
case BPF_JMP:
187
+
if (opcode != BPF_CALL)
188
+
break;
189
+
process_call:
190
+
for (r = BPF_REG_0; r <= BPF_REG_5; r++)
191
+
ci_out[r] = unknown;
192
+
break;
193
+
case BPF_STX:
194
+
if (mode != BPF_ATOMIC)
195
+
break;
196
+
if (insn->imm == BPF_CMPXCHG)
197
+
ci_out[BPF_REG_0] = unknown;
198
+
else if (insn->imm == BPF_LOAD_ACQ)
199
+
*dst = unknown;
200
+
else if (insn->imm & BPF_FETCH)
201
+
*src = unknown;
202
+
break;
203
+
}
204
+
}
205
+
206
+
/* Join function: merge output state into a successor's input state. */
207
+
static bool const_reg_join(struct const_arg_info *ci_target,
208
+
struct const_arg_info *ci_out)
209
+
{
210
+
bool changed = false;
211
+
int r;
212
+
213
+
for (r = 0; r < MAX_BPF_REG; r++) {
214
+
struct const_arg_info *old = &ci_target[r];
215
+
struct const_arg_info *new = &ci_out[r];
216
+
217
+
if (ci_is_unvisited(old) && !ci_is_unvisited(new)) {
218
+
ci_target[r] = *new;
219
+
changed = true;
220
+
} else if (!ci_is_unknown(old) && !ci_is_unvisited(old) &&
221
+
(new->state != old->state || new->val != old->val ||
222
+
new->map_index != old->map_index)) {
223
+
old->state = CONST_ARG_UNKNOWN;
224
+
changed = true;
225
+
}
226
+
}
227
+
return changed;
228
+
}
229
+
230
+
int bpf_compute_const_regs(struct bpf_verifier_env *env)
231
+
{
232
+
struct const_arg_info unknown = { .state = CONST_ARG_UNKNOWN, .val = 0 };
233
+
struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;
234
+
struct bpf_insn *insns = env->prog->insnsi;
235
+
int insn_cnt = env->prog->len;
236
+
struct const_arg_info (*ci_in)[MAX_BPF_REG];
237
+
struct const_arg_info ci_out[MAX_BPF_REG];
238
+
struct bpf_iarray *succ;
239
+
bool changed;
240
+
int i, r;
241
+
242
+
/* kvzalloc zeroes memory, so all entries start as CONST_ARG_UNVISITED (0) */
243
+
ci_in = kvzalloc_objs(*ci_in, insn_cnt, GFP_KERNEL_ACCOUNT);
244
+
if (!ci_in)
245
+
return -ENOMEM;
246
+
247
+
/* Subprogram entries (including main at subprog 0): all registers unknown */
248
+
for (i = 0; i < env->subprog_cnt; i++) {
249
+
int start = env->subprog_info[i].start;
250
+
251
+
for (r = 0; r < MAX_BPF_REG; r++)
252
+
ci_in[start][r] = unknown;
253
+
}
254
+
255
+
redo:
256
+
changed = false;
257
+
for (i = env->cfg.cur_postorder - 1; i >= 0; i--) {
258
+
int idx = env->cfg.insn_postorder[i];
259
+
struct bpf_insn *insn = &insns[idx];
260
+
struct const_arg_info *ci = ci_in[idx];
261
+
262
+
memcpy(ci_out, ci, sizeof(ci_out));
263
+
264
+
const_reg_xfer(env, ci_out, insn, insns, idx);
265
+
266
+
succ = bpf_insn_successors(env, idx);
267
+
for (int s = 0; s < succ->cnt; s++)
268
+
changed |= const_reg_join(ci_in[succ->items[s]], ci_out);
269
+
}
270
+
if (changed)
271
+
goto redo;
272
+
273
+
/* Save computed constants into insn_aux[] if they fit into 32-bit */
274
+
for (i = 0; i < insn_cnt; i++) {
275
+
u16 mask = 0, map_mask = 0, subprog_mask = 0;
276
+
struct bpf_insn_aux_data *aux = &insn_aux[i];
277
+
struct const_arg_info *ci = ci_in[i];
278
+
279
+
for (r = BPF_REG_0; r < ARRAY_SIZE(aux->const_reg_vals); r++) {
280
+
struct const_arg_info *c = &ci[r];
281
+
282
+
switch (c->state) {
283
+
case CONST_ARG_CONST: {
284
+
u64 val = c->val;
285
+
286
+
if (val != (u32)val)
287
+
break;
288
+
mask |= BIT(r);
289
+
aux->const_reg_vals[r] = val;
290
+
break;
291
+
}
292
+
case CONST_ARG_MAP_PTR:
293
+
map_mask |= BIT(r);
294
+
aux->const_reg_vals[r] = c->map_index;
295
+
break;
296
+
case CONST_ARG_SUBPROG:
297
+
subprog_mask |= BIT(r);
298
+
aux->const_reg_vals[r] = c->val;
299
+
break;
300
+
default:
301
+
break;
302
+
}
303
+
}
304
+
aux->const_reg_mask = mask;
305
+
aux->const_reg_map_mask = map_mask;
306
+
aux->const_reg_subprog_mask = subprog_mask;
307
+
}
308
+
309
+
kvfree(ci_in);
310
+
return 0;
311
+
}
312
+
313
+
static int eval_const_branch(u8 opcode, u64 dst_val, u64 src_val)
314
+
{
315
+
switch (BPF_OP(opcode)) {
316
+
case BPF_JEQ: return dst_val == src_val;
317
+
case BPF_JNE: return dst_val != src_val;
318
+
case BPF_JGT: return dst_val > src_val;
319
+
case BPF_JGE: return dst_val >= src_val;
320
+
case BPF_JLT: return dst_val < src_val;
321
+
case BPF_JLE: return dst_val <= src_val;
322
+
case BPF_JSGT: return (s64)dst_val > (s64)src_val;
323
+
case BPF_JSGE: return (s64)dst_val >= (s64)src_val;
324
+
case BPF_JSLT: return (s64)dst_val < (s64)src_val;
325
+
case BPF_JSLE: return (s64)dst_val <= (s64)src_val;
326
+
case BPF_JSET: return (bool)(dst_val & src_val);
327
+
default: return -1;
328
+
}
329
+
}
330
+
331
+
/*
332
+
* Rewrite conditional branches with constant outcomes into unconditional
333
+
* jumps using register values resolved by bpf_compute_const_regs() pass.
334
+
* This eliminates dead edges from the CFG so that compute_live_registers()
335
+
* doesn't propagate liveness through dead code.
336
+
*/
337
+
int bpf_prune_dead_branches(struct bpf_verifier_env *env)
338
+
{
339
+
struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;
340
+
struct bpf_insn *insns = env->prog->insnsi;
341
+
int insn_cnt = env->prog->len;
342
+
bool changed = false;
343
+
int i;
344
+
345
+
for (i = 0; i < insn_cnt; i++) {
346
+
struct bpf_insn_aux_data *aux = &insn_aux[i];
347
+
struct bpf_insn *insn = &insns[i];
348
+
u8 class = BPF_CLASS(insn->code);
349
+
u64 dst_val, src_val;
350
+
int taken;
351
+
352
+
if (!bpf_insn_is_cond_jump(insn->code))
353
+
continue;
354
+
if (bpf_is_may_goto_insn(insn))
355
+
continue;
356
+
357
+
if (!(aux->const_reg_mask & BIT(insn->dst_reg)))
358
+
continue;
359
+
dst_val = aux->const_reg_vals[insn->dst_reg];
360
+
361
+
if (BPF_SRC(insn->code) == BPF_K) {
362
+
src_val = insn->imm;
363
+
} else {
364
+
if (!(aux->const_reg_mask & BIT(insn->src_reg)))
365
+
continue;
366
+
src_val = aux->const_reg_vals[insn->src_reg];
367
+
}
368
+
369
+
if (class == BPF_JMP32) {
370
+
/*
371
+
* The (s32) cast maps the 32-bit range into two u64 sub-ranges:
372
+
* [0x00000000, 0x7FFFFFFF] -> [0x0000000000000000, 0x000000007FFFFFFF]
373
+
* [0x80000000, 0xFFFFFFFF] -> [0xFFFFFFFF80000000, 0xFFFFFFFFFFFFFFFF]
374
+
* The ordering is preserved within each sub-range, and
375
+
* the second sub-range is above the first as u64.
376
+
*/
377
+
dst_val = (s32)dst_val;
378
+
src_val = (s32)src_val;
379
+
}
380
+
381
+
taken = eval_const_branch(insn->code, dst_val, src_val);
382
+
if (taken < 0) {
383
+
bpf_log(&env->log, "Unknown conditional jump %x\n", insn->code);
384
+
return -EFAULT;
385
+
}
386
+
*insn = BPF_JMP_A(taken ? insn->off : 0);
387
+
changed = true;
388
+
}
389
+
390
+
if (!changed)
391
+
return 0;
392
+
/* recompute postorder, since CFG has changed */
393
+
kvfree(env->cfg.insn_postorder);
394
+
env->cfg.insn_postorder = NULL;
395
+
return bpf_compute_postorder(env);
396
+
}
+349
-82
kernel/bpf/verifier.c
+349
-82
kernel/bpf/verifier.c
···
256
256
(poisoned ? BPF_MAP_KEY_POISON : 0ULL);
257
257
}
258
258
259
-
static bool bpf_helper_call(const struct bpf_insn *insn)
260
-
{
261
-
return insn->code == (BPF_JMP | BPF_CALL) &&
262
-
insn->src_reg == 0;
263
-
}
264
-
265
-
static bool bpf_pseudo_call(const struct bpf_insn *insn)
266
-
{
267
-
return insn->code == (BPF_JMP | BPF_CALL) &&
268
-
insn->src_reg == BPF_PSEUDO_CALL;
269
-
}
270
-
271
-
static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
272
-
{
273
-
return insn->code == (BPF_JMP | BPF_CALL) &&
274
-
insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
275
-
}
276
-
277
259
struct bpf_map_desc {
278
260
struct bpf_map *ptr;
279
261
int uid;
···
577
595
return false;
578
596
}
579
597
580
-
static bool is_may_goto_insn(struct bpf_insn *insn)
598
+
bool bpf_is_may_goto_insn(struct bpf_insn *insn)
581
599
{
582
600
return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO;
583
601
}
584
602
585
603
static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx)
586
604
{
587
-
return is_may_goto_insn(&env->prog->insnsi[insn_idx]);
605
+
return bpf_is_may_goto_insn(&env->prog->insnsi[insn_idx]);
588
606
}
589
607
590
608
static bool is_storage_get_function(enum bpf_func_id func_id)
···
2238
2256
static void mark_reg_known_zero(struct bpf_verifier_env *env,
2239
2257
struct bpf_reg_state *regs, u32 regno)
2240
2258
{
2241
-
if (WARN_ON(regno >= MAX_BPF_REG)) {
2242
-
verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
2243
-
/* Something bad happened, let's kill all regs */
2244
-
for (regno = 0; regno < MAX_BPF_REG; regno++)
2245
-
__mark_reg_not_init(env, regs + regno);
2246
-
return;
2247
-
}
2248
2259
__mark_reg_known_zero(regs + regno);
2249
2260
}
2250
2261
···
2911
2936
static void mark_reg_unknown(struct bpf_verifier_env *env,
2912
2937
struct bpf_reg_state *regs, u32 regno)
2913
2938
{
2914
-
if (WARN_ON(regno >= MAX_BPF_REG)) {
2915
-
verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
2916
-
/* Something bad happened, let's kill all regs except FP */
2917
-
for (regno = 0; regno < BPF_REG_FP; regno++)
2918
-
__mark_reg_not_init(env, regs + regno);
2919
-
return;
2920
-
}
2921
2939
__mark_reg_unknown(env, regs + regno);
2922
2940
}
2923
2941
···
2943
2975
static void mark_reg_not_init(struct bpf_verifier_env *env,
2944
2976
struct bpf_reg_state *regs, u32 regno)
2945
2977
{
2946
-
if (WARN_ON(regno >= MAX_BPF_REG)) {
2947
-
verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
2948
-
/* Something bad happened, let's kill all regs except FP */
2949
-
for (regno = 0; regno < BPF_REG_FP; regno++)
2950
-
__mark_reg_not_init(env, regs + regno);
2951
-
return;
2952
-
}
2953
2978
__mark_reg_not_init(env, regs + regno);
2954
2979
}
2955
2980
···
3092
3131
}
3093
3132
3094
3133
/* Find subprogram that starts exactly at 'off' */
3095
-
static int find_subprog(struct bpf_verifier_env *env, int off)
3134
+
int bpf_find_subprog(struct bpf_verifier_env *env, int off)
3096
3135
{
3097
3136
struct bpf_subprog_info *p;
3098
3137
···
3111
3150
verbose(env, "call to invalid destination\n");
3112
3151
return -EINVAL;
3113
3152
}
3114
-
ret = find_subprog(env, off);
3153
+
ret = bpf_find_subprog(env, off);
3115
3154
if (ret >= 0)
3116
3155
return ret;
3117
3156
if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
···
3752
3791
return 0;
3753
3792
}
3754
3793
3794
+
/*
3795
+
* Sort subprogs in topological order so that leaf subprogs come first and
3796
+
* their callers come later. This is a DFS post-order traversal of the call
3797
+
* graph. Scan only reachable instructions (those in the computed postorder) of
3798
+
* the current subprog to discover callees (direct subprogs and sync
3799
+
* callbacks).
3800
+
*/
3801
+
static int sort_subprogs_topo(struct bpf_verifier_env *env)
3802
+
{
3803
+
struct bpf_subprog_info *si = env->subprog_info;
3804
+
int *insn_postorder = env->cfg.insn_postorder;
3805
+
struct bpf_insn *insn = env->prog->insnsi;
3806
+
int cnt = env->subprog_cnt;
3807
+
int *dfs_stack = NULL;
3808
+
int top = 0, order = 0;
3809
+
int i, ret = 0;
3810
+
u8 *color = NULL;
3811
+
3812
+
color = kvzalloc_objs(*color, cnt, GFP_KERNEL_ACCOUNT);
3813
+
dfs_stack = kvmalloc_objs(*dfs_stack, cnt, GFP_KERNEL_ACCOUNT);
3814
+
if (!color || !dfs_stack) {
3815
+
ret = -ENOMEM;
3816
+
goto out;
3817
+
}
3818
+
3819
+
/*
3820
+
* DFS post-order traversal.
3821
+
* Color values: 0 = unvisited, 1 = on stack, 2 = done.
3822
+
*/
3823
+
for (i = 0; i < cnt; i++) {
3824
+
if (color[i])
3825
+
continue;
3826
+
color[i] = 1;
3827
+
dfs_stack[top++] = i;
3828
+
3829
+
while (top > 0) {
3830
+
int cur = dfs_stack[top - 1];
3831
+
int po_start = si[cur].postorder_start;
3832
+
int po_end = si[cur + 1].postorder_start;
3833
+
bool pushed = false;
3834
+
int j;
3835
+
3836
+
for (j = po_start; j < po_end; j++) {
3837
+
int idx = insn_postorder[j];
3838
+
int callee;
3839
+
3840
+
if (!bpf_pseudo_call(&insn[idx]) && !bpf_pseudo_func(&insn[idx]))
3841
+
continue;
3842
+
callee = bpf_find_subprog(env, idx + insn[idx].imm + 1);
3843
+
if (callee < 0) {
3844
+
ret = -EFAULT;
3845
+
goto out;
3846
+
}
3847
+
if (color[callee] == 2)
3848
+
continue;
3849
+
if (color[callee] == 1) {
3850
+
if (bpf_pseudo_func(&insn[idx]))
3851
+
continue;
3852
+
verbose(env, "recursive call from %s() to %s()\n",
3853
+
subprog_name(env, cur),
3854
+
subprog_name(env, callee));
3855
+
ret = -EINVAL;
3856
+
goto out;
3857
+
}
3858
+
color[callee] = 1;
3859
+
dfs_stack[top++] = callee;
3860
+
pushed = true;
3861
+
break;
3862
+
}
3863
+
3864
+
if (!pushed) {
3865
+
color[cur] = 2;
3866
+
env->subprog_topo_order[order++] = cur;
3867
+
top--;
3868
+
}
3869
+
}
3870
+
}
3871
+
3872
+
if (env->log.level & BPF_LOG_LEVEL2)
3873
+
for (i = 0; i < cnt; i++)
3874
+
verbose(env, "topo_order[%d] = %s\n",
3875
+
i, subprog_name(env, env->subprog_topo_order[i]));
3876
+
out:
3877
+
kvfree(dfs_stack);
3878
+
kvfree(color);
3879
+
return ret;
3880
+
}
3881
+
3755
3882
static int mark_stack_slot_obj_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
3756
3883
int spi, int nr_slots)
3757
3884
{
···
4034
3985
struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
4035
3986
struct bpf_reg_state *reg;
4036
3987
bool rw64;
4037
-
4038
-
if (regno >= MAX_BPF_REG) {
4039
-
verbose(env, "R%d is invalid\n", regno);
4040
-
return -EINVAL;
4041
-
}
4042
3988
4043
3989
mark_reg_scratched(env, regno);
4044
3990
···
4279
4235
return btf_name_by_offset(desc_btf, func->name_off);
4280
4236
}
4281
4237
4282
-
static void verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn)
4238
+
void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn)
4283
4239
{
4284
4240
const struct bpf_insn_cbs cbs = {
4285
4241
.cb_call = disasm_kfunc_name,
···
4503
4459
bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
4504
4460
verbose(env, "stack=%s before ", env->tmp_str_buf);
4505
4461
verbose(env, "%d: ", idx);
4506
-
verbose_insn(env, insn);
4462
+
bpf_verbose_insn(env, insn);
4507
4463
}
4508
4464
4509
4465
/* If there is a history record that some registers gained range at this insn,
···
4606
4562
int subprog_insn_idx, subprog;
4607
4563
4608
4564
subprog_insn_idx = idx + insn->imm + 1;
4609
-
subprog = find_subprog(env, subprog_insn_idx);
4565
+
subprog = bpf_find_subprog(env, subprog_insn_idx);
4610
4566
if (subprog < 0)
4611
4567
return -EFAULT;
4612
4568
···
6938
6894
6939
6895
/* find the callee */
6940
6896
next_insn = i + insn[i].imm + 1;
6941
-
sidx = find_subprog(env, next_insn);
6897
+
sidx = bpf_find_subprog(env, next_insn);
6942
6898
if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn))
6943
6899
return -EFAULT;
6944
6900
if (subprog[sidx].is_async_cb) {
···
7073
7029
{
7074
7030
int start = idx + insn->imm + 1, subprog;
7075
7031
7076
-
subprog = find_subprog(env, start);
7032
+
subprog = bpf_find_subprog(env, start);
7077
7033
if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start))
7078
7034
return -EFAULT;
7079
7035
return env->subprog_info[subprog].stack_depth;
···
7320
7276
set_sext32_default_val(reg, size);
7321
7277
}
7322
7278
7323
-
static bool bpf_map_is_rdonly(const struct bpf_map *map)
7279
+
bool bpf_map_is_rdonly(const struct bpf_map *map)
7324
7280
{
7325
7281
/* A map is considered read-only if the following condition are true:
7326
7282
*
···
7340
7296
!bpf_map_write_active(map);
7341
7297
}
7342
7298
7343
-
static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
7344
-
bool is_ldsx)
7299
+
int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
7300
+
bool is_ldsx)
7345
7301
{
7346
7302
void *ptr;
7347
7303
u64 addr;
···
11031
10987
int err, subprog, target_insn;
11032
10988
11033
10989
target_insn = *insn_idx + insn->imm + 1;
11034
-
subprog = find_subprog(env, target_insn);
10990
+
subprog = bpf_find_subprog(env, target_insn);
11035
10991
if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program",
11036
10992
target_insn))
11037
10993
return -EFAULT;
···
14130
14086
meta->kfunc_flags = *kfunc.flags;
14131
14087
14132
14088
return 0;
14089
+
}
14090
+
14091
+
/*
14092
+
* Determine how many bytes a helper accesses through a stack pointer at
14093
+
* argument position @arg (0-based, corresponding to R1-R5).
14094
+
*
14095
+
* Returns:
14096
+
* > 0 known read access size in bytes
14097
+
* 0 doesn't read anything directly
14098
+
* S64_MIN unknown
14099
+
* < 0 known write access of (-return) bytes
14100
+
*/
14101
+
s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn,
14102
+
int arg, int insn_idx)
14103
+
{
14104
+
struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
14105
+
const struct bpf_func_proto *fn;
14106
+
enum bpf_arg_type at;
14107
+
s64 size;
14108
+
14109
+
if (get_helper_proto(env, insn->imm, &fn) < 0)
14110
+
return S64_MIN;
14111
+
14112
+
at = fn->arg_type[arg];
14113
+
14114
+
switch (base_type(at)) {
14115
+
case ARG_PTR_TO_MAP_KEY:
14116
+
case ARG_PTR_TO_MAP_VALUE: {
14117
+
bool is_key = base_type(at) == ARG_PTR_TO_MAP_KEY;
14118
+
u64 val;
14119
+
int i, map_reg;
14120
+
14121
+
for (i = 0; i < arg; i++) {
14122
+
if (base_type(fn->arg_type[i]) == ARG_CONST_MAP_PTR)
14123
+
break;
14124
+
}
14125
+
if (i >= arg)
14126
+
goto scan_all_maps;
14127
+
14128
+
map_reg = BPF_REG_1 + i;
14129
+
14130
+
if (!(aux->const_reg_map_mask & BIT(map_reg)))
14131
+
goto scan_all_maps;
14132
+
14133
+
i = aux->const_reg_vals[map_reg];
14134
+
if (i < env->used_map_cnt) {
14135
+
size = is_key ? env->used_maps[i]->key_size
14136
+
: env->used_maps[i]->value_size;
14137
+
goto out;
14138
+
}
14139
+
scan_all_maps:
14140
+
/*
14141
+
* Map pointer is not known at this call site (e.g. different
14142
+
* maps on merged paths). Conservatively return the largest
14143
+
* key_size or value_size across all maps used by the program.
14144
+
*/
14145
+
val = 0;
14146
+
for (i = 0; i < env->used_map_cnt; i++) {
14147
+
struct bpf_map *map = env->used_maps[i];
14148
+
u32 sz = is_key ? map->key_size : map->value_size;
14149
+
14150
+
if (sz > val)
14151
+
val = sz;
14152
+
if (map->inner_map_meta) {
14153
+
sz = is_key ? map->inner_map_meta->key_size
14154
+
: map->inner_map_meta->value_size;
14155
+
if (sz > val)
14156
+
val = sz;
14157
+
}
14158
+
}
14159
+
if (!val)
14160
+
return S64_MIN;
14161
+
size = val;
14162
+
goto out;
14163
+
}
14164
+
case ARG_PTR_TO_MEM:
14165
+
if (at & MEM_FIXED_SIZE) {
14166
+
size = fn->arg_size[arg];
14167
+
goto out;
14168
+
}
14169
+
if (arg + 1 < ARRAY_SIZE(fn->arg_type) &&
14170
+
arg_type_is_mem_size(fn->arg_type[arg + 1])) {
14171
+
int size_reg = BPF_REG_1 + arg + 1;
14172
+
14173
+
if (aux->const_reg_mask & BIT(size_reg)) {
14174
+
size = (s64)aux->const_reg_vals[size_reg];
14175
+
goto out;
14176
+
}
14177
+
/*
14178
+
* Size arg is const on each path but differs across merged
14179
+
* paths. MAX_BPF_STACK is a safe upper bound for reads.
14180
+
*/
14181
+
if (at & MEM_UNINIT)
14182
+
return 0;
14183
+
return MAX_BPF_STACK;
14184
+
}
14185
+
return S64_MIN;
14186
+
case ARG_PTR_TO_DYNPTR:
14187
+
size = BPF_DYNPTR_SIZE;
14188
+
break;
14189
+
case ARG_PTR_TO_STACK:
14190
+
/*
14191
+
* Only used by bpf_calls_callback() helpers. The helper itself
14192
+
* doesn't access stack. The callback subprog does and it's
14193
+
* analyzed separately.
14194
+
*/
14195
+
return 0;
14196
+
default:
14197
+
return S64_MIN;
14198
+
}
14199
+
out:
14200
+
/*
14201
+
* MEM_UNINIT args are write-only: the helper initializes the
14202
+
* buffer without reading it.
14203
+
*/
14204
+
if (at & MEM_UNINIT)
14205
+
return -size;
14206
+
return size;
14207
+
}
14208
+
14209
+
/*
14210
+
* Determine how many bytes a kfunc accesses through a stack pointer at
14211
+
* argument position @arg (0-based, corresponding to R1-R5).
14212
+
*
14213
+
* Returns:
14214
+
* > 0 known read access size in bytes
14215
+
* 0 doesn't access memory through that argument (ex: not a pointer)
14216
+
* S64_MIN unknown
14217
+
* < 0 known write access of (-return) bytes
14218
+
*/
14219
+
s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn,
14220
+
int arg, int insn_idx)
14221
+
{
14222
+
struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
14223
+
struct bpf_kfunc_call_arg_meta meta;
14224
+
const struct btf_param *args;
14225
+
const struct btf_type *t, *ref_t;
14226
+
const struct btf *btf;
14227
+
u32 nargs, type_size;
14228
+
s64 size;
14229
+
14230
+
if (fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0)
14231
+
return S64_MIN;
14232
+
14233
+
btf = meta.btf;
14234
+
args = btf_params(meta.func_proto);
14235
+
nargs = btf_type_vlen(meta.func_proto);
14236
+
if (arg >= nargs)
14237
+
return 0;
14238
+
14239
+
t = btf_type_skip_modifiers(btf, args[arg].type, NULL);
14240
+
if (!btf_type_is_ptr(t))
14241
+
return 0;
14242
+
14243
+
/* dynptr: fixed 16-byte on-stack representation */
14244
+
if (is_kfunc_arg_dynptr(btf, &args[arg])) {
14245
+
size = BPF_DYNPTR_SIZE;
14246
+
goto out;
14247
+
}
14248
+
14249
+
/* ptr + __sz/__szk pair: size is in the next register */
14250
+
if (arg + 1 < nargs &&
14251
+
(btf_param_match_suffix(btf, &args[arg + 1], "__sz") ||
14252
+
btf_param_match_suffix(btf, &args[arg + 1], "__szk"))) {
14253
+
int size_reg = BPF_REG_1 + arg + 1;
14254
+
14255
+
if (aux->const_reg_mask & BIT(size_reg)) {
14256
+
size = (s64)aux->const_reg_vals[size_reg];
14257
+
goto out;
14258
+
}
14259
+
return MAX_BPF_STACK;
14260
+
}
14261
+
14262
+
/* fixed-size pointed-to type: resolve via BTF */
14263
+
ref_t = btf_type_skip_modifiers(btf, t->type, NULL);
14264
+
if (!IS_ERR(btf_resolve_size(btf, ref_t, &type_size))) {
14265
+
size = type_size;
14266
+
goto out;
14267
+
}
14268
+
14269
+
return S64_MIN;
14270
+
out:
14271
+
/* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */
14272
+
if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW)
14273
+
return -size;
14274
+
if (is_kfunc_arg_uninit(btf, &args[arg]))
14275
+
return -size;
14276
+
return size;
14133
14277
}
14134
14278
14135
14279
/* check special kfuncs and return:
···
18150
17918
18151
17919
if (insn->src_reg == BPF_PSEUDO_FUNC) {
18152
17920
struct bpf_prog_aux *aux = env->prog->aux;
18153
-
u32 subprogno = find_subprog(env,
18154
-
env->insn_idx + insn->imm + 1);
17921
+
u32 subprogno = bpf_find_subprog(env,
17922
+
env->insn_idx + insn->imm + 1);
18155
17923
18156
17924
if (!aux->func_info) {
18157
17925
verbose(env, "missing btf func_info\n");
···
18752
18520
}
18753
18521
}
18754
18522
18755
-
struct call_summary {
18756
-
u8 num_params;
18757
-
bool is_void;
18758
-
bool fastcall;
18759
-
};
18760
-
18761
18523
/* If @call is a kfunc or helper call, fills @cs and returns true,
18762
18524
* otherwise returns false.
18763
18525
*/
18764
-
static bool get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
18765
-
struct call_summary *cs)
18526
+
bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
18527
+
struct bpf_call_summary *cs)
18766
18528
{
18767
18529
struct bpf_kfunc_call_arg_meta meta;
18768
18530
const struct bpf_func_proto *fn;
···
18877
18651
struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx;
18878
18652
struct bpf_insn *call = &env->prog->insnsi[insn_idx];
18879
18653
u32 clobbered_regs_mask;
18880
-
struct call_summary cs;
18654
+
struct bpf_call_summary cs;
18881
18655
u32 expected_regs_mask;
18882
18656
s16 off;
18883
18657
int i;
18884
18658
18885
-
if (!get_call_summary(env, call, &cs))
18659
+
if (!bpf_get_call_summary(env, call, &cs))
18886
18660
return;
18887
18661
18888
18662
/* A bitmask specifying which caller saved registers are clobbered
···
19341
19115
default:
19342
19116
/* conditional jump with two edges */
19343
19117
mark_prune_point(env, t);
19344
-
if (is_may_goto_insn(insn))
19118
+
if (bpf_is_may_goto_insn(insn))
19345
19119
mark_force_checkpoint(env, t);
19346
19120
19347
19121
ret = push_insn(t, t + 1, FALLTHROUGH, env);
···
19448
19222
* [env->subprog_info[i].postorder_start, env->subprog_info[i+1].postorder_start)
19449
19223
* with indices of 'i' instructions in postorder.
19450
19224
*/
19451
-
static int compute_postorder(struct bpf_verifier_env *env)
19225
+
int bpf_compute_postorder(struct bpf_verifier_env *env)
19452
19226
{
19453
19227
u32 cur_postorder, i, top, stack_sz, s;
19454
19228
int *stack = NULL, *postorder = NULL, *state = NULL;
···
21742
21516
verbose_linfo(env, env->insn_idx, "; ");
21743
21517
env->prev_log_pos = env->log.end_pos;
21744
21518
verbose(env, "%d: ", env->insn_idx);
21745
-
verbose_insn(env, insn);
21519
+
bpf_verbose_insn(env, insn);
21746
21520
env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;
21747
21521
env->prev_log_pos = env->log.end_pos;
21748
21522
}
···
21756
21530
21757
21531
sanitize_mark_insn_seen(env);
21758
21532
prev_insn_idx = env->insn_idx;
21533
+
21534
+
/* Sanity check: precomputed constants must match verifier state */
21535
+
if (!state->speculative && insn_aux->const_reg_mask) {
21536
+
struct bpf_reg_state *regs = cur_regs(env);
21537
+
u16 mask = insn_aux->const_reg_mask;
21538
+
21539
+
for (int r = 0; r < ARRAY_SIZE(insn_aux->const_reg_vals); r++) {
21540
+
u32 cval = insn_aux->const_reg_vals[r];
21541
+
21542
+
if (!(mask & BIT(r)))
21543
+
continue;
21544
+
if (regs[r].type != SCALAR_VALUE)
21545
+
continue;
21546
+
if (!tnum_is_const(regs[r].var_off))
21547
+
continue;
21548
+
if (verifier_bug_if((u32)regs[r].var_off.value != cval,
21549
+
env, "const R%d: %u != %llu",
21550
+
r, cval, regs[r].var_off.value))
21551
+
return -EFAULT;
21552
+
}
21553
+
}
21759
21554
21760
21555
/* Reduce verification complexity by stopping speculative path
21761
21556
* verification when a nospec is encountered.
···
22246
21999
return err;
22247
22000
22248
22001
for (i = 0; i < insn_cnt; i++, insn++) {
22002
+
if (insn->dst_reg >= MAX_BPF_REG) {
22003
+
verbose(env, "R%d is invalid\n", insn->dst_reg);
22004
+
return -EINVAL;
22005
+
}
22006
+
if (insn->src_reg >= MAX_BPF_REG) {
22007
+
verbose(env, "R%d is invalid\n", insn->src_reg);
22008
+
return -EINVAL;
22009
+
}
22249
22010
if (BPF_CLASS(insn->code) == BPF_LDX &&
22250
22011
((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) ||
22251
22012
insn->imm != 0)) {
···
22767
22512
}
22768
22513
}
22769
22514
22770
-
static bool insn_is_cond_jump(u8 code)
22515
+
bool bpf_insn_is_cond_jump(u8 code)
22771
22516
{
22772
22517
u8 op;
22773
22518
···
22790
22535
int i;
22791
22536
22792
22537
for (i = 0; i < insn_cnt; i++, insn++) {
22793
-
if (!insn_is_cond_jump(insn->code))
22538
+
if (!bpf_insn_is_cond_jump(insn->code))
22794
22539
continue;
22795
22540
22796
22541
if (!aux_data[i + 1].seen)
···
23286
23031
* need a hard reject of the program. Thus -EFAULT is
23287
23032
* propagated in any case.
23288
23033
*/
23289
-
subprog = find_subprog(env, i + insn->imm + 1);
23034
+
subprog = bpf_find_subprog(env, i + insn->imm + 1);
23290
23035
if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d",
23291
23036
i + insn->imm + 1))
23292
23037
return -EFAULT;
···
23501
23246
if (!bpf_pseudo_call(insn))
23502
23247
continue;
23503
23248
insn->off = env->insn_aux_data[i].call_imm;
23504
-
subprog = find_subprog(env, i + insn->off + 1);
23249
+
subprog = bpf_find_subprog(env, i + insn->off + 1);
23505
23250
insn->imm = subprog;
23506
23251
}
23507
23252
···
24112
23857
goto next_insn;
24113
23858
}
24114
23859
24115
-
if (is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) {
23860
+
if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) {
24116
23861
int stack_off_cnt = -stack_depth - 16;
24117
23862
24118
23863
/*
···
24155
23900
env->prog = prog = new_prog;
24156
23901
insn = new_prog->insnsi + i + delta;
24157
23902
goto next_insn;
24158
-
} else if (is_may_goto_insn(insn)) {
23903
+
} else if (bpf_is_may_goto_insn(insn)) {
24159
23904
int stack_off = -stack_depth - 8;
24160
23905
24161
23906
stack_depth_extra = 8;
···
26049
25794
struct bpf_insn *insn,
26050
25795
struct insn_live_regs *info)
26051
25796
{
26052
-
struct call_summary cs;
25797
+
struct bpf_call_summary cs;
26053
25798
u8 class = BPF_CLASS(insn->code);
26054
25799
u8 code = BPF_OP(insn->code);
26055
25800
u8 mode = BPF_MODE(insn->code);
···
26164
25909
case BPF_CALL:
26165
25910
def = ALL_CALLER_SAVED_REGS;
26166
25911
use = def & ~BIT(BPF_REG_0);
26167
-
if (get_call_summary(env, insn, &cs))
25912
+
if (bpf_get_call_summary(env, insn, &cs))
26168
25913
use = GENMASK(cs.num_params, 1);
26169
25914
break;
26170
25915
default:
···
26264
26009
else
26265
26010
verbose(env, ".");
26266
26011
verbose(env, " ");
26267
-
verbose_insn(env, &insns[i]);
26012
+
bpf_verbose_insn(env, &insns[i]);
26268
26013
if (bpf_is_ldimm64(&insns[i]))
26269
26014
i++;
26270
26015
}
···
26581
26326
if (ret < 0)
26582
26327
goto skip_full_check;
26583
26328
26584
-
ret = compute_postorder(env);
26329
+
ret = bpf_compute_postorder(env);
26585
26330
if (ret < 0)
26586
26331
goto skip_full_check;
26587
26332
···
26591
26336
26592
26337
ret = check_attach_btf_id(env);
26593
26338
if (ret)
26339
+
goto skip_full_check;
26340
+
26341
+
ret = bpf_compute_const_regs(env);
26342
+
if (ret < 0)
26343
+
goto skip_full_check;
26344
+
26345
+
ret = bpf_prune_dead_branches(env);
26346
+
if (ret < 0)
26347
+
goto skip_full_check;
26348
+
26349
+
ret = sort_subprogs_topo(env);
26350
+
if (ret < 0)
26594
26351
goto skip_full_check;
26595
26352
26596
26353
ret = compute_scc(env);
+2
tools/testing/selftests/bpf/prog_tests/verifier.c
+2
tools/testing/selftests/bpf/prog_tests/verifier.c
···
93
93
#include "verifier_stack_ptr.skel.h"
94
94
#include "verifier_store_release.skel.h"
95
95
#include "verifier_subprog_precision.skel.h"
96
+
#include "verifier_subprog_topo.skel.h"
96
97
#include "verifier_subreg.skel.h"
97
98
#include "verifier_tailcall.skel.h"
98
99
#include "verifier_tailcall_jit.skel.h"
···
239
238
void test_verifier_stack_ptr(void) { RUN(verifier_stack_ptr); }
240
239
void test_verifier_store_release(void) { RUN(verifier_store_release); }
241
240
void test_verifier_subprog_precision(void) { RUN(verifier_subprog_precision); }
241
+
void test_verifier_subprog_topo(void) { RUN(verifier_subprog_topo); }
242
242
void test_verifier_subreg(void) { RUN(verifier_subreg); }
243
243
void test_verifier_tailcall(void) { RUN(verifier_tailcall); }
244
244
void test_verifier_tailcall_jit(void) { RUN(verifier_tailcall_jit); }
+1
-2
tools/testing/selftests/bpf/progs/verifier_loops1.c
+1
-2
tools/testing/selftests/bpf/progs/verifier_loops1.c
···
138
138
SEC("tracepoint")
139
139
__description("bounded recursion")
140
140
__failure
141
-
/* verifier limitation in detecting max stack depth */
142
-
__msg("the call stack of 8 frames is too deep !")
141
+
__msg("recursive call from")
143
142
__naked void bounded_recursion(void)
144
143
{
145
144
asm volatile (" \
+13
-7
tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
+13
-7
tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
···
592
592
*/
593
593
SEC("socket")
594
594
__success __log_level(2)
595
-
__msg("11: (1d) if r3 == r4 goto pc+0")
595
+
__msg("14: (1d) if r3 == r4 goto pc+0")
596
596
__msg("frame 0: propagating r3,r4")
597
-
__msg("11: safe")
598
-
__msg("processed 15 insns")
597
+
__msg("14: safe")
598
+
__msg("processed 18 insns")
599
599
__flag(BPF_F_TEST_STATE_FREQ)
600
600
__naked void no_scalar_id_for_const(void)
601
601
{
···
605
605
"if r0 > 7 goto l0_%=;"
606
606
/* possibly generate same scalar ids for r3 and r4 */
607
607
"r1 = 0;"
608
+
"r1 ^= r1;" /* prevent bpf_prune_dead_branches from folding the branch */
608
609
"r1 = r1;"
609
610
"r3 = r1;"
610
611
"r4 = r1;"
···
613
612
"l0_%=:"
614
613
/* possibly generate different scalar ids for r3 and r4 */
615
614
"r1 = 0;"
615
+
"r1 ^= r1;"
616
616
"r2 = 0;"
617
+
"r2 ^= r2;"
617
618
"r3 = r1;"
618
619
"r4 = r2;"
619
620
"l1_%=:"
···
631
628
/* Same as no_scalar_id_for_const() but for 32-bit values */
632
629
SEC("socket")
633
630
__success __log_level(2)
634
-
__msg("11: (1e) if w3 == w4 goto pc+0")
631
+
__msg("14: (1e) if w3 == w4 goto pc+0")
635
632
__msg("frame 0: propagating r3,r4")
636
-
__msg("11: safe")
637
-
__msg("processed 15 insns")
633
+
__msg("14: safe")
634
+
__msg("processed 18 insns")
638
635
__flag(BPF_F_TEST_STATE_FREQ)
639
636
__naked void no_scalar_id_for_const32(void)
640
637
{
···
644
641
"if r0 > 7 goto l0_%=;"
645
642
/* possibly generate same scalar ids for r3 and r4 */
646
643
"w1 = 0;"
644
+
"w1 ^= w1;" /* prevent bpf_prune_dead_branches from folding the branch */
647
645
"w1 = w1;"
648
646
"w3 = w1;"
649
647
"w4 = w1;"
···
652
648
"l0_%=:"
653
649
/* possibly generate different scalar ids for r3 and r4 */
654
650
"w1 = 0;"
651
+
"w1 ^= w1;"
655
652
"w2 = 0;"
653
+
"w2 ^= w2;"
656
654
"w3 = w1;"
657
655
"w4 = w2;"
658
656
"l1_%=:"
659
-
/* predictable jump, marks r1 and r2 precise */
657
+
/* predictable jump, marks r3 and r4 precise */
660
658
"if w3 == w4 goto +0;"
661
659
"r0 = 0;"
662
660
"exit;"
+226
tools/testing/selftests/bpf/progs/verifier_subprog_topo.c
+226
tools/testing/selftests/bpf/progs/verifier_subprog_topo.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
3
+
4
+
#include <linux/bpf.h>
5
+
#include <bpf/bpf_helpers.h>
6
+
#include "bpf_misc.h"
7
+
8
+
/* linear chain main -> A -> B */
9
+
__naked __noinline __used
10
+
static unsigned long linear_b(void)
11
+
{
12
+
asm volatile (
13
+
"r0 = 42;"
14
+
"exit;"
15
+
);
16
+
}
17
+
18
+
__naked __noinline __used
19
+
static unsigned long linear_a(void)
20
+
{
21
+
asm volatile (
22
+
"call linear_b;"
23
+
"exit;"
24
+
);
25
+
}
26
+
27
+
SEC("?raw_tp")
28
+
__success __log_level(2)
29
+
__msg("topo_order[0] = linear_b")
30
+
__msg("topo_order[1] = linear_a")
31
+
__msg("topo_order[2] = topo_linear")
32
+
__naked int topo_linear(void)
33
+
{
34
+
asm volatile (
35
+
"call linear_a;"
36
+
"exit;"
37
+
);
38
+
}
39
+
40
+
/* diamond main -> A, main -> B, A -> C, B -> C */
41
+
__naked __noinline __used
42
+
static unsigned long diamond_c(void)
43
+
{
44
+
asm volatile (
45
+
"r0 = 1;"
46
+
"exit;"
47
+
);
48
+
}
49
+
50
+
__naked __noinline __used
51
+
static unsigned long diamond_b(void)
52
+
{
53
+
asm volatile (
54
+
"call diamond_c;"
55
+
"exit;"
56
+
);
57
+
}
58
+
59
+
__naked __noinline __used
60
+
static unsigned long diamond_a(void)
61
+
{
62
+
asm volatile (
63
+
"call diamond_c;"
64
+
"exit;"
65
+
);
66
+
}
67
+
68
+
SEC("?raw_tp")
69
+
__success __log_level(2)
70
+
__msg("topo_order[0] = diamond_c")
71
+
__msg("topo_order[3] = topo_diamond")
72
+
__naked int topo_diamond(void)
73
+
{
74
+
asm volatile (
75
+
"call diamond_a;"
76
+
"call diamond_b;"
77
+
"exit;"
78
+
);
79
+
}
80
+
81
+
/* main -> global_a (global) -> static_leaf (static, leaf) */
82
+
__naked __noinline __used
83
+
static unsigned long static_leaf(void)
84
+
{
85
+
asm volatile (
86
+
"r0 = 7;"
87
+
"exit;"
88
+
);
89
+
}
90
+
91
+
__noinline __used
92
+
int global_a(int x)
93
+
{
94
+
return static_leaf();
95
+
}
96
+
97
+
SEC("?raw_tp")
98
+
__success __log_level(2)
99
+
__msg("topo_order[0] = static_leaf")
100
+
__msg("topo_order[1] = global_a")
101
+
__msg("topo_order[2] = topo_mixed")
102
+
__naked int topo_mixed(void)
103
+
{
104
+
asm volatile (
105
+
"r1 = 0;"
106
+
"call global_a;"
107
+
"exit;"
108
+
);
109
+
}
110
+
111
+
/*
112
+
* shared static callee from global and main:
113
+
* main -> shared_leaf (static)
114
+
* main -> global_b (global) -> shared_leaf (static)
115
+
*/
116
+
__naked __noinline __used
117
+
static unsigned long shared_leaf(void)
118
+
{
119
+
asm volatile (
120
+
"r0 = 99;"
121
+
"exit;"
122
+
);
123
+
}
124
+
125
+
__noinline __used
126
+
int global_b(int x)
127
+
{
128
+
return shared_leaf();
129
+
}
130
+
131
+
SEC("?raw_tp")
132
+
__success __log_level(2)
133
+
__msg("topo_order[0] = shared_leaf")
134
+
__msg("topo_order[1] = global_b")
135
+
__msg("topo_order[2] = topo_shared")
136
+
__naked int topo_shared(void)
137
+
{
138
+
asm volatile (
139
+
"call shared_leaf;"
140
+
"r1 = 0;"
141
+
"call global_b;"
142
+
"exit;"
143
+
);
144
+
}
145
+
146
+
/* duplicate calls to the same subprog */
147
+
__naked __noinline __used
148
+
static unsigned long dup_leaf(void)
149
+
{
150
+
asm volatile (
151
+
"r0 = 0;"
152
+
"exit;"
153
+
);
154
+
}
155
+
156
+
SEC("?raw_tp")
157
+
__success __log_level(2)
158
+
__msg("topo_order[0] = dup_leaf")
159
+
__msg("topo_order[1] = topo_dup_calls")
160
+
__naked int topo_dup_calls(void)
161
+
{
162
+
asm volatile (
163
+
"call dup_leaf;"
164
+
"call dup_leaf;"
165
+
"exit;"
166
+
);
167
+
}
168
+
169
+
/* main calls bpf_loop() with loop_cb as the callback */
170
+
static int loop_cb(int idx, void *ctx)
171
+
{
172
+
return 0;
173
+
}
174
+
175
+
SEC("?raw_tp")
176
+
__success __log_level(2)
177
+
__msg("topo_order[0] = loop_cb")
178
+
__msg("topo_order[1] = topo_loop_cb")
179
+
int topo_loop_cb(void)
180
+
{
181
+
bpf_loop(1, loop_cb, NULL, 0);
182
+
return 0;
183
+
}
184
+
185
+
/*
186
+
* bpf_loop callback calling another subprog
187
+
* main -> bpf_loop(callback=loop_cb2) -> loop_cb2 -> loop_cb2_leaf
188
+
*/
189
+
__naked __noinline __used
190
+
static unsigned long loop_cb2_leaf(void)
191
+
{
192
+
asm volatile (
193
+
"r0 = 0;"
194
+
"exit;"
195
+
);
196
+
}
197
+
198
+
static int loop_cb2(int idx, void *ctx)
199
+
{
200
+
return loop_cb2_leaf();
201
+
}
202
+
203
+
SEC("?raw_tp")
204
+
__success __log_level(2)
205
+
__msg("topo_order[0] = loop_cb2_leaf")
206
+
__msg("topo_order[1] = loop_cb2")
207
+
__msg("topo_order[2] = topo_loop_cb_chain")
208
+
int topo_loop_cb_chain(void)
209
+
{
210
+
bpf_loop(1, loop_cb2, NULL, 0);
211
+
return 0;
212
+
}
213
+
214
+
/* no calls (single subprog) */
215
+
SEC("?raw_tp")
216
+
__success __log_level(2)
217
+
__msg("topo_order[0] = topo_no_calls")
218
+
__naked int topo_no_calls(void)
219
+
{
220
+
asm volatile (
221
+
"r0 = 0;"
222
+
"exit;"
223
+
);
224
+
}
225
+
226
+
char _license[] SEC("license") = "GPL";
+5
-1
tools/testing/selftests/bpf/progs/verifier_unpriv.c
+5
-1
tools/testing/selftests/bpf/progs/verifier_unpriv.c
···
584
584
{
585
585
asm volatile (" \
586
586
w7 = 0; \
587
-
w7 &= 1; \
587
+
w7 ^= w7; \
588
588
w0 = w7; \
589
589
if r0 == 0 goto l0_%=; \
590
590
r0 = *(u64*)(r7 + 0); \
···
894
894
{
895
895
asm volatile (" \
896
896
r8 = 0; \
897
+
r8 ^= r8; \
897
898
r9 = 0; \
899
+
r9 ^= r9; \
898
900
r0 = r10; \
899
901
r1 = 0; \
900
902
r2 = r10; \
···
934
932
{
935
933
asm volatile (" \
936
934
r8 = 0; \
935
+
r8 ^= r8; \
937
936
r9 = 0; \
937
+
r9 ^= r9; \
938
938
r0 = r10; \
939
939
r1 = 0; \
940
940
r2 = r10; \
+3
-3
tools/testing/selftests/bpf/verifier/calls.c
+3
-3
tools/testing/selftests/bpf/verifier/calls.c
···
455
455
BPF_EXIT_INSN(),
456
456
},
457
457
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
458
-
.errstr = "the call stack of 9 frames is too deep",
458
+
.errstr = "recursive call",
459
459
.result = REJECT,
460
460
},
461
461
{
···
812
812
BPF_EXIT_INSN(),
813
813
},
814
814
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
815
-
.errstr = "the call stack of 9 frames is too deep",
815
+
.errstr = "recursive call",
816
816
.result = REJECT,
817
817
},
818
818
{
···
824
824
BPF_EXIT_INSN(),
825
825
},
826
826
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
827
-
.errstr = "the call stack of 9 frames is too deep",
827
+
.errstr = "recursive call",
828
828
.result = REJECT,
829
829
},
830
830
{
+2
-2
tools/testing/selftests/bpf/verifier/junk_insn.c
+2
-2
tools/testing/selftests/bpf/verifier/junk_insn.c
···
28
28
{
29
29
"junk insn4",
30
30
.insns = {
31
-
BPF_RAW_INSN(-1, -1, -1, -1, -1),
31
+
BPF_RAW_INSN(-1, 0, 0, -1, -1),
32
32
BPF_EXIT_INSN(),
33
33
},
34
34
.errstr = "unknown opcode ff",
···
37
37
{
38
38
"junk insn5",
39
39
.insns = {
40
-
BPF_RAW_INSN(0x7f, -1, -1, -1, -1),
40
+
BPF_RAW_INSN(0x7f, 0, 0, -1, -1),
41
41
BPF_EXIT_INSN(),
42
42
},
43
43
.errstr = "BPF_ALU uses reserved fields",