bpf: verifier: Move desc->imm setup to sort_kfunc_descs_by_imm_off()

Metadata about a kfunc call is added to the kfunc_tab in
add_kfunc_call() but the call instruction itself could get removed by
opt_remove_dead_code() later if it is not reachable.

If the call instruction is removed, specialize_kfunc() is never called
for it and the desc->imm in the kfunc_tab is never initialized for this
kfunc call. In this case, sort_kfunc_descs_by_imm_off(env->prog); in
do_misc_fixups() doesn't sort the table correctly.
This is a problem for s390 as its JIT uses this table to find the
addresses for kfuncs, and if this table is not sorted properly, JIT may
fail to find addresses for valid kfunc calls.

This was exposed by:

commit d869d56ca848 ("bpf: verifier: refactor kfunc specialization")

as before this commit, desc->imm was initialised in add_kfunc_call()
which happens before dead code elimination.

Move desc->imm setup down to sort_kfunc_descs_by_imm_off(), this fixes
the problem and also saves us from having the same logic in
add_kfunc_call() and specialize_kfunc().

Suggested-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20251114154023.12801-1-puranjay@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

authored by

Puranjay Mohan and committed by

Alexei Starovoitov 7 months ago 4f7bc83b a4d31f45

+35 -19

1 changed file

expand all

kernel

bpf

verifier.c

+35 -19

kernel/bpf/verifier.c

··· 3391 3391 return 0; 3392 3392 } 3393 3393 3394 - static void sort_kfunc_descs_by_imm_off(struct bpf_prog *prog) 3394 + static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc) 3395 + { 3396 + unsigned long call_imm; 3397 + 3398 + if (bpf_jit_supports_far_kfunc_call()) { 3399 + call_imm = desc->func_id; 3400 + } else { 3401 + call_imm = BPF_CALL_IMM(desc->addr); 3402 + /* Check whether the relative offset overflows desc->imm */ 3403 + if ((unsigned long)(s32)call_imm != call_imm) { 3404 + verbose(env, "address of kernel func_id %u is out of range\n", 3405 + desc->func_id); 3406 + return -EINVAL; 3407 + } 3408 + } 3409 + desc->imm = call_imm; 3410 + return 0; 3411 + } 3412 + 3413 + static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env) 3395 3414 { 3396 3415 struct bpf_kfunc_desc_tab *tab; 3416 + int i, err; 3397 3417 3398 - tab = prog->aux->kfunc_tab; 3418 + tab = env->prog->aux->kfunc_tab; 3399 3419 if (!tab) 3400 - return; 3420 + return 0; 3421 + 3422 + for (i = 0; i < tab->nr_descs; i++) { 3423 + err = set_kfunc_desc_imm(env, &tab->descs[i]); 3424 + if (err) 3425 + return err; 3426 + } 3401 3427 3402 3428 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 3403 3429 kfunc_desc_cmp_by_imm_off, NULL); 3430 + return 0; 3404 3431 } 3405 3432 3406 3433 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) ··· 22350 22323 bool is_rdonly; 22351 22324 u32 func_id = desc->func_id; 22352 22325 u16 offset = desc->offset; 22353 - unsigned long addr = desc->addr, call_imm; 22326 + unsigned long addr = desc->addr; 22354 22327 22355 22328 if (offset) /* return if module BTF is used */ 22356 - goto set_imm; 22329 + return 0; 22357 22330 22358 22331 if (bpf_dev_bound_kfunc_id(func_id)) { 22359 22332 xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id); ··· 22381 22354 if (!env->insn_aux_data[insn_idx].non_sleepable) 22382 22355 addr = (unsigned long)bpf_dynptr_from_file_sleepable; 22383 22356 } 22384 - 22385 - set_imm: 22386 - if (bpf_jit_supports_far_kfunc_call()) { 22387 - call_imm = func_id; 22388 - } else { 22389 - call_imm = BPF_CALL_IMM(addr); 22390 - /* Check whether the relative offset overflows desc->imm */ 22391 - if ((unsigned long)(s32)call_imm != call_imm) { 22392 - verbose(env, "address of kernel func_id %u is out of range\n", func_id); 22393 - return -EINVAL; 22394 - } 22395 - } 22396 - desc->imm = call_imm; 22397 22357 desc->addr = addr; 22398 22358 return 0; 22399 22359 } ··· 23458 23444 } 23459 23445 } 23460 23446 23461 - sort_kfunc_descs_by_imm_off(env->prog); 23447 + ret = sort_kfunc_descs_by_imm_off(env); 23448 + if (ret) 23449 + return ret; 23462 23450 23463 23451 return 0; 23464 23452 }

Configure Feed

Configure Feed