tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

bpf: Support struct btf_struct_meta via KF_IMPLICIT_ARGS

The following kfuncs currently accept void *meta__ign argument:
* bpf_obj_new_impl
* bpf_obj_drop_impl
* bpf_percpu_obj_new_impl
* bpf_percpu_obj_drop_impl
* bpf_refcount_acquire_impl
* bpf_list_push_back_impl
* bpf_list_push_front_impl
* bpf_rbtree_add_impl

The __ign suffix is an indicator for the verifier to skip the argument
in check_kfunc_args(). Then, in fixup_kfunc_call() the verifier may
set the value of this argument to struct btf_struct_meta *
kptr_struct_meta from insn_aux_data.

BPF programs must pass a dummy NULL value when calling these kfuncs.

Additionally, the list and rbtree _impl kfuncs also accept an implicit
u64 argument, which doesn't require __ign suffix because it's a
scalar, and BPF programs explicitly pass 0.

Add new kfuncs with KF_IMPLICIT_ARGS [1], that correspond to each
_impl kfunc accepting meta__ign. The existing _impl kfuncs remain
unchanged for backwards compatibility.

To support this, add "btf_struct_meta" to the list of recognized
implicit argument types in resolve_btfids.

Implement is_kfunc_arg_implicit() in the verifier, that determines
implicit args by inspecting both a non-_impl BTF prototype of the
kfunc.

Update the special_kfunc_list in the verifier and relevant checks to
support both the old _impl and the new KF_IMPLICIT_ARGS variants of
btf_struct_meta users.

[1] https://lore.kernel.org/bpf/20260120222638.3976562-1-ihor.solodrai@linux.dev/

Signed-off-by: Ihor Solodrai <ihor.solodrai@linux.dev>
Acked-by: Jiri Olsa <jolsa@kernel.org>
Link: https://lore.kernel.org/r/20260327203241.3365046-1-ihor.solodrai@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

authored by

Ihor Solodrai and committed by
Alexei Starovoitov d4570725 5e961eeb

+290 -79
+1 -1
include/linux/btf_ids.h
··· 217 217 218 218 #else 219 219 220 - #define BTF_ID_LIST(name) static u32 __maybe_unused name[64]; 220 + #define BTF_ID_LIST(name) static u32 __maybe_unused name[128]; 221 221 #define BTF_ID(prefix, name) 222 222 #define BTF_ID_FLAGS(prefix, name, ...) 223 223 #define BTF_ID_UNUSED
+161 -23
kernel/bpf/helpers.c
··· 2302 2302 2303 2303 __bpf_kfunc_start_defs(); 2304 2304 2305 - __bpf_kfunc void *bpf_obj_new_impl(u64 local_type_id__k, void *meta__ign) 2305 + /** 2306 + * bpf_obj_new() - allocate an object described by program BTF 2307 + * @local_type_id__k: type ID in program BTF 2308 + * @meta: verifier-supplied struct metadata 2309 + * 2310 + * Allocate an object of the type identified by @local_type_id__k and 2311 + * initialize its special fields. BPF programs can use 2312 + * bpf_core_type_id_local() to provide @local_type_id__k. The verifier 2313 + * rewrites @meta; BPF programs do not set it. 2314 + * 2315 + * Return: Pointer to the allocated object, or %NULL on failure. 2316 + */ 2317 + __bpf_kfunc void *bpf_obj_new(u64 local_type_id__k, struct btf_struct_meta *meta) 2306 2318 { 2307 - struct btf_struct_meta *meta = meta__ign; 2308 2319 u64 size = local_type_id__k; 2309 2320 void *p; 2310 2321 ··· 2324 2313 return NULL; 2325 2314 if (meta) 2326 2315 bpf_obj_init(meta->record, p); 2316 + 2327 2317 return p; 2318 + } 2319 + 2320 + __bpf_kfunc void *bpf_obj_new_impl(u64 local_type_id__k, void *meta__ign) 2321 + { 2322 + return bpf_obj_new(local_type_id__k, meta__ign); 2323 + } 2324 + 2325 + /** 2326 + * bpf_percpu_obj_new() - allocate a percpu object described by program BTF 2327 + * @local_type_id__k: type ID in program BTF 2328 + * @meta: verifier-supplied struct metadata 2329 + * 2330 + * Allocate a percpu object of the type identified by @local_type_id__k. BPF 2331 + * programs can use bpf_core_type_id_local() to provide @local_type_id__k. 2332 + * The verifier rewrites @meta; BPF programs do not set it. 2333 + * 2334 + * Return: Pointer to the allocated percpu object, or %NULL on failure. 2335 + */ 2336 + __bpf_kfunc void *bpf_percpu_obj_new(u64 local_type_id__k, struct btf_struct_meta *meta) 2337 + { 2338 + u64 size = local_type_id__k; 2339 + 2340 + /* The verifier has ensured that meta must be NULL */ 2341 + return bpf_mem_alloc(&bpf_global_percpu_ma, size); 2328 2342 } 2329 2343 2330 2344 __bpf_kfunc void *bpf_percpu_obj_new_impl(u64 local_type_id__k, void *meta__ign) 2331 2345 { 2332 - u64 size = local_type_id__k; 2333 - 2334 - /* The verifier has ensured that meta__ign must be NULL */ 2335 - return bpf_mem_alloc(&bpf_global_percpu_ma, size); 2346 + return bpf_percpu_obj_new(local_type_id__k, meta__ign); 2336 2347 } 2337 2348 2338 2349 /* Must be called under migrate_disable(), as required by bpf_mem_free */ ··· 2380 2347 bpf_mem_free_rcu(ma, p); 2381 2348 } 2382 2349 2383 - __bpf_kfunc void bpf_obj_drop_impl(void *p__alloc, void *meta__ign) 2350 + /** 2351 + * bpf_obj_drop() - drop a previously allocated object 2352 + * @p__alloc: object to free 2353 + * @meta: verifier-supplied struct metadata 2354 + * 2355 + * Destroy special fields in @p__alloc as needed and free the object. The 2356 + * verifier rewrites @meta; BPF programs do not set it. 2357 + */ 2358 + __bpf_kfunc void bpf_obj_drop(void *p__alloc, struct btf_struct_meta *meta) 2384 2359 { 2385 - struct btf_struct_meta *meta = meta__ign; 2386 2360 void *p = p__alloc; 2387 2361 2388 2362 __bpf_obj_drop_impl(p, meta ? meta->record : NULL, false); 2389 2363 } 2390 2364 2391 - __bpf_kfunc void bpf_percpu_obj_drop_impl(void *p__alloc, void *meta__ign) 2365 + __bpf_kfunc void bpf_obj_drop_impl(void *p__alloc, void *meta__ign) 2392 2366 { 2393 - /* The verifier has ensured that meta__ign must be NULL */ 2367 + return bpf_obj_drop(p__alloc, meta__ign); 2368 + } 2369 + 2370 + /** 2371 + * bpf_percpu_obj_drop() - drop a previously allocated percpu object 2372 + * @p__alloc: percpu object to free 2373 + * @meta: verifier-supplied struct metadata 2374 + * 2375 + * Free @p__alloc. The verifier rewrites @meta; BPF programs do not set it. 2376 + */ 2377 + __bpf_kfunc void bpf_percpu_obj_drop(void *p__alloc, struct btf_struct_meta *meta) 2378 + { 2379 + /* The verifier has ensured that meta must be NULL */ 2394 2380 bpf_mem_free_rcu(&bpf_global_percpu_ma, p__alloc); 2395 2381 } 2396 2382 2397 - __bpf_kfunc void *bpf_refcount_acquire_impl(void *p__refcounted_kptr, void *meta__ign) 2383 + __bpf_kfunc void bpf_percpu_obj_drop_impl(void *p__alloc, void *meta__ign) 2398 2384 { 2399 - struct btf_struct_meta *meta = meta__ign; 2385 + bpf_percpu_obj_drop(p__alloc, meta__ign); 2386 + } 2387 + 2388 + /** 2389 + * bpf_refcount_acquire() - turn a local kptr into an owning reference 2390 + * @p__refcounted_kptr: non-owning local kptr 2391 + * @meta: verifier-supplied struct metadata 2392 + * 2393 + * Increment the refcount for @p__refcounted_kptr. The verifier rewrites 2394 + * @meta; BPF programs do not set it. 2395 + * 2396 + * Return: Owning reference to @p__refcounted_kptr, or %NULL on failure. 2397 + */ 2398 + __bpf_kfunc void *bpf_refcount_acquire(void *p__refcounted_kptr, struct btf_struct_meta *meta) 2399 + { 2400 2400 struct bpf_refcount *ref; 2401 2401 2402 2402 /* Could just cast directly to refcount_t *, but need some code using ··· 2443 2377 * in verifier.c 2444 2378 */ 2445 2379 return (void *)p__refcounted_kptr; 2380 + } 2381 + 2382 + __bpf_kfunc void *bpf_refcount_acquire_impl(void *p__refcounted_kptr, void *meta__ign) 2383 + { 2384 + return bpf_refcount_acquire(p__refcounted_kptr, meta__ign); 2446 2385 } 2447 2386 2448 2387 static int __bpf_list_add(struct bpf_list_node_kern *node, ··· 2477 2406 return 0; 2478 2407 } 2479 2408 2409 + /** 2410 + * bpf_list_push_front() - add a node to the front of a BPF linked list 2411 + * @head: list head 2412 + * @node: node to insert 2413 + * @meta: verifier-supplied struct metadata 2414 + * @off: verifier-supplied offset of @node within the containing object 2415 + * 2416 + * Insert @node at the front of @head. The verifier rewrites @meta and @off; 2417 + * BPF programs do not set them. 2418 + * 2419 + * Return: 0 on success, or %-EINVAL if @node is already linked. 2420 + */ 2421 + __bpf_kfunc int bpf_list_push_front(struct bpf_list_head *head, 2422 + struct bpf_list_node *node, 2423 + struct btf_struct_meta *meta, 2424 + u64 off) 2425 + { 2426 + struct bpf_list_node_kern *n = (void *)node; 2427 + 2428 + return __bpf_list_add(n, head, false, meta ? meta->record : NULL, off); 2429 + } 2430 + 2480 2431 __bpf_kfunc int bpf_list_push_front_impl(struct bpf_list_head *head, 2481 2432 struct bpf_list_node *node, 2482 2433 void *meta__ign, u64 off) 2483 2434 { 2484 - struct bpf_list_node_kern *n = (void *)node; 2485 - struct btf_struct_meta *meta = meta__ign; 2435 + return bpf_list_push_front(head, node, meta__ign, off); 2436 + } 2486 2437 2487 - return __bpf_list_add(n, head, false, meta ? meta->record : NULL, off); 2438 + /** 2439 + * bpf_list_push_back() - add a node to the back of a BPF linked list 2440 + * @head: list head 2441 + * @node: node to insert 2442 + * @meta: verifier-supplied struct metadata 2443 + * @off: verifier-supplied offset of @node within the containing object 2444 + * 2445 + * Insert @node at the back of @head. The verifier rewrites @meta and @off; 2446 + * BPF programs do not set them. 2447 + * 2448 + * Return: 0 on success, or %-EINVAL if @node is already linked. 2449 + */ 2450 + __bpf_kfunc int bpf_list_push_back(struct bpf_list_head *head, 2451 + struct bpf_list_node *node, 2452 + struct btf_struct_meta *meta, 2453 + u64 off) 2454 + { 2455 + struct bpf_list_node_kern *n = (void *)node; 2456 + 2457 + return __bpf_list_add(n, head, true, meta ? meta->record : NULL, off); 2488 2458 } 2489 2459 2490 2460 __bpf_kfunc int bpf_list_push_back_impl(struct bpf_list_head *head, 2491 2461 struct bpf_list_node *node, 2492 2462 void *meta__ign, u64 off) 2493 2463 { 2494 - struct bpf_list_node_kern *n = (void *)node; 2495 - struct btf_struct_meta *meta = meta__ign; 2496 - 2497 - return __bpf_list_add(n, head, true, meta ? meta->record : NULL, off); 2464 + return bpf_list_push_back(head, node, meta__ign, off); 2498 2465 } 2499 2466 2500 2467 static struct bpf_list_node *__bpf_list_del(struct bpf_list_head *head, bool tail) ··· 2644 2535 return 0; 2645 2536 } 2646 2537 2538 + /** 2539 + * bpf_rbtree_add() - add a node to a BPF rbtree 2540 + * @root: tree root 2541 + * @node: node to insert 2542 + * @less: comparator used to order nodes 2543 + * @meta: verifier-supplied struct metadata 2544 + * @off: verifier-supplied offset of @node within the containing object 2545 + * 2546 + * Insert @node into @root using @less. The verifier rewrites @meta and @off; 2547 + * BPF programs do not set them. 2548 + * 2549 + * Return: 0 on success, or %-EINVAL if @node is already linked in a tree. 2550 + */ 2551 + __bpf_kfunc int bpf_rbtree_add(struct bpf_rb_root *root, 2552 + struct bpf_rb_node *node, 2553 + bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b), 2554 + struct btf_struct_meta *meta, 2555 + u64 off) 2556 + { 2557 + struct bpf_rb_node_kern *n = (void *)node; 2558 + 2559 + return __bpf_rbtree_add(root, n, (void *)less, meta ? meta->record : NULL, off); 2560 + } 2561 + 2647 2562 __bpf_kfunc int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, 2648 2563 bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b), 2649 2564 void *meta__ign, u64 off) 2650 2565 { 2651 - struct btf_struct_meta *meta = meta__ign; 2652 - struct bpf_rb_node_kern *n = (void *)node; 2653 - 2654 - return __bpf_rbtree_add(root, n, (void *)less, meta ? meta->record : NULL, off); 2566 + return bpf_rbtree_add(root, node, less, meta__ign, off); 2655 2567 } 2656 2568 2657 2569 __bpf_kfunc struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) ··· 4666 4536 #ifdef CONFIG_CRASH_DUMP 4667 4537 BTF_ID_FLAGS(func, crash_kexec, KF_DESTRUCTIVE) 4668 4538 #endif 4539 + BTF_ID_FLAGS(func, bpf_obj_new, KF_ACQUIRE | KF_RET_NULL | KF_IMPLICIT_ARGS) 4669 4540 BTF_ID_FLAGS(func, bpf_obj_new_impl, KF_ACQUIRE | KF_RET_NULL) 4541 + BTF_ID_FLAGS(func, bpf_percpu_obj_new, KF_ACQUIRE | KF_RET_NULL | KF_IMPLICIT_ARGS) 4670 4542 BTF_ID_FLAGS(func, bpf_percpu_obj_new_impl, KF_ACQUIRE | KF_RET_NULL) 4543 + BTF_ID_FLAGS(func, bpf_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS) 4671 4544 BTF_ID_FLAGS(func, bpf_obj_drop_impl, KF_RELEASE) 4545 + BTF_ID_FLAGS(func, bpf_percpu_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS) 4672 4546 BTF_ID_FLAGS(func, bpf_percpu_obj_drop_impl, KF_RELEASE) 4547 + BTF_ID_FLAGS(func, bpf_refcount_acquire, KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_IMPLICIT_ARGS) 4673 4548 BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, KF_ACQUIRE | KF_RET_NULL | KF_RCU) 4549 + BTF_ID_FLAGS(func, bpf_list_push_front, KF_IMPLICIT_ARGS) 4674 4550 BTF_ID_FLAGS(func, bpf_list_push_front_impl) 4551 + BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS) 4675 4552 BTF_ID_FLAGS(func, bpf_list_push_back_impl) 4676 4553 BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL) 4677 4554 BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL) ··· 4687 4550 BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) 4688 4551 BTF_ID_FLAGS(func, bpf_task_release, KF_RELEASE) 4689 4552 BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL) 4553 + BTF_ID_FLAGS(func, bpf_rbtree_add, KF_IMPLICIT_ARGS) 4690 4554 BTF_ID_FLAGS(func, bpf_rbtree_add_impl) 4691 4555 BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL) 4692 4556 BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL)
+125 -53
kernel/bpf/verifier.c
··· 12473 12473 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_PROG_AUX_ID); 12474 12474 } 12475 12475 12476 + /* 12477 + * A kfunc with KF_IMPLICIT_ARGS has two prototypes in BTF: 12478 + * - the _impl prototype with full arg list (meta->func_proto) 12479 + * - the BPF API prototype w/o implicit args (func->type in BTF) 12480 + * To determine whether an argument is implicit, we compare its position 12481 + * against the number of arguments in the prototype w/o implicit args. 12482 + */ 12483 + static bool is_kfunc_arg_implicit(const struct bpf_kfunc_call_arg_meta *meta, u32 arg_idx) 12484 + { 12485 + const struct btf_type *func, *func_proto; 12486 + u32 argn; 12487 + 12488 + if (!(meta->kfunc_flags & KF_IMPLICIT_ARGS)) 12489 + return false; 12490 + 12491 + func = btf_type_by_id(meta->btf, meta->func_id); 12492 + func_proto = btf_type_by_id(meta->btf, func->type); 12493 + argn = btf_type_vlen(func_proto); 12494 + 12495 + return argn <= arg_idx; 12496 + } 12497 + 12476 12498 /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ 12477 12499 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, 12478 12500 const struct btf *btf, ··· 12561 12539 12562 12540 enum special_kfunc_type { 12563 12541 KF_bpf_obj_new_impl, 12542 + KF_bpf_obj_new, 12564 12543 KF_bpf_obj_drop_impl, 12544 + KF_bpf_obj_drop, 12565 12545 KF_bpf_refcount_acquire_impl, 12546 + KF_bpf_refcount_acquire, 12566 12547 KF_bpf_list_push_front_impl, 12548 + KF_bpf_list_push_front, 12567 12549 KF_bpf_list_push_back_impl, 12550 + KF_bpf_list_push_back, 12568 12551 KF_bpf_list_pop_front, 12569 12552 KF_bpf_list_pop_back, 12570 12553 KF_bpf_list_front, ··· 12580 12553 KF_bpf_rcu_read_unlock, 12581 12554 KF_bpf_rbtree_remove, 12582 12555 KF_bpf_rbtree_add_impl, 12556 + KF_bpf_rbtree_add, 12583 12557 KF_bpf_rbtree_first, 12584 12558 KF_bpf_rbtree_root, 12585 12559 KF_bpf_rbtree_left, ··· 12593 12565 KF_bpf_dynptr_slice_rdwr, 12594 12566 KF_bpf_dynptr_clone, 12595 12567 KF_bpf_percpu_obj_new_impl, 12568 + KF_bpf_percpu_obj_new, 12596 12569 KF_bpf_percpu_obj_drop_impl, 12570 + KF_bpf_percpu_obj_drop, 12597 12571 KF_bpf_throw, 12598 12572 KF_bpf_wq_set_callback, 12599 12573 KF_bpf_preempt_disable, ··· 12629 12599 12630 12600 BTF_ID_LIST(special_kfunc_list) 12631 12601 BTF_ID(func, bpf_obj_new_impl) 12602 + BTF_ID(func, bpf_obj_new) 12632 12603 BTF_ID(func, bpf_obj_drop_impl) 12604 + BTF_ID(func, bpf_obj_drop) 12633 12605 BTF_ID(func, bpf_refcount_acquire_impl) 12606 + BTF_ID(func, bpf_refcount_acquire) 12634 12607 BTF_ID(func, bpf_list_push_front_impl) 12608 + BTF_ID(func, bpf_list_push_front) 12635 12609 BTF_ID(func, bpf_list_push_back_impl) 12610 + BTF_ID(func, bpf_list_push_back) 12636 12611 BTF_ID(func, bpf_list_pop_front) 12637 12612 BTF_ID(func, bpf_list_pop_back) 12638 12613 BTF_ID(func, bpf_list_front) ··· 12648 12613 BTF_ID(func, bpf_rcu_read_unlock) 12649 12614 BTF_ID(func, bpf_rbtree_remove) 12650 12615 BTF_ID(func, bpf_rbtree_add_impl) 12616 + BTF_ID(func, bpf_rbtree_add) 12651 12617 BTF_ID(func, bpf_rbtree_first) 12652 12618 BTF_ID(func, bpf_rbtree_root) 12653 12619 BTF_ID(func, bpf_rbtree_left) ··· 12668 12632 BTF_ID(func, bpf_dynptr_slice_rdwr) 12669 12633 BTF_ID(func, bpf_dynptr_clone) 12670 12634 BTF_ID(func, bpf_percpu_obj_new_impl) 12635 + BTF_ID(func, bpf_percpu_obj_new) 12671 12636 BTF_ID(func, bpf_percpu_obj_drop_impl) 12637 + BTF_ID(func, bpf_percpu_obj_drop) 12672 12638 BTF_ID(func, bpf_throw) 12673 12639 BTF_ID(func, bpf_wq_set_callback) 12674 12640 BTF_ID(func, bpf_preempt_disable) ··· 12714 12676 BTF_ID(func, bpf_stream_vprintk) 12715 12677 BTF_ID(func, bpf_stream_print_stack) 12716 12678 12679 + static bool is_bpf_obj_new_kfunc(u32 func_id) 12680 + { 12681 + return func_id == special_kfunc_list[KF_bpf_obj_new] || 12682 + func_id == special_kfunc_list[KF_bpf_obj_new_impl]; 12683 + } 12684 + 12685 + static bool is_bpf_percpu_obj_new_kfunc(u32 func_id) 12686 + { 12687 + return func_id == special_kfunc_list[KF_bpf_percpu_obj_new] || 12688 + func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]; 12689 + } 12690 + 12691 + static bool is_bpf_obj_drop_kfunc(u32 func_id) 12692 + { 12693 + return func_id == special_kfunc_list[KF_bpf_obj_drop] || 12694 + func_id == special_kfunc_list[KF_bpf_obj_drop_impl]; 12695 + } 12696 + 12697 + static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id) 12698 + { 12699 + return func_id == special_kfunc_list[KF_bpf_percpu_obj_drop] || 12700 + func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl]; 12701 + } 12702 + 12703 + static bool is_bpf_refcount_acquire_kfunc(u32 func_id) 12704 + { 12705 + return func_id == special_kfunc_list[KF_bpf_refcount_acquire] || 12706 + func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]; 12707 + } 12708 + 12709 + static bool is_bpf_list_push_kfunc(u32 func_id) 12710 + { 12711 + return func_id == special_kfunc_list[KF_bpf_list_push_front] || 12712 + func_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 12713 + func_id == special_kfunc_list[KF_bpf_list_push_back] || 12714 + func_id == special_kfunc_list[KF_bpf_list_push_back_impl]; 12715 + } 12716 + 12717 + static bool is_bpf_rbtree_add_kfunc(u32 func_id) 12718 + { 12719 + return func_id == special_kfunc_list[KF_bpf_rbtree_add] || 12720 + func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]; 12721 + } 12722 + 12717 12723 static bool is_task_work_add_kfunc(u32 func_id) 12718 12724 { 12719 12725 return func_id == special_kfunc_list[KF_bpf_task_work_schedule_signal] || ··· 12766 12684 12767 12685 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) 12768 12686 { 12769 - if (meta->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] && 12770 - meta->arg_owning_ref) { 12687 + if (is_bpf_refcount_acquire_kfunc(meta->func_id) && meta->arg_owning_ref) 12771 12688 return false; 12772 - } 12773 12689 12774 12690 return meta->kfunc_flags & KF_RET_NULL; 12775 12691 } ··· 13155 13075 13156 13076 static bool is_bpf_list_api_kfunc(u32 btf_id) 13157 13077 { 13158 - return btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 13159 - btf_id == special_kfunc_list[KF_bpf_list_push_back_impl] || 13078 + return is_bpf_list_push_kfunc(btf_id) || 13160 13079 btf_id == special_kfunc_list[KF_bpf_list_pop_front] || 13161 13080 btf_id == special_kfunc_list[KF_bpf_list_pop_back] || 13162 13081 btf_id == special_kfunc_list[KF_bpf_list_front] || ··· 13164 13085 13165 13086 static bool is_bpf_rbtree_api_kfunc(u32 btf_id) 13166 13087 { 13167 - return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] || 13088 + return is_bpf_rbtree_add_kfunc(btf_id) || 13168 13089 btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 13169 13090 btf_id == special_kfunc_list[KF_bpf_rbtree_first] || 13170 13091 btf_id == special_kfunc_list[KF_bpf_rbtree_root] || ··· 13181 13102 13182 13103 static bool is_bpf_graph_api_kfunc(u32 btf_id) 13183 13104 { 13184 - return is_bpf_list_api_kfunc(btf_id) || is_bpf_rbtree_api_kfunc(btf_id) || 13185 - btf_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]; 13105 + return is_bpf_list_api_kfunc(btf_id) || 13106 + is_bpf_rbtree_api_kfunc(btf_id) || 13107 + is_bpf_refcount_acquire_kfunc(btf_id); 13186 13108 } 13187 13109 13188 13110 static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) ··· 13216 13136 13217 13137 static bool is_sync_callback_calling_kfunc(u32 btf_id) 13218 13138 { 13219 - return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]; 13139 + return is_bpf_rbtree_add_kfunc(btf_id); 13220 13140 } 13221 13141 13222 13142 static bool is_async_callback_calling_kfunc(u32 btf_id) ··· 13280 13200 13281 13201 switch (node_field_type) { 13282 13202 case BPF_LIST_NODE: 13283 - ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 13284 - kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_back_impl]); 13203 + ret = is_bpf_list_push_kfunc(kfunc_btf_id); 13285 13204 break; 13286 13205 case BPF_RB_NODE: 13287 - ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 13288 - kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] || 13206 + ret = (is_bpf_rbtree_add_kfunc(kfunc_btf_id) || 13207 + kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 13289 13208 kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_left] || 13290 13209 kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_right]); 13291 13210 break; ··· 13501 13422 bool is_ret_buf_sz = false; 13502 13423 int kf_arg_type; 13503 13424 13504 - t = btf_type_skip_modifiers(btf, args[i].type, NULL); 13505 - 13506 - if (is_kfunc_arg_ignore(btf, &args[i])) 13507 - continue; 13508 - 13509 13425 if (is_kfunc_arg_prog_aux(btf, &args[i])) { 13510 13426 /* Reject repeated use bpf_prog_aux */ 13511 13427 if (meta->arg_prog) { ··· 13511 13437 cur_aux(env)->arg_prog = regno; 13512 13438 continue; 13513 13439 } 13440 + 13441 + if (is_kfunc_arg_ignore(btf, &args[i]) || is_kfunc_arg_implicit(meta, i)) 13442 + continue; 13443 + 13444 + t = btf_type_skip_modifiers(btf, args[i].type, NULL); 13514 13445 13515 13446 if (btf_type_is_scalar(t)) { 13516 13447 if (reg->type != SCALAR_VALUE) { ··· 13691 13612 break; 13692 13613 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 13693 13614 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { 13694 - if (meta->func_id != special_kfunc_list[KF_bpf_obj_drop_impl]) { 13695 - verbose(env, "arg#%d expected for bpf_obj_drop_impl()\n", i); 13615 + if (!is_bpf_obj_drop_kfunc(meta->func_id)) { 13616 + verbose(env, "arg#%d expected for bpf_obj_drop()\n", i); 13696 13617 return -EINVAL; 13697 13618 } 13698 13619 } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { 13699 - if (meta->func_id != special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) { 13700 - verbose(env, "arg#%d expected for bpf_percpu_obj_drop_impl()\n", i); 13620 + if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { 13621 + verbose(env, "arg#%d expected for bpf_percpu_obj_drop()\n", i); 13701 13622 return -EINVAL; 13702 13623 } 13703 13624 } else { ··· 13823 13744 return ret; 13824 13745 break; 13825 13746 case KF_ARG_PTR_TO_RB_NODE: 13826 - if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 13747 + if (is_bpf_rbtree_add_kfunc(meta->func_id)) { 13827 13748 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 13828 13749 verbose(env, "arg#%d expected pointer to allocated object\n", i); 13829 13750 return -EINVAL; ··· 14060 13981 if (meta->btf != btf_vmlinux) 14061 13982 return 0; 14062 13983 14063 - if (meta->func_id == special_kfunc_list[KF_bpf_obj_new_impl] || 14064 - meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { 13984 + if (is_bpf_obj_new_kfunc(meta->func_id) || is_bpf_percpu_obj_new_kfunc(meta->func_id)) { 14065 13985 struct btf_struct_meta *struct_meta; 14066 13986 struct btf *ret_btf; 14067 13987 u32 ret_btf_id; 14068 13988 14069 - if (meta->func_id == special_kfunc_list[KF_bpf_obj_new_impl] && !bpf_global_ma_set) 13989 + if (is_bpf_obj_new_kfunc(meta->func_id) && !bpf_global_ma_set) 14070 13990 return -ENOMEM; 14071 13991 14072 13992 if (((u64)(u32)meta->arg_constant.value) != meta->arg_constant.value) { ··· 14088 14010 return -EINVAL; 14089 14011 } 14090 14012 14091 - if (meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { 14013 + if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { 14092 14014 if (ret_t->size > BPF_GLOBAL_PERCPU_MA_MAX_SIZE) { 14093 14015 verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n", 14094 14016 ret_t->size, BPF_GLOBAL_PERCPU_MA_MAX_SIZE); ··· 14118 14040 } 14119 14041 14120 14042 struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); 14121 - if (meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { 14043 + if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { 14122 14044 if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { 14123 14045 verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); 14124 14046 return -EINVAL; ··· 14134 14056 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 14135 14057 regs[BPF_REG_0].btf = ret_btf; 14136 14058 regs[BPF_REG_0].btf_id = ret_btf_id; 14137 - if (meta->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) 14059 + if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) 14138 14060 regs[BPF_REG_0].type |= MEM_PERCPU; 14139 14061 14140 14062 insn_aux->obj_new_size = ret_t->size; 14141 14063 insn_aux->kptr_struct_meta = struct_meta; 14142 - } else if (meta->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) { 14064 + } else if (is_bpf_refcount_acquire_kfunc(meta->func_id)) { 14143 14065 mark_reg_known_zero(env, regs, BPF_REG_0); 14144 14066 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 14145 14067 regs[BPF_REG_0].btf = meta->arg_btf; ··· 14305 14227 if (err < 0) 14306 14228 return err; 14307 14229 14308 - if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 14230 + if (is_bpf_rbtree_add_kfunc(meta.func_id)) { 14309 14231 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 14310 14232 set_rbtree_add_callback_state); 14311 14233 if (err) { ··· 14409 14331 return err; 14410 14332 } 14411 14333 14412 - if (meta.func_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 14413 - meta.func_id == special_kfunc_list[KF_bpf_list_push_back_impl] || 14414 - meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 14334 + if (is_bpf_list_push_kfunc(meta.func_id) || is_bpf_rbtree_add_kfunc(meta.func_id)) { 14415 14335 release_ref_obj_id = regs[BPF_REG_2].ref_obj_id; 14416 14336 insn_aux->insert_off = regs[BPF_REG_2].var_off.value; 14417 14337 insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id); ··· 14457 14381 t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL); 14458 14382 14459 14383 if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) { 14460 - /* Only exception is bpf_obj_new_impl */ 14461 14384 if (meta.btf != btf_vmlinux || 14462 - (meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl] && 14463 - meta.func_id != special_kfunc_list[KF_bpf_percpu_obj_new_impl] && 14464 - meta.func_id != special_kfunc_list[KF_bpf_refcount_acquire_impl])) { 14385 + (!is_bpf_obj_new_kfunc(meta.func_id) && 14386 + !is_bpf_percpu_obj_new_kfunc(meta.func_id) && 14387 + !is_bpf_refcount_acquire_kfunc(meta.func_id))) { 14465 14388 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); 14466 14389 return -EINVAL; 14467 14390 } ··· 14571 14496 regs[BPF_REG_0].id = ++env->id_gen; 14572 14497 } else if (btf_type_is_void(t)) { 14573 14498 if (meta.btf == btf_vmlinux) { 14574 - if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl] || 14575 - meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) { 14499 + if (is_bpf_obj_drop_kfunc(meta.func_id) || 14500 + is_bpf_percpu_obj_drop_kfunc(meta.func_id)) { 14576 14501 insn_aux->kptr_struct_meta = 14577 14502 btf_find_struct_meta(meta.arg_btf, 14578 14503 meta.arg_btf_id); ··· 23399 23324 if (!bpf_jit_supports_far_kfunc_call()) 23400 23325 insn->imm = BPF_CALL_IMM(desc->addr); 23401 23326 23402 - if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl] || 23403 - desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { 23327 + if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) { 23404 23328 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 23405 23329 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 23406 23330 u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size; 23407 23331 23408 - if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl] && kptr_struct_meta) { 23332 + if (is_bpf_percpu_obj_new_kfunc(desc->func_id) && kptr_struct_meta) { 23409 23333 verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d", 23410 23334 insn_idx); 23411 23335 return -EFAULT; ··· 23415 23341 insn_buf[2] = addr[1]; 23416 23342 insn_buf[3] = *insn; 23417 23343 *cnt = 4; 23418 - } else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl] || 23419 - desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] || 23420 - desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) { 23344 + } else if (is_bpf_obj_drop_kfunc(desc->func_id) || 23345 + is_bpf_percpu_obj_drop_kfunc(desc->func_id) || 23346 + is_bpf_refcount_acquire_kfunc(desc->func_id)) { 23421 23347 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 23422 23348 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 23423 23349 23424 - if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] && kptr_struct_meta) { 23350 + if (is_bpf_percpu_obj_drop_kfunc(desc->func_id) && kptr_struct_meta) { 23425 23351 verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d", 23426 23352 insn_idx); 23427 23353 return -EFAULT; 23428 23354 } 23429 23355 23430 - if (desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] && 23431 - !kptr_struct_meta) { 23356 + if (is_bpf_refcount_acquire_kfunc(desc->func_id) && !kptr_struct_meta) { 23432 23357 verifier_bug(env, "kptr_struct_meta expected at insn_idx %d", 23433 23358 insn_idx); 23434 23359 return -EFAULT; ··· 23437 23364 insn_buf[1] = addr[1]; 23438 23365 insn_buf[2] = *insn; 23439 23366 *cnt = 3; 23440 - } else if (desc->func_id == special_kfunc_list[KF_bpf_list_push_back_impl] || 23441 - desc->func_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 23442 - desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 23367 + } else if (is_bpf_list_push_kfunc(desc->func_id) || 23368 + is_bpf_rbtree_add_kfunc(desc->func_id)) { 23443 23369 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 23444 23370 int struct_meta_reg = BPF_REG_3; 23445 23371 int node_offset_reg = BPF_REG_4; 23446 23372 23447 23373 /* rbtree_add has extra 'less' arg, so args-to-fixup are in diff regs */ 23448 - if (desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 23374 + if (is_bpf_rbtree_add_kfunc(desc->func_id)) { 23449 23375 struct_meta_reg = BPF_REG_4; 23450 23376 node_offset_reg = BPF_REG_5; 23451 23377 }
+1
tools/bpf/resolve_btfids/main.c
··· 1065 1065 { 1066 1066 static const char *const kf_implicit_arg_types[] = { 1067 1067 "bpf_prog_aux", 1068 + "btf_struct_meta", 1068 1069 }; 1069 1070 const struct btf_type *t; 1070 1071 const char *name;
+2 -2
tools/testing/selftests/bpf/progs/percpu_alloc_fail.c
··· 110 110 } 111 111 112 112 SEC("?fentry.s/bpf_fentry_test1") 113 - __failure __msg("arg#0 expected for bpf_percpu_obj_drop_impl()") 113 + __failure __msg("arg#0 expected for bpf_percpu_obj_drop()") 114 114 int BPF_PROG(test_array_map_4) 115 115 { 116 116 struct val_t __percpu_kptr *p; ··· 124 124 } 125 125 126 126 SEC("?fentry.s/bpf_fentry_test1") 127 - __failure __msg("arg#0 expected for bpf_obj_drop_impl()") 127 + __failure __msg("arg#0 expected for bpf_obj_drop()") 128 128 int BPF_PROG(test_array_map_5) 129 129 { 130 130 struct val_t *p;