tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Pull bpf fixes from Daniel Borkmann:

- Fix a bug in the BPF verifier to track changes to packet data
property for global functions (Eduard Zingerman)

- Fix a theoretical BPF prog_array use-after-free in RCU handling of
__uprobe_perf_func (Jann Horn)

- Fix BPF tracing to have an explicit list of tracepoints and their
arguments which need to be annotated as PTR_MAYBE_NULL (Kumar
Kartikeya Dwivedi)

- Fix a logic bug in the bpf_remove_insns code where a potential error
would have been wrongly propagated (Anton Protopopov)

- Avoid deadlock scenarios caused by nested kprobe and fentry BPF
programs (Priya Bala Govindasamy)

- Fix a bug in BPF verifier which was missing a size check for
BTF-based context access (Kumar Kartikeya Dwivedi)

- Fix a crash found by syzbot through an invalid BPF prog_array access
in perf_event_detach_bpf_prog (Jiri Olsa)

- Fix several BPF sockmap bugs including a race causing a refcount
imbalance upon element replace (Michal Luczaj)

- Fix a use-after-free from mismatching BPF program/attachment RCU
flavors (Jann Horn)

* tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (23 commits)
bpf: Avoid deadlock caused by nested kprobe and fentry bpf programs
selftests/bpf: Add tests for raw_tp NULL args
bpf: Augment raw_tp arguments with PTR_MAYBE_NULL
bpf: Revert "bpf: Mark raw_tp arguments with PTR_MAYBE_NULL"
selftests/bpf: Add test for narrow ctx load for pointer args
bpf: Check size for BTF-based ctx access of pointer members
selftests/bpf: extend changes_pkt_data with cases w/o subprograms
bpf: fix null dereference when computing changes_pkt_data of prog w/o subprogs
bpf: Fix theoretical prog_array UAF in __uprobe_perf_func()
bpf: fix potential error return
selftests/bpf: validate that tail call invalidates packet pointers
bpf: consider that tail calls invalidate packet pointers
selftests/bpf: freplace tests for tracking of changes_packet_data
bpf: check changes_pkt_data property for extension programs
selftests/bpf: test for changing packet data from global functions
bpf: track changes_pkt_data property for global functions
bpf: refactor bpf_helper_changes_pkt_data to use helper number
bpf: add find_containing_subprog() utility function
bpf,perf: Fix invalid prog_array access in perf_event_detach_bpf_prog
bpf: Fix UAF via mismatching bpf_prog/attachment RCU flavors
...

Linus Torvalds 35f301dd a0e3919a

+595 -163
+6 -14
include/linux/bpf.h
··· 1527 1527 bool is_extended; /* true if extended by freplace program */ 1528 1528 bool jits_use_priv_stack; 1529 1529 bool priv_stack_requested; 1530 + bool changes_pkt_data; 1530 1531 u64 prog_array_member_cnt; /* counts how many times as member of prog_array */ 1531 1532 struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */ 1532 1533 struct bpf_arena *arena; ··· 2194 2193 * rcu-protected dynamically sized maps. 2195 2194 */ 2196 2195 static __always_inline u32 2197 - bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu, 2196 + bpf_prog_run_array_uprobe(const struct bpf_prog_array *array, 2198 2197 const void *ctx, bpf_prog_run_fn run_prog) 2199 2198 { 2200 2199 const struct bpf_prog_array_item *item; 2201 2200 const struct bpf_prog *prog; 2202 - const struct bpf_prog_array *array; 2203 2201 struct bpf_run_ctx *old_run_ctx; 2204 2202 struct bpf_trace_run_ctx run_ctx; 2205 2203 u32 ret = 1; 2206 2204 2207 2205 might_fault(); 2206 + RCU_LOCKDEP_WARN(!rcu_read_lock_trace_held(), "no rcu lock held"); 2208 2207 2209 - rcu_read_lock_trace(); 2208 + if (unlikely(!array)) 2209 + return ret; 2210 + 2210 2211 migrate_disable(); 2211 2212 2212 2213 run_ctx.is_uprobe = true; 2213 2214 2214 - array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held()); 2215 - if (unlikely(!array)) 2216 - goto out; 2217 2215 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 2218 2216 item = &array->items[0]; 2219 2217 while ((prog = READ_ONCE(item->prog))) { ··· 2227 2227 rcu_read_unlock(); 2228 2228 } 2229 2229 bpf_reset_run_ctx(old_run_ctx); 2230 - out: 2231 2230 migrate_enable(); 2232 - rcu_read_unlock_trace(); 2233 2231 return ret; 2234 2232 } 2235 2233 ··· 3512 3514 static inline bool bpf_is_subprog(const struct bpf_prog *prog) 3513 3515 { 3514 3516 return prog->aux->func_idx != 0; 3515 - } 3516 - 3517 - static inline bool bpf_prog_is_raw_tp(const struct bpf_prog *prog) 3518 - { 3519 - return prog->type == BPF_PROG_TYPE_TRACING && 3520 - prog->expected_attach_type == BPF_TRACE_RAW_TP; 3521 3517 } 3522 3518 3523 3519 #endif /* _LINUX_BPF_H */
+1
include/linux/bpf_verifier.h
··· 659 659 bool args_cached: 1; 660 660 /* true if bpf_fastcall stack region is used by functions that can't be inlined */ 661 661 bool keep_fastcall_stack: 1; 662 + bool changes_pkt_data: 1; 662 663 663 664 enum priv_stack_mode priv_stack_mode; 664 665 u8 arg_cnt;
+1 -1
include/linux/filter.h
··· 1122 1122 bool bpf_jit_supports_private_stack(void); 1123 1123 u64 bpf_arch_uaddress_limit(void); 1124 1124 void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); 1125 - bool bpf_helper_changes_pkt_data(void *func); 1125 + bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id); 1126 1126 1127 1127 static inline bool bpf_dump_raw_ok(const struct cred *cred) 1128 1128 {
+6
kernel/bpf/Makefile
··· 53 53 obj-$(CONFIG_BPF_SYSCALL) += btf_iter.o 54 54 obj-$(CONFIG_BPF_SYSCALL) += btf_relocate.o 55 55 obj-$(CONFIG_BPF_SYSCALL) += kmem_cache_iter.o 56 + 57 + CFLAGS_REMOVE_percpu_freelist.o = $(CC_FLAGS_FTRACE) 58 + CFLAGS_REMOVE_bpf_lru_list.o = $(CC_FLAGS_FTRACE) 59 + CFLAGS_REMOVE_queue_stack_maps.o = $(CC_FLAGS_FTRACE) 60 + CFLAGS_REMOVE_lpm_trie.o = $(CC_FLAGS_FTRACE) 61 + CFLAGS_REMOVE_ringbuf.o = $(CC_FLAGS_FTRACE)
+145 -4
kernel/bpf/btf.c
··· 6439 6439 return off; 6440 6440 } 6441 6441 6442 + struct bpf_raw_tp_null_args { 6443 + const char *func; 6444 + u64 mask; 6445 + }; 6446 + 6447 + static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { 6448 + /* sched */ 6449 + { "sched_pi_setprio", 0x10 }, 6450 + /* ... from sched_numa_pair_template event class */ 6451 + { "sched_stick_numa", 0x100 }, 6452 + { "sched_swap_numa", 0x100 }, 6453 + /* afs */ 6454 + { "afs_make_fs_call", 0x10 }, 6455 + { "afs_make_fs_calli", 0x10 }, 6456 + { "afs_make_fs_call1", 0x10 }, 6457 + { "afs_make_fs_call2", 0x10 }, 6458 + { "afs_protocol_error", 0x1 }, 6459 + { "afs_flock_ev", 0x10 }, 6460 + /* cachefiles */ 6461 + { "cachefiles_lookup", 0x1 | 0x200 }, 6462 + { "cachefiles_unlink", 0x1 }, 6463 + { "cachefiles_rename", 0x1 }, 6464 + { "cachefiles_prep_read", 0x1 }, 6465 + { "cachefiles_mark_active", 0x1 }, 6466 + { "cachefiles_mark_failed", 0x1 }, 6467 + { "cachefiles_mark_inactive", 0x1 }, 6468 + { "cachefiles_vfs_error", 0x1 }, 6469 + { "cachefiles_io_error", 0x1 }, 6470 + { "cachefiles_ondemand_open", 0x1 }, 6471 + { "cachefiles_ondemand_copen", 0x1 }, 6472 + { "cachefiles_ondemand_close", 0x1 }, 6473 + { "cachefiles_ondemand_read", 0x1 }, 6474 + { "cachefiles_ondemand_cread", 0x1 }, 6475 + { "cachefiles_ondemand_fd_write", 0x1 }, 6476 + { "cachefiles_ondemand_fd_release", 0x1 }, 6477 + /* ext4, from ext4__mballoc event class */ 6478 + { "ext4_mballoc_discard", 0x10 }, 6479 + { "ext4_mballoc_free", 0x10 }, 6480 + /* fib */ 6481 + { "fib_table_lookup", 0x100 }, 6482 + /* filelock */ 6483 + /* ... from filelock_lock event class */ 6484 + { "posix_lock_inode", 0x10 }, 6485 + { "fcntl_setlk", 0x10 }, 6486 + { "locks_remove_posix", 0x10 }, 6487 + { "flock_lock_inode", 0x10 }, 6488 + /* ... from filelock_lease event class */ 6489 + { "break_lease_noblock", 0x10 }, 6490 + { "break_lease_block", 0x10 }, 6491 + { "break_lease_unblock", 0x10 }, 6492 + { "generic_delete_lease", 0x10 }, 6493 + { "time_out_leases", 0x10 }, 6494 + /* host1x */ 6495 + { "host1x_cdma_push_gather", 0x10000 }, 6496 + /* huge_memory */ 6497 + { "mm_khugepaged_scan_pmd", 0x10 }, 6498 + { "mm_collapse_huge_page_isolate", 0x1 }, 6499 + { "mm_khugepaged_scan_file", 0x10 }, 6500 + { "mm_khugepaged_collapse_file", 0x10 }, 6501 + /* kmem */ 6502 + { "mm_page_alloc", 0x1 }, 6503 + { "mm_page_pcpu_drain", 0x1 }, 6504 + /* .. from mm_page event class */ 6505 + { "mm_page_alloc_zone_locked", 0x1 }, 6506 + /* netfs */ 6507 + { "netfs_failure", 0x10 }, 6508 + /* power */ 6509 + { "device_pm_callback_start", 0x10 }, 6510 + /* qdisc */ 6511 + { "qdisc_dequeue", 0x1000 }, 6512 + /* rxrpc */ 6513 + { "rxrpc_recvdata", 0x1 }, 6514 + { "rxrpc_resend", 0x10 }, 6515 + /* sunrpc */ 6516 + { "xs_stream_read_data", 0x1 }, 6517 + /* ... from xprt_cong_event event class */ 6518 + { "xprt_reserve_cong", 0x10 }, 6519 + { "xprt_release_cong", 0x10 }, 6520 + { "xprt_get_cong", 0x10 }, 6521 + { "xprt_put_cong", 0x10 }, 6522 + /* tcp */ 6523 + { "tcp_send_reset", 0x11 }, 6524 + /* tegra_apb_dma */ 6525 + { "tegra_dma_tx_status", 0x100 }, 6526 + /* timer_migration */ 6527 + { "tmigr_update_events", 0x1 }, 6528 + /* writeback, from writeback_folio_template event class */ 6529 + { "writeback_dirty_folio", 0x10 }, 6530 + { "folio_wait_writeback", 0x10 }, 6531 + /* rdma */ 6532 + { "mr_integ_alloc", 0x2000 }, 6533 + /* bpf_testmod */ 6534 + { "bpf_testmod_test_read", 0x0 }, 6535 + }; 6536 + 6442 6537 bool btf_ctx_access(int off, int size, enum bpf_access_type type, 6443 6538 const struct bpf_prog *prog, 6444 6539 struct bpf_insn_access_aux *info) ··· 6544 6449 const char *tname = prog->aux->attach_func_name; 6545 6450 struct bpf_verifier_log *log = info->log; 6546 6451 const struct btf_param *args; 6452 + bool ptr_err_raw_tp = false; 6547 6453 const char *tag_value; 6548 6454 u32 nr_args, arg; 6549 6455 int i, ret; ··· 6639 6543 return false; 6640 6544 } 6641 6545 6546 + if (size != sizeof(u64)) { 6547 + bpf_log(log, "func '%s' size %d must be 8\n", 6548 + tname, size); 6549 + return false; 6550 + } 6551 + 6642 6552 /* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */ 6643 6553 for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { 6644 6554 const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; ··· 6690 6588 if (prog_args_trusted(prog)) 6691 6589 info->reg_type |= PTR_TRUSTED; 6692 6590 6693 - /* Raw tracepoint arguments always get marked as maybe NULL */ 6694 - if (bpf_prog_is_raw_tp(prog)) 6591 + if (btf_param_match_suffix(btf, &args[arg], "__nullable")) 6695 6592 info->reg_type |= PTR_MAYBE_NULL; 6696 - else if (btf_param_match_suffix(btf, &args[arg], "__nullable")) 6697 - info->reg_type |= PTR_MAYBE_NULL; 6593 + 6594 + if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 6595 + struct btf *btf = prog->aux->attach_btf; 6596 + const struct btf_type *t; 6597 + const char *tname; 6598 + 6599 + /* BTF lookups cannot fail, return false on error */ 6600 + t = btf_type_by_id(btf, prog->aux->attach_btf_id); 6601 + if (!t) 6602 + return false; 6603 + tname = btf_name_by_offset(btf, t->name_off); 6604 + if (!tname) 6605 + return false; 6606 + /* Checked by bpf_check_attach_target */ 6607 + tname += sizeof("btf_trace_") - 1; 6608 + for (i = 0; i < ARRAY_SIZE(raw_tp_null_args); i++) { 6609 + /* Is this a func with potential NULL args? */ 6610 + if (strcmp(tname, raw_tp_null_args[i].func)) 6611 + continue; 6612 + if (raw_tp_null_args[i].mask & (0x1 << (arg * 4))) 6613 + info->reg_type |= PTR_MAYBE_NULL; 6614 + /* Is the current arg IS_ERR? */ 6615 + if (raw_tp_null_args[i].mask & (0x2 << (arg * 4))) 6616 + ptr_err_raw_tp = true; 6617 + break; 6618 + } 6619 + /* If we don't know NULL-ness specification and the tracepoint 6620 + * is coming from a loadable module, be conservative and mark 6621 + * argument as PTR_MAYBE_NULL. 6622 + */ 6623 + if (i == ARRAY_SIZE(raw_tp_null_args) && btf_is_module(btf)) 6624 + info->reg_type |= PTR_MAYBE_NULL; 6625 + } 6698 6626 6699 6627 if (tgt_prog) { 6700 6628 enum bpf_prog_type tgt_type; ··· 6770 6638 bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n", 6771 6639 tname, arg, info->btf_id, btf_type_str(t), 6772 6640 __btf_name_by_offset(btf, t->name_off)); 6641 + 6642 + /* Perform all checks on the validity of type for this argument, but if 6643 + * we know it can be IS_ERR at runtime, scrub pointer type and mark as 6644 + * scalar. 6645 + */ 6646 + if (ptr_err_raw_tp) { 6647 + bpf_log(log, "marking pointer arg%d as scalar as it may encode error", arg); 6648 + info->reg_type = SCALAR_VALUE; 6649 + } 6773 6650 return true; 6774 6651 } 6775 6652 EXPORT_SYMBOL_GPL(btf_ctx_access);
+6 -2
kernel/bpf/core.c
··· 539 539 540 540 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt) 541 541 { 542 + int err; 543 + 542 544 /* Branch offsets can't overflow when program is shrinking, no need 543 545 * to call bpf_adj_branches(..., true) here 544 546 */ ··· 548 546 sizeof(struct bpf_insn) * (prog->len - off - cnt)); 549 547 prog->len -= cnt; 550 548 551 - return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false)); 549 + err = bpf_adj_branches(prog, off, off + cnt, off, false); 550 + WARN_ON_ONCE(err); 551 + return err; 552 552 } 553 553 554 554 static void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp) ··· 2940 2936 { 2941 2937 } 2942 2938 2943 - bool __weak bpf_helper_changes_pkt_data(void *func) 2939 + bool __weak bpf_helper_changes_pkt_data(enum bpf_func_id func_id) 2944 2940 { 2945 2941 return false; 2946 2942 }
+78 -82
kernel/bpf/verifier.c
··· 420 420 return rec; 421 421 } 422 422 423 - static bool mask_raw_tp_reg_cond(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) { 424 - return reg->type == (PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL) && 425 - bpf_prog_is_raw_tp(env->prog) && !reg->ref_obj_id; 426 - } 427 - 428 - static bool mask_raw_tp_reg(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) 429 - { 430 - if (!mask_raw_tp_reg_cond(env, reg)) 431 - return false; 432 - reg->type &= ~PTR_MAYBE_NULL; 433 - return true; 434 - } 435 - 436 - static void unmask_raw_tp_reg(struct bpf_reg_state *reg, bool result) 437 - { 438 - if (result) 439 - reg->type |= PTR_MAYBE_NULL; 440 - } 441 - 442 423 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog) 443 424 { 444 425 struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux; ··· 2578 2597 ((struct bpf_subprog_info *)b)->start; 2579 2598 } 2580 2599 2600 + /* Find subprogram that contains instruction at 'off' */ 2601 + static struct bpf_subprog_info *find_containing_subprog(struct bpf_verifier_env *env, int off) 2602 + { 2603 + struct bpf_subprog_info *vals = env->subprog_info; 2604 + int l, r, m; 2605 + 2606 + if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0) 2607 + return NULL; 2608 + 2609 + l = 0; 2610 + r = env->subprog_cnt - 1; 2611 + while (l < r) { 2612 + m = l + (r - l + 1) / 2; 2613 + if (vals[m].start <= off) 2614 + l = m; 2615 + else 2616 + r = m - 1; 2617 + } 2618 + return &vals[l]; 2619 + } 2620 + 2621 + /* Find subprogram that starts exactly at 'off' */ 2581 2622 static int find_subprog(struct bpf_verifier_env *env, int off) 2582 2623 { 2583 2624 struct bpf_subprog_info *p; 2584 2625 2585 - p = bsearch(&off, env->subprog_info, env->subprog_cnt, 2586 - sizeof(env->subprog_info[0]), cmp_subprogs); 2587 - if (!p) 2626 + p = find_containing_subprog(env, off); 2627 + if (!p || p->start != off) 2588 2628 return -ENOENT; 2589 2629 return p - env->subprog_info; 2590 - 2591 2630 } 2592 2631 2593 2632 static int add_subprog(struct bpf_verifier_env *env, int off) ··· 6782 6781 const char *field_name = NULL; 6783 6782 enum bpf_type_flag flag = 0; 6784 6783 u32 btf_id = 0; 6785 - bool mask; 6786 6784 int ret; 6787 6785 6788 6786 if (!env->allow_ptr_leaks) { ··· 6853 6853 6854 6854 if (ret < 0) 6855 6855 return ret; 6856 - /* For raw_tp progs, we allow dereference of PTR_MAYBE_NULL 6857 - * trusted PTR_TO_BTF_ID, these are the ones that are possibly 6858 - * arguments to the raw_tp. Since internal checks in for trusted 6859 - * reg in check_ptr_to_btf_access would consider PTR_MAYBE_NULL 6860 - * modifier as problematic, mask it out temporarily for the 6861 - * check. Don't apply this to pointers with ref_obj_id > 0, as 6862 - * those won't be raw_tp args. 6863 - * 6864 - * We may end up applying this relaxation to other trusted 6865 - * PTR_TO_BTF_ID with maybe null flag, since we cannot 6866 - * distinguish PTR_MAYBE_NULL tagged for arguments vs normal 6867 - * tagging, but that should expand allowed behavior, and not 6868 - * cause regression for existing behavior. 6869 - */ 6870 - mask = mask_raw_tp_reg(env, reg); 6856 + 6871 6857 if (ret != PTR_TO_BTF_ID) { 6872 6858 /* just mark; */ 6873 6859 ··· 6914 6928 clear_trusted_flags(&flag); 6915 6929 } 6916 6930 6917 - if (atype == BPF_READ && value_regno >= 0) { 6931 + if (atype == BPF_READ && value_regno >= 0) 6918 6932 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); 6919 - /* We've assigned a new type to regno, so don't undo masking. */ 6920 - if (regno == value_regno) 6921 - mask = false; 6922 - } 6923 - unmask_raw_tp_reg(reg, mask); 6924 6933 6925 6934 return 0; 6926 6935 } ··· 7290 7309 if (!err && t == BPF_READ && value_regno >= 0) 7291 7310 mark_reg_unknown(env, regs, value_regno); 7292 7311 } else if (base_type(reg->type) == PTR_TO_BTF_ID && 7293 - (mask_raw_tp_reg_cond(env, reg) || !type_may_be_null(reg->type))) { 7312 + !type_may_be_null(reg->type)) { 7294 7313 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, 7295 7314 value_regno); 7296 7315 } else if (reg->type == CONST_PTR_TO_MAP) { ··· 8993 9012 enum bpf_reg_type type = reg->type; 8994 9013 u32 *arg_btf_id = NULL; 8995 9014 int err = 0; 8996 - bool mask; 8997 9015 8998 9016 if (arg_type == ARG_DONTCARE) 8999 9017 return 0; ··· 9033 9053 base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) 9034 9054 arg_btf_id = fn->arg_btf_id[arg]; 9035 9055 9036 - mask = mask_raw_tp_reg(env, reg); 9037 9056 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); 9057 + if (err) 9058 + return err; 9038 9059 9039 - err = err ?: check_func_arg_reg_off(env, reg, regno, arg_type); 9040 - unmask_raw_tp_reg(reg, mask); 9060 + err = check_func_arg_reg_off(env, reg, regno, arg_type); 9041 9061 if (err) 9042 9062 return err; 9043 9063 ··· 9832 9852 return ret; 9833 9853 } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { 9834 9854 struct bpf_call_arg_meta meta; 9835 - bool mask; 9836 9855 int err; 9837 9856 9838 9857 if (register_is_null(reg) && type_may_be_null(arg->arg_type)) 9839 9858 continue; 9840 9859 9841 9860 memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ 9842 - mask = mask_raw_tp_reg(env, reg); 9843 9861 err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta); 9844 9862 err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type); 9845 - unmask_raw_tp_reg(reg, mask); 9846 9863 if (err) 9847 9864 return err; 9848 9865 } else { ··· 9999 10022 10000 10023 verbose(env, "Func#%d ('%s') is global and assumed valid.\n", 10001 10024 subprog, sub_name); 10025 + if (env->subprog_info[subprog].changes_pkt_data) 10026 + clear_all_pkt_pointers(env); 10002 10027 /* mark global subprog for verifying after main prog */ 10003 10028 subprog_aux(env, subprog)->called = true; 10004 10029 clear_caller_saved_regs(env, caller->regs); ··· 10687 10708 } 10688 10709 10689 10710 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 10690 - changes_data = bpf_helper_changes_pkt_data(fn->func); 10711 + changes_data = bpf_helper_changes_pkt_data(func_id); 10691 10712 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 10692 10713 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", 10693 10714 func_id_name(func_id), func_id); ··· 12162 12183 enum bpf_arg_type arg_type = ARG_DONTCARE; 12163 12184 u32 regno = i + 1, ref_id, type_size; 12164 12185 bool is_ret_buf_sz = false; 12165 - bool mask = false; 12166 12186 int kf_arg_type; 12167 12187 12168 12188 t = btf_type_skip_modifiers(btf, args[i].type, NULL); ··· 12220 12242 return -EINVAL; 12221 12243 } 12222 12244 12223 - mask = mask_raw_tp_reg(env, reg); 12224 12245 if ((is_kfunc_trusted_args(meta) || is_kfunc_rcu(meta)) && 12225 12246 (register_is_null(reg) || type_may_be_null(reg->type)) && 12226 12247 !is_kfunc_arg_nullable(meta->btf, &args[i])) { 12227 12248 verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i); 12228 - unmask_raw_tp_reg(reg, mask); 12229 12249 return -EACCES; 12230 12250 } 12231 - unmask_raw_tp_reg(reg, mask); 12232 12251 12233 12252 if (reg->ref_obj_id) { 12234 12253 if (is_kfunc_release(meta) && meta->ref_obj_id) { ··· 12283 12308 if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta)) 12284 12309 break; 12285 12310 12286 - /* Allow passing maybe NULL raw_tp arguments to 12287 - * kfuncs for compatibility. Don't apply this to 12288 - * arguments with ref_obj_id > 0. 12289 - */ 12290 - mask = mask_raw_tp_reg(env, reg); 12291 12311 if (!is_trusted_reg(reg)) { 12292 12312 if (!is_kfunc_rcu(meta)) { 12293 12313 verbose(env, "R%d must be referenced or trusted\n", regno); 12294 - unmask_raw_tp_reg(reg, mask); 12295 12314 return -EINVAL; 12296 12315 } 12297 12316 if (!is_rcu_reg(reg)) { 12298 12317 verbose(env, "R%d must be a rcu pointer\n", regno); 12299 - unmask_raw_tp_reg(reg, mask); 12300 12318 return -EINVAL; 12301 12319 } 12302 12320 } 12303 - unmask_raw_tp_reg(reg, mask); 12304 12321 fallthrough; 12305 12322 case KF_ARG_PTR_TO_CTX: 12306 12323 case KF_ARG_PTR_TO_DYNPTR: ··· 12315 12348 12316 12349 if (is_kfunc_release(meta) && reg->ref_obj_id) 12317 12350 arg_type |= OBJ_RELEASE; 12318 - mask = mask_raw_tp_reg(env, reg); 12319 12351 ret = check_func_arg_reg_off(env, reg, regno, arg_type); 12320 - unmask_raw_tp_reg(reg, mask); 12321 12352 if (ret < 0) 12322 12353 return ret; 12323 12354 ··· 12492 12527 ref_tname = btf_name_by_offset(btf, ref_t->name_off); 12493 12528 fallthrough; 12494 12529 case KF_ARG_PTR_TO_BTF_ID: 12495 - mask = mask_raw_tp_reg(env, reg); 12496 12530 /* Only base_type is checked, further checks are done here */ 12497 12531 if ((base_type(reg->type) != PTR_TO_BTF_ID || 12498 12532 (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && ··· 12500 12536 verbose(env, "expected %s or socket\n", 12501 12537 reg_type_str(env, base_type(reg->type) | 12502 12538 (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); 12503 - unmask_raw_tp_reg(reg, mask); 12504 12539 return -EINVAL; 12505 12540 } 12506 12541 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i); 12507 - unmask_raw_tp_reg(reg, mask); 12508 12542 if (ret < 0) 12509 12543 return ret; 12510 12544 break; ··· 13475 13513 */ 13476 13514 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, 13477 13515 struct bpf_insn *insn, 13478 - struct bpf_reg_state *ptr_reg, 13516 + const struct bpf_reg_state *ptr_reg, 13479 13517 const struct bpf_reg_state *off_reg) 13480 13518 { 13481 13519 struct bpf_verifier_state *vstate = env->cur_state; ··· 13489 13527 struct bpf_sanitize_info info = {}; 13490 13528 u8 opcode = BPF_OP(insn->code); 13491 13529 u32 dst = insn->dst_reg; 13492 - bool mask; 13493 13530 int ret; 13494 13531 13495 13532 dst_reg = &regs[dst]; ··· 13515 13554 return -EACCES; 13516 13555 } 13517 13556 13518 - mask = mask_raw_tp_reg(env, ptr_reg); 13519 13557 if (ptr_reg->type & PTR_MAYBE_NULL) { 13520 13558 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 13521 13559 dst, reg_type_str(env, ptr_reg->type)); 13522 - unmask_raw_tp_reg(ptr_reg, mask); 13523 13560 return -EACCES; 13524 13561 } 13525 - unmask_raw_tp_reg(ptr_reg, mask); 13526 13562 13527 13563 switch (base_type(ptr_reg->type)) { 13528 13564 case PTR_TO_CTX: ··· 16184 16226 return 0; 16185 16227 } 16186 16228 16229 + static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off) 16230 + { 16231 + struct bpf_subprog_info *subprog; 16232 + 16233 + subprog = find_containing_subprog(env, off); 16234 + subprog->changes_pkt_data = true; 16235 + } 16236 + 16237 + /* 't' is an index of a call-site. 16238 + * 'w' is a callee entry point. 16239 + * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED. 16240 + * Rely on DFS traversal order and absence of recursive calls to guarantee that 16241 + * callee's change_pkt_data marks would be correct at that moment. 16242 + */ 16243 + static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) 16244 + { 16245 + struct bpf_subprog_info *caller, *callee; 16246 + 16247 + caller = find_containing_subprog(env, t); 16248 + callee = find_containing_subprog(env, w); 16249 + caller->changes_pkt_data |= callee->changes_pkt_data; 16250 + } 16251 + 16187 16252 /* non-recursive DFS pseudo code 16188 16253 * 1 procedure DFS-iterative(G,v): 16189 16254 * 2 label v as discovered ··· 16340 16359 bool visit_callee) 16341 16360 { 16342 16361 int ret, insn_sz; 16362 + int w; 16343 16363 16344 16364 insn_sz = bpf_is_ldimm64(&insns[t]) ? 2 : 1; 16345 16365 ret = push_insn(t, t + insn_sz, FALLTHROUGH, env); ··· 16352 16370 mark_jmp_point(env, t + insn_sz); 16353 16371 16354 16372 if (visit_callee) { 16373 + w = t + insns[t].imm + 1; 16355 16374 mark_prune_point(env, t); 16356 - ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env); 16375 + merge_callee_effects(env, t, w); 16376 + ret = push_insn(t, w, BRANCH, env); 16357 16377 } 16358 16378 return ret; 16359 16379 } ··· 16672 16688 mark_prune_point(env, t); 16673 16689 mark_jmp_point(env, t); 16674 16690 } 16691 + if (bpf_helper_call(insn) && bpf_helper_changes_pkt_data(insn->imm)) 16692 + mark_subprog_changes_pkt_data(env, t); 16675 16693 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 16676 16694 struct bpf_kfunc_call_arg_meta meta; 16677 16695 ··· 16808 16822 } 16809 16823 } 16810 16824 ret = 0; /* cfg looks good */ 16825 + env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data; 16811 16826 16812 16827 err_free: 16813 16828 kvfree(insn_state); ··· 20062 20075 * for this case. 20063 20076 */ 20064 20077 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: 20065 - case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL: 20066 20078 if (type == BPF_READ) { 20067 20079 if (BPF_MODE(insn->code) == BPF_MEM) 20068 20080 insn->code = BPF_LDX | BPF_PROBE_MEM | ··· 20297 20311 func[i]->aux->num_exentries = num_exentries; 20298 20312 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 20299 20313 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; 20314 + func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; 20300 20315 if (!i) 20301 20316 func[i]->aux->exception_boundary = env->seen_exception; 20302 20317 func[i] = bpf_int_jit_compile(func[i]); ··· 22128 22141 } 22129 22142 if (tgt_prog) { 22130 22143 struct bpf_prog_aux *aux = tgt_prog->aux; 22144 + bool tgt_changes_pkt_data; 22131 22145 22132 22146 if (bpf_prog_is_dev_bound(prog->aux) && 22133 22147 !bpf_prog_dev_bound_match(prog, tgt_prog)) { ··· 22161 22173 if (!prog->jit_requested) { 22162 22174 bpf_log(log, 22163 22175 "Extension programs should be JITed\n"); 22176 + return -EINVAL; 22177 + } 22178 + tgt_changes_pkt_data = aux->func 22179 + ? aux->func[subprog]->aux->changes_pkt_data 22180 + : aux->changes_pkt_data; 22181 + if (prog->aux->changes_pkt_data && !tgt_changes_pkt_data) { 22182 + bpf_log(log, 22183 + "Extension program changes packet data, while original does not\n"); 22164 22184 return -EINVAL; 22165 22185 } 22166 22186 } ··· 22636 22640 if (ret < 0) 22637 22641 goto skip_full_check; 22638 22642 22639 - ret = check_attach_btf_id(env); 22640 - if (ret) 22641 - goto skip_full_check; 22642 - 22643 22643 ret = resolve_pseudo_ldimm64(env); 22644 22644 if (ret < 0) 22645 22645 goto skip_full_check; ··· 22648 22656 22649 22657 ret = check_cfg(env); 22650 22658 if (ret < 0) 22659 + goto skip_full_check; 22660 + 22661 + ret = check_attach_btf_id(env); 22662 + if (ret) 22651 22663 goto skip_full_check; 22652 22664 22653 22665 ret = mark_fastcall_patterns(env);
+11
kernel/trace/bpf_trace.c
··· 2250 2250 goto unlock; 2251 2251 2252 2252 old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); 2253 + if (!old_array) 2254 + goto put; 2255 + 2253 2256 ret = bpf_prog_array_copy(old_array, event->prog, NULL, 0, &new_array); 2254 2257 if (ret < 0) { 2255 2258 bpf_prog_array_delete_safe(old_array, event->prog); ··· 2260 2257 rcu_assign_pointer(event->tp_event->prog_array, new_array); 2261 2258 bpf_prog_array_free_sleepable(old_array); 2262 2259 } 2260 + 2261 + put: 2262 + /* 2263 + * It could be that the bpf_prog is not sleepable (and will be freed 2264 + * via normal RCU), but is called from a point that supports sleepable 2265 + * programs and uses tasks-trace-RCU. 2266 + */ 2267 + synchronize_rcu_tasks_trace(); 2263 2268 2264 2269 bpf_prog_put(event->prog); 2265 2270 event->prog = NULL;
+5 -1
kernel/trace/trace_uprobe.c
··· 1402 1402 1403 1403 #ifdef CONFIG_BPF_EVENTS 1404 1404 if (bpf_prog_array_valid(call)) { 1405 + const struct bpf_prog_array *array; 1405 1406 u32 ret; 1406 1407 1407 - ret = bpf_prog_run_array_uprobe(call->prog_array, regs, bpf_prog_run); 1408 + rcu_read_lock_trace(); 1409 + array = rcu_dereference_check(call->prog_array, rcu_read_lock_trace_held()); 1410 + ret = bpf_prog_run_array_uprobe(array, regs, bpf_prog_run); 1411 + rcu_read_unlock_trace(); 1408 1412 if (!ret) 1409 1413 return; 1410 1414 }
+29 -34
net/core/filter.c
··· 7899 7899 7900 7900 #endif /* CONFIG_INET */ 7901 7901 7902 - bool bpf_helper_changes_pkt_data(void *func) 7902 + bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id) 7903 7903 { 7904 - if (func == bpf_skb_vlan_push || 7905 - func == bpf_skb_vlan_pop || 7906 - func == bpf_skb_store_bytes || 7907 - func == bpf_skb_change_proto || 7908 - func == bpf_skb_change_head || 7909 - func == sk_skb_change_head || 7910 - func == bpf_skb_change_tail || 7911 - func == sk_skb_change_tail || 7912 - func == bpf_skb_adjust_room || 7913 - func == sk_skb_adjust_room || 7914 - func == bpf_skb_pull_data || 7915 - func == sk_skb_pull_data || 7916 - func == bpf_clone_redirect || 7917 - func == bpf_l3_csum_replace || 7918 - func == bpf_l4_csum_replace || 7919 - func == bpf_xdp_adjust_head || 7920 - func == bpf_xdp_adjust_meta || 7921 - func == bpf_msg_pull_data || 7922 - func == bpf_msg_push_data || 7923 - func == bpf_msg_pop_data || 7924 - func == bpf_xdp_adjust_tail || 7925 - #if IS_ENABLED(CONFIG_IPV6_SEG6_BPF) 7926 - func == bpf_lwt_seg6_store_bytes || 7927 - func == bpf_lwt_seg6_adjust_srh || 7928 - func == bpf_lwt_seg6_action || 7929 - #endif 7930 - #ifdef CONFIG_INET 7931 - func == bpf_sock_ops_store_hdr_opt || 7932 - #endif 7933 - func == bpf_lwt_in_push_encap || 7934 - func == bpf_lwt_xmit_push_encap) 7904 + switch (func_id) { 7905 + case BPF_FUNC_clone_redirect: 7906 + case BPF_FUNC_l3_csum_replace: 7907 + case BPF_FUNC_l4_csum_replace: 7908 + case BPF_FUNC_lwt_push_encap: 7909 + case BPF_FUNC_lwt_seg6_action: 7910 + case BPF_FUNC_lwt_seg6_adjust_srh: 7911 + case BPF_FUNC_lwt_seg6_store_bytes: 7912 + case BPF_FUNC_msg_pop_data: 7913 + case BPF_FUNC_msg_pull_data: 7914 + case BPF_FUNC_msg_push_data: 7915 + case BPF_FUNC_skb_adjust_room: 7916 + case BPF_FUNC_skb_change_head: 7917 + case BPF_FUNC_skb_change_proto: 7918 + case BPF_FUNC_skb_change_tail: 7919 + case BPF_FUNC_skb_pull_data: 7920 + case BPF_FUNC_skb_store_bytes: 7921 + case BPF_FUNC_skb_vlan_pop: 7922 + case BPF_FUNC_skb_vlan_push: 7923 + case BPF_FUNC_store_hdr_opt: 7924 + case BPF_FUNC_xdp_adjust_head: 7925 + case BPF_FUNC_xdp_adjust_meta: 7926 + case BPF_FUNC_xdp_adjust_tail: 7927 + /* tail-called program could call any of the above */ 7928 + case BPF_FUNC_tail_call: 7935 7929 return true; 7936 - 7937 - return false; 7930 + default: 7931 + return false; 7932 + } 7938 7933 } 7939 7934 7940 7935 const struct bpf_func_proto bpf_event_output_data_proto __weak;
+3 -3
net/core/sock_map.c
··· 159 159 verdict_stop = true; 160 160 list_del(&link->list); 161 161 sk_psock_free_link(link); 162 + break; 162 163 } 163 164 } 164 165 spin_unlock_bh(&psock->link_lock); ··· 412 411 static int __sock_map_delete(struct bpf_stab *stab, struct sock *sk_test, 413 412 struct sock **psk) 414 413 { 415 - struct sock *sk; 414 + struct sock *sk = NULL; 416 415 int err = 0; 417 416 418 417 spin_lock_bh(&stab->lock); 419 - sk = *psk; 420 - if (!sk_test || sk_test == sk) 418 + if (!sk_test || sk_test == *psk) 421 419 sk = xchg(psk, NULL); 422 420 423 421 if (likely(sk))
+107
tools/testing/selftests/bpf/prog_tests/changes_pkt_data.c
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + #include "bpf/libbpf.h" 3 + #include "changes_pkt_data_freplace.skel.h" 4 + #include "changes_pkt_data.skel.h" 5 + #include <test_progs.h> 6 + 7 + static void print_verifier_log(const char *log) 8 + { 9 + if (env.verbosity >= VERBOSE_VERY) 10 + fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log); 11 + } 12 + 13 + static void test_aux(const char *main_prog_name, 14 + const char *to_be_replaced, 15 + const char *replacement, 16 + bool expect_load) 17 + { 18 + struct changes_pkt_data_freplace *freplace = NULL; 19 + struct bpf_program *freplace_prog = NULL; 20 + struct bpf_program *main_prog = NULL; 21 + LIBBPF_OPTS(bpf_object_open_opts, opts); 22 + struct changes_pkt_data *main = NULL; 23 + char log[16*1024]; 24 + int err; 25 + 26 + opts.kernel_log_buf = log; 27 + opts.kernel_log_size = sizeof(log); 28 + if (env.verbosity >= VERBOSE_SUPER) 29 + opts.kernel_log_level = 1 | 2 | 4; 30 + main = changes_pkt_data__open_opts(&opts); 31 + if (!ASSERT_OK_PTR(main, "changes_pkt_data__open")) 32 + goto out; 33 + main_prog = bpf_object__find_program_by_name(main->obj, main_prog_name); 34 + if (!ASSERT_OK_PTR(main_prog, "main_prog")) 35 + goto out; 36 + bpf_program__set_autoload(main_prog, true); 37 + err = changes_pkt_data__load(main); 38 + print_verifier_log(log); 39 + if (!ASSERT_OK(err, "changes_pkt_data__load")) 40 + goto out; 41 + freplace = changes_pkt_data_freplace__open_opts(&opts); 42 + if (!ASSERT_OK_PTR(freplace, "changes_pkt_data_freplace__open")) 43 + goto out; 44 + freplace_prog = bpf_object__find_program_by_name(freplace->obj, replacement); 45 + if (!ASSERT_OK_PTR(freplace_prog, "freplace_prog")) 46 + goto out; 47 + bpf_program__set_autoload(freplace_prog, true); 48 + bpf_program__set_autoattach(freplace_prog, true); 49 + bpf_program__set_attach_target(freplace_prog, 50 + bpf_program__fd(main_prog), 51 + to_be_replaced); 52 + err = changes_pkt_data_freplace__load(freplace); 53 + print_verifier_log(log); 54 + if (expect_load) { 55 + ASSERT_OK(err, "changes_pkt_data_freplace__load"); 56 + } else { 57 + ASSERT_ERR(err, "changes_pkt_data_freplace__load"); 58 + ASSERT_HAS_SUBSTR(log, "Extension program changes packet data", "error log"); 59 + } 60 + 61 + out: 62 + changes_pkt_data_freplace__destroy(freplace); 63 + changes_pkt_data__destroy(main); 64 + } 65 + 66 + /* There are two global subprograms in both changes_pkt_data.skel.h: 67 + * - one changes packet data; 68 + * - another does not. 69 + * It is ok to freplace subprograms that change packet data with those 70 + * that either do or do not. It is only ok to freplace subprograms 71 + * that do not change packet data with those that do not as well. 72 + * The below tests check outcomes for each combination of such freplace. 73 + * Also test a case when main subprogram itself is replaced and is a single 74 + * subprogram in a program. 75 + */ 76 + void test_changes_pkt_data_freplace(void) 77 + { 78 + struct { 79 + const char *main; 80 + const char *to_be_replaced; 81 + bool changes; 82 + } mains[] = { 83 + { "main_with_subprogs", "changes_pkt_data", true }, 84 + { "main_with_subprogs", "does_not_change_pkt_data", false }, 85 + { "main_changes", "main_changes", true }, 86 + { "main_does_not_change", "main_does_not_change", false }, 87 + }; 88 + struct { 89 + const char *func; 90 + bool changes; 91 + } replacements[] = { 92 + { "changes_pkt_data", true }, 93 + { "does_not_change_pkt_data", false } 94 + }; 95 + char buf[64]; 96 + 97 + for (int i = 0; i < ARRAY_SIZE(mains); ++i) { 98 + for (int j = 0; j < ARRAY_SIZE(replacements); ++j) { 99 + snprintf(buf, sizeof(buf), "%s_with_%s", 100 + mains[i].to_be_replaced, replacements[j].func); 101 + if (!test__start_subtest(buf)) 102 + continue; 103 + test_aux(mains[i].main, mains[i].to_be_replaced, replacements[j].func, 104 + mains[i].changes || !replacements[j].changes); 105 + } 106 + } 107 + }
+3
tools/testing/selftests/bpf/prog_tests/raw_tp_null.c
··· 3 3 4 4 #include <test_progs.h> 5 5 #include "raw_tp_null.skel.h" 6 + #include "raw_tp_null_fail.skel.h" 6 7 7 8 void test_raw_tp_null(void) 8 9 { 9 10 struct raw_tp_null *skel; 11 + 12 + RUN_TESTS(raw_tp_null_fail); 10 13 11 14 skel = raw_tp_null__open_and_load(); 12 15 if (!ASSERT_OK_PTR(skel, "raw_tp_null__open_and_load"))
+5 -3
tools/testing/selftests/bpf/prog_tests/sockmap_basic.c
··· 934 934 935 935 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 936 936 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 937 - if (err) 937 + if (err) { 938 + close(tcp); 938 939 goto out; 940 + } 939 941 940 942 for (i = 0; i < 2; i++) { 941 943 err = bpf_map_update_elem(map, &zero, &stream[0], BPF_ANY); ··· 956 954 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 957 955 } 958 956 957 + close(tcp); 959 958 err = bpf_map_delete_elem(map, &zero); 960 - ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 959 + ASSERT_ERR(err, "bpf_map_delete_elem(entry)"); 961 960 962 961 close(stream[0]); 963 962 close(stream[1]); 964 963 out: 965 964 close(dgram); 966 - close(tcp); 967 965 close(udp); 968 966 test_sockmap_pass_prog__destroy(skel); 969 967 }
+39
tools/testing/selftests/bpf/progs/changes_pkt_data.c
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + #include <linux/bpf.h> 4 + #include <bpf/bpf_helpers.h> 5 + 6 + __noinline 7 + long changes_pkt_data(struct __sk_buff *sk) 8 + { 9 + return bpf_skb_pull_data(sk, 0); 10 + } 11 + 12 + __noinline __weak 13 + long does_not_change_pkt_data(struct __sk_buff *sk) 14 + { 15 + return 0; 16 + } 17 + 18 + SEC("?tc") 19 + int main_with_subprogs(struct __sk_buff *sk) 20 + { 21 + changes_pkt_data(sk); 22 + does_not_change_pkt_data(sk); 23 + return 0; 24 + } 25 + 26 + SEC("?tc") 27 + int main_changes(struct __sk_buff *sk) 28 + { 29 + bpf_skb_pull_data(sk, 0); 30 + return 0; 31 + } 32 + 33 + SEC("?tc") 34 + int main_does_not_change(struct __sk_buff *sk) 35 + { 36 + return 0; 37 + } 38 + 39 + char _license[] SEC("license") = "GPL";
+18
tools/testing/selftests/bpf/progs/changes_pkt_data_freplace.c
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + #include <linux/bpf.h> 4 + #include <bpf/bpf_helpers.h> 5 + 6 + SEC("?freplace") 7 + long changes_pkt_data(struct __sk_buff *sk) 8 + { 9 + return bpf_skb_pull_data(sk, 0); 10 + } 11 + 12 + SEC("?freplace") 13 + long does_not_change_pkt_data(struct __sk_buff *sk) 14 + { 15 + return 0; 16 + } 17 + 18 + char _license[] SEC("license") = "GPL";
+9 -10
tools/testing/selftests/bpf/progs/raw_tp_null.c
··· 3 3 4 4 #include <vmlinux.h> 5 5 #include <bpf/bpf_tracing.h> 6 + #include "bpf_misc.h" 6 7 7 8 char _license[] SEC("license") = "GPL"; 8 9 ··· 18 17 if (task->pid != tid) 19 18 return 0; 20 19 21 - i = i + skb->mark + 1; 22 - /* The compiler may move the NULL check before this deref, which causes 23 - * the load to fail as deref of scalar. Prevent that by using a barrier. 20 + /* If dead code elimination kicks in, the increment +=2 will be 21 + * removed. For raw_tp programs attaching to tracepoints in kernel 22 + * modules, we mark input arguments as PTR_MAYBE_NULL, so branch 23 + * prediction should never kick in. 24 24 */ 25 - barrier(); 26 - /* If dead code elimination kicks in, the increment below will 27 - * be removed. For raw_tp programs, we mark input arguments as 28 - * PTR_MAYBE_NULL, so branch prediction should never kick in. 29 - */ 30 - if (!skb) 31 - i += 2; 25 + asm volatile ("%[i] += 1; if %[ctx] != 0 goto +1; %[i] += 2;" 26 + : [i]"+r"(i) 27 + : [ctx]"r"(skb) 28 + : "memory"); 32 29 return 0; 33 30 }
+24
tools/testing/selftests/bpf/progs/raw_tp_null_fail.c
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ 3 + 4 + #include <vmlinux.h> 5 + #include <bpf/bpf_tracing.h> 6 + #include "bpf_misc.h" 7 + 8 + char _license[] SEC("license") = "GPL"; 9 + 10 + /* Ensure module parameter has PTR_MAYBE_NULL */ 11 + SEC("tp_btf/bpf_testmod_test_raw_tp_null") 12 + __failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'") 13 + int test_raw_tp_null_bpf_testmod_test_raw_tp_null_arg_1(void *ctx) { 14 + asm volatile("r1 = *(u64 *)(r1 +0); r1 = *(u64 *)(r1 +0);" ::: __clobber_all); 15 + return 0; 16 + } 17 + 18 + /* Check NULL marking */ 19 + SEC("tp_btf/sched_pi_setprio") 20 + __failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'") 21 + int test_raw_tp_null_sched_pi_setprio_arg_2(void *ctx) { 22 + asm volatile("r1 = *(u64 *)(r1 +8); r1 = *(u64 *)(r1 +0);" ::: __clobber_all); 23 + return 0; 24 + }
+2
tools/testing/selftests/bpf/progs/tc_bpf2bpf.c
··· 11 11 12 12 __sink(skb); 13 13 __sink(ret); 14 + /* let verifier know that 'subprog_tc' can change pointers to skb->data */ 15 + bpf_skb_change_proto(skb, 0, 0); 14 16 return ret; 15 17 } 16 18
+1 -5
tools/testing/selftests/bpf/progs/test_tp_btf_nullable.c
··· 7 7 #include "bpf_misc.h" 8 8 9 9 SEC("tp_btf/bpf_testmod_test_nullable_bare") 10 - /* This used to be a failure test, but raw_tp nullable arguments can now 11 - * directly be dereferenced, whether they have nullable annotation or not, 12 - * and don't need to be explicitly checked. 13 - */ 14 - __success 10 + __failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'") 15 11 int BPF_PROG(handle_tp_btf_nullable_bare1, struct bpf_testmod_test_read_ctx *nullable_ctx) 16 12 { 17 13 return nullable_ctx->len;
+38 -2
tools/testing/selftests/bpf/progs/verifier_btf_ctx_access.c
··· 11 11 __naked void btf_ctx_access_accept(void) 12 12 { 13 13 asm volatile (" \ 14 - r2 = *(u32*)(r1 + 8); /* load 2nd argument value (int pointer) */\ 14 + r2 = *(u64 *)(r1 + 8); /* load 2nd argument value (int pointer) */\ 15 15 r0 = 0; \ 16 16 exit; \ 17 17 " ::: __clobber_all); ··· 23 23 __naked void ctx_access_u32_pointer_accept(void) 24 24 { 25 25 asm volatile (" \ 26 - r2 = *(u32*)(r1 + 0); /* load 1nd argument value (u32 pointer) */\ 26 + r2 = *(u64 *)(r1 + 0); /* load 1nd argument value (u32 pointer) */\ 27 + r0 = 0; \ 28 + exit; \ 29 + " ::: __clobber_all); 30 + } 31 + 32 + SEC("fentry/bpf_fentry_test9") 33 + __description("btf_ctx_access u32 pointer reject u32") 34 + __failure __msg("size 4 must be 8") 35 + __naked void ctx_access_u32_pointer_reject_32(void) 36 + { 37 + asm volatile (" \ 38 + r2 = *(u32 *)(r1 + 0); /* load 1st argument with narrow load */\ 39 + r0 = 0; \ 40 + exit; \ 41 + " ::: __clobber_all); 42 + } 43 + 44 + SEC("fentry/bpf_fentry_test9") 45 + __description("btf_ctx_access u32 pointer reject u16") 46 + __failure __msg("size 2 must be 8") 47 + __naked void ctx_access_u32_pointer_reject_16(void) 48 + { 49 + asm volatile (" \ 50 + r2 = *(u16 *)(r1 + 0); /* load 1st argument with narrow load */\ 51 + r0 = 0; \ 52 + exit; \ 53 + " ::: __clobber_all); 54 + } 55 + 56 + SEC("fentry/bpf_fentry_test9") 57 + __description("btf_ctx_access u32 pointer reject u8") 58 + __failure __msg("size 1 must be 8") 59 + __naked void ctx_access_u32_pointer_reject_8(void) 60 + { 61 + asm volatile (" \ 62 + r2 = *(u8 *)(r1 + 0); /* load 1st argument with narrow load */\ 27 63 r0 = 0; \ 28 64 exit; \ 29 65 " ::: __clobber_all);
+2 -2
tools/testing/selftests/bpf/progs/verifier_d_path.c
··· 11 11 __naked void d_path_accept(void) 12 12 { 13 13 asm volatile (" \ 14 - r1 = *(u32*)(r1 + 0); \ 14 + r1 = *(u64 *)(r1 + 0); \ 15 15 r2 = r10; \ 16 16 r2 += -8; \ 17 17 r6 = 0; \ ··· 31 31 __naked void d_path_reject(void) 32 32 { 33 33 asm volatile (" \ 34 - r1 = *(u32*)(r1 + 0); \ 34 + r1 = *(u64 *)(r1 + 0); \ 35 35 r2 = r10; \ 36 36 r2 += -8; \ 37 37 r6 = 0; \
+56
tools/testing/selftests/bpf/progs/verifier_sock.c
··· 50 50 __uint(map_flags, BPF_F_NO_PREALLOC); 51 51 } sk_storage_map SEC(".maps"); 52 52 53 + struct { 54 + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); 55 + __uint(max_entries, 1); 56 + __uint(key_size, sizeof(__u32)); 57 + __uint(value_size, sizeof(__u32)); 58 + } jmp_table SEC(".maps"); 59 + 53 60 SEC("cgroup/skb") 54 61 __description("skb->sk: no NULL check") 55 62 __failure __msg("invalid mem access 'sock_common_or_null'") ··· 1042 1035 " : 1043 1036 : __imm_const(bpf_sock_src_port, offsetof(struct bpf_sock, src_port)) 1044 1037 : __clobber_all); 1038 + } 1039 + 1040 + __noinline 1041 + long skb_pull_data2(struct __sk_buff *sk, __u32 len) 1042 + { 1043 + return bpf_skb_pull_data(sk, len); 1044 + } 1045 + 1046 + __noinline 1047 + long skb_pull_data1(struct __sk_buff *sk, __u32 len) 1048 + { 1049 + return skb_pull_data2(sk, len); 1050 + } 1051 + 1052 + /* global function calls bpf_skb_pull_data(), which invalidates packet 1053 + * pointers established before global function call. 1054 + */ 1055 + SEC("tc") 1056 + __failure __msg("invalid mem access") 1057 + int invalidate_pkt_pointers_from_global_func(struct __sk_buff *sk) 1058 + { 1059 + int *p = (void *)(long)sk->data; 1060 + 1061 + if ((void *)(p + 1) > (void *)(long)sk->data_end) 1062 + return TCX_DROP; 1063 + skb_pull_data1(sk, 0); 1064 + *p = 42; /* this is unsafe */ 1065 + return TCX_PASS; 1066 + } 1067 + 1068 + __noinline 1069 + int tail_call(struct __sk_buff *sk) 1070 + { 1071 + bpf_tail_call_static(sk, &jmp_table, 0); 1072 + return 0; 1073 + } 1074 + 1075 + /* Tail calls invalidate packet pointers. */ 1076 + SEC("tc") 1077 + __failure __msg("invalid mem access") 1078 + int invalidate_pkt_pointers_by_tail_call(struct __sk_buff *sk) 1079 + { 1080 + int *p = (void *)(long)sk->data; 1081 + 1082 + if ((void *)(p + 1) > (void *)(long)sk->data_end) 1083 + return TCX_DROP; 1084 + tail_call(sk); 1085 + *p = 42; /* this is unsafe */ 1086 + return TCX_PASS; 1045 1087 } 1046 1088 1047 1089 char _license[] SEC("license") = "GPL";