+24 -24

arch/arm64/net/bpf_jit_comp.c

reviewed

··· 267 267 return !(imm & ~0xfff) || !(imm & ~0xfff000); 268 268 } 269 269 270 270 + static inline void emit_a64_add_i(const bool is64, const int dst, const int src, 271 271 + const int tmp, const s32 imm, struct jit_ctx *ctx) 272 272 + { 273 273 + if (is_addsub_imm(imm)) { 274 274 + emit(A64_ADD_I(is64, dst, src, imm), ctx); 275 275 + } else if (is_addsub_imm(-imm)) { 276 276 + emit(A64_SUB_I(is64, dst, src, -imm), ctx); 277 277 + } else { 278 278 + emit_a64_mov_i(is64, tmp, imm, ctx); 279 279 + emit(A64_ADD(is64, dst, src, tmp), ctx); 280 280 + } 281 281 + } 282 282 + 270 283 /* 271 284 * There are 3 types of AArch64 LDR/STR (immediate) instruction: 272 285 * Post-index, Pre-index, Unsigned offset. ··· 661 648 const s16 off = insn->off; 662 649 u8 reg = dst; 663 650 664 664 - if (off || arena) { 665 665 - if (off) { 666 666 - emit_a64_mov_i(1, tmp, off, ctx); 667 667 - emit(A64_ADD(1, tmp, tmp, dst), ctx); 668 668 - reg = tmp; 669 669 - } 670 670 - if (arena) { 671 671 - emit(A64_ADD(1, tmp, reg, arena_vm_base), ctx); 672 672 - reg = tmp; 673 673 - } 651 651 + if (off) { 652 652 + emit_a64_add_i(1, tmp, reg, tmp, off, ctx); 653 653 + reg = tmp; 654 654 + } 655 655 + if (arena) { 656 656 + emit(A64_ADD(1, tmp, reg, arena_vm_base), ctx); 657 657 + reg = tmp; 674 658 } 675 659 676 660 switch (insn->imm) { ··· 733 723 const s32 imm = insn->imm; 734 724 const s16 off = insn->off; 735 725 const bool isdw = BPF_SIZE(code) == BPF_DW; 736 736 - u8 reg; 726 726 + u8 reg = dst; 737 727 s32 jmp_offset; 738 728 739 729 if (BPF_MODE(code) == BPF_PROBE_ATOMIC) { ··· 742 732 return -EINVAL; 743 733 } 744 734 745 745 - if (!off) { 746 746 - reg = dst; 747 747 - } else { 748 748 - emit_a64_mov_i(1, tmp, off, ctx); 749 749 - emit(A64_ADD(1, tmp, tmp, dst), ctx); 735 735 + if (off) { 736 736 + emit_a64_add_i(1, tmp, reg, tmp, off, ctx); 750 737 reg = tmp; 751 738 } 752 739 ··· 1153 1146 /* dst = dst OP imm */ 1154 1147 case BPF_ALU | BPF_ADD | BPF_K: 1155 1148 case BPF_ALU64 | BPF_ADD | BPF_K: 1156 1156 - if (is_addsub_imm(imm)) { 1157 1157 - emit(A64_ADD_I(is64, dst, dst, imm), ctx); 1158 1158 - } else if (is_addsub_imm(-imm)) { 1159 1159 - emit(A64_SUB_I(is64, dst, dst, -imm), ctx); 1160 1160 - } else { 1161 1161 - emit_a64_mov_i(is64, tmp, imm, ctx); 1162 1162 - emit(A64_ADD(is64, dst, dst, tmp), ctx); 1163 1163 - } 1149 1149 + emit_a64_add_i(is64, dst, dst, tmp, imm, ctx); 1164 1150 break; 1165 1151 case BPF_ALU | BPF_SUB | BPF_K: 1166 1152 case BPF_ALU64 | BPF_SUB | BPF_K:

+17

include/linux/bpf.h

reviewed

··· 2299 2299 struct bpf_map *bpf_map_get(u32 ufd); 2300 2300 struct bpf_map *bpf_map_get_with_uref(u32 ufd); 2301 2301 2302 2302 + /* 2303 2303 + * The __bpf_map_get() and __btf_get_by_fd() functions parse a file 2304 2304 + * descriptor and return a corresponding map or btf object. 2305 2305 + * Their names are double underscored to emphasize the fact that they 2306 2306 + * do not increase refcnt. To also increase refcnt use corresponding 2307 2307 + * bpf_map_get() and btf_get_by_fd() functions. 2308 2308 + */ 2309 2309 + 2302 2310 static inline struct bpf_map *__bpf_map_get(struct fd f) 2303 2311 { 2304 2312 if (fd_empty(f)) 2305 2313 return ERR_PTR(-EBADF); 2306 2314 if (unlikely(fd_file(f)->f_op != &bpf_map_fops)) 2315 2315 + return ERR_PTR(-EINVAL); 2316 2316 + return fd_file(f)->private_data; 2317 2317 + } 2318 2318 + 2319 2319 + static inline struct btf *__btf_get_by_fd(struct fd f) 2320 2320 + { 2321 2321 + if (fd_empty(f)) 2322 2322 + return ERR_PTR(-EBADF); 2323 2323 + if (unlikely(fd_file(f)->f_op != &btf_fops)) 2307 2324 return ERR_PTR(-EINVAL); 2308 2325 return fd_file(f)->private_data; 2309 2326 }

+16 -10

include/linux/bpf_verifier.h

reviewed

··· 233 233 */ 234 234 STACK_DYNPTR, 235 235 STACK_ITER, 236 236 + STACK_IRQ_FLAG, 236 237 }; 237 238 238 239 #define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ ··· 255 254 * default to pointer reference on zero initialization of a state. 256 255 */ 257 256 enum ref_state_type { 258 258 - REF_TYPE_PTR = 0, 259 259 - REF_TYPE_LOCK, 257 257 + REF_TYPE_PTR = 1, 258 258 + REF_TYPE_IRQ = 2, 259 259 + REF_TYPE_LOCK = 3, 260 260 } type; 261 261 /* Track each reference created with a unique id, even if the same 262 262 * instruction creates the reference multiple times (eg, via CALL). ··· 317 315 u32 callback_depth; 318 316 319 317 /* The following fields should be last. See copy_func_state() */ 320 320 - int acquired_refs; 321 321 - int active_locks; 322 322 - struct bpf_reference_state *refs; 323 318 /* The state of the stack. Each element of the array describes BPF_REG_SIZE 324 319 * (i.e. 8) bytes worth of stack memory. 325 320 * stack[0] represents bytes [*(r10-8)..*(r10-1)] ··· 369 370 /* call stack tracking */ 370 371 struct bpf_func_state *frame[MAX_CALL_FRAMES]; 371 372 struct bpf_verifier_state *parent; 373 373 + /* Acquired reference states */ 374 374 + struct bpf_reference_state *refs; 372 375 /* 373 376 * 'branches' field is the number of branches left to explore: 374 377 * 0 - all possible paths from this state reached bpf_exit or ··· 420 419 u32 insn_idx; 421 420 u32 curframe; 422 421 423 423 - bool speculative; 422 422 + u32 acquired_refs; 423 423 + u32 active_locks; 424 424 + u32 active_preempt_locks; 425 425 + u32 active_irq_id; 424 426 bool active_rcu_lock; 425 425 - u32 active_preempt_lock; 427 427 + 428 428 + bool speculative; 426 429 /* If this state was ever pointed-to by other state's loop_entry field 427 430 * this flag would be set to true. Used to avoid freeing such states 428 431 * while they are still in use. ··· 985 980 const char *iter_type_str(const struct btf *btf, u32 btf_id); 986 981 const char *iter_state_str(enum bpf_iter_state state); 987 982 988 988 - void print_verifier_state(struct bpf_verifier_env *env, 989 989 - const struct bpf_func_state *state, bool print_all); 990 990 - void print_insn_state(struct bpf_verifier_env *env, const struct bpf_func_state *state); 983 983 + void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, 984 984 + u32 frameno, bool print_all); 985 985 + void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, 986 986 + u32 frameno); 991 987 992 988 #endif /* _LINUX_BPF_VERIFIER_H */

+5

include/linux/btf.h

reviewed

··· 353 353 return btf_type_is_int(t) || btf_type_is_enum(t); 354 354 } 355 355 356 356 + static inline bool btf_type_is_fwd(const struct btf_type *t) 357 357 + { 358 358 + return BTF_INFO_KIND(t->info) == BTF_KIND_FWD; 359 359 + } 360 360 + 356 361 static inline bool btf_type_is_typedef(const struct btf_type *t) 357 362 { 358 363 return BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF;

+10

include/uapi/linux/bpf.h

reviewed

··· 1573 1573 * If provided, prog_flags should have BPF_F_TOKEN_FD flag set. 1574 1574 */ 1575 1575 __s32 prog_token_fd; 1576 1576 + /* The fd_array_cnt can be used to pass the length of the 1577 1577 + * fd_array array. In this case all the [map] file descriptors 1578 1578 + * passed in this array will be bound to the program, even if 1579 1579 + * the maps are not referenced directly. The functionality is 1580 1580 + * similar to the BPF_PROG_BIND_MAP syscall, but maps can be 1581 1581 + * used by the verifier during the program load. If provided, 1582 1582 + * then the fd_array[0,...,fd_array_cnt-1] is expected to be 1583 1583 + * continuous. 1584 1584 + */ 1585 1585 + __u32 fd_array_cnt; 1576 1586 }; 1577 1587 1578 1588 struct { /* anonymous struct used by BPF_OBJ_* commands */

+9 -7

kernel/bpf/arena.c

reviewed

··· 138 138 INIT_LIST_HEAD(&arena->vma_list); 139 139 bpf_map_init_from_attr(&arena->map, attr); 140 140 range_tree_init(&arena->rt); 141 141 - range_tree_set(&arena->rt, 0, attr->max_entries); 141 141 + err = range_tree_set(&arena->rt, 0, attr->max_entries); 142 142 + if (err) { 143 143 + bpf_map_area_free(arena); 144 144 + goto err; 145 145 + } 142 146 mutex_init(&arena->lock); 143 147 144 148 return &arena->map; ··· 222 218 struct vma_list { 223 219 struct vm_area_struct *vma; 224 220 struct list_head head; 225 225 - atomic_t mmap_count; 221 221 + refcount_t mmap_count; 226 222 }; 227 223 228 224 static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma) ··· 232 228 vml = kmalloc(sizeof(*vml), GFP_KERNEL); 233 229 if (!vml) 234 230 return -ENOMEM; 235 235 - atomic_set(&vml->mmap_count, 1); 231 231 + refcount_set(&vml->mmap_count, 1); 236 232 vma->vm_private_data = vml; 237 233 vml->vma = vma; 238 234 list_add(&vml->head, &arena->vma_list); ··· 243 239 { 244 240 struct vma_list *vml = vma->vm_private_data; 245 241 246 246 - atomic_inc(&vml->mmap_count); 242 242 + refcount_inc(&vml->mmap_count); 247 243 } 248 244 249 245 static void arena_vm_close(struct vm_area_struct *vma) ··· 252 248 struct bpf_arena *arena = container_of(map, struct bpf_arena, map); 253 249 struct vma_list *vml = vma->vm_private_data; 254 250 255 255 - if (!atomic_dec_and_test(&vml->mmap_count)) 251 251 + if (!refcount_dec_and_test(&vml->mmap_count)) 256 252 return; 257 253 guard(mutex)(&arena->lock); 258 254 /* update link list under lock */ ··· 260 256 vma->vm_private_data = NULL; 261 257 kfree(vml); 262 258 } 263 263 - 264 264 - #define MT_ENTRY ((void *)&arena_map_ops) /* unused. has to be valid pointer */ 265 259 266 260 static vm_fault_t arena_vm_fault(struct vm_fault *vmf) 267 261 {

+2 -4

kernel/bpf/arraymap.c

reviewed

··· 735 735 u64 ret = 0; 736 736 void *val; 737 737 738 738 + cant_migrate(); 739 739 + 738 740 if (flags != 0) 739 741 return -EINVAL; 740 742 741 743 is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; 742 744 array = container_of(map, struct bpf_array, map); 743 743 - if (is_percpu) 744 744 - migrate_disable(); 745 745 for (i = 0; i < map->max_entries; i++) { 746 746 if (is_percpu) 747 747 val = this_cpu_ptr(array->pptrs[i]); ··· 756 756 break; 757 757 } 758 758 759 759 - if (is_percpu) 760 760 - migrate_enable(); 761 759 return num_elems; 762 760 } 763 761

+7 -8

kernel/bpf/bpf_cgrp_storage.c

reviewed

··· 15 15 16 16 static void bpf_cgrp_storage_lock(void) 17 17 { 18 18 - migrate_disable(); 18 18 + cant_migrate(); 19 19 this_cpu_inc(bpf_cgrp_storage_busy); 20 20 } 21 21 22 22 static void bpf_cgrp_storage_unlock(void) 23 23 { 24 24 this_cpu_dec(bpf_cgrp_storage_busy); 25 25 - migrate_enable(); 26 25 } 27 26 28 27 static bool bpf_cgrp_storage_trylock(void) 29 28 { 30 30 - migrate_disable(); 29 29 + cant_migrate(); 31 30 if (unlikely(this_cpu_inc_return(bpf_cgrp_storage_busy) != 1)) { 32 31 this_cpu_dec(bpf_cgrp_storage_busy); 33 33 - migrate_enable(); 34 32 return false; 35 33 } 36 34 return true; ··· 45 47 { 46 48 struct bpf_local_storage *local_storage; 47 49 50 50 + migrate_disable(); 48 51 rcu_read_lock(); 49 52 local_storage = rcu_dereference(cgroup->bpf_cgrp_storage); 50 50 - if (!local_storage) { 51 51 - rcu_read_unlock(); 52 52 - return; 53 53 - } 53 53 + if (!local_storage) 54 54 + goto out; 54 55 55 56 bpf_cgrp_storage_lock(); 56 57 bpf_local_storage_destroy(local_storage); 57 58 bpf_cgrp_storage_unlock(); 59 59 + out: 58 60 rcu_read_unlock(); 61 61 + migrate_enable(); 59 62 } 60 63 61 64 static struct bpf_local_storage_data *

+5 -4

kernel/bpf/bpf_inode_storage.c

reviewed

··· 62 62 if (!bsb) 63 63 return; 64 64 65 65 + migrate_disable(); 65 66 rcu_read_lock(); 66 67 67 68 local_storage = rcu_dereference(bsb->storage); 68 68 - if (!local_storage) { 69 69 - rcu_read_unlock(); 70 70 - return; 71 71 - } 69 69 + if (!local_storage) 70 70 + goto out; 72 71 73 72 bpf_local_storage_destroy(local_storage); 73 73 + out: 74 74 rcu_read_unlock(); 75 75 + migrate_enable(); 75 76 } 76 77 77 78 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)

+15 -23

kernel/bpf/bpf_local_storage.c

reviewed

··· 81 81 return NULL; 82 82 83 83 if (smap->bpf_ma) { 84 84 - migrate_disable(); 85 84 selem = bpf_mem_cache_alloc_flags(&smap->selem_ma, gfp_flags); 86 86 - migrate_enable(); 87 85 if (selem) 88 86 /* Keep the original bpf_map_kzalloc behavior 89 87 * before started using the bpf_mem_cache_alloc. ··· 172 174 return; 173 175 } 174 176 175 175 - if (smap) { 176 176 - migrate_disable(); 177 177 + if (smap) 177 178 bpf_mem_cache_free(&smap->storage_ma, local_storage); 178 178 - migrate_enable(); 179 179 - } else { 179 179 + else 180 180 /* smap could be NULL if the selem that triggered 181 181 * this 'local_storage' creation had been long gone. 182 182 * In this case, directly do call_rcu(). 183 183 */ 184 184 call_rcu(&local_storage->rcu, bpf_local_storage_free_rcu); 185 185 - } 186 185 } 187 186 188 187 /* rcu tasks trace callback for bpf_ma == false */ ··· 212 217 selem = container_of(rcu, struct bpf_local_storage_elem, rcu); 213 218 /* The bpf_local_storage_map_free will wait for rcu_barrier */ 214 219 smap = rcu_dereference_check(SDATA(selem)->smap, 1); 220 220 + 221 221 + migrate_disable(); 215 222 bpf_obj_free_fields(smap->map.record, SDATA(selem)->data); 223 223 + migrate_enable(); 216 224 bpf_mem_cache_raw_free(selem); 217 225 } 218 226 ··· 254 256 * bpf_mem_cache_free will be able to reuse selem 255 257 * immediately. 256 258 */ 257 257 - migrate_disable(); 258 259 bpf_mem_cache_free(&smap->selem_ma, selem); 259 259 - migrate_enable(); 260 260 return; 261 261 } 262 262 ··· 493 497 if (err) 494 498 return err; 495 499 496 496 - if (smap->bpf_ma) { 497 497 - migrate_disable(); 500 500 + if (smap->bpf_ma) 498 501 storage = bpf_mem_cache_alloc_flags(&smap->storage_ma, gfp_flags); 499 499 - migrate_enable(); 500 500 - } else { 502 502 + else 501 503 storage = bpf_map_kzalloc(&smap->map, sizeof(*storage), 502 504 gfp_flags | __GFP_NOWARN); 503 503 - } 504 504 - 505 505 if (!storage) { 506 506 err = -ENOMEM; 507 507 goto uncharge; ··· 833 841 smap->elem_size = offsetof(struct bpf_local_storage_elem, 834 842 sdata.data[attr->value_size]); 835 843 836 836 - smap->bpf_ma = bpf_ma; 837 837 - if (bpf_ma) { 844 844 + /* In PREEMPT_RT, kmalloc(GFP_ATOMIC) is still not safe in non 845 845 + * preemptible context. Thus, enforce all storages to use 846 846 + * bpf_mem_alloc when CONFIG_PREEMPT_RT is enabled. 847 847 + */ 848 848 + smap->bpf_ma = IS_ENABLED(CONFIG_PREEMPT_RT) ? true : bpf_ma; 849 849 + if (smap->bpf_ma) { 838 850 err = bpf_mem_alloc_init(&smap->selem_ma, smap->elem_size, false); 839 851 if (err) 840 852 goto free_smap; ··· 894 898 while ((selem = hlist_entry_safe( 895 899 rcu_dereference_raw(hlist_first_rcu(&b->list)), 896 900 struct bpf_local_storage_elem, map_node))) { 897 897 - if (busy_counter) { 898 898 - migrate_disable(); 901 901 + if (busy_counter) 899 902 this_cpu_inc(*busy_counter); 900 900 - } 901 903 bpf_selem_unlink(selem, true); 902 902 - if (busy_counter) { 904 904 + if (busy_counter) 903 905 this_cpu_dec(*busy_counter); 904 904 - migrate_enable(); 905 905 - } 906 906 cond_resched_rcu(); 907 907 } 908 908 rcu_read_unlock();

+21

kernel/bpf/bpf_struct_ops.c

reviewed

··· 310 310 kfree(arg_info); 311 311 } 312 312 313 313 + static bool is_module_member(const struct btf *btf, u32 id) 314 314 + { 315 315 + const struct btf_type *t; 316 316 + 317 317 + t = btf_type_resolve_ptr(btf, id, NULL); 318 318 + if (!t) 319 319 + return false; 320 320 + 321 321 + if (!__btf_type_is_struct(t) && !btf_type_is_fwd(t)) 322 322 + return false; 323 323 + 324 324 + return !strcmp(btf_name_by_offset(btf, t->name_off), "module"); 325 325 + } 326 326 + 313 327 int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, 314 328 struct btf *btf, 315 329 struct bpf_verifier_log *log) ··· 399 385 if (__btf_member_bitfield_size(t, member)) { 400 386 pr_warn("bit field member %s in struct %s is not supported\n", 401 387 mname, st_ops->name); 388 388 + err = -EOPNOTSUPP; 389 389 + goto errout; 390 390 + } 391 391 + 392 392 + if (!st_ops_ids[IDX_MODULE_ID] && is_module_member(btf, member->type)) { 393 393 + pr_warn("'struct module' btf id not found. Is CONFIG_MODULES enabled? bpf_struct_ops '%s' needs module support.\n", 394 394 + st_ops->name); 402 395 err = -EOPNOTSUPP; 403 396 goto errout; 404 397 }

+7 -8

kernel/bpf/bpf_task_storage.c

reviewed

··· 24 24 25 25 static void bpf_task_storage_lock(void) 26 26 { 27 27 - migrate_disable(); 27 27 + cant_migrate(); 28 28 this_cpu_inc(bpf_task_storage_busy); 29 29 } 30 30 31 31 static void bpf_task_storage_unlock(void) 32 32 { 33 33 this_cpu_dec(bpf_task_storage_busy); 34 34 - migrate_enable(); 35 34 } 36 35 37 36 static bool bpf_task_storage_trylock(void) 38 37 { 39 39 - migrate_disable(); 38 38 + cant_migrate(); 40 39 if (unlikely(this_cpu_inc_return(bpf_task_storage_busy) != 1)) { 41 40 this_cpu_dec(bpf_task_storage_busy); 42 42 - migrate_enable(); 43 41 return false; 44 42 } 45 43 return true; ··· 70 72 { 71 73 struct bpf_local_storage *local_storage; 72 74 75 75 + migrate_disable(); 73 76 rcu_read_lock(); 74 77 75 78 local_storage = rcu_dereference(task->bpf_storage); 76 76 - if (!local_storage) { 77 77 - rcu_read_unlock(); 78 78 - return; 79 79 - } 79 79 + if (!local_storage) 80 80 + goto out; 80 81 81 82 bpf_task_storage_lock(); 82 83 bpf_local_storage_destroy(local_storage); 83 84 bpf_task_storage_unlock(); 85 85 + out: 84 86 rcu_read_unlock(); 87 87 + migrate_enable(); 85 88 } 86 89 87 90 static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key)

+3 -13

kernel/bpf/btf.c

reviewed

··· 498 498 return t == &btf_void; 499 499 } 500 500 501 501 - static bool btf_type_is_fwd(const struct btf_type *t) 502 502 - { 503 503 - return BTF_INFO_KIND(t->info) == BTF_KIND_FWD; 504 504 - } 505 505 - 506 501 static bool btf_type_is_datasec(const struct btf_type *t) 507 502 { 508 503 return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC; ··· 7882 7887 struct btf *btf; 7883 7888 CLASS(fd, f)(fd); 7884 7889 7885 7885 - if (fd_empty(f)) 7886 7886 - return ERR_PTR(-EBADF); 7887 7887 - 7888 7888 - if (fd_file(f)->f_op != &btf_fops) 7889 7889 - return ERR_PTR(-EINVAL); 7890 7890 - 7891 7891 - btf = fd_file(f)->private_data; 7892 7892 - refcount_inc(&btf->refcnt); 7890 7890 + btf = __btf_get_by_fd(f); 7891 7891 + if (!IS_ERR(btf)) 7892 7892 + refcount_inc(&btf->refcnt); 7893 7893 7894 7894 return btf; 7895 7895 }

-2

kernel/bpf/cpumask.c

reviewed

··· 91 91 if (!refcount_dec_and_test(&cpumask->usage)) 92 92 return; 93 93 94 94 - migrate_disable(); 95 94 bpf_mem_cache_free_rcu(&bpf_cpumask_ma, cpumask); 96 96 - migrate_enable(); 97 95 } 98 96 99 97 __bpf_kfunc void bpf_cpumask_release_dtor(void *cpumask)

+43 -44

kernel/bpf/hashtab.c

reviewed

··· 824 824 hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) 825 825 if (l == tgt_l) { 826 826 hlist_nulls_del_rcu(&l->hash_node); 827 827 - check_and_free_fields(htab, l); 828 827 bpf_map_dec_elem_count(&htab->map); 829 828 break; 830 829 } 831 830 832 831 htab_unlock_bucket(htab, b, tgt_l->hash, flags); 833 832 833 833 + if (l == tgt_l) 834 834 + check_and_free_fields(htab, l); 834 835 return l == tgt_l; 835 836 } 836 837 ··· 898 897 { 899 898 check_and_free_fields(htab, l); 900 899 901 901 - migrate_disable(); 902 900 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) 903 901 bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); 904 902 bpf_mem_cache_free(&htab->ma, l); 905 905 - migrate_enable(); 906 903 } 907 904 908 905 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) ··· 1501 1502 { 1502 1503 int i; 1503 1504 1504 1504 - /* It's called from a worker thread, so disable migration here, 1505 1505 - * since bpf_mem_cache_free() relies on that. 1505 1505 + /* It's called from a worker thread and migration has been disabled, 1506 1506 + * therefore, it is OK to invoke bpf_mem_cache_free() directly. 1506 1507 */ 1507 1507 - migrate_disable(); 1508 1508 for (i = 0; i < htab->n_buckets; i++) { 1509 1509 struct hlist_nulls_head *head = select_bucket(htab, i); 1510 1510 struct hlist_nulls_node *n; ··· 1515 1517 } 1516 1518 cond_resched(); 1517 1519 } 1518 1518 - migrate_enable(); 1519 1520 } 1520 1521 1521 1522 static void htab_free_malloced_timers_and_wq(struct bpf_htab *htab) ··· 1635 1638 l = lookup_elem_raw(head, hash, key, key_size); 1636 1639 if (!l) { 1637 1640 ret = -ENOENT; 1638 1638 - } else { 1639 1639 - if (is_percpu) { 1640 1640 - u32 roundup_value_size = round_up(map->value_size, 8); 1641 1641 - void __percpu *pptr; 1642 1642 - int off = 0, cpu; 1643 1643 - 1644 1644 - pptr = htab_elem_get_ptr(l, key_size); 1645 1645 - for_each_possible_cpu(cpu) { 1646 1646 - copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu)); 1647 1647 - check_and_init_map_value(&htab->map, value + off); 1648 1648 - off += roundup_value_size; 1649 1649 - } 1650 1650 - } else { 1651 1651 - u32 roundup_key_size = round_up(map->key_size, 8); 1652 1652 - 1653 1653 - if (flags & BPF_F_LOCK) 1654 1654 - copy_map_value_locked(map, value, l->key + 1655 1655 - roundup_key_size, 1656 1656 - true); 1657 1657 - else 1658 1658 - copy_map_value(map, value, l->key + 1659 1659 - roundup_key_size); 1660 1660 - /* Zeroing special fields in the temp buffer */ 1661 1661 - check_and_init_map_value(map, value); 1662 1662 - } 1663 1663 - 1664 1664 - hlist_nulls_del_rcu(&l->hash_node); 1665 1665 - if (!is_lru_map) 1666 1666 - free_htab_elem(htab, l); 1641 1641 + goto out_unlock; 1667 1642 } 1668 1643 1644 1644 + if (is_percpu) { 1645 1645 + u32 roundup_value_size = round_up(map->value_size, 8); 1646 1646 + void __percpu *pptr; 1647 1647 + int off = 0, cpu; 1648 1648 + 1649 1649 + pptr = htab_elem_get_ptr(l, key_size); 1650 1650 + for_each_possible_cpu(cpu) { 1651 1651 + copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu)); 1652 1652 + check_and_init_map_value(&htab->map, value + off); 1653 1653 + off += roundup_value_size; 1654 1654 + } 1655 1655 + } else { 1656 1656 + u32 roundup_key_size = round_up(map->key_size, 8); 1657 1657 + 1658 1658 + if (flags & BPF_F_LOCK) 1659 1659 + copy_map_value_locked(map, value, l->key + 1660 1660 + roundup_key_size, 1661 1661 + true); 1662 1662 + else 1663 1663 + copy_map_value(map, value, l->key + 1664 1664 + roundup_key_size); 1665 1665 + /* Zeroing special fields in the temp buffer */ 1666 1666 + check_and_init_map_value(map, value); 1667 1667 + } 1668 1668 + hlist_nulls_del_rcu(&l->hash_node); 1669 1669 + 1670 1670 + out_unlock: 1669 1671 htab_unlock_bucket(htab, b, hash, bflags); 1670 1672 1671 1671 - if (is_lru_map && l) 1672 1672 - htab_lru_push_free(htab, l); 1673 1673 + if (l) { 1674 1674 + if (is_lru_map) 1675 1675 + htab_lru_push_free(htab, l); 1676 1676 + else 1677 1677 + free_htab_elem(htab, l); 1678 1678 + } 1673 1679 1674 1680 return ret; 1675 1681 } ··· 2208 2208 bool is_percpu; 2209 2209 u64 ret = 0; 2210 2210 2211 2211 + cant_migrate(); 2212 2212 + 2211 2213 if (flags != 0) 2212 2214 return -EINVAL; 2213 2215 2214 2216 is_percpu = htab_is_percpu(htab); 2215 2217 2216 2218 roundup_key_size = round_up(map->key_size, 8); 2217 2217 - /* disable migration so percpu value prepared here will be the 2218 2218 - * same as the one seen by the bpf program with bpf_map_lookup_elem(). 2219 2219 + /* migration has been disabled, so percpu value prepared here will be 2220 2220 + * the same as the one seen by the bpf program with 2221 2221 + * bpf_map_lookup_elem(). 2219 2222 */ 2220 2220 - if (is_percpu) 2221 2221 - migrate_disable(); 2222 2223 for (i = 0; i < htab->n_buckets; i++) { 2223 2224 b = &htab->buckets[i]; 2224 2225 rcu_read_lock(); ··· 2245 2244 rcu_read_unlock(); 2246 2245 } 2247 2246 out: 2248 2248 - if (is_percpu) 2249 2249 - migrate_enable(); 2250 2247 return num_elems; 2251 2248 } 2252 2249

+37 -6

kernel/bpf/helpers.c

reviewed

··· 1593 1593 * To avoid these issues, punt to workqueue context when we are in a 1594 1594 * timer callback. 1595 1595 */ 1596 1596 - if (this_cpu_read(hrtimer_running)) 1596 1596 + if (this_cpu_read(hrtimer_running)) { 1597 1597 queue_work(system_unbound_wq, &t->cb.delete_work); 1598 1598 - else 1598 1598 + return; 1599 1599 + } 1600 1600 + 1601 1601 + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 1602 1602 + /* If the timer is running on other CPU, also use a kworker to 1603 1603 + * wait for the completion of the timer instead of trying to 1604 1604 + * acquire a sleepable lock in hrtimer_cancel() to wait for its 1605 1605 + * completion. 1606 1606 + */ 1607 1607 + if (hrtimer_try_to_cancel(&t->timer) >= 0) 1608 1608 + kfree_rcu(t, cb.rcu); 1609 1609 + else 1610 1610 + queue_work(system_unbound_wq, &t->cb.delete_work); 1611 1611 + } else { 1599 1612 bpf_timer_delete_work(&t->cb.delete_work); 1613 1613 + } 1600 1614 } 1601 1615 1602 1616 /* This function is called by map_delete/update_elem for individual element and ··· 2080 2066 /* The contained type can also have resources, including a 2081 2067 * bpf_list_head which needs to be freed. 2082 2068 */ 2083 2083 - migrate_disable(); 2084 2069 __bpf_obj_drop_impl(obj, field->graph_root.value_rec, false); 2085 2085 - migrate_enable(); 2086 2070 } 2087 2071 } 2088 2072 ··· 2117 2105 obj -= field->graph_root.node_offset; 2118 2106 2119 2107 2120 2120 - migrate_disable(); 2121 2108 __bpf_obj_drop_impl(obj, field->graph_root.value_rec, false); 2122 2122 - migrate_enable(); 2123 2109 } 2124 2110 } 2125 2111 ··· 3067 3057 return ret + 1; 3068 3058 } 3069 3059 3060 3060 + /* Keep unsinged long in prototype so that kfunc is usable when emitted to 3061 3061 + * vmlinux.h in BPF programs directly, but note that while in BPF prog, the 3062 3062 + * unsigned long always points to 8-byte region on stack, the kernel may only 3063 3063 + * read and write the 4-bytes on 32-bit. 3064 3064 + */ 3065 3065 + __bpf_kfunc void bpf_local_irq_save(unsigned long *flags__irq_flag) 3066 3066 + { 3067 3067 + local_irq_save(*flags__irq_flag); 3068 3068 + } 3069 3069 + 3070 3070 + __bpf_kfunc void bpf_local_irq_restore(unsigned long *flags__irq_flag) 3071 3071 + { 3072 3072 + local_irq_restore(*flags__irq_flag); 3073 3073 + } 3074 3074 + 3070 3075 __bpf_kfunc_end_defs(); 3071 3076 3072 3077 BTF_KFUNCS_START(generic_btf_ids) ··· 3114 3089 BTF_ID_FLAGS(func, bpf_task_from_pid, KF_ACQUIRE | KF_RET_NULL) 3115 3090 BTF_ID_FLAGS(func, bpf_task_from_vpid, KF_ACQUIRE | KF_RET_NULL) 3116 3091 BTF_ID_FLAGS(func, bpf_throw) 3092 3092 + #ifdef CONFIG_BPF_EVENTS 3117 3093 BTF_ID_FLAGS(func, bpf_send_signal_task, KF_TRUSTED_ARGS) 3094 3094 + #endif 3118 3095 BTF_KFUNCS_END(generic_btf_ids) 3119 3096 3120 3097 static const struct btf_kfunc_id_set generic_kfunc_set = { ··· 3162 3135 BTF_ID_FLAGS(func, bpf_dynptr_is_rdonly) 3163 3136 BTF_ID_FLAGS(func, bpf_dynptr_size) 3164 3137 BTF_ID_FLAGS(func, bpf_dynptr_clone) 3138 3138 + #ifdef CONFIG_NET 3165 3139 BTF_ID_FLAGS(func, bpf_modify_return_test_tp) 3140 3140 + #endif 3166 3141 BTF_ID_FLAGS(func, bpf_wq_init) 3167 3142 BTF_ID_FLAGS(func, bpf_wq_set_callback_impl) 3168 3143 BTF_ID_FLAGS(func, bpf_wq_start) ··· 3178 3149 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) 3179 3150 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) 3180 3151 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) 3152 3152 + BTF_ID_FLAGS(func, bpf_local_irq_save) 3153 3153 + BTF_ID_FLAGS(func, bpf_local_irq_restore) 3181 3154 BTF_KFUNCS_END(common_btf_ids) 3182 3155 3183 3156 static const struct btf_kfunc_id_set common_kfunc_set = {

+12 -9

kernel/bpf/log.c

reviewed

··· 537 537 [STACK_ZERO] = '0', 538 538 [STACK_DYNPTR] = 'd', 539 539 [STACK_ITER] = 'i', 540 540 + [STACK_IRQ_FLAG] = 'f' 540 541 }; 541 542 542 543 static void print_liveness(struct bpf_verifier_env *env, ··· 754 753 verbose(env, ")"); 755 754 } 756 755 757 757 - void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_func_state *state, 758 758 - bool print_all) 756 756 + void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, 757 757 + u32 frameno, bool print_all) 759 758 { 759 759 + const struct bpf_func_state *state = vstate->frame[frameno]; 760 760 const struct bpf_reg_state *reg; 761 761 int i; 762 762 ··· 845 843 break; 846 844 } 847 845 } 848 848 - if (state->acquired_refs && state->refs[0].id) { 849 849 - verbose(env, " refs=%d", state->refs[0].id); 850 850 - for (i = 1; i < state->acquired_refs; i++) 851 851 - if (state->refs[i].id) 852 852 - verbose(env, ",%d", state->refs[i].id); 846 846 + if (vstate->acquired_refs && vstate->refs[0].id) { 847 847 + verbose(env, " refs=%d", vstate->refs[0].id); 848 848 + for (i = 1; i < vstate->acquired_refs; i++) 849 849 + if (vstate->refs[i].id) 850 850 + verbose(env, ",%d", vstate->refs[i].id); 853 851 } 854 852 if (state->in_callback_fn) 855 853 verbose(env, " cb"); ··· 866 864 BPF_LOG_MIN_ALIGNMENT) - pos - 1; 867 865 } 868 866 869 869 - void print_insn_state(struct bpf_verifier_env *env, const struct bpf_func_state *state) 867 867 + void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, 868 868 + u32 frameno) 870 869 { 871 870 if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) { 872 871 /* remove new line character */ ··· 876 873 } else { 877 874 verbose(env, "%d:", env->insn_idx); 878 875 } 879 879 - print_verifier_state(env, state, false); 876 876 + print_verifier_state(env, vstate, frameno, false); 880 877 }

+4 -16

kernel/bpf/lpm_trie.c

reviewed

··· 289 289 } 290 290 291 291 static struct lpm_trie_node *lpm_trie_node_alloc(struct lpm_trie *trie, 292 292 - const void *value, 293 293 - bool disable_migration) 292 292 + const void *value) 294 293 { 295 294 struct lpm_trie_node *node; 296 295 297 297 - if (disable_migration) 298 298 - migrate_disable(); 299 296 node = bpf_mem_cache_alloc(&trie->ma); 300 300 - if (disable_migration) 301 301 - migrate_enable(); 302 297 303 298 if (!node) 304 299 return NULL; ··· 337 342 if (key->prefixlen > trie->max_prefixlen) 338 343 return -EINVAL; 339 344 340 340 - /* Allocate and fill a new node. Need to disable migration before 341 341 - * invoking bpf_mem_cache_alloc(). 342 342 - */ 343 343 - new_node = lpm_trie_node_alloc(trie, value, true); 345 345 + /* Allocate and fill a new node */ 346 346 + new_node = lpm_trie_node_alloc(trie, value); 344 347 if (!new_node) 345 348 return -ENOMEM; 346 349 ··· 418 425 goto out; 419 426 } 420 427 421 421 - /* migration is disabled within the locked scope */ 422 422 - im_node = lpm_trie_node_alloc(trie, NULL, false); 428 428 + im_node = lpm_trie_node_alloc(trie, NULL); 423 429 if (!im_node) { 424 430 trie->n_entries--; 425 431 ret = -ENOMEM; ··· 444 452 out: 445 453 raw_spin_unlock_irqrestore(&trie->lock, irq_flags); 446 454 447 447 - migrate_disable(); 448 455 if (ret) 449 456 bpf_mem_cache_free(&trie->ma, new_node); 450 457 bpf_mem_cache_free_rcu(&trie->ma, free_node); 451 451 - migrate_enable(); 452 458 453 459 return ret; 454 460 } ··· 545 555 out: 546 556 raw_spin_unlock_irqrestore(&trie->lock, irq_flags); 547 557 548 548 - migrate_disable(); 549 558 bpf_mem_cache_free_rcu(&trie->ma, free_parent); 550 559 bpf_mem_cache_free_rcu(&trie->ma, free_node); 551 551 - migrate_enable(); 552 560 553 561 return ret; 554 562 }

-2

kernel/bpf/range_tree.c

reviewed

··· 259 259 260 260 while ((rn = range_it_iter_first(rt, 0, -1U))) { 261 261 range_it_remove(rn, rt); 262 262 - migrate_disable(); 263 262 bpf_mem_free(&bpf_global_ma, rn); 264 264 - migrate_enable(); 265 263 } 266 264 } 267 265

+8 -4

kernel/bpf/syscall.c

reviewed

··· 796 796 if (!btf_is_kernel(field->kptr.btf)) { 797 797 pointee_struct_meta = btf_find_struct_meta(field->kptr.btf, 798 798 field->kptr.btf_id); 799 799 - migrate_disable(); 800 799 __bpf_obj_drop_impl(xchgd_field, pointee_struct_meta ? 801 800 pointee_struct_meta->record : NULL, 802 801 fields[i].type == BPF_KPTR_PERCPU); 803 803 - migrate_enable(); 804 802 } else { 805 803 field->kptr.dtor(xchgd_field); 806 804 } ··· 833 835 struct btf_record *rec = map->record; 834 836 struct btf *btf = map->btf; 835 837 836 836 - /* implementation dependent freeing */ 838 838 + /* implementation dependent freeing. Disabling migration to simplify 839 839 + * the free of values or special fields allocated from bpf memory 840 840 + * allocator. 841 841 + */ 842 842 + migrate_disable(); 837 843 map->ops->map_free(map); 844 844 + migrate_enable(); 845 845 + 838 846 /* Delay freeing of btf_record for maps, as map_free 839 847 * callback usually needs access to them. It is better to do it here 840 848 * than require each callback to do the free itself manually. ··· 2734 2730 } 2735 2731 2736 2732 /* last field in 'union bpf_attr' used by this command */ 2737 2737 - #define BPF_PROG_LOAD_LAST_FIELD prog_token_fd 2733 2733 + #define BPF_PROG_LOAD_LAST_FIELD fd_array_cnt 2738 2734 2739 2735 static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) 2740 2736 {

+844 -364

kernel/bpf/verifier.c

reviewed

··· 196 196 197 197 #define BPF_PRIV_STACK_MIN_SIZE 64 198 198 199 199 - static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx); 199 199 + static int acquire_reference(struct bpf_verifier_env *env, int insn_idx); 200 200 + static int release_reference_nomark(struct bpf_verifier_state *state, int ref_obj_id); 200 201 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id); 201 202 static void invalidate_non_owning_refs(struct bpf_verifier_env *env); 202 203 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); ··· 287 286 u32 ret_btf_id; 288 287 u32 subprogno; 289 288 struct btf_field *kptr_field; 289 289 + s64 const_map_key; 290 290 }; 291 291 292 292 struct bpf_kfunc_call_arg_meta { ··· 643 641 return stack_slot_obj_get_spi(env, reg, "iter", nr_slots); 644 642 } 645 643 644 644 + static int irq_flag_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 645 645 + { 646 646 + return stack_slot_obj_get_spi(env, reg, "irq_flag", 1); 647 647 + } 648 648 + 646 649 static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type) 647 650 { 648 651 switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { ··· 759 752 if (clone_ref_obj_id) 760 753 id = clone_ref_obj_id; 761 754 else 762 762 - id = acquire_reference_state(env, insn_idx); 755 755 + id = acquire_reference(env, insn_idx); 763 756 764 757 if (id < 0) 765 758 return id; ··· 1021 1014 if (spi < 0) 1022 1015 return spi; 1023 1016 1024 1024 - id = acquire_reference_state(env, insn_idx); 1017 1017 + id = acquire_reference(env, insn_idx); 1025 1018 if (id < 0) 1026 1019 return id; 1027 1020 ··· 1143 1136 return 0; 1144 1137 } 1145 1138 1139 1139 + static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx); 1140 1140 + static int release_irq_state(struct bpf_verifier_state *state, int id); 1141 1141 + 1142 1142 + static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, 1143 1143 + struct bpf_kfunc_call_arg_meta *meta, 1144 1144 + struct bpf_reg_state *reg, int insn_idx) 1145 1145 + { 1146 1146 + struct bpf_func_state *state = func(env, reg); 1147 1147 + struct bpf_stack_state *slot; 1148 1148 + struct bpf_reg_state *st; 1149 1149 + int spi, i, id; 1150 1150 + 1151 1151 + spi = irq_flag_get_spi(env, reg); 1152 1152 + if (spi < 0) 1153 1153 + return spi; 1154 1154 + 1155 1155 + id = acquire_irq_state(env, insn_idx); 1156 1156 + if (id < 0) 1157 1157 + return id; 1158 1158 + 1159 1159 + slot = &state->stack[spi]; 1160 1160 + st = &slot->spilled_ptr; 1161 1161 + 1162 1162 + __mark_reg_known_zero(st); 1163 1163 + st->type = PTR_TO_STACK; /* we don't have dedicated reg type */ 1164 1164 + st->live |= REG_LIVE_WRITTEN; 1165 1165 + st->ref_obj_id = id; 1166 1166 + 1167 1167 + for (i = 0; i < BPF_REG_SIZE; i++) 1168 1168 + slot->slot_type[i] = STACK_IRQ_FLAG; 1169 1169 + 1170 1170 + mark_stack_slot_scratched(env, spi); 1171 1171 + return 0; 1172 1172 + } 1173 1173 + 1174 1174 + static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 1175 1175 + { 1176 1176 + struct bpf_func_state *state = func(env, reg); 1177 1177 + struct bpf_stack_state *slot; 1178 1178 + struct bpf_reg_state *st; 1179 1179 + int spi, i, err; 1180 1180 + 1181 1181 + spi = irq_flag_get_spi(env, reg); 1182 1182 + if (spi < 0) 1183 1183 + return spi; 1184 1184 + 1185 1185 + slot = &state->stack[spi]; 1186 1186 + st = &slot->spilled_ptr; 1187 1187 + 1188 1188 + err = release_irq_state(env->cur_state, st->ref_obj_id); 1189 1189 + WARN_ON_ONCE(err && err != -EACCES); 1190 1190 + if (err) { 1191 1191 + int insn_idx = 0; 1192 1192 + 1193 1193 + for (int i = 0; i < env->cur_state->acquired_refs; i++) { 1194 1194 + if (env->cur_state->refs[i].id == env->cur_state->active_irq_id) { 1195 1195 + insn_idx = env->cur_state->refs[i].insn_idx; 1196 1196 + break; 1197 1197 + } 1198 1198 + } 1199 1199 + 1200 1200 + verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n", 1201 1201 + env->cur_state->active_irq_id, insn_idx); 1202 1202 + return err; 1203 1203 + } 1204 1204 + 1205 1205 + __mark_reg_not_init(env, st); 1206 1206 + 1207 1207 + /* see unmark_stack_slots_dynptr() for why we need to set REG_LIVE_WRITTEN */ 1208 1208 + st->live |= REG_LIVE_WRITTEN; 1209 1209 + 1210 1210 + for (i = 0; i < BPF_REG_SIZE; i++) 1211 1211 + slot->slot_type[i] = STACK_INVALID; 1212 1212 + 1213 1213 + mark_stack_slot_scratched(env, spi); 1214 1214 + return 0; 1215 1215 + } 1216 1216 + 1217 1217 + static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 1218 1218 + { 1219 1219 + struct bpf_func_state *state = func(env, reg); 1220 1220 + struct bpf_stack_state *slot; 1221 1221 + int spi, i; 1222 1222 + 1223 1223 + /* For -ERANGE (i.e. spi not falling into allocated stack slots), we 1224 1224 + * will do check_mem_access to check and update stack bounds later, so 1225 1225 + * return true for that case. 1226 1226 + */ 1227 1227 + spi = irq_flag_get_spi(env, reg); 1228 1228 + if (spi == -ERANGE) 1229 1229 + return true; 1230 1230 + if (spi < 0) 1231 1231 + return false; 1232 1232 + 1233 1233 + slot = &state->stack[spi]; 1234 1234 + 1235 1235 + for (i = 0; i < BPF_REG_SIZE; i++) 1236 1236 + if (slot->slot_type[i] == STACK_IRQ_FLAG) 1237 1237 + return false; 1238 1238 + return true; 1239 1239 + } 1240 1240 + 1241 1241 + static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 1242 1242 + { 1243 1243 + struct bpf_func_state *state = func(env, reg); 1244 1244 + struct bpf_stack_state *slot; 1245 1245 + struct bpf_reg_state *st; 1246 1246 + int spi, i; 1247 1247 + 1248 1248 + spi = irq_flag_get_spi(env, reg); 1249 1249 + if (spi < 0) 1250 1250 + return -EINVAL; 1251 1251 + 1252 1252 + slot = &state->stack[spi]; 1253 1253 + st = &slot->spilled_ptr; 1254 1254 + 1255 1255 + if (!st->ref_obj_id) 1256 1256 + return -EINVAL; 1257 1257 + 1258 1258 + for (i = 0; i < BPF_REG_SIZE; i++) 1259 1259 + if (slot->slot_type[i] != STACK_IRQ_FLAG) 1260 1260 + return -EINVAL; 1261 1261 + return 0; 1262 1262 + } 1263 1263 + 1146 1264 /* Check if given stack slot is "special": 1147 1265 * - spilled register state (STACK_SPILL); 1148 1266 * - dynptr state (STACK_DYNPTR); 1149 1267 * - iter state (STACK_ITER). 1268 1268 + * - irq flag state (STACK_IRQ_FLAG) 1150 1269 */ 1151 1270 static bool is_stack_slot_special(const struct bpf_stack_state *stack) 1152 1271 { ··· 1282 1149 case STACK_SPILL: 1283 1150 case STACK_DYNPTR: 1284 1151 case STACK_ITER: 1152 1152 + case STACK_IRQ_FLAG: 1285 1153 return true; 1286 1154 case STACK_INVALID: 1287 1155 case STACK_MISC: ··· 1397 1263 return arr ? arr : ZERO_SIZE_PTR; 1398 1264 } 1399 1265 1400 1400 - static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1266 1266 + static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf_verifier_state *src) 1401 1267 { 1402 1268 dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, 1403 1269 sizeof(struct bpf_reference_state), GFP_KERNEL); 1404 1270 if (!dst->refs) 1405 1271 return -ENOMEM; 1406 1272 1407 1407 - dst->active_locks = src->active_locks; 1408 1273 dst->acquired_refs = src->acquired_refs; 1274 1274 + dst->active_locks = src->active_locks; 1275 1275 + dst->active_preempt_locks = src->active_preempt_locks; 1276 1276 + dst->active_rcu_lock = src->active_rcu_lock; 1277 1277 + dst->active_irq_id = src->active_irq_id; 1409 1278 return 0; 1410 1279 } 1411 1280 ··· 1425 1288 return 0; 1426 1289 } 1427 1290 1428 1428 - static int resize_reference_state(struct bpf_func_state *state, size_t n) 1291 1291 + static int resize_reference_state(struct bpf_verifier_state *state, size_t n) 1429 1292 { 1430 1293 state->refs = realloc_array(state->refs, state->acquired_refs, n, 1431 1294 sizeof(struct bpf_reference_state)); ··· 1468 1331 * On success, returns a valid pointer id to associate with the register 1469 1332 * On failure, returns a negative errno. 1470 1333 */ 1471 1471 - static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 1334 1334 + static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 1472 1335 { 1473 1473 - struct bpf_func_state *state = cur_func(env); 1474 1474 - int new_ofs = state->acquired_refs; 1475 1475 - int id, err; 1476 1476 - 1477 1477 - err = resize_reference_state(state, state->acquired_refs + 1); 1478 1478 - if (err) 1479 1479 - return err; 1480 1480 - id = ++env->id_gen; 1481 1481 - state->refs[new_ofs].type = REF_TYPE_PTR; 1482 1482 - state->refs[new_ofs].id = id; 1483 1483 - state->refs[new_ofs].insn_idx = insn_idx; 1484 1484 - 1485 1485 - return id; 1486 1486 - } 1487 1487 - 1488 1488 - static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type, 1489 1489 - int id, void *ptr) 1490 1490 - { 1491 1491 - struct bpf_func_state *state = cur_func(env); 1336 1336 + struct bpf_verifier_state *state = env->cur_state; 1492 1337 int new_ofs = state->acquired_refs; 1493 1338 int err; 1494 1339 1495 1340 err = resize_reference_state(state, state->acquired_refs + 1); 1496 1341 if (err) 1497 1497 - return err; 1498 1498 - state->refs[new_ofs].type = type; 1499 1499 - state->refs[new_ofs].id = id; 1342 1342 + return NULL; 1500 1343 state->refs[new_ofs].insn_idx = insn_idx; 1501 1501 - state->refs[new_ofs].ptr = ptr; 1344 1344 + 1345 1345 + return &state->refs[new_ofs]; 1346 1346 + } 1347 1347 + 1348 1348 + static int acquire_reference(struct bpf_verifier_env *env, int insn_idx) 1349 1349 + { 1350 1350 + struct bpf_reference_state *s; 1351 1351 + 1352 1352 + s = acquire_reference_state(env, insn_idx); 1353 1353 + if (!s) 1354 1354 + return -ENOMEM; 1355 1355 + s->type = REF_TYPE_PTR; 1356 1356 + s->id = ++env->id_gen; 1357 1357 + return s->id; 1358 1358 + } 1359 1359 + 1360 1360 + static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type, 1361 1361 + int id, void *ptr) 1362 1362 + { 1363 1363 + struct bpf_verifier_state *state = env->cur_state; 1364 1364 + struct bpf_reference_state *s; 1365 1365 + 1366 1366 + s = acquire_reference_state(env, insn_idx); 1367 1367 + s->type = type; 1368 1368 + s->id = id; 1369 1369 + s->ptr = ptr; 1502 1370 1503 1371 state->active_locks++; 1504 1372 return 0; 1505 1373 } 1506 1374 1507 1507 - /* release function corresponding to acquire_reference_state(). Idempotent. */ 1508 1508 - static int release_reference_state(struct bpf_func_state *state, int ptr_id) 1375 1375 + static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx) 1509 1376 { 1510 1510 - int i, last_idx; 1377 1377 + struct bpf_verifier_state *state = env->cur_state; 1378 1378 + struct bpf_reference_state *s; 1511 1379 1512 1512 - last_idx = state->acquired_refs - 1; 1513 1513 - for (i = 0; i < state->acquired_refs; i++) { 1514 1514 - if (state->refs[i].type != REF_TYPE_PTR) 1515 1515 - continue; 1516 1516 - if (state->refs[i].id == ptr_id) { 1517 1517 - if (last_idx && i != last_idx) 1518 1518 - memcpy(&state->refs[i], &state->refs[last_idx], 1519 1519 - sizeof(*state->refs)); 1520 1520 - memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 1521 1521 - state->acquired_refs--; 1522 1522 - return 0; 1523 1523 - } 1524 1524 - } 1525 1525 - return -EINVAL; 1380 1380 + s = acquire_reference_state(env, insn_idx); 1381 1381 + if (!s) 1382 1382 + return -ENOMEM; 1383 1383 + s->type = REF_TYPE_IRQ; 1384 1384 + s->id = ++env->id_gen; 1385 1385 + 1386 1386 + state->active_irq_id = s->id; 1387 1387 + return s->id; 1526 1388 } 1527 1389 1528 1528 - static int release_lock_state(struct bpf_func_state *state, int type, int id, void *ptr) 1390 1390 + static void release_reference_state(struct bpf_verifier_state *state, int idx) 1529 1391 { 1530 1530 - int i, last_idx; 1392 1392 + int last_idx; 1393 1393 + size_t rem; 1531 1394 1395 1395 + /* IRQ state requires the relative ordering of elements remaining the 1396 1396 + * same, since it relies on the refs array to behave as a stack, so that 1397 1397 + * it can detect out-of-order IRQ restore. Hence use memmove to shift 1398 1398 + * the array instead of swapping the final element into the deleted idx. 1399 1399 + */ 1532 1400 last_idx = state->acquired_refs - 1; 1401 1401 + rem = state->acquired_refs - idx - 1; 1402 1402 + if (last_idx && idx != last_idx) 1403 1403 + memmove(&state->refs[idx], &state->refs[idx + 1], sizeof(*state->refs) * rem); 1404 1404 + memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 1405 1405 + state->acquired_refs--; 1406 1406 + return; 1407 1407 + } 1408 1408 + 1409 1409 + static int release_lock_state(struct bpf_verifier_state *state, int type, int id, void *ptr) 1410 1410 + { 1411 1411 + int i; 1412 1412 + 1533 1413 for (i = 0; i < state->acquired_refs; i++) { 1534 1414 if (state->refs[i].type != type) 1535 1415 continue; 1536 1416 if (state->refs[i].id == id && state->refs[i].ptr == ptr) { 1537 1537 - if (last_idx && i != last_idx) 1538 1538 - memcpy(&state->refs[i], &state->refs[last_idx], 1539 1539 - sizeof(*state->refs)); 1540 1540 - memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 1541 1541 - state->acquired_refs--; 1417 1417 + release_reference_state(state, i); 1542 1418 state->active_locks--; 1543 1419 return 0; 1544 1420 } ··· 1559 1409 return -EINVAL; 1560 1410 } 1561 1411 1562 1562 - static struct bpf_reference_state *find_lock_state(struct bpf_verifier_env *env, enum ref_state_type type, 1412 1412 + static int release_irq_state(struct bpf_verifier_state *state, int id) 1413 1413 + { 1414 1414 + u32 prev_id = 0; 1415 1415 + int i; 1416 1416 + 1417 1417 + if (id != state->active_irq_id) 1418 1418 + return -EACCES; 1419 1419 + 1420 1420 + for (i = 0; i < state->acquired_refs; i++) { 1421 1421 + if (state->refs[i].type != REF_TYPE_IRQ) 1422 1422 + continue; 1423 1423 + if (state->refs[i].id == id) { 1424 1424 + release_reference_state(state, i); 1425 1425 + state->active_irq_id = prev_id; 1426 1426 + return 0; 1427 1427 + } else { 1428 1428 + prev_id = state->refs[i].id; 1429 1429 + } 1430 1430 + } 1431 1431 + return -EINVAL; 1432 1432 + } 1433 1433 + 1434 1434 + static struct bpf_reference_state *find_lock_state(struct bpf_verifier_state *state, enum ref_state_type type, 1563 1435 int id, void *ptr) 1564 1436 { 1565 1565 - struct bpf_func_state *state = cur_func(env); 1566 1437 int i; 1567 1438 1568 1439 for (i = 0; i < state->acquired_refs; i++) { 1569 1440 struct bpf_reference_state *s = &state->refs[i]; 1570 1441 1571 1571 - if (s->type == REF_TYPE_PTR || s->type != type) 1442 1442 + if (s->type != type) 1572 1443 continue; 1573 1444 1574 1445 if (s->id == id && s->ptr == ptr) ··· 1602 1431 { 1603 1432 if (!state) 1604 1433 return; 1605 1605 - kfree(state->refs); 1606 1434 kfree(state->stack); 1607 1435 kfree(state); 1608 1436 } ··· 1615 1445 free_func_state(state->frame[i]); 1616 1446 state->frame[i] = NULL; 1617 1447 } 1448 1448 + kfree(state->refs); 1618 1449 if (free_self) 1619 1450 kfree(state); 1620 1451 } ··· 1626 1455 static int copy_func_state(struct bpf_func_state *dst, 1627 1456 const struct bpf_func_state *src) 1628 1457 { 1629 1629 - int err; 1630 1630 - 1631 1631 - memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); 1632 1632 - err = copy_reference_state(dst, src); 1633 1633 - if (err) 1634 1634 - return err; 1458 1458 + memcpy(dst, src, offsetof(struct bpf_func_state, stack)); 1635 1459 return copy_stack_state(dst, src); 1636 1460 } 1637 1461 ··· 1643 1477 free_func_state(dst_state->frame[i]); 1644 1478 dst_state->frame[i] = NULL; 1645 1479 } 1480 1480 + err = copy_reference_state(dst_state, src); 1481 1481 + if (err) 1482 1482 + return err; 1646 1483 dst_state->speculative = src->speculative; 1647 1647 - dst_state->active_rcu_lock = src->active_rcu_lock; 1648 1648 - dst_state->active_preempt_lock = src->active_preempt_lock; 1649 1484 dst_state->in_sleepable = src->in_sleepable; 1650 1485 dst_state->curframe = src->curframe; 1651 1486 dst_state->branches = src->branches; ··· 3373 3206 return 0; 3374 3207 } 3375 3208 3376 3376 - static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 3209 3209 + static int mark_stack_slot_obj_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 3210 3210 + int spi, int nr_slots) 3377 3211 { 3378 3212 struct bpf_func_state *state = func(env, reg); 3379 3379 - int spi, ret; 3213 3213 + int err, i; 3214 3214 + 3215 3215 + for (i = 0; i < nr_slots; i++) { 3216 3216 + struct bpf_reg_state *st = &state->stack[spi - i].spilled_ptr; 3217 3217 + 3218 3218 + err = mark_reg_read(env, st, st->parent, REG_LIVE_READ64); 3219 3219 + if (err) 3220 3220 + return err; 3221 3221 + 3222 3222 + mark_stack_slot_scratched(env, spi - i); 3223 3223 + } 3224 3224 + return 0; 3225 3225 + } 3226 3226 + 3227 3227 + static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 3228 3228 + { 3229 3229 + int spi; 3380 3230 3381 3231 /* For CONST_PTR_TO_DYNPTR, it must have already been done by 3382 3232 * check_reg_arg in check_helper_call and mark_btf_func_reg_size in ··· 3408 3224 * bounds and spi is the first dynptr slot. Simply mark stack slot as 3409 3225 * read. 3410 3226 */ 3411 3411 - ret = mark_reg_read(env, &state->stack[spi].spilled_ptr, 3412 3412 - state->stack[spi].spilled_ptr.parent, REG_LIVE_READ64); 3413 3413 - if (ret) 3414 3414 - return ret; 3415 3415 - return mark_reg_read(env, &state->stack[spi - 1].spilled_ptr, 3416 3416 - state->stack[spi - 1].spilled_ptr.parent, REG_LIVE_READ64); 3227 3227 + return mark_stack_slot_obj_read(env, reg, spi, BPF_DYNPTR_NR_SLOTS); 3417 3228 } 3418 3229 3419 3230 static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 3420 3231 int spi, int nr_slots) 3421 3232 { 3422 3422 - struct bpf_func_state *state = func(env, reg); 3423 3423 - int err, i; 3233 3233 + return mark_stack_slot_obj_read(env, reg, spi, nr_slots); 3234 3234 + } 3424 3235 3425 3425 - for (i = 0; i < nr_slots; i++) { 3426 3426 - struct bpf_reg_state *st = &state->stack[spi - i].spilled_ptr; 3236 3236 + static int mark_irq_flag_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 3237 3237 + { 3238 3238 + int spi; 3427 3239 3428 3428 - err = mark_reg_read(env, st, st->parent, REG_LIVE_READ64); 3429 3429 - if (err) 3430 3430 - return err; 3431 3431 - 3432 3432 - mark_stack_slot_scratched(env, spi - i); 3433 3433 - } 3434 3434 - 3435 3435 - return 0; 3240 3240 + spi = irq_flag_get_spi(env, reg); 3241 3241 + if (spi < 0) 3242 3242 + return spi; 3243 3243 + return mark_stack_slot_obj_read(env, reg, spi, 1); 3436 3244 } 3437 3245 3438 3246 /* This function is supposed to be used by the following 32-bit optimization ··· 4679 4503 fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, 4680 4504 bt_frame_stack_mask(bt, fr)); 4681 4505 verbose(env, "stack=%s: ", env->tmp_str_buf); 4682 4682 - print_verifier_state(env, func, true); 4506 4506 + print_verifier_state(env, st, fr, true); 4683 4507 } 4684 4508 } 4685 4509 ··· 5304 5128 static int check_stack_range_initialized(struct bpf_verifier_env *env, 5305 5129 int regno, int off, int access_size, 5306 5130 bool zero_size_allowed, 5307 5307 - enum bpf_access_src type, 5131 5131 + enum bpf_access_type type, 5308 5132 struct bpf_call_arg_meta *meta); 5309 5133 5310 5134 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) ··· 5337 5161 /* Note that we pass a NULL meta, so raw access will not be permitted. 5338 5162 */ 5339 5163 err = check_stack_range_initialized(env, ptr_regno, off, size, 5340 5340 - false, ACCESS_DIRECT, NULL); 5164 5164 + false, BPF_READ, NULL); 5341 5165 if (err) 5342 5166 return err; 5343 5167 ··· 5677 5501 static bool in_rcu_cs(struct bpf_verifier_env *env) 5678 5502 { 5679 5503 return env->cur_state->active_rcu_lock || 5680 5680 - cur_func(env)->active_locks || 5504 5504 + env->cur_state->active_locks || 5681 5505 !in_sleepable(env); 5682 5506 } 5683 5507 5684 5508 /* Once GCC supports btf_type_tag the following mechanism will be replaced with tag check */ 5685 5509 BTF_SET_START(rcu_protected_types) 5510 5510 + #ifdef CONFIG_NET 5686 5511 BTF_ID(struct, prog_test_ref_kfunc) 5512 5512 + #endif 5687 5513 #ifdef CONFIG_CGROUPS 5688 5514 BTF_ID(struct, cgroup) 5689 5515 #endif ··· 5693 5515 BTF_ID(struct, bpf_cpumask) 5694 5516 #endif 5695 5517 BTF_ID(struct, task_struct) 5518 5518 + #ifdef CONFIG_CRYPTO 5696 5519 BTF_ID(struct, bpf_crypto_ctx) 5520 5520 + #endif 5697 5521 BTF_SET_END(rcu_protected_types) 5698 5522 5699 5523 static bool rcu_protected_object(const struct btf *btf, u32 btf_id) ··· 7191 7011 static int check_stack_access_within_bounds( 7192 7012 struct bpf_verifier_env *env, 7193 7013 int regno, int off, int access_size, 7194 7194 - enum bpf_access_src src, enum bpf_access_type type) 7014 7014 + enum bpf_access_type type) 7195 7015 { 7196 7016 struct bpf_reg_state *regs = cur_regs(env); 7197 7017 struct bpf_reg_state *reg = regs + regno; ··· 7200 7020 int err; 7201 7021 char *err_extra; 7202 7022 7203 7203 - if (src == ACCESS_HELPER) 7204 7204 - /* We don't know if helpers are reading or writing (or both). */ 7205 7205 - err_extra = " indirect access to"; 7206 7206 - else if (type == BPF_READ) 7023 7023 + if (type == BPF_READ) 7207 7024 err_extra = " read from"; 7208 7025 else 7209 7026 err_extra = " write to"; ··· 7418 7241 7419 7242 } else if (reg->type == PTR_TO_STACK) { 7420 7243 /* Basic bounds checks. */ 7421 7421 - err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); 7244 7244 + err = check_stack_access_within_bounds(env, regno, off, size, t); 7422 7245 if (err) 7423 7246 return err; 7424 7247 ··· 7638 7461 static int check_stack_range_initialized( 7639 7462 struct bpf_verifier_env *env, int regno, int off, 7640 7463 int access_size, bool zero_size_allowed, 7641 7641 - enum bpf_access_src type, struct bpf_call_arg_meta *meta) 7464 7464 + enum bpf_access_type type, struct bpf_call_arg_meta *meta) 7642 7465 { 7643 7466 struct bpf_reg_state *reg = reg_state(env, regno); 7644 7467 struct bpf_func_state *state = func(env, reg); 7645 7468 int err, min_off, max_off, i, j, slot, spi; 7646 7646 - char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; 7647 7647 - enum bpf_access_type bounds_check_type; 7648 7469 /* Some accesses can write anything into the stack, others are 7649 7470 * read-only. 7650 7471 */ ··· 7653 7478 return -EACCES; 7654 7479 } 7655 7480 7656 7656 - if (type == ACCESS_HELPER) { 7657 7657 - /* The bounds checks for writes are more permissive than for 7658 7658 - * reads. However, if raw_mode is not set, we'll do extra 7659 7659 - * checks below. 7660 7660 - */ 7661 7661 - bounds_check_type = BPF_WRITE; 7481 7481 + if (type == BPF_WRITE) 7662 7482 clobber = true; 7663 7663 - } else { 7664 7664 - bounds_check_type = BPF_READ; 7665 7665 - } 7666 7666 - err = check_stack_access_within_bounds(env, regno, off, access_size, 7667 7667 - type, bounds_check_type); 7483 7483 + 7484 7484 + err = check_stack_access_within_bounds(env, regno, off, access_size, type); 7668 7485 if (err) 7669 7486 return err; 7670 7487 ··· 7673 7506 char tn_buf[48]; 7674 7507 7675 7508 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 7676 7676 - verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", 7677 7677 - regno, err_extra, tn_buf); 7509 7509 + verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", 7510 7510 + regno, tn_buf); 7678 7511 return -EACCES; 7679 7512 } 7680 7513 /* Only initialized buffer on stack is allowed to be accessed ··· 7727 7560 slot = -i - 1; 7728 7561 spi = slot / BPF_REG_SIZE; 7729 7562 if (state->allocated_stack <= slot) { 7730 7730 - verbose(env, "verifier bug: allocated_stack too small"); 7563 7563 + verbose(env, "verifier bug: allocated_stack too small\n"); 7731 7564 return -EFAULT; 7732 7565 } 7733 7566 ··· 7755 7588 } 7756 7589 7757 7590 if (tnum_is_const(reg->var_off)) { 7758 7758 - verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", 7759 7759 - err_extra, regno, min_off, i - min_off, access_size); 7591 7591 + verbose(env, "invalid read from stack R%d off %d+%d size %d\n", 7592 7592 + regno, min_off, i - min_off, access_size); 7760 7593 } else { 7761 7594 char tn_buf[48]; 7762 7595 7763 7596 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 7764 7764 - verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", 7765 7765 - err_extra, regno, tn_buf, i - min_off, access_size); 7597 7597 + verbose(env, "invalid read from stack R%d var_off %s+%d size %d\n", 7598 7598 + regno, tn_buf, i - min_off, access_size); 7766 7599 } 7767 7600 return -EACCES; 7768 7601 mark: ··· 7837 7670 return check_stack_range_initialized( 7838 7671 env, 7839 7672 regno, reg->off, access_size, 7840 7840 - zero_size_allowed, ACCESS_HELPER, meta); 7673 7673 + zero_size_allowed, access_type, meta); 7841 7674 case PTR_TO_BTF_ID: 7842 7675 return check_ptr_to_btf_access(env, regs, regno, reg->off, 7843 7676 access_size, BPF_READ, -1); ··· 8002 7835 * Since only one bpf_spin_lock is allowed the checks are simpler than 8003 7836 * reg_is_refcounted() logic. The verifier needs to remember only 8004 7837 * one spin_lock instead of array of acquired_refs. 8005 8005 - * cur_func(env)->active_locks remembers which map value element or allocated 7838 7838 + * env->cur_state->active_locks remembers which map value element or allocated 8006 7839 * object got locked and clears it after bpf_spin_unlock. 8007 7840 */ 8008 7841 static int process_spin_lock(struct bpf_verifier_env *env, int regno, 8009 7842 bool is_lock) 8010 7843 { 8011 7844 struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno]; 7845 7845 + struct bpf_verifier_state *cur = env->cur_state; 8012 7846 bool is_const = tnum_is_const(reg->var_off); 8013 8013 - struct bpf_func_state *cur = cur_func(env); 8014 7847 u64 val = reg->var_off.value; 8015 7848 struct bpf_map *map = NULL; 8016 7849 struct btf *btf = NULL; ··· 8077 7910 return -EINVAL; 8078 7911 } 8079 7912 8080 8080 - if (release_lock_state(cur_func(env), REF_TYPE_LOCK, reg->id, ptr)) { 7913 7913 + if (release_lock_state(env->cur_state, REF_TYPE_LOCK, reg->id, ptr)) { 8081 7914 verbose(env, "bpf_spin_unlock of different lock\n"); 8082 7915 return -EINVAL; 8083 7916 } ··· 9149 8982 return 0; 9150 8983 } 9151 8984 8985 8985 + /* Returns constant key value if possible, else negative error */ 8986 8986 + static s64 get_constant_map_key(struct bpf_verifier_env *env, 8987 8987 + struct bpf_reg_state *key, 8988 8988 + u32 key_size) 8989 8989 + { 8990 8990 + struct bpf_func_state *state = func(env, key); 8991 8991 + struct bpf_reg_state *reg; 8992 8992 + int slot, spi, off; 8993 8993 + int spill_size = 0; 8994 8994 + int zero_size = 0; 8995 8995 + int stack_off; 8996 8996 + int i, err; 8997 8997 + u8 *stype; 8998 8998 + 8999 8999 + if (!env->bpf_capable) 9000 9000 + return -EOPNOTSUPP; 9001 9001 + if (key->type != PTR_TO_STACK) 9002 9002 + return -EOPNOTSUPP; 9003 9003 + if (!tnum_is_const(key->var_off)) 9004 9004 + return -EOPNOTSUPP; 9005 9005 + 9006 9006 + stack_off = key->off + key->var_off.value; 9007 9007 + slot = -stack_off - 1; 9008 9008 + spi = slot / BPF_REG_SIZE; 9009 9009 + off = slot % BPF_REG_SIZE; 9010 9010 + stype = state->stack[spi].slot_type; 9011 9011 + 9012 9012 + /* First handle precisely tracked STACK_ZERO */ 9013 9013 + for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--) 9014 9014 + zero_size++; 9015 9015 + if (zero_size >= key_size) 9016 9016 + return 0; 9017 9017 + 9018 9018 + /* Check that stack contains a scalar spill of expected size */ 9019 9019 + if (!is_spilled_scalar_reg(&state->stack[spi])) 9020 9020 + return -EOPNOTSUPP; 9021 9021 + for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--) 9022 9022 + spill_size++; 9023 9023 + if (spill_size != key_size) 9024 9024 + return -EOPNOTSUPP; 9025 9025 + 9026 9026 + reg = &state->stack[spi].spilled_ptr; 9027 9027 + if (!tnum_is_const(reg->var_off)) 9028 9028 + /* Stack value not statically known */ 9029 9029 + return -EOPNOTSUPP; 9030 9030 + 9031 9031 + /* We are relying on a constant value. So mark as precise 9032 9032 + * to prevent pruning on it. 9033 9033 + */ 9034 9034 + bt_set_frame_slot(&env->bt, key->frameno, spi); 9035 9035 + err = mark_chain_precision_batch(env); 9036 9036 + if (err < 0) 9037 9037 + return err; 9038 9038 + 9039 9039 + return reg->var_off.value; 9040 9040 + } 9041 9041 + 9152 9042 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 9153 9043 struct bpf_call_arg_meta *meta, 9154 9044 const struct bpf_func_proto *fn, ··· 9216 8992 enum bpf_arg_type arg_type = fn->arg_type[arg]; 9217 8993 enum bpf_reg_type type = reg->type; 9218 8994 u32 *arg_btf_id = NULL; 8995 8995 + u32 key_size; 9219 8996 int err = 0; 9220 8997 9221 8998 if (arg_type == ARG_DONTCARE) ··· 9350 9125 verbose(env, "invalid map_ptr to access map->key\n"); 9351 9126 return -EACCES; 9352 9127 } 9353 9353 - err = check_helper_mem_access(env, regno, meta->map_ptr->key_size, 9354 9354 - BPF_READ, false, NULL); 9128 9128 + key_size = meta->map_ptr->key_size; 9129 9129 + err = check_helper_mem_access(env, regno, key_size, BPF_READ, false, NULL); 9130 9130 + if (err) 9131 9131 + return err; 9132 9132 + meta->const_map_key = get_constant_map_key(env, reg, key_size); 9133 9133 + if (meta->const_map_key < 0 && meta->const_map_key != -EOPNOTSUPP) 9134 9134 + return meta->const_map_key; 9355 9135 break; 9356 9136 case ARG_PTR_TO_MAP_VALUE: 9357 9137 if (type_may_be_null(arg_type) && register_is_null(reg)) ··· 9888 9658 reg->range = AT_PKT_END; 9889 9659 } 9890 9660 9661 9661 + static int release_reference_nomark(struct bpf_verifier_state *state, int ref_obj_id) 9662 9662 + { 9663 9663 + int i; 9664 9664 + 9665 9665 + for (i = 0; i < state->acquired_refs; i++) { 9666 9666 + if (state->refs[i].type != REF_TYPE_PTR) 9667 9667 + continue; 9668 9668 + if (state->refs[i].id == ref_obj_id) { 9669 9669 + release_reference_state(state, i); 9670 9670 + return 0; 9671 9671 + } 9672 9672 + } 9673 9673 + return -EINVAL; 9674 9674 + } 9675 9675 + 9891 9676 /* The pointer with the specified id has released its reference to kernel 9892 9677 * resources. Identify all copies of the same pointer and clear the reference. 9678 9678 + * 9679 9679 + * This is the release function corresponding to acquire_reference(). Idempotent. 9893 9680 */ 9894 9894 - static int release_reference(struct bpf_verifier_env *env, 9895 9895 - int ref_obj_id) 9681 9681 + static int release_reference(struct bpf_verifier_env *env, int ref_obj_id) 9896 9682 { 9683 9683 + struct bpf_verifier_state *vstate = env->cur_state; 9897 9684 struct bpf_func_state *state; 9898 9685 struct bpf_reg_state *reg; 9899 9686 int err; 9900 9687 9901 9901 - err = release_reference_state(cur_func(env), ref_obj_id); 9688 9688 + err = release_reference_nomark(vstate, ref_obj_id); 9902 9689 if (err) 9903 9690 return err; 9904 9691 9905 9905 - bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 9692 9692 + bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 9906 9693 if (reg->ref_obj_id == ref_obj_id) 9907 9694 mark_reg_invalid(env, reg); 9908 9695 })); ··· 9993 9746 callsite, 9994 9747 state->curframe + 1 /* frameno within this callchain */, 9995 9748 subprog /* subprog number within this prog */); 9996 9996 - /* Transfer references to the callee */ 9997 9997 - err = copy_reference_state(callee, caller); 9998 9998 - err = err ?: set_callee_state_cb(env, caller, callee, callsite); 9749 9749 + err = set_callee_state_cb(env, caller, callee, callsite); 9999 9750 if (err) 10000 9751 goto err_out; 10001 9752 ··· 10223 9978 const char *sub_name = subprog_name(env, subprog); 10224 9979 10225 9980 /* Only global subprogs cannot be called with a lock held. */ 10226 10226 - if (cur_func(env)->active_locks) { 9981 9981 + if (env->cur_state->active_locks) { 10227 9982 verbose(env, "global function calls are not allowed while holding a lock,\n" 10228 9983 "use static function instead\n"); 10229 9984 return -EINVAL; 10230 9985 } 10231 9986 10232 9987 /* Only global subprogs cannot be called with preemption disabled. */ 10233 10233 - if (env->cur_state->active_preempt_lock) { 9988 9988 + if (env->cur_state->active_preempt_locks) { 10234 9989 verbose(env, "global function calls are not allowed with preemption disabled,\n" 9990 9990 + "use static function instead\n"); 9991 9991 + return -EINVAL; 9992 9992 + } 9993 9993 + 9994 9994 + if (env->cur_state->active_irq_id) { 9995 9995 + verbose(env, "global function calls are not allowed with IRQs disabled,\n" 10235 9996 "use static function instead\n"); 10236 9997 return -EINVAL; 10237 9998 } ··· 10278 10027 10279 10028 if (env->log.level & BPF_LOG_LEVEL) { 10280 10029 verbose(env, "caller:\n"); 10281 10281 - print_verifier_state(env, caller, true); 10030 10030 + print_verifier_state(env, state, caller->frameno, true); 10282 10031 verbose(env, "callee:\n"); 10283 10283 - print_verifier_state(env, state->frame[state->curframe], true); 10032 10032 + print_verifier_state(env, state, state->curframe, true); 10284 10033 } 10285 10034 10286 10035 return 0; ··· 10572 10321 caller->regs[BPF_REG_0] = *r0; 10573 10322 } 10574 10323 10575 10575 - /* Transfer references to the caller */ 10576 10576 - err = copy_reference_state(caller, callee); 10577 10577 - if (err) 10578 10578 - return err; 10579 10579 - 10580 10324 /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, 10581 10325 * there function call logic would reschedule callback visit. If iteration 10582 10326 * converges is_state_visited() would prune that visit eventually. ··· 10584 10338 10585 10339 if (env->log.level & BPF_LOG_LEVEL) { 10586 10340 verbose(env, "returning from callee:\n"); 10587 10587 - print_verifier_state(env, callee, true); 10341 10341 + print_verifier_state(env, state, callee->frameno, true); 10588 10342 verbose(env, "to caller at %d:\n", *insn_idx); 10589 10589 - print_verifier_state(env, caller, true); 10343 10343 + print_verifier_state(env, state, caller->frameno, true); 10590 10344 } 10591 10345 /* clear everything in the callee. In case of exceptional exits using 10592 10346 * bpf_throw, this will be done by copy_verifier_state for extra frames. */ ··· 10736 10490 10737 10491 static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit) 10738 10492 { 10739 10739 - struct bpf_func_state *state = cur_func(env); 10493 10493 + struct bpf_verifier_state *state = env->cur_state; 10740 10494 bool refs_lingering = false; 10741 10495 int i; 10742 10496 10743 10743 - if (!exception_exit && state->frameno) 10497 10497 + if (!exception_exit && cur_func(env)->frameno) 10744 10498 return 0; 10745 10499 10746 10500 for (i = 0; i < state->acquired_refs; i++) { ··· 10757 10511 { 10758 10512 int err; 10759 10513 10760 10760 - if (check_lock && cur_func(env)->active_locks) { 10514 10514 + if (check_lock && env->cur_state->active_locks) { 10761 10515 verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix); 10762 10516 return -EINVAL; 10763 10517 } ··· 10768 10522 return err; 10769 10523 } 10770 10524 10525 10525 + if (check_lock && env->cur_state->active_irq_id) { 10526 10526 + verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix); 10527 10527 + return -EINVAL; 10528 10528 + } 10529 10529 + 10771 10530 if (check_lock && env->cur_state->active_rcu_lock) { 10772 10531 verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix); 10773 10532 return -EINVAL; 10774 10533 } 10775 10534 10776 10776 - if (check_lock && env->cur_state->active_preempt_lock) { 10535 10535 + if (check_lock && env->cur_state->active_preempt_locks) { 10777 10536 verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix); 10778 10537 return -EINVAL; 10779 10538 } ··· 10880 10629 state->callback_subprogno == subprogno); 10881 10630 } 10882 10631 10632 10632 + /* Returns whether or not the given map type can potentially elide 10633 10633 + * lookup return value nullness check. This is possible if the key 10634 10634 + * is statically known. 10635 10635 + */ 10636 10636 + static bool can_elide_value_nullness(enum bpf_map_type type) 10637 10637 + { 10638 10638 + switch (type) { 10639 10639 + case BPF_MAP_TYPE_ARRAY: 10640 10640 + case BPF_MAP_TYPE_PERCPU_ARRAY: 10641 10641 + return true; 10642 10642 + default: 10643 10643 + return false; 10644 10644 + } 10645 10645 + } 10646 10646 + 10883 10647 static int get_helper_proto(struct bpf_verifier_env *env, int func_id, 10884 10648 const struct bpf_func_proto **ptr) 10885 10649 { ··· 10981 10715 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; 10982 10716 } 10983 10717 10984 10984 - if (env->cur_state->active_preempt_lock) { 10718 10718 + if (env->cur_state->active_preempt_locks) { 10985 10719 if (fn->might_sleep) { 10986 10720 verbose(env, "sleepable helper %s#%d in non-preemptible region\n", 10721 10721 + func_id_name(func_id), func_id); 10722 10722 + return -EINVAL; 10723 10723 + } 10724 10724 + 10725 10725 + if (in_sleepable(env) && is_storage_get_function(func_id)) 10726 10726 + env->insn_aux_data[insn_idx].storage_get_func_atomic = true; 10727 10727 + } 10728 10728 + 10729 10729 + if (env->cur_state->active_irq_id) { 10730 10730 + if (fn->might_sleep) { 10731 10731 + verbose(env, "sleepable helper %s#%d in IRQ-disabled region\n", 10987 10732 func_id_name(func_id), func_id); 10988 10733 return -EINVAL; 10989 10734 } ··· 11049 10772 struct bpf_func_state *state; 11050 10773 struct bpf_reg_state *reg; 11051 10774 11052 11052 - err = release_reference_state(cur_func(env), ref_obj_id); 10775 10775 + err = release_reference_nomark(env->cur_state, ref_obj_id); 11053 10776 if (!err) { 11054 10777 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 11055 10778 if (reg->ref_obj_id == ref_obj_id) { ··· 11261 10984 "kernel subsystem misconfigured verifier\n"); 11262 10985 return -EINVAL; 11263 10986 } 10987 10987 + 10988 10988 + if (func_id == BPF_FUNC_map_lookup_elem && 10989 10989 + can_elide_value_nullness(meta.map_ptr->map_type) && 10990 10990 + meta.const_map_key >= 0 && 10991 10991 + meta.const_map_key < meta.map_ptr->max_entries) 10992 10992 + ret_flag &= ~PTR_MAYBE_NULL; 10993 10993 + 11264 10994 regs[BPF_REG_0].map_ptr = meta.map_ptr; 11265 10995 regs[BPF_REG_0].map_uid = meta.map_uid; 11266 10996 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; 11267 11267 - if (!type_may_be_null(ret_type) && 10997 10997 + if (!type_may_be_null(ret_flag) && 11268 10998 btf_record_has_field(meta.map_ptr->record, BPF_SPIN_LOCK)) { 11269 10999 regs[BPF_REG_0].id = ++env->id_gen; 11270 11000 } ··· 11389 11105 /* For release_reference() */ 11390 11106 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 11391 11107 } else if (is_acquire_function(func_id, meta.map_ptr)) { 11392 11392 - int id = acquire_reference_state(env, insn_idx); 11108 11108 + int id = acquire_reference(env, insn_idx); 11393 11109 11394 11110 if (id < 0) 11395 11111 return id; ··· 11571 11287 return btf_param_match_suffix(btf, arg, "__str"); 11572 11288 } 11573 11289 11290 11290 + static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param *arg) 11291 11291 + { 11292 11292 + return btf_param_match_suffix(btf, arg, "__irq_flag"); 11293 11293 + } 11294 11294 + 11574 11295 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, 11575 11296 const struct btf_param *arg, 11576 11297 const char *name) ··· 11729 11440 KF_ARG_PTR_TO_CONST_STR, 11730 11441 KF_ARG_PTR_TO_MAP, 11731 11442 KF_ARG_PTR_TO_WORKQUEUE, 11443 11443 + KF_ARG_PTR_TO_IRQ_FLAG, 11732 11444 }; 11733 11445 11734 11446 enum special_kfunc_type { ··· 11761 11471 KF_bpf_iter_css_task_new, 11762 11472 KF_bpf_session_cookie, 11763 11473 KF_bpf_get_kmem_cache, 11474 11474 + KF_bpf_local_irq_save, 11475 11475 + KF_bpf_local_irq_restore, 11476 11476 + KF_bpf_iter_num_new, 11477 11477 + KF_bpf_iter_num_next, 11478 11478 + KF_bpf_iter_num_destroy, 11764 11479 }; 11765 11480 11766 11481 BTF_SET_START(special_kfunc_set) ··· 11781 11486 BTF_ID(func, bpf_rbtree_remove) 11782 11487 BTF_ID(func, bpf_rbtree_add_impl) 11783 11488 BTF_ID(func, bpf_rbtree_first) 11489 11489 + #ifdef CONFIG_NET 11784 11490 BTF_ID(func, bpf_dynptr_from_skb) 11785 11491 BTF_ID(func, bpf_dynptr_from_xdp) 11492 11492 + #endif 11786 11493 BTF_ID(func, bpf_dynptr_slice) 11787 11494 BTF_ID(func, bpf_dynptr_slice_rdwr) 11788 11495 BTF_ID(func, bpf_dynptr_clone) ··· 11812 11515 BTF_ID(func, bpf_rbtree_remove) 11813 11516 BTF_ID(func, bpf_rbtree_add_impl) 11814 11517 BTF_ID(func, bpf_rbtree_first) 11518 11518 + #ifdef CONFIG_NET 11815 11519 BTF_ID(func, bpf_dynptr_from_skb) 11816 11520 BTF_ID(func, bpf_dynptr_from_xdp) 11521 11521 + #else 11522 11522 + BTF_ID_UNUSED 11523 11523 + BTF_ID_UNUSED 11524 11524 + #endif 11817 11525 BTF_ID(func, bpf_dynptr_slice) 11818 11526 BTF_ID(func, bpf_dynptr_slice_rdwr) 11819 11527 BTF_ID(func, bpf_dynptr_clone) ··· 11839 11537 BTF_ID_UNUSED 11840 11538 #endif 11841 11539 BTF_ID(func, bpf_get_kmem_cache) 11540 11540 + BTF_ID(func, bpf_local_irq_save) 11541 11541 + BTF_ID(func, bpf_local_irq_restore) 11542 11542 + BTF_ID(func, bpf_iter_num_new) 11543 11543 + BTF_ID(func, bpf_iter_num_next) 11544 11544 + BTF_ID(func, bpf_iter_num_destroy) 11842 11545 11843 11546 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) 11844 11547 { ··· 11933 11626 11934 11627 if (is_kfunc_arg_wq(meta->btf, &args[argno])) 11935 11628 return KF_ARG_PTR_TO_WORKQUEUE; 11629 11629 + 11630 11630 + if (is_kfunc_arg_irq_flag(meta->btf, &args[argno])) 11631 11631 + return KF_ARG_PTR_TO_IRQ_FLAG; 11936 11632 11937 11633 if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { 11938 11634 if (!btf_type_is_struct(ref_t)) { ··· 12040 11730 return 0; 12041 11731 } 12042 11732 11733 11733 + static int process_irq_flag(struct bpf_verifier_env *env, int regno, 11734 11734 + struct bpf_kfunc_call_arg_meta *meta) 11735 11735 + { 11736 11736 + struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno]; 11737 11737 + bool irq_save; 11738 11738 + int err; 11739 11739 + 11740 11740 + if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_save]) { 11741 11741 + irq_save = true; 11742 11742 + } else if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_restore]) { 11743 11743 + irq_save = false; 11744 11744 + } else { 11745 11745 + verbose(env, "verifier internal error: unknown irq flags kfunc\n"); 11746 11746 + return -EFAULT; 11747 11747 + } 11748 11748 + 11749 11749 + if (irq_save) { 11750 11750 + if (!is_irq_flag_reg_valid_uninit(env, reg)) { 11751 11751 + verbose(env, "expected uninitialized irq flag as arg#%d\n", regno - 1); 11752 11752 + return -EINVAL; 11753 11753 + } 11754 11754 + 11755 11755 + err = check_mem_access(env, env->insn_idx, regno, 0, BPF_DW, BPF_WRITE, -1, false, false); 11756 11756 + if (err) 11757 11757 + return err; 11758 11758 + 11759 11759 + err = mark_stack_slot_irq_flag(env, meta, reg, env->insn_idx); 11760 11760 + if (err) 11761 11761 + return err; 11762 11762 + } else { 11763 11763 + err = is_irq_flag_reg_valid_init(env, reg); 11764 11764 + if (err) { 11765 11765 + verbose(env, "expected an initialized irq flag as arg#%d\n", regno - 1); 11766 11766 + return err; 11767 11767 + } 11768 11768 + 11769 11769 + err = mark_irq_flag_read(env, reg); 11770 11770 + if (err) 11771 11771 + return err; 11772 11772 + 11773 11773 + err = unmark_stack_slot_irq_flag(env, reg); 11774 11774 + if (err) 11775 11775 + return err; 11776 11776 + } 11777 11777 + return 0; 11778 11778 + } 11779 11779 + 11780 11780 + 12043 11781 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 12044 11782 { 12045 11783 struct btf_record *rec = reg_btf_record(reg); 12046 11784 12047 12047 - if (!cur_func(env)->active_locks) { 11785 11785 + if (!env->cur_state->active_locks) { 12048 11786 verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n"); 12049 11787 return -EFAULT; 12050 11788 } ··· 12111 11753 12112 11754 static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id) 12113 11755 { 12114 12114 - struct bpf_func_state *state, *unused; 11756 11756 + struct bpf_verifier_state *state = env->cur_state; 11757 11757 + struct bpf_func_state *unused; 12115 11758 struct bpf_reg_state *reg; 12116 11759 int i; 12117 12117 - 12118 12118 - state = cur_func(env); 12119 11760 12120 11761 if (!ref_obj_id) { 12121 11762 verbose(env, "verifier internal error: ref_obj_id is zero for " ··· 12205 11848 } 12206 11849 id = reg->id; 12207 11850 12208 12208 - if (!cur_func(env)->active_locks) 11851 11851 + if (!env->cur_state->active_locks) 12209 11852 return -EINVAL; 12210 12210 - s = find_lock_state(env, REF_TYPE_LOCK, id, ptr); 11853 11853 + s = find_lock_state(env->cur_state, REF_TYPE_LOCK, id, ptr); 12211 11854 if (!s) { 12212 11855 verbose(env, "held lock and object are not in the same allocation\n"); 12213 11856 return -EINVAL; ··· 12230 11873 btf_id == special_kfunc_list[KF_bpf_rbtree_first]; 12231 11874 } 12232 11875 11876 11876 + static bool is_bpf_iter_num_api_kfunc(u32 btf_id) 11877 11877 + { 11878 11878 + return btf_id == special_kfunc_list[KF_bpf_iter_num_new] || 11879 11879 + btf_id == special_kfunc_list[KF_bpf_iter_num_next] || 11880 11880 + btf_id == special_kfunc_list[KF_bpf_iter_num_destroy]; 11881 11881 + } 11882 11882 + 12233 11883 static bool is_bpf_graph_api_kfunc(u32 btf_id) 12234 11884 { 12235 11885 return is_bpf_list_api_kfunc(btf_id) || is_bpf_rbtree_api_kfunc(btf_id) || 12236 11886 btf_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]; 11887 11887 + } 11888 11888 + 11889 11889 + static bool kfunc_spin_allowed(u32 btf_id) 11890 11890 + { 11891 11891 + return is_bpf_graph_api_kfunc(btf_id) || is_bpf_iter_num_api_kfunc(btf_id); 12237 11892 } 12238 11893 12239 11894 static bool is_sync_callback_calling_kfunc(u32 btf_id) ··· 12676 12307 case KF_ARG_PTR_TO_REFCOUNTED_KPTR: 12677 12308 case KF_ARG_PTR_TO_CONST_STR: 12678 12309 case KF_ARG_PTR_TO_WORKQUEUE: 12310 12310 + case KF_ARG_PTR_TO_IRQ_FLAG: 12679 12311 break; 12680 12312 default: 12681 12313 WARN_ON_ONCE(1); ··· 12966 12596 if (ret < 0) 12967 12597 return ret; 12968 12598 break; 12599 12599 + case KF_ARG_PTR_TO_IRQ_FLAG: 12600 12600 + if (reg->type != PTR_TO_STACK) { 12601 12601 + verbose(env, "arg#%d doesn't point to an irq flag on stack\n", i); 12602 12602 + return -EINVAL; 12603 12603 + } 12604 12604 + ret = process_irq_flag(env, regno, meta); 12605 12605 + if (ret < 0) 12606 12606 + return ret; 12607 12607 + break; 12969 12608 } 12970 12609 } 12971 12610 ··· 13139 12760 return -EINVAL; 13140 12761 } 13141 12762 13142 13142 - if (env->cur_state->active_preempt_lock) { 12763 12763 + if (env->cur_state->active_preempt_locks) { 13143 12764 if (preempt_disable) { 13144 13144 - env->cur_state->active_preempt_lock++; 12765 12765 + env->cur_state->active_preempt_locks++; 13145 12766 } else if (preempt_enable) { 13146 13146 - env->cur_state->active_preempt_lock--; 12767 12767 + env->cur_state->active_preempt_locks--; 13147 12768 } else if (sleepable) { 13148 12769 verbose(env, "kernel func %s is sleepable within non-preemptible region\n", func_name); 13149 12770 return -EACCES; 13150 12771 } 13151 12772 } else if (preempt_disable) { 13152 13152 - env->cur_state->active_preempt_lock++; 12773 12773 + env->cur_state->active_preempt_locks++; 13153 12774 } else if (preempt_enable) { 13154 12775 verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name); 13155 12776 return -EINVAL; 12777 12777 + } 12778 12778 + 12779 12779 + if (env->cur_state->active_irq_id && sleepable) { 12780 12780 + verbose(env, "kernel func %s is sleepable within IRQ-disabled region\n", func_name); 12781 12781 + return -EACCES; 13156 12782 } 13157 12783 13158 12784 /* In case of release function, we get register number of refcounted ··· 13453 13069 } 13454 13070 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); 13455 13071 if (is_kfunc_acquire(&meta)) { 13456 13456 - int id = acquire_reference_state(env, insn_idx); 13072 13072 + int id = acquire_reference(env, insn_idx); 13457 13073 13458 13074 if (id < 0) 13459 13075 return id; ··· 14178 13794 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, 14179 13795 struct bpf_reg_state *src_reg) 14180 13796 { 14181 14181 - s32 smin_val = src_reg->s32_min_value; 14182 14182 - u32 umin_val = src_reg->u32_min_value; 14183 14183 - u32 umax_val = src_reg->u32_max_value; 13797 13797 + s32 *dst_smin = &dst_reg->s32_min_value; 13798 13798 + s32 *dst_smax = &dst_reg->s32_max_value; 13799 13799 + u32 *dst_umin = &dst_reg->u32_min_value; 13800 13800 + u32 *dst_umax = &dst_reg->u32_max_value; 13801 13801 + s32 tmp_prod[4]; 14184 13802 14185 14185 - if (smin_val < 0 || dst_reg->s32_min_value < 0) { 14186 14186 - /* Ain't nobody got time to multiply that sign */ 14187 14187 - __mark_reg32_unbounded(dst_reg); 14188 14188 - return; 14189 14189 - } 14190 14190 - /* Both values are positive, so we can work with unsigned and 14191 14191 - * copy the result to signed (unless it exceeds S32_MAX). 14192 14192 - */ 14193 14193 - if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { 14194 14194 - /* Potential overflow, we know nothing */ 14195 14195 - __mark_reg32_unbounded(dst_reg); 14196 14196 - return; 14197 14197 - } 14198 14198 - dst_reg->u32_min_value *= umin_val; 14199 14199 - dst_reg->u32_max_value *= umax_val; 14200 14200 - if (dst_reg->u32_max_value > S32_MAX) { 13803 13803 + if (check_mul_overflow(*dst_umax, src_reg->u32_max_value, dst_umax) || 13804 13804 + check_mul_overflow(*dst_umin, src_reg->u32_min_value, dst_umin)) { 14201 13805 /* Overflow possible, we know nothing */ 14202 14202 - dst_reg->s32_min_value = S32_MIN; 14203 14203 - dst_reg->s32_max_value = S32_MAX; 13806 13806 + *dst_umin = 0; 13807 13807 + *dst_umax = U32_MAX; 13808 13808 + } 13809 13809 + if (check_mul_overflow(*dst_smin, src_reg->s32_min_value, &tmp_prod[0]) || 13810 13810 + check_mul_overflow(*dst_smin, src_reg->s32_max_value, &tmp_prod[1]) || 13811 13811 + check_mul_overflow(*dst_smax, src_reg->s32_min_value, &tmp_prod[2]) || 13812 13812 + check_mul_overflow(*dst_smax, src_reg->s32_max_value, &tmp_prod[3])) { 13813 13813 + /* Overflow possible, we know nothing */ 13814 13814 + *dst_smin = S32_MIN; 13815 13815 + *dst_smax = S32_MAX; 14204 13816 } else { 14205 14205 - dst_reg->s32_min_value = dst_reg->u32_min_value; 14206 14206 - dst_reg->s32_max_value = dst_reg->u32_max_value; 13817 13817 + *dst_smin = min_array(tmp_prod, 4); 13818 13818 + *dst_smax = max_array(tmp_prod, 4); 14207 13819 } 14208 13820 } 14209 13821 14210 13822 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, 14211 13823 struct bpf_reg_state *src_reg) 14212 13824 { 14213 14213 - s64 smin_val = src_reg->smin_value; 14214 14214 - u64 umin_val = src_reg->umin_value; 14215 14215 - u64 umax_val = src_reg->umax_value; 13825 13825 + s64 *dst_smin = &dst_reg->smin_value; 13826 13826 + s64 *dst_smax = &dst_reg->smax_value; 13827 13827 + u64 *dst_umin = &dst_reg->umin_value; 13828 13828 + u64 *dst_umax = &dst_reg->umax_value; 13829 13829 + s64 tmp_prod[4]; 14216 13830 14217 14217 - if (smin_val < 0 || dst_reg->smin_value < 0) { 14218 14218 - /* Ain't nobody got time to multiply that sign */ 14219 14219 - __mark_reg64_unbounded(dst_reg); 14220 14220 - return; 14221 14221 - } 14222 14222 - /* Both values are positive, so we can work with unsigned and 14223 14223 - * copy the result to signed (unless it exceeds S64_MAX). 14224 14224 - */ 14225 14225 - if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { 14226 14226 - /* Potential overflow, we know nothing */ 14227 14227 - __mark_reg64_unbounded(dst_reg); 14228 14228 - return; 14229 14229 - } 14230 14230 - dst_reg->umin_value *= umin_val; 14231 14231 - dst_reg->umax_value *= umax_val; 14232 14232 - if (dst_reg->umax_value > S64_MAX) { 13831 13831 + if (check_mul_overflow(*dst_umax, src_reg->umax_value, dst_umax) || 13832 13832 + check_mul_overflow(*dst_umin, src_reg->umin_value, dst_umin)) { 14233 13833 /* Overflow possible, we know nothing */ 14234 14234 - dst_reg->smin_value = S64_MIN; 14235 14235 - dst_reg->smax_value = S64_MAX; 13834 13834 + *dst_umin = 0; 13835 13835 + *dst_umax = U64_MAX; 13836 13836 + } 13837 13837 + if (check_mul_overflow(*dst_smin, src_reg->smin_value, &tmp_prod[0]) || 13838 13838 + check_mul_overflow(*dst_smin, src_reg->smax_value, &tmp_prod[1]) || 13839 13839 + check_mul_overflow(*dst_smax, src_reg->smin_value, &tmp_prod[2]) || 13840 13840 + check_mul_overflow(*dst_smax, src_reg->smax_value, &tmp_prod[3])) { 13841 13841 + /* Overflow possible, we know nothing */ 13842 13842 + *dst_smin = S64_MIN; 13843 13843 + *dst_smax = S64_MAX; 14236 13844 } else { 14237 14237 - dst_reg->smin_value = dst_reg->umin_value; 14238 14238 - dst_reg->smax_value = dst_reg->umax_value; 13845 13845 + *dst_smin = min_array(tmp_prod, 4); 13846 13846 + *dst_smax = max_array(tmp_prod, 4); 14239 13847 } 14240 13848 } 14241 13849 ··· 14838 14462 14839 14463 /* Got here implies adding two SCALAR_VALUEs */ 14840 14464 if (WARN_ON_ONCE(ptr_reg)) { 14841 14841 - print_verifier_state(env, state, true); 14465 14465 + print_verifier_state(env, vstate, vstate->curframe, true); 14842 14466 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 14843 14467 return -EINVAL; 14844 14468 } 14845 14469 if (WARN_ON(!src_reg)) { 14846 14846 - print_verifier_state(env, state, true); 14470 14470 + print_verifier_state(env, vstate, vstate->curframe, true); 14847 14471 verbose(env, "verifier internal error: no src_reg\n"); 14848 14472 return -EINVAL; 14849 14473 } ··· 15741 15365 * No one could have freed the reference state before 15742 15366 * doing the NULL check. 15743 15367 */ 15744 15744 - WARN_ON_ONCE(release_reference_state(state, id)); 15368 15368 + WARN_ON_ONCE(release_reference_nomark(vstate, id)); 15745 15369 15746 15370 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 15747 15371 mark_ptr_or_null_reg(state, reg, id, is_null); ··· 15972 15596 15973 15597 if (insn->code != (BPF_JMP | BPF_JCOND) || 15974 15598 insn->src_reg != BPF_MAY_GOTO || 15975 15975 - insn->dst_reg || insn->imm || insn->off == 0) { 15976 15976 - verbose(env, "invalid may_goto off %d imm %d\n", 15977 15977 - insn->off, insn->imm); 15599 15599 + insn->dst_reg || insn->imm) { 15600 15600 + verbose(env, "invalid may_goto imm %d\n", insn->imm); 15978 15601 return -EINVAL; 15979 15602 } 15980 15603 prev_st = find_prev_entry(env, cur_st->parent, idx); ··· 16050 15675 *insn_idx)) 16051 15676 return -EFAULT; 16052 15677 if (env->log.level & BPF_LOG_LEVEL) 16053 16053 - print_insn_state(env, this_branch->frame[this_branch->curframe]); 15678 15678 + print_insn_state(env, this_branch, this_branch->curframe); 16054 15679 *insn_idx += insn->off; 16055 15680 return 0; 16056 15681 } else if (pred == 0) { ··· 16064 15689 *insn_idx)) 16065 15690 return -EFAULT; 16066 15691 if (env->log.level & BPF_LOG_LEVEL) 16067 16067 - print_insn_state(env, this_branch->frame[this_branch->curframe]); 15692 15692 + print_insn_state(env, this_branch, this_branch->curframe); 16068 15693 return 0; 16069 15694 } 16070 15695 ··· 16181 15806 return -EACCES; 16182 15807 } 16183 15808 if (env->log.level & BPF_LOG_LEVEL) 16184 16184 - print_insn_state(env, this_branch->frame[this_branch->curframe]); 15809 15809 + print_insn_state(env, this_branch, this_branch->curframe); 16185 15810 return 0; 16186 15811 } 16187 15812 ··· 18109 17734 !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap)) 18110 17735 return false; 18111 17736 break; 17737 17737 + case STACK_IRQ_FLAG: 17738 17738 + old_reg = &old->stack[spi].spilled_ptr; 17739 17739 + cur_reg = &cur->stack[spi].spilled_ptr; 17740 17740 + if (!check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap)) 17741 17741 + return false; 17742 17742 + break; 18112 17743 case STACK_MISC: 18113 17744 case STACK_ZERO: 18114 17745 case STACK_INVALID: ··· 18127 17746 return true; 18128 17747 } 18129 17748 18130 18130 - static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur, 17749 17749 + static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *cur, 18131 17750 struct bpf_idmap *idmap) 18132 17751 { 18133 17752 int i; 18134 17753 18135 17754 if (old->acquired_refs != cur->acquired_refs) 17755 17755 + return false; 17756 17756 + 17757 17757 + if (old->active_locks != cur->active_locks) 17758 17758 + return false; 17759 17759 + 17760 17760 + if (old->active_preempt_locks != cur->active_preempt_locks) 17761 17761 + return false; 17762 17762 + 17763 17763 + if (old->active_rcu_lock != cur->active_rcu_lock) 17764 17764 + return false; 17765 17765 + 17766 17766 + if (!check_ids(old->active_irq_id, cur->active_irq_id, idmap)) 18136 17767 return false; 18137 17768 18138 17769 for (i = 0; i < old->acquired_refs; i++) { ··· 18153 17760 return false; 18154 17761 switch (old->refs[i].type) { 18155 17762 case REF_TYPE_PTR: 17763 17763 + case REF_TYPE_IRQ: 18156 17764 break; 18157 17765 case REF_TYPE_LOCK: 18158 17766 if (old->refs[i].ptr != cur->refs[i].ptr) ··· 18210 17816 if (!stacksafe(env, old, cur, &env->idmap_scratch, exact)) 18211 17817 return false; 18212 17818 18213 18213 - if (!refsafe(old, cur, &env->idmap_scratch)) 18214 18214 - return false; 18215 18215 - 18216 17819 return true; 18217 17820 } 18218 17821 ··· 18237 17846 if (old->speculative && !cur->speculative) 18238 17847 return false; 18239 17848 18240 18240 - if (old->active_rcu_lock != cur->active_rcu_lock) 18241 18241 - return false; 18242 18242 - 18243 18243 - if (old->active_preempt_lock != cur->active_preempt_lock) 18244 18244 - return false; 18245 18245 - 18246 17849 if (old->in_sleepable != cur->in_sleepable) 17850 17850 + return false; 17851 17851 + 17852 17852 + if (!refsafe(old, cur, &env->idmap_scratch)) 18247 17853 return false; 18248 17854 18249 17855 /* for states to be equal callsites have to be the same ··· 18633 18245 verbose_linfo(env, insn_idx, "; "); 18634 18246 verbose(env, "infinite loop detected at insn %d\n", insn_idx); 18635 18247 verbose(env, "cur state:"); 18636 18636 - print_verifier_state(env, cur->frame[cur->curframe], true); 18248 18248 + print_verifier_state(env, cur, cur->curframe, true); 18637 18249 verbose(env, "old state:"); 18638 18638 - print_verifier_state(env, sl->state.frame[cur->curframe], true); 18250 18250 + print_verifier_state(env, &sl->state, cur->curframe, true); 18639 18251 return -EINVAL; 18640 18252 } 18641 18253 /* if the verifier is processing a loop, avoid adding new state ··· 18991 18603 env->prev_insn_idx, env->insn_idx, 18992 18604 env->cur_state->speculative ? 18993 18605 " (speculative execution)" : ""); 18994 18994 - print_verifier_state(env, state->frame[state->curframe], true); 18606 18606 + print_verifier_state(env, state, state->curframe, true); 18995 18607 do_print_state = false; 18996 18608 } 18997 18609 ··· 19003 18615 }; 19004 18616 19005 18617 if (verifier_state_scratched(env)) 19006 19006 - print_insn_state(env, state->frame[state->curframe]); 18618 18618 + print_insn_state(env, state, state->curframe); 19007 18619 19008 18620 verbose_linfo(env, env->insn_idx, "; "); 19009 18621 env->prev_log_pos = env->log.end_pos; ··· 19135 18747 return -EINVAL; 19136 18748 } 19137 18749 19138 19138 - if (cur_func(env)->active_locks) { 18750 18750 + if (env->cur_state->active_locks) { 19139 18751 if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) || 19140 18752 (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && 19141 19141 - (insn->off != 0 || !is_bpf_graph_api_kfunc(insn->imm)))) { 18753 18753 + (insn->off != 0 || !kfunc_spin_allowed(insn->imm)))) { 19142 18754 verbose(env, "function calls are not allowed while holding a lock\n"); 19143 18755 return -EINVAL; 19144 18756 } ··· 19191 18803 * match caller reference state when it exits. 19192 18804 */ 19193 18805 err = check_resource_leak(env, exception_exit, !env->cur_state->curframe, 19194 19194 - "BPF_EXIT instruction"); 18806 18806 + "BPF_EXIT instruction in main prog"); 19195 18807 if (err) 19196 18808 return err; 19197 18809 ··· 19298 18910 return -ENOENT; 19299 18911 } 19300 18912 18913 18913 + /* 18914 18914 + * Add btf to the used_btfs array and return the index. (If the btf was 18915 18915 + * already added, then just return the index.) Upon successful insertion 18916 18916 + * increase btf refcnt, and, if present, also refcount the corresponding 18917 18917 + * kernel module. 18918 18918 + */ 18919 18919 + static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) 18920 18920 + { 18921 18921 + struct btf_mod_pair *btf_mod; 18922 18922 + int i; 18923 18923 + 18924 18924 + /* check whether we recorded this BTF (and maybe module) already */ 18925 18925 + for (i = 0; i < env->used_btf_cnt; i++) 18926 18926 + if (env->used_btfs[i].btf == btf) 18927 18927 + return i; 18928 18928 + 18929 18929 + if (env->used_btf_cnt >= MAX_USED_BTFS) 18930 18930 + return -E2BIG; 18931 18931 + 18932 18932 + btf_get(btf); 18933 18933 + 18934 18934 + btf_mod = &env->used_btfs[env->used_btf_cnt]; 18935 18935 + btf_mod->btf = btf; 18936 18936 + btf_mod->module = NULL; 18937 18937 + 18938 18938 + /* if we reference variables from kernel module, bump its refcount */ 18939 18939 + if (btf_is_module(btf)) { 18940 18940 + btf_mod->module = btf_try_get_module(btf); 18941 18941 + if (!btf_mod->module) { 18942 18942 + btf_put(btf); 18943 18943 + return -ENXIO; 18944 18944 + } 18945 18945 + } 18946 18946 + 18947 18947 + return env->used_btf_cnt++; 18948 18948 + } 18949 18949 + 19301 18950 /* replace pseudo btf_id with kernel symbol address */ 19302 19302 - static int check_pseudo_btf_id(struct bpf_verifier_env *env, 19303 19303 - struct bpf_insn *insn, 19304 19304 - struct bpf_insn_aux_data *aux) 18951 18951 + static int __check_pseudo_btf_id(struct bpf_verifier_env *env, 18952 18952 + struct bpf_insn *insn, 18953 18953 + struct bpf_insn_aux_data *aux, 18954 18954 + struct btf *btf) 19305 18955 { 19306 18956 const struct btf_var_secinfo *vsi; 19307 18957 const struct btf_type *datasec; 19308 19308 - struct btf_mod_pair *btf_mod; 19309 18958 const struct btf_type *t; 19310 18959 const char *sym_name; 19311 18960 bool percpu = false; 19312 18961 u32 type, id = insn->imm; 19313 19313 - struct btf *btf; 19314 18962 s32 datasec_id; 19315 18963 u64 addr; 19316 19316 - int i, btf_fd, err; 19317 19317 - 19318 19318 - btf_fd = insn[1].imm; 19319 19319 - if (btf_fd) { 19320 19320 - btf = btf_get_by_fd(btf_fd); 19321 19321 - if (IS_ERR(btf)) { 19322 19322 - verbose(env, "invalid module BTF object FD specified.\n"); 19323 19323 - return -EINVAL; 19324 19324 - } 19325 19325 - } else { 19326 19326 - if (!btf_vmlinux) { 19327 19327 - verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 19328 19328 - return -EINVAL; 19329 19329 - } 19330 19330 - btf = btf_vmlinux; 19331 19331 - btf_get(btf); 19332 19332 - } 18964 18964 + int i; 19333 18965 19334 18966 t = btf_type_by_id(btf, id); 19335 18967 if (!t) { 19336 18968 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); 19337 19337 - err = -ENOENT; 19338 19338 - goto err_put; 18969 18969 + return -ENOENT; 19339 18970 } 19340 18971 19341 18972 if (!btf_type_is_var(t) && !btf_type_is_func(t)) { 19342 18973 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id); 19343 19343 - err = -EINVAL; 19344 19344 - goto err_put; 18974 18974 + return -EINVAL; 19345 18975 } 19346 18976 19347 18977 sym_name = btf_name_by_offset(btf, t->name_off); ··· 19367 18961 if (!addr) { 19368 18962 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", 19369 18963 sym_name); 19370 19370 - err = -ENOENT; 19371 19371 - goto err_put; 18964 18964 + return -ENOENT; 19372 18965 } 19373 18966 insn[0].imm = (u32)addr; 19374 18967 insn[1].imm = addr >> 32; ··· 19375 18970 if (btf_type_is_func(t)) { 19376 18971 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 19377 18972 aux->btf_var.mem_size = 0; 19378 19378 - goto check_btf; 18973 18973 + return 0; 19379 18974 } 19380 18975 19381 18976 datasec_id = find_btf_percpu_datasec(btf); ··· 19406 19001 tname = btf_name_by_offset(btf, t->name_off); 19407 19002 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", 19408 19003 tname, PTR_ERR(ret)); 19409 19409 - err = -EINVAL; 19410 19410 - goto err_put; 19004 19004 + return -EINVAL; 19411 19005 } 19412 19006 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 19413 19007 aux->btf_var.mem_size = tsize; ··· 19415 19011 aux->btf_var.btf = btf; 19416 19012 aux->btf_var.btf_id = type; 19417 19013 } 19418 19418 - check_btf: 19419 19419 - /* check whether we recorded this BTF (and maybe module) already */ 19420 19420 - for (i = 0; i < env->used_btf_cnt; i++) { 19421 19421 - if (env->used_btfs[i].btf == btf) { 19422 19422 - btf_put(btf); 19423 19423 - return 0; 19424 19424 - } 19425 19425 - } 19426 19426 - 19427 19427 - if (env->used_btf_cnt >= MAX_USED_BTFS) { 19428 19428 - err = -E2BIG; 19429 19429 - goto err_put; 19430 19430 - } 19431 19431 - 19432 19432 - btf_mod = &env->used_btfs[env->used_btf_cnt]; 19433 19433 - btf_mod->btf = btf; 19434 19434 - btf_mod->module = NULL; 19435 19435 - 19436 19436 - /* if we reference variables from kernel module, bump its refcount */ 19437 19437 - if (btf_is_module(btf)) { 19438 19438 - btf_mod->module = btf_try_get_module(btf); 19439 19439 - if (!btf_mod->module) { 19440 19440 - err = -ENXIO; 19441 19441 - goto err_put; 19442 19442 - } 19443 19443 - } 19444 19444 - 19445 19445 - env->used_btf_cnt++; 19446 19014 19447 19015 return 0; 19448 19448 - err_put: 19449 19449 - btf_put(btf); 19450 19450 - return err; 19016 19016 + } 19017 19017 + 19018 19018 + static int check_pseudo_btf_id(struct bpf_verifier_env *env, 19019 19019 + struct bpf_insn *insn, 19020 19020 + struct bpf_insn_aux_data *aux) 19021 19021 + { 19022 19022 + struct btf *btf; 19023 19023 + int btf_fd; 19024 19024 + int err; 19025 19025 + 19026 19026 + btf_fd = insn[1].imm; 19027 19027 + if (btf_fd) { 19028 19028 + CLASS(fd, f)(btf_fd); 19029 19029 + 19030 19030 + btf = __btf_get_by_fd(f); 19031 19031 + if (IS_ERR(btf)) { 19032 19032 + verbose(env, "invalid module BTF object FD specified.\n"); 19033 19033 + return -EINVAL; 19034 19034 + } 19035 19035 + } else { 19036 19036 + if (!btf_vmlinux) { 19037 19037 + verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 19038 19038 + return -EINVAL; 19039 19039 + } 19040 19040 + btf = btf_vmlinux; 19041 19041 + } 19042 19042 + 19043 19043 + err = __check_pseudo_btf_id(env, insn, aux, btf); 19044 19044 + if (err) 19045 19045 + return err; 19046 19046 + 19047 19047 + err = __add_used_btf(env, btf); 19048 19048 + if (err < 0) 19049 19049 + return err; 19050 19050 + return 0; 19451 19051 } 19452 19052 19453 19053 static bool is_tracing_prog_type(enum bpf_prog_type type) ··· 19466 19058 default: 19467 19059 return false; 19468 19060 } 19061 19061 + } 19062 19062 + 19063 19063 + static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 19064 19064 + { 19065 19065 + return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 19066 19066 + map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 19469 19067 } 19470 19068 19471 19069 static int check_map_prog_compatibility(struct bpf_verifier_env *env, ··· 19552 19138 return -EINVAL; 19553 19139 } 19554 19140 19141 19141 + if (bpf_map_is_cgroup_storage(map) && 19142 19142 + bpf_cgroup_storage_assign(env->prog->aux, map)) { 19143 19143 + verbose(env, "only one cgroup storage of each type is allowed\n"); 19144 19144 + return -EBUSY; 19145 19145 + } 19146 19146 + 19147 19147 + if (map->map_type == BPF_MAP_TYPE_ARENA) { 19148 19148 + if (env->prog->aux->arena) { 19149 19149 + verbose(env, "Only one arena per program\n"); 19150 19150 + return -EBUSY; 19151 19151 + } 19152 19152 + if (!env->allow_ptr_leaks || !env->bpf_capable) { 19153 19153 + verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n"); 19154 19154 + return -EPERM; 19155 19155 + } 19156 19156 + if (!env->prog->jit_requested) { 19157 19157 + verbose(env, "JIT is required to use arena\n"); 19158 19158 + return -EOPNOTSUPP; 19159 19159 + } 19160 19160 + if (!bpf_jit_supports_arena()) { 19161 19161 + verbose(env, "JIT doesn't support arena\n"); 19162 19162 + return -EOPNOTSUPP; 19163 19163 + } 19164 19164 + env->prog->aux->arena = (void *)map; 19165 19165 + if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { 19166 19166 + verbose(env, "arena's user address must be set via map_extra or mmap()\n"); 19167 19167 + return -EINVAL; 19168 19168 + } 19169 19169 + } 19170 19170 + 19555 19171 return 0; 19556 19172 } 19557 19173 19558 19558 - static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 19174 19174 + static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) 19559 19175 { 19560 19560 - return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 19561 19561 - map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 19562 19562 - } 19563 19563 - 19564 19564 - /* Add map behind fd to used maps list, if it's not already there, and return 19565 19565 - * its index. Also set *reused to true if this map was already in the list of 19566 19566 - * used maps. 19567 19567 - * Returns <0 on error, or >= 0 index, on success. 19568 19568 - */ 19569 19569 - static int add_used_map_from_fd(struct bpf_verifier_env *env, int fd, bool *reused) 19570 19570 - { 19571 19571 - CLASS(fd, f)(fd); 19572 19572 - struct bpf_map *map; 19573 19573 - int i; 19574 19574 - 19575 19575 - map = __bpf_map_get(f); 19576 19576 - if (IS_ERR(map)) { 19577 19577 - verbose(env, "fd %d is not pointing to valid bpf_map\n", fd); 19578 19578 - return PTR_ERR(map); 19579 19579 - } 19176 19176 + int i, err; 19580 19177 19581 19178 /* check whether we recorded this map already */ 19582 19582 - for (i = 0; i < env->used_map_cnt; i++) { 19583 19583 - if (env->used_maps[i] == map) { 19584 19584 - *reused = true; 19179 19179 + for (i = 0; i < env->used_map_cnt; i++) 19180 19180 + if (env->used_maps[i] == map) 19585 19181 return i; 19586 19586 - } 19587 19587 - } 19588 19182 19589 19183 if (env->used_map_cnt >= MAX_USED_MAPS) { 19590 19184 verbose(env, "The total number of maps per program has reached the limit of %u\n", 19591 19185 MAX_USED_MAPS); 19592 19186 return -E2BIG; 19593 19187 } 19188 19188 + 19189 19189 + err = check_map_prog_compatibility(env, map, env->prog); 19190 19190 + if (err) 19191 19191 + return err; 19594 19192 19595 19193 if (env->prog->sleepable) 19596 19194 atomic64_inc(&map->sleepable_refcnt); ··· 19614 19188 */ 19615 19189 bpf_map_inc(map); 19616 19190 19617 19617 - *reused = false; 19618 19191 env->used_maps[env->used_map_cnt++] = map; 19619 19192 19620 19193 return env->used_map_cnt - 1; 19194 19194 + } 19195 19195 + 19196 19196 + /* Add map behind fd to used maps list, if it's not already there, and return 19197 19197 + * its index. 19198 19198 + * Returns <0 on error, or >= 0 index, on success. 19199 19199 + */ 19200 19200 + static int add_used_map(struct bpf_verifier_env *env, int fd) 19201 19201 + { 19202 19202 + struct bpf_map *map; 19203 19203 + CLASS(fd, f)(fd); 19204 19204 + 19205 19205 + map = __bpf_map_get(f); 19206 19206 + if (IS_ERR(map)) { 19207 19207 + verbose(env, "fd %d is not pointing to valid bpf_map\n", fd); 19208 19208 + return PTR_ERR(map); 19209 19209 + } 19210 19210 + 19211 19211 + return __add_used_map(env, map); 19621 19212 } 19622 19213 19623 19214 /* find and rewrite pseudo imm in ld_imm64 instructions: ··· 19668 19225 int map_idx; 19669 19226 u64 addr; 19670 19227 u32 fd; 19671 19671 - bool reused; 19672 19228 19673 19229 if (i == insn_cnt - 1 || insn[1].code != 0 || 19674 19230 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || ··· 19728 19286 break; 19729 19287 } 19730 19288 19731 19731 - map_idx = add_used_map_from_fd(env, fd, &reused); 19289 19289 + map_idx = add_used_map(env, fd); 19732 19290 if (map_idx < 0) 19733 19291 return map_idx; 19734 19292 map = env->used_maps[map_idx]; 19735 19293 19736 19294 aux = &env->insn_aux_data[i]; 19737 19295 aux->map_index = map_idx; 19738 19738 - 19739 19739 - err = check_map_prog_compatibility(env, map, env->prog); 19740 19740 - if (err) 19741 19741 - return err; 19742 19296 19743 19297 if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || 19744 19298 insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { ··· 19765 19327 19766 19328 insn[0].imm = (u32)addr; 19767 19329 insn[1].imm = addr >> 32; 19768 19768 - 19769 19769 - /* proceed with extra checks only if its newly added used map */ 19770 19770 - if (reused) 19771 19771 - goto next_insn; 19772 19772 - 19773 19773 - if (bpf_map_is_cgroup_storage(map) && 19774 19774 - bpf_cgroup_storage_assign(env->prog->aux, map)) { 19775 19775 - verbose(env, "only one cgroup storage of each type is allowed\n"); 19776 19776 - return -EBUSY; 19777 19777 - } 19778 19778 - if (map->map_type == BPF_MAP_TYPE_ARENA) { 19779 19779 - if (env->prog->aux->arena) { 19780 19780 - verbose(env, "Only one arena per program\n"); 19781 19781 - return -EBUSY; 19782 19782 - } 19783 19783 - if (!env->allow_ptr_leaks || !env->bpf_capable) { 19784 19784 - verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n"); 19785 19785 - return -EPERM; 19786 19786 - } 19787 19787 - if (!env->prog->jit_requested) { 19788 19788 - verbose(env, "JIT is required to use arena\n"); 19789 19789 - return -EOPNOTSUPP; 19790 19790 - } 19791 19791 - if (!bpf_jit_supports_arena()) { 19792 19792 - verbose(env, "JIT doesn't support arena\n"); 19793 19793 - return -EOPNOTSUPP; 19794 19794 - } 19795 19795 - env->prog->aux->arena = (void *)map; 19796 19796 - if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { 19797 19797 - verbose(env, "arena's user address must be set via map_extra or mmap()\n"); 19798 19798 - return -EINVAL; 19799 19799 - } 19800 19800 - } 19801 19330 19802 19331 next_insn: 19803 19332 insn++; ··· 20184 19779 } 20185 19780 20186 19781 static const struct bpf_insn NOP = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 19782 19782 + static const struct bpf_insn MAY_GOTO_0 = BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0, 0); 20187 19783 20188 19784 static int opt_remove_nops(struct bpf_verifier_env *env) 20189 19785 { 20190 20190 - const struct bpf_insn ja = NOP; 20191 19786 struct bpf_insn *insn = env->prog->insnsi; 20192 19787 int insn_cnt = env->prog->len; 19788 19788 + bool is_may_goto_0, is_ja; 20193 19789 int i, err; 20194 19790 20195 19791 for (i = 0; i < insn_cnt; i++) { 20196 20196 - if (memcmp(&insn[i], &ja, sizeof(ja))) 19792 19792 + is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0)); 19793 19793 + is_ja = !memcmp(&insn[i], &NOP, sizeof(NOP)); 19794 19794 + 19795 19795 + if (!is_may_goto_0 && !is_ja) 20197 19796 continue; 20198 19797 20199 19798 err = verifier_remove_insns(env, i, 1); 20200 19799 if (err) 20201 19800 return err; 20202 19801 insn_cnt--; 20203 20203 - i--; 19802 19802 + /* Go back one insn to catch may_goto +1; may_goto +0 sequence */ 19803 19803 + i -= (is_may_goto_0 && i > 0) ? 2 : 1; 20204 19804 } 20205 19805 20206 19806 return 0; ··· 22954 22544 return btf_vmlinux; 22955 22545 } 22956 22546 22547 22547 + /* 22548 22548 + * The add_fd_from_fd_array() is executed only if fd_array_cnt is non-zero. In 22549 22549 + * this case expect that every file descriptor in the array is either a map or 22550 22550 + * a BTF. Everything else is considered to be trash. 22551 22551 + */ 22552 22552 + static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) 22553 22553 + { 22554 22554 + struct bpf_map *map; 22555 22555 + struct btf *btf; 22556 22556 + CLASS(fd, f)(fd); 22557 22557 + int err; 22558 22558 + 22559 22559 + map = __bpf_map_get(f); 22560 22560 + if (!IS_ERR(map)) { 22561 22561 + err = __add_used_map(env, map); 22562 22562 + if (err < 0) 22563 22563 + return err; 22564 22564 + return 0; 22565 22565 + } 22566 22566 + 22567 22567 + btf = __btf_get_by_fd(f); 22568 22568 + if (!IS_ERR(btf)) { 22569 22569 + err = __add_used_btf(env, btf); 22570 22570 + if (err < 0) 22571 22571 + return err; 22572 22572 + return 0; 22573 22573 + } 22574 22574 + 22575 22575 + verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); 22576 22576 + return PTR_ERR(map); 22577 22577 + } 22578 22578 + 22579 22579 + static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr) 22580 22580 + { 22581 22581 + size_t size = sizeof(int); 22582 22582 + int ret; 22583 22583 + int fd; 22584 22584 + u32 i; 22585 22585 + 22586 22586 + env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); 22587 22587 + 22588 22588 + /* 22589 22589 + * The only difference between old (no fd_array_cnt is given) and new 22590 22590 + * APIs is that in the latter case the fd_array is expected to be 22591 22591 + * continuous and is scanned for map fds right away 22592 22592 + */ 22593 22593 + if (!attr->fd_array_cnt) 22594 22594 + return 0; 22595 22595 + 22596 22596 + /* Check for integer overflow */ 22597 22597 + if (attr->fd_array_cnt >= (U32_MAX / size)) { 22598 22598 + verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt); 22599 22599 + return -EINVAL; 22600 22600 + } 22601 22601 + 22602 22602 + for (i = 0; i < attr->fd_array_cnt; i++) { 22603 22603 + if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size)) 22604 22604 + return -EFAULT; 22605 22605 + 22606 22606 + ret = add_fd_from_fd_array(env, fd); 22607 22607 + if (ret) 22608 22608 + return ret; 22609 22609 + } 22610 22610 + 22611 22611 + return 0; 22612 22612 + } 22613 22613 + 22957 22614 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size) 22958 22615 { 22959 22616 u64 start_time = ktime_get_ns(); ··· 23052 22575 env->insn_aux_data[i].orig_idx = i; 23053 22576 env->prog = *prog; 23054 22577 env->ops = bpf_verifier_ops[env->prog->type]; 23055 23055 - env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); 23056 22578 23057 22579 env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); 23058 22580 env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); ··· 23073 22597 attr->log_size); 23074 22598 if (ret) 23075 22599 goto err_unlock; 22600 22600 + 22601 22601 + ret = process_fd_array(env, attr, uattr); 22602 22602 + if (ret) 22603 22603 + goto skip_full_check; 23076 22604 23077 22605 mark_verifier_state_clean(env); 23078 22606

+33 -25

kernel/trace/bpf_trace.c

reviewed

··· 357 357 .arg3_type = ARG_CONST_SIZE, 358 358 }; 359 359 360 360 - static const struct bpf_func_proto *bpf_get_probe_write_proto(void) 361 361 - { 362 362 - if (!capable(CAP_SYS_ADMIN)) 363 363 - return NULL; 364 364 - 365 365 - pr_warn_ratelimited("%s[%d] is installing a program with bpf_probe_write_user helper that may corrupt user memory!", 366 366 - current->comm, task_pid_nr(current)); 367 367 - 368 368 - return &bpf_probe_write_user_proto; 369 369 - } 370 370 - 371 360 #define MAX_TRACE_PRINTK_VARARGS 3 372 361 #define BPF_TRACE_PRINTK_SIZE 1024 373 362 ··· 843 854 if (unlikely(is_global_init(task))) 844 855 return -EPERM; 845 856 846 846 - if (irqs_disabled()) { 857 857 + if (!preemptible()) { 847 858 /* Do an early check on signal validity. Otherwise, 848 859 * the error is lost in deferred irq_work. 849 860 */ ··· 1434 1445 static const struct bpf_func_proto * 1435 1446 bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1436 1447 { 1448 1448 + const struct bpf_func_proto *func_proto; 1449 1449 + 1437 1450 switch (func_id) { 1438 1451 case BPF_FUNC_map_lookup_elem: 1439 1452 return &bpf_map_lookup_elem_proto; ··· 1477 1486 return &bpf_perf_event_read_proto; 1478 1487 case BPF_FUNC_get_prandom_u32: 1479 1488 return &bpf_get_prandom_u32_proto; 1480 1480 - case BPF_FUNC_probe_write_user: 1481 1481 - return security_locked_down(LOCKDOWN_BPF_WRITE_USER) < 0 ? 1482 1482 - NULL : bpf_get_probe_write_proto(); 1483 1489 case BPF_FUNC_probe_read_user: 1484 1490 return &bpf_probe_read_user_proto; 1485 1491 case BPF_FUNC_probe_read_kernel: ··· 1555 1567 case BPF_FUNC_trace_vprintk: 1556 1568 return bpf_get_trace_vprintk_proto(); 1557 1569 default: 1558 1558 - return bpf_base_func_proto(func_id, prog); 1570 1570 + break; 1571 1571 + } 1572 1572 + 1573 1573 + func_proto = bpf_base_func_proto(func_id, prog); 1574 1574 + if (func_proto) 1575 1575 + return func_proto; 1576 1576 + 1577 1577 + if (!bpf_token_capable(prog->aux->token, CAP_SYS_ADMIN)) 1578 1578 + return NULL; 1579 1579 + 1580 1580 + switch (func_id) { 1581 1581 + case BPF_FUNC_probe_write_user: 1582 1582 + return security_locked_down(LOCKDOWN_BPF_WRITE_USER) < 0 ? 1583 1583 + NULL : &bpf_probe_write_user_proto; 1584 1584 + default: 1585 1585 + return NULL; 1559 1586 } 1560 1587 } 1561 1588 ··· 2246 2243 { 2247 2244 struct bpf_prog_array *old_array; 2248 2245 struct bpf_prog_array *new_array; 2246 2246 + struct bpf_prog *prog = NULL; 2249 2247 int ret; 2250 2248 2251 2249 mutex_lock(&bpf_event_mutex); ··· 2267 2263 } 2268 2264 2269 2265 put: 2270 2270 - /* 2271 2271 - * It could be that the bpf_prog is not sleepable (and will be freed 2272 2272 - * via normal RCU), but is called from a point that supports sleepable 2273 2273 - * programs and uses tasks-trace-RCU. 2274 2274 - */ 2275 2275 - synchronize_rcu_tasks_trace(); 2276 2276 - 2277 2277 - bpf_prog_put(event->prog); 2266 2266 + prog = event->prog; 2278 2267 event->prog = NULL; 2279 2268 2280 2269 unlock: 2281 2270 mutex_unlock(&bpf_event_mutex); 2271 2271 + 2272 2272 + if (prog) { 2273 2273 + /* 2274 2274 + * It could be that the bpf_prog is not sleepable (and will be freed 2275 2275 + * via normal RCU), but is called from a point that supports sleepable 2276 2276 + * programs and uses tasks-trace-RCU. 2277 2277 + */ 2278 2278 + synchronize_rcu_tasks_trace(); 2279 2279 + 2280 2280 + bpf_prog_put(prog); 2281 2281 + } 2282 2282 } 2283 2283 2284 2284 int perf_event_query_prog_array(struct perf_event *event, void __user *info) ··· 2818 2810 2819 2811 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { 2820 2812 bpf_prog_inc_misses_counter(link->link.prog); 2821 2821 - err = 0; 2813 2813 + err = 1; 2822 2814 goto out; 2823 2815 } 2824 2816

+58 -6

lib/test_bpf.c

reviewed

··· 478 478 * to overflow the field size of the native instruction, triggering 479 479 * a branch conversion mechanism in some JITs. 480 480 */ 481 481 - static int __bpf_fill_max_jmp(struct bpf_test *self, int jmp, int imm) 481 481 + static int __bpf_fill_max_jmp(struct bpf_test *self, int jmp, int imm, bool alu32) 482 482 { 483 483 struct bpf_insn *insns; 484 484 int len = S16_MAX + 5; ··· 501 501 }; 502 502 int op = ops[(i >> 1) % ARRAY_SIZE(ops)]; 503 503 504 504 - if (i & 1) 504 504 + if ((i & 1) || alu32) 505 505 insns[i++] = BPF_ALU32_REG(op, R0, R1); 506 506 else 507 507 insns[i++] = BPF_ALU64_REG(op, R0, R1); ··· 516 516 } 517 517 518 518 /* Branch taken by runtime decision */ 519 519 + static int bpf_fill_max_jmp_taken_32(struct bpf_test *self) 520 520 + { 521 521 + return __bpf_fill_max_jmp(self, BPF_JEQ, 1, true); 522 522 + } 523 523 + 519 524 static int bpf_fill_max_jmp_taken(struct bpf_test *self) 520 525 { 521 521 - return __bpf_fill_max_jmp(self, BPF_JEQ, 1); 526 526 + return __bpf_fill_max_jmp(self, BPF_JEQ, 1, false); 522 527 } 523 528 524 529 /* Branch not taken by runtime decision */ 530 530 + static int bpf_fill_max_jmp_not_taken_32(struct bpf_test *self) 531 531 + { 532 532 + return __bpf_fill_max_jmp(self, BPF_JEQ, 0, true); 533 533 + } 534 534 + 525 535 static int bpf_fill_max_jmp_not_taken(struct bpf_test *self) 526 536 { 527 527 - return __bpf_fill_max_jmp(self, BPF_JEQ, 0); 537 537 + return __bpf_fill_max_jmp(self, BPF_JEQ, 0, false); 528 538 } 529 539 530 540 /* Branch always taken, known at JIT time */ 541 541 + static int bpf_fill_max_jmp_always_taken_32(struct bpf_test *self) 542 542 + { 543 543 + return __bpf_fill_max_jmp(self, BPF_JGE, 0, true); 544 544 + } 545 545 + 531 546 static int bpf_fill_max_jmp_always_taken(struct bpf_test *self) 532 547 { 533 533 - return __bpf_fill_max_jmp(self, BPF_JGE, 0); 548 548 + return __bpf_fill_max_jmp(self, BPF_JGE, 0, false); 534 549 } 535 550 536 551 /* Branch never taken, known at JIT time */ 552 552 + static int bpf_fill_max_jmp_never_taken_32(struct bpf_test *self) 553 553 + { 554 554 + return __bpf_fill_max_jmp(self, BPF_JLT, 0, true); 555 555 + } 556 556 + 537 557 static int bpf_fill_max_jmp_never_taken(struct bpf_test *self) 538 558 { 539 539 - return __bpf_fill_max_jmp(self, BPF_JLT, 0); 559 559 + return __bpf_fill_max_jmp(self, BPF_JLT, 0, false); 540 560 } 541 561 542 562 /* ALU result computation used in tests */ ··· 14253 14233 { { 0, 0 } }, 14254 14234 }, 14255 14235 /* Conditional branch conversions */ 14236 14236 + { 14237 14237 + "Long conditional jump: taken at runtime (32 bits)", 14238 14238 + { }, 14239 14239 + INTERNAL | FLAG_NO_DATA, 14240 14240 + { }, 14241 14241 + { { 0, 1 } }, 14242 14242 + .fill_helper = bpf_fill_max_jmp_taken_32, 14243 14243 + }, 14244 14244 + { 14245 14245 + "Long conditional jump: not taken at runtime (32 bits)", 14246 14246 + { }, 14247 14247 + INTERNAL | FLAG_NO_DATA, 14248 14248 + { }, 14249 14249 + { { 0, 2 } }, 14250 14250 + .fill_helper = bpf_fill_max_jmp_not_taken_32, 14251 14251 + }, 14252 14252 + { 14253 14253 + "Long conditional jump: always taken, known at JIT time (32 bits)", 14254 14254 + { }, 14255 14255 + INTERNAL | FLAG_NO_DATA, 14256 14256 + { }, 14257 14257 + { { 0, 1 } }, 14258 14258 + .fill_helper = bpf_fill_max_jmp_always_taken_32, 14259 14259 + }, 14260 14260 + { 14261 14261 + "Long conditional jump: never taken, known at JIT time (32 bits)", 14262 14262 + { }, 14263 14263 + INTERNAL | FLAG_NO_DATA, 14264 14264 + { }, 14265 14265 + { { 0, 2 } }, 14266 14266 + .fill_helper = bpf_fill_max_jmp_never_taken_32, 14267 14267 + }, 14256 14268 { 14257 14269 "Long conditional jump: taken at runtime", 14258 14270 { },

+1

net/bpf/test_run.c

reviewed

··· 1018 1018 case BPF_PROG_TYPE_LWT_IN: 1019 1019 case BPF_PROG_TYPE_LWT_OUT: 1020 1020 case BPF_PROG_TYPE_LWT_XMIT: 1021 1021 + case BPF_PROG_TYPE_CGROUP_SKB: 1021 1022 is_direct_pkt_access = true; 1022 1023 break; 1023 1024 default:

+7 -4

net/core/bpf_sk_storage.c

reviewed

··· 50 50 { 51 51 struct bpf_local_storage *sk_storage; 52 52 53 53 + migrate_disable(); 53 54 rcu_read_lock(); 54 55 sk_storage = rcu_dereference(sk->sk_bpf_storage); 55 55 - if (!sk_storage) { 56 56 - rcu_read_unlock(); 57 57 - return; 58 58 - } 56 56 + if (!sk_storage) 57 57 + goto out; 59 58 60 59 bpf_local_storage_destroy(sk_storage); 60 60 + out: 61 61 rcu_read_unlock(); 62 62 + migrate_enable(); 62 63 } 63 64 64 65 static void bpf_sk_storage_map_free(struct bpf_map *map) ··· 161 160 162 161 RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL); 163 162 163 163 + migrate_disable(); 164 164 rcu_read_lock(); 165 165 sk_storage = rcu_dereference(sk->sk_bpf_storage); 166 166 ··· 214 212 215 213 out: 216 214 rcu_read_unlock(); 215 215 + migrate_enable(); 217 216 218 217 /* In case of an error, don't free anything explicitly here, the 219 218 * caller is responsible to call bpf_sk_storage_free.

+1 -1

net/core/filter.c

reviewed

··· 7651 7651 .gpl_only = false, 7652 7652 .ret_type = RET_INTEGER, 7653 7653 .arg1_type = ARG_PTR_TO_CTX, 7654 7654 - .arg2_type = ARG_PTR_TO_MEM, 7654 7654 + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, 7655 7655 .arg3_type = ARG_CONST_SIZE, 7656 7656 .arg4_type = ARG_ANYTHING, 7657 7657 };

+1 -1

samples/bpf/Makefile

reviewed

··· 123 123 always-y += hbm_out_kern.o 124 124 always-y += hbm_edt_kern.o 125 125 126 126 - TPROGS_CFLAGS = $(TPROGS_USER_CFLAGS) 126 126 + COMMON_CFLAGS = $(TPROGS_USER_CFLAGS) 127 127 TPROGS_LDFLAGS = $(TPROGS_USER_LDFLAGS) 128 128 129 129 ifeq ($(ARCH), arm)

-1

samples/bpf/xdp2skb_meta_kern.c

reviewed

··· 63 63 int _tc_mark(struct __sk_buff *ctx) 64 64 { 65 65 void *data = (void *)(unsigned long)ctx->data; 66 66 - void *data_end = (void *)(unsigned long)ctx->data_end; 67 66 void *data_meta = (void *)(unsigned long)ctx->data_meta; 68 67 struct meta_info *meta = data_meta; 69 68

+5 -1

scripts/link-vmlinux.sh

reviewed

··· 283 283 # fill in BTF IDs 284 284 if is_enabled CONFIG_DEBUG_INFO_BTF; then 285 285 info BTFIDS vmlinux 286 286 - ${RESOLVE_BTFIDS} vmlinux 286 286 + RESOLVE_BTFIDS_ARGS="" 287 287 + if is_enabled CONFIG_WERROR; then 288 288 + RESOLVE_BTFIDS_ARGS=" --fatal_warnings " 289 289 + fi 290 290 + ${RESOLVE_BTFIDS} ${RESOLVE_BTFIDS_ARGS} vmlinux 287 291 fi 288 292 289 293 mksysmap vmlinux System.map

-1

security/bpf/hooks.c

reviewed

··· 13 13 #include <linux/lsm_hook_defs.h> 14 14 #undef LSM_HOOK 15 15 LSM_HOOK_INIT(inode_free_security, bpf_inode_storage_free), 16 16 - LSM_HOOK_INIT(task_free, bpf_task_storage_free), 17 16 }; 18 17 19 18 static const struct lsm_id bpf_lsmid = {

+7 -2

tools/bpf/bpftool/Documentation/bpftool-btf.rst

reviewed

··· 24 24 ============= 25 25 26 26 | **bpftool** **btf** { **show** | **list** } [**id** *BTF_ID*] 27 27 - | **bpftool** **btf dump** *BTF_SRC* [**format** *FORMAT*] 27 27 + | **bpftool** **btf dump** *BTF_SRC* [**format** *FORMAT*] [**root_id** *ROOT_ID*] 28 28 | **bpftool** **btf help** 29 29 | 30 30 | *BTF_SRC* := { **id** *BTF_ID* | **prog** *PROG* | **map** *MAP* [{**key** | **value** | **kv** | **all**}] | **file** *FILE* } ··· 43 43 that hold open file descriptors (FDs) against BTF objects. On such kernels 44 44 bpftool will automatically emit this information as well. 45 45 46 46 - bpftool btf dump *BTF_SRC* 46 46 + bpftool btf dump *BTF_SRC* [format *FORMAT*] [root_id *ROOT_ID*] 47 47 Dump BTF entries from a given *BTF_SRC*. 48 48 49 49 When **id** is specified, BTF object with that ID will be loaded and all ··· 66 66 (**raw**) or C-syntax (**c**) output formats are supported. With C-style 67 67 formatting, the output is sorted by default. Use the **unsorted** option 68 68 to avoid sorting the output. 69 69 + 70 70 + **root_id** option can be used to filter a dump to a single type and all 71 71 + its dependent types. It cannot be used with any other types of filtering 72 72 + (such as the "key", "value", or "kv" arguments when dumping BTF for a map). 73 73 + It can be passed multiple times to dump multiple types. 69 74 70 75 bpftool btf help 71 76 Print short help message.

+6 -1

tools/bpf/bpftool/bash-completion/bpftool

reviewed

··· 930 930 format) 931 931 COMPREPLY=( $( compgen -W "c raw" -- "$cur" ) ) 932 932 ;; 933 933 + root_id) 934 934 + return 0; 935 935 + ;; 933 936 c) 934 934 - COMPREPLY=( $( compgen -W "unsorted" -- "$cur" ) ) 937 937 + COMPREPLY=( $( compgen -W "unsorted root_id" -- "$cur" ) ) 935 938 ;; 936 939 *) 937 940 # emit extra options 938 941 case ${words[3]} in 939 942 id|file) 943 943 + COMPREPLY=( $( compgen -W "root_id" -- "$cur" ) ) 940 944 _bpftool_once_attr 'format' 941 945 ;; 942 946 map|prog) 943 947 if [[ ${words[3]} == "map" ]] && [[ $cword == 6 ]]; then 944 948 COMPREPLY+=( $( compgen -W "key value kv all" -- "$cur" ) ) 945 949 fi 950 950 + COMPREPLY=( $( compgen -W "root_id" -- "$cur" ) ) 946 951 _bpftool_once_attr 'format' 947 952 ;; 948 953 *)

+49 -2

tools/bpf/bpftool/btf.c

reviewed

··· 27 27 #define KFUNC_DECL_TAG "bpf_kfunc" 28 28 #define FASTCALL_DECL_TAG "bpf_fastcall" 29 29 30 30 + #define MAX_ROOT_IDS 16 31 31 + 30 32 static const char * const btf_kind_str[NR_BTF_KINDS] = { 31 33 [BTF_KIND_UNKN] = "UNKNOWN", 32 34 [BTF_KIND_INT] = "INT", ··· 882 880 { 883 881 bool dump_c = false, sort_dump_c = true; 884 882 struct btf *btf = NULL, *base = NULL; 885 885 - __u32 root_type_ids[2]; 883 883 + __u32 root_type_ids[MAX_ROOT_IDS]; 884 884 + bool have_id_filtering; 886 885 int root_type_cnt = 0; 887 886 __u32 btf_id = -1; 888 887 const char *src; 889 888 int fd = -1; 890 889 int err = 0; 890 890 + int i; 891 891 892 892 if (!REQ_ARGS(2)) { 893 893 usage(); ··· 977 973 goto done; 978 974 } 979 975 976 976 + have_id_filtering = !!root_type_cnt; 977 977 + 980 978 while (argc) { 981 979 if (is_prefix(*argv, "format")) { 982 980 NEXT_ARG(); ··· 997 991 err = -EINVAL; 998 992 goto done; 999 993 } 994 994 + NEXT_ARG(); 995 995 + } else if (is_prefix(*argv, "root_id")) { 996 996 + __u32 root_id; 997 997 + char *end; 998 998 + 999 999 + if (have_id_filtering) { 1000 1000 + p_err("cannot use root_id with other type filtering"); 1001 1001 + err = -EINVAL; 1002 1002 + goto done; 1003 1003 + } else if (root_type_cnt == MAX_ROOT_IDS) { 1004 1004 + p_err("only %d root_id are supported", MAX_ROOT_IDS); 1005 1005 + err = -E2BIG; 1006 1006 + goto done; 1007 1007 + } 1008 1008 + 1009 1009 + NEXT_ARG(); 1010 1010 + root_id = strtoul(*argv, &end, 0); 1011 1011 + if (*end) { 1012 1012 + err = -1; 1013 1013 + p_err("can't parse %s as root ID", *argv); 1014 1014 + goto done; 1015 1015 + } 1016 1016 + for (i = 0; i < root_type_cnt; i++) { 1017 1017 + if (root_type_ids[i] == root_id) { 1018 1018 + err = -EINVAL; 1019 1019 + p_err("duplicate root_id %d supplied", root_id); 1020 1020 + goto done; 1021 1021 + } 1022 1022 + } 1023 1023 + root_type_ids[root_type_cnt++] = root_id; 1000 1024 NEXT_ARG(); 1001 1025 } else if (is_prefix(*argv, "unsorted")) { 1002 1026 sort_dump_c = false; ··· 1049 1013 if (!btf) { 1050 1014 err = -errno; 1051 1015 p_err("get btf by id (%u): %s", btf_id, strerror(errno)); 1016 1016 + goto done; 1017 1017 + } 1018 1018 + } 1019 1019 + 1020 1020 + /* Invalid root IDs causes half emitted boilerplate and then unclean 1021 1021 + * exit. It's an ugly user experience, so handle common error here. 1022 1022 + */ 1023 1023 + for (i = 0; i < root_type_cnt; i++) { 1024 1024 + if (root_type_ids[i] >= btf__type_cnt(btf)) { 1025 1025 + err = -EINVAL; 1026 1026 + p_err("invalid root ID: %u", root_type_ids[i]); 1052 1027 goto done; 1053 1028 } 1054 1029 } ··· 1438 1391 1439 1392 fprintf(stderr, 1440 1393 "Usage: %1$s %2$s { show | list } [id BTF_ID]\n" 1441 1441 - " %1$s %2$s dump BTF_SRC [format FORMAT]\n" 1394 1394 + " %1$s %2$s dump BTF_SRC [format FORMAT] [root_id ROOT_ID]\n" 1442 1395 " %1$s %2$s help\n" 1443 1396 "\n" 1444 1397 " BTF_SRC := { id BTF_ID | prog PROG | map MAP [{key | value | kv | all}] | file FILE }\n"

+1

tools/bpf/bpftool/cfg.c

reviewed

··· 302 302 303 303 insn = bb->tail; 304 304 if (!is_jmp_insn(insn->code) || 305 305 + BPF_OP(insn->code) == BPF_CALL || 305 306 BPF_OP(insn->code) == BPF_EXIT) { 306 307 e->dst = bb_next(bb); 307 308 e->flags |= EDGE_FLAG_FALLTHROUGH;

+23

tools/bpf/bpftool/feature.c

reviewed

··· 885 885 "V3_ISA_EXTENSION"); 886 886 } 887 887 888 888 + /* 889 889 + * Probe for the v4 instruction set extension introduced in commit 1f9a1ea821ff 890 890 + * ("bpf: Support new sign-extension load insns"). 891 891 + */ 892 892 + static void 893 893 + probe_v4_isa_extension(const char *define_prefix, __u32 ifindex) 894 894 + { 895 895 + struct bpf_insn insns[5] = { 896 896 + BPF_MOV64_IMM(BPF_REG_0, 0), 897 897 + BPF_JMP32_IMM(BPF_JEQ, BPF_REG_0, 1, 1), 898 898 + BPF_JMP32_A(1), 899 899 + BPF_MOV64_IMM(BPF_REG_0, 1), 900 900 + BPF_EXIT_INSN() 901 901 + }; 902 902 + 903 903 + probe_misc_feature(insns, ARRAY_SIZE(insns), 904 904 + define_prefix, ifindex, 905 905 + "have_v4_isa_extension", 906 906 + "ISA extension v4", 907 907 + "V4_ISA_EXTENSION"); 908 908 + } 909 909 + 888 910 static void 889 911 section_system_config(enum probe_component target, const char *define_prefix) 890 912 { ··· 1051 1029 probe_bounded_loops(define_prefix, ifindex); 1052 1030 probe_v2_isa_extension(define_prefix, ifindex); 1053 1031 probe_v3_isa_extension(define_prefix, ifindex); 1032 1032 + probe_v4_isa_extension(define_prefix, ifindex); 1054 1033 print_end_section(); 1055 1034 } 1056 1035

+10 -2

tools/bpf/resolve_btfids/main.c

reviewed

··· 141 141 }; 142 142 143 143 static int verbose; 144 144 + static int warnings; 144 145 145 146 static int eprintf(int level, int var, const char *fmt, ...) 146 147 { ··· 605 604 if (id->id) { 606 605 pr_info("WARN: multiple IDs found for '%s': %d, %d - using %d\n", 607 606 str, id->id, type_id, id->id); 607 607 + warnings++; 608 608 } else { 609 609 id->id = type_id; 610 610 (*nr)--; ··· 627 625 int i; 628 626 629 627 /* For set, set8, id->id may be 0 */ 630 630 - if (!id->id && !id->is_set && !id->is_set8) 628 628 + if (!id->id && !id->is_set && !id->is_set8) { 631 629 pr_err("WARN: resolve_btfids: unresolved symbol %s\n", id->name); 630 630 + warnings++; 631 631 + } 632 632 633 633 for (i = 0; i < id->addr_cnt; i++) { 634 634 unsigned long addr = id->addr[i]; ··· 786 782 .funcs = RB_ROOT, 787 783 .sets = RB_ROOT, 788 784 }; 785 785 + bool fatal_warnings = false; 789 786 struct option btfid_options[] = { 790 787 OPT_INCR('v', "verbose", &verbose, 791 788 "be more verbose (show errors, etc)"), ··· 794 789 "BTF data"), 795 790 OPT_STRING('b', "btf_base", &obj.base_btf_path, "file", 796 791 "path of file providing base BTF"), 792 792 + OPT_BOOLEAN(0, "fatal_warnings", &fatal_warnings, 793 793 + "turn warnings into errors"), 797 794 OPT_END() 798 795 }; 799 796 int err = -1; ··· 830 823 if (symbols_patch(&obj)) 831 824 goto out; 832 825 833 833 - err = 0; 826 826 + if (!(fatal_warnings && warnings)) 827 827 + err = 0; 834 828 out: 835 829 if (obj.efile.elf) { 836 830 elf_end(obj.efile.elf);

+10

tools/include/linux/filter.h

reviewed

··· 273 273 .off = OFF, \ 274 274 .imm = 0 }) 275 275 276 276 + /* Unconditional jumps, gotol pc + imm32 */ 277 277 + 278 278 + #define BPF_JMP32_A(IMM) \ 279 279 + ((struct bpf_insn) { \ 280 280 + .code = BPF_JMP32 | BPF_JA, \ 281 281 + .dst_reg = 0, \ 282 282 + .src_reg = 0, \ 283 283 + .off = 0, \ 284 284 + .imm = IMM }) 285 285 + 276 286 /* Function call */ 277 287 278 288 #define BPF_EMIT_CALL(FUNC) \

+10

tools/include/uapi/linux/bpf.h

reviewed

··· 1573 1573 * If provided, prog_flags should have BPF_F_TOKEN_FD flag set. 1574 1574 */ 1575 1575 __s32 prog_token_fd; 1576 1576 + /* The fd_array_cnt can be used to pass the length of the 1577 1577 + * fd_array array. In this case all the [map] file descriptors 1578 1578 + * passed in this array will be bound to the program, even if 1579 1579 + * the maps are not referenced directly. The functionality is 1580 1580 + * similar to the BPF_PROG_BIND_MAP syscall, but maps can be 1581 1581 + * used by the verifier during the program load. If provided, 1582 1582 + * then the fd_array[0,...,fd_array_cnt-1] is expected to be 1583 1583 + * continuous. 1584 1584 + */ 1585 1585 + __u32 fd_array_cnt; 1576 1586 }; 1577 1587 1578 1588 struct { /* anonymous struct used by BPF_OBJ_* commands */

+2 -2

tools/include/uapi/linux/if_xdp.h

reviewed

··· 117 117 ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1) 118 118 119 119 /* Request transmit timestamp. Upon completion, put it into tx_timestamp 120 120 - * field of union xsk_tx_metadata. 120 120 + * field of struct xsk_tx_metadata. 121 121 */ 122 122 #define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0) 123 123 124 124 /* Request transmit checksum offload. Checksum start position and offset 125 125 - * are communicated via csum_start and csum_offset fields of union 125 125 + * are communicated via csum_start and csum_offset fields of struct 126 126 * xsk_tx_metadata. 127 127 */ 128 128 #define XDP_TXMD_FLAGS_CHECKSUM (1 << 1)

+2 -1

tools/lib/bpf/bpf.c

reviewed

··· 238 238 const struct bpf_insn *insns, size_t insn_cnt, 239 239 struct bpf_prog_load_opts *opts) 240 240 { 241 241 - const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); 241 241 + const size_t attr_sz = offsetofend(union bpf_attr, fd_array_cnt); 242 242 void *finfo = NULL, *linfo = NULL; 243 243 const char *func_info, *line_info; 244 244 __u32 log_size, log_level, attach_prog_fd, attach_btf_obj_fd; ··· 311 311 attr.line_info_cnt = OPTS_GET(opts, line_info_cnt, 0); 312 312 313 313 attr.fd_array = ptr_to_u64(OPTS_GET(opts, fd_array, NULL)); 314 314 + attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0); 314 315 315 316 if (log_level) { 316 317 attr.log_buf = ptr_to_u64(log_buf);

+4 -1

tools/lib/bpf/bpf.h

reviewed

··· 107 107 */ 108 108 __u32 log_true_size; 109 109 __u32 token_fd; 110 110 + 111 111 + /* if set, provides the length of fd_array */ 112 112 + __u32 fd_array_cnt; 110 113 size_t :0; 111 114 }; 112 112 - #define bpf_prog_load_opts__last_field token_fd 115 115 + #define bpf_prog_load_opts__last_field fd_array_cnt 113 116 114 117 LIBBPF_API int bpf_prog_load(enum bpf_prog_type prog_type, 115 118 const char *prog_name, const char *license,

+2 -1

tools/lib/bpf/btf.c

reviewed

··· 283 283 return -EINVAL; 284 284 } 285 285 if (!btf->base_btf && start[0]) { 286 286 - pr_debug("Invalid BTF string section\n"); 286 286 + pr_debug("Malformed BTF string section, did you forget to provide base BTF?\n"); 287 287 return -EINVAL; 288 288 } 289 289 return 0; ··· 1186 1186 1187 1187 elf = elf_begin(fd, ELF_C_READ, NULL); 1188 1188 if (!elf) { 1189 1189 + err = -LIBBPF_ERRNO__FORMAT; 1189 1190 pr_warn("failed to open %s as ELF file\n", path); 1190 1191 goto done; 1191 1192 }

+1 -1

tools/lib/bpf/btf_relocate.c

reviewed

··· 212 212 * need to match both name and size, otherwise embedding the base 213 213 * struct/union in the split type is invalid. 214 214 */ 215 215 - for (id = r->nr_dist_base_types; id < r->nr_split_types; id++) { 215 215 + for (id = r->nr_dist_base_types; id < r->nr_dist_base_types + r->nr_split_types; id++) { 216 216 err = btf_mark_embedded_composite_type_ids(r, id); 217 217 if (err) 218 218 goto done;

+50 -3

tools/lib/bpf/libbpf.c

reviewed

··· 1731 1731 #ifndef MFD_CLOEXEC 1732 1732 #define MFD_CLOEXEC 0x0001U 1733 1733 #endif 1734 1734 + #ifndef MFD_NOEXEC_SEAL 1735 1735 + #define MFD_NOEXEC_SEAL 0x0008U 1736 1736 + #endif 1734 1737 1735 1738 static int create_placeholder_fd(void) 1736 1739 { 1740 1740 + unsigned int flags = MFD_CLOEXEC | MFD_NOEXEC_SEAL; 1741 1741 + const char *name = "libbpf-placeholder-fd"; 1737 1742 int fd; 1738 1743 1739 1739 - fd = ensure_good_fd(sys_memfd_create("libbpf-placeholder-fd", MFD_CLOEXEC)); 1744 1744 + fd = ensure_good_fd(sys_memfd_create(name, flags)); 1745 1745 + if (fd >= 0) 1746 1746 + return fd; 1747 1747 + else if (errno != EINVAL) 1748 1748 + return -errno; 1749 1749 + 1750 1750 + /* Possibly running on kernel without MFD_NOEXEC_SEAL */ 1751 1751 + fd = ensure_good_fd(sys_memfd_create(name, flags & ~MFD_NOEXEC_SEAL)); 1740 1752 if (fd < 0) 1741 1753 return -errno; 1742 1754 return fd; ··· 11387 11375 struct kprobe_multi_resolve *res = data->res; 11388 11376 int err; 11389 11377 11390 11390 - if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) 11378 11378 + if (!glob_match(sym_name, res->pattern)) 11391 11379 return 0; 11380 11380 + 11381 11381 + if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) { 11382 11382 + /* Some versions of kernel strip out .llvm.<hash> suffix from 11383 11383 + * function names reported in available_filter_functions, but 11384 11384 + * don't do so for kallsyms. While this is clearly a kernel 11385 11385 + * bug (fixed by [0]) we try to accommodate that in libbpf to 11386 11386 + * make multi-kprobe usability a bit better: if no match is 11387 11387 + * found, we will strip .llvm. suffix and try one more time. 11388 11388 + * 11389 11389 + * [0] fb6a421fb615 ("kallsyms: Match symbols exactly with CONFIG_LTO_CLANG") 11390 11390 + */ 11391 11391 + char sym_trim[256], *psym_trim = sym_trim, *sym_sfx; 11392 11392 + 11393 11393 + if (!(sym_sfx = strstr(sym_name, ".llvm."))) 11394 11394 + return 0; 11395 11395 + 11396 11396 + /* psym_trim vs sym_trim dance is done to avoid pointer vs array 11397 11397 + * coercion differences and get proper `const char **` pointer 11398 11398 + * which avail_func_cmp() expects 11399 11399 + */ 11400 11400 + snprintf(sym_trim, sizeof(sym_trim), "%.*s", (int)(sym_sfx - sym_name), sym_name); 11401 11401 + if (!bsearch(&psym_trim, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) 11402 11402 + return 0; 11403 11403 + } 11392 11404 11393 11405 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1); 11394 11406 if (err) ··· 11558 11522 struct bpf_link *link = NULL; 11559 11523 const unsigned long *addrs; 11560 11524 int err, link_fd, prog_fd; 11561 11561 - bool retprobe, session; 11525 11525 + bool retprobe, session, unique_match; 11562 11526 const __u64 *cookies; 11563 11527 const char **syms; 11564 11528 size_t cnt; ··· 11577 11541 addrs = OPTS_GET(opts, addrs, false); 11578 11542 cnt = OPTS_GET(opts, cnt, false); 11579 11543 cookies = OPTS_GET(opts, cookies, false); 11544 11544 + unique_match = OPTS_GET(opts, unique_match, false); 11580 11545 11581 11546 if (!pattern && !addrs && !syms) 11582 11547 return libbpf_err_ptr(-EINVAL); 11583 11548 if (pattern && (addrs || syms || cookies || cnt)) 11584 11549 return libbpf_err_ptr(-EINVAL); 11585 11550 if (!pattern && !cnt) 11551 11551 + return libbpf_err_ptr(-EINVAL); 11552 11552 + if (!pattern && unique_match) 11586 11553 return libbpf_err_ptr(-EINVAL); 11587 11554 if (addrs && syms) 11588 11555 return libbpf_err_ptr(-EINVAL); ··· 11597 11558 err = libbpf_available_kallsyms_parse(&res); 11598 11559 if (err) 11599 11560 goto error; 11561 11561 + 11562 11562 + if (unique_match && res.cnt != 1) { 11563 11563 + pr_warn("prog '%s': failed to find a unique match for '%s' (%zu matches)\n", 11564 11564 + prog->name, pattern, res.cnt); 11565 11565 + err = -EINVAL; 11566 11566 + goto error; 11567 11567 + } 11568 11568 + 11600 11569 addrs = res.addrs; 11601 11570 cnt = res.cnt; 11602 11571 }

+8 -1

tools/lib/bpf/libbpf.h

reviewed

··· 552 552 bool retprobe; 553 553 /* create session kprobes */ 554 554 bool session; 555 555 + /* enforce unique match */ 556 556 + bool unique_match; 555 557 size_t :0; 556 558 }; 557 559 558 558 - #define bpf_kprobe_multi_opts__last_field session 560 560 + #define bpf_kprobe_multi_opts__last_field unique_match 559 561 560 562 LIBBPF_API struct bpf_link * 561 563 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, ··· 1798 1796 struct bpf_linker; 1799 1797 1800 1798 LIBBPF_API struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts); 1799 1799 + LIBBPF_API struct bpf_linker *bpf_linker__new_fd(int fd, struct bpf_linker_opts *opts); 1801 1800 LIBBPF_API int bpf_linker__add_file(struct bpf_linker *linker, 1802 1801 const char *filename, 1803 1802 const struct bpf_linker_file_opts *opts); 1803 1803 + LIBBPF_API int bpf_linker__add_fd(struct bpf_linker *linker, int fd, 1804 1804 + const struct bpf_linker_file_opts *opts); 1805 1805 + LIBBPF_API int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz, 1806 1806 + const struct bpf_linker_file_opts *opts); 1804 1807 LIBBPF_API int bpf_linker__finalize(struct bpf_linker *linker); 1805 1808 LIBBPF_API void bpf_linker__free(struct bpf_linker *linker); 1806 1809

+4

tools/lib/bpf/libbpf.map

reviewed

··· 432 432 } LIBBPF_1.4.0; 433 433 434 434 LIBBPF_1.6.0 { 435 435 + global: 436 436 + bpf_linker__add_buf; 437 437 + bpf_linker__add_fd; 438 438 + bpf_linker__new_fd; 435 439 } LIBBPF_1.5.0;

+181 -67

tools/lib/bpf/linker.c

reviewed

··· 4 4 * 5 5 * Copyright (c) 2021 Facebook 6 6 */ 7 7 + #ifndef _GNU_SOURCE 8 8 + #define _GNU_SOURCE 9 9 + #endif 10 10 + 7 11 #include <stdbool.h> 8 12 #include <stddef.h> 9 13 #include <stdio.h> ··· 20 16 #include <elf.h> 21 17 #include <libelf.h> 22 18 #include <fcntl.h> 19 19 + #include <sys/mman.h> 23 20 #include "libbpf.h" 24 21 #include "btf.h" 25 22 #include "libbpf_internal.h" ··· 157 152 /* global (including extern) ELF symbols */ 158 153 int glob_sym_cnt; 159 154 struct glob_sym *glob_syms; 155 155 + 156 156 + bool fd_is_owned; 160 157 }; 161 158 162 159 #define pr_warn_elf(fmt, ...) \ 163 160 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1)) 164 161 165 165 - static int init_output_elf(struct bpf_linker *linker, const char *file); 162 162 + static int init_output_elf(struct bpf_linker *linker); 166 163 167 167 - static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, 168 168 - const struct bpf_linker_file_opts *opts, 164 164 + static int bpf_linker_add_file(struct bpf_linker *linker, int fd, 165 165 + const char *filename); 166 166 + 167 167 + static int linker_load_obj_file(struct bpf_linker *linker, 169 168 struct src_obj *obj); 170 169 static int linker_sanity_check_elf(struct src_obj *obj); 171 170 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec); ··· 200 191 if (linker->elf) 201 192 elf_end(linker->elf); 202 193 203 203 - if (linker->fd >= 0) 194 194 + if (linker->fd >= 0 && linker->fd_is_owned) 204 195 close(linker->fd); 205 196 206 197 strset__free(linker->strtab_strs); ··· 242 233 if (!linker) 243 234 return errno = ENOMEM, NULL; 244 235 245 245 - linker->fd = -1; 236 236 + linker->filename = strdup(filename); 237 237 + if (!linker->filename) { 238 238 + err = -ENOMEM; 239 239 + goto err_out; 240 240 + } 246 241 247 247 - err = init_output_elf(linker, filename); 242 242 + linker->fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); 243 243 + if (linker->fd < 0) { 244 244 + err = -errno; 245 245 + pr_warn("failed to create '%s': %d\n", filename, err); 246 246 + goto err_out; 247 247 + } 248 248 + linker->fd_is_owned = true; 249 249 + 250 250 + err = init_output_elf(linker); 251 251 + if (err) 252 252 + goto err_out; 253 253 + 254 254 + return linker; 255 255 + 256 256 + err_out: 257 257 + bpf_linker__free(linker); 258 258 + return errno = -err, NULL; 259 259 + } 260 260 + 261 261 + struct bpf_linker *bpf_linker__new_fd(int fd, struct bpf_linker_opts *opts) 262 262 + { 263 263 + struct bpf_linker *linker; 264 264 + char filename[32]; 265 265 + int err; 266 266 + 267 267 + if (fd < 0) 268 268 + return errno = EINVAL, NULL; 269 269 + 270 270 + if (!OPTS_VALID(opts, bpf_linker_opts)) 271 271 + return errno = EINVAL, NULL; 272 272 + 273 273 + if (elf_version(EV_CURRENT) == EV_NONE) { 274 274 + pr_warn_elf("libelf initialization failed"); 275 275 + return errno = EINVAL, NULL; 276 276 + } 277 277 + 278 278 + linker = calloc(1, sizeof(*linker)); 279 279 + if (!linker) 280 280 + return errno = ENOMEM, NULL; 281 281 + 282 282 + snprintf(filename, sizeof(filename), "fd:%d", fd); 283 283 + linker->filename = strdup(filename); 284 284 + if (!linker->filename) { 285 285 + err = -ENOMEM; 286 286 + goto err_out; 287 287 + } 288 288 + 289 289 + linker->fd = fd; 290 290 + linker->fd_is_owned = false; 291 291 + 292 292 + err = init_output_elf(linker); 248 293 if (err) 249 294 goto err_out; 250 295 ··· 357 294 return sym; 358 295 } 359 296 360 360 - static int init_output_elf(struct bpf_linker *linker, const char *file) 297 297 + static int init_output_elf(struct bpf_linker *linker) 361 298 { 362 299 int err, str_off; 363 300 Elf64_Sym *init_sym; 364 301 struct dst_sec *sec; 365 365 - 366 366 - linker->filename = strdup(file); 367 367 - if (!linker->filename) 368 368 - return -ENOMEM; 369 369 - 370 370 - linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); 371 371 - if (linker->fd < 0) { 372 372 - err = -errno; 373 373 - pr_warn("failed to create '%s': %s\n", file, errstr(err)); 374 374 - return err; 375 375 - } 376 302 377 303 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL); 378 304 if (!linker->elf) { ··· 488 436 return 0; 489 437 } 490 438 491 491 - int bpf_linker__add_file(struct bpf_linker *linker, const char *filename, 492 492 - const struct bpf_linker_file_opts *opts) 439 439 + static int bpf_linker_add_file(struct bpf_linker *linker, int fd, 440 440 + const char *filename) 493 441 { 494 442 struct src_obj obj = {}; 495 443 int err = 0; 496 444 497 497 - if (!OPTS_VALID(opts, bpf_linker_file_opts)) 498 498 - return libbpf_err(-EINVAL); 445 445 + obj.filename = filename; 446 446 + obj.fd = fd; 499 447 500 500 - if (!linker->elf) 501 501 - return libbpf_err(-EINVAL); 502 502 - 503 503 - err = err ?: linker_load_obj_file(linker, filename, opts, &obj); 448 448 + err = err ?: linker_load_obj_file(linker, &obj); 504 449 err = err ?: linker_append_sec_data(linker, &obj); 505 450 err = err ?: linker_append_elf_syms(linker, &obj); 506 451 err = err ?: linker_append_elf_relos(linker, &obj); ··· 512 463 free(obj.sym_map); 513 464 if (obj.elf) 514 465 elf_end(obj.elf); 515 515 - if (obj.fd >= 0) 516 516 - close(obj.fd); 517 466 467 467 + return err; 468 468 + } 469 469 + 470 470 + int bpf_linker__add_file(struct bpf_linker *linker, const char *filename, 471 471 + const struct bpf_linker_file_opts *opts) 472 472 + { 473 473 + int fd, err; 474 474 + 475 475 + if (!OPTS_VALID(opts, bpf_linker_file_opts)) 476 476 + return libbpf_err(-EINVAL); 477 477 + 478 478 + if (!linker->elf) 479 479 + return libbpf_err(-EINVAL); 480 480 + 481 481 + fd = open(filename, O_RDONLY | O_CLOEXEC); 482 482 + if (fd < 0) { 483 483 + err = -errno; 484 484 + pr_warn("failed to open file '%s': %s\n", filename, errstr(err)); 485 485 + return libbpf_err(err); 486 486 + } 487 487 + 488 488 + err = bpf_linker_add_file(linker, fd, filename); 489 489 + close(fd); 518 490 return libbpf_err(err); 491 491 + } 492 492 + 493 493 + int bpf_linker__add_fd(struct bpf_linker *linker, int fd, 494 494 + const struct bpf_linker_file_opts *opts) 495 495 + { 496 496 + char filename[32]; 497 497 + int err; 498 498 + 499 499 + if (!OPTS_VALID(opts, bpf_linker_file_opts)) 500 500 + return libbpf_err(-EINVAL); 501 501 + 502 502 + if (!linker->elf) 503 503 + return libbpf_err(-EINVAL); 504 504 + 505 505 + if (fd < 0) 506 506 + return libbpf_err(-EINVAL); 507 507 + 508 508 + snprintf(filename, sizeof(filename), "fd:%d", fd); 509 509 + err = bpf_linker_add_file(linker, fd, filename); 510 510 + return libbpf_err(err); 511 511 + } 512 512 + 513 513 + int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz, 514 514 + const struct bpf_linker_file_opts *opts) 515 515 + { 516 516 + char filename[32]; 517 517 + int fd, written, ret; 518 518 + 519 519 + if (!OPTS_VALID(opts, bpf_linker_file_opts)) 520 520 + return libbpf_err(-EINVAL); 521 521 + 522 522 + if (!linker->elf) 523 523 + return libbpf_err(-EINVAL); 524 524 + 525 525 + snprintf(filename, sizeof(filename), "mem:%p+%zu", buf, buf_sz); 526 526 + 527 527 + fd = memfd_create(filename, 0); 528 528 + if (fd < 0) { 529 529 + ret = -errno; 530 530 + pr_warn("failed to create memfd '%s': %s\n", filename, errstr(ret)); 531 531 + return libbpf_err(ret); 532 532 + } 533 533 + 534 534 + written = 0; 535 535 + while (written < buf_sz) { 536 536 + ret = write(fd, buf, buf_sz); 537 537 + if (ret < 0) { 538 538 + ret = -errno; 539 539 + pr_warn("failed to write '%s': %s\n", filename, errstr(ret)); 540 540 + goto err_out; 541 541 + } 542 542 + written += ret; 543 543 + } 544 544 + 545 545 + ret = bpf_linker_add_file(linker, fd, filename); 546 546 + err_out: 547 547 + close(fd); 548 548 + return libbpf_err(ret); 519 549 } 520 550 521 551 static bool is_dwarf_sec_name(const char *name) ··· 662 534 return sec; 663 535 } 664 536 665 665 - static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, 666 666 - const struct bpf_linker_file_opts *opts, 537 537 + static int linker_load_obj_file(struct bpf_linker *linker, 667 538 struct src_obj *obj) 668 539 { 669 540 int err = 0; ··· 681 554 #error "Unknown __BYTE_ORDER__" 682 555 #endif 683 556 684 684 - pr_debug("linker: adding object file '%s'...\n", filename); 557 557 + pr_debug("linker: adding object file '%s'...\n", obj->filename); 685 558 686 686 - obj->filename = filename; 687 687 - 688 688 - obj->fd = open(filename, O_RDONLY | O_CLOEXEC); 689 689 - if (obj->fd < 0) { 690 690 - err = -errno; 691 691 - pr_warn("failed to open file '%s': %s\n", filename, errstr(err)); 692 692 - return err; 693 693 - } 694 559 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL); 695 560 if (!obj->elf) { 696 696 - err = -errno; 697 697 - pr_warn_elf("failed to parse ELF file '%s'", filename); 698 698 - return err; 561 561 + pr_warn_elf("failed to parse ELF file '%s'", obj->filename); 562 562 + return -EINVAL; 699 563 } 700 564 701 565 /* Sanity check ELF file high-level properties */ 702 566 ehdr = elf64_getehdr(obj->elf); 703 567 if (!ehdr) { 704 704 - err = -errno; 705 705 - pr_warn_elf("failed to get ELF header for %s", filename); 706 706 - return err; 568 568 + pr_warn_elf("failed to get ELF header for %s", obj->filename); 569 569 + return -EINVAL; 707 570 } 708 571 709 572 /* Linker output endianness set by first input object */ 710 573 obj_byteorder = ehdr->e_ident[EI_DATA]; 711 574 if (obj_byteorder != ELFDATA2LSB && obj_byteorder != ELFDATA2MSB) { 712 575 err = -EOPNOTSUPP; 713 713 - pr_warn("unknown byte order of ELF file %s\n", filename); 576 576 + pr_warn("unknown byte order of ELF file %s\n", obj->filename); 714 577 return err; 715 578 } 716 579 if (link_byteorder == ELFDATANONE) { ··· 710 593 obj_byteorder == ELFDATA2MSB ? "big" : "little"); 711 594 } else if (link_byteorder != obj_byteorder) { 712 595 err = -EOPNOTSUPP; 713 713 - pr_warn("byte order mismatch with ELF file %s\n", filename); 596 596 + pr_warn("byte order mismatch with ELF file %s\n", obj->filename); 714 597 return err; 715 598 } 716 599 ··· 718 601 || ehdr->e_machine != EM_BPF 719 602 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) { 720 603 err = -EOPNOTSUPP; 721 721 - pr_warn_elf("unsupported kind of ELF file %s", filename); 604 604 + pr_warn_elf("unsupported kind of ELF file %s", obj->filename); 722 605 return err; 723 606 } 724 607 725 608 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) { 726 726 - err = -errno; 727 727 - pr_warn_elf("failed to get SHSTRTAB section index for %s", filename); 728 728 - return err; 609 609 + pr_warn_elf("failed to get SHSTRTAB section index for %s", obj->filename); 610 610 + return -EINVAL; 729 611 } 730 612 731 613 scn = NULL; ··· 734 618 735 619 shdr = elf64_getshdr(scn); 736 620 if (!shdr) { 737 737 - err = -errno; 738 621 pr_warn_elf("failed to get section #%zu header for %s", 739 739 - sec_idx, filename); 740 740 - return err; 622 622 + sec_idx, obj->filename); 623 623 + return -EINVAL; 741 624 } 742 625 743 626 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name); 744 627 if (!sec_name) { 745 745 - err = -errno; 746 628 pr_warn_elf("failed to get section #%zu name for %s", 747 747 - sec_idx, filename); 748 748 - return err; 629 629 + sec_idx, obj->filename); 630 630 + return -EINVAL; 749 631 } 750 632 751 633 data = elf_getdata(scn, 0); 752 634 if (!data) { 753 753 - err = -errno; 754 635 pr_warn_elf("failed to get section #%zu (%s) data from %s", 755 755 - sec_idx, sec_name, filename); 756 756 - return err; 636 636 + sec_idx, sec_name, obj->filename); 637 637 + return -EINVAL; 757 638 } 758 639 759 640 sec = add_src_sec(obj, sec_name); ··· 785 672 err = libbpf_get_error(obj->btf); 786 673 if (err) { 787 674 pr_warn("failed to parse .BTF from %s: %s\n", 788 788 - filename, errstr(err)); 675 675 + obj->filename, errstr(err)); 789 676 return err; 790 677 } 791 678 sec->skipped = true; ··· 796 683 err = libbpf_get_error(obj->btf_ext); 797 684 if (err) { 798 685 pr_warn("failed to parse .BTF.ext from '%s': %s\n", 799 799 - filename, errstr(err)); 686 686 + obj->filename, errstr(err)); 800 687 return err; 801 688 } 802 689 sec->skipped = true; ··· 813 700 break; 814 701 default: 815 702 pr_warn("unrecognized section #%zu (%s) in %s\n", 816 816 - sec_idx, sec_name, filename); 703 703 + sec_idx, sec_name, obj->filename); 817 704 err = -EINVAL; 818 705 return err; 819 706 } ··· 2793 2680 2794 2681 /* Finalize ELF layout */ 2795 2682 if (elf_update(linker->elf, ELF_C_NULL) < 0) { 2796 2796 - err = -errno; 2683 2683 + err = -EINVAL; 2797 2684 pr_warn_elf("failed to finalize ELF layout"); 2798 2685 return libbpf_err(err); 2799 2686 } 2800 2687 2801 2688 /* Write out final ELF contents */ 2802 2689 if (elf_update(linker->elf, ELF_C_WRITE) < 0) { 2803 2803 - err = -errno; 2690 2690 + err = -EINVAL; 2804 2691 pr_warn_elf("failed to write ELF contents"); 2805 2692 return libbpf_err(err); 2806 2693 } 2807 2694 2808 2695 elf_end(linker->elf); 2809 2809 - close(linker->fd); 2810 2810 - 2811 2696 linker->elf = NULL; 2697 2697 + 2698 2698 + if (linker->fd_is_owned) 2699 2699 + close(linker->fd); 2812 2700 linker->fd = -1; 2813 2701 2814 2702 return 0;

+1 -1

tools/lib/bpf/usdt.c

reviewed

··· 661 661 * [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation 662 662 */ 663 663 usdt_abs_ip = note.loc_addr; 664 664 - if (base_addr) 664 664 + if (base_addr && note.base_addr) 665 665 usdt_abs_ip += base_addr - note.base_addr; 666 666 667 667 /* When attaching uprobes (which is what USDTs basically are)

-1

tools/testing/selftests/bpf/.gitignore

reviewed

··· 18 18 urandom_read 19 19 test_sockmap 20 20 test_lirc_mode2_user 21 21 - test_flow_dissector 22 21 flow_dissector_load 23 22 test_tcpnotify_user 24 23 test_libbpf

+33 -80

tools/testing/selftests/bpf/Makefile

reviewed

··· 41 41 srctree := $(patsubst %/,%,$(dir $(srctree))) 42 42 endif 43 43 44 44 - CFLAGS += -g $(OPT_FLAGS) -rdynamic \ 44 44 + CFLAGS += -g $(OPT_FLAGS) -rdynamic -std=gnu11 \ 45 45 -Wall -Werror -fno-omit-frame-pointer \ 46 46 $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \ 47 47 -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ ··· 53 53 PCAP_LIBS := $(shell $(PKG_CONFIG) --libs libpcap 2>/dev/null) 54 54 LDLIBS += $(PCAP_LIBS) 55 55 CFLAGS += $(PCAP_CFLAGS) 56 56 - 57 57 - # The following tests perform type punning and they may break strict 58 58 - # aliasing rules, which are exploited by both GCC and clang by default 59 59 - # while optimizing. This can lead to broken programs. 60 60 - progs/bind4_prog.c-CFLAGS := -fno-strict-aliasing 61 61 - progs/bind6_prog.c-CFLAGS := -fno-strict-aliasing 62 62 - progs/dynptr_fail.c-CFLAGS := -fno-strict-aliasing 63 63 - progs/linked_list_fail.c-CFLAGS := -fno-strict-aliasing 64 64 - progs/map_kptr_fail.c-CFLAGS := -fno-strict-aliasing 65 65 - progs/syscall.c-CFLAGS := -fno-strict-aliasing 66 66 - progs/test_pkt_md_access.c-CFLAGS := -fno-strict-aliasing 67 67 - progs/test_sk_lookup.c-CFLAGS := -fno-strict-aliasing 68 68 - progs/timer_crash.c-CFLAGS := -fno-strict-aliasing 69 69 - progs/test_global_func9.c-CFLAGS := -fno-strict-aliasing 70 70 - progs/verifier_nocsr.c-CFLAGS := -fno-strict-aliasing 71 56 72 57 # Some utility functions use LLVM libraries 73 58 jit_disasm_helpers.c-CFLAGS = $(LLVM_CFLAGS) ··· 88 103 progs/btf_dump_test_case_padding.c-bpf_gcc-CFLAGS := -Wno-error 89 104 progs/btf_dump_test_case_syntax.c-bpf_gcc-CFLAGS := -Wno-error 90 105 91 91 - # The following tests do type-punning, via the __imm_insn macro, from 92 92 - # `struct bpf_insn' to long and then uses the value. This triggers an 93 93 - # "is used uninitialized" warning in GCC due to strict-aliasing 94 94 - # rules. 95 95 - progs/verifier_ref_tracking.c-bpf_gcc-CFLAGS := -fno-strict-aliasing 96 96 - progs/verifier_unpriv.c-bpf_gcc-CFLAGS := -fno-strict-aliasing 97 97 - progs/verifier_cgroup_storage.c-bpf_gcc-CFLAGS := -fno-strict-aliasing 98 98 - progs/verifier_ld_ind.c-bpf_gcc-CFLAGS := -fno-strict-aliasing 99 99 - progs/verifier_map_ret_val.c-bpf_gcc-CFLAGS := -fno-strict-aliasing 100 100 - progs/verifier_spill_fill.c-bpf_gcc-CFLAGS := -fno-strict-aliasing 101 101 - progs/verifier_subprog_precision.c-bpf_gcc-CFLAGS := -fno-strict-aliasing 102 102 - progs/verifier_uninit.c-bpf_gcc-CFLAGS := -fno-strict-aliasing 103 106 endif 104 107 105 108 ifneq ($(CLANG_CPUV4),) ··· 100 127 101 128 # Order correspond to 'make run_tests' order 102 129 TEST_PROGS := test_kmod.sh \ 103 103 - test_xdp_redirect.sh \ 104 130 test_xdp_redirect_multi.sh \ 105 131 test_tunnel.sh \ 106 132 test_lwt_seg6local.sh \ 107 133 test_lirc_mode2.sh \ 108 108 - test_flow_dissector.sh \ 109 134 test_xdp_vlan_mode_generic.sh \ 110 135 test_xdp_vlan_mode_native.sh \ 111 136 test_lwt_ip_encap.sh \ ··· 121 150 with_tunnels.sh ima_setup.sh verify_sig_setup.sh \ 122 151 test_xdp_vlan.sh test_bpftool.py 123 152 153 153 + TEST_KMODS := bpf_testmod.ko bpf_test_no_cfi.ko bpf_test_modorder_x.ko \ 154 154 + bpf_test_modorder_y.ko 155 155 + TEST_KMOD_TARGETS = $(addprefix $(OUTPUT)/,$(TEST_KMODS)) 156 156 + 124 157 # Compile but not part of 'make run_tests' 125 158 TEST_GEN_PROGS_EXTENDED = \ 126 159 bench \ 127 127 - bpf_testmod.ko \ 128 128 - bpf_test_modorder_x.ko \ 129 129 - bpf_test_modorder_y.ko \ 130 130 - bpf_test_no_cfi.ko \ 131 160 flow_dissector_load \ 132 161 runqslower \ 133 162 test_cpp \ 134 134 - test_flow_dissector \ 135 163 test_lirc_mode2_user \ 136 164 veristat \ 137 165 xdp_features \ ··· 153 183 $(Q)$(RM) -r $(TEST_GEN_PROGS) 154 184 $(Q)$(RM) -r $(TEST_GEN_PROGS_EXTENDED) 155 185 $(Q)$(RM) -r $(TEST_GEN_FILES) 186 186 + $(Q)$(RM) -r $(TEST_KMODS) 156 187 $(Q)$(RM) -r $(EXTRA_CLEAN) 157 157 - $(Q)$(MAKE) -C bpf_testmod clean 188 188 + $(Q)$(MAKE) -C test_kmods clean 158 189 $(Q)$(MAKE) docs-clean 159 190 endef 160 191 ··· 173 202 $(let OUTPUT,$(OUTPUT)/,\ 174 203 $(eval include ../../../build/Makefile.feature)) 175 204 else 176 176 - OUTPUT := $(OUTPUT)/ 205 205 + override OUTPUT := $(OUTPUT)/ 177 206 $(eval include ../../../build/Makefile.feature) 178 178 - OUTPUT := $(patsubst %/,%,$(OUTPUT)) 207 207 + override OUTPUT := $(patsubst %/,%,$(OUTPUT)) 179 208 endif 180 209 endif 181 210 ··· 221 250 # to build individual tests. 222 251 # NOTE: Semicolon at the end is critical to override lib.mk's default static 223 252 # rule for binaries. 224 224 - $(notdir $(TEST_GEN_PROGS) \ 253 253 + $(notdir $(TEST_GEN_PROGS) $(TEST_KMODS) \ 225 254 $(TEST_GEN_PROGS_EXTENDED)): %: $(OUTPUT)/% ; 226 255 227 256 # sort removes libbpf duplicates when not cross-building ··· 275 304 $< -o $@ \ 276 305 $(shell $(PKG_CONFIG) --libs libcrypto 2> /dev/null || echo -lcrypto) 277 306 278 278 - $(OUTPUT)/bpf_testmod.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_testmod/Makefile bpf_testmod/*.[ch]) 279 279 - $(call msg,MOD,,$@) 280 280 - $(Q)$(RM) bpf_testmod/bpf_testmod.ko # force re-compilation 281 281 - $(Q)$(MAKE) $(submake_extras) -C bpf_testmod \ 282 282 - RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \ 307 307 + # This should really be a grouped target, but make versions before 4.3 don't 308 308 + # support that for regular rules. However, pattern matching rules are implicitly 309 309 + # treated as grouped even with older versions of make, so as a workaround, the 310 310 + # subst() turns the rule into a pattern matching rule 311 311 + $(addprefix test_kmods/,$(subst .ko,%ko,$(TEST_KMODS))): $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard test_kmods/Makefile test_kmods/*.[ch]) 312 312 + $(Q)$(RM) test_kmods/*.ko test_kmods/*.mod.o # force re-compilation 313 313 + $(Q)$(MAKE) $(submake_extras) -C test_kmods \ 314 314 + RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \ 283 315 EXTRA_CFLAGS='' EXTRA_LDFLAGS='' 284 284 - $(Q)cp bpf_testmod/bpf_testmod.ko $@ 285 316 286 286 - $(OUTPUT)/bpf_test_no_cfi.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_no_cfi/Makefile bpf_test_no_cfi/*.[ch]) 317 317 + $(TEST_KMOD_TARGETS): $(addprefix test_kmods/,$(TEST_KMODS)) 287 318 $(call msg,MOD,,$@) 288 288 - $(Q)$(RM) bpf_test_no_cfi/bpf_test_no_cfi.ko # force re-compilation 289 289 - $(Q)$(MAKE) $(submake_extras) -C bpf_test_no_cfi \ 290 290 - RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \ 291 291 - EXTRA_CFLAGS='' EXTRA_LDFLAGS='' 292 292 - $(Q)cp bpf_test_no_cfi/bpf_test_no_cfi.ko $@ 293 293 - 294 294 - $(OUTPUT)/bpf_test_modorder_x.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_modorder_x/Makefile bpf_test_modorder_x/*.[ch]) 295 295 - $(call msg,MOD,,$@) 296 296 - $(Q)$(RM) bpf_test_modorder_x/bpf_test_modorder_x.ko # force re-compilation 297 297 - $(Q)$(MAKE) $(submake_extras) -C bpf_test_modorder_x \ 298 298 - RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \ 299 299 - EXTRA_CFLAGS='' EXTRA_LDFLAGS='' 300 300 - $(Q)cp bpf_test_modorder_x/bpf_test_modorder_x.ko $@ 301 301 - 302 302 - $(OUTPUT)/bpf_test_modorder_y.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_modorder_y/Makefile bpf_test_modorder_y/*.[ch]) 303 303 - $(call msg,MOD,,$@) 304 304 - $(Q)$(RM) bpf_test_modorder_y/bpf_test_modorder_y.ko # force re-compilation 305 305 - $(Q)$(MAKE) $(submake_extras) -C bpf_test_modorder_y \ 306 306 - RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \ 307 307 - EXTRA_CFLAGS='' EXTRA_LDFLAGS='' 308 308 - $(Q)cp bpf_test_modorder_y/bpf_test_modorder_y.ko $@ 319 319 + $(Q)cp test_kmods/$(@F) $@ 309 320 310 321 311 322 DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool ··· 432 479 endef 433 480 434 481 # Determine target endianness. 435 435 - IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \ 482 482 + IS_LITTLE_ENDIAN := $(shell $(CC) -dM -E - </dev/null | \ 436 483 grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__') 437 437 - MENDIAN=$(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian) 438 438 - BPF_TARGET_ENDIAN=$(if $(IS_LITTLE_ENDIAN),--target=bpfel,--target=bpfeb) 484 484 + MENDIAN:=$(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian) 485 485 + BPF_TARGET_ENDIAN:=$(if $(IS_LITTLE_ENDIAN),--target=bpfel,--target=bpfeb) 439 486 440 487 ifneq ($(CROSS_COMPILE),) 441 488 CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%)) ··· 445 492 BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \ 446 493 -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR) \ 447 494 -I$(abspath $(OUTPUT)/../usr/include) \ 495 495 + -std=gnu11 \ 496 496 + -fno-strict-aliasing \ 448 497 -Wno-compare-distinct-pointer-types 449 498 # TODO: enable me -Wsign-compare 450 499 ··· 714 759 json_writer.c \ 715 760 flow_dissector_load.h \ 716 761 ip_check_defrag_frags.h 717 717 - TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read $(OUTPUT)/bpf_testmod.ko \ 718 718 - $(OUTPUT)/bpf_test_no_cfi.ko \ 719 719 - $(OUTPUT)/bpf_test_modorder_x.ko \ 720 720 - $(OUTPUT)/bpf_test_modorder_y.ko \ 762 762 + TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ 721 763 $(OUTPUT)/liburandom_read.so \ 722 764 $(OUTPUT)/xdp_synproxy \ 723 765 $(OUTPUT)/sign-file \ 724 766 $(OUTPUT)/uprobe_multi \ 767 767 + $(TEST_KMOD_TARGETS) \ 725 768 ima_setup.sh \ 726 769 verify_sig_setup.sh \ 727 770 $(wildcard progs/btf_dump_test_case_*.c) \ ··· 786 833 $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ 787 834 788 835 # Make sure we are able to include and link libbpf against c++. 836 836 + CXXFLAGS += $(CFLAGS) 837 837 + CXXFLAGS := $(subst -D_GNU_SOURCE=,,$(CXXFLAGS)) 838 838 + CXXFLAGS := $(subst -std=gnu11,-std=gnu++11,$(CXXFLAGS)) 789 839 $(OUTPUT)/test_cpp: test_cpp.cpp $(OUTPUT)/test_core_extern.skel.h $(BPFOBJ) 790 840 $(call msg,CXX,,$@) 791 791 - $(Q)$(CXX) $(subst -D_GNU_SOURCE=,,$(CFLAGS)) $(filter %.a %.o %.cpp,$^) $(LDLIBS) -o $@ 841 841 + $(Q)$(CXX) $(CXXFLAGS) $(filter %.a %.o %.cpp,$^) $(LDLIBS) -o $@ 792 842 793 843 # Benchmark runner 794 844 $(OUTPUT)/bench_%.o: benchs/bench_%.c bench.h $(BPFOBJ) ··· 849 893 850 894 EXTRA_CLEAN := $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) \ 851 895 prog_tests/tests.h map_tests/tests.h verifier/tests.h \ 852 852 - feature bpftool \ 896 896 + feature bpftool $(TEST_KMOD_TARGETS) \ 853 897 $(addprefix $(OUTPUT)/,*.o *.d *.skel.h *.lskel.h *.subskel.h \ 854 854 - no_alu32 cpuv4 bpf_gcc bpf_testmod.ko \ 855 855 - bpf_test_no_cfi.ko \ 856 856 - bpf_test_modorder_x.ko \ 857 857 - bpf_test_modorder_y.ko \ 898 898 + no_alu32 cpuv4 bpf_gcc \ 858 899 liburandom_read.so) \ 859 900 $(OUTPUT)/FEATURE-DUMP.selftests 860 901

-19

tools/testing/selftests/bpf/bpf_test_modorder_x/Makefile

reviewed

··· 1 1 - BPF_TESTMOD_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) 2 2 - KDIR ?= $(abspath $(BPF_TESTMOD_DIR)/../../../../..) 3 3 - 4 4 - ifeq ($(V),1) 5 5 - Q = 6 6 - else 7 7 - Q = @ 8 8 - endif 9 9 - 10 10 - MODULES = bpf_test_modorder_x.ko 11 11 - 12 12 - obj-m += bpf_test_modorder_x.o 13 13 - 14 14 - all: 15 15 - +$(Q)make -C $(KDIR) M=$(BPF_TESTMOD_DIR) modules 16 16 - 17 17 - clean: 18 18 - +$(Q)make -C $(KDIR) M=$(BPF_TESTMOD_DIR) clean 19 19 -

tools/testing/selftests/bpf/bpf_test_modorder_x/bpf_test_modorder_x.c tools/testing/selftests/bpf/test_kmods/bpf_test_modorder_x.c

reviewed

-19

tools/testing/selftests/bpf/bpf_test_modorder_y/Makefile

reviewed

··· 1 1 - BPF_TESTMOD_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) 2 2 - KDIR ?= $(abspath $(BPF_TESTMOD_DIR)/../../../../..) 3 3 - 4 4 - ifeq ($(V),1) 5 5 - Q = 6 6 - else 7 7 - Q = @ 8 8 - endif 9 9 - 10 10 - MODULES = bpf_test_modorder_y.ko 11 11 - 12 12 - obj-m += bpf_test_modorder_y.o 13 13 - 14 14 - all: 15 15 - +$(Q)make -C $(KDIR) M=$(BPF_TESTMOD_DIR) modules 16 16 - 17 17 - clean: 18 18 - +$(Q)make -C $(KDIR) M=$(BPF_TESTMOD_DIR) clean 19 19 -

tools/testing/selftests/bpf/bpf_test_modorder_y/bpf_test_modorder_y.c tools/testing/selftests/bpf/test_kmods/bpf_test_modorder_y.c

reviewed

-19

tools/testing/selftests/bpf/bpf_test_no_cfi/Makefile

reviewed

··· 1 1 - BPF_TEST_NO_CFI_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) 2 2 - KDIR ?= $(abspath $(BPF_TEST_NO_CFI_DIR)/../../../../..) 3 3 - 4 4 - ifeq ($(V),1) 5 5 - Q = 6 6 - else 7 7 - Q = @ 8 8 - endif 9 9 - 10 10 - MODULES = bpf_test_no_cfi.ko 11 11 - 12 12 - obj-m += bpf_test_no_cfi.o 13 13 - 14 14 - all: 15 15 - +$(Q)make -C $(KDIR) M=$(BPF_TEST_NO_CFI_DIR) modules 16 16 - 17 17 - clean: 18 18 - +$(Q)make -C $(KDIR) M=$(BPF_TEST_NO_CFI_DIR) clean 19 19 -

tools/testing/selftests/bpf/bpf_test_no_cfi/bpf_test_no_cfi.c tools/testing/selftests/bpf/test_kmods/bpf_test_no_cfi.c

reviewed

tools/testing/selftests/bpf/bpf_testmod/.gitignore tools/testing/selftests/bpf/test_kmods/.gitignore

reviewed

-20

tools/testing/selftests/bpf/bpf_testmod/Makefile

reviewed

··· 1 1 - BPF_TESTMOD_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) 2 2 - KDIR ?= $(abspath $(BPF_TESTMOD_DIR)/../../../../..) 3 3 - 4 4 - ifeq ($(V),1) 5 5 - Q = 6 6 - else 7 7 - Q = @ 8 8 - endif 9 9 - 10 10 - MODULES = bpf_testmod.ko 11 11 - 12 12 - obj-m += bpf_testmod.o 13 13 - CFLAGS_bpf_testmod.o = -I$(src) 14 14 - 15 15 - all: 16 16 - +$(Q)make -C $(KDIR) M=$(BPF_TESTMOD_DIR) modules 17 17 - 18 18 - clean: 19 19 - +$(Q)make -C $(KDIR) M=$(BPF_TESTMOD_DIR) clean 20 20 -

tools/testing/selftests/bpf/bpf_testmod/bpf_testmod-events.h tools/testing/selftests/bpf/test_kmods/bpf_testmod-events.h

reviewed

tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c tools/testing/selftests/bpf/test_kmods/bpf_testmod.c

reviewed

tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h tools/testing/selftests/bpf/test_kmods/bpf_testmod.h

reviewed

tools/testing/selftests/bpf/bpf_testmod/bpf_testmod_kfunc.h tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h

reviewed

+1

tools/testing/selftests/bpf/config

reviewed

··· 58 58 CONFIG_MPLS_IPTUNNEL=y 59 59 CONFIG_MPLS_ROUTING=y 60 60 CONFIG_MPTCP=y 61 61 + CONFIG_NET_ACT_GACT=y 61 62 CONFIG_NET_ACT_SKBMOD=y 62 63 CONFIG_NET_CLS=y 63 64 CONFIG_NET_CLS_ACT=y

+1 -1

tools/testing/selftests/bpf/network_helpers.c

reviewed

··· 21 21 #include <linux/limits.h> 22 22 23 23 #include <linux/ip.h> 24 24 - #include <linux/udp.h> 24 24 + #include <netinet/udp.h> 25 25 #include <netinet/tcp.h> 26 26 #include <net/if.h> 27 27

+96

tools/testing/selftests/bpf/network_helpers.h

reviewed

··· 14 14 #include <linux/sockios.h> 15 15 #include <linux/err.h> 16 16 #include <netinet/tcp.h> 17 17 + #include <netinet/udp.h> 17 18 #include <bpf/bpf_endian.h> 18 19 #include <net/if.h> 19 20 ··· 106 105 return (__u16)~csum; 107 106 } 108 107 108 108 + static __wsum csum_partial(const void *buf, int len, __wsum sum) 109 109 + { 110 110 + __u16 *p = (__u16 *)buf; 111 111 + int num_u16 = len >> 1; 112 112 + int i; 113 113 + 114 114 + for (i = 0; i < num_u16; i++) 115 115 + sum += p[i]; 116 116 + 117 117 + return sum; 118 118 + } 119 119 + 120 120 + static inline __sum16 build_ip_csum(struct iphdr *iph) 121 121 + { 122 122 + __u32 sum = 0; 123 123 + __u16 *p; 124 124 + 125 125 + iph->check = 0; 126 126 + p = (void *)iph; 127 127 + sum = csum_partial(p, iph->ihl << 2, 0); 128 128 + 129 129 + return csum_fold(sum); 130 130 + } 131 131 + 132 132 + /** 133 133 + * csum_tcpudp_magic - compute IP pseudo-header checksum 134 134 + * 135 135 + * Compute the IPv4 pseudo header checksum. The helper can take a 136 136 + * accumulated sum from the transport layer to accumulate it and directly 137 137 + * return the transport layer 138 138 + * 139 139 + * @saddr: IP source address 140 140 + * @daddr: IP dest address 141 141 + * @len: IP data size 142 142 + * @proto: transport layer protocol 143 143 + * @csum: The accumulated partial sum to add to the computation 144 144 + * 145 145 + * Returns the folded sum 146 146 + */ 109 147 static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, 110 148 __u32 len, __u8 proto, 111 149 __wsum csum) ··· 160 120 return csum_fold((__u32)s); 161 121 } 162 122 123 123 + /** 124 124 + * csum_ipv6_magic - compute IPv6 pseudo-header checksum 125 125 + * 126 126 + * Compute the ipv6 pseudo header checksum. The helper can take a 127 127 + * accumulated sum from the transport layer to accumulate it and directly 128 128 + * return the transport layer 129 129 + * 130 130 + * @saddr: IPv6 source address 131 131 + * @daddr: IPv6 dest address 132 132 + * @len: IPv6 data size 133 133 + * @proto: transport layer protocol 134 134 + * @csum: The accumulated partial sum to add to the computation 135 135 + * 136 136 + * Returns the folded sum 137 137 + */ 163 138 static inline __sum16 csum_ipv6_magic(const struct in6_addr *saddr, 164 139 const struct in6_addr *daddr, 165 140 __u32 len, __u8 proto, ··· 192 137 s = (s & 0xffffffff) + (s >> 32); 193 138 194 139 return csum_fold((__u32)s); 140 140 + } 141 141 + 142 142 + /** 143 143 + * build_udp_v4_csum - compute UDP checksum for UDP over IPv4 144 144 + * 145 145 + * Compute the checksum to embed in UDP header, composed of the sum of IP 146 146 + * pseudo-header checksum, UDP header checksum and UDP data checksum 147 147 + * @iph IP header 148 148 + * @udph UDP header, which must be immediately followed by UDP data 149 149 + * 150 150 + * Returns the total checksum 151 151 + */ 152 152 + 153 153 + static inline __sum16 build_udp_v4_csum(const struct iphdr *iph, 154 154 + const struct udphdr *udph) 155 155 + { 156 156 + unsigned long sum; 157 157 + 158 158 + sum = csum_partial(udph, ntohs(udph->len), 0); 159 159 + return csum_tcpudp_magic(iph->saddr, iph->daddr, ntohs(udph->len), 160 160 + IPPROTO_UDP, sum); 161 161 + } 162 162 + 163 163 + /** 164 164 + * build_udp_v6_csum - compute UDP checksum for UDP over IPv6 165 165 + * 166 166 + * Compute the checksum to embed in UDP header, composed of the sum of IPv6 167 167 + * pseudo-header checksum, UDP header checksum and UDP data checksum 168 168 + * @ip6h IPv6 header 169 169 + * @udph UDP header, which must be immediately followed by UDP data 170 170 + * 171 171 + * Returns the total checksum 172 172 + */ 173 173 + static inline __sum16 build_udp_v6_csum(const struct ipv6hdr *ip6h, 174 174 + const struct udphdr *udph) 175 175 + { 176 176 + unsigned long sum; 177 177 + 178 178 + sum = csum_partial(udph, ntohs(udph->len), 0); 179 179 + return csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, ntohs(udph->len), 180 180 + IPPROTO_UDP, sum); 195 181 } 196 182 197 183 struct tmonitor_ctx;

+74 -2

tools/testing/selftests/bpf/prog_tests/btf_distill.c

reviewed

··· 385 385 "[2] INT 'int' size=8 bits_offset=0 nr_bits=64 encoding=SIGNED"); 386 386 btf5 = btf__new_empty(); 387 387 if (!ASSERT_OK_PTR(btf5, "empty_reloc_btf")) 388 388 - return; 388 388 + goto cleanup; 389 389 btf__add_int(btf5, "int", 4, BTF_INT_SIGNED); /* [1] int */ 390 390 VALIDATE_RAW_BTF( 391 391 btf5, ··· 478 478 "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED"); 479 479 btf5 = btf__new_empty(); 480 480 if (!ASSERT_OK_PTR(btf5, "empty_reloc_btf")) 481 481 - return; 481 481 + goto cleanup; 482 482 btf__add_int(btf5, "int", 4, BTF_INT_SIGNED); /* [1] int */ 483 483 btf__add_int(btf5, "int", 4, BTF_INT_SIGNED); /* [2] int */ 484 484 VALIDATE_RAW_BTF( ··· 601 601 btf__free(base); 602 602 } 603 603 604 604 + /* If a needed composite type, which is the member of composite type 605 605 + * in the split BTF, has a different size in the base BTF we wish to 606 606 + * relocate with, btf__relocate() should error out. 607 607 + */ 608 608 + static void test_distilled_base_embedded_err(void) 609 609 + { 610 610 + struct btf *btf1 = NULL, *btf2 = NULL, *btf3 = NULL, *btf4 = NULL, *btf5 = NULL; 611 611 + 612 612 + btf1 = btf__new_empty(); 613 613 + if (!ASSERT_OK_PTR(btf1, "empty_main_btf")) 614 614 + return; 615 615 + 616 616 + btf__add_int(btf1, "int", 4, BTF_INT_SIGNED); /* [1] int */ 617 617 + btf__add_struct(btf1, "s1", 4); /* [2] struct s1 { */ 618 618 + btf__add_field(btf1, "f1", 1, 0, 0); /* int f1; */ 619 619 + /* } */ 620 620 + VALIDATE_RAW_BTF( 621 621 + btf1, 622 622 + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", 623 623 + "[2] STRUCT 's1' size=4 vlen=1\n" 624 624 + "\t'f1' type_id=1 bits_offset=0"); 625 625 + 626 626 + btf2 = btf__new_empty_split(btf1); 627 627 + if (!ASSERT_OK_PTR(btf2, "empty_split_btf")) 628 628 + goto cleanup; 629 629 + 630 630 + btf__add_struct(btf2, "with_embedded", 8); /* [3] struct with_embedded { */ 631 631 + btf__add_field(btf2, "e1", 2, 0, 0); /* struct s1 e1; */ 632 632 + /* } */ 633 633 + 634 634 + VALIDATE_RAW_BTF( 635 635 + btf2, 636 636 + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", 637 637 + "[2] STRUCT 's1' size=4 vlen=1\n" 638 638 + "\t'f1' type_id=1 bits_offset=0", 639 639 + "[3] STRUCT 'with_embedded' size=8 vlen=1\n" 640 640 + "\t'e1' type_id=2 bits_offset=0"); 641 641 + 642 642 + if (!ASSERT_EQ(0, btf__distill_base(btf2, &btf3, &btf4), 643 643 + "distilled_base") || 644 644 + !ASSERT_OK_PTR(btf3, "distilled_base") || 645 645 + !ASSERT_OK_PTR(btf4, "distilled_split") || 646 646 + !ASSERT_EQ(2, btf__type_cnt(btf3), "distilled_base_type_cnt")) 647 647 + goto cleanup; 648 648 + 649 649 + VALIDATE_RAW_BTF( 650 650 + btf4, 651 651 + "[1] STRUCT 's1' size=4 vlen=0", 652 652 + "[2] STRUCT 'with_embedded' size=8 vlen=1\n" 653 653 + "\t'e1' type_id=1 bits_offset=0"); 654 654 + 655 655 + btf5 = btf__new_empty(); 656 656 + if (!ASSERT_OK_PTR(btf5, "empty_reloc_btf")) 657 657 + goto cleanup; 658 658 + 659 659 + btf__add_int(btf5, "int", 4, BTF_INT_SIGNED); /* [1] int */ 660 660 + /* struct with the same name but different size */ 661 661 + btf__add_struct(btf5, "s1", 8); /* [2] struct s1 { */ 662 662 + btf__add_field(btf5, "f1", 1, 0, 0); /* int f1; */ 663 663 + /* } */ 664 664 + 665 665 + ASSERT_EQ(btf__relocate(btf4, btf5), -EINVAL, "relocate_split"); 666 666 + cleanup: 667 667 + btf__free(btf5); 668 668 + btf__free(btf4); 669 669 + btf__free(btf3); 670 670 + btf__free(btf2); 671 671 + btf__free(btf1); 672 672 + } 673 673 + 604 674 void test_btf_distill(void) 605 675 { 606 676 if (test__start_subtest("distilled_base")) ··· 683 613 test_distilled_base_multi_err(); 684 614 if (test__start_subtest("distilled_base_multi_err2")) 685 615 test_distilled_base_multi_err2(); 616 616 + if (test__start_subtest("distilled_base_embedded_err")) 617 617 + test_distilled_base_embedded_err(); 686 618 if (test__start_subtest("distilled_base_vmlinux")) 687 619 test_distilled_base_vmlinux(); 688 620 if (test__start_subtest("distilled_endianness"))

+28

tools/testing/selftests/bpf/prog_tests/cgroup_skb_direct_packet_access.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + #include <test_progs.h> 4 4 + #include "cgroup_skb_direct_packet_access.skel.h" 5 5 + 6 6 + void test_cgroup_skb_prog_run_direct_packet_access(void) 7 7 + { 8 8 + int err; 9 9 + struct cgroup_skb_direct_packet_access *skel; 10 10 + char test_skb[64] = {}; 11 11 + 12 12 + LIBBPF_OPTS(bpf_test_run_opts, topts, 13 13 + .data_in = test_skb, 14 14 + .data_size_in = sizeof(test_skb), 15 15 + ); 16 16 + 17 17 + skel = cgroup_skb_direct_packet_access__open_and_load(); 18 18 + if (!ASSERT_OK_PTR(skel, "cgroup_skb_direct_packet_access__open_and_load")) 19 19 + return; 20 20 + 21 21 + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.direct_packet_access), &topts); 22 22 + ASSERT_OK(err, "bpf_prog_test_run_opts err"); 23 23 + ASSERT_EQ(topts.retval, 1, "retval"); 24 24 + 25 25 + ASSERT_NEQ(skel->bss->data_end, 0, "data_end"); 26 26 + 27 27 + cgroup_skb_direct_packet_access__destroy(skel); 28 28 + }

+1 -1

tools/testing/selftests/bpf/prog_tests/core_reloc.c

reviewed

··· 2 2 #define _GNU_SOURCE 3 3 #include <test_progs.h> 4 4 #include "progs/core_reloc_types.h" 5 5 - #include "bpf_testmod/bpf_testmod.h" 5 5 + #include "test_kmods/bpf_testmod.h" 6 6 #include <linux/limits.h> 7 7 #include <sys/mman.h> 8 8 #include <sys/syscall.h>

+441

tools/testing/selftests/bpf/prog_tests/fd_array.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + #include <test_progs.h> 4 4 + 5 5 + #include <linux/btf.h> 6 6 + #include <bpf/bpf.h> 7 7 + 8 8 + #include "../test_btf.h" 9 9 + 10 10 + static inline int new_map(void) 11 11 + { 12 12 + const char *name = NULL; 13 13 + __u32 max_entries = 1; 14 14 + __u32 value_size = 8; 15 15 + __u32 key_size = 4; 16 16 + 17 17 + return bpf_map_create(BPF_MAP_TYPE_ARRAY, name, 18 18 + key_size, value_size, 19 19 + max_entries, NULL); 20 20 + } 21 21 + 22 22 + static int new_btf(void) 23 23 + { 24 24 + struct btf_blob { 25 25 + struct btf_header btf_hdr; 26 26 + __u32 types[8]; 27 27 + __u32 str; 28 28 + } raw_btf = { 29 29 + .btf_hdr = { 30 30 + .magic = BTF_MAGIC, 31 31 + .version = BTF_VERSION, 32 32 + .hdr_len = sizeof(struct btf_header), 33 33 + .type_len = sizeof(raw_btf.types), 34 34 + .str_off = offsetof(struct btf_blob, str) - offsetof(struct btf_blob, types), 35 35 + .str_len = sizeof(raw_btf.str), 36 36 + }, 37 37 + .types = { 38 38 + /* long */ 39 39 + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 64, 8), /* [1] */ 40 40 + /* unsigned long */ 41 41 + BTF_TYPE_INT_ENC(0, 0, 0, 64, 8), /* [2] */ 42 42 + }, 43 43 + }; 44 44 + 45 45 + return bpf_btf_load(&raw_btf, sizeof(raw_btf), NULL); 46 46 + } 47 47 + 48 48 + #define Close(FD) do { \ 49 49 + if ((FD) >= 0) { \ 50 50 + close(FD); \ 51 51 + FD = -1; \ 52 52 + } \ 53 53 + } while(0) 54 54 + 55 55 + static bool map_exists(__u32 id) 56 56 + { 57 57 + int fd; 58 58 + 59 59 + fd = bpf_map_get_fd_by_id(id); 60 60 + if (fd >= 0) { 61 61 + close(fd); 62 62 + return true; 63 63 + } 64 64 + return false; 65 65 + } 66 66 + 67 67 + static bool btf_exists(__u32 id) 68 68 + { 69 69 + int fd; 70 70 + 71 71 + fd = bpf_btf_get_fd_by_id(id); 72 72 + if (fd >= 0) { 73 73 + close(fd); 74 74 + return true; 75 75 + } 76 76 + return false; 77 77 + } 78 78 + 79 79 + static inline int bpf_prog_get_map_ids(int prog_fd, __u32 *nr_map_ids, __u32 *map_ids) 80 80 + { 81 81 + __u32 len = sizeof(struct bpf_prog_info); 82 82 + struct bpf_prog_info info; 83 83 + int err; 84 84 + 85 85 + memset(&info, 0, len); 86 86 + info.nr_map_ids = *nr_map_ids, 87 87 + info.map_ids = ptr_to_u64(map_ids), 88 88 + 89 89 + err = bpf_prog_get_info_by_fd(prog_fd, &info, &len); 90 90 + if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd")) 91 91 + return -1; 92 92 + 93 93 + *nr_map_ids = info.nr_map_ids; 94 94 + 95 95 + return 0; 96 96 + } 97 97 + 98 98 + static int __load_test_prog(int map_fd, const int *fd_array, int fd_array_cnt) 99 99 + { 100 100 + /* A trivial program which uses one map */ 101 101 + struct bpf_insn insns[] = { 102 102 + BPF_LD_MAP_FD(BPF_REG_1, map_fd), 103 103 + BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0), 104 104 + BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), 105 105 + BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8), 106 106 + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 107 107 + BPF_MOV64_IMM(BPF_REG_0, 0), 108 108 + BPF_EXIT_INSN(), 109 109 + }; 110 110 + LIBBPF_OPTS(bpf_prog_load_opts, opts); 111 111 + 112 112 + opts.fd_array = fd_array; 113 113 + opts.fd_array_cnt = fd_array_cnt; 114 114 + 115 115 + return bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, ARRAY_SIZE(insns), &opts); 116 116 + } 117 117 + 118 118 + static int load_test_prog(const int *fd_array, int fd_array_cnt) 119 119 + { 120 120 + int map_fd; 121 121 + int ret; 122 122 + 123 123 + map_fd = new_map(); 124 124 + if (!ASSERT_GE(map_fd, 0, "new_map")) 125 125 + return map_fd; 126 126 + 127 127 + ret = __load_test_prog(map_fd, fd_array, fd_array_cnt); 128 128 + close(map_fd); 129 129 + return ret; 130 130 + } 131 131 + 132 132 + static bool check_expected_map_ids(int prog_fd, int expected, __u32 *map_ids, __u32 *nr_map_ids) 133 133 + { 134 134 + int err; 135 135 + 136 136 + err = bpf_prog_get_map_ids(prog_fd, nr_map_ids, map_ids); 137 137 + if (!ASSERT_OK(err, "bpf_prog_get_map_ids")) 138 138 + return false; 139 139 + if (!ASSERT_EQ(*nr_map_ids, expected, "unexpected nr_map_ids")) 140 140 + return false; 141 141 + 142 142 + return true; 143 143 + } 144 144 + 145 145 + /* 146 146 + * Load a program, which uses one map. No fd_array maps are present. 147 147 + * On return only one map is expected to be bound to prog. 148 148 + */ 149 149 + static void check_fd_array_cnt__no_fd_array(void) 150 150 + { 151 151 + __u32 map_ids[16]; 152 152 + __u32 nr_map_ids; 153 153 + int prog_fd = -1; 154 154 + 155 155 + prog_fd = load_test_prog(NULL, 0); 156 156 + if (!ASSERT_GE(prog_fd, 0, "BPF_PROG_LOAD")) 157 157 + return; 158 158 + nr_map_ids = ARRAY_SIZE(map_ids); 159 159 + check_expected_map_ids(prog_fd, 1, map_ids, &nr_map_ids); 160 160 + close(prog_fd); 161 161 + } 162 162 + 163 163 + /* 164 164 + * Load a program, which uses one map, and pass two extra, non-equal, maps in 165 165 + * fd_array with fd_array_cnt=2. On return three maps are expected to be bound 166 166 + * to the program. 167 167 + */ 168 168 + static void check_fd_array_cnt__fd_array_ok(void) 169 169 + { 170 170 + int extra_fds[2] = { -1, -1 }; 171 171 + __u32 map_ids[16]; 172 172 + __u32 nr_map_ids; 173 173 + int prog_fd = -1; 174 174 + 175 175 + extra_fds[0] = new_map(); 176 176 + if (!ASSERT_GE(extra_fds[0], 0, "new_map")) 177 177 + goto cleanup; 178 178 + extra_fds[1] = new_map(); 179 179 + if (!ASSERT_GE(extra_fds[1], 0, "new_map")) 180 180 + goto cleanup; 181 181 + prog_fd = load_test_prog(extra_fds, 2); 182 182 + if (!ASSERT_GE(prog_fd, 0, "BPF_PROG_LOAD")) 183 183 + goto cleanup; 184 184 + nr_map_ids = ARRAY_SIZE(map_ids); 185 185 + if (!check_expected_map_ids(prog_fd, 3, map_ids, &nr_map_ids)) 186 186 + goto cleanup; 187 187 + 188 188 + /* maps should still exist when original file descriptors are closed */ 189 189 + Close(extra_fds[0]); 190 190 + Close(extra_fds[1]); 191 191 + if (!ASSERT_EQ(map_exists(map_ids[0]), true, "map_ids[0] should exist")) 192 192 + goto cleanup; 193 193 + if (!ASSERT_EQ(map_exists(map_ids[1]), true, "map_ids[1] should exist")) 194 194 + goto cleanup; 195 195 + 196 196 + /* some fds might be invalid, so ignore return codes */ 197 197 + cleanup: 198 198 + Close(extra_fds[1]); 199 199 + Close(extra_fds[0]); 200 200 + Close(prog_fd); 201 201 + } 202 202 + 203 203 + /* 204 204 + * Load a program with a few extra maps duplicated in the fd_array. 205 205 + * After the load maps should only be referenced once. 206 206 + */ 207 207 + static void check_fd_array_cnt__duplicated_maps(void) 208 208 + { 209 209 + int extra_fds[4] = { -1, -1, -1, -1 }; 210 210 + __u32 map_ids[16]; 211 211 + __u32 nr_map_ids; 212 212 + int prog_fd = -1; 213 213 + 214 214 + extra_fds[0] = extra_fds[2] = new_map(); 215 215 + if (!ASSERT_GE(extra_fds[0], 0, "new_map")) 216 216 + goto cleanup; 217 217 + extra_fds[1] = extra_fds[3] = new_map(); 218 218 + if (!ASSERT_GE(extra_fds[1], 0, "new_map")) 219 219 + goto cleanup; 220 220 + prog_fd = load_test_prog(extra_fds, 4); 221 221 + if (!ASSERT_GE(prog_fd, 0, "BPF_PROG_LOAD")) 222 222 + goto cleanup; 223 223 + nr_map_ids = ARRAY_SIZE(map_ids); 224 224 + if (!check_expected_map_ids(prog_fd, 3, map_ids, &nr_map_ids)) 225 225 + goto cleanup; 226 226 + 227 227 + /* maps should still exist when original file descriptors are closed */ 228 228 + Close(extra_fds[0]); 229 229 + Close(extra_fds[1]); 230 230 + if (!ASSERT_EQ(map_exists(map_ids[0]), true, "map should exist")) 231 231 + goto cleanup; 232 232 + if (!ASSERT_EQ(map_exists(map_ids[1]), true, "map should exist")) 233 233 + goto cleanup; 234 234 + 235 235 + /* some fds might be invalid, so ignore return codes */ 236 236 + cleanup: 237 237 + Close(extra_fds[1]); 238 238 + Close(extra_fds[0]); 239 239 + Close(prog_fd); 240 240 + } 241 241 + 242 242 + /* 243 243 + * Check that if maps which are referenced by a program are 244 244 + * passed in fd_array, then they will be referenced only once 245 245 + */ 246 246 + static void check_fd_array_cnt__referenced_maps_in_fd_array(void) 247 247 + { 248 248 + int extra_fds[1] = { -1 }; 249 249 + __u32 map_ids[16]; 250 250 + __u32 nr_map_ids; 251 251 + int prog_fd = -1; 252 252 + 253 253 + extra_fds[0] = new_map(); 254 254 + if (!ASSERT_GE(extra_fds[0], 0, "new_map")) 255 255 + goto cleanup; 256 256 + prog_fd = __load_test_prog(extra_fds[0], extra_fds, 1); 257 257 + if (!ASSERT_GE(prog_fd, 0, "BPF_PROG_LOAD")) 258 258 + goto cleanup; 259 259 + nr_map_ids = ARRAY_SIZE(map_ids); 260 260 + if (!check_expected_map_ids(prog_fd, 1, map_ids, &nr_map_ids)) 261 261 + goto cleanup; 262 262 + 263 263 + /* map should still exist when original file descriptor is closed */ 264 264 + Close(extra_fds[0]); 265 265 + if (!ASSERT_EQ(map_exists(map_ids[0]), true, "map should exist")) 266 266 + goto cleanup; 267 267 + 268 268 + /* some fds might be invalid, so ignore return codes */ 269 269 + cleanup: 270 270 + Close(extra_fds[0]); 271 271 + Close(prog_fd); 272 272 + } 273 273 + 274 274 + static int get_btf_id_by_fd(int btf_fd, __u32 *id) 275 275 + { 276 276 + struct bpf_btf_info info; 277 277 + __u32 info_len = sizeof(info); 278 278 + int err; 279 279 + 280 280 + memset(&info, 0, info_len); 281 281 + err = bpf_btf_get_info_by_fd(btf_fd, &info, &info_len); 282 282 + if (err) 283 283 + return err; 284 284 + if (id) 285 285 + *id = info.id; 286 286 + return 0; 287 287 + } 288 288 + 289 289 + /* 290 290 + * Check that fd_array operates properly for btfs. Namely, to check that 291 291 + * passing a btf fd in fd_array increases its reference count, do the 292 292 + * following: 293 293 + * 1) Create a new btf, it's referenced only by a file descriptor, so refcnt=1 294 294 + * 2) Load a BPF prog with fd_array[0] = btf_fd; now btf's refcnt=2 295 295 + * 3) Close the btf_fd, now refcnt=1 296 296 + * Wait and check that BTF stil exists. 297 297 + */ 298 298 + static void check_fd_array_cnt__referenced_btfs(void) 299 299 + { 300 300 + int extra_fds[1] = { -1 }; 301 301 + int prog_fd = -1; 302 302 + __u32 btf_id; 303 303 + int tries; 304 304 + int err; 305 305 + 306 306 + extra_fds[0] = new_btf(); 307 307 + if (!ASSERT_GE(extra_fds[0], 0, "new_btf")) 308 308 + goto cleanup; 309 309 + prog_fd = load_test_prog(extra_fds, 1); 310 310 + if (!ASSERT_GE(prog_fd, 0, "BPF_PROG_LOAD")) 311 311 + goto cleanup; 312 312 + 313 313 + /* btf should still exist when original file descriptor is closed */ 314 314 + err = get_btf_id_by_fd(extra_fds[0], &btf_id); 315 315 + if (!ASSERT_GE(err, 0, "get_btf_id_by_fd")) 316 316 + goto cleanup; 317 317 + 318 318 + Close(extra_fds[0]); 319 319 + 320 320 + if (!ASSERT_GE(kern_sync_rcu(), 0, "kern_sync_rcu 1")) 321 321 + goto cleanup; 322 322 + 323 323 + if (!ASSERT_EQ(btf_exists(btf_id), true, "btf should exist")) 324 324 + goto cleanup; 325 325 + 326 326 + Close(prog_fd); 327 327 + 328 328 + /* The program is freed by a workqueue, so no reliable 329 329 + * way to sync, so just wait a bit (max ~1 second). */ 330 330 + for (tries = 100; tries >= 0; tries--) { 331 331 + usleep(1000); 332 332 + 333 333 + if (!btf_exists(btf_id)) 334 334 + break; 335 335 + 336 336 + if (tries) 337 337 + continue; 338 338 + 339 339 + PRINT_FAIL("btf should have been freed"); 340 340 + } 341 341 + 342 342 + /* some fds might be invalid, so ignore return codes */ 343 343 + cleanup: 344 344 + Close(extra_fds[0]); 345 345 + Close(prog_fd); 346 346 + } 347 347 + 348 348 + /* 349 349 + * Test that a program with trash in fd_array can't be loaded: 350 350 + * only map and BTF file descriptors should be accepted. 351 351 + */ 352 352 + static void check_fd_array_cnt__fd_array_with_trash(void) 353 353 + { 354 354 + int extra_fds[3] = { -1, -1, -1 }; 355 355 + int prog_fd = -1; 356 356 + 357 357 + extra_fds[0] = new_map(); 358 358 + if (!ASSERT_GE(extra_fds[0], 0, "new_map")) 359 359 + goto cleanup; 360 360 + extra_fds[1] = new_btf(); 361 361 + if (!ASSERT_GE(extra_fds[1], 0, "new_btf")) 362 362 + goto cleanup; 363 363 + 364 364 + /* trash 1: not a file descriptor */ 365 365 + extra_fds[2] = 0xbeef; 366 366 + prog_fd = load_test_prog(extra_fds, 3); 367 367 + if (!ASSERT_EQ(prog_fd, -EBADF, "prog should have been rejected with -EBADF")) 368 368 + goto cleanup; 369 369 + 370 370 + /* trash 2: not a map or btf */ 371 371 + extra_fds[2] = socket(AF_INET, SOCK_STREAM, 0); 372 372 + if (!ASSERT_GE(extra_fds[2], 0, "socket")) 373 373 + goto cleanup; 374 374 + 375 375 + prog_fd = load_test_prog(extra_fds, 3); 376 376 + if (!ASSERT_EQ(prog_fd, -EINVAL, "prog should have been rejected with -EINVAL")) 377 377 + goto cleanup; 378 378 + 379 379 + /* Validate that the prog is ok if trash is removed */ 380 380 + Close(extra_fds[2]); 381 381 + extra_fds[2] = new_btf(); 382 382 + if (!ASSERT_GE(extra_fds[2], 0, "new_btf")) 383 383 + goto cleanup; 384 384 + 385 385 + prog_fd = load_test_prog(extra_fds, 3); 386 386 + if (!ASSERT_GE(prog_fd, 0, "prog should have been loaded")) 387 387 + goto cleanup; 388 388 + 389 389 + /* some fds might be invalid, so ignore return codes */ 390 390 + cleanup: 391 391 + Close(extra_fds[2]); 392 392 + Close(extra_fds[1]); 393 393 + Close(extra_fds[0]); 394 394 + } 395 395 + 396 396 + /* 397 397 + * Test that a program with too big fd_array can't be loaded. 398 398 + */ 399 399 + static void check_fd_array_cnt__fd_array_too_big(void) 400 400 + { 401 401 + int extra_fds[65]; 402 402 + int prog_fd = -1; 403 403 + int i; 404 404 + 405 405 + for (i = 0; i < 65; i++) { 406 406 + extra_fds[i] = new_map(); 407 407 + if (!ASSERT_GE(extra_fds[i], 0, "new_map")) 408 408 + goto cleanup_fds; 409 409 + } 410 410 + 411 411 + prog_fd = load_test_prog(extra_fds, 65); 412 412 + ASSERT_EQ(prog_fd, -E2BIG, "prog should have been rejected with -E2BIG"); 413 413 + 414 414 + cleanup_fds: 415 415 + while (i > 0) 416 416 + Close(extra_fds[--i]); 417 417 + } 418 418 + 419 419 + void test_fd_array_cnt(void) 420 420 + { 421 421 + if (test__start_subtest("no-fd-array")) 422 422 + check_fd_array_cnt__no_fd_array(); 423 423 + 424 424 + if (test__start_subtest("fd-array-ok")) 425 425 + check_fd_array_cnt__fd_array_ok(); 426 426 + 427 427 + if (test__start_subtest("fd-array-dup-input")) 428 428 + check_fd_array_cnt__duplicated_maps(); 429 429 + 430 430 + if (test__start_subtest("fd-array-ref-maps-in-array")) 431 431 + check_fd_array_cnt__referenced_maps_in_fd_array(); 432 432 + 433 433 + if (test__start_subtest("fd-array-ref-btfs")) 434 434 + check_fd_array_cnt__referenced_btfs(); 435 435 + 436 436 + if (test__start_subtest("fd-array-trash-input")) 437 437 + check_fd_array_cnt__fd_array_with_trash(); 438 438 + 439 439 + if (test__start_subtest("fd-array-2big")) 440 440 + check_fd_array_cnt__fd_array_too_big(); 441 441 + }

+4

tools/testing/selftests/bpf/prog_tests/fill_link_info.c

reviewed

··· 171 171 /* See also arch_adjust_kprobe_addr(). */ 172 172 if (skel->kconfig->CONFIG_X86_KERNEL_IBT) 173 173 entry_offset = 4; 174 174 + if (skel->kconfig->CONFIG_PPC64 && 175 175 + skel->kconfig->CONFIG_KPROBES_ON_FTRACE && 176 176 + !skel->kconfig->CONFIG_PPC_FTRACE_OUT_OF_LINE) 177 177 + entry_offset = 4; 174 178 err = verify_perf_link_info(link_fd, type, kprobe_addr, 0, entry_offset); 175 179 ASSERT_OK(err, "verify_perf_link_info"); 176 180 } else {

+251 -78

tools/testing/selftests/bpf/prog_tests/flow_dissector.c

reviewed

··· 7 7 8 8 #include "bpf_flow.skel.h" 9 9 10 10 + #define TEST_NS "flow_dissector_ns" 10 11 #define FLOW_CONTINUE_SADDR 0x7f00007f /* 127.0.0.127 */ 12 12 + #define TEST_NAME_MAX_LEN 64 11 13 12 14 #ifndef IP_MF 13 15 #define IP_MF 0x2000 14 16 #endif 15 15 - 16 16 - #define CHECK_FLOW_KEYS(desc, got, expected) \ 17 17 - _CHECK(memcmp(&got, &expected, sizeof(got)) != 0, \ 18 18 - desc, \ 19 19 - topts.duration, \ 20 20 - "nhoff=%u/%u " \ 21 21 - "thoff=%u/%u " \ 22 22 - "addr_proto=0x%x/0x%x " \ 23 23 - "is_frag=%u/%u " \ 24 24 - "is_first_frag=%u/%u " \ 25 25 - "is_encap=%u/%u " \ 26 26 - "ip_proto=0x%x/0x%x " \ 27 27 - "n_proto=0x%x/0x%x " \ 28 28 - "flow_label=0x%x/0x%x " \ 29 29 - "sport=%u/%u " \ 30 30 - "dport=%u/%u\n", \ 31 31 - got.nhoff, expected.nhoff, \ 32 32 - got.thoff, expected.thoff, \ 33 33 - got.addr_proto, expected.addr_proto, \ 34 34 - got.is_frag, expected.is_frag, \ 35 35 - got.is_first_frag, expected.is_first_frag, \ 36 36 - got.is_encap, expected.is_encap, \ 37 37 - got.ip_proto, expected.ip_proto, \ 38 38 - got.n_proto, expected.n_proto, \ 39 39 - got.flow_label, expected.flow_label, \ 40 40 - got.sport, expected.sport, \ 41 41 - got.dport, expected.dport) 42 17 43 18 struct ipv4_pkt { 44 19 struct ethhdr eth; ··· 64 89 struct tcphdr tcp; 65 90 } __packed; 66 91 92 92 + struct gre_base_hdr { 93 93 + __be16 flags; 94 94 + __be16 protocol; 95 95 + } gre_base_hdr; 96 96 + 97 97 + struct gre_minimal_pkt { 98 98 + struct ethhdr eth; 99 99 + struct iphdr iph; 100 100 + struct gre_base_hdr gre_hdr; 101 101 + struct iphdr iph_inner; 102 102 + struct tcphdr tcp; 103 103 + } __packed; 104 104 + 67 105 struct test { 68 106 const char *name; 69 107 union { ··· 86 98 struct ipv6_pkt ipv6; 87 99 struct ipv6_frag_pkt ipv6_frag; 88 100 struct dvlan_ipv6_pkt dvlan_ipv6; 101 101 + struct gre_minimal_pkt gre_minimal; 89 102 } pkt; 90 103 struct bpf_flow_keys keys; 91 104 __u32 flags; ··· 95 106 96 107 #define VLAN_HLEN 4 97 108 98 98 - static __u32 duration; 99 109 struct test tests[] = { 100 110 { 101 111 .name = "ipv4", ··· 432 444 }, 433 445 .retval = BPF_FLOW_DISSECTOR_CONTINUE, 434 446 }, 447 447 + { 448 448 + .name = "ip-gre", 449 449 + .pkt.gre_minimal = { 450 450 + .eth.h_proto = __bpf_constant_htons(ETH_P_IP), 451 451 + .iph.ihl = 5, 452 452 + .iph.protocol = IPPROTO_GRE, 453 453 + .iph.tot_len = __bpf_constant_htons(MAGIC_BYTES), 454 454 + .gre_hdr = { 455 455 + .flags = 0, 456 456 + .protocol = __bpf_constant_htons(ETH_P_IP), 457 457 + }, 458 458 + .iph_inner.ihl = 5, 459 459 + .iph_inner.protocol = IPPROTO_TCP, 460 460 + .iph_inner.tot_len = 461 461 + __bpf_constant_htons(MAGIC_BYTES - 462 462 + sizeof(struct iphdr)), 463 463 + .tcp.doff = 5, 464 464 + .tcp.source = 80, 465 465 + .tcp.dest = 8080, 466 466 + }, 467 467 + .keys = { 468 468 + .nhoff = ETH_HLEN, 469 469 + .thoff = ETH_HLEN + sizeof(struct iphdr) * 2 + 470 470 + sizeof(struct gre_base_hdr), 471 471 + .addr_proto = ETH_P_IP, 472 472 + .ip_proto = IPPROTO_TCP, 473 473 + .n_proto = __bpf_constant_htons(ETH_P_IP), 474 474 + .is_encap = true, 475 475 + .sport = 80, 476 476 + .dport = 8080, 477 477 + }, 478 478 + .retval = BPF_OK, 479 479 + }, 480 480 + { 481 481 + .name = "ip-gre-no-encap", 482 482 + .pkt.ipip = { 483 483 + .eth.h_proto = __bpf_constant_htons(ETH_P_IP), 484 484 + .iph.ihl = 5, 485 485 + .iph.protocol = IPPROTO_GRE, 486 486 + .iph.tot_len = __bpf_constant_htons(MAGIC_BYTES), 487 487 + .iph_inner.ihl = 5, 488 488 + .iph_inner.protocol = IPPROTO_TCP, 489 489 + .iph_inner.tot_len = 490 490 + __bpf_constant_htons(MAGIC_BYTES - 491 491 + sizeof(struct iphdr)), 492 492 + .tcp.doff = 5, 493 493 + .tcp.source = 80, 494 494 + .tcp.dest = 8080, 495 495 + }, 496 496 + .keys = { 497 497 + .flags = BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP, 498 498 + .nhoff = ETH_HLEN, 499 499 + .thoff = ETH_HLEN + sizeof(struct iphdr) 500 500 + + sizeof(struct gre_base_hdr), 501 501 + .addr_proto = ETH_P_IP, 502 502 + .ip_proto = IPPROTO_GRE, 503 503 + .n_proto = __bpf_constant_htons(ETH_P_IP), 504 504 + .is_encap = true, 505 505 + }, 506 506 + .flags = BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP, 507 507 + .retval = BPF_OK, 508 508 + }, 435 509 }; 510 510 + 511 511 + void serial_test_flow_dissector_namespace(void) 512 512 + { 513 513 + struct bpf_flow *skel; 514 514 + struct nstoken *ns; 515 515 + int err, prog_fd; 516 516 + 517 517 + skel = bpf_flow__open_and_load(); 518 518 + if (!ASSERT_OK_PTR(skel, "open/load skeleton")) 519 519 + return; 520 520 + 521 521 + prog_fd = bpf_program__fd(skel->progs._dissect); 522 522 + if (!ASSERT_OK_FD(prog_fd, "get dissector fd")) 523 523 + goto out_destroy_skel; 524 524 + 525 525 + /* We must be able to attach a flow dissector to root namespace */ 526 526 + err = bpf_prog_attach(prog_fd, 0, BPF_FLOW_DISSECTOR, 0); 527 527 + if (!ASSERT_OK(err, "attach on root namespace ok")) 528 528 + goto out_destroy_skel; 529 529 + 530 530 + err = make_netns(TEST_NS); 531 531 + if (!ASSERT_OK(err, "create non-root net namespace")) 532 532 + goto out_destroy_skel; 533 533 + 534 534 + /* We must not be able to additionally attach a flow dissector to a 535 535 + * non-root net namespace 536 536 + */ 537 537 + ns = open_netns(TEST_NS); 538 538 + if (!ASSERT_OK_PTR(ns, "enter non-root net namespace")) 539 539 + goto out_clean_ns; 540 540 + err = bpf_prog_attach(prog_fd, 0, BPF_FLOW_DISSECTOR, 0); 541 541 + if (!ASSERT_ERR(err, 542 542 + "refuse new flow dissector in non-root net namespace")) 543 543 + bpf_prog_detach2(prog_fd, 0, BPF_FLOW_DISSECTOR); 544 544 + else 545 545 + ASSERT_EQ(errno, EEXIST, 546 546 + "refused because of already attached prog"); 547 547 + close_netns(ns); 548 548 + 549 549 + /* If no flow dissector is attached to the root namespace, we must 550 550 + * be able to attach one to a non-root net namespace 551 551 + */ 552 552 + bpf_prog_detach2(prog_fd, 0, BPF_FLOW_DISSECTOR); 553 553 + ns = open_netns(TEST_NS); 554 554 + ASSERT_OK_PTR(ns, "enter non-root net namespace"); 555 555 + err = bpf_prog_attach(prog_fd, 0, BPF_FLOW_DISSECTOR, 0); 556 556 + close_netns(ns); 557 557 + ASSERT_OK(err, "accept new flow dissector in non-root net namespace"); 558 558 + 559 559 + /* If a flow dissector is attached to non-root net namespace, attaching 560 560 + * a flow dissector to root namespace must fail 561 561 + */ 562 562 + err = bpf_prog_attach(prog_fd, 0, BPF_FLOW_DISSECTOR, 0); 563 563 + if (!ASSERT_ERR(err, "refuse new flow dissector on root namespace")) 564 564 + bpf_prog_detach2(prog_fd, 0, BPF_FLOW_DISSECTOR); 565 565 + else 566 566 + ASSERT_EQ(errno, EEXIST, 567 567 + "refused because of already attached prog"); 568 568 + 569 569 + ns = open_netns(TEST_NS); 570 570 + bpf_prog_detach2(prog_fd, 0, BPF_FLOW_DISSECTOR); 571 571 + close_netns(ns); 572 572 + out_clean_ns: 573 573 + remove_netns(TEST_NS); 574 574 + out_destroy_skel: 575 575 + bpf_flow__destroy(skel); 576 576 + } 436 577 437 578 static int create_tap(const char *ifname) 438 579 { ··· 650 533 return 0; 651 534 } 652 535 653 653 - static void run_tests_skb_less(int tap_fd, struct bpf_map *keys) 536 536 + static void run_tests_skb_less(int tap_fd, struct bpf_map *keys, 537 537 + char *test_suffix) 654 538 { 539 539 + char test_name[TEST_NAME_MAX_LEN]; 655 540 int i, err, keys_fd; 656 541 657 542 keys_fd = bpf_map__fd(keys); 658 658 - if (CHECK(keys_fd < 0, "bpf_map__fd", "err %d\n", keys_fd)) 543 543 + if (!ASSERT_OK_FD(keys_fd, "bpf_map__fd")) 659 544 return; 660 545 661 546 for (i = 0; i < ARRAY_SIZE(tests); i++) { 662 547 /* Keep in sync with 'flags' from eth_get_headlen. */ 663 548 __u32 eth_get_headlen_flags = 664 549 BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG; 665 665 - LIBBPF_OPTS(bpf_test_run_opts, topts); 666 550 struct bpf_flow_keys flow_keys = {}; 667 551 __u32 key = (__u32)(tests[i].keys.sport) << 16 | 668 552 tests[i].keys.dport; 553 553 + snprintf(test_name, TEST_NAME_MAX_LEN, "%s-%s", tests[i].name, 554 554 + test_suffix); 555 555 + if (!test__start_subtest(test_name)) 556 556 + continue; 669 557 670 558 /* For skb-less case we can't pass input flags; run 671 559 * only the tests that have a matching set of flags. ··· 680 558 continue; 681 559 682 560 err = tx_tap(tap_fd, &tests[i].pkt, sizeof(tests[i].pkt)); 683 683 - CHECK(err < 0, "tx_tap", "err %d errno %d\n", err, errno); 561 561 + if (!ASSERT_EQ(err, sizeof(tests[i].pkt), "tx_tap")) 562 562 + continue; 684 563 685 564 /* check the stored flow_keys only if BPF_OK expected */ 686 565 if (tests[i].retval != BPF_OK) 687 566 continue; 688 567 689 568 err = bpf_map_lookup_elem(keys_fd, &key, &flow_keys); 690 690 - ASSERT_OK(err, "bpf_map_lookup_elem"); 569 569 + if (!ASSERT_OK(err, "bpf_map_lookup_elem")) 570 570 + continue; 691 571 692 692 - CHECK_FLOW_KEYS(tests[i].name, flow_keys, tests[i].keys); 572 572 + ASSERT_MEMEQ(&flow_keys, &tests[i].keys, 573 573 + sizeof(struct bpf_flow_keys), 574 574 + "returned flow keys"); 693 575 694 576 err = bpf_map_delete_elem(keys_fd, &key); 695 577 ASSERT_OK(err, "bpf_map_delete_elem"); 696 578 } 697 579 } 698 580 699 699 - static void test_skb_less_prog_attach(struct bpf_flow *skel, int tap_fd) 581 581 + void test_flow_dissector_skb_less_direct_attach(void) 700 582 { 701 701 - int err, prog_fd; 583 583 + int err, prog_fd, tap_fd; 584 584 + struct bpf_flow *skel; 585 585 + struct netns_obj *ns; 586 586 + 587 587 + ns = netns_new("flow_dissector_skb_less_indirect_attach_ns", true); 588 588 + if (!ASSERT_OK_PTR(ns, "create and open netns")) 589 589 + return; 590 590 + 591 591 + skel = bpf_flow__open_and_load(); 592 592 + if (!ASSERT_OK_PTR(skel, "open/load skeleton")) 593 593 + goto out_clean_ns; 594 594 + 595 595 + err = init_prog_array(skel->obj, skel->maps.jmp_table); 596 596 + if (!ASSERT_OK(err, "init_prog_array")) 597 597 + goto out_destroy_skel; 702 598 703 599 prog_fd = bpf_program__fd(skel->progs._dissect); 704 704 - if (CHECK(prog_fd < 0, "bpf_program__fd", "err %d\n", prog_fd)) 705 705 - return; 600 600 + if (!ASSERT_OK_FD(prog_fd, "bpf_program__fd")) 601 601 + goto out_destroy_skel; 706 602 707 603 err = bpf_prog_attach(prog_fd, 0, BPF_FLOW_DISSECTOR, 0); 708 708 - if (CHECK(err, "bpf_prog_attach", "err %d errno %d\n", err, errno)) 709 709 - return; 604 604 + if (!ASSERT_OK(err, "bpf_prog_attach")) 605 605 + goto out_destroy_skel; 710 606 711 711 - run_tests_skb_less(tap_fd, skel->maps.last_dissection); 607 607 + tap_fd = create_tap("tap0"); 608 608 + if (!ASSERT_OK_FD(tap_fd, "create_tap")) 609 609 + goto out_destroy_skel; 610 610 + err = ifup("tap0"); 611 611 + if (!ASSERT_OK(err, "ifup")) 612 612 + goto out_close_tap; 613 613 + 614 614 + run_tests_skb_less(tap_fd, skel->maps.last_dissection, 615 615 + "non-skb-direct-attach"); 712 616 713 617 err = bpf_prog_detach2(prog_fd, 0, BPF_FLOW_DISSECTOR); 714 714 - CHECK(err, "bpf_prog_detach2", "err %d errno %d\n", err, errno); 618 618 + ASSERT_OK(err, "bpf_prog_detach2"); 619 619 + 620 620 + out_close_tap: 621 621 + close(tap_fd); 622 622 + out_destroy_skel: 623 623 + bpf_flow__destroy(skel); 624 624 + out_clean_ns: 625 625 + netns_free(ns); 715 626 } 716 627 717 717 - static void test_skb_less_link_create(struct bpf_flow *skel, int tap_fd) 628 628 + void test_flow_dissector_skb_less_indirect_attach(void) 718 629 { 630 630 + int err, net_fd, tap_fd; 631 631 + struct bpf_flow *skel; 719 632 struct bpf_link *link; 720 720 - int err, net_fd; 633 633 + struct netns_obj *ns; 634 634 + 635 635 + ns = netns_new("flow_dissector_skb_less_indirect_attach_ns", true); 636 636 + if (!ASSERT_OK_PTR(ns, "create and open netns")) 637 637 + return; 638 638 + 639 639 + skel = bpf_flow__open_and_load(); 640 640 + if (!ASSERT_OK_PTR(skel, "open/load skeleton")) 641 641 + goto out_clean_ns; 721 642 722 643 net_fd = open("/proc/self/ns/net", O_RDONLY); 723 723 - if (CHECK(net_fd < 0, "open(/proc/self/ns/net)", "err %d\n", errno)) 724 724 - return; 644 644 + if (!ASSERT_OK_FD(net_fd, "open(/proc/self/ns/net")) 645 645 + goto out_destroy_skel; 646 646 + 647 647 + err = init_prog_array(skel->obj, skel->maps.jmp_table); 648 648 + if (!ASSERT_OK(err, "init_prog_array")) 649 649 + goto out_destroy_skel; 650 650 + 651 651 + tap_fd = create_tap("tap0"); 652 652 + if (!ASSERT_OK_FD(tap_fd, "create_tap")) 653 653 + goto out_close_ns; 654 654 + err = ifup("tap0"); 655 655 + if (!ASSERT_OK(err, "ifup")) 656 656 + goto out_close_tap; 725 657 726 658 link = bpf_program__attach_netns(skel->progs._dissect, net_fd); 727 659 if (!ASSERT_OK_PTR(link, "attach_netns")) 728 728 - goto out_close; 660 660 + goto out_close_tap; 729 661 730 730 - run_tests_skb_less(tap_fd, skel->maps.last_dissection); 662 662 + run_tests_skb_less(tap_fd, skel->maps.last_dissection, 663 663 + "non-skb-indirect-attach"); 731 664 732 665 err = bpf_link__destroy(link); 733 733 - CHECK(err, "bpf_link__destroy", "err %d\n", err); 734 734 - out_close: 666 666 + ASSERT_OK(err, "bpf_link__destroy"); 667 667 + 668 668 + out_close_tap: 669 669 + close(tap_fd); 670 670 + out_close_ns: 735 671 close(net_fd); 672 672 + out_destroy_skel: 673 673 + bpf_flow__destroy(skel); 674 674 + out_clean_ns: 675 675 + netns_free(ns); 736 676 } 737 677 738 738 - void test_flow_dissector(void) 678 678 + void test_flow_dissector_skb(void) 739 679 { 740 740 - int i, err, prog_fd, keys_fd = -1, tap_fd; 680 680 + char test_name[TEST_NAME_MAX_LEN]; 741 681 struct bpf_flow *skel; 682 682 + int i, err, prog_fd; 742 683 743 684 skel = bpf_flow__open_and_load(); 744 744 - if (CHECK(!skel, "skel", "failed to open/load skeleton\n")) 685 685 + if (!ASSERT_OK_PTR(skel, "open/load skeleton")) 745 686 return; 746 687 747 747 - prog_fd = bpf_program__fd(skel->progs._dissect); 748 748 - if (CHECK(prog_fd < 0, "bpf_program__fd", "err %d\n", prog_fd)) 749 749 - goto out_destroy_skel; 750 750 - keys_fd = bpf_map__fd(skel->maps.last_dissection); 751 751 - if (CHECK(keys_fd < 0, "bpf_map__fd", "err %d\n", keys_fd)) 752 752 - goto out_destroy_skel; 753 688 err = init_prog_array(skel->obj, skel->maps.jmp_table); 754 754 - if (CHECK(err, "init_prog_array", "err %d\n", err)) 689 689 + if (!ASSERT_OK(err, "init_prog_array")) 690 690 + goto out_destroy_skel; 691 691 + 692 692 + prog_fd = bpf_program__fd(skel->progs._dissect); 693 693 + if (!ASSERT_OK_FD(prog_fd, "bpf_program__fd")) 755 694 goto out_destroy_skel; 756 695 757 696 for (i = 0; i < ARRAY_SIZE(tests); i++) { ··· 823 640 .data_out = &flow_keys, 824 641 ); 825 642 static struct bpf_flow_keys ctx = {}; 643 643 + 644 644 + snprintf(test_name, TEST_NAME_MAX_LEN, "%s-skb", tests[i].name); 645 645 + if (!test__start_subtest(test_name)) 646 646 + continue; 826 647 827 648 if (tests[i].flags) { 828 649 topts.ctx_in = &ctx; ··· 843 656 continue; 844 657 ASSERT_EQ(topts.data_size_out, sizeof(flow_keys), 845 658 "test_run data_size_out"); 846 846 - CHECK_FLOW_KEYS(tests[i].name, flow_keys, tests[i].keys); 659 659 + ASSERT_MEMEQ(&flow_keys, &tests[i].keys, 660 660 + sizeof(struct bpf_flow_keys), 661 661 + "returned flow keys"); 847 662 } 848 663 849 849 - /* Do the same tests but for skb-less flow dissector. 850 850 - * We use a known path in the net/tun driver that calls 851 851 - * eth_get_headlen and we manually export bpf_flow_keys 852 852 - * via BPF map in this case. 853 853 - */ 854 854 - 855 855 - tap_fd = create_tap("tap0"); 856 856 - CHECK(tap_fd < 0, "create_tap", "tap_fd %d errno %d\n", tap_fd, errno); 857 857 - err = ifup("tap0"); 858 858 - CHECK(err, "ifup", "err %d errno %d\n", err, errno); 859 859 - 860 860 - /* Test direct prog attachment */ 861 861 - test_skb_less_prog_attach(skel, tap_fd); 862 862 - /* Test indirect prog attachment via link */ 863 863 - test_skb_less_link_create(skel, tap_fd); 864 864 - 865 865 - close(tap_fd); 866 664 out_destroy_skel: 867 665 bpf_flow__destroy(skel); 868 666 } 667 667 +

+792

tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + #define _GNU_SOURCE 4 4 + #include <stdbool.h> 5 5 + #include <stdlib.h> 6 6 + #include <stdio.h> 7 7 + #include <bpf/bpf.h> 8 8 + #include <linux/bpf.h> 9 9 + #include <bpf/libbpf.h> 10 10 + #include <arpa/inet.h> 11 11 + #include <asm/byteorder.h> 12 12 + #include <netinet/udp.h> 13 13 + #include <poll.h> 14 14 + #include <string.h> 15 15 + #include <sys/ioctl.h> 16 16 + #include <sys/socket.h> 17 17 + #include <sys/time.h> 18 18 + #include <unistd.h> 19 19 + #include "test_progs.h" 20 20 + #include "network_helpers.h" 21 21 + #include "bpf_util.h" 22 22 + #include "bpf_flow.skel.h" 23 23 + 24 24 + #define CFG_PORT_INNER 8000 25 25 + #define CFG_PORT_GUE 6080 26 26 + #define SUBTEST_NAME_MAX_LEN 32 27 27 + #define TEST_NAME_MAX_LEN (32 + SUBTEST_NAME_MAX_LEN) 28 28 + #define MAX_SOURCE_PORTS 3 29 29 + #define TEST_PACKETS_COUNT 10 30 30 + #define TEST_PACKET_LEN 100 31 31 + #define TEST_PACKET_PATTERN 'a' 32 32 + #define TEST_IPV4 "192.168.0.1/32" 33 33 + #define TEST_IPV6 "100::a/128" 34 34 + #define TEST_TUNNEL_REMOTE "127.0.0.2" 35 35 + #define TEST_TUNNEL_LOCAL "127.0.0.1" 36 36 + 37 37 + #define INIT_ADDR4(addr4, port) \ 38 38 + { \ 39 39 + .sin_family = AF_INET, \ 40 40 + .sin_port = __constant_htons(port), \ 41 41 + .sin_addr.s_addr = __constant_htonl(addr4), \ 42 42 + } 43 43 + 44 44 + #define INIT_ADDR6(addr6, port) \ 45 45 + { \ 46 46 + .sin6_family = AF_INET6, \ 47 47 + .sin6_port = __constant_htons(port), \ 48 48 + .sin6_addr = addr6, \ 49 49 + } 50 50 + #define TEST_IN4_SRC_ADDR_DEFAULT INIT_ADDR4(INADDR_LOOPBACK + 2, 0) 51 51 + #define TEST_IN4_DST_ADDR_DEFAULT INIT_ADDR4(INADDR_LOOPBACK, CFG_PORT_INNER) 52 52 + #define TEST_OUT4_SRC_ADDR_DEFAULT INIT_ADDR4(INADDR_LOOPBACK + 1, 0) 53 53 + #define TEST_OUT4_DST_ADDR_DEFAULT INIT_ADDR4(INADDR_LOOPBACK, 0) 54 54 + 55 55 + #define TEST_IN6_SRC_ADDR_DEFAULT INIT_ADDR6(IN6ADDR_LOOPBACK_INIT, 0) 56 56 + #define TEST_IN6_DST_ADDR_DEFAULT \ 57 57 + INIT_ADDR6(IN6ADDR_LOOPBACK_INIT, CFG_PORT_INNER) 58 58 + #define TEST_OUT6_SRC_ADDR_DEFAULT INIT_ADDR6(IN6ADDR_LOOPBACK_INIT, 0) 59 59 + #define TEST_OUT6_DST_ADDR_DEFAULT INIT_ADDR6(IN6ADDR_LOOPBACK_INIT, 0) 60 60 + 61 61 + #define TEST_IN4_SRC_ADDR_DISSECT_CONTINUE INIT_ADDR4(INADDR_LOOPBACK + 126, 0) 62 62 + #define TEST_IN4_SRC_ADDR_IPIP INIT_ADDR4((in_addr_t)0x01010101, 0) 63 63 + #define TEST_IN4_DST_ADDR_IPIP INIT_ADDR4((in_addr_t)0xC0A80001, CFG_PORT_INNER) 64 64 + 65 65 + struct grehdr { 66 66 + uint16_t unused; 67 67 + uint16_t protocol; 68 68 + } __packed; 69 69 + 70 70 + struct guehdr { 71 71 + union { 72 72 + struct { 73 73 + #if defined(__LITTLE_ENDIAN_BITFIELD) 74 74 + __u8 hlen : 5, control : 1, version : 2; 75 75 + #elif defined(__BIG_ENDIAN_BITFIELD) 76 76 + __u8 version : 2, control : 1, hlen : 5; 77 77 + #else 78 78 + #error "Please fix <asm/byteorder.h>" 79 79 + #endif 80 80 + __u8 proto_ctype; 81 81 + __be16 flags; 82 82 + }; 83 83 + __be32 word; 84 84 + }; 85 85 + }; 86 86 + 87 87 + static char buf[ETH_DATA_LEN]; 88 88 + 89 89 + struct test_configuration { 90 90 + char name[SUBTEST_NAME_MAX_LEN]; 91 91 + int (*test_setup)(void); 92 92 + void (*test_teardown)(void); 93 93 + int source_ports[MAX_SOURCE_PORTS]; 94 94 + int cfg_l3_inner; 95 95 + struct sockaddr_in in_saddr4; 96 96 + struct sockaddr_in in_daddr4; 97 97 + struct sockaddr_in6 in_saddr6; 98 98 + struct sockaddr_in6 in_daddr6; 99 99 + int cfg_l3_outer; 100 100 + struct sockaddr_in out_saddr4; 101 101 + struct sockaddr_in out_daddr4; 102 102 + struct sockaddr_in6 out_saddr6; 103 103 + struct sockaddr_in6 out_daddr6; 104 104 + int cfg_encap_proto; 105 105 + uint8_t cfg_dsfield_inner; 106 106 + uint8_t cfg_dsfield_outer; 107 107 + int cfg_l3_extra; 108 108 + struct sockaddr_in extra_saddr4; 109 109 + struct sockaddr_in extra_daddr4; 110 110 + struct sockaddr_in6 extra_saddr6; 111 111 + struct sockaddr_in6 extra_daddr6; 112 112 + }; 113 113 + 114 114 + static unsigned long util_gettime(void) 115 115 + { 116 116 + struct timeval tv; 117 117 + 118 118 + gettimeofday(&tv, NULL); 119 119 + return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); 120 120 + } 121 121 + 122 122 + static void build_ipv4_header(void *header, uint8_t proto, uint32_t src, 123 123 + uint32_t dst, int payload_len, uint8_t tos) 124 124 + { 125 125 + struct iphdr *iph = header; 126 126 + 127 127 + iph->ihl = 5; 128 128 + iph->version = 4; 129 129 + iph->tos = tos; 130 130 + iph->ttl = 8; 131 131 + iph->tot_len = htons(sizeof(*iph) + payload_len); 132 132 + iph->id = htons(1337); 133 133 + iph->protocol = proto; 134 134 + iph->saddr = src; 135 135 + iph->daddr = dst; 136 136 + iph->check = build_ip_csum((void *)iph); 137 137 + } 138 138 + 139 139 + static void ipv6_set_dsfield(struct ipv6hdr *ip6h, uint8_t dsfield) 140 140 + { 141 141 + uint16_t val, *ptr = (uint16_t *)ip6h; 142 142 + 143 143 + val = ntohs(*ptr); 144 144 + val &= 0xF00F; 145 145 + val |= ((uint16_t)dsfield) << 4; 146 146 + *ptr = htons(val); 147 147 + } 148 148 + 149 149 + static void build_ipv6_header(void *header, uint8_t proto, 150 150 + const struct sockaddr_in6 *src, 151 151 + const struct sockaddr_in6 *dst, int payload_len, 152 152 + uint8_t dsfield) 153 153 + { 154 154 + struct ipv6hdr *ip6h = header; 155 155 + 156 156 + ip6h->version = 6; 157 157 + ip6h->payload_len = htons(payload_len); 158 158 + ip6h->nexthdr = proto; 159 159 + ip6h->hop_limit = 8; 160 160 + ipv6_set_dsfield(ip6h, dsfield); 161 161 + 162 162 + memcpy(&ip6h->saddr, &src->sin6_addr, sizeof(ip6h->saddr)); 163 163 + memcpy(&ip6h->daddr, &dst->sin6_addr, sizeof(ip6h->daddr)); 164 164 + } 165 165 + 166 166 + static void build_udp_header(void *header, int payload_len, uint16_t sport, 167 167 + uint16_t dport, int family) 168 168 + { 169 169 + struct udphdr *udph = header; 170 170 + int len = sizeof(*udph) + payload_len; 171 171 + 172 172 + udph->source = htons(sport); 173 173 + udph->dest = htons(dport); 174 174 + udph->len = htons(len); 175 175 + udph->check = 0; 176 176 + if (family == AF_INET) 177 177 + udph->check = build_udp_v4_csum(header - sizeof(struct iphdr), 178 178 + udph); 179 179 + else 180 180 + udph->check = build_udp_v6_csum(header - sizeof(struct ipv6hdr), 181 181 + udph); 182 182 + } 183 183 + 184 184 + static void build_gue_header(void *header, uint8_t proto) 185 185 + { 186 186 + struct guehdr *gueh = header; 187 187 + 188 188 + gueh->proto_ctype = proto; 189 189 + } 190 190 + 191 191 + static void build_gre_header(void *header, uint16_t proto) 192 192 + { 193 193 + struct grehdr *greh = header; 194 194 + 195 195 + greh->protocol = htons(proto); 196 196 + } 197 197 + 198 198 + static int l3_length(int family) 199 199 + { 200 200 + if (family == AF_INET) 201 201 + return sizeof(struct iphdr); 202 202 + else 203 203 + return sizeof(struct ipv6hdr); 204 204 + } 205 205 + 206 206 + static int build_packet(const struct test_configuration *test, uint16_t sport) 207 207 + { 208 208 + int ol3_len = 0, ol4_len = 0, il3_len = 0, il4_len = 0; 209 209 + int el3_len = 0, packet_len; 210 210 + 211 211 + memset(buf, 0, ETH_DATA_LEN); 212 212 + 213 213 + if (test->cfg_l3_extra) 214 214 + el3_len = l3_length(test->cfg_l3_extra); 215 215 + 216 216 + /* calculate header offsets */ 217 217 + if (test->cfg_encap_proto) { 218 218 + ol3_len = l3_length(test->cfg_l3_outer); 219 219 + 220 220 + if (test->cfg_encap_proto == IPPROTO_GRE) 221 221 + ol4_len = sizeof(struct grehdr); 222 222 + else if (test->cfg_encap_proto == IPPROTO_UDP) 223 223 + ol4_len = sizeof(struct udphdr) + sizeof(struct guehdr); 224 224 + } 225 225 + 226 226 + il3_len = l3_length(test->cfg_l3_inner); 227 227 + il4_len = sizeof(struct udphdr); 228 228 + 229 229 + packet_len = el3_len + ol3_len + ol4_len + il3_len + il4_len + 230 230 + TEST_PACKET_LEN; 231 231 + if (!ASSERT_LE(packet_len, sizeof(buf), "check packet size")) 232 232 + return -1; 233 233 + 234 234 + /* 235 235 + * Fill packet from inside out, to calculate correct checksums. 236 236 + * But create ip before udp headers, as udp uses ip for pseudo-sum. 237 237 + */ 238 238 + memset(buf + el3_len + ol3_len + ol4_len + il3_len + il4_len, 239 239 + TEST_PACKET_PATTERN, TEST_PACKET_LEN); 240 240 + 241 241 + /* add zero byte for udp csum padding */ 242 242 + buf[el3_len + ol3_len + ol4_len + il3_len + il4_len + TEST_PACKET_LEN] = 243 243 + 0; 244 244 + 245 245 + switch (test->cfg_l3_inner) { 246 246 + case PF_INET: 247 247 + build_ipv4_header(buf + el3_len + ol3_len + ol4_len, 248 248 + IPPROTO_UDP, test->in_saddr4.sin_addr.s_addr, 249 249 + test->in_daddr4.sin_addr.s_addr, 250 250 + il4_len + TEST_PACKET_LEN, 251 251 + test->cfg_dsfield_inner); 252 252 + break; 253 253 + case PF_INET6: 254 254 + build_ipv6_header(buf + el3_len + ol3_len + ol4_len, 255 255 + IPPROTO_UDP, &test->in_saddr6, 256 256 + &test->in_daddr6, il4_len + TEST_PACKET_LEN, 257 257 + test->cfg_dsfield_inner); 258 258 + break; 259 259 + } 260 260 + 261 261 + build_udp_header(buf + el3_len + ol3_len + ol4_len + il3_len, 262 262 + TEST_PACKET_LEN, sport, CFG_PORT_INNER, 263 263 + test->cfg_l3_inner); 264 264 + 265 265 + if (!test->cfg_encap_proto) 266 266 + return il3_len + il4_len + TEST_PACKET_LEN; 267 267 + 268 268 + switch (test->cfg_l3_outer) { 269 269 + case PF_INET: 270 270 + build_ipv4_header(buf + el3_len, test->cfg_encap_proto, 271 271 + test->out_saddr4.sin_addr.s_addr, 272 272 + test->out_daddr4.sin_addr.s_addr, 273 273 + ol4_len + il3_len + il4_len + TEST_PACKET_LEN, 274 274 + test->cfg_dsfield_outer); 275 275 + break; 276 276 + case PF_INET6: 277 277 + build_ipv6_header(buf + el3_len, test->cfg_encap_proto, 278 278 + &test->out_saddr6, &test->out_daddr6, 279 279 + ol4_len + il3_len + il4_len + TEST_PACKET_LEN, 280 280 + test->cfg_dsfield_outer); 281 281 + break; 282 282 + } 283 283 + 284 284 + switch (test->cfg_encap_proto) { 285 285 + case IPPROTO_UDP: 286 286 + build_gue_header(buf + el3_len + ol3_len + ol4_len - 287 287 + sizeof(struct guehdr), 288 288 + test->cfg_l3_inner == PF_INET ? IPPROTO_IPIP : 289 289 + IPPROTO_IPV6); 290 290 + build_udp_header(buf + el3_len + ol3_len, 291 291 + sizeof(struct guehdr) + il3_len + il4_len + 292 292 + TEST_PACKET_LEN, 293 293 + sport, CFG_PORT_GUE, test->cfg_l3_outer); 294 294 + break; 295 295 + case IPPROTO_GRE: 296 296 + build_gre_header(buf + el3_len + ol3_len, 297 297 + test->cfg_l3_inner == PF_INET ? ETH_P_IP : 298 298 + ETH_P_IPV6); 299 299 + break; 300 300 + } 301 301 + 302 302 + switch (test->cfg_l3_extra) { 303 303 + case PF_INET: 304 304 + build_ipv4_header(buf, 305 305 + test->cfg_l3_outer == PF_INET ? IPPROTO_IPIP : 306 306 + IPPROTO_IPV6, 307 307 + test->extra_saddr4.sin_addr.s_addr, 308 308 + test->extra_daddr4.sin_addr.s_addr, 309 309 + ol3_len + ol4_len + il3_len + il4_len + 310 310 + TEST_PACKET_LEN, 311 311 + 0); 312 312 + break; 313 313 + case PF_INET6: 314 314 + build_ipv6_header(buf, 315 315 + test->cfg_l3_outer == PF_INET ? IPPROTO_IPIP : 316 316 + IPPROTO_IPV6, 317 317 + &test->extra_saddr6, &test->extra_daddr6, 318 318 + ol3_len + ol4_len + il3_len + il4_len + 319 319 + TEST_PACKET_LEN, 320 320 + 0); 321 321 + break; 322 322 + } 323 323 + 324 324 + return el3_len + ol3_len + ol4_len + il3_len + il4_len + 325 325 + TEST_PACKET_LEN; 326 326 + } 327 327 + 328 328 + /* sender transmits encapsulated over RAW or unencap'd over UDP */ 329 329 + static int setup_tx(const struct test_configuration *test) 330 330 + { 331 331 + int family, fd, ret; 332 332 + 333 333 + if (test->cfg_l3_extra) 334 334 + family = test->cfg_l3_extra; 335 335 + else if (test->cfg_l3_outer) 336 336 + family = test->cfg_l3_outer; 337 337 + else 338 338 + family = test->cfg_l3_inner; 339 339 + 340 340 + fd = socket(family, SOCK_RAW, IPPROTO_RAW); 341 341 + if (!ASSERT_OK_FD(fd, "setup tx socket")) 342 342 + return fd; 343 343 + 344 344 + if (test->cfg_l3_extra) { 345 345 + if (test->cfg_l3_extra == PF_INET) 346 346 + ret = connect(fd, (void *)&test->extra_daddr4, 347 347 + sizeof(test->extra_daddr4)); 348 348 + else 349 349 + ret = connect(fd, (void *)&test->extra_daddr6, 350 350 + sizeof(test->extra_daddr6)); 351 351 + if (!ASSERT_OK(ret, "connect")) { 352 352 + close(fd); 353 353 + return ret; 354 354 + } 355 355 + } else if (test->cfg_l3_outer) { 356 356 + /* connect to destination if not encapsulated */ 357 357 + if (test->cfg_l3_outer == PF_INET) 358 358 + ret = connect(fd, (void *)&test->out_daddr4, 359 359 + sizeof(test->out_daddr4)); 360 360 + else 361 361 + ret = connect(fd, (void *)&test->out_daddr6, 362 362 + sizeof(test->out_daddr6)); 363 363 + if (!ASSERT_OK(ret, "connect")) { 364 364 + close(fd); 365 365 + return ret; 366 366 + } 367 367 + } else { 368 368 + /* otherwise using loopback */ 369 369 + if (test->cfg_l3_inner == PF_INET) 370 370 + ret = connect(fd, (void *)&test->in_daddr4, 371 371 + sizeof(test->in_daddr4)); 372 372 + else 373 373 + ret = connect(fd, (void *)&test->in_daddr6, 374 374 + sizeof(test->in_daddr6)); 375 375 + if (!ASSERT_OK(ret, "connect")) { 376 376 + close(fd); 377 377 + return ret; 378 378 + } 379 379 + } 380 380 + 381 381 + return fd; 382 382 + } 383 383 + 384 384 + /* receiver reads unencapsulated UDP */ 385 385 + static int setup_rx(const struct test_configuration *test) 386 386 + { 387 387 + int fd, ret; 388 388 + 389 389 + fd = socket(test->cfg_l3_inner, SOCK_DGRAM, 0); 390 390 + if (!ASSERT_OK_FD(fd, "socket rx")) 391 391 + return fd; 392 392 + 393 393 + if (test->cfg_l3_inner == PF_INET) 394 394 + ret = bind(fd, (void *)&test->in_daddr4, 395 395 + sizeof(test->in_daddr4)); 396 396 + else 397 397 + ret = bind(fd, (void *)&test->in_daddr6, 398 398 + sizeof(test->in_daddr6)); 399 399 + if (!ASSERT_OK(ret, "bind rx")) { 400 400 + close(fd); 401 401 + return ret; 402 402 + } 403 403 + 404 404 + return fd; 405 405 + } 406 406 + 407 407 + static int do_tx(int fd, const char *pkt, int len) 408 408 + { 409 409 + int ret; 410 410 + 411 411 + ret = write(fd, pkt, len); 412 412 + return ret != len; 413 413 + } 414 414 + 415 415 + static int do_poll(int fd, short events, int timeout) 416 416 + { 417 417 + struct pollfd pfd; 418 418 + int ret; 419 419 + 420 420 + pfd.fd = fd; 421 421 + pfd.events = events; 422 422 + 423 423 + ret = poll(&pfd, 1, timeout); 424 424 + return ret; 425 425 + } 426 426 + 427 427 + static int do_rx(int fd) 428 428 + { 429 429 + char rbuf; 430 430 + int ret, num = 0; 431 431 + 432 432 + while (1) { 433 433 + ret = recv(fd, &rbuf, 1, MSG_DONTWAIT); 434 434 + if (ret == -1 && errno == EAGAIN) 435 435 + break; 436 436 + if (ret < 0) 437 437 + return -1; 438 438 + if (!ASSERT_EQ(rbuf, TEST_PACKET_PATTERN, "check pkt pattern")) 439 439 + return -1; 440 440 + num++; 441 441 + } 442 442 + 443 443 + return num; 444 444 + } 445 445 + 446 446 + static int run_test(const struct test_configuration *test, 447 447 + int source_port_index) 448 448 + { 449 449 + int fdt = -1, fdr = -1, len, tx = 0, rx = 0, err; 450 450 + unsigned long tstop, tcur; 451 451 + 452 452 + fdr = setup_rx(test); 453 453 + fdt = setup_tx(test); 454 454 + if (!ASSERT_OK_FD(fdr, "setup rx") || !ASSERT_OK_FD(fdt, "setup tx")) { 455 455 + err = -1; 456 456 + goto out_close_sockets; 457 457 + } 458 458 + 459 459 + len = build_packet(test, 460 460 + (uint16_t)test->source_ports[source_port_index]); 461 461 + if (!ASSERT_GT(len, 0, "build test packet")) 462 462 + return -1; 463 463 + 464 464 + tcur = util_gettime(); 465 465 + tstop = tcur; 466 466 + 467 467 + while (tx < TEST_PACKETS_COUNT) { 468 468 + if (!ASSERT_OK(do_tx(fdt, buf, len), "do_tx")) 469 469 + break; 470 470 + tx++; 471 471 + err = do_rx(fdr); 472 472 + if (!ASSERT_GE(err, 0, "do_rx")) 473 473 + break; 474 474 + rx += err; 475 475 + } 476 476 + 477 477 + /* read straggler packets, if any */ 478 478 + if (rx < tx) { 479 479 + tstop = util_gettime() + 100; 480 480 + while (rx < tx) { 481 481 + tcur = util_gettime(); 482 482 + if (tcur >= tstop) 483 483 + break; 484 484 + 485 485 + err = do_poll(fdr, POLLIN, tstop - tcur); 486 486 + if (err < 0) 487 487 + break; 488 488 + err = do_rx(fdr); 489 489 + if (err >= 0) 490 490 + rx += err; 491 491 + } 492 492 + } 493 493 + 494 494 + out_close_sockets: 495 495 + close(fdt); 496 496 + close(fdr); 497 497 + return rx; 498 498 + } 499 499 + 500 500 + static int attach_and_configure_program(struct bpf_flow *skel) 501 501 + { 502 502 + struct bpf_map *prog_array = skel->maps.jmp_table; 503 503 + int main_prog_fd, sub_prog_fd, map_fd, i, err; 504 504 + struct bpf_program *prog; 505 505 + char prog_name[32]; 506 506 + 507 507 + main_prog_fd = bpf_program__fd(skel->progs._dissect); 508 508 + if (main_prog_fd < 0) 509 509 + return main_prog_fd; 510 510 + 511 511 + err = bpf_prog_attach(main_prog_fd, 0, BPF_FLOW_DISSECTOR, 0); 512 512 + if (err) 513 513 + return err; 514 514 + 515 515 + map_fd = bpf_map__fd(prog_array); 516 516 + if (map_fd < 0) 517 517 + return map_fd; 518 518 + 519 519 + for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 520 520 + snprintf(prog_name, sizeof(prog_name), "flow_dissector_%d", i); 521 521 + 522 522 + prog = bpf_object__find_program_by_name(skel->obj, prog_name); 523 523 + if (!prog) 524 524 + return -1; 525 525 + 526 526 + sub_prog_fd = bpf_program__fd(prog); 527 527 + if (sub_prog_fd < 0) 528 528 + return -1; 529 529 + 530 530 + err = bpf_map_update_elem(map_fd, &i, &sub_prog_fd, BPF_ANY); 531 531 + if (err) 532 532 + return -1; 533 533 + } 534 534 + 535 535 + return main_prog_fd; 536 536 + } 537 537 + 538 538 + static void detach_program(struct bpf_flow *skel, int prog_fd) 539 539 + { 540 540 + bpf_prog_detach2(prog_fd, 0, BPF_FLOW_DISSECTOR); 541 541 + } 542 542 + 543 543 + static int set_port_drop(int pf, bool multi_port) 544 544 + { 545 545 + SYS(fail, "tc qdisc add dev lo ingress"); 546 546 + SYS(fail_delete_qdisc, "tc filter add %s %s %s %s %s %s %s %s %s %s", 547 547 + "dev lo", 548 548 + "parent FFFF:", 549 549 + "protocol", pf == PF_INET6 ? "ipv6" : "ip", 550 550 + "pref 1337", 551 551 + "flower", 552 552 + "ip_proto udp", 553 553 + "src_port", multi_port ? "8-10" : "9", 554 554 + "action drop"); 555 555 + return 0; 556 556 + 557 557 + fail_delete_qdisc: 558 558 + SYS_NOFAIL("tc qdisc del dev lo ingress"); 559 559 + fail: 560 560 + return 1; 561 561 + } 562 562 + 563 563 + static void remove_filter(void) 564 564 + { 565 565 + SYS_NOFAIL("tc filter del dev lo ingress"); 566 566 + SYS_NOFAIL("tc qdisc del dev lo ingress"); 567 567 + } 568 568 + 569 569 + static int ipv4_setup(void) 570 570 + { 571 571 + return set_port_drop(PF_INET, false); 572 572 + } 573 573 + 574 574 + static int ipv6_setup(void) 575 575 + { 576 576 + return set_port_drop(PF_INET6, false); 577 577 + } 578 578 + 579 579 + static int port_range_setup(void) 580 580 + { 581 581 + return set_port_drop(PF_INET, true); 582 582 + } 583 583 + 584 584 + static int set_addresses(void) 585 585 + { 586 586 + SYS(out, "ip -4 addr add %s dev lo", TEST_IPV4); 587 587 + SYS(out_remove_ipv4, "ip -6 addr add %s dev lo", TEST_IPV6); 588 588 + return 0; 589 589 + out_remove_ipv4: 590 590 + SYS_NOFAIL("ip -4 addr del %s dev lo", TEST_IPV4); 591 591 + out: 592 592 + return -1; 593 593 + } 594 594 + 595 595 + static void unset_addresses(void) 596 596 + { 597 597 + SYS_NOFAIL("ip -4 addr del %s dev lo", TEST_IPV4); 598 598 + SYS_NOFAIL("ip -6 addr del %s dev lo", TEST_IPV6); 599 599 + } 600 600 + 601 601 + static int ipip_setup(void) 602 602 + { 603 603 + if (!ASSERT_OK(set_addresses(), "configure addresses")) 604 604 + return -1; 605 605 + if (!ASSERT_OK(set_port_drop(PF_INET, false), "set filter")) 606 606 + goto out_unset_addresses; 607 607 + SYS(out_remove_filter, 608 608 + "ip link add ipip_test type ipip remote %s local %s dev lo", 609 609 + TEST_TUNNEL_REMOTE, TEST_TUNNEL_LOCAL); 610 610 + SYS(out_clean_netif, "ip link set ipip_test up"); 611 611 + return 0; 612 612 + 613 613 + out_clean_netif: 614 614 + SYS_NOFAIL("ip link del ipip_test"); 615 615 + out_remove_filter: 616 616 + remove_filter(); 617 617 + out_unset_addresses: 618 618 + unset_addresses(); 619 619 + return -1; 620 620 + } 621 621 + 622 622 + static void ipip_shutdown(void) 623 623 + { 624 624 + SYS_NOFAIL("ip link del ipip_test"); 625 625 + remove_filter(); 626 626 + unset_addresses(); 627 627 + } 628 628 + 629 629 + static int gre_setup(void) 630 630 + { 631 631 + if (!ASSERT_OK(set_addresses(), "configure addresses")) 632 632 + return -1; 633 633 + if (!ASSERT_OK(set_port_drop(PF_INET, false), "set filter")) 634 634 + goto out_unset_addresses; 635 635 + SYS(out_remove_filter, 636 636 + "ip link add gre_test type gre remote %s local %s dev lo", 637 637 + TEST_TUNNEL_REMOTE, TEST_TUNNEL_LOCAL); 638 638 + SYS(out_clean_netif, "ip link set gre_test up"); 639 639 + return 0; 640 640 + 641 641 + out_clean_netif: 642 642 + SYS_NOFAIL("ip link del ipip_test"); 643 643 + out_remove_filter: 644 644 + remove_filter(); 645 645 + out_unset_addresses: 646 646 + unset_addresses(); 647 647 + return -1; 648 648 + } 649 649 + 650 650 + static void gre_shutdown(void) 651 651 + { 652 652 + SYS_NOFAIL("ip link del gre_test"); 653 653 + remove_filter(); 654 654 + unset_addresses(); 655 655 + } 656 656 + 657 657 + static const struct test_configuration tests_input[] = { 658 658 + { 659 659 + .name = "ipv4", 660 660 + .test_setup = ipv4_setup, 661 661 + .test_teardown = remove_filter, 662 662 + .source_ports = { 8, 9, 10 }, 663 663 + .cfg_l3_inner = PF_INET, 664 664 + .in_saddr4 = TEST_IN4_SRC_ADDR_DEFAULT, 665 665 + .in_daddr4 = TEST_IN4_DST_ADDR_DEFAULT 666 666 + }, 667 667 + { 668 668 + .name = "ipv4_continue_dissect", 669 669 + .test_setup = ipv4_setup, 670 670 + .test_teardown = remove_filter, 671 671 + .source_ports = { 8, 9, 10 }, 672 672 + .cfg_l3_inner = PF_INET, 673 673 + .in_saddr4 = TEST_IN4_SRC_ADDR_DISSECT_CONTINUE, 674 674 + .in_daddr4 = TEST_IN4_DST_ADDR_DEFAULT }, 675 675 + { 676 676 + .name = "ipip", 677 677 + .test_setup = ipip_setup, 678 678 + .test_teardown = ipip_shutdown, 679 679 + .source_ports = { 8, 9, 10 }, 680 680 + .cfg_l3_inner = PF_INET, 681 681 + .in_saddr4 = TEST_IN4_SRC_ADDR_IPIP, 682 682 + .in_daddr4 = TEST_IN4_DST_ADDR_IPIP, 683 683 + .out_saddr4 = TEST_OUT4_SRC_ADDR_DEFAULT, 684 684 + .out_daddr4 = TEST_OUT4_DST_ADDR_DEFAULT, 685 685 + .cfg_l3_outer = PF_INET, 686 686 + .cfg_encap_proto = IPPROTO_IPIP, 687 687 + 688 688 + }, 689 689 + { 690 690 + .name = "gre", 691 691 + .test_setup = gre_setup, 692 692 + .test_teardown = gre_shutdown, 693 693 + .source_ports = { 8, 9, 10 }, 694 694 + .cfg_l3_inner = PF_INET, 695 695 + .in_saddr4 = TEST_IN4_SRC_ADDR_IPIP, 696 696 + .in_daddr4 = TEST_IN4_DST_ADDR_IPIP, 697 697 + .out_saddr4 = TEST_OUT4_SRC_ADDR_DEFAULT, 698 698 + .out_daddr4 = TEST_OUT4_DST_ADDR_DEFAULT, 699 699 + .cfg_l3_outer = PF_INET, 700 700 + .cfg_encap_proto = IPPROTO_GRE, 701 701 + }, 702 702 + { 703 703 + .name = "port_range", 704 704 + .test_setup = port_range_setup, 705 705 + .test_teardown = remove_filter, 706 706 + .source_ports = { 7, 9, 11 }, 707 707 + .cfg_l3_inner = PF_INET, 708 708 + .in_saddr4 = TEST_IN4_SRC_ADDR_DEFAULT, 709 709 + .in_daddr4 = TEST_IN4_DST_ADDR_DEFAULT }, 710 710 + { 711 711 + .name = "ipv6", 712 712 + .test_setup = ipv6_setup, 713 713 + .test_teardown = remove_filter, 714 714 + .source_ports = { 8, 9, 10 }, 715 715 + .cfg_l3_inner = PF_INET6, 716 716 + .in_saddr6 = TEST_IN6_SRC_ADDR_DEFAULT, 717 717 + .in_daddr6 = TEST_IN6_DST_ADDR_DEFAULT 718 718 + }, 719 719 + }; 720 720 + 721 721 + struct test_ctx { 722 722 + struct bpf_flow *skel; 723 723 + struct netns_obj *ns; 724 724 + int prog_fd; 725 725 + }; 726 726 + 727 727 + static int test_global_init(struct test_ctx *ctx) 728 728 + { 729 729 + int err; 730 730 + 731 731 + ctx->skel = bpf_flow__open_and_load(); 732 732 + if (!ASSERT_OK_PTR(ctx->skel, "open and load flow_dissector")) 733 733 + return -1; 734 734 + 735 735 + ctx->ns = netns_new("flow_dissector_classification", true); 736 736 + if (!ASSERT_OK_PTR(ctx->ns, "switch ns")) 737 737 + goto out_destroy_skel; 738 738 + 739 739 + err = write_sysctl("/proc/sys/net/ipv4/conf/default/rp_filter", "0"); 740 740 + err |= write_sysctl("/proc/sys/net/ipv4/conf/all/rp_filter", "0"); 741 741 + err |= write_sysctl("/proc/sys/net/ipv4/conf/lo/rp_filter", "0"); 742 742 + if (!ASSERT_OK(err, "configure net tunables")) 743 743 + goto out_clean_ns; 744 744 + 745 745 + ctx->prog_fd = attach_and_configure_program(ctx->skel); 746 746 + if (!ASSERT_OK_FD(ctx->prog_fd, "attach and configure program")) 747 747 + goto out_clean_ns; 748 748 + return 0; 749 749 + out_clean_ns: 750 750 + netns_free(ctx->ns); 751 751 + out_destroy_skel: 752 752 + bpf_flow__destroy(ctx->skel); 753 753 + return -1; 754 754 + } 755 755 + 756 756 + static void test_global_shutdown(struct test_ctx *ctx) 757 757 + { 758 758 + detach_program(ctx->skel, ctx->prog_fd); 759 759 + netns_free(ctx->ns); 760 760 + bpf_flow__destroy(ctx->skel); 761 761 + } 762 762 + 763 763 + void test_flow_dissector_classification(void) 764 764 + { 765 765 + struct test_ctx ctx; 766 766 + const struct test_configuration *test; 767 767 + int i; 768 768 + 769 769 + if (test_global_init(&ctx)) 770 770 + return; 771 771 + 772 772 + for (i = 0; i < ARRAY_SIZE(tests_input); i++) { 773 773 + if (!test__start_subtest(tests_input[i].name)) 774 774 + continue; 775 775 + test = &tests_input[i]; 776 776 + /* All tests are expected to have one rx-ok port first, 777 777 + * then a non-working rx port, and finally a rx-ok port 778 778 + */ 779 779 + if (test->test_setup && 780 780 + !ASSERT_OK(test->test_setup(), "init filter")) 781 781 + continue; 782 782 + 783 783 + ASSERT_EQ(run_test(test, 0), TEST_PACKETS_COUNT, 784 784 + "test first port"); 785 785 + ASSERT_EQ(run_test(test, 1), 0, "test second port"); 786 786 + ASSERT_EQ(run_test(test, 2), TEST_PACKETS_COUNT, 787 787 + "test third port"); 788 788 + if (test->test_teardown) 789 789 + test->test_teardown(); 790 790 + } 791 791 + test_global_shutdown(&ctx); 792 792 + }

+165

tools/testing/selftests/bpf/prog_tests/free_timer.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (C) 2025. Huawei Technologies Co., Ltd */ 3 3 + #define _GNU_SOURCE 4 4 + #include <unistd.h> 5 5 + #include <sys/syscall.h> 6 6 + #include <test_progs.h> 7 7 + 8 8 + #include "free_timer.skel.h" 9 9 + 10 10 + struct run_ctx { 11 11 + struct bpf_program *start_prog; 12 12 + struct bpf_program *overwrite_prog; 13 13 + pthread_barrier_t notify; 14 14 + int loop; 15 15 + bool start; 16 16 + bool stop; 17 17 + }; 18 18 + 19 19 + static void start_threads(struct run_ctx *ctx) 20 20 + { 21 21 + ctx->start = true; 22 22 + } 23 23 + 24 24 + static void stop_threads(struct run_ctx *ctx) 25 25 + { 26 26 + ctx->stop = true; 27 27 + /* Guarantee the order between ->stop and ->start */ 28 28 + __atomic_store_n(&ctx->start, true, __ATOMIC_RELEASE); 29 29 + } 30 30 + 31 31 + static int wait_for_start(struct run_ctx *ctx) 32 32 + { 33 33 + while (!__atomic_load_n(&ctx->start, __ATOMIC_ACQUIRE)) 34 34 + usleep(10); 35 35 + 36 36 + return ctx->stop; 37 37 + } 38 38 + 39 39 + static void *overwrite_timer_fn(void *arg) 40 40 + { 41 41 + struct run_ctx *ctx = arg; 42 42 + int loop, fd, err; 43 43 + cpu_set_t cpuset; 44 44 + long ret = 0; 45 45 + 46 46 + /* Pin on CPU 0 */ 47 47 + CPU_ZERO(&cpuset); 48 48 + CPU_SET(0, &cpuset); 49 49 + pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset); 50 50 + 51 51 + /* Is the thread being stopped ? */ 52 52 + err = wait_for_start(ctx); 53 53 + if (err) 54 54 + return NULL; 55 55 + 56 56 + fd = bpf_program__fd(ctx->overwrite_prog); 57 57 + loop = ctx->loop; 58 58 + while (loop-- > 0) { 59 59 + LIBBPF_OPTS(bpf_test_run_opts, opts); 60 60 + 61 61 + /* Wait for start thread to complete */ 62 62 + pthread_barrier_wait(&ctx->notify); 63 63 + 64 64 + /* Overwrite timers */ 65 65 + err = bpf_prog_test_run_opts(fd, &opts); 66 66 + if (err) 67 67 + ret |= 1; 68 68 + else if (opts.retval) 69 69 + ret |= 2; 70 70 + 71 71 + /* Notify start thread to start timers */ 72 72 + pthread_barrier_wait(&ctx->notify); 73 73 + } 74 74 + 75 75 + return (void *)ret; 76 76 + } 77 77 + 78 78 + static void *start_timer_fn(void *arg) 79 79 + { 80 80 + struct run_ctx *ctx = arg; 81 81 + int loop, fd, err; 82 82 + cpu_set_t cpuset; 83 83 + long ret = 0; 84 84 + 85 85 + /* Pin on CPU 1 */ 86 86 + CPU_ZERO(&cpuset); 87 87 + CPU_SET(1, &cpuset); 88 88 + pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset); 89 89 + 90 90 + /* Is the thread being stopped ? */ 91 91 + err = wait_for_start(ctx); 92 92 + if (err) 93 93 + return NULL; 94 94 + 95 95 + fd = bpf_program__fd(ctx->start_prog); 96 96 + loop = ctx->loop; 97 97 + while (loop-- > 0) { 98 98 + LIBBPF_OPTS(bpf_test_run_opts, opts); 99 99 + 100 100 + /* Run the prog to start timer */ 101 101 + err = bpf_prog_test_run_opts(fd, &opts); 102 102 + if (err) 103 103 + ret |= 4; 104 104 + else if (opts.retval) 105 105 + ret |= 8; 106 106 + 107 107 + /* Notify overwrite thread to do overwrite */ 108 108 + pthread_barrier_wait(&ctx->notify); 109 109 + 110 110 + /* Wait for overwrite thread to complete */ 111 111 + pthread_barrier_wait(&ctx->notify); 112 112 + } 113 113 + 114 114 + return (void *)ret; 115 115 + } 116 116 + 117 117 + void test_free_timer(void) 118 118 + { 119 119 + struct free_timer *skel; 120 120 + struct bpf_program *prog; 121 121 + struct run_ctx ctx; 122 122 + pthread_t tid[2]; 123 123 + void *ret; 124 124 + int err; 125 125 + 126 126 + skel = free_timer__open_and_load(); 127 127 + if (!ASSERT_OK_PTR(skel, "open_load")) 128 128 + return; 129 129 + 130 130 + memset(&ctx, 0, sizeof(ctx)); 131 131 + 132 132 + prog = bpf_object__find_program_by_name(skel->obj, "start_timer"); 133 133 + if (!ASSERT_OK_PTR(prog, "find start prog")) 134 134 + goto out; 135 135 + ctx.start_prog = prog; 136 136 + 137 137 + prog = bpf_object__find_program_by_name(skel->obj, "overwrite_timer"); 138 138 + if (!ASSERT_OK_PTR(prog, "find overwrite prog")) 139 139 + goto out; 140 140 + ctx.overwrite_prog = prog; 141 141 + 142 142 + pthread_barrier_init(&ctx.notify, NULL, 2); 143 143 + ctx.loop = 10; 144 144 + 145 145 + err = pthread_create(&tid[0], NULL, start_timer_fn, &ctx); 146 146 + if (!ASSERT_OK(err, "create start_timer")) 147 147 + goto out; 148 148 + 149 149 + err = pthread_create(&tid[1], NULL, overwrite_timer_fn, &ctx); 150 150 + if (!ASSERT_OK(err, "create overwrite_timer")) { 151 151 + stop_threads(&ctx); 152 152 + goto out; 153 153 + } 154 154 + 155 155 + start_threads(&ctx); 156 156 + 157 157 + ret = NULL; 158 158 + err = pthread_join(tid[0], &ret); 159 159 + ASSERT_EQ(err | (long)ret, 0, "start_timer"); 160 160 + ret = NULL; 161 161 + err = pthread_join(tid[1], &ret); 162 162 + ASSERT_EQ(err | (long)ret, 0, "overwrite_timer"); 163 163 + out: 164 164 + free_timer__destroy(skel); 165 165 + }

+27

tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c

reviewed

··· 397 397 kprobe_multi_session_cookie__destroy(skel); 398 398 } 399 399 400 400 + static void test_unique_match(void) 401 401 + { 402 402 + LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); 403 403 + struct kprobe_multi *skel = NULL; 404 404 + struct bpf_link *link = NULL; 405 405 + 406 406 + skel = kprobe_multi__open_and_load(); 407 407 + if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load")) 408 408 + return; 409 409 + 410 410 + opts.unique_match = true; 411 411 + skel->bss->pid = getpid(); 412 412 + link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, 413 413 + "bpf_fentry_test*", &opts); 414 414 + if (!ASSERT_ERR_PTR(link, "bpf_program__attach_kprobe_multi_opts")) 415 415 + bpf_link__destroy(link); 416 416 + 417 417 + link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, 418 418 + "bpf_fentry_test8*", &opts); 419 419 + if (ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts")) 420 420 + bpf_link__destroy(link); 421 421 + 422 422 + kprobe_multi__destroy(skel); 423 423 + } 424 424 + 400 425 static size_t symbol_hash(long key, void *ctx __maybe_unused) 401 426 { 402 427 return str_hash((const char *) key); ··· 790 765 test_session_skel_api(); 791 766 if (test__start_subtest("session_cookie")) 792 767 test_session_cookie_skel_api(); 768 768 + if (test__start_subtest("unique_match")) 769 769 + test_unique_match(); 793 770 RUN_TESTS(kprobe_multi_verifier); 794 771 }

+1

tools/testing/selftests/bpf/prog_tests/missed.c

reviewed

··· 85 85 ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test3)), 1, "test3_recursion_misses"); 86 86 ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test4)), 1, "test4_recursion_misses"); 87 87 ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test5)), 1, "test5_recursion_misses"); 88 88 + ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test6)), 1, "test6_recursion_misses"); 88 89 89 90 cleanup: 90 91 missed_kprobe_recursion__destroy(skel);

+1 -1

tools/testing/selftests/bpf/prog_tests/sockopt_sk.c

reviewed

··· 2 2 #include <test_progs.h> 3 3 #include "cgroup_helpers.h" 4 4 5 5 - #include <linux/tcp.h> 5 5 + #include <netinet/tcp.h> 6 6 #include <linux/netlink.h> 7 7 #include "sockopt_sk.skel.h" 8 8

+6

tools/testing/selftests/bpf/prog_tests/verifier.c

reviewed

··· 52 52 #include "verifier_map_ptr_mixing.skel.h" 53 53 #include "verifier_map_ret_val.skel.h" 54 54 #include "verifier_masking.skel.h" 55 55 + #include "verifier_may_goto_1.skel.h" 56 56 + #include "verifier_may_goto_2.skel.h" 55 57 #include "verifier_meta_access.skel.h" 56 58 #include "verifier_movsx.skel.h" 57 59 #include "verifier_mtu.skel.h" ··· 100 98 #include "verifier_xdp_direct_packet_access.skel.h" 101 99 #include "verifier_bits_iter.skel.h" 102 100 #include "verifier_lsm.skel.h" 101 101 + #include "irq.skel.h" 103 102 104 103 #define MAX_ENTRIES 11 105 104 ··· 184 181 void test_verifier_map_ptr_mixing(void) { RUN(verifier_map_ptr_mixing); } 185 182 void test_verifier_map_ret_val(void) { RUN(verifier_map_ret_val); } 186 183 void test_verifier_masking(void) { RUN(verifier_masking); } 184 184 + void test_verifier_may_goto_1(void) { RUN(verifier_may_goto_1); } 185 185 + void test_verifier_may_goto_2(void) { RUN(verifier_may_goto_2); } 187 186 void test_verifier_meta_access(void) { RUN(verifier_meta_access); } 188 187 void test_verifier_movsx(void) { RUN(verifier_movsx); } 189 188 void test_verifier_netfilter_ctx(void) { RUN(verifier_netfilter_ctx); } ··· 230 225 void test_verifier_xdp_direct_packet_access(void) { RUN(verifier_xdp_direct_packet_access); } 231 226 void test_verifier_bits_iter(void) { RUN(verifier_bits_iter); } 232 227 void test_verifier_lsm(void) { RUN(verifier_lsm); } 228 228 + void test_irq(void) { RUN(irq); } 233 229 void test_verifier_mtu(void) { RUN(verifier_mtu); } 234 230 235 231 static int init_test_val_map(struct bpf_object *obj, char *map_name)

+2

tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c

reviewed

··· 82 82 /* SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) */ 83 83 #if defined(__s390x__) 84 84 int tailroom = 512; 85 85 + #elif defined(__powerpc__) 86 86 + int tailroom = 384; 85 87 #else 86 88 int tailroom = 320; 87 89 #endif

+1 -1

tools/testing/selftests/bpf/prog_tests/xdp_bonding.c

reviewed

··· 17 17 #include "network_helpers.h" 18 18 #include <linux/if_bonding.h> 19 19 #include <linux/limits.h> 20 20 - #include <linux/udp.h> 20 20 + #include <netinet/udp.h> 21 21 #include <uapi/linux/netdev.h> 22 22 23 23 #include "xdp_dummy.skel.h"

+165 -1

tools/testing/selftests/bpf/prog_tests/xdp_do_redirect.c

reviewed

··· 7 7 #include <linux/if_link.h> 8 8 #include <linux/ipv6.h> 9 9 #include <linux/in6.h> 10 10 - #include <linux/udp.h> 10 10 + #include <netinet/udp.h> 11 11 #include <bpf/bpf_endian.h> 12 12 #include <uapi/linux/netdev.h> 13 13 #include "test_xdp_do_redirect.skel.h" 14 14 + #include "xdp_dummy.skel.h" 14 15 15 16 struct udp_packet { 16 17 struct ethhdr eth; ··· 247 246 SYS_NOFAIL("ip netns del testns"); 248 247 test_xdp_do_redirect__destroy(skel); 249 248 } 249 249 + 250 250 + #define NS_NB 3 251 251 + #define NS0 "NS0" 252 252 + #define NS1 "NS1" 253 253 + #define NS2 "NS2" 254 254 + #define IPV4_NETWORK "10.1.1" 255 255 + #define VETH1_INDEX 111 256 256 + #define VETH2_INDEX 222 257 257 + 258 258 + struct test_data { 259 259 + struct netns_obj *ns[NS_NB]; 260 260 + u32 xdp_flags; 261 261 + }; 262 262 + 263 263 + static void cleanup(struct test_data *data) 264 264 + { 265 265 + int i; 266 266 + 267 267 + for (i = 0; i < NS_NB; i++) 268 268 + netns_free(data->ns[i]); 269 269 + } 270 270 + 271 271 + /** 272 272 + * ping_setup - 273 273 + * Create two veth peers and forward packets in-between using XDP 274 274 + * 275 275 + * ------------ ------------ 276 276 + * | NS1 | | NS2 | 277 277 + * | veth0 | | veth0 | 278 278 + * | 10.1.1.1 | | 10.1.1.2 | 279 279 + * -----|------ ------|----- 280 280 + * | | 281 281 + * | | 282 282 + * -----|-----------------------|------- 283 283 + * | veth1 veth2 | 284 284 + * | (id:111) (id:222) | 285 285 + * | | | | 286 286 + * | ----- xdp forwarding ----- | 287 287 + * | | 288 288 + * | NS0 | 289 289 + * ------------------------------------- 290 290 + */ 291 291 + static int ping_setup(struct test_data *data) 292 292 + { 293 293 + int i; 294 294 + 295 295 + data->ns[0] = netns_new(NS0, false); 296 296 + if (!ASSERT_OK_PTR(data->ns[0], "create ns")) 297 297 + return -1; 298 298 + 299 299 + for (i = 1; i < NS_NB; i++) { 300 300 + char ns_name[4] = {}; 301 301 + 302 302 + snprintf(ns_name, 4, "NS%d", i); 303 303 + data->ns[i] = netns_new(ns_name, false); 304 304 + if (!ASSERT_OK_PTR(data->ns[i], "create ns")) 305 305 + goto fail; 306 306 + 307 307 + SYS(fail, 308 308 + "ip -n %s link add veth%d index %d%d%d type veth peer name veth0 netns %s", 309 309 + NS0, i, i, i, i, ns_name); 310 310 + SYS(fail, "ip -n %s link set veth%d up", NS0, i); 311 311 + 312 312 + SYS(fail, "ip -n %s addr add %s.%d/24 dev veth0", ns_name, IPV4_NETWORK, i); 313 313 + SYS(fail, "ip -n %s link set veth0 up", ns_name); 314 314 + } 315 315 + 316 316 + return 0; 317 317 + 318 318 + fail: 319 319 + cleanup(data); 320 320 + return -1; 321 321 + } 322 322 + 323 323 + static void ping_test(struct test_data *data) 324 324 + { 325 325 + struct test_xdp_do_redirect *skel = NULL; 326 326 + struct xdp_dummy *skel_dummy = NULL; 327 327 + struct nstoken *nstoken = NULL; 328 328 + int i, ret; 329 329 + 330 330 + skel_dummy = xdp_dummy__open_and_load(); 331 331 + if (!ASSERT_OK_PTR(skel_dummy, "open and load xdp_dummy skeleton")) 332 332 + goto close; 333 333 + 334 334 + for (i = 1; i < NS_NB; i++) { 335 335 + char ns_name[4] = {}; 336 336 + 337 337 + snprintf(ns_name, 4, "NS%d", i); 338 338 + nstoken = open_netns(ns_name); 339 339 + if (!ASSERT_OK_PTR(nstoken, "open ns")) 340 340 + goto close; 341 341 + 342 342 + ret = bpf_xdp_attach(if_nametoindex("veth0"), 343 343 + bpf_program__fd(skel_dummy->progs.xdp_dummy_prog), 344 344 + data->xdp_flags, NULL); 345 345 + if (!ASSERT_GE(ret, 0, "bpf_xdp_attach dummy_prog")) 346 346 + goto close; 347 347 + 348 348 + close_netns(nstoken); 349 349 + nstoken = NULL; 350 350 + } 351 351 + 352 352 + skel = test_xdp_do_redirect__open_and_load(); 353 353 + if (!ASSERT_OK_PTR(skel, "open and load skeleton")) 354 354 + goto close; 355 355 + 356 356 + nstoken = open_netns(NS0); 357 357 + if (!ASSERT_OK_PTR(nstoken, "open NS0")) 358 358 + goto close; 359 359 + 360 360 + ret = bpf_xdp_attach(VETH2_INDEX, 361 361 + bpf_program__fd(skel->progs.xdp_redirect_to_111), 362 362 + data->xdp_flags, NULL); 363 363 + if (!ASSERT_GE(ret, 0, "bpf_xdp_attach")) 364 364 + goto close; 365 365 + 366 366 + ret = bpf_xdp_attach(VETH1_INDEX, 367 367 + bpf_program__fd(skel->progs.xdp_redirect_to_222), 368 368 + data->xdp_flags, NULL); 369 369 + if (!ASSERT_GE(ret, 0, "bpf_xdp_attach")) 370 370 + goto close; 371 371 + 372 372 + close_netns(nstoken); 373 373 + nstoken = NULL; 374 374 + 375 375 + nstoken = open_netns(NS1); 376 376 + if (!ASSERT_OK_PTR(nstoken, "open NS1")) 377 377 + goto close; 378 378 + 379 379 + SYS(close, "ping -c 1 %s.2 > /dev/null", IPV4_NETWORK); 380 380 + 381 381 + close: 382 382 + close_netns(nstoken); 383 383 + xdp_dummy__destroy(skel_dummy); 384 384 + test_xdp_do_redirect__destroy(skel); 385 385 + } 386 386 + 387 387 + 388 388 + static void xdp_redirect_ping(u32 xdp_flags) 389 389 + { 390 390 + struct test_data data = {}; 391 391 + 392 392 + if (ping_setup(&data) < 0) 393 393 + return; 394 394 + 395 395 + data.xdp_flags = xdp_flags; 396 396 + ping_test(&data); 397 397 + cleanup(&data); 398 398 + } 399 399 + 400 400 + void test_xdp_index_redirect(void) 401 401 + { 402 402 + if (test__start_subtest("noflag")) 403 403 + xdp_redirect_ping(0); 404 404 + 405 405 + if (test__start_subtest("drvflag")) 406 406 + xdp_redirect_ping(XDP_FLAGS_DRV_MODE); 407 407 + 408 408 + if (test__start_subtest("skbflag")) 409 409 + xdp_redirect_ping(XDP_FLAGS_SKB_MODE); 410 410 + } 411 411 +

+1 -1

tools/testing/selftests/bpf/prog_tests/xdp_flowtable.c

reviewed

··· 3 3 #include <network_helpers.h> 4 4 #include <bpf/btf.h> 5 5 #include <linux/if_link.h> 6 6 - #include <linux/udp.h> 6 6 + #include <netinet/udp.h> 7 7 #include <net/if.h> 8 8 #include <unistd.h> 9 9

+2 -19

tools/testing/selftests/bpf/prog_tests/xdp_metadata.c

reviewed

··· 10 10 #include <linux/errqueue.h> 11 11 #include <linux/if_link.h> 12 12 #include <linux/net_tstamp.h> 13 13 - #include <linux/udp.h> 13 13 + #include <netinet/udp.h> 14 14 #include <sys/mman.h> 15 15 #include <net/if.h> 16 16 #include <poll.h> ··· 133 133 munmap(xsk->umem_area, UMEM_SIZE); 134 134 } 135 135 136 136 - static void ip_csum(struct iphdr *iph) 137 137 - { 138 138 - __u32 sum = 0; 139 139 - __u16 *p; 140 140 - int i; 141 141 - 142 142 - iph->check = 0; 143 143 - p = (void *)iph; 144 144 - for (i = 0; i < sizeof(*iph) / sizeof(*p); i++) 145 145 - sum += p[i]; 146 146 - 147 147 - while (sum >> 16) 148 148 - sum = (sum & 0xffff) + (sum >> 16); 149 149 - 150 150 - iph->check = ~sum; 151 151 - } 152 152 - 153 136 static int generate_packet(struct xsk *xsk, __u16 dst_port) 154 137 { 155 138 struct xsk_tx_metadata *meta; ··· 175 192 iph->protocol = IPPROTO_UDP; 176 193 ASSERT_EQ(inet_pton(FAMILY, TX_ADDR, &iph->saddr), 1, "inet_pton(TX_ADDR)"); 177 194 ASSERT_EQ(inet_pton(FAMILY, RX_ADDR, &iph->daddr), 1, "inet_pton(RX_ADDR)"); 178 178 - ip_csum(iph); 195 195 + iph->check = build_ip_csum(iph); 179 196 180 197 udph->source = htons(UDP_SOURCE_PORT); 181 198 udph->dest = htons(dst_port);

+1 -1

tools/testing/selftests/bpf/progs/bad_struct_ops.c

reviewed

··· 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 - #include "../bpf_testmod/bpf_testmod.h" 6 6 + #include "../test_kmods/bpf_testmod.h" 7 7 8 8 char _license[] SEC("license") = "GPL"; 9 9

+1 -1

tools/testing/selftests/bpf/progs/cb_refs.c

reviewed

··· 2 2 #include <vmlinux.h> 3 3 #include <bpf/bpf_tracing.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 5 5 + #include "../test_kmods/bpf_testmod_kfunc.h" 6 6 7 7 struct map_value { 8 8 struct prog_test_ref_kfunc __kptr *ptr;

+15

tools/testing/selftests/bpf/progs/cgroup_skb_direct_packet_access.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + #include "vmlinux.h" 4 4 + #include <bpf/bpf_helpers.h> 5 5 + 6 6 + __u32 data_end; 7 7 + 8 8 + SEC("cgroup_skb/ingress") 9 9 + int direct_packet_access(struct __sk_buff *skb) 10 10 + { 11 11 + data_end = skb->data_end; 12 12 + return 1; 13 13 + } 14 14 + 15 15 + char _license[] SEC("license") = "GPL";

+3 -3

tools/testing/selftests/bpf/progs/dynptr_fail.c

reviewed

··· 192 192 193 193 /* Can't add a dynptr to a map */ 194 194 SEC("?raw_tp") 195 195 - __failure __msg("invalid indirect read from stack") 195 195 + __failure __msg("invalid read from stack") 196 196 int add_dynptr_to_map1(void *ctx) 197 197 { 198 198 struct bpf_dynptr ptr; ··· 210 210 211 211 /* Can't add a struct with an embedded dynptr to a map */ 212 212 SEC("?raw_tp") 213 213 - __failure __msg("invalid indirect read from stack") 213 213 + __failure __msg("invalid read from stack") 214 214 int add_dynptr_to_map2(void *ctx) 215 215 { 216 216 struct test_info x; ··· 398 398 * dynptr argument 399 399 */ 400 400 SEC("?raw_tp") 401 401 - __failure __msg("invalid indirect read from stack") 401 401 + __failure __msg("invalid read from stack") 402 402 int invalid_helper1(void *ctx) 403 403 { 404 404 struct bpf_dynptr ptr;

+2 -2

tools/testing/selftests/bpf/progs/epilogue_exit.c

reviewed

··· 4 4 #include <vmlinux.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 #include "bpf_misc.h" 7 7 - #include "../bpf_testmod/bpf_testmod.h" 8 8 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 7 7 + #include "../test_kmods/bpf_testmod.h" 8 8 + #include "../test_kmods/bpf_testmod_kfunc.h" 9 9 10 10 char _license[] SEC("license") = "GPL"; 11 11

+2 -2

tools/testing/selftests/bpf/progs/epilogue_tailcall.c

reviewed

··· 4 4 #include <vmlinux.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 #include "bpf_misc.h" 7 7 - #include "../bpf_testmod/bpf_testmod.h" 8 8 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 7 7 + #include "../test_kmods/bpf_testmod.h" 8 8 + #include "../test_kmods/bpf_testmod_kfunc.h" 9 9 10 10 char _license[] SEC("license") = "GPL"; 11 11

+2 -2

tools/testing/selftests/bpf/progs/exceptions_fail.c

reviewed

··· 131 131 } 132 132 133 133 SEC("?tc") 134 134 - __failure __msg("BPF_EXIT instruction cannot be used inside bpf_rcu_read_lock-ed region") 134 134 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_rcu_read_lock-ed region") 135 135 int reject_with_rcu_read_lock(void *ctx) 136 136 { 137 137 bpf_rcu_read_lock(); ··· 147 147 } 148 148 149 149 SEC("?tc") 150 150 - __failure __msg("BPF_EXIT instruction cannot be used inside bpf_rcu_read_lock-ed region") 150 150 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_rcu_read_lock-ed region") 151 151 int reject_subprog_with_rcu_read_lock(void *ctx) 152 152 { 153 153 bpf_rcu_read_lock();

+71

tools/testing/selftests/bpf/progs/free_timer.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (C) 2025. Huawei Technologies Co., Ltd */ 3 3 + #include <linux/bpf.h> 4 4 + #include <time.h> 5 5 + #include <bpf/bpf_tracing.h> 6 6 + #include <bpf/bpf_helpers.h> 7 7 + 8 8 + #define MAX_ENTRIES 8 9 9 + 10 10 + struct map_value { 11 11 + struct bpf_timer timer; 12 12 + }; 13 13 + 14 14 + struct { 15 15 + __uint(type, BPF_MAP_TYPE_HASH); 16 16 + __type(key, int); 17 17 + __type(value, struct map_value); 18 18 + __uint(max_entries, MAX_ENTRIES); 19 19 + } map SEC(".maps"); 20 20 + 21 21 + static int timer_cb(void *map, void *key, struct map_value *value) 22 22 + { 23 23 + volatile int sum = 0; 24 24 + int i; 25 25 + 26 26 + bpf_for(i, 0, 1024 * 1024) sum += i; 27 27 + 28 28 + return 0; 29 29 + } 30 30 + 31 31 + static int start_cb(int key) 32 32 + { 33 33 + struct map_value *value; 34 34 + 35 35 + value = bpf_map_lookup_elem(&map, (void *)&key); 36 36 + if (!value) 37 37 + return 0; 38 38 + 39 39 + bpf_timer_init(&value->timer, &map, CLOCK_MONOTONIC); 40 40 + bpf_timer_set_callback(&value->timer, timer_cb); 41 41 + /* Hope 100us will be enough to wake-up and run the overwrite thread */ 42 42 + bpf_timer_start(&value->timer, 100000, BPF_F_TIMER_CPU_PIN); 43 43 + 44 44 + return 0; 45 45 + } 46 46 + 47 47 + static int overwrite_cb(int key) 48 48 + { 49 49 + struct map_value zero = {}; 50 50 + 51 51 + /* Free the timer which may run on other CPU */ 52 52 + bpf_map_update_elem(&map, (void *)&key, &zero, BPF_ANY); 53 53 + 54 54 + return 0; 55 55 + } 56 56 + 57 57 + SEC("syscall") 58 58 + int BPF_PROG(start_timer) 59 59 + { 60 60 + bpf_loop(MAX_ENTRIES, start_cb, NULL, 0); 61 61 + return 0; 62 62 + } 63 63 + 64 64 + SEC("syscall") 65 65 + int BPF_PROG(overwrite_timer) 66 66 + { 67 67 + bpf_loop(MAX_ENTRIES, overwrite_cb, NULL, 0); 68 68 + return 0; 69 69 + } 70 70 + 71 71 + char _license[] SEC("license") = "GPL";

+444

tools/testing/selftests/bpf/progs/irq.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ 3 3 + #include <vmlinux.h> 4 4 + #include <bpf/bpf_helpers.h> 5 5 + #include "bpf_misc.h" 6 6 + #include "bpf_experimental.h" 7 7 + 8 8 + unsigned long global_flags; 9 9 + 10 10 + extern void bpf_local_irq_save(unsigned long *) __weak __ksym; 11 11 + extern void bpf_local_irq_restore(unsigned long *) __weak __ksym; 12 12 + extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym; 13 13 + 14 14 + SEC("?tc") 15 15 + __failure __msg("arg#0 doesn't point to an irq flag on stack") 16 16 + int irq_save_bad_arg(struct __sk_buff *ctx) 17 17 + { 18 18 + bpf_local_irq_save(&global_flags); 19 19 + return 0; 20 20 + } 21 21 + 22 22 + SEC("?tc") 23 23 + __failure __msg("arg#0 doesn't point to an irq flag on stack") 24 24 + int irq_restore_bad_arg(struct __sk_buff *ctx) 25 25 + { 26 26 + bpf_local_irq_restore(&global_flags); 27 27 + return 0; 28 28 + } 29 29 + 30 30 + SEC("?tc") 31 31 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 32 32 + int irq_restore_missing_2(struct __sk_buff *ctx) 33 33 + { 34 34 + unsigned long flags1; 35 35 + unsigned long flags2; 36 36 + 37 37 + bpf_local_irq_save(&flags1); 38 38 + bpf_local_irq_save(&flags2); 39 39 + return 0; 40 40 + } 41 41 + 42 42 + SEC("?tc") 43 43 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 44 44 + int irq_restore_missing_3(struct __sk_buff *ctx) 45 45 + { 46 46 + unsigned long flags1; 47 47 + unsigned long flags2; 48 48 + unsigned long flags3; 49 49 + 50 50 + bpf_local_irq_save(&flags1); 51 51 + bpf_local_irq_save(&flags2); 52 52 + bpf_local_irq_save(&flags3); 53 53 + return 0; 54 54 + } 55 55 + 56 56 + SEC("?tc") 57 57 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 58 58 + int irq_restore_missing_3_minus_2(struct __sk_buff *ctx) 59 59 + { 60 60 + unsigned long flags1; 61 61 + unsigned long flags2; 62 62 + unsigned long flags3; 63 63 + 64 64 + bpf_local_irq_save(&flags1); 65 65 + bpf_local_irq_save(&flags2); 66 66 + bpf_local_irq_save(&flags3); 67 67 + bpf_local_irq_restore(&flags3); 68 68 + bpf_local_irq_restore(&flags2); 69 69 + return 0; 70 70 + } 71 71 + 72 72 + static __noinline void local_irq_save(unsigned long *flags) 73 73 + { 74 74 + bpf_local_irq_save(flags); 75 75 + } 76 76 + 77 77 + static __noinline void local_irq_restore(unsigned long *flags) 78 78 + { 79 79 + bpf_local_irq_restore(flags); 80 80 + } 81 81 + 82 82 + SEC("?tc") 83 83 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 84 84 + int irq_restore_missing_1_subprog(struct __sk_buff *ctx) 85 85 + { 86 86 + unsigned long flags; 87 87 + 88 88 + local_irq_save(&flags); 89 89 + return 0; 90 90 + } 91 91 + 92 92 + SEC("?tc") 93 93 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 94 94 + int irq_restore_missing_2_subprog(struct __sk_buff *ctx) 95 95 + { 96 96 + unsigned long flags1; 97 97 + unsigned long flags2; 98 98 + 99 99 + local_irq_save(&flags1); 100 100 + local_irq_save(&flags2); 101 101 + return 0; 102 102 + } 103 103 + 104 104 + SEC("?tc") 105 105 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 106 106 + int irq_restore_missing_3_subprog(struct __sk_buff *ctx) 107 107 + { 108 108 + unsigned long flags1; 109 109 + unsigned long flags2; 110 110 + unsigned long flags3; 111 111 + 112 112 + local_irq_save(&flags1); 113 113 + local_irq_save(&flags2); 114 114 + local_irq_save(&flags3); 115 115 + return 0; 116 116 + } 117 117 + 118 118 + SEC("?tc") 119 119 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 120 120 + int irq_restore_missing_3_minus_2_subprog(struct __sk_buff *ctx) 121 121 + { 122 122 + unsigned long flags1; 123 123 + unsigned long flags2; 124 124 + unsigned long flags3; 125 125 + 126 126 + local_irq_save(&flags1); 127 127 + local_irq_save(&flags2); 128 128 + local_irq_save(&flags3); 129 129 + local_irq_restore(&flags3); 130 130 + local_irq_restore(&flags2); 131 131 + return 0; 132 132 + } 133 133 + 134 134 + SEC("?tc") 135 135 + __success 136 136 + int irq_balance(struct __sk_buff *ctx) 137 137 + { 138 138 + unsigned long flags; 139 139 + 140 140 + local_irq_save(&flags); 141 141 + local_irq_restore(&flags); 142 142 + return 0; 143 143 + } 144 144 + 145 145 + SEC("?tc") 146 146 + __success 147 147 + int irq_balance_n(struct __sk_buff *ctx) 148 148 + { 149 149 + unsigned long flags1; 150 150 + unsigned long flags2; 151 151 + unsigned long flags3; 152 152 + 153 153 + local_irq_save(&flags1); 154 154 + local_irq_save(&flags2); 155 155 + local_irq_save(&flags3); 156 156 + local_irq_restore(&flags3); 157 157 + local_irq_restore(&flags2); 158 158 + local_irq_restore(&flags1); 159 159 + return 0; 160 160 + } 161 161 + 162 162 + static __noinline void local_irq_balance(void) 163 163 + { 164 164 + unsigned long flags; 165 165 + 166 166 + local_irq_save(&flags); 167 167 + local_irq_restore(&flags); 168 168 + } 169 169 + 170 170 + static __noinline void local_irq_balance_n(void) 171 171 + { 172 172 + unsigned long flags1; 173 173 + unsigned long flags2; 174 174 + unsigned long flags3; 175 175 + 176 176 + local_irq_save(&flags1); 177 177 + local_irq_save(&flags2); 178 178 + local_irq_save(&flags3); 179 179 + local_irq_restore(&flags3); 180 180 + local_irq_restore(&flags2); 181 181 + local_irq_restore(&flags1); 182 182 + } 183 183 + 184 184 + SEC("?tc") 185 185 + __success 186 186 + int irq_balance_subprog(struct __sk_buff *ctx) 187 187 + { 188 188 + local_irq_balance(); 189 189 + return 0; 190 190 + } 191 191 + 192 192 + SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") 193 193 + __failure __msg("sleepable helper bpf_copy_from_user#") 194 194 + int irq_sleepable_helper(void *ctx) 195 195 + { 196 196 + unsigned long flags; 197 197 + u32 data; 198 198 + 199 199 + local_irq_save(&flags); 200 200 + bpf_copy_from_user(&data, sizeof(data), NULL); 201 201 + local_irq_restore(&flags); 202 202 + return 0; 203 203 + } 204 204 + 205 205 + SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") 206 206 + __failure __msg("kernel func bpf_copy_from_user_str is sleepable within IRQ-disabled region") 207 207 + int irq_sleepable_kfunc(void *ctx) 208 208 + { 209 209 + unsigned long flags; 210 210 + u32 data; 211 211 + 212 212 + local_irq_save(&flags); 213 213 + bpf_copy_from_user_str(&data, sizeof(data), NULL, 0); 214 214 + local_irq_restore(&flags); 215 215 + return 0; 216 216 + } 217 217 + 218 218 + int __noinline global_local_irq_balance(void) 219 219 + { 220 220 + local_irq_balance_n(); 221 221 + return 0; 222 222 + } 223 223 + 224 224 + SEC("?tc") 225 225 + __failure __msg("global function calls are not allowed with IRQs disabled") 226 226 + int irq_global_subprog(struct __sk_buff *ctx) 227 227 + { 228 228 + unsigned long flags; 229 229 + 230 230 + bpf_local_irq_save(&flags); 231 231 + global_local_irq_balance(); 232 232 + bpf_local_irq_restore(&flags); 233 233 + return 0; 234 234 + } 235 235 + 236 236 + SEC("?tc") 237 237 + __failure __msg("cannot restore irq state out of order") 238 238 + int irq_restore_ooo(struct __sk_buff *ctx) 239 239 + { 240 240 + unsigned long flags1; 241 241 + unsigned long flags2; 242 242 + 243 243 + bpf_local_irq_save(&flags1); 244 244 + bpf_local_irq_save(&flags2); 245 245 + bpf_local_irq_restore(&flags1); 246 246 + bpf_local_irq_restore(&flags2); 247 247 + return 0; 248 248 + } 249 249 + 250 250 + SEC("?tc") 251 251 + __failure __msg("cannot restore irq state out of order") 252 252 + int irq_restore_ooo_3(struct __sk_buff *ctx) 253 253 + { 254 254 + unsigned long flags1; 255 255 + unsigned long flags2; 256 256 + unsigned long flags3; 257 257 + 258 258 + bpf_local_irq_save(&flags1); 259 259 + bpf_local_irq_save(&flags2); 260 260 + bpf_local_irq_restore(&flags2); 261 261 + bpf_local_irq_save(&flags3); 262 262 + bpf_local_irq_restore(&flags1); 263 263 + bpf_local_irq_restore(&flags3); 264 264 + return 0; 265 265 + } 266 266 + 267 267 + static __noinline void local_irq_save_3(unsigned long *flags1, unsigned long *flags2, 268 268 + unsigned long *flags3) 269 269 + { 270 270 + local_irq_save(flags1); 271 271 + local_irq_save(flags2); 272 272 + local_irq_save(flags3); 273 273 + } 274 274 + 275 275 + SEC("?tc") 276 276 + __success 277 277 + int irq_restore_3_subprog(struct __sk_buff *ctx) 278 278 + { 279 279 + unsigned long flags1; 280 280 + unsigned long flags2; 281 281 + unsigned long flags3; 282 282 + 283 283 + local_irq_save_3(&flags1, &flags2, &flags3); 284 284 + bpf_local_irq_restore(&flags3); 285 285 + bpf_local_irq_restore(&flags2); 286 286 + bpf_local_irq_restore(&flags1); 287 287 + return 0; 288 288 + } 289 289 + 290 290 + SEC("?tc") 291 291 + __failure __msg("cannot restore irq state out of order") 292 292 + int irq_restore_4_subprog(struct __sk_buff *ctx) 293 293 + { 294 294 + unsigned long flags1; 295 295 + unsigned long flags2; 296 296 + unsigned long flags3; 297 297 + unsigned long flags4; 298 298 + 299 299 + local_irq_save_3(&flags1, &flags2, &flags3); 300 300 + bpf_local_irq_restore(&flags3); 301 301 + bpf_local_irq_save(&flags4); 302 302 + bpf_local_irq_restore(&flags4); 303 303 + bpf_local_irq_restore(&flags1); 304 304 + return 0; 305 305 + } 306 306 + 307 307 + SEC("?tc") 308 308 + __failure __msg("cannot restore irq state out of order") 309 309 + int irq_restore_ooo_3_subprog(struct __sk_buff *ctx) 310 310 + { 311 311 + unsigned long flags1; 312 312 + unsigned long flags2; 313 313 + unsigned long flags3; 314 314 + 315 315 + local_irq_save_3(&flags1, &flags2, &flags3); 316 316 + bpf_local_irq_restore(&flags3); 317 317 + bpf_local_irq_restore(&flags2); 318 318 + bpf_local_irq_save(&flags3); 319 319 + bpf_local_irq_restore(&flags1); 320 320 + return 0; 321 321 + } 322 322 + 323 323 + SEC("?tc") 324 324 + __failure __msg("expected an initialized") 325 325 + int irq_restore_invalid(struct __sk_buff *ctx) 326 326 + { 327 327 + unsigned long flags1; 328 328 + unsigned long flags = 0xfaceb00c; 329 329 + 330 330 + bpf_local_irq_save(&flags1); 331 331 + bpf_local_irq_restore(&flags); 332 332 + return 0; 333 333 + } 334 334 + 335 335 + SEC("?tc") 336 336 + __failure __msg("expected uninitialized") 337 337 + int irq_save_invalid(struct __sk_buff *ctx) 338 338 + { 339 339 + unsigned long flags1; 340 340 + 341 341 + bpf_local_irq_save(&flags1); 342 342 + bpf_local_irq_save(&flags1); 343 343 + return 0; 344 344 + } 345 345 + 346 346 + SEC("?tc") 347 347 + __failure __msg("expected an initialized") 348 348 + int irq_restore_iter(struct __sk_buff *ctx) 349 349 + { 350 350 + struct bpf_iter_num it; 351 351 + 352 352 + bpf_iter_num_new(&it, 0, 42); 353 353 + bpf_local_irq_restore((unsigned long *)&it); 354 354 + return 0; 355 355 + } 356 356 + 357 357 + SEC("?tc") 358 358 + __failure __msg("Unreleased reference id=1") 359 359 + int irq_save_iter(struct __sk_buff *ctx) 360 360 + { 361 361 + struct bpf_iter_num it; 362 362 + 363 363 + /* Ensure same sized slot has st->ref_obj_id set, so we reject based on 364 364 + * slot_type != STACK_IRQ_FLAG... 365 365 + */ 366 366 + _Static_assert(sizeof(it) == sizeof(unsigned long), "broken iterator size"); 367 367 + 368 368 + bpf_iter_num_new(&it, 0, 42); 369 369 + bpf_local_irq_save((unsigned long *)&it); 370 370 + bpf_local_irq_restore((unsigned long *)&it); 371 371 + return 0; 372 372 + } 373 373 + 374 374 + SEC("?tc") 375 375 + __failure __msg("expected an initialized") 376 376 + int irq_flag_overwrite(struct __sk_buff *ctx) 377 377 + { 378 378 + unsigned long flags; 379 379 + 380 380 + bpf_local_irq_save(&flags); 381 381 + flags = 0xdeadbeef; 382 382 + bpf_local_irq_restore(&flags); 383 383 + return 0; 384 384 + } 385 385 + 386 386 + SEC("?tc") 387 387 + __failure __msg("expected an initialized") 388 388 + int irq_flag_overwrite_partial(struct __sk_buff *ctx) 389 389 + { 390 390 + unsigned long flags; 391 391 + 392 392 + bpf_local_irq_save(&flags); 393 393 + *(((char *)&flags) + 1) = 0xff; 394 394 + bpf_local_irq_restore(&flags); 395 395 + return 0; 396 396 + } 397 397 + 398 398 + SEC("?tc") 399 399 + __failure __msg("cannot restore irq state out of order") 400 400 + int irq_ooo_refs_array(struct __sk_buff *ctx) 401 401 + { 402 402 + unsigned long flags[4]; 403 403 + struct { int i; } *p; 404 404 + 405 405 + /* refs=1 */ 406 406 + bpf_local_irq_save(&flags[0]); 407 407 + 408 408 + /* refs=1,2 */ 409 409 + p = bpf_obj_new(typeof(*p)); 410 410 + if (!p) { 411 411 + bpf_local_irq_restore(&flags[0]); 412 412 + return 0; 413 413 + } 414 414 + 415 415 + /* refs=1,2,3 */ 416 416 + bpf_local_irq_save(&flags[1]); 417 417 + 418 418 + /* refs=1,2,3,4 */ 419 419 + bpf_local_irq_save(&flags[2]); 420 420 + 421 421 + /* Now when we remove ref=2, the verifier must not break the ordering in 422 422 + * the refs array between 1,3,4. With an older implementation, the 423 423 + * verifier would swap the last element with the removed element, but to 424 424 + * maintain the stack property we need to use memmove. 425 425 + */ 426 426 + bpf_obj_drop(p); 427 427 + 428 428 + /* Save and restore to reset active_irq_id to 3, as the ordering is now 429 429 + * refs=1,4,3. When restoring the linear scan will find prev_id in order 430 430 + * as 3 instead of 4. 431 431 + */ 432 432 + bpf_local_irq_save(&flags[3]); 433 433 + bpf_local_irq_restore(&flags[3]); 434 434 + 435 435 + /* With the incorrect implementation, we can release flags[1], flags[2], 436 436 + * and flags[0], i.e. in the wrong order. 437 437 + */ 438 438 + bpf_local_irq_restore(&flags[1]); 439 439 + bpf_local_irq_restore(&flags[2]); 440 440 + bpf_local_irq_restore(&flags[0]); 441 441 + return 0; 442 442 + } 443 443 + 444 444 + char _license[] SEC("license") = "GPL";

+7 -7

tools/testing/selftests/bpf/progs/iters.c

reviewed

··· 524 524 } 525 525 526 526 struct { 527 527 - __uint(type, BPF_MAP_TYPE_ARRAY); 527 527 + __uint(type, BPF_MAP_TYPE_HASH); 528 528 __type(key, int); 529 529 __type(value, int); 530 530 __uint(max_entries, 1000); 531 531 - } arr_map SEC(".maps"); 531 531 + } hash_map SEC(".maps"); 532 532 533 533 SEC("?raw_tp") 534 534 __failure __msg("invalid mem access 'scalar'") ··· 539 539 540 540 MY_PID_GUARD(); 541 541 542 542 - map_val = bpf_map_lookup_elem(&arr_map, &key); 542 542 + map_val = bpf_map_lookup_elem(&hash_map, &key); 543 543 if (!map_val) 544 544 return 0; 545 545 ··· 561 561 562 562 MY_PID_GUARD(); 563 563 564 564 - map_val = bpf_map_lookup_elem(&arr_map, &key); 564 564 + map_val = bpf_map_lookup_elem(&hash_map, &key); 565 565 if (!map_val) 566 566 return 0; 567 567 568 568 bpf_repeat(1000000) { 569 569 - map_val = bpf_map_lookup_elem(&arr_map, &key); 569 569 + map_val = bpf_map_lookup_elem(&hash_map, &key); 570 570 } 571 571 572 572 *map_val = 123; ··· 585 585 MY_PID_GUARD(); 586 586 587 587 bpf_repeat(1000000) { 588 588 - map_val = bpf_map_lookup_elem(&arr_map, &key); 588 588 + map_val = bpf_map_lookup_elem(&hash_map, &key); 589 589 found = true; 590 590 } 591 591 ··· 606 606 MY_PID_GUARD(); 607 607 608 608 bpf_repeat(1000000) { 609 609 - map_val = bpf_map_lookup_elem(&arr_map, &key); 609 609 + map_val = bpf_map_lookup_elem(&hash_map, &key); 610 610 if (map_val) { 611 611 found = true; 612 612 break;

+1 -1

tools/testing/selftests/bpf/progs/iters_testmod.c

reviewed

··· 4 4 #include "bpf_experimental.h" 5 5 #include <bpf/bpf_helpers.h> 6 6 #include "bpf_misc.h" 7 7 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 7 7 + #include "../test_kmods/bpf_testmod_kfunc.h" 8 8 9 9 char _license[] SEC("license") = "GPL"; 10 10

+1 -1

tools/testing/selftests/bpf/progs/jit_probe_mem.c

reviewed

··· 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_tracing.h> 5 5 #include <bpf/bpf_helpers.h> 6 6 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 6 6 + #include "../test_kmods/bpf_testmod_kfunc.h" 7 7 8 8 static struct prog_test_ref_kfunc __kptr *v; 9 9 long total_sum = -1;

+1 -1

tools/testing/selftests/bpf/progs/kfunc_call_destructive.c

reviewed

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 #include <vmlinux.h> 3 3 #include <bpf/bpf_helpers.h> 4 4 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 4 4 + #include "../test_kmods/bpf_testmod_kfunc.h" 5 5 6 6 SEC("tc") 7 7 int kfunc_destructive_test(void)

+1 -1

tools/testing/selftests/bpf/progs/kfunc_call_fail.c

reviewed

··· 2 2 /* Copyright (c) 2021 Facebook */ 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 5 5 + #include "../test_kmods/bpf_testmod_kfunc.h" 6 6 7 7 struct syscall_test_args { 8 8 __u8 data[16];

+1 -1

tools/testing/selftests/bpf/progs/kfunc_call_race.c

reviewed

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 #include <vmlinux.h> 3 3 #include <bpf/bpf_helpers.h> 4 4 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 4 4 + #include "../test_kmods/bpf_testmod_kfunc.h" 5 5 6 6 SEC("tc") 7 7 int kfunc_call_fail(struct __sk_buff *ctx)

+1 -1

tools/testing/selftests/bpf/progs/kfunc_call_test.c

reviewed

··· 2 2 /* Copyright (c) 2021 Facebook */ 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 5 5 + #include "../test_kmods/bpf_testmod_kfunc.h" 6 6 7 7 SEC("tc") 8 8 int kfunc_call_test4(struct __sk_buff *skb)

+1 -1

tools/testing/selftests/bpf/progs/kfunc_call_test_subprog.c

reviewed

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 /* Copyright (c) 2021 Facebook */ 3 3 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 3 3 + #include "../test_kmods/bpf_testmod_kfunc.h" 4 4 5 5 extern const int bpf_prog_active __ksym; 6 6 int active_res = -1;

+1 -1

tools/testing/selftests/bpf/progs/local_kptr_stash.c

reviewed

··· 6 6 #include <bpf/bpf_helpers.h> 7 7 #include <bpf/bpf_core_read.h> 8 8 #include "../bpf_experimental.h" 9 9 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 9 9 + #include "../test_kmods/bpf_testmod_kfunc.h" 10 10 11 11 struct plain_local; 12 12

+1 -1

tools/testing/selftests/bpf/progs/map_kptr.c

reviewed

··· 2 2 #include <vmlinux.h> 3 3 #include <bpf/bpf_tracing.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 5 5 + #include "../test_kmods/bpf_testmod_kfunc.h" 6 6 7 7 struct map_value { 8 8 struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr;

+2 -2

tools/testing/selftests/bpf/progs/map_kptr_fail.c

reviewed

··· 4 4 #include <bpf/bpf_helpers.h> 5 5 #include <bpf/bpf_core_read.h> 6 6 #include "bpf_misc.h" 7 7 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 7 7 + #include "../test_kmods/bpf_testmod_kfunc.h" 8 8 9 9 struct map_value { 10 10 char buf[8]; ··· 345 345 } 346 346 347 347 SEC("?tc") 348 348 - __failure __msg("Unreleased reference id=5 alloc_insn=") 348 348 + __failure __msg("Unreleased reference id=4 alloc_insn=") 349 349 int kptr_xchg_ref_state(struct __sk_buff *ctx) 350 350 { 351 351 struct prog_test_ref_kfunc *p;

+1 -1

tools/testing/selftests/bpf/progs/missed_kprobe.c

reviewed

··· 2 2 #include "vmlinux.h" 3 3 #include <bpf/bpf_helpers.h> 4 4 #include <bpf/bpf_tracing.h> 5 5 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 5 5 + #include "../test_kmods/bpf_testmod_kfunc.h" 6 6 7 7 char _license[] SEC("license") = "GPL"; 8 8

+7 -1

tools/testing/selftests/bpf/progs/missed_kprobe_recursion.c

reviewed

··· 2 2 #include "vmlinux.h" 3 3 #include <bpf/bpf_helpers.h> 4 4 #include <bpf/bpf_tracing.h> 5 5 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 5 5 + #include "../test_kmods/bpf_testmod_kfunc.h" 6 6 7 7 char _license[] SEC("license") = "GPL"; 8 8 ··· 43 43 44 44 SEC("kprobe.multi/bpf_kfunc_common_test") 45 45 int test5(struct pt_regs *ctx) 46 46 + { 47 47 + return 0; 48 48 + } 49 49 + 50 50 + SEC("kprobe.session/bpf_kfunc_common_test") 51 51 + int test6(struct pt_regs *ctx) 46 52 { 47 53 return 0; 48 54 }

+1 -1

tools/testing/selftests/bpf/progs/nested_acquire.c

reviewed

··· 4 4 #include <bpf/bpf_tracing.h> 5 5 #include <bpf/bpf_helpers.h> 6 6 #include "bpf_misc.h" 7 7 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 7 7 + #include "../test_kmods/bpf_testmod_kfunc.h" 8 8 9 9 char _license[] SEC("license") = "GPL"; 10 10

+21 -7

tools/testing/selftests/bpf/progs/preempt_lock.c

reviewed

··· 5 5 #include "bpf_misc.h" 6 6 #include "bpf_experimental.h" 7 7 8 8 + extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym; 9 9 + 8 10 SEC("?tc") 9 9 - __failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region") 11 11 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable-ed region") 10 12 int preempt_lock_missing_1(struct __sk_buff *ctx) 11 13 { 12 14 bpf_preempt_disable(); ··· 16 14 } 17 15 18 16 SEC("?tc") 19 19 - __failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region") 17 17 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable-ed region") 20 18 int preempt_lock_missing_2(struct __sk_buff *ctx) 21 19 { 22 20 bpf_preempt_disable(); ··· 25 23 } 26 24 27 25 SEC("?tc") 28 28 - __failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region") 26 26 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable-ed region") 29 27 int preempt_lock_missing_3(struct __sk_buff *ctx) 30 28 { 31 29 bpf_preempt_disable(); ··· 35 33 } 36 34 37 35 SEC("?tc") 38 38 - __failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region") 36 36 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable-ed region") 39 37 int preempt_lock_missing_3_minus_2(struct __sk_buff *ctx) 40 38 { 41 39 bpf_preempt_disable(); ··· 57 55 } 58 56 59 57 SEC("?tc") 60 60 - __failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region") 58 58 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable-ed region") 61 59 int preempt_lock_missing_1_subprog(struct __sk_buff *ctx) 62 60 { 63 61 preempt_disable(); ··· 65 63 } 66 64 67 65 SEC("?tc") 68 68 - __failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region") 66 66 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable-ed region") 69 67 int preempt_lock_missing_2_subprog(struct __sk_buff *ctx) 70 68 { 71 69 preempt_disable(); ··· 74 72 } 75 73 76 74 SEC("?tc") 77 77 - __failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region") 75 75 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable-ed region") 78 76 int preempt_lock_missing_2_minus_1_subprog(struct __sk_buff *ctx) 79 77 { 80 78 preempt_disable(); ··· 111 109 112 110 bpf_preempt_disable(); 113 111 bpf_copy_from_user(&data, sizeof(data), NULL); 112 112 + bpf_preempt_enable(); 113 113 + return 0; 114 114 + } 115 115 + 116 116 + SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") 117 117 + __failure __msg("kernel func bpf_copy_from_user_str is sleepable within non-preemptible region") 118 118 + int preempt_sleepable_kfunc(void *ctx) 119 119 + { 120 120 + u32 data; 121 121 + 122 122 + bpf_preempt_disable(); 123 123 + bpf_copy_from_user_str(&data, sizeof(data), NULL, 0); 114 124 bpf_preempt_enable(); 115 125 return 0; 116 126 }

+2 -2

tools/testing/selftests/bpf/progs/pro_epilogue.c

reviewed

··· 4 4 #include <vmlinux.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 #include "bpf_misc.h" 7 7 - #include "../bpf_testmod/bpf_testmod.h" 8 8 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 7 7 + #include "../test_kmods/bpf_testmod.h" 8 8 + #include "../test_kmods/bpf_testmod_kfunc.h" 9 9 10 10 char _license[] SEC("license") = "GPL"; 11 11

+2 -2

tools/testing/selftests/bpf/progs/pro_epilogue_goto_start.c

reviewed

··· 4 4 #include <vmlinux.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 #include "bpf_misc.h" 7 7 - #include "../bpf_testmod/bpf_testmod.h" 8 8 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 7 7 + #include "../test_kmods/bpf_testmod.h" 8 8 + #include "../test_kmods/bpf_testmod_kfunc.h" 9 9 10 10 char _license[] SEC("license") = "GPL"; 11 11

+1 -1

tools/testing/selftests/bpf/progs/sock_addr_kern.c

reviewed

··· 2 2 /* Copyright (c) 2024 Google LLC */ 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 5 5 + #include "../test_kmods/bpf_testmod_kfunc.h" 6 6 7 7 SEC("syscall") 8 8 int init_sock(struct init_sock_args *args)

+1 -1

tools/testing/selftests/bpf/progs/struct_ops_detach.c

reviewed

··· 2 2 /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 - #include "../bpf_testmod/bpf_testmod.h" 5 5 + #include "../test_kmods/bpf_testmod.h" 6 6 7 7 char _license[] SEC("license") = "GPL"; 8 8

+1 -1

tools/testing/selftests/bpf/progs/struct_ops_forgotten_cb.c

reviewed

··· 2 2 /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_tracing.h> 5 5 - #include "../bpf_testmod/bpf_testmod.h" 5 5 + #include "../test_kmods/bpf_testmod.h" 6 6 7 7 char _license[] SEC("license") = "GPL"; 8 8

+1 -1

tools/testing/selftests/bpf/progs/struct_ops_maybe_null.c

reviewed

··· 2 2 /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_tracing.h> 5 5 - #include "../bpf_testmod/bpf_testmod.h" 5 5 + #include "../test_kmods/bpf_testmod.h" 6 6 7 7 char _license[] SEC("license") = "GPL"; 8 8

+1 -1

tools/testing/selftests/bpf/progs/struct_ops_maybe_null_fail.c

reviewed

··· 2 2 /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_tracing.h> 5 5 - #include "../bpf_testmod/bpf_testmod.h" 5 5 + #include "../test_kmods/bpf_testmod.h" 6 6 7 7 char _license[] SEC("license") = "GPL"; 8 8

+1 -1

tools/testing/selftests/bpf/progs/struct_ops_module.c

reviewed

··· 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 - #include "../bpf_testmod/bpf_testmod.h" 6 6 + #include "../test_kmods/bpf_testmod.h" 7 7 8 8 char _license[] SEC("license") = "GPL"; 9 9

+1 -1

tools/testing/selftests/bpf/progs/struct_ops_multi_pages.c

reviewed

··· 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 - #include "../bpf_testmod/bpf_testmod.h" 6 6 + #include "../test_kmods/bpf_testmod.h" 7 7 8 8 char _license[] SEC("license") = "GPL"; 9 9

+1 -1

tools/testing/selftests/bpf/progs/struct_ops_nulled_out_cb.c

reviewed

··· 2 2 /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_tracing.h> 5 5 - #include "../bpf_testmod/bpf_testmod.h" 5 5 + #include "../test_kmods/bpf_testmod.h" 6 6 7 7 char _license[] SEC("license") = "GPL"; 8 8

+1 -1

tools/testing/selftests/bpf/progs/struct_ops_private_stack.c

reviewed

··· 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 - #include "../bpf_testmod/bpf_testmod.h" 6 6 + #include "../test_kmods/bpf_testmod.h" 7 7 8 8 char _license[] SEC("license") = "GPL"; 9 9

+1 -1

tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c

reviewed

··· 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 - #include "../bpf_testmod/bpf_testmod.h" 6 6 + #include "../test_kmods/bpf_testmod.h" 7 7 8 8 char _license[] SEC("license") = "GPL"; 9 9

+1 -1

tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c

reviewed

··· 3 3 #include <vmlinux.h> 4 4 #include <bpf/bpf_helpers.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 - #include "../bpf_testmod/bpf_testmod.h" 6 6 + #include "../test_kmods/bpf_testmod.h" 7 7 8 8 char _license[] SEC("license") = "GPL"; 9 9

+3 -3

tools/testing/selftests/bpf/progs/syscall.c

reviewed

··· 76 76 .magic = BTF_MAGIC, 77 77 .version = BTF_VERSION, 78 78 .hdr_len = sizeof(struct btf_header), 79 79 - .type_len = sizeof(__u32) * 8, 80 80 - .str_off = sizeof(__u32) * 8, 81 81 - .str_len = sizeof(__u32), 79 79 + .type_len = sizeof(raw_btf.types), 80 80 + .str_off = offsetof(struct btf_blob, str) - offsetof(struct btf_blob, types), 81 81 + .str_len = sizeof(raw_btf.str), 82 82 }, 83 83 .types = { 84 84 /* long */

+1 -1

tools/testing/selftests/bpf/progs/test_cls_redirect.c

reviewed

··· 15 15 #include <linux/ipv6.h> 16 16 #include <linux/pkt_cls.h> 17 17 #include <linux/tcp.h> 18 18 - #include <linux/udp.h> 18 18 + #include <netinet/udp.h> 19 19 20 20 #include <bpf/bpf_helpers.h> 21 21 #include <bpf/bpf_endian.h>

+1 -1

tools/testing/selftests/bpf/progs/test_cls_redirect.h

reviewed

··· 10 10 #include <linux/in.h> 11 11 #include <linux/ip.h> 12 12 #include <linux/ipv6.h> 13 13 - #include <linux/udp.h> 13 13 + #include <netinet/udp.h> 14 14 15 15 /* offsetof() is used in static asserts, and the libbpf-redefined CO-RE 16 16 * friendly version breaks compilation for older clang versions <= 15

+1 -1

tools/testing/selftests/bpf/progs/test_cls_redirect_dynptr.c

reviewed

··· 15 15 #include <linux/ipv6.h> 16 16 #include <linux/pkt_cls.h> 17 17 #include <linux/tcp.h> 18 18 - #include <linux/udp.h> 18 18 + #include <netinet/udp.h> 19 19 20 20 #include <bpf/bpf_helpers.h> 21 21 #include <bpf/bpf_endian.h>

+10 -3

tools/testing/selftests/bpf/progs/test_fill_link_info.c

reviewed

··· 6 6 #include <stdbool.h> 7 7 8 8 extern bool CONFIG_X86_KERNEL_IBT __kconfig __weak; 9 9 + extern bool CONFIG_PPC_FTRACE_OUT_OF_LINE __kconfig __weak; 10 10 + extern bool CONFIG_KPROBES_ON_FTRACE __kconfig __weak; 11 11 + extern bool CONFIG_PPC64 __kconfig __weak; 9 12 10 10 - /* This function is here to have CONFIG_X86_KERNEL_IBT 11 11 - * used and added to object BTF. 13 13 + /* This function is here to have CONFIG_X86_KERNEL_IBT, 14 14 + * CONFIG_PPC_FTRACE_OUT_OF_LINE, CONFIG_KPROBES_ON_FTRACE, 15 15 + * CONFIG_PPC6 used and added to object BTF. 12 16 */ 13 17 int unused(void) 14 18 { 15 15 - return CONFIG_X86_KERNEL_IBT ? 0 : 1; 19 19 + return CONFIG_X86_KERNEL_IBT || 20 20 + CONFIG_PPC_FTRACE_OUT_OF_LINE || 21 21 + CONFIG_KPROBES_ON_FTRACE || 22 22 + CONFIG_PPC64 ? 0 : 1; 16 23 } 17 24 18 25 SEC("kprobe")

+1 -1

tools/testing/selftests/bpf/progs/test_global_func10.c

reviewed

··· 26 26 } 27 27 28 28 SEC("cgroup_skb/ingress") 29 29 - __failure __msg("invalid indirect access to stack") 29 29 + __failure __msg("invalid read from stack") 30 30 int global_func10(struct __sk_buff *skb) 31 31 { 32 32 const struct Small small = {.x = skb->len };

+1 -1

tools/testing/selftests/bpf/progs/test_kfunc_param_nullable.c

reviewed

··· 4 4 #include <bpf/bpf_helpers.h> 5 5 #include "bpf_misc.h" 6 6 #include "bpf_kfuncs.h" 7 7 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 7 7 + #include "../test_kmods/bpf_testmod_kfunc.h" 8 8 9 9 SEC("tc") 10 10 int kfunc_dynptr_nullable_test1(struct __sk_buff *skb)

+1 -1

tools/testing/selftests/bpf/progs/test_module_attach.c

reviewed

··· 5 5 #include <bpf/bpf_helpers.h> 6 6 #include <bpf/bpf_tracing.h> 7 7 #include <bpf/bpf_core_read.h> 8 8 - #include "../bpf_testmod/bpf_testmod.h" 8 8 + #include "../test_kmods/bpf_testmod.h" 9 9 10 10 __u32 raw_tp_read_sz = 0; 11 11

+1 -1

tools/testing/selftests/bpf/progs/test_tp_btf_nullable.c

reviewed

··· 3 3 #include "vmlinux.h" 4 4 #include <bpf/bpf_helpers.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 - #include "../bpf_testmod/bpf_testmod.h" 6 6 + #include "../test_kmods/bpf_testmod.h" 7 7 #include "bpf_misc.h" 8 8 9 9 SEC("tp_btf/bpf_testmod_test_nullable_bare")

+2

tools/testing/selftests/bpf/progs/test_xdp_adjust_tail_grow.c

reviewed

··· 10 10 /* SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) */ 11 11 #if defined(__TARGET_ARCH_s390) 12 12 int tailroom = 512; 13 13 + #elif defined(__TARGET_ARCH_powerpc) 14 14 + int tailroom = 384; 13 15 #else 14 16 int tailroom = 320; 15 17 #endif

+12

tools/testing/selftests/bpf/progs/test_xdp_do_redirect.c

reviewed

··· 98 98 return XDP_DROP; 99 99 } 100 100 101 101 + SEC("xdp") 102 102 + int xdp_redirect_to_111(struct xdp_md *xdp) 103 103 + { 104 104 + return bpf_redirect(111, 0); 105 105 + } 106 106 + 107 107 + SEC("xdp") 108 108 + int xdp_redirect_to_222(struct xdp_md *xdp) 109 109 + { 110 110 + return bpf_redirect(222, 0); 111 111 + } 112 112 + 101 113 SEC("tc") 102 114 int tc_count_pkts(struct __sk_buff *skb) 103 115 {

-26

tools/testing/selftests/bpf/progs/test_xdp_redirect.c

reviewed

··· 1 1 - /* Copyright (c) 2017 VMware 2 2 - * 3 3 - * This program is free software; you can redistribute it and/or 4 4 - * modify it under the terms of version 2 of the GNU General Public 5 5 - * License as published by the Free Software Foundation. 6 6 - * 7 7 - * This program is distributed in the hope that it will be useful, but 8 8 - * WITHOUT ANY WARRANTY; without even the implied warranty of 9 9 - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 10 - * General Public License for more details. 11 11 - */ 12 12 - #include <linux/bpf.h> 13 13 - #include <bpf/bpf_helpers.h> 14 14 - 15 15 - SEC("redirect_to_111") 16 16 - int xdp_redirect_to_111(struct xdp_md *xdp) 17 17 - { 18 18 - return bpf_redirect(111, 0); 19 19 - } 20 20 - SEC("redirect_to_222") 21 21 - int xdp_redirect_to_222(struct xdp_md *xdp) 22 22 - { 23 23 - return bpf_redirect(222, 0); 24 24 - } 25 25 - 26 26 - char _license[] SEC("license") = "GPL";

+3 -2

tools/testing/selftests/bpf/progs/uninit_stack.c

reviewed

··· 70 70 r1 = r10; \ 71 71 r1 += -128; \ 72 72 r2 = 32; \ 73 73 - call %[bpf_trace_printk]; \ 73 73 + r3 = 0; \ 74 74 + call %[bpf_probe_read_user]; \ 74 75 /* Call to dummy() forces print_verifier_state(..., true), \ 75 76 * thus showing the stack state, matched by __msg(). \ 76 77 */ \ ··· 80 79 exit; \ 81 80 " 82 81 : 83 83 - : __imm(bpf_trace_printk), 82 82 + : __imm(bpf_probe_read_user), 84 83 __imm(dummy) 85 84 : __clobber_all); 86 85 }

+1 -1

tools/testing/selftests/bpf/progs/unsupported_ops.c

reviewed

··· 4 4 #include <vmlinux.h> 5 5 #include <bpf/bpf_tracing.h> 6 6 #include "bpf_misc.h" 7 7 - #include "../bpf_testmod/bpf_testmod.h" 7 7 + #include "../test_kmods/bpf_testmod.h" 8 8 9 9 char _license[] SEC("license") = "GPL"; 10 10

+188

tools/testing/selftests/bpf/progs/verifier_array_access.c

reviewed

··· 29 29 } map_array_wo SEC(".maps"); 30 30 31 31 struct { 32 32 + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 33 33 + __uint(max_entries, 2); 34 34 + __type(key, __u32); 35 35 + __type(value, struct test_val); 36 36 + } map_array_pcpu SEC(".maps"); 37 37 + 38 38 + struct { 39 39 + __uint(type, BPF_MAP_TYPE_ARRAY); 40 40 + __uint(max_entries, 2); 41 41 + __type(key, __u32); 42 42 + __type(value, struct test_val); 43 43 + } map_array SEC(".maps"); 44 44 + 45 45 + struct { 32 46 __uint(type, BPF_MAP_TYPE_HASH); 33 47 __uint(max_entries, 1); 34 48 __type(key, long long); ··· 537 523 __imm(bpf_map_lookup_elem), 538 524 __imm_addr(map_array_wo) 539 525 : __clobber_all); 526 526 + } 527 527 + 528 528 + SEC("socket") 529 529 + __description("valid map access into an array using constant without nullness") 530 530 + __success __retval(4) __log_level(2) 531 531 + __msg("mark_precise: frame0: regs= stack=-8 before {{[0-9]}}: ({{[a-f0-9]+}}) *(u32 *)(r10 -8) = {{(1|r[0-9])}}") 532 532 + unsigned int an_array_with_a_constant_no_nullness(void) 533 533 + { 534 534 + /* Need 8-byte alignment for spill tracking */ 535 535 + __u32 __attribute__((aligned(8))) key = 1; 536 536 + struct test_val *val; 537 537 + 538 538 + val = bpf_map_lookup_elem(&map_array, &key); 539 539 + val->index = offsetof(struct test_val, foo); 540 540 + 541 541 + return val->index; 542 542 + } 543 543 + 544 544 + SEC("socket") 545 545 + __description("valid multiple map access into an array using constant without nullness") 546 546 + __success __retval(8) __log_level(2) 547 547 + __msg("mark_precise: frame0: regs= stack=-8 before {{[0-9]}}: ({{[a-f0-9]+}}) *(u32 *)(r10 -16) = {{(0|r[0-9])}}") 548 548 + __msg("mark_precise: frame0: regs= stack=-8 before {{[0-9]}}: ({{[a-f0-9]+}}) *(u32 *)(r10 -8) = {{(1|r[0-9])}}") 549 549 + unsigned int multiple_array_with_a_constant_no_nullness(void) 550 550 + { 551 551 + __u32 __attribute__((aligned(8))) key = 1; 552 552 + __u32 __attribute__((aligned(8))) key2 = 0; 553 553 + struct test_val *val, *val2; 554 554 + 555 555 + val = bpf_map_lookup_elem(&map_array, &key); 556 556 + val->index = offsetof(struct test_val, foo); 557 557 + 558 558 + val2 = bpf_map_lookup_elem(&map_array, &key2); 559 559 + val2->index = offsetof(struct test_val, foo); 560 560 + 561 561 + return val->index + val2->index; 562 562 + } 563 563 + 564 564 + SEC("socket") 565 565 + __description("valid map access into an array using natural aligned 32-bit constant 0 without nullness") 566 566 + __success __retval(4) 567 567 + unsigned int an_array_with_a_32bit_constant_0_no_nullness(void) 568 568 + { 569 569 + /* Unlike the above tests, 32-bit zeroing is precisely tracked even 570 570 + * if writes are not aligned to BPF_REG_SIZE. This tests that our 571 571 + * STACK_ZERO handling functions. 572 572 + */ 573 573 + struct test_val *val; 574 574 + __u32 key = 0; 575 575 + 576 576 + val = bpf_map_lookup_elem(&map_array, &key); 577 577 + val->index = offsetof(struct test_val, foo); 578 578 + 579 579 + return val->index; 580 580 + } 581 581 + 582 582 + SEC("socket") 583 583 + __description("valid map access into a pcpu array using constant without nullness") 584 584 + __success __retval(4) __log_level(2) 585 585 + __msg("mark_precise: frame0: regs= stack=-8 before {{[0-9]}}: ({{[a-f0-9]+}}) *(u32 *)(r10 -8) = {{(1|r[0-9])}}") 586 586 + unsigned int a_pcpu_array_with_a_constant_no_nullness(void) 587 587 + { 588 588 + __u32 __attribute__((aligned(8))) key = 1; 589 589 + struct test_val *val; 590 590 + 591 591 + val = bpf_map_lookup_elem(&map_array_pcpu, &key); 592 592 + val->index = offsetof(struct test_val, foo); 593 593 + 594 594 + return val->index; 595 595 + } 596 596 + 597 597 + SEC("socket") 598 598 + __description("invalid map access into an array using constant without nullness") 599 599 + __failure __msg("R0 invalid mem access 'map_value_or_null'") 600 600 + unsigned int an_array_with_a_constant_no_nullness_out_of_bounds(void) 601 601 + { 602 602 + /* Out of bounds */ 603 603 + __u32 __attribute__((aligned(8))) key = 3; 604 604 + struct test_val *val; 605 605 + 606 606 + val = bpf_map_lookup_elem(&map_array, &key); 607 607 + val->index = offsetof(struct test_val, foo); 608 608 + 609 609 + return val->index; 610 610 + } 611 611 + 612 612 + SEC("socket") 613 613 + __description("invalid map access into an array using constant smaller than key_size") 614 614 + __failure __msg("R0 invalid mem access 'map_value_or_null'") 615 615 + unsigned int an_array_with_a_constant_too_small(void) 616 616 + { 617 617 + __u32 __attribute__((aligned(8))) key; 618 618 + struct test_val *val; 619 619 + 620 620 + /* Mark entire key as STACK_MISC */ 621 621 + bpf_probe_read_user(&key, sizeof(key), NULL); 622 622 + 623 623 + /* Spilling only the bottom byte results in a tnum const of 1. 624 624 + * We want to check that the verifier rejects it, as the spill is < 4B. 625 625 + */ 626 626 + *(__u8 *)&key = 1; 627 627 + val = bpf_map_lookup_elem(&map_array, &key); 628 628 + 629 629 + /* Should fail, as verifier cannot prove in-bound lookup */ 630 630 + val->index = offsetof(struct test_val, foo); 631 631 + 632 632 + return val->index; 633 633 + } 634 634 + 635 635 + SEC("socket") 636 636 + __description("invalid map access into an array using constant larger than key_size") 637 637 + __failure __msg("R0 invalid mem access 'map_value_or_null'") 638 638 + unsigned int an_array_with_a_constant_too_big(void) 639 639 + { 640 640 + struct test_val *val; 641 641 + __u64 key = 1; 642 642 + 643 643 + /* Even if the constant value is < max_entries, if the spill size is 644 644 + * larger than the key size, the set bits may not be where we expect them 645 645 + * to be on different endian architectures. 646 646 + */ 647 647 + val = bpf_map_lookup_elem(&map_array, &key); 648 648 + val->index = offsetof(struct test_val, foo); 649 649 + 650 650 + return val->index; 651 651 + } 652 652 + 653 653 + SEC("socket") 654 654 + __description("invalid elided lookup using const and non-const key") 655 655 + __failure __msg("R0 invalid mem access 'map_value_or_null'") 656 656 + unsigned int mixed_const_and_non_const_key_lookup(void) 657 657 + { 658 658 + __u32 __attribute__((aligned(8))) key; 659 659 + struct test_val *val; 660 660 + __u32 rand; 661 661 + 662 662 + rand = bpf_get_prandom_u32(); 663 663 + key = rand > 42 ? 1 : rand; 664 664 + val = bpf_map_lookup_elem(&map_array, &key); 665 665 + 666 666 + return val->index; 667 667 + } 668 668 + 669 669 + SEC("socket") 670 670 + __failure __msg("invalid read from stack R2 off=4096 size=4") 671 671 + __naked void key_lookup_at_invalid_fp(void) 672 672 + { 673 673 + asm volatile (" \ 674 674 + r1 = %[map_array] ll; \ 675 675 + r2 = r10; \ 676 676 + r2 += 4096; \ 677 677 + call %[bpf_map_lookup_elem]; \ 678 678 + r0 = *(u64*)(r0 + 0); \ 679 679 + exit; \ 680 680 + " : 681 681 + : __imm(bpf_map_lookup_elem), 682 682 + __imm_addr(map_array) 683 683 + : __clobber_all); 684 684 + } 685 685 + 686 686 + volatile __u32 __attribute__((aligned(8))) global_key; 687 687 + 688 688 + SEC("socket") 689 689 + __description("invalid elided lookup using non-stack key") 690 690 + __failure __msg("R0 invalid mem access 'map_value_or_null'") 691 691 + unsigned int non_stack_key_lookup(void) 692 692 + { 693 693 + struct test_val *val; 694 694 + 695 695 + global_key = 1; 696 696 + val = bpf_map_lookup_elem(&map_array, (void *)&global_key); 697 697 + val->index = offsetof(struct test_val, foo); 698 698 + 699 699 + return val->index; 540 700 } 541 701 542 702 char _license[] SEC("license") = "GPL";

+1 -1

tools/testing/selftests/bpf/progs/verifier_basic_stack.c

reviewed

··· 28 28 SEC("socket") 29 29 __description("uninitialized stack1") 30 30 __success __log_level(4) __msg("stack depth 8") 31 31 - __failure_unpriv __msg_unpriv("invalid indirect read from stack") 31 31 + __failure_unpriv __msg_unpriv("invalid read from stack") 32 32 __naked void uninitialized_stack1(void) 33 33 { 34 34 asm volatile (" \

+134

tools/testing/selftests/bpf/progs/verifier_bounds.c

reviewed

··· 1200 1200 : __clobber_all); 1201 1201 } 1202 1202 1203 1203 + SEC("tc") 1204 1204 + __description("multiply mixed sign bounds. test 1") 1205 1205 + __success __log_level(2) 1206 1206 + __msg("r6 *= r7 {{.*}}; R6_w=scalar(smin=umin=0x1bc16d5cd4927ee1,smax=umax=0x1bc16d674ec80000,smax32=0x7ffffeff,umax32=0xfffffeff,var_off=(0x1bc16d4000000000; 0x3ffffffeff))") 1207 1207 + __naked void mult_mixed0_sign(void) 1208 1208 + { 1209 1209 + asm volatile ( 1210 1210 + "call %[bpf_get_prandom_u32];" 1211 1211 + "r6 = r0;" 1212 1212 + "call %[bpf_get_prandom_u32];" 1213 1213 + "r7 = r0;" 1214 1214 + "r6 &= 0xf;" 1215 1215 + "r6 -= 1000000000;" 1216 1216 + "r7 &= 0xf;" 1217 1217 + "r7 -= 2000000000;" 1218 1218 + "r6 *= r7;" 1219 1219 + "exit" 1220 1220 + : 1221 1221 + : __imm(bpf_get_prandom_u32), 1222 1222 + __imm(bpf_skb_store_bytes) 1223 1223 + : __clobber_all); 1224 1224 + } 1225 1225 + 1226 1226 + SEC("tc") 1227 1227 + __description("multiply mixed sign bounds. test 2") 1228 1228 + __success __log_level(2) 1229 1229 + __msg("r6 *= r7 {{.*}}; R6_w=scalar(smin=smin32=-100,smax=smax32=200)") 1230 1230 + __naked void mult_mixed1_sign(void) 1231 1231 + { 1232 1232 + asm volatile ( 1233 1233 + "call %[bpf_get_prandom_u32];" 1234 1234 + "r6 = r0;" 1235 1235 + "call %[bpf_get_prandom_u32];" 1236 1236 + "r7 = r0;" 1237 1237 + "r6 &= 0xf;" 1238 1238 + "r6 -= 0xa;" 1239 1239 + "r7 &= 0xf;" 1240 1240 + "r7 -= 0x14;" 1241 1241 + "r6 *= r7;" 1242 1242 + "exit" 1243 1243 + : 1244 1244 + : __imm(bpf_get_prandom_u32), 1245 1245 + __imm(bpf_skb_store_bytes) 1246 1246 + : __clobber_all); 1247 1247 + } 1248 1248 + 1249 1249 + SEC("tc") 1250 1250 + __description("multiply negative bounds") 1251 1251 + __success __log_level(2) 1252 1252 + __msg("r6 *= r7 {{.*}}; R6_w=scalar(smin=umin=smin32=umin32=0x3ff280b0,smax=umax=smax32=umax32=0x3fff0001,var_off=(0x3ff00000; 0xf81ff))") 1253 1253 + __naked void mult_sign_bounds(void) 1254 1254 + { 1255 1255 + asm volatile ( 1256 1256 + "r8 = 0x7fff;" 1257 1257 + "call %[bpf_get_prandom_u32];" 1258 1258 + "r6 = r0;" 1259 1259 + "call %[bpf_get_prandom_u32];" 1260 1260 + "r7 = r0;" 1261 1261 + "r6 &= 0xa;" 1262 1262 + "r6 -= r8;" 1263 1263 + "r7 &= 0xf;" 1264 1264 + "r7 -= r8;" 1265 1265 + "r6 *= r7;" 1266 1266 + "exit" 1267 1267 + : 1268 1268 + : __imm(bpf_get_prandom_u32), 1269 1269 + __imm(bpf_skb_store_bytes) 1270 1270 + : __clobber_all); 1271 1271 + } 1272 1272 + 1273 1273 + SEC("tc") 1274 1274 + __description("multiply bounds that don't cross signed boundary") 1275 1275 + __success __log_level(2) 1276 1276 + __msg("r8 *= r6 {{.*}}; R6_w=scalar(smin=smin32=0,smax=umax=smax32=umax32=11,var_off=(0x0; 0xb)) R8_w=scalar(smin=0,smax=umax=0x7b96bb0a94a3a7cd,var_off=(0x0; 0x7fffffffffffffff))") 1277 1277 + __naked void mult_no_sign_crossing(void) 1278 1278 + { 1279 1279 + asm volatile ( 1280 1280 + "r6 = 0xb;" 1281 1281 + "r8 = 0xb3c3f8c99262687 ll;" 1282 1282 + "call %[bpf_get_prandom_u32];" 1283 1283 + "r7 = r0;" 1284 1284 + "r6 &= r7;" 1285 1285 + "r8 *= r6;" 1286 1286 + "exit" 1287 1287 + : 1288 1288 + : __imm(bpf_get_prandom_u32), 1289 1289 + __imm(bpf_skb_store_bytes) 1290 1290 + : __clobber_all); 1291 1291 + } 1292 1292 + 1293 1293 + SEC("tc") 1294 1294 + __description("multiplication overflow, result in unbounded reg. test 1") 1295 1295 + __success __log_level(2) 1296 1296 + __msg("r6 *= r7 {{.*}}; R6_w=scalar()") 1297 1297 + __naked void mult_unsign_ovf(void) 1298 1298 + { 1299 1299 + asm volatile ( 1300 1300 + "r8 = 0x7ffffffffff ll;" 1301 1301 + "call %[bpf_get_prandom_u32];" 1302 1302 + "r6 = r0;" 1303 1303 + "call %[bpf_get_prandom_u32];" 1304 1304 + "r7 = r0;" 1305 1305 + "r6 &= 0x7fffffff;" 1306 1306 + "r7 &= r8;" 1307 1307 + "r6 *= r7;" 1308 1308 + "exit" 1309 1309 + : 1310 1310 + : __imm(bpf_get_prandom_u32), 1311 1311 + __imm(bpf_skb_store_bytes) 1312 1312 + : __clobber_all); 1313 1313 + } 1314 1314 + 1315 1315 + SEC("tc") 1316 1316 + __description("multiplication overflow, result in unbounded reg. test 2") 1317 1317 + __success __log_level(2) 1318 1318 + __msg("r6 *= r7 {{.*}}; R6_w=scalar()") 1319 1319 + __naked void mult_sign_ovf(void) 1320 1320 + { 1321 1321 + asm volatile ( 1322 1322 + "r8 = 0x7ffffffff ll;" 1323 1323 + "call %[bpf_get_prandom_u32];" 1324 1324 + "r6 = r0;" 1325 1325 + "call %[bpf_get_prandom_u32];" 1326 1326 + "r7 = r0;" 1327 1327 + "r6 &= 0xa;" 1328 1328 + "r6 -= r8;" 1329 1329 + "r7 &= 0x7fffffff;" 1330 1330 + "r6 *= r7;" 1331 1331 + "exit" 1332 1332 + : 1333 1333 + : __imm(bpf_get_prandom_u32), 1334 1334 + __imm(bpf_skb_store_bytes) 1335 1335 + : __clobber_all); 1336 1336 + } 1203 1337 char _license[] SEC("license") = "GPL";

+2 -2

tools/testing/selftests/bpf/progs/verifier_const_or.c

reviewed

··· 25 25 26 26 SEC("tracepoint") 27 27 __description("constant register |= constant should not bypass stack boundary checks") 28 28 - __failure __msg("invalid indirect access to stack R1 off=-48 size=58") 28 28 + __failure __msg("invalid write to stack R1 off=-48 size=58") 29 29 __naked void not_bypass_stack_boundary_checks_1(void) 30 30 { 31 31 asm volatile (" \ ··· 62 62 63 63 SEC("tracepoint") 64 64 __description("constant register |= constant register should not bypass stack boundary checks") 65 65 - __failure __msg("invalid indirect access to stack R1 off=-48 size=58") 65 65 + __failure __msg("invalid write to stack R1 off=-48 size=58") 66 66 __naked void not_bypass_stack_boundary_checks_2(void) 67 67 { 68 68 asm volatile (" \

+6 -6

tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c

reviewed

··· 67 67 __description("helper access to variable memory: stack, bitwise AND, zero included") 68 68 /* in privileged mode reads from uninitialized stack locations are permitted */ 69 69 __success __failure_unpriv 70 70 - __msg_unpriv("invalid indirect read from stack R2 off -64+0 size 64") 70 70 + __msg_unpriv("invalid read from stack R2 off -64+0 size 64") 71 71 __retval(0) 72 72 __naked void stack_bitwise_and_zero_included(void) 73 73 { ··· 100 100 101 101 SEC("tracepoint") 102 102 __description("helper access to variable memory: stack, bitwise AND + JMP, wrong max") 103 103 - __failure __msg("invalid indirect access to stack R1 off=-64 size=65") 103 103 + __failure __msg("invalid write to stack R1 off=-64 size=65") 104 104 __naked void bitwise_and_jmp_wrong_max(void) 105 105 { 106 106 asm volatile (" \ ··· 187 187 188 188 SEC("tracepoint") 189 189 __description("helper access to variable memory: stack, JMP, bounds + offset") 190 190 - __failure __msg("invalid indirect access to stack R1 off=-64 size=65") 190 190 + __failure __msg("invalid write to stack R1 off=-64 size=65") 191 191 __naked void memory_stack_jmp_bounds_offset(void) 192 192 { 193 193 asm volatile (" \ ··· 211 211 212 212 SEC("tracepoint") 213 213 __description("helper access to variable memory: stack, JMP, wrong max") 214 214 - __failure __msg("invalid indirect access to stack R1 off=-64 size=65") 214 214 + __failure __msg("invalid write to stack R1 off=-64 size=65") 215 215 __naked void memory_stack_jmp_wrong_max(void) 216 216 { 217 217 asm volatile (" \ ··· 260 260 __description("helper access to variable memory: stack, JMP, no min check") 261 261 /* in privileged mode reads from uninitialized stack locations are permitted */ 262 262 __success __failure_unpriv 263 263 - __msg_unpriv("invalid indirect read from stack R2 off -64+0 size 64") 263 263 + __msg_unpriv("invalid read from stack R2 off -64+0 size 64") 264 264 __retval(0) 265 265 __naked void stack_jmp_no_min_check(void) 266 266 { ··· 750 750 __description("helper access to variable memory: 8 bytes leak") 751 751 /* in privileged mode reads from uninitialized stack locations are permitted */ 752 752 __success __failure_unpriv 753 753 - __msg_unpriv("invalid indirect read from stack R2 off -64+32 size 64") 753 753 + __msg_unpriv("invalid read from stack R2 off -64+32 size 64") 754 754 __retval(0) 755 755 __naked void variable_memory_8_bytes_leak(void) 756 756 {

+1 -1

tools/testing/selftests/bpf/progs/verifier_int_ptr.c

reviewed

··· 96 96 97 97 SEC("cgroup/sysctl") 98 98 __description("arg pointer to long size < sizeof(long)") 99 99 - __failure __msg("invalid indirect access to stack R4 off=-4 size=8") 99 99 + __failure __msg("invalid write to stack R4 off=-4 size=8") 100 100 __naked void to_long_size_sizeof_long(void) 101 101 { 102 102 asm volatile (" \

+1 -1

tools/testing/selftests/bpf/progs/verifier_map_in_map.c

reviewed

··· 47 47 48 48 SEC("xdp") 49 49 __description("map in map state pruning") 50 50 - __success __msg("processed 26 insns") 50 50 + __success __msg("processed 15 insns") 51 51 __log_level(2) __retval(0) __flag(BPF_F_TEST_STATE_FREQ) 52 52 __naked void map_in_map_state_pruning(void) 53 53 {

+97

tools/testing/selftests/bpf/progs/verifier_may_goto_1.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ 3 3 + 4 4 + #include <linux/bpf.h> 5 5 + #include <bpf/bpf_helpers.h> 6 6 + #include "../../../include/linux/filter.h" 7 7 + #include "bpf_misc.h" 8 8 + 9 9 + SEC("raw_tp") 10 10 + __description("may_goto 0") 11 11 + __arch_x86_64 12 12 + __xlated("0: r0 = 1") 13 13 + __xlated("1: exit") 14 14 + __success 15 15 + __naked void may_goto_simple(void) 16 16 + { 17 17 + asm volatile ( 18 18 + ".8byte %[may_goto];" 19 19 + "r0 = 1;" 20 20 + ".8byte %[may_goto];" 21 21 + "exit;" 22 22 + : 23 23 + : __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0 /* offset */, 0)) 24 24 + : __clobber_all); 25 25 + } 26 26 + 27 27 + SEC("raw_tp") 28 28 + __description("batch 2 of may_goto 0") 29 29 + __arch_x86_64 30 30 + __xlated("0: r0 = 1") 31 31 + __xlated("1: exit") 32 32 + __success 33 33 + __naked void may_goto_batch_0(void) 34 34 + { 35 35 + asm volatile ( 36 36 + ".8byte %[may_goto1];" 37 37 + ".8byte %[may_goto1];" 38 38 + "r0 = 1;" 39 39 + ".8byte %[may_goto1];" 40 40 + ".8byte %[may_goto1];" 41 41 + "exit;" 42 42 + : 43 43 + : __imm_insn(may_goto1, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0 /* offset */, 0)) 44 44 + : __clobber_all); 45 45 + } 46 46 + 47 47 + SEC("raw_tp") 48 48 + __description("may_goto batch with offsets 2/1/0") 49 49 + __arch_x86_64 50 50 + __xlated("0: r0 = 1") 51 51 + __xlated("1: exit") 52 52 + __success 53 53 + __naked void may_goto_batch_1(void) 54 54 + { 55 55 + asm volatile ( 56 56 + ".8byte %[may_goto1];" 57 57 + ".8byte %[may_goto2];" 58 58 + ".8byte %[may_goto3];" 59 59 + "r0 = 1;" 60 60 + ".8byte %[may_goto1];" 61 61 + ".8byte %[may_goto2];" 62 62 + ".8byte %[may_goto3];" 63 63 + "exit;" 64 64 + : 65 65 + : __imm_insn(may_goto1, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2 /* offset */, 0)), 66 66 + __imm_insn(may_goto2, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 1 /* offset */, 0)), 67 67 + __imm_insn(may_goto3, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0 /* offset */, 0)) 68 68 + : __clobber_all); 69 69 + } 70 70 + 71 71 + SEC("raw_tp") 72 72 + __description("may_goto batch with offsets 2/0") 73 73 + __arch_x86_64 74 74 + __xlated("0: *(u64 *)(r10 -8) = 8388608") 75 75 + __xlated("1: r11 = *(u64 *)(r10 -8)") 76 76 + __xlated("2: if r11 == 0x0 goto pc+3") 77 77 + __xlated("3: r11 -= 1") 78 78 + __xlated("4: *(u64 *)(r10 -8) = r11") 79 79 + __xlated("5: r0 = 1") 80 80 + __xlated("6: r0 = 2") 81 81 + __xlated("7: exit") 82 82 + __success 83 83 + __naked void may_goto_batch_2(void) 84 84 + { 85 85 + asm volatile ( 86 86 + ".8byte %[may_goto1];" 87 87 + ".8byte %[may_goto3];" 88 88 + "r0 = 1;" 89 89 + "r0 = 2;" 90 90 + "exit;" 91 91 + : 92 92 + : __imm_insn(may_goto1, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2 /* offset */, 0)), 93 93 + __imm_insn(may_goto3, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0 /* offset */, 0)) 94 94 + : __clobber_all); 95 95 + } 96 96 + 97 97 + char _license[] SEC("license") = "GPL";

+28

tools/testing/selftests/bpf/progs/verifier_may_goto_2.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ 3 3 + 4 4 + #include "bpf_misc.h" 5 5 + #include "bpf_experimental.h" 6 6 + 7 7 + int gvar; 8 8 + 9 9 + SEC("raw_tp") 10 10 + __description("C code with may_goto 0") 11 11 + __success 12 12 + int may_goto_c_code(void) 13 13 + { 14 14 + int i, tmp[3]; 15 15 + 16 16 + for (i = 0; i < 3 && can_loop; i++) 17 17 + tmp[i] = 0; 18 18 + 19 19 + for (i = 0; i < 3 && can_loop; i++) 20 20 + tmp[i] = gvar - i; 21 21 + 22 22 + for (i = 0; i < 3 && can_loop; i++) 23 23 + gvar += tmp[i]; 24 24 + 25 25 + return 0; 26 26 + } 27 27 + 28 28 + char _license[] SEC("license") = "GPL";

+1 -1

tools/testing/selftests/bpf/progs/verifier_mtu.c

reviewed

··· 8 8 __description("uninit/mtu: write rejected") 9 9 __success 10 10 __caps_unpriv(CAP_BPF|CAP_NET_ADMIN) 11 11 - __failure_unpriv __msg_unpriv("invalid indirect read from stack") 11 11 + __failure_unpriv __msg_unpriv("invalid read from stack") 12 12 int tc_uninit_mtu(struct __sk_buff *ctx) 13 13 { 14 14 __u32 mtu;

+2 -2

tools/testing/selftests/bpf/progs/verifier_raw_stack.c

reviewed

··· 236 236 237 237 SEC("tc") 238 238 __description("raw_stack: skb_load_bytes, invalid access 1") 239 239 - __failure __msg("invalid indirect access to stack R3 off=-513 size=8") 239 239 + __failure __msg("invalid write to stack R3 off=-513 size=8") 240 240 __naked void load_bytes_invalid_access_1(void) 241 241 { 242 242 asm volatile (" \ ··· 255 255 256 256 SEC("tc") 257 257 __description("raw_stack: skb_load_bytes, invalid access 2") 258 258 - __failure __msg("invalid indirect access to stack R3 off=-1 size=8") 258 258 + __failure __msg("invalid write to stack R3 off=-1 size=8") 259 259 __naked void load_bytes_invalid_access_2(void) 260 260 { 261 261 asm volatile (" \

+27 -1

tools/testing/selftests/bpf/progs/verifier_spin_lock.c

reviewed

··· 187 187 188 188 SEC("cgroup/skb") 189 189 __description("spin_lock: test6 missing unlock") 190 190 - __failure __msg("BPF_EXIT instruction cannot be used inside bpf_spin_lock-ed region") 190 190 + __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_spin_lock-ed region") 191 191 __failure_unpriv __msg_unpriv("") 192 192 __naked void spin_lock_test6_missing_unlock(void) 193 193 { ··· 528 528 __imm(bpf_spin_unlock), 529 529 __imm_addr(map_spin_lock) 530 530 : __clobber_all); 531 531 + } 532 532 + 533 533 + SEC("tc") 534 534 + __description("spin_lock: loop within a locked region") 535 535 + __success __failure_unpriv __msg_unpriv("") 536 536 + __retval(0) 537 537 + int bpf_loop_inside_locked_region(void) 538 538 + { 539 539 + const int zero = 0; 540 540 + struct val *val; 541 541 + int i, j = 0; 542 542 + 543 543 + val = bpf_map_lookup_elem(&map_spin_lock, &zero); 544 544 + if (!val) 545 545 + return -1; 546 546 + 547 547 + bpf_spin_lock(&val->l); 548 548 + bpf_for(i, 0, 10) { 549 549 + j++; 550 550 + /* Silence "unused variable" warnings. */ 551 551 + if (j == 10) 552 552 + break; 553 553 + } 554 554 + bpf_spin_unlock(&val->l); 555 555 + 556 556 + return 0; 531 557 } 532 558 533 559 char _license[] SEC("license") = "GPL";

+1 -1

tools/testing/selftests/bpf/progs/verifier_unpriv.c

reviewed

··· 199 199 SEC("socket") 200 200 __description("unpriv: indirectly pass pointer on stack to helper function") 201 201 __success __failure_unpriv 202 202 - __msg_unpriv("invalid indirect read from stack R2 off -8+0 size 8") 202 202 + __msg_unpriv("invalid read from stack R2 off -8+0 size 8") 203 203 __retval(0) 204 204 __naked void on_stack_to_helper_function(void) 205 205 {

+4 -4

tools/testing/selftests/bpf/progs/verifier_var_off.c

reviewed

··· 203 203 204 204 SEC("sockops") 205 205 __description("indirect variable-offset stack access, unbounded") 206 206 - __failure __msg("invalid unbounded variable-offset indirect access to stack R4") 206 206 + __failure __msg("invalid unbounded variable-offset write to stack R4") 207 207 __naked void variable_offset_stack_access_unbounded(void) 208 208 { 209 209 asm volatile (" \ ··· 236 236 237 237 SEC("lwt_in") 238 238 __description("indirect variable-offset stack access, max out of bound") 239 239 - __failure __msg("invalid variable-offset indirect access to stack R2") 239 239 + __failure __msg("invalid variable-offset read from stack R2") 240 240 __naked void access_max_out_of_bound(void) 241 241 { 242 242 asm volatile (" \ ··· 269 269 */ 270 270 SEC("socket") 271 271 __description("indirect variable-offset stack access, zero-sized, max out of bound") 272 272 - __failure __msg("invalid variable-offset indirect access to stack R1") 272 272 + __failure __msg("invalid variable-offset write to stack R1") 273 273 __naked void zero_sized_access_max_out_of_bound(void) 274 274 { 275 275 asm volatile (" \ ··· 294 294 295 295 SEC("lwt_in") 296 296 __description("indirect variable-offset stack access, min out of bound") 297 297 - __failure __msg("invalid variable-offset indirect access to stack R2") 297 297 + __failure __msg("invalid variable-offset read from stack R2") 298 298 __naked void access_min_out_of_bound(void) 299 299 { 300 300 asm volatile (" \

+1 -1

tools/testing/selftests/bpf/progs/wq.c

reviewed

··· 5 5 #include "bpf_experimental.h" 6 6 #include <bpf/bpf_helpers.h> 7 7 #include "bpf_misc.h" 8 8 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 8 8 + #include "../test_kmods/bpf_testmod_kfunc.h" 9 9 10 10 char _license[] SEC("license") = "GPL"; 11 11

+1 -1

tools/testing/selftests/bpf/progs/wq_failures.c

reviewed

··· 5 5 #include "bpf_experimental.h" 6 6 #include <bpf/bpf_helpers.h> 7 7 #include "bpf_misc.h" 8 8 - #include "../bpf_testmod/bpf_testmod_kfunc.h" 8 8 + #include "../test_kmods/bpf_testmod_kfunc.h" 9 9 10 10 char _license[] SEC("license") = "GPL"; 11 11

+14 -14

tools/testing/selftests/bpf/test_bpftool_synctypes.py

reviewed

··· 66 66 67 67 def __init__(self, reader, array_name): 68 68 self.array_name = array_name 69 69 - self.start_marker = re.compile(f'(static )?const bool {self.array_name}\[.*\] = {{\n') 69 69 + self.start_marker = re.compile(fr'(static )?const bool {self.array_name}\[.*\] = {{\n') 70 70 super().__init__(reader) 71 71 72 72 def search_block(self): ··· 80 80 Parse a block and return data as a dictionary. Items to extract must be 81 81 on separate lines in the file. 82 82 """ 83 83 - pattern = re.compile('\[(BPF_\w*)\]\s*= (true|false),?$') 83 83 + pattern = re.compile(r'\[(BPF_\w*)\]\s*= (true|false),?$') 84 84 entries = set() 85 85 while True: 86 86 line = self.reader.readline() ··· 178 178 @enum_name: name of the enum to parse 179 179 """ 180 180 start_marker = re.compile(f'enum {enum_name} {{\n') 181 181 - pattern = re.compile('^\s*(BPF_\w+),?(\s+/\*.*\*/)?$') 181 181 + pattern = re.compile(r'^\s*(BPF_\w+),?(\s+/\*.*\*/)?$') 182 182 end_marker = re.compile('^};') 183 183 parser = BlockParser(self.reader) 184 184 parser.search_block(start_marker) ··· 226 226 227 227 @block_name: name of the blog to parse, 'TYPE' in the example 228 228 """ 229 229 - start_marker = re.compile(f'\*{block_name}\* := {{') 230 230 - pattern = re.compile('\*\*([\w/-]+)\*\*') 229 229 + start_marker = re.compile(fr'\*{block_name}\* := {{') 230 230 + pattern = re.compile(r'\*\*([\w/-]+)\*\*') 231 231 end_marker = re.compile('}\n') 232 232 return self.__get_description_list(start_marker, pattern, end_marker) 233 233 ··· 245 245 246 246 @block_name: name of the blog to parse, 'TYPE' in the example 247 247 """ 248 248 - start_marker = re.compile(f'"\s*{block_name} := {{') 249 249 - pattern = re.compile('([\w/]+) [|}]') 248 248 + start_marker = re.compile(fr'"\s*{block_name} := {{') 249 249 + pattern = re.compile(r'([\w/]+) [|}]') 250 250 end_marker = re.compile('}') 251 251 return self.__get_description_list(start_marker, pattern, end_marker) 252 252 ··· 264 264 265 265 @macro: macro starting the block, 'HELP_SPEC_OPTIONS' in the example 266 266 """ 267 267 - start_marker = re.compile(f'"\s*{macro}\s*" [|}}]') 268 268 - pattern = re.compile('([\w-]+) ?(?:\||}[ }\]])') 267 267 + start_marker = re.compile(fr'"\s*{macro}\s*" [|}}]') 268 268 + pattern = re.compile(r'([\w-]+) ?(?:\||}[ }\]])') 269 269 end_marker = re.compile('}\\\\n') 270 270 return self.__get_description_list(start_marker, pattern, end_marker) 271 271 ··· 283 283 284 284 @block_name: name of the blog to parse, 'TYPE' in the example 285 285 """ 286 286 - start_marker = re.compile(f'local {block_name}=\'') 287 287 - pattern = re.compile('(?:.*=\')?([\w/]+)') 286 286 + start_marker = re.compile(fr'local {block_name}=\'') 287 287 + pattern = re.compile(r'(?:.*=\')?([\w/]+)') 288 288 end_marker = re.compile('\'$') 289 289 return self.__get_description_list(start_marker, pattern, end_marker) 290 290 ··· 316 316 {'-p', '-d', '--pretty', '--debug', '--json', '-j'} 317 317 """ 318 318 start_marker = re.compile(f'"OPTIONS :=') 319 319 - pattern = re.compile('([\w-]+) ?(?:\||}[ }\]"])') 319 319 + pattern = re.compile(r'([\w-]+) ?(?:\||}[ }\]"])') 320 320 end_marker = re.compile('#define') 321 321 322 322 parser = InlineListParser(self.reader) ··· 338 338 339 339 {'-p', '-d', '--pretty', '--debug', '--json', '-j'} 340 340 """ 341 341 - start_marker = re.compile('\|COMMON_OPTIONS\| replace:: {') 342 342 - pattern = re.compile('\*\*([\w/-]+)\*\*') 341 341 + start_marker = re.compile(r'\|COMMON_OPTIONS\| replace:: {') 342 342 + pattern = re.compile(r'\*\*([\w/-]+)\*\*') 343 343 end_marker = re.compile('}$') 344 344 345 345 parser = InlineListParser(self.reader)

-780

tools/testing/selftests/bpf/test_flow_dissector.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0 2 2 - /* 3 3 - * Inject packets with all sorts of encapsulation into the kernel. 4 4 - * 5 5 - * IPv4/IPv6 outer layer 3 6 6 - * GRE/GUE/BARE outer layer 4, where bare is IPIP/SIT/IPv4-in-IPv6/.. 7 7 - * IPv4/IPv6 inner layer 3 8 8 - */ 9 9 - 10 10 - #define _GNU_SOURCE 11 11 - 12 12 - #include <stddef.h> 13 13 - #include <arpa/inet.h> 14 14 - #include <asm/byteorder.h> 15 15 - #include <error.h> 16 16 - #include <errno.h> 17 17 - #include <linux/if_packet.h> 18 18 - #include <linux/if_ether.h> 19 19 - #include <linux/ipv6.h> 20 20 - #include <netinet/ip.h> 21 21 - #include <netinet/in.h> 22 22 - #include <netinet/udp.h> 23 23 - #include <poll.h> 24 24 - #include <stdbool.h> 25 25 - #include <stdlib.h> 26 26 - #include <stdio.h> 27 27 - #include <string.h> 28 28 - #include <sys/ioctl.h> 29 29 - #include <sys/socket.h> 30 30 - #include <sys/stat.h> 31 31 - #include <sys/time.h> 32 32 - #include <sys/types.h> 33 33 - #include <unistd.h> 34 34 - 35 35 - #define CFG_PORT_INNER 8000 36 36 - 37 37 - /* Add some protocol definitions that do not exist in userspace */ 38 38 - 39 39 - struct grehdr { 40 40 - uint16_t unused; 41 41 - uint16_t protocol; 42 42 - } __attribute__((packed)); 43 43 - 44 44 - struct guehdr { 45 45 - union { 46 46 - struct { 47 47 - #if defined(__LITTLE_ENDIAN_BITFIELD) 48 48 - __u8 hlen:5, 49 49 - control:1, 50 50 - version:2; 51 51 - #elif defined (__BIG_ENDIAN_BITFIELD) 52 52 - __u8 version:2, 53 53 - control:1, 54 54 - hlen:5; 55 55 - #else 56 56 - #error "Please fix <asm/byteorder.h>" 57 57 - #endif 58 58 - __u8 proto_ctype; 59 59 - __be16 flags; 60 60 - }; 61 61 - __be32 word; 62 62 - }; 63 63 - }; 64 64 - 65 65 - static uint8_t cfg_dsfield_inner; 66 66 - static uint8_t cfg_dsfield_outer; 67 67 - static uint8_t cfg_encap_proto; 68 68 - static bool cfg_expect_failure = false; 69 69 - static int cfg_l3_extra = AF_UNSPEC; /* optional SIT prefix */ 70 70 - static int cfg_l3_inner = AF_UNSPEC; 71 71 - static int cfg_l3_outer = AF_UNSPEC; 72 72 - static int cfg_num_pkt = 10; 73 73 - static int cfg_num_secs = 0; 74 74 - static char cfg_payload_char = 'a'; 75 75 - static int cfg_payload_len = 100; 76 76 - static int cfg_port_gue = 6080; 77 77 - static bool cfg_only_rx; 78 78 - static bool cfg_only_tx; 79 79 - static int cfg_src_port = 9; 80 80 - 81 81 - static char buf[ETH_DATA_LEN]; 82 82 - 83 83 - #define INIT_ADDR4(name, addr4, port) \ 84 84 - static struct sockaddr_in name = { \ 85 85 - .sin_family = AF_INET, \ 86 86 - .sin_port = __constant_htons(port), \ 87 87 - .sin_addr.s_addr = __constant_htonl(addr4), \ 88 88 - }; 89 89 - 90 90 - #define INIT_ADDR6(name, addr6, port) \ 91 91 - static struct sockaddr_in6 name = { \ 92 92 - .sin6_family = AF_INET6, \ 93 93 - .sin6_port = __constant_htons(port), \ 94 94 - .sin6_addr = addr6, \ 95 95 - }; 96 96 - 97 97 - INIT_ADDR4(in_daddr4, INADDR_LOOPBACK, CFG_PORT_INNER) 98 98 - INIT_ADDR4(in_saddr4, INADDR_LOOPBACK + 2, 0) 99 99 - INIT_ADDR4(out_daddr4, INADDR_LOOPBACK, 0) 100 100 - INIT_ADDR4(out_saddr4, INADDR_LOOPBACK + 1, 0) 101 101 - INIT_ADDR4(extra_daddr4, INADDR_LOOPBACK, 0) 102 102 - INIT_ADDR4(extra_saddr4, INADDR_LOOPBACK + 1, 0) 103 103 - 104 104 - INIT_ADDR6(in_daddr6, IN6ADDR_LOOPBACK_INIT, CFG_PORT_INNER) 105 105 - INIT_ADDR6(in_saddr6, IN6ADDR_LOOPBACK_INIT, 0) 106 106 - INIT_ADDR6(out_daddr6, IN6ADDR_LOOPBACK_INIT, 0) 107 107 - INIT_ADDR6(out_saddr6, IN6ADDR_LOOPBACK_INIT, 0) 108 108 - INIT_ADDR6(extra_daddr6, IN6ADDR_LOOPBACK_INIT, 0) 109 109 - INIT_ADDR6(extra_saddr6, IN6ADDR_LOOPBACK_INIT, 0) 110 110 - 111 111 - static unsigned long util_gettime(void) 112 112 - { 113 113 - struct timeval tv; 114 114 - 115 115 - gettimeofday(&tv, NULL); 116 116 - return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); 117 117 - } 118 118 - 119 119 - static void util_printaddr(const char *msg, struct sockaddr *addr) 120 120 - { 121 121 - unsigned long off = 0; 122 122 - char nbuf[INET6_ADDRSTRLEN]; 123 123 - 124 124 - switch (addr->sa_family) { 125 125 - case PF_INET: 126 126 - off = __builtin_offsetof(struct sockaddr_in, sin_addr); 127 127 - break; 128 128 - case PF_INET6: 129 129 - off = __builtin_offsetof(struct sockaddr_in6, sin6_addr); 130 130 - break; 131 131 - default: 132 132 - error(1, 0, "printaddr: unsupported family %u\n", 133 133 - addr->sa_family); 134 134 - } 135 135 - 136 136 - if (!inet_ntop(addr->sa_family, ((void *) addr) + off, nbuf, 137 137 - sizeof(nbuf))) 138 138 - error(1, errno, "inet_ntop"); 139 139 - 140 140 - fprintf(stderr, "%s: %s\n", msg, nbuf); 141 141 - } 142 142 - 143 143 - static unsigned long add_csum_hword(const uint16_t *start, int num_u16) 144 144 - { 145 145 - unsigned long sum = 0; 146 146 - int i; 147 147 - 148 148 - for (i = 0; i < num_u16; i++) 149 149 - sum += start[i]; 150 150 - 151 151 - return sum; 152 152 - } 153 153 - 154 154 - static uint16_t build_ip_csum(const uint16_t *start, int num_u16, 155 155 - unsigned long sum) 156 156 - { 157 157 - sum += add_csum_hword(start, num_u16); 158 158 - 159 159 - while (sum >> 16) 160 160 - sum = (sum & 0xffff) + (sum >> 16); 161 161 - 162 162 - return ~sum; 163 163 - } 164 164 - 165 165 - static void build_ipv4_header(void *header, uint8_t proto, 166 166 - uint32_t src, uint32_t dst, 167 167 - int payload_len, uint8_t tos) 168 168 - { 169 169 - struct iphdr *iph = header; 170 170 - 171 171 - iph->ihl = 5; 172 172 - iph->version = 4; 173 173 - iph->tos = tos; 174 174 - iph->ttl = 8; 175 175 - iph->tot_len = htons(sizeof(*iph) + payload_len); 176 176 - iph->id = htons(1337); 177 177 - iph->protocol = proto; 178 178 - iph->saddr = src; 179 179 - iph->daddr = dst; 180 180 - iph->check = build_ip_csum((void *) iph, iph->ihl << 1, 0); 181 181 - } 182 182 - 183 183 - static void ipv6_set_dsfield(struct ipv6hdr *ip6h, uint8_t dsfield) 184 184 - { 185 185 - uint16_t val, *ptr = (uint16_t *)ip6h; 186 186 - 187 187 - val = ntohs(*ptr); 188 188 - val &= 0xF00F; 189 189 - val |= ((uint16_t) dsfield) << 4; 190 190 - *ptr = htons(val); 191 191 - } 192 192 - 193 193 - static void build_ipv6_header(void *header, uint8_t proto, 194 194 - struct sockaddr_in6 *src, 195 195 - struct sockaddr_in6 *dst, 196 196 - int payload_len, uint8_t dsfield) 197 197 - { 198 198 - struct ipv6hdr *ip6h = header; 199 199 - 200 200 - ip6h->version = 6; 201 201 - ip6h->payload_len = htons(payload_len); 202 202 - ip6h->nexthdr = proto; 203 203 - ip6h->hop_limit = 8; 204 204 - ipv6_set_dsfield(ip6h, dsfield); 205 205 - 206 206 - memcpy(&ip6h->saddr, &src->sin6_addr, sizeof(ip6h->saddr)); 207 207 - memcpy(&ip6h->daddr, &dst->sin6_addr, sizeof(ip6h->daddr)); 208 208 - } 209 209 - 210 210 - static uint16_t build_udp_v4_csum(const struct iphdr *iph, 211 211 - const struct udphdr *udph, 212 212 - int num_words) 213 213 - { 214 214 - unsigned long pseudo_sum; 215 215 - int num_u16 = sizeof(iph->saddr); /* halfwords: twice byte len */ 216 216 - 217 217 - pseudo_sum = add_csum_hword((void *) &iph->saddr, num_u16); 218 218 - pseudo_sum += htons(IPPROTO_UDP); 219 219 - pseudo_sum += udph->len; 220 220 - return build_ip_csum((void *) udph, num_words, pseudo_sum); 221 221 - } 222 222 - 223 223 - static uint16_t build_udp_v6_csum(const struct ipv6hdr *ip6h, 224 224 - const struct udphdr *udph, 225 225 - int num_words) 226 226 - { 227 227 - unsigned long pseudo_sum; 228 228 - int num_u16 = sizeof(ip6h->saddr); /* halfwords: twice byte len */ 229 229 - 230 230 - pseudo_sum = add_csum_hword((void *) &ip6h->saddr, num_u16); 231 231 - pseudo_sum += htons(ip6h->nexthdr); 232 232 - pseudo_sum += ip6h->payload_len; 233 233 - return build_ip_csum((void *) udph, num_words, pseudo_sum); 234 234 - } 235 235 - 236 236 - static void build_udp_header(void *header, int payload_len, 237 237 - uint16_t dport, int family) 238 238 - { 239 239 - struct udphdr *udph = header; 240 240 - int len = sizeof(*udph) + payload_len; 241 241 - 242 242 - udph->source = htons(cfg_src_port); 243 243 - udph->dest = htons(dport); 244 244 - udph->len = htons(len); 245 245 - udph->check = 0; 246 246 - if (family == AF_INET) 247 247 - udph->check = build_udp_v4_csum(header - sizeof(struct iphdr), 248 248 - udph, len >> 1); 249 249 - else 250 250 - udph->check = build_udp_v6_csum(header - sizeof(struct ipv6hdr), 251 251 - udph, len >> 1); 252 252 - } 253 253 - 254 254 - static void build_gue_header(void *header, uint8_t proto) 255 255 - { 256 256 - struct guehdr *gueh = header; 257 257 - 258 258 - gueh->proto_ctype = proto; 259 259 - } 260 260 - 261 261 - static void build_gre_header(void *header, uint16_t proto) 262 262 - { 263 263 - struct grehdr *greh = header; 264 264 - 265 265 - greh->protocol = htons(proto); 266 266 - } 267 267 - 268 268 - static int l3_length(int family) 269 269 - { 270 270 - if (family == AF_INET) 271 271 - return sizeof(struct iphdr); 272 272 - else 273 273 - return sizeof(struct ipv6hdr); 274 274 - } 275 275 - 276 276 - static int build_packet(void) 277 277 - { 278 278 - int ol3_len = 0, ol4_len = 0, il3_len = 0, il4_len = 0; 279 279 - int el3_len = 0; 280 280 - 281 281 - if (cfg_l3_extra) 282 282 - el3_len = l3_length(cfg_l3_extra); 283 283 - 284 284 - /* calculate header offsets */ 285 285 - if (cfg_encap_proto) { 286 286 - ol3_len = l3_length(cfg_l3_outer); 287 287 - 288 288 - if (cfg_encap_proto == IPPROTO_GRE) 289 289 - ol4_len = sizeof(struct grehdr); 290 290 - else if (cfg_encap_proto == IPPROTO_UDP) 291 291 - ol4_len = sizeof(struct udphdr) + sizeof(struct guehdr); 292 292 - } 293 293 - 294 294 - il3_len = l3_length(cfg_l3_inner); 295 295 - il4_len = sizeof(struct udphdr); 296 296 - 297 297 - if (el3_len + ol3_len + ol4_len + il3_len + il4_len + cfg_payload_len >= 298 298 - sizeof(buf)) 299 299 - error(1, 0, "packet too large\n"); 300 300 - 301 301 - /* 302 302 - * Fill packet from inside out, to calculate correct checksums. 303 303 - * But create ip before udp headers, as udp uses ip for pseudo-sum. 304 304 - */ 305 305 - memset(buf + el3_len + ol3_len + ol4_len + il3_len + il4_len, 306 306 - cfg_payload_char, cfg_payload_len); 307 307 - 308 308 - /* add zero byte for udp csum padding */ 309 309 - buf[el3_len + ol3_len + ol4_len + il3_len + il4_len + cfg_payload_len] = 0; 310 310 - 311 311 - switch (cfg_l3_inner) { 312 312 - case PF_INET: 313 313 - build_ipv4_header(buf + el3_len + ol3_len + ol4_len, 314 314 - IPPROTO_UDP, 315 315 - in_saddr4.sin_addr.s_addr, 316 316 - in_daddr4.sin_addr.s_addr, 317 317 - il4_len + cfg_payload_len, 318 318 - cfg_dsfield_inner); 319 319 - break; 320 320 - case PF_INET6: 321 321 - build_ipv6_header(buf + el3_len + ol3_len + ol4_len, 322 322 - IPPROTO_UDP, 323 323 - &in_saddr6, &in_daddr6, 324 324 - il4_len + cfg_payload_len, 325 325 - cfg_dsfield_inner); 326 326 - break; 327 327 - } 328 328 - 329 329 - build_udp_header(buf + el3_len + ol3_len + ol4_len + il3_len, 330 330 - cfg_payload_len, CFG_PORT_INNER, cfg_l3_inner); 331 331 - 332 332 - if (!cfg_encap_proto) 333 333 - return il3_len + il4_len + cfg_payload_len; 334 334 - 335 335 - switch (cfg_l3_outer) { 336 336 - case PF_INET: 337 337 - build_ipv4_header(buf + el3_len, cfg_encap_proto, 338 338 - out_saddr4.sin_addr.s_addr, 339 339 - out_daddr4.sin_addr.s_addr, 340 340 - ol4_len + il3_len + il4_len + cfg_payload_len, 341 341 - cfg_dsfield_outer); 342 342 - break; 343 343 - case PF_INET6: 344 344 - build_ipv6_header(buf + el3_len, cfg_encap_proto, 345 345 - &out_saddr6, &out_daddr6, 346 346 - ol4_len + il3_len + il4_len + cfg_payload_len, 347 347 - cfg_dsfield_outer); 348 348 - break; 349 349 - } 350 350 - 351 351 - switch (cfg_encap_proto) { 352 352 - case IPPROTO_UDP: 353 353 - build_gue_header(buf + el3_len + ol3_len + ol4_len - 354 354 - sizeof(struct guehdr), 355 355 - cfg_l3_inner == PF_INET ? IPPROTO_IPIP 356 356 - : IPPROTO_IPV6); 357 357 - build_udp_header(buf + el3_len + ol3_len, 358 358 - sizeof(struct guehdr) + il3_len + il4_len + 359 359 - cfg_payload_len, 360 360 - cfg_port_gue, cfg_l3_outer); 361 361 - break; 362 362 - case IPPROTO_GRE: 363 363 - build_gre_header(buf + el3_len + ol3_len, 364 364 - cfg_l3_inner == PF_INET ? ETH_P_IP 365 365 - : ETH_P_IPV6); 366 366 - break; 367 367 - } 368 368 - 369 369 - switch (cfg_l3_extra) { 370 370 - case PF_INET: 371 371 - build_ipv4_header(buf, 372 372 - cfg_l3_outer == PF_INET ? IPPROTO_IPIP 373 373 - : IPPROTO_IPV6, 374 374 - extra_saddr4.sin_addr.s_addr, 375 375 - extra_daddr4.sin_addr.s_addr, 376 376 - ol3_len + ol4_len + il3_len + il4_len + 377 377 - cfg_payload_len, 0); 378 378 - break; 379 379 - case PF_INET6: 380 380 - build_ipv6_header(buf, 381 381 - cfg_l3_outer == PF_INET ? IPPROTO_IPIP 382 382 - : IPPROTO_IPV6, 383 383 - &extra_saddr6, &extra_daddr6, 384 384 - ol3_len + ol4_len + il3_len + il4_len + 385 385 - cfg_payload_len, 0); 386 386 - break; 387 387 - } 388 388 - 389 389 - return el3_len + ol3_len + ol4_len + il3_len + il4_len + 390 390 - cfg_payload_len; 391 391 - } 392 392 - 393 393 - /* sender transmits encapsulated over RAW or unencap'd over UDP */ 394 394 - static int setup_tx(void) 395 395 - { 396 396 - int family, fd, ret; 397 397 - 398 398 - if (cfg_l3_extra) 399 399 - family = cfg_l3_extra; 400 400 - else if (cfg_l3_outer) 401 401 - family = cfg_l3_outer; 402 402 - else 403 403 - family = cfg_l3_inner; 404 404 - 405 405 - fd = socket(family, SOCK_RAW, IPPROTO_RAW); 406 406 - if (fd == -1) 407 407 - error(1, errno, "socket tx"); 408 408 - 409 409 - if (cfg_l3_extra) { 410 410 - if (cfg_l3_extra == PF_INET) 411 411 - ret = connect(fd, (void *) &extra_daddr4, 412 412 - sizeof(extra_daddr4)); 413 413 - else 414 414 - ret = connect(fd, (void *) &extra_daddr6, 415 415 - sizeof(extra_daddr6)); 416 416 - if (ret) 417 417 - error(1, errno, "connect tx"); 418 418 - } else if (cfg_l3_outer) { 419 419 - /* connect to destination if not encapsulated */ 420 420 - if (cfg_l3_outer == PF_INET) 421 421 - ret = connect(fd, (void *) &out_daddr4, 422 422 - sizeof(out_daddr4)); 423 423 - else 424 424 - ret = connect(fd, (void *) &out_daddr6, 425 425 - sizeof(out_daddr6)); 426 426 - if (ret) 427 427 - error(1, errno, "connect tx"); 428 428 - } else { 429 429 - /* otherwise using loopback */ 430 430 - if (cfg_l3_inner == PF_INET) 431 431 - ret = connect(fd, (void *) &in_daddr4, 432 432 - sizeof(in_daddr4)); 433 433 - else 434 434 - ret = connect(fd, (void *) &in_daddr6, 435 435 - sizeof(in_daddr6)); 436 436 - if (ret) 437 437 - error(1, errno, "connect tx"); 438 438 - } 439 439 - 440 440 - return fd; 441 441 - } 442 442 - 443 443 - /* receiver reads unencapsulated UDP */ 444 444 - static int setup_rx(void) 445 445 - { 446 446 - int fd, ret; 447 447 - 448 448 - fd = socket(cfg_l3_inner, SOCK_DGRAM, 0); 449 449 - if (fd == -1) 450 450 - error(1, errno, "socket rx"); 451 451 - 452 452 - if (cfg_l3_inner == PF_INET) 453 453 - ret = bind(fd, (void *) &in_daddr4, sizeof(in_daddr4)); 454 454 - else 455 455 - ret = bind(fd, (void *) &in_daddr6, sizeof(in_daddr6)); 456 456 - if (ret) 457 457 - error(1, errno, "bind rx"); 458 458 - 459 459 - return fd; 460 460 - } 461 461 - 462 462 - static int do_tx(int fd, const char *pkt, int len) 463 463 - { 464 464 - int ret; 465 465 - 466 466 - ret = write(fd, pkt, len); 467 467 - if (ret == -1) 468 468 - error(1, errno, "send"); 469 469 - if (ret != len) 470 470 - error(1, errno, "send: len (%d < %d)\n", ret, len); 471 471 - 472 472 - return 1; 473 473 - } 474 474 - 475 475 - static int do_poll(int fd, short events, int timeout) 476 476 - { 477 477 - struct pollfd pfd; 478 478 - int ret; 479 479 - 480 480 - pfd.fd = fd; 481 481 - pfd.events = events; 482 482 - 483 483 - ret = poll(&pfd, 1, timeout); 484 484 - if (ret == -1) 485 485 - error(1, errno, "poll"); 486 486 - if (ret && !(pfd.revents & POLLIN)) 487 487 - error(1, errno, "poll: unexpected event 0x%x\n", pfd.revents); 488 488 - 489 489 - return ret; 490 490 - } 491 491 - 492 492 - static int do_rx(int fd) 493 493 - { 494 494 - char rbuf; 495 495 - int ret, num = 0; 496 496 - 497 497 - while (1) { 498 498 - ret = recv(fd, &rbuf, 1, MSG_DONTWAIT); 499 499 - if (ret == -1 && errno == EAGAIN) 500 500 - break; 501 501 - if (ret == -1) 502 502 - error(1, errno, "recv"); 503 503 - if (rbuf != cfg_payload_char) 504 504 - error(1, 0, "recv: payload mismatch"); 505 505 - num++; 506 506 - } 507 507 - 508 508 - return num; 509 509 - } 510 510 - 511 511 - static int do_main(void) 512 512 - { 513 513 - unsigned long tstop, treport, tcur; 514 514 - int fdt = -1, fdr = -1, len, tx = 0, rx = 0; 515 515 - 516 516 - if (!cfg_only_tx) 517 517 - fdr = setup_rx(); 518 518 - if (!cfg_only_rx) 519 519 - fdt = setup_tx(); 520 520 - 521 521 - len = build_packet(); 522 522 - 523 523 - tcur = util_gettime(); 524 524 - treport = tcur + 1000; 525 525 - tstop = tcur + (cfg_num_secs * 1000); 526 526 - 527 527 - while (1) { 528 528 - if (!cfg_only_rx) 529 529 - tx += do_tx(fdt, buf, len); 530 530 - 531 531 - if (!cfg_only_tx) 532 532 - rx += do_rx(fdr); 533 533 - 534 534 - if (cfg_num_secs) { 535 535 - tcur = util_gettime(); 536 536 - if (tcur >= tstop) 537 537 - break; 538 538 - if (tcur >= treport) { 539 539 - fprintf(stderr, "pkts: tx=%u rx=%u\n", tx, rx); 540 540 - tx = 0; 541 541 - rx = 0; 542 542 - treport = tcur + 1000; 543 543 - } 544 544 - } else { 545 545 - if (tx == cfg_num_pkt) 546 546 - break; 547 547 - } 548 548 - } 549 549 - 550 550 - /* read straggler packets, if any */ 551 551 - if (rx < tx) { 552 552 - tstop = util_gettime() + 100; 553 553 - while (rx < tx) { 554 554 - tcur = util_gettime(); 555 555 - if (tcur >= tstop) 556 556 - break; 557 557 - 558 558 - do_poll(fdr, POLLIN, tstop - tcur); 559 559 - rx += do_rx(fdr); 560 560 - } 561 561 - } 562 562 - 563 563 - fprintf(stderr, "pkts: tx=%u rx=%u\n", tx, rx); 564 564 - 565 565 - if (fdr != -1 && close(fdr)) 566 566 - error(1, errno, "close rx"); 567 567 - if (fdt != -1 && close(fdt)) 568 568 - error(1, errno, "close tx"); 569 569 - 570 570 - /* 571 571 - * success (== 0) only if received all packets 572 572 - * unless failure is expected, in which case none must arrive. 573 573 - */ 574 574 - if (cfg_expect_failure) 575 575 - return rx != 0; 576 576 - else 577 577 - return rx != tx; 578 578 - } 579 579 - 580 580 - 581 581 - static void __attribute__((noreturn)) usage(const char *filepath) 582 582 - { 583 583 - fprintf(stderr, "Usage: %s [-e gre|gue|bare|none] [-i 4|6] [-l len] " 584 584 - "[-O 4|6] [-o 4|6] [-n num] [-t secs] [-R] [-T] " 585 585 - "[-s <osrc> [-d <odst>] [-S <isrc>] [-D <idst>] " 586 586 - "[-x <otos>] [-X <itos>] [-f <isport>] [-F]\n", 587 587 - filepath); 588 588 - exit(1); 589 589 - } 590 590 - 591 591 - static void parse_addr(int family, void *addr, const char *optarg) 592 592 - { 593 593 - int ret; 594 594 - 595 595 - ret = inet_pton(family, optarg, addr); 596 596 - if (ret == -1) 597 597 - error(1, errno, "inet_pton"); 598 598 - if (ret == 0) 599 599 - error(1, 0, "inet_pton: bad string"); 600 600 - } 601 601 - 602 602 - static void parse_addr4(struct sockaddr_in *addr, const char *optarg) 603 603 - { 604 604 - parse_addr(AF_INET, &addr->sin_addr, optarg); 605 605 - } 606 606 - 607 607 - static void parse_addr6(struct sockaddr_in6 *addr, const char *optarg) 608 608 - { 609 609 - parse_addr(AF_INET6, &addr->sin6_addr, optarg); 610 610 - } 611 611 - 612 612 - static int parse_protocol_family(const char *filepath, const char *optarg) 613 613 - { 614 614 - if (!strcmp(optarg, "4")) 615 615 - return PF_INET; 616 616 - if (!strcmp(optarg, "6")) 617 617 - return PF_INET6; 618 618 - 619 619 - usage(filepath); 620 620 - } 621 621 - 622 622 - static void parse_opts(int argc, char **argv) 623 623 - { 624 624 - int c; 625 625 - 626 626 - while ((c = getopt(argc, argv, "d:D:e:f:Fhi:l:n:o:O:Rs:S:t:Tx:X:")) != -1) { 627 627 - switch (c) { 628 628 - case 'd': 629 629 - if (cfg_l3_outer == AF_UNSPEC) 630 630 - error(1, 0, "-d must be preceded by -o"); 631 631 - if (cfg_l3_outer == AF_INET) 632 632 - parse_addr4(&out_daddr4, optarg); 633 633 - else 634 634 - parse_addr6(&out_daddr6, optarg); 635 635 - break; 636 636 - case 'D': 637 637 - if (cfg_l3_inner == AF_UNSPEC) 638 638 - error(1, 0, "-D must be preceded by -i"); 639 639 - if (cfg_l3_inner == AF_INET) 640 640 - parse_addr4(&in_daddr4, optarg); 641 641 - else 642 642 - parse_addr6(&in_daddr6, optarg); 643 643 - break; 644 644 - case 'e': 645 645 - if (!strcmp(optarg, "gre")) 646 646 - cfg_encap_proto = IPPROTO_GRE; 647 647 - else if (!strcmp(optarg, "gue")) 648 648 - cfg_encap_proto = IPPROTO_UDP; 649 649 - else if (!strcmp(optarg, "bare")) 650 650 - cfg_encap_proto = IPPROTO_IPIP; 651 651 - else if (!strcmp(optarg, "none")) 652 652 - cfg_encap_proto = IPPROTO_IP; /* == 0 */ 653 653 - else 654 654 - usage(argv[0]); 655 655 - break; 656 656 - case 'f': 657 657 - cfg_src_port = strtol(optarg, NULL, 0); 658 658 - break; 659 659 - case 'F': 660 660 - cfg_expect_failure = true; 661 661 - break; 662 662 - case 'h': 663 663 - usage(argv[0]); 664 664 - break; 665 665 - case 'i': 666 666 - if (!strcmp(optarg, "4")) 667 667 - cfg_l3_inner = PF_INET; 668 668 - else if (!strcmp(optarg, "6")) 669 669 - cfg_l3_inner = PF_INET6; 670 670 - else 671 671 - usage(argv[0]); 672 672 - break; 673 673 - case 'l': 674 674 - cfg_payload_len = strtol(optarg, NULL, 0); 675 675 - break; 676 676 - case 'n': 677 677 - cfg_num_pkt = strtol(optarg, NULL, 0); 678 678 - break; 679 679 - case 'o': 680 680 - cfg_l3_outer = parse_protocol_family(argv[0], optarg); 681 681 - break; 682 682 - case 'O': 683 683 - cfg_l3_extra = parse_protocol_family(argv[0], optarg); 684 684 - break; 685 685 - case 'R': 686 686 - cfg_only_rx = true; 687 687 - break; 688 688 - case 's': 689 689 - if (cfg_l3_outer == AF_INET) 690 690 - parse_addr4(&out_saddr4, optarg); 691 691 - else 692 692 - parse_addr6(&out_saddr6, optarg); 693 693 - break; 694 694 - case 'S': 695 695 - if (cfg_l3_inner == AF_INET) 696 696 - parse_addr4(&in_saddr4, optarg); 697 697 - else 698 698 - parse_addr6(&in_saddr6, optarg); 699 699 - break; 700 700 - case 't': 701 701 - cfg_num_secs = strtol(optarg, NULL, 0); 702 702 - break; 703 703 - case 'T': 704 704 - cfg_only_tx = true; 705 705 - break; 706 706 - case 'x': 707 707 - cfg_dsfield_outer = strtol(optarg, NULL, 0); 708 708 - break; 709 709 - case 'X': 710 710 - cfg_dsfield_inner = strtol(optarg, NULL, 0); 711 711 - break; 712 712 - } 713 713 - } 714 714 - 715 715 - if (cfg_only_rx && cfg_only_tx) 716 716 - error(1, 0, "options: cannot combine rx-only and tx-only"); 717 717 - 718 718 - if (cfg_encap_proto && cfg_l3_outer == AF_UNSPEC) 719 719 - error(1, 0, "options: must specify outer with encap"); 720 720 - else if ((!cfg_encap_proto) && cfg_l3_outer != AF_UNSPEC) 721 721 - error(1, 0, "options: cannot combine no-encap and outer"); 722 722 - else if ((!cfg_encap_proto) && cfg_l3_extra != AF_UNSPEC) 723 723 - error(1, 0, "options: cannot combine no-encap and extra"); 724 724 - 725 725 - if (cfg_l3_inner == AF_UNSPEC) 726 726 - cfg_l3_inner = AF_INET6; 727 727 - if (cfg_l3_inner == AF_INET6 && cfg_encap_proto == IPPROTO_IPIP) 728 728 - cfg_encap_proto = IPPROTO_IPV6; 729 729 - 730 730 - /* RFC 6040 4.2: 731 731 - * on decap, if outer encountered congestion (CE == 0x3), 732 732 - * but inner cannot encode ECN (NoECT == 0x0), then drop packet. 733 733 - */ 734 734 - if (((cfg_dsfield_outer & 0x3) == 0x3) && 735 735 - ((cfg_dsfield_inner & 0x3) == 0x0)) 736 736 - cfg_expect_failure = true; 737 737 - } 738 738 - 739 739 - static void print_opts(void) 740 740 - { 741 741 - if (cfg_l3_inner == PF_INET6) { 742 742 - util_printaddr("inner.dest6", (void *) &in_daddr6); 743 743 - util_printaddr("inner.source6", (void *) &in_saddr6); 744 744 - } else { 745 745 - util_printaddr("inner.dest4", (void *) &in_daddr4); 746 746 - util_printaddr("inner.source4", (void *) &in_saddr4); 747 747 - } 748 748 - 749 749 - if (!cfg_l3_outer) 750 750 - return; 751 751 - 752 752 - fprintf(stderr, "encap proto: %u\n", cfg_encap_proto); 753 753 - 754 754 - if (cfg_l3_outer == PF_INET6) { 755 755 - util_printaddr("outer.dest6", (void *) &out_daddr6); 756 756 - util_printaddr("outer.source6", (void *) &out_saddr6); 757 757 - } else { 758 758 - util_printaddr("outer.dest4", (void *) &out_daddr4); 759 759 - util_printaddr("outer.source4", (void *) &out_saddr4); 760 760 - } 761 761 - 762 762 - if (!cfg_l3_extra) 763 763 - return; 764 764 - 765 765 - if (cfg_l3_outer == PF_INET6) { 766 766 - util_printaddr("extra.dest6", (void *) &extra_daddr6); 767 767 - util_printaddr("extra.source6", (void *) &extra_saddr6); 768 768 - } else { 769 769 - util_printaddr("extra.dest4", (void *) &extra_daddr4); 770 770 - util_printaddr("extra.source4", (void *) &extra_saddr4); 771 771 - } 772 772 - 773 773 - } 774 774 - 775 775 - int main(int argc, char **argv) 776 776 - { 777 777 - parse_opts(argc, argv); 778 778 - print_opts(); 779 779 - return do_main(); 780 780 - }

-178

tools/testing/selftests/bpf/test_flow_dissector.sh

reviewed

··· 1 1 - #!/bin/bash 2 2 - # SPDX-License-Identifier: GPL-2.0 3 3 - # 4 4 - # Load BPF flow dissector and verify it correctly dissects traffic 5 5 - 6 6 - BPF_FILE="bpf_flow.bpf.o" 7 7 - export TESTNAME=test_flow_dissector 8 8 - unmount=0 9 9 - 10 10 - # Kselftest framework requirement - SKIP code is 4. 11 11 - ksft_skip=4 12 12 - 13 13 - msg="skip all tests:" 14 14 - if [ $UID != 0 ]; then 15 15 - echo $msg please run this as root >&2 16 16 - exit $ksft_skip 17 17 - fi 18 18 - 19 19 - # This test needs to be run in a network namespace with in_netns.sh. Check if 20 20 - # this is the case and run it with in_netns.sh if it is being run in the root 21 21 - # namespace. 22 22 - if [[ -z $(ip netns identify $$) ]]; then 23 23 - err=0 24 24 - if bpftool="$(which bpftool)"; then 25 25 - echo "Testing global flow dissector..." 26 26 - 27 27 - $bpftool prog loadall $BPF_FILE /sys/fs/bpf/flow \ 28 28 - type flow_dissector 29 29 - 30 30 - if ! unshare --net $bpftool prog attach pinned \ 31 31 - /sys/fs/bpf/flow/_dissect flow_dissector; then 32 32 - echo "Unexpected unsuccessful attach in namespace" >&2 33 33 - err=1 34 34 - fi 35 35 - 36 36 - $bpftool prog attach pinned /sys/fs/bpf/flow/_dissect \ 37 37 - flow_dissector 38 38 - 39 39 - if unshare --net $bpftool prog attach pinned \ 40 40 - /sys/fs/bpf/flow/_dissect flow_dissector; then 41 41 - echo "Unexpected successful attach in namespace" >&2 42 42 - err=1 43 43 - fi 44 44 - 45 45 - if ! $bpftool prog detach pinned \ 46 46 - /sys/fs/bpf/flow/_dissect flow_dissector; then 47 47 - echo "Failed to detach flow dissector" >&2 48 48 - err=1 49 49 - fi 50 50 - 51 51 - rm -rf /sys/fs/bpf/flow 52 52 - else 53 53 - echo "Skipping root flow dissector test, bpftool not found" >&2 54 54 - fi 55 55 - 56 56 - # Run the rest of the tests in a net namespace. 57 57 - ../net/in_netns.sh "$0" "$@" 58 58 - err=$(( $err + $? )) 59 59 - 60 60 - if (( $err == 0 )); then 61 61 - echo "selftests: $TESTNAME [PASS]"; 62 62 - else 63 63 - echo "selftests: $TESTNAME [FAILED]"; 64 64 - fi 65 65 - 66 66 - exit $err 67 67 - fi 68 68 - 69 69 - # Determine selftest success via shell exit code 70 70 - exit_handler() 71 71 - { 72 72 - set +e 73 73 - 74 74 - # Cleanup 75 75 - tc filter del dev lo ingress pref 1337 2> /dev/null 76 76 - tc qdisc del dev lo ingress 2> /dev/null 77 77 - ./flow_dissector_load -d 2> /dev/null 78 78 - if [ $unmount -ne 0 ]; then 79 79 - umount bpffs 2> /dev/null 80 80 - fi 81 81 - } 82 82 - 83 83 - # Exit script immediately (well catched by trap handler) if any 84 84 - # program/thing exits with a non-zero status. 85 85 - set -e 86 86 - 87 87 - # (Use 'trap -l' to list meaning of numbers) 88 88 - trap exit_handler 0 2 3 6 9 89 89 - 90 90 - # Mount BPF file system 91 91 - if /bin/mount | grep /sys/fs/bpf > /dev/null; then 92 92 - echo "bpffs already mounted" 93 93 - else 94 94 - echo "bpffs not mounted. Mounting..." 95 95 - unmount=1 96 96 - /bin/mount bpffs /sys/fs/bpf -t bpf 97 97 - fi 98 98 - 99 99 - # Attach BPF program 100 100 - ./flow_dissector_load -p $BPF_FILE -s _dissect 101 101 - 102 102 - # Setup 103 103 - tc qdisc add dev lo ingress 104 104 - echo 0 > /proc/sys/net/ipv4/conf/default/rp_filter 105 105 - echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter 106 106 - echo 0 > /proc/sys/net/ipv4/conf/lo/rp_filter 107 107 - 108 108 - echo "Testing IPv4..." 109 109 - # Drops all IP/UDP packets coming from port 9 110 110 - tc filter add dev lo parent ffff: protocol ip pref 1337 flower ip_proto \ 111 111 - udp src_port 9 action drop 112 112 - 113 113 - # Send 10 IPv4/UDP packets from port 8. Filter should not drop any. 114 114 - ./test_flow_dissector -i 4 -f 8 115 115 - # Send 10 IPv4/UDP packets from port 9. Filter should drop all. 116 116 - ./test_flow_dissector -i 4 -f 9 -F 117 117 - # Send 10 IPv4/UDP packets from port 10. Filter should not drop any. 118 118 - ./test_flow_dissector -i 4 -f 10 119 119 - 120 120 - echo "Testing IPv4 from 127.0.0.127 (fallback to generic dissector)..." 121 121 - # Send 10 IPv4/UDP packets from port 8. Filter should not drop any. 122 122 - ./test_flow_dissector -i 4 -S 127.0.0.127 -f 8 123 123 - # Send 10 IPv4/UDP packets from port 9. Filter should drop all. 124 124 - ./test_flow_dissector -i 4 -S 127.0.0.127 -f 9 -F 125 125 - # Send 10 IPv4/UDP packets from port 10. Filter should not drop any. 126 126 - ./test_flow_dissector -i 4 -S 127.0.0.127 -f 10 127 127 - 128 128 - echo "Testing IPIP..." 129 129 - # Send 10 IPv4/IPv4/UDP packets from port 8. Filter should not drop any. 130 130 - ./with_addr.sh ./with_tunnels.sh ./test_flow_dissector -o 4 -e bare -i 4 \ 131 131 - -D 192.168.0.1 -S 1.1.1.1 -f 8 132 132 - # Send 10 IPv4/IPv4/UDP packets from port 9. Filter should drop all. 133 133 - ./with_addr.sh ./with_tunnels.sh ./test_flow_dissector -o 4 -e bare -i 4 \ 134 134 - -D 192.168.0.1 -S 1.1.1.1 -f 9 -F 135 135 - # Send 10 IPv4/IPv4/UDP packets from port 10. Filter should not drop any. 136 136 - ./with_addr.sh ./with_tunnels.sh ./test_flow_dissector -o 4 -e bare -i 4 \ 137 137 - -D 192.168.0.1 -S 1.1.1.1 -f 10 138 138 - 139 139 - echo "Testing IPv4 + GRE..." 140 140 - # Send 10 IPv4/GRE/IPv4/UDP packets from port 8. Filter should not drop any. 141 141 - ./with_addr.sh ./with_tunnels.sh ./test_flow_dissector -o 4 -e gre -i 4 \ 142 142 - -D 192.168.0.1 -S 1.1.1.1 -f 8 143 143 - # Send 10 IPv4/GRE/IPv4/UDP packets from port 9. Filter should drop all. 144 144 - ./with_addr.sh ./with_tunnels.sh ./test_flow_dissector -o 4 -e gre -i 4 \ 145 145 - -D 192.168.0.1 -S 1.1.1.1 -f 9 -F 146 146 - # Send 10 IPv4/GRE/IPv4/UDP packets from port 10. Filter should not drop any. 147 147 - ./with_addr.sh ./with_tunnels.sh ./test_flow_dissector -o 4 -e gre -i 4 \ 148 148 - -D 192.168.0.1 -S 1.1.1.1 -f 10 149 149 - 150 150 - tc filter del dev lo ingress pref 1337 151 151 - 152 152 - echo "Testing port range..." 153 153 - # Drops all IP/UDP packets coming from port 8-10 154 154 - tc filter add dev lo parent ffff: protocol ip pref 1337 flower ip_proto \ 155 155 - udp src_port 8-10 action drop 156 156 - 157 157 - # Send 10 IPv4/UDP packets from port 7. Filter should not drop any. 158 158 - ./test_flow_dissector -i 4 -f 7 159 159 - # Send 10 IPv4/UDP packets from port 9. Filter should drop all. 160 160 - ./test_flow_dissector -i 4 -f 9 -F 161 161 - # Send 10 IPv4/UDP packets from port 11. Filter should not drop any. 162 162 - ./test_flow_dissector -i 4 -f 11 163 163 - 164 164 - tc filter del dev lo ingress pref 1337 165 165 - 166 166 - echo "Testing IPv6..." 167 167 - # Drops all IPv6/UDP packets coming from port 9 168 168 - tc filter add dev lo parent ffff: protocol ipv6 pref 1337 flower ip_proto \ 169 169 - udp src_port 9 action drop 170 170 - 171 171 - # Send 10 IPv6/UDP packets from port 8. Filter should not drop any. 172 172 - ./test_flow_dissector -i 6 -f 8 173 173 - # Send 10 IPv6/UDP packets from port 9. Filter should drop all. 174 174 - ./test_flow_dissector -i 6 -f 9 -F 175 175 - # Send 10 IPv6/UDP packets from port 10. Filter should not drop any. 176 176 - ./test_flow_dissector -i 6 -f 10 177 177 - 178 178 - exit 0

+21

tools/testing/selftests/bpf/test_kmods/Makefile

reviewed

··· 1 1 + TEST_KMOD_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) 2 2 + KDIR ?= $(abspath $(TEST_KMOD_DIR)/../../../../..) 3 3 + 4 4 + ifeq ($(V),1) 5 5 + Q = 6 6 + else 7 7 + Q = @ 8 8 + endif 9 9 + 10 10 + MODULES = bpf_testmod.ko bpf_test_no_cfi.ko bpf_test_modorder_x.ko \ 11 11 + bpf_test_modorder_y.ko 12 12 + 13 13 + $(foreach m,$(MODULES),$(eval obj-m += $(m:.ko=.o))) 14 14 + 15 15 + CFLAGS_bpf_testmod.o = -I$(src) 16 16 + 17 17 + all: 18 18 + $(Q)$(MAKE) -C $(KDIR) M=$(TEST_KMOD_DIR) modules 19 19 + 20 20 + clean: 21 21 + $(Q)$(MAKE) -C $(KDIR) M=$(TEST_KMOD_DIR) clean

+15

tools/testing/selftests/bpf/test_progs.c

reviewed

··· 1282 1282 backtrace_symbols_fd(bt, sz, STDERR_FILENO); 1283 1283 } 1284 1284 1285 1285 + void hexdump(const char *prefix, const void *buf, size_t len) 1286 1286 + { 1287 1287 + for (int i = 0; i < len; i++) { 1288 1288 + if (!(i % 16)) { 1289 1289 + if (i) 1290 1290 + fprintf(stdout, "\n"); 1291 1291 + fprintf(stdout, "%s", prefix); 1292 1292 + } 1293 1293 + if (i && !(i % 8) && (i % 16)) 1294 1294 + fprintf(stdout, "\t"); 1295 1295 + fprintf(stdout, "%02X ", ((uint8_t *)(buf))[i]); 1296 1296 + } 1297 1297 + fprintf(stdout, "\n"); 1298 1298 + } 1299 1299 + 1285 1300 static void sigint_handler(int signum) 1286 1301 { 1287 1302 int i;

+15

tools/testing/selftests/bpf/test_progs.h

reviewed

··· 185 185 void test__skip(void); 186 186 void test__fail(void); 187 187 int test__join_cgroup(const char *path); 188 188 + void hexdump(const char *prefix, const void *buf, size_t len); 188 189 189 190 #define PRINT_FAIL(format...) \ 190 191 ({ \ ··· 342 341 CHECK(!___ok, (name), \ 343 342 "unexpected %s: '%s' is not a substring of '%s'\n", \ 344 343 (name), ___substr, ___str); \ 344 344 + ___ok; \ 345 345 + }) 346 346 + 347 347 + #define ASSERT_MEMEQ(actual, expected, len, name) ({ \ 348 348 + static int duration = 0; \ 349 349 + const void *__act = actual; \ 350 350 + const void *__exp = expected; \ 351 351 + int __len = len; \ 352 352 + bool ___ok = memcmp(__act, __exp, __len) == 0; \ 353 353 + CHECK(!___ok, (name), "unexpected memory mismatch\n"); \ 354 354 + fprintf(stdout, "actual:\n"); \ 355 355 + hexdump("\t", __act, __len); \ 356 356 + fprintf(stdout, "expected:\n"); \ 357 357 + hexdump("\t", __exp, __len); \ 345 358 ___ok; \ 346 359 }) 347 360

+1

tools/testing/selftests/bpf/test_tc_tunnel.sh

reviewed

··· 305 305 client_connect 306 306 verify_data 307 307 server_listen 308 308 + wait_for_port ${port} ${netcat_opt} 308 309 fi 309 310 310 311 # serverside, use BPF for decap

-79

tools/testing/selftests/bpf/test_xdp_redirect.sh

reviewed

··· 1 1 - #!/bin/bash 2 2 - # Create 2 namespaces with two veth peers, and 3 3 - # forward packets in-between using generic XDP 4 4 - # 5 5 - # NS1(veth11) NS2(veth22) 6 6 - # | | 7 7 - # | | 8 8 - # (veth1, ------ (veth2, 9 9 - # id:111) id:222) 10 10 - # | xdp forwarding | 11 11 - # ------------------ 12 12 - 13 13 - readonly NS1="ns1-$(mktemp -u XXXXXX)" 14 14 - readonly NS2="ns2-$(mktemp -u XXXXXX)" 15 15 - ret=0 16 16 - 17 17 - setup() 18 18 - { 19 19 - 20 20 - local xdpmode=$1 21 21 - 22 22 - ip netns add ${NS1} 23 23 - ip netns add ${NS2} 24 24 - 25 25 - ip link add veth1 index 111 type veth peer name veth11 netns ${NS1} 26 26 - ip link add veth2 index 222 type veth peer name veth22 netns ${NS2} 27 27 - 28 28 - ip link set veth1 up 29 29 - ip link set veth2 up 30 30 - ip -n ${NS1} link set dev veth11 up 31 31 - ip -n ${NS2} link set dev veth22 up 32 32 - 33 33 - ip -n ${NS1} addr add 10.1.1.11/24 dev veth11 34 34 - ip -n ${NS2} addr add 10.1.1.22/24 dev veth22 35 35 - } 36 36 - 37 37 - cleanup() 38 38 - { 39 39 - ip link del veth1 2> /dev/null 40 40 - ip link del veth2 2> /dev/null 41 41 - ip netns del ${NS1} 2> /dev/null 42 42 - ip netns del ${NS2} 2> /dev/null 43 43 - } 44 44 - 45 45 - test_xdp_redirect() 46 46 - { 47 47 - local xdpmode=$1 48 48 - 49 49 - setup 50 50 - 51 51 - ip link set dev veth1 $xdpmode off &> /dev/null 52 52 - if [ $? -ne 0 ];then 53 53 - echo "selftests: test_xdp_redirect $xdpmode [SKIP]" 54 54 - return 0 55 55 - fi 56 56 - 57 57 - ip -n ${NS1} link set veth11 $xdpmode obj xdp_dummy.bpf.o sec xdp &> /dev/null 58 58 - ip -n ${NS2} link set veth22 $xdpmode obj xdp_dummy.bpf.o sec xdp &> /dev/null 59 59 - ip link set dev veth1 $xdpmode obj test_xdp_redirect.bpf.o sec redirect_to_222 &> /dev/null 60 60 - ip link set dev veth2 $xdpmode obj test_xdp_redirect.bpf.o sec redirect_to_111 &> /dev/null 61 61 - 62 62 - if ip netns exec ${NS1} ping -c 1 10.1.1.22 &> /dev/null && 63 63 - ip netns exec ${NS2} ping -c 1 10.1.1.11 &> /dev/null; then 64 64 - echo "selftests: test_xdp_redirect $xdpmode [PASS]"; 65 65 - else 66 66 - ret=1 67 67 - echo "selftests: test_xdp_redirect $xdpmode [FAILED]"; 68 68 - fi 69 69 - 70 70 - cleanup 71 71 - } 72 72 - 73 73 - set -e 74 74 - trap cleanup 2 3 6 9 75 75 - 76 76 - test_xdp_redirect xdpgeneric 77 77 - test_xdp_redirect xdpdrv 78 78 - 79 79 - exit $ret

+1 -1

tools/testing/selftests/bpf/verifier/calls.c

reviewed

··· 2252 2252 BPF_EXIT_INSN(), 2253 2253 }, 2254 2254 .fixup_map_hash_48b = { 7 }, 2255 2255 - .errstr_unpriv = "invalid indirect read from stack R2 off -8+0 size 8", 2255 2255 + .errstr_unpriv = "invalid read from stack R2 off -8+0 size 8", 2256 2256 .result_unpriv = REJECT, 2257 2257 /* in privileged mode reads from uninitialized stack locations are permitted */ 2258 2258 .result = ACCEPT,

+1 -1

tools/testing/selftests/bpf/verifier/map_kptr.c

reviewed

··· 373 373 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 374 374 .fixup_map_kptr = { 1 }, 375 375 .result = REJECT, 376 376 - .errstr = "Unreleased reference id=5 alloc_insn=20", 376 376 + .errstr = "Unreleased reference id=4 alloc_insn=20", 377 377 .fixup_kfunc_btf_id = { 378 378 { "bpf_kfunc_call_test_acquire", 15 }, 379 379 }

+149 -10

tools/testing/selftests/bpf/veristat.c

reviewed

··· 21 21 #include <gelf.h> 22 22 #include <float.h> 23 23 #include <math.h> 24 24 + #include <limits.h> 24 25 25 26 #ifndef ARRAY_SIZE 26 27 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) 28 28 + #endif 29 29 + 30 30 + #ifndef max 31 31 + #define max(a, b) ((a) > (b) ? (a) : (b)) 32 32 + #endif 33 33 + 34 34 + #ifndef min 35 35 + #define min(a, b) ((a) < (b) ? (a) : (b)) 27 36 #endif 28 37 29 38 enum stat_id { ··· 43 34 PEAK_STATES, 44 35 MAX_STATES_PER_INSN, 45 36 MARK_READ_MAX_LEN, 37 37 + SIZE, 38 38 + JITED_SIZE, 39 39 + STACK, 40 40 + PROG_TYPE, 41 41 + ATTACH_TYPE, 46 42 47 43 FILE_NAME, 48 44 PROG_NAME, ··· 217 203 "\n" 218 204 "USAGE: veristat <obj-file> [<obj-file>...]\n" 219 205 " OR: veristat -C <baseline.csv> <comparison.csv>\n" 220 220 - " OR: veristat -R <results.csv>\n"; 206 206 + " OR: veristat -R <results.csv>\n" 207 207 + " OR: veristat -vl2 <to_analyze.bpf.o>\n"; 221 208 222 209 enum { 223 210 OPT_LOG_FIXED = 1000, ··· 230 215 { "version", 'V', NULL, 0, "Print version" }, 231 216 { "verbose", 'v', NULL, 0, "Verbose mode" }, 232 217 { "debug", 'd', NULL, 0, "Debug mode (turns on libbpf debug logging)" }, 233 233 - { "log-level", 'l', "LEVEL", 0, "Verifier log level (default 0 for normal mode, 1 for verbose mode)" }, 218 218 + { "log-level", 'l', "LEVEL", 0, "Verifier log level (default 0 for normal mode, 1 for verbose mode, 2 for full verification log)" }, 234 219 { "log-fixed", OPT_LOG_FIXED, NULL, 0, "Disable verifier log rotation" }, 235 220 { "log-size", OPT_LOG_SIZE, "BYTES", 0, "Customize verifier log size (default to 16MB)" }, 236 221 { "top-n", 'n', "N", 0, "Emit only up to first N results." }, ··· 655 640 } 656 641 657 642 static const struct stat_specs default_output_spec = { 658 658 - .spec_cnt = 7, 643 643 + .spec_cnt = 8, 659 644 .ids = { 660 645 FILE_NAME, PROG_NAME, VERDICT, DURATION, 661 661 - TOTAL_INSNS, TOTAL_STATES, PEAK_STATES, 646 646 + TOTAL_INSNS, TOTAL_STATES, SIZE, JITED_SIZE 662 647 }, 663 648 }; 664 649 665 650 static const struct stat_specs default_csv_output_spec = { 666 666 - .spec_cnt = 9, 651 651 + .spec_cnt = 14, 667 652 .ids = { 668 653 FILE_NAME, PROG_NAME, VERDICT, DURATION, 669 654 TOTAL_INSNS, TOTAL_STATES, PEAK_STATES, 670 655 MAX_STATES_PER_INSN, MARK_READ_MAX_LEN, 656 656 + SIZE, JITED_SIZE, PROG_TYPE, ATTACH_TYPE, 657 657 + STACK, 671 658 }, 672 659 }; 673 660 ··· 705 688 [PEAK_STATES] = { "Peak states", {"peak_states"}, }, 706 689 [MAX_STATES_PER_INSN] = { "Max states per insn", {"max_states_per_insn"}, }, 707 690 [MARK_READ_MAX_LEN] = { "Max mark read length", {"max_mark_read_len", "mark_read"}, }, 691 691 + [SIZE] = { "Program size", {"prog_size"}, }, 692 692 + [JITED_SIZE] = { "Jited size", {"prog_size_jited"}, }, 693 693 + [STACK] = {"Stack depth", {"stack_depth", "stack"}, }, 694 694 + [PROG_TYPE] = { "Program type", {"prog_type"}, }, 695 695 + [ATTACH_TYPE] = { "Attach type", {"attach_type", }, }, 708 696 }; 709 697 710 698 static bool parse_stat_id_var(const char *name, size_t len, int *id, ··· 857 835 static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats *s) 858 836 { 859 837 const char *cur; 860 860 - int pos, lines; 838 838 + int pos, lines, sub_stack, cnt = 0; 839 839 + char *state = NULL, *token, stack[512]; 861 840 862 841 buf[buf_sz - 1] = '\0'; 863 842 ··· 876 853 877 854 if (1 == sscanf(cur, "verification time %ld usec\n", &s->stats[DURATION])) 878 855 continue; 879 879 - if (6 == sscanf(cur, "processed %ld insns (limit %*d) max_states_per_insn %ld total_states %ld peak_states %ld mark_read %ld", 856 856 + if (5 == sscanf(cur, "processed %ld insns (limit %*d) max_states_per_insn %ld total_states %ld peak_states %ld mark_read %ld", 880 857 &s->stats[TOTAL_INSNS], 881 858 &s->stats[MAX_STATES_PER_INSN], 882 859 &s->stats[TOTAL_STATES], 883 860 &s->stats[PEAK_STATES], 884 861 &s->stats[MARK_READ_MAX_LEN])) 885 862 continue; 886 886 - } 887 863 864 864 + if (1 == sscanf(cur, "stack depth %511s", stack)) 865 865 + continue; 866 866 + } 867 867 + while ((token = strtok_r(cnt++ ? NULL : stack, "+", &state))) { 868 868 + if (sscanf(token, "%d", &sub_stack) == 0) 869 869 + break; 870 870 + s->stats[STACK] += sub_stack; 871 871 + } 888 872 return 0; 889 873 } 890 874 ··· 914 884 const struct line_cnt *b_cnt = (const struct line_cnt *)b; 915 885 916 886 if (a_cnt->cnt != b_cnt->cnt) 917 917 - return a_cnt->cnt < b_cnt->cnt ? -1 : 1; 887 887 + return a_cnt->cnt > b_cnt->cnt ? -1 : 1; 918 888 return strcmp(a_cnt->line, b_cnt->line); 919 889 } 920 890 ··· 1062 1032 return -ESRCH; 1063 1033 } 1064 1034 1035 1035 + /* Make sure only target program is referenced from struct_ops map, 1036 1036 + * otherwise libbpf would automatically set autocreate for all 1037 1037 + * referenced programs. 1038 1038 + * See libbpf.c:bpf_object_adjust_struct_ops_autoload. 1039 1039 + */ 1040 1040 + static void mask_unrelated_struct_ops_progs(struct bpf_object *obj, 1041 1041 + struct bpf_map *map, 1042 1042 + struct bpf_program *prog) 1043 1043 + { 1044 1044 + struct btf *btf = bpf_object__btf(obj); 1045 1045 + const struct btf_type *t, *mt; 1046 1046 + struct btf_member *m; 1047 1047 + int i, moff; 1048 1048 + size_t data_sz, ptr_sz = sizeof(void *); 1049 1049 + void *data; 1050 1050 + 1051 1051 + t = btf__type_by_id(btf, bpf_map__btf_value_type_id(map)); 1052 1052 + if (!btf_is_struct(t)) 1053 1053 + return; 1054 1054 + 1055 1055 + data = bpf_map__initial_value(map, &data_sz); 1056 1056 + for (i = 0; i < btf_vlen(t); i++) { 1057 1057 + m = &btf_members(t)[i]; 1058 1058 + mt = btf__type_by_id(btf, m->type); 1059 1059 + if (!btf_is_ptr(mt)) 1060 1060 + continue; 1061 1061 + moff = m->offset / 8; 1062 1062 + if (moff + ptr_sz > data_sz) 1063 1063 + continue; 1064 1064 + if (memcmp(data + moff, &prog, ptr_sz) == 0) 1065 1065 + continue; 1066 1066 + memset(data + moff, 0, ptr_sz); 1067 1067 + } 1068 1068 + } 1069 1069 + 1065 1070 static void fixup_obj(struct bpf_object *obj, struct bpf_program *prog, const char *filename) 1066 1071 { 1067 1072 struct bpf_map *map; ··· 1111 1046 case BPF_MAP_TYPE_TASK_STORAGE: 1112 1047 case BPF_MAP_TYPE_INODE_STORAGE: 1113 1048 case BPF_MAP_TYPE_CGROUP_STORAGE: 1049 1049 + break; 1050 1050 + case BPF_MAP_TYPE_STRUCT_OPS: 1051 1051 + mask_unrelated_struct_ops_progs(obj, map, prog); 1114 1052 break; 1115 1053 default: 1116 1054 if (bpf_map__max_entries(map) == 0) ··· 1214 1146 char *buf; 1215 1147 int buf_sz, log_level; 1216 1148 struct verif_stats *stats; 1149 1149 + struct bpf_prog_info info; 1150 1150 + __u32 info_len = sizeof(info); 1217 1151 int err = 0; 1218 1152 void *tmp; 1153 1153 + int fd; 1219 1154 1220 1155 if (!should_process_file_prog(base_filename, bpf_program__name(prog))) { 1221 1156 env.progs_skipped++; ··· 1267 1196 stats->file_name = strdup(base_filename); 1268 1197 stats->prog_name = strdup(bpf_program__name(prog)); 1269 1198 stats->stats[VERDICT] = err == 0; /* 1 - success, 0 - failure */ 1199 1199 + stats->stats[SIZE] = bpf_program__insn_cnt(prog); 1200 1200 + stats->stats[PROG_TYPE] = bpf_program__type(prog); 1201 1201 + stats->stats[ATTACH_TYPE] = bpf_program__expected_attach_type(prog); 1202 1202 + 1203 1203 + memset(&info, 0, info_len); 1204 1204 + fd = bpf_program__fd(prog); 1205 1205 + if (fd > 0 && bpf_prog_get_info_by_fd(fd, &info, &info_len) == 0) 1206 1206 + stats->stats[JITED_SIZE] = info.jited_prog_len; 1207 1207 + 1270 1208 parse_verif_log(buf, buf_sz, stats); 1271 1209 1272 1210 if (env.verbose) { ··· 1389 1309 case PROG_NAME: 1390 1310 cmp = strcmp(s1->prog_name, s2->prog_name); 1391 1311 break; 1312 1312 + case ATTACH_TYPE: 1313 1313 + case PROG_TYPE: 1314 1314 + case SIZE: 1315 1315 + case JITED_SIZE: 1316 1316 + case STACK: 1392 1317 case VERDICT: 1393 1318 case DURATION: 1394 1319 case TOTAL_INSNS: ··· 1608 1523 else 1609 1524 *str = s->stats[VERDICT] ? "success" : "failure"; 1610 1525 break; 1526 1526 + case ATTACH_TYPE: 1527 1527 + if (!s) 1528 1528 + *str = "N/A"; 1529 1529 + else 1530 1530 + *str = libbpf_bpf_attach_type_str(s->stats[ATTACH_TYPE]) ?: "N/A"; 1531 1531 + break; 1532 1532 + case PROG_TYPE: 1533 1533 + if (!s) 1534 1534 + *str = "N/A"; 1535 1535 + else 1536 1536 + *str = libbpf_bpf_prog_type_str(s->stats[PROG_TYPE]) ?: "N/A"; 1537 1537 + break; 1611 1538 case DURATION: 1612 1539 case TOTAL_INSNS: 1613 1540 case TOTAL_STATES: 1614 1541 case PEAK_STATES: 1615 1542 case MAX_STATES_PER_INSN: 1616 1543 case MARK_READ_MAX_LEN: 1544 1544 + case STACK: 1545 1545 + case SIZE: 1546 1546 + case JITED_SIZE: 1617 1547 *val = s ? s->stats[id] : 0; 1618 1548 break; 1619 1549 default: ··· 1712 1612 case TOTAL_STATES: 1713 1613 case PEAK_STATES: 1714 1614 case MAX_STATES_PER_INSN: 1715 1715 - case MARK_READ_MAX_LEN: { 1615 1615 + case MARK_READ_MAX_LEN: 1616 1616 + case SIZE: 1617 1617 + case JITED_SIZE: 1618 1618 + case STACK: { 1716 1619 long val; 1717 1620 int err, n; 1718 1621 ··· 1726 1623 } 1727 1624 1728 1625 st->stats[id] = val; 1626 1626 + break; 1627 1627 + } 1628 1628 + case PROG_TYPE: { 1629 1629 + enum bpf_prog_type prog_type = 0; 1630 1630 + const char *type; 1631 1631 + 1632 1632 + while ((type = libbpf_bpf_prog_type_str(prog_type))) { 1633 1633 + if (strcmp(type, str) == 0) { 1634 1634 + st->stats[id] = prog_type; 1635 1635 + break; 1636 1636 + } 1637 1637 + prog_type++; 1638 1638 + } 1639 1639 + 1640 1640 + if (!type) { 1641 1641 + fprintf(stderr, "Unrecognized prog type %s\n", str); 1642 1642 + return -EINVAL; 1643 1643 + } 1644 1644 + break; 1645 1645 + } 1646 1646 + case ATTACH_TYPE: { 1647 1647 + enum bpf_attach_type attach_type = 0; 1648 1648 + const char *type; 1649 1649 + 1650 1650 + while ((type = libbpf_bpf_attach_type_str(attach_type))) { 1651 1651 + if (strcmp(type, str) == 0) { 1652 1652 + st->stats[id] = attach_type; 1653 1653 + break; 1654 1654 + } 1655 1655 + attach_type++; 1656 1656 + } 1657 1657 + 1658 1658 + if (!type) { 1659 1659 + fprintf(stderr, "Unrecognized attach type %s\n", str); 1660 1660 + return -EINVAL; 1661 1661 + } 1729 1662 break; 1730 1663 } 1731 1664 default:

+1 -1

tools/testing/selftests/bpf/xdp_hw_metadata.c

reviewed

··· 27 27 #include <linux/errqueue.h> 28 28 #include <linux/if_link.h> 29 29 #include <linux/net_tstamp.h> 30 30 - #include <linux/udp.h> 30 30 + #include <netinet/udp.h> 31 31 #include <linux/sockios.h> 32 32 #include <linux/if_xdp.h> 33 33 #include <sys/mman.h>