bpf: Refactor kptr_off_tab into btf_record

+81 -44

include/linux/bpf.h

··· 165 165 }; 166 166 167 167 enum { 168 - /* Support at most 8 pointers in a BPF map value */ 169 - BPF_MAP_VALUE_OFF_MAX = 8, 170 - BPF_MAP_OFF_ARR_MAX = BPF_MAP_VALUE_OFF_MAX + 168 + /* Support at most 8 pointers in a BTF type */ 169 + BTF_FIELDS_MAX = 8, 170 + BPF_MAP_OFF_ARR_MAX = BTF_FIELDS_MAX + 171 171 1 + /* for bpf_spin_lock */ 172 172 1, /* for bpf_timer */ 173 173 }; 174 174 175 - enum bpf_kptr_type { 176 - BPF_KPTR_UNREF, 177 - BPF_KPTR_REF, 175 + enum btf_field_type { 176 + BPF_KPTR_UNREF = (1 << 2), 177 + BPF_KPTR_REF = (1 << 3), 178 + BPF_KPTR = BPF_KPTR_UNREF | BPF_KPTR_REF, 178 179 }; 179 180 180 - struct bpf_map_value_off_desc { 181 + struct btf_field_kptr { 182 + struct btf *btf; 183 + struct module *module; 184 + btf_dtor_kfunc_t dtor; 185 + u32 btf_id; 186 + }; 187 + 188 + struct btf_field { 181 189 u32 offset; 182 - enum bpf_kptr_type type; 183 - struct { 184 - struct btf *btf; 185 - struct module *module; 186 - btf_dtor_kfunc_t dtor; 187 - u32 btf_id; 188 - } kptr; 190 + enum btf_field_type type; 191 + union { 192 + struct btf_field_kptr kptr; 193 + }; 189 194 }; 190 195 191 - struct bpf_map_value_off { 192 - u32 nr_off; 193 - struct bpf_map_value_off_desc off[]; 196 + struct btf_record { 197 + u32 cnt; 198 + u32 field_mask; 199 + struct btf_field fields[]; 194 200 }; 195 201 196 - struct bpf_map_off_arr { 202 + struct btf_field_offs { 197 203 u32 cnt; 198 204 u32 field_off[BPF_MAP_OFF_ARR_MAX]; 199 205 u8 field_sz[BPF_MAP_OFF_ARR_MAX]; ··· 221 215 u64 map_extra; /* any per-map-type extra fields */ 222 216 u32 map_flags; 223 217 int spin_lock_off; /* >=0 valid offset, <0 error */ 224 - struct bpf_map_value_off *kptr_off_tab; 218 + struct btf_record *record; 225 219 int timer_off; /* >=0 valid offset, <0 error */ 226 220 u32 id; 227 221 int numa_node; ··· 233 227 struct obj_cgroup *objcg; 234 228 #endif 235 229 char name[BPF_OBJ_NAME_LEN]; 236 - struct bpf_map_off_arr *off_arr; 230 + struct btf_field_offs *field_offs; 237 231 /* The 3rd and 4th cacheline with misc members to avoid false sharing 238 232 * particularly with refcounting. 239 233 */ ··· 257 251 bool frozen; /* write-once; write-protected by freeze_mutex */ 258 252 }; 259 253 254 + static inline u32 btf_field_type_size(enum btf_field_type type) 255 + { 256 + switch (type) { 257 + case BPF_KPTR_UNREF: 258 + case BPF_KPTR_REF: 259 + return sizeof(u64); 260 + default: 261 + WARN_ON_ONCE(1); 262 + return 0; 263 + } 264 + } 265 + 266 + static inline u32 btf_field_type_align(enum btf_field_type type) 267 + { 268 + switch (type) { 269 + case BPF_KPTR_UNREF: 270 + case BPF_KPTR_REF: 271 + return __alignof__(u64); 272 + default: 273 + WARN_ON_ONCE(1); 274 + return 0; 275 + } 276 + } 277 + 278 + static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type) 279 + { 280 + if (IS_ERR_OR_NULL(rec)) 281 + return false; 282 + return rec->field_mask & type; 283 + } 284 + 260 285 static inline bool map_value_has_spin_lock(const struct bpf_map *map) 261 286 { 262 287 return map->spin_lock_off >= 0; ··· 298 261 return map->timer_off >= 0; 299 262 } 300 263 301 - static inline bool map_value_has_kptrs(const struct bpf_map *map) 302 - { 303 - return !IS_ERR_OR_NULL(map->kptr_off_tab); 304 - } 305 - 306 264 static inline void check_and_init_map_value(struct bpf_map *map, void *dst) 307 265 { 308 266 if (unlikely(map_value_has_spin_lock(map))) 309 267 memset(dst + map->spin_lock_off, 0, sizeof(struct bpf_spin_lock)); 310 268 if (unlikely(map_value_has_timer(map))) 311 269 memset(dst + map->timer_off, 0, sizeof(struct bpf_timer)); 312 - if (unlikely(map_value_has_kptrs(map))) { 313 - struct bpf_map_value_off *tab = map->kptr_off_tab; 270 + if (!IS_ERR_OR_NULL(map->record)) { 271 + struct btf_field *fields = map->record->fields; 272 + u32 cnt = map->record->cnt; 314 273 int i; 315 274 316 - for (i = 0; i < tab->nr_off; i++) 317 - *(u64 *)(dst + tab->off[i].offset) = 0; 275 + for (i = 0; i < cnt; i++) 276 + memset(dst + fields[i].offset, 0, btf_field_type_size(fields[i].type)); 318 277 } 319 278 } 320 279 ··· 336 303 u32 curr_off = 0; 337 304 int i; 338 305 339 - if (likely(!map->off_arr)) { 306 + if (likely(!map->field_offs)) { 340 307 if (long_memcpy) 341 308 bpf_long_memcpy(dst, src, round_up(map->value_size, 8)); 342 309 else ··· 344 311 return; 345 312 } 346 313 347 - for (i = 0; i < map->off_arr->cnt; i++) { 348 - u32 next_off = map->off_arr->field_off[i]; 314 + for (i = 0; i < map->field_offs->cnt; i++) { 315 + u32 next_off = map->field_offs->field_off[i]; 316 + u32 sz = next_off - curr_off; 349 317 350 - memcpy(dst + curr_off, src + curr_off, next_off - curr_off); 351 - curr_off += map->off_arr->field_sz[i]; 318 + memcpy(dst + curr_off, src + curr_off, sz); 319 + curr_off += map->field_offs->field_sz[i]; 352 320 } 353 321 memcpy(dst + curr_off, src + curr_off, map->value_size - curr_off); 354 322 } ··· 369 335 u32 curr_off = 0; 370 336 int i; 371 337 372 - if (likely(!map->off_arr)) { 338 + if (likely(!map->field_offs)) { 373 339 memset(dst, 0, map->value_size); 374 340 return; 375 341 } 376 342 377 - for (i = 0; i < map->off_arr->cnt; i++) { 378 - u32 next_off = map->off_arr->field_off[i]; 343 + for (i = 0; i < map->field_offs->cnt; i++) { 344 + u32 next_off = map->field_offs->field_off[i]; 345 + u32 sz = next_off - curr_off; 379 346 380 - memset(dst + curr_off, 0, next_off - curr_off); 381 - curr_off += map->off_arr->field_sz[i]; 347 + memset(dst + curr_off, 0, sz); 348 + curr_off += map->field_offs->field_sz[i]; 382 349 } 383 350 memset(dst + curr_off, 0, map->value_size - curr_off); 384 351 } ··· 1734 1699 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); 1735 1700 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); 1736 1701 1737 - struct bpf_map_value_off_desc *bpf_map_kptr_off_contains(struct bpf_map *map, u32 offset); 1738 - void bpf_map_free_kptr_off_tab(struct bpf_map *map); 1739 - struct bpf_map_value_off *bpf_map_copy_kptr_off_tab(const struct bpf_map *map); 1740 - bool bpf_map_equal_kptr_off_tab(const struct bpf_map *map_a, const struct bpf_map *map_b); 1741 - void bpf_map_free_kptrs(struct bpf_map *map, void *map_value); 1702 + struct btf_field *btf_record_find(const struct btf_record *rec, 1703 + u32 offset, enum btf_field_type type); 1704 + void btf_record_free(struct btf_record *rec); 1705 + void bpf_map_free_record(struct bpf_map *map); 1706 + struct btf_record *btf_record_dup(const struct btf_record *rec); 1707 + bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b); 1708 + void bpf_obj_free_fields(const struct btf_record *rec, void *obj); 1742 1709 1743 1710 struct bpf_map *bpf_map_get(u32 ufd); 1744 1711 struct bpf_map *bpf_map_get_with_uref(u32 ufd);

+1 -2

include/linux/btf.h

··· 163 163 u32 expected_offset, u32 expected_size); 164 164 int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t); 165 165 int btf_find_timer(const struct btf *btf, const struct btf_type *t); 166 - struct bpf_map_value_off *btf_parse_kptrs(const struct btf *btf, 167 - const struct btf_type *t); 166 + struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type *t); 168 167 bool btf_type_is_void(const struct btf_type *t); 169 168 s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); 170 169 const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,

+6 -7

kernel/bpf/arraymap.c

··· 310 310 { 311 311 if (map_value_has_timer(&arr->map)) 312 312 bpf_timer_cancel_and_free(val + arr->map.timer_off); 313 - if (map_value_has_kptrs(&arr->map)) 314 - bpf_map_free_kptrs(&arr->map, val); 313 + bpf_obj_free_fields(arr->map.record, val); 315 314 } 316 315 317 316 /* Called from syscall or from eBPF program */ ··· 408 409 struct bpf_array *array = container_of(map, struct bpf_array, map); 409 410 int i; 410 411 411 - /* We don't reset or free kptr on uref dropping to zero. */ 412 + /* We don't reset or free fields other than timer on uref dropping to zero. */ 412 413 if (!map_value_has_timer(map)) 413 414 return; 414 415 ··· 422 423 struct bpf_array *array = container_of(map, struct bpf_array, map); 423 424 int i; 424 425 425 - if (map_value_has_kptrs(map)) { 426 + if (!IS_ERR_OR_NULL(map->record)) { 426 427 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 427 428 for (i = 0; i < array->map.max_entries; i++) { 428 429 void __percpu *pptr = array->pptrs[i & array->index_mask]; 429 430 int cpu; 430 431 431 432 for_each_possible_cpu(cpu) { 432 - bpf_map_free_kptrs(map, per_cpu_ptr(pptr, cpu)); 433 + bpf_obj_free_fields(map->record, per_cpu_ptr(pptr, cpu)); 433 434 cond_resched(); 434 435 } 435 436 } 436 437 } else { 437 438 for (i = 0; i < array->map.max_entries; i++) 438 - bpf_map_free_kptrs(map, array_map_elem_ptr(array, i)); 439 + bpf_obj_free_fields(map->record, array_map_elem_ptr(array, i)); 439 440 } 440 - bpf_map_free_kptr_off_tab(map); 441 + bpf_map_free_record(map); 441 442 } 442 443 443 444 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)

+31 -36

kernel/bpf/btf.c

··· 3191 3191 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); 3192 3192 } 3193 3193 3194 - enum btf_field_type { 3194 + enum btf_field_info_type { 3195 3195 BTF_FIELD_SPIN_LOCK, 3196 3196 BTF_FIELD_TIMER, 3197 3197 BTF_FIELD_KPTR, ··· 3203 3203 }; 3204 3204 3205 3205 struct btf_field_info { 3206 - u32 type_id; 3206 + enum btf_field_type type; 3207 3207 u32 off; 3208 - enum bpf_kptr_type type; 3208 + u32 type_id; 3209 3209 }; 3210 3210 3211 3211 static int btf_find_struct(const struct btf *btf, const struct btf_type *t, ··· 3222 3222 static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, 3223 3223 u32 off, int sz, struct btf_field_info *info) 3224 3224 { 3225 - enum bpf_kptr_type type; 3225 + enum btf_field_type type; 3226 3226 u32 res_id; 3227 3227 3228 3228 /* Permit modifiers on the pointer itself */ ··· 3259 3259 3260 3260 static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, 3261 3261 const char *name, int sz, int align, 3262 - enum btf_field_type field_type, 3262 + enum btf_field_info_type field_type, 3263 3263 struct btf_field_info *info, int info_cnt) 3264 3264 { 3265 3265 const struct btf_member *member; ··· 3311 3311 3312 3312 static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, 3313 3313 const char *name, int sz, int align, 3314 - enum btf_field_type field_type, 3314 + enum btf_field_info_type field_type, 3315 3315 struct btf_field_info *info, int info_cnt) 3316 3316 { 3317 3317 const struct btf_var_secinfo *vsi; ··· 3360 3360 } 3361 3361 3362 3362 static int btf_find_field(const struct btf *btf, const struct btf_type *t, 3363 - enum btf_field_type field_type, 3363 + enum btf_field_info_type field_type, 3364 3364 struct btf_field_info *info, int info_cnt) 3365 3365 { 3366 3366 const char *name; ··· 3423 3423 return info.off; 3424 3424 } 3425 3425 3426 - struct bpf_map_value_off *btf_parse_kptrs(const struct btf *btf, 3427 - const struct btf_type *t) 3426 + struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type *t) 3428 3427 { 3429 - struct btf_field_info info_arr[BPF_MAP_VALUE_OFF_MAX]; 3430 - struct bpf_map_value_off *tab; 3428 + struct btf_field_info info_arr[BTF_FIELDS_MAX]; 3431 3429 struct btf *kernel_btf = NULL; 3432 3430 struct module *mod = NULL; 3433 - int ret, i, nr_off; 3431 + struct btf_record *rec; 3432 + int ret, i, cnt; 3434 3433 3435 3434 ret = btf_find_field(btf, t, BTF_FIELD_KPTR, info_arr, ARRAY_SIZE(info_arr)); 3436 3435 if (ret < 0) ··· 3437 3438 if (!ret) 3438 3439 return NULL; 3439 3440 3440 - nr_off = ret; 3441 - tab = kzalloc(offsetof(struct bpf_map_value_off, off[nr_off]), GFP_KERNEL | __GFP_NOWARN); 3442 - if (!tab) 3441 + cnt = ret; 3442 + rec = kzalloc(offsetof(struct btf_record, fields[cnt]), GFP_KERNEL | __GFP_NOWARN); 3443 + if (!rec) 3443 3444 return ERR_PTR(-ENOMEM); 3444 - 3445 - for (i = 0; i < nr_off; i++) { 3445 + rec->cnt = 0; 3446 + for (i = 0; i < cnt; i++) { 3446 3447 const struct btf_type *t; 3447 3448 s32 id; 3448 3449 ··· 3499 3500 ret = -EINVAL; 3500 3501 goto end_mod; 3501 3502 } 3502 - tab->off[i].kptr.dtor = (void *)addr; 3503 + rec->fields[i].kptr.dtor = (void *)addr; 3503 3504 } 3504 3505 3505 - tab->off[i].offset = info_arr[i].off; 3506 - tab->off[i].type = info_arr[i].type; 3507 - tab->off[i].kptr.btf_id = id; 3508 - tab->off[i].kptr.btf = kernel_btf; 3509 - tab->off[i].kptr.module = mod; 3506 + rec->field_mask |= info_arr[i].type; 3507 + rec->fields[i].offset = info_arr[i].off; 3508 + rec->fields[i].type = info_arr[i].type; 3509 + rec->fields[i].kptr.btf_id = id; 3510 + rec->fields[i].kptr.btf = kernel_btf; 3511 + rec->fields[i].kptr.module = mod; 3512 + rec->cnt++; 3510 3513 } 3511 - tab->nr_off = nr_off; 3512 - return tab; 3514 + return rec; 3513 3515 end_mod: 3514 3516 module_put(mod); 3515 3517 end_btf: 3516 3518 btf_put(kernel_btf); 3517 3519 end: 3518 - while (i--) { 3519 - btf_put(tab->off[i].kptr.btf); 3520 - if (tab->off[i].kptr.module) 3521 - module_put(tab->off[i].kptr.module); 3522 - } 3523 - kfree(tab); 3520 + btf_record_free(rec); 3524 3521 return ERR_PTR(ret); 3525 3522 } 3526 3523 ··· 6365 6370 6366 6371 /* kptr_get is only true for kfunc */ 6367 6372 if (i == 0 && kptr_get) { 6368 - struct bpf_map_value_off_desc *off_desc; 6373 + struct btf_field *kptr_field; 6369 6374 6370 6375 if (reg->type != PTR_TO_MAP_VALUE) { 6371 6376 bpf_log(log, "arg#0 expected pointer to map value\n"); ··· 6381 6386 return -EINVAL; 6382 6387 } 6383 6388 6384 - off_desc = bpf_map_kptr_off_contains(reg->map_ptr, reg->off + reg->var_off.value); 6385 - if (!off_desc || off_desc->type != BPF_KPTR_REF) { 6389 + kptr_field = btf_record_find(reg->map_ptr->record, reg->off + reg->var_off.value, BPF_KPTR); 6390 + if (!kptr_field || kptr_field->type != BPF_KPTR_REF) { 6386 6391 bpf_log(log, "arg#0 no referenced kptr at map value offset=%llu\n", 6387 6392 reg->off + reg->var_off.value); 6388 6393 return -EINVAL; ··· 6401 6406 func_name, i, btf_type_str(ref_t), ref_tname); 6402 6407 return -EINVAL; 6403 6408 } 6404 - if (!btf_struct_ids_match(log, btf, ref_id, 0, off_desc->kptr.btf, 6405 - off_desc->kptr.btf_id, true)) { 6409 + if (!btf_struct_ids_match(log, btf, ref_id, 0, kptr_field->kptr.btf, 6410 + kptr_field->kptr.btf_id, true)) { 6406 6411 bpf_log(log, "kernel function %s args#%d expected pointer to %s %s\n", 6407 6412 func_name, i, btf_type_str(ref_t), ref_tname); 6408 6413 return -EINVAL;

+6 -8

kernel/bpf/hashtab.c

··· 238 238 } 239 239 } 240 240 241 - static void htab_free_prealloced_kptrs(struct bpf_htab *htab) 241 + static void htab_free_prealloced_fields(struct bpf_htab *htab) 242 242 { 243 243 u32 num_entries = htab->map.max_entries; 244 244 int i; 245 245 246 - if (!map_value_has_kptrs(&htab->map)) 246 + if (IS_ERR_OR_NULL(htab->map.record)) 247 247 return; 248 248 if (htab_has_extra_elems(htab)) 249 249 num_entries += num_possible_cpus(); 250 - 251 250 for (i = 0; i < num_entries; i++) { 252 251 struct htab_elem *elem; 253 252 254 253 elem = get_htab_elem(htab, i); 255 - bpf_map_free_kptrs(&htab->map, elem->key + round_up(htab->map.key_size, 8)); 254 + bpf_obj_free_fields(htab->map.record, elem->key + round_up(htab->map.key_size, 8)); 256 255 cond_resched(); 257 256 } 258 257 } ··· 765 766 766 767 if (map_value_has_timer(&htab->map)) 767 768 bpf_timer_cancel_and_free(map_value + htab->map.timer_off); 768 - if (map_value_has_kptrs(&htab->map)) 769 - bpf_map_free_kptrs(&htab->map, map_value); 769 + bpf_obj_free_fields(htab->map.record, map_value); 770 770 } 771 771 772 772 /* It is called from the bpf_lru_list when the LRU needs to delete ··· 1515 1517 if (!htab_is_prealloc(htab)) { 1516 1518 delete_all_elements(htab); 1517 1519 } else { 1518 - htab_free_prealloced_kptrs(htab); 1520 + htab_free_prealloced_fields(htab); 1519 1521 prealloc_destroy(htab); 1520 1522 } 1521 1523 1522 - bpf_map_free_kptr_off_tab(map); 1524 + bpf_map_free_record(map); 1523 1525 free_percpu(htab->extra_elems); 1524 1526 bpf_map_area_free(htab->buckets); 1525 1527 bpf_mem_alloc_destroy(&htab->pcpu_ma);

+11 -3

kernel/bpf/map_in_map.c

··· 52 52 inner_map_meta->max_entries = inner_map->max_entries; 53 53 inner_map_meta->spin_lock_off = inner_map->spin_lock_off; 54 54 inner_map_meta->timer_off = inner_map->timer_off; 55 - inner_map_meta->kptr_off_tab = bpf_map_copy_kptr_off_tab(inner_map); 55 + inner_map_meta->record = btf_record_dup(inner_map->record); 56 + if (IS_ERR(inner_map_meta->record)) { 57 + /* btf_record_dup returns NULL or valid pointer in case of 58 + * invalid/empty/valid, but ERR_PTR in case of errors. During 59 + * equality NULL or IS_ERR is equivalent. 60 + */ 61 + fdput(f); 62 + return ERR_CAST(inner_map_meta->record); 63 + } 56 64 if (inner_map->btf) { 57 65 btf_get(inner_map->btf); 58 66 inner_map_meta->btf = inner_map->btf; ··· 80 72 81 73 void bpf_map_meta_free(struct bpf_map *map_meta) 82 74 { 83 - bpf_map_free_kptr_off_tab(map_meta); 75 + bpf_map_free_record(map_meta); 84 76 btf_put(map_meta->btf); 85 77 kfree(map_meta); 86 78 } ··· 94 86 meta0->value_size == meta1->value_size && 95 87 meta0->timer_off == meta1->timer_off && 96 88 meta0->map_flags == meta1->map_flags && 97 - bpf_map_equal_kptr_off_tab(meta0, meta1); 89 + btf_record_equal(meta0->record, meta1->record); 98 90 } 99 91 100 92 void *bpf_map_fd_get_ptr(struct bpf_map *map,

+171 -136

kernel/bpf/syscall.c

··· 495 495 } 496 496 #endif 497 497 498 - static int bpf_map_kptr_off_cmp(const void *a, const void *b) 498 + static int btf_field_cmp(const void *a, const void *b) 499 499 { 500 - const struct bpf_map_value_off_desc *off_desc1 = a, *off_desc2 = b; 500 + const struct btf_field *f1 = a, *f2 = b; 501 501 502 - if (off_desc1->offset < off_desc2->offset) 502 + if (f1->offset < f2->offset) 503 503 return -1; 504 - else if (off_desc1->offset > off_desc2->offset) 504 + else if (f1->offset > f2->offset) 505 505 return 1; 506 506 return 0; 507 507 } 508 508 509 - struct bpf_map_value_off_desc *bpf_map_kptr_off_contains(struct bpf_map *map, u32 offset) 509 + struct btf_field *btf_record_find(const struct btf_record *rec, u32 offset, 510 + enum btf_field_type type) 510 511 { 511 - /* Since members are iterated in btf_find_field in increasing order, 512 - * offsets appended to kptr_off_tab are in increasing order, so we can 513 - * do bsearch to find exact match. 514 - */ 515 - struct bpf_map_value_off *tab; 512 + struct btf_field *field; 516 513 517 - if (!map_value_has_kptrs(map)) 514 + if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & type)) 518 515 return NULL; 519 - tab = map->kptr_off_tab; 520 - return bsearch(&offset, tab->off, tab->nr_off, sizeof(tab->off[0]), bpf_map_kptr_off_cmp); 516 + field = bsearch(&offset, rec->fields, rec->cnt, sizeof(rec->fields[0]), btf_field_cmp); 517 + if (!field || !(field->type & type)) 518 + return NULL; 519 + return field; 521 520 } 522 521 523 - void bpf_map_free_kptr_off_tab(struct bpf_map *map) 522 + void btf_record_free(struct btf_record *rec) 524 523 { 525 - struct bpf_map_value_off *tab = map->kptr_off_tab; 526 524 int i; 527 525 528 - if (!map_value_has_kptrs(map)) 526 + if (IS_ERR_OR_NULL(rec)) 529 527 return; 530 - for (i = 0; i < tab->nr_off; i++) { 531 - if (tab->off[i].kptr.module) 532 - module_put(tab->off[i].kptr.module); 533 - btf_put(tab->off[i].kptr.btf); 534 - } 535 - kfree(tab); 536 - map->kptr_off_tab = NULL; 537 - } 538 - 539 - struct bpf_map_value_off *bpf_map_copy_kptr_off_tab(const struct bpf_map *map) 540 - { 541 - struct bpf_map_value_off *tab = map->kptr_off_tab, *new_tab; 542 - int size, i; 543 - 544 - if (!map_value_has_kptrs(map)) 545 - return ERR_PTR(-ENOENT); 546 - size = offsetof(struct bpf_map_value_off, off[tab->nr_off]); 547 - new_tab = kmemdup(tab, size, GFP_KERNEL | __GFP_NOWARN); 548 - if (!new_tab) 549 - return ERR_PTR(-ENOMEM); 550 - /* Do a deep copy of the kptr_off_tab */ 551 - for (i = 0; i < tab->nr_off; i++) { 552 - btf_get(tab->off[i].kptr.btf); 553 - if (tab->off[i].kptr.module && !try_module_get(tab->off[i].kptr.module)) { 554 - while (i--) { 555 - if (tab->off[i].kptr.module) 556 - module_put(tab->off[i].kptr.module); 557 - btf_put(tab->off[i].kptr.btf); 558 - } 559 - kfree(new_tab); 560 - return ERR_PTR(-ENXIO); 561 - } 562 - } 563 - return new_tab; 564 - } 565 - 566 - bool bpf_map_equal_kptr_off_tab(const struct bpf_map *map_a, const struct bpf_map *map_b) 567 - { 568 - struct bpf_map_value_off *tab_a = map_a->kptr_off_tab, *tab_b = map_b->kptr_off_tab; 569 - bool a_has_kptr = map_value_has_kptrs(map_a), b_has_kptr = map_value_has_kptrs(map_b); 570 - int size; 571 - 572 - if (!a_has_kptr && !b_has_kptr) 573 - return true; 574 - if (a_has_kptr != b_has_kptr) 575 - return false; 576 - if (tab_a->nr_off != tab_b->nr_off) 577 - return false; 578 - size = offsetof(struct bpf_map_value_off, off[tab_a->nr_off]); 579 - return !memcmp(tab_a, tab_b, size); 580 - } 581 - 582 - /* Caller must ensure map_value_has_kptrs is true. Note that this function can 583 - * be called on a map value while the map_value is visible to BPF programs, as 584 - * it ensures the correct synchronization, and we already enforce the same using 585 - * the bpf_kptr_xchg helper on the BPF program side for referenced kptrs. 586 - */ 587 - void bpf_map_free_kptrs(struct bpf_map *map, void *map_value) 588 - { 589 - struct bpf_map_value_off *tab = map->kptr_off_tab; 590 - unsigned long *btf_id_ptr; 591 - int i; 592 - 593 - for (i = 0; i < tab->nr_off; i++) { 594 - struct bpf_map_value_off_desc *off_desc = &tab->off[i]; 595 - unsigned long old_ptr; 596 - 597 - btf_id_ptr = map_value + off_desc->offset; 598 - if (off_desc->type == BPF_KPTR_UNREF) { 599 - u64 *p = (u64 *)btf_id_ptr; 600 - 601 - WRITE_ONCE(*p, 0); 528 + for (i = 0; i < rec->cnt; i++) { 529 + switch (rec->fields[i].type) { 530 + case BPF_KPTR_UNREF: 531 + case BPF_KPTR_REF: 532 + if (rec->fields[i].kptr.module) 533 + module_put(rec->fields[i].kptr.module); 534 + btf_put(rec->fields[i].kptr.btf); 535 + break; 536 + default: 537 + WARN_ON_ONCE(1); 602 538 continue; 603 539 } 604 - old_ptr = xchg(btf_id_ptr, 0); 605 - off_desc->kptr.dtor((void *)old_ptr); 540 + } 541 + kfree(rec); 542 + } 543 + 544 + void bpf_map_free_record(struct bpf_map *map) 545 + { 546 + btf_record_free(map->record); 547 + map->record = NULL; 548 + } 549 + 550 + struct btf_record *btf_record_dup(const struct btf_record *rec) 551 + { 552 + const struct btf_field *fields; 553 + struct btf_record *new_rec; 554 + int ret, size, i; 555 + 556 + if (IS_ERR_OR_NULL(rec)) 557 + return NULL; 558 + size = offsetof(struct btf_record, fields[rec->cnt]); 559 + new_rec = kmemdup(rec, size, GFP_KERNEL | __GFP_NOWARN); 560 + if (!new_rec) 561 + return ERR_PTR(-ENOMEM); 562 + /* Do a deep copy of the btf_record */ 563 + fields = rec->fields; 564 + new_rec->cnt = 0; 565 + for (i = 0; i < rec->cnt; i++) { 566 + switch (fields[i].type) { 567 + case BPF_KPTR_UNREF: 568 + case BPF_KPTR_REF: 569 + btf_get(fields[i].kptr.btf); 570 + if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) { 571 + ret = -ENXIO; 572 + goto free; 573 + } 574 + break; 575 + default: 576 + ret = -EFAULT; 577 + WARN_ON_ONCE(1); 578 + goto free; 579 + } 580 + new_rec->cnt++; 581 + } 582 + return new_rec; 583 + free: 584 + btf_record_free(new_rec); 585 + return ERR_PTR(ret); 586 + } 587 + 588 + bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b) 589 + { 590 + bool a_has_fields = !IS_ERR_OR_NULL(rec_a), b_has_fields = !IS_ERR_OR_NULL(rec_b); 591 + int size; 592 + 593 + if (!a_has_fields && !b_has_fields) 594 + return true; 595 + if (a_has_fields != b_has_fields) 596 + return false; 597 + if (rec_a->cnt != rec_b->cnt) 598 + return false; 599 + size = offsetof(struct btf_record, fields[rec_a->cnt]); 600 + return !memcmp(rec_a, rec_b, size); 601 + } 602 + 603 + void bpf_obj_free_fields(const struct btf_record *rec, void *obj) 604 + { 605 + const struct btf_field *fields; 606 + int i; 607 + 608 + if (IS_ERR_OR_NULL(rec)) 609 + return; 610 + fields = rec->fields; 611 + for (i = 0; i < rec->cnt; i++) { 612 + const struct btf_field *field = &fields[i]; 613 + void *field_ptr = obj + field->offset; 614 + 615 + switch (fields[i].type) { 616 + case BPF_KPTR_UNREF: 617 + WRITE_ONCE(*(u64 *)field_ptr, 0); 618 + break; 619 + case BPF_KPTR_REF: 620 + field->kptr.dtor((void *)xchg((unsigned long *)field_ptr, 0)); 621 + break; 622 + default: 623 + WARN_ON_ONCE(1); 624 + continue; 625 + } 606 626 } 607 627 } 608 628 ··· 632 612 struct bpf_map *map = container_of(work, struct bpf_map, work); 633 613 634 614 security_bpf_map_free(map); 635 - kfree(map->off_arr); 615 + kfree(map->field_offs); 636 616 bpf_map_release_memcg(map); 637 617 /* implementation dependent freeing, map_free callback also does 638 - * bpf_map_free_kptr_off_tab, if needed. 618 + * bpf_map_free_record, if needed. 639 619 */ 640 620 map->ops->map_free(map); 641 621 } ··· 799 779 int err; 800 780 801 781 if (!map->ops->map_mmap || map_value_has_spin_lock(map) || 802 - map_value_has_timer(map) || map_value_has_kptrs(map)) 782 + map_value_has_timer(map) || !IS_ERR_OR_NULL(map->record)) 803 783 return -ENOTSUPP; 804 784 805 785 if (!(vma->vm_flags & VM_SHARED)) ··· 926 906 return -ENOTSUPP; 927 907 } 928 908 929 - static int map_off_arr_cmp(const void *_a, const void *_b, const void *priv) 909 + static int map_field_offs_cmp(const void *_a, const void *_b, const void *priv) 930 910 { 931 911 const u32 a = *(const u32 *)_a; 932 912 const u32 b = *(const u32 *)_b; ··· 938 918 return 0; 939 919 } 940 920 941 - static void map_off_arr_swap(void *_a, void *_b, int size, const void *priv) 921 + static void map_field_offs_swap(void *_a, void *_b, int size, const void *priv) 942 922 { 943 923 struct bpf_map *map = (struct bpf_map *)priv; 944 - u32 *off_base = map->off_arr->field_off; 924 + u32 *off_base = map->field_offs->field_off; 945 925 u32 *a = _a, *b = _b; 946 926 u8 *sz_a, *sz_b; 947 927 948 - sz_a = map->off_arr->field_sz + (a - off_base); 949 - sz_b = map->off_arr->field_sz + (b - off_base); 928 + sz_a = map->field_offs->field_sz + (a - off_base); 929 + sz_b = map->field_offs->field_sz + (b - off_base); 950 930 951 931 swap(*a, *b); 952 932 swap(*sz_a, *sz_b); ··· 956 936 { 957 937 bool has_spin_lock = map_value_has_spin_lock(map); 958 938 bool has_timer = map_value_has_timer(map); 959 - bool has_kptrs = map_value_has_kptrs(map); 960 - struct bpf_map_off_arr *off_arr; 939 + bool has_fields = !IS_ERR_OR_NULL(map->record); 940 + struct btf_field_offs *fo; 961 941 u32 i; 962 942 963 - if (!has_spin_lock && !has_timer && !has_kptrs) { 964 - map->off_arr = NULL; 943 + if (!has_spin_lock && !has_timer && !has_fields) { 944 + map->field_offs = NULL; 965 945 return 0; 966 946 } 967 947 968 - off_arr = kmalloc(sizeof(*map->off_arr), GFP_KERNEL | __GFP_NOWARN); 969 - if (!off_arr) 948 + fo = kmalloc(sizeof(*map->field_offs), GFP_KERNEL | __GFP_NOWARN); 949 + if (!fo) 970 950 return -ENOMEM; 971 - map->off_arr = off_arr; 951 + map->field_offs = fo; 972 952 973 - off_arr->cnt = 0; 953 + fo->cnt = 0; 974 954 if (has_spin_lock) { 975 - i = off_arr->cnt; 955 + i = fo->cnt; 976 956 977 - off_arr->field_off[i] = map->spin_lock_off; 978 - off_arr->field_sz[i] = sizeof(struct bpf_spin_lock); 979 - off_arr->cnt++; 957 + fo->field_off[i] = map->spin_lock_off; 958 + fo->field_sz[i] = sizeof(struct bpf_spin_lock); 959 + fo->cnt++; 980 960 } 981 961 if (has_timer) { 982 - i = off_arr->cnt; 962 + i = fo->cnt; 983 963 984 - off_arr->field_off[i] = map->timer_off; 985 - off_arr->field_sz[i] = sizeof(struct bpf_timer); 986 - off_arr->cnt++; 964 + fo->field_off[i] = map->timer_off; 965 + fo->field_sz[i] = sizeof(struct bpf_timer); 966 + fo->cnt++; 987 967 } 988 - if (has_kptrs) { 989 - struct bpf_map_value_off *tab = map->kptr_off_tab; 990 - u32 *off = &off_arr->field_off[off_arr->cnt]; 991 - u8 *sz = &off_arr->field_sz[off_arr->cnt]; 968 + if (has_fields) { 969 + struct btf_record *rec = map->record; 970 + u32 *off = &fo->field_off[fo->cnt]; 971 + u8 *sz = &fo->field_sz[fo->cnt]; 992 972 993 - for (i = 0; i < tab->nr_off; i++) { 994 - *off++ = tab->off[i].offset; 995 - *sz++ = sizeof(u64); 973 + for (i = 0; i < rec->cnt; i++) { 974 + *off++ = rec->fields[i].offset; 975 + *sz++ = btf_field_type_size(rec->fields[i].type); 996 976 } 997 - off_arr->cnt += tab->nr_off; 977 + fo->cnt += rec->cnt; 998 978 } 999 979 1000 - if (off_arr->cnt == 1) 980 + if (fo->cnt == 1) 1001 981 return 0; 1002 - sort_r(off_arr->field_off, off_arr->cnt, sizeof(off_arr->field_off[0]), 1003 - map_off_arr_cmp, map_off_arr_swap, map); 982 + sort_r(fo->field_off, fo->cnt, sizeof(fo->field_off[0]), 983 + map_field_offs_cmp, map_field_offs_swap, map); 1004 984 return 0; 1005 985 } 1006 986 ··· 1058 1038 return -EOPNOTSUPP; 1059 1039 } 1060 1040 1061 - map->kptr_off_tab = btf_parse_kptrs(btf, value_type); 1062 - if (map_value_has_kptrs(map)) { 1041 + map->record = btf_parse_fields(btf, value_type); 1042 + if (!IS_ERR_OR_NULL(map->record)) { 1043 + int i; 1044 + 1063 1045 if (!bpf_capable()) { 1064 1046 ret = -EPERM; 1065 1047 goto free_map_tab; ··· 1070 1048 ret = -EACCES; 1071 1049 goto free_map_tab; 1072 1050 } 1073 - if (map->map_type != BPF_MAP_TYPE_HASH && 1074 - map->map_type != BPF_MAP_TYPE_LRU_HASH && 1075 - map->map_type != BPF_MAP_TYPE_ARRAY && 1076 - map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) { 1077 - ret = -EOPNOTSUPP; 1078 - goto free_map_tab; 1051 + for (i = 0; i < sizeof(map->record->field_mask) * 8; i++) { 1052 + switch (map->record->field_mask & (1 << i)) { 1053 + case 0: 1054 + continue; 1055 + case BPF_KPTR_UNREF: 1056 + case BPF_KPTR_REF: 1057 + if (map->map_type != BPF_MAP_TYPE_HASH && 1058 + map->map_type != BPF_MAP_TYPE_LRU_HASH && 1059 + map->map_type != BPF_MAP_TYPE_ARRAY && 1060 + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) { 1061 + ret = -EOPNOTSUPP; 1062 + goto free_map_tab; 1063 + } 1064 + break; 1065 + default: 1066 + /* Fail if map_type checks are missing for a field type */ 1067 + ret = -EOPNOTSUPP; 1068 + goto free_map_tab; 1069 + } 1079 1070 } 1080 1071 } 1081 1072 ··· 1100 1065 1101 1066 return ret; 1102 1067 free_map_tab: 1103 - bpf_map_free_kptr_off_tab(map); 1068 + bpf_map_free_record(map); 1104 1069 return ret; 1105 1070 } 1106 1071 ··· 1221 1186 free_map_sec: 1222 1187 security_bpf_map_free(map); 1223 1188 free_map_off_arr: 1224 - kfree(map->off_arr); 1189 + kfree(map->field_offs); 1225 1190 free_map: 1226 1191 btf_put(map->btf); 1227 1192 map->ops->map_free(map); ··· 1918 1883 return PTR_ERR(map); 1919 1884 1920 1885 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS || 1921 - map_value_has_timer(map) || map_value_has_kptrs(map)) { 1886 + map_value_has_timer(map) || !IS_ERR_OR_NULL(map->record)) { 1922 1887 fdput(f); 1923 1888 return -ENOTSUPP; 1924 1889 }

+46 -48

kernel/bpf/verifier.c

··· 262 262 struct btf *ret_btf; 263 263 u32 ret_btf_id; 264 264 u32 subprogno; 265 - struct bpf_map_value_off_desc *kptr_off_desc; 265 + struct btf_field *kptr_field; 266 266 u8 uninit_dynptr_regno; 267 267 }; 268 268 ··· 3674 3674 } 3675 3675 3676 3676 static int map_kptr_match_type(struct bpf_verifier_env *env, 3677 - struct bpf_map_value_off_desc *off_desc, 3677 + struct btf_field *kptr_field, 3678 3678 struct bpf_reg_state *reg, u32 regno) 3679 3679 { 3680 - const char *targ_name = kernel_type_name(off_desc->kptr.btf, off_desc->kptr.btf_id); 3680 + const char *targ_name = kernel_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id); 3681 3681 int perm_flags = PTR_MAYBE_NULL; 3682 3682 const char *reg_name = ""; 3683 3683 3684 3684 /* Only unreferenced case accepts untrusted pointers */ 3685 - if (off_desc->type == BPF_KPTR_UNREF) 3685 + if (kptr_field->type == BPF_KPTR_UNREF) 3686 3686 perm_flags |= PTR_UNTRUSTED; 3687 3687 3688 3688 if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags)) ··· 3729 3729 * strict mode to true for type match. 3730 3730 */ 3731 3731 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 3732 - off_desc->kptr.btf, off_desc->kptr.btf_id, 3733 - off_desc->type == BPF_KPTR_REF)) 3732 + kptr_field->kptr.btf, kptr_field->kptr.btf_id, 3733 + kptr_field->type == BPF_KPTR_REF)) 3734 3734 goto bad_type; 3735 3735 return 0; 3736 3736 bad_type: 3737 3737 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, 3738 3738 reg_type_str(env, reg->type), reg_name); 3739 3739 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); 3740 - if (off_desc->type == BPF_KPTR_UNREF) 3740 + if (kptr_field->type == BPF_KPTR_UNREF) 3741 3741 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), 3742 3742 targ_name); 3743 3743 else ··· 3747 3747 3748 3748 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, 3749 3749 int value_regno, int insn_idx, 3750 - struct bpf_map_value_off_desc *off_desc) 3750 + struct btf_field *kptr_field) 3751 3751 { 3752 3752 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 3753 3753 int class = BPF_CLASS(insn->code); ··· 3757 3757 * - Reject cases where variable offset may touch kptr 3758 3758 * - size of access (must be BPF_DW) 3759 3759 * - tnum_is_const(reg->var_off) 3760 - * - off_desc->offset == off + reg->var_off.value 3760 + * - kptr_field->offset == off + reg->var_off.value 3761 3761 */ 3762 3762 /* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */ 3763 3763 if (BPF_MODE(insn->code) != BPF_MEM) { ··· 3768 3768 /* We only allow loading referenced kptr, since it will be marked as 3769 3769 * untrusted, similar to unreferenced kptr. 3770 3770 */ 3771 - if (class != BPF_LDX && off_desc->type == BPF_KPTR_REF) { 3771 + if (class != BPF_LDX && kptr_field->type == BPF_KPTR_REF) { 3772 3772 verbose(env, "store to referenced kptr disallowed\n"); 3773 3773 return -EACCES; 3774 3774 } ··· 3778 3778 /* We can simply mark the value_regno receiving the pointer 3779 3779 * value from map as PTR_TO_BTF_ID, with the correct type. 3780 3780 */ 3781 - mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, off_desc->kptr.btf, 3782 - off_desc->kptr.btf_id, PTR_MAYBE_NULL | PTR_UNTRUSTED); 3781 + mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf, 3782 + kptr_field->kptr.btf_id, PTR_MAYBE_NULL | PTR_UNTRUSTED); 3783 3783 /* For mark_ptr_or_null_reg */ 3784 3784 val_reg->id = ++env->id_gen; 3785 3785 } else if (class == BPF_STX) { 3786 3786 val_reg = reg_state(env, value_regno); 3787 3787 if (!register_is_null(val_reg) && 3788 - map_kptr_match_type(env, off_desc, val_reg, value_regno)) 3788 + map_kptr_match_type(env, kptr_field, val_reg, value_regno)) 3789 3789 return -EACCES; 3790 3790 } else if (class == BPF_ST) { 3791 3791 if (insn->imm) { 3792 3792 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", 3793 - off_desc->offset); 3793 + kptr_field->offset); 3794 3794 return -EACCES; 3795 3795 } 3796 3796 } else { ··· 3809 3809 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3810 3810 struct bpf_reg_state *reg = &state->regs[regno]; 3811 3811 struct bpf_map *map = reg->map_ptr; 3812 - int err; 3812 + struct btf_record *rec; 3813 + int err, i; 3813 3814 3814 3815 err = check_mem_region_access(env, regno, off, size, map->value_size, 3815 3816 zero_size_allowed); ··· 3840 3839 return -EACCES; 3841 3840 } 3842 3841 } 3843 - if (map_value_has_kptrs(map)) { 3844 - struct bpf_map_value_off *tab = map->kptr_off_tab; 3845 - int i; 3842 + if (IS_ERR_OR_NULL(map->record)) 3843 + return 0; 3844 + rec = map->record; 3845 + for (i = 0; i < rec->cnt; i++) { 3846 + struct btf_field *field = &rec->fields[i]; 3847 + u32 p = field->offset; 3846 3848 3847 - for (i = 0; i < tab->nr_off; i++) { 3848 - u32 p = tab->off[i].offset; 3849 - 3850 - if (reg->smin_value + off < p + sizeof(u64) && 3851 - p < reg->umax_value + off + size) { 3849 + if (reg->smin_value + off < p + btf_field_type_size(field->type) && 3850 + p < reg->umax_value + off + size) { 3851 + switch (field->type) { 3852 + case BPF_KPTR_UNREF: 3853 + case BPF_KPTR_REF: 3852 3854 if (src != ACCESS_DIRECT) { 3853 3855 verbose(env, "kptr cannot be accessed indirectly by helper\n"); 3854 3856 return -EACCES; ··· 3870 3866 return -EACCES; 3871 3867 } 3872 3868 break; 3869 + default: 3870 + verbose(env, "field cannot be accessed directly by load/store\n"); 3871 + return -EACCES; 3873 3872 } 3874 3873 } 3875 3874 } 3876 - return err; 3875 + return 0; 3877 3876 } 3878 3877 3879 3878 #define MAX_PACKET_OFF 0xffff ··· 4749 4742 if (value_regno >= 0) 4750 4743 mark_reg_unknown(env, regs, value_regno); 4751 4744 } else if (reg->type == PTR_TO_MAP_VALUE) { 4752 - struct bpf_map_value_off_desc *kptr_off_desc = NULL; 4745 + struct btf_field *kptr_field = NULL; 4753 4746 4754 4747 if (t == BPF_WRITE && value_regno >= 0 && 4755 4748 is_pointer_value(env, value_regno)) { ··· 4763 4756 if (err) 4764 4757 return err; 4765 4758 if (tnum_is_const(reg->var_off)) 4766 - kptr_off_desc = bpf_map_kptr_off_contains(reg->map_ptr, 4767 - off + reg->var_off.value); 4768 - if (kptr_off_desc) { 4769 - err = check_map_kptr_access(env, regno, value_regno, insn_idx, 4770 - kptr_off_desc); 4759 + kptr_field = btf_record_find(reg->map_ptr->record, 4760 + off + reg->var_off.value, BPF_KPTR); 4761 + if (kptr_field) { 4762 + err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field); 4771 4763 } else if (t == BPF_READ && value_regno >= 0) { 4772 4764 struct bpf_map *map = reg->map_ptr; 4773 4765 ··· 5533 5527 struct bpf_call_arg_meta *meta) 5534 5528 { 5535 5529 struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno]; 5536 - struct bpf_map_value_off_desc *off_desc; 5537 5530 struct bpf_map *map_ptr = reg->map_ptr; 5531 + struct btf_field *kptr_field; 5538 5532 u32 kptr_off; 5539 - int ret; 5540 5533 5541 5534 if (!tnum_is_const(reg->var_off)) { 5542 5535 verbose(env, ··· 5548 5543 map_ptr->name); 5549 5544 return -EINVAL; 5550 5545 } 5551 - if (!map_value_has_kptrs(map_ptr)) { 5552 - ret = PTR_ERR_OR_ZERO(map_ptr->kptr_off_tab); 5553 - if (ret == -E2BIG) 5554 - verbose(env, "map '%s' has more than %d kptr\n", map_ptr->name, 5555 - BPF_MAP_VALUE_OFF_MAX); 5556 - else if (ret == -EEXIST) 5557 - verbose(env, "map '%s' has repeating kptr BTF tags\n", map_ptr->name); 5558 - else 5559 - verbose(env, "map '%s' has no valid kptr\n", map_ptr->name); 5546 + if (!btf_record_has_field(map_ptr->record, BPF_KPTR)) { 5547 + verbose(env, "map '%s' has no valid kptr\n", map_ptr->name); 5560 5548 return -EINVAL; 5561 5549 } 5562 5550 5563 5551 meta->map_ptr = map_ptr; 5564 5552 kptr_off = reg->off + reg->var_off.value; 5565 - off_desc = bpf_map_kptr_off_contains(map_ptr, kptr_off); 5566 - if (!off_desc) { 5553 + kptr_field = btf_record_find(map_ptr->record, kptr_off, BPF_KPTR); 5554 + if (!kptr_field) { 5567 5555 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); 5568 5556 return -EACCES; 5569 5557 } 5570 - if (off_desc->type != BPF_KPTR_REF) { 5558 + if (kptr_field->type != BPF_KPTR_REF) { 5571 5559 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); 5572 5560 return -EACCES; 5573 5561 } 5574 - meta->kptr_off_desc = off_desc; 5562 + meta->kptr_field = kptr_field; 5575 5563 return 0; 5576 5564 } 5577 5565 ··· 5786 5788 } 5787 5789 5788 5790 if (meta->func_id == BPF_FUNC_kptr_xchg) { 5789 - if (map_kptr_match_type(env, meta->kptr_off_desc, reg, regno)) 5791 + if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) 5790 5792 return -EACCES; 5791 5793 } else { 5792 5794 if (arg_btf_id == BPF_PTR_POISON) { ··· 7534 7536 mark_reg_known_zero(env, regs, BPF_REG_0); 7535 7537 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 7536 7538 if (func_id == BPF_FUNC_kptr_xchg) { 7537 - ret_btf = meta.kptr_off_desc->kptr.btf; 7538 - ret_btf_id = meta.kptr_off_desc->kptr.btf_id; 7539 + ret_btf = meta.kptr_field->kptr.btf; 7540 + ret_btf_id = meta.kptr_field->kptr.btf_id; 7539 7541 } else { 7540 7542 if (fn->ret_btf_id == BPF_PTR_POISON) { 7541 7543 verbose(env, "verifier internal error:");

Configure Feed

Configure Feed