bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_array maps

Introduce support for the BPF_F_ALL_CPUS flag in percpu_array maps to
allow updating values for all CPUs with a single value for both
update_elem and update_batch APIs.

Introduce support for the BPF_F_CPU flag in percpu_array maps to allow:

* update value for specified CPU for both update_elem and update_batch
APIs.
* lookup value for specified CPU for both lookup_elem and lookup_batch
APIs.

The BPF_F_CPU flag is passed via:

* map_flags of lookup_elem and update_elem APIs along with embedded cpu
info.
* elem_flags of lookup_batch and update_batch APIs along with embedded
cpu info.

Signed-off-by: Leon Hwang <leon.hwang@linux.dev>
Link: https://lore.kernel.org/r/20260107022022.12843-3-leon.hwang@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

authored by

Leon Hwang and committed by

Alexei Starovoitov 5 months ago 8eb76cb0 2b421662

+32 -10

3 changed files

expand all

include

linux

bpf.h

kernel

bpf

arraymap.c

syscall.c

+7 -2

include/linux/bpf.h

··· 2848 2848 struct bpf_func_state *callee); 2849 2849 2850 2850 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); 2851 - int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); 2851 + int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 flags); 2852 2852 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, 2853 2853 u64 flags); 2854 2854 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, ··· 3917 3917 3918 3918 static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type) 3919 3919 { 3920 - return false; 3920 + switch (map_type) { 3921 + case BPF_MAP_TYPE_PERCPU_ARRAY: 3922 + return true; 3923 + default: 3924 + return false; 3925 + } 3921 3926 } 3922 3927 3923 3928 static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags)

+24 -7

kernel/bpf/arraymap.c

··· 307 307 return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu); 308 308 } 309 309 310 - int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) 310 + int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 map_flags) 311 311 { 312 312 struct bpf_array *array = container_of(map, struct bpf_array, map); 313 313 u32 index = *(u32 *)key; ··· 325 325 size = array->elem_size; 326 326 rcu_read_lock(); 327 327 pptr = array->pptrs[index & array->index_mask]; 328 + if (map_flags & BPF_F_CPU) { 329 + cpu = map_flags >> 32; 330 + copy_map_value(map, value, per_cpu_ptr(pptr, cpu)); 331 + check_and_init_map_value(map, value); 332 + goto unlock; 333 + } 328 334 for_each_possible_cpu(cpu) { 329 335 copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu)); 330 336 check_and_init_map_value(map, value + off); 331 337 off += size; 332 338 } 339 + unlock: 333 340 rcu_read_unlock(); 334 341 return 0; 335 342 } ··· 405 398 struct bpf_array *array = container_of(map, struct bpf_array, map); 406 399 u32 index = *(u32 *)key; 407 400 void __percpu *pptr; 408 - int cpu, off = 0; 401 + void *ptr, *val; 409 402 u32 size; 403 + int cpu; 410 404 411 - if (unlikely(map_flags > BPF_EXIST)) 405 + if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS)) 412 406 /* unknown flags */ 413 407 return -EINVAL; 414 408 ··· 430 422 size = array->elem_size; 431 423 rcu_read_lock(); 432 424 pptr = array->pptrs[index & array->index_mask]; 433 - for_each_possible_cpu(cpu) { 434 - copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off); 435 - bpf_obj_free_fields(array->map.record, per_cpu_ptr(pptr, cpu)); 436 - off += size; 425 + if (map_flags & BPF_F_CPU) { 426 + cpu = map_flags >> 32; 427 + ptr = per_cpu_ptr(pptr, cpu); 428 + copy_map_value(map, ptr, value); 429 + bpf_obj_free_fields(array->map.record, ptr); 430 + goto unlock; 437 431 } 432 + for_each_possible_cpu(cpu) { 433 + ptr = per_cpu_ptr(pptr, cpu); 434 + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; 435 + copy_map_value(map, ptr, val); 436 + bpf_obj_free_fields(array->map.record, ptr); 437 + } 438 + unlock: 438 439 rcu_read_unlock(); 439 440 return 0; 440 441 }

+1 -1

kernel/bpf/syscall.c

··· 318 318 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 319 319 err = bpf_percpu_hash_copy(map, key, value); 320 320 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 321 - err = bpf_percpu_array_copy(map, key, value); 321 + err = bpf_percpu_array_copy(map, key, value, flags); 322 322 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 323 323 err = bpf_percpu_cgroup_storage_copy(map, key, value); 324 324 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {

Configure Feed

Configure Feed