bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_cgroup_storage maps

Introduce BPF_F_ALL_CPUS flag support for percpu_cgroup_storage maps to
allow updating values for all CPUs with a single value for update_elem
API.

Introduce BPF_F_CPU flag support for percpu_cgroup_storage maps to
allow:

* update value for specified CPU for update_elem API.
* lookup value for specified CPU for lookup_elem API.

The BPF_F_CPU flag is passed via map_flags along with embedded cpu info.

Signed-off-by: Leon Hwang <leon.hwang@linux.dev>
Link: https://lore.kernel.org/r/20260107022022.12843-6-leon.hwang@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

authored by

Leon Hwang and committed by

Alexei Starovoitov 5 months ago 47c79f05 8526397c

+22 -8

4 changed files

expand all

include

linux

bpf-cgroup.h

bpf.h

kernel

bpf

local_storage.c

syscall.c

+2 -2

include/linux/bpf-cgroup.h

··· 172 172 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage); 173 173 int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux, struct bpf_map *map); 174 174 175 - int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value); 175 + int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value, u64 flags); 176 176 int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 177 177 void *value, u64 flags); 178 178 ··· 470 470 static inline void bpf_cgroup_storage_free( 471 471 struct bpf_cgroup_storage *storage) {} 472 472 static inline int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, 473 - void *value) { 473 + void *value, u64 flags) { 474 474 return 0; 475 475 } 476 476 static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,

include/linux/bpf.h

··· 3921 3921 case BPF_MAP_TYPE_PERCPU_ARRAY: 3922 3922 case BPF_MAP_TYPE_PERCPU_HASH: 3923 3923 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 3924 + case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 3924 3925 return true; 3925 3926 default: 3926 3927 return false;

+18 -5

kernel/bpf/local_storage.c

··· 180 180 } 181 181 182 182 int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *key, 183 - void *value) 183 + void *value, u64 map_flags) 184 184 { 185 185 struct bpf_cgroup_storage_map *map = map_to_storage(_map); 186 186 struct bpf_cgroup_storage *storage; ··· 198 198 * access 'value_size' of them, so copying rounded areas 199 199 * will not leak any kernel data 200 200 */ 201 + if (map_flags & BPF_F_CPU) { 202 + cpu = map_flags >> 32; 203 + copy_map_value(_map, value, per_cpu_ptr(storage->percpu_buf, cpu)); 204 + goto unlock; 205 + } 201 206 size = round_up(_map->value_size, 8); 202 207 for_each_possible_cpu(cpu) { 203 208 copy_map_value_long(_map, value + off, per_cpu_ptr(storage->percpu_buf, cpu)); 204 209 off += size; 205 210 } 211 + unlock: 206 212 rcu_read_unlock(); 207 213 return 0; 208 214 } ··· 218 212 { 219 213 struct bpf_cgroup_storage_map *map = map_to_storage(_map); 220 214 struct bpf_cgroup_storage *storage; 221 - int cpu, off = 0; 215 + void *val; 222 216 u32 size; 217 + int cpu; 223 218 224 - if (map_flags != BPF_ANY && map_flags != BPF_EXIST) 219 + if ((u32)map_flags & ~(BPF_ANY | BPF_EXIST | BPF_F_CPU | BPF_F_ALL_CPUS)) 225 220 return -EINVAL; 226 221 227 222 rcu_read_lock(); ··· 238 231 * returned or zeros which were zero-filled by percpu_alloc, 239 232 * so no kernel data leaks possible 240 233 */ 234 + if (map_flags & BPF_F_CPU) { 235 + cpu = map_flags >> 32; 236 + copy_map_value(_map, per_cpu_ptr(storage->percpu_buf, cpu), value); 237 + goto unlock; 238 + } 241 239 size = round_up(_map->value_size, 8); 242 240 for_each_possible_cpu(cpu) { 243 - copy_map_value_long(_map, per_cpu_ptr(storage->percpu_buf, cpu), value + off); 244 - off += size; 241 + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; 242 + copy_map_value(_map, per_cpu_ptr(storage->percpu_buf, cpu), val); 245 243 } 244 + unlock: 246 245 rcu_read_unlock(); 247 246 return 0; 248 247 }

+1 -1

kernel/bpf/syscall.c

··· 320 320 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 321 321 err = bpf_percpu_array_copy(map, key, value, flags); 322 322 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { 323 - err = bpf_percpu_cgroup_storage_copy(map, key, value); 323 + err = bpf_percpu_cgroup_storage_copy(map, key, value, flags); 324 324 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 325 325 err = bpf_stackmap_extract(map, key, value, false); 326 326 } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {

Configure Feed

Configure Feed