Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags
Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags and check them for
following APIs:
* 'map_lookup_elem()'
* 'map_update_elem()'
* 'generic_map_lookup_batch()'
* 'generic_map_update_batch()'
And, get the correct value size for these APIs.
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Signed-off-by: Leon Hwang <leon.hwang@linux.dev>
Link: https://lore.kernel.org/r/20260107022022.12843-2-leon.hwang@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
authored by
Leon Hwang
and committed by
Alexei Starovoitov
2b421662
a8d50675
+42
-14
+22
-1
include/linux/bpf.h
+22
-1
include/linux/bpf.h
···
3915
3915
}
3916
3916
#endif
3917
3917
3918
+
static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type)
3919
+
{
3920
+
return false;
3921
+
}
3922
+
3918
3923
static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags)
3919
3924
{
3920
-
if (flags & ~allowed_flags)
3925
+
u32 cpu;
3926
+
3927
+
if ((u32)flags & ~allowed_flags)
3921
3928
return -EINVAL;
3922
3929
3923
3930
if ((flags & BPF_F_LOCK) && !btf_record_has_field(map->record, BPF_SPIN_LOCK))
3924
3931
return -EINVAL;
3932
+
3933
+
if (!(flags & BPF_F_CPU) && flags >> 32)
3934
+
return -EINVAL;
3935
+
3936
+
if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) {
3937
+
if (!bpf_map_supports_cpu_flags(map->map_type))
3938
+
return -EINVAL;
3939
+
if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS))
3940
+
return -EINVAL;
3941
+
3942
+
cpu = flags >> 32;
3943
+
if ((flags & BPF_F_CPU) && cpu >= num_possible_cpus())
3944
+
return -ERANGE;
3945
+
}
3925
3946
3926
3947
return 0;
3927
3948
}
+2
include/uapi/linux/bpf.h
+2
include/uapi/linux/bpf.h
···
1384
1384
BPF_NOEXIST = 1, /* create new element if it didn't exist */
1385
1385
BPF_EXIST = 2, /* update existing element */
1386
1386
BPF_F_LOCK = 4, /* spin_lock-ed map_lookup/map_update */
1387
+
BPF_F_CPU = 8, /* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */
1388
+
BPF_F_ALL_CPUS = 16, /* update value across all CPUs for percpu maps */
1387
1389
};
1388
1390
1389
1391
/* flags for BPF_MAP_CREATE command */
+16
-13
kernel/bpf/syscall.c
+16
-13
kernel/bpf/syscall.c
···
133
133
return atomic64_read(&map->writecnt) != 0;
134
134
}
135
135
136
-
static u32 bpf_map_value_size(const struct bpf_map *map)
136
+
static u32 bpf_map_value_size(const struct bpf_map *map, u64 flags)
137
137
{
138
-
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
139
-
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
140
-
map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
141
-
map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
138
+
if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS))
139
+
return map->value_size;
140
+
else if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
141
+
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
142
+
map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
143
+
map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
142
144
return round_up(map->value_size, 8) * num_possible_cpus();
143
145
else if (IS_FD_MAP(map))
144
146
return sizeof(u32);
···
1731
1729
if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ))
1732
1730
return -EPERM;
1733
1731
1734
-
err = bpf_map_check_op_flags(map, attr->flags, BPF_F_LOCK);
1732
+
err = bpf_map_check_op_flags(map, attr->flags, BPF_F_LOCK | BPF_F_CPU);
1735
1733
if (err)
1736
1734
return err;
1737
1735
···
1739
1737
if (IS_ERR(key))
1740
1738
return PTR_ERR(key);
1741
1739
1742
-
value_size = bpf_map_value_size(map);
1740
+
value_size = bpf_map_value_size(map, attr->flags);
1743
1741
1744
1742
err = -ENOMEM;
1745
1743
value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
···
1806
1804
goto err_put;
1807
1805
}
1808
1806
1809
-
value_size = bpf_map_value_size(map);
1807
+
value_size = bpf_map_value_size(map, attr->flags);
1810
1808
value = kvmemdup_bpfptr(uvalue, value_size);
1811
1809
if (IS_ERR(value)) {
1812
1810
err = PTR_ERR(value);
···
2002
2000
void *key, *value;
2003
2001
int err = 0;
2004
2002
2005
-
err = bpf_map_check_op_flags(map, attr->batch.elem_flags, BPF_F_LOCK);
2003
+
err = bpf_map_check_op_flags(map, attr->batch.elem_flags,
2004
+
BPF_F_LOCK | BPF_F_CPU | BPF_F_ALL_CPUS);
2006
2005
if (err)
2007
2006
return err;
2008
2007
2009
-
value_size = bpf_map_value_size(map);
2008
+
value_size = bpf_map_value_size(map, attr->batch.elem_flags);
2010
2009
2011
2010
max_count = attr->batch.count;
2012
2011
if (!max_count)
···
2062
2059
u32 value_size, cp, max_count;
2063
2060
int err;
2064
2061
2065
-
err = bpf_map_check_op_flags(map, attr->batch.elem_flags, BPF_F_LOCK);
2062
+
err = bpf_map_check_op_flags(map, attr->batch.elem_flags, BPF_F_LOCK | BPF_F_CPU);
2066
2063
if (err)
2067
2064
return err;
2068
2065
2069
-
value_size = bpf_map_value_size(map);
2066
+
value_size = bpf_map_value_size(map, attr->batch.elem_flags);
2070
2067
2071
2068
max_count = attr->batch.count;
2072
2069
if (!max_count)
···
2188
2185
goto err_put;
2189
2186
}
2190
2187
2191
-
value_size = bpf_map_value_size(map);
2188
+
value_size = bpf_map_value_size(map, 0);
2192
2189
2193
2190
err = -ENOMEM;
2194
2191
value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
+2
tools/include/uapi/linux/bpf.h
+2
tools/include/uapi/linux/bpf.h
···
1384
1384
BPF_NOEXIST = 1, /* create new element if it didn't exist */
1385
1385
BPF_EXIST = 2, /* update existing element */
1386
1386
BPF_F_LOCK = 4, /* spin_lock-ed map_lookup/map_update */
1387
+
BPF_F_CPU = 8, /* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */
1388
+
BPF_F_ALL_CPUS = 16, /* update value across all CPUs for percpu maps */
1387
1389
};
1388
1390
1389
1391
/* flags for BPF_MAP_CREATE command */