Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
bpf: Refactor map->off_arr handling
Refactor map->off_arr handling into generic functions that can work on
their own without hardcoding map specific code. The btf_fields_offs
structure is now returned from btf_parse_field_offs, which can be reused
later for types in program BTF.
All functions like copy_map_value, zero_map_value call generic
underlying functions so that they can also be reused later for copying
to values allocated in programs which encode specific fields.
Later, some helper functions will also require access to this
btf_field_offs structure to be able to skip over special fields at
runtime.
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/r/20221103191013.1236066-9-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
authored by
Kumar Kartikeya Dwivedi
and committed by
Alexei Starovoitov
f71b2f64
db559117
+89
-81
+24
-17
include/linux/bpf.h
+24
-17
include/linux/bpf.h
···
341
341
}
342
342
343
343
/* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
344
-
static inline void __copy_map_value(struct bpf_map *map, void *dst, void *src, bool long_memcpy)
344
+
static inline void bpf_obj_memcpy(struct btf_field_offs *foffs,
345
+
void *dst, void *src, u32 size,
346
+
bool long_memcpy)
345
347
{
346
348
u32 curr_off = 0;
347
349
int i;
348
350
349
-
if (likely(!map->field_offs)) {
351
+
if (likely(!foffs)) {
350
352
if (long_memcpy)
351
-
bpf_long_memcpy(dst, src, round_up(map->value_size, 8));
353
+
bpf_long_memcpy(dst, src, round_up(size, 8));
352
354
else
353
-
memcpy(dst, src, map->value_size);
355
+
memcpy(dst, src, size);
354
356
return;
355
357
}
356
358
357
-
for (i = 0; i < map->field_offs->cnt; i++) {
358
-
u32 next_off = map->field_offs->field_off[i];
359
+
for (i = 0; i < foffs->cnt; i++) {
360
+
u32 next_off = foffs->field_off[i];
359
361
u32 sz = next_off - curr_off;
360
362
361
363
memcpy(dst + curr_off, src + curr_off, sz);
362
-
curr_off += map->field_offs->field_sz[i];
364
+
curr_off += foffs->field_sz[i];
363
365
}
364
-
memcpy(dst + curr_off, src + curr_off, map->value_size - curr_off);
366
+
memcpy(dst + curr_off, src + curr_off, size - curr_off);
365
367
}
366
368
367
369
static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
368
370
{
369
-
__copy_map_value(map, dst, src, false);
371
+
bpf_obj_memcpy(map->field_offs, dst, src, map->value_size, false);
370
372
}
371
373
372
374
static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
373
375
{
374
-
__copy_map_value(map, dst, src, true);
376
+
bpf_obj_memcpy(map->field_offs, dst, src, map->value_size, true);
375
377
}
376
378
377
-
static inline void zero_map_value(struct bpf_map *map, void *dst)
379
+
static inline void bpf_obj_memzero(struct btf_field_offs *foffs, void *dst, u32 size)
378
380
{
379
381
u32 curr_off = 0;
380
382
int i;
381
383
382
-
if (likely(!map->field_offs)) {
383
-
memset(dst, 0, map->value_size);
384
+
if (likely(!foffs)) {
385
+
memset(dst, 0, size);
384
386
return;
385
387
}
386
388
387
-
for (i = 0; i < map->field_offs->cnt; i++) {
388
-
u32 next_off = map->field_offs->field_off[i];
389
+
for (i = 0; i < foffs->cnt; i++) {
390
+
u32 next_off = foffs->field_off[i];
389
391
u32 sz = next_off - curr_off;
390
392
391
393
memset(dst + curr_off, 0, sz);
392
-
curr_off += map->field_offs->field_sz[i];
394
+
curr_off += foffs->field_sz[i];
393
395
}
394
-
memset(dst + curr_off, 0, map->value_size - curr_off);
396
+
memset(dst + curr_off, 0, size - curr_off);
397
+
}
398
+
399
+
static inline void zero_map_value(struct bpf_map *map, void *dst)
400
+
{
401
+
bpf_obj_memzero(map->field_offs, dst, map->value_size);
395
402
}
396
403
397
404
void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
+1
include/linux/btf.h
+1
include/linux/btf.h
···
165
165
int btf_find_timer(const struct btf *btf, const struct btf_type *t);
166
166
struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type *t,
167
167
u32 field_mask, u32 value_size);
168
+
struct btf_field_offs *btf_parse_field_offs(struct btf_record *rec);
168
169
bool btf_type_is_void(const struct btf_type *t);
169
170
s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind);
170
171
const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,
+55
kernel/bpf/btf.c
+55
kernel/bpf/btf.c
···
3551
3551
return ERR_PTR(ret);
3552
3552
}
3553
3553
3554
+
static int btf_field_offs_cmp(const void *_a, const void *_b, const void *priv)
3555
+
{
3556
+
const u32 a = *(const u32 *)_a;
3557
+
const u32 b = *(const u32 *)_b;
3558
+
3559
+
if (a < b)
3560
+
return -1;
3561
+
else if (a > b)
3562
+
return 1;
3563
+
return 0;
3564
+
}
3565
+
3566
+
static void btf_field_offs_swap(void *_a, void *_b, int size, const void *priv)
3567
+
{
3568
+
struct btf_field_offs *foffs = (void *)priv;
3569
+
u32 *off_base = foffs->field_off;
3570
+
u32 *a = _a, *b = _b;
3571
+
u8 *sz_a, *sz_b;
3572
+
3573
+
sz_a = foffs->field_sz + (a - off_base);
3574
+
sz_b = foffs->field_sz + (b - off_base);
3575
+
3576
+
swap(*a, *b);
3577
+
swap(*sz_a, *sz_b);
3578
+
}
3579
+
3580
+
struct btf_field_offs *btf_parse_field_offs(struct btf_record *rec)
3581
+
{
3582
+
struct btf_field_offs *foffs;
3583
+
u32 i, *off;
3584
+
u8 *sz;
3585
+
3586
+
BUILD_BUG_ON(ARRAY_SIZE(foffs->field_off) != ARRAY_SIZE(foffs->field_sz));
3587
+
if (IS_ERR_OR_NULL(rec) || WARN_ON_ONCE(rec->cnt > sizeof(foffs->field_off)))
3588
+
return NULL;
3589
+
3590
+
foffs = kzalloc(sizeof(*foffs), GFP_KERNEL | __GFP_NOWARN);
3591
+
if (!foffs)
3592
+
return ERR_PTR(-ENOMEM);
3593
+
3594
+
off = foffs->field_off;
3595
+
sz = foffs->field_sz;
3596
+
for (i = 0; i < rec->cnt; i++) {
3597
+
off[i] = rec->fields[i].offset;
3598
+
sz[i] = btf_field_type_size(rec->fields[i].type);
3599
+
}
3600
+
foffs->cnt = rec->cnt;
3601
+
3602
+
if (foffs->cnt == 1)
3603
+
return foffs;
3604
+
sort_r(foffs->field_off, foffs->cnt, sizeof(foffs->field_off[0]),
3605
+
btf_field_offs_cmp, btf_field_offs_swap, foffs);
3606
+
return foffs;
3607
+
}
3608
+
3554
3609
static void __btf_struct_show(const struct btf *btf, const struct btf_type *t,
3555
3610
u32 type_id, void *data, u8 bits_offset,
3556
3611
struct btf_show *show)
+9
-64
kernel/bpf/syscall.c
+9
-64
kernel/bpf/syscall.c
···
943
943
return -ENOTSUPP;
944
944
}
945
945
946
-
static int map_field_offs_cmp(const void *_a, const void *_b, const void *priv)
947
-
{
948
-
const u32 a = *(const u32 *)_a;
949
-
const u32 b = *(const u32 *)_b;
950
-
951
-
if (a < b)
952
-
return -1;
953
-
else if (a > b)
954
-
return 1;
955
-
return 0;
956
-
}
957
-
958
-
static void map_field_offs_swap(void *_a, void *_b, int size, const void *priv)
959
-
{
960
-
struct bpf_map *map = (struct bpf_map *)priv;
961
-
u32 *off_base = map->field_offs->field_off;
962
-
u32 *a = _a, *b = _b;
963
-
u8 *sz_a, *sz_b;
964
-
965
-
sz_a = map->field_offs->field_sz + (a - off_base);
966
-
sz_b = map->field_offs->field_sz + (b - off_base);
967
-
968
-
swap(*a, *b);
969
-
swap(*sz_a, *sz_b);
970
-
}
971
-
972
-
static int bpf_map_alloc_off_arr(struct bpf_map *map)
973
-
{
974
-
bool has_fields = !IS_ERR_OR_NULL(map->record);
975
-
struct btf_field_offs *fo;
976
-
struct btf_record *rec;
977
-
u32 i, *off;
978
-
u8 *sz;
979
-
980
-
if (!has_fields) {
981
-
map->field_offs = NULL;
982
-
return 0;
983
-
}
984
-
985
-
fo = kzalloc(sizeof(*map->field_offs), GFP_KERNEL | __GFP_NOWARN);
986
-
if (!fo)
987
-
return -ENOMEM;
988
-
map->field_offs = fo;
989
-
990
-
rec = map->record;
991
-
off = fo->field_off;
992
-
sz = fo->field_sz;
993
-
for (i = 0; i < rec->cnt; i++) {
994
-
*off++ = rec->fields[i].offset;
995
-
*sz++ = btf_field_type_size(rec->fields[i].type);
996
-
}
997
-
fo->cnt = rec->cnt;
998
-
999
-
if (fo->cnt == 1)
1000
-
return 0;
1001
-
sort_r(fo->field_off, fo->cnt, sizeof(fo->field_off[0]),
1002
-
map_field_offs_cmp, map_field_offs_swap, map);
1003
-
return 0;
1004
-
}
1005
-
1006
946
static int map_check_btf(struct bpf_map *map, const struct btf *btf,
1007
947
u32 btf_key_id, u32 btf_value_id)
1008
948
{
···
1037
1097
static int map_create(union bpf_attr *attr)
1038
1098
{
1039
1099
int numa_node = bpf_map_attr_numa_node(attr);
1100
+
struct btf_field_offs *foffs;
1040
1101
struct bpf_map *map;
1041
1102
int f_flags;
1042
1103
int err;
···
1117
1176
attr->btf_vmlinux_value_type_id;
1118
1177
}
1119
1178
1120
-
err = bpf_map_alloc_off_arr(map);
1121
-
if (err)
1179
+
1180
+
foffs = btf_parse_field_offs(map->record);
1181
+
if (IS_ERR(foffs)) {
1182
+
err = PTR_ERR(foffs);
1122
1183
goto free_map;
1184
+
}
1185
+
map->field_offs = foffs;
1123
1186
1124
1187
err = security_bpf_map_alloc(map);
1125
1188
if (err)
1126
-
goto free_map_off_arr;
1189
+
goto free_map_field_offs;
1127
1190
1128
1191
err = bpf_map_alloc_id(map);
1129
1192
if (err)
···
1151
1206
1152
1207
free_map_sec:
1153
1208
security_bpf_map_free(map);
1154
-
free_map_off_arr:
1209
+
free_map_field_offs:
1155
1210
kfree(map->field_offs);
1156
1211
free_map:
1157
1212
btf_put(map->btf);