Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'add-kernel-symbol-for-struct_ops-trampoline'
Xu Kuohai says:
====================
Add kernel symbol for struct_ops trampoline.
Without kernel symbol for struct_ops trampoline, the unwinder may
produce unexpected stacktraces. For example, the x86 ORC and FP
unwinder stops stacktrace on a struct_ops trampoline address since
there is no kernel symbol for the address.
v4:
- Add a separate cleanup patch to remove unused member rcu from
bpf_struct_ops_map (patch 1)
- Use funcs_cnt instead of btf_type_vlen(vt) for links memory
calculation in .map_mem_usage (patch 2)
- Include ksyms[] memory in map_mem_usage (patch 3)
- Various fixes in patch 3 (Thanks to Martin)
v3: https://lore.kernel.org/bpf/20241111121641.2679885-1-xukuohai@huaweicloud.com/
- Add a separate cleanup patch to replace links_cnt with funcs_cnt
- Allocate ksyms on-demand in update_elem() to stay with the links
allocation way
- Set ksym name to prog__<struct_ops_name>_<member_name>
v2: https://lore.kernel.org/bpf/20241101111948.1570547-1-xukuohai@huaweicloud.com/
- Refine the commit message for clarity and fix a test bot warning
v1: https://lore.kernel.org/bpf/20241030111533.907289-1-xukuohai@huaweicloud.com/
====================
Reviewed-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20241112145849.3436772-1-xukuohai@huaweicloud.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+114
-16
+2
-1
include/linux/bpf.h
+2
-1
include/linux/bpf.h
···
1402
1402
void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
1403
1403
struct bpf_prog *to);
1404
1404
/* Called only from JIT-enabled code, so there's no need for stubs. */
1405
-
void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym);
1405
+
void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym);
1406
+
void bpf_image_ksym_add(struct bpf_ksym *ksym);
1406
1407
void bpf_image_ksym_del(struct bpf_ksym *ksym);
1407
1408
void bpf_ksym_add(struct bpf_ksym *ksym);
1408
1409
void bpf_ksym_del(struct bpf_ksym *ksym);
+103
-12
kernel/bpf/bpf_struct_ops.c
+103
-12
kernel/bpf/bpf_struct_ops.c
···
23
23
24
24
struct bpf_struct_ops_map {
25
25
struct bpf_map map;
26
-
struct rcu_head rcu;
27
26
const struct bpf_struct_ops_desc *st_ops_desc;
28
27
/* protect map_update */
29
28
struct mutex lock;
···
31
32
* (in kvalue.data).
32
33
*/
33
34
struct bpf_link **links;
34
-
u32 links_cnt;
35
+
/* ksyms for bpf trampolines */
36
+
struct bpf_ksym **ksyms;
37
+
u32 funcs_cnt;
35
38
u32 image_pages_cnt;
36
39
/* image_pages is an array of pages that has all the trampolines
37
40
* that stores the func args before calling the bpf_prog.
···
482
481
{
483
482
u32 i;
484
483
485
-
for (i = 0; i < st_map->links_cnt; i++) {
486
-
if (st_map->links[i]) {
487
-
bpf_link_put(st_map->links[i]);
488
-
st_map->links[i] = NULL;
489
-
}
484
+
for (i = 0; i < st_map->funcs_cnt; i++) {
485
+
if (!st_map->links[i])
486
+
break;
487
+
bpf_link_put(st_map->links[i]);
488
+
st_map->links[i] = NULL;
490
489
}
491
490
}
492
491
···
587
586
return 0;
588
587
}
589
588
589
+
static void bpf_struct_ops_ksym_init(const char *tname, const char *mname,
590
+
void *image, unsigned int size,
591
+
struct bpf_ksym *ksym)
592
+
{
593
+
snprintf(ksym->name, KSYM_NAME_LEN, "bpf__%s_%s", tname, mname);
594
+
INIT_LIST_HEAD_RCU(&ksym->lnode);
595
+
bpf_image_ksym_init(image, size, ksym);
596
+
}
597
+
598
+
static void bpf_struct_ops_map_add_ksyms(struct bpf_struct_ops_map *st_map)
599
+
{
600
+
u32 i;
601
+
602
+
for (i = 0; i < st_map->funcs_cnt; i++) {
603
+
if (!st_map->ksyms[i])
604
+
break;
605
+
bpf_image_ksym_add(st_map->ksyms[i]);
606
+
}
607
+
}
608
+
609
+
static void bpf_struct_ops_map_del_ksyms(struct bpf_struct_ops_map *st_map)
610
+
{
611
+
u32 i;
612
+
613
+
for (i = 0; i < st_map->funcs_cnt; i++) {
614
+
if (!st_map->ksyms[i])
615
+
break;
616
+
bpf_image_ksym_del(st_map->ksyms[i]);
617
+
}
618
+
}
619
+
620
+
static void bpf_struct_ops_map_free_ksyms(struct bpf_struct_ops_map *st_map)
621
+
{
622
+
u32 i;
623
+
624
+
for (i = 0; i < st_map->funcs_cnt; i++) {
625
+
if (!st_map->ksyms[i])
626
+
break;
627
+
kfree(st_map->ksyms[i]);
628
+
st_map->ksyms[i] = NULL;
629
+
}
630
+
}
631
+
590
632
static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
591
633
void *value, u64 flags)
592
634
{
···
645
601
int prog_fd, err;
646
602
u32 i, trampoline_start, image_off = 0;
647
603
void *cur_image = NULL, *image = NULL;
604
+
struct bpf_link **plink;
605
+
struct bpf_ksym **pksym;
606
+
const char *tname, *mname;
648
607
649
608
if (flags)
650
609
return -EINVAL;
···
686
639
udata = &uvalue->data;
687
640
kdata = &kvalue->data;
688
641
642
+
plink = st_map->links;
643
+
pksym = st_map->ksyms;
644
+
tname = btf_name_by_offset(st_map->btf, t->name_off);
689
645
module_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_MODULE_ID]);
690
646
for_each_member(i, t, member) {
691
647
const struct btf_type *mtype, *ptype;
692
648
struct bpf_prog *prog;
693
649
struct bpf_tramp_link *link;
650
+
struct bpf_ksym *ksym;
694
651
u32 moff;
695
652
696
653
moff = __btf_member_bit_offset(t, member) / 8;
654
+
mname = btf_name_by_offset(st_map->btf, member->name_off);
697
655
ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);
698
656
if (ptype == module_type) {
699
657
if (*(void **)(udata + moff))
···
766
714
}
767
715
bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
768
716
&bpf_struct_ops_link_lops, prog);
769
-
st_map->links[i] = &link->link;
717
+
*plink++ = &link->link;
718
+
719
+
ksym = kzalloc(sizeof(*ksym), GFP_USER);
720
+
if (!ksym) {
721
+
err = -ENOMEM;
722
+
goto reset_unlock;
723
+
}
724
+
*pksym++ = ksym;
770
725
771
726
trampoline_start = image_off;
772
727
err = bpf_struct_ops_prepare_trampoline(tlinks, link,
···
794
735
795
736
/* put prog_id to udata */
796
737
*(unsigned long *)(udata + moff) = prog->aux->id;
738
+
739
+
/* init ksym for this trampoline */
740
+
bpf_struct_ops_ksym_init(tname, mname,
741
+
image + trampoline_start,
742
+
image_off - trampoline_start,
743
+
ksym);
797
744
}
798
745
799
746
if (st_ops->validate) {
···
848
783
*/
849
784
850
785
reset_unlock:
786
+
bpf_struct_ops_map_free_ksyms(st_map);
851
787
bpf_struct_ops_map_free_image(st_map);
852
788
bpf_struct_ops_map_put_progs(st_map);
853
789
memset(uvalue, 0, map->value_size);
···
856
790
unlock:
857
791
kfree(tlinks);
858
792
mutex_unlock(&st_map->lock);
793
+
if (!err)
794
+
bpf_struct_ops_map_add_ksyms(st_map);
859
795
return err;
860
796
}
861
797
···
917
849
918
850
if (st_map->links)
919
851
bpf_struct_ops_map_put_progs(st_map);
852
+
if (st_map->ksyms)
853
+
bpf_struct_ops_map_free_ksyms(st_map);
920
854
bpf_map_area_free(st_map->links);
855
+
bpf_map_area_free(st_map->ksyms);
921
856
bpf_struct_ops_map_free_image(st_map);
922
857
bpf_map_area_free(st_map->uvalue);
923
858
bpf_map_area_free(st_map);
···
936
865
*/
937
866
if (btf_is_module(st_map->btf))
938
867
module_put(st_map->st_ops_desc->st_ops->owner);
868
+
869
+
bpf_struct_ops_map_del_ksyms(st_map);
939
870
940
871
/* The struct_ops's function may switch to another struct_ops.
941
872
*
···
966
893
!attr->btf_vmlinux_value_type_id)
967
894
return -EINVAL;
968
895
return 0;
896
+
}
897
+
898
+
static u32 count_func_ptrs(const struct btf *btf, const struct btf_type *t)
899
+
{
900
+
int i;
901
+
u32 count;
902
+
const struct btf_member *member;
903
+
904
+
count = 0;
905
+
for_each_member(i, t, member)
906
+
if (btf_type_resolve_func_ptr(btf, member->type, NULL))
907
+
count++;
908
+
return count;
969
909
}
970
910
971
911
static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
···
1047
961
map = &st_map->map;
1048
962
1049
963
st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
1050
-
st_map->links_cnt = btf_type_vlen(t);
964
+
st_map->funcs_cnt = count_func_ptrs(btf, t);
1051
965
st_map->links =
1052
-
bpf_map_area_alloc(st_map->links_cnt * sizeof(struct bpf_links *),
966
+
bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_link *),
1053
967
NUMA_NO_NODE);
1054
-
if (!st_map->uvalue || !st_map->links) {
968
+
969
+
st_map->ksyms =
970
+
bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_ksym *),
971
+
NUMA_NO_NODE);
972
+
if (!st_map->uvalue || !st_map->links || !st_map->ksyms) {
1055
973
ret = -ENOMEM;
1056
974
goto errout_free;
1057
975
}
···
1084
994
usage = sizeof(*st_map) +
1085
995
vt->size - sizeof(struct bpf_struct_ops_value);
1086
996
usage += vt->size;
1087
-
usage += btf_type_vlen(vt) * sizeof(struct bpf_links *);
997
+
usage += st_map->funcs_cnt * sizeof(struct bpf_link *);
998
+
usage += st_map->funcs_cnt * sizeof(struct bpf_ksym *);
1088
999
usage += PAGE_SIZE;
1089
1000
return usage;
1090
1001
}
+2
-1
kernel/bpf/dispatcher.c
+2
-1
kernel/bpf/dispatcher.c
+7
-2
kernel/bpf/trampoline.c
+7
-2
kernel/bpf/trampoline.c
···
115
115
(ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);
116
116
}
117
117
118
-
void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym)
118
+
void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym)
119
119
{
120
120
ksym->start = (unsigned long) data;
121
121
ksym->end = ksym->start + size;
122
+
}
123
+
124
+
void bpf_image_ksym_add(struct bpf_ksym *ksym)
125
+
{
122
126
bpf_ksym_add(ksym);
123
127
perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,
124
128
PAGE_SIZE, false, ksym->name);
···
381
377
ksym = &im->ksym;
382
378
INIT_LIST_HEAD_RCU(&ksym->lnode);
383
379
snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu", key);
384
-
bpf_image_ksym_add(image, size, ksym);
380
+
bpf_image_ksym_init(image, size, ksym);
381
+
bpf_image_ksym_add(ksym);
385
382
return im;
386
383
387
384
out_free_image: