tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

bpf: Add trampoline ip hash table

Following changes need to lookup trampoline based on its ip address,
adding hash table for that.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20251230145010.103439-8-jolsa@kernel.org

authored by

Jiri Olsa and committed by
Andrii Nakryiko 7d045249 e93672f7

+24 -13
+5 -2
include/linux/bpf.h
··· 1329 1329 }; 1330 1330 1331 1331 struct bpf_trampoline { 1332 - /* hlist for trampoline_table */ 1333 - struct hlist_node hlist; 1332 + /* hlist for trampoline_key_table */ 1333 + struct hlist_node hlist_key; 1334 + /* hlist for trampoline_ip_table */ 1335 + struct hlist_node hlist_ip; 1334 1336 struct ftrace_ops *fops; 1335 1337 /* serializes access to fields of this trampoline */ 1336 1338 struct mutex mutex; 1337 1339 refcount_t refcnt; 1338 1340 u32 flags; 1339 1341 u64 key; 1342 + unsigned long ip; 1340 1343 struct { 1341 1344 struct btf_func_model model; 1342 1345 void *addr;
+19 -11
kernel/bpf/trampoline.c
··· 24 24 #define TRAMPOLINE_HASH_BITS 10 25 25 #define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS) 26 26 27 - static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; 27 + static struct hlist_head trampoline_key_table[TRAMPOLINE_TABLE_SIZE]; 28 + static struct hlist_head trampoline_ip_table[TRAMPOLINE_TABLE_SIZE]; 28 29 29 - /* serializes access to trampoline_table */ 30 + /* serializes access to trampoline tables */ 30 31 static DEFINE_MUTEX(trampoline_mutex); 31 32 32 33 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS ··· 143 142 PAGE_SIZE, true, ksym->name); 144 143 } 145 144 146 - static struct bpf_trampoline *bpf_trampoline_lookup(u64 key) 145 + static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip) 147 146 { 148 147 struct bpf_trampoline *tr; 149 148 struct hlist_head *head; 150 149 int i; 151 150 152 151 mutex_lock(&trampoline_mutex); 153 - head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; 154 - hlist_for_each_entry(tr, head, hlist) { 152 + head = &trampoline_key_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; 153 + hlist_for_each_entry(tr, head, hlist_key) { 155 154 if (tr->key == key) { 156 155 refcount_inc(&tr->refcnt); 157 156 goto out; ··· 172 171 #endif 173 172 174 173 tr->key = key; 175 - INIT_HLIST_NODE(&tr->hlist); 176 - hlist_add_head(&tr->hlist, head); 174 + tr->ip = ftrace_location(ip); 175 + INIT_HLIST_NODE(&tr->hlist_key); 176 + INIT_HLIST_NODE(&tr->hlist_ip); 177 + hlist_add_head(&tr->hlist_key, head); 178 + head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)]; 179 + hlist_add_head(&tr->hlist_ip, head); 177 180 refcount_set(&tr->refcnt, 1); 178 181 mutex_init(&tr->mutex); 179 182 for (i = 0; i < BPF_TRAMP_MAX; i++) ··· 888 883 prog->aux->attach_btf_id); 889 884 890 885 bpf_lsm_find_cgroup_shim(prog, &bpf_func); 891 - tr = bpf_trampoline_lookup(key); 886 + tr = bpf_trampoline_lookup(key, 0); 892 887 if (WARN_ON_ONCE(!tr)) 893 888 return; 894 889 ··· 908 903 { 909 904 struct bpf_trampoline *tr; 910 905 911 - tr = bpf_trampoline_lookup(key); 906 + tr = bpf_trampoline_lookup(key, tgt_info->tgt_addr); 912 907 if (!tr) 913 908 return NULL; 914 909 ··· 944 939 * fexit progs. The fentry-only trampoline will be freed via 945 940 * multiple rcu callbacks. 946 941 */ 947 - hlist_del(&tr->hlist); 942 + hlist_del(&tr->hlist_key); 943 + hlist_del(&tr->hlist_ip); 948 944 if (tr->fops) { 949 945 ftrace_free_filter(tr->fops); 950 946 kfree(tr->fops); ··· 1218 1212 int i; 1219 1213 1220 1214 for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++) 1221 - INIT_HLIST_HEAD(&trampoline_table[i]); 1215 + INIT_HLIST_HEAD(&trampoline_key_table[i]); 1216 + for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++) 1217 + INIT_HLIST_HEAD(&trampoline_ip_table[i]); 1222 1218 return 0; 1223 1219 } 1224 1220 late_initcall(init_trampolines);