tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge branch 'bpf-support-bpf-rbtree-traversal-and-list-peeking'

Martin KaFai Lau says:

====================
bpf: Support bpf rbtree traversal and list peeking

From: Martin KaFai Lau <martin.lau@kernel.org>

The RFC v1 [1] showed a fq qdisc implementation in bpf
that is much closer to the kernel sch_fq.c.

The fq example and bpf qdisc changes are separated out from this set.
This set is to focus on the kfunc and verifier changes that
enable the bpf rbtree traversal and list peeking.

v2:
- Added tests to check that the return value of
the bpf_rbtree_{root,left,right} and bpf_list_{front,back} is
marked as a non_own_ref node pointer. (Kumar)
- Added tests to ensure that the bpf_rbtree_{root,left,right} and
bpf_list_{front,back} must be called after holding the spinlock.
- Squashed the selftests adjustment to the corresponding verifier
changes to avoid bisect failure. (Kumar)
- Separated the bpf qdisc specific changes and fq selftest example
from this set.

[1]: https://lore.kernel.org/bpf/20250418224652.105998-1-martin.lau@linux.dev/
====================

Link: https://patch.msgid.link/20250506015857.817950-1-martin.lau@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

Alexei Starovoitov 9fd06062 62e23f18

+446 -32
+52
kernel/bpf/helpers.c
··· 2293 2293 return __bpf_list_del(head, true); 2294 2294 } 2295 2295 2296 + __bpf_kfunc struct bpf_list_node *bpf_list_front(struct bpf_list_head *head) 2297 + { 2298 + struct list_head *h = (struct list_head *)head; 2299 + 2300 + if (list_empty(h) || unlikely(!h->next)) 2301 + return NULL; 2302 + 2303 + return (struct bpf_list_node *)h->next; 2304 + } 2305 + 2306 + __bpf_kfunc struct bpf_list_node *bpf_list_back(struct bpf_list_head *head) 2307 + { 2308 + struct list_head *h = (struct list_head *)head; 2309 + 2310 + if (list_empty(h) || unlikely(!h->next)) 2311 + return NULL; 2312 + 2313 + return (struct bpf_list_node *)h->prev; 2314 + } 2315 + 2296 2316 __bpf_kfunc struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, 2297 2317 struct bpf_rb_node *node) 2298 2318 { ··· 2384 2364 struct rb_root_cached *r = (struct rb_root_cached *)root; 2385 2365 2386 2366 return (struct bpf_rb_node *)rb_first_cached(r); 2367 + } 2368 + 2369 + __bpf_kfunc struct bpf_rb_node *bpf_rbtree_root(struct bpf_rb_root *root) 2370 + { 2371 + struct rb_root_cached *r = (struct rb_root_cached *)root; 2372 + 2373 + return (struct bpf_rb_node *)r->rb_root.rb_node; 2374 + } 2375 + 2376 + __bpf_kfunc struct bpf_rb_node *bpf_rbtree_left(struct bpf_rb_root *root, struct bpf_rb_node *node) 2377 + { 2378 + struct bpf_rb_node_kern *node_internal = (struct bpf_rb_node_kern *)node; 2379 + 2380 + if (READ_ONCE(node_internal->owner) != root) 2381 + return NULL; 2382 + 2383 + return (struct bpf_rb_node *)node_internal->rb_node.rb_left; 2384 + } 2385 + 2386 + __bpf_kfunc struct bpf_rb_node *bpf_rbtree_right(struct bpf_rb_root *root, struct bpf_rb_node *node) 2387 + { 2388 + struct bpf_rb_node_kern *node_internal = (struct bpf_rb_node_kern *)node; 2389 + 2390 + if (READ_ONCE(node_internal->owner) != root) 2391 + return NULL; 2392 + 2393 + return (struct bpf_rb_node *)node_internal->rb_node.rb_right; 2387 2394 } 2388 2395 2389 2396 /** ··· 3256 3209 BTF_ID_FLAGS(func, bpf_list_push_back_impl) 3257 3210 BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL) 3258 3211 BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL) 3212 + BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL) 3213 + BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL) 3259 3214 BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) 3260 3215 BTF_ID_FLAGS(func, bpf_task_release, KF_RELEASE) 3261 3216 BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL) 3262 3217 BTF_ID_FLAGS(func, bpf_rbtree_add_impl) 3263 3218 BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL) 3219 + BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL) 3220 + BTF_ID_FLAGS(func, bpf_rbtree_left, KF_RET_NULL) 3221 + BTF_ID_FLAGS(func, bpf_rbtree_right, KF_RET_NULL) 3264 3222 3265 3223 #ifdef CONFIG_CGROUPS 3266 3224 BTF_ID_FLAGS(func, bpf_cgroup_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL)
+48 -18
kernel/bpf/verifier.c
··· 11987 11987 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID); 11988 11988 } 11989 11989 11990 + static bool is_rbtree_node_type(const struct btf_type *t) 11991 + { 11992 + return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_RB_NODE_ID]); 11993 + } 11994 + 11995 + static bool is_list_node_type(const struct btf_type *t) 11996 + { 11997 + return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_LIST_NODE_ID]); 11998 + } 11999 + 11990 12000 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf, 11991 12001 const struct btf_param *arg) 11992 12002 { ··· 12079 12069 KF_bpf_list_push_back_impl, 12080 12070 KF_bpf_list_pop_front, 12081 12071 KF_bpf_list_pop_back, 12072 + KF_bpf_list_front, 12073 + KF_bpf_list_back, 12082 12074 KF_bpf_cast_to_kern_ctx, 12083 12075 KF_bpf_rdonly_cast, 12084 12076 KF_bpf_rcu_read_lock, ··· 12088 12076 KF_bpf_rbtree_remove, 12089 12077 KF_bpf_rbtree_add_impl, 12090 12078 KF_bpf_rbtree_first, 12079 + KF_bpf_rbtree_root, 12080 + KF_bpf_rbtree_left, 12081 + KF_bpf_rbtree_right, 12091 12082 KF_bpf_dynptr_from_skb, 12092 12083 KF_bpf_dynptr_from_xdp, 12093 12084 KF_bpf_dynptr_slice, ··· 12126 12111 BTF_ID(func, bpf_list_push_back_impl) 12127 12112 BTF_ID(func, bpf_list_pop_front) 12128 12113 BTF_ID(func, bpf_list_pop_back) 12114 + BTF_ID(func, bpf_list_front) 12115 + BTF_ID(func, bpf_list_back) 12129 12116 BTF_ID(func, bpf_cast_to_kern_ctx) 12130 12117 BTF_ID(func, bpf_rdonly_cast) 12131 12118 BTF_ID(func, bpf_rbtree_remove) 12132 12119 BTF_ID(func, bpf_rbtree_add_impl) 12133 12120 BTF_ID(func, bpf_rbtree_first) 12121 + BTF_ID(func, bpf_rbtree_root) 12122 + BTF_ID(func, bpf_rbtree_left) 12123 + BTF_ID(func, bpf_rbtree_right) 12134 12124 #ifdef CONFIG_NET 12135 12125 BTF_ID(func, bpf_dynptr_from_skb) 12136 12126 BTF_ID(func, bpf_dynptr_from_xdp) ··· 12164 12144 BTF_ID(func, bpf_list_push_back_impl) 12165 12145 BTF_ID(func, bpf_list_pop_front) 12166 12146 BTF_ID(func, bpf_list_pop_back) 12147 + BTF_ID(func, bpf_list_front) 12148 + BTF_ID(func, bpf_list_back) 12167 12149 BTF_ID(func, bpf_cast_to_kern_ctx) 12168 12150 BTF_ID(func, bpf_rdonly_cast) 12169 12151 BTF_ID(func, bpf_rcu_read_lock) ··· 12173 12151 BTF_ID(func, bpf_rbtree_remove) 12174 12152 BTF_ID(func, bpf_rbtree_add_impl) 12175 12153 BTF_ID(func, bpf_rbtree_first) 12154 + BTF_ID(func, bpf_rbtree_root) 12155 + BTF_ID(func, bpf_rbtree_left) 12156 + BTF_ID(func, bpf_rbtree_right) 12176 12157 #ifdef CONFIG_NET 12177 12158 BTF_ID(func, bpf_dynptr_from_skb) 12178 12159 BTF_ID(func, bpf_dynptr_from_xdp) ··· 12604 12579 return btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 12605 12580 btf_id == special_kfunc_list[KF_bpf_list_push_back_impl] || 12606 12581 btf_id == special_kfunc_list[KF_bpf_list_pop_front] || 12607 - btf_id == special_kfunc_list[KF_bpf_list_pop_back]; 12582 + btf_id == special_kfunc_list[KF_bpf_list_pop_back] || 12583 + btf_id == special_kfunc_list[KF_bpf_list_front] || 12584 + btf_id == special_kfunc_list[KF_bpf_list_back]; 12608 12585 } 12609 12586 12610 12587 static bool is_bpf_rbtree_api_kfunc(u32 btf_id) 12611 12588 { 12612 12589 return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] || 12613 12590 btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 12614 - btf_id == special_kfunc_list[KF_bpf_rbtree_first]; 12591 + btf_id == special_kfunc_list[KF_bpf_rbtree_first] || 12592 + btf_id == special_kfunc_list[KF_bpf_rbtree_root] || 12593 + btf_id == special_kfunc_list[KF_bpf_rbtree_left] || 12594 + btf_id == special_kfunc_list[KF_bpf_rbtree_right]; 12615 12595 } 12616 12596 12617 12597 static bool is_bpf_iter_num_api_kfunc(u32 btf_id) ··· 12716 12686 break; 12717 12687 case BPF_RB_NODE: 12718 12688 ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 12719 - kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]); 12689 + kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] || 12690 + kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_left] || 12691 + kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_right]); 12720 12692 break; 12721 12693 default: 12722 12694 verbose(env, "verifier internal error: unexpected graph node argument type %s\n", ··· 13232 13200 return ret; 13233 13201 break; 13234 13202 case KF_ARG_PTR_TO_RB_NODE: 13235 - if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_remove]) { 13236 - if (!type_is_non_owning_ref(reg->type) || reg->ref_obj_id) { 13237 - verbose(env, "rbtree_remove node input must be non-owning ref\n"); 13238 - return -EINVAL; 13239 - } 13240 - if (in_rbtree_lock_required_cb(env)) { 13241 - verbose(env, "rbtree_remove not allowed in rbtree cb\n"); 13242 - return -EINVAL; 13243 - } 13244 - } else { 13203 + if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 13245 13204 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 13246 13205 verbose(env, "arg#%d expected pointer to allocated object\n", i); 13247 13206 return -EINVAL; 13248 13207 } 13249 13208 if (!reg->ref_obj_id) { 13250 13209 verbose(env, "allocated object must be referenced\n"); 13210 + return -EINVAL; 13211 + } 13212 + } else { 13213 + if (!type_is_non_owning_ref(reg->type) && !reg->ref_obj_id) { 13214 + verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name); 13215 + return -EINVAL; 13216 + } 13217 + if (in_rbtree_lock_required_cb(env)) { 13218 + verbose(env, "%s not allowed in rbtree cb\n", func_name); 13251 13219 return -EINVAL; 13252 13220 } 13253 13221 } ··· 13777 13745 insn_aux->kptr_struct_meta = 13778 13746 btf_find_struct_meta(meta.arg_btf, 13779 13747 meta.arg_btf_id); 13780 - } else if (meta.func_id == special_kfunc_list[KF_bpf_list_pop_front] || 13781 - meta.func_id == special_kfunc_list[KF_bpf_list_pop_back]) { 13748 + } else if (is_list_node_type(ptr_type)) { 13782 13749 struct btf_field *field = meta.arg_list_head.field; 13783 13750 13784 13751 mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); 13785 - } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_remove] || 13786 - meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) { 13752 + } else if (is_rbtree_node_type(ptr_type)) { 13787 13753 struct btf_field *field = meta.arg_rbtree_root.field; 13788 13754 13789 13755 mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); ··· 13911 13881 if (is_kfunc_ret_null(&meta)) 13912 13882 regs[BPF_REG_0].id = id; 13913 13883 regs[BPF_REG_0].ref_obj_id = id; 13914 - } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) { 13884 + } else if (is_rbtree_node_type(ptr_type) || is_list_node_type(ptr_type)) { 13915 13885 ref_set_non_owning(env, &regs[BPF_REG_0]); 13916 13886 } 13917 13887
+6
tools/testing/selftests/bpf/prog_tests/linked_list.c
··· 7 7 8 8 #include "linked_list.skel.h" 9 9 #include "linked_list_fail.skel.h" 10 + #include "linked_list_peek.skel.h" 10 11 11 12 static char log_buf[1024 * 1024]; 12 13 ··· 805 804 test_linked_list_success(LIST_IN_LIST, false); 806 805 test_linked_list_success(LIST_IN_LIST, true); 807 806 test_linked_list_success(TEST_ALL, false); 807 + } 808 + 809 + void test_linked_list_peek(void) 810 + { 811 + RUN_TESTS(linked_list_peek); 808 812 }
+6
tools/testing/selftests/bpf/prog_tests/rbtree.c
··· 8 8 #include "rbtree_fail.skel.h" 9 9 #include "rbtree_btf_fail__wrong_node_type.skel.h" 10 10 #include "rbtree_btf_fail__add_wrong_type.skel.h" 11 + #include "rbtree_search.skel.h" 11 12 12 13 static void test_rbtree_add_nodes(void) 13 14 { ··· 187 186 void test_rbtree_fail(void) 188 187 { 189 188 RUN_TESTS(rbtree_fail); 189 + } 190 + 191 + void test_rbtree_search(void) 192 + { 193 + RUN_TESTS(rbtree_search); 190 194 }
+113
tools/testing/selftests/bpf/progs/linked_list_peek.c
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ 3 + 4 + #include <vmlinux.h> 5 + #include <bpf/bpf_helpers.h> 6 + #include "bpf_misc.h" 7 + #include "bpf_experimental.h" 8 + 9 + struct node_data { 10 + struct bpf_list_node l; 11 + int key; 12 + }; 13 + 14 + #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) 15 + private(A) struct bpf_spin_lock glock; 16 + private(A) struct bpf_list_head ghead __contains(node_data, l); 17 + 18 + #define list_entry(ptr, type, member) container_of(ptr, type, member) 19 + #define NR_NODES 16 20 + 21 + int zero = 0; 22 + 23 + SEC("syscall") 24 + __retval(0) 25 + long list_peek(void *ctx) 26 + { 27 + struct bpf_list_node *l_n; 28 + struct node_data *n; 29 + int i, err = 0; 30 + 31 + bpf_spin_lock(&glock); 32 + l_n = bpf_list_front(&ghead); 33 + bpf_spin_unlock(&glock); 34 + if (l_n) 35 + return __LINE__; 36 + 37 + bpf_spin_lock(&glock); 38 + l_n = bpf_list_back(&ghead); 39 + bpf_spin_unlock(&glock); 40 + if (l_n) 41 + return __LINE__; 42 + 43 + for (i = zero; i < NR_NODES && can_loop; i++) { 44 + n = bpf_obj_new(typeof(*n)); 45 + if (!n) 46 + return __LINE__; 47 + n->key = i; 48 + bpf_spin_lock(&glock); 49 + bpf_list_push_back(&ghead, &n->l); 50 + bpf_spin_unlock(&glock); 51 + } 52 + 53 + bpf_spin_lock(&glock); 54 + 55 + l_n = bpf_list_front(&ghead); 56 + if (!l_n) { 57 + err = __LINE__; 58 + goto done; 59 + } 60 + 61 + n = list_entry(l_n, struct node_data, l); 62 + if (n->key != 0) { 63 + err = __LINE__; 64 + goto done; 65 + } 66 + 67 + l_n = bpf_list_back(&ghead); 68 + if (!l_n) { 69 + err = __LINE__; 70 + goto done; 71 + } 72 + 73 + n = list_entry(l_n, struct node_data, l); 74 + if (n->key != NR_NODES - 1) { 75 + err = __LINE__; 76 + goto done; 77 + } 78 + 79 + done: 80 + bpf_spin_unlock(&glock); 81 + return err; 82 + } 83 + 84 + #define TEST_FB(op, dolock) \ 85 + SEC("syscall") \ 86 + __failure __msg(MSG) \ 87 + long test_##op##_spinlock_##dolock(void *ctx) \ 88 + { \ 89 + struct bpf_list_node *l_n; \ 90 + __u64 jiffies = 0; \ 91 + \ 92 + if (dolock) \ 93 + bpf_spin_lock(&glock); \ 94 + l_n = bpf_list_##op(&ghead); \ 95 + if (l_n) \ 96 + jiffies = bpf_jiffies64(); \ 97 + if (dolock) \ 98 + bpf_spin_unlock(&glock); \ 99 + \ 100 + return !!jiffies; \ 101 + } 102 + 103 + #define MSG "call bpf_list_{{(front|back).+}}; R0{{(_w)?}}=ptr_or_null_node_data(id={{[0-9]+}},non_own_ref" 104 + TEST_FB(front, true) 105 + TEST_FB(back, true) 106 + #undef MSG 107 + 108 + #define MSG "bpf_spin_lock at off=0 must be held for bpf_list_head" 109 + TEST_FB(front, false) 110 + TEST_FB(back, false) 111 + #undef MSG 112 + 113 + char _license[] SEC("license") = "GPL";
+15 -14
tools/testing/selftests/bpf/progs/rbtree_fail.c
··· 69 69 } 70 70 71 71 SEC("?tc") 72 - __failure __msg("rbtree_remove node input must be non-owning ref") 72 + __retval(0) 73 73 long rbtree_api_remove_unadded_node(void *ctx) 74 74 { 75 75 struct node_data *n, *m; 76 - struct bpf_rb_node *res; 76 + struct bpf_rb_node *res_n, *res_m; 77 77 78 78 n = bpf_obj_new(typeof(*n)); 79 79 if (!n) ··· 88 88 bpf_spin_lock(&glock); 89 89 bpf_rbtree_add(&groot, &n->node, less); 90 90 91 - /* This remove should pass verifier */ 92 - res = bpf_rbtree_remove(&groot, &n->node); 93 - n = container_of(res, struct node_data, node); 91 + res_n = bpf_rbtree_remove(&groot, &n->node); 94 92 95 - /* This remove shouldn't, m isn't in an rbtree */ 96 - res = bpf_rbtree_remove(&groot, &m->node); 97 - m = container_of(res, struct node_data, node); 93 + res_m = bpf_rbtree_remove(&groot, &m->node); 98 94 bpf_spin_unlock(&glock); 99 95 100 - if (n) 101 - bpf_obj_drop(n); 102 - if (m) 103 - bpf_obj_drop(m); 96 + bpf_obj_drop(m); 97 + if (res_n) 98 + bpf_obj_drop(container_of(res_n, struct node_data, node)); 99 + if (res_m) { 100 + bpf_obj_drop(container_of(res_m, struct node_data, node)); 101 + /* m was not added to the rbtree */ 102 + return 2; 103 + } 104 + 104 105 return 0; 105 106 } 106 107 ··· 179 178 } 180 179 181 180 SEC("?tc") 182 - __failure __msg("rbtree_remove node input must be non-owning ref") 181 + __failure __msg("bpf_rbtree_remove can only take non-owning or refcounted bpf_rb_node pointer") 183 182 long rbtree_api_add_release_unlock_escape(void *ctx) 184 183 { 185 184 struct node_data *n; ··· 203 202 } 204 203 205 204 SEC("?tc") 206 - __failure __msg("rbtree_remove node input must be non-owning ref") 205 + __failure __msg("bpf_rbtree_remove can only take non-owning or refcounted bpf_rb_node pointer") 207 206 long rbtree_api_first_release_unlock_escape(void *ctx) 208 207 { 209 208 struct bpf_rb_node *res;
+206
tools/testing/selftests/bpf/progs/rbtree_search.c
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ 3 + 4 + #include <vmlinux.h> 5 + #include <bpf/bpf_helpers.h> 6 + #include "bpf_misc.h" 7 + #include "bpf_experimental.h" 8 + 9 + struct node_data { 10 + struct bpf_refcount ref; 11 + struct bpf_rb_node r0; 12 + struct bpf_rb_node r1; 13 + int key0; 14 + int key1; 15 + }; 16 + 17 + #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) 18 + private(A) struct bpf_spin_lock glock0; 19 + private(A) struct bpf_rb_root groot0 __contains(node_data, r0); 20 + 21 + private(B) struct bpf_spin_lock glock1; 22 + private(B) struct bpf_rb_root groot1 __contains(node_data, r1); 23 + 24 + #define rb_entry(ptr, type, member) container_of(ptr, type, member) 25 + #define NR_NODES 16 26 + 27 + int zero = 0; 28 + 29 + static bool less0(struct bpf_rb_node *a, const struct bpf_rb_node *b) 30 + { 31 + struct node_data *node_a; 32 + struct node_data *node_b; 33 + 34 + node_a = rb_entry(a, struct node_data, r0); 35 + node_b = rb_entry(b, struct node_data, r0); 36 + 37 + return node_a->key0 < node_b->key0; 38 + } 39 + 40 + static bool less1(struct bpf_rb_node *a, const struct bpf_rb_node *b) 41 + { 42 + struct node_data *node_a; 43 + struct node_data *node_b; 44 + 45 + node_a = rb_entry(a, struct node_data, r1); 46 + node_b = rb_entry(b, struct node_data, r1); 47 + 48 + return node_a->key1 < node_b->key1; 49 + } 50 + 51 + SEC("syscall") 52 + __retval(0) 53 + long rbtree_search(void *ctx) 54 + { 55 + struct bpf_rb_node *rb_n, *rb_m, *gc_ns[NR_NODES]; 56 + long lookup_key = NR_NODES / 2; 57 + struct node_data *n, *m; 58 + int i, nr_gc = 0; 59 + 60 + for (i = zero; i < NR_NODES && can_loop; i++) { 61 + n = bpf_obj_new(typeof(*n)); 62 + if (!n) 63 + return __LINE__; 64 + 65 + m = bpf_refcount_acquire(n); 66 + 67 + n->key0 = i; 68 + m->key1 = i; 69 + 70 + bpf_spin_lock(&glock0); 71 + bpf_rbtree_add(&groot0, &n->r0, less0); 72 + bpf_spin_unlock(&glock0); 73 + 74 + bpf_spin_lock(&glock1); 75 + bpf_rbtree_add(&groot1, &m->r1, less1); 76 + bpf_spin_unlock(&glock1); 77 + } 78 + 79 + n = NULL; 80 + bpf_spin_lock(&glock0); 81 + rb_n = bpf_rbtree_root(&groot0); 82 + while (can_loop) { 83 + if (!rb_n) { 84 + bpf_spin_unlock(&glock0); 85 + return __LINE__; 86 + } 87 + 88 + n = rb_entry(rb_n, struct node_data, r0); 89 + if (lookup_key == n->key0) 90 + break; 91 + if (nr_gc < NR_NODES) 92 + gc_ns[nr_gc++] = rb_n; 93 + if (lookup_key < n->key0) 94 + rb_n = bpf_rbtree_left(&groot0, rb_n); 95 + else 96 + rb_n = bpf_rbtree_right(&groot0, rb_n); 97 + } 98 + 99 + if (!n || lookup_key != n->key0) { 100 + bpf_spin_unlock(&glock0); 101 + return __LINE__; 102 + } 103 + 104 + for (i = 0; i < nr_gc; i++) { 105 + rb_n = gc_ns[i]; 106 + gc_ns[i] = bpf_rbtree_remove(&groot0, rb_n); 107 + } 108 + 109 + m = bpf_refcount_acquire(n); 110 + bpf_spin_unlock(&glock0); 111 + 112 + for (i = 0; i < nr_gc; i++) { 113 + rb_n = gc_ns[i]; 114 + if (rb_n) { 115 + n = rb_entry(rb_n, struct node_data, r0); 116 + bpf_obj_drop(n); 117 + } 118 + } 119 + 120 + if (!m) 121 + return __LINE__; 122 + 123 + bpf_spin_lock(&glock1); 124 + rb_m = bpf_rbtree_remove(&groot1, &m->r1); 125 + bpf_spin_unlock(&glock1); 126 + bpf_obj_drop(m); 127 + if (!rb_m) 128 + return __LINE__; 129 + bpf_obj_drop(rb_entry(rb_m, struct node_data, r1)); 130 + 131 + return 0; 132 + } 133 + 134 + #define TEST_ROOT(dolock) \ 135 + SEC("syscall") \ 136 + __failure __msg(MSG) \ 137 + long test_root_spinlock_##dolock(void *ctx) \ 138 + { \ 139 + struct bpf_rb_node *rb_n; \ 140 + __u64 jiffies = 0; \ 141 + \ 142 + if (dolock) \ 143 + bpf_spin_lock(&glock0); \ 144 + rb_n = bpf_rbtree_root(&groot0); \ 145 + if (rb_n) \ 146 + jiffies = bpf_jiffies64(); \ 147 + if (dolock) \ 148 + bpf_spin_unlock(&glock0); \ 149 + \ 150 + return !!jiffies; \ 151 + } 152 + 153 + #define TEST_LR(op, dolock) \ 154 + SEC("syscall") \ 155 + __failure __msg(MSG) \ 156 + long test_##op##_spinlock_##dolock(void *ctx) \ 157 + { \ 158 + struct bpf_rb_node *rb_n; \ 159 + struct node_data *n; \ 160 + __u64 jiffies = 0; \ 161 + \ 162 + bpf_spin_lock(&glock0); \ 163 + rb_n = bpf_rbtree_root(&groot0); \ 164 + if (!rb_n) { \ 165 + bpf_spin_unlock(&glock0); \ 166 + return 1; \ 167 + } \ 168 + n = rb_entry(rb_n, struct node_data, r0); \ 169 + n = bpf_refcount_acquire(n); \ 170 + bpf_spin_unlock(&glock0); \ 171 + if (!n) \ 172 + return 1; \ 173 + \ 174 + if (dolock) \ 175 + bpf_spin_lock(&glock0); \ 176 + rb_n = bpf_rbtree_##op(&groot0, &n->r0); \ 177 + if (rb_n) \ 178 + jiffies = bpf_jiffies64(); \ 179 + if (dolock) \ 180 + bpf_spin_unlock(&glock0); \ 181 + \ 182 + return !!jiffies; \ 183 + } 184 + 185 + /* 186 + * Use a spearate MSG macro instead of passing to TEST_XXX(..., MSG) 187 + * to ensure the message itself is not in the bpf prog lineinfo 188 + * which the verifier includes in its log. 189 + * Otherwise, the test_loader will incorrectly match the prog lineinfo 190 + * instead of the log generated by the verifier. 191 + */ 192 + #define MSG "call bpf_rbtree_root{{.+}}; R0{{(_w)?}}=rcu_ptr_or_null_node_data(id={{[0-9]+}},non_own_ref" 193 + TEST_ROOT(true) 194 + #undef MSG 195 + #define MSG "call bpf_rbtree_{{(left|right).+}}; R0{{(_w)?}}=rcu_ptr_or_null_node_data(id={{[0-9]+}},non_own_ref" 196 + TEST_LR(left, true) 197 + TEST_LR(right, true) 198 + #undef MSG 199 + 200 + #define MSG "bpf_spin_lock at off=0 must be held for bpf_rb_root" 201 + TEST_ROOT(false) 202 + TEST_LR(left, false) 203 + TEST_LR(right, false) 204 + #undef MSG 205 + 206 + char _license[] SEC("license") = "GPL";