Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3
4#include "vmlinux.h"
5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_tracing.h>
7#include "bpf_tracing_net.h"
8#include "bpf_misc.h"
9
10/* clang considers 'sum += 1' as usage but 'sum++' as non-usage. GCC
11 * is more consistent and considers both 'sum += 1' and 'sum++' as
12 * non-usage. This triggers warnings in the functions below.
13 *
14 * Starting with GCC 16 -Wunused-but-set-variable=2 can be used to
15 * mimic clang's behavior. */
16#if !defined(__clang__) && __GNUC__ > 15
17#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
18#endif
19
20char _license[] SEC("license") = "GPL";
21
22struct {
23 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
24 __uint(map_flags, BPF_F_NO_PREALLOC);
25 __type(key, int);
26 __type(value, long);
27} map_a SEC(".maps");
28
29__u32 user_data, target_pid;
30__s32 key_serial;
31__u64 flags, task_storage_val, cgroup_id;
32
33struct bpf_key *bpf_lookup_user_key(__s32 serial, __u64 flags) __ksym;
34void bpf_key_put(struct bpf_key *key) __ksym;
35void bpf_rcu_read_lock(void) __ksym;
36void bpf_rcu_read_unlock(void) __ksym;
37struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
38void bpf_task_release(struct task_struct *p) __ksym;
39
40SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
41int get_cgroup_id(void *ctx)
42{
43 struct task_struct *task;
44 struct css_set *cgroups;
45
46 task = bpf_get_current_task_btf();
47 if (task->pid != target_pid)
48 return 0;
49
50 /* simulate bpf_get_current_cgroup_id() helper */
51 bpf_rcu_read_lock();
52 cgroups = task->cgroups;
53 if (!cgroups)
54 goto unlock;
55 cgroup_id = cgroups->dfl_cgrp->kn->id;
56unlock:
57 bpf_rcu_read_unlock();
58 return 0;
59}
60
61SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
62int task_succ(void *ctx)
63{
64 struct task_struct *task, *real_parent;
65 long init_val = 2;
66 long *ptr;
67
68 task = bpf_get_current_task_btf();
69 if (task->pid != target_pid)
70 return 0;
71
72 bpf_rcu_read_lock();
73 /* region including helper using rcu ptr real_parent */
74 real_parent = task->real_parent;
75 if (!real_parent)
76 goto out;
77 ptr = bpf_task_storage_get(&map_a, real_parent, &init_val,
78 BPF_LOCAL_STORAGE_GET_F_CREATE);
79 if (!ptr)
80 goto out;
81 ptr = bpf_task_storage_get(&map_a, real_parent, 0, 0);
82 if (!ptr)
83 goto out;
84 task_storage_val = *ptr;
85out:
86 bpf_rcu_read_unlock();
87 return 0;
88}
89
90SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
91int no_lock(void *ctx)
92{
93 struct task_struct *task, *real_parent;
94
95 /* old style ptr_to_btf_id is not allowed in sleepable */
96 task = bpf_get_current_task_btf();
97 real_parent = task->real_parent;
98 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
99 return 0;
100}
101
102SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
103int two_regions(void *ctx)
104{
105 struct task_struct *task, *real_parent;
106
107 /* two regions */
108 task = bpf_get_current_task_btf();
109 bpf_rcu_read_lock();
110 bpf_rcu_read_unlock();
111 bpf_rcu_read_lock();
112 real_parent = task->real_parent;
113 if (!real_parent)
114 goto out;
115 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
116out:
117 bpf_rcu_read_unlock();
118 return 0;
119}
120
121SEC("?fentry/" SYS_PREFIX "sys_getpgid")
122int non_sleepable_1(void *ctx)
123{
124 struct task_struct *task, *real_parent;
125
126 task = bpf_get_current_task_btf();
127 bpf_rcu_read_lock();
128 real_parent = task->real_parent;
129 if (!real_parent)
130 goto out;
131 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
132out:
133 bpf_rcu_read_unlock();
134 return 0;
135}
136
137SEC("?fentry/" SYS_PREFIX "sys_getpgid")
138int non_sleepable_2(void *ctx)
139{
140 struct task_struct *task, *real_parent;
141
142 bpf_rcu_read_lock();
143 task = bpf_get_current_task_btf();
144 bpf_rcu_read_unlock();
145
146 bpf_rcu_read_lock();
147 real_parent = task->real_parent;
148 if (!real_parent)
149 goto out;
150 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
151out:
152 bpf_rcu_read_unlock();
153 return 0;
154}
155
156SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
157int task_acquire(void *ctx)
158{
159 struct task_struct *task, *real_parent, *gparent;
160
161 task = bpf_get_current_task_btf();
162 bpf_rcu_read_lock();
163 real_parent = task->real_parent;
164 if (!real_parent)
165 goto out;
166
167 /* rcu_ptr->rcu_field */
168 gparent = real_parent->real_parent;
169 if (!gparent)
170 goto out;
171
172 /* acquire a reference which can be used outside rcu read lock region */
173 gparent = bpf_task_acquire(gparent);
174 if (!gparent)
175 goto out;
176
177 (void)bpf_task_storage_get(&map_a, gparent, 0, 0);
178 bpf_task_release(gparent);
179out:
180 bpf_rcu_read_unlock();
181 return 0;
182}
183
184SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
185int miss_lock(void *ctx)
186{
187 struct task_struct *task;
188
189 /* missing bpf_rcu_read_lock() */
190 task = bpf_get_current_task_btf();
191 bpf_rcu_read_lock();
192 (void)bpf_task_storage_get(&map_a, task, 0, 0);
193 bpf_rcu_read_unlock();
194 bpf_rcu_read_unlock();
195 return 0;
196}
197
198SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
199int miss_unlock(void *ctx)
200{
201 struct task_struct *task;
202
203 /* missing bpf_rcu_read_unlock() */
204 task = bpf_get_current_task_btf();
205 bpf_rcu_read_lock();
206 (void)bpf_task_storage_get(&map_a, task, 0, 0);
207 return 0;
208}
209
210SEC("?fentry/" SYS_PREFIX "sys_getpgid")
211int non_sleepable_rcu_mismatch(void *ctx)
212{
213 struct task_struct *task, *real_parent;
214
215 task = bpf_get_current_task_btf();
216 /* non-sleepable: missing bpf_rcu_read_unlock() in one path */
217 bpf_rcu_read_lock();
218 real_parent = task->real_parent;
219 if (!real_parent)
220 goto out;
221 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
222 if (real_parent)
223 bpf_rcu_read_unlock();
224out:
225 return 0;
226}
227
228SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
229int inproper_sleepable_helper(void *ctx)
230{
231 struct task_struct *task, *real_parent;
232 struct pt_regs *regs;
233 __u32 value = 0;
234 void *ptr;
235
236 task = bpf_get_current_task_btf();
237 /* sleepable helper in rcu read lock region */
238 bpf_rcu_read_lock();
239 real_parent = task->real_parent;
240 if (!real_parent)
241 goto out;
242 regs = (struct pt_regs *)bpf_task_pt_regs(real_parent);
243 if (!regs)
244 goto out;
245
246 ptr = (void *)PT_REGS_IP(regs);
247 (void)bpf_copy_from_user_task(&value, sizeof(uint32_t), ptr, task, 0);
248 user_data = value;
249 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
250out:
251 bpf_rcu_read_unlock();
252 return 0;
253}
254
255SEC("?lsm.s/bpf")
256int BPF_PROG(inproper_sleepable_kfunc, int cmd, union bpf_attr *attr, unsigned int size,
257 bool kernel)
258{
259 struct bpf_key *bkey;
260
261 /* sleepable kfunc in rcu read lock region */
262 bpf_rcu_read_lock();
263 bkey = bpf_lookup_user_key(key_serial, flags);
264 bpf_rcu_read_unlock();
265 if (!bkey)
266 return -1;
267 bpf_key_put(bkey);
268
269 return 0;
270}
271
272SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
273int nested_rcu_region(void *ctx)
274{
275 struct task_struct *task, *real_parent;
276
277 /* nested rcu read lock regions */
278 task = bpf_get_current_task_btf();
279 bpf_rcu_read_lock();
280 bpf_rcu_read_lock();
281 real_parent = task->real_parent;
282 if (!real_parent)
283 goto out;
284 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
285out:
286 bpf_rcu_read_unlock();
287 bpf_rcu_read_unlock();
288 return 0;
289}
290
291SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
292int nested_rcu_region_unbalanced_1(void *ctx)
293{
294 struct task_struct *task, *real_parent;
295
296 /* nested rcu read lock regions */
297 task = bpf_get_current_task_btf();
298 bpf_rcu_read_lock();
299 bpf_rcu_read_lock();
300 real_parent = task->real_parent;
301 if (!real_parent)
302 goto out;
303 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
304out:
305 bpf_rcu_read_unlock();
306 bpf_rcu_read_unlock();
307 bpf_rcu_read_unlock();
308 return 0;
309}
310
311SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
312int nested_rcu_region_unbalanced_2(void *ctx)
313{
314 struct task_struct *task, *real_parent;
315
316 /* nested rcu read lock regions */
317 task = bpf_get_current_task_btf();
318 bpf_rcu_read_lock();
319 bpf_rcu_read_lock();
320 bpf_rcu_read_lock();
321 real_parent = task->real_parent;
322 if (!real_parent)
323 goto out;
324 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
325out:
326 bpf_rcu_read_unlock();
327 bpf_rcu_read_unlock();
328 return 0;
329}
330
331SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
332int task_trusted_non_rcuptr(void *ctx)
333{
334 struct task_struct *task, *group_leader;
335
336 task = bpf_get_current_task_btf();
337 bpf_rcu_read_lock();
338 /* the pointer group_leader is explicitly marked as trusted */
339 group_leader = task->real_parent->group_leader;
340 (void)bpf_task_storage_get(&map_a, group_leader, 0, 0);
341 bpf_rcu_read_unlock();
342 return 0;
343}
344
345SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
346int task_untrusted_rcuptr(void *ctx)
347{
348 struct task_struct *task, *real_parent;
349
350 task = bpf_get_current_task_btf();
351 bpf_rcu_read_lock();
352 real_parent = task->real_parent;
353 bpf_rcu_read_unlock();
354 /* helper use of rcu ptr outside the rcu read lock region */
355 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
356 return 0;
357}
358
359SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
360int cross_rcu_region(void *ctx)
361{
362 struct task_struct *task, *real_parent;
363
364 /* rcu ptr define/use in different regions */
365 task = bpf_get_current_task_btf();
366 bpf_rcu_read_lock();
367 real_parent = task->real_parent;
368 bpf_rcu_read_unlock();
369 bpf_rcu_read_lock();
370 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
371 bpf_rcu_read_unlock();
372 return 0;
373}
374
375__noinline
376static int static_subprog(void *ctx)
377{
378 volatile int ret = 0;
379
380 if (bpf_get_prandom_u32())
381 return ret + 42;
382 return ret + bpf_get_prandom_u32();
383}
384
385__noinline
386int global_subprog(u64 a)
387{
388 volatile int ret = a;
389
390 return ret + static_subprog(NULL);
391}
392
393__noinline
394static int static_subprog_lock(void *ctx)
395{
396 volatile int ret = 0;
397
398 bpf_rcu_read_lock();
399 if (bpf_get_prandom_u32())
400 return ret + 42;
401 return ret + bpf_get_prandom_u32();
402}
403
404__noinline
405int global_subprog_lock(u64 a)
406{
407 volatile int ret = a;
408
409 return ret + static_subprog_lock(NULL);
410}
411
412__noinline
413static int static_subprog_unlock(void *ctx)
414{
415 volatile int ret = 0;
416
417 bpf_rcu_read_unlock();
418 if (bpf_get_prandom_u32())
419 return ret + 42;
420 return ret + bpf_get_prandom_u32();
421}
422
423__noinline
424int global_subprog_unlock(u64 a)
425{
426 volatile int ret = a;
427
428 return ret + static_subprog_unlock(NULL);
429}
430
431SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
432int rcu_read_lock_subprog(void *ctx)
433{
434 volatile int ret = 0;
435
436 bpf_rcu_read_lock();
437 if (bpf_get_prandom_u32())
438 ret += static_subprog(ctx);
439 bpf_rcu_read_unlock();
440 return 0;
441}
442
443SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
444int rcu_read_lock_global_subprog(void *ctx)
445{
446 volatile int ret = 0;
447
448 bpf_rcu_read_lock();
449 if (bpf_get_prandom_u32())
450 ret += global_subprog(ret);
451 bpf_rcu_read_unlock();
452 return 0;
453}
454
455SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
456int rcu_read_lock_subprog_lock(void *ctx)
457{
458 volatile int ret = 0;
459
460 ret += static_subprog_lock(ctx);
461 bpf_rcu_read_unlock();
462 return 0;
463}
464
465SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
466int rcu_read_lock_global_subprog_lock(void *ctx)
467{
468 volatile int ret = 0;
469
470 ret += global_subprog_lock(ret);
471 bpf_rcu_read_unlock();
472 return 0;
473}
474
475SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
476int rcu_read_lock_subprog_unlock(void *ctx)
477{
478 volatile int ret = 0;
479
480 bpf_rcu_read_lock();
481 ret += static_subprog_unlock(ctx);
482 return 0;
483}
484
485SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
486int rcu_read_lock_global_subprog_unlock(void *ctx)
487{
488 volatile int ret = 0;
489
490 bpf_rcu_read_lock();
491 ret += global_subprog_unlock(ret);
492 return 0;
493}
494
495int __noinline
496global_sleepable_helper_subprog(int i)
497{
498 if (i)
499 bpf_copy_from_user(&i, sizeof(i), NULL);
500 return i;
501}
502
503int __noinline
504global_sleepable_kfunc_subprog(int i)
505{
506 if (i)
507 bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
508 global_subprog(i);
509 return i;
510}
511
512int __noinline
513global_subprog_calling_sleepable_global(int i)
514{
515 if (!i)
516 global_sleepable_kfunc_subprog(i);
517 return i;
518}
519
520SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
521int rcu_read_lock_sleepable_helper_global_subprog(void *ctx)
522{
523 volatile int ret = 0;
524
525 bpf_rcu_read_lock();
526 ret += global_sleepable_helper_subprog(ret);
527 bpf_rcu_read_unlock();
528 return 0;
529}
530
531SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
532int rcu_read_lock_sleepable_kfunc_global_subprog(void *ctx)
533{
534 volatile int ret = 0;
535
536 bpf_rcu_read_lock();
537 ret += global_sleepable_kfunc_subprog(ret);
538 bpf_rcu_read_unlock();
539 return 0;
540}
541
542SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
543int rcu_read_lock_sleepable_global_subprog_indirect(void *ctx)
544{
545 volatile int ret = 0;
546
547 bpf_rcu_read_lock();
548 ret += global_subprog_calling_sleepable_global(ret);
549 bpf_rcu_read_unlock();
550 return 0;
551}