Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>
5#include "bpf_misc.h"
6#include "bpf_experimental.h"
7
8unsigned long global_flags;
9
10extern void bpf_local_irq_save(unsigned long *) __weak __ksym;
11extern void bpf_local_irq_restore(unsigned long *) __weak __ksym;
12extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym;
13
14struct bpf_res_spin_lock lockA __hidden SEC(".data.A");
15struct bpf_res_spin_lock lockB __hidden SEC(".data.B");
16
17SEC("?tc")
18__failure __msg("arg#0 doesn't point to an irq flag on stack")
19int irq_save_bad_arg(struct __sk_buff *ctx)
20{
21 bpf_local_irq_save(&global_flags);
22 return 0;
23}
24
25SEC("?tc")
26__failure __msg("arg#0 doesn't point to an irq flag on stack")
27int irq_restore_bad_arg(struct __sk_buff *ctx)
28{
29 bpf_local_irq_restore(&global_flags);
30 return 0;
31}
32
33SEC("?tc")
34__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
35int irq_restore_missing_2(struct __sk_buff *ctx)
36{
37 unsigned long flags1;
38 unsigned long flags2;
39
40 bpf_local_irq_save(&flags1);
41 bpf_local_irq_save(&flags2);
42 return 0;
43}
44
45SEC("?tc")
46__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
47int irq_restore_missing_3(struct __sk_buff *ctx)
48{
49 unsigned long flags1;
50 unsigned long flags2;
51 unsigned long flags3;
52
53 bpf_local_irq_save(&flags1);
54 bpf_local_irq_save(&flags2);
55 bpf_local_irq_save(&flags3);
56 return 0;
57}
58
59SEC("?tc")
60__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
61int irq_restore_missing_3_minus_2(struct __sk_buff *ctx)
62{
63 unsigned long flags1;
64 unsigned long flags2;
65 unsigned long flags3;
66
67 bpf_local_irq_save(&flags1);
68 bpf_local_irq_save(&flags2);
69 bpf_local_irq_save(&flags3);
70 bpf_local_irq_restore(&flags3);
71 bpf_local_irq_restore(&flags2);
72 return 0;
73}
74
75static __noinline void local_irq_save(unsigned long *flags)
76{
77 bpf_local_irq_save(flags);
78}
79
80static __noinline void local_irq_restore(unsigned long *flags)
81{
82 bpf_local_irq_restore(flags);
83}
84
85SEC("?tc")
86__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
87int irq_restore_missing_1_subprog(struct __sk_buff *ctx)
88{
89 unsigned long flags;
90
91 local_irq_save(&flags);
92 return 0;
93}
94
95SEC("?tc")
96__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
97int irq_restore_missing_2_subprog(struct __sk_buff *ctx)
98{
99 unsigned long flags1;
100 unsigned long flags2;
101
102 local_irq_save(&flags1);
103 local_irq_save(&flags2);
104 return 0;
105}
106
107SEC("?tc")
108__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
109int irq_restore_missing_3_subprog(struct __sk_buff *ctx)
110{
111 unsigned long flags1;
112 unsigned long flags2;
113 unsigned long flags3;
114
115 local_irq_save(&flags1);
116 local_irq_save(&flags2);
117 local_irq_save(&flags3);
118 return 0;
119}
120
121SEC("?tc")
122__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
123int irq_restore_missing_3_minus_2_subprog(struct __sk_buff *ctx)
124{
125 unsigned long flags1;
126 unsigned long flags2;
127 unsigned long flags3;
128
129 local_irq_save(&flags1);
130 local_irq_save(&flags2);
131 local_irq_save(&flags3);
132 local_irq_restore(&flags3);
133 local_irq_restore(&flags2);
134 return 0;
135}
136
137SEC("?tc")
138__success
139int irq_balance(struct __sk_buff *ctx)
140{
141 unsigned long flags;
142
143 local_irq_save(&flags);
144 local_irq_restore(&flags);
145 return 0;
146}
147
148SEC("?tc")
149__success
150int irq_balance_n(struct __sk_buff *ctx)
151{
152 unsigned long flags1;
153 unsigned long flags2;
154 unsigned long flags3;
155
156 local_irq_save(&flags1);
157 local_irq_save(&flags2);
158 local_irq_save(&flags3);
159 local_irq_restore(&flags3);
160 local_irq_restore(&flags2);
161 local_irq_restore(&flags1);
162 return 0;
163}
164
165static __noinline void local_irq_balance(void)
166{
167 unsigned long flags;
168
169 local_irq_save(&flags);
170 local_irq_restore(&flags);
171}
172
173static __noinline void local_irq_balance_n(void)
174{
175 unsigned long flags1;
176 unsigned long flags2;
177 unsigned long flags3;
178
179 local_irq_save(&flags1);
180 local_irq_save(&flags2);
181 local_irq_save(&flags3);
182 local_irq_restore(&flags3);
183 local_irq_restore(&flags2);
184 local_irq_restore(&flags1);
185}
186
187SEC("?tc")
188__success
189int irq_balance_subprog(struct __sk_buff *ctx)
190{
191 local_irq_balance();
192 return 0;
193}
194
195SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
196__failure __msg("sleepable helper bpf_copy_from_user#")
197int irq_sleepable_helper(void *ctx)
198{
199 unsigned long flags;
200 u32 data;
201
202 local_irq_save(&flags);
203 bpf_copy_from_user(&data, sizeof(data), NULL);
204 local_irq_restore(&flags);
205 return 0;
206}
207
208SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
209__failure __msg("kernel func bpf_copy_from_user_str is sleepable within IRQ-disabled region")
210int irq_sleepable_kfunc(void *ctx)
211{
212 unsigned long flags;
213 u32 data;
214
215 local_irq_save(&flags);
216 bpf_copy_from_user_str(&data, sizeof(data), NULL, 0);
217 local_irq_restore(&flags);
218 return 0;
219}
220
221int __noinline global_local_irq_balance(void)
222{
223 local_irq_balance_n();
224 return 0;
225}
226
227SEC("?tc")
228__success
229int irq_global_subprog(struct __sk_buff *ctx)
230{
231 unsigned long flags;
232
233 bpf_local_irq_save(&flags);
234 global_local_irq_balance();
235 bpf_local_irq_restore(&flags);
236 return 0;
237}
238
239SEC("?tc")
240__failure __msg("cannot restore irq state out of order")
241int irq_restore_ooo(struct __sk_buff *ctx)
242{
243 unsigned long flags1;
244 unsigned long flags2;
245
246 bpf_local_irq_save(&flags1);
247 bpf_local_irq_save(&flags2);
248 bpf_local_irq_restore(&flags1);
249 bpf_local_irq_restore(&flags2);
250 return 0;
251}
252
253SEC("?tc")
254__failure __msg("cannot restore irq state out of order")
255int irq_restore_ooo_3(struct __sk_buff *ctx)
256{
257 unsigned long flags1;
258 unsigned long flags2;
259 unsigned long flags3;
260
261 bpf_local_irq_save(&flags1);
262 bpf_local_irq_save(&flags2);
263 bpf_local_irq_restore(&flags2);
264 bpf_local_irq_save(&flags3);
265 bpf_local_irq_restore(&flags1);
266 bpf_local_irq_restore(&flags3);
267 return 0;
268}
269
270static __noinline void local_irq_save_3(unsigned long *flags1, unsigned long *flags2,
271 unsigned long *flags3)
272{
273 local_irq_save(flags1);
274 local_irq_save(flags2);
275 local_irq_save(flags3);
276}
277
278SEC("?tc")
279__success
280int irq_restore_3_subprog(struct __sk_buff *ctx)
281{
282 unsigned long flags1;
283 unsigned long flags2;
284 unsigned long flags3;
285
286 local_irq_save_3(&flags1, &flags2, &flags3);
287 bpf_local_irq_restore(&flags3);
288 bpf_local_irq_restore(&flags2);
289 bpf_local_irq_restore(&flags1);
290 return 0;
291}
292
293SEC("?tc")
294__failure __msg("cannot restore irq state out of order")
295int irq_restore_4_subprog(struct __sk_buff *ctx)
296{
297 unsigned long flags1;
298 unsigned long flags2;
299 unsigned long flags3;
300 unsigned long flags4;
301
302 local_irq_save_3(&flags1, &flags2, &flags3);
303 bpf_local_irq_restore(&flags3);
304 bpf_local_irq_save(&flags4);
305 bpf_local_irq_restore(&flags4);
306 bpf_local_irq_restore(&flags1);
307 return 0;
308}
309
310SEC("?tc")
311__failure __msg("cannot restore irq state out of order")
312int irq_restore_ooo_3_subprog(struct __sk_buff *ctx)
313{
314 unsigned long flags1;
315 unsigned long flags2;
316 unsigned long flags3;
317
318 local_irq_save_3(&flags1, &flags2, &flags3);
319 bpf_local_irq_restore(&flags3);
320 bpf_local_irq_restore(&flags2);
321 bpf_local_irq_save(&flags3);
322 bpf_local_irq_restore(&flags1);
323 return 0;
324}
325
326SEC("?tc")
327__failure __msg("expected an initialized")
328int irq_restore_invalid(struct __sk_buff *ctx)
329{
330 unsigned long flags1;
331 unsigned long flags = 0xfaceb00c;
332
333 bpf_local_irq_save(&flags1);
334 bpf_local_irq_restore(&flags);
335 return 0;
336}
337
338SEC("?tc")
339__failure __msg("expected uninitialized")
340int irq_save_invalid(struct __sk_buff *ctx)
341{
342 unsigned long flags1;
343
344 bpf_local_irq_save(&flags1);
345 bpf_local_irq_save(&flags1);
346 return 0;
347}
348
349SEC("?tc")
350__failure __msg("expected an initialized")
351int irq_restore_iter(struct __sk_buff *ctx)
352{
353 struct bpf_iter_num it;
354
355 bpf_iter_num_new(&it, 0, 42);
356 bpf_local_irq_restore((unsigned long *)&it);
357 return 0;
358}
359
360SEC("?tc")
361__failure __msg("Unreleased reference id=1")
362int irq_save_iter(struct __sk_buff *ctx)
363{
364 struct bpf_iter_num it;
365
366 /* Ensure same sized slot has st->ref_obj_id set, so we reject based on
367 * slot_type != STACK_IRQ_FLAG...
368 */
369 _Static_assert(sizeof(it) == sizeof(unsigned long), "broken iterator size");
370
371 bpf_iter_num_new(&it, 0, 42);
372 bpf_local_irq_save((unsigned long *)&it);
373 bpf_local_irq_restore((unsigned long *)&it);
374 return 0;
375}
376
377SEC("?tc")
378__failure __msg("expected an initialized")
379int irq_flag_overwrite(struct __sk_buff *ctx)
380{
381 unsigned long flags;
382
383 bpf_local_irq_save(&flags);
384 flags = 0xdeadbeef;
385 bpf_local_irq_restore(&flags);
386 return 0;
387}
388
389SEC("?tc")
390__failure __msg("expected an initialized")
391int irq_flag_overwrite_partial(struct __sk_buff *ctx)
392{
393 unsigned long flags;
394
395 bpf_local_irq_save(&flags);
396 *(((char *)&flags) + 1) = 0xff;
397 bpf_local_irq_restore(&flags);
398 return 0;
399}
400
401SEC("?tc")
402__failure __msg("cannot restore irq state out of order")
403int irq_ooo_refs_array(struct __sk_buff *ctx)
404{
405 unsigned long flags[4];
406 struct { int i; } *p;
407
408 /* refs=1 */
409 bpf_local_irq_save(&flags[0]);
410
411 /* refs=1,2 */
412 p = bpf_obj_new(typeof(*p));
413 if (!p) {
414 bpf_local_irq_restore(&flags[0]);
415 return 0;
416 }
417
418 /* refs=1,2,3 */
419 bpf_local_irq_save(&flags[1]);
420
421 /* refs=1,2,3,4 */
422 bpf_local_irq_save(&flags[2]);
423
424 /* Now when we remove ref=2, the verifier must not break the ordering in
425 * the refs array between 1,3,4. With an older implementation, the
426 * verifier would swap the last element with the removed element, but to
427 * maintain the stack property we need to use memmove.
428 */
429 bpf_obj_drop(p);
430
431 /* Save and restore to reset active_irq_id to 3, as the ordering is now
432 * refs=1,4,3. When restoring the linear scan will find prev_id in order
433 * as 3 instead of 4.
434 */
435 bpf_local_irq_save(&flags[3]);
436 bpf_local_irq_restore(&flags[3]);
437
438 /* With the incorrect implementation, we can release flags[1], flags[2],
439 * and flags[0], i.e. in the wrong order.
440 */
441 bpf_local_irq_restore(&flags[1]);
442 bpf_local_irq_restore(&flags[2]);
443 bpf_local_irq_restore(&flags[0]);
444 return 0;
445}
446
447int __noinline
448global_subprog(int i)
449{
450 if (i)
451 bpf_printk("%p", &i);
452 return i;
453}
454
455int __noinline
456global_sleepable_helper_subprog(int i)
457{
458 if (i)
459 bpf_copy_from_user(&i, sizeof(i), NULL);
460 return i;
461}
462
463int __noinline
464global_sleepable_kfunc_subprog(int i)
465{
466 if (i)
467 bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
468 global_subprog(i);
469 return i;
470}
471
472int __noinline
473global_subprog_calling_sleepable_global(int i)
474{
475 if (!i)
476 global_sleepable_kfunc_subprog(i);
477 return i;
478}
479
480SEC("?syscall")
481__success
482int irq_non_sleepable_global_subprog(void *ctx)
483{
484 unsigned long flags;
485
486 bpf_local_irq_save(&flags);
487 global_subprog(0);
488 bpf_local_irq_restore(&flags);
489 return 0;
490}
491
492SEC("?syscall")
493__failure __msg("sleepable global function")
494int irq_sleepable_helper_global_subprog(void *ctx)
495{
496 unsigned long flags;
497
498 bpf_local_irq_save(&flags);
499 global_sleepable_helper_subprog(0);
500 bpf_local_irq_restore(&flags);
501 return 0;
502}
503
504SEC("?syscall")
505__failure __msg("sleepable global function")
506int irq_sleepable_global_subprog_indirect(void *ctx)
507{
508 unsigned long flags;
509
510 bpf_local_irq_save(&flags);
511 global_subprog_calling_sleepable_global(0);
512 bpf_local_irq_restore(&flags);
513 return 0;
514}
515
516SEC("?tc")
517__failure __msg("cannot restore irq state out of order")
518int irq_ooo_lock_cond_inv(struct __sk_buff *ctx)
519{
520 unsigned long flags1, flags2;
521
522 if (bpf_res_spin_lock_irqsave(&lockA, &flags1))
523 return 0;
524 if (bpf_res_spin_lock_irqsave(&lockB, &flags2)) {
525 bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
526 return 0;
527 }
528
529 bpf_res_spin_unlock_irqrestore(&lockB, &flags1);
530 bpf_res_spin_unlock_irqrestore(&lockA, &flags2);
531 return 0;
532}
533
534SEC("?tc")
535__failure __msg("function calls are not allowed")
536int irq_wrong_kfunc_class_1(struct __sk_buff *ctx)
537{
538 unsigned long flags1;
539
540 if (bpf_res_spin_lock_irqsave(&lockA, &flags1))
541 return 0;
542 /* For now, bpf_local_irq_restore is not allowed in critical section,
543 * but this test ensures error will be caught with kfunc_class when it's
544 * opened up. Tested by temporarily permitting this kfunc in critical
545 * section.
546 */
547 bpf_local_irq_restore(&flags1);
548 bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
549 return 0;
550}
551
552SEC("?tc")
553__failure __msg("function calls are not allowed")
554int irq_wrong_kfunc_class_2(struct __sk_buff *ctx)
555{
556 unsigned long flags1, flags2;
557
558 bpf_local_irq_save(&flags1);
559 if (bpf_res_spin_lock_irqsave(&lockA, &flags2))
560 return 0;
561 bpf_local_irq_restore(&flags2);
562 bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
563 return 0;
564}
565
566char _license[] SEC("license") = "GPL";