Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf
Pull bpf fixes from Alexei Starovoitov:
- Fix register equivalence for pointers to packet (Alexei Starovoitov)
- Fix incorrect pruning due to atomic fetch precision tracking (Daniel
Borkmann)
- Fix grace period wait for bpf_link-ed tracepoints (Kumar Kartikeya
Dwivedi)
- Fix use-after-free of sockmap's sk->sk_socket (Kuniyuki Iwashima)
- Reject direct access to nullable PTR_TO_BUF pointers (Qi Tang)
- Reject sleepable kprobe_multi programs at attach time (Varun R
Mallya)
* tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf:
selftests/bpf: Add more precision tracking tests for atomics
bpf: Fix incorrect pruning due to atomic fetch precision tracking
bpf: Reject sleepable kprobe_multi programs at attach time
bpf: reject direct access to nullable PTR_TO_BUF pointers
bpf: sockmap: Fix use-after-free of sk->sk_socket in sk_psock_verdict_data_ready().
bpf: Fix grace period wait for tracepoint bpf_link
bpf: Fix regsafe() for pointers to packet
+432
-12
+4
include/linux/bpf.h
+4
include/linux/bpf.h
···
1854
1854
* target hook is sleepable, we'll go through tasks trace RCU GP and
1855
1855
* then "classic" RCU GP; this need for chaining tasks trace and
1856
1856
* classic RCU GPs is designated by setting bpf_link->sleepable flag
1857
+
*
1858
+
* For non-sleepable tracepoint links we go through SRCU gp instead,
1859
+
* since RCU is not used in that case. Sleepable tracepoints still
1860
+
* follow the scheme above.
1857
1861
*/
1858
1862
void (*dealloc_deferred)(struct bpf_link *link);
1859
1863
int (*detach)(struct bpf_link *link);
+20
include/linux/tracepoint.h
+20
include/linux/tracepoint.h
···
122
122
{
123
123
return tp->ext && tp->ext->faultable;
124
124
}
125
+
/*
126
+
* Run RCU callback with the appropriate grace period wait for non-faultable
127
+
* tracepoints, e.g., those used in atomic context.
128
+
*/
129
+
static inline void call_tracepoint_unregister_atomic(struct rcu_head *rcu, rcu_callback_t func)
130
+
{
131
+
call_srcu(&tracepoint_srcu, rcu, func);
132
+
}
133
+
/*
134
+
* Run RCU callback with the appropriate grace period wait for faultable
135
+
* tracepoints, e.g., those used in syscall context.
136
+
*/
137
+
static inline void call_tracepoint_unregister_syscall(struct rcu_head *rcu, rcu_callback_t func)
138
+
{
139
+
call_rcu_tasks_trace(rcu, func);
140
+
}
125
141
#else
126
142
static inline void tracepoint_synchronize_unregister(void)
127
143
{ }
···
145
129
{
146
130
return false;
147
131
}
132
+
static inline void call_tracepoint_unregister_atomic(struct rcu_head *rcu, rcu_callback_t func)
133
+
{ }
134
+
static inline void call_tracepoint_unregister_syscall(struct rcu_head *rcu, rcu_callback_t func)
135
+
{ }
148
136
#endif
149
137
150
138
#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
+23
-2
kernel/bpf/syscall.c
+23
-2
kernel/bpf/syscall.c
···
3261
3261
bpf_link_dealloc(link);
3262
3262
}
3263
3263
3264
+
static bool bpf_link_is_tracepoint(struct bpf_link *link)
3265
+
{
3266
+
/*
3267
+
* Only these combinations support a tracepoint bpf_link.
3268
+
* BPF_LINK_TYPE_TRACING raw_tp progs are hardcoded to use
3269
+
* bpf_raw_tp_link_lops and thus dealloc_deferred(), see
3270
+
* bpf_raw_tp_link_attach().
3271
+
*/
3272
+
return link->type == BPF_LINK_TYPE_RAW_TRACEPOINT ||
3273
+
(link->type == BPF_LINK_TYPE_TRACING && link->attach_type == BPF_TRACE_RAW_TP);
3274
+
}
3275
+
3264
3276
static void bpf_link_defer_dealloc_mult_rcu_gp(struct rcu_head *rcu)
3265
3277
{
3266
3278
if (rcu_trace_implies_rcu_gp())
···
3291
3279
if (link->prog)
3292
3280
ops->release(link);
3293
3281
if (ops->dealloc_deferred) {
3294
-
/* Schedule BPF link deallocation, which will only then
3282
+
/*
3283
+
* Schedule BPF link deallocation, which will only then
3295
3284
* trigger putting BPF program refcount.
3296
3285
* If underlying BPF program is sleepable or BPF link's target
3297
3286
* attach hookpoint is sleepable or otherwise requires RCU GPs
3298
3287
* to ensure link and its underlying BPF program is not
3299
3288
* reachable anymore, we need to first wait for RCU tasks
3300
-
* trace sync, and then go through "classic" RCU grace period
3289
+
* trace sync, and then go through "classic" RCU grace period.
3290
+
*
3291
+
* For tracepoint BPF links, we need to go through SRCU grace
3292
+
* period wait instead when non-faultable tracepoint is used. We
3293
+
* don't need to chain SRCU grace period waits, however, for the
3294
+
* faultable case, since it exclusively uses RCU Tasks Trace.
3301
3295
*/
3302
3296
if (link->sleepable || (link->prog && link->prog->sleepable))
3303
3297
call_rcu_tasks_trace(&link->rcu, bpf_link_defer_dealloc_mult_rcu_gp);
3298
+
/* We need to do a SRCU grace period wait for non-faultable tracepoint BPF links. */
3299
+
else if (bpf_link_is_tracepoint(link))
3300
+
call_tracepoint_unregister_atomic(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
3304
3301
else
3305
3302
call_rcu(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
3306
3303
} else if (ops->dealloc) {
+32
-5
kernel/bpf/verifier.c
+32
-5
kernel/bpf/verifier.c
···
617
617
insn->imm == BPF_LOAD_ACQ;
618
618
}
619
619
620
+
static bool is_atomic_fetch_insn(const struct bpf_insn *insn)
621
+
{
622
+
return BPF_CLASS(insn->code) == BPF_STX &&
623
+
BPF_MODE(insn->code) == BPF_ATOMIC &&
624
+
(insn->imm & BPF_FETCH);
625
+
}
626
+
620
627
static int __get_spi(s32 off)
621
628
{
622
629
return (-off - 1) / BPF_REG_SIZE;
···
4454
4447
* dreg still needs precision before this insn
4455
4448
*/
4456
4449
}
4457
-
} else if (class == BPF_LDX || is_atomic_load_insn(insn)) {
4458
-
if (!bt_is_reg_set(bt, dreg))
4450
+
} else if (class == BPF_LDX ||
4451
+
is_atomic_load_insn(insn) ||
4452
+
is_atomic_fetch_insn(insn)) {
4453
+
u32 load_reg = dreg;
4454
+
4455
+
/*
4456
+
* Atomic fetch operation writes the old value into
4457
+
* a register (sreg or r0) and if it was tracked for
4458
+
* precision, propagate to the stack slot like we do
4459
+
* in regular ldx.
4460
+
*/
4461
+
if (is_atomic_fetch_insn(insn))
4462
+
load_reg = insn->imm == BPF_CMPXCHG ?
4463
+
BPF_REG_0 : sreg;
4464
+
4465
+
if (!bt_is_reg_set(bt, load_reg))
4459
4466
return 0;
4460
-
bt_clear_reg(bt, dreg);
4467
+
bt_clear_reg(bt, load_reg);
4461
4468
4462
4469
/* scalars can only be spilled into stack w/o losing precision.
4463
4470
* Load from any other memory can be zero extended.
···
7926
7905
} else if (reg->type == CONST_PTR_TO_MAP) {
7927
7906
err = check_ptr_to_map_access(env, regs, regno, off, size, t,
7928
7907
value_regno);
7929
-
} else if (base_type(reg->type) == PTR_TO_BUF) {
7908
+
} else if (base_type(reg->type) == PTR_TO_BUF &&
7909
+
!type_may_be_null(reg->type)) {
7930
7910
bool rdonly_mem = type_is_rdonly_mem(reg->type);
7931
7911
u32 *max_access;
7932
7912
···
19937
19915
* since someone could have accessed through (ptr - k), or
19938
19916
* even done ptr -= k in a register, to get a safe access.
19939
19917
*/
19940
-
if (rold->range > rcur->range)
19918
+
if (rold->range < 0 || rcur->range < 0) {
19919
+
/* special case for [BEYOND|AT]_PKT_END */
19920
+
if (rold->range != rcur->range)
19921
+
return false;
19922
+
} else if (rold->range > rcur->range) {
19941
19923
return false;
19924
+
}
19942
19925
/* If the offsets don't match, we can't trust our alignment;
19943
19926
* nor can we be sure that we won't fall out of range.
19944
19927
*/
+4
kernel/trace/bpf_trace.c
+4
kernel/trace/bpf_trace.c
···
2752
2752
if (!is_kprobe_multi(prog))
2753
2753
return -EINVAL;
2754
2754
2755
+
/* kprobe_multi is not allowed to be sleepable. */
2756
+
if (prog->sleepable)
2757
+
return -EINVAL;
2758
+
2755
2759
/* Writing to context is not allowed for kprobes. */
2756
2760
if (prog->aux->kprobe_write_ctx)
2757
2761
return -EINVAL;
+8
-5
net/core/skmsg.c
+8
-5
net/core/skmsg.c
···
1267
1267
1268
1268
static void sk_psock_verdict_data_ready(struct sock *sk)
1269
1269
{
1270
-
struct socket *sock = sk->sk_socket;
1271
-
const struct proto_ops *ops;
1270
+
const struct proto_ops *ops = NULL;
1271
+
struct socket *sock;
1272
1272
int copied;
1273
1273
1274
1274
trace_sk_data_ready(sk);
1275
1275
1276
-
if (unlikely(!sock))
1277
-
return;
1278
-
ops = READ_ONCE(sock->ops);
1276
+
rcu_read_lock();
1277
+
sock = READ_ONCE(sk->sk_socket);
1278
+
if (likely(sock))
1279
+
ops = READ_ONCE(sock->ops);
1280
+
rcu_read_unlock();
1279
1281
if (!ops || !ops->read_skb)
1280
1282
return;
1283
+
1281
1284
copied = ops->read_skb(sk, sk_psock_verdict_recv);
1282
1285
if (copied >= 0) {
1283
1286
struct sk_psock *psock;
+341
tools/testing/selftests/bpf/progs/verifier_precision.c
+341
tools/testing/selftests/bpf/progs/verifier_precision.c
···
5
5
#include "../../../include/linux/filter.h"
6
6
#include "bpf_misc.h"
7
7
8
+
struct {
9
+
__uint(type, BPF_MAP_TYPE_ARRAY);
10
+
__uint(max_entries, 1);
11
+
__type(key, __u32);
12
+
__type(value, __u64);
13
+
} precision_map SEC(".maps");
14
+
8
15
SEC("?raw_tp")
9
16
__success __log_level(2)
10
17
__msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r1 = r10")
···
306
299
"r0 = -r0;"
307
300
"exit;"
308
301
::: __clobber_all);
302
+
}
303
+
304
+
SEC("?raw_tp")
305
+
__success __log_level(2)
306
+
__msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10")
307
+
__msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_add((u64 *)(r10 -8), r2)")
308
+
__msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
309
+
__msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
310
+
__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
311
+
__naked int bpf_atomic_fetch_add_precision(void)
312
+
{
313
+
asm volatile (
314
+
"r1 = 8;"
315
+
"*(u64 *)(r10 - 8) = r1;"
316
+
"r2 = 0;"
317
+
".8byte %[fetch_add_insn];" /* r2 = atomic_fetch_add(*(u64 *)(r10 - 8), r2) */
318
+
"r3 = r10;"
319
+
"r3 += r2;" /* mark_precise */
320
+
"r0 = 0;"
321
+
"exit;"
322
+
:
323
+
: __imm_insn(fetch_add_insn,
324
+
BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8))
325
+
: __clobber_all);
326
+
}
327
+
328
+
SEC("?raw_tp")
329
+
__success __log_level(2)
330
+
__msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10")
331
+
__msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_xchg((u64 *)(r10 -8), r2)")
332
+
__msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
333
+
__msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
334
+
__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
335
+
__naked int bpf_atomic_xchg_precision(void)
336
+
{
337
+
asm volatile (
338
+
"r1 = 8;"
339
+
"*(u64 *)(r10 - 8) = r1;"
340
+
"r2 = 0;"
341
+
".8byte %[xchg_insn];" /* r2 = atomic_xchg(*(u64 *)(r10 - 8), r2) */
342
+
"r3 = r10;"
343
+
"r3 += r2;" /* mark_precise */
344
+
"r0 = 0;"
345
+
"exit;"
346
+
:
347
+
: __imm_insn(xchg_insn,
348
+
BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_2, -8))
349
+
: __clobber_all);
350
+
}
351
+
352
+
SEC("?raw_tp")
353
+
__success __log_level(2)
354
+
__msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10")
355
+
__msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_or((u64 *)(r10 -8), r2)")
356
+
__msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
357
+
__msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
358
+
__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
359
+
__naked int bpf_atomic_fetch_or_precision(void)
360
+
{
361
+
asm volatile (
362
+
"r1 = 8;"
363
+
"*(u64 *)(r10 - 8) = r1;"
364
+
"r2 = 0;"
365
+
".8byte %[fetch_or_insn];" /* r2 = atomic_fetch_or(*(u64 *)(r10 - 8), r2) */
366
+
"r3 = r10;"
367
+
"r3 += r2;" /* mark_precise */
368
+
"r0 = 0;"
369
+
"exit;"
370
+
:
371
+
: __imm_insn(fetch_or_insn,
372
+
BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8))
373
+
: __clobber_all);
374
+
}
375
+
376
+
SEC("?raw_tp")
377
+
__success __log_level(2)
378
+
__msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10")
379
+
__msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_and((u64 *)(r10 -8), r2)")
380
+
__msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
381
+
__msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
382
+
__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
383
+
__naked int bpf_atomic_fetch_and_precision(void)
384
+
{
385
+
asm volatile (
386
+
"r1 = 8;"
387
+
"*(u64 *)(r10 - 8) = r1;"
388
+
"r2 = 0;"
389
+
".8byte %[fetch_and_insn];" /* r2 = atomic_fetch_and(*(u64 *)(r10 - 8), r2) */
390
+
"r3 = r10;"
391
+
"r3 += r2;" /* mark_precise */
392
+
"r0 = 0;"
393
+
"exit;"
394
+
:
395
+
: __imm_insn(fetch_and_insn,
396
+
BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8))
397
+
: __clobber_all);
398
+
}
399
+
400
+
SEC("?raw_tp")
401
+
__success __log_level(2)
402
+
__msg("mark_precise: frame0: regs=r2 stack= before 4: (bf) r3 = r10")
403
+
__msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_xor((u64 *)(r10 -8), r2)")
404
+
__msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
405
+
__msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
406
+
__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
407
+
__naked int bpf_atomic_fetch_xor_precision(void)
408
+
{
409
+
asm volatile (
410
+
"r1 = 8;"
411
+
"*(u64 *)(r10 - 8) = r1;"
412
+
"r2 = 0;"
413
+
".8byte %[fetch_xor_insn];" /* r2 = atomic_fetch_xor(*(u64 *)(r10 - 8), r2) */
414
+
"r3 = r10;"
415
+
"r3 += r2;" /* mark_precise */
416
+
"r0 = 0;"
417
+
"exit;"
418
+
:
419
+
: __imm_insn(fetch_xor_insn,
420
+
BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8))
421
+
: __clobber_all);
422
+
}
423
+
424
+
SEC("?raw_tp")
425
+
__success __log_level(2)
426
+
__msg("mark_precise: frame0: regs=r0 stack= before 5: (bf) r3 = r10")
427
+
__msg("mark_precise: frame0: regs=r0 stack= before 4: (db) r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2)")
428
+
__msg("mark_precise: frame0: regs= stack=-8 before 3: (b7) r2 = 0")
429
+
__msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r0 = 0")
430
+
__msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
431
+
__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
432
+
__naked int bpf_atomic_cmpxchg_precision(void)
433
+
{
434
+
asm volatile (
435
+
"r1 = 8;"
436
+
"*(u64 *)(r10 - 8) = r1;"
437
+
"r0 = 0;"
438
+
"r2 = 0;"
439
+
".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u64 *)(r10 - 8), r0, r2) */
440
+
"r3 = r10;"
441
+
"r3 += r0;" /* mark_precise */
442
+
"r0 = 0;"
443
+
"exit;"
444
+
:
445
+
: __imm_insn(cmpxchg_insn,
446
+
BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, -8))
447
+
: __clobber_all);
448
+
}
449
+
450
+
/* Regression test for dual precision: Both the fetched value (r2) and
451
+
* a reread of the same stack slot (r3) are tracked for precision. After
452
+
* the atomic operation, the stack slot is STACK_MISC. Thus, the ldx at
453
+
* insn 4 does NOT set INSN_F_STACK_ACCESS. Precision for the stack slot
454
+
* propagates solely through the atomic fetch's load side (insn 3).
455
+
*/
456
+
SEC("?raw_tp")
457
+
__success __log_level(2)
458
+
__msg("mark_precise: frame0: regs=r2,r3 stack= before 4: (79) r3 = *(u64 *)(r10 -8)")
459
+
__msg("mark_precise: frame0: regs=r2 stack= before 3: (db) r2 = atomic64_fetch_add((u64 *)(r10 -8), r2)")
460
+
__msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r2 = 0")
461
+
__msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
462
+
__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
463
+
__naked int bpf_atomic_fetch_add_dual_precision(void)
464
+
{
465
+
asm volatile (
466
+
"r1 = 8;"
467
+
"*(u64 *)(r10 - 8) = r1;"
468
+
"r2 = 0;"
469
+
".8byte %[fetch_add_insn];" /* r2 = atomic_fetch_add(*(u64 *)(r10 - 8), r2) */
470
+
"r3 = *(u64 *)(r10 - 8);"
471
+
"r4 = r2;"
472
+
"r4 += r3;"
473
+
"r4 &= 7;"
474
+
"r5 = r10;"
475
+
"r5 += r4;" /* mark_precise */
476
+
"r0 = 0;"
477
+
"exit;"
478
+
:
479
+
: __imm_insn(fetch_add_insn,
480
+
BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_2, -8))
481
+
: __clobber_all);
482
+
}
483
+
484
+
SEC("?raw_tp")
485
+
__success __log_level(2)
486
+
__msg("mark_precise: frame0: regs=r0,r3 stack= before 5: (79) r3 = *(u64 *)(r10 -8)")
487
+
__msg("mark_precise: frame0: regs=r0 stack= before 4: (db) r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2)")
488
+
__msg("mark_precise: frame0: regs= stack=-8 before 3: (b7) r2 = 0")
489
+
__msg("mark_precise: frame0: regs= stack=-8 before 2: (b7) r0 = 8")
490
+
__msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) *(u64 *)(r10 -8) = r1")
491
+
__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
492
+
__naked int bpf_atomic_cmpxchg_dual_precision(void)
493
+
{
494
+
asm volatile (
495
+
"r1 = 8;"
496
+
"*(u64 *)(r10 - 8) = r1;"
497
+
"r0 = 8;"
498
+
"r2 = 0;"
499
+
".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u64 *)(r10 - 8), r0, r2) */
500
+
"r3 = *(u64 *)(r10 - 8);"
501
+
"r4 = r0;"
502
+
"r4 += r3;"
503
+
"r4 &= 7;"
504
+
"r5 = r10;"
505
+
"r5 += r4;" /* mark_precise */
506
+
"r0 = 0;"
507
+
"exit;"
508
+
:
509
+
: __imm_insn(cmpxchg_insn,
510
+
BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, -8))
511
+
: __clobber_all);
512
+
}
513
+
514
+
SEC("?raw_tp")
515
+
__success __log_level(2)
516
+
__msg("mark_precise: frame0: regs=r1 stack= before 10: (57) r1 &= 7")
517
+
__msg("mark_precise: frame0: regs=r1 stack= before 9: (db) r1 = atomic64_fetch_add((u64 *)(r0 +0), r1)")
518
+
__not_msg("falling back to forcing all scalars precise")
519
+
__naked int bpf_atomic_fetch_add_map_precision(void)
520
+
{
521
+
asm volatile (
522
+
"r1 = 0;"
523
+
"*(u64 *)(r10 - 8) = r1;"
524
+
"r2 = r10;"
525
+
"r2 += -8;"
526
+
"r1 = %[precision_map] ll;"
527
+
"call %[bpf_map_lookup_elem];"
528
+
"if r0 == 0 goto 1f;"
529
+
"r1 = 0;"
530
+
".8byte %[fetch_add_insn];" /* r1 = atomic_fetch_add(*(u64 *)(r0 + 0), r1) */
531
+
"r1 &= 7;"
532
+
"r2 = r10;"
533
+
"r2 += r1;" /* mark_precise */
534
+
"1: r0 = 0;"
535
+
"exit;"
536
+
:
537
+
: __imm_addr(precision_map),
538
+
__imm(bpf_map_lookup_elem),
539
+
__imm_insn(fetch_add_insn,
540
+
BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0))
541
+
: __clobber_all);
542
+
}
543
+
544
+
SEC("?raw_tp")
545
+
__success __log_level(2)
546
+
__msg("mark_precise: frame0: regs=r0 stack= before 12: (57) r0 &= 7")
547
+
__msg("mark_precise: frame0: regs=r0 stack= before 11: (db) r0 = atomic64_cmpxchg((u64 *)(r6 +0), r0, r1)")
548
+
__not_msg("falling back to forcing all scalars precise")
549
+
__naked int bpf_atomic_cmpxchg_map_precision(void)
550
+
{
551
+
asm volatile (
552
+
"r1 = 0;"
553
+
"*(u64 *)(r10 - 8) = r1;"
554
+
"r2 = r10;"
555
+
"r2 += -8;"
556
+
"r1 = %[precision_map] ll;"
557
+
"call %[bpf_map_lookup_elem];"
558
+
"if r0 == 0 goto 1f;"
559
+
"r6 = r0;"
560
+
"r0 = 0;"
561
+
"r1 = 0;"
562
+
".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u64 *)(r6 + 0), r0, r1) */
563
+
"r0 &= 7;"
564
+
"r2 = r10;"
565
+
"r2 += r0;" /* mark_precise */
566
+
"1: r0 = 0;"
567
+
"exit;"
568
+
:
569
+
: __imm_addr(precision_map),
570
+
__imm(bpf_map_lookup_elem),
571
+
__imm_insn(cmpxchg_insn,
572
+
BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 0))
573
+
: __clobber_all);
574
+
}
575
+
576
+
SEC("?raw_tp")
577
+
__success __log_level(2)
578
+
__msg("mark_precise: frame0: regs=r1 stack= before 10: (57) r1 &= 7")
579
+
__msg("mark_precise: frame0: regs=r1 stack= before 9: (c3) r1 = atomic_fetch_add((u32 *)(r0 +0), r1)")
580
+
__not_msg("falling back to forcing all scalars precise")
581
+
__naked int bpf_atomic_fetch_add_32bit_precision(void)
582
+
{
583
+
asm volatile (
584
+
"r1 = 0;"
585
+
"*(u64 *)(r10 - 8) = r1;"
586
+
"r2 = r10;"
587
+
"r2 += -8;"
588
+
"r1 = %[precision_map] ll;"
589
+
"call %[bpf_map_lookup_elem];"
590
+
"if r0 == 0 goto 1f;"
591
+
"r1 = 0;"
592
+
".8byte %[fetch_add_insn];" /* r1 = atomic_fetch_add(*(u32 *)(r0 + 0), r1) */
593
+
"r1 &= 7;"
594
+
"r2 = r10;"
595
+
"r2 += r1;" /* mark_precise */
596
+
"1: r0 = 0;"
597
+
"exit;"
598
+
:
599
+
: __imm_addr(precision_map),
600
+
__imm(bpf_map_lookup_elem),
601
+
__imm_insn(fetch_add_insn,
602
+
BPF_ATOMIC_OP(BPF_W, BPF_ADD | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0))
603
+
: __clobber_all);
604
+
}
605
+
606
+
SEC("?raw_tp")
607
+
__success __log_level(2)
608
+
__msg("mark_precise: frame0: regs=r0 stack= before 12: (57) r0 &= 7")
609
+
__msg("mark_precise: frame0: regs=r0 stack= before 11: (c3) r0 = atomic_cmpxchg((u32 *)(r6 +0), r0, r1)")
610
+
__not_msg("falling back to forcing all scalars precise")
611
+
__naked int bpf_atomic_cmpxchg_32bit_precision(void)
612
+
{
613
+
asm volatile (
614
+
"r1 = 0;"
615
+
"*(u64 *)(r10 - 8) = r1;"
616
+
"r2 = r10;"
617
+
"r2 += -8;"
618
+
"r1 = %[precision_map] ll;"
619
+
"call %[bpf_map_lookup_elem];"
620
+
"if r0 == 0 goto 1f;"
621
+
"r6 = r0;"
622
+
"r0 = 0;"
623
+
"r1 = 0;"
624
+
".8byte %[cmpxchg_insn];" /* r0 = atomic_cmpxchg(*(u32 *)(r6 + 0), r0, r1) */
625
+
"r0 &= 7;"
626
+
"r2 = r10;"
627
+
"r2 += r0;" /* mark_precise */
628
+
"1: r0 = 0;"
629
+
"exit;"
630
+
:
631
+
: __imm_addr(precision_map),
632
+
__imm(bpf_map_lookup_elem),
633
+
__imm_insn(cmpxchg_insn,
634
+
BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 0))
635
+
: __clobber_all);
309
636
}
310
637
311
638
char _license[] SEC("license") = "GPL";