Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'bpf-add-uprobe-session-support'
Jiri Olsa says:
====================
bpf: Add uprobe session support
hi,
this patchset is adding support for session uprobe attachment and
using it through bpf link for bpf programs.
The session means that the uprobe consumer is executed on entry
and return of probed function with additional control:
- entry callback can control execution of the return callback
- entry and return callbacks can share data/cookie
Uprobe changes (on top of perf/core [1] are posted in here [2].
This patchset is based on bpf-next/master and will be merged once
we pull [2] in bpf-next/master.
v9 changes:
- rebased on bpf-next/master with perf/core tag merged (thanks Peter!)
thanks,
jirka
[1] git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git perf/core
[2] https://lore.kernel.org/bpf/20241018202252.693462-1-jolsa@kernel.org/T/#ma43c549c4bf684ca1b17fa638aa5e7cbb46893e9
---
====================
Link: https://lore.kernel.org/r/20241108134544.480660-1-jolsa@kernel.org
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
+656
-65
+1
include/uapi/linux/bpf.h
+1
include/uapi/linux/bpf.h
+7
-2
kernel/bpf/syscall.c
+7
-2
kernel/bpf/syscall.c
···
4103
4103
if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI &&
4104
4104
attach_type != BPF_TRACE_UPROBE_MULTI)
4105
4105
return -EINVAL;
4106
+
if (prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION &&
4107
+
attach_type != BPF_TRACE_UPROBE_SESSION)
4108
+
return -EINVAL;
4106
4109
if (attach_type != BPF_PERF_EVENT &&
4107
4110
attach_type != BPF_TRACE_KPROBE_MULTI &&
4108
4111
attach_type != BPF_TRACE_KPROBE_SESSION &&
4109
-
attach_type != BPF_TRACE_UPROBE_MULTI)
4112
+
attach_type != BPF_TRACE_UPROBE_MULTI &&
4113
+
attach_type != BPF_TRACE_UPROBE_SESSION)
4110
4114
return -EINVAL;
4111
4115
return 0;
4112
4116
case BPF_PROG_TYPE_SCHED_CLS:
···
5363
5359
else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI ||
5364
5360
attr->link_create.attach_type == BPF_TRACE_KPROBE_SESSION)
5365
5361
ret = bpf_kprobe_multi_link_attach(attr, prog);
5366
-
else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI)
5362
+
else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI ||
5363
+
attr->link_create.attach_type == BPF_TRACE_UPROBE_SESSION)
5367
5364
ret = bpf_uprobe_multi_link_attach(attr, prog);
5368
5365
break;
5369
5366
default:
+10
kernel/bpf/verifier.c
+10
kernel/bpf/verifier.c
···
16024
16024
return -ENOTSUPP;
16025
16025
}
16026
16026
break;
16027
+
case BPF_PROG_TYPE_KPROBE:
16028
+
switch (env->prog->expected_attach_type) {
16029
+
case BPF_TRACE_KPROBE_SESSION:
16030
+
case BPF_TRACE_UPROBE_SESSION:
16031
+
range = retval_range(0, 1);
16032
+
break;
16033
+
default:
16034
+
return 0;
16035
+
}
16036
+
break;
16027
16037
case BPF_PROG_TYPE_SK_LOOKUP:
16028
16038
range = retval_range(SK_DROP, SK_PASS);
16029
16039
break;
+41
-16
kernel/trace/bpf_trace.c
+41
-16
kernel/trace/bpf_trace.c
···
1581
1581
return prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;
1582
1582
}
1583
1583
1584
+
static inline bool is_uprobe_multi(const struct bpf_prog *prog)
1585
+
{
1586
+
return prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI ||
1587
+
prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1588
+
}
1589
+
1590
+
static inline bool is_uprobe_session(const struct bpf_prog *prog)
1591
+
{
1592
+
return prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1593
+
}
1594
+
1584
1595
static const struct bpf_func_proto *
1585
1596
kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1586
1597
{
···
1609
1598
case BPF_FUNC_get_func_ip:
1610
1599
if (is_kprobe_multi(prog))
1611
1600
return &bpf_get_func_ip_proto_kprobe_multi;
1612
-
if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)
1601
+
if (is_uprobe_multi(prog))
1613
1602
return &bpf_get_func_ip_proto_uprobe_multi;
1614
1603
return &bpf_get_func_ip_proto_kprobe;
1615
1604
case BPF_FUNC_get_attach_cookie:
1616
1605
if (is_kprobe_multi(prog))
1617
1606
return &bpf_get_attach_cookie_proto_kmulti;
1618
-
if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)
1607
+
if (is_uprobe_multi(prog))
1619
1608
return &bpf_get_attach_cookie_proto_umulti;
1620
1609
return &bpf_get_attach_cookie_proto_trace;
1621
1610
default:
···
3107
3096
u64 cookie;
3108
3097
struct uprobe *uprobe;
3109
3098
struct uprobe_consumer consumer;
3099
+
bool session;
3110
3100
};
3111
3101
3112
3102
struct bpf_uprobe_multi_link {
···
3120
3108
};
3121
3109
3122
3110
struct bpf_uprobe_multi_run_ctx {
3123
-
struct bpf_run_ctx run_ctx;
3111
+
struct bpf_session_run_ctx session_ctx;
3124
3112
unsigned long entry_ip;
3125
3113
struct bpf_uprobe *uprobe;
3126
3114
};
···
3231
3219
3232
3220
static int uprobe_prog_run(struct bpf_uprobe *uprobe,
3233
3221
unsigned long entry_ip,
3234
-
struct pt_regs *regs)
3222
+
struct pt_regs *regs,
3223
+
bool is_return, void *data)
3235
3224
{
3236
3225
struct bpf_uprobe_multi_link *link = uprobe->link;
3237
3226
struct bpf_uprobe_multi_run_ctx run_ctx = {
3227
+
.session_ctx = {
3228
+
.is_return = is_return,
3229
+
.data = data,
3230
+
},
3238
3231
.entry_ip = entry_ip,
3239
3232
.uprobe = uprobe,
3240
3233
};
3241
3234
struct bpf_prog *prog = link->link.prog;
3242
3235
bool sleepable = prog->sleepable;
3243
3236
struct bpf_run_ctx *old_run_ctx;
3244
-
int err = 0;
3237
+
int err;
3245
3238
3246
3239
if (link->task && !same_thread_group(current, link->task))
3247
3240
return 0;
···
3258
3241
3259
3242
migrate_disable();
3260
3243
3261
-
old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
3244
+
old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx);
3262
3245
err = bpf_prog_run(link->link.prog, regs);
3263
3246
bpf_reset_run_ctx(old_run_ctx);
3264
3247
···
3285
3268
__u64 *data)
3286
3269
{
3287
3270
struct bpf_uprobe *uprobe;
3271
+
int ret;
3288
3272
3289
3273
uprobe = container_of(con, struct bpf_uprobe, consumer);
3290
-
return uprobe_prog_run(uprobe, instruction_pointer(regs), regs);
3274
+
ret = uprobe_prog_run(uprobe, instruction_pointer(regs), regs, false, data);
3275
+
if (uprobe->session)
3276
+
return ret ? UPROBE_HANDLER_IGNORE : 0;
3277
+
return 0;
3291
3278
}
3292
3279
3293
3280
static int
···
3301
3280
struct bpf_uprobe *uprobe;
3302
3281
3303
3282
uprobe = container_of(con, struct bpf_uprobe, consumer);
3304
-
return uprobe_prog_run(uprobe, func, regs);
3283
+
uprobe_prog_run(uprobe, func, regs, true, data);
3284
+
return 0;
3305
3285
}
3306
3286
3307
3287
static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
3308
3288
{
3309
3289
struct bpf_uprobe_multi_run_ctx *run_ctx;
3310
3290
3311
-
run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);
3291
+
run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3292
+
session_ctx.run_ctx);
3312
3293
return run_ctx->entry_ip;
3313
3294
}
3314
3295
···
3318
3295
{
3319
3296
struct bpf_uprobe_multi_run_ctx *run_ctx;
3320
3297
3321
-
run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);
3298
+
run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3299
+
session_ctx.run_ctx);
3322
3300
return run_ctx->uprobe->cookie;
3323
3301
}
3324
3302
···
3343
3319
if (sizeof(u64) != sizeof(void *))
3344
3320
return -EOPNOTSUPP;
3345
3321
3346
-
if (prog->expected_attach_type != BPF_TRACE_UPROBE_MULTI)
3322
+
if (!is_uprobe_multi(prog))
3347
3323
return -EINVAL;
3348
3324
3349
3325
flags = attr->link_create.uprobe_multi.flags;
···
3419
3395
3420
3396
uprobes[i].link = link;
3421
3397
3422
-
if (flags & BPF_F_UPROBE_MULTI_RETURN)
3423
-
uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3424
-
else
3398
+
if (!(flags & BPF_F_UPROBE_MULTI_RETURN))
3425
3399
uprobes[i].consumer.handler = uprobe_multi_link_handler;
3426
-
3400
+
if (flags & BPF_F_UPROBE_MULTI_RETURN || is_uprobe_session(prog))
3401
+
uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3402
+
if (is_uprobe_session(prog))
3403
+
uprobes[i].session = true;
3427
3404
if (pid)
3428
3405
uprobes[i].consumer.filter = uprobe_multi_link_filter;
3429
3406
}
···
3513
3488
if (!btf_id_set8_contains(&kprobe_multi_kfunc_set_ids, kfunc_id))
3514
3489
return 0;
3515
3490
3516
-
if (!is_kprobe_session(prog))
3491
+
if (!is_kprobe_session(prog) && !is_uprobe_session(prog))
3517
3492
return -EACCES;
3518
3493
3519
3494
return 0;
+1
tools/include/uapi/linux/bpf.h
+1
tools/include/uapi/linux/bpf.h
+1
tools/lib/bpf/bpf.c
+1
tools/lib/bpf/bpf.c
···
776
776
return libbpf_err(-EINVAL);
777
777
break;
778
778
case BPF_TRACE_UPROBE_MULTI:
779
+
case BPF_TRACE_UPROBE_SESSION:
779
780
attr.link_create.uprobe_multi.flags = OPTS_GET(opts, uprobe_multi.flags, 0);
780
781
attr.link_create.uprobe_multi.cnt = OPTS_GET(opts, uprobe_multi.cnt, 0);
781
782
attr.link_create.uprobe_multi.path = ptr_to_u64(OPTS_GET(opts, uprobe_multi.path, 0));
+17
-2
tools/lib/bpf/libbpf.c
+17
-2
tools/lib/bpf/libbpf.c
···
133
133
[BPF_NETKIT_PRIMARY] = "netkit_primary",
134
134
[BPF_NETKIT_PEER] = "netkit_peer",
135
135
[BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session",
136
+
[BPF_TRACE_UPROBE_SESSION] = "trace_uprobe_session",
136
137
};
137
138
138
139
static const char * const link_type_name[] = {
···
9442
9441
SEC_DEF("kprobe.session+", KPROBE, BPF_TRACE_KPROBE_SESSION, SEC_NONE, attach_kprobe_session),
9443
9442
SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
9444
9443
SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
9444
+
SEC_DEF("uprobe.session+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_NONE, attach_uprobe_multi),
9445
9445
SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
9446
9446
SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
9447
+
SEC_DEF("uprobe.session.s+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_SLEEPABLE, attach_uprobe_multi),
9447
9448
SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
9448
9449
SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
9449
9450
SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt),
···
11767
11764
ret = 0;
11768
11765
break;
11769
11766
case 3:
11767
+
opts.session = str_has_pfx(probe_type, "uprobe.session");
11770
11768
opts.retprobe = str_has_pfx(probe_type, "uretprobe.multi");
11769
+
11771
11770
*link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
11772
11771
ret = libbpf_get_error(*link);
11773
11772
break;
···
12018
12013
const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL;
12019
12014
LIBBPF_OPTS(bpf_link_create_opts, lopts);
12020
12015
unsigned long *resolved_offsets = NULL;
12016
+
enum bpf_attach_type attach_type;
12021
12017
int err = 0, link_fd, prog_fd;
12022
12018
struct bpf_link *link = NULL;
12023
12019
char errmsg[STRERR_BUFSIZE];
12024
12020
char full_path[PATH_MAX];
12021
+
bool retprobe, session;
12025
12022
const __u64 *cookies;
12026
12023
const char **syms;
12027
12024
size_t cnt;
···
12094
12087
offsets = resolved_offsets;
12095
12088
}
12096
12089
12090
+
retprobe = OPTS_GET(opts, retprobe, false);
12091
+
session = OPTS_GET(opts, session, false);
12092
+
12093
+
if (retprobe && session)
12094
+
return libbpf_err_ptr(-EINVAL);
12095
+
12096
+
attach_type = session ? BPF_TRACE_UPROBE_SESSION : BPF_TRACE_UPROBE_MULTI;
12097
+
12097
12098
lopts.uprobe_multi.path = path;
12098
12099
lopts.uprobe_multi.offsets = offsets;
12099
12100
lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets;
12100
12101
lopts.uprobe_multi.cookies = cookies;
12101
12102
lopts.uprobe_multi.cnt = cnt;
12102
-
lopts.uprobe_multi.flags = OPTS_GET(opts, retprobe, false) ? BPF_F_UPROBE_MULTI_RETURN : 0;
12103
+
lopts.uprobe_multi.flags = retprobe ? BPF_F_UPROBE_MULTI_RETURN : 0;
12103
12104
12104
12105
if (pid == 0)
12105
12106
pid = getpid();
···
12121
12106
}
12122
12107
link->detach = &bpf_link__detach_fd;
12123
12108
12124
-
link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts);
12109
+
link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts);
12125
12110
if (link_fd < 0) {
12126
12111
err = -errno;
12127
12112
pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
+3
-1
tools/lib/bpf/libbpf.h
+3
-1
tools/lib/bpf/libbpf.h
···
577
577
size_t cnt;
578
578
/* create return uprobes */
579
579
bool retprobe;
580
+
/* create session kprobes */
581
+
bool session;
580
582
size_t :0;
581
583
};
582
584
583
-
#define bpf_uprobe_multi_opts__last_field retprobe
585
+
#define bpf_uprobe_multi_opts__last_field session
584
586
585
587
/**
586
588
* @brief **bpf_program__attach_uprobe_multi()** attaches a BPF program
+2
tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c
+2
tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c
···
6
6
#include "kprobe_multi_override.skel.h"
7
7
#include "kprobe_multi_session.skel.h"
8
8
#include "kprobe_multi_session_cookie.skel.h"
9
+
#include "kprobe_multi_verifier.skel.h"
9
10
#include "bpf/libbpf_internal.h"
10
11
#include "bpf/hashmap.h"
11
12
···
765
764
test_session_skel_api();
766
765
if (test__start_subtest("session_cookie"))
767
766
test_session_cookie_skel_api();
767
+
RUN_TESTS(kprobe_multi_verifier);
768
768
}
+301
-41
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
+301
-41
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
···
8
8
#include "uprobe_multi_usdt.skel.h"
9
9
#include "uprobe_multi_consumers.skel.h"
10
10
#include "uprobe_multi_pid_filter.skel.h"
11
+
#include "uprobe_multi_session.skel.h"
12
+
#include "uprobe_multi_session_single.skel.h"
13
+
#include "uprobe_multi_session_cookie.skel.h"
14
+
#include "uprobe_multi_session_recursive.skel.h"
15
+
#include "uprobe_multi_verifier.skel.h"
11
16
#include "bpf/libbpf_internal.h"
12
17
#include "testing_helpers.h"
13
18
#include "../sdt.h"
···
37
32
noinline void usdt_trigger(void)
38
33
{
39
34
STAP_PROBE(test, pid_filter_usdt);
35
+
}
36
+
37
+
noinline void uprobe_session_recursive(int i)
38
+
{
39
+
if (i)
40
+
uprobe_session_recursive(i - 1);
40
41
}
41
42
42
43
struct child {
···
789
778
}
790
779
}
791
780
792
-
static int uprobe_attach(struct uprobe_multi_consumers *skel, int idx)
781
+
static int uprobe_attach(struct uprobe_multi_consumers *skel, int idx, unsigned long offset)
793
782
{
794
783
struct bpf_program *prog = get_program(skel, idx);
795
784
struct bpf_link **link = get_link(skel, idx);
···
798
787
if (!prog || !link)
799
788
return -1;
800
789
801
-
/*
802
-
* bit/prog: 0,1 uprobe entry
803
-
* bit/prog: 2,3 uprobe return
804
-
*/
805
-
opts.retprobe = idx == 2 || idx == 3;
790
+
opts.offsets = &offset;
791
+
opts.cnt = 1;
806
792
807
-
*link = bpf_program__attach_uprobe_multi(prog, 0, "/proc/self/exe",
808
-
"uprobe_consumer_test",
809
-
&opts);
793
+
/*
794
+
* bit/prog: 0 uprobe entry
795
+
* bit/prog: 1 uprobe return
796
+
* bit/prog: 2 uprobe session without return
797
+
* bit/prog: 3 uprobe session with return
798
+
*/
799
+
opts.retprobe = idx == 1;
800
+
opts.session = idx == 2 || idx == 3;
801
+
802
+
*link = bpf_program__attach_uprobe_multi(prog, 0, "/proc/self/exe", NULL, &opts);
810
803
if (!ASSERT_OK_PTR(*link, "bpf_program__attach_uprobe_multi"))
811
804
return -1;
812
805
return 0;
···
831
816
832
817
noinline int
833
818
uprobe_consumer_test(struct uprobe_multi_consumers *skel,
834
-
unsigned long before, unsigned long after)
819
+
unsigned long before, unsigned long after,
820
+
unsigned long offset)
835
821
{
836
822
int idx;
837
823
···
845
829
/* ... and attach all new programs in 'after' state */
846
830
for (idx = 0; idx < 4; idx++) {
847
831
if (!test_bit(idx, before) && test_bit(idx, after)) {
848
-
if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_after"))
832
+
if (!ASSERT_OK(uprobe_attach(skel, idx, offset), "uprobe_attach_after"))
849
833
return -1;
850
834
}
851
835
}
852
836
return 0;
853
837
}
854
838
839
+
/*
840
+
* We generate 16 consumer_testX functions that will have uprobe installed on
841
+
* and will be called in separate threads. All function pointer are stored in
842
+
* "consumers" section and each thread will pick one function based on index.
843
+
*/
844
+
845
+
extern const void *__start_consumers;
846
+
847
+
#define __CONSUMER_TEST(func) \
848
+
noinline int func(struct uprobe_multi_consumers *skel, unsigned long before, \
849
+
unsigned long after, unsigned long offset) \
850
+
{ \
851
+
return uprobe_consumer_test(skel, before, after, offset); \
852
+
} \
853
+
void *__ ## func __used __attribute__((section("consumers"))) = (void *) func;
854
+
855
+
#define CONSUMER_TEST(func) __CONSUMER_TEST(func)
856
+
857
+
#define C1 CONSUMER_TEST(__PASTE(consumer_test, __COUNTER__))
858
+
#define C4 C1 C1 C1 C1
859
+
#define C16 C4 C4 C4 C4
860
+
861
+
C16
862
+
863
+
typedef int (*test_t)(struct uprobe_multi_consumers *, unsigned long,
864
+
unsigned long, unsigned long);
865
+
855
866
static int consumer_test(struct uprobe_multi_consumers *skel,
856
-
unsigned long before, unsigned long after)
867
+
unsigned long before, unsigned long after,
868
+
test_t test, unsigned long offset)
857
869
{
858
870
int err, idx, ret = -1;
859
871
···
890
846
/* 'before' is each, we attach uprobe for every set idx */
891
847
for (idx = 0; idx < 4; idx++) {
892
848
if (test_bit(idx, before)) {
893
-
if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_before"))
849
+
if (!ASSERT_OK(uprobe_attach(skel, idx, offset), "uprobe_attach_before"))
894
850
goto cleanup;
895
851
}
896
852
}
897
853
898
-
err = uprobe_consumer_test(skel, before, after);
854
+
err = test(skel, before, after, offset);
899
855
if (!ASSERT_EQ(err, 0, "uprobe_consumer_test"))
900
856
goto cleanup;
901
857
902
858
for (idx = 0; idx < 4; idx++) {
859
+
bool uret_stays, uret_survives;
903
860
const char *fmt = "BUG";
904
861
__u64 val = 0;
905
862
906
-
if (idx < 2) {
863
+
switch (idx) {
864
+
case 0:
907
865
/*
908
866
* uprobe entry
909
867
* +1 if define in 'before'
910
868
*/
911
869
if (test_bit(idx, before))
912
870
val++;
913
-
fmt = "prog 0/1: uprobe";
914
-
} else {
871
+
fmt = "prog 0: uprobe";
872
+
break;
873
+
case 1:
915
874
/*
916
875
* To trigger uretprobe consumer, the uretprobe under test either stayed from
917
876
* before to after (uret_stays + test_bit) or uretprobe instance survived and
918
877
* we have uretprobe active in after (uret_survives + test_bit)
919
878
*/
920
-
921
-
bool uret_stays = before & after & 0b1100;
922
-
bool uret_survives = (before & 0b1100) && (after & 0b1100) && (before & 0b0011);
879
+
uret_stays = before & after & 0b0110;
880
+
uret_survives = ((before & 0b0110) && (after & 0b0110) && (before & 0b1001));
923
881
924
882
if ((uret_stays || uret_survives) && test_bit(idx, after))
925
883
val++;
926
-
927
-
fmt = "idx 2/3: uretprobe";
884
+
fmt = "prog 1: uretprobe";
885
+
break;
886
+
case 2:
887
+
/*
888
+
* session with return
889
+
* +1 if defined in 'before'
890
+
* +1 if defined in 'after'
891
+
*/
892
+
if (test_bit(idx, before)) {
893
+
val++;
894
+
if (test_bit(idx, after))
895
+
val++;
896
+
}
897
+
fmt = "prog 2: session with return";
898
+
break;
899
+
case 3:
900
+
/*
901
+
* session without return
902
+
* +1 if defined in 'before'
903
+
*/
904
+
if (test_bit(idx, before))
905
+
val++;
906
+
fmt = "prog 3: session with NO return";
907
+
break;
928
908
}
929
909
930
910
if (!ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt))
···
964
896
return ret;
965
897
}
966
898
967
-
static void test_consumers(void)
899
+
#define CONSUMER_MAX 16
900
+
901
+
/*
902
+
* Each thread runs 1/16 of the load by running test for single
903
+
* 'before' number (based on thread index) and full scale of
904
+
* 'after' numbers.
905
+
*/
906
+
static void *consumer_thread(void *arg)
968
907
{
908
+
unsigned long idx = (unsigned long) arg;
969
909
struct uprobe_multi_consumers *skel;
970
-
int before, after;
910
+
unsigned long offset;
911
+
const void *func;
912
+
int after;
971
913
972
914
skel = uprobe_multi_consumers__open_and_load();
973
915
if (!ASSERT_OK_PTR(skel, "uprobe_multi_consumers__open_and_load"))
974
-
return;
916
+
return NULL;
917
+
918
+
func = *((&__start_consumers) + idx);
919
+
920
+
offset = get_uprobe_offset(func);
921
+
if (!ASSERT_GE(offset, 0, "uprobe_offset"))
922
+
goto out;
923
+
924
+
for (after = 0; after < CONSUMER_MAX; after++)
925
+
if (consumer_test(skel, idx, after, func, offset))
926
+
goto out;
927
+
928
+
out:
929
+
uprobe_multi_consumers__destroy(skel);
930
+
return NULL;
931
+
}
932
+
933
+
934
+
static void test_consumers(void)
935
+
{
936
+
pthread_t pt[CONSUMER_MAX];
937
+
unsigned long idx;
938
+
int err;
975
939
976
940
/*
977
941
* The idea of this test is to try all possible combinations of
978
942
* uprobes consumers attached on single function.
979
943
*
980
-
* - 2 uprobe entry consumer
981
-
* - 2 uprobe exit consumers
944
+
* - 1 uprobe entry consumer
945
+
* - 1 uprobe exit consumer
946
+
* - 1 uprobe session with return
947
+
* - 1 uprobe session without return
982
948
*
983
949
* The test uses 4 uprobes attached on single function, but that
984
950
* translates into single uprobe with 4 consumers in kernel.
···
1020
918
* The before/after values present the state of attached consumers
1021
919
* before and after the probed function:
1022
920
*
1023
-
* bit/prog 0,1 : uprobe entry
1024
-
* bit/prog 2,3 : uprobe return
921
+
* bit/prog 0 : uprobe entry
922
+
* bit/prog 1 : uprobe return
1025
923
*
1026
924
* For example for:
1027
925
*
1028
-
* before = 0b0101
1029
-
* after = 0b0110
926
+
* before = 0b01
927
+
* after = 0b10
1030
928
*
1031
929
* it means that before we call 'uprobe_consumer_test' we attach
1032
930
* uprobes defined in 'before' value:
1033
931
*
1034
-
* - bit/prog 0: uprobe entry
1035
-
* - bit/prog 2: uprobe return
932
+
* - bit/prog 1: uprobe entry
1036
933
*
1037
934
* uprobe_consumer_test is called and inside it we attach and detach
1038
935
* uprobes based on 'after' value:
1039
936
*
1040
-
* - bit/prog 0: stays untouched
1041
-
* - bit/prog 2: uprobe return is detached
937
+
* - bit/prog 0: is detached
938
+
* - bit/prog 1: is attached
1042
939
*
1043
940
* uprobe_consumer_test returns and we check counters values increased
1044
941
* by bpf programs on each uprobe to match the expected count based on
1045
942
* before/after bits.
1046
943
*/
1047
944
1048
-
for (before = 0; before < 16; before++) {
1049
-
for (after = 0; after < 16; after++)
1050
-
if (consumer_test(skel, before, after))
1051
-
goto out;
945
+
for (idx = 0; idx < CONSUMER_MAX; idx++) {
946
+
err = pthread_create(&pt[idx], NULL, consumer_thread, (void *) idx);
947
+
if (!ASSERT_OK(err, "pthread_create"))
948
+
break;
1052
949
}
1053
950
1054
-
out:
1055
-
uprobe_multi_consumers__destroy(skel);
951
+
while (idx)
952
+
pthread_join(pt[--idx], NULL);
1056
953
}
1057
954
1058
955
static struct bpf_program *uprobe_multi_program(struct uprobe_multi_pid_filter *skel, int idx)
···
1116
1015
run_pid_filter(skel, clone_vm, true);
1117
1016
1118
1017
uprobe_multi_pid_filter__destroy(skel);
1018
+
}
1019
+
1020
+
static void test_session_skel_api(void)
1021
+
{
1022
+
struct uprobe_multi_session *skel = NULL;
1023
+
LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
1024
+
struct bpf_link *link = NULL;
1025
+
int err;
1026
+
1027
+
skel = uprobe_multi_session__open_and_load();
1028
+
if (!ASSERT_OK_PTR(skel, "uprobe_multi_session__open_and_load"))
1029
+
goto cleanup;
1030
+
1031
+
skel->bss->pid = getpid();
1032
+
skel->bss->user_ptr = test_data;
1033
+
1034
+
err = uprobe_multi_session__attach(skel);
1035
+
if (!ASSERT_OK(err, "uprobe_multi_session__attach"))
1036
+
goto cleanup;
1037
+
1038
+
/* trigger all probes */
1039
+
skel->bss->uprobe_multi_func_1_addr = (__u64) uprobe_multi_func_1;
1040
+
skel->bss->uprobe_multi_func_2_addr = (__u64) uprobe_multi_func_2;
1041
+
skel->bss->uprobe_multi_func_3_addr = (__u64) uprobe_multi_func_3;
1042
+
1043
+
uprobe_multi_func_1();
1044
+
uprobe_multi_func_2();
1045
+
uprobe_multi_func_3();
1046
+
1047
+
/*
1048
+
* We expect 2 for uprobe_multi_func_2 because it runs both entry/return probe,
1049
+
* uprobe_multi_func_[13] run just the entry probe. All expected numbers are
1050
+
* doubled, because we run extra test for sleepable session.
1051
+
*/
1052
+
ASSERT_EQ(skel->bss->uprobe_session_result[0], 2, "uprobe_multi_func_1_result");
1053
+
ASSERT_EQ(skel->bss->uprobe_session_result[1], 4, "uprobe_multi_func_2_result");
1054
+
ASSERT_EQ(skel->bss->uprobe_session_result[2], 2, "uprobe_multi_func_3_result");
1055
+
1056
+
/* We expect increase in 3 entry and 1 return session calls -> 4 */
1057
+
ASSERT_EQ(skel->bss->uprobe_multi_sleep_result, 4, "uprobe_multi_sleep_result");
1058
+
1059
+
cleanup:
1060
+
bpf_link__destroy(link);
1061
+
uprobe_multi_session__destroy(skel);
1062
+
}
1063
+
1064
+
static void test_session_single_skel_api(void)
1065
+
{
1066
+
struct uprobe_multi_session_single *skel = NULL;
1067
+
LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
1068
+
int err;
1069
+
1070
+
skel = uprobe_multi_session_single__open_and_load();
1071
+
if (!ASSERT_OK_PTR(skel, "uprobe_multi_session_single__open_and_load"))
1072
+
goto cleanup;
1073
+
1074
+
skel->bss->pid = getpid();
1075
+
1076
+
err = uprobe_multi_session_single__attach(skel);
1077
+
if (!ASSERT_OK(err, "uprobe_multi_session_single__attach"))
1078
+
goto cleanup;
1079
+
1080
+
uprobe_multi_func_1();
1081
+
1082
+
/*
1083
+
* We expect consumer 0 and 2 to trigger just entry handler (value 1)
1084
+
* and consumer 1 to hit both (value 2).
1085
+
*/
1086
+
ASSERT_EQ(skel->bss->uprobe_session_result[0], 1, "uprobe_session_result_0");
1087
+
ASSERT_EQ(skel->bss->uprobe_session_result[1], 2, "uprobe_session_result_1");
1088
+
ASSERT_EQ(skel->bss->uprobe_session_result[2], 1, "uprobe_session_result_2");
1089
+
1090
+
cleanup:
1091
+
uprobe_multi_session_single__destroy(skel);
1092
+
}
1093
+
1094
+
static void test_session_cookie_skel_api(void)
1095
+
{
1096
+
struct uprobe_multi_session_cookie *skel = NULL;
1097
+
int err;
1098
+
1099
+
skel = uprobe_multi_session_cookie__open_and_load();
1100
+
if (!ASSERT_OK_PTR(skel, "uprobe_multi_session_cookie__open_and_load"))
1101
+
goto cleanup;
1102
+
1103
+
skel->bss->pid = getpid();
1104
+
1105
+
err = uprobe_multi_session_cookie__attach(skel);
1106
+
if (!ASSERT_OK(err, "uprobe_multi_session_cookie__attach"))
1107
+
goto cleanup;
1108
+
1109
+
/* trigger all probes */
1110
+
uprobe_multi_func_1();
1111
+
uprobe_multi_func_2();
1112
+
uprobe_multi_func_3();
1113
+
1114
+
ASSERT_EQ(skel->bss->test_uprobe_1_result, 1, "test_uprobe_1_result");
1115
+
ASSERT_EQ(skel->bss->test_uprobe_2_result, 2, "test_uprobe_2_result");
1116
+
ASSERT_EQ(skel->bss->test_uprobe_3_result, 3, "test_uprobe_3_result");
1117
+
1118
+
cleanup:
1119
+
uprobe_multi_session_cookie__destroy(skel);
1120
+
}
1121
+
1122
+
static void test_session_recursive_skel_api(void)
1123
+
{
1124
+
struct uprobe_multi_session_recursive *skel = NULL;
1125
+
int i, err;
1126
+
1127
+
skel = uprobe_multi_session_recursive__open_and_load();
1128
+
if (!ASSERT_OK_PTR(skel, "uprobe_multi_session_recursive__open_and_load"))
1129
+
goto cleanup;
1130
+
1131
+
skel->bss->pid = getpid();
1132
+
1133
+
err = uprobe_multi_session_recursive__attach(skel);
1134
+
if (!ASSERT_OK(err, "uprobe_multi_session_recursive__attach"))
1135
+
goto cleanup;
1136
+
1137
+
for (i = 0; i < ARRAY_SIZE(skel->bss->test_uprobe_cookie_entry); i++)
1138
+
skel->bss->test_uprobe_cookie_entry[i] = i + 1;
1139
+
1140
+
uprobe_session_recursive(5);
1141
+
1142
+
/*
1143
+
* entry uprobe:
1144
+
* uprobe_session_recursive(5) { *cookie = 1, return 0
1145
+
* uprobe_session_recursive(4) { *cookie = 2, return 1
1146
+
* uprobe_session_recursive(3) { *cookie = 3, return 0
1147
+
* uprobe_session_recursive(2) { *cookie = 4, return 1
1148
+
* uprobe_session_recursive(1) { *cookie = 5, return 0
1149
+
* uprobe_session_recursive(0) { *cookie = 6, return 1
1150
+
* return uprobe:
1151
+
* } i = 0 not executed
1152
+
* } i = 1 test_uprobe_cookie_return[0] = 5
1153
+
* } i = 2 not executed
1154
+
* } i = 3 test_uprobe_cookie_return[1] = 3
1155
+
* } i = 4 not executed
1156
+
* } i = 5 test_uprobe_cookie_return[2] = 1
1157
+
*/
1158
+
1159
+
ASSERT_EQ(skel->bss->idx_entry, 6, "idx_entry");
1160
+
ASSERT_EQ(skel->bss->idx_return, 3, "idx_return");
1161
+
1162
+
ASSERT_EQ(skel->bss->test_uprobe_cookie_return[0], 5, "test_uprobe_cookie_return[0]");
1163
+
ASSERT_EQ(skel->bss->test_uprobe_cookie_return[1], 3, "test_uprobe_cookie_return[1]");
1164
+
ASSERT_EQ(skel->bss->test_uprobe_cookie_return[2], 1, "test_uprobe_cookie_return[2]");
1165
+
1166
+
cleanup:
1167
+
uprobe_multi_session_recursive__destroy(skel);
1119
1168
}
1120
1169
1121
1170
static void test_bench_attach_uprobe(void)
···
1364
1113
test_pid_filter_process(false);
1365
1114
if (test__start_subtest("filter_clone_vm"))
1366
1115
test_pid_filter_process(true);
1116
+
if (test__start_subtest("session"))
1117
+
test_session_skel_api();
1118
+
if (test__start_subtest("session_single"))
1119
+
test_session_single_skel_api();
1120
+
if (test__start_subtest("session_cookie"))
1121
+
test_session_cookie_skel_api();
1122
+
if (test__start_subtest("session_cookie_recursive"))
1123
+
test_session_recursive_skel_api();
1124
+
RUN_TESTS(uprobe_multi_verifier);
1367
1125
}
+31
tools/testing/selftests/bpf/progs/kprobe_multi_verifier.c
+31
tools/testing/selftests/bpf/progs/kprobe_multi_verifier.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#include "vmlinux.h"
3
+
#include <bpf/bpf_helpers.h>
4
+
#include <bpf/bpf_tracing.h>
5
+
#include <bpf/usdt.bpf.h>
6
+
#include "bpf_misc.h"
7
+
8
+
char _license[] SEC("license") = "GPL";
9
+
10
+
11
+
SEC("kprobe.session")
12
+
__success
13
+
int kprobe_session_return_0(struct pt_regs *ctx)
14
+
{
15
+
return 0;
16
+
}
17
+
18
+
SEC("kprobe.session")
19
+
__success
20
+
int kprobe_session_return_1(struct pt_regs *ctx)
21
+
{
22
+
return 1;
23
+
}
24
+
25
+
SEC("kprobe.session")
26
+
__failure
27
+
__msg("At program exit the register R0 has smin=2 smax=2 should have been in [0, 1]")
28
+
int kprobe_session_return_2(struct pt_regs *ctx)
29
+
{
30
+
return 2;
31
+
}
+3
-3
tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c
+3
-3
tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c
···
24
24
return 0;
25
25
}
26
26
27
-
SEC("uprobe.multi")
27
+
SEC("uprobe.session")
28
28
int uprobe_2(struct pt_regs *ctx)
29
29
{
30
30
uprobe_result[2]++;
31
31
return 0;
32
32
}
33
33
34
-
SEC("uprobe.multi")
34
+
SEC("uprobe.session")
35
35
int uprobe_3(struct pt_regs *ctx)
36
36
{
37
37
uprobe_result[3]++;
38
-
return 0;
38
+
return 1;
39
39
}
+71
tools/testing/selftests/bpf/progs/uprobe_multi_session.c
+71
tools/testing/selftests/bpf/progs/uprobe_multi_session.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#include <linux/bpf.h>
3
+
#include <bpf/bpf_helpers.h>
4
+
#include <bpf/bpf_tracing.h>
5
+
#include <stdbool.h>
6
+
#include "bpf_kfuncs.h"
7
+
#include "bpf_misc.h"
8
+
9
+
char _license[] SEC("license") = "GPL";
10
+
11
+
__u64 uprobe_multi_func_1_addr = 0;
12
+
__u64 uprobe_multi_func_2_addr = 0;
13
+
__u64 uprobe_multi_func_3_addr = 0;
14
+
15
+
__u64 uprobe_session_result[3] = {};
16
+
__u64 uprobe_multi_sleep_result = 0;
17
+
18
+
void *user_ptr = 0;
19
+
int pid = 0;
20
+
21
+
static int uprobe_multi_check(void *ctx, bool is_return)
22
+
{
23
+
const __u64 funcs[] = {
24
+
uprobe_multi_func_1_addr,
25
+
uprobe_multi_func_2_addr,
26
+
uprobe_multi_func_3_addr,
27
+
};
28
+
unsigned int i;
29
+
__u64 addr;
30
+
31
+
if (bpf_get_current_pid_tgid() >> 32 != pid)
32
+
return 1;
33
+
34
+
addr = bpf_get_func_ip(ctx);
35
+
36
+
for (i = 0; i < ARRAY_SIZE(funcs); i++) {
37
+
if (funcs[i] == addr) {
38
+
uprobe_session_result[i]++;
39
+
break;
40
+
}
41
+
}
42
+
43
+
/* only uprobe_multi_func_2 executes return probe */
44
+
if ((addr == uprobe_multi_func_1_addr) ||
45
+
(addr == uprobe_multi_func_3_addr))
46
+
return 1;
47
+
48
+
return 0;
49
+
}
50
+
51
+
SEC("uprobe.session//proc/self/exe:uprobe_multi_func_*")
52
+
int uprobe(struct pt_regs *ctx)
53
+
{
54
+
return uprobe_multi_check(ctx, bpf_session_is_return());
55
+
}
56
+
57
+
static __always_inline bool verify_sleepable_user_copy(void)
58
+
{
59
+
char data[9];
60
+
61
+
bpf_copy_from_user(data, sizeof(data), user_ptr);
62
+
return bpf_strncmp(data, sizeof(data), "test_data") == 0;
63
+
}
64
+
65
+
SEC("uprobe.session.s//proc/self/exe:uprobe_multi_func_*")
66
+
int uprobe_sleepable(struct pt_regs *ctx)
67
+
{
68
+
if (verify_sleepable_user_copy())
69
+
uprobe_multi_sleep_result++;
70
+
return uprobe_multi_check(ctx, bpf_session_is_return());
71
+
}
+44
tools/testing/selftests/bpf/progs/uprobe_multi_session_recursive.c
+44
tools/testing/selftests/bpf/progs/uprobe_multi_session_recursive.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#include <linux/bpf.h>
3
+
#include <bpf/bpf_helpers.h>
4
+
#include <bpf/bpf_tracing.h>
5
+
#include <stdbool.h>
6
+
#include "bpf_kfuncs.h"
7
+
#include "bpf_misc.h"
8
+
9
+
char _license[] SEC("license") = "GPL";
10
+
11
+
int pid = 0;
12
+
13
+
int idx_entry = 0;
14
+
int idx_return = 0;
15
+
16
+
__u64 test_uprobe_cookie_entry[6];
17
+
__u64 test_uprobe_cookie_return[3];
18
+
19
+
static int check_cookie(void)
20
+
{
21
+
__u64 *cookie = bpf_session_cookie();
22
+
23
+
if (bpf_session_is_return()) {
24
+
if (idx_return >= ARRAY_SIZE(test_uprobe_cookie_return))
25
+
return 1;
26
+
test_uprobe_cookie_return[idx_return++] = *cookie;
27
+
return 0;
28
+
}
29
+
30
+
if (idx_entry >= ARRAY_SIZE(test_uprobe_cookie_entry))
31
+
return 1;
32
+
*cookie = test_uprobe_cookie_entry[idx_entry];
33
+
return idx_entry++ % 2;
34
+
}
35
+
36
+
37
+
SEC("uprobe.session//proc/self/exe:uprobe_session_recursive")
38
+
int uprobe_recursive(struct pt_regs *ctx)
39
+
{
40
+
if (bpf_get_current_pid_tgid() >> 32 != pid)
41
+
return 1;
42
+
43
+
return check_cookie();
44
+
}
+44
tools/testing/selftests/bpf/progs/uprobe_multi_session_single.c
+44
tools/testing/selftests/bpf/progs/uprobe_multi_session_single.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#include <linux/bpf.h>
3
+
#include <bpf/bpf_helpers.h>
4
+
#include <bpf/bpf_tracing.h>
5
+
#include <stdbool.h>
6
+
#include "bpf_kfuncs.h"
7
+
#include "bpf_misc.h"
8
+
9
+
char _license[] SEC("license") = "GPL";
10
+
11
+
__u64 uprobe_session_result[3] = {};
12
+
int pid = 0;
13
+
14
+
static int uprobe_multi_check(void *ctx, int idx)
15
+
{
16
+
if (bpf_get_current_pid_tgid() >> 32 != pid)
17
+
return 1;
18
+
19
+
uprobe_session_result[idx]++;
20
+
21
+
/* only consumer 1 executes return probe */
22
+
if (idx == 0 || idx == 2)
23
+
return 1;
24
+
25
+
return 0;
26
+
}
27
+
28
+
SEC("uprobe.session//proc/self/exe:uprobe_multi_func_1")
29
+
int uprobe_0(struct pt_regs *ctx)
30
+
{
31
+
return uprobe_multi_check(ctx, 0);
32
+
}
33
+
34
+
SEC("uprobe.session//proc/self/exe:uprobe_multi_func_1")
35
+
int uprobe_1(struct pt_regs *ctx)
36
+
{
37
+
return uprobe_multi_check(ctx, 1);
38
+
}
39
+
40
+
SEC("uprobe.session//proc/self/exe:uprobe_multi_func_1")
41
+
int uprobe_2(struct pt_regs *ctx)
42
+
{
43
+
return uprobe_multi_check(ctx, 2);
44
+
}
+31
tools/testing/selftests/bpf/progs/uprobe_multi_verifier.c
+31
tools/testing/selftests/bpf/progs/uprobe_multi_verifier.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#include "vmlinux.h"
3
+
#include <bpf/bpf_helpers.h>
4
+
#include <bpf/bpf_tracing.h>
5
+
#include <bpf/usdt.bpf.h>
6
+
#include "bpf_misc.h"
7
+
8
+
char _license[] SEC("license") = "GPL";
9
+
10
+
11
+
SEC("uprobe.session")
12
+
__success
13
+
int uprobe_sesison_return_0(struct pt_regs *ctx)
14
+
{
15
+
return 0;
16
+
}
17
+
18
+
SEC("uprobe.session")
19
+
__success
20
+
int uprobe_sesison_return_1(struct pt_regs *ctx)
21
+
{
22
+
return 1;
23
+
}
24
+
25
+
SEC("uprobe.session")
26
+
__failure
27
+
__msg("At program exit the register R0 has smin=2 smax=2 should have been in [0, 1]")
28
+
int uprobe_sesison_return_2(struct pt_regs *ctx)
29
+
{
30
+
return 2;
31
+
}