Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'x86-fgraph-bpf-fix-orc-stack-unwind-from-kprobe_multi'
Jiri Olsa says:
====================
x86/fgraph,bpf: Fix ORC stack unwind from kprobe_multi
hi,
Mahe reported missing function from stack trace on top of kprobe multi
program. It turned out the latest fix [1] needs some more fixing.
v2 changes:
- keep the unwind same as for kprobes, attached function
is part of entry probe stacktrace, not kretprobe [Steven]
- several change in trigger bench [Andrii]
- added selftests for standard kprobes and fentry/fexit probes [Andrii]
Note I'll try to add similar stacktrace adjustment for fentry/fexit
in separate patchset to not complicate this change.
thanks,
jirka
[1] https://lore.kernel.org/bpf/20251104215405.168643-1-jolsa@kernel.org/
---
====================
Link: https://patch.msgid.link/20260126211837.472802-1-jolsa@kernel.org
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
+190
-16
+1
-1
arch/x86/include/asm/ftrace.h
+1
-1
arch/x86/include/asm/ftrace.h
+4
-1
arch/x86/kernel/ftrace_64.S
+4
-1
arch/x86/kernel/ftrace_64.S
···
364
364
UNWIND_HINT_UNDEFINED
365
365
ANNOTATE_NOENDBR
366
366
367
+
/* Store original rsp for pt_regs.sp value. */
368
+
movq %rsp, %rdi
369
+
367
370
/* Restore return_to_handler value that got eaten by previous ret instruction. */
368
371
subq $8, %rsp
369
372
UNWIND_HINT_FUNC
···
377
374
movq %rax, RAX(%rsp)
378
375
movq %rdx, RDX(%rsp)
379
376
movq %rbp, RBP(%rsp)
380
-
movq %rsp, RSP(%rsp)
377
+
movq %rdi, RSP(%rsp)
381
378
movq %rsp, %rdi
382
379
383
380
call ftrace_return_to_handler
+4
tools/testing/selftests/bpf/bench.c
+4
tools/testing/selftests/bpf/bench.c
···
265
265
{ "verbose", 'v', NULL, 0, "Verbose debug output"},
266
266
{ "affinity", 'a', NULL, 0, "Set consumer/producer thread affinity"},
267
267
{ "quiet", 'q', NULL, 0, "Be more quiet"},
268
+
{ "stacktrace", 's', NULL, 0, "Get stack trace"},
268
269
{ "prod-affinity", ARG_PROD_AFFINITY_SET, "CPUSET", 0,
269
270
"Set of CPUs for producer threads; implies --affinity"},
270
271
{ "cons-affinity", ARG_CONS_AFFINITY_SET, "CPUSET", 0,
···
350
349
break;
351
350
case 'q':
352
351
env.quiet = true;
352
+
break;
353
+
case 's':
354
+
env.stacktrace = true;
353
355
break;
354
356
case ARG_PROD_AFFINITY_SET:
355
357
env.affinity = true;
+1
tools/testing/selftests/bpf/bench.h
+1
tools/testing/selftests/bpf/bench.h
+1
tools/testing/selftests/bpf/benchs/bench_trigger.c
+1
tools/testing/selftests/bpf/benchs/bench_trigger.c
+115
-5
tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c
+115
-5
tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c
···
74
74
75
75
load_kallsyms();
76
76
77
-
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4,
78
-
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
79
-
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
80
-
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
81
-
ksym_get_addr("bpf_testmod_test_read"));
77
+
if (retprobe) {
78
+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4,
79
+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
80
+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
81
+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
82
+
ksym_get_addr("bpf_testmod_test_read"));
83
+
} else {
84
+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 5,
85
+
ksym_get_addr("bpf_testmod_stacktrace_test"),
86
+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
87
+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
88
+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
89
+
ksym_get_addr("bpf_testmod_test_read"));
90
+
}
82
91
83
92
cleanup:
84
93
stacktrace_ips__destroy(skel);
···
137
128
stacktrace_ips__destroy(skel);
138
129
}
139
130
131
+
static void test_stacktrace_ips_kprobe(bool retprobe)
132
+
{
133
+
LIBBPF_OPTS(bpf_kprobe_opts, opts,
134
+
.retprobe = retprobe
135
+
);
136
+
LIBBPF_OPTS(bpf_test_run_opts, topts);
137
+
struct stacktrace_ips *skel;
138
+
139
+
skel = stacktrace_ips__open_and_load();
140
+
if (!ASSERT_OK_PTR(skel, "stacktrace_ips__open_and_load"))
141
+
return;
142
+
143
+
if (!skel->kconfig->CONFIG_UNWINDER_ORC) {
144
+
test__skip();
145
+
goto cleanup;
146
+
}
147
+
148
+
skel->links.kprobe_test = bpf_program__attach_kprobe_opts(
149
+
skel->progs.kprobe_test,
150
+
"bpf_testmod_stacktrace_test", &opts);
151
+
if (!ASSERT_OK_PTR(skel->links.kprobe_test, "bpf_program__attach_kprobe_opts"))
152
+
goto cleanup;
153
+
154
+
trigger_module_test_read(1);
155
+
156
+
load_kallsyms();
157
+
158
+
if (retprobe) {
159
+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4,
160
+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
161
+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
162
+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
163
+
ksym_get_addr("bpf_testmod_test_read"));
164
+
} else {
165
+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 5,
166
+
ksym_get_addr("bpf_testmod_stacktrace_test"),
167
+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
168
+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
169
+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
170
+
ksym_get_addr("bpf_testmod_test_read"));
171
+
}
172
+
173
+
cleanup:
174
+
stacktrace_ips__destroy(skel);
175
+
}
176
+
177
+
static void test_stacktrace_ips_trampoline(bool retprobe)
178
+
{
179
+
LIBBPF_OPTS(bpf_test_run_opts, topts);
180
+
struct stacktrace_ips *skel;
181
+
182
+
skel = stacktrace_ips__open_and_load();
183
+
if (!ASSERT_OK_PTR(skel, "stacktrace_ips__open_and_load"))
184
+
return;
185
+
186
+
if (!skel->kconfig->CONFIG_UNWINDER_ORC) {
187
+
test__skip();
188
+
goto cleanup;
189
+
}
190
+
191
+
if (retprobe) {
192
+
skel->links.fexit_test = bpf_program__attach_trace(skel->progs.fexit_test);
193
+
if (!ASSERT_OK_PTR(skel->links.fexit_test, "bpf_program__attach_trace"))
194
+
goto cleanup;
195
+
} else {
196
+
skel->links.fentry_test = bpf_program__attach_trace(skel->progs.fentry_test);
197
+
if (!ASSERT_OK_PTR(skel->links.fentry_test, "bpf_program__attach_trace"))
198
+
goto cleanup;
199
+
}
200
+
201
+
trigger_module_test_read(1);
202
+
203
+
load_kallsyms();
204
+
205
+
if (retprobe) {
206
+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4,
207
+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
208
+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
209
+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
210
+
ksym_get_addr("bpf_testmod_test_read"));
211
+
} else {
212
+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 5,
213
+
ksym_get_addr("bpf_testmod_stacktrace_test"),
214
+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
215
+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
216
+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
217
+
ksym_get_addr("bpf_testmod_test_read"));
218
+
}
219
+
220
+
cleanup:
221
+
stacktrace_ips__destroy(skel);
222
+
}
223
+
140
224
static void __test_stacktrace_ips(void)
141
225
{
142
226
if (test__start_subtest("kprobe_multi"))
···
238
136
test_stacktrace_ips_kprobe_multi(true);
239
137
if (test__start_subtest("raw_tp"))
240
138
test_stacktrace_ips_raw_tp();
139
+
if (test__start_subtest("kprobe"))
140
+
test_stacktrace_ips_kprobe(false);
141
+
if (test__start_subtest("kretprobe"))
142
+
test_stacktrace_ips_kprobe(true);
143
+
if (test__start_subtest("fentry"))
144
+
test_stacktrace_ips_trampoline(false);
145
+
if (test__start_subtest("fexit"))
146
+
test_stacktrace_ips_trampoline(true);
241
147
}
242
148
#else
243
149
static void __test_stacktrace_ips(void)
+27
tools/testing/selftests/bpf/progs/stacktrace_ips.c
+27
tools/testing/selftests/bpf/progs/stacktrace_ips.c
···
31
31
32
32
__u32 stack_key;
33
33
34
+
SEC("kprobe")
35
+
int kprobe_test(struct pt_regs *ctx)
36
+
{
37
+
stack_key = bpf_get_stackid(ctx, &stackmap, 0);
38
+
return 0;
39
+
}
40
+
34
41
SEC("kprobe.multi")
35
42
int kprobe_multi_test(struct pt_regs *ctx)
36
43
{
···
50
43
{
51
44
/* Skip ebpf program entry in the stack. */
52
45
stack_key = bpf_get_stackid(ctx, &stackmap, 0);
46
+
return 0;
47
+
}
48
+
49
+
SEC("fentry/bpf_testmod_stacktrace_test")
50
+
int fentry_test(struct pt_regs *ctx)
51
+
{
52
+
/*
53
+
* Skip 2 bpf_program/trampoline stack entries:
54
+
* - bpf_prog_bd1f7a949f55fb03_fentry_test
55
+
* - bpf_trampoline_182536277701
56
+
*/
57
+
stack_key = bpf_get_stackid(ctx, &stackmap, 2);
58
+
return 0;
59
+
}
60
+
61
+
SEC("fexit/bpf_testmod_stacktrace_test")
62
+
int fexit_test(struct pt_regs *ctx)
63
+
{
64
+
/* Skip 2 bpf_program/trampoline stack entries, check fentry_test. */
65
+
stack_key = bpf_get_stackid(ctx, &stackmap, 2);
53
66
return 0;
54
67
}
55
68
+37
-9
tools/testing/selftests/bpf/progs/trigger_bench.c
+37
-9
tools/testing/selftests/bpf/progs/trigger_bench.c
···
25
25
__sync_add_and_fetch(&hits[cpu & CPU_MASK].value, 1);
26
26
}
27
27
28
+
volatile const int stacktrace;
29
+
30
+
typedef __u64 stack_trace_t[128];
31
+
32
+
struct {
33
+
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
34
+
__uint(max_entries, 1);
35
+
__type(key, __u32);
36
+
__type(value, stack_trace_t);
37
+
} stack_heap SEC(".maps");
38
+
39
+
static __always_inline void do_stacktrace(void *ctx)
40
+
{
41
+
if (!stacktrace)
42
+
return;
43
+
44
+
__u64 *ptr = bpf_map_lookup_elem(&stack_heap, &(__u32){0});
45
+
46
+
if (ptr)
47
+
bpf_get_stack(ctx, ptr, sizeof(stack_trace_t), 0);
48
+
}
49
+
50
+
static __always_inline void handle(void *ctx)
51
+
{
52
+
inc_counter();
53
+
do_stacktrace(ctx);
54
+
}
55
+
28
56
SEC("?uprobe")
29
57
int bench_trigger_uprobe(void *ctx)
30
58
{
···
109
81
SEC("?kprobe/bpf_get_numa_node_id")
110
82
int bench_trigger_kprobe(void *ctx)
111
83
{
112
-
inc_counter();
84
+
handle(ctx);
113
85
return 0;
114
86
}
115
87
116
88
SEC("?kretprobe/bpf_get_numa_node_id")
117
89
int bench_trigger_kretprobe(void *ctx)
118
90
{
119
-
inc_counter();
91
+
handle(ctx);
120
92
return 0;
121
93
}
122
94
123
95
SEC("?kprobe.multi/bpf_get_numa_node_id")
124
96
int bench_trigger_kprobe_multi(void *ctx)
125
97
{
126
-
inc_counter();
98
+
handle(ctx);
127
99
return 0;
128
100
}
129
101
···
136
108
SEC("?kretprobe.multi/bpf_get_numa_node_id")
137
109
int bench_trigger_kretprobe_multi(void *ctx)
138
110
{
139
-
inc_counter();
111
+
handle(ctx);
140
112
return 0;
141
113
}
142
114
···
149
121
SEC("?fentry/bpf_get_numa_node_id")
150
122
int bench_trigger_fentry(void *ctx)
151
123
{
152
-
inc_counter();
124
+
handle(ctx);
153
125
return 0;
154
126
}
155
127
156
128
SEC("?fexit/bpf_get_numa_node_id")
157
129
int bench_trigger_fexit(void *ctx)
158
130
{
159
-
inc_counter();
131
+
handle(ctx);
160
132
return 0;
161
133
}
162
134
163
135
SEC("?fmod_ret/bpf_modify_return_test_tp")
164
136
int bench_trigger_fmodret(void *ctx)
165
137
{
166
-
inc_counter();
138
+
handle(ctx);
167
139
return -22;
168
140
}
169
141
170
142
SEC("?tp/bpf_test_run/bpf_trigger_tp")
171
143
int bench_trigger_tp(void *ctx)
172
144
{
173
-
inc_counter();
145
+
handle(ctx);
174
146
return 0;
175
147
}
176
148
177
149
SEC("?raw_tp/bpf_trigger_tp")
178
150
int bench_trigger_rawtp(void *ctx)
179
151
{
180
-
inc_counter();
152
+
handle(ctx);
181
153
return 0;
182
154
}