Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'bpf-arm64-add-fsession-support'
Leon Hwang says:
====================
Similar to commit 98770bd4e6df ("bpf,x86: add fsession support for x86_64"),
add fsession support on arm64.
Patch #1 adds bpf_jit_supports_fsession() to prevent fsession loading
on architectures that do not implement fsession support.
Patch #2 implements fsession support in the arm64 BPF JIT trampoline.
Patch #3 enables the relevant selftests on arm64, including get_func_ip,
and get_func_args.
All enabled tests pass on arm64:
cd tools/testing/selftests/bpf
./test_progs -t fsession
#136/1 fsession_test/fsession_test:OK
#136/2 fsession_test/fsession_reattach:OK
#136/3 fsession_test/fsession_cookie:OK
#136 fsession_test:OK
Summary: 1/3 PASSED, 0 SKIPPED, 0 FAILED
./test_progs -t get_func
#138 get_func_args_test:OK
#139 get_func_ip_test:OK
Summary: 2/0 PASSED, 0 SKIPPED, 0 FAILED
Changes:
v4 -> v5:
* Address comment from Alexei:
* Rename helper bpf_link_prog_session_cookie() to
bpf_prog_calls_session_cookie().
* v4: https://lore.kernel.org/bpf/20260129154953.66915-1-leon.hwang@linux.dev/
v3 -> v4:
* Add a log when !bpf_jit_supports_fsession() in patch #1 (per AI).
* v3: https://lore.kernel.org/bpf/20260129142536.48637-1-leon.hwang@linux.dev/
v2 -> v3:
* Fix typo in subject and patch message of patch #1 (per AI and Chris).
* Collect Acked-by, and Tested-by from Puranjay, thanks.
* v2: https://lore.kernel.org/bpf/20260128150112.8873-1-leon.hwang@linux.dev/
v1 -> v2:
* Add bpf_jit_supports_fsession().
* v1: https://lore.kernel.org/bpf/20260127163344.92819-1-leon.hwang@linux.dev/
====================
Link: https://patch.msgid.link/20260131144950.16294-1-leon.hwang@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+110
-20
+62
-9
arch/arm64/net/bpf_jit_comp.c
+62
-9
arch/arm64/net/bpf_jit_comp.c
···
2510
2510
fentry_links->links[0]->link.type == BPF_LINK_TYPE_STRUCT_OPS;
2511
2511
}
2512
2512
2513
+
static void store_func_meta(struct jit_ctx *ctx, u64 func_meta, int func_meta_off)
2514
+
{
2515
+
emit_a64_mov_i64(A64_R(10), func_meta, ctx);
2516
+
emit(A64_STR64I(A64_R(10), A64_SP, func_meta_off), ctx);
2517
+
}
2518
+
2513
2519
/* Based on the x86's implementation of arch_prepare_bpf_trampoline().
2514
2520
*
2515
2521
* bpf prog and function entry before bpf trampoline hooked:
···
2539
2533
int regs_off;
2540
2534
int retval_off;
2541
2535
int bargs_off;
2542
-
int nfuncargs_off;
2536
+
int func_meta_off;
2543
2537
int ip_off;
2544
2538
int run_ctx_off;
2545
2539
int oargs_off;
···
2550
2544
bool save_ret;
2551
2545
__le32 **branches = NULL;
2552
2546
bool is_struct_ops = is_struct_ops_tramp(fentry);
2547
+
int cookie_off, cookie_cnt, cookie_bargs_off;
2548
+
int fsession_cnt = bpf_fsession_cnt(tlinks);
2549
+
u64 func_meta;
2553
2550
2554
2551
/* trampoline stack layout:
2555
2552
* [ parent ip ]
···
2571
2562
* [ ... ]
2572
2563
* SP + bargs_off [ arg reg 1 ] for bpf
2573
2564
*
2574
-
* SP + nfuncargs_off [ arg regs count ]
2565
+
* SP + func_meta_off [ regs count, etc ]
2575
2566
*
2576
2567
* SP + ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag
2568
+
*
2569
+
* [ stack cookie N ]
2570
+
* [ ... ]
2571
+
* SP + cookie_off [ stack cookie 1 ]
2577
2572
*
2578
2573
* SP + run_ctx_off [ bpf_tramp_run_ctx ]
2579
2574
*
···
2595
2582
/* room for bpf_tramp_run_ctx */
2596
2583
stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
2597
2584
2585
+
cookie_off = stack_size;
2586
+
/* room for session cookies */
2587
+
cookie_cnt = bpf_fsession_cookie_cnt(tlinks);
2588
+
stack_size += cookie_cnt * 8;
2589
+
2598
2590
ip_off = stack_size;
2599
2591
/* room for IP address argument */
2600
2592
if (flags & BPF_TRAMP_F_IP_ARG)
2601
2593
stack_size += 8;
2602
2594
2603
-
nfuncargs_off = stack_size;
2604
-
/* room for args count */
2595
+
func_meta_off = stack_size;
2596
+
/* room for function metadata, such as regs count */
2605
2597
stack_size += 8;
2606
2598
2607
2599
bargs_off = stack_size;
···
2664
2646
emit(A64_STR64I(A64_R(10), A64_SP, ip_off), ctx);
2665
2647
}
2666
2648
2667
-
/* save arg regs count*/
2668
-
emit(A64_MOVZ(1, A64_R(10), nfuncargs, 0), ctx);
2669
-
emit(A64_STR64I(A64_R(10), A64_SP, nfuncargs_off), ctx);
2649
+
/* save function metadata */
2650
+
func_meta = nfuncargs;
2651
+
store_func_meta(ctx, func_meta, func_meta_off);
2670
2652
2671
2653
/* save args for bpf */
2672
2654
save_args(ctx, bargs_off, oargs_off, m, a, false);
···
2684
2666
emit_call((const u64)__bpf_tramp_enter, ctx);
2685
2667
}
2686
2668
2687
-
for (i = 0; i < fentry->nr_links; i++)
2669
+
if (fsession_cnt) {
2670
+
/* clear all the session cookies' value */
2671
+
emit(A64_MOVZ(1, A64_R(10), 0, 0), ctx);
2672
+
for (int i = 0; i < cookie_cnt; i++)
2673
+
emit(A64_STR64I(A64_R(10), A64_SP, cookie_off + 8 * i), ctx);
2674
+
/* clear the return value to make sure fentry always gets 0 */
2675
+
emit(A64_STR64I(A64_R(10), A64_SP, retval_off), ctx);
2676
+
}
2677
+
2678
+
cookie_bargs_off = (bargs_off - cookie_off) / 8;
2679
+
for (i = 0; i < fentry->nr_links; i++) {
2680
+
if (bpf_prog_calls_session_cookie(fentry->links[i])) {
2681
+
u64 meta = func_meta | (cookie_bargs_off << BPF_TRAMP_COOKIE_INDEX_SHIFT);
2682
+
2683
+
store_func_meta(ctx, meta, func_meta_off);
2684
+
cookie_bargs_off--;
2685
+
}
2688
2686
invoke_bpf_prog(ctx, fentry->links[i], bargs_off,
2689
2687
retval_off, run_ctx_off,
2690
2688
flags & BPF_TRAMP_F_RET_FENTRY_RET);
2689
+
}
2691
2690
2692
2691
if (fmod_ret->nr_links) {
2693
2692
branches = kcalloc(fmod_ret->nr_links, sizeof(__le32 *),
···
2736
2701
*branches[i] = cpu_to_le32(A64_CBNZ(1, A64_R(10), offset));
2737
2702
}
2738
2703
2739
-
for (i = 0; i < fexit->nr_links; i++)
2704
+
/* set the "is_return" flag for fsession */
2705
+
func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);
2706
+
if (fsession_cnt)
2707
+
store_func_meta(ctx, func_meta, func_meta_off);
2708
+
2709
+
cookie_bargs_off = (bargs_off - cookie_off) / 8;
2710
+
for (i = 0; i < fexit->nr_links; i++) {
2711
+
if (bpf_prog_calls_session_cookie(fexit->links[i])) {
2712
+
u64 meta = func_meta | (cookie_bargs_off << BPF_TRAMP_COOKIE_INDEX_SHIFT);
2713
+
2714
+
store_func_meta(ctx, meta, func_meta_off);
2715
+
cookie_bargs_off--;
2716
+
}
2740
2717
invoke_bpf_prog(ctx, fexit->links[i], bargs_off, retval_off,
2741
2718
run_ctx_off, false);
2719
+
}
2742
2720
2743
2721
if (flags & BPF_TRAMP_F_CALL_ORIG) {
2744
2722
im->ip_epilogue = ctx->ro_image + ctx->idx;
···
2799
2751
kfree(branches);
2800
2752
2801
2753
return ctx->idx;
2754
+
}
2755
+
2756
+
bool bpf_jit_supports_fsession(void)
2757
+
{
2758
+
return true;
2802
2759
}
2803
2760
2804
2761
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
+5
arch/x86/net/bpf_jit_comp.c
+5
arch/x86/net/bpf_jit_comp.c
+6
-1
include/linux/bpf.h
+6
-1
include/linux/bpf.h
···
2196
2196
return cnt;
2197
2197
}
2198
2198
2199
+
static inline bool bpf_prog_calls_session_cookie(struct bpf_tramp_link *link)
2200
+
{
2201
+
return link->link.prog->call_session_cookie;
2202
+
}
2203
+
2199
2204
static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_links *links)
2200
2205
{
2201
2206
struct bpf_tramp_links fentries = links[BPF_TRAMP_FENTRY];
2202
2207
int cnt = 0;
2203
2208
2204
2209
for (int i = 0; i < links[BPF_TRAMP_FENTRY].nr_links; i++) {
2205
-
if (fentries.links[i]->link.prog->call_session_cookie)
2210
+
if (bpf_prog_calls_session_cookie(fentries.links[i]))
2206
2211
cnt++;
2207
2212
}
2208
2213
+1
include/linux/filter.h
+1
include/linux/filter.h
···
1167
1167
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
1168
1168
bool bpf_jit_supports_private_stack(void);
1169
1169
bool bpf_jit_supports_timed_may_goto(void);
1170
+
bool bpf_jit_supports_fsession(void);
1170
1171
u64 bpf_arch_uaddress_limit(void);
1171
1172
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
1172
1173
u64 arch_bpf_timed_may_goto(void);
+5
kernel/bpf/core.c
+5
kernel/bpf/core.c
···
3144
3144
return false;
3145
3145
}
3146
3146
3147
+
bool __weak bpf_jit_supports_fsession(void)
3148
+
{
3149
+
return false;
3150
+
}
3151
+
3147
3152
u64 __weak bpf_arch_uaddress_limit(void)
3148
3153
{
3149
3154
#if defined(CONFIG_64BIT) && defined(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE)
+5
kernel/bpf/verifier.c
+5
kernel/bpf/verifier.c
···
24828
24828
case BPF_TRACE_FENTRY:
24829
24829
case BPF_TRACE_FEXIT:
24830
24830
case BPF_TRACE_FSESSION:
24831
+
if (prog->expected_attach_type == BPF_TRACE_FSESSION &&
24832
+
!bpf_jit_supports_fsession()) {
24833
+
bpf_log(log, "JIT does not support fsession\n");
24834
+
return -EOPNOTSUPP;
24835
+
}
24831
24836
if (!btf_type_is_func(t)) {
24832
24837
bpf_log(log, "attach_btf_id %u is not a function\n",
24833
24838
btf_id);
+24
-8
tools/testing/selftests/bpf/prog_tests/fsession_test.c
+24
-8
tools/testing/selftests/bpf/prog_tests/fsession_test.c
···
29
29
struct fsession_test *skel = NULL;
30
30
int err;
31
31
32
-
skel = fsession_test__open_and_load();
33
-
if (!ASSERT_OK_PTR(skel, "fsession_test__open_and_load"))
32
+
skel = fsession_test__open();
33
+
if (!ASSERT_OK_PTR(skel, "fsession_test__open"))
34
+
return;
35
+
36
+
err = fsession_test__load(skel);
37
+
if (err == -EOPNOTSUPP) {
38
+
test__skip();
39
+
goto cleanup;
40
+
}
41
+
if (!ASSERT_OK(err, "fsession_test__load"))
34
42
goto cleanup;
35
43
36
44
err = fsession_test__attach(skel);
···
55
47
struct fsession_test *skel = NULL;
56
48
int err;
57
49
58
-
skel = fsession_test__open_and_load();
59
-
if (!ASSERT_OK_PTR(skel, "fsession_test__open_and_load"))
50
+
skel = fsession_test__open();
51
+
if (!ASSERT_OK_PTR(skel, "fsession_test__open"))
52
+
return;
53
+
54
+
err = fsession_test__load(skel);
55
+
if (err == -EOPNOTSUPP) {
56
+
test__skip();
57
+
goto cleanup;
58
+
}
59
+
if (!ASSERT_OK(err, "fsession_test__load"))
60
60
goto cleanup;
61
61
62
62
/* first attach */
···
110
94
bpf_program__set_autoload(skel->progs.test6, false);
111
95
112
96
err = fsession_test__load(skel);
97
+
if (err == -EOPNOTSUPP) {
98
+
test__skip();
99
+
goto cleanup;
100
+
}
113
101
if (!ASSERT_OK(err, "fsession_test__load"))
114
102
goto cleanup;
115
103
···
131
111
132
112
void test_fsession_test(void)
133
113
{
134
-
#if !defined(__x86_64__)
135
-
test__skip();
136
-
return;
137
-
#endif
138
114
if (test__start_subtest("fsession_test"))
139
115
test_fsession_basic();
140
116
if (test__start_subtest("fsession_reattach"))
+1
-1
tools/testing/selftests/bpf/progs/get_func_args_test.c
+1
-1
tools/testing/selftests/bpf/progs/get_func_args_test.c
+1
-1
tools/testing/selftests/bpf/progs/get_func_ip_test.c
+1
-1
tools/testing/selftests/bpf/progs/get_func_ip_test.c