Merge branch 'bpf-fsession-support-for-s390'

+70 -25

arch/s390/net/bpf_jit_comp.c

··· 2487 2487 int ip_off; /* For bpf_get_func_ip(), has to be at 2488 2488 * (ctx - 16) 2489 2489 */ 2490 - int arg_cnt_off; /* For bpf_get_func_arg_cnt(), has to be at 2491 - * (ctx - 8) 2490 + int func_meta_off; /* For bpf_get_func_arg_cnt()/fsession, has 2491 + * to be at (ctx - 8) 2492 2492 */ 2493 2493 int bpf_args_off; /* Offset of BPF_PROG context, which consists 2494 2494 * of BPF arguments followed by return value ··· 2513 2513 EMIT6_IMM(0xc00d0000, dst_reg, val); 2514 2514 } 2515 2515 2516 + static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_off, u64 imm) 2517 + { 2518 + load_imm64(jit, tmp_reg, imm); 2519 + /* stg %tmp_reg,stack_off(%r15) */ 2520 + EMIT6_DISP_LH(0xe3000000, 0x0024, tmp_reg, REG_0, REG_15, stack_off); 2521 + } 2522 + 2516 2523 static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, 2517 2524 const struct btf_func_model *m, 2518 2525 struct bpf_tramp_link *tlink, bool save_ret) ··· 2534 2527 * run_ctx.cookie = tlink->cookie; 2535 2528 */ 2536 2529 2537 - /* %r0 = tlink->cookie */ 2538 - load_imm64(jit, REG_W0, tlink->cookie); 2539 - /* stg %r0,cookie_off(%r15) */ 2540 - EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, REG_0, REG_15, cookie_off); 2530 + emit_store_stack_imm64(jit, REG_W0, cookie_off, tlink->cookie); 2541 2531 2542 2532 /* 2543 2533 * if ((start = __bpf_prog_enter(p, &run_ctx)) == 0) ··· 2592 2588 return 0; 2593 2589 } 2594 2590 2591 + static int invoke_bpf(struct bpf_tramp_jit *tjit, 2592 + const struct btf_func_model *m, 2593 + struct bpf_tramp_links *tl, bool save_ret, 2594 + u64 func_meta, int cookie_off) 2595 + { 2596 + int i, cur_cookie = (tjit->bpf_args_off - cookie_off) / sizeof(u64); 2597 + struct bpf_jit *jit = &tjit->common; 2598 + 2599 + for (i = 0; i < tl->nr_links; i++) { 2600 + if (bpf_prog_calls_session_cookie(tl->links[i])) { 2601 + u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT); 2602 + 2603 + emit_store_stack_imm64(jit, REG_0, tjit->func_meta_off, meta); 2604 + cur_cookie--; 2605 + } 2606 + if (invoke_bpf_prog(tjit, m, tl->links[i], save_ret)) 2607 + return -EINVAL; 2608 + } 2609 + 2610 + return 0; 2611 + } 2612 + 2595 2613 static int alloc_stack(struct bpf_tramp_jit *tjit, size_t size) 2596 2614 { 2597 2615 int stack_offset = tjit->stack_size; ··· 2643 2617 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; 2644 2618 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; 2645 2619 int nr_bpf_args, nr_reg_args, nr_stack_args; 2620 + int cookie_cnt, cookie_off, fsession_cnt; 2646 2621 struct bpf_jit *jit = &tjit->common; 2647 2622 int arg, bpf_arg_off; 2623 + u64 func_meta; 2648 2624 int i, j; 2649 2625 2650 2626 /* Support as many stack arguments as "mvc" instruction can handle. */ ··· 2678 2650 return -ENOTSUPP; 2679 2651 } 2680 2652 2653 + cookie_cnt = bpf_fsession_cookie_cnt(tlinks); 2654 + fsession_cnt = bpf_fsession_cnt(tlinks); 2655 + 2681 2656 /* 2682 2657 * Calculate the stack layout. 2683 2658 */ ··· 2693 2662 tjit->backchain_off = tjit->stack_size - sizeof(u64); 2694 2663 tjit->stack_args_off = alloc_stack(tjit, nr_stack_args * sizeof(u64)); 2695 2664 tjit->reg_args_off = alloc_stack(tjit, nr_reg_args * sizeof(u64)); 2665 + cookie_off = alloc_stack(tjit, cookie_cnt * sizeof(u64)); 2696 2666 tjit->ip_off = alloc_stack(tjit, sizeof(u64)); 2697 - tjit->arg_cnt_off = alloc_stack(tjit, sizeof(u64)); 2667 + tjit->func_meta_off = alloc_stack(tjit, sizeof(u64)); 2698 2668 tjit->bpf_args_off = alloc_stack(tjit, nr_bpf_args * sizeof(u64)); 2699 2669 tjit->retval_off = alloc_stack(tjit, sizeof(u64)); 2700 2670 tjit->r7_r8_off = alloc_stack(tjit, 2 * sizeof(u64)); ··· 2782 2750 * arg_cnt = m->nr_args; 2783 2751 */ 2784 2752 2785 - if (flags & BPF_TRAMP_F_IP_ARG) { 2786 - /* %r0 = func_addr */ 2787 - load_imm64(jit, REG_0, (u64)func_addr); 2788 - /* stg %r0,ip_off(%r15) */ 2789 - EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, 2790 - tjit->ip_off); 2791 - } 2792 - /* lghi %r0,nr_bpf_args */ 2793 - EMIT4_IMM(0xa7090000, REG_0, nr_bpf_args); 2794 - /* stg %r0,arg_cnt_off(%r15) */ 2753 + if (flags & BPF_TRAMP_F_IP_ARG) 2754 + emit_store_stack_imm64(jit, REG_0, tjit->ip_off, (u64)func_addr); 2755 + func_meta = nr_bpf_args; 2756 + /* lghi %r0,func_meta */ 2757 + EMIT4_IMM(0xa7090000, REG_0, func_meta); 2758 + /* stg %r0,func_meta_off(%r15) */ 2795 2759 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, 2796 - tjit->arg_cnt_off); 2760 + tjit->func_meta_off); 2797 2761 2798 2762 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2799 2763 /* ··· 2802 2774 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_enter); 2803 2775 } 2804 2776 2805 - for (i = 0; i < fentry->nr_links; i++) 2806 - if (invoke_bpf_prog(tjit, m, fentry->links[i], 2807 - flags & BPF_TRAMP_F_RET_FENTRY_RET)) 2808 - return -EINVAL; 2777 + if (fsession_cnt) { 2778 + /* Clear all the session cookies' value. */ 2779 + for (i = 0; i < cookie_cnt; i++) 2780 + emit_store_stack_imm64(jit, REG_0, cookie_off + 8 * i, 0); 2781 + /* Clear the return value to make sure fentry always gets 0. */ 2782 + emit_store_stack_imm64(jit, REG_0, tjit->retval_off, 0); 2783 + } 2784 + 2785 + if (invoke_bpf(tjit, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET, 2786 + func_meta, cookie_off)) 2787 + return -EINVAL; 2809 2788 2810 2789 if (fmod_ret->nr_links) { 2811 2790 /* ··· 2889 2854 EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue); 2890 2855 } 2891 2856 2857 + /* Set the "is_return" flag for fsession. */ 2858 + func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); 2859 + if (fsession_cnt) 2860 + emit_store_stack_imm64(jit, REG_W0, tjit->func_meta_off, 2861 + func_meta); 2862 + 2892 2863 /* do_fexit: */ 2893 2864 tjit->do_fexit = jit->prg; 2894 - for (i = 0; i < fexit->nr_links; i++) 2895 - if (invoke_bpf_prog(tjit, m, fexit->links[i], false)) 2896 - return -EINVAL; 2865 + if (invoke_bpf(tjit, m, fexit, false, func_meta, cookie_off)) 2866 + return -EINVAL; 2897 2867 2898 2868 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2899 2869 im->ip_epilogue = jit->prg_buf + jit->prg; ··· 2999 2959 } 3000 2960 3001 2961 bool bpf_jit_supports_arena(void) 2962 + { 2963 + return true; 2964 + } 2965 + 2966 + bool bpf_jit_supports_fsession(void) 3002 2967 { 3003 2968 return true; 3004 2969 }

+24 -1

tools/testing/selftests/bpf/prog_tests/get_func_args_test.c

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 #include <test_progs.h> 3 3 #include "get_func_args_test.skel.h" 4 + #include "get_func_args_fsession_test.skel.h" 4 5 5 6 void test_get_func_args_test(void) 6 7 { ··· 42 41 ASSERT_EQ(skel->bss->test4_result, 1, "test4_result"); 43 42 ASSERT_EQ(skel->bss->test5_result, 1, "test5_result"); 44 43 ASSERT_EQ(skel->bss->test6_result, 1, "test6_result"); 45 - ASSERT_EQ(skel->bss->test7_result, 1, "test7_result"); 46 44 47 45 cleanup: 48 46 get_func_args_test__destroy(skel); 47 + } 48 + 49 + void test_get_func_args_fsession_test(void) 50 + { 51 + struct get_func_args_fsession_test *skel = NULL; 52 + int err; 53 + LIBBPF_OPTS(bpf_test_run_opts, topts); 54 + 55 + skel = get_func_args_fsession_test__open_and_load(); 56 + if (!ASSERT_OK_PTR(skel, "get_func_args_fsession_test__open_and_load")) 57 + return; 58 + 59 + err = get_func_args_fsession_test__attach(skel); 60 + if (!ASSERT_OK(err, "get_func_args_fsession_test__attach")) 61 + goto cleanup; 62 + 63 + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test1), &topts); 64 + ASSERT_OK(err, "test_run"); 65 + ASSERT_EQ(topts.retval, 0, "test_run"); 66 + 67 + ASSERT_EQ(skel->bss->test1_result, 1, "test1_result"); 68 + cleanup: 69 + get_func_args_fsession_test__destroy(skel); 49 70 }

+26 -2

tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c

··· 2 2 #include <test_progs.h> 3 3 #include "get_func_ip_test.skel.h" 4 4 #include "get_func_ip_uprobe_test.skel.h" 5 + #include "get_func_ip_fsession_test.skel.h" 5 6 6 7 static noinline void uprobe_trigger(void) 7 8 { ··· 47 46 ASSERT_EQ(skel->bss->test5_result, 1, "test5_result"); 48 47 ASSERT_EQ(skel->bss->test7_result, 1, "test7_result"); 49 48 ASSERT_EQ(skel->bss->test8_result, 1, "test8_result"); 50 - ASSERT_EQ(skel->bss->test9_entry_result, 1, "test9_entry_result"); 51 - ASSERT_EQ(skel->bss->test9_exit_result, 1, "test9_exit_result"); 52 49 53 50 cleanup: 54 51 get_func_ip_test__destroy(skel); ··· 137 138 { 138 139 test_function_entry(); 139 140 test_function_body(); 141 + } 142 + 143 + void test_get_func_ip_fsession_test(void) 144 + { 145 + struct get_func_ip_fsession_test *skel = NULL; 146 + int err; 147 + LIBBPF_OPTS(bpf_test_run_opts, topts); 148 + 149 + skel = get_func_ip_fsession_test__open_and_load(); 150 + if (!ASSERT_OK_PTR(skel, "get_func_ip_fsession_test__open_and_load")) 151 + return; 152 + 153 + err = get_func_ip_fsession_test__attach(skel); 154 + if (!ASSERT_OK(err, "get_func_ip_fsession_test__attach")) 155 + goto cleanup; 156 + 157 + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test1), &topts); 158 + ASSERT_OK(err, "test_run"); 159 + ASSERT_EQ(topts.retval, 0, "test_run"); 160 + 161 + ASSERT_EQ(skel->bss->test1_entry_result, 1, "test1_entry_result"); 162 + ASSERT_EQ(skel->bss->test1_exit_result, 1, "test1_exit_result"); 163 + 164 + cleanup: 165 + get_func_ip_fsession_test__destroy(skel); 140 166 }

+37

tools/testing/selftests/bpf/progs/get_func_args_fsession_test.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + #include "vmlinux.h" 3 + #include <bpf/bpf_helpers.h> 4 + #include <bpf/bpf_tracing.h> 5 + #include <errno.h> 6 + 7 + char _license[] SEC("license") = "GPL"; 8 + 9 + __u64 test1_result = 0; 10 + 11 + SEC("fsession/bpf_fentry_test1") 12 + int BPF_PROG(test1) 13 + { 14 + __u64 cnt = bpf_get_func_arg_cnt(ctx); 15 + __u64 a = 0, z = 0, ret = 0; 16 + __s64 err; 17 + 18 + test1_result = cnt == 1; 19 + 20 + /* valid arguments */ 21 + err = bpf_get_func_arg(ctx, 0, &a); 22 + test1_result &= err == 0 && ((int) a == 1); 23 + 24 + /* not valid argument */ 25 + err = bpf_get_func_arg(ctx, 1, &z); 26 + test1_result &= err == -EINVAL; 27 + 28 + if (bpf_session_is_return(ctx)) { 29 + err = bpf_get_func_ret(ctx, &ret); 30 + test1_result &= err == 0 && ret == 2; 31 + } else { 32 + err = bpf_get_func_ret(ctx, &ret); 33 + test1_result &= err == 0 && ret == 0; 34 + } 35 + 36 + return 0; 37 + }

-38

tools/testing/selftests/bpf/progs/get_func_args_test.c

··· 165 165 166 166 return 0; 167 167 } 168 - 169 - __u64 test7_result = 0; 170 - #if defined(bpf_target_x86) || defined(bpf_target_arm64) || defined(bpf_target_riscv) 171 - SEC("fsession/bpf_fentry_test1") 172 - int BPF_PROG(test7) 173 - { 174 - __u64 cnt = bpf_get_func_arg_cnt(ctx); 175 - __u64 a = 0, z = 0, ret = 0; 176 - __s64 err; 177 - 178 - test7_result = cnt == 1; 179 - 180 - /* valid arguments */ 181 - err = bpf_get_func_arg(ctx, 0, &a); 182 - test7_result &= err == 0 && ((int) a == 1); 183 - 184 - /* not valid argument */ 185 - err = bpf_get_func_arg(ctx, 1, &z); 186 - test7_result &= err == -EINVAL; 187 - 188 - if (bpf_session_is_return(ctx)) { 189 - err = bpf_get_func_ret(ctx, &ret); 190 - test7_result &= err == 0 && ret == 2; 191 - } else { 192 - err = bpf_get_func_ret(ctx, &ret); 193 - test7_result &= err == 0 && ret == 0; 194 - } 195 - 196 - return 0; 197 - } 198 - #else 199 - SEC("fentry/bpf_fentry_test1") 200 - int BPF_PROG(test7) 201 - { 202 - test7_result = 1; 203 - return 0; 204 - } 205 - #endif

+21

tools/testing/selftests/bpf/progs/get_func_ip_fsession_test.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + #include "vmlinux.h" 3 + #include <bpf/bpf_helpers.h> 4 + #include <bpf/bpf_tracing.h> 5 + 6 + char _license[] SEC("license") = "GPL"; 7 + 8 + __u64 test1_entry_result = 0; 9 + __u64 test1_exit_result = 0; 10 + 11 + SEC("fsession/bpf_fentry_test1") 12 + int BPF_PROG(test1, int a) 13 + { 14 + __u64 addr = bpf_get_func_ip(ctx); 15 + 16 + if (bpf_session_is_return(ctx)) 17 + test1_exit_result = (const void *) addr == &bpf_fentry_test1; 18 + else 19 + test1_entry_result = (const void *) addr == &bpf_fentry_test1; 20 + return 0; 21 + }

-23

tools/testing/selftests/bpf/progs/get_func_ip_test.c

··· 103 103 test8_result = (const void *) addr == (const void *) uprobe_trigger; 104 104 return 0; 105 105 } 106 - 107 - __u64 test9_entry_result = 0; 108 - __u64 test9_exit_result = 0; 109 - #if defined(bpf_target_x86) || defined(bpf_target_arm64) || defined(bpf_target_riscv) 110 - SEC("fsession/bpf_fentry_test1") 111 - int BPF_PROG(test9, int a) 112 - { 113 - __u64 addr = bpf_get_func_ip(ctx); 114 - 115 - if (bpf_session_is_return(ctx)) 116 - test9_exit_result = (const void *) addr == &bpf_fentry_test1; 117 - else 118 - test9_entry_result = (const void *) addr == &bpf_fentry_test1; 119 - return 0; 120 - } 121 - #else 122 - SEC("fentry/bpf_fentry_test1") 123 - int BPF_PROG(test9, int a) 124 - { 125 - test9_entry_result = test9_exit_result = 1; 126 - return 0; 127 - } 128 - #endif

Configure Feed

Configure Feed