s390/bpf: Use direct calls and jumps where possible

After the V!=R rework (commit c98d2ecae08f ("s390/mm: Uncouple physical
vs virtual address spaces")), all kernel code and related data are
allocated within a 4G region, making it possible to use relative
addressing in BPF code more extensively.

Convert as many indirect calls and jumps to direct calls as possible,
namely:

* BPF_CALL
* __bpf_tramp_enter()
* __bpf_tramp_exit()
* __bpf_prog_enter()
* __bpf_prog_exit()
* fentry
* fmod_ret
* fexit
* BPF_TRAMP_F_CALL_ORIG without BPF_TRAMP_F_ORIG_STACK
* Trampoline returns without BPF_TRAMP_F_SKIP_FRAME and
BPF_TRAMP_F_ORIG_STACK

The following indirect calls and jumps remain:

* Prog returns
* Trampoline returns with BPF_TRAMP_F_SKIP_FRAME or
BPF_TRAMP_F_ORIG_STACK
* BPF_TAIL_CALL
* BPF_TRAMP_F_CALL_ORIG with BPF_TRAMP_F_ORIG_STACK

As a result, only one usage of call_r1() remains, so inline it.

Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20250819102116.252203-1-iii@linux.ibm.com

authored by

Ilya Leoshkevich and committed by

Daniel Borkmann 10 months ago b5bbbb70 bf7a6a67

+32 -48

1 changed file

expand all

arch

s390

net

bpf_jit_comp.c

+32 -48

arch/s390/net/bpf_jit_comp.c

··· 675 675 } while (0) 676 676 677 677 /* 678 - * Call r1 either directly or via __s390_indirect_jump_r1 thunk 679 - */ 680 - static void call_r1(struct bpf_jit *jit) 681 - { 682 - if (nospec_uses_trampoline()) 683 - /* brasl %r14,__s390_indirect_jump_r1 */ 684 - EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, 685 - __s390_indirect_jump_r1); 686 - else 687 - /* basr %r14,%r1 */ 688 - EMIT2(0x0d00, REG_14, REG_1); 689 - } 690 - 691 - /* 692 678 * Function epilogue 693 679 */ 694 680 static void bpf_jit_epilogue(struct bpf_jit *jit) ··· 1806 1820 } 1807 1821 } 1808 1822 1809 - /* lgrl %w1,func */ 1810 - EMIT6_PCREL_RILB(0xc4080000, REG_W1, _EMIT_CONST_U64(func)); 1811 - /* %r1() */ 1812 - call_r1(jit); 1823 + /* brasl %r14,func */ 1824 + EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, (void *)func); 1813 1825 /* lgr %b0,%r2: load return value into %b0 */ 1814 1826 EMIT4(0xb9040000, BPF_REG_0, REG_2); 1815 1827 ··· 2518 2534 * goto skip; 2519 2535 */ 2520 2536 2521 - /* %r1 = __bpf_prog_enter */ 2522 - load_imm64(jit, REG_1, (u64)bpf_trampoline_enter(p)); 2523 2537 /* %r2 = p */ 2524 2538 load_imm64(jit, REG_2, (u64)p); 2525 2539 /* la %r3,run_ctx_off(%r15) */ 2526 2540 EMIT4_DISP(0x41000000, REG_3, REG_15, tjit->run_ctx_off); 2527 - /* %r1() */ 2528 - call_r1(jit); 2541 + /* brasl %r14,__bpf_prog_enter */ 2542 + EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_enter(p)); 2529 2543 /* ltgr %r7,%r2 */ 2530 2544 EMIT4(0xb9020000, REG_7, REG_2); 2531 2545 /* brcl 8,skip */ ··· 2534 2552 * retval = bpf_func(args, p->insnsi); 2535 2553 */ 2536 2554 2537 - /* %r1 = p->bpf_func */ 2538 - load_imm64(jit, REG_1, (u64)p->bpf_func); 2539 2555 /* la %r2,bpf_args_off(%r15) */ 2540 2556 EMIT4_DISP(0x41000000, REG_2, REG_15, tjit->bpf_args_off); 2541 2557 /* %r3 = p->insnsi */ 2542 2558 if (!p->jited) 2543 2559 load_imm64(jit, REG_3, (u64)p->insnsi); 2544 - /* %r1() */ 2545 - call_r1(jit); 2560 + /* brasl %r14,p->bpf_func */ 2561 + EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, p->bpf_func); 2546 2562 /* stg %r2,retval_off(%r15) */ 2547 2563 if (save_ret) { 2548 2564 if (sign_extend(jit, REG_2, m->ret_size, m->ret_flags)) ··· 2557 2577 * __bpf_prog_exit(p, start, &run_ctx); 2558 2578 */ 2559 2579 2560 - /* %r1 = __bpf_prog_exit */ 2561 - load_imm64(jit, REG_1, (u64)bpf_trampoline_exit(p)); 2562 2580 /* %r2 = p */ 2563 2581 load_imm64(jit, REG_2, (u64)p); 2564 2582 /* lgr %r3,%r7 */ 2565 2583 EMIT4(0xb9040000, REG_3, REG_7); 2566 2584 /* la %r4,run_ctx_off(%r15) */ 2567 2585 EMIT4_DISP(0x41000000, REG_4, REG_15, tjit->run_ctx_off); 2568 - /* %r1() */ 2569 - call_r1(jit); 2586 + /* brasl %r14,__bpf_prog_exit */ 2587 + EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_exit(p)); 2570 2588 2571 2589 return 0; 2572 2590 } ··· 2724 2746 2725 2747 /* lgr %r8,%r0 */ 2726 2748 EMIT4(0xb9040000, REG_8, REG_0); 2727 - } else { 2728 - /* %r8 = func_addr + S390X_PATCH_SIZE */ 2729 - load_imm64(jit, REG_8, (u64)func_addr + S390X_PATCH_SIZE); 2730 2749 } 2731 2750 2732 2751 /* ··· 2749 2774 * __bpf_tramp_enter(im); 2750 2775 */ 2751 2776 2752 - /* %r1 = __bpf_tramp_enter */ 2753 - load_imm64(jit, REG_1, (u64)__bpf_tramp_enter); 2754 2777 /* %r2 = im */ 2755 2778 load_imm64(jit, REG_2, (u64)im); 2756 - /* %r1() */ 2757 - call_r1(jit); 2779 + /* brasl %r14,__bpf_tramp_enter */ 2780 + EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_enter); 2758 2781 } 2759 2782 2760 2783 for (i = 0; i < fentry->nr_links; i++) ··· 2805 2832 /* mvc tail_call_cnt(4,%r15),tccnt_off(%r15) */ 2806 2833 _EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt), 2807 2834 0xf000 | tjit->tccnt_off); 2808 - /* lgr %r1,%r8 */ 2809 - EMIT4(0xb9040000, REG_1, REG_8); 2810 - /* %r1() */ 2811 - call_r1(jit); 2835 + if (flags & BPF_TRAMP_F_ORIG_STACK) { 2836 + if (nospec_uses_trampoline()) 2837 + /* brasl %r14,__s390_indirect_jump_r8 */ 2838 + EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, 2839 + __s390_indirect_jump_r8); 2840 + else 2841 + /* basr %r14,%r8 */ 2842 + EMIT2(0x0d00, REG_14, REG_8); 2843 + } else { 2844 + /* brasl %r14,func_addr+S390X_PATCH_SIZE */ 2845 + EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, 2846 + func_addr + S390X_PATCH_SIZE); 2847 + } 2812 2848 /* stg %r2,retval_off(%r15) */ 2813 2849 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, 2814 2850 tjit->retval_off); ··· 2848 2866 * __bpf_tramp_exit(im); 2849 2867 */ 2850 2868 2851 - /* %r1 = __bpf_tramp_exit */ 2852 - load_imm64(jit, REG_1, (u64)__bpf_tramp_exit); 2853 2869 /* %r2 = im */ 2854 2870 load_imm64(jit, REG_2, (u64)im); 2855 - /* %r1() */ 2856 - call_r1(jit); 2871 + /* brasl %r14,__bpf_tramp_exit */ 2872 + EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_exit); 2857 2873 } 2858 2874 2859 2875 /* lmg %r2,%rN,reg_args_off(%r15) */ ··· 2860 2880 REG_2 + (nr_reg_args - 1), REG_15, 2861 2881 tjit->reg_args_off); 2862 2882 /* lgr %r1,%r8 */ 2863 - if (!(flags & BPF_TRAMP_F_SKIP_FRAME)) 2883 + if (!(flags & BPF_TRAMP_F_SKIP_FRAME) && 2884 + (flags & BPF_TRAMP_F_ORIG_STACK)) 2864 2885 EMIT4(0xb9040000, REG_1, REG_8); 2865 2886 /* lmg %r7,%r8,r7_r8_off(%r15) */ 2866 2887 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_7, REG_8, REG_15, ··· 2880 2899 EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size); 2881 2900 if (flags & BPF_TRAMP_F_SKIP_FRAME) 2882 2901 EMIT_JUMP_REG(14); 2883 - else 2902 + else if (flags & BPF_TRAMP_F_ORIG_STACK) 2884 2903 EMIT_JUMP_REG(1); 2885 - 2904 + else 2905 + /* brcl 0xf,func_addr+S390X_PATCH_SIZE */ 2906 + EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf, 2907 + func_addr + S390X_PATCH_SIZE); 2886 2908 return 0; 2887 2909 } 2888 2910

Configure Feed

Configure Feed