tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

riscv, bpf: Add ex_insn_off and ex_jmp_off for exception table handling

Add ex_insn_off and ex_jmp_off fields to struct rv_jit_context so that
add_exception_handler() does not need to be immediately followed by the
instruction to add the exception table. ex_insn_off indicates the offset
of the instruction to add the exception table, and ex_jmp_off indicates
the offset to jump over the faulting instruction. This is to prepare for
adding the exception table to atomic instructions later, because some
atomic instructions need to perform zext or other operations.

Signed-off-by: Pu Lehui <pulehui@huawei.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Tested-by: Björn Töpel <bjorn@rivosinc.com>
Reviewed-by: Björn Töpel <bjorn@rivosinc.com>
Acked-by: Björn Töpel <bjorn@kernel.org>
Link: https://lore.kernel.org/bpf/20250719091730.2660197-9-pulehui@huaweicloud.com

authored by

Pu Lehui and committed by
Daniel Borkmann b18f4aae 1c0196b8

+39 -47
+2
arch/riscv/net/bpf_jit.h
··· 78 78 int epilogue_offset; 79 79 int *offset; /* BPF to RV */ 80 80 int nexentries; 81 + int ex_insn_off; 82 + int ex_jmp_off; 81 83 unsigned long flags; 82 84 int stack_size; 83 85 u64 arena_vm_start;
+37 -47
arch/riscv/net/bpf_jit_comp64.c
··· 511 511 } 512 512 } 513 513 514 - static int emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext, 514 + static void emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext, 515 515 struct rv_jit_context *ctx) 516 516 { 517 - int insns_start; 518 - 519 517 if (is_12b_int(off)) { 520 - insns_start = ctx->ninsns; 518 + ctx->ex_insn_off = ctx->ninsns; 521 519 emit_ldx_insn(rd, off, rs, size, sign_ext, ctx); 522 - return ctx->ninsns - insns_start; 520 + ctx->ex_jmp_off = ctx->ninsns; 521 + return; 523 522 } 524 523 525 524 emit_imm(RV_REG_T1, off, ctx); 526 525 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 527 - insns_start = ctx->ninsns; 526 + ctx->ex_insn_off = ctx->ninsns; 528 527 emit_ldx_insn(rd, 0, RV_REG_T1, size, sign_ext, ctx); 529 - return ctx->ninsns - insns_start; 528 + ctx->ex_jmp_off = ctx->ninsns; 530 529 } 531 530 532 - static int emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx) 531 + static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx) 533 532 { 534 - int insns_start; 535 - 536 533 emit_imm(RV_REG_T1, imm, ctx); 537 534 if (is_12b_int(off)) { 538 - insns_start = ctx->ninsns; 535 + ctx->ex_insn_off = ctx->ninsns; 539 536 emit_stx_insn(rd, off, RV_REG_T1, size, ctx); 540 - return ctx->ninsns - insns_start; 537 + ctx->ex_jmp_off = ctx->ninsns; 538 + return; 541 539 } 542 540 543 541 emit_imm(RV_REG_T2, off, ctx); 544 542 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 545 - insns_start = ctx->ninsns; 543 + ctx->ex_insn_off = ctx->ninsns; 546 544 emit_stx_insn(RV_REG_T2, 0, RV_REG_T1, size, ctx); 547 - return ctx->ninsns - insns_start; 545 + ctx->ex_jmp_off = ctx->ninsns; 548 546 } 549 547 550 - static int emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) 548 + static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) 551 549 { 552 - int insns_start; 553 - 554 550 if (is_12b_int(off)) { 555 - insns_start = ctx->ninsns; 551 + ctx->ex_insn_off = ctx->ninsns; 556 552 emit_stx_insn(rd, off, rs, size, ctx); 557 - return ctx->ninsns - insns_start; 553 + ctx->ex_jmp_off = ctx->ninsns; 554 + return; 558 555 } 559 556 560 557 emit_imm(RV_REG_T1, off, ctx); 561 558 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 562 - insns_start = ctx->ninsns; 559 + ctx->ex_insn_off = ctx->ninsns; 563 560 emit_stx_insn(RV_REG_T1, 0, rs, size, ctx); 564 - return ctx->ninsns - insns_start; 561 + ctx->ex_jmp_off = ctx->ninsns; 565 562 } 566 563 567 564 static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn, ··· 697 700 } 698 701 699 702 /* For accesses to BTF pointers, add an entry to the exception table */ 700 - static int add_exception_handler(const struct bpf_insn *insn, 701 - struct rv_jit_context *ctx, 702 - int dst_reg, int insn_len) 703 + static int add_exception_handler(const struct bpf_insn *insn, int dst_reg, 704 + struct rv_jit_context *ctx) 703 705 { 704 706 struct exception_table_entry *ex; 705 707 unsigned long pc; ··· 706 710 off_t fixup_offset; 707 711 708 712 if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable || 709 - (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 710 - BPF_MODE(insn->code) != BPF_PROBE_MEM32)) 713 + ctx->ex_insn_off <= 0 || ctx->ex_jmp_off <= 0) 714 + return 0; 715 + 716 + if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 717 + BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 718 + BPF_MODE(insn->code) != BPF_PROBE_MEM32) 711 719 return 0; 712 720 713 721 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) 714 722 return -EINVAL; 715 723 716 - if (WARN_ON_ONCE(insn_len > ctx->ninsns)) 717 - return -EINVAL; 718 - 719 - if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) 724 + if (WARN_ON_ONCE(ctx->ex_insn_off > ctx->ninsns || ctx->ex_jmp_off > ctx->ninsns)) 720 725 return -EINVAL; 721 726 722 727 ex = &ctx->prog->aux->extable[ctx->nexentries]; 723 - pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len]; 728 + pc = (unsigned long)&ctx->ro_insns[ctx->ex_insn_off]; 724 729 725 730 /* 726 731 * This is the relative offset of the instruction that may fault from ··· 745 748 * that may fault. The execution will jump to this after handling the 746 749 * fault. 747 750 */ 748 - fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); 751 + fixup_offset = (long)&ex->fixup - (long)&ctx->ro_insns[ctx->ex_jmp_off]; 749 752 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset)) 750 753 return -ERANGE; 751 754 ··· 762 765 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 763 766 ex->type = EX_TYPE_BPF; 764 767 768 + ctx->ex_insn_off = 0; 769 + ctx->ex_jmp_off = 0; 765 770 ctx->nexentries++; 766 771 return 0; 767 772 } ··· 1773 1774 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1774 1775 { 1775 1776 bool sign_ext; 1776 - int insn_len; 1777 1777 1778 1778 sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || 1779 1779 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; ··· 1782 1784 rs = RV_REG_T2; 1783 1785 } 1784 1786 1785 - insn_len = emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); 1787 + emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); 1786 1788 1787 - ret = add_exception_handler(insn, ctx, rd, insn_len); 1789 + ret = add_exception_handler(insn, rd, ctx); 1788 1790 if (ret) 1789 1791 return ret; 1790 1792 ··· 1807 1809 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1808 1810 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1809 1811 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1810 - { 1811 - int insn_len; 1812 - 1813 1812 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { 1814 1813 emit_add(RV_REG_T3, rd, RV_REG_ARENA, ctx); 1815 1814 rd = RV_REG_T3; 1816 1815 } 1817 1816 1818 - insn_len = emit_st(rd, off, imm, BPF_SIZE(code), ctx); 1817 + emit_st(rd, off, imm, BPF_SIZE(code), ctx); 1819 1818 1820 - ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, insn_len); 1819 + ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); 1821 1820 if (ret) 1822 1821 return ret; 1823 1822 break; 1824 - } 1825 1823 1826 1824 /* STX: *(size *)(dst + off) = src */ 1827 1825 case BPF_STX | BPF_MEM | BPF_B: ··· 1829 1835 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1830 1836 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1831 1837 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1832 - { 1833 - int insn_len; 1834 - 1835 1838 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { 1836 1839 emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx); 1837 1840 rd = RV_REG_T2; 1838 1841 } 1839 1842 1840 - insn_len = emit_stx(rd, off, rs, BPF_SIZE(code), ctx); 1843 + emit_stx(rd, off, rs, BPF_SIZE(code), ctx); 1841 1844 1842 - ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, insn_len); 1845 + ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); 1843 1846 if (ret) 1844 1847 return ret; 1845 1848 break; 1846 - } 1847 1849 1848 1850 case BPF_STX | BPF_ATOMIC | BPF_B: 1849 1851 case BPF_STX | BPF_ATOMIC | BPF_H: