LoongArch: BPF: Use BPF prog pack allocator

+83 -37

2 changed files

expand all

arch

loongarch

net

bpf_jit.c

bpf_jit.h

+82 -36

arch/loongarch/net/bpf_jit.c

··· 461 461 int dst_reg) 462 462 { 463 463 unsigned long pc; 464 - off_t offset; 464 + off_t ins_offset, fixup_offset; 465 465 struct exception_table_entry *ex; 466 466 467 - if (!ctx->image || !ctx->prog->aux->extable) 467 + if (!ctx->image || !ctx->ro_image || !ctx->prog->aux->extable) 468 468 return 0; 469 469 470 470 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && ··· 475 475 return -EINVAL; 476 476 477 477 ex = &ctx->prog->aux->extable[ctx->num_exentries]; 478 - pc = (unsigned long)&ctx->image[ctx->idx - 1]; 478 + pc = (unsigned long)&ctx->ro_image[ctx->idx - 1]; 479 479 480 - offset = pc - (long)&ex->insn; 481 - if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 480 + /* 481 + * This is the relative offset of the instruction that may fault from 482 + * the exception table itself. This will be written to the exception 483 + * table and if this instruction faults, the destination register will 484 + * be set to '0' and the execution will jump to the next instruction. 485 + */ 486 + ins_offset = pc - (long)&ex->insn; 487 + if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) 482 488 return -ERANGE; 483 - 484 - ex->insn = offset; 485 489 486 490 /* 487 491 * Since the extable follows the program, the fixup offset is always ··· 494 490 * bits. We don't need to worry about buildtime or runtime sort 495 491 * modifying the upper bits because the table is already sorted, and 496 492 * isn't part of the main exception table. 493 + * 494 + * The fixup_offset is set to the next instruction from the instruction 495 + * that may fault. The execution will jump to this after handling the fault. 497 496 */ 498 - offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE); 499 - if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) 497 + fixup_offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE); 498 + if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset)) 500 499 return -ERANGE; 501 500 501 + /* 502 + * The offsets above have been calculated using the RO buffer but we 503 + * need to use the R/W buffer for writes. Switch ex to rw buffer for writing. 504 + */ 505 + ex = (void *)ctx->image + ((void *)ex - (void *)ctx->ro_image); 506 + ex->insn = ins_offset; 507 + ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) | 508 + FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 502 509 ex->type = EX_TYPE_BPF; 503 - ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 504 510 505 511 ctx->num_exentries++; 506 512 ··· 1843 1829 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 1844 1830 { 1845 1831 bool tmp_blinded = false, extra_pass = false; 1846 - u8 *image_ptr; 1832 + u8 *image_ptr, *ro_image_ptr; 1847 1833 int image_size, prog_size, extable_size; 1848 1834 struct jit_ctx ctx; 1849 1835 struct jit_data *jit_data; 1850 1836 struct bpf_binary_header *header; 1837 + struct bpf_binary_header *ro_header; 1851 1838 struct bpf_prog *tmp, *orig_prog = prog; 1852 1839 1853 1840 /* ··· 1883 1868 } 1884 1869 if (jit_data->ctx.offset) { 1885 1870 ctx = jit_data->ctx; 1886 - image_ptr = jit_data->image; 1871 + ro_header = jit_data->ro_header; 1872 + ro_image_ptr = (void *)ctx.ro_image; 1887 1873 header = jit_data->header; 1874 + image_ptr = (void *)header + ((void *)ro_image_ptr - (void *)ro_header); 1888 1875 extra_pass = true; 1889 1876 prog_size = sizeof(u32) * ctx.idx; 1890 1877 goto skip_init_ctx; ··· 1920 1903 prog_size = sizeof(u32) * ctx.idx; 1921 1904 image_size = prog_size + extable_size; 1922 1905 /* Now we know the size of the structure to make */ 1923 - header = bpf_jit_binary_alloc(image_size, &image_ptr, 1924 - sizeof(u32), jit_fill_hole); 1925 - if (header == NULL) { 1906 + ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32), 1907 + &header, &image_ptr, jit_fill_hole); 1908 + if (!ro_header) { 1926 1909 prog = orig_prog; 1927 1910 goto out_offset; 1928 1911 } 1929 1912 1930 1913 /* 2. Now, the actual pass to generate final JIT code */ 1914 + /* 1915 + * Use the image (RW) for writing the JITed instructions. But also save 1916 + * the ro_image (RX) for calculating the offsets in the image. The RW 1917 + * image will be later copied to the RX image from where the program will 1918 + * run. The bpf_jit_binary_pack_finalize() will do this copy in the final 1919 + * step. 1920 + */ 1931 1921 ctx.image = (union loongarch_instruction *)image_ptr; 1922 + ctx.ro_image = (union loongarch_instruction *)ro_image_ptr; 1932 1923 if (extable_size) 1933 - prog->aux->extable = (void *)image_ptr + prog_size; 1924 + prog->aux->extable = (void *)ro_image_ptr + prog_size; 1934 1925 1935 1926 skip_init_ctx: 1936 1927 ctx.idx = 0; ··· 1946 1921 1947 1922 build_prologue(&ctx); 1948 1923 if (build_body(&ctx, extra_pass)) { 1949 - bpf_jit_binary_free(header); 1950 1924 prog = orig_prog; 1951 - goto out_offset; 1925 + goto out_free; 1952 1926 } 1953 1927 build_epilogue(&ctx); 1954 1928 1955 1929 /* 3. Extra pass to validate JITed code */ 1956 1930 if (validate_ctx(&ctx)) { 1957 - bpf_jit_binary_free(header); 1958 1931 prog = orig_prog; 1959 - goto out_offset; 1932 + goto out_free; 1960 1933 } 1961 1934 1962 1935 /* And we're done */ 1963 1936 if (bpf_jit_enable > 1) 1964 1937 bpf_jit_dump(prog->len, prog_size, 2, ctx.image); 1965 1938 1966 - /* Update the icache */ 1967 - flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx)); 1968 - 1969 1939 if (!prog->is_func || extra_pass) { 1970 - int err; 1971 - 1972 1940 if (extra_pass && ctx.idx != jit_data->ctx.idx) { 1973 1941 pr_err_once("multi-func JIT bug %d != %d\n", 1974 1942 ctx.idx, jit_data->ctx.idx); 1975 1943 goto out_free; 1976 1944 } 1977 - err = bpf_jit_binary_lock_ro(header); 1978 - if (err) { 1979 - pr_err_once("bpf_jit_binary_lock_ro() returned %d\n", 1980 - err); 1945 + if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) { 1946 + /* ro_header has been freed */ 1947 + ro_header = NULL; 1948 + prog = orig_prog; 1981 1949 goto out_free; 1982 1950 } 1951 + /* 1952 + * The instructions have now been copied to the ROX region from 1953 + * where they will execute. Now the data cache has to be cleaned 1954 + * to the PoU and the I-cache has to be invalidated for the VAs. 1955 + */ 1956 + bpf_flush_icache(ro_header, ctx.ro_image + ctx.idx); 1983 1957 } else { 1984 1958 jit_data->ctx = ctx; 1985 - jit_data->image = image_ptr; 1986 1959 jit_data->header = header; 1960 + jit_data->ro_header = ro_header; 1987 1961 } 1988 1962 prog->jited = 1; 1989 1963 prog->jited_len = prog_size; 1990 - prog->bpf_func = (void *)ctx.image; 1964 + prog->bpf_func = (void *)ctx.ro_image; 1991 1965 1992 1966 if (!prog->is_func || extra_pass) { 1993 1967 int i; ··· 2006 1982 if (tmp_blinded) 2007 1983 bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); 2008 1984 2009 - 2010 1985 return prog; 2011 1986 2012 1987 out_free: 2013 - bpf_jit_binary_free(header); 2014 - prog->bpf_func = NULL; 2015 - prog->jited = 0; 2016 - prog->jited_len = 0; 1988 + if (header) { 1989 + bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size)); 1990 + bpf_jit_binary_pack_free(ro_header, header); 1991 + } 2017 1992 goto out_offset; 1993 + } 1994 + 1995 + void bpf_jit_free(struct bpf_prog *prog) 1996 + { 1997 + if (prog->jited) { 1998 + struct jit_data *jit_data = prog->aux->jit_data; 1999 + struct bpf_binary_header *hdr; 2000 + 2001 + /* 2002 + * If we fail the final pass of JIT (from jit_subprogs), the 2003 + * program may not be finalized yet. Call finalize here before 2004 + * freeing it. 2005 + */ 2006 + if (jit_data) { 2007 + bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); 2008 + kfree(jit_data); 2009 + } 2010 + hdr = bpf_jit_binary_pack_hdr(prog); 2011 + bpf_jit_binary_pack_free(hdr, NULL); 2012 + WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); 2013 + } 2014 + 2015 + bpf_prog_unlock_free(prog); 2018 2016 } 2019 2017 2020 2018 bool bpf_jit_bypass_spec_v1(void)

+1 -1

arch/loongarch/net/bpf_jit.h

··· 24 24 25 25 struct jit_data { 26 26 struct bpf_binary_header *header; 27 - u8 *image; 27 + struct bpf_binary_header *ro_header; 28 28 struct jit_ctx ctx; 29 29 }; 30 30

Configure Feed

Configure Feed