Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

+15 -26

arch/arc/net/bpf_jit_core.c

··· 79 79 * The JIT pertinent context that is used by different functions. 80 80 * 81 81 * prog: The current eBPF program being handled. 82 - * orig_prog: The original eBPF program before any possible change. 83 82 * jit: The JIT buffer and its length. 84 83 * bpf_header: The JITed program header. "jit.buf" points inside it. 85 84 * emit: If set, opcodes are written to memory; else, a dry-run. ··· 93 94 * need_extra_pass: A forecast if an "extra_pass" will occur. 94 95 * is_extra_pass: Indicates if the current pass is an extra pass. 95 96 * user_bpf_prog: True, if VM opcodes come from a real program. 96 - * blinded: True if "constant blinding" step returned a new "prog". 97 97 * success: Indicates if the whole JIT went OK. 98 98 */ 99 99 struct jit_context { 100 100 struct bpf_prog *prog; 101 - struct bpf_prog *orig_prog; 102 101 struct jit_buffer jit; 103 102 struct bpf_binary_header *bpf_header; 104 103 bool emit; ··· 111 114 bool need_extra_pass; 112 115 bool is_extra_pass; 113 116 bool user_bpf_prog; 114 - bool blinded; 115 117 bool success; 116 118 }; 117 119 ··· 157 161 { 158 162 memset(ctx, 0, sizeof(*ctx)); 159 163 160 - ctx->orig_prog = prog; 161 - 162 - /* If constant blinding was requested but failed, scram. */ 163 - ctx->prog = bpf_jit_blind_constants(prog); 164 - if (IS_ERR(ctx->prog)) 165 - return PTR_ERR(ctx->prog); 166 - ctx->blinded = (ctx->prog != ctx->orig_prog); 164 + ctx->prog = prog; 167 165 168 166 /* If the verifier doesn't zero-extend, then we have to do it. */ 169 167 ctx->do_zext = !ctx->prog->aux->verifier_zext; ··· 204 214 */ 205 215 static void jit_ctx_cleanup(struct jit_context *ctx) 206 216 { 207 - if (ctx->blinded) { 208 - /* if all went well, release the orig_prog. */ 209 - if (ctx->success) 210 - bpf_jit_prog_release_other(ctx->prog, ctx->orig_prog); 211 - else 212 - bpf_jit_prog_release_other(ctx->orig_prog, ctx->prog); 213 - } 214 - 215 217 maybe_free(ctx, (void **)&ctx->bpf2insn); 216 218 maybe_free(ctx, (void **)&ctx->jit_data); 217 219 ··· 211 229 ctx->bpf2insn_valid = false; 212 230 213 231 /* Freeing "bpf_header" is enough. "jit.buf" is a sub-array of it. */ 214 - if (!ctx->success && ctx->bpf_header) { 215 - bpf_jit_binary_free(ctx->bpf_header); 216 - ctx->bpf_header = NULL; 217 - ctx->jit.buf = NULL; 218 - ctx->jit.index = 0; 219 - ctx->jit.len = 0; 232 + if (!ctx->success) { 233 + if (ctx->bpf_header) { 234 + bpf_jit_binary_free(ctx->bpf_header); 235 + ctx->bpf_header = NULL; 236 + ctx->jit.buf = NULL; 237 + ctx->jit.index = 0; 238 + ctx->jit.len = 0; 239 + } 240 + if (ctx->is_extra_pass) { 241 + ctx->prog->bpf_func = NULL; 242 + ctx->prog->jited = 0; 243 + ctx->prog->jited_len = 0; 244 + } 220 245 } 221 246 222 247 ctx->emit = false; ··· 1400 1411 * (re)locations involved that their addresses are not known 1401 1412 * during the first run. 1402 1413 */ 1403 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 1414 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 1404 1415 { 1405 1416 vm_dump(prog); 1406 1417

+14 -35

arch/arm/net/bpf_jit_32.c

··· 1852 1852 { 1853 1853 u64 val = (u32)imm | (u64)insn[1].imm << 32; 1854 1854 1855 + if (insn->src_reg == BPF_PSEUDO_FUNC) 1856 + goto notyet; 1857 + 1855 1858 emit_a32_mov_i64(dst, val, ctx); 1856 1859 1857 1860 return 1; ··· 2058 2055 const s8 *r5 = bpf2a32[BPF_REG_5]; 2059 2056 const u32 func = (u32)__bpf_call_base + (u32)imm; 2060 2057 2058 + if (insn->src_reg == BPF_PSEUDO_CALL) 2059 + goto notyet; 2060 + 2061 2061 emit_a32_mov_r64(true, r0, r1, ctx); 2062 2062 emit_a32_mov_r64(true, r1, r2, ctx); 2063 2063 emit_push_r64(r5, ctx); ··· 2148 2142 return true; 2149 2143 } 2150 2144 2151 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 2145 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 2152 2146 { 2153 - struct bpf_prog *tmp, *orig_prog = prog; 2154 2147 struct bpf_binary_header *header; 2155 - bool tmp_blinded = false; 2156 2148 struct jit_ctx ctx; 2157 2149 unsigned int tmp_idx; 2158 2150 unsigned int image_size; ··· 2160 2156 * the interpreter. 2161 2157 */ 2162 2158 if (!prog->jit_requested) 2163 - return orig_prog; 2164 - 2165 - /* If constant blinding was enabled and we failed during blinding 2166 - * then we must fall back to the interpreter. Otherwise, we save 2167 - * the new JITed code. 2168 - */ 2169 - tmp = bpf_jit_blind_constants(prog); 2170 - 2171 - if (IS_ERR(tmp)) 2172 - return orig_prog; 2173 - if (tmp != prog) { 2174 - tmp_blinded = true; 2175 - prog = tmp; 2176 - } 2159 + return prog; 2177 2160 2178 2161 memset(&ctx, 0, sizeof(ctx)); 2179 2162 ctx.prog = prog; ··· 2170 2179 * we must fall back to the interpreter 2171 2180 */ 2172 2181 ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL); 2173 - if (ctx.offsets == NULL) { 2174 - prog = orig_prog; 2175 - goto out; 2176 - } 2182 + if (ctx.offsets == NULL) 2183 + return prog; 2177 2184 2178 2185 /* 1) fake pass to find in the length of the JITed code, 2179 2186 * to compute ctx->offsets and other context variables ··· 2183 2194 * being successful in the second pass, so just fall back 2184 2195 * to the interpreter. 2185 2196 */ 2186 - if (build_body(&ctx)) { 2187 - prog = orig_prog; 2197 + if (build_body(&ctx)) 2188 2198 goto out_off; 2189 - } 2190 2199 2191 2200 tmp_idx = ctx.idx; 2192 2201 build_prologue(&ctx); ··· 2200 2213 ctx.idx += ctx.imm_count; 2201 2214 if (ctx.imm_count) { 2202 2215 ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL); 2203 - if (ctx.imms == NULL) { 2204 - prog = orig_prog; 2216 + if (ctx.imms == NULL) 2205 2217 goto out_off; 2206 - } 2207 2218 } 2208 2219 #else 2209 2220 /* there's nothing about the epilogue on ARMv7 */ ··· 2223 2238 /* Not able to allocate memory for the structure then 2224 2239 * we must fall back to the interpretation 2225 2240 */ 2226 - if (header == NULL) { 2227 - prog = orig_prog; 2241 + if (header == NULL) 2228 2242 goto out_imms; 2229 - } 2230 2243 2231 2244 /* 2.) Actual pass to generate final JIT code */ 2232 2245 ctx.target = (u32 *) image_ptr; ··· 2261 2278 #endif 2262 2279 out_off: 2263 2280 kfree(ctx.offsets); 2264 - out: 2265 - if (tmp_blinded) 2266 - bpf_jit_prog_release_other(prog, prog == orig_prog ? 2267 - tmp : orig_prog); 2281 + 2268 2282 return prog; 2269 2283 2270 2284 out_free: 2271 2285 image_ptr = NULL; 2272 2286 bpf_jit_binary_free(header); 2273 - prog = orig_prog; 2274 2287 goto out_imms; 2275 2288 } 2276 2289

+2

arch/arm64/lib/insn.c

··· 338 338 long offset; 339 339 340 340 offset = label_imm_common(pc, addr, SZ_1M); 341 + if (offset >= SZ_1M) 342 + return AARCH64_BREAK_FAULT; 341 343 342 344 insn = aarch64_insn_get_bcond_value(); 343 345

+31 -72

arch/arm64/net/bpf_jit_comp.c

··· 18 18 19 19 #include <asm/asm-extable.h> 20 20 #include <asm/byteorder.h> 21 - #include <asm/cacheflush.h> 22 21 #include <asm/cpufeature.h> 23 22 #include <asm/debug-monitors.h> 24 23 #include <asm/insn.h> ··· 34 35 #define ARENA_VM_START (MAX_BPF_JIT_REG + 5) 35 36 36 37 #define check_imm(bits, imm) do { \ 37 - if ((((imm) > 0) && ((imm) >> (bits))) || \ 38 - (((imm) < 0) && (~(imm) >> (bits)))) { \ 38 + if ((((imm) > 0) && ((imm) >> ((bits) - 1))) || \ 39 + (((imm) < 0) && (~(imm) >> ((bits) - 1)))) { \ 39 40 pr_info("[%2d] imm=%d(0x%x) out of range\n", \ 40 41 i, imm, imm); \ 41 42 return -EINVAL; \ ··· 1197 1198 * >0 - successfully JITed a 16-byte eBPF instruction. 1198 1199 * <0 - failed to JIT. 1199 1200 */ 1200 - static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, 1201 - bool extra_pass) 1201 + static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn *insn, 1202 + struct jit_ctx *ctx, bool extra_pass) 1202 1203 { 1203 1204 const u8 code = insn->code; 1204 1205 u8 dst = bpf2a64[insn->dst_reg]; ··· 1222 1223 int off_adj; 1223 1224 int ret; 1224 1225 bool sign_extend; 1226 + 1227 + if (bpf_insn_is_indirect_target(env, ctx->prog, i)) 1228 + emit_bti(A64_BTI_J, ctx); 1225 1229 1226 1230 switch (code) { 1227 1231 /* dst = src */ ··· 1901 1899 return 0; 1902 1900 } 1903 1901 1904 - static int build_body(struct jit_ctx *ctx, bool extra_pass) 1902 + static int build_body(struct bpf_verifier_env *env, struct jit_ctx *ctx, bool extra_pass) 1905 1903 { 1906 1904 const struct bpf_prog *prog = ctx->prog; 1907 1905 int i; ··· 1920 1918 int ret; 1921 1919 1922 1920 ctx->offset[i] = ctx->idx; 1923 - ret = build_insn(insn, ctx, extra_pass); 1921 + ret = build_insn(env, insn, ctx, extra_pass); 1924 1922 if (ret > 0) { 1925 1923 i++; 1926 1924 ctx->offset[i] = ctx->idx; ··· 1963 1961 return 0; 1964 1962 } 1965 1963 1966 - static inline void bpf_flush_icache(void *start, void *end) 1967 - { 1968 - flush_icache_range((unsigned long)start, (unsigned long)end); 1969 - } 1970 - 1971 1964 static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size) 1972 1965 { 1973 1966 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3; ··· 2003 2006 struct jit_ctx ctx; 2004 2007 }; 2005 2008 2006 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 2009 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 2007 2010 { 2008 2011 int image_size, prog_size, extable_size, extable_align, extable_offset; 2009 - struct bpf_prog *tmp, *orig_prog = prog; 2010 2012 struct bpf_binary_header *header; 2011 2013 struct bpf_binary_header *ro_header = NULL; 2012 2014 struct arm64_jit_data *jit_data; 2013 2015 void __percpu *priv_stack_ptr = NULL; 2014 2016 bool was_classic = bpf_prog_was_classic(prog); 2015 2017 int priv_stack_alloc_sz; 2016 - bool tmp_blinded = false; 2017 2018 bool extra_pass = false; 2018 2019 struct jit_ctx ctx; 2019 2020 u8 *image_ptr; ··· 2020 2025 int exentry_idx; 2021 2026 2022 2027 if (!prog->jit_requested) 2023 - return orig_prog; 2024 - 2025 - tmp = bpf_jit_blind_constants(prog); 2026 - /* If blinding was requested and we failed during blinding, 2027 - * we must fall back to the interpreter. 2028 - */ 2029 - if (IS_ERR(tmp)) 2030 - return orig_prog; 2031 - if (tmp != prog) { 2032 - tmp_blinded = true; 2033 - prog = tmp; 2034 - } 2028 + return prog; 2035 2029 2036 2030 jit_data = prog->aux->jit_data; 2037 2031 if (!jit_data) { 2038 2032 jit_data = kzalloc_obj(*jit_data); 2039 - if (!jit_data) { 2040 - prog = orig_prog; 2041 - goto out; 2042 - } 2033 + if (!jit_data) 2034 + return prog; 2043 2035 prog->aux->jit_data = jit_data; 2044 2036 } 2045 2037 priv_stack_ptr = prog->aux->priv_stack_ptr; ··· 2038 2056 priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 16) + 2039 2057 2 * PRIV_STACK_GUARD_SZ; 2040 2058 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 16, GFP_KERNEL); 2041 - if (!priv_stack_ptr) { 2042 - prog = orig_prog; 2059 + if (!priv_stack_ptr) 2043 2060 goto out_priv_stack; 2044 - } 2045 2061 2046 2062 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz); 2047 2063 prog->aux->priv_stack_ptr = priv_stack_ptr; ··· 2059 2079 ctx.prog = prog; 2060 2080 2061 2081 ctx.offset = kvzalloc_objs(int, prog->len + 1); 2062 - if (ctx.offset == NULL) { 2063 - prog = orig_prog; 2082 + if (ctx.offset == NULL) 2064 2083 goto out_off; 2065 - } 2066 2084 2067 2085 ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); 2068 2086 ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); ··· 2073 2095 * BPF line info needs ctx->offset[i] to be the offset of 2074 2096 * instruction[i] in jited image, so build prologue first. 2075 2097 */ 2076 - if (build_prologue(&ctx, was_classic)) { 2077 - prog = orig_prog; 2098 + if (build_prologue(&ctx, was_classic)) 2078 2099 goto out_off; 2079 - } 2080 2100 2081 - if (build_body(&ctx, extra_pass)) { 2082 - prog = orig_prog; 2101 + if (build_body(env, &ctx, extra_pass)) 2083 2102 goto out_off; 2084 - } 2085 2103 2086 2104 ctx.epilogue_offset = ctx.idx; 2087 2105 build_epilogue(&ctx, was_classic); ··· 2095 2121 ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, 2096 2122 sizeof(u64), &header, &image_ptr, 2097 2123 jit_fill_hole); 2098 - if (!ro_header) { 2099 - prog = orig_prog; 2124 + if (!ro_header) 2100 2125 goto out_off; 2101 - } 2102 2126 2103 2127 /* Pass 2: Determine jited position and result for each instruction */ 2104 2128 ··· 2124 2152 /* Dont write body instructions to memory for now */ 2125 2153 ctx.write = false; 2126 2154 2127 - if (build_body(&ctx, extra_pass)) { 2128 - prog = orig_prog; 2155 + if (build_body(env, &ctx, extra_pass)) 2129 2156 goto out_free_hdr; 2130 - } 2131 2157 2132 2158 ctx.epilogue_offset = ctx.idx; 2133 2159 ctx.exentry_idx = exentry_idx; ··· 2133 2163 ctx.write = true; 2134 2164 2135 2165 /* Pass 3: Adjust jump offset and write final image */ 2136 - if (build_body(&ctx, extra_pass) || 2137 - WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset)) { 2138 - prog = orig_prog; 2166 + if (build_body(env, &ctx, extra_pass) || 2167 + WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset)) 2139 2168 goto out_free_hdr; 2140 - } 2141 2169 2142 2170 build_epilogue(&ctx, was_classic); 2143 2171 build_plt(&ctx); 2144 2172 2145 2173 /* Extra pass to validate JITed code. */ 2146 - if (validate_ctx(&ctx)) { 2147 - prog = orig_prog; 2174 + if (validate_ctx(&ctx)) 2148 2175 goto out_free_hdr; 2149 - } 2150 2176 2151 2177 /* update the real prog size */ 2152 2178 prog_size = sizeof(u32) * ctx.idx; ··· 2159 2193 if (extra_pass && ctx.idx > jit_data->ctx.idx) { 2160 2194 pr_err_once("multi-func JIT bug %d > %d\n", 2161 2195 ctx.idx, jit_data->ctx.idx); 2162 - prog->bpf_func = NULL; 2163 - prog->jited = 0; 2164 - prog->jited_len = 0; 2165 2196 goto out_free_hdr; 2166 2197 } 2167 2198 if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) { 2168 - /* ro_header has been freed */ 2199 + /* ro_header and header has been freed */ 2169 2200 ro_header = NULL; 2170 - prog = orig_prog; 2171 - goto out_off; 2201 + header = NULL; 2202 + goto out_free_hdr; 2172 2203 } 2173 - /* 2174 - * The instructions have now been copied to the ROX region from 2175 - * where they will execute. Now the data cache has to be cleaned to 2176 - * the PoU and the I-cache has to be invalidated for the VAs. 2177 - */ 2178 - bpf_flush_icache(ro_header, ctx.ro_image + ctx.idx); 2179 2204 } else { 2180 2205 jit_data->ctx = ctx; 2181 2206 jit_data->ro_image = ro_image_ptr; ··· 2202 2245 kfree(jit_data); 2203 2246 prog->aux->jit_data = NULL; 2204 2247 } 2205 - out: 2206 - if (tmp_blinded) 2207 - bpf_jit_prog_release_other(prog, prog == orig_prog ? 2208 - tmp : orig_prog); 2248 + 2209 2249 return prog; 2210 2250 2211 2251 out_free_hdr: 2252 + if (extra_pass) { 2253 + prog->bpf_func = NULL; 2254 + prog->jited = 0; 2255 + prog->jited_len = 0; 2256 + } 2212 2257 if (header) { 2213 2258 bpf_arch_text_copy(&ro_header->size, &header->size, 2214 2259 sizeof(header->size));

+18 -43

arch/loongarch/net/bpf_jit.c

··· 1920 1920 return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE; 1921 1921 } 1922 1922 1923 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 1923 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 1924 1924 { 1925 - bool tmp_blinded = false, extra_pass = false; 1925 + bool extra_pass = false; 1926 1926 u8 *image_ptr, *ro_image_ptr; 1927 1927 int image_size, prog_size, extable_size; 1928 1928 struct jit_ctx ctx; 1929 1929 struct jit_data *jit_data; 1930 1930 struct bpf_binary_header *header; 1931 1931 struct bpf_binary_header *ro_header; 1932 - struct bpf_prog *tmp, *orig_prog = prog; 1933 1932 1934 1933 /* 1935 1934 * If BPF JIT was not enabled then we must fall back to 1936 1935 * the interpreter. 1937 1936 */ 1938 1937 if (!prog->jit_requested) 1939 - return orig_prog; 1940 - 1941 - tmp = bpf_jit_blind_constants(prog); 1942 - /* 1943 - * If blinding was requested and we failed during blinding, 1944 - * we must fall back to the interpreter. Otherwise, we save 1945 - * the new JITed code. 1946 - */ 1947 - if (IS_ERR(tmp)) 1948 - return orig_prog; 1949 - 1950 - if (tmp != prog) { 1951 - tmp_blinded = true; 1952 - prog = tmp; 1953 - } 1938 + return prog; 1954 1939 1955 1940 jit_data = prog->aux->jit_data; 1956 1941 if (!jit_data) { 1957 1942 jit_data = kzalloc_obj(*jit_data); 1958 - if (!jit_data) { 1959 - prog = orig_prog; 1960 - goto out; 1961 - } 1943 + if (!jit_data) 1944 + return prog; 1962 1945 prog->aux->jit_data = jit_data; 1963 1946 } 1964 1947 if (jit_data->ctx.offset) { ··· 1961 1978 ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); 1962 1979 1963 1980 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); 1964 - if (ctx.offset == NULL) { 1965 - prog = orig_prog; 1981 + if (ctx.offset == NULL) 1966 1982 goto out_offset; 1967 - } 1968 1983 1969 1984 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */ 1970 1985 build_prologue(&ctx); 1971 - if (build_body(&ctx, extra_pass)) { 1972 - prog = orig_prog; 1986 + if (build_body(&ctx, extra_pass)) 1973 1987 goto out_offset; 1974 - } 1975 1988 ctx.epilogue_offset = ctx.idx; 1976 1989 build_epilogue(&ctx); 1977 1990 ··· 1983 2004 /* Now we know the size of the structure to make */ 1984 2005 ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32), 1985 2006 &header, &image_ptr, jit_fill_hole); 1986 - if (!ro_header) { 1987 - prog = orig_prog; 2007 + if (!ro_header) 1988 2008 goto out_offset; 1989 - } 1990 2009 1991 2010 /* 2. Now, the actual pass to generate final JIT code */ 1992 2011 /* ··· 2004 2027 ctx.num_exentries = 0; 2005 2028 2006 2029 build_prologue(&ctx); 2007 - if (build_body(&ctx, extra_pass)) { 2008 - prog = orig_prog; 2030 + if (build_body(&ctx, extra_pass)) 2009 2031 goto out_free; 2010 - } 2011 2032 build_epilogue(&ctx); 2012 2033 2013 2034 /* 3. Extra pass to validate JITed code */ 2014 - if (validate_ctx(&ctx)) { 2015 - prog = orig_prog; 2035 + if (validate_ctx(&ctx)) 2016 2036 goto out_free; 2017 - } 2018 2037 2019 2038 /* And we're done */ 2020 2039 if (bpf_jit_enable > 1) ··· 2023 2050 goto out_free; 2024 2051 } 2025 2052 if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) { 2026 - /* ro_header has been freed */ 2053 + /* ro_header and header have been freed */ 2027 2054 ro_header = NULL; 2028 - prog = orig_prog; 2055 + header = NULL; 2029 2056 goto out_free; 2030 2057 } 2031 2058 /* ··· 2057 2084 prog->aux->jit_data = NULL; 2058 2085 } 2059 2086 2060 - out: 2061 - if (tmp_blinded) 2062 - bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); 2063 - 2064 2087 return prog; 2065 2088 2066 2089 out_free: 2090 + if (extra_pass) { 2091 + prog->bpf_func = NULL; 2092 + prog->jited = 0; 2093 + prog->jited_len = 0; 2094 + } 2095 + 2067 2096 if (header) { 2068 2097 bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size)); 2069 2098 bpf_jit_binary_pack_free(ro_header, header);

+2 -20

arch/mips/net/bpf_jit_comp.c

··· 909 909 return true; 910 910 } 911 911 912 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 912 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 913 913 { 914 - struct bpf_prog *tmp, *orig_prog = prog; 915 914 struct bpf_binary_header *header = NULL; 916 915 struct jit_context ctx; 917 - bool tmp_blinded = false; 918 916 unsigned int tmp_idx; 919 917 unsigned int image_size; 920 918 u8 *image_ptr; ··· 923 925 * the interpreter. 924 926 */ 925 927 if (!prog->jit_requested) 926 - return orig_prog; 927 - /* 928 - * If constant blinding was enabled and we failed during blinding 929 - * then we must fall back to the interpreter. Otherwise, we save 930 - * the new JITed code. 931 - */ 932 - tmp = bpf_jit_blind_constants(prog); 933 - if (IS_ERR(tmp)) 934 - return orig_prog; 935 - if (tmp != prog) { 936 - tmp_blinded = true; 937 - prog = tmp; 938 - } 928 + return prog; 939 929 940 930 memset(&ctx, 0, sizeof(ctx)); 941 931 ctx.program = prog; ··· 1011 1025 prog->jited_len = image_size; 1012 1026 1013 1027 out: 1014 - if (tmp_blinded) 1015 - bpf_jit_prog_release_other(prog, prog == orig_prog ? 1016 - tmp : orig_prog); 1017 1028 kfree(ctx.descriptors); 1018 1029 return prog; 1019 1030 1020 1031 out_err: 1021 - prog = orig_prog; 1022 1032 if (header) 1023 1033 bpf_jit_binary_free(header); 1024 1034 goto out;

+30 -45

arch/parisc/net/bpf_jit_core.c

··· 41 41 return true; 42 42 } 43 43 44 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 44 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 45 45 { 46 46 unsigned int prog_size = 0, extable_size = 0; 47 - bool tmp_blinded = false, extra_pass = false; 48 - struct bpf_prog *tmp, *orig_prog = prog; 47 + bool extra_pass = false; 49 48 int pass = 0, prev_ninsns = 0, prologue_len, i; 50 49 struct hppa_jit_data *jit_data; 51 50 struct hppa_jit_context *ctx; 52 51 53 52 if (!prog->jit_requested) 54 - return orig_prog; 55 - 56 - tmp = bpf_jit_blind_constants(prog); 57 - if (IS_ERR(tmp)) 58 - return orig_prog; 59 - if (tmp != prog) { 60 - tmp_blinded = true; 61 - prog = tmp; 62 - } 53 + return prog; 63 54 64 55 jit_data = prog->aux->jit_data; 65 56 if (!jit_data) { 66 57 jit_data = kzalloc_obj(*jit_data); 67 - if (!jit_data) { 68 - prog = orig_prog; 69 - goto out; 70 - } 58 + if (!jit_data) 59 + return prog; 71 60 prog->aux->jit_data = jit_data; 72 61 } 73 62 ··· 70 81 71 82 ctx->prog = prog; 72 83 ctx->offset = kzalloc_objs(int, prog->len); 73 - if (!ctx->offset) { 74 - prog = orig_prog; 75 - goto out_offset; 76 - } 84 + if (!ctx->offset) 85 + goto out_err; 77 86 for (i = 0; i < prog->len; i++) { 78 87 prev_ninsns += 20; 79 88 ctx->offset[i] = prev_ninsns; ··· 80 93 for (i = 0; i < NR_JIT_ITERATIONS; i++) { 81 94 pass++; 82 95 ctx->ninsns = 0; 83 - if (build_body(ctx, extra_pass, ctx->offset)) { 84 - prog = orig_prog; 85 - goto out_offset; 86 - } 96 + if (build_body(ctx, extra_pass, ctx->offset)) 97 + goto out_err; 87 98 ctx->body_len = ctx->ninsns; 88 99 bpf_jit_build_prologue(ctx); 89 100 ctx->prologue_len = ctx->ninsns - ctx->body_len; ··· 101 116 &jit_data->image, 102 117 sizeof(long), 103 118 bpf_fill_ill_insns); 104 - if (!jit_data->header) { 105 - prog = orig_prog; 106 - goto out_offset; 107 - } 119 + if (!jit_data->header) 120 + goto out_err; 108 121 109 122 ctx->insns = (u32 *)jit_data->image; 110 123 /* ··· 117 134 pr_err("bpf-jit: image did not converge in <%d passes!\n", i); 118 135 if (jit_data->header) 119 136 bpf_jit_binary_free(jit_data->header); 120 - prog = orig_prog; 121 - goto out_offset; 137 + goto out_err; 122 138 } 123 139 124 140 if (extable_size) ··· 130 148 bpf_jit_build_prologue(ctx); 131 149 if (build_body(ctx, extra_pass, NULL)) { 132 150 bpf_jit_binary_free(jit_data->header); 133 - prog = orig_prog; 134 - goto out_offset; 151 + goto out_err; 135 152 } 136 153 bpf_jit_build_epilogue(ctx); 137 154 ··· 141 160 { extern int machine_restart(char *); machine_restart(""); } 142 161 } 143 162 163 + if (!prog->is_func || extra_pass) { 164 + if (bpf_jit_binary_lock_ro(jit_data->header)) { 165 + bpf_jit_binary_free(jit_data->header); 166 + goto out_err; 167 + } 168 + bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns); 169 + } 170 + 144 171 prog->bpf_func = (void *)ctx->insns; 145 172 prog->jited = 1; 146 173 prog->jited_len = prog_size; 147 174 148 - bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns); 149 - 150 175 if (!prog->is_func || extra_pass) { 151 - if (bpf_jit_binary_lock_ro(jit_data->header)) { 152 - bpf_jit_binary_free(jit_data->header); 153 - prog->bpf_func = NULL; 154 - prog->jited = 0; 155 - prog->jited_len = 0; 156 - goto out_offset; 157 - } 158 176 prologue_len = ctx->epilogue_offset - ctx->body_len; 159 177 for (i = 0; i < prog->len; i++) 160 178 ctx->offset[i] += prologue_len; ··· 163 183 kfree(jit_data); 164 184 prog->aux->jit_data = NULL; 165 185 } 166 - out: 186 + 167 187 if (HPPA_JIT_REBOOT) 168 188 { extern int machine_restart(char *); machine_restart(""); } 169 189 170 - if (tmp_blinded) 171 - bpf_jit_prog_release_other(prog, prog == orig_prog ? 172 - tmp : orig_prog); 173 190 return prog; 191 + 192 + out_err: 193 + if (extra_pass) { 194 + prog->bpf_func = NULL; 195 + prog->jited = 0; 196 + prog->jited_len = 0; 197 + } 198 + goto out_offset; 174 199 } 175 200 176 201 u64 hppa_div64(u64 div, u64 divisor)

+27 -47

arch/powerpc/net/bpf_jit_comp.c

··· 162 162 } 163 163 } 164 164 165 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 165 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) 166 166 { 167 167 u32 proglen; 168 168 u32 alloclen; ··· 177 177 void __percpu *priv_stack_ptr = NULL; 178 178 struct bpf_binary_header *fhdr = NULL; 179 179 struct bpf_binary_header *hdr = NULL; 180 - struct bpf_prog *org_fp = fp; 181 - struct bpf_prog *tmp_fp = NULL; 182 - bool bpf_blinded = false; 183 180 bool extra_pass = false; 184 181 u8 *fimage = NULL; 185 182 u32 *fcode_base = NULL; ··· 184 187 u32 fixup_len; 185 188 186 189 if (!fp->jit_requested) 187 - return org_fp; 188 - 189 - tmp_fp = bpf_jit_blind_constants(org_fp); 190 - if (IS_ERR(tmp_fp)) 191 - return org_fp; 192 - 193 - if (tmp_fp != org_fp) { 194 - bpf_blinded = true; 195 - fp = tmp_fp; 196 - } 190 + return fp; 197 191 198 192 jit_data = fp->aux->jit_data; 199 193 if (!jit_data) { 200 194 jit_data = kzalloc_obj(*jit_data); 201 - if (!jit_data) { 202 - fp = org_fp; 203 - goto out; 204 - } 195 + if (!jit_data) 196 + return fp; 205 197 fp->aux->jit_data = jit_data; 206 198 } 207 199 ··· 205 219 priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) + 206 220 2 * PRIV_STACK_GUARD_SZ; 207 221 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_size, 16, GFP_KERNEL); 208 - if (!priv_stack_ptr) { 209 - fp = org_fp; 222 + if (!priv_stack_ptr) 210 223 goto out_priv_stack; 211 - } 212 224 213 225 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_size); 214 226 fp->aux->priv_stack_ptr = priv_stack_ptr; ··· 233 249 } 234 250 235 251 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 236 - if (addrs == NULL) { 237 - fp = org_fp; 238 - goto out_addrs; 239 - } 252 + if (addrs == NULL) 253 + goto out_err; 240 254 241 255 memset(&cgctx, 0, sizeof(struct codegen_context)); 242 256 bpf_jit_init_reg_mapping(&cgctx); ··· 261 279 } 262 280 263 281 /* Scouting faux-generate pass 0 */ 264 - if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { 282 + if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) 265 283 /* We hit something illegal or unsupported. */ 266 - fp = org_fp; 267 - goto out_addrs; 268 - } 284 + goto out_err; 269 285 270 286 /* 271 287 * If we have seen a tail call, we need a second pass. ··· 274 294 */ 275 295 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) { 276 296 cgctx.idx = 0; 277 - if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { 278 - fp = org_fp; 279 - goto out_addrs; 280 - } 297 + if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) 298 + goto out_err; 281 299 } 282 300 283 301 bpf_jit_realloc_regs(&cgctx); ··· 296 318 297 319 fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image, 298 320 bpf_jit_fill_ill_insns); 299 - if (!fhdr) { 300 - fp = org_fp; 301 - goto out_addrs; 302 - } 321 + if (!fhdr) 322 + goto out_err; 303 323 304 324 if (extable_len) 305 325 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len; ··· 316 340 extra_pass)) { 317 341 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size)); 318 342 bpf_jit_binary_pack_free(fhdr, hdr); 319 - fp = org_fp; 320 - goto out_addrs; 343 + goto out_err; 321 344 } 322 345 bpf_jit_build_epilogue(code_base, &cgctx); 323 346 ··· 338 363 ((u64 *)image)[1] = local_paca->kernel_toc; 339 364 #endif 340 365 366 + if (!fp->is_func || extra_pass) { 367 + if (bpf_jit_binary_pack_finalize(fhdr, hdr)) 368 + goto out_err; 369 + } 370 + 341 371 fp->bpf_func = (void *)fimage; 342 372 fp->jited = 1; 343 373 fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE; 344 374 345 375 if (!fp->is_func || extra_pass) { 346 - if (bpf_jit_binary_pack_finalize(fhdr, hdr)) { 347 - fp = org_fp; 348 - goto out_addrs; 349 - } 350 376 bpf_prog_fill_jited_linfo(fp, addrs); 351 377 /* 352 378 * On ABI V1, executable code starts after the function ··· 374 398 jit_data->hdr = hdr; 375 399 } 376 400 377 - out: 378 - if (bpf_blinded) 379 - bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 380 - 381 401 return fp; 402 + 403 + out_err: 404 + if (extra_pass) { 405 + fp->bpf_func = NULL; 406 + fp->jited = 0; 407 + fp->jited_len = 0; 408 + } 409 + goto out_addrs; 382 410 } 383 411 384 412 /*

-6

arch/riscv/net/bpf_jit.h

··· 11 11 12 12 #include <linux/bpf.h> 13 13 #include <linux/filter.h> 14 - #include <asm/cacheflush.h> 15 14 16 15 /* verify runtime detection extension status */ 17 16 #define rv_ext_enabled(ext) \ ··· 102 103 static inline void bpf_fill_ill_insns(void *area, unsigned int size) 103 104 { 104 105 memset(area, 0, size); 105 - } 106 - 107 - static inline void bpf_flush_icache(void *start, void *end) 108 - { 109 - flush_icache_range((unsigned long)start, (unsigned long)end); 110 106 } 111 107 112 108 /* Emit a 4-byte riscv instruction. */

+23 -47

arch/riscv/net/bpf_jit_core.c

··· 41 41 return true; 42 42 } 43 43 44 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 44 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 45 45 { 46 46 unsigned int prog_size = 0, extable_size = 0; 47 - bool tmp_blinded = false, extra_pass = false; 48 - struct bpf_prog *tmp, *orig_prog = prog; 47 + bool extra_pass = false; 49 48 int pass = 0, prev_ninsns = 0, i; 50 49 struct rv_jit_data *jit_data; 51 50 struct rv_jit_context *ctx; 52 51 53 52 if (!prog->jit_requested) 54 - return orig_prog; 55 - 56 - tmp = bpf_jit_blind_constants(prog); 57 - if (IS_ERR(tmp)) 58 - return orig_prog; 59 - if (tmp != prog) { 60 - tmp_blinded = true; 61 - prog = tmp; 62 - } 53 + return prog; 63 54 64 55 jit_data = prog->aux->jit_data; 65 56 if (!jit_data) { 66 57 jit_data = kzalloc_obj(*jit_data); 67 58 if (!jit_data) { 68 - prog = orig_prog; 69 - goto out; 59 + return prog; 70 60 } 71 61 prog->aux->jit_data = jit_data; 72 62 } ··· 73 83 ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); 74 84 ctx->prog = prog; 75 85 ctx->offset = kzalloc_objs(int, prog->len); 76 - if (!ctx->offset) { 77 - prog = orig_prog; 86 + if (!ctx->offset) 78 87 goto out_offset; 79 - } 80 88 81 - if (build_body(ctx, extra_pass, NULL)) { 82 - prog = orig_prog; 89 + if (build_body(ctx, extra_pass, NULL)) 83 90 goto out_offset; 84 - } 85 91 86 92 for (i = 0; i < prog->len; i++) { 87 93 prev_ninsns += 32; ··· 91 105 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); 92 106 ctx->prologue_len = ctx->ninsns; 93 107 94 - if (build_body(ctx, extra_pass, ctx->offset)) { 95 - prog = orig_prog; 108 + if (build_body(ctx, extra_pass, ctx->offset)) 96 109 goto out_offset; 97 - } 98 110 99 111 ctx->epilogue_offset = ctx->ninsns; 100 112 bpf_jit_build_epilogue(ctx); ··· 110 126 &jit_data->ro_image, sizeof(u32), 111 127 &jit_data->header, &jit_data->image, 112 128 bpf_fill_ill_insns); 113 - if (!jit_data->ro_header) { 114 - prog = orig_prog; 129 + if (!jit_data->ro_header) 115 130 goto out_offset; 116 - } 117 131 118 132 /* 119 133 * Use the image(RW) for writing the JITed instructions. But also save ··· 132 150 133 151 if (i == NR_JIT_ITERATIONS) { 134 152 pr_err("bpf-jit: image did not converge in <%d passes!\n", i); 135 - prog = orig_prog; 136 153 goto out_free_hdr; 137 154 } 138 155 ··· 144 163 ctx->nexentries = 0; 145 164 146 165 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); 147 - if (build_body(ctx, extra_pass, NULL)) { 148 - prog = orig_prog; 166 + if (build_body(ctx, extra_pass, NULL)) 149 167 goto out_free_hdr; 150 - } 151 168 bpf_jit_build_epilogue(ctx); 152 169 153 170 if (bpf_jit_enable > 1) 154 171 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns); 172 + 173 + if (!prog->is_func || extra_pass) { 174 + if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) { 175 + /* ro_header has been freed */ 176 + jit_data->ro_header = NULL; 177 + jit_data->header = NULL; 178 + goto out_free_hdr; 179 + } 180 + } 155 181 156 182 prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset(); 157 183 prog->jited = 1; 158 184 prog->jited_len = prog_size - cfi_get_offset(); 159 185 160 186 if (!prog->is_func || extra_pass) { 161 - if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) { 162 - /* ro_header has been freed */ 163 - jit_data->ro_header = NULL; 164 - prog = orig_prog; 165 - goto out_offset; 166 - } 167 - /* 168 - * The instructions have now been copied to the ROX region from 169 - * where they will execute. 170 - * Write any modified data cache blocks out to memory and 171 - * invalidate the corresponding blocks in the instruction cache. 172 - */ 173 - bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns); 174 187 for (i = 0; i < prog->len; i++) 175 188 ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); 176 189 bpf_prog_fill_jited_linfo(prog, ctx->offset); ··· 173 198 kfree(jit_data); 174 199 prog->aux->jit_data = NULL; 175 200 } 176 - out: 177 201 178 - if (tmp_blinded) 179 - bpf_jit_prog_release_other(prog, prog == orig_prog ? 180 - tmp : orig_prog); 181 202 return prog; 182 203 183 204 out_free_hdr: 205 + if (extra_pass) { 206 + prog->bpf_func = NULL; 207 + prog->jited = 0; 208 + prog->jited_len = 0; 209 + } 184 210 if (jit_data->header) { 185 211 bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size, 186 212 sizeof(jit_data->header->size));

+21 -40

arch/s390/net/bpf_jit_comp.c

··· 2312 2312 /* 2313 2313 * Compile eBPF program "fp" 2314 2314 */ 2315 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 2315 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) 2316 2316 { 2317 - struct bpf_prog *tmp, *orig_fp = fp; 2318 2317 struct bpf_binary_header *header; 2319 2318 struct s390_jit_data *jit_data; 2320 - bool tmp_blinded = false; 2321 2319 bool extra_pass = false; 2322 2320 struct bpf_jit jit; 2323 2321 int pass; 2324 2322 2325 2323 if (!fp->jit_requested) 2326 - return orig_fp; 2327 - 2328 - tmp = bpf_jit_blind_constants(fp); 2329 - /* 2330 - * If blinding was requested and we failed during blinding, 2331 - * we must fall back to the interpreter. 2332 - */ 2333 - if (IS_ERR(tmp)) 2334 - return orig_fp; 2335 - if (tmp != fp) { 2336 - tmp_blinded = true; 2337 - fp = tmp; 2338 - } 2324 + return fp; 2339 2325 2340 2326 jit_data = fp->aux->jit_data; 2341 2327 if (!jit_data) { 2342 2328 jit_data = kzalloc_obj(*jit_data); 2343 - if (!jit_data) { 2344 - fp = orig_fp; 2345 - goto out; 2346 - } 2329 + if (!jit_data) 2330 + return fp; 2347 2331 fp->aux->jit_data = jit_data; 2348 2332 } 2349 2333 if (jit_data->ctx.addrs) { ··· 2340 2356 2341 2357 memset(&jit, 0, sizeof(jit)); 2342 2358 jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL); 2343 - if (jit.addrs == NULL) { 2344 - fp = orig_fp; 2345 - goto free_addrs; 2346 - } 2359 + if (jit.addrs == NULL) 2360 + goto out_err; 2347 2361 /* 2348 2362 * Three initial passes: 2349 2363 * - 1/2: Determine clobbered registers 2350 2364 * - 3: Calculate program size and addrs array 2351 2365 */ 2352 2366 for (pass = 1; pass <= 3; pass++) { 2353 - if (bpf_jit_prog(&jit, fp, extra_pass)) { 2354 - fp = orig_fp; 2355 - goto free_addrs; 2356 - } 2367 + if (bpf_jit_prog(&jit, fp, extra_pass)) 2368 + goto out_err; 2357 2369 } 2358 2370 /* 2359 2371 * Final pass: Allocate and generate program 2360 2372 */ 2361 2373 header = bpf_jit_alloc(&jit, fp); 2362 - if (!header) { 2363 - fp = orig_fp; 2364 - goto free_addrs; 2365 - } 2374 + if (!header) 2375 + goto out_err; 2366 2376 skip_init_ctx: 2367 2377 if (bpf_jit_prog(&jit, fp, extra_pass)) { 2368 2378 bpf_jit_binary_free(header); 2369 - fp = orig_fp; 2370 - goto free_addrs; 2379 + goto out_err; 2371 2380 } 2372 2381 if (bpf_jit_enable > 1) { 2373 2382 bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf); ··· 2369 2392 if (!fp->is_func || extra_pass) { 2370 2393 if (bpf_jit_binary_lock_ro(header)) { 2371 2394 bpf_jit_binary_free(header); 2372 - fp = orig_fp; 2373 - goto free_addrs; 2395 + goto out_err; 2374 2396 } 2375 2397 } else { 2376 2398 jit_data->header = header; ··· 2387 2411 kfree(jit_data); 2388 2412 fp->aux->jit_data = NULL; 2389 2413 } 2390 - out: 2391 - if (tmp_blinded) 2392 - bpf_jit_prog_release_other(fp, fp == orig_fp ? 2393 - tmp : orig_fp); 2414 + 2394 2415 return fp; 2416 + 2417 + out_err: 2418 + if (extra_pass) { 2419 + fp->bpf_func = NULL; 2420 + fp->jited = 0; 2421 + fp->jited_len = 0; 2422 + } 2423 + goto free_addrs; 2395 2424 } 2396 2425 2397 2426 bool bpf_jit_supports_kfunc_call(void)

+22 -41

arch/sparc/net/bpf_jit_comp_64.c

··· 1477 1477 struct jit_ctx ctx; 1478 1478 }; 1479 1479 1480 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 1480 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 1481 1481 { 1482 - struct bpf_prog *tmp, *orig_prog = prog; 1483 1482 struct sparc64_jit_data *jit_data; 1484 1483 struct bpf_binary_header *header; 1485 1484 u32 prev_image_size, image_size; 1486 - bool tmp_blinded = false; 1487 1485 bool extra_pass = false; 1488 1486 struct jit_ctx ctx; 1489 1487 u8 *image_ptr; 1490 1488 int pass, i; 1491 1489 1492 1490 if (!prog->jit_requested) 1493 - return orig_prog; 1494 - 1495 - tmp = bpf_jit_blind_constants(prog); 1496 - /* If blinding was requested and we failed during blinding, 1497 - * we must fall back to the interpreter. 1498 - */ 1499 - if (IS_ERR(tmp)) 1500 - return orig_prog; 1501 - if (tmp != prog) { 1502 - tmp_blinded = true; 1503 - prog = tmp; 1504 - } 1491 + return prog; 1505 1492 1506 1493 jit_data = prog->aux->jit_data; 1507 1494 if (!jit_data) { 1508 1495 jit_data = kzalloc_obj(*jit_data); 1509 - if (!jit_data) { 1510 - prog = orig_prog; 1511 - goto out; 1512 - } 1496 + if (!jit_data) 1497 + return prog; 1513 1498 prog->aux->jit_data = jit_data; 1514 1499 } 1515 1500 if (jit_data->ctx.offset) { ··· 1512 1527 ctx.prog = prog; 1513 1528 1514 1529 ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL); 1515 - if (ctx.offset == NULL) { 1516 - prog = orig_prog; 1517 - goto out_off; 1518 - } 1530 + if (ctx.offset == NULL) 1531 + goto out_err; 1519 1532 1520 1533 /* Longest sequence emitted is for bswap32, 12 instructions. Pre-cook 1521 1534 * the offset array so that we converge faster. ··· 1526 1543 ctx.idx = 0; 1527 1544 1528 1545 build_prologue(&ctx); 1529 - if (build_body(&ctx)) { 1530 - prog = orig_prog; 1531 - goto out_off; 1532 - } 1546 + if (build_body(&ctx)) 1547 + goto out_err; 1533 1548 build_epilogue(&ctx); 1534 1549 1535 1550 if (bpf_jit_enable > 1) ··· 1550 1569 image_size = sizeof(u32) * ctx.idx; 1551 1570 header = bpf_jit_binary_alloc(image_size, &image_ptr, 1552 1571 sizeof(u32), jit_fill_hole); 1553 - if (header == NULL) { 1554 - prog = orig_prog; 1555 - goto out_off; 1556 - } 1572 + if (header == NULL) 1573 + goto out_err; 1557 1574 1558 1575 ctx.image = (u32 *)image_ptr; 1559 1576 skip_init_ctx: ··· 1561 1582 1562 1583 if (build_body(&ctx)) { 1563 1584 bpf_jit_binary_free(header); 1564 - prog = orig_prog; 1565 - goto out_off; 1585 + goto out_err; 1566 1586 } 1567 1587 1568 1588 build_epilogue(&ctx); ··· 1570 1592 pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n", 1571 1593 prev_image_size, ctx.idx * 4); 1572 1594 bpf_jit_binary_free(header); 1573 - prog = orig_prog; 1574 - goto out_off; 1595 + goto out_err; 1575 1596 } 1576 1597 1577 1598 if (bpf_jit_enable > 1) ··· 1581 1604 if (!prog->is_func || extra_pass) { 1582 1605 if (bpf_jit_binary_lock_ro(header)) { 1583 1606 bpf_jit_binary_free(header); 1584 - prog = orig_prog; 1585 - goto out_off; 1607 + goto out_err; 1586 1608 } 1587 1609 } else { 1588 1610 jit_data->ctx = ctx; ··· 1600 1624 kfree(jit_data); 1601 1625 prog->aux->jit_data = NULL; 1602 1626 } 1603 - out: 1604 - if (tmp_blinded) 1605 - bpf_jit_prog_release_other(prog, prog == orig_prog ? 1606 - tmp : orig_prog); 1627 + 1607 1628 return prog; 1629 + 1630 + out_err: 1631 + if (extra_pass) { 1632 + prog->bpf_func = NULL; 1633 + prog->jited = 0; 1634 + prog->jited_len = 0; 1635 + } 1636 + goto out_off; 1608 1637 }

+23 -50

arch/x86/net/bpf_jit_comp.c

··· 58 58 #define EMIT_ENDBR() EMIT(gen_endbr(), 4) 59 59 #define EMIT_ENDBR_POISON() EMIT(gen_endbr_poison(), 4) 60 60 #else 61 - #define EMIT_ENDBR() 62 - #define EMIT_ENDBR_POISON() 61 + #define EMIT_ENDBR() do { } while (0) 62 + #define EMIT_ENDBR_POISON() do { } while (0) 63 63 #endif 64 64 65 65 static bool is_imm8(int value) ··· 1649 1649 return 0; 1650 1650 } 1651 1651 1652 - static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image, 1653 - int oldproglen, struct jit_context *ctx, bool jmp_padding) 1652 + static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image, 1653 + u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding) 1654 1654 { 1655 1655 bool tail_call_reachable = bpf_prog->aux->tail_call_reachable; 1656 1656 struct bpf_insn *insn = bpf_prog->insnsi; ··· 1663 1663 void __percpu *priv_stack_ptr; 1664 1664 int i, excnt = 0; 1665 1665 int ilen, proglen = 0; 1666 - u8 *prog = temp; 1666 + u8 *ip, *prog = temp; 1667 1667 u32 stack_depth; 1668 1668 int err; 1669 1669 ··· 1733 1733 if (dst_reg == BPF_REG_FP) 1734 1734 dst_reg = X86_REG_R9; 1735 1735 } 1736 + 1737 + if (bpf_insn_is_indirect_target(env, bpf_prog, i - 1)) 1738 + EMIT_ENDBR(); 1739 + 1740 + ip = image + addrs[i - 1] + (prog - temp); 1736 1741 1737 1742 switch (insn->code) { 1738 1743 /* ALU */ ··· 2445 2440 2446 2441 /* call */ 2447 2442 case BPF_JMP | BPF_CALL: { 2448 - u8 *ip = image + addrs[i - 1]; 2449 - 2450 2443 func = (u8 *) __bpf_call_base + imm32; 2451 2444 if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) { 2452 2445 LOAD_TAIL_CALL_CNT_PTR(stack_depth); ··· 2468 2465 if (imm32) 2469 2466 emit_bpf_tail_call_direct(bpf_prog, 2470 2467 &bpf_prog->aux->poke_tab[imm32 - 1], 2471 - &prog, image + addrs[i - 1], 2468 + &prog, 2469 + ip, 2472 2470 callee_regs_used, 2473 2471 stack_depth, 2474 2472 ctx); ··· 2478 2474 &prog, 2479 2475 callee_regs_used, 2480 2476 stack_depth, 2481 - image + addrs[i - 1], 2477 + ip, 2482 2478 ctx); 2483 2479 break; 2484 2480 ··· 2643 2639 break; 2644 2640 2645 2641 case BPF_JMP | BPF_JA | BPF_X: 2646 - emit_indirect_jump(&prog, insn->dst_reg, image + addrs[i - 1]); 2642 + emit_indirect_jump(&prog, insn->dst_reg, ip); 2647 2643 break; 2648 2644 case BPF_JMP | BPF_JA: 2649 2645 case BPF_JMP32 | BPF_JA: ··· 2733 2729 ctx->cleanup_addr = proglen; 2734 2730 if (bpf_prog_was_classic(bpf_prog) && 2735 2731 !ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) { 2736 - u8 *ip = image + addrs[i - 1]; 2737 - 2738 2732 if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog)) 2739 2733 return -EINVAL; 2740 2734 } ··· 3715 3713 #define MAX_PASSES 20 3716 3714 #define PADDING_PASSES (MAX_PASSES - 5) 3717 3715 3718 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 3716 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 3719 3717 { 3720 3718 struct bpf_binary_header *rw_header = NULL; 3721 3719 struct bpf_binary_header *header = NULL; 3722 - struct bpf_prog *tmp, *orig_prog = prog; 3723 3720 void __percpu *priv_stack_ptr = NULL; 3724 3721 struct x64_jit_data *jit_data; 3725 3722 int priv_stack_alloc_sz; 3726 3723 int proglen, oldproglen = 0; 3727 3724 struct jit_context ctx = {}; 3728 - bool tmp_blinded = false; 3729 3725 bool extra_pass = false; 3730 3726 bool padding = false; 3731 3727 u8 *rw_image = NULL; ··· 3733 3733 int i; 3734 3734 3735 3735 if (!prog->jit_requested) 3736 - return orig_prog; 3737 - 3738 - tmp = bpf_jit_blind_constants(prog); 3739 - /* 3740 - * If blinding was requested and we failed during blinding, 3741 - * we must fall back to the interpreter. 3742 - */ 3743 - if (IS_ERR(tmp)) 3744 - return orig_prog; 3745 - if (tmp != prog) { 3746 - tmp_blinded = true; 3747 - prog = tmp; 3748 - } 3736 + return prog; 3749 3737 3750 3738 jit_data = prog->aux->jit_data; 3751 3739 if (!jit_data) { 3752 3740 jit_data = kzalloc_obj(*jit_data); 3753 - if (!jit_data) { 3754 - prog = orig_prog; 3755 - goto out; 3756 - } 3741 + if (!jit_data) 3742 + return prog; 3757 3743 prog->aux->jit_data = jit_data; 3758 3744 } 3759 3745 priv_stack_ptr = prog->aux->priv_stack_ptr; ··· 3751 3765 priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 8) + 3752 3766 2 * PRIV_STACK_GUARD_SZ; 3753 3767 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 8, GFP_KERNEL); 3754 - if (!priv_stack_ptr) { 3755 - prog = orig_prog; 3768 + if (!priv_stack_ptr) 3756 3769 goto out_priv_stack; 3757 - } 3758 3770 3759 3771 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz); 3760 3772 prog->aux->priv_stack_ptr = priv_stack_ptr; ··· 3770 3786 goto skip_init_addrs; 3771 3787 } 3772 3788 addrs = kvmalloc_objs(*addrs, prog->len + 1); 3773 - if (!addrs) { 3774 - prog = orig_prog; 3789 + if (!addrs) 3775 3790 goto out_addrs; 3776 - } 3777 3791 3778 3792 /* 3779 3793 * Before first pass, make a rough estimation of addrs[] ··· 3793 3811 for (pass = 0; pass < MAX_PASSES || image; pass++) { 3794 3812 if (!padding && pass >= PADDING_PASSES) 3795 3813 padding = true; 3796 - proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx, padding); 3814 + proglen = do_jit(env, prog, addrs, image, rw_image, oldproglen, &ctx, padding); 3797 3815 if (proglen <= 0) { 3798 3816 out_image: 3799 3817 image = NULL; ··· 3802 3820 sizeof(rw_header->size)); 3803 3821 bpf_jit_binary_pack_free(header, rw_header); 3804 3822 } 3805 - /* Fall back to interpreter mode */ 3806 - prog = orig_prog; 3807 3823 if (extra_pass) { 3808 3824 prog->bpf_func = NULL; 3809 3825 prog->jited = 0; ··· 3832 3852 header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size, 3833 3853 &image, align, &rw_header, &rw_image, 3834 3854 jit_fill_hole); 3835 - if (!header) { 3836 - prog = orig_prog; 3855 + if (!header) 3837 3856 goto out_addrs; 3838 - } 3839 3857 prog->aux->extable = (void *) image + roundup(proglen, align); 3840 3858 } 3841 3859 oldproglen = proglen; ··· 3886 3908 prog->bpf_func = (void *)image + cfi_get_offset(); 3887 3909 prog->jited = 1; 3888 3910 prog->jited_len = proglen - cfi_get_offset(); 3889 - } else { 3890 - prog = orig_prog; 3891 3911 } 3892 3912 3893 3913 if (!image || !prog->is_func || extra_pass) { ··· 3901 3925 kfree(jit_data); 3902 3926 prog->aux->jit_data = NULL; 3903 3927 } 3904 - out: 3905 - if (tmp_blinded) 3906 - bpf_jit_prog_release_other(prog, prog == orig_prog ? 3907 - tmp : orig_prog); 3928 + 3908 3929 return prog; 3909 3930 } 3910 3931

+5 -30

arch/x86/net/bpf_jit_comp32.c

··· 2518 2518 return true; 2519 2519 } 2520 2520 2521 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 2521 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 2522 2522 { 2523 2523 struct bpf_binary_header *header = NULL; 2524 - struct bpf_prog *tmp, *orig_prog = prog; 2525 2524 int proglen, oldproglen = 0; 2526 2525 struct jit_context ctx = {}; 2527 - bool tmp_blinded = false; 2528 2526 u8 *image = NULL; 2529 2527 int *addrs; 2530 2528 int pass; 2531 2529 int i; 2532 2530 2533 2531 if (!prog->jit_requested) 2534 - return orig_prog; 2535 - 2536 - tmp = bpf_jit_blind_constants(prog); 2537 - /* 2538 - * If blinding was requested and we failed during blinding, 2539 - * we must fall back to the interpreter. 2540 - */ 2541 - if (IS_ERR(tmp)) 2542 - return orig_prog; 2543 - if (tmp != prog) { 2544 - tmp_blinded = true; 2545 - prog = tmp; 2546 - } 2532 + return prog; 2547 2533 2548 2534 addrs = kmalloc_objs(*addrs, prog->len); 2549 - if (!addrs) { 2550 - prog = orig_prog; 2551 - goto out; 2552 - } 2535 + if (!addrs) 2536 + return prog; 2553 2537 2554 2538 /* 2555 2539 * Before first pass, make a rough estimation of addrs[] ··· 2558 2574 image = NULL; 2559 2575 if (header) 2560 2576 bpf_jit_binary_free(header); 2561 - prog = orig_prog; 2562 2577 goto out_addrs; 2563 2578 } 2564 2579 if (image) { ··· 2571 2588 if (proglen == oldproglen) { 2572 2589 header = bpf_jit_binary_alloc(proglen, &image, 2573 2590 1, jit_fill_hole); 2574 - if (!header) { 2575 - prog = orig_prog; 2591 + if (!header) 2576 2592 goto out_addrs; 2577 - } 2578 2593 } 2579 2594 oldproglen = proglen; 2580 2595 cond_resched(); ··· 2585 2604 prog->bpf_func = (void *)image; 2586 2605 prog->jited = 1; 2587 2606 prog->jited_len = proglen; 2588 - } else { 2589 - prog = orig_prog; 2590 2607 } 2591 2608 2592 2609 out_addrs: 2593 2610 kfree(addrs); 2594 - out: 2595 - if (tmp_blinded) 2596 - bpf_jit_prog_release_other(prog, prog == orig_prog ? 2597 - tmp : orig_prog); 2598 2611 return prog; 2599 2612 } 2600 2613

+2

include/linux/bpf.h

··· 1541 1541 int bpf_jit_charge_modmem(u32 size); 1542 1542 void bpf_jit_uncharge_modmem(u32 size); 1543 1543 bool bpf_prog_has_trampoline(const struct bpf_prog *prog); 1544 + bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog, 1545 + int insn_idx); 1544 1546 #else 1545 1547 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link, 1546 1548 struct bpf_trampoline *tr,

+5 -4

include/linux/bpf_verifier.h

··· 630 630 631 631 /* below fields are initialized once */ 632 632 unsigned int orig_idx; /* original instruction index */ 633 - bool jmp_point; 634 - bool prune_point; 633 + u32 jmp_point:1; 634 + u32 prune_point:1; 635 635 /* ensure we check state equivalence and save state checkpoint and 636 636 * this instruction, regardless of any heuristics 637 637 */ 638 - bool force_checkpoint; 638 + u32 force_checkpoint:1; 639 639 /* true if instruction is a call to a helper function that 640 640 * accepts callback function as a parameter. 641 641 */ 642 - bool calls_callback; 642 + u32 calls_callback:1; 643 + u32 indirect_target:1; /* if it is an indirect jump target */ 643 644 /* 644 645 * CFG strongly connected component this instruction belongs to, 645 646 * zero if it is a singleton SCC.

+48 -2

include/linux/filter.h

··· 1108 1108 return sk_filter_trim_cap(sk, skb, 1); 1109 1109 } 1110 1110 1111 + struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp, 1112 + int *err); 1111 1113 struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err); 1112 1114 void bpf_prog_free(struct bpf_prog *fp); 1113 1115 ··· 1155 1153 ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \ 1156 1154 (void *)__bpf_call_base) 1157 1155 1158 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog); 1156 + struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog); 1159 1157 void bpf_jit_compile(struct bpf_prog *prog); 1160 1158 bool bpf_jit_needs_zext(void); 1161 1159 bool bpf_jit_inlines_helper_call(s32 imm); ··· 1186 1184 1187 1185 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, 1188 1186 const struct bpf_insn *patch, u32 len); 1187 + 1188 + #ifdef CONFIG_BPF_SYSCALL 1189 + struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 1190 + const struct bpf_insn *patch, u32 len); 1191 + struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env); 1192 + void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, 1193 + struct bpf_insn_aux_data *orig_insn_aux); 1194 + #else 1195 + static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 1196 + const struct bpf_insn *patch, u32 len) 1197 + { 1198 + return ERR_PTR(-ENOTSUPP); 1199 + } 1200 + 1201 + static inline struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) 1202 + { 1203 + return NULL; 1204 + } 1205 + 1206 + static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, 1207 + struct bpf_insn_aux_data *orig_insn_aux) 1208 + { 1209 + } 1210 + #endif /* CONFIG_BPF_SYSCALL */ 1211 + 1189 1212 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt); 1190 1213 1191 1214 static inline bool xdp_return_frame_no_direct(void) ··· 1337 1310 1338 1311 const char *bpf_jit_get_prog_name(struct bpf_prog *prog); 1339 1312 1340 - struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp); 1313 + struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog); 1341 1314 void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other); 1315 + 1316 + static inline bool bpf_prog_need_blind(const struct bpf_prog *prog) 1317 + { 1318 + return prog->blinding_requested && !prog->blinded; 1319 + } 1342 1320 1343 1321 static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen, 1344 1322 u32 pass, void *image) ··· 1483 1451 { 1484 1452 } 1485 1453 1454 + static inline bool bpf_prog_need_blind(const struct bpf_prog *prog) 1455 + { 1456 + return false; 1457 + } 1458 + 1459 + static inline 1460 + struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog) 1461 + { 1462 + return prog; 1463 + } 1464 + 1465 + static inline void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) 1466 + { 1467 + } 1486 1468 #endif /* CONFIG_BPF_JIT */ 1487 1469 1488 1470 void bpf_prog_kallsyms_del_all(struct bpf_prog *fp);

+20 -3

kernel/bpf/arena.c

··· 341 341 refcount_inc(&vml->mmap_count); 342 342 } 343 343 344 + static int arena_vm_may_split(struct vm_area_struct *vma, unsigned long addr) 345 + { 346 + return -EINVAL; 347 + } 348 + 349 + static int arena_vm_mremap(struct vm_area_struct *vma) 350 + { 351 + return -EINVAL; 352 + } 353 + 344 354 static void arena_vm_close(struct vm_area_struct *vma) 345 355 { 346 356 struct bpf_map *map = vma->vm_file->private_data; ··· 427 417 428 418 static const struct vm_operations_struct arena_vm_ops = { 429 419 .open = arena_vm_open, 420 + .may_split = arena_vm_may_split, 421 + .mremap = arena_vm_mremap, 430 422 .close = arena_vm_close, 431 423 .fault = arena_vm_fault, 432 424 }; ··· 498 486 arena->user_vm_end = vma->vm_end; 499 487 /* 500 488 * bpf_map_mmap() checks that it's being mmaped as VM_SHARED and 501 - * clears VM_MAYEXEC. Set VM_DONTEXPAND as well to avoid 502 - * potential change of user_vm_start. 489 + * clears VM_MAYEXEC. Set VM_DONTEXPAND to avoid potential change 490 + * of user_vm_start. Set VM_DONTCOPY to prevent arena VMA from 491 + * being copied into the child process on fork. 503 492 */ 504 - vm_flags_set(vma, VM_DONTEXPAND); 493 + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTCOPY); 505 494 vma->vm_ops = &arena_vm_ops; 506 495 return 0; 507 496 } ··· 561 548 long pgoff = 0; 562 549 u32 uaddr32; 563 550 int ret, i; 551 + 552 + if (node_id != NUMA_NO_NODE && 553 + ((unsigned int)node_id >= nr_node_ids || !node_online(node_id))) 554 + return 0; 564 555 565 556 if (page_cnt > page_cnt_max) 566 557 return 0;

+4 -3

kernel/bpf/bpf_struct_ops.c

··· 811 811 goto reset_unlock; 812 812 } 813 813 814 - /* Poison pointer on error instead of return for backward compatibility */ 815 - bpf_prog_assoc_struct_ops(prog, &st_map->map); 816 - 817 814 link = kzalloc_obj(*link, GFP_USER); 818 815 if (!link) { 819 816 bpf_prog_put(prog); ··· 820 823 bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, 821 824 &bpf_struct_ops_link_lops, prog, prog->expected_attach_type); 822 825 *plink++ = &link->link; 826 + 827 + /* Poison pointer on error instead of return for backward compatibility */ 828 + bpf_prog_assoc_struct_ops(prog, &st_map->map); 823 829 824 830 ksym = kzalloc_obj(*ksym, GFP_USER); 825 831 if (!ksym) { ··· 906 906 reset_unlock: 907 907 bpf_struct_ops_map_free_ksyms(st_map); 908 908 bpf_struct_ops_map_free_image(st_map); 909 + bpf_struct_ops_map_dissoc_progs(st_map); 909 910 bpf_struct_ops_map_put_progs(st_map); 910 911 memset(uvalue, 0, map->value_size); 911 912 memset(kvalue, 0, map->value_size);

+99 -39

kernel/bpf/core.c

··· 1491 1491 bpf_prog_clone_free(fp_other); 1492 1492 } 1493 1493 1494 - static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len) 1495 - { 1496 - #ifdef CONFIG_BPF_SYSCALL 1497 - struct bpf_map *map; 1498 - int i; 1499 - 1500 - if (len <= 1) 1501 - return; 1502 - 1503 - for (i = 0; i < prog->aux->used_map_cnt; i++) { 1504 - map = prog->aux->used_maps[i]; 1505 - if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) 1506 - bpf_insn_array_adjust(map, off, len); 1507 - } 1508 - #endif 1509 - } 1510 - 1511 - struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) 1494 + /* 1495 + * Now this function is used only to blind the main prog and must be invoked only when 1496 + * bpf_prog_need_blind() returns true. 1497 + */ 1498 + struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog) 1512 1499 { 1513 1500 struct bpf_insn insn_buff[16], aux[2]; 1514 1501 struct bpf_prog *clone, *tmp; ··· 1503 1516 struct bpf_insn *insn; 1504 1517 int i, rewritten; 1505 1518 1506 - if (!prog->blinding_requested || prog->blinded) 1507 - return prog; 1519 + if (WARN_ON_ONCE(env && env->prog != prog)) 1520 + return ERR_PTR(-EINVAL); 1508 1521 1509 1522 clone = bpf_prog_clone_create(prog, GFP_USER); 1510 1523 if (!clone) 1511 1524 return ERR_PTR(-ENOMEM); 1525 + 1526 + /* make sure bpf_patch_insn_data() patches the correct prog */ 1527 + if (env) 1528 + env->prog = clone; 1512 1529 1513 1530 insn_cnt = clone->len; 1514 1531 insn = clone->insnsi; ··· 1541 1550 if (!rewritten) 1542 1551 continue; 1543 1552 1544 - tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); 1545 - if (IS_ERR(tmp)) { 1553 + if (env) 1554 + tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten); 1555 + else 1556 + tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); 1557 + 1558 + if (IS_ERR_OR_NULL(tmp)) { 1559 + if (env) 1560 + /* restore the original prog */ 1561 + env->prog = prog; 1546 1562 /* Patching may have repointed aux->prog during 1547 1563 * realloc from the original one, so we need to 1548 1564 * fix it up here on error. 1549 1565 */ 1550 1566 bpf_jit_prog_release_other(prog, clone); 1551 - return tmp; 1567 + return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM); 1552 1568 } 1553 1569 1554 1570 clone = tmp; 1555 1571 insn_delta = rewritten - 1; 1556 1572 1557 - /* Instructions arrays must be updated using absolute xlated offsets */ 1558 - adjust_insn_arrays(clone, prog->aux->subprog_start + i, rewritten); 1573 + if (env) 1574 + env->prog = clone; 1559 1575 1560 1576 /* Walk new program and skip insns we just inserted. */ 1561 1577 insn = clone->insnsi + i + insn_delta; ··· 1572 1574 1573 1575 clone->blinded = 1; 1574 1576 return clone; 1577 + } 1578 + 1579 + bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog, 1580 + int insn_idx) 1581 + { 1582 + if (!env) 1583 + return false; 1584 + insn_idx += prog->aux->subprog_start; 1585 + return env->insn_aux_data[insn_idx].indirect_target; 1575 1586 } 1576 1587 #endif /* CONFIG_BPF_JIT */ 1577 1588 ··· 2540 2533 return select_interpreter; 2541 2534 } 2542 2535 2543 - /** 2544 - * bpf_prog_select_runtime - select exec runtime for BPF program 2545 - * @fp: bpf_prog populated with BPF program 2546 - * @err: pointer to error variable 2547 - * 2548 - * Try to JIT eBPF program, if JIT is not available, use interpreter. 2549 - * The BPF program will be executed via bpf_prog_run() function. 2550 - * 2551 - * Return: the &fp argument along with &err set to 0 for success or 2552 - * a negative errno code on failure 2553 - */ 2554 - struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) 2536 + static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 2537 + { 2538 + #ifdef CONFIG_BPF_JIT 2539 + struct bpf_prog *orig_prog; 2540 + struct bpf_insn_aux_data *orig_insn_aux; 2541 + 2542 + if (!bpf_prog_need_blind(prog)) 2543 + return bpf_int_jit_compile(env, prog); 2544 + 2545 + if (env) { 2546 + /* 2547 + * If env is not NULL, we are called from the end of bpf_check(), at this 2548 + * point, only insn_aux_data is used after failure, so it should be restored 2549 + * on failure. 2550 + */ 2551 + orig_insn_aux = bpf_dup_insn_aux_data(env); 2552 + if (!orig_insn_aux) 2553 + return prog; 2554 + } 2555 + 2556 + orig_prog = prog; 2557 + prog = bpf_jit_blind_constants(env, prog); 2558 + /* 2559 + * If blinding was requested and we failed during blinding, we must fall 2560 + * back to the interpreter. 2561 + */ 2562 + if (IS_ERR(prog)) 2563 + goto out_restore; 2564 + 2565 + prog = bpf_int_jit_compile(env, prog); 2566 + if (prog->jited) { 2567 + bpf_jit_prog_release_other(prog, orig_prog); 2568 + if (env) 2569 + vfree(orig_insn_aux); 2570 + return prog; 2571 + } 2572 + 2573 + bpf_jit_prog_release_other(orig_prog, prog); 2574 + 2575 + out_restore: 2576 + prog = orig_prog; 2577 + if (env) 2578 + bpf_restore_insn_aux_data(env, orig_insn_aux); 2579 + #endif 2580 + return prog; 2581 + } 2582 + 2583 + struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp, 2584 + int *err) 2555 2585 { 2556 2586 /* In case of BPF to BPF calls, verifier did all the prep 2557 2587 * work with regards to JITing, etc. ··· 2616 2572 if (*err) 2617 2573 return fp; 2618 2574 2619 - fp = bpf_int_jit_compile(fp); 2575 + fp = bpf_prog_jit_compile(env, fp); 2620 2576 bpf_prog_jit_attempt_done(fp); 2621 2577 if (!fp->jited && jit_needed) { 2622 2578 *err = -ENOTSUPP; ··· 2641 2597 *err = bpf_check_tail_call(fp); 2642 2598 2643 2599 return fp; 2600 + } 2601 + 2602 + /** 2603 + * bpf_prog_select_runtime - select exec runtime for BPF program 2604 + * @fp: bpf_prog populated with BPF program 2605 + * @err: pointer to error variable 2606 + * 2607 + * Try to JIT eBPF program, if JIT is not available, use interpreter. 2608 + * The BPF program will be executed via bpf_prog_run() function. 2609 + * 2610 + * Return: the &fp argument along with &err set to 0 for success or 2611 + * a negative errno code on failure 2612 + */ 2613 + struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) 2614 + { 2615 + return __bpf_prog_select_runtime(NULL, fp, err); 2644 2616 } 2645 2617 EXPORT_SYMBOL_GPL(bpf_prog_select_runtime); 2646 2618 ··· 3145 3085 * It is encouraged to implement bpf_int_jit_compile() instead, so that 3146 3086 * eBPF and implicitly also cBPF can get JITed! 3147 3087 */ 3148 - struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog) 3088 + struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 3149 3089 { 3150 3090 return prog; 3151 3091 }

+138 -25

kernel/bpf/fixups.c

··· 183 183 data[i].seen = old_seen; 184 184 data[i].zext_dst = insn_has_def32(insn + i); 185 185 } 186 + 187 + /* 188 + * The indirect_target flag of the original instruction was moved to the last of the 189 + * new instructions by the above memmove and memset, but the indirect jump target is 190 + * actually the first instruction, so move it back. This also matches with the behavior 191 + * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new 192 + * instruction. 193 + */ 194 + if (data[off + cnt - 1].indirect_target) { 195 + data[off].indirect_target = 1; 196 + data[off + cnt - 1].indirect_target = 0; 197 + } 186 198 } 187 199 188 200 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) ··· 244 232 } 245 233 } 246 234 247 - static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 248 - const struct bpf_insn *patch, u32 len) 235 + struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 236 + const struct bpf_insn *patch, u32 len) 249 237 { 250 238 struct bpf_prog *new_prog; 251 239 struct bpf_insn_aux_data *new_data = NULL; ··· 985 973 return 0; 986 974 } 987 975 988 - int bpf_jit_subprogs(struct bpf_verifier_env *env) 976 + static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env) 977 + { 978 + u32 *starts = NULL; 979 + 980 + starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT); 981 + if (starts) { 982 + for (int i = 0; i < env->subprog_cnt; i++) 983 + starts[i] = env->subprog_info[i].start; 984 + } 985 + return starts; 986 + } 987 + 988 + static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts) 989 + { 990 + for (int i = 0; i < env->subprog_cnt; i++) 991 + env->subprog_info[i].start = orig_starts[i]; 992 + /* restore the start of fake 'exit' subprog as well */ 993 + env->subprog_info[env->subprog_cnt].start = env->prog->len; 994 + } 995 + 996 + struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) 997 + { 998 + size_t size; 999 + void *new_aux; 1000 + 1001 + size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len); 1002 + new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT); 1003 + if (new_aux) 1004 + memcpy(new_aux, env->insn_aux_data, size); 1005 + return new_aux; 1006 + } 1007 + 1008 + void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, 1009 + struct bpf_insn_aux_data *orig_insn_aux) 1010 + { 1011 + /* the expanded elements are zero-filled, so no special handling is required */ 1012 + vfree(env->insn_aux_data); 1013 + env->insn_aux_data = orig_insn_aux; 1014 + } 1015 + 1016 + static int jit_subprogs(struct bpf_verifier_env *env) 989 1017 { 990 1018 struct bpf_prog *prog = env->prog, **func, *tmp; 991 1019 int i, j, subprog_start, subprog_end = 0, len, subprog; ··· 1033 981 struct bpf_insn *insn; 1034 982 void *old_bpf_func; 1035 983 int err, num_exentries; 1036 - int old_len, subprog_start_adjustment = 0; 1037 - 1038 - if (env->subprog_cnt <= 1) 1039 - return 0; 1040 984 1041 985 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1042 986 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) ··· 1101 1053 goto out_free; 1102 1054 func[i]->is_func = 1; 1103 1055 func[i]->sleepable = prog->sleepable; 1056 + func[i]->blinded = prog->blinded; 1104 1057 func[i]->aux->func_idx = i; 1105 1058 /* Below members will be freed only at prog->aux */ 1106 1059 func[i]->aux->btf = prog->aux->btf; 1107 - func[i]->aux->subprog_start = subprog_start + subprog_start_adjustment; 1060 + func[i]->aux->subprog_start = subprog_start; 1108 1061 func[i]->aux->func_info = prog->aux->func_info; 1109 1062 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 1110 1063 func[i]->aux->poke_tab = prog->aux->poke_tab; ··· 1159 1110 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; 1160 1111 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; 1161 1112 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; 1113 + func[i]->aux->token = prog->aux->token; 1162 1114 if (!i) 1163 1115 func[i]->aux->exception_boundary = env->seen_exception; 1164 - 1165 - /* 1166 - * To properly pass the absolute subprog start to jit 1167 - * all instruction adjustments should be accumulated 1168 - */ 1169 - old_len = func[i]->len; 1170 - func[i] = bpf_int_jit_compile(func[i]); 1171 - subprog_start_adjustment += func[i]->len - old_len; 1172 - 1116 + func[i] = bpf_int_jit_compile(env, func[i]); 1173 1117 if (!func[i]->jited) { 1174 1118 err = -ENOTSUPP; 1175 1119 goto out_free; ··· 1206 1164 } 1207 1165 for (i = 0; i < env->subprog_cnt; i++) { 1208 1166 old_bpf_func = func[i]->bpf_func; 1209 - tmp = bpf_int_jit_compile(func[i]); 1167 + tmp = bpf_int_jit_compile(env, func[i]); 1210 1168 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 1211 1169 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 1212 1170 err = -ENOTSUPP; ··· 1288 1246 } 1289 1247 kfree(func); 1290 1248 out_undo_insn: 1249 + bpf_prog_jit_attempt_done(prog); 1250 + return err; 1251 + } 1252 + 1253 + int bpf_jit_subprogs(struct bpf_verifier_env *env) 1254 + { 1255 + int err, i; 1256 + bool blinded = false; 1257 + struct bpf_insn *insn; 1258 + struct bpf_prog *prog, *orig_prog; 1259 + struct bpf_insn_aux_data *orig_insn_aux; 1260 + u32 *orig_subprog_starts; 1261 + 1262 + if (env->subprog_cnt <= 1) 1263 + return 0; 1264 + 1265 + prog = orig_prog = env->prog; 1266 + if (bpf_prog_need_blind(prog)) { 1267 + orig_insn_aux = bpf_dup_insn_aux_data(env); 1268 + if (!orig_insn_aux) { 1269 + err = -ENOMEM; 1270 + goto out_cleanup; 1271 + } 1272 + orig_subprog_starts = bpf_dup_subprog_starts(env); 1273 + if (!orig_subprog_starts) { 1274 + vfree(orig_insn_aux); 1275 + err = -ENOMEM; 1276 + goto out_cleanup; 1277 + } 1278 + prog = bpf_jit_blind_constants(env, prog); 1279 + if (IS_ERR(prog)) { 1280 + err = -ENOMEM; 1281 + prog = orig_prog; 1282 + goto out_restore; 1283 + } 1284 + blinded = true; 1285 + } 1286 + 1287 + err = jit_subprogs(env); 1288 + if (err) 1289 + goto out_jit_err; 1290 + 1291 + if (blinded) { 1292 + bpf_jit_prog_release_other(prog, orig_prog); 1293 + kvfree(orig_subprog_starts); 1294 + vfree(orig_insn_aux); 1295 + } 1296 + 1297 + return 0; 1298 + 1299 + out_jit_err: 1300 + if (blinded) { 1301 + bpf_jit_prog_release_other(orig_prog, prog); 1302 + /* roll back to the clean original prog */ 1303 + prog = env->prog = orig_prog; 1304 + goto out_restore; 1305 + } else { 1306 + if (err != -EFAULT) { 1307 + /* 1308 + * We will fall back to interpreter mode when err is not -EFAULT, before 1309 + * that, insn->off and insn->imm should be restored to their original 1310 + * values since they were modified by jit_subprogs. 1311 + */ 1312 + for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1313 + if (!bpf_pseudo_call(insn)) 1314 + continue; 1315 + insn->off = 0; 1316 + insn->imm = env->insn_aux_data[i].call_imm; 1317 + } 1318 + } 1319 + goto out_cleanup; 1320 + } 1321 + 1322 + out_restore: 1323 + bpf_restore_subprog_starts(env, orig_subprog_starts); 1324 + bpf_restore_insn_aux_data(env, orig_insn_aux); 1325 + kvfree(orig_subprog_starts); 1326 + out_cleanup: 1291 1327 /* cleanup main prog to be interpreted */ 1292 1328 prog->jit_requested = 0; 1293 1329 prog->blinding_requested = 0; 1294 - for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 1295 - if (!bpf_pseudo_call(insn)) 1296 - continue; 1297 - insn->off = 0; 1298 - insn->imm = env->insn_aux_data[i].call_imm; 1299 - } 1300 - bpf_prog_jit_attempt_done(prog); 1301 1330 return err; 1302 1331 } 1303 1332

+16 -1

kernel/bpf/helpers.c

··· 845 845 data->buf = buffers->buf; 846 846 847 847 for (i = 0; i < fmt_size; i++) { 848 - if ((!isprint(fmt[i]) && !isspace(fmt[i])) || !isascii(fmt[i])) { 848 + unsigned char c = fmt[i]; 849 + 850 + /* 851 + * Permit bytes >= 0x80 in plain text so UTF-8 literals can pass 852 + * through unchanged, while still rejecting ASCII control bytes. 853 + */ 854 + if (isascii(c) && !isprint(c) && !isspace(c)) { 849 855 err = -EINVAL; 850 856 goto out; 851 857 } ··· 873 867 * always access fmt[i + 1], in the worst case it will be a 0 874 868 */ 875 869 i++; 870 + c = fmt[i]; 871 + /* 872 + * The format parser below only understands ASCII conversion 873 + * specifiers and modifiers, so reject non-ASCII after '%'. 874 + */ 875 + if (!isascii(c)) { 876 + err = -EINVAL; 877 + goto out; 878 + } 876 879 877 880 /* skip optional "[0 +-][num]" width formatting field */ 878 881 while (fmt[i] == '0' || fmt[i] == '+' || fmt[i] == '-' ||

+61 -51

kernel/bpf/liveness.c

··· 574 574 * 575 575 * precise {frame=N, off=V} -- known absolute frame index and byte offset 576 576 * | 577 - * offset-imprecise {frame=N, off=OFF_IMPRECISE} 577 + * offset-imprecise {frame=N, cnt=0} 578 578 * | -- known frame identity, unknown offset 579 579 * fully-imprecise {frame=ARG_IMPRECISE, mask=bitmask} 580 580 * -- unknown frame identity; .mask is a ··· 607 607 ARG_IMPRECISE = -3, /* lost identity; .mask is arg bitmask */ 608 608 }; 609 609 610 - #define OFF_IMPRECISE S16_MIN /* arg identity known but offset unknown */ 611 - 612 610 /* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */ 613 611 #define MAX_ARG_SPILL_SLOTS 64 614 612 ··· 618 620 static bool arg_is_fp(const struct arg_track *at) 619 621 { 620 622 return at->frame >= 0 || at->frame == ARG_IMPRECISE; 621 - } 622 - 623 - /* 624 - * Clear all tracked callee stack slots overlapping the byte range 625 - * [off, off+sz-1] where off is a negative FP-relative offset. 626 - */ 627 - static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz) 628 - { 629 - struct arg_track none = { .frame = ARG_NONE }; 630 - 631 - if (off == OFF_IMPRECISE) { 632 - for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) 633 - at_stack[i] = none; 634 - return; 635 - } 636 - for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) { 637 - int slot_start = -((i + 1) * 8); 638 - int slot_end = slot_start + 8; 639 - 640 - if (slot_start < off + (int)sz && slot_end > off) 641 - at_stack[i] = none; 642 - } 643 623 } 644 624 645 625 static void verbose_arg_track(struct bpf_verifier_env *env, struct arg_track *at) ··· 839 863 *dst = arg_join_imprecise(*dst, *src); 840 864 } 841 865 842 - static s16 arg_add(s16 off, s64 delta) 866 + static bool arg_add(s16 off, s64 delta, s16 *out) 843 867 { 844 - s64 res; 868 + s16 d = delta; 845 869 846 - if (off == OFF_IMPRECISE) 847 - return OFF_IMPRECISE; 848 - res = (s64)off + delta; 849 - if (res < S16_MIN + 1 || res > S16_MAX) 850 - return OFF_IMPRECISE; 851 - return res; 870 + if (d != delta) 871 + return true; 872 + return check_add_overflow(off, d, out); 852 873 } 853 874 854 875 static void arg_padd(struct arg_track *at, s64 delta) ··· 855 882 if (at->off_cnt == 0) 856 883 return; 857 884 for (i = 0; i < at->off_cnt; i++) { 858 - s16 new_off = arg_add(at->off[i], delta); 885 + s16 new_off; 859 886 860 - if (new_off == OFF_IMPRECISE) { 887 + if (arg_add(at->off[i], delta, &new_off)) { 861 888 at->off_cnt = 0; 862 889 return; 863 890 } ··· 872 899 */ 873 900 static int fp_off_to_slot(s16 off) 874 901 { 875 - if (off == OFF_IMPRECISE) 876 - return -1; 877 902 if (off >= 0 || off < -(int)(MAX_ARG_SPILL_SLOTS * 8)) 878 903 return -1; 879 904 if (off % 8) ··· 901 930 return imp; 902 931 903 932 for (i = 0; i < cnt; i++) { 904 - s16 fp_off = arg_add(at_out[reg].off[i], insn->off); 905 - int slot = fp_off_to_slot(fp_off); 933 + s16 fp_off, slot; 906 934 935 + if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) 936 + return imp; 937 + slot = fp_off_to_slot(fp_off); 907 938 if (slot < 0) 908 939 return imp; 909 940 result = __arg_track_join(result, at_stack_out[slot]); ··· 941 968 return; 942 969 } 943 970 for (i = 0; i < cnt; i++) { 944 - s16 fp_off = arg_add(at_out[reg].off[i], insn->off); 945 - int slot = fp_off_to_slot(fp_off); 971 + s16 fp_off; 972 + int slot; 946 973 974 + if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) 975 + continue; 976 + slot = fp_off_to_slot(fp_off); 947 977 if (slot < 0) 948 978 continue; 949 979 if (cnt == 1) 950 980 at_stack_out[slot] = new_val; 951 981 else 952 982 at_stack_out[slot] = __arg_track_join(at_stack_out[slot], new_val); 983 + } 984 + } 985 + 986 + /* 987 + * Clear all tracked callee stack slots overlapping the byte range 988 + * [off, off+sz-1] where off is a negative FP-relative offset. 989 + */ 990 + static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz, int cnt) 991 + { 992 + struct arg_track none = { .frame = ARG_NONE }; 993 + 994 + if (cnt == 0) { 995 + for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) 996 + at_stack[i] = __arg_track_join(at_stack[i], none); 997 + return; 998 + } 999 + for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) { 1000 + int slot_start = -((i + 1) * 8); 1001 + int slot_end = slot_start + 8; 1002 + 1003 + if (slot_start < off + (int)sz && slot_end > off) { 1004 + if (cnt == 1) 1005 + at_stack[i] = none; 1006 + else 1007 + at_stack[i] = __arg_track_join(at_stack[i], none); 1008 + } 953 1009 } 954 1010 } 955 1011 ··· 992 990 int cnt, i; 993 991 994 992 if (reg == BPF_REG_FP) { 995 - clear_overlapping_stack_slots(at_stack_out, insn->off, sz); 993 + clear_overlapping_stack_slots(at_stack_out, insn->off, sz, 1); 996 994 return; 997 995 } 998 996 cnt = at_out[reg].off_cnt; 999 997 if (cnt == 0) { 1000 - clear_overlapping_stack_slots(at_stack_out, OFF_IMPRECISE, sz); 998 + clear_overlapping_stack_slots(at_stack_out, 0, sz, cnt); 1001 999 return; 1002 1000 } 1003 1001 for (i = 0; i < cnt; i++) { 1004 - s16 fp_off = arg_add(at_out[reg].off[i], insn->off); 1002 + s16 fp_off; 1005 1003 1006 - clear_overlapping_stack_slots(at_stack_out, fp_off, sz); 1004 + if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) { 1005 + clear_overlapping_stack_slots(at_stack_out, 0, sz, 0); 1006 + break; 1007 + } 1008 + clear_overlapping_stack_slots(at_stack_out, fp_off, sz, cnt); 1007 1009 } 1008 1010 } 1009 1011 ··· 1046 1040 } 1047 1041 if (printed) 1048 1042 verbose(env, "\n"); 1043 + } 1044 + 1045 + static bool can_be_local_fp(int depth, int regno, struct arg_track *at) 1046 + { 1047 + return regno == BPF_REG_FP || at->frame == depth || 1048 + (at->frame == ARG_IMPRECISE && (at->mask & BIT(depth))); 1049 1049 } 1050 1050 1051 1051 /* ··· 1123 1111 at_out[r] = none; 1124 1112 } else if (class == BPF_LDX) { 1125 1113 u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); 1126 - bool src_is_local_fp = insn->src_reg == BPF_REG_FP || src->frame == depth || 1127 - (src->frame == ARG_IMPRECISE && (src->mask & BIT(depth))); 1114 + bool src_is_local_fp = can_be_local_fp(depth, insn->src_reg, src); 1128 1115 1129 1116 /* 1130 1117 * Reload from callee stack: if src is current-frame FP-derived ··· 1158 1147 bool dst_is_local_fp; 1159 1148 1160 1149 /* Track spills to current-frame FP-derived callee stack */ 1161 - dst_is_local_fp = insn->dst_reg == BPF_REG_FP || dst->frame == depth; 1150 + dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst); 1162 1151 if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM) 1163 1152 spill_to_stack(insn, at_out, insn->dst_reg, 1164 1153 at_stack_out, src, sz); ··· 1177 1166 } 1178 1167 } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) { 1179 1168 u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); 1180 - bool dst_is_local_fp = insn->dst_reg == BPF_REG_FP || dst->frame == depth; 1169 + bool dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst); 1181 1170 1182 1171 /* BPF_ST to FP-derived dst: clear overlapping stack slots */ 1183 1172 if (dst_is_local_fp) ··· 1327 1316 resolved.off_cnt = ptr->off_cnt; 1328 1317 resolved.frame = ptr->frame; 1329 1318 for (oi = 0; oi < ptr->off_cnt; oi++) { 1330 - resolved.off[oi] = arg_add(ptr->off[oi], insn->off); 1331 - if (resolved.off[oi] == OFF_IMPRECISE) { 1319 + if (arg_add(ptr->off[oi], insn->off, &resolved.off[oi])) { 1332 1320 resolved.off_cnt = 0; 1333 1321 break; 1334 1322 }

-4

kernel/bpf/syscall.c

··· 3083 3083 if (err < 0) 3084 3084 goto free_used_maps; 3085 3085 3086 - prog = bpf_prog_select_runtime(prog, &err); 3087 - if (err < 0) 3088 - goto free_used_maps; 3089 - 3090 3086 err = bpf_prog_mark_insn_arrays_ready(prog); 3091 3087 if (err < 0) 3092 3088 goto free_used_maps;

+19 -7

kernel/bpf/verifier.c

··· 3497 3497 return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno; 3498 3498 } 3499 3499 3500 + static void mark_indirect_target(struct bpf_verifier_env *env, int idx) 3501 + { 3502 + env->insn_aux_data[idx].indirect_target = true; 3503 + } 3504 + 3500 3505 #define LR_FRAMENO_BITS 3 3501 3506 #define LR_SPI_BITS 6 3502 3507 #define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) ··· 4549 4544 int perm_flags; 4550 4545 const char *reg_name = ""; 4551 4546 4547 + if (base_type(reg->type) != PTR_TO_BTF_ID) 4548 + goto bad_type; 4549 + 4552 4550 if (btf_is_kernel(reg->btf)) { 4553 4551 perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU; 4554 4552 ··· 4564 4556 perm_flags |= MEM_PERCPU; 4565 4557 } 4566 4558 4567 - if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags)) 4559 + if (type_flag(reg->type) & ~perm_flags) 4568 4560 goto bad_type; 4569 4561 4570 4562 /* We need to verify reg->type and reg->btf, before accessing reg->btf */ ··· 17553 17545 } 17554 17546 17555 17547 for (i = 0; i < n - 1; i++) { 17548 + mark_indirect_target(env, env->gotox_tmp_buf->items[i]); 17556 17549 other_branch = push_stack(env, env->gotox_tmp_buf->items[i], 17557 17550 env->insn_idx, env->cur_state->speculative); 17558 17551 if (IS_ERR(other_branch)) 17559 17552 return PTR_ERR(other_branch); 17560 17553 } 17561 17554 env->insn_idx = env->gotox_tmp_buf->items[n-1]; 17555 + mark_indirect_target(env, env->insn_idx); 17562 17556 return INSN_IDX_UPDATED; 17563 17557 } 17564 17558 ··· 20165 20155 20166 20156 adjust_btf_func(env); 20167 20157 20158 + /* extension progs temporarily inherit the attach_type of their targets 20159 + for verification purposes, so set it back to zero before returning 20160 + */ 20161 + if (env->prog->type == BPF_PROG_TYPE_EXT) 20162 + env->prog->expected_attach_type = 0; 20163 + 20164 + env->prog = __bpf_prog_select_runtime(env, env->prog, &ret); 20165 + 20168 20166 err_release_maps: 20169 20167 if (ret) 20170 20168 release_insn_arrays(env); ··· 20183 20165 release_maps(env); 20184 20166 if (!env->prog->aux->used_btfs) 20185 20167 release_btfs(env); 20186 - 20187 - /* extension progs temporarily inherit the attach_type of their targets 20188 - for verification purposes, so set it back to zero before returning 20189 - */ 20190 - if (env->prog->type == BPF_PROG_TYPE_EXT) 20191 - env->prog->expected_attach_type = 0; 20192 20168 20193 20169 *prog = env->prog; 20194 20170

+1 -1

net/core/filter.c

··· 503 503 ((unaligned_ok && offset >= 0) || 504 504 (!unaligned_ok && offset >= 0 && 505 505 offset + ip_align >= 0 && 506 - offset + ip_align % size == 0))) { 506 + (offset + ip_align) % size == 0))) { 507 507 bool ldx_off_ok = offset <= S16_MAX; 508 508 509 509 *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);

+2 -2

net/core/sock_map.c

··· 530 530 if (sk_is_tcp(sk)) 531 531 return sk->sk_state != TCP_LISTEN; 532 532 else 533 - return sk->sk_state == TCP_ESTABLISHED; 533 + return READ_ONCE(sk->sk_state) == TCP_ESTABLISHED; 534 534 } 535 535 536 536 static bool sock_map_sk_is_suitable(const struct sock *sk) ··· 543 543 if (sk_is_tcp(sk)) 544 544 return (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_LISTEN); 545 545 if (sk_is_stream_unix(sk)) 546 - return (1 << sk->sk_state) & TCPF_ESTABLISHED; 546 + return (1 << READ_ONCE(sk->sk_state)) & TCPF_ESTABLISHED; 547 547 if (sk_is_vsock(sk) && 548 548 (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET)) 549 549 return (1 << sk->sk_state) & TCPF_ESTABLISHED;

+5 -4

net/unix/af_unix.c

··· 3735 3735 struct bpf_prog *prog; 3736 3736 struct sock *sk = v; 3737 3737 uid_t uid; 3738 - bool slow; 3739 3738 int ret; 3740 3739 3741 3740 if (v == SEQ_START_TOKEN) 3742 3741 return 0; 3743 3742 3744 - slow = lock_sock_fast(sk); 3743 + lock_sock(sk); 3744 + unix_state_lock(sk); 3745 3745 3746 - if (unlikely(sk_unhashed(sk))) { 3746 + if (unlikely(sock_flag(sk, SOCK_DEAD))) { 3747 3747 ret = SEQ_SKIP; 3748 3748 goto unlock; 3749 3749 } ··· 3753 3753 prog = bpf_iter_get_info(&meta, false); 3754 3754 ret = unix_prog_seq_show(prog, &meta, v, uid); 3755 3755 unlock: 3756 - unlock_sock_fast(sk, slow); 3756 + unix_state_unlock(sk); 3757 + release_sock(sk); 3757 3758 return ret; 3758 3759 } 3759 3760

+3

net/unix/unix_bpf.c

··· 185 185 */ 186 186 if (!psock->sk_pair) { 187 187 sk_pair = unix_peer(sk); 188 + if (unlikely(!sk_pair)) 189 + return -EINVAL; 190 + 188 191 sock_hold(sk_pair); 189 192 psock->sk_pair = sk_pair; 190 193 }

+12 -11

tools/lib/bpf/libbpf.c

··· 5852 5852 info.name = ptr_to_u64(name); 5853 5853 info.name_len = sizeof(name); 5854 5854 5855 + btf = NULL; 5855 5856 err = bpf_btf_get_info_by_fd(fd, &info, &len); 5856 5857 if (err) { 5857 5858 err = -errno; 5858 5859 pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err)); 5859 - goto err_out; 5860 + break; 5860 5861 } 5861 5862 5862 5863 /* ignore non-module BTFs */ ··· 5871 5870 if (err) { 5872 5871 pr_warn("failed to load module [%s]'s BTF object #%d: %s\n", 5873 5872 name, id, errstr(err)); 5874 - goto err_out; 5873 + break; 5875 5874 } 5876 5875 5877 5876 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, 5878 5877 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); 5879 5878 if (err) 5880 - goto err_out; 5879 + break; 5881 5880 5882 - mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; 5881 + mod_btf = &obj->btf_modules[obj->btf_module_cnt]; 5883 5882 5884 5883 mod_btf->btf = btf; 5885 5884 mod_btf->id = id; ··· 5887 5886 mod_btf->name = strdup(name); 5888 5887 if (!mod_btf->name) { 5889 5888 err = -ENOMEM; 5890 - goto err_out; 5889 + break; 5891 5890 } 5892 - continue; 5893 - 5894 - err_out: 5895 - close(fd); 5896 - return err; 5891 + obj->btf_module_cnt++; 5897 5892 } 5898 5893 5899 - return 0; 5894 + if (err) { 5895 + btf__free(btf); 5896 + close(fd); 5897 + } 5898 + return err; 5900 5899 } 5901 5900 5902 5901 static struct bpf_core_cand_list *

+1

tools/testing/selftests/bpf/Makefile

··· 751 751 btf_helpers.c \ 752 752 cap_helpers.c \ 753 753 unpriv_helpers.c \ 754 + sysctl_helpers.c \ 754 755 netlink_helpers.c \ 755 756 jit_disasm_helpers.c \ 756 757 io_helpers.c \

+2 -1

tools/testing/selftests/bpf/prog_tests/snprintf.c

··· 114 114 ASSERT_ERR(load_single_snprintf("%--------"), "invalid specifier 5"); 115 115 ASSERT_ERR(load_single_snprintf("%lc"), "invalid specifier 6"); 116 116 ASSERT_ERR(load_single_snprintf("%llc"), "invalid specifier 7"); 117 - ASSERT_ERR(load_single_snprintf("\x80"), "non ascii character"); 117 + ASSERT_OK(load_single_snprintf("\x80"), "non ascii plain text"); 118 + ASSERT_ERR(load_single_snprintf("%\x80"), "non ascii in specifier"); 118 119 ASSERT_ERR(load_single_snprintf("\x1"), "non printable character"); 119 120 ASSERT_ERR(load_single_snprintf("%p%"), "invalid specifier 8"); 120 121 ASSERT_ERR(load_single_snprintf("%s%"), "invalid specifier 9");

+10 -3

tools/testing/selftests/bpf/prog_tests/task_local_data.h

··· 99 99 struct tld_metadata metadata[]; 100 100 }; 101 101 102 + /* 103 + * The unused field ensures map_val.start > 0. On the BPF side, __tld_fetch_key() 104 + * calculates off by summing map_val.start and tld_key_t.off and treats off == 0 105 + * as key not cached. 106 + */ 102 107 struct tld_data_u { 103 - __u64 start; /* offset of tld_data_u->data in a page */ 108 + __u64 unused; 104 109 char data[] __attribute__((aligned(8))); 105 110 }; 106 111 107 112 struct tld_map_value { 108 113 void *data; 109 114 struct tld_meta_u *meta; 115 + __u16 start; /* offset of tld_data_u->data in a page */ 110 116 }; 111 117 112 118 struct tld_meta_u * _Atomic tld_meta_p __attribute__((weak)); ··· 188 182 * is a page in BTF. 189 183 */ 190 184 map_val.data = (void *)(TLD_PAGE_MASK & (intptr_t)data); 191 - data->start = (~TLD_PAGE_MASK & (intptr_t)data) + sizeof(struct tld_data_u); 185 + map_val.start = (~TLD_PAGE_MASK & (intptr_t)data) + sizeof(struct tld_data_u); 192 186 map_val.meta = tld_meta_p; 193 187 194 188 err = bpf_map_update_elem(map_fd, &tid_fd, &map_val, 0); ··· 247 241 * TLD_DYN_DATA_SIZE is allocated for tld_create_key() 248 242 */ 249 243 if (dyn_data) { 250 - if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size) 244 + if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size || 245 + tld_meta_p->size > TLD_PAGE_SIZE - sizeof(struct tld_data_u)) 251 246 return (tld_key_t){-E2BIG}; 252 247 } else { 253 248 if (off + TLD_ROUND_UP(size, 8) > TLD_PAGE_SIZE - sizeof(struct tld_data_u))

+87 -9

tools/testing/selftests/bpf/prog_tests/test_task_local_data.c

··· 3 3 #include <bpf/btf.h> 4 4 #include <test_progs.h> 5 5 6 + /* 7 + * Only a page is pinned to kernel, so the maximum amount of dynamic data 8 + * allowed is page_size - sizeof(struct tld_data_u) - static TLD fields. 9 + */ 10 + #define TLD_DYN_DATA_SIZE_MAX (getpagesize() - sizeof(struct tld_data_u) - 8) 11 + 6 12 #define TLD_FREE_DATA_ON_THREAD_EXIT 7 - #define TLD_DYN_DATA_SIZE (getpagesize() - 8) 13 + #define TLD_DYN_DATA_SIZE TLD_DYN_DATA_SIZE_MAX 8 14 #include "task_local_data.h" 9 15 10 16 struct test_tld_struct { ··· 30 24 * sequentially. Users of task local data library should not touch 31 25 * library internal. 32 26 */ 33 - static void reset_tld(void) 27 + static void reset_tld(__u16 dyn_data_size) 34 28 { 35 29 if (tld_meta_p) { 36 30 /* Remove TLDs created by tld_create_key() */ 37 31 tld_meta_p->cnt = 1; 38 - tld_meta_p->size = TLD_DYN_DATA_SIZE; 32 + tld_meta_p->size = dyn_data_size + 8; 39 33 memset(&tld_meta_p->metadata[1], 0, 40 34 (TLD_MAX_DATA_CNT - 1) * sizeof(struct tld_metadata)); 41 35 } ··· 133 127 tld_key_t key; 134 128 int i, err; 135 129 136 - reset_tld(); 130 + reset_tld(TLD_DYN_DATA_SIZE_MAX); 137 131 138 132 ASSERT_OK(pthread_mutex_init(&global_mutex, NULL), "pthread_mutex_init"); 139 133 ··· 153 147 154 148 /* 155 149 * Shouldn't be able to store data exceed a page. Create a TLD just big 156 - * enough to exceed a page. TLDs already created are int value0, int 157 - * value1, and struct test_tld_struct value2. 150 + * enough to exceed a page. Data already contains struct tld_data_u, 151 + * value0 and value1 of int type, and value 2 of struct test_tld_struct. 158 152 */ 159 - key = tld_create_key("value_not_exist", 160 - TLD_PAGE_SIZE - 2 * sizeof(int) - sizeof(struct test_tld_struct) + 1); 153 + key = tld_create_key("value_not_exist", TLD_PAGE_SIZE + 1 - 154 + sizeof(struct tld_data_u) - 155 + TLD_ROUND_UP(sizeof(int), 8) * 2 - 156 + TLD_ROUND_UP(sizeof(struct test_tld_struct), 8)); 161 157 ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key"); 162 158 163 159 key = tld_create_key("value2", sizeof(struct test_tld_struct)); ··· 247 239 tld_keys[0] = value0_key; 248 240 249 241 for (j = 0; j < 100; j++) { 250 - reset_tld(); 242 + reset_tld(TLD_DYN_DATA_SIZE_MAX); 251 243 252 244 for (i = 0; i < TEST_RACE_THREAD_NUM; i++) { 253 245 /* ··· 296 288 test_task_local_data__destroy(skel); 297 289 } 298 290 291 + static void test_task_local_data_dyn_size(__u16 dyn_data_size) 292 + { 293 + LIBBPF_OPTS(bpf_test_run_opts, opts); 294 + struct test_task_local_data *skel; 295 + int max_keys, i, err, fd, *data; 296 + char name[TLD_NAME_LEN]; 297 + tld_key_t key; 298 + 299 + reset_tld(dyn_data_size); 300 + 301 + skel = test_task_local_data__open_and_load(); 302 + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) 303 + return; 304 + 305 + tld_keys = calloc(TLD_MAX_DATA_CNT, sizeof(tld_key_t)); 306 + if (!ASSERT_OK_PTR(tld_keys, "calloc tld_keys")) 307 + goto out; 308 + 309 + fd = bpf_map__fd(skel->maps.tld_data_map); 310 + 311 + /* Create as many int-sized TLDs as the dynamic data size allows */ 312 + max_keys = dyn_data_size / TLD_ROUND_UP(sizeof(int), 8); 313 + for (i = 0; i < max_keys; i++) { 314 + snprintf(name, TLD_NAME_LEN, "value_%d", i); 315 + tld_keys[i] = tld_create_key(name, sizeof(int)); 316 + if (!ASSERT_FALSE(tld_key_is_err(tld_keys[i]), "tld_create_key")) 317 + goto out; 318 + 319 + data = tld_get_data(fd, tld_keys[i]); 320 + if (!ASSERT_OK_PTR(data, "tld_get_data")) 321 + goto out; 322 + *data = i; 323 + } 324 + 325 + /* The next key should fail with E2BIG */ 326 + key = tld_create_key("overflow", sizeof(int)); 327 + ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key overflow"); 328 + 329 + /* Verify data for value_i do not overlap */ 330 + for (i = 0; i < max_keys; i++) { 331 + data = tld_get_data(fd, tld_keys[i]); 332 + if (!ASSERT_OK_PTR(data, "tld_get_data")) 333 + goto out; 334 + 335 + ASSERT_EQ(*data, i, "tld_get_data value_i"); 336 + } 337 + 338 + /* Verify BPF side can still read the static key */ 339 + data = tld_get_data(fd, value0_key); 340 + if (!ASSERT_OK_PTR(data, "tld_get_data value0")) 341 + goto out; 342 + *data = 0xdeadbeef; 343 + 344 + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.task_main), &opts); 345 + ASSERT_OK(err, "run task_main"); 346 + ASSERT_EQ(skel->bss->test_value0, 0xdeadbeef, "tld_get_data value0"); 347 + 348 + out: 349 + if (tld_keys) { 350 + free(tld_keys); 351 + tld_keys = NULL; 352 + } 353 + tld_free(); 354 + test_task_local_data__destroy(skel); 355 + } 356 + 299 357 void test_task_local_data(void) 300 358 { 301 359 if (test__start_subtest("task_local_data_basic")) 302 360 test_task_local_data_basic(); 303 361 if (test__start_subtest("task_local_data_race")) 304 362 test_task_local_data_race(); 363 + if (test__start_subtest("task_local_data_dyn_size_small")) 364 + test_task_local_data_dyn_size(64); 365 + if (test__start_subtest("task_local_data_dyn_size_zero")) 366 + test_task_local_data_dyn_size(0); 305 367 }

+83 -3

tools/testing/selftests/bpf/prog_tests/token.c

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 /* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ 3 3 #define _GNU_SOURCE 4 - #include <test_progs.h> 5 4 #include <bpf/btf.h> 6 - #include "cap_helpers.h" 7 5 #include <fcntl.h> 8 6 #include <sched.h> 9 7 #include <signal.h> ··· 13 15 #include <sys/stat.h> 14 16 #include <sys/syscall.h> 15 17 #include <sys/un.h> 18 + 19 + #include "bpf_util.h" 20 + #include "cap_helpers.h" 21 + #include "sysctl_helpers.h" 22 + #include "test_progs.h" 23 + #include "trace_helpers.h" 24 + 16 25 #include "priv_map.skel.h" 17 26 #include "priv_prog.skel.h" 18 27 #include "dummy_st_ops_success.skel.h" 28 + #include "token_kallsyms.skel.h" 19 29 #include "token_lsm.skel.h" 20 30 #include "priv_freplace_prog.skel.h" 21 31 ··· 1051 1045 return -EINVAL; 1052 1046 } 1053 1047 1048 + static bool kallsyms_has_bpf_func(struct ksyms *ksyms, const char *func_name) 1049 + { 1050 + char name[256]; 1051 + int i; 1052 + 1053 + for (i = 0; i < ksyms->sym_cnt; i++) { 1054 + if (sscanf(ksyms->syms[i].name, "bpf_prog_%*[^_]_%255s", name) == 1 && 1055 + strcmp(name, func_name) == 0) 1056 + return true; 1057 + } 1058 + return false; 1059 + } 1060 + 1061 + static int userns_obj_priv_prog_kallsyms(int mnt_fd, struct token_lsm *lsm_skel) 1062 + { 1063 + const char *func_names[] = { "xdp_main", "token_ksym_subprog" }; 1064 + LIBBPF_OPTS(bpf_object_open_opts, opts); 1065 + struct token_kallsyms *skel; 1066 + struct ksyms *ksyms = NULL; 1067 + char buf[256]; 1068 + int i, err; 1069 + 1070 + snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd); 1071 + opts.bpf_token_path = buf; 1072 + skel = token_kallsyms__open_opts(&opts); 1073 + if (!ASSERT_OK_PTR(skel, "token_kallsyms__open_opts")) 1074 + return -EINVAL; 1075 + 1076 + err = token_kallsyms__load(skel); 1077 + if (!ASSERT_OK(err, "token_kallsyms__load")) 1078 + goto cleanup; 1079 + 1080 + ksyms = load_kallsyms_local(); 1081 + if (!ASSERT_OK_PTR(ksyms, "load_kallsyms_local")) { 1082 + err = -EINVAL; 1083 + goto cleanup; 1084 + } 1085 + 1086 + for (i = 0; i < ARRAY_SIZE(func_names); i++) { 1087 + if (!ASSERT_TRUE(kallsyms_has_bpf_func(ksyms, func_names[i]), 1088 + func_names[i])) { 1089 + err = -EINVAL; 1090 + break; 1091 + } 1092 + } 1093 + 1094 + cleanup: 1095 + free_kallsyms_local(ksyms); 1096 + token_kallsyms__destroy(skel); 1097 + return err; 1098 + } 1099 + 1054 1100 #define bit(n) (1ULL << (n)) 1055 1101 1056 1102 static int userns_bpf_token_info(int mnt_fd, struct token_lsm *lsm_skel) ··· 1140 1082 return err; 1141 1083 } 1142 1084 1143 - void test_token(void) 1085 + void serial_test_token(void) 1144 1086 { 1145 1087 if (test__start_subtest("map_token")) { 1146 1088 struct bpffs_opts opts = { ··· 1251 1193 }; 1252 1194 1253 1195 subtest_userns(&opts, userns_bpf_token_info); 1196 + } 1197 + if (test__start_subtest("obj_priv_prog_kallsyms")) { 1198 + char perf_paranoid_orig[32] = {}; 1199 + char kptr_restrict_orig[32] = {}; 1200 + struct bpffs_opts opts = { 1201 + .cmds = bit(BPF_BTF_LOAD) | bit(BPF_PROG_LOAD), 1202 + .progs = bit(BPF_PROG_TYPE_XDP), 1203 + .attachs = ~0ULL, 1204 + }; 1205 + 1206 + if (sysctl_set_or_fail("/proc/sys/kernel/perf_event_paranoid", perf_paranoid_orig, "0")) 1207 + goto cleanup; 1208 + if (sysctl_set_or_fail("/proc/sys/kernel/kptr_restrict", kptr_restrict_orig, "0")) 1209 + goto cleanup; 1210 + 1211 + subtest_userns(&opts, userns_obj_priv_prog_kallsyms); 1212 + 1213 + cleanup: 1214 + if (perf_paranoid_orig[0]) 1215 + sysctl_set_or_fail("/proc/sys/kernel/perf_event_paranoid", NULL, perf_paranoid_orig); 1216 + if (kptr_restrict_orig[0]) 1217 + sysctl_set_or_fail("/proc/sys/kernel/kptr_restrict", NULL, kptr_restrict_orig); 1254 1218 } 1255 1219 }

+22 -6

tools/testing/selftests/bpf/prog_tests/trace_printk.c

··· 6 6 #include "trace_printk.lskel.h" 7 7 8 8 #define SEARCHMSG "testing,testing" 9 + #define SEARCHMSG_UTF8 "中文,测试" 9 10 10 11 static void trace_pipe_cb(const char *str, void *data) 11 12 { 12 13 if (strstr(str, SEARCHMSG) != NULL) 13 - (*(int *)data)++; 14 + ((int *)data)[0]++; 15 + if (strstr(str, SEARCHMSG_UTF8)) 16 + ((int *)data)[1]++; 14 17 } 15 18 16 19 void serial_test_trace_printk(void) 17 20 { 18 21 struct trace_printk_lskel__bss *bss; 19 22 struct trace_printk_lskel *skel; 20 - int err = 0, found = 0; 23 + int err = 0, found[2] = {}; 21 24 22 25 skel = trace_printk_lskel__open(); 23 26 if (!ASSERT_OK_PTR(skel, "trace_printk__open")) ··· 49 46 if (!ASSERT_GT(bss->trace_printk_ret, 0, "bss->trace_printk_ret")) 50 47 goto cleanup; 51 48 52 - /* verify our search string is in the trace buffer */ 53 - ASSERT_OK(read_trace_pipe_iter(trace_pipe_cb, &found, 1000), 54 - "read_trace_pipe_iter"); 49 + if (!ASSERT_GT(bss->trace_printk_utf8_ran, 0, "bss->trace_printk_utf8_ran")) 50 + goto cleanup; 55 51 56 - if (!ASSERT_EQ(found, bss->trace_printk_ran, "found")) 52 + if (!ASSERT_GT(bss->trace_printk_utf8_ret, 0, "bss->trace_printk_utf8_ret")) 53 + goto cleanup; 54 + 55 + if (!ASSERT_LT(bss->trace_printk_invalid_spec_ret, 0, 56 + "bss->trace_printk_invalid_spec_ret")) 57 + goto cleanup; 58 + 59 + /* verify our search strings are in the trace buffer */ 60 + ASSERT_OK(read_trace_pipe_iter(trace_pipe_cb, found, 1000), 61 + "read_trace_pipe_iter"); 62 + 63 + if (!ASSERT_EQ(found[0], bss->trace_printk_ran, "found")) 64 + goto cleanup; 65 + 66 + if (!ASSERT_EQ(found[1], bss->trace_printk_utf8_ran, "found_utf8")) 57 67 goto cleanup; 58 68 59 69 cleanup:

+1 -20

tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c

··· 8 8 9 9 #include "cap_helpers.h" 10 10 #include "bpf_util.h" 11 + #include "sysctl_helpers.h" 11 12 12 13 /* Using CAP_LAST_CAP is risky here, since it can get pulled in from 13 14 * an old /usr/include/linux/capability.h and be < CAP_BPF; as a result ··· 35 34 { 36 35 if (ASSERT_EQ(len, sizeof(__u32), "perfbuf_size_valid")) 37 36 got_perfbuf_val = *(__u32 *)data; 38 - } 39 - 40 - static int sysctl_set(const char *sysctl_path, char *old_val, const char *new_val) 41 - { 42 - int ret = 0; 43 - FILE *fp; 44 - 45 - fp = fopen(sysctl_path, "r+"); 46 - if (!fp) 47 - return -errno; 48 - if (old_val && fscanf(fp, "%s", old_val) <= 0) { 49 - ret = -ENOENT; 50 - } else if (!old_val || strcmp(old_val, new_val) != 0) { 51 - fseek(fp, 0, SEEK_SET); 52 - if (fprintf(fp, "%s", new_val) < 0) 53 - ret = -errno; 54 - } 55 - fclose(fp); 56 - 57 - return ret; 58 37 } 59 38 60 39 static void test_unpriv_bpf_disabled_positive(struct test_unpriv_bpf_disabled *skel,

+10

tools/testing/selftests/bpf/progs/bpf_iter_unix.c

··· 7 7 8 8 char _license[] SEC("license") = "GPL"; 9 9 10 + SEC(".maps") struct { 11 + __uint(type, BPF_MAP_TYPE_SOCKMAP); 12 + __uint(max_entries, 1); 13 + __type(key, __u32); 14 + __type(value, __u64); 15 + } sockmap; 16 + 10 17 static long sock_i_ino(const struct sock *sk) 11 18 { 12 19 const struct socket *sk_socket = sk->sk_socket; ··· 82 75 } 83 76 84 77 BPF_SEQ_PRINTF(seq, "\n"); 78 + 79 + /* Test for deadlock. */ 80 + bpf_map_update_elem(&sockmap, &(int){0}, sk, 0); 85 81 86 82 return 0; 87 83 }

+15

tools/testing/selftests/bpf/progs/map_kptr_fail.c

··· 385 385 return 0; 386 386 } 387 387 388 + SEC("?tc") 389 + __failure __msg("invalid kptr access, R") 390 + int reject_scalar_store_to_kptr(struct __sk_buff *ctx) 391 + { 392 + struct map_value *v; 393 + int key = 0; 394 + 395 + v = bpf_map_lookup_elem(&array_map, &key); 396 + if (!v) 397 + return 0; 398 + 399 + *(volatile u64 *)&v->unref_ptr = 0xBADC0DE; 400 + return 0; 401 + } 402 + 388 403 char _license[] SEC("license") = "GPL";

+3 -2

tools/testing/selftests/bpf/progs/task_local_data.bpf.h

··· 86 86 }; 87 87 88 88 struct tld_data_u { 89 - __u64 start; /* offset of tld_data_u->data in a page */ 89 + __u64 unused; 90 90 char data[__PAGE_SIZE - sizeof(__u64)] __attribute__((aligned(8))); 91 91 }; 92 92 93 93 struct tld_map_value { 94 94 struct tld_data_u __uptr *data; 95 95 struct tld_meta_u __uptr *meta; 96 + __u16 start; /* offset of tld_data_u->data in a page */ 96 97 }; 97 98 98 99 typedef struct tld_uptr_dummy { ··· 177 176 if (!tld_obj->data_map || !tld_obj->data_map->data || !tld_obj->data_map->meta) 178 177 return 0; 179 178 180 - start = tld_obj->data_map->data->start; 179 + start = tld_obj->data_map->start; 181 180 cnt = tld_obj->data_map->meta->cnt; 182 181 metadata = tld_obj->data_map->meta->metadata; 183 182

+4 -4

tools/testing/selftests/bpf/progs/timer_start_deadlock.c

··· 27 27 return 0; 28 28 } 29 29 30 - SEC("tp_btf/hrtimer_cancel") 31 - int BPF_PROG(tp_hrtimer_cancel, struct hrtimer *hrtimer) 30 + SEC("tp_btf/hrtimer_start") 31 + int BPF_PROG(tp_hrtimer_start, struct hrtimer *hrtimer, enum hrtimer_mode mode, bool was_armed) 32 32 { 33 33 struct bpf_timer *timer; 34 34 int key = 0; 35 35 36 - if (!in_timer_start) 36 + if (!in_timer_start || !was_armed) 37 37 return 0; 38 38 39 39 tp_called = 1; ··· 60 60 61 61 /* 62 62 * call hrtimer_start() twice, so that 2nd call does 63 - * remove_hrtimer() and trace_hrtimer_cancel() tracepoint. 63 + * trace_hrtimer_start(was_armed=1) tracepoint. 64 64 */ 65 65 in_timer_start = 1; 66 66 bpf_timer_start(timer, 1000000000, 0);

+19

tools/testing/selftests/bpf/progs/token_kallsyms.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ 3 + 4 + #include "vmlinux.h" 5 + #include <bpf/bpf_helpers.h> 6 + 7 + char _license[] SEC("license") = "GPL"; 8 + 9 + __weak 10 + int token_ksym_subprog(void) 11 + { 12 + return 0; 13 + } 14 + 15 + SEC("xdp") 16 + int xdp_main(struct xdp_md *xdp) 17 + { 18 + return token_ksym_subprog(); 19 + }

+10

tools/testing/selftests/bpf/progs/trace_printk.c

··· 10 10 11 11 int trace_printk_ret = 0; 12 12 int trace_printk_ran = 0; 13 + int trace_printk_invalid_spec_ret = 0; 14 + int trace_printk_utf8_ret = 0; 15 + int trace_printk_utf8_ran = 0; 13 16 14 17 const char fmt[] = "Testing,testing %d\n"; 18 + static const char utf8_fmt[] = "中文,测试 %d\n"; 19 + /* Non-ASCII bytes after '%' must still be rejected. */ 20 + static const char invalid_spec_fmt[] = "%\x80\n"; 15 21 16 22 SEC("fentry/" SYS_PREFIX "sys_nanosleep") 17 23 int sys_enter(void *ctx) 18 24 { 19 25 trace_printk_ret = bpf_trace_printk(fmt, sizeof(fmt), 20 26 ++trace_printk_ran); 27 + trace_printk_utf8_ret = bpf_trace_printk(utf8_fmt, sizeof(utf8_fmt), 28 + ++trace_printk_utf8_ran); 29 + trace_printk_invalid_spec_ret = bpf_trace_printk(invalid_spec_fmt, 30 + sizeof(invalid_spec_fmt)); 21 31 return 0; 22 32 }

+193

tools/testing/selftests/bpf/progs/verifier_live_stack.c

··· 2647 2647 "exit;" 2648 2648 ::: __clobber_all); 2649 2649 } 2650 + 2651 + /* 2652 + * Same as spill_join_with_multi_off but the write is BPF_ST (store 2653 + * immediate) instead of BPF_STX. BPF_ST goes through 2654 + * clear_stack_for_all_offs() rather than spill_to_stack(), and that 2655 + * path also needs to join instead of overwriting. 2656 + * 2657 + * fp-8 = &fp-24 2658 + * fp-16 = &fp-32 2659 + * r1 = fp-8 or fp-16 (two offsets from branch) 2660 + * *(u64 *)(r1 + 0) = 0 -- BPF_ST with immediate 2661 + * r0 = *(u64 *)(r10 - 16) -- fill from fp-16 2662 + * r0 = *(u64 *)(r0 + 0) -- deref: should produce use 2663 + */ 2664 + SEC("socket") 2665 + __log_level(2) 2666 + __failure 2667 + __msg("15: (7a) *(u64 *)(r1 +0) = 0 fp-8: fp0-24 -> fp0-24|fp0+0 fp-16: fp0-32 -> fp0-32|fp0+0") 2668 + __msg("17: (79) r0 = *(u64 *)(r0 +0) ; use: fp0-32") 2669 + __naked void st_imm_join_with_multi_off(void) 2670 + { 2671 + asm volatile ( 2672 + "*(u64 *)(r10 - 24) = 0;" 2673 + "*(u64 *)(r10 - 32) = 0;" 2674 + "r1 = r10;" 2675 + "r1 += -24;" 2676 + "*(u64 *)(r10 - 8) = r1;" 2677 + "r1 = r10;" 2678 + "r1 += -32;" 2679 + "*(u64 *)(r10 - 16) = r1;" 2680 + /* create r1 with two candidate offsets: fp-8 or fp-16 */ 2681 + "call %[bpf_get_prandom_u32];" 2682 + "if r0 == 0 goto 1f;" 2683 + "r1 = r10;" 2684 + "r1 += -8;" 2685 + "goto 2f;" 2686 + "1:" 2687 + "r1 = r10;" 2688 + "r1 += -16;" 2689 + "2:" 2690 + /* BPF_ST: store immediate through multi-offset r1 */ 2691 + "*(u64 *)(r1 + 0) = 0;" 2692 + /* read back fp-16 and deref */ 2693 + "r0 = *(u64 *)(r10 - 16);" 2694 + "r0 = *(u64 *)(r0 + 0);" 2695 + "r0 = 0;" 2696 + "exit;" 2697 + :: __imm(bpf_get_prandom_u32) 2698 + : __clobber_all); 2699 + } 2700 + 2701 + /* 2702 + * Check that BPF_ST with a known offset fully overwrites stack slot 2703 + * from the arg tracking point of view. 2704 + */ 2705 + SEC("socket") 2706 + __log_level(2) 2707 + __success 2708 + __msg("5: (7a) *(u64 *)(r1 +0) = 0 fp-8: fp0-16 -> _{{$}}") 2709 + __naked void st_imm_join_with_single_off(void) 2710 + { 2711 + asm volatile ( 2712 + "r2 = r10;" 2713 + "r2 += -16;" 2714 + "*(u64 *)(r10 - 8) = r2;" 2715 + "r1 = r10;" 2716 + "r1 += -8;" 2717 + "*(u64 *)(r1 + 0) = 0;" 2718 + "r0 = 0;" 2719 + "exit;" 2720 + ::: __clobber_all); 2721 + } 2722 + 2723 + /* 2724 + * Same as spill_join_with_imprecise_off but the write is BPF_ST. 2725 + * Use "r2 = -8; r1 += r2" to make arg tracking lose offset 2726 + * precision while the main verifier keeps r1 as fixed-offset. 2727 + * 2728 + * fp-8 = &fp-24 2729 + * fp-16 = &fp-32 2730 + * r1 = fp-8 (imprecise to arg tracking) 2731 + * *(u64 *)(r1 + 0) = 0 -- BPF_ST with immediate 2732 + * r0 = *(u64 *)(r10 - 16) -- fill from fp-16 2733 + * r0 = *(u64 *)(r0 + 0) -- deref: should produce use 2734 + */ 2735 + SEC("socket") 2736 + __log_level(2) 2737 + __success 2738 + __msg("13: (79) r0 = *(u64 *)(r0 +0) ; use: fp0-32") 2739 + __naked void st_imm_join_with_imprecise_off(void) 2740 + { 2741 + asm volatile ( 2742 + "*(u64 *)(r10 - 24) = 0;" 2743 + "*(u64 *)(r10 - 32) = 0;" 2744 + "r1 = r10;" 2745 + "r1 += -24;" 2746 + "*(u64 *)(r10 - 8) = r1;" 2747 + "r1 = r10;" 2748 + "r1 += -32;" 2749 + "*(u64 *)(r10 - 16) = r1;" 2750 + /* r1 = fp-8 but arg tracking sees off_cnt == 0 */ 2751 + "r1 = r10;" 2752 + "r2 = -8;" 2753 + "r1 += r2;" 2754 + /* store immediate through imprecise r1 */ 2755 + "*(u64 *)(r1 + 0) = 0;" 2756 + /* read back fp-16 */ 2757 + "r0 = *(u64 *)(r10 - 16);" 2758 + /* deref: should produce use */ 2759 + "r0 = *(u64 *)(r0 + 0);" 2760 + "r0 = 0;" 2761 + "exit;" 2762 + ::: __clobber_all); 2763 + } 2764 + 2765 + /* 2766 + * Test that spilling through an ARG_IMPRECISE pointer joins with 2767 + * existing at_stack values. Subprog receives r1 = fp0-24 and 2768 + * r2 = map_value, creates an ARG_IMPRECISE pointer by joining caller 2769 + * and callee FP on two branches. 2770 + * 2771 + * Setup: callee spills &fp1-16 to fp1-8 (precise, tracked). 2772 + * Then writes map_value through ARG_IMPRECISE r1 — on path A 2773 + * this hits fp1-8, on path B it hits caller stack. 2774 + * Since spill_to_stack is skipped for ARG_IMPRECISE dst, 2775 + * fp1-8 tracking isn't joined with none. 2776 + * 2777 + * Expected after the imprecise write: 2778 + * - arg tracking should show fp1-8 = fp1-16|fp1+0 (joined with none) 2779 + * - read from fp1-8 and deref should produce use for fp1-16 2780 + * - write through it should NOT produce def for fp1-16 2781 + */ 2782 + SEC("socket") 2783 + __log_level(2) 2784 + __success 2785 + __msg("26: (79) r0 = *(u64 *)(r10 -8) // r1=IMP3 r6=fp0-24 r7=fp1-16 fp-8=fp1-16|fp1+0") 2786 + __naked void imprecise_dst_spill_join(void) 2787 + { 2788 + asm volatile ( 2789 + "*(u64 *)(r10 - 24) = 0;" 2790 + /* map lookup for a valid non-FP pointer */ 2791 + "*(u32 *)(r10 - 32) = 0;" 2792 + "r1 = %[map] ll;" 2793 + "r2 = r10;" 2794 + "r2 += -32;" 2795 + "call %[bpf_map_lookup_elem];" 2796 + "if r0 == 0 goto 1f;" 2797 + /* r1 = &caller_fp-24, r2 = map_value */ 2798 + "r1 = r10;" 2799 + "r1 += -24;" 2800 + "r2 = r0;" 2801 + "call imprecise_dst_spill_join_sub;" 2802 + "1:" 2803 + "r0 = 0;" 2804 + "exit;" 2805 + :: __imm_addr(map), 2806 + __imm(bpf_map_lookup_elem) 2807 + : __clobber_all); 2808 + } 2809 + 2810 + static __used __naked void imprecise_dst_spill_join_sub(void) 2811 + { 2812 + asm volatile ( 2813 + /* r6 = &caller_fp-24 (frame=0), r8 = map_value */ 2814 + "r6 = r1;" 2815 + "r8 = r2;" 2816 + /* spill &fp1-16 to fp1-8: at_stack[0] = fp1-16 */ 2817 + "*(u64 *)(r10 - 16) = 0;" 2818 + "r7 = r10;" 2819 + "r7 += -16;" 2820 + "*(u64 *)(r10 - 8) = r7;" 2821 + /* branch to create ARG_IMPRECISE pointer */ 2822 + "call %[bpf_get_prandom_u32];" 2823 + /* path B: r1 = caller fp-24 (frame=0) */ 2824 + "r1 = r6;" 2825 + "if r0 == 0 goto 1f;" 2826 + /* path A: r1 = callee fp-8 (frame=1) */ 2827 + "r1 = r10;" 2828 + "r1 += -8;" 2829 + "1:" 2830 + /* r1 = ARG_IMPRECISE{mask=BIT(0)|BIT(1)}. 2831 + * Write map_value (non-FP) through r1. On path A this overwrites fp1-8. 2832 + * Should join at_stack[0] with none: fp1-16|fp1+0. 2833 + */ 2834 + "*(u64 *)(r1 + 0) = r8;" 2835 + /* read fp1-8: should be fp1-16|fp1+0 (joined) */ 2836 + "r0 = *(u64 *)(r10 - 8);" 2837 + "*(u64 *)(r0 + 0) = 42;" 2838 + "r0 = 0;" 2839 + "exit;" 2840 + :: __imm(bpf_get_prandom_u32) 2841 + : __clobber_all); 2842 + }

+37

tools/testing/selftests/bpf/sysctl_helpers.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + #include <stdio.h> 3 + #include <errno.h> 4 + #include <string.h> 5 + 6 + #include "sysctl_helpers.h" 7 + #include "test_progs.h" 8 + 9 + int sysctl_set(const char *sysctl_path, char *old_val, const char *new_val) 10 + { 11 + int ret = 0; 12 + FILE *fp; 13 + 14 + fp = fopen(sysctl_path, "r+"); 15 + if (!fp) 16 + return -errno; 17 + if (old_val && fscanf(fp, "%s", old_val) <= 0) { 18 + ret = -ENOENT; 19 + } else if (!old_val || strcmp(old_val, new_val) != 0) { 20 + fseek(fp, 0, SEEK_SET); 21 + if (fprintf(fp, "%s", new_val) < 0) 22 + ret = -errno; 23 + } 24 + fclose(fp); 25 + 26 + return ret; 27 + } 28 + 29 + int sysctl_set_or_fail(const char *sysctl_path, char *old_val, const char *new_val) 30 + { 31 + int err; 32 + 33 + err = sysctl_set(sysctl_path, old_val, new_val); 34 + if (err) 35 + PRINT_FAIL("failed to set %s to %s: %s\n", sysctl_path, new_val, strerror(-err)); 36 + return err; 37 + }

+8

tools/testing/selftests/bpf/sysctl_helpers.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + #ifndef __SYSCTL_HELPERS_H 3 + #define __SYSCTL_HELPERS_H 4 + 5 + int sysctl_set(const char *sysctl_path, char *old_val, const char *new_val); 6 + int sysctl_set_or_fail(const char *sysctl_path, char *old_val, const char *new_val); 7 + 8 + #endif

Configure Feed

Configure Feed