Merge branch 'bpf-unify-state-pruning-handling-of-invalid-misc-stack-slots'

Eduard Zingerman says:

====================
bpf: unify state pruning handling of invalid/misc stack slots

This change unifies states pruning handling of NOT_INIT registers,
STACK_INVALID/STACK_MISC stack slots for regular and iterator/callback
based loop cases.

The change results in a modest verifier performance improvement:

========= selftests: master vs loop-stack-misc-pruning =========

File Program Insns (A) Insns (B) Insns (DIFF)
------------------------------- -------------------- --------- --------- ----------------
test_tcp_custom_syncookie.bpf.o tcp_custom_syncookie 38307 18430 -19877 (-51.89%)
xdp_synproxy_kern.bpf.o syncookie_tc 23035 19067 -3968 (-17.23%)
xdp_synproxy_kern.bpf.o syncookie_xdp 21022 18516 -2506 (-11.92%)

Total progs: 4173
Old success: 2520
New success: 2521
total_insns diff min: -99.99%
total_insns diff max: 0.00%
0 -> value: 0
value -> 0: 0
total_insns abs max old: 837,487
total_insns abs max new: 837,487
-100 .. -90 %: 1
-60 .. -50 %: 3
-50 .. -40 %: 2
-40 .. -30 %: 2
-30 .. -20 %: 8
-20 .. -10 %: 4
-10 .. 0 %: 5
0 .. 5 %: 4148

========= scx: master vs loop-stack-misc-pruning =========

File Program Insns (A) Insns (B) Insns (DIFF)
------------------------- ---------------- --------- --------- ----------------
scx_arena_selftests.bpf.o arena_selftest 257545 243678 -13867 (-5.38%)
scx_chaos.bpf.o chaos_dispatch 13989 12804 -1185 (-8.47%)
scx_layered.bpf.o layered_dispatch 27600 13925 -13675 (-49.55%)

Total progs: 305
Old success: 292
New success: 292
total_insns diff min: -49.55%
total_insns diff max: 0.00%
0 -> value: 0
value -> 0: 0
total_insns abs max old: 257,545
total_insns abs max new: 243,678
-50 .. -45 %: 7
-30 .. -20 %: 5
-20 .. -10 %: 14
-10 .. 0 %: 18
0 .. 5 %: 261

There is also a significant verifier performance improvement for some
bpf_loop() heavy Meta internal programs (~ -40% processed instructions).
====================

Link: https://patch.msgid.link/20251230-loop-stack-misc-pruning-v1-0-585cfd6cec51@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

Alexei Starovoitov 5 months ago c0e4a193 ccaa6d2c

+69 -6

2 changed files

expand all

kernel

bpf

verifier.c

tools

testing

selftests

bpf

progs

iters.c

+4 -6

kernel/bpf/verifier.c

··· 19086 19086 if (exact == EXACT) 19087 19087 return regs_exact(rold, rcur, idmap); 19088 19088 19089 - if (rold->type == NOT_INIT) { 19090 - if (exact == NOT_EXACT || rcur->type == NOT_INIT) 19091 - /* explored state can't have used this */ 19092 - return true; 19093 - } 19089 + if (rold->type == NOT_INIT) 19090 + /* explored state can't have used this */ 19091 + return true; 19094 19092 19095 19093 /* Enforce that register types have to match exactly, including their 19096 19094 * modifiers (like PTR_MAYBE_NULL, MEM_RDONLY, etc), as a general ··· 19257 19259 19258 19260 spi = i / BPF_REG_SIZE; 19259 19261 19260 - if (exact != NOT_EXACT && 19262 + if (exact == EXACT && 19261 19263 (i >= cur->allocated_stack || 19262 19264 old->stack[spi].slot_type[i % BPF_REG_SIZE] != 19263 19265 cur->stack[spi].slot_type[i % BPF_REG_SIZE]))

+65

tools/testing/selftests/bpf/progs/iters.c

··· 1997 1997 "goto 2b;" 1998 1998 : 1999 1999 : __imm(bpf_get_prandom_u32) 2000 + ); 2001 + } 2002 + 2003 + SEC("raw_tp") 2004 + __success 2005 + __naked int stack_misc_vs_scalar_in_a_loop(void) 2006 + { 2007 + asm volatile( 2008 + "*(u8 *)(r10 - 15) = 1;" /* This marks stack slot fp[-16] as STACK_MISC. */ 2009 + "*(u8 *)(r10 - 23) = 1;" 2010 + "*(u8 *)(r10 - 31) = 1;" 2011 + "*(u8 *)(r10 - 39) = 1;" 2012 + "*(u8 *)(r10 - 47) = 1;" 2013 + "*(u8 *)(r10 - 55) = 1;" 2014 + "*(u8 *)(r10 - 63) = 1;" 2015 + "*(u8 *)(r10 - 71) = 1;" 2016 + "*(u8 *)(r10 - 79) = 1;" 2017 + "r1 = r10;" 2018 + "r1 += -8;" 2019 + "r2 = 0;" 2020 + "r3 = 10;" 2021 + "call %[bpf_iter_num_new];" 2022 + "loop_%=:" 2023 + "r1 = r10;" 2024 + "r1 += -8;" 2025 + "call %[bpf_iter_num_next];" 2026 + "if r0 == 0 goto loop_end_%=;" 2027 + 2028 + #define maybe_change_stack_slot(off) \ 2029 + "call %[bpf_get_prandom_u32];" \ 2030 + "if r0 == 42 goto +1;" \ 2031 + "goto +1;" \ 2032 + "*(u64 *)(r10 " #off ") = r0;" 2033 + 2034 + /* 2035 + * When comparing verifier states fp[-16] will be 2036 + * either STACK_MISC or SCALAR. Pruning logic should 2037 + * consider old STACK_MISC equivalent to current SCALAR 2038 + * to avoid states explosion. 2039 + */ 2040 + maybe_change_stack_slot(-16) 2041 + maybe_change_stack_slot(-24) 2042 + maybe_change_stack_slot(-32) 2043 + maybe_change_stack_slot(-40) 2044 + maybe_change_stack_slot(-48) 2045 + maybe_change_stack_slot(-56) 2046 + maybe_change_stack_slot(-64) 2047 + maybe_change_stack_slot(-72) 2048 + maybe_change_stack_slot(-80) 2049 + 2050 + #undef maybe_change_stack_slot 2051 + 2052 + "goto loop_%=;" 2053 + "loop_end_%=:" 2054 + "r1 = r10;" 2055 + "r1 += -8;" 2056 + "call %[bpf_iter_num_destroy];" 2057 + "r0 = 0;" 2058 + "exit;" 2059 + : 2060 + : __imm(bpf_get_prandom_u32), 2061 + __imm(bpf_iter_num_new), 2062 + __imm(bpf_iter_num_next), 2063 + __imm(bpf_iter_num_destroy), 2064 + __imm_addr(amap) 2000 2065 : __clobber_all 2001 2066 ); 2002 2067 }

Configure Feed

Configure Feed