+1 -1

Documentation/bpf/prog_flow_dissector.rst

reviewed

··· 113 113 used by ``eth_get_headlen`` to estimate length of all headers for GRO. 114 114 * ``BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL`` - tells BPF flow dissector to 115 115 stop parsing as soon as it reaches IPv6 flow label; used by 116 116 - ``___skb_get_hash`` and ``__skb_get_hash_symmetric`` to get flow hash. 116 116 + ``___skb_get_hash`` to get flow hash. 117 117 * ``BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP`` - tells BPF flow dissector to stop 118 118 parsing as soon as it reaches encapsulated headers; used by routing 119 119 infrastructure.

+21

Documentation/netlink/specs/netdev.yaml

reviewed

··· 42 42 doc: 43 43 This feature informs if netdev implements non-linear XDP buffer 44 44 support in ndo_xdp_xmit callback. 45 45 + - 46 46 + type: flags 47 47 + name: xdp-rx-metadata 48 48 + render-max: true 49 49 + entries: 50 50 + - 51 51 + name: timestamp 52 52 + doc: 53 53 + Device is capable of exposing receive HW timestamp via bpf_xdp_metadata_rx_timestamp(). 54 54 + - 55 55 + name: hash 56 56 + doc: 57 57 + Device is capable of exposing receive packet hash via bpf_xdp_metadata_rx_hash(). 45 58 46 59 attribute-sets: 47 60 - ··· 81 68 type: u32 82 69 checks: 83 70 min: 1 71 71 + - 72 72 + name: xdp-rx-metadata-features 73 73 + doc: Bitmask of supported XDP receive metadata features. 74 74 + See Documentation/networking/xdp-rx-metadata.rst for more details. 75 75 + type: u64 76 76 + enum: xdp-rx-metadata 77 77 + enum-as-flags: true 84 78 85 79 operations: 86 80 list: ··· 104 84 - ifindex 105 85 - xdp-features 106 86 - xdp-zc-max-segs 87 87 + - xdp-rx-metadata-features 107 88 dump: 108 89 reply: *dev-all 109 90 -

+7

Documentation/networking/xdp-rx-metadata.rst

reviewed

··· 105 105 Adding programs that access metadata kfuncs to the ``BPF_MAP_TYPE_PROG_ARRAY`` 106 106 is currently not supported. 107 107 108 108 + Supported Devices 109 109 + ================= 110 110 + 111 111 + It is possible to query which kfunc the particular netdev implements via 112 112 + netlink. See ``xdp-rx-metadata-features`` attribute set in 113 113 + ``Documentation/netlink/specs/netdev.yaml``. 114 114 + 108 115 Example 109 116 ======= 110 117

+3 -2

MAINTAINERS

reviewed

··· 3596 3596 F: drivers/iio/accel/bma400* 3597 3597 3598 3598 BPF JIT for ARM 3599 3599 - M: Shubham Bansal <illusionist.neo@gmail.com> 3599 3599 + M: Russell King <linux@armlinux.org.uk> 3600 3600 + M: Puranjay Mohan <puranjay12@gmail.com> 3600 3601 L: bpf@vger.kernel.org 3601 3601 - S: Odd Fixes 3602 3602 + S: Maintained 3602 3603 F: arch/arm/net/ 3603 3604 3604 3605 BPF JIT for ARM64

+260 -20

arch/arm/net/bpf_jit_32.c

reviewed

··· 2 2 /* 3 3 * Just-In-Time compiler for eBPF filters on 32bit ARM 4 4 * 5 5 + * Copyright (c) 2023 Puranjay Mohan <puranjay12@gmail.com> 5 6 * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com> 6 7 * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com> 7 8 */ ··· 16 15 #include <linux/string.h> 17 16 #include <linux/slab.h> 18 17 #include <linux/if_vlan.h> 18 18 + #include <linux/math64.h> 19 19 20 20 #include <asm/cacheflush.h> 21 21 #include <asm/hwcap.h> ··· 230 228 return dividend % divisor; 231 229 } 232 230 231 231 + static s32 jit_sdiv32(s32 dividend, s32 divisor) 232 232 + { 233 233 + return dividend / divisor; 234 234 + } 235 235 + 236 236 + static s32 jit_smod32(s32 dividend, s32 divisor) 237 237 + { 238 238 + return dividend % divisor; 239 239 + } 240 240 + 241 241 + /* Wrappers for 64-bit div/mod */ 242 242 + static u64 jit_udiv64(u64 dividend, u64 divisor) 243 243 + { 244 244 + return div64_u64(dividend, divisor); 245 245 + } 246 246 + 247 247 + static u64 jit_mod64(u64 dividend, u64 divisor) 248 248 + { 249 249 + u64 rem; 250 250 + 251 251 + div64_u64_rem(dividend, divisor, &rem); 252 252 + return rem; 253 253 + } 254 254 + 255 255 + static s64 jit_sdiv64(s64 dividend, s64 divisor) 256 256 + { 257 257 + return div64_s64(dividend, divisor); 258 258 + } 259 259 + 260 260 + static s64 jit_smod64(s64 dividend, s64 divisor) 261 261 + { 262 262 + u64 q; 263 263 + 264 264 + q = div64_s64(dividend, divisor); 265 265 + 266 266 + return dividend - q * divisor; 267 267 + } 268 268 + 233 269 static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx) 234 270 { 235 271 inst |= (cond << 28); ··· 372 332 #define ARM_LDRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDRB_I, rt, rn, off) 373 333 #define ARM_LDRD_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRD_I, rt, rn, off) 374 334 #define ARM_LDRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRH_I, rt, rn, off) 335 335 + 336 336 + #define ARM_LDRSH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRSH_I, rt, rn, off) 337 337 + #define ARM_LDRSB_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRSB_I, rt, rn, off) 375 338 376 339 #define ARM_STR_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STR_I, rt, rn, off) 377 340 #define ARM_STRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STRB_I, rt, rn, off) ··· 517 474 return to - from - 2; 518 475 } 519 476 520 520 - static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op) 477 477 + static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op, u8 sign) 521 478 { 522 479 const int exclude_mask = BIT(ARM_R0) | BIT(ARM_R1); 523 480 const s8 *tmp = bpf2a32[TMP_REG_1]; 481 481 + u32 dst; 524 482 525 483 #if __LINUX_ARM_ARCH__ == 7 526 484 if (elf_hwcap & HWCAP_IDIVA) { 527 527 - if (op == BPF_DIV) 528 528 - emit(ARM_UDIV(rd, rm, rn), ctx); 529 529 - else { 530 530 - emit(ARM_UDIV(ARM_IP, rm, rn), ctx); 485 485 + if (op == BPF_DIV) { 486 486 + emit(sign ? ARM_SDIV(rd, rm, rn) : ARM_UDIV(rd, rm, rn), ctx); 487 487 + } else { 488 488 + emit(sign ? ARM_SDIV(ARM_IP, rm, rn) : ARM_UDIV(ARM_IP, rm, rn), ctx); 531 489 emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx); 532 490 } 533 491 return; ··· 556 512 emit(ARM_PUSH(CALLER_MASK & ~exclude_mask), ctx); 557 513 558 514 /* Call appropriate function */ 559 559 - emit_mov_i(ARM_IP, op == BPF_DIV ? 560 560 - (u32)jit_udiv32 : (u32)jit_mod32, ctx); 515 515 + if (sign) { 516 516 + if (op == BPF_DIV) 517 517 + dst = (u32)jit_sdiv32; 518 518 + else 519 519 + dst = (u32)jit_smod32; 520 520 + } else { 521 521 + if (op == BPF_DIV) 522 522 + dst = (u32)jit_udiv32; 523 523 + else 524 524 + dst = (u32)jit_mod32; 525 525 + } 526 526 + 527 527 + emit_mov_i(ARM_IP, dst, ctx); 561 528 emit_blx_r(ARM_IP, ctx); 562 529 563 530 /* Restore caller-saved registers from stack */ ··· 583 528 emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx); 584 529 if (rm != ARM_R0) 585 530 emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx); 531 531 + } 532 532 + 533 533 + static inline void emit_udivmod64(const s8 *rd, const s8 *rm, const s8 *rn, struct jit_ctx *ctx, 534 534 + u8 op, u8 sign) 535 535 + { 536 536 + u32 dst; 537 537 + 538 538 + /* Push caller-saved registers on stack */ 539 539 + emit(ARM_PUSH(CALLER_MASK), ctx); 540 540 + 541 541 + /* 542 542 + * As we are implementing 64-bit div/mod as function calls, We need to put the dividend in 543 543 + * R0-R1 and the divisor in R2-R3. As we have already pushed these registers on the stack, 544 544 + * we can recover them later after returning from the function call. 545 545 + */ 546 546 + if (rm[1] != ARM_R0 || rn[1] != ARM_R2) { 547 547 + /* 548 548 + * Move Rm to {R1, R0} if it is not already there. 549 549 + */ 550 550 + if (rm[1] != ARM_R0) { 551 551 + if (rn[1] == ARM_R0) 552 552 + emit(ARM_PUSH(BIT(ARM_R0) | BIT(ARM_R1)), ctx); 553 553 + emit(ARM_MOV_R(ARM_R1, rm[0]), ctx); 554 554 + emit(ARM_MOV_R(ARM_R0, rm[1]), ctx); 555 555 + if (rn[1] == ARM_R0) { 556 556 + emit(ARM_POP(BIT(ARM_R2) | BIT(ARM_R3)), ctx); 557 557 + goto cont; 558 558 + } 559 559 + } 560 560 + /* 561 561 + * Move Rn to {R3, R2} if it is not already there. 562 562 + */ 563 563 + if (rn[1] != ARM_R2) { 564 564 + emit(ARM_MOV_R(ARM_R3, rn[0]), ctx); 565 565 + emit(ARM_MOV_R(ARM_R2, rn[1]), ctx); 566 566 + } 567 567 + } 568 568 + 569 569 + cont: 570 570 + 571 571 + /* Call appropriate function */ 572 572 + if (sign) { 573 573 + if (op == BPF_DIV) 574 574 + dst = (u32)jit_sdiv64; 575 575 + else 576 576 + dst = (u32)jit_smod64; 577 577 + } else { 578 578 + if (op == BPF_DIV) 579 579 + dst = (u32)jit_udiv64; 580 580 + else 581 581 + dst = (u32)jit_mod64; 582 582 + } 583 583 + 584 584 + emit_mov_i(ARM_IP, dst, ctx); 585 585 + emit_blx_r(ARM_IP, ctx); 586 586 + 587 587 + /* Save return value */ 588 588 + if (rd[1] != ARM_R0) { 589 589 + emit(ARM_MOV_R(rd[0], ARM_R1), ctx); 590 590 + emit(ARM_MOV_R(rd[1], ARM_R0), ctx); 591 591 + } 592 592 + 593 593 + /* Recover {R3, R2} and {R1, R0} from stack if they are not Rd */ 594 594 + if (rd[1] != ARM_R0 && rd[1] != ARM_R2) { 595 595 + emit(ARM_POP(CALLER_MASK), ctx); 596 596 + } else if (rd[1] != ARM_R0) { 597 597 + emit(ARM_POP(BIT(ARM_R0) | BIT(ARM_R1)), ctx); 598 598 + emit(ARM_ADD_I(ARM_SP, ARM_SP, 8), ctx); 599 599 + } else { 600 600 + emit(ARM_ADD_I(ARM_SP, ARM_SP, 8), ctx); 601 601 + emit(ARM_POP(BIT(ARM_R2) | BIT(ARM_R3)), ctx); 602 602 + } 586 603 } 587 604 588 605 /* Is the translated BPF register on stack? */ ··· 871 744 } 872 745 873 746 /* dst = src (4 bytes)*/ 874 874 - static inline void emit_a32_mov_r(const s8 dst, const s8 src, 747 747 + static inline void emit_a32_mov_r(const s8 dst, const s8 src, const u8 off, 875 748 struct jit_ctx *ctx) { 876 749 const s8 *tmp = bpf2a32[TMP_REG_1]; 877 750 s8 rt; 878 751 879 752 rt = arm_bpf_get_reg32(src, tmp[0], ctx); 753 753 + if (off && off != 32) { 754 754 + emit(ARM_LSL_I(rt, rt, 32 - off), ctx); 755 755 + emit(ARM_ASR_I(rt, rt, 32 - off), ctx); 756 756 + } 880 757 arm_bpf_put_reg32(dst, rt, ctx); 881 758 } 882 759 ··· 889 758 const s8 src[], 890 759 struct jit_ctx *ctx) { 891 760 if (!is64) { 892 892 - emit_a32_mov_r(dst_lo, src_lo, ctx); 761 761 + emit_a32_mov_r(dst_lo, src_lo, 0, ctx); 893 762 if (!ctx->prog->aux->verifier_zext) 894 763 /* Zero out high 4 bytes */ 895 764 emit_a32_mov_i(dst_hi, 0, ctx); 896 765 } else if (__LINUX_ARM_ARCH__ < 6 && 897 766 ctx->cpu_architecture < CPU_ARCH_ARMv5TE) { 898 767 /* complete 8 byte move */ 899 899 - emit_a32_mov_r(dst_lo, src_lo, ctx); 900 900 - emit_a32_mov_r(dst_hi, src_hi, ctx); 768 768 + emit_a32_mov_r(dst_lo, src_lo, 0, ctx); 769 769 + emit_a32_mov_r(dst_hi, src_hi, 0, ctx); 901 770 } else if (is_stacked(src_lo) && is_stacked(dst_lo)) { 902 771 const u8 *tmp = bpf2a32[TMP_REG_1]; 903 772 ··· 910 779 } else { 911 780 emit(ARM_MOV_R(dst[0], src[0]), ctx); 912 781 emit(ARM_MOV_R(dst[1], src[1]), ctx); 782 782 + } 783 783 + } 784 784 + 785 785 + /* dst = (signed)src */ 786 786 + static inline void emit_a32_movsx_r64(const bool is64, const u8 off, const s8 dst[], const s8 src[], 787 787 + struct jit_ctx *ctx) { 788 788 + const s8 *tmp = bpf2a32[TMP_REG_1]; 789 789 + const s8 *rt; 790 790 + 791 791 + rt = arm_bpf_get_reg64(dst, tmp, ctx); 792 792 + 793 793 + emit_a32_mov_r(dst_lo, src_lo, off, ctx); 794 794 + if (!is64) { 795 795 + if (!ctx->prog->aux->verifier_zext) 796 796 + /* Zero out high 4 bytes */ 797 797 + emit_a32_mov_i(dst_hi, 0, ctx); 798 798 + } else { 799 799 + emit(ARM_ASR_I(rt[0], rt[1], 31), ctx); 913 800 } 914 801 } 915 802 ··· 1175 1026 return -off_max <= off && off <= off_max; 1176 1027 } 1177 1028 1029 1029 + static bool is_ldst_imm8(s16 off, const u8 size) 1030 1030 + { 1031 1031 + s16 off_max = 0; 1032 1032 + 1033 1033 + switch (size) { 1034 1034 + case BPF_B: 1035 1035 + off_max = 0xff; 1036 1036 + break; 1037 1037 + case BPF_W: 1038 1038 + off_max = 0xfff; 1039 1039 + break; 1040 1040 + case BPF_H: 1041 1041 + off_max = 0xff; 1042 1042 + break; 1043 1043 + } 1044 1044 + return -off_max <= off && off <= off_max; 1045 1045 + } 1046 1046 + 1178 1047 /* *(size *)(dst + off) = src */ 1179 1048 static inline void emit_str_r(const s8 dst, const s8 src[], 1180 1049 s16 off, struct jit_ctx *ctx, const u8 sz){ ··· 1269 1102 emit(ARM_LDR_I(rd[0], rm, off + 4), ctx); 1270 1103 break; 1271 1104 } 1105 1105 + arm_bpf_put_reg64(dst, rd, ctx); 1106 1106 + } 1107 1107 + 1108 1108 + /* dst = *(signed size*)(src + off) */ 1109 1109 + static inline void emit_ldsx_r(const s8 dst[], const s8 src, 1110 1110 + s16 off, struct jit_ctx *ctx, const u8 sz){ 1111 1111 + const s8 *tmp = bpf2a32[TMP_REG_1]; 1112 1112 + const s8 *rd = is_stacked(dst_lo) ? tmp : dst; 1113 1113 + s8 rm = src; 1114 1114 + int add_off; 1115 1115 + 1116 1116 + if (!is_ldst_imm8(off, sz)) { 1117 1117 + /* 1118 1118 + * offset does not fit in the load/store immediate, 1119 1119 + * construct an ADD instruction to apply the offset. 1120 1120 + */ 1121 1121 + add_off = imm8m(off); 1122 1122 + if (add_off > 0) { 1123 1123 + emit(ARM_ADD_I(tmp[0], src, add_off), ctx); 1124 1124 + rm = tmp[0]; 1125 1125 + } else { 1126 1126 + emit_a32_mov_i(tmp[0], off, ctx); 1127 1127 + emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx); 1128 1128 + rm = tmp[0]; 1129 1129 + } 1130 1130 + off = 0; 1131 1131 + } 1132 1132 + 1133 1133 + switch (sz) { 1134 1134 + case BPF_B: 1135 1135 + /* Load a Byte with sign extension*/ 1136 1136 + emit(ARM_LDRSB_I(rd[1], rm, off), ctx); 1137 1137 + break; 1138 1138 + case BPF_H: 1139 1139 + /* Load a HalfWord with sign extension*/ 1140 1140 + emit(ARM_LDRSH_I(rd[1], rm, off), ctx); 1141 1141 + break; 1142 1142 + case BPF_W: 1143 1143 + /* Load a Word*/ 1144 1144 + emit(ARM_LDR_I(rd[1], rm, off), ctx); 1145 1145 + break; 1146 1146 + } 1147 1147 + /* Carry the sign extension to upper 32 bits */ 1148 1148 + emit(ARM_ASR_I(rd[0], rd[1], 31), ctx); 1272 1149 arm_bpf_put_reg64(dst, rd, ctx); 1273 1150 } 1274 1151 ··· 1596 1385 emit_a32_mov_i(dst_hi, 0, ctx); 1597 1386 break; 1598 1387 } 1599 1599 - emit_a32_mov_r64(is64, dst, src, ctx); 1388 1388 + if (insn->off) 1389 1389 + emit_a32_movsx_r64(is64, insn->off, dst, src, ctx); 1390 1390 + else 1391 1391 + emit_a32_mov_r64(is64, dst, src, ctx); 1600 1392 break; 1601 1393 case BPF_K: 1602 1394 /* Sign-extend immediate value to destination reg */ ··· 1675 1461 rt = src_lo; 1676 1462 break; 1677 1463 } 1678 1678 - emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code)); 1464 1464 + emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code), off); 1679 1465 arm_bpf_put_reg32(dst_lo, rd_lo, ctx); 1680 1466 if (!ctx->prog->aux->verifier_zext) 1681 1467 emit_a32_mov_i(dst_hi, 0, ctx); ··· 1684 1470 case BPF_ALU64 | BPF_DIV | BPF_X: 1685 1471 case BPF_ALU64 | BPF_MOD | BPF_K: 1686 1472 case BPF_ALU64 | BPF_MOD | BPF_X: 1687 1687 - goto notyet; 1473 1473 + rd = arm_bpf_get_reg64(dst, tmp2, ctx); 1474 1474 + switch (BPF_SRC(code)) { 1475 1475 + case BPF_X: 1476 1476 + rs = arm_bpf_get_reg64(src, tmp, ctx); 1477 1477 + break; 1478 1478 + case BPF_K: 1479 1479 + rs = tmp; 1480 1480 + emit_a32_mov_se_i64(is64, rs, imm, ctx); 1481 1481 + break; 1482 1482 + } 1483 1483 + emit_udivmod64(rd, rd, rs, ctx, BPF_OP(code), off); 1484 1484 + arm_bpf_put_reg64(dst, rd, ctx); 1485 1485 + break; 1688 1486 /* dst = dst << imm */ 1689 1487 /* dst = dst >> imm */ 1690 1488 /* dst = dst >> imm (signed) */ ··· 1771 1545 break; 1772 1546 /* dst = htole(dst) */ 1773 1547 /* dst = htobe(dst) */ 1774 1774 - case BPF_ALU | BPF_END | BPF_FROM_LE: 1775 1775 - case BPF_ALU | BPF_END | BPF_FROM_BE: 1548 1548 + case BPF_ALU | BPF_END | BPF_FROM_LE: /* also BPF_TO_LE */ 1549 1549 + case BPF_ALU | BPF_END | BPF_FROM_BE: /* also BPF_TO_BE */ 1550 1550 + /* dst = bswap(dst) */ 1551 1551 + case BPF_ALU64 | BPF_END | BPF_FROM_LE: /* also BPF_TO_LE */ 1776 1552 rd = arm_bpf_get_reg64(dst, tmp, ctx); 1777 1777 - if (BPF_SRC(code) == BPF_FROM_LE) 1553 1553 + if (BPF_SRC(code) == BPF_FROM_LE && BPF_CLASS(code) != BPF_ALU64) 1778 1554 goto emit_bswap_uxt; 1779 1555 switch (imm) { 1780 1556 case 16: ··· 1831 1603 case BPF_LDX | BPF_MEM | BPF_H: 1832 1604 case BPF_LDX | BPF_MEM | BPF_B: 1833 1605 case BPF_LDX | BPF_MEM | BPF_DW: 1606 1606 + /* LDSX: dst = *(signed size *)(src + off) */ 1607 1607 + case BPF_LDX | BPF_MEMSX | BPF_B: 1608 1608 + case BPF_LDX | BPF_MEMSX | BPF_H: 1609 1609 + case BPF_LDX | BPF_MEMSX | BPF_W: 1834 1610 rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); 1835 1835 - emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code)); 1611 1611 + if (BPF_MODE(insn->code) == BPF_MEMSX) 1612 1612 + emit_ldsx_r(dst, rn, off, ctx, BPF_SIZE(code)); 1613 1613 + else 1614 1614 + emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code)); 1836 1615 break; 1837 1616 /* speculation barrier */ 1838 1617 case BPF_ST | BPF_NOSPEC: ··· 1996 1761 break; 1997 1762 /* JMP OFF */ 1998 1763 case BPF_JMP | BPF_JA: 1764 1764 + case BPF_JMP32 | BPF_JA: 1999 1765 { 2000 2000 - if (off == 0) 1766 1766 + if (BPF_CLASS(code) == BPF_JMP32 && imm != 0) 1767 1767 + jmp_offset = bpf2a32_offset(i + imm, i, ctx); 1768 1768 + else if (BPF_CLASS(code) == BPF_JMP && off != 0) 1769 1769 + jmp_offset = bpf2a32_offset(i + off, i, ctx); 1770 1770 + else 2001 1771 break; 2002 2002 - jmp_offset = bpf2a32_offset(i+off, i, ctx); 1772 1772 + 2003 1773 check_imm24(jmp_offset); 2004 1774 emit(ARM_B(jmp_offset), ctx); 2005 1775 break;

+4

arch/arm/net/bpf_jit_32.h

reviewed

··· 79 79 #define ARM_INST_LDST__IMM12 0x00000fff 80 80 #define ARM_INST_LDRB_I 0x05500000 81 81 #define ARM_INST_LDRB_R 0x07d00000 82 82 + #define ARM_INST_LDRSB_I 0x015000d0 82 83 #define ARM_INST_LDRD_I 0x014000d0 83 84 #define ARM_INST_LDRH_I 0x015000b0 84 85 #define ARM_INST_LDRH_R 0x019000b0 86 86 + #define ARM_INST_LDRSH_I 0x015000f0 85 87 #define ARM_INST_LDR_I 0x05100000 86 88 #define ARM_INST_LDR_R 0x07900000 87 89 ··· 139 137 #define ARM_INST_TST_I 0x03100000 140 138 141 139 #define ARM_INST_UDIV 0x0730f010 140 140 + #define ARM_INST_SDIV 0x0710f010 142 141 143 142 #define ARM_INST_UMULL 0x00800090 144 143 ··· 268 265 #define ARM_TST_I(rn, imm) _AL3_I(ARM_INST_TST, 0, rn, imm) 269 266 270 267 #define ARM_UDIV(rd, rn, rm) (ARM_INST_UDIV | (rd) << 16 | (rn) | (rm) << 8) 268 268 + #define ARM_SDIV(rd, rn, rm) (ARM_INST_SDIV | (rd) << 16 | (rn) | (rm) << 8) 271 269 272 270 #define ARM_UMULL(rd_lo, rd_hi, rn, rm) (ARM_INST_UMULL | (rd_hi) << 16 \ 273 271 | (rd_lo) << 12 | (rm) << 8 | rn)

+1 -1

arch/arm64/net/bpf_jit_comp.c

reviewed

··· 288 288 static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) 289 289 { 290 290 const struct bpf_prog *prog = ctx->prog; 291 291 - const bool is_main_prog = prog->aux->func_idx == 0; 291 291 + const bool is_main_prog = !bpf_is_subprog(prog); 292 292 const u8 r6 = bpf2a64[BPF_REG_6]; 293 293 const u8 r7 = bpf2a64[BPF_REG_7]; 294 294 const u8 r8 = bpf2a64[BPF_REG_8];

+1 -1

arch/s390/net/bpf_jit_comp.c

reviewed

··· 556 556 EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt); 557 557 jit->prologue_plt_ret = jit->prg; 558 558 559 559 - if (fp->aux->func_idx == 0) { 559 559 + if (!bpf_is_subprog(fp)) { 560 560 /* Initialize the tail call counter in the main program. */ 561 561 /* xc STK_OFF_TCCNT(4,%r15),STK_OFF_TCCNT(%r15) */ 562 562 _EMIT6(0xd703f000 | STK_OFF_TCCNT, 0xf000 | STK_OFF_TCCNT);

+131 -18

arch/x86/net/bpf_jit_comp.c

reviewed

··· 16 16 #include <asm/set_memory.h> 17 17 #include <asm/nospec-branch.h> 18 18 #include <asm/text-patching.h> 19 19 + #include <asm/unwind.h> 20 20 + 21 21 + static bool all_callee_regs_used[4] = {true, true, true, true}; 19 22 20 23 static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) 21 24 { ··· 258 255 /* Number of bytes that will be skipped on tailcall */ 259 256 #define X86_TAIL_CALL_OFFSET (11 + ENDBR_INSN_SIZE) 260 257 258 258 + static void push_r12(u8 **pprog) 259 259 + { 260 260 + u8 *prog = *pprog; 261 261 + 262 262 + EMIT2(0x41, 0x54); /* push r12 */ 263 263 + *pprog = prog; 264 264 + } 265 265 + 261 266 static void push_callee_regs(u8 **pprog, bool *callee_regs_used) 262 267 { 263 268 u8 *prog = *pprog; ··· 278 267 EMIT2(0x41, 0x56); /* push r14 */ 279 268 if (callee_regs_used[3]) 280 269 EMIT2(0x41, 0x57); /* push r15 */ 270 270 + *pprog = prog; 271 271 + } 272 272 + 273 273 + static void pop_r12(u8 **pprog) 274 274 + { 275 275 + u8 *prog = *pprog; 276 276 + 277 277 + EMIT2(0x41, 0x5C); /* pop r12 */ 281 278 *pprog = prog; 282 279 } 283 280 ··· 310 291 * while jumping to another program 311 292 */ 312 293 static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, 313 313 - bool tail_call_reachable, bool is_subprog) 294 294 + bool tail_call_reachable, bool is_subprog, 295 295 + bool is_exception_cb) 314 296 { 315 297 u8 *prog = *pprog; 316 298 ··· 323 303 prog += X86_PATCH_SIZE; 324 304 if (!ebpf_from_cbpf) { 325 305 if (tail_call_reachable && !is_subprog) 306 306 + /* When it's the entry of the whole tailcall context, 307 307 + * zeroing rax means initialising tail_call_cnt. 308 308 + */ 326 309 EMIT2(0x31, 0xC0); /* xor eax, eax */ 327 310 else 311 311 + /* Keep the same instruction layout. */ 328 312 EMIT2(0x66, 0x90); /* nop2 */ 329 313 } 330 330 - EMIT1(0x55); /* push rbp */ 331 331 - EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ 314 314 + /* Exception callback receives FP as third parameter */ 315 315 + if (is_exception_cb) { 316 316 + EMIT3(0x48, 0x89, 0xF4); /* mov rsp, rsi */ 317 317 + EMIT3(0x48, 0x89, 0xD5); /* mov rbp, rdx */ 318 318 + /* The main frame must have exception_boundary as true, so we 319 319 + * first restore those callee-saved regs from stack, before 320 320 + * reusing the stack frame. 321 321 + */ 322 322 + pop_callee_regs(&prog, all_callee_regs_used); 323 323 + pop_r12(&prog); 324 324 + /* Reset the stack frame. */ 325 325 + EMIT3(0x48, 0x89, 0xEC); /* mov rsp, rbp */ 326 326 + } else { 327 327 + EMIT1(0x55); /* push rbp */ 328 328 + EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ 329 329 + } 332 330 333 331 /* X86_TAIL_CALL_OFFSET is here */ 334 332 EMIT_ENDBR(); ··· 505 467 * goto *(prog->bpf_func + prologue_size); 506 468 * out: 507 469 */ 508 508 - static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, 470 470 + static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, 471 471 + u8 **pprog, bool *callee_regs_used, 509 472 u32 stack_depth, u8 *ip, 510 473 struct jit_context *ctx) 511 474 { ··· 556 517 offset = ctx->tail_call_indirect_label - (prog + 2 - start); 557 518 EMIT2(X86_JE, offset); /* je out */ 558 519 559 559 - pop_callee_regs(&prog, callee_regs_used); 520 520 + if (bpf_prog->aux->exception_boundary) { 521 521 + pop_callee_regs(&prog, all_callee_regs_used); 522 522 + pop_r12(&prog); 523 523 + } else { 524 524 + pop_callee_regs(&prog, callee_regs_used); 525 525 + } 560 526 561 527 EMIT1(0x58); /* pop rax */ 562 528 if (stack_depth) ··· 585 541 *pprog = prog; 586 542 } 587 543 588 588 - static void emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, 544 544 + static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, 545 545 + struct bpf_jit_poke_descriptor *poke, 589 546 u8 **pprog, u8 *ip, 590 547 bool *callee_regs_used, u32 stack_depth, 591 548 struct jit_context *ctx) ··· 615 570 emit_jump(&prog, (u8 *)poke->tailcall_target + X86_PATCH_SIZE, 616 571 poke->tailcall_bypass); 617 572 618 618 - pop_callee_regs(&prog, callee_regs_used); 573 573 + if (bpf_prog->aux->exception_boundary) { 574 574 + pop_callee_regs(&prog, all_callee_regs_used); 575 575 + pop_r12(&prog); 576 576 + } else { 577 577 + pop_callee_regs(&prog, callee_regs_used); 578 578 + } 579 579 + 619 580 EMIT1(0x58); /* pop rax */ 620 581 if (stack_depth) 621 582 EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8)); ··· 1069 1018 1070 1019 #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp))) 1071 1020 1021 1021 + /* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */ 1022 1022 + #define RESTORE_TAIL_CALL_CNT(stack) \ 1023 1023 + EMIT3_off32(0x48, 0x8B, 0x85, -round_up(stack, 8) - 8) 1024 1024 + 1072 1025 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image, 1073 1026 int oldproglen, struct jit_context *ctx, bool jmp_padding) 1074 1027 { ··· 1096 1041 1097 1042 emit_prologue(&prog, bpf_prog->aux->stack_depth, 1098 1043 bpf_prog_was_classic(bpf_prog), tail_call_reachable, 1099 1099 - bpf_prog->aux->func_idx != 0); 1100 1100 - push_callee_regs(&prog, callee_regs_used); 1044 1044 + bpf_is_subprog(bpf_prog), bpf_prog->aux->exception_cb); 1045 1045 + /* Exception callback will clobber callee regs for its own use, and 1046 1046 + * restore the original callee regs from main prog's stack frame. 1047 1047 + */ 1048 1048 + if (bpf_prog->aux->exception_boundary) { 1049 1049 + /* We also need to save r12, which is not mapped to any BPF 1050 1050 + * register, as we throw after entry into the kernel, which may 1051 1051 + * overwrite r12. 1052 1052 + */ 1053 1053 + push_r12(&prog); 1054 1054 + push_callee_regs(&prog, all_callee_regs_used); 1055 1055 + } else { 1056 1056 + push_callee_regs(&prog, callee_regs_used); 1057 1057 + } 1101 1058 1102 1059 ilen = prog - temp; 1103 1060 if (rw_image) ··· 1690 1623 1691 1624 func = (u8 *) __bpf_call_base + imm32; 1692 1625 if (tail_call_reachable) { 1693 1693 - /* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */ 1694 1694 - EMIT3_off32(0x48, 0x8B, 0x85, 1695 1695 - -round_up(bpf_prog->aux->stack_depth, 8) - 8); 1626 1626 + RESTORE_TAIL_CALL_CNT(bpf_prog->aux->stack_depth); 1696 1627 if (!imm32) 1697 1628 return -EINVAL; 1698 1629 offs = 7 + x86_call_depth_emit_accounting(&prog, func); ··· 1706 1641 1707 1642 case BPF_JMP | BPF_TAIL_CALL: 1708 1643 if (imm32) 1709 1709 - emit_bpf_tail_call_direct(&bpf_prog->aux->poke_tab[imm32 - 1], 1644 1644 + emit_bpf_tail_call_direct(bpf_prog, 1645 1645 + &bpf_prog->aux->poke_tab[imm32 - 1], 1710 1646 &prog, image + addrs[i - 1], 1711 1647 callee_regs_used, 1712 1648 bpf_prog->aux->stack_depth, 1713 1649 ctx); 1714 1650 else 1715 1715 - emit_bpf_tail_call_indirect(&prog, 1651 1651 + emit_bpf_tail_call_indirect(bpf_prog, 1652 1652 + &prog, 1716 1653 callee_regs_used, 1717 1654 bpf_prog->aux->stack_depth, 1718 1655 image + addrs[i - 1], ··· 1967 1900 seen_exit = true; 1968 1901 /* Update cleanup_addr */ 1969 1902 ctx->cleanup_addr = proglen; 1970 1970 - pop_callee_regs(&prog, callee_regs_used); 1903 1903 + if (bpf_prog->aux->exception_boundary) { 1904 1904 + pop_callee_regs(&prog, all_callee_regs_used); 1905 1905 + pop_r12(&prog); 1906 1906 + } else { 1907 1907 + pop_callee_regs(&prog, callee_regs_used); 1908 1908 + } 1971 1909 EMIT1(0xC9); /* leave */ 1972 1910 emit_return(&prog, image + addrs[i - 1] + (prog - temp)); 1973 1911 break; ··· 2472 2400 * [ ... ] 2473 2401 * [ stack_arg2 ] 2474 2402 * RBP - arg_stack_off [ stack_arg1 ] 2403 2403 + * RSP [ tail_call_cnt ] BPF_TRAMP_F_TAIL_CALL_CTX 2475 2404 */ 2476 2405 2477 2406 /* room for return value of orig_call or fentry prog */ ··· 2537 2464 else 2538 2465 /* sub rsp, stack_size */ 2539 2466 EMIT4(0x48, 0x83, 0xEC, stack_size); 2467 2467 + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 2468 2468 + EMIT1(0x50); /* push rax */ 2540 2469 /* mov QWORD PTR [rbp - rbx_off], rbx */ 2541 2470 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off); 2542 2471 ··· 2591 2516 restore_regs(m, &prog, regs_off); 2592 2517 save_args(m, &prog, arg_stack_off, true); 2593 2518 2519 2519 + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 2520 2520 + /* Before calling the original function, restore the 2521 2521 + * tail_call_cnt from stack to rax. 2522 2522 + */ 2523 2523 + RESTORE_TAIL_CALL_CNT(stack_size); 2524 2524 + 2594 2525 if (flags & BPF_TRAMP_F_ORIG_STACK) { 2595 2595 - emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8); 2596 2596 - EMIT2(0xff, 0xd0); /* call *rax */ 2526 2526 + emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, 8); 2527 2527 + EMIT2(0xff, 0xd3); /* call *rbx */ 2597 2528 } else { 2598 2529 /* call original function */ 2599 2530 if (emit_rsb_call(&prog, orig_call, prog)) { ··· 2650 2569 ret = -EINVAL; 2651 2570 goto cleanup; 2652 2571 } 2653 2653 - } 2572 2572 + } else if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 2573 2573 + /* Before running the original function, restore the 2574 2574 + * tail_call_cnt from stack to rax. 2575 2575 + */ 2576 2576 + RESTORE_TAIL_CALL_CNT(stack_size); 2577 2577 + 2654 2578 /* restore return value of orig_call or fentry prog back into RAX */ 2655 2579 if (save_ret) 2656 2580 emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8); ··· 2998 2912 } 2999 2913 3000 2914 bpf_prog_unlock_free(prog); 2915 2915 + } 2916 2916 + 2917 2917 + bool bpf_jit_supports_exceptions(void) 2918 2918 + { 2919 2919 + /* We unwind through both kernel frames (starting from within bpf_throw 2920 2920 + * call) and BPF frames. Therefore we require one of ORC or FP unwinder 2921 2921 + * to be enabled to walk kernel frames and reach BPF frames in the stack 2922 2922 + * trace. 2923 2923 + */ 2924 2924 + return IS_ENABLED(CONFIG_UNWINDER_ORC) || IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER); 2925 2925 + } 2926 2926 + 2927 2927 + void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie) 2928 2928 + { 2929 2929 + #if defined(CONFIG_UNWINDER_ORC) || defined(CONFIG_UNWINDER_FRAME_POINTER) 2930 2930 + struct unwind_state state; 2931 2931 + unsigned long addr; 2932 2932 + 2933 2933 + for (unwind_start(&state, current, NULL, NULL); !unwind_done(&state); 2934 2934 + unwind_next_frame(&state)) { 2935 2935 + addr = unwind_get_return_address(&state); 2936 2936 + if (!addr || !consume_fn(cookie, (u64)addr, (u64)state.sp, (u64)state.bp)) 2937 2937 + break; 2938 2938 + } 2939 2939 + return; 2940 2940 + #endif 2941 2941 + WARN(1, "verification of programs using bpf_throw should have failed\n"); 3001 2942 }

+31 -9

include/linux/bpf.h

reviewed

··· 55 55 extern struct idr btf_idr; 56 56 extern spinlock_t btf_idr_lock; 57 57 extern struct kobject *btf_kobj; 58 58 - extern struct bpf_mem_alloc bpf_global_ma; 59 59 - extern bool bpf_global_ma_set; 58 58 + extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma; 59 59 + extern bool bpf_global_ma_set, bpf_global_percpu_ma_set; 60 60 61 61 typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64); 62 62 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, ··· 180 180 BPF_TIMER = (1 << 1), 181 181 BPF_KPTR_UNREF = (1 << 2), 182 182 BPF_KPTR_REF = (1 << 3), 183 183 - BPF_KPTR = BPF_KPTR_UNREF | BPF_KPTR_REF, 184 184 - BPF_LIST_HEAD = (1 << 4), 185 185 - BPF_LIST_NODE = (1 << 5), 186 186 - BPF_RB_ROOT = (1 << 6), 187 187 - BPF_RB_NODE = (1 << 7), 183 183 + BPF_KPTR_PERCPU = (1 << 4), 184 184 + BPF_KPTR = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU, 185 185 + BPF_LIST_HEAD = (1 << 5), 186 186 + BPF_LIST_NODE = (1 << 6), 187 187 + BPF_RB_ROOT = (1 << 7), 188 188 + BPF_RB_NODE = (1 << 8), 188 189 BPF_GRAPH_NODE_OR_ROOT = BPF_LIST_NODE | BPF_LIST_HEAD | 189 190 BPF_RB_NODE | BPF_RB_ROOT, 190 190 - BPF_REFCOUNT = (1 << 8), 191 191 + BPF_REFCOUNT = (1 << 9), 191 192 }; 192 193 193 194 typedef void (*btf_dtor_kfunc_t)(void *); ··· 301 300 case BPF_KPTR_UNREF: 302 301 case BPF_KPTR_REF: 303 302 return "kptr"; 303 303 + case BPF_KPTR_PERCPU: 304 304 + return "percpu_kptr"; 304 305 case BPF_LIST_HEAD: 305 306 return "bpf_list_head"; 306 307 case BPF_LIST_NODE: ··· 328 325 return sizeof(struct bpf_timer); 329 326 case BPF_KPTR_UNREF: 330 327 case BPF_KPTR_REF: 328 328 + case BPF_KPTR_PERCPU: 331 329 return sizeof(u64); 332 330 case BPF_LIST_HEAD: 333 331 return sizeof(struct bpf_list_head); ··· 355 351 return __alignof__(struct bpf_timer); 356 352 case BPF_KPTR_UNREF: 357 353 case BPF_KPTR_REF: 354 354 + case BPF_KPTR_PERCPU: 358 355 return __alignof__(u64); 359 356 case BPF_LIST_HEAD: 360 357 return __alignof__(struct bpf_list_head); ··· 394 389 case BPF_TIMER: 395 390 case BPF_KPTR_UNREF: 396 391 case BPF_KPTR_REF: 392 392 + case BPF_KPTR_PERCPU: 397 393 break; 398 394 default: 399 395 WARN_ON_ONCE(1); ··· 1035 1029 */ 1036 1030 #define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6) 1037 1031 1032 1032 + /* Indicate that current trampoline is in a tail call context. Then, it has to 1033 1033 + * cache and restore tail_call_cnt to avoid infinite tail call loop. 1034 1034 + */ 1035 1035 + #define BPF_TRAMP_F_TAIL_CALL_CTX BIT(7) 1036 1036 + 1038 1037 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 1039 1038 * bytes on x86. 1040 1039 */ ··· 1389 1378 u32 stack_depth; 1390 1379 u32 id; 1391 1380 u32 func_cnt; /* used by non-func prog as the number of func progs */ 1381 1381 + u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */ 1392 1382 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ 1393 1383 u32 attach_btf_id; /* in-kernel BTF type id to attach to */ 1394 1384 u32 ctx_arg_info_size; ··· 1410 1398 bool sleepable; 1411 1399 bool tail_call_reachable; 1412 1400 bool xdp_has_frags; 1401 1401 + bool exception_cb; 1402 1402 + bool exception_boundary; 1413 1403 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ 1414 1404 const struct btf_type *attach_func_proto; 1415 1405 /* function name for valid attach_btf_id */ ··· 1434 1420 int cgroup_atype; /* enum cgroup_bpf_attach_type */ 1435 1421 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 1436 1422 char name[BPF_OBJ_NAME_LEN]; 1423 1423 + unsigned int (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp); 1437 1424 #ifdef CONFIG_SECURITY 1438 1425 void *security; 1439 1426 #endif ··· 2422 2407 int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, 2423 2408 struct bpf_reg_state *regs); 2424 2409 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, 2425 2425 - struct bpf_reg_state *reg); 2410 2410 + struct bpf_reg_state *reg, bool is_ex_cb); 2426 2411 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, 2427 2412 struct btf *btf, const struct btf_type *t); 2413 2413 + const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, 2414 2414 + int comp_idx, const char *tag_key); 2428 2415 2429 2416 struct bpf_prog *bpf_prog_by_id(u32 id); 2430 2417 struct bpf_link *bpf_link_by_id(u32 id); ··· 3198 3181 if (memcg_bpf_enabled()) 3199 3182 return flags | __GFP_ACCOUNT; 3200 3183 return flags; 3184 3184 + } 3185 3185 + 3186 3186 + static inline bool bpf_is_subprog(const struct bpf_prog *prog) 3187 3187 + { 3188 3188 + return prog->aux->func_idx != 0; 3201 3189 } 3202 3190 3203 3191 #endif /* _LINUX_BPF_H */

+8 -1

include/linux/bpf_verifier.h

reviewed

··· 300 300 bool in_callback_fn; 301 301 struct tnum callback_ret_range; 302 302 bool in_async_callback_fn; 303 303 + bool in_exception_callback_fn; 303 304 304 305 /* The following fields should be last. See copy_func_state() */ 305 306 int acquired_refs; ··· 481 480 bool zext_dst; /* this insn zero extends dst reg */ 482 481 bool storage_get_func_atomic; /* bpf_*_storage_get() with atomic memory alloc */ 483 482 bool is_iter_next; /* bpf_iter_<type>_next() kfunc call */ 483 483 + bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ 484 484 u8 alu_state; /* used in combination with alu_limit */ 485 485 486 486 /* below fields are initialized once */ ··· 542 540 bool has_tail_call; 543 541 bool tail_call_reachable; 544 542 bool has_ld_abs; 543 543 + bool is_cb; 545 544 bool is_async_cb; 545 545 + bool is_exception_cb; 546 546 }; 547 547 548 548 struct bpf_verifier_env; ··· 591 587 u32 used_map_cnt; /* number of used maps */ 592 588 u32 used_btf_cnt; /* number of used BTF objects */ 593 589 u32 id_gen; /* used to generate unique reg IDs */ 590 590 + u32 hidden_subprog_cnt; /* number of hidden subprogs */ 591 591 + int exception_callback_subprog; 594 592 bool explore_alu_limits; 595 593 bool allow_ptr_leaks; 596 594 bool allow_uninit_stack; ··· 600 594 bool bypass_spec_v1; 601 595 bool bypass_spec_v4; 602 596 bool seen_direct_write; 597 597 + bool seen_exception; 603 598 struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ 604 599 const struct bpf_line_info *prev_linfo; 605 600 struct bpf_verifier_log log; 606 606 - struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 1]; 601 601 + struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ 607 602 union { 608 603 struct bpf_idmap idmap_scratch; 609 604 struct bpf_idset idset_scratch;

+54 -4

include/linux/filter.h

reviewed

··· 117 117 118 118 /* ALU ops on immediates, bpf_add|sub|...: dst_reg += imm32 */ 119 119 120 120 - #define BPF_ALU64_IMM(OP, DST, IMM) \ 120 120 + #define BPF_ALU64_IMM_OFF(OP, DST, IMM, OFF) \ 121 121 ((struct bpf_insn) { \ 122 122 .code = BPF_ALU64 | BPF_OP(OP) | BPF_K, \ 123 123 .dst_reg = DST, \ 124 124 .src_reg = 0, \ 125 125 - .off = 0, \ 125 125 + .off = OFF, \ 126 126 .imm = IMM }) 127 127 + #define BPF_ALU64_IMM(OP, DST, IMM) \ 128 128 + BPF_ALU64_IMM_OFF(OP, DST, IMM, 0) 127 129 128 128 - #define BPF_ALU32_IMM(OP, DST, IMM) \ 130 130 + #define BPF_ALU32_IMM_OFF(OP, DST, IMM, OFF) \ 129 131 ((struct bpf_insn) { \ 130 132 .code = BPF_ALU | BPF_OP(OP) | BPF_K, \ 131 133 .dst_reg = DST, \ 132 134 .src_reg = 0, \ 133 133 - .off = 0, \ 135 135 + .off = OFF, \ 134 136 .imm = IMM }) 137 137 + #define BPF_ALU32_IMM(OP, DST, IMM) \ 138 138 + BPF_ALU32_IMM_OFF(OP, DST, IMM, 0) 135 139 136 140 /* Endianess conversion, cpu_to_{l,b}e(), {l,b}e_to_cpu() */ 137 141 138 142 #define BPF_ENDIAN(TYPE, DST, LEN) \ 139 143 ((struct bpf_insn) { \ 140 144 .code = BPF_ALU | BPF_END | BPF_SRC(TYPE), \ 145 145 + .dst_reg = DST, \ 146 146 + .src_reg = 0, \ 147 147 + .off = 0, \ 148 148 + .imm = LEN }) 149 149 + 150 150 + /* Byte Swap, bswap16/32/64 */ 151 151 + 152 152 + #define BPF_BSWAP(DST, LEN) \ 153 153 + ((struct bpf_insn) { \ 154 154 + .code = BPF_ALU64 | BPF_END | BPF_SRC(BPF_TO_LE), \ 141 155 .dst_reg = DST, \ 142 156 .src_reg = 0, \ 143 157 .off = 0, \ ··· 192 178 .src_reg = 0, \ 193 179 .off = 0, \ 194 180 .imm = IMM }) 181 181 + 182 182 + /* Short form of movsx, dst_reg = (s8,s16,s32)src_reg */ 183 183 + 184 184 + #define BPF_MOVSX64_REG(DST, SRC, OFF) \ 185 185 + ((struct bpf_insn) { \ 186 186 + .code = BPF_ALU64 | BPF_MOV | BPF_X, \ 187 187 + .dst_reg = DST, \ 188 188 + .src_reg = SRC, \ 189 189 + .off = OFF, \ 190 190 + .imm = 0 }) 191 191 + 192 192 + #define BPF_MOVSX32_REG(DST, SRC, OFF) \ 193 193 + ((struct bpf_insn) { \ 194 194 + .code = BPF_ALU | BPF_MOV | BPF_X, \ 195 195 + .dst_reg = DST, \ 196 196 + .src_reg = SRC, \ 197 197 + .off = OFF, \ 198 198 + .imm = 0 }) 195 199 196 200 /* Special form of mov32, used for doing explicit zero extension on dst. */ 197 201 #define BPF_ZEXT_REG(DST) \ ··· 290 258 #define BPF_LDX_MEM(SIZE, DST, SRC, OFF) \ 291 259 ((struct bpf_insn) { \ 292 260 .code = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEM, \ 261 261 + .dst_reg = DST, \ 262 262 + .src_reg = SRC, \ 263 263 + .off = OFF, \ 264 264 + .imm = 0 }) 265 265 + 266 266 + /* Memory load, dst_reg = *(signed size *) (src_reg + off16) */ 267 267 + 268 268 + #define BPF_LDX_MEMSX(SIZE, DST, SRC, OFF) \ 269 269 + ((struct bpf_insn) { \ 270 270 + .code = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEMSX, \ 293 271 .dst_reg = DST, \ 294 272 .src_reg = SRC, \ 295 273 .off = OFF, \ ··· 954 912 bool bpf_jit_supports_subprog_tailcalls(void); 955 913 bool bpf_jit_supports_kfunc_call(void); 956 914 bool bpf_jit_supports_far_kfunc_call(void); 915 915 + bool bpf_jit_supports_exceptions(void); 916 916 + void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); 957 917 bool bpf_helper_changes_pkt_data(void *func); 958 918 959 919 static inline bool bpf_dump_raw_ok(const struct cred *cred) ··· 1171 1127 bool is_bpf_text_address(unsigned long addr); 1172 1128 int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, 1173 1129 char *sym); 1130 1130 + struct bpf_prog *bpf_prog_ksym_find(unsigned long addr); 1174 1131 1175 1132 static inline const char * 1176 1133 bpf_address_lookup(unsigned long addr, unsigned long *size, ··· 1237 1192 char *type, char *sym) 1238 1193 { 1239 1194 return -ERANGE; 1195 1195 + } 1196 1196 + 1197 1197 + static inline struct bpf_prog *bpf_prog_ksym_find(unsigned long addr) 1198 1198 + { 1199 1199 + return NULL; 1240 1200 } 1241 1201 1242 1202 static inline const char *

+2

include/linux/kasan.h

reviewed

··· 285 285 286 286 #if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK) 287 287 void kasan_unpoison_task_stack(struct task_struct *task); 288 288 + asmlinkage void kasan_unpoison_task_stack_below(const void *watermark); 288 289 #else 289 290 static inline void kasan_unpoison_task_stack(struct task_struct *task) {} 291 291 + static inline void kasan_unpoison_task_stack_below(const void *watermark) {} 290 292 #endif 291 293 292 294 #ifdef CONFIG_KASAN_GENERIC

+15 -4

include/net/xdp.h

reviewed

··· 383 383 384 384 #define DEV_MAP_BULK_SIZE XDP_BULK_QUEUE_SIZE 385 385 386 386 + /* Define the relationship between xdp-rx-metadata kfunc and 387 387 + * various other entities: 388 388 + * - xdp_rx_metadata enum 389 389 + * - netdev netlink enum (Documentation/netlink/specs/netdev.yaml) 390 390 + * - kfunc name 391 391 + * - xdp_metadata_ops field 392 392 + */ 386 393 #define XDP_METADATA_KFUNC_xxx \ 387 394 XDP_METADATA_KFUNC(XDP_METADATA_KFUNC_RX_TIMESTAMP, \ 388 388 - bpf_xdp_metadata_rx_timestamp) \ 395 395 + NETDEV_XDP_RX_METADATA_TIMESTAMP, \ 396 396 + bpf_xdp_metadata_rx_timestamp, \ 397 397 + xmo_rx_timestamp) \ 389 398 XDP_METADATA_KFUNC(XDP_METADATA_KFUNC_RX_HASH, \ 390 390 - bpf_xdp_metadata_rx_hash) \ 399 399 + NETDEV_XDP_RX_METADATA_HASH, \ 400 400 + bpf_xdp_metadata_rx_hash, \ 401 401 + xmo_rx_hash) \ 391 402 392 392 - enum { 393 393 - #define XDP_METADATA_KFUNC(name, _) name, 403 403 + enum xdp_rx_metadata { 404 404 + #define XDP_METADATA_KFUNC(name, _, __, ___) name, 394 405 XDP_METADATA_KFUNC_xxx 395 406 #undef XDP_METADATA_KFUNC 396 407 MAX_XDP_METADATA_KFUNC,

+2

include/net/xdp_sock.h

reviewed

··· 14 14 #include <linux/mm.h> 15 15 #include <net/sock.h> 16 16 17 17 + #define XDP_UMEM_SG_FLAG (1 << 1) 18 18 + 17 19 struct net_device; 18 20 struct xsk_queue; 19 21 struct xdp_buff;

+8 -1

include/uapi/linux/bpf.h

reviewed

··· 932 932 */ 933 933 BPF_MAP_TYPE_CGROUP_STORAGE = BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED, 934 934 BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, 935 935 - BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, 935 935 + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED, 936 936 + /* BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE is available to bpf programs 937 937 + * attaching to a cgroup. The new mechanism (BPF_MAP_TYPE_CGRP_STORAGE + 938 938 + * local percpu kptr) supports all BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE 939 939 + * functionality and more. So mark * BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE 940 940 + * deprecated. 941 941 + */ 942 942 + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE = BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED, 936 943 BPF_MAP_TYPE_QUEUE, 937 944 BPF_MAP_TYPE_STACK, 938 945 BPF_MAP_TYPE_SK_STORAGE,

+16

include/uapi/linux/netdev.h

reviewed

··· 38 38 NETDEV_XDP_ACT_MASK = 127, 39 39 }; 40 40 41 41 + /** 42 42 + * enum netdev_xdp_rx_metadata 43 43 + * @NETDEV_XDP_RX_METADATA_TIMESTAMP: Device is capable of exposing receive HW 44 44 + * timestamp via bpf_xdp_metadata_rx_timestamp(). 45 45 + * @NETDEV_XDP_RX_METADATA_HASH: Device is capable of exposing receive packet 46 46 + * hash via bpf_xdp_metadata_rx_hash(). 47 47 + */ 48 48 + enum netdev_xdp_rx_metadata { 49 49 + NETDEV_XDP_RX_METADATA_TIMESTAMP = 1, 50 50 + NETDEV_XDP_RX_METADATA_HASH = 2, 51 51 + 52 52 + /* private: */ 53 53 + NETDEV_XDP_RX_METADATA_MASK = 3, 54 54 + }; 55 55 + 41 56 enum { 42 57 NETDEV_A_DEV_IFINDEX = 1, 43 58 NETDEV_A_DEV_PAD, 44 59 NETDEV_A_DEV_XDP_FEATURES, 45 60 NETDEV_A_DEV_XDP_ZC_MAX_SEGS, 61 61 + NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, 46 62 47 63 __NETDEV_A_DEV_MAX, 48 64 NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)

+22 -4

kernel/bpf/bpf_struct_ops.c

reviewed

··· 615 615 if (st_map->links) 616 616 bpf_struct_ops_map_put_progs(st_map); 617 617 bpf_map_area_free(st_map->links); 618 618 - bpf_jit_free_exec(st_map->image); 618 618 + if (st_map->image) { 619 619 + bpf_jit_free_exec(st_map->image); 620 620 + bpf_jit_uncharge_modmem(PAGE_SIZE); 621 621 + } 619 622 bpf_map_area_free(st_map->uvalue); 620 623 bpf_map_area_free(st_map); 621 624 } ··· 660 657 struct bpf_struct_ops_map *st_map; 661 658 const struct btf_type *t, *vt; 662 659 struct bpf_map *map; 660 660 + int ret; 663 661 664 662 st_ops = bpf_struct_ops_find_value(attr->btf_vmlinux_value_type_id); 665 663 if (!st_ops) ··· 685 681 st_map->st_ops = st_ops; 686 682 map = &st_map->map; 687 683 684 684 + ret = bpf_jit_charge_modmem(PAGE_SIZE); 685 685 + if (ret) { 686 686 + __bpf_struct_ops_map_free(map); 687 687 + return ERR_PTR(ret); 688 688 + } 689 689 + 690 690 + st_map->image = bpf_jit_alloc_exec(PAGE_SIZE); 691 691 + if (!st_map->image) { 692 692 + /* __bpf_struct_ops_map_free() uses st_map->image as flag 693 693 + * for "charged or not". In this case, we need to unchange 694 694 + * here. 695 695 + */ 696 696 + bpf_jit_uncharge_modmem(PAGE_SIZE); 697 697 + __bpf_struct_ops_map_free(map); 698 698 + return ERR_PTR(-ENOMEM); 699 699 + } 688 700 st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE); 689 701 st_map->links = 690 702 bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_links *), 691 703 NUMA_NO_NODE); 692 692 - st_map->image = bpf_jit_alloc_exec(PAGE_SIZE); 693 693 - if (!st_map->uvalue || !st_map->links || !st_map->image) { 704 704 + if (!st_map->uvalue || !st_map->links) { 694 705 __bpf_struct_ops_map_free(map); 695 706 return ERR_PTR(-ENOMEM); 696 707 } ··· 926 907 kfree(link); 927 908 return err; 928 909 } 929 929 -

+26 -8

kernel/bpf/btf.c

reviewed

··· 3293 3293 type = BPF_KPTR_UNREF; 3294 3294 else if (!strcmp("kptr", __btf_name_by_offset(btf, t->name_off))) 3295 3295 type = BPF_KPTR_REF; 3296 3296 + else if (!strcmp("percpu_kptr", __btf_name_by_offset(btf, t->name_off))) 3297 3297 + type = BPF_KPTR_PERCPU; 3296 3298 else 3297 3299 return -EINVAL; 3298 3300 ··· 3310 3308 return BTF_FIELD_FOUND; 3311 3309 } 3312 3310 3313 3313 - static const char *btf_find_decl_tag_value(const struct btf *btf, 3314 3314 - const struct btf_type *pt, 3315 3315 - int comp_idx, const char *tag_key) 3311 3311 + const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, 3312 3312 + int comp_idx, const char *tag_key) 3316 3313 { 3314 3314 + const char *value = NULL; 3317 3315 int i; 3318 3316 3319 3317 for (i = 1; i < btf_nr_types(btf); i++) { ··· 3327 3325 continue; 3328 3326 if (strncmp(__btf_name_by_offset(btf, t->name_off), tag_key, len)) 3329 3327 continue; 3330 3330 - return __btf_name_by_offset(btf, t->name_off) + len; 3328 3328 + /* Prevent duplicate entries for same type */ 3329 3329 + if (value) 3330 3330 + return ERR_PTR(-EEXIST); 3331 3331 + value = __btf_name_by_offset(btf, t->name_off) + len; 3331 3332 } 3332 3332 - return NULL; 3333 3333 + if (!value) 3334 3334 + return ERR_PTR(-ENOENT); 3335 3335 + return value; 3333 3336 } 3334 3337 3335 3338 static int ··· 3352 3345 if (t->size != sz) 3353 3346 return BTF_FIELD_IGNORE; 3354 3347 value_type = btf_find_decl_tag_value(btf, pt, comp_idx, "contains:"); 3355 3355 - if (!value_type) 3348 3348 + if (IS_ERR(value_type)) 3356 3349 return -EINVAL; 3357 3350 node_field_name = strstr(value_type, ":"); 3358 3351 if (!node_field_name) ··· 3464 3457 break; 3465 3458 case BPF_KPTR_UNREF: 3466 3459 case BPF_KPTR_REF: 3460 3460 + case BPF_KPTR_PERCPU: 3467 3461 ret = btf_find_kptr(btf, member_type, off, sz, 3468 3462 idx < info_cnt ? &info[idx] : &tmp); 3469 3463 if (ret < 0) ··· 3531 3523 break; 3532 3524 case BPF_KPTR_UNREF: 3533 3525 case BPF_KPTR_REF: 3526 3526 + case BPF_KPTR_PERCPU: 3534 3527 ret = btf_find_kptr(btf, var_type, off, sz, 3535 3528 idx < info_cnt ? &info[idx] : &tmp); 3536 3529 if (ret < 0) ··· 3792 3783 break; 3793 3784 case BPF_KPTR_UNREF: 3794 3785 case BPF_KPTR_REF: 3786 3786 + case BPF_KPTR_PERCPU: 3795 3787 ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]); 3796 3788 if (ret < 0) 3797 3789 goto end; ··· 6959 6949 * (either PTR_TO_CTX or SCALAR_VALUE). 6960 6950 */ 6961 6951 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, 6962 6962 - struct bpf_reg_state *regs) 6952 6952 + struct bpf_reg_state *regs, bool is_ex_cb) 6963 6953 { 6964 6954 struct bpf_verifier_log *log = &env->log; 6965 6955 struct bpf_prog *prog = env->prog; ··· 7016 7006 tname, nargs, MAX_BPF_FUNC_REG_ARGS); 7017 7007 return -EINVAL; 7018 7008 } 7019 7019 - /* check that function returns int */ 7009 7009 + /* check that function returns int, exception cb also requires this */ 7020 7010 t = btf_type_by_id(btf, t->type); 7021 7011 while (btf_type_is_modifier(t)) 7022 7012 t = btf_type_by_id(btf, t->type); ··· 7063 7053 } 7064 7054 bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n", 7065 7055 i, btf_type_str(t), tname); 7056 7056 + return -EINVAL; 7057 7057 + } 7058 7058 + /* We have already ensured that the callback returns an integer, just 7059 7059 + * like all global subprogs. We need to determine it only has a single 7060 7060 + * scalar argument. 7061 7061 + */ 7062 7062 + if (is_ex_cb && (nargs != 1 || regs[BPF_REG_1].type != SCALAR_VALUE)) { 7063 7063 + bpf_log(log, "exception cb only supports single integer argument\n"); 7066 7064 return -EINVAL; 7067 7065 } 7068 7066 return 0;

+26 -11

kernel/bpf/core.c

reviewed

··· 64 64 #define OFF insn->off 65 65 #define IMM insn->imm 66 66 67 67 - struct bpf_mem_alloc bpf_global_ma; 68 68 - bool bpf_global_ma_set; 67 67 + struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma; 68 68 + bool bpf_global_ma_set, bpf_global_percpu_ma_set; 69 69 70 70 /* No hurry in this branch 71 71 * ··· 212 212 const struct bpf_line_info *linfo; 213 213 void **jited_linfo; 214 214 215 215 - if (!prog->aux->jited_linfo) 215 215 + if (!prog->aux->jited_linfo || prog->aux->func_idx > prog->aux->func_cnt) 216 216 /* Userspace did not provide linfo */ 217 217 return; 218 218 ··· 539 539 { 540 540 int i; 541 541 542 542 - for (i = 0; i < fp->aux->func_cnt; i++) 542 542 + for (i = 0; i < fp->aux->real_func_cnt; i++) 543 543 bpf_prog_kallsyms_del(fp->aux->func[i]); 544 544 } 545 545 ··· 589 589 sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); 590 590 591 591 /* prog->aux->name will be ignored if full btf name is available */ 592 592 - if (prog->aux->func_info_cnt) { 592 592 + if (prog->aux->func_info_cnt && prog->aux->func_idx < prog->aux->func_info_cnt) { 593 593 type = btf_type_by_id(prog->aux->btf, 594 594 prog->aux->func_info[prog->aux->func_idx].type_id); 595 595 func_name = btf_name_by_offset(prog->aux->btf, type->name_off); ··· 623 623 624 624 if (val < ksym->start) 625 625 return -1; 626 626 - if (val >= ksym->end) 626 626 + /* Ensure that we detect return addresses as part of the program, when 627 627 + * the final instruction is a call for a program part of the stack 628 628 + * trace. Therefore, do val > ksym->end instead of val >= ksym->end. 629 629 + */ 630 630 + if (val > ksym->end) 627 631 return 1; 628 632 629 633 return 0; ··· 737 733 return ret; 738 734 } 739 735 740 740 - static struct bpf_prog *bpf_prog_ksym_find(unsigned long addr) 736 736 + struct bpf_prog *bpf_prog_ksym_find(unsigned long addr) 741 737 { 742 738 struct bpf_ksym *ksym = bpf_ksym_find(addr); 743 739 ··· 1212 1208 if (!extra_pass) 1213 1209 addr = NULL; 1214 1210 else if (prog->aux->func && 1215 1215 - off >= 0 && off < prog->aux->func_cnt) 1211 1211 + off >= 0 && off < prog->aux->real_func_cnt) 1216 1212 addr = (u8 *)prog->aux->func[off]->bpf_func; 1217 1213 else 1218 1214 return -EINVAL; ··· 2725 2721 #endif 2726 2722 if (aux->dst_trampoline) 2727 2723 bpf_trampoline_put(aux->dst_trampoline); 2728 2728 - for (i = 0; i < aux->func_cnt; i++) { 2724 2724 + for (i = 0; i < aux->real_func_cnt; i++) { 2729 2725 /* We can just unlink the subprog poke descriptor table as 2730 2726 * it was originally linked to the main program and is also 2731 2727 * released along with it. ··· 2733 2729 aux->func[i]->aux->poke_tab = NULL; 2734 2730 bpf_jit_free(aux->func[i]); 2735 2731 } 2736 2736 - if (aux->func_cnt) { 2732 2732 + if (aux->real_func_cnt) { 2737 2733 kfree(aux->func); 2738 2734 bpf_prog_unlock_free(aux->prog); 2739 2735 } else { ··· 2918 2914 return -ENOTSUPP; 2919 2915 } 2920 2916 2917 2917 + bool __weak bpf_jit_supports_exceptions(void) 2918 2918 + { 2919 2919 + return false; 2920 2920 + } 2921 2921 + 2922 2922 + void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie) 2923 2923 + { 2924 2924 + } 2925 2925 + 2921 2926 #ifdef CONFIG_BPF_SYSCALL 2922 2927 static int __init bpf_global_ma_init(void) 2923 2928 { ··· 2934 2921 2935 2922 ret = bpf_mem_alloc_init(&bpf_global_ma, 0, false); 2936 2923 bpf_global_ma_set = !ret; 2937 2937 - return ret; 2924 2924 + ret = bpf_mem_alloc_init(&bpf_global_percpu_ma, 0, true); 2925 2925 + bpf_global_percpu_ma_set = !ret; 2926 2926 + return !bpf_global_ma_set || !bpf_global_percpu_ma_set; 2938 2927 } 2939 2928 late_initcall(bpf_global_ma_init); 2940 2929 #endif

+61

kernel/bpf/helpers.c

reviewed

··· 22 22 #include <linux/security.h> 23 23 #include <linux/btf_ids.h> 24 24 #include <linux/bpf_mem_alloc.h> 25 25 + #include <linux/kasan.h> 25 26 26 27 #include "../../lib/kstrtox.h" 27 28 ··· 1903 1902 return p; 1904 1903 } 1905 1904 1905 1905 + __bpf_kfunc void *bpf_percpu_obj_new_impl(u64 local_type_id__k, void *meta__ign) 1906 1906 + { 1907 1907 + u64 size = local_type_id__k; 1908 1908 + 1909 1909 + /* The verifier has ensured that meta__ign must be NULL */ 1910 1910 + return bpf_mem_alloc(&bpf_global_percpu_ma, size); 1911 1911 + } 1912 1912 + 1906 1913 /* Must be called under migrate_disable(), as required by bpf_mem_free */ 1907 1914 void __bpf_obj_drop_impl(void *p, const struct btf_record *rec) 1908 1915 { ··· 1937 1928 void *p = p__alloc; 1938 1929 1939 1930 __bpf_obj_drop_impl(p, meta ? meta->record : NULL); 1931 1931 + } 1932 1932 + 1933 1933 + __bpf_kfunc void bpf_percpu_obj_drop_impl(void *p__alloc, void *meta__ign) 1934 1934 + { 1935 1935 + /* The verifier has ensured that meta__ign must be NULL */ 1936 1936 + bpf_mem_free_rcu(&bpf_global_percpu_ma, p__alloc); 1940 1937 } 1941 1938 1942 1939 __bpf_kfunc void *bpf_refcount_acquire_impl(void *p__refcounted_kptr, void *meta__ign) ··· 2450 2435 rcu_read_unlock(); 2451 2436 } 2452 2437 2438 2438 + struct bpf_throw_ctx { 2439 2439 + struct bpf_prog_aux *aux; 2440 2440 + u64 sp; 2441 2441 + u64 bp; 2442 2442 + int cnt; 2443 2443 + }; 2444 2444 + 2445 2445 + static bool bpf_stack_walker(void *cookie, u64 ip, u64 sp, u64 bp) 2446 2446 + { 2447 2447 + struct bpf_throw_ctx *ctx = cookie; 2448 2448 + struct bpf_prog *prog; 2449 2449 + 2450 2450 + if (!is_bpf_text_address(ip)) 2451 2451 + return !ctx->cnt; 2452 2452 + prog = bpf_prog_ksym_find(ip); 2453 2453 + ctx->cnt++; 2454 2454 + if (bpf_is_subprog(prog)) 2455 2455 + return true; 2456 2456 + ctx->aux = prog->aux; 2457 2457 + ctx->sp = sp; 2458 2458 + ctx->bp = bp; 2459 2459 + return false; 2460 2460 + } 2461 2461 + 2462 2462 + __bpf_kfunc void bpf_throw(u64 cookie) 2463 2463 + { 2464 2464 + struct bpf_throw_ctx ctx = {}; 2465 2465 + 2466 2466 + arch_bpf_stack_walk(bpf_stack_walker, &ctx); 2467 2467 + WARN_ON_ONCE(!ctx.aux); 2468 2468 + if (ctx.aux) 2469 2469 + WARN_ON_ONCE(!ctx.aux->exception_boundary); 2470 2470 + WARN_ON_ONCE(!ctx.bp); 2471 2471 + WARN_ON_ONCE(!ctx.cnt); 2472 2472 + /* Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning 2473 2473 + * deeper stack depths than ctx.sp as we do not return from bpf_throw, 2474 2474 + * which skips compiler generated instrumentation to do the same. 2475 2475 + */ 2476 2476 + kasan_unpoison_task_stack_below((void *)ctx.sp); 2477 2477 + ctx.aux->bpf_exception_cb(cookie, ctx.sp, ctx.bp); 2478 2478 + WARN(1, "A call to BPF exception callback should never return\n"); 2479 2479 + } 2480 2480 + 2453 2481 __diag_pop(); 2454 2482 2455 2483 BTF_SET8_START(generic_btf_ids) ··· 2500 2442 BTF_ID_FLAGS(func, crash_kexec, KF_DESTRUCTIVE) 2501 2443 #endif 2502 2444 BTF_ID_FLAGS(func, bpf_obj_new_impl, KF_ACQUIRE | KF_RET_NULL) 2445 2445 + BTF_ID_FLAGS(func, bpf_percpu_obj_new_impl, KF_ACQUIRE | KF_RET_NULL) 2503 2446 BTF_ID_FLAGS(func, bpf_obj_drop_impl, KF_RELEASE) 2447 2447 + BTF_ID_FLAGS(func, bpf_percpu_obj_drop_impl, KF_RELEASE) 2504 2448 BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, KF_ACQUIRE | KF_RET_NULL) 2505 2449 BTF_ID_FLAGS(func, bpf_list_push_front_impl) 2506 2450 BTF_ID_FLAGS(func, bpf_list_push_back_impl) ··· 2522 2462 BTF_ID_FLAGS(func, bpf_task_under_cgroup, KF_RCU) 2523 2463 #endif 2524 2464 BTF_ID_FLAGS(func, bpf_task_from_pid, KF_ACQUIRE | KF_RET_NULL) 2465 2465 + BTF_ID_FLAGS(func, bpf_throw) 2525 2466 BTF_SET8_END(generic_btf_ids) 2526 2467 2527 2468 static const struct btf_kfunc_id_set generic_kfunc_set = {

+19 -11

kernel/bpf/memalloc.c

reviewed

··· 499 499 struct obj_cgroup *objcg = NULL; 500 500 int cpu, i, unit_size, percpu_size = 0; 501 501 502 502 + /* room for llist_node and per-cpu pointer */ 503 503 + if (percpu) 504 504 + percpu_size = LLIST_NODE_SZ + sizeof(void *); 505 505 + 502 506 if (size) { 503 507 pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL); 504 508 if (!pc) 505 509 return -ENOMEM; 506 510 507 507 - if (percpu) 508 508 - /* room for llist_node and per-cpu pointer */ 509 509 - percpu_size = LLIST_NODE_SZ + sizeof(void *); 510 510 - else 511 511 + if (!percpu) 511 512 size += LLIST_NODE_SZ; /* room for llist_node */ 512 513 unit_size = size; 513 514 ··· 528 527 return 0; 529 528 } 530 529 531 531 - /* size == 0 && percpu is an invalid combination */ 532 532 - if (WARN_ON_ONCE(percpu)) 533 533 - return -EINVAL; 534 534 - 535 530 pcc = __alloc_percpu_gfp(sizeof(*cc), 8, GFP_KERNEL); 536 531 if (!pcc) 537 532 return -ENOMEM; ··· 540 543 c = &cc->cache[i]; 541 544 c->unit_size = sizes[i]; 542 545 c->objcg = objcg; 546 546 + c->percpu_size = percpu_size; 543 547 c->tgt = c; 544 548 prefill_mem_cache(c, cpu); 545 549 } ··· 732 734 } 733 735 } 734 736 local_dec(&c->active); 735 735 - local_irq_restore(flags); 736 737 737 738 WARN_ON(cnt < 0); 738 739 739 740 if (cnt < c->low_watermark) 740 741 irq_work_raise(c); 742 742 + /* Enable IRQ after the enqueue of irq work completes, so irq work 743 743 + * will run after IRQ is enabled and free_llist may be refilled by 744 744 + * irq work before other task preempts current task. 745 745 + */ 746 746 + local_irq_restore(flags); 747 747 + 741 748 return llnode; 742 749 } 743 750 ··· 778 775 llist_add(llnode, &c->free_llist_extra); 779 776 } 780 777 local_dec(&c->active); 781 781 - local_irq_restore(flags); 782 778 783 779 if (cnt > c->high_watermark) 784 780 /* free few objects from current cpu into global kmalloc pool */ 785 781 irq_work_raise(c); 782 782 + /* Enable IRQ after irq_work_raise() completes, otherwise when current 783 783 + * task is preempted by task which does unit_alloc(), unit_alloc() may 784 784 + * return NULL unexpectedly because irq work is already pending but can 785 785 + * not been triggered and free_llist can not be refilled timely. 786 786 + */ 787 787 + local_irq_restore(flags); 786 788 } 787 789 788 790 static void notrace unit_free_rcu(struct bpf_mem_cache *c, void *ptr) ··· 805 797 llist_add(llnode, &c->free_llist_extra_rcu); 806 798 } 807 799 local_dec(&c->active); 808 808 - local_irq_restore(flags); 809 800 810 801 if (!atomic_read(&c->call_rcu_in_progress)) 811 802 irq_work_raise(c); 803 803 + local_irq_restore(flags); 812 804 } 813 805 814 806 /* Called from BPF program or from sys_bpf syscall.

+13 -5

kernel/bpf/offload.c

reviewed

··· 232 232 attr->prog_type != BPF_PROG_TYPE_XDP) 233 233 return -EINVAL; 234 234 235 235 - if (attr->prog_flags & ~BPF_F_XDP_DEV_BOUND_ONLY) 235 235 + if (attr->prog_flags & ~(BPF_F_XDP_DEV_BOUND_ONLY | BPF_F_XDP_HAS_FRAGS)) 236 236 + return -EINVAL; 237 237 + 238 238 + /* Frags are allowed only if program is dev-bound-only, but not 239 239 + * if it is requesting bpf offload. 240 240 + */ 241 241 + if (attr->prog_flags & BPF_F_XDP_HAS_FRAGS && 242 242 + !(attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY)) 236 243 return -EINVAL; 237 244 238 245 if (attr->prog_type == BPF_PROG_TYPE_SCHED_CLS && ··· 852 845 if (!ops) 853 846 goto out; 854 847 855 855 - if (func_id == bpf_xdp_metadata_kfunc_id(XDP_METADATA_KFUNC_RX_TIMESTAMP)) 856 856 - p = ops->xmo_rx_timestamp; 857 857 - else if (func_id == bpf_xdp_metadata_kfunc_id(XDP_METADATA_KFUNC_RX_HASH)) 858 858 - p = ops->xmo_rx_hash; 848 848 + #define XDP_METADATA_KFUNC(name, _, __, xmo) \ 849 849 + if (func_id == bpf_xdp_metadata_kfunc_id(name)) p = ops->xmo; 850 850 + XDP_METADATA_KFUNC_xxx 851 851 + #undef XDP_METADATA_KFUNC 852 852 + 859 853 out: 860 854 up_read(&bpf_devs_lock); 861 855

+5 -1

kernel/bpf/syscall.c

reviewed

··· 514 514 switch (rec->fields[i].type) { 515 515 case BPF_KPTR_UNREF: 516 516 case BPF_KPTR_REF: 517 517 + case BPF_KPTR_PERCPU: 517 518 if (rec->fields[i].kptr.module) 518 519 module_put(rec->fields[i].kptr.module); 519 520 btf_put(rec->fields[i].kptr.btf); ··· 561 560 switch (fields[i].type) { 562 561 case BPF_KPTR_UNREF: 563 562 case BPF_KPTR_REF: 563 563 + case BPF_KPTR_PERCPU: 564 564 btf_get(fields[i].kptr.btf); 565 565 if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) { 566 566 ret = -ENXIO; ··· 652 650 WRITE_ONCE(*(u64 *)field_ptr, 0); 653 651 break; 654 652 case BPF_KPTR_REF: 653 653 + case BPF_KPTR_PERCPU: 655 654 xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0); 656 655 if (!xchgd_field) 657 656 break; ··· 1048 1045 break; 1049 1046 case BPF_KPTR_UNREF: 1050 1047 case BPF_KPTR_REF: 1048 1048 + case BPF_KPTR_PERCPU: 1051 1049 case BPF_REFCOUNT: 1052 1050 if (map->map_type != BPF_MAP_TYPE_HASH && 1053 1051 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && ··· 2749 2745 * period before we can tear down JIT memory since symbols 2750 2746 * are already exposed under kallsyms. 2751 2747 */ 2752 2752 - __bpf_prog_put_noref(prog, prog->aux->func_cnt); 2748 2748 + __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); 2753 2749 return err; 2754 2750 free_prog_sec: 2755 2751 free_uid(prog->aux->user);

+10 -30

kernel/bpf/task_iter.c

reviewed

··· 35 35 u32 *tid, 36 36 bool skip_if_dup_files) 37 37 { 38 38 - struct task_struct *task, *next_task; 38 38 + struct task_struct *task; 39 39 struct pid *pid; 40 40 - u32 saved_tid; 40 40 + u32 next_tid; 41 41 42 42 if (!*tid) { 43 43 /* The first time, the iterator calls this function. */ 44 44 pid = find_pid_ns(common->pid, common->ns); 45 45 - if (!pid) 46 46 - return NULL; 47 47 - 48 45 task = get_pid_task(pid, PIDTYPE_TGID); 49 46 if (!task) 50 47 return NULL; ··· 63 66 return task; 64 67 } 65 68 66 66 - pid = find_pid_ns(common->pid_visiting, common->ns); 67 67 - if (!pid) 68 68 - return NULL; 69 69 - 70 70 - task = get_pid_task(pid, PIDTYPE_PID); 69 69 + task = find_task_by_pid_ns(common->pid_visiting, common->ns); 71 70 if (!task) 72 71 return NULL; 73 72 74 73 retry: 75 75 - if (!pid_alive(task)) { 76 76 - put_task_struct(task); 77 77 - return NULL; 78 78 - } 74 74 + task = next_thread(task); 79 75 80 80 - next_task = next_thread(task); 81 81 - put_task_struct(task); 82 82 - if (!next_task) 83 83 - return NULL; 84 84 - 85 85 - saved_tid = *tid; 86 86 - *tid = __task_pid_nr_ns(next_task, PIDTYPE_PID, common->ns); 87 87 - if (!*tid || *tid == common->pid) { 76 76 + next_tid = __task_pid_nr_ns(task, PIDTYPE_PID, common->ns); 77 77 + if (!next_tid || next_tid == common->pid) { 88 78 /* Run out of tasks of a process. The tasks of a 89 79 * thread_group are linked as circular linked list. 90 80 */ 91 91 - *tid = saved_tid; 92 81 return NULL; 93 82 } 94 83 95 95 - get_task_struct(next_task); 96 96 - common->pid_visiting = *tid; 97 97 - 98 98 - if (skip_if_dup_files && task->files == task->group_leader->files) { 99 99 - task = next_task; 84 84 + if (skip_if_dup_files && task->files == task->group_leader->files) 100 85 goto retry; 101 101 - } 102 86 103 103 - return next_task; 87 87 + *tid = common->pid_visiting = next_tid; 88 88 + get_task_struct(task); 89 89 + return task; 104 90 } 105 91 106 92 static struct task_struct *task_seq_get_next(struct bpf_iter_seq_task_common *common,

+2 -2

kernel/bpf/trampoline.c

reviewed

··· 415 415 goto out; 416 416 } 417 417 418 418 - /* clear all bits except SHARE_IPMODIFY */ 419 419 - tr->flags &= BPF_TRAMP_F_SHARE_IPMODIFY; 418 418 + /* clear all bits except SHARE_IPMODIFY and TAIL_CALL_CTX */ 419 419 + tr->flags &= (BPF_TRAMP_F_SHARE_IPMODIFY | BPF_TRAMP_F_TAIL_CALL_CTX); 420 420 421 421 if (tlinks[BPF_TRAMP_FEXIT].nr_links || 422 422 tlinks[BPF_TRAMP_MODIFY_RETURN].nr_links) {

+549 -96

kernel/bpf/verifier.c

reviewed

··· 304 304 /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, 305 305 * generally to pass info about user-defined local kptr types to later 306 306 * verification logic 307 307 - * bpf_obj_drop 307 307 + * bpf_obj_drop/bpf_percpu_obj_drop 308 308 * Record the local kptr type to be drop'd 309 309 * bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type) 310 310 * Record the local kptr type to be refcount_incr'd and use ··· 543 543 } 544 544 545 545 static bool is_callback_calling_kfunc(u32 btf_id); 546 546 + static bool is_bpf_throw_kfunc(struct bpf_insn *insn); 546 547 547 548 static bool is_callback_calling_function(enum bpf_func_id func_id) 548 549 { ··· 1749 1748 return -ENOMEM; 1750 1749 dst_state->jmp_history_cnt = src->jmp_history_cnt; 1751 1750 1752 1752 - /* if dst has more stack frames then src frame, free them */ 1751 1751 + /* if dst has more stack frames then src frame, free them, this is also 1752 1752 + * necessary in case of exceptional exits using bpf_throw. 1753 1753 + */ 1753 1754 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 1754 1755 free_func_state(dst_state->frame[i]); 1755 1756 dst_state->frame[i] = NULL; ··· 2457 2454 return env->subprog_cnt - 1; 2458 2455 } 2459 2456 2457 2457 + static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env) 2458 2458 + { 2459 2459 + struct bpf_prog_aux *aux = env->prog->aux; 2460 2460 + struct btf *btf = aux->btf; 2461 2461 + const struct btf_type *t; 2462 2462 + u32 main_btf_id, id; 2463 2463 + const char *name; 2464 2464 + int ret, i; 2465 2465 + 2466 2466 + /* Non-zero func_info_cnt implies valid btf */ 2467 2467 + if (!aux->func_info_cnt) 2468 2468 + return 0; 2469 2469 + main_btf_id = aux->func_info[0].type_id; 2470 2470 + 2471 2471 + t = btf_type_by_id(btf, main_btf_id); 2472 2472 + if (!t) { 2473 2473 + verbose(env, "invalid btf id for main subprog in func_info\n"); 2474 2474 + return -EINVAL; 2475 2475 + } 2476 2476 + 2477 2477 + name = btf_find_decl_tag_value(btf, t, -1, "exception_callback:"); 2478 2478 + if (IS_ERR(name)) { 2479 2479 + ret = PTR_ERR(name); 2480 2480 + /* If there is no tag present, there is no exception callback */ 2481 2481 + if (ret == -ENOENT) 2482 2482 + ret = 0; 2483 2483 + else if (ret == -EEXIST) 2484 2484 + verbose(env, "multiple exception callback tags for main subprog\n"); 2485 2485 + return ret; 2486 2486 + } 2487 2487 + 2488 2488 + ret = btf_find_by_name_kind(btf, name, BTF_KIND_FUNC); 2489 2489 + if (ret < 0) { 2490 2490 + verbose(env, "exception callback '%s' could not be found in BTF\n", name); 2491 2491 + return ret; 2492 2492 + } 2493 2493 + id = ret; 2494 2494 + t = btf_type_by_id(btf, id); 2495 2495 + if (btf_func_linkage(t) != BTF_FUNC_GLOBAL) { 2496 2496 + verbose(env, "exception callback '%s' must have global linkage\n", name); 2497 2497 + return -EINVAL; 2498 2498 + } 2499 2499 + ret = 0; 2500 2500 + for (i = 0; i < aux->func_info_cnt; i++) { 2501 2501 + if (aux->func_info[i].type_id != id) 2502 2502 + continue; 2503 2503 + ret = aux->func_info[i].insn_off; 2504 2504 + /* Further func_info and subprog checks will also happen 2505 2505 + * later, so assume this is the right insn_off for now. 2506 2506 + */ 2507 2507 + if (!ret) { 2508 2508 + verbose(env, "invalid exception callback insn_off in func_info: 0\n"); 2509 2509 + ret = -EINVAL; 2510 2510 + } 2511 2511 + } 2512 2512 + if (!ret) { 2513 2513 + verbose(env, "exception callback type id not found in func_info\n"); 2514 2514 + ret = -EINVAL; 2515 2515 + } 2516 2516 + return ret; 2517 2517 + } 2518 2518 + 2460 2519 #define MAX_KFUNC_DESCS 256 2461 2520 #define MAX_KFUNC_BTFS 256 2462 2521 ··· 2858 2793 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) 2859 2794 { 2860 2795 struct bpf_subprog_info *subprog = env->subprog_info; 2796 2796 + int i, ret, insn_cnt = env->prog->len, ex_cb_insn; 2861 2797 struct bpf_insn *insn = env->prog->insnsi; 2862 2862 - int i, ret, insn_cnt = env->prog->len; 2863 2798 2864 2799 /* Add entry function. */ 2865 2800 ret = add_subprog(env, 0); ··· 2883 2818 2884 2819 if (ret < 0) 2885 2820 return ret; 2821 2821 + } 2822 2822 + 2823 2823 + ret = bpf_find_exception_callback_insn_off(env); 2824 2824 + if (ret < 0) 2825 2825 + return ret; 2826 2826 + ex_cb_insn = ret; 2827 2827 + 2828 2828 + /* If ex_cb_insn > 0, this means that the main program has a subprog 2829 2829 + * marked using BTF decl tag to serve as the exception callback. 2830 2830 + */ 2831 2831 + if (ex_cb_insn) { 2832 2832 + ret = add_subprog(env, ex_cb_insn); 2833 2833 + if (ret < 0) 2834 2834 + return ret; 2835 2835 + for (i = 1; i < env->subprog_cnt; i++) { 2836 2836 + if (env->subprog_info[i].start != ex_cb_insn) 2837 2837 + continue; 2838 2838 + env->exception_callback_subprog = i; 2839 2839 + break; 2840 2840 + } 2886 2841 } 2887 2842 2888 2843 /* Add a fake 'exit' subprog which could simplify subprog iteration ··· 2953 2868 if (i == subprog_end - 1) { 2954 2869 /* to avoid fall-through from one subprog into another 2955 2870 * the last insn of the subprog should be either exit 2956 2956 - * or unconditional jump back 2871 2871 + * or unconditional jump back or bpf_throw call 2957 2872 */ 2958 2873 if (code != (BPF_JMP | BPF_EXIT) && 2959 2874 code != (BPF_JMP32 | BPF_JA) && ··· 5086 5001 perm_flags |= PTR_UNTRUSTED; 5087 5002 } else { 5088 5003 perm_flags = PTR_MAYBE_NULL | MEM_ALLOC; 5004 5004 + if (kptr_field->type == BPF_KPTR_PERCPU) 5005 5005 + perm_flags |= MEM_PERCPU; 5089 5006 } 5090 5007 5091 5008 if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags)) ··· 5131 5044 */ 5132 5045 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 5133 5046 kptr_field->kptr.btf, kptr_field->kptr.btf_id, 5134 5134 - kptr_field->type == BPF_KPTR_REF)) 5047 5047 + kptr_field->type != BPF_KPTR_UNREF)) 5135 5048 goto bad_type; 5136 5049 return 0; 5137 5050 bad_type: ··· 5175 5088 { 5176 5089 const struct btf_field_kptr *kptr = &field->kptr; 5177 5090 5178 5178 - return field->type == BPF_KPTR_REF && rcu_protected_object(kptr->btf, kptr->btf_id); 5091 5091 + return field->type == BPF_KPTR_PERCPU || 5092 5092 + (field->type == BPF_KPTR_REF && rcu_protected_object(kptr->btf, kptr->btf_id)); 5093 5093 + } 5094 5094 + 5095 5095 + static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field) 5096 5096 + { 5097 5097 + if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) { 5098 5098 + if (kptr_field->type != BPF_KPTR_PERCPU) 5099 5099 + return PTR_MAYBE_NULL | MEM_RCU; 5100 5100 + return PTR_MAYBE_NULL | MEM_RCU | MEM_PERCPU; 5101 5101 + } 5102 5102 + return PTR_MAYBE_NULL | PTR_UNTRUSTED; 5179 5103 } 5180 5104 5181 5105 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, ··· 5212 5114 /* We only allow loading referenced kptr, since it will be marked as 5213 5115 * untrusted, similar to unreferenced kptr. 5214 5116 */ 5215 5215 - if (class != BPF_LDX && kptr_field->type == BPF_KPTR_REF) { 5117 5117 + if (class != BPF_LDX && 5118 5118 + (kptr_field->type == BPF_KPTR_REF || kptr_field->type == BPF_KPTR_PERCPU)) { 5216 5119 verbose(env, "store to referenced kptr disallowed\n"); 5217 5120 return -EACCES; 5218 5121 } ··· 5224 5125 * value from map as PTR_TO_BTF_ID, with the correct type. 5225 5126 */ 5226 5127 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf, 5227 5227 - kptr_field->kptr.btf_id, 5228 5228 - rcu_safe_kptr(kptr_field) && in_rcu_cs(env) ? 5229 5229 - PTR_MAYBE_NULL | MEM_RCU : 5230 5230 - PTR_MAYBE_NULL | PTR_UNTRUSTED); 5128 5128 + kptr_field->kptr.btf_id, btf_ld_kptr_type(env, kptr_field)); 5231 5129 /* For mark_ptr_or_null_reg */ 5232 5130 val_reg->id = ++env->id_gen; 5233 5131 } else if (class == BPF_STX) { ··· 5278 5182 switch (field->type) { 5279 5183 case BPF_KPTR_UNREF: 5280 5184 case BPF_KPTR_REF: 5185 5185 + case BPF_KPTR_PERCPU: 5281 5186 if (src != ACCESS_DIRECT) { 5282 5187 verbose(env, "kptr cannot be accessed indirectly by helper\n"); 5283 5188 return -EACCES; ··· 5746 5649 for (; i < subprog_end; i++) { 5747 5650 int next_insn, sidx; 5748 5651 5652 5652 + if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { 5653 5653 + bool err = false; 5654 5654 + 5655 5655 + if (!is_bpf_throw_kfunc(insn + i)) 5656 5656 + continue; 5657 5657 + if (subprog[idx].is_cb) 5658 5658 + err = true; 5659 5659 + for (int c = 0; c < frame && !err; c++) { 5660 5660 + if (subprog[ret_prog[c]].is_cb) { 5661 5661 + err = true; 5662 5662 + break; 5663 5663 + } 5664 5664 + } 5665 5665 + if (!err) 5666 5666 + continue; 5667 5667 + verbose(env, 5668 5668 + "bpf_throw kfunc (insn %d) cannot be called from callback subprog %d\n", 5669 5669 + i, idx); 5670 5670 + return -EINVAL; 5671 5671 + } 5672 5672 + 5749 5673 if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) 5750 5674 continue; 5751 5675 /* remember insn and function to return to */ ··· 5789 5671 /* async callbacks don't increase bpf prog stack size unless called directly */ 5790 5672 if (!bpf_pseudo_call(insn + i)) 5791 5673 continue; 5674 5674 + if (subprog[sidx].is_exception_cb) { 5675 5675 + verbose(env, "insn %d cannot call exception cb directly\n", i); 5676 5676 + return -EINVAL; 5677 5677 + } 5792 5678 } 5793 5679 i = next_insn; 5794 5680 idx = sidx; ··· 5814 5692 * tail call counter throughout bpf2bpf calls combined with tailcalls 5815 5693 */ 5816 5694 if (tail_call_reachable) 5817 5817 - for (j = 0; j < frame; j++) 5695 5695 + for (j = 0; j < frame; j++) { 5696 5696 + if (subprog[ret_prog[j]].is_exception_cb) { 5697 5697 + verbose(env, "cannot tail call within exception cb\n"); 5698 5698 + return -EINVAL; 5699 5699 + } 5818 5700 subprog[ret_prog[j]].tail_call_reachable = true; 5701 5701 + } 5819 5702 if (subprog[0].tail_call_reachable) 5820 5703 env->prog->aux->tail_call_reachable = true; 5821 5704 ··· 6336 6209 } 6337 6210 6338 6211 if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) && 6339 6339 - !reg->ref_obj_id) { 6212 6212 + !(reg->type & MEM_RCU) && !reg->ref_obj_id) { 6340 6213 verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n"); 6341 6214 return -EFAULT; 6342 6215 } ··· 7447 7320 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); 7448 7321 return -EACCES; 7449 7322 } 7450 7450 - if (kptr_field->type != BPF_KPTR_REF) { 7323 7323 + if (kptr_field->type != BPF_KPTR_REF && kptr_field->type != BPF_KPTR_PERCPU) { 7451 7324 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); 7452 7325 return -EACCES; 7453 7326 } ··· 7880 7753 static const struct bpf_reg_types percpu_btf_ptr_types = { 7881 7754 .types = { 7882 7755 PTR_TO_BTF_ID | MEM_PERCPU, 7756 7756 + PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU, 7883 7757 PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED, 7884 7758 } 7885 7759 }; ··· 7959 7831 if (base_type(arg_type) == ARG_PTR_TO_MEM) 7960 7832 type &= ~DYNPTR_TYPE_FLAG_MASK; 7961 7833 7962 7962 - if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type)) 7834 7834 + if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type)) { 7963 7835 type &= ~MEM_ALLOC; 7836 7836 + type &= ~MEM_PERCPU; 7837 7837 + } 7964 7838 7965 7839 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { 7966 7840 expected = compatible->types[i]; ··· 8045 7915 break; 8046 7916 } 8047 7917 case PTR_TO_BTF_ID | MEM_ALLOC: 7918 7918 + case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC: 8048 7919 if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock && 8049 7920 meta->func_id != BPF_FUNC_kptr_xchg) { 8050 7921 verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n"); ··· 8057 7926 } 8058 7927 break; 8059 7928 case PTR_TO_BTF_ID | MEM_PERCPU: 7929 7929 + case PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU: 8060 7930 case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED: 8061 7931 /* Handled by helper specific checks */ 8062 7932 break; ··· 9034 8902 * callbacks 9035 8903 */ 9036 8904 if (set_callee_state_cb != set_callee_state) { 8905 8905 + env->subprog_info[subprog].is_cb = true; 9037 8906 if (bpf_pseudo_kfunc_call(insn) && 9038 8907 !is_callback_calling_kfunc(insn->imm)) { 9039 8908 verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n", ··· 9424 9291 verbose(env, "to caller at %d:\n", *insn_idx); 9425 9292 print_verifier_state(env, caller, true); 9426 9293 } 9427 9427 - /* clear everything in the callee */ 9294 9294 + /* clear everything in the callee. In case of exceptional exits using 9295 9295 + * bpf_throw, this will be done by copy_verifier_state for extra frames. */ 9428 9296 free_func_state(callee); 9429 9297 state->frame[state->curframe--] = NULL; 9430 9298 return 0; ··· 9549 9415 return 0; 9550 9416 } 9551 9417 9552 9552 - static int check_reference_leak(struct bpf_verifier_env *env) 9418 9418 + static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit) 9553 9419 { 9554 9420 struct bpf_func_state *state = cur_func(env); 9555 9421 bool refs_lingering = false; 9556 9422 int i; 9557 9423 9558 9558 - if (state->frameno && !state->in_callback_fn) 9424 9424 + if (!exception_exit && state->frameno && !state->in_callback_fn) 9559 9425 return 0; 9560 9426 9561 9427 for (i = 0; i < state->acquired_refs; i++) { 9562 9562 - if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 9428 9428 + if (!exception_exit && state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 9563 9429 continue; 9564 9430 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 9565 9431 state->refs[i].id, state->refs[i].insn_idx); ··· 9666 9532 int *insn_idx_p) 9667 9533 { 9668 9534 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 9535 9535 + bool returns_cpu_specific_alloc_ptr = false; 9669 9536 const struct bpf_func_proto *fn = NULL; 9670 9537 enum bpf_return_type ret_type; 9671 9538 enum bpf_type_flag ret_flag; ··· 9777 9642 return -EFAULT; 9778 9643 } 9779 9644 err = unmark_stack_slots_dynptr(env, &regs[meta.release_regno]); 9645 9645 + } else if (func_id == BPF_FUNC_kptr_xchg && meta.ref_obj_id) { 9646 9646 + u32 ref_obj_id = meta.ref_obj_id; 9647 9647 + bool in_rcu = in_rcu_cs(env); 9648 9648 + struct bpf_func_state *state; 9649 9649 + struct bpf_reg_state *reg; 9650 9650 + 9651 9651 + err = release_reference_state(cur_func(env), ref_obj_id); 9652 9652 + if (!err) { 9653 9653 + bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 9654 9654 + if (reg->ref_obj_id == ref_obj_id) { 9655 9655 + if (in_rcu && (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { 9656 9656 + reg->ref_obj_id = 0; 9657 9657 + reg->type &= ~MEM_ALLOC; 9658 9658 + reg->type |= MEM_RCU; 9659 9659 + } else { 9660 9660 + mark_reg_invalid(env, reg); 9661 9661 + } 9662 9662 + } 9663 9663 + })); 9664 9664 + } 9780 9665 } else if (meta.ref_obj_id) { 9781 9666 err = release_reference(env, meta.ref_obj_id); 9782 9667 } else if (register_is_null(&regs[meta.release_regno])) { ··· 9814 9659 9815 9660 switch (func_id) { 9816 9661 case BPF_FUNC_tail_call: 9817 9817 - err = check_reference_leak(env); 9662 9662 + err = check_reference_leak(env, false); 9818 9663 if (err) { 9819 9664 verbose(env, "tail_call would lead to reference leak\n"); 9820 9665 return err; ··· 9925 9770 9926 9771 break; 9927 9772 } 9773 9773 + case BPF_FUNC_per_cpu_ptr: 9774 9774 + case BPF_FUNC_this_cpu_ptr: 9775 9775 + { 9776 9776 + struct bpf_reg_state *reg = &regs[BPF_REG_1]; 9777 9777 + const struct btf_type *type; 9778 9778 + 9779 9779 + if (reg->type & MEM_RCU) { 9780 9780 + type = btf_type_by_id(reg->btf, reg->btf_id); 9781 9781 + if (!type || !btf_type_is_struct(type)) { 9782 9782 + verbose(env, "Helper has invalid btf/btf_id in R1\n"); 9783 9783 + return -EFAULT; 9784 9784 + } 9785 9785 + returns_cpu_specific_alloc_ptr = true; 9786 9786 + env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true; 9787 9787 + } 9788 9788 + break; 9789 9789 + } 9928 9790 case BPF_FUNC_user_ringbuf_drain: 9929 9791 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 9930 9792 set_user_ringbuf_callback_state); ··· 10031 9859 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 10032 9860 regs[BPF_REG_0].mem_size = tsize; 10033 9861 } else { 10034 10034 - /* MEM_RDONLY may be carried from ret_flag, but it 10035 10035 - * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise 10036 10036 - * it will confuse the check of PTR_TO_BTF_ID in 10037 10037 - * check_mem_access(). 10038 10038 - */ 10039 10039 - ret_flag &= ~MEM_RDONLY; 9862 9862 + if (returns_cpu_specific_alloc_ptr) { 9863 9863 + regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU; 9864 9864 + } else { 9865 9865 + /* MEM_RDONLY may be carried from ret_flag, but it 9866 9866 + * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise 9867 9867 + * it will confuse the check of PTR_TO_BTF_ID in 9868 9868 + * check_mem_access(). 9869 9869 + */ 9870 9870 + ret_flag &= ~MEM_RDONLY; 9871 9871 + regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 9872 9872 + } 10040 9873 10041 10041 - regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 10042 9874 regs[BPF_REG_0].btf = meta.ret_btf; 10043 9875 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 10044 9876 } ··· 10058 9882 if (func_id == BPF_FUNC_kptr_xchg) { 10059 9883 ret_btf = meta.kptr_field->kptr.btf; 10060 9884 ret_btf_id = meta.kptr_field->kptr.btf_id; 10061 10061 - if (!btf_is_kernel(ret_btf)) 9885 9885 + if (!btf_is_kernel(ret_btf)) { 10062 9886 regs[BPF_REG_0].type |= MEM_ALLOC; 9887 9887 + if (meta.kptr_field->type == BPF_KPTR_PERCPU) 9888 9888 + regs[BPF_REG_0].type |= MEM_PERCPU; 9889 9889 + } 10063 9890 } else { 10064 9891 if (fn->ret_btf_id == BPF_PTR_POISON) { 10065 9892 verbose(env, "verifier internal error:"); ··· 10447 10268 KF_bpf_dynptr_slice, 10448 10269 KF_bpf_dynptr_slice_rdwr, 10449 10270 KF_bpf_dynptr_clone, 10271 10271 + KF_bpf_percpu_obj_new_impl, 10272 10272 + KF_bpf_percpu_obj_drop_impl, 10273 10273 + KF_bpf_throw, 10450 10274 }; 10451 10275 10452 10276 BTF_SET_START(special_kfunc_set) ··· 10470 10288 BTF_ID(func, bpf_dynptr_slice) 10471 10289 BTF_ID(func, bpf_dynptr_slice_rdwr) 10472 10290 BTF_ID(func, bpf_dynptr_clone) 10291 10291 + BTF_ID(func, bpf_percpu_obj_new_impl) 10292 10292 + BTF_ID(func, bpf_percpu_obj_drop_impl) 10293 10293 + BTF_ID(func, bpf_throw) 10473 10294 BTF_SET_END(special_kfunc_set) 10474 10295 10475 10296 BTF_ID_LIST(special_kfunc_list) ··· 10495 10310 BTF_ID(func, bpf_dynptr_slice) 10496 10311 BTF_ID(func, bpf_dynptr_slice_rdwr) 10497 10312 BTF_ID(func, bpf_dynptr_clone) 10313 10313 + BTF_ID(func, bpf_percpu_obj_new_impl) 10314 10314 + BTF_ID(func, bpf_percpu_obj_drop_impl) 10315 10315 + BTF_ID(func, bpf_throw) 10498 10316 10499 10317 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) 10500 10318 { ··· 10813 10625 static bool is_callback_calling_kfunc(u32 btf_id) 10814 10626 { 10815 10627 return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]; 10628 10628 + } 10629 10629 + 10630 10630 + static bool is_bpf_throw_kfunc(struct bpf_insn *insn) 10631 10631 + { 10632 10632 + return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && 10633 10633 + insn->imm == special_kfunc_list[KF_bpf_throw]; 10816 10634 } 10817 10635 10818 10636 static bool is_rbtree_lock_required_kfunc(u32 btf_id) ··· 11198 11004 } 11199 11005 break; 11200 11006 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 11201 11201 - if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 11007 11007 + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { 11008 11008 + if (meta->func_id != special_kfunc_list[KF_bpf_obj_drop_impl]) { 11009 11009 + verbose(env, "arg#%d expected for bpf_obj_drop_impl()\n", i); 11010 11010 + return -EINVAL; 11011 11011 + } 11012 11012 + } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { 11013 11013 + if (meta->func_id != special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) { 11014 11014 + verbose(env, "arg#%d expected for bpf_percpu_obj_drop_impl()\n", i); 11015 11015 + return -EINVAL; 11016 11016 + } 11017 11017 + } else { 11202 11018 verbose(env, "arg#%d expected pointer to allocated object\n", i); 11203 11019 return -EINVAL; 11204 11020 } ··· 11216 11012 verbose(env, "allocated object must be referenced\n"); 11217 11013 return -EINVAL; 11218 11014 } 11219 11219 - if (meta->btf == btf_vmlinux && 11220 11220 - meta->func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 11015 11015 + if (meta->btf == btf_vmlinux) { 11221 11016 meta->arg_btf = reg->btf; 11222 11017 meta->arg_btf_id = reg->btf_id; 11223 11018 } ··· 11407 11204 break; 11408 11205 } 11409 11206 case KF_ARG_PTR_TO_CALLBACK: 11207 11207 + if (reg->type != PTR_TO_FUNC) { 11208 11208 + verbose(env, "arg%d expected pointer to func\n", i); 11209 11209 + return -EINVAL; 11210 11210 + } 11410 11211 meta->subprogno = reg->subprogno; 11411 11212 break; 11412 11213 case KF_ARG_PTR_TO_REFCOUNTED_KPTR: ··· 11488 11281 11489 11282 return 0; 11490 11283 } 11284 11284 + 11285 11285 + static int check_return_code(struct bpf_verifier_env *env, int regno); 11491 11286 11492 11287 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 11493 11288 int *insn_idx_p) ··· 11612 11403 } 11613 11404 } 11614 11405 11406 11406 + if (meta.func_id == special_kfunc_list[KF_bpf_throw]) { 11407 11407 + if (!bpf_jit_supports_exceptions()) { 11408 11408 + verbose(env, "JIT does not support calling kfunc %s#%d\n", 11409 11409 + func_name, meta.func_id); 11410 11410 + return -ENOTSUPP; 11411 11411 + } 11412 11412 + env->seen_exception = true; 11413 11413 + 11414 11414 + /* In the case of the default callback, the cookie value passed 11415 11415 + * to bpf_throw becomes the return value of the program. 11416 11416 + */ 11417 11417 + if (!env->exception_callback_subprog) { 11418 11418 + err = check_return_code(env, BPF_REG_1); 11419 11419 + if (err < 0) 11420 11420 + return err; 11421 11421 + } 11422 11422 + } 11423 11423 + 11615 11424 for (i = 0; i < CALLER_SAVED_REGS; i++) 11616 11425 mark_reg_not_init(env, regs, caller_saved[i]); 11617 11426 ··· 11640 11413 /* Only exception is bpf_obj_new_impl */ 11641 11414 if (meta.btf != btf_vmlinux || 11642 11415 (meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl] && 11416 11416 + meta.func_id != special_kfunc_list[KF_bpf_percpu_obj_new_impl] && 11643 11417 meta.func_id != special_kfunc_list[KF_bpf_refcount_acquire_impl])) { 11644 11418 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); 11645 11419 return -EINVAL; ··· 11654 11426 ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); 11655 11427 11656 11428 if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) { 11657 11657 - if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl]) { 11429 11429 + if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl] || 11430 11430 + meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { 11431 11431 + struct btf_struct_meta *struct_meta; 11658 11432 struct btf *ret_btf; 11659 11433 u32 ret_btf_id; 11660 11434 11661 11661 - if (unlikely(!bpf_global_ma_set)) 11435 11435 + if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl] && !bpf_global_ma_set) 11436 11436 + return -ENOMEM; 11437 11437 + 11438 11438 + if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl] && !bpf_global_percpu_ma_set) 11662 11439 return -ENOMEM; 11663 11440 11664 11441 if (((u64)(u32)meta.arg_constant.value) != meta.arg_constant.value) { ··· 11676 11443 11677 11444 /* This may be NULL due to user not supplying a BTF */ 11678 11445 if (!ret_btf) { 11679 11679 - verbose(env, "bpf_obj_new requires prog BTF\n"); 11446 11446 + verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n"); 11680 11447 return -EINVAL; 11681 11448 } 11682 11449 11683 11450 ret_t = btf_type_by_id(ret_btf, ret_btf_id); 11684 11451 if (!ret_t || !__btf_type_is_struct(ret_t)) { 11685 11685 - verbose(env, "bpf_obj_new type ID argument must be of a struct\n"); 11452 11452 + verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n"); 11686 11453 return -EINVAL; 11454 11454 + } 11455 11455 + 11456 11456 + struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); 11457 11457 + if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { 11458 11458 + if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { 11459 11459 + verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); 11460 11460 + return -EINVAL; 11461 11461 + } 11462 11462 + 11463 11463 + if (struct_meta) { 11464 11464 + verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n"); 11465 11465 + return -EINVAL; 11466 11466 + } 11687 11467 } 11688 11468 11689 11469 mark_reg_known_zero(env, regs, BPF_REG_0); 11690 11470 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 11691 11471 regs[BPF_REG_0].btf = ret_btf; 11692 11472 regs[BPF_REG_0].btf_id = ret_btf_id; 11473 11473 + if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) 11474 11474 + regs[BPF_REG_0].type |= MEM_PERCPU; 11693 11475 11694 11476 insn_aux->obj_new_size = ret_t->size; 11695 11695 - insn_aux->kptr_struct_meta = 11696 11696 - btf_find_struct_meta(ret_btf, ret_btf_id); 11477 11477 + insn_aux->kptr_struct_meta = struct_meta; 11697 11478 } else if (meta.func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) { 11698 11479 mark_reg_known_zero(env, regs, BPF_REG_0); 11699 11480 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; ··· 11844 11597 regs[BPF_REG_0].id = ++env->id_gen; 11845 11598 } else if (btf_type_is_void(t)) { 11846 11599 if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) { 11847 11847 - if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 11600 11600 + if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl] || 11601 11601 + meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) { 11848 11602 insn_aux->kptr_struct_meta = 11849 11603 btf_find_struct_meta(meta.arg_btf, 11850 11604 meta.arg_btf_id); ··· 14675 14427 * gen_ld_abs() may terminate the program at runtime, leading to 14676 14428 * reference leak. 14677 14429 */ 14678 14678 - err = check_reference_leak(env); 14430 14430 + err = check_reference_leak(env, false); 14679 14431 if (err) { 14680 14432 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); 14681 14433 return err; ··· 14724 14476 return 0; 14725 14477 } 14726 14478 14727 14727 - static int check_return_code(struct bpf_verifier_env *env) 14479 14479 + static int check_return_code(struct bpf_verifier_env *env, int regno) 14728 14480 { 14729 14481 struct tnum enforce_attach_type_range = tnum_unknown; 14730 14482 const struct bpf_prog *prog = env->prog; ··· 14736 14488 const bool is_subprog = frame->subprogno; 14737 14489 14738 14490 /* LSM and struct_ops func-ptr's return type could be "void" */ 14739 14739 - if (!is_subprog) { 14491 14491 + if (!is_subprog || frame->in_exception_callback_fn) { 14740 14492 switch (prog_type) { 14741 14493 case BPF_PROG_TYPE_LSM: 14742 14494 if (prog->expected_attach_type == BPF_LSM_CGROUP) ··· 14758 14510 * of bpf_exit, which means that program wrote 14759 14511 * something into it earlier 14760 14512 */ 14761 14761 - err = check_reg_arg(env, BPF_REG_0, SRC_OP); 14513 14513 + err = check_reg_arg(env, regno, SRC_OP); 14762 14514 if (err) 14763 14515 return err; 14764 14516 14765 14765 - if (is_pointer_value(env, BPF_REG_0)) { 14766 14766 - verbose(env, "R0 leaks addr as return value\n"); 14517 14517 + if (is_pointer_value(env, regno)) { 14518 14518 + verbose(env, "R%d leaks addr as return value\n", regno); 14767 14519 return -EACCES; 14768 14520 } 14769 14521 14770 14770 - reg = cur_regs(env) + BPF_REG_0; 14522 14522 + reg = cur_regs(env) + regno; 14771 14523 14772 14524 if (frame->in_async_callback_fn) { 14773 14525 /* enforce return zero from async callbacks like timer */ 14774 14526 if (reg->type != SCALAR_VALUE) { 14775 14775 - verbose(env, "In async callback the register R0 is not a known value (%s)\n", 14776 14776 - reg_type_str(env, reg->type)); 14527 14527 + verbose(env, "In async callback the register R%d is not a known value (%s)\n", 14528 14528 + regno, reg_type_str(env, reg->type)); 14777 14529 return -EINVAL; 14778 14530 } 14779 14531 ··· 14784 14536 return 0; 14785 14537 } 14786 14538 14787 14787 - if (is_subprog) { 14539 14539 + if (is_subprog && !frame->in_exception_callback_fn) { 14788 14540 if (reg->type != SCALAR_VALUE) { 14789 14789 - verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", 14790 14790 - reg_type_str(env, reg->type)); 14541 14541 + verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", 14542 14542 + regno, reg_type_str(env, reg->type)); 14791 14543 return -EINVAL; 14792 14544 } 14793 14545 return 0; ··· 14869 14621 } 14870 14622 14871 14623 if (reg->type != SCALAR_VALUE) { 14872 14872 - verbose(env, "At program exit the register R0 is not a known value (%s)\n", 14873 14873 - reg_type_str(env, reg->type)); 14624 14624 + verbose(env, "At program exit the register R%d is not a known value (%s)\n", 14625 14625 + regno, reg_type_str(env, reg->type)); 14874 14626 return -EINVAL; 14875 14627 } 14876 14628 ··· 15141 14893 { 15142 14894 int insn_cnt = env->prog->len; 15143 14895 int *insn_stack, *insn_state; 15144 15144 - int ret = 0; 15145 15145 - int i; 14896 14896 + int ex_insn_beg, i, ret = 0; 14897 14897 + bool ex_done = false; 15146 14898 15147 14899 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 15148 14900 if (!insn_state) ··· 15158 14910 insn_stack[0] = 0; /* 0 is the first instruction */ 15159 14911 env->cfg.cur_stack = 1; 15160 14912 14913 14913 + walk_cfg: 15161 14914 while (env->cfg.cur_stack > 0) { 15162 14915 int t = insn_stack[env->cfg.cur_stack - 1]; 15163 14916 ··· 15183 14934 verbose(env, "pop stack internal bug\n"); 15184 14935 ret = -EFAULT; 15185 14936 goto err_free; 14937 14937 + } 14938 14938 + 14939 14939 + if (env->exception_callback_subprog && !ex_done) { 14940 14940 + ex_insn_beg = env->subprog_info[env->exception_callback_subprog].start; 14941 14941 + 14942 14942 + insn_state[ex_insn_beg] = DISCOVERED; 14943 14943 + insn_stack[0] = ex_insn_beg; 14944 14944 + env->cfg.cur_stack = 1; 14945 14945 + ex_done = true; 14946 14946 + goto walk_cfg; 15186 14947 } 15187 14948 15188 14949 for (i = 0; i < insn_cnt; i++) { ··· 15232 14973 #define MIN_BPF_FUNCINFO_SIZE 8 15233 14974 #define MAX_FUNCINFO_REC_SIZE 252 15234 14975 15235 15235 - static int check_btf_func(struct bpf_verifier_env *env, 15236 15236 - const union bpf_attr *attr, 15237 15237 - bpfptr_t uattr) 14976 14976 + static int check_btf_func_early(struct bpf_verifier_env *env, 14977 14977 + const union bpf_attr *attr, 14978 14978 + bpfptr_t uattr) 15238 14979 { 15239 15239 - const struct btf_type *type, *func_proto, *ret_type; 15240 15240 - u32 i, nfuncs, urec_size, min_size; 15241 14980 u32 krec_size = sizeof(struct bpf_func_info); 14981 14981 + const struct btf_type *type, *func_proto; 14982 14982 + u32 i, nfuncs, urec_size, min_size; 15242 14983 struct bpf_func_info *krecord; 15243 15243 - struct bpf_func_info_aux *info_aux = NULL; 15244 14984 struct bpf_prog *prog; 15245 14985 const struct btf *btf; 15246 15246 - bpfptr_t urecord; 15247 14986 u32 prev_offset = 0; 15248 15248 - bool scalar_return; 14987 14987 + bpfptr_t urecord; 15249 14988 int ret = -ENOMEM; 15250 14989 15251 14990 nfuncs = attr->func_info_cnt; ··· 15251 14994 if (check_abnormal_return(env)) 15252 14995 return -EINVAL; 15253 14996 return 0; 15254 15254 - } 15255 15255 - 15256 15256 - if (nfuncs != env->subprog_cnt) { 15257 15257 - verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 15258 15258 - return -EINVAL; 15259 14997 } 15260 14998 15261 14999 urec_size = attr->func_info_rec_size; ··· 15270 15018 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); 15271 15019 if (!krecord) 15272 15020 return -ENOMEM; 15273 15273 - info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); 15274 15274 - if (!info_aux) 15275 15275 - goto err_free; 15276 15021 15277 15022 for (i = 0; i < nfuncs; i++) { 15278 15023 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); ··· 15308 15059 goto err_free; 15309 15060 } 15310 15061 15311 15311 - if (env->subprog_info[i].start != krecord[i].insn_off) { 15312 15312 - verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 15313 15313 - goto err_free; 15314 15314 - } 15315 15315 - 15316 15062 /* check type_id */ 15317 15063 type = btf_type_by_id(btf, krecord[i].type_id); 15318 15064 if (!type || !btf_type_is_func(type)) { ··· 15315 15071 krecord[i].type_id); 15316 15072 goto err_free; 15317 15073 } 15318 15318 - info_aux[i].linkage = BTF_INFO_VLEN(type->info); 15319 15074 15320 15075 func_proto = btf_type_by_id(btf, type->type); 15321 15076 if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) 15322 15077 /* btf_func_check() already verified it during BTF load */ 15323 15078 goto err_free; 15079 15079 + 15080 15080 + prev_offset = krecord[i].insn_off; 15081 15081 + bpfptr_add(&urecord, urec_size); 15082 15082 + } 15083 15083 + 15084 15084 + prog->aux->func_info = krecord; 15085 15085 + prog->aux->func_info_cnt = nfuncs; 15086 15086 + return 0; 15087 15087 + 15088 15088 + err_free: 15089 15089 + kvfree(krecord); 15090 15090 + return ret; 15091 15091 + } 15092 15092 + 15093 15093 + static int check_btf_func(struct bpf_verifier_env *env, 15094 15094 + const union bpf_attr *attr, 15095 15095 + bpfptr_t uattr) 15096 15096 + { 15097 15097 + const struct btf_type *type, *func_proto, *ret_type; 15098 15098 + u32 i, nfuncs, urec_size, min_size; 15099 15099 + u32 krec_size = sizeof(struct bpf_func_info); 15100 15100 + struct bpf_func_info *krecord; 15101 15101 + struct bpf_func_info_aux *info_aux = NULL; 15102 15102 + struct bpf_prog *prog; 15103 15103 + const struct btf *btf; 15104 15104 + bpfptr_t urecord; 15105 15105 + u32 prev_offset = 0; 15106 15106 + bool scalar_return; 15107 15107 + int ret = -ENOMEM; 15108 15108 + 15109 15109 + nfuncs = attr->func_info_cnt; 15110 15110 + if (!nfuncs) { 15111 15111 + if (check_abnormal_return(env)) 15112 15112 + return -EINVAL; 15113 15113 + return 0; 15114 15114 + } 15115 15115 + if (nfuncs != env->subprog_cnt) { 15116 15116 + verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 15117 15117 + return -EINVAL; 15118 15118 + } 15119 15119 + 15120 15120 + urec_size = attr->func_info_rec_size; 15121 15121 + 15122 15122 + prog = env->prog; 15123 15123 + btf = prog->aux->btf; 15124 15124 + 15125 15125 + urecord = make_bpfptr(attr->func_info, uattr.is_kernel); 15126 15126 + min_size = min_t(u32, krec_size, urec_size); 15127 15127 + 15128 15128 + krecord = prog->aux->func_info; 15129 15129 + info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); 15130 15130 + if (!info_aux) 15131 15131 + return -ENOMEM; 15132 15132 + 15133 15133 + for (i = 0; i < nfuncs; i++) { 15134 15134 + /* check insn_off */ 15135 15135 + ret = -EINVAL; 15136 15136 + 15137 15137 + if (env->subprog_info[i].start != krecord[i].insn_off) { 15138 15138 + verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 15139 15139 + goto err_free; 15140 15140 + } 15141 15141 + 15142 15142 + /* Already checked type_id */ 15143 15143 + type = btf_type_by_id(btf, krecord[i].type_id); 15144 15144 + info_aux[i].linkage = BTF_INFO_VLEN(type->info); 15145 15145 + /* Already checked func_proto */ 15146 15146 + func_proto = btf_type_by_id(btf, type->type); 15147 15147 + 15324 15148 ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); 15325 15149 scalar_return = 15326 15150 btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type); ··· 15405 15093 bpfptr_add(&urecord, urec_size); 15406 15094 } 15407 15095 15408 15408 - prog->aux->func_info = krecord; 15409 15409 - prog->aux->func_info_cnt = nfuncs; 15410 15096 prog->aux->func_info_aux = info_aux; 15411 15097 return 0; 15412 15098 15413 15099 err_free: 15414 15414 - kvfree(krecord); 15415 15100 kfree(info_aux); 15416 15101 return ret; 15417 15102 } ··· 15421 15112 if (!aux->func_info) 15422 15113 return; 15423 15114 15424 15424 - for (i = 0; i < env->subprog_cnt; i++) 15115 15115 + /* func_info is not available for hidden subprogs */ 15116 15116 + for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++) 15425 15117 aux->func_info[i].insn_off = env->subprog_info[i].start; 15426 15118 } 15427 15119 ··· 15626 15316 return err; 15627 15317 } 15628 15318 15629 15629 - static int check_btf_info(struct bpf_verifier_env *env, 15630 15630 - const union bpf_attr *attr, 15631 15631 - bpfptr_t uattr) 15319 15319 + static int check_btf_info_early(struct bpf_verifier_env *env, 15320 15320 + const union bpf_attr *attr, 15321 15321 + bpfptr_t uattr) 15632 15322 { 15633 15323 struct btf *btf; 15634 15324 int err; ··· 15647 15337 return -EACCES; 15648 15338 } 15649 15339 env->prog->aux->btf = btf; 15340 15340 + 15341 15341 + err = check_btf_func_early(env, attr, uattr); 15342 15342 + if (err) 15343 15343 + return err; 15344 15344 + return 0; 15345 15345 + } 15346 15346 + 15347 15347 + static int check_btf_info(struct bpf_verifier_env *env, 15348 15348 + const union bpf_attr *attr, 15349 15349 + bpfptr_t uattr) 15350 15350 + { 15351 15351 + int err; 15352 15352 + 15353 15353 + if (!attr->func_info_cnt && !attr->line_info_cnt) { 15354 15354 + if (check_abnormal_return(env)) 15355 15355 + return -EINVAL; 15356 15356 + return 0; 15357 15357 + } 15650 15358 15651 15359 err = check_btf_func(env, attr, uattr); 15652 15360 if (err) ··· 16768 16440 int prev_insn_idx = -1; 16769 16441 16770 16442 for (;;) { 16443 16443 + bool exception_exit = false; 16771 16444 struct bpf_insn *insn; 16772 16445 u8 class; 16773 16446 int err; ··· 16983 16654 return -EINVAL; 16984 16655 } 16985 16656 } 16986 16986 - if (insn->src_reg == BPF_PSEUDO_CALL) 16657 16657 + if (insn->src_reg == BPF_PSEUDO_CALL) { 16987 16658 err = check_func_call(env, insn, &env->insn_idx); 16988 16988 - else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) 16659 16659 + } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 16989 16660 err = check_kfunc_call(env, insn, &env->insn_idx); 16990 16990 - else 16661 16661 + if (!err && is_bpf_throw_kfunc(insn)) { 16662 16662 + exception_exit = true; 16663 16663 + goto process_bpf_exit_full; 16664 16664 + } 16665 16665 + } else { 16991 16666 err = check_helper_call(env, insn, &env->insn_idx); 16667 16667 + } 16992 16668 if (err) 16993 16669 return err; 16994 16670 ··· 17023 16689 verbose(env, "BPF_EXIT uses reserved fields\n"); 17024 16690 return -EINVAL; 17025 16691 } 17026 17026 - 16692 16692 + process_bpf_exit_full: 17027 16693 if (env->cur_state->active_lock.ptr && 17028 16694 !in_rbtree_lock_required_cb(env)) { 17029 16695 verbose(env, "bpf_spin_unlock is missing\n"); ··· 17042 16708 * function, for which reference_state must 17043 16709 * match caller reference state when it exits. 17044 16710 */ 17045 17045 - err = check_reference_leak(env); 16711 16711 + err = check_reference_leak(env, exception_exit); 17046 16712 if (err) 17047 16713 return err; 16714 16714 + 16715 16715 + /* The side effect of the prepare_func_exit 16716 16716 + * which is being skipped is that it frees 16717 16717 + * bpf_func_state. Typically, process_bpf_exit 16718 16718 + * will only be hit with outermost exit. 16719 16719 + * copy_verifier_state in pop_stack will handle 16720 16720 + * freeing of any extra bpf_func_state left over 16721 16721 + * from not processing all nested function 16722 16722 + * exits. We also skip return code checks as 16723 16723 + * they are not needed for exceptional exits. 16724 16724 + */ 16725 16725 + if (exception_exit) 16726 16726 + goto process_bpf_exit; 17048 16727 17049 16728 if (state->curframe) { 17050 16729 /* exit from nested function */ ··· 17068 16721 continue; 17069 16722 } 17070 16723 17071 17071 - err = check_return_code(env); 16724 16724 + err = check_return_code(env, BPF_REG_0); 17072 16725 if (err) 17073 16726 return err; 17074 16727 process_bpf_exit: ··· 18361 18014 } 18362 18015 func[i]->aux->num_exentries = num_exentries; 18363 18016 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 18017 18017 + func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; 18018 18018 + if (!i) 18019 18019 + func[i]->aux->exception_boundary = env->seen_exception; 18364 18020 func[i] = bpf_int_jit_compile(func[i]); 18365 18021 if (!func[i]->jited) { 18366 18022 err = -ENOTSUPP; ··· 18403 18053 * the call instruction, as an index for this list 18404 18054 */ 18405 18055 func[i]->aux->func = func; 18406 18406 - func[i]->aux->func_cnt = env->subprog_cnt; 18056 18056 + func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 18057 18057 + func[i]->aux->real_func_cnt = env->subprog_cnt; 18407 18058 } 18408 18059 for (i = 0; i < env->subprog_cnt; i++) { 18409 18060 old_bpf_func = func[i]->bpf_func; ··· 18450 18099 prog->aux->extable = func[0]->aux->extable; 18451 18100 prog->aux->num_exentries = func[0]->aux->num_exentries; 18452 18101 prog->aux->func = func; 18453 18453 - prog->aux->func_cnt = env->subprog_cnt; 18102 18102 + prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 18103 18103 + prog->aux->real_func_cnt = env->subprog_cnt; 18104 18104 + prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; 18105 18105 + prog->aux->exception_boundary = func[0]->aux->exception_boundary; 18454 18106 bpf_prog_jit_attempt_done(prog); 18455 18107 return 0; 18456 18108 out_free: ··· 18620 18266 insn->imm = BPF_CALL_IMM(desc->addr); 18621 18267 if (insn->off) 18622 18268 return 0; 18623 18623 - if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl]) { 18269 18269 + if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl] || 18270 18270 + desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { 18624 18271 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 18625 18272 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 18626 18273 u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size; 18274 18274 + 18275 18275 + if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl] && kptr_struct_meta) { 18276 18276 + verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n", 18277 18277 + insn_idx); 18278 18278 + return -EFAULT; 18279 18279 + } 18627 18280 18628 18281 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size); 18629 18282 insn_buf[1] = addr[0]; ··· 18638 18277 insn_buf[3] = *insn; 18639 18278 *cnt = 4; 18640 18279 } else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl] || 18280 18280 + desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] || 18641 18281 desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) { 18642 18282 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 18643 18283 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 18284 18284 + 18285 18285 + if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] && kptr_struct_meta) { 18286 18286 + verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n", 18287 18287 + insn_idx); 18288 18288 + return -EFAULT; 18289 18289 + } 18644 18290 18645 18291 if (desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] && 18646 18292 !kptr_struct_meta) { ··· 18689 18321 return 0; 18690 18322 } 18691 18323 18324 18324 + /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */ 18325 18325 + static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) 18326 18326 + { 18327 18327 + struct bpf_subprog_info *info = env->subprog_info; 18328 18328 + int cnt = env->subprog_cnt; 18329 18329 + struct bpf_prog *prog; 18330 18330 + 18331 18331 + /* We only reserve one slot for hidden subprogs in subprog_info. */ 18332 18332 + if (env->hidden_subprog_cnt) { 18333 18333 + verbose(env, "verifier internal error: only one hidden subprog supported\n"); 18334 18334 + return -EFAULT; 18335 18335 + } 18336 18336 + /* We're not patching any existing instruction, just appending the new 18337 18337 + * ones for the hidden subprog. Hence all of the adjustment operations 18338 18338 + * in bpf_patch_insn_data are no-ops. 18339 18339 + */ 18340 18340 + prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); 18341 18341 + if (!prog) 18342 18342 + return -ENOMEM; 18343 18343 + env->prog = prog; 18344 18344 + info[cnt + 1].start = info[cnt].start; 18345 18345 + info[cnt].start = prog->len - len + 1; 18346 18346 + env->subprog_cnt++; 18347 18347 + env->hidden_subprog_cnt++; 18348 18348 + return 0; 18349 18349 + } 18350 18350 + 18692 18351 /* Do various post-verification rewrites in a single program pass. 18693 18352 * These rewrites simplify JIT and interpreter implementations. 18694 18353 */ ··· 18733 18338 struct bpf_prog *new_prog; 18734 18339 struct bpf_map *map_ptr; 18735 18340 int i, ret, cnt, delta = 0; 18341 18341 + 18342 18342 + if (env->seen_exception && !env->exception_callback_subprog) { 18343 18343 + struct bpf_insn patch[] = { 18344 18344 + env->prog->insnsi[insn_cnt - 1], 18345 18345 + BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), 18346 18346 + BPF_EXIT_INSN(), 18347 18347 + }; 18348 18348 + 18349 18349 + ret = add_hidden_subprog(env, patch, ARRAY_SIZE(patch)); 18350 18350 + if (ret < 0) 18351 18351 + return ret; 18352 18352 + prog = env->prog; 18353 18353 + insn = prog->insnsi; 18354 18354 + 18355 18355 + env->exception_callback_subprog = env->subprog_cnt - 1; 18356 18356 + /* Don't update insn_cnt, as add_hidden_subprog always appends insns */ 18357 18357 + env->subprog_info[env->exception_callback_subprog].is_cb = true; 18358 18358 + env->subprog_info[env->exception_callback_subprog].is_async_cb = true; 18359 18359 + env->subprog_info[env->exception_callback_subprog].is_exception_cb = true; 18360 18360 + } 18736 18361 18737 18362 for (i = 0; i < insn_cnt; i++, insn++) { 18738 18363 /* Make divide-by-zero exceptions impossible. */ ··· 19010 18595 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC); 19011 18596 else 19012 18597 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL); 18598 18598 + insn_buf[1] = *insn; 18599 18599 + cnt = 2; 18600 18600 + 18601 18601 + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 18602 18602 + if (!new_prog) 18603 18603 + return -ENOMEM; 18604 18604 + 18605 18605 + delta += cnt - 1; 18606 18606 + env->prog = prog = new_prog; 18607 18607 + insn = new_prog->insnsi + i + delta; 18608 18608 + goto patch_call_imm; 18609 18609 + } 18610 18610 + 18611 18611 + /* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */ 18612 18612 + if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) { 18613 18613 + /* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data, 18614 18614 + * bpf_mem_alloc() returns a ptr to the percpu data ptr. 18615 18615 + */ 18616 18616 + insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 19013 18617 insn_buf[1] = *insn; 19014 18618 cnt = 2; 19015 18619 ··· 19451 19017 } 19452 19018 } 19453 19019 19454 19454 - static int do_check_common(struct bpf_verifier_env *env, int subprog) 19020 19020 + static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_ex_cb) 19455 19021 { 19456 19022 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 19457 19023 struct bpf_verifier_state *state; ··· 19482 19048 19483 19049 regs = state->frame[state->curframe]->regs; 19484 19050 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { 19485 19485 - ret = btf_prepare_func_args(env, subprog, regs); 19051 19051 + ret = btf_prepare_func_args(env, subprog, regs, is_ex_cb); 19486 19052 if (ret) 19487 19053 goto out; 19488 19054 for (i = BPF_REG_1; i <= BPF_REG_5; i++) { ··· 19497 19063 regs[i].mem_size = mem_size; 19498 19064 regs[i].id = ++env->id_gen; 19499 19065 } 19066 19066 + } 19067 19067 + if (is_ex_cb) { 19068 19068 + state->frame[0]->in_exception_callback_fn = true; 19069 19069 + env->subprog_info[subprog].is_cb = true; 19070 19070 + env->subprog_info[subprog].is_async_cb = true; 19071 19071 + env->subprog_info[subprog].is_exception_cb = true; 19500 19072 } 19501 19073 } else { 19502 19074 /* 1st arg to a function */ ··· 19568 19128 continue; 19569 19129 env->insn_idx = env->subprog_info[i].start; 19570 19130 WARN_ON_ONCE(env->insn_idx == 0); 19571 19571 - ret = do_check_common(env, i); 19131 19131 + ret = do_check_common(env, i, env->exception_callback_subprog == i); 19572 19132 if (ret) { 19573 19133 return ret; 19574 19134 } else if (env->log.level & BPF_LOG_LEVEL) { ··· 19585 19145 int ret; 19586 19146 19587 19147 env->insn_idx = 0; 19588 19588 - ret = do_check_common(env, 0); 19148 19148 + ret = do_check_common(env, 0, false); 19589 19149 if (!ret) 19590 19150 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 19591 19151 return ret; ··· 19752 19312 } 19753 19313 if (subprog == -1) { 19754 19314 bpf_log(log, "Subprog %s doesn't exist\n", tname); 19315 19315 + return -EINVAL; 19316 19316 + } 19317 19317 + if (aux->func && aux->func[subprog]->aux->exception_cb) { 19318 19318 + bpf_log(log, 19319 19319 + "%s programs cannot attach to exception callback\n", 19320 19320 + prog_extension ? "Extension" : "FENTRY/FEXIT"); 19755 19321 return -EINVAL; 19756 19322 } 19757 19323 conservative = aux->func_info_aux[subprog].unreliable; ··· 20089 19643 if (!tr) 20090 19644 return -ENOMEM; 20091 19645 19646 19646 + if (tgt_prog && tgt_prog->aux->tail_call_reachable) 19647 19647 + tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX; 19648 19648 + 20092 19649 prog->aux->dst_trampoline = tr; 20093 19650 return 0; 20094 19651 } ··· 20185 19736 GFP_USER); 20186 19737 ret = -ENOMEM; 20187 19738 if (!env->explored_states) 19739 19739 + goto skip_full_check; 19740 19740 + 19741 19741 + ret = check_btf_info_early(env, attr, uattr); 19742 19742 + if (ret < 0) 20188 19743 goto skip_full_check; 20189 19744 20190 19745 ret = add_subprog_and_kfunc(env);

+371

lib/test_bpf.c

reviewed

··· 5111 5111 { }, 5112 5112 { { 0, 0xffffffff } } 5113 5113 }, 5114 5114 + /* MOVSX32 */ 5115 5115 + { 5116 5116 + "ALU_MOVSX | BPF_B", 5117 5117 + .u.insns_int = { 5118 5118 + BPF_LD_IMM64(R2, 0x00000000ffffffefLL), 5119 5119 + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), 5120 5120 + BPF_MOVSX32_REG(R1, R3, 8), 5121 5121 + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), 5122 5122 + BPF_MOV32_IMM(R0, 2), 5123 5123 + BPF_EXIT_INSN(), 5124 5124 + BPF_MOV32_IMM(R0, 1), 5125 5125 + BPF_EXIT_INSN(), 5126 5126 + }, 5127 5127 + INTERNAL, 5128 5128 + { }, 5129 5129 + { { 0, 0x1 } }, 5130 5130 + }, 5131 5131 + { 5132 5132 + "ALU_MOVSX | BPF_H", 5133 5133 + .u.insns_int = { 5134 5134 + BPF_LD_IMM64(R2, 0x00000000ffffbeefLL), 5135 5135 + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), 5136 5136 + BPF_MOVSX32_REG(R1, R3, 16), 5137 5137 + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), 5138 5138 + BPF_MOV32_IMM(R0, 2), 5139 5139 + BPF_EXIT_INSN(), 5140 5140 + BPF_MOV32_IMM(R0, 1), 5141 5141 + BPF_EXIT_INSN(), 5142 5142 + }, 5143 5143 + INTERNAL, 5144 5144 + { }, 5145 5145 + { { 0, 0x1 } }, 5146 5146 + }, 5147 5147 + { 5148 5148 + "ALU_MOVSX | BPF_W", 5149 5149 + .u.insns_int = { 5150 5150 + BPF_LD_IMM64(R2, 0x00000000deadbeefLL), 5151 5151 + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), 5152 5152 + BPF_MOVSX32_REG(R1, R3, 32), 5153 5153 + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), 5154 5154 + BPF_MOV32_IMM(R0, 2), 5155 5155 + BPF_EXIT_INSN(), 5156 5156 + BPF_MOV32_IMM(R0, 1), 5157 5157 + BPF_EXIT_INSN(), 5158 5158 + }, 5159 5159 + INTERNAL, 5160 5160 + { }, 5161 5161 + { { 0, 0x1 } }, 5162 5162 + }, 5163 5163 + /* MOVSX64 REG */ 5164 5164 + { 5165 5165 + "ALU64_MOVSX | BPF_B", 5166 5166 + .u.insns_int = { 5167 5167 + BPF_LD_IMM64(R2, 0xffffffffffffffefLL), 5168 5168 + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), 5169 5169 + BPF_MOVSX64_REG(R1, R3, 8), 5170 5170 + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), 5171 5171 + BPF_MOV32_IMM(R0, 2), 5172 5172 + BPF_EXIT_INSN(), 5173 5173 + BPF_MOV32_IMM(R0, 1), 5174 5174 + BPF_EXIT_INSN(), 5175 5175 + }, 5176 5176 + INTERNAL, 5177 5177 + { }, 5178 5178 + { { 0, 0x1 } }, 5179 5179 + }, 5180 5180 + { 5181 5181 + "ALU64_MOVSX | BPF_H", 5182 5182 + .u.insns_int = { 5183 5183 + BPF_LD_IMM64(R2, 0xffffffffffffbeefLL), 5184 5184 + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), 5185 5185 + BPF_MOVSX64_REG(R1, R3, 16), 5186 5186 + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), 5187 5187 + BPF_MOV32_IMM(R0, 2), 5188 5188 + BPF_EXIT_INSN(), 5189 5189 + BPF_MOV32_IMM(R0, 1), 5190 5190 + BPF_EXIT_INSN(), 5191 5191 + }, 5192 5192 + INTERNAL, 5193 5193 + { }, 5194 5194 + { { 0, 0x1 } }, 5195 5195 + }, 5196 5196 + { 5197 5197 + "ALU64_MOVSX | BPF_W", 5198 5198 + .u.insns_int = { 5199 5199 + BPF_LD_IMM64(R2, 0xffffffffdeadbeefLL), 5200 5200 + BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), 5201 5201 + BPF_MOVSX64_REG(R1, R3, 32), 5202 5202 + BPF_JMP_REG(BPF_JEQ, R2, R1, 2), 5203 5203 + BPF_MOV32_IMM(R0, 2), 5204 5204 + BPF_EXIT_INSN(), 5205 5205 + BPF_MOV32_IMM(R0, 1), 5206 5206 + BPF_EXIT_INSN(), 5207 5207 + }, 5208 5208 + INTERNAL, 5209 5209 + { }, 5210 5210 + { { 0, 0x1 } }, 5211 5211 + }, 5114 5212 /* BPF_ALU | BPF_ADD | BPF_X */ 5115 5213 { 5116 5214 "ALU_ADD_X: 1 + 2 = 3", ··· 6202 6104 INTERNAL, 6203 6105 { }, 6204 6106 { { 0, 2 } }, 6107 6107 + }, 6108 6108 + /* BPF_ALU | BPF_DIV | BPF_X off=1 (SDIV) */ 6109 6109 + { 6110 6110 + "ALU_SDIV_X: -6 / 2 = -3", 6111 6111 + .u.insns_int = { 6112 6112 + BPF_LD_IMM64(R0, -6), 6113 6113 + BPF_ALU32_IMM(BPF_MOV, R1, 2), 6114 6114 + BPF_ALU32_REG_OFF(BPF_DIV, R0, R1, 1), 6115 6115 + BPF_EXIT_INSN(), 6116 6116 + }, 6117 6117 + INTERNAL, 6118 6118 + { }, 6119 6119 + { { 0, -3 } }, 6120 6120 + }, 6121 6121 + /* BPF_ALU | BPF_DIV | BPF_K off=1 (SDIV) */ 6122 6122 + { 6123 6123 + "ALU_SDIV_K: -6 / 2 = -3", 6124 6124 + .u.insns_int = { 6125 6125 + BPF_LD_IMM64(R0, -6), 6126 6126 + BPF_ALU32_IMM_OFF(BPF_DIV, R0, 2, 1), 6127 6127 + BPF_EXIT_INSN(), 6128 6128 + }, 6129 6129 + INTERNAL, 6130 6130 + { }, 6131 6131 + { { 0, -3 } }, 6132 6132 + }, 6133 6133 + /* BPF_ALU64 | BPF_DIV | BPF_X off=1 (SDIV64) */ 6134 6134 + { 6135 6135 + "ALU64_SDIV_X: -6 / 2 = -3", 6136 6136 + .u.insns_int = { 6137 6137 + BPF_LD_IMM64(R0, -6), 6138 6138 + BPF_ALU32_IMM(BPF_MOV, R1, 2), 6139 6139 + BPF_ALU64_REG_OFF(BPF_DIV, R0, R1, 1), 6140 6140 + BPF_EXIT_INSN(), 6141 6141 + }, 6142 6142 + INTERNAL, 6143 6143 + { }, 6144 6144 + { { 0, -3 } }, 6145 6145 + }, 6146 6146 + /* BPF_ALU64 | BPF_DIV | BPF_K off=1 (SDIV64) */ 6147 6147 + { 6148 6148 + "ALU64_SDIV_K: -6 / 2 = -3", 6149 6149 + .u.insns_int = { 6150 6150 + BPF_LD_IMM64(R0, -6), 6151 6151 + BPF_ALU64_IMM_OFF(BPF_DIV, R0, 2, 1), 6152 6152 + BPF_EXIT_INSN(), 6153 6153 + }, 6154 6154 + INTERNAL, 6155 6155 + { }, 6156 6156 + { { 0, -3 } }, 6157 6157 + }, 6158 6158 + /* BPF_ALU | BPF_MOD | BPF_X off=1 (SMOD) */ 6159 6159 + { 6160 6160 + "ALU_SMOD_X: -7 % 2 = -1", 6161 6161 + .u.insns_int = { 6162 6162 + BPF_LD_IMM64(R0, -7), 6163 6163 + BPF_ALU32_IMM(BPF_MOV, R1, 2), 6164 6164 + BPF_ALU32_REG_OFF(BPF_MOD, R0, R1, 1), 6165 6165 + BPF_EXIT_INSN(), 6166 6166 + }, 6167 6167 + INTERNAL, 6168 6168 + { }, 6169 6169 + { { 0, -1 } }, 6170 6170 + }, 6171 6171 + /* BPF_ALU | BPF_MOD | BPF_K off=1 (SMOD) */ 6172 6172 + { 6173 6173 + "ALU_SMOD_K: -7 % 2 = -1", 6174 6174 + .u.insns_int = { 6175 6175 + BPF_LD_IMM64(R0, -7), 6176 6176 + BPF_ALU32_IMM_OFF(BPF_MOD, R0, 2, 1), 6177 6177 + BPF_EXIT_INSN(), 6178 6178 + }, 6179 6179 + INTERNAL, 6180 6180 + { }, 6181 6181 + { { 0, -1 } }, 6182 6182 + }, 6183 6183 + /* BPF_ALU64 | BPF_MOD | BPF_X off=1 (SMOD64) */ 6184 6184 + { 6185 6185 + "ALU64_SMOD_X: -7 % 2 = -1", 6186 6186 + .u.insns_int = { 6187 6187 + BPF_LD_IMM64(R0, -7), 6188 6188 + BPF_ALU32_IMM(BPF_MOV, R1, 2), 6189 6189 + BPF_ALU64_REG_OFF(BPF_MOD, R0, R1, 1), 6190 6190 + BPF_EXIT_INSN(), 6191 6191 + }, 6192 6192 + INTERNAL, 6193 6193 + { }, 6194 6194 + { { 0, -1 } }, 6195 6195 + }, 6196 6196 + /* BPF_ALU64 | BPF_MOD | BPF_K off=1 (SMOD64) */ 6197 6197 + { 6198 6198 + "ALU64_SMOD_X: -7 % 2 = -1", 6199 6199 + .u.insns_int = { 6200 6200 + BPF_LD_IMM64(R0, -7), 6201 6201 + BPF_ALU64_IMM_OFF(BPF_MOD, R0, 2, 1), 6202 6202 + BPF_EXIT_INSN(), 6203 6203 + }, 6204 6204 + INTERNAL, 6205 6205 + { }, 6206 6206 + { { 0, -1 } }, 6205 6207 }, 6206 6208 /* BPF_ALU | BPF_AND | BPF_X */ 6207 6209 { ··· 8035 7837 { }, 8036 7838 { { 0, (u32) (cpu_to_le64(0xfedcba9876543210ULL) >> 32) } }, 8037 7839 }, 7840 7840 + /* BSWAP */ 7841 7841 + { 7842 7842 + "BSWAP 16: 0x0123456789abcdef -> 0xefcd", 7843 7843 + .u.insns_int = { 7844 7844 + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), 7845 7845 + BPF_BSWAP(R0, 16), 7846 7846 + BPF_EXIT_INSN(), 7847 7847 + }, 7848 7848 + INTERNAL, 7849 7849 + { }, 7850 7850 + { { 0, 0xefcd } }, 7851 7851 + }, 7852 7852 + { 7853 7853 + "BSWAP 32: 0x0123456789abcdef -> 0xefcdab89", 7854 7854 + .u.insns_int = { 7855 7855 + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), 7856 7856 + BPF_BSWAP(R0, 32), 7857 7857 + BPF_ALU64_REG(BPF_MOV, R1, R0), 7858 7858 + BPF_ALU64_IMM(BPF_RSH, R1, 32), 7859 7859 + BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ 7860 7860 + BPF_EXIT_INSN(), 7861 7861 + }, 7862 7862 + INTERNAL, 7863 7863 + { }, 7864 7864 + { { 0, 0xefcdab89 } }, 7865 7865 + }, 7866 7866 + { 7867 7867 + "BSWAP 64: 0x0123456789abcdef -> 0x67452301", 7868 7868 + .u.insns_int = { 7869 7869 + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), 7870 7870 + BPF_BSWAP(R0, 64), 7871 7871 + BPF_EXIT_INSN(), 7872 7872 + }, 7873 7873 + INTERNAL, 7874 7874 + { }, 7875 7875 + { { 0, 0x67452301 } }, 7876 7876 + }, 7877 7877 + { 7878 7878 + "BSWAP 64: 0x0123456789abcdef >> 32 -> 0xefcdab89", 7879 7879 + .u.insns_int = { 7880 7880 + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), 7881 7881 + BPF_BSWAP(R0, 64), 7882 7882 + BPF_ALU64_IMM(BPF_RSH, R0, 32), 7883 7883 + BPF_EXIT_INSN(), 7884 7884 + }, 7885 7885 + INTERNAL, 7886 7886 + { }, 7887 7887 + { { 0, 0xefcdab89 } }, 7888 7888 + }, 7889 7889 + /* BSWAP, reversed */ 7890 7890 + { 7891 7891 + "BSWAP 16: 0xfedcba9876543210 -> 0x1032", 7892 7892 + .u.insns_int = { 7893 7893 + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), 7894 7894 + BPF_BSWAP(R0, 16), 7895 7895 + BPF_EXIT_INSN(), 7896 7896 + }, 7897 7897 + INTERNAL, 7898 7898 + { }, 7899 7899 + { { 0, 0x1032 } }, 7900 7900 + }, 7901 7901 + { 7902 7902 + "BSWAP 32: 0xfedcba9876543210 -> 0x10325476", 7903 7903 + .u.insns_int = { 7904 7904 + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), 7905 7905 + BPF_BSWAP(R0, 32), 7906 7906 + BPF_ALU64_REG(BPF_MOV, R1, R0), 7907 7907 + BPF_ALU64_IMM(BPF_RSH, R1, 32), 7908 7908 + BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ 7909 7909 + BPF_EXIT_INSN(), 7910 7910 + }, 7911 7911 + INTERNAL, 7912 7912 + { }, 7913 7913 + { { 0, 0x10325476 } }, 7914 7914 + }, 7915 7915 + { 7916 7916 + "BSWAP 64: 0xfedcba9876543210 -> 0x98badcfe", 7917 7917 + .u.insns_int = { 7918 7918 + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), 7919 7919 + BPF_BSWAP(R0, 64), 7920 7920 + BPF_EXIT_INSN(), 7921 7921 + }, 7922 7922 + INTERNAL, 7923 7923 + { }, 7924 7924 + { { 0, 0x98badcfe } }, 7925 7925 + }, 7926 7926 + { 7927 7927 + "BSWAP 64: 0xfedcba9876543210 >> 32 -> 0x10325476", 7928 7928 + .u.insns_int = { 7929 7929 + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), 7930 7930 + BPF_BSWAP(R0, 64), 7931 7931 + BPF_ALU64_IMM(BPF_RSH, R0, 32), 7932 7932 + BPF_EXIT_INSN(), 7933 7933 + }, 7934 7934 + INTERNAL, 7935 7935 + { }, 7936 7936 + { { 0, 0x10325476 } }, 7937 7937 + }, 8038 7938 /* BPF_LDX_MEM B/H/W/DW */ 8039 7939 { 8040 7940 "BPF_LDX_MEM | BPF_B, base", ··· 8523 8227 { }, 8524 8228 { { 32, 0 } }, 8525 8229 .stack_depth = 0, 8230 8230 + }, 8231 8231 + /* BPF_LDX_MEMSX B/H/W */ 8232 8232 + { 8233 8233 + "BPF_LDX_MEMSX | BPF_B", 8234 8234 + .u.insns_int = { 8235 8235 + BPF_LD_IMM64(R1, 0xdead0000000000f0ULL), 8236 8236 + BPF_LD_IMM64(R2, 0xfffffffffffffff0ULL), 8237 8237 + BPF_STX_MEM(BPF_DW, R10, R1, -8), 8238 8238 + #ifdef __BIG_ENDIAN 8239 8239 + BPF_LDX_MEMSX(BPF_B, R0, R10, -1), 8240 8240 + #else 8241 8241 + BPF_LDX_MEMSX(BPF_B, R0, R10, -8), 8242 8242 + #endif 8243 8243 + BPF_JMP_REG(BPF_JNE, R0, R2, 1), 8244 8244 + BPF_ALU64_IMM(BPF_MOV, R0, 0), 8245 8245 + BPF_EXIT_INSN(), 8246 8246 + }, 8247 8247 + INTERNAL, 8248 8248 + { }, 8249 8249 + { { 0, 0 } }, 8250 8250 + .stack_depth = 8, 8251 8251 + }, 8252 8252 + { 8253 8253 + "BPF_LDX_MEMSX | BPF_H", 8254 8254 + .u.insns_int = { 8255 8255 + BPF_LD_IMM64(R1, 0xdead00000000f123ULL), 8256 8256 + BPF_LD_IMM64(R2, 0xfffffffffffff123ULL), 8257 8257 + BPF_STX_MEM(BPF_DW, R10, R1, -8), 8258 8258 + #ifdef __BIG_ENDIAN 8259 8259 + BPF_LDX_MEMSX(BPF_H, R0, R10, -2), 8260 8260 + #else 8261 8261 + BPF_LDX_MEMSX(BPF_H, R0, R10, -8), 8262 8262 + #endif 8263 8263 + BPF_JMP_REG(BPF_JNE, R0, R2, 1), 8264 8264 + BPF_ALU64_IMM(BPF_MOV, R0, 0), 8265 8265 + BPF_EXIT_INSN(), 8266 8266 + }, 8267 8267 + INTERNAL, 8268 8268 + { }, 8269 8269 + { { 0, 0 } }, 8270 8270 + .stack_depth = 8, 8271 8271 + }, 8272 8272 + { 8273 8273 + "BPF_LDX_MEMSX | BPF_W", 8274 8274 + .u.insns_int = { 8275 8275 + BPF_LD_IMM64(R1, 0x00000000deadbeefULL), 8276 8276 + BPF_LD_IMM64(R2, 0xffffffffdeadbeefULL), 8277 8277 + BPF_STX_MEM(BPF_DW, R10, R1, -8), 8278 8278 + #ifdef __BIG_ENDIAN 8279 8279 + BPF_LDX_MEMSX(BPF_W, R0, R10, -4), 8280 8280 + #else 8281 8281 + BPF_LDX_MEMSX(BPF_W, R0, R10, -8), 8282 8282 + #endif 8283 8283 + BPF_JMP_REG(BPF_JNE, R0, R2, 1), 8284 8284 + BPF_ALU64_IMM(BPF_MOV, R0, 0), 8285 8285 + BPF_EXIT_INSN(), 8286 8286 + }, 8287 8287 + INTERNAL, 8288 8288 + { }, 8289 8289 + { { 0, 0 } }, 8290 8290 + .stack_depth = 8, 8526 8291 }, 8527 8292 /* BPF_STX_MEM B/H/W/DW */ 8528 8293 { ··· 9823 9466 .u.insns_int = { 9824 9467 BPF_ALU32_IMM(BPF_MOV, R0, 0), 9825 9468 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 9469 9469 + BPF_EXIT_INSN(), 9470 9470 + BPF_ALU32_IMM(BPF_MOV, R0, 1), 9471 9471 + BPF_EXIT_INSN(), 9472 9472 + }, 9473 9473 + INTERNAL, 9474 9474 + { }, 9475 9475 + { { 0, 1 } }, 9476 9476 + }, 9477 9477 + /* BPF_JMP32 | BPF_JA */ 9478 9478 + { 9479 9479 + "JMP32_JA: Unconditional jump: if (true) return 1", 9480 9480 + .u.insns_int = { 9481 9481 + BPF_ALU32_IMM(BPF_MOV, R0, 0), 9482 9482 + BPF_JMP32_IMM(BPF_JA, 0, 1, 0), 9826 9483 BPF_EXIT_INSN(), 9827 9484 BPF_ALU32_IMM(BPF_MOV, R0, 1), 9828 9485 BPF_EXIT_INSN(),

-1

mm/kasan/kasan.h

reviewed

··· 564 564 * code. Declared here to avoid warnings about missing declarations. 565 565 */ 566 566 567 567 - asmlinkage void kasan_unpoison_task_stack_below(const void *watermark); 568 567 void __asan_register_globals(void *globals, ssize_t size); 569 568 void __asan_unregister_globals(void *globals, ssize_t size); 570 569 void __asan_handle_no_return(void);

+11 -1

net/core/netdev-genl.c

reviewed

··· 5 5 #include <linux/rtnetlink.h> 6 6 #include <net/net_namespace.h> 7 7 #include <net/sock.h> 8 8 + #include <net/xdp.h> 8 9 9 10 #include "netdev-genl-gen.h" 10 11 ··· 13 12 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, 14 13 const struct genl_info *info) 15 14 { 15 15 + u64 xdp_rx_meta = 0; 16 16 void *hdr; 17 17 18 18 hdr = genlmsg_iput(rsp, info); 19 19 if (!hdr) 20 20 return -EMSGSIZE; 21 21 22 22 + #define XDP_METADATA_KFUNC(_, flag, __, xmo) \ 23 23 + if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \ 24 24 + xdp_rx_meta |= flag; 25 25 + XDP_METADATA_KFUNC_xxx 26 26 + #undef XDP_METADATA_KFUNC 27 27 + 22 28 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) || 23 29 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES, 24 24 - netdev->xdp_features, NETDEV_A_DEV_PAD)) { 30 30 + netdev->xdp_features, NETDEV_A_DEV_PAD) || 31 31 + nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, 32 32 + xdp_rx_meta, NETDEV_A_DEV_PAD)) { 25 33 genlmsg_cancel(rsp, hdr); 26 34 return -EINVAL; 27 35 }

+2 -2

net/core/xdp.c

reviewed

··· 741 741 __diag_pop(); 742 742 743 743 BTF_SET8_START(xdp_metadata_kfunc_ids) 744 744 - #define XDP_METADATA_KFUNC(_, name) BTF_ID_FLAGS(func, name, KF_TRUSTED_ARGS) 744 744 + #define XDP_METADATA_KFUNC(_, __, name, ___) BTF_ID_FLAGS(func, name, KF_TRUSTED_ARGS) 745 745 XDP_METADATA_KFUNC_xxx 746 746 #undef XDP_METADATA_KFUNC 747 747 BTF_SET8_END(xdp_metadata_kfunc_ids) ··· 752 752 }; 753 753 754 754 BTF_ID_LIST(xdp_metadata_kfunc_ids_unsorted) 755 755 - #define XDP_METADATA_KFUNC(name, str) BTF_ID(func, str) 755 755 + #define XDP_METADATA_KFUNC(name, _, str, __) BTF_ID(func, str) 756 756 XDP_METADATA_KFUNC_xxx 757 757 #undef XDP_METADATA_KFUNC 758 758

+1 -1

net/xdp/xsk.c

reviewed

··· 1228 1228 1229 1229 xs->dev = dev; 1230 1230 xs->zc = xs->umem->zc; 1231 1231 - xs->sg = !!(flags & XDP_USE_SG); 1231 1231 + xs->sg = !!(xs->umem->flags & XDP_UMEM_SG_FLAG); 1232 1232 xs->queue_id = qid; 1233 1233 xp_add_xsk(xs->pool, xs); 1234 1234

+3

net/xdp/xsk_buff_pool.c

reviewed

··· 170 170 if (err) 171 171 return err; 172 172 173 173 + if (flags & XDP_USE_SG) 174 174 + pool->umem->flags |= XDP_UMEM_SG_FLAG; 175 175 + 173 176 if (flags & XDP_USE_NEED_WAKEUP) 174 177 pool->uses_need_wakeup = true; 175 178 /* Tx needs to be explicitly woken up the first time. Also

+4

samples/bpf/Makefile

reviewed

··· 175 175 TPROGS_CFLAGS += -I$(LIBBPF_INCLUDE) 176 176 TPROGS_CFLAGS += -I$(srctree)/tools/include 177 177 TPROGS_CFLAGS += -I$(srctree)/tools/perf 178 178 + TPROGS_CFLAGS += -I$(srctree)/tools/lib 178 179 TPROGS_CFLAGS += -DHAVE_ATTR_TEST=0 179 180 180 181 ifdef SYSROOT ··· 315 314 316 315 $(obj)/$(XDP_SAMPLE): TPROGS_CFLAGS = $(XDP_SAMPLE_CFLAGS) 317 316 $(obj)/$(XDP_SAMPLE): $(src)/xdp_sample_user.h $(src)/xdp_sample_shared.h 317 317 + # Override includes for trace_helpers.o because __must_check won't be defined 318 318 + # in our include path. 319 319 + $(obj)/$(TRACE_HELPERS): TPROGS_CFLAGS := $(TPROGS_CFLAGS) -D__must_check= 318 320 319 321 -include $(BPF_SAMPLES_PATH)/Makefile.target 320 322

+1 -1

tools/bpf/bpftool/gen.c

reviewed

··· 1209 1209 codegen("\ 1210 1210 \n\ 1211 1211 \n\ 1212 1212 - s->data = (void *)%2$s__elf_bytes(&s->data_sz); \n\ 1212 1212 + s->data = %2$s__elf_bytes(&s->data_sz); \n\ 1213 1213 \n\ 1214 1214 obj->skeleton = s; \n\ 1215 1215 return 0; \n\

+8 -1

tools/include/uapi/linux/bpf.h

reviewed

··· 932 932 */ 933 933 BPF_MAP_TYPE_CGROUP_STORAGE = BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED, 934 934 BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, 935 935 - BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, 935 935 + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED, 936 936 + /* BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE is available to bpf programs 937 937 + * attaching to a cgroup. The new mechanism (BPF_MAP_TYPE_CGRP_STORAGE + 938 938 + * local percpu kptr) supports all BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE 939 939 + * functionality and more. So mark * BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE 940 940 + * deprecated. 941 941 + */ 942 942 + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE = BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED, 936 943 BPF_MAP_TYPE_QUEUE, 937 944 BPF_MAP_TYPE_STACK, 938 945 BPF_MAP_TYPE_SK_STORAGE,

+16

tools/include/uapi/linux/netdev.h

reviewed

··· 38 38 NETDEV_XDP_ACT_MASK = 127, 39 39 }; 40 40 41 41 + /** 42 42 + * enum netdev_xdp_rx_metadata 43 43 + * @NETDEV_XDP_RX_METADATA_TIMESTAMP: Device is capable of exposing receive HW 44 44 + * timestamp via bpf_xdp_metadata_rx_timestamp(). 45 45 + * @NETDEV_XDP_RX_METADATA_HASH: Device is capable of exposing receive packet 46 46 + * hash via bpf_xdp_metadata_rx_hash(). 47 47 + */ 48 48 + enum netdev_xdp_rx_metadata { 49 49 + NETDEV_XDP_RX_METADATA_TIMESTAMP = 1, 50 50 + NETDEV_XDP_RX_METADATA_HASH = 2, 51 51 + 52 52 + /* private: */ 53 53 + NETDEV_XDP_RX_METADATA_MASK = 3, 54 54 + }; 55 55 + 41 56 enum { 42 57 NETDEV_A_DEV_IFINDEX = 1, 43 58 NETDEV_A_DEV_PAD, 44 59 NETDEV_A_DEV_XDP_FEATURES, 45 60 NETDEV_A_DEV_XDP_ZC_MAX_SEGS, 61 61 + NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, 46 62 47 63 __NETDEV_A_DEV_MAX, 48 64 NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)

+1

tools/lib/bpf/bpf_helpers.h

reviewed

··· 181 181 #define __ksym __attribute__((section(".ksyms"))) 182 182 #define __kptr_untrusted __attribute__((btf_type_tag("kptr_untrusted"))) 183 183 #define __kptr __attribute__((btf_type_tag("kptr"))) 184 184 + #define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr"))) 184 185 185 186 #define bpf_ksym_exists(sym) ({ \ 186 187 _Static_assert(!__builtin_constant_p(!!sym), #sym " should be marked as __weak"); \

+160

tools/lib/bpf/btf.c

reviewed

··· 448 448 return 0; 449 449 } 450 450 451 451 + static int btf_validate_str(const struct btf *btf, __u32 str_off, const char *what, __u32 type_id) 452 452 + { 453 453 + const char *s; 454 454 + 455 455 + s = btf__str_by_offset(btf, str_off); 456 456 + if (!s) { 457 457 + pr_warn("btf: type [%u]: invalid %s (string offset %u)\n", type_id, what, str_off); 458 458 + return -EINVAL; 459 459 + } 460 460 + 461 461 + return 0; 462 462 + } 463 463 + 464 464 + static int btf_validate_id(const struct btf *btf, __u32 id, __u32 ctx_id) 465 465 + { 466 466 + const struct btf_type *t; 467 467 + 468 468 + t = btf__type_by_id(btf, id); 469 469 + if (!t) { 470 470 + pr_warn("btf: type [%u]: invalid referenced type ID %u\n", ctx_id, id); 471 471 + return -EINVAL; 472 472 + } 473 473 + 474 474 + return 0; 475 475 + } 476 476 + 477 477 + static int btf_validate_type(const struct btf *btf, const struct btf_type *t, __u32 id) 478 478 + { 479 479 + __u32 kind = btf_kind(t); 480 480 + int err, i, n; 481 481 + 482 482 + err = btf_validate_str(btf, t->name_off, "type name", id); 483 483 + if (err) 484 484 + return err; 485 485 + 486 486 + switch (kind) { 487 487 + case BTF_KIND_UNKN: 488 488 + case BTF_KIND_INT: 489 489 + case BTF_KIND_FWD: 490 490 + case BTF_KIND_FLOAT: 491 491 + break; 492 492 + case BTF_KIND_PTR: 493 493 + case BTF_KIND_TYPEDEF: 494 494 + case BTF_KIND_VOLATILE: 495 495 + case BTF_KIND_CONST: 496 496 + case BTF_KIND_RESTRICT: 497 497 + case BTF_KIND_VAR: 498 498 + case BTF_KIND_DECL_TAG: 499 499 + case BTF_KIND_TYPE_TAG: 500 500 + err = btf_validate_id(btf, t->type, id); 501 501 + if (err) 502 502 + return err; 503 503 + break; 504 504 + case BTF_KIND_ARRAY: { 505 505 + const struct btf_array *a = btf_array(t); 506 506 + 507 507 + err = btf_validate_id(btf, a->type, id); 508 508 + err = err ?: btf_validate_id(btf, a->index_type, id); 509 509 + if (err) 510 510 + return err; 511 511 + break; 512 512 + } 513 513 + case BTF_KIND_STRUCT: 514 514 + case BTF_KIND_UNION: { 515 515 + const struct btf_member *m = btf_members(t); 516 516 + 517 517 + n = btf_vlen(t); 518 518 + for (i = 0; i < n; i++, m++) { 519 519 + err = btf_validate_str(btf, m->name_off, "field name", id); 520 520 + err = err ?: btf_validate_id(btf, m->type, id); 521 521 + if (err) 522 522 + return err; 523 523 + } 524 524 + break; 525 525 + } 526 526 + case BTF_KIND_ENUM: { 527 527 + const struct btf_enum *m = btf_enum(t); 528 528 + 529 529 + n = btf_vlen(t); 530 530 + for (i = 0; i < n; i++, m++) { 531 531 + err = btf_validate_str(btf, m->name_off, "enum name", id); 532 532 + if (err) 533 533 + return err; 534 534 + } 535 535 + break; 536 536 + } 537 537 + case BTF_KIND_ENUM64: { 538 538 + const struct btf_enum64 *m = btf_enum64(t); 539 539 + 540 540 + n = btf_vlen(t); 541 541 + for (i = 0; i < n; i++, m++) { 542 542 + err = btf_validate_str(btf, m->name_off, "enum name", id); 543 543 + if (err) 544 544 + return err; 545 545 + } 546 546 + break; 547 547 + } 548 548 + case BTF_KIND_FUNC: { 549 549 + const struct btf_type *ft; 550 550 + 551 551 + err = btf_validate_id(btf, t->type, id); 552 552 + if (err) 553 553 + return err; 554 554 + ft = btf__type_by_id(btf, t->type); 555 555 + if (btf_kind(ft) != BTF_KIND_FUNC_PROTO) { 556 556 + pr_warn("btf: type [%u]: referenced type [%u] is not FUNC_PROTO\n", id, t->type); 557 557 + return -EINVAL; 558 558 + } 559 559 + break; 560 560 + } 561 561 + case BTF_KIND_FUNC_PROTO: { 562 562 + const struct btf_param *m = btf_params(t); 563 563 + 564 564 + n = btf_vlen(t); 565 565 + for (i = 0; i < n; i++, m++) { 566 566 + err = btf_validate_str(btf, m->name_off, "param name", id); 567 567 + err = err ?: btf_validate_id(btf, m->type, id); 568 568 + if (err) 569 569 + return err; 570 570 + } 571 571 + break; 572 572 + } 573 573 + case BTF_KIND_DATASEC: { 574 574 + const struct btf_var_secinfo *m = btf_var_secinfos(t); 575 575 + 576 576 + n = btf_vlen(t); 577 577 + for (i = 0; i < n; i++, m++) { 578 578 + err = btf_validate_id(btf, m->type, id); 579 579 + if (err) 580 580 + return err; 581 581 + } 582 582 + break; 583 583 + } 584 584 + default: 585 585 + pr_warn("btf: type [%u]: unrecognized kind %u\n", id, kind); 586 586 + return -EINVAL; 587 587 + } 588 588 + return 0; 589 589 + } 590 590 + 591 591 + /* Validate basic sanity of BTF. It's intentionally less thorough than 592 592 + * kernel's validation and validates only properties of BTF that libbpf relies 593 593 + * on to be correct (e.g., valid type IDs, valid string offsets, etc) 594 594 + */ 595 595 + static int btf_sanity_check(const struct btf *btf) 596 596 + { 597 597 + const struct btf_type *t; 598 598 + __u32 i, n = btf__type_cnt(btf); 599 599 + int err; 600 600 + 601 601 + for (i = 1; i < n; i++) { 602 602 + t = btf_type_by_id(btf, i); 603 603 + err = btf_validate_type(btf, t, i); 604 604 + if (err) 605 605 + return err; 606 606 + } 607 607 + return 0; 608 608 + } 609 609 + 451 610 __u32 btf__type_cnt(const struct btf *btf) 452 611 { 453 612 return btf->start_id + btf->nr_types; ··· 1061 902 1062 903 err = btf_parse_str_sec(btf); 1063 904 err = err ?: btf_parse_type_sec(btf); 905 905 + err = err ?: btf_sanity_check(btf); 1064 906 if (err) 1065 907 goto done; 1066 908

+142 -24

tools/lib/bpf/libbpf.c

reviewed

··· 436 436 int fd; 437 437 bool autoload; 438 438 bool autoattach; 439 439 + bool sym_global; 439 440 bool mark_btf_static; 440 441 enum bpf_prog_type type; 441 442 enum bpf_attach_type expected_attach_type; 443 443 + int exception_cb_idx; 442 444 443 445 int prog_ifindex; 444 446 __u32 attach_btf_obj_fd; ··· 767 765 768 766 prog->type = BPF_PROG_TYPE_UNSPEC; 769 767 prog->fd = -1; 768 768 + prog->exception_cb_idx = -1; 770 769 771 770 /* libbpf's convention for SEC("?abc...") is that it's just like 772 771 * SEC("abc...") but the corresponding bpf_program starts out with ··· 874 871 if (err) 875 872 return err; 876 873 874 874 + if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL) 875 875 + prog->sym_global = true; 876 876 + 877 877 /* if function is a global/weak symbol, but has restricted 878 878 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC 879 879 * as static to enable more permissive BPF verification mode 880 880 * with more outside context available to BPF verifier 881 881 */ 882 882 - if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL 883 883 - && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN 884 884 - || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) 882 882 + if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN 883 883 + || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) 885 884 prog->mark_btf_static = true; 886 885 887 886 nr_progs++; ··· 3146 3141 break; 3147 3142 } 3148 3143 } 3144 3144 + 3145 3145 + if (!kernel_supports(obj, FEAT_BTF_DECL_TAG)) 3146 3146 + goto skip_exception_cb; 3147 3147 + for (i = 0; i < obj->nr_programs; i++) { 3148 3148 + struct bpf_program *prog = &obj->programs[i]; 3149 3149 + int j, k, n; 3150 3150 + 3151 3151 + if (prog_is_subprog(obj, prog)) 3152 3152 + continue; 3153 3153 + n = btf__type_cnt(obj->btf); 3154 3154 + for (j = 1; j < n; j++) { 3155 3155 + const char *str = "exception_callback:", *name; 3156 3156 + size_t len = strlen(str); 3157 3157 + struct btf_type *t; 3158 3158 + 3159 3159 + t = btf_type_by_id(obj->btf, j); 3160 3160 + if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1) 3161 3161 + continue; 3162 3162 + 3163 3163 + name = btf__str_by_offset(obj->btf, t->name_off); 3164 3164 + if (strncmp(name, str, len)) 3165 3165 + continue; 3166 3166 + 3167 3167 + t = btf_type_by_id(obj->btf, t->type); 3168 3168 + if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) { 3169 3169 + pr_warn("prog '%s': exception_callback:<value> decl tag not applied to the main program\n", 3170 3170 + prog->name); 3171 3171 + return -EINVAL; 3172 3172 + } 3173 3173 + if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off))) 3174 3174 + continue; 3175 3175 + /* Multiple callbacks are specified for the same prog, 3176 3176 + * the verifier will eventually return an error for this 3177 3177 + * case, hence simply skip appending a subprog. 3178 3178 + */ 3179 3179 + if (prog->exception_cb_idx >= 0) { 3180 3180 + prog->exception_cb_idx = -1; 3181 3181 + break; 3182 3182 + } 3183 3183 + 3184 3184 + name += len; 3185 3185 + if (str_is_empty(name)) { 3186 3186 + pr_warn("prog '%s': exception_callback:<value> decl tag contains empty value\n", 3187 3187 + prog->name); 3188 3188 + return -EINVAL; 3189 3189 + } 3190 3190 + 3191 3191 + for (k = 0; k < obj->nr_programs; k++) { 3192 3192 + struct bpf_program *subprog = &obj->programs[k]; 3193 3193 + 3194 3194 + if (!prog_is_subprog(obj, subprog)) 3195 3195 + continue; 3196 3196 + if (strcmp(name, subprog->name)) 3197 3197 + continue; 3198 3198 + /* Enforce non-hidden, as from verifier point of 3199 3199 + * view it expects global functions, whereas the 3200 3200 + * mark_btf_static fixes up linkage as static. 3201 3201 + */ 3202 3202 + if (!subprog->sym_global || subprog->mark_btf_static) { 3203 3203 + pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n", 3204 3204 + prog->name, subprog->name); 3205 3205 + return -EINVAL; 3206 3206 + } 3207 3207 + /* Let's see if we already saw a static exception callback with the same name */ 3208 3208 + if (prog->exception_cb_idx >= 0) { 3209 3209 + pr_warn("prog '%s': multiple subprogs with same name as exception callback '%s'\n", 3210 3210 + prog->name, subprog->name); 3211 3211 + return -EINVAL; 3212 3212 + } 3213 3213 + prog->exception_cb_idx = k; 3214 3214 + break; 3215 3215 + } 3216 3216 + 3217 3217 + if (prog->exception_cb_idx >= 0) 3218 3218 + continue; 3219 3219 + pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name); 3220 3220 + return -ENOENT; 3221 3221 + } 3222 3222 + } 3223 3223 + skip_exception_cb: 3149 3224 3150 3225 sanitize = btf_needs_sanitization(obj); 3151 3226 if (sanitize) { ··· 6320 6235 } 6321 6236 6322 6237 static int 6238 6238 + bpf_object__append_subprog_code(struct bpf_object *obj, struct bpf_program *main_prog, 6239 6239 + struct bpf_program *subprog) 6240 6240 + { 6241 6241 + struct bpf_insn *insns; 6242 6242 + size_t new_cnt; 6243 6243 + int err; 6244 6244 + 6245 6245 + subprog->sub_insn_off = main_prog->insns_cnt; 6246 6246 + 6247 6247 + new_cnt = main_prog->insns_cnt + subprog->insns_cnt; 6248 6248 + insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); 6249 6249 + if (!insns) { 6250 6250 + pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); 6251 6251 + return -ENOMEM; 6252 6252 + } 6253 6253 + main_prog->insns = insns; 6254 6254 + main_prog->insns_cnt = new_cnt; 6255 6255 + 6256 6256 + memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, 6257 6257 + subprog->insns_cnt * sizeof(*insns)); 6258 6258 + 6259 6259 + pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", 6260 6260 + main_prog->name, subprog->insns_cnt, subprog->name); 6261 6261 + 6262 6262 + /* The subprog insns are now appended. Append its relos too. */ 6263 6263 + err = append_subprog_relos(main_prog, subprog); 6264 6264 + if (err) 6265 6265 + return err; 6266 6266 + return 0; 6267 6267 + } 6268 6268 + 6269 6269 + static int 6323 6270 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, 6324 6271 struct bpf_program *prog) 6325 6272 { 6326 6326 - size_t sub_insn_idx, insn_idx, new_cnt; 6273 6273 + size_t sub_insn_idx, insn_idx; 6327 6274 struct bpf_program *subprog; 6328 6328 - struct bpf_insn *insns, *insn; 6329 6275 struct reloc_desc *relo; 6276 6276 + struct bpf_insn *insn; 6330 6277 int err; 6331 6278 6332 6279 err = reloc_prog_func_and_line_info(obj, main_prog, prog); ··· 6433 6316 * and relocate. 6434 6317 */ 6435 6318 if (subprog->sub_insn_off == 0) { 6436 6436 - subprog->sub_insn_off = main_prog->insns_cnt; 6437 6437 - 6438 6438 - new_cnt = main_prog->insns_cnt + subprog->insns_cnt; 6439 6439 - insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); 6440 6440 - if (!insns) { 6441 6441 - pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); 6442 6442 - return -ENOMEM; 6443 6443 - } 6444 6444 - main_prog->insns = insns; 6445 6445 - main_prog->insns_cnt = new_cnt; 6446 6446 - 6447 6447 - memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, 6448 6448 - subprog->insns_cnt * sizeof(*insns)); 6449 6449 - 6450 6450 - pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", 6451 6451 - main_prog->name, subprog->insns_cnt, subprog->name); 6452 6452 - 6453 6453 - /* The subprog insns are now appended. Append its relos too. */ 6454 6454 - err = append_subprog_relos(main_prog, subprog); 6319 6319 + err = bpf_object__append_subprog_code(obj, main_prog, subprog); 6455 6320 if (err) 6456 6321 return err; 6457 6322 err = bpf_object__reloc_code(obj, main_prog, subprog); ··· 6666 6567 pr_warn("prog '%s': failed to relocate calls: %d\n", 6667 6568 prog->name, err); 6668 6569 return err; 6570 6570 + } 6571 6571 + 6572 6572 + /* Now, also append exception callback if it has not been done already. */ 6573 6573 + if (prog->exception_cb_idx >= 0) { 6574 6574 + struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx]; 6575 6575 + 6576 6576 + /* Calling exception callback directly is disallowed, which the 6577 6577 + * verifier will reject later. In case it was processed already, 6578 6578 + * we can skip this step, otherwise for all other valid cases we 6579 6579 + * have to append exception callback now. 6580 6580 + */ 6581 6581 + if (subprog->sub_insn_off == 0) { 6582 6582 + err = bpf_object__append_subprog_code(obj, prog, subprog); 6583 6583 + if (err) 6584 6584 + return err; 6585 6585 + err = bpf_object__reloc_code(obj, prog, subprog); 6586 6586 + if (err) 6587 6587 + return err; 6588 6588 + } 6669 6589 } 6670 6590 } 6671 6591 /* Process data relos for main programs */

+19

tools/net/ynl/generated/netdev-user.c

reviewed

··· 45 45 return netdev_xdp_act_strmap[value]; 46 46 } 47 47 48 48 + static const char * const netdev_xdp_rx_metadata_strmap[] = { 49 49 + [0] = "timestamp", 50 50 + [1] = "hash", 51 51 + }; 52 52 + 53 53 + const char *netdev_xdp_rx_metadata_str(enum netdev_xdp_rx_metadata value) 54 54 + { 55 55 + value = ffs(value) - 1; 56 56 + if (value < 0 || value >= (int)MNL_ARRAY_SIZE(netdev_xdp_rx_metadata_strmap)) 57 57 + return NULL; 58 58 + return netdev_xdp_rx_metadata_strmap[value]; 59 59 + } 60 60 + 48 61 /* Policies */ 49 62 struct ynl_policy_attr netdev_dev_policy[NETDEV_A_DEV_MAX + 1] = { 50 63 [NETDEV_A_DEV_IFINDEX] = { .name = "ifindex", .type = YNL_PT_U32, }, 51 64 [NETDEV_A_DEV_PAD] = { .name = "pad", .type = YNL_PT_IGNORE, }, 52 65 [NETDEV_A_DEV_XDP_FEATURES] = { .name = "xdp-features", .type = YNL_PT_U64, }, 53 66 [NETDEV_A_DEV_XDP_ZC_MAX_SEGS] = { .name = "xdp-zc-max-segs", .type = YNL_PT_U32, }, 67 67 + [NETDEV_A_DEV_XDP_RX_METADATA_FEATURES] = { .name = "xdp-rx-metadata-features", .type = YNL_PT_U64, }, 54 68 }; 55 69 56 70 struct ynl_policy_nest netdev_dev_nest = { ··· 111 97 return MNL_CB_ERROR; 112 98 dst->_present.xdp_zc_max_segs = 1; 113 99 dst->xdp_zc_max_segs = mnl_attr_get_u32(attr); 100 100 + } else if (type == NETDEV_A_DEV_XDP_RX_METADATA_FEATURES) { 101 101 + if (ynl_attr_validate(yarg, attr)) 102 102 + return MNL_CB_ERROR; 103 103 + dst->_present.xdp_rx_metadata_features = 1; 104 104 + dst->xdp_rx_metadata_features = mnl_attr_get_u64(attr); 114 105 } 115 106 } 116 107

+3

tools/net/ynl/generated/netdev-user.h

reviewed

··· 18 18 /* Enums */ 19 19 const char *netdev_op_str(int op); 20 20 const char *netdev_xdp_act_str(enum netdev_xdp_act value); 21 21 + const char *netdev_xdp_rx_metadata_str(enum netdev_xdp_rx_metadata value); 21 22 22 23 /* Common nested types */ 23 24 /* ============== NETDEV_CMD_DEV_GET ============== */ ··· 49 48 __u32 ifindex:1; 50 49 __u32 xdp_features:1; 51 50 __u32 xdp_zc_max_segs:1; 51 51 + __u32 xdp_rx_metadata_features:1; 52 52 } _present; 53 53 54 54 __u32 ifindex; 55 55 __u64 xdp_features; 56 56 __u32 xdp_zc_max_segs; 57 57 + __u64 xdp_rx_metadata_features; 57 58 }; 58 59 59 60 void netdev_dev_get_rsp_free(struct netdev_dev_get_rsp *rsp);

+1 -1

tools/net/ynl/samples/Makefile

reviewed

··· 4 4 5 5 CC=gcc 6 6 CFLAGS=-std=gnu11 -O2 -W -Wall -Wextra -Wno-unused-parameter -Wshadow \ 7 7 - -I../lib/ -I../generated/ -idirafter $(UAPI_PATH) 7 7 + -I../../../include/uapi -I../lib/ -I../generated/ -idirafter $(UAPI_PATH) 8 8 ifeq ("$(DEBUG)","1") 9 9 CFLAGS += -g -fsanitize=address -fsanitize=leak -static-libasan 10 10 endif

+7 -1

tools/net/ynl/samples/netdev.c

reviewed

··· 32 32 if (!d->_present.xdp_features) 33 33 return; 34 34 35 35 - printf("%llx:", d->xdp_features); 35 35 + printf("xdp-features (%llx):", d->xdp_features); 36 36 for (int i = 0; d->xdp_features > 1U << i; i++) { 37 37 if (d->xdp_features & (1U << i)) 38 38 printf(" %s", netdev_xdp_act_str(1 << i)); 39 39 + } 40 40 + 41 41 + printf(" xdp-rx-metadata-features (%llx):", d->xdp_rx_metadata_features); 42 42 + for (int i = 0; d->xdp_rx_metadata_features > 1U << i; i++) { 43 43 + if (d->xdp_rx_metadata_features & (1U << i)) 44 44 + printf(" %s", netdev_xdp_rx_metadata_str(1 << i)); 39 45 } 40 46 41 47 printf(" xdp-zc-max-segs=%u", d->xdp_zc_max_segs);

+1

tools/testing/selftests/bpf/DENYLIST.aarch64

reviewed

··· 1 1 bpf_cookie/multi_kprobe_attach_api # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3 2 2 bpf_cookie/multi_kprobe_link_api # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3 3 3 + exceptions # JIT does not support calling kfunc bpf_throw: -524 3 4 fexit_sleep # The test never returns. The remaining tests cannot start. 4 5 kprobe_multi_bench_attach # bpf_program__attach_kprobe_multi_opts unexpected error: -95 5 6 kprobe_multi_test/attach_api_addrs # bpf_program__attach_kprobe_multi_opts unexpected error: -95

+1

tools/testing/selftests/bpf/DENYLIST.s390x

reviewed

··· 6 6 cgrp_local_storage # prog_attach unexpected error: -524 (trampoline) 7 7 dynptr/test_dynptr_skb_data 8 8 dynptr/test_skb_readonly 9 9 + exceptions # JIT does not support calling kfunc bpf_throw (exceptions) 9 10 fexit_sleep # fexit_skel_load fexit skeleton failed (trampoline) 10 11 get_stack_raw_tp # user_stack corrupted user stack (no backchain userspace) 11 12 iters/testmod_seq* # s390x doesn't support kfuncs in modules yet

+319

tools/testing/selftests/bpf/bpf_experimental.h

reviewed

··· 131 131 */ 132 132 extern struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; 133 133 134 134 + /* Description 135 135 + * Allocates a percpu object of the type represented by 'local_type_id' in 136 136 + * program BTF. User may use the bpf_core_type_id_local macro to pass the 137 137 + * type ID of a struct in program BTF. 138 138 + * 139 139 + * The 'local_type_id' parameter must be a known constant. 140 140 + * The 'meta' parameter is rewritten by the verifier, no need for BPF 141 141 + * program to set it. 142 142 + * Returns 143 143 + * A pointer to a percpu object of the type corresponding to the passed in 144 144 + * 'local_type_id', or NULL on failure. 145 145 + */ 146 146 + extern void *bpf_percpu_obj_new_impl(__u64 local_type_id, void *meta) __ksym; 147 147 + 148 148 + /* Convenience macro to wrap over bpf_percpu_obj_new_impl */ 149 149 + #define bpf_percpu_obj_new(type) ((type __percpu_kptr *)bpf_percpu_obj_new_impl(bpf_core_type_id_local(type), NULL)) 150 150 + 151 151 + /* Description 152 152 + * Free an allocated percpu object. All fields of the object that require 153 153 + * destruction will be destructed before the storage is freed. 154 154 + * 155 155 + * The 'meta' parameter is rewritten by the verifier, no need for BPF 156 156 + * program to set it. 157 157 + * Returns 158 158 + * Void. 159 159 + */ 160 160 + extern void bpf_percpu_obj_drop_impl(void *kptr, void *meta) __ksym; 161 161 + 162 162 + /* Convenience macro to wrap over bpf_obj_drop_impl */ 163 163 + #define bpf_percpu_obj_drop(kptr) bpf_percpu_obj_drop_impl(kptr, NULL) 164 164 + 165 165 + /* Description 166 166 + * Throw a BPF exception from the program, immediately terminating its 167 167 + * execution and unwinding the stack. The supplied 'cookie' parameter 168 168 + * will be the return value of the program when an exception is thrown, 169 169 + * and the default exception callback is used. Otherwise, if an exception 170 170 + * callback is set using the '__exception_cb(callback)' declaration tag 171 171 + * on the main program, the 'cookie' parameter will be the callback's only 172 172 + * input argument. 173 173 + * 174 174 + * Thus, in case of default exception callback, 'cookie' is subjected to 175 175 + * constraints on the program's return value (as with R0 on exit). 176 176 + * Otherwise, the return value of the marked exception callback will be 177 177 + * subjected to the same checks. 178 178 + * 179 179 + * Note that throwing an exception with lingering resources (locks, 180 180 + * references, etc.) will lead to a verification error. 181 181 + * 182 182 + * Note that callbacks *cannot* call this helper. 183 183 + * Returns 184 184 + * Never. 185 185 + * Throws 186 186 + * An exception with the specified 'cookie' value. 187 187 + */ 188 188 + extern void bpf_throw(u64 cookie) __ksym; 189 189 + 190 190 + /* This macro must be used to mark the exception callback corresponding to the 191 191 + * main program. For example: 192 192 + * 193 193 + * int exception_cb(u64 cookie) { 194 194 + * return cookie; 195 195 + * } 196 196 + * 197 197 + * SEC("tc") 198 198 + * __exception_cb(exception_cb) 199 199 + * int main_prog(struct __sk_buff *ctx) { 200 200 + * ... 201 201 + * return TC_ACT_OK; 202 202 + * } 203 203 + * 204 204 + * Here, exception callback for the main program will be 'exception_cb'. Note 205 205 + * that this attribute can only be used once, and multiple exception callbacks 206 206 + * specified for the main program will lead to verification error. 207 207 + */ 208 208 + #define __exception_cb(name) __attribute__((btf_decl_tag("exception_callback:" #name))) 209 209 + 210 210 + #define __bpf_assert_signed(x) _Generic((x), \ 211 211 + unsigned long: 0, \ 212 212 + unsigned long long: 0, \ 213 213 + signed long: 1, \ 214 214 + signed long long: 1 \ 215 215 + ) 216 216 + 217 217 + #define __bpf_assert_check(LHS, op, RHS) \ 218 218 + _Static_assert(sizeof(&(LHS)), "1st argument must be an lvalue expression"); \ 219 219 + _Static_assert(sizeof(LHS) == 8, "Only 8-byte integers are supported\n"); \ 220 220 + _Static_assert(__builtin_constant_p(__bpf_assert_signed(LHS)), "internal static assert"); \ 221 221 + _Static_assert(__builtin_constant_p((RHS)), "2nd argument must be a constant expression") 222 222 + 223 223 + #define __bpf_assert(LHS, op, cons, RHS, VAL) \ 224 224 + ({ \ 225 225 + (void)bpf_throw; \ 226 226 + asm volatile ("if %[lhs] " op " %[rhs] goto +2; r1 = %[value]; call bpf_throw" \ 227 227 + : : [lhs] "r"(LHS), [rhs] cons(RHS), [value] "ri"(VAL) : ); \ 228 228 + }) 229 229 + 230 230 + #define __bpf_assert_op_sign(LHS, op, cons, RHS, VAL, supp_sign) \ 231 231 + ({ \ 232 232 + __bpf_assert_check(LHS, op, RHS); \ 233 233 + if (__bpf_assert_signed(LHS) && !(supp_sign)) \ 234 234 + __bpf_assert(LHS, "s" #op, cons, RHS, VAL); \ 235 235 + else \ 236 236 + __bpf_assert(LHS, #op, cons, RHS, VAL); \ 237 237 + }) 238 238 + 239 239 + #define __bpf_assert_op(LHS, op, RHS, VAL, supp_sign) \ 240 240 + ({ \ 241 241 + if (sizeof(typeof(RHS)) == 8) { \ 242 242 + const typeof(RHS) rhs_var = (RHS); \ 243 243 + __bpf_assert_op_sign(LHS, op, "r", rhs_var, VAL, supp_sign); \ 244 244 + } else { \ 245 245 + __bpf_assert_op_sign(LHS, op, "i", RHS, VAL, supp_sign); \ 246 246 + } \ 247 247 + }) 248 248 + 249 249 + /* Description 250 250 + * Assert that a conditional expression is true. 251 251 + * Returns 252 252 + * Void. 253 253 + * Throws 254 254 + * An exception with the value zero when the assertion fails. 255 255 + */ 256 256 + #define bpf_assert(cond) if (!(cond)) bpf_throw(0); 257 257 + 258 258 + /* Description 259 259 + * Assert that a conditional expression is true. 260 260 + * Returns 261 261 + * Void. 262 262 + * Throws 263 263 + * An exception with the specified value when the assertion fails. 264 264 + */ 265 265 + #define bpf_assert_with(cond, value) if (!(cond)) bpf_throw(value); 266 266 + 267 267 + /* Description 268 268 + * Assert that LHS is equal to RHS. This statement updates the known value 269 269 + * of LHS during verification. Note that RHS must be a constant value, and 270 270 + * must fit within the data type of LHS. 271 271 + * Returns 272 272 + * Void. 273 273 + * Throws 274 274 + * An exception with the value zero when the assertion fails. 275 275 + */ 276 276 + #define bpf_assert_eq(LHS, RHS) \ 277 277 + ({ \ 278 278 + barrier_var(LHS); \ 279 279 + __bpf_assert_op(LHS, ==, RHS, 0, true); \ 280 280 + }) 281 281 + 282 282 + /* Description 283 283 + * Assert that LHS is equal to RHS. This statement updates the known value 284 284 + * of LHS during verification. Note that RHS must be a constant value, and 285 285 + * must fit within the data type of LHS. 286 286 + * Returns 287 287 + * Void. 288 288 + * Throws 289 289 + * An exception with the specified value when the assertion fails. 290 290 + */ 291 291 + #define bpf_assert_eq_with(LHS, RHS, value) \ 292 292 + ({ \ 293 293 + barrier_var(LHS); \ 294 294 + __bpf_assert_op(LHS, ==, RHS, value, true); \ 295 295 + }) 296 296 + 297 297 + /* Description 298 298 + * Assert that LHS is less than RHS. This statement updates the known 299 299 + * bounds of LHS during verification. Note that RHS must be a constant 300 300 + * value, and must fit within the data type of LHS. 301 301 + * Returns 302 302 + * Void. 303 303 + * Throws 304 304 + * An exception with the value zero when the assertion fails. 305 305 + */ 306 306 + #define bpf_assert_lt(LHS, RHS) \ 307 307 + ({ \ 308 308 + barrier_var(LHS); \ 309 309 + __bpf_assert_op(LHS, <, RHS, 0, false); \ 310 310 + }) 311 311 + 312 312 + /* Description 313 313 + * Assert that LHS is less than RHS. This statement updates the known 314 314 + * bounds of LHS during verification. Note that RHS must be a constant 315 315 + * value, and must fit within the data type of LHS. 316 316 + * Returns 317 317 + * Void. 318 318 + * Throws 319 319 + * An exception with the specified value when the assertion fails. 320 320 + */ 321 321 + #define bpf_assert_lt_with(LHS, RHS, value) \ 322 322 + ({ \ 323 323 + barrier_var(LHS); \ 324 324 + __bpf_assert_op(LHS, <, RHS, value, false); \ 325 325 + }) 326 326 + 327 327 + /* Description 328 328 + * Assert that LHS is greater than RHS. This statement updates the known 329 329 + * bounds of LHS during verification. Note that RHS must be a constant 330 330 + * value, and must fit within the data type of LHS. 331 331 + * Returns 332 332 + * Void. 333 333 + * Throws 334 334 + * An exception with the value zero when the assertion fails. 335 335 + */ 336 336 + #define bpf_assert_gt(LHS, RHS) \ 337 337 + ({ \ 338 338 + barrier_var(LHS); \ 339 339 + __bpf_assert_op(LHS, >, RHS, 0, false); \ 340 340 + }) 341 341 + 342 342 + /* Description 343 343 + * Assert that LHS is greater than RHS. This statement updates the known 344 344 + * bounds of LHS during verification. Note that RHS must be a constant 345 345 + * value, and must fit within the data type of LHS. 346 346 + * Returns 347 347 + * Void. 348 348 + * Throws 349 349 + * An exception with the specified value when the assertion fails. 350 350 + */ 351 351 + #define bpf_assert_gt_with(LHS, RHS, value) \ 352 352 + ({ \ 353 353 + barrier_var(LHS); \ 354 354 + __bpf_assert_op(LHS, >, RHS, value, false); \ 355 355 + }) 356 356 + 357 357 + /* Description 358 358 + * Assert that LHS is less than or equal to RHS. This statement updates the 359 359 + * known bounds of LHS during verification. Note that RHS must be a 360 360 + * constant value, and must fit within the data type of LHS. 361 361 + * Returns 362 362 + * Void. 363 363 + * Throws 364 364 + * An exception with the value zero when the assertion fails. 365 365 + */ 366 366 + #define bpf_assert_le(LHS, RHS) \ 367 367 + ({ \ 368 368 + barrier_var(LHS); \ 369 369 + __bpf_assert_op(LHS, <=, RHS, 0, false); \ 370 370 + }) 371 371 + 372 372 + /* Description 373 373 + * Assert that LHS is less than or equal to RHS. This statement updates the 374 374 + * known bounds of LHS during verification. Note that RHS must be a 375 375 + * constant value, and must fit within the data type of LHS. 376 376 + * Returns 377 377 + * Void. 378 378 + * Throws 379 379 + * An exception with the specified value when the assertion fails. 380 380 + */ 381 381 + #define bpf_assert_le_with(LHS, RHS, value) \ 382 382 + ({ \ 383 383 + barrier_var(LHS); \ 384 384 + __bpf_assert_op(LHS, <=, RHS, value, false); \ 385 385 + }) 386 386 + 387 387 + /* Description 388 388 + * Assert that LHS is greater than or equal to RHS. This statement updates 389 389 + * the known bounds of LHS during verification. Note that RHS must be a 390 390 + * constant value, and must fit within the data type of LHS. 391 391 + * Returns 392 392 + * Void. 393 393 + * Throws 394 394 + * An exception with the value zero when the assertion fails. 395 395 + */ 396 396 + #define bpf_assert_ge(LHS, RHS) \ 397 397 + ({ \ 398 398 + barrier_var(LHS); \ 399 399 + __bpf_assert_op(LHS, >=, RHS, 0, false); \ 400 400 + }) 401 401 + 402 402 + /* Description 403 403 + * Assert that LHS is greater than or equal to RHS. This statement updates 404 404 + * the known bounds of LHS during verification. Note that RHS must be a 405 405 + * constant value, and must fit within the data type of LHS. 406 406 + * Returns 407 407 + * Void. 408 408 + * Throws 409 409 + * An exception with the specified value when the assertion fails. 410 410 + */ 411 411 + #define bpf_assert_ge_with(LHS, RHS, value) \ 412 412 + ({ \ 413 413 + barrier_var(LHS); \ 414 414 + __bpf_assert_op(LHS, >=, RHS, value, false); \ 415 415 + }) 416 416 + 417 417 + /* Description 418 418 + * Assert that LHS is in the range [BEG, END] (inclusive of both). This 419 419 + * statement updates the known bounds of LHS during verification. Note 420 420 + * that both BEG and END must be constant values, and must fit within the 421 421 + * data type of LHS. 422 422 + * Returns 423 423 + * Void. 424 424 + * Throws 425 425 + * An exception with the value zero when the assertion fails. 426 426 + */ 427 427 + #define bpf_assert_range(LHS, BEG, END) \ 428 428 + ({ \ 429 429 + _Static_assert(BEG <= END, "BEG must be <= END"); \ 430 430 + barrier_var(LHS); \ 431 431 + __bpf_assert_op(LHS, >=, BEG, 0, false); \ 432 432 + __bpf_assert_op(LHS, <=, END, 0, false); \ 433 433 + }) 434 434 + 435 435 + /* Description 436 436 + * Assert that LHS is in the range [BEG, END] (inclusive of both). This 437 437 + * statement updates the known bounds of LHS during verification. Note 438 438 + * that both BEG and END must be constant values, and must fit within the 439 439 + * data type of LHS. 440 440 + * Returns 441 441 + * Void. 442 442 + * Throws 443 443 + * An exception with the specified value when the assertion fails. 444 444 + */ 445 445 + #define bpf_assert_range_with(LHS, BEG, END, value) \ 446 446 + ({ \ 447 447 + _Static_assert(BEG <= END, "BEG must be <= END"); \ 448 448 + barrier_var(LHS); \ 449 449 + __bpf_assert_op(LHS, >=, BEG, value, false); \ 450 450 + __bpf_assert_op(LHS, <=, END, value, false); \ 451 451 + }) 452 452 + 134 453 #endif

+2 -2

tools/testing/selftests/bpf/prog_tests/btf.c

reviewed

··· 7296 7296 BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ 7297 7297 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(2), 1), 7298 7298 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(3), 1), 7299 7299 - BTF_FUNC_ENC(NAME_NTH(4), 2), /* [4] */ 7299 7299 + BTF_FUNC_ENC(NAME_NTH(4), 3), /* [4] */ 7300 7300 /* tag -> t */ 7301 7301 BTF_DECL_TAG_ENC(NAME_NTH(5), 2, -1), /* [5] */ 7302 7302 BTF_DECL_TAG_ENC(NAME_NTH(5), 2, -1), /* [6] */ ··· 7317 7317 BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ 7318 7318 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(2), 1), 7319 7319 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(3), 1), 7320 7320 - BTF_FUNC_ENC(NAME_NTH(4), 2), /* [4] */ 7320 7320 + BTF_FUNC_ENC(NAME_NTH(4), 3), /* [4] */ 7321 7321 BTF_DECL_TAG_ENC(NAME_NTH(5), 2, -1), /* [5] */ 7322 7322 BTF_DECL_TAG_ENC(NAME_NTH(5), 4, -1), /* [6] */ 7323 7323 BTF_DECL_TAG_ENC(NAME_NTH(5), 4, 1), /* [7] */

+408

tools/testing/selftests/bpf/prog_tests/exceptions.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + #include <test_progs.h> 3 3 + #include <network_helpers.h> 4 4 + 5 5 + #include "exceptions.skel.h" 6 6 + #include "exceptions_ext.skel.h" 7 7 + #include "exceptions_fail.skel.h" 8 8 + #include "exceptions_assert.skel.h" 9 9 + 10 10 + static char log_buf[1024 * 1024]; 11 11 + 12 12 + static void test_exceptions_failure(void) 13 13 + { 14 14 + RUN_TESTS(exceptions_fail); 15 15 + } 16 16 + 17 17 + static void test_exceptions_success(void) 18 18 + { 19 19 + LIBBPF_OPTS(bpf_test_run_opts, ropts, 20 20 + .data_in = &pkt_v4, 21 21 + .data_size_in = sizeof(pkt_v4), 22 22 + .repeat = 1, 23 23 + ); 24 24 + struct exceptions_ext *eskel = NULL; 25 25 + struct exceptions *skel; 26 26 + int ret; 27 27 + 28 28 + skel = exceptions__open(); 29 29 + if (!ASSERT_OK_PTR(skel, "exceptions__open")) 30 30 + return; 31 31 + 32 32 + ret = exceptions__load(skel); 33 33 + if (!ASSERT_OK(ret, "exceptions__load")) 34 34 + goto done; 35 35 + 36 36 + if (!ASSERT_OK(bpf_map_update_elem(bpf_map__fd(skel->maps.jmp_table), &(int){0}, 37 37 + &(int){bpf_program__fd(skel->progs.exception_tail_call_target)}, BPF_ANY), 38 38 + "bpf_map_update_elem jmp_table")) 39 39 + goto done; 40 40 + 41 41 + #define RUN_SUCCESS(_prog, return_val) \ 42 42 + if (!test__start_subtest(#_prog)) goto _prog##_##return_val; \ 43 43 + ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs._prog), &ropts); \ 44 44 + ASSERT_OK(ret, #_prog " prog run ret"); \ 45 45 + ASSERT_EQ(ropts.retval, return_val, #_prog " prog run retval"); \ 46 46 + _prog##_##return_val: 47 47 + 48 48 + RUN_SUCCESS(exception_throw_always_1, 64); 49 49 + RUN_SUCCESS(exception_throw_always_2, 32); 50 50 + RUN_SUCCESS(exception_throw_unwind_1, 16); 51 51 + RUN_SUCCESS(exception_throw_unwind_2, 32); 52 52 + RUN_SUCCESS(exception_throw_default, 0); 53 53 + RUN_SUCCESS(exception_throw_default_value, 5); 54 54 + RUN_SUCCESS(exception_tail_call, 24); 55 55 + RUN_SUCCESS(exception_ext, 0); 56 56 + RUN_SUCCESS(exception_ext_mod_cb_runtime, 35); 57 57 + RUN_SUCCESS(exception_throw_subprog, 1); 58 58 + RUN_SUCCESS(exception_assert_nz_gfunc, 1); 59 59 + RUN_SUCCESS(exception_assert_zero_gfunc, 1); 60 60 + RUN_SUCCESS(exception_assert_neg_gfunc, 1); 61 61 + RUN_SUCCESS(exception_assert_pos_gfunc, 1); 62 62 + RUN_SUCCESS(exception_assert_negeq_gfunc, 1); 63 63 + RUN_SUCCESS(exception_assert_poseq_gfunc, 1); 64 64 + RUN_SUCCESS(exception_assert_nz_gfunc_with, 1); 65 65 + RUN_SUCCESS(exception_assert_zero_gfunc_with, 1); 66 66 + RUN_SUCCESS(exception_assert_neg_gfunc_with, 1); 67 67 + RUN_SUCCESS(exception_assert_pos_gfunc_with, 1); 68 68 + RUN_SUCCESS(exception_assert_negeq_gfunc_with, 1); 69 69 + RUN_SUCCESS(exception_assert_poseq_gfunc_with, 1); 70 70 + RUN_SUCCESS(exception_bad_assert_nz_gfunc, 0); 71 71 + RUN_SUCCESS(exception_bad_assert_zero_gfunc, 0); 72 72 + RUN_SUCCESS(exception_bad_assert_neg_gfunc, 0); 73 73 + RUN_SUCCESS(exception_bad_assert_pos_gfunc, 0); 74 74 + RUN_SUCCESS(exception_bad_assert_negeq_gfunc, 0); 75 75 + RUN_SUCCESS(exception_bad_assert_poseq_gfunc, 0); 76 76 + RUN_SUCCESS(exception_bad_assert_nz_gfunc_with, 100); 77 77 + RUN_SUCCESS(exception_bad_assert_zero_gfunc_with, 105); 78 78 + RUN_SUCCESS(exception_bad_assert_neg_gfunc_with, 200); 79 79 + RUN_SUCCESS(exception_bad_assert_pos_gfunc_with, 0); 80 80 + RUN_SUCCESS(exception_bad_assert_negeq_gfunc_with, 101); 81 81 + RUN_SUCCESS(exception_bad_assert_poseq_gfunc_with, 99); 82 82 + RUN_SUCCESS(exception_assert_range, 1); 83 83 + RUN_SUCCESS(exception_assert_range_with, 1); 84 84 + RUN_SUCCESS(exception_bad_assert_range, 0); 85 85 + RUN_SUCCESS(exception_bad_assert_range_with, 10); 86 86 + 87 87 + #define RUN_EXT(load_ret, attach_err, expr, msg, after_link) \ 88 88 + { \ 89 89 + LIBBPF_OPTS(bpf_object_open_opts, o, .kernel_log_buf = log_buf, \ 90 90 + .kernel_log_size = sizeof(log_buf), \ 91 91 + .kernel_log_level = 2); \ 92 92 + exceptions_ext__destroy(eskel); \ 93 93 + eskel = exceptions_ext__open_opts(&o); \ 94 94 + struct bpf_program *prog = NULL; \ 95 95 + struct bpf_link *link = NULL; \ 96 96 + if (!ASSERT_OK_PTR(eskel, "exceptions_ext__open")) \ 97 97 + goto done; \ 98 98 + (expr); \ 99 99 + ASSERT_OK_PTR(bpf_program__name(prog), bpf_program__name(prog)); \ 100 100 + if (!ASSERT_EQ(exceptions_ext__load(eskel), load_ret, \ 101 101 + "exceptions_ext__load")) { \ 102 102 + printf("%s\n", log_buf); \ 103 103 + goto done; \ 104 104 + } \ 105 105 + if (load_ret != 0) { \ 106 106 + printf("%s\n", log_buf); \ 107 107 + if (!ASSERT_OK_PTR(strstr(log_buf, msg), "strstr")) \ 108 108 + goto done; \ 109 109 + } \ 110 110 + if (!load_ret && attach_err) { \ 111 111 + if (!ASSERT_ERR_PTR(link = bpf_program__attach(prog), "attach err")) \ 112 112 + goto done; \ 113 113 + } else if (!load_ret) { \ 114 114 + if (!ASSERT_OK_PTR(link = bpf_program__attach(prog), "attach ok")) \ 115 115 + goto done; \ 116 116 + (void)(after_link); \ 117 117 + bpf_link__destroy(link); \ 118 118 + } \ 119 119 + } 120 120 + 121 121 + if (test__start_subtest("non-throwing fentry -> exception_cb")) 122 122 + RUN_EXT(-EINVAL, true, ({ 123 123 + prog = eskel->progs.pfentry; 124 124 + bpf_program__set_autoload(prog, true); 125 125 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 126 126 + bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), 127 127 + "exception_cb_mod"), "set_attach_target")) 128 128 + goto done; 129 129 + }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); 130 130 + 131 131 + if (test__start_subtest("throwing fentry -> exception_cb")) 132 132 + RUN_EXT(-EINVAL, true, ({ 133 133 + prog = eskel->progs.throwing_fentry; 134 134 + bpf_program__set_autoload(prog, true); 135 135 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 136 136 + bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), 137 137 + "exception_cb_mod"), "set_attach_target")) 138 138 + goto done; 139 139 + }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); 140 140 + 141 141 + if (test__start_subtest("non-throwing fexit -> exception_cb")) 142 142 + RUN_EXT(-EINVAL, true, ({ 143 143 + prog = eskel->progs.pfexit; 144 144 + bpf_program__set_autoload(prog, true); 145 145 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 146 146 + bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), 147 147 + "exception_cb_mod"), "set_attach_target")) 148 148 + goto done; 149 149 + }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); 150 150 + 151 151 + if (test__start_subtest("throwing fexit -> exception_cb")) 152 152 + RUN_EXT(-EINVAL, true, ({ 153 153 + prog = eskel->progs.throwing_fexit; 154 154 + bpf_program__set_autoload(prog, true); 155 155 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 156 156 + bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), 157 157 + "exception_cb_mod"), "set_attach_target")) 158 158 + goto done; 159 159 + }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); 160 160 + 161 161 + if (test__start_subtest("throwing extension (with custom cb) -> exception_cb")) 162 162 + RUN_EXT(-EINVAL, true, ({ 163 163 + prog = eskel->progs.throwing_exception_cb_extension; 164 164 + bpf_program__set_autoload(prog, true); 165 165 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 166 166 + bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), 167 167 + "exception_cb_mod"), "set_attach_target")) 168 168 + goto done; 169 169 + }), "Extension programs cannot attach to exception callback", 0); 170 170 + 171 171 + if (test__start_subtest("throwing extension -> global func in exception_cb")) 172 172 + RUN_EXT(0, false, ({ 173 173 + prog = eskel->progs.throwing_exception_cb_extension; 174 174 + bpf_program__set_autoload(prog, true); 175 175 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 176 176 + bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), 177 177 + "exception_cb_mod_global"), "set_attach_target")) 178 178 + goto done; 179 179 + }), "", ({ RUN_SUCCESS(exception_ext_mod_cb_runtime, 131); })); 180 180 + 181 181 + if (test__start_subtest("throwing extension (with custom cb) -> global func in exception_cb")) 182 182 + RUN_EXT(0, false, ({ 183 183 + prog = eskel->progs.throwing_extension; 184 184 + bpf_program__set_autoload(prog, true); 185 185 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 186 186 + bpf_program__fd(skel->progs.exception_ext), 187 187 + "exception_ext_global"), "set_attach_target")) 188 188 + goto done; 189 189 + }), "", ({ RUN_SUCCESS(exception_ext, 128); })); 190 190 + 191 191 + if (test__start_subtest("non-throwing fentry -> non-throwing subprog")) 192 192 + /* non-throwing fentry -> non-throwing subprog : OK */ 193 193 + RUN_EXT(0, false, ({ 194 194 + prog = eskel->progs.pfentry; 195 195 + bpf_program__set_autoload(prog, true); 196 196 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 197 197 + bpf_program__fd(skel->progs.exception_throw_subprog), 198 198 + "subprog"), "set_attach_target")) 199 199 + goto done; 200 200 + }), "", 0); 201 201 + 202 202 + if (test__start_subtest("throwing fentry -> non-throwing subprog")) 203 203 + /* throwing fentry -> non-throwing subprog : OK */ 204 204 + RUN_EXT(0, false, ({ 205 205 + prog = eskel->progs.throwing_fentry; 206 206 + bpf_program__set_autoload(prog, true); 207 207 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 208 208 + bpf_program__fd(skel->progs.exception_throw_subprog), 209 209 + "subprog"), "set_attach_target")) 210 210 + goto done; 211 211 + }), "", 0); 212 212 + 213 213 + if (test__start_subtest("non-throwing fentry -> throwing subprog")) 214 214 + /* non-throwing fentry -> throwing subprog : OK */ 215 215 + RUN_EXT(0, false, ({ 216 216 + prog = eskel->progs.pfentry; 217 217 + bpf_program__set_autoload(prog, true); 218 218 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 219 219 + bpf_program__fd(skel->progs.exception_throw_subprog), 220 220 + "throwing_subprog"), "set_attach_target")) 221 221 + goto done; 222 222 + }), "", 0); 223 223 + 224 224 + if (test__start_subtest("throwing fentry -> throwing subprog")) 225 225 + /* throwing fentry -> throwing subprog : OK */ 226 226 + RUN_EXT(0, false, ({ 227 227 + prog = eskel->progs.throwing_fentry; 228 228 + bpf_program__set_autoload(prog, true); 229 229 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 230 230 + bpf_program__fd(skel->progs.exception_throw_subprog), 231 231 + "throwing_subprog"), "set_attach_target")) 232 232 + goto done; 233 233 + }), "", 0); 234 234 + 235 235 + if (test__start_subtest("non-throwing fexit -> non-throwing subprog")) 236 236 + /* non-throwing fexit -> non-throwing subprog : OK */ 237 237 + RUN_EXT(0, false, ({ 238 238 + prog = eskel->progs.pfexit; 239 239 + bpf_program__set_autoload(prog, true); 240 240 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 241 241 + bpf_program__fd(skel->progs.exception_throw_subprog), 242 242 + "subprog"), "set_attach_target")) 243 243 + goto done; 244 244 + }), "", 0); 245 245 + 246 246 + if (test__start_subtest("throwing fexit -> non-throwing subprog")) 247 247 + /* throwing fexit -> non-throwing subprog : OK */ 248 248 + RUN_EXT(0, false, ({ 249 249 + prog = eskel->progs.throwing_fexit; 250 250 + bpf_program__set_autoload(prog, true); 251 251 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 252 252 + bpf_program__fd(skel->progs.exception_throw_subprog), 253 253 + "subprog"), "set_attach_target")) 254 254 + goto done; 255 255 + }), "", 0); 256 256 + 257 257 + if (test__start_subtest("non-throwing fexit -> throwing subprog")) 258 258 + /* non-throwing fexit -> throwing subprog : OK */ 259 259 + RUN_EXT(0, false, ({ 260 260 + prog = eskel->progs.pfexit; 261 261 + bpf_program__set_autoload(prog, true); 262 262 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 263 263 + bpf_program__fd(skel->progs.exception_throw_subprog), 264 264 + "throwing_subprog"), "set_attach_target")) 265 265 + goto done; 266 266 + }), "", 0); 267 267 + 268 268 + if (test__start_subtest("throwing fexit -> throwing subprog")) 269 269 + /* throwing fexit -> throwing subprog : OK */ 270 270 + RUN_EXT(0, false, ({ 271 271 + prog = eskel->progs.throwing_fexit; 272 272 + bpf_program__set_autoload(prog, true); 273 273 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 274 274 + bpf_program__fd(skel->progs.exception_throw_subprog), 275 275 + "throwing_subprog"), "set_attach_target")) 276 276 + goto done; 277 277 + }), "", 0); 278 278 + 279 279 + /* fmod_ret not allowed for subprog - Check so we remember to handle its 280 280 + * throwing specification compatibility with target when supported. 281 281 + */ 282 282 + if (test__start_subtest("non-throwing fmod_ret -> non-throwing subprog")) 283 283 + RUN_EXT(-EINVAL, true, ({ 284 284 + prog = eskel->progs.pfmod_ret; 285 285 + bpf_program__set_autoload(prog, true); 286 286 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 287 287 + bpf_program__fd(skel->progs.exception_throw_subprog), 288 288 + "subprog"), "set_attach_target")) 289 289 + goto done; 290 290 + }), "can't modify return codes of BPF program", 0); 291 291 + 292 292 + /* fmod_ret not allowed for subprog - Check so we remember to handle its 293 293 + * throwing specification compatibility with target when supported. 294 294 + */ 295 295 + if (test__start_subtest("non-throwing fmod_ret -> non-throwing global subprog")) 296 296 + RUN_EXT(-EINVAL, true, ({ 297 297 + prog = eskel->progs.pfmod_ret; 298 298 + bpf_program__set_autoload(prog, true); 299 299 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 300 300 + bpf_program__fd(skel->progs.exception_throw_subprog), 301 301 + "global_subprog"), "set_attach_target")) 302 302 + goto done; 303 303 + }), "can't modify return codes of BPF program", 0); 304 304 + 305 305 + if (test__start_subtest("non-throwing extension -> non-throwing subprog")) 306 306 + /* non-throwing extension -> non-throwing subprog : BAD (!global) */ 307 307 + RUN_EXT(-EINVAL, true, ({ 308 308 + prog = eskel->progs.extension; 309 309 + bpf_program__set_autoload(prog, true); 310 310 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 311 311 + bpf_program__fd(skel->progs.exception_throw_subprog), 312 312 + "subprog"), "set_attach_target")) 313 313 + goto done; 314 314 + }), "subprog() is not a global function", 0); 315 315 + 316 316 + if (test__start_subtest("non-throwing extension -> throwing subprog")) 317 317 + /* non-throwing extension -> throwing subprog : BAD (!global) */ 318 318 + RUN_EXT(-EINVAL, true, ({ 319 319 + prog = eskel->progs.extension; 320 320 + bpf_program__set_autoload(prog, true); 321 321 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 322 322 + bpf_program__fd(skel->progs.exception_throw_subprog), 323 323 + "throwing_subprog"), "set_attach_target")) 324 324 + goto done; 325 325 + }), "throwing_subprog() is not a global function", 0); 326 326 + 327 327 + if (test__start_subtest("non-throwing extension -> non-throwing subprog")) 328 328 + /* non-throwing extension -> non-throwing global subprog : OK */ 329 329 + RUN_EXT(0, false, ({ 330 330 + prog = eskel->progs.extension; 331 331 + bpf_program__set_autoload(prog, true); 332 332 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 333 333 + bpf_program__fd(skel->progs.exception_throw_subprog), 334 334 + "global_subprog"), "set_attach_target")) 335 335 + goto done; 336 336 + }), "", 0); 337 337 + 338 338 + if (test__start_subtest("non-throwing extension -> throwing global subprog")) 339 339 + /* non-throwing extension -> throwing global subprog : OK */ 340 340 + RUN_EXT(0, false, ({ 341 341 + prog = eskel->progs.extension; 342 342 + bpf_program__set_autoload(prog, true); 343 343 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 344 344 + bpf_program__fd(skel->progs.exception_throw_subprog), 345 345 + "throwing_global_subprog"), "set_attach_target")) 346 346 + goto done; 347 347 + }), "", 0); 348 348 + 349 349 + if (test__start_subtest("throwing extension -> throwing global subprog")) 350 350 + /* throwing extension -> throwing global subprog : OK */ 351 351 + RUN_EXT(0, false, ({ 352 352 + prog = eskel->progs.throwing_extension; 353 353 + bpf_program__set_autoload(prog, true); 354 354 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 355 355 + bpf_program__fd(skel->progs.exception_throw_subprog), 356 356 + "throwing_global_subprog"), "set_attach_target")) 357 357 + goto done; 358 358 + }), "", 0); 359 359 + 360 360 + if (test__start_subtest("throwing extension -> non-throwing global subprog")) 361 361 + /* throwing extension -> non-throwing global subprog : OK */ 362 362 + RUN_EXT(0, false, ({ 363 363 + prog = eskel->progs.throwing_extension; 364 364 + bpf_program__set_autoload(prog, true); 365 365 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 366 366 + bpf_program__fd(skel->progs.exception_throw_subprog), 367 367 + "global_subprog"), "set_attach_target")) 368 368 + goto done; 369 369 + }), "", 0); 370 370 + 371 371 + if (test__start_subtest("non-throwing extension -> main subprog")) 372 372 + /* non-throwing extension -> main subprog : OK */ 373 373 + RUN_EXT(0, false, ({ 374 374 + prog = eskel->progs.extension; 375 375 + bpf_program__set_autoload(prog, true); 376 376 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 377 377 + bpf_program__fd(skel->progs.exception_throw_subprog), 378 378 + "exception_throw_subprog"), "set_attach_target")) 379 379 + goto done; 380 380 + }), "", 0); 381 381 + 382 382 + if (test__start_subtest("throwing extension -> main subprog")) 383 383 + /* throwing extension -> main subprog : OK */ 384 384 + RUN_EXT(0, false, ({ 385 385 + prog = eskel->progs.throwing_extension; 386 386 + bpf_program__set_autoload(prog, true); 387 387 + if (!ASSERT_OK(bpf_program__set_attach_target(prog, 388 388 + bpf_program__fd(skel->progs.exception_throw_subprog), 389 389 + "exception_throw_subprog"), "set_attach_target")) 390 390 + goto done; 391 391 + }), "", 0); 392 392 + 393 393 + done: 394 394 + exceptions_ext__destroy(eskel); 395 395 + exceptions__destroy(skel); 396 396 + } 397 397 + 398 398 + static void test_exceptions_assertions(void) 399 399 + { 400 400 + RUN_TESTS(exceptions_assert); 401 401 + } 402 402 + 403 403 + void test_exceptions(void) 404 404 + { 405 405 + test_exceptions_success(); 406 406 + test_exceptions_failure(); 407 407 + test_exceptions_assertions(); 408 408 + }

+1 -1

tools/testing/selftests/bpf/prog_tests/fill_link_info.c

reviewed

··· 308 308 return; 309 309 310 310 /* load kallsyms to compare the addr */ 311 311 - if (!ASSERT_OK(load_kallsyms_refresh(), "load_kallsyms_refresh")) 311 311 + if (!ASSERT_OK(load_kallsyms(), "load_kallsyms")) 312 312 goto cleanup; 313 313 314 314 kprobe_addr = ksym_get_addr(KPROBE_FUNC);

+13 -7

tools/testing/selftests/bpf/prog_tests/kprobe_multi_testmod_test.c

reviewed

··· 4 4 #include "trace_helpers.h" 5 5 #include "bpf/libbpf_internal.h" 6 6 7 7 + static struct ksyms *ksyms; 8 8 + 7 9 static void kprobe_multi_testmod_check(struct kprobe_multi *skel) 8 10 { 9 11 ASSERT_EQ(skel->bss->kprobe_testmod_test1_result, 1, "kprobe_test1_result"); ··· 52 50 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); 53 51 unsigned long long addrs[3]; 54 52 55 55 - addrs[0] = ksym_get_addr("bpf_testmod_fentry_test1"); 56 56 - ASSERT_NEQ(addrs[0], 0, "ksym_get_addr"); 57 57 - addrs[1] = ksym_get_addr("bpf_testmod_fentry_test2"); 58 58 - ASSERT_NEQ(addrs[1], 0, "ksym_get_addr"); 59 59 - addrs[2] = ksym_get_addr("bpf_testmod_fentry_test3"); 60 60 - ASSERT_NEQ(addrs[2], 0, "ksym_get_addr"); 53 53 + addrs[0] = ksym_get_addr_local(ksyms, "bpf_testmod_fentry_test1"); 54 54 + ASSERT_NEQ(addrs[0], 0, "ksym_get_addr_local"); 55 55 + addrs[1] = ksym_get_addr_local(ksyms, "bpf_testmod_fentry_test2"); 56 56 + ASSERT_NEQ(addrs[1], 0, "ksym_get_addr_local"); 57 57 + addrs[2] = ksym_get_addr_local(ksyms, "bpf_testmod_fentry_test3"); 58 58 + ASSERT_NEQ(addrs[2], 0, "ksym_get_addr_local"); 61 59 62 60 opts.addrs = (const unsigned long *) addrs; 63 61 opts.cnt = ARRAY_SIZE(addrs); ··· 81 79 82 80 void serial_test_kprobe_multi_testmod_test(void) 83 81 { 84 84 - if (!ASSERT_OK(load_kallsyms_refresh(), "load_kallsyms_refresh")) 82 82 + ksyms = load_kallsyms_local(); 83 83 + if (!ASSERT_OK_PTR(ksyms, "load_kallsyms_local")) 85 84 return; 86 85 87 86 if (test__start_subtest("testmod_attach_api_syms")) 88 87 test_testmod_attach_api_syms(); 88 88 + 89 89 if (test__start_subtest("testmod_attach_api_addrs")) 90 90 test_testmod_attach_api_addrs(); 91 91 + 92 92 + free_kallsyms_local(ksyms); 91 93 }

+5 -1

tools/testing/selftests/bpf/prog_tests/libbpf_str.c

reviewed

··· 142 142 /* Special case for map_type_name BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED 143 143 * where it and BPF_MAP_TYPE_CGROUP_STORAGE have the same enum value 144 144 * (map_type). For this enum value, libbpf_bpf_map_type_str() picks 145 145 - * BPF_MAP_TYPE_CGROUP_STORAGE. 145 145 + * BPF_MAP_TYPE_CGROUP_STORAGE. The same for 146 146 + * BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED and 147 147 + * BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE. 146 148 */ 147 149 if (strcmp(map_type_name, "BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED") == 0) 150 150 + continue; 151 151 + if (strcmp(map_type_name, "BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED") == 0) 148 152 continue; 149 153 150 154 ASSERT_STREQ(buf, map_type_name, "exp_str_value");

+2 -2

tools/testing/selftests/bpf/prog_tests/linked_list.c

reviewed

··· 65 65 { "map_compat_raw_tp", "tracing progs cannot use bpf_{list_head,rb_root} yet" }, 66 66 { "map_compat_raw_tp_w", "tracing progs cannot use bpf_{list_head,rb_root} yet" }, 67 67 { "obj_type_id_oor", "local type ID argument must be in range [0, U32_MAX]" }, 68 68 - { "obj_new_no_composite", "bpf_obj_new type ID argument must be of a struct" }, 69 69 - { "obj_new_no_struct", "bpf_obj_new type ID argument must be of a struct" }, 68 68 + { "obj_new_no_composite", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" }, 69 69 + { "obj_new_no_struct", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" }, 70 70 { "obj_drop_non_zero_off", "R1 must have zero offset when passed to release func" }, 71 71 { "new_null_ret", "R0 invalid mem access 'ptr_or_null_'" }, 72 72 { "obj_new_acq", "Unreleased reference id=" },

+5

tools/testing/selftests/bpf/prog_tests/module_fentry_shadow.c

reviewed

··· 61 61 int link_fd[2] = {}; 62 62 __s32 btf_id[2] = {}; 63 63 64 64 + if (!env.has_testmod) { 65 65 + test__skip(); 66 66 + return; 67 67 + } 68 68 + 64 69 LIBBPF_OPTS(bpf_prog_load_opts, load_opts, 65 70 .expected_attach_type = BPF_TRACE_FENTRY, 66 71 );

+125

tools/testing/selftests/bpf/prog_tests/percpu_alloc.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + #include <test_progs.h> 3 3 + #include "percpu_alloc_array.skel.h" 4 4 + #include "percpu_alloc_cgrp_local_storage.skel.h" 5 5 + #include "percpu_alloc_fail.skel.h" 6 6 + 7 7 + static void test_array(void) 8 8 + { 9 9 + struct percpu_alloc_array *skel; 10 10 + int err, prog_fd; 11 11 + LIBBPF_OPTS(bpf_test_run_opts, topts); 12 12 + 13 13 + skel = percpu_alloc_array__open(); 14 14 + if (!ASSERT_OK_PTR(skel, "percpu_alloc_array__open")) 15 15 + return; 16 16 + 17 17 + bpf_program__set_autoload(skel->progs.test_array_map_1, true); 18 18 + bpf_program__set_autoload(skel->progs.test_array_map_2, true); 19 19 + bpf_program__set_autoload(skel->progs.test_array_map_3, true); 20 20 + bpf_program__set_autoload(skel->progs.test_array_map_4, true); 21 21 + 22 22 + skel->rodata->nr_cpus = libbpf_num_possible_cpus(); 23 23 + 24 24 + err = percpu_alloc_array__load(skel); 25 25 + if (!ASSERT_OK(err, "percpu_alloc_array__load")) 26 26 + goto out; 27 27 + 28 28 + err = percpu_alloc_array__attach(skel); 29 29 + if (!ASSERT_OK(err, "percpu_alloc_array__attach")) 30 30 + goto out; 31 31 + 32 32 + prog_fd = bpf_program__fd(skel->progs.test_array_map_1); 33 33 + err = bpf_prog_test_run_opts(prog_fd, &topts); 34 34 + ASSERT_OK(err, "test_run array_map 1-4"); 35 35 + ASSERT_EQ(topts.retval, 0, "test_run array_map 1-4"); 36 36 + ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d"); 37 37 + ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c"); 38 38 + out: 39 39 + percpu_alloc_array__destroy(skel); 40 40 + } 41 41 + 42 42 + static void test_array_sleepable(void) 43 43 + { 44 44 + struct percpu_alloc_array *skel; 45 45 + int err, prog_fd; 46 46 + LIBBPF_OPTS(bpf_test_run_opts, topts); 47 47 + 48 48 + skel = percpu_alloc_array__open(); 49 49 + if (!ASSERT_OK_PTR(skel, "percpu_alloc__open")) 50 50 + return; 51 51 + 52 52 + bpf_program__set_autoload(skel->progs.test_array_map_10, true); 53 53 + 54 54 + skel->rodata->nr_cpus = libbpf_num_possible_cpus(); 55 55 + 56 56 + err = percpu_alloc_array__load(skel); 57 57 + if (!ASSERT_OK(err, "percpu_alloc_array__load")) 58 58 + goto out; 59 59 + 60 60 + err = percpu_alloc_array__attach(skel); 61 61 + if (!ASSERT_OK(err, "percpu_alloc_array__attach")) 62 62 + goto out; 63 63 + 64 64 + prog_fd = bpf_program__fd(skel->progs.test_array_map_10); 65 65 + err = bpf_prog_test_run_opts(prog_fd, &topts); 66 66 + ASSERT_OK(err, "test_run array_map_10"); 67 67 + ASSERT_EQ(topts.retval, 0, "test_run array_map_10"); 68 68 + ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d"); 69 69 + ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c"); 70 70 + out: 71 71 + percpu_alloc_array__destroy(skel); 72 72 + } 73 73 + 74 74 + static void test_cgrp_local_storage(void) 75 75 + { 76 76 + struct percpu_alloc_cgrp_local_storage *skel; 77 77 + int err, cgroup_fd, prog_fd; 78 78 + LIBBPF_OPTS(bpf_test_run_opts, topts); 79 79 + 80 80 + cgroup_fd = test__join_cgroup("/percpu_alloc"); 81 81 + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup /percpu_alloc")) 82 82 + return; 83 83 + 84 84 + skel = percpu_alloc_cgrp_local_storage__open(); 85 85 + if (!ASSERT_OK_PTR(skel, "percpu_alloc_cgrp_local_storage__open")) 86 86 + goto close_fd; 87 87 + 88 88 + skel->rodata->nr_cpus = libbpf_num_possible_cpus(); 89 89 + 90 90 + err = percpu_alloc_cgrp_local_storage__load(skel); 91 91 + if (!ASSERT_OK(err, "percpu_alloc_cgrp_local_storage__load")) 92 92 + goto destroy_skel; 93 93 + 94 94 + err = percpu_alloc_cgrp_local_storage__attach(skel); 95 95 + if (!ASSERT_OK(err, "percpu_alloc_cgrp_local_storage__attach")) 96 96 + goto destroy_skel; 97 97 + 98 98 + prog_fd = bpf_program__fd(skel->progs.test_cgrp_local_storage_1); 99 99 + err = bpf_prog_test_run_opts(prog_fd, &topts); 100 100 + ASSERT_OK(err, "test_run cgrp_local_storage 1-3"); 101 101 + ASSERT_EQ(topts.retval, 0, "test_run cgrp_local_storage 1-3"); 102 102 + ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d"); 103 103 + ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c"); 104 104 + 105 105 + destroy_skel: 106 106 + percpu_alloc_cgrp_local_storage__destroy(skel); 107 107 + close_fd: 108 108 + close(cgroup_fd); 109 109 + } 110 110 + 111 111 + static void test_failure(void) { 112 112 + RUN_TESTS(percpu_alloc_fail); 113 113 + } 114 114 + 115 115 + void test_percpu_alloc(void) 116 116 + { 117 117 + if (test__start_subtest("array")) 118 118 + test_array(); 119 119 + if (test__start_subtest("array_sleepable")) 120 120 + test_array_sleepable(); 121 121 + if (test__start_subtest("cgrp_local_storage")) 122 122 + test_cgrp_local_storage(); 123 123 + if (test__start_subtest("failure_tests")) 124 124 + test_failure(); 125 125 + }

+89

tools/testing/selftests/bpf/prog_tests/preempted_bpf_ma_op.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (C) 2023. Huawei Technologies Co., Ltd */ 3 3 + #define _GNU_SOURCE 4 4 + #include <sched.h> 5 5 + #include <pthread.h> 6 6 + #include <stdbool.h> 7 7 + #include <test_progs.h> 8 8 + 9 9 + #include "preempted_bpf_ma_op.skel.h" 10 10 + 11 11 + #define ALLOC_THREAD_NR 4 12 12 + #define ALLOC_LOOP_NR 512 13 13 + 14 14 + struct alloc_ctx { 15 15 + /* output */ 16 16 + int run_err; 17 17 + /* input */ 18 18 + int fd; 19 19 + bool *nomem_err; 20 20 + }; 21 21 + 22 22 + static void *run_alloc_prog(void *data) 23 23 + { 24 24 + struct alloc_ctx *ctx = data; 25 25 + cpu_set_t cpu_set; 26 26 + int i; 27 27 + 28 28 + CPU_ZERO(&cpu_set); 29 29 + CPU_SET(0, &cpu_set); 30 30 + pthread_setaffinity_np(pthread_self(), sizeof(cpu_set), &cpu_set); 31 31 + 32 32 + for (i = 0; i < ALLOC_LOOP_NR && !*ctx->nomem_err; i++) { 33 33 + LIBBPF_OPTS(bpf_test_run_opts, topts); 34 34 + int err; 35 35 + 36 36 + err = bpf_prog_test_run_opts(ctx->fd, &topts); 37 37 + ctx->run_err |= err | topts.retval; 38 38 + } 39 39 + 40 40 + return NULL; 41 41 + } 42 42 + 43 43 + void test_preempted_bpf_ma_op(void) 44 44 + { 45 45 + struct alloc_ctx ctx[ALLOC_THREAD_NR]; 46 46 + struct preempted_bpf_ma_op *skel; 47 47 + pthread_t tid[ALLOC_THREAD_NR]; 48 48 + int i, err; 49 49 + 50 50 + skel = preempted_bpf_ma_op__open_and_load(); 51 51 + if (!ASSERT_OK_PTR(skel, "open_and_load")) 52 52 + return; 53 53 + 54 54 + err = preempted_bpf_ma_op__attach(skel); 55 55 + if (!ASSERT_OK(err, "attach")) 56 56 + goto out; 57 57 + 58 58 + for (i = 0; i < ARRAY_SIZE(ctx); i++) { 59 59 + struct bpf_program *prog; 60 60 + char name[8]; 61 61 + 62 62 + snprintf(name, sizeof(name), "test%d", i); 63 63 + prog = bpf_object__find_program_by_name(skel->obj, name); 64 64 + if (!ASSERT_OK_PTR(prog, "no test prog")) 65 65 + goto out; 66 66 + 67 67 + ctx[i].run_err = 0; 68 68 + ctx[i].fd = bpf_program__fd(prog); 69 69 + ctx[i].nomem_err = &skel->bss->nomem_err; 70 70 + } 71 71 + 72 72 + memset(tid, 0, sizeof(tid)); 73 73 + for (i = 0; i < ARRAY_SIZE(tid); i++) { 74 74 + err = pthread_create(&tid[i], NULL, run_alloc_prog, &ctx[i]); 75 75 + if (!ASSERT_OK(err, "pthread_create")) 76 76 + break; 77 77 + } 78 78 + 79 79 + for (i = 0; i < ARRAY_SIZE(tid); i++) { 80 80 + if (!tid[i]) 81 81 + break; 82 82 + pthread_join(tid[i], NULL); 83 83 + ASSERT_EQ(ctx[i].run_err, 0, "run prog err"); 84 84 + } 85 85 + 86 86 + ASSERT_FALSE(skel->bss->nomem_err, "ENOMEM"); 87 87 + out: 88 88 + preempted_bpf_ma_op__destroy(skel); 89 89 + }

+249 -20

tools/testing/selftests/bpf/prog_tests/tailcalls.c

reviewed

··· 218 218 bpf_object__close(obj); 219 219 } 220 220 221 221 - static void test_tailcall_count(const char *which) 221 221 + static void test_tailcall_count(const char *which, bool test_fentry, 222 222 + bool test_fexit) 222 223 { 224 224 + struct bpf_object *obj = NULL, *fentry_obj = NULL, *fexit_obj = NULL; 225 225 + struct bpf_link *fentry_link = NULL, *fexit_link = NULL; 223 226 int err, map_fd, prog_fd, main_fd, data_fd, i, val; 224 227 struct bpf_map *prog_array, *data_map; 225 228 struct bpf_program *prog; 226 226 - struct bpf_object *obj; 227 229 char buff[128] = {}; 228 230 LIBBPF_OPTS(bpf_test_run_opts, topts, 229 231 .data_in = buff, ··· 267 265 if (CHECK_FAIL(err)) 268 266 goto out; 269 267 268 268 + if (test_fentry) { 269 269 + fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o", 270 270 + NULL); 271 271 + if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file")) 272 272 + goto out; 273 273 + 274 274 + prog = bpf_object__find_program_by_name(fentry_obj, "fentry"); 275 275 + if (!ASSERT_OK_PTR(prog, "find fentry prog")) 276 276 + goto out; 277 277 + 278 278 + err = bpf_program__set_attach_target(prog, prog_fd, 279 279 + "subprog_tail"); 280 280 + if (!ASSERT_OK(err, "set_attach_target subprog_tail")) 281 281 + goto out; 282 282 + 283 283 + err = bpf_object__load(fentry_obj); 284 284 + if (!ASSERT_OK(err, "load fentry_obj")) 285 285 + goto out; 286 286 + 287 287 + fentry_link = bpf_program__attach_trace(prog); 288 288 + if (!ASSERT_OK_PTR(fentry_link, "attach_trace")) 289 289 + goto out; 290 290 + } 291 291 + 292 292 + if (test_fexit) { 293 293 + fexit_obj = bpf_object__open_file("tailcall_bpf2bpf_fexit.bpf.o", 294 294 + NULL); 295 295 + if (!ASSERT_OK_PTR(fexit_obj, "open fexit_obj file")) 296 296 + goto out; 297 297 + 298 298 + prog = bpf_object__find_program_by_name(fexit_obj, "fexit"); 299 299 + if (!ASSERT_OK_PTR(prog, "find fexit prog")) 300 300 + goto out; 301 301 + 302 302 + err = bpf_program__set_attach_target(prog, prog_fd, 303 303 + "subprog_tail"); 304 304 + if (!ASSERT_OK(err, "set_attach_target subprog_tail")) 305 305 + goto out; 306 306 + 307 307 + err = bpf_object__load(fexit_obj); 308 308 + if (!ASSERT_OK(err, "load fexit_obj")) 309 309 + goto out; 310 310 + 311 311 + fexit_link = bpf_program__attach_trace(prog); 312 312 + if (!ASSERT_OK_PTR(fexit_link, "attach_trace")) 313 313 + goto out; 314 314 + } 315 315 + 270 316 err = bpf_prog_test_run_opts(main_fd, &topts); 271 317 ASSERT_OK(err, "tailcall"); 272 318 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 273 319 274 320 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 275 321 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 276 276 - return; 322 322 + goto out; 277 323 278 324 data_fd = bpf_map__fd(data_map); 279 279 - if (CHECK_FAIL(map_fd < 0)) 280 280 - return; 325 325 + if (CHECK_FAIL(data_fd < 0)) 326 326 + goto out; 281 327 282 328 i = 0; 283 329 err = bpf_map_lookup_elem(data_fd, &i, &val); 284 330 ASSERT_OK(err, "tailcall count"); 285 331 ASSERT_EQ(val, 33, "tailcall count"); 332 332 + 333 333 + if (test_fentry) { 334 334 + data_map = bpf_object__find_map_by_name(fentry_obj, ".bss"); 335 335 + if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 336 336 + "find tailcall_bpf2bpf_fentry.bss map")) 337 337 + goto out; 338 338 + 339 339 + data_fd = bpf_map__fd(data_map); 340 340 + if (!ASSERT_FALSE(data_fd < 0, 341 341 + "find tailcall_bpf2bpf_fentry.bss map fd")) 342 342 + goto out; 343 343 + 344 344 + i = 0; 345 345 + err = bpf_map_lookup_elem(data_fd, &i, &val); 346 346 + ASSERT_OK(err, "fentry count"); 347 347 + ASSERT_EQ(val, 33, "fentry count"); 348 348 + } 349 349 + 350 350 + if (test_fexit) { 351 351 + data_map = bpf_object__find_map_by_name(fexit_obj, ".bss"); 352 352 + if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 353 353 + "find tailcall_bpf2bpf_fexit.bss map")) 354 354 + goto out; 355 355 + 356 356 + data_fd = bpf_map__fd(data_map); 357 357 + if (!ASSERT_FALSE(data_fd < 0, 358 358 + "find tailcall_bpf2bpf_fexit.bss map fd")) 359 359 + goto out; 360 360 + 361 361 + i = 0; 362 362 + err = bpf_map_lookup_elem(data_fd, &i, &val); 363 363 + ASSERT_OK(err, "fexit count"); 364 364 + ASSERT_EQ(val, 33, "fexit count"); 365 365 + } 286 366 287 367 i = 0; 288 368 err = bpf_map_delete_elem(map_fd, &i); ··· 375 291 ASSERT_OK(err, "tailcall"); 376 292 ASSERT_OK(topts.retval, "tailcall retval"); 377 293 out: 294 294 + bpf_link__destroy(fentry_link); 295 295 + bpf_link__destroy(fexit_link); 296 296 + bpf_object__close(fentry_obj); 297 297 + bpf_object__close(fexit_obj); 378 298 bpf_object__close(obj); 379 299 } 380 300 ··· 387 299 */ 388 300 static void test_tailcall_3(void) 389 301 { 390 390 - test_tailcall_count("tailcall3.bpf.o"); 302 302 + test_tailcall_count("tailcall3.bpf.o", false, false); 391 303 } 392 304 393 305 /* test_tailcall_6 checks that the count value of the tail call limit ··· 395 307 */ 396 308 static void test_tailcall_6(void) 397 309 { 398 398 - test_tailcall_count("tailcall6.bpf.o"); 310 310 + test_tailcall_count("tailcall6.bpf.o", false, false); 399 311 } 400 312 401 313 /* test_tailcall_4 checks that the kernel properly selects indirect jump ··· 440 352 441 353 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 442 354 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 443 443 - return; 355 355 + goto out; 444 356 445 357 data_fd = bpf_map__fd(data_map); 446 446 - if (CHECK_FAIL(map_fd < 0)) 447 447 - return; 358 358 + if (CHECK_FAIL(data_fd < 0)) 359 359 + goto out; 448 360 449 361 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 450 362 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); ··· 530 442 531 443 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 532 444 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 533 533 - return; 445 445 + goto out; 534 446 535 447 data_fd = bpf_map__fd(data_map); 536 536 - if (CHECK_FAIL(map_fd < 0)) 537 537 - return; 448 448 + if (CHECK_FAIL(data_fd < 0)) 449 449 + goto out; 538 450 539 451 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 540 452 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); ··· 719 631 720 632 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 721 633 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 722 722 - return; 634 634 + goto out; 723 635 724 636 data_fd = bpf_map__fd(data_map); 725 725 - if (CHECK_FAIL(map_fd < 0)) 726 726 - return; 637 637 + if (CHECK_FAIL(data_fd < 0)) 638 638 + goto out; 727 639 728 640 i = 0; 729 641 err = bpf_map_lookup_elem(data_fd, &i, &val); ··· 893 805 894 806 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 895 807 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 896 896 - return; 808 808 + goto out; 897 809 898 810 data_fd = bpf_map__fd(data_map); 899 899 - if (CHECK_FAIL(map_fd < 0)) 900 900 - return; 811 811 + if (CHECK_FAIL(data_fd < 0)) 812 812 + goto out; 901 813 902 814 i = 0; 903 815 val.noise = noise; ··· 960 872 ASSERT_EQ(topts.retval, 0, "tailcall retval"); 961 873 962 874 data_fd = bpf_map__fd(obj->maps.bss); 963 963 - if (!ASSERT_GE(map_fd, 0, "bss map fd")) 875 875 + if (!ASSERT_GE(data_fd, 0, "bss map fd")) 964 876 goto out; 965 877 966 878 i = 0; ··· 970 882 971 883 out: 972 884 tailcall_bpf2bpf6__destroy(obj); 885 885 + } 886 886 + 887 887 + /* test_tailcall_bpf2bpf_fentry checks that the count value of the tail call 888 888 + * limit enforcement matches with expectations when tailcall is preceded with 889 889 + * bpf2bpf call, and the bpf2bpf call is traced by fentry. 890 890 + */ 891 891 + static void test_tailcall_bpf2bpf_fentry(void) 892 892 + { 893 893 + test_tailcall_count("tailcall_bpf2bpf2.bpf.o", true, false); 894 894 + } 895 895 + 896 896 + /* test_tailcall_bpf2bpf_fexit checks that the count value of the tail call 897 897 + * limit enforcement matches with expectations when tailcall is preceded with 898 898 + * bpf2bpf call, and the bpf2bpf call is traced by fexit. 899 899 + */ 900 900 + static void test_tailcall_bpf2bpf_fexit(void) 901 901 + { 902 902 + test_tailcall_count("tailcall_bpf2bpf2.bpf.o", false, true); 903 903 + } 904 904 + 905 905 + /* test_tailcall_bpf2bpf_fentry_fexit checks that the count value of the tail 906 906 + * call limit enforcement matches with expectations when tailcall is preceded 907 907 + * with bpf2bpf call, and the bpf2bpf call is traced by both fentry and fexit. 908 908 + */ 909 909 + static void test_tailcall_bpf2bpf_fentry_fexit(void) 910 910 + { 911 911 + test_tailcall_count("tailcall_bpf2bpf2.bpf.o", true, true); 912 912 + } 913 913 + 914 914 + /* test_tailcall_bpf2bpf_fentry_entry checks that the count value of the tail 915 915 + * call limit enforcement matches with expectations when tailcall is preceded 916 916 + * with bpf2bpf call, and the bpf2bpf caller is traced by fentry. 917 917 + */ 918 918 + static void test_tailcall_bpf2bpf_fentry_entry(void) 919 919 + { 920 920 + struct bpf_object *tgt_obj = NULL, *fentry_obj = NULL; 921 921 + int err, map_fd, prog_fd, data_fd, i, val; 922 922 + struct bpf_map *prog_array, *data_map; 923 923 + struct bpf_link *fentry_link = NULL; 924 924 + struct bpf_program *prog; 925 925 + char buff[128] = {}; 926 926 + 927 927 + LIBBPF_OPTS(bpf_test_run_opts, topts, 928 928 + .data_in = buff, 929 929 + .data_size_in = sizeof(buff), 930 930 + .repeat = 1, 931 931 + ); 932 932 + 933 933 + err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o", 934 934 + BPF_PROG_TYPE_SCHED_CLS, 935 935 + &tgt_obj, &prog_fd); 936 936 + if (!ASSERT_OK(err, "load tgt_obj")) 937 937 + return; 938 938 + 939 939 + prog_array = bpf_object__find_map_by_name(tgt_obj, "jmp_table"); 940 940 + if (!ASSERT_OK_PTR(prog_array, "find jmp_table map")) 941 941 + goto out; 942 942 + 943 943 + map_fd = bpf_map__fd(prog_array); 944 944 + if (!ASSERT_FALSE(map_fd < 0, "find jmp_table map fd")) 945 945 + goto out; 946 946 + 947 947 + prog = bpf_object__find_program_by_name(tgt_obj, "classifier_0"); 948 948 + if (!ASSERT_OK_PTR(prog, "find classifier_0 prog")) 949 949 + goto out; 950 950 + 951 951 + prog_fd = bpf_program__fd(prog); 952 952 + if (!ASSERT_FALSE(prog_fd < 0, "find classifier_0 prog fd")) 953 953 + goto out; 954 954 + 955 955 + i = 0; 956 956 + err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 957 957 + if (!ASSERT_OK(err, "update jmp_table")) 958 958 + goto out; 959 959 + 960 960 + fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o", 961 961 + NULL); 962 962 + if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file")) 963 963 + goto out; 964 964 + 965 965 + prog = bpf_object__find_program_by_name(fentry_obj, "fentry"); 966 966 + if (!ASSERT_OK_PTR(prog, "find fentry prog")) 967 967 + goto out; 968 968 + 969 969 + err = bpf_program__set_attach_target(prog, prog_fd, "classifier_0"); 970 970 + if (!ASSERT_OK(err, "set_attach_target classifier_0")) 971 971 + goto out; 972 972 + 973 973 + err = bpf_object__load(fentry_obj); 974 974 + if (!ASSERT_OK(err, "load fentry_obj")) 975 975 + goto out; 976 976 + 977 977 + fentry_link = bpf_program__attach_trace(prog); 978 978 + if (!ASSERT_OK_PTR(fentry_link, "attach_trace")) 979 979 + goto out; 980 980 + 981 981 + err = bpf_prog_test_run_opts(prog_fd, &topts); 982 982 + ASSERT_OK(err, "tailcall"); 983 983 + ASSERT_EQ(topts.retval, 1, "tailcall retval"); 984 984 + 985 985 + data_map = bpf_object__find_map_by_name(tgt_obj, "tailcall.bss"); 986 986 + if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 987 987 + "find tailcall.bss map")) 988 988 + goto out; 989 989 + 990 990 + data_fd = bpf_map__fd(data_map); 991 991 + if (!ASSERT_FALSE(data_fd < 0, "find tailcall.bss map fd")) 992 992 + goto out; 993 993 + 994 994 + i = 0; 995 995 + err = bpf_map_lookup_elem(data_fd, &i, &val); 996 996 + ASSERT_OK(err, "tailcall count"); 997 997 + ASSERT_EQ(val, 34, "tailcall count"); 998 998 + 999 999 + data_map = bpf_object__find_map_by_name(fentry_obj, ".bss"); 1000 1000 + if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 1001 1001 + "find tailcall_bpf2bpf_fentry.bss map")) 1002 1002 + goto out; 1003 1003 + 1004 1004 + data_fd = bpf_map__fd(data_map); 1005 1005 + if (!ASSERT_FALSE(data_fd < 0, 1006 1006 + "find tailcall_bpf2bpf_fentry.bss map fd")) 1007 1007 + goto out; 1008 1008 + 1009 1009 + i = 0; 1010 1010 + err = bpf_map_lookup_elem(data_fd, &i, &val); 1011 1011 + ASSERT_OK(err, "fentry count"); 1012 1012 + ASSERT_EQ(val, 1, "fentry count"); 1013 1013 + 1014 1014 + out: 1015 1015 + bpf_link__destroy(fentry_link); 1016 1016 + bpf_object__close(fentry_obj); 1017 1017 + bpf_object__close(tgt_obj); 973 1018 } 974 1019 975 1020 void test_tailcalls(void) ··· 1131 910 test_tailcall_bpf2bpf_4(true); 1132 911 if (test__start_subtest("tailcall_bpf2bpf_6")) 1133 912 test_tailcall_bpf2bpf_6(); 913 913 + if (test__start_subtest("tailcall_bpf2bpf_fentry")) 914 914 + test_tailcall_bpf2bpf_fentry(); 915 915 + if (test__start_subtest("tailcall_bpf2bpf_fexit")) 916 916 + test_tailcall_bpf2bpf_fexit(); 917 917 + if (test__start_subtest("tailcall_bpf2bpf_fentry_fexit")) 918 918 + test_tailcall_bpf2bpf_fentry_fexit(); 919 919 + if (test__start_subtest("tailcall_bpf2bpf_fentry_entry")) 920 920 + test_tailcall_bpf2bpf_fentry_entry(); 1134 921 }

+368

tools/testing/selftests/bpf/progs/exceptions.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + #include <vmlinux.h> 3 3 + #include <bpf/bpf_tracing.h> 4 4 + #include <bpf/bpf_helpers.h> 5 5 + #include <bpf/bpf_core_read.h> 6 6 + #include <bpf/bpf_endian.h> 7 7 + #include "bpf_misc.h" 8 8 + #include "bpf_experimental.h" 9 9 + 10 10 + #ifndef ETH_P_IP 11 11 + #define ETH_P_IP 0x0800 12 12 + #endif 13 13 + 14 14 + struct { 15 15 + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); 16 16 + __uint(max_entries, 4); 17 17 + __uint(key_size, sizeof(__u32)); 18 18 + __uint(value_size, sizeof(__u32)); 19 19 + } jmp_table SEC(".maps"); 20 20 + 21 21 + static __noinline int static_func(u64 i) 22 22 + { 23 23 + bpf_throw(32); 24 24 + return i; 25 25 + } 26 26 + 27 27 + __noinline int global2static_simple(u64 i) 28 28 + { 29 29 + static_func(i + 2); 30 30 + return i - 1; 31 31 + } 32 32 + 33 33 + __noinline int global2static(u64 i) 34 34 + { 35 35 + if (i == ETH_P_IP) 36 36 + bpf_throw(16); 37 37 + return static_func(i); 38 38 + } 39 39 + 40 40 + static __noinline int static2global(u64 i) 41 41 + { 42 42 + return global2static(i) + i; 43 43 + } 44 44 + 45 45 + SEC("tc") 46 46 + int exception_throw_always_1(struct __sk_buff *ctx) 47 47 + { 48 48 + bpf_throw(64); 49 49 + return 0; 50 50 + } 51 51 + 52 52 + /* In this case, the global func will never be seen executing after call to 53 53 + * static subprog, hence verifier will DCE the remaining instructions. Ensure we 54 54 + * are resilient to that. 55 55 + */ 56 56 + SEC("tc") 57 57 + int exception_throw_always_2(struct __sk_buff *ctx) 58 58 + { 59 59 + return global2static_simple(ctx->protocol); 60 60 + } 61 61 + 62 62 + SEC("tc") 63 63 + int exception_throw_unwind_1(struct __sk_buff *ctx) 64 64 + { 65 65 + return static2global(bpf_ntohs(ctx->protocol)); 66 66 + } 67 67 + 68 68 + SEC("tc") 69 69 + int exception_throw_unwind_2(struct __sk_buff *ctx) 70 70 + { 71 71 + return static2global(bpf_ntohs(ctx->protocol) - 1); 72 72 + } 73 73 + 74 74 + SEC("tc") 75 75 + int exception_throw_default(struct __sk_buff *ctx) 76 76 + { 77 77 + bpf_throw(0); 78 78 + return 1; 79 79 + } 80 80 + 81 81 + SEC("tc") 82 82 + int exception_throw_default_value(struct __sk_buff *ctx) 83 83 + { 84 84 + bpf_throw(5); 85 85 + return 1; 86 86 + } 87 87 + 88 88 + SEC("tc") 89 89 + int exception_tail_call_target(struct __sk_buff *ctx) 90 90 + { 91 91 + bpf_throw(16); 92 92 + return 0; 93 93 + } 94 94 + 95 95 + static __noinline 96 96 + int exception_tail_call_subprog(struct __sk_buff *ctx) 97 97 + { 98 98 + volatile int ret = 10; 99 99 + 100 100 + bpf_tail_call_static(ctx, &jmp_table, 0); 101 101 + return ret; 102 102 + } 103 103 + 104 104 + SEC("tc") 105 105 + int exception_tail_call(struct __sk_buff *ctx) { 106 106 + volatile int ret = 0; 107 107 + 108 108 + ret = exception_tail_call_subprog(ctx); 109 109 + return ret + 8; 110 110 + } 111 111 + 112 112 + __noinline int exception_ext_global(struct __sk_buff *ctx) 113 113 + { 114 114 + volatile int ret = 0; 115 115 + 116 116 + return ret; 117 117 + } 118 118 + 119 119 + static __noinline int exception_ext_static(struct __sk_buff *ctx) 120 120 + { 121 121 + return exception_ext_global(ctx); 122 122 + } 123 123 + 124 124 + SEC("tc") 125 125 + int exception_ext(struct __sk_buff *ctx) 126 126 + { 127 127 + return exception_ext_static(ctx); 128 128 + } 129 129 + 130 130 + __noinline int exception_cb_mod_global(u64 cookie) 131 131 + { 132 132 + volatile int ret = 0; 133 133 + 134 134 + return ret; 135 135 + } 136 136 + 137 137 + /* Example of how the exception callback supplied during verification can still 138 138 + * introduce extensions by calling to dummy global functions, and alter runtime 139 139 + * behavior. 140 140 + * 141 141 + * Right now we don't allow freplace attachment to exception callback itself, 142 142 + * but if the need arises this restriction is technically feasible to relax in 143 143 + * the future. 144 144 + */ 145 145 + __noinline int exception_cb_mod(u64 cookie) 146 146 + { 147 147 + return exception_cb_mod_global(cookie) + cookie + 10; 148 148 + } 149 149 + 150 150 + SEC("tc") 151 151 + __exception_cb(exception_cb_mod) 152 152 + int exception_ext_mod_cb_runtime(struct __sk_buff *ctx) 153 153 + { 154 154 + bpf_throw(25); 155 155 + return 0; 156 156 + } 157 157 + 158 158 + __noinline static int subprog(struct __sk_buff *ctx) 159 159 + { 160 160 + return bpf_ktime_get_ns(); 161 161 + } 162 162 + 163 163 + __noinline static int throwing_subprog(struct __sk_buff *ctx) 164 164 + { 165 165 + if (ctx->tstamp) 166 166 + bpf_throw(0); 167 167 + return bpf_ktime_get_ns(); 168 168 + } 169 169 + 170 170 + __noinline int global_subprog(struct __sk_buff *ctx) 171 171 + { 172 172 + return bpf_ktime_get_ns(); 173 173 + } 174 174 + 175 175 + __noinline int throwing_global_subprog(struct __sk_buff *ctx) 176 176 + { 177 177 + if (ctx->tstamp) 178 178 + bpf_throw(0); 179 179 + return bpf_ktime_get_ns(); 180 180 + } 181 181 + 182 182 + SEC("tc") 183 183 + int exception_throw_subprog(struct __sk_buff *ctx) 184 184 + { 185 185 + switch (ctx->protocol) { 186 186 + case 1: 187 187 + return subprog(ctx); 188 188 + case 2: 189 189 + return global_subprog(ctx); 190 190 + case 3: 191 191 + return throwing_subprog(ctx); 192 192 + case 4: 193 193 + return throwing_global_subprog(ctx); 194 194 + default: 195 195 + break; 196 196 + } 197 197 + bpf_throw(1); 198 198 + return 0; 199 199 + } 200 200 + 201 201 + __noinline int assert_nz_gfunc(u64 c) 202 202 + { 203 203 + volatile u64 cookie = c; 204 204 + 205 205 + bpf_assert(cookie != 0); 206 206 + return 0; 207 207 + } 208 208 + 209 209 + __noinline int assert_zero_gfunc(u64 c) 210 210 + { 211 211 + volatile u64 cookie = c; 212 212 + 213 213 + bpf_assert_eq(cookie, 0); 214 214 + return 0; 215 215 + } 216 216 + 217 217 + __noinline int assert_neg_gfunc(s64 c) 218 218 + { 219 219 + volatile s64 cookie = c; 220 220 + 221 221 + bpf_assert_lt(cookie, 0); 222 222 + return 0; 223 223 + } 224 224 + 225 225 + __noinline int assert_pos_gfunc(s64 c) 226 226 + { 227 227 + volatile s64 cookie = c; 228 228 + 229 229 + bpf_assert_gt(cookie, 0); 230 230 + return 0; 231 231 + } 232 232 + 233 233 + __noinline int assert_negeq_gfunc(s64 c) 234 234 + { 235 235 + volatile s64 cookie = c; 236 236 + 237 237 + bpf_assert_le(cookie, -1); 238 238 + return 0; 239 239 + } 240 240 + 241 241 + __noinline int assert_poseq_gfunc(s64 c) 242 242 + { 243 243 + volatile s64 cookie = c; 244 244 + 245 245 + bpf_assert_ge(cookie, 1); 246 246 + return 0; 247 247 + } 248 248 + 249 249 + __noinline int assert_nz_gfunc_with(u64 c) 250 250 + { 251 251 + volatile u64 cookie = c; 252 252 + 253 253 + bpf_assert_with(cookie != 0, cookie + 100); 254 254 + return 0; 255 255 + } 256 256 + 257 257 + __noinline int assert_zero_gfunc_with(u64 c) 258 258 + { 259 259 + volatile u64 cookie = c; 260 260 + 261 261 + bpf_assert_eq_with(cookie, 0, cookie + 100); 262 262 + return 0; 263 263 + } 264 264 + 265 265 + __noinline int assert_neg_gfunc_with(s64 c) 266 266 + { 267 267 + volatile s64 cookie = c; 268 268 + 269 269 + bpf_assert_lt_with(cookie, 0, cookie + 100); 270 270 + return 0; 271 271 + } 272 272 + 273 273 + __noinline int assert_pos_gfunc_with(s64 c) 274 274 + { 275 275 + volatile s64 cookie = c; 276 276 + 277 277 + bpf_assert_gt_with(cookie, 0, cookie + 100); 278 278 + return 0; 279 279 + } 280 280 + 281 281 + __noinline int assert_negeq_gfunc_with(s64 c) 282 282 + { 283 283 + volatile s64 cookie = c; 284 284 + 285 285 + bpf_assert_le_with(cookie, -1, cookie + 100); 286 286 + return 0; 287 287 + } 288 288 + 289 289 + __noinline int assert_poseq_gfunc_with(s64 c) 290 290 + { 291 291 + volatile s64 cookie = c; 292 292 + 293 293 + bpf_assert_ge_with(cookie, 1, cookie + 100); 294 294 + return 0; 295 295 + } 296 296 + 297 297 + #define check_assert(name, cookie, tag) \ 298 298 + SEC("tc") \ 299 299 + int exception##tag##name(struct __sk_buff *ctx) \ 300 300 + { \ 301 301 + return name(cookie) + 1; \ 302 302 + } 303 303 + 304 304 + check_assert(assert_nz_gfunc, 5, _); 305 305 + check_assert(assert_zero_gfunc, 0, _); 306 306 + check_assert(assert_neg_gfunc, -100, _); 307 307 + check_assert(assert_pos_gfunc, 100, _); 308 308 + check_assert(assert_negeq_gfunc, -1, _); 309 309 + check_assert(assert_poseq_gfunc, 1, _); 310 310 + 311 311 + check_assert(assert_nz_gfunc_with, 5, _); 312 312 + check_assert(assert_zero_gfunc_with, 0, _); 313 313 + check_assert(assert_neg_gfunc_with, -100, _); 314 314 + check_assert(assert_pos_gfunc_with, 100, _); 315 315 + check_assert(assert_negeq_gfunc_with, -1, _); 316 316 + check_assert(assert_poseq_gfunc_with, 1, _); 317 317 + 318 318 + check_assert(assert_nz_gfunc, 0, _bad_); 319 319 + check_assert(assert_zero_gfunc, 5, _bad_); 320 320 + check_assert(assert_neg_gfunc, 100, _bad_); 321 321 + check_assert(assert_pos_gfunc, -100, _bad_); 322 322 + check_assert(assert_negeq_gfunc, 1, _bad_); 323 323 + check_assert(assert_poseq_gfunc, -1, _bad_); 324 324 + 325 325 + check_assert(assert_nz_gfunc_with, 0, _bad_); 326 326 + check_assert(assert_zero_gfunc_with, 5, _bad_); 327 327 + check_assert(assert_neg_gfunc_with, 100, _bad_); 328 328 + check_assert(assert_pos_gfunc_with, -100, _bad_); 329 329 + check_assert(assert_negeq_gfunc_with, 1, _bad_); 330 330 + check_assert(assert_poseq_gfunc_with, -1, _bad_); 331 331 + 332 332 + SEC("tc") 333 333 + int exception_assert_range(struct __sk_buff *ctx) 334 334 + { 335 335 + u64 time = bpf_ktime_get_ns(); 336 336 + 337 337 + bpf_assert_range(time, 0, ~0ULL); 338 338 + return 1; 339 339 + } 340 340 + 341 341 + SEC("tc") 342 342 + int exception_assert_range_with(struct __sk_buff *ctx) 343 343 + { 344 344 + u64 time = bpf_ktime_get_ns(); 345 345 + 346 346 + bpf_assert_range_with(time, 0, ~0ULL, 10); 347 347 + return 1; 348 348 + } 349 349 + 350 350 + SEC("tc") 351 351 + int exception_bad_assert_range(struct __sk_buff *ctx) 352 352 + { 353 353 + u64 time = bpf_ktime_get_ns(); 354 354 + 355 355 + bpf_assert_range(time, -100, 100); 356 356 + return 1; 357 357 + } 358 358 + 359 359 + SEC("tc") 360 360 + int exception_bad_assert_range_with(struct __sk_buff *ctx) 361 361 + { 362 362 + u64 time = bpf_ktime_get_ns(); 363 363 + 364 364 + bpf_assert_range_with(time, -1000, 1000, 10); 365 365 + return 1; 366 366 + } 367 367 + 368 368 + char _license[] SEC("license") = "GPL";

+135

tools/testing/selftests/bpf/progs/exceptions_assert.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + #include <vmlinux.h> 3 3 + #include <limits.h> 4 4 + #include <bpf/bpf_tracing.h> 5 5 + #include <bpf/bpf_helpers.h> 6 6 + #include <bpf/bpf_core_read.h> 7 7 + #include <bpf/bpf_endian.h> 8 8 + #include "bpf_misc.h" 9 9 + #include "bpf_experimental.h" 10 10 + 11 11 + #define check_assert(type, op, name, value) \ 12 12 + SEC("?tc") \ 13 13 + __log_level(2) __failure \ 14 14 + int check_assert_##op##_##name(void *ctx) \ 15 15 + { \ 16 16 + type num = bpf_ktime_get_ns(); \ 17 17 + bpf_assert_##op(num, value); \ 18 18 + return *(u64 *)num; \ 19 19 + } 20 20 + 21 21 + __msg(": R0_w=-2147483648 R10=fp0") 22 22 + check_assert(s64, eq, int_min, INT_MIN); 23 23 + __msg(": R0_w=2147483647 R10=fp0") 24 24 + check_assert(s64, eq, int_max, INT_MAX); 25 25 + __msg(": R0_w=0 R10=fp0") 26 26 + check_assert(s64, eq, zero, 0); 27 27 + __msg(": R0_w=-9223372036854775808 R1_w=-9223372036854775808 R10=fp0") 28 28 + check_assert(s64, eq, llong_min, LLONG_MIN); 29 29 + __msg(": R0_w=9223372036854775807 R1_w=9223372036854775807 R10=fp0") 30 30 + check_assert(s64, eq, llong_max, LLONG_MAX); 31 31 + 32 32 + __msg(": R0_w=scalar(smax=2147483646) R10=fp0") 33 33 + check_assert(s64, lt, pos, INT_MAX); 34 34 + __msg(": R0_w=scalar(umin=9223372036854775808,var_off=(0x8000000000000000; 0x7fffffffffffffff))") 35 35 + check_assert(s64, lt, zero, 0); 36 36 + __msg(": R0_w=scalar(umin=9223372036854775808,umax=18446744071562067967,var_off=(0x8000000000000000; 0x7fffffffffffffff))") 37 37 + check_assert(s64, lt, neg, INT_MIN); 38 38 + 39 39 + __msg(": R0_w=scalar(smax=2147483647) R10=fp0") 40 40 + check_assert(s64, le, pos, INT_MAX); 41 41 + __msg(": R0_w=scalar(smax=0) R10=fp0") 42 42 + check_assert(s64, le, zero, 0); 43 43 + __msg(": R0_w=scalar(umin=9223372036854775808,umax=18446744071562067968,var_off=(0x8000000000000000; 0x7fffffffffffffff))") 44 44 + check_assert(s64, le, neg, INT_MIN); 45 45 + 46 46 + __msg(": R0_w=scalar(umin=2147483648,umax=9223372036854775807,var_off=(0x0; 0x7fffffffffffffff))") 47 47 + check_assert(s64, gt, pos, INT_MAX); 48 48 + __msg(": R0_w=scalar(umin=1,umax=9223372036854775807,var_off=(0x0; 0x7fffffffffffffff))") 49 49 + check_assert(s64, gt, zero, 0); 50 50 + __msg(": R0_w=scalar(smin=-2147483647) R10=fp0") 51 51 + check_assert(s64, gt, neg, INT_MIN); 52 52 + 53 53 + __msg(": R0_w=scalar(umin=2147483647,umax=9223372036854775807,var_off=(0x0; 0x7fffffffffffffff))") 54 54 + check_assert(s64, ge, pos, INT_MAX); 55 55 + __msg(": R0_w=scalar(umax=9223372036854775807,var_off=(0x0; 0x7fffffffffffffff)) R10=fp0") 56 56 + check_assert(s64, ge, zero, 0); 57 57 + __msg(": R0_w=scalar(smin=-2147483648) R10=fp0") 58 58 + check_assert(s64, ge, neg, INT_MIN); 59 59 + 60 60 + SEC("?tc") 61 61 + __log_level(2) __failure 62 62 + __msg(": R0=0 R1=ctx(off=0,imm=0) R2=scalar(smin=-2147483646,smax=2147483645) R10=fp0") 63 63 + int check_assert_range_s64(struct __sk_buff *ctx) 64 64 + { 65 65 + struct bpf_sock *sk = ctx->sk; 66 66 + s64 num; 67 67 + 68 68 + _Static_assert(_Generic((sk->rx_queue_mapping), s32: 1, default: 0), "type match"); 69 69 + if (!sk) 70 70 + return 0; 71 71 + num = sk->rx_queue_mapping; 72 72 + bpf_assert_range(num, INT_MIN + 2, INT_MAX - 2); 73 73 + return *((u8 *)ctx + num); 74 74 + } 75 75 + 76 76 + SEC("?tc") 77 77 + __log_level(2) __failure 78 78 + __msg(": R1=ctx(off=0,imm=0) R2=scalar(umin=4096,umax=8192,var_off=(0x0; 0x3fff))") 79 79 + int check_assert_range_u64(struct __sk_buff *ctx) 80 80 + { 81 81 + u64 num = ctx->len; 82 82 + 83 83 + bpf_assert_range(num, 4096, 8192); 84 84 + return *((u8 *)ctx + num); 85 85 + } 86 86 + 87 87 + SEC("?tc") 88 88 + __log_level(2) __failure 89 89 + __msg(": R0=0 R1=ctx(off=0,imm=0) R2=4096 R10=fp0") 90 90 + int check_assert_single_range_s64(struct __sk_buff *ctx) 91 91 + { 92 92 + struct bpf_sock *sk = ctx->sk; 93 93 + s64 num; 94 94 + 95 95 + _Static_assert(_Generic((sk->rx_queue_mapping), s32: 1, default: 0), "type match"); 96 96 + if (!sk) 97 97 + return 0; 98 98 + num = sk->rx_queue_mapping; 99 99 + 100 100 + bpf_assert_range(num, 4096, 4096); 101 101 + return *((u8 *)ctx + num); 102 102 + } 103 103 + 104 104 + SEC("?tc") 105 105 + __log_level(2) __failure 106 106 + __msg(": R1=ctx(off=0,imm=0) R2=4096 R10=fp0") 107 107 + int check_assert_single_range_u64(struct __sk_buff *ctx) 108 108 + { 109 109 + u64 num = ctx->len; 110 110 + 111 111 + bpf_assert_range(num, 4096, 4096); 112 112 + return *((u8 *)ctx + num); 113 113 + } 114 114 + 115 115 + SEC("?tc") 116 116 + __log_level(2) __failure 117 117 + __msg(": R1=pkt(off=64,r=64,imm=0) R2=pkt_end(off=0,imm=0) R6=pkt(off=0,r=64,imm=0) R10=fp0") 118 118 + int check_assert_generic(struct __sk_buff *ctx) 119 119 + { 120 120 + u8 *data_end = (void *)(long)ctx->data_end; 121 121 + u8 *data = (void *)(long)ctx->data; 122 122 + 123 123 + bpf_assert(data + 64 <= data_end); 124 124 + return data[128]; 125 125 + } 126 126 + 127 127 + SEC("?fentry/bpf_check") 128 128 + __failure __msg("At program exit the register R0 has value (0x40; 0x0)") 129 129 + int check_assert_with_return(void *ctx) 130 130 + { 131 131 + bpf_assert_with(!ctx, 64); 132 132 + return 0; 133 133 + } 134 134 + 135 135 + char _license[] SEC("license") = "GPL";

+72

tools/testing/selftests/bpf/progs/exceptions_ext.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + #include <vmlinux.h> 3 3 + #include <bpf/bpf_helpers.h> 4 4 + #include "bpf_experimental.h" 5 5 + 6 6 + SEC("?fentry") 7 7 + int pfentry(void *ctx) 8 8 + { 9 9 + return 0; 10 10 + } 11 11 + 12 12 + SEC("?fentry") 13 13 + int throwing_fentry(void *ctx) 14 14 + { 15 15 + bpf_throw(0); 16 16 + return 0; 17 17 + } 18 18 + 19 19 + __noinline int exception_cb(u64 cookie) 20 20 + { 21 21 + return cookie + 64; 22 22 + } 23 23 + 24 24 + SEC("?freplace") 25 25 + int extension(struct __sk_buff *ctx) 26 26 + { 27 27 + return 0; 28 28 + } 29 29 + 30 30 + SEC("?freplace") 31 31 + __exception_cb(exception_cb) 32 32 + int throwing_exception_cb_extension(u64 cookie) 33 33 + { 34 34 + bpf_throw(32); 35 35 + return 0; 36 36 + } 37 37 + 38 38 + SEC("?freplace") 39 39 + __exception_cb(exception_cb) 40 40 + int throwing_extension(struct __sk_buff *ctx) 41 41 + { 42 42 + bpf_throw(64); 43 43 + return 0; 44 44 + } 45 45 + 46 46 + SEC("?fexit") 47 47 + int pfexit(void *ctx) 48 48 + { 49 49 + return 0; 50 50 + } 51 51 + 52 52 + SEC("?fexit") 53 53 + int throwing_fexit(void *ctx) 54 54 + { 55 55 + bpf_throw(0); 56 56 + return 0; 57 57 + } 58 58 + 59 59 + SEC("?fmod_ret") 60 60 + int pfmod_ret(void *ctx) 61 61 + { 62 62 + return 0; 63 63 + } 64 64 + 65 65 + SEC("?fmod_ret") 66 66 + int throwing_fmod_ret(void *ctx) 67 67 + { 68 68 + bpf_throw(0); 69 69 + return 0; 70 70 + } 71 71 + 72 72 + char _license[] SEC("license") = "GPL";

+347

tools/testing/selftests/bpf/progs/exceptions_fail.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + #include <vmlinux.h> 3 3 + #include <bpf/bpf_tracing.h> 4 4 + #include <bpf/bpf_helpers.h> 5 5 + #include <bpf/bpf_core_read.h> 6 6 + 7 7 + #include "bpf_misc.h" 8 8 + #include "bpf_experimental.h" 9 9 + 10 10 + extern void bpf_rcu_read_lock(void) __ksym; 11 11 + 12 12 + #define private(name) SEC(".bss." #name) __hidden __attribute__((aligned(8))) 13 13 + 14 14 + struct foo { 15 15 + struct bpf_rb_node node; 16 16 + }; 17 17 + 18 18 + struct hmap_elem { 19 19 + struct bpf_timer timer; 20 20 + }; 21 21 + 22 22 + struct { 23 23 + __uint(type, BPF_MAP_TYPE_HASH); 24 24 + __uint(max_entries, 64); 25 25 + __type(key, int); 26 26 + __type(value, struct hmap_elem); 27 27 + } hmap SEC(".maps"); 28 28 + 29 29 + private(A) struct bpf_spin_lock lock; 30 30 + private(A) struct bpf_rb_root rbtree __contains(foo, node); 31 31 + 32 32 + __noinline void *exception_cb_bad_ret_type(u64 cookie) 33 33 + { 34 34 + return NULL; 35 35 + } 36 36 + 37 37 + __noinline int exception_cb_bad_arg_0(void) 38 38 + { 39 39 + return 0; 40 40 + } 41 41 + 42 42 + __noinline int exception_cb_bad_arg_2(int a, int b) 43 43 + { 44 44 + return 0; 45 45 + } 46 46 + 47 47 + __noinline int exception_cb_ok_arg_small(int a) 48 48 + { 49 49 + return 0; 50 50 + } 51 51 + 52 52 + SEC("?tc") 53 53 + __exception_cb(exception_cb_bad_ret_type) 54 54 + __failure __msg("Global function exception_cb_bad_ret_type() doesn't return scalar.") 55 55 + int reject_exception_cb_type_1(struct __sk_buff *ctx) 56 56 + { 57 57 + bpf_throw(0); 58 58 + return 0; 59 59 + } 60 60 + 61 61 + SEC("?tc") 62 62 + __exception_cb(exception_cb_bad_arg_0) 63 63 + __failure __msg("exception cb only supports single integer argument") 64 64 + int reject_exception_cb_type_2(struct __sk_buff *ctx) 65 65 + { 66 66 + bpf_throw(0); 67 67 + return 0; 68 68 + } 69 69 + 70 70 + SEC("?tc") 71 71 + __exception_cb(exception_cb_bad_arg_2) 72 72 + __failure __msg("exception cb only supports single integer argument") 73 73 + int reject_exception_cb_type_3(struct __sk_buff *ctx) 74 74 + { 75 75 + bpf_throw(0); 76 76 + return 0; 77 77 + } 78 78 + 79 79 + SEC("?tc") 80 80 + __exception_cb(exception_cb_ok_arg_small) 81 81 + __success 82 82 + int reject_exception_cb_type_4(struct __sk_buff *ctx) 83 83 + { 84 84 + bpf_throw(0); 85 85 + return 0; 86 86 + } 87 87 + 88 88 + __noinline 89 89 + static int timer_cb(void *map, int *key, struct bpf_timer *timer) 90 90 + { 91 91 + bpf_throw(0); 92 92 + return 0; 93 93 + } 94 94 + 95 95 + SEC("?tc") 96 96 + __failure __msg("cannot be called from callback subprog") 97 97 + int reject_async_callback_throw(struct __sk_buff *ctx) 98 98 + { 99 99 + struct hmap_elem *elem; 100 100 + 101 101 + elem = bpf_map_lookup_elem(&hmap, &(int){0}); 102 102 + if (!elem) 103 103 + return 0; 104 104 + return bpf_timer_set_callback(&elem->timer, timer_cb); 105 105 + } 106 106 + 107 107 + __noinline static int subprog_lock(struct __sk_buff *ctx) 108 108 + { 109 109 + volatile int ret = 0; 110 110 + 111 111 + bpf_spin_lock(&lock); 112 112 + if (ctx->len) 113 113 + bpf_throw(0); 114 114 + return ret; 115 115 + } 116 116 + 117 117 + SEC("?tc") 118 118 + __failure __msg("function calls are not allowed while holding a lock") 119 119 + int reject_with_lock(void *ctx) 120 120 + { 121 121 + bpf_spin_lock(&lock); 122 122 + bpf_throw(0); 123 123 + return 0; 124 124 + } 125 125 + 126 126 + SEC("?tc") 127 127 + __failure __msg("function calls are not allowed while holding a lock") 128 128 + int reject_subprog_with_lock(void *ctx) 129 129 + { 130 130 + return subprog_lock(ctx); 131 131 + } 132 132 + 133 133 + SEC("?tc") 134 134 + __failure __msg("bpf_rcu_read_unlock is missing") 135 135 + int reject_with_rcu_read_lock(void *ctx) 136 136 + { 137 137 + bpf_rcu_read_lock(); 138 138 + bpf_throw(0); 139 139 + return 0; 140 140 + } 141 141 + 142 142 + __noinline static int throwing_subprog(struct __sk_buff *ctx) 143 143 + { 144 144 + if (ctx->len) 145 145 + bpf_throw(0); 146 146 + return 0; 147 147 + } 148 148 + 149 149 + SEC("?tc") 150 150 + __failure __msg("bpf_rcu_read_unlock is missing") 151 151 + int reject_subprog_with_rcu_read_lock(void *ctx) 152 152 + { 153 153 + bpf_rcu_read_lock(); 154 154 + return throwing_subprog(ctx); 155 155 + } 156 156 + 157 157 + static bool rbless(struct bpf_rb_node *n1, const struct bpf_rb_node *n2) 158 158 + { 159 159 + bpf_throw(0); 160 160 + return true; 161 161 + } 162 162 + 163 163 + SEC("?tc") 164 164 + __failure __msg("function calls are not allowed while holding a lock") 165 165 + int reject_with_rbtree_add_throw(void *ctx) 166 166 + { 167 167 + struct foo *f; 168 168 + 169 169 + f = bpf_obj_new(typeof(*f)); 170 170 + if (!f) 171 171 + return 0; 172 172 + bpf_spin_lock(&lock); 173 173 + bpf_rbtree_add(&rbtree, &f->node, rbless); 174 174 + return 0; 175 175 + } 176 176 + 177 177 + SEC("?tc") 178 178 + __failure __msg("Unreleased reference") 179 179 + int reject_with_reference(void *ctx) 180 180 + { 181 181 + struct foo *f; 182 182 + 183 183 + f = bpf_obj_new(typeof(*f)); 184 184 + if (!f) 185 185 + return 0; 186 186 + bpf_throw(0); 187 187 + return 0; 188 188 + } 189 189 + 190 190 + __noinline static int subprog_ref(struct __sk_buff *ctx) 191 191 + { 192 192 + struct foo *f; 193 193 + 194 194 + f = bpf_obj_new(typeof(*f)); 195 195 + if (!f) 196 196 + return 0; 197 197 + bpf_throw(0); 198 198 + return 0; 199 199 + } 200 200 + 201 201 + __noinline static int subprog_cb_ref(u32 i, void *ctx) 202 202 + { 203 203 + bpf_throw(0); 204 204 + return 0; 205 205 + } 206 206 + 207 207 + SEC("?tc") 208 208 + __failure __msg("Unreleased reference") 209 209 + int reject_with_cb_reference(void *ctx) 210 210 + { 211 211 + struct foo *f; 212 212 + 213 213 + f = bpf_obj_new(typeof(*f)); 214 214 + if (!f) 215 215 + return 0; 216 216 + bpf_loop(5, subprog_cb_ref, NULL, 0); 217 217 + return 0; 218 218 + } 219 219 + 220 220 + SEC("?tc") 221 221 + __failure __msg("cannot be called from callback") 222 222 + int reject_with_cb(void *ctx) 223 223 + { 224 224 + bpf_loop(5, subprog_cb_ref, NULL, 0); 225 225 + return 0; 226 226 + } 227 227 + 228 228 + SEC("?tc") 229 229 + __failure __msg("Unreleased reference") 230 230 + int reject_with_subprog_reference(void *ctx) 231 231 + { 232 232 + return subprog_ref(ctx) + 1; 233 233 + } 234 234 + 235 235 + __noinline int throwing_exception_cb(u64 c) 236 236 + { 237 237 + bpf_throw(0); 238 238 + return c; 239 239 + } 240 240 + 241 241 + __noinline int exception_cb1(u64 c) 242 242 + { 243 243 + return c; 244 244 + } 245 245 + 246 246 + __noinline int exception_cb2(u64 c) 247 247 + { 248 248 + return c; 249 249 + } 250 250 + 251 251 + static __noinline int static_func(struct __sk_buff *ctx) 252 252 + { 253 253 + return exception_cb1(ctx->tstamp); 254 254 + } 255 255 + 256 256 + __noinline int global_func(struct __sk_buff *ctx) 257 257 + { 258 258 + return exception_cb1(ctx->tstamp); 259 259 + } 260 260 + 261 261 + SEC("?tc") 262 262 + __exception_cb(throwing_exception_cb) 263 263 + __failure __msg("cannot be called from callback subprog") 264 264 + int reject_throwing_exception_cb(struct __sk_buff *ctx) 265 265 + { 266 266 + return 0; 267 267 + } 268 268 + 269 269 + SEC("?tc") 270 270 + __exception_cb(exception_cb1) 271 271 + __failure __msg("cannot call exception cb directly") 272 272 + int reject_exception_cb_call_global_func(struct __sk_buff *ctx) 273 273 + { 274 274 + return global_func(ctx); 275 275 + } 276 276 + 277 277 + SEC("?tc") 278 278 + __exception_cb(exception_cb1) 279 279 + __failure __msg("cannot call exception cb directly") 280 280 + int reject_exception_cb_call_static_func(struct __sk_buff *ctx) 281 281 + { 282 282 + return static_func(ctx); 283 283 + } 284 284 + 285 285 + SEC("?tc") 286 286 + __exception_cb(exception_cb1) 287 287 + __exception_cb(exception_cb2) 288 288 + __failure __msg("multiple exception callback tags for main subprog") 289 289 + int reject_multiple_exception_cb(struct __sk_buff *ctx) 290 290 + { 291 291 + bpf_throw(0); 292 292 + return 16; 293 293 + } 294 294 + 295 295 + __noinline int exception_cb_bad_ret(u64 c) 296 296 + { 297 297 + return c; 298 298 + } 299 299 + 300 300 + SEC("?fentry/bpf_check") 301 301 + __exception_cb(exception_cb_bad_ret) 302 302 + __failure __msg("At program exit the register R0 has unknown scalar value should") 303 303 + int reject_set_exception_cb_bad_ret1(void *ctx) 304 304 + { 305 305 + return 0; 306 306 + } 307 307 + 308 308 + SEC("?fentry/bpf_check") 309 309 + __failure __msg("At program exit the register R0 has value (0x40; 0x0) should") 310 310 + int reject_set_exception_cb_bad_ret2(void *ctx) 311 311 + { 312 312 + bpf_throw(64); 313 313 + return 0; 314 314 + } 315 315 + 316 316 + __noinline static int loop_cb1(u32 index, int *ctx) 317 317 + { 318 318 + bpf_throw(0); 319 319 + return 0; 320 320 + } 321 321 + 322 322 + __noinline static int loop_cb2(u32 index, int *ctx) 323 323 + { 324 324 + bpf_throw(0); 325 325 + return 0; 326 326 + } 327 327 + 328 328 + SEC("?tc") 329 329 + __failure __msg("cannot be called from callback") 330 330 + int reject_exception_throw_cb(struct __sk_buff *ctx) 331 331 + { 332 332 + bpf_loop(5, loop_cb1, NULL, 0); 333 333 + return 0; 334 334 + } 335 335 + 336 336 + SEC("?tc") 337 337 + __failure __msg("cannot be called from callback") 338 338 + int reject_exception_throw_cb_diff(struct __sk_buff *ctx) 339 339 + { 340 340 + if (ctx->protocol) 341 341 + bpf_loop(5, loop_cb1, NULL, 0); 342 342 + else 343 343 + bpf_loop(5, loop_cb2, NULL, 0); 344 344 + return 0; 345 345 + } 346 346 + 347 347 + char _license[] SEC("license") = "GPL";

+183

tools/testing/selftests/bpf/progs/percpu_alloc_array.c

reviewed

··· 1 1 + #include "bpf_experimental.h" 2 2 + 3 3 + struct val_t { 4 4 + long b, c, d; 5 5 + }; 6 6 + 7 7 + struct elem { 8 8 + long sum; 9 9 + struct val_t __percpu_kptr *pc; 10 10 + }; 11 11 + 12 12 + struct { 13 13 + __uint(type, BPF_MAP_TYPE_ARRAY); 14 14 + __uint(max_entries, 1); 15 15 + __type(key, int); 16 16 + __type(value, struct elem); 17 17 + } array SEC(".maps"); 18 18 + 19 19 + void bpf_rcu_read_lock(void) __ksym; 20 20 + void bpf_rcu_read_unlock(void) __ksym; 21 21 + 22 22 + const volatile int nr_cpus; 23 23 + 24 24 + /* Initialize the percpu object */ 25 25 + SEC("?fentry/bpf_fentry_test1") 26 26 + int BPF_PROG(test_array_map_1) 27 27 + { 28 28 + struct val_t __percpu_kptr *p; 29 29 + struct elem *e; 30 30 + int index = 0; 31 31 + 32 32 + e = bpf_map_lookup_elem(&array, &index); 33 33 + if (!e) 34 34 + return 0; 35 35 + 36 36 + p = bpf_percpu_obj_new(struct val_t); 37 37 + if (!p) 38 38 + return 0; 39 39 + 40 40 + p = bpf_kptr_xchg(&e->pc, p); 41 41 + if (p) 42 42 + bpf_percpu_obj_drop(p); 43 43 + 44 44 + return 0; 45 45 + } 46 46 + 47 47 + /* Update percpu data */ 48 48 + SEC("?fentry/bpf_fentry_test2") 49 49 + int BPF_PROG(test_array_map_2) 50 50 + { 51 51 + struct val_t __percpu_kptr *p; 52 52 + struct val_t *v; 53 53 + struct elem *e; 54 54 + int index = 0; 55 55 + 56 56 + e = bpf_map_lookup_elem(&array, &index); 57 57 + if (!e) 58 58 + return 0; 59 59 + 60 60 + p = e->pc; 61 61 + if (!p) 62 62 + return 0; 63 63 + 64 64 + v = bpf_per_cpu_ptr(p, 0); 65 65 + if (!v) 66 66 + return 0; 67 67 + v->c = 1; 68 68 + v->d = 2; 69 69 + 70 70 + return 0; 71 71 + } 72 72 + 73 73 + int cpu0_field_d, sum_field_c; 74 74 + 75 75 + /* Summarize percpu data */ 76 76 + SEC("?fentry/bpf_fentry_test3") 77 77 + int BPF_PROG(test_array_map_3) 78 78 + { 79 79 + struct val_t __percpu_kptr *p; 80 80 + int i, index = 0; 81 81 + struct val_t *v; 82 82 + struct elem *e; 83 83 + 84 84 + e = bpf_map_lookup_elem(&array, &index); 85 85 + if (!e) 86 86 + return 0; 87 87 + 88 88 + p = e->pc; 89 89 + if (!p) 90 90 + return 0; 91 91 + 92 92 + bpf_for(i, 0, nr_cpus) { 93 93 + v = bpf_per_cpu_ptr(p, i); 94 94 + if (v) { 95 95 + if (i == 0) 96 96 + cpu0_field_d = v->d; 97 97 + sum_field_c += v->c; 98 98 + } 99 99 + } 100 100 + 101 101 + return 0; 102 102 + } 103 103 + 104 104 + /* Explicitly free allocated percpu data */ 105 105 + SEC("?fentry/bpf_fentry_test4") 106 106 + int BPF_PROG(test_array_map_4) 107 107 + { 108 108 + struct val_t __percpu_kptr *p; 109 109 + struct elem *e; 110 110 + int index = 0; 111 111 + 112 112 + e = bpf_map_lookup_elem(&array, &index); 113 113 + if (!e) 114 114 + return 0; 115 115 + 116 116 + /* delete */ 117 117 + p = bpf_kptr_xchg(&e->pc, NULL); 118 118 + if (p) { 119 119 + bpf_percpu_obj_drop(p); 120 120 + } 121 121 + 122 122 + return 0; 123 123 + } 124 124 + 125 125 + SEC("?fentry.s/bpf_fentry_test1") 126 126 + int BPF_PROG(test_array_map_10) 127 127 + { 128 128 + struct val_t __percpu_kptr *p, *p1; 129 129 + int i, index = 0; 130 130 + struct val_t *v; 131 131 + struct elem *e; 132 132 + 133 133 + e = bpf_map_lookup_elem(&array, &index); 134 134 + if (!e) 135 135 + return 0; 136 136 + 137 137 + bpf_rcu_read_lock(); 138 138 + p = e->pc; 139 139 + if (!p) { 140 140 + p = bpf_percpu_obj_new(struct val_t); 141 141 + if (!p) 142 142 + goto out; 143 143 + 144 144 + p1 = bpf_kptr_xchg(&e->pc, p); 145 145 + if (p1) { 146 146 + /* race condition */ 147 147 + bpf_percpu_obj_drop(p1); 148 148 + } 149 149 + } 150 150 + 151 151 + v = bpf_this_cpu_ptr(p); 152 152 + v->c = 3; 153 153 + v = bpf_this_cpu_ptr(p); 154 154 + v->c = 0; 155 155 + 156 156 + v = bpf_per_cpu_ptr(p, 0); 157 157 + if (!v) 158 158 + goto out; 159 159 + v->c = 1; 160 160 + v->d = 2; 161 161 + 162 162 + /* delete */ 163 163 + p1 = bpf_kptr_xchg(&e->pc, NULL); 164 164 + if (!p1) 165 165 + goto out; 166 166 + 167 167 + bpf_for(i, 0, nr_cpus) { 168 168 + v = bpf_per_cpu_ptr(p, i); 169 169 + if (v) { 170 170 + if (i == 0) 171 171 + cpu0_field_d = v->d; 172 172 + sum_field_c += v->c; 173 173 + } 174 174 + } 175 175 + 176 176 + /* finally release p */ 177 177 + bpf_percpu_obj_drop(p1); 178 178 + out: 179 179 + bpf_rcu_read_unlock(); 180 180 + return 0; 181 181 + } 182 182 + 183 183 + char _license[] SEC("license") = "GPL";

+105

tools/testing/selftests/bpf/progs/percpu_alloc_cgrp_local_storage.c

reviewed

··· 1 1 + #include "bpf_experimental.h" 2 2 + 3 3 + struct val_t { 4 4 + long b, c, d; 5 5 + }; 6 6 + 7 7 + struct elem { 8 8 + long sum; 9 9 + struct val_t __percpu_kptr *pc; 10 10 + }; 11 11 + 12 12 + struct { 13 13 + __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); 14 14 + __uint(map_flags, BPF_F_NO_PREALLOC); 15 15 + __type(key, int); 16 16 + __type(value, struct elem); 17 17 + } cgrp SEC(".maps"); 18 18 + 19 19 + const volatile int nr_cpus; 20 20 + 21 21 + /* Initialize the percpu object */ 22 22 + SEC("fentry/bpf_fentry_test1") 23 23 + int BPF_PROG(test_cgrp_local_storage_1) 24 24 + { 25 25 + struct task_struct *task; 26 26 + struct val_t __percpu_kptr *p; 27 27 + struct elem *e; 28 28 + 29 29 + task = bpf_get_current_task_btf(); 30 30 + e = bpf_cgrp_storage_get(&cgrp, task->cgroups->dfl_cgrp, 0, 31 31 + BPF_LOCAL_STORAGE_GET_F_CREATE); 32 32 + if (!e) 33 33 + return 0; 34 34 + 35 35 + p = bpf_percpu_obj_new(struct val_t); 36 36 + if (!p) 37 37 + return 0; 38 38 + 39 39 + p = bpf_kptr_xchg(&e->pc, p); 40 40 + if (p) 41 41 + bpf_percpu_obj_drop(p); 42 42 + 43 43 + return 0; 44 44 + } 45 45 + 46 46 + /* Percpu data collection */ 47 47 + SEC("fentry/bpf_fentry_test2") 48 48 + int BPF_PROG(test_cgrp_local_storage_2) 49 49 + { 50 50 + struct task_struct *task; 51 51 + struct val_t __percpu_kptr *p; 52 52 + struct val_t *v; 53 53 + struct elem *e; 54 54 + 55 55 + task = bpf_get_current_task_btf(); 56 56 + e = bpf_cgrp_storage_get(&cgrp, task->cgroups->dfl_cgrp, 0, 0); 57 57 + if (!e) 58 58 + return 0; 59 59 + 60 60 + p = e->pc; 61 61 + if (!p) 62 62 + return 0; 63 63 + 64 64 + v = bpf_per_cpu_ptr(p, 0); 65 65 + if (!v) 66 66 + return 0; 67 67 + v->c = 1; 68 68 + v->d = 2; 69 69 + return 0; 70 70 + } 71 71 + 72 72 + int cpu0_field_d, sum_field_c; 73 73 + 74 74 + /* Summarize percpu data collection */ 75 75 + SEC("fentry/bpf_fentry_test3") 76 76 + int BPF_PROG(test_cgrp_local_storage_3) 77 77 + { 78 78 + struct task_struct *task; 79 79 + struct val_t __percpu_kptr *p; 80 80 + struct val_t *v; 81 81 + struct elem *e; 82 82 + int i; 83 83 + 84 84 + task = bpf_get_current_task_btf(); 85 85 + e = bpf_cgrp_storage_get(&cgrp, task->cgroups->dfl_cgrp, 0, 0); 86 86 + if (!e) 87 87 + return 0; 88 88 + 89 89 + p = e->pc; 90 90 + if (!p) 91 91 + return 0; 92 92 + 93 93 + bpf_for(i, 0, nr_cpus) { 94 94 + v = bpf_per_cpu_ptr(p, i); 95 95 + if (v) { 96 96 + if (i == 0) 97 97 + cpu0_field_d = v->d; 98 98 + sum_field_c += v->c; 99 99 + } 100 100 + } 101 101 + 102 102 + return 0; 103 103 + } 104 104 + 105 105 + char _license[] SEC("license") = "GPL";

+164

tools/testing/selftests/bpf/progs/percpu_alloc_fail.c

reviewed

··· 1 1 + #include "bpf_experimental.h" 2 2 + #include "bpf_misc.h" 3 3 + 4 4 + struct val_t { 5 5 + long b, c, d; 6 6 + }; 7 7 + 8 8 + struct val2_t { 9 9 + long b; 10 10 + }; 11 11 + 12 12 + struct val_with_ptr_t { 13 13 + char *p; 14 14 + }; 15 15 + 16 16 + struct val_with_rb_root_t { 17 17 + struct bpf_spin_lock lock; 18 18 + }; 19 19 + 20 20 + struct elem { 21 21 + long sum; 22 22 + struct val_t __percpu_kptr *pc; 23 23 + }; 24 24 + 25 25 + struct { 26 26 + __uint(type, BPF_MAP_TYPE_ARRAY); 27 27 + __uint(max_entries, 1); 28 28 + __type(key, int); 29 29 + __type(value, struct elem); 30 30 + } array SEC(".maps"); 31 31 + 32 32 + long ret; 33 33 + 34 34 + SEC("?fentry/bpf_fentry_test1") 35 35 + __failure __msg("store to referenced kptr disallowed") 36 36 + int BPF_PROG(test_array_map_1) 37 37 + { 38 38 + struct val_t __percpu_kptr *p; 39 39 + struct elem *e; 40 40 + int index = 0; 41 41 + 42 42 + e = bpf_map_lookup_elem(&array, &index); 43 43 + if (!e) 44 44 + return 0; 45 45 + 46 46 + p = bpf_percpu_obj_new(struct val_t); 47 47 + if (!p) 48 48 + return 0; 49 49 + 50 50 + p = bpf_kptr_xchg(&e->pc, p); 51 51 + if (p) 52 52 + bpf_percpu_obj_drop(p); 53 53 + 54 54 + e->pc = (struct val_t __percpu_kptr *)ret; 55 55 + return 0; 56 56 + } 57 57 + 58 58 + SEC("?fentry/bpf_fentry_test1") 59 59 + __failure __msg("invalid kptr access, R2 type=percpu_ptr_val2_t expected=ptr_val_t") 60 60 + int BPF_PROG(test_array_map_2) 61 61 + { 62 62 + struct val2_t __percpu_kptr *p2; 63 63 + struct val_t __percpu_kptr *p; 64 64 + struct elem *e; 65 65 + int index = 0; 66 66 + 67 67 + e = bpf_map_lookup_elem(&array, &index); 68 68 + if (!e) 69 69 + return 0; 70 70 + 71 71 + p2 = bpf_percpu_obj_new(struct val2_t); 72 72 + if (!p2) 73 73 + return 0; 74 74 + 75 75 + p = bpf_kptr_xchg(&e->pc, p2); 76 76 + if (p) 77 77 + bpf_percpu_obj_drop(p); 78 78 + 79 79 + return 0; 80 80 + } 81 81 + 82 82 + SEC("?fentry.s/bpf_fentry_test1") 83 83 + __failure __msg("R1 type=scalar expected=percpu_ptr_, percpu_rcu_ptr_, percpu_trusted_ptr_") 84 84 + int BPF_PROG(test_array_map_3) 85 85 + { 86 86 + struct val_t __percpu_kptr *p, *p1; 87 87 + struct val_t *v; 88 88 + struct elem *e; 89 89 + int index = 0; 90 90 + 91 91 + e = bpf_map_lookup_elem(&array, &index); 92 92 + if (!e) 93 93 + return 0; 94 94 + 95 95 + p = bpf_percpu_obj_new(struct val_t); 96 96 + if (!p) 97 97 + return 0; 98 98 + 99 99 + p1 = bpf_kptr_xchg(&e->pc, p); 100 100 + if (p1) 101 101 + bpf_percpu_obj_drop(p1); 102 102 + 103 103 + v = bpf_this_cpu_ptr(p); 104 104 + ret = v->b; 105 105 + return 0; 106 106 + } 107 107 + 108 108 + SEC("?fentry.s/bpf_fentry_test1") 109 109 + __failure __msg("arg#0 expected for bpf_percpu_obj_drop_impl()") 110 110 + int BPF_PROG(test_array_map_4) 111 111 + { 112 112 + struct val_t __percpu_kptr *p; 113 113 + 114 114 + p = bpf_percpu_obj_new(struct val_t); 115 115 + if (!p) 116 116 + return 0; 117 117 + 118 118 + bpf_obj_drop(p); 119 119 + return 0; 120 120 + } 121 121 + 122 122 + SEC("?fentry.s/bpf_fentry_test1") 123 123 + __failure __msg("arg#0 expected for bpf_obj_drop_impl()") 124 124 + int BPF_PROG(test_array_map_5) 125 125 + { 126 126 + struct val_t *p; 127 127 + 128 128 + p = bpf_obj_new(struct val_t); 129 129 + if (!p) 130 130 + return 0; 131 131 + 132 132 + bpf_percpu_obj_drop(p); 133 133 + return 0; 134 134 + } 135 135 + 136 136 + SEC("?fentry.s/bpf_fentry_test1") 137 137 + __failure __msg("bpf_percpu_obj_new type ID argument must be of a struct of scalars") 138 138 + int BPF_PROG(test_array_map_6) 139 139 + { 140 140 + struct val_with_ptr_t __percpu_kptr *p; 141 141 + 142 142 + p = bpf_percpu_obj_new(struct val_with_ptr_t); 143 143 + if (!p) 144 144 + return 0; 145 145 + 146 146 + bpf_percpu_obj_drop(p); 147 147 + return 0; 148 148 + } 149 149 + 150 150 + SEC("?fentry.s/bpf_fentry_test1") 151 151 + __failure __msg("bpf_percpu_obj_new type ID argument must not contain special fields") 152 152 + int BPF_PROG(test_array_map_7) 153 153 + { 154 154 + struct val_with_rb_root_t __percpu_kptr *p; 155 155 + 156 156 + p = bpf_percpu_obj_new(struct val_with_rb_root_t); 157 157 + if (!p) 158 158 + return 0; 159 159 + 160 160 + bpf_percpu_obj_drop(p); 161 161 + return 0; 162 162 + } 163 163 + 164 164 + char _license[] SEC("license") = "GPL";

+106

tools/testing/selftests/bpf/progs/preempted_bpf_ma_op.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (C) 2023. Huawei Technologies Co., Ltd */ 3 3 + #include <vmlinux.h> 4 4 + #include <bpf/bpf_tracing.h> 5 5 + #include <bpf/bpf_helpers.h> 6 6 + 7 7 + #include "bpf_experimental.h" 8 8 + 9 9 + struct bin_data { 10 10 + char data[256]; 11 11 + struct bpf_spin_lock lock; 12 12 + }; 13 13 + 14 14 + struct map_value { 15 15 + struct bin_data __kptr * data; 16 16 + }; 17 17 + 18 18 + struct { 19 19 + __uint(type, BPF_MAP_TYPE_ARRAY); 20 20 + __type(key, int); 21 21 + __type(value, struct map_value); 22 22 + __uint(max_entries, 2048); 23 23 + } array SEC(".maps"); 24 24 + 25 25 + char _license[] SEC("license") = "GPL"; 26 26 + 27 27 + bool nomem_err = false; 28 28 + 29 29 + static int del_array(unsigned int i, int *from) 30 30 + { 31 31 + struct map_value *value; 32 32 + struct bin_data *old; 33 33 + 34 34 + value = bpf_map_lookup_elem(&array, from); 35 35 + if (!value) 36 36 + return 1; 37 37 + 38 38 + old = bpf_kptr_xchg(&value->data, NULL); 39 39 + if (old) 40 40 + bpf_obj_drop(old); 41 41 + 42 42 + (*from)++; 43 43 + return 0; 44 44 + } 45 45 + 46 46 + static int add_array(unsigned int i, int *from) 47 47 + { 48 48 + struct bin_data *old, *new; 49 49 + struct map_value *value; 50 50 + 51 51 + value = bpf_map_lookup_elem(&array, from); 52 52 + if (!value) 53 53 + return 1; 54 54 + 55 55 + new = bpf_obj_new(typeof(*new)); 56 56 + if (!new) { 57 57 + nomem_err = true; 58 58 + return 1; 59 59 + } 60 60 + 61 61 + old = bpf_kptr_xchg(&value->data, new); 62 62 + if (old) 63 63 + bpf_obj_drop(old); 64 64 + 65 65 + (*from)++; 66 66 + return 0; 67 67 + } 68 68 + 69 69 + static void del_then_add_array(int from) 70 70 + { 71 71 + int i; 72 72 + 73 73 + i = from; 74 74 + bpf_loop(512, del_array, &i, 0); 75 75 + 76 76 + i = from; 77 77 + bpf_loop(512, add_array, &i, 0); 78 78 + } 79 79 + 80 80 + SEC("fentry/bpf_fentry_test1") 81 81 + int BPF_PROG2(test0, int, a) 82 82 + { 83 83 + del_then_add_array(0); 84 84 + return 0; 85 85 + } 86 86 + 87 87 + SEC("fentry/bpf_fentry_test2") 88 88 + int BPF_PROG2(test1, int, a, u64, b) 89 89 + { 90 90 + del_then_add_array(512); 91 91 + return 0; 92 92 + } 93 93 + 94 94 + SEC("fentry/bpf_fentry_test3") 95 95 + int BPF_PROG2(test2, char, a, int, b, u64, c) 96 96 + { 97 97 + del_then_add_array(1024); 98 98 + return 0; 99 99 + } 100 100 + 101 101 + SEC("fentry/bpf_fentry_test4") 102 102 + int BPF_PROG2(test3, void *, a, char, b, int, c, u64, d) 103 103 + { 104 104 + del_then_add_array(1536); 105 105 + return 0; 106 106 + }

+18

tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_fentry.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright Leon Hwang */ 3 3 + 4 4 + #include "vmlinux.h" 5 5 + #include <bpf/bpf_helpers.h> 6 6 + #include <bpf/bpf_tracing.h> 7 7 + 8 8 + int count = 0; 9 9 + 10 10 + SEC("fentry/subprog_tail") 11 11 + int BPF_PROG(fentry, struct sk_buff *skb) 12 12 + { 13 13 + count++; 14 14 + 15 15 + return 0; 16 16 + } 17 17 + 18 18 + char _license[] SEC("license") = "GPL";

+18

tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_fexit.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright Leon Hwang */ 3 3 + 4 4 + #include "vmlinux.h" 5 5 + #include <bpf/bpf_helpers.h> 6 6 + #include <bpf/bpf_tracing.h> 7 7 + 8 8 + int count = 0; 9 9 + 10 10 + SEC("fexit/subprog_tail") 11 11 + int BPF_PROG(fexit, struct sk_buff *skb) 12 12 + { 13 13 + count++; 14 14 + 15 15 + return 0; 16 16 + } 17 17 + 18 18 + char _license[] SEC("license") = "GPL";

+2 -1

tools/testing/selftests/bpf/progs/verifier_bswap.c

reviewed

··· 5 5 #include "bpf_misc.h" 6 6 7 7 #if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ 8 8 - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18 8 8 + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || defined(__TARGET_ARCH_arm)) && \ 9 9 + __clang_major__ >= 18 9 10 10 11 SEC("socket") 11 12 __description("BSWAP, 16")

+2 -1

tools/testing/selftests/bpf/progs/verifier_gotol.c

reviewed

··· 5 5 #include "bpf_misc.h" 6 6 7 7 #if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ 8 8 - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18 8 8 + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || defined(__TARGET_ARCH_arm)) && \ 9 9 + __clang_major__ >= 18 9 10 10 11 SEC("socket") 11 12 __description("gotol, small_imm")

+2 -1

tools/testing/selftests/bpf/progs/verifier_ldsx.c

reviewed

··· 5 5 #include "bpf_misc.h" 6 6 7 7 #if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ 8 8 - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18 8 8 + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || defined(__TARGET_ARCH_arm)) && \ 9 9 + __clang_major__ >= 18 9 10 10 11 SEC("socket") 11 12 __description("LDSX, S8")

+2 -1

tools/testing/selftests/bpf/progs/verifier_movsx.c

reviewed

··· 5 5 #include "bpf_misc.h" 6 6 7 7 #if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ 8 8 - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18 8 8 + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || defined(__TARGET_ARCH_arm)) && \ 9 9 + __clang_major__ >= 18 9 10 10 11 SEC("socket") 11 12 __description("MOV32SX, S8")

+2 -1

tools/testing/selftests/bpf/progs/verifier_sdiv.c

reviewed

··· 5 5 #include "bpf_misc.h" 6 6 7 7 #if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ 8 8 - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18 8 8 + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || defined(__TARGET_ARCH_arm)) && \ 9 9 + __clang_major__ >= 18 9 10 10 11 SEC("socket") 11 12 __description("SDIV32, non-zero imm divisor, check 1")

+9

tools/testing/selftests/bpf/test_bpftool_synctypes.py

reviewed

··· 509 509 source_map_types.remove('cgroup_storage_deprecated') 510 510 source_map_types.add('cgroup_storage') 511 511 512 512 + # The same applied to BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED and 513 513 + # BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE which share the same enum value 514 514 + # and source_map_types picks 515 515 + # BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED/percpu_cgroup_storage_deprecated. 516 516 + # Replace 'percpu_cgroup_storage_deprecated' with 'percpu_cgroup_storage' 517 517 + # so it aligns with what `bpftool map help` shows. 518 518 + source_map_types.remove('percpu_cgroup_storage_deprecated') 519 519 + source_map_types.add('percpu_cgroup_storage') 520 520 + 512 521 help_map_types = map_info.get_map_help() 513 522 help_map_options = map_info.get_options() 514 523 map_info.close()

+39 -1

tools/testing/selftests/bpf/test_xsk.sh

reviewed

··· 73 73 # 74 74 # Run test suite for physical device in loopback mode 75 75 # sudo ./test_xsk.sh -i IFACE 76 76 + # 77 77 + # Run test suite in a specific mode only [skb,drv,zc] 78 78 + # sudo ./test_xsk.sh -m MODE 79 79 + # 80 80 + # List available tests 81 81 + # ./test_xsk.sh -l 82 82 + # 83 83 + # Run a specific test from the test suite 84 84 + # sudo ./test_xsk.sh -t TEST_NAME 85 85 + # 86 86 + # Display the available command line options 87 87 + # ./test_xsk.sh -h 76 88 77 89 . xsk_prereqs.sh 78 90 79 91 ETH="" 80 92 81 81 - while getopts "vi:d" flag 93 93 + while getopts "vi:dm:lt:h" flag 82 94 do 83 95 case "${flag}" in 84 96 v) verbose=1;; 85 97 d) debug=1;; 86 98 i) ETH=${OPTARG};; 99 99 + m) MODE=${OPTARG};; 100 100 + l) list=1;; 101 101 + t) TEST=${OPTARG};; 102 102 + h) help=1;; 87 103 esac 88 104 done 89 105 ··· 147 131 ip link set ${VETH0} up 148 132 } 149 133 134 134 + if [[ $list -eq 1 ]]; then 135 135 + ./${XSKOBJ} -l 136 136 + exit 137 137 + fi 138 138 + 139 139 + if [[ $help -eq 1 ]]; then 140 140 + ./${XSKOBJ} 141 141 + exit 142 142 + fi 143 143 + 150 144 if [ ! -z $ETH ]; then 151 145 VETH0=${ETH} 152 146 VETH1=${ETH} ··· 179 153 ARGS+="-v " 180 154 fi 181 155 156 156 + if [ -n "$MODE" ]; then 157 157 + ARGS+="-m ${MODE} " 158 158 + fi 159 159 + 160 160 + if [ -n "$TEST" ]; then 161 161 + ARGS+="-t ${TEST} " 162 162 + fi 163 163 + 182 164 retval=$? 183 165 test_status $retval "${TEST_NAME}" 184 166 ··· 207 173 cleanup_exit ${VETH0} ${VETH1} 208 174 else 209 175 cleanup_iface ${ETH} ${MTU} 176 176 + fi 177 177 + 178 178 + if [[ $list -eq 1 ]]; then 179 179 + exit 210 180 fi 211 181 212 182 TEST_NAME="XSK_SELFTESTS_${VETH0}_BUSY_POLL"

+102 -40

tools/testing/selftests/bpf/trace_helpers.c

reviewed

··· 7 7 #include <errno.h> 8 8 #include <fcntl.h> 9 9 #include <poll.h> 10 10 + #include <pthread.h> 10 11 #include <unistd.h> 11 12 #include <linux/perf_event.h> 12 13 #include <sys/mman.h> ··· 15 14 #include <linux/limits.h> 16 15 #include <libelf.h> 17 16 #include <gelf.h> 17 17 + #include "bpf/libbpf_internal.h" 18 18 19 19 #define TRACEFS_PIPE "/sys/kernel/tracing/trace_pipe" 20 20 #define DEBUGFS_PIPE "/sys/kernel/debug/tracing/trace_pipe" 21 21 22 22 - #define MAX_SYMS 400000 23 23 - static struct ksym syms[MAX_SYMS]; 24 24 - static int sym_cnt; 22 22 + struct ksyms { 23 23 + struct ksym *syms; 24 24 + size_t sym_cap; 25 25 + size_t sym_cnt; 26 26 + }; 27 27 + 28 28 + static struct ksyms *ksyms; 29 29 + static pthread_mutex_t ksyms_mutex = PTHREAD_MUTEX_INITIALIZER; 30 30 + 31 31 + static int ksyms__add_symbol(struct ksyms *ksyms, const char *name, 32 32 + unsigned long addr) 33 33 + { 34 34 + void *tmp; 35 35 + 36 36 + tmp = strdup(name); 37 37 + if (!tmp) 38 38 + return -ENOMEM; 39 39 + ksyms->syms[ksyms->sym_cnt].addr = addr; 40 40 + ksyms->syms[ksyms->sym_cnt].name = tmp; 41 41 + ksyms->sym_cnt++; 42 42 + return 0; 43 43 + } 44 44 + 45 45 + void free_kallsyms_local(struct ksyms *ksyms) 46 46 + { 47 47 + unsigned int i; 48 48 + 49 49 + if (!ksyms) 50 50 + return; 51 51 + 52 52 + if (!ksyms->syms) { 53 53 + free(ksyms); 54 54 + return; 55 55 + } 56 56 + 57 57 + for (i = 0; i < ksyms->sym_cnt; i++) 58 58 + free(ksyms->syms[i].name); 59 59 + free(ksyms->syms); 60 60 + free(ksyms); 61 61 + } 25 62 26 63 static int ksym_cmp(const void *p1, const void *p2) 27 64 { 28 65 return ((struct ksym *)p1)->addr - ((struct ksym *)p2)->addr; 29 66 } 30 67 31 31 - int load_kallsyms_refresh(void) 68 68 + struct ksyms *load_kallsyms_local(void) 32 69 { 33 70 FILE *f; 34 71 char func[256], buf[256]; 35 72 char symbol; 36 73 void *addr; 37 37 - int i = 0; 38 38 - 39 39 - sym_cnt = 0; 74 74 + int ret; 75 75 + struct ksyms *ksyms; 40 76 41 77 f = fopen("/proc/kallsyms", "r"); 42 78 if (!f) 43 43 - return -ENOENT; 79 79 + return NULL; 80 80 + 81 81 + ksyms = calloc(1, sizeof(struct ksyms)); 82 82 + if (!ksyms) { 83 83 + fclose(f); 84 84 + return NULL; 85 85 + } 44 86 45 87 while (fgets(buf, sizeof(buf), f)) { 46 88 if (sscanf(buf, "%p %c %s", &addr, &symbol, func) != 3) 47 89 break; 48 90 if (!addr) 49 91 continue; 50 50 - if (i >= MAX_SYMS) 51 51 - return -EFBIG; 52 92 53 53 - syms[i].addr = (long) addr; 54 54 - syms[i].name = strdup(func); 55 55 - i++; 93 93 + ret = libbpf_ensure_mem((void **) &ksyms->syms, &ksyms->sym_cap, 94 94 + sizeof(struct ksym), ksyms->sym_cnt + 1); 95 95 + if (ret) 96 96 + goto error; 97 97 + ret = ksyms__add_symbol(ksyms, func, (unsigned long)addr); 98 98 + if (ret) 99 99 + goto error; 56 100 } 57 101 fclose(f); 58 58 - sym_cnt = i; 59 59 - qsort(syms, sym_cnt, sizeof(struct ksym), ksym_cmp); 60 60 - return 0; 102 102 + qsort(ksyms->syms, ksyms->sym_cnt, sizeof(struct ksym), ksym_cmp); 103 103 + return ksyms; 104 104 + 105 105 + error: 106 106 + fclose(f); 107 107 + free_kallsyms_local(ksyms); 108 108 + return NULL; 61 109 } 62 110 63 111 int load_kallsyms(void) 64 112 { 65 65 - /* 66 66 - * This is called/used from multiplace places, 67 67 - * load symbols just once. 68 68 - */ 69 69 - if (sym_cnt) 70 70 - return 0; 71 71 - return load_kallsyms_refresh(); 113 113 + pthread_mutex_lock(&ksyms_mutex); 114 114 + if (!ksyms) 115 115 + ksyms = load_kallsyms_local(); 116 116 + pthread_mutex_unlock(&ksyms_mutex); 117 117 + return ksyms ? 0 : 1; 72 118 } 73 119 74 74 - struct ksym *ksym_search(long key) 120 120 + struct ksym *ksym_search_local(struct ksyms *ksyms, long key) 75 121 { 76 76 - int start = 0, end = sym_cnt; 122 122 + int start = 0, end = ksyms->sym_cnt; 77 123 int result; 78 124 79 125 /* kallsyms not loaded. return NULL */ 80 80 - if (sym_cnt <= 0) 126 126 + if (ksyms->sym_cnt <= 0) 81 127 return NULL; 82 128 83 129 while (start < end) { 84 130 size_t mid = start + (end - start) / 2; 85 131 86 86 - result = key - syms[mid].addr; 132 132 + result = key - ksyms->syms[mid].addr; 87 133 if (result < 0) 88 134 end = mid; 89 135 else if (result > 0) 90 136 start = mid + 1; 91 137 else 92 92 - return &syms[mid]; 138 138 + return &ksyms->syms[mid]; 93 139 } 94 140 95 95 - if (start >= 1 && syms[start - 1].addr < key && 96 96 - key < syms[start].addr) 141 141 + if (start >= 1 && ksyms->syms[start - 1].addr < key && 142 142 + key < ksyms->syms[start].addr) 97 143 /* valid ksym */ 98 98 - return &syms[start - 1]; 144 144 + return &ksyms->syms[start - 1]; 99 145 100 146 /* out of range. return _stext */ 101 101 - return &syms[0]; 147 147 + return &ksyms->syms[0]; 148 148 + } 149 149 + 150 150 + struct ksym *ksym_search(long key) 151 151 + { 152 152 + if (!ksyms) 153 153 + return NULL; 154 154 + return ksym_search_local(ksyms, key); 155 155 + } 156 156 + 157 157 + long ksym_get_addr_local(struct ksyms *ksyms, const char *name) 158 158 + { 159 159 + int i; 160 160 + 161 161 + for (i = 0; i < ksyms->sym_cnt; i++) { 162 162 + if (strcmp(ksyms->syms[i].name, name) == 0) 163 163 + return ksyms->syms[i].addr; 164 164 + } 165 165 + 166 166 + return 0; 102 167 } 103 168 104 169 long ksym_get_addr(const char *name) 105 170 { 106 106 - int i; 107 107 - 108 108 - for (i = 0; i < sym_cnt; i++) { 109 109 - if (strcmp(syms[i].name, name) == 0) 110 110 - return syms[i].addr; 111 111 - } 112 112 - 113 113 - return 0; 171 171 + if (!ksyms) 172 172 + return 0; 173 173 + return ksym_get_addr_local(ksyms, name); 114 174 } 115 175 116 176 /* open kallsyms and read symbol addresses on the fly. Without caching all symbols,

+6 -2

tools/testing/selftests/bpf/trace_helpers.h

reviewed

··· 11 11 long addr; 12 12 char *name; 13 13 }; 14 14 + struct ksyms; 14 15 15 16 int load_kallsyms(void); 16 16 - int load_kallsyms_refresh(void); 17 17 - 18 17 struct ksym *ksym_search(long key); 19 18 long ksym_get_addr(const char *name); 19 19 + 20 20 + struct ksyms *load_kallsyms_local(void); 21 21 + struct ksym *ksym_search_local(struct ksyms *ksyms, long key); 22 22 + long ksym_get_addr_local(struct ksyms *ksyms, const char *name); 23 23 + void free_kallsyms_local(struct ksyms *ksyms); 20 24 21 25 /* open kallsyms and find addresses on the fly, faster than load + search. */ 22 26 int kallsyms_find(const char *sym, unsigned long long *addr);

+6 -4

tools/testing/selftests/bpf/xsk_prereqs.sh

reviewed

··· 83 83 fi 84 84 85 85 ./${XSKOBJ} -i ${VETH0} -i ${VETH1} ${ARGS} 86 86 - 87 86 retval=$? 88 88 - test_status $retval "${TEST_NAME}" 89 89 - statusList+=($retval) 90 90 - nameList+=(${TEST_NAME}) 87 87 + 88 88 + if [[ $list -ne 1 ]]; then 89 89 + test_status $retval "${TEST_NAME}" 90 90 + statusList+=($retval) 91 91 + nameList+=(${TEST_NAME}) 92 92 + fi 91 93 }

+312 -217

tools/testing/selftests/bpf/xskxceiver.c

reviewed

··· 107 107 static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62"; 108 108 static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61"; 109 109 110 110 + static bool opt_verbose; 111 111 + static bool opt_print_tests; 112 112 + static enum test_mode opt_mode = TEST_MODE_ALL; 113 113 + static u32 opt_run_test = RUN_ALL_TESTS; 114 114 + 110 115 static void __exit_with_error(int error, const char *file, const char *func, int line) 111 116 { 112 117 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, ··· 315 310 {"interface", required_argument, 0, 'i'}, 316 311 {"busy-poll", no_argument, 0, 'b'}, 317 312 {"verbose", no_argument, 0, 'v'}, 313 313 + {"mode", required_argument, 0, 'm'}, 314 314 + {"list", no_argument, 0, 'l'}, 315 315 + {"test", required_argument, 0, 't'}, 316 316 + {"help", no_argument, 0, 'h'}, 318 317 {0, 0, 0, 0} 319 318 }; 320 319 321 321 - static void usage(const char *prog) 320 320 + static void print_usage(char **argv) 322 321 { 323 322 const char *str = 324 324 - " Usage: %s [OPTIONS]\n" 323 323 + " Usage: xskxceiver [OPTIONS]\n" 325 324 " Options:\n" 326 325 " -i, --interface Use interface\n" 327 326 " -v, --verbose Verbose output\n" 328 328 - " -b, --busy-poll Enable busy poll\n"; 327 327 + " -b, --busy-poll Enable busy poll\n" 328 328 + " -m, --mode Run only mode skb, drv, or zc\n" 329 329 + " -l, --list List all available tests\n" 330 330 + " -t, --test Run a specific test. Enter number from -l option.\n" 331 331 + " -h, --help Display this help and exit\n"; 329 332 330 330 - ksft_print_msg(str, prog); 333 333 + ksft_print_msg(str, basename(argv[0])); 334 334 + ksft_exit_xfail(); 331 335 } 332 336 333 337 static bool validate_interface(struct ifobject *ifobj) ··· 356 342 opterr = 0; 357 343 358 344 for (;;) { 359 359 - c = getopt_long(argc, argv, "i:vb", long_options, &option_index); 345 345 + c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index); 360 346 if (c == -1) 361 347 break; 362 348 ··· 385 371 ifobj_tx->busy_poll = true; 386 372 ifobj_rx->busy_poll = true; 387 373 break; 374 374 + case 'm': 375 375 + if (!strncmp("skb", optarg, strlen(optarg))) 376 376 + opt_mode = TEST_MODE_SKB; 377 377 + else if (!strncmp("drv", optarg, strlen(optarg))) 378 378 + opt_mode = TEST_MODE_DRV; 379 379 + else if (!strncmp("zc", optarg, strlen(optarg))) 380 380 + opt_mode = TEST_MODE_ZC; 381 381 + else 382 382 + print_usage(argv); 383 383 + break; 384 384 + case 'l': 385 385 + opt_print_tests = true; 386 386 + break; 387 387 + case 't': 388 388 + errno = 0; 389 389 + opt_run_test = strtol(optarg, NULL, 0); 390 390 + if (errno) 391 391 + print_usage(argv); 392 392 + break; 393 393 + case 'h': 388 394 default: 389 389 - usage(basename(argv[0])); 390 390 - ksft_exit_xfail(); 395 395 + print_usage(argv); 391 396 } 392 397 } 393 398 } ··· 460 427 } 461 428 462 429 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, 463 463 - struct ifobject *ifobj_rx, enum test_mode mode) 430 430 + struct ifobject *ifobj_rx, enum test_mode mode, 431 431 + const struct test_spec *test_to_run) 464 432 { 465 433 struct pkt_stream *tx_pkt_stream; 466 434 struct pkt_stream *rx_pkt_stream; ··· 483 449 ifobj->bind_flags |= XDP_COPY; 484 450 } 485 451 452 452 + strncpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE); 453 453 + test->test_func = test_to_run->test_func; 486 454 test->mode = mode; 487 455 __test_spec_init(test, ifobj_tx, ifobj_rx); 488 456 } ··· 492 456 static void test_spec_reset(struct test_spec *test) 493 457 { 494 458 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); 495 495 - } 496 496 - 497 497 - static void test_spec_set_name(struct test_spec *test, const char *name) 498 498 - { 499 499 - strncpy(test->name, name, MAX_TEST_NAME_SIZE); 500 459 } 501 460 502 461 static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, ··· 778 747 len = 0; 779 748 } 780 749 750 750 + print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n", 751 751 + pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb); 752 752 + 781 753 if (pkt->valid && pkt->len > pkt_stream->max_pkt_len) 782 754 pkt_stream->max_pkt_len = pkt->len; 783 755 pkt_nb++; ··· 811 777 812 778 seqnum = ntohl(*data) & 0xffff; 813 779 pkt_nb = ntohl(*data) >> 16; 814 814 - fprintf(stdout, "%u:%u ", pkt_nb, seqnum); 780 780 + ksft_print_msg("%u:%u ", pkt_nb, seqnum); 815 781 data++; 816 782 } 817 783 } ··· 823 789 824 790 if (eth_header) { 825 791 /*extract L2 frame */ 826 826 - fprintf(stdout, "DEBUG>> L2: dst mac: "); 792 792 + ksft_print_msg("DEBUG>> L2: dst mac: "); 827 793 for (i = 0; i < ETH_ALEN; i++) 828 828 - fprintf(stdout, "%02X", ethhdr->h_dest[i]); 794 794 + ksft_print_msg("%02X", ethhdr->h_dest[i]); 829 795 830 830 - fprintf(stdout, "\nDEBUG>> L2: src mac: "); 796 796 + ksft_print_msg("\nDEBUG>> L2: src mac: "); 831 797 for (i = 0; i < ETH_ALEN; i++) 832 832 - fprintf(stdout, "%02X", ethhdr->h_source[i]); 798 798 + ksft_print_msg("%02X", ethhdr->h_source[i]); 833 799 834 800 data = pkt + PKT_HDR_SIZE; 835 801 } else { ··· 837 803 } 838 804 839 805 /*extract L5 frame */ 840 840 - fprintf(stdout, "\nDEBUG>> L5: seqnum: "); 806 806 + ksft_print_msg("\nDEBUG>> L5: seqnum: "); 841 807 pkt_print_data(data, PKT_DUMP_NB_TO_PRINT); 842 842 - fprintf(stdout, "...."); 808 808 + ksft_print_msg("...."); 843 809 if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) { 844 844 - fprintf(stdout, "\n.... "); 810 810 + ksft_print_msg("\n.... "); 845 811 pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT, 846 812 PKT_DUMP_NB_TO_PRINT); 847 813 } 848 848 - fprintf(stdout, "\n---------------------------------------\n"); 814 814 + ksft_print_msg("\n---------------------------------------\n"); 849 815 } 850 816 851 817 static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr) ··· 950 916 return true; 951 917 } 952 918 953 953 - static void kick_tx(struct xsk_socket_info *xsk) 919 919 + static int kick_tx(struct xsk_socket_info *xsk) 954 920 { 955 921 int ret; 956 922 957 923 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); 958 924 if (ret >= 0) 959 959 - return; 925 925 + return TEST_PASS; 960 926 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { 961 927 usleep(100); 962 962 - return; 928 928 + return TEST_PASS; 963 929 } 964 964 - exit_with_error(errno); 930 930 + return TEST_FAILURE; 965 931 } 966 932 967 967 - static void kick_rx(struct xsk_socket_info *xsk) 933 933 + static int kick_rx(struct xsk_socket_info *xsk) 968 934 { 969 935 int ret; 970 936 971 937 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); 972 938 if (ret < 0) 973 973 - exit_with_error(errno); 939 939 + return TEST_FAILURE; 940 940 + 941 941 + return TEST_PASS; 974 942 } 975 943 976 944 static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) 977 945 { 978 946 unsigned int rcvd; 979 947 u32 idx; 948 948 + int ret; 980 949 981 981 - if (xsk_ring_prod__needs_wakeup(&xsk->tx)) 982 982 - kick_tx(xsk); 950 950 + if (xsk_ring_prod__needs_wakeup(&xsk->tx)) { 951 951 + ret = kick_tx(xsk); 952 952 + if (ret) 953 953 + return TEST_FAILURE; 954 954 + } 983 955 984 956 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); 985 957 if (rcvd) { ··· 1033 993 return TEST_FAILURE; 1034 994 } 1035 995 1036 1036 - kick_rx(xsk); 996 996 + ret = kick_rx(xsk); 997 997 + if (ret) 998 998 + return TEST_FAILURE; 999 999 + 1037 1000 if (ifobj->use_poll) { 1038 1001 ret = poll(fds, 1, POLL_TMOUT); 1039 1002 if (ret < 0) 1040 1040 - exit_with_error(errno); 1003 1003 + return TEST_FAILURE; 1041 1004 1042 1005 if (!ret) { 1043 1006 if (!is_umem_valid(test->ifobj_tx)) ··· 1061 1018 if (ifobj->use_fill_ring) { 1062 1019 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 1063 1020 while (ret != rcvd) { 1064 1064 - if (ret < 0) 1065 1065 - exit_with_error(-ret); 1066 1021 if (xsk_ring_prod__needs_wakeup(&umem->fq)) { 1067 1022 ret = poll(fds, 1, POLL_TMOUT); 1068 1023 if (ret < 0) 1069 1069 - exit_with_error(errno); 1024 1024 + return TEST_FAILURE; 1070 1025 } 1071 1026 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 1072 1027 } ··· 1082 1041 __func__, addr, desc->len); 1083 1042 return TEST_FAILURE; 1084 1043 } 1044 1044 + 1045 1045 + print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n", 1046 1046 + addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid); 1085 1047 1086 1048 if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) || 1087 1049 !is_offset_correct(umem, pkt, addr) || ··· 1148 1104 buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); 1149 1105 /* pkts_in_flight might be negative if many invalid packets are sent */ 1150 1106 if (pkts_in_flight >= (int)((umem_size(umem) - BATCH_SIZE * buffer_len) / buffer_len)) { 1151 1151 - kick_tx(xsk); 1107 1107 + ret = kick_tx(xsk); 1108 1108 + if (ret) 1109 1109 + return TEST_FAILURE; 1152 1110 return TEST_CONTINUE; 1153 1111 } 1154 1112 ··· 1211 1165 bytes_written); 1212 1166 bytes_written += tx_desc->len; 1213 1167 1168 1168 + print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n", 1169 1169 + tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb); 1170 1170 + 1214 1171 if (nb_frags_left) { 1215 1172 i++; 1216 1173 if (pkt_stream->verbatim) ··· 1256 1207 return TEST_CONTINUE; 1257 1208 } 1258 1209 1259 1259 - static void wait_for_tx_completion(struct xsk_socket_info *xsk) 1210 1210 + static int wait_for_tx_completion(struct xsk_socket_info *xsk) 1260 1211 { 1261 1261 - while (xsk->outstanding_tx) 1212 1212 + struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; 1213 1213 + int ret; 1214 1214 + 1215 1215 + ret = gettimeofday(&tv_now, NULL); 1216 1216 + if (ret) 1217 1217 + exit_with_error(errno); 1218 1218 + timeradd(&tv_now, &tv_timeout, &tv_end); 1219 1219 + 1220 1220 + while (xsk->outstanding_tx) { 1221 1221 + ret = gettimeofday(&tv_now, NULL); 1222 1222 + if (ret) 1223 1223 + exit_with_error(errno); 1224 1224 + if (timercmp(&tv_now, &tv_end, >)) { 1225 1225 + ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__); 1226 1226 + return TEST_FAILURE; 1227 1227 + } 1228 1228 + 1262 1229 complete_pkts(xsk, BATCH_SIZE); 1230 1230 + } 1231 1231 + 1232 1232 + return TEST_PASS; 1263 1233 } 1264 1234 1265 1235 static int send_pkts(struct test_spec *test, struct ifobject *ifobject) ··· 1301 1233 return ret; 1302 1234 } 1303 1235 1304 1304 - wait_for_tx_completion(ifobject->xsk); 1305 1305 - return TEST_PASS; 1236 1236 + return wait_for_tx_completion(ifobject->xsk); 1306 1237 } 1307 1238 1308 1239 static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) ··· 1333 1266 struct xdp_statistics stats; 1334 1267 int err; 1335 1268 1336 1336 - kick_rx(ifobject->xsk); 1269 1269 + err = kick_rx(ifobject->xsk); 1270 1270 + if (err) 1271 1271 + return TEST_FAILURE; 1337 1272 1338 1273 err = get_xsk_stats(xsk, &stats); 1339 1274 if (err) ··· 1361 1292 int err; 1362 1293 1363 1294 usleep(1000); 1364 1364 - kick_rx(ifobject->xsk); 1295 1295 + err = kick_rx(ifobject->xsk); 1296 1296 + if (err) 1297 1297 + return TEST_FAILURE; 1365 1298 1366 1299 err = get_xsk_stats(xsk, &stats); 1367 1300 if (err) ··· 1382 1311 int err; 1383 1312 1384 1313 usleep(1000); 1385 1385 - kick_rx(ifobject->xsk); 1314 1314 + err = kick_rx(ifobject->xsk); 1315 1315 + if (err) 1316 1316 + return TEST_FAILURE; 1386 1317 1387 1318 err = get_xsk_stats(xsk, &stats); 1388 1319 if (err) ··· 1548 1475 thread_common_ops_tx(test, ifobject); 1549 1476 } 1550 1477 1551 1551 - print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts, 1552 1552 - ifobject->ifname); 1553 1478 err = send_pkts(test, ifobject); 1554 1479 1555 1480 if (!err && ifobject->validation_func) ··· 1571 1500 xsk_clear_xskmap(ifobject->xskmap); 1572 1501 err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk); 1573 1502 if (err) { 1574 1574 - printf("Error: Failed to update xskmap, error %s\n", strerror(-err)); 1503 1503 + ksft_print_msg("Error: Failed to update xskmap, error %s\n", 1504 1504 + strerror(-err)); 1575 1505 exit_with_error(-err); 1576 1506 } 1577 1507 } ··· 1636 1564 xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode)); 1637 1565 err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode)); 1638 1566 if (err) { 1639 1639 - printf("Error attaching XDP program\n"); 1567 1567 + ksft_print_msg("Error attaching XDP program\n"); 1640 1568 exit_with_error(-err); 1641 1569 } 1642 1570 ··· 1754 1682 { 1755 1683 int i; 1756 1684 1757 1757 - test_spec_set_name(test, "TEARDOWN"); 1758 1685 for (i = 0; i < MAX_TEARDOWN_ITER; i++) { 1759 1686 if (testapp_validate_traffic(test)) 1760 1687 return TEST_FAILURE; ··· 1775 1704 *ifobj2 = tmp_ifobj; 1776 1705 } 1777 1706 1778 1778 - static int testapp_bidi(struct test_spec *test) 1707 1707 + static int testapp_bidirectional(struct test_spec *test) 1779 1708 { 1780 1709 int res; 1781 1710 1782 1782 - test_spec_set_name(test, "BIDIRECTIONAL"); 1783 1711 test->ifobj_tx->rx_on = true; 1784 1712 test->ifobj_rx->tx_on = true; 1785 1713 test->total_steps = 2; 1786 1714 if (testapp_validate_traffic(test)) 1787 1715 return TEST_FAILURE; 1788 1716 1789 1789 - print_verbose("Switching Tx/Rx vectors\n"); 1717 1717 + print_verbose("Switching Tx/Rx direction\n"); 1790 1718 swap_directions(&test->ifobj_rx, &test->ifobj_tx); 1791 1719 res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx); 1792 1720 ··· 1793 1723 return res; 1794 1724 } 1795 1725 1796 1796 - static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx) 1726 1726 + static int swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx) 1797 1727 { 1798 1728 int ret; 1799 1729 ··· 1804 1734 1805 1735 ret = xsk_update_xskmap(ifobj_rx->xskmap, ifobj_rx->xsk->xsk); 1806 1736 if (ret) 1807 1807 - exit_with_error(errno); 1737 1737 + return TEST_FAILURE; 1738 1738 + 1739 1739 + return TEST_PASS; 1808 1740 } 1809 1741 1810 1810 - static int testapp_bpf_res(struct test_spec *test) 1742 1742 + static int testapp_xdp_prog_cleanup(struct test_spec *test) 1811 1743 { 1812 1812 - test_spec_set_name(test, "BPF_RES"); 1813 1744 test->total_steps = 2; 1814 1745 test->nb_sockets = 2; 1815 1746 if (testapp_validate_traffic(test)) 1816 1747 return TEST_FAILURE; 1817 1748 1818 1818 - swap_xsk_resources(test->ifobj_tx, test->ifobj_rx); 1749 1749 + if (swap_xsk_resources(test->ifobj_tx, test->ifobj_rx)) 1750 1750 + return TEST_FAILURE; 1819 1751 return testapp_validate_traffic(test); 1820 1752 } 1821 1753 1822 1754 static int testapp_headroom(struct test_spec *test) 1823 1755 { 1824 1824 - test_spec_set_name(test, "UMEM_HEADROOM"); 1825 1756 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; 1826 1757 return testapp_validate_traffic(test); 1827 1758 } 1828 1759 1829 1760 static int testapp_stats_rx_dropped(struct test_spec *test) 1830 1761 { 1831 1831 - test_spec_set_name(test, "STAT_RX_DROPPED"); 1832 1762 if (test->mode == TEST_MODE_ZC) { 1833 1763 ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n"); 1834 1764 return TEST_SKIP; ··· 1844 1774 1845 1775 static int testapp_stats_tx_invalid_descs(struct test_spec *test) 1846 1776 { 1847 1847 - test_spec_set_name(test, "STAT_TX_INVALID"); 1848 1777 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0); 1849 1778 test->ifobj_tx->validation_func = validate_tx_invalid_descs; 1850 1779 return testapp_validate_traffic(test); ··· 1851 1782 1852 1783 static int testapp_stats_rx_full(struct test_spec *test) 1853 1784 { 1854 1854 - test_spec_set_name(test, "STAT_RX_FULL"); 1855 1785 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); 1856 1786 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, 1857 1787 DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); ··· 1863 1795 1864 1796 static int testapp_stats_fill_empty(struct test_spec *test) 1865 1797 { 1866 1866 - test_spec_set_name(test, "STAT_RX_FILL_EMPTY"); 1867 1798 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); 1868 1799 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, 1869 1800 DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); ··· 1872 1805 return testapp_validate_traffic(test); 1873 1806 } 1874 1807 1875 1875 - static int testapp_unaligned(struct test_spec *test) 1808 1808 + static int testapp_send_receive_unaligned(struct test_spec *test) 1876 1809 { 1877 1877 - test_spec_set_name(test, "UNALIGNED_MODE"); 1878 1810 test->ifobj_tx->umem->unaligned_mode = true; 1879 1811 test->ifobj_rx->umem->unaligned_mode = true; 1880 1812 /* Let half of the packets straddle a 4K buffer boundary */ ··· 1882 1816 return testapp_validate_traffic(test); 1883 1817 } 1884 1818 1885 1885 - static int testapp_unaligned_mb(struct test_spec *test) 1819 1819 + static int testapp_send_receive_unaligned_mb(struct test_spec *test) 1886 1820 { 1887 1887 - test_spec_set_name(test, "UNALIGNED_MODE_9K"); 1888 1821 test->mtu = MAX_ETH_JUMBO_SIZE; 1889 1822 test->ifobj_tx->umem->unaligned_mode = true; 1890 1823 test->ifobj_rx->umem->unaligned_mode = true; ··· 1899 1834 return testapp_validate_traffic(test); 1900 1835 } 1901 1836 1902 1902 - static int testapp_multi_buffer(struct test_spec *test) 1837 1837 + static int testapp_send_receive_mb(struct test_spec *test) 1903 1838 { 1904 1904 - test_spec_set_name(test, "RUN_TO_COMPLETION_9K_PACKETS"); 1905 1839 test->mtu = MAX_ETH_JUMBO_SIZE; 1906 1840 pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); 1907 1841 ··· 1997 1933 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; 1998 1934 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; 1999 1935 2000 2000 - test_spec_set_name(test, "XDP_DROP_HALF"); 2001 1936 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop, 2002 1937 skel_rx->maps.xsk, skel_tx->maps.xsk); 2003 1938 ··· 2004 1941 return testapp_validate_traffic(test); 2005 1942 } 2006 1943 2007 2007 - static int testapp_xdp_metadata_count(struct test_spec *test) 1944 1944 + static int testapp_xdp_metadata_copy(struct test_spec *test) 2008 1945 { 2009 1946 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; 2010 1947 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; ··· 2018 1955 test->ifobj_rx->use_metadata = true; 2019 1956 2020 1957 data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss"); 2021 2021 - if (!data_map || !bpf_map__is_internal(data_map)) 2022 2022 - exit_with_error(ENOMEM); 1958 1958 + if (!data_map || !bpf_map__is_internal(data_map)) { 1959 1959 + ksft_print_msg("Error: could not find bss section of XDP program\n"); 1960 1960 + return TEST_FAILURE; 1961 1961 + } 2023 1962 2024 2024 - if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY)) 2025 2025 - exit_with_error(errno); 1963 1963 + if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY)) { 1964 1964 + ksft_print_msg("Error: could not update count element\n"); 1965 1965 + return TEST_FAILURE; 1966 1966 + } 2026 1967 2027 1968 return testapp_validate_traffic(test); 2028 1969 } 2029 1970 2030 1971 static int testapp_poll_txq_tmout(struct test_spec *test) 2031 1972 { 2032 2032 - test_spec_set_name(test, "POLL_TXQ_FULL"); 2033 2033 - 2034 1973 test->ifobj_tx->use_poll = true; 2035 1974 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ 2036 1975 test->ifobj_tx->umem->frame_size = 2048; ··· 2042 1977 2043 1978 static int testapp_poll_rxq_tmout(struct test_spec *test) 2044 1979 { 2045 2045 - test_spec_set_name(test, "POLL_RXQ_EMPTY"); 2046 1980 test->ifobj_rx->use_poll = true; 2047 1981 return testapp_validate_traffic_single_thread(test, test->ifobj_rx); 2048 1982 } ··· 2051 1987 struct pkt pkts[2 * XSK_DESC__MAX_SKB_FRAGS + 2] = {}; 2052 1988 u32 max_frags, i; 2053 1989 2054 2054 - test_spec_set_name(test, "TOO_MANY_FRAGS"); 2055 1990 if (test->mode == TEST_MODE_ZC) 2056 1991 max_frags = test->ifobj_tx->xdp_zc_max_segs; 2057 1992 else ··· 2130 2067 2131 2068 err = xsk_load_xdp_programs(ifobj); 2132 2069 if (err) { 2133 2133 - printf("Error loading XDP program\n"); 2070 2070 + ksft_print_msg("Error loading XDP program\n"); 2134 2071 exit_with_error(err); 2135 2072 } 2136 2073 ··· 2154 2091 } 2155 2092 } 2156 2093 2157 2157 - static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type) 2094 2094 + static int testapp_send_receive(struct test_spec *test) 2158 2095 { 2159 2159 - int ret = TEST_SKIP; 2096 2096 + return testapp_validate_traffic(test); 2097 2097 + } 2160 2098 2161 2161 - switch (type) { 2162 2162 - case TEST_TYPE_STATS_RX_DROPPED: 2163 2163 - ret = testapp_stats_rx_dropped(test); 2164 2164 - break; 2165 2165 - case TEST_TYPE_STATS_TX_INVALID_DESCS: 2166 2166 - ret = testapp_stats_tx_invalid_descs(test); 2167 2167 - break; 2168 2168 - case TEST_TYPE_STATS_RX_FULL: 2169 2169 - ret = testapp_stats_rx_full(test); 2170 2170 - break; 2171 2171 - case TEST_TYPE_STATS_FILL_EMPTY: 2172 2172 - ret = testapp_stats_fill_empty(test); 2173 2173 - break; 2174 2174 - case TEST_TYPE_TEARDOWN: 2175 2175 - ret = testapp_teardown(test); 2176 2176 - break; 2177 2177 - case TEST_TYPE_BIDI: 2178 2178 - ret = testapp_bidi(test); 2179 2179 - break; 2180 2180 - case TEST_TYPE_BPF_RES: 2181 2181 - ret = testapp_bpf_res(test); 2182 2182 - break; 2183 2183 - case TEST_TYPE_RUN_TO_COMPLETION: 2184 2184 - test_spec_set_name(test, "RUN_TO_COMPLETION"); 2185 2185 - ret = testapp_validate_traffic(test); 2186 2186 - break; 2187 2187 - case TEST_TYPE_RUN_TO_COMPLETION_MB: 2188 2188 - ret = testapp_multi_buffer(test); 2189 2189 - break; 2190 2190 - case TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT: 2191 2191 - test_spec_set_name(test, "RUN_TO_COMPLETION_SINGLE_PKT"); 2192 2192 - ret = testapp_single_pkt(test); 2193 2193 - break; 2194 2194 - case TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME: 2195 2195 - test_spec_set_name(test, "RUN_TO_COMPLETION_2K_FRAME_SIZE"); 2196 2196 - test->ifobj_tx->umem->frame_size = 2048; 2197 2197 - test->ifobj_rx->umem->frame_size = 2048; 2198 2198 - pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE); 2199 2199 - ret = testapp_validate_traffic(test); 2200 2200 - break; 2201 2201 - case TEST_TYPE_RX_POLL: 2202 2202 - test->ifobj_rx->use_poll = true; 2203 2203 - test_spec_set_name(test, "POLL_RX"); 2204 2204 - ret = testapp_validate_traffic(test); 2205 2205 - break; 2206 2206 - case TEST_TYPE_TX_POLL: 2207 2207 - test->ifobj_tx->use_poll = true; 2208 2208 - test_spec_set_name(test, "POLL_TX"); 2209 2209 - ret = testapp_validate_traffic(test); 2210 2210 - break; 2211 2211 - case TEST_TYPE_POLL_TXQ_TMOUT: 2212 2212 - ret = testapp_poll_txq_tmout(test); 2213 2213 - break; 2214 2214 - case TEST_TYPE_POLL_RXQ_TMOUT: 2215 2215 - ret = testapp_poll_rxq_tmout(test); 2216 2216 - break; 2217 2217 - case TEST_TYPE_ALIGNED_INV_DESC: 2218 2218 - test_spec_set_name(test, "ALIGNED_INV_DESC"); 2219 2219 - ret = testapp_invalid_desc(test); 2220 2220 - break; 2221 2221 - case TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME: 2222 2222 - test_spec_set_name(test, "ALIGNED_INV_DESC_2K_FRAME_SIZE"); 2223 2223 - test->ifobj_tx->umem->frame_size = 2048; 2224 2224 - test->ifobj_rx->umem->frame_size = 2048; 2225 2225 - ret = testapp_invalid_desc(test); 2226 2226 - break; 2227 2227 - case TEST_TYPE_UNALIGNED_INV_DESC: 2228 2228 - test_spec_set_name(test, "UNALIGNED_INV_DESC"); 2229 2229 - test->ifobj_tx->umem->unaligned_mode = true; 2230 2230 - test->ifobj_rx->umem->unaligned_mode = true; 2231 2231 - ret = testapp_invalid_desc(test); 2232 2232 - break; 2233 2233 - case TEST_TYPE_UNALIGNED_INV_DESC_4K1_FRAME: { 2234 2234 - u64 page_size, umem_size; 2099 2099 + static int testapp_send_receive_2k_frame(struct test_spec *test) 2100 2100 + { 2101 2101 + test->ifobj_tx->umem->frame_size = 2048; 2102 2102 + test->ifobj_rx->umem->frame_size = 2048; 2103 2103 + pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE); 2104 2104 + return testapp_validate_traffic(test); 2105 2105 + } 2235 2106 2236 2236 - test_spec_set_name(test, "UNALIGNED_INV_DESC_4K1_FRAME_SIZE"); 2237 2237 - /* Odd frame size so the UMEM doesn't end near a page boundary. */ 2238 2238 - test->ifobj_tx->umem->frame_size = 4001; 2239 2239 - test->ifobj_rx->umem->frame_size = 4001; 2240 2240 - test->ifobj_tx->umem->unaligned_mode = true; 2241 2241 - test->ifobj_rx->umem->unaligned_mode = true; 2242 2242 - /* This test exists to test descriptors that staddle the end of 2243 2243 - * the UMEM but not a page. 2244 2244 - */ 2245 2245 - page_size = sysconf(_SC_PAGESIZE); 2246 2246 - umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; 2247 2247 - assert(umem_size % page_size > MIN_PKT_SIZE); 2248 2248 - assert(umem_size % page_size < page_size - MIN_PKT_SIZE); 2249 2249 - ret = testapp_invalid_desc(test); 2250 2250 - break; 2251 2251 - } 2252 2252 - case TEST_TYPE_ALIGNED_INV_DESC_MB: 2253 2253 - test_spec_set_name(test, "ALIGNED_INV_DESC_MULTI_BUFF"); 2254 2254 - ret = testapp_invalid_desc_mb(test); 2255 2255 - break; 2256 2256 - case TEST_TYPE_UNALIGNED_INV_DESC_MB: 2257 2257 - test_spec_set_name(test, "UNALIGNED_INV_DESC_MULTI_BUFF"); 2258 2258 - test->ifobj_tx->umem->unaligned_mode = true; 2259 2259 - test->ifobj_rx->umem->unaligned_mode = true; 2260 2260 - ret = testapp_invalid_desc_mb(test); 2261 2261 - break; 2262 2262 - case TEST_TYPE_UNALIGNED: 2263 2263 - ret = testapp_unaligned(test); 2264 2264 - break; 2265 2265 - case TEST_TYPE_UNALIGNED_MB: 2266 2266 - ret = testapp_unaligned_mb(test); 2267 2267 - break; 2268 2268 - case TEST_TYPE_HEADROOM: 2269 2269 - ret = testapp_headroom(test); 2270 2270 - break; 2271 2271 - case TEST_TYPE_XDP_DROP_HALF: 2272 2272 - ret = testapp_xdp_drop(test); 2273 2273 - break; 2274 2274 - case TEST_TYPE_XDP_METADATA_COUNT: 2275 2275 - test_spec_set_name(test, "XDP_METADATA_COUNT"); 2276 2276 - ret = testapp_xdp_metadata_count(test); 2277 2277 - break; 2278 2278 - case TEST_TYPE_XDP_METADATA_COUNT_MB: 2279 2279 - test_spec_set_name(test, "XDP_METADATA_COUNT_MULTI_BUFF"); 2280 2280 - test->mtu = MAX_ETH_JUMBO_SIZE; 2281 2281 - ret = testapp_xdp_metadata_count(test); 2282 2282 - break; 2283 2283 - case TEST_TYPE_TOO_MANY_FRAGS: 2284 2284 - ret = testapp_too_many_frags(test); 2285 2285 - break; 2286 2286 - default: 2287 2287 - break; 2288 2288 - } 2107 2107 + static int testapp_poll_rx(struct test_spec *test) 2108 2108 + { 2109 2109 + test->ifobj_rx->use_poll = true; 2110 2110 + return testapp_validate_traffic(test); 2111 2111 + } 2112 2112 + 2113 2113 + static int testapp_poll_tx(struct test_spec *test) 2114 2114 + { 2115 2115 + test->ifobj_tx->use_poll = true; 2116 2116 + return testapp_validate_traffic(test); 2117 2117 + } 2118 2118 + 2119 2119 + static int testapp_aligned_inv_desc(struct test_spec *test) 2120 2120 + { 2121 2121 + return testapp_invalid_desc(test); 2122 2122 + } 2123 2123 + 2124 2124 + static int testapp_aligned_inv_desc_2k_frame(struct test_spec *test) 2125 2125 + { 2126 2126 + test->ifobj_tx->umem->frame_size = 2048; 2127 2127 + test->ifobj_rx->umem->frame_size = 2048; 2128 2128 + return testapp_invalid_desc(test); 2129 2129 + } 2130 2130 + 2131 2131 + static int testapp_unaligned_inv_desc(struct test_spec *test) 2132 2132 + { 2133 2133 + test->ifobj_tx->umem->unaligned_mode = true; 2134 2134 + test->ifobj_rx->umem->unaligned_mode = true; 2135 2135 + return testapp_invalid_desc(test); 2136 2136 + } 2137 2137 + 2138 2138 + static int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test) 2139 2139 + { 2140 2140 + u64 page_size, umem_size; 2141 2141 + 2142 2142 + /* Odd frame size so the UMEM doesn't end near a page boundary. */ 2143 2143 + test->ifobj_tx->umem->frame_size = 4001; 2144 2144 + test->ifobj_rx->umem->frame_size = 4001; 2145 2145 + test->ifobj_tx->umem->unaligned_mode = true; 2146 2146 + test->ifobj_rx->umem->unaligned_mode = true; 2147 2147 + /* This test exists to test descriptors that staddle the end of 2148 2148 + * the UMEM but not a page. 2149 2149 + */ 2150 2150 + page_size = sysconf(_SC_PAGESIZE); 2151 2151 + umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; 2152 2152 + assert(umem_size % page_size > MIN_PKT_SIZE); 2153 2153 + assert(umem_size % page_size < page_size - MIN_PKT_SIZE); 2154 2154 + 2155 2155 + return testapp_invalid_desc(test); 2156 2156 + } 2157 2157 + 2158 2158 + static int testapp_aligned_inv_desc_mb(struct test_spec *test) 2159 2159 + { 2160 2160 + return testapp_invalid_desc_mb(test); 2161 2161 + } 2162 2162 + 2163 2163 + static int testapp_unaligned_inv_desc_mb(struct test_spec *test) 2164 2164 + { 2165 2165 + test->ifobj_tx->umem->unaligned_mode = true; 2166 2166 + test->ifobj_rx->umem->unaligned_mode = true; 2167 2167 + return testapp_invalid_desc_mb(test); 2168 2168 + } 2169 2169 + 2170 2170 + static int testapp_xdp_metadata(struct test_spec *test) 2171 2171 + { 2172 2172 + return testapp_xdp_metadata_copy(test); 2173 2173 + } 2174 2174 + 2175 2175 + static int testapp_xdp_metadata_mb(struct test_spec *test) 2176 2176 + { 2177 2177 + test->mtu = MAX_ETH_JUMBO_SIZE; 2178 2178 + return testapp_xdp_metadata_copy(test); 2179 2179 + } 2180 2180 + 2181 2181 + static void run_pkt_test(struct test_spec *test) 2182 2182 + { 2183 2183 + int ret; 2184 2184 + 2185 2185 + ret = test->test_func(test); 2289 2186 2290 2187 if (ret == TEST_PASS) 2291 2188 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), ··· 2313 2290 return true; 2314 2291 } 2315 2292 2293 2293 + static const struct test_spec tests[] = { 2294 2294 + {.name = "SEND_RECEIVE", .test_func = testapp_send_receive}, 2295 2295 + {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame}, 2296 2296 + {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt}, 2297 2297 + {.name = "POLL_RX", .test_func = testapp_poll_rx}, 2298 2298 + {.name = "POLL_TX", .test_func = testapp_poll_tx}, 2299 2299 + {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout}, 2300 2300 + {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout}, 2301 2301 + {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned}, 2302 2302 + {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc}, 2303 2303 + {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame}, 2304 2304 + {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc}, 2305 2305 + {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE", 2306 2306 + .test_func = testapp_unaligned_inv_desc_4001_frame}, 2307 2307 + {.name = "UMEM_HEADROOM", .test_func = testapp_headroom}, 2308 2308 + {.name = "TEARDOWN", .test_func = testapp_teardown}, 2309 2309 + {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional}, 2310 2310 + {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped}, 2311 2311 + {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs}, 2312 2312 + {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full}, 2313 2313 + {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty}, 2314 2314 + {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup}, 2315 2315 + {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop}, 2316 2316 + {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata}, 2317 2317 + {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb}, 2318 2318 + {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb}, 2319 2319 + {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS", 2320 2320 + .test_func = testapp_send_receive_unaligned_mb}, 2321 2321 + {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb}, 2322 2322 + {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb}, 2323 2323 + {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags}, 2324 2324 + }; 2325 2325 + 2326 2326 + static void print_tests(void) 2327 2327 + { 2328 2328 + u32 i; 2329 2329 + 2330 2330 + printf("Tests:\n"); 2331 2331 + for (i = 0; i < ARRAY_SIZE(tests); i++) 2332 2332 + printf("%u: %s\n", i, tests[i].name); 2333 2333 + } 2334 2334 + 2316 2335 int main(int argc, char **argv) 2317 2336 { 2318 2337 struct pkt_stream *rx_pkt_stream_default; 2319 2338 struct pkt_stream *tx_pkt_stream_default; 2320 2339 struct ifobject *ifobj_tx, *ifobj_rx; 2340 2340 + u32 i, j, failed_tests = 0, nb_tests; 2321 2341 int modes = TEST_MODE_SKB + 1; 2322 2322 - u32 i, j, failed_tests = 0; 2323 2342 struct test_spec test; 2324 2343 bool shared_netdev; 2325 2344 ··· 2379 2314 2380 2315 parse_command_line(ifobj_tx, ifobj_rx, argc, argv); 2381 2316 2317 2317 + if (opt_print_tests) { 2318 2318 + print_tests(); 2319 2319 + ksft_exit_xpass(); 2320 2320 + } 2321 2321 + if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= ARRAY_SIZE(tests)) { 2322 2322 + ksft_print_msg("Error: test %u does not exist.\n", opt_run_test); 2323 2323 + ksft_exit_xfail(); 2324 2324 + } 2325 2325 + 2382 2326 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex); 2383 2327 ifobj_tx->shared_umem = shared_netdev; 2384 2328 ifobj_rx->shared_umem = shared_netdev; 2385 2329 2386 2386 - if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) { 2387 2387 - usage(basename(argv[0])); 2388 2388 - ksft_exit_xfail(); 2389 2389 - } 2330 2330 + if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) 2331 2331 + print_usage(argv); 2390 2332 2391 2333 if (is_xdp_supported(ifobj_tx->ifindex)) { 2392 2334 modes++; ··· 2404 2332 init_iface(ifobj_rx, MAC1, MAC2, worker_testapp_validate_rx); 2405 2333 init_iface(ifobj_tx, MAC2, MAC1, worker_testapp_validate_tx); 2406 2334 2407 2407 - test_spec_init(&test, ifobj_tx, ifobj_rx, 0); 2335 2335 + test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]); 2408 2336 tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, MIN_PKT_SIZE); 2409 2337 rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, MIN_PKT_SIZE); 2410 2338 if (!tx_pkt_stream_default || !rx_pkt_stream_default) ··· 2412 2340 test.tx_pkt_stream_default = tx_pkt_stream_default; 2413 2341 test.rx_pkt_stream_default = rx_pkt_stream_default; 2414 2342 2415 2415 - ksft_set_plan(modes * TEST_TYPE_MAX); 2343 2343 + if (opt_run_test == RUN_ALL_TESTS) 2344 2344 + nb_tests = ARRAY_SIZE(tests); 2345 2345 + else 2346 2346 + nb_tests = 1; 2347 2347 + if (opt_mode == TEST_MODE_ALL) { 2348 2348 + ksft_set_plan(modes * nb_tests); 2349 2349 + } else { 2350 2350 + if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) { 2351 2351 + ksft_print_msg("Error: XDP_DRV mode not supported.\n"); 2352 2352 + ksft_exit_xfail(); 2353 2353 + } 2354 2354 + if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) { 2355 2355 + ksft_print_msg("Error: zero-copy mode not supported.\n"); 2356 2356 + ksft_exit_xfail(); 2357 2357 + } 2358 2358 + 2359 2359 + ksft_set_plan(nb_tests); 2360 2360 + } 2416 2361 2417 2362 for (i = 0; i < modes; i++) { 2418 2418 - for (j = 0; j < TEST_TYPE_MAX; j++) { 2419 2419 - test_spec_init(&test, ifobj_tx, ifobj_rx, i); 2420 2420 - run_pkt_test(&test, i, j); 2363 2363 + if (opt_mode != TEST_MODE_ALL && i != opt_mode) 2364 2364 + continue; 2365 2365 + 2366 2366 + for (j = 0; j < ARRAY_SIZE(tests); j++) { 2367 2367 + if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test) 2368 2368 + continue; 2369 2369 + 2370 2370 + test_spec_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]); 2371 2371 + run_pkt_test(&test); 2421 2372 usleep(USLEEP_MAX); 2422 2373 2423 2374 if (test.fail)

+8 -36

tools/testing/selftests/bpf/xskxceiver.h

reviewed

··· 5 5 #ifndef XSKXCEIVER_H_ 6 6 #define XSKXCEIVER_H_ 7 7 8 8 + #include <limits.h> 9 9 + 8 10 #include "xsk_xdp_progs.skel.h" 9 11 10 12 #ifndef SOL_XDP ··· 36 34 #define MAX_INTERFACES 2 37 35 #define MAX_INTERFACE_NAME_CHARS 16 38 36 #define MAX_SOCKETS 2 39 39 - #define MAX_TEST_NAME_SIZE 32 37 37 + #define MAX_TEST_NAME_SIZE 48 40 38 #define MAX_TEARDOWN_ITER 10 41 39 #define PKT_HDR_SIZE (sizeof(struct ethhdr) + 2) /* Just to align the data in the packet */ 42 40 #define MIN_PKT_SIZE 64 ··· 58 56 #define XSK_DESC__MAX_SKB_FRAGS 18 59 57 #define HUGEPAGE_SIZE (2 * 1024 * 1024) 60 58 #define PKT_DUMP_NB_TO_PRINT 16 59 59 + #define RUN_ALL_TESTS UINT_MAX 61 60 62 61 #define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0) 63 62 ··· 66 63 TEST_MODE_SKB, 67 64 TEST_MODE_DRV, 68 65 TEST_MODE_ZC, 69 69 - TEST_MODE_MAX 66 66 + TEST_MODE_ALL 70 67 }; 71 71 - 72 72 - enum test_type { 73 73 - TEST_TYPE_RUN_TO_COMPLETION, 74 74 - TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME, 75 75 - TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT, 76 76 - TEST_TYPE_RX_POLL, 77 77 - TEST_TYPE_TX_POLL, 78 78 - TEST_TYPE_POLL_RXQ_TMOUT, 79 79 - TEST_TYPE_POLL_TXQ_TMOUT, 80 80 - TEST_TYPE_UNALIGNED, 81 81 - TEST_TYPE_ALIGNED_INV_DESC, 82 82 - TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME, 83 83 - TEST_TYPE_UNALIGNED_INV_DESC, 84 84 - TEST_TYPE_UNALIGNED_INV_DESC_4K1_FRAME, 85 85 - TEST_TYPE_HEADROOM, 86 86 - TEST_TYPE_TEARDOWN, 87 87 - TEST_TYPE_BIDI, 88 88 - TEST_TYPE_STATS_RX_DROPPED, 89 89 - TEST_TYPE_STATS_TX_INVALID_DESCS, 90 90 - TEST_TYPE_STATS_RX_FULL, 91 91 - TEST_TYPE_STATS_FILL_EMPTY, 92 92 - TEST_TYPE_BPF_RES, 93 93 - TEST_TYPE_XDP_DROP_HALF, 94 94 - TEST_TYPE_XDP_METADATA_COUNT, 95 95 - TEST_TYPE_XDP_METADATA_COUNT_MB, 96 96 - TEST_TYPE_RUN_TO_COMPLETION_MB, 97 97 - TEST_TYPE_UNALIGNED_MB, 98 98 - TEST_TYPE_ALIGNED_INV_DESC_MB, 99 99 - TEST_TYPE_UNALIGNED_INV_DESC_MB, 100 100 - TEST_TYPE_TOO_MANY_FRAGS, 101 101 - TEST_TYPE_MAX 102 102 - }; 103 103 - 104 104 - static bool opt_verbose; 105 68 106 69 struct xsk_umem_info { 107 70 struct xsk_ring_prod fq; ··· 108 139 }; 109 140 110 141 struct ifobject; 142 142 + struct test_spec; 111 143 typedef int (*validation_func_t)(struct ifobject *ifobj); 112 144 typedef void *(*thread_func_t)(void *arg); 145 145 + typedef int (*test_func_t)(struct test_spec *test); 113 146 114 147 struct ifobject { 115 148 char ifname[MAX_INTERFACE_NAME_CHARS]; ··· 153 182 struct bpf_program *xdp_prog_tx; 154 183 struct bpf_map *xskmap_rx; 155 184 struct bpf_map *xskmap_tx; 185 185 + test_func_t test_func; 156 186 int mtu; 157 187 u16 total_steps; 158 188 u16 current_step;