+15 -2

Documentation/bpf/btf.rst

reviewed

··· 84 84 #define BTF_KIND_FUNC_PROTO 13 /* Function Proto */ 85 85 #define BTF_KIND_VAR 14 /* Variable */ 86 86 #define BTF_KIND_DATASEC 15 /* Section */ 87 87 + #define BTF_KIND_FLOAT 16 /* Floating point */ 87 88 88 89 Note that the type section encodes debug info, not just pure types. 89 90 ``BTF_KIND_FUNC`` is not a type, and it represents a defined subprogram. ··· 96 95 /* "info" bits arrangement 97 96 * bits 0-15: vlen (e.g. # of struct's members) 98 97 * bits 16-23: unused 99 99 - * bits 24-27: kind (e.g. int, ptr, array...etc) 100 100 - * bits 28-30: unused 98 98 + * bits 24-28: kind (e.g. int, ptr, array...etc) 99 99 + * bits 29-30: unused 101 100 * bit 31: kind_flag, currently used by 102 101 * struct, union and fwd 103 102 */ ··· 452 451 * ``type``: the type of the BTF_KIND_VAR variable 453 452 * ``offset``: the in-section offset of the variable 454 453 * ``size``: the size of the variable in bytes 454 454 + 455 455 + 2.2.16 BTF_KIND_FLOAT 456 456 + ~~~~~~~~~~~~~~~~~~~~~ 457 457 + 458 458 + ``struct btf_type`` encoding requirement: 459 459 + * ``name_off``: any valid offset 460 460 + * ``info.kind_flag``: 0 461 461 + * ``info.kind``: BTF_KIND_FLOAT 462 462 + * ``info.vlen``: 0 463 463 + * ``size``: the size of the float type in bytes: 2, 4, 8, 12 or 16. 464 464 + 465 465 + No additional type data follow ``btf_type``. 455 466 456 467 3. BTF Kernel API 457 468 *****************

+6 -3

Documentation/bpf/index.rst

reviewed

··· 12 12 The Cilium project also maintains a `BPF and XDP Reference Guide`_ 13 13 that goes into great technical depth about the BPF Architecture. 14 14 15 15 - The primary info for the bpf syscall is available in the `man-pages`_ 16 16 - for `bpf(2)`_. 17 17 - 18 15 BPF Type Format (BTF) 19 16 ===================== 20 17 ··· 32 35 bpf_design_QA 33 36 bpf_devel_QA 34 37 38 38 + Syscall API 39 39 + =========== 40 40 + 41 41 + The primary info for the bpf syscall is available in the `man-pages`_ 42 42 + for `bpf(2)`_. For more information about the userspace API, see 43 43 + Documentation/userspace-api/ebpf/index.rst. 35 44 36 45 Helper functions 37 46 ================

+17

Documentation/userspace-api/ebpf/index.rst

reviewed

··· 1 1 + .. SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + eBPF Userspace API 4 4 + ================== 5 5 + 6 6 + eBPF is a kernel mechanism to provide a sandboxed runtime environment in the 7 7 + Linux kernel for runtime extension and instrumentation without changing kernel 8 8 + source code or loading kernel modules. eBPF programs can be attached to various 9 9 + kernel subsystems, including networking, tracing and Linux security modules 10 10 + (LSM). 11 11 + 12 12 + For internal kernel documentation on eBPF, see Documentation/bpf/index.rst. 13 13 + 14 14 + .. toctree:: 15 15 + :maxdepth: 1 16 16 + 17 17 + syscall

+24

Documentation/userspace-api/ebpf/syscall.rst

reviewed

··· 1 1 + .. SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + eBPF Syscall 4 4 + ------------ 5 5 + 6 6 + :Authors: - Alexei Starovoitov <ast@kernel.org> 7 7 + - Joe Stringer <joe@wand.net.nz> 8 8 + - Michael Kerrisk <mtk.manpages@gmail.com> 9 9 + 10 10 + The primary info for the bpf syscall is available in the `man-pages`_ 11 11 + for `bpf(2)`_. 12 12 + 13 13 + bpf() subcommand reference 14 14 + ~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 15 + 16 16 + .. kernel-doc:: include/uapi/linux/bpf.h 17 17 + :doc: eBPF Syscall Preamble 18 18 + 19 19 + .. kernel-doc:: include/uapi/linux/bpf.h 20 20 + :doc: eBPF Syscall Commands 21 21 + 22 22 + .. Links: 23 23 + .. _man-pages: https://www.kernel.org/doc/man-pages/ 24 24 + .. _bpf(2): https://man7.org/linux/man-pages/man2/bpf.2.html

+1

Documentation/userspace-api/index.rst

reviewed

··· 21 21 unshare 22 22 spec_ctrl 23 23 accelerators/ocxl 24 24 + ebpf/index 24 25 ioctl/index 25 26 iommu 26 27 media/index

+2

MAINTAINERS

reviewed

··· 3233 3233 T: git git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git 3234 3234 F: Documentation/bpf/ 3235 3235 F: Documentation/networking/filter.rst 3236 3236 + F: Documentation/userspace-api/ebpf/ 3236 3237 F: arch/*/net/* 3237 3238 F: include/linux/bpf* 3238 3239 F: include/linux/filter.h ··· 3248 3247 F: net/sched/act_bpf.c 3249 3248 F: net/sched/cls_bpf.c 3250 3249 F: samples/bpf/ 3250 3250 + F: scripts/bpf_doc.py 3251 3251 F: tools/bpf/ 3252 3252 F: tools/lib/bpf/ 3253 3253 F: tools/testing/selftests/bpf/

+2 -1

drivers/net/virtio_net.c

reviewed

··· 2973 2973 return -ENOMEM; 2974 2974 2975 2975 /* Set up network device as normal. */ 2976 2976 - dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE; 2976 2976 + dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE | 2977 2977 + IFF_TX_SKB_NO_LINEAR; 2977 2978 dev->netdev_ops = &virtnet_netdev; 2978 2979 dev->features = NETIF_F_HIGHDMA; 2979 2980

+54 -40

include/linux/bpf.h

reviewed

··· 39 39 struct bpf_local_storage_map; 40 40 struct kobject; 41 41 struct mem_cgroup; 42 42 + struct bpf_func_state; 42 43 43 44 extern struct idr btf_idr; 44 45 extern spinlock_t btf_idr_lock; ··· 118 117 void *owner, u32 size); 119 118 struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner); 120 119 120 120 + /* Misc helpers.*/ 121 121 + int (*map_redirect)(struct bpf_map *map, u32 ifindex, u64 flags); 122 122 + 121 123 /* map_meta_equal must be implemented for maps that can be 122 124 * used as an inner map. It is a runtime check to ensure 123 125 * an inner map can be inserted to an outer map. ··· 132 128 */ 133 129 bool (*map_meta_equal)(const struct bpf_map *meta0, 134 130 const struct bpf_map *meta1); 131 131 + 132 132 + 133 133 + int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env, 134 134 + struct bpf_func_state *caller, 135 135 + struct bpf_func_state *callee); 136 136 + int (*map_for_each_callback)(struct bpf_map *map, void *callback_fn, 137 137 + void *callback_ctx, u64 flags); 135 138 136 139 /* BTF name and id of struct allocated by map_alloc */ 137 140 const char * const map_btf_name; ··· 306 295 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */ 307 296 ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */ 308 297 ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */ 298 298 + ARG_PTR_TO_FUNC, /* pointer to a bpf program function */ 299 299 + ARG_PTR_TO_STACK_OR_NULL, /* pointer to stack or NULL */ 309 300 __BPF_ARG_TYPE_MAX, 310 301 }; 311 302 ··· 424 411 PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */ 425 412 PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */ 426 413 PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */ 414 414 + PTR_TO_FUNC, /* reg points to a bpf program function */ 415 415 + PTR_TO_MAP_KEY, /* reg points to a map element key */ 427 416 }; 428 417 429 418 /* The information passed from prog-specific *_is_valid_access ··· 520 505 * See include/trace/bpf_probe.h 521 506 */ 522 507 #define MAX_BPF_FUNC_ARGS 12 508 508 + 509 509 + /* The maximum number of arguments passed through registers 510 510 + * a single function may have. 511 511 + */ 512 512 + #define MAX_BPF_FUNC_REG_ARGS 5 523 513 524 514 struct btf_func_model { 525 515 u8 ret_size; ··· 1400 1380 int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, 1401 1381 struct bpf_link_info *info); 1402 1382 1383 1383 + int map_set_for_each_callback_args(struct bpf_verifier_env *env, 1384 1384 + struct bpf_func_state *caller, 1385 1385 + struct bpf_func_state *callee); 1386 1386 + 1403 1387 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); 1404 1388 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); 1405 1389 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, ··· 1453 1429 /* Map specifics */ 1454 1430 struct xdp_buff; 1455 1431 struct sk_buff; 1432 1432 + struct bpf_dtab_netdev; 1433 1433 + struct bpf_cpu_map_entry; 1456 1434 1457 1457 - struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key); 1458 1458 - struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key); 1459 1435 void __dev_flush(void); 1460 1436 int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, 1461 1437 struct net_device *dev_rx); ··· 1465 1441 struct bpf_prog *xdp_prog); 1466 1442 bool dev_map_can_have_prog(struct bpf_map *map); 1467 1443 1468 1468 - struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key); 1469 1444 void __cpu_map_flush(void); 1470 1445 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, 1471 1446 struct net_device *dev_rx); ··· 1493 1470 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, 1494 1471 const union bpf_attr *kattr, 1495 1472 union bpf_attr __user *uattr); 1473 1473 + int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, 1474 1474 + const union bpf_attr *kattr, 1475 1475 + union bpf_attr __user *uattr); 1496 1476 bool btf_ctx_access(int off, int size, enum bpf_access_type type, 1497 1477 const struct bpf_prog *prog, 1498 1478 struct bpf_insn_access_aux *info); ··· 1525 1499 struct bpf_link *bpf_link_by_id(u32 id); 1526 1500 1527 1501 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); 1502 1502 + void bpf_task_storage_free(struct task_struct *task); 1528 1503 #else /* !CONFIG_BPF_SYSCALL */ 1529 1504 static inline struct bpf_prog *bpf_prog_get(u32 ufd) 1530 1505 { ··· 1595 1568 return -EOPNOTSUPP; 1596 1569 } 1597 1570 1598 1598 - static inline struct net_device *__dev_map_lookup_elem(struct bpf_map *map, 1599 1599 - u32 key) 1600 1600 - { 1601 1601 - return NULL; 1602 1602 - } 1603 1603 - 1604 1604 - static inline struct net_device *__dev_map_hash_lookup_elem(struct bpf_map *map, 1605 1605 - u32 key) 1606 1606 - { 1607 1607 - return NULL; 1608 1608 - } 1609 1571 static inline bool dev_map_can_have_prog(struct bpf_map *map) 1610 1572 { 1611 1573 return false; ··· 1606 1590 1607 1591 struct xdp_buff; 1608 1592 struct bpf_dtab_netdev; 1593 1593 + struct bpf_cpu_map_entry; 1609 1594 1610 1595 static inline 1611 1596 int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, ··· 1629 1612 struct bpf_prog *xdp_prog) 1630 1613 { 1631 1614 return 0; 1632 1632 - } 1633 1633 - 1634 1634 - static inline 1635 1635 - struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) 1636 1636 - { 1637 1637 - return NULL; 1638 1615 } 1639 1616 1640 1617 static inline void __cpu_map_flush(void) ··· 1681 1670 return -ENOTSUPP; 1682 1671 } 1683 1672 1673 1673 + static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, 1674 1674 + const union bpf_attr *kattr, 1675 1675 + union bpf_attr __user *uattr) 1676 1676 + { 1677 1677 + return -ENOTSUPP; 1678 1678 + } 1679 1679 + 1684 1680 static inline void bpf_map_put(struct bpf_map *map) 1685 1681 { 1686 1682 } ··· 1701 1683 bpf_base_func_proto(enum bpf_func_id func_id) 1702 1684 { 1703 1685 return NULL; 1686 1686 + } 1687 1687 + 1688 1688 + static inline void bpf_task_storage_free(struct task_struct *task) 1689 1689 + { 1704 1690 } 1705 1691 #endif /* CONFIG_BPF_SYSCALL */ 1706 1692 ··· 1790 1768 } 1791 1769 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ 1792 1770 1793 1793 - #if defined(CONFIG_BPF_STREAM_PARSER) 1794 1794 - int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, 1795 1795 - struct bpf_prog *old, u32 which); 1771 1771 + #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) 1796 1772 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); 1797 1773 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); 1798 1774 int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags); 1799 1775 void sock_map_unhash(struct sock *sk); 1800 1776 void sock_map_close(struct sock *sk, long timeout); 1777 1777 + 1778 1778 + void bpf_sk_reuseport_detach(struct sock *sk); 1779 1779 + int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, 1780 1780 + void *value); 1781 1781 + int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, 1782 1782 + void *value, u64 map_flags); 1801 1783 #else 1802 1802 - static inline int sock_map_prog_update(struct bpf_map *map, 1803 1803 - struct bpf_prog *prog, 1804 1804 - struct bpf_prog *old, u32 which) 1784 1784 + static inline void bpf_sk_reuseport_detach(struct sock *sk) 1805 1785 { 1806 1806 - return -EOPNOTSUPP; 1807 1786 } 1808 1787 1788 1788 + #ifdef CONFIG_BPF_SYSCALL 1809 1789 static inline int sock_map_get_from_fd(const union bpf_attr *attr, 1810 1790 struct bpf_prog *prog) 1811 1791 { ··· 1825 1801 { 1826 1802 return -EOPNOTSUPP; 1827 1803 } 1828 1828 - #endif /* CONFIG_BPF_STREAM_PARSER */ 1829 1804 1830 1830 - #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) 1831 1831 - void bpf_sk_reuseport_detach(struct sock *sk); 1832 1832 - int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, 1833 1833 - void *value); 1834 1834 - int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, 1835 1835 - void *value, u64 map_flags); 1836 1836 - #else 1837 1837 - static inline void bpf_sk_reuseport_detach(struct sock *sk) 1838 1838 - { 1839 1839 - } 1840 1840 - 1841 1841 - #ifdef CONFIG_BPF_SYSCALL 1842 1805 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, 1843 1806 void *key, void *value) 1844 1807 { ··· 1897 1886 extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto; 1898 1887 extern const struct bpf_func_proto bpf_sock_from_file_proto; 1899 1888 extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto; 1889 1889 + extern const struct bpf_func_proto bpf_task_storage_get_proto; 1890 1890 + extern const struct bpf_func_proto bpf_task_storage_delete_proto; 1891 1891 + extern const struct bpf_func_proto bpf_for_each_map_elem_proto; 1900 1892 1901 1893 const struct bpf_func_proto *bpf_tracing_func_proto( 1902 1894 enum bpf_func_id func_id, const struct bpf_prog *prog);

+2 -1

include/linux/bpf_local_storage.h

reviewed

··· 126 126 struct bpf_local_storage_map *smap, 127 127 bool cacheit_lockit); 128 128 129 129 - void bpf_local_storage_map_free(struct bpf_local_storage_map *smap); 129 129 + void bpf_local_storage_map_free(struct bpf_local_storage_map *smap, 130 130 + int __percpu *busy_counter); 130 131 131 132 int bpf_local_storage_map_check_btf(const struct bpf_map *map, 132 133 const struct btf *btf,

-22

include/linux/bpf_lsm.h

reviewed

··· 38 38 return inode->i_security + bpf_lsm_blob_sizes.lbs_inode; 39 39 } 40 40 41 41 - static inline struct bpf_storage_blob *bpf_task( 42 42 - const struct task_struct *task) 43 43 - { 44 44 - if (unlikely(!task->security)) 45 45 - return NULL; 46 46 - 47 47 - return task->security + bpf_lsm_blob_sizes.lbs_task; 48 48 - } 49 49 - 50 41 extern const struct bpf_func_proto bpf_inode_storage_get_proto; 51 42 extern const struct bpf_func_proto bpf_inode_storage_delete_proto; 52 52 - extern const struct bpf_func_proto bpf_task_storage_get_proto; 53 53 - extern const struct bpf_func_proto bpf_task_storage_delete_proto; 54 43 void bpf_inode_storage_free(struct inode *inode); 55 55 - void bpf_task_storage_free(struct task_struct *task); 56 44 57 45 #else /* !CONFIG_BPF_LSM */ 58 46 ··· 61 73 return NULL; 62 74 } 63 75 64 64 - static inline struct bpf_storage_blob *bpf_task( 65 65 - const struct task_struct *task) 66 66 - { 67 67 - return NULL; 68 68 - } 69 69 - 70 76 static inline void bpf_inode_storage_free(struct inode *inode) 71 71 - { 72 72 - } 73 73 - 74 74 - static inline void bpf_task_storage_free(struct task_struct *task) 75 77 { 76 78 } 77 79

+3 -5

include/linux/bpf_types.h

reviewed

··· 103 103 BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP, dev_map_ops) 104 104 BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP_HASH, dev_map_hash_ops) 105 105 BPF_MAP_TYPE(BPF_MAP_TYPE_SK_STORAGE, sk_storage_map_ops) 106 106 - #if defined(CONFIG_BPF_STREAM_PARSER) 107 107 - BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKMAP, sock_map_ops) 108 108 - BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKHASH, sock_hash_ops) 109 109 - #endif 110 106 #ifdef CONFIG_BPF_LSM 111 107 BPF_MAP_TYPE(BPF_MAP_TYPE_INODE_STORAGE, inode_storage_map_ops) 112 112 - BPF_MAP_TYPE(BPF_MAP_TYPE_TASK_STORAGE, task_storage_map_ops) 113 108 #endif 109 109 + BPF_MAP_TYPE(BPF_MAP_TYPE_TASK_STORAGE, task_storage_map_ops) 114 110 BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops) 115 111 #if defined(CONFIG_XDP_SOCKETS) 116 112 BPF_MAP_TYPE(BPF_MAP_TYPE_XSKMAP, xsk_map_ops) 117 113 #endif 118 114 #ifdef CONFIG_INET 115 115 + BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKMAP, sock_map_ops) 116 116 + BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKHASH, sock_hash_ops) 119 117 BPF_MAP_TYPE(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, reuseport_array_ops) 120 118 #endif 121 119 #endif

+3

include/linux/bpf_verifier.h

reviewed

··· 68 68 unsigned long raw1; 69 69 unsigned long raw2; 70 70 } raw; 71 71 + 72 72 + u32 subprogno; /* for PTR_TO_FUNC */ 71 73 }; 72 74 /* For PTR_TO_PACKET, used to find other pointers with the same variable 73 75 * offset, so they can share range knowledge. ··· 206 204 int acquired_refs; 207 205 struct bpf_reference_state *refs; 208 206 int allocated_stack; 207 207 + bool in_callback_fn; 209 208 struct bpf_stack_state *stack; 210 209 }; 211 210

+30 -1

include/linux/filter.h

reviewed

··· 646 646 u32 flags; 647 647 u32 tgt_index; 648 648 void *tgt_value; 649 649 - struct bpf_map *map; 649 649 + u32 map_id; 650 650 + enum bpf_map_type map_type; 650 651 u32 kern_flags; 651 652 struct bpf_nh_params nh; 652 653 }; ··· 1472 1471 return no_reuseport; 1473 1472 } 1474 1473 #endif /* IS_ENABLED(CONFIG_IPV6) */ 1474 1474 + 1475 1475 + static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u32 ifindex, u64 flags, 1476 1476 + void *lookup_elem(struct bpf_map *map, u32 key)) 1477 1477 + { 1478 1478 + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); 1479 1479 + 1480 1480 + /* Lower bits of the flags are used as return code on lookup failure */ 1481 1481 + if (unlikely(flags > XDP_TX)) 1482 1482 + return XDP_ABORTED; 1483 1483 + 1484 1484 + ri->tgt_value = lookup_elem(map, ifindex); 1485 1485 + if (unlikely(!ri->tgt_value)) { 1486 1486 + /* If the lookup fails we want to clear out the state in the 1487 1487 + * redirect_info struct completely, so that if an eBPF program 1488 1488 + * performs multiple lookups, the last one always takes 1489 1489 + * precedence. 1490 1490 + */ 1491 1491 + ri->map_id = INT_MAX; /* Valid map id idr range: [1,INT_MAX[ */ 1492 1492 + ri->map_type = BPF_MAP_TYPE_UNSPEC; 1493 1493 + return flags; 1494 1494 + } 1495 1495 + 1496 1496 + ri->tgt_index = ifindex; 1497 1497 + ri->map_id = map->id; 1498 1498 + ri->map_type = map->map_type; 1499 1499 + 1500 1500 + return XDP_REDIRECT; 1501 1501 + } 1475 1502 1476 1503 #endif /* __LINUX_FILTER_H__ */

+5

include/linux/netdevice.h

reviewed

··· 1518 1518 * @IFF_FAILOVER_SLAVE: device is lower dev of a failover master device 1519 1519 * @IFF_L3MDEV_RX_HANDLER: only invoke the rx handler of L3 master device 1520 1520 * @IFF_LIVE_RENAME_OK: rename is allowed while device is up and running 1521 1521 + * @IFF_TX_SKB_NO_LINEAR: device/driver is capable of xmitting frames with 1522 1522 + * skb_headlen(skb) == 0 (data starts from frag0) 1521 1523 */ 1522 1524 enum netdev_priv_flags { 1523 1525 IFF_802_1Q_VLAN = 1<<0, ··· 1553 1551 IFF_FAILOVER_SLAVE = 1<<28, 1554 1552 IFF_L3MDEV_RX_HANDLER = 1<<29, 1555 1553 IFF_LIVE_RENAME_OK = 1<<30, 1554 1554 + IFF_TX_SKB_NO_LINEAR = 1<<31, 1556 1555 }; 1557 1556 1558 1557 #define IFF_802_1Q_VLAN IFF_802_1Q_VLAN ··· 1580 1577 #define IFF_L3MDEV_SLAVE IFF_L3MDEV_SLAVE 1581 1578 #define IFF_TEAM IFF_TEAM 1582 1579 #define IFF_RXFH_CONFIGURED IFF_RXFH_CONFIGURED 1580 1580 + #define IFF_PHONY_HEADROOM IFF_PHONY_HEADROOM 1583 1581 #define IFF_MACSEC IFF_MACSEC 1584 1582 #define IFF_NO_RX_HANDLER IFF_NO_RX_HANDLER 1585 1583 #define IFF_FAILOVER IFF_FAILOVER 1586 1584 #define IFF_FAILOVER_SLAVE IFF_FAILOVER_SLAVE 1587 1585 #define IFF_L3MDEV_RX_HANDLER IFF_L3MDEV_RX_HANDLER 1588 1586 #define IFF_LIVE_RENAME_OK IFF_LIVE_RENAME_OK 1587 1587 + #define IFF_TX_SKB_NO_LINEAR IFF_TX_SKB_NO_LINEAR 1589 1588 1590 1589 /* Specifies the type of the struct net_device::ml_priv pointer */ 1591 1590 enum netdev_ml_priv_type {

+5

include/linux/sched.h

reviewed

··· 42 42 struct backing_dev_info; 43 43 struct bio_list; 44 44 struct blk_plug; 45 45 + struct bpf_local_storage; 45 46 struct capture_control; 46 47 struct cfs_rq; 47 48 struct fs_struct; ··· 1351 1350 #ifdef CONFIG_SECURITY 1352 1351 /* Used by LSM modules for access restriction: */ 1353 1352 void *security; 1353 1353 + #endif 1354 1354 + #ifdef CONFIG_BPF_SYSCALL 1355 1355 + /* Used by BPF task local storage */ 1356 1356 + struct bpf_local_storage __rcu *bpf_storage; 1354 1357 #endif 1355 1358 1356 1359 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK

+4

include/linux/skbuff.h

reviewed

··· 656 656 * @protocol: Packet protocol from driver 657 657 * @destructor: Destruct function 658 658 * @tcp_tsorted_anchor: list structure for TCP (tp->tsorted_sent_queue) 659 659 + * @_sk_redir: socket redirection information for skmsg 659 660 * @_nfct: Associated connection, if any (with nfctinfo bits) 660 661 * @nf_bridge: Saved data about a bridged frame - see br_netfilter.c 661 662 * @skb_iif: ifindex of device we arrived on ··· 756 755 void (*destructor)(struct sk_buff *skb); 757 756 }; 758 757 struct list_head tcp_tsorted_anchor; 758 758 + #ifdef CONFIG_NET_SOCK_MSG 759 759 + unsigned long _sk_redir; 760 760 + #endif 759 761 }; 760 762 761 763 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)

+65 -17

include/linux/skmsg.h

reviewed

··· 56 56 57 57 struct sk_psock_progs { 58 58 struct bpf_prog *msg_parser; 59 59 - struct bpf_prog *skb_parser; 60 60 - struct bpf_prog *skb_verdict; 59 59 + struct bpf_prog *stream_parser; 60 60 + struct bpf_prog *stream_verdict; 61 61 }; 62 62 63 63 enum sk_psock_state_bits { ··· 68 68 struct list_head list; 69 69 struct bpf_map *map; 70 70 void *link_raw; 71 71 - }; 72 72 - 73 73 - struct sk_psock_parser { 74 74 - struct strparser strp; 75 75 - bool enabled; 76 76 - void (*saved_data_ready)(struct sock *sk); 77 71 }; 78 72 79 73 struct sk_psock_work_state { ··· 84 90 u32 eval; 85 91 struct sk_msg *cork; 86 92 struct sk_psock_progs progs; 87 87 - struct sk_psock_parser parser; 93 93 + #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER) 94 94 + struct strparser strp; 95 95 + #endif 88 96 struct sk_buff_head ingress_skb; 89 97 struct list_head ingress_msg; 90 98 unsigned long state; ··· 96 100 void (*saved_unhash)(struct sock *sk); 97 101 void (*saved_close)(struct sock *sk, long timeout); 98 102 void (*saved_write_space)(struct sock *sk); 103 103 + void (*saved_data_ready)(struct sock *sk); 99 104 struct proto *sk_proto; 100 105 struct sk_psock_work_state work_state; 101 106 struct work_struct work; ··· 302 305 303 306 struct sk_psock *sk_psock_init(struct sock *sk, int node); 304 307 308 308 + #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER) 305 309 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock); 306 310 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock); 307 311 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock); 312 312 + #else 313 313 + static inline int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock) 314 314 + { 315 315 + return -EOPNOTSUPP; 316 316 + } 317 317 + 318 318 + static inline void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) 319 319 + { 320 320 + } 321 321 + 322 322 + static inline void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) 323 323 + { 324 324 + } 325 325 + #endif 326 326 + 308 327 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock); 309 328 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock); 310 329 ··· 339 326 } 340 327 341 328 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock); 342 342 - 343 343 - void __sk_psock_purge_ingress_msg(struct sk_psock *psock); 344 329 345 330 static inline void sk_psock_cork_free(struct sk_psock *psock) 346 331 { ··· 400 389 return psock; 401 390 } 402 391 403 403 - void sk_psock_stop(struct sock *sk, struct sk_psock *psock); 404 392 void sk_psock_drop(struct sock *sk, struct sk_psock *psock); 405 393 406 394 static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock) ··· 410 400 411 401 static inline void sk_psock_data_ready(struct sock *sk, struct sk_psock *psock) 412 402 { 413 413 - if (psock->parser.enabled) 414 414 - psock->parser.saved_data_ready(sk); 403 403 + if (psock->saved_data_ready) 404 404 + psock->saved_data_ready(sk); 415 405 else 416 406 sk->sk_data_ready(sk); 417 407 } ··· 440 430 static inline void psock_progs_drop(struct sk_psock_progs *progs) 441 431 { 442 432 psock_set_prog(&progs->msg_parser, NULL); 443 443 - psock_set_prog(&progs->skb_parser, NULL); 444 444 - psock_set_prog(&progs->skb_verdict, NULL); 433 433 + psock_set_prog(&progs->stream_parser, NULL); 434 434 + psock_set_prog(&progs->stream_verdict, NULL); 445 435 } 446 436 447 437 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb); ··· 450 440 { 451 441 if (!psock) 452 442 return false; 453 453 - return psock->parser.enabled; 443 443 + return !!psock->saved_data_ready; 454 444 } 445 445 + 446 446 + #if IS_ENABLED(CONFIG_NET_SOCK_MSG) 447 447 + 448 448 + /* We only have one bit so far. */ 449 449 + #define BPF_F_PTR_MASK ~(BPF_F_INGRESS) 450 450 + 451 451 + static inline bool skb_bpf_ingress(const struct sk_buff *skb) 452 452 + { 453 453 + unsigned long sk_redir = skb->_sk_redir; 454 454 + 455 455 + return sk_redir & BPF_F_INGRESS; 456 456 + } 457 457 + 458 458 + static inline void skb_bpf_set_ingress(struct sk_buff *skb) 459 459 + { 460 460 + skb->_sk_redir |= BPF_F_INGRESS; 461 461 + } 462 462 + 463 463 + static inline void skb_bpf_set_redir(struct sk_buff *skb, struct sock *sk_redir, 464 464 + bool ingress) 465 465 + { 466 466 + skb->_sk_redir = (unsigned long)sk_redir; 467 467 + if (ingress) 468 468 + skb->_sk_redir |= BPF_F_INGRESS; 469 469 + } 470 470 + 471 471 + static inline struct sock *skb_bpf_redirect_fetch(const struct sk_buff *skb) 472 472 + { 473 473 + unsigned long sk_redir = skb->_sk_redir; 474 474 + 475 475 + return (struct sock *)(sk_redir & BPF_F_PTR_MASK); 476 476 + } 477 477 + 478 478 + static inline void skb_bpf_redirect_clear(struct sk_buff *skb) 479 479 + { 480 480 + skb->_sk_redir = 0; 481 481 + } 482 482 + #endif /* CONFIG_NET_SOCK_MSG */ 455 483 #endif /* _LINUX_SKMSG_H */

+9 -32

include/net/tcp.h

reviewed

··· 883 883 struct inet6_skb_parm h6; 884 884 #endif 885 885 } header; /* For incoming skbs */ 886 886 - struct { 887 887 - __u32 flags; 888 888 - struct sock *sk_redir; 889 889 - void *data_end; 890 890 - } bpf; 891 886 }; 892 887 }; 893 888 894 889 #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0])) 895 895 - 896 896 - static inline void bpf_compute_data_end_sk_skb(struct sk_buff *skb) 897 897 - { 898 898 - TCP_SKB_CB(skb)->bpf.data_end = skb->data + skb_headlen(skb); 899 899 - } 900 900 - 901 901 - static inline bool tcp_skb_bpf_ingress(const struct sk_buff *skb) 902 902 - { 903 903 - return TCP_SKB_CB(skb)->bpf.flags & BPF_F_INGRESS; 904 904 - } 905 905 - 906 906 - static inline struct sock *tcp_skb_bpf_redirect_fetch(struct sk_buff *skb) 907 907 - { 908 908 - return TCP_SKB_CB(skb)->bpf.sk_redir; 909 909 - } 910 910 - 911 911 - static inline void tcp_skb_bpf_redirect_clear(struct sk_buff *skb) 912 912 - { 913 913 - TCP_SKB_CB(skb)->bpf.sk_redir = NULL; 914 914 - } 915 890 916 891 extern const struct inet_connection_sock_af_ops ipv4_specific; 917 892 ··· 2197 2222 __MODULE_INFO(alias, alias_userspace, name); \ 2198 2223 __MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name) 2199 2224 2225 2225 + #ifdef CONFIG_NET_SOCK_MSG 2200 2226 struct sk_msg; 2201 2227 struct sk_psock; 2202 2228 2203 2203 - #ifdef CONFIG_BPF_STREAM_PARSER 2229 2229 + #ifdef CONFIG_BPF_SYSCALL 2204 2230 struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock); 2205 2231 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk); 2206 2206 - #else 2207 2207 - static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) 2208 2208 - { 2209 2209 - } 2210 2210 - #endif /* CONFIG_BPF_STREAM_PARSER */ 2232 2232 + #endif /* CONFIG_BPF_SYSCALL */ 2211 2233 2212 2212 - #ifdef CONFIG_NET_SOCK_MSG 2213 2234 int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes, 2214 2235 int flags); 2215 2236 int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock, 2216 2237 struct msghdr *msg, int len, int flags); 2217 2238 #endif /* CONFIG_NET_SOCK_MSG */ 2239 2239 + 2240 2240 + #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG) 2241 2241 + static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) 2242 2242 + { 2243 2243 + } 2244 2244 + #endif 2218 2245 2219 2246 #ifdef CONFIG_CGROUP_BPF 2220 2247 static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,

+2 -2

include/net/udp.h

reviewed

··· 515 515 return segs; 516 516 } 517 517 518 518 - #ifdef CONFIG_BPF_STREAM_PARSER 518 518 + #ifdef CONFIG_BPF_SYSCALL 519 519 struct sk_psock; 520 520 struct proto *udp_bpf_get_proto(struct sock *sk, struct sk_psock *psock); 521 521 - #endif /* BPF_STREAM_PARSER */ 521 521 + #endif 522 522 523 523 #endif /* _UDP_H */

-19

include/net/xdp_sock.h

reviewed

··· 80 80 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp); 81 81 void __xsk_map_flush(void); 82 82 83 83 - static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, 84 84 - u32 key) 85 85 - { 86 86 - struct xsk_map *m = container_of(map, struct xsk_map, map); 87 87 - struct xdp_sock *xs; 88 88 - 89 89 - if (key >= map->max_entries) 90 90 - return NULL; 91 91 - 92 92 - xs = READ_ONCE(m->xsk_map[key]); 93 93 - return xs; 94 94 - } 95 95 - 96 83 #else 97 84 98 85 static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) ··· 94 107 95 108 static inline void __xsk_map_flush(void) 96 109 { 97 97 - } 98 98 - 99 99 - static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, 100 100 - u32 key) 101 101 - { 102 102 - return NULL; 103 110 } 104 111 105 112 #endif /* CONFIG_XDP_SOCKETS */

+35 -27

include/trace/events/xdp.h

reviewed

··· 86 86 }; 87 87 #endif /* __DEVMAP_OBJ_TYPE */ 88 88 89 89 - #define devmap_ifindex(tgt, map) \ 90 90 - (((map->map_type == BPF_MAP_TYPE_DEVMAP || \ 91 91 - map->map_type == BPF_MAP_TYPE_DEVMAP_HASH)) ? \ 92 92 - ((struct _bpf_dtab_netdev *)tgt)->dev->ifindex : 0) 93 93 - 94 89 DECLARE_EVENT_CLASS(xdp_redirect_template, 95 90 96 91 TP_PROTO(const struct net_device *dev, 97 92 const struct bpf_prog *xdp, 98 93 const void *tgt, int err, 99 99 - const struct bpf_map *map, u32 index), 94 94 + enum bpf_map_type map_type, 95 95 + u32 map_id, u32 index), 100 96 101 101 - TP_ARGS(dev, xdp, tgt, err, map, index), 97 97 + TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index), 102 98 103 99 TP_STRUCT__entry( 104 100 __field(int, prog_id) ··· 107 111 ), 108 112 109 113 TP_fast_assign( 114 114 + u32 ifindex = 0, map_index = index; 115 115 + 116 116 + if (map_type == BPF_MAP_TYPE_DEVMAP || map_type == BPF_MAP_TYPE_DEVMAP_HASH) { 117 117 + ifindex = ((struct _bpf_dtab_netdev *)tgt)->dev->ifindex; 118 118 + } else if (map_type == BPF_MAP_TYPE_UNSPEC && map_id == INT_MAX) { 119 119 + ifindex = index; 120 120 + map_index = 0; 121 121 + } 122 122 + 110 123 __entry->prog_id = xdp->aux->id; 111 124 __entry->act = XDP_REDIRECT; 112 125 __entry->ifindex = dev->ifindex; 113 126 __entry->err = err; 114 114 - __entry->to_ifindex = map ? devmap_ifindex(tgt, map) : 115 115 - index; 116 116 - __entry->map_id = map ? map->id : 0; 117 117 - __entry->map_index = map ? index : 0; 127 127 + __entry->to_ifindex = ifindex; 128 128 + __entry->map_id = map_id; 129 129 + __entry->map_index = map_index; 118 130 ), 119 131 120 132 TP_printk("prog_id=%d action=%s ifindex=%d to_ifindex=%d err=%d" ··· 137 133 TP_PROTO(const struct net_device *dev, 138 134 const struct bpf_prog *xdp, 139 135 const void *tgt, int err, 140 140 - const struct bpf_map *map, u32 index), 141 141 - TP_ARGS(dev, xdp, tgt, err, map, index) 136 136 + enum bpf_map_type map_type, 137 137 + u32 map_id, u32 index), 138 138 + TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index) 142 139 ); 143 140 144 141 DEFINE_EVENT(xdp_redirect_template, xdp_redirect_err, 145 142 TP_PROTO(const struct net_device *dev, 146 143 const struct bpf_prog *xdp, 147 144 const void *tgt, int err, 148 148 - const struct bpf_map *map, u32 index), 149 149 - TP_ARGS(dev, xdp, tgt, err, map, index) 145 145 + enum bpf_map_type map_type, 146 146 + u32 map_id, u32 index), 147 147 + TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index) 150 148 ); 151 149 152 152 - #define _trace_xdp_redirect(dev, xdp, to) \ 153 153 - trace_xdp_redirect(dev, xdp, NULL, 0, NULL, to) 150 150 + #define _trace_xdp_redirect(dev, xdp, to) \ 151 151 + trace_xdp_redirect(dev, xdp, NULL, 0, BPF_MAP_TYPE_UNSPEC, INT_MAX, to) 154 152 155 155 - #define _trace_xdp_redirect_err(dev, xdp, to, err) \ 156 156 - trace_xdp_redirect_err(dev, xdp, NULL, err, NULL, to) 153 153 + #define _trace_xdp_redirect_err(dev, xdp, to, err) \ 154 154 + trace_xdp_redirect_err(dev, xdp, NULL, err, BPF_MAP_TYPE_UNSPEC, INT_MAX, to) 157 155 158 158 - #define _trace_xdp_redirect_map(dev, xdp, to, map, index) \ 159 159 - trace_xdp_redirect(dev, xdp, to, 0, map, index) 156 156 + #define _trace_xdp_redirect_map(dev, xdp, to, map_type, map_id, index) \ 157 157 + trace_xdp_redirect(dev, xdp, to, 0, map_type, map_id, index) 160 158 161 161 - #define _trace_xdp_redirect_map_err(dev, xdp, to, map, index, err) \ 162 162 - trace_xdp_redirect_err(dev, xdp, to, err, map, index) 159 159 + #define _trace_xdp_redirect_map_err(dev, xdp, to, map_type, map_id, index, err) \ 160 160 + trace_xdp_redirect_err(dev, xdp, to, err, map_type, map_id, index) 163 161 164 162 /* not used anymore, but kept around so as not to break old programs */ 165 163 DEFINE_EVENT(xdp_redirect_template, xdp_redirect_map, 166 164 TP_PROTO(const struct net_device *dev, 167 165 const struct bpf_prog *xdp, 168 166 const void *tgt, int err, 169 169 - const struct bpf_map *map, u32 index), 170 170 - TP_ARGS(dev, xdp, tgt, err, map, index) 167 167 + enum bpf_map_type map_type, 168 168 + u32 map_id, u32 index), 169 169 + TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index) 171 170 ); 172 171 173 172 DEFINE_EVENT(xdp_redirect_template, xdp_redirect_map_err, 174 173 TP_PROTO(const struct net_device *dev, 175 174 const struct bpf_prog *xdp, 176 175 const void *tgt, int err, 177 177 - const struct bpf_map *map, u32 index), 178 178 - TP_ARGS(dev, xdp, tgt, err, map, index) 176 176 + enum bpf_map_type map_type, 177 177 + u32 map_id, u32 index), 178 178 + TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index) 179 179 ); 180 180 181 181 TRACE_EVENT(xdp_cpumap_kthread,

+759 -3

include/uapi/linux/bpf.h

reviewed

··· 93 93 } map; 94 94 }; 95 95 96 96 - /* BPF syscall commands, see bpf(2) man-page for details. */ 96 96 + /* BPF syscall commands, see bpf(2) man-page for more details. */ 97 97 + /** 98 98 + * DOC: eBPF Syscall Preamble 99 99 + * 100 100 + * The operation to be performed by the **bpf**\ () system call is determined 101 101 + * by the *cmd* argument. Each operation takes an accompanying argument, 102 102 + * provided via *attr*, which is a pointer to a union of type *bpf_attr* (see 103 103 + * below). The size argument is the size of the union pointed to by *attr*. 104 104 + */ 105 105 + /** 106 106 + * DOC: eBPF Syscall Commands 107 107 + * 108 108 + * BPF_MAP_CREATE 109 109 + * Description 110 110 + * Create a map and return a file descriptor that refers to the 111 111 + * map. The close-on-exec file descriptor flag (see **fcntl**\ (2)) 112 112 + * is automatically enabled for the new file descriptor. 113 113 + * 114 114 + * Applying **close**\ (2) to the file descriptor returned by 115 115 + * **BPF_MAP_CREATE** will delete the map (but see NOTES). 116 116 + * 117 117 + * Return 118 118 + * A new file descriptor (a nonnegative integer), or -1 if an 119 119 + * error occurred (in which case, *errno* is set appropriately). 120 120 + * 121 121 + * BPF_MAP_LOOKUP_ELEM 122 122 + * Description 123 123 + * Look up an element with a given *key* in the map referred to 124 124 + * by the file descriptor *map_fd*. 125 125 + * 126 126 + * The *flags* argument may be specified as one of the 127 127 + * following: 128 128 + * 129 129 + * **BPF_F_LOCK** 130 130 + * Look up the value of a spin-locked map without 131 131 + * returning the lock. This must be specified if the 132 132 + * elements contain a spinlock. 133 133 + * 134 134 + * Return 135 135 + * Returns zero on success. On error, -1 is returned and *errno* 136 136 + * is set appropriately. 137 137 + * 138 138 + * BPF_MAP_UPDATE_ELEM 139 139 + * Description 140 140 + * Create or update an element (key/value pair) in a specified map. 141 141 + * 142 142 + * The *flags* argument should be specified as one of the 143 143 + * following: 144 144 + * 145 145 + * **BPF_ANY** 146 146 + * Create a new element or update an existing element. 147 147 + * **BPF_NOEXIST** 148 148 + * Create a new element only if it did not exist. 149 149 + * **BPF_EXIST** 150 150 + * Update an existing element. 151 151 + * **BPF_F_LOCK** 152 152 + * Update a spin_lock-ed map element. 153 153 + * 154 154 + * Return 155 155 + * Returns zero on success. On error, -1 is returned and *errno* 156 156 + * is set appropriately. 157 157 + * 158 158 + * May set *errno* to **EINVAL**, **EPERM**, **ENOMEM**, 159 159 + * **E2BIG**, **EEXIST**, or **ENOENT**. 160 160 + * 161 161 + * **E2BIG** 162 162 + * The number of elements in the map reached the 163 163 + * *max_entries* limit specified at map creation time. 164 164 + * **EEXIST** 165 165 + * If *flags* specifies **BPF_NOEXIST** and the element 166 166 + * with *key* already exists in the map. 167 167 + * **ENOENT** 168 168 + * If *flags* specifies **BPF_EXIST** and the element with 169 169 + * *key* does not exist in the map. 170 170 + * 171 171 + * BPF_MAP_DELETE_ELEM 172 172 + * Description 173 173 + * Look up and delete an element by key in a specified map. 174 174 + * 175 175 + * Return 176 176 + * Returns zero on success. On error, -1 is returned and *errno* 177 177 + * is set appropriately. 178 178 + * 179 179 + * BPF_MAP_GET_NEXT_KEY 180 180 + * Description 181 181 + * Look up an element by key in a specified map and return the key 182 182 + * of the next element. Can be used to iterate over all elements 183 183 + * in the map. 184 184 + * 185 185 + * Return 186 186 + * Returns zero on success. On error, -1 is returned and *errno* 187 187 + * is set appropriately. 188 188 + * 189 189 + * The following cases can be used to iterate over all elements of 190 190 + * the map: 191 191 + * 192 192 + * * If *key* is not found, the operation returns zero and sets 193 193 + * the *next_key* pointer to the key of the first element. 194 194 + * * If *key* is found, the operation returns zero and sets the 195 195 + * *next_key* pointer to the key of the next element. 196 196 + * * If *key* is the last element, returns -1 and *errno* is set 197 197 + * to **ENOENT**. 198 198 + * 199 199 + * May set *errno* to **ENOMEM**, **EFAULT**, **EPERM**, or 200 200 + * **EINVAL** on error. 201 201 + * 202 202 + * BPF_PROG_LOAD 203 203 + * Description 204 204 + * Verify and load an eBPF program, returning a new file 205 205 + * descriptor associated with the program. 206 206 + * 207 207 + * Applying **close**\ (2) to the file descriptor returned by 208 208 + * **BPF_PROG_LOAD** will unload the eBPF program (but see NOTES). 209 209 + * 210 210 + * The close-on-exec file descriptor flag (see **fcntl**\ (2)) is 211 211 + * automatically enabled for the new file descriptor. 212 212 + * 213 213 + * Return 214 214 + * A new file descriptor (a nonnegative integer), or -1 if an 215 215 + * error occurred (in which case, *errno* is set appropriately). 216 216 + * 217 217 + * BPF_OBJ_PIN 218 218 + * Description 219 219 + * Pin an eBPF program or map referred by the specified *bpf_fd* 220 220 + * to the provided *pathname* on the filesystem. 221 221 + * 222 222 + * The *pathname* argument must not contain a dot ("."). 223 223 + * 224 224 + * On success, *pathname* retains a reference to the eBPF object, 225 225 + * preventing deallocation of the object when the original 226 226 + * *bpf_fd* is closed. This allow the eBPF object to live beyond 227 227 + * **close**\ (\ *bpf_fd*\ ), and hence the lifetime of the parent 228 228 + * process. 229 229 + * 230 230 + * Applying **unlink**\ (2) or similar calls to the *pathname* 231 231 + * unpins the object from the filesystem, removing the reference. 232 232 + * If no other file descriptors or filesystem nodes refer to the 233 233 + * same object, it will be deallocated (see NOTES). 234 234 + * 235 235 + * The filesystem type for the parent directory of *pathname* must 236 236 + * be **BPF_FS_MAGIC**. 237 237 + * 238 238 + * Return 239 239 + * Returns zero on success. On error, -1 is returned and *errno* 240 240 + * is set appropriately. 241 241 + * 242 242 + * BPF_OBJ_GET 243 243 + * Description 244 244 + * Open a file descriptor for the eBPF object pinned to the 245 245 + * specified *pathname*. 246 246 + * 247 247 + * Return 248 248 + * A new file descriptor (a nonnegative integer), or -1 if an 249 249 + * error occurred (in which case, *errno* is set appropriately). 250 250 + * 251 251 + * BPF_PROG_ATTACH 252 252 + * Description 253 253 + * Attach an eBPF program to a *target_fd* at the specified 254 254 + * *attach_type* hook. 255 255 + * 256 256 + * The *attach_type* specifies the eBPF attachment point to 257 257 + * attach the program to, and must be one of *bpf_attach_type* 258 258 + * (see below). 259 259 + * 260 260 + * The *attach_bpf_fd* must be a valid file descriptor for a 261 261 + * loaded eBPF program of a cgroup, flow dissector, LIRC, sockmap 262 262 + * or sock_ops type corresponding to the specified *attach_type*. 263 263 + * 264 264 + * The *target_fd* must be a valid file descriptor for a kernel 265 265 + * object which depends on the attach type of *attach_bpf_fd*: 266 266 + * 267 267 + * **BPF_PROG_TYPE_CGROUP_DEVICE**, 268 268 + * **BPF_PROG_TYPE_CGROUP_SKB**, 269 269 + * **BPF_PROG_TYPE_CGROUP_SOCK**, 270 270 + * **BPF_PROG_TYPE_CGROUP_SOCK_ADDR**, 271 271 + * **BPF_PROG_TYPE_CGROUP_SOCKOPT**, 272 272 + * **BPF_PROG_TYPE_CGROUP_SYSCTL**, 273 273 + * **BPF_PROG_TYPE_SOCK_OPS** 274 274 + * 275 275 + * Control Group v2 hierarchy with the eBPF controller 276 276 + * enabled. Requires the kernel to be compiled with 277 277 + * **CONFIG_CGROUP_BPF**. 278 278 + * 279 279 + * **BPF_PROG_TYPE_FLOW_DISSECTOR** 280 280 + * 281 281 + * Network namespace (eg /proc/self/ns/net). 282 282 + * 283 283 + * **BPF_PROG_TYPE_LIRC_MODE2** 284 284 + * 285 285 + * LIRC device path (eg /dev/lircN). Requires the kernel 286 286 + * to be compiled with **CONFIG_BPF_LIRC_MODE2**. 287 287 + * 288 288 + * **BPF_PROG_TYPE_SK_SKB**, 289 289 + * **BPF_PROG_TYPE_SK_MSG** 290 290 + * 291 291 + * eBPF map of socket type (eg **BPF_MAP_TYPE_SOCKHASH**). 292 292 + * 293 293 + * Return 294 294 + * Returns zero on success. On error, -1 is returned and *errno* 295 295 + * is set appropriately. 296 296 + * 297 297 + * BPF_PROG_DETACH 298 298 + * Description 299 299 + * Detach the eBPF program associated with the *target_fd* at the 300 300 + * hook specified by *attach_type*. The program must have been 301 301 + * previously attached using **BPF_PROG_ATTACH**. 302 302 + * 303 303 + * Return 304 304 + * Returns zero on success. On error, -1 is returned and *errno* 305 305 + * is set appropriately. 306 306 + * 307 307 + * BPF_PROG_TEST_RUN 308 308 + * Description 309 309 + * Run the eBPF program associated with the *prog_fd* a *repeat* 310 310 + * number of times against a provided program context *ctx_in* and 311 311 + * data *data_in*, and return the modified program context 312 312 + * *ctx_out*, *data_out* (for example, packet data), result of the 313 313 + * execution *retval*, and *duration* of the test run. 314 314 + * 315 315 + * Return 316 316 + * Returns zero on success. On error, -1 is returned and *errno* 317 317 + * is set appropriately. 318 318 + * 319 319 + * **ENOSPC** 320 320 + * Either *data_size_out* or *ctx_size_out* is too small. 321 321 + * **ENOTSUPP** 322 322 + * This command is not supported by the program type of 323 323 + * the program referred to by *prog_fd*. 324 324 + * 325 325 + * BPF_PROG_GET_NEXT_ID 326 326 + * Description 327 327 + * Fetch the next eBPF program currently loaded into the kernel. 328 328 + * 329 329 + * Looks for the eBPF program with an id greater than *start_id* 330 330 + * and updates *next_id* on success. If no other eBPF programs 331 331 + * remain with ids higher than *start_id*, returns -1 and sets 332 332 + * *errno* to **ENOENT**. 333 333 + * 334 334 + * Return 335 335 + * Returns zero on success. On error, or when no id remains, -1 336 336 + * is returned and *errno* is set appropriately. 337 337 + * 338 338 + * BPF_MAP_GET_NEXT_ID 339 339 + * Description 340 340 + * Fetch the next eBPF map currently loaded into the kernel. 341 341 + * 342 342 + * Looks for the eBPF map with an id greater than *start_id* 343 343 + * and updates *next_id* on success. If no other eBPF maps 344 344 + * remain with ids higher than *start_id*, returns -1 and sets 345 345 + * *errno* to **ENOENT**. 346 346 + * 347 347 + * Return 348 348 + * Returns zero on success. On error, or when no id remains, -1 349 349 + * is returned and *errno* is set appropriately. 350 350 + * 351 351 + * BPF_PROG_GET_FD_BY_ID 352 352 + * Description 353 353 + * Open a file descriptor for the eBPF program corresponding to 354 354 + * *prog_id*. 355 355 + * 356 356 + * Return 357 357 + * A new file descriptor (a nonnegative integer), or -1 if an 358 358 + * error occurred (in which case, *errno* is set appropriately). 359 359 + * 360 360 + * BPF_MAP_GET_FD_BY_ID 361 361 + * Description 362 362 + * Open a file descriptor for the eBPF map corresponding to 363 363 + * *map_id*. 364 364 + * 365 365 + * Return 366 366 + * A new file descriptor (a nonnegative integer), or -1 if an 367 367 + * error occurred (in which case, *errno* is set appropriately). 368 368 + * 369 369 + * BPF_OBJ_GET_INFO_BY_FD 370 370 + * Description 371 371 + * Obtain information about the eBPF object corresponding to 372 372 + * *bpf_fd*. 373 373 + * 374 374 + * Populates up to *info_len* bytes of *info*, which will be in 375 375 + * one of the following formats depending on the eBPF object type 376 376 + * of *bpf_fd*: 377 377 + * 378 378 + * * **struct bpf_prog_info** 379 379 + * * **struct bpf_map_info** 380 380 + * * **struct bpf_btf_info** 381 381 + * * **struct bpf_link_info** 382 382 + * 383 383 + * Return 384 384 + * Returns zero on success. On error, -1 is returned and *errno* 385 385 + * is set appropriately. 386 386 + * 387 387 + * BPF_PROG_QUERY 388 388 + * Description 389 389 + * Obtain information about eBPF programs associated with the 390 390 + * specified *attach_type* hook. 391 391 + * 392 392 + * The *target_fd* must be a valid file descriptor for a kernel 393 393 + * object which depends on the attach type of *attach_bpf_fd*: 394 394 + * 395 395 + * **BPF_PROG_TYPE_CGROUP_DEVICE**, 396 396 + * **BPF_PROG_TYPE_CGROUP_SKB**, 397 397 + * **BPF_PROG_TYPE_CGROUP_SOCK**, 398 398 + * **BPF_PROG_TYPE_CGROUP_SOCK_ADDR**, 399 399 + * **BPF_PROG_TYPE_CGROUP_SOCKOPT**, 400 400 + * **BPF_PROG_TYPE_CGROUP_SYSCTL**, 401 401 + * **BPF_PROG_TYPE_SOCK_OPS** 402 402 + * 403 403 + * Control Group v2 hierarchy with the eBPF controller 404 404 + * enabled. Requires the kernel to be compiled with 405 405 + * **CONFIG_CGROUP_BPF**. 406 406 + * 407 407 + * **BPF_PROG_TYPE_FLOW_DISSECTOR** 408 408 + * 409 409 + * Network namespace (eg /proc/self/ns/net). 410 410 + * 411 411 + * **BPF_PROG_TYPE_LIRC_MODE2** 412 412 + * 413 413 + * LIRC device path (eg /dev/lircN). Requires the kernel 414 414 + * to be compiled with **CONFIG_BPF_LIRC_MODE2**. 415 415 + * 416 416 + * **BPF_PROG_QUERY** always fetches the number of programs 417 417 + * attached and the *attach_flags* which were used to attach those 418 418 + * programs. Additionally, if *prog_ids* is nonzero and the number 419 419 + * of attached programs is less than *prog_cnt*, populates 420 420 + * *prog_ids* with the eBPF program ids of the programs attached 421 421 + * at *target_fd*. 422 422 + * 423 423 + * The following flags may alter the result: 424 424 + * 425 425 + * **BPF_F_QUERY_EFFECTIVE** 426 426 + * Only return information regarding programs which are 427 427 + * currently effective at the specified *target_fd*. 428 428 + * 429 429 + * Return 430 430 + * Returns zero on success. On error, -1 is returned and *errno* 431 431 + * is set appropriately. 432 432 + * 433 433 + * BPF_RAW_TRACEPOINT_OPEN 434 434 + * Description 435 435 + * Attach an eBPF program to a tracepoint *name* to access kernel 436 436 + * internal arguments of the tracepoint in their raw form. 437 437 + * 438 438 + * The *prog_fd* must be a valid file descriptor associated with 439 439 + * a loaded eBPF program of type **BPF_PROG_TYPE_RAW_TRACEPOINT**. 440 440 + * 441 441 + * No ABI guarantees are made about the content of tracepoint 442 442 + * arguments exposed to the corresponding eBPF program. 443 443 + * 444 444 + * Applying **close**\ (2) to the file descriptor returned by 445 445 + * **BPF_RAW_TRACEPOINT_OPEN** will delete the map (but see NOTES). 446 446 + * 447 447 + * Return 448 448 + * A new file descriptor (a nonnegative integer), or -1 if an 449 449 + * error occurred (in which case, *errno* is set appropriately). 450 450 + * 451 451 + * BPF_BTF_LOAD 452 452 + * Description 453 453 + * Verify and load BPF Type Format (BTF) metadata into the kernel, 454 454 + * returning a new file descriptor associated with the metadata. 455 455 + * BTF is described in more detail at 456 456 + * https://www.kernel.org/doc/html/latest/bpf/btf.html. 457 457 + * 458 458 + * The *btf* parameter must point to valid memory providing 459 459 + * *btf_size* bytes of BTF binary metadata. 460 460 + * 461 461 + * The returned file descriptor can be passed to other **bpf**\ () 462 462 + * subcommands such as **BPF_PROG_LOAD** or **BPF_MAP_CREATE** to 463 463 + * associate the BTF with those objects. 464 464 + * 465 465 + * Similar to **BPF_PROG_LOAD**, **BPF_BTF_LOAD** has optional 466 466 + * parameters to specify a *btf_log_buf*, *btf_log_size* and 467 467 + * *btf_log_level* which allow the kernel to return freeform log 468 468 + * output regarding the BTF verification process. 469 469 + * 470 470 + * Return 471 471 + * A new file descriptor (a nonnegative integer), or -1 if an 472 472 + * error occurred (in which case, *errno* is set appropriately). 473 473 + * 474 474 + * BPF_BTF_GET_FD_BY_ID 475 475 + * Description 476 476 + * Open a file descriptor for the BPF Type Format (BTF) 477 477 + * corresponding to *btf_id*. 478 478 + * 479 479 + * Return 480 480 + * A new file descriptor (a nonnegative integer), or -1 if an 481 481 + * error occurred (in which case, *errno* is set appropriately). 482 482 + * 483 483 + * BPF_TASK_FD_QUERY 484 484 + * Description 485 485 + * Obtain information about eBPF programs associated with the 486 486 + * target process identified by *pid* and *fd*. 487 487 + * 488 488 + * If the *pid* and *fd* are associated with a tracepoint, kprobe 489 489 + * or uprobe perf event, then the *prog_id* and *fd_type* will 490 490 + * be populated with the eBPF program id and file descriptor type 491 491 + * of type **bpf_task_fd_type**. If associated with a kprobe or 492 492 + * uprobe, the *probe_offset* and *probe_addr* will also be 493 493 + * populated. Optionally, if *buf* is provided, then up to 494 494 + * *buf_len* bytes of *buf* will be populated with the name of 495 495 + * the tracepoint, kprobe or uprobe. 496 496 + * 497 497 + * The resulting *prog_id* may be introspected in deeper detail 498 498 + * using **BPF_PROG_GET_FD_BY_ID** and **BPF_OBJ_GET_INFO_BY_FD**. 499 499 + * 500 500 + * Return 501 501 + * Returns zero on success. On error, -1 is returned and *errno* 502 502 + * is set appropriately. 503 503 + * 504 504 + * BPF_MAP_LOOKUP_AND_DELETE_ELEM 505 505 + * Description 506 506 + * Look up an element with the given *key* in the map referred to 507 507 + * by the file descriptor *fd*, and if found, delete the element. 508 508 + * 509 509 + * The **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map types 510 510 + * implement this command as a "pop" operation, deleting the top 511 511 + * element rather than one corresponding to *key*. 512 512 + * The *key* and *key_len* parameters should be zeroed when 513 513 + * issuing this operation for these map types. 514 514 + * 515 515 + * This command is only valid for the following map types: 516 516 + * * **BPF_MAP_TYPE_QUEUE** 517 517 + * * **BPF_MAP_TYPE_STACK** 518 518 + * 519 519 + * Return 520 520 + * Returns zero on success. On error, -1 is returned and *errno* 521 521 + * is set appropriately. 522 522 + * 523 523 + * BPF_MAP_FREEZE 524 524 + * Description 525 525 + * Freeze the permissions of the specified map. 526 526 + * 527 527 + * Write permissions may be frozen by passing zero *flags*. 528 528 + * Upon success, no future syscall invocations may alter the 529 529 + * map state of *map_fd*. Write operations from eBPF programs 530 530 + * are still possible for a frozen map. 531 531 + * 532 532 + * Not supported for maps of type **BPF_MAP_TYPE_STRUCT_OPS**. 533 533 + * 534 534 + * Return 535 535 + * Returns zero on success. On error, -1 is returned and *errno* 536 536 + * is set appropriately. 537 537 + * 538 538 + * BPF_BTF_GET_NEXT_ID 539 539 + * Description 540 540 + * Fetch the next BPF Type Format (BTF) object currently loaded 541 541 + * into the kernel. 542 542 + * 543 543 + * Looks for the BTF object with an id greater than *start_id* 544 544 + * and updates *next_id* on success. If no other BTF objects 545 545 + * remain with ids higher than *start_id*, returns -1 and sets 546 546 + * *errno* to **ENOENT**. 547 547 + * 548 548 + * Return 549 549 + * Returns zero on success. On error, or when no id remains, -1 550 550 + * is returned and *errno* is set appropriately. 551 551 + * 552 552 + * BPF_MAP_LOOKUP_BATCH 553 553 + * Description 554 554 + * Iterate and fetch multiple elements in a map. 555 555 + * 556 556 + * Two opaque values are used to manage batch operations, 557 557 + * *in_batch* and *out_batch*. Initially, *in_batch* must be set 558 558 + * to NULL to begin the batched operation. After each subsequent 559 559 + * **BPF_MAP_LOOKUP_BATCH**, the caller should pass the resultant 560 560 + * *out_batch* as the *in_batch* for the next operation to 561 561 + * continue iteration from the current point. 562 562 + * 563 563 + * The *keys* and *values* are output parameters which must point 564 564 + * to memory large enough to hold *count* items based on the key 565 565 + * and value size of the map *map_fd*. The *keys* buffer must be 566 566 + * of *key_size* * *count*. The *values* buffer must be of 567 567 + * *value_size* * *count*. 568 568 + * 569 569 + * The *elem_flags* argument may be specified as one of the 570 570 + * following: 571 571 + * 572 572 + * **BPF_F_LOCK** 573 573 + * Look up the value of a spin-locked map without 574 574 + * returning the lock. This must be specified if the 575 575 + * elements contain a spinlock. 576 576 + * 577 577 + * On success, *count* elements from the map are copied into the 578 578 + * user buffer, with the keys copied into *keys* and the values 579 579 + * copied into the corresponding indices in *values*. 580 580 + * 581 581 + * If an error is returned and *errno* is not **EFAULT**, *count* 582 582 + * is set to the number of successfully processed elements. 583 583 + * 584 584 + * Return 585 585 + * Returns zero on success. On error, -1 is returned and *errno* 586 586 + * is set appropriately. 587 587 + * 588 588 + * May set *errno* to **ENOSPC** to indicate that *keys* or 589 589 + * *values* is too small to dump an entire bucket during 590 590 + * iteration of a hash-based map type. 591 591 + * 592 592 + * BPF_MAP_LOOKUP_AND_DELETE_BATCH 593 593 + * Description 594 594 + * Iterate and delete all elements in a map. 595 595 + * 596 596 + * This operation has the same behavior as 597 597 + * **BPF_MAP_LOOKUP_BATCH** with two exceptions: 598 598 + * 599 599 + * * Every element that is successfully returned is also deleted 600 600 + * from the map. This is at least *count* elements. Note that 601 601 + * *count* is both an input and an output parameter. 602 602 + * * Upon returning with *errno* set to **EFAULT**, up to 603 603 + * *count* elements may be deleted without returning the keys 604 604 + * and values of the deleted elements. 605 605 + * 606 606 + * Return 607 607 + * Returns zero on success. On error, -1 is returned and *errno* 608 608 + * is set appropriately. 609 609 + * 610 610 + * BPF_MAP_UPDATE_BATCH 611 611 + * Description 612 612 + * Update multiple elements in a map by *key*. 613 613 + * 614 614 + * The *keys* and *values* are input parameters which must point 615 615 + * to memory large enough to hold *count* items based on the key 616 616 + * and value size of the map *map_fd*. The *keys* buffer must be 617 617 + * of *key_size* * *count*. The *values* buffer must be of 618 618 + * *value_size* * *count*. 619 619 + * 620 620 + * Each element specified in *keys* is sequentially updated to the 621 621 + * value in the corresponding index in *values*. The *in_batch* 622 622 + * and *out_batch* parameters are ignored and should be zeroed. 623 623 + * 624 624 + * The *elem_flags* argument should be specified as one of the 625 625 + * following: 626 626 + * 627 627 + * **BPF_ANY** 628 628 + * Create new elements or update a existing elements. 629 629 + * **BPF_NOEXIST** 630 630 + * Create new elements only if they do not exist. 631 631 + * **BPF_EXIST** 632 632 + * Update existing elements. 633 633 + * **BPF_F_LOCK** 634 634 + * Update spin_lock-ed map elements. This must be 635 635 + * specified if the map value contains a spinlock. 636 636 + * 637 637 + * On success, *count* elements from the map are updated. 638 638 + * 639 639 + * If an error is returned and *errno* is not **EFAULT**, *count* 640 640 + * is set to the number of successfully processed elements. 641 641 + * 642 642 + * Return 643 643 + * Returns zero on success. On error, -1 is returned and *errno* 644 644 + * is set appropriately. 645 645 + * 646 646 + * May set *errno* to **EINVAL**, **EPERM**, **ENOMEM**, or 647 647 + * **E2BIG**. **E2BIG** indicates that the number of elements in 648 648 + * the map reached the *max_entries* limit specified at map 649 649 + * creation time. 650 650 + * 651 651 + * May set *errno* to one of the following error codes under 652 652 + * specific circumstances: 653 653 + * 654 654 + * **EEXIST** 655 655 + * If *flags* specifies **BPF_NOEXIST** and the element 656 656 + * with *key* already exists in the map. 657 657 + * **ENOENT** 658 658 + * If *flags* specifies **BPF_EXIST** and the element with 659 659 + * *key* does not exist in the map. 660 660 + * 661 661 + * BPF_MAP_DELETE_BATCH 662 662 + * Description 663 663 + * Delete multiple elements in a map by *key*. 664 664 + * 665 665 + * The *keys* parameter is an input parameter which must point 666 666 + * to memory large enough to hold *count* items based on the key 667 667 + * size of the map *map_fd*, that is, *key_size* * *count*. 668 668 + * 669 669 + * Each element specified in *keys* is sequentially deleted. The 670 670 + * *in_batch*, *out_batch*, and *values* parameters are ignored 671 671 + * and should be zeroed. 672 672 + * 673 673 + * The *elem_flags* argument may be specified as one of the 674 674 + * following: 675 675 + * 676 676 + * **BPF_F_LOCK** 677 677 + * Look up the value of a spin-locked map without 678 678 + * returning the lock. This must be specified if the 679 679 + * elements contain a spinlock. 680 680 + * 681 681 + * On success, *count* elements from the map are updated. 682 682 + * 683 683 + * If an error is returned and *errno* is not **EFAULT**, *count* 684 684 + * is set to the number of successfully processed elements. If 685 685 + * *errno* is **EFAULT**, up to *count* elements may be been 686 686 + * deleted. 687 687 + * 688 688 + * Return 689 689 + * Returns zero on success. On error, -1 is returned and *errno* 690 690 + * is set appropriately. 691 691 + * 692 692 + * BPF_LINK_CREATE 693 693 + * Description 694 694 + * Attach an eBPF program to a *target_fd* at the specified 695 695 + * *attach_type* hook and return a file descriptor handle for 696 696 + * managing the link. 697 697 + * 698 698 + * Return 699 699 + * A new file descriptor (a nonnegative integer), or -1 if an 700 700 + * error occurred (in which case, *errno* is set appropriately). 701 701 + * 702 702 + * BPF_LINK_UPDATE 703 703 + * Description 704 704 + * Update the eBPF program in the specified *link_fd* to 705 705 + * *new_prog_fd*. 706 706 + * 707 707 + * Return 708 708 + * Returns zero on success. On error, -1 is returned and *errno* 709 709 + * is set appropriately. 710 710 + * 711 711 + * BPF_LINK_GET_FD_BY_ID 712 712 + * Description 713 713 + * Open a file descriptor for the eBPF Link corresponding to 714 714 + * *link_id*. 715 715 + * 716 716 + * Return 717 717 + * A new file descriptor (a nonnegative integer), or -1 if an 718 718 + * error occurred (in which case, *errno* is set appropriately). 719 719 + * 720 720 + * BPF_LINK_GET_NEXT_ID 721 721 + * Description 722 722 + * Fetch the next eBPF link currently loaded into the kernel. 723 723 + * 724 724 + * Looks for the eBPF link with an id greater than *start_id* 725 725 + * and updates *next_id* on success. If no other eBPF links 726 726 + * remain with ids higher than *start_id*, returns -1 and sets 727 727 + * *errno* to **ENOENT**. 728 728 + * 729 729 + * Return 730 730 + * Returns zero on success. On error, or when no id remains, -1 731 731 + * is returned and *errno* is set appropriately. 732 732 + * 733 733 + * BPF_ENABLE_STATS 734 734 + * Description 735 735 + * Enable eBPF runtime statistics gathering. 736 736 + * 737 737 + * Runtime statistics gathering for the eBPF runtime is disabled 738 738 + * by default to minimize the corresponding performance overhead. 739 739 + * This command enables statistics globally. 740 740 + * 741 741 + * Multiple programs may independently enable statistics. 742 742 + * After gathering the desired statistics, eBPF runtime statistics 743 743 + * may be disabled again by calling **close**\ (2) for the file 744 744 + * descriptor returned by this function. Statistics will only be 745 745 + * disabled system-wide when all outstanding file descriptors 746 746 + * returned by prior calls for this subcommand are closed. 747 747 + * 748 748 + * Return 749 749 + * A new file descriptor (a nonnegative integer), or -1 if an 750 750 + * error occurred (in which case, *errno* is set appropriately). 751 751 + * 752 752 + * BPF_ITER_CREATE 753 753 + * Description 754 754 + * Create an iterator on top of the specified *link_fd* (as 755 755 + * previously created using **BPF_LINK_CREATE**) and return a 756 756 + * file descriptor that can be used to trigger the iteration. 757 757 + * 758 758 + * If the resulting file descriptor is pinned to the filesystem 759 759 + * using **BPF_OBJ_PIN**, then subsequent **read**\ (2) syscalls 760 760 + * for that path will trigger the iterator to read kernel state 761 761 + * using the eBPF program attached to *link_fd*. 762 762 + * 763 763 + * Return 764 764 + * A new file descriptor (a nonnegative integer), or -1 if an 765 765 + * error occurred (in which case, *errno* is set appropriately). 766 766 + * 767 767 + * BPF_LINK_DETACH 768 768 + * Description 769 769 + * Forcefully detach the specified *link_fd* from its 770 770 + * corresponding attachment point. 771 771 + * 772 772 + * Return 773 773 + * Returns zero on success. On error, -1 is returned and *errno* 774 774 + * is set appropriately. 775 775 + * 776 776 + * BPF_PROG_BIND_MAP 777 777 + * Description 778 778 + * Bind a map to the lifetime of an eBPF program. 779 779 + * 780 780 + * The map identified by *map_fd* is bound to the program 781 781 + * identified by *prog_fd* and only released when *prog_fd* is 782 782 + * released. This may be used in cases where metadata should be 783 783 + * associated with a program which otherwise does not contain any 784 784 + * references to the map (for example, embedded in the eBPF 785 785 + * program instructions). 786 786 + * 787 787 + * Return 788 788 + * Returns zero on success. On error, -1 is returned and *errno* 789 789 + * is set appropriately. 790 790 + * 791 791 + * NOTES 792 792 + * eBPF objects (maps and programs) can be shared between processes. 793 793 + * 794 794 + * * After **fork**\ (2), the child inherits file descriptors 795 795 + * referring to the same eBPF objects. 796 796 + * * File descriptors referring to eBPF objects can be transferred over 797 797 + * **unix**\ (7) domain sockets. 798 798 + * * File descriptors referring to eBPF objects can be duplicated in the 799 799 + * usual way, using **dup**\ (2) and similar calls. 800 800 + * * File descriptors referring to eBPF objects can be pinned to the 801 801 + * filesystem using the **BPF_OBJ_PIN** command of **bpf**\ (2). 802 802 + * 803 803 + * An eBPF object is deallocated only after all file descriptors referring 804 804 + * to the object have been closed and no references remain pinned to the 805 805 + * filesystem or attached (for example, bound to a program or device). 806 806 + */ 97 807 enum bpf_cmd { 98 808 BPF_MAP_CREATE, 99 809 BPF_MAP_LOOKUP_ELEM, ··· 1103 393 * is struct/union. 1104 394 */ 1105 395 #define BPF_PSEUDO_BTF_ID 3 396 396 + /* insn[0].src_reg: BPF_PSEUDO_FUNC 397 397 + * insn[0].imm: insn offset to the func 398 398 + * insn[1].imm: 0 399 399 + * insn[0].off: 0 400 400 + * insn[1].off: 0 401 401 + * ldimm64 rewrite: address of the function 402 402 + * verifier type: PTR_TO_FUNC. 403 403 + */ 404 404 + #define BPF_PSEUDO_FUNC 4 1106 405 1107 406 /* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative 1108 407 * offset to another bpf function ··· 1439 720 * parsed and used to produce a manual page. The workflow is the following, 1440 721 * and requires the rst2man utility: 1441 722 * 1442 1442 - * $ ./scripts/bpf_helpers_doc.py \ 723 723 + * $ ./scripts/bpf_doc.py \ 1443 724 * --filename include/uapi/linux/bpf.h > /tmp/bpf-helpers.rst 1444 725 * $ rst2man /tmp/bpf-helpers.rst > /tmp/bpf-helpers.7 1445 726 * $ man /tmp/bpf-helpers.7 ··· 2483 1764 * * **BPF_F_ADJ_ROOM_ENCAP_L2**\ (*len*): 2484 1765 * Use with ENCAP_L3/L4 flags to further specify the tunnel 2485 1766 * type; *len* is the length of the inner MAC header. 1767 1767 + * 1768 1768 + * * **BPF_F_ADJ_ROOM_ENCAP_L2_ETH**: 1769 1769 + * Use with BPF_F_ADJ_ROOM_ENCAP_L2 flag to further specify the 1770 1770 + * L2 type as Ethernet. 2486 1771 * 2487 1772 * A call to this helper is susceptible to change the underlying 2488 1773 * packet buffer. Therefore, at load time, all checks on pointers ··· 4632 3909 * * **BPF_MTU_CHK_RET_FRAG_NEEDED** 4633 3910 * * **BPF_MTU_CHK_RET_SEGS_TOOBIG** 4634 3911 * 3912 3912 + * long bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, void *callback_ctx, u64 flags) 3913 3913 + * Description 3914 3914 + * For each element in **map**, call **callback_fn** function with 3915 3915 + * **map**, **callback_ctx** and other map-specific parameters. 3916 3916 + * The **callback_fn** should be a static function and 3917 3917 + * the **callback_ctx** should be a pointer to the stack. 3918 3918 + * The **flags** is used to control certain aspects of the helper. 3919 3919 + * Currently, the **flags** must be 0. 3920 3920 + * 3921 3921 + * The following are a list of supported map types and their 3922 3922 + * respective expected callback signatures: 3923 3923 + * 3924 3924 + * BPF_MAP_TYPE_HASH, BPF_MAP_TYPE_PERCPU_HASH, 3925 3925 + * BPF_MAP_TYPE_LRU_HASH, BPF_MAP_TYPE_LRU_PERCPU_HASH, 3926 3926 + * BPF_MAP_TYPE_ARRAY, BPF_MAP_TYPE_PERCPU_ARRAY 3927 3927 + * 3928 3928 + * long (\*callback_fn)(struct bpf_map \*map, const void \*key, void \*value, void \*ctx); 3929 3929 + * 3930 3930 + * For per_cpu maps, the map_value is the value on the cpu where the 3931 3931 + * bpf_prog is running. 3932 3932 + * 3933 3933 + * If **callback_fn** return 0, the helper will continue to the next 3934 3934 + * element. If return value is 1, the helper will skip the rest of 3935 3935 + * elements and return. Other return values are not used now. 3936 3936 + * 3937 3937 + * Return 3938 3938 + * The number of traversed map elements for success, **-EINVAL** for 3939 3939 + * invalid **flags**. 4635 3940 */ 4636 3941 #define __BPF_FUNC_MAPPER(FN) \ 4637 3942 FN(unspec), \ ··· 4826 4075 FN(ima_inode_hash), \ 4827 4076 FN(sock_from_file), \ 4828 4077 FN(check_mtu), \ 4078 4078 + FN(for_each_map_elem), \ 4829 4079 /* */ 4830 4080 4831 4081 /* integer value in 'imm' field of BPF_CALL instruction selects which helper ··· 4920 4168 BPF_F_ADJ_ROOM_ENCAP_L4_GRE = (1ULL << 3), 4921 4169 BPF_F_ADJ_ROOM_ENCAP_L4_UDP = (1ULL << 4), 4922 4170 BPF_F_ADJ_ROOM_NO_CSUM_RESET = (1ULL << 5), 4171 4171 + BPF_F_ADJ_ROOM_ENCAP_L2_ETH = (1ULL << 6), 4923 4172 }; 4924 4173 4925 4174 enum { ··· 5958 5205 5959 5206 /* User accessible data for SK_LOOKUP programs. Add new fields at the end. */ 5960 5207 struct bpf_sk_lookup { 5961 5961 - __bpf_md_ptr(struct bpf_sock *, sk); /* Selected socket */ 5208 5208 + union { 5209 5209 + __bpf_md_ptr(struct bpf_sock *, sk); /* Selected socket */ 5210 5210 + __u64 cookie; /* Non-zero if socket was selected in PROG_TEST_RUN */ 5211 5211 + }; 5962 5212 5963 5213 __u32 family; /* Protocol family (AF_INET, AF_INET6) */ 5964 5214 __u32 protocol; /* IP protocol (IPPROTO_TCP, IPPROTO_UDP) */

+3 -2

include/uapi/linux/btf.h

reviewed

··· 52 52 }; 53 53 }; 54 54 55 55 - #define BTF_INFO_KIND(info) (((info) >> 24) & 0x0f) 55 55 + #define BTF_INFO_KIND(info) (((info) >> 24) & 0x1f) 56 56 #define BTF_INFO_VLEN(info) ((info) & 0xffff) 57 57 #define BTF_INFO_KFLAG(info) ((info) >> 31) 58 58 ··· 72 72 #define BTF_KIND_FUNC_PROTO 13 /* Function Proto */ 73 73 #define BTF_KIND_VAR 14 /* Variable */ 74 74 #define BTF_KIND_DATASEC 15 /* Section */ 75 75 - #define BTF_KIND_MAX BTF_KIND_DATASEC 75 75 + #define BTF_KIND_FLOAT 16 /* Floating point */ 76 76 + #define BTF_KIND_MAX BTF_KIND_FLOAT 76 77 #define NR_BTF_KINDS (BTF_KIND_MAX + 1) 77 78 78 79 /* For some specific BTF_KIND, "struct btf_type" is immediately

+1

init/Kconfig

reviewed

··· 1709 1709 select BPF 1710 1710 select IRQ_WORK 1711 1711 select TASKS_TRACE_RCU 1712 1712 + select NET_SOCK_MSG if INET 1712 1713 default n 1713 1714 help 1714 1715 Enable the bpf() system call that allows to manipulate eBPF

+1 -2

kernel/bpf/Makefile

reviewed

··· 9 9 obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_iter.o map_iter.o task_iter.o prog_iter.o 10 10 obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o 11 11 obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o 12 12 + obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o 12 13 obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o 13 13 - obj-${CONFIG_BPF_LSM} += bpf_task_storage.o 14 14 obj-$(CONFIG_BPF_SYSCALL) += disasm.o 15 15 obj-$(CONFIG_BPF_JIT) += trampoline.o 16 16 obj-$(CONFIG_BPF_SYSCALL) += btf.o ··· 18 18 ifeq ($(CONFIG_NET),y) 19 19 obj-$(CONFIG_BPF_SYSCALL) += devmap.o 20 20 obj-$(CONFIG_BPF_SYSCALL) += cpumap.o 21 21 - obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o 22 21 obj-$(CONFIG_BPF_SYSCALL) += offload.o 23 22 obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o 24 23 endif

+40

kernel/bpf/arraymap.c

reviewed

··· 625 625 .seq_priv_size = sizeof(struct bpf_iter_seq_array_map_info), 626 626 }; 627 627 628 628 + static int bpf_for_each_array_elem(struct bpf_map *map, void *callback_fn, 629 629 + void *callback_ctx, u64 flags) 630 630 + { 631 631 + u32 i, key, num_elems = 0; 632 632 + struct bpf_array *array; 633 633 + bool is_percpu; 634 634 + u64 ret = 0; 635 635 + void *val; 636 636 + 637 637 + if (flags != 0) 638 638 + return -EINVAL; 639 639 + 640 640 + is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; 641 641 + array = container_of(map, struct bpf_array, map); 642 642 + if (is_percpu) 643 643 + migrate_disable(); 644 644 + for (i = 0; i < map->max_entries; i++) { 645 645 + if (is_percpu) 646 646 + val = this_cpu_ptr(array->pptrs[i]); 647 647 + else 648 648 + val = array->value + array->elem_size * i; 649 649 + num_elems++; 650 650 + key = i; 651 651 + ret = BPF_CAST_CALL(callback_fn)((u64)(long)map, 652 652 + (u64)(long)&key, (u64)(long)val, 653 653 + (u64)(long)callback_ctx, 0); 654 654 + /* return value: 0 - continue, 1 - stop and return */ 655 655 + if (ret) 656 656 + break; 657 657 + } 658 658 + 659 659 + if (is_percpu) 660 660 + migrate_enable(); 661 661 + return num_elems; 662 662 + } 663 663 + 628 664 static int array_map_btf_id; 629 665 const struct bpf_map_ops array_map_ops = { 630 666 .map_meta_equal = array_map_meta_equal, ··· 679 643 .map_check_btf = array_map_check_btf, 680 644 .map_lookup_batch = generic_map_lookup_batch, 681 645 .map_update_batch = generic_map_update_batch, 646 646 + .map_set_for_each_callback_args = map_set_for_each_callback_args, 647 647 + .map_for_each_callback = bpf_for_each_array_elem, 682 648 .map_btf_name = "bpf_array", 683 649 .map_btf_id = &array_map_btf_id, 684 650 .iter_seq_info = &iter_seq_info, ··· 698 660 .map_delete_elem = array_map_delete_elem, 699 661 .map_seq_show_elem = percpu_array_map_seq_show_elem, 700 662 .map_check_btf = array_map_check_btf, 663 663 + .map_set_for_each_callback_args = map_set_for_each_callback_args, 664 664 + .map_for_each_callback = bpf_for_each_array_elem, 701 665 .map_btf_name = "bpf_array", 702 666 .map_btf_id = &percpu_array_map_btf_id, 703 667 .iter_seq_info = &iter_seq_info,

+1 -1

kernel/bpf/bpf_inode_storage.c

reviewed

··· 237 237 238 238 smap = (struct bpf_local_storage_map *)map; 239 239 bpf_local_storage_cache_idx_free(&inode_cache, smap->cache_idx); 240 240 - bpf_local_storage_map_free(smap); 240 240 + bpf_local_storage_map_free(smap, NULL); 241 241 } 242 242 243 243 static int inode_storage_map_btf_id;

+16

kernel/bpf/bpf_iter.c

reviewed

··· 675 675 */ 676 676 return ret == 0 ? 0 : -EAGAIN; 677 677 } 678 678 + 679 679 + BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn, 680 680 + void *, callback_ctx, u64, flags) 681 681 + { 682 682 + return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags); 683 683 + } 684 684 + 685 685 + const struct bpf_func_proto bpf_for_each_map_elem_proto = { 686 686 + .func = bpf_for_each_map_elem, 687 687 + .gpl_only = false, 688 688 + .ret_type = RET_INTEGER, 689 689 + .arg1_type = ARG_CONST_MAP_PTR, 690 690 + .arg2_type = ARG_PTR_TO_FUNC, 691 691 + .arg3_type = ARG_PTR_TO_STACK_OR_NULL, 692 692 + .arg4_type = ARG_ANYTHING, 693 693 + };

+27 -12

kernel/bpf/bpf_local_storage.c

reviewed

··· 140 140 { 141 141 struct bpf_local_storage *local_storage; 142 142 bool free_local_storage = false; 143 143 + unsigned long flags; 143 144 144 145 if (unlikely(!selem_linked_to_storage(selem))) 145 146 /* selem has already been unlinked from sk */ 146 147 return; 147 148 148 149 local_storage = rcu_dereference(selem->local_storage); 149 149 - raw_spin_lock_bh(&local_storage->lock); 150 150 + raw_spin_lock_irqsave(&local_storage->lock, flags); 150 151 if (likely(selem_linked_to_storage(selem))) 151 152 free_local_storage = bpf_selem_unlink_storage_nolock( 152 153 local_storage, selem, true); 153 153 - raw_spin_unlock_bh(&local_storage->lock); 154 154 + raw_spin_unlock_irqrestore(&local_storage->lock, flags); 154 155 155 156 if (free_local_storage) 156 157 kfree_rcu(local_storage, rcu); ··· 168 167 { 169 168 struct bpf_local_storage_map *smap; 170 169 struct bpf_local_storage_map_bucket *b; 170 170 + unsigned long flags; 171 171 172 172 if (unlikely(!selem_linked_to_map(selem))) 173 173 /* selem has already be unlinked from smap */ ··· 176 174 177 175 smap = rcu_dereference(SDATA(selem)->smap); 178 176 b = select_bucket(smap, selem); 179 179 - raw_spin_lock_bh(&b->lock); 177 177 + raw_spin_lock_irqsave(&b->lock, flags); 180 178 if (likely(selem_linked_to_map(selem))) 181 179 hlist_del_init_rcu(&selem->map_node); 182 182 - raw_spin_unlock_bh(&b->lock); 180 180 + raw_spin_unlock_irqrestore(&b->lock, flags); 183 181 } 184 182 185 183 void bpf_selem_link_map(struct bpf_local_storage_map *smap, 186 184 struct bpf_local_storage_elem *selem) 187 185 { 188 186 struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem); 187 187 + unsigned long flags; 189 188 190 190 - raw_spin_lock_bh(&b->lock); 189 189 + raw_spin_lock_irqsave(&b->lock, flags); 191 190 RCU_INIT_POINTER(SDATA(selem)->smap, smap); 192 191 hlist_add_head_rcu(&selem->map_node, &b->list); 193 193 - raw_spin_unlock_bh(&b->lock); 192 192 + raw_spin_unlock_irqrestore(&b->lock, flags); 194 193 } 195 194 196 195 void bpf_selem_unlink(struct bpf_local_storage_elem *selem) ··· 227 224 228 225 sdata = SDATA(selem); 229 226 if (cacheit_lockit) { 227 227 + unsigned long flags; 228 228 + 230 229 /* spinlock is needed to avoid racing with the 231 230 * parallel delete. Otherwise, publishing an already 232 231 * deleted sdata to the cache will become a use-after-free 233 232 * problem in the next bpf_local_storage_lookup(). 234 233 */ 235 235 - raw_spin_lock_bh(&local_storage->lock); 234 234 + raw_spin_lock_irqsave(&local_storage->lock, flags); 236 235 if (selem_linked_to_storage(selem)) 237 236 rcu_assign_pointer(local_storage->cache[smap->cache_idx], 238 237 sdata); 239 239 - raw_spin_unlock_bh(&local_storage->lock); 238 238 + raw_spin_unlock_irqrestore(&local_storage->lock, flags); 240 239 } 241 240 242 241 return sdata; ··· 332 327 struct bpf_local_storage_data *old_sdata = NULL; 333 328 struct bpf_local_storage_elem *selem; 334 329 struct bpf_local_storage *local_storage; 330 330 + unsigned long flags; 335 331 int err; 336 332 337 333 /* BPF_EXIST and BPF_NOEXIST cannot be both set */ ··· 380 374 } 381 375 } 382 376 383 383 - raw_spin_lock_bh(&local_storage->lock); 377 377 + raw_spin_lock_irqsave(&local_storage->lock, flags); 384 378 385 379 /* Recheck local_storage->list under local_storage->lock */ 386 380 if (unlikely(hlist_empty(&local_storage->list))) { ··· 434 428 } 435 429 436 430 unlock: 437 437 - raw_spin_unlock_bh(&local_storage->lock); 431 431 + raw_spin_unlock_irqrestore(&local_storage->lock, flags); 438 432 return SDATA(selem); 439 433 440 434 unlock_err: 441 441 - raw_spin_unlock_bh(&local_storage->lock); 435 435 + raw_spin_unlock_irqrestore(&local_storage->lock, flags); 442 436 return ERR_PTR(err); 443 437 } 444 438 ··· 474 468 spin_unlock(&cache->idx_lock); 475 469 } 476 470 477 477 - void bpf_local_storage_map_free(struct bpf_local_storage_map *smap) 471 471 + void bpf_local_storage_map_free(struct bpf_local_storage_map *smap, 472 472 + int __percpu *busy_counter) 478 473 { 479 474 struct bpf_local_storage_elem *selem; 480 475 struct bpf_local_storage_map_bucket *b; ··· 504 497 while ((selem = hlist_entry_safe( 505 498 rcu_dereference_raw(hlist_first_rcu(&b->list)), 506 499 struct bpf_local_storage_elem, map_node))) { 500 500 + if (busy_counter) { 501 501 + migrate_disable(); 502 502 + __this_cpu_inc(*busy_counter); 503 503 + } 507 504 bpf_selem_unlink(selem); 505 505 + if (busy_counter) { 506 506 + __this_cpu_dec(*busy_counter); 507 507 + migrate_enable(); 508 508 + } 508 509 cond_resched_rcu(); 509 510 } 510 511 rcu_read_unlock();

-4

kernel/bpf/bpf_lsm.c

reviewed

··· 115 115 return &bpf_spin_lock_proto; 116 116 case BPF_FUNC_spin_unlock: 117 117 return &bpf_spin_unlock_proto; 118 118 - case BPF_FUNC_task_storage_get: 119 119 - return &bpf_task_storage_get_proto; 120 120 - case BPF_FUNC_task_storage_delete: 121 121 - return &bpf_task_storage_delete_proto; 122 118 case BPF_FUNC_bprm_opts_set: 123 119 return &bpf_bprm_opts_set_proto; 124 120 case BPF_FUNC_ima_inode_hash:

+61 -39

kernel/bpf/bpf_task_storage.c

reviewed

··· 15 15 #include <linux/bpf_local_storage.h> 16 16 #include <linux/filter.h> 17 17 #include <uapi/linux/btf.h> 18 18 - #include <linux/bpf_lsm.h> 19 18 #include <linux/btf_ids.h> 20 19 #include <linux/fdtable.h> 21 20 22 21 DEFINE_BPF_STORAGE_CACHE(task_cache); 23 22 23 23 + DEFINE_PER_CPU(int, bpf_task_storage_busy); 24 24 + 25 25 + static void bpf_task_storage_lock(void) 26 26 + { 27 27 + migrate_disable(); 28 28 + __this_cpu_inc(bpf_task_storage_busy); 29 29 + } 30 30 + 31 31 + static void bpf_task_storage_unlock(void) 32 32 + { 33 33 + __this_cpu_dec(bpf_task_storage_busy); 34 34 + migrate_enable(); 35 35 + } 36 36 + 37 37 + static bool bpf_task_storage_trylock(void) 38 38 + { 39 39 + migrate_disable(); 40 40 + if (unlikely(__this_cpu_inc_return(bpf_task_storage_busy) != 1)) { 41 41 + __this_cpu_dec(bpf_task_storage_busy); 42 42 + migrate_enable(); 43 43 + return false; 44 44 + } 45 45 + return true; 46 46 + } 47 47 + 24 48 static struct bpf_local_storage __rcu **task_storage_ptr(void *owner) 25 49 { 26 50 struct task_struct *task = owner; 27 27 - struct bpf_storage_blob *bsb; 28 51 29 29 - bsb = bpf_task(task); 30 30 - if (!bsb) 31 31 - return NULL; 32 32 - return &bsb->storage; 52 52 + return &task->bpf_storage; 33 53 } 34 54 35 55 static struct bpf_local_storage_data * ··· 58 38 { 59 39 struct bpf_local_storage *task_storage; 60 40 struct bpf_local_storage_map *smap; 61 61 - struct bpf_storage_blob *bsb; 62 41 63 63 - bsb = bpf_task(task); 64 64 - if (!bsb) 65 65 - return NULL; 66 66 - 67 67 - task_storage = rcu_dereference(bsb->storage); 42 42 + task_storage = rcu_dereference(task->bpf_storage); 68 43 if (!task_storage) 69 44 return NULL; 70 45 ··· 72 57 struct bpf_local_storage_elem *selem; 73 58 struct bpf_local_storage *local_storage; 74 59 bool free_task_storage = false; 75 75 - struct bpf_storage_blob *bsb; 76 60 struct hlist_node *n; 77 77 - 78 78 - bsb = bpf_task(task); 79 79 - if (!bsb) 80 80 - return; 61 61 + unsigned long flags; 81 62 82 63 rcu_read_lock(); 83 64 84 84 - local_storage = rcu_dereference(bsb->storage); 65 65 + local_storage = rcu_dereference(task->bpf_storage); 85 66 if (!local_storage) { 86 67 rcu_read_unlock(); 87 68 return; ··· 92 81 * when unlinking elem from the local_storage->list and 93 82 * the map's bucket->list. 94 83 */ 95 95 - raw_spin_lock_bh(&local_storage->lock); 84 84 + bpf_task_storage_lock(); 85 85 + raw_spin_lock_irqsave(&local_storage->lock, flags); 96 86 hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) { 97 87 /* Always unlink from map before unlinking from 98 88 * local_storage. ··· 102 90 free_task_storage = bpf_selem_unlink_storage_nolock( 103 91 local_storage, selem, false); 104 92 } 105 105 - raw_spin_unlock_bh(&local_storage->lock); 93 93 + raw_spin_unlock_irqrestore(&local_storage->lock, flags); 94 94 + bpf_task_storage_unlock(); 106 95 rcu_read_unlock(); 107 96 108 97 /* free_task_storage should always be true as long as ··· 136 123 goto out; 137 124 } 138 125 126 126 + bpf_task_storage_lock(); 139 127 sdata = task_storage_lookup(task, map, true); 128 128 + bpf_task_storage_unlock(); 140 129 put_pid(pid); 141 130 return sdata ? sdata->data : NULL; 142 131 out: ··· 165 150 */ 166 151 WARN_ON_ONCE(!rcu_read_lock_held()); 167 152 task = pid_task(pid, PIDTYPE_PID); 168 168 - if (!task || !task_storage_ptr(task)) { 153 153 + if (!task) { 169 154 err = -ENOENT; 170 155 goto out; 171 156 } 172 157 158 158 + bpf_task_storage_lock(); 173 159 sdata = bpf_local_storage_update( 174 160 task, (struct bpf_local_storage_map *)map, value, map_flags); 161 161 + bpf_task_storage_unlock(); 175 162 176 163 err = PTR_ERR_OR_ZERO(sdata); 177 164 out: ··· 216 199 goto out; 217 200 } 218 201 202 202 + bpf_task_storage_lock(); 219 203 err = task_storage_delete(task, map); 204 204 + bpf_task_storage_unlock(); 220 205 out: 221 206 put_pid(pid); 222 207 return err; ··· 232 213 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) 233 214 return (unsigned long)NULL; 234 215 235 235 - /* explicitly check that the task_storage_ptr is not 236 236 - * NULL as task_storage_lookup returns NULL in this case and 237 237 - * bpf_local_storage_update expects the owner to have a 238 238 - * valid storage pointer. 239 239 - */ 240 240 - if (!task || !task_storage_ptr(task)) 216 216 + if (!task) 217 217 + return (unsigned long)NULL; 218 218 + 219 219 + if (!bpf_task_storage_trylock()) 241 220 return (unsigned long)NULL; 242 221 243 222 sdata = task_storage_lookup(task, map, true); 244 223 if (sdata) 245 245 - return (unsigned long)sdata->data; 224 224 + goto unlock; 246 225 247 247 - /* This helper must only be called from places where the lifetime of the task 248 248 - * is guaranteed. Either by being refcounted or by being protected 249 249 - * by an RCU read-side critical section. 250 250 - */ 251 251 - if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) { 226 226 + /* only allocate new storage, when the task is refcounted */ 227 227 + if (refcount_read(&task->usage) && 228 228 + (flags & BPF_LOCAL_STORAGE_GET_F_CREATE)) 252 229 sdata = bpf_local_storage_update( 253 230 task, (struct bpf_local_storage_map *)map, value, 254 231 BPF_NOEXIST); 255 255 - return IS_ERR(sdata) ? (unsigned long)NULL : 256 256 - (unsigned long)sdata->data; 257 257 - } 258 232 259 259 - return (unsigned long)NULL; 233 233 + unlock: 234 234 + bpf_task_storage_unlock(); 235 235 + return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : 236 236 + (unsigned long)sdata->data; 260 237 } 261 238 262 239 BPF_CALL_2(bpf_task_storage_delete, struct bpf_map *, map, struct task_struct *, 263 240 task) 264 241 { 242 242 + int ret; 243 243 + 265 244 if (!task) 266 245 return -EINVAL; 246 246 + 247 247 + if (!bpf_task_storage_trylock()) 248 248 + return -EBUSY; 267 249 268 250 /* This helper must only be called from places where the lifetime of the task 269 251 * is guaranteed. Either by being refcounted or by being protected 270 252 * by an RCU read-side critical section. 271 253 */ 272 272 - return task_storage_delete(task, map); 254 254 + ret = task_storage_delete(task, map); 255 255 + bpf_task_storage_unlock(); 256 256 + return ret; 273 257 } 274 258 275 259 static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key) ··· 298 276 299 277 smap = (struct bpf_local_storage_map *)map; 300 278 bpf_local_storage_cache_idx_free(&task_cache, smap->cache_idx); 301 301 - bpf_local_storage_map_free(smap); 279 279 + bpf_local_storage_map_free(smap, &bpf_task_storage_busy); 302 280 } 303 281 304 282 static int task_storage_map_btf_id;

+95 -13

kernel/bpf/btf.c

reviewed

··· 173 173 #define BITS_ROUNDUP_BYTES(bits) \ 174 174 (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) 175 175 176 176 - #define BTF_INFO_MASK 0x8f00ffff 176 176 + #define BTF_INFO_MASK 0x9f00ffff 177 177 #define BTF_INT_MASK 0x0fffffff 178 178 #define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE) 179 179 #define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET) ··· 280 280 [BTF_KIND_FUNC_PROTO] = "FUNC_PROTO", 281 281 [BTF_KIND_VAR] = "VAR", 282 282 [BTF_KIND_DATASEC] = "DATASEC", 283 283 + [BTF_KIND_FLOAT] = "FLOAT", 283 284 }; 284 285 285 286 static const char *btf_type_str(const struct btf_type *t) ··· 575 574 case BTF_KIND_UNION: 576 575 case BTF_KIND_ENUM: 577 576 case BTF_KIND_DATASEC: 577 577 + case BTF_KIND_FLOAT: 578 578 return true; 579 579 } 580 580 ··· 1706 1704 case BTF_KIND_STRUCT: 1707 1705 case BTF_KIND_UNION: 1708 1706 case BTF_KIND_ENUM: 1707 1707 + case BTF_KIND_FLOAT: 1709 1708 size = type->size; 1710 1709 goto resolved; 1711 1710 ··· 1852 1849 return -EINVAL; 1853 1850 } 1854 1851 1855 1855 - /* Used for ptr, array and struct/union type members. 1852 1852 + /* Used for ptr, array struct/union and float type members. 1856 1853 * int, enum and modifier types have their specific callback functions. 1857 1854 */ 1858 1855 static int btf_generic_check_kflag_member(struct btf_verifier_env *env, ··· 3678 3675 .show = btf_datasec_show, 3679 3676 }; 3680 3677 3678 3678 + static s32 btf_float_check_meta(struct btf_verifier_env *env, 3679 3679 + const struct btf_type *t, 3680 3680 + u32 meta_left) 3681 3681 + { 3682 3682 + if (btf_type_vlen(t)) { 3683 3683 + btf_verifier_log_type(env, t, "vlen != 0"); 3684 3684 + return -EINVAL; 3685 3685 + } 3686 3686 + 3687 3687 + if (btf_type_kflag(t)) { 3688 3688 + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); 3689 3689 + return -EINVAL; 3690 3690 + } 3691 3691 + 3692 3692 + if (t->size != 2 && t->size != 4 && t->size != 8 && t->size != 12 && 3693 3693 + t->size != 16) { 3694 3694 + btf_verifier_log_type(env, t, "Invalid type_size"); 3695 3695 + return -EINVAL; 3696 3696 + } 3697 3697 + 3698 3698 + btf_verifier_log_type(env, t, NULL); 3699 3699 + 3700 3700 + return 0; 3701 3701 + } 3702 3702 + 3703 3703 + static int btf_float_check_member(struct btf_verifier_env *env, 3704 3704 + const struct btf_type *struct_type, 3705 3705 + const struct btf_member *member, 3706 3706 + const struct btf_type *member_type) 3707 3707 + { 3708 3708 + u64 start_offset_bytes; 3709 3709 + u64 end_offset_bytes; 3710 3710 + u64 misalign_bits; 3711 3711 + u64 align_bytes; 3712 3712 + u64 align_bits; 3713 3713 + 3714 3714 + /* Different architectures have different alignment requirements, so 3715 3715 + * here we check only for the reasonable minimum. This way we ensure 3716 3716 + * that types after CO-RE can pass the kernel BTF verifier. 3717 3717 + */ 3718 3718 + align_bytes = min_t(u64, sizeof(void *), member_type->size); 3719 3719 + align_bits = align_bytes * BITS_PER_BYTE; 3720 3720 + div64_u64_rem(member->offset, align_bits, &misalign_bits); 3721 3721 + if (misalign_bits) { 3722 3722 + btf_verifier_log_member(env, struct_type, member, 3723 3723 + "Member is not properly aligned"); 3724 3724 + return -EINVAL; 3725 3725 + } 3726 3726 + 3727 3727 + start_offset_bytes = member->offset / BITS_PER_BYTE; 3728 3728 + end_offset_bytes = start_offset_bytes + member_type->size; 3729 3729 + if (end_offset_bytes > struct_type->size) { 3730 3730 + btf_verifier_log_member(env, struct_type, member, 3731 3731 + "Member exceeds struct_size"); 3732 3732 + return -EINVAL; 3733 3733 + } 3734 3734 + 3735 3735 + return 0; 3736 3736 + } 3737 3737 + 3738 3738 + static void btf_float_log(struct btf_verifier_env *env, 3739 3739 + const struct btf_type *t) 3740 3740 + { 3741 3741 + btf_verifier_log(env, "size=%u", t->size); 3742 3742 + } 3743 3743 + 3744 3744 + static const struct btf_kind_operations float_ops = { 3745 3745 + .check_meta = btf_float_check_meta, 3746 3746 + .resolve = btf_df_resolve, 3747 3747 + .check_member = btf_float_check_member, 3748 3748 + .check_kflag_member = btf_generic_check_kflag_member, 3749 3749 + .log_details = btf_float_log, 3750 3750 + .show = btf_df_show, 3751 3751 + }; 3752 3752 + 3681 3753 static int btf_func_proto_check(struct btf_verifier_env *env, 3682 3754 const struct btf_type *t) 3683 3755 { ··· 3886 3808 [BTF_KIND_FUNC_PROTO] = &func_proto_ops, 3887 3809 [BTF_KIND_VAR] = &var_ops, 3888 3810 [BTF_KIND_DATASEC] = &datasec_ops, 3811 3811 + [BTF_KIND_FLOAT] = &float_ops, 3889 3812 }; 3890 3813 3891 3814 static s32 btf_check_meta(struct btf_verifier_env *env, ··· 4671 4592 } 4672 4593 arg = off / 8; 4673 4594 args = (const struct btf_param *)(t + 1); 4674 4674 - /* if (t == NULL) Fall back to default BPF prog with 5 u64 arguments */ 4675 4675 - nr_args = t ? btf_type_vlen(t) : 5; 4595 4595 + /* if (t == NULL) Fall back to default BPF prog with 4596 4596 + * MAX_BPF_FUNC_REG_ARGS u64 arguments. 4597 4597 + */ 4598 4598 + nr_args = t ? btf_type_vlen(t) : MAX_BPF_FUNC_REG_ARGS; 4676 4599 if (prog->aux->attach_btf_trace) { 4677 4600 /* skip first 'void *__data' argument in btf_trace_##name typedef */ 4678 4601 args++; ··· 4730 4649 } 4731 4650 } else { 4732 4651 if (!t) 4733 4733 - /* Default prog with 5 args */ 4652 4652 + /* Default prog with MAX_BPF_FUNC_REG_ARGS args */ 4734 4653 return true; 4735 4654 t = btf_type_by_id(btf, args[arg].type); 4736 4655 } ··· 5181 5100 5182 5101 if (!func) { 5183 5102 /* BTF function prototype doesn't match the verifier types. 5184 5184 - * Fall back to 5 u64 args. 5103 5103 + * Fall back to MAX_BPF_FUNC_REG_ARGS u64 args. 5185 5104 */ 5186 5186 - for (i = 0; i < 5; i++) 5105 5105 + for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) 5187 5106 m->arg_size[i] = 8; 5188 5107 m->ret_size = 8; 5189 5189 - m->nr_args = 5; 5108 5108 + m->nr_args = MAX_BPF_FUNC_REG_ARGS; 5190 5109 return 0; 5191 5110 } 5192 5111 args = (const struct btf_param *)(func + 1); ··· 5409 5328 } 5410 5329 args = (const struct btf_param *)(t + 1); 5411 5330 nargs = btf_type_vlen(t); 5412 5412 - if (nargs > 5) { 5413 5413 - bpf_log(log, "Function %s has %d > 5 args\n", tname, nargs); 5331 5331 + if (nargs > MAX_BPF_FUNC_REG_ARGS) { 5332 5332 + bpf_log(log, "Function %s has %d > %d args\n", tname, nargs, 5333 5333 + MAX_BPF_FUNC_REG_ARGS); 5414 5334 goto out; 5415 5335 } 5416 5336 ··· 5540 5458 } 5541 5459 args = (const struct btf_param *)(t + 1); 5542 5460 nargs = btf_type_vlen(t); 5543 5543 - if (nargs > 5) { 5544 5544 - bpf_log(log, "Global function %s() with %d > 5 args. Buggy compiler.\n", 5545 5545 - tname, nargs); 5461 5461 + if (nargs > MAX_BPF_FUNC_REG_ARGS) { 5462 5462 + bpf_log(log, "Global function %s() with %d > %d args. Buggy compiler.\n", 5463 5463 + tname, nargs, MAX_BPF_FUNC_REG_ARGS); 5546 5464 return -EINVAL; 5547 5465 } 5548 5466 /* check that function returns int */

+7 -2

kernel/bpf/cpumap.c

reviewed

··· 543 543 * complete. 544 544 */ 545 545 546 546 - bpf_clear_redirect_map(map); 547 546 synchronize_rcu(); 548 547 549 548 /* For cpu_map the remote CPUs can still be using the entries ··· 562 563 kfree(cmap); 563 564 } 564 565 565 565 - struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) 566 566 + static void *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) 566 567 { 567 568 struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map); 568 569 struct bpf_cpu_map_entry *rcpu; ··· 599 600 return 0; 600 601 } 601 602 603 603 + static int cpu_map_redirect(struct bpf_map *map, u32 ifindex, u64 flags) 604 604 + { 605 605 + return __bpf_xdp_redirect_map(map, ifindex, flags, __cpu_map_lookup_elem); 606 606 + } 607 607 + 602 608 static int cpu_map_btf_id; 603 609 const struct bpf_map_ops cpu_map_ops = { 604 610 .map_meta_equal = bpf_map_meta_equal, ··· 616 612 .map_check_btf = map_check_no_btf, 617 613 .map_btf_name = "bpf_cpu_map", 618 614 .map_btf_id = &cpu_map_btf_id, 615 615 + .map_redirect = cpu_map_redirect, 619 616 }; 620 617 621 618 static void bq_flush_to_queue(struct xdp_bulk_queue *bq)

+14 -3

kernel/bpf/devmap.c

reviewed

··· 197 197 list_del_rcu(&dtab->list); 198 198 spin_unlock(&dev_map_lock); 199 199 200 200 - bpf_clear_redirect_map(map); 201 200 synchronize_rcu(); 202 201 203 202 /* Make sure prior __dev_map_entry_free() have completed. */ ··· 257 258 return 0; 258 259 } 259 260 260 260 - struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key) 261 261 + static void *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key) 261 262 { 262 263 struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); 263 264 struct hlist_head *head = dev_map_index_hash(dtab, key); ··· 391 392 * update happens in parallel here a dev_put wont happen until after reading the 392 393 * ifindex. 393 394 */ 394 394 - struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key) 395 395 + static void *__dev_map_lookup_elem(struct bpf_map *map, u32 key) 395 396 { 396 397 struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); 397 398 struct bpf_dtab_netdev *obj; ··· 734 735 map, key, value, map_flags); 735 736 } 736 737 738 738 + static int dev_map_redirect(struct bpf_map *map, u32 ifindex, u64 flags) 739 739 + { 740 740 + return __bpf_xdp_redirect_map(map, ifindex, flags, __dev_map_lookup_elem); 741 741 + } 742 742 + 743 743 + static int dev_hash_map_redirect(struct bpf_map *map, u32 ifindex, u64 flags) 744 744 + { 745 745 + return __bpf_xdp_redirect_map(map, ifindex, flags, __dev_map_hash_lookup_elem); 746 746 + } 747 747 + 737 748 static int dev_map_btf_id; 738 749 const struct bpf_map_ops dev_map_ops = { 739 750 .map_meta_equal = bpf_map_meta_equal, ··· 756 747 .map_check_btf = map_check_no_btf, 757 748 .map_btf_name = "bpf_dtab", 758 749 .map_btf_id = &dev_map_btf_id, 750 750 + .map_redirect = dev_map_redirect, 759 751 }; 760 752 761 753 static int dev_map_hash_map_btf_id; ··· 771 761 .map_check_btf = map_check_no_btf, 772 762 .map_btf_name = "bpf_dtab", 773 763 .map_btf_id = &dev_map_hash_map_btf_id, 764 764 + .map_redirect = dev_hash_map_redirect, 774 765 }; 775 766 776 767 static void dev_map_hash_remove_netdev(struct bpf_dtab *dtab,

+65

kernel/bpf/hashtab.c

reviewed

··· 1869 1869 .seq_priv_size = sizeof(struct bpf_iter_seq_hash_map_info), 1870 1870 }; 1871 1871 1872 1872 + static int bpf_for_each_hash_elem(struct bpf_map *map, void *callback_fn, 1873 1873 + void *callback_ctx, u64 flags) 1874 1874 + { 1875 1875 + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); 1876 1876 + struct hlist_nulls_head *head; 1877 1877 + struct hlist_nulls_node *n; 1878 1878 + struct htab_elem *elem; 1879 1879 + u32 roundup_key_size; 1880 1880 + int i, num_elems = 0; 1881 1881 + void __percpu *pptr; 1882 1882 + struct bucket *b; 1883 1883 + void *key, *val; 1884 1884 + bool is_percpu; 1885 1885 + u64 ret = 0; 1886 1886 + 1887 1887 + if (flags != 0) 1888 1888 + return -EINVAL; 1889 1889 + 1890 1890 + is_percpu = htab_is_percpu(htab); 1891 1891 + 1892 1892 + roundup_key_size = round_up(map->key_size, 8); 1893 1893 + /* disable migration so percpu value prepared here will be the 1894 1894 + * same as the one seen by the bpf program with bpf_map_lookup_elem(). 1895 1895 + */ 1896 1896 + if (is_percpu) 1897 1897 + migrate_disable(); 1898 1898 + for (i = 0; i < htab->n_buckets; i++) { 1899 1899 + b = &htab->buckets[i]; 1900 1900 + rcu_read_lock(); 1901 1901 + head = &b->head; 1902 1902 + hlist_nulls_for_each_entry_rcu(elem, n, head, hash_node) { 1903 1903 + key = elem->key; 1904 1904 + if (is_percpu) { 1905 1905 + /* current cpu value for percpu map */ 1906 1906 + pptr = htab_elem_get_ptr(elem, map->key_size); 1907 1907 + val = this_cpu_ptr(pptr); 1908 1908 + } else { 1909 1909 + val = elem->key + roundup_key_size; 1910 1910 + } 1911 1911 + num_elems++; 1912 1912 + ret = BPF_CAST_CALL(callback_fn)((u64)(long)map, 1913 1913 + (u64)(long)key, (u64)(long)val, 1914 1914 + (u64)(long)callback_ctx, 0); 1915 1915 + /* return value: 0 - continue, 1 - stop and return */ 1916 1916 + if (ret) { 1917 1917 + rcu_read_unlock(); 1918 1918 + goto out; 1919 1919 + } 1920 1920 + } 1921 1921 + rcu_read_unlock(); 1922 1922 + } 1923 1923 + out: 1924 1924 + if (is_percpu) 1925 1925 + migrate_enable(); 1926 1926 + return num_elems; 1927 1927 + } 1928 1928 + 1872 1929 static int htab_map_btf_id; 1873 1930 const struct bpf_map_ops htab_map_ops = { 1874 1931 .map_meta_equal = bpf_map_meta_equal, ··· 1938 1881 .map_delete_elem = htab_map_delete_elem, 1939 1882 .map_gen_lookup = htab_map_gen_lookup, 1940 1883 .map_seq_show_elem = htab_map_seq_show_elem, 1884 1884 + .map_set_for_each_callback_args = map_set_for_each_callback_args, 1885 1885 + .map_for_each_callback = bpf_for_each_hash_elem, 1941 1886 BATCH_OPS(htab), 1942 1887 .map_btf_name = "bpf_htab", 1943 1888 .map_btf_id = &htab_map_btf_id, ··· 1959 1900 .map_delete_elem = htab_lru_map_delete_elem, 1960 1901 .map_gen_lookup = htab_lru_map_gen_lookup, 1961 1902 .map_seq_show_elem = htab_map_seq_show_elem, 1903 1903 + .map_set_for_each_callback_args = map_set_for_each_callback_args, 1904 1904 + .map_for_each_callback = bpf_for_each_hash_elem, 1962 1905 BATCH_OPS(htab_lru), 1963 1906 .map_btf_name = "bpf_htab", 1964 1907 .map_btf_id = &htab_lru_map_btf_id, ··· 2080 2019 .map_update_elem = htab_percpu_map_update_elem, 2081 2020 .map_delete_elem = htab_map_delete_elem, 2082 2021 .map_seq_show_elem = htab_percpu_map_seq_show_elem, 2022 2022 + .map_set_for_each_callback_args = map_set_for_each_callback_args, 2023 2023 + .map_for_each_callback = bpf_for_each_hash_elem, 2083 2024 BATCH_OPS(htab_percpu), 2084 2025 .map_btf_name = "bpf_htab", 2085 2026 .map_btf_id = &htab_percpu_map_btf_id, ··· 2099 2036 .map_update_elem = htab_lru_percpu_map_update_elem, 2100 2037 .map_delete_elem = htab_lru_map_delete_elem, 2101 2038 .map_seq_show_elem = htab_percpu_map_seq_show_elem, 2039 2039 + .map_set_for_each_callback_args = map_set_for_each_callback_args, 2040 2040 + .map_for_each_callback = bpf_for_each_hash_elem, 2102 2041 BATCH_OPS(htab_lru_percpu), 2103 2042 .map_btf_name = "bpf_htab", 2104 2043 .map_btf_id = &htab_lru_percpu_map_btf_id,

+2

kernel/bpf/helpers.c

reviewed

··· 708 708 return &bpf_ringbuf_discard_proto; 709 709 case BPF_FUNC_ringbuf_query: 710 710 return &bpf_ringbuf_query_proto; 711 711 + case BPF_FUNC_for_each_map_elem: 712 712 + return &bpf_for_each_map_elem_proto; 711 713 default: 712 714 break; 713 715 }

+329 -64

kernel/bpf/verifier.c

reviewed

··· 234 234 insn->src_reg == BPF_PSEUDO_CALL; 235 235 } 236 236 237 237 + static bool bpf_pseudo_func(const struct bpf_insn *insn) 238 238 + { 239 239 + return insn->code == (BPF_LD | BPF_IMM | BPF_DW) && 240 240 + insn->src_reg == BPF_PSEUDO_FUNC; 241 241 + } 242 242 + 237 243 struct bpf_call_arg_meta { 238 244 struct bpf_map *map_ptr; 239 245 bool raw_mode; ··· 254 248 u32 btf_id; 255 249 struct btf *ret_btf; 256 250 u32 ret_btf_id; 251 251 + u32 subprogno; 257 252 }; 258 253 259 254 struct btf *btf_vmlinux; ··· 397 390 env->prev_linfo = linfo; 398 391 } 399 392 393 393 + static void verbose_invalid_scalar(struct bpf_verifier_env *env, 394 394 + struct bpf_reg_state *reg, 395 395 + struct tnum *range, const char *ctx, 396 396 + const char *reg_name) 397 397 + { 398 398 + char tn_buf[48]; 399 399 + 400 400 + verbose(env, "At %s the register %s ", ctx, reg_name); 401 401 + if (!tnum_is_unknown(reg->var_off)) { 402 402 + tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 403 403 + verbose(env, "has value %s", tn_buf); 404 404 + } else { 405 405 + verbose(env, "has unknown scalar value"); 406 406 + } 407 407 + tnum_strn(tn_buf, sizeof(tn_buf), *range); 408 408 + verbose(env, " should have been in %s\n", tn_buf); 409 409 + } 410 410 + 400 411 static bool type_is_pkt_pointer(enum bpf_reg_type type) 401 412 { 402 413 return type == PTR_TO_PACKET || ··· 434 409 return type == PTR_TO_SOCKET || 435 410 type == PTR_TO_TCP_SOCK || 436 411 type == PTR_TO_MAP_VALUE || 412 412 + type == PTR_TO_MAP_KEY || 437 413 type == PTR_TO_SOCK_COMMON; 438 414 } 439 415 ··· 477 451 type == ARG_PTR_TO_MEM_OR_NULL || 478 452 type == ARG_PTR_TO_CTX_OR_NULL || 479 453 type == ARG_PTR_TO_SOCKET_OR_NULL || 480 480 - type == ARG_PTR_TO_ALLOC_MEM_OR_NULL; 454 454 + type == ARG_PTR_TO_ALLOC_MEM_OR_NULL || 455 455 + type == ARG_PTR_TO_STACK_OR_NULL; 481 456 } 482 457 483 458 /* Determine whether the function releases some resources allocated by another ··· 568 541 [PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null", 569 542 [PTR_TO_RDWR_BUF] = "rdwr_buf", 570 543 [PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null", 544 544 + [PTR_TO_FUNC] = "func", 545 545 + [PTR_TO_MAP_KEY] = "map_key", 571 546 }; 572 547 573 548 static char slot_type_char[] = { ··· 641 612 if (type_is_pkt_pointer(t)) 642 613 verbose(env, ",r=%d", reg->range); 643 614 else if (t == CONST_PTR_TO_MAP || 615 615 + t == PTR_TO_MAP_KEY || 644 616 t == PTR_TO_MAP_VALUE || 645 617 t == PTR_TO_MAP_VALUE_OR_NULL) 646 618 verbose(env, ",ks=%d,vs=%d", ··· 1549 1519 } 1550 1520 ret = find_subprog(env, off); 1551 1521 if (ret >= 0) 1552 1552 - return 0; 1522 1522 + return ret; 1553 1523 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 1554 1524 verbose(env, "too many subprograms\n"); 1555 1525 return -E2BIG; ··· 1557 1527 env->subprog_info[env->subprog_cnt++].start = off; 1558 1528 sort(env->subprog_info, env->subprog_cnt, 1559 1529 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); 1560 1560 - return 0; 1530 1530 + return env->subprog_cnt - 1; 1561 1531 } 1562 1532 1563 1533 static int check_subprogs(struct bpf_verifier_env *env) ··· 1574 1544 1575 1545 /* determine subprog starts. The end is one before the next starts */ 1576 1546 for (i = 0; i < insn_cnt; i++) { 1547 1547 + if (bpf_pseudo_func(insn + i)) { 1548 1548 + if (!env->bpf_capable) { 1549 1549 + verbose(env, 1550 1550 + "function pointers are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 1551 1551 + return -EPERM; 1552 1552 + } 1553 1553 + ret = add_subprog(env, i + insn[i].imm + 1); 1554 1554 + if (ret < 0) 1555 1555 + return ret; 1556 1556 + /* remember subprog */ 1557 1557 + insn[i + 1].imm = ret; 1558 1558 + continue; 1559 1559 + } 1577 1560 if (!bpf_pseudo_call(insn + i)) 1578 1561 continue; 1579 1562 if (!env->bpf_capable) { ··· 2338 2295 case PTR_TO_PERCPU_BTF_ID: 2339 2296 case PTR_TO_MEM: 2340 2297 case PTR_TO_MEM_OR_NULL: 2298 2298 + case PTR_TO_FUNC: 2299 2299 + case PTR_TO_MAP_KEY: 2341 2300 return true; 2342 2301 default: 2343 2302 return false; ··· 2944 2899 2945 2900 reg = &cur_regs(env)[regno]; 2946 2901 switch (reg->type) { 2902 2902 + case PTR_TO_MAP_KEY: 2903 2903 + verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", 2904 2904 + mem_size, off, size); 2905 2905 + break; 2947 2906 case PTR_TO_MAP_VALUE: 2948 2907 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 2949 2908 mem_size, off, size); ··· 3353 3304 case PTR_TO_FLOW_KEYS: 3354 3305 pointer_desc = "flow keys "; 3355 3306 break; 3307 3307 + case PTR_TO_MAP_KEY: 3308 3308 + pointer_desc = "key "; 3309 3309 + break; 3356 3310 case PTR_TO_MAP_VALUE: 3357 3311 pointer_desc = "value "; 3358 3312 break; ··· 3457 3405 continue_func: 3458 3406 subprog_end = subprog[idx + 1].start; 3459 3407 for (; i < subprog_end; i++) { 3460 3460 - if (!bpf_pseudo_call(insn + i)) 3408 3408 + if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) 3461 3409 continue; 3462 3410 /* remember insn and function to return to */ 3463 3411 ret_insn[frame] = i + 1; ··· 3894 3842 /* for access checks, reg->off is just part of off */ 3895 3843 off += reg->off; 3896 3844 3897 3897 - if (reg->type == PTR_TO_MAP_VALUE) { 3845 3845 + if (reg->type == PTR_TO_MAP_KEY) { 3846 3846 + if (t == BPF_WRITE) { 3847 3847 + verbose(env, "write to change key R%d not allowed\n", regno); 3848 3848 + return -EACCES; 3849 3849 + } 3850 3850 + 3851 3851 + err = check_mem_region_access(env, regno, off, size, 3852 3852 + reg->map_ptr->key_size, false); 3853 3853 + if (err) 3854 3854 + return err; 3855 3855 + if (value_regno >= 0) 3856 3856 + mark_reg_unknown(env, regs, value_regno); 3857 3857 + } else if (reg->type == PTR_TO_MAP_VALUE) { 3898 3858 if (t == BPF_WRITE && value_regno >= 0 && 3899 3859 is_pointer_value(env, value_regno)) { 3900 3860 verbose(env, "R%d leaks addr into map\n", value_regno); ··· 4322 4258 case PTR_TO_PACKET_META: 4323 4259 return check_packet_access(env, regno, reg->off, access_size, 4324 4260 zero_size_allowed); 4261 4261 + case PTR_TO_MAP_KEY: 4262 4262 + return check_mem_region_access(env, regno, reg->off, access_size, 4263 4263 + reg->map_ptr->key_size, false); 4325 4264 case PTR_TO_MAP_VALUE: 4326 4265 if (check_map_access_type(env, regno, reg->off, access_size, 4327 4266 meta && meta->raw_mode ? BPF_WRITE : ··· 4541 4474 PTR_TO_STACK, 4542 4475 PTR_TO_PACKET, 4543 4476 PTR_TO_PACKET_META, 4477 4477 + PTR_TO_MAP_KEY, 4544 4478 PTR_TO_MAP_VALUE, 4545 4479 }, 4546 4480 }; ··· 4573 4505 PTR_TO_STACK, 4574 4506 PTR_TO_PACKET, 4575 4507 PTR_TO_PACKET_META, 4508 4508 + PTR_TO_MAP_KEY, 4576 4509 PTR_TO_MAP_VALUE, 4577 4510 PTR_TO_MEM, 4578 4511 PTR_TO_RDONLY_BUF, ··· 4586 4517 PTR_TO_STACK, 4587 4518 PTR_TO_PACKET, 4588 4519 PTR_TO_PACKET_META, 4520 4520 + PTR_TO_MAP_KEY, 4589 4521 PTR_TO_MAP_VALUE, 4590 4522 }, 4591 4523 }; ··· 4599 4529 static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } }; 4600 4530 static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } }; 4601 4531 static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } }; 4532 4532 + static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; 4533 4533 + static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; 4602 4534 4603 4535 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { 4604 4536 [ARG_PTR_TO_MAP_KEY] = &map_key_value_types, ··· 4629 4557 [ARG_PTR_TO_INT] = &int_ptr_types, 4630 4558 [ARG_PTR_TO_LONG] = &int_ptr_types, 4631 4559 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, 4560 4560 + [ARG_PTR_TO_FUNC] = &func_ptr_types, 4561 4561 + [ARG_PTR_TO_STACK_OR_NULL] = &stack_ptr_types, 4632 4562 }; 4633 4563 4634 4564 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, ··· 4812 4738 verbose(env, "verifier internal error\n"); 4813 4739 return -EFAULT; 4814 4740 } 4741 4741 + } else if (arg_type == ARG_PTR_TO_FUNC) { 4742 4742 + meta->subprogno = reg->subprogno; 4815 4743 } else if (arg_type_is_mem_ptr(arg_type)) { 4816 4744 /* The access to this pointer is only checked when we hit the 4817 4745 * next is_mem_size argument below. ··· 5334 5258 } 5335 5259 } 5336 5260 5337 5337 - static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 5338 5338 - int *insn_idx) 5261 5261 + typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, 5262 5262 + struct bpf_func_state *caller, 5263 5263 + struct bpf_func_state *callee, 5264 5264 + int insn_idx); 5265 5265 + 5266 5266 + static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 5267 5267 + int *insn_idx, int subprog, 5268 5268 + set_callee_state_fn set_callee_state_cb) 5339 5269 { 5340 5270 struct bpf_verifier_state *state = env->cur_state; 5341 5271 struct bpf_func_info_aux *func_info_aux; 5342 5272 struct bpf_func_state *caller, *callee; 5343 5343 - int i, err, subprog, target_insn; 5273 5273 + int err; 5344 5274 bool is_global = false; 5345 5275 5346 5276 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 5347 5277 verbose(env, "the call stack of %d frames is too deep\n", 5348 5278 state->curframe + 2); 5349 5279 return -E2BIG; 5350 5350 - } 5351 5351 - 5352 5352 - target_insn = *insn_idx + insn->imm; 5353 5353 - subprog = find_subprog(env, target_insn + 1); 5354 5354 - if (subprog < 0) { 5355 5355 - verbose(env, "verifier bug. No program starts at insn %d\n", 5356 5356 - target_insn + 1); 5357 5357 - return -EFAULT; 5358 5280 } 5359 5281 5360 5282 caller = state->frame[state->curframe]; ··· 5409 5335 if (err) 5410 5336 return err; 5411 5337 5412 5412 - /* copy r1 - r5 args that callee can access. The copy includes parent 5413 5413 - * pointers, which connects us up to the liveness chain 5414 5414 - */ 5415 5415 - for (i = BPF_REG_1; i <= BPF_REG_5; i++) 5416 5416 - callee->regs[i] = caller->regs[i]; 5338 5338 + err = set_callee_state_cb(env, caller, callee, *insn_idx); 5339 5339 + if (err) 5340 5340 + return err; 5417 5341 5418 5342 clear_caller_saved_regs(env, caller->regs); 5419 5343 ··· 5419 5347 state->curframe++; 5420 5348 5421 5349 /* and go analyze first insn of the callee */ 5422 5422 - *insn_idx = target_insn; 5350 5350 + *insn_idx = env->subprog_info[subprog].start - 1; 5423 5351 5424 5352 if (env->log.level & BPF_LOG_LEVEL) { 5425 5353 verbose(env, "caller:\n"); ··· 5427 5355 verbose(env, "callee:\n"); 5428 5356 print_verifier_state(env, callee); 5429 5357 } 5358 5358 + return 0; 5359 5359 + } 5360 5360 + 5361 5361 + int map_set_for_each_callback_args(struct bpf_verifier_env *env, 5362 5362 + struct bpf_func_state *caller, 5363 5363 + struct bpf_func_state *callee) 5364 5364 + { 5365 5365 + /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, 5366 5366 + * void *callback_ctx, u64 flags); 5367 5367 + * callback_fn(struct bpf_map *map, void *key, void *value, 5368 5368 + * void *callback_ctx); 5369 5369 + */ 5370 5370 + callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 5371 5371 + 5372 5372 + callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 5373 5373 + __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 5374 5374 + callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; 5375 5375 + 5376 5376 + callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 5377 5377 + __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 5378 5378 + callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; 5379 5379 + 5380 5380 + /* pointer to stack or null */ 5381 5381 + callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; 5382 5382 + 5383 5383 + /* unused */ 5384 5384 + __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 5385 5385 + return 0; 5386 5386 + } 5387 5387 + 5388 5388 + static int set_callee_state(struct bpf_verifier_env *env, 5389 5389 + struct bpf_func_state *caller, 5390 5390 + struct bpf_func_state *callee, int insn_idx) 5391 5391 + { 5392 5392 + int i; 5393 5393 + 5394 5394 + /* copy r1 - r5 args that callee can access. The copy includes parent 5395 5395 + * pointers, which connects us up to the liveness chain 5396 5396 + */ 5397 5397 + for (i = BPF_REG_1; i <= BPF_REG_5; i++) 5398 5398 + callee->regs[i] = caller->regs[i]; 5399 5399 + return 0; 5400 5400 + } 5401 5401 + 5402 5402 + static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 5403 5403 + int *insn_idx) 5404 5404 + { 5405 5405 + int subprog, target_insn; 5406 5406 + 5407 5407 + target_insn = *insn_idx + insn->imm + 1; 5408 5408 + subprog = find_subprog(env, target_insn); 5409 5409 + if (subprog < 0) { 5410 5410 + verbose(env, "verifier bug. No program starts at insn %d\n", 5411 5411 + target_insn); 5412 5412 + return -EFAULT; 5413 5413 + } 5414 5414 + 5415 5415 + return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); 5416 5416 + } 5417 5417 + 5418 5418 + static int set_map_elem_callback_state(struct bpf_verifier_env *env, 5419 5419 + struct bpf_func_state *caller, 5420 5420 + struct bpf_func_state *callee, 5421 5421 + int insn_idx) 5422 5422 + { 5423 5423 + struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; 5424 5424 + struct bpf_map *map; 5425 5425 + int err; 5426 5426 + 5427 5427 + if (bpf_map_ptr_poisoned(insn_aux)) { 5428 5428 + verbose(env, "tail_call abusing map_ptr\n"); 5429 5429 + return -EINVAL; 5430 5430 + } 5431 5431 + 5432 5432 + map = BPF_MAP_PTR(insn_aux->map_ptr_state); 5433 5433 + if (!map->ops->map_set_for_each_callback_args || 5434 5434 + !map->ops->map_for_each_callback) { 5435 5435 + verbose(env, "callback function not allowed for map\n"); 5436 5436 + return -ENOTSUPP; 5437 5437 + } 5438 5438 + 5439 5439 + err = map->ops->map_set_for_each_callback_args(env, caller, callee); 5440 5440 + if (err) 5441 5441 + return err; 5442 5442 + 5443 5443 + callee->in_callback_fn = true; 5430 5444 return 0; 5431 5445 } 5432 5446 ··· 5538 5380 5539 5381 state->curframe--; 5540 5382 caller = state->frame[state->curframe]; 5541 5541 - /* return to the caller whatever r0 had in the callee */ 5542 5542 - caller->regs[BPF_REG_0] = *r0; 5383 5383 + if (callee->in_callback_fn) { 5384 5384 + /* enforce R0 return value range [0, 1]. */ 5385 5385 + struct tnum range = tnum_range(0, 1); 5386 5386 + 5387 5387 + if (r0->type != SCALAR_VALUE) { 5388 5388 + verbose(env, "R0 not a scalar value\n"); 5389 5389 + return -EACCES; 5390 5390 + } 5391 5391 + if (!tnum_in(range, r0->var_off)) { 5392 5392 + verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); 5393 5393 + return -EINVAL; 5394 5394 + } 5395 5395 + } else { 5396 5396 + /* return to the caller whatever r0 had in the callee */ 5397 5397 + caller->regs[BPF_REG_0] = *r0; 5398 5398 + } 5543 5399 5544 5400 /* Transfer references to the caller */ 5545 5401 err = transfer_reference_state(caller, callee); ··· 5608 5436 func_id != BPF_FUNC_map_delete_elem && 5609 5437 func_id != BPF_FUNC_map_push_elem && 5610 5438 func_id != BPF_FUNC_map_pop_elem && 5611 5611 - func_id != BPF_FUNC_map_peek_elem) 5439 5439 + func_id != BPF_FUNC_map_peek_elem && 5440 5440 + func_id != BPF_FUNC_for_each_map_elem && 5441 5441 + func_id != BPF_FUNC_redirect_map) 5612 5442 return 0; 5613 5443 5614 5444 if (map == NULL) { ··· 5691 5517 return state->acquired_refs ? -EINVAL : 0; 5692 5518 } 5693 5519 5694 5694 - static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx) 5520 5520 + static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 5521 5521 + int *insn_idx_p) 5695 5522 { 5696 5523 const struct bpf_func_proto *fn = NULL; 5697 5524 struct bpf_reg_state *regs; 5698 5525 struct bpf_call_arg_meta meta; 5526 5526 + int insn_idx = *insn_idx_p; 5699 5527 bool changes_data; 5700 5700 - int i, err; 5528 5528 + int i, err, func_id; 5701 5529 5702 5530 /* find function prototype */ 5531 5531 + func_id = insn->imm; 5703 5532 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { 5704 5533 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), 5705 5534 func_id); ··· 5748 5571 5749 5572 meta.func_id = func_id; 5750 5573 /* check args */ 5751 5751 - for (i = 0; i < 5; i++) { 5574 5574 + for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 5752 5575 err = check_func_arg(env, i, &meta, fn); 5753 5576 if (err) 5754 5577 return err; ··· 5796 5619 !register_is_null(&regs[BPF_REG_2])) { 5797 5620 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 5798 5621 return -EINVAL; 5622 5622 + } 5623 5623 + 5624 5624 + if (func_id == BPF_FUNC_for_each_map_elem) { 5625 5625 + err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 5626 5626 + set_map_elem_callback_state); 5627 5627 + if (err < 0) 5628 5628 + return -EINVAL; 5799 5629 } 5800 5630 5801 5631 /* reset caller saved regs */ ··· 6058 5874 else 6059 5875 *ptr_limit = -off; 6060 5876 return 0; 5877 5877 + case PTR_TO_MAP_KEY: 5878 5878 + /* Currently, this code is not exercised as the only use 5879 5879 + * is bpf_for_each_map_elem() helper which requires 5880 5880 + * bpf_capble. The code has been tested manually for 5881 5881 + * future use. 5882 5882 + */ 5883 5883 + if (mask_to_left) { 5884 5884 + *ptr_limit = ptr_reg->umax_value + ptr_reg->off; 5885 5885 + } else { 5886 5886 + off = ptr_reg->smin_value + ptr_reg->off; 5887 5887 + *ptr_limit = ptr_reg->map_ptr->key_size - off; 5888 5888 + } 5889 5889 + return 0; 6061 5890 case PTR_TO_MAP_VALUE: 6062 5891 if (mask_to_left) { 6063 5892 *ptr_limit = ptr_reg->umax_value + ptr_reg->off; ··· 6101 5904 aux->alu_limit != alu_limit)) 6102 5905 return -EACCES; 6103 5906 6104 6104 - /* Corresponding fixup done in fixup_bpf_calls(). */ 5907 5907 + /* Corresponding fixup done in do_misc_fixups(). */ 6105 5908 aux->alu_state = alu_state; 6106 5909 aux->alu_limit = alu_limit; 6107 5910 return 0; ··· 6272 6075 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 6273 6076 dst, reg_type_str[ptr_reg->type]); 6274 6077 return -EACCES; 6078 6078 + case PTR_TO_MAP_KEY: 6275 6079 case PTR_TO_MAP_VALUE: 6276 6080 if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) { 6277 6081 verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n", ··· 8452 8254 return 0; 8453 8255 } 8454 8256 8257 8257 + if (insn->src_reg == BPF_PSEUDO_FUNC) { 8258 8258 + struct bpf_prog_aux *aux = env->prog->aux; 8259 8259 + u32 subprogno = insn[1].imm; 8260 8260 + 8261 8261 + if (!aux->func_info) { 8262 8262 + verbose(env, "missing btf func_info\n"); 8263 8263 + return -EINVAL; 8264 8264 + } 8265 8265 + if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { 8266 8266 + verbose(env, "callback function not static\n"); 8267 8267 + return -EINVAL; 8268 8268 + } 8269 8269 + 8270 8270 + dst_reg->type = PTR_TO_FUNC; 8271 8271 + dst_reg->subprogno = subprogno; 8272 8272 + return 0; 8273 8273 + } 8274 8274 + 8455 8275 map = env->used_maps[aux->map_index]; 8456 8276 mark_reg_known_zero(env, regs, insn->dst_reg); 8457 8277 dst_reg->map_ptr = map; ··· 8698 8482 } 8699 8483 8700 8484 if (!tnum_in(range, reg->var_off)) { 8701 8701 - char tn_buf[48]; 8702 8702 - 8703 8703 - verbose(env, "At program exit the register R0 "); 8704 8704 - if (!tnum_is_unknown(reg->var_off)) { 8705 8705 - tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 8706 8706 - verbose(env, "has value %s", tn_buf); 8707 8707 - } else { 8708 8708 - verbose(env, "has unknown scalar value"); 8709 8709 - } 8710 8710 - tnum_strn(tn_buf, sizeof(tn_buf), range); 8711 8711 - verbose(env, " should have been in %s\n", tn_buf); 8485 8485 + verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); 8712 8486 return -EINVAL; 8713 8487 } 8714 8488 ··· 8825 8619 return DONE_EXPLORING; 8826 8620 } 8827 8621 8622 8622 + static int visit_func_call_insn(int t, int insn_cnt, 8623 8623 + struct bpf_insn *insns, 8624 8624 + struct bpf_verifier_env *env, 8625 8625 + bool visit_callee) 8626 8626 + { 8627 8627 + int ret; 8628 8628 + 8629 8629 + ret = push_insn(t, t + 1, FALLTHROUGH, env, false); 8630 8630 + if (ret) 8631 8631 + return ret; 8632 8632 + 8633 8633 + if (t + 1 < insn_cnt) 8634 8634 + init_explored_state(env, t + 1); 8635 8635 + if (visit_callee) { 8636 8636 + init_explored_state(env, t); 8637 8637 + ret = push_insn(t, t + insns[t].imm + 1, BRANCH, 8638 8638 + env, false); 8639 8639 + } 8640 8640 + return ret; 8641 8641 + } 8642 8642 + 8828 8643 /* Visits the instruction at index t and returns one of the following: 8829 8644 * < 0 - an error occurred 8830 8645 * DONE_EXPLORING - the instruction was fully explored ··· 8855 8628 { 8856 8629 struct bpf_insn *insns = env->prog->insnsi; 8857 8630 int ret; 8631 8631 + 8632 8632 + if (bpf_pseudo_func(insns + t)) 8633 8633 + return visit_func_call_insn(t, insn_cnt, insns, env, true); 8858 8634 8859 8635 /* All non-branch instructions have a single fall-through edge. */ 8860 8636 if (BPF_CLASS(insns[t].code) != BPF_JMP && ··· 8869 8639 return DONE_EXPLORING; 8870 8640 8871 8641 case BPF_CALL: 8872 8872 - ret = push_insn(t, t + 1, FALLTHROUGH, env, false); 8873 8873 - if (ret) 8874 8874 - return ret; 8875 8875 - 8876 8876 - if (t + 1 < insn_cnt) 8877 8877 - init_explored_state(env, t + 1); 8878 8878 - if (insns[t].src_reg == BPF_PSEUDO_CALL) { 8879 8879 - init_explored_state(env, t); 8880 8880 - ret = push_insn(t, t + insns[t].imm + 1, BRANCH, 8881 8881 - env, false); 8882 8882 - } 8883 8883 - return ret; 8642 8642 + return visit_func_call_insn(t, insn_cnt, insns, env, 8643 8643 + insns[t].src_reg == BPF_PSEUDO_CALL); 8884 8644 8885 8645 case BPF_JA: 8886 8646 if (BPF_SRC(insns[t].code) != BPF_K) ··· 9479 9259 */ 9480 9260 return false; 9481 9261 } 9262 9262 + case PTR_TO_MAP_KEY: 9482 9263 case PTR_TO_MAP_VALUE: 9483 9264 /* If the new min/max/var_off satisfy the old ones and 9484 9265 * everything else matches, we are OK. ··· 10326 10105 if (insn->src_reg == BPF_PSEUDO_CALL) 10327 10106 err = check_func_call(env, insn, &env->insn_idx); 10328 10107 else 10329 10329 - err = check_helper_call(env, insn->imm, env->insn_idx); 10108 10108 + err = check_helper_call(env, insn, &env->insn_idx); 10330 10109 if (err) 10331 10110 return err; 10332 10332 - 10333 10111 } else if (opcode == BPF_JA) { 10334 10112 if (BPF_SRC(insn->code) != BPF_K || 10335 10113 insn->imm != 0 || ··· 10757 10537 goto next_insn; 10758 10538 } 10759 10539 10540 10540 + if (insn[0].src_reg == BPF_PSEUDO_FUNC) { 10541 10541 + aux = &env->insn_aux_data[i]; 10542 10542 + aux->ptr_type = PTR_TO_FUNC; 10543 10543 + goto next_insn; 10544 10544 + } 10545 10545 + 10760 10546 /* In final convert_pseudo_ld_imm64() step, this is 10761 10547 * converted into regular 64-bit imm load insn. 10762 10548 */ ··· 10895 10669 int insn_cnt = env->prog->len; 10896 10670 int i; 10897 10671 10898 10898 - for (i = 0; i < insn_cnt; i++, insn++) 10899 10899 - if (insn->code == (BPF_LD | BPF_IMM | BPF_DW)) 10900 10900 - insn->src_reg = 0; 10672 10672 + for (i = 0; i < insn_cnt; i++, insn++) { 10673 10673 + if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) 10674 10674 + continue; 10675 10675 + if (insn->src_reg == BPF_PSEUDO_FUNC) 10676 10676 + continue; 10677 10677 + insn->src_reg = 0; 10678 10678 + } 10901 10679 } 10902 10680 10903 10681 /* single env->prog->insni[off] instruction was replaced with the range ··· 11540 11310 return 0; 11541 11311 11542 11312 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 11313 11313 + if (bpf_pseudo_func(insn)) { 11314 11314 + env->insn_aux_data[i].call_imm = insn->imm; 11315 11315 + /* subprog is encoded in insn[1].imm */ 11316 11316 + continue; 11317 11317 + } 11318 11318 + 11543 11319 if (!bpf_pseudo_call(insn)) 11544 11320 continue; 11545 11321 /* Upon error here we cannot fall back to interpreter but ··· 11675 11439 for (i = 0; i < env->subprog_cnt; i++) { 11676 11440 insn = func[i]->insnsi; 11677 11441 for (j = 0; j < func[i]->len; j++, insn++) { 11442 11442 + if (bpf_pseudo_func(insn)) { 11443 11443 + subprog = insn[1].imm; 11444 11444 + insn[0].imm = (u32)(long)func[subprog]->bpf_func; 11445 11445 + insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 11446 11446 + continue; 11447 11447 + } 11678 11448 if (!bpf_pseudo_call(insn)) 11679 11449 continue; 11680 11450 subprog = insn->off; ··· 11726 11484 * later look the same as if they were interpreted only. 11727 11485 */ 11728 11486 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 11487 11487 + if (bpf_pseudo_func(insn)) { 11488 11488 + insn[0].imm = env->insn_aux_data[i].call_imm; 11489 11489 + insn[1].imm = find_subprog(env, i + insn[0].imm + 1); 11490 11490 + continue; 11491 11491 + } 11729 11492 if (!bpf_pseudo_call(insn)) 11730 11493 continue; 11731 11494 insn->off = env->insn_aux_data[i].call_imm; ··· 11795 11548 return -EINVAL; 11796 11549 } 11797 11550 for (i = 0; i < prog->len; i++, insn++) { 11551 11551 + if (bpf_pseudo_func(insn)) { 11552 11552 + /* When JIT fails the progs with callback calls 11553 11553 + * have to be rejected, since interpreter doesn't support them yet. 11554 11554 + */ 11555 11555 + verbose(env, "callbacks are not allowed in non-JITed programs\n"); 11556 11556 + return -EINVAL; 11557 11557 + } 11558 11558 + 11798 11559 if (!bpf_pseudo_call(insn)) 11799 11560 continue; 11800 11561 depth = get_callee_stack_depth(env, insn, i); ··· 11815 11560 return err; 11816 11561 } 11817 11562 11818 11818 - /* fixup insn->imm field of bpf_call instructions 11819 11819 - * and inline eligible helpers as explicit sequence of BPF instructions 11820 11820 - * 11821 11821 - * this function is called after eBPF program passed verification 11563 11563 + /* Do various post-verification rewrites in a single program pass. 11564 11564 + * These rewrites simplify JIT and interpreter implementations. 11822 11565 */ 11823 11823 - static int fixup_bpf_calls(struct bpf_verifier_env *env) 11566 11566 + static int do_misc_fixups(struct bpf_verifier_env *env) 11824 11567 { 11825 11568 struct bpf_prog *prog = env->prog; 11826 11569 bool expect_blinding = bpf_jit_blinding_enabled(prog); ··· 11833 11580 int i, ret, cnt, delta = 0; 11834 11581 11835 11582 for (i = 0; i < insn_cnt; i++, insn++) { 11583 11583 + /* Make divide-by-zero exceptions impossible. */ 11836 11584 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 11837 11585 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 11838 11586 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || ··· 11874 11620 continue; 11875 11621 } 11876 11622 11623 11623 + /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 11877 11624 if (BPF_CLASS(insn->code) == BPF_LD && 11878 11625 (BPF_MODE(insn->code) == BPF_ABS || 11879 11626 BPF_MODE(insn->code) == BPF_IND)) { ··· 11894 11639 continue; 11895 11640 } 11896 11641 11642 11642 + /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 11897 11643 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 11898 11644 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 11899 11645 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; ··· 12043 11787 insn->imm == BPF_FUNC_map_delete_elem || 12044 11788 insn->imm == BPF_FUNC_map_push_elem || 12045 11789 insn->imm == BPF_FUNC_map_pop_elem || 12046 12046 - insn->imm == BPF_FUNC_map_peek_elem)) { 11790 11790 + insn->imm == BPF_FUNC_map_peek_elem || 11791 11791 + insn->imm == BPF_FUNC_redirect_map)) { 12047 11792 aux = &env->insn_aux_data[i + delta]; 12048 11793 if (bpf_map_ptr_poisoned(aux)) 12049 11794 goto patch_call_imm; ··· 12086 11829 (int (*)(struct bpf_map *map, void *value))NULL)); 12087 11830 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 12088 11831 (int (*)(struct bpf_map *map, void *value))NULL)); 11832 11832 + BUILD_BUG_ON(!__same_type(ops->map_redirect, 11833 11833 + (int (*)(struct bpf_map *map, u32 ifindex, u64 flags))NULL)); 11834 11834 + 12089 11835 patch_map_ops_generic: 12090 11836 switch (insn->imm) { 12091 11837 case BPF_FUNC_map_lookup_elem: ··· 12115 11855 insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - 12116 11856 __bpf_call_base; 12117 11857 continue; 11858 11858 + case BPF_FUNC_redirect_map: 11859 11859 + insn->imm = BPF_CAST_CALL(ops->map_redirect) - 11860 11860 + __bpf_call_base; 11861 11861 + continue; 12118 11862 } 12119 11863 12120 11864 goto patch_call_imm; 12121 11865 } 12122 11866 11867 11867 + /* Implement bpf_jiffies64 inline. */ 12123 11868 if (prog->jit_requested && BITS_PER_LONG == 64 && 12124 11869 insn->imm == BPF_FUNC_jiffies64) { 12125 11870 struct bpf_insn ld_jiffies_addr[2] = { ··· 12935 12670 ret = convert_ctx_accesses(env); 12936 12671 12937 12672 if (ret == 0) 12938 12938 - ret = fixup_bpf_calls(env); 12673 12673 + ret = do_misc_fixups(env); 12939 12674 12940 12675 /* do 32-bit optimization after insn patching has done so those patched 12941 12676 * insns could be handled correctly.

+5

kernel/fork.c

reviewed

··· 96 96 #include <linux/kasan.h> 97 97 #include <linux/scs.h> 98 98 #include <linux/io_uring.h> 99 99 + #include <linux/bpf.h> 99 100 100 101 #include <asm/pgalloc.h> 101 102 #include <linux/uaccess.h> ··· 735 734 cgroup_free(tsk); 736 735 task_numa_free(tsk, true); 737 736 security_task_free(tsk); 737 737 + bpf_task_storage_free(tsk); 738 738 exit_creds(tsk); 739 739 delayacct_tsk_free(tsk); 740 740 put_signal_struct(tsk->signal); ··· 2065 2063 #ifdef CONFIG_BCACHE 2066 2064 p->sequential_io = 0; 2067 2065 p->sequential_io_avg = 0; 2066 2066 + #endif 2067 2067 + #ifdef CONFIG_BPF_SYSCALL 2068 2068 + RCU_INIT_POINTER(p->bpf_storage, NULL); 2068 2069 #endif 2069 2070 2070 2071 /* Perform scheduler related setup. Assign this task to a CPU. */

+6

kernel/trace/bpf_trace.c

reviewed

··· 1367 1367 return &bpf_per_cpu_ptr_proto; 1368 1368 case BPF_FUNC_this_cpu_ptr: 1369 1369 return &bpf_this_cpu_ptr_proto; 1370 1370 + case BPF_FUNC_task_storage_get: 1371 1371 + return &bpf_task_storage_get_proto; 1372 1372 + case BPF_FUNC_task_storage_delete: 1373 1373 + return &bpf_task_storage_delete_proto; 1374 1374 + case BPF_FUNC_for_each_map_elem: 1375 1375 + return &bpf_for_each_map_elem_proto; 1370 1376 default: 1371 1377 return NULL; 1372 1378 }

+1 -5

net/Kconfig

reviewed

··· 317 317 select STREAM_PARSER 318 318 select NET_SOCK_MSG 319 319 help 320 320 - Enabling this allows a stream parser to be used with 320 320 + Enabling this allows a TCP stream parser to be used with 321 321 BPF_MAP_TYPE_SOCKMAP. 322 322 - 323 323 - BPF_MAP_TYPE_SOCKMAP provides a map type to use with network sockets. 324 324 - It can be used to enforce socket policy, implement socket redirects, 325 325 - etc. 326 322 327 323 config NET_FLOW_LIMIT 328 324 bool

+183 -63

net/bpf/test_run.c

reviewed

··· 10 10 #include <net/bpf_sk_storage.h> 11 11 #include <net/sock.h> 12 12 #include <net/tcp.h> 13 13 + #include <net/net_namespace.h> 13 14 #include <linux/error-injection.h> 14 15 #include <linux/smp.h> 16 16 + #include <linux/sock_diag.h> 15 17 16 18 #define CREATE_TRACE_POINTS 17 19 #include <trace/events/bpf_test_run.h> 20 20 + 21 21 + struct bpf_test_timer { 22 22 + enum { NO_PREEMPT, NO_MIGRATE } mode; 23 23 + u32 i; 24 24 + u64 time_start, time_spent; 25 25 + }; 26 26 + 27 27 + static void bpf_test_timer_enter(struct bpf_test_timer *t) 28 28 + __acquires(rcu) 29 29 + { 30 30 + rcu_read_lock(); 31 31 + if (t->mode == NO_PREEMPT) 32 32 + preempt_disable(); 33 33 + else 34 34 + migrate_disable(); 35 35 + 36 36 + t->time_start = ktime_get_ns(); 37 37 + } 38 38 + 39 39 + static void bpf_test_timer_leave(struct bpf_test_timer *t) 40 40 + __releases(rcu) 41 41 + { 42 42 + t->time_start = 0; 43 43 + 44 44 + if (t->mode == NO_PREEMPT) 45 45 + preempt_enable(); 46 46 + else 47 47 + migrate_enable(); 48 48 + rcu_read_unlock(); 49 49 + } 50 50 + 51 51 + static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration) 52 52 + __must_hold(rcu) 53 53 + { 54 54 + t->i++; 55 55 + if (t->i >= repeat) { 56 56 + /* We're done. */ 57 57 + t->time_spent += ktime_get_ns() - t->time_start; 58 58 + do_div(t->time_spent, t->i); 59 59 + *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent; 60 60 + *err = 0; 61 61 + goto reset; 62 62 + } 63 63 + 64 64 + if (signal_pending(current)) { 65 65 + /* During iteration: we've been cancelled, abort. */ 66 66 + *err = -EINTR; 67 67 + goto reset; 68 68 + } 69 69 + 70 70 + if (need_resched()) { 71 71 + /* During iteration: we need to reschedule between runs. */ 72 72 + t->time_spent += ktime_get_ns() - t->time_start; 73 73 + bpf_test_timer_leave(t); 74 74 + cond_resched(); 75 75 + bpf_test_timer_enter(t); 76 76 + } 77 77 + 78 78 + /* Do another round. */ 79 79 + return true; 80 80 + 81 81 + reset: 82 82 + t->i = 0; 83 83 + return false; 84 84 + } 18 85 19 86 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 20 87 u32 *retval, u32 *time, bool xdp) 21 88 { 22 89 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL }; 90 90 + struct bpf_test_timer t = { NO_MIGRATE }; 23 91 enum bpf_cgroup_storage_type stype; 24 24 - u64 time_start, time_spent = 0; 25 25 - int ret = 0; 26 26 - u32 i; 92 92 + int ret; 27 93 28 94 for_each_cgroup_storage_type(stype) { 29 95 storage[stype] = bpf_cgroup_storage_alloc(prog, stype); ··· 104 38 if (!repeat) 105 39 repeat = 1; 106 40 107 107 - rcu_read_lock(); 108 108 - migrate_disable(); 109 109 - time_start = ktime_get_ns(); 110 110 - for (i = 0; i < repeat; i++) { 41 41 + bpf_test_timer_enter(&t); 42 42 + do { 111 43 bpf_cgroup_storage_set(storage); 112 44 113 45 if (xdp) 114 46 *retval = bpf_prog_run_xdp(prog, ctx); 115 47 else 116 48 *retval = BPF_PROG_RUN(prog, ctx); 117 117 - 118 118 - if (signal_pending(current)) { 119 119 - ret = -EINTR; 120 120 - break; 121 121 - } 122 122 - 123 123 - if (need_resched()) { 124 124 - time_spent += ktime_get_ns() - time_start; 125 125 - migrate_enable(); 126 126 - rcu_read_unlock(); 127 127 - 128 128 - cond_resched(); 129 129 - 130 130 - rcu_read_lock(); 131 131 - migrate_disable(); 132 132 - time_start = ktime_get_ns(); 133 133 - } 134 134 - } 135 135 - time_spent += ktime_get_ns() - time_start; 136 136 - migrate_enable(); 137 137 - rcu_read_unlock(); 138 138 - 139 139 - do_div(time_spent, repeat); 140 140 - *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 49 49 + } while (bpf_test_timer_continue(&t, repeat, &ret, time)); 50 50 + bpf_test_timer_leave(&t); 141 51 142 52 for_each_cgroup_storage_type(stype) 143 53 bpf_cgroup_storage_free(storage[stype]); ··· 716 674 const union bpf_attr *kattr, 717 675 union bpf_attr __user *uattr) 718 676 { 677 677 + struct bpf_test_timer t = { NO_PREEMPT }; 719 678 u32 size = kattr->test.data_size_in; 720 679 struct bpf_flow_dissector ctx = {}; 721 680 u32 repeat = kattr->test.repeat; 722 681 struct bpf_flow_keys *user_ctx; 723 682 struct bpf_flow_keys flow_keys; 724 724 - u64 time_start, time_spent = 0; 725 683 const struct ethhdr *eth; 726 684 unsigned int flags = 0; 727 685 u32 retval, duration; 728 686 void *data; 729 687 int ret; 730 730 - u32 i; 731 688 732 689 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) 733 690 return -EINVAL; ··· 762 721 ctx.data = data; 763 722 ctx.data_end = (__u8 *)data + size; 764 723 765 765 - rcu_read_lock(); 766 766 - preempt_disable(); 767 767 - time_start = ktime_get_ns(); 768 768 - for (i = 0; i < repeat; i++) { 724 724 + bpf_test_timer_enter(&t); 725 725 + do { 769 726 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 770 727 size, flags); 728 728 + } while (bpf_test_timer_continue(&t, repeat, &ret, &duration)); 729 729 + bpf_test_timer_leave(&t); 771 730 772 772 - if (signal_pending(current)) { 773 773 - preempt_enable(); 774 774 - rcu_read_unlock(); 775 775 - 776 776 - ret = -EINTR; 777 777 - goto out; 778 778 - } 779 779 - 780 780 - if (need_resched()) { 781 781 - time_spent += ktime_get_ns() - time_start; 782 782 - preempt_enable(); 783 783 - rcu_read_unlock(); 784 784 - 785 785 - cond_resched(); 786 786 - 787 787 - rcu_read_lock(); 788 788 - preempt_disable(); 789 789 - time_start = ktime_get_ns(); 790 790 - } 791 791 - } 792 792 - time_spent += ktime_get_ns() - time_start; 793 793 - preempt_enable(); 794 794 - rcu_read_unlock(); 795 795 - 796 796 - do_div(time_spent, repeat); 797 797 - duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 731 731 + if (ret < 0) 732 732 + goto out; 798 733 799 734 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), 800 735 retval, duration); ··· 781 764 out: 782 765 kfree(user_ctx); 783 766 kfree(data); 767 767 + return ret; 768 768 + } 769 769 + 770 770 + int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr, 771 771 + union bpf_attr __user *uattr) 772 772 + { 773 773 + struct bpf_test_timer t = { NO_PREEMPT }; 774 774 + struct bpf_prog_array *progs = NULL; 775 775 + struct bpf_sk_lookup_kern ctx = {}; 776 776 + u32 repeat = kattr->test.repeat; 777 777 + struct bpf_sk_lookup *user_ctx; 778 778 + u32 retval, duration; 779 779 + int ret = -EINVAL; 780 780 + 781 781 + if (prog->type != BPF_PROG_TYPE_SK_LOOKUP) 782 782 + return -EINVAL; 783 783 + 784 784 + if (kattr->test.flags || kattr->test.cpu) 785 785 + return -EINVAL; 786 786 + 787 787 + if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || 788 788 + kattr->test.data_size_out) 789 789 + return -EINVAL; 790 790 + 791 791 + if (!repeat) 792 792 + repeat = 1; 793 793 + 794 794 + user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx)); 795 795 + if (IS_ERR(user_ctx)) 796 796 + return PTR_ERR(user_ctx); 797 797 + 798 798 + if (!user_ctx) 799 799 + return -EINVAL; 800 800 + 801 801 + if (user_ctx->sk) 802 802 + goto out; 803 803 + 804 804 + if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx))) 805 805 + goto out; 806 806 + 807 807 + if (user_ctx->local_port > U16_MAX || user_ctx->remote_port > U16_MAX) { 808 808 + ret = -ERANGE; 809 809 + goto out; 810 810 + } 811 811 + 812 812 + ctx.family = (u16)user_ctx->family; 813 813 + ctx.protocol = (u16)user_ctx->protocol; 814 814 + ctx.dport = (u16)user_ctx->local_port; 815 815 + ctx.sport = (__force __be16)user_ctx->remote_port; 816 816 + 817 817 + switch (ctx.family) { 818 818 + case AF_INET: 819 819 + ctx.v4.daddr = (__force __be32)user_ctx->local_ip4; 820 820 + ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4; 821 821 + break; 822 822 + 823 823 + #if IS_ENABLED(CONFIG_IPV6) 824 824 + case AF_INET6: 825 825 + ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6; 826 826 + ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6; 827 827 + break; 828 828 + #endif 829 829 + 830 830 + default: 831 831 + ret = -EAFNOSUPPORT; 832 832 + goto out; 833 833 + } 834 834 + 835 835 + progs = bpf_prog_array_alloc(1, GFP_KERNEL); 836 836 + if (!progs) { 837 837 + ret = -ENOMEM; 838 838 + goto out; 839 839 + } 840 840 + 841 841 + progs->items[0].prog = prog; 842 842 + 843 843 + bpf_test_timer_enter(&t); 844 844 + do { 845 845 + ctx.selected_sk = NULL; 846 846 + retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, BPF_PROG_RUN); 847 847 + } while (bpf_test_timer_continue(&t, repeat, &ret, &duration)); 848 848 + bpf_test_timer_leave(&t); 849 849 + 850 850 + if (ret < 0) 851 851 + goto out; 852 852 + 853 853 + user_ctx->cookie = 0; 854 854 + if (ctx.selected_sk) { 855 855 + if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { 856 856 + ret = -EOPNOTSUPP; 857 857 + goto out; 858 858 + } 859 859 + 860 860 + user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); 861 861 + } 862 862 + 863 863 + ret = bpf_test_finish(kattr, uattr, NULL, 0, retval, duration); 864 864 + if (!ret) 865 865 + ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx)); 866 866 + 867 867 + out: 868 868 + bpf_prog_array_free(progs); 869 869 + kfree(user_ctx); 784 870 return ret; 785 871 }

+4 -2

net/core/Makefile

reviewed

··· 16 16 obj-y += net-sysfs.o 17 17 obj-$(CONFIG_PAGE_POOL) += page_pool.o 18 18 obj-$(CONFIG_PROC_FS) += net-procfs.o 19 19 - obj-$(CONFIG_NET_SOCK_MSG) += skmsg.o 20 19 obj-$(CONFIG_NET_PKTGEN) += pktgen.o 21 20 obj-$(CONFIG_NETPOLL) += netpoll.o 22 21 obj-$(CONFIG_FIB_RULES) += fib_rules.o ··· 27 28 obj-$(CONFIG_CGROUP_NET_CLASSID) += netclassid_cgroup.o 28 29 obj-$(CONFIG_LWTUNNEL) += lwtunnel.o 29 30 obj-$(CONFIG_LWTUNNEL_BPF) += lwt_bpf.o 30 30 - obj-$(CONFIG_BPF_STREAM_PARSER) += sock_map.o 31 31 obj-$(CONFIG_DST_CACHE) += dst_cache.o 32 32 obj-$(CONFIG_HWBM) += hwbm.o 33 33 obj-$(CONFIG_NET_DEVLINK) += devlink.o 34 34 obj-$(CONFIG_GRO_CELLS) += gro_cells.o 35 35 obj-$(CONFIG_FAILOVER) += failover.o 36 36 + ifeq ($(CONFIG_INET),y) 37 37 + obj-$(CONFIG_NET_SOCK_MSG) += skmsg.o 38 38 + obj-$(CONFIG_BPF_SYSCALL) += sock_map.o 39 39 + endif 36 40 obj-$(CONFIG_BPF_SYSCALL) += bpf_sk_storage.o

+1 -1

net/core/bpf_sk_storage.c

reviewed

··· 89 89 90 90 smap = (struct bpf_local_storage_map *)map; 91 91 bpf_local_storage_cache_idx_free(&sk_cache, smap->cache_idx); 92 92 - bpf_local_storage_map_free(smap); 92 92 + bpf_local_storage_map_free(smap, NULL); 93 93 } 94 94 95 95 static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr)

+114 -153

net/core/filter.c

reviewed

··· 1863 1863 static inline int sk_skb_try_make_writable(struct sk_buff *skb, 1864 1864 unsigned int write_len) 1865 1865 { 1866 1866 - int err = __bpf_try_make_writable(skb, write_len); 1867 1867 - 1868 1868 - bpf_compute_data_end_sk_skb(skb); 1869 1869 - return err; 1866 1866 + return __bpf_try_make_writable(skb, write_len); 1870 1867 } 1871 1868 1872 1869 BPF_CALL_2(sk_skb_pull_data, struct sk_buff *, skb, u32, len) ··· 3409 3412 BPF_F_ADJ_ROOM_ENCAP_L3_MASK | \ 3410 3413 BPF_F_ADJ_ROOM_ENCAP_L4_GRE | \ 3411 3414 BPF_F_ADJ_ROOM_ENCAP_L4_UDP | \ 3415 3415 + BPF_F_ADJ_ROOM_ENCAP_L2_ETH | \ 3412 3416 BPF_F_ADJ_ROOM_ENCAP_L2( \ 3413 3417 BPF_ADJ_ROOM_ENCAP_L2_MASK)) 3414 3418 ··· 3446 3448 flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP) 3447 3449 return -EINVAL; 3448 3450 3451 3451 + if (flags & BPF_F_ADJ_ROOM_ENCAP_L2_ETH && 3452 3452 + inner_mac_len < ETH_HLEN) 3453 3453 + return -EINVAL; 3454 3454 + 3449 3455 if (skb->encapsulation) 3450 3456 return -EALREADY; 3451 3457 ··· 3468 3466 skb->inner_mac_header = inner_net - inner_mac_len; 3469 3467 skb->inner_network_header = inner_net; 3470 3468 skb->inner_transport_header = inner_trans; 3471 3471 - skb_set_inner_protocol(skb, skb->protocol); 3469 3469 + 3470 3470 + if (flags & BPF_F_ADJ_ROOM_ENCAP_L2_ETH) 3471 3471 + skb_set_inner_protocol(skb, htons(ETH_P_TEB)); 3472 3472 + else 3473 3473 + skb_set_inner_protocol(skb, skb->protocol); 3472 3474 3473 3475 skb->encapsulation = 1; 3474 3476 skb_set_network_header(skb, mac_len); ··· 3583 3577 return -ENOMEM; 3584 3578 __skb_pull(skb, len_diff_abs); 3585 3579 } 3586 3586 - bpf_compute_data_end_sk_skb(skb); 3587 3580 if (tls_sw_has_ctx_rx(skb->sk)) { 3588 3581 struct strp_msg *rxm = strp_msg(skb); 3589 3582 ··· 3747 3742 BPF_CALL_3(sk_skb_change_tail, struct sk_buff *, skb, u32, new_len, 3748 3743 u64, flags) 3749 3744 { 3750 3750 - int ret = __bpf_skb_change_tail(skb, new_len, flags); 3751 3751 - 3752 3752 - bpf_compute_data_end_sk_skb(skb); 3753 3753 - return ret; 3745 3745 + return __bpf_skb_change_tail(skb, new_len, flags); 3754 3746 } 3755 3747 3756 3748 static const struct bpf_func_proto sk_skb_change_tail_proto = { ··· 3810 3808 BPF_CALL_3(sk_skb_change_head, struct sk_buff *, skb, u32, head_room, 3811 3809 u64, flags) 3812 3810 { 3813 3813 - int ret = __bpf_skb_change_head(skb, head_room, flags); 3814 3814 - 3815 3815 - bpf_compute_data_end_sk_skb(skb); 3816 3816 - return ret; 3811 3811 + return __bpf_skb_change_head(skb, head_room, flags); 3817 3812 } 3818 3813 3819 3814 static const struct bpf_func_proto sk_skb_change_head_proto = { ··· 3918 3919 .arg2_type = ARG_ANYTHING, 3919 3920 }; 3920 3921 3921 3921 - static int __bpf_tx_xdp_map(struct net_device *dev_rx, void *fwd, 3922 3922 - struct bpf_map *map, struct xdp_buff *xdp) 3923 3923 - { 3924 3924 - switch (map->map_type) { 3925 3925 - case BPF_MAP_TYPE_DEVMAP: 3926 3926 - case BPF_MAP_TYPE_DEVMAP_HASH: 3927 3927 - return dev_map_enqueue(fwd, xdp, dev_rx); 3928 3928 - case BPF_MAP_TYPE_CPUMAP: 3929 3929 - return cpu_map_enqueue(fwd, xdp, dev_rx); 3930 3930 - case BPF_MAP_TYPE_XSKMAP: 3931 3931 - return __xsk_map_redirect(fwd, xdp); 3932 3932 - default: 3933 3933 - return -EBADRQC; 3934 3934 - } 3935 3935 - return 0; 3936 3936 - } 3937 3937 - 3938 3922 void xdp_do_flush(void) 3939 3923 { 3940 3924 __dev_flush(); ··· 3926 3944 } 3927 3945 EXPORT_SYMBOL_GPL(xdp_do_flush); 3928 3946 3929 3929 - static inline void *__xdp_map_lookup_elem(struct bpf_map *map, u32 index) 3930 3930 - { 3931 3931 - switch (map->map_type) { 3932 3932 - case BPF_MAP_TYPE_DEVMAP: 3933 3933 - return __dev_map_lookup_elem(map, index); 3934 3934 - case BPF_MAP_TYPE_DEVMAP_HASH: 3935 3935 - return __dev_map_hash_lookup_elem(map, index); 3936 3936 - case BPF_MAP_TYPE_CPUMAP: 3937 3937 - return __cpu_map_lookup_elem(map, index); 3938 3938 - case BPF_MAP_TYPE_XSKMAP: 3939 3939 - return __xsk_map_lookup_elem(map, index); 3940 3940 - default: 3941 3941 - return NULL; 3942 3942 - } 3943 3943 - } 3944 3944 - 3945 3945 - void bpf_clear_redirect_map(struct bpf_map *map) 3946 3946 - { 3947 3947 - struct bpf_redirect_info *ri; 3948 3948 - int cpu; 3949 3949 - 3950 3950 - for_each_possible_cpu(cpu) { 3951 3951 - ri = per_cpu_ptr(&bpf_redirect_info, cpu); 3952 3952 - /* Avoid polluting remote cacheline due to writes if 3953 3953 - * not needed. Once we pass this test, we need the 3954 3954 - * cmpxchg() to make sure it hasn't been changed in 3955 3955 - * the meantime by remote CPU. 3956 3956 - */ 3957 3957 - if (unlikely(READ_ONCE(ri->map) == map)) 3958 3958 - cmpxchg(&ri->map, map, NULL); 3959 3959 - } 3960 3960 - } 3961 3961 - 3962 3947 int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, 3963 3948 struct bpf_prog *xdp_prog) 3964 3949 { 3965 3950 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); 3966 3966 - struct bpf_map *map = READ_ONCE(ri->map); 3967 3967 - u32 index = ri->tgt_index; 3951 3951 + enum bpf_map_type map_type = ri->map_type; 3968 3952 void *fwd = ri->tgt_value; 3953 3953 + u32 map_id = ri->map_id; 3969 3954 int err; 3970 3955 3971 3971 - ri->tgt_index = 0; 3972 3972 - ri->tgt_value = NULL; 3973 3973 - WRITE_ONCE(ri->map, NULL); 3956 3956 + ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */ 3957 3957 + ri->map_type = BPF_MAP_TYPE_UNSPEC; 3974 3958 3975 3975 - if (unlikely(!map)) { 3976 3976 - fwd = dev_get_by_index_rcu(dev_net(dev), index); 3977 3977 - if (unlikely(!fwd)) { 3978 3978 - err = -EINVAL; 3979 3979 - goto err; 3959 3959 + switch (map_type) { 3960 3960 + case BPF_MAP_TYPE_DEVMAP: 3961 3961 + fallthrough; 3962 3962 + case BPF_MAP_TYPE_DEVMAP_HASH: 3963 3963 + err = dev_map_enqueue(fwd, xdp, dev); 3964 3964 + break; 3965 3965 + case BPF_MAP_TYPE_CPUMAP: 3966 3966 + err = cpu_map_enqueue(fwd, xdp, dev); 3967 3967 + break; 3968 3968 + case BPF_MAP_TYPE_XSKMAP: 3969 3969 + err = __xsk_map_redirect(fwd, xdp); 3970 3970 + break; 3971 3971 + case BPF_MAP_TYPE_UNSPEC: 3972 3972 + if (map_id == INT_MAX) { 3973 3973 + fwd = dev_get_by_index_rcu(dev_net(dev), ri->tgt_index); 3974 3974 + if (unlikely(!fwd)) { 3975 3975 + err = -EINVAL; 3976 3976 + break; 3977 3977 + } 3978 3978 + err = dev_xdp_enqueue(fwd, xdp, dev); 3979 3979 + break; 3980 3980 } 3981 3981 - 3982 3982 - err = dev_xdp_enqueue(fwd, xdp, dev); 3983 3983 - } else { 3984 3984 - err = __bpf_tx_xdp_map(dev, fwd, map, xdp); 3981 3981 + fallthrough; 3982 3982 + default: 3983 3983 + err = -EBADRQC; 3985 3984 } 3986 3985 3987 3986 if (unlikely(err)) 3988 3987 goto err; 3989 3988 3990 3990 - _trace_xdp_redirect_map(dev, xdp_prog, fwd, map, index); 3989 3989 + _trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index); 3991 3990 return 0; 3992 3991 err: 3993 3993 - _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map, index, err); 3992 3992 + _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err); 3994 3993 return err; 3995 3994 } 3996 3995 EXPORT_SYMBOL_GPL(xdp_do_redirect); ··· 3980 4017 struct sk_buff *skb, 3981 4018 struct xdp_buff *xdp, 3982 4019 struct bpf_prog *xdp_prog, 3983 3983 - struct bpf_map *map) 4020 4020 + void *fwd, 4021 4021 + enum bpf_map_type map_type, u32 map_id) 3984 4022 { 3985 4023 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); 3986 3986 - u32 index = ri->tgt_index; 3987 3987 - void *fwd = ri->tgt_value; 3988 3988 - int err = 0; 4024 4024 + int err; 3989 4025 3990 3990 - ri->tgt_index = 0; 3991 3991 - ri->tgt_value = NULL; 3992 3992 - WRITE_ONCE(ri->map, NULL); 3993 3993 - 3994 3994 - if (map->map_type == BPF_MAP_TYPE_DEVMAP || 3995 3995 - map->map_type == BPF_MAP_TYPE_DEVMAP_HASH) { 3996 3996 - struct bpf_dtab_netdev *dst = fwd; 3997 3997 - 3998 3998 - err = dev_map_generic_redirect(dst, skb, xdp_prog); 4026 4026 + switch (map_type) { 4027 4027 + case BPF_MAP_TYPE_DEVMAP: 4028 4028 + fallthrough; 4029 4029 + case BPF_MAP_TYPE_DEVMAP_HASH: 4030 4030 + err = dev_map_generic_redirect(fwd, skb, xdp_prog); 3999 4031 if (unlikely(err)) 4000 4032 goto err; 4001 4001 - } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { 4002 4002 - struct xdp_sock *xs = fwd; 4003 4003 - 4004 4004 - err = xsk_generic_rcv(xs, xdp); 4033 4033 + break; 4034 4034 + case BPF_MAP_TYPE_XSKMAP: 4035 4035 + err = xsk_generic_rcv(fwd, xdp); 4005 4036 if (err) 4006 4037 goto err; 4007 4038 consume_skb(skb); 4008 4008 - } else { 4039 4039 + break; 4040 4040 + default: 4009 4041 /* TODO: Handle BPF_MAP_TYPE_CPUMAP */ 4010 4042 err = -EBADRQC; 4011 4043 goto err; 4012 4044 } 4013 4045 4014 4014 - _trace_xdp_redirect_map(dev, xdp_prog, fwd, map, index); 4046 4046 + _trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index); 4015 4047 return 0; 4016 4048 err: 4017 4017 - _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map, index, err); 4049 4049 + _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err); 4018 4050 return err; 4019 4051 } 4020 4052 ··· 4017 4059 struct xdp_buff *xdp, struct bpf_prog *xdp_prog) 4018 4060 { 4019 4061 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); 4020 4020 - struct bpf_map *map = READ_ONCE(ri->map); 4021 4021 - u32 index = ri->tgt_index; 4022 4022 - struct net_device *fwd; 4023 4023 - int err = 0; 4062 4062 + enum bpf_map_type map_type = ri->map_type; 4063 4063 + void *fwd = ri->tgt_value; 4064 4064 + u32 map_id = ri->map_id; 4065 4065 + int err; 4024 4066 4025 4025 - if (map) 4026 4026 - return xdp_do_generic_redirect_map(dev, skb, xdp, xdp_prog, 4027 4027 - map); 4028 4028 - ri->tgt_index = 0; 4029 4029 - fwd = dev_get_by_index_rcu(dev_net(dev), index); 4030 4030 - if (unlikely(!fwd)) { 4031 4031 - err = -EINVAL; 4032 4032 - goto err; 4067 4067 + ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */ 4068 4068 + ri->map_type = BPF_MAP_TYPE_UNSPEC; 4069 4069 + 4070 4070 + if (map_type == BPF_MAP_TYPE_UNSPEC && map_id == INT_MAX) { 4071 4071 + fwd = dev_get_by_index_rcu(dev_net(dev), ri->tgt_index); 4072 4072 + if (unlikely(!fwd)) { 4073 4073 + err = -EINVAL; 4074 4074 + goto err; 4075 4075 + } 4076 4076 + 4077 4077 + err = xdp_ok_fwd_dev(fwd, skb->len); 4078 4078 + if (unlikely(err)) 4079 4079 + goto err; 4080 4080 + 4081 4081 + skb->dev = fwd; 4082 4082 + _trace_xdp_redirect(dev, xdp_prog, ri->tgt_index); 4083 4083 + generic_xdp_tx(skb, xdp_prog); 4084 4084 + return 0; 4033 4085 } 4034 4086 4035 4035 - err = xdp_ok_fwd_dev(fwd, skb->len); 4036 4036 - if (unlikely(err)) 4037 4037 - goto err; 4038 4038 - 4039 4039 - skb->dev = fwd; 4040 4040 - _trace_xdp_redirect(dev, xdp_prog, index); 4041 4041 - generic_xdp_tx(skb, xdp_prog); 4042 4042 - return 0; 4087 4087 + return xdp_do_generic_redirect_map(dev, skb, xdp, xdp_prog, fwd, map_type, map_id); 4043 4088 err: 4044 4044 - _trace_xdp_redirect_err(dev, xdp_prog, index, err); 4089 4089 + _trace_xdp_redirect_err(dev, xdp_prog, ri->tgt_index, err); 4045 4090 return err; 4046 4091 } 4047 4092 ··· 4055 4094 if (unlikely(flags)) 4056 4095 return XDP_ABORTED; 4057 4096 4058 4058 - ri->flags = flags; 4097 4097 + /* NB! Map type UNSPEC and map_id == INT_MAX (never generated 4098 4098 + * by map_idr) is used for ifindex based XDP redirect. 4099 4099 + */ 4059 4100 ri->tgt_index = ifindex; 4060 4060 - ri->tgt_value = NULL; 4061 4061 - WRITE_ONCE(ri->map, NULL); 4101 4101 + ri->map_id = INT_MAX; 4102 4102 + ri->map_type = BPF_MAP_TYPE_UNSPEC; 4062 4103 4063 4104 return XDP_REDIRECT; 4064 4105 } ··· 4076 4113 BPF_CALL_3(bpf_xdp_redirect_map, struct bpf_map *, map, u32, ifindex, 4077 4114 u64, flags) 4078 4115 { 4079 4079 - struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); 4080 4080 - 4081 4081 - /* Lower bits of the flags are used as return code on lookup failure */ 4082 4082 - if (unlikely(flags > XDP_TX)) 4083 4083 - return XDP_ABORTED; 4084 4084 - 4085 4085 - ri->tgt_value = __xdp_map_lookup_elem(map, ifindex); 4086 4086 - if (unlikely(!ri->tgt_value)) { 4087 4087 - /* If the lookup fails we want to clear out the state in the 4088 4088 - * redirect_info struct completely, so that if an eBPF program 4089 4089 - * performs multiple lookups, the last one always takes 4090 4090 - * precedence. 4091 4091 - */ 4092 4092 - WRITE_ONCE(ri->map, NULL); 4093 4093 - return flags; 4094 4094 - } 4095 4095 - 4096 4096 - ri->flags = flags; 4097 4097 - ri->tgt_index = ifindex; 4098 4098 - WRITE_ONCE(ri->map, map); 4099 4099 - 4100 4100 - return XDP_REDIRECT; 4116 4116 + return map->ops->map_redirect(map, ifindex, flags); 4101 4117 } 4102 4118 4103 4119 static const struct bpf_func_proto bpf_xdp_redirect_map_proto = { ··· 9597 9655 return insn - insn_buf; 9598 9656 } 9599 9657 9658 9658 + /* data_end = skb->data + skb_headlen() */ 9659 9659 + static struct bpf_insn *bpf_convert_data_end_access(const struct bpf_insn *si, 9660 9660 + struct bpf_insn *insn) 9661 9661 + { 9662 9662 + /* si->dst_reg = skb->data */ 9663 9663 + *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data), 9664 9664 + si->dst_reg, si->src_reg, 9665 9665 + offsetof(struct sk_buff, data)); 9666 9666 + /* AX = skb->len */ 9667 9667 + *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, len), 9668 9668 + BPF_REG_AX, si->src_reg, 9669 9669 + offsetof(struct sk_buff, len)); 9670 9670 + /* si->dst_reg = skb->data + skb->len */ 9671 9671 + *insn++ = BPF_ALU64_REG(BPF_ADD, si->dst_reg, BPF_REG_AX); 9672 9672 + /* AX = skb->data_len */ 9673 9673 + *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data_len), 9674 9674 + BPF_REG_AX, si->src_reg, 9675 9675 + offsetof(struct sk_buff, data_len)); 9676 9676 + /* si->dst_reg = skb->data + skb->len - skb->data_len */ 9677 9677 + *insn++ = BPF_ALU64_REG(BPF_SUB, si->dst_reg, BPF_REG_AX); 9678 9678 + 9679 9679 + return insn; 9680 9680 + } 9681 9681 + 9600 9682 static u32 sk_skb_convert_ctx_access(enum bpf_access_type type, 9601 9683 const struct bpf_insn *si, 9602 9684 struct bpf_insn *insn_buf, 9603 9685 struct bpf_prog *prog, u32 *target_size) 9604 9686 { 9605 9687 struct bpf_insn *insn = insn_buf; 9606 9606 - int off; 9607 9688 9608 9689 switch (si->off) { 9609 9690 case offsetof(struct __sk_buff, data_end): 9610 9610 - off = si->off; 9611 9611 - off -= offsetof(struct __sk_buff, data_end); 9612 9612 - off += offsetof(struct sk_buff, cb); 9613 9613 - off += offsetof(struct tcp_skb_cb, bpf.data_end); 9614 9614 - *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, 9615 9615 - si->src_reg, off); 9691 9691 + insn = bpf_convert_data_end_access(si, insn); 9616 9692 break; 9617 9693 default: 9618 9694 return bpf_convert_ctx_access(type, si, insn_buf, prog, ··· 10409 10449 } 10410 10450 10411 10451 const struct bpf_prog_ops sk_lookup_prog_ops = { 10452 10452 + .test_run = bpf_prog_test_run_sk_lookup, 10412 10453 }; 10413 10454 10414 10455 const struct bpf_verifier_ops sk_lookup_verifier_ops = {

+102 -110

net/core/skmsg.c

reviewed

··· 525 525 len = skb->len; 526 526 off = 0; 527 527 start: 528 528 - ingress = tcp_skb_bpf_ingress(skb); 528 528 + ingress = skb_bpf_ingress(skb); 529 529 + skb_bpf_redirect_clear(skb); 529 530 do { 530 531 ret = -EIO; 531 532 if (likely(psock->sk->sk_socket)) ··· 619 618 return link; 620 619 } 621 620 622 622 - void __sk_psock_purge_ingress_msg(struct sk_psock *psock) 621 621 + static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) 623 622 { 624 623 struct sk_msg *msg, *tmp; 625 624 ··· 632 631 633 632 static void sk_psock_zap_ingress(struct sk_psock *psock) 634 633 { 635 635 - __skb_queue_purge(&psock->ingress_skb); 634 634 + struct sk_buff *skb; 635 635 + 636 636 + while ((skb = __skb_dequeue(&psock->ingress_skb)) != NULL) { 637 637 + skb_bpf_redirect_clear(skb); 638 638 + kfree_skb(skb); 639 639 + } 636 640 __sk_psock_purge_ingress_msg(psock); 637 641 } 638 642 ··· 651 645 } 652 646 } 653 647 648 648 + static void sk_psock_done_strp(struct sk_psock *psock); 649 649 + 654 650 static void sk_psock_destroy_deferred(struct work_struct *gc) 655 651 { 656 652 struct sk_psock *psock = container_of(gc, struct sk_psock, gc); 657 653 658 654 /* No sk_callback_lock since already detached. */ 659 655 660 660 - /* Parser has been stopped */ 661 661 - if (psock->progs.skb_parser) 662 662 - strp_done(&psock->parser.strp); 656 656 + sk_psock_done_strp(psock); 663 657 664 658 cancel_work_sync(&psock->work); 665 659 ··· 691 685 write_lock_bh(&sk->sk_callback_lock); 692 686 sk_psock_restore_proto(sk, psock); 693 687 rcu_assign_sk_user_data(sk, NULL); 694 694 - if (psock->progs.skb_parser) 688 688 + if (psock->progs.stream_parser) 695 689 sk_psock_stop_strp(sk, psock); 696 696 - else if (psock->progs.skb_verdict) 690 690 + else if (psock->progs.stream_verdict) 697 691 sk_psock_stop_verdict(sk, psock); 698 692 write_unlock_bh(&sk->sk_callback_lock); 699 693 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); ··· 749 743 } 750 744 EXPORT_SYMBOL_GPL(sk_psock_msg_verdict); 751 745 752 752 - static int sk_psock_bpf_run(struct sk_psock *psock, struct bpf_prog *prog, 753 753 - struct sk_buff *skb) 754 754 - { 755 755 - bpf_compute_data_end_sk_skb(skb); 756 756 - return bpf_prog_run_pin_on_cpu(prog, skb); 757 757 - } 758 758 - 759 759 - static struct sk_psock *sk_psock_from_strp(struct strparser *strp) 760 760 - { 761 761 - struct sk_psock_parser *parser; 762 762 - 763 763 - parser = container_of(strp, struct sk_psock_parser, strp); 764 764 - return container_of(parser, struct sk_psock, parser); 765 765 - } 766 766 - 767 746 static void sk_psock_skb_redirect(struct sk_buff *skb) 768 747 { 769 748 struct sk_psock *psock_other; 770 749 struct sock *sk_other; 771 750 772 772 - sk_other = tcp_skb_bpf_redirect_fetch(skb); 751 751 + sk_other = skb_bpf_redirect_fetch(skb); 773 752 /* This error is a buggy BPF program, it returned a redirect 774 753 * return code, but then didn't set a redirect interface. 775 754 */ ··· 797 806 int ret = __SK_PASS; 798 807 799 808 rcu_read_lock(); 800 800 - prog = READ_ONCE(psock->progs.skb_verdict); 809 809 + prog = READ_ONCE(psock->progs.stream_verdict); 801 810 if (likely(prog)) { 802 811 /* We skip full set_owner_r here because if we do a SK_PASS 803 812 * or SK_DROP we can skip skb memory accounting and use the 804 813 * TLS context. 805 814 */ 806 815 skb->sk = psock->sk; 807 807 - tcp_skb_bpf_redirect_clear(skb); 808 808 - ret = sk_psock_bpf_run(psock, prog, skb); 809 809 - ret = sk_psock_map_verd(ret, tcp_skb_bpf_redirect_fetch(skb)); 816 816 + skb_dst_drop(skb); 817 817 + skb_bpf_redirect_clear(skb); 818 818 + ret = bpf_prog_run_pin_on_cpu(prog, skb); 819 819 + ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb)); 810 820 skb->sk = NULL; 811 821 } 812 822 sk_psock_tls_verdict_apply(skb, psock->sk, ret); ··· 819 827 static void sk_psock_verdict_apply(struct sk_psock *psock, 820 828 struct sk_buff *skb, int verdict) 821 829 { 822 822 - struct tcp_skb_cb *tcp; 823 830 struct sock *sk_other; 824 831 int err = -EIO; 825 832 ··· 830 839 goto out_free; 831 840 } 832 841 833 833 - tcp = TCP_SKB_CB(skb); 834 834 - tcp->bpf.flags |= BPF_F_INGRESS; 842 842 + skb_bpf_set_ingress(skb); 835 843 836 844 /* If the queue is empty then we can submit directly 837 845 * into the msg queue. If its not empty we have to ··· 856 866 } 857 867 } 858 868 869 869 + static void sk_psock_write_space(struct sock *sk) 870 870 + { 871 871 + struct sk_psock *psock; 872 872 + void (*write_space)(struct sock *sk) = NULL; 873 873 + 874 874 + rcu_read_lock(); 875 875 + psock = sk_psock(sk); 876 876 + if (likely(psock)) { 877 877 + if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) 878 878 + schedule_work(&psock->work); 879 879 + write_space = psock->saved_write_space; 880 880 + } 881 881 + rcu_read_unlock(); 882 882 + if (write_space) 883 883 + write_space(sk); 884 884 + } 885 885 + 886 886 + #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER) 859 887 static void sk_psock_strp_read(struct strparser *strp, struct sk_buff *skb) 860 888 { 861 889 struct sk_psock *psock; ··· 889 881 goto out; 890 882 } 891 883 skb_set_owner_r(skb, sk); 892 892 - prog = READ_ONCE(psock->progs.skb_verdict); 884 884 + prog = READ_ONCE(psock->progs.stream_verdict); 893 885 if (likely(prog)) { 894 894 - tcp_skb_bpf_redirect_clear(skb); 895 895 - ret = sk_psock_bpf_run(psock, prog, skb); 896 896 - ret = sk_psock_map_verd(ret, tcp_skb_bpf_redirect_fetch(skb)); 886 886 + skb_dst_drop(skb); 887 887 + skb_bpf_redirect_clear(skb); 888 888 + ret = bpf_prog_run_pin_on_cpu(prog, skb); 889 889 + ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb)); 897 890 } 898 891 sk_psock_verdict_apply(psock, skb, ret); 899 892 out: ··· 908 899 909 900 static int sk_psock_strp_parse(struct strparser *strp, struct sk_buff *skb) 910 901 { 911 911 - struct sk_psock *psock = sk_psock_from_strp(strp); 902 902 + struct sk_psock *psock = container_of(strp, struct sk_psock, strp); 912 903 struct bpf_prog *prog; 913 904 int ret = skb->len; 914 905 915 906 rcu_read_lock(); 916 916 - prog = READ_ONCE(psock->progs.skb_parser); 907 907 + prog = READ_ONCE(psock->progs.stream_parser); 917 908 if (likely(prog)) { 918 909 skb->sk = psock->sk; 919 919 - ret = sk_psock_bpf_run(psock, prog, skb); 910 910 + ret = bpf_prog_run_pin_on_cpu(prog, skb); 920 911 skb->sk = NULL; 921 912 } 922 913 rcu_read_unlock(); ··· 932 923 psock = sk_psock(sk); 933 924 if (likely(psock)) { 934 925 if (tls_sw_has_ctx_rx(sk)) { 935 935 - psock->parser.saved_data_ready(sk); 926 926 + psock->saved_data_ready(sk); 936 927 } else { 937 928 write_lock_bh(&sk->sk_callback_lock); 938 938 - strp_data_ready(&psock->parser.strp); 929 929 + strp_data_ready(&psock->strp); 939 930 write_unlock_bh(&sk->sk_callback_lock); 940 931 } 941 932 } 942 933 rcu_read_unlock(); 943 934 } 935 935 + 936 936 + int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock) 937 937 + { 938 938 + static const struct strp_callbacks cb = { 939 939 + .rcv_msg = sk_psock_strp_read, 940 940 + .read_sock_done = sk_psock_strp_read_done, 941 941 + .parse_msg = sk_psock_strp_parse, 942 942 + }; 943 943 + 944 944 + return strp_init(&psock->strp, sk, &cb); 945 945 + } 946 946 + 947 947 + void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) 948 948 + { 949 949 + if (psock->saved_data_ready) 950 950 + return; 951 951 + 952 952 + psock->saved_data_ready = sk->sk_data_ready; 953 953 + sk->sk_data_ready = sk_psock_strp_data_ready; 954 954 + sk->sk_write_space = sk_psock_write_space; 955 955 + } 956 956 + 957 957 + void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) 958 958 + { 959 959 + if (!psock->saved_data_ready) 960 960 + return; 961 961 + 962 962 + sk->sk_data_ready = psock->saved_data_ready; 963 963 + psock->saved_data_ready = NULL; 964 964 + strp_stop(&psock->strp); 965 965 + } 966 966 + 967 967 + static void sk_psock_done_strp(struct sk_psock *psock) 968 968 + { 969 969 + /* Parser has been stopped */ 970 970 + if (psock->progs.stream_parser) 971 971 + strp_done(&psock->strp); 972 972 + } 973 973 + #else 974 974 + static void sk_psock_done_strp(struct sk_psock *psock) 975 975 + { 976 976 + } 977 977 + #endif /* CONFIG_BPF_STREAM_PARSER */ 944 978 945 979 static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb, 946 980 unsigned int offset, size_t orig_len) ··· 1009 957 goto out; 1010 958 } 1011 959 skb_set_owner_r(skb, sk); 1012 1012 - prog = READ_ONCE(psock->progs.skb_verdict); 960 960 + prog = READ_ONCE(psock->progs.stream_verdict); 1013 961 if (likely(prog)) { 1014 1014 - tcp_skb_bpf_redirect_clear(skb); 1015 1015 - ret = sk_psock_bpf_run(psock, prog, skb); 1016 1016 - ret = sk_psock_map_verd(ret, tcp_skb_bpf_redirect_fetch(skb)); 962 962 + skb_dst_drop(skb); 963 963 + skb_bpf_redirect_clear(skb); 964 964 + ret = bpf_prog_run_pin_on_cpu(prog, skb); 965 965 + ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb)); 1017 966 } 1018 967 sk_psock_verdict_apply(psock, skb, ret); 1019 968 out: ··· 1037 984 sock->ops->read_sock(sk, &desc, sk_psock_verdict_recv); 1038 985 } 1039 986 1040 1040 - static void sk_psock_write_space(struct sock *sk) 1041 1041 - { 1042 1042 - struct sk_psock *psock; 1043 1043 - void (*write_space)(struct sock *sk) = NULL; 1044 1044 - 1045 1045 - rcu_read_lock(); 1046 1046 - psock = sk_psock(sk); 1047 1047 - if (likely(psock)) { 1048 1048 - if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) 1049 1049 - schedule_work(&psock->work); 1050 1050 - write_space = psock->saved_write_space; 1051 1051 - } 1052 1052 - rcu_read_unlock(); 1053 1053 - if (write_space) 1054 1054 - write_space(sk); 1055 1055 - } 1056 1056 - 1057 1057 - int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock) 1058 1058 - { 1059 1059 - static const struct strp_callbacks cb = { 1060 1060 - .rcv_msg = sk_psock_strp_read, 1061 1061 - .read_sock_done = sk_psock_strp_read_done, 1062 1062 - .parse_msg = sk_psock_strp_parse, 1063 1063 - }; 1064 1064 - 1065 1065 - psock->parser.enabled = false; 1066 1066 - return strp_init(&psock->parser.strp, sk, &cb); 1067 1067 - } 1068 1068 - 1069 987 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock) 1070 988 { 1071 1071 - struct sk_psock_parser *parser = &psock->parser; 1072 1072 - 1073 1073 - if (parser->enabled) 989 989 + if (psock->saved_data_ready) 1074 990 return; 1075 991 1076 1076 - parser->saved_data_ready = sk->sk_data_ready; 992 992 + psock->saved_data_ready = sk->sk_data_ready; 1077 993 sk->sk_data_ready = sk_psock_verdict_data_ready; 1078 994 sk->sk_write_space = sk_psock_write_space; 1079 1079 - parser->enabled = true; 1080 1080 - } 1081 1081 - 1082 1082 - void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) 1083 1083 - { 1084 1084 - struct sk_psock_parser *parser = &psock->parser; 1085 1085 - 1086 1086 - if (parser->enabled) 1087 1087 - return; 1088 1088 - 1089 1089 - parser->saved_data_ready = sk->sk_data_ready; 1090 1090 - sk->sk_data_ready = sk_psock_strp_data_ready; 1091 1091 - sk->sk_write_space = sk_psock_write_space; 1092 1092 - parser->enabled = true; 1093 1093 - } 1094 1094 - 1095 1095 - void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) 1096 1096 - { 1097 1097 - struct sk_psock_parser *parser = &psock->parser; 1098 1098 - 1099 1099 - if (!parser->enabled) 1100 1100 - return; 1101 1101 - 1102 1102 - sk->sk_data_ready = parser->saved_data_ready; 1103 1103 - parser->saved_data_ready = NULL; 1104 1104 - strp_stop(&parser->strp); 1105 1105 - parser->enabled = false; 1106 995 } 1107 996 1108 997 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) 1109 998 { 1110 1110 - struct sk_psock_parser *parser = &psock->parser; 1111 1111 - 1112 1112 - if (!parser->enabled) 999 999 + if (!psock->saved_data_ready) 1113 1000 return; 1114 1001 1115 1115 - sk->sk_data_ready = parser->saved_data_ready; 1116 1116 - parser->saved_data_ready = NULL; 1117 1117 - parser->enabled = false; 1002 1002 + sk->sk_data_ready = psock->saved_data_ready; 1003 1003 + psock->saved_data_ready = NULL; 1118 1004 }

+39 -38

net/core/sock_map.c

reviewed

··· 24 24 #define SOCK_CREATE_FLAG_MASK \ 25 25 (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) 26 26 27 27 + static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, 28 28 + struct bpf_prog *old, u32 which); 29 29 + 27 30 static struct bpf_map *sock_map_alloc(union bpf_attr *attr) 28 31 { 29 32 struct bpf_stab *stab; ··· 151 148 struct bpf_map *map = link->map; 152 149 struct bpf_stab *stab = container_of(map, struct bpf_stab, 153 150 map); 154 154 - if (psock->parser.enabled && stab->progs.skb_parser) 151 151 + if (psock->saved_data_ready && stab->progs.stream_parser) 155 152 strp_stop = true; 156 156 - if (psock->parser.enabled && stab->progs.skb_verdict) 153 153 + if (psock->saved_data_ready && stab->progs.stream_verdict) 157 154 verdict_stop = true; 158 155 list_del(&link->list); 159 156 sk_psock_free_link(link); ··· 227 224 static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs, 228 225 struct sock *sk) 229 226 { 230 230 - struct bpf_prog *msg_parser, *skb_parser, *skb_verdict; 227 227 + struct bpf_prog *msg_parser, *stream_parser, *stream_verdict; 231 228 struct sk_psock *psock; 232 229 int ret; 233 230 234 234 - skb_verdict = READ_ONCE(progs->skb_verdict); 235 235 - if (skb_verdict) { 236 236 - skb_verdict = bpf_prog_inc_not_zero(skb_verdict); 237 237 - if (IS_ERR(skb_verdict)) 238 238 - return PTR_ERR(skb_verdict); 231 231 + stream_verdict = READ_ONCE(progs->stream_verdict); 232 232 + if (stream_verdict) { 233 233 + stream_verdict = bpf_prog_inc_not_zero(stream_verdict); 234 234 + if (IS_ERR(stream_verdict)) 235 235 + return PTR_ERR(stream_verdict); 239 236 } 240 237 241 241 - skb_parser = READ_ONCE(progs->skb_parser); 242 242 - if (skb_parser) { 243 243 - skb_parser = bpf_prog_inc_not_zero(skb_parser); 244 244 - if (IS_ERR(skb_parser)) { 245 245 - ret = PTR_ERR(skb_parser); 246 246 - goto out_put_skb_verdict; 238 238 + stream_parser = READ_ONCE(progs->stream_parser); 239 239 + if (stream_parser) { 240 240 + stream_parser = bpf_prog_inc_not_zero(stream_parser); 241 241 + if (IS_ERR(stream_parser)) { 242 242 + ret = PTR_ERR(stream_parser); 243 243 + goto out_put_stream_verdict; 247 244 } 248 245 } 249 246 ··· 252 249 msg_parser = bpf_prog_inc_not_zero(msg_parser); 253 250 if (IS_ERR(msg_parser)) { 254 251 ret = PTR_ERR(msg_parser); 255 255 - goto out_put_skb_parser; 252 252 + goto out_put_stream_parser; 256 253 } 257 254 } 258 255 ··· 264 261 265 262 if (psock) { 266 263 if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) || 267 267 - (skb_parser && READ_ONCE(psock->progs.skb_parser)) || 268 268 - (skb_verdict && READ_ONCE(psock->progs.skb_verdict))) { 264 264 + (stream_parser && READ_ONCE(psock->progs.stream_parser)) || 265 265 + (stream_verdict && READ_ONCE(psock->progs.stream_verdict))) { 269 266 sk_psock_put(sk, psock); 270 267 ret = -EBUSY; 271 268 goto out_progs; ··· 286 283 goto out_drop; 287 284 288 285 write_lock_bh(&sk->sk_callback_lock); 289 289 - if (skb_parser && skb_verdict && !psock->parser.enabled) { 286 286 + if (stream_parser && stream_verdict && !psock->saved_data_ready) { 290 287 ret = sk_psock_init_strp(sk, psock); 291 288 if (ret) 292 289 goto out_unlock_drop; 293 293 - psock_set_prog(&psock->progs.skb_verdict, skb_verdict); 294 294 - psock_set_prog(&psock->progs.skb_parser, skb_parser); 290 290 + psock_set_prog(&psock->progs.stream_verdict, stream_verdict); 291 291 + psock_set_prog(&psock->progs.stream_parser, stream_parser); 295 292 sk_psock_start_strp(sk, psock); 296 296 - } else if (!skb_parser && skb_verdict && !psock->parser.enabled) { 297 297 - psock_set_prog(&psock->progs.skb_verdict, skb_verdict); 293 293 + } else if (!stream_parser && stream_verdict && !psock->saved_data_ready) { 294 294 + psock_set_prog(&psock->progs.stream_verdict, stream_verdict); 298 295 sk_psock_start_verdict(sk,psock); 299 296 } 300 297 write_unlock_bh(&sk->sk_callback_lock); ··· 306 303 out_progs: 307 304 if (msg_parser) 308 305 bpf_prog_put(msg_parser); 309 309 - out_put_skb_parser: 310 310 - if (skb_parser) 311 311 - bpf_prog_put(skb_parser); 312 312 - out_put_skb_verdict: 313 313 - if (skb_verdict) 314 314 - bpf_prog_put(skb_verdict); 306 306 + out_put_stream_parser: 307 307 + if (stream_parser) 308 308 + bpf_prog_put(stream_parser); 309 309 + out_put_stream_verdict: 310 310 + if (stream_verdict) 311 311 + bpf_prog_put(stream_verdict); 315 312 return ret; 316 313 } 317 314 ··· 660 657 BPF_CALL_4(bpf_sk_redirect_map, struct sk_buff *, skb, 661 658 struct bpf_map *, map, u32, key, u64, flags) 662 659 { 663 663 - struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 664 660 struct sock *sk; 665 661 666 662 if (unlikely(flags & ~(BPF_F_INGRESS))) ··· 669 667 if (unlikely(!sk || !sock_map_redirect_allowed(sk))) 670 668 return SK_DROP; 671 669 672 672 - tcb->bpf.flags = flags; 673 673 - tcb->bpf.sk_redir = sk; 670 670 + skb_bpf_set_redir(skb, sk, flags & BPF_F_INGRESS); 674 671 return SK_PASS; 675 672 } 676 673 ··· 1251 1250 BPF_CALL_4(bpf_sk_redirect_hash, struct sk_buff *, skb, 1252 1251 struct bpf_map *, map, void *, key, u64, flags) 1253 1252 { 1254 1254 - struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 1255 1253 struct sock *sk; 1256 1254 1257 1255 if (unlikely(flags & ~(BPF_F_INGRESS))) ··· 1260 1260 if (unlikely(!sk || !sock_map_redirect_allowed(sk))) 1261 1261 return SK_DROP; 1262 1262 1263 1263 - tcb->bpf.flags = flags; 1264 1264 - tcb->bpf.sk_redir = sk; 1263 1263 + skb_bpf_set_redir(skb, sk, flags & BPF_F_INGRESS); 1265 1264 return SK_PASS; 1266 1265 } 1267 1266 ··· 1447 1448 return NULL; 1448 1449 } 1449 1450 1450 1450 - int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, 1451 1451 - struct bpf_prog *old, u32 which) 1451 1451 + static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, 1452 1452 + struct bpf_prog *old, u32 which) 1452 1453 { 1453 1454 struct sk_psock_progs *progs = sock_map_progs(map); 1454 1455 struct bpf_prog **pprog; ··· 1460 1461 case BPF_SK_MSG_VERDICT: 1461 1462 pprog = &progs->msg_parser; 1462 1463 break; 1464 1464 + #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER) 1463 1465 case BPF_SK_SKB_STREAM_PARSER: 1464 1464 - pprog = &progs->skb_parser; 1466 1466 + pprog = &progs->stream_parser; 1465 1467 break; 1468 1468 + #endif 1466 1469 case BPF_SK_SKB_STREAM_VERDICT: 1467 1467 - pprog = &progs->skb_verdict; 1470 1470 + pprog = &progs->stream_verdict; 1468 1471 break; 1469 1472 default: 1470 1473 return -EOPNOTSUPP;

+1 -1

net/ipv4/Makefile

reviewed

··· 62 62 obj-$(CONFIG_TCP_CONG_YEAH) += tcp_yeah.o 63 63 obj-$(CONFIG_TCP_CONG_ILLINOIS) += tcp_illinois.o 64 64 obj-$(CONFIG_NET_SOCK_MSG) += tcp_bpf.o 65 65 - obj-$(CONFIG_BPF_STREAM_PARSER) += udp_bpf.o 65 65 + obj-$(CONFIG_BPF_SYSCALL) += udp_bpf.o 66 66 obj-$(CONFIG_NETLABEL) += cipso_ipv4.o 67 67 68 68 obj-$(CONFIG_XFRM) += xfrm4_policy.o xfrm4_state.o xfrm4_input.o \

+2 -2

net/ipv4/tcp_bpf.c

reviewed

··· 229 229 } 230 230 EXPORT_SYMBOL_GPL(tcp_bpf_sendmsg_redir); 231 231 232 232 - #ifdef CONFIG_BPF_STREAM_PARSER 232 232 + #ifdef CONFIG_BPF_SYSCALL 233 233 static bool tcp_bpf_stream_read(const struct sock *sk) 234 234 { 235 235 struct sk_psock *psock; ··· 629 629 if (prot == &tcp_bpf_prots[family][TCP_BPF_BASE]) 630 630 newsk->sk_prot = sk->sk_prot_creator; 631 631 } 632 632 - #endif /* CONFIG_BPF_STREAM_PARSER */ 632 632 + #endif /* CONFIG_BPF_SYSCALL */

+96 -18

net/xdp/xsk.c

reviewed

··· 445 445 sock_wfree(skb); 446 446 } 447 447 448 448 + static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, 449 449 + struct xdp_desc *desc) 450 450 + { 451 451 + struct xsk_buff_pool *pool = xs->pool; 452 452 + u32 hr, len, ts, offset, copy, copied; 453 453 + struct sk_buff *skb; 454 454 + struct page *page; 455 455 + void *buffer; 456 456 + int err, i; 457 457 + u64 addr; 458 458 + 459 459 + hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(xs->dev->needed_headroom)); 460 460 + 461 461 + skb = sock_alloc_send_skb(&xs->sk, hr, 1, &err); 462 462 + if (unlikely(!skb)) 463 463 + return ERR_PTR(err); 464 464 + 465 465 + skb_reserve(skb, hr); 466 466 + 467 467 + addr = desc->addr; 468 468 + len = desc->len; 469 469 + ts = pool->unaligned ? len : pool->chunk_size; 470 470 + 471 471 + buffer = xsk_buff_raw_get_data(pool, addr); 472 472 + offset = offset_in_page(buffer); 473 473 + addr = buffer - pool->addrs; 474 474 + 475 475 + for (copied = 0, i = 0; copied < len; i++) { 476 476 + page = pool->umem->pgs[addr >> PAGE_SHIFT]; 477 477 + get_page(page); 478 478 + 479 479 + copy = min_t(u32, PAGE_SIZE - offset, len - copied); 480 480 + skb_fill_page_desc(skb, i, page, offset, copy); 481 481 + 482 482 + copied += copy; 483 483 + addr += copy; 484 484 + offset = 0; 485 485 + } 486 486 + 487 487 + skb->len += len; 488 488 + skb->data_len += len; 489 489 + skb->truesize += ts; 490 490 + 491 491 + refcount_add(ts, &xs->sk.sk_wmem_alloc); 492 492 + 493 493 + return skb; 494 494 + } 495 495 + 496 496 + static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, 497 497 + struct xdp_desc *desc) 498 498 + { 499 499 + struct net_device *dev = xs->dev; 500 500 + struct sk_buff *skb; 501 501 + 502 502 + if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) { 503 503 + skb = xsk_build_skb_zerocopy(xs, desc); 504 504 + if (IS_ERR(skb)) 505 505 + return skb; 506 506 + } else { 507 507 + u32 hr, tr, len; 508 508 + void *buffer; 509 509 + int err; 510 510 + 511 511 + hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom)); 512 512 + tr = dev->needed_tailroom; 513 513 + len = desc->len; 514 514 + 515 515 + skb = sock_alloc_send_skb(&xs->sk, hr + len + tr, 1, &err); 516 516 + if (unlikely(!skb)) 517 517 + return ERR_PTR(err); 518 518 + 519 519 + skb_reserve(skb, hr); 520 520 + skb_put(skb, len); 521 521 + 522 522 + buffer = xsk_buff_raw_get_data(xs->pool, desc->addr); 523 523 + err = skb_store_bits(skb, 0, buffer, len); 524 524 + if (unlikely(err)) { 525 525 + kfree_skb(skb); 526 526 + return ERR_PTR(err); 527 527 + } 528 528 + } 529 529 + 530 530 + skb->dev = dev; 531 531 + skb->priority = xs->sk.sk_priority; 532 532 + skb->mark = xs->sk.sk_mark; 533 533 + skb_shinfo(skb)->destructor_arg = (void *)(long)desc->addr; 534 534 + skb->destructor = xsk_destruct_skb; 535 535 + 536 536 + return skb; 537 537 + } 538 538 + 448 539 static int xsk_generic_xmit(struct sock *sk) 449 540 { 450 541 struct xdp_sock *xs = xdp_sk(sk); ··· 552 461 goto out; 553 462 554 463 while (xskq_cons_peek_desc(xs->tx, &desc, xs->pool)) { 555 555 - char *buffer; 556 556 - u64 addr; 557 557 - u32 len; 558 558 - 559 464 if (max_batch-- == 0) { 560 465 err = -EAGAIN; 561 466 goto out; 562 467 } 563 468 564 564 - len = desc.len; 565 565 - skb = sock_alloc_send_skb(sk, len, 1, &err); 566 566 - if (unlikely(!skb)) 469 469 + skb = xsk_build_skb(xs, &desc); 470 470 + if (IS_ERR(skb)) { 471 471 + err = PTR_ERR(skb); 567 472 goto out; 473 473 + } 568 474 569 569 - skb_put(skb, len); 570 570 - addr = desc.addr; 571 571 - buffer = xsk_buff_raw_get_data(xs->pool, addr); 572 572 - err = skb_store_bits(skb, 0, buffer, len); 573 475 /* This is the backpressure mechanism for the Tx path. 574 476 * Reserve space in the completion queue and only proceed 575 477 * if there is space in it. This avoids having to implement 576 478 * any buffering in the Tx path. 577 479 */ 578 480 spin_lock_irqsave(&xs->pool->cq_lock, flags); 579 579 - if (unlikely(err) || xskq_prod_reserve(xs->pool->cq)) { 481 481 + if (xskq_prod_reserve(xs->pool->cq)) { 580 482 spin_unlock_irqrestore(&xs->pool->cq_lock, flags); 581 483 kfree_skb(skb); 582 484 goto out; 583 485 } 584 486 spin_unlock_irqrestore(&xs->pool->cq_lock, flags); 585 585 - 586 586 - skb->dev = xs->dev; 587 587 - skb->priority = sk->sk_priority; 588 588 - skb->mark = sk->sk_mark; 589 589 - skb_shinfo(skb)->destructor_arg = (void *)(long)desc.addr; 590 590 - skb->destructor = xsk_destruct_skb; 591 487 592 488 err = __dev_direct_xmit(skb, xs->queue_id); 593 489 if (err == NETDEV_TX_BUSY) {

+13 -17

net/xdp/xsk_queue.h

reviewed

··· 47 47 u64 queue_empty_descs; 48 48 }; 49 49 50 50 - /* The structure of the shared state of the rings are the same as the 51 51 - * ring buffer in kernel/events/ring_buffer.c. For the Rx and completion 52 52 - * ring, the kernel is the producer and user space is the consumer. For 53 53 - * the Tx and fill rings, the kernel is the consumer and user space is 54 54 - * the producer. 50 50 + /* The structure of the shared state of the rings are a simple 51 51 + * circular buffer, as outlined in 52 52 + * Documentation/core-api/circular-buffers.rst. For the Rx and 53 53 + * completion ring, the kernel is the producer and user space is the 54 54 + * consumer. For the Tx and fill rings, the kernel is the consumer and 55 55 + * user space is the producer. 55 56 * 56 57 * producer consumer 57 58 * 58 58 - * if (LOAD ->consumer) { LOAD ->producer 59 59 - * (A) smp_rmb() (C) 59 59 + * if (LOAD ->consumer) { (A) LOAD.acq ->producer (C) 60 60 * STORE $data LOAD $data 61 61 - * smp_wmb() (B) smp_mb() (D) 62 62 - * STORE ->producer STORE ->consumer 61 61 + * STORE.rel ->producer (B) STORE.rel ->consumer (D) 63 62 * } 64 63 * 65 64 * (A) pairs with (D), and (B) pairs with (C). ··· 77 78 * 78 79 * (A) is a control dependency that separates the load of ->consumer 79 80 * from the stores of $data. In case ->consumer indicates there is no 80 80 - * room in the buffer to store $data we do not. So no barrier is needed. 81 81 + * room in the buffer to store $data we do not. The dependency will 82 82 + * order both of the stores after the loads. So no barrier is needed. 81 83 * 82 84 * (D) protects the load of the data to be observed to happen after the 83 85 * store of the consumer pointer. If we did not have this memory ··· 227 227 228 228 static inline void __xskq_cons_release(struct xsk_queue *q) 229 229 { 230 230 - smp_mb(); /* D, matches A */ 231 231 - WRITE_ONCE(q->ring->consumer, q->cached_cons); 230 230 + smp_store_release(&q->ring->consumer, q->cached_cons); /* D, matchees A */ 232 231 } 233 232 234 233 static inline void __xskq_cons_peek(struct xsk_queue *q) 235 234 { 236 235 /* Refresh the local pointer */ 237 237 - q->cached_prod = READ_ONCE(q->ring->producer); 238 238 - smp_rmb(); /* C, matches B */ 236 236 + q->cached_prod = smp_load_acquire(&q->ring->producer); /* C, matches B */ 239 237 } 240 238 241 239 static inline void xskq_cons_get_entries(struct xsk_queue *q) ··· 395 397 396 398 static inline void __xskq_prod_submit(struct xsk_queue *q, u32 idx) 397 399 { 398 398 - smp_wmb(); /* B, matches C */ 399 399 - 400 400 - WRITE_ONCE(q->ring->producer, idx); 400 400 + smp_store_release(&q->ring->producer, idx); /* B, matches C */ 401 401 } 402 402 403 403 static inline void xskq_prod_submit(struct xsk_queue *q)

+16 -1

net/xdp/xskmap.c

reviewed

··· 87 87 { 88 88 struct xsk_map *m = container_of(map, struct xsk_map, map); 89 89 90 90 - bpf_clear_redirect_map(map); 91 90 synchronize_net(); 92 91 bpf_map_area_free(m); 93 92 } ··· 122 123 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 123 124 *insn++ = BPF_MOV64_IMM(ret, 0); 124 125 return insn - insn_buf; 126 126 + } 127 127 + 128 128 + static void *__xsk_map_lookup_elem(struct bpf_map *map, u32 key) 129 129 + { 130 130 + struct xsk_map *m = container_of(map, struct xsk_map, map); 131 131 + 132 132 + if (key >= map->max_entries) 133 133 + return NULL; 134 134 + 135 135 + return READ_ONCE(m->xsk_map[key]); 125 136 } 126 137 127 138 static void *xsk_map_lookup_elem(struct bpf_map *map, void *key) ··· 224 215 return 0; 225 216 } 226 217 218 218 + static int xsk_map_redirect(struct bpf_map *map, u32 ifindex, u64 flags) 219 219 + { 220 220 + return __bpf_xdp_redirect_map(map, ifindex, flags, __xsk_map_lookup_elem); 221 221 + } 222 222 + 227 223 void xsk_map_try_sock_delete(struct xsk_map *map, struct xdp_sock *xs, 228 224 struct xdp_sock **map_entry) 229 225 { ··· 261 247 .map_check_btf = map_check_no_btf, 262 248 .map_btf_name = "xsk_map", 263 249 .map_btf_id = &xsk_map_btf_id, 250 250 + .map_redirect = xsk_map_redirect, 264 251 };

+154 -37

scripts/bpf_helpers_doc.py scripts/bpf_doc.py

reviewed

··· 2 2 # SPDX-License-Identifier: GPL-2.0-only 3 3 # 4 4 # Copyright (C) 2018-2019 Netronome Systems, Inc. 5 5 + # Copyright (C) 2021 Isovalent, Inc. 5 6 6 7 # In case user attempts to run with Python 2. 7 8 from __future__ import print_function ··· 14 13 class NoHelperFound(BaseException): 15 14 pass 16 15 16 16 + class NoSyscallCommandFound(BaseException): 17 17 + pass 18 18 + 17 19 class ParsingError(BaseException): 18 20 def __init__(self, line='<line not provided>', reader=None): 19 21 if reader: ··· 26 22 else: 27 23 BaseException.__init__(self, 'Error parsing line: %s' % line) 28 24 29 29 - class Helper(object): 25 25 + 26 26 + class APIElement(object): 30 27 """ 31 31 - An object representing the description of an eBPF helper function. 32 32 - @proto: function prototype of the helper function 33 33 - @desc: textual description of the helper function 34 34 - @ret: description of the return value of the helper function 28 28 + An object representing the description of an aspect of the eBPF API. 29 29 + @proto: prototype of the API symbol 30 30 + @desc: textual description of the symbol 31 31 + @ret: (optional) description of any associated return value 35 32 """ 36 33 def __init__(self, proto='', desc='', ret=''): 37 34 self.proto = proto 38 35 self.desc = desc 39 36 self.ret = ret 40 37 38 38 + 39 39 + class Helper(APIElement): 40 40 + """ 41 41 + An object representing the description of an eBPF helper function. 42 42 + @proto: function prototype of the helper function 43 43 + @desc: textual description of the helper function 44 44 + @ret: description of the return value of the helper function 45 45 + """ 41 46 def proto_break_down(self): 42 47 """ 43 48 Break down helper function protocol into smaller chunks: return type, ··· 73 60 74 61 return res 75 62 63 63 + 76 64 class HeaderParser(object): 77 65 """ 78 66 An object used to parse a file in order to extract the documentation of a ··· 86 72 self.reader = open(filename, 'r') 87 73 self.line = '' 88 74 self.helpers = [] 75 75 + self.commands = [] 76 76 + 77 77 + def parse_element(self): 78 78 + proto = self.parse_symbol() 79 79 + desc = self.parse_desc() 80 80 + ret = self.parse_ret() 81 81 + return APIElement(proto=proto, desc=desc, ret=ret) 89 82 90 83 def parse_helper(self): 91 84 proto = self.parse_proto() 92 85 desc = self.parse_desc() 93 86 ret = self.parse_ret() 94 87 return Helper(proto=proto, desc=desc, ret=ret) 88 88 + 89 89 + def parse_symbol(self): 90 90 + p = re.compile(' \* ?(.+)$') 91 91 + capture = p.match(self.line) 92 92 + if not capture: 93 93 + raise NoSyscallCommandFound 94 94 + end_re = re.compile(' \* ?NOTES$') 95 95 + end = end_re.match(self.line) 96 96 + if end: 97 97 + raise NoSyscallCommandFound 98 98 + self.line = self.reader.readline() 99 99 + return capture.group(1) 95 100 96 101 def parse_proto(self): 97 102 # Argument can be of shape: ··· 173 140 break 174 141 return ret 175 142 176 176 - def run(self): 177 177 - # Advance to start of helper function descriptions. 178 178 - offset = self.reader.read().find('* Start of BPF helper function descriptions:') 143 143 + def seek_to(self, target, help_message): 144 144 + self.reader.seek(0) 145 145 + offset = self.reader.read().find(target) 179 146 if offset == -1: 180 180 - raise Exception('Could not find start of eBPF helper descriptions list') 147 147 + raise Exception(help_message) 181 148 self.reader.seek(offset) 182 149 self.reader.readline() 183 150 self.reader.readline() 184 151 self.line = self.reader.readline() 185 152 153 153 + def parse_syscall(self): 154 154 + self.seek_to('* DOC: eBPF Syscall Commands', 155 155 + 'Could not find start of eBPF syscall descriptions list') 156 156 + while True: 157 157 + try: 158 158 + command = self.parse_element() 159 159 + self.commands.append(command) 160 160 + except NoSyscallCommandFound: 161 161 + break 162 162 + 163 163 + def parse_helpers(self): 164 164 + self.seek_to('* Start of BPF helper function descriptions:', 165 165 + 'Could not find start of eBPF helper descriptions list') 186 166 while True: 187 167 try: 188 168 helper = self.parse_helper() ··· 203 157 except NoHelperFound: 204 158 break 205 159 160 160 + def run(self): 161 161 + self.parse_syscall() 162 162 + self.parse_helpers() 206 163 self.reader.close() 207 164 208 165 ############################################################################### ··· 214 165 """ 215 166 A generic class for printers. Printers should be created with an array of 216 167 Helper objects, and implement a way to print them in the desired fashion. 217 217 - @helpers: array of Helper objects to print to standard output 168 168 + @parser: A HeaderParser with objects to print to standard output 218 169 """ 219 219 - def __init__(self, helpers): 220 220 - self.helpers = helpers 170 170 + def __init__(self, parser): 171 171 + self.parser = parser 172 172 + self.elements = [] 221 173 222 174 def print_header(self): 223 175 pass ··· 231 181 232 182 def print_all(self): 233 183 self.print_header() 234 234 - for helper in self.helpers: 235 235 - self.print_one(helper) 184 184 + for elem in self.elements: 185 185 + self.print_one(elem) 236 186 self.print_footer() 187 187 + 237 188 238 189 class PrinterRST(Printer): 239 190 """ 240 240 - A printer for dumping collected information about helpers as a ReStructured 241 241 - Text page compatible with the rst2man program, which can be used to 242 242 - generate a manual page for the helpers. 243 243 - @helpers: array of Helper objects to print to standard output 191 191 + A generic class for printers that print ReStructured Text. Printers should 192 192 + be created with a HeaderParser object, and implement a way to print API 193 193 + elements in the desired fashion. 194 194 + @parser: A HeaderParser with objects to print to standard output 244 195 """ 245 245 - def print_header(self): 246 246 - header = '''\ 196 196 + def __init__(self, parser): 197 197 + self.parser = parser 198 198 + 199 199 + def print_license(self): 200 200 + license = '''\ 247 201 .. Copyright (C) All BPF authors and contributors from 2014 to present. 248 202 .. See git log include/uapi/linux/bpf.h in kernel tree for details. 249 203 .. ··· 275 221 .. 276 222 .. Please do not edit this file. It was generated from the documentation 277 223 .. located in file include/uapi/linux/bpf.h of the Linux kernel sources 278 278 - .. (helpers description), and from scripts/bpf_helpers_doc.py in the same 224 224 + .. (helpers description), and from scripts/bpf_doc.py in the same 279 225 .. repository (header and footer). 226 226 + ''' 227 227 + print(license) 280 228 229 229 + def print_elem(self, elem): 230 230 + if (elem.desc): 231 231 + print('\tDescription') 232 232 + # Do not strip all newline characters: formatted code at the end of 233 233 + # a section must be followed by a blank line. 234 234 + for line in re.sub('\n$', '', elem.desc, count=1).split('\n'): 235 235 + print('{}{}'.format('\t\t' if line else '', line)) 236 236 + 237 237 + if (elem.ret): 238 238 + print('\tReturn') 239 239 + for line in elem.ret.rstrip().split('\n'): 240 240 + print('{}{}'.format('\t\t' if line else '', line)) 241 241 + 242 242 + print('') 243 243 + 244 244 + 245 245 + class PrinterHelpersRST(PrinterRST): 246 246 + """ 247 247 + A printer for dumping collected information about helpers as a ReStructured 248 248 + Text page compatible with the rst2man program, which can be used to 249 249 + generate a manual page for the helpers. 250 250 + @parser: A HeaderParser with Helper objects to print to standard output 251 251 + """ 252 252 + def __init__(self, parser): 253 253 + self.elements = parser.helpers 254 254 + 255 255 + def print_header(self): 256 256 + header = '''\ 281 257 =========== 282 258 BPF-HELPERS 283 259 =========== ··· 348 264 HELPERS 349 265 ======= 350 266 ''' 267 267 + PrinterRST.print_license(self) 351 268 print(header) 352 269 353 270 def print_footer(self): ··· 465 380 466 381 def print_one(self, helper): 467 382 self.print_proto(helper) 383 383 + self.print_elem(helper) 468 384 469 469 - if (helper.desc): 470 470 - print('\tDescription') 471 471 - # Do not strip all newline characters: formatted code at the end of 472 472 - # a section must be followed by a blank line. 473 473 - for line in re.sub('\n$', '', helper.desc, count=1).split('\n'): 474 474 - print('{}{}'.format('\t\t' if line else '', line)) 475 385 476 476 - if (helper.ret): 477 477 - print('\tReturn') 478 478 - for line in helper.ret.rstrip().split('\n'): 479 479 - print('{}{}'.format('\t\t' if line else '', line)) 386 386 + class PrinterSyscallRST(PrinterRST): 387 387 + """ 388 388 + A printer for dumping collected information about the syscall API as a 389 389 + ReStructured Text page compatible with the rst2man program, which can be 390 390 + used to generate a manual page for the syscall. 391 391 + @parser: A HeaderParser with APIElement objects to print to standard 392 392 + output 393 393 + """ 394 394 + def __init__(self, parser): 395 395 + self.elements = parser.commands 480 396 481 481 - print('') 397 397 + def print_header(self): 398 398 + header = '''\ 399 399 + === 400 400 + bpf 401 401 + === 402 402 + ------------------------------------------------------------------------------- 403 403 + Perform a command on an extended BPF object 404 404 + ------------------------------------------------------------------------------- 405 405 + 406 406 + :Manual section: 2 407 407 + 408 408 + COMMANDS 409 409 + ======== 410 410 + ''' 411 411 + PrinterRST.print_license(self) 412 412 + print(header) 413 413 + 414 414 + def print_one(self, command): 415 415 + print('**%s**' % (command.proto)) 416 416 + self.print_elem(command) 417 417 + 482 418 483 419 class PrinterHelpers(Printer): 484 420 """ 485 421 A printer for dumping collected information about helpers as C header to 486 422 be included from BPF program. 487 487 - @helpers: array of Helper objects to print to standard output 423 423 + @parser: A HeaderParser with Helper objects to print to standard output 488 424 """ 425 425 + def __init__(self, parser): 426 426 + self.elements = parser.helpers 489 427 490 428 type_fwds = [ 491 429 'struct bpf_fib_lookup', ··· 619 511 620 512 def print_header(self): 621 513 header = '''\ 622 622 - /* This is auto-generated file. See bpf_helpers_doc.py for details. */ 514 514 + /* This is auto-generated file. See bpf_doc.py for details. */ 623 515 624 516 /* Forward declarations of BPF structs */''' 625 517 ··· 697 589 linuxRoot = os.path.dirname(os.path.dirname(script)) 698 590 bpfh = os.path.join(linuxRoot, 'include/uapi/linux/bpf.h') 699 591 592 592 + printers = { 593 593 + 'helpers': PrinterHelpersRST, 594 594 + 'syscall': PrinterSyscallRST, 595 595 + } 596 596 + 700 597 argParser = argparse.ArgumentParser(description=""" 701 701 - Parse eBPF header file and generate documentation for eBPF helper functions. 598 598 + Parse eBPF header file and generate documentation for the eBPF API. 702 599 The RST-formatted output produced can be turned into a manual page with the 703 600 rst2man utility. 704 601 """) ··· 714 601 default=bpfh) 715 602 else: 716 603 argParser.add_argument('--filename', help='path to include/uapi/linux/bpf.h') 604 604 + argParser.add_argument('target', nargs='?', default='helpers', 605 605 + choices=printers.keys(), help='eBPF API target') 717 606 args = argParser.parse_args() 718 607 719 608 # Parse file. ··· 724 609 725 610 # Print formatted output to standard output. 726 611 if args.header: 727 727 - printer = PrinterHelpers(headerParser.helpers) 612 612 + if args.target != 'helpers': 613 613 + raise NotImplementedError('Only helpers header generation is supported') 614 614 + printer = PrinterHelpers(headerParser) 728 615 else: 729 729 - printer = PrinterRST(headerParser.helpers) 616 616 + printer = printers[args.target](headerParser) 730 617 printer.print_all()

-60

tools/bpf/Makefile.helpers

reviewed

··· 1 1 - # SPDX-License-Identifier: GPL-2.0-only 2 2 - ifndef allow-override 3 3 - include ../scripts/Makefile.include 4 4 - include ../scripts/utilities.mak 5 5 - else 6 6 - # Assume Makefile.helpers is being run from bpftool/Documentation 7 7 - # subdirectory. Go up two more directories to fetch bpf.h header and 8 8 - # associated script. 9 9 - UP2DIR := ../../ 10 10 - endif 11 11 - 12 12 - INSTALL ?= install 13 13 - RM ?= rm -f 14 14 - RMDIR ?= rmdir --ignore-fail-on-non-empty 15 15 - 16 16 - ifeq ($(V),1) 17 17 - Q = 18 18 - else 19 19 - Q = @ 20 20 - endif 21 21 - 22 22 - prefix ?= /usr/local 23 23 - mandir ?= $(prefix)/man 24 24 - man7dir = $(mandir)/man7 25 25 - 26 26 - HELPERS_RST = bpf-helpers.rst 27 27 - MAN7_RST = $(HELPERS_RST) 28 28 - 29 29 - _DOC_MAN7 = $(patsubst %.rst,%.7,$(MAN7_RST)) 30 30 - DOC_MAN7 = $(addprefix $(OUTPUT),$(_DOC_MAN7)) 31 31 - 32 32 - helpers: man7 33 33 - man7: $(DOC_MAN7) 34 34 - 35 35 - RST2MAN_DEP := $(shell command -v rst2man 2>/dev/null) 36 36 - 37 37 - $(OUTPUT)$(HELPERS_RST): $(UP2DIR)../../include/uapi/linux/bpf.h 38 38 - $(QUIET_GEN)$(UP2DIR)../../scripts/bpf_helpers_doc.py --filename $< > $@ 39 39 - 40 40 - $(OUTPUT)%.7: $(OUTPUT)%.rst 41 41 - ifndef RST2MAN_DEP 42 42 - $(error "rst2man not found, but required to generate man pages") 43 43 - endif 44 44 - $(QUIET_GEN)rst2man $< > $@ 45 45 - 46 46 - helpers-clean: 47 47 - $(call QUIET_CLEAN, eBPF_helpers-manpage) 48 48 - $(Q)$(RM) $(DOC_MAN7) $(OUTPUT)$(HELPERS_RST) 49 49 - 50 50 - helpers-install: helpers 51 51 - $(call QUIET_INSTALL, eBPF_helpers-manpage) 52 52 - $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(man7dir) 53 53 - $(Q)$(INSTALL) -m 644 $(DOC_MAN7) $(DESTDIR)$(man7dir) 54 54 - 55 55 - helpers-uninstall: 56 56 - $(call QUIET_UNINST, eBPF_helpers-manpage) 57 57 - $(Q)$(RM) $(addprefix $(DESTDIR)$(man7dir)/,$(_DOC_MAN7)) 58 58 - $(Q)$(RMDIR) $(DESTDIR)$(man7dir) 59 59 - 60 60 - .PHONY: helpers helpers-clean helpers-install helpers-uninstall

+1 -1

tools/bpf/bpf_dbg.c

reviewed

··· 1198 1198 else 1199 1199 return CMD_OK; 1200 1200 bpf_reset(); 1201 1201 - } while (pcap_next_pkt() && (!has_limit || (has_limit && ++i < pkts))); 1201 1201 + } while (pcap_next_pkt() && (!has_limit || (++i < pkts))); 1202 1202 1203 1203 rl_printf("bpf passes:%u fails:%u\n", pass, fail); 1204 1204

+8 -6

tools/bpf/bpf_exp.y

reviewed

··· 185 185 | OP_LDXB number '*' '(' '[' number ']' '&' number ')' { 186 186 if ($2 != 4 || $9 != 0xf) { 187 187 fprintf(stderr, "ldxb offset not supported!\n"); 188 188 - exit(0); 188 188 + exit(1); 189 189 } else { 190 190 bpf_set_curr_instr(BPF_LDX | BPF_MSH | BPF_B, 0, 0, $6); } } 191 191 | OP_LDX number '*' '(' '[' number ']' '&' number ')' { 192 192 if ($2 != 4 || $9 != 0xf) { 193 193 fprintf(stderr, "ldxb offset not supported!\n"); 194 194 - exit(0); 194 194 + exit(1); 195 195 } else { 196 196 bpf_set_curr_instr(BPF_LDX | BPF_MSH | BPF_B, 0, 0, $6); } } 197 197 ; ··· 472 472 { 473 473 if (curr_instr >= BPF_MAXINSNS) { 474 474 fprintf(stderr, "only max %u insns allowed!\n", BPF_MAXINSNS); 475 475 - exit(0); 475 475 + exit(1); 476 476 } 477 477 } 478 478 ··· 522 522 523 523 if (ret == -ENOENT) { 524 524 fprintf(stderr, "no such label \'%s\'!\n", label); 525 525 - exit(0); 525 525 + exit(1); 526 526 } 527 527 528 528 return ret; ··· 549 549 { 550 550 int delta = off - i - 1; 551 551 552 552 - if (delta < 0 || delta > 255) 553 553 - fprintf(stderr, "warning: insn #%d jumps to insn #%d, " 552 552 + if (delta < 0 || delta > 255) { 553 553 + fprintf(stderr, "error: insn #%d jumps to insn #%d, " 554 554 "which is out of range\n", i, off); 555 555 + exit(1); 556 556 + } 555 557 return (uint8_t) delta; 556 558 } 557 559

-1

tools/bpf/bpftool/.gitignore

reviewed

··· 3 3 /bootstrap/ 4 4 /bpftool 5 5 bpftool*.8 6 6 - bpf-helpers.* 7 6 FEATURE-DUMP.bpftool 8 7 feature 9 8 libbpf

+4 -7

tools/bpf/bpftool/Documentation/Makefile

reviewed

··· 16 16 mandir ?= $(prefix)/man 17 17 man8dir = $(mandir)/man8 18 18 19 19 - # Load targets for building eBPF helpers man page. 20 20 - include ../../Makefile.helpers 21 21 - 22 19 MAN8_RST = $(wildcard bpftool*.rst) 23 20 24 21 _DOC_MAN8 = $(patsubst %.rst,%.8,$(MAN8_RST)) 25 22 DOC_MAN8 = $(addprefix $(OUTPUT),$(_DOC_MAN8)) 26 23 27 27 - man: man8 helpers 24 24 + man: man8 28 25 man8: $(DOC_MAN8) 29 26 30 27 RST2MAN_DEP := $(shell command -v rst2man 2>/dev/null) ··· 43 46 endif 44 47 $(QUIET_GEN)( cat $< ; printf "%b" $(call see_also,$<) ) | rst2man $(RST2MAN_OPTS) > $@ 45 48 46 46 - clean: helpers-clean 49 49 + clean: 47 50 $(call QUIET_CLEAN, Documentation) 48 51 $(Q)$(RM) $(DOC_MAN8) 49 52 50 50 - install: man helpers-install 53 53 + install: man 51 54 $(call QUIET_INSTALL, Documentation-man) 52 55 $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(man8dir) 53 56 $(Q)$(INSTALL) -m 644 $(DOC_MAN8) $(DESTDIR)$(man8dir) 54 57 55 55 - uninstall: helpers-uninstall 58 58 + uninstall: 56 59 $(call QUIET_UNINST, Documentation-man) 57 60 $(Q)$(RM) $(addprefix $(DESTDIR)$(man8dir)/,$(_DOC_MAN8)) 58 61 $(Q)$(RMDIR) $(DESTDIR)$(man8dir)

+8

tools/bpf/bpftool/btf.c

reviewed

··· 36 36 [BTF_KIND_FUNC_PROTO] = "FUNC_PROTO", 37 37 [BTF_KIND_VAR] = "VAR", 38 38 [BTF_KIND_DATASEC] = "DATASEC", 39 39 + [BTF_KIND_FLOAT] = "FLOAT", 39 40 }; 40 41 41 42 struct btf_attach_table { ··· 326 325 } 327 326 if (json_output) 328 327 jsonw_end_array(w); 328 328 + break; 329 329 + } 330 330 + case BTF_KIND_FLOAT: { 331 331 + if (json_output) 332 332 + jsonw_uint_field(w, "size", t->size); 333 333 + else 334 334 + printf(" size=%u", t->size); 329 335 break; 330 336 } 331 337 default:

+1

tools/bpf/bpftool/btf_dumper.c

reviewed

··· 596 596 switch (BTF_INFO_KIND(t->info)) { 597 597 case BTF_KIND_INT: 598 598 case BTF_KIND_TYPEDEF: 599 599 + case BTF_KIND_FLOAT: 599 600 BTF_PRINT_ARG("%s ", btf__name_by_offset(btf, t->name_off)); 600 601 break; 601 602 case BTF_KIND_STRUCT:

+4

tools/bpf/bpftool/feature.c

reviewed

··· 336 336 { "CONFIG_BPF_JIT", }, 337 337 /* Avoid compiling eBPF interpreter (use JIT only) */ 338 338 { "CONFIG_BPF_JIT_ALWAYS_ON", }, 339 339 + /* Kernel BTF debug information available */ 340 340 + { "CONFIG_DEBUG_INFO_BTF", }, 341 341 + /* Kernel module BTF debug information available */ 342 342 + { "CONFIG_DEBUG_INFO_BTF_MODULES", }, 339 343 340 344 /* cgroups */ 341 345 { "CONFIG_CGROUPS", },

+3

tools/bpf/bpftool/xlated_dumper.c

reviewed

··· 196 196 else if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) 197 197 snprintf(dd->scratch_buff, sizeof(dd->scratch_buff), 198 198 "map[id:%u][0]+%u", insn->imm, (insn + 1)->imm); 199 199 + else if (insn->src_reg == BPF_PSEUDO_FUNC) 200 200 + snprintf(dd->scratch_buff, sizeof(dd->scratch_buff), 201 201 + "subprog[%+d]", insn->imm); 199 202 else 200 203 snprintf(dd->scratch_buff, sizeof(dd->scratch_buff), 201 204 "0x%llx", (unsigned long long)full_imm);

+8 -1

tools/bpf/runqslower/Makefile

reviewed

··· 16 16 17 17 # Try to detect best kernel BTF source 18 18 KERNEL_REL := $(shell uname -r) 19 19 - VMLINUX_BTF_PATHS := /sys/kernel/btf/vmlinux /boot/vmlinux-$(KERNEL_REL) 19 19 + VMLINUX_BTF_PATHS := $(if $(O),$(O)/vmlinux) \ 20 20 + $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ 21 21 + ../../../vmlinux /sys/kernel/btf/vmlinux \ 22 22 + /boot/vmlinux-$(KERNEL_REL) 20 23 VMLINUX_BTF_PATH := $(or $(VMLINUX_BTF),$(firstword \ 21 24 $(wildcard $(VMLINUX_BTF_PATHS)))) 22 25 ··· 69 66 $(QUIET_MKDIR)mkdir -p $@ 70 67 71 68 $(OUTPUT)/vmlinux.h: $(VMLINUX_BTF_PATH) | $(OUTPUT) $(BPFTOOL) 69 69 + ifeq ($(VMLINUX_H),) 72 70 $(Q)if [ ! -e "$(VMLINUX_BTF_PATH)" ] ; then \ 73 71 echo "Couldn't find kernel BTF; set VMLINUX_BTF to" \ 74 72 "specify its location." >&2; \ 75 73 exit 1;\ 76 74 fi 77 75 $(QUIET_GEN)$(BPFTOOL) btf dump file $(VMLINUX_BTF_PATH) format c > $@ 76 76 + else 77 77 + $(Q)cp "$(VMLINUX_H)" $@ 78 78 + endif 78 79 79 80 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(BPFOBJ_OUTPUT) 80 81 $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(BPFOBJ_OUTPUT) $@

+21 -12

tools/bpf/runqslower/runqslower.bpf.c

reviewed

··· 11 11 const volatile pid_t targ_pid = 0; 12 12 13 13 struct { 14 14 - __uint(type, BPF_MAP_TYPE_HASH); 15 15 - __uint(max_entries, 10240); 16 16 - __type(key, u32); 14 14 + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); 15 15 + __uint(map_flags, BPF_F_NO_PREALLOC); 16 16 + __type(key, int); 17 17 __type(value, u64); 18 18 } start SEC(".maps"); 19 19 ··· 25 25 26 26 /* record enqueue timestamp */ 27 27 __always_inline 28 28 - static int trace_enqueue(u32 tgid, u32 pid) 28 28 + static int trace_enqueue(struct task_struct *t) 29 29 { 30 30 - u64 ts; 30 30 + u32 pid = t->pid; 31 31 + u64 *ptr; 31 32 32 33 if (!pid || (targ_pid && targ_pid != pid)) 33 34 return 0; 34 35 35 35 - ts = bpf_ktime_get_ns(); 36 36 - bpf_map_update_elem(&start, &pid, &ts, 0); 36 36 + ptr = bpf_task_storage_get(&start, t, 0, 37 37 + BPF_LOCAL_STORAGE_GET_F_CREATE); 38 38 + if (!ptr) 39 39 + return 0; 40 40 + 41 41 + *ptr = bpf_ktime_get_ns(); 37 42 return 0; 38 43 } 39 44 ··· 48 43 /* TP_PROTO(struct task_struct *p) */ 49 44 struct task_struct *p = (void *)ctx[0]; 50 45 51 51 - return trace_enqueue(p->tgid, p->pid); 46 46 + return trace_enqueue(p); 52 47 } 53 48 54 49 SEC("tp_btf/sched_wakeup_new") ··· 57 52 /* TP_PROTO(struct task_struct *p) */ 58 53 struct task_struct *p = (void *)ctx[0]; 59 54 60 60 - return trace_enqueue(p->tgid, p->pid); 55 55 + return trace_enqueue(p); 61 56 } 62 57 63 58 SEC("tp_btf/sched_switch") ··· 75 70 76 71 /* ivcsw: treat like an enqueue event and store timestamp */ 77 72 if (prev->state == TASK_RUNNING) 78 78 - trace_enqueue(prev->tgid, prev->pid); 73 73 + trace_enqueue(prev); 79 74 80 75 pid = next->pid; 81 76 77 77 + /* For pid mismatch, save a bpf_task_storage_get */ 78 78 + if (!pid || (targ_pid && targ_pid != pid)) 79 79 + return 0; 80 80 + 82 81 /* fetch timestamp and calculate delta */ 83 83 - tsp = bpf_map_lookup_elem(&start, &pid); 82 82 + tsp = bpf_task_storage_get(&start, next, 0, 0); 84 83 if (!tsp) 85 84 return 0; /* missed enqueue */ 86 85 ··· 100 91 bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, 101 92 &event, sizeof(event)); 102 93 103 103 - bpf_map_delete_elem(&start, &pid); 94 94 + bpf_task_storage_delete(&start, next); 104 95 return 0; 105 96 } 106 97

+759 -3

tools/include/uapi/linux/bpf.h

reviewed

··· 93 93 } map; 94 94 }; 95 95 96 96 - /* BPF syscall commands, see bpf(2) man-page for details. */ 96 96 + /* BPF syscall commands, see bpf(2) man-page for more details. */ 97 97 + /** 98 98 + * DOC: eBPF Syscall Preamble 99 99 + * 100 100 + * The operation to be performed by the **bpf**\ () system call is determined 101 101 + * by the *cmd* argument. Each operation takes an accompanying argument, 102 102 + * provided via *attr*, which is a pointer to a union of type *bpf_attr* (see 103 103 + * below). The size argument is the size of the union pointed to by *attr*. 104 104 + */ 105 105 + /** 106 106 + * DOC: eBPF Syscall Commands 107 107 + * 108 108 + * BPF_MAP_CREATE 109 109 + * Description 110 110 + * Create a map and return a file descriptor that refers to the 111 111 + * map. The close-on-exec file descriptor flag (see **fcntl**\ (2)) 112 112 + * is automatically enabled for the new file descriptor. 113 113 + * 114 114 + * Applying **close**\ (2) to the file descriptor returned by 115 115 + * **BPF_MAP_CREATE** will delete the map (but see NOTES). 116 116 + * 117 117 + * Return 118 118 + * A new file descriptor (a nonnegative integer), or -1 if an 119 119 + * error occurred (in which case, *errno* is set appropriately). 120 120 + * 121 121 + * BPF_MAP_LOOKUP_ELEM 122 122 + * Description 123 123 + * Look up an element with a given *key* in the map referred to 124 124 + * by the file descriptor *map_fd*. 125 125 + * 126 126 + * The *flags* argument may be specified as one of the 127 127 + * following: 128 128 + * 129 129 + * **BPF_F_LOCK** 130 130 + * Look up the value of a spin-locked map without 131 131 + * returning the lock. This must be specified if the 132 132 + * elements contain a spinlock. 133 133 + * 134 134 + * Return 135 135 + * Returns zero on success. On error, -1 is returned and *errno* 136 136 + * is set appropriately. 137 137 + * 138 138 + * BPF_MAP_UPDATE_ELEM 139 139 + * Description 140 140 + * Create or update an element (key/value pair) in a specified map. 141 141 + * 142 142 + * The *flags* argument should be specified as one of the 143 143 + * following: 144 144 + * 145 145 + * **BPF_ANY** 146 146 + * Create a new element or update an existing element. 147 147 + * **BPF_NOEXIST** 148 148 + * Create a new element only if it did not exist. 149 149 + * **BPF_EXIST** 150 150 + * Update an existing element. 151 151 + * **BPF_F_LOCK** 152 152 + * Update a spin_lock-ed map element. 153 153 + * 154 154 + * Return 155 155 + * Returns zero on success. On error, -1 is returned and *errno* 156 156 + * is set appropriately. 157 157 + * 158 158 + * May set *errno* to **EINVAL**, **EPERM**, **ENOMEM**, 159 159 + * **E2BIG**, **EEXIST**, or **ENOENT**. 160 160 + * 161 161 + * **E2BIG** 162 162 + * The number of elements in the map reached the 163 163 + * *max_entries* limit specified at map creation time. 164 164 + * **EEXIST** 165 165 + * If *flags* specifies **BPF_NOEXIST** and the element 166 166 + * with *key* already exists in the map. 167 167 + * **ENOENT** 168 168 + * If *flags* specifies **BPF_EXIST** and the element with 169 169 + * *key* does not exist in the map. 170 170 + * 171 171 + * BPF_MAP_DELETE_ELEM 172 172 + * Description 173 173 + * Look up and delete an element by key in a specified map. 174 174 + * 175 175 + * Return 176 176 + * Returns zero on success. On error, -1 is returned and *errno* 177 177 + * is set appropriately. 178 178 + * 179 179 + * BPF_MAP_GET_NEXT_KEY 180 180 + * Description 181 181 + * Look up an element by key in a specified map and return the key 182 182 + * of the next element. Can be used to iterate over all elements 183 183 + * in the map. 184 184 + * 185 185 + * Return 186 186 + * Returns zero on success. On error, -1 is returned and *errno* 187 187 + * is set appropriately. 188 188 + * 189 189 + * The following cases can be used to iterate over all elements of 190 190 + * the map: 191 191 + * 192 192 + * * If *key* is not found, the operation returns zero and sets 193 193 + * the *next_key* pointer to the key of the first element. 194 194 + * * If *key* is found, the operation returns zero and sets the 195 195 + * *next_key* pointer to the key of the next element. 196 196 + * * If *key* is the last element, returns -1 and *errno* is set 197 197 + * to **ENOENT**. 198 198 + * 199 199 + * May set *errno* to **ENOMEM**, **EFAULT**, **EPERM**, or 200 200 + * **EINVAL** on error. 201 201 + * 202 202 + * BPF_PROG_LOAD 203 203 + * Description 204 204 + * Verify and load an eBPF program, returning a new file 205 205 + * descriptor associated with the program. 206 206 + * 207 207 + * Applying **close**\ (2) to the file descriptor returned by 208 208 + * **BPF_PROG_LOAD** will unload the eBPF program (but see NOTES). 209 209 + * 210 210 + * The close-on-exec file descriptor flag (see **fcntl**\ (2)) is 211 211 + * automatically enabled for the new file descriptor. 212 212 + * 213 213 + * Return 214 214 + * A new file descriptor (a nonnegative integer), or -1 if an 215 215 + * error occurred (in which case, *errno* is set appropriately). 216 216 + * 217 217 + * BPF_OBJ_PIN 218 218 + * Description 219 219 + * Pin an eBPF program or map referred by the specified *bpf_fd* 220 220 + * to the provided *pathname* on the filesystem. 221 221 + * 222 222 + * The *pathname* argument must not contain a dot ("."). 223 223 + * 224 224 + * On success, *pathname* retains a reference to the eBPF object, 225 225 + * preventing deallocation of the object when the original 226 226 + * *bpf_fd* is closed. This allow the eBPF object to live beyond 227 227 + * **close**\ (\ *bpf_fd*\ ), and hence the lifetime of the parent 228 228 + * process. 229 229 + * 230 230 + * Applying **unlink**\ (2) or similar calls to the *pathname* 231 231 + * unpins the object from the filesystem, removing the reference. 232 232 + * If no other file descriptors or filesystem nodes refer to the 233 233 + * same object, it will be deallocated (see NOTES). 234 234 + * 235 235 + * The filesystem type for the parent directory of *pathname* must 236 236 + * be **BPF_FS_MAGIC**. 237 237 + * 238 238 + * Return 239 239 + * Returns zero on success. On error, -1 is returned and *errno* 240 240 + * is set appropriately. 241 241 + * 242 242 + * BPF_OBJ_GET 243 243 + * Description 244 244 + * Open a file descriptor for the eBPF object pinned to the 245 245 + * specified *pathname*. 246 246 + * 247 247 + * Return 248 248 + * A new file descriptor (a nonnegative integer), or -1 if an 249 249 + * error occurred (in which case, *errno* is set appropriately). 250 250 + * 251 251 + * BPF_PROG_ATTACH 252 252 + * Description 253 253 + * Attach an eBPF program to a *target_fd* at the specified 254 254 + * *attach_type* hook. 255 255 + * 256 256 + * The *attach_type* specifies the eBPF attachment point to 257 257 + * attach the program to, and must be one of *bpf_attach_type* 258 258 + * (see below). 259 259 + * 260 260 + * The *attach_bpf_fd* must be a valid file descriptor for a 261 261 + * loaded eBPF program of a cgroup, flow dissector, LIRC, sockmap 262 262 + * or sock_ops type corresponding to the specified *attach_type*. 263 263 + * 264 264 + * The *target_fd* must be a valid file descriptor for a kernel 265 265 + * object which depends on the attach type of *attach_bpf_fd*: 266 266 + * 267 267 + * **BPF_PROG_TYPE_CGROUP_DEVICE**, 268 268 + * **BPF_PROG_TYPE_CGROUP_SKB**, 269 269 + * **BPF_PROG_TYPE_CGROUP_SOCK**, 270 270 + * **BPF_PROG_TYPE_CGROUP_SOCK_ADDR**, 271 271 + * **BPF_PROG_TYPE_CGROUP_SOCKOPT**, 272 272 + * **BPF_PROG_TYPE_CGROUP_SYSCTL**, 273 273 + * **BPF_PROG_TYPE_SOCK_OPS** 274 274 + * 275 275 + * Control Group v2 hierarchy with the eBPF controller 276 276 + * enabled. Requires the kernel to be compiled with 277 277 + * **CONFIG_CGROUP_BPF**. 278 278 + * 279 279 + * **BPF_PROG_TYPE_FLOW_DISSECTOR** 280 280 + * 281 281 + * Network namespace (eg /proc/self/ns/net). 282 282 + * 283 283 + * **BPF_PROG_TYPE_LIRC_MODE2** 284 284 + * 285 285 + * LIRC device path (eg /dev/lircN). Requires the kernel 286 286 + * to be compiled with **CONFIG_BPF_LIRC_MODE2**. 287 287 + * 288 288 + * **BPF_PROG_TYPE_SK_SKB**, 289 289 + * **BPF_PROG_TYPE_SK_MSG** 290 290 + * 291 291 + * eBPF map of socket type (eg **BPF_MAP_TYPE_SOCKHASH**). 292 292 + * 293 293 + * Return 294 294 + * Returns zero on success. On error, -1 is returned and *errno* 295 295 + * is set appropriately. 296 296 + * 297 297 + * BPF_PROG_DETACH 298 298 + * Description 299 299 + * Detach the eBPF program associated with the *target_fd* at the 300 300 + * hook specified by *attach_type*. The program must have been 301 301 + * previously attached using **BPF_PROG_ATTACH**. 302 302 + * 303 303 + * Return 304 304 + * Returns zero on success. On error, -1 is returned and *errno* 305 305 + * is set appropriately. 306 306 + * 307 307 + * BPF_PROG_TEST_RUN 308 308 + * Description 309 309 + * Run the eBPF program associated with the *prog_fd* a *repeat* 310 310 + * number of times against a provided program context *ctx_in* and 311 311 + * data *data_in*, and return the modified program context 312 312 + * *ctx_out*, *data_out* (for example, packet data), result of the 313 313 + * execution *retval*, and *duration* of the test run. 314 314 + * 315 315 + * Return 316 316 + * Returns zero on success. On error, -1 is returned and *errno* 317 317 + * is set appropriately. 318 318 + * 319 319 + * **ENOSPC** 320 320 + * Either *data_size_out* or *ctx_size_out* is too small. 321 321 + * **ENOTSUPP** 322 322 + * This command is not supported by the program type of 323 323 + * the program referred to by *prog_fd*. 324 324 + * 325 325 + * BPF_PROG_GET_NEXT_ID 326 326 + * Description 327 327 + * Fetch the next eBPF program currently loaded into the kernel. 328 328 + * 329 329 + * Looks for the eBPF program with an id greater than *start_id* 330 330 + * and updates *next_id* on success. If no other eBPF programs 331 331 + * remain with ids higher than *start_id*, returns -1 and sets 332 332 + * *errno* to **ENOENT**. 333 333 + * 334 334 + * Return 335 335 + * Returns zero on success. On error, or when no id remains, -1 336 336 + * is returned and *errno* is set appropriately. 337 337 + * 338 338 + * BPF_MAP_GET_NEXT_ID 339 339 + * Description 340 340 + * Fetch the next eBPF map currently loaded into the kernel. 341 341 + * 342 342 + * Looks for the eBPF map with an id greater than *start_id* 343 343 + * and updates *next_id* on success. If no other eBPF maps 344 344 + * remain with ids higher than *start_id*, returns -1 and sets 345 345 + * *errno* to **ENOENT**. 346 346 + * 347 347 + * Return 348 348 + * Returns zero on success. On error, or when no id remains, -1 349 349 + * is returned and *errno* is set appropriately. 350 350 + * 351 351 + * BPF_PROG_GET_FD_BY_ID 352 352 + * Description 353 353 + * Open a file descriptor for the eBPF program corresponding to 354 354 + * *prog_id*. 355 355 + * 356 356 + * Return 357 357 + * A new file descriptor (a nonnegative integer), or -1 if an 358 358 + * error occurred (in which case, *errno* is set appropriately). 359 359 + * 360 360 + * BPF_MAP_GET_FD_BY_ID 361 361 + * Description 362 362 + * Open a file descriptor for the eBPF map corresponding to 363 363 + * *map_id*. 364 364 + * 365 365 + * Return 366 366 + * A new file descriptor (a nonnegative integer), or -1 if an 367 367 + * error occurred (in which case, *errno* is set appropriately). 368 368 + * 369 369 + * BPF_OBJ_GET_INFO_BY_FD 370 370 + * Description 371 371 + * Obtain information about the eBPF object corresponding to 372 372 + * *bpf_fd*. 373 373 + * 374 374 + * Populates up to *info_len* bytes of *info*, which will be in 375 375 + * one of the following formats depending on the eBPF object type 376 376 + * of *bpf_fd*: 377 377 + * 378 378 + * * **struct bpf_prog_info** 379 379 + * * **struct bpf_map_info** 380 380 + * * **struct bpf_btf_info** 381 381 + * * **struct bpf_link_info** 382 382 + * 383 383 + * Return 384 384 + * Returns zero on success. On error, -1 is returned and *errno* 385 385 + * is set appropriately. 386 386 + * 387 387 + * BPF_PROG_QUERY 388 388 + * Description 389 389 + * Obtain information about eBPF programs associated with the 390 390 + * specified *attach_type* hook. 391 391 + * 392 392 + * The *target_fd* must be a valid file descriptor for a kernel 393 393 + * object which depends on the attach type of *attach_bpf_fd*: 394 394 + * 395 395 + * **BPF_PROG_TYPE_CGROUP_DEVICE**, 396 396 + * **BPF_PROG_TYPE_CGROUP_SKB**, 397 397 + * **BPF_PROG_TYPE_CGROUP_SOCK**, 398 398 + * **BPF_PROG_TYPE_CGROUP_SOCK_ADDR**, 399 399 + * **BPF_PROG_TYPE_CGROUP_SOCKOPT**, 400 400 + * **BPF_PROG_TYPE_CGROUP_SYSCTL**, 401 401 + * **BPF_PROG_TYPE_SOCK_OPS** 402 402 + * 403 403 + * Control Group v2 hierarchy with the eBPF controller 404 404 + * enabled. Requires the kernel to be compiled with 405 405 + * **CONFIG_CGROUP_BPF**. 406 406 + * 407 407 + * **BPF_PROG_TYPE_FLOW_DISSECTOR** 408 408 + * 409 409 + * Network namespace (eg /proc/self/ns/net). 410 410 + * 411 411 + * **BPF_PROG_TYPE_LIRC_MODE2** 412 412 + * 413 413 + * LIRC device path (eg /dev/lircN). Requires the kernel 414 414 + * to be compiled with **CONFIG_BPF_LIRC_MODE2**. 415 415 + * 416 416 + * **BPF_PROG_QUERY** always fetches the number of programs 417 417 + * attached and the *attach_flags* which were used to attach those 418 418 + * programs. Additionally, if *prog_ids* is nonzero and the number 419 419 + * of attached programs is less than *prog_cnt*, populates 420 420 + * *prog_ids* with the eBPF program ids of the programs attached 421 421 + * at *target_fd*. 422 422 + * 423 423 + * The following flags may alter the result: 424 424 + * 425 425 + * **BPF_F_QUERY_EFFECTIVE** 426 426 + * Only return information regarding programs which are 427 427 + * currently effective at the specified *target_fd*. 428 428 + * 429 429 + * Return 430 430 + * Returns zero on success. On error, -1 is returned and *errno* 431 431 + * is set appropriately. 432 432 + * 433 433 + * BPF_RAW_TRACEPOINT_OPEN 434 434 + * Description 435 435 + * Attach an eBPF program to a tracepoint *name* to access kernel 436 436 + * internal arguments of the tracepoint in their raw form. 437 437 + * 438 438 + * The *prog_fd* must be a valid file descriptor associated with 439 439 + * a loaded eBPF program of type **BPF_PROG_TYPE_RAW_TRACEPOINT**. 440 440 + * 441 441 + * No ABI guarantees are made about the content of tracepoint 442 442 + * arguments exposed to the corresponding eBPF program. 443 443 + * 444 444 + * Applying **close**\ (2) to the file descriptor returned by 445 445 + * **BPF_RAW_TRACEPOINT_OPEN** will delete the map (but see NOTES). 446 446 + * 447 447 + * Return 448 448 + * A new file descriptor (a nonnegative integer), or -1 if an 449 449 + * error occurred (in which case, *errno* is set appropriately). 450 450 + * 451 451 + * BPF_BTF_LOAD 452 452 + * Description 453 453 + * Verify and load BPF Type Format (BTF) metadata into the kernel, 454 454 + * returning a new file descriptor associated with the metadata. 455 455 + * BTF is described in more detail at 456 456 + * https://www.kernel.org/doc/html/latest/bpf/btf.html. 457 457 + * 458 458 + * The *btf* parameter must point to valid memory providing 459 459 + * *btf_size* bytes of BTF binary metadata. 460 460 + * 461 461 + * The returned file descriptor can be passed to other **bpf**\ () 462 462 + * subcommands such as **BPF_PROG_LOAD** or **BPF_MAP_CREATE** to 463 463 + * associate the BTF with those objects. 464 464 + * 465 465 + * Similar to **BPF_PROG_LOAD**, **BPF_BTF_LOAD** has optional 466 466 + * parameters to specify a *btf_log_buf*, *btf_log_size* and 467 467 + * *btf_log_level* which allow the kernel to return freeform log 468 468 + * output regarding the BTF verification process. 469 469 + * 470 470 + * Return 471 471 + * A new file descriptor (a nonnegative integer), or -1 if an 472 472 + * error occurred (in which case, *errno* is set appropriately). 473 473 + * 474 474 + * BPF_BTF_GET_FD_BY_ID 475 475 + * Description 476 476 + * Open a file descriptor for the BPF Type Format (BTF) 477 477 + * corresponding to *btf_id*. 478 478 + * 479 479 + * Return 480 480 + * A new file descriptor (a nonnegative integer), or -1 if an 481 481 + * error occurred (in which case, *errno* is set appropriately). 482 482 + * 483 483 + * BPF_TASK_FD_QUERY 484 484 + * Description 485 485 + * Obtain information about eBPF programs associated with the 486 486 + * target process identified by *pid* and *fd*. 487 487 + * 488 488 + * If the *pid* and *fd* are associated with a tracepoint, kprobe 489 489 + * or uprobe perf event, then the *prog_id* and *fd_type* will 490 490 + * be populated with the eBPF program id and file descriptor type 491 491 + * of type **bpf_task_fd_type**. If associated with a kprobe or 492 492 + * uprobe, the *probe_offset* and *probe_addr* will also be 493 493 + * populated. Optionally, if *buf* is provided, then up to 494 494 + * *buf_len* bytes of *buf* will be populated with the name of 495 495 + * the tracepoint, kprobe or uprobe. 496 496 + * 497 497 + * The resulting *prog_id* may be introspected in deeper detail 498 498 + * using **BPF_PROG_GET_FD_BY_ID** and **BPF_OBJ_GET_INFO_BY_FD**. 499 499 + * 500 500 + * Return 501 501 + * Returns zero on success. On error, -1 is returned and *errno* 502 502 + * is set appropriately. 503 503 + * 504 504 + * BPF_MAP_LOOKUP_AND_DELETE_ELEM 505 505 + * Description 506 506 + * Look up an element with the given *key* in the map referred to 507 507 + * by the file descriptor *fd*, and if found, delete the element. 508 508 + * 509 509 + * The **BPF_MAP_TYPE_QUEUE** and **BPF_MAP_TYPE_STACK** map types 510 510 + * implement this command as a "pop" operation, deleting the top 511 511 + * element rather than one corresponding to *key*. 512 512 + * The *key* and *key_len* parameters should be zeroed when 513 513 + * issuing this operation for these map types. 514 514 + * 515 515 + * This command is only valid for the following map types: 516 516 + * * **BPF_MAP_TYPE_QUEUE** 517 517 + * * **BPF_MAP_TYPE_STACK** 518 518 + * 519 519 + * Return 520 520 + * Returns zero on success. On error, -1 is returned and *errno* 521 521 + * is set appropriately. 522 522 + * 523 523 + * BPF_MAP_FREEZE 524 524 + * Description 525 525 + * Freeze the permissions of the specified map. 526 526 + * 527 527 + * Write permissions may be frozen by passing zero *flags*. 528 528 + * Upon success, no future syscall invocations may alter the 529 529 + * map state of *map_fd*. Write operations from eBPF programs 530 530 + * are still possible for a frozen map. 531 531 + * 532 532 + * Not supported for maps of type **BPF_MAP_TYPE_STRUCT_OPS**. 533 533 + * 534 534 + * Return 535 535 + * Returns zero on success. On error, -1 is returned and *errno* 536 536 + * is set appropriately. 537 537 + * 538 538 + * BPF_BTF_GET_NEXT_ID 539 539 + * Description 540 540 + * Fetch the next BPF Type Format (BTF) object currently loaded 541 541 + * into the kernel. 542 542 + * 543 543 + * Looks for the BTF object with an id greater than *start_id* 544 544 + * and updates *next_id* on success. If no other BTF objects 545 545 + * remain with ids higher than *start_id*, returns -1 and sets 546 546 + * *errno* to **ENOENT**. 547 547 + * 548 548 + * Return 549 549 + * Returns zero on success. On error, or when no id remains, -1 550 550 + * is returned and *errno* is set appropriately. 551 551 + * 552 552 + * BPF_MAP_LOOKUP_BATCH 553 553 + * Description 554 554 + * Iterate and fetch multiple elements in a map. 555 555 + * 556 556 + * Two opaque values are used to manage batch operations, 557 557 + * *in_batch* and *out_batch*. Initially, *in_batch* must be set 558 558 + * to NULL to begin the batched operation. After each subsequent 559 559 + * **BPF_MAP_LOOKUP_BATCH**, the caller should pass the resultant 560 560 + * *out_batch* as the *in_batch* for the next operation to 561 561 + * continue iteration from the current point. 562 562 + * 563 563 + * The *keys* and *values* are output parameters which must point 564 564 + * to memory large enough to hold *count* items based on the key 565 565 + * and value size of the map *map_fd*. The *keys* buffer must be 566 566 + * of *key_size* * *count*. The *values* buffer must be of 567 567 + * *value_size* * *count*. 568 568 + * 569 569 + * The *elem_flags* argument may be specified as one of the 570 570 + * following: 571 571 + * 572 572 + * **BPF_F_LOCK** 573 573 + * Look up the value of a spin-locked map without 574 574 + * returning the lock. This must be specified if the 575 575 + * elements contain a spinlock. 576 576 + * 577 577 + * On success, *count* elements from the map are copied into the 578 578 + * user buffer, with the keys copied into *keys* and the values 579 579 + * copied into the corresponding indices in *values*. 580 580 + * 581 581 + * If an error is returned and *errno* is not **EFAULT**, *count* 582 582 + * is set to the number of successfully processed elements. 583 583 + * 584 584 + * Return 585 585 + * Returns zero on success. On error, -1 is returned and *errno* 586 586 + * is set appropriately. 587 587 + * 588 588 + * May set *errno* to **ENOSPC** to indicate that *keys* or 589 589 + * *values* is too small to dump an entire bucket during 590 590 + * iteration of a hash-based map type. 591 591 + * 592 592 + * BPF_MAP_LOOKUP_AND_DELETE_BATCH 593 593 + * Description 594 594 + * Iterate and delete all elements in a map. 595 595 + * 596 596 + * This operation has the same behavior as 597 597 + * **BPF_MAP_LOOKUP_BATCH** with two exceptions: 598 598 + * 599 599 + * * Every element that is successfully returned is also deleted 600 600 + * from the map. This is at least *count* elements. Note that 601 601 + * *count* is both an input and an output parameter. 602 602 + * * Upon returning with *errno* set to **EFAULT**, up to 603 603 + * *count* elements may be deleted without returning the keys 604 604 + * and values of the deleted elements. 605 605 + * 606 606 + * Return 607 607 + * Returns zero on success. On error, -1 is returned and *errno* 608 608 + * is set appropriately. 609 609 + * 610 610 + * BPF_MAP_UPDATE_BATCH 611 611 + * Description 612 612 + * Update multiple elements in a map by *key*. 613 613 + * 614 614 + * The *keys* and *values* are input parameters which must point 615 615 + * to memory large enough to hold *count* items based on the key 616 616 + * and value size of the map *map_fd*. The *keys* buffer must be 617 617 + * of *key_size* * *count*. The *values* buffer must be of 618 618 + * *value_size* * *count*. 619 619 + * 620 620 + * Each element specified in *keys* is sequentially updated to the 621 621 + * value in the corresponding index in *values*. The *in_batch* 622 622 + * and *out_batch* parameters are ignored and should be zeroed. 623 623 + * 624 624 + * The *elem_flags* argument should be specified as one of the 625 625 + * following: 626 626 + * 627 627 + * **BPF_ANY** 628 628 + * Create new elements or update a existing elements. 629 629 + * **BPF_NOEXIST** 630 630 + * Create new elements only if they do not exist. 631 631 + * **BPF_EXIST** 632 632 + * Update existing elements. 633 633 + * **BPF_F_LOCK** 634 634 + * Update spin_lock-ed map elements. This must be 635 635 + * specified if the map value contains a spinlock. 636 636 + * 637 637 + * On success, *count* elements from the map are updated. 638 638 + * 639 639 + * If an error is returned and *errno* is not **EFAULT**, *count* 640 640 + * is set to the number of successfully processed elements. 641 641 + * 642 642 + * Return 643 643 + * Returns zero on success. On error, -1 is returned and *errno* 644 644 + * is set appropriately. 645 645 + * 646 646 + * May set *errno* to **EINVAL**, **EPERM**, **ENOMEM**, or 647 647 + * **E2BIG**. **E2BIG** indicates that the number of elements in 648 648 + * the map reached the *max_entries* limit specified at map 649 649 + * creation time. 650 650 + * 651 651 + * May set *errno* to one of the following error codes under 652 652 + * specific circumstances: 653 653 + * 654 654 + * **EEXIST** 655 655 + * If *flags* specifies **BPF_NOEXIST** and the element 656 656 + * with *key* already exists in the map. 657 657 + * **ENOENT** 658 658 + * If *flags* specifies **BPF_EXIST** and the element with 659 659 + * *key* does not exist in the map. 660 660 + * 661 661 + * BPF_MAP_DELETE_BATCH 662 662 + * Description 663 663 + * Delete multiple elements in a map by *key*. 664 664 + * 665 665 + * The *keys* parameter is an input parameter which must point 666 666 + * to memory large enough to hold *count* items based on the key 667 667 + * size of the map *map_fd*, that is, *key_size* * *count*. 668 668 + * 669 669 + * Each element specified in *keys* is sequentially deleted. The 670 670 + * *in_batch*, *out_batch*, and *values* parameters are ignored 671 671 + * and should be zeroed. 672 672 + * 673 673 + * The *elem_flags* argument may be specified as one of the 674 674 + * following: 675 675 + * 676 676 + * **BPF_F_LOCK** 677 677 + * Look up the value of a spin-locked map without 678 678 + * returning the lock. This must be specified if the 679 679 + * elements contain a spinlock. 680 680 + * 681 681 + * On success, *count* elements from the map are updated. 682 682 + * 683 683 + * If an error is returned and *errno* is not **EFAULT**, *count* 684 684 + * is set to the number of successfully processed elements. If 685 685 + * *errno* is **EFAULT**, up to *count* elements may be been 686 686 + * deleted. 687 687 + * 688 688 + * Return 689 689 + * Returns zero on success. On error, -1 is returned and *errno* 690 690 + * is set appropriately. 691 691 + * 692 692 + * BPF_LINK_CREATE 693 693 + * Description 694 694 + * Attach an eBPF program to a *target_fd* at the specified 695 695 + * *attach_type* hook and return a file descriptor handle for 696 696 + * managing the link. 697 697 + * 698 698 + * Return 699 699 + * A new file descriptor (a nonnegative integer), or -1 if an 700 700 + * error occurred (in which case, *errno* is set appropriately). 701 701 + * 702 702 + * BPF_LINK_UPDATE 703 703 + * Description 704 704 + * Update the eBPF program in the specified *link_fd* to 705 705 + * *new_prog_fd*. 706 706 + * 707 707 + * Return 708 708 + * Returns zero on success. On error, -1 is returned and *errno* 709 709 + * is set appropriately. 710 710 + * 711 711 + * BPF_LINK_GET_FD_BY_ID 712 712 + * Description 713 713 + * Open a file descriptor for the eBPF Link corresponding to 714 714 + * *link_id*. 715 715 + * 716 716 + * Return 717 717 + * A new file descriptor (a nonnegative integer), or -1 if an 718 718 + * error occurred (in which case, *errno* is set appropriately). 719 719 + * 720 720 + * BPF_LINK_GET_NEXT_ID 721 721 + * Description 722 722 + * Fetch the next eBPF link currently loaded into the kernel. 723 723 + * 724 724 + * Looks for the eBPF link with an id greater than *start_id* 725 725 + * and updates *next_id* on success. If no other eBPF links 726 726 + * remain with ids higher than *start_id*, returns -1 and sets 727 727 + * *errno* to **ENOENT**. 728 728 + * 729 729 + * Return 730 730 + * Returns zero on success. On error, or when no id remains, -1 731 731 + * is returned and *errno* is set appropriately. 732 732 + * 733 733 + * BPF_ENABLE_STATS 734 734 + * Description 735 735 + * Enable eBPF runtime statistics gathering. 736 736 + * 737 737 + * Runtime statistics gathering for the eBPF runtime is disabled 738 738 + * by default to minimize the corresponding performance overhead. 739 739 + * This command enables statistics globally. 740 740 + * 741 741 + * Multiple programs may independently enable statistics. 742 742 + * After gathering the desired statistics, eBPF runtime statistics 743 743 + * may be disabled again by calling **close**\ (2) for the file 744 744 + * descriptor returned by this function. Statistics will only be 745 745 + * disabled system-wide when all outstanding file descriptors 746 746 + * returned by prior calls for this subcommand are closed. 747 747 + * 748 748 + * Return 749 749 + * A new file descriptor (a nonnegative integer), or -1 if an 750 750 + * error occurred (in which case, *errno* is set appropriately). 751 751 + * 752 752 + * BPF_ITER_CREATE 753 753 + * Description 754 754 + * Create an iterator on top of the specified *link_fd* (as 755 755 + * previously created using **BPF_LINK_CREATE**) and return a 756 756 + * file descriptor that can be used to trigger the iteration. 757 757 + * 758 758 + * If the resulting file descriptor is pinned to the filesystem 759 759 + * using **BPF_OBJ_PIN**, then subsequent **read**\ (2) syscalls 760 760 + * for that path will trigger the iterator to read kernel state 761 761 + * using the eBPF program attached to *link_fd*. 762 762 + * 763 763 + * Return 764 764 + * A new file descriptor (a nonnegative integer), or -1 if an 765 765 + * error occurred (in which case, *errno* is set appropriately). 766 766 + * 767 767 + * BPF_LINK_DETACH 768 768 + * Description 769 769 + * Forcefully detach the specified *link_fd* from its 770 770 + * corresponding attachment point. 771 771 + * 772 772 + * Return 773 773 + * Returns zero on success. On error, -1 is returned and *errno* 774 774 + * is set appropriately. 775 775 + * 776 776 + * BPF_PROG_BIND_MAP 777 777 + * Description 778 778 + * Bind a map to the lifetime of an eBPF program. 779 779 + * 780 780 + * The map identified by *map_fd* is bound to the program 781 781 + * identified by *prog_fd* and only released when *prog_fd* is 782 782 + * released. This may be used in cases where metadata should be 783 783 + * associated with a program which otherwise does not contain any 784 784 + * references to the map (for example, embedded in the eBPF 785 785 + * program instructions). 786 786 + * 787 787 + * Return 788 788 + * Returns zero on success. On error, -1 is returned and *errno* 789 789 + * is set appropriately. 790 790 + * 791 791 + * NOTES 792 792 + * eBPF objects (maps and programs) can be shared between processes. 793 793 + * 794 794 + * * After **fork**\ (2), the child inherits file descriptors 795 795 + * referring to the same eBPF objects. 796 796 + * * File descriptors referring to eBPF objects can be transferred over 797 797 + * **unix**\ (7) domain sockets. 798 798 + * * File descriptors referring to eBPF objects can be duplicated in the 799 799 + * usual way, using **dup**\ (2) and similar calls. 800 800 + * * File descriptors referring to eBPF objects can be pinned to the 801 801 + * filesystem using the **BPF_OBJ_PIN** command of **bpf**\ (2). 802 802 + * 803 803 + * An eBPF object is deallocated only after all file descriptors referring 804 804 + * to the object have been closed and no references remain pinned to the 805 805 + * filesystem or attached (for example, bound to a program or device). 806 806 + */ 97 807 enum bpf_cmd { 98 808 BPF_MAP_CREATE, 99 809 BPF_MAP_LOOKUP_ELEM, ··· 1103 393 * is struct/union. 1104 394 */ 1105 395 #define BPF_PSEUDO_BTF_ID 3 396 396 + /* insn[0].src_reg: BPF_PSEUDO_FUNC 397 397 + * insn[0].imm: insn offset to the func 398 398 + * insn[1].imm: 0 399 399 + * insn[0].off: 0 400 400 + * insn[1].off: 0 401 401 + * ldimm64 rewrite: address of the function 402 402 + * verifier type: PTR_TO_FUNC. 403 403 + */ 404 404 + #define BPF_PSEUDO_FUNC 4 1106 405 1107 406 /* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative 1108 407 * offset to another bpf function ··· 1439 720 * parsed and used to produce a manual page. The workflow is the following, 1440 721 * and requires the rst2man utility: 1441 722 * 1442 1442 - * $ ./scripts/bpf_helpers_doc.py \ 723 723 + * $ ./scripts/bpf_doc.py \ 1443 724 * --filename include/uapi/linux/bpf.h > /tmp/bpf-helpers.rst 1444 725 * $ rst2man /tmp/bpf-helpers.rst > /tmp/bpf-helpers.7 1445 726 * $ man /tmp/bpf-helpers.7 ··· 2483 1764 * * **BPF_F_ADJ_ROOM_ENCAP_L2**\ (*len*): 2484 1765 * Use with ENCAP_L3/L4 flags to further specify the tunnel 2485 1766 * type; *len* is the length of the inner MAC header. 1767 1767 + * 1768 1768 + * * **BPF_F_ADJ_ROOM_ENCAP_L2_ETH**: 1769 1769 + * Use with BPF_F_ADJ_ROOM_ENCAP_L2 flag to further specify the 1770 1770 + * L2 type as Ethernet. 2486 1771 * 2487 1772 * A call to this helper is susceptible to change the underlying 2488 1773 * packet buffer. Therefore, at load time, all checks on pointers ··· 4632 3909 * * **BPF_MTU_CHK_RET_FRAG_NEEDED** 4633 3910 * * **BPF_MTU_CHK_RET_SEGS_TOOBIG** 4634 3911 * 3912 3912 + * long bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, void *callback_ctx, u64 flags) 3913 3913 + * Description 3914 3914 + * For each element in **map**, call **callback_fn** function with 3915 3915 + * **map**, **callback_ctx** and other map-specific parameters. 3916 3916 + * The **callback_fn** should be a static function and 3917 3917 + * the **callback_ctx** should be a pointer to the stack. 3918 3918 + * The **flags** is used to control certain aspects of the helper. 3919 3919 + * Currently, the **flags** must be 0. 3920 3920 + * 3921 3921 + * The following are a list of supported map types and their 3922 3922 + * respective expected callback signatures: 3923 3923 + * 3924 3924 + * BPF_MAP_TYPE_HASH, BPF_MAP_TYPE_PERCPU_HASH, 3925 3925 + * BPF_MAP_TYPE_LRU_HASH, BPF_MAP_TYPE_LRU_PERCPU_HASH, 3926 3926 + * BPF_MAP_TYPE_ARRAY, BPF_MAP_TYPE_PERCPU_ARRAY 3927 3927 + * 3928 3928 + * long (\*callback_fn)(struct bpf_map \*map, const void \*key, void \*value, void \*ctx); 3929 3929 + * 3930 3930 + * For per_cpu maps, the map_value is the value on the cpu where the 3931 3931 + * bpf_prog is running. 3932 3932 + * 3933 3933 + * If **callback_fn** return 0, the helper will continue to the next 3934 3934 + * element. If return value is 1, the helper will skip the rest of 3935 3935 + * elements and return. Other return values are not used now. 3936 3936 + * 3937 3937 + * Return 3938 3938 + * The number of traversed map elements for success, **-EINVAL** for 3939 3939 + * invalid **flags**. 4635 3940 */ 4636 3941 #define __BPF_FUNC_MAPPER(FN) \ 4637 3942 FN(unspec), \ ··· 4826 4075 FN(ima_inode_hash), \ 4827 4076 FN(sock_from_file), \ 4828 4077 FN(check_mtu), \ 4078 4078 + FN(for_each_map_elem), \ 4829 4079 /* */ 4830 4080 4831 4081 /* integer value in 'imm' field of BPF_CALL instruction selects which helper ··· 4920 4168 BPF_F_ADJ_ROOM_ENCAP_L4_GRE = (1ULL << 3), 4921 4169 BPF_F_ADJ_ROOM_ENCAP_L4_UDP = (1ULL << 4), 4922 4170 BPF_F_ADJ_ROOM_NO_CSUM_RESET = (1ULL << 5), 4171 4171 + BPF_F_ADJ_ROOM_ENCAP_L2_ETH = (1ULL << 6), 4923 4172 }; 4924 4173 4925 4174 enum { ··· 5958 5205 5959 5206 /* User accessible data for SK_LOOKUP programs. Add new fields at the end. */ 5960 5207 struct bpf_sk_lookup { 5961 5961 - __bpf_md_ptr(struct bpf_sock *, sk); /* Selected socket */ 5208 5208 + union { 5209 5209 + __bpf_md_ptr(struct bpf_sock *, sk); /* Selected socket */ 5210 5210 + __u64 cookie; /* Non-zero if socket was selected in PROG_TEST_RUN */ 5211 5211 + }; 5962 5212 5963 5213 __u32 family; /* Protocol family (AF_INET, AF_INET6) */ 5964 5214 __u32 protocol; /* IP protocol (IPPROTO_TCP, IPPROTO_UDP) */

+3 -2

tools/include/uapi/linux/btf.h

reviewed

··· 52 52 }; 53 53 }; 54 54 55 55 - #define BTF_INFO_KIND(info) (((info) >> 24) & 0x0f) 55 55 + #define BTF_INFO_KIND(info) (((info) >> 24) & 0x1f) 56 56 #define BTF_INFO_VLEN(info) ((info) & 0xffff) 57 57 #define BTF_INFO_KFLAG(info) ((info) >> 31) 58 58 ··· 72 72 #define BTF_KIND_FUNC_PROTO 13 /* Function Proto */ 73 73 #define BTF_KIND_VAR 14 /* Variable */ 74 74 #define BTF_KIND_DATASEC 15 /* Section */ 75 75 - #define BTF_KIND_MAX BTF_KIND_DATASEC 75 75 + #define BTF_KIND_FLOAT 16 /* Floating point */ 76 76 + #define BTF_KIND_MAX BTF_KIND_FLOAT 76 77 #define NR_BTF_KINDS (BTF_KIND_MAX + 1) 77 78 78 79 /* For some specific BTF_KIND, "struct btf_type" is immediately

+1 -1

tools/lib/bpf/Makefile

reviewed

··· 158 158 $(Q)$(MAKE) $(build)=libbpf OUTPUT=$(STATIC_OBJDIR) 159 159 160 160 $(BPF_HELPER_DEFS): $(srctree)/tools/include/uapi/linux/bpf.h 161 161 - $(QUIET_GEN)$(srctree)/scripts/bpf_helpers_doc.py --header \ 161 161 + $(QUIET_GEN)$(srctree)/scripts/bpf_doc.py --header \ 162 162 --file $(srctree)/tools/include/uapi/linux/bpf.h > $(BPF_HELPER_DEFS) 163 163 164 164 $(OUTPUT)libbpf.so: $(OUTPUT)libbpf.so.$(LIBBPF_VERSION)

+50 -1

tools/lib/bpf/btf.c

reviewed

··· 291 291 case BTF_KIND_PTR: 292 292 case BTF_KIND_TYPEDEF: 293 293 case BTF_KIND_FUNC: 294 294 + case BTF_KIND_FLOAT: 294 295 return base_size; 295 296 case BTF_KIND_INT: 296 297 return base_size + sizeof(__u32); ··· 339 338 case BTF_KIND_PTR: 340 339 case BTF_KIND_TYPEDEF: 341 340 case BTF_KIND_FUNC: 341 341 + case BTF_KIND_FLOAT: 342 342 return 0; 343 343 case BTF_KIND_INT: 344 344 *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1)); ··· 580 578 case BTF_KIND_UNION: 581 579 case BTF_KIND_ENUM: 582 580 case BTF_KIND_DATASEC: 581 581 + case BTF_KIND_FLOAT: 583 582 size = t->size; 584 583 goto done; 585 584 case BTF_KIND_PTR: ··· 624 621 switch (kind) { 625 622 case BTF_KIND_INT: 626 623 case BTF_KIND_ENUM: 624 624 + case BTF_KIND_FLOAT: 627 625 return min(btf_ptr_sz(btf), (size_t)t->size); 628 626 case BTF_KIND_PTR: 629 627 return btf_ptr_sz(btf); ··· 1760 1756 return btf_commit_type(btf, sz); 1761 1757 } 1762 1758 1759 1759 + /* 1760 1760 + * Append new BTF_KIND_FLOAT type with: 1761 1761 + * - *name* - non-empty, non-NULL type name; 1762 1762 + * - *sz* - size of the type, in bytes; 1763 1763 + * Returns: 1764 1764 + * - >0, type ID of newly added BTF type; 1765 1765 + * - <0, on error. 1766 1766 + */ 1767 1767 + int btf__add_float(struct btf *btf, const char *name, size_t byte_sz) 1768 1768 + { 1769 1769 + struct btf_type *t; 1770 1770 + int sz, name_off; 1771 1771 + 1772 1772 + /* non-empty name */ 1773 1773 + if (!name || !name[0]) 1774 1774 + return -EINVAL; 1775 1775 + 1776 1776 + /* byte_sz must be one of the explicitly allowed values */ 1777 1777 + if (byte_sz != 2 && byte_sz != 4 && byte_sz != 8 && byte_sz != 12 && 1778 1778 + byte_sz != 16) 1779 1779 + return -EINVAL; 1780 1780 + 1781 1781 + if (btf_ensure_modifiable(btf)) 1782 1782 + return -ENOMEM; 1783 1783 + 1784 1784 + sz = sizeof(struct btf_type); 1785 1785 + t = btf_add_type_mem(btf, sz); 1786 1786 + if (!t) 1787 1787 + return -ENOMEM; 1788 1788 + 1789 1789 + name_off = btf__add_str(btf, name); 1790 1790 + if (name_off < 0) 1791 1791 + return name_off; 1792 1792 + 1793 1793 + t->name_off = name_off; 1794 1794 + t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0); 1795 1795 + t->size = byte_sz; 1796 1796 + 1797 1797 + return btf_commit_type(btf, sz); 1798 1798 + } 1799 1799 + 1763 1800 /* it's completely legal to append BTF types with type IDs pointing forward to 1764 1801 * types that haven't been appended yet, so we only make sure that id looks 1765 1802 * sane, we can't guarantee that ID will always be valid ··· 1928 1883 * - *byte_sz* - size of the struct, in bytes; 1929 1884 * 1930 1885 * Struct initially has no fields in it. Fields can be added by 1931 1931 - * btf__add_field() right after btf__add_struct() succeeds. 1886 1886 + * btf__add_field() right after btf__add_struct() succeeds. 1932 1887 * 1933 1888 * Returns: 1934 1889 * - >0, type ID of newly added BTF type; ··· 3671 3626 case BTF_KIND_FWD: 3672 3627 case BTF_KIND_TYPEDEF: 3673 3628 case BTF_KIND_FUNC: 3629 3629 + case BTF_KIND_FLOAT: 3674 3630 h = btf_hash_common(t); 3675 3631 break; 3676 3632 case BTF_KIND_INT: ··· 3768 3722 break; 3769 3723 3770 3724 case BTF_KIND_FWD: 3725 3725 + case BTF_KIND_FLOAT: 3771 3726 h = btf_hash_common(t); 3772 3727 for_each_dedup_cand(d, hash_entry, h) { 3773 3728 cand_id = (__u32)(long)hash_entry->value; ··· 4030 3983 return btf_compat_enum(cand_type, canon_type); 4031 3984 4032 3985 case BTF_KIND_FWD: 3986 3986 + case BTF_KIND_FLOAT: 4033 3987 return btf_equal_common(cand_type, canon_type); 4034 3988 4035 3989 case BTF_KIND_CONST: ··· 4527 4479 switch (btf_kind(t)) { 4528 4480 case BTF_KIND_INT: 4529 4481 case BTF_KIND_ENUM: 4482 4482 + case BTF_KIND_FLOAT: 4530 4483 break; 4531 4484 4532 4485 case BTF_KIND_FWD:

+6

tools/lib/bpf/btf.h

reviewed

··· 95 95 LIBBPF_API int btf__add_str(struct btf *btf, const char *s); 96 96 97 97 LIBBPF_API int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding); 98 98 + LIBBPF_API int btf__add_float(struct btf *btf, const char *name, size_t byte_sz); 98 99 LIBBPF_API int btf__add_ptr(struct btf *btf, int ref_type_id); 99 100 LIBBPF_API int btf__add_array(struct btf *btf, 100 101 int index_type_id, int elem_type_id, __u32 nr_elems); ··· 293 292 static inline bool btf_is_datasec(const struct btf_type *t) 294 293 { 295 294 return btf_kind(t) == BTF_KIND_DATASEC; 295 295 + } 296 296 + 297 297 + static inline bool btf_is_float(const struct btf_type *t) 298 298 + { 299 299 + return btf_kind(t) == BTF_KIND_FLOAT; 296 300 } 297 301 298 302 static inline __u8 btf_int_encoding(const struct btf_type *t)

+4

tools/lib/bpf/btf_dump.c

reviewed

··· 279 279 case BTF_KIND_INT: 280 280 case BTF_KIND_ENUM: 281 281 case BTF_KIND_FWD: 282 282 + case BTF_KIND_FLOAT: 282 283 break; 283 284 284 285 case BTF_KIND_VOLATILE: ··· 454 453 455 454 switch (btf_kind(t)) { 456 455 case BTF_KIND_INT: 456 456 + case BTF_KIND_FLOAT: 457 457 tstate->order_state = ORDERED; 458 458 return 0; 459 459 ··· 1135 1133 case BTF_KIND_STRUCT: 1136 1134 case BTF_KIND_UNION: 1137 1135 case BTF_KIND_TYPEDEF: 1136 1136 + case BTF_KIND_FLOAT: 1138 1137 goto done; 1139 1138 default: 1140 1139 pr_warn("unexpected type in decl chain, kind:%u, id:[%u]\n", ··· 1250 1247 1251 1248 switch (kind) { 1252 1249 case BTF_KIND_INT: 1250 1250 + case BTF_KIND_FLOAT: 1253 1251 btf_dump_emit_mods(d, decls); 1254 1252 name = btf_name_of(d, t->name_off); 1255 1253 btf_dump_printf(d, "%s", name);

+95 -10

tools/lib/bpf/libbpf.c

reviewed

··· 178 178 FEAT_PROG_BIND_MAP, 179 179 /* Kernel support for module BTFs */ 180 180 FEAT_MODULE_BTF, 181 181 + /* BTF_KIND_FLOAT support */ 182 182 + FEAT_BTF_FLOAT, 181 183 __FEAT_CNT, 182 184 }; 183 185 ··· 190 188 RELO_CALL, 191 189 RELO_DATA, 192 190 RELO_EXTERN, 191 191 + RELO_SUBPROG_ADDR, 193 192 }; 194 193 195 194 struct reloc_desc { ··· 575 572 insn->src_reg == BPF_PSEUDO_CALL && 576 573 insn->dst_reg == 0 && 577 574 insn->off == 0; 575 575 + } 576 576 + 577 577 + static bool is_ldimm64(struct bpf_insn *insn) 578 578 + { 579 579 + return insn->code == (BPF_LD | BPF_IMM | BPF_DW); 580 580 + } 581 581 + 582 582 + static bool insn_is_pseudo_func(struct bpf_insn *insn) 583 583 + { 584 584 + return is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC; 578 585 } 579 586 580 587 static int ··· 1948 1935 case BTF_KIND_FUNC_PROTO: return "func_proto"; 1949 1936 case BTF_KIND_VAR: return "var"; 1950 1937 case BTF_KIND_DATASEC: return "datasec"; 1938 1938 + case BTF_KIND_FLOAT: return "float"; 1951 1939 default: return "unknown"; 1952 1940 } 1953 1941 } ··· 2398 2384 { 2399 2385 bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC); 2400 2386 bool has_datasec = kernel_supports(FEAT_BTF_DATASEC); 2387 2387 + bool has_float = kernel_supports(FEAT_BTF_FLOAT); 2401 2388 bool has_func = kernel_supports(FEAT_BTF_FUNC); 2402 2389 2403 2403 - return !has_func || !has_datasec || !has_func_global; 2390 2390 + return !has_func || !has_datasec || !has_func_global || !has_float; 2404 2391 } 2405 2392 2406 2393 static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) 2407 2394 { 2408 2395 bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC); 2409 2396 bool has_datasec = kernel_supports(FEAT_BTF_DATASEC); 2397 2397 + bool has_float = kernel_supports(FEAT_BTF_FLOAT); 2410 2398 bool has_func = kernel_supports(FEAT_BTF_FUNC); 2411 2399 struct btf_type *t; 2412 2400 int i, j, vlen; ··· 2461 2445 } else if (!has_func_global && btf_is_func(t)) { 2462 2446 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ 2463 2447 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); 2448 2448 + } else if (!has_float && btf_is_float(t)) { 2449 2449 + /* replace FLOAT with an equally-sized empty STRUCT; 2450 2450 + * since C compilers do not accept e.g. "float" as a 2451 2451 + * valid struct name, make it anonymous 2452 2452 + */ 2453 2453 + t->name_off = 0; 2454 2454 + t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); 2464 2455 } 2465 2456 } 2466 2457 } ··· 2997 2974 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE; 2998 2975 } 2999 2976 2977 2977 + static bool sym_is_subprog(const GElf_Sym *sym, int text_shndx) 2978 2978 + { 2979 2979 + int bind = GELF_ST_BIND(sym->st_info); 2980 2980 + int type = GELF_ST_TYPE(sym->st_info); 2981 2981 + 2982 2982 + /* in .text section */ 2983 2983 + if (sym->st_shndx != text_shndx) 2984 2984 + return false; 2985 2985 + 2986 2986 + /* local function */ 2987 2987 + if (bind == STB_LOCAL && type == STT_SECTION) 2988 2988 + return true; 2989 2989 + 2990 2990 + /* global function */ 2991 2991 + return bind == STB_GLOBAL && type == STT_FUNC; 2992 2992 + } 2993 2993 + 3000 2994 static int find_extern_btf_id(const struct btf *btf, const char *ext_name) 3001 2995 { 3002 2996 const struct btf_type *t; ··· 3435 3395 return 0; 3436 3396 } 3437 3397 3438 3438 - if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) { 3398 3398 + if (!is_ldimm64(insn)) { 3439 3399 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", 3440 3400 prog->name, sym_name, insn_idx, insn->code); 3441 3401 return -LIBBPF_ERRNO__RELOC; ··· 3468 3428 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n", 3469 3429 prog->name, sym_name, shdr_idx); 3470 3430 return -LIBBPF_ERRNO__RELOC; 3431 3431 + } 3432 3432 + 3433 3433 + /* loading subprog addresses */ 3434 3434 + if (sym_is_subprog(sym, obj->efile.text_shndx)) { 3435 3435 + /* global_func: sym->st_value = offset in the section, insn->imm = 0. 3436 3436 + * local_func: sym->st_value = 0, insn->imm = offset in the section. 3437 3437 + */ 3438 3438 + if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { 3439 3439 + pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n", 3440 3440 + prog->name, sym_name, (size_t)sym->st_value, insn->imm); 3441 3441 + return -LIBBPF_ERRNO__RELOC; 3442 3442 + } 3443 3443 + 3444 3444 + reloc_desc->type = RELO_SUBPROG_ADDR; 3445 3445 + reloc_desc->insn_idx = insn_idx; 3446 3446 + reloc_desc->sym_off = sym->st_value; 3447 3447 + return 0; 3471 3448 } 3472 3449 3473 3450 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); ··· 3939 3882 strs, sizeof(strs))); 3940 3883 } 3941 3884 3885 3885 + static int probe_kern_btf_float(void) 3886 3886 + { 3887 3887 + static const char strs[] = "\0float"; 3888 3888 + __u32 types[] = { 3889 3889 + /* float */ 3890 3890 + BTF_TYPE_FLOAT_ENC(1, 4), 3891 3891 + }; 3892 3892 + 3893 3893 + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 3894 3894 + strs, sizeof(strs))); 3895 3895 + } 3896 3896 + 3942 3897 static int probe_kern_array_mmap(void) 3943 3898 { 3944 3899 struct bpf_create_map_attr attr = { ··· 4129 4060 }, 4130 4061 [FEAT_MODULE_BTF] = { 4131 4062 "module BTF support", probe_module_btf, 4063 4063 + }, 4064 4064 + [FEAT_BTF_FLOAT] = { 4065 4065 + "BTF_KIND_FLOAT support", probe_kern_btf_float, 4132 4066 }, 4133 4067 }; 4134 4068 ··· 5638 5566 insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */ 5639 5567 } 5640 5568 5641 5641 - static bool is_ldimm64(struct bpf_insn *insn) 5642 5642 - { 5643 5643 - return insn->code == (BPF_LD | BPF_IMM | BPF_DW); 5644 5644 - } 5645 5645 - 5646 5569 static int insn_bpf_size_to_bytes(struct bpf_insn *insn) 5647 5570 { 5648 5571 switch (BPF_SIZE(insn->code)) { ··· 6239 6172 } 6240 6173 relo->processed = true; 6241 6174 break; 6175 6175 + case RELO_SUBPROG_ADDR: 6176 6176 + insn[0].src_reg = BPF_PSEUDO_FUNC; 6177 6177 + /* will be handled as a follow up pass */ 6178 6178 + break; 6242 6179 case RELO_CALL: 6243 6180 /* will be handled as a follow up pass */ 6244 6181 break; ··· 6429 6358 6430 6359 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { 6431 6360 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 6432 6432 - if (!insn_is_subprog_call(insn)) 6361 6361 + if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn)) 6433 6362 continue; 6434 6363 6435 6364 relo = find_prog_insn_relo(prog, insn_idx); 6436 6436 - if (relo && relo->type != RELO_CALL) { 6365 6365 + if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { 6437 6366 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", 6438 6367 prog->name, insn_idx, relo->type); 6439 6368 return -LIBBPF_ERRNO__RELOC; ··· 6445 6374 * call always has imm = -1, but for static functions 6446 6375 * relocation is against STT_SECTION and insn->imm 6447 6376 * points to a start of a static function 6377 6377 + * 6378 6378 + * for subprog addr relocation, the relo->sym_off + insn->imm is 6379 6379 + * the byte offset in the corresponding section. 6448 6380 */ 6449 6449 - sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; 6381 6381 + if (relo->type == RELO_CALL) 6382 6382 + sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; 6383 6383 + else 6384 6384 + sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; 6385 6385 + } else if (insn_is_pseudo_func(insn)) { 6386 6386 + /* 6387 6387 + * RELO_SUBPROG_ADDR relo is always emitted even if both 6388 6388 + * functions are in the same section, so it shouldn't reach here. 6389 6389 + */ 6390 6390 + pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n", 6391 6391 + prog->name, insn_idx); 6392 6392 + return -LIBBPF_ERRNO__RELOC; 6450 6393 } else { 6451 6394 /* if subprogram call is to a static function within 6452 6395 * the same ELF section, there won't be any relocation

+5

tools/lib/bpf/libbpf.map

reviewed

··· 350 350 xsk_setup_xdp_prog; 351 351 xsk_socket__update_xskmap; 352 352 } LIBBPF_0.2.0; 353 353 + 354 354 + LIBBPF_0.4.0 { 355 355 + global: 356 356 + btf__add_float; 357 357 + } LIBBPF_0.3.0;

+2

tools/lib/bpf/libbpf_internal.h

reviewed

··· 31 31 #define BTF_MEMBER_ENC(name, type, bits_offset) (name), (type), (bits_offset) 32 32 #define BTF_PARAM_ENC(name, type) (name), (type) 33 33 #define BTF_VAR_SECINFO_ENC(type, offset, size) (type), (offset), (size) 34 34 + #define BTF_TYPE_FLOAT_ENC(name, sz) \ 35 35 + BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz) 34 36 35 37 #ifndef likely 36 38 #define likely(x) __builtin_expect(!!(x), 1)

+50 -22

tools/lib/bpf/libbpf_util.h

reviewed

··· 5 5 #define __LIBBPF_LIBBPF_UTIL_H 6 6 7 7 #include <stdbool.h> 8 8 + #include <linux/compiler.h> 8 9 9 10 #ifdef __cplusplus 10 11 extern "C" { ··· 16 15 * application that uses libbpf. 17 16 */ 18 17 #if defined(__i386__) || defined(__x86_64__) 19 19 - # define libbpf_smp_rmb() asm volatile("" : : : "memory") 20 20 - # define libbpf_smp_wmb() asm volatile("" : : : "memory") 21 21 - # define libbpf_smp_mb() \ 22 22 - asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc") 23 23 - /* Hinders stores to be observed before older loads. */ 24 24 - # define libbpf_smp_rwmb() asm volatile("" : : : "memory") 18 18 + # define libbpf_smp_store_release(p, v) \ 19 19 + do { \ 20 20 + asm volatile("" : : : "memory"); \ 21 21 + WRITE_ONCE(*p, v); \ 22 22 + } while (0) 23 23 + # define libbpf_smp_load_acquire(p) \ 24 24 + ({ \ 25 25 + typeof(*p) ___p1 = READ_ONCE(*p); \ 26 26 + asm volatile("" : : : "memory"); \ 27 27 + ___p1; \ 28 28 + }) 25 29 #elif defined(__aarch64__) 26 26 - # define libbpf_smp_rmb() asm volatile("dmb ishld" : : : "memory") 27 27 - # define libbpf_smp_wmb() asm volatile("dmb ishst" : : : "memory") 28 28 - # define libbpf_smp_mb() asm volatile("dmb ish" : : : "memory") 29 29 - # define libbpf_smp_rwmb() libbpf_smp_mb() 30 30 - #elif defined(__arm__) 31 31 - /* These are only valid for armv7 and above */ 32 32 - # define libbpf_smp_rmb() asm volatile("dmb ish" : : : "memory") 33 33 - # define libbpf_smp_wmb() asm volatile("dmb ishst" : : : "memory") 34 34 - # define libbpf_smp_mb() asm volatile("dmb ish" : : : "memory") 35 35 - # define libbpf_smp_rwmb() libbpf_smp_mb() 36 36 - #else 37 37 - /* Architecture missing native barrier functions. */ 38 38 - # define libbpf_smp_rmb() __sync_synchronize() 39 39 - # define libbpf_smp_wmb() __sync_synchronize() 40 40 - # define libbpf_smp_mb() __sync_synchronize() 41 41 - # define libbpf_smp_rwmb() __sync_synchronize() 30 30 + # define libbpf_smp_store_release(p, v) \ 31 31 + asm volatile ("stlr %w1, %0" : "=Q" (*p) : "r" (v) : "memory") 32 32 + # define libbpf_smp_load_acquire(p) \ 33 33 + ({ \ 34 34 + typeof(*p) ___p1; \ 35 35 + asm volatile ("ldar %w0, %1" \ 36 36 + : "=r" (___p1) : "Q" (*p) : "memory"); \ 37 37 + ___p1; \ 38 38 + }) 39 39 + #elif defined(__riscv) 40 40 + # define libbpf_smp_store_release(p, v) \ 41 41 + do { \ 42 42 + asm volatile ("fence rw,w" : : : "memory"); \ 43 43 + WRITE_ONCE(*p, v); \ 44 44 + } while (0) 45 45 + # define libbpf_smp_load_acquire(p) \ 46 46 + ({ \ 47 47 + typeof(*p) ___p1 = READ_ONCE(*p); \ 48 48 + asm volatile ("fence r,rw" : : : "memory"); \ 49 49 + ___p1; \ 50 50 + }) 51 51 + #endif 52 52 + 53 53 + #ifndef libbpf_smp_store_release 54 54 + #define libbpf_smp_store_release(p, v) \ 55 55 + do { \ 56 56 + __sync_synchronize(); \ 57 57 + WRITE_ONCE(*p, v); \ 58 58 + } while (0) 59 59 + #endif 60 60 + 61 61 + #ifndef libbpf_smp_load_acquire 62 62 + #define libbpf_smp_load_acquire(p) \ 63 63 + ({ \ 64 64 + typeof(*p) ___p1 = READ_ONCE(*p); \ 65 65 + __sync_synchronize(); \ 66 66 + ___p1; \ 67 67 + }) 42 68 #endif 43 69 44 70 #ifdef __cplusplus

+5 -12

tools/lib/bpf/xsk.h

reviewed

··· 96 96 * this function. Without this optimization it whould have been 97 97 * free_entries = r->cached_prod - r->cached_cons + r->size. 98 98 */ 99 99 - r->cached_cons = *r->consumer + r->size; 99 99 + r->cached_cons = libbpf_smp_load_acquire(r->consumer); 100 100 + r->cached_cons += r->size; 100 101 101 102 return r->cached_cons - r->cached_prod; 102 103 } ··· 107 106 __u32 entries = r->cached_prod - r->cached_cons; 108 107 109 108 if (entries == 0) { 110 110 - r->cached_prod = *r->producer; 109 109 + r->cached_prod = libbpf_smp_load_acquire(r->producer); 111 110 entries = r->cached_prod - r->cached_cons; 112 111 } 113 112 ··· 130 129 /* Make sure everything has been written to the ring before indicating 131 130 * this to the kernel by writing the producer pointer. 132 131 */ 133 133 - libbpf_smp_wmb(); 134 134 - 135 135 - *prod->producer += nb; 132 132 + libbpf_smp_store_release(prod->producer, *prod->producer + nb); 136 133 } 137 134 138 135 static inline __u32 xsk_ring_cons__peek(struct xsk_ring_cons *cons, __u32 nb, __u32 *idx) ··· 138 139 __u32 entries = xsk_cons_nb_avail(cons, nb); 139 140 140 141 if (entries > 0) { 141 141 - /* Make sure we do not speculatively read the data before 142 142 - * we have received the packet buffers from the ring. 143 143 - */ 144 144 - libbpf_smp_rmb(); 145 145 - 146 142 *idx = cons->cached_cons; 147 143 cons->cached_cons += entries; 148 144 } ··· 155 161 /* Make sure data has been read before indicating we are done 156 162 * with the entries by updating the consumer pointer. 157 163 */ 158 158 - libbpf_smp_rwmb(); 164 164 + libbpf_smp_store_release(cons->consumer, *cons->consumer + nb); 159 165 160 160 - *cons->consumer += nb; 161 166 } 162 167 163 168 static inline void *xsk_umem__get_data(void *umem_area, __u64 addr)

+1 -1

tools/perf/MANIFEST

reviewed

··· 20 20 tools/lib/str_error_r.c 21 21 tools/lib/vsprintf.c 22 22 tools/lib/zalloc.c 23 23 - scripts/bpf_helpers_doc.py 23 23 + scripts/bpf_doc.py

+2

tools/testing/selftests/bpf/.gitignore

reviewed

··· 1 1 # SPDX-License-Identifier: GPL-2.0-only 2 2 + bpf-helpers* 3 3 + bpf-syscall* 2 4 test_verifier 3 5 test_maps 4 6 test_lru_map

+19 -8

tools/testing/selftests/bpf/Makefile

reviewed

··· 68 68 test_bpftool_build.sh \ 69 69 test_bpftool.sh \ 70 70 test_bpftool_metadata.sh \ 71 71 + test_doc_build.sh \ 71 72 test_xsk.sh 72 73 73 74 TEST_PROGS_EXTENDED := with_addr.sh \ ··· 104 103 $(call msg,CLEAN) 105 104 $(Q)$(RM) -r $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED) $(TEST_GEN_FILES) $(EXTRA_CLEAN) 106 105 $(Q)$(MAKE) -C bpf_testmod clean 106 106 + $(Q)$(MAKE) docs-clean 107 107 endef 108 108 109 109 include ../lib.mk ··· 182 180 cp $(SCRATCH_DIR)/runqslower $@ 183 181 184 182 $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(OUTPUT)/test_stub.o $(BPFOBJ) 183 183 + $(TEST_GEN_FILES): docs 185 184 186 185 $(OUTPUT)/test_dev_cgroup: cgroup_helpers.c 187 186 $(OUTPUT)/test_skb_cgroup_id_user: cgroup_helpers.c ··· 203 200 CC=$(HOSTCC) LD=$(HOSTLD) \ 204 201 OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \ 205 202 prefix= DESTDIR=$(HOST_SCRATCH_DIR)/ install 206 206 - $(Q)mkdir -p $(BUILD_DIR)/bpftool/Documentation 207 207 - $(Q)RST2MAN_OPTS="--exit-status=1" $(MAKE) $(submake_extras) \ 208 208 - -C $(BPFTOOLDIR)/Documentation \ 209 209 - OUTPUT=$(BUILD_DIR)/bpftool/Documentation/ \ 210 210 - prefix= DESTDIR=$(SCRATCH_DIR)/ install 203 203 + 204 204 + docs: 205 205 + $(Q)RST2MAN_OPTS="--exit-status=1" $(MAKE) $(submake_extras) \ 206 206 + -f Makefile.docs \ 207 207 + prefix= OUTPUT=$(OUTPUT)/ DESTDIR=$(OUTPUT)/ $@ 208 208 + 209 209 + docs-clean: 210 210 + $(Q)$(MAKE) $(submake_extras) \ 211 211 + -f Makefile.docs \ 212 212 + prefix= OUTPUT=$(OUTPUT)/ DESTDIR=$(OUTPUT)/ $@ 211 213 212 214 $(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ 213 215 ../../../include/uapi/linux/bpf.h \ ··· 390 382 $$(call msg,EXT-OBJ,$(TRUNNER_BINARY),$$@) 391 383 $(Q)$$(CC) $$(CFLAGS) -c $$< $$(LDLIBS) -o $$@ 392 384 393 393 - # only copy extra resources if in flavored build 385 385 + # non-flavored in-srctree builds receive special treatment, in particular, we 386 386 + # do not need to copy extra resources (see e.g. test_btf_dump_case()) 394 387 $(TRUNNER_BINARY)-extras: $(TRUNNER_EXTRA_FILES) | $(TRUNNER_OUTPUT) 395 395 - ifneq ($2,) 388 388 + ifneq ($2:$(OUTPUT),:$(shell pwd)) 396 389 $$(call msg,EXT-COPY,$(TRUNNER_BINARY),$(TRUNNER_EXTRA_FILES)) 397 397 - $(Q)cp -a $$^ $(TRUNNER_OUTPUT)/ 390 390 + $(Q)rsync -aq $$^ $(TRUNNER_OUTPUT)/ 398 391 endif 399 392 400 393 $(OUTPUT)/$(TRUNNER_BINARY): $(TRUNNER_TEST_OBJS) \ ··· 485 476 prog_tests/tests.h map_tests/tests.h verifier/tests.h \ 486 477 feature \ 487 478 $(addprefix $(OUTPUT)/,*.o *.skel.h no_alu32 bpf_gcc bpf_testmod.ko) 479 479 + 480 480 + .PHONY: docs docs-clean

+82

tools/testing/selftests/bpf/Makefile.docs

reviewed

··· 1 1 + # SPDX-License-Identifier: GPL-2.0-only 2 2 + 3 3 + include ../../../scripts/Makefile.include 4 4 + include ../../../scripts/utilities.mak 5 5 + 6 6 + INSTALL ?= install 7 7 + RM ?= rm -f 8 8 + RMDIR ?= rmdir --ignore-fail-on-non-empty 9 9 + 10 10 + ifeq ($(V),1) 11 11 + Q = 12 12 + else 13 13 + Q = @ 14 14 + endif 15 15 + 16 16 + prefix ?= /usr/local 17 17 + mandir ?= $(prefix)/man 18 18 + man2dir = $(mandir)/man2 19 19 + man7dir = $(mandir)/man7 20 20 + 21 21 + SYSCALL_RST = bpf-syscall.rst 22 22 + MAN2_RST = $(SYSCALL_RST) 23 23 + 24 24 + HELPERS_RST = bpf-helpers.rst 25 25 + MAN7_RST = $(HELPERS_RST) 26 26 + 27 27 + _DOC_MAN2 = $(patsubst %.rst,%.2,$(MAN2_RST)) 28 28 + DOC_MAN2 = $(addprefix $(OUTPUT),$(_DOC_MAN2)) 29 29 + 30 30 + _DOC_MAN7 = $(patsubst %.rst,%.7,$(MAN7_RST)) 31 31 + DOC_MAN7 = $(addprefix $(OUTPUT),$(_DOC_MAN7)) 32 32 + 33 33 + DOCTARGETS := helpers syscall 34 34 + 35 35 + docs: $(DOCTARGETS) 36 36 + syscall: man2 37 37 + helpers: man7 38 38 + man2: $(DOC_MAN2) 39 39 + man7: $(DOC_MAN7) 40 40 + 41 41 + RST2MAN_DEP := $(shell command -v rst2man 2>/dev/null) 42 42 + 43 43 + # Configure make rules for the man page bpf-$1.$2. 44 44 + # $1 - target for scripts/bpf_doc.py 45 45 + # $2 - man page section to generate the troff file 46 46 + define DOCS_RULES = 47 47 + $(OUTPUT)bpf-$1.rst: ../../../../include/uapi/linux/bpf.h 48 48 + $$(QUIET_GEN)../../../../scripts/bpf_doc.py $1 \ 49 49 + --filename $$< > $$@ 50 50 + 51 51 + $(OUTPUT)%.$2: $(OUTPUT)%.rst 52 52 + ifndef RST2MAN_DEP 53 53 + $$(error "rst2man not found, but required to generate man pages") 54 54 + endif 55 55 + $$(QUIET_GEN)rst2man $$< > $$@ 56 56 + 57 57 + docs-clean-$1: 58 58 + $$(call QUIET_CLEAN, eBPF_$1-manpage) 59 59 + $(Q)$(RM) $$(DOC_MAN$2) $(OUTPUT)bpf-$1.rst 60 60 + 61 61 + docs-install-$1: docs 62 62 + $$(call QUIET_INSTALL, eBPF_$1-manpage) 63 63 + $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$$(man$2dir) 64 64 + $(Q)$(INSTALL) -m 644 $$(DOC_MAN$2) $(DESTDIR)$$(man$2dir) 65 65 + 66 66 + docs-uninstall-$1: 67 67 + $$(call QUIET_UNINST, eBPF_$1-manpage) 68 68 + $(Q)$(RM) $$(addprefix $(DESTDIR)$$(man$2dir)/,$$(_DOC_MAN$2)) 69 69 + $(Q)$(RMDIR) $(DESTDIR)$$(man$2dir) 70 70 + 71 71 + .PHONY: $1 docs-clean-$1 docs-install-$1 docs-uninstall-$1 72 72 + endef 73 73 + 74 74 + # Create the make targets to generate manual pages by name and section 75 75 + $(eval $(call DOCS_RULES,helpers,7)) 76 76 + $(eval $(call DOCS_RULES,syscall,2)) 77 77 + 78 78 + docs-clean: $(foreach doctarget,$(DOCTARGETS), docs-clean-$(doctarget)) 79 79 + docs-install: $(foreach doctarget,$(DOCTARGETS), docs-install-$(doctarget)) 80 80 + docs-uninstall: $(foreach doctarget,$(DOCTARGETS), docs-uninstall-$(doctarget)) 81 81 + 82 82 + .PHONY: docs docs-clean docs-install docs-uninstall man2 man7

+48

tools/testing/selftests/bpf/README.rst

reviewed

··· 111 111 112 112 __ https://reviews.llvm.org/D78466 113 113 114 114 + bpf_verif_scale/loop6.o test failure with Clang 12 115 115 + ================================================== 116 116 + 117 117 + With Clang 12, the following bpf_verif_scale test failed: 118 118 + * ``bpf_verif_scale/loop6.o`` 119 119 + 120 120 + The verifier output looks like 121 121 + 122 122 + .. code-block:: c 123 123 + 124 124 + R1 type=ctx expected=fp 125 125 + The sequence of 8193 jumps is too complex. 126 126 + 127 127 + The reason is compiler generating the following code 128 128 + 129 129 + .. code-block:: c 130 130 + 131 131 + ; for (i = 0; (i < VIRTIO_MAX_SGS) && (i < num); i++) { 132 132 + 14: 16 05 40 00 00 00 00 00 if w5 == 0 goto +64 <LBB0_6> 133 133 + 15: bc 51 00 00 00 00 00 00 w1 = w5 134 134 + 16: 04 01 00 00 ff ff ff ff w1 += -1 135 135 + 17: 67 05 00 00 20 00 00 00 r5 <<= 32 136 136 + 18: 77 05 00 00 20 00 00 00 r5 >>= 32 137 137 + 19: a6 01 01 00 05 00 00 00 if w1 < 5 goto +1 <LBB0_4> 138 138 + 20: b7 05 00 00 06 00 00 00 r5 = 6 139 139 + 00000000000000a8 <LBB0_4>: 140 140 + 21: b7 02 00 00 00 00 00 00 r2 = 0 141 141 + 22: b7 01 00 00 00 00 00 00 r1 = 0 142 142 + ; for (i = 0; (i < VIRTIO_MAX_SGS) && (i < num); i++) { 143 143 + 23: 7b 1a e0 ff 00 00 00 00 *(u64 *)(r10 - 32) = r1 144 144 + 24: 7b 5a c0 ff 00 00 00 00 *(u64 *)(r10 - 64) = r5 145 145 + 146 146 + Note that insn #15 has w1 = w5 and w1 is refined later but 147 147 + r5(w5) is eventually saved on stack at insn #24 for later use. 148 148 + This cause later verifier failure. The bug has been `fixed`__ in 149 149 + Clang 13. 150 150 + 151 151 + __ https://reviews.llvm.org/D97479 152 152 + 114 153 BPF CO-RE-based tests and Clang version 115 154 ======================================= 116 155 ··· 170 131 .. _2: https://reviews.llvm.org/D85174 171 132 .. _3: https://reviews.llvm.org/D83878 172 133 .. _4: https://reviews.llvm.org/D83242 134 134 + 135 135 + Floating-point tests and Clang version 136 136 + ====================================== 137 137 + 138 138 + Certain selftests, e.g. core_reloc, require support for the floating-point 139 139 + types, which was introduced in `Clang 13`__. The older Clang versions will 140 140 + either crash when compiling these tests, or generate an incorrect BTF. 141 141 + 142 142 + __ https://reviews.llvm.org/D83289

+4

tools/testing/selftests/bpf/btf_helpers.c

reviewed

··· 23 23 [BTF_KIND_FUNC_PROTO] = "FUNC_PROTO", 24 24 [BTF_KIND_VAR] = "VAR", 25 25 [BTF_KIND_DATASEC] = "DATASEC", 26 26 + [BTF_KIND_FLOAT] = "FLOAT", 26 27 }; 27 28 28 29 static const char *btf_kind_str(__u16 kind) ··· 174 173 } 175 174 break; 176 175 } 176 176 + case BTF_KIND_FLOAT: 177 177 + fprintf(out, " size=%u", t->size); 178 178 + break; 177 179 default: 178 180 break; 179 181 }

+39 -1

tools/testing/selftests/bpf/prog_tests/attach_probe.c

reviewed

··· 2 2 #include <test_progs.h> 3 3 #include "test_attach_probe.skel.h" 4 4 5 5 + #if defined(__powerpc64__) && defined(_CALL_ELF) && _CALL_ELF == 2 6 6 + 7 7 + #define OP_RT_RA_MASK 0xffff0000UL 8 8 + #define LIS_R2 0x3c400000UL 9 9 + #define ADDIS_R2_R12 0x3c4c0000UL 10 10 + #define ADDI_R2_R2 0x38420000UL 11 11 + 12 12 + static ssize_t get_offset(ssize_t addr, ssize_t base) 13 13 + { 14 14 + u32 *insn = (u32 *) addr; 15 15 + 16 16 + /* 17 17 + * A PPC64 ABIv2 function may have a local and a global entry 18 18 + * point. We need to use the local entry point when patching 19 19 + * functions, so identify and step over the global entry point 20 20 + * sequence. 21 21 + * 22 22 + * The global entry point sequence is always of the form: 23 23 + * 24 24 + * addis r2,r12,XXXX 25 25 + * addi r2,r2,XXXX 26 26 + * 27 27 + * A linker optimisation may convert the addis to lis: 28 28 + * 29 29 + * lis r2,XXXX 30 30 + * addi r2,r2,XXXX 31 31 + */ 32 32 + if ((((*insn & OP_RT_RA_MASK) == ADDIS_R2_R12) || 33 33 + ((*insn & OP_RT_RA_MASK) == LIS_R2)) && 34 34 + ((*(insn + 1) & OP_RT_RA_MASK) == ADDI_R2_R2)) 35 35 + return (ssize_t)(insn + 2) - base; 36 36 + else 37 37 + return addr - base; 38 38 + } 39 39 + #else 40 40 + #define get_offset(addr, base) (addr - base) 41 41 + #endif 42 42 + 5 43 ssize_t get_base_addr() { 6 44 size_t start, offset; 7 45 char buf[256]; ··· 74 36 if (CHECK(base_addr < 0, "get_base_addr", 75 37 "failed to find base addr: %zd", base_addr)) 76 38 return; 77 77 - uprobe_offset = (size_t)&get_base_addr - base_addr; 39 39 + uprobe_offset = get_offset((size_t)&get_base_addr, base_addr); 78 40 79 41 skel = test_attach_probe__open_and_load(); 80 42 if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))

+1

tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c

reviewed

··· 76 76 { "loop2.o", BPF_PROG_TYPE_RAW_TRACEPOINT }, 77 77 { "loop4.o", BPF_PROG_TYPE_SCHED_CLS }, 78 78 { "loop5.o", BPF_PROG_TYPE_SCHED_CLS }, 79 79 + { "loop6.o", BPF_PROG_TYPE_KPROBE }, 79 80 80 81 /* partial unroll. 19k insn in a loop. 81 82 * Total program size 20.8k insn.

+163 -13

tools/testing/selftests/bpf/prog_tests/btf.c

reviewed

··· 1903 1903 .raw_types = { 1904 1904 /* int */ /* [1] */ 1905 1905 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), 1906 1906 - BTF_TYPE_ENC(0, 0x10000000, 4), 1906 1906 + BTF_TYPE_ENC(0, 0x20000000, 4), 1907 1907 BTF_END_RAW, 1908 1908 }, 1909 1909 .str_sec = "", ··· 3529 3529 .key_type_id = 1, 3530 3530 .value_type_id = 1, 3531 3531 .max_entries = 1, 3532 3532 + }, 3533 3533 + 3534 3534 + { 3535 3535 + .descr = "float test #1, well-formed", 3536 3536 + .raw_types = { 3537 3537 + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), 3538 3538 + /* [1] */ 3539 3539 + BTF_TYPE_FLOAT_ENC(NAME_TBD, 2), /* [2] */ 3540 3540 + BTF_TYPE_FLOAT_ENC(NAME_TBD, 4), /* [3] */ 3541 3541 + BTF_TYPE_FLOAT_ENC(NAME_TBD, 8), /* [4] */ 3542 3542 + BTF_TYPE_FLOAT_ENC(NAME_TBD, 12), /* [5] */ 3543 3543 + BTF_TYPE_FLOAT_ENC(NAME_TBD, 16), /* [6] */ 3544 3544 + BTF_STRUCT_ENC(NAME_TBD, 5, 48), /* [7] */ 3545 3545 + BTF_MEMBER_ENC(NAME_TBD, 2, 0), 3546 3546 + BTF_MEMBER_ENC(NAME_TBD, 3, 32), 3547 3547 + BTF_MEMBER_ENC(NAME_TBD, 4, 64), 3548 3548 + BTF_MEMBER_ENC(NAME_TBD, 5, 128), 3549 3549 + BTF_MEMBER_ENC(NAME_TBD, 6, 256), 3550 3550 + BTF_END_RAW, 3551 3551 + }, 3552 3552 + BTF_STR_SEC("\0int\0_Float16\0float\0double\0_Float80\0long_double" 3553 3553 + "\0floats\0a\0b\0c\0d\0e"), 3554 3554 + .map_type = BPF_MAP_TYPE_ARRAY, 3555 3555 + .map_name = "float_type_check_btf", 3556 3556 + .key_size = sizeof(int), 3557 3557 + .value_size = 48, 3558 3558 + .key_type_id = 1, 3559 3559 + .value_type_id = 7, 3560 3560 + .max_entries = 1, 3561 3561 + }, 3562 3562 + { 3563 3563 + .descr = "float test #2, invalid vlen", 3564 3564 + .raw_types = { 3565 3565 + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), 3566 3566 + /* [1] */ 3567 3567 + BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 1), 4), 3568 3568 + /* [2] */ 3569 3569 + BTF_END_RAW, 3570 3570 + }, 3571 3571 + BTF_STR_SEC("\0int\0float"), 3572 3572 + .map_type = BPF_MAP_TYPE_ARRAY, 3573 3573 + .map_name = "float_type_check_btf", 3574 3574 + .key_size = sizeof(int), 3575 3575 + .value_size = 4, 3576 3576 + .key_type_id = 1, 3577 3577 + .value_type_id = 2, 3578 3578 + .max_entries = 1, 3579 3579 + .btf_load_err = true, 3580 3580 + .err_str = "vlen != 0", 3581 3581 + }, 3582 3582 + { 3583 3583 + .descr = "float test #3, invalid kind_flag", 3584 3584 + .raw_types = { 3585 3585 + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), 3586 3586 + /* [1] */ 3587 3587 + BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_FLOAT, 1, 0), 4), 3588 3588 + /* [2] */ 3589 3589 + BTF_END_RAW, 3590 3590 + }, 3591 3591 + BTF_STR_SEC("\0int\0float"), 3592 3592 + .map_type = BPF_MAP_TYPE_ARRAY, 3593 3593 + .map_name = "float_type_check_btf", 3594 3594 + .key_size = sizeof(int), 3595 3595 + .value_size = 4, 3596 3596 + .key_type_id = 1, 3597 3597 + .value_type_id = 2, 3598 3598 + .max_entries = 1, 3599 3599 + .btf_load_err = true, 3600 3600 + .err_str = "Invalid btf_info kind_flag", 3601 3601 + }, 3602 3602 + { 3603 3603 + .descr = "float test #4, member does not fit", 3604 3604 + .raw_types = { 3605 3605 + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), 3606 3606 + /* [1] */ 3607 3607 + BTF_TYPE_FLOAT_ENC(NAME_TBD, 4), /* [2] */ 3608 3608 + BTF_STRUCT_ENC(NAME_TBD, 1, 2), /* [3] */ 3609 3609 + BTF_MEMBER_ENC(NAME_TBD, 2, 0), 3610 3610 + BTF_END_RAW, 3611 3611 + }, 3612 3612 + BTF_STR_SEC("\0int\0float\0floats\0x"), 3613 3613 + .map_type = BPF_MAP_TYPE_ARRAY, 3614 3614 + .map_name = "float_type_check_btf", 3615 3615 + .key_size = sizeof(int), 3616 3616 + .value_size = 4, 3617 3617 + .key_type_id = 1, 3618 3618 + .value_type_id = 3, 3619 3619 + .max_entries = 1, 3620 3620 + .btf_load_err = true, 3621 3621 + .err_str = "Member exceeds struct_size", 3622 3622 + }, 3623 3623 + { 3624 3624 + .descr = "float test #5, member is not properly aligned", 3625 3625 + .raw_types = { 3626 3626 + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), 3627 3627 + /* [1] */ 3628 3628 + BTF_TYPE_FLOAT_ENC(NAME_TBD, 4), /* [2] */ 3629 3629 + BTF_STRUCT_ENC(NAME_TBD, 1, 8), /* [3] */ 3630 3630 + BTF_MEMBER_ENC(NAME_TBD, 2, 8), 3631 3631 + BTF_END_RAW, 3632 3632 + }, 3633 3633 + BTF_STR_SEC("\0int\0float\0floats\0x"), 3634 3634 + .map_type = BPF_MAP_TYPE_ARRAY, 3635 3635 + .map_name = "float_type_check_btf", 3636 3636 + .key_size = sizeof(int), 3637 3637 + .value_size = 4, 3638 3638 + .key_type_id = 1, 3639 3639 + .value_type_id = 3, 3640 3640 + .max_entries = 1, 3641 3641 + .btf_load_err = true, 3642 3642 + .err_str = "Member is not properly aligned", 3643 3643 + }, 3644 3644 + { 3645 3645 + .descr = "float test #6, invalid size", 3646 3646 + .raw_types = { 3647 3647 + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), 3648 3648 + /* [1] */ 3649 3649 + BTF_TYPE_FLOAT_ENC(NAME_TBD, 6), /* [2] */ 3650 3650 + BTF_END_RAW, 3651 3651 + }, 3652 3652 + BTF_STR_SEC("\0int\0float"), 3653 3653 + .map_type = BPF_MAP_TYPE_ARRAY, 3654 3654 + .map_name = "float_type_check_btf", 3655 3655 + .key_size = sizeof(int), 3656 3656 + .value_size = 6, 3657 3657 + .key_type_id = 1, 3658 3658 + .value_type_id = 2, 3659 3659 + .max_entries = 1, 3660 3660 + .btf_load_err = true, 3661 3661 + .err_str = "Invalid type_size", 3532 3662 }, 3533 3663 3534 3664 }; /* struct btf_raw_test raw_tests[] */ ··· 6411 6281 /* int[16] */ 6412 6282 BTF_TYPE_ARRAY_ENC(1, 1, 16), /* [2] */ 6413 6283 /* struct s { */ 6414 6414 - BTF_STRUCT_ENC(NAME_NTH(2), 4, 84), /* [3] */ 6284 6284 + BTF_STRUCT_ENC(NAME_NTH(2), 5, 88), /* [3] */ 6415 6285 BTF_MEMBER_ENC(NAME_NTH(3), 4, 0), /* struct s *next; */ 6416 6286 BTF_MEMBER_ENC(NAME_NTH(4), 5, 64), /* const int *a; */ 6417 6287 BTF_MEMBER_ENC(NAME_NTH(5), 2, 128), /* int b[16]; */ 6418 6288 BTF_MEMBER_ENC(NAME_NTH(6), 1, 640), /* int c; */ 6289 6289 + BTF_MEMBER_ENC(NAME_NTH(8), 13, 672), /* float d; */ 6419 6290 /* ptr -> [3] struct s */ 6420 6291 BTF_PTR_ENC(3), /* [4] */ 6421 6292 /* ptr -> [6] const int */ ··· 6427 6296 /* full copy of the above */ 6428 6297 BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [7] */ 6429 6298 BTF_TYPE_ARRAY_ENC(7, 7, 16), /* [8] */ 6430 6430 - BTF_STRUCT_ENC(NAME_NTH(2), 4, 84), /* [9] */ 6299 6299 + BTF_STRUCT_ENC(NAME_NTH(2), 5, 88), /* [9] */ 6431 6300 BTF_MEMBER_ENC(NAME_NTH(3), 10, 0), 6432 6301 BTF_MEMBER_ENC(NAME_NTH(4), 11, 64), 6433 6302 BTF_MEMBER_ENC(NAME_NTH(5), 8, 128), 6434 6303 BTF_MEMBER_ENC(NAME_NTH(6), 7, 640), 6304 6304 + BTF_MEMBER_ENC(NAME_NTH(8), 13, 672), 6435 6305 BTF_PTR_ENC(9), /* [10] */ 6436 6306 BTF_PTR_ENC(12), /* [11] */ 6437 6307 BTF_CONST_ENC(7), /* [12] */ 6308 6308 + BTF_TYPE_FLOAT_ENC(NAME_NTH(7), 4), /* [13] */ 6438 6309 BTF_END_RAW, 6439 6310 }, 6440 6440 - BTF_STR_SEC("\0int\0s\0next\0a\0b\0c\0"), 6311 6311 + BTF_STR_SEC("\0int\0s\0next\0a\0b\0c\0float\0d"), 6441 6312 }, 6442 6313 .expect = { 6443 6314 .raw_types = { 6444 6315 /* int */ 6445 6445 - BTF_TYPE_INT_ENC(NAME_NTH(4), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 6316 6316 + BTF_TYPE_INT_ENC(NAME_NTH(5), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 6446 6317 /* int[16] */ 6447 6318 BTF_TYPE_ARRAY_ENC(1, 1, 16), /* [2] */ 6448 6319 /* struct s { */ 6449 6449 - BTF_STRUCT_ENC(NAME_NTH(6), 4, 84), /* [3] */ 6450 6450 - BTF_MEMBER_ENC(NAME_NTH(5), 4, 0), /* struct s *next; */ 6320 6320 + BTF_STRUCT_ENC(NAME_NTH(8), 5, 88), /* [3] */ 6321 6321 + BTF_MEMBER_ENC(NAME_NTH(7), 4, 0), /* struct s *next; */ 6451 6322 BTF_MEMBER_ENC(NAME_NTH(1), 5, 64), /* const int *a; */ 6452 6323 BTF_MEMBER_ENC(NAME_NTH(2), 2, 128), /* int b[16]; */ 6453 6324 BTF_MEMBER_ENC(NAME_NTH(3), 1, 640), /* int c; */ 6325 6325 + BTF_MEMBER_ENC(NAME_NTH(4), 7, 672), /* float d; */ 6454 6326 /* ptr -> [3] struct s */ 6455 6327 BTF_PTR_ENC(3), /* [4] */ 6456 6328 /* ptr -> [6] const int */ 6457 6329 BTF_PTR_ENC(6), /* [5] */ 6458 6330 /* const -> [1] int */ 6459 6331 BTF_CONST_ENC(1), /* [6] */ 6332 6332 + BTF_TYPE_FLOAT_ENC(NAME_NTH(7), 4), /* [7] */ 6460 6333 BTF_END_RAW, 6461 6334 }, 6462 6462 - BTF_STR_SEC("\0a\0b\0c\0int\0next\0s"), 6335 6335 + BTF_STR_SEC("\0a\0b\0c\0d\0int\0float\0next\0s"), 6463 6336 }, 6464 6337 .opts = { 6465 6338 .dont_resolve_fwds = false, ··· 6584 6449 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), 6585 6450 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 8), 6586 6451 BTF_FUNC_ENC(NAME_TBD, 12), /* [13] func */ 6452 6452 + BTF_TYPE_FLOAT_ENC(NAME_TBD, 2), /* [14] float */ 6587 6453 BTF_END_RAW, 6588 6454 }, 6589 6589 - BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M"), 6455 6455 + BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N"), 6590 6456 }, 6591 6457 .expect = { 6592 6458 .raw_types = { ··· 6610 6474 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), 6611 6475 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 8), 6612 6476 BTF_FUNC_ENC(NAME_TBD, 12), /* [13] func */ 6477 6477 + BTF_TYPE_FLOAT_ENC(NAME_TBD, 2), /* [14] float */ 6613 6478 BTF_END_RAW, 6614 6479 }, 6615 6615 - BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M"), 6480 6480 + BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N"), 6616 6481 }, 6617 6482 .opts = { 6618 6483 .dont_resolve_fwds = false, 6619 6484 }, 6620 6485 }, 6621 6486 { 6622 6622 - .descr = "dedup: no int duplicates", 6487 6487 + .descr = "dedup: no int/float duplicates", 6623 6488 .input = { 6624 6489 .raw_types = { 6625 6490 BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 8), ··· 6635 6498 BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 27, 8), 6636 6499 /* different byte size */ 6637 6500 BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), 6501 6501 + /* all allowed sizes */ 6502 6502 + BTF_TYPE_FLOAT_ENC(NAME_NTH(3), 2), 6503 6503 + BTF_TYPE_FLOAT_ENC(NAME_NTH(3), 4), 6504 6504 + BTF_TYPE_FLOAT_ENC(NAME_NTH(3), 8), 6505 6505 + BTF_TYPE_FLOAT_ENC(NAME_NTH(3), 12), 6506 6506 + BTF_TYPE_FLOAT_ENC(NAME_NTH(3), 16), 6638 6507 BTF_END_RAW, 6639 6508 }, 6640 6640 - BTF_STR_SEC("\0int\0some other int"), 6509 6509 + BTF_STR_SEC("\0int\0some other int\0float"), 6641 6510 }, 6642 6511 .expect = { 6643 6512 .raw_types = { ··· 6659 6516 BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 27, 8), 6660 6517 /* different byte size */ 6661 6518 BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), 6519 6519 + /* all allowed sizes */ 6520 6520 + BTF_TYPE_FLOAT_ENC(NAME_NTH(3), 2), 6521 6521 + BTF_TYPE_FLOAT_ENC(NAME_NTH(3), 4), 6522 6522 + BTF_TYPE_FLOAT_ENC(NAME_NTH(3), 8), 6523 6523 + BTF_TYPE_FLOAT_ENC(NAME_NTH(3), 12), 6524 6524 + BTF_TYPE_FLOAT_ENC(NAME_NTH(3), 16), 6662 6525 BTF_END_RAW, 6663 6526 }, 6664 6664 - BTF_STR_SEC("\0int\0some other int"), 6527 6527 + BTF_STR_SEC("\0int\0some other int\0float"), 6665 6528 }, 6666 6529 .opts = { 6667 6530 .dont_resolve_fwds = false, ··· 6779 6630 case BTF_KIND_PTR: 6780 6631 case BTF_KIND_TYPEDEF: 6781 6632 case BTF_KIND_FUNC: 6633 6633 + case BTF_KIND_FLOAT: 6782 6634 return base_size; 6783 6635 case BTF_KIND_INT: 6784 6636 return base_size + sizeof(__u32);

+1

tools/testing/selftests/bpf/prog_tests/core_reloc.c

reviewed

··· 266 266 .arr_elem_sz = sizeof(((type *)0)->arr_field[0]), \ 267 267 .ptr_sz = 8, /* always 8-byte pointer for BPF */ \ 268 268 .enum_sz = sizeof(((type *)0)->enum_field), \ 269 269 + .float_sz = sizeof(((type *)0)->float_field), \ 269 270 } 270 271 271 272 #define SIZE_CASE(name) { \

+130

tools/testing/selftests/bpf/prog_tests/for_each.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2021 Facebook */ 3 3 + #include <test_progs.h> 4 4 + #include <network_helpers.h> 5 5 + #include "for_each_hash_map_elem.skel.h" 6 6 + #include "for_each_array_map_elem.skel.h" 7 7 + 8 8 + static unsigned int duration; 9 9 + 10 10 + static void test_hash_map(void) 11 11 + { 12 12 + int i, err, hashmap_fd, max_entries, percpu_map_fd; 13 13 + struct for_each_hash_map_elem *skel; 14 14 + __u64 *percpu_valbuf = NULL; 15 15 + __u32 key, num_cpus, retval; 16 16 + __u64 val; 17 17 + 18 18 + skel = for_each_hash_map_elem__open_and_load(); 19 19 + if (!ASSERT_OK_PTR(skel, "for_each_hash_map_elem__open_and_load")) 20 20 + return; 21 21 + 22 22 + hashmap_fd = bpf_map__fd(skel->maps.hashmap); 23 23 + max_entries = bpf_map__max_entries(skel->maps.hashmap); 24 24 + for (i = 0; i < max_entries; i++) { 25 25 + key = i; 26 26 + val = i + 1; 27 27 + err = bpf_map_update_elem(hashmap_fd, &key, &val, BPF_ANY); 28 28 + if (!ASSERT_OK(err, "map_update")) 29 29 + goto out; 30 30 + } 31 31 + 32 32 + num_cpus = bpf_num_possible_cpus(); 33 33 + percpu_map_fd = bpf_map__fd(skel->maps.percpu_map); 34 34 + percpu_valbuf = malloc(sizeof(__u64) * num_cpus); 35 35 + if (!ASSERT_OK_PTR(percpu_valbuf, "percpu_valbuf")) 36 36 + goto out; 37 37 + 38 38 + key = 1; 39 39 + for (i = 0; i < num_cpus; i++) 40 40 + percpu_valbuf[i] = i + 1; 41 41 + err = bpf_map_update_elem(percpu_map_fd, &key, percpu_valbuf, BPF_ANY); 42 42 + if (!ASSERT_OK(err, "percpu_map_update")) 43 43 + goto out; 44 44 + 45 45 + err = bpf_prog_test_run(bpf_program__fd(skel->progs.test_pkt_access), 46 46 + 1, &pkt_v4, sizeof(pkt_v4), NULL, NULL, 47 47 + &retval, &duration); 48 48 + if (CHECK(err || retval, "ipv4", "err %d errno %d retval %d\n", 49 49 + err, errno, retval)) 50 50 + goto out; 51 51 + 52 52 + ASSERT_EQ(skel->bss->hashmap_output, 4, "hashmap_output"); 53 53 + ASSERT_EQ(skel->bss->hashmap_elems, max_entries, "hashmap_elems"); 54 54 + 55 55 + key = 1; 56 56 + err = bpf_map_lookup_elem(hashmap_fd, &key, &val); 57 57 + ASSERT_ERR(err, "hashmap_lookup"); 58 58 + 59 59 + ASSERT_EQ(skel->bss->percpu_called, 1, "percpu_called"); 60 60 + ASSERT_LT(skel->bss->cpu, num_cpus, "num_cpus"); 61 61 + ASSERT_EQ(skel->bss->percpu_map_elems, 1, "percpu_map_elems"); 62 62 + ASSERT_EQ(skel->bss->percpu_key, 1, "percpu_key"); 63 63 + ASSERT_EQ(skel->bss->percpu_val, skel->bss->cpu + 1, "percpu_val"); 64 64 + ASSERT_EQ(skel->bss->percpu_output, 100, "percpu_output"); 65 65 + out: 66 66 + free(percpu_valbuf); 67 67 + for_each_hash_map_elem__destroy(skel); 68 68 + } 69 69 + 70 70 + static void test_array_map(void) 71 71 + { 72 72 + __u32 key, num_cpus, max_entries, retval; 73 73 + int i, arraymap_fd, percpu_map_fd, err; 74 74 + struct for_each_array_map_elem *skel; 75 75 + __u64 *percpu_valbuf = NULL; 76 76 + __u64 val, expected_total; 77 77 + 78 78 + skel = for_each_array_map_elem__open_and_load(); 79 79 + if (!ASSERT_OK_PTR(skel, "for_each_array_map_elem__open_and_load")) 80 80 + return; 81 81 + 82 82 + arraymap_fd = bpf_map__fd(skel->maps.arraymap); 83 83 + expected_total = 0; 84 84 + max_entries = bpf_map__max_entries(skel->maps.arraymap); 85 85 + for (i = 0; i < max_entries; i++) { 86 86 + key = i; 87 87 + val = i + 1; 88 88 + /* skip the last iteration for expected total */ 89 89 + if (i != max_entries - 1) 90 90 + expected_total += val; 91 91 + err = bpf_map_update_elem(arraymap_fd, &key, &val, BPF_ANY); 92 92 + if (!ASSERT_OK(err, "map_update")) 93 93 + goto out; 94 94 + } 95 95 + 96 96 + num_cpus = bpf_num_possible_cpus(); 97 97 + percpu_map_fd = bpf_map__fd(skel->maps.percpu_map); 98 98 + percpu_valbuf = malloc(sizeof(__u64) * num_cpus); 99 99 + if (!ASSERT_OK_PTR(percpu_valbuf, "percpu_valbuf")) 100 100 + goto out; 101 101 + 102 102 + key = 0; 103 103 + for (i = 0; i < num_cpus; i++) 104 104 + percpu_valbuf[i] = i + 1; 105 105 + err = bpf_map_update_elem(percpu_map_fd, &key, percpu_valbuf, BPF_ANY); 106 106 + if (!ASSERT_OK(err, "percpu_map_update")) 107 107 + goto out; 108 108 + 109 109 + err = bpf_prog_test_run(bpf_program__fd(skel->progs.test_pkt_access), 110 110 + 1, &pkt_v4, sizeof(pkt_v4), NULL, NULL, 111 111 + &retval, &duration); 112 112 + if (CHECK(err || retval, "ipv4", "err %d errno %d retval %d\n", 113 113 + err, errno, retval)) 114 114 + goto out; 115 115 + 116 116 + ASSERT_EQ(skel->bss->arraymap_output, expected_total, "array_output"); 117 117 + ASSERT_EQ(skel->bss->cpu + 1, skel->bss->percpu_val, "percpu_val"); 118 118 + 119 119 + out: 120 120 + free(percpu_valbuf); 121 121 + for_each_array_map_elem__destroy(skel); 122 122 + } 123 123 + 124 124 + void test_for_each(void) 125 125 + { 126 126 + if (test__start_subtest("hash_map")) 127 127 + test_hash_map(); 128 128 + if (test__start_subtest("array_map")) 129 129 + test_array_map(); 130 130 + }

+42 -9

tools/testing/selftests/bpf/prog_tests/prog_run_xattr.c

reviewed

··· 2 2 #include <test_progs.h> 3 3 #include <network_helpers.h> 4 4 5 5 + #include "test_pkt_access.skel.h" 6 6 + 7 7 + static const __u32 duration; 8 8 + 9 9 + static void check_run_cnt(int prog_fd, __u64 run_cnt) 10 10 + { 11 11 + struct bpf_prog_info info = {}; 12 12 + __u32 info_len = sizeof(info); 13 13 + int err; 14 14 + 15 15 + err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); 16 16 + if (CHECK(err, "get_prog_info", "failed to get bpf_prog_info for fd %d\n", prog_fd)) 17 17 + return; 18 18 + 19 19 + CHECK(run_cnt != info.run_cnt, "run_cnt", 20 20 + "incorrect number of repetitions, want %llu have %llu\n", run_cnt, info.run_cnt); 21 21 + } 22 22 + 5 23 void test_prog_run_xattr(void) 6 24 { 7 7 - const char *file = "./test_pkt_access.o"; 8 8 - struct bpf_object *obj; 9 9 - char buf[10]; 10 10 - int err; 25 25 + struct test_pkt_access *skel; 26 26 + int err, stats_fd = -1; 27 27 + char buf[10] = {}; 28 28 + __u64 run_cnt = 0; 29 29 + 11 30 struct bpf_prog_test_run_attr tattr = { 12 31 .repeat = 1, 13 32 .data_in = &pkt_v4, ··· 35 16 .data_size_out = 5, 36 17 }; 37 18 38 38 - err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, 39 39 - &tattr.prog_fd); 40 40 - if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno)) 19 19 + stats_fd = bpf_enable_stats(BPF_STATS_RUN_TIME); 20 20 + if (CHECK_ATTR(stats_fd < 0, "enable_stats", "failed %d\n", errno)) 41 21 return; 42 22 43 43 - memset(buf, 0, sizeof(buf)); 23 23 + skel = test_pkt_access__open_and_load(); 24 24 + if (CHECK_ATTR(!skel, "open_and_load", "failed\n")) 25 25 + goto cleanup; 26 26 + 27 27 + tattr.prog_fd = bpf_program__fd(skel->progs.test_pkt_access); 44 28 45 29 err = bpf_prog_test_run_xattr(&tattr); 46 30 CHECK_ATTR(err != -1 || errno != ENOSPC || tattr.retval, "run", ··· 56 34 CHECK_ATTR(buf[5] != 0, "overflow", 57 35 "BPF_PROG_TEST_RUN ignored size hint\n"); 58 36 37 37 + run_cnt += tattr.repeat; 38 38 + check_run_cnt(tattr.prog_fd, run_cnt); 39 39 + 59 40 tattr.data_out = NULL; 60 41 tattr.data_size_out = 0; 42 42 + tattr.repeat = 2; 61 43 errno = 0; 62 44 63 45 err = bpf_prog_test_run_xattr(&tattr); ··· 72 46 err = bpf_prog_test_run_xattr(&tattr); 73 47 CHECK_ATTR(err != -EINVAL, "run_wrong_size_out", "err %d\n", err); 74 48 75 75 - bpf_object__close(obj); 49 49 + run_cnt += tattr.repeat; 50 50 + check_run_cnt(tattr.prog_fd, run_cnt); 51 51 + 52 52 + cleanup: 53 53 + if (skel) 54 54 + test_pkt_access__destroy(skel); 55 55 + if (stats_fd != -1) 56 56 + close(stats_fd); 76 57 }

+68 -15

tools/testing/selftests/bpf/prog_tests/sk_lookup.c

reviewed

··· 241 241 return -1; 242 242 } 243 243 244 244 + static __u64 socket_cookie(int fd) 245 245 + { 246 246 + __u64 cookie; 247 247 + socklen_t cookie_len = sizeof(cookie); 248 248 + 249 249 + if (CHECK(getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &cookie_len) < 0, 250 250 + "getsockopt(SO_COOKIE)", "%s\n", strerror(errno))) 251 251 + return 0; 252 252 + return cookie; 253 253 + } 254 254 + 255 255 + static int fill_sk_lookup_ctx(struct bpf_sk_lookup *ctx, const char *local_ip, __u16 local_port, 256 256 + const char *remote_ip, __u16 remote_port) 257 257 + { 258 258 + void *local, *remote; 259 259 + int err; 260 260 + 261 261 + memset(ctx, 0, sizeof(*ctx)); 262 262 + ctx->local_port = local_port; 263 263 + ctx->remote_port = htons(remote_port); 264 264 + 265 265 + if (is_ipv6(local_ip)) { 266 266 + ctx->family = AF_INET6; 267 267 + local = &ctx->local_ip6[0]; 268 268 + remote = &ctx->remote_ip6[0]; 269 269 + } else { 270 270 + ctx->family = AF_INET; 271 271 + local = &ctx->local_ip4; 272 272 + remote = &ctx->remote_ip4; 273 273 + } 274 274 + 275 275 + err = inet_pton(ctx->family, local_ip, local); 276 276 + if (CHECK(err != 1, "inet_pton", "local_ip failed\n")) 277 277 + return 1; 278 278 + 279 279 + err = inet_pton(ctx->family, remote_ip, remote); 280 280 + if (CHECK(err != 1, "inet_pton", "remote_ip failed\n")) 281 281 + return 1; 282 282 + 283 283 + return 0; 284 284 + } 285 285 + 244 286 static int send_byte(int fd) 245 287 { 246 288 ssize_t n; ··· 1051 1009 1052 1010 static void run_sk_assign(struct test_sk_lookup *skel, 1053 1011 struct bpf_program *lookup_prog, 1054 1054 - const char *listen_ip, const char *connect_ip) 1012 1012 + const char *remote_ip, const char *local_ip) 1055 1013 { 1056 1056 - int client_fd, peer_fd, server_fds[MAX_SERVERS] = { -1 }; 1057 1057 - struct bpf_link *lookup_link; 1014 1014 + int server_fds[MAX_SERVERS] = { -1 }; 1015 1015 + struct bpf_sk_lookup ctx; 1016 1016 + __u64 server_cookie; 1058 1017 int i, err; 1059 1018 1060 1060 - lookup_link = attach_lookup_prog(lookup_prog); 1061 1061 - if (!lookup_link) 1019 1019 + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, 1020 1020 + .ctx_in = &ctx, 1021 1021 + .ctx_size_in = sizeof(ctx), 1022 1022 + .ctx_out = &ctx, 1023 1023 + .ctx_size_out = sizeof(ctx), 1024 1024 + ); 1025 1025 + 1026 1026 + if (fill_sk_lookup_ctx(&ctx, local_ip, EXT_PORT, remote_ip, INT_PORT)) 1062 1027 return; 1063 1028 1029 1029 + ctx.protocol = IPPROTO_TCP; 1030 1030 + 1064 1031 for (i = 0; i < ARRAY_SIZE(server_fds); i++) { 1065 1065 - server_fds[i] = make_server(SOCK_STREAM, listen_ip, 0, NULL); 1032 1032 + server_fds[i] = make_server(SOCK_STREAM, local_ip, 0, NULL); 1066 1033 if (server_fds[i] < 0) 1067 1034 goto close_servers; 1068 1035 ··· 1081 1030 goto close_servers; 1082 1031 } 1083 1032 1084 1084 - client_fd = make_client(SOCK_STREAM, connect_ip, EXT_PORT); 1085 1085 - if (client_fd < 0) 1033 1033 + server_cookie = socket_cookie(server_fds[SERVER_B]); 1034 1034 + if (!server_cookie) 1035 1035 + return; 1036 1036 + 1037 1037 + err = bpf_prog_test_run_opts(bpf_program__fd(lookup_prog), &opts); 1038 1038 + if (CHECK(err, "test_run", "failed with error %d\n", errno)) 1086 1039 goto close_servers; 1087 1040 1088 1088 - peer_fd = accept(server_fds[SERVER_B], NULL, NULL); 1089 1089 - if (CHECK(peer_fd < 0, "accept", "failed\n")) 1090 1090 - goto close_client; 1041 1041 + if (CHECK(ctx.cookie == 0, "ctx.cookie", "no socket selected\n")) 1042 1042 + goto close_servers; 1091 1043 1092 1092 - close(peer_fd); 1093 1093 - close_client: 1094 1094 - close(client_fd); 1044 1044 + CHECK(ctx.cookie != server_cookie, "ctx.cookie", 1045 1045 + "selected sk %llu instead of %llu\n", ctx.cookie, server_cookie); 1046 1046 + 1095 1047 close_servers: 1096 1048 for (i = 0; i < ARRAY_SIZE(server_fds); i++) { 1097 1049 if (server_fds[i] != -1) 1098 1050 close(server_fds[i]); 1099 1051 } 1100 1100 - bpf_link__destroy(lookup_link); 1101 1052 } 1102 1053 1103 1054 static void run_sk_assign_v4(struct test_sk_lookup *skel,

+4 -4

tools/testing/selftests/bpf/prog_tests/sockmap_listen.c

reviewed

··· 1014 1014 struct bpf_map *inner_map, int family, 1015 1015 int sotype) 1016 1016 { 1017 1017 - int verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 1018 1018 - int parser = bpf_program__fd(skel->progs.prog_skb_parser); 1017 1017 + int verdict = bpf_program__fd(skel->progs.prog_stream_verdict); 1018 1018 + int parser = bpf_program__fd(skel->progs.prog_stream_parser); 1019 1019 int verdict_map = bpf_map__fd(skel->maps.verdict_map); 1020 1020 int sock_map = bpf_map__fd(inner_map); 1021 1021 int err; ··· 1125 1125 struct bpf_map *inner_map, int family, 1126 1126 int sotype) 1127 1127 { 1128 1128 - int verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 1129 1129 - int parser = bpf_program__fd(skel->progs.prog_skb_parser); 1128 1128 + int verdict = bpf_program__fd(skel->progs.prog_stream_verdict); 1129 1129 + int parser = bpf_program__fd(skel->progs.prog_stream_parser); 1130 1130 int verdict_map = bpf_map__fd(skel->maps.verdict_map); 1131 1131 int sock_map = bpf_map__fd(inner_map); 1132 1132 int err;

+92

tools/testing/selftests/bpf/prog_tests/task_local_storage.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2021 Facebook */ 3 3 + 4 4 + #define _GNU_SOURCE /* See feature_test_macros(7) */ 5 5 + #include <unistd.h> 6 6 + #include <sys/syscall.h> /* For SYS_xxx definitions */ 7 7 + #include <sys/types.h> 8 8 + #include <test_progs.h> 9 9 + #include "task_local_storage.skel.h" 10 10 + #include "task_local_storage_exit_creds.skel.h" 11 11 + #include "task_ls_recursion.skel.h" 12 12 + 13 13 + static void test_sys_enter_exit(void) 14 14 + { 15 15 + struct task_local_storage *skel; 16 16 + int err; 17 17 + 18 18 + skel = task_local_storage__open_and_load(); 19 19 + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) 20 20 + return; 21 21 + 22 22 + skel->bss->target_pid = syscall(SYS_gettid); 23 23 + 24 24 + err = task_local_storage__attach(skel); 25 25 + if (!ASSERT_OK(err, "skel_attach")) 26 26 + goto out; 27 27 + 28 28 + syscall(SYS_gettid); 29 29 + syscall(SYS_gettid); 30 30 + 31 31 + /* 3x syscalls: 1x attach and 2x gettid */ 32 32 + ASSERT_EQ(skel->bss->enter_cnt, 3, "enter_cnt"); 33 33 + ASSERT_EQ(skel->bss->exit_cnt, 3, "exit_cnt"); 34 34 + ASSERT_EQ(skel->bss->mismatch_cnt, 0, "mismatch_cnt"); 35 35 + out: 36 36 + task_local_storage__destroy(skel); 37 37 + } 38 38 + 39 39 + static void test_exit_creds(void) 40 40 + { 41 41 + struct task_local_storage_exit_creds *skel; 42 42 + int err; 43 43 + 44 44 + skel = task_local_storage_exit_creds__open_and_load(); 45 45 + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) 46 46 + return; 47 47 + 48 48 + err = task_local_storage_exit_creds__attach(skel); 49 49 + if (!ASSERT_OK(err, "skel_attach")) 50 50 + goto out; 51 51 + 52 52 + /* trigger at least one exit_creds() */ 53 53 + if (CHECK_FAIL(system("ls > /dev/null"))) 54 54 + goto out; 55 55 + 56 56 + /* sync rcu to make sure exit_creds() is called for "ls" */ 57 57 + kern_sync_rcu(); 58 58 + ASSERT_EQ(skel->bss->valid_ptr_count, 0, "valid_ptr_count"); 59 59 + ASSERT_NEQ(skel->bss->null_ptr_count, 0, "null_ptr_count"); 60 60 + out: 61 61 + task_local_storage_exit_creds__destroy(skel); 62 62 + } 63 63 + 64 64 + static void test_recursion(void) 65 65 + { 66 66 + struct task_ls_recursion *skel; 67 67 + int err; 68 68 + 69 69 + skel = task_ls_recursion__open_and_load(); 70 70 + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) 71 71 + return; 72 72 + 73 73 + err = task_ls_recursion__attach(skel); 74 74 + if (!ASSERT_OK(err, "skel_attach")) 75 75 + goto out; 76 76 + 77 77 + /* trigger sys_enter, make sure it does not cause deadlock */ 78 78 + syscall(SYS_gettid); 79 79 + 80 80 + out: 81 81 + task_ls_recursion__destroy(skel); 82 82 + } 83 83 + 84 84 + void test_task_local_storage(void) 85 85 + { 86 86 + if (test__start_subtest("sys_enter_exit")) 87 87 + test_sys_enter_exit(); 88 88 + if (test__start_subtest("exit_creds")) 89 89 + test_exit_creds(); 90 90 + if (test__start_subtest("recursion")) 91 91 + test_recursion(); 92 92 + }

+7

tools/testing/selftests/bpf/progs/btf_dump_test_case_syntax.c

reviewed

··· 205 205 int t[11]; 206 206 }; 207 207 208 208 + struct float_struct { 209 209 + float f; 210 210 + const double *d; 211 211 + volatile long double *ld; 212 212 + }; 213 213 + 208 214 struct root_struct { 209 215 enum e1 _1; 210 216 enum e2 _2; ··· 225 219 union_fwd_t *_12; 226 220 union_fwd_ptr_t _13; 227 221 struct struct_with_embedded_stuff _14; 222 222 + struct float_struct _15; 228 223 }; 229 224 230 225 /* ------ END-EXPECTED-OUTPUT ------ */

+5

tools/testing/selftests/bpf/progs/core_reloc_types.h

reviewed

··· 807 807 int arr_elem_sz; 808 808 int ptr_sz; 809 809 int enum_sz; 810 810 + int float_sz; 810 811 }; 811 812 812 813 struct core_reloc_size { ··· 817 816 int arr_field[4]; 818 817 void *ptr_field; 819 818 enum { VALUE = 123 } enum_field; 819 819 + float float_field; 820 820 }; 821 821 822 822 struct core_reloc_size___diff_sz { ··· 827 825 char arr_field[10]; 828 826 void *ptr_field; 829 827 enum { OTHER_VALUE = 0xFFFFFFFFFFFFFFFF } enum_field; 828 828 + double float_field; 830 829 }; 831 830 832 831 /* Error case of two candidates with the fields (int_field) at the same ··· 842 839 int arr_field[4]; 843 840 void *ptr_field; 844 841 enum { VALUE___1 = 123 } enum_field; 842 842 + float float_field; 845 843 }; 846 844 847 845 struct core_reloc_size___err_ambiguous2 { ··· 854 850 int arr_field[4]; 855 851 void *ptr_field; 856 852 enum { VALUE___2 = 123 } enum_field; 853 853 + float float_field; 857 854 }; 858 855 859 856 /*

+61

tools/testing/selftests/bpf/progs/for_each_array_map_elem.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2021 Facebook */ 3 3 + #include "vmlinux.h" 4 4 + #include <bpf/bpf_helpers.h> 5 5 + 6 6 + char _license[] SEC("license") = "GPL"; 7 7 + 8 8 + struct { 9 9 + __uint(type, BPF_MAP_TYPE_ARRAY); 10 10 + __uint(max_entries, 3); 11 11 + __type(key, __u32); 12 12 + __type(value, __u64); 13 13 + } arraymap SEC(".maps"); 14 14 + 15 15 + struct { 16 16 + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 17 17 + __uint(max_entries, 1); 18 18 + __type(key, __u32); 19 19 + __type(value, __u64); 20 20 + } percpu_map SEC(".maps"); 21 21 + 22 22 + struct callback_ctx { 23 23 + int output; 24 24 + }; 25 25 + 26 26 + static __u64 27 27 + check_array_elem(struct bpf_map *map, __u32 *key, __u64 *val, 28 28 + struct callback_ctx *data) 29 29 + { 30 30 + data->output += *val; 31 31 + if (*key == 1) 32 32 + return 1; /* stop the iteration */ 33 33 + return 0; 34 34 + } 35 35 + 36 36 + __u32 cpu = 0; 37 37 + __u64 percpu_val = 0; 38 38 + 39 39 + static __u64 40 40 + check_percpu_elem(struct bpf_map *map, __u32 *key, __u64 *val, 41 41 + struct callback_ctx *data) 42 42 + { 43 43 + cpu = bpf_get_smp_processor_id(); 44 44 + percpu_val = *val; 45 45 + return 0; 46 46 + } 47 47 + 48 48 + u32 arraymap_output = 0; 49 49 + 50 50 + SEC("classifier") 51 51 + int test_pkt_access(struct __sk_buff *skb) 52 52 + { 53 53 + struct callback_ctx data; 54 54 + 55 55 + data.output = 0; 56 56 + bpf_for_each_map_elem(&arraymap, check_array_elem, &data, 0); 57 57 + arraymap_output = data.output; 58 58 + 59 59 + bpf_for_each_map_elem(&percpu_map, check_percpu_elem, (void *)0, 0); 60 60 + return 0; 61 61 + }

+95

tools/testing/selftests/bpf/progs/for_each_hash_map_elem.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2021 Facebook */ 3 3 + #include "vmlinux.h" 4 4 + #include <bpf/bpf_helpers.h> 5 5 + 6 6 + char _license[] SEC("license") = "GPL"; 7 7 + 8 8 + struct { 9 9 + __uint(type, BPF_MAP_TYPE_HASH); 10 10 + __uint(max_entries, 3); 11 11 + __type(key, __u32); 12 12 + __type(value, __u64); 13 13 + } hashmap SEC(".maps"); 14 14 + 15 15 + struct { 16 16 + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); 17 17 + __uint(max_entries, 1); 18 18 + __type(key, __u32); 19 19 + __type(value, __u64); 20 20 + } percpu_map SEC(".maps"); 21 21 + 22 22 + struct callback_ctx { 23 23 + struct __sk_buff *ctx; 24 24 + int input; 25 25 + int output; 26 26 + }; 27 27 + 28 28 + static __u64 29 29 + check_hash_elem(struct bpf_map *map, __u32 *key, __u64 *val, 30 30 + struct callback_ctx *data) 31 31 + { 32 32 + struct __sk_buff *skb = data->ctx; 33 33 + __u32 k; 34 34 + __u64 v; 35 35 + 36 36 + if (skb) { 37 37 + k = *key; 38 38 + v = *val; 39 39 + if (skb->len == 10000 && k == 10 && v == 10) 40 40 + data->output = 3; /* impossible path */ 41 41 + else 42 42 + data->output = 4; 43 43 + } else { 44 44 + data->output = data->input; 45 45 + bpf_map_delete_elem(map, key); 46 46 + } 47 47 + 48 48 + return 0; 49 49 + } 50 50 + 51 51 + __u32 cpu = 0; 52 52 + __u32 percpu_called = 0; 53 53 + __u32 percpu_key = 0; 54 54 + __u64 percpu_val = 0; 55 55 + int percpu_output = 0; 56 56 + 57 57 + static __u64 58 58 + check_percpu_elem(struct bpf_map *map, __u32 *key, __u64 *val, 59 59 + struct callback_ctx *unused) 60 60 + { 61 61 + struct callback_ctx data; 62 62 + 63 63 + percpu_called++; 64 64 + cpu = bpf_get_smp_processor_id(); 65 65 + percpu_key = *key; 66 66 + percpu_val = *val; 67 67 + 68 68 + data.ctx = 0; 69 69 + data.input = 100; 70 70 + data.output = 0; 71 71 + bpf_for_each_map_elem(&hashmap, check_hash_elem, &data, 0); 72 72 + percpu_output = data.output; 73 73 + 74 74 + return 0; 75 75 + } 76 76 + 77 77 + int hashmap_output = 0; 78 78 + int hashmap_elems = 0; 79 79 + int percpu_map_elems = 0; 80 80 + 81 81 + SEC("classifier") 82 82 + int test_pkt_access(struct __sk_buff *skb) 83 83 + { 84 84 + struct callback_ctx data; 85 85 + 86 86 + data.ctx = skb; 87 87 + data.input = 10; 88 88 + data.output = 0; 89 89 + hashmap_elems = bpf_for_each_map_elem(&hashmap, check_hash_elem, &data, 0); 90 90 + hashmap_output = data.output; 91 91 + 92 92 + percpu_map_elems = bpf_for_each_map_elem(&percpu_map, check_percpu_elem, 93 93 + (void *)0, 0); 94 94 + return 0; 95 95 + }

+99

tools/testing/selftests/bpf/progs/loop6.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + #include <linux/ptrace.h> 4 4 + #include <stddef.h> 5 5 + #include <linux/bpf.h> 6 6 + #include <bpf/bpf_helpers.h> 7 7 + #include <bpf/bpf_tracing.h> 8 8 + 9 9 + char _license[] SEC("license") = "GPL"; 10 10 + 11 11 + /* typically virtio scsi has max SGs of 6 */ 12 12 + #define VIRTIO_MAX_SGS 6 13 13 + 14 14 + /* Verifier will fail with SG_MAX = 128. The failure can be 15 15 + * workarounded with a smaller SG_MAX, e.g. 10. 16 16 + */ 17 17 + #define WORKAROUND 18 18 + #ifdef WORKAROUND 19 19 + #define SG_MAX 10 20 20 + #else 21 21 + /* typically virtio blk has max SEG of 128 */ 22 22 + #define SG_MAX 128 23 23 + #endif 24 24 + 25 25 + #define SG_CHAIN 0x01UL 26 26 + #define SG_END 0x02UL 27 27 + 28 28 + struct scatterlist { 29 29 + unsigned long page_link; 30 30 + unsigned int offset; 31 31 + unsigned int length; 32 32 + }; 33 33 + 34 34 + #define sg_is_chain(sg) ((sg)->page_link & SG_CHAIN) 35 35 + #define sg_is_last(sg) ((sg)->page_link & SG_END) 36 36 + #define sg_chain_ptr(sg) \ 37 37 + ((struct scatterlist *) ((sg)->page_link & ~(SG_CHAIN | SG_END))) 38 38 + 39 39 + static inline struct scatterlist *__sg_next(struct scatterlist *sgp) 40 40 + { 41 41 + struct scatterlist sg; 42 42 + 43 43 + bpf_probe_read_kernel(&sg, sizeof(sg), sgp); 44 44 + if (sg_is_last(&sg)) 45 45 + return NULL; 46 46 + 47 47 + sgp++; 48 48 + 49 49 + bpf_probe_read_kernel(&sg, sizeof(sg), sgp); 50 50 + if (sg_is_chain(&sg)) 51 51 + sgp = sg_chain_ptr(&sg); 52 52 + 53 53 + return sgp; 54 54 + } 55 55 + 56 56 + static inline struct scatterlist *get_sgp(struct scatterlist **sgs, int i) 57 57 + { 58 58 + struct scatterlist *sgp; 59 59 + 60 60 + bpf_probe_read_kernel(&sgp, sizeof(sgp), sgs + i); 61 61 + return sgp; 62 62 + } 63 63 + 64 64 + int config = 0; 65 65 + int result = 0; 66 66 + 67 67 + SEC("kprobe/virtqueue_add_sgs") 68 68 + int BPF_KPROBE(trace_virtqueue_add_sgs, void *unused, struct scatterlist **sgs, 69 69 + unsigned int out_sgs, unsigned int in_sgs) 70 70 + { 71 71 + struct scatterlist *sgp = NULL; 72 72 + __u64 length1 = 0, length2 = 0; 73 73 + unsigned int i, n, len; 74 74 + 75 75 + if (config != 0) 76 76 + return 0; 77 77 + 78 78 + for (i = 0; (i < VIRTIO_MAX_SGS) && (i < out_sgs); i++) { 79 79 + for (n = 0, sgp = get_sgp(sgs, i); sgp && (n < SG_MAX); 80 80 + sgp = __sg_next(sgp)) { 81 81 + bpf_probe_read_kernel(&len, sizeof(len), &sgp->length); 82 82 + length1 += len; 83 83 + n++; 84 84 + } 85 85 + } 86 86 + 87 87 + for (i = 0; (i < VIRTIO_MAX_SGS) && (i < in_sgs); i++) { 88 88 + for (n = 0, sgp = get_sgp(sgs, i); sgp && (n < SG_MAX); 89 89 + sgp = __sg_next(sgp)) { 90 90 + bpf_probe_read_kernel(&len, sizeof(len), &sgp->length); 91 91 + length2 += len; 92 92 + n++; 93 93 + } 94 94 + } 95 95 + 96 96 + config = 1; 97 97 + result = length2 - length1; 98 98 + return 0; 99 99 + }

+64

tools/testing/selftests/bpf/progs/task_local_storage.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2021 Facebook */ 3 3 + 4 4 + #include "vmlinux.h" 5 5 + #include <bpf/bpf_helpers.h> 6 6 + #include <bpf/bpf_tracing.h> 7 7 + 8 8 + char _license[] SEC("license") = "GPL"; 9 9 + 10 10 + struct { 11 11 + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); 12 12 + __uint(map_flags, BPF_F_NO_PREALLOC); 13 13 + __type(key, int); 14 14 + __type(value, long); 15 15 + } enter_id SEC(".maps"); 16 16 + 17 17 + #define MAGIC_VALUE 0xabcd1234 18 18 + 19 19 + pid_t target_pid = 0; 20 20 + int mismatch_cnt = 0; 21 21 + int enter_cnt = 0; 22 22 + int exit_cnt = 0; 23 23 + 24 24 + SEC("tp_btf/sys_enter") 25 25 + int BPF_PROG(on_enter, struct pt_regs *regs, long id) 26 26 + { 27 27 + struct task_struct *task; 28 28 + long *ptr; 29 29 + 30 30 + task = bpf_get_current_task_btf(); 31 31 + if (task->pid != target_pid) 32 32 + return 0; 33 33 + 34 34 + ptr = bpf_task_storage_get(&enter_id, task, 0, 35 35 + BPF_LOCAL_STORAGE_GET_F_CREATE); 36 36 + if (!ptr) 37 37 + return 0; 38 38 + 39 39 + __sync_fetch_and_add(&enter_cnt, 1); 40 40 + *ptr = MAGIC_VALUE + enter_cnt; 41 41 + 42 42 + return 0; 43 43 + } 44 44 + 45 45 + SEC("tp_btf/sys_exit") 46 46 + int BPF_PROG(on_exit, struct pt_regs *regs, long id) 47 47 + { 48 48 + struct task_struct *task; 49 49 + long *ptr; 50 50 + 51 51 + task = bpf_get_current_task_btf(); 52 52 + if (task->pid != target_pid) 53 53 + return 0; 54 54 + 55 55 + ptr = bpf_task_storage_get(&enter_id, task, 0, 56 56 + BPF_LOCAL_STORAGE_GET_F_CREATE); 57 57 + if (!ptr) 58 58 + return 0; 59 59 + 60 60 + __sync_fetch_and_add(&exit_cnt, 1); 61 61 + if (*ptr != MAGIC_VALUE + exit_cnt) 62 62 + __sync_fetch_and_add(&mismatch_cnt, 1); 63 63 + return 0; 64 64 + }

+32

tools/testing/selftests/bpf/progs/task_local_storage_exit_creds.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2021 Facebook */ 3 3 + 4 4 + #include "vmlinux.h" 5 5 + #include <bpf/bpf_helpers.h> 6 6 + #include <bpf/bpf_tracing.h> 7 7 + 8 8 + char _license[] SEC("license") = "GPL"; 9 9 + 10 10 + struct { 11 11 + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); 12 12 + __uint(map_flags, BPF_F_NO_PREALLOC); 13 13 + __type(key, int); 14 14 + __type(value, __u64); 15 15 + } task_storage SEC(".maps"); 16 16 + 17 17 + int valid_ptr_count = 0; 18 18 + int null_ptr_count = 0; 19 19 + 20 20 + SEC("fentry/exit_creds") 21 21 + int BPF_PROG(trace_exit_creds, struct task_struct *task) 22 22 + { 23 23 + __u64 *ptr; 24 24 + 25 25 + ptr = bpf_task_storage_get(&task_storage, task, 0, 26 26 + BPF_LOCAL_STORAGE_GET_F_CREATE); 27 27 + if (ptr) 28 28 + __sync_fetch_and_add(&valid_ptr_count, 1); 29 29 + else 30 30 + __sync_fetch_and_add(&null_ptr_count, 1); 31 31 + return 0; 32 32 + }

+70

tools/testing/selftests/bpf/progs/task_ls_recursion.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2021 Facebook */ 3 3 + 4 4 + #include "vmlinux.h" 5 5 + #include <bpf/bpf_helpers.h> 6 6 + #include <bpf/bpf_tracing.h> 7 7 + 8 8 + char _license[] SEC("license") = "GPL"; 9 9 + 10 10 + struct { 11 11 + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); 12 12 + __uint(map_flags, BPF_F_NO_PREALLOC); 13 13 + __type(key, int); 14 14 + __type(value, long); 15 15 + } map_a SEC(".maps"); 16 16 + 17 17 + struct { 18 18 + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); 19 19 + __uint(map_flags, BPF_F_NO_PREALLOC); 20 20 + __type(key, int); 21 21 + __type(value, long); 22 22 + } map_b SEC(".maps"); 23 23 + 24 24 + SEC("fentry/bpf_local_storage_lookup") 25 25 + int BPF_PROG(on_lookup) 26 26 + { 27 27 + struct task_struct *task = bpf_get_current_task_btf(); 28 28 + 29 29 + bpf_task_storage_delete(&map_a, task); 30 30 + bpf_task_storage_delete(&map_b, task); 31 31 + return 0; 32 32 + } 33 33 + 34 34 + SEC("fentry/bpf_local_storage_update") 35 35 + int BPF_PROG(on_update) 36 36 + { 37 37 + struct task_struct *task = bpf_get_current_task_btf(); 38 38 + long *ptr; 39 39 + 40 40 + ptr = bpf_task_storage_get(&map_a, task, 0, 41 41 + BPF_LOCAL_STORAGE_GET_F_CREATE); 42 42 + if (ptr) 43 43 + *ptr += 1; 44 44 + 45 45 + ptr = bpf_task_storage_get(&map_b, task, 0, 46 46 + BPF_LOCAL_STORAGE_GET_F_CREATE); 47 47 + if (ptr) 48 48 + *ptr += 1; 49 49 + 50 50 + return 0; 51 51 + } 52 52 + 53 53 + SEC("tp_btf/sys_enter") 54 54 + int BPF_PROG(on_enter, struct pt_regs *regs, long id) 55 55 + { 56 56 + struct task_struct *task; 57 57 + long *ptr; 58 58 + 59 59 + task = bpf_get_current_task_btf(); 60 60 + ptr = bpf_task_storage_get(&map_a, task, 0, 61 61 + BPF_LOCAL_STORAGE_GET_F_CREATE); 62 62 + if (ptr) 63 63 + *ptr = 200; 64 64 + 65 65 + ptr = bpf_task_storage_get(&map_b, task, 0, 66 66 + BPF_LOCAL_STORAGE_GET_F_CREATE); 67 67 + if (ptr) 68 68 + *ptr = 100; 69 69 + return 0; 70 70 + }

+3

tools/testing/selftests/bpf/progs/test_core_reloc_size.c

reviewed

··· 21 21 int arr_elem_sz; 22 22 int ptr_sz; 23 23 int enum_sz; 24 24 + int float_sz; 24 25 }; 25 26 26 27 struct core_reloc_size { ··· 31 30 int arr_field[4]; 32 31 void *ptr_field; 33 32 enum { VALUE = 123 } enum_field; 33 33 + float float_field; 34 34 }; 35 35 36 36 SEC("raw_tracepoint/sys_enter") ··· 47 45 out->arr_elem_sz = bpf_core_field_size(in->arr_field[0]); 48 46 out->ptr_sz = bpf_core_field_size(in->ptr_field); 49 47 out->enum_sz = bpf_core_field_size(in->enum_field); 48 48 + out->float_sz = bpf_core_field_size(in->float_field); 50 49 51 50 return 0; 52 51 }

+41 -21

tools/testing/selftests/bpf/progs/test_sk_lookup.c

reviewed

··· 64 64 static const __u32 KEY_SERVER_A = SERVER_A; 65 65 static const __u32 KEY_SERVER_B = SERVER_B; 66 66 67 67 + static const __u16 SRC_PORT = bpf_htons(8008); 68 68 + static const __u32 SRC_IP4 = IP4(127, 0, 0, 2); 69 69 + static const __u32 SRC_IP6[] = IP6(0xfd000000, 0x0, 0x0, 0x00000002); 70 70 + 67 71 static const __u16 DST_PORT = 7007; /* Host byte order */ 68 72 static const __u32 DST_IP4 = IP4(127, 0, 0, 1); 69 73 static const __u32 DST_IP6[] = IP6(0xfd000000, 0x0, 0x0, 0x00000001); ··· 402 398 if (LSW(ctx->protocol, 0) != IPPROTO_TCP) 403 399 return SK_DROP; 404 400 405 405 - /* Narrow loads from remote_port field. Expect non-0 value. */ 406 406 - if (LSB(ctx->remote_port, 0) == 0 && LSB(ctx->remote_port, 1) == 0 && 407 407 - LSB(ctx->remote_port, 2) == 0 && LSB(ctx->remote_port, 3) == 0) 401 401 + /* Narrow loads from remote_port field. Expect SRC_PORT. */ 402 402 + if (LSB(ctx->remote_port, 0) != ((SRC_PORT >> 0) & 0xff) || 403 403 + LSB(ctx->remote_port, 1) != ((SRC_PORT >> 8) & 0xff) || 404 404 + LSB(ctx->remote_port, 2) != 0 || LSB(ctx->remote_port, 3) != 0) 408 405 return SK_DROP; 409 409 - if (LSW(ctx->remote_port, 0) == 0) 406 406 + if (LSW(ctx->remote_port, 0) != SRC_PORT) 410 407 return SK_DROP; 411 408 412 409 /* Narrow loads from local_port field. Expect DST_PORT. */ ··· 420 415 421 416 /* Narrow loads from IPv4 fields */ 422 417 if (v4) { 423 423 - /* Expect non-0.0.0.0 in remote_ip4 */ 424 424 - if (LSB(ctx->remote_ip4, 0) == 0 && LSB(ctx->remote_ip4, 1) == 0 && 425 425 - LSB(ctx->remote_ip4, 2) == 0 && LSB(ctx->remote_ip4, 3) == 0) 418 418 + /* Expect SRC_IP4 in remote_ip4 */ 419 419 + if (LSB(ctx->remote_ip4, 0) != ((SRC_IP4 >> 0) & 0xff) || 420 420 + LSB(ctx->remote_ip4, 1) != ((SRC_IP4 >> 8) & 0xff) || 421 421 + LSB(ctx->remote_ip4, 2) != ((SRC_IP4 >> 16) & 0xff) || 422 422 + LSB(ctx->remote_ip4, 3) != ((SRC_IP4 >> 24) & 0xff)) 426 423 return SK_DROP; 427 427 - if (LSW(ctx->remote_ip4, 0) == 0 && LSW(ctx->remote_ip4, 1) == 0) 424 424 + if (LSW(ctx->remote_ip4, 0) != ((SRC_IP4 >> 0) & 0xffff) || 425 425 + LSW(ctx->remote_ip4, 1) != ((SRC_IP4 >> 16) & 0xffff)) 428 426 return SK_DROP; 429 427 430 428 /* Expect DST_IP4 in local_ip4 */ ··· 456 448 457 449 /* Narrow loads from IPv6 fields */ 458 450 if (!v4) { 459 459 - /* Expect non-:: IP in remote_ip6 */ 460 460 - if (LSB(ctx->remote_ip6[0], 0) == 0 && LSB(ctx->remote_ip6[0], 1) == 0 && 461 461 - LSB(ctx->remote_ip6[0], 2) == 0 && LSB(ctx->remote_ip6[0], 3) == 0 && 462 462 - LSB(ctx->remote_ip6[1], 0) == 0 && LSB(ctx->remote_ip6[1], 1) == 0 && 463 463 - LSB(ctx->remote_ip6[1], 2) == 0 && LSB(ctx->remote_ip6[1], 3) == 0 && 464 464 - LSB(ctx->remote_ip6[2], 0) == 0 && LSB(ctx->remote_ip6[2], 1) == 0 && 465 465 - LSB(ctx->remote_ip6[2], 2) == 0 && LSB(ctx->remote_ip6[2], 3) == 0 && 466 466 - LSB(ctx->remote_ip6[3], 0) == 0 && LSB(ctx->remote_ip6[3], 1) == 0 && 467 467 - LSB(ctx->remote_ip6[3], 2) == 0 && LSB(ctx->remote_ip6[3], 3) == 0) 451 451 + /* Expect SRC_IP6 in remote_ip6 */ 452 452 + if (LSB(ctx->remote_ip6[0], 0) != ((SRC_IP6[0] >> 0) & 0xff) || 453 453 + LSB(ctx->remote_ip6[0], 1) != ((SRC_IP6[0] >> 8) & 0xff) || 454 454 + LSB(ctx->remote_ip6[0], 2) != ((SRC_IP6[0] >> 16) & 0xff) || 455 455 + LSB(ctx->remote_ip6[0], 3) != ((SRC_IP6[0] >> 24) & 0xff) || 456 456 + LSB(ctx->remote_ip6[1], 0) != ((SRC_IP6[1] >> 0) & 0xff) || 457 457 + LSB(ctx->remote_ip6[1], 1) != ((SRC_IP6[1] >> 8) & 0xff) || 458 458 + LSB(ctx->remote_ip6[1], 2) != ((SRC_IP6[1] >> 16) & 0xff) || 459 459 + LSB(ctx->remote_ip6[1], 3) != ((SRC_IP6[1] >> 24) & 0xff) || 460 460 + LSB(ctx->remote_ip6[2], 0) != ((SRC_IP6[2] >> 0) & 0xff) || 461 461 + LSB(ctx->remote_ip6[2], 1) != ((SRC_IP6[2] >> 8) & 0xff) || 462 462 + LSB(ctx->remote_ip6[2], 2) != ((SRC_IP6[2] >> 16) & 0xff) || 463 463 + LSB(ctx->remote_ip6[2], 3) != ((SRC_IP6[2] >> 24) & 0xff) || 464 464 + LSB(ctx->remote_ip6[3], 0) != ((SRC_IP6[3] >> 0) & 0xff) || 465 465 + LSB(ctx->remote_ip6[3], 1) != ((SRC_IP6[3] >> 8) & 0xff) || 466 466 + LSB(ctx->remote_ip6[3], 2) != ((SRC_IP6[3] >> 16) & 0xff) || 467 467 + LSB(ctx->remote_ip6[3], 3) != ((SRC_IP6[3] >> 24) & 0xff)) 468 468 return SK_DROP; 469 469 - if (LSW(ctx->remote_ip6[0], 0) == 0 && LSW(ctx->remote_ip6[0], 1) == 0 && 470 470 - LSW(ctx->remote_ip6[1], 0) == 0 && LSW(ctx->remote_ip6[1], 1) == 0 && 471 471 - LSW(ctx->remote_ip6[2], 0) == 0 && LSW(ctx->remote_ip6[2], 1) == 0 && 472 472 - LSW(ctx->remote_ip6[3], 0) == 0 && LSW(ctx->remote_ip6[3], 1) == 0) 469 469 + if (LSW(ctx->remote_ip6[0], 0) != ((SRC_IP6[0] >> 0) & 0xffff) || 470 470 + LSW(ctx->remote_ip6[0], 1) != ((SRC_IP6[0] >> 16) & 0xffff) || 471 471 + LSW(ctx->remote_ip6[1], 0) != ((SRC_IP6[1] >> 0) & 0xffff) || 472 472 + LSW(ctx->remote_ip6[1], 1) != ((SRC_IP6[1] >> 16) & 0xffff) || 473 473 + LSW(ctx->remote_ip6[2], 0) != ((SRC_IP6[2] >> 0) & 0xffff) || 474 474 + LSW(ctx->remote_ip6[2], 1) != ((SRC_IP6[2] >> 16) & 0xffff) || 475 475 + LSW(ctx->remote_ip6[3], 0) != ((SRC_IP6[3] >> 0) & 0xffff) || 476 476 + LSW(ctx->remote_ip6[3], 1) != ((SRC_IP6[3] >> 16) & 0xffff)) 473 477 return SK_DROP; 474 478 /* Expect DST_IP6 in local_ip6 */ 475 479 if (LSB(ctx->local_ip6[0], 0) != ((DST_IP6[0] >> 0) & 0xff) ||

+2 -2

tools/testing/selftests/bpf/progs/test_sockmap_listen.c

reviewed

··· 31 31 static volatile bool test_sockmap; /* toggled by user-space */ 32 32 33 33 SEC("sk_skb/stream_parser") 34 34 - int prog_skb_parser(struct __sk_buff *skb) 34 34 + int prog_stream_parser(struct __sk_buff *skb) 35 35 { 36 36 return skb->len; 37 37 } 38 38 39 39 SEC("sk_skb/stream_verdict") 40 40 - int prog_skb_verdict(struct __sk_buff *skb) 40 40 + int prog_stream_verdict(struct __sk_buff *skb) 41 41 { 42 42 unsigned int *count; 43 43 __u32 zero = 0;

+99 -14

tools/testing/selftests/bpf/progs/test_tc_tunnel.c

reviewed

··· 24 24 25 25 static const int cfg_udp_src = 20000; 26 26 27 27 + #define L2_PAD_SZ (sizeof(struct vxlanhdr) + ETH_HLEN) 28 28 + 27 29 #define UDP_PORT 5555 28 30 #define MPLS_OVER_UDP_PORT 6635 29 31 #define ETH_OVER_UDP_PORT 7777 32 32 + #define VXLAN_UDP_PORT 8472 33 33 + 34 34 + #define EXTPROTO_VXLAN 0x1 35 35 + 36 36 + #define VXLAN_N_VID (1u << 24) 37 37 + #define VXLAN_VNI_MASK bpf_htonl((VXLAN_N_VID - 1) << 8) 38 38 + #define VXLAN_FLAGS 0x8 39 39 + #define VXLAN_VNI 1 30 40 31 41 /* MPLS label 1000 with S bit (last label) set and ttl of 255. */ 32 42 static const __u32 mpls_label = __bpf_constant_htonl(1000 << 12 | 33 43 MPLS_LS_S_MASK | 0xff); 44 44 + 45 45 + struct vxlanhdr { 46 46 + __be32 vx_flags; 47 47 + __be32 vx_vni; 48 48 + } __attribute__((packed)); 34 49 35 50 struct gre_hdr { 36 51 __be16 flags; ··· 60 45 struct v4hdr { 61 46 struct iphdr ip; 62 47 union l4hdr l4hdr; 63 63 - __u8 pad[16]; /* enough space for L2 header */ 48 48 + __u8 pad[L2_PAD_SZ]; /* space for L2 header / vxlan header ... */ 64 49 } __attribute__((packed)); 65 50 66 51 struct v6hdr { 67 52 struct ipv6hdr ip; 68 53 union l4hdr l4hdr; 69 69 - __u8 pad[16]; /* enough space for L2 header */ 54 54 + __u8 pad[L2_PAD_SZ]; /* space for L2 header / vxlan header ... */ 70 55 } __attribute__((packed)); 71 56 72 57 static __always_inline void set_ipv4_csum(struct iphdr *iph) ··· 84 69 iph->check = ~((csum & 0xffff) + (csum >> 16)); 85 70 } 86 71 87 87 - static __always_inline int encap_ipv4(struct __sk_buff *skb, __u8 encap_proto, 88 88 - __u16 l2_proto) 72 72 + static __always_inline int __encap_ipv4(struct __sk_buff *skb, __u8 encap_proto, 73 73 + __u16 l2_proto, __u16 ext_proto) 89 74 { 90 75 __u16 udp_dst = UDP_PORT; 91 76 struct iphdr iph_inner; 92 77 struct v4hdr h_outer; 93 78 struct tcphdr tcph; 94 79 int olen, l2_len; 80 80 + __u8 *l2_hdr = NULL; 95 81 int tcp_off; 96 82 __u64 flags; 97 83 ··· 157 141 break; 158 142 case ETH_P_TEB: 159 143 l2_len = ETH_HLEN; 160 160 - udp_dst = ETH_OVER_UDP_PORT; 144 144 + if (ext_proto & EXTPROTO_VXLAN) { 145 145 + udp_dst = VXLAN_UDP_PORT; 146 146 + l2_len += sizeof(struct vxlanhdr); 147 147 + } else 148 148 + udp_dst = ETH_OVER_UDP_PORT; 161 149 break; 162 150 } 163 151 flags |= BPF_F_ADJ_ROOM_ENCAP_L2(l2_len); ··· 191 171 } 192 172 193 173 /* add L2 encap (if specified) */ 174 174 + l2_hdr = (__u8 *)&h_outer + olen; 194 175 switch (l2_proto) { 195 176 case ETH_P_MPLS_UC: 196 196 - *((__u32 *)((__u8 *)&h_outer + olen)) = mpls_label; 177 177 + *(__u32 *)l2_hdr = mpls_label; 197 178 break; 198 179 case ETH_P_TEB: 199 199 - if (bpf_skb_load_bytes(skb, 0, (__u8 *)&h_outer + olen, 200 200 - ETH_HLEN)) 180 180 + flags |= BPF_F_ADJ_ROOM_ENCAP_L2_ETH; 181 181 + 182 182 + if (ext_proto & EXTPROTO_VXLAN) { 183 183 + struct vxlanhdr *vxlan_hdr = (struct vxlanhdr *)l2_hdr; 184 184 + 185 185 + vxlan_hdr->vx_flags = VXLAN_FLAGS; 186 186 + vxlan_hdr->vx_vni = bpf_htonl((VXLAN_VNI & VXLAN_VNI_MASK) << 8); 187 187 + 188 188 + l2_hdr += sizeof(struct vxlanhdr); 189 189 + } 190 190 + 191 191 + if (bpf_skb_load_bytes(skb, 0, l2_hdr, ETH_HLEN)) 201 192 return TC_ACT_SHOT; 193 193 + 202 194 break; 203 195 } 204 196 olen += l2_len; ··· 246 214 return TC_ACT_OK; 247 215 } 248 216 249 249 - static __always_inline int encap_ipv6(struct __sk_buff *skb, __u8 encap_proto, 217 217 + static __always_inline int encap_ipv4(struct __sk_buff *skb, __u8 encap_proto, 250 218 __u16 l2_proto) 219 219 + { 220 220 + return __encap_ipv4(skb, encap_proto, l2_proto, 0); 221 221 + } 222 222 + 223 223 + static __always_inline int __encap_ipv6(struct __sk_buff *skb, __u8 encap_proto, 224 224 + __u16 l2_proto, __u16 ext_proto) 251 225 { 252 226 __u16 udp_dst = UDP_PORT; 253 227 struct ipv6hdr iph_inner; 254 228 struct v6hdr h_outer; 255 229 struct tcphdr tcph; 256 230 int olen, l2_len; 231 231 + __u8 *l2_hdr = NULL; 257 232 __u16 tot_len; 258 233 __u64 flags; 259 234 ··· 288 249 break; 289 250 case ETH_P_TEB: 290 251 l2_len = ETH_HLEN; 291 291 - udp_dst = ETH_OVER_UDP_PORT; 252 252 + if (ext_proto & EXTPROTO_VXLAN) { 253 253 + udp_dst = VXLAN_UDP_PORT; 254 254 + l2_len += sizeof(struct vxlanhdr); 255 255 + } else 256 256 + udp_dst = ETH_OVER_UDP_PORT; 292 257 break; 293 258 } 294 259 flags |= BPF_F_ADJ_ROOM_ENCAP_L2(l2_len); ··· 310 267 h_outer.l4hdr.udp.source = __bpf_constant_htons(cfg_udp_src); 311 268 h_outer.l4hdr.udp.dest = bpf_htons(udp_dst); 312 269 tot_len = bpf_ntohs(iph_inner.payload_len) + sizeof(iph_inner) + 313 313 - sizeof(h_outer.l4hdr.udp); 270 270 + sizeof(h_outer.l4hdr.udp) + l2_len; 314 271 h_outer.l4hdr.udp.check = 0; 315 272 h_outer.l4hdr.udp.len = bpf_htons(tot_len); 316 273 break; ··· 321 278 } 322 279 323 280 /* add L2 encap (if specified) */ 281 281 + l2_hdr = (__u8 *)&h_outer + olen; 324 282 switch (l2_proto) { 325 283 case ETH_P_MPLS_UC: 326 326 - *((__u32 *)((__u8 *)&h_outer + olen)) = mpls_label; 284 284 + *(__u32 *)l2_hdr = mpls_label; 327 285 break; 328 286 case ETH_P_TEB: 329 329 - if (bpf_skb_load_bytes(skb, 0, (__u8 *)&h_outer + olen, 330 330 - ETH_HLEN)) 287 287 + flags |= BPF_F_ADJ_ROOM_ENCAP_L2_ETH; 288 288 + 289 289 + if (ext_proto & EXTPROTO_VXLAN) { 290 290 + struct vxlanhdr *vxlan_hdr = (struct vxlanhdr *)l2_hdr; 291 291 + 292 292 + vxlan_hdr->vx_flags = VXLAN_FLAGS; 293 293 + vxlan_hdr->vx_vni = bpf_htonl((VXLAN_VNI & VXLAN_VNI_MASK) << 8); 294 294 + 295 295 + l2_hdr += sizeof(struct vxlanhdr); 296 296 + } 297 297 + 298 298 + if (bpf_skb_load_bytes(skb, 0, l2_hdr, ETH_HLEN)) 331 299 return TC_ACT_SHOT; 332 300 break; 333 301 } ··· 361 307 return TC_ACT_SHOT; 362 308 363 309 return TC_ACT_OK; 310 310 + } 311 311 + 312 312 + static __always_inline int encap_ipv6(struct __sk_buff *skb, __u8 encap_proto, 313 313 + __u16 l2_proto) 314 314 + { 315 315 + return __encap_ipv6(skb, encap_proto, l2_proto, 0); 364 316 } 365 317 366 318 SEC("encap_ipip_none") ··· 428 368 { 429 369 if (skb->protocol == __bpf_constant_htons(ETH_P_IP)) 430 370 return encap_ipv4(skb, IPPROTO_UDP, ETH_P_TEB); 371 371 + else 372 372 + return TC_ACT_OK; 373 373 + } 374 374 + 375 375 + SEC("encap_vxlan_eth") 376 376 + int __encap_vxlan_eth(struct __sk_buff *skb) 377 377 + { 378 378 + if (skb->protocol == __bpf_constant_htons(ETH_P_IP)) 379 379 + return __encap_ipv4(skb, IPPROTO_UDP, 380 380 + ETH_P_TEB, 381 381 + EXTPROTO_VXLAN); 431 382 else 432 383 return TC_ACT_OK; 433 384 } ··· 515 444 return TC_ACT_OK; 516 445 } 517 446 447 447 + SEC("encap_ip6vxlan_eth") 448 448 + int __encap_ip6vxlan_eth(struct __sk_buff *skb) 449 449 + { 450 450 + if (skb->protocol == __bpf_constant_htons(ETH_P_IPV6)) 451 451 + return __encap_ipv6(skb, IPPROTO_UDP, 452 452 + ETH_P_TEB, 453 453 + EXTPROTO_VXLAN); 454 454 + else 455 455 + return TC_ACT_OK; 456 456 + } 457 457 + 518 458 static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) 519 459 { 520 460 char buf[sizeof(struct v6hdr)]; ··· 560 478 break; 561 479 case ETH_OVER_UDP_PORT: 562 480 olen += ETH_HLEN; 481 481 + break; 482 482 + case VXLAN_UDP_PORT: 483 483 + olen += ETH_HLEN + sizeof(struct vxlanhdr); 563 484 break; 564 485 } 565 486 break;

-21

tools/testing/selftests/bpf/test_bpftool_build.sh

reviewed

··· 85 85 echo 86 86 } 87 87 88 88 - make_doc_and_clean() { 89 89 - echo -e "\$PWD: $PWD" 90 90 - echo -e "command: make -s $* doc >/dev/null" 91 91 - RST2MAN_OPTS="--exit-status=1" make $J -s $* doc 92 92 - if [ $? -ne 0 ] ; then 93 93 - ERROR=1 94 94 - printf "FAILURE: Errors or warnings when building documentation\n" 95 95 - fi 96 96 - ( 97 97 - if [ $# -ge 1 ] ; then 98 98 - cd ${@: -1} 99 99 - fi 100 100 - make -s doc-clean 101 101 - ) 102 102 - echo 103 103 - } 104 104 - 105 88 echo "Trying to build bpftool" 106 89 echo -e "... through kbuild\n" 107 90 ··· 145 162 make_with_tmpdir OUTPUT 146 163 147 164 make_with_tmpdir O 148 148 - 149 149 - echo -e "Checking documentation build\n" 150 150 - # From tools/bpf/bpftool 151 151 - make_doc_and_clean

+3

tools/testing/selftests/bpf/test_btf.h

reviewed

··· 66 66 #define BTF_FUNC_ENC(name, func_proto) \ 67 67 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), func_proto) 68 68 69 69 + #define BTF_TYPE_FLOAT_ENC(name, sz) \ 70 70 + BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz) 71 71 + 69 72 #endif /* _TEST_BTF_H */

+13

tools/testing/selftests/bpf/test_doc_build.sh

reviewed

··· 1 1 + #!/bin/bash 2 2 + # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 3 3 + 4 4 + # Assume script is located under tools/testing/selftests/bpf/. We want to start 5 5 + # build attempts from the top of kernel repository. 6 6 + SCRIPT_REL_PATH=$(realpath --relative-to=$PWD $0) 7 7 + SCRIPT_REL_DIR=$(dirname $SCRIPT_REL_PATH) 8 8 + KDIR_ROOT_DIR=$(realpath $PWD/$SCRIPT_REL_DIR/../../../../) 9 9 + cd $KDIR_ROOT_DIR 10 10 + 11 11 + for tgt in docs docs-clean; do 12 12 + make -s -C $PWD/$SCRIPT_REL_DIR $tgt; 13 13 + done

+11

tools/testing/selftests/bpf/test_progs.h

reviewed

··· 152 152 ___ok; \ 153 153 }) 154 154 155 155 + #define ASSERT_LT(actual, expected, name) ({ \ 156 156 + static int duration = 0; \ 157 157 + typeof(actual) ___act = (actual); \ 158 158 + typeof(expected) ___exp = (expected); \ 159 159 + bool ___ok = ___act < ___exp; \ 160 160 + CHECK(!___ok, (name), \ 161 161 + "unexpected %s: actual %lld >= expected %lld\n", \ 162 162 + (name), (long long)(___act), (long long)(___exp)); \ 163 163 + ___ok; \ 164 164 + }) 165 165 + 155 166 #define ASSERT_STREQ(actual, expected, name) ({ \ 156 167 static int duration = 0; \ 157 168 const char *___act = actual; \

+1 -1

tools/testing/selftests/bpf/test_sockmap.c

reviewed

··· 732 732 * socket is not a valid test. So in this case lets not 733 733 * enable kTLS but still run the test. 734 734 */ 735 735 - if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) { 735 735 + if (!txmsg_redir || txmsg_ingress) { 736 736 err = sockmap_init_ktls(opt->verbose, rx_fd); 737 737 if (err) 738 738 return err;

+12 -3

tools/testing/selftests/bpf/test_tc_tunnel.sh

reviewed

··· 44 44 # clamp route to reserve room for tunnel headers 45 45 ip -netns "${ns1}" -4 route flush table main 46 46 ip -netns "${ns1}" -6 route flush table main 47 47 - ip -netns "${ns1}" -4 route add "${ns2_v4}" mtu 1458 dev veth1 48 48 - ip -netns "${ns1}" -6 route add "${ns2_v6}" mtu 1438 dev veth1 47 47 + ip -netns "${ns1}" -4 route add "${ns2_v4}" mtu 1450 dev veth1 48 48 + ip -netns "${ns1}" -6 route add "${ns2_v6}" mtu 1430 dev veth1 49 49 50 50 sleep 1 51 51 ··· 104 104 105 105 echo "sit" 106 106 $0 ipv6 sit none 100 107 107 + 108 108 + echo "ip4 vxlan" 109 109 + $0 ipv4 vxlan eth 2000 110 110 + 111 111 + echo "ip6 vxlan" 112 112 + $0 ipv6 ip6vxlan eth 2000 107 113 108 114 for mac in none mpls eth ; do 109 115 echo "ip gre $mac" ··· 220 214 targs="encap fou encap-sport auto encap-dport $dport" 221 215 elif [[ "$tuntype" =~ "gre" && "$mac" == "eth" ]]; then 222 216 ttype=$gretaptype 217 217 + elif [[ "$tuntype" =~ "vxlan" && "$mac" == "eth" ]]; then 218 218 + ttype="vxlan" 219 219 + targs="id 1 dstport 8472 udp6zerocsumrx" 223 220 else 224 221 ttype=$tuntype 225 222 targs="" ··· 251 242 elif [[ "$tuntype" =~ "udp" && "$mac" == "eth" ]]; then 252 243 # No support for TEB fou tunnel; expect failure. 253 244 expect_tun_fail=1 254 254 - elif [[ "$tuntype" =~ "gre" && "$mac" == "eth" ]]; then 245 245 + elif [[ "$tuntype" =~ (gre|vxlan) && "$mac" == "eth" ]]; then 255 246 # Share ethernet address between tunnel/veth2 so L2 decap works. 256 247 ethaddr=$(ip netns exec "${ns2}" ip link show veth2 | \ 257 248 awk '/ether/ { print $2 }')

+2 -2

tools/testing/selftests/bpf/test_verifier.c

reviewed

··· 105 105 enum bpf_prog_type prog_type; 106 106 uint8_t flags; 107 107 void (*fill_helper)(struct bpf_test *self); 108 108 - uint8_t runs; 108 108 + int runs; 109 109 #define bpf_testdata_struct_t \ 110 110 struct { \ 111 111 uint32_t retval, retval_unpriv; \ ··· 1165 1165 1166 1166 run_errs = 0; 1167 1167 run_successes = 0; 1168 1168 - if (!alignment_prevented_execution && fd_prog >= 0) { 1168 1168 + if (!alignment_prevented_execution && fd_prog >= 0 && test->runs >= 0) { 1169 1169 uint32_t expected_val; 1170 1170 int i; 1171 1171

+25 -110

tools/testing/selftests/bpf/test_xsk.sh

reviewed

··· 71 71 # 72 72 # Run (full output without color-coding): 73 73 # sudo ./test_xsk.sh 74 74 + # 75 75 + # Run with verbose output: 76 76 + # sudo ./test_xsk.sh -v 77 77 + # 78 78 + # Run and dump packet contents: 79 79 + # sudo ./test_xsk.sh -D 74 80 75 81 . xsk_prereqs.sh 76 82 77 77 - while getopts c flag 83 83 + while getopts "cvD" flag 78 84 do 79 85 case "${flag}" in 80 86 c) colorconsole=1;; 87 87 + v) verbose=1;; 88 88 + D) dump_pkts=1;; 81 89 esac 82 90 done 83 91 ··· 103 95 MTU=1500 104 96 105 97 setup_vethPairs() { 106 106 - echo "setting up ${VETH0}: namespace: ${NS0}" 98 98 + if [[ $verbose -eq 1 ]]; then 99 99 + echo "setting up ${VETH0}: namespace: ${NS0}" 100 100 + fi 107 101 ip netns add ${NS1} 108 102 ip link add ${VETH0} type veth peer name ${VETH1} 109 103 if [ -f /proc/net/if_inet6 ]; then 110 104 echo 1 > /proc/sys/net/ipv6/conf/${VETH0}/disable_ipv6 111 105 fi 112 112 - echo "setting up ${VETH1}: namespace: ${NS1}" 106 106 + if [[ $verbose -eq 1 ]]; then 107 107 + echo "setting up ${VETH1}: namespace: ${NS1}" 108 108 + fi 113 109 ip link set ${VETH1} netns ${NS1} 114 110 ip netns exec ${NS1} ip link set ${VETH1} mtu ${MTU} 115 111 ip link set ${VETH0} mtu ${MTU} ··· 137 125 138 126 validate_veth_spec_file 139 127 140 140 - echo "Spec file created: ${SPECFILE}" 128 128 + if [[ $verbose -eq 1 ]]; then 129 129 + echo "Spec file created: ${SPECFILE}" 130 130 + VERBOSE_ARG="-v" 131 131 + fi 132 132 + 133 133 + if [[ $dump_pkts -eq 1 ]]; then 134 134 + DUMP_PKTS_ARG="-D" 135 135 + fi 141 136 142 137 test_status $retval "${TEST_NAME}" 143 138 ··· 152 133 153 134 statusList=() 154 135 155 155 - ### TEST 1 156 156 - TEST_NAME="XSK KSELFTEST FRAMEWORK" 136 136 + TEST_NAME="XSK KSELFTESTS" 157 137 158 158 - echo "Switching interfaces [${VETH0}, ${VETH1}] to XDP Generic mode" 159 159 - vethXDPgeneric ${VETH0} ${VETH1} ${NS1} 160 160 - 161 161 - retval=$? 162 162 - if [ $retval -eq 0 ]; then 163 163 - echo "Switching interfaces [${VETH0}, ${VETH1}] to XDP Native mode" 164 164 - vethXDPnative ${VETH0} ${VETH1} ${NS1} 165 165 - fi 166 166 - 167 167 - retval=$? 168 168 - test_status $retval "${TEST_NAME}" 169 169 - statusList+=($retval) 170 170 - 171 171 - ### TEST 2 172 172 - TEST_NAME="SKB NOPOLL" 173 173 - 174 174 - vethXDPgeneric ${VETH0} ${VETH1} ${NS1} 175 175 - 176 176 - params=("-S") 177 177 - execxdpxceiver params 178 178 - 179 179 - retval=$? 180 180 - test_status $retval "${TEST_NAME}" 181 181 - statusList+=($retval) 182 182 - 183 183 - ### TEST 3 184 184 - TEST_NAME="SKB POLL" 185 185 - 186 186 - vethXDPgeneric ${VETH0} ${VETH1} ${NS1} 187 187 - 188 188 - params=("-S" "-p") 189 189 - execxdpxceiver params 190 190 - 191 191 - retval=$? 192 192 - test_status $retval "${TEST_NAME}" 193 193 - statusList+=($retval) 194 194 - 195 195 - ### TEST 4 196 196 - TEST_NAME="DRV NOPOLL" 197 197 - 198 198 - vethXDPnative ${VETH0} ${VETH1} ${NS1} 199 199 - 200 200 - params=("-N") 201 201 - execxdpxceiver params 202 202 - 203 203 - retval=$? 204 204 - test_status $retval "${TEST_NAME}" 205 205 - statusList+=($retval) 206 206 - 207 207 - ### TEST 5 208 208 - TEST_NAME="DRV POLL" 209 209 - 210 210 - vethXDPnative ${VETH0} ${VETH1} ${NS1} 211 211 - 212 212 - params=("-N" "-p") 213 213 - execxdpxceiver params 214 214 - 215 215 - retval=$? 216 216 - test_status $retval "${TEST_NAME}" 217 217 - statusList+=($retval) 218 218 - 219 219 - ### TEST 6 220 220 - TEST_NAME="SKB SOCKET TEARDOWN" 221 221 - 222 222 - vethXDPgeneric ${VETH0} ${VETH1} ${NS1} 223 223 - 224 224 - params=("-S" "-T") 225 225 - execxdpxceiver params 226 226 - 227 227 - retval=$? 228 228 - test_status $retval "${TEST_NAME}" 229 229 - statusList+=($retval) 230 230 - 231 231 - ### TEST 7 232 232 - TEST_NAME="DRV SOCKET TEARDOWN" 233 233 - 234 234 - vethXDPnative ${VETH0} ${VETH1} ${NS1} 235 235 - 236 236 - params=("-N" "-T") 237 237 - execxdpxceiver params 238 238 - 239 239 - retval=$? 240 240 - test_status $retval "${TEST_NAME}" 241 241 - statusList+=($retval) 242 242 - 243 243 - ### TEST 8 244 244 - TEST_NAME="SKB BIDIRECTIONAL SOCKETS" 245 245 - 246 246 - vethXDPgeneric ${VETH0} ${VETH1} ${NS1} 247 247 - 248 248 - params=("-S" "-B") 249 249 - execxdpxceiver params 250 250 - 251 251 - retval=$? 252 252 - test_status $retval "${TEST_NAME}" 253 253 - statusList+=($retval) 254 254 - 255 255 - ### TEST 9 256 256 - TEST_NAME="DRV BIDIRECTIONAL SOCKETS" 257 257 - 258 258 - vethXDPnative ${VETH0} ${VETH1} ${NS1} 259 259 - 260 260 - params=("-N" "-B") 261 261 - execxdpxceiver params 138 138 + execxdpxceiver 262 139 263 140 retval=$? 264 141 test_status $retval "${TEST_NAME}"

+1

tools/testing/selftests/bpf/verifier/ctx_sk_lookup.c

reviewed

··· 239 239 .result = ACCEPT, 240 240 .prog_type = BPF_PROG_TYPE_SK_LOOKUP, 241 241 .expected_attach_type = BPF_SK_LOOKUP, 242 242 + .runs = -1, 242 243 }, 243 244 /* invalid 8-byte reads from a 4-byte fields in bpf_sk_lookup */ 244 245 {

+19 -7

tools/testing/selftests/bpf/vmtest.sh

reviewed

··· 17 17 KCONFIG_API_URL="https://api.github.com/repos/libbpf/libbpf/contents/travis-ci/vmtest/configs/latest.config" 18 18 INDEX_URL="https://raw.githubusercontent.com/libbpf/libbpf/master/travis-ci/vmtest/configs/INDEX" 19 19 NUM_COMPILE_JOBS="$(nproc)" 20 20 + LOG_FILE_BASE="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S")" 21 21 + LOG_FILE="${LOG_FILE_BASE}.log" 22 22 + EXIT_STATUS_FILE="${LOG_FILE_BASE}.exit_status" 20 23 21 24 usage() 22 25 { ··· 149 146 local init_script_dir="${OUTPUT_DIR}/${MOUNT_DIR}/etc/rcS.d" 150 147 local init_script="${init_script_dir}/S50-startup" 151 148 local command="$1" 152 152 - local log_file="$2" 153 149 154 150 mount_image 155 151 ··· 165 163 sudo bash -c "cat >${init_script}" <<EOF 166 164 #!/bin/bash 167 165 166 166 + # Have a default value in the exit status file 167 167 + # incase the VM is forcefully stopped. 168 168 + echo "130" > "/root/${EXIT_STATUS_FILE}" 169 169 + 168 170 { 169 171 cd /root/bpf 170 172 echo ${command} 171 173 stdbuf -oL -eL ${command} 172 172 - } 2>&1 | tee /root/${log_file} 174 174 + echo "\$?" > "/root/${EXIT_STATUS_FILE}" 175 175 + } 2>&1 | tee "/root/${LOG_FILE}" 173 176 poweroff -f 174 177 EOF 175 178 ··· 228 221 copy_logs() 229 222 { 230 223 local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}" 231 231 - local log_file="${mount_dir}/root/$1" 224 224 + local log_file="${mount_dir}/root/${LOG_FILE}" 225 225 + local exit_status_file="${mount_dir}/root/${EXIT_STATUS_FILE}" 232 226 233 227 mount_image 234 228 sudo cp ${log_file} "${OUTPUT_DIR}" 229 229 + sudo cp ${exit_status_file} "${OUTPUT_DIR}" 235 230 sudo rm -f ${log_file} 236 231 unmount_image 237 232 } ··· 272 263 { 273 264 local script_dir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" 274 265 local kernel_checkout=$(realpath "${script_dir}"/../../../../) 275 275 - local log_file="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S.log")" 276 266 # By default the script searches for the kernel in the checkout directory but 277 267 # it also obeys environment variables O= and KBUILD_OUTPUT= 278 268 local kernel_bzimage="${kernel_checkout}/${X86_BZIMAGE}" ··· 355 347 fi 356 348 357 349 update_selftests "${kernel_checkout}" "${make_command}" 358 358 - update_init_script "${command}" "${log_file}" 350 350 + update_init_script "${command}" 359 351 run_vm "${kernel_bzimage}" 360 360 - copy_logs "${log_file}" 361 361 - echo "Logs saved in ${OUTPUT_DIR}/${log_file}" 352 352 + copy_logs 353 353 + echo "Logs saved in ${OUTPUT_DIR}/${LOG_FILE}" 362 354 } 363 355 364 356 catch() 365 357 { 366 358 local exit_code=$1 359 359 + local exit_status_file="${OUTPUT_DIR}/${EXIT_STATUS_FILE}" 367 360 # This is just a cleanup and the directory may 368 361 # have already been unmounted. So, don't let this 369 362 # clobber the error code we intend to return. 370 363 unmount_image || true 364 364 + if [[ -f "${exit_status_file}" ]]; then 365 365 + exit_code="$(cat ${exit_status_file})" 366 366 + fi 371 367 exit ${exit_code} 372 368 } 373 369

+272 -108

tools/testing/selftests/bpf/xdpxceiver.c

reviewed

··· 18 18 * These selftests test AF_XDP SKB and Native/DRV modes using veth 19 19 * Virtual Ethernet interfaces. 20 20 * 21 21 - * The following tests are run: 22 22 - * 23 23 - * 1. AF_XDP SKB mode 24 24 - * Generic mode XDP is driver independent, used when the driver does 25 25 - * not have support for XDP. Works on any netdevice using sockets and 26 26 - * generic XDP path. XDP hook from netif_receive_skb(). 21 21 + * For each mode, the following tests are run: 27 22 * a. nopoll - soft-irq processing 28 23 * b. poll - using poll() syscall 29 24 * c. Socket Teardown ··· 28 33 * Configure sockets as bi-directional tx/rx sockets, sets up fill and 29 34 * completion rings on each socket, tx/rx in both directions. Only nopoll 30 35 * mode is used 36 36 + * e. Statistics 37 37 + * Trigger some error conditions and ensure that the appropriate statistics 38 38 + * are incremented. Within this test, the following statistics are tested: 39 39 + * i. rx dropped 40 40 + * Increase the UMEM frame headroom to a value which results in 41 41 + * insufficient space in the rx buffer for both the packet and the headroom. 42 42 + * ii. tx invalid 43 43 + * Set the 'len' field of tx descriptors to an invalid value (umem frame 44 44 + * size + 1). 45 45 + * iii. rx ring full 46 46 + * Reduce the size of the RX ring to a fraction of the fill ring size. 47 47 + * iv. fill queue empty 48 48 + * Do not populate the fill queue and then try to receive pkts. 31 49 * 32 32 - * 2. AF_XDP DRV/Native mode 33 33 - * Works on any netdevice with XDP_REDIRECT support, driver dependent. Processes 34 34 - * packets before SKB allocation. Provides better performance than SKB. Driver 35 35 - * hook available just after DMA of buffer descriptor. 36 36 - * a. nopoll 37 37 - * b. poll 38 38 - * c. Socket Teardown 39 39 - * d. Bi-directional sockets 40 40 - * - Only copy mode is supported because veth does not currently support 41 41 - * zero-copy mode 42 42 - * 43 43 - * Total tests: 8 50 50 + * Total tests: 10 44 51 * 45 52 * Flow: 46 53 * ----- ··· 55 58 * - Rx thread verifies if all 10k packets were received and delivered in-order, 56 59 * and have the right content 57 60 * 58 58 - * Enable/disable debug mode: 61 61 + * Enable/disable packet dump mode: 59 62 * -------------------------- 60 63 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add 61 64 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D") ··· 95 98 96 99 static void __exit_with_error(int error, const char *file, const char *func, int line) 97 100 { 98 98 - ksft_test_result_fail 99 99 - ("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, strerror(error)); 100 100 - ksft_exit_xfail(); 101 101 + if (configured_mode == TEST_MODE_UNCONFIGURED) { 102 102 + ksft_exit_fail_msg 103 103 + ("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, strerror(error)); 104 104 + } else { 105 105 + ksft_test_result_fail 106 106 + ("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, strerror(error)); 107 107 + ksft_exit_xfail(); 108 108 + } 101 109 } 102 110 103 111 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) 104 112 105 113 #define print_ksft_result(void)\ 106 106 - (ksft_test_result_pass("PASS: %s %s %s%s\n", uut ? "DRV" : "SKB", opt_poll ? "POLL" :\ 107 107 - "NOPOLL", opt_teardown ? "Socket Teardown" : "",\ 108 108 - opt_bidi ? "Bi-directional Sockets" : "")) 114 114 + (ksft_test_result_pass("PASS: %s %s %s%s%s\n", configured_mode ? "DRV" : "SKB",\ 115 115 + test_type == TEST_TYPE_POLL ? "POLL" : "NOPOLL",\ 116 116 + test_type == TEST_TYPE_TEARDOWN ? "Socket Teardown" : "",\ 117 117 + test_type == TEST_TYPE_BIDI ? "Bi-directional Sockets" : "",\ 118 118 + test_type == TEST_TYPE_STATS ? "Stats" : "")) 109 119 110 120 static void pthread_init_mutex(void) 111 121 { ··· 274 270 static void xsk_configure_umem(struct ifobject *data, void *buffer, u64 size) 275 271 { 276 272 int ret; 273 273 + struct xsk_umem_config cfg = { 274 274 + .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, 275 275 + .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, 276 276 + .frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE, 277 277 + .frame_headroom = frame_headroom, 278 278 + .flags = XSK_UMEM__DEFAULT_FLAGS 279 279 + }; 277 280 278 281 data->umem = calloc(1, sizeof(struct xsk_umem_info)); 279 282 if (!data->umem) 280 283 exit_with_error(errno); 281 284 282 285 ret = xsk_umem__create(&data->umem->umem, buffer, size, 283 283 - &data->umem->fq, &data->umem->cq, NULL); 286 286 + &data->umem->fq, &data->umem->cq, &cfg); 284 287 if (ret) 285 288 exit_with_error(ret); 286 289 ··· 319 308 exit_with_error(errno); 320 309 321 310 ifobject->xsk->umem = ifobject->umem; 322 322 - cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS; 311 311 + cfg.rx_size = rxqsize; 323 312 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; 324 313 cfg.libbpf_flags = 0; 325 325 - cfg.xdp_flags = opt_xdp_flags; 326 326 - cfg.bind_flags = opt_xdp_bind_flags; 314 314 + cfg.xdp_flags = xdp_flags; 315 315 + cfg.bind_flags = xdp_bind_flags; 327 316 328 328 - if (!opt_bidi) { 317 317 + if (test_type != TEST_TYPE_BIDI) { 329 318 rxr = (ifobject->fv.vector == rx) ? &ifobject->xsk->rx : NULL; 330 319 txr = (ifobject->fv.vector == tx) ? &ifobject->xsk->tx : NULL; 331 320 } else { ··· 345 334 static struct option long_options[] = { 346 335 {"interface", required_argument, 0, 'i'}, 347 336 {"queue", optional_argument, 0, 'q'}, 348 348 - {"poll", no_argument, 0, 'p'}, 349 349 - {"xdp-skb", no_argument, 0, 'S'}, 350 350 - {"xdp-native", no_argument, 0, 'N'}, 351 351 - {"copy", no_argument, 0, 'c'}, 352 352 - {"tear-down", no_argument, 0, 'T'}, 353 353 - {"bidi", optional_argument, 0, 'B'}, 354 354 - {"debug", optional_argument, 0, 'D'}, 337 337 + {"dump-pkts", optional_argument, 0, 'D'}, 338 338 + {"verbose", no_argument, 0, 'v'}, 355 339 {"tx-pkt-count", optional_argument, 0, 'C'}, 356 340 {0, 0, 0, 0} 357 341 }; ··· 358 352 " Options:\n" 359 353 " -i, --interface Use interface\n" 360 354 " -q, --queue=n Use queue n (default 0)\n" 361 361 - " -p, --poll Use poll syscall\n" 362 362 - " -S, --xdp-skb=n Use XDP SKB mode\n" 363 363 - " -N, --xdp-native=n Enforce XDP DRV (native) mode\n" 364 364 - " -c, --copy Force copy mode\n" 365 365 - " -T, --tear-down Tear down sockets by repeatedly recreating them\n" 366 366 - " -B, --bidi Bi-directional sockets test\n" 367 367 - " -D, --debug Debug mode - dump packets L2 - L5\n" 355 355 + " -D, --dump-pkts Dump packets L2 - L5\n" 356 356 + " -v, --verbose Verbose output\n" 368 357 " -C, --tx-pkt-count=n Number of packets to send\n"; 369 358 ksft_print_msg(str, prog); 370 359 } ··· 393 392 ksft_test_result_fail("ERROR: [%s] interface \"%s\" does not exist\n", 394 393 __func__, ifdict[targs->idx]->ifname); 395 394 } else { 396 396 - ksft_print_msg("Interface found: %s\n", ifdict[targs->idx]->ifname); 395 395 + print_verbose("Interface found: %s\n", ifdict[targs->idx]->ifname); 397 396 targs->retptr = true; 398 397 } 399 398 } ··· 423 422 pthread_join(ns_thread, NULL); 424 423 425 424 if (targs->retptr) 426 426 - ksft_print_msg("NS switched: %s\n", ifdict[i]->nsname); 425 425 + print_verbose("NS switched: %s\n", ifdict[i]->nsname); 427 426 428 427 free(targs); 429 428 } else { ··· 433 432 ("ERROR: interface \"%s\" does not exist\n", ifdict[i]->ifname); 434 433 ret = false; 435 434 } else { 436 436 - ksft_print_msg("Interface found: %s\n", ifdict[i]->ifname); 435 435 + print_verbose("Interface found: %s\n", ifdict[i]->ifname); 437 436 } 438 437 } 439 438 } ··· 447 446 opterr = 0; 448 447 449 448 for (;;) { 450 450 - c = getopt_long(argc, argv, "i:q:pSNcTBDC:", long_options, &option_index); 449 449 + c = getopt_long(argc, argv, "i:q:DC:v", long_options, &option_index); 451 450 452 451 if (c == -1) 453 452 break; ··· 470 469 case 'q': 471 470 opt_queue = atoi(optarg); 472 471 break; 473 473 - case 'p': 474 474 - opt_poll = 1; 475 475 - break; 476 476 - case 'S': 477 477 - opt_xdp_flags |= XDP_FLAGS_SKB_MODE; 478 478 - opt_xdp_bind_flags |= XDP_COPY; 479 479 - uut = ORDER_CONTENT_VALIDATE_XDP_SKB; 480 480 - break; 481 481 - case 'N': 482 482 - opt_xdp_flags |= XDP_FLAGS_DRV_MODE; 483 483 - opt_xdp_bind_flags |= XDP_COPY; 484 484 - uut = ORDER_CONTENT_VALIDATE_XDP_DRV; 485 485 - break; 486 486 - case 'c': 487 487 - opt_xdp_bind_flags |= XDP_COPY; 488 488 - break; 489 489 - case 'T': 490 490 - opt_teardown = 1; 491 491 - break; 492 492 - case 'B': 493 493 - opt_bidi = 1; 494 494 - break; 495 472 case 'D': 496 473 debug_pkt_dump = 1; 497 474 break; 498 475 case 'C': 499 476 opt_pkt_count = atoi(optarg); 500 477 break; 478 478 + case 'v': 479 479 + opt_verbose = 1; 480 480 + break; 501 481 default: 502 482 usage(basename(argv[0])); 503 483 ksft_exit_xfail(); 504 484 } 485 485 + } 486 486 + 487 487 + if (!opt_pkt_count) { 488 488 + print_verbose("No tx-pkt-count specified, using default %u\n", DEFAULT_PKT_CNT); 489 489 + opt_pkt_count = DEFAULT_PKT_CNT; 505 490 } 506 491 507 492 if (!validate_interfaces()) { ··· 586 599 { 587 600 u32 idx; 588 601 unsigned int i; 602 602 + bool tx_invalid_test = stat_test_type == STAT_TEST_TX_INVALID; 603 603 + u32 len = tx_invalid_test ? XSK_UMEM__DEFAULT_FRAME_SIZE + 1 : PKT_SIZE; 589 604 590 605 while (xsk_ring_prod__reserve(&xsk->tx, batch_size, &idx) < batch_size) 591 606 complete_tx_only(xsk, batch_size); ··· 596 607 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); 597 608 598 609 tx_desc->addr = (*frameptr + i) << XSK_UMEM__DEFAULT_FRAME_SHIFT; 599 599 - tx_desc->len = PKT_SIZE; 610 610 + tx_desc->len = len; 600 611 } 601 612 602 613 xsk_ring_prod__submit(&xsk->tx, batch_size); 603 603 - xsk->outstanding_tx += batch_size; 614 614 + if (!tx_invalid_test) { 615 615 + xsk->outstanding_tx += batch_size; 616 616 + } else { 617 617 + if (!NEED_WAKEUP || xsk_ring_prod__needs_wakeup(&xsk->tx)) 618 618 + kick_tx(xsk); 619 619 + } 604 620 *frameptr += batch_size; 605 621 *frameptr %= num_frames; 606 622 complete_tx_only(xsk, batch_size); ··· 648 654 while ((opt_pkt_count && pkt_cnt < opt_pkt_count) || !opt_pkt_count) { 649 655 int batch_size = get_batch_size(pkt_cnt); 650 656 651 651 - if (opt_poll) { 657 657 + if (test_type == TEST_TYPE_POLL) { 652 658 ret = poll(fds, 1, POLL_TMOUT); 653 659 if (ret <= 0) 654 660 continue; ··· 708 714 int payload = *((uint32_t *)(pkt_buf[iter]->payload + PKT_HDR_SIZE)); 709 715 710 716 if (payload == EOT) { 711 711 - ksft_print_msg("End-of-transmission frame received\n"); 717 717 + print_verbose("End-of-transmission frame received\n"); 712 718 fprintf(stdout, "---------------------------------------\n"); 713 719 break; 714 720 } 715 721 fprintf(stdout, "DEBUG>> L5: payload: %d\n", payload); 716 722 fprintf(stdout, "---------------------------------------\n"); 723 723 + } 724 724 + } 725 725 + 726 726 + static void worker_stats_validate(struct ifobject *ifobject) 727 727 + { 728 728 + struct xdp_statistics stats; 729 729 + socklen_t optlen; 730 730 + int err; 731 731 + struct xsk_socket *xsk = stat_test_type == STAT_TEST_TX_INVALID ? 732 732 + ifdict[!ifobject->ifdict_index]->xsk->xsk : 733 733 + ifobject->xsk->xsk; 734 734 + int fd = xsk_socket__fd(xsk); 735 735 + unsigned long xsk_stat = 0, expected_stat = opt_pkt_count; 736 736 + 737 737 + sigvar = 0; 738 738 + 739 739 + optlen = sizeof(stats); 740 740 + err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); 741 741 + if (err) 742 742 + return; 743 743 + 744 744 + if (optlen == sizeof(struct xdp_statistics)) { 745 745 + switch (stat_test_type) { 746 746 + case STAT_TEST_RX_DROPPED: 747 747 + xsk_stat = stats.rx_dropped; 748 748 + break; 749 749 + case STAT_TEST_TX_INVALID: 750 750 + xsk_stat = stats.tx_invalid_descs; 751 751 + break; 752 752 + case STAT_TEST_RX_FULL: 753 753 + xsk_stat = stats.rx_ring_full; 754 754 + expected_stat -= RX_FULL_RXQSIZE; 755 755 + break; 756 756 + case STAT_TEST_RX_FILL_EMPTY: 757 757 + xsk_stat = stats.rx_fill_ring_empty_descs; 758 758 + break; 759 759 + default: 760 760 + break; 761 761 + } 762 762 + 763 763 + if (xsk_stat == expected_stat) 764 764 + sigvar = 1; 717 765 } 718 766 } 719 767 ··· 782 746 } 783 747 784 748 if (payloadseqnum == EOT) { 785 785 - ksft_print_msg("End-of-transmission frame received: PASS\n"); 749 749 + print_verbose("End-of-transmission frame received: PASS\n"); 786 750 sigvar = 1; 787 751 break; 788 752 } ··· 872 836 usleep(USLEEP_MAX); 873 837 } 874 838 875 875 - ksft_print_msg("Interface [%s] vector [Tx]\n", ifobject->ifname); 839 839 + print_verbose("Interface [%s] vector [Tx]\n", ifobject->ifname); 876 840 for (int i = 0; i < num_frames; i++) { 877 841 /*send EOT frame */ 878 842 if (i == (num_frames - 1)) ··· 886 850 gen_eth_frame(ifobject->umem, i * XSK_UMEM__DEFAULT_FRAME_SIZE); 887 851 } 888 852 889 889 - ksft_print_msg("Sending %d packets on interface %s\n", 853 853 + print_verbose("Sending %d packets on interface %s\n", 890 854 (opt_pkt_count - 1), ifobject->ifname); 891 855 tx_only_all(ifobject); 892 856 } else if (ifobject->fv.vector == rx) { ··· 896 860 if (!bidi_pass) 897 861 thread_common_ops(ifobject, bufs, &sync_mutex_tx, &spinning_rx); 898 862 899 899 - ksft_print_msg("Interface [%s] vector [Rx]\n", ifobject->ifname); 900 900 - xsk_populate_fill_ring(ifobject->umem); 863 863 + print_verbose("Interface [%s] vector [Rx]\n", ifobject->ifname); 864 864 + if (stat_test_type != STAT_TEST_RX_FILL_EMPTY) 865 865 + xsk_populate_fill_ring(ifobject->umem); 901 866 902 867 TAILQ_INIT(&head); 903 868 if (debug_pkt_dump) { ··· 915 878 pthread_mutex_unlock(&sync_mutex); 916 879 917 880 while (1) { 918 918 - if (opt_poll) { 881 881 + if (test_type == TEST_TYPE_POLL) { 919 882 ret = poll(fds, 1, POLL_TMOUT); 920 883 if (ret <= 0) 921 884 continue; 922 885 } 923 923 - rx_pkt(ifobject->xsk, fds); 924 924 - worker_pkt_validate(); 886 886 + 887 887 + if (test_type != TEST_TYPE_STATS) { 888 888 + rx_pkt(ifobject->xsk, fds); 889 889 + worker_pkt_validate(); 890 890 + } else { 891 891 + worker_stats_validate(ifobject); 892 892 + } 925 893 926 894 if (sigvar) 927 895 break; 928 896 } 929 897 930 930 - ksft_print_msg("Received %d packets on interface %s\n", 931 931 - pkt_counter, ifobject->ifname); 898 898 + if (test_type != TEST_TYPE_STATS) 899 899 + print_verbose("Received %d packets on interface %s\n", 900 900 + pkt_counter, ifobject->ifname); 932 901 933 933 - if (opt_teardown) 934 934 - ksft_print_msg("Destroying socket\n"); 902 902 + if (test_type == TEST_TYPE_TEARDOWN) 903 903 + print_verbose("Destroying socket\n"); 935 904 } 936 905 937 937 - if (!opt_bidi || bidi_pass) { 906 906 + if ((test_type != TEST_TYPE_BIDI) || bidi_pass) { 938 907 xsk_socket__delete(ifobject->xsk->xsk); 939 908 (void)xsk_umem__delete(ifobject->umem->umem); 940 909 } ··· 950 907 static void testapp_validate(void) 951 908 { 952 909 struct timespec max_wait = { 0, 0 }; 910 910 + bool bidi = test_type == TEST_TYPE_BIDI; 953 911 954 912 pthread_attr_init(&attr); 955 913 pthread_attr_setstacksize(&attr, THREAD_STACK); 956 914 957 957 - if (opt_bidi && bidi_pass) { 915 915 + if ((test_type == TEST_TYPE_BIDI) && bidi_pass) { 958 916 pthread_init_mutex(); 959 917 if (!switching_notify) { 960 960 - ksft_print_msg("Switching Tx/Rx vectors\n"); 918 918 + print_verbose("Switching Tx/Rx vectors\n"); 961 919 switching_notify++; 962 920 } 963 921 } ··· 966 922 pthread_mutex_lock(&sync_mutex); 967 923 968 924 /*Spawn RX thread */ 969 969 - if (!opt_bidi || !bidi_pass) { 925 925 + if (!bidi || !bidi_pass) { 970 926 if (pthread_create(&t0, &attr, worker_testapp_validate, ifdict[1])) 971 927 exit_with_error(errno); 972 972 - } else if (opt_bidi && bidi_pass) { 928 928 + } else if (bidi && bidi_pass) { 973 929 /*switch Tx/Rx vectors */ 974 930 ifdict[0]->fv.vector = rx; 975 931 if (pthread_create(&t0, &attr, worker_testapp_validate, ifdict[0])) ··· 986 942 pthread_mutex_unlock(&sync_mutex); 987 943 988 944 /*Spawn TX thread */ 989 989 - if (!opt_bidi || !bidi_pass) { 945 945 + if (!bidi || !bidi_pass) { 990 946 if (pthread_create(&t1, &attr, worker_testapp_validate, ifdict[0])) 991 947 exit_with_error(errno); 992 992 - } else if (opt_bidi && bidi_pass) { 948 948 + } else if (bidi && bidi_pass) { 993 949 /*switch Tx/Rx vectors */ 994 950 ifdict[1]->fv.vector = tx; 995 951 if (pthread_create(&t1, &attr, worker_testapp_validate, ifdict[1])) ··· 1008 964 free(pkt_buf); 1009 965 } 1010 966 1011 1011 - if (!opt_teardown && !opt_bidi) 967 967 + if (!(test_type == TEST_TYPE_TEARDOWN) && !bidi && !(test_type == TEST_TYPE_STATS)) 1012 968 print_ksft_result(); 1013 969 } 1014 970 1015 971 static void testapp_sockets(void) 1016 972 { 1017 1017 - for (int i = 0; i < (opt_teardown ? MAX_TEARDOWN_ITER : MAX_BIDI_ITER); i++) { 973 973 + for (int i = 0; i < ((test_type == TEST_TYPE_TEARDOWN) ? MAX_TEARDOWN_ITER : MAX_BIDI_ITER); 974 974 + i++) { 1018 975 pkt_counter = 0; 1019 976 prev_pkt = -1; 1020 977 sigvar = 0; 1021 1021 - ksft_print_msg("Creating socket\n"); 978 978 + print_verbose("Creating socket\n"); 1022 979 testapp_validate(); 1023 1023 - opt_bidi ? bidi_pass++ : bidi_pass; 980 980 + test_type == TEST_TYPE_BIDI ? bidi_pass++ : bidi_pass; 981 981 + } 982 982 + 983 983 + print_ksft_result(); 984 984 + } 985 985 + 986 986 + static void testapp_stats(void) 987 987 + { 988 988 + for (int i = 0; i < STAT_TEST_TYPE_MAX; i++) { 989 989 + stat_test_type = i; 990 990 + 991 991 + /* reset defaults */ 992 992 + rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 993 993 + frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM; 994 994 + 995 995 + switch (stat_test_type) { 996 996 + case STAT_TEST_RX_DROPPED: 997 997 + frame_headroom = XSK_UMEM__DEFAULT_FRAME_SIZE - 998 998 + XDP_PACKET_HEADROOM - 1; 999 999 + break; 1000 1000 + case STAT_TEST_RX_FULL: 1001 1001 + rxqsize = RX_FULL_RXQSIZE; 1002 1002 + break; 1003 1003 + default: 1004 1004 + break; 1005 1005 + } 1006 1006 + testapp_validate(); 1024 1007 } 1025 1008 1026 1009 print_ksft_result(); ··· 1074 1003 ifdict[1]->src_port = ifaceconfig->dst_port; 1075 1004 } 1076 1005 1006 1006 + static void *nsdisablemodethread(void *args) 1007 1007 + { 1008 1008 + struct targs *targs = args; 1009 1009 + 1010 1010 + targs->retptr = false; 1011 1011 + 1012 1012 + if (switch_namespace(targs->idx)) { 1013 1013 + targs->retptr = bpf_set_link_xdp_fd(ifdict[targs->idx]->ifindex, -1, targs->flags); 1014 1014 + } else { 1015 1015 + targs->retptr = errno; 1016 1016 + print_verbose("Failed to switch namespace to %s\n", ifdict[targs->idx]->nsname); 1017 1017 + } 1018 1018 + 1019 1019 + pthread_exit(NULL); 1020 1020 + } 1021 1021 + 1022 1022 + static void disable_xdp_mode(int mode) 1023 1023 + { 1024 1024 + int err = 0; 1025 1025 + __u32 flags = XDP_FLAGS_UPDATE_IF_NOEXIST | mode; 1026 1026 + char *mode_str = mode & XDP_FLAGS_SKB_MODE ? "skb" : "drv"; 1027 1027 + 1028 1028 + for (int i = 0; i < MAX_INTERFACES; i++) { 1029 1029 + if (strcmp(ifdict[i]->nsname, "")) { 1030 1030 + struct targs *targs; 1031 1031 + 1032 1032 + targs = malloc(sizeof(*targs)); 1033 1033 + memset(targs, 0, sizeof(*targs)); 1034 1034 + if (!targs) 1035 1035 + exit_with_error(errno); 1036 1036 + 1037 1037 + targs->idx = i; 1038 1038 + targs->flags = flags; 1039 1039 + if (pthread_create(&ns_thread, NULL, nsdisablemodethread, targs)) 1040 1040 + exit_with_error(errno); 1041 1041 + 1042 1042 + pthread_join(ns_thread, NULL); 1043 1043 + err = targs->retptr; 1044 1044 + free(targs); 1045 1045 + } else { 1046 1046 + err = bpf_set_link_xdp_fd(ifdict[i]->ifindex, -1, flags); 1047 1047 + } 1048 1048 + 1049 1049 + if (err) { 1050 1050 + print_verbose("Failed to disable %s mode on interface %s\n", 1051 1051 + mode_str, ifdict[i]->ifname); 1052 1052 + exit_with_error(err); 1053 1053 + } 1054 1054 + 1055 1055 + print_verbose("Disabled %s mode for interface: %s\n", mode_str, ifdict[i]->ifname); 1056 1056 + configured_mode = mode & XDP_FLAGS_SKB_MODE ? TEST_MODE_DRV : TEST_MODE_SKB; 1057 1057 + } 1058 1058 + } 1059 1059 + 1060 1060 + static void run_pkt_test(int mode, int type) 1061 1061 + { 1062 1062 + test_type = type; 1063 1063 + 1064 1064 + /* reset defaults after potential previous test */ 1065 1065 + xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; 1066 1066 + pkt_counter = 0; 1067 1067 + switching_notify = 0; 1068 1068 + bidi_pass = 0; 1069 1069 + prev_pkt = -1; 1070 1070 + ifdict[0]->fv.vector = tx; 1071 1071 + ifdict[1]->fv.vector = rx; 1072 1072 + sigvar = 0; 1073 1073 + stat_test_type = -1; 1074 1074 + rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 1075 1075 + frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM; 1076 1076 + 1077 1077 + switch (mode) { 1078 1078 + case (TEST_MODE_SKB): 1079 1079 + if (configured_mode == TEST_MODE_DRV) 1080 1080 + disable_xdp_mode(XDP_FLAGS_DRV_MODE); 1081 1081 + xdp_flags |= XDP_FLAGS_SKB_MODE; 1082 1082 + break; 1083 1083 + case (TEST_MODE_DRV): 1084 1084 + if (configured_mode == TEST_MODE_SKB) 1085 1085 + disable_xdp_mode(XDP_FLAGS_SKB_MODE); 1086 1086 + xdp_flags |= XDP_FLAGS_DRV_MODE; 1087 1087 + break; 1088 1088 + default: 1089 1089 + break; 1090 1090 + } 1091 1091 + 1092 1092 + pthread_init_mutex(); 1093 1093 + 1094 1094 + if (test_type == TEST_TYPE_STATS) 1095 1095 + testapp_stats(); 1096 1096 + else if ((test_type != TEST_TYPE_TEARDOWN) && (test_type != TEST_TYPE_BIDI)) 1097 1097 + testapp_validate(); 1098 1098 + else 1099 1099 + testapp_sockets(); 1100 1100 + 1101 1101 + pthread_destroy_mutex(); 1102 1102 + } 1103 1103 + 1077 1104 int main(int argc, char **argv) 1078 1105 { 1079 1106 struct rlimit _rlim = { RLIM_INFINITY, RLIM_INFINITY }; ··· 1185 1016 const char *IP2 = "192.168.100.161"; 1186 1017 u16 UDP_DST_PORT = 2020; 1187 1018 u16 UDP_SRC_PORT = 2121; 1019 1019 + int i, j; 1188 1020 1189 1021 ifaceconfig = malloc(sizeof(struct ifaceconfigobj)); 1190 1022 memcpy(ifaceconfig->dst_mac, MAC1, ETH_ALEN); ··· 1211 1041 1212 1042 init_iface_config(ifaceconfig); 1213 1043 1214 1214 - pthread_init_mutex(); 1044 1044 + disable_xdp_mode(XDP_FLAGS_DRV_MODE); 1215 1045 1216 1216 - ksft_set_plan(1); 1046 1046 + ksft_set_plan(TEST_MODE_MAX * TEST_TYPE_MAX); 1217 1047 1218 1218 - if (!opt_teardown && !opt_bidi) { 1219 1219 - testapp_validate(); 1220 1220 - } else if (opt_teardown && opt_bidi) { 1221 1221 - ksft_test_result_fail("ERROR: parameters -T and -B cannot be used together\n"); 1222 1222 - ksft_exit_xfail(); 1223 1223 - } else { 1224 1224 - testapp_sockets(); 1048 1048 + for (i = 0; i < TEST_MODE_MAX; i++) { 1049 1049 + for (j = 0; j < TEST_TYPE_MAX; j++) 1050 1050 + run_pkt_test(i, j); 1225 1051 } 1226 1052 1227 1053 for (int i = 0; i < MAX_INTERFACES; i++) 1228 1054 free(ifdict[i]); 1229 1229 - 1230 1230 - pthread_destroy_mutex(); 1231 1055 1232 1056 ksft_exit_pass(); 1233 1057

+42 -15

tools/testing/selftests/bpf/xdpxceiver.h

reviewed

··· 41 41 #define BATCH_SIZE 64 42 42 #define POLL_TMOUT 1000 43 43 #define NEED_WAKEUP true 44 44 + #define DEFAULT_PKT_CNT 10000 45 45 + #define RX_FULL_RXQSIZE 32 46 46 + 47 47 + #define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0) 44 48 45 49 typedef __u32 u32; 46 50 typedef __u16 u16; 47 51 typedef __u8 u8; 48 52 49 49 - enum TESTS { 50 50 - ORDER_CONTENT_VALIDATE_XDP_SKB = 0, 51 51 - ORDER_CONTENT_VALIDATE_XDP_DRV = 1, 53 53 + enum TEST_MODES { 54 54 + TEST_MODE_UNCONFIGURED = -1, 55 55 + TEST_MODE_SKB, 56 56 + TEST_MODE_DRV, 57 57 + TEST_MODE_MAX 52 58 }; 53 59 54 54 - u8 uut; 55 55 - u8 debug_pkt_dump; 56 56 - u32 num_frames; 57 57 - u8 switching_notify; 58 58 - u8 bidi_pass; 60 60 + enum TEST_TYPES { 61 61 + TEST_TYPE_NOPOLL, 62 62 + TEST_TYPE_POLL, 63 63 + TEST_TYPE_TEARDOWN, 64 64 + TEST_TYPE_BIDI, 65 65 + TEST_TYPE_STATS, 66 66 + TEST_TYPE_MAX 67 67 + }; 59 68 60 60 - static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; 69 69 + enum STAT_TEST_TYPES { 70 70 + STAT_TEST_RX_DROPPED, 71 71 + STAT_TEST_TX_INVALID, 72 72 + STAT_TEST_RX_FULL, 73 73 + STAT_TEST_RX_FILL_EMPTY, 74 74 + STAT_TEST_TYPE_MAX 75 75 + }; 76 76 + 77 77 + static int configured_mode = TEST_MODE_UNCONFIGURED; 78 78 + static u8 debug_pkt_dump; 79 79 + static u32 num_frames; 80 80 + static u8 switching_notify; 81 81 + static u8 bidi_pass; 82 82 + static int test_type; 83 83 + 61 84 static int opt_queue; 62 85 static int opt_pkt_count; 63 63 - static int opt_poll; 64 64 - static int opt_teardown; 65 65 - static int opt_bidi; 66 66 - static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP; 86 86 + static u8 opt_verbose; 87 87 + 88 88 + static u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; 89 89 + static u32 xdp_bind_flags = XDP_USE_NEED_WAKEUP | XDP_COPY; 67 90 static u8 pkt_data[XSK_UMEM__DEFAULT_FRAME_SIZE]; 68 91 static u32 pkt_counter; 69 69 - static u32 prev_pkt = -1; 92 92 + static long prev_pkt = -1; 70 93 static int sigvar; 94 94 + static int stat_test_type; 95 95 + static u32 rxqsize; 96 96 + static u32 frame_headroom; 71 97 72 98 struct xsk_umem_info { 73 99 struct xsk_ring_prod fq; ··· 163 137 pthread_attr_t attr; 164 138 165 139 struct targs { 166 166 - bool retptr; 140 140 + u8 retptr; 167 141 int idx; 142 142 + u32 flags; 168 143 }; 169 144 170 145 TAILQ_HEAD(head_s, pkt) head = TAILQ_HEAD_INITIALIZER(head);

+3 -27

tools/testing/selftests/bpf/xsk_prereqs.sh

reviewed

··· 82 82 { 83 83 if [ $(ip netns show | grep $3 &>/dev/null; echo $?;) == 0 ]; then 84 84 [ $(ip netns exec $3 ip link show $2 &>/dev/null; echo $?;) == 0 ] && 85 85 - { echo "removing link $1:$2"; ip netns exec $3 ip link del $2; } 86 86 - echo "removing ns $3" 85 85 + { ip netns exec $3 ip link del $2; } 87 86 ip netns del $3 88 87 fi 89 88 #Once we delete a veth pair node, the entire veth pair is removed, 90 89 #this is just to be cautious just incase the NS does not exist then 91 90 #veth node inside NS won't get removed so we explicitly remove it 92 91 [ $(ip link show $1 &>/dev/null; echo $?;) == 0 ] && 93 93 - { echo "removing link $1"; ip link del $1; } 92 92 + { ip link del $1; } 94 93 if [ -f ${SPECFILE} ]; then 95 95 - echo "removing spec file:" ${SPECFILE} 96 94 rm -f ${SPECFILE} 97 95 fi 98 96 } 99 97 100 98 cleanup_exit() 101 99 { 102 102 - echo "cleaning up..." 103 100 clear_configs $1 $2 $3 104 101 } 105 102 ··· 105 108 [ ! $(type -P ip) ] && { echo "'ip' not found. Skipping tests."; test_exit $ksft_skip 1; } 106 109 } 107 110 108 108 - vethXDPgeneric() 109 109 - { 110 110 - ip link set dev $1 xdpdrv off 111 111 - ip netns exec $3 ip link set dev $2 xdpdrv off 112 112 - } 113 113 - 114 114 - vethXDPnative() 115 115 - { 116 116 - ip link set dev $1 xdpgeneric off 117 117 - ip netns exec $3 ip link set dev $2 xdpgeneric off 118 118 - } 119 119 - 120 111 execxdpxceiver() 121 112 { 122 122 - local -a 'paramkeys=("${!'"$1"'[@]}")' copy 123 123 - paramkeysstr=${paramkeys[*]} 124 124 - 125 125 - for index in $paramkeysstr; 126 126 - do 127 127 - current=$1"[$index]" 128 128 - copy[$index]=${!current} 129 129 - done 130 130 - 131 131 - ./${XSKOBJ} -i ${VETH0} -i ${VETH1},${NS1} ${copy[*]} -C ${NUMPKTS} 113 113 + ./${XSKOBJ} -i ${VETH0} -i ${VETH1},${NS1} -C ${NUMPKTS} ${VERBOSE_ARG} ${DUMP_PKTS_ARG} 132 114 }