bpf: remove ipv6_bpf_stub completely and use direct function calls

+2

include/net/ipv6.h

··· 1149 1149 void inet6_cleanup_sock(struct sock *sk); 1150 1150 void inet6_sock_destruct(struct sock *sk); 1151 1151 int inet6_release(struct socket *sock); 1152 + int __inet6_bind(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len, 1153 + u32 flags); 1152 1154 int inet6_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int addr_len); 1153 1155 int inet6_bind_sk(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len); 1154 1156 int inet6_getname(struct socket *sock, struct sockaddr *uaddr,

-21

include/net/ipv6_stubs.h

··· 77 77 __u32 mark, struct ipv6_txoptions *opt, int tclass, u32 priority); 78 78 }; 79 79 extern const struct ipv6_stub *ipv6_stub __read_mostly; 80 - 81 - /* A stub used by bpf helpers. Similarly ugly as ipv6_stub */ 82 - struct ipv6_bpf_stub { 83 - int (*inet6_bind)(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len, 84 - u32 flags); 85 - struct sock *(*udp6_lib_lookup)(const struct net *net, 86 - const struct in6_addr *saddr, __be16 sport, 87 - const struct in6_addr *daddr, __be16 dport, 88 - int dif, int sdif, struct sk_buff *skb); 89 - int (*ipv6_setsockopt)(struct sock *sk, int level, int optname, 90 - sockptr_t optval, unsigned int optlen); 91 - int (*ipv6_getsockopt)(struct sock *sk, int level, int optname, 92 - sockptr_t optval, sockptr_t optlen); 93 - int (*ipv6_dev_get_saddr)(struct net *net, 94 - const struct net_device *dst_dev, 95 - const struct in6_addr *daddr, 96 - unsigned int prefs, 97 - struct in6_addr *saddr); 98 - }; 99 - extern const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly; 100 - 101 80 #endif

+24 -34

net/core/filter.c

··· 73 73 #include <net/seg6.h> 74 74 #include <net/seg6_local.h> 75 75 #include <net/lwtunnel.h> 76 - #include <net/ipv6_stubs.h> 77 76 #include <net/bpf_sk_storage.h> 78 77 #include <net/transp_v6.h> 79 78 #include <linux/btf_ids.h> ··· 2278 2279 .saddr = ip6h->saddr, 2279 2280 }; 2280 2281 2281 - dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &fl6, NULL); 2282 + dst = ip6_dst_lookup_flow(net, NULL, &fl6, NULL); 2282 2283 if (IS_ERR(dst)) 2283 2284 goto out_drop; 2284 2285 ··· 5576 5577 } 5577 5578 5578 5579 if (getopt) 5579 - return ipv6_bpf_stub->ipv6_getsockopt(sk, SOL_IPV6, optname, 5580 - KERNEL_SOCKPTR(optval), 5581 - KERNEL_SOCKPTR(optlen)); 5580 + return do_ipv6_getsockopt(sk, SOL_IPV6, optname, 5581 + KERNEL_SOCKPTR(optval), 5582 + KERNEL_SOCKPTR(optlen)); 5582 5583 5583 - return ipv6_bpf_stub->ipv6_setsockopt(sk, SOL_IPV6, optname, 5584 - KERNEL_SOCKPTR(optval), *optlen); 5584 + return do_ipv6_setsockopt(sk, SOL_IPV6, optname, 5585 + KERNEL_SOCKPTR(optval), *optlen); 5585 5586 } 5586 5587 5587 5588 static int __bpf_setsockopt(struct sock *sk, int level, int optname, ··· 5980 5981 .arg2_type = ARG_ANYTHING, 5981 5982 }; 5982 5983 5983 - const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly; 5984 - EXPORT_SYMBOL_GPL(ipv6_bpf_stub); 5985 - 5986 5984 BPF_CALL_3(bpf_bind, struct bpf_sock_addr_kern *, ctx, struct sockaddr *, addr, 5987 5985 int, addr_len) 5988 5986 { ··· 6003 6007 return err; 6004 6008 if (((struct sockaddr_in6 *)addr)->sin6_port == htons(0)) 6005 6009 flags |= BIND_FORCE_ADDRESS_NO_PORT; 6006 - /* ipv6_bpf_stub cannot be NULL, since it's called from 6007 - * bpf_cgroup_inet6_connect hook and ipv6 is already loaded 6008 - */ 6009 - return ipv6_bpf_stub->inet6_bind(sk, (struct sockaddr_unsized *)addr, 6010 - addr_len, flags); 6010 + 6011 + return __inet6_bind(sk, (struct sockaddr_unsized *)addr, 6012 + addr_len, flags); 6011 6013 #endif /* CONFIG_IPV6 */ 6012 6014 } 6013 6015 #endif /* CONFIG_INET */ ··· 6093 6099 static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, 6094 6100 u32 flags, bool check_mtu) 6095 6101 { 6102 + struct neighbour *neigh = NULL; 6096 6103 struct fib_nh_common *nhc; 6097 6104 struct in_device *in_dev; 6098 - struct neighbour *neigh; 6099 6105 struct net_device *dev; 6100 6106 struct fib_result res; 6101 6107 struct flowi4 fl4; ··· 6215 6221 if (likely(nhc->nhc_gw_family != AF_INET6)) 6216 6222 neigh = __ipv4_neigh_lookup_noref(dev, 6217 6223 (__force u32)params->ipv4_dst); 6218 - else 6219 - neigh = __ipv6_neigh_lookup_noref_stub(dev, params->ipv6_dst); 6224 + else if (IS_ENABLED(CONFIG_IPV6)) 6225 + neigh = __ipv6_neigh_lookup_noref(dev, params->ipv6_dst); 6220 6226 6221 6227 if (!neigh || !(READ_ONCE(neigh->nud_state) & NUD_VALID)) 6222 6228 return BPF_FIB_LKUP_RET_NO_NEIGH; ··· 6284 6290 params->tbid = 0; 6285 6291 } 6286 6292 6287 - tb = ipv6_stub->fib6_get_table(net, tbid); 6293 + tb = fib6_get_table(net, tbid); 6288 6294 if (unlikely(!tb)) 6289 6295 return BPF_FIB_LKUP_RET_NOT_FWDED; 6290 6296 6291 - err = ipv6_stub->fib6_table_lookup(net, tb, oif, &fl6, &res, 6292 - strict); 6297 + err = fib6_table_lookup(net, tb, oif, &fl6, &res, strict); 6293 6298 } else { 6294 6299 if (flags & BPF_FIB_LOOKUP_MARK) 6295 6300 fl6.flowi6_mark = params->mark; ··· 6298 6305 fl6.flowi6_tun_key.tun_id = 0; 6299 6306 fl6.flowi6_uid = sock_net_uid(net, NULL); 6300 6307 6301 - err = ipv6_stub->fib6_lookup(net, oif, &fl6, &res, strict); 6308 + err = fib6_lookup(net, oif, &fl6, &res, strict); 6302 6309 } 6303 6310 6304 6311 if (unlikely(err || IS_ERR_OR_NULL(res.f6i) || ··· 6319 6326 return BPF_FIB_LKUP_RET_NOT_FWDED; 6320 6327 } 6321 6328 6322 - ipv6_stub->fib6_select_path(net, &res, &fl6, fl6.flowi6_oif, 6323 - fl6.flowi6_oif != 0, NULL, strict); 6329 + fib6_select_path(net, &res, &fl6, fl6.flowi6_oif, 6330 + fl6.flowi6_oif != 0, NULL, strict); 6324 6331 6325 6332 if (check_mtu) { 6326 - mtu = ipv6_stub->ip6_mtu_from_fib6(&res, dst, src); 6333 + mtu = ip6_mtu_from_fib6(&res, dst, src); 6327 6334 if (params->tot_len > mtu) { 6328 6335 params->mtu_result = mtu; /* union with tot_len */ 6329 6336 return BPF_FIB_LKUP_RET_FRAG_NEEDED; ··· 6344 6351 if (res.f6i->fib6_prefsrc.plen) { 6345 6352 *src = res.f6i->fib6_prefsrc.addr; 6346 6353 } else { 6347 - err = ipv6_bpf_stub->ipv6_dev_get_saddr(net, dev, 6348 - &fl6.daddr, 0, 6349 - src); 6354 + err = ipv6_dev_get_saddr(net, dev, &fl6.daddr, 0, src); 6350 6355 if (err) 6351 6356 return BPF_FIB_LKUP_RET_NO_SRC_ADDR; 6352 6357 } ··· 6356 6365 /* xdp and cls_bpf programs are run in RCU-bh so rcu_read_lock_bh is 6357 6366 * not needed here. 6358 6367 */ 6359 - neigh = __ipv6_neigh_lookup_noref_stub(dev, dst); 6368 + neigh = __ipv6_neigh_lookup_noref(dev, dst); 6360 6369 if (!neigh || !(READ_ONCE(neigh->nud_state) & NUD_VALID)) 6361 6370 return BPF_FIB_LKUP_RET_NO_NEIGH; 6362 6371 memcpy(params->dmac, neigh->ha, ETH_ALEN); ··· 6891 6900 src6, tuple->ipv6.sport, 6892 6901 dst6, ntohs(tuple->ipv6.dport), 6893 6902 dif, sdif, &refcounted); 6894 - else if (likely(ipv6_bpf_stub)) 6895 - sk = ipv6_bpf_stub->udp6_lib_lookup(net, 6896 - src6, tuple->ipv6.sport, 6897 - dst6, tuple->ipv6.dport, 6898 - dif, sdif, NULL); 6903 + else if (likely(ipv6_mod_enabled())) 6904 + sk = __udp6_lib_lookup(net, src6, tuple->ipv6.sport, 6905 + dst6, tuple->ipv6.dport, 6906 + dif, sdif, NULL); 6899 6907 #endif 6900 6908 } 6901 6909

+7 -3

net/core/lwt_bpf.c

··· 13 13 #include <net/gre.h> 14 14 #include <net/ip.h> 15 15 #include <net/ip6_route.h> 16 - #include <net/ipv6_stubs.h> 17 16 18 17 struct bpf_lwt_prog { 19 18 struct bpf_prog *prog; ··· 102 103 dev_put(dev); 103 104 } else if (skb->protocol == htons(ETH_P_IPV6)) { 104 105 skb_dst_drop(skb); 105 - err = ipv6_stub->ipv6_route_input(skb); 106 + if (IS_ENABLED(CONFIG_IPV6)) { 107 + ip6_route_input(skb); 108 + err = skb_dst(skb)->error; 109 + } else { 110 + err = -EAFNOSUPPORT; 111 + } 106 112 } else { 107 113 err = -EAFNOSUPPORT; 108 114 } ··· 237 233 fl6.daddr = iph6->daddr; 238 234 fl6.saddr = iph6->saddr; 239 235 240 - dst = ipv6_stub->ipv6_dst_lookup_flow(net, skb->sk, &fl6, NULL); 236 + dst = ip6_dst_lookup_flow(net, skb->sk, &fl6, NULL); 241 237 if (IS_ERR(dst)) { 242 238 err = PTR_ERR(dst); 243 239 goto err;

+2 -11

net/ipv6/af_inet6.c

··· 264 264 goto out; 265 265 } 266 266 267 - static int __inet6_bind(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len, 268 - u32 flags) 267 + int __inet6_bind(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len, 268 + u32 flags) 269 269 { 270 270 struct sockaddr_in6 *addr = (struct sockaddr_in6 *)uaddr; 271 271 struct inet_sock *inet = inet_sk(sk); ··· 1032 1032 .ip6_xmit = ip6_xmit, 1033 1033 }; 1034 1034 1035 - static const struct ipv6_bpf_stub ipv6_bpf_stub_impl = { 1036 - .inet6_bind = __inet6_bind, 1037 - .udp6_lib_lookup = __udp6_lib_lookup, 1038 - .ipv6_setsockopt = do_ipv6_setsockopt, 1039 - .ipv6_getsockopt = do_ipv6_getsockopt, 1040 - .ipv6_dev_get_saddr = ipv6_dev_get_saddr, 1041 - }; 1042 - 1043 1035 static int __init inet6_init(void) 1044 1036 { 1045 1037 struct list_head *r; ··· 1191 1199 /* ensure that ipv6 stubs are visible only after ipv6 is ready */ 1192 1200 wmb(); 1193 1201 ipv6_stub = &ipv6_stub_impl; 1194 - ipv6_bpf_stub = &ipv6_bpf_stub_impl; 1195 1202 out: 1196 1203 return err; 1197 1204

Configure Feed

Configure Feed