Merge branch 'ipv6-Use-fib6_result-for-fib_lookups'

+15 -9

include/net/ip6_fib.h

··· 190 190 unsigned short rt6i_nfheader_len; 191 191 }; 192 192 193 + struct fib6_result { 194 + struct fib6_nh *nh; 195 + struct fib6_info *f6i; 196 + u32 fib6_flags; 197 + u8 fib6_type; 198 + }; 199 + 193 200 #define for_each_fib6_node_rt_rcu(fn) \ 194 201 for (rt = rcu_dereference((fn)->leaf); rt; \ 195 202 rt = rcu_dereference(rt->fib6_next)) ··· 391 384 /* called with rcu lock held; can return error pointer 392 385 * caller needs to select path 393 386 */ 394 - struct fib6_info *fib6_lookup(struct net *net, int oif, struct flowi6 *fl6, 395 - int flags); 387 + int fib6_lookup(struct net *net, int oif, struct flowi6 *fl6, 388 + struct fib6_result *res, int flags); 396 389 397 390 /* called with rcu lock held; caller needs to select path */ 398 - struct fib6_info *fib6_table_lookup(struct net *net, struct fib6_table *table, 399 - int oif, struct flowi6 *fl6, int strict); 391 + int fib6_table_lookup(struct net *net, struct fib6_table *table, 392 + int oif, struct flowi6 *fl6, struct fib6_result *res, 393 + int strict); 400 394 401 - struct fib6_info *fib6_multipath_select(const struct net *net, 402 - struct fib6_info *match, 403 - struct flowi6 *fl6, int oif, 404 - const struct sk_buff *skb, int strict); 405 - 395 + void fib6_select_path(const struct net *net, struct fib6_result *res, 396 + struct flowi6 *fl6, int oif, bool have_oif_match, 397 + const struct sk_buff *skb, int strict); 406 398 struct fib6_node *fib6_node_lookup(struct fib6_node *root, 407 399 const struct in6_addr *daddr, 408 400 const struct in6_addr *saddr);

+3 -2

include/net/ip6_route.h

··· 302 302 return mtu; 303 303 } 304 304 305 - u32 ip6_mtu_from_fib6(struct fib6_info *f6i, struct in6_addr *daddr, 306 - struct in6_addr *saddr); 305 + u32 ip6_mtu_from_fib6(const struct fib6_result *res, 306 + const struct in6_addr *daddr, 307 + const struct in6_addr *saddr); 307 308 308 309 struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw, 309 310 struct net_device *dev, struct sk_buff *skb,

+12 -13

include/net/ipv6_stubs.h

··· 14 14 struct fib6_info; 15 15 struct fib6_nh; 16 16 struct fib6_config; 17 + struct fib6_result; 17 18 18 19 /* This is ugly, ideally these symbols should be built 19 20 * into the core kernel. ··· 29 28 int (*ipv6_route_input)(struct sk_buff *skb); 30 29 31 30 struct fib6_table *(*fib6_get_table)(struct net *net, u32 id); 32 - struct fib6_info *(*fib6_lookup)(struct net *net, int oif, 33 - struct flowi6 *fl6, int flags); 34 - struct fib6_info *(*fib6_table_lookup)(struct net *net, 35 - struct fib6_table *table, 36 - int oif, struct flowi6 *fl6, 37 - int flags); 38 - struct fib6_info *(*fib6_multipath_select)(const struct net *net, 39 - struct fib6_info *f6i, 40 - struct flowi6 *fl6, int oif, 41 - const struct sk_buff *skb, 42 - int strict); 43 - u32 (*ip6_mtu_from_fib6)(struct fib6_info *f6i, struct in6_addr *daddr, 44 - struct in6_addr *saddr); 31 + int (*fib6_lookup)(struct net *net, int oif, struct flowi6 *fl6, 32 + struct fib6_result *res, int flags); 33 + int (*fib6_table_lookup)(struct net *net, struct fib6_table *table, 34 + int oif, struct flowi6 *fl6, 35 + struct fib6_result *res, int flags); 36 + void (*fib6_select_path)(const struct net *net, struct fib6_result *res, 37 + struct flowi6 *fl6, int oif, bool oif_match, 38 + const struct sk_buff *skb, int strict); 39 + u32 (*ip6_mtu_from_fib6)(const struct fib6_result *res, 40 + const struct in6_addr *daddr, 41 + const struct in6_addr *saddr); 45 42 46 43 int (*fib6_nh_init)(struct net *net, struct fib6_nh *fib6_nh, 47 44 struct fib6_config *cfg, gfp_t gfp_flags,

+8 -8

include/trace/events/fib6.h

··· 12 12 13 13 TRACE_EVENT(fib6_table_lookup, 14 14 15 - TP_PROTO(const struct net *net, const struct fib6_info *f6i, 15 + TP_PROTO(const struct net *net, const struct fib6_result *res, 16 16 struct fib6_table *table, const struct flowi6 *flp), 17 17 18 - TP_ARGS(net, f6i, table, flp), 18 + TP_ARGS(net, res, table, flp), 19 19 20 20 TP_STRUCT__entry( 21 21 __field( u32, tb_id ) ··· 39 39 struct in6_addr *in6; 40 40 41 41 __entry->tb_id = table->tb6_id; 42 - __entry->err = ip6_rt_type_to_error(f6i->fib6_type); 42 + __entry->err = ip6_rt_type_to_error(res->fib6_type); 43 43 __entry->oif = flp->flowi6_oif; 44 44 __entry->iif = flp->flowi6_iif; 45 45 __entry->tos = ip6_tclass(flp->flowlabel); ··· 62 62 __entry->dport = 0; 63 63 } 64 64 65 - if (f6i->fib6_nh.fib_nh_dev) { 66 - __assign_str(name, f6i->fib6_nh.fib_nh_dev); 65 + if (res->nh && res->nh->fib_nh_dev) { 66 + __assign_str(name, res->nh->fib_nh_dev); 67 67 } else { 68 68 __assign_str(name, "-"); 69 69 } 70 - if (f6i == net->ipv6.fib6_null_entry) { 70 + if (res->f6i == net->ipv6.fib6_null_entry) { 71 71 struct in6_addr in6_zero = {}; 72 72 73 73 in6 = (struct in6_addr *)__entry->gw; 74 74 *in6 = in6_zero; 75 75 76 - } else if (f6i) { 76 + } else if (res->nh) { 77 77 in6 = (struct in6_addr *)__entry->gw; 78 - *in6 = f6i->fib6_nh.fib_nh_gw6; 78 + *in6 = res->nh->fib_nh_gw6; 79 79 } 80 80 ), 81 81

+27 -29

net/core/filter.c

··· 4679 4679 struct in6_addr *src = (struct in6_addr *) params->ipv6_src; 4680 4680 struct in6_addr *dst = (struct in6_addr *) params->ipv6_dst; 4681 4681 struct neighbour *neigh; 4682 + struct fib6_result res; 4682 4683 struct net_device *dev; 4683 4684 struct inet6_dev *idev; 4684 - struct fib6_info *f6i; 4685 4685 struct flowi6 fl6; 4686 4686 int strict = 0; 4687 - int oif; 4687 + int oif, err; 4688 4688 u32 mtu; 4689 4689 4690 4690 /* link local addresses are never forwarded */ ··· 4726 4726 if (unlikely(!tb)) 4727 4727 return BPF_FIB_LKUP_RET_NOT_FWDED; 4728 4728 4729 - f6i = ipv6_stub->fib6_table_lookup(net, tb, oif, &fl6, strict); 4729 + err = ipv6_stub->fib6_table_lookup(net, tb, oif, &fl6, &res, 4730 + strict); 4730 4731 } else { 4731 4732 fl6.flowi6_mark = 0; 4732 4733 fl6.flowi6_secid = 0; 4733 4734 fl6.flowi6_tun_key.tun_id = 0; 4734 4735 fl6.flowi6_uid = sock_net_uid(net, NULL); 4735 4736 4736 - f6i = ipv6_stub->fib6_lookup(net, oif, &fl6, strict); 4737 + err = ipv6_stub->fib6_lookup(net, oif, &fl6, &res, strict); 4737 4738 } 4738 4739 4739 - if (unlikely(IS_ERR_OR_NULL(f6i) || f6i == net->ipv6.fib6_null_entry)) 4740 + if (unlikely(err || IS_ERR_OR_NULL(res.f6i) || 4741 + res.f6i == net->ipv6.fib6_null_entry)) 4740 4742 return BPF_FIB_LKUP_RET_NOT_FWDED; 4741 4743 4742 - if (unlikely(f6i->fib6_flags & RTF_REJECT)) { 4743 - switch (f6i->fib6_type) { 4744 - case RTN_BLACKHOLE: 4745 - return BPF_FIB_LKUP_RET_BLACKHOLE; 4746 - case RTN_UNREACHABLE: 4747 - return BPF_FIB_LKUP_RET_UNREACHABLE; 4748 - case RTN_PROHIBIT: 4749 - return BPF_FIB_LKUP_RET_PROHIBIT; 4750 - default: 4751 - return BPF_FIB_LKUP_RET_NOT_FWDED; 4752 - } 4744 + switch (res.fib6_type) { 4745 + /* only unicast is forwarded */ 4746 + case RTN_UNICAST: 4747 + break; 4748 + case RTN_BLACKHOLE: 4749 + return BPF_FIB_LKUP_RET_BLACKHOLE; 4750 + case RTN_UNREACHABLE: 4751 + return BPF_FIB_LKUP_RET_UNREACHABLE; 4752 + case RTN_PROHIBIT: 4753 + return BPF_FIB_LKUP_RET_PROHIBIT; 4754 + default: 4755 + return BPF_FIB_LKUP_RET_NOT_FWDED; 4753 4756 } 4754 4757 4755 - if (f6i->fib6_type != RTN_UNICAST) 4756 - return BPF_FIB_LKUP_RET_NOT_FWDED; 4757 - 4758 - if (f6i->fib6_nsiblings && fl6.flowi6_oif == 0) 4759 - f6i = ipv6_stub->fib6_multipath_select(net, f6i, &fl6, 4760 - fl6.flowi6_oif, NULL, 4761 - strict); 4758 + ipv6_stub->fib6_select_path(net, &res, &fl6, fl6.flowi6_oif, 4759 + fl6.flowi6_oif != 0, NULL, strict); 4762 4760 4763 4761 if (check_mtu) { 4764 - mtu = ipv6_stub->ip6_mtu_from_fib6(f6i, dst, src); 4762 + mtu = ipv6_stub->ip6_mtu_from_fib6(&res, dst, src); 4765 4763 if (params->tot_len > mtu) 4766 4764 return BPF_FIB_LKUP_RET_FRAG_NEEDED; 4767 4765 } 4768 4766 4769 - if (f6i->fib6_nh.fib_nh_lws) 4767 + if (res.nh->fib_nh_lws) 4770 4768 return BPF_FIB_LKUP_RET_UNSUPP_LWT; 4771 4769 4772 - if (f6i->fib6_nh.fib_nh_gw_family) 4773 - *dst = f6i->fib6_nh.fib_nh_gw6; 4770 + if (res.nh->fib_nh_gw_family) 4771 + *dst = res.nh->fib_nh_gw6; 4774 4772 4775 - dev = f6i->fib6_nh.fib_nh_dev; 4776 - params->rt_metric = f6i->fib6_metric; 4773 + dev = res.nh->fib_nh_dev; 4774 + params->rt_metric = res.f6i->fib6_metric; 4777 4775 4778 4776 /* xdp and cls_bpf programs are run in RCU-bh so rcu_read_lock_bh is 4779 4777 * not needed here.

+15 -14

net/ipv6/addrconf_core.c

··· 144 144 return NULL; 145 145 } 146 146 147 - static struct fib6_info * 147 + static int 148 148 eafnosupport_fib6_table_lookup(struct net *net, struct fib6_table *table, 149 - int oif, struct flowi6 *fl6, int flags) 149 + int oif, struct flowi6 *fl6, 150 + struct fib6_result *res, int flags) 150 151 { 151 - return NULL; 152 + return -EAFNOSUPPORT; 152 153 } 153 154 154 - static struct fib6_info * 155 + static int 155 156 eafnosupport_fib6_lookup(struct net *net, int oif, struct flowi6 *fl6, 156 - int flags) 157 + struct fib6_result *res, int flags) 157 158 { 158 - return NULL; 159 + return -EAFNOSUPPORT; 159 160 } 160 161 161 - static struct fib6_info * 162 - eafnosupport_fib6_multipath_select(const struct net *net, struct fib6_info *f6i, 163 - struct flowi6 *fl6, int oif, 164 - const struct sk_buff *skb, int strict) 162 + static void 163 + eafnosupport_fib6_select_path(const struct net *net, struct fib6_result *res, 164 + struct flowi6 *fl6, int oif, bool have_oif_match, 165 + const struct sk_buff *skb, int strict) 165 166 { 166 - return f6i; 167 167 } 168 168 169 169 static u32 170 - eafnosupport_ip6_mtu_from_fib6(struct fib6_info *f6i, struct in6_addr *daddr, 171 - struct in6_addr *saddr) 170 + eafnosupport_ip6_mtu_from_fib6(const struct fib6_result *res, 171 + const struct in6_addr *daddr, 172 + const struct in6_addr *saddr) 172 173 { 173 174 return 0; 174 175 } ··· 188 187 .fib6_get_table = eafnosupport_fib6_get_table, 189 188 .fib6_table_lookup = eafnosupport_fib6_table_lookup, 190 189 .fib6_lookup = eafnosupport_fib6_lookup, 191 - .fib6_multipath_select = eafnosupport_fib6_multipath_select, 190 + .fib6_select_path = eafnosupport_fib6_select_path, 192 191 .ip6_mtu_from_fib6 = eafnosupport_ip6_mtu_from_fib6, 193 192 .fib6_nh_init = eafnosupport_fib6_nh_init, 194 193 };

+1 -1

net/ipv6/af_inet6.c

··· 917 917 .fib6_get_table = fib6_get_table, 918 918 .fib6_table_lookup = fib6_table_lookup, 919 919 .fib6_lookup = fib6_lookup, 920 - .fib6_multipath_select = fib6_multipath_select, 920 + .fib6_select_path = fib6_select_path, 921 921 .ip6_mtu_from_fib6 = ip6_mtu_from_fib6, 922 922 .fib6_nh_init = fib6_nh_init, 923 923 .fib6_nh_release = fib6_nh_release,

+13 -21

net/ipv6/fib6_rules.c

··· 61 61 } 62 62 63 63 /* called with rcu lock held; no reference taken on fib6_info */ 64 - struct fib6_info *fib6_lookup(struct net *net, int oif, struct flowi6 *fl6, 65 - int flags) 64 + int fib6_lookup(struct net *net, int oif, struct flowi6 *fl6, 65 + struct fib6_result *res, int flags) 66 66 { 67 - struct fib6_info *f6i; 68 67 int err; 69 68 70 69 if (net->ipv6.fib6_has_custom_rules) { 71 70 struct fib_lookup_arg arg = { 72 71 .lookup_ptr = fib6_table_lookup, 73 72 .lookup_data = &oif, 73 + .result = res, 74 74 .flags = FIB_LOOKUP_NOREF, 75 75 }; 76 76 ··· 78 78 79 79 err = fib_rules_lookup(net->ipv6.fib6_rules_ops, 80 80 flowi6_to_flowi(fl6), flags, &arg); 81 - if (err) 82 - return ERR_PTR(err); 83 - 84 - f6i = arg.result ? : net->ipv6.fib6_null_entry; 85 81 } else { 86 - f6i = fib6_table_lookup(net, net->ipv6.fib6_local_tbl, 87 - oif, fl6, flags); 88 - if (!f6i || f6i == net->ipv6.fib6_null_entry) 89 - f6i = fib6_table_lookup(net, net->ipv6.fib6_main_tbl, 90 - oif, fl6, flags); 82 + err = fib6_table_lookup(net, net->ipv6.fib6_local_tbl, oif, 83 + fl6, res, flags); 84 + if (err || res->f6i == net->ipv6.fib6_null_entry) 85 + err = fib6_table_lookup(net, net->ipv6.fib6_main_tbl, 86 + oif, fl6, res, flags); 91 87 } 92 88 93 - return f6i; 89 + return err; 94 90 } 95 91 96 92 struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6, ··· 153 157 static int fib6_rule_action_alt(struct fib_rule *rule, struct flowi *flp, 154 158 int flags, struct fib_lookup_arg *arg) 155 159 { 160 + struct fib6_result *res = arg->result; 156 161 struct flowi6 *flp6 = &flp->u.ip6; 157 162 struct net *net = rule->fr_net; 158 163 struct fib6_table *table; 159 - struct fib6_info *f6i; 160 164 int err = -EAGAIN, *oif; 161 165 u32 tb_id; 162 166 ··· 178 182 return -EAGAIN; 179 183 180 184 oif = (int *)arg->lookup_data; 181 - f6i = fib6_table_lookup(net, table, *oif, flp6, flags); 182 - if (f6i != net->ipv6.fib6_null_entry) { 185 + err = fib6_table_lookup(net, table, *oif, flp6, res, flags); 186 + if (!err && res->f6i != net->ipv6.fib6_null_entry) 183 187 err = fib6_rule_saddr(net, rule, flags, flp6, 184 - fib6_info_nh_dev(f6i)); 185 - 186 - if (likely(!err)) 187 - arg->result = f6i; 188 - } 188 + res->nh->fib_nh_dev); 189 189 190 190 return err; 191 191 }

+4 -3

net/ipv6/ip6_fib.c

··· 354 354 } 355 355 356 356 /* called with rcu lock held; no reference taken on fib6_info */ 357 - struct fib6_info *fib6_lookup(struct net *net, int oif, struct flowi6 *fl6, 358 - int flags) 357 + int fib6_lookup(struct net *net, int oif, struct flowi6 *fl6, 358 + struct fib6_result *res, int flags) 359 359 { 360 - return fib6_table_lookup(net, net->ipv6.fib6_main_tbl, oif, fl6, flags); 360 + return fib6_table_lookup(net, net->ipv6.fib6_main_tbl, oif, fl6, 361 + res, flags); 361 362 } 362 363 363 364 static void __net_init fib6_tables_init(struct net *net)

+252 -195

net/ipv6/route.c

··· 110 110 struct in6_addr *dest, struct in6_addr *src, 111 111 int iif, int type, u32 portid, u32 seq, 112 112 unsigned int flags); 113 - static struct rt6_info *rt6_find_cached_rt(struct fib6_info *rt, 113 + static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 114 114 struct in6_addr *daddr, 115 115 struct in6_addr *saddr); 116 116 ··· 428 428 return false; 429 429 } 430 430 431 - struct fib6_info *fib6_multipath_select(const struct net *net, 432 - struct fib6_info *match, 433 - struct flowi6 *fl6, int oif, 434 - const struct sk_buff *skb, 435 - int strict) 431 + void fib6_select_path(const struct net *net, struct fib6_result *res, 432 + struct flowi6 *fl6, int oif, bool have_oif_match, 433 + const struct sk_buff *skb, int strict) 436 434 { 437 435 struct fib6_info *sibling, *next_sibling; 436 + struct fib6_info *match = res->f6i; 437 + 438 + if (!match->fib6_nsiblings || have_oif_match) 439 + goto out; 438 440 439 441 /* We might have already computed the hash for ICMPv6 errors. In such 440 442 * case it will always be non-zero. Otherwise now is the time to do it. ··· 445 443 fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL); 446 444 447 445 if (fl6->mp_hash <= atomic_read(&match->fib6_nh.fib_nh_upper_bound)) 448 - return match; 446 + goto out; 449 447 450 448 list_for_each_entry_safe(sibling, next_sibling, &match->fib6_siblings, 451 449 fib6_siblings) { ··· 461 459 break; 462 460 } 463 461 464 - return match; 462 + out: 463 + res->f6i = match; 464 + res->nh = &match->fib6_nh; 465 465 } 466 466 467 467 /* ··· 491 487 return false; 492 488 } 493 489 494 - static inline struct fib6_info *rt6_device_match(struct net *net, 495 - struct fib6_info *rt, 496 - const struct in6_addr *saddr, 497 - int oif, 498 - int flags) 490 + static void rt6_device_match(struct net *net, struct fib6_result *res, 491 + const struct in6_addr *saddr, int oif, int flags) 499 492 { 500 - const struct fib6_nh *nh; 501 - struct fib6_info *sprt; 493 + struct fib6_info *f6i = res->f6i; 494 + struct fib6_info *spf6i; 495 + struct fib6_nh *nh; 502 496 503 - if (!oif && ipv6_addr_any(saddr) && 504 - !(rt->fib6_nh.fib_nh_flags & RTNH_F_DEAD)) 505 - return rt; 506 - 507 - for (sprt = rt; sprt; sprt = rcu_dereference(sprt->fib6_next)) { 508 - nh = &sprt->fib6_nh; 509 - if (__rt6_device_match(net, nh, saddr, oif, flags)) 510 - return sprt; 497 + if (!oif && ipv6_addr_any(saddr)) { 498 + nh = &f6i->fib6_nh; 499 + if (!(nh->fib_nh_flags & RTNH_F_DEAD)) 500 + goto out; 511 501 } 512 502 513 - if (oif && flags & RT6_LOOKUP_F_IFACE) 514 - return net->ipv6.fib6_null_entry; 503 + for (spf6i = f6i; spf6i; spf6i = rcu_dereference(spf6i->fib6_next)) { 504 + nh = &spf6i->fib6_nh; 505 + if (__rt6_device_match(net, nh, saddr, oif, flags)) { 506 + res->f6i = spf6i; 507 + goto out; 508 + } 509 + } 515 510 516 - return rt->fib6_nh.fib_nh_flags & RTNH_F_DEAD ? net->ipv6.fib6_null_entry : rt; 511 + if (oif && flags & RT6_LOOKUP_F_IFACE) { 512 + res->f6i = net->ipv6.fib6_null_entry; 513 + nh = &res->f6i->fib6_nh; 514 + goto out; 515 + } 516 + 517 + nh = &f6i->fib6_nh; 518 + if (nh->fib_nh_flags & RTNH_F_DEAD) { 519 + res->f6i = net->ipv6.fib6_null_entry; 520 + nh = &res->f6i->fib6_nh; 521 + } 522 + out: 523 + res->nh = nh; 524 + res->fib6_type = res->f6i->fib6_type; 525 + res->fib6_flags = res->f6i->fib6_flags; 517 526 } 518 527 519 528 #ifdef CONFIG_IPV6_ROUTER_PREF ··· 697 680 return rc; 698 681 } 699 682 700 - static void __find_rr_leaf(struct fib6_info *rt_start, 683 + static void __find_rr_leaf(struct fib6_info *f6i_start, 701 684 struct fib6_info *nomatch, u32 metric, 702 - struct fib6_info **match, struct fib6_info **cont, 685 + struct fib6_result *res, struct fib6_info **cont, 703 686 int oif, int strict, bool *do_rr, int *mpri) 704 687 { 705 - struct fib6_info *rt; 688 + struct fib6_info *f6i; 706 689 707 - for (rt = rt_start; 708 - rt && rt != nomatch; 709 - rt = rcu_dereference(rt->fib6_next)) { 690 + for (f6i = f6i_start; 691 + f6i && f6i != nomatch; 692 + f6i = rcu_dereference(f6i->fib6_next)) { 710 693 struct fib6_nh *nh; 711 694 712 - if (cont && rt->fib6_metric != metric) { 713 - *cont = rt; 695 + if (cont && f6i->fib6_metric != metric) { 696 + *cont = f6i; 714 697 return; 715 698 } 716 699 717 - if (fib6_check_expired(rt)) 700 + if (fib6_check_expired(f6i)) 718 701 continue; 719 702 720 - nh = &rt->fib6_nh; 721 - if (find_match(nh, rt->fib6_flags, oif, strict, mpri, do_rr)) 722 - *match = rt; 703 + nh = &f6i->fib6_nh; 704 + if (find_match(nh, f6i->fib6_flags, oif, strict, mpri, do_rr)) { 705 + res->f6i = f6i; 706 + res->nh = nh; 707 + res->fib6_flags = f6i->fib6_flags; 708 + res->fib6_type = f6i->fib6_type; 709 + } 723 710 } 724 711 } 725 712 726 - static struct fib6_info *find_rr_leaf(struct fib6_node *fn, 727 - struct fib6_info *leaf, 728 - struct fib6_info *rr_head, 729 - u32 metric, int oif, int strict, 730 - bool *do_rr) 713 + static void find_rr_leaf(struct fib6_node *fn, struct fib6_info *leaf, 714 + struct fib6_info *rr_head, int oif, int strict, 715 + bool *do_rr, struct fib6_result *res) 731 716 { 732 - struct fib6_info *match = NULL, *cont = NULL; 717 + u32 metric = rr_head->fib6_metric; 718 + struct fib6_info *cont = NULL; 733 719 int mpri = -1; 734 720 735 - __find_rr_leaf(rr_head, NULL, metric, &match, &cont, 721 + __find_rr_leaf(rr_head, NULL, metric, res, &cont, 736 722 oif, strict, do_rr, &mpri); 737 723 738 - __find_rr_leaf(leaf, rr_head, metric, &match, &cont, 724 + __find_rr_leaf(leaf, rr_head, metric, res, &cont, 739 725 oif, strict, do_rr, &mpri); 740 726 741 - if (match || !cont) 742 - return match; 727 + if (res->f6i || !cont) 728 + return; 743 729 744 - __find_rr_leaf(cont, NULL, metric, &match, NULL, 730 + __find_rr_leaf(cont, NULL, metric, res, NULL, 745 731 oif, strict, do_rr, &mpri); 746 - 747 - return match; 748 732 } 749 733 750 - static struct fib6_info *rt6_select(struct net *net, struct fib6_node *fn, 751 - int oif, int strict) 734 + static void rt6_select(struct net *net, struct fib6_node *fn, int oif, 735 + struct fib6_result *res, int strict) 752 736 { 753 737 struct fib6_info *leaf = rcu_dereference(fn->leaf); 754 - struct fib6_info *match, *rt0; 738 + struct fib6_info *rt0; 755 739 bool do_rr = false; 756 740 int key_plen; 757 741 742 + /* make sure this function or its helpers sets f6i */ 743 + res->f6i = NULL; 744 + 758 745 if (!leaf || leaf == net->ipv6.fib6_null_entry) 759 - return net->ipv6.fib6_null_entry; 746 + goto out; 760 747 761 748 rt0 = rcu_dereference(fn->rr_ptr); 762 749 if (!rt0) ··· 777 756 key_plen = rt0->fib6_src.plen; 778 757 #endif 779 758 if (fn->fn_bit != key_plen) 780 - return net->ipv6.fib6_null_entry; 759 + goto out; 781 760 782 - match = find_rr_leaf(fn, leaf, rt0, rt0->fib6_metric, oif, strict, 783 - &do_rr); 784 - 761 + find_rr_leaf(fn, leaf, rt0, oif, strict, &do_rr, res); 785 762 if (do_rr) { 786 763 struct fib6_info *next = rcu_dereference(rt0->fib6_next); 787 764 ··· 796 777 } 797 778 } 798 779 799 - return match ? match : net->ipv6.fib6_null_entry; 780 + out: 781 + if (!res->f6i) { 782 + res->f6i = net->ipv6.fib6_null_entry; 783 + res->nh = &res->f6i->fib6_nh; 784 + res->fib6_flags = res->f6i->fib6_flags; 785 + res->fib6_type = res->f6i->fib6_type; 786 + } 800 787 } 801 788 802 - static bool rt6_is_gw_or_nonexthop(const struct fib6_info *rt) 789 + static bool rt6_is_gw_or_nonexthop(const struct fib6_result *res) 803 790 { 804 - return (rt->fib6_flags & RTF_NONEXTHOP) || rt->fib6_nh.fib_nh_gw_family; 791 + return (res->f6i->fib6_flags & RTF_NONEXTHOP) || 792 + res->nh->fib_nh_gw_family; 805 793 } 806 794 807 795 #ifdef CONFIG_IPV6_ROUTE_INFO ··· 892 866 */ 893 867 894 868 /* called with rcu_lock held */ 895 - static struct net_device *ip6_rt_get_dev_rcu(struct fib6_info *rt) 869 + static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res) 896 870 { 897 - struct net_device *dev = rt->fib6_nh.fib_nh_dev; 871 + struct net_device *dev = res->nh->fib_nh_dev; 898 872 899 - if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) { 873 + if (res->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) { 900 874 /* for copies of local routes, dst->dev needs to be the 901 875 * device if it is a master device, the master device if 902 876 * device is enslaved, and the loopback as the default 903 877 */ 904 878 if (netif_is_l3_slave(dev) && 905 - !rt6_need_strict(&rt->fib6_dst.addr)) 879 + !rt6_need_strict(&res->f6i->fib6_dst.addr)) 906 880 dev = l3mdev_master_dev_rcu(dev); 907 881 else if (!netif_is_l3_master(dev)) 908 882 dev = dev_net(dev)->loopback_dev; ··· 948 922 return flags; 949 923 } 950 924 951 - static void ip6_rt_init_dst_reject(struct rt6_info *rt, struct fib6_info *ort) 925 + static void ip6_rt_init_dst_reject(struct rt6_info *rt, u8 fib6_type) 952 926 { 953 - rt->dst.error = ip6_rt_type_to_error(ort->fib6_type); 927 + rt->dst.error = ip6_rt_type_to_error(fib6_type); 954 928 955 - switch (ort->fib6_type) { 929 + switch (fib6_type) { 956 930 case RTN_BLACKHOLE: 957 931 rt->dst.output = dst_discard_out; 958 932 rt->dst.input = dst_discard; ··· 970 944 } 971 945 } 972 946 973 - static void ip6_rt_init_dst(struct rt6_info *rt, struct fib6_info *ort) 947 + static void ip6_rt_init_dst(struct rt6_info *rt, const struct fib6_result *res) 974 948 { 975 - if (ort->fib6_flags & RTF_REJECT) { 976 - ip6_rt_init_dst_reject(rt, ort); 949 + struct fib6_info *f6i = res->f6i; 950 + 951 + if (res->fib6_flags & RTF_REJECT) { 952 + ip6_rt_init_dst_reject(rt, res->fib6_type); 977 953 return; 978 954 } 979 955 980 956 rt->dst.error = 0; 981 957 rt->dst.output = ip6_output; 982 958 983 - if (ort->fib6_type == RTN_LOCAL || ort->fib6_type == RTN_ANYCAST) { 959 + if (res->fib6_type == RTN_LOCAL || res->fib6_type == RTN_ANYCAST) { 984 960 rt->dst.input = ip6_input; 985 - } else if (ipv6_addr_type(&ort->fib6_dst.addr) & IPV6_ADDR_MULTICAST) { 961 + } else if (ipv6_addr_type(&f6i->fib6_dst.addr) & IPV6_ADDR_MULTICAST) { 986 962 rt->dst.input = ip6_mc_input; 987 963 } else { 988 964 rt->dst.input = ip6_forward; 989 965 } 990 966 991 - if (ort->fib6_nh.fib_nh_lws) { 992 - rt->dst.lwtstate = lwtstate_get(ort->fib6_nh.fib_nh_lws); 967 + if (res->nh->fib_nh_lws) { 968 + rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws); 993 969 lwtunnel_set_redirect(&rt->dst); 994 970 } 995 971 ··· 1006 978 ip_dst_init_metrics(&rt->dst, from->fib6_metrics); 1007 979 } 1008 980 1009 - /* Caller must already hold reference to @ort */ 1010 - static void ip6_rt_copy_init(struct rt6_info *rt, struct fib6_info *ort) 981 + /* Caller must already hold reference to f6i in result */ 982 + static void ip6_rt_copy_init(struct rt6_info *rt, const struct fib6_result *res) 1011 983 { 1012 - struct net_device *dev = fib6_info_nh_dev(ort); 984 + const struct fib6_nh *nh = res->nh; 985 + const struct net_device *dev = nh->fib_nh_dev; 986 + struct fib6_info *f6i = res->f6i; 1013 987 1014 - ip6_rt_init_dst(rt, ort); 988 + ip6_rt_init_dst(rt, res); 1015 989 1016 - rt->rt6i_dst = ort->fib6_dst; 990 + rt->rt6i_dst = f6i->fib6_dst; 1017 991 rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL; 1018 - rt->rt6i_flags = ort->fib6_flags; 1019 - if (ort->fib6_nh.fib_nh_gw_family) { 1020 - rt->rt6i_gateway = ort->fib6_nh.fib_nh_gw6; 992 + rt->rt6i_flags = res->fib6_flags; 993 + if (nh->fib_nh_gw_family) { 994 + rt->rt6i_gateway = nh->fib_nh_gw6; 1021 995 rt->rt6i_flags |= RTF_GATEWAY; 1022 996 } 1023 - rt6_set_from(rt, ort); 997 + rt6_set_from(rt, f6i); 1024 998 #ifdef CONFIG_IPV6_SUBTREES 1025 - rt->rt6i_src = ort->fib6_src; 999 + rt->rt6i_src = f6i->fib6_src; 1026 1000 #endif 1027 1001 } 1028 1002 ··· 1063 1033 } 1064 1034 1065 1035 /* called with rcu_lock held */ 1066 - static struct rt6_info *ip6_create_rt_rcu(struct fib6_info *rt) 1036 + static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res) 1067 1037 { 1068 - unsigned short flags = fib6_info_dst_flags(rt); 1069 - struct net_device *dev = rt->fib6_nh.fib_nh_dev; 1038 + struct net_device *dev = res->nh->fib_nh_dev; 1039 + struct fib6_info *f6i = res->f6i; 1040 + unsigned short flags; 1070 1041 struct rt6_info *nrt; 1071 1042 1072 - if (!fib6_info_hold_safe(rt)) 1043 + if (!fib6_info_hold_safe(f6i)) 1073 1044 goto fallback; 1074 1045 1046 + flags = fib6_info_dst_flags(f6i); 1075 1047 nrt = ip6_dst_alloc(dev_net(dev), dev, flags); 1076 1048 if (!nrt) { 1077 - fib6_info_release(rt); 1049 + fib6_info_release(f6i); 1078 1050 goto fallback; 1079 1051 } 1080 1052 1081 - ip6_rt_copy_init(nrt, rt); 1053 + ip6_rt_copy_init(nrt, res); 1082 1054 return nrt; 1083 1055 1084 1056 fallback: ··· 1095 1063 const struct sk_buff *skb, 1096 1064 int flags) 1097 1065 { 1098 - struct fib6_info *f6i; 1066 + struct fib6_result res = {}; 1099 1067 struct fib6_node *fn; 1100 1068 struct rt6_info *rt; 1101 1069 ··· 1105 1073 rcu_read_lock(); 1106 1074 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 1107 1075 restart: 1108 - f6i = rcu_dereference(fn->leaf); 1109 - if (!f6i) 1110 - f6i = net->ipv6.fib6_null_entry; 1076 + res.f6i = rcu_dereference(fn->leaf); 1077 + if (!res.f6i) 1078 + res.f6i = net->ipv6.fib6_null_entry; 1111 1079 else 1112 - f6i = rt6_device_match(net, f6i, &fl6->saddr, 1113 - fl6->flowi6_oif, flags); 1080 + rt6_device_match(net, &res, &fl6->saddr, fl6->flowi6_oif, 1081 + flags); 1114 1082 1115 - if (f6i == net->ipv6.fib6_null_entry) { 1083 + if (res.f6i == net->ipv6.fib6_null_entry) { 1116 1084 fn = fib6_backtrack(fn, &fl6->saddr); 1117 1085 if (fn) 1118 1086 goto restart; ··· 1122 1090 goto out; 1123 1091 } 1124 1092 1125 - if (f6i->fib6_nsiblings && fl6->flowi6_oif == 0) 1126 - f6i = fib6_multipath_select(net, f6i, fl6, fl6->flowi6_oif, skb, 1127 - flags); 1093 + fib6_select_path(net, &res, fl6, fl6->flowi6_oif, 1094 + fl6->flowi6_oif != 0, skb, flags); 1095 + 1128 1096 /* Search through exception table */ 1129 - rt = rt6_find_cached_rt(f6i, &fl6->daddr, &fl6->saddr); 1097 + rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 1130 1098 if (rt) { 1131 1099 if (ip6_hold_safe(net, &rt)) 1132 1100 dst_use_noref(&rt->dst, jiffies); 1133 1101 } else { 1134 - rt = ip6_create_rt_rcu(f6i); 1102 + rt = ip6_create_rt_rcu(&res); 1135 1103 } 1136 1104 1137 1105 out: 1138 - trace_fib6_table_lookup(net, f6i, table, fl6); 1106 + trace_fib6_table_lookup(net, &res, table, fl6); 1139 1107 1140 1108 rcu_read_unlock(); 1141 1109 ··· 1202 1170 return __ip6_ins_rt(rt, &info, NULL); 1203 1171 } 1204 1172 1205 - static struct rt6_info *ip6_rt_cache_alloc(struct fib6_info *ort, 1173 + static struct rt6_info *ip6_rt_cache_alloc(const struct fib6_result *res, 1206 1174 const struct in6_addr *daddr, 1207 1175 const struct in6_addr *saddr) 1208 1176 { 1177 + struct fib6_info *f6i = res->f6i; 1209 1178 struct net_device *dev; 1210 1179 struct rt6_info *rt; 1211 1180 ··· 1214 1181 * Clone the route. 1215 1182 */ 1216 1183 1217 - if (!fib6_info_hold_safe(ort)) 1184 + if (!fib6_info_hold_safe(f6i)) 1218 1185 return NULL; 1219 1186 1220 - dev = ip6_rt_get_dev_rcu(ort); 1187 + dev = ip6_rt_get_dev_rcu(res); 1221 1188 rt = ip6_dst_alloc(dev_net(dev), dev, 0); 1222 1189 if (!rt) { 1223 - fib6_info_release(ort); 1190 + fib6_info_release(f6i); 1224 1191 return NULL; 1225 1192 } 1226 1193 1227 - ip6_rt_copy_init(rt, ort); 1194 + ip6_rt_copy_init(rt, res); 1228 1195 rt->rt6i_flags |= RTF_CACHE; 1229 1196 rt->dst.flags |= DST_HOST; 1230 1197 rt->rt6i_dst.addr = *daddr; 1231 1198 rt->rt6i_dst.plen = 128; 1232 1199 1233 - if (!rt6_is_gw_or_nonexthop(ort)) { 1234 - if (ort->fib6_dst.plen != 128 && 1235 - ipv6_addr_equal(&ort->fib6_dst.addr, daddr)) 1200 + if (!rt6_is_gw_or_nonexthop(res)) { 1201 + if (f6i->fib6_dst.plen != 128 && 1202 + ipv6_addr_equal(&f6i->fib6_dst.addr, daddr)) 1236 1203 rt->rt6i_flags |= RTF_ANYCAST; 1237 1204 #ifdef CONFIG_IPV6_SUBTREES 1238 1205 if (rt->rt6i_src.plen && saddr) { ··· 1245 1212 return rt; 1246 1213 } 1247 1214 1248 - static struct rt6_info *ip6_rt_pcpu_alloc(struct fib6_info *rt) 1215 + static struct rt6_info *ip6_rt_pcpu_alloc(const struct fib6_result *res) 1249 1216 { 1250 - unsigned short flags = fib6_info_dst_flags(rt); 1217 + struct fib6_info *f6i = res->f6i; 1218 + unsigned short flags = fib6_info_dst_flags(f6i); 1251 1219 struct net_device *dev; 1252 1220 struct rt6_info *pcpu_rt; 1253 1221 1254 - if (!fib6_info_hold_safe(rt)) 1222 + if (!fib6_info_hold_safe(f6i)) 1255 1223 return NULL; 1256 1224 1257 1225 rcu_read_lock(); 1258 - dev = ip6_rt_get_dev_rcu(rt); 1226 + dev = ip6_rt_get_dev_rcu(res); 1259 1227 pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags); 1260 1228 rcu_read_unlock(); 1261 1229 if (!pcpu_rt) { 1262 - fib6_info_release(rt); 1230 + fib6_info_release(f6i); 1263 1231 return NULL; 1264 1232 } 1265 - ip6_rt_copy_init(pcpu_rt, rt); 1233 + ip6_rt_copy_init(pcpu_rt, res); 1266 1234 pcpu_rt->rt6i_flags |= RTF_PCPU; 1267 1235 return pcpu_rt; 1268 1236 } 1269 1237 1270 1238 /* It should be called with rcu_read_lock() acquired */ 1271 - static struct rt6_info *rt6_get_pcpu_route(struct fib6_info *rt) 1239 + static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res) 1272 1240 { 1273 1241 struct rt6_info *pcpu_rt, **p; 1274 1242 1275 - p = this_cpu_ptr(rt->rt6i_pcpu); 1243 + p = this_cpu_ptr(res->f6i->rt6i_pcpu); 1276 1244 pcpu_rt = *p; 1277 1245 1278 1246 if (pcpu_rt) ··· 1283 1249 } 1284 1250 1285 1251 static struct rt6_info *rt6_make_pcpu_route(struct net *net, 1286 - struct fib6_info *rt) 1252 + const struct fib6_result *res) 1287 1253 { 1288 1254 struct rt6_info *pcpu_rt, *prev, **p; 1289 1255 1290 - pcpu_rt = ip6_rt_pcpu_alloc(rt); 1256 + pcpu_rt = ip6_rt_pcpu_alloc(res); 1291 1257 if (!pcpu_rt) { 1292 1258 dst_hold(&net->ipv6.ip6_null_entry->dst); 1293 1259 return net->ipv6.ip6_null_entry; 1294 1260 } 1295 1261 1296 1262 dst_hold(&pcpu_rt->dst); 1297 - p = this_cpu_ptr(rt->rt6i_pcpu); 1263 + p = this_cpu_ptr(res->f6i->rt6i_pcpu); 1298 1264 prev = cmpxchg(p, NULL, pcpu_rt); 1299 1265 BUG_ON(prev); 1300 1266 ··· 1437 1403 return NULL; 1438 1404 } 1439 1405 1440 - static unsigned int fib6_mtu(const struct fib6_info *rt) 1406 + static unsigned int fib6_mtu(const struct fib6_result *res) 1441 1407 { 1408 + const struct fib6_nh *nh = res->nh; 1442 1409 unsigned int mtu; 1443 1410 1444 - if (rt->fib6_pmtu) { 1445 - mtu = rt->fib6_pmtu; 1411 + if (res->f6i->fib6_pmtu) { 1412 + mtu = res->f6i->fib6_pmtu; 1446 1413 } else { 1447 - struct net_device *dev = fib6_info_nh_dev(rt); 1414 + struct net_device *dev = nh->fib_nh_dev; 1448 1415 struct inet6_dev *idev; 1449 1416 1450 1417 rcu_read_lock(); ··· 1456 1421 1457 1422 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 1458 1423 1459 - return mtu - lwtunnel_headroom(rt->fib6_nh.fib_nh_lws, mtu); 1424 + return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 1460 1425 } 1461 1426 1462 1427 static int rt6_insert_exception(struct rt6_info *nrt, 1463 - struct fib6_info *ort) 1428 + const struct fib6_result *res) 1464 1429 { 1465 1430 struct net *net = dev_net(nrt->dst.dev); 1466 1431 struct rt6_exception_bucket *bucket; 1467 1432 struct in6_addr *src_key = NULL; 1468 1433 struct rt6_exception *rt6_ex; 1434 + struct fib6_info *f6i = res->f6i; 1469 1435 int err = 0; 1470 1436 1471 1437 spin_lock_bh(&rt6_exception_lock); 1472 1438 1473 - if (ort->exception_bucket_flushed) { 1439 + if (f6i->exception_bucket_flushed) { 1474 1440 err = -EINVAL; 1475 1441 goto out; 1476 1442 } 1477 1443 1478 - bucket = rcu_dereference_protected(ort->rt6i_exception_bucket, 1444 + bucket = rcu_dereference_protected(f6i->rt6i_exception_bucket, 1479 1445 lockdep_is_held(&rt6_exception_lock)); 1480 1446 if (!bucket) { 1481 1447 bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket), ··· 1485 1449 err = -ENOMEM; 1486 1450 goto out; 1487 1451 } 1488 - rcu_assign_pointer(ort->rt6i_exception_bucket, bucket); 1452 + rcu_assign_pointer(f6i->rt6i_exception_bucket, bucket); 1489 1453 } 1490 1454 1491 1455 #ifdef CONFIG_IPV6_SUBTREES 1492 - /* rt6i_src.plen != 0 indicates ort is in subtree 1456 + /* fib6_src.plen != 0 indicates f6i is in subtree 1493 1457 * and exception table is indexed by a hash of 1494 - * both rt6i_dst and rt6i_src. 1458 + * both fib6_dst and fib6_src. 1495 1459 * Otherwise, the exception table is indexed by 1496 - * a hash of only rt6i_dst. 1460 + * a hash of only fib6_dst. 1497 1461 */ 1498 - if (ort->fib6_src.plen) 1462 + if (f6i->fib6_src.plen) 1499 1463 src_key = &nrt->rt6i_src.addr; 1500 1464 #endif 1501 - /* rt6_mtu_change() might lower mtu on ort. 1465 + /* rt6_mtu_change() might lower mtu on f6i. 1502 1466 * Only insert this exception route if its mtu 1503 - * is less than ort's mtu value. 1467 + * is less than f6i's mtu value. 1504 1468 */ 1505 - if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(ort)) { 1469 + if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(res)) { 1506 1470 err = -EINVAL; 1507 1471 goto out; 1508 1472 } ··· 1531 1495 1532 1496 /* Update fn->fn_sernum to invalidate all cached dst */ 1533 1497 if (!err) { 1534 - spin_lock_bh(&ort->fib6_table->tb6_lock); 1535 - fib6_update_sernum(net, ort); 1536 - spin_unlock_bh(&ort->fib6_table->tb6_lock); 1498 + spin_lock_bh(&f6i->fib6_table->tb6_lock); 1499 + fib6_update_sernum(net, f6i); 1500 + spin_unlock_bh(&f6i->fib6_table->tb6_lock); 1537 1501 fib6_force_start_gc(net); 1538 1502 } 1539 1503 ··· 1570 1534 /* Find cached rt in the hash table inside passed in rt 1571 1535 * Caller has to hold rcu_read_lock() 1572 1536 */ 1573 - static struct rt6_info *rt6_find_cached_rt(struct fib6_info *rt, 1537 + static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 1574 1538 struct in6_addr *daddr, 1575 1539 struct in6_addr *saddr) 1576 1540 { 1577 1541 struct rt6_exception_bucket *bucket; 1578 1542 struct in6_addr *src_key = NULL; 1579 1543 struct rt6_exception *rt6_ex; 1580 - struct rt6_info *res = NULL; 1544 + struct rt6_info *ret = NULL; 1581 1545 1582 - bucket = rcu_dereference(rt->rt6i_exception_bucket); 1546 + bucket = rcu_dereference(res->f6i->rt6i_exception_bucket); 1583 1547 1584 1548 #ifdef CONFIG_IPV6_SUBTREES 1585 - /* rt6i_src.plen != 0 indicates rt is in subtree 1549 + /* fib6i_src.plen != 0 indicates f6i is in subtree 1586 1550 * and exception table is indexed by a hash of 1587 - * both rt6i_dst and rt6i_src. 1551 + * both fib6_dst and fib6_src. 1588 1552 * Otherwise, the exception table is indexed by 1589 - * a hash of only rt6i_dst. 1553 + * a hash of only fib6_dst. 1590 1554 */ 1591 - if (rt->fib6_src.plen) 1555 + if (res->f6i->fib6_src.plen) 1592 1556 src_key = saddr; 1593 1557 #endif 1594 1558 rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key); 1595 1559 1596 1560 if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i)) 1597 - res = rt6_ex->rt6i; 1561 + ret = rt6_ex->rt6i; 1598 1562 1599 - return res; 1563 + return ret; 1600 1564 } 1601 1565 1602 1566 /* Remove the passed in cached rt from the hash table that contains it */ ··· 1844 1808 } 1845 1809 1846 1810 /* must be called with rcu lock held */ 1847 - struct fib6_info *fib6_table_lookup(struct net *net, struct fib6_table *table, 1848 - int oif, struct flowi6 *fl6, int strict) 1811 + int fib6_table_lookup(struct net *net, struct fib6_table *table, int oif, 1812 + struct flowi6 *fl6, struct fib6_result *res, int strict) 1849 1813 { 1850 1814 struct fib6_node *fn, *saved_fn; 1851 - struct fib6_info *f6i; 1852 1815 1853 1816 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 1854 1817 saved_fn = fn; ··· 1856 1821 oif = 0; 1857 1822 1858 1823 redo_rt6_select: 1859 - f6i = rt6_select(net, fn, oif, strict); 1860 - if (f6i == net->ipv6.fib6_null_entry) { 1824 + rt6_select(net, fn, oif, res, strict); 1825 + if (res->f6i == net->ipv6.fib6_null_entry) { 1861 1826 fn = fib6_backtrack(fn, &fl6->saddr); 1862 1827 if (fn) 1863 1828 goto redo_rt6_select; ··· 1869 1834 } 1870 1835 } 1871 1836 1872 - trace_fib6_table_lookup(net, f6i, table, fl6); 1837 + trace_fib6_table_lookup(net, res, table, fl6); 1873 1838 1874 - return f6i; 1839 + return 0; 1875 1840 } 1876 1841 1877 1842 struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table, 1878 1843 int oif, struct flowi6 *fl6, 1879 1844 const struct sk_buff *skb, int flags) 1880 1845 { 1881 - struct fib6_info *f6i; 1846 + struct fib6_result res = {}; 1882 1847 struct rt6_info *rt; 1883 1848 int strict = 0; 1884 1849 ··· 1889 1854 1890 1855 rcu_read_lock(); 1891 1856 1892 - f6i = fib6_table_lookup(net, table, oif, fl6, strict); 1893 - if (f6i == net->ipv6.fib6_null_entry) { 1857 + fib6_table_lookup(net, table, oif, fl6, &res, strict); 1858 + if (res.f6i == net->ipv6.fib6_null_entry) { 1894 1859 rt = net->ipv6.ip6_null_entry; 1895 1860 rcu_read_unlock(); 1896 1861 dst_hold(&rt->dst); 1897 1862 return rt; 1898 1863 } 1899 1864 1900 - if (f6i->fib6_nsiblings) 1901 - f6i = fib6_multipath_select(net, f6i, fl6, oif, skb, strict); 1865 + fib6_select_path(net, &res, fl6, oif, false, skb, strict); 1902 1866 1903 1867 /*Search through exception table */ 1904 - rt = rt6_find_cached_rt(f6i, &fl6->daddr, &fl6->saddr); 1868 + rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 1905 1869 if (rt) { 1906 1870 if (ip6_hold_safe(net, &rt)) 1907 1871 dst_use_noref(&rt->dst, jiffies); ··· 1908 1874 rcu_read_unlock(); 1909 1875 return rt; 1910 1876 } else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) && 1911 - !f6i->fib6_nh.fib_nh_gw_family)) { 1877 + !res.nh->fib_nh_gw_family)) { 1912 1878 /* Create a RTF_CACHE clone which will not be 1913 1879 * owned by the fib6 tree. It is for the special case where 1914 1880 * the daddr in the skb during the neighbor look-up is different ··· 1916 1882 */ 1917 1883 struct rt6_info *uncached_rt; 1918 1884 1919 - uncached_rt = ip6_rt_cache_alloc(f6i, &fl6->daddr, NULL); 1885 + uncached_rt = ip6_rt_cache_alloc(&res, &fl6->daddr, NULL); 1920 1886 1921 1887 rcu_read_unlock(); 1922 1888 ··· 1938 1904 struct rt6_info *pcpu_rt; 1939 1905 1940 1906 local_bh_disable(); 1941 - pcpu_rt = rt6_get_pcpu_route(f6i); 1907 + pcpu_rt = rt6_get_pcpu_route(&res); 1942 1908 1943 1909 if (!pcpu_rt) 1944 - pcpu_rt = rt6_make_pcpu_route(net, f6i); 1910 + pcpu_rt = rt6_make_pcpu_route(net, &res); 1945 1911 1946 1912 local_bh_enable(); 1947 1913 rcu_read_unlock(); ··· 2360 2326 if (rt6->rt6i_flags & RTF_CACHE) 2361 2327 rt6_update_exception_stamp_rt(rt6); 2362 2328 } else if (daddr) { 2363 - struct fib6_info *from; 2329 + struct fib6_result res = {}; 2364 2330 struct rt6_info *nrt6; 2365 2331 2366 2332 rcu_read_lock(); 2367 - from = rcu_dereference(rt6->from); 2368 - if (!from) { 2333 + res.f6i = rcu_dereference(rt6->from); 2334 + if (!res.f6i) { 2369 2335 rcu_read_unlock(); 2370 2336 return; 2371 2337 } 2372 - nrt6 = ip6_rt_cache_alloc(from, daddr, saddr); 2338 + res.nh = &res.f6i->fib6_nh; 2339 + res.fib6_flags = res.f6i->fib6_flags; 2340 + res.fib6_type = res.f6i->fib6_type; 2341 + 2342 + nrt6 = ip6_rt_cache_alloc(&res, daddr, saddr); 2373 2343 if (nrt6) { 2374 2344 rt6_do_update_pmtu(nrt6, mtu); 2375 - if (rt6_insert_exception(nrt6, from)) 2345 + if (rt6_insert_exception(nrt6, &res)) 2376 2346 dst_release_immediate(&nrt6->dst); 2377 2347 } 2378 2348 rcu_read_unlock(); ··· 2449 2411 NULL); 2450 2412 } 2451 2413 2452 - static bool ip6_redirect_nh_match(struct fib6_info *f6i, 2453 - struct fib6_nh *nh, 2414 + static bool ip6_redirect_nh_match(const struct fib6_result *res, 2454 2415 struct flowi6 *fl6, 2455 2416 const struct in6_addr *gw, 2456 2417 struct rt6_info **ret) 2457 2418 { 2419 + const struct fib6_nh *nh = res->nh; 2420 + 2458 2421 if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family || 2459 2422 fl6->flowi6_oif != nh->fib_nh_dev->ifindex) 2460 2423 return false; ··· 2468 2429 if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) { 2469 2430 struct rt6_info *rt_cache; 2470 2431 2471 - rt_cache = rt6_find_cached_rt(f6i, &fl6->daddr, &fl6->saddr); 2432 + rt_cache = rt6_find_cached_rt(res, &fl6->daddr, &fl6->saddr); 2472 2433 if (rt_cache && 2473 2434 ipv6_addr_equal(gw, &rt_cache->rt6i_gateway)) { 2474 2435 *ret = rt_cache; ··· 2493 2454 { 2494 2455 struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6; 2495 2456 struct rt6_info *ret = NULL; 2457 + struct fib6_result res = {}; 2496 2458 struct fib6_info *rt; 2497 2459 struct fib6_node *fn; 2498 2460 ··· 2511 2471 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 2512 2472 restart: 2513 2473 for_each_fib6_node_rt_rcu(fn) { 2474 + res.f6i = rt; 2475 + res.nh = &rt->fib6_nh; 2476 + 2514 2477 if (fib6_check_expired(rt)) 2515 2478 continue; 2516 2479 if (rt->fib6_flags & RTF_REJECT) 2517 2480 break; 2518 - if (ip6_redirect_nh_match(rt, &rt->fib6_nh, fl6, 2519 - &rdfl->gateway, &ret)) 2481 + if (ip6_redirect_nh_match(&res, fl6, &rdfl->gateway, &ret)) 2520 2482 goto out; 2521 2483 } 2522 2484 ··· 2535 2493 goto restart; 2536 2494 } 2537 2495 2496 + res.f6i = rt; 2497 + res.nh = &rt->fib6_nh; 2538 2498 out: 2539 - if (ret) 2499 + if (ret) { 2540 2500 ip6_hold_safe(net, &ret); 2541 - else 2542 - ret = ip6_create_rt_rcu(rt); 2501 + } else { 2502 + res.fib6_flags = res.f6i->fib6_flags; 2503 + res.fib6_type = res.f6i->fib6_type; 2504 + ret = ip6_create_rt_rcu(&res); 2505 + } 2543 2506 2544 2507 rcu_read_unlock(); 2545 2508 2546 - trace_fib6_table_lookup(net, rt, table, fl6); 2509 + trace_fib6_table_lookup(net, &res, table, fl6); 2547 2510 return ret; 2548 2511 }; 2549 2512 ··· 2666 2619 * based on ip6_dst_mtu_forward and exception logic of 2667 2620 * rt6_find_cached_rt; called with rcu_read_lock 2668 2621 */ 2669 - u32 ip6_mtu_from_fib6(struct fib6_info *f6i, struct in6_addr *daddr, 2670 - struct in6_addr *saddr) 2622 + u32 ip6_mtu_from_fib6(const struct fib6_result *res, 2623 + const struct in6_addr *daddr, 2624 + const struct in6_addr *saddr) 2671 2625 { 2672 2626 struct rt6_exception_bucket *bucket; 2627 + const struct fib6_nh *nh = res->nh; 2628 + struct fib6_info *f6i = res->f6i; 2629 + const struct in6_addr *src_key; 2673 2630 struct rt6_exception *rt6_ex; 2674 - struct in6_addr *src_key; 2675 2631 struct inet6_dev *idev; 2676 2632 u32 mtu = 0; 2677 2633 ··· 2696 2646 mtu = dst_metric_raw(&rt6_ex->rt6i->dst, RTAX_MTU); 2697 2647 2698 2648 if (likely(!mtu)) { 2699 - struct net_device *dev = fib6_info_nh_dev(f6i); 2649 + struct net_device *dev = nh->fib_nh_dev; 2700 2650 2701 2651 mtu = IPV6_MIN_MTU; 2702 2652 idev = __in6_dev_get(dev); ··· 2706 2656 2707 2657 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 2708 2658 out: 2709 - return mtu - lwtunnel_headroom(fib6_info_nh_lwt(f6i), mtu); 2659 + return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 2710 2660 } 2711 2661 2712 2662 struct dst_entry *icmp6_dst_alloc(struct net_device *dev, ··· 3358 3308 struct fib6_nh *nh; 3359 3309 3360 3310 if (cfg->fc_flags & RTF_CACHE) { 3311 + struct fib6_result res = { 3312 + .f6i = rt, 3313 + }; 3361 3314 int rc; 3362 3315 3363 - rt_cache = rt6_find_cached_rt(rt, &cfg->fc_dst, 3316 + rt_cache = rt6_find_cached_rt(&res, 3317 + &cfg->fc_dst, 3364 3318 &cfg->fc_src); 3365 3319 if (rt_cache) { 3366 3320 rc = ip6_del_cached_rt(rt_cache, cfg); ··· 3408 3354 { 3409 3355 struct netevent_redirect netevent; 3410 3356 struct rt6_info *rt, *nrt = NULL; 3357 + struct fib6_result res = {}; 3411 3358 struct ndisc_options ndopts; 3412 3359 struct inet6_dev *in6_dev; 3413 3360 struct neighbour *neigh; 3414 - struct fib6_info *from; 3415 3361 struct rd_msg *msg; 3416 3362 int optlen, on_link; 3417 3363 u8 *lladdr; ··· 3494 3440 NDISC_REDIRECT, &ndopts); 3495 3441 3496 3442 rcu_read_lock(); 3497 - from = rcu_dereference(rt->from); 3443 + res.f6i = rcu_dereference(rt->from); 3498 3444 /* This fib6_info_hold() is safe here because we hold reference to rt 3499 3445 * and rt already holds reference to fib6_info. 3500 3446 */ 3501 - fib6_info_hold(from); 3447 + fib6_info_hold(res.f6i); 3502 3448 rcu_read_unlock(); 3503 3449 3504 - nrt = ip6_rt_cache_alloc(from, &msg->dest, NULL); 3450 + res.nh = &res.f6i->fib6_nh; 3451 + res.fib6_flags = res.f6i->fib6_flags; 3452 + res.fib6_type = res.f6i->fib6_type; 3453 + nrt = ip6_rt_cache_alloc(&res, &msg->dest, NULL); 3505 3454 if (!nrt) 3506 3455 goto out; 3507 3456 ··· 3518 3461 * a cached route because rt6_insert_exception() will 3519 3462 * takes care of it 3520 3463 */ 3521 - if (rt6_insert_exception(nrt, from)) { 3464 + if (rt6_insert_exception(nrt, &res)) { 3522 3465 dst_release_immediate(&nrt->dst); 3523 3466 goto out; 3524 3467 } ··· 3530 3473 call_netevent_notifiers(NETEVENT_REDIRECT, &netevent); 3531 3474 3532 3475 out: 3533 - fib6_info_release(from); 3476 + fib6_info_release(res.f6i); 3534 3477 neigh_release(neigh); 3535 3478 } 3536 3479

Configure Feed

Configure Feed