tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge branch 'ipv6-misc-changes-in-output-path'

Eric Dumazet says:

====================
ipv6: misc changes in output path

Small optimizations mostly in ip6_xmit() path.

TX performance increases by about 3 %.

Patches 5-7: add dst4_mtu() and dst6_mtu() to save space.

Last patch colocates inet6_cork in inet_cork_full.

This series reduces kernel size by 494 bytes on x86_64:

scripts/bloat-o-meter -t vmlinux.old vmlinux.new
add/remove: 4/2 grow/shrink: 9/23 up/down: 665/-1159 (-494)
Function old new delta
ip6_finish_output_gso_slowpath_drop - 197 +197
ip6_xmit 1452 1595 +143
do_ipv6_getsockopt 2855 2950 +95
kzalloc_noprof - 55 +55
ip4ip6_err 918 955 +37
__icmp_send 1499 1532 +33
do_ip_getsockopt 2573 2605 +32
__ip6_append_data 4109 4137 +28
__pfx_kzalloc_noprof - 16 +16
__pfx_ip6_finish_output_gso_slowpath_drop - 16 +16
ipmr_prepare_xmit 1232 1238 +6
ip6_forward 1905 1909 +4
ip6_cork_release 108 111 +3
ipv6_push_nfrag_opts 489 486 -3
ipv6_push_frag_opts 90 87 -3
ip6_finish_output2 1446 1437 -9
ip6_tnl_xmit 2639 2627 -12
ip6_default_advmss 176 160 -16
__ip6_rt_update_pmtu 1087 1071 -16
tcp_v6_syn_recv_sock 1715 1696 -19
tcp_v4_syn_recv_sock 1107 1088 -19
__ip_make_skb 1339 1320 -19
ip_setup_cork 406 385 -21
ip6_setup_cork 732 710 -22
rawv6_push_pending_frames 581 556 -25
ip6_push_pending_frames 184 157 -27
udpv6_splice_eof 203 170 -33
ip6_flush_pending_frames 220 183 -37
ip6_append_data 349 312 -37
udp_v6_push_pending_frames 155 115 -40
sit_tunnel_xmit 1957 1914 -43
__pfx_dst_mtu 64 - -64
tcp_v4_mtu_reduced 289 220 -69
tcp_v6_mtu_reduced 209 139 -70
ip6_make_skb 574 484 -90
ip6_finish_output 827 697 -130
dst_mtu 160 - -160
fib6_nh_mtu_change 511 336 -175
Total: Before=22584400, After=22583906, chg -0.00%
====================

Link: https://patch.msgid.link/20260130210303.3888261-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

Jakub Kicinski 8755aae4 e0221553

+144 -125
-8
include/linux/ipv6.h
··· 205 205 struct ipv6_ac_socklist; 206 206 struct ipv6_fl_socklist; 207 207 208 - struct inet6_cork { 209 - struct ipv6_txoptions *opt; 210 - u8 hop_limit; 211 - u8 tclass; 212 - u8 dontfrag:1; 213 - }; 214 - 215 208 /* struct ipv6_pinfo - ipv6 private area */ 216 209 struct ipv6_pinfo { 217 210 /* Used in tx path (inet6_csk_route_socket(), ip6_xmit()) */ ··· 260 267 261 268 struct sk_buff *pktoptions; 262 269 struct sk_buff *rxpmtu; 263 - struct inet6_cork cork; 264 270 265 271 struct ipv6_mc_socklist __rcu *ipv6_mc_list; 266 272 struct ipv6_ac_socklist *ipv6_ac_list;
+6
include/net/dst.h
··· 219 219 return INDIRECT_CALL_INET(dst->ops->mtu, ip6_mtu, ipv4_mtu, dst); 220 220 } 221 221 222 + /* Variant of dst_mtu() for IPv4 users. */ 223 + static inline u32 dst4_mtu(const struct dst_entry *dst) 224 + { 225 + return INDIRECT_CALL_1(dst->ops->mtu, ipv4_mtu, dst); 226 + } 227 + 222 228 /* RTT metrics are stored in milliseconds for user ABI, but used as jiffies */ 223 229 static inline unsigned long dst_metric_rtt(const struct dst_entry *dst, int metric) 224 230 {
+10
include/net/inet_sock.h
··· 159 159 #endif 160 160 } 161 161 162 + struct inet6_cork { 163 + struct ipv6_txoptions *opt; 164 + u8 hop_limit; 165 + u8 tclass; 166 + u8 dontfrag:1; 167 + }; 168 + 162 169 struct inet_cork { 163 170 unsigned int flags; 164 171 __be32 addr; ··· 186 179 struct inet_cork_full { 187 180 struct inet_cork base; 188 181 struct flowi fl; 182 + #if IS_ENABLED(CONFIG_IPV6) 183 + struct inet6_cork base6; 184 + #endif 189 185 }; 190 186 191 187 struct ip_mc_socklist;
+6
include/net/ip6_route.h
··· 266 266 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, 267 267 int (*output)(struct net *, struct sock *, struct sk_buff *)); 268 268 269 + /* Variant of dst_mtu() for IPv6 users */ 270 + static inline u32 dst6_mtu(const struct dst_entry *dst) 271 + { 272 + return INDIRECT_CALL_1(dst->ops->mtu, ip6_mtu, dst); 273 + } 274 + 269 275 static inline unsigned int ip6_skb_dst_mtu(const struct sk_buff *skb) 270 276 { 271 277 const struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ?
+7 -9
include/net/ipv6.h
··· 1107 1107 int ip6_send_skb(struct sk_buff *skb); 1108 1108 1109 1109 struct sk_buff *__ip6_make_skb(struct sock *sk, struct sk_buff_head *queue, 1110 - struct inet_cork_full *cork, 1111 - struct inet6_cork *v6_cork); 1110 + struct inet_cork_full *cork); 1112 1111 struct sk_buff *ip6_make_skb(struct sock *sk, 1113 1112 int getfrag(void *from, char *to, int offset, 1114 1113 int len, int odd, struct sk_buff *skb), ··· 1118 1119 1119 1120 static inline struct sk_buff *ip6_finish_skb(struct sock *sk) 1120 1121 { 1121 - return __ip6_make_skb(sk, &sk->sk_write_queue, &inet_sk(sk)->cork, 1122 - &inet6_sk(sk)->cork); 1122 + return __ip6_make_skb(sk, &sk->sk_write_queue, &inet_sk(sk)->cork); 1123 1123 } 1124 1124 1125 1125 int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst, ··· 1149 1151 * Extension header (options) processing 1150 1152 */ 1151 1153 1152 - void ipv6_push_nfrag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, 1153 - u8 *proto, struct in6_addr **daddr_p, 1154 - struct in6_addr *saddr); 1155 - void ipv6_push_frag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, 1156 - u8 *proto); 1154 + u8 ipv6_push_nfrag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, 1155 + u8 proto, struct in6_addr **daddr_p, 1156 + struct in6_addr *saddr); 1157 + u8 ipv6_push_frag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, 1158 + u8 proto); 1157 1159 1158 1160 int ipv6_skip_exthdr(const struct sk_buff *, int start, u8 *nexthdrp, 1159 1161 __be16 *frag_offp);
+1 -1
net/ipv4/icmp.c
··· 945 945 946 946 /* RFC says return as much as we can without exceeding 576 bytes. */ 947 947 948 - room = dst_mtu(&rt->dst); 948 + room = dst4_mtu(&rt->dst); 949 949 if (room > 576) 950 950 room = 576; 951 951 room -= sizeof(struct iphdr) + icmp_param->replyopts.opt.optlen;
+2 -2
net/ipv4/ip_output.c
··· 1300 1300 return -EFAULT; 1301 1301 1302 1302 cork->fragsize = ip_sk_use_pmtu(sk) ? 1303 - dst_mtu(&rt->dst) : READ_ONCE(rt->dst.dev->mtu); 1303 + dst4_mtu(&rt->dst) : READ_ONCE(rt->dst.dev->mtu); 1304 1304 1305 1305 if (!inetdev_valid_mtu(cork->fragsize)) 1306 1306 return -ENETUNREACH; ··· 1439 1439 pmtudisc = READ_ONCE(inet->pmtudisc); 1440 1440 if (pmtudisc == IP_PMTUDISC_DO || 1441 1441 pmtudisc == IP_PMTUDISC_PROBE || 1442 - (skb->len <= dst_mtu(&rt->dst) && 1442 + (skb->len <= dst4_mtu(&rt->dst) && 1443 1443 ip_dont_fragment(sk, &rt->dst))) 1444 1444 df = htons(IP_DF); 1445 1445
+1 -1
net/ipv4/ip_sockglue.c
··· 1634 1634 val = 0; 1635 1635 dst = sk_dst_get(sk); 1636 1636 if (dst) { 1637 - val = dst_mtu(dst); 1637 + val = dst4_mtu(dst); 1638 1638 dst_release(dst); 1639 1639 } 1640 1640 if (!val)
+1 -1
net/ipv4/ipmr.c
··· 1895 1895 return -1; 1896 1896 } 1897 1897 1898 - if (skb->len+encap > dst_mtu(&rt->dst) && (ntohs(iph->frag_off) & IP_DF)) { 1898 + if (skb->len+encap > dst4_mtu(&rt->dst) && (ntohs(iph->frag_off) & IP_DF)) { 1899 1899 /* Do not fragment multicasts. Alas, IPv4 does not 1900 1900 * allow to send ICMP, so that packets will disappear 1901 1901 * to blackhole.
+1 -1
net/ipv4/netfilter/nf_reject_ipv4.c
··· 303 303 goto free_nskb; 304 304 305 305 /* "Never happens" */ 306 - if (nskb->len > dst_mtu(skb_dst(nskb))) 306 + if (nskb->len > dst4_mtu(skb_dst(nskb))) 307 307 goto free_nskb; 308 308 309 309 nf_ct_attach(nskb, oldskb);
+6 -7
net/ipv4/tcp_ipv4.c
··· 374 374 { 375 375 struct inet_sock *inet = inet_sk(sk); 376 376 struct dst_entry *dst; 377 - u32 mtu; 377 + u32 mtu, dmtu; 378 378 379 379 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) 380 380 return; ··· 386 386 /* Something is about to be wrong... Remember soft error 387 387 * for the case, if this connection will not able to recover. 388 388 */ 389 - if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst)) 389 + dmtu = dst4_mtu(dst); 390 + if (mtu < dmtu && ip_dont_fragment(sk, dst)) 390 391 WRITE_ONCE(sk->sk_err_soft, EMSGSIZE); 391 - 392 - mtu = dst_mtu(dst); 393 392 394 393 if (inet->pmtudisc != IP_PMTUDISC_DONT && 395 394 ip_sk_accept_pmtu(sk) && 396 - inet_csk(sk)->icsk_pmtu_cookie > mtu) { 397 - tcp_sync_mss(sk, mtu); 395 + inet_csk(sk)->icsk_pmtu_cookie > dmtu) { 396 + tcp_sync_mss(sk, dmtu); 398 397 399 398 /* Resend the TCP packet because it's 400 399 * clear that the old packet has been ··· 1759 1760 1760 1761 tcp_ca_openreq_child(newsk, dst); 1761 1762 1762 - tcp_sync_mss(newsk, dst_mtu(dst)); 1763 + tcp_sync_mss(newsk, dst4_mtu(dst)); 1763 1764 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); 1764 1765 1765 1766 tcp_initialize_rcv_mss(newsk);
+29 -26
net/ipv6/exthdrs.c
··· 1074 1074 * for headers. 1075 1075 */ 1076 1076 1077 - static void ipv6_push_rthdr0(struct sk_buff *skb, u8 *proto, 1078 - struct ipv6_rt_hdr *opt, 1079 - struct in6_addr **addr_p, struct in6_addr *saddr) 1077 + static u8 ipv6_push_rthdr0(struct sk_buff *skb, u8 proto, 1078 + struct ipv6_rt_hdr *opt, 1079 + struct in6_addr **addr_p, struct in6_addr *saddr) 1080 1080 { 1081 1081 struct rt0_hdr *phdr, *ihdr; 1082 1082 int hops; ··· 1095 1095 phdr->addr[hops - 1] = **addr_p; 1096 1096 *addr_p = ihdr->addr; 1097 1097 1098 - phdr->rt_hdr.nexthdr = *proto; 1099 - *proto = NEXTHDR_ROUTING; 1098 + phdr->rt_hdr.nexthdr = proto; 1099 + return NEXTHDR_ROUTING; 1100 1100 } 1101 1101 1102 - static void ipv6_push_rthdr4(struct sk_buff *skb, u8 *proto, 1103 - struct ipv6_rt_hdr *opt, 1104 - struct in6_addr **addr_p, struct in6_addr *saddr) 1102 + static u8 ipv6_push_rthdr4(struct sk_buff *skb, u8 proto, 1103 + struct ipv6_rt_hdr *opt, 1104 + struct in6_addr **addr_p, struct in6_addr *saddr) 1105 1105 { 1106 1106 struct ipv6_sr_hdr *sr_phdr, *sr_ihdr; 1107 1107 int plen, hops; ··· 1144 1144 } 1145 1145 #endif 1146 1146 1147 - sr_phdr->nexthdr = *proto; 1148 - *proto = NEXTHDR_ROUTING; 1147 + sr_phdr->nexthdr = proto; 1148 + return NEXTHDR_ROUTING; 1149 1149 } 1150 1150 1151 - static void ipv6_push_rthdr(struct sk_buff *skb, u8 *proto, 1152 - struct ipv6_rt_hdr *opt, 1153 - struct in6_addr **addr_p, struct in6_addr *saddr) 1151 + static u8 ipv6_push_rthdr(struct sk_buff *skb, u8 proto, 1152 + struct ipv6_rt_hdr *opt, 1153 + struct in6_addr **addr_p, struct in6_addr *saddr) 1154 1154 { 1155 1155 switch (opt->type) { 1156 1156 case IPV6_SRCRT_TYPE_0: 1157 1157 case IPV6_SRCRT_STRICT: 1158 1158 case IPV6_SRCRT_TYPE_2: 1159 - ipv6_push_rthdr0(skb, proto, opt, addr_p, saddr); 1159 + proto = ipv6_push_rthdr0(skb, proto, opt, addr_p, saddr); 1160 1160 break; 1161 1161 case IPV6_SRCRT_TYPE_4: 1162 - ipv6_push_rthdr4(skb, proto, opt, addr_p, saddr); 1162 + proto = ipv6_push_rthdr4(skb, proto, opt, addr_p, saddr); 1163 1163 break; 1164 1164 default: 1165 1165 break; 1166 1166 } 1167 + return proto; 1167 1168 } 1168 1169 1169 - static void ipv6_push_exthdr(struct sk_buff *skb, u8 *proto, u8 type, struct ipv6_opt_hdr *opt) 1170 + static u8 ipv6_push_exthdr(struct sk_buff *skb, u8 proto, u8 type, struct ipv6_opt_hdr *opt) 1170 1171 { 1171 1172 struct ipv6_opt_hdr *h = skb_push(skb, ipv6_optlen(opt)); 1172 1173 1173 1174 memcpy(h, opt, ipv6_optlen(opt)); 1174 - h->nexthdr = *proto; 1175 - *proto = type; 1175 + h->nexthdr = proto; 1176 + return type; 1176 1177 } 1177 1178 1178 - void ipv6_push_nfrag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, 1179 - u8 *proto, 1180 - struct in6_addr **daddr, struct in6_addr *saddr) 1179 + u8 ipv6_push_nfrag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, 1180 + u8 proto, 1181 + struct in6_addr **daddr, struct in6_addr *saddr) 1181 1182 { 1182 1183 if (opt->srcrt) { 1183 - ipv6_push_rthdr(skb, proto, opt->srcrt, daddr, saddr); 1184 + proto = ipv6_push_rthdr(skb, proto, opt->srcrt, daddr, saddr); 1184 1185 /* 1185 1186 * IPV6_RTHDRDSTOPTS is ignored 1186 1187 * unless IPV6_RTHDR is set (RFC3542). 1187 1188 */ 1188 1189 if (opt->dst0opt) 1189 - ipv6_push_exthdr(skb, proto, NEXTHDR_DEST, opt->dst0opt); 1190 + proto = ipv6_push_exthdr(skb, proto, NEXTHDR_DEST, opt->dst0opt); 1190 1191 } 1191 1192 if (opt->hopopt) 1192 - ipv6_push_exthdr(skb, proto, NEXTHDR_HOP, opt->hopopt); 1193 + proto = ipv6_push_exthdr(skb, proto, NEXTHDR_HOP, opt->hopopt); 1194 + return proto; 1193 1195 } 1194 1196 1195 - void ipv6_push_frag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, u8 *proto) 1197 + u8 ipv6_push_frag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, u8 proto) 1196 1198 { 1197 1199 if (opt->dst1opt) 1198 - ipv6_push_exthdr(skb, proto, NEXTHDR_DEST, opt->dst1opt); 1200 + proto = ipv6_push_exthdr(skb, proto, NEXTHDR_DEST, opt->dst1opt); 1201 + return proto; 1199 1202 } 1200 1203 EXPORT_SYMBOL(ipv6_push_frag_opts); 1201 1204
+1 -1
net/ipv6/ip6_gre.c
··· 1057 1057 /* TooBig packet may have updated dst->dev's mtu */ 1058 1058 if (!t->parms.collect_md && dst) { 1059 1059 mtu = READ_ONCE(dst_dev(dst)->mtu); 1060 - if (dst_mtu(dst) > mtu) 1060 + if (dst6_mtu(dst) > mtu) 1061 1061 dst->ops->update_pmtu(dst, NULL, skb, mtu, false); 1062 1062 } 1063 1063 err = ip6_tnl_xmit(skb, dev, dsfield, &fl6, encap_limit, &mtu,
+55 -53
net/ipv6/ip6_output.c
··· 80 80 81 81 hdr = ipv6_hdr(skb); 82 82 daddr = &hdr->daddr; 83 - if (ipv6_addr_is_multicast(daddr)) { 83 + if (unlikely(ipv6_addr_is_multicast(daddr))) { 84 84 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) && 85 85 ((mroute6_is_socket(net, skb) && 86 86 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) || ··· 179 179 static int ip6_finish_output_gso(struct net *net, struct sock *sk, 180 180 struct sk_buff *skb, unsigned int mtu) 181 181 { 182 - if (!(IP6CB(skb)->flags & IP6SKB_FAKEJUMBO) && 183 - !skb_gso_validate_network_len(skb, mtu)) 182 + if (unlikely(!(IP6CB(skb)->flags & IP6SKB_FAKEJUMBO) && 183 + !skb_gso_validate_network_len(skb, mtu))) 184 184 return ip6_finish_output_gso_slowpath_drop(net, sk, skb, mtu); 185 185 186 186 return ip6_finish_output2(net, sk, skb); ··· 202 202 if (skb_is_gso(skb)) 203 203 return ip6_finish_output_gso(net, sk, skb, mtu); 204 204 205 - if (skb->len > mtu || 206 - (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size)) 205 + if (unlikely(skb->len > mtu || 206 + (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size))) 207 207 return ip6_fragment(net, sk, skb, ip6_finish_output2); 208 208 209 209 return ip6_finish_output2(net, sk, skb); ··· 301 301 } 302 302 } 303 303 304 - if (opt) { 304 + if (unlikely(opt)) { 305 305 seg_len += opt->opt_nflen + opt->opt_flen; 306 306 307 307 if (opt->opt_flen) 308 - ipv6_push_frag_opts(skb, opt, &proto); 308 + proto = ipv6_push_frag_opts(skb, opt, proto); 309 309 310 310 if (opt->opt_nflen) 311 - ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop, 312 - &fl6->saddr); 311 + proto = ipv6_push_nfrag_opts(skb, opt, proto, 312 + &first_hop, 313 + &fl6->saddr); 313 314 } 314 315 315 316 if (unlikely(seg_len > IPV6_MAXPLEN)) { 316 - hop_jumbo = skb_push(skb, hoplen); 317 + hop_jumbo = __skb_push(skb, hoplen); 317 318 318 319 hop_jumbo->nexthdr = proto; 319 320 hop_jumbo->hdrlen = 0; ··· 327 326 IP6CB(skb)->flags |= IP6SKB_FAKEJUMBO; 328 327 } 329 328 330 - skb_push(skb, sizeof(struct ipv6hdr)); 329 + __skb_push(skb, sizeof(struct ipv6hdr)); 331 330 skb_reset_network_header(skb); 332 331 hdr = ipv6_hdr(skb); 333 332 ··· 353 352 skb->priority = priority; 354 353 skb->mark = mark; 355 354 356 - mtu = dst_mtu(dst); 357 - if ((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb)) { 355 + mtu = dst6_mtu(dst); 356 + if (likely((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb))) { 358 357 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTREQUESTS); 359 358 360 359 /* if egress device is enslaved to an L3 master device pass the ··· 383 382 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu); 384 383 385 384 IP6_INC_STATS(net, idev, IPSTATS_MIB_FRAGFAILS); 386 - kfree_skb(skb); 385 + kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG); 387 386 unlock: 388 387 rcu_read_unlock(); 389 388 return ret; ··· 654 653 if (mtu < IPV6_MIN_MTU) 655 654 mtu = IPV6_MIN_MTU; 656 655 657 - if (ip6_pkt_too_big(skb, mtu)) { 656 + if (unlikely(ip6_pkt_too_big(skb, mtu))) { 658 657 /* Again, force OUTPUT device used as source address */ 659 658 skb->dev = dev; 660 659 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); ··· 1353 1352 } 1354 1353 1355 1354 static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork, 1356 - struct inet6_cork *v6_cork, struct ipcm6_cookie *ipc6, 1355 + struct ipcm6_cookie *ipc6, 1357 1356 struct rt6_info *rt) 1358 1357 { 1358 + struct ipv6_txoptions *nopt, *opt = ipc6->opt; 1359 + struct inet6_cork *v6_cork = &cork->base6; 1359 1360 struct ipv6_pinfo *np = inet6_sk(sk); 1360 1361 unsigned int mtu, frag_size; 1361 - struct ipv6_txoptions *nopt, *opt = ipc6->opt; 1362 1362 1363 1363 /* callers pass dst together with a reference, set it first so 1364 1364 * ip6_cork_release() can put it down even in case of an error. ··· 1369 1367 /* 1370 1368 * setup for corking 1371 1369 */ 1372 - if (opt) { 1370 + if (unlikely(opt)) { 1373 1371 if (WARN_ON(v6_cork->opt)) 1374 1372 return -EINVAL; 1375 1373 ··· 1404 1402 v6_cork->dontfrag = ipc6->dontfrag; 1405 1403 if (rt->dst.flags & DST_XFRM_TUNNEL) 1406 1404 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ? 1407 - READ_ONCE(rt->dst.dev->mtu) : dst_mtu(&rt->dst); 1405 + READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(&rt->dst); 1408 1406 else 1409 1407 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ? 1410 - READ_ONCE(rt->dst.dev->mtu) : dst_mtu(xfrm_dst_path(&rt->dst)); 1408 + READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(xfrm_dst_path(&rt->dst)); 1411 1409 1412 1410 frag_size = READ_ONCE(np->frag_size); 1413 1411 if (frag_size && frag_size < mtu) ··· 1432 1430 static int __ip6_append_data(struct sock *sk, 1433 1431 struct sk_buff_head *queue, 1434 1432 struct inet_cork_full *cork_full, 1435 - struct inet6_cork *v6_cork, 1436 1433 struct page_frag *pfrag, 1437 1434 int getfrag(void *from, char *to, int offset, 1438 1435 int len, int odd, struct sk_buff *skb), 1439 1436 void *from, size_t length, int transhdrlen, 1440 1437 unsigned int flags) 1441 1438 { 1442 - struct sk_buff *skb, *skb_prev = NULL; 1439 + unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu, pmtu; 1440 + struct inet6_cork *v6_cork = &cork_full->base6; 1443 1441 struct inet_cork *cork = &cork_full->base; 1444 1442 struct flowi6 *fl6 = &cork_full->fl.u.ip6; 1445 - unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu, pmtu; 1443 + struct sk_buff *skb, *skb_prev = NULL; 1446 1444 struct ubuf_info *uarg = NULL; 1447 1445 int exthdrlen = 0; 1448 1446 int dst_exthdrlen = 0; ··· 1845 1843 struct rt6_info *rt, unsigned int flags) 1846 1844 { 1847 1845 struct inet_sock *inet = inet_sk(sk); 1848 - struct ipv6_pinfo *np = inet6_sk(sk); 1849 1846 int exthdrlen; 1850 1847 int err; 1851 1848 ··· 1855 1854 * setup for corking 1856 1855 */ 1857 1856 dst_hold(&rt->dst); 1858 - err = ip6_setup_cork(sk, &inet->cork, &np->cork, 1857 + err = ip6_setup_cork(sk, &inet->cork, 1859 1858 ipc6, rt); 1860 1859 if (err) 1861 1860 return err; ··· 1869 1868 } 1870 1869 1871 1870 return __ip6_append_data(sk, &sk->sk_write_queue, &inet->cork, 1872 - &np->cork, sk_page_frag(sk), getfrag, 1871 + sk_page_frag(sk), getfrag, 1873 1872 from, length, transhdrlen, flags); 1874 1873 } 1875 1874 EXPORT_SYMBOL_GPL(ip6_append_data); ··· 1882 1881 skb_dst_set(skb, dst); 1883 1882 } 1884 1883 1885 - static void ip6_cork_release(struct inet_cork_full *cork, 1886 - struct inet6_cork *v6_cork) 1884 + static void ip6_cork_release(struct inet_cork_full *cork) 1887 1885 { 1888 - if (v6_cork->opt) { 1886 + struct inet6_cork *v6_cork = &cork->base6; 1887 + 1888 + if (unlikely(v6_cork->opt)) { 1889 1889 struct ipv6_txoptions *opt = v6_cork->opt; 1890 1890 1891 1891 kfree(opt->dst0opt); ··· 1905 1903 1906 1904 struct sk_buff *__ip6_make_skb(struct sock *sk, 1907 1905 struct sk_buff_head *queue, 1908 - struct inet_cork_full *cork, 1909 - struct inet6_cork *v6_cork) 1906 + struct inet_cork_full *cork) 1910 1907 { 1911 1908 struct sk_buff *skb, *tmp_skb; 1912 1909 struct sk_buff **tail_skb; 1913 1910 struct in6_addr *final_dst; 1914 1911 struct net *net = sock_net(sk); 1915 1912 struct ipv6hdr *hdr; 1916 - struct ipv6_txoptions *opt = v6_cork->opt; 1913 + struct ipv6_txoptions *opt; 1917 1914 struct rt6_info *rt = dst_rt6_info(cork->base.dst); 1918 1915 struct flowi6 *fl6 = &cork->fl.u.ip6; 1919 1916 unsigned char proto = fl6->flowi6_proto; ··· 1941 1940 __skb_pull(skb, skb_network_header_len(skb)); 1942 1941 1943 1942 final_dst = &fl6->daddr; 1944 - if (opt && opt->opt_flen) 1945 - ipv6_push_frag_opts(skb, opt, &proto); 1946 - if (opt && opt->opt_nflen) 1947 - ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst, &fl6->saddr); 1948 - 1943 + opt = cork->base6.opt; 1944 + if (unlikely(opt)) { 1945 + if (opt->opt_flen) 1946 + proto = ipv6_push_frag_opts(skb, opt, proto); 1947 + if (opt->opt_nflen) 1948 + proto = ipv6_push_nfrag_opts(skb, opt, proto, 1949 + &final_dst, &fl6->saddr); 1950 + } 1949 1951 skb_push(skb, sizeof(struct ipv6hdr)); 1950 1952 skb_reset_network_header(skb); 1951 1953 hdr = ipv6_hdr(skb); 1952 1954 1953 - ip6_flow_hdr(hdr, v6_cork->tclass, 1955 + ip6_flow_hdr(hdr, cork->base6.tclass, 1954 1956 ip6_make_flowlabel(net, skb, fl6->flowlabel, 1955 1957 ip6_autoflowlabel(net, sk), fl6)); 1956 - hdr->hop_limit = v6_cork->hop_limit; 1958 + hdr->hop_limit = cork->base6.hop_limit; 1957 1959 hdr->nexthdr = proto; 1958 1960 hdr->saddr = fl6->saddr; 1959 1961 hdr->daddr = *final_dst; ··· 1970 1966 1971 1967 ip6_cork_steal_dst(skb, cork); 1972 1968 IP6_INC_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUTREQUESTS); 1973 - if (proto == IPPROTO_ICMPV6) { 1969 + if (unlikely(proto == IPPROTO_ICMPV6)) { 1974 1970 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 1975 1971 u8 icmp6_type; 1976 1972 ··· 1983 1979 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS); 1984 1980 } 1985 1981 1986 - ip6_cork_release(cork, v6_cork); 1982 + ip6_cork_release(cork); 1987 1983 out: 1988 1984 return skb; 1989 1985 } ··· 2022 2018 2023 2019 static void __ip6_flush_pending_frames(struct sock *sk, 2024 2020 struct sk_buff_head *queue, 2025 - struct inet_cork_full *cork, 2026 - struct inet6_cork *v6_cork) 2021 + struct inet_cork_full *cork) 2027 2022 { 2028 2023 struct sk_buff *skb; 2029 2024 ··· 2033 2030 kfree_skb(skb); 2034 2031 } 2035 2032 2036 - ip6_cork_release(cork, v6_cork); 2033 + ip6_cork_release(cork); 2037 2034 } 2038 2035 2039 2036 void ip6_flush_pending_frames(struct sock *sk) 2040 2037 { 2041 2038 __ip6_flush_pending_frames(sk, &sk->sk_write_queue, 2042 - &inet_sk(sk)->cork, &inet6_sk(sk)->cork); 2039 + &inet_sk(sk)->cork); 2043 2040 } 2044 2041 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames); 2045 2042 ··· 2050 2047 struct ipcm6_cookie *ipc6, struct rt6_info *rt, 2051 2048 unsigned int flags, struct inet_cork_full *cork) 2052 2049 { 2053 - struct inet6_cork v6_cork; 2054 - struct sk_buff_head queue; 2055 2050 int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0); 2051 + struct sk_buff_head queue; 2056 2052 int err; 2057 2053 2058 2054 if (flags & MSG_PROBE) { ··· 2064 2062 cork->base.flags = 0; 2065 2063 cork->base.addr = 0; 2066 2064 cork->base.opt = NULL; 2067 - v6_cork.opt = NULL; 2068 - err = ip6_setup_cork(sk, cork, &v6_cork, ipc6, rt); 2065 + cork->base6.opt = NULL; 2066 + err = ip6_setup_cork(sk, cork, ipc6, rt); 2069 2067 if (err) { 2070 - ip6_cork_release(cork, &v6_cork); 2068 + ip6_cork_release(cork); 2071 2069 return ERR_PTR(err); 2072 2070 } 2073 2071 2074 - err = __ip6_append_data(sk, &queue, cork, &v6_cork, 2072 + err = __ip6_append_data(sk, &queue, cork, 2075 2073 &current->task_frag, getfrag, from, 2076 2074 length + exthdrlen, transhdrlen + exthdrlen, 2077 2075 flags); 2078 2076 if (err) { 2079 - __ip6_flush_pending_frames(sk, &queue, cork, &v6_cork); 2077 + __ip6_flush_pending_frames(sk, &queue, cork); 2080 2078 return ERR_PTR(err); 2081 2079 } 2082 2080 2083 - return __ip6_make_skb(sk, &queue, cork, &v6_cork); 2081 + return __ip6_make_skb(sk, &queue, cork); 2084 2082 }
+3 -3
net/ipv6/ip6_tunnel.c
··· 638 638 639 639 /* change mtu on this route */ 640 640 if (rel_type == ICMP_DEST_UNREACH && rel_code == ICMP_FRAG_NEEDED) { 641 - if (rel_info > dst_mtu(skb_dst(skb2))) 641 + if (rel_info > dst6_mtu(skb_dst(skb2))) 642 642 goto out; 643 643 644 644 skb_dst_update_pmtu_no_confirm(skb2, rel_info); ··· 1187 1187 t->parms.name); 1188 1188 goto tx_err_dst_release; 1189 1189 } 1190 - mtu = dst_mtu(dst) - eth_hlen - psh_hlen - t->tun_hlen; 1190 + mtu = dst6_mtu(dst) - eth_hlen - psh_hlen - t->tun_hlen; 1191 1191 if (encap_limit >= 0) { 1192 1192 max_headroom += 8; 1193 1193 mtu -= 8; ··· 1265 1265 1266 1266 if (encap_limit >= 0) { 1267 1267 init_tel_txopt(&opt, encap_limit); 1268 - ipv6_push_frag_opts(skb, &opt.ops, &proto); 1268 + proto = ipv6_push_frag_opts(skb, &opt.ops, proto); 1269 1269 } 1270 1270 1271 1271 skb_push(skb, sizeof(struct ipv6hdr));
+2 -2
net/ipv6/ipv6_sockglue.c
··· 1184 1184 rcu_read_lock(); 1185 1185 dst = __sk_dst_get(sk); 1186 1186 if (dst) 1187 - val = dst_mtu(dst); 1187 + val = dst6_mtu(dst); 1188 1188 rcu_read_unlock(); 1189 1189 if (!val) 1190 1190 return -ENOTCONN; ··· 1283 1283 rcu_read_lock(); 1284 1284 dst = __sk_dst_get(sk); 1285 1285 if (dst) 1286 - mtuinfo.ip6m_mtu = dst_mtu(dst); 1286 + mtuinfo.ip6m_mtu = dst6_mtu(dst); 1287 1287 rcu_read_unlock(); 1288 1288 if (!mtuinfo.ip6m_mtu) 1289 1289 return -ENOTCONN;
+1 -1
net/ipv6/raw.c
··· 529 529 530 530 offset = rp->offset; 531 531 total_len = inet_sk(sk)->cork.base.length; 532 - opt = inet6_sk(sk)->cork.opt; 532 + opt = inet_sk(sk)->cork.base6.opt; 533 533 total_len -= opt ? opt->opt_flen : 0; 534 534 535 535 if (offset >= total_len - 1) {
+6 -4
net/ipv6/route.c
··· 2049 2049 static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev, 2050 2050 struct rt6_info *rt, int mtu) 2051 2051 { 2052 + u32 dmtu = dst6_mtu(&rt->dst); 2053 + 2052 2054 /* If the new MTU is lower than the route PMTU, this new MTU will be the 2053 2055 * lowest MTU in the path: always allow updating the route PMTU to 2054 2056 * reflect PMTU decreases. ··· 2061 2059 * handle this. 2062 2060 */ 2063 2061 2064 - if (dst_mtu(&rt->dst) >= mtu) 2062 + if (dmtu >= mtu) 2065 2063 return true; 2066 2064 2067 - if (dst_mtu(&rt->dst) == idev->cnf.mtu6) 2065 + if (dmtu == idev->cnf.mtu6) 2068 2066 return true; 2069 2067 2070 2068 return false; ··· 2934 2932 2935 2933 if (mtu < IPV6_MIN_MTU) 2936 2934 return; 2937 - if (mtu >= dst_mtu(dst)) 2935 + if (mtu >= dst6_mtu(dst)) 2938 2936 return; 2939 2937 2940 2938 if (!rt6_cache_allowed_for_pmtu(rt6)) { ··· 3250 3248 3251 3249 static unsigned int ip6_default_advmss(const struct dst_entry *dst) 3252 3250 { 3253 - unsigned int mtu = dst_mtu(dst); 3251 + unsigned int mtu = dst6_mtu(dst); 3254 3252 struct net *net; 3255 3253 3256 3254 mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
+1 -1
net/ipv6/sit.c
··· 962 962 } 963 963 964 964 if (df) { 965 - mtu = dst_mtu(&rt->dst) - t_hlen; 965 + mtu = dst4_mtu(&rt->dst) - t_hlen; 966 966 967 967 if (mtu < IPV4_MIN_MTU) { 968 968 DEV_STATS_INC(dev, collisions);
+5 -4
net/ipv6/tcp_ipv6.c
··· 351 351 static void tcp_v6_mtu_reduced(struct sock *sk) 352 352 { 353 353 struct dst_entry *dst; 354 - u32 mtu; 354 + u32 mtu, dmtu; 355 355 356 356 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) 357 357 return; ··· 368 368 if (!dst) 369 369 return; 370 370 371 - if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) { 372 - tcp_sync_mss(sk, dst_mtu(dst)); 371 + dmtu = dst6_mtu(dst); 372 + if (inet_csk(sk)->icsk_pmtu_cookie > dmtu) { 373 + tcp_sync_mss(sk, dmtu); 373 374 tcp_simple_retransmit(sk); 374 375 } 375 376 } ··· 1468 1467 1469 1468 tcp_ca_openreq_child(newsk, dst); 1470 1469 1471 - tcp_sync_mss(newsk, dst_mtu(dst)); 1470 + tcp_sync_mss(newsk, dst6_mtu(dst)); 1472 1471 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); 1473 1472 1474 1473 tcp_initialize_rcv_mss(newsk);