Merge branch 'tcp-refactor-bhash2' · tjh.dev/kernel@5f12303

+8 -13

include/net/inet_hashtables.h

··· 88 88 unsigned short fast_sk_family; 89 89 bool fast_ipv6_only; 90 90 struct hlist_node node; 91 - struct hlist_head owners; 91 + struct hlist_head bhash2; 92 92 }; 93 93 94 94 struct inet_bind2_bucket { ··· 96 96 int l3mdev; 97 97 unsigned short port; 98 98 #if IS_ENABLED(CONFIG_IPV6) 99 - unsigned short family; 99 + unsigned short addr_type; 100 + struct in6_addr v6_rcv_saddr; 101 + #define rcv_saddr v6_rcv_saddr.s6_addr32[3] 102 + #else 103 + __be32 rcv_saddr; 100 104 #endif 101 - union { 102 - #if IS_ENABLED(CONFIG_IPV6) 103 - struct in6_addr v6_rcv_saddr; 104 - #endif 105 - __be32 rcv_saddr; 106 - }; 107 105 /* Node in the bhash2 inet_bind_hashbucket chain */ 108 106 struct hlist_node node; 107 + struct hlist_node bhash_node; 109 108 /* List of sockets hashed to this bucket */ 110 109 struct hlist_head owners; 111 - /* bhash has twsk in owners, but bhash2 has twsk in 112 - * deathrow not to add a member in struct sock_common. 113 - */ 114 - struct hlist_head deathrow; 115 110 }; 116 111 117 112 static inline struct net *ib_net(const struct inet_bind_bucket *ib) ··· 236 241 struct inet_bind2_bucket * 237 242 inet_bind2_bucket_create(struct kmem_cache *cachep, struct net *net, 238 243 struct inet_bind_hashbucket *head, 239 - unsigned short port, int l3mdev, 244 + struct inet_bind_bucket *tb, 240 245 const struct sock *sk); 241 246 242 247 void inet_bind2_bucket_destroy(struct kmem_cache *cachep,

-4

include/net/inet_timewait_sock.h

··· 75 75 struct timer_list tw_timer; 76 76 struct inet_bind_bucket *tw_tb; 77 77 struct inet_bind2_bucket *tw_tb2; 78 - struct hlist_node tw_bind2_node; 79 78 }; 80 79 #define tw_tclass tw_tos 81 - 82 - #define twsk_for_each_bound_bhash2(__tw, list) \ 83 - hlist_for_each_entry(__tw, list, tw_bind2_node) 84 80 85 81 static inline struct inet_timewait_sock *inet_twsk(const struct sock *sk) 86 82 {

-5

include/net/ipv6.h

··· 784 784 cpu_to_be32(0x0000ffff))) == 0UL; 785 785 } 786 786 787 - static inline bool ipv6_addr_v4mapped_any(const struct in6_addr *a) 788 - { 789 - return ipv6_addr_v4mapped(a) && ipv4_is_zeronet(a->s6_addr32[3]); 790 - } 791 - 792 787 static inline bool ipv6_addr_v4mapped_loopback(const struct in6_addr *a) 793 788 { 794 789 return ipv6_addr_v4mapped(a) && ipv4_is_loopback(a->s6_addr32[3]);

-14

include/net/sock.h

··· 352 352 * @sk_txtime_report_errors: set report errors mode for SO_TXTIME 353 353 * @sk_txtime_unused: unused txtime flags 354 354 * @ns_tracker: tracker for netns reference 355 - * @sk_bind2_node: bind node in the bhash2 table 356 355 */ 357 356 struct sock { 358 357 /* ··· 543 544 #endif 544 545 struct rcu_head sk_rcu; 545 546 netns_tracker ns_tracker; 546 - struct hlist_node sk_bind2_node; 547 547 }; 548 548 549 549 enum sk_pacing { ··· 871 873 hlist_add_head(&sk->sk_bind_node, list); 872 874 } 873 875 874 - static inline void __sk_del_bind2_node(struct sock *sk) 875 - { 876 - __hlist_del(&sk->sk_bind2_node); 877 - } 878 - 879 - static inline void sk_add_bind2_node(struct sock *sk, struct hlist_head *list) 880 - { 881 - hlist_add_head(&sk->sk_bind2_node, list); 882 - } 883 - 884 876 #define sk_for_each(__sk, list) \ 885 877 hlist_for_each_entry(__sk, list, sk_node) 886 878 #define sk_for_each_rcu(__sk, list) \ ··· 888 900 hlist_for_each_entry_safe(__sk, tmp, list, sk_node) 889 901 #define sk_for_each_bound(__sk, list) \ 890 902 hlist_for_each_entry(__sk, list, sk_bind_node) 891 - #define sk_for_each_bound_bhash2(__sk, list) \ 892 - hlist_for_each_entry(__sk, list, sk_bind2_node) 893 903 894 904 /** 895 905 * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset

+36 -41

net/ipv4/inet_connection_sock.c

··· 159 159 if (sk->sk_family == AF_INET6) { 160 160 int addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr); 161 161 162 - return addr_type != IPV6_ADDR_ANY && 163 - addr_type != IPV6_ADDR_MAPPED; 162 + if (addr_type == IPV6_ADDR_ANY) 163 + return false; 164 + 165 + if (addr_type != IPV6_ADDR_MAPPED) 166 + return true; 164 167 } 165 168 #endif 166 169 return sk->sk_rcv_saddr != htonl(INADDR_ANY); ··· 216 213 bool relax, bool reuseport_cb_ok, 217 214 bool reuseport_ok) 218 215 { 219 - struct inet_timewait_sock *tw2; 220 216 struct sock *sk2; 221 217 222 - sk_for_each_bound_bhash2(sk2, &tb2->owners) { 223 - if (__inet_bhash2_conflict(sk, sk2, sk_uid, relax, 224 - reuseport_cb_ok, reuseport_ok)) 225 - return true; 226 - } 227 - 228 - twsk_for_each_bound_bhash2(tw2, &tb2->deathrow) { 229 - sk2 = (struct sock *)tw2; 230 - 218 + sk_for_each_bound(sk2, &tb2->owners) { 231 219 if (__inet_bhash2_conflict(sk, sk2, sk_uid, relax, 232 220 reuseport_cb_ok, reuseport_ok)) 233 221 return true; ··· 227 233 return false; 228 234 } 229 235 236 + #define sk_for_each_bound_bhash(__sk, __tb2, __tb) \ 237 + hlist_for_each_entry(__tb2, &(__tb)->bhash2, bhash_node) \ 238 + sk_for_each_bound(sk2, &(__tb2)->owners) 239 + 230 240 /* This should be called only when the tb and tb2 hashbuckets' locks are held */ 231 241 static int inet_csk_bind_conflict(const struct sock *sk, 232 242 const struct inet_bind_bucket *tb, 233 243 const struct inet_bind2_bucket *tb2, /* may be null */ 234 244 bool relax, bool reuseport_ok) 235 245 { 236 - bool reuseport_cb_ok; 237 - struct sock_reuseport *reuseport_cb; 238 246 kuid_t uid = sock_i_uid((struct sock *)sk); 247 + struct sock_reuseport *reuseport_cb; 248 + bool reuseport_cb_ok; 249 + struct sock *sk2; 239 250 240 251 rcu_read_lock(); 241 252 reuseport_cb = rcu_dereference(sk->sk_reuseport_cb); ··· 248 249 reuseport_cb_ok = !reuseport_cb || READ_ONCE(reuseport_cb->num_closed_socks); 249 250 rcu_read_unlock(); 250 251 251 - /* 252 - * Unlike other sk lookup places we do not check 253 - * for sk_net here, since _all_ the socks listed 254 - * in tb->owners and tb2->owners list belong 255 - * to the same net - the one this bucket belongs to. 256 - */ 257 - 258 - if (!inet_use_bhash2_on_bind(sk)) { 259 - struct sock *sk2; 260 - 261 - sk_for_each_bound(sk2, &tb->owners) 262 - if (inet_bind_conflict(sk, sk2, uid, relax, 263 - reuseport_cb_ok, reuseport_ok) && 264 - inet_rcv_saddr_equal(sk, sk2, true)) 265 - return true; 266 - 267 - return false; 268 - } 269 - 270 252 /* Conflicts with an existing IPV6_ADDR_ANY (if ipv6) or INADDR_ANY (if 271 253 * ipv4) should have been checked already. We need to do these two 272 254 * checks separately because their spinlocks have to be acquired/released 273 255 * independently of each other, to prevent possible deadlocks 274 256 */ 275 - return tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok, 276 - reuseport_ok); 257 + if (inet_use_bhash2_on_bind(sk)) 258 + return tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, 259 + reuseport_cb_ok, reuseport_ok); 260 + 261 + /* Unlike other sk lookup places we do not check 262 + * for sk_net here, since _all_ the socks listed 263 + * in tb->owners and tb2->owners list belong 264 + * to the same net - the one this bucket belongs to. 265 + */ 266 + sk_for_each_bound_bhash(sk2, tb2, tb) { 267 + if (!inet_bind_conflict(sk, sk2, uid, relax, reuseport_cb_ok, reuseport_ok)) 268 + continue; 269 + 270 + if (inet_rcv_saddr_equal(sk, sk2, true)) 271 + return true; 272 + } 273 + 274 + return false; 277 275 } 278 276 279 277 /* Determine if there is a bind conflict with an existing IPV6_ADDR_ANY (if ipv6) or ··· 453 457 kuid_t uid = sock_i_uid(sk); 454 458 bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN; 455 459 456 - if (hlist_empty(&tb->owners)) { 460 + if (hlist_empty(&tb->bhash2)) { 457 461 tb->fastreuse = reuse; 458 462 if (sk->sk_reuseport) { 459 463 tb->fastreuseport = FASTREUSEPORT_ANY; ··· 545 549 } 546 550 547 551 if (!found_port) { 548 - if (!hlist_empty(&tb->owners)) { 552 + if (!hlist_empty(&tb->bhash2)) { 549 553 if (sk->sk_reuse == SK_FORCE_REUSE || 550 554 (tb->fastreuse > 0 && reuse) || 551 555 sk_reuseport_match(tb, sk)) ··· 565 569 566 570 if (!tb2) { 567 571 tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep, 568 - net, head2, port, l3mdev, sk); 572 + net, head2, tb, sk); 569 573 if (!tb2) 570 574 goto fail_unlock; 571 575 bhash2_created = true; ··· 587 591 588 592 fail_unlock: 589 593 if (ret) { 594 + if (bhash2_created) 595 + inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep, tb2); 590 596 if (bhash_created) 591 597 inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb); 592 - if (bhash2_created) 593 - inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep, 594 - tb2); 595 598 } 596 599 if (head2_lock_acquired) 597 600 spin_unlock(&head2->lock);

+1 -1

net/ipv4/inet_diag.c

··· 1104 1104 if (!net_eq(ib2_net(tb2), net)) 1105 1105 continue; 1106 1106 1107 - sk_for_each_bound_bhash2(sk, &tb2->owners) { 1107 + sk_for_each_bound(sk, &tb2->owners) { 1108 1108 struct inet_sock *inet = inet_sk(sk); 1109 1109 1110 1110 if (num < s_num)

+46 -52

net/ipv4/inet_hashtables.c

··· 76 76 tb->port = snum; 77 77 tb->fastreuse = 0; 78 78 tb->fastreuseport = 0; 79 - INIT_HLIST_HEAD(&tb->owners); 79 + INIT_HLIST_HEAD(&tb->bhash2); 80 80 hlist_add_head(&tb->node, &head->chain); 81 81 } 82 82 return tb; ··· 87 87 */ 88 88 void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb) 89 89 { 90 - if (hlist_empty(&tb->owners)) { 90 + if (hlist_empty(&tb->bhash2)) { 91 91 __hlist_del(&tb->node); 92 92 kmem_cache_free(cachep, tb); 93 93 } ··· 100 100 tb->l3mdev == l3mdev; 101 101 } 102 102 103 - static void inet_bind2_bucket_init(struct inet_bind2_bucket *tb, 103 + static void inet_bind2_bucket_init(struct inet_bind2_bucket *tb2, 104 104 struct net *net, 105 105 struct inet_bind_hashbucket *head, 106 - unsigned short port, int l3mdev, 106 + struct inet_bind_bucket *tb, 107 107 const struct sock *sk) 108 108 { 109 - write_pnet(&tb->ib_net, net); 110 - tb->l3mdev = l3mdev; 111 - tb->port = port; 109 + write_pnet(&tb2->ib_net, net); 110 + tb2->l3mdev = tb->l3mdev; 111 + tb2->port = tb->port; 112 112 #if IS_ENABLED(CONFIG_IPV6) 113 - tb->family = sk->sk_family; 114 - if (sk->sk_family == AF_INET6) 115 - tb->v6_rcv_saddr = sk->sk_v6_rcv_saddr; 116 - else 113 + BUILD_BUG_ON(USHRT_MAX < (IPV6_ADDR_ANY | IPV6_ADDR_MAPPED)); 114 + if (sk->sk_family == AF_INET6) { 115 + tb2->addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr); 116 + tb2->v6_rcv_saddr = sk->sk_v6_rcv_saddr; 117 + } else { 118 + tb2->addr_type = IPV6_ADDR_MAPPED; 119 + ipv6_addr_set_v4mapped(sk->sk_rcv_saddr, &tb2->v6_rcv_saddr); 120 + } 121 + #else 122 + tb2->rcv_saddr = sk->sk_rcv_saddr; 117 123 #endif 118 - tb->rcv_saddr = sk->sk_rcv_saddr; 119 - INIT_HLIST_HEAD(&tb->owners); 120 - INIT_HLIST_HEAD(&tb->deathrow); 121 - hlist_add_head(&tb->node, &head->chain); 124 + INIT_HLIST_HEAD(&tb2->owners); 125 + hlist_add_head(&tb2->node, &head->chain); 126 + hlist_add_head(&tb2->bhash_node, &tb->bhash2); 122 127 } 123 128 124 129 struct inet_bind2_bucket *inet_bind2_bucket_create(struct kmem_cache *cachep, 125 130 struct net *net, 126 131 struct inet_bind_hashbucket *head, 127 - unsigned short port, 128 - int l3mdev, 132 + struct inet_bind_bucket *tb, 129 133 const struct sock *sk) 130 134 { 131 - struct inet_bind2_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC); 135 + struct inet_bind2_bucket *tb2 = kmem_cache_alloc(cachep, GFP_ATOMIC); 132 136 133 - if (tb) 134 - inet_bind2_bucket_init(tb, net, head, port, l3mdev, sk); 137 + if (tb2) 138 + inet_bind2_bucket_init(tb2, net, head, tb, sk); 135 139 136 - return tb; 140 + return tb2; 137 141 } 138 142 139 143 /* Caller must hold hashbucket lock for this tb with local BH disabled */ 140 144 void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb) 141 145 { 142 - if (hlist_empty(&tb->owners) && hlist_empty(&tb->deathrow)) { 146 + if (hlist_empty(&tb->owners)) { 143 147 __hlist_del(&tb->node); 148 + __hlist_del(&tb->bhash_node); 144 149 kmem_cache_free(cachep, tb); 145 150 } 146 151 } ··· 154 149 const struct sock *sk) 155 150 { 156 151 #if IS_ENABLED(CONFIG_IPV6) 157 - if (sk->sk_family != tb2->family) { 158 - if (sk->sk_family == AF_INET) 159 - return ipv6_addr_v4mapped(&tb2->v6_rcv_saddr) && 160 - tb2->v6_rcv_saddr.s6_addr32[3] == sk->sk_rcv_saddr; 161 - 162 - return ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr) && 163 - sk->sk_v6_rcv_saddr.s6_addr32[3] == tb2->rcv_saddr; 164 - } 165 - 166 152 if (sk->sk_family == AF_INET6) 167 - return ipv6_addr_equal(&tb2->v6_rcv_saddr, 168 - &sk->sk_v6_rcv_saddr); 153 + return ipv6_addr_equal(&tb2->v6_rcv_saddr, &sk->sk_v6_rcv_saddr); 154 + 155 + if (tb2->addr_type != IPV6_ADDR_MAPPED) 156 + return false; 169 157 #endif 170 158 return tb2->rcv_saddr == sk->sk_rcv_saddr; 171 159 } ··· 167 169 struct inet_bind2_bucket *tb2, unsigned short port) 168 170 { 169 171 inet_sk(sk)->inet_num = port; 170 - sk_add_bind_node(sk, &tb->owners); 171 172 inet_csk(sk)->icsk_bind_hash = tb; 172 - sk_add_bind2_node(sk, &tb2->owners); 173 173 inet_csk(sk)->icsk_bind2_hash = tb2; 174 + sk_add_bind_node(sk, &tb2->owners); 174 175 } 175 176 176 177 /* ··· 189 192 190 193 spin_lock(&head->lock); 191 194 tb = inet_csk(sk)->icsk_bind_hash; 192 - __sk_del_bind_node(sk); 193 195 inet_csk(sk)->icsk_bind_hash = NULL; 194 196 inet_sk(sk)->inet_num = 0; 195 - inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb); 196 197 197 198 spin_lock(&head2->lock); 198 199 if (inet_csk(sk)->icsk_bind2_hash) { 199 200 struct inet_bind2_bucket *tb2 = inet_csk(sk)->icsk_bind2_hash; 200 201 201 - __sk_del_bind2_node(sk); 202 + __sk_del_bind_node(sk); 202 203 inet_csk(sk)->icsk_bind2_hash = NULL; 203 204 inet_bind2_bucket_destroy(hashinfo->bind2_bucket_cachep, tb2); 204 205 } 205 206 spin_unlock(&head2->lock); 206 207 208 + inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb); 207 209 spin_unlock(&head->lock); 208 210 } 209 211 ··· 271 275 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, child); 272 276 if (!tb2) { 273 277 tb2 = inet_bind2_bucket_create(table->bind2_bucket_cachep, 274 - net, head2, port, 275 - l3mdev, child); 278 + net, head2, tb, child); 276 279 if (!tb2) 277 280 goto error; 278 281 } ··· 831 836 return false; 832 837 833 838 #if IS_ENABLED(CONFIG_IPV6) 834 - if (sk->sk_family != tb->family) { 835 - if (sk->sk_family == AF_INET) 836 - return ipv6_addr_any(&tb->v6_rcv_saddr) || 837 - ipv6_addr_v4mapped_any(&tb->v6_rcv_saddr); 839 + if (tb->addr_type == IPV6_ADDR_ANY) 840 + return true; 838 841 842 + if (tb->addr_type != IPV6_ADDR_MAPPED) 839 843 return false; 840 - } 841 844 842 - if (sk->sk_family == AF_INET6) 843 - return ipv6_addr_any(&tb->v6_rcv_saddr); 845 + if (sk->sk_family == AF_INET6 && 846 + !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) 847 + return false; 844 848 #endif 845 849 return tb->rcv_saddr == 0; 846 850 } ··· 936 942 spin_lock_bh(&head->lock); 937 943 938 944 spin_lock(&head2->lock); 939 - __sk_del_bind2_node(sk); 945 + __sk_del_bind_node(sk); 940 946 inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep, inet_csk(sk)->icsk_bind2_hash); 941 947 spin_unlock(&head2->lock); 942 948 ··· 951 957 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); 952 958 if (!tb2) { 953 959 tb2 = new_tb2; 954 - inet_bind2_bucket_init(tb2, net, head2, port, l3mdev, sk); 960 + inet_bind2_bucket_init(tb2, net, head2, inet_csk(sk)->icsk_bind_hash, sk); 955 961 } 956 - sk_add_bind2_node(sk, &tb2->owners); 957 962 inet_csk(sk)->icsk_bind2_hash = tb2; 963 + sk_add_bind_node(sk, &tb2->owners); 958 964 spin_unlock(&head2->lock); 959 965 960 966 spin_unlock_bh(&head->lock); ··· 1058 1064 if (tb->fastreuse >= 0 || 1059 1065 tb->fastreuseport >= 0) 1060 1066 goto next_port; 1061 - WARN_ON(hlist_empty(&tb->owners)); 1067 + WARN_ON(hlist_empty(&tb->bhash2)); 1062 1068 if (!check_established(death_row, sk, 1063 1069 port, &tw)) 1064 1070 goto ok; ··· 1098 1104 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); 1099 1105 if (!tb2) { 1100 1106 tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep, net, 1101 - head2, port, l3mdev, sk); 1107 + head2, tb, sk); 1102 1108 if (!tb2) 1103 1109 goto error; 1104 1110 }

+3 -18

net/ipv4/inet_timewait_sock.c

··· 35 35 if (!tb) 36 36 return; 37 37 38 - __hlist_del(&tw->tw_bind_node); 38 + __sk_del_bind_node((struct sock *)tw); 39 39 tw->tw_tb = NULL; 40 - inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb); 41 - 42 - __hlist_del(&tw->tw_bind2_node); 43 40 tw->tw_tb2 = NULL; 44 41 inet_bind2_bucket_destroy(hashinfo->bind2_bucket_cachep, tb2); 42 + inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb); 45 43 46 44 __sock_put((struct sock *)tw); 47 45 } ··· 92 94 hlist_nulls_add_head_rcu(&tw->tw_node, list); 93 95 } 94 96 95 - static void inet_twsk_add_bind_node(struct inet_timewait_sock *tw, 96 - struct hlist_head *list) 97 - { 98 - hlist_add_head(&tw->tw_bind_node, list); 99 - } 100 - 101 - static void inet_twsk_add_bind2_node(struct inet_timewait_sock *tw, 102 - struct hlist_head *list) 103 - { 104 - hlist_add_head(&tw->tw_bind2_node, list); 105 - } 106 - 107 97 /* 108 98 * Enter the time wait state. This is called with locally disabled BH. 109 99 * Essentially we whip up a timewait bucket, copy the relevant info into it ··· 119 133 120 134 tw->tw_tb = icsk->icsk_bind_hash; 121 135 WARN_ON(!icsk->icsk_bind_hash); 122 - inet_twsk_add_bind_node(tw, &tw->tw_tb->owners); 123 136 124 137 tw->tw_tb2 = icsk->icsk_bind2_hash; 125 138 WARN_ON(!icsk->icsk_bind2_hash); 126 - inet_twsk_add_bind2_node(tw, &tw->tw_tb2->deathrow); 139 + sk_add_bind_node((struct sock *)tw, &tw->tw_tb2->owners); 127 140 128 141 spin_unlock(&bhead2->lock); 129 142 spin_unlock(&bhead->lock);

Configure Feed

Configure Feed