Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'tcp-take-care-of-tcp_get_timestamping_opt_stats-races'
Eric Dumazet says:
====================
tcp: take care of tcp_get_timestamping_opt_stats() races
tcp_get_timestamping_opt_stats() does not own the socket lock,
this is intentional.
It calls tcp_get_info_chrono_stats() while other threads could
change chrono fields in tcp_chrono_set(). It also reads many
tcp socket fields that can be modified by other cpus/threads.
I do not think we need coherent TCP socket state snapshot
in tcp_get_timestamping_opt_stats().
Add READ_ONCE()/WRITE_ONCE() or data_race() annotations.
Note that icsk_ca_state is a bitfield, thus not covered
in this series.
====================
Link: https://patch.msgid.link/20260416200319.3608680-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+106
-83
+8
-4
include/net/tcp.h
+8
-4
include/net/tcp.h
···
1513
1513
static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val)
1514
1514
{
1515
1515
WARN_ON_ONCE((int)val <= 0);
1516
-
tp->snd_cwnd = val;
1516
+
WRITE_ONCE(tp->snd_cwnd, val);
1517
1517
}
1518
1518
1519
1519
static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
···
2208
2208
const u32 now = tcp_jiffies32;
2209
2209
enum tcp_chrono old = tp->chrono_type;
2210
2210
2211
+
/* Following WRITE_ONCE()s pair with READ_ONCE()s in
2212
+
* tcp_get_info_chrono_stats().
2213
+
*/
2211
2214
if (old > TCP_CHRONO_UNSPEC)
2212
-
tp->chrono_stat[old - 1] += now - tp->chrono_start;
2213
-
tp->chrono_start = now;
2214
-
tp->chrono_type = new;
2215
+
WRITE_ONCE(tp->chrono_stat[old - 1],
2216
+
tp->chrono_stat[old - 1] + now - tp->chrono_start);
2217
+
WRITE_ONCE(tp->chrono_start, now);
2218
+
WRITE_ONCE(tp->chrono_type, new);
2215
2219
}
2216
2220
2217
2221
static inline void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type)
+1
-1
include/net/tcp_ecn.h
+1
-1
include/net/tcp_ecn.h
+1
-1
net/core/filter.c
+1
-1
net/core/filter.c
+37
-25
net/ipv4/tcp.c
+37
-25
net/ipv4/tcp.c
···
3425
3425
icsk->icsk_rto = TCP_TIMEOUT_INIT;
3426
3426
WRITE_ONCE(icsk->icsk_rto_min, TCP_RTO_MIN);
3427
3427
WRITE_ONCE(icsk->icsk_delack_max, TCP_DELACK_MAX);
3428
-
tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
3428
+
WRITE_ONCE(tp->snd_ssthresh, TCP_INFINITE_SSTHRESH);
3429
3429
tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
3430
3430
tp->snd_cwnd_cnt = 0;
3431
3431
tp->is_cwnd_limited = 0;
···
3623
3623
if (delta && sk->sk_state == TCP_ESTABLISHED) {
3624
3624
s64 srtt = (s64)tp->srtt_us + delta;
3625
3625
3626
-
tp->srtt_us = clamp_t(s64, srtt, 1, ~0U);
3626
+
WRITE_ONCE(tp->srtt_us,
3627
+
clamp_t(s64, srtt, 1, ~0U));
3627
3628
3628
3629
/* Note: does not deal with non zero icsk_backoff */
3629
3630
tcp_set_rto(sk);
···
4192
4191
struct tcp_info *info)
4193
4192
{
4194
4193
u64 stats[__TCP_CHRONO_MAX], total = 0;
4195
-
enum tcp_chrono i;
4194
+
enum tcp_chrono i, cur;
4196
4195
4196
+
/* Following READ_ONCE()s pair with WRITE_ONCE()s in tcp_chrono_set().
4197
+
* This is because socket lock might not be owned by us at this point.
4198
+
* This is best effort, tcp_get_timestamping_opt_stats() can
4199
+
* see wrong values. A real fix would be too costly for TCP fast path.
4200
+
*/
4201
+
cur = READ_ONCE(tp->chrono_type);
4197
4202
for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) {
4198
-
stats[i] = tp->chrono_stat[i - 1];
4199
-
if (i == tp->chrono_type)
4200
-
stats[i] += tcp_jiffies32 - tp->chrono_start;
4203
+
stats[i] = READ_ONCE(tp->chrono_stat[i - 1]);
4204
+
if (i == cur)
4205
+
stats[i] += tcp_jiffies32 - READ_ONCE(tp->chrono_start);
4201
4206
stats[i] *= USEC_PER_SEC / HZ;
4202
4207
total += stats[i];
4203
4208
}
···
4435
4428
nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED,
4436
4429
info.tcpi_sndbuf_limited, TCP_NLA_PAD);
4437
4430
nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT,
4438
-
tp->data_segs_out, TCP_NLA_PAD);
4431
+
READ_ONCE(tp->data_segs_out), TCP_NLA_PAD);
4439
4432
nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS,
4440
-
tp->total_retrans, TCP_NLA_PAD);
4433
+
READ_ONCE(tp->total_retrans), TCP_NLA_PAD);
4441
4434
4442
4435
rate = READ_ONCE(sk->sk_pacing_rate);
4443
4436
rate64 = (rate != ~0UL) ? rate : ~0ULL;
···
4446
4439
rate64 = tcp_compute_delivery_rate(tp);
4447
4440
nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD);
4448
4441
4449
-
nla_put_u32(stats, TCP_NLA_SND_CWND, tcp_snd_cwnd(tp));
4450
-
nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering);
4451
-
nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp));
4442
+
nla_put_u32(stats, TCP_NLA_SND_CWND, READ_ONCE(tp->snd_cwnd));
4443
+
nla_put_u32(stats, TCP_NLA_REORDERING, READ_ONCE(tp->reordering));
4444
+
nla_put_u32(stats, TCP_NLA_MIN_RTT, data_race(tcp_min_rtt(tp)));
4452
4445
4453
4446
nla_put_u8(stats, TCP_NLA_RECUR_RETRANS,
4454
4447
READ_ONCE(inet_csk(sk)->icsk_retransmits));
4455
-
nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited);
4456
-
nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh);
4457
-
nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered);
4458
-
nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce);
4448
+
nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, data_race(!!tp->rate_app_limited));
4449
+
nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, READ_ONCE(tp->snd_ssthresh));
4450
+
nla_put_u32(stats, TCP_NLA_DELIVERED, READ_ONCE(tp->delivered));
4451
+
nla_put_u32(stats, TCP_NLA_DELIVERED_CE, READ_ONCE(tp->delivered_ce));
4459
4452
4460
-
nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una);
4453
+
nla_put_u32(stats, TCP_NLA_SNDQ_SIZE,
4454
+
max_t(int, 0,
4455
+
READ_ONCE(tp->write_seq) - READ_ONCE(tp->snd_una)));
4461
4456
nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state);
4462
4457
4463
-
nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent,
4458
+
nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, READ_ONCE(tp->bytes_sent),
4464
4459
TCP_NLA_PAD);
4465
-
nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans,
4466
-
TCP_NLA_PAD);
4467
-
nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups);
4468
-
nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen);
4469
-
nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3);
4470
-
nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash);
4460
+
nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS,
4461
+
READ_ONCE(tp->bytes_retrans), TCP_NLA_PAD);
4462
+
nla_put_u32(stats, TCP_NLA_DSACK_DUPS, READ_ONCE(tp->dsack_dups));
4463
+
nla_put_u32(stats, TCP_NLA_REORD_SEEN, READ_ONCE(tp->reord_seen));
4464
+
nla_put_u32(stats, TCP_NLA_SRTT, READ_ONCE(tp->srtt_us) >> 3);
4465
+
nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH,
4466
+
READ_ONCE(tp->timeout_rehash));
4471
4467
nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT,
4472
-
max_t(int, 0, tp->write_seq - tp->snd_nxt));
4468
+
max_t(int, 0,
4469
+
READ_ONCE(tp->write_seq) - READ_ONCE(tp->snd_nxt)));
4473
4470
nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns,
4474
4471
TCP_NLA_PAD);
4475
4472
if (ack_skb)
4476
4473
nla_put_u8(stats, TCP_NLA_TTL,
4477
4474
tcp_skb_ttl_or_hop_limit(ack_skb));
4478
4475
4479
-
nla_put_u32(stats, TCP_NLA_REHASH, tp->plb_rehash + tp->timeout_rehash);
4476
+
nla_put_u32(stats, TCP_NLA_REHASH,
4477
+
READ_ONCE(tp->plb_rehash) + READ_ONCE(tp->timeout_rehash));
4480
4478
return stats;
4481
4479
}
4482
4480
+3
-3
net/ipv4/tcp_bbr.c
+3
-3
net/ipv4/tcp_bbr.c
···
897
897
898
898
if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) {
899
899
bbr->mode = BBR_DRAIN; /* drain queue we created */
900
-
tcp_sk(sk)->snd_ssthresh =
901
-
bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT);
900
+
WRITE_ONCE(tcp_sk(sk)->snd_ssthresh,
901
+
bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT));
902
902
} /* fall through to check if in-flight is already small: */
903
903
if (bbr->mode == BBR_DRAIN &&
904
904
bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <=
···
1043
1043
struct bbr *bbr = inet_csk_ca(sk);
1044
1044
1045
1045
bbr->prior_cwnd = 0;
1046
-
tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
1046
+
WRITE_ONCE(tp->snd_ssthresh, TCP_INFINITE_SSTHRESH);
1047
1047
bbr->rtt_cnt = 0;
1048
1048
bbr->next_rtt_delivered = tp->delivered;
1049
1049
bbr->prev_ca_state = TCP_CA_Open;
+1
-1
net/ipv4/tcp_bic.c
+1
-1
net/ipv4/tcp_bic.c
+2
-2
net/ipv4/tcp_cdg.c
+2
-2
net/ipv4/tcp_cdg.c
···
162
162
NET_ADD_STATS(sock_net(sk),
163
163
LINUX_MIB_TCPHYSTARTTRAINCWND,
164
164
tcp_snd_cwnd(tp));
165
-
tp->snd_ssthresh = tcp_snd_cwnd(tp);
165
+
WRITE_ONCE(tp->snd_ssthresh, tcp_snd_cwnd(tp));
166
166
return;
167
167
}
168
168
}
···
181
181
NET_ADD_STATS(sock_net(sk),
182
182
LINUX_MIB_TCPHYSTARTDELAYCWND,
183
183
tcp_snd_cwnd(tp));
184
-
tp->snd_ssthresh = tcp_snd_cwnd(tp);
184
+
WRITE_ONCE(tp->snd_ssthresh, tcp_snd_cwnd(tp));
185
185
}
186
186
}
187
187
}
+3
-3
net/ipv4/tcp_cubic.c
+3
-3
net/ipv4/tcp_cubic.c
···
136
136
bictcp_hystart_reset(sk);
137
137
138
138
if (!hystart && initial_ssthresh)
139
-
tcp_sk(sk)->snd_ssthresh = initial_ssthresh;
139
+
WRITE_ONCE(tcp_sk(sk)->snd_ssthresh, initial_ssthresh);
140
140
}
141
141
142
142
__bpf_kfunc static void cubictcp_cwnd_event_tx_start(struct sock *sk)
···
420
420
NET_ADD_STATS(sock_net(sk),
421
421
LINUX_MIB_TCPHYSTARTTRAINCWND,
422
422
tcp_snd_cwnd(tp));
423
-
tp->snd_ssthresh = tcp_snd_cwnd(tp);
423
+
WRITE_ONCE(tp->snd_ssthresh, tcp_snd_cwnd(tp));
424
424
}
425
425
}
426
426
}
···
440
440
NET_ADD_STATS(sock_net(sk),
441
441
LINUX_MIB_TCPHYSTARTDELAYCWND,
442
442
tcp_snd_cwnd(tp));
443
-
tp->snd_ssthresh = tcp_snd_cwnd(tp);
443
+
WRITE_ONCE(tp->snd_ssthresh, tcp_snd_cwnd(tp));
444
444
}
445
445
}
446
446
}
+1
-1
net/ipv4/tcp_dctcp.c
+1
-1
net/ipv4/tcp_dctcp.c
···
177
177
struct tcp_sock *tp = tcp_sk(sk);
178
178
179
179
ca->loss_cwnd = tcp_snd_cwnd(tp);
180
-
tp->snd_ssthresh = max(tcp_snd_cwnd(tp) >> 1U, 2U);
180
+
WRITE_ONCE(tp->snd_ssthresh, max(tcp_snd_cwnd(tp) >> 1U, 2U));
181
181
}
182
182
183
183
__bpf_kfunc static void dctcp_state(struct sock *sk, u8 new_state)
+22
-20
net/ipv4/tcp_input.c
+22
-20
net/ipv4/tcp_input.c
···
476
476
477
477
static void tcp_count_delivered_ce(struct tcp_sock *tp, u32 ecn_count)
478
478
{
479
-
tp->delivered_ce += ecn_count;
479
+
WRITE_ONCE(tp->delivered_ce, tp->delivered_ce + ecn_count);
480
480
}
481
481
482
482
/* Updates the delivered and delivered_ce counts */
483
483
static void tcp_count_delivered(struct tcp_sock *tp, u32 delivered,
484
484
bool ece_ack)
485
485
{
486
-
tp->delivered += delivered;
486
+
WRITE_ONCE(tp->delivered, tp->delivered + delivered);
487
487
if (tcp_ecn_mode_rfc3168(tp) && ece_ack)
488
488
tcp_count_delivered_ce(tp, delivered);
489
489
}
···
1132
1132
1133
1133
tcp_bpf_rtt(sk, mrtt_us, srtt);
1134
1134
}
1135
-
tp->srtt_us = max(1U, srtt);
1135
+
WRITE_ONCE(tp->srtt_us, max(1U, srtt));
1136
1136
}
1137
1137
1138
1138
void tcp_update_pacing_rate(struct sock *sk)
···
1246
1246
else if (tp->tlp_high_seq && tp->tlp_high_seq == end_seq)
1247
1247
state->flag |= FLAG_DSACK_TLP;
1248
1248
1249
-
tp->dsack_dups += dup_segs;
1249
+
WRITE_ONCE(tp->dsack_dups, tp->dsack_dups + dup_segs);
1250
1250
/* Skip the DSACK if dup segs weren't retransmitted by sender */
1251
1251
if (tp->dsack_dups > tp->total_retrans)
1252
1252
return 0;
···
1293
1293
tp->sacked_out,
1294
1294
tp->undo_marker ? tp->undo_retrans : 0);
1295
1295
#endif
1296
-
tp->reordering = min_t(u32, (metric + mss - 1) / mss,
1297
-
READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering));
1296
+
WRITE_ONCE(tp->reordering,
1297
+
min_t(u32, (metric + mss - 1) / mss,
1298
+
READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering)));
1298
1299
}
1299
1300
1300
1301
/* This exciting event is worth to be remembered. 8) */
1301
-
tp->reord_seen++;
1302
+
WRITE_ONCE(tp->reord_seen, tp->reord_seen + 1);
1302
1303
NET_INC_STATS(sock_net(sk),
1303
1304
ts ? LINUX_MIB_TCPTSREORDER : LINUX_MIB_TCPSACKREORDER);
1304
1305
}
···
2440
2439
if (!tcp_limit_reno_sacked(tp))
2441
2440
return;
2442
2441
2443
-
tp->reordering = min_t(u32, tp->packets_out + addend,
2444
-
READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering));
2445
-
tp->reord_seen++;
2442
+
WRITE_ONCE(tp->reordering,
2443
+
min_t(u32, tp->packets_out + addend,
2444
+
READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering)));
2445
+
WRITE_ONCE(tp->reord_seen, tp->reord_seen + 1);
2446
2446
NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRENOREORDER);
2447
2447
}
2448
2448
···
2567
2565
(icsk->icsk_ca_state == TCP_CA_Loss && !icsk->icsk_retransmits)) {
2568
2566
tp->prior_ssthresh = tcp_current_ssthresh(sk);
2569
2567
tp->prior_cwnd = tcp_snd_cwnd(tp);
2570
-
tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk);
2568
+
WRITE_ONCE(tp->snd_ssthresh, icsk->icsk_ca_ops->ssthresh(sk));
2571
2569
tcp_ca_event(sk, CA_EVENT_LOSS);
2572
2570
tcp_init_undo(tp);
2573
2571
}
···
2581
2579
reordering = READ_ONCE(net->ipv4.sysctl_tcp_reordering);
2582
2580
if (icsk->icsk_ca_state <= TCP_CA_Disorder &&
2583
2581
tp->sacked_out >= reordering)
2584
-
tp->reordering = min_t(unsigned int, tp->reordering,
2585
-
reordering);
2582
+
WRITE_ONCE(tp->reordering,
2583
+
min_t(unsigned int, tp->reordering, reordering));
2586
2584
2587
2585
tcp_set_ca_state(sk, TCP_CA_Loss);
2588
2586
tp->high_seq = tp->snd_nxt;
···
2860
2858
tcp_snd_cwnd_set(tp, icsk->icsk_ca_ops->undo_cwnd(sk));
2861
2859
2862
2860
if (tp->prior_ssthresh > tp->snd_ssthresh) {
2863
-
tp->snd_ssthresh = tp->prior_ssthresh;
2861
+
WRITE_ONCE(tp->snd_ssthresh, tp->prior_ssthresh);
2864
2862
tcp_ecn_withdraw_cwr(tp);
2865
2863
}
2866
2864
}
···
2978
2976
tp->prior_cwnd = tcp_snd_cwnd(tp);
2979
2977
tp->prr_delivered = 0;
2980
2978
tp->prr_out = 0;
2981
-
tp->snd_ssthresh = inet_csk(sk)->icsk_ca_ops->ssthresh(sk);
2979
+
WRITE_ONCE(tp->snd_ssthresh, inet_csk(sk)->icsk_ca_ops->ssthresh(sk));
2982
2980
tcp_ecn_queue_cwr(tp);
2983
2981
}
2984
2982
···
3120
3118
3121
3119
if (icsk->icsk_ca_state != TCP_CA_Loss) {
3122
3120
tp->high_seq = tp->snd_nxt;
3123
-
tp->snd_ssthresh = tcp_current_ssthresh(sk);
3121
+
WRITE_ONCE(tp->snd_ssthresh, tcp_current_ssthresh(sk));
3124
3122
tp->prior_ssthresh = 0;
3125
3123
tp->undo_marker = 0;
3126
3124
tcp_set_ca_state(sk, TCP_CA_Loss);
···
3912
3910
sock_owned_by_me((struct sock *)tp);
3913
3911
tp->bytes_acked += delta;
3914
3912
tcp_snd_sne_update(tp, ack);
3915
-
tp->snd_una = ack;
3913
+
WRITE_ONCE(tp->snd_una, ack);
3916
3914
}
3917
3915
3918
3916
static void tcp_rcv_sne_update(struct tcp_sock *tp, u32 seq)
···
6779
6777
NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVE);
6780
6778
/* SYN-data is counted as two separate packets in tcp_ack() */
6781
6779
if (tp->delivered > 1)
6782
-
--tp->delivered;
6780
+
WRITE_ONCE(tp->delivered, tp->delivered - 1);
6783
6781
}
6784
6782
6785
6783
tcp_fastopen_add_skb(sk, synack);
···
7212
7210
SKB_DR_SET(reason, NOT_SPECIFIED);
7213
7211
switch (sk->sk_state) {
7214
7212
case TCP_SYN_RECV:
7215
-
tp->delivered++; /* SYN-ACK delivery isn't tracked in tcp_ack */
7213
+
WRITE_ONCE(tp->delivered, tp->delivered + 1); /* SYN-ACK delivery isn't tracked in tcp_ack */
7216
7214
if (!tp->srtt_us)
7217
7215
tcp_synack_rtt_meas(sk, req);
7218
7216
···
7240
7238
if (sk->sk_socket)
7241
7239
sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);
7242
7240
7243
-
tp->snd_una = TCP_SKB_CB(skb)->ack_seq;
7241
+
WRITE_ONCE(tp->snd_una, TCP_SKB_CB(skb)->ack_seq);
7244
7242
tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale;
7245
7243
tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
7246
7244
+3
-3
net/ipv4/tcp_metrics.c
+3
-3
net/ipv4/tcp_metrics.c
···
490
490
val = READ_ONCE(net->ipv4.sysctl_tcp_no_ssthresh_metrics_save) ?
491
491
0 : tcp_metric_get(tm, TCP_METRIC_SSTHRESH);
492
492
if (val) {
493
-
tp->snd_ssthresh = val;
493
+
WRITE_ONCE(tp->snd_ssthresh, val);
494
494
if (tp->snd_ssthresh > tp->snd_cwnd_clamp)
495
-
tp->snd_ssthresh = tp->snd_cwnd_clamp;
495
+
WRITE_ONCE(tp->snd_ssthresh, tp->snd_cwnd_clamp);
496
496
}
497
497
val = tcp_metric_get(tm, TCP_METRIC_REORDERING);
498
498
if (val && tp->reordering != val)
499
-
tp->reordering = val;
499
+
WRITE_ONCE(tp->reordering, val);
500
500
501
501
crtt = tcp_metric_get(tm, TCP_METRIC_RTT);
502
502
rcu_read_unlock();
+2
-2
net/ipv4/tcp_nv.c
+2
-2
net/ipv4/tcp_nv.c
···
396
396
397
397
/* We have enough data to determine we are congested */
398
398
ca->nv_allow_cwnd_growth = 0;
399
-
tp->snd_ssthresh =
400
-
(nv_ssthresh_factor * max_win) >> 3;
399
+
WRITE_ONCE(tp->snd_ssthresh,
400
+
(nv_ssthresh_factor * max_win) >> 3);
401
401
if (tcp_snd_cwnd(tp) - max_win > 2) {
402
402
/* gap > 2, we do exponential cwnd decrease */
403
403
int dec;
+11
-8
net/ipv4/tcp_output.c
+11
-8
net/ipv4/tcp_output.c
···
171
171
172
172
tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
173
173
174
-
tp->snd_ssthresh = tcp_current_ssthresh(sk);
174
+
WRITE_ONCE(tp->snd_ssthresh, tcp_current_ssthresh(sk));
175
175
restart_cwnd = min(restart_cwnd, cwnd);
176
176
177
177
while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
···
1688
1688
1689
1689
if (skb->len != tcp_header_size) {
1690
1690
tcp_event_data_sent(tp, sk);
1691
-
tp->data_segs_out += tcp_skb_pcount(skb);
1692
-
tp->bytes_sent += skb->len - tcp_header_size;
1691
+
WRITE_ONCE(tp->data_segs_out,
1692
+
tp->data_segs_out + tcp_skb_pcount(skb));
1693
+
WRITE_ONCE(tp->bytes_sent,
1694
+
tp->bytes_sent + skb->len - tcp_header_size);
1693
1695
}
1694
1696
1695
1697
if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
···
2144
2142
u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk));
2145
2143
u32 win_used = max(tp->snd_cwnd_used, init_win);
2146
2144
if (win_used < tcp_snd_cwnd(tp)) {
2147
-
tp->snd_ssthresh = tcp_current_ssthresh(sk);
2145
+
WRITE_ONCE(tp->snd_ssthresh, tcp_current_ssthresh(sk));
2148
2146
tcp_snd_cwnd_set(tp, (tcp_snd_cwnd(tp) + win_used) >> 1);
2149
2147
}
2150
2148
tp->snd_cwnd_used = 0;
···
3644
3642
TCP_ADD_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS, segs);
3645
3643
if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
3646
3644
__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
3647
-
tp->total_retrans += segs;
3648
-
tp->bytes_retrans += skb->len;
3645
+
WRITE_ONCE(tp->total_retrans, tp->total_retrans + segs);
3646
+
WRITE_ONCE(tp->bytes_retrans, tp->bytes_retrans + skb->len);
3649
3647
3650
3648
/* make sure skb->data is aligned on arches that require it
3651
3649
* and check if ack-trimming & collapsing extended the headroom
···
4154
4152
tp->snd_wnd = 0;
4155
4153
tcp_init_wl(tp, 0);
4156
4154
tcp_write_queue_purge(sk);
4157
-
tp->snd_una = tp->write_seq;
4155
+
WRITE_ONCE(tp->snd_una, tp->write_seq);
4158
4156
tp->snd_sml = tp->write_seq;
4159
4157
tp->snd_up = tp->write_seq;
4160
4158
WRITE_ONCE(tp->snd_nxt, tp->write_seq);
···
4648
4646
* However in this case, we are dealing with a passive fastopen
4649
4647
* socket thus we can change total_retrans value.
4650
4648
*/
4651
-
tcp_sk_rw(sk)->total_retrans++;
4649
+
WRITE_ONCE(tcp_sk_rw(sk)->total_retrans,
4650
+
tcp_sk_rw(sk)->total_retrans + 1);
4652
4651
}
4653
4652
trace_tcp_retransmit_synack(sk, req);
4654
4653
WRITE_ONCE(req->num_retrans, req->num_retrans + 1);
+1
-1
net/ipv4/tcp_plb.c
+1
-1
net/ipv4/tcp_plb.c
+1
-1
net/ipv4/tcp_timer.c
+1
-1
net/ipv4/tcp_timer.c
+5
-4
net/ipv4/tcp_vegas.c
+5
-4
net/ipv4/tcp_vegas.c
···
245
245
*/
246
246
tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp),
247
247
(u32)target_cwnd + 1));
248
-
tp->snd_ssthresh = tcp_vegas_ssthresh(tp);
248
+
WRITE_ONCE(tp->snd_ssthresh,
249
+
tcp_vegas_ssthresh(tp));
249
250
250
251
} else if (tcp_in_slow_start(tp)) {
251
252
/* Slow start. */
···
262
261
* we slow down.
263
262
*/
264
263
tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - 1);
265
-
tp->snd_ssthresh
266
-
= tcp_vegas_ssthresh(tp);
264
+
WRITE_ONCE(tp->snd_ssthresh,
265
+
tcp_vegas_ssthresh(tp));
267
266
} else if (diff < alpha) {
268
267
/* We don't have enough extra packets
269
268
* in the network, so speed up.
···
281
280
else if (tcp_snd_cwnd(tp) > tp->snd_cwnd_clamp)
282
281
tcp_snd_cwnd_set(tp, tp->snd_cwnd_clamp);
283
282
284
-
tp->snd_ssthresh = tcp_current_ssthresh(sk);
283
+
WRITE_ONCE(tp->snd_ssthresh, tcp_current_ssthresh(sk));
285
284
}
286
285
287
286
/* Wipe the slate clean for the next RTT. */
+2
-2
net/ipv4/tcp_westwood.c
+2
-2
net/ipv4/tcp_westwood.c
···
244
244
245
245
switch (event) {
246
246
case CA_EVENT_COMPLETE_CWR:
247
-
tp->snd_ssthresh = tcp_westwood_bw_rttmin(sk);
247
+
WRITE_ONCE(tp->snd_ssthresh, tcp_westwood_bw_rttmin(sk));
248
248
tcp_snd_cwnd_set(tp, tp->snd_ssthresh);
249
249
break;
250
250
case CA_EVENT_LOSS:
251
-
tp->snd_ssthresh = tcp_westwood_bw_rttmin(sk);
251
+
WRITE_ONCE(tp->snd_ssthresh, tcp_westwood_bw_rttmin(sk));
252
252
/* Update RTT_min when next ack arrives */
253
253
w->reset_rtt_min = 1;
254
254
break;