Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
rxrpc: Use the new rxrpc_tx_queue struct to more efficiently process ACKs
With the change in the structure of the transmission buffer to store
buffers in bunches of 32 or 64 (BITS_PER_LONG) we can place sets of
per-buffer flags into the rxrpc_tx_queue struct rather than storing them in
rxrpc_tx_buf, thereby vastly increasing efficiency when assessing the SACK
table in an ACK packet.
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Marc Dionne <marc.dionne@auristor.com>
cc: linux-afs@lists.infradead.org
Link: https://patch.msgid.link/20241204074710.990092-24-dhowells@redhat.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
authored by
David Howells
and committed by
Jakub Kicinski
9b052c6b
f7dd0dc9
+356
-208
+76
-12
include/trace/events/rxrpc.h
+76
-12
include/trace/events/rxrpc.h
···
132
132
EM(rxrpc_skb_get_call_rx, "GET call-rx ") \
133
133
EM(rxrpc_skb_get_conn_secured, "GET conn-secd") \
134
134
EM(rxrpc_skb_get_conn_work, "GET conn-work") \
135
-
EM(rxrpc_skb_get_last_nack, "GET last-nack") \
136
135
EM(rxrpc_skb_get_local_work, "GET locl-work") \
137
136
EM(rxrpc_skb_get_reject_work, "GET rej-work ") \
138
137
EM(rxrpc_skb_get_to_recvmsg, "GET to-recv ") \
···
146
147
EM(rxrpc_skb_put_error_report, "PUT error-rep") \
147
148
EM(rxrpc_skb_put_input, "PUT input ") \
148
149
EM(rxrpc_skb_put_jumbo_subpacket, "PUT jumbo-sub") \
149
-
EM(rxrpc_skb_put_last_nack, "PUT last-nack") \
150
150
EM(rxrpc_skb_put_purge, "PUT purge ") \
151
151
EM(rxrpc_skb_put_rotate, "PUT rotate ") \
152
152
EM(rxrpc_skb_put_unknown, "PUT unknown ") \
···
497
499
EM(rxrpc_pmtud_reduce_icmp, "Icmp ") \
498
500
E_(rxrpc_pmtud_reduce_route, "Route")
499
501
502
+
#define rxrpc_rotate_traces \
503
+
EM(rxrpc_rotate_trace_hack, "hard-ack") \
504
+
EM(rxrpc_rotate_trace_sack, "soft-ack") \
505
+
E_(rxrpc_rotate_trace_snak, "soft-nack")
506
+
500
507
/*
501
508
* Generate enums for tracing information.
502
509
*/
···
528
525
enum rxrpc_receive_trace { rxrpc_receive_traces } __mode(byte);
529
526
enum rxrpc_recvmsg_trace { rxrpc_recvmsg_traces } __mode(byte);
530
527
enum rxrpc_req_ack_trace { rxrpc_req_ack_traces } __mode(byte);
528
+
enum rxrpc_rotate_trace { rxrpc_rotate_traces } __mode(byte);
531
529
enum rxrpc_rtt_rx_trace { rxrpc_rtt_rx_traces } __mode(byte);
532
530
enum rxrpc_rtt_tx_trace { rxrpc_rtt_tx_traces } __mode(byte);
533
531
enum rxrpc_sack_trace { rxrpc_sack_traces } __mode(byte);
···
566
562
rxrpc_receive_traces;
567
563
rxrpc_recvmsg_traces;
568
564
rxrpc_req_ack_traces;
565
+
rxrpc_rotate_traces;
569
566
rxrpc_rtt_rx_traces;
570
567
rxrpc_rtt_tx_traces;
571
568
rxrpc_sack_traces;
···
1672
1667
1673
1668
TP_STRUCT__entry(
1674
1669
__field(unsigned int, call)
1670
+
__field(unsigned int, qbase)
1675
1671
__field(rxrpc_seq_t, seq)
1676
1672
__field(rxrpc_serial_t, serial)
1677
1673
__field(ktime_t, expiry)
···
1680
1674
1681
1675
TP_fast_assign(
1682
1676
__entry->call = call->debug_id;
1677
+
__entry->qbase = req->tq->qbase;
1683
1678
__entry->seq = req->seq;
1684
1679
__entry->serial = txb->serial;
1685
1680
__entry->expiry = expiry;
1686
1681
),
1687
1682
1688
-
TP_printk("c=%08x q=%x r=%x xp=%lld",
1683
+
TP_printk("c=%08x tq=%x q=%x r=%x xp=%lld",
1689
1684
__entry->call,
1685
+
__entry->qbase,
1690
1686
__entry->seq,
1691
1687
__entry->serial,
1692
1688
ktime_to_us(__entry->expiry))
···
1732
1724
memcpy(&__entry->sum, summary, sizeof(__entry->sum));
1733
1725
),
1734
1726
1735
-
TP_printk("c=%08x r=%08x %s q=%08x %s cw=%u ss=%u nA=%u,%u+%u,%u b=%u u=%u d=%u l=%x%s%s%s",
1727
+
TP_printk("c=%08x r=%08x %s q=%08x %s cw=%u ss=%u A=%u+%u/%u+%u r=%u b=%u u=%u d=%u l=%x%s%s%s",
1736
1728
__entry->call,
1737
1729
__entry->ack_serial,
1738
1730
__print_symbolic(__entry->sum.ack_reason, rxrpc_ack_names),
···
1740
1732
__print_symbolic(__entry->ca_state, rxrpc_ca_states),
1741
1733
__entry->cwnd,
1742
1734
__entry->ssthresh,
1743
-
__entry->nr_sacks, __entry->sum.nr_retained_snacks,
1744
-
__entry->sum.nr_new_sacks,
1745
-
__entry->sum.nr_new_snacks,
1735
+
__entry->nr_sacks, __entry->sum.nr_new_sacks,
1736
+
__entry->nr_snacks, __entry->sum.nr_new_snacks,
1737
+
__entry->sum.nr_new_hacks,
1746
1738
__entry->top - __entry->hard_ack,
1747
1739
__entry->cumul_acks,
1748
1740
__entry->dup_acks,
···
1858
1850
&__entry->srx.transport)
1859
1851
);
1860
1852
1861
-
TRACE_EVENT(rxrpc_resend,
1862
-
TP_PROTO(struct rxrpc_call *call, struct sk_buff *ack),
1853
+
TRACE_EVENT(rxrpc_apply_acks,
1854
+
TP_PROTO(struct rxrpc_call *call, struct rxrpc_txqueue *tq),
1863
1855
1864
-
TP_ARGS(call, ack),
1856
+
TP_ARGS(call, tq),
1857
+
1858
+
TP_STRUCT__entry(
1859
+
__field(unsigned int, call)
1860
+
__field(unsigned int, nr_rep)
1861
+
__field(rxrpc_seq_t, qbase)
1862
+
__field(unsigned long, acks)
1863
+
),
1864
+
1865
+
TP_fast_assign(
1866
+
__entry->call = call->debug_id;
1867
+
__entry->qbase = tq->qbase;
1868
+
__entry->acks = tq->segment_acked;
1869
+
__entry->nr_rep = tq->nr_reported_acks;
1870
+
),
1871
+
1872
+
TP_printk("c=%08x tq=%x acks=%016lx rep=%u",
1873
+
__entry->call,
1874
+
__entry->qbase,
1875
+
__entry->acks,
1876
+
__entry->nr_rep)
1877
+
);
1878
+
1879
+
TRACE_EVENT(rxrpc_resend,
1880
+
TP_PROTO(struct rxrpc_call *call, rxrpc_serial_t ack_serial),
1881
+
1882
+
TP_ARGS(call, ack_serial),
1865
1883
1866
1884
TP_STRUCT__entry(
1867
1885
__field(unsigned int, call)
···
1897
1863
),
1898
1864
1899
1865
TP_fast_assign(
1900
-
struct rxrpc_skb_priv *sp = ack ? rxrpc_skb(ack) : NULL;
1901
1866
__entry->call = call->debug_id;
1902
1867
__entry->seq = call->acks_hard_ack;
1903
1868
__entry->transmitted = call->tx_transmitted;
1904
-
__entry->ack_serial = sp ? sp->hdr.serial : 0;
1869
+
__entry->ack_serial = ack_serial;
1905
1870
),
1906
1871
1907
1872
TP_printk("c=%08x r=%x q=%x tq=%x",
···
1908
1875
__entry->ack_serial,
1909
1876
__entry->seq,
1910
1877
__entry->transmitted)
1878
+
);
1879
+
1880
+
TRACE_EVENT(rxrpc_rotate,
1881
+
TP_PROTO(struct rxrpc_call *call, struct rxrpc_txqueue *tq,
1882
+
struct rxrpc_ack_summary *summary, rxrpc_seq_t seq,
1883
+
enum rxrpc_rotate_trace trace),
1884
+
1885
+
TP_ARGS(call, tq, summary, seq, trace),
1886
+
1887
+
TP_STRUCT__entry(
1888
+
__field(unsigned int, call)
1889
+
__field(rxrpc_seq_t, qbase)
1890
+
__field(rxrpc_seq_t, seq)
1891
+
__field(unsigned int, nr_rep)
1892
+
__field(enum rxrpc_rotate_trace, trace)
1893
+
),
1894
+
1895
+
TP_fast_assign(
1896
+
__entry->call = call->debug_id;
1897
+
__entry->qbase = tq->qbase;
1898
+
__entry->seq = seq;
1899
+
__entry->nr_rep = tq->nr_reported_acks;
1900
+
__entry->trace = trace;
1901
+
),
1902
+
1903
+
TP_printk("c=%08x tq=%x q=%x nr=%x %s",
1904
+
__entry->call,
1905
+
__entry->qbase,
1906
+
__entry->seq,
1907
+
__entry->nr_rep,
1908
+
__print_symbolic(__entry->trace, rxrpc_rotate_traces))
1911
1909
);
1912
1910
1913
1911
TRACE_EVENT(rxrpc_rx_icmp,
+17
-6
net/rxrpc/ar-internal.h
+17
-6
net/rxrpc/ar-internal.h
···
214
214
rxrpc_seq_t first_ack; /* First packet in acks table */
215
215
rxrpc_seq_t prev_ack; /* Highest seq seen */
216
216
rxrpc_serial_t acked_serial; /* Packet in response to (or 0) */
217
+
u16 nr_acks; /* Number of acks+nacks */
217
218
u8 reason; /* Reason for ack */
218
-
u8 nr_acks; /* Number of acks+nacks */
219
-
u8 nr_nacks; /* Number of nacks */
220
219
} ack;
221
220
};
222
221
struct rxrpc_host_header hdr; /* RxRPC packet header from this packet */
···
733
734
u16 cong_dup_acks; /* Count of ACKs showing missing packets */
734
735
u16 cong_cumul_acks; /* Cumulative ACK count */
735
736
ktime_t cong_tstamp; /* Last time cwnd was changed */
736
-
struct sk_buff *cong_last_nack; /* Last ACK with nacks received */
737
737
738
738
/* Receive-phase ACK management (ACKs we send). */
739
739
u8 ackr_reason; /* reason to ACK */
···
773
775
u16 nr_new_hacks; /* Number of rotated new ACKs */
774
776
u16 nr_new_sacks; /* Number of new soft ACKs in packet */
775
777
u16 nr_new_snacks; /* Number of new soft nacks in packet */
776
-
u16 nr_retained_snacks; /* Number of nacks retained between ACKs */
777
778
u8 ack_reason;
778
-
bool saw_snacks:1; /* T if we saw a soft NACK */
779
779
bool new_low_snack:1; /* T if new low soft NACK found */
780
780
bool retrans_timeo:1; /* T if reTx due to timeout happened */
781
+
bool need_retransmit:1; /* T if we need transmission */
781
782
u8 /*enum rxrpc_congest_change*/ change;
782
783
};
783
784
···
855
858
struct rxrpc_txqueue *next;
856
859
ktime_t xmit_ts_base;
857
860
rxrpc_seq_t qbase;
861
+
u8 nr_reported_acks; /* Number of segments explicitly acked/nacked */
862
+
unsigned long segment_acked; /* Bit-per-buf: Set if ACK'd */
863
+
unsigned long segment_lost; /* Bit-per-buf: Set if declared lost */
864
+
unsigned long segment_retransmitted; /* Bit-per-buf: Set if retransmitted */
858
865
859
866
/* The arrays we want to pack into as few cache lines as possible. */
860
867
struct {
···
936
935
enum rxrpc_propose_ack_trace why);
937
936
void rxrpc_propose_delay_ACK(struct rxrpc_call *, rxrpc_serial_t,
938
937
enum rxrpc_propose_ack_trace);
939
-
void rxrpc_resend(struct rxrpc_call *call, struct sk_buff *ack_skb);
938
+
void rxrpc_resend(struct rxrpc_call *call, rxrpc_serial_t ack_serial, bool ping_response);
940
939
941
940
bool rxrpc_input_call_event(struct rxrpc_call *call);
942
941
···
1382
1381
static inline bool after_eq(u32 seq1, u32 seq2)
1383
1382
{
1384
1383
return (s32)(seq1 - seq2) >= 0;
1384
+
}
1385
+
1386
+
static inline u32 earliest(u32 seq1, u32 seq2)
1387
+
{
1388
+
return before(seq1, seq2) ? seq1 : seq2;
1389
+
}
1390
+
1391
+
static inline u32 latest(u32 seq1, u32 seq2)
1392
+
{
1393
+
return after(seq1, seq2) ? seq1 : seq2;
1385
1394
}
1386
1395
1387
1396
static inline void rxrpc_queue_rx_call_packet(struct rxrpc_call *call, struct sk_buff *skb)
+90
-97
net/rxrpc/call_event.c
+90
-97
net/rxrpc/call_event.c
···
65
65
/*
66
66
* Retransmit one or more packets.
67
67
*/
68
-
static void rxrpc_retransmit_data(struct rxrpc_call *call,
68
+
static bool rxrpc_retransmit_data(struct rxrpc_call *call,
69
69
struct rxrpc_send_data_req *req,
70
-
ktime_t rto)
70
+
ktime_t rto, bool skip_too_young)
71
71
{
72
72
struct rxrpc_txqueue *tq = req->tq;
73
73
unsigned int ix = req->seq & RXRPC_TXQ_MASK;
···
78
78
79
79
xmit_ts = ktime_add_us(tq->xmit_ts_base, tq->segment_xmit_ts[ix]);
80
80
resend_at = ktime_add(xmit_ts, rto);
81
-
trace_rxrpc_retransmit(call, req, txb,
82
-
ktime_sub(resend_at, req->now));
81
+
trace_rxrpc_retransmit(call, req, txb, ktime_sub(resend_at, req->now));
82
+
if (skip_too_young && ktime_after(resend_at, req->now))
83
+
return false;
83
84
85
+
__set_bit(ix, &tq->segment_retransmitted);
84
86
txb->flags |= RXRPC_TXBUF_RESENT;
85
87
rxrpc_send_data_packet(call, req);
86
88
rxrpc_inc_stat(call->rxnet, stat_tx_data_retrans);
···
91
89
req->n = 0;
92
90
req->did_send = true;
93
91
req->now = ktime_get_real();
92
+
return true;
94
93
}
95
94
96
95
/*
97
96
* Perform retransmission of NAK'd and unack'd packets.
98
97
*/
99
-
void rxrpc_resend(struct rxrpc_call *call, struct sk_buff *ack_skb)
98
+
void rxrpc_resend(struct rxrpc_call *call, rxrpc_serial_t ack_serial, bool ping_response)
100
99
{
101
100
struct rxrpc_send_data_req req = {
102
101
.now = ktime_get_real(),
103
102
};
104
-
struct rxrpc_ackpacket *ack = NULL;
105
-
struct rxrpc_skb_priv *sp;
106
-
struct rxrpc_txqueue *tq;
107
-
struct rxrpc_txbuf *txb;
108
-
rxrpc_seq_t transmitted = call->tx_transmitted, seq;
109
-
ktime_t next_resend = KTIME_MAX, rto = ns_to_ktime(call->peer->rto_us * NSEC_PER_USEC);
110
-
ktime_t resend_at = KTIME_MAX, delay;
111
-
bool unacked = false, did_send = false;
112
-
unsigned int qix;
103
+
struct rxrpc_txqueue *tq = call->tx_queue;
104
+
ktime_t lowest_xmit_ts = KTIME_MAX, rto = ns_to_ktime(call->peer->rto_us * NSEC_PER_USEC);
105
+
bool unacked = false;
113
106
114
107
_enter("{%d,%d}", call->tx_bottom, call->tx_top);
115
108
116
-
if (call->tx_bottom == call->tx_top)
117
-
goto no_resend;
109
+
if (call->tx_bottom == call->tx_top) {
110
+
call->resend_at = KTIME_MAX;
111
+
trace_rxrpc_timer_can(call, rxrpc_timer_trace_resend);
112
+
return;
113
+
}
118
114
119
-
trace_rxrpc_resend(call, ack_skb);
120
-
tq = call->tx_queue;
121
-
seq = call->tx_bottom;
115
+
trace_rxrpc_resend(call, ack_serial);
122
116
123
-
/* Scan the soft ACK table and resend any explicitly NAK'd packets. */
124
-
if (ack_skb) {
125
-
sp = rxrpc_skb(ack_skb);
126
-
ack = (void *)ack_skb->data + sizeof(struct rxrpc_wire_header);
117
+
/* Scan the transmission queue, looking for explicitly NAK'd packets. */
118
+
do {
119
+
unsigned long naks = ~tq->segment_acked;
120
+
rxrpc_seq_t tq_top = tq->qbase + RXRPC_NR_TXQUEUE - 1;
127
121
128
-
for (int i = 0; i < sp->ack.nr_acks; i++) {
129
-
rxrpc_seq_t aseq;
122
+
if (after(tq->qbase, call->tx_transmitted))
123
+
break;
130
124
131
-
if (ack->acks[i] & 1)
132
-
continue;
133
-
aseq = sp->ack.first_ack + i;
134
-
while (after_eq(aseq, tq->qbase + RXRPC_NR_TXQUEUE))
135
-
tq = tq->next;
136
-
seq = aseq;
137
-
qix = seq - tq->qbase;
138
-
txb = tq->bufs[qix];
139
-
if (after(seq, transmitted))
140
-
goto no_further_resend;
125
+
if (tq->nr_reported_acks < RXRPC_NR_TXQUEUE)
126
+
naks &= (1UL << tq->nr_reported_acks) - 1;
141
127
142
-
resend_at = ktime_add_us(tq->xmit_ts_base, tq->segment_xmit_ts[qix]);
143
-
resend_at = ktime_add(resend_at, rto);
144
-
if (after(txb->serial, call->acks_highest_serial)) {
145
-
if (ktime_after(resend_at, req.now) &&
146
-
ktime_before(resend_at, next_resend))
147
-
next_resend = resend_at;
128
+
_debug("retr %16lx %u c=%08x [%x]",
129
+
tq->segment_acked, tq->nr_reported_acks, call->debug_id, tq->qbase);
130
+
_debug("nack %16lx", naks);
131
+
132
+
while (naks) {
133
+
unsigned int ix = __ffs(naks);
134
+
struct rxrpc_txbuf *txb = tq->bufs[ix];
135
+
136
+
__clear_bit(ix, &naks);
137
+
if (after(txb->serial, call->acks_highest_serial))
148
138
continue; /* Ack point not yet reached */
149
-
}
150
139
151
140
rxrpc_see_txbuf(txb, rxrpc_txbuf_see_unacked);
152
141
153
142
req.tq = tq;
154
-
req.seq = seq;
143
+
req.seq = tq->qbase + ix;
155
144
req.n = 1;
156
-
rxrpc_retransmit_data(call, &req, rto);
157
-
158
-
if (after_eq(seq, call->tx_top))
159
-
goto no_further_resend;
160
-
}
161
-
}
162
-
163
-
/* Fast-forward through the Tx queue to the point the peer says it has
164
-
* seen. Anything between the soft-ACK table and that point will get
165
-
* ACK'd or NACK'd in due course, so don't worry about it here; here we
166
-
* need to consider retransmitting anything beyond that point.
167
-
*/
168
-
seq = call->acks_prev_seq;
169
-
if (after_eq(seq, call->tx_transmitted))
170
-
goto no_further_resend;
171
-
seq++;
172
-
173
-
while (after_eq(seq, tq->qbase + RXRPC_NR_TXQUEUE))
174
-
tq = tq->next;
175
-
176
-
while (before_eq(seq, call->tx_transmitted)) {
177
-
qix = seq - tq->qbase;
178
-
if (qix >= RXRPC_NR_TXQUEUE) {
179
-
tq = tq->next;
180
-
continue;
181
-
}
182
-
txb = tq->bufs[qix];
183
-
resend_at = ktime_add_us(tq->xmit_ts_base, tq->segment_xmit_ts[qix]);
184
-
resend_at = ktime_add(resend_at, rto);
185
-
186
-
if (ack && ack->reason == RXRPC_ACK_PING_RESPONSE &&
187
-
before(txb->serial, ntohl(ack->serial)))
188
-
goto do_resend; /* Wasn't accounted for by a more recent ping. */
189
-
190
-
if (ktime_after(resend_at, req.now)) {
191
-
if (ktime_before(resend_at, next_resend))
192
-
next_resend = resend_at;
193
-
seq++;
194
-
continue;
145
+
rxrpc_retransmit_data(call, &req, rto, false);
195
146
}
196
147
197
-
do_resend:
198
-
unacked = true;
148
+
/* Anything after the soft-ACK table up to and including
149
+
* ack.previousPacket will get ACK'd or NACK'd in due course,
150
+
* so don't worry about those here. We do, however, need to
151
+
* consider retransmitting anything beyond that point.
152
+
*/
153
+
if (tq->nr_reported_acks < RXRPC_NR_TXQUEUE &&
154
+
after(tq_top, call->acks_prev_seq)) {
155
+
rxrpc_seq_t start = latest(call->acks_prev_seq,
156
+
tq->qbase + tq->nr_reported_acks);
157
+
rxrpc_seq_t stop = earliest(tq_top, call->tx_transmitted);
199
158
200
-
req.tq = tq;
201
-
req.seq = seq;
202
-
req.n = 1;
203
-
rxrpc_retransmit_data(call, &req, rto);
204
-
seq++;
205
-
}
159
+
_debug("unrep %x-%x", start, stop);
160
+
for (rxrpc_seq_t seq = start; before(seq, stop); seq++) {
161
+
struct rxrpc_txbuf *txb = tq->bufs[seq & RXRPC_TXQ_MASK];
206
162
207
-
no_further_resend:
208
-
no_resend:
209
-
if (resend_at < KTIME_MAX) {
210
-
delay = rxrpc_get_rto_backoff(call->peer, did_send);
211
-
resend_at = ktime_add(resend_at, delay);
163
+
if (ping_response &&
164
+
before(txb->serial, call->acks_highest_serial))
165
+
break; /* Wasn't accounted for by a more recent ping. */
166
+
req.tq = tq;
167
+
req.seq = seq;
168
+
req.n = 1;
169
+
if (rxrpc_retransmit_data(call, &req, rto, true))
170
+
unacked = true;
171
+
}
172
+
}
173
+
174
+
/* Work out the next retransmission timeout. */
175
+
if (ktime_before(tq->xmit_ts_base, lowest_xmit_ts)) {
176
+
unsigned int lowest_us = UINT_MAX;
177
+
178
+
for (int i = 0; i < RXRPC_NR_TXQUEUE; i++)
179
+
if (!test_bit(i, &tq->segment_acked) &&
180
+
tq->segment_xmit_ts[i] < lowest_us)
181
+
lowest_us = tq->segment_xmit_ts[i];
182
+
_debug("lowest[%x] %llx %u", tq->qbase, tq->xmit_ts_base, lowest_us);
183
+
184
+
if (lowest_us != UINT_MAX) {
185
+
ktime_t lowest_ns = ktime_add_us(tq->xmit_ts_base, lowest_us);
186
+
187
+
if (ktime_before(lowest_ns, lowest_xmit_ts))
188
+
lowest_xmit_ts = lowest_ns;
189
+
}
190
+
}
191
+
} while ((tq = tq->next));
192
+
193
+
if (lowest_xmit_ts < KTIME_MAX) {
194
+
ktime_t delay = rxrpc_get_rto_backoff(call->peer, req.did_send);
195
+
ktime_t resend_at = ktime_add(lowest_xmit_ts, delay);
196
+
197
+
_debug("delay %llu %lld", delay, ktime_sub(resend_at, req.now));
198
+
call->resend_at = resend_at;
212
199
trace_rxrpc_timer_set(call, resend_at - req.now,
213
200
rxrpc_timer_trace_resend_reset);
201
+
} else {
202
+
call->resend_at = KTIME_MAX;
203
+
trace_rxrpc_timer_can(call, rxrpc_timer_trace_resend);
214
204
}
215
-
call->resend_at = resend_at;
216
205
217
206
if (unacked)
218
207
rxrpc_congestion_timeout(call);
···
487
494
if (resend &&
488
495
__rxrpc_call_state(call) != RXRPC_CALL_CLIENT_RECV_REPLY &&
489
496
!test_bit(RXRPC_CALL_TX_ALL_ACKED, &call->flags))
490
-
rxrpc_resend(call, NULL);
497
+
rxrpc_resend(call, 0, false);
491
498
492
499
if (test_and_clear_bit(RXRPC_CALL_RX_IS_IDLE, &call->flags))
493
500
rxrpc_send_ACK(call, RXRPC_ACK_IDLE, 0,
-1
net/rxrpc/call_object.c
-1
net/rxrpc/call_object.c
+161
-91
net/rxrpc/input.c
+161
-91
net/rxrpc/input.c
···
34
34
struct rxrpc_ack_summary *summary,
35
35
rxrpc_serial_t acked_serial)
36
36
{
37
-
bool resend = false;
38
-
39
37
summary->change = rxrpc_cong_no_change;
40
38
summary->in_flight = (call->tx_top - call->tx_bottom) - call->acks_nr_sacks;
41
39
···
50
52
}
51
53
52
54
call->cong_cumul_acks += summary->nr_new_sacks;
55
+
call->cong_cumul_acks += summary->nr_new_hacks;
53
56
if (call->cong_cumul_acks > 255)
54
57
call->cong_cumul_acks = 255;
55
58
56
59
switch (call->cong_ca_state) {
57
60
case RXRPC_CA_SLOW_START:
58
-
if (summary->saw_snacks)
61
+
if (call->acks_nr_snacks > 0)
59
62
goto packet_loss_detected;
60
63
if (call->cong_cumul_acks > 0)
61
64
call->cong_cwnd += 1;
···
67
68
goto out;
68
69
69
70
case RXRPC_CA_CONGEST_AVOIDANCE:
70
-
if (summary->saw_snacks)
71
+
if (call->acks_nr_snacks > 0)
71
72
goto packet_loss_detected;
72
73
73
74
/* We analyse the number of packets that get ACK'd per RTT
···
86
87
goto out;
87
88
88
89
case RXRPC_CA_PACKET_LOSS:
89
-
if (!summary->saw_snacks)
90
+
if (call->acks_nr_snacks == 0)
90
91
goto resume_normality;
91
92
92
93
if (summary->new_low_snack) {
···
107
108
call->cong_cwnd = call->cong_ssthresh + 3;
108
109
call->cong_extra = 0;
109
110
call->cong_dup_acks = 0;
110
-
resend = true;
111
+
summary->need_retransmit = true;
111
112
goto out;
112
113
113
114
case RXRPC_CA_FAST_RETRANSMIT:
···
118
119
if (call->cong_dup_acks == 2) {
119
120
summary->change = rxrpc_cong_retransmit_again;
120
121
call->cong_dup_acks = 0;
121
-
resend = true;
122
+
summary->need_retransmit = true;
122
123
}
123
124
} else {
124
125
summary->change = rxrpc_cong_progress;
125
126
call->cong_cwnd = call->cong_ssthresh;
126
-
if (!summary->saw_snacks)
127
+
if (call->acks_nr_snacks == 0)
127
128
goto resume_normality;
128
129
}
129
130
goto out;
···
148
149
if (call->cong_cwnd >= RXRPC_TX_MAX_WINDOW)
149
150
call->cong_cwnd = RXRPC_TX_MAX_WINDOW;
150
151
trace_rxrpc_congest(call, summary, acked_serial);
151
-
if (resend)
152
-
rxrpc_resend(call, skb);
153
152
return;
154
153
155
154
packet_loss_detected:
···
209
212
trace_rxrpc_tx_rotate(call, seq, to);
210
213
trace_rxrpc_tq(call, tq, seq, rxrpc_tq_rotate);
211
214
215
+
if (call->acks_lowest_nak == call->tx_bottom) {
216
+
call->acks_lowest_nak = to;
217
+
} else if (after(to, call->acks_lowest_nak)) {
218
+
summary->new_low_snack = true;
219
+
call->acks_lowest_nak = to;
220
+
}
221
+
212
222
/* We may have a left over fully-consumed buffer at the front that we
213
223
* couldn't drop before (rotate_and_keep below).
214
224
*/
···
235
231
set_bit(RXRPC_CALL_TX_LAST, &call->flags);
236
232
rot_last = true;
237
233
}
234
+
235
+
if (ix == tq->nr_reported_acks) {
236
+
/* Packet directly hard ACK'd. */
237
+
tq->nr_reported_acks++;
238
+
summary->nr_new_hacks++;
239
+
__set_bit(ix, &tq->segment_acked);
240
+
trace_rxrpc_rotate(call, tq, summary, seq, rxrpc_rotate_trace_hack);
241
+
} else if (test_bit(ix, &tq->segment_acked)) {
242
+
/* Soft ACK -> hard ACK. */
243
+
call->acks_nr_sacks--;
244
+
trace_rxrpc_rotate(call, tq, summary, seq, rxrpc_rotate_trace_sack);
245
+
} else {
246
+
/* Soft NAK -> hard ACK. */
247
+
call->acks_nr_snacks--;
248
+
summary->nr_new_hacks++;
249
+
__set_bit(ix, &tq->segment_acked);
250
+
trace_rxrpc_rotate(call, tq, summary, seq, rxrpc_rotate_trace_snak);
251
+
}
252
+
238
253
rxrpc_put_txbuf(tq->bufs[ix], rxrpc_txbuf_put_rotated);
239
254
tq->bufs[ix] = NULL;
240
255
···
291
268
292
269
_debug("%x,%x,%x,%d", to, call->tx_bottom, call->tx_top, rot_last);
293
270
294
-
if (call->acks_lowest_nak == call->tx_bottom) {
295
-
call->acks_lowest_nak = to;
296
-
} else if (after(to, call->acks_lowest_nak)) {
297
-
summary->new_low_snack = true;
298
-
call->acks_lowest_nak = to;
299
-
}
300
-
301
271
wake_up(&call->waitq);
302
272
return rot_last;
303
273
}
···
308
292
309
293
call->resend_at = KTIME_MAX;
310
294
trace_rxrpc_timer_can(call, rxrpc_timer_trace_resend);
311
-
312
-
if (unlikely(call->cong_last_nack)) {
313
-
rxrpc_free_skb(call->cong_last_nack, rxrpc_skb_put_last_nack);
314
-
call->cong_last_nack = NULL;
315
-
}
316
295
317
296
switch (__rxrpc_call_state(call)) {
318
297
case RXRPC_CALL_CLIENT_SEND_REQUEST:
···
781
770
wake_up(&call->waitq);
782
771
}
783
772
773
+
#if defined(CONFIG_X86) && __GNUC__ && !defined(__clang__)
774
+
/* Clang doesn't support the %z constraint modifier */
775
+
#define shiftr_adv_rotr(shift_from, rotate_into) ({ \
776
+
asm(" shr%z1 %1\n" \
777
+
" inc %0\n" \
778
+
" rcr%z2 %2\n" \
779
+
: "+d"(shift_from), "+m"(*(shift_from)), "+rm"(rotate_into) \
780
+
); \
781
+
})
782
+
#else
783
+
#define shiftr_adv_rotr(shift_from, rotate_into) ({ \
784
+
typeof(rotate_into) __bit0 = *(shift_from) & 1; \
785
+
*(shift_from) >>= 1; \
786
+
shift_from++; \
787
+
rotate_into >>= 1; \
788
+
rotate_into |= __bit0 << (sizeof(rotate_into) * 8 - 1); \
789
+
})
790
+
#endif
791
+
784
792
/*
785
-
* Determine how many nacks from the previous ACK have now been satisfied.
793
+
* Process a batch of soft ACKs specific to a transmission queue segment.
786
794
*/
787
-
static rxrpc_seq_t rxrpc_input_check_prev_ack(struct rxrpc_call *call,
788
-
struct rxrpc_ack_summary *summary,
789
-
rxrpc_seq_t hard_ack)
795
+
static void rxrpc_input_soft_ack_tq(struct rxrpc_call *call,
796
+
struct rxrpc_ack_summary *summary,
797
+
struct rxrpc_txqueue *tq,
798
+
unsigned long extracted_acks,
799
+
int nr_reported,
800
+
rxrpc_seq_t seq,
801
+
rxrpc_seq_t *lowest_nak)
790
802
{
791
-
struct sk_buff *skb = call->cong_last_nack;
792
-
struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
793
-
unsigned int i, new_acks = 0, retained_nacks = 0;
794
-
rxrpc_seq_t seq = hard_ack + 1, old_seq = sp->ack.first_ack;
795
-
u8 *acks = skb->data + sizeof(struct rxrpc_wire_header) + sizeof(struct rxrpc_ackpacket);
803
+
unsigned long old_reported, flipped, new_acks, a_to_n, n_to_a;
804
+
int new, a, n;
796
805
797
-
if (after_eq(seq, old_seq + sp->ack.nr_acks)) {
798
-
summary->nr_new_sacks += sp->ack.nr_nacks;
799
-
summary->nr_new_sacks += seq - (old_seq + sp->ack.nr_acks);
800
-
summary->nr_retained_snacks = 0;
801
-
} else if (seq == old_seq) {
802
-
summary->nr_retained_snacks = sp->ack.nr_nacks;
803
-
} else {
804
-
for (i = 0; i < sp->ack.nr_acks; i++) {
805
-
if (acks[i] == RXRPC_ACK_TYPE_NACK) {
806
-
if (before(old_seq + i, seq))
807
-
new_acks++;
808
-
else
809
-
retained_nacks++;
810
-
}
811
-
}
806
+
old_reported = ~0UL >> (RXRPC_NR_TXQUEUE - tq->nr_reported_acks);
807
+
_enter("{%x,%lx,%d},%lx,%d,%x",
808
+
tq->qbase, tq->segment_acked, tq->nr_reported_acks,
809
+
extracted_acks, nr_reported, seq);
812
810
813
-
summary->nr_new_sacks += new_acks;
814
-
summary->nr_retained_snacks = retained_nacks;
811
+
_debug("[%x]", tq->qbase);
812
+
_debug("tq %16lx %u", tq->segment_acked, tq->nr_reported_acks);
813
+
_debug("sack %16lx %u", extracted_acks, nr_reported);
814
+
815
+
/* See how many previously logged ACKs/NAKs have flipped. */
816
+
flipped = (tq->segment_acked ^ extracted_acks) & old_reported;
817
+
if (flipped) {
818
+
n_to_a = ~tq->segment_acked & flipped; /* Old NAK -> ACK */
819
+
a_to_n = tq->segment_acked & flipped; /* Old ACK -> NAK */
820
+
a = hweight_long(n_to_a);
821
+
n = hweight_long(a_to_n);
822
+
_debug("flip %16lx", flipped);
823
+
_debug("ntoa %16lx %d", n_to_a, a);
824
+
_debug("aton %16lx %d", a_to_n, n);
825
+
call->acks_nr_sacks += a - n;
826
+
call->acks_nr_snacks += n - a;
827
+
summary->nr_new_sacks += a;
828
+
summary->nr_new_snacks += n;
815
829
}
816
830
817
-
return old_seq + sp->ack.nr_acks - 1;
831
+
/* See how many new ACKs/NAKs have been acquired. */
832
+
new = nr_reported - tq->nr_reported_acks;
833
+
if (new > 0) {
834
+
new_acks = extracted_acks & ~old_reported;
835
+
if (new_acks) {
836
+
a = hweight_long(new_acks);
837
+
n = new - a;
838
+
_debug("new_a %16lx new=%d a=%d n=%d", new_acks, new, a, n);
839
+
call->acks_nr_sacks += a;
840
+
call->acks_nr_snacks += n;
841
+
summary->nr_new_sacks += a;
842
+
summary->nr_new_snacks += n;
843
+
} else {
844
+
call->acks_nr_snacks += new;
845
+
summary->nr_new_snacks += new;
846
+
}
847
+
}
848
+
849
+
tq->nr_reported_acks = nr_reported;
850
+
tq->segment_acked = extracted_acks;
851
+
trace_rxrpc_apply_acks(call, tq);
852
+
853
+
if (extracted_acks != ~0UL) {
854
+
rxrpc_seq_t lowest = seq + ffz(extracted_acks);
855
+
856
+
if (before(lowest, *lowest_nak))
857
+
*lowest_nak = lowest;
858
+
}
818
859
}
819
860
820
861
/*
···
880
817
*/
881
818
static void rxrpc_input_soft_acks(struct rxrpc_call *call,
882
819
struct rxrpc_ack_summary *summary,
883
-
struct sk_buff *skb,
884
-
rxrpc_seq_t since)
820
+
struct sk_buff *skb)
885
821
{
886
822
struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
887
-
unsigned int i, old_nacks = 0;
888
-
rxrpc_seq_t lowest_nak = call->acks_hard_ack + sp->ack.nr_acks + 1;
889
-
rxrpc_seq_t seq = call->acks_hard_ack;
823
+
struct rxrpc_txqueue *tq = call->tx_queue;
824
+
unsigned long extracted = ~0UL;
825
+
unsigned int nr = 0;
826
+
rxrpc_seq_t seq = call->acks_hard_ack + 1;
827
+
rxrpc_seq_t lowest_nak = seq + sp->ack.nr_acks;
890
828
u8 *acks = skb->data + sizeof(struct rxrpc_wire_header) + sizeof(struct rxrpc_ackpacket);
891
829
892
-
for (i = 0; i < sp->ack.nr_acks; i++) {
893
-
seq++;
894
-
if (acks[i] == RXRPC_ACK_TYPE_ACK) {
895
-
call->acks_nr_sacks++;
896
-
if (after(seq, since))
897
-
summary->nr_new_sacks++;
898
-
} else {
899
-
summary->saw_snacks = true;
900
-
if (before_eq(seq, since)) {
901
-
/* Overlap with previous ACK */
902
-
old_nacks++;
903
-
} else {
904
-
summary->nr_new_snacks++;
905
-
sp->ack.nr_nacks++;
906
-
}
830
+
_enter("%x,%x,%u", tq->qbase, seq, sp->ack.nr_acks);
907
831
908
-
if (before(seq, lowest_nak))
909
-
lowest_nak = seq;
832
+
while (after(seq, tq->qbase + RXRPC_NR_TXQUEUE - 1))
833
+
tq = tq->next;
834
+
835
+
for (unsigned int i = 0; i < sp->ack.nr_acks; i++) {
836
+
/* Decant ACKs until we hit a txqueue boundary. */
837
+
shiftr_adv_rotr(acks, extracted);
838
+
if (i == 256) {
839
+
acks -= i;
840
+
i = 0;
910
841
}
842
+
seq++;
843
+
nr++;
844
+
if ((seq & RXRPC_TXQ_MASK) != 0)
845
+
continue;
846
+
847
+
_debug("bound %16lx %u", extracted, nr);
848
+
849
+
rxrpc_input_soft_ack_tq(call, summary, tq, extracted, RXRPC_NR_TXQUEUE,
850
+
seq - RXRPC_NR_TXQUEUE, &lowest_nak);
851
+
extracted = ~0UL;
852
+
nr = 0;
853
+
tq = tq->next;
854
+
prefetch(tq);
911
855
}
912
856
913
-
if (lowest_nak != call->acks_lowest_nak) {
914
-
call->acks_lowest_nak = lowest_nak;
915
-
summary->new_low_snack = true;
857
+
if (nr) {
858
+
unsigned int nr_reported = seq & RXRPC_TXQ_MASK;
859
+
860
+
extracted >>= RXRPC_NR_TXQUEUE - nr_reported;
861
+
_debug("tail %16lx %u", extracted, nr_reported);
862
+
rxrpc_input_soft_ack_tq(call, summary, tq, extracted, nr_reported,
863
+
seq & ~RXRPC_TXQ_MASK, &lowest_nak);
916
864
}
917
865
918
866
/* We *can* have more nacks than we did - the peer is permitted to drop
···
931
857
* possible for the nack distribution to change whilst the number of
932
858
* nacks stays the same or goes down.
933
859
*/
934
-
if (old_nacks < summary->nr_retained_snacks)
935
-
summary->nr_new_sacks += summary->nr_retained_snacks - old_nacks;
936
-
summary->nr_retained_snacks = old_nacks;
860
+
if (lowest_nak != call->acks_lowest_nak) {
861
+
call->acks_lowest_nak = lowest_nak;
862
+
summary->new_low_snack = true;
863
+
}
864
+
865
+
_debug("summary A=%d+%d N=%d+%d",
866
+
call->acks_nr_sacks, summary->nr_new_sacks,
867
+
call->acks_nr_snacks, summary->nr_new_snacks);
937
868
}
938
869
939
870
/*
···
981
902
struct rxrpc_acktrailer trailer;
982
903
struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
983
904
rxrpc_serial_t ack_serial, acked_serial;
984
-
rxrpc_seq_t first_soft_ack, hard_ack, prev_pkt, since;
905
+
rxrpc_seq_t first_soft_ack, hard_ack, prev_pkt;
985
906
int nr_acks, offset, ioffset;
986
907
987
908
_enter("");
···
999
920
1000
921
trace_rxrpc_rx_ack(call, sp);
1001
922
rxrpc_inc_stat(call->rxnet, stat_rx_acks[summary.ack_reason]);
923
+
prefetch(call->tx_queue);
1002
924
1003
925
if (acked_serial != 0) {
1004
926
switch (summary.ack_reason) {
···
1060
980
if (nr_acks > 0)
1061
981
skb_condense(skb);
1062
982
1063
-
if (call->cong_last_nack) {
1064
-
since = rxrpc_input_check_prev_ack(call, &summary, hard_ack);
1065
-
rxrpc_free_skb(call->cong_last_nack, rxrpc_skb_put_last_nack);
1066
-
call->cong_last_nack = NULL;
1067
-
} else {
1068
-
summary.nr_new_sacks = hard_ack - call->acks_hard_ack;
1069
-
call->acks_lowest_nak = hard_ack + nr_acks;
1070
-
since = hard_ack;
1071
-
}
1072
-
1073
983
call->acks_latest_ts = skb->tstamp;
1074
984
call->acks_hard_ack = hard_ack;
1075
985
call->acks_prev_seq = prev_pkt;
···
1107
1037
if (nr_acks > 0) {
1108
1038
if (offset > (int)skb->len - nr_acks)
1109
1039
return rxrpc_proto_abort(call, 0, rxrpc_eproto_ackr_short_sack);
1110
-
rxrpc_input_soft_acks(call, &summary, skb, since);
1111
-
rxrpc_get_skb(skb, rxrpc_skb_get_last_nack);
1112
-
call->cong_last_nack = skb;
1040
+
rxrpc_input_soft_acks(call, &summary, skb);
1113
1041
}
1114
1042
1115
1043
if (test_bit(RXRPC_CALL_TX_LAST, &call->flags) &&
···
1117
1049
rxrpc_propose_ack_ping_for_lost_reply);
1118
1050
1119
1051
rxrpc_congestion_management(call, skb, &summary, acked_serial);
1052
+
if (summary.need_retransmit)
1053
+
rxrpc_resend(call, ack_serial, summary.ack_reason == RXRPC_ACK_PING_RESPONSE);
1120
1054
1121
1055
send_response:
1122
1056
if (summary.ack_reason == RXRPC_ACK_PING)
+9
-1
net/rxrpc/output.c
+9
-1
net/rxrpc/output.c
···
461
461
len += sizeof(*jumbo);
462
462
}
463
463
464
-
trace_rxrpc_tx_data(call, txb->seq, txb->serial, flags, false);
464
+
trace_rxrpc_tx_data(call, txb->seq, txb->serial, txb->flags | flags, false);
465
465
kv->iov_len = len;
466
466
return len;
467
467
}
···
522
522
}
523
523
524
524
/* Set timeouts */
525
+
if (call->peer->rtt_count > 1) {
526
+
ktime_t delay = rxrpc_get_rto_backoff(call->peer, false);
527
+
528
+
call->ack_lost_at = ktime_add(req->now, delay);
529
+
trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_lost_ack);
530
+
}
531
+
525
532
if (!test_and_set_bit(RXRPC_CALL_BEGAN_RX_TIMER, &call->flags)) {
526
533
ktime_t delay = ms_to_ktime(READ_ONCE(call->next_rx_timo));
527
534
···
603
596
ret = 0;
604
597
trace_rxrpc_tx_data(call, txb->seq, txb->serial,
605
598
txb->flags, true);
599
+
conn->peer->last_tx_at = ktime_get_seconds();
606
600
goto done;
607
601
}
608
602
}