Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
selftests/bpf: Sanitize the SEC and inline usages in the bpf-tcp-cc tests
It is needed to remove the BPF_STRUCT_OPS usages from the tcp-cc tests
because it is defined in bpf_tcp_helpers.h which is going to be retired.
While at it, this patch consolidates all tcp-cc struct_ops programs to
use the SEC("struct_ops") + BPF_PROG().
It also removes the unnecessary __always_inline usages from the
tcp-cc tests.
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20240509175026.3423614-5-martin.lau@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
authored by
Martin KaFai Lau
and committed by
Alexei Starovoitov
7d3851a3
cc5b18ce
+77
-75
+15
-13
tools/testing/selftests/bpf/progs/bpf_cc_cubic.c
+15
-13
tools/testing/selftests/bpf/progs/bpf_cc_cubic.c
···
17
17
#include <bpf/bpf_helpers.h>
18
18
#include <bpf/bpf_tracing.h>
19
19
20
-
#define BPF_STRUCT_OPS(name, args...) \
21
-
SEC("struct_ops/"#name) \
22
-
BPF_PROG(name, args)
23
-
24
20
#define USEC_PER_SEC 1000000UL
25
21
#define TCP_PACING_SS_RATIO (200)
26
22
#define TCP_PACING_CA_RATIO (120)
···
110
114
return flag & FLAG_DATA_ACKED;
111
115
}
112
116
113
-
void BPF_STRUCT_OPS(bpf_cubic_init, struct sock *sk)
117
+
SEC("struct_ops")
118
+
void BPF_PROG(bpf_cubic_init, struct sock *sk)
114
119
{
115
120
cubictcp_init(sk);
116
121
}
117
122
118
-
void BPF_STRUCT_OPS(bpf_cubic_cwnd_event, struct sock *sk, enum tcp_ca_event event)
123
+
SEC("struct_ops")
124
+
void BPF_PROG(bpf_cubic_cwnd_event, struct sock *sk, enum tcp_ca_event event)
119
125
{
120
126
cubictcp_cwnd_event(sk, event);
121
127
}
122
128
123
-
void BPF_STRUCT_OPS(bpf_cubic_cong_control, struct sock *sk, __u32 ack, int flag,
124
-
const struct rate_sample *rs)
129
+
SEC("struct_ops")
130
+
void BPF_PROG(bpf_cubic_cong_control, struct sock *sk, __u32 ack, int flag,
131
+
const struct rate_sample *rs)
125
132
{
126
133
struct tcp_sock *tp = tcp_sk(sk);
127
134
···
150
151
tcp_update_pacing_rate(sk);
151
152
}
152
153
153
-
__u32 BPF_STRUCT_OPS(bpf_cubic_recalc_ssthresh, struct sock *sk)
154
+
SEC("struct_ops")
155
+
__u32 BPF_PROG(bpf_cubic_recalc_ssthresh, struct sock *sk)
154
156
{
155
157
return cubictcp_recalc_ssthresh(sk);
156
158
}
157
159
158
-
void BPF_STRUCT_OPS(bpf_cubic_state, struct sock *sk, __u8 new_state)
160
+
SEC("struct_ops")
161
+
void BPF_PROG(bpf_cubic_state, struct sock *sk, __u8 new_state)
159
162
{
160
163
cubictcp_state(sk, new_state);
161
164
}
162
165
163
-
void BPF_STRUCT_OPS(bpf_cubic_acked, struct sock *sk,
164
-
const struct ack_sample *sample)
166
+
SEC("struct_ops")
167
+
void BPF_PROG(bpf_cubic_acked, struct sock *sk, const struct ack_sample *sample)
165
168
{
166
169
cubictcp_acked(sk, sample);
167
170
}
168
171
169
-
__u32 BPF_STRUCT_OPS(bpf_cubic_undo_cwnd, struct sock *sk)
172
+
SEC("struct_ops")
173
+
__u32 BPF_PROG(bpf_cubic_undo_cwnd, struct sock *sk)
170
174
{
171
175
return tcp_reno_undo_cwnd(sk);
172
176
}
+21
-21
tools/testing/selftests/bpf/progs/bpf_cubic.c
+21
-21
tools/testing/selftests/bpf/progs/bpf_cubic.c
···
91
91
__u32 curr_rtt; /* the minimum rtt of current round */
92
92
};
93
93
94
-
static inline void bictcp_reset(struct bictcp *ca)
94
+
static void bictcp_reset(struct bictcp *ca)
95
95
{
96
96
ca->cnt = 0;
97
97
ca->last_max_cwnd = 0;
···
112
112
#define USEC_PER_SEC 1000000UL
113
113
#define USEC_PER_JIFFY (USEC_PER_SEC / HZ)
114
114
115
-
static __always_inline __u64 div64_u64(__u64 dividend, __u64 divisor)
115
+
static __u64 div64_u64(__u64 dividend, __u64 divisor)
116
116
{
117
117
return dividend / divisor;
118
118
}
···
120
120
#define div64_ul div64_u64
121
121
122
122
#define BITS_PER_U64 (sizeof(__u64) * 8)
123
-
static __always_inline int fls64(__u64 x)
123
+
static int fls64(__u64 x)
124
124
{
125
125
int num = BITS_PER_U64 - 1;
126
126
···
153
153
return num + 1;
154
154
}
155
155
156
-
static __always_inline __u32 bictcp_clock_us(const struct sock *sk)
156
+
static __u32 bictcp_clock_us(const struct sock *sk)
157
157
{
158
158
return tcp_sk(sk)->tcp_mstamp;
159
159
}
160
160
161
-
static __always_inline void bictcp_hystart_reset(struct sock *sk)
161
+
static void bictcp_hystart_reset(struct sock *sk)
162
162
{
163
163
struct tcp_sock *tp = tcp_sk(sk);
164
164
struct bictcp *ca = inet_csk_ca(sk);
···
169
169
ca->sample_cnt = 0;
170
170
}
171
171
172
-
/* "struct_ops/" prefix is a requirement */
173
-
SEC("struct_ops/bpf_cubic_init")
172
+
SEC("struct_ops")
174
173
void BPF_PROG(bpf_cubic_init, struct sock *sk)
175
174
{
176
175
struct bictcp *ca = inet_csk_ca(sk);
···
183
184
tcp_sk(sk)->snd_ssthresh = initial_ssthresh;
184
185
}
185
186
186
-
/* "struct_ops" prefix is a requirement */
187
-
SEC("struct_ops/bpf_cubic_cwnd_event")
187
+
SEC("struct_ops")
188
188
void BPF_PROG(bpf_cubic_cwnd_event, struct sock *sk, enum tcp_ca_event event)
189
189
{
190
190
if (event == CA_EVENT_TX_START) {
···
228
230
* Newton-Raphson iteration.
229
231
* Avg err ~= 0.195%
230
232
*/
231
-
static __always_inline __u32 cubic_root(__u64 a)
233
+
static __u32 cubic_root(__u64 a)
232
234
{
233
235
__u32 x, b, shift;
234
236
···
261
263
/*
262
264
* Compute congestion window to use.
263
265
*/
264
-
static __always_inline void bictcp_update(struct bictcp *ca, __u32 cwnd,
265
-
__u32 acked)
266
+
static void bictcp_update(struct bictcp *ca, __u32 cwnd, __u32 acked)
266
267
{
267
268
__u32 delta, bic_target, max_cnt;
268
269
__u64 offs, t;
···
374
377
ca->cnt = max(ca->cnt, 2U);
375
378
}
376
379
377
-
/* Or simply use the BPF_STRUCT_OPS to avoid the SEC boiler plate. */
378
-
void BPF_STRUCT_OPS(bpf_cubic_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
380
+
SEC("struct_ops")
381
+
void BPF_PROG(bpf_cubic_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
379
382
{
380
383
struct tcp_sock *tp = tcp_sk(sk);
381
384
struct bictcp *ca = inet_csk_ca(sk);
···
394
397
tcp_cong_avoid_ai(tp, ca->cnt, acked);
395
398
}
396
399
397
-
__u32 BPF_STRUCT_OPS(bpf_cubic_recalc_ssthresh, struct sock *sk)
400
+
SEC("struct_ops")
401
+
__u32 BPF_PROG(bpf_cubic_recalc_ssthresh, struct sock *sk)
398
402
{
399
403
const struct tcp_sock *tp = tcp_sk(sk);
400
404
struct bictcp *ca = inet_csk_ca(sk);
···
412
414
return max((tp->snd_cwnd * beta) / BICTCP_BETA_SCALE, 2U);
413
415
}
414
416
415
-
void BPF_STRUCT_OPS(bpf_cubic_state, struct sock *sk, __u8 new_state)
417
+
SEC("struct_ops")
418
+
void BPF_PROG(bpf_cubic_state, struct sock *sk, __u8 new_state)
416
419
{
417
420
if (new_state == TCP_CA_Loss) {
418
421
bictcp_reset(inet_csk_ca(sk));
···
432
433
* We apply another 100% factor because @rate is doubled at this point.
433
434
* We cap the cushion to 1ms.
434
435
*/
435
-
static __always_inline __u32 hystart_ack_delay(struct sock *sk)
436
+
static __u32 hystart_ack_delay(struct sock *sk)
436
437
{
437
438
unsigned long rate;
438
439
···
443
444
div64_ul((__u64)GSO_MAX_SIZE * 4 * USEC_PER_SEC, rate));
444
445
}
445
446
446
-
static __always_inline void hystart_update(struct sock *sk, __u32 delay)
447
+
static void hystart_update(struct sock *sk, __u32 delay)
447
448
{
448
449
struct tcp_sock *tp = tcp_sk(sk);
449
450
struct bictcp *ca = inet_csk_ca(sk);
···
491
492
492
493
int bpf_cubic_acked_called = 0;
493
494
494
-
void BPF_STRUCT_OPS(bpf_cubic_acked, struct sock *sk,
495
-
const struct ack_sample *sample)
495
+
SEC("struct_ops")
496
+
void BPF_PROG(bpf_cubic_acked, struct sock *sk, const struct ack_sample *sample)
496
497
{
497
498
const struct tcp_sock *tp = tcp_sk(sk);
498
499
struct bictcp *ca = inet_csk_ca(sk);
···
523
524
524
525
extern __u32 tcp_reno_undo_cwnd(struct sock *sk) __ksym;
525
526
526
-
__u32 BPF_STRUCT_OPS(bpf_cubic_undo_cwnd, struct sock *sk)
527
+
SEC("struct_ops")
528
+
__u32 BPF_PROG(bpf_cubic_undo_cwnd, struct sock *sk)
527
529
{
528
530
return tcp_reno_undo_cwnd(sk);
529
531
}
+12
-14
tools/testing/selftests/bpf/progs/bpf_dctcp.c
+12
-14
tools/testing/selftests/bpf/progs/bpf_dctcp.c
···
48
48
static unsigned int dctcp_shift_g = 4; /* g = 1/2^4 */
49
49
static unsigned int dctcp_alpha_on_init = DCTCP_MAX_ALPHA;
50
50
51
-
static __always_inline void dctcp_reset(const struct tcp_sock *tp,
52
-
struct dctcp *ca)
51
+
static void dctcp_reset(const struct tcp_sock *tp, struct dctcp *ca)
53
52
{
54
53
ca->next_seq = tp->snd_nxt;
55
54
···
56
57
ca->old_delivered_ce = tp->delivered_ce;
57
58
}
58
59
59
-
SEC("struct_ops/dctcp_init")
60
+
SEC("struct_ops")
60
61
void BPF_PROG(dctcp_init, struct sock *sk)
61
62
{
62
63
const struct tcp_sock *tp = tcp_sk(sk);
···
103
104
dctcp_reset(tp, ca);
104
105
}
105
106
106
-
SEC("struct_ops/dctcp_ssthresh")
107
+
SEC("struct_ops")
107
108
__u32 BPF_PROG(dctcp_ssthresh, struct sock *sk)
108
109
{
109
110
struct dctcp *ca = inet_csk_ca(sk);
···
113
114
return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U);
114
115
}
115
116
116
-
SEC("struct_ops/dctcp_update_alpha")
117
+
SEC("struct_ops")
117
118
void BPF_PROG(dctcp_update_alpha, struct sock *sk, __u32 flags)
118
119
{
119
120
const struct tcp_sock *tp = tcp_sk(sk);
···
143
144
}
144
145
}
145
146
146
-
static __always_inline void dctcp_react_to_loss(struct sock *sk)
147
+
static void dctcp_react_to_loss(struct sock *sk)
147
148
{
148
149
struct dctcp *ca = inet_csk_ca(sk);
149
150
struct tcp_sock *tp = tcp_sk(sk);
···
152
153
tp->snd_ssthresh = max(tp->snd_cwnd >> 1U, 2U);
153
154
}
154
155
155
-
SEC("struct_ops/dctcp_state")
156
+
SEC("struct_ops")
156
157
void BPF_PROG(dctcp_state, struct sock *sk, __u8 new_state)
157
158
{
158
159
if (new_state == TCP_CA_Recovery &&
···
163
164
*/
164
165
}
165
166
166
-
static __always_inline void dctcp_ece_ack_cwr(struct sock *sk, __u32 ce_state)
167
+
static void dctcp_ece_ack_cwr(struct sock *sk, __u32 ce_state)
167
168
{
168
169
struct tcp_sock *tp = tcp_sk(sk);
169
170
···
178
179
* S: 0 <- last pkt was non-CE
179
180
* 1 <- last pkt was CE
180
181
*/
181
-
static __always_inline
182
-
void dctcp_ece_ack_update(struct sock *sk, enum tcp_ca_event evt,
183
-
__u32 *prior_rcv_nxt, __u32 *ce_state)
182
+
static void dctcp_ece_ack_update(struct sock *sk, enum tcp_ca_event evt,
183
+
__u32 *prior_rcv_nxt, __u32 *ce_state)
184
184
{
185
185
__u32 new_ce_state = (evt == CA_EVENT_ECN_IS_CE) ? 1 : 0;
186
186
···
199
201
dctcp_ece_ack_cwr(sk, new_ce_state);
200
202
}
201
203
202
-
SEC("struct_ops/dctcp_cwnd_event")
204
+
SEC("struct_ops")
203
205
void BPF_PROG(dctcp_cwnd_event, struct sock *sk, enum tcp_ca_event ev)
204
206
{
205
207
struct dctcp *ca = inet_csk_ca(sk);
···
218
220
}
219
221
}
220
222
221
-
SEC("struct_ops/dctcp_cwnd_undo")
223
+
SEC("struct_ops")
222
224
__u32 BPF_PROG(dctcp_cwnd_undo, struct sock *sk)
223
225
{
224
226
const struct dctcp *ca = inet_csk_ca(sk);
···
228
230
229
231
extern void tcp_reno_cong_avoid(struct sock *sk, __u32 ack, __u32 acked) __ksym;
230
232
231
-
SEC("struct_ops/dctcp_reno_cong_avoid")
233
+
SEC("struct_ops")
232
234
void BPF_PROG(dctcp_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
233
235
{
234
236
tcp_reno_cong_avoid(sk, ack, acked);
+2
-1
tools/testing/selftests/bpf/progs/bpf_dctcp_release.c
+2
-1
tools/testing/selftests/bpf/progs/bpf_dctcp_release.c
···
13
13
char _license[] SEC("license") = "GPL";
14
14
const char cubic[] = "cubic";
15
15
16
-
void BPF_STRUCT_OPS(dctcp_nouse_release, struct sock *sk)
16
+
SEC("struct_ops")
17
+
void BPF_PROG(dctcp_nouse_release, struct sock *sk)
17
18
{
18
19
bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
19
20
(void *)cubic, sizeof(cubic));
+2
-1
tools/testing/selftests/bpf/progs/bpf_tcp_nogpl.c
+2
-1
tools/testing/selftests/bpf/progs/bpf_tcp_nogpl.c
+2
-2
tools/testing/selftests/bpf/progs/tcp_ca_incompl_cong_ops.c
+2
-2
tools/testing/selftests/bpf/progs/tcp_ca_incompl_cong_ops.c
···
6
6
7
7
char _license[] SEC("license") = "GPL";
8
8
9
-
SEC("struct_ops/incompl_cong_ops_ssthresh")
9
+
SEC("struct_ops")
10
10
__u32 BPF_PROG(incompl_cong_ops_ssthresh, struct sock *sk)
11
11
{
12
12
return tcp_sk(sk)->snd_ssthresh;
13
13
}
14
14
15
-
SEC("struct_ops/incompl_cong_ops_undo_cwnd")
15
+
SEC("struct_ops")
16
16
__u32 BPF_PROG(incompl_cong_ops_undo_cwnd, struct sock *sk)
17
17
{
18
18
return tcp_sk(sk)->snd_cwnd;
+11
-11
tools/testing/selftests/bpf/progs/tcp_ca_kfunc.c
+11
-11
tools/testing/selftests/bpf/progs/tcp_ca_kfunc.c
···
27
27
extern void cubictcp_cwnd_event(struct sock *sk, enum tcp_ca_event event) __ksym;
28
28
extern void cubictcp_acked(struct sock *sk, const struct ack_sample *sample) __ksym;
29
29
30
-
SEC("struct_ops/init")
30
+
SEC("struct_ops")
31
31
void BPF_PROG(init, struct sock *sk)
32
32
{
33
33
bbr_init(sk);
···
35
35
cubictcp_init(sk);
36
36
}
37
37
38
-
SEC("struct_ops/in_ack_event")
38
+
SEC("struct_ops")
39
39
void BPF_PROG(in_ack_event, struct sock *sk, u32 flags)
40
40
{
41
41
dctcp_update_alpha(sk, flags);
42
42
}
43
43
44
-
SEC("struct_ops/cong_control")
44
+
SEC("struct_ops")
45
45
void BPF_PROG(cong_control, struct sock *sk, u32 ack, int flag, const struct rate_sample *rs)
46
46
{
47
47
bbr_main(sk, ack, flag, rs);
48
48
}
49
49
50
-
SEC("struct_ops/cong_avoid")
50
+
SEC("struct_ops")
51
51
void BPF_PROG(cong_avoid, struct sock *sk, u32 ack, u32 acked)
52
52
{
53
53
cubictcp_cong_avoid(sk, ack, acked);
54
54
}
55
55
56
-
SEC("struct_ops/sndbuf_expand")
56
+
SEC("struct_ops")
57
57
u32 BPF_PROG(sndbuf_expand, struct sock *sk)
58
58
{
59
59
return bbr_sndbuf_expand(sk);
60
60
}
61
61
62
-
SEC("struct_ops/undo_cwnd")
62
+
SEC("struct_ops")
63
63
u32 BPF_PROG(undo_cwnd, struct sock *sk)
64
64
{
65
65
bbr_undo_cwnd(sk);
66
66
return dctcp_cwnd_undo(sk);
67
67
}
68
68
69
-
SEC("struct_ops/cwnd_event")
69
+
SEC("struct_ops")
70
70
void BPF_PROG(cwnd_event, struct sock *sk, enum tcp_ca_event event)
71
71
{
72
72
bbr_cwnd_event(sk, event);
···
74
74
cubictcp_cwnd_event(sk, event);
75
75
}
76
76
77
-
SEC("struct_ops/ssthresh")
77
+
SEC("struct_ops")
78
78
u32 BPF_PROG(ssthresh, struct sock *sk)
79
79
{
80
80
bbr_ssthresh(sk);
···
82
82
return cubictcp_recalc_ssthresh(sk);
83
83
}
84
84
85
-
SEC("struct_ops/min_tso_segs")
85
+
SEC("struct_ops")
86
86
u32 BPF_PROG(min_tso_segs, struct sock *sk)
87
87
{
88
88
return bbr_min_tso_segs(sk);
89
89
}
90
90
91
-
SEC("struct_ops/set_state")
91
+
SEC("struct_ops")
92
92
void BPF_PROG(set_state, struct sock *sk, u8 new_state)
93
93
{
94
94
bbr_set_state(sk, new_state);
···
96
96
cubictcp_state(sk, new_state);
97
97
}
98
98
99
-
SEC("struct_ops/pkts_acked")
99
+
SEC("struct_ops")
100
100
void BPF_PROG(pkts_acked, struct sock *sk, const struct ack_sample *sample)
101
101
{
102
102
cubictcp_acked(sk, sample);
+1
-1
tools/testing/selftests/bpf/progs/tcp_ca_unsupp_cong_op.c
+1
-1
tools/testing/selftests/bpf/progs/tcp_ca_unsupp_cong_op.c
+5
-5
tools/testing/selftests/bpf/progs/tcp_ca_update.c
+5
-5
tools/testing/selftests/bpf/progs/tcp_ca_update.c
···
9
9
int ca1_cnt = 0;
10
10
int ca2_cnt = 0;
11
11
12
-
SEC("struct_ops/ca_update_1_init")
12
+
SEC("struct_ops")
13
13
void BPF_PROG(ca_update_1_init, struct sock *sk)
14
14
{
15
15
ca1_cnt++;
16
16
}
17
17
18
-
SEC("struct_ops/ca_update_2_init")
18
+
SEC("struct_ops")
19
19
void BPF_PROG(ca_update_2_init, struct sock *sk)
20
20
{
21
21
ca2_cnt++;
22
22
}
23
23
24
-
SEC("struct_ops/ca_update_cong_control")
24
+
SEC("struct_ops")
25
25
void BPF_PROG(ca_update_cong_control, struct sock *sk,
26
26
const struct rate_sample *rs)
27
27
{
28
28
}
29
29
30
-
SEC("struct_ops/ca_update_ssthresh")
30
+
SEC("struct_ops")
31
31
__u32 BPF_PROG(ca_update_ssthresh, struct sock *sk)
32
32
{
33
33
return tcp_sk(sk)->snd_ssthresh;
34
34
}
35
35
36
-
SEC("struct_ops/ca_update_undo_cwnd")
36
+
SEC("struct_ops")
37
37
__u32 BPF_PROG(ca_update_undo_cwnd, struct sock *sk)
38
38
{
39
39
return tcp_sk(sk)->snd_cwnd;
+6
-6
tools/testing/selftests/bpf/progs/tcp_ca_write_sk_pacing.c
+6
-6
tools/testing/selftests/bpf/progs/tcp_ca_write_sk_pacing.c
···
10
10
11
11
#define min(a, b) ((a) < (b) ? (a) : (b))
12
12
13
-
static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
13
+
static unsigned int tcp_left_out(const struct tcp_sock *tp)
14
14
{
15
15
return tp->sacked_out + tp->lost_out;
16
16
}
17
17
18
-
static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
18
+
static unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
19
19
{
20
20
return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
21
21
}
22
22
23
-
SEC("struct_ops/write_sk_pacing_init")
23
+
SEC("struct_ops")
24
24
void BPF_PROG(write_sk_pacing_init, struct sock *sk)
25
25
{
26
26
#ifdef ENABLE_ATOMICS_TESTS
···
31
31
#endif
32
32
}
33
33
34
-
SEC("struct_ops/write_sk_pacing_cong_control")
34
+
SEC("struct_ops")
35
35
void BPF_PROG(write_sk_pacing_cong_control, struct sock *sk,
36
36
const struct rate_sample *rs)
37
37
{
···
43
43
tp->app_limited = (tp->delivered + tcp_packets_in_flight(tp)) ?: 1;
44
44
}
45
45
46
-
SEC("struct_ops/write_sk_pacing_ssthresh")
46
+
SEC("struct_ops")
47
47
__u32 BPF_PROG(write_sk_pacing_ssthresh, struct sock *sk)
48
48
{
49
49
return tcp_sk(sk)->snd_ssthresh;
50
50
}
51
51
52
-
SEC("struct_ops/write_sk_pacing_undo_cwnd")
52
+
SEC("struct_ops")
53
53
__u32 BPF_PROG(write_sk_pacing_undo_cwnd, struct sock *sk)
54
54
{
55
55
return tcp_sk(sk)->snd_cwnd;