Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
bpf: net_sched: Add a qdisc watchdog timer
Add a watchdog timer to bpf qdisc. The watchdog can be used to schedule
the execution of qdisc through kfunc, bpf_qdisc_schedule(). It can be
useful for building traffic shaping scheduling algorithm, where the time
the next packet will be dequeued is known.
The implementation relies on struct_ops gen_prologue/epilogue to patch bpf
programs provided by users. Operator specific prologue/epilogue kfuncs
are introduced instead of watchdog kfuncs so that it is easier to extend
prologue/epilogue in the future (writing C vs BPF bytecode).
Signed-off-by: Amery Hung <amery.hung@bytedance.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Acked-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://patch.msgid.link/20250409214606.2000194-5-ameryhung@gmail.com
authored by
Amery Hung
and committed by
Martin KaFai Lau
7a2dafda
870c2858
+105
-1
+105
-1
net/sched/bpf_qdisc.c
+105
-1
net/sched/bpf_qdisc.c
···
13
13
14
14
static struct bpf_struct_ops bpf_Qdisc_ops;
15
15
16
+
struct bpf_sched_data {
17
+
struct qdisc_watchdog watchdog;
18
+
};
19
+
16
20
struct bpf_sk_buff_ptr {
17
21
struct sk_buff *skb;
18
22
};
···
130
126
return 0;
131
127
}
132
128
129
+
BTF_ID_LIST(bpf_qdisc_init_prologue_ids)
130
+
BTF_ID(func, bpf_qdisc_init_prologue)
131
+
132
+
static int bpf_qdisc_gen_prologue(struct bpf_insn *insn_buf, bool direct_write,
133
+
const struct bpf_prog *prog)
134
+
{
135
+
struct bpf_insn *insn = insn_buf;
136
+
137
+
if (prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, init))
138
+
return 0;
139
+
140
+
/* r6 = r1; // r6 will be "u64 *ctx". r1 is "u64 *ctx".
141
+
* r1 = r1[0]; // r1 will be "struct Qdisc *sch"
142
+
* r0 = bpf_qdisc_init_prologue(r1);
143
+
* r1 = r6; // r1 will be "u64 *ctx".
144
+
*/
145
+
*insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
146
+
*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
147
+
*insn++ = BPF_CALL_KFUNC(0, bpf_qdisc_init_prologue_ids[0]);
148
+
*insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
149
+
*insn++ = prog->insnsi[0];
150
+
151
+
return insn - insn_buf;
152
+
}
153
+
154
+
BTF_ID_LIST(bpf_qdisc_reset_destroy_epilogue_ids)
155
+
BTF_ID(func, bpf_qdisc_reset_destroy_epilogue)
156
+
157
+
static int bpf_qdisc_gen_epilogue(struct bpf_insn *insn_buf, const struct bpf_prog *prog,
158
+
s16 ctx_stack_off)
159
+
{
160
+
struct bpf_insn *insn = insn_buf;
161
+
162
+
if (prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, reset) &&
163
+
prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, destroy))
164
+
return 0;
165
+
166
+
/* r1 = stack[ctx_stack_off]; // r1 will be "u64 *ctx"
167
+
* r1 = r1[0]; // r1 will be "struct Qdisc *sch"
168
+
* r0 = bpf_qdisc_reset_destroy_epilogue(r1);
169
+
* BPF_EXIT;
170
+
*/
171
+
*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_FP, ctx_stack_off);
172
+
*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
173
+
*insn++ = BPF_CALL_KFUNC(0, bpf_qdisc_reset_destroy_epilogue_ids[0]);
174
+
*insn++ = BPF_EXIT_INSN();
175
+
176
+
return insn - insn_buf;
177
+
}
178
+
133
179
__bpf_kfunc_start_defs();
134
180
135
181
/* bpf_skb_get_hash - Get the flow hash of an skb.
···
208
154
__qdisc_drop(skb, (struct sk_buff **)to_free_list);
209
155
}
210
156
157
+
/* bpf_qdisc_watchdog_schedule - Schedule a qdisc to a later time using a timer.
158
+
* @sch: The qdisc to be scheduled.
159
+
* @expire: The expiry time of the timer.
160
+
* @delta_ns: The slack range of the timer.
161
+
*/
162
+
__bpf_kfunc void bpf_qdisc_watchdog_schedule(struct Qdisc *sch, u64 expire, u64 delta_ns)
163
+
{
164
+
struct bpf_sched_data *q = qdisc_priv(sch);
165
+
166
+
qdisc_watchdog_schedule_range_ns(&q->watchdog, expire, delta_ns);
167
+
}
168
+
169
+
/* bpf_qdisc_init_prologue - Hidden kfunc called in prologue of .init. */
170
+
__bpf_kfunc void bpf_qdisc_init_prologue(struct Qdisc *sch)
171
+
{
172
+
struct bpf_sched_data *q = qdisc_priv(sch);
173
+
174
+
qdisc_watchdog_init(&q->watchdog, sch);
175
+
}
176
+
177
+
/* bpf_qdisc_reset_destroy_epilogue - Hidden kfunc called in epilogue of .reset
178
+
* and .destroy
179
+
*/
180
+
__bpf_kfunc void bpf_qdisc_reset_destroy_epilogue(struct Qdisc *sch)
181
+
{
182
+
struct bpf_sched_data *q = qdisc_priv(sch);
183
+
184
+
qdisc_watchdog_cancel(&q->watchdog);
185
+
}
186
+
211
187
__bpf_kfunc_end_defs();
212
188
213
189
BTF_KFUNCS_START(qdisc_kfunc_ids)
···
245
161
BTF_ID_FLAGS(func, bpf_kfree_skb, KF_RELEASE)
246
162
BTF_ID_FLAGS(func, bpf_qdisc_skb_drop, KF_RELEASE)
247
163
BTF_ID_FLAGS(func, bpf_dynptr_from_skb, KF_TRUSTED_ARGS)
164
+
BTF_ID_FLAGS(func, bpf_qdisc_watchdog_schedule, KF_TRUSTED_ARGS)
165
+
BTF_ID_FLAGS(func, bpf_qdisc_init_prologue, KF_TRUSTED_ARGS)
166
+
BTF_ID_FLAGS(func, bpf_qdisc_reset_destroy_epilogue, KF_TRUSTED_ARGS)
248
167
BTF_KFUNCS_END(qdisc_kfunc_ids)
249
168
250
169
BTF_SET_START(qdisc_common_kfunc_set)
···
258
171
259
172
BTF_SET_START(qdisc_enqueue_kfunc_set)
260
173
BTF_ID(func, bpf_qdisc_skb_drop)
174
+
BTF_ID(func, bpf_qdisc_watchdog_schedule)
261
175
BTF_SET_END(qdisc_enqueue_kfunc_set)
176
+
177
+
BTF_SET_START(qdisc_dequeue_kfunc_set)
178
+
BTF_ID(func, bpf_qdisc_watchdog_schedule)
179
+
BTF_SET_END(qdisc_dequeue_kfunc_set)
262
180
263
181
enum qdisc_ops_kf_flags {
264
182
QDISC_OPS_KF_COMMON = 0,
265
183
QDISC_OPS_KF_ENQUEUE = 1 << 0,
184
+
QDISC_OPS_KF_DEQUEUE = 1 << 1,
266
185
};
267
186
268
187
static const u32 qdisc_ops_context_flags[] = {
269
188
[QDISC_OP_IDX(enqueue)] = QDISC_OPS_KF_ENQUEUE,
270
-
[QDISC_OP_IDX(dequeue)] = QDISC_OPS_KF_COMMON,
189
+
[QDISC_OP_IDX(dequeue)] = QDISC_OPS_KF_DEQUEUE,
271
190
[QDISC_OP_IDX(init)] = QDISC_OPS_KF_COMMON,
272
191
[QDISC_OP_IDX(reset)] = QDISC_OPS_KF_COMMON,
273
192
[QDISC_OP_IDX(destroy)] = QDISC_OPS_KF_COMMON,
···
296
203
btf_id_set_contains(&qdisc_enqueue_kfunc_set, kfunc_id))
297
204
return 0;
298
205
206
+
if ((flags & QDISC_OPS_KF_DEQUEUE) &&
207
+
btf_id_set_contains(&qdisc_dequeue_kfunc_set, kfunc_id))
208
+
return 0;
209
+
299
210
if (btf_id_set_contains(&qdisc_common_kfunc_set, kfunc_id))
300
211
return 0;
301
212
···
316
219
.get_func_proto = bpf_base_func_proto,
317
220
.is_valid_access = bpf_qdisc_is_valid_access,
318
221
.btf_struct_access = bpf_qdisc_btf_struct_access,
222
+
.gen_prologue = bpf_qdisc_gen_prologue,
223
+
.gen_epilogue = bpf_qdisc_gen_epilogue,
319
224
};
320
225
321
226
static int bpf_qdisc_init_member(const struct btf_type *t,
···
333
234
334
235
moff = __btf_member_bit_offset(t, member) / 8;
335
236
switch (moff) {
237
+
case offsetof(struct Qdisc_ops, priv_size):
238
+
if (uqdisc_ops->priv_size)
239
+
return -EINVAL;
240
+
qdisc_ops->priv_size = sizeof(struct bpf_sched_data);
241
+
return 1;
336
242
case offsetof(struct Qdisc_ops, peek):
337
243
qdisc_ops->peek = qdisc_peek_dequeued;
338
244
return 0;