Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-bpf-restrictions.4-20260206' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux
Pull io_uring bpf filters from Jens Axboe:
"This adds support for both cBPF filters for io_uring, as well as task
inherited restrictions and filters.
seccomp and io_uring don't play along nicely, as most of the
interesting data to filter on resides somewhat out-of-band, in the
submission queue ring.
As a result, things like containers and systemd that apply seccomp
filters, can't filter io_uring operations.
That leaves them with just one choice if filtering is critical -
filter the actual io_uring_setup(2) system call to simply disallow
io_uring. That's rather unfortunate, and has limited us because of it.
io_uring already has some filtering support. It requires the ring to
be setup in a disabled state, and then a filter set can be applied.
This filter set is completely bi-modal - an opcode is either enabled
or it's not. Once a filter set is registered, the ring can be enabled.
This is very restrictive, and it's not useful at all to systemd or
containers which really want both broader and more specific control.
This first adds support for cBPF filters for opcodes, which enables
tighter control over what exactly a specific opcode may do. As
examples, specific support is added for IORING_OP_OPENAT/OPENAT2,
allowing filtering on resolve flags. And another example is added for
IORING_OP_SOCKET, allowing filtering on domain/type/protocol. These
are both common use cases. cBPF was chosen rather than eBPF, because
the latter is often restricted in containers as well.
These filters are run post the init phase of the request, which allows
filters to even dip into data that is being passed in struct in user
memory, as the init side of requests make that data stable by bringing
it into the kernel. This allows filtering without needing to copy this
data twice, or have filters etc know about the exact layout of the
user data. The filters get the already copied and sanitized data
passed.
On top of that support is added for per-task filters, meaning that any
ring created with a task that has a per-task filter will get those
filters applied when it's created. These filters are inherited across
fork as well. Once a filter has been registered, any further added
filters may only further restrict what operations are permitted.
Filters cannot change the return value of an operation, they can only
permit or deny it based on the contents"
* tag 'io_uring-bpf-restrictions.4-20260206' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux:
io_uring: allow registration of per-task restrictions
io_uring: add task fork hook
io_uring/bpf_filter: add ref counts to struct io_bpf_filter
io_uring/bpf_filter: cache lookup table in ctx->bpf_filters
io_uring/bpf_filter: allow filtering on contents of struct open_how
io_uring/net: allow filtering on IORING_OP_SOCKET data
io_uring: add support for BPF filtering for opcode restrictions
+789
-10
+13
-1
include/linux/io_uring.h
+13
-1
include/linux/io_uring.h
···
12
12
void io_uring_unreg_ringfd(void);
13
13
const char *io_uring_get_opcode(u8 opcode);
14
14
bool io_is_uring_fops(struct file *file);
15
+
int __io_uring_fork(struct task_struct *tsk);
15
16
16
17
static inline void io_uring_files_cancel(void)
17
18
{
···
26
25
}
27
26
static inline void io_uring_free(struct task_struct *tsk)
28
27
{
29
-
if (tsk->io_uring)
28
+
if (tsk->io_uring || tsk->io_uring_restrict)
30
29
__io_uring_free(tsk);
30
+
}
31
+
static inline int io_uring_fork(struct task_struct *tsk)
32
+
{
33
+
if (tsk->io_uring_restrict)
34
+
return __io_uring_fork(tsk);
35
+
36
+
return 0;
31
37
}
32
38
#else
33
39
static inline void io_uring_task_cancel(void)
···
53
45
static inline bool io_is_uring_fops(struct file *file)
54
46
{
55
47
return false;
48
+
}
49
+
static inline int io_uring_fork(struct task_struct *tsk)
50
+
{
51
+
return 0;
56
52
}
57
53
#endif
58
54
+13
include/linux/io_uring_types.h
+13
include/linux/io_uring_types.h
···
219
219
struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
220
220
};
221
221
222
+
struct io_bpf_filter;
223
+
struct io_bpf_filters {
224
+
refcount_t refs; /* ref for ->bpf_filters */
225
+
spinlock_t lock; /* protects ->bpf_filters modifications */
226
+
struct io_bpf_filter __rcu **filters;
227
+
struct rcu_head rcu_head;
228
+
};
229
+
222
230
struct io_restriction {
223
231
DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
224
232
DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
233
+
struct io_bpf_filters *bpf_filters;
234
+
/* ->bpf_filters needs COW on modification */
235
+
bool bpf_filters_cow;
225
236
u8 sqe_flags_allowed;
226
237
u8 sqe_flags_required;
227
238
/* IORING_OP_* restrictions exist */
···
289
278
290
279
struct task_struct *submitter_task;
291
280
struct io_rings *rings;
281
+
/* cache of ->restrictions.bpf_filters->filters */
282
+
struct io_bpf_filter __rcu **bpf_filters;
292
283
struct percpu_ref refs;
293
284
294
285
clockid_t clockid;
+1
include/linux/sched.h
+1
include/linux/sched.h
+10
include/uapi/linux/io_uring.h
+10
include/uapi/linux/io_uring.h
···
712
712
/* auxiliary zcrx configuration, see enum zcrx_ctrl_op */
713
713
IORING_REGISTER_ZCRX_CTRL = 36,
714
714
715
+
/* register bpf filtering programs */
716
+
IORING_REGISTER_BPF_FILTER = 37,
717
+
715
718
/* this goes last */
716
719
IORING_REGISTER_LAST,
717
720
···
818
815
};
819
816
__u8 resv;
820
817
__u32 resv2[3];
818
+
};
819
+
820
+
struct io_uring_task_restriction {
821
+
__u16 flags;
822
+
__u16 nr_res;
823
+
__u32 resv[3];
824
+
__DECLARE_FLEX_ARRAY(struct io_uring_restriction, restrictions);
821
825
};
822
826
823
827
struct io_uring_clock_register {
+62
include/uapi/linux/io_uring/bpf_filter.h
+62
include/uapi/linux/io_uring/bpf_filter.h
···
1
+
/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */
2
+
/*
3
+
* Header file for the io_uring BPF filters.
4
+
*/
5
+
#ifndef LINUX_IO_URING_BPF_FILTER_H
6
+
#define LINUX_IO_URING_BPF_FILTER_H
7
+
8
+
#include <linux/types.h>
9
+
10
+
/*
11
+
* Struct passed to filters.
12
+
*/
13
+
struct io_uring_bpf_ctx {
14
+
__u64 user_data;
15
+
__u8 opcode;
16
+
__u8 sqe_flags;
17
+
__u8 pdu_size; /* size of aux data for filter */
18
+
__u8 pad[5];
19
+
union {
20
+
struct {
21
+
__u32 family;
22
+
__u32 type;
23
+
__u32 protocol;
24
+
} socket;
25
+
struct {
26
+
__u64 flags;
27
+
__u64 mode;
28
+
__u64 resolve;
29
+
} open;
30
+
};
31
+
};
32
+
33
+
enum {
34
+
/*
35
+
* If set, any currently unset opcode will have a deny filter attached
36
+
*/
37
+
IO_URING_BPF_FILTER_DENY_REST = 1,
38
+
};
39
+
40
+
struct io_uring_bpf_filter {
41
+
__u32 opcode; /* io_uring opcode to filter */
42
+
__u32 flags;
43
+
__u32 filter_len; /* number of BPF instructions */
44
+
__u32 resv;
45
+
__u64 filter_ptr; /* pointer to BPF filter */
46
+
__u64 resv2[5];
47
+
};
48
+
49
+
enum {
50
+
IO_URING_BPF_CMD_FILTER = 1,
51
+
};
52
+
53
+
struct io_uring_bpf {
54
+
__u16 cmd_type; /* IO_URING_BPF_* values */
55
+
__u16 cmd_flags; /* none so far */
56
+
__u32 resv;
57
+
union {
58
+
struct io_uring_bpf_filter filter;
59
+
};
60
+
};
61
+
62
+
#endif
+5
io_uring/Kconfig
+5
io_uring/Kconfig
+1
io_uring/Makefile
+1
io_uring/Makefile
+430
io_uring/bpf_filter.c
+430
io_uring/bpf_filter.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* BPF filter support for io_uring. Supports SQE opcodes for now.
4
+
*/
5
+
#include <linux/kernel.h>
6
+
#include <linux/errno.h>
7
+
#include <linux/io_uring.h>
8
+
#include <linux/filter.h>
9
+
#include <linux/bpf.h>
10
+
#include <uapi/linux/io_uring.h>
11
+
12
+
#include "io_uring.h"
13
+
#include "bpf_filter.h"
14
+
#include "net.h"
15
+
#include "openclose.h"
16
+
17
+
struct io_bpf_filter {
18
+
refcount_t refs;
19
+
struct bpf_prog *prog;
20
+
struct io_bpf_filter *next;
21
+
};
22
+
23
+
/* Deny if this is set as the filter */
24
+
static const struct io_bpf_filter dummy_filter;
25
+
26
+
static void io_uring_populate_bpf_ctx(struct io_uring_bpf_ctx *bctx,
27
+
struct io_kiocb *req)
28
+
{
29
+
bctx->opcode = req->opcode;
30
+
bctx->sqe_flags = (__force int) req->flags & SQE_VALID_FLAGS;
31
+
bctx->user_data = req->cqe.user_data;
32
+
/* clear residual, anything from pdu_size and below */
33
+
memset((void *) bctx + offsetof(struct io_uring_bpf_ctx, pdu_size), 0,
34
+
sizeof(*bctx) - offsetof(struct io_uring_bpf_ctx, pdu_size));
35
+
36
+
/*
37
+
* Opcodes can provide a handler fo populating more data into bctx,
38
+
* for filters to use.
39
+
*/
40
+
switch (req->opcode) {
41
+
case IORING_OP_SOCKET:
42
+
bctx->pdu_size = sizeof(bctx->socket);
43
+
io_socket_bpf_populate(bctx, req);
44
+
break;
45
+
case IORING_OP_OPENAT:
46
+
case IORING_OP_OPENAT2:
47
+
bctx->pdu_size = sizeof(bctx->open);
48
+
io_openat_bpf_populate(bctx, req);
49
+
break;
50
+
}
51
+
}
52
+
53
+
/*
54
+
* Run registered filters for a given opcode. For filters, a return of 0 denies
55
+
* execution of the request, a return of 1 allows it. If any filter for an
56
+
* opcode returns 0, filter processing is stopped, and the request is denied.
57
+
* This also stops the processing of filters.
58
+
*
59
+
* __io_uring_run_bpf_filters() returns 0 on success, allow running the
60
+
* request, and -EACCES when a request is denied.
61
+
*/
62
+
int __io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters,
63
+
struct io_kiocb *req)
64
+
{
65
+
struct io_bpf_filter *filter;
66
+
struct io_uring_bpf_ctx bpf_ctx;
67
+
int ret;
68
+
69
+
/* Fast check for existence of filters outside of RCU */
70
+
if (!rcu_access_pointer(filters[req->opcode]))
71
+
return 0;
72
+
73
+
/*
74
+
* req->opcode has already been validated to be within the range
75
+
* of what we expect, io_init_req() does this.
76
+
*/
77
+
guard(rcu)();
78
+
filter = rcu_dereference(filters[req->opcode]);
79
+
if (!filter)
80
+
return 0;
81
+
else if (filter == &dummy_filter)
82
+
return -EACCES;
83
+
84
+
io_uring_populate_bpf_ctx(&bpf_ctx, req);
85
+
86
+
/*
87
+
* Iterate registered filters. The opcode is allowed IFF all filters
88
+
* return 1. If any filter returns denied, opcode will be denied.
89
+
*/
90
+
do {
91
+
if (filter == &dummy_filter)
92
+
return -EACCES;
93
+
ret = bpf_prog_run(filter->prog, &bpf_ctx);
94
+
if (!ret)
95
+
return -EACCES;
96
+
filter = filter->next;
97
+
} while (filter);
98
+
99
+
return 0;
100
+
}
101
+
102
+
static void io_free_bpf_filters(struct rcu_head *head)
103
+
{
104
+
struct io_bpf_filter __rcu **filter;
105
+
struct io_bpf_filters *filters;
106
+
int i;
107
+
108
+
filters = container_of(head, struct io_bpf_filters, rcu_head);
109
+
scoped_guard(spinlock, &filters->lock) {
110
+
filter = filters->filters;
111
+
if (!filter)
112
+
return;
113
+
}
114
+
115
+
for (i = 0; i < IORING_OP_LAST; i++) {
116
+
struct io_bpf_filter *f;
117
+
118
+
rcu_read_lock();
119
+
f = rcu_dereference(filter[i]);
120
+
while (f) {
121
+
struct io_bpf_filter *next = f->next;
122
+
123
+
/*
124
+
* Even if stacked, dummy filter will always be last
125
+
* as it can only get installed into an empty spot.
126
+
*/
127
+
if (f == &dummy_filter)
128
+
break;
129
+
130
+
/* Someone still holds a ref, stop iterating. */
131
+
if (!refcount_dec_and_test(&f->refs))
132
+
break;
133
+
134
+
bpf_prog_destroy(f->prog);
135
+
kfree(f);
136
+
f = next;
137
+
}
138
+
rcu_read_unlock();
139
+
}
140
+
kfree(filters->filters);
141
+
kfree(filters);
142
+
}
143
+
144
+
static void __io_put_bpf_filters(struct io_bpf_filters *filters)
145
+
{
146
+
if (refcount_dec_and_test(&filters->refs))
147
+
call_rcu(&filters->rcu_head, io_free_bpf_filters);
148
+
}
149
+
150
+
void io_put_bpf_filters(struct io_restriction *res)
151
+
{
152
+
if (res->bpf_filters)
153
+
__io_put_bpf_filters(res->bpf_filters);
154
+
}
155
+
156
+
static struct io_bpf_filters *io_new_bpf_filters(void)
157
+
{
158
+
struct io_bpf_filters *filters __free(kfree) = NULL;
159
+
160
+
filters = kzalloc(sizeof(*filters), GFP_KERNEL_ACCOUNT);
161
+
if (!filters)
162
+
return ERR_PTR(-ENOMEM);
163
+
164
+
filters->filters = kcalloc(IORING_OP_LAST,
165
+
sizeof(struct io_bpf_filter *),
166
+
GFP_KERNEL_ACCOUNT);
167
+
if (!filters->filters)
168
+
return ERR_PTR(-ENOMEM);
169
+
170
+
refcount_set(&filters->refs, 1);
171
+
spin_lock_init(&filters->lock);
172
+
return no_free_ptr(filters);
173
+
}
174
+
175
+
/*
176
+
* Validate classic BPF filter instructions. Only allow a safe subset of
177
+
* operations - no packet data access, just context field loads and basic
178
+
* ALU/jump operations.
179
+
*/
180
+
static int io_uring_check_cbpf_filter(struct sock_filter *filter,
181
+
unsigned int flen)
182
+
{
183
+
int pc;
184
+
185
+
for (pc = 0; pc < flen; pc++) {
186
+
struct sock_filter *ftest = &filter[pc];
187
+
u16 code = ftest->code;
188
+
u32 k = ftest->k;
189
+
190
+
switch (code) {
191
+
case BPF_LD | BPF_W | BPF_ABS:
192
+
ftest->code = BPF_LDX | BPF_W | BPF_ABS;
193
+
/* 32-bit aligned and not out of bounds. */
194
+
if (k >= sizeof(struct io_uring_bpf_ctx) || k & 3)
195
+
return -EINVAL;
196
+
continue;
197
+
case BPF_LD | BPF_W | BPF_LEN:
198
+
ftest->code = BPF_LD | BPF_IMM;
199
+
ftest->k = sizeof(struct io_uring_bpf_ctx);
200
+
continue;
201
+
case BPF_LDX | BPF_W | BPF_LEN:
202
+
ftest->code = BPF_LDX | BPF_IMM;
203
+
ftest->k = sizeof(struct io_uring_bpf_ctx);
204
+
continue;
205
+
/* Explicitly include allowed calls. */
206
+
case BPF_RET | BPF_K:
207
+
case BPF_RET | BPF_A:
208
+
case BPF_ALU | BPF_ADD | BPF_K:
209
+
case BPF_ALU | BPF_ADD | BPF_X:
210
+
case BPF_ALU | BPF_SUB | BPF_K:
211
+
case BPF_ALU | BPF_SUB | BPF_X:
212
+
case BPF_ALU | BPF_MUL | BPF_K:
213
+
case BPF_ALU | BPF_MUL | BPF_X:
214
+
case BPF_ALU | BPF_DIV | BPF_K:
215
+
case BPF_ALU | BPF_DIV | BPF_X:
216
+
case BPF_ALU | BPF_AND | BPF_K:
217
+
case BPF_ALU | BPF_AND | BPF_X:
218
+
case BPF_ALU | BPF_OR | BPF_K:
219
+
case BPF_ALU | BPF_OR | BPF_X:
220
+
case BPF_ALU | BPF_XOR | BPF_K:
221
+
case BPF_ALU | BPF_XOR | BPF_X:
222
+
case BPF_ALU | BPF_LSH | BPF_K:
223
+
case BPF_ALU | BPF_LSH | BPF_X:
224
+
case BPF_ALU | BPF_RSH | BPF_K:
225
+
case BPF_ALU | BPF_RSH | BPF_X:
226
+
case BPF_ALU | BPF_NEG:
227
+
case BPF_LD | BPF_IMM:
228
+
case BPF_LDX | BPF_IMM:
229
+
case BPF_MISC | BPF_TAX:
230
+
case BPF_MISC | BPF_TXA:
231
+
case BPF_LD | BPF_MEM:
232
+
case BPF_LDX | BPF_MEM:
233
+
case BPF_ST:
234
+
case BPF_STX:
235
+
case BPF_JMP | BPF_JA:
236
+
case BPF_JMP | BPF_JEQ | BPF_K:
237
+
case BPF_JMP | BPF_JEQ | BPF_X:
238
+
case BPF_JMP | BPF_JGE | BPF_K:
239
+
case BPF_JMP | BPF_JGE | BPF_X:
240
+
case BPF_JMP | BPF_JGT | BPF_K:
241
+
case BPF_JMP | BPF_JGT | BPF_X:
242
+
case BPF_JMP | BPF_JSET | BPF_K:
243
+
case BPF_JMP | BPF_JSET | BPF_X:
244
+
continue;
245
+
default:
246
+
return -EINVAL;
247
+
}
248
+
}
249
+
return 0;
250
+
}
251
+
252
+
void io_bpf_filter_clone(struct io_restriction *dst, struct io_restriction *src)
253
+
{
254
+
if (!src->bpf_filters)
255
+
return;
256
+
257
+
rcu_read_lock();
258
+
/*
259
+
* If the src filter is going away, just ignore it.
260
+
*/
261
+
if (refcount_inc_not_zero(&src->bpf_filters->refs)) {
262
+
dst->bpf_filters = src->bpf_filters;
263
+
dst->bpf_filters_cow = true;
264
+
}
265
+
rcu_read_unlock();
266
+
}
267
+
268
+
/*
269
+
* Allocate a new struct io_bpf_filters. Used when a filter is cloned and
270
+
* modifications need to be made.
271
+
*/
272
+
static struct io_bpf_filters *io_bpf_filter_cow(struct io_restriction *src)
273
+
{
274
+
struct io_bpf_filters *filters;
275
+
struct io_bpf_filter *srcf;
276
+
int i;
277
+
278
+
filters = io_new_bpf_filters();
279
+
if (IS_ERR(filters))
280
+
return filters;
281
+
282
+
/*
283
+
* Iterate filters from src and assign in destination. Grabbing
284
+
* a reference is enough, we don't need to duplicate the memory.
285
+
* This is safe because filters are only ever appended to the
286
+
* front of the list, hence the only memory ever touched inside
287
+
* a filter is the refcount.
288
+
*/
289
+
rcu_read_lock();
290
+
for (i = 0; i < IORING_OP_LAST; i++) {
291
+
srcf = rcu_dereference(src->bpf_filters->filters[i]);
292
+
if (!srcf) {
293
+
continue;
294
+
} else if (srcf == &dummy_filter) {
295
+
rcu_assign_pointer(filters->filters[i], &dummy_filter);
296
+
continue;
297
+
}
298
+
299
+
/*
300
+
* Getting a ref on the first node is enough, putting the
301
+
* filter and iterating nodes to free will stop on the first
302
+
* one that doesn't hit zero when dropping.
303
+
*/
304
+
if (!refcount_inc_not_zero(&srcf->refs))
305
+
goto err;
306
+
rcu_assign_pointer(filters->filters[i], srcf);
307
+
}
308
+
rcu_read_unlock();
309
+
return filters;
310
+
err:
311
+
rcu_read_unlock();
312
+
__io_put_bpf_filters(filters);
313
+
return ERR_PTR(-EBUSY);
314
+
}
315
+
316
+
#define IO_URING_BPF_FILTER_FLAGS IO_URING_BPF_FILTER_DENY_REST
317
+
318
+
int io_register_bpf_filter(struct io_restriction *res,
319
+
struct io_uring_bpf __user *arg)
320
+
{
321
+
struct io_bpf_filters *filters, *old_filters = NULL;
322
+
struct io_bpf_filter *filter, *old_filter;
323
+
struct io_uring_bpf reg;
324
+
struct bpf_prog *prog;
325
+
struct sock_fprog fprog;
326
+
int ret;
327
+
328
+
if (copy_from_user(®, arg, sizeof(reg)))
329
+
return -EFAULT;
330
+
if (reg.cmd_type != IO_URING_BPF_CMD_FILTER)
331
+
return -EINVAL;
332
+
if (reg.cmd_flags || reg.resv)
333
+
return -EINVAL;
334
+
335
+
if (reg.filter.opcode >= IORING_OP_LAST)
336
+
return -EINVAL;
337
+
if (reg.filter.flags & ~IO_URING_BPF_FILTER_FLAGS)
338
+
return -EINVAL;
339
+
if (reg.filter.resv)
340
+
return -EINVAL;
341
+
if (!mem_is_zero(reg.filter.resv2, sizeof(reg.filter.resv2)))
342
+
return -EINVAL;
343
+
if (!reg.filter.filter_len || reg.filter.filter_len > BPF_MAXINSNS)
344
+
return -EINVAL;
345
+
346
+
fprog.len = reg.filter.filter_len;
347
+
fprog.filter = u64_to_user_ptr(reg.filter.filter_ptr);
348
+
349
+
ret = bpf_prog_create_from_user(&prog, &fprog,
350
+
io_uring_check_cbpf_filter, false);
351
+
if (ret)
352
+
return ret;
353
+
354
+
/*
355
+
* No existing filters, allocate set.
356
+
*/
357
+
filters = res->bpf_filters;
358
+
if (!filters) {
359
+
filters = io_new_bpf_filters();
360
+
if (IS_ERR(filters)) {
361
+
ret = PTR_ERR(filters);
362
+
goto err_prog;
363
+
}
364
+
} else if (res->bpf_filters_cow) {
365
+
filters = io_bpf_filter_cow(res);
366
+
if (IS_ERR(filters)) {
367
+
ret = PTR_ERR(filters);
368
+
goto err_prog;
369
+
}
370
+
/*
371
+
* Stash old filters, we'll put them once we know we'll
372
+
* succeed. Until then, res->bpf_filters is left untouched.
373
+
*/
374
+
old_filters = res->bpf_filters;
375
+
}
376
+
377
+
filter = kzalloc(sizeof(*filter), GFP_KERNEL_ACCOUNT);
378
+
if (!filter) {
379
+
ret = -ENOMEM;
380
+
goto err;
381
+
}
382
+
refcount_set(&filter->refs, 1);
383
+
filter->prog = prog;
384
+
385
+
/*
386
+
* Success - install the new filter set now. If we did COW, put
387
+
* the old filters as we're replacing them.
388
+
*/
389
+
if (old_filters) {
390
+
__io_put_bpf_filters(old_filters);
391
+
res->bpf_filters_cow = false;
392
+
}
393
+
res->bpf_filters = filters;
394
+
395
+
/*
396
+
* Insert filter - if the current opcode already has a filter
397
+
* attached, add to the set.
398
+
*/
399
+
rcu_read_lock();
400
+
spin_lock_bh(&filters->lock);
401
+
old_filter = rcu_dereference(filters->filters[reg.filter.opcode]);
402
+
if (old_filter)
403
+
filter->next = old_filter;
404
+
rcu_assign_pointer(filters->filters[reg.filter.opcode], filter);
405
+
406
+
/*
407
+
* If IO_URING_BPF_FILTER_DENY_REST is set, fill any unregistered
408
+
* opcode with the dummy filter. That will cause them to be denied.
409
+
*/
410
+
if (reg.filter.flags & IO_URING_BPF_FILTER_DENY_REST) {
411
+
for (int i = 0; i < IORING_OP_LAST; i++) {
412
+
if (i == reg.filter.opcode)
413
+
continue;
414
+
old_filter = rcu_dereference(filters->filters[i]);
415
+
if (old_filter)
416
+
continue;
417
+
rcu_assign_pointer(filters->filters[i], &dummy_filter);
418
+
}
419
+
}
420
+
421
+
spin_unlock_bh(&filters->lock);
422
+
rcu_read_unlock();
423
+
return 0;
424
+
err:
425
+
if (filters != res->bpf_filters)
426
+
__io_put_bpf_filters(filters);
427
+
err_prog:
428
+
bpf_prog_destroy(prog);
429
+
return ret;
430
+
}
+48
io_uring/bpf_filter.h
+48
io_uring/bpf_filter.h
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#ifndef IO_URING_BPF_FILTER_H
3
+
#define IO_URING_BPF_FILTER_H
4
+
5
+
#include <uapi/linux/io_uring/bpf_filter.h>
6
+
7
+
#ifdef CONFIG_IO_URING_BPF
8
+
9
+
int __io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters, struct io_kiocb *req);
10
+
11
+
int io_register_bpf_filter(struct io_restriction *res,
12
+
struct io_uring_bpf __user *arg);
13
+
14
+
void io_put_bpf_filters(struct io_restriction *res);
15
+
16
+
void io_bpf_filter_clone(struct io_restriction *dst, struct io_restriction *src);
17
+
18
+
static inline int io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters,
19
+
struct io_kiocb *req)
20
+
{
21
+
if (filters)
22
+
return __io_uring_run_bpf_filters(filters, req);
23
+
24
+
return 0;
25
+
}
26
+
27
+
#else
28
+
29
+
static inline int io_register_bpf_filter(struct io_restriction *res,
30
+
struct io_uring_bpf __user *arg)
31
+
{
32
+
return -EINVAL;
33
+
}
34
+
static inline int io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters,
35
+
struct io_kiocb *req)
36
+
{
37
+
return 0;
38
+
}
39
+
static inline void io_put_bpf_filters(struct io_restriction *res)
40
+
{
41
+
}
42
+
static inline void io_bpf_filter_clone(struct io_restriction *dst,
43
+
struct io_restriction *src)
44
+
{
45
+
}
46
+
#endif /* CONFIG_IO_URING_BPF */
47
+
48
+
#endif
+48
io_uring/io_uring.c
+48
io_uring/io_uring.c
···
94
94
#include "alloc_cache.h"
95
95
#include "eventfd.h"
96
96
#include "wait.h"
97
+
#include "bpf_filter.h"
97
98
98
99
#define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \
99
100
IOSQE_IO_HARDLINK | IOSQE_ASYNC)
···
1876
1875
if (unlikely(ret))
1877
1876
return io_submit_fail_init(sqe, req, ret);
1878
1877
1878
+
if (unlikely(ctx->bpf_filters)) {
1879
+
ret = io_uring_run_bpf_filters(ctx->bpf_filters, req);
1880
+
if (ret)
1881
+
return io_submit_fail_init(sqe, req, ret);
1882
+
}
1883
+
1879
1884
trace_io_uring_submit_req(req);
1880
1885
1881
1886
/*
···
2178
2171
percpu_ref_exit(&ctx->refs);
2179
2172
free_uid(ctx->user);
2180
2173
io_req_caches_free(ctx);
2174
+
2175
+
if (ctx->restrictions.bpf_filters) {
2176
+
WARN_ON_ONCE(ctx->bpf_filters !=
2177
+
ctx->restrictions.bpf_filters->filters);
2178
+
} else {
2179
+
WARN_ON_ONCE(ctx->bpf_filters);
2180
+
}
2181
+
io_put_bpf_filters(&ctx->restrictions);
2181
2182
2182
2183
WARN_ON_ONCE(ctx->nr_req_allocated);
2183
2184
···
2900
2885
return 0;
2901
2886
}
2902
2887
2888
+
void io_restriction_clone(struct io_restriction *dst, struct io_restriction *src)
2889
+
{
2890
+
memcpy(&dst->register_op, &src->register_op, sizeof(dst->register_op));
2891
+
memcpy(&dst->sqe_op, &src->sqe_op, sizeof(dst->sqe_op));
2892
+
dst->sqe_flags_allowed = src->sqe_flags_allowed;
2893
+
dst->sqe_flags_required = src->sqe_flags_required;
2894
+
dst->op_registered = src->op_registered;
2895
+
dst->reg_registered = src->reg_registered;
2896
+
2897
+
io_bpf_filter_clone(dst, src);
2898
+
}
2899
+
2900
+
static void io_ctx_restriction_clone(struct io_ring_ctx *ctx,
2901
+
struct io_restriction *src)
2902
+
{
2903
+
struct io_restriction *dst = &ctx->restrictions;
2904
+
2905
+
io_restriction_clone(dst, src);
2906
+
if (dst->bpf_filters)
2907
+
WRITE_ONCE(ctx->bpf_filters, dst->bpf_filters->filters);
2908
+
if (dst->op_registered)
2909
+
ctx->op_restricted = 1;
2910
+
if (dst->reg_registered)
2911
+
ctx->reg_restricted = 1;
2912
+
}
2913
+
2903
2914
static __cold int io_uring_create(struct io_ctx_config *config)
2904
2915
{
2905
2916
struct io_uring_params *p = &config->p;
···
2985
2944
ctx->notify_method = TWA_SIGNAL_NO_IPI;
2986
2945
else
2987
2946
ctx->notify_method = TWA_SIGNAL;
2947
+
2948
+
/*
2949
+
* If the current task has restrictions enabled, then copy them to
2950
+
* our newly created ring and mark it as registered.
2951
+
*/
2952
+
if (current->io_uring_restrict)
2953
+
io_ctx_restriction_clone(ctx, current->io_uring_restrict);
2988
2954
2989
2955
/*
2990
2956
* This is just grabbed for accounting purposes. When a process exits,
+1
io_uring/io_uring.h
+1
io_uring/io_uring.h
···
199
199
bool __io_alloc_req_refill(struct io_ring_ctx *ctx);
200
200
201
201
void io_activate_pollwq(struct io_ring_ctx *ctx);
202
+
void io_restriction_clone(struct io_restriction *dst, struct io_restriction *src);
202
203
203
204
static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx)
204
205
{
+9
io_uring/net.c
+9
io_uring/net.c
···
1703
1703
return IOU_COMPLETE;
1704
1704
}
1705
1705
1706
+
void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
1707
+
{
1708
+
struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1709
+
1710
+
bctx->socket.family = sock->domain;
1711
+
bctx->socket.type = sock->type;
1712
+
bctx->socket.protocol = sock->protocol;
1713
+
}
1714
+
1706
1715
int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1707
1716
{
1708
1717
struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
+6
io_uring/net.h
+6
io_uring/net.h
···
3
3
#include <linux/net.h>
4
4
#include <linux/uio.h>
5
5
#include <linux/io_uring_types.h>
6
+
#include <uapi/linux/io_uring/bpf_filter.h>
6
7
7
8
struct io_async_msghdr {
8
9
#if defined(CONFIG_NET)
···
45
44
46
45
int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
47
46
int io_socket(struct io_kiocb *req, unsigned int issue_flags);
47
+
void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req);
48
48
49
49
int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
50
50
int io_connect(struct io_kiocb *req, unsigned int issue_flags);
···
64
62
void io_netmsg_cache_free(const void *entry);
65
63
#else
66
64
static inline void io_netmsg_cache_free(const void *entry)
65
+
{
66
+
}
67
+
static inline void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx,
68
+
struct io_kiocb *req)
67
69
{
68
70
}
69
71
#endif
+9
io_uring/openclose.c
+9
io_uring/openclose.c
···
82
82
return 0;
83
83
}
84
84
85
+
void io_openat_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
86
+
{
87
+
struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
88
+
89
+
bctx->open.flags = open->how.flags;
90
+
bctx->open.mode = open->how.mode;
91
+
bctx->open.resolve = open->how.resolve;
92
+
}
93
+
85
94
int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
86
95
{
87
96
struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
+3
io_uring/openclose.h
+3
io_uring/openclose.h
···
1
1
// SPDX-License-Identifier: GPL-2.0
2
2
3
+
#include "bpf_filter.h"
4
+
3
5
int __io_close_fixed(struct io_ring_ctx *ctx, unsigned int issue_flags,
4
6
unsigned int offset);
5
7
6
8
int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
7
9
int io_openat(struct io_kiocb *req, unsigned int issue_flags);
8
10
void io_open_cleanup(struct io_kiocb *req);
11
+
void io_openat_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req);
9
12
10
13
int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
11
14
int io_openat2(struct io_kiocb *req, unsigned int issue_flags);
+91
io_uring/register.c
+91
io_uring/register.c
···
33
33
#include "memmap.h"
34
34
#include "zcrx.h"
35
35
#include "query.h"
36
+
#include "bpf_filter.h"
36
37
37
38
#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
38
39
IORING_REGISTER_LAST + IORING_OP_LAST)
···
187
186
ctx->op_restricted = 1;
188
187
if (ctx->restrictions.reg_registered)
189
188
ctx->reg_restricted = 1;
189
+
return 0;
190
+
}
191
+
192
+
static int io_register_restrictions_task(void __user *arg, unsigned int nr_args)
193
+
{
194
+
struct io_uring_task_restriction __user *ures = arg;
195
+
struct io_uring_task_restriction tres;
196
+
struct io_restriction *res;
197
+
int ret;
198
+
199
+
/* Disallow if task already has registered restrictions */
200
+
if (current->io_uring_restrict)
201
+
return -EPERM;
202
+
/*
203
+
* Similar to seccomp, disallow setting a filter if task_no_new_privs
204
+
* is true and we're not CAP_SYS_ADMIN.
205
+
*/
206
+
if (!task_no_new_privs(current) &&
207
+
!ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN))
208
+
return -EACCES;
209
+
if (nr_args != 1)
210
+
return -EINVAL;
211
+
212
+
if (copy_from_user(&tres, arg, sizeof(tres)))
213
+
return -EFAULT;
214
+
215
+
if (tres.flags)
216
+
return -EINVAL;
217
+
if (!mem_is_zero(tres.resv, sizeof(tres.resv)))
218
+
return -EINVAL;
219
+
220
+
res = kzalloc(sizeof(*res), GFP_KERNEL_ACCOUNT);
221
+
if (!res)
222
+
return -ENOMEM;
223
+
224
+
ret = io_parse_restrictions(ures->restrictions, tres.nr_res, res);
225
+
if (ret < 0) {
226
+
kfree(res);
227
+
return ret;
228
+
}
229
+
current->io_uring_restrict = res;
230
+
return 0;
231
+
}
232
+
233
+
static int io_register_bpf_filter_task(void __user *arg, unsigned int nr_args)
234
+
{
235
+
struct io_restriction *res;
236
+
int ret;
237
+
238
+
/*
239
+
* Similar to seccomp, disallow setting a filter if task_no_new_privs
240
+
* is true and we're not CAP_SYS_ADMIN.
241
+
*/
242
+
if (!task_no_new_privs(current) &&
243
+
!ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN))
244
+
return -EACCES;
245
+
246
+
if (nr_args != 1)
247
+
return -EINVAL;
248
+
249
+
/* If no task restrictions exist, setup a new set */
250
+
res = current->io_uring_restrict;
251
+
if (!res) {
252
+
res = kzalloc(sizeof(*res), GFP_KERNEL_ACCOUNT);
253
+
if (!res)
254
+
return -ENOMEM;
255
+
}
256
+
257
+
ret = io_register_bpf_filter(res, arg);
258
+
if (ret) {
259
+
if (res != current->io_uring_restrict)
260
+
kfree(res);
261
+
return ret;
262
+
}
263
+
if (!current->io_uring_restrict)
264
+
current->io_uring_restrict = res;
190
265
return 0;
191
266
}
192
267
···
909
832
case IORING_REGISTER_ZCRX_CTRL:
910
833
ret = io_zcrx_ctrl(ctx, arg, nr_args);
911
834
break;
835
+
case IORING_REGISTER_BPF_FILTER:
836
+
ret = -EINVAL;
837
+
838
+
if (nr_args != 1)
839
+
break;
840
+
ret = io_register_bpf_filter(&ctx->restrictions, arg);
841
+
if (!ret)
842
+
WRITE_ONCE(ctx->bpf_filters,
843
+
ctx->restrictions.bpf_filters->filters);
844
+
break;
912
845
default:
913
846
ret = -EINVAL;
914
847
break;
···
990
903
return io_uring_register_send_msg_ring(arg, nr_args);
991
904
case IORING_REGISTER_QUERY:
992
905
return io_query(arg, nr_args);
906
+
case IORING_REGISTER_RESTRICTIONS:
907
+
return io_register_restrictions_task(arg, nr_args);
908
+
case IORING_REGISTER_BPF_FILTER:
909
+
return io_register_bpf_filter_task(arg, nr_args);
993
910
}
994
911
return -EINVAL;
995
912
}
+33
-9
io_uring/tctx.c
+33
-9
io_uring/tctx.c
···
11
11
12
12
#include "io_uring.h"
13
13
#include "tctx.h"
14
+
#include "bpf_filter.h"
14
15
15
16
static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
16
17
struct task_struct *task)
···
55
54
* node is stored in the xarray. Until that gets sorted out, attempt
56
55
* an iteration here and warn if any entries are found.
57
56
*/
58
-
xa_for_each(&tctx->xa, index, node) {
59
-
WARN_ON_ONCE(1);
60
-
break;
61
-
}
62
-
WARN_ON_ONCE(tctx->io_wq);
63
-
WARN_ON_ONCE(tctx->cached_refs);
57
+
if (tctx) {
58
+
xa_for_each(&tctx->xa, index, node) {
59
+
WARN_ON_ONCE(1);
60
+
break;
61
+
}
62
+
WARN_ON_ONCE(tctx->io_wq);
63
+
WARN_ON_ONCE(tctx->cached_refs);
64
64
65
-
percpu_counter_destroy(&tctx->inflight);
66
-
kfree(tctx);
67
-
tsk->io_uring = NULL;
65
+
percpu_counter_destroy(&tctx->inflight);
66
+
kfree(tctx);
67
+
tsk->io_uring = NULL;
68
+
}
69
+
if (tsk->io_uring_restrict) {
70
+
io_put_bpf_filters(tsk->io_uring_restrict);
71
+
kfree(tsk->io_uring_restrict);
72
+
tsk->io_uring_restrict = NULL;
73
+
}
68
74
}
69
75
70
76
__cold int io_uring_alloc_task_context(struct task_struct *task,
···
369
361
}
370
362
371
363
return i ? i : ret;
364
+
}
365
+
366
+
int __io_uring_fork(struct task_struct *tsk)
367
+
{
368
+
struct io_restriction *res, *src = tsk->io_uring_restrict;
369
+
370
+
/* Don't leave it dangling on error */
371
+
tsk->io_uring_restrict = NULL;
372
+
373
+
res = kzalloc(sizeof(*res), GFP_KERNEL_ACCOUNT);
374
+
if (!res)
375
+
return -ENOMEM;
376
+
377
+
tsk->io_uring_restrict = res;
378
+
io_restriction_clone(res, src);
379
+
return 0;
372
380
}
+6
kernel/fork.c
+6
kernel/fork.c
···
97
97
#include <linux/kasan.h>
98
98
#include <linux/scs.h>
99
99
#include <linux/io_uring.h>
100
+
#include <linux/io_uring_types.h>
100
101
#include <linux/bpf.h>
101
102
#include <linux/stackprotector.h>
102
103
#include <linux/user_events.h>
···
2127
2126
2128
2127
#ifdef CONFIG_IO_URING
2129
2128
p->io_uring = NULL;
2129
+
retval = io_uring_fork(p);
2130
+
if (unlikely(retval))
2131
+
goto bad_fork_cleanup_delayacct;
2132
+
retval = -EAGAIN;
2130
2133
#endif
2131
2134
2132
2135
p->default_timer_slack_ns = current->timer_slack_ns;
···
2527
2522
mpol_put(p->mempolicy);
2528
2523
#endif
2529
2524
bad_fork_cleanup_delayacct:
2525
+
io_uring_free(p);
2530
2526
delayacct_tsk_free(p);
2531
2527
bad_fork_cleanup_count:
2532
2528
dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);