Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/bpf_filter: cache lookup table in ctx->bpf_filters
Currently a few pointer dereferences need to be made to both check if
BPF filters are installed, and then also to retrieve the actual filter
for the opcode. Cache the table in ctx->bpf_filters to avoid that.
Add a bit of debug info on ring exit to show if we ever got this wrong.
Small risk of that given that the table is currently only updated in one
spot, but once task forking is enabled, that will add one more spot.
Reviewed-by: Christian Brauner <brauner@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+23
-10
+2
include/linux/io_uring_types.h
+2
include/linux/io_uring_types.h
+4
-3
io_uring/bpf_filter.c
+4
-3
io_uring/bpf_filter.c
···
58
58
* __io_uring_run_bpf_filters() returns 0 on success, allow running the
59
59
* request, and -EACCES when a request is denied.
60
60
*/
61
-
int __io_uring_run_bpf_filters(struct io_restriction *res, struct io_kiocb *req)
61
+
int __io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters,
62
+
struct io_kiocb *req)
62
63
{
63
64
struct io_bpf_filter *filter;
64
65
struct io_uring_bpf_ctx bpf_ctx;
65
66
int ret;
66
67
67
68
/* Fast check for existence of filters outside of RCU */
68
-
if (!rcu_access_pointer(res->bpf_filters->filters[req->opcode]))
69
+
if (!rcu_access_pointer(filters[req->opcode]))
69
70
return 0;
70
71
71
72
/*
···
74
73
* of what we expect, io_init_req() does this.
75
74
*/
76
75
guard(rcu)();
77
-
filter = rcu_dereference(res->bpf_filters->filters[req->opcode]);
76
+
filter = rcu_dereference(filters[req->opcode]);
78
77
if (!filter)
79
78
return 0;
80
79
else if (filter == &dummy_filter)
+5
-5
io_uring/bpf_filter.h
+5
-5
io_uring/bpf_filter.h
···
6
6
7
7
#ifdef CONFIG_IO_URING_BPF
8
8
9
-
int __io_uring_run_bpf_filters(struct io_restriction *res, struct io_kiocb *req);
9
+
int __io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters, struct io_kiocb *req);
10
10
11
11
int io_register_bpf_filter(struct io_restriction *res,
12
12
struct io_uring_bpf __user *arg);
13
13
14
14
void io_put_bpf_filters(struct io_restriction *res);
15
15
16
-
static inline int io_uring_run_bpf_filters(struct io_restriction *res,
16
+
static inline int io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters,
17
17
struct io_kiocb *req)
18
18
{
19
-
if (res->bpf_filters)
20
-
return __io_uring_run_bpf_filters(res, req);
19
+
if (filters)
20
+
return __io_uring_run_bpf_filters(filters, req);
21
21
22
22
return 0;
23
23
}
···
29
29
{
30
30
return -EINVAL;
31
31
}
32
-
static inline int io_uring_run_bpf_filters(struct io_restriction *res,
32
+
static inline int io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters,
33
33
struct io_kiocb *req)
34
34
{
35
35
return 0;
+9
-2
io_uring/io_uring.c
+9
-2
io_uring/io_uring.c
···
1875
1875
if (unlikely(ret))
1876
1876
return io_submit_fail_init(sqe, req, ret);
1877
1877
1878
-
if (unlikely(ctx->restrictions.bpf_filters)) {
1879
-
ret = io_uring_run_bpf_filters(&ctx->restrictions, req);
1878
+
if (unlikely(ctx->bpf_filters)) {
1879
+
ret = io_uring_run_bpf_filters(ctx->bpf_filters, req);
1880
1880
if (ret)
1881
1881
return io_submit_fail_init(sqe, req, ret);
1882
1882
}
···
2168
2168
percpu_ref_exit(&ctx->refs);
2169
2169
free_uid(ctx->user);
2170
2170
io_req_caches_free(ctx);
2171
+
2172
+
if (ctx->restrictions.bpf_filters) {
2173
+
WARN_ON_ONCE(ctx->bpf_filters !=
2174
+
ctx->restrictions.bpf_filters->filters);
2175
+
} else {
2176
+
WARN_ON_ONCE(ctx->bpf_filters);
2177
+
}
2171
2178
io_put_bpf_filters(&ctx->restrictions);
2172
2179
2173
2180
WARN_ON_ONCE(ctx->nr_req_allocated);