Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring: get rid of alloc cache init_once handling
init_once is called when an object doesn't come from the cache, and
hence needs initial clearing of certain members. While the whole
struct could get cleared by memset() in that case, a few of the cache
members are large enough that this may cause unnecessary overhead if
the caches used aren't large enough to satisfy the workload. For those
cases, some churn of kmalloc+kfree is to be expected.
Ensure that the 3 users that need clearing put the members they need
cleared at the start of the struct, and wrap the rest of the struct in
a struct group so the offset is known.
While at it, improve the interaction with KASAN such that when/if
KASAN writes to members inside the struct that should be retained over
caching, it won't trip over itself. For rw and net, the retaining of
the iovec over caching is disabled if KASAN is enabled. A helper will
free and clear those members in that case.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+91
-93
+1
-1
include/linux/io_uring/cmd.h
+1
-1
include/linux/io_uring/cmd.h
+2
-1
include/linux/io_uring_types.h
+2
-1
include/linux/io_uring_types.h
+34
-9
io_uring/alloc_cache.h
+34
-9
io_uring/alloc_cache.h
···
6
6
*/
7
7
#define IO_ALLOC_CACHE_MAX 128
8
8
9
+
#if defined(CONFIG_KASAN)
10
+
static inline void io_alloc_cache_kasan(struct iovec **iov, int *nr)
11
+
{
12
+
kfree(*iov);
13
+
*iov = NULL;
14
+
*nr = 0;
15
+
}
16
+
#else
17
+
static inline void io_alloc_cache_kasan(struct iovec **iov, int *nr)
18
+
{
19
+
}
20
+
#endif
21
+
9
22
static inline bool io_alloc_cache_put(struct io_alloc_cache *cache,
10
23
void *entry)
11
24
{
···
36
23
if (cache->nr_cached) {
37
24
void *entry = cache->entries[--cache->nr_cached];
38
25
26
+
/*
27
+
* If KASAN is enabled, always clear the initial bytes that
28
+
* must be zeroed post alloc, in case any of them overlap
29
+
* with KASAN storage.
30
+
*/
31
+
#if defined(CONFIG_KASAN)
39
32
kasan_mempool_unpoison_object(entry, cache->elem_size);
33
+
if (cache->init_clear)
34
+
memset(entry, 0, cache->init_clear);
35
+
#endif
40
36
return entry;
41
37
}
42
38
43
39
return NULL;
44
40
}
45
41
46
-
static inline void *io_cache_alloc(struct io_alloc_cache *cache, gfp_t gfp,
47
-
void (*init_once)(void *obj))
42
+
static inline void *io_cache_alloc(struct io_alloc_cache *cache, gfp_t gfp)
48
43
{
49
-
if (unlikely(!cache->nr_cached)) {
50
-
void *obj = kmalloc(cache->elem_size, gfp);
44
+
void *obj;
51
45
52
-
if (obj && init_once)
53
-
init_once(obj);
46
+
obj = io_alloc_cache_get(cache);
47
+
if (obj)
54
48
return obj;
55
-
}
56
-
return io_alloc_cache_get(cache);
49
+
50
+
obj = kmalloc(cache->elem_size, gfp);
51
+
if (obj && cache->init_clear)
52
+
memset(obj, 0, cache->init_clear);
53
+
return obj;
57
54
}
58
55
59
56
/* returns false if the cache was initialized properly */
60
57
static inline bool io_alloc_cache_init(struct io_alloc_cache *cache,
61
-
unsigned max_nr, size_t size)
58
+
unsigned max_nr, unsigned int size,
59
+
unsigned int init_bytes)
62
60
{
63
61
cache->entries = kvmalloc_array(max_nr, sizeof(void *), GFP_KERNEL);
64
62
if (cache->entries) {
65
63
cache->nr_cached = 0;
66
64
cache->max_cached = max_nr;
67
65
cache->elem_size = size;
66
+
cache->init_clear = init_bytes;
68
67
return false;
69
68
}
70
69
return true;
+2
-2
io_uring/futex.c
+2
-2
io_uring/futex.c
···
36
36
bool io_futex_cache_init(struct io_ring_ctx *ctx)
37
37
{
38
38
return io_alloc_cache_init(&ctx->futex_cache, IO_FUTEX_ALLOC_CACHE_MAX,
39
-
sizeof(struct io_futex_data));
39
+
sizeof(struct io_futex_data), 0);
40
40
}
41
41
42
42
void io_futex_cache_free(struct io_ring_ctx *ctx)
···
320
320
}
321
321
322
322
io_ring_submit_lock(ctx, issue_flags);
323
-
ifd = io_cache_alloc(&ctx->futex_cache, GFP_NOWAIT, NULL);
323
+
ifd = io_cache_alloc(&ctx->futex_cache, GFP_NOWAIT);
324
324
if (!ifd) {
325
325
ret = -ENOMEM;
326
326
goto done_unlock;
+7
-5
io_uring/io_uring.c
+7
-5
io_uring/io_uring.c
···
315
315
INIT_LIST_HEAD(&ctx->cq_overflow_list);
316
316
INIT_LIST_HEAD(&ctx->io_buffers_cache);
317
317
ret = io_alloc_cache_init(&ctx->apoll_cache, IO_POLL_ALLOC_CACHE_MAX,
318
-
sizeof(struct async_poll));
318
+
sizeof(struct async_poll), 0);
319
319
ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
320
-
sizeof(struct io_async_msghdr));
320
+
sizeof(struct io_async_msghdr),
321
+
offsetof(struct io_async_msghdr, clear));
321
322
ret |= io_alloc_cache_init(&ctx->rw_cache, IO_ALLOC_CACHE_MAX,
322
-
sizeof(struct io_async_rw));
323
+
sizeof(struct io_async_rw),
324
+
offsetof(struct io_async_rw, clear));
323
325
ret |= io_alloc_cache_init(&ctx->uring_cache, IO_ALLOC_CACHE_MAX,
324
-
sizeof(struct io_uring_cmd_data));
326
+
sizeof(struct io_uring_cmd_data), 0);
325
327
spin_lock_init(&ctx->msg_lock);
326
328
ret |= io_alloc_cache_init(&ctx->msg_cache, IO_ALLOC_CACHE_MAX,
327
-
sizeof(struct io_kiocb));
329
+
sizeof(struct io_kiocb), 0);
328
330
ret |= io_futex_cache_init(ctx);
329
331
if (ret)
330
332
goto free_ref;
+2
-3
io_uring/io_uring.h
+2
-3
io_uring/io_uring.h
···
226
226
}
227
227
228
228
static inline void *io_uring_alloc_async_data(struct io_alloc_cache *cache,
229
-
struct io_kiocb *req,
230
-
void (*init_once)(void *obj))
229
+
struct io_kiocb *req)
231
230
{
232
-
req->async_data = io_cache_alloc(cache, GFP_KERNEL, init_once);
231
+
req->async_data = io_cache_alloc(cache, GFP_KERNEL);
233
232
if (req->async_data)
234
233
req->flags |= REQ_F_ASYNC_DATA;
235
234
return req->async_data;
+6
-22
io_uring/net.c
+6
-22
io_uring/net.c
···
137
137
static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
138
138
{
139
139
struct io_async_msghdr *hdr = req->async_data;
140
-
struct iovec *iov;
141
140
142
141
/* can't recycle, ensure we free the iovec if we have one */
143
142
if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
···
145
146
}
146
147
147
148
/* Let normal cleanup path reap it if we fail adding to the cache */
148
-
iov = hdr->free_iov;
149
+
io_alloc_cache_kasan(&hdr->free_iov, &hdr->free_iov_nr);
149
150
if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr)) {
150
-
if (iov)
151
-
kasan_mempool_poison_object(iov);
152
151
req->async_data = NULL;
153
152
req->flags &= ~REQ_F_ASYNC_DATA;
154
153
}
155
-
}
156
-
157
-
static void io_msg_async_data_init(void *obj)
158
-
{
159
-
struct io_async_msghdr *hdr = (struct io_async_msghdr *)obj;
160
-
161
-
hdr->free_iov = NULL;
162
-
hdr->free_iov_nr = 0;
163
154
}
164
155
165
156
static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)
···
157
168
struct io_ring_ctx *ctx = req->ctx;
158
169
struct io_async_msghdr *hdr;
159
170
160
-
hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req,
161
-
io_msg_async_data_init);
171
+
hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req);
162
172
if (!hdr)
163
173
return NULL;
164
174
165
175
/* If the async data was cached, we might have an iov cached inside. */
166
-
if (hdr->free_iov) {
167
-
kasan_mempool_unpoison_object(hdr->free_iov,
168
-
hdr->free_iov_nr * sizeof(struct iovec));
176
+
if (hdr->free_iov)
169
177
req->flags |= REQ_F_NEED_CLEANUP;
170
-
}
171
178
return hdr;
172
179
}
173
180
···
1798
1813
{
1799
1814
struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry;
1800
1815
1801
-
if (kmsg->free_iov) {
1802
-
kasan_mempool_unpoison_object(kmsg->free_iov,
1803
-
kmsg->free_iov_nr * sizeof(struct iovec));
1816
+
#if !defined(CONFIG_KASAN)
1817
+
if (kmsg->free_iov)
1804
1818
io_netmsg_iovec_free(kmsg);
1805
-
}
1819
+
#endif
1806
1820
kfree(kmsg);
1807
1821
}
1808
1822
#endif
+12
-8
io_uring/net.h
+12
-8
io_uring/net.h
···
5
5
6
6
struct io_async_msghdr {
7
7
#if defined(CONFIG_NET)
8
-
struct iovec fast_iov;
9
-
/* points to an allocated iov, if NULL we use fast_iov instead */
10
8
struct iovec *free_iov;
9
+
/* points to an allocated iov, if NULL we use fast_iov instead */
11
10
int free_iov_nr;
12
-
int namelen;
13
-
__kernel_size_t controllen;
14
-
__kernel_size_t payloadlen;
15
-
struct sockaddr __user *uaddr;
16
-
struct msghdr msg;
17
-
struct sockaddr_storage addr;
11
+
struct_group(clear,
12
+
int namelen;
13
+
struct iovec fast_iov;
14
+
__kernel_size_t controllen;
15
+
__kernel_size_t payloadlen;
16
+
struct sockaddr __user *uaddr;
17
+
struct msghdr msg;
18
+
struct sockaddr_storage addr;
19
+
);
20
+
#else
21
+
struct_group(clear);
18
22
#endif
19
23
};
20
24
+1
-1
io_uring/poll.c
+1
-1
io_uring/poll.c
···
650
650
kfree(apoll->double_poll);
651
651
} else {
652
652
if (!(issue_flags & IO_URING_F_UNLOCKED))
653
-
apoll = io_cache_alloc(&ctx->apoll_cache, GFP_ATOMIC, NULL);
653
+
apoll = io_cache_alloc(&ctx->apoll_cache, GFP_ATOMIC);
654
654
else
655
655
apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
656
656
if (!apoll)
+6
-21
io_uring/rw.c
+6
-21
io_uring/rw.c
···
158
158
static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)
159
159
{
160
160
struct io_async_rw *rw = req->async_data;
161
-
struct iovec *iov;
162
161
163
162
if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
164
163
io_rw_iovec_free(rw);
165
164
return;
166
165
}
167
-
iov = rw->free_iovec;
166
+
io_alloc_cache_kasan(&rw->free_iovec, &rw->free_iov_nr);
168
167
if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) {
169
-
if (iov)
170
-
kasan_mempool_poison_object(iov);
171
168
req->async_data = NULL;
172
169
req->flags &= ~REQ_F_ASYNC_DATA;
173
170
}
···
205
208
}
206
209
}
207
210
208
-
static void io_rw_async_data_init(void *obj)
209
-
{
210
-
struct io_async_rw *rw = (struct io_async_rw *)obj;
211
-
212
-
rw->free_iovec = NULL;
213
-
rw->bytes_done = 0;
214
-
}
215
-
216
211
static int io_rw_alloc_async(struct io_kiocb *req)
217
212
{
218
213
struct io_ring_ctx *ctx = req->ctx;
219
214
struct io_async_rw *rw;
220
215
221
-
rw = io_uring_alloc_async_data(&ctx->rw_cache, req, io_rw_async_data_init);
216
+
rw = io_uring_alloc_async_data(&ctx->rw_cache, req);
222
217
if (!rw)
223
218
return -ENOMEM;
224
-
if (rw->free_iovec) {
225
-
kasan_mempool_unpoison_object(rw->free_iovec,
226
-
rw->free_iov_nr * sizeof(struct iovec));
219
+
if (rw->free_iovec)
227
220
req->flags |= REQ_F_NEED_CLEANUP;
228
-
}
229
221
rw->bytes_done = 0;
230
222
return 0;
231
223
}
···
1309
1323
{
1310
1324
struct io_async_rw *rw = (struct io_async_rw *) entry;
1311
1325
1312
-
if (rw->free_iovec) {
1313
-
kasan_mempool_unpoison_object(rw->free_iovec,
1314
-
rw->free_iov_nr * sizeof(struct iovec));
1326
+
#if !defined(CONFIG_KASAN)
1327
+
if (rw->free_iovec)
1315
1328
io_rw_iovec_free(rw);
1316
-
}
1329
+
#endif
1317
1330
kfree(rw);
1318
1331
}
+16
-11
io_uring/rw.h
+16
-11
io_uring/rw.h
···
9
9
10
10
struct io_async_rw {
11
11
size_t bytes_done;
12
-
struct iov_iter iter;
13
-
struct iov_iter_state iter_state;
14
-
struct iovec fast_iov;
15
12
struct iovec *free_iovec;
16
-
int free_iov_nr;
17
-
/* wpq is for buffered io, while meta fields are used with direct io */
18
-
union {
19
-
struct wait_page_queue wpq;
20
-
struct {
21
-
struct uio_meta meta;
22
-
struct io_meta_state meta_state;
13
+
struct_group(clear,
14
+
struct iov_iter iter;
15
+
struct iov_iter_state iter_state;
16
+
struct iovec fast_iov;
17
+
int free_iov_nr;
18
+
/*
19
+
* wpq is for buffered io, while meta fields are used with
20
+
* direct io
21
+
*/
22
+
union {
23
+
struct wait_page_queue wpq;
24
+
struct {
25
+
struct uio_meta meta;
26
+
struct io_meta_state meta_state;
27
+
};
23
28
};
24
-
};
29
+
);
25
30
};
26
31
27
32
int io_prep_read_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe);
+2
-9
io_uring/uring_cmd.c
+2
-9
io_uring/uring_cmd.c
···
168
168
}
169
169
EXPORT_SYMBOL_GPL(io_uring_cmd_done);
170
170
171
-
static void io_uring_cmd_init_once(void *obj)
172
-
{
173
-
struct io_uring_cmd_data *data = obj;
174
-
175
-
data->op_data = NULL;
176
-
}
177
-
178
171
static int io_uring_cmd_prep_setup(struct io_kiocb *req,
179
172
const struct io_uring_sqe *sqe)
180
173
{
181
174
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
182
175
struct io_uring_cmd_data *cache;
183
176
184
-
cache = io_uring_alloc_async_data(&req->ctx->uring_cache, req,
185
-
io_uring_cmd_init_once);
177
+
cache = io_uring_alloc_async_data(&req->ctx->uring_cache, req);
186
178
if (!cache)
187
179
return -ENOMEM;
180
+
cache->op_data = NULL;
188
181
189
182
if (!(req->flags & REQ_F_FORCE_ASYNC)) {
190
183
/* defer memcpy until we need it */