Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/rsrc: add io_rsrc_node_lookup() helper
There are lots of spots open-coding this functionality, add a generic
helper that does the node lookup in a speculation safe way.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+57
-59
+5
-3
io_uring/cancel.c
+5
-3
io_uring/cancel.c
···
240
240
/* fixed must be grabbed every time since we drop the uring_lock */
241
241
if ((cd->flags & IORING_ASYNC_CANCEL_FD) &&
242
242
(cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
243
-
if (unlikely(fd >= ctx->file_table.data.nr))
243
+
struct io_rsrc_node *node;
244
+
245
+
node = io_rsrc_node_lookup(&ctx->file_table.data, fd);
246
+
if (unlikely(!node))
244
247
return -EBADF;
245
-
fd = array_index_nospec(fd, ctx->file_table.data.nr);
246
-
cd->file = io_file_from_index(&ctx->file_table, fd);
248
+
cd->file = io_slot_file(node);
247
249
if (!cd->file)
248
250
return -EBADF;
249
251
}
+9
-7
io_uring/filetable.c
+9
-7
io_uring/filetable.c
···
58
58
u32 slot_index)
59
59
__must_hold(&req->ctx->uring_lock)
60
60
{
61
-
struct io_rsrc_node *node;
61
+
struct io_rsrc_node *node, *old_node;
62
62
63
63
if (io_is_uring_fops(file))
64
64
return -EBADF;
···
71
71
if (!node)
72
72
return -ENOMEM;
73
73
74
-
slot_index = array_index_nospec(slot_index, ctx->file_table.data.nr);
75
-
if (ctx->file_table.data.nodes[slot_index])
76
-
io_put_rsrc_node(ctx->file_table.data.nodes[slot_index]);
74
+
old_node = io_rsrc_node_lookup(&ctx->file_table.data, slot_index);
75
+
if (old_node)
76
+
io_put_rsrc_node(old_node);
77
77
else
78
78
io_file_bitmap_set(&ctx->file_table, slot_index);
79
79
···
123
123
124
124
int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
125
125
{
126
+
struct io_rsrc_node *node;
127
+
126
128
if (unlikely(!ctx->file_table.data.nr))
127
129
return -ENXIO;
128
130
if (offset >= ctx->file_table.data.nr)
129
131
return -EINVAL;
130
132
131
-
offset = array_index_nospec(offset, ctx->file_table.data.nr);
132
-
if (!ctx->file_table.data.nodes[offset])
133
+
node = io_rsrc_node_lookup(&ctx->file_table.data, offset);
134
+
if (!node)
133
135
return -EBADF;
134
-
io_put_rsrc_node(ctx->file_table.data.nodes[offset]);
136
+
io_put_rsrc_node(node);
135
137
ctx->file_table.data.nodes[offset] = NULL;
136
138
io_file_bitmap_clear(&ctx->file_table, offset);
137
139
return 0;
+1
-1
io_uring/filetable.h
+1
-1
io_uring/filetable.h
+1
-5
io_uring/io_uring.c
+1
-5
io_uring/io_uring.c
···
1879
1879
struct file *file = NULL;
1880
1880
1881
1881
io_ring_submit_lock(ctx, issue_flags);
1882
-
if (unlikely((unsigned int)fd >= ctx->file_table.data.nr))
1883
-
goto out;
1884
-
fd = array_index_nospec(fd, ctx->file_table.data.nr);
1885
-
node = ctx->file_table.data.nodes[fd];
1882
+
node = io_rsrc_node_lookup(&ctx->file_table.data, fd);
1886
1883
if (node) {
1887
1884
io_req_assign_rsrc_node(req, node);
1888
1885
req->flags |= io_slot_flags(node);
1889
1886
file = io_slot_file(node);
1890
1887
}
1891
-
out:
1892
1888
io_ring_submit_unlock(ctx, issue_flags);
1893
1889
return file;
1894
1890
}
+15
-16
io_uring/msg_ring.c
+15
-16
io_uring/msg_ring.c
···
172
172
return __io_msg_ring_data(target_ctx, msg, issue_flags);
173
173
}
174
174
175
-
static struct file *io_msg_grab_file(struct io_kiocb *req, unsigned int issue_flags)
175
+
static int io_msg_grab_file(struct io_kiocb *req, unsigned int issue_flags)
176
176
{
177
177
struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
178
178
struct io_ring_ctx *ctx = req->ctx;
179
-
struct file *file = NULL;
180
-
int idx = msg->src_fd;
179
+
struct io_rsrc_node *node;
180
+
int ret = -EBADF;
181
181
182
182
io_ring_submit_lock(ctx, issue_flags);
183
-
if (likely(idx < ctx->file_table.data.nr)) {
184
-
idx = array_index_nospec(idx, ctx->file_table.data.nr);
185
-
file = io_file_from_index(&ctx->file_table, idx);
186
-
if (file)
187
-
get_file(file);
183
+
node = io_rsrc_node_lookup(&ctx->file_table.data, msg->src_fd);
184
+
if (node) {
185
+
msg->src_file = io_slot_file(node);
186
+
if (msg->src_file)
187
+
get_file(msg->src_file);
188
+
req->flags |= REQ_F_NEED_CLEANUP;
189
+
ret = 0;
188
190
}
189
191
io_ring_submit_unlock(ctx, issue_flags);
190
-
return file;
192
+
return ret;
191
193
}
192
194
193
195
static int io_msg_install_complete(struct io_kiocb *req, unsigned int issue_flags)
···
258
256
struct io_ring_ctx *target_ctx = req->file->private_data;
259
257
struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
260
258
struct io_ring_ctx *ctx = req->ctx;
261
-
struct file *src_file = msg->src_file;
262
259
263
260
if (msg->len)
264
261
return -EINVAL;
···
265
264
return -EINVAL;
266
265
if (target_ctx->flags & IORING_SETUP_R_DISABLED)
267
266
return -EBADFD;
268
-
if (!src_file) {
269
-
src_file = io_msg_grab_file(req, issue_flags);
270
-
if (!src_file)
271
-
return -EBADF;
272
-
msg->src_file = src_file;
273
-
req->flags |= REQ_F_NEED_CLEANUP;
267
+
if (!msg->src_file) {
268
+
int ret = io_msg_grab_file(req, issue_flags);
269
+
if (unlikely(ret))
270
+
return ret;
274
271
}
275
272
276
273
if (io_msg_need_remote(target_ctx))
+2
-4
io_uring/net.c
+2
-4
io_uring/net.c
···
1343
1343
if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
1344
1344
struct io_ring_ctx *ctx = req->ctx;
1345
1345
struct io_rsrc_node *node;
1346
-
int idx;
1347
1346
1348
1347
ret = -EFAULT;
1349
1348
io_ring_submit_lock(ctx, issue_flags);
1350
-
if (sr->buf_index < ctx->buf_table.nr) {
1351
-
idx = array_index_nospec(sr->buf_index, ctx->buf_table.nr);
1352
-
node = ctx->buf_table.nodes[idx];
1349
+
node = io_rsrc_node_lookup(&ctx->buf_table, sr->buf_index);
1350
+
if (node) {
1353
1351
io_req_assign_rsrc_node(sr->notif, node);
1354
1352
ret = 0;
1355
1353
}
+2
-4
io_uring/nop.c
+2
-4
io_uring/nop.c
···
62
62
if (nop->flags & IORING_NOP_FIXED_BUFFER) {
63
63
struct io_ring_ctx *ctx = req->ctx;
64
64
struct io_rsrc_node *node;
65
-
int idx;
66
65
67
66
ret = -EFAULT;
68
67
io_ring_submit_lock(ctx, issue_flags);
69
-
if (nop->buffer < ctx->buf_table.nr) {
70
-
idx = array_index_nospec(nop->buffer, ctx->buf_table.nr);
71
-
node = READ_ONCE(ctx->buf_table.nodes[idx]);
68
+
node = io_rsrc_node_lookup(&ctx->buf_table, nop->buffer);
69
+
if (node) {
72
70
io_req_assign_rsrc_node(req, node);
73
71
ret = 0;
74
72
}
+7
-5
io_uring/rsrc.c
+7
-5
io_uring/rsrc.c
···
181
181
return -EINVAL;
182
182
183
183
for (done = 0; done < nr_args; done++) {
184
+
struct io_rsrc_node *node;
184
185
u64 tag = 0;
185
186
186
187
if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
···
196
195
if (fd == IORING_REGISTER_FILES_SKIP)
197
196
continue;
198
197
199
-
i = array_index_nospec(up->offset + done, ctx->file_table.data.nr);
200
-
if (ctx->file_table.data.nodes[i]) {
201
-
io_put_rsrc_node(ctx->file_table.data.nodes[i]);
198
+
i = up->offset + done;
199
+
node = io_rsrc_node_lookup(&ctx->file_table.data, i);
200
+
if (node) {
201
+
io_put_rsrc_node(node);
202
202
ctx->file_table.data.nodes[i] = NULL;
203
203
io_file_bitmap_clear(&ctx->file_table, i);
204
204
}
···
960
958
goto out_unlock;
961
959
962
960
for (i = 0; i < nbufs; i++) {
963
-
struct io_rsrc_node *src_node = src_ctx->buf_table.nodes[i];
964
-
struct io_rsrc_node *dst_node;
961
+
struct io_rsrc_node *dst_node, *src_node;
965
962
963
+
src_node = io_rsrc_node_lookup(&src_ctx->buf_table, i);
966
964
if (src_node == rsrc_empty_node) {
967
965
dst_node = rsrc_empty_node;
968
966
} else {
+8
io_uring/rsrc.h
+8
io_uring/rsrc.h
···
70
70
extern const struct io_rsrc_node empty_node;
71
71
#define rsrc_empty_node (struct io_rsrc_node *) &empty_node
72
72
73
+
static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data,
74
+
int index)
75
+
{
76
+
if (index < data->nr)
77
+
return data->nodes[array_index_nospec(index, data->nr)];
78
+
return NULL;
79
+
}
80
+
73
81
static inline void io_put_rsrc_node(struct io_rsrc_node *node)
74
82
{
75
83
if (node != rsrc_empty_node && !--node->refs)
+2
-4
io_uring/rw.c
+2
-4
io_uring/rw.c
···
332
332
struct io_ring_ctx *ctx = req->ctx;
333
333
struct io_rsrc_node *node;
334
334
struct io_async_rw *io;
335
-
u16 index;
336
335
int ret;
337
336
338
337
ret = io_prep_rw(req, sqe, ddir, false);
339
338
if (unlikely(ret))
340
339
return ret;
341
340
342
-
if (unlikely(req->buf_index >= ctx->buf_table.nr))
341
+
node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index);
342
+
if (!node)
343
343
return -EFAULT;
344
-
index = array_index_nospec(req->buf_index, ctx->buf_table.nr);
345
-
node = ctx->buf_table.nodes[index];
346
344
io_req_assign_rsrc_node(req, node);
347
345
348
346
io = req->async_data;
+1
-5
io_uring/splice.c
+1
-5
io_uring/splice.c
···
66
66
return io_file_get_normal(req, sp->splice_fd_in);
67
67
68
68
io_ring_submit_lock(ctx, issue_flags);
69
-
if (unlikely(sp->splice_fd_in >= ctx->file_table.data.nr))
70
-
goto out;
71
-
sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->file_table.data.nr);
72
-
node = ctx->file_table.data.nodes[sp->splice_fd_in];
69
+
node = io_rsrc_node_lookup(&ctx->file_table.data, sp->splice_fd_in);
73
70
if (node) {
74
71
node->refs++;
75
72
sp->rsrc_node = node;
76
73
file = io_slot_file(node);
77
74
req->flags |= REQ_F_NEED_CLEANUP;
78
75
}
79
-
out:
80
76
io_ring_submit_unlock(ctx, issue_flags);
81
77
return file;
82
78
}
+4
-5
io_uring/uring_cmd.c
+4
-5
io_uring/uring_cmd.c
···
209
209
210
210
if (ioucmd->flags & IORING_URING_CMD_FIXED) {
211
211
struct io_ring_ctx *ctx = req->ctx;
212
-
u16 index;
212
+
struct io_rsrc_node *node;
213
213
214
-
index = READ_ONCE(sqe->buf_index);
215
-
if (unlikely(index >= ctx->buf_table.nr))
214
+
node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index);
215
+
if (unlikely(!node))
216
216
return -EFAULT;
217
-
req->buf_index = array_index_nospec(index, ctx->buf_table.nr);
218
217
/*
219
218
* Pi node upfront, prior to io_uring_cmd_import_fixed()
220
219
* being called. This prevents destruction of the mapped buffer
221
220
* we'll need at actual import time.
222
221
*/
223
-
io_req_assign_rsrc_node(req, ctx->buf_table.nodes[req->buf_index]);
222
+
io_req_assign_rsrc_node(req, node);
224
223
}
225
224
ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
226
225