Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/rsrc: unify file and buffer resource tables
For files, there's nr_user_files/file_table/file_data, and buffers have
nr_user_bufs/user_bufs/buf_data. There's no reason why file_table and
file_data can't be the same thing, and ditto for the buffer side. That
gets rid of more io_ring_ctx state that's in two spots rather than just
being in one spot, as it should be. Put all the registered file data in
one locations, and ditto on the buffer front.
This also avoids having both io_rsrc_data->nodes being an allocated
array, and ->user_bufs[] or ->file_table.nodes. There's no reason to
have this information duplicated. Keep it in one spot, io_rsrc_data,
along with how many resources are available.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+123
-212
+7
-8
include/linux/io_uring_types.h
+7
-8
include/linux/io_uring_types.h
···
55
55
int cancel_seq;
56
56
};
57
57
58
+
struct io_rsrc_data {
59
+
unsigned int nr;
60
+
struct io_rsrc_node **nodes;
61
+
};
62
+
58
63
struct io_file_table {
59
-
struct io_rsrc_node **nodes;
64
+
struct io_rsrc_data data;
60
65
unsigned long *bitmap;
61
66
unsigned int alloc_hint;
62
67
};
···
281
276
struct io_wq_work_list iopoll_list;
282
277
283
278
struct io_file_table file_table;
284
-
struct io_rsrc_node **user_bufs;
285
-
unsigned nr_user_files;
286
-
unsigned nr_user_bufs;
279
+
struct io_rsrc_data buf_table;
287
280
288
281
struct io_submit_state submit_state;
289
282
···
368
365
/* Keep this last, we don't need it for the fast path */
369
366
struct wait_queue_head poll_wq;
370
367
struct io_restriction restrictions;
371
-
372
-
/* slow path rsrc auxilary data, used by update/register */
373
-
struct io_rsrc_data *file_data;
374
-
struct io_rsrc_data *buf_data;
375
368
376
369
u32 pers_next;
377
370
struct xarray personalities;
+2
-2
io_uring/cancel.c
+2
-2
io_uring/cancel.c
···
240
240
/* fixed must be grabbed every time since we drop the uring_lock */
241
241
if ((cd->flags & IORING_ASYNC_CANCEL_FD) &&
242
242
(cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
243
-
if (unlikely(fd >= ctx->nr_user_files))
243
+
if (unlikely(fd >= ctx->file_table.data.nr))
244
244
return -EBADF;
245
-
fd = array_index_nospec(fd, ctx->nr_user_files);
245
+
fd = array_index_nospec(fd, ctx->file_table.data.nr);
246
246
cd->file = io_file_from_index(&ctx->file_table, fd);
247
247
if (!cd->file)
248
248
return -EBADF;
+5
-5
io_uring/fdinfo.c
+5
-5
io_uring/fdinfo.c
···
165
165
seq_printf(m, "SqThreadCpu:\t%d\n", sq_cpu);
166
166
seq_printf(m, "SqTotalTime:\t%llu\n", sq_total_time);
167
167
seq_printf(m, "SqWorkTime:\t%llu\n", sq_work_time);
168
-
seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
169
-
for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
168
+
seq_printf(m, "UserFiles:\t%u\n", ctx->file_table.data.nr);
169
+
for (i = 0; has_lock && i < ctx->file_table.data.nr; i++) {
170
170
struct file *f = io_file_from_index(&ctx->file_table, i);
171
171
172
172
if (f)
···
174
174
else
175
175
seq_printf(m, "%5u: <none>\n", i);
176
176
}
177
-
seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
178
-
for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
179
-
struct io_mapped_ubuf *buf = ctx->user_bufs[i]->buf;
177
+
seq_printf(m, "UserBufs:\t%u\n", ctx->buf_table.nr);
178
+
for (i = 0; has_lock && i < ctx->buf_table.nr; i++) {
179
+
struct io_mapped_ubuf *buf = ctx->buf_table.nodes[i]->buf;
180
180
181
181
seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, buf->len);
182
182
}
+19
-25
io_uring/filetable.c
+19
-25
io_uring/filetable.c
···
38
38
39
39
bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
40
40
{
41
-
table->nodes = kvmalloc_array(nr_files, sizeof(struct io_src_node *),
42
-
GFP_KERNEL_ACCOUNT | __GFP_ZERO);
43
-
if (unlikely(!table->nodes))
41
+
if (io_rsrc_data_alloc(&table->data, nr_files))
44
42
return false;
45
-
46
43
table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
47
-
if (unlikely(!table->bitmap)) {
48
-
kvfree(table->nodes);
49
-
return false;
50
-
}
51
-
52
-
return true;
44
+
if (table->bitmap)
45
+
return true;
46
+
io_rsrc_data_free(&table->data);
47
+
return false;
53
48
}
54
49
55
50
void io_free_file_tables(struct io_file_table *table)
56
51
{
57
-
kvfree(table->nodes);
52
+
io_rsrc_data_free(&table->data);
58
53
bitmap_free(table->bitmap);
59
-
table->nodes = NULL;
60
54
table->bitmap = NULL;
61
55
}
62
56
···
62
68
63
69
if (io_is_uring_fops(file))
64
70
return -EBADF;
65
-
if (!ctx->file_data)
71
+
if (!ctx->file_table.data.nr)
66
72
return -ENXIO;
67
-
if (slot_index >= ctx->nr_user_files)
73
+
if (slot_index >= ctx->file_table.data.nr)
68
74
return -EINVAL;
69
75
70
76
node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);
71
77
if (!node)
72
78
return -ENOMEM;
73
79
74
-
slot_index = array_index_nospec(slot_index, ctx->nr_user_files);
75
-
if (ctx->file_table.nodes[slot_index])
76
-
io_put_rsrc_node(ctx->file_table.nodes[slot_index]);
80
+
slot_index = array_index_nospec(slot_index, ctx->file_table.data.nr);
81
+
if (ctx->file_table.data.nodes[slot_index])
82
+
io_put_rsrc_node(ctx->file_table.data.nodes[slot_index]);
77
83
else
78
84
io_file_bitmap_set(&ctx->file_table, slot_index);
79
85
80
-
ctx->file_table.nodes[slot_index] = node;
86
+
ctx->file_table.data.nodes[slot_index] = node;
81
87
io_fixed_file_set(node, file);
82
88
return 0;
83
89
}
···
123
129
124
130
int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
125
131
{
126
-
if (unlikely(!ctx->file_data))
132
+
if (unlikely(!ctx->file_table.data.nr))
127
133
return -ENXIO;
128
-
if (offset >= ctx->nr_user_files)
134
+
if (offset >= ctx->file_table.data.nr)
129
135
return -EINVAL;
130
136
131
-
offset = array_index_nospec(offset, ctx->nr_user_files);
132
-
if (!ctx->file_table.nodes[offset])
137
+
offset = array_index_nospec(offset, ctx->file_table.data.nr);
138
+
if (!ctx->file_table.data.nodes[offset])
133
139
return -EBADF;
134
-
io_put_rsrc_node(ctx->file_table.nodes[offset]);
135
-
ctx->file_table.nodes[offset] = NULL;
140
+
io_put_rsrc_node(ctx->file_table.data.nodes[offset]);
141
+
ctx->file_table.data.nodes[offset] = NULL;
136
142
io_file_bitmap_clear(&ctx->file_table, offset);
137
143
return 0;
138
144
}
···
147
153
return -EFAULT;
148
154
if (check_add_overflow(range.off, range.len, &end))
149
155
return -EOVERFLOW;
150
-
if (range.resv || end > ctx->nr_user_files)
156
+
if (range.resv || end > ctx->file_table.data.nr)
151
157
return -EINVAL;
152
158
153
159
io_file_table_set_alloc_range(ctx, range.off, range.len);
+1
-1
io_uring/filetable.h
+1
-1
io_uring/filetable.h
+3
-4
io_uring/io_uring.c
+3
-4
io_uring/io_uring.c
···
1879
1879
struct file *file = NULL;
1880
1880
1881
1881
io_ring_submit_lock(ctx, issue_flags);
1882
-
1883
-
if (unlikely((unsigned int)fd >= ctx->nr_user_files))
1882
+
if (unlikely((unsigned int)fd >= ctx->file_table.data.nr))
1884
1883
goto out;
1885
-
fd = array_index_nospec(fd, ctx->nr_user_files);
1886
-
node = ctx->file_table.nodes[fd];
1884
+
fd = array_index_nospec(fd, ctx->file_table.data.nr);
1885
+
node = ctx->file_table.data.nodes[fd];
1887
1886
if (node) {
1888
1887
io_req_assign_rsrc_node(req, node);
1889
1888
req->flags |= io_slot_flags(node);
+2
-2
io_uring/msg_ring.c
+2
-2
io_uring/msg_ring.c
···
180
180
int idx = msg->src_fd;
181
181
182
182
io_ring_submit_lock(ctx, issue_flags);
183
-
if (likely(idx < ctx->nr_user_files)) {
184
-
idx = array_index_nospec(idx, ctx->nr_user_files);
183
+
if (likely(idx < ctx->file_table.data.nr)) {
184
+
idx = array_index_nospec(idx, ctx->file_table.data.nr);
185
185
file = io_file_from_index(&ctx->file_table, idx);
186
186
if (file)
187
187
get_file(file);
+3
-3
io_uring/net.c
+3
-3
io_uring/net.c
···
1347
1347
1348
1348
ret = -EFAULT;
1349
1349
io_ring_submit_lock(ctx, issue_flags);
1350
-
if (sr->buf_index < ctx->nr_user_bufs) {
1351
-
idx = array_index_nospec(sr->buf_index, ctx->nr_user_bufs);
1352
-
node = ctx->user_bufs[idx];
1350
+
if (sr->buf_index < ctx->buf_table.nr) {
1351
+
idx = array_index_nospec(sr->buf_index, ctx->buf_table.nr);
1352
+
node = ctx->buf_table.nodes[idx];
1353
1353
io_req_assign_rsrc_node(sr->notif, node);
1354
1354
ret = 0;
1355
1355
}
+3
-3
io_uring/nop.c
+3
-3
io_uring/nop.c
···
66
66
67
67
ret = -EFAULT;
68
68
io_ring_submit_lock(ctx, issue_flags);
69
-
if (nop->buffer < ctx->nr_user_bufs) {
70
-
idx = array_index_nospec(nop->buffer, ctx->nr_user_bufs);
71
-
node = READ_ONCE(ctx->user_bufs[idx]);
69
+
if (nop->buffer < ctx->buf_table.nr) {
70
+
idx = array_index_nospec(nop->buffer, ctx->buf_table.nr);
71
+
node = READ_ONCE(ctx->buf_table.nodes[idx]);
72
72
io_req_assign_rsrc_node(req, node);
73
73
ret = 0;
74
74
}
+2
-1
io_uring/register.c
+2
-1
io_uring/register.c
···
937
937
mutex_lock(&ctx->uring_lock);
938
938
ret = __io_uring_register(ctx, opcode, arg, nr_args);
939
939
mutex_unlock(&ctx->uring_lock);
940
-
trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ret);
940
+
trace_io_uring_register(ctx, opcode, ctx->file_table.data.nr,
941
+
ctx->buf_table.nr, ret);
941
942
if (!use_registered_ring)
942
943
fput(file);
943
944
return ret;
+65
-144
io_uring/rsrc.c
+65
-144
io_uring/rsrc.c
···
142
142
return node;
143
143
}
144
144
145
-
static void io_rsrc_data_free(struct io_rsrc_data *data)
145
+
__cold void io_rsrc_data_free(struct io_rsrc_data *data)
146
146
{
147
-
int i;
148
-
149
-
for (i = 0; i < data->nr; i++) {
150
-
struct io_rsrc_node *node = data->nodes[i];
151
-
152
-
if (node)
153
-
io_put_rsrc_node(node);
147
+
if (!data->nr)
148
+
return;
149
+
while (data->nr--) {
150
+
if (data->nodes[data->nr])
151
+
io_put_rsrc_node(data->nodes[data->nr]);
154
152
}
155
153
kvfree(data->nodes);
156
-
kfree(data);
154
+
data->nodes = NULL;
155
+
data->nr = 0;
157
156
}
158
157
159
-
__cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, unsigned nr,
160
-
struct io_rsrc_data **pdata)
158
+
__cold int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr)
161
159
{
162
-
struct io_rsrc_data *data;
163
-
164
-
data = kzalloc(sizeof(*data), GFP_KERNEL);
165
-
if (!data)
166
-
return -ENOMEM;
167
-
168
160
data->nodes = kvmalloc_array(nr, sizeof(struct io_rsrc_node *),
169
-
GFP_KERNEL | __GFP_ZERO);
170
-
if (!data->nodes) {
171
-
kfree(data);
172
-
return -ENOMEM;
161
+
GFP_KERNEL_ACCOUNT | __GFP_ZERO);
162
+
if (data->nodes) {
163
+
data->nr = nr;
164
+
return 0;
173
165
}
174
-
175
-
data->nr = nr;
176
-
*pdata = data;
177
-
return 0;
166
+
return -ENOMEM;
178
167
}
179
168
180
169
static int __io_sqe_files_update(struct io_ring_ctx *ctx,
···
175
186
int fd, i, err = 0;
176
187
unsigned int done;
177
188
178
-
if (!ctx->file_data)
189
+
if (!ctx->file_table.data.nr)
179
190
return -ENXIO;
180
-
if (up->offset + nr_args > ctx->nr_user_files)
191
+
if (up->offset + nr_args > ctx->file_table.data.nr)
181
192
return -EINVAL;
182
193
183
194
for (done = 0; done < nr_args; done++) {
···
195
206
if (fd == IORING_REGISTER_FILES_SKIP)
196
207
continue;
197
208
198
-
i = array_index_nospec(up->offset + done, ctx->nr_user_files);
199
-
if (ctx->file_table.nodes[i]) {
200
-
io_put_rsrc_node(ctx->file_table.nodes[i]);
201
-
ctx->file_table.nodes[i] = NULL;
209
+
i = array_index_nospec(up->offset + done, ctx->file_table.data.nr);
210
+
if (ctx->file_table.data.nodes[i]) {
211
+
io_put_rsrc_node(ctx->file_table.data.nodes[i]);
212
+
ctx->file_table.data.nodes[i] = NULL;
202
213
io_file_bitmap_clear(&ctx->file_table, i);
203
214
}
204
215
if (fd != -1) {
···
223
234
fput(file);
224
235
break;
225
236
}
226
-
ctx->file_table.nodes[i] = node;
237
+
ctx->file_table.data.nodes[i] = node;
227
238
if (tag)
228
239
node->tag = tag;
229
240
io_fixed_file_set(node, file);
···
245
256
__u32 done;
246
257
int i, err;
247
258
248
-
if (!ctx->buf_data)
259
+
if (!ctx->buf_table.nr)
249
260
return -ENXIO;
250
-
if (up->offset + nr_args > ctx->nr_user_bufs)
261
+
if (up->offset + nr_args > ctx->buf_table.nr)
251
262
return -EINVAL;
252
263
253
264
for (done = 0; done < nr_args; done++) {
···
271
282
err = -EINVAL;
272
283
break;
273
284
}
274
-
i = array_index_nospec(up->offset + done, ctx->nr_user_bufs);
275
285
node = io_sqe_buffer_register(ctx, iov, &last_hpage);
276
286
if (IS_ERR(node)) {
277
287
err = PTR_ERR(node);
278
288
break;
279
289
}
280
-
if (ctx->user_bufs[i])
281
-
io_put_rsrc_node(ctx->user_bufs[i]);
290
+
i = array_index_nospec(up->offset + done, ctx->buf_table.nr);
291
+
if (ctx->buf_table.nodes[i])
292
+
io_put_rsrc_node(ctx->buf_table.nodes[i]);
282
293
283
-
ctx->user_bufs[i] = node;
294
+
ctx->buf_table.nodes[i] = node;
284
295
if (tag)
285
296
node->tag = tag;
286
297
if (ctx->compat)
···
398
409
struct file *file;
399
410
int ret, fd;
400
411
401
-
if (!req->ctx->file_data)
412
+
if (!req->ctx->file_table.data.nr)
402
413
return -ENXIO;
403
414
404
415
for (done = 0; done < up->nr_args; done++) {
···
483
494
kfree(node);
484
495
}
485
496
486
-
static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
497
+
int io_sqe_files_unregister(struct io_ring_ctx *ctx)
487
498
{
488
-
int i;
489
-
490
-
lockdep_assert_held(&ctx->uring_lock);
491
-
492
-
for (i = 0; i < ctx->nr_user_files; i++) {
493
-
struct io_rsrc_node *node = ctx->file_table.nodes[i];
494
-
495
-
if (node) {
496
-
io_put_rsrc_node(node);
497
-
io_file_bitmap_clear(&ctx->file_table, i);
498
-
ctx->file_table.nodes[i] = NULL;
499
-
}
500
-
}
499
+
if (!ctx->file_table.data.nr)
500
+
return -ENXIO;
501
501
502
502
io_free_file_tables(&ctx->file_table);
503
503
io_file_table_set_alloc_range(ctx, 0, 0);
504
-
io_rsrc_data_free(ctx->file_data);
505
-
ctx->file_data = NULL;
506
-
ctx->nr_user_files = 0;
507
-
}
508
-
509
-
int io_sqe_files_unregister(struct io_ring_ctx *ctx)
510
-
{
511
-
if (!ctx->file_data)
512
-
return -ENXIO;
513
-
514
-
__io_sqe_files_unregister(ctx);
515
504
return 0;
516
505
}
517
506
···
501
534
int fd, ret;
502
535
unsigned i;
503
536
504
-
if (ctx->file_data)
537
+
if (ctx->file_table.data.nr)
505
538
return -EBUSY;
506
539
if (!nr_args)
507
540
return -EINVAL;
···
509
542
return -EMFILE;
510
543
if (nr_args > rlimit(RLIMIT_NOFILE))
511
544
return -EMFILE;
512
-
ret = io_rsrc_data_alloc(ctx, nr_args, &ctx->file_data);
513
-
if (ret)
514
-
return ret;
515
-
516
-
if (!io_alloc_file_tables(&ctx->file_table, nr_args)) {
517
-
io_rsrc_data_free(ctx->file_data);
518
-
ctx->file_data = NULL;
545
+
if (!io_alloc_file_tables(&ctx->file_table, nr_args))
519
546
return -ENOMEM;
520
-
}
521
547
522
-
for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
548
+
for (i = 0; i < nr_args; i++) {
523
549
struct io_rsrc_node *node;
524
550
u64 tag = 0;
525
551
···
549
589
}
550
590
if (tag)
551
591
node->tag = tag;
552
-
ctx->file_table.nodes[i] = node;
592
+
ctx->file_table.data.nodes[i] = node;
553
593
io_fixed_file_set(node, file);
554
594
io_file_bitmap_set(&ctx->file_table, i);
555
595
}
556
596
557
597
/* default it to the whole table */
558
-
io_file_table_set_alloc_range(ctx, 0, ctx->nr_user_files);
598
+
io_file_table_set_alloc_range(ctx, 0, ctx->file_table.data.nr);
559
599
return 0;
560
600
fail:
561
-
__io_sqe_files_unregister(ctx);
601
+
io_sqe_files_unregister(ctx);
562
602
return ret;
563
-
}
564
-
565
-
static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
566
-
{
567
-
unsigned int i;
568
-
569
-
lockdep_assert_held(&ctx->uring_lock);
570
-
571
-
for (i = 0; i < ctx->nr_user_bufs; i++) {
572
-
if (ctx->user_bufs[i]) {
573
-
io_put_rsrc_node(ctx->user_bufs[i]);
574
-
ctx->user_bufs[i] = NULL;
575
-
}
576
-
}
577
-
kvfree(ctx->user_bufs);
578
-
ctx->user_bufs = NULL;
579
-
io_rsrc_data_free(ctx->buf_data);
580
-
ctx->buf_data = NULL;
581
-
ctx->nr_user_bufs = 0;
582
603
}
583
604
584
605
int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
585
606
{
586
-
if (!ctx->buf_data)
607
+
if (!ctx->buf_table.nr)
587
608
return -ENXIO;
588
-
589
-
__io_sqe_buffers_unregister(ctx);
609
+
io_rsrc_data_free(&ctx->buf_table);
590
610
return 0;
591
611
}
592
612
···
593
653
}
594
654
595
655
/* check previously registered pages */
596
-
for (i = 0; i < ctx->nr_user_bufs; i++) {
597
-
struct io_rsrc_node *node = ctx->user_bufs[i];
656
+
for (i = 0; i < ctx->buf_table.nr; i++) {
657
+
struct io_rsrc_node *node = ctx->buf_table.nodes[i];
598
658
struct io_mapped_ubuf *imu = node->buf;
599
659
600
660
for (j = 0; j < imu->nr_bvecs; j++) {
···
745
805
struct io_imu_folio_data data;
746
806
bool coalesced;
747
807
808
+
if (!iov->iov_base)
809
+
return rsrc_empty_node;
810
+
748
811
node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
749
812
if (!node)
750
813
return ERR_PTR(-ENOMEM);
···
807
864
return node;
808
865
}
809
866
810
-
static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
811
-
{
812
-
ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL);
813
-
return ctx->user_bufs ? 0 : -ENOMEM;
814
-
}
815
-
816
867
int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
817
868
unsigned int nr_args, u64 __user *tags)
818
869
{
819
870
struct page *last_hpage = NULL;
820
-
struct io_rsrc_data *data;
871
+
struct io_rsrc_data data;
821
872
struct iovec fast_iov, *iov = &fast_iov;
822
873
const struct iovec __user *uvec;
823
874
int i, ret;
824
875
825
876
BUILD_BUG_ON(IORING_MAX_REG_BUFFERS >= (1u << 16));
826
877
827
-
if (ctx->user_bufs)
878
+
if (ctx->buf_table.nr)
828
879
return -EBUSY;
829
880
if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
830
881
return -EINVAL;
831
-
ret = io_rsrc_data_alloc(ctx, nr_args, &data);
882
+
ret = io_rsrc_data_alloc(&data, nr_args);
832
883
if (ret)
833
884
return ret;
834
-
ret = io_buffers_map_alloc(ctx, nr_args);
835
-
if (ret) {
836
-
io_rsrc_data_free(data);
837
-
return ret;
838
-
}
839
885
840
886
if (!arg)
841
887
memset(iov, 0, sizeof(*iov));
842
888
843
-
for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
889
+
for (i = 0; i < nr_args; i++) {
844
890
struct io_rsrc_node *node;
845
891
u64 tag = 0;
846
892
···
867
935
}
868
936
if (tag)
869
937
node->tag = tag;
870
-
ctx->user_bufs[i] = node;
938
+
data.nodes[i] = node;
871
939
}
872
940
873
-
WARN_ON_ONCE(ctx->buf_data);
874
-
875
-
ctx->buf_data = data;
941
+
ctx->buf_table = data;
876
942
if (ret)
877
-
__io_sqe_buffers_unregister(ctx);
943
+
io_sqe_buffers_unregister(ctx);
878
944
return ret;
879
945
}
880
946
···
939
1009
940
1010
static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx)
941
1011
{
942
-
struct io_rsrc_node **user_bufs;
943
-
struct io_rsrc_data *data;
1012
+
struct io_rsrc_data data;
944
1013
int i, ret, nbufs;
945
1014
946
1015
/*
···
950
1021
951
1022
mutex_lock(&src_ctx->uring_lock);
952
1023
ret = -ENXIO;
953
-
nbufs = src_ctx->nr_user_bufs;
1024
+
nbufs = src_ctx->buf_table.nr;
954
1025
if (!nbufs)
955
1026
goto out_unlock;
956
-
ret = io_rsrc_data_alloc(ctx, nbufs, &data);
1027
+
ret = io_rsrc_data_alloc(&data, nbufs);
957
1028
if (ret)
958
1029
goto out_unlock;
959
1030
960
-
ret = -ENOMEM;
961
-
user_bufs = kvmalloc_array(nbufs, sizeof(struct io_rsrc_node *),
962
-
GFP_KERNEL | __GFP_ZERO);
963
-
if (!user_bufs)
964
-
goto out_free_data;
965
-
966
1031
for (i = 0; i < nbufs; i++) {
967
-
struct io_rsrc_node *src_node = src_ctx->user_bufs[i];
1032
+
struct io_rsrc_node *src_node = src_ctx->buf_table.nodes[i];
968
1033
struct io_rsrc_node *dst_node;
969
1034
970
1035
if (src_node == rsrc_empty_node) {
971
1036
dst_node = rsrc_empty_node;
972
1037
} else {
973
1038
dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
974
-
if (!dst_node)
1039
+
if (!dst_node) {
1040
+
ret = -ENOMEM;
975
1041
goto out_put_free;
1042
+
}
976
1043
977
1044
refcount_inc(&src_node->buf->refs);
978
1045
dst_node->buf = src_node->buf;
979
1046
}
980
-
user_bufs[i] = dst_node;
1047
+
data.nodes[i] = dst_node;
981
1048
}
982
1049
983
1050
/* Have a ref on the bufs now, drop src lock and re-grab our own lock */
984
1051
mutex_unlock(&src_ctx->uring_lock);
985
1052
mutex_lock(&ctx->uring_lock);
986
-
if (!ctx->user_bufs) {
987
-
ctx->user_bufs = user_bufs;
988
-
ctx->buf_data = data;
989
-
ctx->nr_user_bufs = nbufs;
1053
+
if (!ctx->buf_table.nr) {
1054
+
ctx->buf_table = data;
990
1055
return 0;
991
1056
}
992
1057
···
991
1068
i = nbufs;
992
1069
out_put_free:
993
1070
while (i--) {
994
-
io_buffer_unmap(src_ctx, user_bufs[i]);
995
-
kfree(user_bufs[i]);
1071
+
io_buffer_unmap(src_ctx, data.nodes[i]);
1072
+
kfree(data.nodes[i]);
996
1073
}
997
-
kvfree(user_bufs);
998
-
out_free_data:
999
-
io_rsrc_data_free(data);
1074
+
io_rsrc_data_free(&data);
1000
1075
out_unlock:
1001
1076
mutex_unlock(&src_ctx->uring_lock);
1002
1077
mutex_lock(&ctx->uring_lock);
···
1015
1094
struct file *file;
1016
1095
int ret;
1017
1096
1018
-
if (ctx->user_bufs || ctx->nr_user_bufs)
1097
+
if (ctx->buf_table.nr)
1019
1098
return -EBUSY;
1020
1099
if (copy_from_user(&buf, arg, sizeof(buf)))
1021
1100
return -EFAULT;
+2
-5
io_uring/rsrc.h
+2
-5
io_uring/rsrc.h
···
13
13
IORING_RSRC_BUFFER = 1,
14
14
};
15
15
16
-
struct io_rsrc_data {
17
-
unsigned int nr;
18
-
struct io_rsrc_node **nodes;
19
-
};
20
-
21
16
struct io_rsrc_node {
22
17
struct io_ring_ctx *ctx;
23
18
int refs;
···
45
50
46
51
struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type);
47
52
void io_free_rsrc_node(struct io_rsrc_node *node);
53
+
void io_rsrc_data_free(struct io_rsrc_data *data);
54
+
int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr);
48
55
49
56
int io_import_fixed(int ddir, struct iov_iter *iter,
50
57
struct io_mapped_ubuf *imu,
+3
-3
io_uring/rw.c
+3
-3
io_uring/rw.c
···
339
339
if (unlikely(ret))
340
340
return ret;
341
341
342
-
if (unlikely(req->buf_index >= ctx->nr_user_bufs))
342
+
if (unlikely(req->buf_index >= ctx->buf_table.nr))
343
343
return -EFAULT;
344
-
index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);
345
-
node = ctx->user_bufs[index];
344
+
index = array_index_nospec(req->buf_index, ctx->buf_table.nr);
345
+
node = ctx->buf_table.nodes[index];
346
346
io_req_assign_rsrc_node(req, node);
347
347
348
348
io = req->async_data;
+3
-3
io_uring/splice.c
+3
-3
io_uring/splice.c
···
66
66
return io_file_get_normal(req, sp->splice_fd_in);
67
67
68
68
io_ring_submit_lock(ctx, issue_flags);
69
-
if (unlikely(sp->splice_fd_in >= ctx->nr_user_files))
69
+
if (unlikely(sp->splice_fd_in >= ctx->file_table.data.nr))
70
70
goto out;
71
-
sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->nr_user_files);
72
-
node = ctx->file_table.nodes[sp->splice_fd_in];
71
+
sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->file_table.data.nr);
72
+
node = ctx->file_table.data.nodes[sp->splice_fd_in];
73
73
if (node) {
74
74
node->refs++;
75
75
sp->rsrc_node = node;
+3
-3
io_uring/uring_cmd.c
+3
-3
io_uring/uring_cmd.c
···
212
212
u16 index;
213
213
214
214
index = READ_ONCE(sqe->buf_index);
215
-
if (unlikely(index >= ctx->nr_user_bufs))
215
+
if (unlikely(index >= ctx->buf_table.nr))
216
216
return -EFAULT;
217
-
req->buf_index = array_index_nospec(index, ctx->nr_user_bufs);
217
+
req->buf_index = array_index_nospec(index, ctx->buf_table.nr);
218
218
/*
219
219
* Pi node upfront, prior to io_uring_cmd_import_fixed()
220
220
* being called. This prevents destruction of the mapped buffer
221
221
* we'll need at actual import time.
222
222
*/
223
-
io_req_assign_rsrc_node(req, ctx->user_bufs[req->buf_index]);
223
+
io_req_assign_rsrc_node(req, ctx->buf_table.nodes[req->buf_index]);
224
224
}
225
225
ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
226
226