Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/rsrc: add an empty io_rsrc_node for sparse buffer entries
Rather than allocate an io_rsrc_node for an empty/sparse buffer entry,
add a const entry that can be used for that. This just needs checking
for writing the tag, and the put check needs to check for that sparse
node rather than NULL for validity.
This avoids allocating rsrc nodes for sparse buffer entries.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+41
-29
+2
-2
io_uring/io_uring.c
+2
-2
io_uring/io_uring.c
···
2032
2032
req->flags = (__force io_req_flags_t) sqe_flags;
2033
2033
req->cqe.user_data = READ_ONCE(sqe->user_data);
2034
2034
req->file = NULL;
2035
-
req->rsrc_nodes[IORING_RSRC_FILE] = NULL;
2036
-
req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
2035
+
req->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node;
2036
+
req->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node;
2037
2037
req->task = current;
2038
2038
req->cancel_seq_set = false;
2039
2039
+2
-2
io_uring/notif.c
+2
-2
io_uring/notif.c
···
117
117
notif->file = NULL;
118
118
notif->task = current;
119
119
io_get_task_refs(1);
120
-
notif->rsrc_nodes[IORING_RSRC_FILE] = NULL;
121
-
notif->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
120
+
notif->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node;
121
+
notif->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node;
122
122
123
123
nd = io_notif_to_data(notif);
124
124
nd->zc_report = false;
+28
-21
io_uring/rsrc.c
+28
-21
io_uring/rsrc.c
···
38
38
.len = UINT_MAX,
39
39
};
40
40
41
+
const struct io_rsrc_node empty_node = {
42
+
.type = IORING_RSRC_BUFFER,
43
+
.buf = (struct io_mapped_ubuf *) &dummy_ubuf,
44
+
};
45
+
41
46
int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
42
47
{
43
48
unsigned long page_limit, cur_pages, new_pages;
···
149
144
for (i = 0; i < data->nr; i++) {
150
145
struct io_rsrc_node *node = data->nodes[i];
151
146
152
-
io_put_rsrc_node(node);
147
+
if (node)
148
+
io_put_rsrc_node(node);
153
149
}
154
150
kvfree(data->nodes);
155
151
kfree(data);
···
235
229
break;
236
230
}
237
231
ctx->file_table.nodes[i] = node;
238
-
node->tag = tag;
232
+
if (tag)
233
+
node->tag = tag;
239
234
io_fixed_file_set(node, file);
240
235
io_file_bitmap_set(&ctx->file_table, i);
241
236
}
···
288
281
err = PTR_ERR(node);
289
282
break;
290
283
}
291
-
io_put_rsrc_node(ctx->user_bufs[i]);
284
+
if (ctx->user_bufs[i])
285
+
io_put_rsrc_node(ctx->user_bufs[i]);
292
286
293
287
ctx->user_bufs[i] = node;
294
-
node->tag = tag;
288
+
if (tag)
289
+
node->tag = tag;
295
290
if (ctx->compat)
296
291
user_data += sizeof(struct compat_iovec);
297
292
else
···
609
600
lockdep_assert_held(&ctx->uring_lock);
610
601
611
602
for (i = 0; i < ctx->nr_user_bufs; i++) {
612
-
io_put_rsrc_node(ctx->user_bufs[i]);
613
-
ctx->user_bufs[i] = NULL;
603
+
if (ctx->user_bufs[i]) {
604
+
io_put_rsrc_node(ctx->user_bufs[i]);
605
+
ctx->user_bufs[i] = NULL;
606
+
}
614
607
}
615
608
kvfree(ctx->user_bufs);
616
609
ctx->user_bufs = NULL;
···
810
799
return ERR_PTR(-ENOMEM);
811
800
node->buf = NULL;
812
801
813
-
if (!iov->iov_base) {
814
-
node->buf = (struct io_mapped_ubuf *) &dummy_ubuf;
815
-
return node;
816
-
}
817
-
818
802
ret = -ENOMEM;
819
803
pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len,
820
804
&nr_pages);
···
933
927
ret = PTR_ERR(node);
934
928
break;
935
929
}
936
-
node->tag = tag;
930
+
if (tag)
931
+
node->tag = tag;
937
932
ctx->user_bufs[i] = node;
938
933
}
939
934
···
1035
1028
goto out_free_data;
1036
1029
1037
1030
for (i = 0; i < nbufs; i++) {
1038
-
struct io_mapped_ubuf *imu = src_ctx->user_bufs[i]->buf;
1031
+
struct io_rsrc_node *src_node = src_ctx->user_bufs[i];
1039
1032
struct io_rsrc_node *dst_node;
1040
1033
1041
-
dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
1042
-
if (!dst_node)
1043
-
goto out_put_free;
1044
-
1045
-
if (imu == &dummy_ubuf) {
1046
-
dst_node->buf = (struct io_mapped_ubuf *) &dummy_ubuf;
1034
+
if (src_node == rsrc_empty_node) {
1035
+
dst_node = rsrc_empty_node;
1047
1036
} else {
1048
-
refcount_inc(&imu->refs);
1049
-
dst_node->buf = imu;
1037
+
dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
1038
+
if (!dst_node)
1039
+
goto out_put_free;
1040
+
1041
+
refcount_inc(&src_node->buf->refs);
1042
+
dst_node->buf = src_node->buf;
1050
1043
}
1051
1044
user_bufs[i] = dst_node;
1052
1045
}
+8
-3
io_uring/rsrc.h
+8
-3
io_uring/rsrc.h
···
70
70
int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
71
71
unsigned int size, unsigned int type);
72
72
73
+
extern const struct io_rsrc_node empty_node;
74
+
#define rsrc_empty_node (struct io_rsrc_node *) &empty_node
75
+
73
76
static inline void io_put_rsrc_node(struct io_rsrc_node *node)
74
77
{
75
-
if (node && !--node->refs)
78
+
if (node != rsrc_empty_node && !--node->refs)
76
79
io_free_rsrc_node(node);
77
80
}
78
81
···
88
85
static inline void io_req_assign_rsrc_node(struct io_kiocb *req,
89
86
struct io_rsrc_node *node)
90
87
{
91
-
node->refs++;
92
-
req->rsrc_nodes[node->type] = node;
88
+
if (node != rsrc_empty_node) {
89
+
node->refs++;
90
+
req->rsrc_nodes[node->type] = node;
91
+
}
93
92
}
94
93
95
94
int io_files_update(struct io_kiocb *req, unsigned int issue_flags);