Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/kbuf: introduce struct io_br_sel
Rather than return addresses directly from buffer selection, add a
struct around it. No functional changes in this patch, it's in
preparation for storing more buffer related information locally, rather
than in struct io_kiocb.
Link: https://lore.kernel.org/r/20250821020750.598432-7-axboe@kernel.dk
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+59
-38
+13
-13
io_uring/kbuf.c
+13
-13
io_uring/kbuf.c
···
151
151
return 1;
152
152
}
153
153
154
-
static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
155
-
struct io_buffer_list *bl,
156
-
unsigned int issue_flags)
154
+
static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len,
155
+
struct io_buffer_list *bl,
156
+
unsigned int issue_flags)
157
157
{
158
158
struct io_uring_buf_ring *br = bl->buf_ring;
159
159
__u16 tail, head = bl->head;
160
+
struct io_br_sel sel = { };
160
161
struct io_uring_buf *buf;
161
-
void __user *ret;
162
162
163
163
tail = smp_load_acquire(&br->tail);
164
164
if (unlikely(tail == head))
165
-
return NULL;
165
+
return sel;
166
166
167
167
if (head + 1 == tail)
168
168
req->flags |= REQ_F_BL_EMPTY;
···
173
173
req->flags |= REQ_F_BUFFER_RING | REQ_F_BUFFERS_COMMIT;
174
174
req->buf_list = bl;
175
175
req->buf_index = buf->bid;
176
-
ret = u64_to_user_ptr(buf->addr);
176
+
sel.addr = u64_to_user_ptr(buf->addr);
177
177
178
178
if (issue_flags & IO_URING_F_UNLOCKED || !io_file_can_poll(req)) {
179
179
/*
···
189
189
io_kbuf_commit(req, bl, *len, 1);
190
190
req->buf_list = NULL;
191
191
}
192
-
return ret;
192
+
return sel;
193
193
}
194
194
195
-
void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
196
-
unsigned buf_group, unsigned int issue_flags)
195
+
struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len,
196
+
unsigned buf_group, unsigned int issue_flags)
197
197
{
198
198
struct io_ring_ctx *ctx = req->ctx;
199
+
struct io_br_sel sel = { };
199
200
struct io_buffer_list *bl;
200
-
void __user *ret = NULL;
201
201
202
202
io_ring_submit_lock(req->ctx, issue_flags);
203
203
204
204
bl = io_buffer_get_list(ctx, buf_group);
205
205
if (likely(bl)) {
206
206
if (bl->flags & IOBL_BUF_RING)
207
-
ret = io_ring_buffer_select(req, len, bl, issue_flags);
207
+
sel = io_ring_buffer_select(req, len, bl, issue_flags);
208
208
else
209
-
ret = io_provided_buffer_select(req, len, bl);
209
+
sel.addr = io_provided_buffer_select(req, len, bl);
210
210
}
211
211
io_ring_submit_unlock(req->ctx, issue_flags);
212
-
return ret;
212
+
return sel;
213
213
}
214
214
215
215
/* cap it at a reasonable 256, will be one page even for 4K */
+17
-2
io_uring/kbuf.h
+17
-2
io_uring/kbuf.h
···
62
62
unsigned short partial_map;
63
63
};
64
64
65
-
void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
66
-
unsigned buf_group, unsigned int issue_flags);
65
+
/*
66
+
* Return value from io_buffer_list selection. Just returns the error or
67
+
* user address for now, will be extended to return the buffer list in the
68
+
* future.
69
+
*/
70
+
struct io_br_sel {
71
+
/*
72
+
* Some selection parts return the user address, others return an error.
73
+
*/
74
+
union {
75
+
void __user *addr;
76
+
ssize_t val;
77
+
};
78
+
};
79
+
80
+
struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len,
81
+
unsigned buf_group, unsigned int issue_flags);
67
82
int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,
68
83
unsigned int issue_flags);
69
84
int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg);
+9
-9
io_uring/net.c
+9
-9
io_uring/net.c
···
1035
1035
1036
1036
retry_multishot:
1037
1037
if (io_do_buffer_select(req)) {
1038
-
void __user *buf;
1038
+
struct io_br_sel sel;
1039
1039
size_t len = sr->len;
1040
1040
1041
-
buf = io_buffer_select(req, &len, sr->buf_group, issue_flags);
1042
-
if (!buf)
1041
+
sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
1042
+
if (!sel.addr)
1043
1043
return -ENOBUFS;
1044
1044
1045
1045
if (req->flags & REQ_F_APOLL_MULTISHOT) {
1046
-
ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len);
1046
+
ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len);
1047
1047
if (ret) {
1048
1048
io_kbuf_recycle(req, req->buf_list, issue_flags);
1049
1049
return ret;
1050
1050
}
1051
1051
}
1052
1052
1053
-
iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, buf, len);
1053
+
iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len);
1054
1054
}
1055
1055
1056
1056
kmsg->msg.msg_get_inq = 1;
···
1153
1153
iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,
1154
1154
arg.out_len);
1155
1155
} else {
1156
-
void __user *buf;
1156
+
struct io_br_sel sel;
1157
1157
1158
1158
*len = sr->len;
1159
-
buf = io_buffer_select(req, len, sr->buf_group, issue_flags);
1160
-
if (!buf)
1159
+
sel = io_buffer_select(req, len, sr->buf_group, issue_flags);
1160
+
if (!sel.addr)
1161
1161
return -ENOBUFS;
1162
-
sr->buf = buf;
1162
+
sr->buf = sel.addr;
1163
1163
sr->len = *len;
1164
1164
map_ubuf:
1165
1165
ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
+20
-14
io_uring/rw.c
+20
-14
io_uring/rw.c
···
107
107
}
108
108
109
109
static int __io_import_rw_buffer(int ddir, struct io_kiocb *req,
110
-
struct io_async_rw *io,
111
-
unsigned int issue_flags)
110
+
struct io_async_rw *io, struct io_br_sel *sel,
111
+
unsigned int issue_flags)
112
112
{
113
113
const struct io_issue_def *def = &io_issue_defs[req->opcode];
114
114
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
115
-
void __user *buf = u64_to_user_ptr(rw->addr);
116
115
size_t sqe_len = rw->len;
117
116
117
+
sel->addr = u64_to_user_ptr(rw->addr);
118
118
if (def->vectored && !(req->flags & REQ_F_BUFFER_SELECT))
119
-
return io_import_vec(ddir, req, io, buf, sqe_len);
119
+
return io_import_vec(ddir, req, io, sel->addr, sqe_len);
120
120
121
121
if (io_do_buffer_select(req)) {
122
-
buf = io_buffer_select(req, &sqe_len, io->buf_group, issue_flags);
123
-
if (!buf)
122
+
*sel = io_buffer_select(req, &sqe_len, io->buf_group, issue_flags);
123
+
if (!sel->addr)
124
124
return -ENOBUFS;
125
-
rw->addr = (unsigned long) buf;
125
+
rw->addr = (unsigned long) sel->addr;
126
126
rw->len = sqe_len;
127
127
}
128
-
return import_ubuf(ddir, buf, sqe_len, &io->iter);
128
+
return import_ubuf(ddir, sel->addr, sqe_len, &io->iter);
129
129
}
130
130
131
131
static inline int io_import_rw_buffer(int rw, struct io_kiocb *req,
132
132
struct io_async_rw *io,
133
+
struct io_br_sel *sel,
133
134
unsigned int issue_flags)
134
135
{
135
136
int ret;
136
137
137
-
ret = __io_import_rw_buffer(rw, req, io, issue_flags);
138
+
ret = __io_import_rw_buffer(rw, req, io, sel, issue_flags);
138
139
if (unlikely(ret < 0))
139
140
return ret;
140
141
···
307
306
308
307
static int io_rw_do_import(struct io_kiocb *req, int ddir)
309
308
{
309
+
struct io_br_sel sel = { };
310
+
310
311
if (io_do_buffer_select(req))
311
312
return 0;
312
313
313
-
return io_import_rw_buffer(ddir, req, req->async_data, 0);
314
+
return io_import_rw_buffer(ddir, req, req->async_data, &sel, 0);
314
315
}
315
316
316
317
static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
···
902
899
return 0;
903
900
}
904
901
905
-
static int __io_read(struct io_kiocb *req, unsigned int issue_flags)
902
+
static int __io_read(struct io_kiocb *req, struct io_br_sel *sel,
903
+
unsigned int issue_flags)
906
904
{
907
905
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
908
906
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
···
917
913
if (unlikely(ret))
918
914
return ret;
919
915
} else if (io_do_buffer_select(req)) {
920
-
ret = io_import_rw_buffer(ITER_DEST, req, io, issue_flags);
916
+
ret = io_import_rw_buffer(ITER_DEST, req, io, sel, issue_flags);
921
917
if (unlikely(ret < 0))
922
918
return ret;
923
919
}
···
1019
1015
1020
1016
int io_read(struct io_kiocb *req, unsigned int issue_flags)
1021
1017
{
1018
+
struct io_br_sel sel = { };
1022
1019
int ret;
1023
1020
1024
-
ret = __io_read(req, issue_flags);
1021
+
ret = __io_read(req, &sel, issue_flags);
1025
1022
if (ret >= 0)
1026
1023
return kiocb_done(req, ret, issue_flags);
1027
1024
···
1032
1027
int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
1033
1028
{
1034
1029
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
1030
+
struct io_br_sel sel = { };
1035
1031
unsigned int cflags = 0;
1036
1032
int ret;
1037
1033
···
1044
1038
1045
1039
/* make it sync, multishot doesn't support async execution */
1046
1040
rw->kiocb.ki_complete = NULL;
1047
-
ret = __io_read(req, issue_flags);
1041
+
ret = __io_read(req, &sel, issue_flags);
1048
1042
1049
1043
/*
1050
1044
* If we get -EAGAIN, recycle our buffer and just let normal poll