Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/kbuf: pass in struct io_buffer_list to commit/recycle helpers
Rather than have this implied being in the io_kiocb, pass it in directly
so it's immediately obvious where these users of ->buf_list are coming
from.
Link: https://lore.kernel.org/r/20250821020750.598432-6-axboe@kernel.dk
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+45
-40
+3
-3
io_uring/io_uring.c
+3
-3
io_uring/io_uring.c
···
1007
1007
lockdep_assert_held(&req->ctx->uring_lock);
1008
1008
1009
1009
req_set_fail(req);
1010
-
io_req_set_res(req, res, io_put_kbuf(req, res));
1010
+
io_req_set_res(req, res, io_put_kbuf(req, res, req->buf_list));
1011
1011
if (def->fail)
1012
1012
def->fail(req);
1013
1013
io_req_complete_defer(req);
···
2025
2025
2026
2026
switch (io_arm_poll_handler(req, 0)) {
2027
2027
case IO_APOLL_READY:
2028
-
io_kbuf_recycle(req, 0);
2028
+
io_kbuf_recycle(req, req->buf_list, 0);
2029
2029
io_req_task_queue(req);
2030
2030
break;
2031
2031
case IO_APOLL_ABORTED:
2032
-
io_kbuf_recycle(req, 0);
2032
+
io_kbuf_recycle(req, req->buf_list, 0);
2033
2033
io_queue_iowq(req);
2034
2034
break;
2035
2035
case IO_APOLL_OK:
+5
-4
io_uring/kbuf.c
+5
-4
io_uring/kbuf.c
···
354
354
return io_provided_buffers_select(req, &arg->max_len, bl, arg->iovs);
355
355
}
356
356
357
-
static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)
357
+
static inline bool __io_put_kbuf_ring(struct io_kiocb *req,
358
+
struct io_buffer_list *bl, int len, int nr)
358
359
{
359
-
struct io_buffer_list *bl = req->buf_list;
360
360
bool ret = true;
361
361
362
362
if (bl)
···
366
366
return ret;
367
367
}
368
368
369
-
unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs)
369
+
unsigned int __io_put_kbufs(struct io_kiocb *req, struct io_buffer_list *bl,
370
+
int len, int nbufs)
370
371
{
371
372
unsigned int ret;
372
373
···
378
377
return ret;
379
378
}
380
379
381
-
if (!__io_put_kbuf_ring(req, len, nbufs))
380
+
if (!__io_put_kbuf_ring(req, bl, len, nbufs))
382
381
ret |= IORING_CQE_F_BUF_MORE;
383
382
return ret;
384
383
}
+13
-9
io_uring/kbuf.h
+13
-9
io_uring/kbuf.h
···
80
80
bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
81
81
void io_kbuf_drop_legacy(struct io_kiocb *req);
82
82
83
-
unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs);
83
+
unsigned int __io_put_kbufs(struct io_kiocb *req, struct io_buffer_list *bl,
84
+
int len, int nbufs);
84
85
bool io_kbuf_commit(struct io_kiocb *req,
85
86
struct io_buffer_list *bl, int len, int nr);
86
87
87
88
struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx,
88
89
unsigned int bgid);
89
90
90
-
static inline bool io_kbuf_recycle_ring(struct io_kiocb *req)
91
+
static inline bool io_kbuf_recycle_ring(struct io_kiocb *req,
92
+
struct io_buffer_list *bl)
91
93
{
92
94
/*
93
95
* We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
···
98
96
* The exception is partial io, that case we should increment bl->head
99
97
* to monopolize the buffer.
100
98
*/
101
-
if (req->buf_list) {
99
+
if (bl) {
102
100
req->flags &= ~(REQ_F_BUFFER_RING|REQ_F_BUFFERS_COMMIT);
103
101
return true;
104
102
}
···
112
110
return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING));
113
111
}
114
112
115
-
static inline bool io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
113
+
static inline bool io_kbuf_recycle(struct io_kiocb *req, struct io_buffer_list *bl,
114
+
unsigned issue_flags)
116
115
{
117
116
if (req->flags & REQ_F_BL_NO_RECYCLE)
118
117
return false;
119
118
if (req->flags & REQ_F_BUFFER_SELECTED)
120
119
return io_kbuf_recycle_legacy(req, issue_flags);
121
120
if (req->flags & REQ_F_BUFFER_RING)
122
-
return io_kbuf_recycle_ring(req);
121
+
return io_kbuf_recycle_ring(req, bl);
123
122
return false;
124
123
}
125
124
126
-
static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len)
125
+
static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len,
126
+
struct io_buffer_list *bl)
127
127
{
128
128
if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))
129
129
return 0;
130
-
return __io_put_kbufs(req, len, 1);
130
+
return __io_put_kbufs(req, bl, len, 1);
131
131
}
132
132
133
133
static inline unsigned int io_put_kbufs(struct io_kiocb *req, int len,
134
-
int nbufs)
134
+
struct io_buffer_list *bl, int nbufs)
135
135
{
136
136
if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))
137
137
return 0;
138
-
return __io_put_kbufs(req, len, nbufs);
138
+
return __io_put_kbufs(req, bl, len, nbufs);
139
139
}
140
140
#endif
+14
-14
io_uring/net.c
+14
-14
io_uring/net.c
···
494
494
return nbufs;
495
495
}
496
496
497
-
static int io_net_kbuf_recyle(struct io_kiocb *req,
497
+
static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl,
498
498
struct io_async_msghdr *kmsg, int len)
499
499
{
500
500
req->flags |= REQ_F_BL_NO_RECYCLE;
501
501
if (req->flags & REQ_F_BUFFERS_COMMIT)
502
-
io_kbuf_commit(req, req->buf_list, len, io_bundle_nbufs(kmsg, len));
502
+
io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len));
503
503
return IOU_RETRY;
504
504
}
505
505
···
511
511
unsigned int cflags;
512
512
513
513
if (!(sr->flags & IORING_RECVSEND_BUNDLE)) {
514
-
cflags = io_put_kbuf(req, *ret);
514
+
cflags = io_put_kbuf(req, *ret, req->buf_list);
515
515
goto finish;
516
516
}
517
517
518
-
cflags = io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret));
518
+
cflags = io_put_kbufs(req, *ret, req->buf_list, io_bundle_nbufs(kmsg, *ret));
519
519
520
520
if (bundle_finished || req->flags & REQ_F_BL_EMPTY)
521
521
goto finish;
···
681
681
sr->len -= ret;
682
682
sr->buf += ret;
683
683
sr->done_io += ret;
684
-
return io_net_kbuf_recyle(req, kmsg, ret);
684
+
return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret);
685
685
}
686
686
if (ret == -ERESTARTSYS)
687
687
ret = -EINTR;
···
871
871
if (sr->flags & IORING_RECVSEND_BUNDLE) {
872
872
size_t this_ret = *ret - sr->done_io;
873
873
874
-
cflags |= io_put_kbufs(req, this_ret, io_bundle_nbufs(kmsg, this_ret));
874
+
cflags |= io_put_kbufs(req, this_ret, req->buf_list, io_bundle_nbufs(kmsg, this_ret));
875
875
if (sr->flags & IORING_RECV_RETRY)
876
876
cflags = req->cqe.flags | (cflags & CQE_F_MASK);
877
877
if (sr->mshot_len && *ret >= sr->mshot_len)
···
893
893
return false;
894
894
}
895
895
} else {
896
-
cflags |= io_put_kbuf(req, *ret);
896
+
cflags |= io_put_kbuf(req, *ret, req->buf_list);
897
897
}
898
898
899
899
/*
···
1045
1045
if (req->flags & REQ_F_APOLL_MULTISHOT) {
1046
1046
ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len);
1047
1047
if (ret) {
1048
-
io_kbuf_recycle(req, issue_flags);
1048
+
io_kbuf_recycle(req, req->buf_list, issue_flags);
1049
1049
return ret;
1050
1050
}
1051
1051
}
···
1070
1070
if (ret < min_ret) {
1071
1071
if (ret == -EAGAIN && force_nonblock) {
1072
1072
if (issue_flags & IO_URING_F_MULTISHOT)
1073
-
io_kbuf_recycle(req, issue_flags);
1073
+
io_kbuf_recycle(req, req->buf_list, issue_flags);
1074
1074
1075
1075
return IOU_RETRY;
1076
1076
}
1077
1077
if (ret > 0 && io_net_retry(sock, flags)) {
1078
1078
sr->done_io += ret;
1079
-
return io_net_kbuf_recyle(req, kmsg, ret);
1079
+
return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret);
1080
1080
}
1081
1081
if (ret == -ERESTARTSYS)
1082
1082
ret = -EINTR;
···
1090
1090
else if (sr->done_io)
1091
1091
ret = sr->done_io;
1092
1092
else
1093
-
io_kbuf_recycle(req, issue_flags);
1093
+
io_kbuf_recycle(req, req->buf_list, issue_flags);
1094
1094
1095
1095
if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags))
1096
1096
goto retry_multishot;
···
1214
1214
if (ret < min_ret) {
1215
1215
if (ret == -EAGAIN && force_nonblock) {
1216
1216
if (issue_flags & IO_URING_F_MULTISHOT)
1217
-
io_kbuf_recycle(req, issue_flags);
1217
+
io_kbuf_recycle(req, req->buf_list, issue_flags);
1218
1218
1219
1219
return IOU_RETRY;
1220
1220
}
···
1222
1222
sr->len -= ret;
1223
1223
sr->buf += ret;
1224
1224
sr->done_io += ret;
1225
-
return io_net_kbuf_recyle(req, kmsg, ret);
1225
+
return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret);
1226
1226
}
1227
1227
if (ret == -ERESTARTSYS)
1228
1228
ret = -EINTR;
···
1238
1238
else if (sr->done_io)
1239
1239
ret = sr->done_io;
1240
1240
else
1241
-
io_kbuf_recycle(req, issue_flags);
1241
+
io_kbuf_recycle(req, req->buf_list, issue_flags);
1242
1242
1243
1243
if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags))
1244
1244
goto retry_multishot;
+3
-3
io_uring/poll.c
+3
-3
io_uring/poll.c
···
316
316
317
317
ret = io_poll_check_events(req, tw);
318
318
if (ret == IOU_POLL_NO_ACTION) {
319
-
io_kbuf_recycle(req, 0);
319
+
io_kbuf_recycle(req, req->buf_list, 0);
320
320
return;
321
321
} else if (ret == IOU_POLL_REQUEUE) {
322
-
io_kbuf_recycle(req, 0);
322
+
io_kbuf_recycle(req, req->buf_list, 0);
323
323
__io_poll_execute(req, 0);
324
324
return;
325
325
}
···
686
686
req->flags |= REQ_F_POLLED;
687
687
ipt.pt._qproc = io_async_queue_proc;
688
688
689
-
io_kbuf_recycle(req, issue_flags);
689
+
io_kbuf_recycle(req, req->buf_list, issue_flags);
690
690
691
691
ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags);
692
692
if (ret)
+7
-7
io_uring/rw.c
+7
-7
io_uring/rw.c
···
576
576
io_req_io_end(req);
577
577
578
578
if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))
579
-
req->cqe.flags |= io_put_kbuf(req, req->cqe.res);
579
+
req->cqe.flags |= io_put_kbuf(req, req->cqe.res, req->buf_list);
580
580
581
581
io_req_rw_cleanup(req, 0);
582
582
io_req_task_complete(req, tw);
···
659
659
* from the submission path.
660
660
*/
661
661
io_req_io_end(req);
662
-
io_req_set_res(req, final_ret, io_put_kbuf(req, ret));
662
+
io_req_set_res(req, final_ret, io_put_kbuf(req, ret, req->buf_list));
663
663
io_req_rw_cleanup(req, issue_flags);
664
664
return IOU_COMPLETE;
665
665
} else {
···
1049
1049
* Reset rw->len to 0 again to avoid clamping future mshot
1050
1050
* reads, in case the buffer size varies.
1051
1051
*/
1052
-
if (io_kbuf_recycle(req, issue_flags))
1052
+
if (io_kbuf_recycle(req, req->buf_list, issue_flags))
1053
1053
rw->len = 0;
1054
1054
return IOU_RETRY;
1055
1055
} else if (ret <= 0) {
1056
-
io_kbuf_recycle(req, issue_flags);
1056
+
io_kbuf_recycle(req, req->buf_list, issue_flags);
1057
1057
if (ret < 0)
1058
1058
req_set_fail(req);
1059
1059
} else if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
1060
-
cflags = io_put_kbuf(req, ret);
1060
+
cflags = io_put_kbuf(req, ret, req->buf_list);
1061
1061
} else {
1062
1062
/*
1063
1063
* Any successful return value will keep the multishot read
···
1065
1065
* we fail to post a CQE, or multishot is no longer set, then
1066
1066
* jump to the termination path. This request is then done.
1067
1067
*/
1068
-
cflags = io_put_kbuf(req, ret);
1068
+
cflags = io_put_kbuf(req, ret, req->buf_list);
1069
1069
rw->len = 0; /* similarly to above, reset len to 0 */
1070
1070
1071
1071
if (io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) {
···
1362
1362
if (!smp_load_acquire(&req->iopoll_completed))
1363
1363
break;
1364
1364
nr_events++;
1365
-
req->cqe.flags = io_put_kbuf(req, req->cqe.res);
1365
+
req->cqe.flags = io_put_kbuf(req, req->cqe.res, req->buf_list);
1366
1366
if (req->opcode != IORING_OP_URING_CMD)
1367
1367
io_req_rw_cleanup(req, 0);
1368
1368
}