Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

io_uring/kbuf: pass in struct io_buffer_list to commit/recycle helpers

Rather than have this implied being in the io_kiocb, pass it in directly
so it's immediately obvious where these users of ->buf_list are coming
from.

Link: https://lore.kernel.org/r/20250821020750.598432-6-axboe@kernel.dk
Signed-off-by: Jens Axboe <axboe@kernel.dk>

Jens Axboe 1b5add75 b22743f2

+45 -40
+3 -3
io_uring/io_uring.c
··· 1007 1007 lockdep_assert_held(&req->ctx->uring_lock); 1008 1008 1009 1009 req_set_fail(req); 1010 - io_req_set_res(req, res, io_put_kbuf(req, res)); 1010 + io_req_set_res(req, res, io_put_kbuf(req, res, req->buf_list)); 1011 1011 if (def->fail) 1012 1012 def->fail(req); 1013 1013 io_req_complete_defer(req); ··· 2025 2025 2026 2026 switch (io_arm_poll_handler(req, 0)) { 2027 2027 case IO_APOLL_READY: 2028 - io_kbuf_recycle(req, 0); 2028 + io_kbuf_recycle(req, req->buf_list, 0); 2029 2029 io_req_task_queue(req); 2030 2030 break; 2031 2031 case IO_APOLL_ABORTED: 2032 - io_kbuf_recycle(req, 0); 2032 + io_kbuf_recycle(req, req->buf_list, 0); 2033 2033 io_queue_iowq(req); 2034 2034 break; 2035 2035 case IO_APOLL_OK:
+5 -4
io_uring/kbuf.c
··· 354 354 return io_provided_buffers_select(req, &arg->max_len, bl, arg->iovs); 355 355 } 356 356 357 - static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr) 357 + static inline bool __io_put_kbuf_ring(struct io_kiocb *req, 358 + struct io_buffer_list *bl, int len, int nr) 358 359 { 359 - struct io_buffer_list *bl = req->buf_list; 360 360 bool ret = true; 361 361 362 362 if (bl) ··· 366 366 return ret; 367 367 } 368 368 369 - unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs) 369 + unsigned int __io_put_kbufs(struct io_kiocb *req, struct io_buffer_list *bl, 370 + int len, int nbufs) 370 371 { 371 372 unsigned int ret; 372 373 ··· 378 377 return ret; 379 378 } 380 379 381 - if (!__io_put_kbuf_ring(req, len, nbufs)) 380 + if (!__io_put_kbuf_ring(req, bl, len, nbufs)) 382 381 ret |= IORING_CQE_F_BUF_MORE; 383 382 return ret; 384 383 }
+13 -9
io_uring/kbuf.h
··· 80 80 bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags); 81 81 void io_kbuf_drop_legacy(struct io_kiocb *req); 82 82 83 - unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs); 83 + unsigned int __io_put_kbufs(struct io_kiocb *req, struct io_buffer_list *bl, 84 + int len, int nbufs); 84 85 bool io_kbuf_commit(struct io_kiocb *req, 85 86 struct io_buffer_list *bl, int len, int nr); 86 87 87 88 struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx, 88 89 unsigned int bgid); 89 90 90 - static inline bool io_kbuf_recycle_ring(struct io_kiocb *req) 91 + static inline bool io_kbuf_recycle_ring(struct io_kiocb *req, 92 + struct io_buffer_list *bl) 91 93 { 92 94 /* 93 95 * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear ··· 98 96 * The exception is partial io, that case we should increment bl->head 99 97 * to monopolize the buffer. 100 98 */ 101 - if (req->buf_list) { 99 + if (bl) { 102 100 req->flags &= ~(REQ_F_BUFFER_RING|REQ_F_BUFFERS_COMMIT); 103 101 return true; 104 102 } ··· 112 110 return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)); 113 111 } 114 112 115 - static inline bool io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) 113 + static inline bool io_kbuf_recycle(struct io_kiocb *req, struct io_buffer_list *bl, 114 + unsigned issue_flags) 116 115 { 117 116 if (req->flags & REQ_F_BL_NO_RECYCLE) 118 117 return false; 119 118 if (req->flags & REQ_F_BUFFER_SELECTED) 120 119 return io_kbuf_recycle_legacy(req, issue_flags); 121 120 if (req->flags & REQ_F_BUFFER_RING) 122 - return io_kbuf_recycle_ring(req); 121 + return io_kbuf_recycle_ring(req, bl); 123 122 return false; 124 123 } 125 124 126 - static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len) 125 + static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len, 126 + struct io_buffer_list *bl) 127 127 { 128 128 if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED))) 129 129 return 0; 130 - return __io_put_kbufs(req, len, 1); 130 + return __io_put_kbufs(req, bl, len, 1); 131 131 } 132 132 133 133 static inline unsigned int io_put_kbufs(struct io_kiocb *req, int len, 134 - int nbufs) 134 + struct io_buffer_list *bl, int nbufs) 135 135 { 136 136 if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED))) 137 137 return 0; 138 - return __io_put_kbufs(req, len, nbufs); 138 + return __io_put_kbufs(req, bl, len, nbufs); 139 139 } 140 140 #endif
+14 -14
io_uring/net.c
··· 494 494 return nbufs; 495 495 } 496 496 497 - static int io_net_kbuf_recyle(struct io_kiocb *req, 497 + static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl, 498 498 struct io_async_msghdr *kmsg, int len) 499 499 { 500 500 req->flags |= REQ_F_BL_NO_RECYCLE; 501 501 if (req->flags & REQ_F_BUFFERS_COMMIT) 502 - io_kbuf_commit(req, req->buf_list, len, io_bundle_nbufs(kmsg, len)); 502 + io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len)); 503 503 return IOU_RETRY; 504 504 } 505 505 ··· 511 511 unsigned int cflags; 512 512 513 513 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) { 514 - cflags = io_put_kbuf(req, *ret); 514 + cflags = io_put_kbuf(req, *ret, req->buf_list); 515 515 goto finish; 516 516 } 517 517 518 - cflags = io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret)); 518 + cflags = io_put_kbufs(req, *ret, req->buf_list, io_bundle_nbufs(kmsg, *ret)); 519 519 520 520 if (bundle_finished || req->flags & REQ_F_BL_EMPTY) 521 521 goto finish; ··· 681 681 sr->len -= ret; 682 682 sr->buf += ret; 683 683 sr->done_io += ret; 684 - return io_net_kbuf_recyle(req, kmsg, ret); 684 + return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret); 685 685 } 686 686 if (ret == -ERESTARTSYS) 687 687 ret = -EINTR; ··· 871 871 if (sr->flags & IORING_RECVSEND_BUNDLE) { 872 872 size_t this_ret = *ret - sr->done_io; 873 873 874 - cflags |= io_put_kbufs(req, this_ret, io_bundle_nbufs(kmsg, this_ret)); 874 + cflags |= io_put_kbufs(req, this_ret, req->buf_list, io_bundle_nbufs(kmsg, this_ret)); 875 875 if (sr->flags & IORING_RECV_RETRY) 876 876 cflags = req->cqe.flags | (cflags & CQE_F_MASK); 877 877 if (sr->mshot_len && *ret >= sr->mshot_len) ··· 893 893 return false; 894 894 } 895 895 } else { 896 - cflags |= io_put_kbuf(req, *ret); 896 + cflags |= io_put_kbuf(req, *ret, req->buf_list); 897 897 } 898 898 899 899 /* ··· 1045 1045 if (req->flags & REQ_F_APOLL_MULTISHOT) { 1046 1046 ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len); 1047 1047 if (ret) { 1048 - io_kbuf_recycle(req, issue_flags); 1048 + io_kbuf_recycle(req, req->buf_list, issue_flags); 1049 1049 return ret; 1050 1050 } 1051 1051 } ··· 1070 1070 if (ret < min_ret) { 1071 1071 if (ret == -EAGAIN && force_nonblock) { 1072 1072 if (issue_flags & IO_URING_F_MULTISHOT) 1073 - io_kbuf_recycle(req, issue_flags); 1073 + io_kbuf_recycle(req, req->buf_list, issue_flags); 1074 1074 1075 1075 return IOU_RETRY; 1076 1076 } 1077 1077 if (ret > 0 && io_net_retry(sock, flags)) { 1078 1078 sr->done_io += ret; 1079 - return io_net_kbuf_recyle(req, kmsg, ret); 1079 + return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret); 1080 1080 } 1081 1081 if (ret == -ERESTARTSYS) 1082 1082 ret = -EINTR; ··· 1090 1090 else if (sr->done_io) 1091 1091 ret = sr->done_io; 1092 1092 else 1093 - io_kbuf_recycle(req, issue_flags); 1093 + io_kbuf_recycle(req, req->buf_list, issue_flags); 1094 1094 1095 1095 if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags)) 1096 1096 goto retry_multishot; ··· 1214 1214 if (ret < min_ret) { 1215 1215 if (ret == -EAGAIN && force_nonblock) { 1216 1216 if (issue_flags & IO_URING_F_MULTISHOT) 1217 - io_kbuf_recycle(req, issue_flags); 1217 + io_kbuf_recycle(req, req->buf_list, issue_flags); 1218 1218 1219 1219 return IOU_RETRY; 1220 1220 } ··· 1222 1222 sr->len -= ret; 1223 1223 sr->buf += ret; 1224 1224 sr->done_io += ret; 1225 - return io_net_kbuf_recyle(req, kmsg, ret); 1225 + return io_net_kbuf_recyle(req, req->buf_list, kmsg, ret); 1226 1226 } 1227 1227 if (ret == -ERESTARTSYS) 1228 1228 ret = -EINTR; ··· 1238 1238 else if (sr->done_io) 1239 1239 ret = sr->done_io; 1240 1240 else 1241 - io_kbuf_recycle(req, issue_flags); 1241 + io_kbuf_recycle(req, req->buf_list, issue_flags); 1242 1242 1243 1243 if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags)) 1244 1244 goto retry_multishot;
+3 -3
io_uring/poll.c
··· 316 316 317 317 ret = io_poll_check_events(req, tw); 318 318 if (ret == IOU_POLL_NO_ACTION) { 319 - io_kbuf_recycle(req, 0); 319 + io_kbuf_recycle(req, req->buf_list, 0); 320 320 return; 321 321 } else if (ret == IOU_POLL_REQUEUE) { 322 - io_kbuf_recycle(req, 0); 322 + io_kbuf_recycle(req, req->buf_list, 0); 323 323 __io_poll_execute(req, 0); 324 324 return; 325 325 } ··· 686 686 req->flags |= REQ_F_POLLED; 687 687 ipt.pt._qproc = io_async_queue_proc; 688 688 689 - io_kbuf_recycle(req, issue_flags); 689 + io_kbuf_recycle(req, req->buf_list, issue_flags); 690 690 691 691 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags); 692 692 if (ret)
+7 -7
io_uring/rw.c
··· 576 576 io_req_io_end(req); 577 577 578 578 if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) 579 - req->cqe.flags |= io_put_kbuf(req, req->cqe.res); 579 + req->cqe.flags |= io_put_kbuf(req, req->cqe.res, req->buf_list); 580 580 581 581 io_req_rw_cleanup(req, 0); 582 582 io_req_task_complete(req, tw); ··· 659 659 * from the submission path. 660 660 */ 661 661 io_req_io_end(req); 662 - io_req_set_res(req, final_ret, io_put_kbuf(req, ret)); 662 + io_req_set_res(req, final_ret, io_put_kbuf(req, ret, req->buf_list)); 663 663 io_req_rw_cleanup(req, issue_flags); 664 664 return IOU_COMPLETE; 665 665 } else { ··· 1049 1049 * Reset rw->len to 0 again to avoid clamping future mshot 1050 1050 * reads, in case the buffer size varies. 1051 1051 */ 1052 - if (io_kbuf_recycle(req, issue_flags)) 1052 + if (io_kbuf_recycle(req, req->buf_list, issue_flags)) 1053 1053 rw->len = 0; 1054 1054 return IOU_RETRY; 1055 1055 } else if (ret <= 0) { 1056 - io_kbuf_recycle(req, issue_flags); 1056 + io_kbuf_recycle(req, req->buf_list, issue_flags); 1057 1057 if (ret < 0) 1058 1058 req_set_fail(req); 1059 1059 } else if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 1060 - cflags = io_put_kbuf(req, ret); 1060 + cflags = io_put_kbuf(req, ret, req->buf_list); 1061 1061 } else { 1062 1062 /* 1063 1063 * Any successful return value will keep the multishot read ··· 1065 1065 * we fail to post a CQE, or multishot is no longer set, then 1066 1066 * jump to the termination path. This request is then done. 1067 1067 */ 1068 - cflags = io_put_kbuf(req, ret); 1068 + cflags = io_put_kbuf(req, ret, req->buf_list); 1069 1069 rw->len = 0; /* similarly to above, reset len to 0 */ 1070 1070 1071 1071 if (io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) { ··· 1362 1362 if (!smp_load_acquire(&req->iopoll_completed)) 1363 1363 break; 1364 1364 nr_events++; 1365 - req->cqe.flags = io_put_kbuf(req, req->cqe.res); 1365 + req->cqe.flags = io_put_kbuf(req, req->cqe.res, req->buf_list); 1366 1366 if (req->opcode != IORING_OP_URING_CMD) 1367 1367 io_req_rw_cleanup(req, 0); 1368 1368 }