tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

io_uring/rw: defer reg buf vec import

Import registered buffers for vectored reads and writes later at issue
time as we now do for other fixed ops.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/e8491c976e4ab83a4e3dc428e9fe7555e59583b8.1741362889.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Pavel Begunkov and committed by
Jens Axboe 835c4bdf bdabba04

+37 -8
+3
include/linux/io_uring_types.h
··· 502 502 REQ_F_BUFFERS_COMMIT_BIT, 503 503 REQ_F_BUF_NODE_BIT, 504 504 REQ_F_HAS_METADATA_BIT, 505 + REQ_F_IMPORT_BUFFER_BIT, 505 506 506 507 /* not a real bit, just to check we're not overflowing the space */ 507 508 __REQ_F_LAST_BIT, ··· 585 584 REQ_F_BUF_NODE = IO_REQ_FLAG(REQ_F_BUF_NODE_BIT), 586 585 /* request has read/write metadata assigned */ 587 586 REQ_F_HAS_METADATA = IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT), 587 + /* resolve padded iovec to registered buffers */ 588 + REQ_F_IMPORT_BUFFER = IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT), 588 589 }; 589 590 590 591 typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);
+34 -8
io_uring/rw.c
··· 381 381 return __io_prep_rw(req, sqe, ITER_SOURCE); 382 382 } 383 383 384 - static int io_rw_prep_reg_vec(struct io_kiocb *req, int ddir) 384 + static int io_rw_import_reg_vec(struct io_kiocb *req, 385 + struct io_async_rw *io, 386 + int ddir, unsigned int issue_flags) 387 + { 388 + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); 389 + unsigned uvec_segs = rw->len; 390 + unsigned iovec_off = io->vec.nr - uvec_segs; 391 + int ret; 392 + 393 + ret = io_import_reg_vec(ddir, &io->iter, req, &io->vec, 394 + uvec_segs, iovec_off, issue_flags); 395 + if (unlikely(ret)) 396 + return ret; 397 + iov_iter_save_state(&io->iter, &io->iter_state); 398 + req->flags &= ~REQ_F_IMPORT_BUFFER; 399 + return 0; 400 + } 401 + 402 + static int io_rw_prep_reg_vec(struct io_kiocb *req) 385 403 { 386 404 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); 387 405 struct io_async_rw *io = req->async_data; ··· 424 406 if (IS_ERR(res)) 425 407 return PTR_ERR(res); 426 408 427 - ret = io_import_reg_vec(ddir, &io->iter, req, &io->vec, 428 - uvec_segs, iovec_off, 0); 429 - iov_iter_save_state(&io->iter, &io->iter_state); 430 - return ret; 409 + req->flags |= REQ_F_IMPORT_BUFFER; 410 + return 0; 431 411 } 432 412 433 413 int io_prep_readv_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) ··· 435 419 ret = __io_prep_rw(req, sqe, ITER_DEST); 436 420 if (unlikely(ret)) 437 421 return ret; 438 - return io_rw_prep_reg_vec(req, ITER_DEST); 422 + return io_rw_prep_reg_vec(req); 439 423 } 440 424 441 425 int io_prep_writev_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) ··· 445 429 ret = __io_prep_rw(req, sqe, ITER_SOURCE); 446 430 if (unlikely(ret)) 447 431 return ret; 448 - return io_rw_prep_reg_vec(req, ITER_SOURCE); 432 + return io_rw_prep_reg_vec(req); 449 433 } 450 434 451 435 /* ··· 922 906 ssize_t ret; 923 907 loff_t *ppos; 924 908 925 - if (io_do_buffer_select(req)) { 909 + if (req->flags & REQ_F_IMPORT_BUFFER) { 910 + ret = io_rw_import_reg_vec(req, io, ITER_DEST, issue_flags); 911 + if (unlikely(ret)) 912 + return ret; 913 + } else if (io_do_buffer_select(req)) { 926 914 ret = io_import_rw_buffer(ITER_DEST, req, io, issue_flags); 927 915 if (unlikely(ret < 0)) 928 916 return ret; ··· 1136 1116 struct kiocb *kiocb = &rw->kiocb; 1137 1117 ssize_t ret, ret2; 1138 1118 loff_t *ppos; 1119 + 1120 + if (req->flags & REQ_F_IMPORT_BUFFER) { 1121 + ret = io_rw_import_reg_vec(req, io, ITER_SOURCE, issue_flags); 1122 + if (unlikely(ret)) 1123 + return ret; 1124 + } 1139 1125 1140 1126 ret = io_rw_init_file(req, FMODE_WRITE, WRITE); 1141 1127 if (unlikely(ret))