Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

io_uring: split out splice related operations

This splits out splice and tee support.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

+154 -130
+1 -1
io_uring/Makefile
··· 2 2 # 3 3 # Makefile for io_uring 4 4 5 - obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o 5 + obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o 6 6 obj-$(CONFIG_IO_WQ) += io-wq.o
+4 -129
io_uring/io_uring.c
··· 95 95 #include "xattr.h" 96 96 #include "nop.h" 97 97 #include "fs.h" 98 + #include "splice.h" 98 99 99 100 #define IORING_MAX_ENTRIES 32768 100 101 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) ··· 437 436 struct epoll_event event; 438 437 }; 439 438 440 - struct io_splice { 441 - struct file *file_out; 442 - loff_t off_out; 443 - loff_t off_in; 444 - u64 len; 445 - int splice_fd_in; 446 - unsigned int flags; 447 - }; 448 - 449 439 struct io_provide_buf { 450 440 struct file *file; 451 441 __u64 addr; ··· 588 596 struct io_uring_rsrc_update2 *up, 589 597 unsigned nr_args); 590 598 static void io_clean_op(struct io_kiocb *req); 591 - static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 592 - unsigned issue_flags); 593 - static struct file *io_file_get_normal(struct io_kiocb *req, int fd); 594 599 static void io_queue_sqe(struct io_kiocb *req); 595 600 static void io_rsrc_put_work(struct work_struct *work); 596 601 ··· 1065 1076 static inline bool req_has_async_data(struct io_kiocb *req) 1066 1077 { 1067 1078 return req->flags & REQ_F_ASYNC_DATA; 1068 - } 1069 - 1070 - static inline void req_set_fail(struct io_kiocb *req) 1071 - { 1072 - req->flags |= REQ_F_FAIL; 1073 - if (req->flags & REQ_F_CQE_SKIP) { 1074 - req->flags &= ~REQ_F_CQE_SKIP; 1075 - req->flags |= REQ_F_SKIP_LINK_CQES; 1076 - } 1077 1079 } 1078 1080 1079 1081 static inline void req_fail_link_node(struct io_kiocb *req, int res) ··· 1919 1939 1920 1940 node = wq_stack_extract(&ctx->submit_state.free_list); 1921 1941 return container_of(node, struct io_kiocb, comp_list); 1922 - } 1923 - 1924 - static inline void io_put_file(struct file *file) 1925 - { 1926 - if (file) 1927 - fput(file); 1928 1942 } 1929 1943 1930 1944 static inline void io_dismantle_req(struct io_kiocb *req) ··· 3891 3917 } 3892 3918 3893 3919 return IOU_ISSUE_SKIP_COMPLETE; 3894 - } 3895 - 3896 - static int __io_splice_prep(struct io_kiocb *req, 3897 - const struct io_uring_sqe *sqe) 3898 - { 3899 - struct io_splice *sp = io_kiocb_to_cmd(req); 3900 - unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL; 3901 - 3902 - sp->len = READ_ONCE(sqe->len); 3903 - sp->flags = READ_ONCE(sqe->splice_flags); 3904 - if (unlikely(sp->flags & ~valid_flags)) 3905 - return -EINVAL; 3906 - sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in); 3907 - return 0; 3908 - } 3909 - 3910 - static int io_tee_prep(struct io_kiocb *req, 3911 - const struct io_uring_sqe *sqe) 3912 - { 3913 - if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off)) 3914 - return -EINVAL; 3915 - return __io_splice_prep(req, sqe); 3916 - } 3917 - 3918 - static int io_tee(struct io_kiocb *req, unsigned int issue_flags) 3919 - { 3920 - struct io_splice *sp = io_kiocb_to_cmd(req); 3921 - struct file *out = sp->file_out; 3922 - unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; 3923 - struct file *in; 3924 - long ret = 0; 3925 - 3926 - if (issue_flags & IO_URING_F_NONBLOCK) 3927 - return -EAGAIN; 3928 - 3929 - if (sp->flags & SPLICE_F_FD_IN_FIXED) 3930 - in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags); 3931 - else 3932 - in = io_file_get_normal(req, sp->splice_fd_in); 3933 - if (!in) { 3934 - ret = -EBADF; 3935 - goto done; 3936 - } 3937 - 3938 - if (sp->len) 3939 - ret = do_tee(in, out, sp->len, flags); 3940 - 3941 - if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) 3942 - io_put_file(in); 3943 - done: 3944 - if (ret != sp->len) 3945 - req_set_fail(req); 3946 - io_req_set_res(req, ret, 0); 3947 - return IOU_OK; 3948 - } 3949 - 3950 - static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 3951 - { 3952 - struct io_splice *sp = io_kiocb_to_cmd(req); 3953 - 3954 - sp->off_in = READ_ONCE(sqe->splice_off_in); 3955 - sp->off_out = READ_ONCE(sqe->off); 3956 - return __io_splice_prep(req, sqe); 3957 - } 3958 - 3959 - static int io_splice(struct io_kiocb *req, unsigned int issue_flags) 3960 - { 3961 - struct io_splice *sp = io_kiocb_to_cmd(req); 3962 - struct file *out = sp->file_out; 3963 - unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; 3964 - loff_t *poff_in, *poff_out; 3965 - struct file *in; 3966 - long ret = 0; 3967 - 3968 - if (issue_flags & IO_URING_F_NONBLOCK) 3969 - return -EAGAIN; 3970 - 3971 - if (sp->flags & SPLICE_F_FD_IN_FIXED) 3972 - in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags); 3973 - else 3974 - in = io_file_get_normal(req, sp->splice_fd_in); 3975 - if (!in) { 3976 - ret = -EBADF; 3977 - goto done; 3978 - } 3979 - 3980 - poff_in = (sp->off_in == -1) ? NULL : &sp->off_in; 3981 - poff_out = (sp->off_out == -1) ? NULL : &sp->off_out; 3982 - 3983 - if (sp->len) 3984 - ret = do_splice(in, poff_in, out, poff_out, sp->len, flags); 3985 - 3986 - if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) 3987 - io_put_file(in); 3988 - done: 3989 - if (ret != sp->len) 3990 - req_set_fail(req); 3991 - io_req_set_res(req, ret, 0); 3992 - return IOU_OK; 3993 3920 } 3994 3921 3995 3922 static int io_msg_ring_prep(struct io_kiocb *req, ··· 7032 7157 file_slot->file_ptr = file_ptr; 7033 7158 } 7034 7159 7035 - static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 7036 - unsigned int issue_flags) 7160 + inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 7161 + unsigned int issue_flags) 7037 7162 { 7038 7163 struct io_ring_ctx *ctx = req->ctx; 7039 7164 struct file *file = NULL; ··· 7056 7181 return file; 7057 7182 } 7058 7183 7059 - static struct file *io_file_get_normal(struct io_kiocb *req, int fd) 7184 + struct file *io_file_get_normal(struct io_kiocb *req, int fd) 7060 7185 { 7061 7186 struct file *file = fget(fd); 7062 7187
+19
io_uring/io_uring.h
··· 9 9 IOU_ISSUE_SKIP_COMPLETE = -EIOCBQUEUED, 10 10 }; 11 11 12 + static inline void req_set_fail(struct io_kiocb *req) 13 + { 14 + req->flags |= REQ_F_FAIL; 15 + if (req->flags & REQ_F_CQE_SKIP) { 16 + req->flags &= ~REQ_F_CQE_SKIP; 17 + req->flags |= REQ_F_SKIP_LINK_CQES; 18 + } 19 + } 20 + 12 21 static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags) 13 22 { 14 23 req->cqe.res = res; 15 24 req->cqe.flags = cflags; 16 25 } 26 + 27 + static inline void io_put_file(struct file *file) 28 + { 29 + if (file) 30 + fput(file); 31 + } 32 + 33 + struct file *io_file_get_normal(struct io_kiocb *req, int fd); 34 + struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 35 + unsigned issue_flags); 17 36 18 37 #endif
+123
io_uring/splice.c
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + #include <linux/kernel.h> 3 + #include <linux/errno.h> 4 + #include <linux/fs.h> 5 + #include <linux/file.h> 6 + #include <linux/mm.h> 7 + #include <linux/slab.h> 8 + #include <linux/namei.h> 9 + #include <linux/io_uring.h> 10 + #include <linux/splice.h> 11 + 12 + #include <uapi/linux/io_uring.h> 13 + 14 + #include "io_uring_types.h" 15 + #include "io_uring.h" 16 + #include "splice.h" 17 + 18 + struct io_splice { 19 + struct file *file_out; 20 + loff_t off_out; 21 + loff_t off_in; 22 + u64 len; 23 + int splice_fd_in; 24 + unsigned int flags; 25 + }; 26 + 27 + static int __io_splice_prep(struct io_kiocb *req, 28 + const struct io_uring_sqe *sqe) 29 + { 30 + struct io_splice *sp = io_kiocb_to_cmd(req); 31 + unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL; 32 + 33 + sp->len = READ_ONCE(sqe->len); 34 + sp->flags = READ_ONCE(sqe->splice_flags); 35 + if (unlikely(sp->flags & ~valid_flags)) 36 + return -EINVAL; 37 + sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in); 38 + return 0; 39 + } 40 + 41 + int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 42 + { 43 + if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off)) 44 + return -EINVAL; 45 + return __io_splice_prep(req, sqe); 46 + } 47 + 48 + int io_tee(struct io_kiocb *req, unsigned int issue_flags) 49 + { 50 + struct io_splice *sp = io_kiocb_to_cmd(req); 51 + struct file *out = sp->file_out; 52 + unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; 53 + struct file *in; 54 + long ret = 0; 55 + 56 + if (issue_flags & IO_URING_F_NONBLOCK) 57 + return -EAGAIN; 58 + 59 + if (sp->flags & SPLICE_F_FD_IN_FIXED) 60 + in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags); 61 + else 62 + in = io_file_get_normal(req, sp->splice_fd_in); 63 + if (!in) { 64 + ret = -EBADF; 65 + goto done; 66 + } 67 + 68 + if (sp->len) 69 + ret = do_tee(in, out, sp->len, flags); 70 + 71 + if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) 72 + io_put_file(in); 73 + done: 74 + if (ret != sp->len) 75 + req_set_fail(req); 76 + io_req_set_res(req, ret, 0); 77 + return IOU_OK; 78 + } 79 + 80 + int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 81 + { 82 + struct io_splice *sp = io_kiocb_to_cmd(req); 83 + 84 + sp->off_in = READ_ONCE(sqe->splice_off_in); 85 + sp->off_out = READ_ONCE(sqe->off); 86 + return __io_splice_prep(req, sqe); 87 + } 88 + 89 + int io_splice(struct io_kiocb *req, unsigned int issue_flags) 90 + { 91 + struct io_splice *sp = io_kiocb_to_cmd(req); 92 + struct file *out = sp->file_out; 93 + unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; 94 + loff_t *poff_in, *poff_out; 95 + struct file *in; 96 + long ret = 0; 97 + 98 + if (issue_flags & IO_URING_F_NONBLOCK) 99 + return -EAGAIN; 100 + 101 + if (sp->flags & SPLICE_F_FD_IN_FIXED) 102 + in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags); 103 + else 104 + in = io_file_get_normal(req, sp->splice_fd_in); 105 + if (!in) { 106 + ret = -EBADF; 107 + goto done; 108 + } 109 + 110 + poff_in = (sp->off_in == -1) ? NULL : &sp->off_in; 111 + poff_out = (sp->off_out == -1) ? NULL : &sp->off_out; 112 + 113 + if (sp->len) 114 + ret = do_splice(in, poff_in, out, poff_out, sp->len, flags); 115 + 116 + if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) 117 + io_put_file(in); 118 + done: 119 + if (ret != sp->len) 120 + req_set_fail(req); 121 + io_req_set_res(req, ret, 0); 122 + return IOU_OK; 123 + }
+7
io_uring/splice.h
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 4 + int io_tee(struct io_kiocb *req, unsigned int issue_flags); 5 + 6 + int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 7 + int io_splice(struct io_kiocb *req, unsigned int issue_flags);