Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring: split out splice related operations
This splits out splice and tee support.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+154
-130
+1
-1
io_uring/Makefile
+1
-1
io_uring/Makefile
+4
-129
io_uring/io_uring.c
+4
-129
io_uring/io_uring.c
···
95
95
#include "xattr.h"
96
96
#include "nop.h"
97
97
#include "fs.h"
98
+
#include "splice.h"
98
99
99
100
#define IORING_MAX_ENTRIES 32768
100
101
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
···
437
436
struct epoll_event event;
438
437
};
439
438
440
-
struct io_splice {
441
-
struct file *file_out;
442
-
loff_t off_out;
443
-
loff_t off_in;
444
-
u64 len;
445
-
int splice_fd_in;
446
-
unsigned int flags;
447
-
};
448
-
449
439
struct io_provide_buf {
450
440
struct file *file;
451
441
__u64 addr;
···
588
596
struct io_uring_rsrc_update2 *up,
589
597
unsigned nr_args);
590
598
static void io_clean_op(struct io_kiocb *req);
591
-
static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
592
-
unsigned issue_flags);
593
-
static struct file *io_file_get_normal(struct io_kiocb *req, int fd);
594
599
static void io_queue_sqe(struct io_kiocb *req);
595
600
static void io_rsrc_put_work(struct work_struct *work);
596
601
···
1065
1076
static inline bool req_has_async_data(struct io_kiocb *req)
1066
1077
{
1067
1078
return req->flags & REQ_F_ASYNC_DATA;
1068
-
}
1069
-
1070
-
static inline void req_set_fail(struct io_kiocb *req)
1071
-
{
1072
-
req->flags |= REQ_F_FAIL;
1073
-
if (req->flags & REQ_F_CQE_SKIP) {
1074
-
req->flags &= ~REQ_F_CQE_SKIP;
1075
-
req->flags |= REQ_F_SKIP_LINK_CQES;
1076
-
}
1077
1079
}
1078
1080
1079
1081
static inline void req_fail_link_node(struct io_kiocb *req, int res)
···
1919
1939
1920
1940
node = wq_stack_extract(&ctx->submit_state.free_list);
1921
1941
return container_of(node, struct io_kiocb, comp_list);
1922
-
}
1923
-
1924
-
static inline void io_put_file(struct file *file)
1925
-
{
1926
-
if (file)
1927
-
fput(file);
1928
1942
}
1929
1943
1930
1944
static inline void io_dismantle_req(struct io_kiocb *req)
···
3891
3917
}
3892
3918
3893
3919
return IOU_ISSUE_SKIP_COMPLETE;
3894
-
}
3895
-
3896
-
static int __io_splice_prep(struct io_kiocb *req,
3897
-
const struct io_uring_sqe *sqe)
3898
-
{
3899
-
struct io_splice *sp = io_kiocb_to_cmd(req);
3900
-
unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
3901
-
3902
-
sp->len = READ_ONCE(sqe->len);
3903
-
sp->flags = READ_ONCE(sqe->splice_flags);
3904
-
if (unlikely(sp->flags & ~valid_flags))
3905
-
return -EINVAL;
3906
-
sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
3907
-
return 0;
3908
-
}
3909
-
3910
-
static int io_tee_prep(struct io_kiocb *req,
3911
-
const struct io_uring_sqe *sqe)
3912
-
{
3913
-
if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
3914
-
return -EINVAL;
3915
-
return __io_splice_prep(req, sqe);
3916
-
}
3917
-
3918
-
static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
3919
-
{
3920
-
struct io_splice *sp = io_kiocb_to_cmd(req);
3921
-
struct file *out = sp->file_out;
3922
-
unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3923
-
struct file *in;
3924
-
long ret = 0;
3925
-
3926
-
if (issue_flags & IO_URING_F_NONBLOCK)
3927
-
return -EAGAIN;
3928
-
3929
-
if (sp->flags & SPLICE_F_FD_IN_FIXED)
3930
-
in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags);
3931
-
else
3932
-
in = io_file_get_normal(req, sp->splice_fd_in);
3933
-
if (!in) {
3934
-
ret = -EBADF;
3935
-
goto done;
3936
-
}
3937
-
3938
-
if (sp->len)
3939
-
ret = do_tee(in, out, sp->len, flags);
3940
-
3941
-
if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3942
-
io_put_file(in);
3943
-
done:
3944
-
if (ret != sp->len)
3945
-
req_set_fail(req);
3946
-
io_req_set_res(req, ret, 0);
3947
-
return IOU_OK;
3948
-
}
3949
-
3950
-
static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3951
-
{
3952
-
struct io_splice *sp = io_kiocb_to_cmd(req);
3953
-
3954
-
sp->off_in = READ_ONCE(sqe->splice_off_in);
3955
-
sp->off_out = READ_ONCE(sqe->off);
3956
-
return __io_splice_prep(req, sqe);
3957
-
}
3958
-
3959
-
static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
3960
-
{
3961
-
struct io_splice *sp = io_kiocb_to_cmd(req);
3962
-
struct file *out = sp->file_out;
3963
-
unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3964
-
loff_t *poff_in, *poff_out;
3965
-
struct file *in;
3966
-
long ret = 0;
3967
-
3968
-
if (issue_flags & IO_URING_F_NONBLOCK)
3969
-
return -EAGAIN;
3970
-
3971
-
if (sp->flags & SPLICE_F_FD_IN_FIXED)
3972
-
in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags);
3973
-
else
3974
-
in = io_file_get_normal(req, sp->splice_fd_in);
3975
-
if (!in) {
3976
-
ret = -EBADF;
3977
-
goto done;
3978
-
}
3979
-
3980
-
poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
3981
-
poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
3982
-
3983
-
if (sp->len)
3984
-
ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
3985
-
3986
-
if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3987
-
io_put_file(in);
3988
-
done:
3989
-
if (ret != sp->len)
3990
-
req_set_fail(req);
3991
-
io_req_set_res(req, ret, 0);
3992
-
return IOU_OK;
3993
3920
}
3994
3921
3995
3922
static int io_msg_ring_prep(struct io_kiocb *req,
···
7032
7157
file_slot->file_ptr = file_ptr;
7033
7158
}
7034
7159
7035
-
static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
7036
-
unsigned int issue_flags)
7160
+
inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
7161
+
unsigned int issue_flags)
7037
7162
{
7038
7163
struct io_ring_ctx *ctx = req->ctx;
7039
7164
struct file *file = NULL;
···
7056
7181
return file;
7057
7182
}
7058
7183
7059
-
static struct file *io_file_get_normal(struct io_kiocb *req, int fd)
7184
+
struct file *io_file_get_normal(struct io_kiocb *req, int fd)
7060
7185
{
7061
7186
struct file *file = fget(fd);
7062
7187
+19
io_uring/io_uring.h
+19
io_uring/io_uring.h
···
9
9
IOU_ISSUE_SKIP_COMPLETE = -EIOCBQUEUED,
10
10
};
11
11
12
+
static inline void req_set_fail(struct io_kiocb *req)
13
+
{
14
+
req->flags |= REQ_F_FAIL;
15
+
if (req->flags & REQ_F_CQE_SKIP) {
16
+
req->flags &= ~REQ_F_CQE_SKIP;
17
+
req->flags |= REQ_F_SKIP_LINK_CQES;
18
+
}
19
+
}
20
+
12
21
static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags)
13
22
{
14
23
req->cqe.res = res;
15
24
req->cqe.flags = cflags;
16
25
}
26
+
27
+
static inline void io_put_file(struct file *file)
28
+
{
29
+
if (file)
30
+
fput(file);
31
+
}
32
+
33
+
struct file *io_file_get_normal(struct io_kiocb *req, int fd);
34
+
struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
35
+
unsigned issue_flags);
17
36
18
37
#endif
+123
io_uring/splice.c
+123
io_uring/splice.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#include <linux/kernel.h>
3
+
#include <linux/errno.h>
4
+
#include <linux/fs.h>
5
+
#include <linux/file.h>
6
+
#include <linux/mm.h>
7
+
#include <linux/slab.h>
8
+
#include <linux/namei.h>
9
+
#include <linux/io_uring.h>
10
+
#include <linux/splice.h>
11
+
12
+
#include <uapi/linux/io_uring.h>
13
+
14
+
#include "io_uring_types.h"
15
+
#include "io_uring.h"
16
+
#include "splice.h"
17
+
18
+
struct io_splice {
19
+
struct file *file_out;
20
+
loff_t off_out;
21
+
loff_t off_in;
22
+
u64 len;
23
+
int splice_fd_in;
24
+
unsigned int flags;
25
+
};
26
+
27
+
static int __io_splice_prep(struct io_kiocb *req,
28
+
const struct io_uring_sqe *sqe)
29
+
{
30
+
struct io_splice *sp = io_kiocb_to_cmd(req);
31
+
unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
32
+
33
+
sp->len = READ_ONCE(sqe->len);
34
+
sp->flags = READ_ONCE(sqe->splice_flags);
35
+
if (unlikely(sp->flags & ~valid_flags))
36
+
return -EINVAL;
37
+
sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
38
+
return 0;
39
+
}
40
+
41
+
int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
42
+
{
43
+
if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
44
+
return -EINVAL;
45
+
return __io_splice_prep(req, sqe);
46
+
}
47
+
48
+
int io_tee(struct io_kiocb *req, unsigned int issue_flags)
49
+
{
50
+
struct io_splice *sp = io_kiocb_to_cmd(req);
51
+
struct file *out = sp->file_out;
52
+
unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
53
+
struct file *in;
54
+
long ret = 0;
55
+
56
+
if (issue_flags & IO_URING_F_NONBLOCK)
57
+
return -EAGAIN;
58
+
59
+
if (sp->flags & SPLICE_F_FD_IN_FIXED)
60
+
in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags);
61
+
else
62
+
in = io_file_get_normal(req, sp->splice_fd_in);
63
+
if (!in) {
64
+
ret = -EBADF;
65
+
goto done;
66
+
}
67
+
68
+
if (sp->len)
69
+
ret = do_tee(in, out, sp->len, flags);
70
+
71
+
if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
72
+
io_put_file(in);
73
+
done:
74
+
if (ret != sp->len)
75
+
req_set_fail(req);
76
+
io_req_set_res(req, ret, 0);
77
+
return IOU_OK;
78
+
}
79
+
80
+
int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
81
+
{
82
+
struct io_splice *sp = io_kiocb_to_cmd(req);
83
+
84
+
sp->off_in = READ_ONCE(sqe->splice_off_in);
85
+
sp->off_out = READ_ONCE(sqe->off);
86
+
return __io_splice_prep(req, sqe);
87
+
}
88
+
89
+
int io_splice(struct io_kiocb *req, unsigned int issue_flags)
90
+
{
91
+
struct io_splice *sp = io_kiocb_to_cmd(req);
92
+
struct file *out = sp->file_out;
93
+
unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
94
+
loff_t *poff_in, *poff_out;
95
+
struct file *in;
96
+
long ret = 0;
97
+
98
+
if (issue_flags & IO_URING_F_NONBLOCK)
99
+
return -EAGAIN;
100
+
101
+
if (sp->flags & SPLICE_F_FD_IN_FIXED)
102
+
in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags);
103
+
else
104
+
in = io_file_get_normal(req, sp->splice_fd_in);
105
+
if (!in) {
106
+
ret = -EBADF;
107
+
goto done;
108
+
}
109
+
110
+
poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
111
+
poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
112
+
113
+
if (sp->len)
114
+
ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
115
+
116
+
if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
117
+
io_put_file(in);
118
+
done:
119
+
if (ret != sp->len)
120
+
req_set_fail(req);
121
+
io_req_set_res(req, ret, 0);
122
+
return IOU_OK;
123
+
}
+7
io_uring/splice.h
+7
io_uring/splice.h
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
3
+
int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
4
+
int io_tee(struct io_kiocb *req, unsigned int issue_flags);
5
+
6
+
int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
7
+
int io_splice(struct io_kiocb *req, unsigned int issue_flags);