Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring: move uring_cmd handling to its own file
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+142
-124
+1
-1
io_uring/Makefile
+1
-1
io_uring/Makefile
+4
-123
io_uring/io_uring.c
+4
-123
io_uring/io_uring.c
···
99
99
#include "sync.h"
100
100
#include "advise.h"
101
101
#include "openclose.h"
102
+
#include "uring_cmd.h"
102
103
103
104
#define IORING_MAX_ENTRIES 32768
104
105
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
···
473
472
u32 flags;
474
473
int seq;
475
474
};
476
-
477
-
/*
478
-
* The URING_CMD payload starts at 'cmd' in the first sqe, and continues into
479
-
* the following sqe if SQE128 is used.
480
-
*/
481
-
#define uring_cmd_pdu_size(is_sqe128) \
482
-
((1 + !!(is_sqe128)) * sizeof(struct io_uring_sqe) - \
483
-
offsetof(struct io_uring_sqe, cmd))
484
475
485
476
struct io_op_def {
486
477
/* needs req->file assigned */
···
979
986
matched = io_match_linked(head);
980
987
}
981
988
return matched;
982
-
}
983
-
984
-
static inline bool req_has_async_data(struct io_kiocb *req)
985
-
{
986
-
return req->flags & REQ_F_ASYNC_DATA;
987
989
}
988
990
989
991
static inline void req_fail_link_node(struct io_kiocb *req, int res)
···
1731
1743
io_cqring_ev_posted(ctx);
1732
1744
}
1733
1745
1734
-
static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags)
1746
+
inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags)
1735
1747
{
1736
1748
if (issue_flags & IO_URING_F_COMPLETE_DEFER)
1737
1749
req->flags |= REQ_F_COMPLETE_INLINE;
···
2139
2151
}
2140
2152
}
2141
2153
2142
-
static void io_req_task_work_add(struct io_kiocb *req)
2154
+
void io_req_task_work_add(struct io_kiocb *req)
2143
2155
{
2144
2156
struct io_uring_task *tctx = req->task->io_uring;
2145
2157
···
3256
3268
}
3257
3269
}
3258
3270
3259
-
static inline bool io_alloc_async_data(struct io_kiocb *req)
3271
+
bool io_alloc_async_data(struct io_kiocb *req)
3260
3272
{
3261
3273
WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
3262
3274
req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
···
3700
3712
if (iovec)
3701
3713
kfree(iovec);
3702
3714
return ret;
3703
-
}
3704
-
3705
-
static void io_uring_cmd_work(struct io_kiocb *req, bool *locked)
3706
-
{
3707
-
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
3708
-
3709
-
ioucmd->task_work_cb(ioucmd);
3710
-
}
3711
-
3712
-
void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
3713
-
void (*task_work_cb)(struct io_uring_cmd *))
3714
-
{
3715
-
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
3716
-
3717
-
ioucmd->task_work_cb = task_work_cb;
3718
-
req->io_task_work.func = io_uring_cmd_work;
3719
-
io_req_task_work_add(req);
3720
-
}
3721
-
EXPORT_SYMBOL_GPL(io_uring_cmd_complete_in_task);
3722
-
3723
-
static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
3724
-
u64 extra1, u64 extra2)
3725
-
{
3726
-
req->extra1 = extra1;
3727
-
req->extra2 = extra2;
3728
-
req->flags |= REQ_F_CQE32_INIT;
3729
-
}
3730
-
3731
-
/*
3732
-
* Called by consumers of io_uring_cmd, if they originally returned
3733
-
* -EIOCBQUEUED upon receiving the command.
3734
-
*/
3735
-
void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2)
3736
-
{
3737
-
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
3738
-
3739
-
if (ret < 0)
3740
-
req_set_fail(req);
3741
-
3742
-
io_req_set_res(req, 0, ret);
3743
-
if (req->ctx->flags & IORING_SETUP_CQE32)
3744
-
io_req_set_cqe32_extra(req, res2, 0);
3745
-
__io_req_complete(req, 0);
3746
-
}
3747
-
EXPORT_SYMBOL_GPL(io_uring_cmd_done);
3748
-
3749
-
static int io_uring_cmd_prep_async(struct io_kiocb *req)
3750
-
{
3751
-
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
3752
-
size_t cmd_size;
3753
-
3754
-
cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128);
3755
-
3756
-
memcpy(req->async_data, ioucmd->cmd, cmd_size);
3757
-
return 0;
3758
-
}
3759
-
3760
-
static int io_uring_cmd_prep(struct io_kiocb *req,
3761
-
const struct io_uring_sqe *sqe)
3762
-
{
3763
-
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
3764
-
3765
-
if (sqe->rw_flags || sqe->__pad1)
3766
-
return -EINVAL;
3767
-
ioucmd->cmd = sqe->cmd;
3768
-
ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
3769
-
return 0;
3770
-
}
3771
-
3772
-
static int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
3773
-
{
3774
-
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
3775
-
struct io_ring_ctx *ctx = req->ctx;
3776
-
struct file *file = req->file;
3777
-
int ret;
3778
-
3779
-
if (!req->file->f_op->uring_cmd)
3780
-
return -EOPNOTSUPP;
3781
-
3782
-
if (ctx->flags & IORING_SETUP_SQE128)
3783
-
issue_flags |= IO_URING_F_SQE128;
3784
-
if (ctx->flags & IORING_SETUP_CQE32)
3785
-
issue_flags |= IO_URING_F_CQE32;
3786
-
if (ctx->flags & IORING_SETUP_IOPOLL)
3787
-
issue_flags |= IO_URING_F_IOPOLL;
3788
-
3789
-
if (req_has_async_data(req))
3790
-
ioucmd->cmd = req->async_data;
3791
-
3792
-
ret = file->f_op->uring_cmd(ioucmd, issue_flags);
3793
-
if (ret == -EAGAIN) {
3794
-
if (!req_has_async_data(req)) {
3795
-
if (io_alloc_async_data(req))
3796
-
return -ENOMEM;
3797
-
io_uring_cmd_prep_async(req);
3798
-
}
3799
-
return -EAGAIN;
3800
-
}
3801
-
3802
-
if (ret != -EIOCBQUEUED) {
3803
-
io_uring_cmd_done(ioucmd, ret, 0);
3804
-
return IOU_OK;
3805
-
}
3806
-
3807
-
return IOU_ISSUE_SKIP_COMPLETE;
3808
3715
}
3809
3716
3810
3717
static int io_msg_ring_prep(struct io_kiocb *req,
···
11415
11532
BUILD_BUG_ON(__REQ_F_LAST_BIT > 8 * sizeof(int));
11416
11533
11417
11534
BUILD_BUG_ON(sizeof(atomic_t) != sizeof(u32));
11418
-
11419
-
BUILD_BUG_ON(sizeof(struct io_uring_cmd) > 64);
11420
11535
11421
11536
for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
11422
11537
BUG_ON(!io_op_defs[i].prep);
+9
io_uring/io_uring.h
+9
io_uring/io_uring.h
···
25
25
req->cqe.flags = cflags;
26
26
}
27
27
28
+
static inline bool req_has_async_data(struct io_kiocb *req)
29
+
{
30
+
return req->flags & REQ_F_ASYNC_DATA;
31
+
}
32
+
28
33
static inline void io_put_file(struct file *file)
29
34
{
30
35
if (file)
···
58
53
lockdep_assert_held(&ctx->uring_lock);
59
54
}
60
55
56
+
void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
57
+
61
58
struct file *io_file_get_normal(struct io_kiocb *req, int fd);
62
59
struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
63
60
unsigned issue_flags);
···
72
65
void io_rsrc_node_switch(struct io_ring_ctx *ctx,
73
66
struct io_rsrc_data *data_to_kill);
74
67
bool io_is_uring_fops(struct file *file);
68
+
bool io_alloc_async_data(struct io_kiocb *req);
69
+
void io_req_task_work_add(struct io_kiocb *req);
75
70
76
71
#endif
+115
io_uring/uring_cmd.c
+115
io_uring/uring_cmd.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#include <linux/kernel.h>
3
+
#include <linux/errno.h>
4
+
#include <linux/file.h>
5
+
#include <linux/io_uring.h>
6
+
7
+
#include <uapi/linux/io_uring.h>
8
+
9
+
#include "io_uring_types.h"
10
+
#include "io_uring.h"
11
+
#include "uring_cmd.h"
12
+
13
+
static void io_uring_cmd_work(struct io_kiocb *req, bool *locked)
14
+
{
15
+
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
16
+
17
+
ioucmd->task_work_cb(ioucmd);
18
+
}
19
+
20
+
void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
21
+
void (*task_work_cb)(struct io_uring_cmd *))
22
+
{
23
+
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
24
+
25
+
ioucmd->task_work_cb = task_work_cb;
26
+
req->io_task_work.func = io_uring_cmd_work;
27
+
io_req_task_work_add(req);
28
+
}
29
+
EXPORT_SYMBOL_GPL(io_uring_cmd_complete_in_task);
30
+
31
+
static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
32
+
u64 extra1, u64 extra2)
33
+
{
34
+
req->extra1 = extra1;
35
+
req->extra2 = extra2;
36
+
req->flags |= REQ_F_CQE32_INIT;
37
+
}
38
+
39
+
/*
40
+
* Called by consumers of io_uring_cmd, if they originally returned
41
+
* -EIOCBQUEUED upon receiving the command.
42
+
*/
43
+
void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2)
44
+
{
45
+
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
46
+
47
+
if (ret < 0)
48
+
req_set_fail(req);
49
+
50
+
io_req_set_res(req, 0, ret);
51
+
if (req->ctx->flags & IORING_SETUP_CQE32)
52
+
io_req_set_cqe32_extra(req, res2, 0);
53
+
__io_req_complete(req, 0);
54
+
}
55
+
EXPORT_SYMBOL_GPL(io_uring_cmd_done);
56
+
57
+
int io_uring_cmd_prep_async(struct io_kiocb *req)
58
+
{
59
+
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
60
+
size_t cmd_size;
61
+
62
+
cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128);
63
+
64
+
memcpy(req->async_data, ioucmd->cmd, cmd_size);
65
+
return 0;
66
+
}
67
+
68
+
int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
69
+
{
70
+
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
71
+
72
+
if (sqe->rw_flags || sqe->__pad1)
73
+
return -EINVAL;
74
+
ioucmd->cmd = sqe->cmd;
75
+
ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
76
+
return 0;
77
+
}
78
+
79
+
int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
80
+
{
81
+
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
82
+
struct io_ring_ctx *ctx = req->ctx;
83
+
struct file *file = req->file;
84
+
int ret;
85
+
86
+
if (!req->file->f_op->uring_cmd)
87
+
return -EOPNOTSUPP;
88
+
89
+
if (ctx->flags & IORING_SETUP_SQE128)
90
+
issue_flags |= IO_URING_F_SQE128;
91
+
if (ctx->flags & IORING_SETUP_CQE32)
92
+
issue_flags |= IO_URING_F_CQE32;
93
+
if (ctx->flags & IORING_SETUP_IOPOLL)
94
+
issue_flags |= IO_URING_F_IOPOLL;
95
+
96
+
if (req_has_async_data(req))
97
+
ioucmd->cmd = req->async_data;
98
+
99
+
ret = file->f_op->uring_cmd(ioucmd, issue_flags);
100
+
if (ret == -EAGAIN) {
101
+
if (!req_has_async_data(req)) {
102
+
if (io_alloc_async_data(req))
103
+
return -ENOMEM;
104
+
io_uring_cmd_prep_async(req);
105
+
}
106
+
return -EAGAIN;
107
+
}
108
+
109
+
if (ret != -EIOCBQUEUED) {
110
+
io_uring_cmd_done(ioucmd, ret, 0);
111
+
return IOU_OK;
112
+
}
113
+
114
+
return IOU_ISSUE_SKIP_COMPLETE;
115
+
}
+13
io_uring/uring_cmd.h
+13
io_uring/uring_cmd.h
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
3
+
int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags);
4
+
int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
5
+
int io_uring_cmd_prep_async(struct io_kiocb *req);
6
+
7
+
/*
8
+
* The URING_CMD payload starts at 'cmd' in the first sqe, and continues into
9
+
* the following sqe if SQE128 is used.
10
+
*/
11
+
#define uring_cmd_pdu_size(is_sqe128) \
12
+
((1 + !!(is_sqe128)) * sizeof(struct io_uring_sqe) - \
13
+
offsetof(struct io_uring_sqe, cmd))