Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-6.2-2023-01-20' of git://git.kernel.dk/linux
Pull io_uring fixes from Jens Axboe:
"Fixes for the MSG_RING opcode. Nothing really major:
- Fix an overflow missing serialization around posting CQEs to the
target ring (me)
- Disable MSG_RING on a ring that isn't enabled yet. There's nothing
really wrong with allowing it, but 1) it's somewhat odd as nobody
can receive them yet, and 2) it means that using the right delivery
mechanism might change. As nobody should be sending CQEs to a ring
that isn't enabled yet, let's just disable it (Pavel)
- Tweak to when we decide to post remotely or not for MSG_RING
(Pavel)"
* tag 'io_uring-6.2-2023-01-20' of git://git.kernel.dk/linux:
io_uring/msg_ring: fix remote queue to disabled ring
io_uring/msg_ring: fix flagging remote execution
io_uring/msg_ring: fix missing lock on overflow for IOPOLL
io_uring/msg_ring: move double lock/unlock helpers higher up
+100
-66
+2
-2
io_uring/io_uring.c
+2
-2
io_uring/io_uring.c
···
3674
3674
3675
3675
if (ctx->flags & IORING_SETUP_SINGLE_ISSUER
3676
3676
&& !(ctx->flags & IORING_SETUP_R_DISABLED))
3677
-
ctx->submitter_task = get_task_struct(current);
3677
+
WRITE_ONCE(ctx->submitter_task, get_task_struct(current));
3678
3678
3679
3679
file = io_uring_get_file(ctx);
3680
3680
if (IS_ERR(file)) {
···
3868
3868
return -EBADFD;
3869
3869
3870
3870
if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task)
3871
-
ctx->submitter_task = get_task_struct(current);
3871
+
WRITE_ONCE(ctx->submitter_task, get_task_struct(current));
3872
3872
3873
3873
if (ctx->restrictions.registered)
3874
3874
ctx->restricted = 1;
+98
-64
io_uring/msg_ring.c
+98
-64
io_uring/msg_ring.c
···
25
25
u32 flags;
26
26
};
27
27
28
-
void io_msg_ring_cleanup(struct io_kiocb *req)
29
-
{
30
-
struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
31
-
32
-
if (WARN_ON_ONCE(!msg->src_file))
33
-
return;
34
-
35
-
fput(msg->src_file);
36
-
msg->src_file = NULL;
37
-
}
38
-
39
-
static void io_msg_tw_complete(struct callback_head *head)
40
-
{
41
-
struct io_msg *msg = container_of(head, struct io_msg, tw);
42
-
struct io_kiocb *req = cmd_to_io_kiocb(msg);
43
-
struct io_ring_ctx *target_ctx = req->file->private_data;
44
-
int ret = 0;
45
-
46
-
if (current->flags & PF_EXITING)
47
-
ret = -EOWNERDEAD;
48
-
else if (!io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0))
49
-
ret = -EOVERFLOW;
50
-
51
-
if (ret < 0)
52
-
req_set_fail(req);
53
-
io_req_queue_tw_complete(req, ret);
54
-
}
55
-
56
-
static int io_msg_ring_data(struct io_kiocb *req)
57
-
{
58
-
struct io_ring_ctx *target_ctx = req->file->private_data;
59
-
struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
60
-
61
-
if (msg->src_fd || msg->dst_fd || msg->flags)
62
-
return -EINVAL;
63
-
64
-
if (target_ctx->task_complete && current != target_ctx->submitter_task) {
65
-
init_task_work(&msg->tw, io_msg_tw_complete);
66
-
if (task_work_add(target_ctx->submitter_task, &msg->tw,
67
-
TWA_SIGNAL_NO_IPI))
68
-
return -EOWNERDEAD;
69
-
70
-
atomic_or(IORING_SQ_TASKRUN, &target_ctx->rings->sq_flags);
71
-
return IOU_ISSUE_SKIP_COMPLETE;
72
-
}
73
-
74
-
if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0))
75
-
return 0;
76
-
77
-
return -EOVERFLOW;
78
-
}
79
-
80
-
static void io_double_unlock_ctx(struct io_ring_ctx *octx,
81
-
unsigned int issue_flags)
28
+
static void io_double_unlock_ctx(struct io_ring_ctx *octx)
82
29
{
83
30
mutex_unlock(&octx->uring_lock);
84
31
}
···
45
98
}
46
99
mutex_lock(&octx->uring_lock);
47
100
return 0;
101
+
}
102
+
103
+
void io_msg_ring_cleanup(struct io_kiocb *req)
104
+
{
105
+
struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
106
+
107
+
if (WARN_ON_ONCE(!msg->src_file))
108
+
return;
109
+
110
+
fput(msg->src_file);
111
+
msg->src_file = NULL;
112
+
}
113
+
114
+
static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)
115
+
{
116
+
if (!target_ctx->task_complete)
117
+
return false;
118
+
return current != target_ctx->submitter_task;
119
+
}
120
+
121
+
static int io_msg_exec_remote(struct io_kiocb *req, task_work_func_t func)
122
+
{
123
+
struct io_ring_ctx *ctx = req->file->private_data;
124
+
struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
125
+
struct task_struct *task = READ_ONCE(ctx->submitter_task);
126
+
127
+
if (unlikely(!task))
128
+
return -EOWNERDEAD;
129
+
130
+
init_task_work(&msg->tw, func);
131
+
if (task_work_add(ctx->submitter_task, &msg->tw, TWA_SIGNAL))
132
+
return -EOWNERDEAD;
133
+
134
+
return IOU_ISSUE_SKIP_COMPLETE;
135
+
}
136
+
137
+
static void io_msg_tw_complete(struct callback_head *head)
138
+
{
139
+
struct io_msg *msg = container_of(head, struct io_msg, tw);
140
+
struct io_kiocb *req = cmd_to_io_kiocb(msg);
141
+
struct io_ring_ctx *target_ctx = req->file->private_data;
142
+
int ret = 0;
143
+
144
+
if (current->flags & PF_EXITING) {
145
+
ret = -EOWNERDEAD;
146
+
} else {
147
+
/*
148
+
* If the target ring is using IOPOLL mode, then we need to be
149
+
* holding the uring_lock for posting completions. Other ring
150
+
* types rely on the regular completion locking, which is
151
+
* handled while posting.
152
+
*/
153
+
if (target_ctx->flags & IORING_SETUP_IOPOLL)
154
+
mutex_lock(&target_ctx->uring_lock);
155
+
if (!io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0))
156
+
ret = -EOVERFLOW;
157
+
if (target_ctx->flags & IORING_SETUP_IOPOLL)
158
+
mutex_unlock(&target_ctx->uring_lock);
159
+
}
160
+
161
+
if (ret < 0)
162
+
req_set_fail(req);
163
+
io_req_queue_tw_complete(req, ret);
164
+
}
165
+
166
+
static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags)
167
+
{
168
+
struct io_ring_ctx *target_ctx = req->file->private_data;
169
+
struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
170
+
int ret;
171
+
172
+
if (msg->src_fd || msg->dst_fd || msg->flags)
173
+
return -EINVAL;
174
+
if (target_ctx->flags & IORING_SETUP_R_DISABLED)
175
+
return -EBADFD;
176
+
177
+
if (io_msg_need_remote(target_ctx))
178
+
return io_msg_exec_remote(req, io_msg_tw_complete);
179
+
180
+
ret = -EOVERFLOW;
181
+
if (target_ctx->flags & IORING_SETUP_IOPOLL) {
182
+
if (unlikely(io_double_lock_ctx(target_ctx, issue_flags)))
183
+
return -EAGAIN;
184
+
if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0))
185
+
ret = 0;
186
+
io_double_unlock_ctx(target_ctx);
187
+
} else {
188
+
if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0))
189
+
ret = 0;
190
+
}
191
+
return ret;
48
192
}
49
193
50
194
static struct file *io_msg_grab_file(struct io_kiocb *req, unsigned int issue_flags)
···
186
148
if (!io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0))
187
149
ret = -EOVERFLOW;
188
150
out_unlock:
189
-
io_double_unlock_ctx(target_ctx, issue_flags);
151
+
io_double_unlock_ctx(target_ctx);
190
152
return ret;
191
153
}
192
154
···
212
174
213
175
if (target_ctx == ctx)
214
176
return -EINVAL;
177
+
if (target_ctx->flags & IORING_SETUP_R_DISABLED)
178
+
return -EBADFD;
215
179
if (!src_file) {
216
180
src_file = io_msg_grab_file(req, issue_flags);
217
181
if (!src_file)
···
222
182
req->flags |= REQ_F_NEED_CLEANUP;
223
183
}
224
184
225
-
if (target_ctx->task_complete && current != target_ctx->submitter_task) {
226
-
init_task_work(&msg->tw, io_msg_tw_fd_complete);
227
-
if (task_work_add(target_ctx->submitter_task, &msg->tw,
228
-
TWA_SIGNAL))
229
-
return -EOWNERDEAD;
230
-
231
-
return IOU_ISSUE_SKIP_COMPLETE;
232
-
}
185
+
if (io_msg_need_remote(target_ctx))
186
+
return io_msg_exec_remote(req, io_msg_tw_fd_complete);
233
187
return io_msg_install_complete(req, issue_flags);
234
188
}
235
189
···
258
224
259
225
switch (msg->cmd) {
260
226
case IORING_MSG_DATA:
261
-
ret = io_msg_ring_data(req);
227
+
ret = io_msg_ring_data(req, issue_flags);
262
228
break;
263
229
case IORING_MSG_SEND_FD:
264
230
ret = io_msg_send_fd(req, issue_flags);