Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring: unify task_work cancelation checks
Rather than do per-tw checking, which needs to dip into the task_struct
for checking flags, do it upfront before running task_work. This places
a 'cancel' member in io_tw_token_t, which is assigned before running
task_work for that given ctx.
This is both more efficient in doing it upfront rather than for every
task_work, and it means that io_should_terminate_tw() can be made
private in io_uring.c rather than need to be called by various
callbacks of task_work.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+24
-23
+1
include/linux/io_uring_types.h
+1
include/linux/io_uring_types.h
+20
-7
io_uring/io_uring.c
+20
-7
io_uring/io_uring.c
···
265
265
complete(&ctx->ref_comp);
266
266
}
267
267
268
+
/*
269
+
* Terminate the request if either of these conditions are true:
270
+
*
271
+
* 1) It's being executed by the original task, but that task is marked
272
+
* with PF_EXITING as it's exiting.
273
+
* 2) PF_KTHREAD is set, in which case the invoker of the task_work is
274
+
* our fallback task_work.
275
+
* 3) The ring has been closed and is going away.
276
+
*/
277
+
static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx)
278
+
{
279
+
return (current->flags & (PF_EXITING | PF_KTHREAD)) || percpu_ref_is_dying(&ctx->refs);
280
+
}
281
+
268
282
static __cold void io_fallback_req_func(struct work_struct *work)
269
283
{
270
284
struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
···
289
275
290
276
percpu_ref_get(&ctx->refs);
291
277
mutex_lock(&ctx->uring_lock);
292
-
llist_for_each_entry_safe(req, tmp, node, io_task_work.node)
278
+
llist_for_each_entry_safe(req, tmp, node, io_task_work.node) {
279
+
ts.cancel = io_should_terminate_tw(req->ctx);
293
280
req->io_task_work.func(req, ts);
281
+
}
294
282
io_submit_flush_completions(ctx);
295
283
mutex_unlock(&ctx->uring_lock);
296
284
percpu_ref_put(&ctx->refs);
···
1163
1147
ctx = req->ctx;
1164
1148
mutex_lock(&ctx->uring_lock);
1165
1149
percpu_ref_get(&ctx->refs);
1150
+
ts.cancel = io_should_terminate_tw(ctx);
1166
1151
}
1167
1152
INDIRECT_CALL_2(req->io_task_work.func,
1168
1153
io_poll_task_func, io_req_rw_complete,
···
1221
1204
unsigned int *count)
1222
1205
{
1223
1206
struct llist_node *node;
1224
-
1225
-
if (unlikely(current->flags & PF_EXITING)) {
1226
-
io_fallback_tw(tctx, true);
1227
-
return NULL;
1228
-
}
1229
1207
1230
1208
node = llist_del_all(&tctx->task_list);
1231
1209
if (node) {
···
1411
1399
if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
1412
1400
atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1413
1401
again:
1402
+
tw.cancel = io_should_terminate_tw(ctx);
1414
1403
min_events -= ret;
1415
1404
ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events);
1416
1405
if (ctx->retry_llist.first)
···
1471
1458
struct io_ring_ctx *ctx = req->ctx;
1472
1459
1473
1460
io_tw_lock(ctx, tw);
1474
-
if (unlikely(io_should_terminate_tw(ctx)))
1461
+
if (unlikely(tw.cancel))
1475
1462
io_req_defer_failed(req, -EFAULT);
1476
1463
else if (req->flags & REQ_F_FORCE_ASYNC)
1477
1464
io_queue_iowq(req);
-13
io_uring/io_uring.h
-13
io_uring/io_uring.h
···
558
558
ctx->submitter_task == current);
559
559
}
560
560
561
-
/*
562
-
* Terminate the request if either of these conditions are true:
563
-
*
564
-
* 1) It's being executed by the original task, but that task is marked
565
-
* with PF_EXITING as it's exiting.
566
-
* 2) PF_KTHREAD is set, in which case the invoker of the task_work is
567
-
* our fallback task_work.
568
-
*/
569
-
static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx)
570
-
{
571
-
return (current->flags & (PF_KTHREAD | PF_EXITING)) || percpu_ref_is_dying(&ctx->refs);
572
-
}
573
-
574
561
static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res)
575
562
{
576
563
io_req_set_res(req, res, 0);
+1
-1
io_uring/poll.c
+1
-1
io_uring/poll.c
+1
-1
io_uring/timeout.c
+1
-1
io_uring/timeout.c
+1
-1
io_uring/uring_cmd.c
+1
-1
io_uring/uring_cmd.c
···
118
118
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
119
119
unsigned int flags = IO_URING_F_COMPLETE_DEFER;
120
120
121
-
if (io_should_terminate_tw(req->ctx))
121
+
if (unlikely(tw.cancel))
122
122
flags |= IO_URING_F_TASK_DEAD;
123
123
124
124
/* task_work executor checks the deffered list completion */