Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

io_uring: move struct io_kiocb from task_struct to io_uring_task

Rather than store the task_struct itself in struct io_kiocb, store
the io_uring specific task_struct. The life times are the same in terms
of io_uring, and this avoids doing some dereferences through the
task_struct. For the hot path of putting local task references, we can
deref req->tctx instead, which we'll need anyway in that function
regardless of whether it's local or remote references.

This is mostly straight forward, except the original task PF_EXITING
check needs a bit of tweaking. task_work is _always_ run from the
originating task, except in the fallback case, where it's run from a
kernel thread. Replace the potentially racy (in case of fallback work)
checks for req->task->flags with current->flags. It's either the still
the original task, in which case PF_EXITING will be sane, or it has
PF_KTHREAD set, in which case it's fallback work. Both cases should
prevent moving forward with the given request.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

+48 -36
+1 -1
include/linux/io_uring/cmd.h
··· 110 110 111 111 static inline struct task_struct *io_uring_cmd_get_task(struct io_uring_cmd *cmd) 112 112 { 113 - return cmd_to_io_kiocb(cmd)->task; 113 + return cmd_to_io_kiocb(cmd)->tctx->task; 114 114 } 115 115 116 116 #endif /* _LINUX_IO_URING_CMD_H */
+2 -1
include/linux/io_uring_types.h
··· 84 84 /* submission side */ 85 85 int cached_refs; 86 86 const struct io_ring_ctx *last; 87 + struct task_struct *task; 87 88 struct io_wq *io_wq; 88 89 struct file *registered_rings[IO_RINGFD_REG_MAX]; 89 90 ··· 626 625 struct io_cqe cqe; 627 626 628 627 struct io_ring_ctx *ctx; 629 - struct task_struct *task; 628 + struct io_uring_task *tctx; 630 629 631 630 union { 632 631 /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
+1 -1
io_uring/cancel.c
··· 205 205 .opcode = cancel->opcode, 206 206 .seq = atomic_inc_return(&req->ctx->cancel_seq), 207 207 }; 208 - struct io_uring_task *tctx = req->task->io_uring; 208 + struct io_uring_task *tctx = req->tctx; 209 209 int ret; 210 210 211 211 if (cd.flags & IORING_ASYNC_CANCEL_FD) {
+1 -1
io_uring/fdinfo.c
··· 203 203 204 204 hlist_for_each_entry(req, &hb->list, hash_node) 205 205 seq_printf(m, " op=%d, task_works=%d\n", req->opcode, 206 - task_work_pending(req->task)); 206 + task_work_pending(req->tctx->task)); 207 207 } 208 208 209 209 if (has_lock)
+15 -19
io_uring/io_uring.c
··· 206 206 { 207 207 bool matched; 208 208 209 - if (tctx && head->task->io_uring != tctx) 209 + if (tctx && head->tctx != tctx) 210 210 return false; 211 211 if (cancel_all) 212 212 return true; ··· 407 407 kfree(req->apoll); 408 408 req->apoll = NULL; 409 409 } 410 - if (req->flags & REQ_F_INFLIGHT) { 411 - struct io_uring_task *tctx = req->task->io_uring; 412 - 413 - atomic_dec(&tctx->inflight_tracked); 414 - } 410 + if (req->flags & REQ_F_INFLIGHT) 411 + atomic_dec(&req->tctx->inflight_tracked); 415 412 if (req->flags & REQ_F_CREDS) 416 413 put_cred(req->creds); 417 414 if (req->flags & REQ_F_ASYNC_DATA) { ··· 422 425 { 423 426 if (!(req->flags & REQ_F_INFLIGHT)) { 424 427 req->flags |= REQ_F_INFLIGHT; 425 - atomic_inc(&req->task->io_uring->inflight_tracked); 428 + atomic_inc(&req->tctx->inflight_tracked); 426 429 } 427 430 } 428 431 ··· 511 514 static void io_queue_iowq(struct io_kiocb *req) 512 515 { 513 516 struct io_kiocb *link = io_prep_linked_timeout(req); 514 - struct io_uring_task *tctx = req->task->io_uring; 517 + struct io_uring_task *tctx = req->tctx; 515 518 516 519 BUG_ON(!tctx); 517 520 BUG_ON(!tctx->io_wq); ··· 526 529 * procedure rather than attempt to run this request (or create a new 527 530 * worker for it). 528 531 */ 529 - if (WARN_ON_ONCE(!same_thread_group(req->task, current))) 532 + if (WARN_ON_ONCE(!same_thread_group(tctx->task, current))) 530 533 atomic_or(IO_WQ_WORK_CANCEL, &req->work.flags); 531 534 532 535 trace_io_uring_queue_async_work(req, io_wq_is_hashed(&req->work)); ··· 675 678 } 676 679 677 680 /* must to be called somewhat shortly after putting a request */ 678 - static inline void io_put_task(struct task_struct *task) 681 + static inline void io_put_task(struct io_kiocb *req) 679 682 { 680 - struct io_uring_task *tctx = task->io_uring; 683 + struct io_uring_task *tctx = req->tctx; 681 684 682 - if (likely(task == current)) { 685 + if (likely(tctx->task == current)) { 683 686 tctx->cached_refs++; 684 687 } else { 685 688 percpu_counter_sub(&tctx->inflight, 1); 686 689 if (unlikely(atomic_read(&tctx->in_cancel))) 687 690 wake_up(&tctx->wait); 688 - put_task_struct(task); 691 + put_task_struct(tctx->task); 689 692 } 690 693 } 691 694 ··· 1204 1207 1205 1208 static void io_req_normal_work_add(struct io_kiocb *req) 1206 1209 { 1207 - struct io_uring_task *tctx = req->task->io_uring; 1210 + struct io_uring_task *tctx = req->tctx; 1208 1211 struct io_ring_ctx *ctx = req->ctx; 1209 1212 1210 1213 /* task_work already pending, we're done */ ··· 1223 1226 return; 1224 1227 } 1225 1228 1226 - if (likely(!task_work_add(req->task, &tctx->task_work, ctx->notify_method))) 1229 + if (likely(!task_work_add(tctx->task, &tctx->task_work, ctx->notify_method))) 1227 1230 return; 1228 1231 1229 1232 io_fallback_tw(tctx, false); ··· 1340 1343 void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts) 1341 1344 { 1342 1345 io_tw_lock(req->ctx, ts); 1343 - /* req->task == current here, checking PF_EXITING is safe */ 1344 - if (unlikely(req->task->flags & PF_EXITING)) 1346 + if (unlikely(io_should_terminate_tw())) 1345 1347 io_req_defer_failed(req, -EFAULT); 1346 1348 else if (req->flags & REQ_F_FORCE_ASYNC) 1347 1349 io_queue_iowq(req); ··· 1399 1403 } 1400 1404 io_put_file(req); 1401 1405 io_req_put_rsrc_nodes(req); 1402 - io_put_task(req->task); 1406 + io_put_task(req); 1403 1407 1404 1408 node = req->comp_list.next; 1405 1409 io_req_add_to_cache(req, ctx); ··· 2015 2019 req->flags = (__force io_req_flags_t) sqe_flags; 2016 2020 req->cqe.user_data = READ_ONCE(sqe->user_data); 2017 2021 req->file = NULL; 2018 - req->task = current; 2022 + req->tctx = current->io_uring; 2019 2023 req->cancel_seq_set = false; 2020 2024 2021 2025 if (unlikely(opcode >= IORING_OP_LAST)) {
+13
io_uring/io_uring.h
··· 426 426 ctx->submitter_task == current); 427 427 } 428 428 429 + /* 430 + * Terminate the request if either of these conditions are true: 431 + * 432 + * 1) It's being executed by the original task, but that task is marked 433 + * with PF_EXITING as it's exiting. 434 + * 2) PF_KTHREAD is set, in which case the invoker of the task_work is 435 + * our fallback task_work. 436 + */ 437 + static inline bool io_should_terminate_tw(void) 438 + { 439 + return current->flags & (PF_KTHREAD | PF_EXITING); 440 + } 441 + 429 442 static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res) 430 443 { 431 444 io_req_set_res(req, res, 0);
+2 -2
io_uring/msg_ring.c
··· 89 89 static int io_msg_remote_post(struct io_ring_ctx *ctx, struct io_kiocb *req, 90 90 int res, u32 cflags, u64 user_data) 91 91 { 92 - req->task = READ_ONCE(ctx->submitter_task); 93 - if (!req->task) { 92 + req->tctx = READ_ONCE(ctx->submitter_task->io_uring); 93 + if (!req->tctx) { 94 94 kmem_cache_free(req_cachep, req); 95 95 return -EOWNERDEAD; 96 96 }
+2 -2
io_uring/notif.c
··· 89 89 90 90 /* make sure all noifications can be finished in the same task_work */ 91 91 if (unlikely(notif->ctx != prev_notif->ctx || 92 - notif->task != prev_notif->task)) 92 + notif->tctx != prev_notif->tctx)) 93 93 return -EEXIST; 94 94 95 95 nd->head = prev_nd->head; ··· 115 115 notif->opcode = IORING_OP_NOP; 116 116 notif->flags = 0; 117 117 notif->file = NULL; 118 - notif->task = current; 118 + notif->tctx = current->io_uring; 119 119 io_get_task_refs(1); 120 120 notif->file_node = NULL; 121 121 notif->buf_node = NULL;
+1 -2
io_uring/poll.c
··· 224 224 { 225 225 int v; 226 226 227 - /* req->task == current here, checking PF_EXITING is safe */ 228 - if (unlikely(req->task->flags & PF_EXITING)) 227 + if (unlikely(io_should_terminate_tw())) 229 228 return -ECANCELED; 230 229 231 230 do {
+1 -1
io_uring/rw.c
··· 435 435 * Play it safe and assume not safe to re-import and reissue if we're 436 436 * not in the original thread group (or in task context). 437 437 */ 438 - if (!same_thread_group(req->task, current) || !in_task()) 438 + if (!same_thread_group(req->tctx->task, current) || !in_task()) 439 439 return false; 440 440 return true; 441 441 }
+1
io_uring/tctx.c
··· 81 81 return ret; 82 82 } 83 83 84 + tctx->task = task; 84 85 xa_init(&tctx->xa); 85 86 init_waitqueue_head(&tctx->wait); 86 87 atomic_set(&tctx->in_cancel, 0);
+6 -4
io_uring/timeout.c
··· 300 300 { 301 301 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 302 302 struct io_kiocb *prev = timeout->prev; 303 - int ret = -ENOENT; 303 + int ret; 304 304 305 305 if (prev) { 306 - if (!(req->task->flags & PF_EXITING)) { 306 + if (!io_should_terminate_tw()) { 307 307 struct io_cancel_data cd = { 308 308 .ctx = req->ctx, 309 309 .data = prev->cqe.user_data, 310 310 }; 311 311 312 - ret = io_try_cancel(req->task->io_uring, &cd, 0); 312 + ret = io_try_cancel(req->tctx, &cd, 0); 313 + } else { 314 + ret = -ECANCELED; 313 315 } 314 316 io_req_set_res(req, ret ?: -ETIME, 0); 315 317 io_req_task_complete(req, ts); ··· 645 643 { 646 644 struct io_kiocb *req; 647 645 648 - if (tctx && head->task->io_uring != tctx) 646 + if (tctx && head->tctx != tctx) 649 647 return false; 650 648 if (cancel_all) 651 649 return true;
+1 -1
io_uring/uring_cmd.c
··· 61 61 struct io_uring_cmd); 62 62 struct file *file = req->file; 63 63 64 - if (!cancel_all && req->task->io_uring != tctx) 64 + if (!cancel_all && req->tctx != tctx) 65 65 continue; 66 66 67 67 if (cmd->flags & IORING_URING_CMD_CANCELABLE) {
+1 -1
io_uring/waitid.c
··· 331 331 hlist_add_head(&req->hash_node, &ctx->waitid_list); 332 332 333 333 init_waitqueue_func_entry(&iwa->wo.child_wait, io_waitid_wait); 334 - iwa->wo.child_wait.private = req->task; 334 + iwa->wo.child_wait.private = req->tctx->task; 335 335 iw->head = &current->signal->wait_chldexit; 336 336 add_wait_queue(iw->head, &iwa->wo.child_wait); 337 337