Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

io_uring: move timeout opcodes and handling into its own file

Signed-off-by: Jens Axboe <axboe@kernel.dk>

+701 -660
+1 -1
io_uring/Makefile
··· 5 5 obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \ 6 6 sync.o advise.o filetable.o \ 7 7 openclose.o uring_cmd.o epoll.o \ 8 - statx.o net.o msg_ring.o 8 + statx.o net.o msg_ring.o timeout.o 9 9 obj-$(CONFIG_IO_WQ) += io-wq.o
+9 -658
io_uring/io_uring.c
··· 105 105 #include "statx.h" 106 106 #include "net.h" 107 107 #include "msg_ring.h" 108 + #include "timeout.h" 108 109 109 110 #define IORING_MAX_ENTRIES 32768 110 111 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) ··· 289 288 bool update_user_data; 290 289 }; 291 290 292 - struct io_timeout_data { 293 - struct io_kiocb *req; 294 - struct hrtimer timer; 295 - struct timespec64 ts; 296 - enum hrtimer_mode mode; 297 - u32 flags; 298 - }; 299 - 300 291 struct io_cancel { 301 292 struct file *file; 302 293 u64 addr; 303 294 u32 flags; 304 295 s32 fd; 305 - }; 306 - 307 - struct io_timeout { 308 - struct file *file; 309 - u32 off; 310 - u32 target_seq; 311 - struct list_head list; 312 - /* head of the link, used by linked timeouts only */ 313 - struct io_kiocb *head; 314 - /* for linked completions */ 315 - struct io_kiocb *prev; 316 - }; 317 - 318 - struct io_timeout_rem { 319 - struct file *file; 320 - u64 addr; 321 - 322 - /* timeout update */ 323 - struct timespec64 ts; 324 - u32 flags; 325 - bool ltimeout; 326 296 }; 327 297 328 298 struct io_rw { ··· 360 388 u32 seq; 361 389 }; 362 390 363 - struct io_cancel_data { 364 - struct io_ring_ctx *ctx; 365 - union { 366 - u64 data; 367 - struct file *file; 368 - }; 369 - u32 flags; 370 - int seq; 371 - }; 372 - 373 391 struct io_op_def { 374 392 /* needs req->file assigned */ 375 393 unsigned needs_file : 1; ··· 398 436 #define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL) 399 437 #define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK) 400 438 401 - static bool io_disarm_next(struct io_kiocb *req); 402 439 static void io_uring_del_tctx_node(unsigned long index); 403 440 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, 404 441 struct task_struct *task, ··· 405 444 static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd); 406 445 407 446 static void io_dismantle_req(struct io_kiocb *req); 408 - static void io_queue_linked_timeout(struct io_kiocb *req); 409 447 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, 410 448 struct io_uring_rsrc_update2 *up, 411 449 unsigned nr_args); ··· 416 456 static void __io_submit_flush_completions(struct io_ring_ctx *ctx); 417 457 static int io_req_prep_async(struct io_kiocb *req); 418 458 419 - static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer); 420 459 static void io_eventfd_signal(struct io_ring_ctx *ctx); 421 - static void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags); 422 460 423 461 static struct kmem_cache *req_cachep; 424 462 ··· 566 608 *locked = true; 567 609 } 568 610 } 569 - 570 - #define io_for_each_link(pos, head) \ 571 - for (pos = (head); pos; pos = pos->link) 572 611 573 612 static inline void io_submit_flush_completions(struct io_ring_ctx *ctx) 574 613 { ··· 758 803 io_ring_submit_unlock(ctx, issue_flags); 759 804 } 760 805 761 - static bool io_match_task(struct io_kiocb *head, struct task_struct *task, 762 - bool cancel_all) 763 - __must_hold(&req->ctx->timeout_lock) 764 - { 765 - struct io_kiocb *req; 766 - 767 - if (task && head->task != task) 768 - return false; 769 - if (cancel_all) 770 - return true; 771 - 772 - io_for_each_link(req, head) { 773 - if (req->flags & REQ_F_INFLIGHT) 774 - return true; 775 - } 776 - return false; 777 - } 778 - 779 806 static bool io_match_linked(struct io_kiocb *head) 780 807 { 781 808 struct io_kiocb *req; ··· 812 875 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); 813 876 814 877 complete(&ctx->ref_comp); 815 - } 816 - 817 - static inline bool io_is_timeout_noseq(struct io_kiocb *req) 818 - { 819 - struct io_timeout *timeout = io_kiocb_to_cmd(req); 820 - 821 - return !timeout->off; 822 878 } 823 879 824 880 static __cold void io_fallback_req_func(struct work_struct *work) ··· 1050 1120 io_queue_linked_timeout(link); 1051 1121 } 1052 1122 1053 - static void io_kill_timeout(struct io_kiocb *req, int status) 1054 - __must_hold(&req->ctx->completion_lock) 1055 - __must_hold(&req->ctx->timeout_lock) 1056 - { 1057 - struct io_timeout_data *io = req->async_data; 1058 - 1059 - if (hrtimer_try_to_cancel(&io->timer) != -1) { 1060 - struct io_timeout *timeout = io_kiocb_to_cmd(req); 1061 - 1062 - if (status) 1063 - req_set_fail(req); 1064 - atomic_set(&req->ctx->cq_timeouts, 1065 - atomic_read(&req->ctx->cq_timeouts) + 1); 1066 - list_del_init(&timeout->list); 1067 - io_req_tw_post_queue(req, status, 0); 1068 - } 1069 - } 1070 - 1071 1123 static __cold void io_queue_deferred(struct io_ring_ctx *ctx) 1072 1124 { 1073 1125 while (!list_empty(&ctx->defer_list)) { ··· 1062 1150 io_req_task_queue(de->req); 1063 1151 kfree(de); 1064 1152 } 1065 - } 1066 - 1067 - static __cold void io_flush_timeouts(struct io_ring_ctx *ctx) 1068 - __must_hold(&ctx->completion_lock) 1069 - { 1070 - u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); 1071 - struct io_timeout *timeout, *tmp; 1072 - 1073 - spin_lock_irq(&ctx->timeout_lock); 1074 - list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { 1075 - struct io_kiocb *req = cmd_to_io_kiocb(timeout); 1076 - u32 events_needed, events_got; 1077 - 1078 - if (io_is_timeout_noseq(req)) 1079 - break; 1080 - 1081 - /* 1082 - * Since seq can easily wrap around over time, subtract 1083 - * the last seq at which timeouts were flushed before comparing. 1084 - * Assuming not more than 2^31-1 events have happened since, 1085 - * these subtractions won't have wrapped, so we can check if 1086 - * target is in [last_seq, current_seq] by comparing the two. 1087 - */ 1088 - events_needed = timeout->target_seq - ctx->cq_last_tm_flush; 1089 - events_got = seq - ctx->cq_last_tm_flush; 1090 - if (events_got < events_needed) 1091 - break; 1092 - 1093 - io_kill_timeout(req, 0); 1094 - } 1095 - ctx->cq_last_tm_flush = seq; 1096 - spin_unlock_irq(&ctx->timeout_lock); 1097 1153 } 1098 1154 1099 1155 static void __io_commit_cqring_flush(struct io_ring_ctx *ctx) ··· 1465 1585 } 1466 1586 } 1467 1587 1468 - static void __io_req_complete_post(struct io_kiocb *req) 1588 + void __io_req_complete_post(struct io_kiocb *req) 1469 1589 { 1470 1590 if (!(req->flags & REQ_F_CQE_SKIP)) 1471 1591 __io_fill_cqe_req(req->ctx, req); 1472 1592 __io_req_complete_put(req); 1473 1593 } 1474 1594 1475 - static void io_req_complete_post(struct io_kiocb *req) 1595 + void io_req_complete_post(struct io_kiocb *req) 1476 1596 { 1477 1597 struct io_ring_ctx *ctx = req->ctx; 1478 1598 ··· 1597 1717 io_put_file(req->file); 1598 1718 } 1599 1719 1600 - static __cold void io_free_req(struct io_kiocb *req) 1720 + __cold void io_free_req(struct io_kiocb *req) 1601 1721 { 1602 1722 struct io_ring_ctx *ctx = req->ctx; 1603 1723 ··· 1609 1729 wq_list_add_head(&req->comp_list, &ctx->locked_free_list); 1610 1730 ctx->locked_free_nr++; 1611 1731 spin_unlock(&ctx->completion_lock); 1612 - } 1613 - 1614 - static inline void io_remove_next_linked(struct io_kiocb *req) 1615 - { 1616 - struct io_kiocb *nxt = req->link; 1617 - 1618 - req->link = nxt->link; 1619 - nxt->link = NULL; 1620 - } 1621 - 1622 - static struct io_kiocb *io_disarm_linked_timeout(struct io_kiocb *req) 1623 - __must_hold(&req->ctx->completion_lock) 1624 - __must_hold(&req->ctx->timeout_lock) 1625 - { 1626 - struct io_kiocb *link = req->link; 1627 - 1628 - if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { 1629 - struct io_timeout_data *io = link->async_data; 1630 - struct io_timeout *timeout = io_kiocb_to_cmd(link); 1631 - 1632 - io_remove_next_linked(req); 1633 - timeout->head = NULL; 1634 - if (hrtimer_try_to_cancel(&io->timer) != -1) { 1635 - list_del(&timeout->list); 1636 - return link; 1637 - } 1638 - } 1639 - return NULL; 1640 - } 1641 - 1642 - static void io_fail_links(struct io_kiocb *req) 1643 - __must_hold(&req->ctx->completion_lock) 1644 - { 1645 - struct io_kiocb *nxt, *link = req->link; 1646 - bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES; 1647 - 1648 - req->link = NULL; 1649 - while (link) { 1650 - long res = -ECANCELED; 1651 - 1652 - if (link->flags & REQ_F_FAIL) 1653 - res = link->cqe.res; 1654 - 1655 - nxt = link->link; 1656 - link->link = NULL; 1657 - 1658 - trace_io_uring_fail_link(req->ctx, req, req->cqe.user_data, 1659 - req->opcode, link); 1660 - 1661 - if (ignore_cqes) 1662 - link->flags |= REQ_F_CQE_SKIP; 1663 - else 1664 - link->flags &= ~REQ_F_CQE_SKIP; 1665 - io_req_set_res(link, res, 0); 1666 - __io_req_complete_post(link); 1667 - link = nxt; 1668 - } 1669 - } 1670 - 1671 - static bool io_disarm_next(struct io_kiocb *req) 1672 - __must_hold(&req->ctx->completion_lock) 1673 - { 1674 - struct io_kiocb *link = NULL; 1675 - bool posted = false; 1676 - 1677 - if (req->flags & REQ_F_ARM_LTIMEOUT) { 1678 - link = req->link; 1679 - req->flags &= ~REQ_F_ARM_LTIMEOUT; 1680 - if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { 1681 - io_remove_next_linked(req); 1682 - io_req_tw_post_queue(link, -ECANCELED, 0); 1683 - posted = true; 1684 - } 1685 - } else if (req->flags & REQ_F_LINK_TIMEOUT) { 1686 - struct io_ring_ctx *ctx = req->ctx; 1687 - 1688 - spin_lock_irq(&ctx->timeout_lock); 1689 - link = io_disarm_linked_timeout(req); 1690 - spin_unlock_irq(&ctx->timeout_lock); 1691 - if (link) { 1692 - posted = true; 1693 - io_req_tw_post_queue(link, -ECANCELED, 0); 1694 - } 1695 - } 1696 - if (unlikely((req->flags & REQ_F_FAIL) && 1697 - !(req->flags & REQ_F_HARDLINK))) { 1698 - posted |= (req->link != NULL); 1699 - io_fail_links(req); 1700 - } 1701 - return posted; 1702 1732 } 1703 1733 1704 1734 static void __io_req_find_next_prep(struct io_kiocb *req) ··· 1823 2033 io_req_complete_post(req); 1824 2034 } 1825 2035 1826 - static void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags) 2036 + void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags) 1827 2037 { 1828 2038 io_req_set_res(req, res, cflags); 1829 2039 req->io_task_work.func = io_req_tw_post; ··· 1847 2057 io_req_complete_failed(req, -EFAULT); 1848 2058 } 1849 2059 1850 - static void io_req_task_queue_fail(struct io_kiocb *req, int ret) 2060 + void io_req_task_queue_fail(struct io_kiocb *req, int ret) 1851 2061 { 1852 2062 io_req_set_res(req, ret, 0); 1853 2063 req->io_task_work.func = io_req_task_cancel; ··· 1866 2076 io_req_task_work_add(req); 1867 2077 } 1868 2078 1869 - static void io_queue_next(struct io_kiocb *req) 2079 + void io_queue_next(struct io_kiocb *req) 1870 2080 { 1871 2081 struct io_kiocb *nxt = io_req_find_next(req); 1872 2082 ··· 1965 2175 io_free_req(req); 1966 2176 } 1967 2177 return nxt; 1968 - } 1969 - 1970 - static inline void io_put_req(struct io_kiocb *req) 1971 - { 1972 - if (req_ref_put_and_test(req)) { 1973 - io_queue_next(req); 1974 - io_free_req(req); 1975 - } 1976 2178 } 1977 2179 1978 2180 static unsigned io_cqring_events(struct io_ring_ctx *ctx) ··· 2233 2451 return false; 2234 2452 } 2235 2453 2236 - static inline void io_req_task_complete(struct io_kiocb *req, bool *locked) 2454 + inline void io_req_task_complete(struct io_kiocb *req, bool *locked) 2237 2455 { 2238 2456 if (*locked) { 2239 2457 req->cqe.flags |= io_put_kbuf(req, 0); ··· 4382 4600 return IOU_OK; 4383 4601 } 4384 4602 4385 - static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) 4386 - { 4387 - struct io_timeout_data *data = container_of(timer, 4388 - struct io_timeout_data, timer); 4389 - struct io_kiocb *req = data->req; 4390 - struct io_timeout *timeout = io_kiocb_to_cmd(req); 4391 - struct io_ring_ctx *ctx = req->ctx; 4392 - unsigned long flags; 4393 - 4394 - spin_lock_irqsave(&ctx->timeout_lock, flags); 4395 - list_del_init(&timeout->list); 4396 - atomic_set(&req->ctx->cq_timeouts, 4397 - atomic_read(&req->ctx->cq_timeouts) + 1); 4398 - spin_unlock_irqrestore(&ctx->timeout_lock, flags); 4399 - 4400 - if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS)) 4401 - req_set_fail(req); 4402 - 4403 - io_req_set_res(req, -ETIME, 0); 4404 - req->io_task_work.func = io_req_task_complete; 4405 - io_req_task_work_add(req); 4406 - return HRTIMER_NORESTART; 4407 - } 4408 - 4409 - static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, 4410 - struct io_cancel_data *cd) 4411 - __must_hold(&ctx->timeout_lock) 4412 - { 4413 - struct io_timeout *timeout; 4414 - struct io_timeout_data *io; 4415 - struct io_kiocb *req = NULL; 4416 - 4417 - list_for_each_entry(timeout, &ctx->timeout_list, list) { 4418 - struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); 4419 - 4420 - if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) && 4421 - cd->data != tmp->cqe.user_data) 4422 - continue; 4423 - if (cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY)) { 4424 - if (cd->seq == tmp->work.cancel_seq) 4425 - continue; 4426 - tmp->work.cancel_seq = cd->seq; 4427 - } 4428 - req = tmp; 4429 - break; 4430 - } 4431 - if (!req) 4432 - return ERR_PTR(-ENOENT); 4433 - 4434 - io = req->async_data; 4435 - if (hrtimer_try_to_cancel(&io->timer) == -1) 4436 - return ERR_PTR(-EALREADY); 4437 - timeout = io_kiocb_to_cmd(req); 4438 - list_del_init(&timeout->list); 4439 - return req; 4440 - } 4441 - 4442 - static int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) 4443 - __must_hold(&ctx->completion_lock) 4444 - { 4445 - struct io_kiocb *req; 4446 - 4447 - spin_lock_irq(&ctx->timeout_lock); 4448 - req = io_timeout_extract(ctx, cd); 4449 - spin_unlock_irq(&ctx->timeout_lock); 4450 - 4451 - if (IS_ERR(req)) 4452 - return PTR_ERR(req); 4453 - io_req_task_queue_fail(req, -ECANCELED); 4454 - return 0; 4455 - } 4456 - 4457 - static clockid_t io_timeout_get_clock(struct io_timeout_data *data) 4458 - { 4459 - switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) { 4460 - case IORING_TIMEOUT_BOOTTIME: 4461 - return CLOCK_BOOTTIME; 4462 - case IORING_TIMEOUT_REALTIME: 4463 - return CLOCK_REALTIME; 4464 - default: 4465 - /* can't happen, vetted at prep time */ 4466 - WARN_ON_ONCE(1); 4467 - fallthrough; 4468 - case 0: 4469 - return CLOCK_MONOTONIC; 4470 - } 4471 - } 4472 - 4473 - static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, 4474 - struct timespec64 *ts, enum hrtimer_mode mode) 4475 - __must_hold(&ctx->timeout_lock) 4476 - { 4477 - struct io_timeout_data *io; 4478 - struct io_timeout *timeout; 4479 - struct io_kiocb *req = NULL; 4480 - 4481 - list_for_each_entry(timeout, &ctx->ltimeout_list, list) { 4482 - struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); 4483 - 4484 - if (user_data == tmp->cqe.user_data) { 4485 - req = tmp; 4486 - break; 4487 - } 4488 - } 4489 - if (!req) 4490 - return -ENOENT; 4491 - 4492 - io = req->async_data; 4493 - if (hrtimer_try_to_cancel(&io->timer) == -1) 4494 - return -EALREADY; 4495 - hrtimer_init(&io->timer, io_timeout_get_clock(io), mode); 4496 - io->timer.function = io_link_timeout_fn; 4497 - hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode); 4498 - return 0; 4499 - } 4500 - 4501 - static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, 4502 - struct timespec64 *ts, enum hrtimer_mode mode) 4503 - __must_hold(&ctx->timeout_lock) 4504 - { 4505 - struct io_cancel_data cd = { .data = user_data, }; 4506 - struct io_kiocb *req = io_timeout_extract(ctx, &cd); 4507 - struct io_timeout *timeout = io_kiocb_to_cmd(req); 4508 - struct io_timeout_data *data; 4509 - 4510 - if (IS_ERR(req)) 4511 - return PTR_ERR(req); 4512 - 4513 - timeout->off = 0; /* noseq */ 4514 - data = req->async_data; 4515 - list_add_tail(&timeout->list, &ctx->timeout_list); 4516 - hrtimer_init(&data->timer, io_timeout_get_clock(data), mode); 4517 - data->timer.function = io_timeout_fn; 4518 - hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode); 4519 - return 0; 4520 - } 4521 - 4522 - static int io_timeout_remove_prep(struct io_kiocb *req, 4523 - const struct io_uring_sqe *sqe) 4524 - { 4525 - struct io_timeout_rem *tr = io_kiocb_to_cmd(req); 4526 - 4527 - if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) 4528 - return -EINVAL; 4529 - if (sqe->buf_index || sqe->len || sqe->splice_fd_in) 4530 - return -EINVAL; 4531 - 4532 - tr->ltimeout = false; 4533 - tr->addr = READ_ONCE(sqe->addr); 4534 - tr->flags = READ_ONCE(sqe->timeout_flags); 4535 - if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) { 4536 - if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1) 4537 - return -EINVAL; 4538 - if (tr->flags & IORING_LINK_TIMEOUT_UPDATE) 4539 - tr->ltimeout = true; 4540 - if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS)) 4541 - return -EINVAL; 4542 - if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2))) 4543 - return -EFAULT; 4544 - if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0) 4545 - return -EINVAL; 4546 - } else if (tr->flags) { 4547 - /* timeout removal doesn't support flags */ 4548 - return -EINVAL; 4549 - } 4550 - 4551 - return 0; 4552 - } 4553 - 4554 - static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) 4555 - { 4556 - return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS 4557 - : HRTIMER_MODE_REL; 4558 - } 4559 - 4560 - /* 4561 - * Remove or update an existing timeout command 4562 - */ 4563 - static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) 4564 - { 4565 - struct io_timeout_rem *tr = io_kiocb_to_cmd(req); 4566 - struct io_ring_ctx *ctx = req->ctx; 4567 - int ret; 4568 - 4569 - if (!(tr->flags & IORING_TIMEOUT_UPDATE)) { 4570 - struct io_cancel_data cd = { .data = tr->addr, }; 4571 - 4572 - spin_lock(&ctx->completion_lock); 4573 - ret = io_timeout_cancel(ctx, &cd); 4574 - spin_unlock(&ctx->completion_lock); 4575 - } else { 4576 - enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags); 4577 - 4578 - spin_lock_irq(&ctx->timeout_lock); 4579 - if (tr->ltimeout) 4580 - ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode); 4581 - else 4582 - ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode); 4583 - spin_unlock_irq(&ctx->timeout_lock); 4584 - } 4585 - 4586 - if (ret < 0) 4587 - req_set_fail(req); 4588 - io_req_set_res(req, ret, 0); 4589 - return IOU_OK; 4590 - } 4591 - 4592 - static int __io_timeout_prep(struct io_kiocb *req, 4593 - const struct io_uring_sqe *sqe, 4594 - bool is_timeout_link) 4595 - { 4596 - struct io_timeout *timeout = io_kiocb_to_cmd(req); 4597 - struct io_timeout_data *data; 4598 - unsigned flags; 4599 - u32 off = READ_ONCE(sqe->off); 4600 - 4601 - if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in) 4602 - return -EINVAL; 4603 - if (off && is_timeout_link) 4604 - return -EINVAL; 4605 - flags = READ_ONCE(sqe->timeout_flags); 4606 - if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK | 4607 - IORING_TIMEOUT_ETIME_SUCCESS)) 4608 - return -EINVAL; 4609 - /* more than one clock specified is invalid, obviously */ 4610 - if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1) 4611 - return -EINVAL; 4612 - 4613 - INIT_LIST_HEAD(&timeout->list); 4614 - timeout->off = off; 4615 - if (unlikely(off && !req->ctx->off_timeout_used)) 4616 - req->ctx->off_timeout_used = true; 4617 - 4618 - if (WARN_ON_ONCE(req_has_async_data(req))) 4619 - return -EFAULT; 4620 - if (io_alloc_async_data(req)) 4621 - return -ENOMEM; 4622 - 4623 - data = req->async_data; 4624 - data->req = req; 4625 - data->flags = flags; 4626 - 4627 - if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) 4628 - return -EFAULT; 4629 - 4630 - if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0) 4631 - return -EINVAL; 4632 - 4633 - INIT_LIST_HEAD(&timeout->list); 4634 - data->mode = io_translate_timeout_mode(flags); 4635 - hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode); 4636 - 4637 - if (is_timeout_link) { 4638 - struct io_submit_link *link = &req->ctx->submit_state.link; 4639 - 4640 - if (!link->head) 4641 - return -EINVAL; 4642 - if (link->last->opcode == IORING_OP_LINK_TIMEOUT) 4643 - return -EINVAL; 4644 - timeout->head = link->last; 4645 - link->last->flags |= REQ_F_ARM_LTIMEOUT; 4646 - } 4647 - return 0; 4648 - } 4649 - 4650 - static int io_timeout_prep(struct io_kiocb *req, 4651 - const struct io_uring_sqe *sqe) 4652 - { 4653 - return __io_timeout_prep(req, sqe, false); 4654 - } 4655 - 4656 - static int io_link_timeout_prep(struct io_kiocb *req, 4657 - const struct io_uring_sqe *sqe) 4658 - { 4659 - return __io_timeout_prep(req, sqe, true); 4660 - } 4661 - 4662 - static int io_timeout(struct io_kiocb *req, unsigned int issue_flags) 4663 - { 4664 - struct io_timeout *timeout = io_kiocb_to_cmd(req); 4665 - struct io_ring_ctx *ctx = req->ctx; 4666 - struct io_timeout_data *data = req->async_data; 4667 - struct list_head *entry; 4668 - u32 tail, off = timeout->off; 4669 - 4670 - spin_lock_irq(&ctx->timeout_lock); 4671 - 4672 - /* 4673 - * sqe->off holds how many events that need to occur for this 4674 - * timeout event to be satisfied. If it isn't set, then this is 4675 - * a pure timeout request, sequence isn't used. 4676 - */ 4677 - if (io_is_timeout_noseq(req)) { 4678 - entry = ctx->timeout_list.prev; 4679 - goto add; 4680 - } 4681 - 4682 - tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); 4683 - timeout->target_seq = tail + off; 4684 - 4685 - /* Update the last seq here in case io_flush_timeouts() hasn't. 4686 - * This is safe because ->completion_lock is held, and submissions 4687 - * and completions are never mixed in the same ->completion_lock section. 4688 - */ 4689 - ctx->cq_last_tm_flush = tail; 4690 - 4691 - /* 4692 - * Insertion sort, ensuring the first entry in the list is always 4693 - * the one we need first. 4694 - */ 4695 - list_for_each_prev(entry, &ctx->timeout_list) { 4696 - struct io_timeout *nextt = list_entry(entry, struct io_timeout, list); 4697 - struct io_kiocb *nxt = cmd_to_io_kiocb(nextt); 4698 - 4699 - if (io_is_timeout_noseq(nxt)) 4700 - continue; 4701 - /* nxt.seq is behind @tail, otherwise would've been completed */ 4702 - if (off >= nextt->target_seq - tail) 4703 - break; 4704 - } 4705 - add: 4706 - list_add(&timeout->list, entry); 4707 - data->timer.function = io_timeout_fn; 4708 - hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); 4709 - spin_unlock_irq(&ctx->timeout_lock); 4710 - return IOU_ISSUE_SKIP_COMPLETE; 4711 - } 4712 - 4713 4603 static bool io_cancel_cb(struct io_wq_work *work, void *data) 4714 4604 { 4715 4605 struct io_kiocb *req = container_of(work, struct io_kiocb, work); ··· 4433 4979 return ret; 4434 4980 } 4435 4981 4436 - static int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd) 4982 + int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd) 4437 4983 { 4438 4984 struct io_ring_ctx *ctx = req->ctx; 4439 4985 int ret; ··· 4914 5460 if (file && file->f_op == &io_uring_fops) 4915 5461 io_req_track_inflight(req); 4916 5462 return file; 4917 - } 4918 - 4919 - static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) 4920 - { 4921 - struct io_timeout *timeout = io_kiocb_to_cmd(req); 4922 - struct io_kiocb *prev = timeout->prev; 4923 - int ret = -ENOENT; 4924 - 4925 - if (prev) { 4926 - if (!(req->task->flags & PF_EXITING)) { 4927 - struct io_cancel_data cd = { 4928 - .ctx = req->ctx, 4929 - .data = prev->cqe.user_data, 4930 - }; 4931 - 4932 - ret = io_try_cancel(req, &cd); 4933 - } 4934 - io_req_set_res(req, ret ?: -ETIME, 0); 4935 - io_req_complete_post(req); 4936 - io_put_req(prev); 4937 - } else { 4938 - io_req_set_res(req, -ETIME, 0); 4939 - io_req_complete_post(req); 4940 - } 4941 - } 4942 - 4943 - static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) 4944 - { 4945 - struct io_timeout_data *data = container_of(timer, 4946 - struct io_timeout_data, timer); 4947 - struct io_kiocb *prev, *req = data->req; 4948 - struct io_timeout *timeout = io_kiocb_to_cmd(req); 4949 - struct io_ring_ctx *ctx = req->ctx; 4950 - unsigned long flags; 4951 - 4952 - spin_lock_irqsave(&ctx->timeout_lock, flags); 4953 - prev = timeout->head; 4954 - timeout->head = NULL; 4955 - 4956 - /* 4957 - * We don't expect the list to be empty, that will only happen if we 4958 - * race with the completion of the linked work. 4959 - */ 4960 - if (prev) { 4961 - io_remove_next_linked(prev); 4962 - if (!req_ref_inc_not_zero(prev)) 4963 - prev = NULL; 4964 - } 4965 - list_del(&timeout->list); 4966 - timeout->prev = prev; 4967 - spin_unlock_irqrestore(&ctx->timeout_lock, flags); 4968 - 4969 - req->io_task_work.func = io_req_task_link_timeout; 4970 - io_req_task_work_add(req); 4971 - return HRTIMER_NORESTART; 4972 - } 4973 - 4974 - static void io_queue_linked_timeout(struct io_kiocb *req) 4975 - { 4976 - struct io_timeout *timeout = io_kiocb_to_cmd(req); 4977 - struct io_ring_ctx *ctx = req->ctx; 4978 - 4979 - spin_lock_irq(&ctx->timeout_lock); 4980 - /* 4981 - * If the back reference is NULL, then our linked request finished 4982 - * before we got a chance to setup the timer 4983 - */ 4984 - if (timeout->head) { 4985 - struct io_timeout_data *data = req->async_data; 4986 - 4987 - data->timer.function = io_link_timeout_fn; 4988 - hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), 4989 - data->mode); 4990 - list_add_tail(&timeout->list, &ctx->ltimeout_list); 4991 - } 4992 - spin_unlock_irq(&ctx->timeout_lock); 4993 - /* drop submission reference */ 4994 - io_put_req(req); 4995 5463 } 4996 5464 4997 5465 static void io_queue_async(struct io_kiocb *req, int ret) ··· 7490 8114 spin_unlock(&ctx->completion_lock); 7491 8115 7492 8116 io_ring_ctx_free(ctx); 7493 - } 7494 - 7495 - /* Returns true if we found and killed one or more timeouts */ 7496 - static __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, 7497 - struct task_struct *tsk, bool cancel_all) 7498 - { 7499 - struct io_timeout *timeout, *tmp; 7500 - int canceled = 0; 7501 - 7502 - spin_lock(&ctx->completion_lock); 7503 - spin_lock_irq(&ctx->timeout_lock); 7504 - list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { 7505 - struct io_kiocb *req = cmd_to_io_kiocb(timeout); 7506 - 7507 - if (io_match_task(req, tsk, cancel_all)) { 7508 - io_kill_timeout(req, -ECANCELED); 7509 - canceled++; 7510 - } 7511 - } 7512 - spin_unlock_irq(&ctx->timeout_lock); 7513 - io_commit_cqring(ctx); 7514 - spin_unlock(&ctx->completion_lock); 7515 - if (canceled != 0) 7516 - io_cqring_ev_posted(ctx); 7517 - return canceled != 0; 7518 8117 } 7519 8118 7520 8119 static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
+12 -1
io_uring/io_uring.h
··· 65 65 } 66 66 67 67 void __io_req_complete(struct io_kiocb *req, unsigned issue_flags); 68 - 68 + void io_req_complete_post(struct io_kiocb *req); 69 + void __io_req_complete_post(struct io_kiocb *req); 69 70 bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, 70 71 u32 cflags); 71 72 void io_cqring_ev_posted(struct io_ring_ctx *ctx); ··· 97 96 bool io_is_uring_fops(struct file *file); 98 97 bool io_alloc_async_data(struct io_kiocb *req); 99 98 void io_req_task_work_add(struct io_kiocb *req); 99 + void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags); 100 + void io_req_task_complete(struct io_kiocb *req, bool *locked); 101 + void io_req_task_queue_fail(struct io_kiocb *req, int ret); 102 + int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd); 103 + 104 + void io_free_req(struct io_kiocb *req); 105 + void io_queue_next(struct io_kiocb *req); 106 + 107 + #define io_for_each_link(pos, head) \ 108 + for (pos = (head); pos; pos = pos->link) 100 109 101 110 #endif
+10
io_uring/io_uring_types.h
··· 488 488 struct io_wq_work work; 489 489 }; 490 490 491 + struct io_cancel_data { 492 + struct io_ring_ctx *ctx; 493 + union { 494 + u64 data; 495 + struct file *file; 496 + }; 497 + u32 flags; 498 + int seq; 499 + }; 500 + 491 501 #endif
+634
io_uring/timeout.c
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + #include <linux/kernel.h> 3 + #include <linux/errno.h> 4 + #include <linux/file.h> 5 + #include <linux/io_uring.h> 6 + 7 + #include <trace/events/io_uring.h> 8 + 9 + #include <uapi/linux/io_uring.h> 10 + 11 + #include "io_uring_types.h" 12 + #include "io_uring.h" 13 + #include "refs.h" 14 + #include "timeout.h" 15 + 16 + struct io_timeout { 17 + struct file *file; 18 + u32 off; 19 + u32 target_seq; 20 + struct list_head list; 21 + /* head of the link, used by linked timeouts only */ 22 + struct io_kiocb *head; 23 + /* for linked completions */ 24 + struct io_kiocb *prev; 25 + }; 26 + 27 + struct io_timeout_rem { 28 + struct file *file; 29 + u64 addr; 30 + 31 + /* timeout update */ 32 + struct timespec64 ts; 33 + u32 flags; 34 + bool ltimeout; 35 + }; 36 + 37 + static inline bool io_is_timeout_noseq(struct io_kiocb *req) 38 + { 39 + struct io_timeout *timeout = io_kiocb_to_cmd(req); 40 + 41 + return !timeout->off; 42 + } 43 + 44 + static inline void io_put_req(struct io_kiocb *req) 45 + { 46 + if (req_ref_put_and_test(req)) { 47 + io_queue_next(req); 48 + io_free_req(req); 49 + } 50 + } 51 + 52 + static void io_kill_timeout(struct io_kiocb *req, int status) 53 + __must_hold(&req->ctx->completion_lock) 54 + __must_hold(&req->ctx->timeout_lock) 55 + { 56 + struct io_timeout_data *io = req->async_data; 57 + 58 + if (hrtimer_try_to_cancel(&io->timer) != -1) { 59 + struct io_timeout *timeout = io_kiocb_to_cmd(req); 60 + 61 + if (status) 62 + req_set_fail(req); 63 + atomic_set(&req->ctx->cq_timeouts, 64 + atomic_read(&req->ctx->cq_timeouts) + 1); 65 + list_del_init(&timeout->list); 66 + io_req_tw_post_queue(req, status, 0); 67 + } 68 + } 69 + 70 + __cold void io_flush_timeouts(struct io_ring_ctx *ctx) 71 + __must_hold(&ctx->completion_lock) 72 + { 73 + u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); 74 + struct io_timeout *timeout, *tmp; 75 + 76 + spin_lock_irq(&ctx->timeout_lock); 77 + list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { 78 + struct io_kiocb *req = cmd_to_io_kiocb(timeout); 79 + u32 events_needed, events_got; 80 + 81 + if (io_is_timeout_noseq(req)) 82 + break; 83 + 84 + /* 85 + * Since seq can easily wrap around over time, subtract 86 + * the last seq at which timeouts were flushed before comparing. 87 + * Assuming not more than 2^31-1 events have happened since, 88 + * these subtractions won't have wrapped, so we can check if 89 + * target is in [last_seq, current_seq] by comparing the two. 90 + */ 91 + events_needed = timeout->target_seq - ctx->cq_last_tm_flush; 92 + events_got = seq - ctx->cq_last_tm_flush; 93 + if (events_got < events_needed) 94 + break; 95 + 96 + io_kill_timeout(req, 0); 97 + } 98 + ctx->cq_last_tm_flush = seq; 99 + spin_unlock_irq(&ctx->timeout_lock); 100 + } 101 + 102 + static void io_fail_links(struct io_kiocb *req) 103 + __must_hold(&req->ctx->completion_lock) 104 + { 105 + struct io_kiocb *nxt, *link = req->link; 106 + bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES; 107 + 108 + req->link = NULL; 109 + while (link) { 110 + long res = -ECANCELED; 111 + 112 + if (link->flags & REQ_F_FAIL) 113 + res = link->cqe.res; 114 + 115 + nxt = link->link; 116 + link->link = NULL; 117 + 118 + trace_io_uring_fail_link(req->ctx, req, req->cqe.user_data, 119 + req->opcode, link); 120 + 121 + if (ignore_cqes) 122 + link->flags |= REQ_F_CQE_SKIP; 123 + else 124 + link->flags &= ~REQ_F_CQE_SKIP; 125 + io_req_set_res(link, res, 0); 126 + __io_req_complete_post(link); 127 + link = nxt; 128 + } 129 + } 130 + 131 + static inline void io_remove_next_linked(struct io_kiocb *req) 132 + { 133 + struct io_kiocb *nxt = req->link; 134 + 135 + req->link = nxt->link; 136 + nxt->link = NULL; 137 + } 138 + 139 + bool io_disarm_next(struct io_kiocb *req) 140 + __must_hold(&req->ctx->completion_lock) 141 + { 142 + struct io_kiocb *link = NULL; 143 + bool posted = false; 144 + 145 + if (req->flags & REQ_F_ARM_LTIMEOUT) { 146 + link = req->link; 147 + req->flags &= ~REQ_F_ARM_LTIMEOUT; 148 + if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { 149 + io_remove_next_linked(req); 150 + io_req_tw_post_queue(link, -ECANCELED, 0); 151 + posted = true; 152 + } 153 + } else if (req->flags & REQ_F_LINK_TIMEOUT) { 154 + struct io_ring_ctx *ctx = req->ctx; 155 + 156 + spin_lock_irq(&ctx->timeout_lock); 157 + link = io_disarm_linked_timeout(req); 158 + spin_unlock_irq(&ctx->timeout_lock); 159 + if (link) { 160 + posted = true; 161 + io_req_tw_post_queue(link, -ECANCELED, 0); 162 + } 163 + } 164 + if (unlikely((req->flags & REQ_F_FAIL) && 165 + !(req->flags & REQ_F_HARDLINK))) { 166 + posted |= (req->link != NULL); 167 + io_fail_links(req); 168 + } 169 + return posted; 170 + } 171 + 172 + struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, 173 + struct io_kiocb *link) 174 + __must_hold(&req->ctx->completion_lock) 175 + __must_hold(&req->ctx->timeout_lock) 176 + { 177 + struct io_timeout_data *io = link->async_data; 178 + struct io_timeout *timeout = io_kiocb_to_cmd(link); 179 + 180 + io_remove_next_linked(req); 181 + timeout->head = NULL; 182 + if (hrtimer_try_to_cancel(&io->timer) != -1) { 183 + list_del(&timeout->list); 184 + return link; 185 + } 186 + 187 + return NULL; 188 + } 189 + 190 + static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) 191 + { 192 + struct io_timeout_data *data = container_of(timer, 193 + struct io_timeout_data, timer); 194 + struct io_kiocb *req = data->req; 195 + struct io_timeout *timeout = io_kiocb_to_cmd(req); 196 + struct io_ring_ctx *ctx = req->ctx; 197 + unsigned long flags; 198 + 199 + spin_lock_irqsave(&ctx->timeout_lock, flags); 200 + list_del_init(&timeout->list); 201 + atomic_set(&req->ctx->cq_timeouts, 202 + atomic_read(&req->ctx->cq_timeouts) + 1); 203 + spin_unlock_irqrestore(&ctx->timeout_lock, flags); 204 + 205 + if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS)) 206 + req_set_fail(req); 207 + 208 + io_req_set_res(req, -ETIME, 0); 209 + req->io_task_work.func = io_req_task_complete; 210 + io_req_task_work_add(req); 211 + return HRTIMER_NORESTART; 212 + } 213 + 214 + static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, 215 + struct io_cancel_data *cd) 216 + __must_hold(&ctx->timeout_lock) 217 + { 218 + struct io_timeout *timeout; 219 + struct io_timeout_data *io; 220 + struct io_kiocb *req = NULL; 221 + 222 + list_for_each_entry(timeout, &ctx->timeout_list, list) { 223 + struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); 224 + 225 + if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) && 226 + cd->data != tmp->cqe.user_data) 227 + continue; 228 + if (cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY)) { 229 + if (cd->seq == tmp->work.cancel_seq) 230 + continue; 231 + tmp->work.cancel_seq = cd->seq; 232 + } 233 + req = tmp; 234 + break; 235 + } 236 + if (!req) 237 + return ERR_PTR(-ENOENT); 238 + 239 + io = req->async_data; 240 + if (hrtimer_try_to_cancel(&io->timer) == -1) 241 + return ERR_PTR(-EALREADY); 242 + timeout = io_kiocb_to_cmd(req); 243 + list_del_init(&timeout->list); 244 + return req; 245 + } 246 + 247 + int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) 248 + __must_hold(&ctx->completion_lock) 249 + { 250 + struct io_kiocb *req; 251 + 252 + spin_lock_irq(&ctx->timeout_lock); 253 + req = io_timeout_extract(ctx, cd); 254 + spin_unlock_irq(&ctx->timeout_lock); 255 + 256 + if (IS_ERR(req)) 257 + return PTR_ERR(req); 258 + io_req_task_queue_fail(req, -ECANCELED); 259 + return 0; 260 + } 261 + 262 + static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) 263 + { 264 + struct io_timeout *timeout = io_kiocb_to_cmd(req); 265 + struct io_kiocb *prev = timeout->prev; 266 + int ret = -ENOENT; 267 + 268 + if (prev) { 269 + if (!(req->task->flags & PF_EXITING)) { 270 + struct io_cancel_data cd = { 271 + .ctx = req->ctx, 272 + .data = prev->cqe.user_data, 273 + }; 274 + 275 + ret = io_try_cancel(req, &cd); 276 + } 277 + io_req_set_res(req, ret ?: -ETIME, 0); 278 + io_req_complete_post(req); 279 + io_put_req(prev); 280 + } else { 281 + io_req_set_res(req, -ETIME, 0); 282 + io_req_complete_post(req); 283 + } 284 + } 285 + 286 + static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) 287 + { 288 + struct io_timeout_data *data = container_of(timer, 289 + struct io_timeout_data, timer); 290 + struct io_kiocb *prev, *req = data->req; 291 + struct io_timeout *timeout = io_kiocb_to_cmd(req); 292 + struct io_ring_ctx *ctx = req->ctx; 293 + unsigned long flags; 294 + 295 + spin_lock_irqsave(&ctx->timeout_lock, flags); 296 + prev = timeout->head; 297 + timeout->head = NULL; 298 + 299 + /* 300 + * We don't expect the list to be empty, that will only happen if we 301 + * race with the completion of the linked work. 302 + */ 303 + if (prev) { 304 + io_remove_next_linked(prev); 305 + if (!req_ref_inc_not_zero(prev)) 306 + prev = NULL; 307 + } 308 + list_del(&timeout->list); 309 + timeout->prev = prev; 310 + spin_unlock_irqrestore(&ctx->timeout_lock, flags); 311 + 312 + req->io_task_work.func = io_req_task_link_timeout; 313 + io_req_task_work_add(req); 314 + return HRTIMER_NORESTART; 315 + } 316 + 317 + static clockid_t io_timeout_get_clock(struct io_timeout_data *data) 318 + { 319 + switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) { 320 + case IORING_TIMEOUT_BOOTTIME: 321 + return CLOCK_BOOTTIME; 322 + case IORING_TIMEOUT_REALTIME: 323 + return CLOCK_REALTIME; 324 + default: 325 + /* can't happen, vetted at prep time */ 326 + WARN_ON_ONCE(1); 327 + fallthrough; 328 + case 0: 329 + return CLOCK_MONOTONIC; 330 + } 331 + } 332 + 333 + static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, 334 + struct timespec64 *ts, enum hrtimer_mode mode) 335 + __must_hold(&ctx->timeout_lock) 336 + { 337 + struct io_timeout_data *io; 338 + struct io_timeout *timeout; 339 + struct io_kiocb *req = NULL; 340 + 341 + list_for_each_entry(timeout, &ctx->ltimeout_list, list) { 342 + struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); 343 + 344 + if (user_data == tmp->cqe.user_data) { 345 + req = tmp; 346 + break; 347 + } 348 + } 349 + if (!req) 350 + return -ENOENT; 351 + 352 + io = req->async_data; 353 + if (hrtimer_try_to_cancel(&io->timer) == -1) 354 + return -EALREADY; 355 + hrtimer_init(&io->timer, io_timeout_get_clock(io), mode); 356 + io->timer.function = io_link_timeout_fn; 357 + hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode); 358 + return 0; 359 + } 360 + 361 + static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, 362 + struct timespec64 *ts, enum hrtimer_mode mode) 363 + __must_hold(&ctx->timeout_lock) 364 + { 365 + struct io_cancel_data cd = { .data = user_data, }; 366 + struct io_kiocb *req = io_timeout_extract(ctx, &cd); 367 + struct io_timeout *timeout = io_kiocb_to_cmd(req); 368 + struct io_timeout_data *data; 369 + 370 + if (IS_ERR(req)) 371 + return PTR_ERR(req); 372 + 373 + timeout->off = 0; /* noseq */ 374 + data = req->async_data; 375 + list_add_tail(&timeout->list, &ctx->timeout_list); 376 + hrtimer_init(&data->timer, io_timeout_get_clock(data), mode); 377 + data->timer.function = io_timeout_fn; 378 + hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode); 379 + return 0; 380 + } 381 + 382 + int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 383 + { 384 + struct io_timeout_rem *tr = io_kiocb_to_cmd(req); 385 + 386 + if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) 387 + return -EINVAL; 388 + if (sqe->buf_index || sqe->len || sqe->splice_fd_in) 389 + return -EINVAL; 390 + 391 + tr->ltimeout = false; 392 + tr->addr = READ_ONCE(sqe->addr); 393 + tr->flags = READ_ONCE(sqe->timeout_flags); 394 + if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) { 395 + if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1) 396 + return -EINVAL; 397 + if (tr->flags & IORING_LINK_TIMEOUT_UPDATE) 398 + tr->ltimeout = true; 399 + if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS)) 400 + return -EINVAL; 401 + if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2))) 402 + return -EFAULT; 403 + if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0) 404 + return -EINVAL; 405 + } else if (tr->flags) { 406 + /* timeout removal doesn't support flags */ 407 + return -EINVAL; 408 + } 409 + 410 + return 0; 411 + } 412 + 413 + static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) 414 + { 415 + return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS 416 + : HRTIMER_MODE_REL; 417 + } 418 + 419 + /* 420 + * Remove or update an existing timeout command 421 + */ 422 + int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) 423 + { 424 + struct io_timeout_rem *tr = io_kiocb_to_cmd(req); 425 + struct io_ring_ctx *ctx = req->ctx; 426 + int ret; 427 + 428 + if (!(tr->flags & IORING_TIMEOUT_UPDATE)) { 429 + struct io_cancel_data cd = { .data = tr->addr, }; 430 + 431 + spin_lock(&ctx->completion_lock); 432 + ret = io_timeout_cancel(ctx, &cd); 433 + spin_unlock(&ctx->completion_lock); 434 + } else { 435 + enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags); 436 + 437 + spin_lock_irq(&ctx->timeout_lock); 438 + if (tr->ltimeout) 439 + ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode); 440 + else 441 + ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode); 442 + spin_unlock_irq(&ctx->timeout_lock); 443 + } 444 + 445 + if (ret < 0) 446 + req_set_fail(req); 447 + io_req_set_res(req, ret, 0); 448 + return IOU_OK; 449 + } 450 + 451 + static int __io_timeout_prep(struct io_kiocb *req, 452 + const struct io_uring_sqe *sqe, 453 + bool is_timeout_link) 454 + { 455 + struct io_timeout *timeout = io_kiocb_to_cmd(req); 456 + struct io_timeout_data *data; 457 + unsigned flags; 458 + u32 off = READ_ONCE(sqe->off); 459 + 460 + if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in) 461 + return -EINVAL; 462 + if (off && is_timeout_link) 463 + return -EINVAL; 464 + flags = READ_ONCE(sqe->timeout_flags); 465 + if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK | 466 + IORING_TIMEOUT_ETIME_SUCCESS)) 467 + return -EINVAL; 468 + /* more than one clock specified is invalid, obviously */ 469 + if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1) 470 + return -EINVAL; 471 + 472 + INIT_LIST_HEAD(&timeout->list); 473 + timeout->off = off; 474 + if (unlikely(off && !req->ctx->off_timeout_used)) 475 + req->ctx->off_timeout_used = true; 476 + 477 + if (WARN_ON_ONCE(req_has_async_data(req))) 478 + return -EFAULT; 479 + if (io_alloc_async_data(req)) 480 + return -ENOMEM; 481 + 482 + data = req->async_data; 483 + data->req = req; 484 + data->flags = flags; 485 + 486 + if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) 487 + return -EFAULT; 488 + 489 + if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0) 490 + return -EINVAL; 491 + 492 + INIT_LIST_HEAD(&timeout->list); 493 + data->mode = io_translate_timeout_mode(flags); 494 + hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode); 495 + 496 + if (is_timeout_link) { 497 + struct io_submit_link *link = &req->ctx->submit_state.link; 498 + 499 + if (!link->head) 500 + return -EINVAL; 501 + if (link->last->opcode == IORING_OP_LINK_TIMEOUT) 502 + return -EINVAL; 503 + timeout->head = link->last; 504 + link->last->flags |= REQ_F_ARM_LTIMEOUT; 505 + } 506 + return 0; 507 + } 508 + 509 + int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 510 + { 511 + return __io_timeout_prep(req, sqe, false); 512 + } 513 + 514 + int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 515 + { 516 + return __io_timeout_prep(req, sqe, true); 517 + } 518 + 519 + int io_timeout(struct io_kiocb *req, unsigned int issue_flags) 520 + { 521 + struct io_timeout *timeout = io_kiocb_to_cmd(req); 522 + struct io_ring_ctx *ctx = req->ctx; 523 + struct io_timeout_data *data = req->async_data; 524 + struct list_head *entry; 525 + u32 tail, off = timeout->off; 526 + 527 + spin_lock_irq(&ctx->timeout_lock); 528 + 529 + /* 530 + * sqe->off holds how many events that need to occur for this 531 + * timeout event to be satisfied. If it isn't set, then this is 532 + * a pure timeout request, sequence isn't used. 533 + */ 534 + if (io_is_timeout_noseq(req)) { 535 + entry = ctx->timeout_list.prev; 536 + goto add; 537 + } 538 + 539 + tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); 540 + timeout->target_seq = tail + off; 541 + 542 + /* Update the last seq here in case io_flush_timeouts() hasn't. 543 + * This is safe because ->completion_lock is held, and submissions 544 + * and completions are never mixed in the same ->completion_lock section. 545 + */ 546 + ctx->cq_last_tm_flush = tail; 547 + 548 + /* 549 + * Insertion sort, ensuring the first entry in the list is always 550 + * the one we need first. 551 + */ 552 + list_for_each_prev(entry, &ctx->timeout_list) { 553 + struct io_timeout *nextt = list_entry(entry, struct io_timeout, list); 554 + struct io_kiocb *nxt = cmd_to_io_kiocb(nextt); 555 + 556 + if (io_is_timeout_noseq(nxt)) 557 + continue; 558 + /* nxt.seq is behind @tail, otherwise would've been completed */ 559 + if (off >= nextt->target_seq - tail) 560 + break; 561 + } 562 + add: 563 + list_add(&timeout->list, entry); 564 + data->timer.function = io_timeout_fn; 565 + hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); 566 + spin_unlock_irq(&ctx->timeout_lock); 567 + return IOU_ISSUE_SKIP_COMPLETE; 568 + } 569 + 570 + void io_queue_linked_timeout(struct io_kiocb *req) 571 + { 572 + struct io_timeout *timeout = io_kiocb_to_cmd(req); 573 + struct io_ring_ctx *ctx = req->ctx; 574 + 575 + spin_lock_irq(&ctx->timeout_lock); 576 + /* 577 + * If the back reference is NULL, then our linked request finished 578 + * before we got a chance to setup the timer 579 + */ 580 + if (timeout->head) { 581 + struct io_timeout_data *data = req->async_data; 582 + 583 + data->timer.function = io_link_timeout_fn; 584 + hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), 585 + data->mode); 586 + list_add_tail(&timeout->list, &ctx->ltimeout_list); 587 + } 588 + spin_unlock_irq(&ctx->timeout_lock); 589 + /* drop submission reference */ 590 + io_put_req(req); 591 + } 592 + 593 + static bool io_match_task(struct io_kiocb *head, struct task_struct *task, 594 + bool cancel_all) 595 + __must_hold(&req->ctx->timeout_lock) 596 + { 597 + struct io_kiocb *req; 598 + 599 + if (task && head->task != task) 600 + return false; 601 + if (cancel_all) 602 + return true; 603 + 604 + io_for_each_link(req, head) { 605 + if (req->flags & REQ_F_INFLIGHT) 606 + return true; 607 + } 608 + return false; 609 + } 610 + 611 + /* Returns true if we found and killed one or more timeouts */ 612 + __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, 613 + bool cancel_all) 614 + { 615 + struct io_timeout *timeout, *tmp; 616 + int canceled = 0; 617 + 618 + spin_lock(&ctx->completion_lock); 619 + spin_lock_irq(&ctx->timeout_lock); 620 + list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { 621 + struct io_kiocb *req = cmd_to_io_kiocb(timeout); 622 + 623 + if (io_match_task(req, tsk, cancel_all)) { 624 + io_kill_timeout(req, -ECANCELED); 625 + canceled++; 626 + } 627 + } 628 + spin_unlock_irq(&ctx->timeout_lock); 629 + io_commit_cqring(ctx); 630 + spin_unlock(&ctx->completion_lock); 631 + if (canceled != 0) 632 + io_cqring_ev_posted(ctx); 633 + return canceled != 0; 634 + }
+35
io_uring/timeout.h
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + struct io_timeout_data { 4 + struct io_kiocb *req; 5 + struct hrtimer timer; 6 + struct timespec64 ts; 7 + enum hrtimer_mode mode; 8 + u32 flags; 9 + }; 10 + 11 + struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, 12 + struct io_kiocb *link); 13 + 14 + static inline struct io_kiocb *io_disarm_linked_timeout(struct io_kiocb *req) 15 + { 16 + struct io_kiocb *link = req->link; 17 + 18 + if (link && link->opcode == IORING_OP_LINK_TIMEOUT) 19 + return __io_disarm_linked_timeout(req, link); 20 + 21 + return NULL; 22 + } 23 + 24 + __cold void io_flush_timeouts(struct io_ring_ctx *ctx); 25 + int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd); 26 + __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, 27 + bool cancel_all); 28 + void io_queue_linked_timeout(struct io_kiocb *req); 29 + bool io_disarm_next(struct io_kiocb *req); 30 + 31 + int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 32 + int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 33 + int io_timeout(struct io_kiocb *req, unsigned int issue_flags); 34 + int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 35 + int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags);