Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring: move timeout opcodes and handling into its own file
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+701
-660
+1
-1
io_uring/Makefile
+1
-1
io_uring/Makefile
+9
-658
io_uring/io_uring.c
+9
-658
io_uring/io_uring.c
···
105
105
#include "statx.h"
106
106
#include "net.h"
107
107
#include "msg_ring.h"
108
+
#include "timeout.h"
108
109
109
110
#define IORING_MAX_ENTRIES 32768
110
111
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
···
289
288
bool update_user_data;
290
289
};
291
290
292
-
struct io_timeout_data {
293
-
struct io_kiocb *req;
294
-
struct hrtimer timer;
295
-
struct timespec64 ts;
296
-
enum hrtimer_mode mode;
297
-
u32 flags;
298
-
};
299
-
300
291
struct io_cancel {
301
292
struct file *file;
302
293
u64 addr;
303
294
u32 flags;
304
295
s32 fd;
305
-
};
306
-
307
-
struct io_timeout {
308
-
struct file *file;
309
-
u32 off;
310
-
u32 target_seq;
311
-
struct list_head list;
312
-
/* head of the link, used by linked timeouts only */
313
-
struct io_kiocb *head;
314
-
/* for linked completions */
315
-
struct io_kiocb *prev;
316
-
};
317
-
318
-
struct io_timeout_rem {
319
-
struct file *file;
320
-
u64 addr;
321
-
322
-
/* timeout update */
323
-
struct timespec64 ts;
324
-
u32 flags;
325
-
bool ltimeout;
326
296
};
327
297
328
298
struct io_rw {
···
360
388
u32 seq;
361
389
};
362
390
363
-
struct io_cancel_data {
364
-
struct io_ring_ctx *ctx;
365
-
union {
366
-
u64 data;
367
-
struct file *file;
368
-
};
369
-
u32 flags;
370
-
int seq;
371
-
};
372
-
373
391
struct io_op_def {
374
392
/* needs req->file assigned */
375
393
unsigned needs_file : 1;
···
398
436
#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
399
437
#define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK)
400
438
401
-
static bool io_disarm_next(struct io_kiocb *req);
402
439
static void io_uring_del_tctx_node(unsigned long index);
403
440
static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
404
441
struct task_struct *task,
···
405
444
static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
406
445
407
446
static void io_dismantle_req(struct io_kiocb *req);
408
-
static void io_queue_linked_timeout(struct io_kiocb *req);
409
447
static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
410
448
struct io_uring_rsrc_update2 *up,
411
449
unsigned nr_args);
···
416
456
static void __io_submit_flush_completions(struct io_ring_ctx *ctx);
417
457
static int io_req_prep_async(struct io_kiocb *req);
418
458
419
-
static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer);
420
459
static void io_eventfd_signal(struct io_ring_ctx *ctx);
421
-
static void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags);
422
460
423
461
static struct kmem_cache *req_cachep;
424
462
···
566
608
*locked = true;
567
609
}
568
610
}
569
-
570
-
#define io_for_each_link(pos, head) \
571
-
for (pos = (head); pos; pos = pos->link)
572
611
573
612
static inline void io_submit_flush_completions(struct io_ring_ctx *ctx)
574
613
{
···
758
803
io_ring_submit_unlock(ctx, issue_flags);
759
804
}
760
805
761
-
static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
762
-
bool cancel_all)
763
-
__must_hold(&req->ctx->timeout_lock)
764
-
{
765
-
struct io_kiocb *req;
766
-
767
-
if (task && head->task != task)
768
-
return false;
769
-
if (cancel_all)
770
-
return true;
771
-
772
-
io_for_each_link(req, head) {
773
-
if (req->flags & REQ_F_INFLIGHT)
774
-
return true;
775
-
}
776
-
return false;
777
-
}
778
-
779
806
static bool io_match_linked(struct io_kiocb *head)
780
807
{
781
808
struct io_kiocb *req;
···
812
875
struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
813
876
814
877
complete(&ctx->ref_comp);
815
-
}
816
-
817
-
static inline bool io_is_timeout_noseq(struct io_kiocb *req)
818
-
{
819
-
struct io_timeout *timeout = io_kiocb_to_cmd(req);
820
-
821
-
return !timeout->off;
822
878
}
823
879
824
880
static __cold void io_fallback_req_func(struct work_struct *work)
···
1050
1120
io_queue_linked_timeout(link);
1051
1121
}
1052
1122
1053
-
static void io_kill_timeout(struct io_kiocb *req, int status)
1054
-
__must_hold(&req->ctx->completion_lock)
1055
-
__must_hold(&req->ctx->timeout_lock)
1056
-
{
1057
-
struct io_timeout_data *io = req->async_data;
1058
-
1059
-
if (hrtimer_try_to_cancel(&io->timer) != -1) {
1060
-
struct io_timeout *timeout = io_kiocb_to_cmd(req);
1061
-
1062
-
if (status)
1063
-
req_set_fail(req);
1064
-
atomic_set(&req->ctx->cq_timeouts,
1065
-
atomic_read(&req->ctx->cq_timeouts) + 1);
1066
-
list_del_init(&timeout->list);
1067
-
io_req_tw_post_queue(req, status, 0);
1068
-
}
1069
-
}
1070
-
1071
1123
static __cold void io_queue_deferred(struct io_ring_ctx *ctx)
1072
1124
{
1073
1125
while (!list_empty(&ctx->defer_list)) {
···
1062
1150
io_req_task_queue(de->req);
1063
1151
kfree(de);
1064
1152
}
1065
-
}
1066
-
1067
-
static __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
1068
-
__must_hold(&ctx->completion_lock)
1069
-
{
1070
-
u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
1071
-
struct io_timeout *timeout, *tmp;
1072
-
1073
-
spin_lock_irq(&ctx->timeout_lock);
1074
-
list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
1075
-
struct io_kiocb *req = cmd_to_io_kiocb(timeout);
1076
-
u32 events_needed, events_got;
1077
-
1078
-
if (io_is_timeout_noseq(req))
1079
-
break;
1080
-
1081
-
/*
1082
-
* Since seq can easily wrap around over time, subtract
1083
-
* the last seq at which timeouts were flushed before comparing.
1084
-
* Assuming not more than 2^31-1 events have happened since,
1085
-
* these subtractions won't have wrapped, so we can check if
1086
-
* target is in [last_seq, current_seq] by comparing the two.
1087
-
*/
1088
-
events_needed = timeout->target_seq - ctx->cq_last_tm_flush;
1089
-
events_got = seq - ctx->cq_last_tm_flush;
1090
-
if (events_got < events_needed)
1091
-
break;
1092
-
1093
-
io_kill_timeout(req, 0);
1094
-
}
1095
-
ctx->cq_last_tm_flush = seq;
1096
-
spin_unlock_irq(&ctx->timeout_lock);
1097
1153
}
1098
1154
1099
1155
static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
···
1465
1585
}
1466
1586
}
1467
1587
1468
-
static void __io_req_complete_post(struct io_kiocb *req)
1588
+
void __io_req_complete_post(struct io_kiocb *req)
1469
1589
{
1470
1590
if (!(req->flags & REQ_F_CQE_SKIP))
1471
1591
__io_fill_cqe_req(req->ctx, req);
1472
1592
__io_req_complete_put(req);
1473
1593
}
1474
1594
1475
-
static void io_req_complete_post(struct io_kiocb *req)
1595
+
void io_req_complete_post(struct io_kiocb *req)
1476
1596
{
1477
1597
struct io_ring_ctx *ctx = req->ctx;
1478
1598
···
1597
1717
io_put_file(req->file);
1598
1718
}
1599
1719
1600
-
static __cold void io_free_req(struct io_kiocb *req)
1720
+
__cold void io_free_req(struct io_kiocb *req)
1601
1721
{
1602
1722
struct io_ring_ctx *ctx = req->ctx;
1603
1723
···
1609
1729
wq_list_add_head(&req->comp_list, &ctx->locked_free_list);
1610
1730
ctx->locked_free_nr++;
1611
1731
spin_unlock(&ctx->completion_lock);
1612
-
}
1613
-
1614
-
static inline void io_remove_next_linked(struct io_kiocb *req)
1615
-
{
1616
-
struct io_kiocb *nxt = req->link;
1617
-
1618
-
req->link = nxt->link;
1619
-
nxt->link = NULL;
1620
-
}
1621
-
1622
-
static struct io_kiocb *io_disarm_linked_timeout(struct io_kiocb *req)
1623
-
__must_hold(&req->ctx->completion_lock)
1624
-
__must_hold(&req->ctx->timeout_lock)
1625
-
{
1626
-
struct io_kiocb *link = req->link;
1627
-
1628
-
if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
1629
-
struct io_timeout_data *io = link->async_data;
1630
-
struct io_timeout *timeout = io_kiocb_to_cmd(link);
1631
-
1632
-
io_remove_next_linked(req);
1633
-
timeout->head = NULL;
1634
-
if (hrtimer_try_to_cancel(&io->timer) != -1) {
1635
-
list_del(&timeout->list);
1636
-
return link;
1637
-
}
1638
-
}
1639
-
return NULL;
1640
-
}
1641
-
1642
-
static void io_fail_links(struct io_kiocb *req)
1643
-
__must_hold(&req->ctx->completion_lock)
1644
-
{
1645
-
struct io_kiocb *nxt, *link = req->link;
1646
-
bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES;
1647
-
1648
-
req->link = NULL;
1649
-
while (link) {
1650
-
long res = -ECANCELED;
1651
-
1652
-
if (link->flags & REQ_F_FAIL)
1653
-
res = link->cqe.res;
1654
-
1655
-
nxt = link->link;
1656
-
link->link = NULL;
1657
-
1658
-
trace_io_uring_fail_link(req->ctx, req, req->cqe.user_data,
1659
-
req->opcode, link);
1660
-
1661
-
if (ignore_cqes)
1662
-
link->flags |= REQ_F_CQE_SKIP;
1663
-
else
1664
-
link->flags &= ~REQ_F_CQE_SKIP;
1665
-
io_req_set_res(link, res, 0);
1666
-
__io_req_complete_post(link);
1667
-
link = nxt;
1668
-
}
1669
-
}
1670
-
1671
-
static bool io_disarm_next(struct io_kiocb *req)
1672
-
__must_hold(&req->ctx->completion_lock)
1673
-
{
1674
-
struct io_kiocb *link = NULL;
1675
-
bool posted = false;
1676
-
1677
-
if (req->flags & REQ_F_ARM_LTIMEOUT) {
1678
-
link = req->link;
1679
-
req->flags &= ~REQ_F_ARM_LTIMEOUT;
1680
-
if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
1681
-
io_remove_next_linked(req);
1682
-
io_req_tw_post_queue(link, -ECANCELED, 0);
1683
-
posted = true;
1684
-
}
1685
-
} else if (req->flags & REQ_F_LINK_TIMEOUT) {
1686
-
struct io_ring_ctx *ctx = req->ctx;
1687
-
1688
-
spin_lock_irq(&ctx->timeout_lock);
1689
-
link = io_disarm_linked_timeout(req);
1690
-
spin_unlock_irq(&ctx->timeout_lock);
1691
-
if (link) {
1692
-
posted = true;
1693
-
io_req_tw_post_queue(link, -ECANCELED, 0);
1694
-
}
1695
-
}
1696
-
if (unlikely((req->flags & REQ_F_FAIL) &&
1697
-
!(req->flags & REQ_F_HARDLINK))) {
1698
-
posted |= (req->link != NULL);
1699
-
io_fail_links(req);
1700
-
}
1701
-
return posted;
1702
1732
}
1703
1733
1704
1734
static void __io_req_find_next_prep(struct io_kiocb *req)
···
1823
2033
io_req_complete_post(req);
1824
2034
}
1825
2035
1826
-
static void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags)
2036
+
void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags)
1827
2037
{
1828
2038
io_req_set_res(req, res, cflags);
1829
2039
req->io_task_work.func = io_req_tw_post;
···
1847
2057
io_req_complete_failed(req, -EFAULT);
1848
2058
}
1849
2059
1850
-
static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
2060
+
void io_req_task_queue_fail(struct io_kiocb *req, int ret)
1851
2061
{
1852
2062
io_req_set_res(req, ret, 0);
1853
2063
req->io_task_work.func = io_req_task_cancel;
···
1866
2076
io_req_task_work_add(req);
1867
2077
}
1868
2078
1869
-
static void io_queue_next(struct io_kiocb *req)
2079
+
void io_queue_next(struct io_kiocb *req)
1870
2080
{
1871
2081
struct io_kiocb *nxt = io_req_find_next(req);
1872
2082
···
1965
2175
io_free_req(req);
1966
2176
}
1967
2177
return nxt;
1968
-
}
1969
-
1970
-
static inline void io_put_req(struct io_kiocb *req)
1971
-
{
1972
-
if (req_ref_put_and_test(req)) {
1973
-
io_queue_next(req);
1974
-
io_free_req(req);
1975
-
}
1976
2178
}
1977
2179
1978
2180
static unsigned io_cqring_events(struct io_ring_ctx *ctx)
···
2233
2451
return false;
2234
2452
}
2235
2453
2236
-
static inline void io_req_task_complete(struct io_kiocb *req, bool *locked)
2454
+
inline void io_req_task_complete(struct io_kiocb *req, bool *locked)
2237
2455
{
2238
2456
if (*locked) {
2239
2457
req->cqe.flags |= io_put_kbuf(req, 0);
···
4382
4600
return IOU_OK;
4383
4601
}
4384
4602
4385
-
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
4386
-
{
4387
-
struct io_timeout_data *data = container_of(timer,
4388
-
struct io_timeout_data, timer);
4389
-
struct io_kiocb *req = data->req;
4390
-
struct io_timeout *timeout = io_kiocb_to_cmd(req);
4391
-
struct io_ring_ctx *ctx = req->ctx;
4392
-
unsigned long flags;
4393
-
4394
-
spin_lock_irqsave(&ctx->timeout_lock, flags);
4395
-
list_del_init(&timeout->list);
4396
-
atomic_set(&req->ctx->cq_timeouts,
4397
-
atomic_read(&req->ctx->cq_timeouts) + 1);
4398
-
spin_unlock_irqrestore(&ctx->timeout_lock, flags);
4399
-
4400
-
if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS))
4401
-
req_set_fail(req);
4402
-
4403
-
io_req_set_res(req, -ETIME, 0);
4404
-
req->io_task_work.func = io_req_task_complete;
4405
-
io_req_task_work_add(req);
4406
-
return HRTIMER_NORESTART;
4407
-
}
4408
-
4409
-
static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
4410
-
struct io_cancel_data *cd)
4411
-
__must_hold(&ctx->timeout_lock)
4412
-
{
4413
-
struct io_timeout *timeout;
4414
-
struct io_timeout_data *io;
4415
-
struct io_kiocb *req = NULL;
4416
-
4417
-
list_for_each_entry(timeout, &ctx->timeout_list, list) {
4418
-
struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
4419
-
4420
-
if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) &&
4421
-
cd->data != tmp->cqe.user_data)
4422
-
continue;
4423
-
if (cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY)) {
4424
-
if (cd->seq == tmp->work.cancel_seq)
4425
-
continue;
4426
-
tmp->work.cancel_seq = cd->seq;
4427
-
}
4428
-
req = tmp;
4429
-
break;
4430
-
}
4431
-
if (!req)
4432
-
return ERR_PTR(-ENOENT);
4433
-
4434
-
io = req->async_data;
4435
-
if (hrtimer_try_to_cancel(&io->timer) == -1)
4436
-
return ERR_PTR(-EALREADY);
4437
-
timeout = io_kiocb_to_cmd(req);
4438
-
list_del_init(&timeout->list);
4439
-
return req;
4440
-
}
4441
-
4442
-
static int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
4443
-
__must_hold(&ctx->completion_lock)
4444
-
{
4445
-
struct io_kiocb *req;
4446
-
4447
-
spin_lock_irq(&ctx->timeout_lock);
4448
-
req = io_timeout_extract(ctx, cd);
4449
-
spin_unlock_irq(&ctx->timeout_lock);
4450
-
4451
-
if (IS_ERR(req))
4452
-
return PTR_ERR(req);
4453
-
io_req_task_queue_fail(req, -ECANCELED);
4454
-
return 0;
4455
-
}
4456
-
4457
-
static clockid_t io_timeout_get_clock(struct io_timeout_data *data)
4458
-
{
4459
-
switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) {
4460
-
case IORING_TIMEOUT_BOOTTIME:
4461
-
return CLOCK_BOOTTIME;
4462
-
case IORING_TIMEOUT_REALTIME:
4463
-
return CLOCK_REALTIME;
4464
-
default:
4465
-
/* can't happen, vetted at prep time */
4466
-
WARN_ON_ONCE(1);
4467
-
fallthrough;
4468
-
case 0:
4469
-
return CLOCK_MONOTONIC;
4470
-
}
4471
-
}
4472
-
4473
-
static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
4474
-
struct timespec64 *ts, enum hrtimer_mode mode)
4475
-
__must_hold(&ctx->timeout_lock)
4476
-
{
4477
-
struct io_timeout_data *io;
4478
-
struct io_timeout *timeout;
4479
-
struct io_kiocb *req = NULL;
4480
-
4481
-
list_for_each_entry(timeout, &ctx->ltimeout_list, list) {
4482
-
struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
4483
-
4484
-
if (user_data == tmp->cqe.user_data) {
4485
-
req = tmp;
4486
-
break;
4487
-
}
4488
-
}
4489
-
if (!req)
4490
-
return -ENOENT;
4491
-
4492
-
io = req->async_data;
4493
-
if (hrtimer_try_to_cancel(&io->timer) == -1)
4494
-
return -EALREADY;
4495
-
hrtimer_init(&io->timer, io_timeout_get_clock(io), mode);
4496
-
io->timer.function = io_link_timeout_fn;
4497
-
hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode);
4498
-
return 0;
4499
-
}
4500
-
4501
-
static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
4502
-
struct timespec64 *ts, enum hrtimer_mode mode)
4503
-
__must_hold(&ctx->timeout_lock)
4504
-
{
4505
-
struct io_cancel_data cd = { .data = user_data, };
4506
-
struct io_kiocb *req = io_timeout_extract(ctx, &cd);
4507
-
struct io_timeout *timeout = io_kiocb_to_cmd(req);
4508
-
struct io_timeout_data *data;
4509
-
4510
-
if (IS_ERR(req))
4511
-
return PTR_ERR(req);
4512
-
4513
-
timeout->off = 0; /* noseq */
4514
-
data = req->async_data;
4515
-
list_add_tail(&timeout->list, &ctx->timeout_list);
4516
-
hrtimer_init(&data->timer, io_timeout_get_clock(data), mode);
4517
-
data->timer.function = io_timeout_fn;
4518
-
hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
4519
-
return 0;
4520
-
}
4521
-
4522
-
static int io_timeout_remove_prep(struct io_kiocb *req,
4523
-
const struct io_uring_sqe *sqe)
4524
-
{
4525
-
struct io_timeout_rem *tr = io_kiocb_to_cmd(req);
4526
-
4527
-
if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
4528
-
return -EINVAL;
4529
-
if (sqe->buf_index || sqe->len || sqe->splice_fd_in)
4530
-
return -EINVAL;
4531
-
4532
-
tr->ltimeout = false;
4533
-
tr->addr = READ_ONCE(sqe->addr);
4534
-
tr->flags = READ_ONCE(sqe->timeout_flags);
4535
-
if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) {
4536
-
if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
4537
-
return -EINVAL;
4538
-
if (tr->flags & IORING_LINK_TIMEOUT_UPDATE)
4539
-
tr->ltimeout = true;
4540
-
if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS))
4541
-
return -EINVAL;
4542
-
if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
4543
-
return -EFAULT;
4544
-
if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0)
4545
-
return -EINVAL;
4546
-
} else if (tr->flags) {
4547
-
/* timeout removal doesn't support flags */
4548
-
return -EINVAL;
4549
-
}
4550
-
4551
-
return 0;
4552
-
}
4553
-
4554
-
static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
4555
-
{
4556
-
return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
4557
-
: HRTIMER_MODE_REL;
4558
-
}
4559
-
4560
-
/*
4561
-
* Remove or update an existing timeout command
4562
-
*/
4563
-
static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
4564
-
{
4565
-
struct io_timeout_rem *tr = io_kiocb_to_cmd(req);
4566
-
struct io_ring_ctx *ctx = req->ctx;
4567
-
int ret;
4568
-
4569
-
if (!(tr->flags & IORING_TIMEOUT_UPDATE)) {
4570
-
struct io_cancel_data cd = { .data = tr->addr, };
4571
-
4572
-
spin_lock(&ctx->completion_lock);
4573
-
ret = io_timeout_cancel(ctx, &cd);
4574
-
spin_unlock(&ctx->completion_lock);
4575
-
} else {
4576
-
enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);
4577
-
4578
-
spin_lock_irq(&ctx->timeout_lock);
4579
-
if (tr->ltimeout)
4580
-
ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode);
4581
-
else
4582
-
ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode);
4583
-
spin_unlock_irq(&ctx->timeout_lock);
4584
-
}
4585
-
4586
-
if (ret < 0)
4587
-
req_set_fail(req);
4588
-
io_req_set_res(req, ret, 0);
4589
-
return IOU_OK;
4590
-
}
4591
-
4592
-
static int __io_timeout_prep(struct io_kiocb *req,
4593
-
const struct io_uring_sqe *sqe,
4594
-
bool is_timeout_link)
4595
-
{
4596
-
struct io_timeout *timeout = io_kiocb_to_cmd(req);
4597
-
struct io_timeout_data *data;
4598
-
unsigned flags;
4599
-
u32 off = READ_ONCE(sqe->off);
4600
-
4601
-
if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in)
4602
-
return -EINVAL;
4603
-
if (off && is_timeout_link)
4604
-
return -EINVAL;
4605
-
flags = READ_ONCE(sqe->timeout_flags);
4606
-
if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK |
4607
-
IORING_TIMEOUT_ETIME_SUCCESS))
4608
-
return -EINVAL;
4609
-
/* more than one clock specified is invalid, obviously */
4610
-
if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
4611
-
return -EINVAL;
4612
-
4613
-
INIT_LIST_HEAD(&timeout->list);
4614
-
timeout->off = off;
4615
-
if (unlikely(off && !req->ctx->off_timeout_used))
4616
-
req->ctx->off_timeout_used = true;
4617
-
4618
-
if (WARN_ON_ONCE(req_has_async_data(req)))
4619
-
return -EFAULT;
4620
-
if (io_alloc_async_data(req))
4621
-
return -ENOMEM;
4622
-
4623
-
data = req->async_data;
4624
-
data->req = req;
4625
-
data->flags = flags;
4626
-
4627
-
if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
4628
-
return -EFAULT;
4629
-
4630
-
if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0)
4631
-
return -EINVAL;
4632
-
4633
-
INIT_LIST_HEAD(&timeout->list);
4634
-
data->mode = io_translate_timeout_mode(flags);
4635
-
hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode);
4636
-
4637
-
if (is_timeout_link) {
4638
-
struct io_submit_link *link = &req->ctx->submit_state.link;
4639
-
4640
-
if (!link->head)
4641
-
return -EINVAL;
4642
-
if (link->last->opcode == IORING_OP_LINK_TIMEOUT)
4643
-
return -EINVAL;
4644
-
timeout->head = link->last;
4645
-
link->last->flags |= REQ_F_ARM_LTIMEOUT;
4646
-
}
4647
-
return 0;
4648
-
}
4649
-
4650
-
static int io_timeout_prep(struct io_kiocb *req,
4651
-
const struct io_uring_sqe *sqe)
4652
-
{
4653
-
return __io_timeout_prep(req, sqe, false);
4654
-
}
4655
-
4656
-
static int io_link_timeout_prep(struct io_kiocb *req,
4657
-
const struct io_uring_sqe *sqe)
4658
-
{
4659
-
return __io_timeout_prep(req, sqe, true);
4660
-
}
4661
-
4662
-
static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
4663
-
{
4664
-
struct io_timeout *timeout = io_kiocb_to_cmd(req);
4665
-
struct io_ring_ctx *ctx = req->ctx;
4666
-
struct io_timeout_data *data = req->async_data;
4667
-
struct list_head *entry;
4668
-
u32 tail, off = timeout->off;
4669
-
4670
-
spin_lock_irq(&ctx->timeout_lock);
4671
-
4672
-
/*
4673
-
* sqe->off holds how many events that need to occur for this
4674
-
* timeout event to be satisfied. If it isn't set, then this is
4675
-
* a pure timeout request, sequence isn't used.
4676
-
*/
4677
-
if (io_is_timeout_noseq(req)) {
4678
-
entry = ctx->timeout_list.prev;
4679
-
goto add;
4680
-
}
4681
-
4682
-
tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
4683
-
timeout->target_seq = tail + off;
4684
-
4685
-
/* Update the last seq here in case io_flush_timeouts() hasn't.
4686
-
* This is safe because ->completion_lock is held, and submissions
4687
-
* and completions are never mixed in the same ->completion_lock section.
4688
-
*/
4689
-
ctx->cq_last_tm_flush = tail;
4690
-
4691
-
/*
4692
-
* Insertion sort, ensuring the first entry in the list is always
4693
-
* the one we need first.
4694
-
*/
4695
-
list_for_each_prev(entry, &ctx->timeout_list) {
4696
-
struct io_timeout *nextt = list_entry(entry, struct io_timeout, list);
4697
-
struct io_kiocb *nxt = cmd_to_io_kiocb(nextt);
4698
-
4699
-
if (io_is_timeout_noseq(nxt))
4700
-
continue;
4701
-
/* nxt.seq is behind @tail, otherwise would've been completed */
4702
-
if (off >= nextt->target_seq - tail)
4703
-
break;
4704
-
}
4705
-
add:
4706
-
list_add(&timeout->list, entry);
4707
-
data->timer.function = io_timeout_fn;
4708
-
hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
4709
-
spin_unlock_irq(&ctx->timeout_lock);
4710
-
return IOU_ISSUE_SKIP_COMPLETE;
4711
-
}
4712
-
4713
4603
static bool io_cancel_cb(struct io_wq_work *work, void *data)
4714
4604
{
4715
4605
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
···
4433
4979
return ret;
4434
4980
}
4435
4981
4436
-
static int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd)
4982
+
int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd)
4437
4983
{
4438
4984
struct io_ring_ctx *ctx = req->ctx;
4439
4985
int ret;
···
4914
5460
if (file && file->f_op == &io_uring_fops)
4915
5461
io_req_track_inflight(req);
4916
5462
return file;
4917
-
}
4918
-
4919
-
static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)
4920
-
{
4921
-
struct io_timeout *timeout = io_kiocb_to_cmd(req);
4922
-
struct io_kiocb *prev = timeout->prev;
4923
-
int ret = -ENOENT;
4924
-
4925
-
if (prev) {
4926
-
if (!(req->task->flags & PF_EXITING)) {
4927
-
struct io_cancel_data cd = {
4928
-
.ctx = req->ctx,
4929
-
.data = prev->cqe.user_data,
4930
-
};
4931
-
4932
-
ret = io_try_cancel(req, &cd);
4933
-
}
4934
-
io_req_set_res(req, ret ?: -ETIME, 0);
4935
-
io_req_complete_post(req);
4936
-
io_put_req(prev);
4937
-
} else {
4938
-
io_req_set_res(req, -ETIME, 0);
4939
-
io_req_complete_post(req);
4940
-
}
4941
-
}
4942
-
4943
-
static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
4944
-
{
4945
-
struct io_timeout_data *data = container_of(timer,
4946
-
struct io_timeout_data, timer);
4947
-
struct io_kiocb *prev, *req = data->req;
4948
-
struct io_timeout *timeout = io_kiocb_to_cmd(req);
4949
-
struct io_ring_ctx *ctx = req->ctx;
4950
-
unsigned long flags;
4951
-
4952
-
spin_lock_irqsave(&ctx->timeout_lock, flags);
4953
-
prev = timeout->head;
4954
-
timeout->head = NULL;
4955
-
4956
-
/*
4957
-
* We don't expect the list to be empty, that will only happen if we
4958
-
* race with the completion of the linked work.
4959
-
*/
4960
-
if (prev) {
4961
-
io_remove_next_linked(prev);
4962
-
if (!req_ref_inc_not_zero(prev))
4963
-
prev = NULL;
4964
-
}
4965
-
list_del(&timeout->list);
4966
-
timeout->prev = prev;
4967
-
spin_unlock_irqrestore(&ctx->timeout_lock, flags);
4968
-
4969
-
req->io_task_work.func = io_req_task_link_timeout;
4970
-
io_req_task_work_add(req);
4971
-
return HRTIMER_NORESTART;
4972
-
}
4973
-
4974
-
static void io_queue_linked_timeout(struct io_kiocb *req)
4975
-
{
4976
-
struct io_timeout *timeout = io_kiocb_to_cmd(req);
4977
-
struct io_ring_ctx *ctx = req->ctx;
4978
-
4979
-
spin_lock_irq(&ctx->timeout_lock);
4980
-
/*
4981
-
* If the back reference is NULL, then our linked request finished
4982
-
* before we got a chance to setup the timer
4983
-
*/
4984
-
if (timeout->head) {
4985
-
struct io_timeout_data *data = req->async_data;
4986
-
4987
-
data->timer.function = io_link_timeout_fn;
4988
-
hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
4989
-
data->mode);
4990
-
list_add_tail(&timeout->list, &ctx->ltimeout_list);
4991
-
}
4992
-
spin_unlock_irq(&ctx->timeout_lock);
4993
-
/* drop submission reference */
4994
-
io_put_req(req);
4995
5463
}
4996
5464
4997
5465
static void io_queue_async(struct io_kiocb *req, int ret)
···
7490
8114
spin_unlock(&ctx->completion_lock);
7491
8115
7492
8116
io_ring_ctx_free(ctx);
7493
-
}
7494
-
7495
-
/* Returns true if we found and killed one or more timeouts */
7496
-
static __cold bool io_kill_timeouts(struct io_ring_ctx *ctx,
7497
-
struct task_struct *tsk, bool cancel_all)
7498
-
{
7499
-
struct io_timeout *timeout, *tmp;
7500
-
int canceled = 0;
7501
-
7502
-
spin_lock(&ctx->completion_lock);
7503
-
spin_lock_irq(&ctx->timeout_lock);
7504
-
list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
7505
-
struct io_kiocb *req = cmd_to_io_kiocb(timeout);
7506
-
7507
-
if (io_match_task(req, tsk, cancel_all)) {
7508
-
io_kill_timeout(req, -ECANCELED);
7509
-
canceled++;
7510
-
}
7511
-
}
7512
-
spin_unlock_irq(&ctx->timeout_lock);
7513
-
io_commit_cqring(ctx);
7514
-
spin_unlock(&ctx->completion_lock);
7515
-
if (canceled != 0)
7516
-
io_cqring_ev_posted(ctx);
7517
-
return canceled != 0;
7518
8117
}
7519
8118
7520
8119
static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
+12
-1
io_uring/io_uring.h
+12
-1
io_uring/io_uring.h
···
65
65
}
66
66
67
67
void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
68
-
68
+
void io_req_complete_post(struct io_kiocb *req);
69
+
void __io_req_complete_post(struct io_kiocb *req);
69
70
bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
70
71
u32 cflags);
71
72
void io_cqring_ev_posted(struct io_ring_ctx *ctx);
···
97
96
bool io_is_uring_fops(struct file *file);
98
97
bool io_alloc_async_data(struct io_kiocb *req);
99
98
void io_req_task_work_add(struct io_kiocb *req);
99
+
void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags);
100
+
void io_req_task_complete(struct io_kiocb *req, bool *locked);
101
+
void io_req_task_queue_fail(struct io_kiocb *req, int ret);
102
+
int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd);
103
+
104
+
void io_free_req(struct io_kiocb *req);
105
+
void io_queue_next(struct io_kiocb *req);
106
+
107
+
#define io_for_each_link(pos, head) \
108
+
for (pos = (head); pos; pos = pos->link)
100
109
101
110
#endif
+10
io_uring/io_uring_types.h
+10
io_uring/io_uring_types.h
+634
io_uring/timeout.c
+634
io_uring/timeout.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#include <linux/kernel.h>
3
+
#include <linux/errno.h>
4
+
#include <linux/file.h>
5
+
#include <linux/io_uring.h>
6
+
7
+
#include <trace/events/io_uring.h>
8
+
9
+
#include <uapi/linux/io_uring.h>
10
+
11
+
#include "io_uring_types.h"
12
+
#include "io_uring.h"
13
+
#include "refs.h"
14
+
#include "timeout.h"
15
+
16
+
struct io_timeout {
17
+
struct file *file;
18
+
u32 off;
19
+
u32 target_seq;
20
+
struct list_head list;
21
+
/* head of the link, used by linked timeouts only */
22
+
struct io_kiocb *head;
23
+
/* for linked completions */
24
+
struct io_kiocb *prev;
25
+
};
26
+
27
+
struct io_timeout_rem {
28
+
struct file *file;
29
+
u64 addr;
30
+
31
+
/* timeout update */
32
+
struct timespec64 ts;
33
+
u32 flags;
34
+
bool ltimeout;
35
+
};
36
+
37
+
static inline bool io_is_timeout_noseq(struct io_kiocb *req)
38
+
{
39
+
struct io_timeout *timeout = io_kiocb_to_cmd(req);
40
+
41
+
return !timeout->off;
42
+
}
43
+
44
+
static inline void io_put_req(struct io_kiocb *req)
45
+
{
46
+
if (req_ref_put_and_test(req)) {
47
+
io_queue_next(req);
48
+
io_free_req(req);
49
+
}
50
+
}
51
+
52
+
static void io_kill_timeout(struct io_kiocb *req, int status)
53
+
__must_hold(&req->ctx->completion_lock)
54
+
__must_hold(&req->ctx->timeout_lock)
55
+
{
56
+
struct io_timeout_data *io = req->async_data;
57
+
58
+
if (hrtimer_try_to_cancel(&io->timer) != -1) {
59
+
struct io_timeout *timeout = io_kiocb_to_cmd(req);
60
+
61
+
if (status)
62
+
req_set_fail(req);
63
+
atomic_set(&req->ctx->cq_timeouts,
64
+
atomic_read(&req->ctx->cq_timeouts) + 1);
65
+
list_del_init(&timeout->list);
66
+
io_req_tw_post_queue(req, status, 0);
67
+
}
68
+
}
69
+
70
+
__cold void io_flush_timeouts(struct io_ring_ctx *ctx)
71
+
__must_hold(&ctx->completion_lock)
72
+
{
73
+
u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
74
+
struct io_timeout *timeout, *tmp;
75
+
76
+
spin_lock_irq(&ctx->timeout_lock);
77
+
list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
78
+
struct io_kiocb *req = cmd_to_io_kiocb(timeout);
79
+
u32 events_needed, events_got;
80
+
81
+
if (io_is_timeout_noseq(req))
82
+
break;
83
+
84
+
/*
85
+
* Since seq can easily wrap around over time, subtract
86
+
* the last seq at which timeouts were flushed before comparing.
87
+
* Assuming not more than 2^31-1 events have happened since,
88
+
* these subtractions won't have wrapped, so we can check if
89
+
* target is in [last_seq, current_seq] by comparing the two.
90
+
*/
91
+
events_needed = timeout->target_seq - ctx->cq_last_tm_flush;
92
+
events_got = seq - ctx->cq_last_tm_flush;
93
+
if (events_got < events_needed)
94
+
break;
95
+
96
+
io_kill_timeout(req, 0);
97
+
}
98
+
ctx->cq_last_tm_flush = seq;
99
+
spin_unlock_irq(&ctx->timeout_lock);
100
+
}
101
+
102
+
static void io_fail_links(struct io_kiocb *req)
103
+
__must_hold(&req->ctx->completion_lock)
104
+
{
105
+
struct io_kiocb *nxt, *link = req->link;
106
+
bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES;
107
+
108
+
req->link = NULL;
109
+
while (link) {
110
+
long res = -ECANCELED;
111
+
112
+
if (link->flags & REQ_F_FAIL)
113
+
res = link->cqe.res;
114
+
115
+
nxt = link->link;
116
+
link->link = NULL;
117
+
118
+
trace_io_uring_fail_link(req->ctx, req, req->cqe.user_data,
119
+
req->opcode, link);
120
+
121
+
if (ignore_cqes)
122
+
link->flags |= REQ_F_CQE_SKIP;
123
+
else
124
+
link->flags &= ~REQ_F_CQE_SKIP;
125
+
io_req_set_res(link, res, 0);
126
+
__io_req_complete_post(link);
127
+
link = nxt;
128
+
}
129
+
}
130
+
131
+
static inline void io_remove_next_linked(struct io_kiocb *req)
132
+
{
133
+
struct io_kiocb *nxt = req->link;
134
+
135
+
req->link = nxt->link;
136
+
nxt->link = NULL;
137
+
}
138
+
139
+
bool io_disarm_next(struct io_kiocb *req)
140
+
__must_hold(&req->ctx->completion_lock)
141
+
{
142
+
struct io_kiocb *link = NULL;
143
+
bool posted = false;
144
+
145
+
if (req->flags & REQ_F_ARM_LTIMEOUT) {
146
+
link = req->link;
147
+
req->flags &= ~REQ_F_ARM_LTIMEOUT;
148
+
if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
149
+
io_remove_next_linked(req);
150
+
io_req_tw_post_queue(link, -ECANCELED, 0);
151
+
posted = true;
152
+
}
153
+
} else if (req->flags & REQ_F_LINK_TIMEOUT) {
154
+
struct io_ring_ctx *ctx = req->ctx;
155
+
156
+
spin_lock_irq(&ctx->timeout_lock);
157
+
link = io_disarm_linked_timeout(req);
158
+
spin_unlock_irq(&ctx->timeout_lock);
159
+
if (link) {
160
+
posted = true;
161
+
io_req_tw_post_queue(link, -ECANCELED, 0);
162
+
}
163
+
}
164
+
if (unlikely((req->flags & REQ_F_FAIL) &&
165
+
!(req->flags & REQ_F_HARDLINK))) {
166
+
posted |= (req->link != NULL);
167
+
io_fail_links(req);
168
+
}
169
+
return posted;
170
+
}
171
+
172
+
struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
173
+
struct io_kiocb *link)
174
+
__must_hold(&req->ctx->completion_lock)
175
+
__must_hold(&req->ctx->timeout_lock)
176
+
{
177
+
struct io_timeout_data *io = link->async_data;
178
+
struct io_timeout *timeout = io_kiocb_to_cmd(link);
179
+
180
+
io_remove_next_linked(req);
181
+
timeout->head = NULL;
182
+
if (hrtimer_try_to_cancel(&io->timer) != -1) {
183
+
list_del(&timeout->list);
184
+
return link;
185
+
}
186
+
187
+
return NULL;
188
+
}
189
+
190
+
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
191
+
{
192
+
struct io_timeout_data *data = container_of(timer,
193
+
struct io_timeout_data, timer);
194
+
struct io_kiocb *req = data->req;
195
+
struct io_timeout *timeout = io_kiocb_to_cmd(req);
196
+
struct io_ring_ctx *ctx = req->ctx;
197
+
unsigned long flags;
198
+
199
+
spin_lock_irqsave(&ctx->timeout_lock, flags);
200
+
list_del_init(&timeout->list);
201
+
atomic_set(&req->ctx->cq_timeouts,
202
+
atomic_read(&req->ctx->cq_timeouts) + 1);
203
+
spin_unlock_irqrestore(&ctx->timeout_lock, flags);
204
+
205
+
if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS))
206
+
req_set_fail(req);
207
+
208
+
io_req_set_res(req, -ETIME, 0);
209
+
req->io_task_work.func = io_req_task_complete;
210
+
io_req_task_work_add(req);
211
+
return HRTIMER_NORESTART;
212
+
}
213
+
214
+
static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
215
+
struct io_cancel_data *cd)
216
+
__must_hold(&ctx->timeout_lock)
217
+
{
218
+
struct io_timeout *timeout;
219
+
struct io_timeout_data *io;
220
+
struct io_kiocb *req = NULL;
221
+
222
+
list_for_each_entry(timeout, &ctx->timeout_list, list) {
223
+
struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
224
+
225
+
if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) &&
226
+
cd->data != tmp->cqe.user_data)
227
+
continue;
228
+
if (cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY)) {
229
+
if (cd->seq == tmp->work.cancel_seq)
230
+
continue;
231
+
tmp->work.cancel_seq = cd->seq;
232
+
}
233
+
req = tmp;
234
+
break;
235
+
}
236
+
if (!req)
237
+
return ERR_PTR(-ENOENT);
238
+
239
+
io = req->async_data;
240
+
if (hrtimer_try_to_cancel(&io->timer) == -1)
241
+
return ERR_PTR(-EALREADY);
242
+
timeout = io_kiocb_to_cmd(req);
243
+
list_del_init(&timeout->list);
244
+
return req;
245
+
}
246
+
247
+
int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
248
+
__must_hold(&ctx->completion_lock)
249
+
{
250
+
struct io_kiocb *req;
251
+
252
+
spin_lock_irq(&ctx->timeout_lock);
253
+
req = io_timeout_extract(ctx, cd);
254
+
spin_unlock_irq(&ctx->timeout_lock);
255
+
256
+
if (IS_ERR(req))
257
+
return PTR_ERR(req);
258
+
io_req_task_queue_fail(req, -ECANCELED);
259
+
return 0;
260
+
}
261
+
262
+
static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)
263
+
{
264
+
struct io_timeout *timeout = io_kiocb_to_cmd(req);
265
+
struct io_kiocb *prev = timeout->prev;
266
+
int ret = -ENOENT;
267
+
268
+
if (prev) {
269
+
if (!(req->task->flags & PF_EXITING)) {
270
+
struct io_cancel_data cd = {
271
+
.ctx = req->ctx,
272
+
.data = prev->cqe.user_data,
273
+
};
274
+
275
+
ret = io_try_cancel(req, &cd);
276
+
}
277
+
io_req_set_res(req, ret ?: -ETIME, 0);
278
+
io_req_complete_post(req);
279
+
io_put_req(prev);
280
+
} else {
281
+
io_req_set_res(req, -ETIME, 0);
282
+
io_req_complete_post(req);
283
+
}
284
+
}
285
+
286
+
static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
287
+
{
288
+
struct io_timeout_data *data = container_of(timer,
289
+
struct io_timeout_data, timer);
290
+
struct io_kiocb *prev, *req = data->req;
291
+
struct io_timeout *timeout = io_kiocb_to_cmd(req);
292
+
struct io_ring_ctx *ctx = req->ctx;
293
+
unsigned long flags;
294
+
295
+
spin_lock_irqsave(&ctx->timeout_lock, flags);
296
+
prev = timeout->head;
297
+
timeout->head = NULL;
298
+
299
+
/*
300
+
* We don't expect the list to be empty, that will only happen if we
301
+
* race with the completion of the linked work.
302
+
*/
303
+
if (prev) {
304
+
io_remove_next_linked(prev);
305
+
if (!req_ref_inc_not_zero(prev))
306
+
prev = NULL;
307
+
}
308
+
list_del(&timeout->list);
309
+
timeout->prev = prev;
310
+
spin_unlock_irqrestore(&ctx->timeout_lock, flags);
311
+
312
+
req->io_task_work.func = io_req_task_link_timeout;
313
+
io_req_task_work_add(req);
314
+
return HRTIMER_NORESTART;
315
+
}
316
+
317
+
static clockid_t io_timeout_get_clock(struct io_timeout_data *data)
318
+
{
319
+
switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) {
320
+
case IORING_TIMEOUT_BOOTTIME:
321
+
return CLOCK_BOOTTIME;
322
+
case IORING_TIMEOUT_REALTIME:
323
+
return CLOCK_REALTIME;
324
+
default:
325
+
/* can't happen, vetted at prep time */
326
+
WARN_ON_ONCE(1);
327
+
fallthrough;
328
+
case 0:
329
+
return CLOCK_MONOTONIC;
330
+
}
331
+
}
332
+
333
+
static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
334
+
struct timespec64 *ts, enum hrtimer_mode mode)
335
+
__must_hold(&ctx->timeout_lock)
336
+
{
337
+
struct io_timeout_data *io;
338
+
struct io_timeout *timeout;
339
+
struct io_kiocb *req = NULL;
340
+
341
+
list_for_each_entry(timeout, &ctx->ltimeout_list, list) {
342
+
struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
343
+
344
+
if (user_data == tmp->cqe.user_data) {
345
+
req = tmp;
346
+
break;
347
+
}
348
+
}
349
+
if (!req)
350
+
return -ENOENT;
351
+
352
+
io = req->async_data;
353
+
if (hrtimer_try_to_cancel(&io->timer) == -1)
354
+
return -EALREADY;
355
+
hrtimer_init(&io->timer, io_timeout_get_clock(io), mode);
356
+
io->timer.function = io_link_timeout_fn;
357
+
hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode);
358
+
return 0;
359
+
}
360
+
361
+
static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
362
+
struct timespec64 *ts, enum hrtimer_mode mode)
363
+
__must_hold(&ctx->timeout_lock)
364
+
{
365
+
struct io_cancel_data cd = { .data = user_data, };
366
+
struct io_kiocb *req = io_timeout_extract(ctx, &cd);
367
+
struct io_timeout *timeout = io_kiocb_to_cmd(req);
368
+
struct io_timeout_data *data;
369
+
370
+
if (IS_ERR(req))
371
+
return PTR_ERR(req);
372
+
373
+
timeout->off = 0; /* noseq */
374
+
data = req->async_data;
375
+
list_add_tail(&timeout->list, &ctx->timeout_list);
376
+
hrtimer_init(&data->timer, io_timeout_get_clock(data), mode);
377
+
data->timer.function = io_timeout_fn;
378
+
hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
379
+
return 0;
380
+
}
381
+
382
+
int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
383
+
{
384
+
struct io_timeout_rem *tr = io_kiocb_to_cmd(req);
385
+
386
+
if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
387
+
return -EINVAL;
388
+
if (sqe->buf_index || sqe->len || sqe->splice_fd_in)
389
+
return -EINVAL;
390
+
391
+
tr->ltimeout = false;
392
+
tr->addr = READ_ONCE(sqe->addr);
393
+
tr->flags = READ_ONCE(sqe->timeout_flags);
394
+
if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) {
395
+
if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
396
+
return -EINVAL;
397
+
if (tr->flags & IORING_LINK_TIMEOUT_UPDATE)
398
+
tr->ltimeout = true;
399
+
if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS))
400
+
return -EINVAL;
401
+
if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
402
+
return -EFAULT;
403
+
if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0)
404
+
return -EINVAL;
405
+
} else if (tr->flags) {
406
+
/* timeout removal doesn't support flags */
407
+
return -EINVAL;
408
+
}
409
+
410
+
return 0;
411
+
}
412
+
413
+
static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
414
+
{
415
+
return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
416
+
: HRTIMER_MODE_REL;
417
+
}
418
+
419
+
/*
420
+
* Remove or update an existing timeout command
421
+
*/
422
+
int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
423
+
{
424
+
struct io_timeout_rem *tr = io_kiocb_to_cmd(req);
425
+
struct io_ring_ctx *ctx = req->ctx;
426
+
int ret;
427
+
428
+
if (!(tr->flags & IORING_TIMEOUT_UPDATE)) {
429
+
struct io_cancel_data cd = { .data = tr->addr, };
430
+
431
+
spin_lock(&ctx->completion_lock);
432
+
ret = io_timeout_cancel(ctx, &cd);
433
+
spin_unlock(&ctx->completion_lock);
434
+
} else {
435
+
enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);
436
+
437
+
spin_lock_irq(&ctx->timeout_lock);
438
+
if (tr->ltimeout)
439
+
ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode);
440
+
else
441
+
ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode);
442
+
spin_unlock_irq(&ctx->timeout_lock);
443
+
}
444
+
445
+
if (ret < 0)
446
+
req_set_fail(req);
447
+
io_req_set_res(req, ret, 0);
448
+
return IOU_OK;
449
+
}
450
+
451
+
static int __io_timeout_prep(struct io_kiocb *req,
452
+
const struct io_uring_sqe *sqe,
453
+
bool is_timeout_link)
454
+
{
455
+
struct io_timeout *timeout = io_kiocb_to_cmd(req);
456
+
struct io_timeout_data *data;
457
+
unsigned flags;
458
+
u32 off = READ_ONCE(sqe->off);
459
+
460
+
if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in)
461
+
return -EINVAL;
462
+
if (off && is_timeout_link)
463
+
return -EINVAL;
464
+
flags = READ_ONCE(sqe->timeout_flags);
465
+
if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK |
466
+
IORING_TIMEOUT_ETIME_SUCCESS))
467
+
return -EINVAL;
468
+
/* more than one clock specified is invalid, obviously */
469
+
if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
470
+
return -EINVAL;
471
+
472
+
INIT_LIST_HEAD(&timeout->list);
473
+
timeout->off = off;
474
+
if (unlikely(off && !req->ctx->off_timeout_used))
475
+
req->ctx->off_timeout_used = true;
476
+
477
+
if (WARN_ON_ONCE(req_has_async_data(req)))
478
+
return -EFAULT;
479
+
if (io_alloc_async_data(req))
480
+
return -ENOMEM;
481
+
482
+
data = req->async_data;
483
+
data->req = req;
484
+
data->flags = flags;
485
+
486
+
if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
487
+
return -EFAULT;
488
+
489
+
if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0)
490
+
return -EINVAL;
491
+
492
+
INIT_LIST_HEAD(&timeout->list);
493
+
data->mode = io_translate_timeout_mode(flags);
494
+
hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode);
495
+
496
+
if (is_timeout_link) {
497
+
struct io_submit_link *link = &req->ctx->submit_state.link;
498
+
499
+
if (!link->head)
500
+
return -EINVAL;
501
+
if (link->last->opcode == IORING_OP_LINK_TIMEOUT)
502
+
return -EINVAL;
503
+
timeout->head = link->last;
504
+
link->last->flags |= REQ_F_ARM_LTIMEOUT;
505
+
}
506
+
return 0;
507
+
}
508
+
509
+
int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
510
+
{
511
+
return __io_timeout_prep(req, sqe, false);
512
+
}
513
+
514
+
int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
515
+
{
516
+
return __io_timeout_prep(req, sqe, true);
517
+
}
518
+
519
+
int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
520
+
{
521
+
struct io_timeout *timeout = io_kiocb_to_cmd(req);
522
+
struct io_ring_ctx *ctx = req->ctx;
523
+
struct io_timeout_data *data = req->async_data;
524
+
struct list_head *entry;
525
+
u32 tail, off = timeout->off;
526
+
527
+
spin_lock_irq(&ctx->timeout_lock);
528
+
529
+
/*
530
+
* sqe->off holds how many events that need to occur for this
531
+
* timeout event to be satisfied. If it isn't set, then this is
532
+
* a pure timeout request, sequence isn't used.
533
+
*/
534
+
if (io_is_timeout_noseq(req)) {
535
+
entry = ctx->timeout_list.prev;
536
+
goto add;
537
+
}
538
+
539
+
tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
540
+
timeout->target_seq = tail + off;
541
+
542
+
/* Update the last seq here in case io_flush_timeouts() hasn't.
543
+
* This is safe because ->completion_lock is held, and submissions
544
+
* and completions are never mixed in the same ->completion_lock section.
545
+
*/
546
+
ctx->cq_last_tm_flush = tail;
547
+
548
+
/*
549
+
* Insertion sort, ensuring the first entry in the list is always
550
+
* the one we need first.
551
+
*/
552
+
list_for_each_prev(entry, &ctx->timeout_list) {
553
+
struct io_timeout *nextt = list_entry(entry, struct io_timeout, list);
554
+
struct io_kiocb *nxt = cmd_to_io_kiocb(nextt);
555
+
556
+
if (io_is_timeout_noseq(nxt))
557
+
continue;
558
+
/* nxt.seq is behind @tail, otherwise would've been completed */
559
+
if (off >= nextt->target_seq - tail)
560
+
break;
561
+
}
562
+
add:
563
+
list_add(&timeout->list, entry);
564
+
data->timer.function = io_timeout_fn;
565
+
hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
566
+
spin_unlock_irq(&ctx->timeout_lock);
567
+
return IOU_ISSUE_SKIP_COMPLETE;
568
+
}
569
+
570
+
void io_queue_linked_timeout(struct io_kiocb *req)
571
+
{
572
+
struct io_timeout *timeout = io_kiocb_to_cmd(req);
573
+
struct io_ring_ctx *ctx = req->ctx;
574
+
575
+
spin_lock_irq(&ctx->timeout_lock);
576
+
/*
577
+
* If the back reference is NULL, then our linked request finished
578
+
* before we got a chance to setup the timer
579
+
*/
580
+
if (timeout->head) {
581
+
struct io_timeout_data *data = req->async_data;
582
+
583
+
data->timer.function = io_link_timeout_fn;
584
+
hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
585
+
data->mode);
586
+
list_add_tail(&timeout->list, &ctx->ltimeout_list);
587
+
}
588
+
spin_unlock_irq(&ctx->timeout_lock);
589
+
/* drop submission reference */
590
+
io_put_req(req);
591
+
}
592
+
593
+
static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
594
+
bool cancel_all)
595
+
__must_hold(&req->ctx->timeout_lock)
596
+
{
597
+
struct io_kiocb *req;
598
+
599
+
if (task && head->task != task)
600
+
return false;
601
+
if (cancel_all)
602
+
return true;
603
+
604
+
io_for_each_link(req, head) {
605
+
if (req->flags & REQ_F_INFLIGHT)
606
+
return true;
607
+
}
608
+
return false;
609
+
}
610
+
611
+
/* Returns true if we found and killed one or more timeouts */
612
+
__cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
613
+
bool cancel_all)
614
+
{
615
+
struct io_timeout *timeout, *tmp;
616
+
int canceled = 0;
617
+
618
+
spin_lock(&ctx->completion_lock);
619
+
spin_lock_irq(&ctx->timeout_lock);
620
+
list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
621
+
struct io_kiocb *req = cmd_to_io_kiocb(timeout);
622
+
623
+
if (io_match_task(req, tsk, cancel_all)) {
624
+
io_kill_timeout(req, -ECANCELED);
625
+
canceled++;
626
+
}
627
+
}
628
+
spin_unlock_irq(&ctx->timeout_lock);
629
+
io_commit_cqring(ctx);
630
+
spin_unlock(&ctx->completion_lock);
631
+
if (canceled != 0)
632
+
io_cqring_ev_posted(ctx);
633
+
return canceled != 0;
634
+
}
+35
io_uring/timeout.h
+35
io_uring/timeout.h
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
3
+
struct io_timeout_data {
4
+
struct io_kiocb *req;
5
+
struct hrtimer timer;
6
+
struct timespec64 ts;
7
+
enum hrtimer_mode mode;
8
+
u32 flags;
9
+
};
10
+
11
+
struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
12
+
struct io_kiocb *link);
13
+
14
+
static inline struct io_kiocb *io_disarm_linked_timeout(struct io_kiocb *req)
15
+
{
16
+
struct io_kiocb *link = req->link;
17
+
18
+
if (link && link->opcode == IORING_OP_LINK_TIMEOUT)
19
+
return __io_disarm_linked_timeout(req, link);
20
+
21
+
return NULL;
22
+
}
23
+
24
+
__cold void io_flush_timeouts(struct io_ring_ctx *ctx);
25
+
int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd);
26
+
__cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
27
+
bool cancel_all);
28
+
void io_queue_linked_timeout(struct io_kiocb *req);
29
+
bool io_disarm_next(struct io_kiocb *req);
30
+
31
+
int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
32
+
int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
33
+
int io_timeout(struct io_kiocb *req, unsigned int issue_flags);
34
+
int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
35
+
int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags);