Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-6.1-2022-11-25' of git://git.kernel.dk/linux
Pull io_uring fixes from Jens Axboe:
- A few poll related fixes. One fixing a race condition between poll
cancelation and trigger, and one making the overflow handling a bit
more robust (Lin, Pavel)
- Fix an fput() for error handling in the direct file table (Lin)
- Fix for a regression introduced in this cycle, where we don't always
get TIF_NOTIFY_SIGNAL cleared appropriately (me)
* tag 'io_uring-6.1-2022-11-25' of git://git.kernel.dk/linux:
io_uring: clear TIF_NOTIFY_SIGNAL if set and task_work not available
io_uring/poll: fix poll_refs race with cancelation
io_uring/filetable: fix file reference underflow
io_uring: make poll refs more robust
io_uring: cmpxchg for poll arm refs release
+47
-11
-2
io_uring/filetable.c
-2
io_uring/filetable.c
+7
-2
io_uring/io_uring.h
+7
-2
io_uring/io_uring.h
···
238
238
239
239
static inline int io_run_task_work(void)
240
240
{
241
+
/*
242
+
* Always check-and-clear the task_work notification signal. With how
243
+
* signaling works for task_work, we can find it set with nothing to
244
+
* run. We need to clear it for that case, like get_signal() does.
245
+
*/
246
+
if (test_thread_flag(TIF_NOTIFY_SIGNAL))
247
+
clear_notify_signal();
241
248
if (task_work_pending(current)) {
242
-
if (test_thread_flag(TIF_NOTIFY_SIGNAL))
243
-
clear_notify_signal();
244
249
__set_current_state(TASK_RUNNING);
245
250
task_work_run();
246
251
return 1;
+40
-7
io_uring/poll.c
+40
-7
io_uring/poll.c
···
40
40
};
41
41
42
42
#define IO_POLL_CANCEL_FLAG BIT(31)
43
-
#define IO_POLL_REF_MASK GENMASK(30, 0)
43
+
#define IO_POLL_RETRY_FLAG BIT(30)
44
+
#define IO_POLL_REF_MASK GENMASK(29, 0)
45
+
46
+
/*
47
+
* We usually have 1-2 refs taken, 128 is more than enough and we want to
48
+
* maximise the margin between this amount and the moment when it overflows.
49
+
*/
50
+
#define IO_POLL_REF_BIAS 128
44
51
45
52
#define IO_WQE_F_DOUBLE 1
46
53
···
65
58
return priv & IO_WQE_F_DOUBLE;
66
59
}
67
60
61
+
static bool io_poll_get_ownership_slowpath(struct io_kiocb *req)
62
+
{
63
+
int v;
64
+
65
+
/*
66
+
* poll_refs are already elevated and we don't have much hope for
67
+
* grabbing the ownership. Instead of incrementing set a retry flag
68
+
* to notify the loop that there might have been some change.
69
+
*/
70
+
v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs);
71
+
if (v & IO_POLL_REF_MASK)
72
+
return false;
73
+
return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);
74
+
}
75
+
68
76
/*
69
77
* If refs part of ->poll_refs (see IO_POLL_REF_MASK) is 0, it's free. We can
70
78
* bump it and acquire ownership. It's disallowed to modify requests while not
···
88
66
*/
89
67
static inline bool io_poll_get_ownership(struct io_kiocb *req)
90
68
{
69
+
if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS))
70
+
return io_poll_get_ownership_slowpath(req);
91
71
return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);
92
72
}
93
73
···
259
235
*/
260
236
if ((v & IO_POLL_REF_MASK) != 1)
261
237
req->cqe.res = 0;
238
+
if (v & IO_POLL_RETRY_FLAG) {
239
+
req->cqe.res = 0;
240
+
/*
241
+
* We won't find new events that came in between
242
+
* vfs_poll and the ref put unless we clear the flag
243
+
* in advance.
244
+
*/
245
+
atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs);
246
+
v &= ~IO_POLL_RETRY_FLAG;
247
+
}
262
248
263
249
/* the mask was stashed in __io_poll_execute */
264
250
if (!req->cqe.res) {
···
308
274
* Release all references, retry if someone tried to restart
309
275
* task_work while we were executing it.
310
276
*/
311
-
} while (atomic_sub_return(v & IO_POLL_REF_MASK, &req->poll_refs));
277
+
} while (atomic_sub_return(v & IO_POLL_REF_MASK, &req->poll_refs) &
278
+
IO_POLL_REF_MASK);
312
279
313
280
return IOU_POLL_NO_ACTION;
314
281
}
···
553
518
unsigned issue_flags)
554
519
{
555
520
struct io_ring_ctx *ctx = req->ctx;
556
-
int v;
557
521
558
522
INIT_HLIST_NODE(&req->hash_node);
559
523
req->work.cancel_seq = atomic_read(&ctx->cancel_seq);
···
620
586
621
587
if (ipt->owning) {
622
588
/*
623
-
* Release ownership. If someone tried to queue a tw while it was
624
-
* locked, kick it off for them.
589
+
* Try to release ownership. If we see a change of state, e.g.
590
+
* poll was waken up, queue up a tw, it'll deal with it.
625
591
*/
626
-
v = atomic_dec_return(&req->poll_refs);
627
-
if (unlikely(v & IO_POLL_REF_MASK))
592
+
if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1)
628
593
__io_poll_execute(req, 0);
629
594
}
630
595
return 0;