Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-5.14-2021-07-30' of git://git.kernel.dk/linux-block
Pull io_uring fixes from Jens Axboe:
- A fix for block backed reissue (me)
- Reissue context hardening (me)
- Async link locking fix (Pavel)
* tag 'io_uring-5.14-2021-07-30' of git://git.kernel.dk/linux-block:
io_uring: fix poll requests leaking second poll entries
io_uring: don't block level reissue off completion path
io_uring: always reissue from task_work context
io_uring: fix race in unified task_work running
io_uring: fix io_prep_async_link locking
+32
-8
+32
-8
fs/io_uring.c
+32
-8
fs/io_uring.c
···
1279
1279
{
1280
1280
struct io_kiocb *cur;
1281
1281
1282
-
io_for_each_link(cur, req)
1283
-
io_prep_async_work(cur);
1282
+
if (req->flags & REQ_F_LINK_TIMEOUT) {
1283
+
struct io_ring_ctx *ctx = req->ctx;
1284
+
1285
+
spin_lock_irq(&ctx->completion_lock);
1286
+
io_for_each_link(cur, req)
1287
+
io_prep_async_work(cur);
1288
+
spin_unlock_irq(&ctx->completion_lock);
1289
+
} else {
1290
+
io_for_each_link(cur, req)
1291
+
io_prep_async_work(cur);
1292
+
}
1284
1293
}
1285
1294
1286
1295
static void io_queue_async_work(struct io_kiocb *req)
···
1959
1950
node = next;
1960
1951
}
1961
1952
if (wq_list_empty(&tctx->task_list)) {
1953
+
spin_lock_irq(&tctx->task_lock);
1962
1954
clear_bit(0, &tctx->task_state);
1963
-
if (wq_list_empty(&tctx->task_list))
1955
+
if (wq_list_empty(&tctx->task_list)) {
1956
+
spin_unlock_irq(&tctx->task_lock);
1964
1957
break;
1958
+
}
1959
+
spin_unlock_irq(&tctx->task_lock);
1965
1960
/* another tctx_task_work() is enqueued, yield */
1966
1961
if (test_and_set_bit(0, &tctx->task_state))
1967
1962
break;
···
2057
2044
static void io_req_task_queue(struct io_kiocb *req)
2058
2045
{
2059
2046
req->io_task_work.func = io_req_task_submit;
2047
+
io_req_task_work_add(req);
2048
+
}
2049
+
2050
+
static void io_req_task_queue_reissue(struct io_kiocb *req)
2051
+
{
2052
+
req->io_task_work.func = io_queue_async_work;
2060
2053
io_req_task_work_add(req);
2061
2054
}
2062
2055
···
2254
2235
!(req->flags & REQ_F_DONT_REISSUE)) {
2255
2236
req->iopoll_completed = 0;
2256
2237
req_ref_get(req);
2257
-
io_queue_async_work(req);
2238
+
io_req_task_queue_reissue(req);
2258
2239
continue;
2259
2240
}
2260
2241
···
2446
2427
* -EAGAIN.
2447
2428
*/
2448
2429
if (percpu_ref_is_dying(&ctx->refs))
2430
+
return false;
2431
+
/*
2432
+
* Play it safe and assume not safe to re-import and reissue if we're
2433
+
* not in the original thread group (or in task context).
2434
+
*/
2435
+
if (!same_thread_group(req->task, current) || !in_task())
2449
2436
return false;
2450
2437
return true;
2451
2438
}
···
2783
2758
req->flags &= ~REQ_F_REISSUE;
2784
2759
if (io_resubmit_prep(req)) {
2785
2760
req_ref_get(req);
2786
-
io_queue_async_work(req);
2761
+
io_req_task_queue_reissue(req);
2787
2762
} else {
2788
2763
int cflags = 0;
2789
2764
···
4939
4914
if (req->poll.events & EPOLLONESHOT)
4940
4915
flags = 0;
4941
4916
if (!io_cqring_fill_event(ctx, req->user_data, error, flags)) {
4942
-
io_poll_remove_waitqs(req);
4943
4917
req->poll.done = true;
4944
4918
flags = 0;
4945
4919
}
···
4961
4937
4962
4938
done = io_poll_complete(req, req->result);
4963
4939
if (done) {
4940
+
io_poll_remove_double(req);
4964
4941
hash_del(&req->hash_node);
4965
4942
} else {
4966
4943
req->result = 0;
···
5149
5124
ipt->error = -EINVAL;
5150
5125
5151
5126
spin_lock_irq(&ctx->completion_lock);
5152
-
if (ipt->error)
5127
+
if (ipt->error || (mask && (poll->events & EPOLLONESHOT)))
5153
5128
io_poll_remove_double(req);
5154
5129
if (likely(poll->head)) {
5155
5130
spin_lock(&poll->head->lock);
···
5221
5196
ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
5222
5197
io_async_wake);
5223
5198
if (ret || ipt.error) {
5224
-
io_poll_remove_double(req);
5225
5199
spin_unlock_irq(&ctx->completion_lock);
5226
5200
if (ret)
5227
5201
return IO_APOLL_READY;