Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-5.10-2020-10-30' of git://git.kernel.dk/linux-block
Pull io_uring fixes from Jens Axboe:
- Fixes for linked timeouts (Pavel)
- Set IO_WQ_WORK_CONCURRENT early for async offload (Pavel)
- Two minor simplifications that make the code easier to read and
follow (Pavel)
* tag 'io_uring-5.10-2020-10-30' of git://git.kernel.dk/linux-block:
io_uring: use type appropriate io_kiocb handler for double poll
io_uring: simplify __io_queue_sqe()
io_uring: simplify nxt propagation in io_queue_sqe
io_uring: don't miss setting IO_WQ_WORK_CONCURRENT
io_uring: don't defer put of cancelled ltimeout
io_uring: always clear LINK_TIMEOUT after cancel
io_uring: don't adjust LINK_HEAD in cancel ltimeout
io_uring: remove opcode check on ltimeout kill
+38
-70
+38
-70
fs/io_uring.c
+38
-70
fs/io_uring.c
···
1365
1365
io_req_init_async(req);
1366
1366
id = req->work.identity;
1367
1367
1368
+
if (req->flags & REQ_F_FORCE_ASYNC)
1369
+
req->work.flags |= IO_WQ_WORK_CONCURRENT;
1370
+
1368
1371
if (req->flags & REQ_F_ISREG) {
1369
1372
if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))
1370
1373
io_wq_hash_work(&req->work, file_inode(req->file));
···
1849
1846
percpu_ref_put(&ctx->refs);
1850
1847
}
1851
1848
1852
-
static bool io_link_cancel_timeout(struct io_kiocb *req)
1849
+
static void io_kill_linked_timeout(struct io_kiocb *req)
1853
1850
{
1854
-
struct io_timeout_data *io = req->async_data;
1855
1851
struct io_ring_ctx *ctx = req->ctx;
1856
-
int ret;
1857
-
1858
-
ret = hrtimer_try_to_cancel(&io->timer);
1859
-
if (ret != -1) {
1860
-
io_cqring_fill_event(req, -ECANCELED);
1861
-
io_commit_cqring(ctx);
1862
-
req->flags &= ~REQ_F_LINK_HEAD;
1863
-
io_put_req_deferred(req, 1);
1864
-
return true;
1865
-
}
1866
-
1867
-
return false;
1868
-
}
1869
-
1870
-
static bool __io_kill_linked_timeout(struct io_kiocb *req)
1871
-
{
1872
1852
struct io_kiocb *link;
1873
-
bool wake_ev;
1853
+
bool cancelled = false;
1854
+
unsigned long flags;
1874
1855
1875
-
if (list_empty(&req->link_list))
1876
-
return false;
1877
-
link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
1878
-
if (link->opcode != IORING_OP_LINK_TIMEOUT)
1879
-
return false;
1856
+
spin_lock_irqsave(&ctx->completion_lock, flags);
1857
+
link = list_first_entry_or_null(&req->link_list, struct io_kiocb,
1858
+
link_list);
1880
1859
/*
1881
1860
* Can happen if a linked timeout fired and link had been like
1882
1861
* req -> link t-out -> link t-out [-> ...]
1883
1862
*/
1884
-
if (!(link->flags & REQ_F_LTIMEOUT_ACTIVE))
1885
-
return false;
1863
+
if (link && (link->flags & REQ_F_LTIMEOUT_ACTIVE)) {
1864
+
struct io_timeout_data *io = link->async_data;
1865
+
int ret;
1886
1866
1887
-
list_del_init(&link->link_list);
1888
-
wake_ev = io_link_cancel_timeout(link);
1867
+
list_del_init(&link->link_list);
1868
+
ret = hrtimer_try_to_cancel(&io->timer);
1869
+
if (ret != -1) {
1870
+
io_cqring_fill_event(link, -ECANCELED);
1871
+
io_commit_cqring(ctx);
1872
+
cancelled = true;
1873
+
}
1874
+
}
1889
1875
req->flags &= ~REQ_F_LINK_TIMEOUT;
1890
-
return wake_ev;
1891
-
}
1892
-
1893
-
static void io_kill_linked_timeout(struct io_kiocb *req)
1894
-
{
1895
-
struct io_ring_ctx *ctx = req->ctx;
1896
-
unsigned long flags;
1897
-
bool wake_ev;
1898
-
1899
-
spin_lock_irqsave(&ctx->completion_lock, flags);
1900
-
wake_ev = __io_kill_linked_timeout(req);
1901
1876
spin_unlock_irqrestore(&ctx->completion_lock, flags);
1902
1877
1903
-
if (wake_ev)
1878
+
if (cancelled) {
1904
1879
io_cqring_ev_posted(ctx);
1880
+
io_put_req(link);
1881
+
}
1905
1882
}
1906
1883
1907
1884
static struct io_kiocb *io_req_link_next(struct io_kiocb *req)
···
4960
4977
/* make sure double remove sees this as being gone */
4961
4978
wait->private = NULL;
4962
4979
spin_unlock(&poll->head->lock);
4963
-
if (!done)
4964
-
__io_async_wake(req, poll, mask, io_poll_task_func);
4980
+
if (!done) {
4981
+
/* use wait func handler, so it matches the rq type */
4982
+
poll->wait.func(&poll->wait, mode, sync, key);
4983
+
}
4965
4984
}
4966
4985
refcount_dec(&req->refs);
4967
4986
return 1;
···
6165
6180
static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs)
6166
6181
{
6167
6182
struct io_kiocb *linked_timeout;
6168
-
struct io_kiocb *nxt;
6169
6183
const struct cred *old_creds = NULL;
6170
6184
int ret;
6171
6185
···
6190
6206
*/
6191
6207
if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
6192
6208
if (!io_arm_poll_handler(req)) {
6193
-
punt:
6194
6209
/*
6195
6210
* Queued up for async execution, worker will release
6196
6211
* submit reference when the iocb is actually submitted.
···
6199
6216
6200
6217
if (linked_timeout)
6201
6218
io_queue_linked_timeout(linked_timeout);
6202
-
goto exit;
6203
-
}
6219
+
} else if (likely(!ret)) {
6220
+
/* drop submission reference */
6221
+
req = io_put_req_find_next(req);
6222
+
if (linked_timeout)
6223
+
io_queue_linked_timeout(linked_timeout);
6204
6224
6205
-
if (unlikely(ret)) {
6225
+
if (req) {
6226
+
if (!(req->flags & REQ_F_FORCE_ASYNC))
6227
+
goto again;
6228
+
io_queue_async_work(req);
6229
+
}
6230
+
} else {
6206
6231
/* un-prep timeout, so it'll be killed as any other linked */
6207
6232
req->flags &= ~REQ_F_LINK_TIMEOUT;
6208
6233
req_set_fail_links(req);
6209
6234
io_put_req(req);
6210
6235
io_req_complete(req, ret);
6211
-
goto exit;
6212
6236
}
6213
6237
6214
-
/* drop submission reference */
6215
-
nxt = io_put_req_find_next(req);
6216
-
if (linked_timeout)
6217
-
io_queue_linked_timeout(linked_timeout);
6218
-
6219
-
if (nxt) {
6220
-
req = nxt;
6221
-
6222
-
if (req->flags & REQ_F_FORCE_ASYNC) {
6223
-
linked_timeout = NULL;
6224
-
goto punt;
6225
-
}
6226
-
goto again;
6227
-
}
6228
-
exit:
6229
6238
if (old_creds)
6230
6239
revert_creds(old_creds);
6231
6240
}
···
6241
6266
if (unlikely(ret))
6242
6267
goto fail_req;
6243
6268
}
6244
-
6245
-
/*
6246
-
* Never try inline submit of IOSQE_ASYNC is set, go straight
6247
-
* to async execution.
6248
-
*/
6249
-
io_req_init_async(req);
6250
-
req->work.flags |= IO_WQ_WORK_CONCURRENT;
6251
6269
io_queue_async_work(req);
6252
6270
} else {
6253
6271
if (sqe) {