Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-5.7-2020-05-22' of git://git.kernel.dk/linux-block
Pull io_uring fixes from Jens Axboe:
"A small collection of small fixes that should go into this release:
- Two fixes for async request preparation (Pavel)
- Busy clear fix for SQPOLL (Xiaoguang)
- Don't use kiocb->private for O_DIRECT buf index, some file systems
use it (Bijan)
- Kill dead check in io_splice()
- Ensure sqo_wait is initialized early
- Cancel task_work if we fail adding to original process
- Only add (IO)pollable requests to iopoll list, fixing a regression
in this merge window"
* tag 'io_uring-5.7-2020-05-22' of git://git.kernel.dk/linux-block:
io_uring: reset -EBUSY error when io sq thread is waken up
io_uring: don't add non-IO requests to iopoll pending list
io_uring: don't use kiocb.private to store buf_index
io_uring: cancel work if task_work_add() fails
io_uring: remove dead check in io_splice()
io_uring: fix FORCE_ASYNC req preparation
io_uring: don't prepare DRAIN reqs twice
io_uring: initialize ctx->sqo_wait earlier
+34
-26
+34
-26
fs/io_uring.c
+34
-26
fs/io_uring.c
···
619
619
bool needs_fixed_file;
620
620
u8 opcode;
621
621
622
+
u16 buf_index;
623
+
622
624
struct io_ring_ctx *ctx;
623
625
struct list_head list;
624
626
unsigned int flags;
···
926
924
goto err;
927
925
928
926
ctx->flags = p->flags;
927
+
init_waitqueue_head(&ctx->sqo_wait);
929
928
init_waitqueue_head(&ctx->cq_wait);
930
929
INIT_LIST_HEAD(&ctx->cq_overflow_list);
931
930
init_completion(&ctx->completions[0]);
···
2103
2100
2104
2101
req->rw.addr = READ_ONCE(sqe->addr);
2105
2102
req->rw.len = READ_ONCE(sqe->len);
2106
-
/* we own ->private, reuse it for the buffer index / buffer ID */
2107
-
req->rw.kiocb.private = (void *) (unsigned long)
2108
-
READ_ONCE(sqe->buf_index);
2103
+
req->buf_index = READ_ONCE(sqe->buf_index);
2109
2104
return 0;
2110
2105
}
2111
2106
···
2146
2145
struct io_ring_ctx *ctx = req->ctx;
2147
2146
size_t len = req->rw.len;
2148
2147
struct io_mapped_ubuf *imu;
2149
-
unsigned index, buf_index;
2148
+
u16 index, buf_index;
2150
2149
size_t offset;
2151
2150
u64 buf_addr;
2152
2151
···
2154
2153
if (unlikely(!ctx->user_bufs))
2155
2154
return -EFAULT;
2156
2155
2157
-
buf_index = (unsigned long) req->rw.kiocb.private;
2156
+
buf_index = req->buf_index;
2158
2157
if (unlikely(buf_index >= ctx->nr_user_bufs))
2159
2158
return -EFAULT;
2160
2159
···
2270
2269
bool needs_lock)
2271
2270
{
2272
2271
struct io_buffer *kbuf;
2273
-
int bgid;
2272
+
u16 bgid;
2274
2273
2275
2274
kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2276
-
bgid = (int) (unsigned long) req->rw.kiocb.private;
2275
+
bgid = req->buf_index;
2277
2276
kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
2278
2277
if (IS_ERR(kbuf))
2279
2278
return kbuf;
···
2364
2363
}
2365
2364
2366
2365
/* buffer index only valid with fixed read/write, or buffer select */
2367
-
if (req->rw.kiocb.private && !(req->flags & REQ_F_BUFFER_SELECT))
2366
+
if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
2368
2367
return -EINVAL;
2369
2368
2370
2369
if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
···
2772
2771
poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
2773
2772
poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
2774
2773
2775
-
if (sp->len) {
2774
+
if (sp->len)
2776
2775
ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
2777
-
if (force_nonblock && ret == -EAGAIN)
2778
-
return -EAGAIN;
2779
-
}
2780
2776
2781
2777
io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
2782
2778
req->flags &= ~REQ_F_NEED_CLEANUP;
···
4135
4137
req->result = mask;
4136
4138
init_task_work(&req->task_work, func);
4137
4139
/*
4138
-
* If this fails, then the task is exiting. Punt to one of the io-wq
4139
-
* threads to ensure the work gets run, we can't always rely on exit
4140
-
* cancelation taking care of this.
4140
+
* If this fails, then the task is exiting. When a task exits, the
4141
+
* work gets canceled, so just cancel this request as well instead
4142
+
* of executing it. We can't safely execute it anyway, as we may not
4143
+
* have the needed state needed for it anyway.
4141
4144
*/
4142
4145
ret = task_work_add(tsk, &req->task_work, true);
4143
4146
if (unlikely(ret)) {
4147
+
WRITE_ONCE(poll->canceled, true);
4144
4148
tsk = io_wq_get_task(req->ctx->io_wq);
4145
4149
task_work_add(tsk, &req->task_work, true);
4146
4150
}
···
5013
5013
if (!req_need_defer(req) && list_empty_careful(&ctx->defer_list))
5014
5014
return 0;
5015
5015
5016
-
if (!req->io && io_alloc_async_ctx(req))
5017
-
return -EAGAIN;
5018
-
5019
-
ret = io_req_defer_prep(req, sqe);
5020
-
if (ret < 0)
5021
-
return ret;
5016
+
if (!req->io) {
5017
+
if (io_alloc_async_ctx(req))
5018
+
return -EAGAIN;
5019
+
ret = io_req_defer_prep(req, sqe);
5020
+
if (ret < 0)
5021
+
return ret;
5022
+
}
5022
5023
5023
5024
spin_lock_irq(&ctx->completion_lock);
5024
5025
if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
···
5306
5305
if (ret)
5307
5306
return ret;
5308
5307
5309
-
if (ctx->flags & IORING_SETUP_IOPOLL) {
5308
+
/* If the op doesn't have a file, we're not polling for it */
5309
+
if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) {
5310
5310
const bool in_async = io_wq_current_is_worker();
5311
5311
5312
5312
if (req->result == -EAGAIN)
···
5608
5606
io_double_put_req(req);
5609
5607
}
5610
5608
} else if (req->flags & REQ_F_FORCE_ASYNC) {
5611
-
ret = io_req_defer_prep(req, sqe);
5612
-
if (unlikely(ret < 0))
5613
-
goto fail_req;
5609
+
if (!req->io) {
5610
+
ret = -EAGAIN;
5611
+
if (io_alloc_async_ctx(req))
5612
+
goto fail_req;
5613
+
ret = io_req_defer_prep(req, sqe);
5614
+
if (unlikely(ret < 0))
5615
+
goto fail_req;
5616
+
}
5617
+
5614
5618
/*
5615
5619
* Never try inline submit of IOSQE_ASYNC is set, go straight
5616
5620
* to async execution.
···
6032
6024
finish_wait(&ctx->sqo_wait, &wait);
6033
6025
6034
6026
ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
6027
+
ret = 0;
6035
6028
continue;
6036
6029
}
6037
6030
finish_wait(&ctx->sqo_wait, &wait);
···
6846
6837
{
6847
6838
int ret;
6848
6839
6849
-
init_waitqueue_head(&ctx->sqo_wait);
6850
6840
mmgrab(current->mm);
6851
6841
ctx->sqo_mm = current->mm;
6852
6842