Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-5.16-2021-12-10' of git://git.kernel.dk/linux-block
Pull io_uring fixes from Jens Axboe:
"A few fixes that are all bound for stable:
- Two syzbot reports for io-wq that turned out to be separate fixes,
but ultimately very closely related
- io_uring task_work running on cancelations"
* tag 'io_uring-5.16-2021-12-10' of git://git.kernel.dk/linux-block:
io-wq: check for wq exit after adding new worker task_work
io_uring: ensure task_work gets run as part of cancelations
io-wq: remove spurious bit clear on task_work addition
+27
-8
+23
-6
fs/io-wq.c
+23
-6
fs/io-wq.c
···
142
142
struct io_wqe_acct *acct,
143
143
struct io_cb_cancel_data *match);
144
144
static void create_worker_cb(struct callback_head *cb);
145
+
static void io_wq_cancel_tw_create(struct io_wq *wq);
145
146
146
147
static bool io_worker_get(struct io_worker *worker)
147
148
{
···
358
357
test_and_set_bit_lock(0, &worker->create_state))
359
358
goto fail_release;
360
359
360
+
atomic_inc(&wq->worker_refs);
361
361
init_task_work(&worker->create_work, func);
362
362
worker->create_index = acct->index;
363
363
if (!task_work_add(wq->task, &worker->create_work, TWA_SIGNAL)) {
364
-
clear_bit_unlock(0, &worker->create_state);
364
+
/*
365
+
* EXIT may have been set after checking it above, check after
366
+
* adding the task_work and remove any creation item if it is
367
+
* now set. wq exit does that too, but we can have added this
368
+
* work item after we canceled in io_wq_exit_workers().
369
+
*/
370
+
if (test_bit(IO_WQ_BIT_EXIT, &wq->state))
371
+
io_wq_cancel_tw_create(wq);
372
+
io_worker_ref_put(wq);
365
373
return true;
366
374
}
375
+
io_worker_ref_put(wq);
367
376
clear_bit_unlock(0, &worker->create_state);
368
377
fail_release:
369
378
io_worker_release(worker);
···
1209
1198
set_bit(IO_WQ_BIT_EXIT, &wq->state);
1210
1199
}
1211
1200
1212
-
static void io_wq_exit_workers(struct io_wq *wq)
1201
+
static void io_wq_cancel_tw_create(struct io_wq *wq)
1213
1202
{
1214
1203
struct callback_head *cb;
1215
-
int node;
1216
-
1217
-
if (!wq->task)
1218
-
return;
1219
1204
1220
1205
while ((cb = task_work_cancel_match(wq->task, io_task_work_match, wq)) != NULL) {
1221
1206
struct io_worker *worker;
···
1219
1212
worker = container_of(cb, struct io_worker, create_work);
1220
1213
io_worker_cancel_cb(worker);
1221
1214
}
1215
+
}
1216
+
1217
+
static void io_wq_exit_workers(struct io_wq *wq)
1218
+
{
1219
+
int node;
1220
+
1221
+
if (!wq->task)
1222
+
return;
1223
+
1224
+
io_wq_cancel_tw_create(wq);
1222
1225
1223
1226
rcu_read_lock();
1224
1227
for_each_node(node) {
+4
-2
fs/io_uring.c
+4
-2
fs/io_uring.c
···
9824
9824
9825
9825
/*
9826
9826
* Find any io_uring ctx that this task has registered or done IO on, and cancel
9827
-
* requests. @sqd should be not-null IIF it's an SQPOLL thread cancellation.
9827
+
* requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation.
9828
9828
*/
9829
9829
static __cold void io_uring_cancel_generic(bool cancel_all,
9830
9830
struct io_sq_data *sqd)
···
9866
9866
cancel_all);
9867
9867
}
9868
9868
9869
-
prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
9869
+
prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE);
9870
+
io_run_task_work();
9870
9871
io_uring_drop_tctx_refs(current);
9872
+
9871
9873
/*
9872
9874
* If we've seen completions, retry without waiting. This
9873
9875
* avoids a race where a completion comes in before we did