Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-5.6-2020-03-07' of git://git.kernel.dk/linux-block
Pull io_uring fixes from Jens Axboe:
"Here are a few io_uring fixes that should go into this release. This
contains:
- Removal of (now) unused io_wq_flush() and associated flag (Pavel)
- Fix cancelation lockup with linked timeouts (Pavel)
- Fix for potential use-after-free when freeing percpu ref for fixed
file sets
- io-wq cancelation fixups (Pavel)"
* tag 'io_uring-5.6-2020-03-07' of git://git.kernel.dk/linux-block:
io_uring: fix lockup with timeouts
io_uring: free fixed_file_data after RCU grace period
io-wq: remove io_wq_flush and IO_WQ_WORK_INTERNAL
io-wq: fix IO_WQ_WORK_NO_CANCEL cancellation
+38
-47
+15
-43
fs/io-wq.c
+15
-43
fs/io-wq.c
···
502
502
if (worker->mm)
503
503
work->flags |= IO_WQ_WORK_HAS_MM;
504
504
505
-
if (wq->get_work && !(work->flags & IO_WQ_WORK_INTERNAL)) {
505
+
if (wq->get_work) {
506
506
put_work = work;
507
507
wq->get_work(work);
508
508
}
···
747
747
return true;
748
748
}
749
749
750
+
static void io_run_cancel(struct io_wq_work *work)
751
+
{
752
+
do {
753
+
struct io_wq_work *old_work = work;
754
+
755
+
work->flags |= IO_WQ_WORK_CANCEL;
756
+
work->func(&work);
757
+
work = (work == old_work) ? NULL : work;
758
+
} while (work);
759
+
}
760
+
750
761
static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)
751
762
{
752
763
struct io_wqe_acct *acct = io_work_get_acct(wqe, work);
···
771
760
* It's close enough to not be an issue, fork() has the same delay.
772
761
*/
773
762
if (unlikely(!io_wq_can_queue(wqe, acct, work))) {
774
-
work->flags |= IO_WQ_WORK_CANCEL;
775
-
work->func(&work);
763
+
io_run_cancel(work);
776
764
return;
777
765
}
778
766
···
910
900
spin_unlock_irqrestore(&wqe->lock, flags);
911
901
912
902
if (found) {
913
-
work->flags |= IO_WQ_WORK_CANCEL;
914
-
work->func(&work);
903
+
io_run_cancel(work);
915
904
return IO_WQ_CANCEL_OK;
916
905
}
917
906
···
985
976
spin_unlock_irqrestore(&wqe->lock, flags);
986
977
987
978
if (found) {
988
-
work->flags |= IO_WQ_WORK_CANCEL;
989
-
work->func(&work);
979
+
io_run_cancel(work);
990
980
return IO_WQ_CANCEL_OK;
991
981
}
992
982
···
1055
1047
}
1056
1048
1057
1049
return ret;
1058
-
}
1059
-
1060
-
struct io_wq_flush_data {
1061
-
struct io_wq_work work;
1062
-
struct completion done;
1063
-
};
1064
-
1065
-
static void io_wq_flush_func(struct io_wq_work **workptr)
1066
-
{
1067
-
struct io_wq_work *work = *workptr;
1068
-
struct io_wq_flush_data *data;
1069
-
1070
-
data = container_of(work, struct io_wq_flush_data, work);
1071
-
complete(&data->done);
1072
-
}
1073
-
1074
-
/*
1075
-
* Doesn't wait for previously queued work to finish. When this completes,
1076
-
* it just means that previously queued work was started.
1077
-
*/
1078
-
void io_wq_flush(struct io_wq *wq)
1079
-
{
1080
-
struct io_wq_flush_data data;
1081
-
int node;
1082
-
1083
-
for_each_node(node) {
1084
-
struct io_wqe *wqe = wq->wqes[node];
1085
-
1086
-
if (!node_online(node))
1087
-
continue;
1088
-
init_completion(&data.done);
1089
-
INIT_IO_WORK(&data.work, io_wq_flush_func);
1090
-
data.work.flags |= IO_WQ_WORK_INTERNAL;
1091
-
io_wqe_enqueue(wqe, &data.work);
1092
-
wait_for_completion(&data.done);
1093
-
}
1094
1050
}
1095
1051
1096
1052
struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
-2
fs/io-wq.h
-2
fs/io-wq.h
···
8
8
IO_WQ_WORK_HAS_MM = 2,
9
9
IO_WQ_WORK_HASHED = 4,
10
10
IO_WQ_WORK_UNBOUND = 32,
11
-
IO_WQ_WORK_INTERNAL = 64,
12
11
IO_WQ_WORK_CB = 128,
13
12
IO_WQ_WORK_NO_CANCEL = 256,
14
13
IO_WQ_WORK_CONCURRENT = 512,
···
99
100
100
101
void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work);
101
102
void io_wq_enqueue_hashed(struct io_wq *wq, struct io_wq_work *work, void *val);
102
-
void io_wq_flush(struct io_wq *wq);
103
103
104
104
void io_wq_cancel_all(struct io_wq *wq);
105
105
enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork);
+23
-2
fs/io_uring.c
+23
-2
fs/io_uring.c
···
191
191
struct llist_head put_llist;
192
192
struct work_struct ref_work;
193
193
struct completion done;
194
+
struct rcu_head rcu;
194
195
};
195
196
196
197
struct io_ring_ctx {
···
1000
999
if (ret != -1) {
1001
1000
atomic_inc(&req->ctx->cq_timeouts);
1002
1001
list_del_init(&req->list);
1002
+
req->flags |= REQ_F_COMP_LOCKED;
1003
1003
io_cqring_fill_event(req, 0);
1004
1004
io_put_req(req);
1005
1005
}
···
5331
5329
complete(&data->done);
5332
5330
}
5333
5331
5332
+
static void __io_file_ref_exit_and_free(struct rcu_head *rcu)
5333
+
{
5334
+
struct fixed_file_data *data = container_of(rcu, struct fixed_file_data,
5335
+
rcu);
5336
+
percpu_ref_exit(&data->refs);
5337
+
kfree(data);
5338
+
}
5339
+
5340
+
static void io_file_ref_exit_and_free(struct rcu_head *rcu)
5341
+
{
5342
+
/*
5343
+
* We need to order our exit+free call against the potentially
5344
+
* existing call_rcu() for switching to atomic. One way to do that
5345
+
* is to have this rcu callback queue the final put and free, as we
5346
+
* could otherwise have a pre-existing atomic switch complete _after_
5347
+
* the free callback we queued.
5348
+
*/
5349
+
call_rcu(rcu, __io_file_ref_exit_and_free);
5350
+
}
5351
+
5334
5352
static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
5335
5353
{
5336
5354
struct fixed_file_data *data = ctx->file_data;
···
5363
5341
flush_work(&data->ref_work);
5364
5342
wait_for_completion(&data->done);
5365
5343
io_ring_file_ref_flush(data);
5366
-
percpu_ref_exit(&data->refs);
5367
5344
5368
5345
__io_sqe_files_unregister(ctx);
5369
5346
nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
5370
5347
for (i = 0; i < nr_tables; i++)
5371
5348
kfree(data->table[i].files);
5372
5349
kfree(data->table);
5373
-
kfree(data);
5350
+
call_rcu(&data->rcu, io_file_ref_exit_and_free);
5374
5351
ctx->file_data = NULL;
5375
5352
ctx->nr_user_files = 0;
5376
5353
return 0;