Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-5.13-2021-05-28' of git://git.kernel.dk/linux-block
Pull io_uring fixes from Jens Axboe:
"A few minor fixes:
- Fix an issue with hashed wait removal on exit (Zqiang, Pavel)
- Fix a recent data race introduced in this series (Marco)"
* tag 'io_uring-5.13-2021-05-28' of git://git.kernel.dk/linux-block:
io_uring: fix data race to avoid potential NULL-deref
io-wq: Fix UAF when wakeup wqe in hash waitqueue
io_uring/io-wq: close io-wq full-stop gap
+29
-17
+15
-14
fs/io-wq.c
+15
-14
fs/io-wq.c
···
979
979
return cwd->wqe->wq == data;
980
980
}
981
981
982
+
void io_wq_exit_start(struct io_wq *wq)
983
+
{
984
+
set_bit(IO_WQ_BIT_EXIT, &wq->state);
985
+
}
986
+
982
987
static void io_wq_exit_workers(struct io_wq *wq)
983
988
{
984
989
struct callback_head *cb;
985
990
int node;
986
-
987
-
set_bit(IO_WQ_BIT_EXIT, &wq->state);
988
991
989
992
if (!wq->task)
990
993
return;
···
1006
1003
struct io_wqe *wqe = wq->wqes[node];
1007
1004
1008
1005
io_wq_for_each_worker(wqe, io_wq_worker_wake, NULL);
1009
-
spin_lock_irq(&wq->hash->wait.lock);
1010
-
list_del_init(&wq->wqes[node]->wait.entry);
1011
-
spin_unlock_irq(&wq->hash->wait.lock);
1012
1006
}
1013
1007
rcu_read_unlock();
1014
1008
io_worker_ref_put(wq);
1015
1009
wait_for_completion(&wq->worker_done);
1010
+
1011
+
for_each_node(node) {
1012
+
spin_lock_irq(&wq->hash->wait.lock);
1013
+
list_del_init(&wq->wqes[node]->wait.entry);
1014
+
spin_unlock_irq(&wq->hash->wait.lock);
1015
+
}
1016
1016
put_task_struct(wq->task);
1017
1017
wq->task = NULL;
1018
1018
}
···
1025
1019
int node;
1026
1020
1027
1021
cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node);
1028
-
1029
-
io_wq_exit_workers(wq);
1030
1022
1031
1023
for_each_node(node) {
1032
1024
struct io_wqe *wqe = wq->wqes[node];
···
1040
1036
kfree(wq);
1041
1037
}
1042
1038
1043
-
void io_wq_put(struct io_wq *wq)
1044
-
{
1045
-
if (refcount_dec_and_test(&wq->refs))
1046
-
io_wq_destroy(wq);
1047
-
}
1048
-
1049
1039
void io_wq_put_and_exit(struct io_wq *wq)
1050
1040
{
1041
+
WARN_ON_ONCE(!test_bit(IO_WQ_BIT_EXIT, &wq->state));
1042
+
1051
1043
io_wq_exit_workers(wq);
1052
-
io_wq_put(wq);
1044
+
if (refcount_dec_and_test(&wq->refs))
1045
+
io_wq_destroy(wq);
1053
1046
}
1054
1047
1055
1048
static bool io_wq_worker_affinity(struct io_worker *worker, void *data)
+1
-1
fs/io-wq.h
+1
-1
fs/io-wq.h
···
122
122
};
123
123
124
124
struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data);
125
-
void io_wq_put(struct io_wq *wq);
125
+
void io_wq_exit_start(struct io_wq *wq);
126
126
void io_wq_put_and_exit(struct io_wq *wq);
127
127
128
128
void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work);
+13
-2
fs/io_uring.c
+13
-2
fs/io_uring.c
···
9039
9039
struct io_tctx_node *node;
9040
9040
unsigned long index;
9041
9041
9042
-
tctx->io_wq = NULL;
9043
9042
xa_for_each(&tctx->xa, index, node)
9044
9043
io_uring_del_task_file(index);
9045
-
if (wq)
9044
+
if (wq) {
9045
+
/*
9046
+
* Must be after io_uring_del_task_file() (removes nodes under
9047
+
* uring_lock) to avoid race with io_uring_try_cancel_iowq().
9048
+
*/
9049
+
tctx->io_wq = NULL;
9046
9050
io_wq_put_and_exit(wq);
9051
+
}
9047
9052
}
9048
9053
9049
9054
static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
···
9083
9078
9084
9079
if (!current->io_uring)
9085
9080
return;
9081
+
if (tctx->io_wq)
9082
+
io_wq_exit_start(tctx->io_wq);
9083
+
9086
9084
WARN_ON_ONCE(!sqd || sqd->thread != current);
9087
9085
9088
9086
atomic_inc(&tctx->in_idle);
···
9119
9111
struct io_uring_task *tctx = current->io_uring;
9120
9112
DEFINE_WAIT(wait);
9121
9113
s64 inflight;
9114
+
9115
+
if (tctx->io_wq)
9116
+
io_wq_exit_start(tctx->io_wq);
9122
9117
9123
9118
/* make sure overflow events are dropped */
9124
9119
atomic_inc(&tctx->in_idle);