Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-6.1-2022-10-28' of git://git.kernel.dk/linux
Pull io_uring fix from Jens Axboe:
"Just a fix for a locking regression introduced with the deferred
task_work running from this merge window"
* tag 'io_uring-6.1-2022-10-28' of git://git.kernel.dk/linux:
io_uring: unlock if __io_run_local_work locked inside
io_uring: use io_run_local_work_locked helper
+16
-8
+5
-6
io_uring/io_uring.c
+5
-6
io_uring/io_uring.c
···
1173
1173
}
1174
1174
}
1175
1175
1176
-
int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
1176
+
int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked)
1177
1177
{
1178
1178
struct llist_node *node;
1179
1179
struct llist_node fake;
···
1192
1192
struct io_kiocb *req = container_of(node, struct io_kiocb,
1193
1193
io_task_work.node);
1194
1194
prefetch(container_of(next, struct io_kiocb, io_task_work.node));
1195
-
req->io_task_work.func(req, &locked);
1195
+
req->io_task_work.func(req, locked);
1196
1196
ret++;
1197
1197
node = next;
1198
1198
}
···
1208
1208
goto again;
1209
1209
}
1210
1210
1211
-
if (locked)
1211
+
if (*locked)
1212
1212
io_submit_flush_completions(ctx);
1213
1213
trace_io_uring_local_work_run(ctx, ret, loops);
1214
1214
return ret;
···
1225
1225
1226
1226
__set_current_state(TASK_RUNNING);
1227
1227
locked = mutex_trylock(&ctx->uring_lock);
1228
-
ret = __io_run_local_work(ctx, locked);
1228
+
ret = __io_run_local_work(ctx, &locked);
1229
1229
if (locked)
1230
1230
mutex_unlock(&ctx->uring_lock);
1231
1231
···
1446
1446
io_task_work_pending(ctx)) {
1447
1447
u32 tail = ctx->cached_cq_tail;
1448
1448
1449
-
if (!llist_empty(&ctx->work_llist))
1450
-
__io_run_local_work(ctx, true);
1449
+
(void) io_run_local_work_locked(ctx);
1451
1450
1452
1451
if (task_work_pending(current) ||
1453
1452
wq_list_empty(&ctx->iopoll_list)) {
+11
-2
io_uring/io_uring.h
+11
-2
io_uring/io_uring.h
···
27
27
struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow);
28
28
bool io_req_cqe_overflow(struct io_kiocb *req);
29
29
int io_run_task_work_sig(struct io_ring_ctx *ctx);
30
-
int __io_run_local_work(struct io_ring_ctx *ctx, bool locked);
30
+
int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked);
31
31
int io_run_local_work(struct io_ring_ctx *ctx);
32
32
void io_req_complete_failed(struct io_kiocb *req, s32 res);
33
33
void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
···
277
277
278
278
static inline int io_run_local_work_locked(struct io_ring_ctx *ctx)
279
279
{
280
+
bool locked;
281
+
int ret;
282
+
280
283
if (llist_empty(&ctx->work_llist))
281
284
return 0;
282
-
return __io_run_local_work(ctx, true);
285
+
286
+
locked = true;
287
+
ret = __io_run_local_work(ctx, &locked);
288
+
/* shouldn't happen! */
289
+
if (WARN_ON_ONCE(!locked))
290
+
mutex_lock(&ctx->uring_lock);
291
+
return ret;
283
292
}
284
293
285
294
static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)