Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-2023-01-06' of git://git.kernel.dk/linux
Pull io_uring fixes from Jens Axboe:
"A few minor fixes that should go into the 6.2 release:
- Fix for a memory leak in io-wq worker creation, if we ultimately
end up canceling the worker creation before it gets created (me)
- lockdep annotations for the CQ locking (Pavel)
- A regression fix for CQ timeout handling (Pavel)
- Ring pinning around deferred task_work fix (Pavel)
- A trivial member move in struct io_ring_ctx, saving us some memory
(me)"
* tag 'io_uring-2023-01-06' of git://git.kernel.dk/linux:
io_uring: fix CQ waiting timeout handling
io_uring: move 'poll_multi_queue' bool in io_ring_ctx
io_uring: lockdep annotate CQ locking
io_uring: pin context while queueing deferred tw
io_uring/io-wq: free worker if task_work creation is canceled
+30
-8
+2
-1
include/linux/io_uring_types.h
+2
-1
include/linux/io_uring_types.h
···
292
292
struct {
293
293
spinlock_t completion_lock;
294
294
295
+
bool poll_multi_queue;
296
+
295
297
/*
296
298
* ->iopoll_list is protected by the ctx->uring_lock for
297
299
* io_uring instances that don't use IORING_SETUP_SQPOLL.
···
302
300
*/
303
301
struct io_wq_work_list iopoll_list;
304
302
struct io_hash_table cancel_table;
305
-
bool poll_multi_queue;
306
303
307
304
struct llist_head work_llist;
308
305
+1
io_uring/io-wq.c
+1
io_uring/io-wq.c
+12
-7
io_uring/io_uring.c
+12
-7
io_uring/io_uring.c
···
731
731
size_t ocq_size = sizeof(struct io_overflow_cqe);
732
732
bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
733
733
734
+
lockdep_assert_held(&ctx->completion_lock);
735
+
734
736
if (is_cqe32)
735
737
ocq_size += sizeof(struct io_uring_cqe);
736
738
···
821
819
u32 cflags)
822
820
{
823
821
struct io_uring_cqe *cqe;
824
-
825
-
if (!ctx->task_complete)
826
-
lockdep_assert_held(&ctx->completion_lock);
827
822
828
823
ctx->cq_extra++;
829
824
···
1235
1236
{
1236
1237
struct io_ring_ctx *ctx = req->ctx;
1237
1238
1238
-
if (!llist_add(&req->io_task_work.node, &ctx->work_llist))
1239
+
percpu_ref_get(&ctx->refs);
1240
+
1241
+
if (!llist_add(&req->io_task_work.node, &ctx->work_llist)) {
1242
+
percpu_ref_put(&ctx->refs);
1239
1243
return;
1244
+
}
1240
1245
/* need it for the following io_cqring_wake() */
1241
1246
smp_mb__after_atomic();
1242
1247
1243
1248
if (unlikely(atomic_read(&req->task->io_uring->in_idle))) {
1244
1249
io_move_task_work_from_local(ctx);
1250
+
percpu_ref_put(&ctx->refs);
1245
1251
return;
1246
1252
}
1247
1253
···
1256
1252
if (ctx->has_evfd)
1257
1253
io_eventfd_signal(ctx);
1258
1254
__io_cqring_wake(ctx);
1255
+
percpu_ref_put(&ctx->refs);
1259
1256
}
1260
1257
1261
1258
void __io_req_task_work_add(struct io_kiocb *req, bool allow_local)
···
2470
2465
/* when returns >0, the caller should retry */
2471
2466
static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
2472
2467
struct io_wait_queue *iowq,
2473
-
ktime_t timeout)
2468
+
ktime_t *timeout)
2474
2469
{
2475
2470
int ret;
2476
2471
unsigned long check_cq;
···
2488
2483
if (check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT))
2489
2484
return -EBADR;
2490
2485
}
2491
-
if (!schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS))
2486
+
if (!schedule_hrtimeout(timeout, HRTIMER_MODE_ABS))
2492
2487
return -ETIME;
2493
2488
2494
2489
/*
···
2564
2559
}
2565
2560
prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
2566
2561
TASK_INTERRUPTIBLE);
2567
-
ret = io_cqring_wait_schedule(ctx, &iowq, timeout);
2562
+
ret = io_cqring_wait_schedule(ctx, &iowq, &timeout);
2568
2563
if (__io_cqring_events_user(ctx) >= min_events)
2569
2564
break;
2570
2565
cond_resched();
+15
io_uring/io_uring.h
+15
io_uring/io_uring.h
···
79
79
bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,
80
80
bool cancel_all);
81
81
82
+
#define io_lockdep_assert_cq_locked(ctx) \
83
+
do { \
84
+
if (ctx->flags & IORING_SETUP_IOPOLL) { \
85
+
lockdep_assert_held(&ctx->uring_lock); \
86
+
} else if (!ctx->task_complete) { \
87
+
lockdep_assert_held(&ctx->completion_lock); \
88
+
} else if (ctx->submitter_task->flags & PF_EXITING) { \
89
+
lockdep_assert(current_work()); \
90
+
} else { \
91
+
lockdep_assert(current == ctx->submitter_task); \
92
+
} \
93
+
} while (0)
94
+
82
95
static inline void io_req_task_work_add(struct io_kiocb *req)
83
96
{
84
97
__io_req_task_work_add(req, true);
···
105
92
static inline struct io_uring_cqe *io_get_cqe_overflow(struct io_ring_ctx *ctx,
106
93
bool overflow)
107
94
{
95
+
io_lockdep_assert_cq_locked(ctx);
96
+
108
97
if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {
109
98
struct io_uring_cqe *cqe = ctx->cqe_cached;
110
99