Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring: introduce type alias for io_tw_state
In preparation for changing how io_tw_state is passed, introduce a type
alias io_tw_token_t for struct io_tw_state *. This allows for changing
the representation in one place, without having to update the many
functions that just forward their struct io_tw_state * argument.
Also add a comment to struct io_tw_state to explain its purpose.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
Link: https://lore.kernel.org/r/20250217022511.1150145-1-csander@purestorage.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Caleb Sander Mateos
and committed by
Jens Axboe
bcf8a029
496f56bf
+66
-56
+8
-1
include/linux/io_uring_types.h
+8
-1
include/linux/io_uring_types.h
···
436
436
struct io_mapped_region param_region;
437
437
};
438
438
439
+
/*
440
+
* Token indicating function is called in task work context:
441
+
* ctx->uring_lock is held and any completions generated will be flushed.
442
+
* ONLY core io_uring.c should instantiate this struct.
443
+
*/
439
444
struct io_tw_state {
440
445
};
446
+
/* Alias to use in code that doesn't instantiate struct io_tw_state */
447
+
typedef struct io_tw_state *io_tw_token_t;
441
448
442
449
enum {
443
450
REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
···
570
563
REQ_F_HAS_METADATA = IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),
571
564
};
572
565
573
-
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);
566
+
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);
574
567
575
568
struct io_task_work {
576
569
struct llist_node node;
+8
-8
io_uring/futex.c
+8
-8
io_uring/futex.c
···
44
44
io_alloc_cache_free(&ctx->futex_cache, kfree);
45
45
}
46
46
47
-
static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)
47
+
static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
48
48
{
49
49
req->async_data = NULL;
50
50
hlist_del_init(&req->hash_node);
51
-
io_req_task_complete(req, ts);
51
+
io_req_task_complete(req, tw);
52
52
}
53
53
54
-
static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)
54
+
static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
55
55
{
56
56
struct io_futex_data *ifd = req->async_data;
57
57
struct io_ring_ctx *ctx = req->ctx;
58
58
59
-
io_tw_lock(ctx, ts);
59
+
io_tw_lock(ctx, tw);
60
60
if (!io_alloc_cache_put(&ctx->futex_cache, ifd))
61
61
kfree(ifd);
62
-
__io_futex_complete(req, ts);
62
+
__io_futex_complete(req, tw);
63
63
}
64
64
65
-
static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)
65
+
static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw)
66
66
{
67
67
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
68
68
struct futex_vector *futexv = req->async_data;
69
69
70
-
io_tw_lock(req->ctx, ts);
70
+
io_tw_lock(req->ctx, tw);
71
71
72
72
if (!iof->futexv_unqueued) {
73
73
int res;
···
79
79
80
80
kfree(req->async_data);
81
81
req->flags &= ~REQ_F_ASYNC_DATA;
82
-
__io_futex_complete(req, ts);
82
+
__io_futex_complete(req, tw);
83
83
}
84
84
85
85
static bool io_futexv_claim(struct io_futex *iof)
+14
-14
io_uring/io_uring.c
+14
-14
io_uring/io_uring.c
···
543
543
io_queue_linked_timeout(link);
544
544
}
545
545
546
-
static void io_req_queue_iowq_tw(struct io_kiocb *req, struct io_tw_state *ts)
546
+
static void io_req_queue_iowq_tw(struct io_kiocb *req, io_tw_token_t tw)
547
547
{
548
548
io_queue_iowq(req);
549
549
}
···
1022
1022
return nxt;
1023
1023
}
1024
1024
1025
-
static void ctx_flush_and_put(struct io_ring_ctx *ctx, struct io_tw_state *ts)
1025
+
static void ctx_flush_and_put(struct io_ring_ctx *ctx, io_tw_token_t tw)
1026
1026
{
1027
1027
if (!ctx)
1028
1028
return;
···
1277
1277
}
1278
1278
1279
1279
static int __io_run_local_work_loop(struct llist_node **node,
1280
-
struct io_tw_state *ts,
1280
+
io_tw_token_t tw,
1281
1281
int events)
1282
1282
{
1283
1283
int ret = 0;
···
1288
1288
io_task_work.node);
1289
1289
INDIRECT_CALL_2(req->io_task_work.func,
1290
1290
io_poll_task_func, io_req_rw_complete,
1291
-
req, ts);
1291
+
req, tw);
1292
1292
*node = next;
1293
1293
if (++ret >= events)
1294
1294
break;
···
1297
1297
return ret;
1298
1298
}
1299
1299
1300
-
static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,
1300
+
static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw,
1301
1301
int min_events, int max_events)
1302
1302
{
1303
1303
struct llist_node *node;
···
1310
1310
atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1311
1311
again:
1312
1312
min_events -= ret;
1313
-
ret = __io_run_local_work_loop(&ctx->retry_llist.first, ts, max_events);
1313
+
ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events);
1314
1314
if (ctx->retry_llist.first)
1315
1315
goto retry_done;
1316
1316
···
1319
1319
* running the pending items.
1320
1320
*/
1321
1321
node = llist_reverse_order(llist_del_all(&ctx->work_llist));
1322
-
ret += __io_run_local_work_loop(&node, ts, max_events - ret);
1322
+
ret += __io_run_local_work_loop(&node, tw, max_events - ret);
1323
1323
ctx->retry_llist.first = node;
1324
1324
loops++;
1325
1325
···
1357
1357
return ret;
1358
1358
}
1359
1359
1360
-
static void io_req_task_cancel(struct io_kiocb *req, struct io_tw_state *ts)
1360
+
static void io_req_task_cancel(struct io_kiocb *req, io_tw_token_t tw)
1361
1361
{
1362
-
io_tw_lock(req->ctx, ts);
1362
+
io_tw_lock(req->ctx, tw);
1363
1363
io_req_defer_failed(req, req->cqe.res);
1364
1364
}
1365
1365
1366
-
void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts)
1366
+
void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw)
1367
1367
{
1368
-
io_tw_lock(req->ctx, ts);
1368
+
io_tw_lock(req->ctx, tw);
1369
1369
if (unlikely(io_should_terminate_tw()))
1370
1370
io_req_defer_failed(req, -EFAULT);
1371
1371
else if (req->flags & REQ_F_FORCE_ASYNC)
···
1583
1583
return 0;
1584
1584
}
1585
1585
1586
-
void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts)
1586
+
void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw)
1587
1587
{
1588
1588
io_req_complete_defer(req);
1589
1589
}
···
1763
1763
return ret;
1764
1764
}
1765
1765
1766
-
int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts)
1766
+
int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)
1767
1767
{
1768
-
io_tw_lock(req->ctx, ts);
1768
+
io_tw_lock(req->ctx, tw);
1769
1769
return io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_MULTISHOT|
1770
1770
IO_URING_F_COMPLETE_DEFER);
1771
1771
}
+4
-4
io_uring/io_uring.h
+4
-4
io_uring/io_uring.h
···
90
90
unsigned flags);
91
91
bool io_alloc_async_data(struct io_kiocb *req);
92
92
void io_req_task_queue(struct io_kiocb *req);
93
-
void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts);
93
+
void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw);
94
94
void io_req_task_queue_fail(struct io_kiocb *req, int ret);
95
-
void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts);
95
+
void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw);
96
96
struct llist_node *io_handle_tw_list(struct llist_node *node, unsigned int *count, unsigned int max_entries);
97
97
struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count);
98
98
void tctx_task_work(struct callback_head *cb);
···
104
104
int start, int end);
105
105
void io_req_queue_iowq(struct io_kiocb *req);
106
106
107
-
int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts);
107
+
int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw);
108
108
int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
109
109
int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);
110
110
void __io_submit_flush_completions(struct io_ring_ctx *ctx);
···
376
376
return task_work_pending(current) || io_local_work_pending(ctx);
377
377
}
378
378
379
-
static inline void io_tw_lock(struct io_ring_ctx *ctx, struct io_tw_state *ts)
379
+
static inline void io_tw_lock(struct io_ring_ctx *ctx, io_tw_token_t tw)
380
380
{
381
381
lockdep_assert_held(&ctx->uring_lock);
382
382
}
+1
-1
io_uring/msg_ring.c
+1
-1
io_uring/msg_ring.c
+2
-2
io_uring/notif.c
+2
-2
io_uring/notif.c
···
11
11
12
12
static const struct ubuf_info_ops io_ubuf_ops;
13
13
14
-
static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
14
+
static void io_notif_tw_complete(struct io_kiocb *notif, io_tw_token_t tw)
15
15
{
16
16
struct io_notif_data *nd = io_notif_to_data(notif);
17
17
···
29
29
}
30
30
31
31
nd = nd->next;
32
-
io_req_task_complete(notif, ts);
32
+
io_req_task_complete(notif, tw);
33
33
} while (nd);
34
34
}
35
35
+9
-9
io_uring/poll.c
+9
-9
io_uring/poll.c
···
220
220
* req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot
221
221
* poll and that the result is stored in req->cqe.
222
222
*/
223
-
static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
223
+
static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
224
224
{
225
225
int v;
226
226
···
288
288
return IOU_POLL_REMOVE_POLL_USE_RES;
289
289
}
290
290
} else {
291
-
int ret = io_poll_issue(req, ts);
291
+
int ret = io_poll_issue(req, tw);
292
292
if (ret == IOU_STOP_MULTISHOT)
293
293
return IOU_POLL_REMOVE_POLL_USE_RES;
294
294
else if (ret == IOU_REQUEUE)
···
311
311
return IOU_POLL_NO_ACTION;
312
312
}
313
313
314
-
void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
314
+
void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw)
315
315
{
316
316
int ret;
317
317
318
-
ret = io_poll_check_events(req, ts);
318
+
ret = io_poll_check_events(req, tw);
319
319
if (ret == IOU_POLL_NO_ACTION) {
320
320
io_kbuf_recycle(req, 0);
321
321
return;
···
335
335
poll = io_kiocb_to_cmd(req, struct io_poll);
336
336
req->cqe.res = mangle_poll(req->cqe.res & poll->events);
337
337
} else if (ret == IOU_POLL_REISSUE) {
338
-
io_req_task_submit(req, ts);
338
+
io_req_task_submit(req, tw);
339
339
return;
340
340
} else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) {
341
341
req->cqe.res = ret;
···
343
343
}
344
344
345
345
io_req_set_res(req, req->cqe.res, 0);
346
-
io_req_task_complete(req, ts);
346
+
io_req_task_complete(req, tw);
347
347
} else {
348
-
io_tw_lock(req->ctx, ts);
348
+
io_tw_lock(req->ctx, tw);
349
349
350
350
if (ret == IOU_POLL_REMOVE_POLL_USE_RES)
351
-
io_req_task_complete(req, ts);
351
+
io_req_task_complete(req, tw);
352
352
else if (ret == IOU_POLL_DONE || ret == IOU_POLL_REISSUE)
353
-
io_req_task_submit(req, ts);
353
+
io_req_task_submit(req, tw);
354
354
else
355
355
io_req_defer_failed(req, ret);
356
356
}
+3
-1
io_uring/poll.h
+3
-1
io_uring/poll.h
···
1
1
// SPDX-License-Identifier: GPL-2.0
2
2
3
+
#include <linux/io_uring_types.h>
4
+
3
5
#define IO_POLL_ALLOC_CACHE_MAX 32
4
6
5
7
enum {
···
45
43
bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
46
44
bool cancel_all);
47
45
48
-
void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts);
46
+
void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw);
+2
-2
io_uring/rw.c
+2
-2
io_uring/rw.c
···
511
511
return res;
512
512
}
513
513
514
-
void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts)
514
+
void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw)
515
515
{
516
516
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
517
517
struct kiocb *kiocb = &rw->kiocb;
···
528
528
req->cqe.flags |= io_put_kbuf(req, req->cqe.res, 0);
529
529
530
530
io_req_rw_cleanup(req, 0);
531
-
io_req_task_complete(req, ts);
531
+
io_req_task_complete(req, tw);
532
532
}
533
533
534
534
static void io_complete_rw(struct kiocb *kiocb, long res)
+2
-1
io_uring/rw.h
+2
-1
io_uring/rw.h
···
1
1
// SPDX-License-Identifier: GPL-2.0
2
2
3
+
#include <linux/io_uring_types.h>
3
4
#include <linux/pagemap.h>
4
5
5
6
struct io_meta_state {
···
40
39
int io_write(struct io_kiocb *req, unsigned int issue_flags);
41
40
void io_readv_writev_cleanup(struct io_kiocb *req);
42
41
void io_rw_fail(struct io_kiocb *req);
43
-
void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts);
42
+
void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw);
44
43
int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
45
44
int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags);
46
45
void io_rw_cache_free(const void *entry);
+8
-8
io_uring/timeout.c
+8
-8
io_uring/timeout.c
···
65
65
66
66
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer);
67
67
68
-
static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)
68
+
static void io_timeout_complete(struct io_kiocb *req, io_tw_token_t tw)
69
69
{
70
70
struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
71
71
struct io_timeout_data *data = req->async_data;
···
82
82
}
83
83
}
84
84
85
-
io_req_task_complete(req, ts);
85
+
io_req_task_complete(req, tw);
86
86
}
87
87
88
88
static __cold bool io_flush_killed_timeouts(struct list_head *list, int err)
···
154
154
io_flush_killed_timeouts(&list, 0);
155
155
}
156
156
157
-
static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts)
157
+
static void io_req_tw_fail_links(struct io_kiocb *link, io_tw_token_t tw)
158
158
{
159
-
io_tw_lock(link->ctx, ts);
159
+
io_tw_lock(link->ctx, tw);
160
160
while (link) {
161
161
struct io_kiocb *nxt = link->link;
162
162
long res = -ECANCELED;
···
165
165
res = link->cqe.res;
166
166
link->link = NULL;
167
167
io_req_set_res(link, res, 0);
168
-
io_req_task_complete(link, ts);
168
+
io_req_task_complete(link, tw);
169
169
link = nxt;
170
170
}
171
171
}
···
312
312
return 0;
313
313
}
314
314
315
-
static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts)
315
+
static void io_req_task_link_timeout(struct io_kiocb *req, io_tw_token_t tw)
316
316
{
317
317
struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
318
318
struct io_kiocb *prev = timeout->prev;
···
330
330
ret = -ECANCELED;
331
331
}
332
332
io_req_set_res(req, ret ?: -ETIME, 0);
333
-
io_req_task_complete(req, ts);
333
+
io_req_task_complete(req, tw);
334
334
io_put_req(prev);
335
335
} else {
336
336
io_req_set_res(req, -ETIME, 0);
337
-
io_req_task_complete(req, ts);
337
+
io_req_task_complete(req, tw);
338
338
}
339
339
}
340
340
+1
-1
io_uring/uring_cmd.c
+1
-1
io_uring/uring_cmd.c
···
102
102
}
103
103
EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
104
104
105
-
static void io_uring_cmd_work(struct io_kiocb *req, struct io_tw_state *ts)
105
+
static void io_uring_cmd_work(struct io_kiocb *req, io_tw_token_t tw)
106
106
{
107
107
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
108
108
unsigned int flags = IO_URING_F_COMPLETE_DEFER;
+4
-4
io_uring/waitid.c
+4
-4
io_uring/waitid.c
···
16
16
#include "waitid.h"
17
17
#include "../kernel/exit.h"
18
18
19
-
static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts);
19
+
static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw);
20
20
21
21
#define IO_WAITID_CANCEL_FLAG BIT(31)
22
22
#define IO_WAITID_REF_MASK GENMASK(30, 0)
···
185
185
return true;
186
186
}
187
187
188
-
static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts)
188
+
static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw)
189
189
{
190
190
struct io_waitid_async *iwa = req->async_data;
191
191
struct io_ring_ctx *ctx = req->ctx;
192
192
int ret;
193
193
194
-
io_tw_lock(ctx, ts);
194
+
io_tw_lock(ctx, tw);
195
195
196
196
ret = __do_wait(&iwa->wo);
197
197
···
221
221
}
222
222
223
223
io_waitid_complete(req, ret);
224
-
io_req_task_complete(req, ts);
224
+
io_req_task_complete(req, tw);
225
225
}
226
226
227
227
static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,