Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/rsrc: split io_kiocb node type assignments
Currently the io_rsrc_node assignment in io_kiocb is an array of two
pointers, as two nodes may be assigned to a request - one file node,
and one buffer node. However, the buffer node can co-exist with the
provided buffers, as currently it's not supported to use both provided
and registered buffers at the same time.
This crucially brings struct io_kiocb down to 4 cache lines again, as
before it spilled into the 5th cacheline.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+29
-17
+6
-1
include/linux/io_uring_types.h
+6
-1
include/linux/io_uring_types.h
···
475
475
REQ_F_BL_EMPTY_BIT,
476
476
REQ_F_BL_NO_RECYCLE_BIT,
477
477
REQ_F_BUFFERS_COMMIT_BIT,
478
+
REQ_F_BUF_NODE_BIT,
478
479
479
480
/* not a real bit, just to check we're not overflowing the space */
480
481
__REQ_F_LAST_BIT,
···
554
553
REQ_F_BL_NO_RECYCLE = IO_REQ_FLAG(REQ_F_BL_NO_RECYCLE_BIT),
555
554
/* buffer ring head needs incrementing on put */
556
555
REQ_F_BUFFERS_COMMIT = IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT),
556
+
/* buf node is valid */
557
+
REQ_F_BUF_NODE = IO_REQ_FLAG(REQ_F_BUF_NODE_BIT),
557
558
};
558
559
559
560
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);
···
636
633
* REQ_F_BUFFER_RING is set.
637
634
*/
638
635
struct io_buffer_list *buf_list;
636
+
637
+
struct io_rsrc_node *buf_node;
639
638
};
640
639
641
640
union {
···
647
642
__poll_t apoll_events;
648
643
};
649
644
650
-
struct io_rsrc_node *rsrc_nodes[2];
645
+
struct io_rsrc_node *file_node;
651
646
652
647
atomic_t refs;
653
648
bool cancel_seq_set;
+3
-3
io_uring/io_uring.c
+3
-3
io_uring/io_uring.c
···
948
948
static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)
949
949
{
950
950
req->ctx = ctx;
951
-
req->rsrc_nodes[IORING_RSRC_FILE] = NULL;
952
-
req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
951
+
req->buf_node = NULL;
952
+
req->file_node = NULL;
953
953
req->link = NULL;
954
954
req->async_data = NULL;
955
955
/* not necessary, but safer to zero */
···
1882
1882
io_ring_submit_lock(ctx, issue_flags);
1883
1883
node = io_rsrc_node_lookup(&ctx->file_table.data, fd);
1884
1884
if (node) {
1885
-
io_req_assign_rsrc_node(req, node);
1885
+
io_req_assign_rsrc_node(&req->file_node, node);
1886
1886
req->flags |= io_slot_flags(node);
1887
1887
file = io_slot_file(node);
1888
1888
}
+2
-1
io_uring/net.c
+2
-1
io_uring/net.c
···
1348
1348
io_ring_submit_lock(ctx, issue_flags);
1349
1349
node = io_rsrc_node_lookup(&ctx->buf_table, sr->buf_index);
1350
1350
if (node) {
1351
-
io_req_assign_rsrc_node(sr->notif, node);
1351
+
io_req_assign_rsrc_node(&sr->notif->buf_node, node);
1352
+
sr->notif->flags |= REQ_F_BUF_NODE;
1352
1353
ret = 0;
1353
1354
}
1354
1355
io_ring_submit_unlock(ctx, issue_flags);
+2
-1
io_uring/nop.c
+2
-1
io_uring/nop.c
···
67
67
io_ring_submit_lock(ctx, issue_flags);
68
68
node = io_rsrc_node_lookup(&ctx->buf_table, nop->buffer);
69
69
if (node) {
70
-
io_req_assign_rsrc_node(req, node);
70
+
io_req_assign_rsrc_node(&req->buf_node, node);
71
+
req->flags |= REQ_F_BUF_NODE;
71
72
ret = 0;
72
73
}
73
74
io_ring_submit_unlock(ctx, issue_flags);
+2
-2
io_uring/notif.c
+2
-2
io_uring/notif.c
···
117
117
notif->file = NULL;
118
118
notif->task = current;
119
119
io_get_task_refs(1);
120
-
notif->rsrc_nodes[IORING_RSRC_FILE] = NULL;
121
-
notif->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
120
+
notif->file_node = NULL;
121
+
notif->buf_node = NULL;
122
122
123
123
nd = io_notif_to_data(notif);
124
124
nd->zc_report = false;
+10
-6
io_uring/rsrc.h
+10
-6
io_uring/rsrc.h
···
95
95
96
96
static inline void io_req_put_rsrc_nodes(struct io_kiocb *req)
97
97
{
98
-
io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_FILE]);
99
-
io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_BUFFER]);
100
-
req->rsrc_nodes[IORING_RSRC_FILE] = NULL;
101
-
req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
98
+
if (req->file_node) {
99
+
io_put_rsrc_node(req->file_node);
100
+
req->file_node = NULL;
101
+
}
102
+
if (req->flags & REQ_F_BUF_NODE) {
103
+
io_put_rsrc_node(req->buf_node);
104
+
req->buf_node = NULL;
105
+
}
102
106
}
103
107
104
108
static inline struct io_ring_ctx *io_rsrc_node_ctx(struct io_rsrc_node *node)
···
115
111
return node->ctx_ptr & IORING_RSRC_TYPE_MASK;
116
112
}
117
113
118
-
static inline void io_req_assign_rsrc_node(struct io_kiocb *req,
114
+
static inline void io_req_assign_rsrc_node(struct io_rsrc_node **dst_node,
119
115
struct io_rsrc_node *node)
120
116
{
121
117
node->refs++;
122
-
req->rsrc_nodes[io_rsrc_node_type(node)] = node;
118
+
*dst_node = node;
123
119
}
124
120
125
121
int io_files_update(struct io_kiocb *req, unsigned int issue_flags);
+2
-1
io_uring/rw.c
+2
-1
io_uring/rw.c
···
341
341
node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index);
342
342
if (!node)
343
343
return -EFAULT;
344
-
io_req_assign_rsrc_node(req, node);
344
+
io_req_assign_rsrc_node(&req->buf_node, node);
345
+
req->flags |= REQ_F_BUF_NODE;
345
346
346
347
io = req->async_data;
347
348
ret = io_import_fixed(ddir, &io->iter, node->buf, rw->addr, rw->len);
+2
-2
io_uring/uring_cmd.c
+2
-2
io_uring/uring_cmd.c
···
219
219
* being called. This prevents destruction of the mapped buffer
220
220
* we'll need at actual import time.
221
221
*/
222
-
io_req_assign_rsrc_node(req, node);
222
+
io_req_assign_rsrc_node(&req->buf_node, node);
223
223
}
224
224
ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
225
225
···
275
275
struct iov_iter *iter, void *ioucmd)
276
276
{
277
277
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
278
-
struct io_rsrc_node *node = req->rsrc_nodes[IORING_RSRC_BUFFER];
278
+
struct io_rsrc_node *node = req->buf_node;
279
279
280
280
/* Must have had rsrc_node assigned at prep time */
281
281
if (node)