Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/kbuf: uninline __io_put_kbufs
__io_put_kbufs() and other helper functions are too large to be inlined,
compilers would normally refuse to do so. Uninline it and move together
with io_kbuf_commit into kbuf.c.
io_kbuf_commitSigned-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/3dade7f55ad590e811aff83b1ec55c9c04e17b2b.1738724373.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Pavel Begunkov
and committed by
Jens Axboe
5d3e5124
54e00d9a
+70
-64
+60
io_uring/kbuf.c
+60
io_uring/kbuf.c
···
20
20
/* BIDs are addressed by a 16-bit field in a CQE */
21
21
#define MAX_BIDS_PER_BGID (1 << 16)
22
22
23
+
/* Mapped buffer ring, return io_uring_buf from head */
24
+
#define io_ring_head_to_buf(br, head, mask) &(br)->bufs[(head) & (mask)]
25
+
23
26
struct io_provide_buf {
24
27
struct file *file;
25
28
__u64 addr;
···
31
28
__u32 nbufs;
32
29
__u16 bid;
33
30
};
31
+
32
+
bool io_kbuf_commit(struct io_kiocb *req,
33
+
struct io_buffer_list *bl, int len, int nr)
34
+
{
35
+
if (unlikely(!(req->flags & REQ_F_BUFFERS_COMMIT)))
36
+
return true;
37
+
38
+
req->flags &= ~REQ_F_BUFFERS_COMMIT;
39
+
40
+
if (unlikely(len < 0))
41
+
return true;
42
+
43
+
if (bl->flags & IOBL_INC) {
44
+
struct io_uring_buf *buf;
45
+
46
+
buf = io_ring_head_to_buf(bl->buf_ring, bl->head, bl->mask);
47
+
if (WARN_ON_ONCE(len > buf->len))
48
+
len = buf->len;
49
+
buf->len -= len;
50
+
if (buf->len) {
51
+
buf->addr += len;
52
+
return false;
53
+
}
54
+
}
55
+
56
+
bl->head += nr;
57
+
return true;
58
+
}
34
59
35
60
static inline struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx,
36
61
unsigned int bgid)
···
352
321
353
322
/* don't support multiple buffer selections for legacy */
354
323
return io_provided_buffers_select(req, &arg->max_len, bl, arg->iovs);
324
+
}
325
+
326
+
static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)
327
+
{
328
+
struct io_buffer_list *bl = req->buf_list;
329
+
bool ret = true;
330
+
331
+
if (bl) {
332
+
ret = io_kbuf_commit(req, bl, len, nr);
333
+
req->buf_index = bl->bgid;
334
+
}
335
+
req->flags &= ~REQ_F_BUFFER_RING;
336
+
return ret;
337
+
}
338
+
339
+
unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs)
340
+
{
341
+
unsigned int ret;
342
+
343
+
ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
344
+
345
+
if (unlikely(!(req->flags & REQ_F_BUFFER_RING))) {
346
+
io_kbuf_drop_legacy(req);
347
+
return ret;
348
+
}
349
+
350
+
if (!__io_put_kbuf_ring(req, len, nbufs))
351
+
ret |= IORING_CQE_F_BUF_MORE;
352
+
return ret;
355
353
}
356
354
357
355
static int __io_remove_buffers(struct io_ring_ctx *ctx,
+10
-64
io_uring/kbuf.h
+10
-64
io_uring/kbuf.h
···
77
77
bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
78
78
void io_kbuf_drop_legacy(struct io_kiocb *req);
79
79
80
+
unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs);
81
+
bool io_kbuf_commit(struct io_kiocb *req,
82
+
struct io_buffer_list *bl, int len, int nr);
83
+
80
84
struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx,
81
85
unsigned int bgid);
82
86
···
119
115
return false;
120
116
}
121
117
122
-
/* Mapped buffer ring, return io_uring_buf from head */
123
-
#define io_ring_head_to_buf(br, head, mask) &(br)->bufs[(head) & (mask)]
124
-
125
-
static inline bool io_kbuf_commit(struct io_kiocb *req,
126
-
struct io_buffer_list *bl, int len, int nr)
127
-
{
128
-
if (unlikely(!(req->flags & REQ_F_BUFFERS_COMMIT)))
129
-
return true;
130
-
131
-
req->flags &= ~REQ_F_BUFFERS_COMMIT;
132
-
133
-
if (unlikely(len < 0))
134
-
return true;
135
-
136
-
if (bl->flags & IOBL_INC) {
137
-
struct io_uring_buf *buf;
138
-
139
-
buf = io_ring_head_to_buf(bl->buf_ring, bl->head, bl->mask);
140
-
if (WARN_ON_ONCE(len > buf->len))
141
-
len = buf->len;
142
-
buf->len -= len;
143
-
if (buf->len) {
144
-
buf->addr += len;
145
-
return false;
146
-
}
147
-
}
148
-
149
-
bl->head += nr;
150
-
return true;
151
-
}
152
-
153
-
static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)
154
-
{
155
-
struct io_buffer_list *bl = req->buf_list;
156
-
bool ret = true;
157
-
158
-
if (bl) {
159
-
ret = io_kbuf_commit(req, bl, len, nr);
160
-
req->buf_index = bl->bgid;
161
-
}
162
-
req->flags &= ~REQ_F_BUFFER_RING;
163
-
return ret;
164
-
}
165
-
166
-
static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,
167
-
int nbufs, unsigned issue_flags)
168
-
{
169
-
unsigned int ret;
170
-
171
-
if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))
172
-
return 0;
173
-
174
-
ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
175
-
if (req->flags & REQ_F_BUFFER_RING) {
176
-
if (!__io_put_kbuf_ring(req, len, nbufs))
177
-
ret |= IORING_CQE_F_BUF_MORE;
178
-
} else {
179
-
io_kbuf_drop_legacy(req);
180
-
}
181
-
return ret;
182
-
}
183
-
184
118
static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len,
185
119
unsigned issue_flags)
186
120
{
187
-
return __io_put_kbufs(req, len, 1, issue_flags);
121
+
if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))
122
+
return 0;
123
+
return __io_put_kbufs(req, len, 1);
188
124
}
189
125
190
126
static inline unsigned int io_put_kbufs(struct io_kiocb *req, int len,
191
127
int nbufs, unsigned issue_flags)
192
128
{
193
-
return __io_put_kbufs(req, len, nbufs, issue_flags);
129
+
if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))
130
+
return 0;
131
+
return __io_put_kbufs(req, len, nbufs);
194
132
}
195
133
#endif