Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/kbuf: cleanup passing back cflags
We have various functions calculating the CQE cflags we need to pass
back, but it's all the same everywhere. Make a number of the putting
functions void, and just have the two main helps for this, io_put_kbuf()
and io_put_kbuf_comp() calculate the actual mask and pass it back.
While at it, cleanup how we put REQ_F_BUFFER_RING buffers. Before
this change, we would call into __io_put_kbuf() only to go right back
in to the header defined functions. As clearing this type of buffer
is just re-assigning the buf_index and incrementing the head, this
is very wasteful.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+31
-24
+4
-10
io_uring/kbuf.c
+4
-10
io_uring/kbuf.c
···
102
102
return true;
103
103
}
104
104
105
-
unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags)
105
+
void __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags)
106
106
{
107
-
unsigned int cflags;
108
-
109
107
/*
110
108
* We can add this buffer back to two lists:
111
109
*
···
116
118
* We migrate buffers from the comp_list to the issue cache list
117
119
* when we need one.
118
120
*/
119
-
if (req->flags & REQ_F_BUFFER_RING) {
120
-
/* no buffers to recycle for this case */
121
-
cflags = __io_put_kbuf_list(req, NULL);
122
-
} else if (issue_flags & IO_URING_F_UNLOCKED) {
121
+
if (issue_flags & IO_URING_F_UNLOCKED) {
123
122
struct io_ring_ctx *ctx = req->ctx;
124
123
125
124
spin_lock(&ctx->completion_lock);
126
-
cflags = __io_put_kbuf_list(req, &ctx->io_buffers_comp);
125
+
__io_put_kbuf_list(req, &ctx->io_buffers_comp);
127
126
spin_unlock(&ctx->completion_lock);
128
127
} else {
129
128
lockdep_assert_held(&req->ctx->uring_lock);
130
129
131
-
cflags = __io_put_kbuf_list(req, &req->ctx->io_buffers_cache);
130
+
__io_put_kbuf_list(req, &req->ctx->io_buffers_cache);
132
131
}
133
-
return cflags;
134
132
}
135
133
136
134
static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len,
+27
-14
io_uring/kbuf.h
+27
-14
io_uring/kbuf.h
···
57
57
58
58
void io_kbuf_mmap_list_free(struct io_ring_ctx *ctx);
59
59
60
-
unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags);
60
+
void __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags);
61
61
62
62
bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
63
63
···
108
108
return false;
109
109
}
110
110
111
-
static inline unsigned int __io_put_kbuf_list(struct io_kiocb *req,
112
-
struct list_head *list)
111
+
static inline void __io_put_kbuf_ring(struct io_kiocb *req)
113
112
{
114
-
unsigned int ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
113
+
if (req->buf_list) {
114
+
req->buf_index = req->buf_list->bgid;
115
+
req->buf_list->head++;
116
+
}
117
+
req->flags &= ~REQ_F_BUFFER_RING;
118
+
}
115
119
120
+
static inline void __io_put_kbuf_list(struct io_kiocb *req,
121
+
struct list_head *list)
122
+
{
116
123
if (req->flags & REQ_F_BUFFER_RING) {
117
-
if (req->buf_list) {
118
-
req->buf_index = req->buf_list->bgid;
119
-
req->buf_list->head++;
120
-
}
121
-
req->flags &= ~REQ_F_BUFFER_RING;
124
+
__io_put_kbuf_ring(req);
122
125
} else {
123
126
req->buf_index = req->kbuf->bgid;
124
127
list_add(&req->kbuf->list, list);
125
128
req->flags &= ~REQ_F_BUFFER_SELECTED;
126
129
}
127
-
128
-
return ret;
129
130
}
130
131
131
132
static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req)
132
133
{
134
+
unsigned int ret;
135
+
133
136
lockdep_assert_held(&req->ctx->completion_lock);
134
137
135
138
if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
136
139
return 0;
137
-
return __io_put_kbuf_list(req, &req->ctx->io_buffers_comp);
140
+
141
+
ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
142
+
__io_put_kbuf_list(req, &req->ctx->io_buffers_comp);
143
+
return ret;
138
144
}
139
145
140
146
static inline unsigned int io_put_kbuf(struct io_kiocb *req,
141
147
unsigned issue_flags)
142
148
{
149
+
unsigned int ret;
143
150
144
-
if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
151
+
if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))
145
152
return 0;
146
-
return __io_put_kbuf(req, issue_flags);
153
+
154
+
ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
155
+
if (req->flags & REQ_F_BUFFER_RING)
156
+
__io_put_kbuf_ring(req);
157
+
else
158
+
__io_put_kbuf(req, issue_flags);
159
+
return ret;
147
160
}
148
161
#endif