Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
selftests: ublk: enable zero copy for stripe target
Use io_uring vectored fixed kernel buffer for handling stripe IO.
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/20250325135155.935398-5-ming.lei@redhat.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Ming Lei
and committed by
Jens Axboe
57ed58c1
1045afae
+53
-17
+1
tools/testing/selftests/ublk/Makefile
+1
tools/testing/selftests/ublk/Makefile
+52
-17
tools/testing/selftests/ublk/stripe.c
+52
-17
tools/testing/selftests/ublk/stripe.c
···
111
111
}
112
112
}
113
113
114
-
static inline enum io_uring_op stripe_to_uring_op(const struct ublksrv_io_desc *iod)
114
+
static inline enum io_uring_op stripe_to_uring_op(
115
+
const struct ublksrv_io_desc *iod, int zc)
115
116
{
116
117
unsigned ublk_op = ublksrv_get_op(iod);
117
118
118
119
if (ublk_op == UBLK_IO_OP_READ)
119
-
return IORING_OP_READV;
120
+
return zc ? IORING_OP_READV_FIXED : IORING_OP_READV;
120
121
else if (ublk_op == UBLK_IO_OP_WRITE)
121
-
return IORING_OP_WRITEV;
122
+
return zc ? IORING_OP_WRITEV_FIXED : IORING_OP_WRITEV;
122
123
assert(0);
123
124
}
124
125
125
126
static int stripe_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag)
126
127
{
127
128
const struct stripe_conf *conf = get_chunk_shift(q);
128
-
enum io_uring_op op = stripe_to_uring_op(iod);
129
+
int zc = !!(ublk_queue_use_zc(q) != 0);
130
+
enum io_uring_op op = stripe_to_uring_op(iod, zc);
129
131
struct io_uring_sqe *sqe[NR_STRIPE];
130
132
struct stripe_array *s = alloc_stripe_array(conf, iod);
131
133
struct ublk_io *io = ublk_get_io(q, tag);
132
-
int i;
134
+
int i, extra = zc ? 2 : 0;
133
135
134
136
io->private_data = s;
135
137
calculate_stripe_array(conf, iod, s);
136
138
137
-
ublk_queue_alloc_sqes(q, sqe, s->nr);
138
-
for (i = 0; i < s->nr; i++) {
139
-
struct stripe *t = &s->s[i];
139
+
ublk_queue_alloc_sqes(q, sqe, s->nr + extra);
140
+
141
+
if (zc) {
142
+
io_uring_prep_buf_register(sqe[0], 0, tag, q->q_id, tag);
143
+
sqe[0]->flags |= IOSQE_CQE_SKIP_SUCCESS | IOSQE_IO_HARDLINK;
144
+
sqe[0]->user_data = build_user_data(tag,
145
+
ublk_cmd_op_nr(sqe[0]->cmd_op), 0, 1);
146
+
}
147
+
148
+
for (i = zc; i < s->nr + extra - zc; i++) {
149
+
struct stripe *t = &s->s[i - zc];
140
150
141
151
io_uring_prep_rw(op, sqe[i],
142
152
t->seq + 1,
143
153
(void *)t->vec,
144
154
t->nr_vec,
145
155
t->start << 9);
146
-
io_uring_sqe_set_flags(sqe[i], IOSQE_FIXED_FILE);
156
+
if (zc) {
157
+
sqe[i]->buf_index = tag;
158
+
io_uring_sqe_set_flags(sqe[i],
159
+
IOSQE_FIXED_FILE | IOSQE_IO_HARDLINK);
160
+
} else {
161
+
io_uring_sqe_set_flags(sqe[i], IOSQE_FIXED_FILE);
162
+
}
147
163
/* bit63 marks us as tgt io */
148
-
sqe[i]->user_data = build_user_data(tag, ublksrv_get_op(iod), i, 1);
164
+
sqe[i]->user_data = build_user_data(tag, ublksrv_get_op(iod), i - zc, 1);
149
165
}
150
-
return s->nr;
166
+
if (zc) {
167
+
struct io_uring_sqe *unreg = sqe[s->nr + 1];
168
+
169
+
io_uring_prep_buf_unregister(unreg, 0, tag, q->q_id, tag);
170
+
unreg->user_data = build_user_data(tag, ublk_cmd_op_nr(unreg->cmd_op), 0, 1);
171
+
}
172
+
173
+
/* register buffer is skip_success */
174
+
return s->nr + zc;
151
175
}
152
176
153
177
static int handle_flush(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag)
···
232
208
struct ublk_io *io = ublk_get_io(q, tag);
233
209
int res = cqe->res;
234
210
235
-
if (res < 0) {
211
+
if (res < 0 || op != ublk_cmd_op_nr(UBLK_U_IO_UNREGISTER_IO_BUF)) {
236
212
if (!io->result)
237
213
io->result = res;
238
-
ublk_err("%s: io failure %d tag %u\n", __func__, res, tag);
214
+
if (res < 0)
215
+
ublk_err("%s: io failure %d tag %u\n", __func__, res, tag);
239
216
}
217
+
218
+
/* buffer register op is IOSQE_CQE_SKIP_SUCCESS */
219
+
if (op == ublk_cmd_op_nr(UBLK_U_IO_REGISTER_IO_BUF))
220
+
io->tgt_ios += 1;
240
221
241
222
/* fail short READ/WRITE simply */
242
223
if (op == UBLK_IO_OP_READ || op == UBLK_IO_OP_WRITE) {
243
224
unsigned seq = user_data_to_tgt_data(cqe->user_data);
244
225
struct stripe_array *s = io->private_data;
245
226
246
-
if (res < s->s[seq].vec->iov_len)
227
+
if (res < s->s[seq].nr_sects << 9) {
247
228
io->result = -EIO;
229
+
ublk_err("%s: short rw op %u res %d exp %u tag %u\n",
230
+
__func__, op, res, s->s[seq].vec->iov_len, tag);
231
+
}
248
232
}
249
233
250
234
if (ublk_completed_tgt_io(q, tag)) {
···
285
253
struct stripe_conf *conf;
286
254
unsigned chunk_shift;
287
255
loff_t bytes = 0;
288
-
int ret, i;
256
+
int ret, i, mul = 1;
289
257
290
258
if ((chunk_size & (chunk_size - 1)) || !chunk_size) {
291
259
ublk_err("invalid chunk size %u\n", chunk_size);
···
327
295
dev->tgt.dev_size = bytes;
328
296
p.basic.dev_sectors = bytes >> 9;
329
297
dev->tgt.params = p;
330
-
dev->tgt.sq_depth = dev->dev_info.queue_depth * conf->nr_files;
331
-
dev->tgt.cq_depth = dev->dev_info.queue_depth * conf->nr_files;
298
+
299
+
if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY)
300
+
mul = 2;
301
+
dev->tgt.sq_depth = mul * dev->dev_info.queue_depth * conf->nr_files;
302
+
dev->tgt.cq_depth = mul * dev->dev_info.queue_depth * conf->nr_files;
332
303
333
304
printf("%s: shift %u files %u\n", __func__, conf->shift, conf->nr_files);
334
305