Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring: add infra for importing vectored reg buffers
Add io_import_reg_vec(), which will be responsible for importing
vectored registered buffers. The function might reallocate the vector,
but it'd try to do the conversion in place first, which is why it's
required of the user to pad the iovec to the right border of the cache.
Overlapping also depends on struct iovec being larger than bvec, which
is not the case on e.g. 32 bit architectures. Don't try to complicate
this case and make sure vectors never overlap, it'll be improved later.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/60bd246b1249476a6996407c1dbc38ef6febad14.1741362889.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Pavel Begunkov
and committed by
Jens Axboe
9ef4cbbc
e1d49959
+138
-2
+5
-2
include/linux/io_uring_types.h
+5
-2
include/linux/io_uring_types.h
+128
io_uring/rsrc.c
+128
io_uring/rsrc.c
···
1269
1269
iv->iovec = NULL;
1270
1270
iv->nr = 0;
1271
1271
}
1272
+
1273
+
int io_vec_realloc(struct iou_vec *iv, unsigned nr_entries)
1274
+
{
1275
+
gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
1276
+
struct iovec *iov;
1277
+
1278
+
iov = kmalloc_array(nr_entries, sizeof(iov[0]), gfp);
1279
+
if (!iov)
1280
+
return -ENOMEM;
1281
+
1282
+
io_vec_free(iv);
1283
+
iv->iovec = iov;
1284
+
iv->nr = nr_entries;
1285
+
return 0;
1286
+
}
1287
+
1288
+
static int io_vec_fill_bvec(int ddir, struct iov_iter *iter,
1289
+
struct io_mapped_ubuf *imu,
1290
+
struct iovec *iovec, unsigned nr_iovs,
1291
+
struct iou_vec *vec)
1292
+
{
1293
+
unsigned long folio_size = 1 << imu->folio_shift;
1294
+
unsigned long folio_mask = folio_size - 1;
1295
+
u64 folio_addr = imu->ubuf & ~folio_mask;
1296
+
struct bio_vec *res_bvec = vec->bvec;
1297
+
size_t total_len = 0;
1298
+
unsigned bvec_idx = 0;
1299
+
unsigned iov_idx;
1300
+
1301
+
for (iov_idx = 0; iov_idx < nr_iovs; iov_idx++) {
1302
+
size_t iov_len = iovec[iov_idx].iov_len;
1303
+
u64 buf_addr = (u64)(uintptr_t)iovec[iov_idx].iov_base;
1304
+
struct bio_vec *src_bvec;
1305
+
size_t offset;
1306
+
u64 buf_end;
1307
+
1308
+
if (unlikely(check_add_overflow(buf_addr, (u64)iov_len, &buf_end)))
1309
+
return -EFAULT;
1310
+
if (unlikely(buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len)))
1311
+
return -EFAULT;
1312
+
if (unlikely(!iov_len))
1313
+
return -EFAULT;
1314
+
if (unlikely(check_add_overflow(total_len, iov_len, &total_len)))
1315
+
return -EOVERFLOW;
1316
+
1317
+
/* by using folio address it also accounts for bvec offset */
1318
+
offset = buf_addr - folio_addr;
1319
+
src_bvec = imu->bvec + (offset >> imu->folio_shift);
1320
+
offset &= folio_mask;
1321
+
1322
+
for (; iov_len; offset = 0, bvec_idx++, src_bvec++) {
1323
+
size_t seg_size = min_t(size_t, iov_len,
1324
+
folio_size - offset);
1325
+
1326
+
bvec_set_page(&res_bvec[bvec_idx],
1327
+
src_bvec->bv_page, seg_size, offset);
1328
+
iov_len -= seg_size;
1329
+
}
1330
+
}
1331
+
if (total_len > MAX_RW_COUNT)
1332
+
return -EINVAL;
1333
+
1334
+
iov_iter_bvec(iter, ddir, res_bvec, bvec_idx, total_len);
1335
+
return 0;
1336
+
}
1337
+
1338
+
static int io_estimate_bvec_size(struct iovec *iov, unsigned nr_iovs,
1339
+
struct io_mapped_ubuf *imu)
1340
+
{
1341
+
unsigned shift = imu->folio_shift;
1342
+
size_t max_segs = 0;
1343
+
unsigned i;
1344
+
1345
+
for (i = 0; i < nr_iovs; i++)
1346
+
max_segs += (iov[i].iov_len >> shift) + 2;
1347
+
return max_segs;
1348
+
}
1349
+
1350
+
int io_import_reg_vec(int ddir, struct iov_iter *iter,
1351
+
struct io_kiocb *req, struct iou_vec *vec,
1352
+
unsigned nr_iovs, unsigned iovec_off,
1353
+
unsigned issue_flags)
1354
+
{
1355
+
struct io_rsrc_node *node;
1356
+
struct io_mapped_ubuf *imu;
1357
+
struct iovec *iov;
1358
+
unsigned nr_segs;
1359
+
1360
+
node = io_find_buf_node(req, issue_flags);
1361
+
if (!node)
1362
+
return -EFAULT;
1363
+
imu = node->buf;
1364
+
if (imu->is_kbuf)
1365
+
return -EOPNOTSUPP;
1366
+
if (!(imu->dir & (1 << ddir)))
1367
+
return -EFAULT;
1368
+
1369
+
iov = vec->iovec + iovec_off;
1370
+
nr_segs = io_estimate_bvec_size(iov, nr_iovs, imu);
1371
+
1372
+
if (sizeof(struct bio_vec) > sizeof(struct iovec)) {
1373
+
size_t bvec_bytes;
1374
+
1375
+
bvec_bytes = nr_segs * sizeof(struct bio_vec);
1376
+
nr_segs = (bvec_bytes + sizeof(*iov) - 1) / sizeof(*iov);
1377
+
nr_segs += nr_iovs;
1378
+
}
1379
+
1380
+
if (WARN_ON_ONCE(iovec_off + nr_iovs != vec->nr) ||
1381
+
nr_segs > vec->nr) {
1382
+
struct iou_vec tmp_vec = {};
1383
+
int ret;
1384
+
1385
+
ret = io_vec_realloc(&tmp_vec, nr_segs);
1386
+
if (ret)
1387
+
return ret;
1388
+
1389
+
iovec_off = tmp_vec.nr - nr_iovs;
1390
+
memcpy(tmp_vec.iovec + iovec_off, iov, sizeof(*iov) * nr_iovs);
1391
+
io_vec_free(vec);
1392
+
1393
+
*vec = tmp_vec;
1394
+
iov = vec->iovec + iovec_off;
1395
+
req->flags |= REQ_F_NEED_CLEANUP;
1396
+
}
1397
+
1398
+
return io_vec_fill_bvec(ddir, iter, imu, iov, nr_iovs, vec);
1399
+
}
+5
io_uring/rsrc.h
+5
io_uring/rsrc.h
···
61
61
int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
62
62
u64 buf_addr, size_t len, int ddir,
63
63
unsigned issue_flags);
64
+
int io_import_reg_vec(int ddir, struct iov_iter *iter,
65
+
struct io_kiocb *req, struct iou_vec *vec,
66
+
unsigned nr_iovs, unsigned iovec_off,
67
+
unsigned issue_flags);
64
68
65
69
int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg);
66
70
int io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
···
150
146
}
151
147
152
148
void io_vec_free(struct iou_vec *iv);
149
+
int io_vec_realloc(struct iou_vec *iv, unsigned nr_entries);
153
150
154
151
static inline void io_vec_reset_iovec(struct iou_vec *iv,
155
152
struct iovec *iovec, unsigned nr)