Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring: support vectored kernel fixed buffer
io_uring has supported fixed kernel buffer via io_buffer_register_bvec()
and io_buffer_unregister_bvec().
The vectored fixed buffer has been ready, so it is natural to support
fixed kernel buffer, one use case is ublk.
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/20250325135155.935398-4-ming.lei@redhat.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Ming Lei
and committed by
Jens Axboe
1045afae
149974fd
+88
-3
+88
-3
io_uring/rsrc.c
+88
-3
io_uring/rsrc.c
···
1362
1362
return max_segs;
1363
1363
}
1364
1364
1365
+
static int io_vec_fill_kern_bvec(int ddir, struct iov_iter *iter,
1366
+
struct io_mapped_ubuf *imu,
1367
+
struct iovec *iovec, unsigned nr_iovs,
1368
+
struct iou_vec *vec)
1369
+
{
1370
+
const struct bio_vec *src_bvec = imu->bvec;
1371
+
struct bio_vec *res_bvec = vec->bvec;
1372
+
unsigned res_idx = 0;
1373
+
size_t total_len = 0;
1374
+
unsigned iov_idx;
1375
+
1376
+
for (iov_idx = 0; iov_idx < nr_iovs; iov_idx++) {
1377
+
size_t offset = (size_t)(uintptr_t)iovec[iov_idx].iov_base;
1378
+
size_t iov_len = iovec[iov_idx].iov_len;
1379
+
struct bvec_iter bi = {
1380
+
.bi_size = offset + iov_len,
1381
+
};
1382
+
struct bio_vec bv;
1383
+
1384
+
bvec_iter_advance(src_bvec, &bi, offset);
1385
+
for_each_mp_bvec(bv, src_bvec, bi, bi)
1386
+
res_bvec[res_idx++] = bv;
1387
+
total_len += iov_len;
1388
+
}
1389
+
iov_iter_bvec(iter, ddir, res_bvec, res_idx, total_len);
1390
+
return 0;
1391
+
}
1392
+
1393
+
static int iov_kern_bvec_size(const struct iovec *iov,
1394
+
const struct io_mapped_ubuf *imu,
1395
+
unsigned int *nr_seg)
1396
+
{
1397
+
size_t offset = (size_t)(uintptr_t)iov->iov_base;
1398
+
const struct bio_vec *bvec = imu->bvec;
1399
+
int start = 0, i = 0;
1400
+
size_t off = 0;
1401
+
int ret;
1402
+
1403
+
ret = validate_fixed_range(offset, iov->iov_len, imu);
1404
+
if (unlikely(ret))
1405
+
return ret;
1406
+
1407
+
for (i = 0; off < offset + iov->iov_len && i < imu->nr_bvecs;
1408
+
off += bvec[i].bv_len, i++) {
1409
+
if (offset >= off && offset < off + bvec[i].bv_len)
1410
+
start = i;
1411
+
}
1412
+
*nr_seg = i - start;
1413
+
return 0;
1414
+
}
1415
+
1416
+
static int io_kern_bvec_size(struct iovec *iov, unsigned nr_iovs,
1417
+
struct io_mapped_ubuf *imu, unsigned *nr_segs)
1418
+
{
1419
+
unsigned max_segs = 0;
1420
+
size_t total_len = 0;
1421
+
unsigned i;
1422
+
int ret;
1423
+
1424
+
*nr_segs = 0;
1425
+
for (i = 0; i < nr_iovs; i++) {
1426
+
if (unlikely(!iov[i].iov_len))
1427
+
return -EFAULT;
1428
+
if (unlikely(check_add_overflow(total_len, iov[i].iov_len,
1429
+
&total_len)))
1430
+
return -EOVERFLOW;
1431
+
ret = iov_kern_bvec_size(&iov[i], imu, &max_segs);
1432
+
if (unlikely(ret))
1433
+
return ret;
1434
+
*nr_segs += max_segs;
1435
+
}
1436
+
if (total_len > MAX_RW_COUNT)
1437
+
return -EINVAL;
1438
+
return 0;
1439
+
}
1440
+
1365
1441
int io_import_reg_vec(int ddir, struct iov_iter *iter,
1366
1442
struct io_kiocb *req, struct iou_vec *vec,
1367
1443
unsigned nr_iovs, unsigned issue_flags)
···
1452
1376
if (!node)
1453
1377
return -EFAULT;
1454
1378
imu = node->buf;
1455
-
if (imu->is_kbuf)
1456
-
return -EOPNOTSUPP;
1457
1379
if (!(imu->dir & (1 << ddir)))
1458
1380
return -EFAULT;
1459
1381
1460
1382
iovec_off = vec->nr - nr_iovs;
1461
1383
iov = vec->iovec + iovec_off;
1462
-
nr_segs = io_estimate_bvec_size(iov, nr_iovs, imu);
1384
+
1385
+
if (imu->is_kbuf) {
1386
+
int ret = io_kern_bvec_size(iov, nr_iovs, imu, &nr_segs);
1387
+
1388
+
if (unlikely(ret))
1389
+
return ret;
1390
+
} else {
1391
+
nr_segs = io_estimate_bvec_size(iov, nr_iovs, imu);
1392
+
}
1463
1393
1464
1394
if (sizeof(struct bio_vec) > sizeof(struct iovec)) {
1465
1395
size_t bvec_bytes;
···
1491
1409
iov = vec->iovec + iovec_off;
1492
1410
req->flags |= REQ_F_NEED_CLEANUP;
1493
1411
}
1412
+
1413
+
if (imu->is_kbuf)
1414
+
return io_vec_fill_kern_bvec(ddir, iter, imu, iov, nr_iovs, vec);
1494
1415
1495
1416
return io_vec_fill_bvec(ddir, iter, imu, iov, nr_iovs, vec);
1496
1417
}