Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
block: add size alignment to bio_iov_iter_get_pages
The block layer tries to align bio vectors to the block device's logical
block size. Some cases don't have a block device, or we may need to
align to something larger, which we can't derive it from the queue
limits. Have the caller specify what they want, or allow any length
alignment if nothing was specified. Since the most common use case
relies on the block device's limits, a helper function is provided.
Signed-off-by: Keith Busch <kbusch@kernel.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Keith Busch
and committed by
Jens Axboe
743bf2e0
fec2e705
+30
-13
+11
-8
block/bio.c
+11
-8
block/bio.c
···
1208
1208
* For a multi-segment *iter, this function only adds pages from the next
1209
1209
* non-empty segment of the iov iterator.
1210
1210
*/
1211
-
static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
1211
+
static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
1212
+
unsigned len_align_mask)
1212
1213
{
1213
1214
iov_iter_extraction_t extraction_flags = 0;
1214
1215
unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
···
1218
1217
struct page **pages = (struct page **)bv;
1219
1218
ssize_t size;
1220
1219
unsigned int num_pages, i = 0;
1221
-
size_t offset, folio_offset, left, len;
1220
+
size_t offset, folio_offset, left, len, trim;
1222
1221
int ret = 0;
1223
1222
1224
1223
/*
···
1247
1246
1248
1247
nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE);
1249
1248
1250
-
if (bio->bi_bdev) {
1251
-
size_t trim = size & (bdev_logical_block_size(bio->bi_bdev) - 1);
1249
+
trim = size & len_align_mask;
1250
+
if (trim) {
1252
1251
iov_iter_revert(iter, trim);
1253
1252
size -= trim;
1254
1253
}
···
1303
1302
}
1304
1303
1305
1304
/**
1306
-
* bio_iov_iter_get_pages - add user or kernel pages to a bio
1305
+
* bio_iov_iter_get_pages_aligned - add user or kernel pages to a bio
1307
1306
* @bio: bio to add pages to
1308
1307
* @iter: iov iterator describing the region to be added
1308
+
* @len_align_mask: the mask to align each vector size to, 0 for any length
1309
1309
*
1310
1310
* This takes either an iterator pointing to user memory, or one pointing to
1311
1311
* kernel pages (BVEC iterator). If we're adding user pages, we pin them and
···
1323
1321
* MM encounters an error pinning the requested pages, it stops. Error
1324
1322
* is returned only if 0 pages could be pinned.
1325
1323
*/
1326
-
int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
1324
+
int bio_iov_iter_get_pages_aligned(struct bio *bio, struct iov_iter *iter,
1325
+
unsigned len_align_mask)
1327
1326
{
1328
1327
int ret = 0;
1329
1328
···
1340
1337
if (iov_iter_extract_will_pin(iter))
1341
1338
bio_set_flag(bio, BIO_PAGE_PINNED);
1342
1339
do {
1343
-
ret = __bio_iov_iter_get_pages(bio, iter);
1340
+
ret = __bio_iov_iter_get_pages(bio, iter, len_align_mask);
1344
1341
} while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
1345
1342
1346
1343
return bio->bi_vcnt ? 0 : ret;
1347
1344
}
1348
-
EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);
1345
+
EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages_aligned);
1349
1346
1350
1347
static void submit_bio_wait_endio(struct bio *bio)
1351
1348
{
+3
-3
block/fops.c
+3
-3
block/fops.c
···
78
78
if (iocb->ki_flags & IOCB_ATOMIC)
79
79
bio.bi_opf |= REQ_ATOMIC;
80
80
81
-
ret = bio_iov_iter_get_pages(&bio, iter);
81
+
ret = bio_iov_iter_get_bdev_pages(&bio, iter, bdev);
82
82
if (unlikely(ret))
83
83
goto out;
84
84
ret = bio.bi_iter.bi_size;
···
212
212
bio->bi_end_io = blkdev_bio_end_io;
213
213
bio->bi_ioprio = iocb->ki_ioprio;
214
214
215
-
ret = bio_iov_iter_get_pages(bio, iter);
215
+
ret = bio_iov_iter_get_bdev_pages(bio, iter, bdev);
216
216
if (unlikely(ret)) {
217
217
bio->bi_status = BLK_STS_IOERR;
218
218
bio_endio(bio);
···
348
348
*/
349
349
bio_iov_bvec_set(bio, iter);
350
350
} else {
351
-
ret = bio_iov_iter_get_pages(bio, iter);
351
+
ret = bio_iov_iter_get_bdev_pages(bio, iter, bdev);
352
352
if (unlikely(ret))
353
353
goto out_bio_put;
354
354
}
+1
-1
fs/iomap/direct-io.c
+1
-1
fs/iomap/direct-io.c
···
434
434
bio->bi_private = dio;
435
435
bio->bi_end_io = iomap_dio_bio_end_io;
436
436
437
-
ret = bio_iov_iter_get_pages(bio, dio->submit.iter);
437
+
ret = bio_iov_iter_get_bdev_pages(bio, dio->submit.iter, iomap->bdev);
438
438
if (unlikely(ret)) {
439
439
/*
440
440
* We have to stop part way through an IO. We must fall
+8
-1
include/linux/bio.h
+8
-1
include/linux/bio.h
···
446
446
int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data,
447
447
size_t len, enum req_op op);
448
448
449
-
int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
449
+
int bio_iov_iter_get_pages_aligned(struct bio *bio, struct iov_iter *iter,
450
+
unsigned len_align_mask);
451
+
452
+
static inline int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
453
+
{
454
+
return bio_iov_iter_get_pages_aligned(bio, iter, 0);
455
+
}
456
+
450
457
void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter);
451
458
void __bio_release_pages(struct bio *bio, bool mark_dirty);
452
459
extern void bio_set_pages_dirty(struct bio *bio);
+7
include/linux/blkdev.h
+7
include/linux/blkdev.h
···
1877
1877
return bio_split_io_at(bio, lim, segs, max_bytes, lim->dma_alignment);
1878
1878
}
1879
1879
1880
+
static inline int bio_iov_iter_get_bdev_pages(struct bio *bio,
1881
+
struct iov_iter *iter, struct block_device *bdev)
1882
+
{
1883
+
return bio_iov_iter_get_pages_aligned(bio, iter,
1884
+
bdev_logical_block_size(bdev) - 1);
1885
+
}
1886
+
1880
1887
#define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { }
1881
1888
1882
1889
#endif /* _LINUX_BLKDEV_H */