Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
iomap: split out the per-bio logic from iomap_dio_bio_iter
Factor out a separate helper that builds and submits a single bio.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <dlemoal@kernel.org>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Tested-by: Anuj Gupta <anuj20.g@samsung.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Christoph Hellwig
and committed by
Jens Axboe
2631c946
6e7a6c80
+59
-52
+59
-52
fs/iomap/direct-io.c
+59
-52
fs/iomap/direct-io.c
···
302
302
return 0;
303
303
}
304
304
305
+
static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
306
+
struct iomap_dio *dio, loff_t pos, unsigned int alignment,
307
+
blk_opf_t op)
308
+
{
309
+
struct bio *bio;
310
+
ssize_t ret;
311
+
312
+
bio = iomap_dio_alloc_bio(iter, dio,
313
+
bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS),
314
+
op);
315
+
fscrypt_set_bio_crypt_ctx(bio, iter->inode,
316
+
pos >> iter->inode->i_blkbits, GFP_KERNEL);
317
+
bio->bi_iter.bi_sector = iomap_sector(&iter->iomap, pos);
318
+
bio->bi_write_hint = iter->inode->i_write_hint;
319
+
bio->bi_ioprio = dio->iocb->ki_ioprio;
320
+
bio->bi_private = dio;
321
+
bio->bi_end_io = iomap_dio_bio_end_io;
322
+
323
+
ret = bio_iov_iter_get_pages(bio, dio->submit.iter, alignment - 1);
324
+
if (unlikely(ret))
325
+
goto out_put_bio;
326
+
ret = bio->bi_iter.bi_size;
327
+
328
+
/*
329
+
* An atomic write bio must cover the complete length. If it doesn't,
330
+
* error out.
331
+
*/
332
+
if ((op & REQ_ATOMIC) && WARN_ON_ONCE(ret != iomap_length(iter))) {
333
+
ret = -EINVAL;
334
+
goto out_put_bio;
335
+
}
336
+
337
+
if (dio->flags & IOMAP_DIO_WRITE)
338
+
task_io_account_write(ret);
339
+
else if (dio->flags & IOMAP_DIO_DIRTY)
340
+
bio_set_pages_dirty(bio);
341
+
342
+
/*
343
+
* We can only poll for single bio I/Os.
344
+
*/
345
+
if (iov_iter_count(dio->submit.iter))
346
+
dio->iocb->ki_flags &= ~IOCB_HIPRI;
347
+
iomap_dio_submit_bio(iter, dio, bio, pos);
348
+
return ret;
349
+
350
+
out_put_bio:
351
+
bio_put(bio);
352
+
return ret;
353
+
}
354
+
305
355
static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
306
356
{
307
357
const struct iomap *iomap = &iter->iomap;
···
360
310
const loff_t length = iomap_length(iter);
361
311
loff_t pos = iter->pos;
362
312
blk_opf_t bio_opf = REQ_SYNC | REQ_IDLE;
363
-
struct bio *bio;
364
313
bool need_zeroout = false;
365
-
int ret = 0;
366
314
u64 copied = 0;
367
315
size_t orig_count;
368
316
unsigned int alignment;
317
+
ssize_t ret = 0;
369
318
370
319
/*
371
320
* File systems that write out of place and always allocate new blocks
···
490
441
}
491
442
492
443
do {
493
-
size_t n;
494
-
495
444
/*
496
445
* If completions already occurred and reported errors, give up now and
497
446
* don't bother submitting more bios.
498
447
*/
499
-
if (unlikely(data_race(dio->error))) {
500
-
ret = 0;
448
+
if (unlikely(data_race(dio->error)))
501
449
goto out;
502
-
}
503
450
504
-
bio = iomap_dio_alloc_bio(iter, dio,
505
-
bio_iov_vecs_to_alloc(dio->submit.iter,
506
-
BIO_MAX_VECS), bio_opf);
507
-
fscrypt_set_bio_crypt_ctx(bio, inode, pos >> inode->i_blkbits,
508
-
GFP_KERNEL);
509
-
bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
510
-
bio->bi_write_hint = inode->i_write_hint;
511
-
bio->bi_ioprio = dio->iocb->ki_ioprio;
512
-
bio->bi_private = dio;
513
-
bio->bi_end_io = iomap_dio_bio_end_io;
514
-
515
-
ret = bio_iov_iter_get_pages(bio, dio->submit.iter,
516
-
alignment - 1);
517
-
if (unlikely(ret)) {
451
+
ret = iomap_dio_bio_iter_one(iter, dio, pos, alignment, bio_opf);
452
+
if (unlikely(ret < 0)) {
518
453
/*
519
454
* We have to stop part way through an IO. We must fall
520
455
* through to the sub-block tail zeroing here, otherwise
521
456
* this short IO may expose stale data in the tail of
522
457
* the block we haven't written data to.
523
458
*/
524
-
bio_put(bio);
525
-
goto zero_tail;
459
+
break;
526
460
}
527
-
528
-
n = bio->bi_iter.bi_size;
529
-
if (WARN_ON_ONCE((bio_opf & REQ_ATOMIC) && n != length)) {
530
-
/*
531
-
* An atomic write bio must cover the complete length,
532
-
* which it doesn't, so error. We may need to zero out
533
-
* the tail (complete FS block), similar to when
534
-
* bio_iov_iter_get_pages() returns an error, above.
535
-
*/
536
-
ret = -EINVAL;
537
-
bio_put(bio);
538
-
goto zero_tail;
539
-
}
540
-
if (dio->flags & IOMAP_DIO_WRITE)
541
-
task_io_account_write(n);
542
-
else if (dio->flags & IOMAP_DIO_DIRTY)
543
-
bio_set_pages_dirty(bio);
544
-
545
-
dio->size += n;
546
-
copied += n;
547
-
548
-
/*
549
-
* We can only poll for single bio I/Os.
550
-
*/
551
-
if (iov_iter_count(dio->submit.iter))
552
-
dio->iocb->ki_flags &= ~IOCB_HIPRI;
553
-
iomap_dio_submit_bio(iter, dio, bio, pos);
554
-
pos += n;
461
+
dio->size += ret;
462
+
copied += ret;
463
+
pos += ret;
464
+
ret = 0;
555
465
} while (iov_iter_count(dio->submit.iter));
556
466
557
467
/*
···
519
511
* the block tail in the latter case, we can expose stale data via mmap
520
512
* reads of the EOF block.
521
513
*/
522
-
zero_tail:
523
514
if (need_zeroout ||
524
515
((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
525
516
/* zero out from the end of the write to the end of the block */