Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
fscrypt: keep multiple bios in flight in fscrypt_zeroout_range_inline_crypt
This should slightly improve performance for large zeroing operations,
but more importantly prepares for blk-crypto refactoring that requires
all fscrypt users to call submit_bio directly.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Eric Biggers <ebiggers@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Christoph Hellwig
and committed by
Jens Axboe
bc26e2ef
c22756a9
+54
-32
+54
-32
fs/crypto/bio.c
+54
-32
fs/crypto/bio.c
···
47
47
}
48
48
EXPORT_SYMBOL(fscrypt_decrypt_bio);
49
49
50
+
struct fscrypt_zero_done {
51
+
atomic_t pending;
52
+
blk_status_t status;
53
+
struct completion done;
54
+
};
55
+
56
+
static void fscrypt_zeroout_range_done(struct fscrypt_zero_done *done)
57
+
{
58
+
if (atomic_dec_and_test(&done->pending))
59
+
complete(&done->done);
60
+
}
61
+
62
+
static void fscrypt_zeroout_range_end_io(struct bio *bio)
63
+
{
64
+
struct fscrypt_zero_done *done = bio->bi_private;
65
+
66
+
if (bio->bi_status)
67
+
cmpxchg(&done->status, 0, bio->bi_status);
68
+
fscrypt_zeroout_range_done(done);
69
+
bio_put(bio);
70
+
}
71
+
50
72
static int fscrypt_zeroout_range_inline_crypt(const struct inode *inode,
51
73
pgoff_t lblk, sector_t sector,
52
74
unsigned int len)
53
75
{
54
76
const unsigned int blockbits = inode->i_blkbits;
55
77
const unsigned int blocks_per_page = 1 << (PAGE_SHIFT - blockbits);
56
-
struct bio *bio;
57
-
int ret, err = 0;
58
-
int num_pages = 0;
59
-
60
-
/* This always succeeds since __GFP_DIRECT_RECLAIM is set. */
61
-
bio = bio_alloc(inode->i_sb->s_bdev, BIO_MAX_VECS, REQ_OP_WRITE,
62
-
GFP_NOFS);
78
+
struct fscrypt_zero_done done = {
79
+
.pending = ATOMIC_INIT(1),
80
+
.done = COMPLETION_INITIALIZER_ONSTACK(done.done),
81
+
};
63
82
64
83
while (len) {
65
-
unsigned int blocks_this_page = min(len, blocks_per_page);
66
-
unsigned int bytes_this_page = blocks_this_page << blockbits;
84
+
struct bio *bio;
85
+
unsigned int n;
67
86
68
-
if (num_pages == 0) {
69
-
fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOFS);
70
-
bio->bi_iter.bi_sector = sector;
87
+
bio = bio_alloc(inode->i_sb->s_bdev, BIO_MAX_VECS, REQ_OP_WRITE,
88
+
GFP_NOFS);
89
+
bio->bi_iter.bi_sector = sector;
90
+
bio->bi_private = &done;
91
+
bio->bi_end_io = fscrypt_zeroout_range_end_io;
92
+
fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOFS);
93
+
94
+
for (n = 0; n < BIO_MAX_VECS; n++) {
95
+
unsigned int blocks_this_page =
96
+
min(len, blocks_per_page);
97
+
unsigned int bytes_this_page = blocks_this_page << blockbits;
98
+
99
+
__bio_add_page(bio, ZERO_PAGE(0), bytes_this_page, 0);
100
+
len -= blocks_this_page;
101
+
lblk += blocks_this_page;
102
+
sector += (bytes_this_page >> SECTOR_SHIFT);
103
+
if (!len || !fscrypt_mergeable_bio(bio, inode, lblk))
104
+
break;
71
105
}
72
-
ret = bio_add_page(bio, ZERO_PAGE(0), bytes_this_page, 0);
73
-
if (WARN_ON_ONCE(ret != bytes_this_page)) {
74
-
err = -EIO;
75
-
goto out;
76
-
}
77
-
num_pages++;
78
-
len -= blocks_this_page;
79
-
lblk += blocks_this_page;
80
-
sector += (bytes_this_page >> SECTOR_SHIFT);
81
-
if (num_pages == BIO_MAX_VECS || !len ||
82
-
!fscrypt_mergeable_bio(bio, inode, lblk)) {
83
-
err = submit_bio_wait(bio);
84
-
if (err)
85
-
goto out;
86
-
bio_reset(bio, inode->i_sb->s_bdev, REQ_OP_WRITE);
87
-
num_pages = 0;
88
-
}
106
+
107
+
atomic_inc(&done.pending);
108
+
submit_bio(bio);
89
109
}
90
-
out:
91
-
bio_put(bio);
92
-
return err;
110
+
111
+
fscrypt_zeroout_range_done(&done);
112
+
113
+
wait_for_completion(&done.done);
114
+
return blk_status_to_errno(done.status);
93
115
}
94
116
95
117
/**