Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
blk-crypto: use mempool_alloc_bulk for encrypted bio page allocation
Calling mempool_alloc in a loop is not safe unless the maximum allocation
size times the maximum number of threads using it is less than the
minimum pool size. Use the new mempool_alloc_bulk helper to allocate
all missing elements in one pass to remove this deadlock risk. This
also means that non-pool allocations now use alloc_pages_bulk which can
be significantly faster than a loop over individual page allocations.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Eric Biggers <ebiggers@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Christoph Hellwig
and committed by
Jens Axboe
3d939695
2f655dcb
+62
-14
+62
-14
block/blk-crypto-fallback.c
+62
-14
block/blk-crypto-fallback.c
···
22
22
#include "blk-cgroup.h"
23
23
#include "blk-crypto-internal.h"
24
24
25
-
static unsigned int num_prealloc_bounce_pg = 32;
25
+
static unsigned int num_prealloc_bounce_pg = BIO_MAX_VECS;
26
26
module_param(num_prealloc_bounce_pg, uint, 0);
27
27
MODULE_PARM_DESC(num_prealloc_bounce_pg,
28
28
"Number of preallocated bounce pages for the blk-crypto crypto API fallback");
···
144
144
static void blk_crypto_fallback_encrypt_endio(struct bio *enc_bio)
145
145
{
146
146
struct bio *src_bio = enc_bio->bi_private;
147
-
int i;
147
+
struct page **pages = (struct page **)enc_bio->bi_io_vec;
148
+
struct bio_vec *bv;
149
+
unsigned int i;
148
150
149
-
for (i = 0; i < enc_bio->bi_vcnt; i++)
150
-
mempool_free(enc_bio->bi_io_vec[i].bv_page,
151
-
blk_crypto_bounce_page_pool);
151
+
/*
152
+
* Use the same trick as the alloc side to avoid the need for an extra
153
+
* pages array.
154
+
*/
155
+
bio_for_each_bvec_all(bv, enc_bio, i)
156
+
pages[i] = bv->bv_page;
157
+
158
+
i = mempool_free_bulk(blk_crypto_bounce_page_pool, (void **)pages,
159
+
enc_bio->bi_vcnt);
160
+
if (i < enc_bio->bi_vcnt)
161
+
release_pages(pages + i, enc_bio->bi_vcnt - i);
152
162
153
163
if (enc_bio->bi_status)
154
164
cmpxchg(&src_bio->bi_status, 0, enc_bio->bi_status);
···
167
157
bio_endio(src_bio);
168
158
}
169
159
160
+
#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *))
161
+
170
162
static struct bio *blk_crypto_alloc_enc_bio(struct bio *bio_src,
171
-
unsigned int nr_segs)
163
+
unsigned int nr_segs, struct page ***pages_ret)
172
164
{
165
+
unsigned int memflags = memalloc_noio_save();
166
+
unsigned int nr_allocated;
167
+
struct page **pages;
173
168
struct bio *bio;
174
169
175
170
bio = bio_alloc_bioset(bio_src->bi_bdev, nr_segs, bio_src->bi_opf,
···
188
173
bio->bi_write_stream = bio_src->bi_write_stream;
189
174
bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
190
175
bio_clone_blkg_association(bio, bio_src);
176
+
177
+
/*
178
+
* Move page array up in the allocated memory for the bio vecs as far as
179
+
* possible so that we can start filling biovecs from the beginning
180
+
* without overwriting the temporary page array.
181
+
*/
182
+
static_assert(PAGE_PTRS_PER_BVEC > 1);
183
+
pages = (struct page **)bio->bi_io_vec;
184
+
pages += nr_segs * (PAGE_PTRS_PER_BVEC - 1);
185
+
186
+
/*
187
+
* Try a bulk allocation first. This could leave random pages in the
188
+
* array unallocated, but we'll fix that up later in mempool_alloc_bulk.
189
+
*
190
+
* Note: alloc_pages_bulk needs the array to be zeroed, as it assumes
191
+
* any non-zero slot already contains a valid allocation.
192
+
*/
193
+
memset(pages, 0, sizeof(struct page *) * nr_segs);
194
+
nr_allocated = alloc_pages_bulk(GFP_KERNEL, nr_segs, pages);
195
+
if (nr_allocated < nr_segs)
196
+
mempool_alloc_bulk(blk_crypto_bounce_page_pool, (void **)pages,
197
+
nr_segs, nr_allocated);
198
+
memalloc_noio_restore(memflags);
199
+
*pages_ret = pages;
191
200
return bio;
192
201
}
193
202
···
248
209
struct scatterlist src, dst;
249
210
union blk_crypto_iv iv;
250
211
unsigned int nr_enc_pages, enc_idx;
212
+
struct page **enc_pages;
251
213
struct bio *enc_bio;
252
214
unsigned int i;
253
215
···
271
231
*/
272
232
new_bio:
273
233
nr_enc_pages = min(bio_segments(src_bio), BIO_MAX_VECS);
274
-
enc_bio = blk_crypto_alloc_enc_bio(src_bio, nr_enc_pages);
234
+
enc_bio = blk_crypto_alloc_enc_bio(src_bio, nr_enc_pages, &enc_pages);
275
235
enc_idx = 0;
276
236
for (;;) {
277
237
struct bio_vec src_bv =
278
238
bio_iter_iovec(src_bio, src_bio->bi_iter);
279
-
struct page *enc_page;
239
+
struct page *enc_page = enc_pages[enc_idx];
280
240
281
-
enc_page = mempool_alloc(blk_crypto_bounce_page_pool,
282
-
GFP_NOIO);
283
241
__bio_add_page(enc_bio, enc_page, src_bv.bv_len,
284
242
src_bv.bv_offset);
285
243
···
296
258
*/
297
259
for (i = 0; i < src_bv.bv_len; i += data_unit_size) {
298
260
blk_crypto_dun_to_iv(curr_dun, &iv);
299
-
if (crypto_skcipher_encrypt(ciph_req)) {
300
-
bio_io_error(enc_bio);
301
-
return;
302
-
}
261
+
if (crypto_skcipher_encrypt(ciph_req))
262
+
goto out_free_enc_bio;
303
263
bio_crypt_dun_increment(curr_dun, 1);
304
264
src.offset += data_unit_size;
305
265
dst.offset += data_unit_size;
···
323
287
}
324
288
325
289
submit_bio(enc_bio);
290
+
return;
291
+
292
+
out_free_enc_bio:
293
+
/*
294
+
* Add the remaining pages to the bio so that the normal completion path
295
+
* in blk_crypto_fallback_encrypt_endio frees them. The exact data
296
+
* layout does not matter for that, so don't bother iterating the source
297
+
* bio.
298
+
*/
299
+
for (; enc_idx < nr_enc_pages; enc_idx++)
300
+
__bio_add_page(enc_bio, enc_pages[enc_idx], PAGE_SIZE, 0);
301
+
bio_io_error(enc_bio);
326
302
}
327
303
328
304
/*