Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
blk-crypto: submit the encrypted bio in blk_crypto_fallback_bio_prep
Restructure blk_crypto_fallback_bio_prep so that it always submits the
encrypted bio instead of passing it back to the caller, which allows
to simplify the calling conventions for blk_crypto_fallback_bio_prep and
blk_crypto_bio_prep so that they never have to return a bio, and can
use a true return value to indicate that the caller should submit the
bio, and false that the blk-crypto code consumed it.
The submission is handled by the on-stack bio list in the current
task_struct by the block layer and does not cause additional stack
usage or major overhead. It also prepares for the following optimization
and fixes for the blk-crypto fallback write path.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Eric Biggers <ebiggers@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Christoph Hellwig
and committed by
Jens Axboe
aefc2a1f
a3cc978e
+67
-77
+1
-1
block/blk-core.c
+1
-1
block/blk-core.c
+33
-37
block/blk-crypto-fallback.c
+33
-37
block/blk-crypto-fallback.c
···
250
250
251
251
/*
252
252
* The crypto API fallback's encryption routine.
253
-
* Allocate a bounce bio for encryption, encrypt the input bio using crypto API,
254
-
* and replace *bio_ptr with the bounce bio. May split input bio if it's too
255
-
* large. Returns true on success. Returns false and sets bio->bi_status on
256
-
* error.
253
+
*
254
+
* Allocate one or more bios for encryption, encrypt the input bio using the
255
+
* crypto API, and submit the encrypted bios. Sets bio->bi_status and
256
+
* completes the source bio on error
257
257
*/
258
-
static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr)
258
+
static void blk_crypto_fallback_encrypt_bio(struct bio *src_bio)
259
259
{
260
-
struct bio *src_bio, *enc_bio;
260
+
struct bio *enc_bio;
261
261
struct bio_crypt_ctx *bc;
262
262
struct blk_crypto_keyslot *slot;
263
263
int data_unit_size;
···
267
267
struct scatterlist src, dst;
268
268
union blk_crypto_iv iv;
269
269
unsigned int i, j;
270
-
bool ret = false;
271
270
blk_status_t blk_st;
272
271
273
272
/* Split the bio if it's too big for single page bvec */
274
-
if (!blk_crypto_fallback_split_bio_if_needed(bio_ptr))
275
-
return false;
273
+
if (!blk_crypto_fallback_split_bio_if_needed(&src_bio))
274
+
goto out_endio;
276
275
277
-
src_bio = *bio_ptr;
278
276
bc = src_bio->bi_crypt_context;
279
277
data_unit_size = bc->bc_key->crypto_cfg.data_unit_size;
280
278
···
280
282
enc_bio = blk_crypto_fallback_clone_bio(src_bio);
281
283
if (!enc_bio) {
282
284
src_bio->bi_status = BLK_STS_RESOURCE;
283
-
return false;
285
+
goto out_endio;
284
286
}
285
287
286
288
/*
···
343
345
344
346
enc_bio->bi_private = src_bio;
345
347
enc_bio->bi_end_io = blk_crypto_fallback_encrypt_endio;
346
-
*bio_ptr = enc_bio;
347
-
ret = true;
348
-
349
-
enc_bio = NULL;
350
-
goto out_free_ciph_req;
348
+
skcipher_request_free(ciph_req);
349
+
blk_crypto_put_keyslot(slot);
350
+
submit_bio(enc_bio);
351
+
return;
351
352
352
353
out_free_bounce_pages:
353
354
while (i > 0)
354
355
mempool_free(enc_bio->bi_io_vec[--i].bv_page,
355
356
blk_crypto_bounce_page_pool);
356
-
out_free_ciph_req:
357
357
skcipher_request_free(ciph_req);
358
358
out_release_keyslot:
359
359
blk_crypto_put_keyslot(slot);
360
360
out_put_enc_bio:
361
-
if (enc_bio)
362
-
bio_uninit(enc_bio);
361
+
bio_uninit(enc_bio);
363
362
kfree(enc_bio);
364
-
return ret;
363
+
out_endio:
364
+
bio_endio(src_bio);
365
365
}
366
366
367
367
/*
···
462
466
463
467
/**
464
468
* blk_crypto_fallback_bio_prep - Prepare a bio to use fallback en/decryption
469
+
* @bio: bio to prepare
465
470
*
466
-
* @bio_ptr: pointer to the bio to prepare
471
+
* If bio is doing a WRITE operation, allocate one or more bios to contain the
472
+
* encrypted payload and submit them.
467
473
*
468
-
* If bio is doing a WRITE operation, this splits the bio into two parts if it's
469
-
* too big (see blk_crypto_fallback_split_bio_if_needed()). It then allocates a
470
-
* bounce bio for the first part, encrypts it, and updates bio_ptr to point to
471
-
* the bounce bio.
472
-
*
473
-
* For a READ operation, we mark the bio for decryption by using bi_private and
474
+
* For a READ operation, mark the bio for decryption by using bi_private and
474
475
* bi_end_io.
475
476
*
476
-
* In either case, this function will make the bio look like a regular bio (i.e.
477
-
* as if no encryption context was ever specified) for the purposes of the rest
478
-
* of the stack except for blk-integrity (blk-integrity and blk-crypto are not
479
-
* currently supported together).
477
+
* In either case, this function will make the submitted bio(s) look like
478
+
* regular bios (i.e. as if no encryption context was ever specified) for the
479
+
* purposes of the rest of the stack except for blk-integrity (blk-integrity and
480
+
* blk-crypto are not currently supported together).
480
481
*
481
-
* Return: true on success. Sets bio->bi_status and returns false on error.
482
+
* Return: true if @bio should be submitted to the driver by the caller, else
483
+
* false. Sets bio->bi_status, calls bio_endio and returns false on error.
482
484
*/
483
-
bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr)
485
+
bool blk_crypto_fallback_bio_prep(struct bio *bio)
484
486
{
485
-
struct bio *bio = *bio_ptr;
486
487
struct bio_crypt_ctx *bc = bio->bi_crypt_context;
487
488
struct bio_fallback_crypt_ctx *f_ctx;
488
489
489
490
if (WARN_ON_ONCE(!tfms_inited[bc->bc_key->crypto_cfg.crypto_mode])) {
490
491
/* User didn't call blk_crypto_start_using_key() first */
491
-
bio->bi_status = BLK_STS_IOERR;
492
+
bio_io_error(bio);
492
493
return false;
493
494
}
494
495
495
496
if (!__blk_crypto_cfg_supported(blk_crypto_fallback_profile,
496
497
&bc->bc_key->crypto_cfg)) {
497
498
bio->bi_status = BLK_STS_NOTSUPP;
499
+
bio_endio(bio);
498
500
return false;
499
501
}
500
502
501
-
if (bio_data_dir(bio) == WRITE)
502
-
return blk_crypto_fallback_encrypt_bio(bio_ptr);
503
+
if (bio_data_dir(bio) == WRITE) {
504
+
blk_crypto_fallback_encrypt_bio(bio);
505
+
return false;
506
+
}
503
507
504
508
/*
505
509
* bio READ case: Set up a f_ctx in the bio's bi_private and set the
+6
-13
block/blk-crypto-internal.h
+6
-13
block/blk-crypto-internal.h
···
165
165
#endif
166
166
}
167
167
168
-
bool __blk_crypto_bio_prep(struct bio **bio_ptr);
169
-
static inline bool blk_crypto_bio_prep(struct bio **bio_ptr)
168
+
bool __blk_crypto_bio_prep(struct bio *bio);
169
+
static inline bool blk_crypto_bio_prep(struct bio *bio)
170
170
{
171
-
if (bio_has_crypt_ctx(*bio_ptr))
172
-
return __blk_crypto_bio_prep(bio_ptr);
171
+
if (bio_has_crypt_ctx(bio))
172
+
return __blk_crypto_bio_prep(bio);
173
173
return true;
174
174
}
175
175
···
215
215
return 0;
216
216
}
217
217
218
+
bool blk_crypto_fallback_bio_prep(struct bio *bio);
219
+
218
220
#ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK
219
221
220
222
int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num);
221
-
222
-
bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr);
223
223
224
224
int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key);
225
225
···
230
230
{
231
231
pr_warn_once("crypto API fallback is disabled\n");
232
232
return -ENOPKG;
233
-
}
234
-
235
-
static inline bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr)
236
-
{
237
-
pr_warn_once("crypto API fallback disabled; failing request.\n");
238
-
(*bio_ptr)->bi_status = BLK_STS_NOTSUPP;
239
-
return false;
240
233
}
241
234
242
235
static inline int
+27
-26
block/blk-crypto.c
+27
-26
block/blk-crypto.c
···
260
260
261
261
/**
262
262
* __blk_crypto_bio_prep - Prepare bio for inline encryption
263
-
*
264
-
* @bio_ptr: pointer to original bio pointer
263
+
* @bio: bio to prepare
265
264
*
266
265
* If the bio crypt context provided for the bio is supported by the underlying
267
266
* device's inline encryption hardware, do nothing.
268
267
*
269
268
* Otherwise, try to perform en/decryption for this bio by falling back to the
270
-
* kernel crypto API. When the crypto API fallback is used for encryption,
271
-
* blk-crypto may choose to split the bio into 2 - the first one that will
272
-
* continue to be processed and the second one that will be resubmitted via
273
-
* submit_bio_noacct. A bounce bio will be allocated to encrypt the contents
274
-
* of the aforementioned "first one", and *bio_ptr will be updated to this
275
-
* bounce bio.
269
+
* kernel crypto API. For encryption this means submitting newly allocated
270
+
* bios for the encrypted payload while keeping back the source bio until they
271
+
* complete, while for reads the decryption happens in-place by a hooked in
272
+
* completion handler.
276
273
*
277
274
* Caller must ensure bio has bio_crypt_ctx.
278
275
*
279
-
* Return: true on success; false on error (and bio->bi_status will be set
280
-
* appropriately, and bio_endio() will have been called so bio
281
-
* submission should abort).
276
+
* Return: true if @bio should be submitted to the driver by the caller, else
277
+
* false. Sets bio->bi_status, calls bio_endio and returns false on error.
282
278
*/
283
-
bool __blk_crypto_bio_prep(struct bio **bio_ptr)
279
+
bool __blk_crypto_bio_prep(struct bio *bio)
284
280
{
285
-
struct bio *bio = *bio_ptr;
286
281
const struct blk_crypto_key *bc_key = bio->bi_crypt_context->bc_key;
282
+
struct block_device *bdev = bio->bi_bdev;
287
283
288
284
/* Error if bio has no data. */
289
285
if (WARN_ON_ONCE(!bio_has_data(bio))) {
290
-
bio->bi_status = BLK_STS_IOERR;
291
-
goto fail;
286
+
bio_io_error(bio);
287
+
return false;
292
288
}
293
289
294
290
if (!bio_crypt_check_alignment(bio)) {
295
291
bio->bi_status = BLK_STS_INVAL;
296
-
goto fail;
292
+
bio_endio(bio);
293
+
return false;
297
294
}
298
295
299
296
/*
300
-
* Success if device supports the encryption context, or if we succeeded
301
-
* in falling back to the crypto API.
297
+
* If the device does not natively support the encryption context, try to use
298
+
* the fallback if available.
302
299
*/
303
-
if (blk_crypto_config_supported_natively(bio->bi_bdev,
304
-
&bc_key->crypto_cfg))
305
-
return true;
306
-
if (blk_crypto_fallback_bio_prep(bio_ptr))
307
-
return true;
308
-
fail:
309
-
bio_endio(*bio_ptr);
310
-
return false;
300
+
if (!blk_crypto_config_supported_natively(bdev, &bc_key->crypto_cfg)) {
301
+
if (!IS_ENABLED(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK)) {
302
+
pr_warn_once("%pg: crypto API fallback disabled; failing request.\n",
303
+
bdev);
304
+
bio->bi_status = BLK_STS_NOTSUPP;
305
+
bio_endio(bio);
306
+
return false;
307
+
}
308
+
return blk_crypto_fallback_bio_prep(bio);
309
+
}
310
+
311
+
return true;
311
312
}
312
313
313
314
int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,