Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
blk-crypto: use on-stack skcipher requests for fallback en/decryption
Allocating a skcipher request dynamically can deadlock or cause
unexpected I/O failures when called from writeback context. Avoid the
allocation entirely by using on-stack skciphers, similar to what the
non-blk-crypto fscrypt path already does.
This drops the incomplete support for asynchronous algorithms, which
previously could be used, but only synchronously.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Eric Biggers <ebiggers@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Christoph Hellwig
and committed by
Jens Axboe
2f655dcb
b37fbce4
+94
-115
+94
-115
block/blk-crypto-fallback.c
+94
-115
block/blk-crypto-fallback.c
···
75
75
76
76
static struct blk_crypto_fallback_keyslot {
77
77
enum blk_crypto_mode_num crypto_mode;
78
-
struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX];
78
+
struct crypto_sync_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX];
79
79
} *blk_crypto_keyslots;
80
80
81
81
static struct blk_crypto_profile *blk_crypto_fallback_profile;
···
98
98
WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID);
99
99
100
100
/* Clear the key in the skcipher */
101
-
err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key,
101
+
err = crypto_sync_skcipher_setkey(slotp->tfms[crypto_mode], blank_key,
102
102
blk_crypto_modes[crypto_mode].keysize);
103
103
WARN_ON(err);
104
104
slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID;
···
119
119
blk_crypto_fallback_evict_keyslot(slot);
120
120
121
121
slotp->crypto_mode = crypto_mode;
122
-
err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key->bytes,
122
+
err = crypto_sync_skcipher_setkey(slotp->tfms[crypto_mode], key->bytes,
123
123
key->size);
124
124
if (err) {
125
125
blk_crypto_fallback_evict_keyslot(slot);
···
176
176
return bio;
177
177
}
178
178
179
-
static bool
180
-
blk_crypto_fallback_alloc_cipher_req(struct blk_crypto_keyslot *slot,
181
-
struct skcipher_request **ciph_req_ret,
182
-
struct crypto_wait *wait)
179
+
static struct crypto_sync_skcipher *
180
+
blk_crypto_fallback_tfm(struct blk_crypto_keyslot *slot)
183
181
{
184
-
struct skcipher_request *ciph_req;
185
-
const struct blk_crypto_fallback_keyslot *slotp;
186
-
int keyslot_idx = blk_crypto_keyslot_index(slot);
182
+
const struct blk_crypto_fallback_keyslot *slotp =
183
+
&blk_crypto_keyslots[blk_crypto_keyslot_index(slot)];
187
184
188
-
slotp = &blk_crypto_keyslots[keyslot_idx];
189
-
ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode],
190
-
GFP_NOIO);
191
-
if (!ciph_req)
192
-
return false;
193
-
194
-
skcipher_request_set_callback(ciph_req,
195
-
CRYPTO_TFM_REQ_MAY_BACKLOG |
196
-
CRYPTO_TFM_REQ_MAY_SLEEP,
197
-
crypto_req_done, wait);
198
-
*ciph_req_ret = ciph_req;
199
-
200
-
return true;
185
+
return slotp->tfms[slotp->crypto_mode];
201
186
}
202
187
203
188
union blk_crypto_iv {
···
199
214
iv->dun[i] = cpu_to_le64(dun[i]);
200
215
}
201
216
202
-
/*
203
-
* The crypto API fallback's encryption routine.
204
-
*
205
-
* Allocate one or more bios for encryption, encrypt the input bio using the
206
-
* crypto API, and submit the encrypted bios. Sets bio->bi_status and
207
-
* completes the source bio on error
208
-
*/
209
-
static void blk_crypto_fallback_encrypt_bio(struct bio *src_bio)
217
+
static void __blk_crypto_fallback_encrypt_bio(struct bio *src_bio,
218
+
struct crypto_sync_skcipher *tfm)
210
219
{
211
220
struct bio_crypt_ctx *bc = src_bio->bi_crypt_context;
212
221
int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size;
213
-
struct skcipher_request *ciph_req = NULL;
214
-
struct blk_crypto_keyslot *slot;
215
-
DECLARE_CRYPTO_WAIT(wait);
222
+
SYNC_SKCIPHER_REQUEST_ON_STACK(ciph_req, tfm);
216
223
u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
217
224
struct scatterlist src, dst;
218
225
union blk_crypto_iv iv;
219
226
unsigned int nr_enc_pages, enc_idx;
220
227
struct bio *enc_bio;
221
-
blk_status_t status;
222
228
unsigned int i;
223
229
224
-
/*
225
-
* Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for
226
-
* this bio's algorithm and key.
227
-
*/
228
-
status = blk_crypto_get_keyslot(blk_crypto_fallback_profile,
229
-
bc->bc_key, &slot);
230
-
if (status != BLK_STS_OK) {
231
-
src_bio->bi_status = status;
232
-
bio_endio(src_bio);
233
-
return;
234
-
}
235
-
236
-
/* and then allocate an skcipher_request for it */
237
-
if (!blk_crypto_fallback_alloc_cipher_req(slot, &ciph_req, &wait)) {
238
-
src_bio->bi_status = BLK_STS_RESOURCE;
239
-
bio_endio(src_bio);
240
-
goto out_release_keyslot;
241
-
}
230
+
skcipher_request_set_callback(ciph_req,
231
+
CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
232
+
NULL, NULL);
242
233
243
234
memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun));
244
235
sg_init_table(&src, 1);
···
258
297
*/
259
298
for (i = 0; i < src_bv.bv_len; i += data_unit_size) {
260
299
blk_crypto_dun_to_iv(curr_dun, &iv);
261
-
if (crypto_wait_req(crypto_skcipher_encrypt(ciph_req),
262
-
&wait)) {
300
+
if (crypto_skcipher_encrypt(ciph_req)) {
263
301
bio_io_error(enc_bio);
264
-
goto out_free_request;
302
+
return;
265
303
}
266
304
bio_crypt_dun_increment(curr_dun, 1);
267
305
src.offset += data_unit_size;
···
287
327
}
288
328
289
329
submit_bio(enc_bio);
290
-
out_free_request:
291
-
skcipher_request_free(ciph_req);
292
-
out_release_keyslot:
330
+
}
331
+
332
+
/*
333
+
* The crypto API fallback's encryption routine.
334
+
*
335
+
* Allocate one or more bios for encryption, encrypt the input bio using the
336
+
* crypto API, and submit the encrypted bios. Sets bio->bi_status and
337
+
* completes the source bio on error
338
+
*/
339
+
static void blk_crypto_fallback_encrypt_bio(struct bio *src_bio)
340
+
{
341
+
struct bio_crypt_ctx *bc = src_bio->bi_crypt_context;
342
+
struct blk_crypto_keyslot *slot;
343
+
blk_status_t status;
344
+
345
+
status = blk_crypto_get_keyslot(blk_crypto_fallback_profile,
346
+
bc->bc_key, &slot);
347
+
if (status != BLK_STS_OK) {
348
+
src_bio->bi_status = status;
349
+
bio_endio(src_bio);
350
+
return;
351
+
}
352
+
__blk_crypto_fallback_encrypt_bio(src_bio,
353
+
blk_crypto_fallback_tfm(slot));
293
354
blk_crypto_put_keyslot(slot);
355
+
}
356
+
357
+
static blk_status_t __blk_crypto_fallback_decrypt_bio(struct bio *bio,
358
+
struct bio_crypt_ctx *bc, struct bvec_iter iter,
359
+
struct crypto_sync_skcipher *tfm)
360
+
{
361
+
SYNC_SKCIPHER_REQUEST_ON_STACK(ciph_req, tfm);
362
+
u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
363
+
union blk_crypto_iv iv;
364
+
struct scatterlist sg;
365
+
struct bio_vec bv;
366
+
const int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size;
367
+
unsigned int i;
368
+
369
+
skcipher_request_set_callback(ciph_req,
370
+
CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
371
+
NULL, NULL);
372
+
373
+
memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun));
374
+
sg_init_table(&sg, 1);
375
+
skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size,
376
+
iv.bytes);
377
+
378
+
/* Decrypt each segment in the bio */
379
+
__bio_for_each_segment(bv, bio, iter, iter) {
380
+
struct page *page = bv.bv_page;
381
+
382
+
sg_set_page(&sg, page, data_unit_size, bv.bv_offset);
383
+
384
+
/* Decrypt each data unit in the segment */
385
+
for (i = 0; i < bv.bv_len; i += data_unit_size) {
386
+
blk_crypto_dun_to_iv(curr_dun, &iv);
387
+
if (crypto_skcipher_decrypt(ciph_req))
388
+
return BLK_STS_IOERR;
389
+
bio_crypt_dun_increment(curr_dun, 1);
390
+
sg.offset += data_unit_size;
391
+
}
392
+
}
393
+
394
+
return BLK_STS_OK;
294
395
}
295
396
296
397
/*
297
398
* The crypto API fallback's main decryption routine.
399
+
*
298
400
* Decrypts input bio in place, and calls bio_endio on the bio.
299
401
*/
300
402
static void blk_crypto_fallback_decrypt_bio(struct work_struct *work)
···
366
344
struct bio *bio = f_ctx->bio;
367
345
struct bio_crypt_ctx *bc = &f_ctx->crypt_ctx;
368
346
struct blk_crypto_keyslot *slot;
369
-
struct skcipher_request *ciph_req = NULL;
370
-
DECLARE_CRYPTO_WAIT(wait);
371
-
u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
372
-
union blk_crypto_iv iv;
373
-
struct scatterlist sg;
374
-
struct bio_vec bv;
375
-
struct bvec_iter iter;
376
-
const int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size;
377
-
unsigned int i;
378
-
blk_status_t blk_st;
347
+
blk_status_t status;
379
348
380
-
/*
381
-
* Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for
382
-
* this bio's algorithm and key.
383
-
*/
384
-
blk_st = blk_crypto_get_keyslot(blk_crypto_fallback_profile,
349
+
status = blk_crypto_get_keyslot(blk_crypto_fallback_profile,
385
350
bc->bc_key, &slot);
386
-
if (blk_st != BLK_STS_OK) {
387
-
bio->bi_status = blk_st;
388
-
goto out_no_keyslot;
351
+
if (status == BLK_STS_OK) {
352
+
status = __blk_crypto_fallback_decrypt_bio(bio, bc,
353
+
f_ctx->crypt_iter,
354
+
blk_crypto_fallback_tfm(slot));
355
+
blk_crypto_put_keyslot(slot);
389
356
}
390
-
391
-
/* and then allocate an skcipher_request for it */
392
-
if (!blk_crypto_fallback_alloc_cipher_req(slot, &ciph_req, &wait)) {
393
-
bio->bi_status = BLK_STS_RESOURCE;
394
-
goto out;
395
-
}
396
-
397
-
memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun));
398
-
sg_init_table(&sg, 1);
399
-
skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size,
400
-
iv.bytes);
401
-
402
-
/* Decrypt each segment in the bio */
403
-
__bio_for_each_segment(bv, bio, iter, f_ctx->crypt_iter) {
404
-
struct page *page = bv.bv_page;
405
-
406
-
sg_set_page(&sg, page, data_unit_size, bv.bv_offset);
407
-
408
-
/* Decrypt each data unit in the segment */
409
-
for (i = 0; i < bv.bv_len; i += data_unit_size) {
410
-
blk_crypto_dun_to_iv(curr_dun, &iv);
411
-
if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req),
412
-
&wait)) {
413
-
bio->bi_status = BLK_STS_IOERR;
414
-
goto out;
415
-
}
416
-
bio_crypt_dun_increment(curr_dun, 1);
417
-
sg.offset += data_unit_size;
418
-
}
419
-
}
420
-
421
-
out:
422
-
skcipher_request_free(ciph_req);
423
-
blk_crypto_put_keyslot(slot);
424
-
out_no_keyslot:
425
357
mempool_free(f_ctx, bio_fallback_crypt_ctx_pool);
358
+
359
+
bio->bi_status = status;
426
360
bio_endio(bio);
427
361
}
428
362
···
586
608
587
609
for (i = 0; i < blk_crypto_num_keyslots; i++) {
588
610
slotp = &blk_crypto_keyslots[i];
589
-
slotp->tfms[mode_num] = crypto_alloc_skcipher(cipher_str, 0, 0);
611
+
slotp->tfms[mode_num] = crypto_alloc_sync_skcipher(cipher_str,
612
+
0, 0);
590
613
if (IS_ERR(slotp->tfms[mode_num])) {
591
614
err = PTR_ERR(slotp->tfms[mode_num]);
592
615
if (err == -ENOENT) {
···
599
620
goto out_free_tfms;
600
621
}
601
622
602
-
crypto_skcipher_set_flags(slotp->tfms[mode_num],
623
+
crypto_sync_skcipher_set_flags(slotp->tfms[mode_num],
603
624
CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
604
625
}
605
626
···
613
634
out_free_tfms:
614
635
for (i = 0; i < blk_crypto_num_keyslots; i++) {
615
636
slotp = &blk_crypto_keyslots[i];
616
-
crypto_free_skcipher(slotp->tfms[mode_num]);
637
+
crypto_free_sync_skcipher(slotp->tfms[mode_num]);
617
638
slotp->tfms[mode_num] = NULL;
618
639
}
619
640
out: