blk-crypto: optimize data unit alignment checking

Avoid the relatively high overhead of constructing and walking per-page
segment bio_vecs for data unit alignment checking by merging the checks
into existing loops.

For hardware support crypto, perform the check in bio_split_io_at, which
already contains a similar alignment check applied for all I/O. This
means bio-based drivers that do not call bio_split_to_limits, should they
ever grow blk-crypto support, need to implement the check themselves,
just like all other queue limits checks.

For blk-crypto-fallback do it in the encryption/decryption loops. This
means alignment errors for decryption will only be detected after I/O
has completed, but that seems like a worthwhile trade off.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Eric Biggers <ebiggers@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Christoph Hellwig and committed by

Jens Axboe 5 months ago 66e5a11d 3d939695

+21 -25

3 changed files

expand all

block

blk-crypto-fallback.c

blk-crypto.c

blk-merge.c

+13 -2

block/blk-crypto-fallback.c

··· 278 278 bio_iter_iovec(src_bio, src_bio->bi_iter); 279 279 struct page *enc_page = enc_pages[enc_idx]; 280 280 281 + if (!IS_ALIGNED(src_bv.bv_len | src_bv.bv_offset, 282 + data_unit_size)) { 283 + enc_bio->bi_status = BLK_STS_INVAL; 284 + goto out_free_enc_bio; 285 + } 286 + 281 287 __bio_add_page(enc_bio, enc_page, src_bv.bv_len, 282 288 src_bv.bv_offset); 283 289 ··· 302 296 */ 303 297 for (i = 0; i < src_bv.bv_len; i += data_unit_size) { 304 298 blk_crypto_dun_to_iv(curr_dun, &iv); 305 - if (crypto_skcipher_encrypt(ciph_req)) 299 + if (crypto_skcipher_encrypt(ciph_req)) { 300 + enc_bio->bi_status = BLK_STS_IOERR; 306 301 goto out_free_enc_bio; 302 + } 307 303 bio_crypt_dun_increment(curr_dun, 1); 308 304 src.offset += data_unit_size; 309 305 dst.offset += data_unit_size; ··· 342 334 */ 343 335 for (; enc_idx < nr_enc_pages; enc_idx++) 344 336 __bio_add_page(enc_bio, enc_pages[enc_idx], PAGE_SIZE, 0); 345 - bio_io_error(enc_bio); 337 + bio_endio(enc_bio); 346 338 } 347 339 348 340 /* ··· 394 386 /* Decrypt each segment in the bio */ 395 387 __bio_for_each_segment(bv, bio, iter, iter) { 396 388 struct page *page = bv.bv_page; 389 + 390 + if (!IS_ALIGNED(bv.bv_len | bv.bv_offset, data_unit_size)) 391 + return BLK_STS_INVAL; 397 392 398 393 sg_set_page(&sg, page, data_unit_size, bv.bv_offset); 399 394

-22

block/blk-crypto.c

··· 219 219 return !bc1 || bio_crypt_dun_is_contiguous(bc1, bc1_bytes, bc2->bc_dun); 220 220 } 221 221 222 - /* Check that all I/O segments are data unit aligned. */ 223 - static bool bio_crypt_check_alignment(struct bio *bio) 224 - { 225 - const unsigned int data_unit_size = 226 - bio->bi_crypt_context->bc_key->crypto_cfg.data_unit_size; 227 - struct bvec_iter iter; 228 - struct bio_vec bv; 229 - 230 - bio_for_each_segment(bv, bio, iter) { 231 - if (!IS_ALIGNED(bv.bv_len | bv.bv_offset, data_unit_size)) 232 - return false; 233 - } 234 - 235 - return true; 236 - } 237 - 238 222 blk_status_t __blk_crypto_rq_get_keyslot(struct request *rq) 239 223 { 240 224 return blk_crypto_get_keyslot(rq->q->crypto_profile, ··· 268 284 /* Error if bio has no data. */ 269 285 if (WARN_ON_ONCE(!bio_has_data(bio))) { 270 286 bio_io_error(bio); 271 - return false; 272 - } 273 - 274 - if (!bio_crypt_check_alignment(bio)) { 275 - bio->bi_status = BLK_STS_INVAL; 276 - bio_endio(bio); 277 287 return false; 278 288 } 279 289

+8 -1

block/blk-merge.c

··· 324 324 int bio_split_io_at(struct bio *bio, const struct queue_limits *lim, 325 325 unsigned *segs, unsigned max_bytes, unsigned len_align_mask) 326 326 { 327 + struct bio_crypt_ctx *bc = bio_crypt_ctx(bio); 327 328 struct bio_vec bv, bvprv, *bvprvp = NULL; 328 329 unsigned nsegs = 0, bytes = 0, gaps = 0; 329 330 struct bvec_iter iter; 331 + unsigned start_align_mask = lim->dma_alignment; 332 + 333 + if (bc) { 334 + start_align_mask |= (bc->bc_key->crypto_cfg.data_unit_size - 1); 335 + len_align_mask |= (bc->bc_key->crypto_cfg.data_unit_size - 1); 336 + } 330 337 331 338 bio_for_each_bvec(bv, bio, iter) { 332 - if (bv.bv_offset & lim->dma_alignment || 339 + if (bv.bv_offset & start_align_mask || 333 340 bv.bv_len & len_align_mask) 334 341 return -EINVAL; 335 342

Configure Feed

Configure Feed