Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

dm-integrity: enable asynchronous hash interface

This commit enables the asynchronous hash interface in dm-integrity.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>

+199 -44
+199 -44
drivers/md/dm-integrity.c
··· 224 224 int failed; 225 225 226 226 struct crypto_shash *internal_shash; 227 + struct crypto_ahash *internal_ahash; 227 228 unsigned int internal_hash_digestsize; 228 229 229 230 struct dm_target *ti; ··· 280 279 bool fix_hmac; 281 280 bool legacy_recalculate; 282 281 282 + mempool_t ahash_req_pool; 283 + struct ahash_request *journal_ahash_req; 284 + 283 285 struct alg_spec internal_hash_alg; 284 286 struct alg_spec journal_crypt_alg; 285 287 struct alg_spec journal_mac_alg; ··· 332 328 unsigned payload_len; 333 329 bool integrity_payload_from_mempool; 334 330 bool integrity_range_locked; 331 + 332 + struct ahash_request *ahash_req; 335 333 }; 336 334 337 335 struct journal_completion { ··· 360 354 static struct kmem_cache *journal_io_cache; 361 355 362 356 #define JOURNAL_IO_MEMPOOL 32 357 + #define AHASH_MEMPOOL 32 363 358 364 359 #ifdef DEBUG_PRINT 365 360 #define DEBUG_print(x, ...) printk(KERN_DEBUG x, ##__VA_ARGS__) ··· 1643 1636 dec_in_flight(dio); 1644 1637 } 1645 1638 1646 - static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector, 1647 - const char *data, unsigned offset, char *result) 1639 + static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t sector, 1640 + const char *data, unsigned offset, char *result) 1648 1641 { 1649 1642 __le64 sector_le = cpu_to_le64(sector); 1650 1643 SHASH_DESC_ON_STACK(req, ic->internal_shash); ··· 1696 1689 get_random_bytes(result, ic->tag_size); 1697 1690 } 1698 1691 1692 + static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ahash_request **ahash_req, 1693 + sector_t sector, struct page *page, unsigned offset, char *result) 1694 + { 1695 + __le64 sector_le = cpu_to_le64(sector); 1696 + struct ahash_request *req; 1697 + DECLARE_CRYPTO_WAIT(wait); 1698 + struct scatterlist sg[3], *s = sg; 1699 + int r; 1700 + unsigned int digest_size; 1701 + unsigned int nbytes = 0; 1702 + 1703 + might_sleep(); 1704 + 1705 + req = *ahash_req; 1706 + if (unlikely(!req)) { 1707 + req = mempool_alloc(&ic->ahash_req_pool, GFP_NOIO); 1708 + *ahash_req = req; 1709 + } 1710 + 1711 + ahash_request_set_tfm(req, ic->internal_ahash); 1712 + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &wait); 1713 + 1714 + if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { 1715 + sg_init_table(sg, 3); 1716 + sg_set_buf(s, (const __u8 *)&ic->sb->salt, SALT_SIZE); 1717 + nbytes += SALT_SIZE; 1718 + s++; 1719 + } else { 1720 + sg_init_table(sg, 2); 1721 + } 1722 + 1723 + if (likely(!is_vmalloc_addr(&sector_le))) { 1724 + sg_set_buf(s, &sector_le, sizeof(sector_le)); 1725 + } else { 1726 + struct page *sec_page = vmalloc_to_page(&sector_le); 1727 + unsigned int sec_off = offset_in_page(&sector_le); 1728 + sg_set_page(s, sec_page, sizeof(sector_le), sec_off); 1729 + } 1730 + nbytes += sizeof(sector_le); 1731 + s++; 1732 + 1733 + sg_set_page(s, page, ic->sectors_per_block << SECTOR_SHIFT, offset); 1734 + nbytes += ic->sectors_per_block << SECTOR_SHIFT; 1735 + 1736 + ahash_request_set_crypt(req, sg, result, nbytes); 1737 + 1738 + r = crypto_wait_req(crypto_ahash_digest(req), &wait); 1739 + if (unlikely(r)) { 1740 + dm_integrity_io_error(ic, "crypto_ahash_digest", r); 1741 + goto failed; 1742 + } 1743 + 1744 + digest_size = ic->internal_hash_digestsize; 1745 + if (unlikely(digest_size < ic->tag_size)) 1746 + memset(result + digest_size, 0, ic->tag_size - digest_size); 1747 + 1748 + return; 1749 + 1750 + failed: 1751 + /* this shouldn't happen anyway, the hash functions have no reason to fail */ 1752 + get_random_bytes(result, ic->tag_size); 1753 + } 1754 + 1755 + static void integrity_sector_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req, 1756 + sector_t sector, const char *data, unsigned offset, char *result) 1757 + { 1758 + if (likely(ic->internal_shash != NULL)) 1759 + integrity_sector_checksum_shash(ic, sector, data, offset, result); 1760 + else 1761 + integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data, offset, result); 1762 + } 1763 + 1699 1764 static void *integrity_kmap(struct dm_integrity_c *ic, struct page *p) 1700 1765 { 1701 - return kmap_local_page(p); 1766 + if (likely(ic->internal_shash != NULL)) 1767 + return kmap_local_page(p); 1768 + else 1769 + return p; 1702 1770 } 1703 1771 1704 1772 static void integrity_kunmap(struct dm_integrity_c *ic, const void *ptr) 1705 1773 { 1706 - kunmap_local(ptr); 1774 + if (likely(ic->internal_shash != NULL)) 1775 + kunmap_local(ptr); 1707 1776 } 1708 1777 1709 1778 static void *integrity_identity(struct dm_integrity_c *ic, void *data) ··· 1788 1705 BUG_ON(offset_in_page(data)); 1789 1706 BUG_ON(!virt_addr_valid(data)); 1790 1707 #endif 1791 - return data; 1708 + if (likely(ic->internal_shash != NULL)) 1709 + return data; 1710 + else 1711 + return virt_to_page(data); 1792 1712 } 1793 1713 1794 1714 static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum) ··· 1845 1759 goto free_ret; 1846 1760 } 1847 1761 1848 - integrity_sector_checksum(ic, logical_sector, integrity_identity(ic, buffer), buffer_offset, checksum); 1762 + integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, integrity_identity(ic, buffer), buffer_offset, checksum); 1849 1763 r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block, 1850 1764 &dio->metadata_offset, ic->tag_size, TAG_CMP); 1851 1765 if (r) { ··· 1949 1863 pos = 0; 1950 1864 checksums_ptr = checksums; 1951 1865 do { 1952 - integrity_sector_checksum(ic, sector, mem, bv_copy.bv_offset + pos, checksums_ptr); 1866 + integrity_sector_checksum(ic, &dio->ahash_req, sector, mem, bv_copy.bv_offset + pos, checksums_ptr); 1953 1867 checksums_ptr += ic->tag_size; 1954 1868 sectors_to_process -= ic->sectors_per_block; 1955 1869 pos += ic->sectors_per_block << SECTOR_SHIFT; ··· 2057 1971 dio->ic = ic; 2058 1972 dio->bi_status = 0; 2059 1973 dio->op = bio_op(bio); 1974 + dio->ahash_req = NULL; 2060 1975 2061 1976 if (ic->mode == 'I') { 2062 1977 bio->bi_iter.bi_sector = dm_target_offset(ic->ti, bio->bi_iter.bi_sector); ··· 2227 2140 if (unlikely(digest_size > ic->tag_size)) { 2228 2141 char checksums_onstack[HASH_MAX_DIGESTSIZE]; 2229 2142 2230 - integrity_sector_checksum(ic, logical_sector, js_page, js_offset, checksums_onstack); 2143 + integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, js_page, js_offset, checksums_onstack); 2231 2144 memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size); 2232 2145 } else 2233 - integrity_sector_checksum(ic, logical_sector, js_page, js_offset, journal_entry_tag(ic, je)); 2146 + integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, js_page, js_offset, journal_entry_tag(ic, je)); 2234 2147 } 2235 2148 2236 2149 journal_entry_set_sector(je, logical_sector); ··· 2606 2519 const char *mem = integrity_kmap(ic, bv.bv_page); 2607 2520 if (ic->tag_size < ic->tuple_size) 2608 2521 memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tuple_size); 2609 - integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, dio->integrity_payload + pos); 2522 + integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, dio->integrity_payload + pos); 2610 2523 integrity_kunmap(ic, mem); 2611 2524 pos += ic->tuple_size; 2612 2525 bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT); ··· 2686 2599 } 2687 2600 bio_put(outgoing_bio); 2688 2601 2689 - integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, integrity_identity(ic, outgoing_data), 0, digest); 2602 + integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, integrity_identity(ic, outgoing_data), 0, digest); 2690 2603 if (unlikely(crypto_memneq(digest, dio->integrity_payload, min(ic->internal_hash_digestsize, ic->tag_size)))) { 2691 2604 DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx", 2692 2605 ic->dev->bdev, dio->bio_details.bi_iter.bi_sector); ··· 2710 2623 bio_endio(bio); 2711 2624 } 2712 2625 2626 + static inline bool dm_integrity_check(struct dm_integrity_c *ic, struct dm_integrity_io *dio) 2627 + { 2628 + struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 2629 + unsigned pos = 0; 2630 + 2631 + while (dio->bio_details.bi_iter.bi_size) { 2632 + char digest[HASH_MAX_DIGESTSIZE]; 2633 + struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); 2634 + char *mem = integrity_kmap(ic, bv.bv_page); 2635 + integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest); 2636 + if (unlikely(crypto_memneq(digest, dio->integrity_payload + pos, 2637 + min(ic->internal_hash_digestsize, ic->tag_size)))) { 2638 + integrity_kunmap(ic, mem); 2639 + dm_integrity_free_payload(dio); 2640 + INIT_WORK(&dio->work, dm_integrity_inline_recheck); 2641 + queue_work(ic->offload_wq, &dio->work); 2642 + return false; 2643 + } 2644 + integrity_kunmap(ic, mem); 2645 + pos += ic->tuple_size; 2646 + bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT); 2647 + } 2648 + 2649 + return true; 2650 + } 2651 + 2652 + static void dm_integrity_inline_async_check(struct work_struct *w) 2653 + { 2654 + struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); 2655 + struct dm_integrity_c *ic = dio->ic; 2656 + struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 2657 + 2658 + if (likely(dm_integrity_check(ic, dio))) 2659 + bio_endio(bio); 2660 + } 2661 + 2713 2662 static int dm_integrity_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status) 2714 2663 { 2715 2664 struct dm_integrity_c *ic = ti->private; 2665 + struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); 2716 2666 if (ic->mode == 'I') { 2717 - struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); 2718 - if (dio->op == REQ_OP_READ && likely(*status == BLK_STS_OK)) { 2719 - unsigned pos = 0; 2667 + if (dio->op == REQ_OP_READ && likely(*status == BLK_STS_OK) && likely(dio->bio_details.bi_iter.bi_size != 0)) { 2720 2668 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && 2721 2669 unlikely(dio->integrity_range_locked)) 2722 - goto skip_check; 2723 - while (dio->bio_details.bi_iter.bi_size) { 2724 - char digest[HASH_MAX_DIGESTSIZE]; 2725 - struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); 2726 - char *mem = integrity_kmap(ic, bv.bv_page); 2727 - integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest); 2728 - if (unlikely(crypto_memneq(digest, dio->integrity_payload + pos, 2729 - min(ic->internal_hash_digestsize, ic->tag_size)))) { 2730 - integrity_kunmap(ic, mem); 2731 - dm_integrity_free_payload(dio); 2732 - INIT_WORK(&dio->work, dm_integrity_inline_recheck); 2733 - queue_work(ic->offload_wq, &dio->work); 2670 + goto skip_check; 2671 + if (likely(ic->internal_shash != NULL)) { 2672 + if (unlikely(!dm_integrity_check(ic, dio))) 2734 2673 return DM_ENDIO_INCOMPLETE; 2735 - } 2736 - integrity_kunmap(ic, mem); 2737 - pos += ic->tuple_size; 2738 - bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT); 2674 + } else { 2675 + INIT_WORK(&dio->work, dm_integrity_inline_async_check); 2676 + queue_work(ic->offload_wq, &dio->work); 2677 + return DM_ENDIO_INCOMPLETE; 2739 2678 } 2740 2679 } 2741 2680 skip_check: ··· 2769 2656 if (unlikely(dio->integrity_range_locked)) 2770 2657 remove_range(ic, &dio->range); 2771 2658 } 2659 + if (unlikely(dio->ahash_req)) 2660 + mempool_free(dio->ahash_req, &ic->ahash_req_pool); 2772 2661 return DM_ENDIO_DONE; 2773 2662 } 2774 2663 ··· 3031 2916 void *js_page = integrity_identity(ic, (char *)js - offset_in_page(js)); 3032 2917 unsigned js_offset = offset_in_page(js); 3033 2918 3034 - integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block), 2919 + integrity_sector_checksum(ic, &ic->journal_ahash_req, sec + ((l - j) << ic->sb->log2_sectors_per_block), 3035 2920 js_page, js_offset, test_tag); 3036 2921 if (unlikely(crypto_memneq(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) { 3037 2922 dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ); ··· 3115 3000 size_t recalc_tags_size; 3116 3001 u8 *recalc_buffer = NULL; 3117 3002 u8 *recalc_tags = NULL; 3003 + struct ahash_request *ahash_req = NULL; 3118 3004 struct dm_integrity_range range; 3119 3005 struct dm_io_request io_req; 3120 3006 struct dm_io_region io_loc; ··· 3229 3113 void *ptr = recalc_buffer + (i << SECTOR_SHIFT); 3230 3114 void *ptr_page = integrity_identity(ic, (char *)ptr - offset_in_page(ptr)); 3231 3115 unsigned ptr_offset = offset_in_page(ptr); 3232 - integrity_sector_checksum(ic, logical_sector + i, ptr_page, ptr_offset, t); 3116 + integrity_sector_checksum(ic, &ahash_req, logical_sector + i, ptr_page, ptr_offset, t); 3233 3117 t += ic->tag_size; 3234 3118 } 3235 3119 ··· 3273 3157 free_ret: 3274 3158 kfree(recalc_buffer); 3275 3159 kvfree(recalc_tags); 3160 + mempool_free(ahash_req, &ic->ahash_req_pool); 3276 3161 } 3277 3162 3278 3163 static void integrity_recalc_inline(struct work_struct *w) ··· 3282 3165 size_t recalc_tags_size; 3283 3166 u8 *recalc_buffer = NULL; 3284 3167 u8 *recalc_tags = NULL; 3168 + struct ahash_request *ahash_req = NULL; 3285 3169 struct dm_integrity_range range; 3286 3170 struct bio *bio; 3287 3171 struct bio_integrity_payload *bip; ··· 3355 3237 void *ptr_page = integrity_identity(ic, (char *)ptr - offset_in_page(ptr)); 3356 3238 unsigned ptr_offset = offset_in_page(ptr); 3357 3239 memset(t, 0, ic->tuple_size); 3358 - integrity_sector_checksum(ic, range.logical_sector + i, ptr_page, ptr_offset, t); 3240 + integrity_sector_checksum(ic, &ahash_req, range.logical_sector + i, ptr_page, ptr_offset, t); 3359 3241 t += ic->tuple_size; 3360 3242 } 3361 3243 ··· 3407 3289 free_ret: 3408 3290 kfree(recalc_buffer); 3409 3291 kfree(recalc_tags); 3292 + mempool_free(ahash_req, &ic->ahash_req_pool); 3410 3293 } 3411 3294 3412 3295 static void bitmap_block_work(struct work_struct *w) ··· 4348 4229 return -ENOMEM; 4349 4230 } 4350 4231 4351 - static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error, 4352 - char *error_alg, char *error_key) 4232 + static int get_mac(struct crypto_shash **shash, struct crypto_ahash **ahash, 4233 + struct alg_spec *a, char **error, char *error_alg, char *error_key) 4353 4234 { 4354 4235 int r; 4355 4236 4356 4237 if (a->alg_string) { 4357 - *hash = crypto_alloc_shash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY); 4358 - if (IS_ERR(*hash)) { 4238 + if (ahash) { 4239 + *ahash = crypto_alloc_ahash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY); 4240 + if (IS_ERR(*ahash)) { 4241 + *ahash = NULL; 4242 + goto try_shash; 4243 + } 4244 + 4245 + if (a->key) { 4246 + r = crypto_ahash_setkey(*ahash, a->key, a->key_size); 4247 + if (r) { 4248 + *error = error_key; 4249 + return r; 4250 + } 4251 + } else if (crypto_ahash_get_flags(*ahash) & CRYPTO_TFM_NEED_KEY) { 4252 + *error = error_key; 4253 + return -ENOKEY; 4254 + } 4255 + 4256 + return 0; 4257 + } 4258 + 4259 + try_shash: 4260 + *shash = crypto_alloc_shash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY); 4261 + if (IS_ERR(*shash)) { 4359 4262 *error = error_alg; 4360 - r = PTR_ERR(*hash); 4361 - *hash = NULL; 4263 + r = PTR_ERR(*shash); 4264 + *shash = NULL; 4362 4265 return r; 4363 4266 } 4364 4267 4365 4268 if (a->key) { 4366 - r = crypto_shash_setkey(*hash, a->key, a->key_size); 4269 + r = crypto_shash_setkey(*shash, a->key, a->key_size); 4367 4270 if (r) { 4368 4271 *error = error_key; 4369 4272 return r; 4370 4273 } 4371 - } else if (crypto_shash_get_flags(*hash) & CRYPTO_TFM_NEED_KEY) { 4274 + } else if (crypto_shash_get_flags(*shash) & CRYPTO_TFM_NEED_KEY) { 4372 4275 *error = error_key; 4373 4276 return -ENOKEY; 4374 4277 } ··· 4850 4709 buffer_sectors = 1; 4851 4710 ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT); 4852 4711 4853 - r = get_mac(&ic->internal_shash, &ic->internal_hash_alg, &ti->error, 4712 + r = get_mac(&ic->internal_shash, &ic->internal_ahash, &ic->internal_hash_alg, &ti->error, 4854 4713 "Invalid internal hash", "Error setting internal hash key"); 4855 4714 if (r) 4856 4715 goto bad; ··· 4858 4717 ic->internal_hash = true; 4859 4718 ic->internal_hash_digestsize = crypto_shash_digestsize(ic->internal_shash); 4860 4719 } 4720 + if (ic->internal_ahash) { 4721 + ic->internal_hash = true; 4722 + ic->internal_hash_digestsize = crypto_ahash_digestsize(ic->internal_ahash); 4723 + r = mempool_init_kmalloc_pool(&ic->ahash_req_pool, AHASH_MEMPOOL, 4724 + sizeof(struct ahash_request) + crypto_ahash_reqsize(ic->internal_ahash)); 4725 + if (r) { 4726 + ti->error = "Cannot allocate mempool"; 4727 + goto bad; 4728 + } 4729 + } 4861 4730 4862 - r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error, 4731 + r = get_mac(&ic->journal_mac, NULL, &ic->journal_mac_alg, &ti->error, 4863 4732 "Invalid journal mac", "Error setting journal mac key"); 4864 4733 if (r) 4865 4734 goto bad; ··· 5352 5201 kvfree(ic->bbs); 5353 5202 if (ic->bufio) 5354 5203 dm_bufio_client_destroy(ic->bufio); 5204 + mempool_free(ic->journal_ahash_req, &ic->ahash_req_pool); 5205 + mempool_exit(&ic->ahash_req_pool); 5355 5206 bioset_exit(&ic->recalc_bios); 5356 5207 bioset_exit(&ic->recheck_bios); 5357 5208 mempool_exit(&ic->recheck_pool); ··· 5393 5240 5394 5241 if (ic->internal_shash) 5395 5242 crypto_free_shash(ic->internal_shash); 5243 + if (ic->internal_ahash) 5244 + crypto_free_ahash(ic->internal_ahash); 5396 5245 free_alg(&ic->internal_hash_alg); 5397 5246 5398 5247 if (ic->journal_crypt) ··· 5411 5256 5412 5257 static struct target_type integrity_target = { 5413 5258 .name = "integrity", 5414 - .version = {1, 13, 0}, 5259 + .version = {1, 14, 0}, 5415 5260 .module = THIS_MODULE, 5416 5261 .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY, 5417 5262 .ctr = dm_integrity_ctr,