Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
dm-integrity: enable asynchronous hash interface
This commit enables the asynchronous hash interface in dm-integrity.
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
+199
-44
+199
-44
drivers/md/dm-integrity.c
+199
-44
drivers/md/dm-integrity.c
···
224
224
int failed;
225
225
226
226
struct crypto_shash *internal_shash;
227
+
struct crypto_ahash *internal_ahash;
227
228
unsigned int internal_hash_digestsize;
228
229
229
230
struct dm_target *ti;
···
280
279
bool fix_hmac;
281
280
bool legacy_recalculate;
282
281
282
+
mempool_t ahash_req_pool;
283
+
struct ahash_request *journal_ahash_req;
284
+
283
285
struct alg_spec internal_hash_alg;
284
286
struct alg_spec journal_crypt_alg;
285
287
struct alg_spec journal_mac_alg;
···
332
328
unsigned payload_len;
333
329
bool integrity_payload_from_mempool;
334
330
bool integrity_range_locked;
331
+
332
+
struct ahash_request *ahash_req;
335
333
};
336
334
337
335
struct journal_completion {
···
360
354
static struct kmem_cache *journal_io_cache;
361
355
362
356
#define JOURNAL_IO_MEMPOOL 32
357
+
#define AHASH_MEMPOOL 32
363
358
364
359
#ifdef DEBUG_PRINT
365
360
#define DEBUG_print(x, ...) printk(KERN_DEBUG x, ##__VA_ARGS__)
···
1643
1636
dec_in_flight(dio);
1644
1637
}
1645
1638
1646
-
static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector,
1647
-
const char *data, unsigned offset, char *result)
1639
+
static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t sector,
1640
+
const char *data, unsigned offset, char *result)
1648
1641
{
1649
1642
__le64 sector_le = cpu_to_le64(sector);
1650
1643
SHASH_DESC_ON_STACK(req, ic->internal_shash);
···
1696
1689
get_random_bytes(result, ic->tag_size);
1697
1690
}
1698
1691
1692
+
static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ahash_request **ahash_req,
1693
+
sector_t sector, struct page *page, unsigned offset, char *result)
1694
+
{
1695
+
__le64 sector_le = cpu_to_le64(sector);
1696
+
struct ahash_request *req;
1697
+
DECLARE_CRYPTO_WAIT(wait);
1698
+
struct scatterlist sg[3], *s = sg;
1699
+
int r;
1700
+
unsigned int digest_size;
1701
+
unsigned int nbytes = 0;
1702
+
1703
+
might_sleep();
1704
+
1705
+
req = *ahash_req;
1706
+
if (unlikely(!req)) {
1707
+
req = mempool_alloc(&ic->ahash_req_pool, GFP_NOIO);
1708
+
*ahash_req = req;
1709
+
}
1710
+
1711
+
ahash_request_set_tfm(req, ic->internal_ahash);
1712
+
ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &wait);
1713
+
1714
+
if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
1715
+
sg_init_table(sg, 3);
1716
+
sg_set_buf(s, (const __u8 *)&ic->sb->salt, SALT_SIZE);
1717
+
nbytes += SALT_SIZE;
1718
+
s++;
1719
+
} else {
1720
+
sg_init_table(sg, 2);
1721
+
}
1722
+
1723
+
if (likely(!is_vmalloc_addr(§or_le))) {
1724
+
sg_set_buf(s, §or_le, sizeof(sector_le));
1725
+
} else {
1726
+
struct page *sec_page = vmalloc_to_page(§or_le);
1727
+
unsigned int sec_off = offset_in_page(§or_le);
1728
+
sg_set_page(s, sec_page, sizeof(sector_le), sec_off);
1729
+
}
1730
+
nbytes += sizeof(sector_le);
1731
+
s++;
1732
+
1733
+
sg_set_page(s, page, ic->sectors_per_block << SECTOR_SHIFT, offset);
1734
+
nbytes += ic->sectors_per_block << SECTOR_SHIFT;
1735
+
1736
+
ahash_request_set_crypt(req, sg, result, nbytes);
1737
+
1738
+
r = crypto_wait_req(crypto_ahash_digest(req), &wait);
1739
+
if (unlikely(r)) {
1740
+
dm_integrity_io_error(ic, "crypto_ahash_digest", r);
1741
+
goto failed;
1742
+
}
1743
+
1744
+
digest_size = ic->internal_hash_digestsize;
1745
+
if (unlikely(digest_size < ic->tag_size))
1746
+
memset(result + digest_size, 0, ic->tag_size - digest_size);
1747
+
1748
+
return;
1749
+
1750
+
failed:
1751
+
/* this shouldn't happen anyway, the hash functions have no reason to fail */
1752
+
get_random_bytes(result, ic->tag_size);
1753
+
}
1754
+
1755
+
static void integrity_sector_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req,
1756
+
sector_t sector, const char *data, unsigned offset, char *result)
1757
+
{
1758
+
if (likely(ic->internal_shash != NULL))
1759
+
integrity_sector_checksum_shash(ic, sector, data, offset, result);
1760
+
else
1761
+
integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data, offset, result);
1762
+
}
1763
+
1699
1764
static void *integrity_kmap(struct dm_integrity_c *ic, struct page *p)
1700
1765
{
1701
-
return kmap_local_page(p);
1766
+
if (likely(ic->internal_shash != NULL))
1767
+
return kmap_local_page(p);
1768
+
else
1769
+
return p;
1702
1770
}
1703
1771
1704
1772
static void integrity_kunmap(struct dm_integrity_c *ic, const void *ptr)
1705
1773
{
1706
-
kunmap_local(ptr);
1774
+
if (likely(ic->internal_shash != NULL))
1775
+
kunmap_local(ptr);
1707
1776
}
1708
1777
1709
1778
static void *integrity_identity(struct dm_integrity_c *ic, void *data)
···
1788
1705
BUG_ON(offset_in_page(data));
1789
1706
BUG_ON(!virt_addr_valid(data));
1790
1707
#endif
1791
-
return data;
1708
+
if (likely(ic->internal_shash != NULL))
1709
+
return data;
1710
+
else
1711
+
return virt_to_page(data);
1792
1712
}
1793
1713
1794
1714
static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum)
···
1845
1759
goto free_ret;
1846
1760
}
1847
1761
1848
-
integrity_sector_checksum(ic, logical_sector, integrity_identity(ic, buffer), buffer_offset, checksum);
1762
+
integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, integrity_identity(ic, buffer), buffer_offset, checksum);
1849
1763
r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block,
1850
1764
&dio->metadata_offset, ic->tag_size, TAG_CMP);
1851
1765
if (r) {
···
1949
1863
pos = 0;
1950
1864
checksums_ptr = checksums;
1951
1865
do {
1952
-
integrity_sector_checksum(ic, sector, mem, bv_copy.bv_offset + pos, checksums_ptr);
1866
+
integrity_sector_checksum(ic, &dio->ahash_req, sector, mem, bv_copy.bv_offset + pos, checksums_ptr);
1953
1867
checksums_ptr += ic->tag_size;
1954
1868
sectors_to_process -= ic->sectors_per_block;
1955
1869
pos += ic->sectors_per_block << SECTOR_SHIFT;
···
2057
1971
dio->ic = ic;
2058
1972
dio->bi_status = 0;
2059
1973
dio->op = bio_op(bio);
1974
+
dio->ahash_req = NULL;
2060
1975
2061
1976
if (ic->mode == 'I') {
2062
1977
bio->bi_iter.bi_sector = dm_target_offset(ic->ti, bio->bi_iter.bi_sector);
···
2227
2140
if (unlikely(digest_size > ic->tag_size)) {
2228
2141
char checksums_onstack[HASH_MAX_DIGESTSIZE];
2229
2142
2230
-
integrity_sector_checksum(ic, logical_sector, js_page, js_offset, checksums_onstack);
2143
+
integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, js_page, js_offset, checksums_onstack);
2231
2144
memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size);
2232
2145
} else
2233
-
integrity_sector_checksum(ic, logical_sector, js_page, js_offset, journal_entry_tag(ic, je));
2146
+
integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, js_page, js_offset, journal_entry_tag(ic, je));
2234
2147
}
2235
2148
2236
2149
journal_entry_set_sector(je, logical_sector);
···
2606
2519
const char *mem = integrity_kmap(ic, bv.bv_page);
2607
2520
if (ic->tag_size < ic->tuple_size)
2608
2521
memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tuple_size);
2609
-
integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, dio->integrity_payload + pos);
2522
+
integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, dio->integrity_payload + pos);
2610
2523
integrity_kunmap(ic, mem);
2611
2524
pos += ic->tuple_size;
2612
2525
bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);
···
2686
2599
}
2687
2600
bio_put(outgoing_bio);
2688
2601
2689
-
integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, integrity_identity(ic, outgoing_data), 0, digest);
2602
+
integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, integrity_identity(ic, outgoing_data), 0, digest);
2690
2603
if (unlikely(crypto_memneq(digest, dio->integrity_payload, min(ic->internal_hash_digestsize, ic->tag_size)))) {
2691
2604
DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx",
2692
2605
ic->dev->bdev, dio->bio_details.bi_iter.bi_sector);
···
2710
2623
bio_endio(bio);
2711
2624
}
2712
2625
2626
+
static inline bool dm_integrity_check(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
2627
+
{
2628
+
struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
2629
+
unsigned pos = 0;
2630
+
2631
+
while (dio->bio_details.bi_iter.bi_size) {
2632
+
char digest[HASH_MAX_DIGESTSIZE];
2633
+
struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);
2634
+
char *mem = integrity_kmap(ic, bv.bv_page);
2635
+
integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest);
2636
+
if (unlikely(crypto_memneq(digest, dio->integrity_payload + pos,
2637
+
min(ic->internal_hash_digestsize, ic->tag_size)))) {
2638
+
integrity_kunmap(ic, mem);
2639
+
dm_integrity_free_payload(dio);
2640
+
INIT_WORK(&dio->work, dm_integrity_inline_recheck);
2641
+
queue_work(ic->offload_wq, &dio->work);
2642
+
return false;
2643
+
}
2644
+
integrity_kunmap(ic, mem);
2645
+
pos += ic->tuple_size;
2646
+
bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);
2647
+
}
2648
+
2649
+
return true;
2650
+
}
2651
+
2652
+
static void dm_integrity_inline_async_check(struct work_struct *w)
2653
+
{
2654
+
struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
2655
+
struct dm_integrity_c *ic = dio->ic;
2656
+
struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
2657
+
2658
+
if (likely(dm_integrity_check(ic, dio)))
2659
+
bio_endio(bio);
2660
+
}
2661
+
2713
2662
static int dm_integrity_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status)
2714
2663
{
2715
2664
struct dm_integrity_c *ic = ti->private;
2665
+
struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
2716
2666
if (ic->mode == 'I') {
2717
-
struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
2718
-
if (dio->op == REQ_OP_READ && likely(*status == BLK_STS_OK)) {
2719
-
unsigned pos = 0;
2667
+
if (dio->op == REQ_OP_READ && likely(*status == BLK_STS_OK) && likely(dio->bio_details.bi_iter.bi_size != 0)) {
2720
2668
if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
2721
2669
unlikely(dio->integrity_range_locked))
2722
-
goto skip_check;
2723
-
while (dio->bio_details.bi_iter.bi_size) {
2724
-
char digest[HASH_MAX_DIGESTSIZE];
2725
-
struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);
2726
-
char *mem = integrity_kmap(ic, bv.bv_page);
2727
-
integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest);
2728
-
if (unlikely(crypto_memneq(digest, dio->integrity_payload + pos,
2729
-
min(ic->internal_hash_digestsize, ic->tag_size)))) {
2730
-
integrity_kunmap(ic, mem);
2731
-
dm_integrity_free_payload(dio);
2732
-
INIT_WORK(&dio->work, dm_integrity_inline_recheck);
2733
-
queue_work(ic->offload_wq, &dio->work);
2670
+
goto skip_check;
2671
+
if (likely(ic->internal_shash != NULL)) {
2672
+
if (unlikely(!dm_integrity_check(ic, dio)))
2734
2673
return DM_ENDIO_INCOMPLETE;
2735
-
}
2736
-
integrity_kunmap(ic, mem);
2737
-
pos += ic->tuple_size;
2738
-
bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);
2674
+
} else {
2675
+
INIT_WORK(&dio->work, dm_integrity_inline_async_check);
2676
+
queue_work(ic->offload_wq, &dio->work);
2677
+
return DM_ENDIO_INCOMPLETE;
2739
2678
}
2740
2679
}
2741
2680
skip_check:
···
2769
2656
if (unlikely(dio->integrity_range_locked))
2770
2657
remove_range(ic, &dio->range);
2771
2658
}
2659
+
if (unlikely(dio->ahash_req))
2660
+
mempool_free(dio->ahash_req, &ic->ahash_req_pool);
2772
2661
return DM_ENDIO_DONE;
2773
2662
}
2774
2663
···
3031
2916
void *js_page = integrity_identity(ic, (char *)js - offset_in_page(js));
3032
2917
unsigned js_offset = offset_in_page(js);
3033
2918
3034
-
integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block),
2919
+
integrity_sector_checksum(ic, &ic->journal_ahash_req, sec + ((l - j) << ic->sb->log2_sectors_per_block),
3035
2920
js_page, js_offset, test_tag);
3036
2921
if (unlikely(crypto_memneq(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) {
3037
2922
dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ);
···
3115
3000
size_t recalc_tags_size;
3116
3001
u8 *recalc_buffer = NULL;
3117
3002
u8 *recalc_tags = NULL;
3003
+
struct ahash_request *ahash_req = NULL;
3118
3004
struct dm_integrity_range range;
3119
3005
struct dm_io_request io_req;
3120
3006
struct dm_io_region io_loc;
···
3229
3113
void *ptr = recalc_buffer + (i << SECTOR_SHIFT);
3230
3114
void *ptr_page = integrity_identity(ic, (char *)ptr - offset_in_page(ptr));
3231
3115
unsigned ptr_offset = offset_in_page(ptr);
3232
-
integrity_sector_checksum(ic, logical_sector + i, ptr_page, ptr_offset, t);
3116
+
integrity_sector_checksum(ic, &ahash_req, logical_sector + i, ptr_page, ptr_offset, t);
3233
3117
t += ic->tag_size;
3234
3118
}
3235
3119
···
3273
3157
free_ret:
3274
3158
kfree(recalc_buffer);
3275
3159
kvfree(recalc_tags);
3160
+
mempool_free(ahash_req, &ic->ahash_req_pool);
3276
3161
}
3277
3162
3278
3163
static void integrity_recalc_inline(struct work_struct *w)
···
3282
3165
size_t recalc_tags_size;
3283
3166
u8 *recalc_buffer = NULL;
3284
3167
u8 *recalc_tags = NULL;
3168
+
struct ahash_request *ahash_req = NULL;
3285
3169
struct dm_integrity_range range;
3286
3170
struct bio *bio;
3287
3171
struct bio_integrity_payload *bip;
···
3355
3237
void *ptr_page = integrity_identity(ic, (char *)ptr - offset_in_page(ptr));
3356
3238
unsigned ptr_offset = offset_in_page(ptr);
3357
3239
memset(t, 0, ic->tuple_size);
3358
-
integrity_sector_checksum(ic, range.logical_sector + i, ptr_page, ptr_offset, t);
3240
+
integrity_sector_checksum(ic, &ahash_req, range.logical_sector + i, ptr_page, ptr_offset, t);
3359
3241
t += ic->tuple_size;
3360
3242
}
3361
3243
···
3407
3289
free_ret:
3408
3290
kfree(recalc_buffer);
3409
3291
kfree(recalc_tags);
3292
+
mempool_free(ahash_req, &ic->ahash_req_pool);
3410
3293
}
3411
3294
3412
3295
static void bitmap_block_work(struct work_struct *w)
···
4348
4229
return -ENOMEM;
4349
4230
}
4350
4231
4351
-
static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error,
4352
-
char *error_alg, char *error_key)
4232
+
static int get_mac(struct crypto_shash **shash, struct crypto_ahash **ahash,
4233
+
struct alg_spec *a, char **error, char *error_alg, char *error_key)
4353
4234
{
4354
4235
int r;
4355
4236
4356
4237
if (a->alg_string) {
4357
-
*hash = crypto_alloc_shash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
4358
-
if (IS_ERR(*hash)) {
4238
+
if (ahash) {
4239
+
*ahash = crypto_alloc_ahash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
4240
+
if (IS_ERR(*ahash)) {
4241
+
*ahash = NULL;
4242
+
goto try_shash;
4243
+
}
4244
+
4245
+
if (a->key) {
4246
+
r = crypto_ahash_setkey(*ahash, a->key, a->key_size);
4247
+
if (r) {
4248
+
*error = error_key;
4249
+
return r;
4250
+
}
4251
+
} else if (crypto_ahash_get_flags(*ahash) & CRYPTO_TFM_NEED_KEY) {
4252
+
*error = error_key;
4253
+
return -ENOKEY;
4254
+
}
4255
+
4256
+
return 0;
4257
+
}
4258
+
4259
+
try_shash:
4260
+
*shash = crypto_alloc_shash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
4261
+
if (IS_ERR(*shash)) {
4359
4262
*error = error_alg;
4360
-
r = PTR_ERR(*hash);
4361
-
*hash = NULL;
4263
+
r = PTR_ERR(*shash);
4264
+
*shash = NULL;
4362
4265
return r;
4363
4266
}
4364
4267
4365
4268
if (a->key) {
4366
-
r = crypto_shash_setkey(*hash, a->key, a->key_size);
4269
+
r = crypto_shash_setkey(*shash, a->key, a->key_size);
4367
4270
if (r) {
4368
4271
*error = error_key;
4369
4272
return r;
4370
4273
}
4371
-
} else if (crypto_shash_get_flags(*hash) & CRYPTO_TFM_NEED_KEY) {
4274
+
} else if (crypto_shash_get_flags(*shash) & CRYPTO_TFM_NEED_KEY) {
4372
4275
*error = error_key;
4373
4276
return -ENOKEY;
4374
4277
}
···
4850
4709
buffer_sectors = 1;
4851
4710
ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT);
4852
4711
4853
-
r = get_mac(&ic->internal_shash, &ic->internal_hash_alg, &ti->error,
4712
+
r = get_mac(&ic->internal_shash, &ic->internal_ahash, &ic->internal_hash_alg, &ti->error,
4854
4713
"Invalid internal hash", "Error setting internal hash key");
4855
4714
if (r)
4856
4715
goto bad;
···
4858
4717
ic->internal_hash = true;
4859
4718
ic->internal_hash_digestsize = crypto_shash_digestsize(ic->internal_shash);
4860
4719
}
4720
+
if (ic->internal_ahash) {
4721
+
ic->internal_hash = true;
4722
+
ic->internal_hash_digestsize = crypto_ahash_digestsize(ic->internal_ahash);
4723
+
r = mempool_init_kmalloc_pool(&ic->ahash_req_pool, AHASH_MEMPOOL,
4724
+
sizeof(struct ahash_request) + crypto_ahash_reqsize(ic->internal_ahash));
4725
+
if (r) {
4726
+
ti->error = "Cannot allocate mempool";
4727
+
goto bad;
4728
+
}
4729
+
}
4861
4730
4862
-
r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error,
4731
+
r = get_mac(&ic->journal_mac, NULL, &ic->journal_mac_alg, &ti->error,
4863
4732
"Invalid journal mac", "Error setting journal mac key");
4864
4733
if (r)
4865
4734
goto bad;
···
5352
5201
kvfree(ic->bbs);
5353
5202
if (ic->bufio)
5354
5203
dm_bufio_client_destroy(ic->bufio);
5204
+
mempool_free(ic->journal_ahash_req, &ic->ahash_req_pool);
5205
+
mempool_exit(&ic->ahash_req_pool);
5355
5206
bioset_exit(&ic->recalc_bios);
5356
5207
bioset_exit(&ic->recheck_bios);
5357
5208
mempool_exit(&ic->recheck_pool);
···
5393
5240
5394
5241
if (ic->internal_shash)
5395
5242
crypto_free_shash(ic->internal_shash);
5243
+
if (ic->internal_ahash)
5244
+
crypto_free_ahash(ic->internal_ahash);
5396
5245
free_alg(&ic->internal_hash_alg);
5397
5246
5398
5247
if (ic->journal_crypt)
···
5411
5256
5412
5257
static struct target_type integrity_target = {
5413
5258
.name = "integrity",
5414
-
.version = {1, 13, 0},
5259
+
.version = {1, 14, 0},
5415
5260
.module = THIS_MODULE,
5416
5261
.features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
5417
5262
.ctr = dm_integrity_ctr,