Merge tag 'for-7.0/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

+1 -1

Documentation/admin-guide/device-mapper/dm-raid.rst

··· 433 433 8192 1960886272 linear 8:0 0 2048 # previous data segment 434 434 435 435 # Mapping table for e.g. raid5_rs reshape causing the size of the raid device to double-fold once the reshape finishes. 436 - # Check the status output (e.g. "dmsetup status $RaidDev") for progess. 436 + # Check the status output (e.g. "dmsetup status $RaidDev") for progress. 437 437 438 438 0 $((2 * 1960886272)) raid raid5 7 0 region_size 2048 data_offset 8192 delta_disk 1 2 /dev/dm-0 /dev/dm-1 /dev/dm-2 /dev/dm-3 439 439

+7

Documentation/admin-guide/kernel-parameters.txt

··· 1372 1372 For details see: 1373 1373 Documentation/admin-guide/hw-vuln/reg-file-data-sampling.rst 1374 1374 1375 + dm_verity.keyring_unsealed= 1376 + [KNL] When set to 1, leave the dm-verity keyring 1377 + unsealed after initialization so userspace can 1378 + provision keys. Once the keyring is restricted 1379 + it becomes active and is searched during signature 1380 + verification. 1381 + 1375 1382 driver_async_probe= [KNL] 1376 1383 List of driver names to be probed asynchronously. * 1377 1384 matches with all driver names. If * is specified, the

+2 -2

drivers/md/Kconfig

··· 549 549 select CRYPTO_HASH 550 550 select CRYPTO_LIB_SHA256 551 551 select DM_BUFIO 552 + select REED_SOLOMON if DM_VERITY_FEC 553 + select REED_SOLOMON_DEC8 if DM_VERITY_FEC 552 554 help 553 555 This device-mapper target creates a read-only device that 554 556 transparently validates the data on one underlying device against ··· 600 598 config DM_VERITY_FEC 601 599 bool "Verity forward error correction support" 602 600 depends on DM_VERITY 603 - select REED_SOLOMON 604 - select REED_SOLOMON_DEC8 605 601 help 606 602 Add forward error correction support to dm-verity. This option 607 603 makes it possible to use pre-generated error correction data to

+102 -77

drivers/md/dm-bufio.c

··· 369 369 * - IO 370 370 * - Eviction or cache sizing. 371 371 * 372 - * cache_get() and cache_put() are threadsafe, you do not need to 373 - * protect these calls with a surrounding mutex. All the other 372 + * cache_get() and cache_put_and_wake() are threadsafe, you do not need 373 + * to protect these calls with a surrounding mutex. All the other 374 374 * methods are not threadsafe; they do use locking primitives, but 375 375 * only enough to ensure get/put are threadsafe. 376 376 */ ··· 401 401 return dm_hash_locks_index(block, num_locks); 402 402 } 403 403 404 - static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t block) 404 + /* Get the buffer tree in the cache for the given block. Doesn't lock it. */ 405 + static inline struct buffer_tree *cache_get_tree(struct dm_buffer_cache *bc, 406 + sector_t block) 405 407 { 406 - if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 407 - read_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 408 - else 409 - down_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 408 + return &bc->trees[cache_index(block, bc->num_locks)]; 410 409 } 411 410 412 - static inline void cache_read_unlock(struct dm_buffer_cache *bc, sector_t block) 411 + /* Lock the given buffer tree in the cache for reading. */ 412 + static inline void cache_read_lock(struct dm_buffer_cache *bc, 413 + struct buffer_tree *tree) 413 414 { 414 415 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 415 - read_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 416 + read_lock_bh(&tree->u.spinlock); 416 417 else 417 - up_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 418 + down_read(&tree->u.lock); 418 419 } 419 420 420 - static inline void cache_write_lock(struct dm_buffer_cache *bc, sector_t block) 421 + /* Unlock the given buffer tree in the cache for reading. */ 422 + static inline void cache_read_unlock(struct dm_buffer_cache *bc, 423 + struct buffer_tree *tree) 421 424 { 422 425 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 423 - write_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 426 + read_unlock_bh(&tree->u.spinlock); 424 427 else 425 - down_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 428 + up_read(&tree->u.lock); 426 429 } 427 430 428 - static inline void cache_write_unlock(struct dm_buffer_cache *bc, sector_t block) 431 + /* Lock the given buffer tree in the cache for writing. */ 432 + static inline void cache_write_lock(struct dm_buffer_cache *bc, 433 + struct buffer_tree *tree) 429 434 { 430 435 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 431 - write_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 436 + write_lock_bh(&tree->u.spinlock); 432 437 else 433 - up_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 438 + down_write(&tree->u.lock); 439 + } 440 + 441 + /* Unlock the given buffer tree in the cache for writing. */ 442 + static inline void cache_write_unlock(struct dm_buffer_cache *bc, 443 + struct buffer_tree *tree) 444 + { 445 + if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 446 + write_unlock_bh(&tree->u.spinlock); 447 + else 448 + up_write(&tree->u.lock); 434 449 } 435 450 436 451 /* ··· 617 602 WRITE_ONCE(b->last_accessed, jiffies); 618 603 } 619 604 620 - static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, sector_t block) 605 + static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, 606 + struct buffer_tree *tree, sector_t block) 621 607 { 622 608 struct dm_buffer *b; 623 609 624 - cache_read_lock(bc, block); 625 - b = __cache_get(&bc->trees[cache_index(block, bc->num_locks)].root, block); 610 + /* Assuming tree == cache_get_tree(bc, block) */ 611 + cache_read_lock(bc, tree); 612 + b = __cache_get(&tree->root, block); 626 613 if (b) { 627 614 lru_reference(&b->lru); 628 615 __cache_inc_buffer(b); 629 616 } 630 - cache_read_unlock(bc, block); 617 + cache_read_unlock(bc, tree); 631 618 632 619 return b; 633 - } 634 - 635 - /*--------------*/ 636 - 637 - /* 638 - * Returns true if the hold count hits zero. 639 - * threadsafe 640 - */ 641 - static bool cache_put(struct dm_buffer_cache *bc, struct dm_buffer *b) 642 - { 643 - bool r; 644 - 645 - cache_read_lock(bc, b->block); 646 - BUG_ON(!atomic_read(&b->hold_count)); 647 - r = atomic_dec_and_test(&b->hold_count); 648 - cache_read_unlock(bc, b->block); 649 - 650 - return r; 651 620 } 652 621 653 622 /*--------------*/ ··· 680 681 681 682 b = le_to_buffer(le); 682 683 /* __evict_pred will have locked the appropriate tree. */ 683 - rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); 684 + rb_erase(&b->node, &cache_get_tree(bc, b->block)->root); 684 685 685 686 return b; 686 687 } ··· 703 704 /* 704 705 * Mark a buffer as clean or dirty. Not threadsafe. 705 706 */ 706 - static void cache_mark(struct dm_buffer_cache *bc, struct dm_buffer *b, int list_mode) 707 + static void cache_mark(struct dm_buffer_cache *bc, struct buffer_tree *tree, 708 + struct dm_buffer *b, int list_mode) 707 709 { 708 - cache_write_lock(bc, b->block); 710 + /* Assuming tree == cache_get_tree(bc, b->block) */ 711 + cache_write_lock(bc, tree); 709 712 if (list_mode != b->list_mode) { 710 713 lru_remove(&bc->lru[b->list_mode], &b->lru); 711 714 b->list_mode = list_mode; 712 715 lru_insert(&bc->lru[b->list_mode], &b->lru); 713 716 } 714 - cache_write_unlock(bc, b->block); 717 + cache_write_unlock(bc, tree); 715 718 } 716 719 717 720 /*--------------*/ ··· 839 838 return true; 840 839 } 841 840 842 - static bool cache_insert(struct dm_buffer_cache *bc, struct dm_buffer *b) 841 + static bool cache_insert(struct dm_buffer_cache *bc, struct buffer_tree *tree, 842 + struct dm_buffer *b) 843 843 { 844 844 bool r; 845 845 846 846 if (WARN_ON_ONCE(b->list_mode >= LIST_SIZE)) 847 847 return false; 848 848 849 - cache_write_lock(bc, b->block); 849 + /* Assuming tree == cache_get_tree(bc, b->block) */ 850 + cache_write_lock(bc, tree); 850 851 BUG_ON(atomic_read(&b->hold_count) != 1); 851 - r = __cache_insert(&bc->trees[cache_index(b->block, bc->num_locks)].root, b); 852 + r = __cache_insert(&tree->root, b); 852 853 if (r) 853 854 lru_insert(&bc->lru[b->list_mode], &b->lru); 854 - cache_write_unlock(bc, b->block); 855 + cache_write_unlock(bc, tree); 855 856 856 857 return r; 857 858 } ··· 866 863 * 867 864 * Not threadsafe. 868 865 */ 869 - static bool cache_remove(struct dm_buffer_cache *bc, struct dm_buffer *b) 866 + static bool cache_remove(struct dm_buffer_cache *bc, struct buffer_tree *tree, 867 + struct dm_buffer *b) 870 868 { 871 869 bool r; 872 870 873 - cache_write_lock(bc, b->block); 871 + /* Assuming tree == cache_get_tree(bc, b->block) */ 872 + cache_write_lock(bc, tree); 874 873 875 874 if (atomic_read(&b->hold_count) != 1) { 876 875 r = false; 877 876 } else { 878 877 r = true; 879 - rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); 878 + rb_erase(&b->node, &tree->root); 880 879 lru_remove(&bc->lru[b->list_mode], &b->lru); 881 880 } 882 881 883 - cache_write_unlock(bc, b->block); 882 + cache_write_unlock(bc, tree); 884 883 885 884 return r; 886 885 } ··· 1748 1743 *-------------------------------------------------------------- 1749 1744 */ 1750 1745 1751 - static void cache_put_and_wake(struct dm_bufio_client *c, struct dm_buffer *b) 1746 + static void cache_put_and_wake(struct dm_bufio_client *c, 1747 + struct buffer_tree *tree, struct dm_buffer *b) 1752 1748 { 1749 + bool wake; 1750 + 1751 + /* Assuming tree == cache_get_tree(&c->cache, b->block) */ 1752 + cache_read_lock(&c->cache, tree); 1753 + BUG_ON(!atomic_read(&b->hold_count)); 1754 + wake = atomic_dec_and_test(&b->hold_count); 1755 + cache_read_unlock(&c->cache, tree); 1756 + 1753 1757 /* 1754 1758 * Relying on waitqueue_active() is racey, but we sleep 1755 1759 * with schedule_timeout anyway. 1756 1760 */ 1757 - if (cache_put(&c->cache, b) && 1758 - unlikely(waitqueue_active(&c->free_buffer_wait))) 1761 + if (wake && unlikely(waitqueue_active(&c->free_buffer_wait))) 1759 1762 wake_up(&c->free_buffer_wait); 1760 1763 } 1761 1764 ··· 1771 1758 * This assumes you have already checked the cache to see if the buffer 1772 1759 * is already present (it will recheck after dropping the lock for allocation). 1773 1760 */ 1774 - static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block, 1761 + static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, 1762 + struct buffer_tree *tree, sector_t block, 1775 1763 enum new_flag nf, int *need_submit, 1776 1764 struct list_head *write_list) 1777 1765 { ··· 1792 1778 * We've had a period where the mutex was unlocked, so need to 1793 1779 * recheck the buffer tree. 1794 1780 */ 1795 - b = cache_get(&c->cache, block); 1781 + b = cache_get(&c->cache, tree, block); 1796 1782 if (b) { 1797 1783 __free_buffer_wake(new_b); 1798 1784 goto found_buffer; ··· 1820 1806 * is set. Otherwise another thread could get it and use 1821 1807 * it before it had been read. 1822 1808 */ 1823 - cache_insert(&c->cache, b); 1809 + cache_insert(&c->cache, tree, b); 1824 1810 1825 1811 return b; 1826 1812 1827 1813 found_buffer: 1828 1814 if (nf == NF_PREFETCH) { 1829 - cache_put_and_wake(c, b); 1815 + cache_put_and_wake(c, tree, b); 1830 1816 return NULL; 1831 1817 } 1832 1818 ··· 1838 1824 * the same buffer, it would deadlock if we waited. 1839 1825 */ 1840 1826 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { 1841 - cache_put_and_wake(c, b); 1827 + cache_put_and_wake(c, tree, b); 1842 1828 return NULL; 1843 1829 } 1844 1830 ··· 1872 1858 enum new_flag nf, struct dm_buffer **bp, 1873 1859 unsigned short ioprio) 1874 1860 { 1861 + struct buffer_tree *tree; 1875 1862 int need_submit = 0; 1876 1863 struct dm_buffer *b; 1877 1864 ··· 1884 1869 * Fast path, hopefully the block is already in the cache. No need 1885 1870 * to get the client lock for this. 1886 1871 */ 1887 - b = cache_get(&c->cache, block); 1872 + tree = cache_get_tree(&c->cache, block); 1873 + b = cache_get(&c->cache, tree, block); 1888 1874 if (b) { 1889 1875 if (nf == NF_PREFETCH) { 1890 - cache_put_and_wake(c, b); 1876 + cache_put_and_wake(c, tree, b); 1891 1877 return NULL; 1892 1878 } 1893 1879 ··· 1900 1884 * the same buffer, it would deadlock if we waited. 1901 1885 */ 1902 1886 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { 1903 - cache_put_and_wake(c, b); 1887 + cache_put_and_wake(c, tree, b); 1904 1888 return NULL; 1905 1889 } 1906 1890 } ··· 1910 1894 return NULL; 1911 1895 1912 1896 dm_bufio_lock(c); 1913 - b = __bufio_new(c, block, nf, &need_submit, &write_list); 1897 + b = __bufio_new(c, tree, block, nf, &need_submit, &write_list); 1914 1898 dm_bufio_unlock(c); 1915 1899 } 1916 1900 ··· 1997 1981 blk_start_plug(&plug); 1998 1982 1999 1983 for (; n_blocks--; block++) { 2000 - int need_submit; 1984 + struct buffer_tree *tree; 2001 1985 struct dm_buffer *b; 1986 + int need_submit; 2002 1987 2003 - b = cache_get(&c->cache, block); 1988 + tree = cache_get_tree(&c->cache, block); 1989 + b = cache_get(&c->cache, tree, block); 2004 1990 if (b) { 2005 1991 /* already in cache */ 2006 - cache_put_and_wake(c, b); 1992 + cache_put_and_wake(c, tree, b); 2007 1993 continue; 2008 1994 } 2009 1995 2010 1996 dm_bufio_lock(c); 2011 - b = __bufio_new(c, block, NF_PREFETCH, &need_submit, 1997 + b = __bufio_new(c, tree, block, NF_PREFETCH, &need_submit, 2012 1998 &write_list); 2013 1999 if (unlikely(!list_empty(&write_list))) { 2014 2000 dm_bufio_unlock(c); ··· 2055 2037 void dm_bufio_release(struct dm_buffer *b) 2056 2038 { 2057 2039 struct dm_bufio_client *c = b->c; 2040 + struct buffer_tree *tree = cache_get_tree(&c->cache, b->block); 2058 2041 2059 2042 /* 2060 2043 * If there were errors on the buffer, and the buffer is not ··· 2069 2050 dm_bufio_lock(c); 2070 2051 2071 2052 /* cache remove can fail if there are other holders */ 2072 - if (cache_remove(&c->cache, b)) { 2053 + if (cache_remove(&c->cache, tree, b)) { 2073 2054 __free_buffer_wake(b); 2074 2055 dm_bufio_unlock(c); 2075 2056 return; ··· 2078 2059 dm_bufio_unlock(c); 2079 2060 } 2080 2061 2081 - cache_put_and_wake(c, b); 2062 + cache_put_and_wake(c, tree, b); 2082 2063 } 2083 2064 EXPORT_SYMBOL_GPL(dm_bufio_release); 2084 2065 ··· 2097 2078 if (!test_and_set_bit(B_DIRTY, &b->state)) { 2098 2079 b->dirty_start = start; 2099 2080 b->dirty_end = end; 2100 - cache_mark(&c->cache, b, LIST_DIRTY); 2081 + cache_mark(&c->cache, cache_get_tree(&c->cache, b->block), b, 2082 + LIST_DIRTY); 2101 2083 } else { 2102 2084 if (start < b->dirty_start) 2103 2085 b->dirty_start = start; ··· 2163 2143 lru_iter_begin(&c->cache.lru[LIST_DIRTY], &it); 2164 2144 while ((e = lru_iter_next(&it, is_writing, c))) { 2165 2145 struct dm_buffer *b = le_to_buffer(e); 2146 + struct buffer_tree *tree; 2166 2147 __cache_inc_buffer(b); 2167 2148 2168 2149 BUG_ON(test_bit(B_READING, &b->state)); ··· 2177 2156 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); 2178 2157 } 2179 2158 2180 - if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state)) 2181 - cache_mark(&c->cache, b, LIST_CLEAN); 2159 + tree = cache_get_tree(&c->cache, b->block); 2182 2160 2183 - cache_put_and_wake(c, b); 2161 + if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state)) 2162 + cache_mark(&c->cache, tree, b, LIST_CLEAN); 2163 + 2164 + cache_put_and_wake(c, tree, b); 2184 2165 2185 2166 cond_resched(); 2186 2167 } ··· 2250 2227 2251 2228 static void forget_buffer(struct dm_bufio_client *c, sector_t block) 2252 2229 { 2230 + struct buffer_tree *tree = cache_get_tree(&c->cache, block); 2253 2231 struct dm_buffer *b; 2254 2232 2255 - b = cache_get(&c->cache, block); 2233 + b = cache_get(&c->cache, tree, block); 2256 2234 if (b) { 2257 2235 if (likely(!smp_load_acquire(&b->state))) { 2258 - if (cache_remove(&c->cache, b)) 2236 + if (cache_remove(&c->cache, tree, b)) 2259 2237 __free_buffer_wake(b); 2260 2238 else 2261 - cache_put_and_wake(c, b); 2239 + cache_put_and_wake(c, tree, b); 2262 2240 } else { 2263 - cache_put_and_wake(c, b); 2241 + cache_put_and_wake(c, tree, b); 2264 2242 } 2265 2243 } 2266 2244 } ··· 2857 2833 __cache_size_refresh(); 2858 2834 mutex_unlock(&dm_bufio_clients_lock); 2859 2835 2860 - dm_bufio_wq = alloc_workqueue("dm_bufio_cache", WQ_MEM_RECLAIM, 0); 2836 + dm_bufio_wq = alloc_workqueue("dm_bufio_cache", 2837 + WQ_MEM_RECLAIM | WQ_PERCPU, 0); 2861 2838 if (!dm_bufio_wq) 2862 2839 return -ENOMEM; 2863 2840

+2 -8

drivers/md/dm-cache-target.c

··· 2099 2099 static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as, 2100 2100 char **error) 2101 2101 { 2102 - sector_t origin_sectors; 2103 2102 int r; 2104 2103 2105 2104 if (!at_least_one_arg(as, error)) ··· 2109 2110 if (r) { 2110 2111 *error = "Error opening origin device"; 2111 2112 return r; 2112 - } 2113 - 2114 - origin_sectors = get_dev_size(ca->origin_dev); 2115 - if (ca->ti->len > origin_sectors) { 2116 - *error = "Device size larger than cached device"; 2117 - return -EINVAL; 2118 2113 } 2119 2114 2120 2115 return 0; ··· 2526 2533 goto bad; 2527 2534 } 2528 2535 2529 - cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0); 2536 + cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, 2537 + WQ_MEM_RECLAIM | WQ_PERCPU, 0); 2530 2538 if (!cache->wq) { 2531 2539 *error = "could not create workqueue for metadata object"; 2532 2540 goto bad;

+2 -17

drivers/md/dm-clone-target.c

··· 1697 1697 static int parse_dest_dev(struct clone *clone, struct dm_arg_set *as, char **error) 1698 1698 { 1699 1699 int r; 1700 - sector_t dest_dev_size; 1701 1700 1702 1701 r = dm_get_device(clone->ti, dm_shift_arg(as), 1703 1702 BLK_OPEN_READ | BLK_OPEN_WRITE, &clone->dest_dev); ··· 1705 1706 return r; 1706 1707 } 1707 1708 1708 - dest_dev_size = get_dev_size(clone->dest_dev); 1709 - if (dest_dev_size < clone->ti->len) { 1710 - dm_put_device(clone->ti, clone->dest_dev); 1711 - *error = "Device size larger than destination device"; 1712 - return -EINVAL; 1713 - } 1714 - 1715 1709 return 0; 1716 1710 } 1717 1711 1718 1712 static int parse_source_dev(struct clone *clone, struct dm_arg_set *as, char **error) 1719 1713 { 1720 1714 int r; 1721 - sector_t source_dev_size; 1722 1715 1723 1716 r = dm_get_device(clone->ti, dm_shift_arg(as), BLK_OPEN_READ, 1724 1717 &clone->source_dev); 1725 1718 if (r) { 1726 1719 *error = "Error opening source device"; 1727 1720 return r; 1728 - } 1729 - 1730 - source_dev_size = get_dev_size(clone->source_dev); 1731 - if (source_dev_size < clone->ti->len) { 1732 - dm_put_device(clone->ti, clone->source_dev); 1733 - *error = "Device size larger than source device"; 1734 - return -EINVAL; 1735 1721 } 1736 1722 1737 1723 return 0; ··· 1861 1877 clone->hydration_offset = 0; 1862 1878 atomic_set(&clone->hydrations_in_flight, 0); 1863 1879 1864 - clone->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0); 1880 + clone->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, 1881 + WQ_MEM_RECLAIM | WQ_PERCPU, 0); 1865 1882 if (!clone->wq) { 1866 1883 ti->error = "Failed to allocate workqueue"; 1867 1884 r = -ENOMEM;

-1

drivers/md/dm-core.h

··· 215 215 216 216 /* a list of devices used by this table */ 217 217 struct list_head devices; 218 - struct rw_semaphore devices_lock; 219 218 220 219 /* events get handed up using this callback */ 221 220 void (*event_fn)(void *data);

+4 -2

drivers/md/dm-crypt.c

··· 3400 3400 if (test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags)) 3401 3401 common_wq_flags |= WQ_HIGHPRI; 3402 3402 3403 - cc->io_queue = alloc_workqueue("kcryptd_io-%s-%d", common_wq_flags, 1, devname, wq_id); 3403 + cc->io_queue = alloc_workqueue("kcryptd_io-%s-%d", 3404 + common_wq_flags | WQ_PERCPU, 1, 3405 + devname, wq_id); 3404 3406 if (!cc->io_queue) { 3405 3407 ti->error = "Couldn't create kcryptd io queue"; 3406 3408 goto bad; ··· 3410 3408 3411 3409 if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) { 3412 3410 cc->crypt_queue = alloc_workqueue("kcryptd-%s-%d", 3413 - common_wq_flags | WQ_CPU_INTENSIVE, 3411 + common_wq_flags | WQ_CPU_INTENSIVE | WQ_PERCPU, 3414 3412 1, devname, wq_id); 3415 3413 } else { 3416 3414 /*

+3 -1

drivers/md/dm-delay.c

··· 290 290 } else { 291 291 timer_setup(&dc->delay_timer, handle_delayed_timer, 0); 292 292 INIT_WORK(&dc->flush_expired_bios, flush_expired_bios); 293 - dc->kdelayd_wq = alloc_workqueue("kdelayd", WQ_MEM_RECLAIM, 0); 293 + dc->kdelayd_wq = alloc_workqueue("kdelayd", 294 + WQ_MEM_RECLAIM | WQ_PERCPU, 295 + 0); 294 296 if (!dc->kdelayd_wq) { 295 297 ret = -EINVAL; 296 298 DMERR("Couldn't start kdelayd");

+1 -1

drivers/md/dm-exception-store.c

··· 116 116 if (!__find_exception_store_type(type->name)) 117 117 list_add(&type->list, &_exception_store_types); 118 118 else 119 - r = -EEXIST; 119 + r = -EBUSY; 120 120 spin_unlock(&_lock); 121 121 122 122 return r;

+24 -6

drivers/md/dm-integrity.c

··· 2411 2411 2412 2412 new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); 2413 2413 if (unlikely(new_pos != NOT_FOUND) || 2414 - unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) { 2414 + unlikely(next_sector < dio->range.logical_sector + dio->range.n_sectors)) { 2415 2415 remove_range_unlocked(ic, &dio->range); 2416 2416 spin_unlock_irq(&ic->endio_wait.lock); 2417 2417 queue_work(ic->commit_wq, &ic->commit_work); ··· 3788 3788 struct dm_integrity_c *ic = ti->private; 3789 3789 __u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors); 3790 3790 int r; 3791 + __le32 flags; 3791 3792 3792 3793 DEBUG_print("resume\n"); 3793 3794 3794 3795 ic->wrote_to_journal = false; 3795 3796 3797 + flags = ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING); 3798 + r = sync_rw_sb(ic, REQ_OP_READ); 3799 + if (r) 3800 + dm_integrity_io_error(ic, "reading superblock", r); 3801 + if ((ic->sb->flags & flags) != flags) { 3802 + ic->sb->flags |= flags; 3803 + r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); 3804 + if (unlikely(r)) 3805 + dm_integrity_io_error(ic, "writing superblock", r); 3806 + } 3807 + 3796 3808 if (ic->provided_data_sectors != old_provided_data_sectors) { 3797 3809 if (ic->provided_data_sectors > old_provided_data_sectors && 3798 3810 ic->mode == 'B' && 3811 + ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP) && 3799 3812 ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) { 3800 3813 rw_journal_sectors(ic, REQ_OP_READ, 0, 3801 3814 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); ··· 5003 4990 } 5004 4991 5005 4992 ic->metadata_wq = alloc_workqueue("dm-integrity-metadata", 5006 - WQ_MEM_RECLAIM, METADATA_WORKQUEUE_MAX_ACTIVE); 4993 + WQ_MEM_RECLAIM | WQ_PERCPU, 4994 + METADATA_WORKQUEUE_MAX_ACTIVE); 5007 4995 if (!ic->metadata_wq) { 5008 4996 ti->error = "Cannot allocate workqueue"; 5009 4997 r = -ENOMEM; ··· 5022 5008 goto bad; 5023 5009 } 5024 5010 5025 - ic->offload_wq = alloc_workqueue("dm-integrity-offload", WQ_MEM_RECLAIM, 5011 + ic->offload_wq = alloc_workqueue("dm-integrity-offload", 5012 + WQ_MEM_RECLAIM | WQ_PERCPU, 5026 5013 METADATA_WORKQUEUE_MAX_ACTIVE); 5027 5014 if (!ic->offload_wq) { 5028 5015 ti->error = "Cannot allocate workqueue"; ··· 5031 5016 goto bad; 5032 5017 } 5033 5018 5034 - ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1); 5019 + ic->commit_wq = alloc_workqueue("dm-integrity-commit", 5020 + WQ_MEM_RECLAIM | WQ_PERCPU, 1); 5035 5021 if (!ic->commit_wq) { 5036 5022 ti->error = "Cannot allocate workqueue"; 5037 5023 r = -ENOMEM; ··· 5041 5025 INIT_WORK(&ic->commit_work, integrity_commit); 5042 5026 5043 5027 if (ic->mode == 'J' || ic->mode == 'B') { 5044 - ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1); 5028 + ic->writer_wq = alloc_workqueue("dm-integrity-writer", 5029 + WQ_MEM_RECLAIM | WQ_PERCPU, 1); 5045 5030 if (!ic->writer_wq) { 5046 5031 ti->error = "Cannot allocate workqueue"; 5047 5032 r = -ENOMEM; ··· 5214 5197 } 5215 5198 5216 5199 if (ic->internal_hash) { 5217 - ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", WQ_MEM_RECLAIM, 1); 5200 + ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", 5201 + WQ_MEM_RECLAIM | WQ_PERCPU, 1); 5218 5202 if (!ic->recalc_wq) { 5219 5203 ti->error = "Cannot allocate workqueue"; 5220 5204 r = -ENOMEM;

+1 -6

drivers/md/dm-ioctl.c

··· 1648 1648 struct dm_dev_internal *dd; 1649 1649 struct dm_target_deps *deps; 1650 1650 1651 - down_read(&table->devices_lock); 1652 - 1653 1651 deps = get_result_buffer(param, param_size, &len); 1654 1652 1655 1653 /* ··· 1662 1664 needed = struct_size(deps, dev, count); 1663 1665 if (len < needed) { 1664 1666 param->flags |= DM_BUFFER_FULL_FLAG; 1665 - goto out; 1667 + return; 1666 1668 } 1667 1669 1668 1670 /* ··· 1674 1676 deps->dev[count++] = huge_encode_dev(dd->dm_dev->bdev->bd_dev); 1675 1677 1676 1678 param->data_size = param->data_start + needed; 1677 - 1678 - out: 1679 - up_read(&table->devices_lock); 1680 1679 } 1681 1680 1682 1681 static int table_deps(struct file *filp, struct dm_ioctl *param, size_t param_size)

+2 -1

drivers/md/dm-kcopyd.c

··· 934 934 goto bad_slab; 935 935 936 936 INIT_WORK(&kc->kcopyd_work, do_work); 937 - kc->kcopyd_wq = alloc_workqueue("kcopyd", WQ_MEM_RECLAIM, 0); 937 + kc->kcopyd_wq = alloc_workqueue("kcopyd", WQ_MEM_RECLAIM | WQ_PERCPU, 938 + 0); 938 939 if (!kc->kcopyd_wq) { 939 940 r = -ENOMEM; 940 941 goto bad_workqueue;

+2 -1

drivers/md/dm-log-userspace-base.c

··· 299 299 } 300 300 301 301 if (lc->integrated_flush) { 302 - lc->dmlog_wq = alloc_workqueue("dmlogd", WQ_MEM_RECLAIM, 0); 302 + lc->dmlog_wq = alloc_workqueue("dmlogd", 303 + WQ_MEM_RECLAIM | WQ_PERCPU, 0); 303 304 if (!lc->dmlog_wq) { 304 305 DMERR("couldn't start dmlogd"); 305 306 r = -ENOMEM;

+1 -1

drivers/md/dm-log.c

··· 121 121 if (!__find_dirty_log_type(type->name)) 122 122 list_add(&type->list, &_log_types); 123 123 else 124 - r = -EEXIST; 124 + r = -EBUSY; 125 125 spin_unlock(&_lock); 126 126 127 127 return r;

+14 -13

drivers/md/dm-mpath.c

··· 225 225 mutex_init(&m->work_mutex); 226 226 227 227 m->queue_mode = DM_TYPE_NONE; 228 + m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; 228 229 229 230 m->ti = ti; 230 231 ti->private = m; ··· 252 251 set_bit(MPATHF_QUEUE_IO, &m->flags); 253 252 atomic_set(&m->pg_init_in_progress, 0); 254 253 atomic_set(&m->pg_init_count, 0); 255 - m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; 256 254 init_waitqueue_head(&m->pg_init_wait); 257 255 init_waitqueue_head(&m->probe_wait); 258 256 ··· 960 960 attached_handler_name = NULL; 961 961 } else { 962 962 r = PTR_ERR(attached_handler_name); 963 - goto bad; 963 + ti->error = "error allocating handler name"; 964 + goto bad_put_device; 964 965 } 965 966 } 966 967 if (attached_handler_name || m->hw_handler_name) { 967 968 INIT_DELAYED_WORK(&p->activate_path, activate_path_work); 968 969 r = setup_scsi_dh(p->path.dev->bdev, m, &attached_handler_name, &ti->error); 969 970 kfree(attached_handler_name); 970 - if (r) { 971 - dm_put_device(ti, p->path.dev); 972 - goto bad; 973 - } 971 + if (r) 972 + goto bad_put_device; 974 973 } 975 974 976 975 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error); 977 - if (r) { 978 - dm_put_device(ti, p->path.dev); 979 - goto bad; 980 - } 976 + if (r) 977 + goto bad_put_device; 981 978 982 979 return p; 983 - bad: 980 + 981 + bad_put_device: 982 + dm_put_device(ti, p->path.dev); 983 + bad: 984 984 free_pgpath(p); 985 985 return ERR_PTR(r); 986 986 } ··· 2328 2328 { 2329 2329 int r = -ENOMEM; 2330 2330 2331 - kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0); 2331 + kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM | WQ_PERCPU, 2332 + 0); 2332 2333 if (!kmultipathd) { 2333 2334 DMERR("failed to create workqueue kmpathd"); 2334 2335 goto bad_alloc_kmultipathd; ··· 2348 2347 goto bad_alloc_kmpath_handlerd; 2349 2348 } 2350 2349 2351 - dm_mpath_wq = alloc_workqueue("dm_mpath_wq", 0, 0); 2350 + dm_mpath_wq = alloc_workqueue("dm_mpath_wq", WQ_PERCPU, 0); 2352 2351 if (!dm_mpath_wq) { 2353 2352 DMERR("failed to create workqueue dm_mpath_wq"); 2354 2353 goto bad_alloc_dm_mpath_wq;

+1 -1

drivers/md/dm-path-selector.c

··· 107 107 108 108 if (__find_path_selector_type(pst->name)) { 109 109 kfree(psi); 110 - r = -EEXIST; 110 + r = -EBUSY; 111 111 } else 112 112 list_add(&psi->list, &_path_selectors); 113 113

+3 -2

drivers/md/dm-raid1.c

··· 1128 1128 ti->num_discard_bios = 1; 1129 1129 ti->per_io_data_size = sizeof(struct dm_raid1_bio_record); 1130 1130 1131 - ms->kmirrord_wq = alloc_workqueue("kmirrord", WQ_MEM_RECLAIM, 0); 1131 + ms->kmirrord_wq = alloc_workqueue("kmirrord", 1132 + WQ_MEM_RECLAIM | WQ_PERCPU, 0); 1132 1133 if (!ms->kmirrord_wq) { 1133 1134 DMERR("couldn't start kmirrord"); 1134 1135 r = -ENOMEM; ··· 1501 1500 { 1502 1501 int r; 1503 1502 1504 - dm_raid1_wq = alloc_workqueue("dm_raid1_wq", 0, 0); 1503 + dm_raid1_wq = alloc_workqueue("dm_raid1_wq", WQ_PERCPU, 0); 1505 1504 if (!dm_raid1_wq) { 1506 1505 DMERR("Failed to alloc workqueue"); 1507 1506 return -ENOMEM;

+11 -5

drivers/md/dm-rq.c

··· 109 109 */ 110 110 tio->completed += nr_bytes; 111 111 112 + if (!is_last) 113 + return; 114 + /* 115 + * At this moment we know this is the last bio of the cloned request, 116 + * and all cloned bios have been released, so reset the clone request's 117 + * bio pointer to avoid double free. 118 + */ 119 + tio->clone->bio = NULL; 120 + exit: 112 121 /* 113 122 * Update the original request. 114 123 * Do not use blk_mq_end_request() here, because it may complete 115 124 * the original request before the clone, and break the ordering. 116 125 */ 117 - if (is_last) 118 - exit: 119 - blk_update_request(tio->orig, BLK_STS_OK, tio->completed); 126 + blk_update_request(tio->orig, BLK_STS_OK, tio->completed); 120 127 } 121 128 122 129 static struct dm_rq_target_io *tio_from_request(struct request *rq) ··· 285 278 struct dm_rq_target_io *tio = tio_from_request(rq); 286 279 287 280 tio->error = error; 288 - if (likely(!blk_should_fake_timeout(rq->q))) 289 - blk_mq_complete_request(rq); 281 + blk_mq_complete_request(rq); 290 282 } 291 283 292 284 /*

+2 -1

drivers/md/dm-snap-persistent.c

··· 871 871 atomic_set(&ps->pending_count, 0); 872 872 ps->callbacks = NULL; 873 873 874 - ps->metadata_wq = alloc_workqueue("ksnaphd", WQ_MEM_RECLAIM, 0); 874 + ps->metadata_wq = alloc_workqueue("ksnaphd", 875 + WQ_MEM_RECLAIM | WQ_PERCPU, 0); 875 876 if (!ps->metadata_wq) { 876 877 DMERR("couldn't start header metadata update thread"); 877 878 r = -ENOMEM;

+10 -2

drivers/md/dm-stripe.c

··· 456 456 struct queue_limits *limits) 457 457 { 458 458 struct stripe_c *sc = ti->private; 459 - unsigned int io_min, io_opt; 459 + unsigned int io_min, io_opt, max_hw_discard_sectors = limits->max_hw_discard_sectors; 460 460 461 461 limits->chunk_sectors = sc->chunk_size; 462 462 ··· 464 464 !check_mul_overflow(io_min, sc->stripes, &io_opt)) { 465 465 limits->io_min = io_min; 466 466 limits->io_opt = io_opt; 467 + } 468 + if (max_hw_discard_sectors >= sc->chunk_size) { 469 + if (!check_mul_overflow(max_hw_discard_sectors, sc->stripes, &max_hw_discard_sectors)) { 470 + max_hw_discard_sectors = rounddown(max_hw_discard_sectors, 471 + sc->chunk_size * sc->stripes); 472 + limits->max_hw_discard_sectors = max_hw_discard_sectors; 473 + } else 474 + limits->max_hw_discard_sectors = UINT_MAX >> SECTOR_SHIFT; 467 475 } 468 476 } 469 477 ··· 497 489 { 498 490 int r; 499 491 500 - dm_stripe_wq = alloc_workqueue("dm_stripe_wq", 0, 0); 492 + dm_stripe_wq = alloc_workqueue("dm_stripe_wq", WQ_PERCPU, 0); 501 493 if (!dm_stripe_wq) 502 494 return -ENOMEM; 503 495 r = dm_register_target(&stripe_target);

+11 -33

drivers/md/dm-table.c

··· 139 139 return -ENOMEM; 140 140 141 141 INIT_LIST_HEAD(&t->devices); 142 - init_rwsem(&t->devices_lock); 143 142 144 143 if (!num_targets) 145 144 num_targets = KEYS_PER_NODE; ··· 379 380 if (dev == disk_devt(t->md->disk)) 380 381 return -EINVAL; 381 382 382 - down_write(&t->devices_lock); 383 - 384 383 dd = find_device(&t->devices, dev); 385 384 if (!dd) { 386 385 dd = kmalloc(sizeof(*dd), GFP_KERNEL); 387 - if (!dd) { 388 - r = -ENOMEM; 389 - goto unlock_ret_r; 390 - } 386 + if (!dd) 387 + return -ENOMEM; 391 388 392 389 r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev); 393 390 if (r) { 394 391 kfree(dd); 395 - goto unlock_ret_r; 392 + return r; 396 393 } 397 394 398 395 refcount_set(&dd->count, 1); ··· 398 403 } else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) { 399 404 r = upgrade_mode(dd, mode, t->md); 400 405 if (r) 401 - goto unlock_ret_r; 406 + return r; 402 407 } 403 408 refcount_inc(&dd->count); 404 409 out: 405 - up_write(&t->devices_lock); 406 410 *result = dd->dm_dev; 407 411 return 0; 408 - 409 - unlock_ret_r: 410 - up_write(&t->devices_lock); 411 - return r; 412 412 } 413 413 EXPORT_SYMBOL(dm_get_device); 414 414 ··· 454 464 void dm_put_device(struct dm_target *ti, struct dm_dev *d) 455 465 { 456 466 int found = 0; 457 - struct dm_table *t = ti->table; 458 - struct list_head *devices = &t->devices; 467 + struct list_head *devices = &ti->table->devices; 459 468 struct dm_dev_internal *dd; 460 - 461 - down_write(&t->devices_lock); 462 469 463 470 list_for_each_entry(dd, devices, list) { 464 471 if (dd->dm_dev == d) { ··· 465 478 } 466 479 if (!found) { 467 480 DMERR("%s: device %s not in table devices list", 468 - dm_device_name(t->md), d->name); 469 - goto unlock_ret; 481 + dm_device_name(ti->table->md), d->name); 482 + return; 470 483 } 471 484 if (refcount_dec_and_test(&dd->count)) { 472 - dm_put_table_device(t->md, d); 485 + dm_put_table_device(ti->table->md, d); 473 486 list_del(&dd->list); 474 487 kfree(dd); 475 488 } 476 - 477 - unlock_ret: 478 - up_write(&t->devices_lock); 479 489 } 480 490 EXPORT_SYMBOL(dm_put_device); 481 491 ··· 1221 1237 bdev_get_queue(bdev)->crypto_profile; 1222 1238 int err = -EOPNOTSUPP; 1223 1239 1224 - if (!args->err) 1225 - return 0; 1226 - 1227 1240 switch (args->op) { 1228 1241 case DERIVE_SW_SECRET: 1229 1242 err = blk_crypto_derive_sw_secret( ··· 1247 1266 break; 1248 1267 } 1249 1268 args->err = err; 1250 - 1251 - /* Try another device in case this fails. */ 1252 - return 0; 1269 + return 1; /* No need to continue the iteration. */ 1253 1270 } 1254 1271 1255 1272 static int dm_exec_wrappedkey_op(struct blk_crypto_profile *profile, ··· 1273 1294 * declared on all underlying devices. Thus, all the underlying devices 1274 1295 * should support all wrapped key operations and they should behave 1275 1296 * identically, i.e. work with the same keys. So, just executing the 1276 - * operation on the first device on which it works suffices for now. 1297 + * operation on the first device suffices for now. 1277 1298 */ 1278 1299 for (i = 0; i < t->num_targets; i++) { 1279 1300 ti = dm_table_get_target(t, i); 1280 1301 if (!ti->type->iterate_devices) 1281 1302 continue; 1282 - ti->type->iterate_devices(ti, dm_wrappedkey_op_callback, args); 1283 - if (!args->err) 1303 + if (ti->type->iterate_devices(ti, dm_wrappedkey_op_callback, args) != 0) 1284 1304 break; 1285 1305 } 1286 1306 out:

+1 -1

drivers/md/dm-target.c

··· 88 88 if (__find_target_type(tt->name)) { 89 89 DMERR("%s: '%s' target already registered", 90 90 __func__, tt->name); 91 - rv = -EEXIST; 91 + rv = -EBUSY; 92 92 } else { 93 93 list_add(&tt->list, &_targets); 94 94 }

+1 -1

drivers/md/dm-unstripe.c

··· 117 117 static sector_t map_to_core(struct dm_target *ti, struct bio *bio) 118 118 { 119 119 struct unstripe_c *uc = ti->private; 120 - sector_t sector = bio->bi_iter.bi_sector; 120 + sector_t sector = dm_target_offset(ti, bio->bi_iter.bi_sector); 121 121 sector_t tmp_sector = sector; 122 122 123 123 /* Shift us up to the right "row" on the stripe */

+2 -2

drivers/md/dm-vdo/data-vio.h

··· 88 88 }; 89 89 90 90 /* 91 - * Where a data_vio is on the compression path; advance_compression_stage() depends on the order of 92 - * this enum. 91 + * Where a data_vio is on the compression path; advance_data_vio_compression_stage() 92 + * depends on the order of this enum. 93 93 */ 94 94 enum data_vio_compression_stage { 95 95 /* A data_vio which has not yet entered the compression path */

+6 -5

drivers/md/dm-vdo/encodings.c

··· 172 172 * VDO_INCORRECT_COMPONENT if the component ids don't match, 173 173 * VDO_UNSUPPORTED_VERSION if the versions or sizes don't match. 174 174 */ 175 - int vdo_validate_header(const struct header *expected_header, 176 - const struct header *actual_header, bool exact_size, 177 - const char *name) 175 + static int vdo_validate_header(const struct header *expected_header, 176 + const struct header *actual_header, 177 + bool exact_size, const char *name) 178 178 { 179 179 int result; 180 180 ··· 210 210 *offset += sizeof(packed); 211 211 } 212 212 213 - void vdo_encode_header(u8 *buffer, size_t *offset, const struct header *header) 213 + static void vdo_encode_header(u8 *buffer, size_t *offset, 214 + const struct header *header) 214 215 { 215 216 struct packed_header packed = vdo_pack_header(header); 216 217 ··· 229 228 *version = vdo_unpack_version_number(packed); 230 229 } 231 230 232 - void vdo_decode_header(u8 *buffer, size_t *offset, struct header *header) 231 + static void vdo_decode_header(u8 *buffer, size_t *offset, struct header *header) 233 232 { 234 233 struct packed_header packed; 235 234

-25

drivers/md/dm-vdo/encodings.h

··· 708 708 } 709 709 710 710 /** 711 - * vdo_is_upgradable_version() - Check whether an actual version is upgradable to an expected 712 - * version. 713 - * @expected_version: The expected version. 714 - * @actual_version: The version being validated. 715 - * 716 - * An actual version is upgradable if its major number is expected but its minor number differs, 717 - * and the expected version's minor number is greater than the actual version's minor number. 718 - * 719 - * Return: true if the actual version is upgradable. 720 - */ 721 - static inline bool vdo_is_upgradable_version(struct version_number expected_version, 722 - struct version_number actual_version) 723 - { 724 - return ((expected_version.major_version == actual_version.major_version) && 725 - (expected_version.minor_version > actual_version.minor_version)); 726 - } 727 - 728 - int __must_check vdo_validate_header(const struct header *expected_header, 729 - const struct header *actual_header, bool exact_size, 730 - const char *component_name); 731 - 732 - void vdo_encode_header(u8 *buffer, size_t *offset, const struct header *header); 733 - void vdo_decode_header(u8 *buffer, size_t *offset, struct header *header); 734 - 735 - /** 736 711 * vdo_pack_version_number() - Convert a version_number to its packed on-disk representation. 737 712 * @version: The version number to convert. 738 713 *

+53 -81

drivers/md/dm-verity-fec.c

··· 11 11 #define DM_MSG_PREFIX "verity-fec" 12 12 13 13 /* 14 - * If error correction has been configured, returns true. 14 + * When correcting a data block, the FEC code performs optimally when it can 15 + * collect all the associated RS blocks at the same time. As each byte is part 16 + * of a different RS block, there are '1 << data_dev_block_bits' RS blocks. 17 + * There are '1 << DM_VERITY_FEC_BUF_RS_BITS' RS blocks per buffer, so that 18 + * gives '1 << (data_dev_block_bits - DM_VERITY_FEC_BUF_RS_BITS)' buffers. 15 19 */ 16 - bool verity_fec_is_enabled(struct dm_verity *v) 20 + static inline unsigned int fec_max_nbufs(struct dm_verity *v) 17 21 { 18 - return v->fec && v->fec->dev; 19 - } 20 - 21 - /* 22 - * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable 23 - * length fields. 24 - */ 25 - static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io) 26 - { 27 - return (struct dm_verity_fec_io *) 28 - ((char *)io + io->v->ti->per_io_data_size - sizeof(struct dm_verity_fec_io)); 22 + return 1 << (v->data_dev_block_bits - DM_VERITY_FEC_BUF_RS_BITS); 29 23 } 30 24 31 25 /* ··· 62 68 63 69 return res; 64 70 } 65 - 66 - /* Loop over each preallocated buffer slot. */ 67 - #define fec_for_each_prealloc_buffer(__i) \ 68 - for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++) 69 - 70 - /* Loop over each extra buffer slot. */ 71 - #define fec_for_each_extra_buffer(io, __i) \ 72 - for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++) 73 71 74 72 /* Loop over each allocated buffer. */ 75 73 #define fec_for_each_buffer(io, __i) \ ··· 197 211 int i, j, target_index = -1; 198 212 struct dm_buffer *buf; 199 213 struct dm_bufio_client *bufio; 200 - struct dm_verity_fec_io *fio = fec_io(io); 214 + struct dm_verity_fec_io *fio = io->fec_io; 201 215 u64 block, ileaved; 202 216 u8 *bbuf, *rs_block; 203 217 u8 want_digest[HASH_MAX_DIGESTSIZE]; ··· 293 307 } 294 308 295 309 /* 296 - * Allocate RS control structure and FEC buffers from preallocated mempools, 297 - * and attempt to allocate as many extra buffers as available. 310 + * Allocate and initialize a struct dm_verity_fec_io to use for FEC for a bio. 311 + * This runs the first time a block needs to be corrected for a bio. In the 312 + * common case where no block needs to be corrected, this code never runs. 313 + * 314 + * This always succeeds, as all required allocations are done from mempools. 315 + * Additional buffers are also allocated opportunistically to improve error 316 + * correction performance, but these aren't required to succeed. 298 317 */ 299 - static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) 318 + static struct dm_verity_fec_io *fec_alloc_and_init_io(struct dm_verity *v) 300 319 { 320 + const unsigned int max_nbufs = fec_max_nbufs(v); 321 + struct dm_verity_fec *f = v->fec; 322 + struct dm_verity_fec_io *fio; 301 323 unsigned int n; 302 324 303 - if (!fio->rs) 304 - fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO); 325 + fio = mempool_alloc(&f->fio_pool, GFP_NOIO); 326 + fio->rs = mempool_alloc(&f->rs_pool, GFP_NOIO); 305 327 306 - fec_for_each_prealloc_buffer(n) { 307 - if (fio->bufs[n]) 308 - continue; 309 - 310 - fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOIO); 311 - } 328 + fio->bufs[0] = mempool_alloc(&f->prealloc_pool, GFP_NOIO); 312 329 313 330 /* try to allocate the maximum number of buffers */ 314 - fec_for_each_extra_buffer(fio, n) { 315 - if (fio->bufs[n]) 316 - continue; 317 - 318 - fio->bufs[n] = kmem_cache_alloc(v->fec->cache, GFP_NOWAIT); 331 + for (n = 1; n < max_nbufs; n++) { 332 + fio->bufs[n] = kmem_cache_alloc(f->cache, GFP_NOWAIT); 319 333 /* we can manage with even one buffer if necessary */ 320 334 if (unlikely(!fio->bufs[n])) 321 335 break; 322 336 } 323 337 fio->nbufs = n; 324 338 325 - if (!fio->output) 326 - fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO); 327 - 328 - return 0; 339 + fio->output = mempool_alloc(&f->output_pool, GFP_NOIO); 340 + fio->level = 0; 341 + return fio; 329 342 } 330 343 331 344 /* ··· 352 367 { 353 368 int r, neras = 0; 354 369 unsigned int pos; 355 - 356 - r = fec_alloc_bufs(v, fio); 357 - if (unlikely(r < 0)) 358 - return r; 359 370 360 371 for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { 361 372 fec_init_bufs(v, fio); ··· 389 408 sector_t block, u8 *dest) 390 409 { 391 410 int r; 392 - struct dm_verity_fec_io *fio = fec_io(io); 411 + struct dm_verity_fec_io *fio; 393 412 u64 offset, res, rsb; 394 413 395 414 if (!verity_fec_is_enabled(v)) 396 415 return -EOPNOTSUPP; 416 + 417 + fio = io->fec_io; 418 + if (!fio) 419 + fio = io->fec_io = fec_alloc_and_init_io(v); 397 420 398 421 if (fio->level) 399 422 return -EIO; ··· 448 463 /* 449 464 * Clean up per-bio data. 450 465 */ 451 - void verity_fec_finish_io(struct dm_verity_io *io) 466 + void __verity_fec_finish_io(struct dm_verity_io *io) 452 467 { 453 468 unsigned int n; 454 469 struct dm_verity_fec *f = io->v->fec; 455 - struct dm_verity_fec_io *fio = fec_io(io); 456 - 457 - if (!verity_fec_is_enabled(io->v)) 458 - return; 470 + struct dm_verity_fec_io *fio = io->fec_io; 459 471 460 472 mempool_free(fio->rs, &f->rs_pool); 461 473 462 - fec_for_each_prealloc_buffer(n) 463 - mempool_free(fio->bufs[n], &f->prealloc_pool); 474 + mempool_free(fio->bufs[0], &f->prealloc_pool); 464 475 465 - fec_for_each_extra_buffer(fio, n) 466 - if (fio->bufs[n]) 467 - kmem_cache_free(f->cache, fio->bufs[n]); 476 + for (n = 1; n < fio->nbufs; n++) 477 + kmem_cache_free(f->cache, fio->bufs[n]); 468 478 469 479 mempool_free(fio->output, &f->output_pool); 470 - } 471 480 472 - /* 473 - * Initialize per-bio data. 474 - */ 475 - void verity_fec_init_io(struct dm_verity_io *io) 476 - { 477 - struct dm_verity_fec_io *fio = fec_io(io); 478 - 479 - if (!verity_fec_is_enabled(io->v)) 480 - return; 481 - 482 - fio->rs = NULL; 483 - memset(fio->bufs, 0, sizeof(fio->bufs)); 484 - fio->nbufs = 0; 485 - fio->output = NULL; 486 - fio->level = 0; 481 + mempool_free(fio, &f->fio_pool); 482 + io->fec_io = NULL; 487 483 } 488 484 489 485 /* ··· 495 529 if (!verity_fec_is_enabled(v)) 496 530 goto out; 497 531 532 + mempool_exit(&f->fio_pool); 498 533 mempool_exit(&f->rs_pool); 499 534 mempool_exit(&f->prealloc_pool); 500 535 mempool_exit(&f->output_pool); 501 536 kmem_cache_destroy(f->cache); 502 537 503 - if (f->data_bufio) 538 + if (!IS_ERR_OR_NULL(f->data_bufio)) 504 539 dm_bufio_client_destroy(f->data_bufio); 505 - if (f->bufio) 540 + if (!IS_ERR_OR_NULL(f->bufio)) 506 541 dm_bufio_client_destroy(f->bufio); 507 542 508 543 if (f->dev) ··· 725 758 return -E2BIG; 726 759 } 727 760 761 + /* Preallocate some dm_verity_fec_io structures */ 762 + ret = mempool_init_kmalloc_pool(&f->fio_pool, num_online_cpus(), 763 + struct_size((struct dm_verity_fec_io *)0, 764 + bufs, fec_max_nbufs(v))); 765 + if (ret) { 766 + ti->error = "Cannot allocate FEC IO pool"; 767 + return ret; 768 + } 769 + 728 770 /* Preallocate an rs_control structure for each worker thread */ 729 771 ret = mempool_init(&f->rs_pool, num_online_cpus(), fec_rs_alloc, 730 772 fec_rs_free, (void *) v); ··· 750 774 return -ENOMEM; 751 775 } 752 776 753 - /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */ 754 - ret = mempool_init_slab_pool(&f->prealloc_pool, num_online_cpus() * 755 - DM_VERITY_FEC_BUF_PREALLOC, 777 + /* Preallocate one buffer for each thread */ 778 + ret = mempool_init_slab_pool(&f->prealloc_pool, num_online_cpus(), 756 779 f->cache); 757 780 if (ret) { 758 781 ti->error = "Cannot allocate FEC buffer prealloc pool"; ··· 765 790 ti->error = "Cannot allocate FEC output pool"; 766 791 return ret; 767 792 } 768 - 769 - /* Reserve space for our per-bio data */ 770 - ti->per_io_data_size += sizeof(struct dm_verity_fec_io); 771 793 772 794 return 0; 773 795 }

+26 -9

drivers/md/dm-verity-fec.h

··· 17 17 #define DM_VERITY_FEC_MIN_RSN 231 /* ~10% space overhead */ 18 18 19 19 /* buffers for deinterleaving and decoding */ 20 - #define DM_VERITY_FEC_BUF_PREALLOC 1 /* buffers to preallocate */ 21 20 #define DM_VERITY_FEC_BUF_RS_BITS 4 /* 1 << RS blocks per buffer */ 22 - /* we need buffers for at most 1 << block size RS blocks */ 23 - #define DM_VERITY_FEC_BUF_MAX \ 24 - (1 << (PAGE_SHIFT - DM_VERITY_FEC_BUF_RS_BITS)) 25 21 26 22 #define DM_VERITY_OPT_FEC_DEV "use_fec_from_device" 27 23 #define DM_VERITY_OPT_FEC_BLOCKS "fec_blocks" ··· 36 40 sector_t hash_blocks; /* blocks covered after v->hash_start */ 37 41 unsigned char roots; /* number of parity bytes, M-N of RS(M, N) */ 38 42 unsigned char rsn; /* N of RS(M, N) */ 43 + mempool_t fio_pool; /* mempool for dm_verity_fec_io */ 39 44 mempool_t rs_pool; /* mempool for fio->rs */ 40 45 mempool_t prealloc_pool; /* mempool for preallocated buffers */ 41 46 mempool_t output_pool; /* mempool for output */ ··· 48 51 struct dm_verity_fec_io { 49 52 struct rs_control *rs; /* Reed-Solomon state */ 50 53 int erasures[DM_VERITY_FEC_MAX_RSN]; /* erasures for decode_rs8 */ 51 - u8 *bufs[DM_VERITY_FEC_BUF_MAX]; /* bufs for deinterleaving */ 52 - unsigned int nbufs; /* number of buffers allocated */ 53 54 u8 *output; /* buffer for corrected output */ 54 55 unsigned int level; /* recursion level */ 56 + unsigned int nbufs; /* number of buffers allocated */ 57 + /* 58 + * Buffers for deinterleaving RS blocks. Each buffer has space for 59 + * the data bytes of (1 << DM_VERITY_FEC_BUF_RS_BITS) RS blocks. The 60 + * array length is fec_max_nbufs(v), and we try to allocate that many 61 + * buffers. However, in low-memory situations we may be unable to 62 + * allocate all buffers. 'nbufs' holds the number actually allocated. 63 + */ 64 + u8 *bufs[]; 55 65 }; 56 66 57 67 #ifdef CONFIG_DM_VERITY_FEC ··· 66 62 /* each feature parameter requires a value */ 67 63 #define DM_VERITY_OPTS_FEC 8 68 64 69 - extern bool verity_fec_is_enabled(struct dm_verity *v); 65 + /* Returns true if forward error correction is enabled. */ 66 + static inline bool verity_fec_is_enabled(struct dm_verity *v) 67 + { 68 + return v->fec && v->fec->dev; 69 + } 70 70 71 71 extern int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, 72 72 enum verity_block_type type, const u8 *want_digest, ··· 79 71 extern unsigned int verity_fec_status_table(struct dm_verity *v, unsigned int sz, 80 72 char *result, unsigned int maxlen); 81 73 82 - extern void verity_fec_finish_io(struct dm_verity_io *io); 83 - extern void verity_fec_init_io(struct dm_verity_io *io); 74 + extern void __verity_fec_finish_io(struct dm_verity_io *io); 75 + static inline void verity_fec_finish_io(struct dm_verity_io *io) 76 + { 77 + if (unlikely(io->fec_io)) 78 + __verity_fec_finish_io(io); 79 + } 80 + 81 + static inline void verity_fec_init_io(struct dm_verity_io *io) 82 + { 83 + io->fec_io = NULL; 84 + } 84 85 85 86 extern bool verity_is_fec_opt_arg(const char *arg_name); 86 87 extern int verity_fec_parse_opt_args(struct dm_arg_set *as,

+44 -21

drivers/md/dm-verity-target.c

··· 254 254 data = dm_bufio_get(v->bufio, hash_block, &buf); 255 255 if (IS_ERR_OR_NULL(data)) { 256 256 /* 257 - * In tasklet and the hash was not in the bufio cache. 258 - * Return early and resume execution from a work-queue 259 - * to read the hash from disk. 257 + * In softirq and the hash was not in the bufio cache. 258 + * Return early and resume execution from a kworker to 259 + * read the hash from disk. 260 260 */ 261 261 return -EAGAIN; 262 262 } ··· 303 303 else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { 304 304 /* 305 305 * Error handling code (FEC included) cannot be run in a 306 - * tasklet since it may sleep, so fallback to work-queue. 306 + * softirq since it may sleep, so fallback to a kworker. 307 307 */ 308 308 r = -EAGAIN; 309 309 goto release_ret_r; ··· 425 425 426 426 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { 427 427 /* 428 - * Error handling code (FEC included) cannot be run in the 429 - * BH workqueue, so fallback to a standard workqueue. 428 + * Error handling code (FEC included) cannot be run in a 429 + * softirq since it may sleep, so fallback to a kworker. 430 430 */ 431 431 return -EAGAIN; 432 432 } ··· 435 435 set_bit(blkno, v->validated_blocks); 436 436 return 0; 437 437 } 438 - #if defined(CONFIG_DM_VERITY_FEC) 439 438 if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA, want_digest, 440 439 blkno, data) == 0) 441 440 return 0; 442 - #endif 443 441 if (bio->bi_status) 444 442 return -EIO; /* Error correction failed; Just return error */ 445 443 ··· 519 521 520 522 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { 521 523 /* 522 - * Copy the iterator in case we need to restart 523 - * verification in a work-queue. 524 + * Copy the iterator in case we need to restart verification in 525 + * a kworker. 524 526 */ 525 527 iter_copy = io->iter; 526 528 iter = &iter_copy; ··· 528 530 iter = &io->iter; 529 531 530 532 for (b = 0; b < io->n_blocks; 531 - b++, bio_advance_iter(bio, iter, block_size)) { 533 + b++, bio_advance_iter_single(bio, iter, block_size)) { 532 534 sector_t blkno = io->block + b; 533 535 struct pending_block *block; 534 536 bool is_zero; ··· 617 619 bio->bi_end_io = io->orig_bi_end_io; 618 620 bio->bi_status = status; 619 621 620 - if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh) 621 - verity_fec_finish_io(io); 622 + verity_fec_finish_io(io); 622 623 623 624 if (unlikely(status != BLK_STS_OK) && 624 625 unlikely(!(bio->bi_opf & REQ_RAHEAD)) && ··· 651 654 652 655 static void verity_bh_work(struct work_struct *w) 653 656 { 654 - struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work); 657 + struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); 655 658 int err; 656 659 657 660 io->in_bh = true; 658 661 err = verity_verify_io(io); 659 662 if (err == -EAGAIN || err == -ENOMEM) { 660 - /* fallback to retrying with work-queue */ 663 + /* fallback to retrying in a kworker */ 661 664 INIT_WORK(&io->work, verity_work); 662 665 queue_work(io->v->verify_wq, &io->work); 663 666 return; ··· 690 693 if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq && 691 694 verity_use_bh(bytes, ioprio)) { 692 695 if (in_hardirq() || irqs_disabled()) { 693 - INIT_WORK(&io->bh_work, verity_bh_work); 694 - queue_work(system_bh_wq, &io->bh_work); 696 + INIT_WORK(&io->work, verity_bh_work); 697 + queue_work(system_bh_wq, &io->work); 695 698 } else { 696 - verity_bh_work(&io->bh_work); 699 + verity_bh_work(&io->work); 697 700 } 698 701 } else { 699 702 INIT_WORK(&io->work, verity_work); ··· 1644 1647 * reducing wait times when reading from a dm-verity device. 1645 1648 * 1646 1649 * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI 1647 - * allows verify_wq to preempt softirq since verification in BH workqueue 1650 + * allows verify_wq to preempt softirq since verification in softirq 1648 1651 * will fall-back to using it for error handling (or if the bufio cache 1649 1652 * doesn't have required hashes). 1650 1653 */ 1651 - v->verify_wq = alloc_workqueue("kverityd", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 1654 + v->verify_wq = alloc_workqueue("kverityd", 1655 + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 1656 + 0); 1652 1657 if (!v->verify_wq) { 1653 1658 ti->error = "Cannot allocate workqueue"; 1654 1659 r = -ENOMEM; ··· 1802 1803 .preresume = verity_preresume, 1803 1804 #endif /* CONFIG_SECURITY */ 1804 1805 }; 1805 - module_dm(verity); 1806 + 1807 + static int __init dm_verity_init(void) 1808 + { 1809 + int r; 1810 + 1811 + r = dm_verity_verify_sig_init(); 1812 + if (r) 1813 + return r; 1814 + 1815 + r = dm_register_target(&verity_target); 1816 + if (r) { 1817 + dm_verity_verify_sig_exit(); 1818 + return r; 1819 + } 1820 + 1821 + return 0; 1822 + } 1823 + module_init(dm_verity_init); 1824 + 1825 + static void __exit dm_verity_exit(void) 1826 + { 1827 + dm_unregister_target(&verity_target); 1828 + dm_verity_verify_sig_exit(); 1829 + } 1830 + module_exit(dm_verity_exit); 1806 1831 1807 1832 /* 1808 1833 * Check whether a DM target is a verity target.

+45

drivers/md/dm-verity-verify-sig.c

··· 7 7 */ 8 8 #include <linux/device-mapper.h> 9 9 #include <linux/verification.h> 10 + #include <linux/key.h> 10 11 #include <keys/user-type.h> 11 12 #include <linux/module.h> 12 13 #include "dm-verity.h" 13 14 #include "dm-verity-verify-sig.h" 14 15 15 16 #define DM_VERITY_VERIFY_ERR(s) DM_VERITY_ROOT_HASH_VERIFICATION " " s 17 + 18 + static struct key *dm_verity_keyring; 19 + 20 + static bool dm_verity_keyring_unsealed __ro_after_init; 21 + module_param_named(keyring_unsealed, dm_verity_keyring_unsealed, bool, 0444); 22 + MODULE_PARM_DESC(keyring_unsealed, "Leave the dm-verity keyring unsealed"); 16 23 17 24 static bool require_signatures; 18 25 module_param(require_signatures, bool, 0444); ··· 150 143 VERIFYING_UNSPECIFIED_SIGNATURE, NULL, NULL); 151 144 #endif 152 145 146 + if (ret != -ENOKEY && ret != -EKEYREJECTED) 147 + return ret; 148 + 149 + if (dm_verity_keyring->keys.nr_leaves_on_tree && 150 + dm_verity_keyring->restrict_link) 151 + ret = verify_pkcs7_signature(root_hash, root_hash_len, 152 + sig_data, sig_len, 153 + dm_verity_keyring, 154 + VERIFYING_UNSPECIFIED_SIGNATURE, 155 + NULL, NULL); 156 + 153 157 return ret; 154 158 } 155 159 ··· 169 151 kfree(sig_opts->sig); 170 152 sig_opts->sig = NULL; 171 153 sig_opts->sig_size = 0; 154 + } 155 + 156 + int __init dm_verity_verify_sig_init(void) 157 + { 158 + dm_verity_keyring = keyring_alloc(".dm-verity", 159 + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 160 + current_cred(), 161 + KEY_POS_SEARCH | 162 + KEY_USR_VIEW | KEY_USR_READ | 163 + KEY_USR_WRITE | KEY_USR_SEARCH | 164 + KEY_USR_SETATTR, 165 + KEY_ALLOC_NOT_IN_QUOTA, 166 + NULL, NULL); 167 + if (IS_ERR(dm_verity_keyring)) 168 + panic("dm-verity can't allocate keyring\n"); 169 + 170 + if (!dm_verity_keyring_unsealed && 171 + keyring_restrict(make_key_ref(dm_verity_keyring, true), NULL, NULL)) 172 + panic("dm-verity can't seal keyring\n"); 173 + 174 + return 0; 175 + } 176 + 177 + void dm_verity_verify_sig_exit(void) 178 + { 179 + key_revoke(dm_verity_keyring); 180 + key_put(dm_verity_keyring); 172 181 }

+12

drivers/md/dm-verity-verify-sig.h

··· 30 30 31 31 void verity_verify_sig_opts_cleanup(struct dm_verity_sig_opts *sig_opts); 32 32 33 + int __init dm_verity_verify_sig_init(void); 34 + void dm_verity_verify_sig_exit(void); 35 + 33 36 #else 34 37 35 38 #define DM_VERITY_ROOT_HASH_VERIFICATION_OPTS 0 ··· 56 53 } 57 54 58 55 static inline void verity_verify_sig_opts_cleanup(struct dm_verity_sig_opts *sig_opts) 56 + { 57 + } 58 + 59 + static inline int dm_verity_verify_sig_init(void) 60 + { 61 + return 0; 62 + } 63 + 64 + static inline void dm_verity_verify_sig_exit(void) 59 65 { 60 66 } 61 67

+4 -1

drivers/md/dm-verity.h

··· 104 104 bool in_bh; 105 105 bool had_mismatch; 106 106 107 + #ifdef CONFIG_DM_VERITY_FEC 108 + struct dm_verity_fec_io *fec_io; 109 + #endif 110 + 107 111 struct work_struct work; 108 - struct work_struct bh_work; 109 112 110 113 u8 tmp_digest[HASH_MAX_DIGESTSIZE]; 111 114

+2 -1

drivers/md/dm-writecache.c

··· 2275 2275 goto bad; 2276 2276 } 2277 2277 2278 - wc->writeback_wq = alloc_workqueue("writecache-writeback", WQ_MEM_RECLAIM, 1); 2278 + wc->writeback_wq = alloc_workqueue("writecache-writeback", 2279 + WQ_MEM_RECLAIM | WQ_PERCPU, 1); 2279 2280 if (!wc->writeback_wq) { 2280 2281 r = -ENOMEM; 2281 2282 ti->error = "Could not allocate writeback workqueue";

+7 -4

drivers/md/dm-zone.c

··· 50 50 { 51 51 struct mapped_device *md = disk->private_data; 52 52 struct dm_table *map; 53 - struct dm_table *zone_revalidate_map = md->zone_revalidate_map; 53 + struct dm_table *zone_revalidate_map = READ_ONCE(md->zone_revalidate_map); 54 54 int srcu_idx, ret = -EIO; 55 55 bool put_table = false; 56 56 ··· 60 60 * Zone revalidation during __bind() is in progress, but this 61 61 * call is from a different process 62 62 */ 63 - if (dm_suspended_md(md)) 64 - return -EAGAIN; 65 - 66 63 map = dm_get_live_table(md, &srcu_idx); 67 64 put_table = true; 65 + 66 + if (dm_suspended_md(md)) { 67 + ret = -EAGAIN; 68 + goto do_put_table; 69 + } 68 70 } else { 69 71 /* Zone revalidation during __bind() */ 70 72 map = zone_revalidate_map; ··· 81 79 ret = dm_blk_do_report_zones(md, map, nr_zones, &dm_args); 82 80 } 83 81 82 + do_put_table: 84 83 if (put_table) 85 84 dm_put_live_table(md, srcu_idx); 86 85

+4 -1

drivers/md/dm.c

··· 1364 1364 if (!tgt_clone) 1365 1365 tgt_clone = clone; 1366 1366 1367 + bio_clone_blkg_association(tgt_clone, io->orig_bio); 1368 + 1367 1369 /* 1368 1370 * Account io->origin_bio to DM dev on behalf of target 1369 1371 * that took ownership of IO with DM_MAPIO_SUBMITTED. ··· 2368 2366 2369 2367 format_dev_t(md->name, MKDEV(_major, minor)); 2370 2368 2371 - md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name); 2369 + md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM | WQ_PERCPU, 0, 2370 + md->name); 2372 2371 if (!md->wq) 2373 2372 goto bad; 2374 2373

+2 -2

drivers/md/md.c

··· 10503 10503 goto err_bitmap; 10504 10504 10505 10505 ret = -ENOMEM; 10506 - md_wq = alloc_workqueue("md", WQ_MEM_RECLAIM, 0); 10506 + md_wq = alloc_workqueue("md", WQ_MEM_RECLAIM | WQ_PERCPU, 0); 10507 10507 if (!md_wq) 10508 10508 goto err_wq; 10509 10509 10510 - md_misc_wq = alloc_workqueue("md_misc", 0, 0); 10510 + md_misc_wq = alloc_workqueue("md_misc", WQ_PERCPU, 0); 10511 10511 if (!md_misc_wq) 10512 10512 goto err_misc_wq; 10513 10513

+5

tools/testing/selftests/dm-verity/Makefile

··· 1 + # SPDX-License-Identifier: GPL-2.0 2 + 3 + TEST_PROGS := test-dm-verity-keyring.sh 4 + 5 + include ../lib.mk

+10

tools/testing/selftests/dm-verity/config

··· 1 + CONFIG_BLK_DEV_DM=y 2 + CONFIG_DM_VERITY=m 3 + CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG=y 4 + CONFIG_MODULE_UNLOAD=y 5 + CONFIG_KEYS=y 6 + CONFIG_ASYMMETRIC_KEY_TYPE=y 7 + CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y 8 + CONFIG_X509_CERTIFICATE_PARSER=y 9 + CONFIG_PKCS7_MESSAGE_PARSER=y 10 + CONFIG_SYSTEM_DATA_VERIFICATION=y

+873

tools/testing/selftests/dm-verity/test-dm-verity-keyring.sh

··· 1 + #!/bin/bash 2 + # SPDX-License-Identifier: GPL-2.0 3 + # 4 + # Test script for dm-verity keyring functionality 5 + # 6 + # This script has two modes depending on kernel configuration: 7 + # 8 + # 1. keyring_unsealed=1 AND require_signatures=1: 9 + # - Upload a test key to the .dm-verity keyring 10 + # - Seal the keyring 11 + # - Create a dm-verity device with a signed root hash 12 + # - Verify signature verification works 13 + # 14 + # 2. keyring_unsealed=0 (default) OR require_signatures=0: 15 + # - Verify the keyring is already sealed (if unsealed=0) 16 + # - Verify keys cannot be added to a sealed keyring 17 + # - Verify the keyring is inactive (not used for verification) 18 + # 19 + # Requirements: 20 + # - Root privileges 21 + # - openssl 22 + # - veritysetup (cryptsetup) 23 + # - keyctl (keyutils) 24 + 25 + set -e 26 + 27 + WORK_DIR="" 28 + DATA_DEV="" 29 + HASH_DEV="" 30 + DM_NAME="verity-test-$$" 31 + CLEANUP_DONE=0 32 + 33 + # Module parameters (detected at runtime) 34 + KEYRING_UNSEALED="" 35 + REQUIRE_SIGNATURES="" 36 + 37 + # Colors for output 38 + RED='\033[0;31m' 39 + GREEN='\033[0;32m' 40 + YELLOW='\033[1;33m' 41 + NC='\033[0m' # No Color 42 + 43 + log_info() { 44 + echo -e "${GREEN}[INFO]${NC} $*" 45 + } 46 + 47 + log_warn() { 48 + echo -e "${YELLOW}[WARN]${NC} $*" 49 + } 50 + 51 + log_error() { 52 + echo -e "${RED}[ERROR]${NC} $*" >&2 53 + } 54 + 55 + log_pass() { 56 + echo -e "${GREEN}[PASS]${NC} $*" 57 + } 58 + 59 + log_fail() { 60 + echo -e "${RED}[FAIL]${NC} $*" >&2 61 + } 62 + 63 + log_skip() { 64 + echo -e "${YELLOW}[SKIP]${NC} $*" 65 + } 66 + 67 + cleanup() { 68 + if [ "$CLEANUP_DONE" -eq 1 ]; then 69 + return 70 + fi 71 + CLEANUP_DONE=1 72 + 73 + log_info "Cleaning up..." 74 + 75 + # Remove dm-verity device if it exists 76 + if dmsetup info "$DM_NAME" &>/dev/null; then 77 + dmsetup remove "$DM_NAME" 2>/dev/null || true 78 + fi 79 + 80 + # Detach loop devices 81 + if [ -n "$DATA_DEV" ] && [[ "$DATA_DEV" == /dev/loop* ]]; then 82 + losetup -d "$DATA_DEV" 2>/dev/null || true 83 + fi 84 + if [ -n "$HASH_DEV" ] && [[ "$HASH_DEV" == /dev/loop* ]]; then 85 + losetup -d "$HASH_DEV" 2>/dev/null || true 86 + fi 87 + 88 + # Remove work directory 89 + if [ -n "$WORK_DIR" ] && [ -d "$WORK_DIR" ]; then 90 + rm -rf "$WORK_DIR" 91 + fi 92 + } 93 + 94 + trap cleanup EXIT 95 + 96 + die() { 97 + log_error "$*" 98 + exit 1 99 + } 100 + 101 + find_dm_verity_keyring() { 102 + # The .dm-verity keyring is not linked to user-accessible keyrings, 103 + # so we need to find it via /proc/keys 104 + local serial_hex 105 + serial_hex=$(awk '/\.dm-verity/ {print $1}' /proc/keys 2>/dev/null) 106 + 107 + if [ -z "$serial_hex" ]; then 108 + return 1 109 + fi 110 + 111 + # Convert hex to decimal for keyctl 112 + echo $((16#$serial_hex)) 113 + } 114 + 115 + get_module_param() { 116 + local param="$1" 117 + local path="/sys/module/dm_verity/parameters/$param" 118 + 119 + if [ -f "$path" ]; then 120 + cat "$path" 121 + else 122 + echo "" 123 + fi 124 + } 125 + 126 + check_requirements() { 127 + log_info "Checking requirements..." 128 + 129 + # Check for root 130 + if [ "$(id -u)" -ne 0 ]; then 131 + die "This script must be run as root" 132 + fi 133 + 134 + # Check for required tools 135 + for cmd in openssl veritysetup keyctl losetup dmsetup dd awk; do 136 + if ! command -v "$cmd" &>/dev/null; then 137 + die "Required command not found: $cmd" 138 + fi 139 + done 140 + 141 + # Check for dm-verity module 142 + if ! modprobe -n dm-verity &>/dev/null; then 143 + die "dm-verity module not available" 144 + fi 145 + 146 + # Verify OpenSSL can create signatures 147 + # OpenSSL cms -sign with -binary -outform DER creates detached signatures by default 148 + log_info "Using OpenSSL for PKCS#7 signatures" 149 + } 150 + 151 + load_dm_verity_module() { 152 + local keyring_unsealed="${1:-0}" 153 + local require_signatures="${2:-0}" 154 + 155 + log_info "Loading dm-verity module with keyring_unsealed=$keyring_unsealed require_signatures=$require_signatures" 156 + 157 + # Unload if already loaded 158 + if lsmod | grep -q '^dm_verity'; then 159 + log_info "Unloading existing dm-verity module..." 160 + modprobe -r dm-verity 2>/dev/null || \ 161 + die "Failed to unload dm-verity module (may be in use)" 162 + sleep 1 163 + fi 164 + 165 + # Load with specified parameters 166 + modprobe dm-verity keyring_unsealed="$keyring_unsealed" require_signatures="$require_signatures" || \ 167 + die "Failed to load dm-verity module" 168 + 169 + # Wait for keyring to be created (poll with timeout) 170 + local keyring_id="" 171 + local timeout=50 # 5 seconds (50 * 0.1s) 172 + while [ $timeout -gt 0 ]; do 173 + keyring_id=$(find_dm_verity_keyring) && break 174 + sleep 0.1 175 + timeout=$((timeout - 1)) 176 + done 177 + 178 + if [ -z "$keyring_id" ]; then 179 + die "dm-verity keyring not found after module load (timeout)" 180 + fi 181 + 182 + log_info "Found .dm-verity keyring: $keyring_id" 183 + echo "$keyring_id" > "$WORK_DIR/keyring_id" 184 + 185 + # Read and display module parameters 186 + KEYRING_UNSEALED=$(get_module_param "keyring_unsealed") 187 + REQUIRE_SIGNATURES=$(get_module_param "require_signatures") 188 + 189 + log_info "Module parameters:" 190 + log_info " keyring_unsealed=$KEYRING_UNSEALED" 191 + log_info " require_signatures=$REQUIRE_SIGNATURES" 192 + } 193 + 194 + unload_dm_verity_module() { 195 + log_info "Unloading dm-verity module..." 196 + 197 + # Clean up any dm-verity devices first 198 + local dm_dev 199 + while read -r dm_dev _; do 200 + [ -n "$dm_dev" ] || continue 201 + log_info "Removing dm-verity device: $dm_dev" 202 + dmsetup remove "$dm_dev" 2>/dev/null || true 203 + done < <(dmsetup ls --target verity 2>/dev/null) 204 + 205 + if lsmod | grep -q '^dm_verity'; then 206 + modprobe -r dm-verity 2>/dev/null || \ 207 + log_warn "Failed to unload dm-verity module" 208 + sleep 1 209 + fi 210 + } 211 + 212 + generate_keys() { 213 + log_info "Generating signing key pair..." 214 + 215 + # Generate private key (2048-bit for faster test execution) 216 + openssl genrsa -out "$WORK_DIR/private.pem" 2048 2>/dev/null 217 + 218 + # Create OpenSSL config for certificate extensions 219 + # The kernel requires digitalSignature key usage for signature verification 220 + # Both subjectKeyIdentifier and authorityKeyIdentifier are needed for 221 + # the kernel to match keys in the keyring (especially for self-signed certs) 222 + cat > "$WORK_DIR/openssl.cnf" << 'EOF' 223 + [req] 224 + distinguished_name = req_distinguished_name 225 + x509_extensions = v3_ca 226 + prompt = no 227 + 228 + [req_distinguished_name] 229 + CN = dm-verity-test-key 230 + 231 + [v3_ca] 232 + basicConstraints = critical,CA:FALSE 233 + keyUsage = digitalSignature 234 + subjectKeyIdentifier = hash 235 + authorityKeyIdentifier = keyid 236 + EOF 237 + 238 + # Generate self-signed certificate with proper extensions 239 + openssl req -new -x509 -key "$WORK_DIR/private.pem" \ 240 + -out "$WORK_DIR/cert.pem" -days 365 \ 241 + -config "$WORK_DIR/openssl.cnf" 2>/dev/null 242 + 243 + # Convert certificate to DER format for kernel 244 + openssl x509 -in "$WORK_DIR/cert.pem" -outform DER \ 245 + -out "$WORK_DIR/cert.der" 246 + 247 + # Show certificate info for debugging 248 + log_info "Certificate details:" 249 + openssl x509 -in "$WORK_DIR/cert.pem" -noout -text 2>/dev/null | \ 250 + grep -E "Subject:|Issuer:|Key Usage|Extended" | head -10 251 + 252 + log_info "Keys generated successfully" 253 + } 254 + 255 + seal_keyring() { 256 + log_info "Sealing the .dm-verity keyring..." 257 + 258 + local keyring_id 259 + keyring_id=$(cat "$WORK_DIR/keyring_id") 260 + 261 + keyctl restrict_keyring "$keyring_id" || \ 262 + die "Failed to seal keyring" 263 + 264 + log_info "Keyring sealed successfully" 265 + } 266 + 267 + create_test_device() { 268 + log_info "Creating test device images..." 269 + 270 + # Create data image with random content (8MB is sufficient for testing) 271 + dd if=/dev/urandom of="$WORK_DIR/data.img" bs=1M count=8 status=none 272 + 273 + # Create hash image (will be populated by veritysetup) 274 + dd if=/dev/zero of="$WORK_DIR/hash.img" bs=1M count=1 status=none 275 + 276 + # Setup loop devices 277 + DATA_DEV=$(losetup --find --show "$WORK_DIR/data.img") 278 + HASH_DEV=$(losetup --find --show "$WORK_DIR/hash.img") 279 + 280 + log_info "Data device: $DATA_DEV" 281 + log_info "Hash device: $HASH_DEV" 282 + } 283 + 284 + create_verity_hash() { 285 + log_info "Creating dm-verity hash tree..." 286 + 287 + local root_hash output 288 + output=$(veritysetup format "$DATA_DEV" "$HASH_DEV" 2>&1) 289 + root_hash=$(echo "$output" | grep "Root hash:" | awk '{print $3}') 290 + 291 + if [ -z "$root_hash" ]; then 292 + log_error "veritysetup format output:" 293 + echo "$output" | sed 's/^/ /' 294 + die "Failed to get root hash from veritysetup format" 295 + fi 296 + 297 + echo "$root_hash" > "$WORK_DIR/root_hash" 298 + log_info "Root hash: $root_hash" 299 + } 300 + 301 + create_detached_signature() { 302 + local infile="$1" 303 + local outfile="$2" 304 + local cert="$3" 305 + local key="$4" 306 + 307 + # Use openssl smime (not cms) for PKCS#7 signatures compatible with kernel 308 + # Flags from working veritysetup example: 309 + # -nocerts: don't include certificate in signature 310 + # -noattr: no signed attributes 311 + # -binary: binary input mode 312 + if openssl smime -sign -nocerts -noattr -binary \ 313 + -in "$infile" \ 314 + -inkey "$key" \ 315 + -signer "$cert" \ 316 + -outform der \ 317 + -out "$outfile" 2>/dev/null; then 318 + return 0 319 + fi 320 + 321 + log_error "Failed to create signature" 322 + return 1 323 + } 324 + 325 + activate_verity_device() { 326 + local with_sig="$1" 327 + local root_hash 328 + root_hash=$(cat "$WORK_DIR/root_hash") 329 + 330 + # Clear dmesg and capture any kernel messages during activation 331 + dmesg -C 2>/dev/null || true 332 + 333 + if [ "$with_sig" = "yes" ]; then 334 + log_info "Activating dm-verity device with signature..." 335 + veritysetup open "$DATA_DEV" "$DM_NAME" "$HASH_DEV" "$root_hash" \ 336 + --root-hash-signature="$WORK_DIR/root_hash.p7s" 2>&1 337 + local ret=$? 338 + else 339 + log_info "Activating dm-verity device without signature..." 340 + veritysetup open "$DATA_DEV" "$DM_NAME" "$HASH_DEV" "$root_hash" 2>&1 341 + local ret=$? 342 + fi 343 + 344 + # Show relevant kernel messages 345 + local kmsg 346 + kmsg=$(dmesg 2>/dev/null | grep -i -E 'verity|pkcs|signature|asymmetric|key' | tail -10) 347 + if [ -n "$kmsg" ]; then 348 + log_info "Kernel messages:" 349 + echo "$kmsg" | while read -r line; do echo " $line"; done 350 + fi 351 + 352 + return $ret 353 + } 354 + 355 + deactivate_verity_device() { 356 + if dmsetup info "$DM_NAME" &>/dev/null; then 357 + dmsetup remove "$DM_NAME" 2>/dev/null || true 358 + fi 359 + } 360 + 361 + show_keyring_status() { 362 + log_info "Keyring status:" 363 + 364 + local keyring_id 365 + keyring_id=$(find_dm_verity_keyring) || true 366 + 367 + if [ -n "$keyring_id" ]; then 368 + echo " Keyring ID: $keyring_id" 369 + keyctl show "$keyring_id" 2>/dev/null || true 370 + grep '\.dm-verity' /proc/keys 2>/dev/null || true 371 + fi 372 + } 373 + 374 + list_keyring_keys() { 375 + log_info "Keys in .dm-verity keyring:" 376 + 377 + local keyring_id 378 + keyring_id=$(cat "$WORK_DIR/keyring_id" 2>/dev/null) || \ 379 + keyring_id=$(find_dm_verity_keyring) || true 380 + 381 + if [ -z "$keyring_id" ]; then 382 + log_warn "Could not find keyring" 383 + return 384 + fi 385 + 386 + # List all keys in the keyring 387 + local keys 388 + keys=$(keyctl list "$keyring_id" 2>/dev/null) 389 + if [ -z "$keys" ] || [ "$keys" = "keyring is empty" ]; then 390 + echo " (empty)" 391 + else 392 + echo "$keys" | while read -r line; do 393 + echo " $line" 394 + done 395 + 396 + # Show detailed info for each key 397 + log_info "Key details:" 398 + keyctl list "$keyring_id" 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]+$' | while read -r key_id; do 399 + echo " Key $key_id:" 400 + keyctl describe "$key_id" 2>/dev/null | sed 's/^/ /' 401 + done 402 + fi 403 + } 404 + 405 + generate_named_key() { 406 + local name="$1" 407 + local key_dir="$WORK_DIR/keys/$name" 408 + 409 + mkdir -p "$key_dir" 410 + 411 + # Log to stderr so it doesn't interfere with return value 412 + echo "[INFO] Generating key pair: $name" >&2 413 + 414 + # Generate private key 415 + openssl genrsa -out "$key_dir/private.pem" 2048 2>/dev/null 416 + 417 + # Create OpenSSL config for certificate extensions 418 + # Both subjectKeyIdentifier and authorityKeyIdentifier are needed for 419 + # the kernel to match keys in the keyring (especially for self-signed certs) 420 + cat > "$key_dir/openssl.cnf" << EOF 421 + [req] 422 + distinguished_name = req_distinguished_name 423 + x509_extensions = v3_ca 424 + prompt = no 425 + 426 + [req_distinguished_name] 427 + CN = dm-verity-test-$name 428 + 429 + [v3_ca] 430 + basicConstraints = critical,CA:FALSE 431 + keyUsage = digitalSignature 432 + subjectKeyIdentifier = hash 433 + authorityKeyIdentifier = keyid 434 + EOF 435 + 436 + # Generate self-signed certificate with proper extensions 437 + openssl req -new -x509 -key "$key_dir/private.pem" \ 438 + -out "$key_dir/cert.pem" -days 365 \ 439 + -config "$key_dir/openssl.cnf" 2>/dev/null 440 + 441 + # Convert certificate to DER format for kernel 442 + openssl x509 -in "$key_dir/cert.pem" -outform DER \ 443 + -out "$key_dir/cert.der" 444 + 445 + # Return the key directory path (only this goes to stdout) 446 + echo "$key_dir" 447 + } 448 + 449 + upload_named_key() { 450 + local name="$1" 451 + local key_dir="$2" 452 + 453 + local keyring_id 454 + keyring_id=$(cat "$WORK_DIR/keyring_id") 455 + 456 + log_info "Uploading key '$name' to keyring..." 457 + 458 + local key_id 459 + if key_id=$(keyctl padd asymmetric "$name" "$keyring_id" \ 460 + < "$key_dir/cert.der" 2>&1); then 461 + log_info "Key '$name' uploaded with ID: $key_id" 462 + echo "$key_id" > "$key_dir/key_id" 463 + return 0 464 + else 465 + log_error "Failed to upload key '$name': $key_id" 466 + return 1 467 + fi 468 + } 469 + 470 + # 471 + # Test: Verify sealed keyring rejects key additions 472 + # 473 + test_sealed_keyring_rejects_keys() { 474 + log_info "TEST: Verify sealed keyring rejects key additions" 475 + 476 + local keyring_id 477 + keyring_id=$(cat "$WORK_DIR/keyring_id") 478 + 479 + generate_keys 480 + 481 + # Try to add a key - should fail 482 + if keyctl padd asymmetric "dm-verity-test" "$keyring_id" \ 483 + < "$WORK_DIR/cert.der" 2>/dev/null; then 484 + log_fail "Key addition should have been rejected on sealed keyring" 485 + return 1 486 + else 487 + log_pass "Sealed keyring correctly rejected key addition" 488 + return 0 489 + fi 490 + } 491 + 492 + # 493 + # Test: Multiple keys in keyring 494 + # 495 + test_multiple_keys() { 496 + log_info "TEST: Multiple keys in keyring" 497 + 498 + local key1_dir key2_dir key3_dir 499 + 500 + # Generate three different keys 501 + key1_dir=$(generate_named_key "vendor-a") 502 + key2_dir=$(generate_named_key "vendor-b") 503 + key3_dir=$(generate_named_key "vendor-c") 504 + 505 + # Upload all three keys 506 + upload_named_key "vendor-a" "$key1_dir" || return 1 507 + upload_named_key "vendor-b" "$key2_dir" || return 1 508 + upload_named_key "vendor-c" "$key3_dir" || return 1 509 + 510 + log_info "" 511 + log_info "Keys in keyring before sealing:" 512 + list_keyring_keys 513 + show_keyring_status 514 + 515 + # Seal the keyring 516 + log_info "" 517 + seal_keyring 518 + 519 + # List keys after sealing 520 + log_info "" 521 + log_info "Keys in keyring after sealing:" 522 + list_keyring_keys 523 + show_keyring_status 524 + 525 + log_pass "Key upload and keyring sealing succeeded" 526 + 527 + # Create test device 528 + log_info "" 529 + create_test_device 530 + create_verity_hash 531 + 532 + # Test 1: Sign with key1, should verify successfully 533 + log_info "" 534 + log_info "Sub-test: Verify with vendor-a key" 535 + if ! sign_root_hash_with_key "$key1_dir"; then 536 + log_fail "Failed to sign with vendor-a key" 537 + return 1 538 + fi 539 + if activate_verity_device "yes"; then 540 + log_pass "Verification with vendor-a key succeeded" 541 + deactivate_verity_device 542 + else 543 + log_fail "Verification with vendor-a key should succeed" 544 + return 1 545 + fi 546 + 547 + # Test 2: Sign with key2, should also verify successfully 548 + log_info "" 549 + log_info "Sub-test: Verify with vendor-b key" 550 + if ! sign_root_hash_with_key "$key2_dir"; then 551 + log_fail "Failed to sign with vendor-b key" 552 + return 1 553 + fi 554 + if activate_verity_device "yes"; then 555 + log_pass "Verification with vendor-b key succeeded" 556 + deactivate_verity_device 557 + else 558 + log_fail "Verification with vendor-b key should succeed" 559 + return 1 560 + fi 561 + 562 + # Test 3: Sign with key3, should also verify successfully 563 + log_info "" 564 + log_info "Sub-test: Verify with vendor-c key" 565 + if ! sign_root_hash_with_key "$key3_dir"; then 566 + log_fail "Failed to sign with vendor-c key" 567 + return 1 568 + fi 569 + if activate_verity_device "yes"; then 570 + log_pass "Verification with vendor-c key succeeded" 571 + deactivate_verity_device 572 + else 573 + log_fail "Verification with vendor-c key should succeed" 574 + return 1 575 + fi 576 + 577 + # Test 4: Generate a key NOT in the keyring, should fail 578 + log_info "" 579 + log_info "Sub-test: Verify with unknown key (should fail)" 580 + local unknown_key_dir 581 + unknown_key_dir=$(generate_named_key "unknown-vendor") 582 + if ! sign_root_hash_with_key "$unknown_key_dir"; then 583 + log_fail "Failed to sign with unknown-vendor key" 584 + return 1 585 + fi 586 + if activate_verity_device "yes"; then 587 + log_fail "Verification with unknown key should fail" 588 + deactivate_verity_device 589 + return 1 590 + else 591 + log_pass "Verification with unknown key correctly rejected" 592 + fi 593 + 594 + log_info "" 595 + log_pass "Multiple keys test completed successfully" 596 + return 0 597 + } 598 + 599 + sign_root_hash_with_key() { 600 + local key_dir="$1" 601 + 602 + local root_hash 603 + root_hash=$(cat "$WORK_DIR/root_hash") 604 + 605 + # Create the data to sign (hex string, not binary) 606 + echo -n "$root_hash" > "$WORK_DIR/root_hash.txt" 607 + 608 + # Debug: show exactly what we're signing 609 + log_info "Root hash (hex): $root_hash" 610 + log_info "Root hash hex string size: $(wc -c < "$WORK_DIR/root_hash.txt") bytes" 611 + 612 + # Create detached PKCS#7 signature 613 + if ! create_detached_signature "$WORK_DIR/root_hash.txt" "$WORK_DIR/root_hash.p7s" \ 614 + "$key_dir/cert.pem" "$key_dir/private.pem"; then 615 + log_error "Failed to sign root hash with key from $key_dir" 616 + return 1 617 + fi 618 + 619 + # Debug: show signing certificate info 620 + log_info "Signed with certificate:" 621 + openssl x509 -in "$key_dir/cert.pem" -noout -subject 2>/dev/null | sed 's/^/ /' 622 + 623 + # Debug: verify signature locally 624 + # -nointern: cert not in signature, use -certfile 625 + # -noverify: skip certificate chain validation (self-signed) 626 + if openssl smime -verify -binary -inform der -nointern -noverify \ 627 + -in "$WORK_DIR/root_hash.p7s" \ 628 + -content "$WORK_DIR/root_hash.txt" \ 629 + -certfile "$key_dir/cert.pem" \ 630 + -out /dev/null 2>/dev/null; then 631 + log_info "Local signature verification: PASSED" 632 + else 633 + log_warn "Local signature verification: FAILED" 634 + fi 635 + return 0 636 + } 637 + 638 + # 639 + # Test: Verify corrupted signatures are rejected 640 + # 641 + test_corrupted_signature() { 642 + log_info "TEST: Verify corrupted signatures are rejected" 643 + 644 + # This test requires a valid setup from test_multiple_keys or similar 645 + # It modifies the signature file and verifies rejection 646 + 647 + if [ ! -f "$WORK_DIR/root_hash.p7s" ]; then 648 + log_warn "No signature file found, skipping corrupted signature test" 649 + return 0 650 + fi 651 + 652 + # Save original signature 653 + cp "$WORK_DIR/root_hash.p7s" "$WORK_DIR/root_hash.p7s.orig" 654 + 655 + # Test 1: Truncated signature 656 + log_info "Sub-test: Truncated signature (should fail)" 657 + head -c 100 "$WORK_DIR/root_hash.p7s.orig" > "$WORK_DIR/root_hash.p7s" 658 + if activate_verity_device "yes"; then 659 + log_fail "Truncated signature should be rejected" 660 + deactivate_verity_device 661 + cp "$WORK_DIR/root_hash.p7s.orig" "$WORK_DIR/root_hash.p7s" 662 + return 1 663 + else 664 + log_pass "Truncated signature correctly rejected" 665 + fi 666 + 667 + # Test 2: Corrupted signature (flip some bytes) 668 + log_info "Sub-test: Corrupted signature bytes (should fail)" 669 + cp "$WORK_DIR/root_hash.p7s.orig" "$WORK_DIR/root_hash.p7s" 670 + # Corrupt bytes in the middle of the signature 671 + local sig_size 672 + sig_size=$(wc -c < "$WORK_DIR/root_hash.p7s") 673 + local corrupt_offset=$((sig_size / 2)) 674 + printf '\xff\xff\xff\xff' | dd of="$WORK_DIR/root_hash.p7s" bs=1 seek=$corrupt_offset conv=notrunc 2>/dev/null 675 + if activate_verity_device "yes"; then 676 + log_fail "Corrupted signature should be rejected" 677 + deactivate_verity_device 678 + cp "$WORK_DIR/root_hash.p7s.orig" "$WORK_DIR/root_hash.p7s" 679 + return 1 680 + else 681 + log_pass "Corrupted signature correctly rejected" 682 + fi 683 + 684 + # Test 3: Signature over wrong data (sign different content) 685 + log_info "Sub-test: Signature over wrong data (should fail)" 686 + # Create a different root hash (all zeros as hex string) 687 + printf '%064d' 0 > "$WORK_DIR/wrong_hash.txt" 688 + # Get the first key directory that was used 689 + local key_dir="$WORK_DIR/keys/vendor-a" 690 + if [ -d "$key_dir" ]; then 691 + create_detached_signature "$WORK_DIR/wrong_hash.txt" "$WORK_DIR/root_hash.p7s" \ 692 + "$key_dir/cert.pem" "$key_dir/private.pem" 693 + if activate_verity_device "yes"; then 694 + log_fail "Signature over wrong data should be rejected" 695 + deactivate_verity_device 696 + cp "$WORK_DIR/root_hash.p7s.orig" "$WORK_DIR/root_hash.p7s" 697 + return 1 698 + else 699 + log_pass "Signature over wrong data correctly rejected" 700 + fi 701 + else 702 + log_warn "Key directory not found, skipping wrong data test" 703 + fi 704 + 705 + # Restore original signature 706 + cp "$WORK_DIR/root_hash.p7s.orig" "$WORK_DIR/root_hash.p7s" 707 + 708 + log_pass "Corrupted signature test completed successfully" 709 + return 0 710 + } 711 + 712 + # 713 + # Test: Verify keyring is sealed when keyring_unsealed=0 714 + # 715 + test_keyring_sealed_by_default() { 716 + log_info "TEST: Verify keyring is sealed by default (keyring_unsealed=0)" 717 + 718 + local keyring_id 719 + keyring_id=$(cat "$WORK_DIR/keyring_id") 720 + 721 + log_info "Current keyring state (should be empty and sealed):" 722 + list_keyring_keys 723 + show_keyring_status 724 + 725 + generate_keys 726 + 727 + # Try to add a key - should fail if keyring is sealed 728 + log_info "Attempting to add key to sealed keyring..." 729 + if keyctl padd asymmetric "dm-verity-test" "$keyring_id" \ 730 + < "$WORK_DIR/cert.der" 2>/dev/null; then 731 + log_fail "Keyring should be sealed when keyring_unsealed=0" 732 + list_keyring_keys 733 + return 1 734 + else 735 + log_pass "Keyring is correctly sealed when keyring_unsealed=0" 736 + log_info "Keyring state after failed add attempt:" 737 + list_keyring_keys 738 + return 0 739 + fi 740 + } 741 + 742 + # 743 + # Test: Verify dm-verity keyring is inactive when sealed empty 744 + # 745 + test_keyring_inactive_when_empty() { 746 + log_info "TEST: Verify dm-verity keyring is inactive when sealed empty" 747 + 748 + # When keyring_unsealed=0, the keyring is sealed immediately while empty 749 + # This means it should NOT be used for verification (nr_leaves_on_tree=0) 750 + 751 + log_info "Keyring state (should be empty and sealed):" 752 + list_keyring_keys 753 + show_keyring_status 754 + 755 + create_test_device 756 + create_verity_hash 757 + 758 + # Without any keys in the dm-verity keyring, and with it sealed, 759 + # verification should fall through to the secondary/platform keyrings 760 + # and likely succeed (if require_signatures=0) or fail (if =1) 761 + 762 + log_info "Sub-test: Device activation with sealed empty keyring" 763 + if [ "$REQUIRE_SIGNATURES" = "Y" ] || [ "$REQUIRE_SIGNATURES" = "1" ]; then 764 + if activate_verity_device "no"; then 765 + log_fail "Device should NOT activate without signature when require_signatures=1" 766 + deactivate_verity_device 767 + return 1 768 + else 769 + log_pass "Device correctly rejected (require_signatures=1, no valid signature)" 770 + fi 771 + else 772 + if activate_verity_device "no"; then 773 + log_pass "Device activated (require_signatures=0, empty dm-verity keyring is inactive)" 774 + deactivate_verity_device 775 + else 776 + log_fail "Device should activate when require_signatures=0" 777 + return 1 778 + fi 779 + fi 780 + 781 + return 0 782 + } 783 + 784 + main() { 785 + local rc=0 786 + 787 + log_info "=== dm-verity keyring test ===" 788 + log_info "" 789 + 790 + # Create work directory 791 + WORK_DIR=$(mktemp -d -t dm-verity-test.XXXXXX) 792 + log_info "Work directory: $WORK_DIR" 793 + 794 + check_requirements 795 + 796 + # 797 + # Test 1: UNSEALED keyring mode (keyring_unsealed=1) 798 + # 799 + log_info "" 800 + log_info "========================================" 801 + log_info "=== TEST MODE: UNSEALED KEYRING ===" 802 + log_info "========================================" 803 + log_info "" 804 + 805 + load_dm_verity_module 1 1 # keyring_unsealed=1, require_signatures=1 806 + show_keyring_status 807 + 808 + log_info "" 809 + if ! test_multiple_keys; then 810 + rc=1 811 + fi 812 + 813 + # After sealing, verify it rejects new keys 814 + log_info "" 815 + if ! test_sealed_keyring_rejects_keys; then 816 + rc=1 817 + fi 818 + 819 + # Test corrupted signatures are rejected 820 + log_info "" 821 + if ! test_corrupted_signature; then 822 + rc=1 823 + fi 824 + 825 + # Clean up devices before reloading module 826 + deactivate_verity_device 827 + if [ -n "$DATA_DEV" ] && [[ "$DATA_DEV" == /dev/loop* ]]; then 828 + losetup -d "$DATA_DEV" 2>/dev/null || true 829 + DATA_DEV="" 830 + fi 831 + if [ -n "$HASH_DEV" ] && [[ "$HASH_DEV" == /dev/loop* ]]; then 832 + losetup -d "$HASH_DEV" 2>/dev/null || true 833 + HASH_DEV="" 834 + fi 835 + 836 + # 837 + # Test 2: SEALED keyring mode (keyring_unsealed=0, default) 838 + # 839 + log_info "" 840 + log_info "========================================" 841 + log_info "=== TEST MODE: SEALED KEYRING (default) ===" 842 + log_info "========================================" 843 + log_info "" 844 + 845 + load_dm_verity_module 0 0 # keyring_unsealed=0, require_signatures=0 846 + show_keyring_status 847 + 848 + log_info "" 849 + if ! test_keyring_sealed_by_default; then 850 + rc=1 851 + fi 852 + 853 + log_info "" 854 + if ! test_keyring_inactive_when_empty; then 855 + rc=1 856 + fi 857 + 858 + # 859 + # Summary 860 + # 861 + log_info "" 862 + log_info "========================================" 863 + if [ $rc -eq 0 ]; then 864 + log_info "=== All tests PASSED ===" 865 + else 866 + log_error "=== Some tests FAILED ===" 867 + fi 868 + log_info "========================================" 869 + 870 + return $rc 871 + } 872 + 873 + main "$@"

Configure Feed

Configure Feed