dm-bufio: avoid redundant buffer_tree lookups

+92 -56

1 changed file

expand all

drivers

dm-bufio.c

+92 -56

drivers/md/dm-bufio.c

··· 401 401 return dm_hash_locks_index(block, num_locks); 402 402 } 403 403 404 - static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t block) 404 + /* Get the buffer tree in the cache for the given block. Doesn't lock it. */ 405 + static inline struct buffer_tree *cache_get_tree(struct dm_buffer_cache *bc, 406 + sector_t block) 405 407 { 406 - if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 407 - read_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 408 - else 409 - down_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 408 + return &bc->trees[cache_index(block, bc->num_locks)]; 410 409 } 411 410 412 - static inline void cache_read_unlock(struct dm_buffer_cache *bc, sector_t block) 411 + /* Lock the given buffer tree in the cache for reading. */ 412 + static inline void cache_read_lock(struct dm_buffer_cache *bc, 413 + struct buffer_tree *tree) 413 414 { 414 415 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 415 - read_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 416 + read_lock_bh(&tree->u.spinlock); 416 417 else 417 - up_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 418 + down_read(&tree->u.lock); 418 419 } 419 420 420 - static inline void cache_write_lock(struct dm_buffer_cache *bc, sector_t block) 421 + /* Unlock the given buffer tree in the cache for reading. */ 422 + static inline void cache_read_unlock(struct dm_buffer_cache *bc, 423 + struct buffer_tree *tree) 421 424 { 422 425 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 423 - write_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 426 + read_unlock_bh(&tree->u.spinlock); 424 427 else 425 - down_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 428 + up_read(&tree->u.lock); 426 429 } 427 430 428 - static inline void cache_write_unlock(struct dm_buffer_cache *bc, sector_t block) 431 + /* Lock the given buffer tree in the cache for writing. */ 432 + static inline void cache_write_lock(struct dm_buffer_cache *bc, 433 + struct buffer_tree *tree) 429 434 { 430 435 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 431 - write_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 436 + write_lock_bh(&tree->u.spinlock); 432 437 else 433 - up_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 438 + down_write(&tree->u.lock); 439 + } 440 + 441 + /* Unlock the given buffer tree in the cache for writing. */ 442 + static inline void cache_write_unlock(struct dm_buffer_cache *bc, 443 + struct buffer_tree *tree) 444 + { 445 + if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 446 + write_unlock_bh(&tree->u.spinlock); 447 + else 448 + up_write(&tree->u.lock); 434 449 } 435 450 436 451 /* ··· 617 602 WRITE_ONCE(b->last_accessed, jiffies); 618 603 } 619 604 620 - static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, sector_t block) 605 + static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, 606 + struct buffer_tree *tree, sector_t block) 621 607 { 622 608 struct dm_buffer *b; 623 609 624 - cache_read_lock(bc, block); 625 - b = __cache_get(&bc->trees[cache_index(block, bc->num_locks)].root, block); 610 + /* Assuming tree == cache_get_tree(bc, block) */ 611 + cache_read_lock(bc, tree); 612 + b = __cache_get(&tree->root, block); 626 613 if (b) { 627 614 lru_reference(&b->lru); 628 615 __cache_inc_buffer(b); 629 616 } 630 - cache_read_unlock(bc, block); 617 + cache_read_unlock(bc, tree); 631 618 632 619 return b; 633 620 } ··· 680 663 681 664 b = le_to_buffer(le); 682 665 /* __evict_pred will have locked the appropriate tree. */ 683 - rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); 666 + rb_erase(&b->node, &cache_get_tree(bc, b->block)->root); 684 667 685 668 return b; 686 669 } ··· 703 686 /* 704 687 * Mark a buffer as clean or dirty. Not threadsafe. 705 688 */ 706 - static void cache_mark(struct dm_buffer_cache *bc, struct dm_buffer *b, int list_mode) 689 + static void cache_mark(struct dm_buffer_cache *bc, struct buffer_tree *tree, 690 + struct dm_buffer *b, int list_mode) 707 691 { 708 - cache_write_lock(bc, b->block); 692 + /* Assuming tree == cache_get_tree(bc, b->block) */ 693 + cache_write_lock(bc, tree); 709 694 if (list_mode != b->list_mode) { 710 695 lru_remove(&bc->lru[b->list_mode], &b->lru); 711 696 b->list_mode = list_mode; 712 697 lru_insert(&bc->lru[b->list_mode], &b->lru); 713 698 } 714 - cache_write_unlock(bc, b->block); 699 + cache_write_unlock(bc, tree); 715 700 } 716 701 717 702 /*--------------*/ ··· 839 820 return true; 840 821 } 841 822 842 - static bool cache_insert(struct dm_buffer_cache *bc, struct dm_buffer *b) 823 + static bool cache_insert(struct dm_buffer_cache *bc, struct buffer_tree *tree, 824 + struct dm_buffer *b) 843 825 { 844 826 bool r; 845 827 846 828 if (WARN_ON_ONCE(b->list_mode >= LIST_SIZE)) 847 829 return false; 848 830 849 - cache_write_lock(bc, b->block); 831 + /* Assuming tree == cache_get_tree(bc, b->block) */ 832 + cache_write_lock(bc, tree); 850 833 BUG_ON(atomic_read(&b->hold_count) != 1); 851 - r = __cache_insert(&bc->trees[cache_index(b->block, bc->num_locks)].root, b); 834 + r = __cache_insert(&tree->root, b); 852 835 if (r) 853 836 lru_insert(&bc->lru[b->list_mode], &b->lru); 854 - cache_write_unlock(bc, b->block); 837 + cache_write_unlock(bc, tree); 855 838 856 839 return r; 857 840 } ··· 866 845 * 867 846 * Not threadsafe. 868 847 */ 869 - static bool cache_remove(struct dm_buffer_cache *bc, struct dm_buffer *b) 848 + static bool cache_remove(struct dm_buffer_cache *bc, struct buffer_tree *tree, 849 + struct dm_buffer *b) 870 850 { 871 851 bool r; 872 852 873 - cache_write_lock(bc, b->block); 853 + /* Assuming tree == cache_get_tree(bc, b->block) */ 854 + cache_write_lock(bc, tree); 874 855 875 856 if (atomic_read(&b->hold_count) != 1) { 876 857 r = false; 877 858 } else { 878 859 r = true; 879 - rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); 860 + rb_erase(&b->node, &tree->root); 880 861 lru_remove(&bc->lru[b->list_mode], &b->lru); 881 862 } 882 863 883 - cache_write_unlock(bc, b->block); 864 + cache_write_unlock(bc, tree); 884 865 885 866 return r; 886 867 } ··· 1748 1725 *-------------------------------------------------------------- 1749 1726 */ 1750 1727 1751 - static void cache_put_and_wake(struct dm_bufio_client *c, struct dm_buffer *b) 1728 + static void cache_put_and_wake(struct dm_bufio_client *c, 1729 + struct buffer_tree *tree, struct dm_buffer *b) 1752 1730 { 1753 1731 bool wake; 1754 1732 1755 - cache_read_lock(&c->cache, b->block); 1733 + /* Assuming tree == cache_get_tree(&c->cache, b->block) */ 1734 + cache_read_lock(&c->cache, tree); 1756 1735 BUG_ON(!atomic_read(&b->hold_count)); 1757 1736 wake = atomic_dec_and_test(&b->hold_count); 1758 - cache_read_unlock(&c->cache, b->block); 1737 + cache_read_unlock(&c->cache, tree); 1759 1738 1760 1739 /* 1761 1740 * Relying on waitqueue_active() is racey, but we sleep ··· 1771 1746 * This assumes you have already checked the cache to see if the buffer 1772 1747 * is already present (it will recheck after dropping the lock for allocation). 1773 1748 */ 1774 - static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block, 1749 + static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, 1750 + struct buffer_tree *tree, sector_t block, 1775 1751 enum new_flag nf, int *need_submit, 1776 1752 struct list_head *write_list) 1777 1753 { ··· 1792 1766 * We've had a period where the mutex was unlocked, so need to 1793 1767 * recheck the buffer tree. 1794 1768 */ 1795 - b = cache_get(&c->cache, block); 1769 + b = cache_get(&c->cache, tree, block); 1796 1770 if (b) { 1797 1771 __free_buffer_wake(new_b); 1798 1772 goto found_buffer; ··· 1820 1794 * is set. Otherwise another thread could get it and use 1821 1795 * it before it had been read. 1822 1796 */ 1823 - cache_insert(&c->cache, b); 1797 + cache_insert(&c->cache, tree, b); 1824 1798 1825 1799 return b; 1826 1800 1827 1801 found_buffer: 1828 1802 if (nf == NF_PREFETCH) { 1829 - cache_put_and_wake(c, b); 1803 + cache_put_and_wake(c, tree, b); 1830 1804 return NULL; 1831 1805 } 1832 1806 ··· 1838 1812 * the same buffer, it would deadlock if we waited. 1839 1813 */ 1840 1814 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { 1841 - cache_put_and_wake(c, b); 1815 + cache_put_and_wake(c, tree, b); 1842 1816 return NULL; 1843 1817 } 1844 1818 ··· 1872 1846 enum new_flag nf, struct dm_buffer **bp, 1873 1847 unsigned short ioprio) 1874 1848 { 1849 + struct buffer_tree *tree; 1875 1850 int need_submit = 0; 1876 1851 struct dm_buffer *b; 1877 1852 ··· 1884 1857 * Fast path, hopefully the block is already in the cache. No need 1885 1858 * to get the client lock for this. 1886 1859 */ 1887 - b = cache_get(&c->cache, block); 1860 + tree = cache_get_tree(&c->cache, block); 1861 + b = cache_get(&c->cache, tree, block); 1888 1862 if (b) { 1889 1863 if (nf == NF_PREFETCH) { 1890 - cache_put_and_wake(c, b); 1864 + cache_put_and_wake(c, tree, b); 1891 1865 return NULL; 1892 1866 } 1893 1867 ··· 1900 1872 * the same buffer, it would deadlock if we waited. 1901 1873 */ 1902 1874 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { 1903 - cache_put_and_wake(c, b); 1875 + cache_put_and_wake(c, tree, b); 1904 1876 return NULL; 1905 1877 } 1906 1878 } ··· 1910 1882 return NULL; 1911 1883 1912 1884 dm_bufio_lock(c); 1913 - b = __bufio_new(c, block, nf, &need_submit, &write_list); 1885 + b = __bufio_new(c, tree, block, nf, &need_submit, &write_list); 1914 1886 dm_bufio_unlock(c); 1915 1887 } 1916 1888 ··· 1997 1969 blk_start_plug(&plug); 1998 1970 1999 1971 for (; n_blocks--; block++) { 2000 - int need_submit; 1972 + struct buffer_tree *tree; 2001 1973 struct dm_buffer *b; 1974 + int need_submit; 2002 1975 2003 - b = cache_get(&c->cache, block); 1976 + tree = cache_get_tree(&c->cache, block); 1977 + b = cache_get(&c->cache, tree, block); 2004 1978 if (b) { 2005 1979 /* already in cache */ 2006 - cache_put_and_wake(c, b); 1980 + cache_put_and_wake(c, tree, b); 2007 1981 continue; 2008 1982 } 2009 1983 2010 1984 dm_bufio_lock(c); 2011 - b = __bufio_new(c, block, NF_PREFETCH, &need_submit, 1985 + b = __bufio_new(c, tree, block, NF_PREFETCH, &need_submit, 2012 1986 &write_list); 2013 1987 if (unlikely(!list_empty(&write_list))) { 2014 1988 dm_bufio_unlock(c); ··· 2055 2025 void dm_bufio_release(struct dm_buffer *b) 2056 2026 { 2057 2027 struct dm_bufio_client *c = b->c; 2028 + struct buffer_tree *tree = cache_get_tree(&c->cache, b->block); 2058 2029 2059 2030 /* 2060 2031 * If there were errors on the buffer, and the buffer is not ··· 2069 2038 dm_bufio_lock(c); 2070 2039 2071 2040 /* cache remove can fail if there are other holders */ 2072 - if (cache_remove(&c->cache, b)) { 2041 + if (cache_remove(&c->cache, tree, b)) { 2073 2042 __free_buffer_wake(b); 2074 2043 dm_bufio_unlock(c); 2075 2044 return; ··· 2078 2047 dm_bufio_unlock(c); 2079 2048 } 2080 2049 2081 - cache_put_and_wake(c, b); 2050 + cache_put_and_wake(c, tree, b); 2082 2051 } 2083 2052 EXPORT_SYMBOL_GPL(dm_bufio_release); 2084 2053 ··· 2097 2066 if (!test_and_set_bit(B_DIRTY, &b->state)) { 2098 2067 b->dirty_start = start; 2099 2068 b->dirty_end = end; 2100 - cache_mark(&c->cache, b, LIST_DIRTY); 2069 + cache_mark(&c->cache, cache_get_tree(&c->cache, b->block), b, 2070 + LIST_DIRTY); 2101 2071 } else { 2102 2072 if (start < b->dirty_start) 2103 2073 b->dirty_start = start; ··· 2163 2131 lru_iter_begin(&c->cache.lru[LIST_DIRTY], &it); 2164 2132 while ((e = lru_iter_next(&it, is_writing, c))) { 2165 2133 struct dm_buffer *b = le_to_buffer(e); 2134 + struct buffer_tree *tree; 2166 2135 __cache_inc_buffer(b); 2167 2136 2168 2137 BUG_ON(test_bit(B_READING, &b->state)); ··· 2177 2144 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); 2178 2145 } 2179 2146 2180 - if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state)) 2181 - cache_mark(&c->cache, b, LIST_CLEAN); 2147 + tree = cache_get_tree(&c->cache, b->block); 2182 2148 2183 - cache_put_and_wake(c, b); 2149 + if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state)) 2150 + cache_mark(&c->cache, tree, b, LIST_CLEAN); 2151 + 2152 + cache_put_and_wake(c, tree, b); 2184 2153 2185 2154 cond_resched(); 2186 2155 } ··· 2250 2215 2251 2216 static void forget_buffer(struct dm_bufio_client *c, sector_t block) 2252 2217 { 2218 + struct buffer_tree *tree = cache_get_tree(&c->cache, block); 2253 2219 struct dm_buffer *b; 2254 2220 2255 - b = cache_get(&c->cache, block); 2221 + b = cache_get(&c->cache, tree, block); 2256 2222 if (b) { 2257 2223 if (likely(!smp_load_acquire(&b->state))) { 2258 - if (cache_remove(&c->cache, b)) 2224 + if (cache_remove(&c->cache, tree, b)) 2259 2225 __free_buffer_wake(b); 2260 2226 else 2261 - cache_put_and_wake(c, b); 2227 + cache_put_and_wake(c, tree, b); 2262 2228 } else { 2263 - cache_put_and_wake(c, b); 2229 + cache_put_and_wake(c, tree, b); 2264 2230 } 2265 2231 } 2266 2232 }

Configure Feed

Configure Feed