maple_tree: remove maple big node and subtree structs

-1184

1 changed file

expand all

lib

maple_tree.c

-1184

lib/maple_tree.c

··· 133 133 }; 134 134 #define mt_min_slot_count(x) mt_min_slots[mte_node_type(x)] 135 135 136 - #define MAPLE_BIG_NODE_SLOTS (MAPLE_RANGE64_SLOTS * 2 + 2) 137 - #define MAPLE_BIG_NODE_GAPS (MAPLE_ARANGE64_SLOTS * 2 + 1) 138 - 139 - struct maple_big_node { 140 - unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1]; 141 - union { 142 - struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS]; 143 - struct { 144 - unsigned long padding[MAPLE_BIG_NODE_GAPS]; 145 - unsigned long gap[MAPLE_BIG_NODE_GAPS]; 146 - }; 147 - }; 148 - unsigned char b_end; 149 - enum maple_type type; 150 - }; 151 - 152 - /* 153 - * The maple_subtree_state is used to build a tree to replace a segment of an 154 - * existing tree in a more atomic way. Any walkers of the older tree will hit a 155 - * dead node and restart on updates. 156 - */ 157 - struct maple_subtree_state { 158 - struct ma_state *orig_l; /* Original left side of subtree */ 159 - struct ma_state *orig_r; /* Original right side of subtree */ 160 - struct ma_state *l; /* New left side of subtree */ 161 - struct ma_state *m; /* New middle of subtree (rare) */ 162 - struct ma_state *r; /* New right side of subtree */ 163 - struct ma_topiary *free; /* nodes to be freed */ 164 - struct ma_topiary *destroy; /* Nodes to be destroyed (walked and freed) */ 165 - struct maple_big_node *bn; 166 - }; 167 - 168 - #ifdef CONFIG_KASAN_STACK 169 - /* Prevent mas_wr_bnode() from exceeding the stack frame limit */ 170 - #define noinline_for_kasan noinline_for_stack 171 - #else 172 - #define noinline_for_kasan inline 173 - #endif 174 - 175 136 /* Functions */ 176 137 static inline struct maple_node *mt_alloc_one(gfp_t gfp) 177 138 { ··· 1631 1670 } 1632 1671 1633 1672 /* 1634 - * mab_shift_right() - Shift the data in mab right. Note, does not clean out the 1635 - * old data or set b_node->b_end. 1636 - * @b_node: the maple_big_node 1637 - * @shift: the shift count 1638 - */ 1639 - static inline void mab_shift_right(struct maple_big_node *b_node, 1640 - unsigned char shift) 1641 - { 1642 - unsigned long size = b_node->b_end * sizeof(unsigned long); 1643 - 1644 - memmove(b_node->pivot + shift, b_node->pivot, size); 1645 - memmove(b_node->slot + shift, b_node->slot, size); 1646 - if (b_node->type == maple_arange_64) 1647 - memmove(b_node->gap + shift, b_node->gap, size); 1648 - } 1649 - 1650 - /* 1651 - * mab_middle_node() - Check if a middle node is needed (unlikely) 1652 - * @b_node: the maple_big_node that contains the data. 1653 - * @split: the potential split location 1654 - * @slot_count: the size that can be stored in a single node being considered. 1655 - * 1656 - * Return: true if a middle node is required. 1657 - */ 1658 - static inline bool mab_middle_node(struct maple_big_node *b_node, int split, 1659 - unsigned char slot_count) 1660 - { 1661 - unsigned char size = b_node->b_end; 1662 - 1663 - if (size >= 2 * slot_count) 1664 - return true; 1665 - 1666 - if (!b_node->slot[split] && (size >= 2 * slot_count - 1)) 1667 - return true; 1668 - 1669 - return false; 1670 - } 1671 - 1672 - /* 1673 - * mab_no_null_split() - ensure the split doesn't fall on a NULL 1674 - * @b_node: the maple_big_node with the data 1675 - * @split: the suggested split location 1676 - * @slot_count: the number of slots in the node being considered. 1677 - * 1678 - * Return: the split location. 1679 - */ 1680 - static inline int mab_no_null_split(struct maple_big_node *b_node, 1681 - unsigned char split, unsigned char slot_count) 1682 - { 1683 - if (!b_node->slot[split]) { 1684 - /* 1685 - * If the split is less than the max slot && the right side will 1686 - * still be sufficient, then increment the split on NULL. 1687 - */ 1688 - if ((split < slot_count - 1) && 1689 - (b_node->b_end - split) > (mt_min_slots[b_node->type])) 1690 - split++; 1691 - else 1692 - split--; 1693 - } 1694 - return split; 1695 - } 1696 - 1697 - /* 1698 - * mab_calc_split() - Calculate the split location and if there needs to be two 1699 - * splits. 1700 - * @mas: The maple state 1701 - * @bn: The maple_big_node with the data 1702 - * @mid_split: The second split, if required. 0 otherwise. 1703 - * 1704 - * Return: The first split location. The middle split is set in @mid_split. 1705 - */ 1706 - static inline int mab_calc_split(struct ma_state *mas, 1707 - struct maple_big_node *bn, unsigned char *mid_split) 1708 - { 1709 - unsigned char b_end = bn->b_end; 1710 - int split = b_end / 2; /* Assume equal split. */ 1711 - unsigned char slot_count = mt_slots[bn->type]; 1712 - 1713 - /* 1714 - * To support gap tracking, all NULL entries are kept together and a node cannot 1715 - * end on a NULL entry, with the exception of the left-most leaf. The 1716 - * limitation means that the split of a node must be checked for this condition 1717 - * and be able to put more data in one direction or the other. 1718 - * 1719 - * Although extremely rare, it is possible to enter what is known as the 3-way 1720 - * split scenario. The 3-way split comes about by means of a store of a range 1721 - * that overwrites the end and beginning of two full nodes. The result is a set 1722 - * of entries that cannot be stored in 2 nodes. Sometimes, these two nodes can 1723 - * also be located in different parent nodes which are also full. This can 1724 - * carry upwards all the way to the root in the worst case. 1725 - */ 1726 - if (unlikely(mab_middle_node(bn, split, slot_count))) { 1727 - split = b_end / 3; 1728 - *mid_split = split * 2; 1729 - } else { 1730 - *mid_split = 0; 1731 - } 1732 - 1733 - /* Avoid ending a node on a NULL entry */ 1734 - split = mab_no_null_split(bn, split, slot_count); 1735 - 1736 - if (unlikely(*mid_split)) 1737 - *mid_split = mab_no_null_split(bn, *mid_split, slot_count); 1738 - 1739 - return split; 1740 - } 1741 - 1742 - /* 1743 - * mas_mab_cp() - Copy data from a maple state inclusively to a maple_big_node 1744 - * and set @b_node->b_end to the next free slot. 1745 - * @mas: The maple state 1746 - * @mas_start: The starting slot to copy 1747 - * @mas_end: The end slot to copy (inclusively) 1748 - * @b_node: The maple_big_node to place the data 1749 - * @mab_start: The starting location in maple_big_node to store the data. 1750 - */ 1751 - static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start, 1752 - unsigned char mas_end, struct maple_big_node *b_node, 1753 - unsigned char mab_start) 1754 - { 1755 - enum maple_type mt; 1756 - struct maple_node *node; 1757 - void __rcu **slots; 1758 - unsigned long *pivots, *gaps; 1759 - int i = mas_start, j = mab_start; 1760 - unsigned char piv_end; 1761 - 1762 - node = mas_mn(mas); 1763 - mt = mte_node_type(mas->node); 1764 - pivots = ma_pivots(node, mt); 1765 - if (!i) { 1766 - b_node->pivot[j] = pivots[i++]; 1767 - if (unlikely(i > mas_end)) 1768 - goto complete; 1769 - j++; 1770 - } 1771 - 1772 - piv_end = min(mas_end, mt_pivots[mt]); 1773 - for (; i < piv_end; i++, j++) { 1774 - b_node->pivot[j] = pivots[i]; 1775 - if (unlikely(!b_node->pivot[j])) 1776 - goto complete; 1777 - 1778 - if (unlikely(mas->max == b_node->pivot[j])) 1779 - goto complete; 1780 - } 1781 - 1782 - b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt); 1783 - 1784 - complete: 1785 - b_node->b_end = ++j; 1786 - j -= mab_start; 1787 - slots = ma_slots(node, mt); 1788 - memcpy(b_node->slot + mab_start, slots + mas_start, sizeof(void *) * j); 1789 - if (!ma_is_leaf(mt) && mt_is_alloc(mas->tree)) { 1790 - gaps = ma_gaps(node, mt); 1791 - memcpy(b_node->gap + mab_start, gaps + mas_start, 1792 - sizeof(unsigned long) * j); 1793 - } 1794 - } 1795 - 1796 - /* 1797 1673 * mas_leaf_set_meta() - Set the metadata of a leaf if possible. 1798 1674 * @node: The maple node 1799 1675 * @mt: The maple type ··· 1641 1843 { 1642 1844 if (end < mt_slots[mt] - 1) 1643 1845 ma_set_meta(node, mt, 0, end); 1644 - } 1645 - 1646 - /* 1647 - * mab_mas_cp() - Copy data from maple_big_node to a maple encoded node. 1648 - * @b_node: the maple_big_node that has the data 1649 - * @mab_start: the start location in @b_node. 1650 - * @mab_end: The end location in @b_node (inclusively) 1651 - * @mas: The maple state with the maple encoded node. 1652 - */ 1653 - static inline void mab_mas_cp(struct maple_big_node *b_node, 1654 - unsigned char mab_start, unsigned char mab_end, 1655 - struct ma_state *mas, bool new_max) 1656 - { 1657 - int i, j = 0; 1658 - enum maple_type mt = mte_node_type(mas->node); 1659 - struct maple_node *node = mte_to_node(mas->node); 1660 - void __rcu **slots = ma_slots(node, mt); 1661 - unsigned long *pivots = ma_pivots(node, mt); 1662 - unsigned long *gaps = NULL; 1663 - unsigned char end; 1664 - 1665 - if (mab_end - mab_start > mt_pivots[mt]) 1666 - mab_end--; 1667 - 1668 - if (!pivots[mt_pivots[mt] - 1]) 1669 - slots[mt_pivots[mt]] = NULL; 1670 - 1671 - i = mab_start; 1672 - do { 1673 - pivots[j++] = b_node->pivot[i++]; 1674 - } while (i <= mab_end && likely(b_node->pivot[i])); 1675 - 1676 - memcpy(slots, b_node->slot + mab_start, 1677 - sizeof(void *) * (i - mab_start)); 1678 - 1679 - if (new_max) 1680 - mas->max = b_node->pivot[i - 1]; 1681 - 1682 - end = j - 1; 1683 - if (likely(!ma_is_leaf(mt) && mt_is_alloc(mas->tree))) { 1684 - unsigned long max_gap = 0; 1685 - unsigned char offset = 0; 1686 - 1687 - gaps = ma_gaps(node, mt); 1688 - do { 1689 - gaps[--j] = b_node->gap[--i]; 1690 - if (gaps[j] > max_gap) { 1691 - offset = j; 1692 - max_gap = gaps[j]; 1693 - } 1694 - } while (j); 1695 - 1696 - ma_set_meta(node, mt, offset, end); 1697 - } else { 1698 - mas_leaf_set_meta(node, mt, end); 1699 - } 1700 - } 1701 - 1702 - /* 1703 - * mas_store_b_node() - Store an @entry into the b_node while also copying the 1704 - * data from a maple encoded node. 1705 - * @wr_mas: the maple write state 1706 - * @b_node: the maple_big_node to fill with data 1707 - * @offset_end: the offset to end copying 1708 - * 1709 - * Return: The actual end of the data stored in @b_node 1710 - */ 1711 - static noinline_for_kasan void mas_store_b_node(struct ma_wr_state *wr_mas, 1712 - struct maple_big_node *b_node, unsigned char offset_end) 1713 - { 1714 - unsigned char slot; 1715 - unsigned char b_end; 1716 - /* Possible underflow of piv will wrap back to 0 before use. */ 1717 - unsigned long piv; 1718 - struct ma_state *mas = wr_mas->mas; 1719 - 1720 - b_node->type = wr_mas->type; 1721 - b_end = 0; 1722 - slot = mas->offset; 1723 - if (slot) { 1724 - /* Copy start data up to insert. */ 1725 - mas_mab_cp(mas, 0, slot - 1, b_node, 0); 1726 - b_end = b_node->b_end; 1727 - piv = b_node->pivot[b_end - 1]; 1728 - } else 1729 - piv = mas->min - 1; 1730 - 1731 - if (piv + 1 < mas->index) { 1732 - /* Handle range starting after old range */ 1733 - b_node->slot[b_end] = wr_mas->content; 1734 - if (!wr_mas->content) 1735 - b_node->gap[b_end] = mas->index - 1 - piv; 1736 - b_node->pivot[b_end++] = mas->index - 1; 1737 - } 1738 - 1739 - /* Store the new entry. */ 1740 - mas->offset = b_end; 1741 - b_node->slot[b_end] = wr_mas->entry; 1742 - b_node->pivot[b_end] = mas->last; 1743 - 1744 - /* Appended. */ 1745 - if (mas->last >= mas->max) 1746 - goto b_end; 1747 - 1748 - /* Handle new range ending before old range ends */ 1749 - piv = mas_safe_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type); 1750 - if (piv > mas->last) { 1751 - if (offset_end != slot) 1752 - wr_mas->content = mas_slot_locked(mas, wr_mas->slots, 1753 - offset_end); 1754 - 1755 - b_node->slot[++b_end] = wr_mas->content; 1756 - if (!wr_mas->content) 1757 - b_node->gap[b_end] = piv - mas->last + 1; 1758 - b_node->pivot[b_end] = piv; 1759 - } 1760 - 1761 - slot = offset_end + 1; 1762 - if (slot > mas->end) 1763 - goto b_end; 1764 - 1765 - /* Copy end data to the end of the node. */ 1766 - mas_mab_cp(mas, slot, mas->end + 1, b_node, ++b_end); 1767 - b_node->b_end--; 1768 - return; 1769 - 1770 - b_end: 1771 - b_node->b_end = b_end; 1772 1846 } 1773 1847 1774 1848 /* ··· 1688 2018 } 1689 2019 1690 2020 /* 1691 - * mas_node_or_none() - Set the enode and state. 1692 - * @mas: the maple state 1693 - * @enode: The encoded maple node. 1694 - * 1695 - * Set the node to the enode and the status. 1696 - */ 1697 - static inline void mas_node_or_none(struct ma_state *mas, 1698 - struct maple_enode *enode) 1699 - { 1700 - if (enode) { 1701 - mas->node = enode; 1702 - mas->status = ma_active; 1703 - } else { 1704 - mas->node = NULL; 1705 - mas->status = ma_none; 1706 - } 1707 - } 1708 - 1709 - /* 1710 2021 * mas_wr_node_walk() - Find the correct offset for the index in the @mas. 1711 2022 * If @mas->index cannot be found within the containing 1712 2023 * node, we traverse to the last entry in the node. ··· 1718 2067 wr_mas->r_max = offset < count ? wr_mas->pivots[offset] : mas->max; 1719 2068 wr_mas->r_min = mas_safe_min(mas, wr_mas->pivots, offset); 1720 2069 wr_mas->offset_end = mas->offset = offset; 1721 - } 1722 - 1723 - /* 1724 - * mast_rebalance_next() - Rebalance against the next node 1725 - * @mast: The maple subtree state 1726 - */ 1727 - static inline void mast_rebalance_next(struct maple_subtree_state *mast) 1728 - { 1729 - unsigned char b_end = mast->bn->b_end; 1730 - 1731 - mas_mab_cp(mast->orig_r, 0, mt_slot_count(mast->orig_r->node), 1732 - mast->bn, b_end); 1733 - mast->orig_r->last = mast->orig_r->max; 1734 - } 1735 - 1736 - /* 1737 - * mast_rebalance_prev() - Rebalance against the previous node 1738 - * @mast: The maple subtree state 1739 - */ 1740 - static inline void mast_rebalance_prev(struct maple_subtree_state *mast) 1741 - { 1742 - unsigned char end = mas_data_end(mast->orig_l) + 1; 1743 - unsigned char b_end = mast->bn->b_end; 1744 - 1745 - mab_shift_right(mast->bn, end); 1746 - mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0); 1747 - mast->l->min = mast->orig_l->min; 1748 - mast->orig_l->index = mast->orig_l->min; 1749 - mast->bn->b_end = end + b_end; 1750 - mast->l->offset += end; 1751 - } 1752 - 1753 - /* 1754 - * mast_spanning_rebalance() - Rebalance nodes with nearest neighbour favouring 1755 - * the node to the right. Checking the nodes to the right then the left at each 1756 - * level upwards until root is reached. 1757 - * Data is copied into the @mast->bn. 1758 - * @mast: The maple_subtree_state. 1759 - */ 1760 - static inline 1761 - bool mast_spanning_rebalance(struct maple_subtree_state *mast) 1762 - { 1763 - struct ma_state r_tmp = *mast->orig_r; 1764 - struct ma_state l_tmp = *mast->orig_l; 1765 - unsigned char depth = 0; 1766 - 1767 - do { 1768 - mas_ascend(mast->orig_r); 1769 - mas_ascend(mast->orig_l); 1770 - depth++; 1771 - if (mast->orig_r->offset < mas_data_end(mast->orig_r)) { 1772 - mast->orig_r->offset++; 1773 - do { 1774 - mas_descend(mast->orig_r); 1775 - mast->orig_r->offset = 0; 1776 - } while (--depth); 1777 - 1778 - mast_rebalance_next(mast); 1779 - *mast->orig_l = l_tmp; 1780 - return true; 1781 - } else if (mast->orig_l->offset != 0) { 1782 - mast->orig_l->offset--; 1783 - do { 1784 - mas_descend(mast->orig_l); 1785 - mast->orig_l->offset = 1786 - mas_data_end(mast->orig_l); 1787 - } while (--depth); 1788 - 1789 - mast_rebalance_prev(mast); 1790 - *mast->orig_r = r_tmp; 1791 - return true; 1792 - } 1793 - } while (!mte_is_root(mast->orig_r->node)); 1794 - 1795 - *mast->orig_r = r_tmp; 1796 - *mast->orig_l = l_tmp; 1797 - return false; 1798 - } 1799 - 1800 - /* 1801 - * mast_ascend() - Ascend the original left and right maple states. 1802 - * @mast: the maple subtree state. 1803 - * 1804 - * Ascend the original left and right sides. Set the offsets to point to the 1805 - * data already in the new tree (@mast->l and @mast->r). 1806 - */ 1807 - static inline void mast_ascend(struct maple_subtree_state *mast) 1808 - { 1809 - MA_WR_STATE(wr_mas, mast->orig_r, NULL); 1810 - mas_ascend(mast->orig_l); 1811 - mas_ascend(mast->orig_r); 1812 - 1813 - mast->orig_r->offset = 0; 1814 - mast->orig_r->index = mast->r->max; 1815 - /* last should be larger than or equal to index */ 1816 - if (mast->orig_r->last < mast->orig_r->index) 1817 - mast->orig_r->last = mast->orig_r->index; 1818 - 1819 - wr_mas.type = mte_node_type(mast->orig_r->node); 1820 - mas_wr_node_walk(&wr_mas); 1821 - /* Set up the left side of things */ 1822 - mast->orig_l->offset = 0; 1823 - mast->orig_l->index = mast->l->min; 1824 - wr_mas.mas = mast->orig_l; 1825 - wr_mas.type = mte_node_type(mast->orig_l->node); 1826 - mas_wr_node_walk(&wr_mas); 1827 - 1828 - mast->bn->type = wr_mas.type; 1829 - } 1830 - 1831 - /* 1832 - * mas_new_ma_node() - Create and return a new maple node. Helper function. 1833 - * @mas: the maple state with the allocations. 1834 - * @b_node: the maple_big_node with the type encoding. 1835 - * 1836 - * Use the node type from the maple_big_node to allocate a new node from the 1837 - * ma_state. This function exists mainly for code readability. 1838 - * 1839 - * Return: A new maple encoded node 1840 - */ 1841 - static inline struct maple_enode 1842 - *mas_new_ma_node(struct ma_state *mas, struct maple_big_node *b_node) 1843 - { 1844 - return mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), b_node->type); 1845 - } 1846 - 1847 - /* 1848 - * mas_mab_to_node() - Set up right and middle nodes 1849 - * 1850 - * @mas: the maple state that contains the allocations. 1851 - * @b_node: the node which contains the data. 1852 - * @left: The pointer which will have the left node 1853 - * @right: The pointer which may have the right node 1854 - * @middle: the pointer which may have the middle node (rare) 1855 - * @mid_split: the split location for the middle node 1856 - * 1857 - * Return: the split of left. 1858 - */ 1859 - static inline unsigned char mas_mab_to_node(struct ma_state *mas, 1860 - struct maple_big_node *b_node, struct maple_enode **left, 1861 - struct maple_enode **right, struct maple_enode **middle, 1862 - unsigned char *mid_split) 1863 - { 1864 - unsigned char split = 0; 1865 - unsigned char slot_count = mt_slots[b_node->type]; 1866 - 1867 - *left = mas_new_ma_node(mas, b_node); 1868 - *right = NULL; 1869 - *middle = NULL; 1870 - *mid_split = 0; 1871 - 1872 - if (b_node->b_end < slot_count) { 1873 - split = b_node->b_end; 1874 - } else { 1875 - split = mab_calc_split(mas, b_node, mid_split); 1876 - *right = mas_new_ma_node(mas, b_node); 1877 - } 1878 - 1879 - if (*mid_split) 1880 - *middle = mas_new_ma_node(mas, b_node); 1881 - 1882 - return split; 1883 - 1884 - } 1885 - 1886 - /* 1887 - * mab_set_b_end() - Add entry to b_node at b_node->b_end and increment the end 1888 - * pointer. 1889 - * @b_node: the big node to add the entry 1890 - * @mas: the maple state to get the pivot (mas->max) 1891 - * @entry: the entry to add, if NULL nothing happens. 1892 - */ 1893 - static inline void mab_set_b_end(struct maple_big_node *b_node, 1894 - struct ma_state *mas, 1895 - void *entry) 1896 - { 1897 - if (!entry) 1898 - return; 1899 - 1900 - b_node->slot[b_node->b_end] = entry; 1901 - if (mt_is_alloc(mas->tree)) 1902 - b_node->gap[b_node->b_end] = mas_max_gap(mas); 1903 - b_node->pivot[b_node->b_end++] = mas->max; 1904 - } 1905 - 1906 - /* 1907 - * mas_set_split_parent() - combine_then_separate helper function. Sets the parent 1908 - * of @mas->node to either @left or @right, depending on @slot and @split 1909 - * 1910 - * @mas: the maple state with the node that needs a parent 1911 - * @left: possible parent 1 1912 - * @right: possible parent 2 1913 - * @slot: the slot the mas->node was placed 1914 - * @split: the split location between @left and @right 1915 - */ 1916 - static inline void mas_set_split_parent(struct ma_state *mas, 1917 - struct maple_enode *left, 1918 - struct maple_enode *right, 1919 - unsigned char *slot, unsigned char split) 1920 - { 1921 - if (mas_is_none(mas)) 1922 - return; 1923 - 1924 - if ((*slot) <= split) 1925 - mas_set_parent(mas, mas->node, left, *slot); 1926 - else if (right) 1927 - mas_set_parent(mas, mas->node, right, (*slot) - split - 1); 1928 - 1929 - (*slot)++; 1930 - } 1931 - 1932 - /* 1933 - * mte_mid_split_check() - Check if the next node passes the mid-split 1934 - * @l: Pointer to left encoded maple node. 1935 - * @m: Pointer to middle encoded maple node. 1936 - * @r: Pointer to right encoded maple node. 1937 - * @slot: The offset 1938 - * @split: The split location. 1939 - * @mid_split: The middle split. 1940 - */ 1941 - static inline void mte_mid_split_check(struct maple_enode **l, 1942 - struct maple_enode **r, 1943 - struct maple_enode *right, 1944 - unsigned char slot, 1945 - unsigned char *split, 1946 - unsigned char mid_split) 1947 - { 1948 - if (*r == right) 1949 - return; 1950 - 1951 - if (slot < mid_split) 1952 - return; 1953 - 1954 - *l = *r; 1955 - *r = right; 1956 - *split = mid_split; 1957 2070 } 1958 2071 1959 2072 static inline void rebalance_sib(struct ma_state *parent, struct ma_state *sib) ··· 1769 2354 } while (!mte_is_root(r_tmp.node)); 1770 2355 1771 2356 WARN_ON_ONCE(1); 1772 - } 1773 - 1774 - /* 1775 - * mast_set_split_parents() - Helper function to set three nodes parents. Slot 1776 - * is taken from @mast->l. 1777 - * @mast: the maple subtree state 1778 - * @left: the left node 1779 - * @right: the right node 1780 - * @split: the split location. 1781 - */ 1782 - static inline void mast_set_split_parents(struct maple_subtree_state *mast, 1783 - struct maple_enode *left, 1784 - struct maple_enode *middle, 1785 - struct maple_enode *right, 1786 - unsigned char split, 1787 - unsigned char mid_split) 1788 - { 1789 - unsigned char slot; 1790 - struct maple_enode *l = left; 1791 - struct maple_enode *r = right; 1792 - 1793 - if (mas_is_none(mast->l)) 1794 - return; 1795 - 1796 - if (middle) 1797 - r = middle; 1798 - 1799 - slot = mast->l->offset; 1800 - 1801 - mte_mid_split_check(&l, &r, right, slot, &split, mid_split); 1802 - mas_set_split_parent(mast->l, l, r, &slot, split); 1803 - 1804 - mte_mid_split_check(&l, &r, right, slot, &split, mid_split); 1805 - mas_set_split_parent(mast->m, l, r, &slot, split); 1806 - 1807 - mte_mid_split_check(&l, &r, right, slot, &split, mid_split); 1808 - mas_set_split_parent(mast->r, l, r, &slot, split); 1809 2357 } 1810 2358 1811 2359 /* ··· 2024 2646 ma_set_meta(node, mt, gap_slot, end - 1); 2025 2647 else if (end <= max_end - 1) 2026 2648 ma_set_meta(node, mt, gap_slot, end - 1); 2027 - } 2028 - 2029 - /* 2030 - * mast_cp_to_nodes() - Copy data out to nodes. 2031 - * @mast: The maple subtree state 2032 - * @left: The left encoded maple node 2033 - * @middle: The middle encoded maple node 2034 - * @right: The right encoded maple node 2035 - * @split: The location to split between left and (middle ? middle : right) 2036 - * @mid_split: The location to split between middle and right. 2037 - */ 2038 - static inline void mast_cp_to_nodes(struct maple_subtree_state *mast, 2039 - struct maple_enode *left, struct maple_enode *middle, 2040 - struct maple_enode *right, unsigned char split, unsigned char mid_split) 2041 - { 2042 - bool new_lmax = true; 2043 - 2044 - mas_node_or_none(mast->l, left); 2045 - mas_node_or_none(mast->m, middle); 2046 - mas_node_or_none(mast->r, right); 2047 - 2048 - mast->l->min = mast->orig_l->min; 2049 - if (split == mast->bn->b_end) { 2050 - mast->l->max = mast->orig_r->max; 2051 - new_lmax = false; 2052 - } 2053 - 2054 - mab_mas_cp(mast->bn, 0, split, mast->l, new_lmax); 2055 - 2056 - if (middle) { 2057 - mab_mas_cp(mast->bn, 1 + split, mid_split, mast->m, true); 2058 - mast->m->min = mast->bn->pivot[split] + 1; 2059 - split = mid_split; 2060 - } 2061 - 2062 - mast->r->max = mast->orig_r->max; 2063 - if (right) { 2064 - mab_mas_cp(mast->bn, 1 + split, mast->bn->b_end, mast->r, false); 2065 - mast->r->min = mast->bn->pivot[split] + 1; 2066 - } 2067 - } 2068 - 2069 - /* 2070 - * mast_combine_cp_left - Copy in the original left side of the tree into the 2071 - * combined data set in the maple subtree state big node. 2072 - * @mast: The maple subtree state 2073 - */ 2074 - static inline void mast_combine_cp_left(struct maple_subtree_state *mast) 2075 - { 2076 - unsigned char l_slot = mast->orig_l->offset; 2077 - 2078 - if (!l_slot) 2079 - return; 2080 - 2081 - mas_mab_cp(mast->orig_l, 0, l_slot - 1, mast->bn, 0); 2082 - } 2083 - 2084 - /* 2085 - * mast_combine_cp_right: Copy in the original right side of the tree into the 2086 - * combined data set in the maple subtree state big node. 2087 - * @mast: The maple subtree state 2088 - */ 2089 - static inline void mast_combine_cp_right(struct maple_subtree_state *mast) 2090 - { 2091 - if (mast->bn->pivot[mast->bn->b_end - 1] >= mast->orig_r->max) 2092 - return; 2093 - 2094 - mas_mab_cp(mast->orig_r, mast->orig_r->offset + 1, 2095 - mt_slot_count(mast->orig_r->node), mast->bn, 2096 - mast->bn->b_end); 2097 - mast->orig_r->last = mast->orig_r->max; 2098 - } 2099 - 2100 - /* 2101 - * mast_sufficient: Check if the maple subtree state has enough data in the big 2102 - * node to create at least one sufficient node 2103 - * @mast: the maple subtree state 2104 - */ 2105 - static inline bool mast_sufficient(struct maple_subtree_state *mast) 2106 - { 2107 - if (mast->bn->b_end > mt_min_slot_count(mast->orig_l->node)) 2108 - return true; 2109 - 2110 - return false; 2111 - } 2112 - 2113 - /* 2114 - * mast_overflow: Check if there is too much data in the subtree state for a 2115 - * single node. 2116 - * @mast: The maple subtree state 2117 - */ 2118 - static inline bool mast_overflow(struct maple_subtree_state *mast) 2119 - { 2120 - if (mast->bn->b_end > mt_slot_count(mast->orig_l->node)) 2121 - return true; 2122 - 2123 - return false; 2124 2649 } 2125 2650 2126 2651 static inline void *mtree_range_walk(struct ma_state *mas) ··· 2585 3304 cp->max = max; 2586 3305 } 2587 3306 2588 - static void mas_spanning_rebalance_loop(struct ma_state *mas, 2589 - struct maple_subtree_state *mast, unsigned char count) 2590 - { 2591 - 2592 - unsigned char split, mid_split; 2593 - unsigned char slot = 0; 2594 - unsigned char new_height = 0; /* used if node is a new root */ 2595 - struct maple_enode *left = NULL, *middle = NULL, *right = NULL; 2596 - struct maple_enode *old_enode; 2597 - 2598 - /* 2599 - * Each level of the tree is examined and balanced, pushing data to the left or 2600 - * right, or rebalancing against left or right nodes is employed to avoid 2601 - * rippling up the tree to limit the amount of churn. Once a new sub-section of 2602 - * the tree is created, there may be a mix of new and old nodes. The old nodes 2603 - * will have the incorrect parent pointers and currently be in two trees: the 2604 - * original tree and the partially new tree. To remedy the parent pointers in 2605 - * the old tree, the new data is swapped into the active tree and a walk down 2606 - * the tree is performed and the parent pointers are updated. 2607 - * See mas_topiary_replace() for more information. 2608 - */ 2609 - while (count--) { 2610 - mast->bn->b_end--; 2611 - mast->bn->type = mte_node_type(mast->orig_l->node); 2612 - split = mas_mab_to_node(mas, mast->bn, &left, &right, &middle, 2613 - &mid_split); 2614 - mast_set_split_parents(mast, left, middle, right, split, 2615 - mid_split); 2616 - mast_cp_to_nodes(mast, left, middle, right, split, mid_split); 2617 - new_height++; 2618 - 2619 - /* 2620 - * Copy data from next level in the tree to mast->bn from next 2621 - * iteration 2622 - */ 2623 - memset(mast->bn, 0, sizeof(struct maple_big_node)); 2624 - mast->bn->type = mte_node_type(left); 2625 - 2626 - /* Root already stored in l->node. */ 2627 - if (mas_is_root_limits(mast->l)) 2628 - goto new_root; 2629 - 2630 - mast_ascend(mast); 2631 - mast_combine_cp_left(mast); 2632 - mast->l->offset = mast->bn->b_end; 2633 - mab_set_b_end(mast->bn, mast->l, left); 2634 - mab_set_b_end(mast->bn, mast->m, middle); 2635 - mab_set_b_end(mast->bn, mast->r, right); 2636 - 2637 - /* Copy anything necessary out of the right node. */ 2638 - mast_combine_cp_right(mast); 2639 - mast->orig_l->last = mast->orig_l->max; 2640 - 2641 - if (mast_sufficient(mast)) { 2642 - if (mast_overflow(mast)) 2643 - continue; 2644 - 2645 - if (mast->orig_l->node == mast->orig_r->node) { 2646 - /* 2647 - * The data in b_node should be stored in one 2648 - * node and in the tree 2649 - */ 2650 - slot = mast->l->offset; 2651 - break; 2652 - } 2653 - 2654 - continue; 2655 - } 2656 - 2657 - /* May be a new root stored in mast->bn */ 2658 - if (mas_is_root_limits(mast->orig_l)) 2659 - break; 2660 - 2661 - mast_spanning_rebalance(mast); 2662 - 2663 - /* rebalancing from other nodes may require another loop. */ 2664 - if (!count) 2665 - count++; 2666 - } 2667 - 2668 - mast->l->node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), 2669 - mte_node_type(mast->orig_l->node)); 2670 - 2671 - mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true); 2672 - new_height++; 2673 - mas_set_parent(mas, left, mast->l->node, slot); 2674 - if (middle) 2675 - mas_set_parent(mas, middle, mast->l->node, ++slot); 2676 - 2677 - if (right) 2678 - mas_set_parent(mas, right, mast->l->node, ++slot); 2679 - 2680 - if (mas_is_root_limits(mast->l)) { 2681 - new_root: 2682 - mas_mn(mast->l)->parent = ma_parent_ptr(mas_tree_parent(mas)); 2683 - while (!mte_is_root(mast->orig_l->node)) 2684 - mast_ascend(mast); 2685 - } else { 2686 - mas_mn(mast->l)->parent = mas_mn(mast->orig_l)->parent; 2687 - } 2688 - 2689 - old_enode = mast->orig_l->node; 2690 - mas->depth = mast->l->depth; 2691 - mas->node = mast->l->node; 2692 - mas->min = mast->l->min; 2693 - mas->max = mast->l->max; 2694 - mas->offset = mast->l->offset; 2695 - mas_wmb_replace(mas, old_enode, new_height); 2696 - mtree_range_walk(mas); 2697 - } 2698 - 2699 - /* 2700 - * mas_spanning_rebalance() - Rebalance across two nodes which may not be peers. 2701 - * @mas: The starting maple state 2702 - * @mast: The maple_subtree_state, keeps track of 4 maple states. 2703 - * @count: The estimated count of iterations needed. 2704 - * 2705 - * Follow the tree upwards from @l_mas and @r_mas for @count, or until the root 2706 - * is hit. First @b_node is split into two entries which are inserted into the 2707 - * next iteration of the loop. @b_node is returned populated with the final 2708 - * iteration. @mas is used to obtain allocations. orig_l_mas keeps track of the 2709 - * nodes that will remain active by using orig_l_mas->index and orig_l_mas->last 2710 - * to account of what has been copied into the new sub-tree. The update of 2711 - * orig_l_mas->last is used in mas_consume to find the slots that will need to 2712 - * be either freed or destroyed. orig_l_mas->depth keeps track of the height of 2713 - * the new sub-tree in case the sub-tree becomes the full tree. 2714 - */ 2715 - static void mas_spanning_rebalance(struct ma_state *mas, 2716 - struct maple_subtree_state *mast, unsigned char count) 2717 - { 2718 - 2719 - MA_STATE(l_mas, mas->tree, mas->index, mas->index); 2720 - MA_STATE(r_mas, mas->tree, mas->index, mas->last); 2721 - MA_STATE(m_mas, mas->tree, mas->index, mas->index); 2722 - 2723 - /* 2724 - * The tree needs to be rebalanced and leaves need to be kept at the same level. 2725 - * Rebalancing is done by use of the ``struct maple_topiary``. 2726 - */ 2727 - mast->l = &l_mas; 2728 - mast->m = &m_mas; 2729 - mast->r = &r_mas; 2730 - l_mas.status = r_mas.status = m_mas.status = ma_none; 2731 - 2732 - /* Check if this is not root and has sufficient data. */ 2733 - if (((mast->orig_l->min != 0) || (mast->orig_r->max != ULONG_MAX)) && 2734 - unlikely(mast->bn->b_end <= mt_min_slots[mast->bn->type])) 2735 - mast_spanning_rebalance(mast); 2736 - 2737 - mas_spanning_rebalance_loop(mas, mast, count); 2738 - } 2739 - 2740 3307 static inline bool cp_is_new_root(struct maple_copy *cp, struct ma_state *mas) 2741 3308 { 2742 3309 if (cp->min || cp->max != ULONG_MAX) ··· 2721 3592 copy_tree_location(parent, mas); 2722 3593 wr_mas_setup(wr_mas, mas); 2723 3594 return true; 2724 - } 2725 - 2726 - /* 2727 - * mas_rebalance() - Rebalance a given node. 2728 - * @mas: The maple state 2729 - * @b_node: The big maple node. 2730 - * 2731 - * Rebalance two nodes into a single node or two new nodes that are sufficient. 2732 - * Continue upwards until tree is sufficient. 2733 - */ 2734 - static inline void mas_rebalance(struct ma_state *mas, 2735 - struct maple_big_node *b_node) 2736 - { 2737 - char empty_count = mas_mt_height(mas); 2738 - struct maple_subtree_state mast; 2739 - unsigned char shift, b_end = ++b_node->b_end; 2740 - 2741 - MA_STATE(l_mas, mas->tree, mas->index, mas->last); 2742 - MA_STATE(r_mas, mas->tree, mas->index, mas->last); 2743 - 2744 - trace_ma_op(TP_FCT, mas); 2745 - 2746 - /* 2747 - * Rebalancing occurs if a node is insufficient. Data is rebalanced 2748 - * against the node to the right if it exists, otherwise the node to the 2749 - * left of this node is rebalanced against this node. If rebalancing 2750 - * causes just one node to be produced instead of two, then the parent 2751 - * is also examined and rebalanced if it is insufficient. Every level 2752 - * tries to combine the data in the same way. If one node contains the 2753 - * entire range of the tree, then that node is used as a new root node. 2754 - */ 2755 - 2756 - mast.orig_l = &l_mas; 2757 - mast.orig_r = &r_mas; 2758 - mast.bn = b_node; 2759 - mast.bn->type = mte_node_type(mas->node); 2760 - 2761 - l_mas = r_mas = *mas; 2762 - 2763 - if (mas_next_sibling(&r_mas)) { 2764 - mas_mab_cp(&r_mas, 0, mt_slot_count(r_mas.node), b_node, b_end); 2765 - r_mas.last = r_mas.index = r_mas.max; 2766 - } else { 2767 - mas_prev_sibling(&l_mas); 2768 - shift = mas_data_end(&l_mas) + 1; 2769 - mab_shift_right(b_node, shift); 2770 - mas->offset += shift; 2771 - mas_mab_cp(&l_mas, 0, shift - 1, b_node, 0); 2772 - b_node->b_end = shift + b_end; 2773 - l_mas.index = l_mas.last = l_mas.min; 2774 - } 2775 - 2776 - return mas_spanning_rebalance(mas, &mast, empty_count); 2777 - } 2778 - 2779 - /* 2780 - * mas_split_final_node() - Split the final node in a subtree operation. 2781 - * @mast: the maple subtree state 2782 - * @mas: The maple state 2783 - */ 2784 - static inline void mas_split_final_node(struct maple_subtree_state *mast, 2785 - struct ma_state *mas) 2786 - { 2787 - struct maple_enode *ancestor; 2788 - 2789 - if (mte_is_root(mas->node)) { 2790 - if (mt_is_alloc(mas->tree)) 2791 - mast->bn->type = maple_arange_64; 2792 - else 2793 - mast->bn->type = maple_range_64; 2794 - } 2795 - /* 2796 - * Only a single node is used here, could be root. 2797 - * The Big_node data should just fit in a single node. 2798 - */ 2799 - ancestor = mas_new_ma_node(mas, mast->bn); 2800 - mas_set_parent(mas, mast->l->node, ancestor, mast->l->offset); 2801 - mas_set_parent(mas, mast->r->node, ancestor, mast->r->offset); 2802 - mte_to_node(ancestor)->parent = mas_mn(mas)->parent; 2803 - 2804 - mast->l->node = ancestor; 2805 - mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true); 2806 - mas->offset = mast->bn->b_end - 1; 2807 - } 2808 - 2809 - /* 2810 - * mast_fill_bnode() - Copy data into the big node in the subtree state 2811 - * @mast: The maple subtree state 2812 - * @mas: the maple state 2813 - * @skip: The number of entries to skip for new nodes insertion. 2814 - */ 2815 - static inline void mast_fill_bnode(struct maple_subtree_state *mast, 2816 - struct ma_state *mas, 2817 - unsigned char skip) 2818 - { 2819 - bool cp = true; 2820 - unsigned char split; 2821 - 2822 - memset(mast->bn, 0, sizeof(struct maple_big_node)); 2823 - 2824 - if (mte_is_root(mas->node)) { 2825 - cp = false; 2826 - } else { 2827 - mas_ascend(mas); 2828 - mas->offset = mte_parent_slot(mas->node); 2829 - } 2830 - 2831 - if (cp && mast->l->offset) 2832 - mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0); 2833 - 2834 - split = mast->bn->b_end; 2835 - mab_set_b_end(mast->bn, mast->l, mast->l->node); 2836 - mast->r->offset = mast->bn->b_end; 2837 - mab_set_b_end(mast->bn, mast->r, mast->r->node); 2838 - if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max) 2839 - cp = false; 2840 - 2841 - if (cp) 2842 - mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1, 2843 - mast->bn, mast->bn->b_end); 2844 - 2845 - mast->bn->b_end--; 2846 - mast->bn->type = mte_node_type(mas->node); 2847 - } 2848 - 2849 - /* 2850 - * mast_split_data() - Split the data in the subtree state big node into regular 2851 - * nodes. 2852 - * @mast: The maple subtree state 2853 - * @mas: The maple state 2854 - * @split: The location to split the big node 2855 - */ 2856 - static inline void mast_split_data(struct maple_subtree_state *mast, 2857 - struct ma_state *mas, unsigned char split) 2858 - { 2859 - unsigned char p_slot; 2860 - 2861 - mab_mas_cp(mast->bn, 0, split, mast->l, true); 2862 - mte_set_pivot(mast->r->node, 0, mast->r->max); 2863 - mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false); 2864 - mast->l->offset = mte_parent_slot(mas->node); 2865 - mast->l->max = mast->bn->pivot[split]; 2866 - mast->r->min = mast->l->max + 1; 2867 - if (mte_is_leaf(mas->node)) 2868 - return; 2869 - 2870 - p_slot = mast->orig_l->offset; 2871 - mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node, 2872 - &p_slot, split); 2873 - mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node, 2874 - &p_slot, split); 2875 - } 2876 - 2877 - /* 2878 - * mas_push_data() - Instead of splitting a node, it is beneficial to push the 2879 - * data to the right or left node if there is room. 2880 - * @mas: The maple state 2881 - * @mast: The maple subtree state 2882 - * @left: Push left or not. 2883 - * 2884 - * Keeping the height of the tree low means faster lookups. 2885 - * 2886 - * Return: True if pushed, false otherwise. 2887 - */ 2888 - static inline bool mas_push_data(struct ma_state *mas, 2889 - struct maple_subtree_state *mast, bool left) 2890 - { 2891 - unsigned char slot_total = mast->bn->b_end; 2892 - unsigned char end, space, split; 2893 - 2894 - MA_STATE(tmp_mas, mas->tree, mas->index, mas->last); 2895 - tmp_mas = *mas; 2896 - tmp_mas.depth = mast->l->depth; 2897 - 2898 - if (left && !mas_prev_sibling(&tmp_mas)) 2899 - return false; 2900 - else if (!left && !mas_next_sibling(&tmp_mas)) 2901 - return false; 2902 - 2903 - end = mas_data_end(&tmp_mas); 2904 - slot_total += end; 2905 - space = 2 * mt_slot_count(mas->node) - 2; 2906 - /* -2 instead of -1 to ensure there isn't a triple split */ 2907 - if (ma_is_leaf(mast->bn->type)) 2908 - space--; 2909 - 2910 - if (mas->max == ULONG_MAX) 2911 - space--; 2912 - 2913 - if (slot_total >= space) 2914 - return false; 2915 - 2916 - /* Get the data; Fill mast->bn */ 2917 - mast->bn->b_end++; 2918 - if (left) { 2919 - mab_shift_right(mast->bn, end + 1); 2920 - mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0); 2921 - mast->bn->b_end = slot_total + 1; 2922 - } else { 2923 - mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end); 2924 - } 2925 - 2926 - /* Configure mast for splitting of mast->bn */ 2927 - split = mt_slots[mast->bn->type] - 2; 2928 - if (left) { 2929 - /* Switch mas to prev node */ 2930 - *mas = tmp_mas; 2931 - /* Start using mast->l for the left side. */ 2932 - tmp_mas.node = mast->l->node; 2933 - *mast->l = tmp_mas; 2934 - } else { 2935 - tmp_mas.node = mast->r->node; 2936 - *mast->r = tmp_mas; 2937 - split = slot_total - split; 2938 - } 2939 - split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]); 2940 - /* Update parent slot for split calculation. */ 2941 - if (left) 2942 - mast->orig_l->offset += end + 1; 2943 - 2944 - mast_split_data(mast, mas, split); 2945 - mast_fill_bnode(mast, mas, 2); 2946 - mas_split_final_node(mast, mas); 2947 - return true; 2948 - } 2949 - 2950 - /* 2951 - * mas_split() - Split data that is too big for one node into two. 2952 - * @mas: The maple state 2953 - * @b_node: The maple big node 2954 - */ 2955 - static void mas_split(struct ma_state *mas, struct maple_big_node *b_node) 2956 - { 2957 - struct maple_subtree_state mast; 2958 - int height = 0; 2959 - unsigned int orig_height = mas_mt_height(mas); 2960 - unsigned char mid_split, split = 0; 2961 - struct maple_enode *old; 2962 - 2963 - /* 2964 - * Splitting is handled differently from any other B-tree; the Maple 2965 - * Tree splits upwards. Splitting up means that the split operation 2966 - * occurs when the walk of the tree hits the leaves and not on the way 2967 - * down. The reason for splitting up is that it is impossible to know 2968 - * how much space will be needed until the leaf is (or leaves are) 2969 - * reached. Since overwriting data is allowed and a range could 2970 - * overwrite more than one range or result in changing one entry into 3 2971 - * entries, it is impossible to know if a split is required until the 2972 - * data is examined. 2973 - * 2974 - * Splitting is a balancing act between keeping allocations to a minimum 2975 - * and avoiding a 'jitter' event where a tree is expanded to make room 2976 - * for an entry followed by a contraction when the entry is removed. To 2977 - * accomplish the balance, there are empty slots remaining in both left 2978 - * and right nodes after a split. 2979 - */ 2980 - MA_STATE(l_mas, mas->tree, mas->index, mas->last); 2981 - MA_STATE(r_mas, mas->tree, mas->index, mas->last); 2982 - MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last); 2983 - MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last); 2984 - 2985 - trace_ma_op(TP_FCT, mas); 2986 - 2987 - mast.l = &l_mas; 2988 - mast.r = &r_mas; 2989 - mast.orig_l = &prev_l_mas; 2990 - mast.orig_r = &prev_r_mas; 2991 - mast.bn = b_node; 2992 - 2993 - while (height++ <= orig_height) { 2994 - if (mt_slots[b_node->type] > b_node->b_end) { 2995 - mas_split_final_node(&mast, mas); 2996 - break; 2997 - } 2998 - 2999 - l_mas = r_mas = *mas; 3000 - l_mas.node = mas_new_ma_node(mas, b_node); 3001 - r_mas.node = mas_new_ma_node(mas, b_node); 3002 - /* 3003 - * Another way that 'jitter' is avoided is to terminate a split up early if the 3004 - * left or right node has space to spare. This is referred to as "pushing left" 3005 - * or "pushing right" and is similar to the B* tree, except the nodes left or 3006 - * right can rarely be reused due to RCU, but the ripple upwards is halted which 3007 - * is a significant savings. 3008 - */ 3009 - /* Try to push left. */ 3010 - if (mas_push_data(mas, &mast, true)) { 3011 - height++; 3012 - break; 3013 - } 3014 - /* Try to push right. */ 3015 - if (mas_push_data(mas, &mast, false)) { 3016 - height++; 3017 - break; 3018 - } 3019 - 3020 - split = mab_calc_split(mas, b_node, &mid_split); 3021 - mast_split_data(&mast, mas, split); 3022 - /* 3023 - * Usually correct, mab_mas_cp in the above call overwrites 3024 - * r->max. 3025 - */ 3026 - mast.r->max = mas->max; 3027 - mast_fill_bnode(&mast, mas, 1); 3028 - prev_l_mas = *mast.l; 3029 - prev_r_mas = *mast.r; 3030 - } 3031 - 3032 - /* Set the original node as dead */ 3033 - old = mas->node; 3034 - mas->node = l_mas.node; 3035 - mas_wmb_replace(mas, old, height); 3036 - mtree_range_walk(mas); 3037 3595 } 3038 3596 3039 3597 /*

Configure Feed

Configure Feed