tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

maple_tree: introduce maple_copy node and use it in mas_spanning_rebalance()

Introduce an internal-memory only node type called maple_copy to
facilitate internal copy operations. Use it in mas_spanning_rebalance()
for just the leaf nodes. Initially, the maple_copy node is used to
configure the source nodes and copy the data into the big_node.

The maple_copy contains a list of source entries with start and end
offsets. One of the maple_copy entries can be itself with an offset of 0
to 2, representing the data where the store partially overwrites entries,
or fully overwrites the entry. The side effect is that the source nodes
no longer have to worry about partially copying the existing offset if it
is not fully overwritten.

This is in preparation of removal of the maple big_node, but for the time
being the data is copied to the big node to limit the change size.

Link: https://lkml.kernel.org/r/20260130205935.2559335-12-Liam.Howlett@oracle.com
Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Andrew Ballance <andrewjballance@gmail.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Christian Kujau <lists@nerdbynature.de>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Liam R. Howlett and committed by
Andrew Morton 9ec1e972 6b74d44b

+157 -9
+26
include/linux/maple_tree.h
··· 139 139 maple_leaf_64, 140 140 maple_range_64, 141 141 maple_arange_64, 142 + maple_copy, 142 143 }; 143 144 144 145 enum store_type { ··· 153 152 wr_append, 154 153 wr_node_store, 155 154 wr_slot_store, 155 + }; 156 + 157 + struct maple_copy { 158 + struct { 159 + struct maple_node *node; 160 + unsigned long max; 161 + unsigned char start; 162 + unsigned char end; 163 + enum maple_type mt; 164 + } src[4]; 165 + /* Simulated node */ 166 + void __rcu *slot[3]; 167 + unsigned long min; 168 + union { 169 + unsigned long pivot[3]; 170 + struct { 171 + void *_pad[2]; 172 + unsigned long max; 173 + }; 174 + }; 175 + unsigned char end; 176 + 177 + /*Avoid passing these around */ 178 + unsigned char s_count; 156 179 }; 157 180 158 181 /** ··· 324 299 }; 325 300 struct maple_range_64 mr64; 326 301 struct maple_arange_64 ma64; 302 + struct maple_copy cp; 327 303 }; 328 304 }; 329 305
+131 -9
lib/maple_tree.c
··· 605 605 case maple_range_64: 606 606 case maple_leaf_64: 607 607 return node->mr64.pivot; 608 + case maple_copy: 609 + return node->cp.pivot; 608 610 case maple_dense: 609 611 return NULL; 610 612 } ··· 626 624 switch (type) { 627 625 case maple_arange_64: 628 626 return node->ma64.gap; 627 + case maple_copy: 629 628 case maple_range_64: 630 629 case maple_leaf_64: 631 630 case maple_dense: ··· 693 690 case maple_arange_64: 694 691 node->ma64.pivot[piv] = val; 695 692 break; 693 + case maple_copy: 696 694 case maple_dense: 697 695 break; 698 696 } ··· 715 711 case maple_range_64: 716 712 case maple_leaf_64: 717 713 return mn->mr64.slot; 714 + case maple_copy: 715 + return mn->cp.slot; 718 716 case maple_dense: 719 717 return mn->slot; 720 718 } ··· 2601 2595 return NULL; 2602 2596 } 2603 2597 2598 + /* 2599 + * cp_leaf_init() - Initialize a maple_copy node for the leaf level of a 2600 + * spanning store 2601 + * @cp: The maple copy node 2602 + * @mas: The maple state 2603 + * @l_wr_mas: The left write state of the spanning store 2604 + * @r_wr_mas: The right write state of the spanning store 2605 + */ 2606 + static inline void cp_leaf_init(struct maple_copy *cp, 2607 + struct ma_state *mas, struct ma_wr_state *l_wr_mas, 2608 + struct ma_wr_state *r_wr_mas) 2609 + { 2610 + unsigned char end = 0; 2611 + 2612 + /* 2613 + * WARNING: The use of RCU_INIT_POINTER() makes it extremely important 2614 + * to not expose the maple_copy node to any readers. Exposure may 2615 + * result in buggy code when a compiler reorders the instructions. 2616 + */ 2617 + 2618 + /* Create entries to insert including split entries to left and right */ 2619 + if (l_wr_mas->r_min < mas->index) { 2620 + end++; 2621 + RCU_INIT_POINTER(cp->slot[0], l_wr_mas->content); 2622 + cp->pivot[0] = mas->index - 1; 2623 + } 2624 + RCU_INIT_POINTER(cp->slot[end], l_wr_mas->entry); 2625 + cp->pivot[end] = mas->last; 2626 + 2627 + if (r_wr_mas->end_piv > mas->last) { 2628 + end++; 2629 + RCU_INIT_POINTER(cp->slot[end], 2630 + r_wr_mas->slots[r_wr_mas->offset_end]); 2631 + cp->pivot[end] = r_wr_mas->end_piv; 2632 + } 2633 + 2634 + cp->min = l_wr_mas->r_min; 2635 + cp->max = cp->pivot[end]; 2636 + cp->end = end; 2637 + } 2638 + 2639 + static inline void append_wr_mas_cp(struct maple_copy *cp, 2640 + struct ma_wr_state *wr_mas, unsigned char start, unsigned char end) 2641 + { 2642 + unsigned char count; 2643 + 2644 + count = cp->s_count; 2645 + cp->src[count].node = wr_mas->node; 2646 + cp->src[count].mt = wr_mas->type; 2647 + if (wr_mas->mas->end <= end) 2648 + cp->src[count].max = wr_mas->mas->max; 2649 + else 2650 + cp->src[count].max = wr_mas->pivots[end]; 2651 + 2652 + cp->src[count].start = start; 2653 + cp->src[count].end = end; 2654 + cp->s_count++; 2655 + } 2656 + 2657 + static inline void init_cp_src(struct maple_copy *cp) 2658 + { 2659 + cp->src[cp->s_count].node = ma_mnode_ptr(cp); 2660 + cp->src[cp->s_count].mt = maple_copy; 2661 + cp->src[cp->s_count].max = cp->max; 2662 + cp->src[cp->s_count].start = 0; 2663 + cp->src[cp->s_count].end = cp->end; 2664 + cp->s_count++; 2665 + } 2666 + 2667 + static inline 2668 + void cp_data_write(struct maple_copy *cp, struct maple_big_node *b_node) 2669 + { 2670 + struct maple_node *src; 2671 + unsigned char s; 2672 + unsigned char src_end, s_offset; 2673 + unsigned long *b_pivots, *cp_pivots; 2674 + void __rcu **b_slots, **cp_slots; 2675 + enum maple_type s_mt; 2676 + 2677 + b_node->b_end = 0; 2678 + 2679 + s = 0; 2680 + b_pivots = b_node->pivot; 2681 + b_slots = (void __rcu **)b_node->slot; 2682 + do { 2683 + unsigned char size; 2684 + 2685 + src = cp->src[s].node; 2686 + s_mt = cp->src[s].mt; 2687 + s_offset = cp->src[s].start; 2688 + src_end = cp->src[s].end; 2689 + size = src_end - s_offset + 1; 2690 + cp_pivots = ma_pivots(src, s_mt) + s_offset; 2691 + cp_slots = ma_slots(src, s_mt) + s_offset; 2692 + memcpy(b_slots, cp_slots, size * sizeof(void __rcu *)); 2693 + if (size > 1) 2694 + memcpy(b_pivots, cp_pivots, (size - 1) * sizeof(unsigned long)); 2695 + b_pivots[size - 1] = cp->src[s].max; 2696 + b_pivots += size; 2697 + b_slots += size; 2698 + b_node->b_end += size; 2699 + } while (++s < cp->s_count); 2700 + } 2701 + 2604 2702 static void mas_spanning_rebalance_loop(struct ma_state *mas, 2605 2703 struct maple_subtree_state *mast, unsigned char count) 2606 2704 { ··· 2860 2750 2861 2751 2862 2752 static noinline void mas_wr_spanning_rebalance(struct ma_state *mas, 2863 - struct ma_wr_state *wr_mas, struct ma_wr_state *r_wr_mas) 2753 + struct ma_wr_state *l_wr_mas, struct ma_wr_state *r_wr_mas) 2864 2754 { 2865 2755 struct maple_subtree_state mast; 2866 2756 struct maple_big_node b_node; 2757 + struct maple_copy cp; 2867 2758 unsigned char height; 2868 2759 MA_STATE(l_mas, mas->tree, mas->index, mas->index); 2869 2760 MA_STATE(r_mas, mas->tree, mas->index, mas->last); ··· 2876 2765 mast.orig_l = &mast_l_mas; 2877 2766 mast.orig_r = r_wr_mas->mas; 2878 2767 memset(&b_node, 0, sizeof(struct maple_big_node)); 2879 - /* Copy l_mas and store the value in b_node. */ 2880 - mas_store_b_node(wr_mas, &b_node, mast.orig_l->end); 2881 - /* Copy r_mas into b_node if there is anything to copy. */ 2882 - if (mast.orig_r->max > mast.orig_r->last) 2883 - mas_mab_cp(mast.orig_r, mast.orig_r->offset, 2884 - mast.orig_r->end, &b_node, b_node.b_end + 1); 2885 - else 2886 - b_node.b_end++; 2768 + cp.s_count = 0; 2769 + cp_leaf_init(&cp, mas, l_wr_mas, r_wr_mas); 2770 + /* Copy left 0 - offset */ 2771 + if (l_wr_mas->mas->offset) { 2772 + unsigned char off = l_wr_mas->mas->offset - 1; 2887 2773 2774 + append_wr_mas_cp(&cp, l_wr_mas, 0, off); 2775 + cp.src[cp.s_count - 1].max = cp.min - 1; 2776 + } 2777 + 2778 + init_cp_src(&cp); 2779 + 2780 + /* Copy right from offset_end + 1 to end */ 2781 + if (r_wr_mas->mas->end != r_wr_mas->offset_end) 2782 + append_wr_mas_cp(&cp, r_wr_mas, r_wr_mas->offset_end + 1, 2783 + r_wr_mas->mas->end); 2784 + 2785 + 2786 + b_node.type = l_wr_mas->type; 2787 + cp_data_write(&cp, &b_node); 2888 2788 /* Stop spanning searches by searching for just index. */ 2889 2789 mast.orig_l->last = mas->index; 2890 2790