maple_tree: start using maple copy node for destination

+14

include/linux/maple_tree.h

··· 156 156 }; 157 157 158 158 struct maple_copy { 159 + /* 160 + * min, max, and pivots are values 161 + * start, end, split are indexes into arrays 162 + * data is a size 163 + */ 164 + 165 + struct { 166 + struct maple_node *node; 167 + unsigned long max; 168 + enum maple_type mt; 169 + } dst[3]; 159 170 struct { 160 171 struct maple_node *node; 161 172 unsigned long max; ··· 189 178 190 179 /*Avoid passing these around */ 191 180 unsigned char s_count; 181 + unsigned char d_count; 182 + unsigned char split; 192 183 unsigned char data; 184 + unsigned char height; 193 185 }; 194 186 195 187 /**

+443 -181

lib/maple_tree.c

··· 353 353 (type << MAPLE_ENODE_TYPE_SHIFT) | MAPLE_ENODE_NULL); 354 354 } 355 355 356 + static inline void ma_init_slot(void __rcu **slot, const struct maple_node *mn, 357 + const enum maple_type mt) 358 + { 359 + /* WARNING: this is unsafe if the slot is exposed to readers. */ 360 + RCU_INIT_POINTER(*slot, (void *)mt_mk_node(mn, mt)); 361 + } 362 + 356 363 static inline void *mte_mk_root(const struct maple_enode *node) 357 364 { 358 365 return (void *)((unsigned long)node | MAPLE_ROOT_NODE); ··· 1327 1320 wr_mas->r_max = mas_safe_pivot(mas, wr_mas->pivots, mas->offset, 1328 1321 wr_mas->type); 1329 1322 } 1323 + 1324 + static inline 1325 + void wr_mas_ascend(struct ma_wr_state *wr_mas) 1326 + { 1327 + struct ma_state *mas = wr_mas->mas; 1328 + 1329 + mas_ascend(mas); 1330 + wr_mas_setup(wr_mas, mas); 1331 + mas->end = ma_data_end(wr_mas->node, wr_mas->type, wr_mas->pivots, 1332 + mas->max); 1333 + /* Careful, this may be wrong.. */ 1334 + wr_mas->end_piv = wr_mas->r_max; 1335 + wr_mas->offset_end = mas->offset; 1336 + } 1337 + 1330 1338 static inline unsigned long ma_leaf_max_gap(struct maple_node *mn, 1331 1339 enum maple_type mt, unsigned long min, unsigned long max, 1332 1340 unsigned long *pivots, void __rcu **slots) ··· 2530 2508 } 2531 2509 2532 2510 /* 2511 + * node_copy() - Copy from one node to another. 2512 + * 2513 + * @mas: The maple state 2514 + * @src: The source node 2515 + * @start: The offset into the src to start copying 2516 + * @size: The size to copy (non-zero) 2517 + * @s_max: The source node max 2518 + * @s_mt: The source maple node type 2519 + * @dst: The destination 2520 + * @d_start: The start location in the destination node 2521 + * @d_mt: The destination maple node type 2522 + */ 2523 + static inline 2524 + unsigned long node_copy(struct ma_state *mas, struct maple_node *src, 2525 + unsigned char start, unsigned char size, unsigned long s_max, 2526 + enum maple_type s_mt, struct maple_node *dst, unsigned char d_start, 2527 + enum maple_type d_mt) 2528 + { 2529 + unsigned long *s_pivots, *d_pivots; 2530 + void __rcu **s_slots, **d_slots; 2531 + unsigned long *s_gaps, *d_gaps; 2532 + unsigned long d_max; 2533 + 2534 + d_slots = ma_slots(dst, d_mt) + d_start; 2535 + d_pivots = ma_pivots(dst, d_mt) + d_start; 2536 + s_slots = ma_slots(src, s_mt) + start; 2537 + s_pivots = ma_pivots(src, s_mt) + start; 2538 + memcpy(d_slots, s_slots, size * sizeof(void __rcu *)); 2539 + if (!ma_is_leaf(d_mt) && s_mt == maple_copy) { 2540 + struct maple_enode *edst = mt_mk_node(dst, d_mt); 2541 + 2542 + 2543 + for (int i = 0; i < size; i++) 2544 + mas_set_parent(mas, 2545 + mt_slot_locked(mas->tree, d_slots, i), 2546 + edst, d_start + i); 2547 + } 2548 + 2549 + d_gaps = ma_gaps(dst, d_mt); 2550 + if (d_gaps) { 2551 + s_gaps = ma_gaps(src, s_mt) + start; 2552 + d_gaps += d_start; 2553 + memcpy(d_gaps, s_gaps, size * sizeof(unsigned long)); 2554 + } 2555 + 2556 + if (start + size - 1 < mt_pivots[s_mt]) 2557 + d_max = s_pivots[size - 1]; 2558 + else 2559 + d_max = s_max; 2560 + 2561 + if (d_start + size <= mt_pivots[d_mt]) 2562 + d_pivots[size - 1] = d_max; 2563 + 2564 + size--; 2565 + if (size) 2566 + memcpy(d_pivots, s_pivots, size * sizeof(unsigned long)); 2567 + 2568 + return d_max; 2569 + } 2570 + 2571 + /* 2572 + * node_finalise() - Zero out unused area and populate metadata 2573 + * @node: The maple node 2574 + * @mt: The maple node type 2575 + * @end: The end of the used area 2576 + */ 2577 + static inline 2578 + void node_finalise(struct maple_node *node, enum maple_type mt, 2579 + unsigned char end) 2580 + { 2581 + unsigned char max_end = mt_slots[mt]; 2582 + unsigned char size; 2583 + unsigned long *gaps; 2584 + unsigned char gap_slot; 2585 + 2586 + gaps = ma_gaps(node, mt); 2587 + if (end < max_end - 1) { 2588 + size = max_end - end; 2589 + memset(ma_slots(node, mt) + end, 0, size * sizeof(void *)); 2590 + 2591 + if (gaps) 2592 + memset(gaps + end, 0, size * sizeof(unsigned long)); 2593 + 2594 + if (--size) 2595 + memset(ma_pivots(node, mt) + end, 0, size * sizeof(unsigned long)); 2596 + } 2597 + 2598 + gap_slot = 0; 2599 + if (gaps && !ma_is_leaf(mt)) { 2600 + unsigned long max_gap; 2601 + 2602 + max_gap = 0; 2603 + for (int i = 0; i <= end; i++) 2604 + if (gaps[i] > max_gap) { 2605 + gap_slot = i; 2606 + max_gap = gaps[i]; 2607 + } 2608 + } 2609 + 2610 + if (mt == maple_arange_64) 2611 + ma_set_meta(node, mt, gap_slot, end - 1); 2612 + else if (end <= max_end - 1) 2613 + ma_set_meta(node, mt, gap_slot, end - 1); 2614 + } 2615 + 2616 + /* 2533 2617 * mast_cp_to_nodes() - Copy data out to nodes. 2534 2618 * @mast: The maple subtree state 2535 2619 * @left: The left encoded maple node ··· 2812 2684 * result in buggy code when a compiler reorders the instructions. 2813 2685 */ 2814 2686 2687 + cp->height = 1; 2815 2688 /* Create entries to insert including split entries to left and right */ 2816 2689 if (l_wr_mas->r_min < mas->index) { 2817 2690 end++; ··· 2853 2724 cp->data += cp->end + 1; 2854 2725 /* Data from right (offset + 1 to end), +1 for zero */ 2855 2726 cp->data += r_wr_mas->mas->end - r_wr_mas->offset_end; 2727 + } 2728 + 2729 + /* 2730 + * spanning_data() - Calculate the @cp data and populate @sib if insufficient 2731 + * @cp: The maple copy node 2732 + * @l_wr_mas: The left write maple state 2733 + * @r_wr_mas: The right write maple state 2734 + * @sib: The maple state of the sibling. 2735 + * 2736 + * Note: @cp->data is a size and not indexed by 0. @sib->end may be set to 0 to 2737 + * indicate it will not be used. 2738 + */ 2739 + static inline void spanning_data(struct maple_copy *cp, 2740 + struct ma_wr_state *l_wr_mas, struct ma_wr_state *r_wr_mas, 2741 + struct ma_state *sib) 2742 + { 2743 + cp_data_calc(cp, l_wr_mas, r_wr_mas); 2744 + if (((l_wr_mas->mas->min != 0) || (r_wr_mas->mas->max != ULONG_MAX)) && 2745 + (cp->data <= mt_min_slots[l_wr_mas->type])) { 2746 + spanning_sib(l_wr_mas, r_wr_mas, sib); 2747 + cp->data += sib->end + 1; 2748 + } else { 2749 + sib->end = 0; 2750 + } 2751 + } 2752 + 2753 + /* 2754 + * dst_setup() - Set up one or more destinations for the new data. 2755 + * @cp: The maple copy node 2756 + * @mas: The maple state 2757 + * @mt: The source node type 2758 + */ 2759 + static inline 2760 + void dst_setup(struct maple_copy *cp, struct ma_state *mas, enum maple_type mt) 2761 + { 2762 + /* Data is 1 indexed, every src has +1 added. */ 2763 + 2764 + if (cp->data <= mt_slots[mt]) { 2765 + cp->split = cp->data - 1; 2766 + cp->d_count = 1; 2767 + goto node_setup; 2768 + } 2769 + 2770 + cp->split = (cp->data - 1) / 2; 2771 + cp->d_count = 2; 2772 + if (cp->data < mt_slots[mt] * 2) 2773 + goto node_setup; 2774 + 2775 + if (cp->data == mt_slots[mt] * 2) { 2776 + unsigned char off; 2777 + unsigned char s; 2778 + 2779 + if (!ma_is_leaf(mt)) 2780 + goto node_setup; 2781 + 2782 + /* 2783 + * Leaf nodes are a bit tricky because we cannot assume the data 2784 + * can fit due to the NULL limitation on node ends. 2785 + */ 2786 + off = cp->split; 2787 + for (s = 0; s < cp->s_count; s++) { 2788 + unsigned char s_off; 2789 + 2790 + s_off = cp->src[s].end - cp->src[s].start; 2791 + if (s_off >= off) 2792 + break; 2793 + 2794 + s_off++; 2795 + off -= s_off; 2796 + } 2797 + 2798 + off += cp->src[s].start; 2799 + if (ma_slots(cp->src[s].node, cp->src[s].mt)[off]) 2800 + goto node_setup; 2801 + 2802 + cp->split++; 2803 + if (cp->split < mt_slots[mt]) 2804 + goto node_setup; 2805 + 2806 + cp->split -= 2; 2807 + if (cp->data - 2 - cp->split < mt_slots[mt]) 2808 + goto node_setup; 2809 + 2810 + } 2811 + 2812 + /* No other choice but to 3-way split the data */ 2813 + cp->split = (cp->data + 2) / 3; 2814 + cp->d_count = 3; 2815 + 2816 + node_setup: 2817 + for (int i = 0; i < cp->d_count; i++) { 2818 + cp->dst[i].mt = mt; 2819 + cp->dst[i].node = ma_mnode_ptr(mas_pop_node(mas)); 2820 + } 2856 2821 } 2857 2822 2858 2823 static inline void append_mas_cp(struct maple_copy *cp, ··· 3036 2813 } 3037 2814 3038 2815 static inline 3039 - void cp_data_write(struct maple_copy *cp, struct maple_big_node *b_node) 2816 + void cp_data_write(struct maple_copy *cp, struct ma_state *mas) 3040 2817 { 3041 - struct maple_node *src; 3042 - unsigned char s; 2818 + struct maple_node *dst, *src; 2819 + unsigned char s, d; 2820 + unsigned char dst_offset; 2821 + unsigned char data_offset; 3043 2822 unsigned char src_end, s_offset; 3044 - unsigned long *b_pivots, *cp_pivots; 3045 - void __rcu **b_slots, **cp_slots; 3046 - enum maple_type s_mt; 2823 + unsigned char split; 2824 + unsigned long s_max, d_max; 2825 + unsigned char dst_size; 2826 + enum maple_type s_mt, d_mt; 3047 2827 3048 - b_node->b_end = 0; 3049 - 3050 - s = 0; 3051 - b_pivots = b_node->pivot; 3052 - b_slots = (void __rcu **)b_node->slot; 2828 + data_offset = 0; 2829 + s = d = 0; 2830 + /* Readability help */ 2831 + src = cp->src[s].node; 2832 + dst = cp->dst[d].node; 2833 + s_offset = cp->src[s].start; 2834 + src_end = cp->src[s].end; 2835 + split = cp->split; 2836 + s_max = cp->src[s].max; 2837 + s_mt = cp->src[s].mt; 2838 + d_mt = cp->dst[d].mt; 3053 2839 do { 3054 - unsigned char size; 2840 + dst_offset = 0; 2841 + d_max = 0; 2842 + dst = cp->dst[d].node; 2843 + d_mt = cp->dst[d].mt; 2844 + dst_size = split + 1; 3055 2845 3056 - src = cp->src[s].node; 3057 - s_mt = cp->src[s].mt; 3058 - s_offset = cp->src[s].start; 3059 - src_end = cp->src[s].end; 3060 - size = src_end - s_offset + 1; 3061 - cp_pivots = ma_pivots(src, s_mt) + s_offset; 3062 - cp_slots = ma_slots(src, s_mt) + s_offset; 3063 - memcpy(b_slots, cp_slots, size * sizeof(void __rcu *)); 3064 - if (size > 1) 3065 - memcpy(b_pivots, cp_pivots, (size - 1) * sizeof(unsigned long)); 3066 - b_pivots[size - 1] = cp->src[s].max; 3067 - b_pivots += size; 3068 - b_slots += size; 3069 - b_node->b_end += size; 3070 - } while (++s < cp->s_count); 2846 + while (dst_size) { 2847 + unsigned char size; 2848 + 2849 + if (src_end - s_offset + 1 < dst_size) 2850 + size = src_end - s_offset + 1; 2851 + else 2852 + size = dst_size; 2853 + 2854 + d_max = node_copy(mas, src, s_offset, size, s_max, s_mt, 2855 + dst, dst_offset, d_mt); 2856 + 2857 + dst_offset += size; 2858 + s_offset += size; 2859 + if (s_offset > src_end) { 2860 + /* This source is exhausted */ 2861 + s++; 2862 + if (s >= cp->s_count) { 2863 + cp->dst[d].max = d_max; 2864 + node_finalise(dst, d_mt, dst_offset); 2865 + return; 2866 + } 2867 + /* Reset local src */ 2868 + src = cp->src[s].node; 2869 + s_offset = cp->src[s].start; 2870 + src_end = cp->src[s].end; 2871 + s_max = cp->src[s].max; 2872 + s_mt = cp->src[s].mt; 2873 + } 2874 + 2875 + dst_size -= size; 2876 + data_offset += size; 2877 + } 2878 + 2879 + split = cp->split; 2880 + cp->dst[d].max = d_max; 2881 + /* Handle null entries */ 2882 + if (cp->dst[d].max != ULONG_MAX && 2883 + !ma_slots(dst, d_mt)[dst_offset - 1]) { 2884 + if (s_offset == cp->src[s].start) { 2885 + s--; 2886 + src = cp->src[s].node; 2887 + src_end = cp->src[s].end; 2888 + s_max = cp->src[s].max; 2889 + s_mt = cp->src[s].mt; 2890 + s_offset = src_end; 2891 + } else { 2892 + s_offset--; 2893 + } 2894 + /* Set dst max and clear pivot */ 2895 + split++; 2896 + data_offset--; 2897 + dst_offset--; 2898 + cp->dst[d].max = ma_pivots(dst, d_mt)[dst_offset - 1]; 2899 + } 2900 + 2901 + node_finalise(dst, d_mt, dst_offset); 2902 + ++d; /* Next destination */ 2903 + if (d == cp->d_count - 1) 2904 + split = cp->data - data_offset; 2905 + 2906 + if (d >= cp->d_count) { 2907 + WARN_ON(data_offset < cp->data); 2908 + return; 2909 + } 2910 + 2911 + } while (data_offset <= cp->data); 2912 + } 2913 + 2914 + /* 2915 + * cp_dst_to_slots() - Migrate the maple copy destination to the maple copy 2916 + * slots 2917 + * @cp: The maple copy node 2918 + * @min: The minimal value represented 2919 + * @max: The maximum value represented 2920 + * @mas: The maple state 2921 + */ 2922 + static inline void cp_dst_to_slots(struct maple_copy *cp, unsigned long min, 2923 + unsigned long max, struct ma_state *mas) 2924 + { 2925 + unsigned char d; 2926 + unsigned long slot_min = min; 2927 + 2928 + for (d = 0; d < cp->d_count; d++) { 2929 + struct maple_node *mn = cp->dst[d].node; 2930 + enum maple_type mt = cp->dst[d].mt; 2931 + unsigned long slot_max = cp->dst[d].max; 2932 + 2933 + /* 2934 + * Warning, see cp_leaf_init() comment and rcu_assign_pointer() 2935 + * documentation. Since these are new nodes, there are no 2936 + * read-side operations that can view them until they are 2937 + * inserted into the tree after an rcu_assign_pointer() call. 2938 + */ 2939 + ma_init_slot(&cp->slot[d], mn, mt); 2940 + cp->pivot[d] = slot_max; 2941 + if (mt_is_alloc(mas->tree)) { 2942 + if (ma_is_leaf(mt)) { 2943 + cp->gap[d] = ma_leaf_max_gap(mn, mt, slot_min, 2944 + slot_max, ma_pivots(mn, mt), 2945 + ma_slots(mn, mt)); 2946 + } else { 2947 + unsigned long *gaps = ma_gaps(mn, mt); 2948 + 2949 + if (gaps) { 2950 + unsigned char gap_slot; 2951 + 2952 + gap_slot = ma_meta_gap(mn); 2953 + cp->gap[d] = gaps[gap_slot]; 2954 + } 2955 + } 2956 + } 2957 + slot_min = slot_max + 1; 2958 + } 2959 + 2960 + cp->end = cp->d_count - 1; 2961 + cp->min = min; 2962 + cp->max = max; 3071 2963 } 3072 2964 3073 2965 static void mas_spanning_rebalance_loop(struct ma_state *mas, ··· 3338 3000 mas_spanning_rebalance_loop(mas, mast, count); 3339 3001 } 3340 3002 3003 + /* 3004 + * spanning_ascend() - See if a spanning store operation has to keep walking up 3005 + * the tree 3006 + * @cp: The maple_copy node 3007 + * @l_wr_mas: The left maple write state 3008 + * @r_wr_mas: The right maple write state 3009 + * @sib: the maple state of the sibling 3010 + * 3011 + * Returns: True if another iteration is necessary. 3012 + */ 3013 + static bool spanning_ascend(struct maple_copy *cp, struct ma_state *mas, 3014 + struct ma_wr_state *l_wr_mas, struct ma_wr_state *r_wr_mas, 3015 + struct ma_state *sib) 3016 + { 3017 + if (sib->end) { 3018 + if (sib->max < l_wr_mas->mas->min) 3019 + *l_wr_mas->mas = *sib; 3020 + else 3021 + *r_wr_mas->mas = *sib; 3022 + } 3023 + 3024 + cp_dst_to_slots(cp, l_wr_mas->mas->min, r_wr_mas->mas->max, mas); 3025 + if (!cp->min && cp->max == ULONG_MAX) { 3026 + /* New root */ 3027 + if (cp->d_count != 1) { 3028 + enum maple_type mt = maple_arange_64; 3029 + 3030 + if (!mt_is_alloc(mas->tree)) 3031 + mt = maple_range_64; 3032 + 3033 + cp->data = cp->d_count; 3034 + cp->s_count = 0; 3035 + dst_setup(cp, mas, mt); 3036 + init_cp_src(cp); 3037 + node_copy(mas, cp->src[0].node, 0, cp->data, cp->max, maple_copy, 3038 + cp->dst[0].node, 0, mt); 3039 + node_finalise(cp->dst[0].node, mt, cp->end + 1); 3040 + /* 3041 + * Warning, see cp_leaf_init() comment and rcu_assign_pointer() 3042 + * documentation. Since this is a new root, there are no 3043 + * read-side operations that can view it until it is insert into 3044 + * the tree after an rcu_assign_pointer() call. 3045 + */ 3046 + ma_init_slot(&cp->slot[0], cp->dst[0].node, mt); 3047 + cp->height++; 3048 + } 3049 + WARN_ON_ONCE(cp->dst[0].node != mte_to_node( 3050 + mt_slot_locked(mas->tree, cp->slot, 0))); 3051 + cp->dst[0].node->parent = ma_parent_ptr(mas_tree_parent(mas)); 3052 + mas->min = 0; 3053 + mas->max = ULONG_MAX; 3054 + mas->depth = 0; 3055 + mas->node = mas_root_locked(mas); 3056 + return false; 3057 + } 3058 + 3059 + /* Converged and has a single destination */ 3060 + if ((cp->d_count == 1) && 3061 + (l_wr_mas->mas->node == r_wr_mas->mas->node)) { 3062 + cp->dst[0].node->parent = ma_parent_ptr(mas_mn(mas)->parent); 3063 + return false; 3064 + } 3065 + 3066 + cp->height++; 3067 + wr_mas_ascend(l_wr_mas); 3068 + wr_mas_ascend(r_wr_mas); 3069 + return true; 3070 + } 3341 3071 3342 3072 static noinline void mas_wr_spanning_rebalance(struct ma_state *mas, 3343 3073 struct ma_wr_state *l_wr_mas, struct ma_wr_state *r_wr_mas) 3344 3074 { 3345 3075 3346 - unsigned char split, mid_split; 3347 - unsigned char slot = 0; 3348 - unsigned char new_height = 0; /* used if node is a new root */ 3349 - struct maple_enode *left = NULL, *middle = NULL, *right = NULL; 3350 3076 struct maple_enode *old_enode; 3351 - 3352 - struct maple_subtree_state mast; 3353 - struct maple_big_node b_node; 3354 3077 struct maple_copy cp; 3355 - unsigned char height; 3356 3078 struct ma_state sib; 3357 - MA_STATE(l_mas, mas->tree, mas->index, mas->index); 3358 - MA_STATE(r_mas, mas->tree, mas->index, mas->last); 3359 - MA_STATE(m_mas, mas->tree, mas->index, mas->index); 3360 - MA_STATE(mast_l_mas, NULL, 0, 0); 3361 3079 3362 - 3363 - memset(&b_node, 0, sizeof(struct maple_big_node)); 3364 - mast_l_mas = *mas; 3365 - cp.s_count = 0; 3366 3080 cp_leaf_init(&cp, mas, l_wr_mas, r_wr_mas); 3367 - cp_data_calc(&cp, l_wr_mas, r_wr_mas); 3368 - if (((l_wr_mas->mas->min != 0) || (r_wr_mas->mas->max != ULONG_MAX)) && 3369 - (cp.data <= mt_min_slots[l_wr_mas->type])) { 3370 - spanning_sib(l_wr_mas, r_wr_mas, &sib); 3371 - cp.data += sib.end + 1; 3372 - } else { 3373 - sib.end = 0; 3374 - } 3081 + do { 3082 + spanning_data(&cp, l_wr_mas, r_wr_mas, &sib); 3083 + multi_src_setup(&cp, l_wr_mas, r_wr_mas, &sib); 3084 + dst_setup(&cp, mas, l_wr_mas->type); 3085 + cp_data_write(&cp, mas); 3086 + } while (spanning_ascend(&cp, mas, l_wr_mas, r_wr_mas, &sib)); 3375 3087 3376 - multi_src_setup(&cp, l_wr_mas, r_wr_mas, &sib); 3377 - b_node.type = l_wr_mas->type; 3378 - cp_data_write(&cp, &b_node); 3379 - if (sib.end) { 3380 - if (sib.max < l_wr_mas->mas->min) { 3381 - *l_wr_mas->mas = sib; 3382 - wr_mas_setup(l_wr_mas, &sib); 3383 - mast_l_mas = sib; 3384 - } else { 3385 - *r_wr_mas->mas = sib; 3386 - wr_mas_setup(r_wr_mas, &sib); 3387 - } 3388 - } 3389 - 3390 - mast.orig_l = &mast_l_mas; 3391 - mast.orig_r = r_wr_mas->mas; 3392 - /* Stop spanning searches by searching for just index. */ 3393 - mast.orig_l->last = mas->index; 3394 - 3395 - mast.bn = &b_node; 3396 - /* Combine l_mas and r_mas and split them up evenly again. */ 3397 - 3398 - /* 3399 - * The tree needs to be rebalanced and leaves need to be kept at the same level. 3400 - * Rebalancing is done by use of the ``struct maple_topiary``. 3401 - */ 3402 - mast.l = &l_mas; 3403 - mast.m = &m_mas; 3404 - mast.r = &r_mas; 3405 - l_mas.status = r_mas.status = m_mas.status = ma_none; 3406 - height = mas_mt_height(mas) + 1; 3407 - 3408 - /* 3409 - * Each level of the tree is examined and balanced, pushing data to the left or 3410 - * right, or rebalancing against left or right nodes is employed to avoid 3411 - * rippling up the tree to limit the amount of churn. Once a new sub-section of 3412 - * the tree is created, there may be a mix of new and old nodes. The old nodes 3413 - * will have the incorrect parent pointers and currently be in two trees: the 3414 - * original tree and the partially new tree. To remedy the parent pointers in 3415 - * the old tree, the new data is swapped into the active tree and a walk down 3416 - * the tree is performed and the parent pointers are updated. 3417 - * See mas_topiary_replace() for more information. 3418 - */ 3419 - while (height--) { 3420 - mast.bn->b_end--; 3421 - mast.bn->type = mte_node_type(mast.orig_l->node); 3422 - split = mas_mab_to_node(mas, mast.bn, &left, &right, &middle, 3423 - &mid_split); 3424 - mast_set_split_parents(&mast, left, middle, right, split, 3425 - mid_split); 3426 - mast_cp_to_nodes(&mast, left, middle, right, split, mid_split); 3427 - new_height++; 3428 - 3429 - /* 3430 - * Copy data from next level in the tree to mast.bn from next 3431 - * iteration 3432 - */ 3433 - memset(mast.bn, 0, sizeof(struct maple_big_node)); 3434 - mast.bn->type = mte_node_type(left); 3435 - 3436 - /* Root already stored in l->node. */ 3437 - if (mas_is_root_limits(mast.l)) 3438 - goto new_root; 3439 - 3440 - mast_ascend(&mast); 3441 - mast_combine_cp_left(&mast); 3442 - mast.l->offset = mast.bn->b_end; 3443 - mab_set_b_end(mast.bn, mast.l, left); 3444 - mab_set_b_end(mast.bn, mast.m, middle); 3445 - mab_set_b_end(mast.bn, mast.r, right); 3446 - 3447 - /* Copy anything necessary out of the right node. */ 3448 - mast_combine_cp_right(&mast); 3449 - mast.orig_l->last = mast.orig_l->max; 3450 - 3451 - if (mast_sufficient(&mast)) { 3452 - if (mast_overflow(&mast)) 3453 - continue; 3454 - 3455 - if (mast.orig_l->node == mast.orig_r->node) { 3456 - /* 3457 - * The data in b_node should be stored in one 3458 - * node and in the tree 3459 - */ 3460 - slot = mast.l->offset; 3461 - break; 3462 - } 3463 - 3464 - continue; 3465 - } 3466 - 3467 - /* May be a new root stored in mast.bn */ 3468 - if (mas_is_root_limits(mast.orig_l)) 3469 - break; 3470 - 3471 - mast_spanning_rebalance(&mast); 3472 - 3473 - /* rebalancing from other nodes may require another loop. */ 3474 - if (!height) 3475 - height++; 3476 - } 3477 - 3478 - mast.l->node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), 3479 - mte_node_type(mast.orig_l->node)); 3480 - 3481 - mab_mas_cp(mast.bn, 0, mt_slots[mast.bn->type] - 1, mast.l, true); 3482 - new_height++; 3483 - mas_set_parent(mas, left, mast.l->node, slot); 3484 - if (middle) 3485 - mas_set_parent(mas, middle, mast.l->node, ++slot); 3486 - 3487 - if (right) 3488 - mas_set_parent(mas, right, mast.l->node, ++slot); 3489 - 3490 - if (mas_is_root_limits(mast.l)) { 3491 - new_root: 3492 - mas_mn(mast.l)->parent = ma_parent_ptr(mas_tree_parent(mas)); 3493 - while (!mte_is_root(mast.orig_l->node)) 3494 - mast_ascend(&mast); 3495 - } else { 3496 - mas_mn(mast.l)->parent = mas_mn(mast.orig_l)->parent; 3497 - } 3498 - 3499 - old_enode = mast.orig_l->node; 3500 - mas->depth = mast.l->depth; 3501 - mas->node = mast.l->node; 3502 - mas->min = mast.l->min; 3503 - mas->max = mast.l->max; 3504 - mas->offset = mast.l->offset; 3505 - mas_wmb_replace(mas, old_enode, new_height); 3088 + old_enode = mas->node; 3089 + mas->node = mt_slot_locked(mas->tree, cp.slot, 0); 3090 + mas_wmb_replace(mas, old_enode, cp.height); 3506 3091 mtree_range_walk(mas); 3507 3092 } 3093 + 3508 3094 /* 3509 3095 * mas_rebalance() - Rebalance a given node. 3510 3096 * @mas: The maple state

+1 -1

tools/testing/radix-tree/maple.c

··· 35508 35508 /* Store a value across a node boundary that causes a 3 way split */ 35509 35509 35510 35510 if (MAPLE_32BIT) 35511 - i = 49590; /* 0xc1b6 */ 35511 + i = 49430; /* 0xc116 */ 35512 35512 else 35513 35513 i = 49670; /* 0xC206 */ 35514 35514

Configure Feed

Configure Feed