Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
maple_tree: change initial big node setup in mas_wr_spanning_rebalance()
Instead of copying the data into the big node and finding out that the
data may need to be moved or appended to, calculate the data space up
front (in the maple copy node) and set up another source for the copy.
The additional copy source is tracked in the maple state sib (short for
sibling), and is put into the maple write states for future operations
after the data is in the big node.
To facilitate the newly moved node, some initial setup of the maple
subtree state are relocated after the potential shift caused by the new
way of rebalancing against a sibling.
Link: https://lkml.kernel.org/r/20260130205935.2559335-15-Liam.Howlett@oracle.com
Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Andrew Ballance <andrewjballance@gmail.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Christian Kujau <lists@nerdbynature.de>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Liam R. Howlett
and committed by
Andrew Morton
6953038c
f141d566
+153
-23
+1
include/linux/maple_tree.h
+1
include/linux/maple_tree.h
+152
-23
lib/maple_tree.c
+152
-23
lib/maple_tree.c
···
1304
1304
return mt_pivots[type];
1305
1305
}
1306
1306
1307
+
static inline
1308
+
void wr_mas_setup(struct ma_wr_state *wr_mas, struct ma_state *mas)
1309
+
{
1310
+
wr_mas->node = mas_mn(mas);
1311
+
wr_mas->type = mte_node_type(mas->node);
1312
+
wr_mas->pivots = ma_pivots(wr_mas->node, wr_mas->type);
1313
+
wr_mas->slots = ma_slots(wr_mas->node, wr_mas->type);
1314
+
wr_mas->r_min = mas_safe_min(mas, wr_mas->pivots, mas->offset);
1315
+
wr_mas->r_max = mas_safe_pivot(mas, wr_mas->pivots, mas->offset,
1316
+
wr_mas->type);
1317
+
}
1318
+
1307
1319
/*
1308
1320
* mas_leaf_max_gap() - Returns the largest gap in a leaf node
1309
1321
* @mas: the maple state
···
2270
2258
*split = mid_split;
2271
2259
}
2272
2260
2261
+
static inline
2262
+
void spanning_sib(struct ma_wr_state *l_wr_mas,
2263
+
struct ma_wr_state *r_wr_mas, struct ma_state *nneighbour)
2264
+
{
2265
+
struct ma_state l_tmp = *l_wr_mas->mas;
2266
+
struct ma_state r_tmp = *r_wr_mas->mas;
2267
+
unsigned char depth = 0;
2268
+
2269
+
do {
2270
+
mas_ascend(&r_tmp);
2271
+
mas_ascend(&l_tmp);
2272
+
depth++;
2273
+
if (r_tmp.offset < mas_data_end(&r_tmp)) {
2274
+
r_tmp.offset++;
2275
+
mas_descend(&r_tmp);
2276
+
r_tmp.offset = 0;
2277
+
while (--depth)
2278
+
mas_descend(&r_tmp);
2279
+
2280
+
r_tmp.end = mas_data_end(&r_tmp);
2281
+
*nneighbour = r_tmp;
2282
+
return;
2283
+
} else if (l_tmp.offset) {
2284
+
l_tmp.offset--;
2285
+
do {
2286
+
mas_descend(&l_tmp);
2287
+
l_tmp.offset = mas_data_end(&l_tmp);
2288
+
} while (--depth);
2289
+
2290
+
l_tmp.end = l_tmp.offset;
2291
+
*nneighbour = l_tmp;
2292
+
return;
2293
+
}
2294
+
} while (!mte_is_root(r_tmp.node));
2295
+
2296
+
WARN_ON_ONCE(1);
2297
+
}
2298
+
2273
2299
/*
2274
2300
* mast_set_split_parents() - Helper function to set three nodes parents. Slot
2275
2301
* is taken from @mast->l.
···
2692
2642
cp->end = end;
2693
2643
}
2694
2644
2645
+
/*
2646
+
* cp_data_calc() - Calculate the size of the data (1 indexed).
2647
+
* @cp: The maple copy struct with the new data populated.
2648
+
* @l_wr_mas: The maple write state containing the data to the left of the write
2649
+
* @r_wr_mas: The maple write state containing the data to the right of the
2650
+
* write
2651
+
*
2652
+
* cp->data is a size (not indexed by 0).
2653
+
*/
2654
+
static inline void cp_data_calc(struct maple_copy *cp,
2655
+
struct ma_wr_state *l_wr_mas, struct ma_wr_state *r_wr_mas)
2656
+
{
2657
+
2658
+
/* Add 1 every time for the 0th element */
2659
+
cp->data = l_wr_mas->mas->offset;
2660
+
/* Add the new data and any partial overwrites */
2661
+
cp->data += cp->end + 1;
2662
+
/* Data from right (offset + 1 to end), +1 for zero */
2663
+
cp->data += r_wr_mas->mas->end - r_wr_mas->offset_end;
2664
+
}
2665
+
2666
+
static inline void append_mas_cp(struct maple_copy *cp,
2667
+
struct ma_state *mas, unsigned char start, unsigned char end)
2668
+
{
2669
+
struct maple_node *node;
2670
+
enum maple_type mt;
2671
+
unsigned char count;
2672
+
2673
+
count = cp->s_count;
2674
+
node = mas_mn(mas);
2675
+
mt = mte_node_type(mas->node);
2676
+
cp->src[count].node = node;
2677
+
cp->src[count].mt = mt;
2678
+
if (mas->end <= end)
2679
+
cp->src[count].max = mas->max;
2680
+
else
2681
+
cp->src[count].max = ma_pivots(node, mt)[end];
2682
+
2683
+
cp->src[count].start = start;
2684
+
cp->src[count].end = end;
2685
+
cp->s_count++;
2686
+
}
2687
+
2695
2688
static inline void append_wr_mas_cp(struct maple_copy *cp,
2696
2689
struct ma_wr_state *wr_mas, unsigned char start, unsigned char end)
2697
2690
{
···
2761
2668
cp->src[cp->s_count].start = 0;
2762
2669
cp->src[cp->s_count].end = cp->end;
2763
2670
cp->s_count++;
2671
+
}
2672
+
2673
+
/*
2674
+
* multi_src_setup() - Set the @cp node up with multiple sources to copy from.
2675
+
* @cp: The maple copy node
2676
+
* @l_wr_mas: The left write maple state
2677
+
* @r_wr_mas: The right write maple state
2678
+
* @sib: The sibling maple state
2679
+
*
2680
+
* Note: @sib->end == 0 indicates no sibling will be used.
2681
+
*/
2682
+
static inline
2683
+
void multi_src_setup(struct maple_copy *cp, struct ma_wr_state *l_wr_mas,
2684
+
struct ma_wr_state *r_wr_mas, struct ma_state *sib)
2685
+
{
2686
+
cp->s_count = 0;
2687
+
if (sib->end && sib->max < l_wr_mas->mas->min)
2688
+
append_mas_cp(cp, sib, 0, sib->end);
2689
+
2690
+
/* Copy left 0 - offset */
2691
+
if (l_wr_mas->mas->offset) {
2692
+
unsigned char off = l_wr_mas->mas->offset - 1;
2693
+
2694
+
append_wr_mas_cp(cp, l_wr_mas, 0, off);
2695
+
cp->src[cp->s_count - 1].max = cp->min - 1;
2696
+
}
2697
+
2698
+
init_cp_src(cp);
2699
+
2700
+
/* Copy right either from offset or offset + 1 pending on r_max */
2701
+
if (r_wr_mas->mas->end != r_wr_mas->offset_end)
2702
+
append_wr_mas_cp(cp, r_wr_mas, r_wr_mas->offset_end + 1,
2703
+
r_wr_mas->mas->end);
2704
+
2705
+
if (sib->end && sib->min > r_wr_mas->mas->max)
2706
+
append_mas_cp(cp, sib, 0, sib->end);
2764
2707
}
2765
2708
2766
2709
static inline
···
3002
2873
struct maple_big_node b_node;
3003
2874
struct maple_copy cp;
3004
2875
unsigned char height;
2876
+
struct ma_state sib;
3005
2877
MA_STATE(l_mas, mas->tree, mas->index, mas->index);
3006
2878
MA_STATE(r_mas, mas->tree, mas->index, mas->last);
3007
2879
MA_STATE(m_mas, mas->tree, mas->index, mas->index);
3008
2880
MA_STATE(mast_l_mas, NULL, 0, 0);
3009
2881
3010
2882
3011
-
mast_l_mas = *mas;
3012
-
mast.orig_l = &mast_l_mas;
3013
-
mast.orig_r = r_wr_mas->mas;
3014
2883
memset(&b_node, 0, sizeof(struct maple_big_node));
2884
+
mast_l_mas = *mas;
3015
2885
cp.s_count = 0;
3016
2886
cp_leaf_init(&cp, mas, l_wr_mas, r_wr_mas);
3017
-
/* Copy left 0 - offset */
3018
-
if (l_wr_mas->mas->offset) {
3019
-
unsigned char off = l_wr_mas->mas->offset - 1;
3020
-
3021
-
append_wr_mas_cp(&cp, l_wr_mas, 0, off);
3022
-
cp.src[cp.s_count - 1].max = cp.min - 1;
2887
+
cp_data_calc(&cp, l_wr_mas, r_wr_mas);
2888
+
if (((l_wr_mas->mas->min != 0) || (r_wr_mas->mas->max != ULONG_MAX)) &&
2889
+
(cp.data <= mt_min_slots[l_wr_mas->type])) {
2890
+
spanning_sib(l_wr_mas, r_wr_mas, &sib);
2891
+
cp.data += sib.end + 1;
2892
+
} else {
2893
+
sib.end = 0;
3023
2894
}
3024
2895
3025
-
init_cp_src(&cp);
3026
-
3027
-
/* Copy right from offset_end + 1 to end */
3028
-
if (r_wr_mas->mas->end != r_wr_mas->offset_end)
3029
-
append_wr_mas_cp(&cp, r_wr_mas, r_wr_mas->offset_end + 1,
3030
-
r_wr_mas->mas->end);
3031
-
3032
-
2896
+
multi_src_setup(&cp, l_wr_mas, r_wr_mas, &sib);
3033
2897
b_node.type = l_wr_mas->type;
3034
2898
cp_data_write(&cp, &b_node);
2899
+
if (sib.end) {
2900
+
if (sib.max < l_wr_mas->mas->min) {
2901
+
*l_wr_mas->mas = sib;
2902
+
wr_mas_setup(l_wr_mas, &sib);
2903
+
mast_l_mas = sib;
2904
+
} else {
2905
+
*r_wr_mas->mas = sib;
2906
+
wr_mas_setup(r_wr_mas, &sib);
2907
+
}
2908
+
}
2909
+
2910
+
mast.orig_l = &mast_l_mas;
2911
+
mast.orig_r = r_wr_mas->mas;
3035
2912
/* Stop spanning searches by searching for just index. */
3036
2913
mast.orig_l->last = mas->index;
3037
2914
···
3052
2917
mast.m = &m_mas;
3053
2918
mast.r = &r_mas;
3054
2919
l_mas.status = r_mas.status = m_mas.status = ma_none;
3055
-
3056
-
/* Check if this is not root and has sufficient data. */
3057
-
if (((mast.orig_l->min != 0) || (mast.orig_r->max != ULONG_MAX)) &&
3058
-
unlikely(mast.bn->b_end <= mt_min_slots[mast.bn->type]))
3059
-
mast_spanning_rebalance(&mast);
3060
-
3061
2920
height = mas_mt_height(mas) + 1;
3062
2921
3063
2922
/*