Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
maple_tree: use maple copy node for mas_wr_split()
Instead of using the maple big node, use the maple copy node for reduced
stack usage and aligning with mas_wr_rebalance() and
mas_wr_spanning_store().
Splitting a node is similar to rebalancing, but a new evaluation of when
to ascend is needed. The only other difference is that the data is pushed
and never rebalanced at each level.
The testing must also align with the changes to this commit to ensure the
test suite continues to pass.
Link: https://lkml.kernel.org/r/20260130205935.2559335-27-Liam.Howlett@oracle.com
Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Andrew Ballance <andrewjballance@gmail.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Christian Kujau <lists@nerdbynature.de>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Liam R. Howlett
and committed by
Andrew Morton
280b792c
11e7f22f
+149
-16
+93
-6
lib/maple_tree.c
+93
-6
lib/maple_tree.c
···
4543
4543
}
4544
4544
4545
4545
/*
4546
+
* split_ascend() - See if a split operation has to keep walking up the tree
4547
+
* @cp: The maple_copy node
4548
+
* @wr_mas: The maple write state
4549
+
* @sib: the maple state of the sibling
4550
+
*
4551
+
* Return: true if another split operation on the next level is needed, false
4552
+
* otherwise
4553
+
*/
4554
+
static inline bool split_ascend(struct maple_copy *cp,
4555
+
struct ma_wr_state *wr_mas, struct ma_state *sib,
4556
+
struct ma_state *parent)
4557
+
{
4558
+
struct ma_state *mas;
4559
+
unsigned long min, max;
4560
+
4561
+
mas = wr_mas->mas;
4562
+
min = mas->min; /* push right, or normal split */
4563
+
max = mas->max;
4564
+
wr_mas->offset_end = parent->offset;
4565
+
if (sib->end) {
4566
+
if (sib->max < mas->min) {
4567
+
min = sib->min; /* push left */
4568
+
parent->offset--;
4569
+
} else {
4570
+
max = sib->max; /* push right */
4571
+
wr_mas->offset_end++;
4572
+
}
4573
+
}
4574
+
4575
+
cp_dst_to_slots(cp, min, max, mas);
4576
+
if (cp_is_new_root(cp, mas))
4577
+
return false;
4578
+
4579
+
if (cp_converged(cp, mas, sib))
4580
+
return false;
4581
+
4582
+
cp->height++;
4583
+
copy_tree_location(parent, mas);
4584
+
wr_mas_setup(wr_mas, mas);
4585
+
return true;
4586
+
}
4587
+
4588
+
/*
4589
+
* split_data() - Calculate the @cp data, populate @sib if the data can be
4590
+
* pushed into a sibling.
4591
+
* @cp: The maple copy node
4592
+
* @wr_mas: The left write maple state
4593
+
* @sib: The maple state of the sibling.
4594
+
*
4595
+
* Note: @cp->data is a size and not indexed by 0. @sib->end may be set to 0 to
4596
+
* indicate it will not be used.
4597
+
*
4598
+
*/
4599
+
static inline void split_data(struct maple_copy *cp,
4600
+
struct ma_wr_state *wr_mas, struct ma_state *sib,
4601
+
struct ma_state *parent)
4602
+
{
4603
+
cp_data_calc(cp, wr_mas, wr_mas);
4604
+
if (cp->data <= mt_slots[wr_mas->type]) {
4605
+
sib->end = 0;
4606
+
return;
4607
+
}
4608
+
4609
+
push_data_sib(cp, wr_mas->mas, sib, parent);
4610
+
if (sib->end)
4611
+
cp->data += sib->end + 1;
4612
+
}
4613
+
4614
+
/*
4546
4615
* mas_wr_split() - Expand one node into two
4547
4616
* @wr_mas: The write maple state
4548
4617
*/
4549
-
static noinline_for_kasan void mas_wr_split(struct ma_wr_state *wr_mas)
4618
+
static void mas_wr_split(struct ma_wr_state *wr_mas)
4550
4619
{
4551
-
struct maple_big_node b_node;
4620
+
struct maple_enode *old_enode;
4621
+
struct ma_state parent;
4622
+
struct ma_state *mas;
4623
+
struct maple_copy cp;
4624
+
struct ma_state sib;
4552
4625
4626
+
mas = wr_mas->mas;
4553
4627
trace_ma_write(TP_FCT, wr_mas->mas, 0, wr_mas->entry);
4554
-
memset(&b_node, 0, sizeof(struct maple_big_node));
4555
-
mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end);
4556
-
WARN_ON_ONCE(wr_mas->mas->store_type != wr_split_store);
4557
-
return mas_split(wr_mas->mas, &b_node);
4628
+
parent = *mas;
4629
+
cp_leaf_init(&cp, mas, wr_mas, wr_mas);
4630
+
do {
4631
+
if (!mte_is_root(parent.node)) {
4632
+
mas_ascend(&parent);
4633
+
parent.end = mas_data_end(&parent);
4634
+
}
4635
+
split_data(&cp, wr_mas, &sib, &parent);
4636
+
multi_src_setup(&cp, wr_mas, wr_mas, &sib);
4637
+
dst_setup(&cp, mas, wr_mas->type);
4638
+
cp_data_write(&cp, mas);
4639
+
} while (split_ascend(&cp, wr_mas, &sib, &parent));
4640
+
4641
+
old_enode = mas->node;
4642
+
mas->node = mt_slot_locked(mas->tree, cp.slot, 0);
4643
+
mas_wmb_replace(mas, old_enode, cp.height);
4644
+
mtree_range_walk(mas);
4558
4645
}
4559
4646
4560
4647
/*
+45
-10
lib/test_maple_tree.c
+45
-10
lib/test_maple_tree.c
···
1024
1024
mt_set_non_kernel(10);
1025
1025
check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0);
1026
1026
MT_BUG_ON(mt, !mt_height(mt));
1027
+
mt_validate(mt);
1027
1028
mtree_destroy(mt);
1028
1029
1029
1030
/* Create tree of 1-200 */
···
1032
1031
/* Store 45-168 */
1033
1032
check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0);
1034
1033
MT_BUG_ON(mt, !mt_height(mt));
1034
+
mt_validate(mt);
1035
1035
mtree_destroy(mt);
1036
1036
1037
1037
check_seq(mt, 30, false);
1038
1038
check_store_range(mt, 6, 18, xa_mk_value(6), 0);
1039
1039
MT_BUG_ON(mt, !mt_height(mt));
1040
+
mt_validate(mt);
1040
1041
mtree_destroy(mt);
1041
1042
1042
1043
/* Overwrite across multiple levels. */
···
1064
1061
check_load(mt, r[13] + 1, xa_mk_value(r[13] + 1));
1065
1062
check_load(mt, 135, NULL);
1066
1063
check_load(mt, 140, NULL);
1064
+
mt_validate(mt);
1067
1065
mt_set_non_kernel(0);
1068
1066
MT_BUG_ON(mt, !mt_height(mt));
1069
1067
mtree_destroy(mt);
···
1289
1285
MT_BUG_ON(mt, mt_height(mt) >= 4);
1290
1286
}
1291
1287
/* Cause a 3 child split all the way up the tree. */
1292
-
for (i = 5; i < 215; i += 10)
1288
+
for (i = 5; i < 215; i += 10) {
1293
1289
check_store_range(mt, 11450 + i, 11450 + i + 1, NULL, 0);
1294
-
for (i = 5; i < 65; i += 10)
1290
+
mt_validate(mt);
1291
+
}
1292
+
for (i = 5; i < 65; i += 10) {
1295
1293
check_store_range(mt, 11770 + i, 11770 + i + 1, NULL, 0);
1294
+
mt_validate(mt);
1295
+
}
1296
1296
1297
1297
MT_BUG_ON(mt, mt_height(mt) >= 4);
1298
-
for (i = 5; i < 45; i += 10)
1298
+
for (i = 5; i < 45; i += 10) {
1299
1299
check_store_range(mt, 11700 + i, 11700 + i + 1, NULL, 0);
1300
+
mt_validate(mt);
1301
+
}
1300
1302
if (!MAPLE_32BIT)
1301
1303
MT_BUG_ON(mt, mt_height(mt) < 4);
1302
1304
mtree_destroy(mt);
···
1314
1304
val2 = (i+1)*10;
1315
1305
check_store_range(mt, val, val2, xa_mk_value(val), 0);
1316
1306
MT_BUG_ON(mt, mt_height(mt) >= 4);
1307
+
mt_validate(mt);
1317
1308
}
1318
1309
/* Fill parents and leaves before split. */
1319
-
for (i = 5; i < 455; i += 10)
1320
-
check_store_range(mt, 7800 + i, 7800 + i + 1, NULL, 0);
1310
+
val = 7660;
1311
+
for (i = 5; i < 490; i += 5) {
1312
+
val += 5;
1313
+
check_store_range(mt, val, val + 1, NULL, 0);
1314
+
mt_validate(mt);
1315
+
MT_BUG_ON(mt, mt_height(mt) >= 4);
1316
+
}
1321
1317
1322
-
for (i = 1; i < 16; i++)
1323
-
check_store_range(mt, 8185 + i, 8185 + i + 1,
1324
-
xa_mk_value(8185+i), 0);
1325
-
MT_BUG_ON(mt, mt_height(mt) >= 4);
1318
+
val = 9460;
1319
+
/* Fill parents and leaves before split. */
1320
+
for (i = 1; i < 10; i++) {
1321
+
val++;
1322
+
check_store_range(mt, val, val + 1, xa_mk_value(val), 0);
1323
+
mt_validate(mt);
1324
+
}
1325
+
1326
+
val = 8000;
1327
+
for (i = 1; i < 14; i++) {
1328
+
val++;
1329
+
check_store_range(mt, val, val + 1, xa_mk_value(val), 0);
1330
+
mt_validate(mt);
1331
+
}
1332
+
1333
+
1334
+
check_store_range(mt, 8051, 8051, xa_mk_value(8081), 0);
1335
+
check_store_range(mt, 8052, 8052, xa_mk_value(8082), 0);
1336
+
check_store_range(mt, 8083, 8083, xa_mk_value(8083), 0);
1337
+
check_store_range(mt, 8084, 8084, xa_mk_value(8084), 0);
1338
+
check_store_range(mt, 8085, 8085, xa_mk_value(8085), 0);
1326
1339
/* triple split across multiple levels. */
1327
-
check_store_range(mt, 8184, 8184, xa_mk_value(8184), 0);
1340
+
check_store_range(mt, 8099, 8100, xa_mk_value(1), 0);
1341
+
1342
+
mt_validate(mt);
1328
1343
if (!MAPLE_32BIT)
1329
1344
MT_BUG_ON(mt, mt_height(mt) != 4);
1330
1345
}
+11
tools/testing/radix-tree/maple.c
+11
tools/testing/radix-tree/maple.c
···
35406
35406
mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
35407
35407
for (i = 0; i <= max; i++)
35408
35408
mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
35409
+
35409
35410
mtree_lock(mt);
35411
+
if (MAPLE_32BIT) {
35412
+
i = 47811;
35413
+
do {
35414
+
mas_set(&mas, i);
35415
+
mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL);
35416
+
i++;
35417
+
mas_ascend(&mas);
35418
+
} while (mas_data_end(&mas) < mt_slot_count(mas.node) - 1);
35419
+
}
35420
+
35410
35421
mas_set(&mas, 47606);
35411
35422
mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL);
35412
35423
mas_set(&mas, 47607);