tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

maple_tree: Add single node allocation support to maple state

The fast path through a write will require replacing a single node in
the tree. Using a sheaf (32 nodes) is too heavy for the fast path, so
special case the node store operation by just allocating one node in the
maple state.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

authored by

Liam R. Howlett and committed by
Vlastimil Babka 6bf377b0 9b05890a

+52 -10
+3 -1
include/linux/maple_tree.h
··· 443 443 unsigned long min; /* The minimum index of this node - implied pivot min */ 444 444 unsigned long max; /* The maximum index of this node - implied pivot max */ 445 445 struct slab_sheaf *sheaf; /* Allocated nodes for this operation */ 446 + struct maple_node *alloc; /* A single allocated node for fast path writes */ 446 447 unsigned long node_request; /* The number of nodes to allocate for this operation */ 447 448 enum maple_status status; /* The status of the state (active, start, none, etc) */ 448 449 unsigned char depth; /* depth of tree descent during write */ ··· 492 491 .status = ma_start, \ 493 492 .min = 0, \ 494 493 .max = ULONG_MAX, \ 495 - .node_request = 0, \ 496 494 .sheaf = NULL, \ 495 + .alloc = NULL, \ 496 + .node_request = 0, \ 497 497 .mas_flags = 0, \ 498 498 .store_type = wr_invalid, \ 499 499 }
+41 -6
lib/maple_tree.c
··· 1073 1073 * 1074 1074 * Return: A pointer to a maple node. 1075 1075 */ 1076 - static inline struct maple_node *mas_pop_node(struct ma_state *mas) 1076 + static __always_inline struct maple_node *mas_pop_node(struct ma_state *mas) 1077 1077 { 1078 1078 struct maple_node *ret; 1079 + 1080 + if (mas->alloc) { 1081 + ret = mas->alloc; 1082 + mas->alloc = NULL; 1083 + goto out; 1084 + } 1079 1085 1080 1086 if (WARN_ON_ONCE(!mas->sheaf)) 1081 1087 return NULL; 1082 1088 1083 1089 ret = kmem_cache_alloc_from_sheaf(maple_node_cache, GFP_NOWAIT, mas->sheaf); 1084 - memset(ret, 0, sizeof(*ret)); 1085 1090 1091 + out: 1092 + memset(ret, 0, sizeof(*ret)); 1086 1093 return ret; 1087 1094 } 1088 1095 ··· 1100 1093 */ 1101 1094 static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp) 1102 1095 { 1103 - if (unlikely(mas->sheaf)) { 1104 - unsigned long refill = mas->node_request; 1096 + if (!mas->node_request) 1097 + return; 1105 1098 1099 + if (mas->node_request == 1) { 1100 + if (mas->sheaf) 1101 + goto use_sheaf; 1102 + 1103 + if (mas->alloc) 1104 + return; 1105 + 1106 + mas->alloc = mt_alloc_one(gfp); 1107 + if (!mas->alloc) 1108 + goto error; 1109 + 1110 + mas->node_request = 0; 1111 + return; 1112 + } 1113 + 1114 + use_sheaf: 1115 + if (unlikely(mas->alloc)) { 1116 + kfree(mas->alloc); 1117 + mas->alloc = NULL; 1118 + } 1119 + 1120 + if (mas->sheaf) { 1121 + unsigned long refill; 1122 + 1123 + refill = mas->node_request; 1106 1124 if (kmem_cache_sheaf_size(mas->sheaf) >= refill) { 1107 1125 mas->node_request = 0; 1108 1126 return; ··· 5212 5180 mas->node_request = 0; 5213 5181 if (mas->sheaf) 5214 5182 mt_return_sheaf(mas->sheaf); 5215 - 5216 5183 mas->sheaf = NULL; 5184 + 5185 + if (mas->alloc) 5186 + kfree(mas->alloc); 5187 + mas->alloc = NULL; 5217 5188 } 5218 5189 EXPORT_SYMBOL_GPL(mas_destroy); 5219 5190 ··· 5851 5816 mas_alloc_nodes(mas, gfp); 5852 5817 } 5853 5818 5854 - if (!mas->sheaf) 5819 + if (!mas->sheaf && !mas->alloc) 5855 5820 return false; 5856 5821 5857 5822 mas->status = ma_start;
+8 -3
tools/testing/radix-tree/maple.c
··· 35085 35085 35086 35086 static int mas_allocated(struct ma_state *mas) 35087 35087 { 35088 - if (mas->sheaf) 35089 - return kmem_cache_sheaf_size(mas->sheaf); 35088 + int total = 0; 35090 35089 35091 - return 0; 35090 + if (mas->alloc) 35091 + total++; 35092 + 35093 + if (mas->sheaf) 35094 + total += kmem_cache_sheaf_size(mas->sheaf); 35095 + 35096 + return total; 35092 35097 } 35093 35098 /* Preallocation testing */ 35094 35099 static noinline void __init check_prealloc(struct maple_tree *mt)