rbtree: Provide rbtree with links

Some RB tree users require quick access to the next and the previous node,
e.g. to check whether a modification of the node results in a change of the
nodes position in the tree. If the node position does not change, then the
modification can happen in place without going through a full enqueue
requeue cycle. A upcoming use case for this are the timer queues of the
hrtimer subsystem as they can optimize for timers which are frequently
rearmed while enqueued.

This can be obviously achieved with rb_next() and rb_prev(), but those
turned out to be quite expensive for hotpath operations depending on the
tree depth.

Add a linked RB tree variant where add() and erase() maintain the links
between the nodes. Like the cached variant it provides a pointer to the
left most node in the root.

It intentionally does not use a [h]list head as there is no real need for
true list operations as the list is strictly coupled to the tree and
and cannot be manipulated independently.

It sets the nodes previous pointer to NULL for the left most node and the
next pointer to NULL for the right most node. This allows a quick check
especially for the left most node without consulting the list head address,
which creates better code.

Aside of the rb_leftmost cached pointer this could trivially provide a
rb_rightmost pointer as well, but there is no usage for that (yet).

Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260224163431.668401024@kernel.org

authored by

Thomas Gleixner and committed by

Peter Zijlstra 3 months ago 67104794 3601a1d8

+105 -9

3 changed files

expand all

include

linux

rbtree.h

rbtree_types.h

lib

rbtree.c

+72 -9

include/linux/rbtree.h

··· 35 35 #define RB_CLEAR_NODE(node) \ 36 36 ((node)->__rb_parent_color = (unsigned long)(node)) 37 37 38 + #define RB_EMPTY_LINKED_NODE(lnode) RB_EMPTY_NODE(&(lnode)->node) 39 + #define RB_CLEAR_LINKED_NODE(lnode) ({ \ 40 + RB_CLEAR_NODE(&(lnode)->node); \ 41 + (lnode)->prev = (lnode)->next = NULL; \ 42 + }) 38 43 39 44 extern void rb_insert_color(struct rb_node *, struct rb_root *); 40 45 extern void rb_erase(struct rb_node *, struct rb_root *); 41 - 46 + extern bool rb_erase_linked(struct rb_node_linked *, struct rb_root_linked *); 42 47 43 48 /* Find logical next and previous nodes in a tree */ 44 49 extern struct rb_node *rb_next(const struct rb_node *); ··· 218 213 return leftmost ? node : NULL; 219 214 } 220 215 221 - /** 222 - * rb_add() - insert @node into @tree 223 - * @node: node to insert 224 - * @tree: tree to insert @node into 225 - * @less: operator defining the (partial) node order 226 - */ 227 216 static __always_inline void 228 - rb_add(struct rb_node *node, struct rb_root *tree, 229 - bool (*less)(struct rb_node *, const struct rb_node *)) 217 + __rb_add(struct rb_node *node, struct rb_root *tree, 218 + bool (*less)(struct rb_node *, const struct rb_node *), 219 + void (*linkop)(struct rb_node *, struct rb_node *, struct rb_node **)) 230 220 { 231 221 struct rb_node **link = &tree->rb_node; 232 222 struct rb_node *parent = NULL; ··· 234 234 link = &parent->rb_right; 235 235 } 236 236 237 + linkop(node, parent, link); 237 238 rb_link_node(node, parent, link); 238 239 rb_insert_color(node, tree); 240 + } 241 + 242 + #define __node_2_linked_node(_n) \ 243 + rb_entry((_n), struct rb_node_linked, node) 244 + 245 + static inline void 246 + rb_link_linked_node(struct rb_node *node, struct rb_node *parent, struct rb_node **link) 247 + { 248 + if (!parent) 249 + return; 250 + 251 + struct rb_node_linked *nnew = __node_2_linked_node(node); 252 + struct rb_node_linked *npar = __node_2_linked_node(parent); 253 + 254 + if (link == &parent->rb_left) { 255 + nnew->prev = npar->prev; 256 + nnew->next = npar; 257 + npar->prev = nnew; 258 + if (nnew->prev) 259 + nnew->prev->next = nnew; 260 + } else { 261 + nnew->next = npar->next; 262 + nnew->prev = npar; 263 + npar->next = nnew; 264 + if (nnew->next) 265 + nnew->next->prev = nnew; 266 + } 267 + } 268 + 269 + /** 270 + * rb_add_linked() - insert @node into the leftmost linked tree @tree 271 + * @node: node to insert 272 + * @tree: linked tree to insert @node into 273 + * @less: operator defining the (partial) node order 274 + * 275 + * Returns @true when @node is the new leftmost, @false otherwise. 276 + */ 277 + static __always_inline bool 278 + rb_add_linked(struct rb_node_linked *node, struct rb_root_linked *tree, 279 + bool (*less)(struct rb_node *, const struct rb_node *)) 280 + { 281 + __rb_add(&node->node, &tree->rb_root, less, rb_link_linked_node); 282 + if (!node->prev) 283 + tree->rb_leftmost = node; 284 + return !node->prev; 285 + } 286 + 287 + /* Empty linkop function which is optimized away by the compiler */ 288 + static __always_inline void 289 + rb_link_noop(struct rb_node *n, struct rb_node *p, struct rb_node **l) { } 290 + 291 + /** 292 + * rb_add() - insert @node into @tree 293 + * @node: node to insert 294 + * @tree: tree to insert @node into 295 + * @less: operator defining the (partial) node order 296 + */ 297 + static __always_inline void 298 + rb_add(struct rb_node *node, struct rb_root *tree, 299 + bool (*less)(struct rb_node *, const struct rb_node *)) 300 + { 301 + __rb_add(node, tree, less, rb_link_noop); 239 302 } 240 303 241 304 /**

+16

include/linux/rbtree_types.h

··· 9 9 } __attribute__((aligned(sizeof(long)))); 10 10 /* The alignment might seem pointless, but allegedly CRIS needs it */ 11 11 12 + struct rb_node_linked { 13 + struct rb_node node; 14 + struct rb_node_linked *prev; 15 + struct rb_node_linked *next; 16 + }; 17 + 12 18 struct rb_root { 13 19 struct rb_node *rb_node; 14 20 }; ··· 34 28 struct rb_node *rb_leftmost; 35 29 }; 36 30 31 + /* 32 + * Leftmost tree with links. This would allow a trivial rb_rightmost update, 33 + * but that has been omitted due to the lack of users. 34 + */ 35 + struct rb_root_linked { 36 + struct rb_root rb_root; 37 + struct rb_node_linked *rb_leftmost; 38 + }; 39 + 37 40 #define RB_ROOT (struct rb_root) { NULL, } 38 41 #define RB_ROOT_CACHED (struct rb_root_cached) { {NULL, }, NULL } 42 + #define RB_ROOT_LINKED (struct rb_root_linked) { {NULL, }, NULL } 39 43 40 44 #endif

+17

lib/rbtree.c

··· 446 446 } 447 447 EXPORT_SYMBOL(rb_erase); 448 448 449 + bool rb_erase_linked(struct rb_node_linked *node, struct rb_root_linked *root) 450 + { 451 + if (node->prev) 452 + node->prev->next = node->next; 453 + else 454 + root->rb_leftmost = node->next; 455 + 456 + if (node->next) 457 + node->next->prev = node->prev; 458 + 459 + rb_erase(&node->node, &root->rb_root); 460 + RB_CLEAR_LINKED_NODE(node); 461 + 462 + return !!root->rb_leftmost; 463 + } 464 + EXPORT_SYMBOL_GPL(rb_erase_linked); 465 + 449 466 /* 450 467 * Augmented rbtree manipulation functions. 451 468 *

Configure Feed

Configure Feed