Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/generic-radix-tree.c: genradix_ptr_inlined()
Provide an inlined fast path
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
+76
-63
+75
include/linux/generic-radix-tree.h
+75
include/linux/generic-radix-tree.h
···
48
48
#define GENRADIX_NODE_SHIFT 9
49
49
#define GENRADIX_NODE_SIZE (1U << GENRADIX_NODE_SHIFT)
50
50
51
+
#define GENRADIX_ARY (GENRADIX_NODE_SIZE / sizeof(struct genradix_node *))
52
+
#define GENRADIX_ARY_SHIFT ilog2(GENRADIX_ARY)
53
+
54
+
/* depth that's needed for a genradix that can address up to ULONG_MAX: */
55
+
#define GENRADIX_MAX_DEPTH \
56
+
DIV_ROUND_UP(BITS_PER_LONG - GENRADIX_NODE_SHIFT, GENRADIX_ARY_SHIFT)
57
+
58
+
#define GENRADIX_DEPTH_MASK \
59
+
((unsigned long) (roundup_pow_of_two(GENRADIX_MAX_DEPTH + 1) - 1))
60
+
61
+
static inline int genradix_depth_shift(unsigned depth)
62
+
{
63
+
return GENRADIX_NODE_SHIFT + GENRADIX_ARY_SHIFT * depth;
64
+
}
65
+
66
+
/*
67
+
* Returns size (of data, in bytes) that a tree of a given depth holds:
68
+
*/
69
+
static inline size_t genradix_depth_size(unsigned depth)
70
+
{
71
+
return 1UL << genradix_depth_shift(depth);
72
+
}
73
+
74
+
static inline unsigned genradix_root_to_depth(struct genradix_root *r)
75
+
{
76
+
return (unsigned long) r & GENRADIX_DEPTH_MASK;
77
+
}
78
+
79
+
static inline struct genradix_node *genradix_root_to_node(struct genradix_root *r)
80
+
{
81
+
return (void *) ((unsigned long) r & ~GENRADIX_DEPTH_MASK);
82
+
}
83
+
84
+
struct genradix_node {
85
+
union {
86
+
/* Interior node: */
87
+
struct genradix_node *children[GENRADIX_ARY];
88
+
89
+
/* Leaf: */
90
+
u8 data[GENRADIX_NODE_SIZE];
91
+
};
92
+
};
93
+
51
94
struct __genradix {
52
95
struct genradix_root *root;
53
96
};
···
171
128
#define __genradix_idx_to_offset(_radix, _idx) \
172
129
__idx_to_offset(_idx, __genradix_obj_size(_radix))
173
130
131
+
static inline void *__genradix_ptr_inlined(struct __genradix *radix, size_t offset)
132
+
{
133
+
struct genradix_root *r = READ_ONCE(radix->root);
134
+
struct genradix_node *n = genradix_root_to_node(r);
135
+
unsigned level = genradix_root_to_depth(r);
136
+
unsigned shift = genradix_depth_shift(level);
137
+
138
+
if (unlikely(ilog2(offset) >= genradix_depth_shift(level)))
139
+
return NULL;
140
+
141
+
while (n && shift > GENRADIX_NODE_SHIFT) {
142
+
shift -= GENRADIX_ARY_SHIFT;
143
+
n = n->children[offset >> shift];
144
+
offset &= (1UL << shift) - 1;
145
+
}
146
+
147
+
return n ? &n->data[offset] : NULL;
148
+
}
149
+
150
+
#define genradix_ptr_inlined(_radix, _idx) \
151
+
(__genradix_cast(_radix) \
152
+
__genradix_ptr_inlined(&(_radix)->tree, \
153
+
__genradix_idx_to_offset(_radix, _idx)))
154
+
174
155
void *__genradix_ptr(struct __genradix *, size_t);
175
156
176
157
/**
···
210
143
__genradix_idx_to_offset(_radix, _idx)))
211
144
212
145
void *__genradix_ptr_alloc(struct __genradix *, size_t, gfp_t);
146
+
147
+
#define genradix_ptr_alloc_inlined(_radix, _idx, _gfp) \
148
+
(__genradix_cast(_radix) \
149
+
(__genradix_ptr_inlined(&(_radix)->tree, \
150
+
__genradix_idx_to_offset(_radix, _idx)) ?: \
151
+
__genradix_ptr_alloc(&(_radix)->tree, \
152
+
__genradix_idx_to_offset(_radix, _idx), \
153
+
_gfp)))
213
154
214
155
/**
215
156
* genradix_ptr_alloc - get a pointer to a genradix entry, allocating it
+1
-63
lib/generic-radix-tree.c
+1
-63
lib/generic-radix-tree.c
···
5
5
#include <linux/gfp.h>
6
6
#include <linux/kmemleak.h>
7
7
8
-
#define GENRADIX_ARY (GENRADIX_NODE_SIZE / sizeof(struct genradix_node *))
9
-
#define GENRADIX_ARY_SHIFT ilog2(GENRADIX_ARY)
10
-
11
-
struct genradix_node {
12
-
union {
13
-
/* Interior node: */
14
-
struct genradix_node *children[GENRADIX_ARY];
15
-
16
-
/* Leaf: */
17
-
u8 data[GENRADIX_NODE_SIZE];
18
-
};
19
-
};
20
-
21
-
static inline int genradix_depth_shift(unsigned depth)
22
-
{
23
-
return GENRADIX_NODE_SHIFT + GENRADIX_ARY_SHIFT * depth;
24
-
}
25
-
26
-
/*
27
-
* Returns size (of data, in bytes) that a tree of a given depth holds:
28
-
*/
29
-
static inline size_t genradix_depth_size(unsigned depth)
30
-
{
31
-
return 1UL << genradix_depth_shift(depth);
32
-
}
33
-
34
-
/* depth that's needed for a genradix that can address up to ULONG_MAX: */
35
-
#define GENRADIX_MAX_DEPTH \
36
-
DIV_ROUND_UP(BITS_PER_LONG - GENRADIX_NODE_SHIFT, GENRADIX_ARY_SHIFT)
37
-
38
-
#define GENRADIX_DEPTH_MASK \
39
-
((unsigned long) (roundup_pow_of_two(GENRADIX_MAX_DEPTH + 1) - 1))
40
-
41
-
static inline unsigned genradix_root_to_depth(struct genradix_root *r)
42
-
{
43
-
return (unsigned long) r & GENRADIX_DEPTH_MASK;
44
-
}
45
-
46
-
static inline struct genradix_node *genradix_root_to_node(struct genradix_root *r)
47
-
{
48
-
return (void *) ((unsigned long) r & ~GENRADIX_DEPTH_MASK);
49
-
}
50
-
51
8
/*
52
9
* Returns pointer to the specified byte @offset within @radix, or NULL if not
53
10
* allocated
54
11
*/
55
12
void *__genradix_ptr(struct __genradix *radix, size_t offset)
56
13
{
57
-
struct genradix_root *r = READ_ONCE(radix->root);
58
-
struct genradix_node *n = genradix_root_to_node(r);
59
-
unsigned level = genradix_root_to_depth(r);
60
-
61
-
if (ilog2(offset) >= genradix_depth_shift(level))
62
-
return NULL;
63
-
64
-
while (1) {
65
-
if (!n)
66
-
return NULL;
67
-
if (!level)
68
-
break;
69
-
70
-
level--;
71
-
72
-
n = n->children[offset >> genradix_depth_shift(level)];
73
-
offset &= genradix_depth_size(level) - 1;
74
-
}
75
-
76
-
return &n->data[offset];
14
+
return __genradix_ptr_inlined(radix, offset);
77
15
}
78
16
EXPORT_SYMBOL(__genradix_ptr);
79
17