Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
binder: use bitmap for faster descriptor lookup
When creating new binder references, the driver assigns a descriptor id
that is shared with userspace. Regrettably, the driver needs to keep the
descriptors small enough to accommodate userspace potentially using them
as Vector indexes. Currently, the driver performs a linear search on the
rb-tree of references to find the smallest available descriptor id. This
approach, however, scales poorly as the number of references grows.
This patch introduces the usage of bitmaps to boost the performance of
descriptor assignments. This optimization results in notable performance
gains, particularly in processes with a large number of references. The
following benchmark with 100,000 references showcases the difference in
latency between the dbitmap implementation and the legacy approach:
[ 587.145098] get_ref_desc_olocked: 15us (dbitmap on)
[ 602.788623] get_ref_desc_olocked: 47343us (dbitmap off)
Note the bitmap size is dynamically adjusted in line with the number of
references, ensuring efficient memory usage. In cases where growing the
bitmap is not possible, the driver falls back to the slow legacy method.
A previous attempt to solve this issue was proposed in [1]. However,
such method involved adding new ioctls which isn't great, plus older
userspace code would not have benefited from the optimizations either.
Link: https://lore.kernel.org/all/20240417191418.1341988-1-cmllamas@google.com/ [1]
Cc: Tim Murray <timmurray@google.com>
Cc: Arve Hjønnevåg <arve@android.com>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Martijn Coenen <maco@android.com>
Cc: Todd Kjos <tkjos@android.com>
Cc: John Stultz <jstultz@google.com>
Cc: Steven Moreland <smoreland@google.com>
Suggested-by: Nick Chen <chenjia3@oppo.com>
Reviewed-by: Alice Ryhl <aliceryhl@google.com>
Signed-off-by: Carlos Llamas <cmllamas@google.com>
Link: https://lore.kernel.org/r/20240612042535.1556708-1-cmllamas@google.com
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
authored by
Carlos Llamas
and committed by
Greg Kroah-Hartman
15d9da3f
7269d767
+279
-14
+99
-13
drivers/android/binder.c
+99
-13
drivers/android/binder.c
···
1045
1045
return NULL;
1046
1046
}
1047
1047
1048
+
/* Find the smallest unused descriptor the "slow way" */
1049
+
static u32 slow_desc_lookup_olocked(struct binder_proc *proc)
1050
+
{
1051
+
struct binder_ref *ref;
1052
+
struct rb_node *n;
1053
+
u32 desc;
1054
+
1055
+
desc = 1;
1056
+
for (n = rb_first(&proc->refs_by_desc); n; n = rb_next(n)) {
1057
+
ref = rb_entry(n, struct binder_ref, rb_node_desc);
1058
+
if (ref->data.desc > desc)
1059
+
break;
1060
+
desc = ref->data.desc + 1;
1061
+
}
1062
+
1063
+
return desc;
1064
+
}
1065
+
1066
+
/*
1067
+
* Find an available reference descriptor ID. The proc->outer_lock might
1068
+
* be released in the process, in which case -EAGAIN is returned and the
1069
+
* @desc should be considered invalid.
1070
+
*/
1071
+
static int get_ref_desc_olocked(struct binder_proc *proc,
1072
+
struct binder_node *node,
1073
+
u32 *desc)
1074
+
{
1075
+
struct dbitmap *dmap = &proc->dmap;
1076
+
unsigned long *new, bit;
1077
+
unsigned int nbits;
1078
+
1079
+
/* 0 is reserved for the context manager */
1080
+
if (node == proc->context->binder_context_mgr_node) {
1081
+
*desc = 0;
1082
+
return 0;
1083
+
}
1084
+
1085
+
if (!dbitmap_enabled(dmap)) {
1086
+
*desc = slow_desc_lookup_olocked(proc);
1087
+
return 0;
1088
+
}
1089
+
1090
+
if (dbitmap_acquire_first_zero_bit(dmap, &bit) == 0) {
1091
+
*desc = bit;
1092
+
return 0;
1093
+
}
1094
+
1095
+
/*
1096
+
* The dbitmap is full and needs to grow. The proc->outer_lock
1097
+
* is briefly released to allocate the new bitmap safely.
1098
+
*/
1099
+
nbits = dbitmap_grow_nbits(dmap);
1100
+
binder_proc_unlock(proc);
1101
+
new = bitmap_zalloc(nbits, GFP_KERNEL);
1102
+
binder_proc_lock(proc);
1103
+
dbitmap_grow(dmap, new, nbits);
1104
+
1105
+
return -EAGAIN;
1106
+
}
1107
+
1048
1108
/**
1049
1109
* binder_get_ref_for_node_olocked() - get the ref associated with given node
1050
1110
* @proc: binder_proc that owns the ref
···
1128
1068
struct binder_node *node,
1129
1069
struct binder_ref *new_ref)
1130
1070
{
1131
-
struct binder_context *context = proc->context;
1132
-
struct rb_node **p = &proc->refs_by_node.rb_node;
1133
-
struct rb_node *parent = NULL;
1134
1071
struct binder_ref *ref;
1135
-
struct rb_node *n;
1072
+
struct rb_node *parent;
1073
+
struct rb_node **p;
1074
+
u32 desc;
1136
1075
1076
+
retry:
1077
+
p = &proc->refs_by_node.rb_node;
1078
+
parent = NULL;
1137
1079
while (*p) {
1138
1080
parent = *p;
1139
1081
ref = rb_entry(parent, struct binder_ref, rb_node_node);
···
1150
1088
if (!new_ref)
1151
1089
return NULL;
1152
1090
1091
+
/* might release the proc->outer_lock */
1092
+
if (get_ref_desc_olocked(proc, node, &desc) == -EAGAIN)
1093
+
goto retry;
1094
+
1153
1095
binder_stats_created(BINDER_STAT_REF);
1154
1096
new_ref->data.debug_id = atomic_inc_return(&binder_last_id);
1155
1097
new_ref->proc = proc;
···
1161
1095
rb_link_node(&new_ref->rb_node_node, parent, p);
1162
1096
rb_insert_color(&new_ref->rb_node_node, &proc->refs_by_node);
1163
1097
1164
-
new_ref->data.desc = (node == context->binder_context_mgr_node) ? 0 : 1;
1165
-
for (n = rb_first(&proc->refs_by_desc); n != NULL; n = rb_next(n)) {
1166
-
ref = rb_entry(n, struct binder_ref, rb_node_desc);
1167
-
if (ref->data.desc > new_ref->data.desc)
1168
-
break;
1169
-
new_ref->data.desc = ref->data.desc + 1;
1170
-
}
1171
-
1098
+
new_ref->data.desc = desc;
1172
1099
p = &proc->refs_by_desc.rb_node;
1173
1100
while (*p) {
1174
1101
parent = *p;
···
1190
1131
1191
1132
static void binder_cleanup_ref_olocked(struct binder_ref *ref)
1192
1133
{
1134
+
struct dbitmap *dmap = &ref->proc->dmap;
1193
1135
bool delete_node = false;
1194
1136
1195
1137
binder_debug(BINDER_DEBUG_INTERNAL_REFS,
···
1198
1138
ref->proc->pid, ref->data.debug_id, ref->data.desc,
1199
1139
ref->node->debug_id);
1200
1140
1141
+
if (dbitmap_enabled(dmap))
1142
+
dbitmap_clear_bit(dmap, ref->data.desc);
1201
1143
rb_erase(&ref->rb_node_desc, &ref->proc->refs_by_desc);
1202
1144
rb_erase(&ref->rb_node_node, &ref->proc->refs_by_node);
1203
1145
···
1360
1298
kfree(ref);
1361
1299
}
1362
1300
1301
+
/* shrink descriptor bitmap if needed */
1302
+
static void try_shrink_dmap(struct binder_proc *proc)
1303
+
{
1304
+
unsigned long *new;
1305
+
int nbits;
1306
+
1307
+
binder_proc_lock(proc);
1308
+
nbits = dbitmap_shrink_nbits(&proc->dmap);
1309
+
binder_proc_unlock(proc);
1310
+
1311
+
if (!nbits)
1312
+
return;
1313
+
1314
+
new = bitmap_zalloc(nbits, GFP_KERNEL);
1315
+
binder_proc_lock(proc);
1316
+
dbitmap_shrink(&proc->dmap, new, nbits);
1317
+
binder_proc_unlock(proc);
1318
+
}
1319
+
1363
1320
/**
1364
1321
* binder_update_ref_for_handle() - inc/dec the ref for given handle
1365
1322
* @proc: proc containing the ref
···
1415
1334
*rdata = ref->data;
1416
1335
binder_proc_unlock(proc);
1417
1336
1418
-
if (delete_ref)
1337
+
if (delete_ref) {
1419
1338
binder_free_ref(ref);
1339
+
try_shrink_dmap(proc);
1340
+
}
1420
1341
return ret;
1421
1342
1422
1343
err_no_ref:
···
5014
4931
put_task_struct(proc->tsk);
5015
4932
put_cred(proc->cred);
5016
4933
binder_stats_deleted(BINDER_STAT_PROC);
4934
+
dbitmap_free(&proc->dmap);
5017
4935
kfree(proc);
5018
4936
}
5019
4937
···
5718
5634
proc = kzalloc(sizeof(*proc), GFP_KERNEL);
5719
5635
if (proc == NULL)
5720
5636
return -ENOMEM;
5637
+
5638
+
dbitmap_init(&proc->dmap);
5721
5639
spin_lock_init(&proc->inner_lock);
5722
5640
spin_lock_init(&proc->outer_lock);
5723
5641
get_task_struct(current->group_leader);
+4
-1
drivers/android/binder_internal.h
+4
-1
drivers/android/binder_internal.h
···
14
14
#include <linux/uidgid.h>
15
15
#include <uapi/linux/android/binderfs.h>
16
16
#include "binder_alloc.h"
17
+
#include "dbitmap.h"
17
18
18
19
struct binder_context {
19
20
struct binder_node *binder_context_mgr_node;
···
369
368
* @freeze_wait: waitqueue of processes waiting for all outstanding
370
369
* transactions to be processed
371
370
* (protected by @inner_lock)
371
+
* @dmap dbitmap to manage available reference descriptors
372
+
* (protected by @outer_lock)
372
373
* @todo: list of work for this process
373
374
* (protected by @inner_lock)
374
375
* @stats: per-process binder statistics
···
420
417
bool sync_recv;
421
418
bool async_recv;
422
419
wait_queue_head_t freeze_wait;
423
-
420
+
struct dbitmap dmap;
424
421
struct list_head todo;
425
422
struct binder_stats stats;
426
423
struct list_head delivered_death;
+176
drivers/android/dbitmap.h
+176
drivers/android/dbitmap.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* Copyright 2024 Google LLC
4
+
*
5
+
* dbitmap - dynamically sized bitmap library.
6
+
*
7
+
* Used by the binder driver to optimize the allocation of the smallest
8
+
* available descriptor ID. Each bit in the bitmap represents the state
9
+
* of an ID, with the exception of BIT(0) which is used exclusively to
10
+
* reference binder's context manager.
11
+
*
12
+
* A dbitmap can grow or shrink as needed. This part has been designed
13
+
* considering that users might need to briefly release their locks in
14
+
* order to allocate memory for the new bitmap. These operations then,
15
+
* are verified to determine if the grow or shrink is sill valid.
16
+
*
17
+
* This library does not provide protection against concurrent access
18
+
* by itself. Binder uses the proc->outer_lock for this purpose.
19
+
*/
20
+
21
+
#ifndef _LINUX_DBITMAP_H
22
+
#define _LINUX_DBITMAP_H
23
+
#include <linux/bitmap.h>
24
+
25
+
#define NBITS_MIN BITS_PER_TYPE(unsigned long)
26
+
27
+
struct dbitmap {
28
+
unsigned int nbits;
29
+
unsigned long *map;
30
+
};
31
+
32
+
static inline int dbitmap_enabled(struct dbitmap *dmap)
33
+
{
34
+
return !!dmap->nbits;
35
+
}
36
+
37
+
static inline void dbitmap_free(struct dbitmap *dmap)
38
+
{
39
+
dmap->nbits = 0;
40
+
kfree(dmap->map);
41
+
}
42
+
43
+
/* Returns the nbits that a dbitmap can shrink to, 0 if not possible. */
44
+
static inline unsigned int dbitmap_shrink_nbits(struct dbitmap *dmap)
45
+
{
46
+
unsigned int bit;
47
+
48
+
if (dmap->nbits <= NBITS_MIN)
49
+
return 0;
50
+
51
+
/*
52
+
* Determine if the bitmap can shrink based on the position of
53
+
* its last set bit. If the bit is within the first quarter of
54
+
* the bitmap then shrinking is possible. In this case, the
55
+
* bitmap should shrink to half its current size.
56
+
*/
57
+
bit = find_last_bit(dmap->map, dmap->nbits);
58
+
if (bit < (dmap->nbits >> 2))
59
+
return dmap->nbits >> 1;
60
+
61
+
/*
62
+
* Note that find_last_bit() returns dmap->nbits when no bits
63
+
* are set. While this is technically not possible here since
64
+
* BIT(0) is always set, this check is left for extra safety.
65
+
*/
66
+
if (bit == dmap->nbits)
67
+
return NBITS_MIN;
68
+
69
+
return 0;
70
+
}
71
+
72
+
/* Replace the internal bitmap with a new one of different size */
73
+
static inline void
74
+
dbitmap_replace(struct dbitmap *dmap, unsigned long *new, unsigned int nbits)
75
+
{
76
+
bitmap_copy(new, dmap->map, min(dmap->nbits, nbits));
77
+
kfree(dmap->map);
78
+
dmap->map = new;
79
+
dmap->nbits = nbits;
80
+
}
81
+
82
+
static inline void
83
+
dbitmap_shrink(struct dbitmap *dmap, unsigned long *new, unsigned int nbits)
84
+
{
85
+
if (!new)
86
+
return;
87
+
88
+
/*
89
+
* Verify that shrinking to @nbits is still possible. The @new
90
+
* bitmap might have been allocated without locks, so this call
91
+
* could now be outdated. In this case, free @new and move on.
92
+
*/
93
+
if (!dbitmap_enabled(dmap) || dbitmap_shrink_nbits(dmap) != nbits) {
94
+
kfree(new);
95
+
return;
96
+
}
97
+
98
+
dbitmap_replace(dmap, new, nbits);
99
+
}
100
+
101
+
/* Returns the nbits that a dbitmap can grow to. */
102
+
static inline unsigned int dbitmap_grow_nbits(struct dbitmap *dmap)
103
+
{
104
+
return dmap->nbits << 1;
105
+
}
106
+
107
+
static inline void
108
+
dbitmap_grow(struct dbitmap *dmap, unsigned long *new, unsigned int nbits)
109
+
{
110
+
/*
111
+
* Verify that growing to @nbits is still possible. The @new
112
+
* bitmap might have been allocated without locks, so this call
113
+
* could now be outdated. In this case, free @new and move on.
114
+
*/
115
+
if (!dbitmap_enabled(dmap) || nbits <= dmap->nbits) {
116
+
kfree(new);
117
+
return;
118
+
}
119
+
120
+
/*
121
+
* Check for ENOMEM after confirming the grow operation is still
122
+
* required. This ensures we only disable the dbitmap when it's
123
+
* necessary. Once the dbitmap is disabled, binder will fallback
124
+
* to slow_desc_lookup_olocked().
125
+
*/
126
+
if (!new) {
127
+
dbitmap_free(dmap);
128
+
return;
129
+
}
130
+
131
+
dbitmap_replace(dmap, new, nbits);
132
+
}
133
+
134
+
/*
135
+
* Finds and sets the first zero bit in the bitmap. Upon success @bit
136
+
* is populated with the index and 0 is returned. Otherwise, -ENOSPC
137
+
* is returned to indicate that a dbitmap_grow() is needed.
138
+
*/
139
+
static inline int
140
+
dbitmap_acquire_first_zero_bit(struct dbitmap *dmap, unsigned long *bit)
141
+
{
142
+
unsigned long n;
143
+
144
+
n = find_first_zero_bit(dmap->map, dmap->nbits);
145
+
if (n == dmap->nbits)
146
+
return -ENOSPC;
147
+
148
+
*bit = n;
149
+
set_bit(n, dmap->map);
150
+
151
+
return 0;
152
+
}
153
+
154
+
static inline void
155
+
dbitmap_clear_bit(struct dbitmap *dmap, unsigned long bit)
156
+
{
157
+
/* BIT(0) should always set for the context manager */
158
+
if (bit)
159
+
clear_bit(bit, dmap->map);
160
+
}
161
+
162
+
static inline int dbitmap_init(struct dbitmap *dmap)
163
+
{
164
+
dmap->map = bitmap_zalloc(NBITS_MIN, GFP_KERNEL);
165
+
if (!dmap->map) {
166
+
dmap->nbits = 0;
167
+
return -ENOMEM;
168
+
}
169
+
170
+
dmap->nbits = NBITS_MIN;
171
+
/* BIT(0) is reserved for the context manager */
172
+
set_bit(0, dmap->map);
173
+
174
+
return 0;
175
+
}
176
+
#endif