Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
debugobjects: Rework object allocation
The current allocation scheme tries to allocate from the per CPU pool
first. If that fails it allocates one object from the global pool and then
refills the per CPU pool from the global pool.
That is in the way of switching the pool management to batch mode as the
global pool needs to be a strict stack of batches, which does not allow
to allocate single objects.
Rework the code to refill the per CPU pool first and then allocate the
object from the refilled batch. Also try to allocate from the to free pool
first to avoid freeing and reallocating objects.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Zhen Lei <thunder.leizhen@huawei.com>
Link: https://lore.kernel.org/all/20241007164913.893554162@linutronix.de
+69
-75
+69
-75
lib/debugobjects.c
+69
-75
lib/debugobjects.c
···
141
141
return pool_count(pool) < pool->min_cnt / 2;
142
142
}
143
143
144
+
static bool pool_move_batch(struct obj_pool *dst, struct obj_pool *src)
145
+
{
146
+
if (dst->cnt + ODEBUG_BATCH_SIZE > dst->max_cnt || !src->cnt)
147
+
return false;
148
+
149
+
for (int i = 0; i < ODEBUG_BATCH_SIZE && src->cnt; i++) {
150
+
struct hlist_node *node = src->objects.first;
151
+
152
+
WRITE_ONCE(src->cnt, src->cnt - 1);
153
+
WRITE_ONCE(dst->cnt, dst->cnt + 1);
154
+
155
+
hlist_del(node);
156
+
hlist_add_head(node, &dst->objects);
157
+
}
158
+
return true;
159
+
}
160
+
161
+
static struct debug_obj *__alloc_object(struct hlist_head *list)
162
+
{
163
+
struct debug_obj *obj;
164
+
165
+
if (unlikely(!list->first))
166
+
return NULL;
167
+
168
+
obj = hlist_entry(list->first, typeof(*obj), node);
169
+
hlist_del(&obj->node);
170
+
return obj;
171
+
}
172
+
173
+
static struct debug_obj *pcpu_alloc(void)
174
+
{
175
+
struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu);
176
+
177
+
lockdep_assert_irqs_disabled();
178
+
179
+
for (;;) {
180
+
struct debug_obj *obj = __alloc_object(&pcp->objects);
181
+
182
+
if (likely(obj)) {
183
+
pcp->cnt--;
184
+
return obj;
185
+
}
186
+
187
+
guard(raw_spinlock)(&pool_lock);
188
+
if (!pool_move_batch(pcp, &pool_to_free)) {
189
+
if (!pool_move_batch(pcp, &pool_global))
190
+
return NULL;
191
+
}
192
+
obj_pool_used += pcp->cnt;
193
+
194
+
if (obj_pool_used > obj_pool_max_used)
195
+
obj_pool_max_used = obj_pool_used;
196
+
197
+
if (pool_global.cnt < obj_pool_min_free)
198
+
obj_pool_min_free = pool_global.cnt;
199
+
}
200
+
}
201
+
144
202
static void free_object_list(struct hlist_head *head)
145
203
{
146
204
struct hlist_node *tmp;
···
216
158
static void fill_pool_from_freelist(void)
217
159
{
218
160
static unsigned long state;
219
-
struct debug_obj *obj;
220
161
221
162
/*
222
163
* Reuse objs from the global obj_to_free list; they will be
···
237
180
if (test_bit(0, &state) || test_and_set_bit(0, &state))
238
181
return;
239
182
240
-
guard(raw_spinlock)(&pool_lock);
241
-
/*
242
-
* Recheck with the lock held as the worker thread might have
243
-
* won the race and freed the global free list already.
244
-
*/
245
-
while (pool_to_free.cnt && (pool_global.cnt < pool_global.min_cnt)) {
246
-
obj = hlist_entry(pool_to_free.objects.first, typeof(*obj), node);
247
-
hlist_del(&obj->node);
248
-
WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt - 1);
249
-
hlist_add_head(&obj->node, &pool_global.objects);
250
-
WRITE_ONCE(pool_global.cnt, pool_global.cnt + 1);
183
+
/* Avoid taking the lock when there is no work to do */
184
+
while (pool_should_refill(&pool_global) && pool_count(&pool_to_free)) {
185
+
guard(raw_spinlock)(&pool_lock);
186
+
/* Move a batch if possible */
187
+
pool_move_batch(&pool_global, &pool_to_free);
251
188
}
252
189
clear_bit(0, &state);
253
190
}
···
302
251
return NULL;
303
252
}
304
253
305
-
/*
306
-
* Allocate a new object from the hlist
307
-
*/
308
-
static struct debug_obj *__alloc_object(struct hlist_head *list)
254
+
static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b,
255
+
const struct debug_obj_descr *descr)
309
256
{
310
-
struct debug_obj *obj = NULL;
311
-
312
-
if (list->first) {
313
-
obj = hlist_entry(list->first, typeof(*obj), node);
314
-
hlist_del(&obj->node);
315
-
}
316
-
317
-
return obj;
318
-
}
319
-
320
-
static struct debug_obj *
321
-
alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr)
322
-
{
323
-
struct obj_pool *percpu_pool = this_cpu_ptr(&pool_pcpu);
324
257
struct debug_obj *obj;
325
258
326
-
if (likely(obj_cache)) {
327
-
obj = __alloc_object(&percpu_pool->objects);
328
-
if (obj) {
329
-
percpu_pool->cnt--;
330
-
goto init_obj;
331
-
}
332
-
} else {
259
+
if (likely(obj_cache))
260
+
obj = pcpu_alloc();
261
+
else
333
262
obj = __alloc_object(&pool_boot);
334
-
goto init_obj;
335
-
}
336
263
337
-
raw_spin_lock(&pool_lock);
338
-
obj = __alloc_object(&pool_global.objects);
339
-
if (obj) {
340
-
obj_pool_used++;
341
-
WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
342
-
343
-
/*
344
-
* Looking ahead, allocate one batch of debug objects and
345
-
* put them into the percpu free pool.
346
-
*/
347
-
if (likely(obj_cache)) {
348
-
int i;
349
-
350
-
for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
351
-
struct debug_obj *obj2;
352
-
353
-
obj2 = __alloc_object(&pool_global.objects);
354
-
if (!obj2)
355
-
break;
356
-
hlist_add_head(&obj2->node, &percpu_pool->objects);
357
-
percpu_pool->cnt++;
358
-
obj_pool_used++;
359
-
WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
360
-
}
361
-
}
362
-
363
-
if (obj_pool_used > obj_pool_max_used)
364
-
obj_pool_max_used = obj_pool_used;
365
-
366
-
if (pool_global.cnt < obj_pool_min_free)
367
-
obj_pool_min_free = pool_global.cnt;
368
-
}
369
-
raw_spin_unlock(&pool_lock);
370
-
371
-
init_obj:
372
-
if (obj) {
264
+
if (likely(obj)) {
373
265
obj->object = addr;
374
266
obj->descr = descr;
375
267
obj->state = ODEBUG_STATE_NONE;