Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
debugobjects: Prepare kmem_cache allocations for batching
Allocate a batch and then push it into the pool. Utilize the
debug_obj::last_node pointer for keeping track of the batch boundary.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/all/20241007164914.198647184@linutronix.de
+49
-31
+49
-31
lib/debugobjects.c
+49
-31
lib/debugobjects.c
···
164
164
return true;
165
165
}
166
166
167
+
static bool pool_push_batch(struct obj_pool *dst, struct hlist_head *head)
168
+
{
169
+
struct hlist_node *last;
170
+
struct debug_obj *obj;
171
+
172
+
if (dst->cnt >= dst->max_cnt)
173
+
return false;
174
+
175
+
obj = hlist_entry(head->first, typeof(*obj), node);
176
+
last = obj->batch_last;
177
+
178
+
hlist_splice_init(head, last, &dst->objects);
179
+
WRITE_ONCE(dst->cnt, dst->cnt + ODEBUG_BATCH_SIZE);
180
+
return true;
181
+
}
182
+
167
183
static bool pool_pop_batch(struct hlist_head *head, struct obj_pool *src)
168
184
{
169
185
if (!src->cnt)
···
304
288
clear_bit(0, &state);
305
289
}
306
290
291
+
static bool kmem_alloc_batch(struct hlist_head *head, struct kmem_cache *cache, gfp_t gfp)
292
+
{
293
+
struct hlist_node *last = NULL;
294
+
struct debug_obj *obj;
295
+
296
+
for (int cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
297
+
obj = kmem_cache_zalloc(cache, gfp);
298
+
if (!obj) {
299
+
free_object_list(head);
300
+
return false;
301
+
}
302
+
debug_objects_allocated++;
303
+
304
+
if (!last)
305
+
last = &obj->node;
306
+
obj->batch_last = last;
307
+
308
+
hlist_add_head(&obj->node, head);
309
+
}
310
+
return true;
311
+
}
312
+
307
313
static void fill_pool(void)
308
314
{
309
315
static atomic_t cpus_allocating;
···
340
302
341
303
atomic_inc(&cpus_allocating);
342
304
while (pool_should_refill(&pool_global)) {
343
-
struct debug_obj *new, *last = NULL;
344
305
HLIST_HEAD(head);
345
-
int cnt;
346
306
347
-
for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
348
-
new = kmem_cache_zalloc(obj_cache, __GFP_HIGH | __GFP_NOWARN);
349
-
if (!new)
350
-
break;
351
-
hlist_add_head(&new->node, &head);
352
-
if (!last)
353
-
last = new;
354
-
}
355
-
if (!cnt)
307
+
if (!kmem_alloc_batch(&head, obj_cache, __GFP_HIGH | __GFP_NOWARN))
356
308
break;
357
309
358
310
guard(raw_spinlock_irqsave)(&pool_lock);
359
-
hlist_splice_init(&head, &last->node, &pool_global.objects);
360
-
debug_objects_allocated += cnt;
361
-
WRITE_ONCE(pool_global.cnt, pool_global.cnt + cnt);
311
+
if (!pool_push_batch(&pool_global, &head))
312
+
pool_push_batch(&pool_to_free, &head);
362
313
}
363
314
atomic_dec(&cpus_allocating);
364
315
}
···
1329
1302
static bool __init debug_objects_replace_static_objects(struct kmem_cache *cache)
1330
1303
{
1331
1304
struct debug_bucket *db = obj_hash;
1332
-
struct debug_obj *obj, *new;
1333
1305
struct hlist_node *tmp;
1306
+
struct debug_obj *obj;
1334
1307
HLIST_HEAD(objects);
1335
1308
int i;
1336
1309
1337
-
for (i = 0; i < ODEBUG_POOL_SIZE; i++) {
1338
-
obj = kmem_cache_zalloc(cache, GFP_KERNEL);
1339
-
if (!obj)
1310
+
for (i = 0; i < ODEBUG_POOL_SIZE; i += ODEBUG_BATCH_SIZE) {
1311
+
if (!kmem_alloc_batch(&objects, cache, GFP_KERNEL))
1340
1312
goto free;
1341
-
hlist_add_head(&obj->node, &objects);
1313
+
pool_push_batch(&pool_global, &objects);
1342
1314
}
1343
1315
1344
-
debug_objects_allocated = ODEBUG_POOL_SIZE;
1345
-
pool_global.cnt = ODEBUG_POOL_SIZE;
1346
-
1347
-
/*
1348
-
* Move the allocated objects to the global pool and disconnect the
1349
-
* boot pool.
1350
-
*/
1351
-
hlist_move_list(&objects, &pool_global.objects);
1316
+
/* Disconnect the boot pool. */
1352
1317
pool_boot.first = NULL;
1353
1318
1354
1319
/* Replace the active object references */
···
1348
1329
hlist_move_list(&db->list, &objects);
1349
1330
1350
1331
hlist_for_each_entry(obj, &objects, node) {
1351
-
new = hlist_entry(pool_global.objects.first, typeof(*obj), node);
1352
-
hlist_del(&new->node);
1353
-
pool_global.cnt--;
1332
+
struct debug_obj *new = pcpu_alloc();
1333
+
1354
1334
/* copy object data */
1355
1335
*new = *obj;
1356
1336
hlist_add_head(&new->node, &db->list);
···
1358
1340
return true;
1359
1341
free:
1360
1342
/* Can't use free_object_list() as the cache is not populated yet */
1361
-
hlist_for_each_entry_safe(obj, tmp, &objects, node) {
1343
+
hlist_for_each_entry_safe(obj, tmp, &pool_global.objects, node) {
1362
1344
hlist_del(&obj->node);
1363
1345
kmem_cache_free(cache, obj);
1364
1346
}