Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
debugobjects: Move pools into a datastructure
The contention on the global pool lock can be reduced by strict batch
processing where batches of objects are moved from one list head to another
instead of moving them object by object. This also reduces the cache
footprint because it avoids the list walk and dirties at maximum three
cache lines instead of potentially up to eighteen.
To prepare for that, move the hlist head and related counters into a
struct.
No functional change.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Zhen Lei <thunder.leizhen@huawei.com>
Link: https://lore.kernel.org/all/20241007164913.646171170@linutronix.de
+78
-62
+78
-62
lib/debugobjects.c
+78
-62
lib/debugobjects.c
···
52
52
int obj_free;
53
53
};
54
54
55
+
struct obj_pool {
56
+
struct hlist_head objects;
57
+
unsigned int cnt;
58
+
} ____cacheline_aligned;
59
+
55
60
static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
56
61
57
62
static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];
···
65
60
66
61
static DEFINE_RAW_SPINLOCK(pool_lock);
67
62
68
-
static HLIST_HEAD(obj_pool);
69
-
static HLIST_HEAD(obj_to_free);
63
+
static struct obj_pool pool_global;
64
+
static struct obj_pool pool_to_free;
70
65
71
66
/*
72
67
* Because of the presence of percpu free pools, obj_pool_free will
···
76
71
* can be off.
77
72
*/
78
73
static int __data_racy obj_pool_min_free = ODEBUG_POOL_SIZE;
79
-
static int __data_racy obj_pool_free = ODEBUG_POOL_SIZE;
80
74
static int obj_pool_used;
81
75
static int __data_racy obj_pool_max_used;
82
76
static bool obj_freeing;
83
-
/* The number of objs on the global free list */
84
-
static int obj_nr_tofree;
85
77
86
78
static int __data_racy debug_objects_maxchain __read_mostly;
87
79
static int __data_racy __maybe_unused debug_objects_maxchecked __read_mostly;
···
126
124
[ODEBUG_STATE_NOTAVAILABLE] = "not available",
127
125
};
128
126
127
+
static __always_inline unsigned int pool_count(struct obj_pool *pool)
128
+
{
129
+
return READ_ONCE(pool->cnt);
130
+
}
131
+
132
+
static inline bool pool_global_should_refill(void)
133
+
{
134
+
return READ_ONCE(pool_global.cnt) < debug_objects_pool_min_level;
135
+
}
136
+
137
+
static inline bool pool_global_must_refill(void)
138
+
{
139
+
return READ_ONCE(pool_global.cnt) < (debug_objects_pool_min_level / 2);
140
+
}
141
+
129
142
static void free_object_list(struct hlist_head *head)
130
143
{
131
144
struct hlist_node *tmp;
···
163
146
/*
164
147
* Reuse objs from the global obj_to_free list; they will be
165
148
* reinitialized when allocating.
166
-
*
167
-
* obj_nr_tofree is checked locklessly; the READ_ONCE() pairs with
168
-
* the WRITE_ONCE() in pool_lock critical sections.
169
149
*/
170
-
if (!READ_ONCE(obj_nr_tofree))
150
+
if (!pool_count(&pool_to_free))
171
151
return;
172
152
173
153
/*
···
185
171
* Recheck with the lock held as the worker thread might have
186
172
* won the race and freed the global free list already.
187
173
*/
188
-
while (obj_nr_tofree && (obj_pool_free < debug_objects_pool_min_level)) {
189
-
obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
174
+
while (pool_to_free.cnt && (pool_global.cnt < debug_objects_pool_min_level)) {
175
+
obj = hlist_entry(pool_to_free.objects.first, typeof(*obj), node);
190
176
hlist_del(&obj->node);
191
-
WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
192
-
hlist_add_head(&obj->node, &obj_pool);
193
-
WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
177
+
WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt - 1);
178
+
hlist_add_head(&obj->node, &pool_global.objects);
179
+
WRITE_ONCE(pool_global.cnt, pool_global.cnt + 1);
194
180
}
195
181
clear_bit(0, &state);
196
182
}
···
204
190
* - One other CPU is already allocating
205
191
* - the global pool has not reached the critical level yet
206
192
*/
207
-
if (READ_ONCE(obj_pool_free) > (debug_objects_pool_min_level / 2) &&
208
-
atomic_read(&cpus_allocating))
193
+
if (!pool_global_must_refill() && atomic_read(&cpus_allocating))
209
194
return;
210
195
211
196
atomic_inc(&cpus_allocating);
212
-
while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) {
197
+
while (pool_global_should_refill()) {
213
198
struct debug_obj *new, *last = NULL;
214
199
HLIST_HEAD(head);
215
200
int cnt;
···
225
212
break;
226
213
227
214
guard(raw_spinlock_irqsave)(&pool_lock);
228
-
hlist_splice_init(&head, &last->node, &obj_pool);
215
+
hlist_splice_init(&head, &last->node, &pool_global.objects);
229
216
debug_objects_allocated += cnt;
230
-
WRITE_ONCE(obj_pool_free, obj_pool_free + cnt);
217
+
WRITE_ONCE(pool_global.cnt, pool_global.cnt + cnt);
231
218
}
232
219
atomic_dec(&cpus_allocating);
233
220
}
···
281
268
}
282
269
283
270
raw_spin_lock(&pool_lock);
284
-
obj = __alloc_object(&obj_pool);
271
+
obj = __alloc_object(&pool_global.objects);
285
272
if (obj) {
286
273
obj_pool_used++;
287
-
WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
274
+
WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
288
275
289
276
/*
290
277
* Looking ahead, allocate one batch of debug objects and
···
296
283
for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
297
284
struct debug_obj *obj2;
298
285
299
-
obj2 = __alloc_object(&obj_pool);
286
+
obj2 = __alloc_object(&pool_global.objects);
300
287
if (!obj2)
301
288
break;
302
-
hlist_add_head(&obj2->node,
303
-
&percpu_pool->free_objs);
289
+
hlist_add_head(&obj2->node, &percpu_pool->free_objs);
304
290
percpu_pool->obj_free++;
305
291
obj_pool_used++;
306
-
WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
292
+
WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
307
293
}
308
294
}
309
295
310
296
if (obj_pool_used > obj_pool_max_used)
311
297
obj_pool_max_used = obj_pool_used;
312
298
313
-
if (obj_pool_free < obj_pool_min_free)
314
-
obj_pool_min_free = obj_pool_free;
299
+
if (pool_global.cnt < obj_pool_min_free)
300
+
obj_pool_min_free = pool_global.cnt;
315
301
}
316
302
raw_spin_unlock(&pool_lock);
317
303
···
341
329
if (!raw_spin_trylock_irqsave(&pool_lock, flags))
342
330
return;
343
331
344
-
if (obj_pool_free >= debug_objects_pool_size)
332
+
if (pool_global.cnt >= debug_objects_pool_size)
345
333
goto free_objs;
346
334
347
335
/*
···
351
339
* may be gearing up to use more and more objects, don't free any
352
340
* of them until the next round.
353
341
*/
354
-
while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
355
-
obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
342
+
while (pool_to_free.cnt && pool_global.cnt < debug_objects_pool_size) {
343
+
obj = hlist_entry(pool_to_free.objects.first, typeof(*obj), node);
356
344
hlist_del(&obj->node);
357
-
hlist_add_head(&obj->node, &obj_pool);
358
-
WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
359
-
WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
345
+
hlist_add_head(&obj->node, &pool_global.objects);
346
+
WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt - 1);
347
+
WRITE_ONCE(pool_global.cnt, pool_global.cnt + 1);
360
348
}
361
349
raw_spin_unlock_irqrestore(&pool_lock, flags);
362
350
return;
···
367
355
* list. Move remaining free objs to a temporary list to free the
368
356
* memory outside the pool_lock held region.
369
357
*/
370
-
if (obj_nr_tofree) {
371
-
hlist_move_list(&obj_to_free, &tofree);
372
-
WRITE_ONCE(obj_nr_tofree, 0);
358
+
if (pool_to_free.cnt) {
359
+
hlist_move_list(&pool_to_free.objects, &tofree);
360
+
WRITE_ONCE(pool_to_free.cnt, 0);
373
361
}
374
362
raw_spin_unlock_irqrestore(&pool_lock, flags);
375
363
···
412
400
413
401
free_to_obj_pool:
414
402
raw_spin_lock(&pool_lock);
415
-
work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&
416
-
(obj_nr_tofree < ODEBUG_FREE_WORK_MAX);
403
+
work = (pool_global.cnt > debug_objects_pool_size) && obj_cache &&
404
+
(pool_to_free.cnt < ODEBUG_FREE_WORK_MAX);
417
405
obj_pool_used--;
418
406
419
407
if (work) {
420
-
WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
421
-
hlist_add_head(&obj->node, &obj_to_free);
408
+
WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt + 1);
409
+
hlist_add_head(&obj->node, &pool_to_free.objects);
422
410
if (lookahead_count) {
423
-
WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count);
411
+
WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt + lookahead_count);
424
412
obj_pool_used -= lookahead_count;
425
413
while (lookahead_count) {
426
414
hlist_add_head(&objs[--lookahead_count]->node,
427
-
&obj_to_free);
415
+
&pool_to_free.objects);
428
416
}
429
417
}
430
418
431
-
if ((obj_pool_free > debug_objects_pool_size) &&
432
-
(obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {
419
+
if ((pool_global.cnt > debug_objects_pool_size) &&
420
+
(pool_to_free.cnt < ODEBUG_FREE_WORK_MAX)) {
433
421
int i;
434
422
435
423
/*
436
424
* Free one more batch of objects from obj_pool.
437
425
*/
438
426
for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
439
-
obj = __alloc_object(&obj_pool);
440
-
hlist_add_head(&obj->node, &obj_to_free);
441
-
WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
442
-
WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
427
+
obj = __alloc_object(&pool_global.objects);
428
+
hlist_add_head(&obj->node, &pool_to_free.objects);
429
+
WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
430
+
WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt + 1);
443
431
}
444
432
}
445
433
} else {
446
-
WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
447
-
hlist_add_head(&obj->node, &obj_pool);
434
+
WRITE_ONCE(pool_global.cnt, pool_global.cnt + 1);
435
+
hlist_add_head(&obj->node, &pool_global.objects);
448
436
if (lookahead_count) {
449
-
WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count);
437
+
WRITE_ONCE(pool_global.cnt, pool_global.cnt + lookahead_count);
450
438
obj_pool_used -= lookahead_count;
451
439
while (lookahead_count) {
452
440
hlist_add_head(&objs[--lookahead_count]->node,
453
-
&obj_pool);
441
+
&pool_global.objects);
454
442
}
455
443
}
456
444
}
···
465
453
static void free_object(struct debug_obj *obj)
466
454
{
467
455
__free_object(obj);
468
-
if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
456
+
if (!READ_ONCE(obj_freeing) && pool_count(&pool_to_free)) {
469
457
WRITE_ONCE(obj_freeing, true);
470
458
schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
471
459
}
···
634
622
if (unlikely(!obj_cache))
635
623
return;
636
624
637
-
if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))
625
+
if (likely(!pool_global_should_refill()))
638
626
return;
639
627
640
628
/* Try reusing objects from obj_to_free_list */
641
629
fill_pool_from_freelist();
642
630
643
-
if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))
631
+
if (likely(!pool_global_should_refill()))
644
632
return;
645
633
646
634
/*
···
1052
1040
debug_objects_maxchecked = objs_checked;
1053
1041
1054
1042
/* Schedule work to actually kmem_cache_free() objects */
1055
-
if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
1043
+
if (!READ_ONCE(obj_freeing) && pool_count(&pool_to_free)) {
1056
1044
WRITE_ONCE(obj_freeing, true);
1057
1045
schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
1058
1046
}
···
1078
1066
seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);
1079
1067
seq_printf(m, "warnings :%d\n", debug_objects_warnings);
1080
1068
seq_printf(m, "fixups :%d\n", debug_objects_fixups);
1081
-
seq_printf(m, "pool_free :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free);
1069
+
seq_printf(m, "pool_free :%d\n", pool_count(&pool_global) + obj_percpu_free);
1082
1070
seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
1083
1071
seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
1084
1072
seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free);
1085
1073
seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
1086
-
seq_printf(m, "on_free_list :%d\n", READ_ONCE(obj_nr_tofree));
1074
+
seq_printf(m, "on_free_list :%d\n", pool_count(&pool_to_free));
1087
1075
seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
1088
1076
seq_printf(m, "objs_freed :%d\n", debug_objects_freed);
1089
1077
return 0;
···
1342
1330
raw_spin_lock_init(&obj_hash[i].lock);
1343
1331
1344
1332
for (i = 0; i < ODEBUG_POOL_SIZE; i++)
1345
-
hlist_add_head(&obj_static_pool[i].node, &obj_pool);
1333
+
hlist_add_head(&obj_static_pool[i].node, &pool_global.objects);
1334
+
1335
+
pool_global.cnt = ODEBUG_POOL_SIZE;
1346
1336
}
1347
1337
1348
1338
/*
···
1368
1354
hlist_add_head(&obj->node, &objects);
1369
1355
}
1370
1356
1371
-
debug_objects_allocated += i;
1357
+
debug_objects_allocated = ODEBUG_POOL_SIZE;
1358
+
pool_global.cnt = ODEBUG_POOL_SIZE;
1372
1359
1373
1360
/*
1374
1361
* Replace the statically allocated objects list with the allocated
1375
1362
* objects list.
1376
1363
*/
1377
-
hlist_move_list(&objects, &obj_pool);
1364
+
hlist_move_list(&objects, &pool_global.objects);
1378
1365
1379
1366
/* Replace the active object references */
1380
1367
for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
1381
1368
hlist_move_list(&db->list, &objects);
1382
1369
1383
1370
hlist_for_each_entry(obj, &objects, node) {
1384
-
new = hlist_entry(obj_pool.first, typeof(*obj), node);
1371
+
new = hlist_entry(pool_global.objects.first, typeof(*obj), node);
1385
1372
hlist_del(&new->node);
1373
+
pool_global.cnt--;
1386
1374
/* copy object data */
1387
1375
*new = *obj;
1388
1376
hlist_add_head(&new->node, &db->list);