Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
debugobjects: Move min/max count into pool struct
Having the accounting in the datastructure is better in terms of cache
lines and allows more optimizations later on.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Zhen Lei <thunder.leizhen@huawei.com>
Link: https://lore.kernel.org/all/20241007164913.831908427@linutronix.de
+31
-24
+31
-24
lib/debugobjects.c
+31
-24
lib/debugobjects.c
···
46
46
struct obj_pool {
47
47
struct hlist_head objects;
48
48
unsigned int cnt;
49
+
unsigned int min_cnt;
50
+
unsigned int max_cnt;
49
51
} ____cacheline_aligned;
50
52
51
-
static DEFINE_PER_CPU(struct obj_pool, pool_pcpu);
53
+
54
+
static DEFINE_PER_CPU_ALIGNED(struct obj_pool, pool_pcpu) = {
55
+
.max_cnt = ODEBUG_POOL_PERCPU_SIZE,
56
+
};
52
57
53
58
static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];
54
59
···
61
56
62
57
static DEFINE_RAW_SPINLOCK(pool_lock);
63
58
64
-
static struct obj_pool pool_global;
65
-
static struct obj_pool pool_to_free;
59
+
static struct obj_pool pool_global = {
60
+
.min_cnt = ODEBUG_POOL_MIN_LEVEL,
61
+
.max_cnt = ODEBUG_POOL_SIZE,
62
+
};
63
+
64
+
static struct obj_pool pool_to_free = {
65
+
.max_cnt = UINT_MAX,
66
+
};
66
67
67
68
static HLIST_HEAD(pool_boot);
68
69
···
90
79
static int __data_racy debug_objects_warnings __read_mostly;
91
80
static bool __data_racy debug_objects_enabled __read_mostly
92
81
= CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;
93
-
static int debug_objects_pool_size __ro_after_init
94
-
= ODEBUG_POOL_SIZE;
95
-
static int debug_objects_pool_min_level __ro_after_init
96
-
= ODEBUG_POOL_MIN_LEVEL;
97
82
98
-
static const struct debug_obj_descr *descr_test __read_mostly;
99
-
static struct kmem_cache *obj_cache __ro_after_init;
83
+
static const struct debug_obj_descr *descr_test __read_mostly;
84
+
static struct kmem_cache *obj_cache __ro_after_init;
100
85
101
86
/*
102
87
* Track numbers of kmem_cache_alloc()/free() calls done.
···
131
124
return READ_ONCE(pool->cnt);
132
125
}
133
126
134
-
static inline bool pool_global_should_refill(void)
127
+
static __always_inline bool pool_should_refill(struct obj_pool *pool)
135
128
{
136
-
return READ_ONCE(pool_global.cnt) < debug_objects_pool_min_level;
129
+
return pool_count(pool) < pool->min_cnt;
137
130
}
138
131
139
-
static inline bool pool_global_must_refill(void)
132
+
static __always_inline bool pool_must_refill(struct obj_pool *pool)
140
133
{
141
-
return READ_ONCE(pool_global.cnt) < (debug_objects_pool_min_level / 2);
134
+
return pool_count(pool) < pool->min_cnt / 2;
142
135
}
143
136
144
137
static void free_object_list(struct hlist_head *head)
···
185
178
* Recheck with the lock held as the worker thread might have
186
179
* won the race and freed the global free list already.
187
180
*/
188
-
while (pool_to_free.cnt && (pool_global.cnt < debug_objects_pool_min_level)) {
181
+
while (pool_to_free.cnt && (pool_global.cnt < pool_global.min_cnt)) {
189
182
obj = hlist_entry(pool_to_free.objects.first, typeof(*obj), node);
190
183
hlist_del(&obj->node);
191
184
WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt - 1);
···
204
197
* - One other CPU is already allocating
205
198
* - the global pool has not reached the critical level yet
206
199
*/
207
-
if (!pool_global_must_refill() && atomic_read(&cpus_allocating))
200
+
if (!pool_must_refill(&pool_global) && atomic_read(&cpus_allocating))
208
201
return;
209
202
210
203
atomic_inc(&cpus_allocating);
211
-
while (pool_global_should_refill()) {
204
+
while (pool_should_refill(&pool_global)) {
212
205
struct debug_obj *new, *last = NULL;
213
206
HLIST_HEAD(head);
214
207
int cnt;
···
344
337
if (!raw_spin_trylock_irqsave(&pool_lock, flags))
345
338
return;
346
339
347
-
if (pool_global.cnt >= debug_objects_pool_size)
340
+
if (pool_global.cnt >= pool_global.max_cnt)
348
341
goto free_objs;
349
342
350
343
/*
···
354
347
* may be gearing up to use more and more objects, don't free any
355
348
* of them until the next round.
356
349
*/
357
-
while (pool_to_free.cnt && pool_global.cnt < debug_objects_pool_size) {
350
+
while (pool_to_free.cnt && pool_global.cnt < pool_global.max_cnt) {
358
351
obj = hlist_entry(pool_to_free.objects.first, typeof(*obj), node);
359
352
hlist_del(&obj->node);
360
353
hlist_add_head(&obj->node, &pool_global.objects);
···
415
408
}
416
409
417
410
raw_spin_lock(&pool_lock);
418
-
work = (pool_global.cnt > debug_objects_pool_size) && obj_cache &&
411
+
work = (pool_global.cnt > pool_global.max_cnt) && obj_cache &&
419
412
(pool_to_free.cnt < ODEBUG_FREE_WORK_MAX);
420
413
obj_pool_used--;
421
414
···
431
424
}
432
425
}
433
426
434
-
if ((pool_global.cnt > debug_objects_pool_size) &&
427
+
if ((pool_global.cnt > pool_global.max_cnt) &&
435
428
(pool_to_free.cnt < ODEBUG_FREE_WORK_MAX)) {
436
429
int i;
437
430
···
636
629
if (unlikely(!obj_cache))
637
630
return;
638
631
639
-
if (likely(!pool_global_should_refill()))
632
+
if (likely(!pool_should_refill(&pool_global)))
640
633
return;
641
634
642
635
/* Try reusing objects from obj_to_free_list */
643
636
fill_pool_from_freelist();
644
637
645
-
if (likely(!pool_global_should_refill()))
638
+
if (likely(!pool_should_refill(&pool_global)))
646
639
return;
647
640
648
641
/*
···
1434
1427
* system.
1435
1428
*/
1436
1429
extras = num_possible_cpus() * ODEBUG_BATCH_SIZE;
1437
-
debug_objects_pool_size += extras;
1438
-
debug_objects_pool_min_level += extras;
1430
+
pool_global.max_cnt += extras;
1431
+
pool_global.min_cnt += extras;
1439
1432
1440
1433
/* Everything worked. Expose the cache */
1441
1434
obj_cache = cache;