Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
debugobjects: Move pool statistics into global_pool struct
Keep it along with the pool as that's a hot cache line anyway and it makes
the code more comprehensible.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Zhen Lei <thunder.leizhen@huawei.com>
Link: https://lore.kernel.org/all/20241007164914.318776207@linutronix.de
+51
-34
+51
-34
lib/debugobjects.c
+51
-34
lib/debugobjects.c
···
47
47
raw_spinlock_t lock;
48
48
};
49
49
50
+
struct pool_stats {
51
+
unsigned int cur_used;
52
+
unsigned int max_used;
53
+
unsigned int min_fill;
54
+
};
55
+
50
56
struct obj_pool {
51
57
struct hlist_head objects;
52
58
unsigned int cnt;
53
59
unsigned int min_cnt;
54
60
unsigned int max_cnt;
61
+
struct pool_stats stats;
55
62
} ____cacheline_aligned;
56
63
57
64
···
73
66
static DEFINE_RAW_SPINLOCK(pool_lock);
74
67
75
68
static struct obj_pool pool_global = {
76
-
.min_cnt = ODEBUG_POOL_MIN_LEVEL,
77
-
.max_cnt = ODEBUG_POOL_SIZE,
69
+
.min_cnt = ODEBUG_POOL_MIN_LEVEL,
70
+
.max_cnt = ODEBUG_POOL_SIZE,
71
+
.stats = {
72
+
.min_fill = ODEBUG_POOL_SIZE,
73
+
},
78
74
};
79
75
80
76
static struct obj_pool pool_to_free = {
···
86
76
87
77
static HLIST_HEAD(pool_boot);
88
78
89
-
/*
90
-
* Because of the presence of percpu free pools, obj_pool_free will
91
-
* under-count those in the percpu free pools. Similarly, obj_pool_used
92
-
* will over-count those in the percpu free pools. Adjustments will be
93
-
* made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
94
-
* can be off.
95
-
*/
96
-
static int __data_racy obj_pool_min_free = ODEBUG_POOL_SIZE;
97
-
static int obj_pool_used;
98
-
static int __data_racy obj_pool_max_used;
99
79
static bool obj_freeing;
100
80
101
81
static int __data_racy debug_objects_maxchain __read_mostly;
···
231
231
return obj;
232
232
}
233
233
234
+
static void pcpu_refill_stats(void)
235
+
{
236
+
struct pool_stats *stats = &pool_global.stats;
237
+
238
+
WRITE_ONCE(stats->cur_used, stats->cur_used + ODEBUG_BATCH_SIZE);
239
+
240
+
if (stats->cur_used > stats->max_used)
241
+
stats->max_used = stats->cur_used;
242
+
243
+
if (pool_global.cnt < stats->min_fill)
244
+
stats->min_fill = pool_global.cnt;
245
+
}
246
+
234
247
static struct debug_obj *pcpu_alloc(void)
235
248
{
236
249
struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu);
···
263
250
if (!pool_move_batch(pcp, &pool_global))
264
251
return NULL;
265
252
}
266
-
obj_pool_used += ODEBUG_BATCH_SIZE;
267
-
268
-
if (obj_pool_used > obj_pool_max_used)
269
-
obj_pool_max_used = obj_pool_used;
270
-
271
-
if (pool_global.cnt < obj_pool_min_free)
272
-
obj_pool_min_free = pool_global.cnt;
253
+
pcpu_refill_stats();
273
254
}
274
255
}
275
256
···
292
285
/* Try to fit the batch into the pool_global first */
293
286
if (!pool_move_batch(&pool_global, pcp))
294
287
pool_move_batch(&pool_to_free, pcp);
295
-
obj_pool_used -= ODEBUG_BATCH_SIZE;
288
+
WRITE_ONCE(pool_global.stats.cur_used, pool_global.stats.cur_used - ODEBUG_BATCH_SIZE);
296
289
}
297
290
298
291
static void free_object_list(struct hlist_head *head)
···
1081
1074
1082
1075
static int debug_stats_show(struct seq_file *m, void *v)
1083
1076
{
1084
-
int cpu, obj_percpu_free = 0;
1077
+
unsigned int cpu, pool_used, pcp_free = 0;
1085
1078
1079
+
/*
1080
+
* pool_global.stats.cur_used is the number of batches currently
1081
+
* handed out to per CPU pools. Convert it to number of objects
1082
+
* and subtract the number of free objects in the per CPU pools.
1083
+
* As this is lockless the number is an estimate.
1084
+
*/
1086
1085
for_each_possible_cpu(cpu)
1087
-
obj_percpu_free += per_cpu(pool_pcpu.cnt, cpu);
1086
+
pcp_free += per_cpu(pool_pcpu.cnt, cpu);
1088
1087
1089
-
seq_printf(m, "max_chain :%d\n", debug_objects_maxchain);
1090
-
seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);
1091
-
seq_printf(m, "warnings :%d\n", debug_objects_warnings);
1092
-
seq_printf(m, "fixups :%d\n", debug_objects_fixups);
1093
-
seq_printf(m, "pool_free :%d\n", pool_count(&pool_global) + obj_percpu_free);
1094
-
seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
1095
-
seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
1096
-
seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free);
1097
-
seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
1098
-
seq_printf(m, "on_free_list :%d\n", pool_count(&pool_to_free));
1099
-
seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
1100
-
seq_printf(m, "objs_freed :%d\n", debug_objects_freed);
1088
+
pool_used = data_race(pool_global.stats.cur_used);
1089
+
pcp_free = min(pool_used, pcp_free);
1090
+
pool_used -= pcp_free;
1091
+
1092
+
seq_printf(m, "max_chain : %d\n", debug_objects_maxchain);
1093
+
seq_printf(m, "max_checked : %d\n", debug_objects_maxchecked);
1094
+
seq_printf(m, "warnings : %d\n", debug_objects_warnings);
1095
+
seq_printf(m, "fixups : %d\n", debug_objects_fixups);
1096
+
seq_printf(m, "pool_free : %u\n", pool_count(&pool_global) + pcp_free);
1097
+
seq_printf(m, "pool_pcp_free : %u\n", pcp_free);
1098
+
seq_printf(m, "pool_min_free : %u\n", data_race(pool_global.stats.min_fill));
1099
+
seq_printf(m, "pool_used : %u\n", pool_used);
1100
+
seq_printf(m, "pool_max_used : %u\n", data_race(pool_global.stats.max_used));
1101
+
seq_printf(m, "on_free_list : %u\n", pool_count(&pool_to_free));
1102
+
seq_printf(m, "objs_allocated: %d\n", debug_objects_allocated);
1103
+
seq_printf(m, "objs_freed : %d\n", debug_objects_freed);
1101
1104
return 0;
1102
1105
}
1103
1106
DEFINE_SHOW_ATTRIBUTE(debug_stats);