debugobjects: Track object usage to avoid premature freeing of objects

The freelist is freed at a constant rate independent of the actual usage
requirements. That's bad in scenarios where usage comes in bursts. The end
of a burst puts the objects on the free list and freeing proceeds even when
the next burst which requires objects started again.

Keep track of the usage with a exponentially wheighted moving average and
take that into account in the worker function which frees objects from the
free list.

This further reduces the kmem_cache allocation/free rate for a full kernel
compile:

kmem_cache_alloc() kmem_cache_free()
Baseline: 225k 173k
Usage: 170k 117k

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Zhen Lei <thunder.leizhen@huawei.com>
Link: https://lore.kernel.org/all/87bjznhme2.ffs@tglx

Thomas Gleixner 2 years ago ff8d523c 13f9ca72

+40 -5

1 changed file

expand all

lib

debugobjects.c

+40 -5

lib/debugobjects.c

··· 13 13 #include <linux/hash.h> 14 14 #include <linux/kmemleak.h> 15 15 #include <linux/sched.h> 16 + #include <linux/sched/loadavg.h> 16 17 #include <linux/sched/task_stack.h> 17 18 #include <linux/seq_file.h> 18 19 #include <linux/slab.h> ··· 87 86 88 87 static HLIST_HEAD(pool_boot); 89 88 89 + static unsigned long avg_usage; 90 90 static bool obj_freeing; 91 91 92 92 static int __data_racy debug_objects_maxchain __read_mostly; ··· 429 427 return NULL; 430 428 } 431 429 430 + static void calc_usage(void) 431 + { 432 + static DEFINE_RAW_SPINLOCK(avg_lock); 433 + static unsigned long avg_period; 434 + unsigned long cur, now = jiffies; 435 + 436 + if (!time_after_eq(now, READ_ONCE(avg_period))) 437 + return; 438 + 439 + if (!raw_spin_trylock(&avg_lock)) 440 + return; 441 + 442 + WRITE_ONCE(avg_period, now + msecs_to_jiffies(10)); 443 + cur = READ_ONCE(pool_global.stats.cur_used) * ODEBUG_FREE_WORK_MAX; 444 + WRITE_ONCE(avg_usage, calc_load(avg_usage, EXP_5, cur)); 445 + raw_spin_unlock(&avg_lock); 446 + } 447 + 432 448 static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b, 433 449 const struct debug_obj_descr *descr) 434 450 { 435 451 struct debug_obj *obj; 452 + 453 + calc_usage(); 436 454 437 455 if (static_branch_likely(&obj_cache_enabled)) 438 456 obj = pcpu_alloc(); ··· 472 450 /* workqueue function to free objects. */ 473 451 static void free_obj_work(struct work_struct *work) 474 452 { 475 - bool free = true; 453 + static unsigned long last_use_avg; 454 + unsigned long cur_used, last_used, delta; 455 + unsigned int max_free = 0; 476 456 477 457 WRITE_ONCE(obj_freeing, false); 458 + 459 + /* Rate limit freeing based on current use average */ 460 + cur_used = READ_ONCE(avg_usage); 461 + last_used = last_use_avg; 462 + last_use_avg = cur_used; 478 463 479 464 if (!pool_count(&pool_to_free)) 480 465 return; 481 466 482 - for (unsigned int cnt = 0; cnt < ODEBUG_FREE_WORK_MAX; cnt++) { 467 + if (cur_used <= last_used) { 468 + delta = (last_used - cur_used) / ODEBUG_FREE_WORK_MAX; 469 + max_free = min(delta, ODEBUG_FREE_WORK_MAX); 470 + } 471 + 472 + for (int cnt = 0; cnt < ODEBUG_FREE_WORK_MAX; cnt++) { 483 473 HLIST_HEAD(tofree); 484 474 485 475 /* Acquire and drop the lock for each batch */ ··· 502 468 /* Refill the global pool if possible */ 503 469 if (pool_move_batch(&pool_global, &pool_to_free)) { 504 470 /* Don't free as there seems to be demand */ 505 - free = false; 506 - } else if (free) { 471 + max_free = 0; 472 + } else if (max_free) { 507 473 pool_pop_batch(&tofree, &pool_to_free); 474 + max_free--; 508 475 } else { 509 476 return; 510 477 } ··· 1145 1110 for_each_possible_cpu(cpu) 1146 1111 pcp_free += per_cpu(pool_pcpu.cnt, cpu); 1147 1112 1148 - pool_used = data_race(pool_global.stats.cur_used); 1113 + pool_used = READ_ONCE(pool_global.stats.cur_used); 1149 1114 pcp_free = min(pool_used, pcp_free); 1150 1115 pool_used -= pcp_free; 1151 1116

Configure Feed

Configure Feed