debugobjects: Reduce parallel pool fill attempts

The contention on the global pool_lock can be massive when the global pool
needs to be refilled and many CPUs try to handle this.

Address this by:

- splitting the refill from free list and allocation.

Refill from free list has no constraints vs. the context on RT, so
it can be tried outside of the RT specific preemptible() guard

- Let only one CPU handle the free list

- Let only one CPU do allocations unless the pool level is below
half of the minimum fill level.

Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Zhen Lei <thunder.leizhen@huawei.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/all/20240911083521.2257-4-thunder.leizhen@huawei.com-
Link: https://lore.kernel.org/all/20241007164913.582118421@linutronix.de

--
lib/debugobjects.c | 84 +++++++++++++++++++++++++++++++++++++----------------
1 file changed, 59 insertions(+), 25 deletions(-)

authored by

Zhen Lei and committed by

Thomas Gleixner 2 years ago d8c6cd3a 661cc28b

+61 -27

1 changed file

expand all

lib

debugobjects.c

+61 -27

lib/debugobjects.c

··· 138 138 debug_objects_freed += cnt; 139 139 } 140 140 141 - static void fill_pool(void) 141 + static void fill_pool_from_freelist(void) 142 142 { 143 - gfp_t gfp = __GFP_HIGH | __GFP_NOWARN; 143 + static unsigned long state; 144 144 struct debug_obj *obj; 145 - unsigned long flags; 146 - 147 - if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level)) 148 - return; 149 145 150 146 /* 151 147 * Reuse objs from the global obj_to_free list; they will be ··· 150 154 * obj_nr_tofree is checked locklessly; the READ_ONCE() pairs with 151 155 * the WRITE_ONCE() in pool_lock critical sections. 152 156 */ 153 - if (READ_ONCE(obj_nr_tofree)) { 154 - raw_spin_lock_irqsave(&pool_lock, flags); 155 - /* 156 - * Recheck with the lock held as the worker thread might have 157 - * won the race and freed the global free list already. 158 - */ 159 - while (obj_nr_tofree && (obj_pool_free < debug_objects_pool_min_level)) { 160 - obj = hlist_entry(obj_to_free.first, typeof(*obj), node); 161 - hlist_del(&obj->node); 162 - WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1); 163 - hlist_add_head(&obj->node, &obj_pool); 164 - WRITE_ONCE(obj_pool_free, obj_pool_free + 1); 165 - } 166 - raw_spin_unlock_irqrestore(&pool_lock, flags); 167 - } 168 - 169 - if (unlikely(!obj_cache)) 157 + if (!READ_ONCE(obj_nr_tofree)) 170 158 return; 171 159 160 + /* 161 + * Prevent the context from being scheduled or interrupted after 162 + * setting the state flag; 163 + */ 164 + guard(irqsave)(); 165 + 166 + /* 167 + * Avoid lock contention on &pool_lock and avoid making the cache 168 + * line exclusive by testing the bit before attempting to set it. 169 + */ 170 + if (test_bit(0, &state) || test_and_set_bit(0, &state)) 171 + return; 172 + 173 + guard(raw_spinlock)(&pool_lock); 174 + /* 175 + * Recheck with the lock held as the worker thread might have 176 + * won the race and freed the global free list already. 177 + */ 178 + while (obj_nr_tofree && (obj_pool_free < debug_objects_pool_min_level)) { 179 + obj = hlist_entry(obj_to_free.first, typeof(*obj), node); 180 + hlist_del(&obj->node); 181 + WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1); 182 + hlist_add_head(&obj->node, &obj_pool); 183 + WRITE_ONCE(obj_pool_free, obj_pool_free + 1); 184 + } 185 + clear_bit(0, &state); 186 + } 187 + 188 + static void fill_pool(void) 189 + { 190 + static atomic_t cpus_allocating; 191 + 192 + /* 193 + * Avoid allocation and lock contention when: 194 + * - One other CPU is already allocating 195 + * - the global pool has not reached the critical level yet 196 + */ 197 + if (READ_ONCE(obj_pool_free) > (debug_objects_pool_min_level / 2) && 198 + atomic_read(&cpus_allocating)) 199 + return; 200 + 201 + atomic_inc(&cpus_allocating); 172 202 while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) { 173 203 struct debug_obj *new, *last = NULL; 174 204 HLIST_HEAD(head); 175 205 int cnt; 176 206 177 207 for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) { 178 - new = kmem_cache_zalloc(obj_cache, gfp); 208 + new = kmem_cache_zalloc(obj_cache, __GFP_HIGH | __GFP_NOWARN); 179 209 if (!new) 180 210 break; 181 211 hlist_add_head(&new->node, &head); ··· 209 187 last = new; 210 188 } 211 189 if (!cnt) 212 - return; 190 + break; 213 191 214 - raw_spin_lock_irqsave(&pool_lock, flags); 192 + guard(raw_spinlock_irqsave)(&pool_lock); 215 193 hlist_splice_init(&head, &last->node, &obj_pool); 216 194 debug_objects_allocated += cnt; 217 195 WRITE_ONCE(obj_pool_free, obj_pool_free + cnt); 218 - raw_spin_unlock_irqrestore(&pool_lock, flags); 219 196 } 197 + atomic_dec(&cpus_allocating); 220 198 } 221 199 222 200 /* ··· 619 597 620 598 static void debug_objects_fill_pool(void) 621 599 { 600 + if (unlikely(!obj_cache)) 601 + return; 602 + 603 + if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level)) 604 + return; 605 + 606 + /* Try reusing objects from obj_to_free_list */ 607 + fill_pool_from_freelist(); 608 + 609 + if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level)) 610 + return; 611 + 622 612 /* 623 613 * On RT enabled kernels the pool refill must happen in preemptible 624 614 * context -- for !RT kernels we rely on the fact that spinlock_t and

Configure Feed

Configure Feed