debugobjects: Implement batch processing

Adding and removing single objects in a loop is bad in terms of lock
contention and cache line accesses.

To implement batching, record the last object in a batch in the object
itself. This is trivialy possible as hlists are strictly stacks. At a batch
boundary, when the first object is added to the list the object stores a
pointer to itself in debug_obj::batch_last. When the next object is added
to the list then the batch_last pointer is retrieved from the first object
in the list and stored in the to be added one.

That means for batch processing the first object always has a pointer to
the last object in a batch, which allows to move batches in a cache line
efficient way and reduces the lock held time.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Zhen Lei <thunder.leizhen@huawei.com>
Link: https://lore.kernel.org/all/20241007164914.258995000@linutronix.de

Thomas Gleixner 2 years ago f57ebb92 aebbfe07

+46 -15

1 changed file

expand all

lib

debugobjects.c

+46 -15

lib/debugobjects.c

··· 149 149 150 150 static bool pool_move_batch(struct obj_pool *dst, struct obj_pool *src) 151 151 { 152 - if (dst->cnt + ODEBUG_BATCH_SIZE > dst->max_cnt || !src->cnt) 152 + struct hlist_node *last, *next_batch, *first_batch; 153 + struct debug_obj *obj; 154 + 155 + if (dst->cnt >= dst->max_cnt || !src->cnt) 153 156 return false; 154 157 155 - for (int i = 0; i < ODEBUG_BATCH_SIZE && src->cnt; i++) { 156 - struct hlist_node *node = src->objects.first; 158 + first_batch = src->objects.first; 159 + obj = hlist_entry(first_batch, typeof(*obj), node); 160 + last = obj->batch_last; 161 + next_batch = last->next; 157 162 158 - WRITE_ONCE(src->cnt, src->cnt - 1); 159 - WRITE_ONCE(dst->cnt, dst->cnt + 1); 163 + /* Move the next batch to the front of the source pool */ 164 + src->objects.first = next_batch; 165 + if (next_batch) 166 + next_batch->pprev = &src->objects.first; 160 167 161 - hlist_del(node); 162 - hlist_add_head(node, &dst->objects); 163 - } 168 + /* Add the extracted batch to the destination pool */ 169 + last->next = dst->objects.first; 170 + if (last->next) 171 + last->next->pprev = &last->next; 172 + first_batch->pprev = &dst->objects.first; 173 + dst->objects.first = first_batch; 174 + 175 + WRITE_ONCE(src->cnt, src->cnt - ODEBUG_BATCH_SIZE); 176 + WRITE_ONCE(dst->cnt, dst->cnt + ODEBUG_BATCH_SIZE); 164 177 return true; 165 178 } 166 179 ··· 195 182 196 183 static bool pool_pop_batch(struct hlist_head *head, struct obj_pool *src) 197 184 { 185 + struct hlist_node *last, *next; 186 + struct debug_obj *obj; 187 + 198 188 if (!src->cnt) 199 189 return false; 200 190 201 - for (int i = 0; src->cnt && i < ODEBUG_BATCH_SIZE; i++) { 202 - struct hlist_node *node = src->objects.first; 191 + /* Move the complete list to the head */ 192 + hlist_move_list(&src->objects, head); 203 193 204 - WRITE_ONCE(src->cnt, src->cnt - 1); 205 - hlist_del(node); 206 - hlist_add_head(node, head); 207 - } 194 + obj = hlist_entry(head->first, typeof(*obj), node); 195 + last = obj->batch_last; 196 + next = last->next; 197 + /* Disconnect the batch from the list */ 198 + last->next = NULL; 199 + 200 + /* Move the node after last back to the source pool. */ 201 + src->objects.first = next; 202 + if (next) 203 + next->pprev = &src->objects.first; 204 + 205 + WRITE_ONCE(src->cnt, src->cnt - ODEBUG_BATCH_SIZE); 208 206 return true; 209 207 } 210 208 ··· 250 226 if (!pool_move_batch(pcp, &pool_global)) 251 227 return NULL; 252 228 } 253 - obj_pool_used += pcp->cnt; 229 + obj_pool_used += ODEBUG_BATCH_SIZE; 254 230 255 231 if (obj_pool_used > obj_pool_max_used) 256 232 obj_pool_max_used = obj_pool_used; ··· 263 239 static void pcpu_free(struct debug_obj *obj) 264 240 { 265 241 struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu); 242 + struct debug_obj *first; 266 243 267 244 lockdep_assert_irqs_disabled(); 268 245 246 + if (!(pcp->cnt % ODEBUG_BATCH_SIZE)) { 247 + obj->batch_last = &obj->node; 248 + } else { 249 + first = hlist_entry(pcp->objects.first, typeof(*first), node); 250 + obj->batch_last = first->batch_last; 251 + } 269 252 hlist_add_head(&obj->node, &pcp->objects); 270 253 pcp->cnt++; 271 254

Configure Feed

Configure Feed