tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

trace/pid_list: optimize pid_list->lock contention

When the system has many cores and task switching is frequent,
setting set_ftrace_pid can cause frequent pid_list->lock contention
and high system sys usage.

For example, in a 288-core VM environment, we observed 267 CPUs
experiencing contention on pid_list->lock, with stack traces showing:

#4 [ffffa6226fb4bc70] native_queued_spin_lock_slowpath at ffffffff99cd4b7e
#5 [ffffa6226fb4bc90] _raw_spin_lock_irqsave at ffffffff99cd3e36
#6 [ffffa6226fb4bca0] trace_pid_list_is_set at ffffffff99267554
#7 [ffffa6226fb4bcc0] trace_ignore_this_task at ffffffff9925c288
#8 [ffffa6226fb4bcd8] ftrace_filter_pid_sched_switch_probe at ffffffff99246efe
#9 [ffffa6226fb4bcf0] __schedule at ffffffff99ccd161

Replaces the existing spinlock with a seqlock to allow concurrent readers,
while maintaining write exclusivity.

Link: https://patch.msgid.link/20251113000252.1058144-1-leonylgao@gmail.com
Reviewed-by: Huang Cun <cunhuang@tencent.com>
Signed-off-by: Yongliang Gao <leonylgao@tencent.com>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>

authored by

Yongliang Gao and committed by
Steven Rostedt (Google) 97e047f4 e29aa918

+22 -9
+21 -9
kernel/trace/pid_list.c
··· 3 3 * Copyright (C) 2021 VMware Inc, Steven Rostedt <rostedt@goodmis.org> 4 4 */ 5 5 #include <linux/spinlock.h> 6 + #include <linux/seqlock.h> 6 7 #include <linux/irq_work.h> 7 8 #include <linux/slab.h> 8 9 #include "trace.h" ··· 127 126 { 128 127 union upper_chunk *upper_chunk; 129 128 union lower_chunk *lower_chunk; 130 - unsigned long flags; 129 + unsigned int seq; 131 130 unsigned int upper1; 132 131 unsigned int upper2; 133 132 unsigned int lower; ··· 139 138 if (pid_split(pid, &upper1, &upper2, &lower) < 0) 140 139 return false; 141 140 142 - raw_spin_lock_irqsave(&pid_list->lock, flags); 143 - upper_chunk = pid_list->upper[upper1]; 144 - if (upper_chunk) { 145 - lower_chunk = upper_chunk->data[upper2]; 146 - if (lower_chunk) 147 - ret = test_bit(lower, lower_chunk->data); 148 - } 149 - raw_spin_unlock_irqrestore(&pid_list->lock, flags); 141 + do { 142 + seq = read_seqcount_begin(&pid_list->seqcount); 143 + ret = false; 144 + upper_chunk = pid_list->upper[upper1]; 145 + if (upper_chunk) { 146 + lower_chunk = upper_chunk->data[upper2]; 147 + if (lower_chunk) 148 + ret = test_bit(lower, lower_chunk->data); 149 + } 150 + } while (read_seqcount_retry(&pid_list->seqcount, seq)); 150 151 151 152 return ret; 152 153 } ··· 181 178 return -EINVAL; 182 179 183 180 raw_spin_lock_irqsave(&pid_list->lock, flags); 181 + write_seqcount_begin(&pid_list->seqcount); 184 182 upper_chunk = pid_list->upper[upper1]; 185 183 if (!upper_chunk) { 186 184 upper_chunk = get_upper_chunk(pid_list); ··· 203 199 set_bit(lower, lower_chunk->data); 204 200 ret = 0; 205 201 out: 202 + write_seqcount_end(&pid_list->seqcount); 206 203 raw_spin_unlock_irqrestore(&pid_list->lock, flags); 207 204 return ret; 208 205 } ··· 235 230 return -EINVAL; 236 231 237 232 raw_spin_lock_irqsave(&pid_list->lock, flags); 233 + write_seqcount_begin(&pid_list->seqcount); 238 234 upper_chunk = pid_list->upper[upper1]; 239 235 if (!upper_chunk) 240 236 goto out; ··· 256 250 } 257 251 } 258 252 out: 253 + write_seqcount_end(&pid_list->seqcount); 259 254 raw_spin_unlock_irqrestore(&pid_list->lock, flags); 260 255 return 0; 261 256 } ··· 347 340 348 341 again: 349 342 raw_spin_lock(&pid_list->lock); 343 + write_seqcount_begin(&pid_list->seqcount); 350 344 upper_count = CHUNK_ALLOC - pid_list->free_upper_chunks; 351 345 lower_count = CHUNK_ALLOC - pid_list->free_lower_chunks; 346 + write_seqcount_end(&pid_list->seqcount); 352 347 raw_spin_unlock(&pid_list->lock); 353 348 354 349 if (upper_count <= 0 && lower_count <= 0) ··· 379 370 } 380 371 381 372 raw_spin_lock(&pid_list->lock); 373 + write_seqcount_begin(&pid_list->seqcount); 382 374 if (upper) { 383 375 *upper_next = pid_list->upper_list; 384 376 pid_list->upper_list = upper; ··· 390 380 pid_list->lower_list = lower; 391 381 pid_list->free_lower_chunks += lcnt; 392 382 } 383 + write_seqcount_end(&pid_list->seqcount); 393 384 raw_spin_unlock(&pid_list->lock); 394 385 395 386 /* ··· 430 419 init_irq_work(&pid_list->refill_irqwork, pid_list_refill_irq); 431 420 432 421 raw_spin_lock_init(&pid_list->lock); 422 + seqcount_raw_spinlock_init(&pid_list->seqcount, &pid_list->lock); 433 423 434 424 for (i = 0; i < CHUNK_ALLOC; i++) { 435 425 union upper_chunk *chunk;
+1
kernel/trace/pid_list.h
··· 76 76 }; 77 77 78 78 struct trace_pid_list { 79 + seqcount_raw_spinlock_t seqcount; 79 80 raw_spinlock_t lock; 80 81 struct irq_work refill_irqwork; 81 82 union upper_chunk *upper[UPPER1_SIZE]; // 1 or 2K in size