tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

tracing: Convert the per CPU "disabled" counter to local from atomic

The per CPU "disabled" counter is used for the latency tracers and stack
tracers to make sure that their accounting isn't messed up by an NMI or
interrupt coming in and affecting the same CPU data. But the counter is an
atomic_t type. As it only needs to synchronize against the current CPU,
switch it over to local_t type.

Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Link: https://lore.kernel.org/20250505212236.394925376@goodmis.org
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>

Steven Rostedt 90633c34 cf64792f

+25 -25
+1 -1
kernel/trace/trace.h
··· 183 183 * the trace, etc.) 184 184 */ 185 185 struct trace_array_cpu { 186 - atomic_t disabled; 186 + local_t disabled; 187 187 void *buffer_page; /* ring buffer spare */ 188 188 189 189 unsigned long entries;
+4 -4
kernel/trace/trace_functions.c
··· 291 291 parent_ip = function_get_true_parent_ip(parent_ip, fregs); 292 292 cpu = raw_smp_processor_id(); 293 293 data = per_cpu_ptr(tr->array_buffer.data, cpu); 294 - disabled = atomic_inc_return(&data->disabled); 294 + disabled = local_inc_return(&data->disabled); 295 295 296 296 if (likely(disabled == 1)) { 297 297 trace_ctx = tracing_gen_ctx_flags(flags); ··· 303 303 __trace_stack(tr, trace_ctx, skip); 304 304 } 305 305 306 - atomic_dec(&data->disabled); 306 + local_dec(&data->disabled); 307 307 local_irq_restore(flags); 308 308 } 309 309 ··· 402 402 parent_ip = function_get_true_parent_ip(parent_ip, fregs); 403 403 cpu = raw_smp_processor_id(); 404 404 data = per_cpu_ptr(tr->array_buffer.data, cpu); 405 - disabled = atomic_inc_return(&data->disabled); 405 + disabled = local_inc_return(&data->disabled); 406 406 407 407 if (likely(disabled == 1)) { 408 408 last_info = per_cpu_ptr(tr->last_func_repeats, cpu); ··· 417 417 } 418 418 419 419 out: 420 - atomic_dec(&data->disabled); 420 + local_dec(&data->disabled); 421 421 local_irq_restore(flags); 422 422 } 423 423
+11 -11
kernel/trace/trace_irqsoff.c
··· 123 123 return 0; 124 124 125 125 *data = per_cpu_ptr(tr->array_buffer.data, cpu); 126 - disabled = atomic_inc_return(&(*data)->disabled); 126 + disabled = local_inc_return(&(*data)->disabled); 127 127 128 128 if (likely(disabled == 1)) 129 129 return 1; 130 130 131 - atomic_dec(&(*data)->disabled); 131 + local_dec(&(*data)->disabled); 132 132 133 133 return 0; 134 134 } ··· 152 152 153 153 trace_function(tr, ip, parent_ip, trace_ctx, fregs); 154 154 155 - atomic_dec(&data->disabled); 155 + local_dec(&data->disabled); 156 156 } 157 157 #endif /* CONFIG_FUNCTION_TRACER */ 158 158 ··· 209 209 210 210 trace_ctx = tracing_gen_ctx_flags(flags); 211 211 ret = __trace_graph_entry(tr, trace, trace_ctx); 212 - atomic_dec(&data->disabled); 212 + local_dec(&data->disabled); 213 213 214 214 return ret; 215 215 } ··· 238 238 239 239 trace_ctx = tracing_gen_ctx_flags(flags); 240 240 __trace_graph_return(tr, trace, trace_ctx, *calltime, rettime); 241 - atomic_dec(&data->disabled); 241 + local_dec(&data->disabled); 242 242 } 243 243 244 244 static struct fgraph_ops fgraph_ops = { ··· 408 408 409 409 data = per_cpu_ptr(tr->array_buffer.data, cpu); 410 410 411 - if (unlikely(!data) || atomic_read(&data->disabled)) 411 + if (unlikely(!data) || local_read(&data->disabled)) 412 412 return; 413 413 414 - atomic_inc(&data->disabled); 414 + local_inc(&data->disabled); 415 415 416 416 data->critical_sequence = max_sequence; 417 417 data->preempt_timestamp = ftrace_now(cpu); ··· 421 421 422 422 per_cpu(tracing_cpu, cpu) = 1; 423 423 424 - atomic_dec(&data->disabled); 424 + local_dec(&data->disabled); 425 425 } 426 426 427 427 static nokprobe_inline void ··· 445 445 data = per_cpu_ptr(tr->array_buffer.data, cpu); 446 446 447 447 if (unlikely(!data) || 448 - !data->critical_start || atomic_read(&data->disabled)) 448 + !data->critical_start || local_read(&data->disabled)) 449 449 return; 450 450 451 - atomic_inc(&data->disabled); 451 + local_inc(&data->disabled); 452 452 453 453 trace_ctx = tracing_gen_ctx(); 454 454 __trace_function(tr, ip, parent_ip, trace_ctx); 455 455 check_critical_timing(tr, data, parent_ip ? : ip, cpu); 456 456 data->critical_start = 0; 457 - atomic_dec(&data->disabled); 457 + local_dec(&data->disabled); 458 458 } 459 459 460 460 /* start and stop critical timings used to for stoppage (in idle) */
+9 -9
kernel/trace/trace_sched_wakeup.c
··· 83 83 goto out_enable; 84 84 85 85 *data = per_cpu_ptr(tr->array_buffer.data, cpu); 86 - disabled = atomic_inc_return(&(*data)->disabled); 86 + disabled = local_inc_return(&(*data)->disabled); 87 87 if (unlikely(disabled != 1)) 88 88 goto out; 89 89 90 90 return 1; 91 91 92 92 out: 93 - atomic_dec(&(*data)->disabled); 93 + local_dec(&(*data)->disabled); 94 94 95 95 out_enable: 96 96 preempt_enable_notrace(); ··· 144 144 *calltime = trace_clock_local(); 145 145 146 146 ret = __trace_graph_entry(tr, trace, trace_ctx); 147 - atomic_dec(&data->disabled); 147 + local_dec(&data->disabled); 148 148 preempt_enable_notrace(); 149 149 150 150 return ret; ··· 173 173 return; 174 174 175 175 __trace_graph_return(tr, trace, trace_ctx, *calltime, rettime); 176 - atomic_dec(&data->disabled); 176 + local_dec(&data->disabled); 177 177 178 178 preempt_enable_notrace(); 179 179 return; ··· 243 243 trace_function(tr, ip, parent_ip, trace_ctx, fregs); 244 244 local_irq_restore(flags); 245 245 246 - atomic_dec(&data->disabled); 246 + local_dec(&data->disabled); 247 247 preempt_enable_notrace(); 248 248 } 249 249 ··· 471 471 472 472 /* disable local data, not wakeup_cpu data */ 473 473 cpu = raw_smp_processor_id(); 474 - disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); 474 + disabled = local_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); 475 475 if (likely(disabled != 1)) 476 476 goto out; 477 477 ··· 508 508 arch_spin_unlock(&wakeup_lock); 509 509 local_irq_restore(flags); 510 510 out: 511 - atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); 511 + local_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); 512 512 } 513 513 514 514 static void __wakeup_reset(struct trace_array *tr) ··· 563 563 (!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio))) 564 564 return; 565 565 566 - disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); 566 + disabled = local_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); 567 567 if (unlikely(disabled != 1)) 568 568 goto out; 569 569 ··· 610 610 out_locked: 611 611 arch_spin_unlock(&wakeup_lock); 612 612 out: 613 - atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); 613 + local_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); 614 614 } 615 615 616 616 static void start_wakeup_tracer(struct trace_array *tr)