Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'stop-machine.2025.07.23a' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu
Pull stop-machine documentation updates from Paul McKenney:
- Improve kernel-doc function-header comments
- Document preemption and stop_machine() mutual exclusion (Joel
Fernandes)
* tag 'stop-machine.2025.07.23a' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu:
smp: Document preemption and stop_machine() mutual exclusion
stop_machine: Improve kernel-doc function-header comments
+58
-32
+41
-23
include/linux/stop_machine.h
+41
-23
include/linux/stop_machine.h
···
88
88
#endif /* CONFIG_SMP */
89
89
90
90
/*
91
-
* stop_machine "Bogolock": stop the entire machine, disable
92
-
* interrupts. This is a very heavy lock, which is equivalent to
93
-
* grabbing every spinlock (and more). So the "read" side to such a
94
-
* lock is anything which disables preemption.
91
+
* stop_machine "Bogolock": stop the entire machine, disable interrupts.
92
+
* This is a very heavy lock, which is equivalent to grabbing every raw
93
+
* spinlock (and more). So the "read" side to such a lock is anything
94
+
* which disables preemption.
95
95
*/
96
96
#if defined(CONFIG_SMP) || defined(CONFIG_HOTPLUG_CPU)
97
97
98
98
/**
99
99
* stop_machine: freeze the machine on all CPUs and run this function
100
100
* @fn: the function to run
101
-
* @data: the data ptr for the @fn()
102
-
* @cpus: the cpus to run the @fn() on (NULL = any online cpu)
101
+
* @data: the data ptr to pass to @fn()
102
+
* @cpus: the cpus to run @fn() on (NULL = run on each online CPU)
103
103
*
104
-
* Description: This causes a thread to be scheduled on every cpu,
105
-
* each of which disables interrupts. The result is that no one is
106
-
* holding a spinlock or inside any other preempt-disabled region when
107
-
* @fn() runs.
104
+
* Description: This causes a thread to be scheduled on every CPU, which
105
+
* will run with interrupts disabled. Each CPU specified by @cpus will
106
+
* run @fn. While @fn is executing, there will no other CPUs holding
107
+
* a raw spinlock or running within any other type of preempt-disabled
108
+
* region of code.
108
109
*
109
-
* This can be thought of as a very heavy write lock, equivalent to
110
-
* grabbing every spinlock in the kernel.
110
+
* When @cpus specifies only a single CPU, this can be thought of as
111
+
* a reader-writer lock where readers disable preemption (for example,
112
+
* by holding a raw spinlock) and where the insanely heavy writers run
113
+
* @fn while also preventing any other CPU from doing any useful work.
114
+
* These writers can also be thought of as having implicitly grabbed every
115
+
* raw spinlock in the kernel.
111
116
*
112
-
* Protects against CPU hotplug.
117
+
* When @fn is a no-op, this can be thought of as an RCU implementation
118
+
* where readers again disable preemption and writers use stop_machine()
119
+
* in place of synchronize_rcu(), albeit with orders of magnitude more
120
+
* disruption than even that of synchronize_rcu_expedited().
121
+
*
122
+
* Although only one stop_machine() operation can proceed at a time,
123
+
* the possibility of blocking in cpus_read_lock() means that the caller
124
+
* cannot usefully rely on this serialization.
125
+
*
126
+
* Return: 0 if all invocations of @fn return zero. Otherwise, the
127
+
* value returned by an arbitrarily chosen member of the set of calls to
128
+
* @fn that returned non-zero.
113
129
*/
114
130
int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus);
115
131
116
132
/**
117
133
* stop_machine_cpuslocked: freeze the machine on all CPUs and run this function
118
134
* @fn: the function to run
119
-
* @data: the data ptr for the @fn()
120
-
* @cpus: the cpus to run the @fn() on (NULL = any online cpu)
135
+
* @data: the data ptr to pass to @fn()
136
+
* @cpus: the cpus to run @fn() on (NULL = run on each online CPU)
121
137
*
122
-
* Same as above. Must be called from with in a cpus_read_lock() protected
123
-
* region. Avoids nested calls to cpus_read_lock().
138
+
* Same as above. Avoids nested calls to cpus_read_lock().
139
+
*
140
+
* Context: Must be called from within a cpus_read_lock() protected region.
124
141
*/
125
142
int stop_machine_cpuslocked(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus);
126
143
127
144
/**
128
145
* stop_core_cpuslocked: - stop all threads on just one core
129
146
* @cpu: any cpu in the targeted core
130
-
* @fn: the function to run
131
-
* @data: the data ptr for @fn()
147
+
* @fn: the function to run on each CPU in the core containing @cpu
148
+
* @data: the data ptr to pass to @fn()
132
149
*
133
-
* Same as above, but instead of every CPU, only the logical CPUs of a
134
-
* single core are affected.
150
+
* Same as above, but instead of every CPU, only the logical CPUs of the
151
+
* single core containing @cpu are affected.
135
152
*
136
153
* Context: Must be called from within a cpus_read_lock() protected region.
137
154
*
138
-
* Return: 0 if all executions of @fn returned 0, any non zero return
139
-
* value if any returned non zero.
155
+
* Return: 0 if all invocations of @fn return zero. Otherwise, the
156
+
* value returned by an arbitrarily chosen member of the set of calls to
157
+
* @fn that returned non-zero.
140
158
*/
141
159
int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data);
142
160
+17
-9
kernel/smp.c
+17
-9
kernel/smp.c
···
86
86
int smpcfd_dying_cpu(unsigned int cpu)
87
87
{
88
88
/*
89
-
* The IPIs for the smp-call-function callbacks queued by other
90
-
* CPUs might arrive late, either due to hardware latencies or
91
-
* because this CPU disabled interrupts (inside stop-machine)
92
-
* before the IPIs were sent. So flush out any pending callbacks
93
-
* explicitly (without waiting for the IPIs to arrive), to
94
-
* ensure that the outgoing CPU doesn't go offline with work
95
-
* still pending.
89
+
* The IPIs for the smp-call-function callbacks queued by other CPUs
90
+
* might arrive late, either due to hardware latencies or because this
91
+
* CPU disabled interrupts (inside stop-machine) before the IPIs were
92
+
* sent. So flush out any pending callbacks explicitly (without waiting
93
+
* for the IPIs to arrive), to ensure that the outgoing CPU doesn't go
94
+
* offline with work still pending.
95
+
*
96
+
* This runs with interrupts disabled inside the stopper task invoked by
97
+
* stop_machine(), ensuring mutually exclusive CPU offlining and IPI flush.
96
98
*/
97
99
__flush_smp_call_function_queue(false);
98
100
irq_work_run();
···
420
418
*/
421
419
static int generic_exec_single(int cpu, call_single_data_t *csd)
422
420
{
421
+
/*
422
+
* Preemption already disabled here so stopper cannot run on this CPU,
423
+
* ensuring mutually exclusive CPU offlining and last IPI flush.
424
+
*/
423
425
if (cpu == smp_processor_id()) {
424
426
smp_call_func_t func = csd->func;
425
427
void *info = csd->info;
···
644
638
int err;
645
639
646
640
/*
647
-
* prevent preemption and reschedule on another processor,
648
-
* as well as CPU removal
641
+
* Prevent preemption and reschedule on another CPU, as well as CPU
642
+
* removal. This prevents stopper from running on this CPU, thus
643
+
* providing mutual exclusion of the below cpu_online() check and
644
+
* IPI sending ensuring IPI are not missed by CPU going offline.
649
645
*/
650
646
this_cpu = get_cpu();
651
647