Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'x86-core-2024-01-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 core updates from Ingo Molnar:
- Add comments about the magic behind the shadow STI
before MWAIT in __sti_mwait().
- Fix possible unintended timer delays caused by a race
in mwait_idle_with_hints().
* tag 'x86-core-2024-01-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
x86: Fix CPUIDLE_FLAG_IRQ_ENABLE leaking timer reprogram
x86: Add a comment about the "magic" behind shadow sti before mwait
+25
-14
+18
-2
arch/x86/include/asm/mwait.h
+18
-2
arch/x86/include/asm/mwait.h
···
87
87
:: "a" (eax), "b" (ebx), "c" (ecx));
88
88
}
89
89
90
+
/*
91
+
* Re-enable interrupts right upon calling mwait in such a way that
92
+
* no interrupt can fire _before_ the execution of mwait, ie: no
93
+
* instruction must be placed between "sti" and "mwait".
94
+
*
95
+
* This is necessary because if an interrupt queues a timer before
96
+
* executing mwait, it would otherwise go unnoticed and the next tick
97
+
* would not be reprogrammed accordingly before mwait ever wakes up.
98
+
*/
90
99
static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx)
91
100
{
92
101
mds_idle_clear_cpu_buffers();
···
124
115
}
125
116
126
117
__monitor((void *)¤t_thread_info()->flags, 0, 0);
127
-
if (!need_resched())
128
-
__mwait(eax, ecx);
118
+
119
+
if (!need_resched()) {
120
+
if (ecx & 1) {
121
+
__mwait(eax, ecx);
122
+
} else {
123
+
__sti_mwait(eax, ecx);
124
+
raw_local_irq_disable();
125
+
}
126
+
}
129
127
}
130
128
current_clr_polling();
131
129
}
+7
-12
drivers/idle/intel_idle.c
+7
-12
drivers/idle/intel_idle.c
···
131
131
#define MWAIT2flg(eax) ((eax & 0xFF) << 24)
132
132
133
133
static __always_inline int __intel_idle(struct cpuidle_device *dev,
134
-
struct cpuidle_driver *drv, int index)
134
+
struct cpuidle_driver *drv,
135
+
int index, bool irqoff)
135
136
{
136
137
struct cpuidle_state *state = &drv->states[index];
137
138
unsigned long eax = flg2MWAIT(state->flags);
138
-
unsigned long ecx = 1; /* break on interrupt flag */
139
+
unsigned long ecx = 1*irqoff; /* break on interrupt flag */
139
140
140
141
mwait_idle_with_hints(eax, ecx);
141
142
···
160
159
static __cpuidle int intel_idle(struct cpuidle_device *dev,
161
160
struct cpuidle_driver *drv, int index)
162
161
{
163
-
return __intel_idle(dev, drv, index);
162
+
return __intel_idle(dev, drv, index, true);
164
163
}
165
164
166
165
static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
167
166
struct cpuidle_driver *drv, int index)
168
167
{
169
-
int ret;
170
-
171
-
raw_local_irq_enable();
172
-
ret = __intel_idle(dev, drv, index);
173
-
raw_local_irq_disable();
174
-
175
-
return ret;
168
+
return __intel_idle(dev, drv, index, false);
176
169
}
177
170
178
171
static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
···
179
184
if (smt_active)
180
185
__update_spec_ctrl(0);
181
186
182
-
ret = __intel_idle(dev, drv, index);
187
+
ret = __intel_idle(dev, drv, index, true);
183
188
184
189
if (smt_active)
185
190
__update_spec_ctrl(spec_ctrl);
···
191
196
struct cpuidle_driver *drv, int index)
192
197
{
193
198
fpu_idle_fpregs();
194
-
return __intel_idle(dev, drv, index);
199
+
return __intel_idle(dev, drv, index, true);
195
200
}
196
201
197
202
/**