Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'locking-urgent-2020-11-29' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking fixes from Thomas Gleixner:
"Two more places which invoke tracing from RCU disabled regions in the
idle path.
Similar to the entry path the low level idle functions have to be
non-instrumentable"
* tag 'locking-urgent-2020-11-29' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
intel_idle: Fix intel_idle() vs tracing
sched/idle: Fix arch_cpu_idle() vs tracing
+84
-55
+1
-1
arch/alpha/kernel/process.c
+1
-1
arch/alpha/kernel/process.c
+1
-1
arch/arm/kernel/process.c
+1
-1
arch/arm/kernel/process.c
+1
-1
arch/arm64/kernel/process.c
+1
-1
arch/arm64/kernel/process.c
+1
-1
arch/csky/kernel/process.c
+1
-1
arch/csky/kernel/process.c
+1
-1
arch/h8300/kernel/process.c
+1
-1
arch/h8300/kernel/process.c
+1
-1
arch/hexagon/kernel/process.c
+1
-1
arch/hexagon/kernel/process.c
+1
-1
arch/ia64/kernel/process.c
+1
-1
arch/ia64/kernel/process.c
+1
-1
arch/microblaze/kernel/process.c
+1
-1
arch/microblaze/kernel/process.c
+6
-6
arch/mips/kernel/idle.c
+6
-6
arch/mips/kernel/idle.c
···
33
33
{
34
34
unsigned long cfg = read_c0_conf();
35
35
write_c0_conf(cfg | R30XX_CONF_HALT);
36
-
local_irq_enable();
36
+
raw_local_irq_enable();
37
37
}
38
38
39
39
static void __cpuidle r39xx_wait(void)
40
40
{
41
41
if (!need_resched())
42
42
write_c0_conf(read_c0_conf() | TX39_CONF_HALT);
43
-
local_irq_enable();
43
+
raw_local_irq_enable();
44
44
}
45
45
46
46
void __cpuidle r4k_wait(void)
47
47
{
48
-
local_irq_enable();
48
+
raw_local_irq_enable();
49
49
__r4k_wait();
50
50
}
51
51
···
64
64
" .set arch=r4000 \n"
65
65
" wait \n"
66
66
" .set pop \n");
67
-
local_irq_enable();
67
+
raw_local_irq_enable();
68
68
}
69
69
70
70
/*
···
84
84
" wait \n"
85
85
" mtc0 $1, $12 # stalls until W stage \n"
86
86
" .set pop \n");
87
-
local_irq_enable();
87
+
raw_local_irq_enable();
88
88
}
89
89
90
90
/*
···
257
257
if (cpu_wait)
258
258
cpu_wait();
259
259
else
260
-
local_irq_enable();
260
+
raw_local_irq_enable();
261
261
}
262
262
263
263
#ifdef CONFIG_CPU_IDLE
+1
-1
arch/nios2/kernel/process.c
+1
-1
arch/nios2/kernel/process.c
+1
-1
arch/openrisc/kernel/process.c
+1
-1
arch/openrisc/kernel/process.c
+1
-1
arch/parisc/kernel/process.c
+1
-1
arch/parisc/kernel/process.c
+2
-2
arch/powerpc/kernel/idle.c
+2
-2
arch/powerpc/kernel/idle.c
+1
-1
arch/riscv/kernel/process.c
+1
-1
arch/riscv/kernel/process.c
+3
-3
arch/s390/kernel/idle.c
+3
-3
arch/s390/kernel/idle.c
···
33
33
PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
34
34
clear_cpu_flag(CIF_NOHZ_DELAY);
35
35
36
-
local_irq_save(flags);
36
+
raw_local_irq_save(flags);
37
37
/* Call the assembler magic in entry.S */
38
38
psw_idle(idle, psw_mask);
39
-
local_irq_restore(flags);
39
+
raw_local_irq_restore(flags);
40
40
41
41
/* Account time spent with enabled wait psw loaded as idle time. */
42
42
raw_write_seqcount_begin(&idle->seqcount);
···
123
123
void arch_cpu_idle(void)
124
124
{
125
125
enabled_wait();
126
-
local_irq_enable();
126
+
raw_local_irq_enable();
127
127
}
128
128
129
129
void arch_cpu_idle_exit(void)
+1
-1
arch/sh/kernel/idle.c
+1
-1
arch/sh/kernel/idle.c
+2
-2
arch/sparc/kernel/leon_pmc.c
+2
-2
arch/sparc/kernel/leon_pmc.c
···
50
50
register unsigned int address = (unsigned int)leon3_irqctrl_regs;
51
51
52
52
/* Interrupts need to be enabled to not hang the CPU */
53
-
local_irq_enable();
53
+
raw_local_irq_enable();
54
54
55
55
__asm__ __volatile__ (
56
56
"wr %%g0, %%asr19\n"
···
66
66
static void pmc_leon_idle(void)
67
67
{
68
68
/* Interrupts need to be enabled to not hang the CPU */
69
-
local_irq_enable();
69
+
raw_local_irq_enable();
70
70
71
71
/* For systems without power-down, this will be no-op */
72
72
__asm__ __volatile__ ("wr %g0, %asr19\n\t");
+1
-1
arch/sparc/kernel/process_32.c
+1
-1
arch/sparc/kernel/process_32.c
+2
-2
arch/sparc/kernel/process_64.c
+2
-2
arch/sparc/kernel/process_64.c
···
62
62
{
63
63
if (tlb_type != hypervisor) {
64
64
touch_nmi_watchdog();
65
-
local_irq_enable();
65
+
raw_local_irq_enable();
66
66
} else {
67
67
unsigned long pstate;
68
68
69
-
local_irq_enable();
69
+
raw_local_irq_enable();
70
70
71
71
/* The sun4v sleeping code requires that we have PSTATE.IE cleared over
72
72
* the cpu sleep hypervisor call.
+1
-1
arch/um/kernel/process.c
+1
-1
arch/um/kernel/process.c
-2
arch/x86/include/asm/mwait.h
-2
arch/x86/include/asm/mwait.h
+7
-5
arch/x86/kernel/process.c
+7
-5
arch/x86/kernel/process.c
···
685
685
*/
686
686
void __cpuidle default_idle(void)
687
687
{
688
-
safe_halt();
688
+
raw_safe_halt();
689
689
}
690
690
#if defined(CONFIG_APM_MODULE) || defined(CONFIG_HALTPOLL_CPUIDLE_MODULE)
691
691
EXPORT_SYMBOL(default_idle);
···
736
736
/*
737
737
* AMD Erratum 400 aware idle routine. We handle it the same way as C3 power
738
738
* states (local apic timer and TSC stop).
739
+
*
740
+
* XXX this function is completely buggered vs RCU and tracing.
739
741
*/
740
742
static void amd_e400_idle(void)
741
743
{
···
759
757
* The switch back from broadcast mode needs to be called with
760
758
* interrupts disabled.
761
759
*/
762
-
local_irq_disable();
760
+
raw_local_irq_disable();
763
761
tick_broadcast_exit();
764
-
local_irq_enable();
762
+
raw_local_irq_enable();
765
763
}
766
764
767
765
/*
···
803
801
if (!need_resched())
804
802
__sti_mwait(0, 0);
805
803
else
806
-
local_irq_enable();
804
+
raw_local_irq_enable();
807
805
} else {
808
-
local_irq_enable();
806
+
raw_local_irq_enable();
809
807
}
810
808
__current_clr_polling();
811
809
}
+20
-17
drivers/idle/intel_idle.c
+20
-17
drivers/idle/intel_idle.c
···
126
126
struct cpuidle_state *state = &drv->states[index];
127
127
unsigned long eax = flg2MWAIT(state->flags);
128
128
unsigned long ecx = 1; /* break on interrupt flag */
129
-
bool tick;
130
-
131
-
if (!static_cpu_has(X86_FEATURE_ARAT)) {
132
-
/*
133
-
* Switch over to one-shot tick broadcast if the target C-state
134
-
* is deeper than C1.
135
-
*/
136
-
if ((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) {
137
-
tick = true;
138
-
tick_broadcast_enter();
139
-
} else {
140
-
tick = false;
141
-
}
142
-
}
143
129
144
130
mwait_idle_with_hints(eax, ecx);
145
-
146
-
if (!static_cpu_has(X86_FEATURE_ARAT) && tick)
147
-
tick_broadcast_exit();
148
131
149
132
return index;
150
133
}
···
1210
1227
return false;
1211
1228
}
1212
1229
1230
+
static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1231
+
{
1232
+
unsigned long eax = flg2MWAIT(state->flags);
1233
+
1234
+
if (boot_cpu_has(X86_FEATURE_ARAT))
1235
+
return false;
1236
+
1237
+
/*
1238
+
* Switch over to one-shot tick broadcast if the target C-state
1239
+
* is deeper than C1.
1240
+
*/
1241
+
return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1242
+
}
1243
+
1213
1244
static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1214
1245
{
1215
1246
int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
···
1265
1268
1266
1269
if (disabled_states_mask & BIT(cstate))
1267
1270
state->flags |= CPUIDLE_FLAG_OFF;
1271
+
1272
+
if (intel_idle_state_needs_timer_stop(state))
1273
+
state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1268
1274
1269
1275
state->enter = intel_idle;
1270
1276
state->enter_s2idle = intel_idle_s2idle;
···
1506
1506
intel_idle_off_by_default(mwait_hint) &&
1507
1507
!(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
1508
1508
drv->states[drv->state_count].flags |= CPUIDLE_FLAG_OFF;
1509
+
1510
+
if (intel_idle_state_needs_timer_stop(&drv->states[drv->state_count]))
1511
+
drv->states[drv->state_count].flags |= CPUIDLE_FLAG_TIMER_STOP;
1509
1512
1510
1513
drv->state_count++;
1511
1514
}
+27
-1
kernel/sched/idle.c
+27
-1
kernel/sched/idle.c
···
78
78
void __weak arch_cpu_idle(void)
79
79
{
80
80
cpu_idle_force_poll = 1;
81
-
local_irq_enable();
81
+
raw_local_irq_enable();
82
82
}
83
83
84
84
/**
···
94
94
95
95
trace_cpu_idle(1, smp_processor_id());
96
96
stop_critical_timings();
97
+
98
+
/*
99
+
* arch_cpu_idle() is supposed to enable IRQs, however
100
+
* we can't do that because of RCU and tracing.
101
+
*
102
+
* Trace IRQs enable here, then switch off RCU, and have
103
+
* arch_cpu_idle() use raw_local_irq_enable(). Note that
104
+
* rcu_idle_enter() relies on lockdep IRQ state, so switch that
105
+
* last -- this is very similar to the entry code.
106
+
*/
107
+
trace_hardirqs_on_prepare();
108
+
lockdep_hardirqs_on_prepare(_THIS_IP_);
97
109
rcu_idle_enter();
110
+
lockdep_hardirqs_on(_THIS_IP_);
111
+
98
112
arch_cpu_idle();
113
+
114
+
/*
115
+
* OK, so IRQs are enabled here, but RCU needs them disabled to
116
+
* turn itself back on.. funny thing is that disabling IRQs
117
+
* will cause tracing, which needs RCU. Jump through hoops to
118
+
* make it 'work'.
119
+
*/
120
+
raw_local_irq_disable();
121
+
lockdep_hardirqs_off(_THIS_IP_);
99
122
rcu_idle_exit();
123
+
lockdep_hardirqs_on(_THIS_IP_);
124
+
raw_local_irq_enable();
125
+
100
126
start_critical_timings();
101
127
trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
102
128
}