Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'locking-urgent-2020-08-30' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking fixes from Thomas Gleixner:
"A set of fixes for lockdep, tracing and RCU:
- Prevent recursion by using raw_cpu_* operations
- Fixup the interrupt state in the cpu idle code to be consistent
- Push rcu_idle_enter/exit() invocations deeper into the idle path so
that the lock operations are inside the RCU watching sections
- Move trace_cpu_idle() into generic code so it's called before RCU
goes idle.
- Handle raw_local_irq* vs. local_irq* operations correctly
- Move the tracepoints out from under the lockdep recursion handling
which turned out to be fragile and inconsistent"
* tag 'locking-urgent-2020-08-30' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
lockdep,trace: Expose tracepoints
lockdep: Only trace IRQ edges
mips: Implement arch_irqs_disabled()
arm64: Implement arch_irqs_disabled()
nds32: Implement arch_irqs_disabled()
locking/lockdep: Cleanup
x86/entry: Remove unused THUNKs
cpuidle: Move trace_cpu_idle() into generic code
cpuidle: Make CPUIDLE_FLAG_TLB_FLUSHED generic
sched,idle,rcu: Push rcu_idle deeper into the idle path
cpuidle: Fixup IRQ state
lockdep: Use raw_cpu_*() for per-cpu variables
+124
-123
-4
arch/arm/mach-omap2/pm34xx.c
-4
arch/arm/mach-omap2/pm34xx.c
+5
arch/arm64/include/asm/irqflags.h
+5
arch/arm64/include/asm/irqflags.h
-2
arch/arm64/kernel/process.c
-2
arch/arm64/kernel/process.c
···
123
123
* This should do all the clock switching and wait for interrupt
124
124
* tricks
125
125
*/
126
-
trace_cpu_idle_rcuidle(1, smp_processor_id());
127
126
cpu_do_idle();
128
127
local_irq_enable();
129
-
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
130
128
}
131
129
132
130
#ifdef CONFIG_HOTPLUG_CPU
+5
arch/mips/include/asm/irqflags.h
+5
arch/mips/include/asm/irqflags.h
+5
arch/nds32/include/asm/irqflags.h
+5
arch/nds32/include/asm/irqflags.h
+4
-7
arch/powerpc/include/asm/hw_irq.h
+4
-7
arch/powerpc/include/asm/hw_irq.h
···
200
200
#define powerpc_local_irq_pmu_save(flags) \
201
201
do { \
202
202
raw_local_irq_pmu_save(flags); \
203
-
trace_hardirqs_off(); \
203
+
if (!raw_irqs_disabled_flags(flags)) \
204
+
trace_hardirqs_off(); \
204
205
} while(0)
205
206
#define powerpc_local_irq_pmu_restore(flags) \
206
207
do { \
207
-
if (raw_irqs_disabled_flags(flags)) { \
208
-
raw_local_irq_pmu_restore(flags); \
209
-
trace_hardirqs_off(); \
210
-
} else { \
208
+
if (!raw_irqs_disabled_flags(flags)) \
211
209
trace_hardirqs_on(); \
212
-
raw_local_irq_pmu_restore(flags); \
213
-
} \
210
+
raw_local_irq_pmu_restore(flags); \
214
211
} while(0)
215
212
#else
216
213
#define powerpc_local_irq_pmu_save(flags) \
+1
-2
arch/s390/kernel/idle.c
+1
-2
arch/s390/kernel/idle.c
···
33
33
PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
34
34
clear_cpu_flag(CIF_NOHZ_DELAY);
35
35
36
-
trace_cpu_idle_rcuidle(1, smp_processor_id());
37
36
local_irq_save(flags);
38
37
/* Call the assembler magic in entry.S */
39
38
psw_idle(idle, psw_mask);
40
39
local_irq_restore(flags);
41
-
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
42
40
43
41
/* Account time spent with enabled wait psw loaded as idle time. */
42
+
/* XXX seqcount has tracepoints that require RCU */
44
43
write_seqcount_begin(&idle->seqcount);
45
44
idle_time = idle->clock_idle_exit - idle->clock_idle_enter;
46
45
idle->clock_idle_enter = idle->clock_idle_exit = 0ULL;
-5
arch/x86/entry/thunk_32.S
-5
arch/x86/entry/thunk_32.S
···
29
29
SYM_CODE_END(\name)
30
30
.endm
31
31
32
-
#ifdef CONFIG_TRACE_IRQFLAGS
33
-
THUNK trace_hardirqs_on_thunk,trace_hardirqs_on_caller,1
34
-
THUNK trace_hardirqs_off_thunk,trace_hardirqs_off_caller,1
35
-
#endif
36
-
37
32
#ifdef CONFIG_PREEMPTION
38
33
THUNK preempt_schedule_thunk, preempt_schedule
39
34
THUNK preempt_schedule_notrace_thunk, preempt_schedule_notrace
+1
arch/x86/include/asm/mmu.h
+1
arch/x86/include/asm/mmu.h
-4
arch/x86/kernel/process.c
-4
arch/x86/kernel/process.c
···
684
684
*/
685
685
void __cpuidle default_idle(void)
686
686
{
687
-
trace_cpu_idle_rcuidle(1, smp_processor_id());
688
687
safe_halt();
689
-
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
690
688
}
691
689
#if defined(CONFIG_APM_MODULE) || defined(CONFIG_HALTPOLL_CPUIDLE_MODULE)
692
690
EXPORT_SYMBOL(default_idle);
···
790
792
static __cpuidle void mwait_idle(void)
791
793
{
792
794
if (!current_set_polling_and_test()) {
793
-
trace_cpu_idle_rcuidle(1, smp_processor_id());
794
795
if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
795
796
mb(); /* quirk */
796
797
clflush((void *)¤t_thread_info()->flags);
···
801
804
__sti_mwait(0, 0);
802
805
else
803
806
local_irq_enable();
804
-
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
805
807
} else {
806
808
local_irq_enable();
807
809
}
+2
-11
arch/x86/mm/tlb.c
+2
-11
arch/x86/mm/tlb.c
···
555
555
this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen);
556
556
load_new_mm_cr3(next->pgd, new_asid, true);
557
557
558
-
/*
559
-
* NB: This gets called via leave_mm() in the idle path
560
-
* where RCU functions differently. Tracing normally
561
-
* uses RCU, so we need to use the _rcuidle variant.
562
-
*
563
-
* (There is no good reason for this. The idle code should
564
-
* be rearranged to call this before rcu_idle_enter().)
565
-
*/
566
-
trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
558
+
trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
567
559
} else {
568
560
/* The new ASID is already up to date. */
569
561
load_new_mm_cr3(next->pgd, new_asid, false);
570
562
571
-
/* See above wrt _rcuidle. */
572
-
trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, 0);
563
+
trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, 0);
573
564
}
574
565
575
566
/* Make sure we write CR3 before loaded_mm. */
+14
-5
drivers/cpuidle/cpuidle.c
+14
-5
drivers/cpuidle/cpuidle.c
···
22
22
#include <linux/module.h>
23
23
#include <linux/suspend.h>
24
24
#include <linux/tick.h>
25
+
#include <linux/mmu_context.h>
25
26
#include <trace/events/power.h>
26
27
27
28
#include "cpuidle.h"
···
146
145
* executing it contains RCU usage regarded as invalid in the idle
147
146
* context, so tell RCU about that.
148
147
*/
149
-
RCU_NONIDLE(tick_freeze());
148
+
tick_freeze();
150
149
/*
151
150
* The state used here cannot be a "coupled" one, because the "coupled"
152
151
* cpuidle mechanism enables interrupts and doing that with timekeeping
153
152
* suspended is generally unsafe.
154
153
*/
155
154
stop_critical_timings();
155
+
rcu_idle_enter();
156
156
drv->states[index].enter_s2idle(dev, drv, index);
157
-
WARN_ON(!irqs_disabled());
157
+
if (WARN_ON_ONCE(!irqs_disabled()))
158
+
local_irq_disable();
158
159
/*
159
160
* timekeeping_resume() that will be called by tick_unfreeze() for the
160
161
* first CPU executing it calls functions containing RCU read-side
161
162
* critical sections, so tell RCU about that.
162
163
*/
163
-
RCU_NONIDLE(tick_unfreeze());
164
+
rcu_idle_exit();
165
+
tick_unfreeze();
164
166
start_critical_timings();
165
167
166
168
time_end = ns_to_ktime(local_clock());
···
229
225
broadcast = false;
230
226
}
231
227
228
+
if (target_state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
229
+
leave_mm(dev->cpu);
230
+
232
231
/* Take note of the planned idle state. */
233
232
sched_idle_set_state(target_state);
234
233
235
-
trace_cpu_idle_rcuidle(index, dev->cpu);
234
+
trace_cpu_idle(index, dev->cpu);
236
235
time_start = ns_to_ktime(local_clock());
237
236
238
237
stop_critical_timings();
238
+
rcu_idle_enter();
239
239
entered_state = target_state->enter(dev, drv, index);
240
+
rcu_idle_exit();
240
241
start_critical_timings();
241
242
242
243
sched_clock_idle_wakeup_event();
243
244
time_end = ns_to_ktime(local_clock());
244
-
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
245
+
trace_cpu_idle(PWR_EVENT_EXIT, dev->cpu);
245
246
246
247
/* The cpu is no longer idle or about to enter idle. */
247
248
sched_idle_set_state(NULL);
-16
drivers/idle/intel_idle.c
-16
drivers/idle/intel_idle.c
···
90
90
#define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15)
91
91
92
92
/*
93
-
* Set this flag for states where the HW flushes the TLB for us
94
-
* and so we don't need cross-calls to keep it consistent.
95
-
* If this flag is set, SW flushes the TLB, so even if the
96
-
* HW doesn't do the flushing, this flag is safe to use.
97
-
*/
98
-
#define CPUIDLE_FLAG_TLB_FLUSHED BIT(16)
99
-
100
-
/*
101
93
* MWAIT takes an 8-bit "hint" in EAX "suggesting"
102
94
* the C-state (top nibble) and sub-state (bottom nibble)
103
95
* 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
···
123
131
unsigned long eax = flg2MWAIT(state->flags);
124
132
unsigned long ecx = 1; /* break on interrupt flag */
125
133
bool tick;
126
-
int cpu = smp_processor_id();
127
-
128
-
/*
129
-
* leave_mm() to avoid costly and often unnecessary wakeups
130
-
* for flushing the user TLB's associated with the active mm.
131
-
*/
132
-
if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
133
-
leave_mm(cpu);
134
134
135
135
if (!static_cpu_has(X86_FEATURE_ARAT)) {
136
136
/*
+7
-6
include/linux/cpuidle.h
+7
-6
include/linux/cpuidle.h
···
75
75
};
76
76
77
77
/* Idle State Flags */
78
-
#define CPUIDLE_FLAG_NONE (0x00)
79
-
#define CPUIDLE_FLAG_POLLING BIT(0) /* polling state */
80
-
#define CPUIDLE_FLAG_COUPLED BIT(1) /* state applies to multiple cpus */
81
-
#define CPUIDLE_FLAG_TIMER_STOP BIT(2) /* timer is stopped on this state */
82
-
#define CPUIDLE_FLAG_UNUSABLE BIT(3) /* avoid using this state */
83
-
#define CPUIDLE_FLAG_OFF BIT(4) /* disable this state by default */
78
+
#define CPUIDLE_FLAG_NONE (0x00)
79
+
#define CPUIDLE_FLAG_POLLING BIT(0) /* polling state */
80
+
#define CPUIDLE_FLAG_COUPLED BIT(1) /* state applies to multiple cpus */
81
+
#define CPUIDLE_FLAG_TIMER_STOP BIT(2) /* timer is stopped on this state */
82
+
#define CPUIDLE_FLAG_UNUSABLE BIT(3) /* avoid using this state */
83
+
#define CPUIDLE_FLAG_OFF BIT(4) /* disable this state by default */
84
+
#define CPUIDLE_FLAG_TLB_FLUSHED BIT(5) /* idle-state flushes TLBs */
84
85
85
86
struct cpuidle_device_kobj;
86
87
struct cpuidle_state_kobj;
+39
-34
include/linux/irqflags.h
+39
-34
include/linux/irqflags.h
···
49
49
DECLARE_PER_CPU(int, hardirqs_enabled);
50
50
DECLARE_PER_CPU(int, hardirq_context);
51
51
52
-
extern void trace_hardirqs_on_prepare(void);
53
-
extern void trace_hardirqs_off_finish(void);
54
-
extern void trace_hardirqs_on(void);
55
-
extern void trace_hardirqs_off(void);
56
-
# define lockdep_hardirq_context() (this_cpu_read(hardirq_context))
52
+
extern void trace_hardirqs_on_prepare(void);
53
+
extern void trace_hardirqs_off_finish(void);
54
+
extern void trace_hardirqs_on(void);
55
+
extern void trace_hardirqs_off(void);
56
+
57
+
# define lockdep_hardirq_context() (raw_cpu_read(hardirq_context))
57
58
# define lockdep_softirq_context(p) ((p)->softirq_context)
58
59
# define lockdep_hardirqs_enabled() (this_cpu_read(hardirqs_enabled))
59
60
# define lockdep_softirqs_enabled(p) ((p)->softirqs_enabled)
60
61
# define lockdep_hardirq_enter() \
61
62
do { \
62
-
if (this_cpu_inc_return(hardirq_context) == 1) \
63
+
if (__this_cpu_inc_return(hardirq_context) == 1)\
63
64
current->hardirq_threaded = 0; \
64
65
} while (0)
65
66
# define lockdep_hardirq_threaded() \
···
69
68
} while (0)
70
69
# define lockdep_hardirq_exit() \
71
70
do { \
72
-
this_cpu_dec(hardirq_context); \
71
+
__this_cpu_dec(hardirq_context); \
73
72
} while (0)
74
73
# define lockdep_softirq_enter() \
75
74
do { \
···
121
120
#else
122
121
# define trace_hardirqs_on_prepare() do { } while (0)
123
122
# define trace_hardirqs_off_finish() do { } while (0)
124
-
# define trace_hardirqs_on() do { } while (0)
125
-
# define trace_hardirqs_off() do { } while (0)
126
-
# define lockdep_hardirq_context() 0
127
-
# define lockdep_softirq_context(p) 0
128
-
# define lockdep_hardirqs_enabled() 0
129
-
# define lockdep_softirqs_enabled(p) 0
130
-
# define lockdep_hardirq_enter() do { } while (0)
131
-
# define lockdep_hardirq_threaded() do { } while (0)
132
-
# define lockdep_hardirq_exit() do { } while (0)
133
-
# define lockdep_softirq_enter() do { } while (0)
134
-
# define lockdep_softirq_exit() do { } while (0)
123
+
# define trace_hardirqs_on() do { } while (0)
124
+
# define trace_hardirqs_off() do { } while (0)
125
+
# define lockdep_hardirq_context() 0
126
+
# define lockdep_softirq_context(p) 0
127
+
# define lockdep_hardirqs_enabled() 0
128
+
# define lockdep_softirqs_enabled(p) 0
129
+
# define lockdep_hardirq_enter() do { } while (0)
130
+
# define lockdep_hardirq_threaded() do { } while (0)
131
+
# define lockdep_hardirq_exit() do { } while (0)
132
+
# define lockdep_softirq_enter() do { } while (0)
133
+
# define lockdep_softirq_exit() do { } while (0)
135
134
# define lockdep_hrtimer_enter(__hrtimer) false
136
135
# define lockdep_hrtimer_exit(__context) do { } while (0)
137
136
# define lockdep_posixtimer_enter() do { } while (0)
···
182
181
* if !TRACE_IRQFLAGS.
183
182
*/
184
183
#ifdef CONFIG_TRACE_IRQFLAGS
185
-
#define local_irq_enable() \
186
-
do { trace_hardirqs_on(); raw_local_irq_enable(); } while (0)
187
-
#define local_irq_disable() \
188
-
do { raw_local_irq_disable(); trace_hardirqs_off(); } while (0)
184
+
185
+
#define local_irq_enable() \
186
+
do { \
187
+
trace_hardirqs_on(); \
188
+
raw_local_irq_enable(); \
189
+
} while (0)
190
+
191
+
#define local_irq_disable() \
192
+
do { \
193
+
bool was_disabled = raw_irqs_disabled();\
194
+
raw_local_irq_disable(); \
195
+
if (!was_disabled) \
196
+
trace_hardirqs_off(); \
197
+
} while (0)
198
+
189
199
#define local_irq_save(flags) \
190
200
do { \
191
201
raw_local_irq_save(flags); \
192
-
trace_hardirqs_off(); \
202
+
if (!raw_irqs_disabled_flags(flags)) \
203
+
trace_hardirqs_off(); \
193
204
} while (0)
194
-
195
205
196
206
#define local_irq_restore(flags) \
197
207
do { \
198
-
if (raw_irqs_disabled_flags(flags)) { \
199
-
raw_local_irq_restore(flags); \
200
-
trace_hardirqs_off(); \
201
-
} else { \
208
+
if (!raw_irqs_disabled_flags(flags)) \
202
209
trace_hardirqs_on(); \
203
-
raw_local_irq_restore(flags); \
204
-
} \
210
+
raw_local_irq_restore(flags); \
205
211
} while (0)
206
212
207
213
#define safe_halt() \
···
222
214
223
215
#define local_irq_enable() do { raw_local_irq_enable(); } while (0)
224
216
#define local_irq_disable() do { raw_local_irq_disable(); } while (0)
225
-
#define local_irq_save(flags) \
226
-
do { \
227
-
raw_local_irq_save(flags); \
228
-
} while (0)
217
+
#define local_irq_save(flags) do { raw_local_irq_save(flags); } while (0)
229
218
#define local_irq_restore(flags) do { raw_local_irq_restore(flags); } while (0)
230
219
#define safe_halt() do { raw_safe_halt(); } while (0)
231
220
+13
-5
include/linux/lockdep.h
+13
-5
include/linux/lockdep.h
···
535
535
DECLARE_PER_CPU(int, hardirqs_enabled);
536
536
DECLARE_PER_CPU(int, hardirq_context);
537
537
538
+
/*
539
+
* The below lockdep_assert_*() macros use raw_cpu_read() to access the above
540
+
* per-cpu variables. This is required because this_cpu_read() will potentially
541
+
* call into preempt/irq-disable and that obviously isn't right. This is also
542
+
* correct because when IRQs are enabled, it doesn't matter if we accidentally
543
+
* read the value from our previous CPU.
544
+
*/
545
+
538
546
#define lockdep_assert_irqs_enabled() \
539
547
do { \
540
-
WARN_ON_ONCE(debug_locks && !this_cpu_read(hardirqs_enabled)); \
548
+
WARN_ON_ONCE(debug_locks && !raw_cpu_read(hardirqs_enabled)); \
541
549
} while (0)
542
550
543
551
#define lockdep_assert_irqs_disabled() \
544
552
do { \
545
-
WARN_ON_ONCE(debug_locks && this_cpu_read(hardirqs_enabled)); \
553
+
WARN_ON_ONCE(debug_locks && raw_cpu_read(hardirqs_enabled)); \
546
554
} while (0)
547
555
548
556
#define lockdep_assert_in_irq() \
549
557
do { \
550
-
WARN_ON_ONCE(debug_locks && !this_cpu_read(hardirq_context)); \
558
+
WARN_ON_ONCE(debug_locks && !raw_cpu_read(hardirq_context)); \
551
559
} while (0)
552
560
553
561
#define lockdep_assert_preemption_enabled() \
···
563
555
WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT) && \
564
556
debug_locks && \
565
557
(preempt_count() != 0 || \
566
-
!this_cpu_read(hardirqs_enabled))); \
558
+
!raw_cpu_read(hardirqs_enabled))); \
567
559
} while (0)
568
560
569
561
#define lockdep_assert_preemption_disabled() \
···
571
563
WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT) && \
572
564
debug_locks && \
573
565
(preempt_count() == 0 && \
574
-
this_cpu_read(hardirqs_enabled))); \
566
+
raw_cpu_read(hardirqs_enabled))); \
575
567
} while (0)
576
568
577
569
#else
+5
include/linux/mmu_context.h
+5
include/linux/mmu_context.h
···
3
3
#define _LINUX_MMU_CONTEXT_H
4
4
5
5
#include <asm/mmu_context.h>
6
+
#include <asm/mmu.h>
6
7
7
8
/* Architectures that care about IRQ state in switch_mm can override this. */
8
9
#ifndef switch_mm_irqs_off
9
10
# define switch_mm_irqs_off switch_mm
11
+
#endif
12
+
13
+
#ifndef leave_mm
14
+
static inline void leave_mm(int cpu) { }
10
15
#endif
11
16
12
17
#endif
+11
-7
kernel/locking/lockdep.c
+11
-7
kernel/locking/lockdep.c
···
3756
3756
3757
3757
skip_checks:
3758
3758
/* we'll do an OFF -> ON transition: */
3759
-
this_cpu_write(hardirqs_enabled, 1);
3759
+
__this_cpu_write(hardirqs_enabled, 1);
3760
3760
trace->hardirq_enable_ip = ip;
3761
3761
trace->hardirq_enable_event = ++trace->irq_events;
3762
3762
debug_atomic_inc(hardirqs_on_events);
···
3795
3795
/*
3796
3796
* We have done an ON -> OFF transition:
3797
3797
*/
3798
-
this_cpu_write(hardirqs_enabled, 0);
3798
+
__this_cpu_write(hardirqs_enabled, 0);
3799
3799
trace->hardirq_disable_ip = ip;
3800
3800
trace->hardirq_disable_event = ++trace->irq_events;
3801
3801
debug_atomic_inc(hardirqs_off_events);
···
4977
4977
{
4978
4978
unsigned long flags;
4979
4979
4980
+
trace_lock_acquire(lock, subclass, trylock, read, check, nest_lock, ip);
4981
+
4980
4982
if (unlikely(current->lockdep_recursion)) {
4981
4983
/* XXX allow trylock from NMI ?!? */
4982
4984
if (lockdep_nmi() && !trylock) {
···
5003
5001
check_flags(flags);
5004
5002
5005
5003
current->lockdep_recursion++;
5006
-
trace_lock_acquire(lock, subclass, trylock, read, check, nest_lock, ip);
5007
5004
__lock_acquire(lock, subclass, trylock, read, check,
5008
5005
irqs_disabled_flags(flags), nest_lock, ip, 0, 0);
5009
5006
lockdep_recursion_finish();
···
5014
5013
{
5015
5014
unsigned long flags;
5016
5015
5016
+
trace_lock_release(lock, ip);
5017
+
5017
5018
if (unlikely(current->lockdep_recursion))
5018
5019
return;
5019
5020
5020
5021
raw_local_irq_save(flags);
5021
5022
check_flags(flags);
5023
+
5022
5024
current->lockdep_recursion++;
5023
-
trace_lock_release(lock, ip);
5024
5025
if (__lock_release(lock, ip))
5025
5026
check_chain_key(current);
5026
5027
lockdep_recursion_finish();
···
5208
5205
hlock->holdtime_stamp = now;
5209
5206
}
5210
5207
5211
-
trace_lock_acquired(lock, ip);
5212
-
5213
5208
stats = get_lock_stats(hlock_class(hlock));
5214
5209
if (waittime) {
5215
5210
if (hlock->read)
···
5226
5225
{
5227
5226
unsigned long flags;
5228
5227
5228
+
trace_lock_acquired(lock, ip);
5229
+
5229
5230
if (unlikely(!lock_stat || !debug_locks))
5230
5231
return;
5231
5232
···
5237
5234
raw_local_irq_save(flags);
5238
5235
check_flags(flags);
5239
5236
current->lockdep_recursion++;
5240
-
trace_lock_contended(lock, ip);
5241
5237
__lock_contended(lock, ip);
5242
5238
lockdep_recursion_finish();
5243
5239
raw_local_irq_restore(flags);
···
5246
5244
void lock_acquired(struct lockdep_map *lock, unsigned long ip)
5247
5245
{
5248
5246
unsigned long flags;
5247
+
5248
+
trace_lock_contended(lock, ip);
5249
5249
5250
5250
if (unlikely(!lock_stat || !debug_locks))
5251
5251
return;
+12
-15
kernel/sched/idle.c
+12
-15
kernel/sched/idle.c
···
54
54
55
55
static noinline int __cpuidle cpu_idle_poll(void)
56
56
{
57
-
rcu_idle_enter();
58
-
trace_cpu_idle_rcuidle(0, smp_processor_id());
59
-
local_irq_enable();
57
+
trace_cpu_idle(0, smp_processor_id());
60
58
stop_critical_timings();
59
+
rcu_idle_enter();
60
+
local_irq_enable();
61
61
62
62
while (!tif_need_resched() &&
63
-
(cpu_idle_force_poll || tick_check_broadcast_expired()))
63
+
(cpu_idle_force_poll || tick_check_broadcast_expired()))
64
64
cpu_relax();
65
-
start_critical_timings();
66
-
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
65
+
67
66
rcu_idle_exit();
67
+
start_critical_timings();
68
+
trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
68
69
69
70
return 1;
70
71
}
···
91
90
if (current_clr_polling_and_test()) {
92
91
local_irq_enable();
93
92
} else {
93
+
94
+
trace_cpu_idle(1, smp_processor_id());
94
95
stop_critical_timings();
96
+
rcu_idle_enter();
95
97
arch_cpu_idle();
98
+
rcu_idle_exit();
96
99
start_critical_timings();
100
+
trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
97
101
}
98
102
}
99
103
···
164
158
165
159
if (cpuidle_not_available(drv, dev)) {
166
160
tick_nohz_idle_stop_tick();
167
-
rcu_idle_enter();
168
161
169
162
default_idle_call();
170
163
goto exit_idle;
···
183
178
u64 max_latency_ns;
184
179
185
180
if (idle_should_enter_s2idle()) {
186
-
rcu_idle_enter();
187
181
188
182
entered_state = call_cpuidle_s2idle(drv, dev);
189
183
if (entered_state > 0)
190
184
goto exit_idle;
191
-
192
-
rcu_idle_exit();
193
185
194
186
max_latency_ns = U64_MAX;
195
187
} else {
···
194
192
}
195
193
196
194
tick_nohz_idle_stop_tick();
197
-
rcu_idle_enter();
198
195
199
196
next_state = cpuidle_find_deepest_state(drv, dev, max_latency_ns);
200
197
call_cpuidle(drv, dev, next_state);
···
210
209
else
211
210
tick_nohz_idle_retain_tick();
212
211
213
-
rcu_idle_enter();
214
-
215
212
entered_state = call_cpuidle(drv, dev, next_state);
216
213
/*
217
214
* Give the governor an opportunity to reflect on the outcome
···
225
226
*/
226
227
if (WARN_ON_ONCE(irqs_disabled()))
227
228
local_irq_enable();
228
-
229
-
rcu_idle_exit();
230
229
}
231
230
232
231
/*