Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull scheduler fixes from Thomas Gleixner:
"This update provides:
- make the scheduler clock switch to unstable mode smooth so the
timestamps stay at microseconds granularity instead of switching to
tick granularity.
- unbreak perf test tsc by taking the new offset into account which
was added in order to proveide better sched clock continuity
- switching sched clock to unstable mode runs all clock related
computations which affect the sched clock output itself from a work
queue. In case of preemption sched clock uses half updated data and
provides wrong timestamps. Keep the math in the protected context
and delegate only the static key switch to workqueue context.
- remove a duplicate header include"
* 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
sched/headers: Remove duplicate #include <linux/sched/debug.h> line
sched/clock: Fix broken stable to unstable transfer
sched/clock, x86/perf: Fix "perf test tsc"
sched/clock: Fix clear_sched_clock_stable() preempt wobbly
+44
-31
+6
-3
arch/x86/events/core.c
+6
-3
arch/x86/events/core.c
···
2256
2256
struct perf_event_mmap_page *userpg, u64 now)
2257
2257
{
2258
2258
struct cyc2ns_data *data;
2259
+
u64 offset;
2259
2260
2260
2261
userpg->cap_user_time = 0;
2261
2262
userpg->cap_user_time_zero = 0;
···
2264
2263
!!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED);
2265
2264
userpg->pmc_width = x86_pmu.cntval_bits;
2266
2265
2267
-
if (!sched_clock_stable())
2266
+
if (!using_native_sched_clock() || !sched_clock_stable())
2268
2267
return;
2269
2268
2270
2269
data = cyc2ns_read_begin();
2270
+
2271
+
offset = data->cyc2ns_offset + __sched_clock_offset;
2271
2272
2272
2273
/*
2273
2274
* Internal timekeeping for enabled/running/stopped times
···
2278
2275
userpg->cap_user_time = 1;
2279
2276
userpg->time_mult = data->cyc2ns_mul;
2280
2277
userpg->time_shift = data->cyc2ns_shift;
2281
-
userpg->time_offset = data->cyc2ns_offset - now;
2278
+
userpg->time_offset = offset - now;
2282
2279
2283
2280
/*
2284
2281
* cap_user_time_zero doesn't make sense when we're using a different
···
2286
2283
*/
2287
2284
if (!event->attr.use_clockid) {
2288
2285
userpg->cap_user_time_zero = 1;
2289
-
userpg->time_zero = data->cyc2ns_offset;
2286
+
userpg->time_zero = offset;
2290
2287
}
2291
2288
2292
2289
cyc2ns_read_end(data);
+2
arch/x86/include/asm/timer.h
+2
arch/x86/include/asm/timer.h
+2
-2
arch/x86/kernel/tsc.c
+2
-2
arch/x86/kernel/tsc.c
···
328
328
return paravirt_sched_clock();
329
329
}
330
330
331
-
static inline bool using_native_sched_clock(void)
331
+
bool using_native_sched_clock(void)
332
332
{
333
333
return pv_time_ops.sched_clock == native_sched_clock;
334
334
}
···
336
336
unsigned long long
337
337
sched_clock(void) __attribute__((alias("native_sched_clock")));
338
338
339
-
static inline bool using_native_sched_clock(void) { return true; }
339
+
bool using_native_sched_clock(void) { return true; }
340
340
#endif
341
341
342
342
int check_tsc_unstable(void)
-1
drivers/tty/vt/keyboard.c
-1
drivers/tty/vt/keyboard.c
+7
-6
include/linux/sched/clock.h
+7
-6
include/linux/sched/clock.h
···
54
54
}
55
55
#else
56
56
extern void sched_clock_init_late(void);
57
-
/*
58
-
* Architectures can set this to 1 if they have specified
59
-
* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK in their arch Kconfig,
60
-
* but then during bootup it turns out that sched_clock()
61
-
* is reliable after all:
62
-
*/
63
57
extern int sched_clock_stable(void);
64
58
extern void clear_sched_clock_stable(void);
59
+
60
+
/*
61
+
* When sched_clock_stable(), __sched_clock_offset provides the offset
62
+
* between local_clock() and sched_clock().
63
+
*/
64
+
extern u64 __sched_clock_offset;
65
+
65
66
66
67
extern void sched_clock_tick(void);
67
68
extern void sched_clock_idle_sleep_event(void);
+27
-19
kernel/sched/clock.c
+27
-19
kernel/sched/clock.c
···
96
96
static int __sched_clock_stable_early = 1;
97
97
98
98
/*
99
-
* We want: ktime_get_ns() + gtod_offset == sched_clock() + raw_offset
99
+
* We want: ktime_get_ns() + __gtod_offset == sched_clock() + __sched_clock_offset
100
100
*/
101
-
static __read_mostly u64 raw_offset;
102
-
static __read_mostly u64 gtod_offset;
101
+
__read_mostly u64 __sched_clock_offset;
102
+
static __read_mostly u64 __gtod_offset;
103
103
104
104
struct sched_clock_data {
105
105
u64 tick_raw;
···
131
131
/*
132
132
* Attempt to make the (initial) unstable->stable transition continuous.
133
133
*/
134
-
raw_offset = (scd->tick_gtod + gtod_offset) - (scd->tick_raw);
134
+
__sched_clock_offset = (scd->tick_gtod + __gtod_offset) - (scd->tick_raw);
135
135
136
136
printk(KERN_INFO "sched_clock: Marking stable (%lld, %lld)->(%lld, %lld)\n",
137
-
scd->tick_gtod, gtod_offset,
138
-
scd->tick_raw, raw_offset);
137
+
scd->tick_gtod, __gtod_offset,
138
+
scd->tick_raw, __sched_clock_offset);
139
139
140
140
static_branch_enable(&__sched_clock_stable);
141
141
tick_dep_clear(TICK_DEP_BIT_CLOCK_UNSTABLE);
142
142
}
143
143
144
-
static void __clear_sched_clock_stable(struct work_struct *work)
144
+
static void __sched_clock_work(struct work_struct *work)
145
+
{
146
+
static_branch_disable(&__sched_clock_stable);
147
+
}
148
+
149
+
static DECLARE_WORK(sched_clock_work, __sched_clock_work);
150
+
151
+
static void __clear_sched_clock_stable(void)
145
152
{
146
153
struct sched_clock_data *scd = this_scd();
147
154
···
161
154
*
162
155
* Still do what we can.
163
156
*/
164
-
gtod_offset = (scd->tick_raw + raw_offset) - (scd->tick_gtod);
157
+
__gtod_offset = (scd->tick_raw + __sched_clock_offset) - (scd->tick_gtod);
165
158
166
159
printk(KERN_INFO "sched_clock: Marking unstable (%lld, %lld)<-(%lld, %lld)\n",
167
-
scd->tick_gtod, gtod_offset,
168
-
scd->tick_raw, raw_offset);
160
+
scd->tick_gtod, __gtod_offset,
161
+
scd->tick_raw, __sched_clock_offset);
169
162
170
-
static_branch_disable(&__sched_clock_stable);
171
163
tick_dep_set(TICK_DEP_BIT_CLOCK_UNSTABLE);
172
-
}
173
164
174
-
static DECLARE_WORK(sched_clock_work, __clear_sched_clock_stable);
165
+
if (sched_clock_stable())
166
+
schedule_work(&sched_clock_work);
167
+
}
175
168
176
169
void clear_sched_clock_stable(void)
177
170
{
···
180
173
smp_mb(); /* matches sched_clock_init_late() */
181
174
182
175
if (sched_clock_running == 2)
183
-
schedule_work(&sched_clock_work);
176
+
__clear_sched_clock_stable();
184
177
}
185
178
186
179
void sched_clock_init_late(void)
···
221
214
*/
222
215
static u64 sched_clock_local(struct sched_clock_data *scd)
223
216
{
224
-
u64 now, clock, old_clock, min_clock, max_clock;
217
+
u64 now, clock, old_clock, min_clock, max_clock, gtod;
225
218
s64 delta;
226
219
227
220
again:
···
238
231
* scd->tick_gtod + TICK_NSEC);
239
232
*/
240
233
241
-
clock = scd->tick_gtod + gtod_offset + delta;
242
-
min_clock = wrap_max(scd->tick_gtod, old_clock);
243
-
max_clock = wrap_max(old_clock, scd->tick_gtod + TICK_NSEC);
234
+
gtod = scd->tick_gtod + __gtod_offset;
235
+
clock = gtod + delta;
236
+
min_clock = wrap_max(gtod, old_clock);
237
+
max_clock = wrap_max(old_clock, gtod + TICK_NSEC);
244
238
245
239
clock = wrap_max(clock, min_clock);
246
240
clock = wrap_min(clock, max_clock);
···
325
317
u64 clock;
326
318
327
319
if (sched_clock_stable())
328
-
return sched_clock() + raw_offset;
320
+
return sched_clock() + __sched_clock_offset;
329
321
330
322
if (unlikely(!sched_clock_running))
331
323
return 0ull;