timekeeping: Allow inlining clocksource::read()

On some architectures clocksource::read() boils down to a single
instruction, so the indirect function call is just a massive overhead
especially with speculative execution mitigations in effect.

Allow architectures to enable conditional inlining of that read to avoid
that by:

- providing a static branch to switch to the inlined variant

- disabling the branch before clocksource changes

- enabling the branch after a clocksource change, when the clocksource
indicates in a feature flag that it is the one which provides the
inlined variant

This is intentionally not a static call as that would only remove the
indirect call, but not the rest of the overhead.

Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260224163429.675151545@kernel.org

authored by

Thomas Gleixner and committed by

Peter Zijlstra 3 months ago 2e27beeb 70802807

+62 -21

3 changed files

expand all

include

linux

clocksource.h

kernel

time

Kconfig

timekeeping.c

include/linux/clocksource.h

··· 149 149 #define CLOCK_SOURCE_SUSPEND_NONSTOP 0x80 150 150 #define CLOCK_SOURCE_RESELECT 0x100 151 151 #define CLOCK_SOURCE_VERIFY_PERCPU 0x200 152 + #define CLOCK_SOURCE_CAN_INLINE_READ 0x400 153 + 152 154 /* simplify initialization of mask field */ 153 155 #define CLOCKSOURCE_MASK(bits) GENMASK_ULL((bits) - 1, 0) 154 156

kernel/time/Kconfig

··· 17 17 config ARCH_CLOCKSOURCE_INIT 18 18 bool 19 19 20 + config ARCH_WANTS_CLOCKSOURCE_READ_INLINE 21 + bool 22 + 20 23 # Timekeeping vsyscall support 21 24 config GENERIC_TIME_VSYSCALL 22 25 bool

+57 -21

kernel/time/timekeeping.c

··· 3 3 * Kernel timekeeping code and accessor functions. Based on code from 4 4 * timer.c, moved in commit 8524070b7982. 5 5 */ 6 - #include <linux/timekeeper_internal.h> 7 - #include <linux/module.h> 8 - #include <linux/interrupt.h> 9 - #include <linux/kobject.h> 10 - #include <linux/percpu.h> 11 - #include <linux/init.h> 12 - #include <linux/mm.h> 13 - #include <linux/nmi.h> 14 - #include <linux/sched.h> 15 - #include <linux/sched/loadavg.h> 16 - #include <linux/sched/clock.h> 17 - #include <linux/syscore_ops.h> 6 + #include <linux/audit.h> 18 7 #include <linux/clocksource.h> 8 + #include <linux/compiler.h> 19 9 #include <linux/jiffies.h> 10 + #include <linux/kobject.h> 11 + #include <linux/module.h> 12 + #include <linux/nmi.h> 13 + #include <linux/pvclock_gtod.h> 14 + #include <linux/random.h> 15 + #include <linux/sched/clock.h> 16 + #include <linux/sched/loadavg.h> 17 + #include <linux/static_key.h> 18 + #include <linux/stop_machine.h> 19 + #include <linux/syscore_ops.h> 20 + #include <linux/tick.h> 20 21 #include <linux/time.h> 21 22 #include <linux/timex.h> 22 - #include <linux/tick.h> 23 - #include <linux/stop_machine.h> 24 - #include <linux/pvclock_gtod.h> 25 - #include <linux/compiler.h> 26 - #include <linux/audit.h> 27 - #include <linux/random.h> 23 + #include <linux/timekeeper_internal.h> 28 24 29 25 #include <vdso/auxclock.h> 30 26 31 27 #include "tick-internal.h" 32 - #include "ntp_internal.h" 33 28 #include "timekeeping_internal.h" 29 + #include "ntp_internal.h" 34 30 35 31 #define TK_CLEAR_NTP (1 << 0) 36 32 #define TK_CLOCK_WAS_SET (1 << 1) ··· 271 275 tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot); 272 276 } 273 277 278 + #ifdef CONFIG_ARCH_WANTS_CLOCKSOURCE_READ_INLINE 279 + #include <asm/clock_inlined.h> 280 + 281 + static DEFINE_STATIC_KEY_FALSE(clocksource_read_inlined); 282 + 274 283 /* 275 284 * tk_clock_read - atomic clocksource read() helper 276 285 * ··· 289 288 * a read of the fast-timekeeper tkrs (which is protected by its own locking 290 289 * and update logic). 291 290 */ 292 - static inline u64 tk_clock_read(const struct tk_read_base *tkr) 291 + static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr) 292 + { 293 + struct clocksource *clock = READ_ONCE(tkr->clock); 294 + 295 + if (static_branch_likely(&clocksource_read_inlined)) 296 + return arch_inlined_clocksource_read(clock); 297 + 298 + return clock->read(clock); 299 + } 300 + 301 + static inline void clocksource_disable_inline_read(void) 302 + { 303 + static_branch_disable(&clocksource_read_inlined); 304 + } 305 + 306 + static inline void clocksource_enable_inline_read(void) 307 + { 308 + static_branch_enable(&clocksource_read_inlined); 309 + } 310 + #else 311 + static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr) 293 312 { 294 313 struct clocksource *clock = READ_ONCE(tkr->clock); 295 314 296 315 return clock->read(clock); 297 316 } 317 + static inline void clocksource_disable_inline_read(void) { } 318 + static inline void clocksource_enable_inline_read(void) { } 319 + #endif 298 320 299 321 /** 300 322 * tk_setup_internals - Set up internals to use clocksource clock. ··· 399 375 return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift); 400 376 } 401 377 402 - static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles) 378 + static __always_inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles) 403 379 { 404 380 /* Calculate the delta since the last update_wall_time() */ 405 381 u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask; ··· 1655 1631 1656 1632 if (tk->tkr_mono.clock == clock) 1657 1633 return 0; 1634 + 1635 + /* Disable inlined reads accross the clocksource switch */ 1636 + clocksource_disable_inline_read(); 1637 + 1658 1638 stop_machine(change_clocksource, clock, NULL); 1639 + 1640 + /* 1641 + * If the clocksource has been selected and supports inlined reads 1642 + * enable the branch. 1643 + */ 1644 + if (tk->tkr_mono.clock == clock && clock->flags & CLOCK_SOURCE_CAN_INLINE_READ) 1645 + clocksource_enable_inline_read(); 1646 + 1659 1647 tick_clock_notify(); 1660 1648 return tk->tkr_mono.clock == clock ? 0 : -1; 1661 1649 }

Configure Feed

Configure Feed