hrtimer: Provide a static branch based hrtimer_hres_enabled()

The scheduler evaluates this via hrtimer_is_hres_active() every time it has
to update HRTICK. This needs to follow three pointers, which is expensive.

Provide a static branch based mechanism to avoid that.

Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260224163429.136503358@kernel.org

authored by

Thomas Gleixner and committed by

Peter Zijlstra 4 months ago 0a93d308 d19ff16c

+34 -7

2 changed files

expand all

include

linux

hrtimer.h

kernel

time

hrtimer.c

+9 -4

include/linux/hrtimer.h

··· 153 153 } 154 154 155 155 #ifdef CONFIG_HIGH_RES_TIMERS 156 + extern unsigned int hrtimer_resolution; 156 157 struct clock_event_device; 157 158 158 159 extern void hrtimer_interrupt(struct clock_event_device *dev); 159 160 160 - extern unsigned int hrtimer_resolution; 161 + extern struct static_key_false hrtimer_highres_enabled_key; 161 162 162 - #else 163 + static inline bool hrtimer_highres_enabled(void) 164 + { 165 + return static_branch_likely(&hrtimer_highres_enabled_key); 166 + } 163 167 168 + #else /* CONFIG_HIGH_RES_TIMERS */ 164 169 #define hrtimer_resolution (unsigned int)LOW_RES_NSEC 165 - 166 - #endif 170 + static inline bool hrtimer_highres_enabled(void) { return false; } 171 + #endif /* !CONFIG_HIGH_RES_TIMERS */ 167 172 168 173 static inline ktime_t 169 174 __hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now)

+25 -3

kernel/time/hrtimer.c

··· 126 126 return likely(base->online); 127 127 } 128 128 129 + #ifdef CONFIG_HIGH_RES_TIMERS 130 + DEFINE_STATIC_KEY_FALSE(hrtimer_highres_enabled_key); 131 + 132 + static void hrtimer_hres_workfn(struct work_struct *work) 133 + { 134 + static_branch_enable(&hrtimer_highres_enabled_key); 135 + } 136 + 137 + static DECLARE_WORK(hrtimer_hres_work, hrtimer_hres_workfn); 138 + 139 + static inline void hrtimer_schedule_hres_work(void) 140 + { 141 + if (!hrtimer_highres_enabled()) 142 + schedule_work(&hrtimer_hres_work); 143 + } 144 + #else 145 + static inline void hrtimer_schedule_hres_work(void) { } 146 + #endif 147 + 129 148 /* 130 149 * Functions and macros which are different for UP/SMP systems are kept in a 131 150 * single place ··· 668 649 } 669 650 670 651 /* 671 - * Is the high resolution mode active ? 652 + * Is the high resolution mode active in the CPU base. This cannot use the 653 + * static key as the CPUs are switched to high resolution mode 654 + * asynchronously. 672 655 */ 673 656 static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base) 674 657 { ··· 771 750 tick_setup_sched_timer(true); 772 751 /* "Retrigger" the interrupt to get things going */ 773 752 retrigger_next_event(NULL); 753 + hrtimer_schedule_hres_work(); 774 754 } 775 755 776 756 #else ··· 969 947 */ 970 948 void clock_was_set(unsigned int bases) 971 949 { 972 - struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases); 973 950 cpumask_var_t mask; 974 951 int cpu; 975 952 976 - if (!hrtimer_hres_active(cpu_base) && !tick_nohz_is_active()) 953 + if (!hrtimer_highres_enabled() && !tick_nohz_is_active()) 977 954 goto out_timerfd; 978 955 979 956 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { ··· 983 962 /* Avoid interrupting CPUs if possible */ 984 963 cpus_read_lock(); 985 964 for_each_online_cpu(cpu) { 965 + struct hrtimer_cpu_base *cpu_base; 986 966 unsigned long flags; 987 967 988 968 cpu_base = &per_cpu(hrtimer_bases, cpu);

Configure Feed

Configure Feed