hrtimer: Keep track of first expiring timer per clock base

Evaluating the next expiry time of all clock bases is cache line expensive
as the expiry time of the first expiring timer is not cached in the base
and requires to access the timer itself, which is definitely in a different
cache line.

It's way more efficient to keep track of the expiry time on enqueue and
dequeue operations as the relevant data is already in the cache at that
point.

Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260224163431.404839710@kernel.org

authored by

Thomas Gleixner and committed by

Peter Zijlstra 3 months ago eddffab8 b95c4442

+36 -3

2 changed files

expand all

include

linux

hrtimer_defs.h

kernel

time

hrtimer.c

include/linux/hrtimer_defs.h

··· 19 19 * timer to a base on another cpu. 20 20 * @clockid: clock id for per_cpu support 21 21 * @seq: seqcount around __run_hrtimer 22 + * @expires_next: Absolute time of the next event in this clock base 22 23 * @running: pointer to the currently running hrtimer 23 24 * @active: red black tree root node for the active timers 24 25 * @offset: offset of this clock to the monotonic base ··· 29 28 unsigned int index; 30 29 clockid_t clockid; 31 30 seqcount_raw_spinlock_t seq; 31 + ktime_t expires_next; 32 32 struct hrtimer *running; 33 33 struct timerqueue_head active; 34 34 ktime_t offset;

+34 -3

kernel/time/hrtimer.c

··· 1107 1107 /* Pairs with the lockless read in hrtimer_is_queued() */ 1108 1108 WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED); 1109 1109 1110 - return timerqueue_add(&base->active, &timer->node); 1110 + if (!timerqueue_add(&base->active, &timer->node)) 1111 + return false; 1112 + 1113 + base->expires_next = hrtimer_get_expires(timer); 1114 + return true; 1115 + } 1116 + 1117 + static inline void base_update_next_timer(struct hrtimer_clock_base *base) 1118 + { 1119 + struct timerqueue_node *next = timerqueue_getnext(&base->active); 1120 + 1121 + base->expires_next = next ? next->expires : KTIME_MAX; 1111 1122 } 1112 1123 1113 1124 /* ··· 1133 1122 bool newstate, bool reprogram) 1134 1123 { 1135 1124 struct hrtimer_cpu_base *cpu_base = base->cpu_base; 1125 + bool was_first; 1136 1126 1137 1127 lockdep_assert_held(&cpu_base->lock); 1138 1128 ··· 1143 1131 /* Pairs with the lockless read in hrtimer_is_queued() */ 1144 1132 WRITE_ONCE(timer->is_queued, newstate); 1145 1133 1134 + was_first = &timer->node == timerqueue_getnext(&base->active); 1135 + 1146 1136 if (!timerqueue_del(&base->active, &timer->node)) 1147 1137 cpu_base->active_bases &= ~(1 << base->index); 1138 + 1139 + /* Nothing to update if this was not the first timer in the base */ 1140 + if (!was_first) 1141 + return; 1142 + 1143 + base_update_next_timer(base); 1148 1144 1149 1145 /* 1150 1146 * If reprogram is false don't update cpu_base->next_timer and do not ··· 1202 1182 remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *base, 1203 1183 const enum hrtimer_mode mode, ktime_t expires, u64 delta_ns) 1204 1184 { 1185 + bool was_first = false; 1186 + 1205 1187 /* Remove it from the timer queue if active */ 1206 1188 if (timer->is_queued) { 1207 1189 debug_hrtimer_deactivate(timer); 1190 + was_first = &timer->node == timerqueue_getnext(&base->active); 1208 1191 timerqueue_del(&base->active, &timer->node); 1209 1192 } 1210 1193 ··· 1220 1197 /* Pairs with the lockless read in hrtimer_is_queued() */ 1221 1198 WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED); 1222 1199 1223 - /* Returns true if this is the first expiring timer */ 1224 - return timerqueue_add(&base->active, &timer->node); 1200 + /* If it's the first expiring timer now or again, update base */ 1201 + if (timerqueue_add(&base->active, &timer->node)) { 1202 + base->expires_next = expires; 1203 + return true; 1204 + } 1205 + 1206 + if (was_first) 1207 + base_update_next_timer(base); 1208 + 1209 + return false; 1225 1210 } 1226 1211 1227 1212 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,

Configure Feed

Configure Feed