Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
hrtimer: Separate remove/enqueue handling for local timers
As the base switch can be avoided completely when the base stays the same
the remove/enqueue handling can be more streamlined.
Split it out into a separate function which handles both in one go which is
way more efficient and makes the code simpler to follow.
Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260224163430.737600486@kernel.org
authored by
Thomas Gleixner
and committed by
Peter Zijlstra
85a690d1
c9391914
+43
-29
+43
-29
kernel/time/hrtimer.c
+43
-29
kernel/time/hrtimer.c
···
1147
1147
}
1148
1148
1149
1149
static inline bool remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
1150
-
bool restart, bool keep_base)
1150
+
bool newstate)
1151
1151
{
1152
-
bool queued_state = timer->is_queued;
1153
-
1154
1152
lockdep_assert_held(&base->cpu_base->lock);
1155
1153
1156
-
if (queued_state) {
1154
+
if (timer->is_queued) {
1157
1155
bool reprogram;
1158
1156
1159
1157
debug_hrtimer_deactivate(timer);
···
1166
1168
*/
1167
1169
reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
1168
1170
1169
-
/*
1170
-
* If the timer is not restarted then reprogramming is
1171
-
* required if the timer is local. If it is local and about
1172
-
* to be restarted, avoid programming it twice (on removal
1173
-
* and a moment later when it's requeued).
1174
-
*/
1175
-
if (!restart)
1176
-
queued_state = HRTIMER_STATE_INACTIVE;
1177
-
else
1178
-
reprogram &= !keep_base;
1179
-
1180
-
__remove_hrtimer(timer, base, queued_state, reprogram);
1171
+
__remove_hrtimer(timer, base, newstate, reprogram);
1181
1172
return true;
1182
1173
}
1183
1174
return false;
1175
+
}
1176
+
1177
+
static inline bool
1178
+
remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
1179
+
const enum hrtimer_mode mode, ktime_t expires, u64 delta_ns)
1180
+
{
1181
+
/* Remove it from the timer queue if active */
1182
+
if (timer->is_queued) {
1183
+
debug_hrtimer_deactivate(timer);
1184
+
timerqueue_del(&base->active, &timer->node);
1185
+
}
1186
+
1187
+
/* Set the new expiry time */
1188
+
hrtimer_set_expires_range_ns(timer, expires, delta_ns);
1189
+
1190
+
debug_activate(timer, mode, timer->is_queued);
1191
+
base->cpu_base->active_bases |= 1 << base->index;
1192
+
1193
+
/* Pairs with the lockless read in hrtimer_is_queued() */
1194
+
WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED);
1195
+
1196
+
/* Returns true if this is the first expiring timer */
1197
+
return timerqueue_add(&base->active, &timer->node);
1184
1198
}
1185
1199
1186
1200
static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
···
1277
1267
const enum hrtimer_mode mode, struct hrtimer_clock_base *base)
1278
1268
{
1279
1269
struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases);
1280
-
bool is_pinned, first, was_first, was_armed, keep_base = false;
1270
+
bool is_pinned, first, was_first, keep_base = false;
1281
1271
struct hrtimer_cpu_base *cpu_base = base->cpu_base;
1282
1272
1283
1273
was_first = cpu_base->next_timer == timer;
···
1293
1283
keep_base = hrtimer_keep_base(timer, is_local, was_first, is_pinned);
1294
1284
}
1295
1285
1286
+
/* Calculate absolute expiry time for relative timers */
1287
+
if (mode & HRTIMER_MODE_REL)
1288
+
tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid));
1289
+
/* Compensate for low resolution granularity */
1290
+
tim = hrtimer_update_lowres(timer, tim, mode);
1291
+
1296
1292
/*
1297
1293
* Remove an active timer from the queue. In case it is not queued
1298
1294
* on the current CPU, make sure that remove_hrtimer() updates the
···
1313
1297
* @keep_base is also true if the timer callback is running on a
1314
1298
* remote CPU and for local pinned timers.
1315
1299
*/
1316
-
was_armed = remove_hrtimer(timer, base, true, keep_base);
1300
+
if (likely(keep_base)) {
1301
+
first = remove_and_enqueue_same_base(timer, base, mode, tim, delta_ns);
1302
+
} else {
1303
+
/* Keep the ENQUEUED state in case it is queued */
1304
+
bool was_armed = remove_hrtimer(timer, base, HRTIMER_STATE_ENQUEUED);
1317
1305
1318
-
if (mode & HRTIMER_MODE_REL)
1319
-
tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid));
1306
+
hrtimer_set_expires_range_ns(timer, tim, delta_ns);
1320
1307
1321
-
tim = hrtimer_update_lowres(timer, tim, mode);
1322
-
1323
-
hrtimer_set_expires_range_ns(timer, tim, delta_ns);
1324
-
1325
-
/* Switch the timer base, if necessary: */
1326
-
if (!keep_base) {
1308
+
/* Switch the timer base, if necessary: */
1327
1309
base = switch_hrtimer_base(timer, base, is_pinned);
1328
1310
cpu_base = base->cpu_base;
1329
-
}
1330
1311
1331
-
first = enqueue_hrtimer(timer, base, mode, was_armed);
1312
+
first = enqueue_hrtimer(timer, base, mode, was_armed);
1313
+
}
1332
1314
1333
1315
/*
1334
1316
* If the hrtimer interrupt is running, then it will reevaluate the
···
1446
1432
base = lock_hrtimer_base(timer, &flags);
1447
1433
1448
1434
if (!hrtimer_callback_running(timer)) {
1449
-
ret = remove_hrtimer(timer, base, false, false);
1435
+
ret = remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE);
1450
1436
if (ret)
1451
1437
trace_hrtimer_cancel(timer);
1452
1438
}