Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
hrtimer: Avoid double reprogramming in __hrtimer_start_range_ns()
If __hrtimer_start_range_ns() is invoked with an already armed hrtimer then
the timer has to be canceled first and then added back. If the timer is the
first expiring timer then on removal the clockevent device is reprogrammed
to the next expiring timer to avoid that the pending expiry fires needlessly.
If the new expiry time ends up to be the first expiry again then the clock
event device has to reprogrammed again.
Avoid this by checking whether the timer is the first to expire and in that
case, keep the timer on the current CPU and delay the reprogramming up to
the point where the timer has been enqueued again.
Reported-by: Lorenzo Colitti <lorenzo@google.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/r/20210713135157.873137732@linutronix.de
+53
-7
+53
-7
kernel/time/hrtimer.c
+53
-7
kernel/time/hrtimer.c
···
1030
1030
* remove hrtimer, called with base lock held
1031
1031
*/
1032
1032
static inline int
1033
-
remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, bool restart)
1033
+
remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
1034
+
bool restart, bool keep_local)
1034
1035
{
1035
1036
u8 state = timer->state;
1036
1037
1037
1038
if (state & HRTIMER_STATE_ENQUEUED) {
1038
-
int reprogram;
1039
+
bool reprogram;
1039
1040
1040
1041
/*
1041
1042
* Remove the timer and force reprogramming when high
···
1049
1048
debug_deactivate(timer);
1050
1049
reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
1051
1050
1051
+
/*
1052
+
* If the timer is not restarted then reprogramming is
1053
+
* required if the timer is local. If it is local and about
1054
+
* to be restarted, avoid programming it twice (on removal
1055
+
* and a moment later when it's requeued).
1056
+
*/
1052
1057
if (!restart)
1053
1058
state = HRTIMER_STATE_INACTIVE;
1059
+
else
1060
+
reprogram &= !keep_local;
1054
1061
1055
1062
__remove_hrtimer(timer, base, state, reprogram);
1056
1063
return 1;
···
1112
1103
struct hrtimer_clock_base *base)
1113
1104
{
1114
1105
struct hrtimer_clock_base *new_base;
1106
+
bool force_local, first;
1115
1107
1116
-
/* Remove an active timer from the queue: */
1117
-
remove_hrtimer(timer, base, true);
1108
+
/*
1109
+
* If the timer is on the local cpu base and is the first expiring
1110
+
* timer then this might end up reprogramming the hardware twice
1111
+
* (on removal and on enqueue). To avoid that by prevent the
1112
+
* reprogram on removal, keep the timer local to the current CPU
1113
+
* and enforce reprogramming after it is queued no matter whether
1114
+
* it is the new first expiring timer again or not.
1115
+
*/
1116
+
force_local = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
1117
+
force_local &= base->cpu_base->next_timer == timer;
1118
+
1119
+
/*
1120
+
* Remove an active timer from the queue. In case it is not queued
1121
+
* on the current CPU, make sure that remove_hrtimer() updates the
1122
+
* remote data correctly.
1123
+
*
1124
+
* If it's on the current CPU and the first expiring timer, then
1125
+
* skip reprogramming, keep the timer local and enforce
1126
+
* reprogramming later if it was the first expiring timer. This
1127
+
* avoids programming the underlying clock event twice (once at
1128
+
* removal and once after enqueue).
1129
+
*/
1130
+
remove_hrtimer(timer, base, true, force_local);
1118
1131
1119
1132
if (mode & HRTIMER_MODE_REL)
1120
1133
tim = ktime_add_safe(tim, base->get_time());
···
1146
1115
hrtimer_set_expires_range_ns(timer, tim, delta_ns);
1147
1116
1148
1117
/* Switch the timer base, if necessary: */
1149
-
new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);
1118
+
if (!force_local) {
1119
+
new_base = switch_hrtimer_base(timer, base,
1120
+
mode & HRTIMER_MODE_PINNED);
1121
+
} else {
1122
+
new_base = base;
1123
+
}
1150
1124
1151
-
return enqueue_hrtimer(timer, new_base, mode);
1125
+
first = enqueue_hrtimer(timer, new_base, mode);
1126
+
if (!force_local)
1127
+
return first;
1128
+
1129
+
/*
1130
+
* Timer was forced to stay on the current CPU to avoid
1131
+
* reprogramming on removal and enqueue. Force reprogram the
1132
+
* hardware by evaluating the new first expiring timer.
1133
+
*/
1134
+
hrtimer_force_reprogram(new_base->cpu_base, 1);
1135
+
return 0;
1152
1136
}
1153
1137
1154
1138
/**
···
1229
1183
base = lock_hrtimer_base(timer, &flags);
1230
1184
1231
1185
if (!hrtimer_callback_running(timer))
1232
-
ret = remove_hrtimer(timer, base, false);
1186
+
ret = remove_hrtimer(timer, base, false, false);
1233
1187
1234
1188
unlock_hrtimer_base(timer, &flags);
1235
1189