Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
hrtimer: Avoid re-evaluation when nothing changed
Most times there is no change between hrtimer_interrupt() deferring the rearm
and the invocation of hrtimer_rearm_deferred(). In those cases it's a pointless
exercise to re-evaluate the next expiring timer.
Cache the required data and use it if nothing changed.
Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260224163431.338569372@kernel.org
authored by
Thomas Gleixner
and committed by
Peter Zijlstra
b95c4442
15dd3a94
+58
-40
+27
-26
include/linux/hrtimer_defs.h
+27
-26
include/linux/hrtimer_defs.h
···
47
47
48
48
/**
49
49
* struct hrtimer_cpu_base - the per cpu clock bases
50
-
* @lock: lock protecting the base and associated clock bases
51
-
* and timers
52
-
* @cpu: cpu number
53
-
* @active_bases: Bitfield to mark bases with active timers
54
-
* @clock_was_set_seq: Sequence counter of clock was set events
55
-
* @hres_active: State of high resolution mode
56
-
* @deferred_rearm: A deferred rearm is pending
57
-
* @hang_detected: The last hrtimer interrupt detected a hang
58
-
* @softirq_activated: displays, if the softirq is raised - update of softirq
59
-
* related settings is not required then.
60
-
* @nr_events: Total number of hrtimer interrupt events
61
-
* @nr_retries: Total number of hrtimer interrupt retries
62
-
* @nr_hangs: Total number of hrtimer interrupt hangs
63
-
* @max_hang_time: Maximum time spent in hrtimer_interrupt
64
-
* @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are
65
-
* expired
66
-
* @online: CPU is online from an hrtimers point of view
67
-
* @timer_waiters: A hrtimer_cancel() invocation waits for the timer
68
-
* callback to finish.
69
-
* @expires_next: absolute time of the next event, is required for remote
70
-
* hrtimer enqueue; it is the total first expiry time (hard
71
-
* and soft hrtimer are taken into account)
72
-
* @next_timer: Pointer to the first expiring timer
73
-
* @softirq_expires_next: Time to check, if soft queues needs also to be expired
74
-
* @softirq_next_timer: Pointer to the first expiring softirq based timer
75
-
* @clock_base: array of clock bases for this cpu
50
+
* @lock: lock protecting the base and associated clock bases and timers
51
+
* @cpu: cpu number
52
+
* @active_bases: Bitfield to mark bases with active timers
53
+
* @clock_was_set_seq: Sequence counter of clock was set events
54
+
* @hres_active: State of high resolution mode
55
+
* @deferred_rearm: A deferred rearm is pending
56
+
* @deferred_needs_update: The deferred rearm must re-evaluate the first timer
57
+
* @hang_detected: The last hrtimer interrupt detected a hang
58
+
* @softirq_activated: displays, if the softirq is raised - update of softirq
59
+
* related settings is not required then.
60
+
* @nr_events: Total number of hrtimer interrupt events
61
+
* @nr_retries: Total number of hrtimer interrupt retries
62
+
* @nr_hangs: Total number of hrtimer interrupt hangs
63
+
* @max_hang_time: Maximum time spent in hrtimer_interrupt
64
+
* @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are expired
65
+
* @online: CPU is online from an hrtimers point of view
66
+
* @timer_waiters: A hrtimer_cancel() waiters for the timer callback to finish.
67
+
* @expires_next: Absolute time of the next event, is required for remote
68
+
* hrtimer enqueue; it is the total first expiry time (hard
69
+
* and soft hrtimer are taken into account)
70
+
* @next_timer: Pointer to the first expiring timer
71
+
* @softirq_expires_next: Time to check, if soft queues needs also to be expired
72
+
* @softirq_next_timer: Pointer to the first expiring softirq based timer
73
+
* @deferred_expires_next: Cached expires next value for deferred rearm
74
+
* @clock_base: Array of clock bases for this cpu
76
75
*
77
76
* Note: next_timer is just an optimization for __remove_hrtimer().
78
77
* Do not dereference the pointer because it is not reliable on
···
84
85
unsigned int clock_was_set_seq;
85
86
bool hres_active;
86
87
bool deferred_rearm;
88
+
bool deferred_needs_update;
87
89
bool hang_detected;
88
90
bool softirq_activated;
89
91
bool online;
···
102
102
struct hrtimer *next_timer;
103
103
ktime_t softirq_expires_next;
104
104
struct hrtimer *softirq_next_timer;
105
+
ktime_t deferred_expires_next;
105
106
struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES];
106
107
call_single_data_t csd;
107
108
} ____cacheline_aligned;
+31
-14
kernel/time/hrtimer.c
+31
-14
kernel/time/hrtimer.c
···
919
919
return false;
920
920
921
921
/* If a deferred rearm is pending the remote CPU will take care of it */
922
-
if (cpu_base->deferred_rearm)
922
+
if (cpu_base->deferred_rearm) {
923
+
cpu_base->deferred_needs_update = true;
923
924
return false;
925
+
}
924
926
925
927
/*
926
928
* Walk the affected clock bases and check whether the first expiring
···
1143
1141
* a local timer is removed to be immediately restarted. That's handled
1144
1142
* at the call site.
1145
1143
*/
1146
-
if (reprogram && timer == cpu_base->next_timer && !timer->is_lazy)
1144
+
if (!reprogram || timer != cpu_base->next_timer || timer->is_lazy)
1145
+
return;
1146
+
1147
+
if (cpu_base->deferred_rearm)
1148
+
cpu_base->deferred_needs_update = true;
1149
+
else
1147
1150
hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);
1148
1151
}
1149
1152
···
1335
1328
}
1336
1329
1337
1330
/* If a deferred rearm is pending skip reprogramming the device */
1338
-
if (cpu_base->deferred_rearm)
1331
+
if (cpu_base->deferred_rearm) {
1332
+
cpu_base->deferred_needs_update = true;
1339
1333
return false;
1334
+
}
1340
1335
1341
1336
if (!was_first || cpu_base != this_cpu_base) {
1342
1337
/*
···
1948
1939
* Very similar to hrtimer_force_reprogram(), except it deals with
1949
1940
* deferred_rearm and hang_detected.
1950
1941
*/
1951
-
static void hrtimer_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t now,
1952
-
ktime_t expires_next, bool deferred)
1942
+
static void hrtimer_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next, bool deferred)
1953
1943
{
1954
1944
cpu_base->expires_next = expires_next;
1955
1945
cpu_base->deferred_rearm = false;
···
1958
1950
* Give the system a chance to do something else than looping
1959
1951
* on hrtimer interrupts.
1960
1952
*/
1961
-
expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC);
1953
+
expires_next = ktime_add_ns(ktime_get(), 100 * NSEC_PER_MSEC);
1962
1954
cpu_base->hang_detected = false;
1963
1955
}
1964
1956
hrtimer_rearm_event(expires_next, deferred);
···
1968
1960
void __hrtimer_rearm_deferred(void)
1969
1961
{
1970
1962
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1971
-
ktime_t now, expires_next;
1963
+
ktime_t expires_next;
1972
1964
1973
1965
if (!cpu_base->deferred_rearm)
1974
1966
return;
1975
1967
1976
1968
guard(raw_spinlock)(&cpu_base->lock);
1977
-
now = hrtimer_update_base(cpu_base);
1978
-
expires_next = hrtimer_update_next_event(cpu_base);
1979
-
hrtimer_rearm(cpu_base, now, expires_next, true);
1969
+
if (cpu_base->deferred_needs_update) {
1970
+
hrtimer_update_base(cpu_base);
1971
+
expires_next = hrtimer_update_next_event(cpu_base);
1972
+
} else {
1973
+
/* No timer added/removed. Use the cached value */
1974
+
expires_next = cpu_base->deferred_expires_next;
1975
+
}
1976
+
hrtimer_rearm(cpu_base, expires_next, true);
1980
1977
}
1981
1978
1982
1979
static __always_inline void
1983
-
hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t now, ktime_t expires_next)
1980
+
hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next)
1984
1981
{
1982
+
/* hrtimer_interrupt() just re-evaluated the first expiring timer */
1983
+
cpu_base->deferred_needs_update = false;
1984
+
/* Cache the expiry time */
1985
+
cpu_base->deferred_expires_next = expires_next;
1985
1986
set_thread_flag(TIF_HRTIMER_REARM);
1986
1987
}
1987
1988
#else /* CONFIG_HRTIMER_REARM_DEFERRED */
1988
1989
static __always_inline void
1989
-
hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t now, ktime_t expires_next)
1990
+
hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next)
1990
1991
{
1991
-
hrtimer_rearm(cpu_base, now, expires_next, false);
1992
+
hrtimer_rearm(cpu_base, expires_next, false);
1992
1993
}
1993
1994
#endif /* !CONFIG_HRTIMER_REARM_DEFERRED */
1994
1995
···
2058
2041
cpu_base->hang_detected = true;
2059
2042
}
2060
2043
2061
-
hrtimer_interrupt_rearm(cpu_base, now, expires_next);
2044
+
hrtimer_interrupt_rearm(cpu_base, expires_next);
2062
2045
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
2063
2046
}
2064
2047