Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
hrtimer: Rework next event evaluation
The per clock base cached expiry time allows to do a more efficient
evaluation of the next expiry on a CPU.
Separate the reprogramming evaluation from the NOHZ idle evaluation which
needs to exclude the NOHZ timer to keep the reprogramming path lean and
clean.
Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260224163431.468186893@kernel.org
authored by
Thomas Gleixner
and committed by
Peter Zijlstra
2bd1cc24
eddffab8
+74
-56
+74
-56
kernel/time/hrtimer.c
+74
-56
kernel/time/hrtimer.c
···
546
546
#define for_each_active_base(base, cpu_base, active) \
547
547
while ((base = __next_base((cpu_base), &(active))))
548
548
549
-
static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
550
-
const struct hrtimer *exclude,
551
-
unsigned int active, ktime_t expires_next)
549
+
#if defined(CONFIG_NO_HZ_COMMON)
550
+
/*
551
+
* Same as hrtimer_bases_next_event() below, but skips the excluded timer and
552
+
* does not update cpu_base->next_timer/expires.
553
+
*/
554
+
static ktime_t hrtimer_bases_next_event_without(struct hrtimer_cpu_base *cpu_base,
555
+
const struct hrtimer *exclude,
556
+
unsigned int active, ktime_t expires_next)
557
+
{
558
+
struct hrtimer_clock_base *base;
559
+
ktime_t expires;
560
+
561
+
lockdep_assert_held(&cpu_base->lock);
562
+
563
+
for_each_active_base(base, cpu_base, active) {
564
+
expires = ktime_sub(base->expires_next, base->offset);
565
+
if (expires >= expires_next)
566
+
continue;
567
+
568
+
/*
569
+
* If the excluded timer is the first on this base evaluate the
570
+
* next timer.
571
+
*/
572
+
struct timerqueue_node *node = timerqueue_getnext(&base->active);
573
+
574
+
if (unlikely(&exclude->node == node)) {
575
+
node = timerqueue_iterate_next(node);
576
+
if (!node)
577
+
continue;
578
+
expires = ktime_sub(node->expires, base->offset);
579
+
if (expires >= expires_next)
580
+
continue;
581
+
}
582
+
expires_next = expires;
583
+
}
584
+
/* If base->offset changed, the result might be negative */
585
+
return max(expires_next, 0);
586
+
}
587
+
#endif
588
+
589
+
static __always_inline struct hrtimer *clock_base_next_timer(struct hrtimer_clock_base *base)
590
+
{
591
+
struct timerqueue_node *next = timerqueue_getnext(&base->active);
592
+
593
+
return container_of(next, struct hrtimer, node);
594
+
}
595
+
596
+
/* Find the base with the earliest expiry */
597
+
static void hrtimer_bases_first(struct hrtimer_cpu_base *cpu_base,unsigned int active,
598
+
ktime_t *expires_next, struct hrtimer **next_timer)
552
599
{
553
600
struct hrtimer_clock_base *base;
554
601
ktime_t expires;
555
602
556
603
for_each_active_base(base, cpu_base, active) {
557
-
struct timerqueue_node *next;
558
-
struct hrtimer *timer;
559
-
560
-
next = timerqueue_getnext(&base->active);
561
-
timer = container_of(next, struct hrtimer, node);
562
-
if (timer == exclude) {
563
-
/* Get to the next timer in the queue. */
564
-
next = timerqueue_iterate_next(next);
565
-
if (!next)
566
-
continue;
567
-
568
-
timer = container_of(next, struct hrtimer, node);
569
-
}
570
-
expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
571
-
if (expires < expires_next) {
572
-
expires_next = expires;
573
-
574
-
/* Skip cpu_base update if a timer is being excluded. */
575
-
if (exclude)
576
-
continue;
577
-
578
-
if (timer->is_soft)
579
-
cpu_base->softirq_next_timer = timer;
580
-
else
581
-
cpu_base->next_timer = timer;
604
+
expires = ktime_sub(base->expires_next, base->offset);
605
+
if (expires < *expires_next) {
606
+
*expires_next = expires;
607
+
*next_timer = clock_base_next_timer(base);
582
608
}
583
609
}
584
-
/*
585
-
* clock_was_set() might have changed base->offset of any of
586
-
* the clock bases so the result might be negative. Fix it up
587
-
* to prevent a false positive in clockevents_program_event().
588
-
*/
589
-
if (expires_next < 0)
590
-
expires_next = 0;
591
-
return expires_next;
592
610
}
593
611
594
612
/*
···
635
617
ktime_t expires_next = KTIME_MAX;
636
618
unsigned int active;
637
619
620
+
lockdep_assert_held(&cpu_base->lock);
621
+
638
622
if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
639
623
active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
640
-
cpu_base->softirq_next_timer = NULL;
641
-
expires_next = __hrtimer_next_event_base(cpu_base, NULL, active, KTIME_MAX);
642
-
next_timer = cpu_base->softirq_next_timer;
624
+
if (active)
625
+
hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer);
626
+
cpu_base->softirq_next_timer = next_timer;
643
627
}
644
628
645
629
if (active_mask & HRTIMER_ACTIVE_HARD) {
646
630
active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
631
+
if (active)
632
+
hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer);
647
633
cpu_base->next_timer = next_timer;
648
-
expires_next = __hrtimer_next_event_base(cpu_base, NULL, active, expires_next);
649
634
}
650
-
return expires_next;
635
+
return max(expires_next, 0);
651
636
}
652
637
653
638
static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
···
745
724
hrtimer_rearm_event(expires_next, false);
746
725
}
747
726
748
-
/*
749
-
* Reprogram the event source with checking both queues for the
750
-
* next event
751
-
* Called with interrupts disabled and base->lock held
752
-
*/
727
+
/* Reprogram the event source with a evaluation of all clock bases */
753
728
static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip_equal)
754
729
{
755
730
ktime_t expires_next = hrtimer_update_next_event(cpu_base);
···
1679
1662
{
1680
1663
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1681
1664
u64 expires = KTIME_MAX;
1665
+
unsigned int active;
1682
1666
1683
1667
guard(raw_spinlock_irqsave)(&cpu_base->lock);
1684
-
if (hrtimer_hres_active(cpu_base)) {
1685
-
unsigned int active;
1668
+
if (!hrtimer_hres_active(cpu_base))
1669
+
return expires;
1686
1670
1687
-
if (!cpu_base->softirq_activated) {
1688
-
active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
1689
-
expires = __hrtimer_next_event_base(cpu_base, exclude, active, KTIME_MAX);
1690
-
}
1691
-
active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
1692
-
expires = __hrtimer_next_event_base(cpu_base, exclude, active, expires);
1693
-
}
1694
-
return expires;
1671
+
active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
1672
+
if (active && !cpu_base->softirq_activated)
1673
+
expires = hrtimer_bases_next_event_without(cpu_base, exclude, active, KTIME_MAX);
1674
+
1675
+
active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
1676
+
if (!active)
1677
+
return expires;
1678
+
return hrtimer_bases_next_event_without(cpu_base, exclude, active, expires);
1695
1679
}
1696
1680
#endif
1697
1681