tick/broadcast: Prefer per-cpu oneshot wakeup timers to broadcast

Some SoCs have two per-cpu timer implementations where the timer with the
higher rating stops in deep idle (i.e. suffers from CLOCK_EVT_FEAT_C3STOP)
but is otherwise preferable to the timer with the lower rating. In such a
design, selecting the higher rated devices relies on a global broadcast
timer and IPIs to wake up from deep idle states.

To avoid the reliance on a global broadcast timer and also to reduce the
overhead associated with the IPI wakeups, extend
tick_install_broadcast_device() to manage per-cpu wakeup timers separately
from the broadcast device.

For now, these timers remain unused.

Signed-off-by: Will Deacon <will@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/r/20210524221818.15850-4-will@kernel.org

authored by

Will Deacon and committed by

Thomas Gleixner 5 years ago c94a8537 e5007c28

+61 -4

3 changed files

expand all

kernel

time

tick-broadcast.c

tick-common.c

tick-internal.h

+58 -1

kernel/time/tick-broadcast.c

··· 33 33 static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(tick_broadcast_lock); 34 34 35 35 #ifdef CONFIG_TICK_ONESHOT 36 + static DEFINE_PER_CPU(struct clock_event_device *, tick_oneshot_wakeup_device); 37 + 36 38 static void tick_broadcast_setup_oneshot(struct clock_event_device *bc); 37 39 static void tick_broadcast_clear_oneshot(int cpu); 38 40 static void tick_resume_broadcast_oneshot(struct clock_event_device *bc); ··· 90 88 return !curdev || newdev->rating > curdev->rating; 91 89 } 92 90 91 + #ifdef CONFIG_TICK_ONESHOT 92 + static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu) 93 + { 94 + return per_cpu(tick_oneshot_wakeup_device, cpu); 95 + } 96 + 97 + static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev, 98 + int cpu) 99 + { 100 + struct clock_event_device *curdev = tick_get_oneshot_wakeup_device(cpu); 101 + 102 + if (!newdev) 103 + goto set_device; 104 + 105 + if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) || 106 + (newdev->features & CLOCK_EVT_FEAT_C3STOP)) 107 + return false; 108 + 109 + if (!(newdev->features & CLOCK_EVT_FEAT_PERCPU) || 110 + !(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) 111 + return false; 112 + 113 + if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu))) 114 + return false; 115 + 116 + if (curdev && newdev->rating <= curdev->rating) 117 + return false; 118 + 119 + if (!try_module_get(newdev->owner)) 120 + return false; 121 + 122 + set_device: 123 + clockevents_exchange_device(curdev, newdev); 124 + per_cpu(tick_oneshot_wakeup_device, cpu) = newdev; 125 + return true; 126 + } 127 + #else 128 + static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu) 129 + { 130 + return NULL; 131 + } 132 + 133 + static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev, 134 + int cpu) 135 + { 136 + return false; 137 + } 138 + #endif 139 + 93 140 /* 94 141 * Conditionally install/replace broadcast device 95 142 */ 96 - void tick_install_broadcast_device(struct clock_event_device *dev) 143 + void tick_install_broadcast_device(struct clock_event_device *dev, int cpu) 97 144 { 98 145 struct clock_event_device *cur = tick_broadcast_device.evtdev; 146 + 147 + if (tick_set_oneshot_wakeup_device(dev, cpu)) 148 + return; 99 149 100 150 if (!tick_check_broadcast_device(cur, dev)) 101 151 return; ··· 1050 996 */ 1051 997 static void tick_broadcast_oneshot_offline(unsigned int cpu) 1052 998 { 999 + if (tick_get_oneshot_wakeup_device(cpu)) 1000 + tick_set_oneshot_wakeup_device(NULL, cpu); 1001 + 1053 1002 /* 1054 1003 * Clear the broadcast masks for the dead cpu, but do not stop 1055 1004 * the broadcast device!

+1 -1

kernel/time/tick-common.c

··· 373 373 /* 374 374 * Can the new device be used as a broadcast device ? 375 375 */ 376 - tick_install_broadcast_device(newdev); 376 + tick_install_broadcast_device(newdev, cpu); 377 377 } 378 378 379 379 /**

+2 -2

kernel/time/tick-internal.h

··· 61 61 /* Broadcasting support */ 62 62 # ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 63 63 extern int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu); 64 - extern void tick_install_broadcast_device(struct clock_event_device *dev); 64 + extern void tick_install_broadcast_device(struct clock_event_device *dev, int cpu); 65 65 extern int tick_is_broadcast_device(struct clock_event_device *dev); 66 66 extern void tick_suspend_broadcast(void); 67 67 extern void tick_resume_broadcast(void); ··· 72 72 extern struct tick_device *tick_get_broadcast_device(void); 73 73 extern struct cpumask *tick_get_broadcast_mask(void); 74 74 # else /* !CONFIG_GENERIC_CLOCKEVENTS_BROADCAST: */ 75 - static inline void tick_install_broadcast_device(struct clock_event_device *dev) { } 75 + static inline void tick_install_broadcast_device(struct clock_event_device *dev, int cpu) { } 76 76 static inline int tick_is_broadcast_device(struct clock_event_device *dev) { return 0; } 77 77 static inline int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) { return 0; } 78 78 static inline void tick_do_periodic_broadcast(struct clock_event_device *d) { }

Configure Feed

Configure Feed