hrtimer: Use linked timerqueue · tjh.dev/kernel@b7418e6

+8 -8

include/linux/hrtimer_defs.h

··· 25 25 * @offset: offset of this clock to the monotonic base 26 26 */ 27 27 struct hrtimer_clock_base { 28 - struct hrtimer_cpu_base *cpu_base; 29 - unsigned int index; 30 - clockid_t clockid; 31 - seqcount_raw_spinlock_t seq; 32 - ktime_t expires_next; 33 - struct hrtimer *running; 34 - struct timerqueue_head active; 35 - ktime_t offset; 28 + struct hrtimer_cpu_base *cpu_base; 29 + unsigned int index; 30 + clockid_t clockid; 31 + seqcount_raw_spinlock_t seq; 32 + ktime_t expires_next; 33 + struct hrtimer *running; 34 + struct timerqueue_linked_head active; 35 + ktime_t offset; 36 36 } __hrtimer_clock_base_align; 37 37 38 38 enum hrtimer_base_type {

+4 -4

include/linux/hrtimer_types.h

··· 17 17 18 18 /** 19 19 * struct hrtimer - the basic hrtimer structure 20 - * @node: timerqueue node, which also manages node.expires, 20 + * @node: Linked timerqueue node, which also manages node.expires, 21 21 * the absolute expiry time in the hrtimers internal 22 22 * representation. The time is related to the clock on 23 23 * which the timer is based. Is setup by adding ··· 39 39 * The hrtimer structure must be initialized by hrtimer_setup() 40 40 */ 41 41 struct hrtimer { 42 - struct timerqueue_node node; 43 - ktime_t _softexpires; 44 - enum hrtimer_restart (*__private function)(struct hrtimer *); 42 + struct timerqueue_linked_node node; 45 43 struct hrtimer_clock_base *base; 46 44 bool is_queued; 47 45 bool is_rel; 48 46 bool is_soft; 49 47 bool is_hard; 50 48 bool is_lazy; 49 + ktime_t _softexpires; 50 + enum hrtimer_restart (*__private function)(struct hrtimer *); 51 51 }; 52 52 53 53 #endif /* _LINUX_HRTIMER_TYPES_H */

+17 -17

kernel/time/hrtimer.c

··· 557 557 * If the excluded timer is the first on this base evaluate the 558 558 * next timer. 559 559 */ 560 - struct timerqueue_node *node = timerqueue_getnext(&base->active); 560 + struct timerqueue_linked_node *node = timerqueue_linked_first(&base->active); 561 561 562 562 if (unlikely(&exclude->node == node)) { 563 - node = timerqueue_iterate_next(node); 563 + node = timerqueue_linked_next(node); 564 564 if (!node) 565 565 continue; 566 566 expires = ktime_sub(node->expires, base->offset); ··· 576 576 577 577 static __always_inline struct hrtimer *clock_base_next_timer(struct hrtimer_clock_base *base) 578 578 { 579 - struct timerqueue_node *next = timerqueue_getnext(&base->active); 579 + struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active); 580 580 581 581 return container_of(next, struct hrtimer, node); 582 582 } ··· 938 938 active &= cpu_base->active_bases; 939 939 940 940 for_each_active_base(base, cpu_base, active) { 941 - struct timerqueue_node *next; 941 + struct timerqueue_linked_node *next; 942 942 943 - next = timerqueue_getnext(&base->active); 943 + next = timerqueue_linked_first(&base->active); 944 944 expires = ktime_sub(next->expires, base->offset); 945 945 if (expires < cpu_base->expires_next) 946 946 return true; ··· 1112 1112 /* Pairs with the lockless read in hrtimer_is_queued() */ 1113 1113 WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED); 1114 1114 1115 - if (!timerqueue_add(&base->active, &timer->node)) 1115 + if (!timerqueue_linked_add(&base->active, &timer->node)) 1116 1116 return false; 1117 1117 1118 1118 base->expires_next = hrtimer_get_expires(timer); ··· 1121 1121 1122 1122 static inline void base_update_next_timer(struct hrtimer_clock_base *base) 1123 1123 { 1124 - struct timerqueue_node *next = timerqueue_getnext(&base->active); 1124 + struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active); 1125 1125 1126 1126 base->expires_next = next ? next->expires : KTIME_MAX; 1127 1127 } ··· 1148 1148 /* Pairs with the lockless read in hrtimer_is_queued() */ 1149 1149 WRITE_ONCE(timer->is_queued, newstate); 1150 1150 1151 - was_first = &timer->node == timerqueue_getnext(&base->active); 1151 + was_first = !timerqueue_linked_prev(&timer->node); 1152 1152 1153 - if (!timerqueue_del(&base->active, &timer->node)) 1153 + if (!timerqueue_linked_del(&base->active, &timer->node)) 1154 1154 cpu_base->active_bases &= ~(1 << base->index); 1155 1155 1156 1156 /* Nothing to update if this was not the first timer in the base */ ··· 1212 1212 /* Remove it from the timer queue if active */ 1213 1213 if (timer->is_queued) { 1214 1214 debug_hrtimer_deactivate(timer); 1215 - was_first = &timer->node == timerqueue_getnext(&base->active); 1216 - timerqueue_del(&base->active, &timer->node); 1215 + was_first = !timerqueue_linked_prev(&timer->node); 1216 + timerqueue_linked_del(&base->active, &timer->node); 1217 1217 } 1218 1218 1219 1219 /* Set the new expiry time */ ··· 1226 1226 WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED); 1227 1227 1228 1228 /* If it's the first expiring timer now or again, update base */ 1229 - if (timerqueue_add(&base->active, &timer->node)) { 1229 + if (timerqueue_linked_add(&base->active, &timer->node)) { 1230 1230 base->expires_next = expires; 1231 1231 return true; 1232 1232 } ··· 1758 1758 timer->is_hard = !!(mode & HRTIMER_MODE_HARD); 1759 1759 timer->is_lazy = !!(mode & HRTIMER_MODE_LAZY_REARM); 1760 1760 timer->base = &cpu_base->clock_base[base]; 1761 - timerqueue_init(&timer->node); 1761 + timerqueue_linked_init(&timer->node); 1762 1762 1763 1763 if (WARN_ON_ONCE(!fn)) 1764 1764 ACCESS_PRIVATE(timer, function) = hrtimer_dummy_timeout; ··· 1923 1923 1924 1924 static __always_inline struct hrtimer *clock_base_next_timer_safe(struct hrtimer_clock_base *base) 1925 1925 { 1926 - struct timerqueue_node *next = timerqueue_getnext(&base->active); 1926 + struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active); 1927 1927 1928 1928 return next ? container_of(next, struct hrtimer, node) : NULL; 1929 1929 } ··· 2369 2369 2370 2370 clock_b->cpu_base = cpu_base; 2371 2371 seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock); 2372 - timerqueue_init_head(&clock_b->active); 2372 + timerqueue_linked_init_head(&clock_b->active); 2373 2373 } 2374 2374 2375 2375 cpu_base->cpu = cpu; ··· 2399 2399 static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, 2400 2400 struct hrtimer_clock_base *new_base) 2401 2401 { 2402 - struct timerqueue_node *node; 2402 + struct timerqueue_linked_node *node; 2403 2403 struct hrtimer *timer; 2404 2404 2405 - while ((node = timerqueue_getnext(&old_base->active))) { 2405 + while ((node = timerqueue_linked_first(&old_base->active))) { 2406 2406 timer = container_of(node, struct hrtimer, node); 2407 2407 BUG_ON(hrtimer_callback_running(timer)); 2408 2408 debug_hrtimer_deactivate(timer);

+4 -6

kernel/time/timer_list.c

··· 56 56 (long long)(ktime_to_ns(hrtimer_get_expires(timer)) - now)); 57 57 } 58 58 59 - static void 60 - print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, 61 - u64 now) 59 + static void print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, u64 now) 62 60 { 61 + struct timerqueue_linked_node *curr; 63 62 struct hrtimer *timer, tmp; 64 63 unsigned long next = 0, i; 65 - struct timerqueue_node *curr; 66 64 unsigned long flags; 67 65 68 66 next_one: ··· 70 72 71 73 raw_spin_lock_irqsave(&base->cpu_base->lock, flags); 72 74 73 - curr = timerqueue_getnext(&base->active); 75 + curr = timerqueue_linked_first(&base->active); 74 76 /* 75 77 * Crude but we have to do this O(N*N) thing, because 76 78 * we have to unlock the base when printing: 77 79 */ 78 80 while (curr && i < next) { 79 - curr = timerqueue_iterate_next(curr); 81 + curr = timerqueue_linked_next(curr); 80 82 i++; 81 83 } 82 84

Configure Feed

Configure Feed