Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2#ifndef _LINUX_HRTIMER_REARM_H
3#define _LINUX_HRTIMER_REARM_H
4
5#ifdef CONFIG_HRTIMER_REARM_DEFERRED
6#include <linux/thread_info.h>
7
8void __hrtimer_rearm_deferred(void);
9
10/*
11 * This is purely CPU local, so check the TIF bit first to avoid the overhead of
12 * the atomic test_and_clear_bit() operation for the common case where the bit
13 * is not set.
14 */
15static __always_inline bool hrtimer_test_and_clear_rearm_deferred_tif(unsigned long tif_work)
16{
17 lockdep_assert_irqs_disabled();
18
19 if (unlikely(tif_work & _TIF_HRTIMER_REARM)) {
20 clear_thread_flag(TIF_HRTIMER_REARM);
21 return true;
22 }
23 return false;
24}
25
26#define TIF_REARM_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | _TIF_HRTIMER_REARM)
27
28/* Invoked from the exit to user before invoking exit_to_user_mode_loop() */
29static __always_inline bool
30hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask)
31{
32 /* Help the compiler to optimize the function out for syscall returns */
33 if (!(tif_mask & _TIF_HRTIMER_REARM))
34 return false;
35 /*
36 * Rearm the timer if none of the resched flags is set before going into
37 * the loop which re-enables interrupts.
38 */
39 if (unlikely((*tif_work & TIF_REARM_MASK) == _TIF_HRTIMER_REARM)) {
40 clear_thread_flag(TIF_HRTIMER_REARM);
41 __hrtimer_rearm_deferred();
42 /* Don't go into the loop if HRTIMER_REARM was the only flag */
43 *tif_work &= ~TIF_HRTIMER_REARM;
44 return !*tif_work;
45 }
46 return false;
47}
48
49/* Invoked from the time slice extension decision function */
50static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work)
51{
52 if (hrtimer_test_and_clear_rearm_deferred_tif(tif_work))
53 __hrtimer_rearm_deferred();
54}
55
56/*
57 * This is to be called on all irqentry_exit() paths that will enable
58 * interrupts.
59 */
60static __always_inline void hrtimer_rearm_deferred(void)
61{
62 hrtimer_rearm_deferred_tif(read_thread_flags());
63}
64
65/*
66 * Invoked from the scheduler on entry to __schedule() so it can defer
67 * rearming after the load balancing callbacks which might change hrtick.
68 */
69static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void)
70{
71 return hrtimer_test_and_clear_rearm_deferred_tif(read_thread_flags());
72}
73
74#else /* CONFIG_HRTIMER_REARM_DEFERRED */
75static __always_inline void __hrtimer_rearm_deferred(void) { }
76static __always_inline void hrtimer_rearm_deferred(void) { }
77static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work) { }
78static __always_inline bool
79hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask) { return false; }
80static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void) { return false; }
81#endif /* !CONFIG_HRTIMER_REARM_DEFERRED */
82
83#endif