Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
entry: Hook up rseq time slice extension
Wire the grant decision function up in exit_to_user_mode_loop()
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20251215155709.258157362@linutronix.de
authored by
Thomas Gleixner
and committed by
Peter Zijlstra
3c78aaec
dfb630f5
+25
-2
+25
-2
kernel/entry/common.c
+25
-2
kernel/entry/common.c
···
17
17
#define EXIT_TO_USER_MODE_WORK_LOOP (EXIT_TO_USER_MODE_WORK)
18
18
#endif
19
19
20
+
/* TIF bits, which prevent a time slice extension. */
21
+
#ifdef CONFIG_PREEMPT_RT
22
+
/*
23
+
* Since rseq slice ext has a direct correlation to the worst case
24
+
* scheduling latency (schedule is delayed after all), only have it affect
25
+
* LAZY reschedules on PREEMPT_RT for now.
26
+
*
27
+
* However, since this delay is only applicable to userspace, a value
28
+
* for rseq_slice_extension_nsec that is strictly less than the worst case
29
+
* kernel space preempt_disable() region, should mean the scheduling latency
30
+
* is not affected, even for !LAZY.
31
+
*
32
+
* However, since this value depends on the hardware at hand, it cannot be
33
+
* pre-determined in any sensible way. Hence punt on this problem for now.
34
+
*/
35
+
# define TIF_SLICE_EXT_SCHED (_TIF_NEED_RESCHED_LAZY)
36
+
#else
37
+
# define TIF_SLICE_EXT_SCHED (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)
38
+
#endif
39
+
#define TIF_SLICE_EXT_DENY (EXIT_TO_USER_MODE_WORK & ~TIF_SLICE_EXT_SCHED)
40
+
20
41
static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *regs,
21
42
unsigned long ti_work)
22
43
{
···
49
28
50
29
local_irq_enable_exit_to_user(ti_work);
51
30
52
-
if (ti_work & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY))
53
-
schedule();
31
+
if (ti_work & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)) {
32
+
if (!rseq_grant_slice_extension(ti_work & TIF_SLICE_EXT_DENY))
33
+
schedule();
34
+
}
54
35
55
36
if (ti_work & _TIF_UPROBE)
56
37
uprobe_notify_resume(regs);