Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

sched/preempt: Add PREEMPT_DYNAMIC using static keys

Where an architecture selects HAVE_STATIC_CALL but not
HAVE_STATIC_CALL_INLINE, each static call has an out-of-line trampoline
which will either branch to a callee or return to the caller.

On such architectures, a number of constraints can conspire to make
those trampolines more complicated and potentially less useful than we'd
like. For example:

* Hardware and software control flow integrity schemes can require the
addition of "landing pad" instructions (e.g. `BTI` for arm64), which
will also be present at the "real" callee.

* Limited branch ranges can require that trampolines generate or load an
address into a register and perform an indirect branch (or at least
have a slow path that does so). This loses some of the benefits of
having a direct branch.

* Interaction with SW CFI schemes can be complicated and fragile, e.g.
requiring that we can recognise idiomatic codegen and remove
indirections understand, at least until clang proves more helpful
mechanisms for dealing with this.

For PREEMPT_DYNAMIC, we don't need the full power of static calls, as we
really only need to enable/disable specific preemption functions. We can
achieve the same effect without a number of the pain points above by
using static keys to fold early returns into the preemption functions
themselves rather than in an out-of-line trampoline, effectively
inlining the trampoline into the start of the function.

For arm64, this results in good code generation. For example, the
dynamic_cond_resched() wrapper looks as follows when enabled. When
disabled, the first `B` is replaced with a `NOP`, resulting in an early
return.

| <dynamic_cond_resched>:
| bti c
| b <dynamic_cond_resched+0x10> // or `nop`
| mov w0, #0x0
| ret
| mrs x0, sp_el0
| ldr x0, [x0, #8]
| cbnz x0, <dynamic_cond_resched+0x8>
| paciasp
| stp x29, x30, [sp, #-16]!
| mov x29, sp
| bl <preempt_schedule_common>
| mov w0, #0x1
| ldp x29, x30, [sp], #16
| autiasp
| ret

... compared to the regular form of the function:

| <__cond_resched>:
| bti c
| mrs x0, sp_el0
| ldr x1, [x0, #8]
| cbz x1, <__cond_resched+0x18>
| mov w0, #0x0
| ret
| paciasp
| stp x29, x30, [sp, #-16]!
| mov x29, sp
| bl <preempt_schedule_common>
| mov w0, #0x1
| ldp x29, x30, [sp], #16
| autiasp
| ret

Any architecture which implements static keys should be able to use this
to implement PREEMPT_DYNAMIC with similar cost to non-inlined static
calls. Since this is likely to have greater overhead than (inlined)
static calls, PREEMPT_DYNAMIC is only defaulted to enabled when
HAVE_PREEMPT_DYNAMIC_CALL is selected.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Acked-by: Frederic Weisbecker <frederic@kernel.org>
Link: https://lore.kernel.org/r/20220214165216.2231574-6-mark.rutland@arm.com

authored by

Mark Rutland and committed by
Peter Zijlstra
99cf983c 33c64734

+122 -11
+33 -3
arch/Kconfig
··· 1278 1278 1279 1279 config HAVE_PREEMPT_DYNAMIC 1280 1280 bool 1281 + 1282 + config HAVE_PREEMPT_DYNAMIC_CALL 1283 + bool 1281 1284 depends on HAVE_STATIC_CALL 1285 + select HAVE_PREEMPT_DYNAMIC 1282 1286 help 1283 - Select this if the architecture support boot time preempt setting 1284 - on top of static calls. It is strongly advised to support inline 1285 - static call to avoid any overhead. 1287 + An architecture should select this if it can handle the preemption 1288 + model being selected at boot time using static calls. 1289 + 1290 + Where an architecture selects HAVE_STATIC_CALL_INLINE, any call to a 1291 + preemption function will be patched directly. 1292 + 1293 + Where an architecture does not select HAVE_STATIC_CALL_INLINE, any 1294 + call to a preemption function will go through a trampoline, and the 1295 + trampoline will be patched. 1296 + 1297 + It is strongly advised to support inline static call to avoid any 1298 + overhead. 1299 + 1300 + config HAVE_PREEMPT_DYNAMIC_KEY 1301 + bool 1302 + depends on HAVE_ARCH_JUMP_LABEL && CC_HAS_ASM_GOTO 1303 + select HAVE_PREEMPT_DYNAMIC 1304 + help 1305 + An architecture should select this if it can handle the preemption 1306 + model being selected at boot time using static keys. 1307 + 1308 + Each preemption function will be given an early return based on a 1309 + static key. This should have slightly lower overhead than non-inline 1310 + static calls, as this effectively inlines each trampoline into the 1311 + start of its callee. This may avoid redundant work, and may 1312 + integrate better with CFI schemes. 1313 + 1314 + This will have greater overhead than using inline static calls as 1315 + the call to the preemption function cannot be entirely elided. 1286 1316 1287 1317 config ARCH_WANT_LD_ORPHAN_WARN 1288 1318 bool
+1 -1
arch/x86/Kconfig
··· 245 245 select HAVE_STACK_VALIDATION if X86_64 246 246 select HAVE_STATIC_CALL 247 247 select HAVE_STATIC_CALL_INLINE if HAVE_STACK_VALIDATION 248 - select HAVE_PREEMPT_DYNAMIC 248 + select HAVE_PREEMPT_DYNAMIC_CALL 249 249 select HAVE_RSEQ 250 250 select HAVE_SYSCALL_TRACEPOINTS 251 251 select HAVE_UNSTABLE_SCHED_CLOCK
+8 -2
include/linux/entry-common.h
··· 456 456 */ 457 457 void raw_irqentry_exit_cond_resched(void); 458 458 #ifdef CONFIG_PREEMPT_DYNAMIC 459 + #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 459 460 #define irqentry_exit_cond_resched_dynamic_enabled raw_irqentry_exit_cond_resched 460 461 #define irqentry_exit_cond_resched_dynamic_disabled NULL 461 462 DECLARE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); 462 463 #define irqentry_exit_cond_resched() static_call(irqentry_exit_cond_resched)() 463 - #else 464 - #define irqentry_exit_cond_resched() raw_irqentry_exit_cond_resched() 464 + #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 465 + DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); 466 + void dynamic_irqentry_exit_cond_resched(void); 467 + #define irqentry_exit_cond_resched() dynamic_irqentry_exit_cond_resched() 465 468 #endif 469 + #else /* CONFIG_PREEMPT_DYNAMIC */ 470 + #define irqentry_exit_cond_resched() raw_irqentry_exit_cond_resched() 471 + #endif /* CONFIG_PREEMPT_DYNAMIC */ 466 472 467 473 /** 468 474 * irqentry_exit - Handle return from exception that used irqentry_enter()
+6 -1
include/linux/kernel.h
··· 99 99 extern int __cond_resched(void); 100 100 # define might_resched() __cond_resched() 101 101 102 - #elif defined(CONFIG_PREEMPT_DYNAMIC) 102 + #elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 103 103 104 104 extern int __cond_resched(void); 105 105 ··· 109 109 { 110 110 static_call_mod(might_resched)(); 111 111 } 112 + 113 + #elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 114 + 115 + extern int dynamic_might_resched(void); 116 + # define might_resched() dynamic_might_resched() 112 117 113 118 #else 114 119
+9 -1
include/linux/sched.h
··· 2020 2020 #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) 2021 2021 extern int __cond_resched(void); 2022 2022 2023 - #ifdef CONFIG_PREEMPT_DYNAMIC 2023 + #if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 2024 2024 2025 2025 DECLARE_STATIC_CALL(cond_resched, __cond_resched); 2026 2026 2027 2027 static __always_inline int _cond_resched(void) 2028 2028 { 2029 2029 return static_call_mod(cond_resched)(); 2030 + } 2031 + 2032 + #elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 2033 + extern int dynamic_cond_resched(void); 2034 + 2035 + static __always_inline int _cond_resched(void) 2036 + { 2037 + return dynamic_cond_resched(); 2030 2038 } 2031 2039 2032 2040 #else
+2 -1
kernel/Kconfig.preempt
··· 96 96 config PREEMPT_DYNAMIC 97 97 bool "Preemption behaviour defined on boot" 98 98 depends on HAVE_PREEMPT_DYNAMIC && !PREEMPT_RT 99 + select JUMP_LABEL if HAVE_PREEMPT_DYNAMIC_KEY 99 100 select PREEMPT_BUILD 100 - default y 101 + default y if HAVE_PREEMPT_DYNAMIC_CALL 101 102 help 102 103 This option allows to define the preemption model on the kernel 103 104 command line parameter and thus override the default preemption
+11
kernel/entry/common.c
··· 3 3 #include <linux/context_tracking.h> 4 4 #include <linux/entry-common.h> 5 5 #include <linux/highmem.h> 6 + #include <linux/jump_label.h> 6 7 #include <linux/livepatch.h> 7 8 #include <linux/audit.h> 8 9 #include <linux/tick.h> ··· 393 392 } 394 393 } 395 394 #ifdef CONFIG_PREEMPT_DYNAMIC 395 + #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 396 396 DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); 397 + #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 398 + DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); 399 + void dynamic_irqentry_exit_cond_resched(void) 400 + { 401 + if (!static_key_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) 402 + return; 403 + raw_irqentry_exit_cond_resched(); 404 + } 405 + #endif 397 406 #endif 398 407 399 408 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
+52 -2
kernel/sched/core.c
··· 14 14 15 15 #include <linux/nospec.h> 16 16 #include <linux/blkdev.h> 17 + #include <linux/jump_label.h> 17 18 #include <linux/kcov.h> 18 19 #include <linux/scs.h> 19 20 ··· 6485 6484 */ 6486 6485 if (likely(!preemptible())) 6487 6486 return; 6488 - 6489 6487 preempt_schedule_common(); 6490 6488 } 6491 6489 NOKPROBE_SYMBOL(preempt_schedule); 6492 6490 EXPORT_SYMBOL(preempt_schedule); 6493 6491 6494 6492 #ifdef CONFIG_PREEMPT_DYNAMIC 6493 + #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 6495 6494 #ifndef preempt_schedule_dynamic_enabled 6496 6495 #define preempt_schedule_dynamic_enabled preempt_schedule 6497 6496 #define preempt_schedule_dynamic_disabled NULL 6498 6497 #endif 6499 6498 DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); 6500 6499 EXPORT_STATIC_CALL_TRAMP(preempt_schedule); 6500 + #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 6501 + static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule); 6502 + void __sched notrace dynamic_preempt_schedule(void) 6503 + { 6504 + if (!static_branch_unlikely(&sk_dynamic_preempt_schedule)) 6505 + return; 6506 + preempt_schedule(); 6507 + } 6508 + NOKPROBE_SYMBOL(dynamic_preempt_schedule); 6509 + EXPORT_SYMBOL(dynamic_preempt_schedule); 6501 6510 #endif 6502 - 6511 + #endif 6503 6512 6504 6513 /** 6505 6514 * preempt_schedule_notrace - preempt_schedule called by tracing ··· 6564 6553 EXPORT_SYMBOL_GPL(preempt_schedule_notrace); 6565 6554 6566 6555 #ifdef CONFIG_PREEMPT_DYNAMIC 6556 + #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 6567 6557 #ifndef preempt_schedule_notrace_dynamic_enabled 6568 6558 #define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace 6569 6559 #define preempt_schedule_notrace_dynamic_disabled NULL 6570 6560 #endif 6571 6561 DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); 6572 6562 EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); 6563 + #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 6564 + static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace); 6565 + void __sched notrace dynamic_preempt_schedule_notrace(void) 6566 + { 6567 + if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace)) 6568 + return; 6569 + preempt_schedule_notrace(); 6570 + } 6571 + NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace); 6572 + EXPORT_SYMBOL(dynamic_preempt_schedule_notrace); 6573 + #endif 6573 6574 #endif 6574 6575 6575 6576 #endif /* CONFIG_PREEMPTION */ ··· 8091 8068 #endif 8092 8069 8093 8070 #ifdef CONFIG_PREEMPT_DYNAMIC 8071 + #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 8094 8072 #define cond_resched_dynamic_enabled __cond_resched 8095 8073 #define cond_resched_dynamic_disabled ((void *)&__static_call_return0) 8096 8074 DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); ··· 8101 8077 #define might_resched_dynamic_disabled ((void *)&__static_call_return0) 8102 8078 DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); 8103 8079 EXPORT_STATIC_CALL_TRAMP(might_resched); 8080 + #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 8081 + static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); 8082 + int __sched dynamic_cond_resched(void) 8083 + { 8084 + if (!static_branch_unlikely(&sk_dynamic_cond_resched)) 8085 + return 0; 8086 + return __cond_resched(); 8087 + } 8088 + EXPORT_SYMBOL(dynamic_cond_resched); 8089 + 8090 + static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched); 8091 + int __sched dynamic_might_resched(void) 8092 + { 8093 + if (!static_branch_unlikely(&sk_dynamic_might_resched)) 8094 + return 0; 8095 + return __cond_resched(); 8096 + } 8097 + EXPORT_SYMBOL(dynamic_might_resched); 8098 + #endif 8104 8099 #endif 8105 8100 8106 8101 /* ··· 8249 8206 return -EINVAL; 8250 8207 } 8251 8208 8209 + #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 8252 8210 #define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled) 8253 8211 #define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled) 8212 + #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 8213 + #define preempt_dynamic_enable(f) static_key_enable(&sk_dynamic_##f.key) 8214 + #define preempt_dynamic_disable(f) static_key_disable(&sk_dynamic_##f.key) 8215 + #else 8216 + #error "Unsupported PREEMPT_DYNAMIC mechanism" 8217 + #endif 8254 8218 8255 8219 void sched_dynamic_update(int mode) 8256 8220 {