sched,livepatch: Untangle cond_resched() and live-patching

+5 -9

include/linux/livepatch_sched.h

··· 3 3 #define _LINUX_LIVEPATCH_SCHED_H_ 4 4 5 5 #include <linux/jump_label.h> 6 - #include <linux/static_call_types.h> 6 + #include <linux/sched.h> 7 7 8 8 #ifdef CONFIG_LIVEPATCH 9 9 10 10 void __klp_sched_try_switch(void); 11 11 12 - #if !defined(CONFIG_PREEMPT_DYNAMIC) || !defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 13 - 14 12 DECLARE_STATIC_KEY_FALSE(klp_sched_try_switch_key); 15 13 16 - static __always_inline void klp_sched_try_switch(void) 14 + static __always_inline void klp_sched_try_switch(struct task_struct *curr) 17 15 { 18 - if (static_branch_unlikely(&klp_sched_try_switch_key)) 16 + if (static_branch_unlikely(&klp_sched_try_switch_key) && 17 + READ_ONCE(curr->__state) & TASK_FREEZABLE) 19 18 __klp_sched_try_switch(); 20 19 } 21 20 22 - #endif /* !CONFIG_PREEMPT_DYNAMIC || !CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ 23 - 24 21 #else /* !CONFIG_LIVEPATCH */ 25 - static inline void klp_sched_try_switch(void) {} 26 - static inline void __klp_sched_try_switch(void) {} 22 + static inline void klp_sched_try_switch(struct task_struct *curr) {} 27 23 #endif /* CONFIG_LIVEPATCH */ 28 24 29 25 #endif /* _LINUX_LIVEPATCH_SCHED_H_ */

-6

include/linux/sched.h

··· 44 44 #include <linux/seqlock_types.h> 45 45 #include <linux/kcsan.h> 46 46 #include <linux/rv.h> 47 - #include <linux/livepatch_sched.h> 48 47 #include <linux/uidgid_types.h> 49 48 #include <linux/tracepoint-defs.h> 50 49 #include <asm/kmap_size.h> ··· 2088 2089 2089 2090 #if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 2090 2091 2091 - void sched_dynamic_klp_enable(void); 2092 - void sched_dynamic_klp_disable(void); 2093 - 2094 2092 DECLARE_STATIC_CALL(cond_resched, __cond_resched); 2095 2093 2096 2094 static __always_inline int _cond_resched(void) ··· 2108 2112 2109 2113 static inline int _cond_resched(void) 2110 2114 { 2111 - klp_sched_try_switch(); 2112 2115 return __cond_resched(); 2113 2116 } 2114 2117 ··· 2117 2122 2118 2123 static inline int _cond_resched(void) 2119 2124 { 2120 - klp_sched_try_switch(); 2121 2125 return 0; 2122 2126 } 2123 2127

+15 -36

kernel/livepatch/transition.c

··· 29 29 30 30 /* 31 31 * When a livepatch is in progress, enable klp stack checking in 32 - * cond_resched(). This helps CPU-bound kthreads get patched. 32 + * schedule(). This helps CPU-bound kthreads get patched. 33 33 */ 34 - #if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 35 - 36 - #define klp_cond_resched_enable() sched_dynamic_klp_enable() 37 - #define klp_cond_resched_disable() sched_dynamic_klp_disable() 38 - 39 - #else /* !CONFIG_PREEMPT_DYNAMIC || !CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ 40 34 41 35 DEFINE_STATIC_KEY_FALSE(klp_sched_try_switch_key); 42 - EXPORT_SYMBOL(klp_sched_try_switch_key); 43 36 44 - #define klp_cond_resched_enable() static_branch_enable(&klp_sched_try_switch_key) 45 - #define klp_cond_resched_disable() static_branch_disable(&klp_sched_try_switch_key) 46 - 47 - #endif /* CONFIG_PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ 37 + #define klp_resched_enable() static_branch_enable(&klp_sched_try_switch_key) 38 + #define klp_resched_disable() static_branch_disable(&klp_sched_try_switch_key) 48 39 49 40 /* 50 41 * This work can be performed periodically to finish patching or unpatching any ··· 356 365 357 366 void __klp_sched_try_switch(void) 358 367 { 368 + /* 369 + * This function is called from __schedule() while a context switch is 370 + * about to happen. Preemption is already disabled and klp_mutex 371 + * can't be acquired. 372 + * Disabled preemption is used to prevent racing with other callers of 373 + * klp_try_switch_task(). Thanks to task_call_func() they won't be 374 + * able to switch to this task while it's running. 375 + */ 376 + lockdep_assert_preemption_disabled(); 377 + 359 378 if (likely(!klp_patch_pending(current))) 360 379 return; 361 - 362 - /* 363 - * This function is called from cond_resched() which is called in many 364 - * places throughout the kernel. Using the klp_mutex here might 365 - * deadlock. 366 - * 367 - * Instead, disable preemption to prevent racing with other callers of 368 - * klp_try_switch_task(). Thanks to task_call_func() they won't be 369 - * able to switch this task while it's running. 370 - */ 371 - preempt_disable(); 372 - 373 - /* 374 - * Make sure current didn't get patched between the above check and 375 - * preempt_disable(). 376 - */ 377 - if (unlikely(!klp_patch_pending(current))) 378 - goto out; 379 380 380 381 /* 381 382 * Enforce the order of the TIF_PATCH_PENDING read above and the ··· 378 395 smp_rmb(); 379 396 380 397 klp_try_switch_task(current); 381 - 382 - out: 383 - preempt_enable(); 384 398 } 385 - EXPORT_SYMBOL(__klp_sched_try_switch); 386 399 387 400 /* 388 401 * Sends a fake signal to all non-kthread tasks with TIF_PATCH_PENDING set. ··· 487 508 } 488 509 489 510 /* Done! Now cleanup the data structures. */ 490 - klp_cond_resched_disable(); 511 + klp_resched_disable(); 491 512 patch = klp_transition_patch; 492 513 klp_complete_transition(); 493 514 ··· 539 560 set_tsk_thread_flag(task, TIF_PATCH_PENDING); 540 561 } 541 562 542 - klp_cond_resched_enable(); 563 + klp_resched_enable(); 543 564 544 565 klp_signals_cnt = 0; 545 566 }

+8 -42

kernel/sched/core.c

··· 66 66 #include <linux/vtime.h> 67 67 #include <linux/wait_api.h> 68 68 #include <linux/workqueue_api.h> 69 + #include <linux/livepatch_sched.h> 69 70 70 71 #ifdef CONFIG_PREEMPT_DYNAMIC 71 72 # ifdef CONFIG_GENERIC_ENTRY ··· 6677 6676 if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) 6678 6677 hrtick_clear(rq); 6679 6678 6679 + klp_sched_try_switch(prev); 6680 + 6680 6681 local_irq_disable(); 6681 6682 rcu_note_context_switch(preempt); 6682 6683 ··· 7339 7336 static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); 7340 7337 int __sched dynamic_cond_resched(void) 7341 7338 { 7342 - klp_sched_try_switch(); 7343 7339 if (!static_branch_unlikely(&sk_dynamic_cond_resched)) 7344 7340 return 0; 7345 7341 return __cond_resched(); ··· 7510 7508 #endif 7511 7509 7512 7510 static DEFINE_MUTEX(sched_dynamic_mutex); 7513 - static bool klp_override; 7514 7511 7515 7512 static void __sched_dynamic_update(int mode) 7516 7513 { ··· 7517 7516 * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in 7518 7517 * the ZERO state, which is invalid. 7519 7518 */ 7520 - if (!klp_override) 7521 - preempt_dynamic_enable(cond_resched); 7519 + preempt_dynamic_enable(cond_resched); 7522 7520 preempt_dynamic_enable(might_resched); 7523 7521 preempt_dynamic_enable(preempt_schedule); 7524 7522 preempt_dynamic_enable(preempt_schedule_notrace); ··· 7526 7526 7527 7527 switch (mode) { 7528 7528 case preempt_dynamic_none: 7529 - if (!klp_override) 7530 - preempt_dynamic_enable(cond_resched); 7529 + preempt_dynamic_enable(cond_resched); 7531 7530 preempt_dynamic_disable(might_resched); 7532 7531 preempt_dynamic_disable(preempt_schedule); 7533 7532 preempt_dynamic_disable(preempt_schedule_notrace); ··· 7537 7538 break; 7538 7539 7539 7540 case preempt_dynamic_voluntary: 7540 - if (!klp_override) 7541 - preempt_dynamic_enable(cond_resched); 7541 + preempt_dynamic_enable(cond_resched); 7542 7542 preempt_dynamic_enable(might_resched); 7543 7543 preempt_dynamic_disable(preempt_schedule); 7544 7544 preempt_dynamic_disable(preempt_schedule_notrace); ··· 7548 7550 break; 7549 7551 7550 7552 case preempt_dynamic_full: 7551 - if (!klp_override) 7552 - preempt_dynamic_disable(cond_resched); 7553 + preempt_dynamic_disable(cond_resched); 7553 7554 preempt_dynamic_disable(might_resched); 7554 7555 preempt_dynamic_enable(preempt_schedule); 7555 7556 preempt_dynamic_enable(preempt_schedule_notrace); ··· 7559 7562 break; 7560 7563 7561 7564 case preempt_dynamic_lazy: 7562 - if (!klp_override) 7563 - preempt_dynamic_disable(cond_resched); 7565 + preempt_dynamic_disable(cond_resched); 7564 7566 preempt_dynamic_disable(might_resched); 7565 7567 preempt_dynamic_enable(preempt_schedule); 7566 7568 preempt_dynamic_enable(preempt_schedule_notrace); ··· 7579 7583 __sched_dynamic_update(mode); 7580 7584 mutex_unlock(&sched_dynamic_mutex); 7581 7585 } 7582 - 7583 - #ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL 7584 - 7585 - static int klp_cond_resched(void) 7586 - { 7587 - __klp_sched_try_switch(); 7588 - return __cond_resched(); 7589 - } 7590 - 7591 - void sched_dynamic_klp_enable(void) 7592 - { 7593 - mutex_lock(&sched_dynamic_mutex); 7594 - 7595 - klp_override = true; 7596 - static_call_update(cond_resched, klp_cond_resched); 7597 - 7598 - mutex_unlock(&sched_dynamic_mutex); 7599 - } 7600 - 7601 - void sched_dynamic_klp_disable(void) 7602 - { 7603 - mutex_lock(&sched_dynamic_mutex); 7604 - 7605 - klp_override = false; 7606 - __sched_dynamic_update(preempt_dynamic_mode); 7607 - 7608 - mutex_unlock(&sched_dynamic_mutex); 7609 - } 7610 - 7611 - #endif /* CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ 7612 7586 7613 7587 static int __init setup_preempt_mode(char *str) 7614 7588 {

Configure Feed

Configure Feed