tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

sched_ext: Simplify breather mechanism with scx_aborting flag

The breather mechanism was introduced in 62dcbab8b0ef ("sched_ext: Avoid
live-locking bypass mode switching") and e32c260195e6 ("sched_ext: Enable the
ops breather and eject BPF scheduler on softlockup") to prevent live-locks by
injecting delays when CPUs are trapped in dispatch paths.

Currently, it uses scx_breather_depth (atomic_t) and scx_in_softlockup
(unsigned long) with separate increment/decrement and cleanup operations. The
breather is only activated when aborting, so tie it directly to the exit
mechanism. Replace both variables with scx_aborting flag set when exit is
claimed and cleared after bypass is enabled. Introduce scx_claim_exit() to
consolidate exit_kind claiming and breather enablement. This eliminates
scx_clear_softlockup() and simplifies scx_softlockup() and scx_bypass().

The breather mechanism will be replaced by a different abort mechanism in a
future patch. This simplification prepares for that change.

Reviewed-by: Dan Schatzberg <schatzberg.dan@gmail.com>
Reviewed-by: Emil Tsalapatis <emil@etsalapatis.com>
Acked-by: Andrea Righi <arighi@nvidia.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

Tejun Heo a69040ed 61debc25

+25 -29
+25 -29
kernel/sched/ext.c
··· 33 33 DEFINE_STATIC_KEY_FALSE(__scx_enabled); 34 34 DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); 35 35 static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); 36 - static unsigned long scx_in_softlockup; 37 - static atomic_t scx_breather_depth = ATOMIC_INIT(0); 38 36 static int scx_bypass_depth; 37 + static bool scx_aborting; 39 38 static bool scx_init_task_enabled; 40 39 static bool scx_switching_all; 41 40 DEFINE_STATIC_KEY_FALSE(__scx_switched_all); ··· 1830 1831 1831 1832 lockdep_assert_rq_held(rq); 1832 1833 1833 - if (likely(!atomic_read(&scx_breather_depth))) 1834 + if (likely(!READ_ONCE(scx_aborting))) 1834 1835 return; 1835 1836 1836 1837 raw_spin_rq_unlock(rq); ··· 1839 1840 1840 1841 do { 1841 1842 int cnt = 1024; 1842 - while (atomic_read(&scx_breather_depth) && --cnt) 1843 + while (READ_ONCE(scx_aborting) && --cnt) 1843 1844 cpu_relax(); 1844 - } while (atomic_read(&scx_breather_depth) && 1845 + } while (READ_ONCE(scx_aborting) && 1845 1846 time_before64(ktime_get_ns(), until)); 1846 1847 1847 1848 raw_spin_rq_lock(rq); ··· 3740 3741 goto out_unlock; 3741 3742 } 3742 3743 3743 - /* allow only one instance, cleared at the end of scx_bypass() */ 3744 - if (test_and_set_bit(0, &scx_in_softlockup)) 3745 - goto out_unlock; 3746 - 3747 3744 printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU%d stuck for %us, disabling \"%s\"\n", 3748 3745 smp_processor_id(), dur_s, scx_root->ops.name); 3749 - 3750 - /* 3751 - * Some CPUs may be trapped in the dispatch paths. Enable breather 3752 - * immediately; otherwise, we might even be able to get to scx_bypass(). 3753 - */ 3754 - atomic_inc(&scx_breather_depth); 3755 3746 3756 3747 scx_error(sch, "soft lockup - CPU#%d stuck for %us", smp_processor_id(), dur_s); 3757 3748 out_unlock: 3758 3749 rcu_read_unlock(); 3759 - } 3760 - 3761 - static void scx_clear_softlockup(void) 3762 - { 3763 - if (test_and_clear_bit(0, &scx_in_softlockup)) 3764 - atomic_dec(&scx_breather_depth); 3765 3750 } 3766 3751 3767 3752 /** ··· 3810 3827 ktime_get_ns() - bypass_timestamp); 3811 3828 } 3812 3829 3813 - atomic_inc(&scx_breather_depth); 3814 - 3815 3830 /* 3816 3831 * No task property is changing. We just need to make sure all currently 3817 3832 * queued tasks are re-queued according to the new scx_rq_bypassing() ··· 3865 3884 raw_spin_rq_unlock(rq); 3866 3885 } 3867 3886 3868 - atomic_dec(&scx_breather_depth); 3869 3887 unlock: 3870 3888 raw_spin_unlock_irqrestore(&bypass_lock, flags); 3871 - scx_clear_softlockup(); 3872 3889 } 3873 3890 3874 3891 static void free_exit_info(struct scx_exit_info *ei) ··· 3961 3982 3962 3983 /* guarantee forward progress by bypassing scx_ops */ 3963 3984 scx_bypass(true); 3985 + WRITE_ONCE(scx_aborting, false); 3964 3986 3965 3987 switch (scx_set_enable_state(SCX_DISABLING)) { 3966 3988 case SCX_DISABLING: ··· 4084 4104 scx_bypass(false); 4085 4105 } 4086 4106 4087 - static void scx_disable(enum scx_exit_kind kind) 4107 + static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) 4088 4108 { 4089 4109 int none = SCX_EXIT_NONE; 4110 + 4111 + if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) 4112 + return false; 4113 + 4114 + /* 4115 + * Some CPUs may be trapped in the dispatch paths. Enable breather 4116 + * immediately; otherwise, we might not even be able to get to 4117 + * scx_bypass(). 4118 + */ 4119 + WRITE_ONCE(scx_aborting, true); 4120 + return true; 4121 + } 4122 + 4123 + static void scx_disable(enum scx_exit_kind kind) 4124 + { 4090 4125 struct scx_sched *sch; 4091 4126 4092 4127 if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) ··· 4110 4115 rcu_read_lock(); 4111 4116 sch = rcu_dereference(scx_root); 4112 4117 if (sch) { 4113 - atomic_try_cmpxchg(&sch->exit_kind, &none, kind); 4118 + scx_claim_exit(sch, kind); 4114 4119 kthread_queue_work(sch->helper, &sch->disable_work); 4115 4120 } 4116 4121 rcu_read_unlock(); ··· 4431 4436 const char *fmt, va_list args) 4432 4437 { 4433 4438 struct scx_exit_info *ei = sch->exit_info; 4434 - int none = SCX_EXIT_NONE; 4435 4439 4436 - if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) 4440 + if (!scx_claim_exit(sch, kind)) 4437 4441 return; 4438 4442 4439 4443 ei->exit_code = exit_code; ··· 4648 4654 */ 4649 4655 WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); 4650 4656 WARN_ON_ONCE(scx_root); 4657 + if (WARN_ON_ONCE(READ_ONCE(scx_aborting))) 4658 + WRITE_ONCE(scx_aborting, false); 4651 4659 4652 4660 atomic_long_set(&scx_nr_rejected, 0); 4653 4661