Merge branch 'for-7.0-fixes' into for-7.1

+54 -26

2 changed files

expand all

kernel

sched

ext.c

ext_idle.c

+35 -14

kernel/sched/ext.c

··· 1590 1590 p->scx.dsq = dsq; 1591 1591 1592 1592 /* 1593 - * scx.ddsp_dsq_id and scx.ddsp_enq_flags are only relevant on the 1594 - * direct dispatch path, but we clear them here because the direct 1595 - * dispatch verdict may be overridden on the enqueue path during e.g. 1596 - * bypass. 1597 - */ 1598 - p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; 1599 - p->scx.ddsp_enq_flags = 0; 1600 - 1601 - /* 1602 1593 * Update custody and call ops.dequeue() before clearing ops_state: 1603 1594 * once ops_state is cleared, waiters in ops_dequeue() can proceed 1604 1595 * and dequeue_task_scx() will RMW p->scx.flags. If we clear ··· 1770 1779 p->scx.ddsp_enq_flags = enq_flags; 1771 1780 } 1772 1781 1782 + /* 1783 + * Clear @p direct dispatch state when leaving the scheduler. 1784 + * 1785 + * Direct dispatch state must be cleared in the following cases: 1786 + * - direct_dispatch(): cleared on the synchronous enqueue path, deferred 1787 + * dispatch keeps the state until consumed 1788 + * - process_ddsp_deferred_locals(): cleared after consuming deferred state, 1789 + * - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch 1790 + * verdict is ignored (local/global/bypass) 1791 + * - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred 1792 + * cancellation and holding_cpu races 1793 + * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by 1794 + * the scx_bypass() loop, so that stale state is not reused by a subsequent 1795 + * scheduler instance 1796 + */ 1797 + static inline void clear_direct_dispatch(struct task_struct *p) 1798 + { 1799 + p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; 1800 + p->scx.ddsp_enq_flags = 0; 1801 + } 1802 + 1773 1803 static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, 1774 1804 u64 enq_flags) 1775 1805 { 1776 1806 struct rq *rq = task_rq(p); 1777 1807 struct scx_dispatch_q *dsq = 1778 1808 find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); 1809 + u64 ddsp_enq_flags; 1779 1810 1780 1811 touch_core_sched_dispatch(rq, p); 1781 1812 ··· 1838 1825 return; 1839 1826 } 1840 1827 1841 - dispatch_enqueue(sch, rq, dsq, p, 1842 - p->scx.ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); 1828 + ddsp_enq_flags = p->scx.ddsp_enq_flags; 1829 + clear_direct_dispatch(p); 1830 + 1831 + dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); 1843 1832 } 1844 1833 1845 1834 static bool scx_rq_online(struct rq *rq) ··· 1964 1949 */ 1965 1950 touch_core_sched(rq, p); 1966 1951 refill_task_slice_dfl(sch, p); 1952 + clear_direct_dispatch(p); 1967 1953 dispatch_enqueue(sch, rq, dsq, p, enq_flags); 1968 1954 } 1969 1955 ··· 2158 2142 sub_nr_running(rq, 1); 2159 2143 2160 2144 dispatch_dequeue(rq, p); 2145 + clear_direct_dispatch(p); 2161 2146 return true; 2162 2147 } 2163 2148 ··· 3681 3664 lockdep_assert_rq_held(rq); 3682 3665 WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED); 3683 3666 3667 + clear_direct_dispatch(p); 3668 + 3684 3669 if (SCX_HAS_OP(sch, disable)) 3685 3670 SCX_CALL_OP_TASK(sch, SCX_KF_REST, disable, rq, p); 3686 3671 scx_set_task_state(p, SCX_TASK_READY); ··· 3967 3948 struct task_struct, scx.dsq_list.node))) { 3968 3949 struct scx_sched *sch = scx_task_sched(p); 3969 3950 struct scx_dispatch_q *dsq; 3951 + u64 dsq_id = p->scx.ddsp_dsq_id; 3952 + u64 enq_flags = p->scx.ddsp_enq_flags; 3970 3953 3971 3954 list_del_init(&p->scx.dsq_list.node); 3955 + clear_direct_dispatch(p); 3972 3956 3973 - dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); 3957 + dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); 3974 3958 if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) 3975 - dispatch_to_local_dsq(sch, rq, dsq, p, 3976 - p->scx.ddsp_enq_flags); 3959 + dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); 3977 3960 } 3978 3961 } 3979 3962

+19 -12

kernel/sched/ext_idle.c

··· 881 881 * code. 882 882 * 883 883 * We can't simply check whether @p->migration_disabled is set in a 884 - * sched_ext callback, because migration is always disabled for the current 885 - * task while running BPF code. 884 + * sched_ext callback, because the BPF prolog (__bpf_prog_enter) may disable 885 + * migration for the current task while running BPF code. 886 886 * 887 - * The prolog (__bpf_prog_enter) and epilog (__bpf_prog_exit) respectively 888 - * disable and re-enable migration. For this reason, the current task 889 - * inside a sched_ext callback is always a migration-disabled task. 887 + * Since the BPF prolog calls migrate_disable() only when CONFIG_PREEMPT_RCU 888 + * is enabled (via rcu_read_lock_dont_migrate()), migration_disabled == 1 for 889 + * the current task is ambiguous only in that case: it could be from the BPF 890 + * prolog rather than a real migrate_disable() call. 890 891 * 891 - * Therefore, when @p->migration_disabled == 1, check whether @p is the 892 - * current task or not: if it is, then migration was not disabled before 893 - * entering the callback, otherwise migration was disabled. 892 + * Without CONFIG_PREEMPT_RCU, the BPF prolog never calls migrate_disable(), 893 + * so migration_disabled == 1 always means the task is truly 894 + * migration-disabled. 895 + * 896 + * Therefore, when migration_disabled == 1 and CONFIG_PREEMPT_RCU is enabled, 897 + * check whether @p is the current task or not: if it is, then migration was 898 + * not disabled before entering the callback, otherwise migration was disabled. 894 899 * 895 900 * Returns true if @p is migration-disabled, false otherwise. 896 901 */ 897 902 static bool is_bpf_migration_disabled(const struct task_struct *p) 898 903 { 899 - if (p->migration_disabled == 1) 900 - return p != current; 901 - else 902 - return p->migration_disabled; 904 + if (p->migration_disabled == 1) { 905 + if (IS_ENABLED(CONFIG_PREEMPT_RCU)) 906 + return p != current; 907 + return true; 908 + } 909 + return p->migration_disabled; 903 910 } 904 911 905 912 static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p,

Configure Feed

Configure Feed