Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
sched_ext: Relocate functions in kernel/sched/ext.c
Relocate functions to ease the removal of switch_class_scx(). No functional
changes.
Signed-off-by: Tejun Heo <tj@kernel.org>
+78
-78
+78
-78
kernel/sched/ext.c
+78
-78
kernel/sched/ext.c
···
2642
2642
return ret;
2643
2643
}
2644
2644
2645
+
static void process_ddsp_deferred_locals(struct rq *rq)
2646
+
{
2647
+
struct task_struct *p;
2648
+
2649
+
lockdep_assert_rq_held(rq);
2650
+
2651
+
/*
2652
+
* Now that @rq can be unlocked, execute the deferred enqueueing of
2653
+
* tasks directly dispatched to the local DSQs of other CPUs. See
2654
+
* direct_dispatch(). Keep popping from the head instead of using
2655
+
* list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq
2656
+
* temporarily.
2657
+
*/
2658
+
while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals,
2659
+
struct task_struct, scx.dsq_list.node))) {
2660
+
s32 ret;
2661
+
2662
+
list_del_init(&p->scx.dsq_list.node);
2663
+
2664
+
ret = dispatch_to_local_dsq(rq, p->scx.ddsp_dsq_id, p,
2665
+
p->scx.ddsp_enq_flags);
2666
+
WARN_ON_ONCE(ret == DTL_NOT_LOCAL);
2667
+
}
2668
+
}
2669
+
2645
2670
static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
2646
2671
{
2647
2672
if (p->scx.flags & SCX_TASK_QUEUED) {
···
2709
2684
}
2710
2685
}
2711
2686
2712
-
static void process_ddsp_deferred_locals(struct rq *rq)
2687
+
static enum scx_cpu_preempt_reason
2688
+
preempt_reason_from_class(const struct sched_class *class)
2713
2689
{
2714
-
struct task_struct *p;
2690
+
#ifdef CONFIG_SMP
2691
+
if (class == &stop_sched_class)
2692
+
return SCX_CPU_PREEMPT_STOP;
2693
+
#endif
2694
+
if (class == &dl_sched_class)
2695
+
return SCX_CPU_PREEMPT_DL;
2696
+
if (class == &rt_sched_class)
2697
+
return SCX_CPU_PREEMPT_RT;
2698
+
return SCX_CPU_PREEMPT_UNKNOWN;
2699
+
}
2715
2700
2716
-
lockdep_assert_rq_held(rq);
2701
+
static void switch_class_scx(struct rq *rq, struct task_struct *next)
2702
+
{
2703
+
const struct sched_class *next_class = next->sched_class;
2704
+
2705
+
if (!scx_enabled())
2706
+
return;
2707
+
#ifdef CONFIG_SMP
2708
+
/*
2709
+
* Pairs with the smp_load_acquire() issued by a CPU in
2710
+
* kick_cpus_irq_workfn() who is waiting for this CPU to perform a
2711
+
* resched.
2712
+
*/
2713
+
smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1);
2714
+
#endif
2715
+
if (!static_branch_unlikely(&scx_ops_cpu_preempt))
2716
+
return;
2717
2717
2718
2718
/*
2719
-
* Now that @rq can be unlocked, execute the deferred enqueueing of
2720
-
* tasks directly dispatched to the local DSQs of other CPUs. See
2721
-
* direct_dispatch(). Keep popping from the head instead of using
2722
-
* list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq
2723
-
* temporarily.
2719
+
* The callback is conceptually meant to convey that the CPU is no
2720
+
* longer under the control of SCX. Therefore, don't invoke the callback
2721
+
* if the next class is below SCX (in which case the BPF scheduler has
2722
+
* actively decided not to schedule any tasks on the CPU).
2724
2723
*/
2725
-
while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals,
2726
-
struct task_struct, scx.dsq_list.node))) {
2727
-
s32 ret;
2724
+
if (sched_class_above(&ext_sched_class, next_class))
2725
+
return;
2728
2726
2729
-
list_del_init(&p->scx.dsq_list.node);
2727
+
/*
2728
+
* At this point we know that SCX was preempted by a higher priority
2729
+
* sched_class, so invoke the ->cpu_release() callback if we have not
2730
+
* done so already. We only send the callback once between SCX being
2731
+
* preempted, and it regaining control of the CPU.
2732
+
*
2733
+
* ->cpu_release() complements ->cpu_acquire(), which is emitted the
2734
+
* next time that balance_scx() is invoked.
2735
+
*/
2736
+
if (!rq->scx.cpu_released) {
2737
+
if (SCX_HAS_OP(cpu_release)) {
2738
+
struct scx_cpu_release_args args = {
2739
+
.reason = preempt_reason_from_class(next_class),
2740
+
.task = next,
2741
+
};
2730
2742
2731
-
ret = dispatch_to_local_dsq(rq, p->scx.ddsp_dsq_id, p,
2732
-
p->scx.ddsp_enq_flags);
2733
-
WARN_ON_ONCE(ret == DTL_NOT_LOCAL);
2743
+
SCX_CALL_OP(SCX_KF_CPU_RELEASE,
2744
+
cpu_release, cpu_of(rq), &args);
2745
+
}
2746
+
rq->scx.cpu_released = true;
2734
2747
}
2735
2748
}
2736
2749
···
2883
2820
return time_after64(a->scx.core_sched_at, b->scx.core_sched_at);
2884
2821
}
2885
2822
#endif /* CONFIG_SCHED_CORE */
2886
-
2887
-
static enum scx_cpu_preempt_reason
2888
-
preempt_reason_from_class(const struct sched_class *class)
2889
-
{
2890
-
#ifdef CONFIG_SMP
2891
-
if (class == &stop_sched_class)
2892
-
return SCX_CPU_PREEMPT_STOP;
2893
-
#endif
2894
-
if (class == &dl_sched_class)
2895
-
return SCX_CPU_PREEMPT_DL;
2896
-
if (class == &rt_sched_class)
2897
-
return SCX_CPU_PREEMPT_RT;
2898
-
return SCX_CPU_PREEMPT_UNKNOWN;
2899
-
}
2900
-
2901
-
static void switch_class_scx(struct rq *rq, struct task_struct *next)
2902
-
{
2903
-
const struct sched_class *next_class = next->sched_class;
2904
-
2905
-
if (!scx_enabled())
2906
-
return;
2907
-
#ifdef CONFIG_SMP
2908
-
/*
2909
-
* Pairs with the smp_load_acquire() issued by a CPU in
2910
-
* kick_cpus_irq_workfn() who is waiting for this CPU to perform a
2911
-
* resched.
2912
-
*/
2913
-
smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1);
2914
-
#endif
2915
-
if (!static_branch_unlikely(&scx_ops_cpu_preempt))
2916
-
return;
2917
-
2918
-
/*
2919
-
* The callback is conceptually meant to convey that the CPU is no
2920
-
* longer under the control of SCX. Therefore, don't invoke the callback
2921
-
* if the next class is below SCX (in which case the BPF scheduler has
2922
-
* actively decided not to schedule any tasks on the CPU).
2923
-
*/
2924
-
if (sched_class_above(&ext_sched_class, next_class))
2925
-
return;
2926
-
2927
-
/*
2928
-
* At this point we know that SCX was preempted by a higher priority
2929
-
* sched_class, so invoke the ->cpu_release() callback if we have not
2930
-
* done so already. We only send the callback once between SCX being
2931
-
* preempted, and it regaining control of the CPU.
2932
-
*
2933
-
* ->cpu_release() complements ->cpu_acquire(), which is emitted the
2934
-
* next time that balance_scx() is invoked.
2935
-
*/
2936
-
if (!rq->scx.cpu_released) {
2937
-
if (SCX_HAS_OP(cpu_release)) {
2938
-
struct scx_cpu_release_args args = {
2939
-
.reason = preempt_reason_from_class(next_class),
2940
-
.task = next,
2941
-
};
2942
-
2943
-
SCX_CALL_OP(SCX_KF_CPU_RELEASE,
2944
-
cpu_release, cpu_of(rq), &args);
2945
-
}
2946
-
rq->scx.cpu_released = true;
2947
-
}
2948
-
}
2949
2823
2950
2824
#ifdef CONFIG_SMP
2951
2825