Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
sched_ext: Add SCX_TASK_REENQ_REASON flags
SCX_ENQ_REENQ indicates that a task is being re-enqueued but doesn't tell the
BPF scheduler why. Add SCX_TASK_REENQ_REASON flags using bits 12-13 of
p->scx.flags to communicate the reason during ops.enqueue():
- NONE: Not being reenqueued
- KFUNC: Reenqueued by scx_bpf_dsq_reenq() and friends
More reasons will be added.
Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Andrea Righi <arighi@nvidia.com>
+40
-10
+15
include/linux/sched/ext.h
+15
include/linux/sched/ext.h
···
118
118
SCX_TASK_READY = 2 << SCX_TASK_STATE_SHIFT,
119
119
SCX_TASK_ENABLED = 3 << SCX_TASK_STATE_SHIFT,
120
120
121
+
/*
122
+
* Bits 12 and 13 are used to carry reenqueue reason. In addition to
123
+
* %SCX_ENQ_REENQ flag, ops.enqueue() can also test for
124
+
* %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues.
125
+
*
126
+
* NONE not being reenqueued
127
+
* KFUNC reenqueued by scx_bpf_dsq_reenq() and friends
128
+
*/
129
+
SCX_TASK_REENQ_REASON_SHIFT = 12,
130
+
SCX_TASK_REENQ_REASON_BITS = 2,
131
+
SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT,
132
+
133
+
SCX_TASK_REENQ_NONE = 0 << SCX_TASK_REENQ_REASON_SHIFT,
134
+
SCX_TASK_REENQ_KFUNC = 1 << SCX_TASK_REENQ_REASON_SHIFT,
135
+
121
136
/* iteration cursor, not a task */
122
137
SCX_TASK_CURSOR = 1 << 31,
123
138
};
+22
-3
kernel/sched/ext.c
+22
-3
kernel/sched/ext.c
···
3728
3728
}
3729
3729
}
3730
3730
3731
-
static bool task_should_reenq(struct task_struct *p, u64 reenq_flags)
3731
+
static bool task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason)
3732
3732
{
3733
+
*reason = SCX_TASK_REENQ_KFUNC;
3734
+
3733
3735
if (reenq_flags & SCX_REENQ_ANY)
3734
3736
return true;
3735
3737
return false;
···
3753
3751
list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list,
3754
3752
scx.dsq_list.node) {
3755
3753
struct scx_sched *task_sch = scx_task_sched(p);
3754
+
u32 reason;
3756
3755
3757
3756
/*
3758
3757
* If @p is being migrated, @p's current CPU may not agree with
···
3772
3769
if (!scx_is_descendant(task_sch, sch))
3773
3770
continue;
3774
3771
3775
-
if (!task_should_reenq(p, reenq_flags))
3772
+
if (!task_should_reenq(p, reenq_flags, &reason))
3776
3773
continue;
3777
3774
3778
3775
dispatch_dequeue(rq, p);
3776
+
3777
+
if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
3778
+
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
3779
+
p->scx.flags |= reason;
3780
+
3779
3781
list_add_tail(&p->scx.dsq_list.node, &tasks);
3780
3782
}
3781
3783
3782
3784
list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
3783
3785
list_del_init(&p->scx.dsq_list.node);
3786
+
3784
3787
do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
3788
+
3789
+
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
3785
3790
nr_enqueued++;
3786
3791
}
3787
3792
···
3843
3832
3844
3833
while (likely(!READ_ONCE(sch->bypass_depth))) {
3845
3834
struct rq *task_rq;
3835
+
u32 reason;
3846
3836
3847
3837
p = nldsq_cursor_next_task(&cursor, dsq);
3848
3838
if (!p)
3849
3839
break;
3850
3840
3851
-
if (!task_should_reenq(p, reenq_flags))
3841
+
if (!task_should_reenq(p, reenq_flags, &reason))
3852
3842
continue;
3853
3843
3854
3844
task_rq = task_rq(p);
···
3872
3860
/* @p is on @dsq, its rq and @dsq are locked */
3873
3861
dispatch_dequeue_locked(p, dsq);
3874
3862
raw_spin_unlock(&dsq->lock);
3863
+
3864
+
if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
3865
+
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
3866
+
p->scx.flags |= reason;
3867
+
3875
3868
do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1);
3869
+
3870
+
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
3876
3871
3877
3872
if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) {
3878
3873
raw_spin_rq_unlock(locked_rq);
+3
-7
kernel/sched/ext_internal.h
+3
-7
kernel/sched/ext_internal.h
···
1080
1080
SCX_ENQ_PREEMPT = 1LLU << 32,
1081
1081
1082
1082
/*
1083
-
* The task being enqueued was previously enqueued on the current CPU's
1084
-
* %SCX_DSQ_LOCAL, but was removed from it in a call to the
1085
-
* scx_bpf_reenqueue_local() kfunc. If scx_bpf_reenqueue_local() was
1086
-
* invoked in a ->cpu_release() callback, and the task is again
1087
-
* dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the
1088
-
* task will not be scheduled on the CPU until at least the next invocation
1089
-
* of the ->cpu_acquire() callback.
1083
+
* The task being enqueued was previously enqueued on a DSQ, but was
1084
+
* removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find
1085
+
* out why a given task is being reenqueued.
1090
1086
*/
1091
1087
SCX_ENQ_REENQ = 1LLU << 40,
1092
1088