Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
sched_ext: Convert deferred_reenq_locals from llist to regular list
The deferred reenqueue local mechanism uses an llist (lockless list) for
collecting schedulers that need their local DSQs re-enqueued. Convert to a
regular list protected by a raw_spinlock.
The llist was used for its lockless properties, but the upcoming changes to
support remote reenqueue require more complex list operations that are
difficult to implement correctly with lockless data structures. A spinlock-
protected regular list provides the necessary flexibility.
Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Andrea Righi <arighi@nvidia.com>
+41
-21
+38
-19
kernel/sched/ext.c
+38
-19
kernel/sched/ext.c
···
3639
3639
return nr_enqueued;
3640
3640
}
3641
3641
3642
+
static void process_deferred_reenq_locals(struct rq *rq)
3643
+
{
3644
+
lockdep_assert_rq_held(rq);
3645
+
3646
+
while (true) {
3647
+
struct scx_sched *sch;
3648
+
3649
+
scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) {
3650
+
struct scx_sched_pcpu *sch_pcpu =
3651
+
list_first_entry_or_null(&rq->scx.deferred_reenq_locals,
3652
+
struct scx_sched_pcpu,
3653
+
deferred_reenq_local_node);
3654
+
if (!sch_pcpu)
3655
+
return;
3656
+
3657
+
sch = sch_pcpu->sch;
3658
+
list_del_init(&sch_pcpu->deferred_reenq_local_node);
3659
+
}
3660
+
3661
+
reenq_local(sch, rq);
3662
+
}
3663
+
}
3664
+
3642
3665
static void run_deferred(struct rq *rq)
3643
3666
{
3644
3667
process_ddsp_deferred_locals(rq);
3645
3668
3646
-
if (!llist_empty(&rq->scx.deferred_reenq_locals)) {
3647
-
struct llist_node *llist =
3648
-
llist_del_all(&rq->scx.deferred_reenq_locals);
3649
-
struct scx_sched_pcpu *pos, *next;
3650
-
3651
-
llist_for_each_entry_safe(pos, next, llist,
3652
-
deferred_reenq_locals_node) {
3653
-
init_llist_node(&pos->deferred_reenq_locals_node);
3654
-
reenq_local(pos->sch, rq);
3655
-
}
3656
-
}
3669
+
if (!list_empty(&rq->scx.deferred_reenq_locals))
3670
+
process_deferred_reenq_locals(rq);
3657
3671
}
3658
3672
3659
3673
#ifdef CONFIG_NO_HZ_FULL
···
4193
4179
4194
4180
/*
4195
4181
* $sch would have entered bypass mode before the RCU grace period. As
4196
-
* that blocks new deferrals, all deferred_reenq_locals_node's must be
4182
+
* that blocks new deferrals, all deferred_reenq_local_node's must be
4197
4183
* off-list by now.
4198
4184
*/
4199
4185
for_each_possible_cpu(cpu) {
4200
4186
struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
4201
4187
4202
-
WARN_ON_ONCE(llist_on_list(&pcpu->deferred_reenq_locals_node));
4188
+
WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local_node));
4203
4189
}
4204
4190
4205
4191
free_percpu(sch->pcpu);
···
5812
5798
struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
5813
5799
5814
5800
pcpu->sch = sch;
5815
-
init_llist_node(&pcpu->deferred_reenq_locals_node);
5801
+
INIT_LIST_HEAD(&pcpu->deferred_reenq_local_node);
5816
5802
}
5817
5803
5818
5804
sch->helper = kthread_run_worker(0, "sched_ext_helper");
···
7139
7125
BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n));
7140
7126
BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n));
7141
7127
BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n));
7142
-
init_llist_head(&rq->scx.deferred_reenq_locals);
7128
+
raw_spin_lock_init(&rq->scx.deferred_reenq_lock);
7129
+
INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals);
7143
7130
rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn);
7144
7131
rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn);
7145
7132
···
8372
8357
unsigned long flags;
8373
8358
struct scx_sched *sch;
8374
8359
struct rq *rq;
8375
-
struct llist_node *lnode;
8376
8360
8377
8361
raw_local_irq_save(flags);
8378
8362
···
8387
8373
goto out_irq_restore;
8388
8374
8389
8375
rq = this_rq();
8390
-
lnode = &this_cpu_ptr(sch->pcpu)->deferred_reenq_locals_node;
8391
-
if (!llist_on_list(lnode))
8392
-
llist_add(lnode, &rq->scx.deferred_reenq_locals);
8376
+
scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) {
8377
+
struct scx_sched_pcpu *pcpu = this_cpu_ptr(sch->pcpu);
8378
+
8379
+
if (list_empty(&pcpu->deferred_reenq_local_node))
8380
+
list_move_tail(&pcpu->deferred_reenq_local_node,
8381
+
&rq->scx.deferred_reenq_locals);
8382
+
}
8383
+
8393
8384
schedule_deferred(rq);
8394
8385
out_irq_restore:
8395
8386
raw_local_irq_restore(flags);
+1
-1
kernel/sched/ext_internal.h
+1
-1
kernel/sched/ext_internal.h
+2
-1
kernel/sched/sched.h
+2
-1
kernel/sched/sched.h
···
808
808
809
809
struct task_struct *sub_dispatch_prev;
810
810
811
-
struct llist_head deferred_reenq_locals;
811
+
raw_spinlock_t deferred_reenq_lock;
812
+
struct list_head deferred_reenq_locals; /* scheds requesting reenq of local DSQ */
812
813
struct balance_callback deferred_bal_cb;
813
814
struct irq_work deferred_irq_work;
814
815
struct irq_work kick_cpus_irq_work;