tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

sched_ext: Optimize schedule_dsq_reenq() with lockless fast path

schedule_dsq_reenq() always acquires deferred_reenq_lock to queue a reenqueue
request. Add a lockless fast-path to skip lock acquisition when the request is
already pending with the required flags set.

Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Andrea Righi <arighi@nvidia.com>

Tejun Heo a90449b1 84b1a0ea

+36 -8
+36 -8
kernel/sched/ext.c
··· 1174 1174 struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq)); 1175 1175 struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local; 1176 1176 1177 - scoped_guard (raw_spinlock_irqsave, &rq->scx.deferred_reenq_lock) { 1177 + /* 1178 + * Pairs with smp_mb() in process_deferred_reenq_locals() and 1179 + * guarantees that there is a reenq_local() afterwards. 1180 + */ 1181 + smp_mb(); 1182 + 1183 + if (list_empty(&drl->node) || 1184 + (READ_ONCE(drl->flags) & reenq_flags) != reenq_flags) { 1185 + 1186 + guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); 1187 + 1178 1188 if (list_empty(&drl->node)) 1179 1189 list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals); 1180 - drl->flags |= reenq_flags; 1190 + WRITE_ONCE(drl->flags, drl->flags | reenq_flags); 1181 1191 } 1182 1192 1183 1193 schedule_deferred(rq); ··· 1196 1186 struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); 1197 1187 struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user; 1198 1188 1199 - scoped_guard (raw_spinlock_irqsave, &rq->scx.deferred_reenq_lock) { 1189 + /* 1190 + * Pairs with smp_mb() in process_deferred_reenq_users() and 1191 + * guarantees that there is a reenq_user() afterwards. 1192 + */ 1193 + smp_mb(); 1194 + 1195 + if (list_empty(&dru->node) || 1196 + (READ_ONCE(dru->flags) & reenq_flags) != reenq_flags) { 1197 + 1198 + guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); 1199 + 1200 1200 if (list_empty(&dru->node)) 1201 1201 list_move_tail(&dru->node, &rq->scx.deferred_reenq_users); 1202 - dru->flags |= reenq_flags; 1202 + WRITE_ONCE(dru->flags, dru->flags | reenq_flags); 1203 1203 } 1204 1204 1205 1205 schedule_deferred(rq); ··· 3793 3773 3794 3774 while (true) { 3795 3775 struct scx_sched *sch; 3796 - u64 reenq_flags = 0; 3776 + u64 reenq_flags; 3797 3777 3798 3778 scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { 3799 3779 struct scx_deferred_reenq_local *drl = ··· 3808 3788 sch_pcpu = container_of(drl, struct scx_sched_pcpu, 3809 3789 deferred_reenq_local); 3810 3790 sch = sch_pcpu->sch; 3811 - swap(drl->flags, reenq_flags); 3791 + reenq_flags = drl->flags; 3792 + WRITE_ONCE(drl->flags, 0); 3812 3793 list_del_init(&drl->node); 3813 3794 } 3795 + 3796 + /* see schedule_dsq_reenq() */ 3797 + smp_mb(); 3814 3798 3815 3799 reenq_local(sch, rq, reenq_flags); 3816 3800 } ··· 3889 3865 3890 3866 while (true) { 3891 3867 struct scx_dispatch_q *dsq; 3892 - u64 reenq_flags = 0; 3868 + u64 reenq_flags; 3893 3869 3894 3870 scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { 3895 3871 struct scx_deferred_reenq_user *dru = ··· 3904 3880 dsq_pcpu = container_of(dru, struct scx_dsq_pcpu, 3905 3881 deferred_reenq_user); 3906 3882 dsq = dsq_pcpu->dsq; 3907 - swap(dru->flags, reenq_flags); 3883 + reenq_flags = dru->flags; 3884 + WRITE_ONCE(dru->flags, 0); 3908 3885 list_del_init(&dru->node); 3909 3886 } 3887 + 3888 + /* see schedule_dsq_reenq() */ 3889 + smp_mb(); 3910 3890 3911 3891 BUG_ON(dsq->id & SCX_DSQ_FLAG_BUILTIN); 3912 3892 reenq_user(rq, dsq, reenq_flags);