Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
sched_ext: Remove scx_ops_enq_* static_keys
scx_ops_enq_last/exiting/migration_disabled are used to encode the
corresponding SCX_OPS_ flags into static_keys. These flags aren't hot enough
for static_key usage to make any meaningful difference and are made
static_keys mostly because there was no reason not to. However, global
static_keys can't work with the planned hierarchical multiple scheduler
support. Remove the static_keys and test the ops flags directly.
In repeated hackbench runs before and after static_keys removal on an AMD
Ryzen 3900X, I couldn't tell any measurable performance difference.
Signed-off-by: Tejun Heo <tj@kernel.org>
Acked-by: Changwoo Min <changwoo@igalia.com>
Acked-by: Andrea Righi <arighi@nvidia.com>
+5
-17
+5
-17
kernel/sched/ext.c
+5
-17
kernel/sched/ext.c
···
924
924
static bool scx_warned_zero_slice;
925
925
926
926
DEFINE_STATIC_KEY_FALSE(scx_ops_allow_queued_wakeup);
927
-
static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last);
928
-
static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting);
929
-
static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_migration_disabled);
930
927
static DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt);
931
928
932
929
static struct static_key_false scx_has_op[SCX_OPI_END] =
···
2141
2144
goto direct;
2142
2145
2143
2146
/* see %SCX_OPS_ENQ_EXITING */
2144
-
if (!static_branch_unlikely(&scx_ops_enq_exiting) &&
2147
+
if (!(scx_ops.flags & SCX_OPS_ENQ_EXITING) &&
2145
2148
unlikely(p->flags & PF_EXITING)) {
2146
2149
__scx_add_event(SCX_EV_ENQ_SKIP_EXITING, 1);
2147
2150
goto local;
2148
2151
}
2149
2152
2150
2153
/* see %SCX_OPS_ENQ_MIGRATION_DISABLED */
2151
-
if (!static_branch_unlikely(&scx_ops_enq_migration_disabled) &&
2154
+
if (!(scx_ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) &&
2152
2155
is_migration_disabled(p)) {
2153
2156
__scx_add_event(SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1);
2154
2157
goto local;
···
3019
3022
* Didn't find another task to run. Keep running @prev unless
3020
3023
* %SCX_OPS_ENQ_LAST is in effect.
3021
3024
*/
3022
-
if (prev_on_rq && (!static_branch_unlikely(&scx_ops_enq_last) ||
3023
-
scx_rq_bypassing(rq))) {
3025
+
if (prev_on_rq &&
3026
+
(!(scx_ops.flags & SCX_OPS_ENQ_LAST) || scx_rq_bypassing(rq))) {
3024
3027
rq->scx.flags |= SCX_RQ_BAL_KEEP;
3025
3028
__scx_add_event(SCX_EV_DISPATCH_KEEP_LAST, 1);
3026
3029
goto has_tasks;
···
3225
3228
* which should trigger an explicit follow-up scheduling event.
3226
3229
*/
3227
3230
if (sched_class_above(&ext_sched_class, next->sched_class)) {
3228
-
WARN_ON_ONCE(!static_branch_unlikely(&scx_ops_enq_last));
3231
+
WARN_ON_ONCE(!(scx_ops.flags & SCX_OPS_ENQ_LAST));
3229
3232
do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
3230
3233
} else {
3231
3234
do_enqueue_task(rq, p, 0, -1);
···
4725
4728
for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)
4726
4729
static_branch_disable(&scx_has_op[i]);
4727
4730
static_branch_disable(&scx_ops_allow_queued_wakeup);
4728
-
static_branch_disable(&scx_ops_enq_last);
4729
-
static_branch_disable(&scx_ops_enq_exiting);
4730
-
static_branch_disable(&scx_ops_enq_migration_disabled);
4731
4731
static_branch_disable(&scx_ops_cpu_preempt);
4732
4732
scx_idle_disable();
4733
4733
synchronize_rcu();
···
5366
5372
5367
5373
if (ops->flags & SCX_OPS_ALLOW_QUEUED_WAKEUP)
5368
5374
static_branch_enable(&scx_ops_allow_queued_wakeup);
5369
-
if (ops->flags & SCX_OPS_ENQ_LAST)
5370
-
static_branch_enable(&scx_ops_enq_last);
5371
-
if (ops->flags & SCX_OPS_ENQ_EXITING)
5372
-
static_branch_enable(&scx_ops_enq_exiting);
5373
-
if (ops->flags & SCX_OPS_ENQ_MIGRATION_DISABLED)
5374
-
static_branch_enable(&scx_ops_enq_migration_disabled);
5375
5375
if (scx_ops.cpu_acquire || scx_ops.cpu_release)
5376
5376
static_branch_enable(&scx_ops_cpu_preempt);
5377
5377