tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

sched_ext: Remove runtime kfunc mask enforcement

Now that scx_kfunc_context_filter enforces context-sensitive kfunc
restrictions at BPF load time, the per-task runtime enforcement via
scx_kf_mask is redundant. Remove it entirely:

- Delete enum scx_kf_mask, the kf_mask field on sched_ext_entity, and
the scx_kf_allow()/scx_kf_disallow()/scx_kf_allowed() helpers along
with the higher_bits()/highest_bit() helpers they used.
- Strip the @mask parameter (and the BUILD_BUG_ON checks) from the
SCX_CALL_OP[_RET]/SCX_CALL_OP_TASK[_RET]/SCX_CALL_OP_2TASKS_RET
macros and update every call site. Reflow call sites that were
wrapped only to fit the old 5-arg form and now collapse onto a single
line under ~100 cols.
- Remove the in-kfunc scx_kf_allowed() runtime checks from
scx_dsq_insert_preamble(), scx_dsq_move(), scx_bpf_dispatch_nr_slots(),
scx_bpf_dispatch_cancel(), scx_bpf_dsq_move_to_local___v2(),
scx_bpf_sub_dispatch(), scx_bpf_reenqueue_local(), and the per-call
guard inside select_cpu_from_kfunc().

scx_bpf_task_cgroup() and scx_kf_allowed_on_arg_tasks() were already
cleaned up in the "drop redundant rq-locked check" patch.
scx_kf_allowed_if_unlocked() was rewritten in the preceding "decouple"
patch. No further changes to those helpers here.

Co-developed-by: Juntong Deng <juntong.deng@outlook.com>
Signed-off-by: Juntong Deng <juntong.deng@outlook.com>
Signed-off-by: Cheng-Yang Chou <yphbchou0911@gmail.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Andrea Righi <arighi@nvidia.com>

authored by

Cheng-Yang Chou and committed by
Tejun Heo 7cd9a5d7 d1d3c1c6

+58 -218
-28
include/linux/sched/ext.h
··· 147 147 SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ 148 148 }; 149 149 150 - /* 151 - * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from 152 - * everywhere and the following bits track which kfunc sets are currently 153 - * allowed for %current. This simple per-task tracking works because SCX ops 154 - * nest in a limited way. BPF will likely implement a way to allow and disallow 155 - * kfuncs depending on the calling context which will replace this manual 156 - * mechanism. See scx_kf_allow(). 157 - */ 158 - enum scx_kf_mask { 159 - SCX_KF_UNLOCKED = 0, /* sleepable and not rq locked */ 160 - /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ 161 - SCX_KF_CPU_RELEASE = 1 << 0, /* ops.cpu_release() */ 162 - /* 163 - * ops.dispatch() may release rq lock temporarily and thus ENQUEUE and 164 - * SELECT_CPU may be nested inside. ops.dequeue (in REST) may also be 165 - * nested inside DISPATCH. 166 - */ 167 - SCX_KF_DISPATCH = 1 << 1, /* ops.dispatch() */ 168 - SCX_KF_ENQUEUE = 1 << 2, /* ops.enqueue() and ops.select_cpu() */ 169 - SCX_KF_SELECT_CPU = 1 << 3, /* ops.select_cpu() */ 170 - SCX_KF_REST = 1 << 4, /* other rq-locked operations */ 171 - 172 - __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | 173 - SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU | SCX_KF_REST, 174 - __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU | SCX_KF_REST, 175 - }; 176 - 177 150 enum scx_dsq_lnode_flags { 178 151 SCX_DSQ_LNODE_ITER_CURSOR = 1 << 0, 179 152 ··· 194 221 s32 sticky_cpu; 195 222 s32 holding_cpu; 196 223 s32 selected_cpu; 197 - u32 kf_mask; /* see scx_kf_mask above */ 198 224 struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ 199 225 200 226 struct list_head runnable_node; /* rq->scx.runnable_list */
+57 -187
kernel/sched/ext.c
··· 229 229 return -(long)jiffies_to_msecs(now - at); 230 230 } 231 231 232 - /* if the highest set bit is N, return a mask with bits [N+1, 31] set */ 233 - static u32 higher_bits(u32 flags) 234 - { 235 - return ~((1 << fls(flags)) - 1); 236 - } 237 - 238 - /* return the mask with only the highest bit set */ 239 - static u32 highest_bit(u32 flags) 240 - { 241 - int bit = fls(flags); 242 - return ((u64)1 << bit) >> 1; 243 - } 244 - 245 232 static bool u32_before(u32 a, u32 b) 246 233 { 247 234 return (s32)(a - b) < 0; ··· 450 463 } 451 464 452 465 /* 453 - * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX 454 - * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate 455 - * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check 456 - * whether it's running from an allowed context. 457 - * 458 - * @mask is constant, always inline to cull the mask calculations. 459 - */ 460 - static __always_inline void scx_kf_allow(u32 mask) 461 - { 462 - /* nesting is allowed only in increasing scx_kf_mask order */ 463 - WARN_ONCE((mask | higher_bits(mask)) & current->scx.kf_mask, 464 - "invalid nesting current->scx.kf_mask=0x%x mask=0x%x\n", 465 - current->scx.kf_mask, mask); 466 - current->scx.kf_mask |= mask; 467 - barrier(); 468 - } 469 - 470 - static void scx_kf_disallow(u32 mask) 471 - { 472 - barrier(); 473 - current->scx.kf_mask &= ~mask; 474 - } 475 - 476 - /* 477 466 * Track the rq currently locked. 478 467 * 479 468 * This allows kfuncs to safely operate on rq from any scx ops callback, ··· 469 506 __this_cpu_write(scx_locked_rq_state, rq); 470 507 } 471 508 472 - #define SCX_CALL_OP(sch, mask, op, rq, args...) \ 509 + #define SCX_CALL_OP(sch, op, rq, args...) \ 473 510 do { \ 474 511 if (rq) \ 475 512 update_locked_rq(rq); \ 476 - if (mask) { \ 477 - scx_kf_allow(mask); \ 478 - (sch)->ops.op(args); \ 479 - scx_kf_disallow(mask); \ 480 - } else { \ 481 - (sch)->ops.op(args); \ 482 - } \ 513 + (sch)->ops.op(args); \ 483 514 if (rq) \ 484 515 update_locked_rq(NULL); \ 485 516 } while (0) 486 517 487 - #define SCX_CALL_OP_RET(sch, mask, op, rq, args...) \ 518 + #define SCX_CALL_OP_RET(sch, op, rq, args...) \ 488 519 ({ \ 489 520 __typeof__((sch)->ops.op(args)) __ret; \ 490 521 \ 491 522 if (rq) \ 492 523 update_locked_rq(rq); \ 493 - if (mask) { \ 494 - scx_kf_allow(mask); \ 495 - __ret = (sch)->ops.op(args); \ 496 - scx_kf_disallow(mask); \ 497 - } else { \ 498 - __ret = (sch)->ops.op(args); \ 499 - } \ 524 + __ret = (sch)->ops.op(args); \ 500 525 if (rq) \ 501 526 update_locked_rq(NULL); \ 502 527 __ret; \ ··· 504 553 * 505 554 * These macros only work for non-nesting ops since kf_tasks[] is not stacked. 506 555 */ 507 - #define SCX_CALL_OP_TASK(sch, mask, op, rq, task, args...) \ 556 + #define SCX_CALL_OP_TASK(sch, op, rq, task, args...) \ 508 557 do { \ 509 - BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ 510 558 current->scx.kf_tasks[0] = task; \ 511 - SCX_CALL_OP((sch), mask, op, rq, task, ##args); \ 559 + SCX_CALL_OP((sch), op, rq, task, ##args); \ 512 560 current->scx.kf_tasks[0] = NULL; \ 513 561 } while (0) 514 562 515 - #define SCX_CALL_OP_TASK_RET(sch, mask, op, rq, task, args...) \ 563 + #define SCX_CALL_OP_TASK_RET(sch, op, rq, task, args...) \ 516 564 ({ \ 517 565 __typeof__((sch)->ops.op(task, ##args)) __ret; \ 518 - BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ 519 566 current->scx.kf_tasks[0] = task; \ 520 - __ret = SCX_CALL_OP_RET((sch), mask, op, rq, task, ##args); \ 567 + __ret = SCX_CALL_OP_RET((sch), op, rq, task, ##args); \ 521 568 current->scx.kf_tasks[0] = NULL; \ 522 569 __ret; \ 523 570 }) 524 571 525 - #define SCX_CALL_OP_2TASKS_RET(sch, mask, op, rq, task0, task1, args...) \ 572 + #define SCX_CALL_OP_2TASKS_RET(sch, op, rq, task0, task1, args...) \ 526 573 ({ \ 527 574 __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ 528 - BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ 529 575 current->scx.kf_tasks[0] = task0; \ 530 576 current->scx.kf_tasks[1] = task1; \ 531 - __ret = SCX_CALL_OP_RET((sch), mask, op, rq, task0, task1, ##args); \ 577 + __ret = SCX_CALL_OP_RET((sch), op, rq, task0, task1, ##args); \ 532 578 current->scx.kf_tasks[0] = NULL; \ 533 579 current->scx.kf_tasks[1] = NULL; \ 534 580 __ret; \ 535 581 }) 536 - 537 - /* @mask is constant, always inline to cull unnecessary branches */ 538 - static __always_inline bool scx_kf_allowed(struct scx_sched *sch, u32 mask) 539 - { 540 - if (unlikely(!(current->scx.kf_mask & mask))) { 541 - scx_error(sch, "kfunc with mask 0x%x called from an operation only allowing 0x%x", 542 - mask, current->scx.kf_mask); 543 - return false; 544 - } 545 - 546 - /* 547 - * Enforce nesting boundaries. e.g. A kfunc which can be called from 548 - * DISPATCH must not be called if we're running DEQUEUE which is nested 549 - * inside ops.dispatch(). We don't need to check boundaries for any 550 - * blocking kfuncs as the verifier ensures they're only called from 551 - * sleepable progs. 552 - */ 553 - if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && 554 - (current->scx.kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { 555 - scx_error(sch, "cpu_release kfunc called from a nested operation"); 556 - return false; 557 - } 558 - 559 - if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && 560 - (current->scx.kf_mask & higher_bits(SCX_KF_DISPATCH)))) { 561 - scx_error(sch, "dispatch kfunc called from a nested operation"); 562 - return false; 563 - } 564 - 565 - return true; 566 - } 567 582 568 583 /* see SCX_CALL_OP_TASK() */ 569 584 static __always_inline bool scx_kf_allowed_on_arg_tasks(struct scx_sched *sch, ··· 1378 1461 return; 1379 1462 1380 1463 if (SCX_HAS_OP(sch, dequeue)) 1381 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, dequeue, rq, p, deq_flags); 1464 + SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); 1382 1465 1383 1466 p->scx.flags &= ~SCX_TASK_IN_CUSTODY; 1384 1467 } ··· 1837 1920 WARN_ON_ONCE(*ddsp_taskp); 1838 1921 *ddsp_taskp = p; 1839 1922 1840 - SCX_CALL_OP_TASK(sch, SCX_KF_ENQUEUE, enqueue, rq, p, enq_flags); 1923 + SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags); 1841 1924 1842 1925 *ddsp_taskp = NULL; 1843 1926 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) ··· 1941 2024 add_nr_running(rq, 1); 1942 2025 1943 2026 if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) 1944 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, runnable, rq, p, enq_flags); 2027 + SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); 1945 2028 1946 2029 if (enq_flags & SCX_ENQ_WAKEUP) 1947 2030 touch_core_sched(rq, p); ··· 2058 2141 */ 2059 2142 if (SCX_HAS_OP(sch, stopping) && task_current(rq, p)) { 2060 2143 update_curr_scx(rq); 2061 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, stopping, rq, p, false); 2144 + SCX_CALL_OP_TASK(sch, stopping, rq, p, false); 2062 2145 } 2063 2146 2064 2147 if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) 2065 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, quiescent, rq, p, deq_flags); 2148 + SCX_CALL_OP_TASK(sch, quiescent, rq, p, deq_flags); 2066 2149 2067 2150 if (deq_flags & SCX_DEQ_SLEEP) 2068 2151 p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; ··· 2084 2167 struct scx_sched *sch = scx_task_sched(p); 2085 2168 2086 2169 if (SCX_HAS_OP(sch, yield)) 2087 - SCX_CALL_OP_2TASKS_RET(sch, SCX_KF_REST, yield, rq, p, NULL); 2170 + SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); 2088 2171 else 2089 2172 p->scx.slice = 0; 2090 2173 } ··· 2095 2178 struct scx_sched *sch = scx_task_sched(from); 2096 2179 2097 2180 if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to)) 2098 - return SCX_CALL_OP_2TASKS_RET(sch, SCX_KF_REST, yield, rq, 2099 - from, to); 2181 + return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to); 2100 2182 else 2101 2183 return false; 2102 2184 } ··· 2715 2799 dspc->nr_tasks = 0; 2716 2800 2717 2801 if (nested) { 2718 - /* 2719 - * If nested, don't update kf_mask as the originating 2720 - * invocation would already have set it up. 2721 - */ 2722 - SCX_CALL_OP(sch, 0, dispatch, rq, cpu, 2723 - prev_on_sch ? prev : NULL); 2802 + SCX_CALL_OP(sch, dispatch, rq, cpu, prev_on_sch ? prev : NULL); 2724 2803 } else { 2725 - /* 2726 - * If not nested, stash @prev so that nested invocations 2727 - * can access it. 2728 - */ 2804 + /* stash @prev so that nested invocations can access it */ 2729 2805 rq->scx.sub_dispatch_prev = prev; 2730 - SCX_CALL_OP(sch, SCX_KF_DISPATCH, dispatch, rq, cpu, 2731 - prev_on_sch ? prev : NULL); 2806 + SCX_CALL_OP(sch, dispatch, rq, cpu, prev_on_sch ? prev : NULL); 2732 2807 rq->scx.sub_dispatch_prev = NULL; 2733 2808 } 2734 2809 ··· 2778 2871 * emitted in switch_class(). 2779 2872 */ 2780 2873 if (SCX_HAS_OP(sch, cpu_acquire)) 2781 - SCX_CALL_OP(sch, SCX_KF_REST, cpu_acquire, rq, cpu, NULL); 2874 + SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL); 2782 2875 rq->scx.cpu_released = false; 2783 2876 } 2784 2877 ··· 2857 2950 2858 2951 /* see dequeue_task_scx() on why we skip when !QUEUED */ 2859 2952 if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) 2860 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, running, rq, p); 2953 + SCX_CALL_OP_TASK(sch, running, rq, p); 2861 2954 2862 2955 clr_task_runnable(p, true); 2863 2956 ··· 2929 3022 .task = next, 2930 3023 }; 2931 3024 2932 - SCX_CALL_OP(sch, SCX_KF_CPU_RELEASE, cpu_release, rq, 2933 - cpu_of(rq), &args); 3025 + SCX_CALL_OP(sch, cpu_release, rq, cpu_of(rq), &args); 2934 3026 } 2935 3027 rq->scx.cpu_released = true; 2936 3028 } ··· 2947 3041 2948 3042 /* see dequeue_task_scx() on why we skip when !QUEUED */ 2949 3043 if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) 2950 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, stopping, rq, p, true); 3044 + SCX_CALL_OP_TASK(sch, stopping, rq, p, true); 2951 3045 2952 3046 if (p->scx.flags & SCX_TASK_QUEUED) { 2953 3047 set_task_runnable(rq, p); ··· 3177 3271 */ 3178 3272 if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) && 3179 3273 !scx_bypassing(sch_a, task_cpu(a))) 3180 - return SCX_CALL_OP_2TASKS_RET(sch_a, SCX_KF_REST, core_sched_before, 3274 + return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before, 3181 3275 NULL, 3182 3276 (struct task_struct *)a, 3183 3277 (struct task_struct *)b); ··· 3214 3308 *ddsp_taskp = p; 3215 3309 3216 3310 this_rq()->scx.in_select_cpu = true; 3217 - cpu = SCX_CALL_OP_TASK_RET(sch, 3218 - SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU, 3219 - select_cpu, NULL, p, prev_cpu, 3220 - wake_flags); 3311 + cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p, prev_cpu, wake_flags); 3221 3312 this_rq()->scx.in_select_cpu = false; 3222 3313 p->scx.selected_cpu = cpu; 3223 3314 *ddsp_taskp = NULL; ··· 3264 3361 * designation pointless. Cast it away when calling the operation. 3265 3362 */ 3266 3363 if (SCX_HAS_OP(sch, set_cpumask)) 3267 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, set_cpumask, task_rq(p), 3268 - p, (struct cpumask *)p->cpus_ptr); 3364 + SCX_CALL_OP_TASK(sch, set_cpumask, task_rq(p), p, (struct cpumask *)p->cpus_ptr); 3269 3365 } 3270 3366 3271 3367 static void handle_hotplug(struct rq *rq, bool online) ··· 3286 3384 scx_idle_update_selcpu_topology(&sch->ops); 3287 3385 3288 3386 if (online && SCX_HAS_OP(sch, cpu_online)) 3289 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cpu_online, NULL, cpu); 3387 + SCX_CALL_OP(sch, cpu_online, NULL, cpu); 3290 3388 else if (!online && SCX_HAS_OP(sch, cpu_offline)) 3291 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cpu_offline, NULL, cpu); 3389 + SCX_CALL_OP(sch, cpu_offline, NULL, cpu); 3292 3390 else 3293 3391 scx_exit(sch, SCX_EXIT_UNREG_KERN, 3294 3392 SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, ··· 3406 3504 curr->scx.slice = 0; 3407 3505 touch_core_sched(rq, curr); 3408 3506 } else if (SCX_HAS_OP(sch, tick)) { 3409 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, tick, rq, curr); 3507 + SCX_CALL_OP_TASK(sch, tick, rq, curr); 3410 3508 } 3411 3509 3412 3510 if (!curr->scx.slice) ··· 3482 3580 .fork = fork, 3483 3581 }; 3484 3582 3485 - ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, init_task, NULL, 3486 - p, &args); 3583 + ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args); 3487 3584 if (unlikely(ret)) { 3488 3585 ret = ops_sanitize_err(sch, "init_task", ret); 3489 3586 return ret; ··· 3563 3662 p->scx.weight = sched_weight_to_cgroup(weight); 3564 3663 3565 3664 if (SCX_HAS_OP(sch, enable)) 3566 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, enable, rq, p); 3665 + SCX_CALL_OP_TASK(sch, enable, rq, p); 3567 3666 3568 3667 if (SCX_HAS_OP(sch, set_weight)) 3569 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, set_weight, rq, 3570 - p, p->scx.weight); 3668 + SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); 3571 3669 } 3572 3670 3573 3671 static void scx_enable_task(struct scx_sched *sch, struct task_struct *p) ··· 3585 3685 clear_direct_dispatch(p); 3586 3686 3587 3687 if (SCX_HAS_OP(sch, disable)) 3588 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, disable, rq, p); 3688 + SCX_CALL_OP_TASK(sch, disable, rq, p); 3589 3689 scx_set_task_state(p, SCX_TASK_READY); 3590 3690 3591 3691 /* ··· 3623 3723 } 3624 3724 3625 3725 if (SCX_HAS_OP(sch, exit_task)) 3626 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, exit_task, task_rq(p), 3627 - p, &args); 3726 + SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); 3628 3727 } 3629 3728 3630 3729 static void scx_disable_and_exit_task(struct scx_sched *sch, ··· 3802 3903 3803 3904 p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); 3804 3905 if (SCX_HAS_OP(sch, set_weight)) 3805 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, set_weight, rq, 3806 - p, p->scx.weight); 3906 + SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); 3807 3907 } 3808 3908 3809 3909 static void prio_changed_scx(struct rq *rq, struct task_struct *p, u64 oldprio) ··· 3823 3925 * different scheduler class. Keep the BPF scheduler up-to-date. 3824 3926 */ 3825 3927 if (SCX_HAS_OP(sch, set_cpumask)) 3826 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, set_cpumask, rq, 3827 - p, (struct cpumask *)p->cpus_ptr); 3928 + SCX_CALL_OP_TASK(sch, set_cpumask, rq, p, (struct cpumask *)p->cpus_ptr); 3828 3929 } 3829 3930 3830 3931 static void switched_from_scx(struct rq *rq, struct task_struct *p) ··· 4206 4309 .bw_quota_us = tg->scx.bw_quota_us, 4207 4310 .bw_burst_us = tg->scx.bw_burst_us }; 4208 4311 4209 - ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, cgroup_init, 4312 + ret = SCX_CALL_OP_RET(sch, cgroup_init, 4210 4313 NULL, tg->css.cgroup, &args); 4211 4314 if (ret) 4212 4315 ret = ops_sanitize_err(sch, "cgroup_init", ret); ··· 4228 4331 4229 4332 if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_exit) && 4230 4333 (tg->scx.flags & SCX_TG_INITED)) 4231 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_exit, NULL, 4232 - tg->css.cgroup); 4334 + SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup); 4233 4335 tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); 4234 4336 } 4235 4337 ··· 4257 4361 continue; 4258 4362 4259 4363 if (SCX_HAS_OP(sch, cgroup_prep_move)) { 4260 - ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, 4261 - cgroup_prep_move, NULL, 4364 + ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL, 4262 4365 p, from, css->cgroup); 4263 4366 if (ret) 4264 4367 goto err; ··· 4272 4377 cgroup_taskset_for_each(p, css, tset) { 4273 4378 if (SCX_HAS_OP(sch, cgroup_cancel_move) && 4274 4379 p->scx.cgrp_moving_from) 4275 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_cancel_move, NULL, 4380 + SCX_CALL_OP(sch, cgroup_cancel_move, NULL, 4276 4381 p, p->scx.cgrp_moving_from, css->cgroup); 4277 4382 p->scx.cgrp_moving_from = NULL; 4278 4383 } ··· 4293 4398 */ 4294 4399 if (SCX_HAS_OP(sch, cgroup_move) && 4295 4400 !WARN_ON_ONCE(!p->scx.cgrp_moving_from)) 4296 - SCX_CALL_OP_TASK(sch, SCX_KF_REST, cgroup_move, task_rq(p), 4401 + SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), 4297 4402 p, p->scx.cgrp_moving_from, 4298 4403 tg_cgrp(task_group(p))); 4299 4404 p->scx.cgrp_moving_from = NULL; ··· 4311 4416 cgroup_taskset_for_each(p, css, tset) { 4312 4417 if (SCX_HAS_OP(sch, cgroup_cancel_move) && 4313 4418 p->scx.cgrp_moving_from) 4314 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_cancel_move, NULL, 4419 + SCX_CALL_OP(sch, cgroup_cancel_move, NULL, 4315 4420 p, p->scx.cgrp_moving_from, css->cgroup); 4316 4421 p->scx.cgrp_moving_from = NULL; 4317 4422 } ··· 4325 4430 4326 4431 if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && 4327 4432 tg->scx.weight != weight) 4328 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_set_weight, NULL, 4329 - tg_cgrp(tg), weight); 4433 + SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight); 4330 4434 4331 4435 tg->scx.weight = weight; 4332 4436 ··· 4339 4445 percpu_down_read(&scx_cgroup_ops_rwsem); 4340 4446 4341 4447 if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) 4342 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_set_idle, NULL, 4343 - tg_cgrp(tg), idle); 4448 + SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); 4344 4449 4345 4450 /* Update the task group's idle state */ 4346 4451 tg->scx.idle = idle; ··· 4358 4465 (tg->scx.bw_period_us != period_us || 4359 4466 tg->scx.bw_quota_us != quota_us || 4360 4467 tg->scx.bw_burst_us != burst_us)) 4361 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_set_bandwidth, NULL, 4468 + SCX_CALL_OP(sch, cgroup_set_bandwidth, NULL, 4362 4469 tg_cgrp(tg), period_us, quota_us, burst_us); 4363 4470 4364 4471 tg->scx.bw_period_us = period_us; ··· 4583 4690 if (!sch->ops.cgroup_exit) 4584 4691 continue; 4585 4692 4586 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_exit, NULL, 4587 - css->cgroup); 4693 + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); 4588 4694 } 4589 4695 } 4590 4696 ··· 4614 4722 continue; 4615 4723 } 4616 4724 4617 - ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, cgroup_init, NULL, 4725 + ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, 4618 4726 css->cgroup, &args); 4619 4727 if (ret) { 4620 4728 scx_error(sch, "ops.cgroup_init() failed (%d)", ret); ··· 5687 5795 .ops = &sch->ops, 5688 5796 .cgroup_path = sch->cgrp_path, 5689 5797 }; 5690 - SCX_CALL_OP(parent, SCX_KF_UNLOCKED, sub_detach, NULL, 5798 + SCX_CALL_OP(parent, sub_detach, NULL, 5691 5799 &sub_detach_args); 5692 5800 } 5693 5801 5694 5802 if (sch->ops.exit) 5695 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, exit, NULL, sch->exit_info); 5803 + SCX_CALL_OP(sch, exit, NULL, sch->exit_info); 5696 5804 kobject_del(&sch->kobj); 5697 5805 } 5698 5806 #else /* CONFIG_EXT_SUB_SCHED */ ··· 5807 5915 } 5808 5916 5809 5917 if (sch->ops.exit) 5810 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, exit, NULL, ei); 5918 + SCX_CALL_OP(sch, exit, NULL, ei); 5811 5919 5812 5920 scx_unlink_sched(sch); 5813 5921 ··· 6070 6178 6071 6179 if (SCX_HAS_OP(sch, dump_task)) { 6072 6180 ops_dump_init(s, " "); 6073 - SCX_CALL_OP(sch, SCX_KF_REST, dump_task, NULL, dctx, p); 6181 + SCX_CALL_OP(sch, dump_task, NULL, dctx, p); 6074 6182 ops_dump_exit(); 6075 6183 } 6076 6184 ··· 6134 6242 6135 6243 if (SCX_HAS_OP(sch, dump)) { 6136 6244 ops_dump_init(&s, ""); 6137 - SCX_CALL_OP(sch, SCX_KF_UNLOCKED, dump, NULL, &dctx); 6245 + SCX_CALL_OP(sch, dump, NULL, &dctx); 6138 6246 ops_dump_exit(); 6139 6247 } 6140 6248 ··· 6194 6302 used = seq_buf_used(&ns); 6195 6303 if (SCX_HAS_OP(sch, dump_cpu)) { 6196 6304 ops_dump_init(&ns, " "); 6197 - SCX_CALL_OP(sch, SCX_KF_REST, dump_cpu, NULL, 6305 + SCX_CALL_OP(sch, dump_cpu, NULL, 6198 6306 &dctx, cpu, idle); 6199 6307 ops_dump_exit(); 6200 6308 } ··· 6640 6748 scx_idle_enable(ops); 6641 6749 6642 6750 if (sch->ops.init) { 6643 - ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, init, NULL); 6751 + ret = SCX_CALL_OP_RET(sch, init, NULL); 6644 6752 if (ret) { 6645 6753 ret = ops_sanitize_err(sch, "init", ret); 6646 6754 cpus_read_unlock(); ··· 6912 7020 } 6913 7021 6914 7022 if (sch->ops.init) { 6915 - ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, init, NULL); 7023 + ret = SCX_CALL_OP_RET(sch, init, NULL); 6916 7024 if (ret) { 6917 7025 ret = ops_sanitize_err(sch, "init", ret); 6918 7026 scx_error(sch, "ops.init() failed (%d)", ret); ··· 6929 7037 .cgroup_path = sch->cgrp_path, 6930 7038 }; 6931 7039 6932 - ret = SCX_CALL_OP_RET(parent, SCX_KF_UNLOCKED, sub_attach, NULL, 7040 + ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, 6933 7041 &sub_attach_args); 6934 7042 if (ret) { 6935 7043 ret = ops_sanitize_err(sch, "sub_attach", ret); ··· 7783 7891 static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p, 7784 7892 u64 dsq_id, u64 *enq_flags) 7785 7893 { 7786 - if (!scx_kf_allowed(sch, SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) 7787 - return false; 7788 - 7789 7894 lockdep_assert_irqs_disabled(); 7790 7895 7791 7896 if (unlikely(!p)) { ··· 8035 8146 bool in_balance; 8036 8147 unsigned long flags; 8037 8148 8038 - if ((scx_locked_rq() || this_rq()->scx.in_select_cpu) && 8039 - !scx_kf_allowed(sch, SCX_KF_DISPATCH)) 8040 - return false; 8041 - 8042 8149 if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags)) 8043 8150 return false; 8044 8151 ··· 8129 8244 if (unlikely(!sch)) 8130 8245 return 0; 8131 8246 8132 - if (!scx_kf_allowed(sch, SCX_KF_DISPATCH)) 8133 - return 0; 8134 - 8135 8247 return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor); 8136 8248 } 8137 8249 ··· 8148 8266 8149 8267 sch = scx_prog_sched(aux); 8150 8268 if (unlikely(!sch)) 8151 - return; 8152 - 8153 - if (!scx_kf_allowed(sch, SCX_KF_DISPATCH)) 8154 8269 return; 8155 8270 8156 8271 dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; ··· 8194 8315 8195 8316 sch = scx_prog_sched(aux); 8196 8317 if (unlikely(!sch)) 8197 - return false; 8198 - 8199 - if (!scx_kf_allowed(sch, SCX_KF_DISPATCH)) 8200 8318 return false; 8201 8319 8202 8320 if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags)) ··· 8349 8473 if (unlikely(!parent)) 8350 8474 return false; 8351 8475 8352 - if (!scx_kf_allowed(parent, SCX_KF_DISPATCH)) 8353 - return false; 8354 - 8355 8476 child = scx_find_sub_sched(cgroup_id); 8356 8477 8357 8478 if (unlikely(!child)) ··· 8406 8533 guard(rcu)(); 8407 8534 sch = scx_prog_sched(aux); 8408 8535 if (unlikely(!sch)) 8409 - return 0; 8410 - 8411 - if (!scx_kf_allowed(sch, SCX_KF_CPU_RELEASE)) 8412 8536 return 0; 8413 8537 8414 8538 rq = cpu_rq(smp_processor_id());
+1 -3
kernel/sched/ext_idle.c
··· 789 789 */ 790 790 if (SCX_HAS_OP(sch, update_idle) && do_notify && 791 791 !scx_bypassing(sch, cpu_of(rq))) 792 - SCX_CALL_OP(sch, SCX_KF_REST, update_idle, rq, cpu_of(rq), idle); 792 + SCX_CALL_OP(sch, update_idle, rq, cpu_of(rq), idle); 793 793 } 794 794 795 795 static void reset_idle_masks(struct sched_ext_ops *ops) ··· 937 937 } else if (!scx_locked_rq()) { 938 938 raw_spin_lock_irqsave(&p->pi_lock, irq_flags); 939 939 we_locked = true; 940 - } else if (!scx_kf_allowed(sch, SCX_KF_ENQUEUE)) { 941 - return -EPERM; 942 940 } 943 941 944 942 /*