sched_ext: Move internal type and accessor definitions to ext_internal.h

+1

kernel/sched/build_policy.c

··· 58 58 #include "deadline.c" 59 59 60 60 #ifdef CONFIG_SCHED_CLASS_EXT 61 + # include "ext_internal.h" 61 62 # include "ext.c" 62 63 # include "ext_idle.c" 63 64 #endif

-1034

kernel/sched/ext.c

··· 9 9 #include <linux/btf_ids.h> 10 10 #include "ext_idle.h" 11 11 12 - #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) 13 - 14 - enum scx_consts { 15 - SCX_DSP_DFL_MAX_BATCH = 32, 16 - SCX_DSP_MAX_LOOPS = 32, 17 - SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, 18 - 19 - SCX_EXIT_BT_LEN = 64, 20 - SCX_EXIT_MSG_LEN = 1024, 21 - SCX_EXIT_DUMP_DFL_LEN = 32768, 22 - 23 - SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, 24 - 25 - /* 26 - * Iterating all tasks may take a while. Periodically drop 27 - * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. 28 - */ 29 - SCX_TASK_ITER_BATCH = 32, 30 - }; 31 - 32 - enum scx_exit_kind { 33 - SCX_EXIT_NONE, 34 - SCX_EXIT_DONE, 35 - 36 - SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */ 37 - SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */ 38 - SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ 39 - SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ 40 - 41 - SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ 42 - SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ 43 - SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ 44 - }; 45 - 46 - /* 47 - * An exit code can be specified when exiting with scx_bpf_exit() or scx_exit(), 48 - * corresponding to exit_kind UNREG_BPF and UNREG_KERN respectively. The codes 49 - * are 64bit of the format: 50 - * 51 - * Bits: [63 .. 48 47 .. 32 31 .. 0] 52 - * [ SYS ACT ] [ SYS RSN ] [ USR ] 53 - * 54 - * SYS ACT: System-defined exit actions 55 - * SYS RSN: System-defined exit reasons 56 - * USR : User-defined exit codes and reasons 57 - * 58 - * Using the above, users may communicate intention and context by ORing system 59 - * actions and/or system reasons with a user-defined exit code. 60 - */ 61 - enum scx_exit_code { 62 - /* Reasons */ 63 - SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, 64 - 65 - /* Actions */ 66 - SCX_ECODE_ACT_RESTART = 1LLU << 48, 67 - }; 68 - 69 - /* 70 - * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is 71 - * being disabled. 72 - */ 73 - struct scx_exit_info { 74 - /* %SCX_EXIT_* - broad category of the exit reason */ 75 - enum scx_exit_kind kind; 76 - 77 - /* exit code if gracefully exiting */ 78 - s64 exit_code; 79 - 80 - /* textual representation of the above */ 81 - const char *reason; 82 - 83 - /* backtrace if exiting due to an error */ 84 - unsigned long *bt; 85 - u32 bt_len; 86 - 87 - /* informational message */ 88 - char *msg; 89 - 90 - /* debug dump */ 91 - char *dump; 92 - }; 93 - 94 - /* sched_ext_ops.flags */ 95 - enum scx_ops_flags { 96 - /* 97 - * Keep built-in idle tracking even if ops.update_idle() is implemented. 98 - */ 99 - SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, 100 - 101 - /* 102 - * By default, if there are no other task to run on the CPU, ext core 103 - * keeps running the current task even after its slice expires. If this 104 - * flag is specified, such tasks are passed to ops.enqueue() with 105 - * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. 106 - */ 107 - SCX_OPS_ENQ_LAST = 1LLU << 1, 108 - 109 - /* 110 - * An exiting task may schedule after PF_EXITING is set. In such cases, 111 - * bpf_task_from_pid() may not be able to find the task and if the BPF 112 - * scheduler depends on pid lookup for dispatching, the task will be 113 - * lost leading to various issues including RCU grace period stalls. 114 - * 115 - * To mask this problem, by default, unhashed tasks are automatically 116 - * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't 117 - * depend on pid lookups and wants to handle these tasks directly, the 118 - * following flag can be used. 119 - */ 120 - SCX_OPS_ENQ_EXITING = 1LLU << 2, 121 - 122 - /* 123 - * If set, only tasks with policy set to SCHED_EXT are attached to 124 - * sched_ext. If clear, SCHED_NORMAL tasks are also included. 125 - */ 126 - SCX_OPS_SWITCH_PARTIAL = 1LLU << 3, 127 - 128 - /* 129 - * A migration disabled task can only execute on its current CPU. By 130 - * default, such tasks are automatically put on the CPU's local DSQ with 131 - * the default slice on enqueue. If this ops flag is set, they also go 132 - * through ops.enqueue(). 133 - * 134 - * A migration disabled task never invokes ops.select_cpu() as it can 135 - * only select the current CPU. Also, p->cpus_ptr will only contain its 136 - * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr 137 - * and thus may disagree with cpumask_weight(p->cpus_ptr). 138 - */ 139 - SCX_OPS_ENQ_MIGRATION_DISABLED = 1LLU << 4, 140 - 141 - /* 142 - * Queued wakeup (ttwu_queue) is a wakeup optimization that invokes 143 - * ops.enqueue() on the ops.select_cpu() selected or the wakee's 144 - * previous CPU via IPI (inter-processor interrupt) to reduce cacheline 145 - * transfers. When this optimization is enabled, ops.select_cpu() is 146 - * skipped in some cases (when racing against the wakee switching out). 147 - * As the BPF scheduler may depend on ops.select_cpu() being invoked 148 - * during wakeups, queued wakeup is disabled by default. 149 - * 150 - * If this ops flag is set, queued wakeup optimization is enabled and 151 - * the BPF scheduler must be able to handle ops.enqueue() invoked on the 152 - * wakee's CPU without preceding ops.select_cpu() even for tasks which 153 - * may be executed on multiple CPUs. 154 - */ 155 - SCX_OPS_ALLOW_QUEUED_WAKEUP = 1LLU << 5, 156 - 157 - /* 158 - * If set, enable per-node idle cpumasks. If clear, use a single global 159 - * flat idle cpumask. 160 - */ 161 - SCX_OPS_BUILTIN_IDLE_PER_NODE = 1LLU << 6, 162 - 163 - /* 164 - * CPU cgroup support flags 165 - */ 166 - SCX_OPS_HAS_CGROUP_WEIGHT = 1LLU << 16, /* DEPRECATED, will be removed on 6.18 */ 167 - 168 - SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | 169 - SCX_OPS_ENQ_LAST | 170 - SCX_OPS_ENQ_EXITING | 171 - SCX_OPS_ENQ_MIGRATION_DISABLED | 172 - SCX_OPS_ALLOW_QUEUED_WAKEUP | 173 - SCX_OPS_SWITCH_PARTIAL | 174 - SCX_OPS_BUILTIN_IDLE_PER_NODE | 175 - SCX_OPS_HAS_CGROUP_WEIGHT, 176 - 177 - /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ 178 - __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, 179 - 180 - SCX_OPS_HAS_CPU_PREEMPT = 1LLU << 56, 181 - }; 182 - 183 - /* argument container for ops.init_task() */ 184 - struct scx_init_task_args { 185 - /* 186 - * Set if ops.init_task() is being invoked on the fork path, as opposed 187 - * to the scheduler transition path. 188 - */ 189 - bool fork; 190 - #ifdef CONFIG_EXT_GROUP_SCHED 191 - /* the cgroup the task is joining */ 192 - struct cgroup *cgroup; 193 - #endif 194 - }; 195 - 196 - /* argument container for ops.exit_task() */ 197 - struct scx_exit_task_args { 198 - /* Whether the task exited before running on sched_ext. */ 199 - bool cancelled; 200 - }; 201 - 202 - /* argument container for ops->cgroup_init() */ 203 - struct scx_cgroup_init_args { 204 - /* the weight of the cgroup [1..10000] */ 205 - u32 weight; 206 - 207 - /* bandwidth control parameters from cpu.max and cpu.max.burst */ 208 - u64 bw_period_us; 209 - u64 bw_quota_us; 210 - u64 bw_burst_us; 211 - }; 212 - 213 - enum scx_cpu_preempt_reason { 214 - /* next task is being scheduled by &sched_class_rt */ 215 - SCX_CPU_PREEMPT_RT, 216 - /* next task is being scheduled by &sched_class_dl */ 217 - SCX_CPU_PREEMPT_DL, 218 - /* next task is being scheduled by &sched_class_stop */ 219 - SCX_CPU_PREEMPT_STOP, 220 - /* unknown reason for SCX being preempted */ 221 - SCX_CPU_PREEMPT_UNKNOWN, 222 - }; 223 - 224 - /* 225 - * Argument container for ops->cpu_acquire(). Currently empty, but may be 226 - * expanded in the future. 227 - */ 228 - struct scx_cpu_acquire_args {}; 229 - 230 - /* argument container for ops->cpu_release() */ 231 - struct scx_cpu_release_args { 232 - /* the reason the CPU was preempted */ 233 - enum scx_cpu_preempt_reason reason; 234 - 235 - /* the task that's going to be scheduled on the CPU */ 236 - struct task_struct *task; 237 - }; 238 - 239 - /* 240 - * Informational context provided to dump operations. 241 - */ 242 - struct scx_dump_ctx { 243 - enum scx_exit_kind kind; 244 - s64 exit_code; 245 - const char *reason; 246 - u64 at_ns; 247 - u64 at_jiffies; 248 - }; 249 - 250 - /** 251 - * struct sched_ext_ops - Operation table for BPF scheduler implementation 252 - * 253 - * A BPF scheduler can implement an arbitrary scheduling policy by 254 - * implementing and loading operations in this table. Note that a userland 255 - * scheduling policy can also be implemented using the BPF scheduler 256 - * as a shim layer. 257 - */ 258 - struct sched_ext_ops { 259 - /** 260 - * @select_cpu: Pick the target CPU for a task which is being woken up 261 - * @p: task being woken up 262 - * @prev_cpu: the cpu @p was on before sleeping 263 - * @wake_flags: SCX_WAKE_* 264 - * 265 - * Decision made here isn't final. @p may be moved to any CPU while it 266 - * is getting dispatched for execution later. However, as @p is not on 267 - * the rq at this point, getting the eventual execution CPU right here 268 - * saves a small bit of overhead down the line. 269 - * 270 - * If an idle CPU is returned, the CPU is kicked and will try to 271 - * dispatch. While an explicit custom mechanism can be added, 272 - * select_cpu() serves as the default way to wake up idle CPUs. 273 - * 274 - * @p may be inserted into a DSQ directly by calling 275 - * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped. 276 - * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ 277 - * of the CPU returned by this operation. 278 - * 279 - * Note that select_cpu() is never called for tasks that can only run 280 - * on a single CPU or tasks with migration disabled, as they don't have 281 - * the option to select a different CPU. See select_task_rq() for 282 - * details. 283 - */ 284 - s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); 285 - 286 - /** 287 - * @enqueue: Enqueue a task on the BPF scheduler 288 - * @p: task being enqueued 289 - * @enq_flags: %SCX_ENQ_* 290 - * 291 - * @p is ready to run. Insert directly into a DSQ by calling 292 - * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly 293 - * inserted, the bpf scheduler owns @p and if it fails to dispatch @p, 294 - * the task will stall. 295 - * 296 - * If @p was inserted into a DSQ from ops.select_cpu(), this callback is 297 - * skipped. 298 - */ 299 - void (*enqueue)(struct task_struct *p, u64 enq_flags); 300 - 301 - /** 302 - * @dequeue: Remove a task from the BPF scheduler 303 - * @p: task being dequeued 304 - * @deq_flags: %SCX_DEQ_* 305 - * 306 - * Remove @p from the BPF scheduler. This is usually called to isolate 307 - * the task while updating its scheduling properties (e.g. priority). 308 - * 309 - * The ext core keeps track of whether the BPF side owns a given task or 310 - * not and can gracefully ignore spurious dispatches from BPF side, 311 - * which makes it safe to not implement this method. However, depending 312 - * on the scheduling logic, this can lead to confusing behaviors - e.g. 313 - * scheduling position not being updated across a priority change. 314 - */ 315 - void (*dequeue)(struct task_struct *p, u64 deq_flags); 316 - 317 - /** 318 - * @dispatch: Dispatch tasks from the BPF scheduler and/or user DSQs 319 - * @cpu: CPU to dispatch tasks for 320 - * @prev: previous task being switched out 321 - * 322 - * Called when a CPU's local dsq is empty. The operation should dispatch 323 - * one or more tasks from the BPF scheduler into the DSQs using 324 - * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ 325 - * using scx_bpf_dsq_move_to_local(). 326 - * 327 - * The maximum number of times scx_bpf_dsq_insert() can be called 328 - * without an intervening scx_bpf_dsq_move_to_local() is specified by 329 - * ops.dispatch_max_batch. See the comments on top of the two functions 330 - * for more details. 331 - * 332 - * When not %NULL, @prev is an SCX task with its slice depleted. If 333 - * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in 334 - * @prev->scx.flags, it is not enqueued yet and will be enqueued after 335 - * ops.dispatch() returns. To keep executing @prev, return without 336 - * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST. 337 - */ 338 - void (*dispatch)(s32 cpu, struct task_struct *prev); 339 - 340 - /** 341 - * @tick: Periodic tick 342 - * @p: task running currently 343 - * 344 - * This operation is called every 1/HZ seconds on CPUs which are 345 - * executing an SCX task. Setting @p->scx.slice to 0 will trigger an 346 - * immediate dispatch cycle on the CPU. 347 - */ 348 - void (*tick)(struct task_struct *p); 349 - 350 - /** 351 - * @runnable: A task is becoming runnable on its associated CPU 352 - * @p: task becoming runnable 353 - * @enq_flags: %SCX_ENQ_* 354 - * 355 - * This and the following three functions can be used to track a task's 356 - * execution state transitions. A task becomes ->runnable() on a CPU, 357 - * and then goes through one or more ->running() and ->stopping() pairs 358 - * as it runs on the CPU, and eventually becomes ->quiescent() when it's 359 - * done running on the CPU. 360 - * 361 - * @p is becoming runnable on the CPU because it's 362 - * 363 - * - waking up (%SCX_ENQ_WAKEUP) 364 - * - being moved from another CPU 365 - * - being restored after temporarily taken off the queue for an 366 - * attribute change. 367 - * 368 - * This and ->enqueue() are related but not coupled. This operation 369 - * notifies @p's state transition and may not be followed by ->enqueue() 370 - * e.g. when @p is being dispatched to a remote CPU, or when @p is 371 - * being enqueued on a CPU experiencing a hotplug event. Likewise, a 372 - * task may be ->enqueue()'d without being preceded by this operation 373 - * e.g. after exhausting its slice. 374 - */ 375 - void (*runnable)(struct task_struct *p, u64 enq_flags); 376 - 377 - /** 378 - * @running: A task is starting to run on its associated CPU 379 - * @p: task starting to run 380 - * 381 - * Note that this callback may be called from a CPU other than the 382 - * one the task is going to run on. This can happen when a task 383 - * property is changed (i.e., affinity), since scx_next_task_scx(), 384 - * which triggers this callback, may run on a CPU different from 385 - * the task's assigned CPU. 386 - * 387 - * Therefore, always use scx_bpf_task_cpu(@p) to determine the 388 - * target CPU the task is going to use. 389 - * 390 - * See ->runnable() for explanation on the task state notifiers. 391 - */ 392 - void (*running)(struct task_struct *p); 393 - 394 - /** 395 - * @stopping: A task is stopping execution 396 - * @p: task stopping to run 397 - * @runnable: is task @p still runnable? 398 - * 399 - * Note that this callback may be called from a CPU other than the 400 - * one the task was running on. This can happen when a task 401 - * property is changed (i.e., affinity), since dequeue_task_scx(), 402 - * which triggers this callback, may run on a CPU different from 403 - * the task's assigned CPU. 404 - * 405 - * Therefore, always use scx_bpf_task_cpu(@p) to retrieve the CPU 406 - * the task was running on. 407 - * 408 - * See ->runnable() for explanation on the task state notifiers. If 409 - * !@runnable, ->quiescent() will be invoked after this operation 410 - * returns. 411 - */ 412 - void (*stopping)(struct task_struct *p, bool runnable); 413 - 414 - /** 415 - * @quiescent: A task is becoming not runnable on its associated CPU 416 - * @p: task becoming not runnable 417 - * @deq_flags: %SCX_DEQ_* 418 - * 419 - * See ->runnable() for explanation on the task state notifiers. 420 - * 421 - * @p is becoming quiescent on the CPU because it's 422 - * 423 - * - sleeping (%SCX_DEQ_SLEEP) 424 - * - being moved to another CPU 425 - * - being temporarily taken off the queue for an attribute change 426 - * (%SCX_DEQ_SAVE) 427 - * 428 - * This and ->dequeue() are related but not coupled. This operation 429 - * notifies @p's state transition and may not be preceded by ->dequeue() 430 - * e.g. when @p is being dispatched to a remote CPU. 431 - */ 432 - void (*quiescent)(struct task_struct *p, u64 deq_flags); 433 - 434 - /** 435 - * @yield: Yield CPU 436 - * @from: yielding task 437 - * @to: optional yield target task 438 - * 439 - * If @to is NULL, @from is yielding the CPU to other runnable tasks. 440 - * The BPF scheduler should ensure that other available tasks are 441 - * dispatched before the yielding task. Return value is ignored in this 442 - * case. 443 - * 444 - * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf 445 - * scheduler can implement the request, return %true; otherwise, %false. 446 - */ 447 - bool (*yield)(struct task_struct *from, struct task_struct *to); 448 - 449 - /** 450 - * @core_sched_before: Task ordering for core-sched 451 - * @a: task A 452 - * @b: task B 453 - * 454 - * Used by core-sched to determine the ordering between two tasks. See 455 - * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on 456 - * core-sched. 457 - * 458 - * Both @a and @b are runnable and may or may not currently be queued on 459 - * the BPF scheduler. Should return %true if @a should run before @b. 460 - * %false if there's no required ordering or @b should run before @a. 461 - * 462 - * If not specified, the default is ordering them according to when they 463 - * became runnable. 464 - */ 465 - bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); 466 - 467 - /** 468 - * @set_weight: Set task weight 469 - * @p: task to set weight for 470 - * @weight: new weight [1..10000] 471 - * 472 - * Update @p's weight to @weight. 473 - */ 474 - void (*set_weight)(struct task_struct *p, u32 weight); 475 - 476 - /** 477 - * @set_cpumask: Set CPU affinity 478 - * @p: task to set CPU affinity for 479 - * @cpumask: cpumask of cpus that @p can run on 480 - * 481 - * Update @p's CPU affinity to @cpumask. 482 - */ 483 - void (*set_cpumask)(struct task_struct *p, 484 - const struct cpumask *cpumask); 485 - 486 - /** 487 - * @update_idle: Update the idle state of a CPU 488 - * @cpu: CPU to update the idle state for 489 - * @idle: whether entering or exiting the idle state 490 - * 491 - * This operation is called when @rq's CPU goes or leaves the idle 492 - * state. By default, implementing this operation disables the built-in 493 - * idle CPU tracking and the following helpers become unavailable: 494 - * 495 - * - scx_bpf_select_cpu_dfl() 496 - * - scx_bpf_select_cpu_and() 497 - * - scx_bpf_test_and_clear_cpu_idle() 498 - * - scx_bpf_pick_idle_cpu() 499 - * 500 - * The user also must implement ops.select_cpu() as the default 501 - * implementation relies on scx_bpf_select_cpu_dfl(). 502 - * 503 - * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle 504 - * tracking. 505 - */ 506 - void (*update_idle)(s32 cpu, bool idle); 507 - 508 - /** 509 - * @cpu_acquire: A CPU is becoming available to the BPF scheduler 510 - * @cpu: The CPU being acquired by the BPF scheduler. 511 - * @args: Acquire arguments, see the struct definition. 512 - * 513 - * A CPU that was previously released from the BPF scheduler is now once 514 - * again under its control. 515 - */ 516 - void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); 517 - 518 - /** 519 - * @cpu_release: A CPU is taken away from the BPF scheduler 520 - * @cpu: The CPU being released by the BPF scheduler. 521 - * @args: Release arguments, see the struct definition. 522 - * 523 - * The specified CPU is no longer under the control of the BPF 524 - * scheduler. This could be because it was preempted by a higher 525 - * priority sched_class, though there may be other reasons as well. The 526 - * caller should consult @args->reason to determine the cause. 527 - */ 528 - void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); 529 - 530 - /** 531 - * @init_task: Initialize a task to run in a BPF scheduler 532 - * @p: task to initialize for BPF scheduling 533 - * @args: init arguments, see the struct definition 534 - * 535 - * Either we're loading a BPF scheduler or a new task is being forked. 536 - * Initialize @p for BPF scheduling. This operation may block and can 537 - * be used for allocations, and is called exactly once for a task. 538 - * 539 - * Return 0 for success, -errno for failure. An error return while 540 - * loading will abort loading of the BPF scheduler. During a fork, it 541 - * will abort that specific fork. 542 - */ 543 - s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); 544 - 545 - /** 546 - * @exit_task: Exit a previously-running task from the system 547 - * @p: task to exit 548 - * @args: exit arguments, see the struct definition 549 - * 550 - * @p is exiting or the BPF scheduler is being unloaded. Perform any 551 - * necessary cleanup for @p. 552 - */ 553 - void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); 554 - 555 - /** 556 - * @enable: Enable BPF scheduling for a task 557 - * @p: task to enable BPF scheduling for 558 - * 559 - * Enable @p for BPF scheduling. enable() is called on @p any time it 560 - * enters SCX, and is always paired with a matching disable(). 561 - */ 562 - void (*enable)(struct task_struct *p); 563 - 564 - /** 565 - * @disable: Disable BPF scheduling for a task 566 - * @p: task to disable BPF scheduling for 567 - * 568 - * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. 569 - * Disable BPF scheduling for @p. A disable() call is always matched 570 - * with a prior enable() call. 571 - */ 572 - void (*disable)(struct task_struct *p); 573 - 574 - /** 575 - * @dump: Dump BPF scheduler state on error 576 - * @ctx: debug dump context 577 - * 578 - * Use scx_bpf_dump() to generate BPF scheduler specific debug dump. 579 - */ 580 - void (*dump)(struct scx_dump_ctx *ctx); 581 - 582 - /** 583 - * @dump_cpu: Dump BPF scheduler state for a CPU on error 584 - * @ctx: debug dump context 585 - * @cpu: CPU to generate debug dump for 586 - * @idle: @cpu is currently idle without any runnable tasks 587 - * 588 - * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for 589 - * @cpu. If @idle is %true and this operation doesn't produce any 590 - * output, @cpu is skipped for dump. 591 - */ 592 - void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle); 593 - 594 - /** 595 - * @dump_task: Dump BPF scheduler state for a runnable task on error 596 - * @ctx: debug dump context 597 - * @p: runnable task to generate debug dump for 598 - * 599 - * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for 600 - * @p. 601 - */ 602 - void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); 603 - 604 - #ifdef CONFIG_EXT_GROUP_SCHED 605 - /** 606 - * @cgroup_init: Initialize a cgroup 607 - * @cgrp: cgroup being initialized 608 - * @args: init arguments, see the struct definition 609 - * 610 - * Either the BPF scheduler is being loaded or @cgrp created, initialize 611 - * @cgrp for sched_ext. This operation may block. 612 - * 613 - * Return 0 for success, -errno for failure. An error return while 614 - * loading will abort loading of the BPF scheduler. During cgroup 615 - * creation, it will abort the specific cgroup creation. 616 - */ 617 - s32 (*cgroup_init)(struct cgroup *cgrp, 618 - struct scx_cgroup_init_args *args); 619 - 620 - /** 621 - * @cgroup_exit: Exit a cgroup 622 - * @cgrp: cgroup being exited 623 - * 624 - * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit 625 - * @cgrp for sched_ext. This operation my block. 626 - */ 627 - void (*cgroup_exit)(struct cgroup *cgrp); 628 - 629 - /** 630 - * @cgroup_prep_move: Prepare a task to be moved to a different cgroup 631 - * @p: task being moved 632 - * @from: cgroup @p is being moved from 633 - * @to: cgroup @p is being moved to 634 - * 635 - * Prepare @p for move from cgroup @from to @to. This operation may 636 - * block and can be used for allocations. 637 - * 638 - * Return 0 for success, -errno for failure. An error return aborts the 639 - * migration. 640 - */ 641 - s32 (*cgroup_prep_move)(struct task_struct *p, 642 - struct cgroup *from, struct cgroup *to); 643 - 644 - /** 645 - * @cgroup_move: Commit cgroup move 646 - * @p: task being moved 647 - * @from: cgroup @p is being moved from 648 - * @to: cgroup @p is being moved to 649 - * 650 - * Commit the move. @p is dequeued during this operation. 651 - */ 652 - void (*cgroup_move)(struct task_struct *p, 653 - struct cgroup *from, struct cgroup *to); 654 - 655 - /** 656 - * @cgroup_cancel_move: Cancel cgroup move 657 - * @p: task whose cgroup move is being canceled 658 - * @from: cgroup @p was being moved from 659 - * @to: cgroup @p was being moved to 660 - * 661 - * @p was cgroup_prep_move()'d but failed before reaching cgroup_move(). 662 - * Undo the preparation. 663 - */ 664 - void (*cgroup_cancel_move)(struct task_struct *p, 665 - struct cgroup *from, struct cgroup *to); 666 - 667 - /** 668 - * @cgroup_set_weight: A cgroup's weight is being changed 669 - * @cgrp: cgroup whose weight is being updated 670 - * @weight: new weight [1..10000] 671 - * 672 - * Update @cgrp's weight to @weight. 673 - */ 674 - void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); 675 - 676 - /** 677 - * @cgroup_set_bandwidth: A cgroup's bandwidth is being changed 678 - * @cgrp: cgroup whose bandwidth is being updated 679 - * @period_us: bandwidth control period 680 - * @quota_us: bandwidth control quota 681 - * @burst_us: bandwidth control burst 682 - * 683 - * Update @cgrp's bandwidth control parameters. This is from the cpu.max 684 - * cgroup interface. 685 - * 686 - * @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled 687 - * to. For example, if @period_us is 1_000_000 and @quota_us is 688 - * 2_500_000. @cgrp is entitled to 2.5 CPUs. @burst_us can be 689 - * interpreted in the same fashion and specifies how much @cgrp can 690 - * burst temporarily. The specific control mechanism and thus the 691 - * interpretation of @period_us and burstiness is upto to the BPF 692 - * scheduler. 693 - */ 694 - void (*cgroup_set_bandwidth)(struct cgroup *cgrp, 695 - u64 period_us, u64 quota_us, u64 burst_us); 696 - 697 - #endif /* CONFIG_EXT_GROUP_SCHED */ 698 - 699 - /* 700 - * All online ops must come before ops.cpu_online(). 701 - */ 702 - 703 - /** 704 - * @cpu_online: A CPU became online 705 - * @cpu: CPU which just came up 706 - * 707 - * @cpu just came online. @cpu will not call ops.enqueue() or 708 - * ops.dispatch(), nor run tasks associated with other CPUs beforehand. 709 - */ 710 - void (*cpu_online)(s32 cpu); 711 - 712 - /** 713 - * @cpu_offline: A CPU is going offline 714 - * @cpu: CPU which is going offline 715 - * 716 - * @cpu is going offline. @cpu will not call ops.enqueue() or 717 - * ops.dispatch(), nor run tasks associated with other CPUs afterwards. 718 - */ 719 - void (*cpu_offline)(s32 cpu); 720 - 721 - /* 722 - * All CPU hotplug ops must come before ops.init(). 723 - */ 724 - 725 - /** 726 - * @init: Initialize the BPF scheduler 727 - */ 728 - s32 (*init)(void); 729 - 730 - /** 731 - * @exit: Clean up after the BPF scheduler 732 - * @info: Exit info 733 - * 734 - * ops.exit() is also called on ops.init() failure, which is a bit 735 - * unusual. This is to allow rich reporting through @info on how 736 - * ops.init() failed. 737 - */ 738 - void (*exit)(struct scx_exit_info *info); 739 - 740 - /** 741 - * @dispatch_max_batch: Max nr of tasks that dispatch() can dispatch 742 - */ 743 - u32 dispatch_max_batch; 744 - 745 - /** 746 - * @flags: %SCX_OPS_* flags 747 - */ 748 - u64 flags; 749 - 750 - /** 751 - * @timeout_ms: The maximum amount of time, in milliseconds, that a 752 - * runnable task should be able to wait before being scheduled. The 753 - * maximum timeout may not exceed the default timeout of 30 seconds. 754 - * 755 - * Defaults to the maximum allowed timeout value of 30 seconds. 756 - */ 757 - u32 timeout_ms; 758 - 759 - /** 760 - * @exit_dump_len: scx_exit_info.dump buffer length. If 0, the default 761 - * value of 32768 is used. 762 - */ 763 - u32 exit_dump_len; 764 - 765 - /** 766 - * @hotplug_seq: A sequence number that may be set by the scheduler to 767 - * detect when a hotplug event has occurred during the loading process. 768 - * If 0, no detection occurs. Otherwise, the scheduler will fail to 769 - * load if the sequence number does not match @scx_hotplug_seq on the 770 - * enable path. 771 - */ 772 - u64 hotplug_seq; 773 - 774 - /** 775 - * @name: BPF scheduler's name 776 - * 777 - * Must be a non-zero valid BPF object name including only isalnum(), 778 - * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the 779 - * BPF scheduler is enabled. 780 - */ 781 - char name[SCX_OPS_NAME_LEN]; 782 - 783 - /* internal use only, must be NULL */ 784 - void *priv; 785 - }; 786 - 787 - enum scx_opi { 788 - SCX_OPI_BEGIN = 0, 789 - SCX_OPI_NORMAL_BEGIN = 0, 790 - SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), 791 - SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), 792 - SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), 793 - SCX_OPI_END = SCX_OP_IDX(init), 794 - }; 795 - 796 - /* 797 - * Collection of event counters. Event types are placed in descending order. 798 - */ 799 - struct scx_event_stats { 800 - /* 801 - * If ops.select_cpu() returns a CPU which can't be used by the task, 802 - * the core scheduler code silently picks a fallback CPU. 803 - */ 804 - s64 SCX_EV_SELECT_CPU_FALLBACK; 805 - 806 - /* 807 - * When dispatching to a local DSQ, the CPU may have gone offline in 808 - * the meantime. In this case, the task is bounced to the global DSQ. 809 - */ 810 - s64 SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE; 811 - 812 - /* 813 - * If SCX_OPS_ENQ_LAST is not set, the number of times that a task 814 - * continued to run because there were no other tasks on the CPU. 815 - */ 816 - s64 SCX_EV_DISPATCH_KEEP_LAST; 817 - 818 - /* 819 - * If SCX_OPS_ENQ_EXITING is not set, the number of times that a task 820 - * is dispatched to a local DSQ when exiting. 821 - */ 822 - s64 SCX_EV_ENQ_SKIP_EXITING; 823 - 824 - /* 825 - * If SCX_OPS_ENQ_MIGRATION_DISABLED is not set, the number of times a 826 - * migration disabled task skips ops.enqueue() and is dispatched to its 827 - * local DSQ. 828 - */ 829 - s64 SCX_EV_ENQ_SKIP_MIGRATION_DISABLED; 830 - 831 - /* 832 - * Total number of times a task's time slice was refilled with the 833 - * default value (SCX_SLICE_DFL). 834 - */ 835 - s64 SCX_EV_REFILL_SLICE_DFL; 836 - 837 - /* 838 - * The total duration of bypass modes in nanoseconds. 839 - */ 840 - s64 SCX_EV_BYPASS_DURATION; 841 - 842 - /* 843 - * The number of tasks dispatched in the bypassing mode. 844 - */ 845 - s64 SCX_EV_BYPASS_DISPATCH; 846 - 847 - /* 848 - * The number of times the bypassing mode has been activated. 849 - */ 850 - s64 SCX_EV_BYPASS_ACTIVATE; 851 - }; 852 - 853 - struct scx_sched { 854 - struct sched_ext_ops ops; 855 - DECLARE_BITMAP(has_op, SCX_OPI_END); 856 - 857 - /* 858 - * Dispatch queues. 859 - * 860 - * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. 861 - * This is to avoid live-locking in bypass mode where all tasks are 862 - * dispatched to %SCX_DSQ_GLOBAL and all CPUs consume from it. If 863 - * per-node split isn't sufficient, it can be further split. 864 - */ 865 - struct rhashtable dsq_hash; 866 - struct scx_dispatch_q **global_dsqs; 867 - 868 - /* 869 - * The event counters are in a per-CPU variable to minimize the 870 - * accounting overhead. A system-wide view on the event counter is 871 - * constructed when requested by scx_bpf_events(). 872 - */ 873 - struct scx_event_stats __percpu *event_stats_cpu; 874 - 875 - bool warned_zero_slice; 876 - 877 - atomic_t exit_kind; 878 - struct scx_exit_info *exit_info; 879 - 880 - struct kobject kobj; 881 - 882 - struct kthread_worker *helper; 883 - struct irq_work error_irq_work; 884 - struct kthread_work disable_work; 885 - struct rcu_work rcu_work; 886 - }; 887 - 888 - enum scx_wake_flags { 889 - /* expose select WF_* flags as enums */ 890 - SCX_WAKE_FORK = WF_FORK, 891 - SCX_WAKE_TTWU = WF_TTWU, 892 - SCX_WAKE_SYNC = WF_SYNC, 893 - }; 894 - 895 - enum scx_enq_flags { 896 - /* expose select ENQUEUE_* flags as enums */ 897 - SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, 898 - SCX_ENQ_HEAD = ENQUEUE_HEAD, 899 - SCX_ENQ_CPU_SELECTED = ENQUEUE_RQ_SELECTED, 900 - 901 - /* high 32bits are SCX specific */ 902 - 903 - /* 904 - * Set the following to trigger preemption when calling 905 - * scx_bpf_dsq_insert() with a local dsq as the target. The slice of the 906 - * current task is cleared to zero and the CPU is kicked into the 907 - * scheduling path. Implies %SCX_ENQ_HEAD. 908 - */ 909 - SCX_ENQ_PREEMPT = 1LLU << 32, 910 - 911 - /* 912 - * The task being enqueued was previously enqueued on the current CPU's 913 - * %SCX_DSQ_LOCAL, but was removed from it in a call to the 914 - * scx_bpf_reenqueue_local() kfunc. If scx_bpf_reenqueue_local() was 915 - * invoked in a ->cpu_release() callback, and the task is again 916 - * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the 917 - * task will not be scheduled on the CPU until at least the next invocation 918 - * of the ->cpu_acquire() callback. 919 - */ 920 - SCX_ENQ_REENQ = 1LLU << 40, 921 - 922 - /* 923 - * The task being enqueued is the only task available for the cpu. By 924 - * default, ext core keeps executing such tasks but when 925 - * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the 926 - * %SCX_ENQ_LAST flag set. 927 - * 928 - * The BPF scheduler is responsible for triggering a follow-up 929 - * scheduling event. Otherwise, Execution may stall. 930 - */ 931 - SCX_ENQ_LAST = 1LLU << 41, 932 - 933 - /* high 8 bits are internal */ 934 - __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, 935 - 936 - SCX_ENQ_CLEAR_OPSS = 1LLU << 56, 937 - SCX_ENQ_DSQ_PRIQ = 1LLU << 57, 938 - }; 939 - 940 - enum scx_deq_flags { 941 - /* expose select DEQUEUE_* flags as enums */ 942 - SCX_DEQ_SLEEP = DEQUEUE_SLEEP, 943 - 944 - /* high 32bits are SCX specific */ 945 - 946 - /* 947 - * The generic core-sched layer decided to execute the task even though 948 - * it hasn't been dispatched yet. Dequeue from the BPF side. 949 - */ 950 - SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, 951 - }; 952 - 953 - enum scx_pick_idle_cpu_flags { 954 - SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */ 955 - SCX_PICK_IDLE_IN_NODE = 1LLU << 1, /* pick a CPU in the same target NUMA node */ 956 - }; 957 - 958 - enum scx_kick_flags { 959 - /* 960 - * Kick the target CPU if idle. Guarantees that the target CPU goes 961 - * through at least one full scheduling cycle before going idle. If the 962 - * target CPU can be determined to be currently not idle and going to go 963 - * through a scheduling cycle before going idle, noop. 964 - */ 965 - SCX_KICK_IDLE = 1LLU << 0, 966 - 967 - /* 968 - * Preempt the current task and execute the dispatch path. If the 969 - * current task of the target CPU is an SCX task, its ->scx.slice is 970 - * cleared to zero before the scheduling path is invoked so that the 971 - * task expires and the dispatch path is invoked. 972 - */ 973 - SCX_KICK_PREEMPT = 1LLU << 1, 974 - 975 - /* 976 - * Wait for the CPU to be rescheduled. The scx_bpf_kick_cpu() call will 977 - * return after the target CPU finishes picking the next task. 978 - */ 979 - SCX_KICK_WAIT = 1LLU << 2, 980 - }; 981 - 982 - enum scx_tg_flags { 983 - SCX_TG_ONLINE = 1U << 0, 984 - SCX_TG_INITED = 1U << 1, 985 - }; 986 - 987 - enum scx_enable_state { 988 - SCX_ENABLING, 989 - SCX_ENABLED, 990 - SCX_DISABLING, 991 - SCX_DISABLED, 992 - }; 993 - 994 - static const char *scx_enable_state_str[] = { 995 - [SCX_ENABLING] = "enabling", 996 - [SCX_ENABLED] = "enabled", 997 - [SCX_DISABLING] = "disabling", 998 - [SCX_DISABLED] = "disabled", 999 - }; 1000 - 1001 - /* 1002 - * sched_ext_entity->ops_state 1003 - * 1004 - * Used to track the task ownership between the SCX core and the BPF scheduler. 1005 - * State transitions look as follows: 1006 - * 1007 - * NONE -> QUEUEING -> QUEUED -> DISPATCHING 1008 - * ^ | | 1009 - * | v v 1010 - * \-------------------------------/ 1011 - * 1012 - * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call 1013 - * sites for explanations on the conditions being waited upon and why they are 1014 - * safe. Transitions out of them into NONE or QUEUED must store_release and the 1015 - * waiters should load_acquire. 1016 - * 1017 - * Tracking scx_ops_state enables sched_ext core to reliably determine whether 1018 - * any given task can be dispatched by the BPF scheduler at all times and thus 1019 - * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler 1020 - * to try to dispatch any task anytime regardless of its state as the SCX core 1021 - * can safely reject invalid dispatches. 1022 - */ 1023 - enum scx_ops_state { 1024 - SCX_OPSS_NONE, /* owned by the SCX core */ 1025 - SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ 1026 - SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ 1027 - SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ 1028 - 1029 - /* 1030 - * QSEQ brands each QUEUED instance so that, when dispatch races 1031 - * dequeue/requeue, the dispatcher can tell whether it still has a claim 1032 - * on the task being dispatched. 1033 - * 1034 - * As some 32bit archs can't do 64bit store_release/load_acquire, 1035 - * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on 1036 - * 32bit machines. The dispatch race window QSEQ protects is very narrow 1037 - * and runs with IRQ disabled. 30 bits should be sufficient. 1038 - */ 1039 - SCX_OPSS_QSEQ_SHIFT = 2, 1040 - }; 1041 - 1042 - /* Use macros to ensure that the type is unsigned long for the masks */ 1043 - #define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) 1044 - #define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) 1045 - 1046 12 /* 1047 13 * NOTE: sched_ext is in the process of growing multiple scheduler support and 1048 14 * scx_root usage is in a transitional state. Naked dereferences are safe if the

-23

kernel/sched/ext.h

··· 8 8 */ 9 9 #ifdef CONFIG_SCHED_CLASS_EXT 10 10 11 - static inline bool scx_kf_allowed_if_unlocked(void) 12 - { 13 - return !current->scx.kf_mask; 14 - } 15 - 16 - static inline bool scx_rq_bypassing(struct rq *rq) 17 - { 18 - return unlikely(rq->scx.flags & SCX_RQ_BYPASSING); 19 - } 20 - 21 - DECLARE_STATIC_KEY_FALSE(scx_ops_allow_queued_wakeup); 22 - 23 - DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); 24 - 25 - /* 26 - * Return the rq currently locked from an scx callback, or NULL if no rq is 27 - * locked. 28 - */ 29 - static inline struct rq *scx_locked_rq(void) 30 - { 31 - return __this_cpu_read(scx_locked_rq_state); 32 - } 33 - 34 11 void scx_tick(struct rq *rq); 35 12 void init_scx_entity(struct sched_ext_entity *scx); 36 13 void scx_pre_fork(struct task_struct *p);

+1061

kernel/sched/ext_internal.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + /* 3 + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst 4 + * 5 + * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. 6 + * Copyright (c) 2025 Tejun Heo <tj@kernel.org> 7 + */ 8 + #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) 9 + 10 + enum scx_consts { 11 + SCX_DSP_DFL_MAX_BATCH = 32, 12 + SCX_DSP_MAX_LOOPS = 32, 13 + SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, 14 + 15 + SCX_EXIT_BT_LEN = 64, 16 + SCX_EXIT_MSG_LEN = 1024, 17 + SCX_EXIT_DUMP_DFL_LEN = 32768, 18 + 19 + SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, 20 + 21 + /* 22 + * Iterating all tasks may take a while. Periodically drop 23 + * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. 24 + */ 25 + SCX_TASK_ITER_BATCH = 32, 26 + }; 27 + 28 + enum scx_exit_kind { 29 + SCX_EXIT_NONE, 30 + SCX_EXIT_DONE, 31 + 32 + SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */ 33 + SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */ 34 + SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ 35 + SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ 36 + 37 + SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ 38 + SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ 39 + SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ 40 + }; 41 + 42 + /* 43 + * An exit code can be specified when exiting with scx_bpf_exit() or scx_exit(), 44 + * corresponding to exit_kind UNREG_BPF and UNREG_KERN respectively. The codes 45 + * are 64bit of the format: 46 + * 47 + * Bits: [63 .. 48 47 .. 32 31 .. 0] 48 + * [ SYS ACT ] [ SYS RSN ] [ USR ] 49 + * 50 + * SYS ACT: System-defined exit actions 51 + * SYS RSN: System-defined exit reasons 52 + * USR : User-defined exit codes and reasons 53 + * 54 + * Using the above, users may communicate intention and context by ORing system 55 + * actions and/or system reasons with a user-defined exit code. 56 + */ 57 + enum scx_exit_code { 58 + /* Reasons */ 59 + SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, 60 + 61 + /* Actions */ 62 + SCX_ECODE_ACT_RESTART = 1LLU << 48, 63 + }; 64 + 65 + /* 66 + * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is 67 + * being disabled. 68 + */ 69 + struct scx_exit_info { 70 + /* %SCX_EXIT_* - broad category of the exit reason */ 71 + enum scx_exit_kind kind; 72 + 73 + /* exit code if gracefully exiting */ 74 + s64 exit_code; 75 + 76 + /* textual representation of the above */ 77 + const char *reason; 78 + 79 + /* backtrace if exiting due to an error */ 80 + unsigned long *bt; 81 + u32 bt_len; 82 + 83 + /* informational message */ 84 + char *msg; 85 + 86 + /* debug dump */ 87 + char *dump; 88 + }; 89 + 90 + /* sched_ext_ops.flags */ 91 + enum scx_ops_flags { 92 + /* 93 + * Keep built-in idle tracking even if ops.update_idle() is implemented. 94 + */ 95 + SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, 96 + 97 + /* 98 + * By default, if there are no other task to run on the CPU, ext core 99 + * keeps running the current task even after its slice expires. If this 100 + * flag is specified, such tasks are passed to ops.enqueue() with 101 + * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. 102 + */ 103 + SCX_OPS_ENQ_LAST = 1LLU << 1, 104 + 105 + /* 106 + * An exiting task may schedule after PF_EXITING is set. In such cases, 107 + * bpf_task_from_pid() may not be able to find the task and if the BPF 108 + * scheduler depends on pid lookup for dispatching, the task will be 109 + * lost leading to various issues including RCU grace period stalls. 110 + * 111 + * To mask this problem, by default, unhashed tasks are automatically 112 + * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't 113 + * depend on pid lookups and wants to handle these tasks directly, the 114 + * following flag can be used. 115 + */ 116 + SCX_OPS_ENQ_EXITING = 1LLU << 2, 117 + 118 + /* 119 + * If set, only tasks with policy set to SCHED_EXT are attached to 120 + * sched_ext. If clear, SCHED_NORMAL tasks are also included. 121 + */ 122 + SCX_OPS_SWITCH_PARTIAL = 1LLU << 3, 123 + 124 + /* 125 + * A migration disabled task can only execute on its current CPU. By 126 + * default, such tasks are automatically put on the CPU's local DSQ with 127 + * the default slice on enqueue. If this ops flag is set, they also go 128 + * through ops.enqueue(). 129 + * 130 + * A migration disabled task never invokes ops.select_cpu() as it can 131 + * only select the current CPU. Also, p->cpus_ptr will only contain its 132 + * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr 133 + * and thus may disagree with cpumask_weight(p->cpus_ptr). 134 + */ 135 + SCX_OPS_ENQ_MIGRATION_DISABLED = 1LLU << 4, 136 + 137 + /* 138 + * Queued wakeup (ttwu_queue) is a wakeup optimization that invokes 139 + * ops.enqueue() on the ops.select_cpu() selected or the wakee's 140 + * previous CPU via IPI (inter-processor interrupt) to reduce cacheline 141 + * transfers. When this optimization is enabled, ops.select_cpu() is 142 + * skipped in some cases (when racing against the wakee switching out). 143 + * As the BPF scheduler may depend on ops.select_cpu() being invoked 144 + * during wakeups, queued wakeup is disabled by default. 145 + * 146 + * If this ops flag is set, queued wakeup optimization is enabled and 147 + * the BPF scheduler must be able to handle ops.enqueue() invoked on the 148 + * wakee's CPU without preceding ops.select_cpu() even for tasks which 149 + * may be executed on multiple CPUs. 150 + */ 151 + SCX_OPS_ALLOW_QUEUED_WAKEUP = 1LLU << 5, 152 + 153 + /* 154 + * If set, enable per-node idle cpumasks. If clear, use a single global 155 + * flat idle cpumask. 156 + */ 157 + SCX_OPS_BUILTIN_IDLE_PER_NODE = 1LLU << 6, 158 + 159 + /* 160 + * CPU cgroup support flags 161 + */ 162 + SCX_OPS_HAS_CGROUP_WEIGHT = 1LLU << 16, /* DEPRECATED, will be removed on 6.18 */ 163 + 164 + SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | 165 + SCX_OPS_ENQ_LAST | 166 + SCX_OPS_ENQ_EXITING | 167 + SCX_OPS_ENQ_MIGRATION_DISABLED | 168 + SCX_OPS_ALLOW_QUEUED_WAKEUP | 169 + SCX_OPS_SWITCH_PARTIAL | 170 + SCX_OPS_BUILTIN_IDLE_PER_NODE | 171 + SCX_OPS_HAS_CGROUP_WEIGHT, 172 + 173 + /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ 174 + __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, 175 + 176 + SCX_OPS_HAS_CPU_PREEMPT = 1LLU << 56, 177 + }; 178 + 179 + /* argument container for ops.init_task() */ 180 + struct scx_init_task_args { 181 + /* 182 + * Set if ops.init_task() is being invoked on the fork path, as opposed 183 + * to the scheduler transition path. 184 + */ 185 + bool fork; 186 + #ifdef CONFIG_EXT_GROUP_SCHED 187 + /* the cgroup the task is joining */ 188 + struct cgroup *cgroup; 189 + #endif 190 + }; 191 + 192 + /* argument container for ops.exit_task() */ 193 + struct scx_exit_task_args { 194 + /* Whether the task exited before running on sched_ext. */ 195 + bool cancelled; 196 + }; 197 + 198 + /* argument container for ops->cgroup_init() */ 199 + struct scx_cgroup_init_args { 200 + /* the weight of the cgroup [1..10000] */ 201 + u32 weight; 202 + 203 + /* bandwidth control parameters from cpu.max and cpu.max.burst */ 204 + u64 bw_period_us; 205 + u64 bw_quota_us; 206 + u64 bw_burst_us; 207 + }; 208 + 209 + enum scx_cpu_preempt_reason { 210 + /* next task is being scheduled by &sched_class_rt */ 211 + SCX_CPU_PREEMPT_RT, 212 + /* next task is being scheduled by &sched_class_dl */ 213 + SCX_CPU_PREEMPT_DL, 214 + /* next task is being scheduled by &sched_class_stop */ 215 + SCX_CPU_PREEMPT_STOP, 216 + /* unknown reason for SCX being preempted */ 217 + SCX_CPU_PREEMPT_UNKNOWN, 218 + }; 219 + 220 + /* 221 + * Argument container for ops->cpu_acquire(). Currently empty, but may be 222 + * expanded in the future. 223 + */ 224 + struct scx_cpu_acquire_args {}; 225 + 226 + /* argument container for ops->cpu_release() */ 227 + struct scx_cpu_release_args { 228 + /* the reason the CPU was preempted */ 229 + enum scx_cpu_preempt_reason reason; 230 + 231 + /* the task that's going to be scheduled on the CPU */ 232 + struct task_struct *task; 233 + }; 234 + 235 + /* 236 + * Informational context provided to dump operations. 237 + */ 238 + struct scx_dump_ctx { 239 + enum scx_exit_kind kind; 240 + s64 exit_code; 241 + const char *reason; 242 + u64 at_ns; 243 + u64 at_jiffies; 244 + }; 245 + 246 + /** 247 + * struct sched_ext_ops - Operation table for BPF scheduler implementation 248 + * 249 + * A BPF scheduler can implement an arbitrary scheduling policy by 250 + * implementing and loading operations in this table. Note that a userland 251 + * scheduling policy can also be implemented using the BPF scheduler 252 + * as a shim layer. 253 + */ 254 + struct sched_ext_ops { 255 + /** 256 + * @select_cpu: Pick the target CPU for a task which is being woken up 257 + * @p: task being woken up 258 + * @prev_cpu: the cpu @p was on before sleeping 259 + * @wake_flags: SCX_WAKE_* 260 + * 261 + * Decision made here isn't final. @p may be moved to any CPU while it 262 + * is getting dispatched for execution later. However, as @p is not on 263 + * the rq at this point, getting the eventual execution CPU right here 264 + * saves a small bit of overhead down the line. 265 + * 266 + * If an idle CPU is returned, the CPU is kicked and will try to 267 + * dispatch. While an explicit custom mechanism can be added, 268 + * select_cpu() serves as the default way to wake up idle CPUs. 269 + * 270 + * @p may be inserted into a DSQ directly by calling 271 + * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped. 272 + * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ 273 + * of the CPU returned by this operation. 274 + * 275 + * Note that select_cpu() is never called for tasks that can only run 276 + * on a single CPU or tasks with migration disabled, as they don't have 277 + * the option to select a different CPU. See select_task_rq() for 278 + * details. 279 + */ 280 + s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); 281 + 282 + /** 283 + * @enqueue: Enqueue a task on the BPF scheduler 284 + * @p: task being enqueued 285 + * @enq_flags: %SCX_ENQ_* 286 + * 287 + * @p is ready to run. Insert directly into a DSQ by calling 288 + * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly 289 + * inserted, the bpf scheduler owns @p and if it fails to dispatch @p, 290 + * the task will stall. 291 + * 292 + * If @p was inserted into a DSQ from ops.select_cpu(), this callback is 293 + * skipped. 294 + */ 295 + void (*enqueue)(struct task_struct *p, u64 enq_flags); 296 + 297 + /** 298 + * @dequeue: Remove a task from the BPF scheduler 299 + * @p: task being dequeued 300 + * @deq_flags: %SCX_DEQ_* 301 + * 302 + * Remove @p from the BPF scheduler. This is usually called to isolate 303 + * the task while updating its scheduling properties (e.g. priority). 304 + * 305 + * The ext core keeps track of whether the BPF side owns a given task or 306 + * not and can gracefully ignore spurious dispatches from BPF side, 307 + * which makes it safe to not implement this method. However, depending 308 + * on the scheduling logic, this can lead to confusing behaviors - e.g. 309 + * scheduling position not being updated across a priority change. 310 + */ 311 + void (*dequeue)(struct task_struct *p, u64 deq_flags); 312 + 313 + /** 314 + * @dispatch: Dispatch tasks from the BPF scheduler and/or user DSQs 315 + * @cpu: CPU to dispatch tasks for 316 + * @prev: previous task being switched out 317 + * 318 + * Called when a CPU's local dsq is empty. The operation should dispatch 319 + * one or more tasks from the BPF scheduler into the DSQs using 320 + * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ 321 + * using scx_bpf_dsq_move_to_local(). 322 + * 323 + * The maximum number of times scx_bpf_dsq_insert() can be called 324 + * without an intervening scx_bpf_dsq_move_to_local() is specified by 325 + * ops.dispatch_max_batch. See the comments on top of the two functions 326 + * for more details. 327 + * 328 + * When not %NULL, @prev is an SCX task with its slice depleted. If 329 + * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in 330 + * @prev->scx.flags, it is not enqueued yet and will be enqueued after 331 + * ops.dispatch() returns. To keep executing @prev, return without 332 + * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST. 333 + */ 334 + void (*dispatch)(s32 cpu, struct task_struct *prev); 335 + 336 + /** 337 + * @tick: Periodic tick 338 + * @p: task running currently 339 + * 340 + * This operation is called every 1/HZ seconds on CPUs which are 341 + * executing an SCX task. Setting @p->scx.slice to 0 will trigger an 342 + * immediate dispatch cycle on the CPU. 343 + */ 344 + void (*tick)(struct task_struct *p); 345 + 346 + /** 347 + * @runnable: A task is becoming runnable on its associated CPU 348 + * @p: task becoming runnable 349 + * @enq_flags: %SCX_ENQ_* 350 + * 351 + * This and the following three functions can be used to track a task's 352 + * execution state transitions. A task becomes ->runnable() on a CPU, 353 + * and then goes through one or more ->running() and ->stopping() pairs 354 + * as it runs on the CPU, and eventually becomes ->quiescent() when it's 355 + * done running on the CPU. 356 + * 357 + * @p is becoming runnable on the CPU because it's 358 + * 359 + * - waking up (%SCX_ENQ_WAKEUP) 360 + * - being moved from another CPU 361 + * - being restored after temporarily taken off the queue for an 362 + * attribute change. 363 + * 364 + * This and ->enqueue() are related but not coupled. This operation 365 + * notifies @p's state transition and may not be followed by ->enqueue() 366 + * e.g. when @p is being dispatched to a remote CPU, or when @p is 367 + * being enqueued on a CPU experiencing a hotplug event. Likewise, a 368 + * task may be ->enqueue()'d without being preceded by this operation 369 + * e.g. after exhausting its slice. 370 + */ 371 + void (*runnable)(struct task_struct *p, u64 enq_flags); 372 + 373 + /** 374 + * @running: A task is starting to run on its associated CPU 375 + * @p: task starting to run 376 + * 377 + * Note that this callback may be called from a CPU other than the 378 + * one the task is going to run on. This can happen when a task 379 + * property is changed (i.e., affinity), since scx_next_task_scx(), 380 + * which triggers this callback, may run on a CPU different from 381 + * the task's assigned CPU. 382 + * 383 + * Therefore, always use scx_bpf_task_cpu(@p) to determine the 384 + * target CPU the task is going to use. 385 + * 386 + * See ->runnable() for explanation on the task state notifiers. 387 + */ 388 + void (*running)(struct task_struct *p); 389 + 390 + /** 391 + * @stopping: A task is stopping execution 392 + * @p: task stopping to run 393 + * @runnable: is task @p still runnable? 394 + * 395 + * Note that this callback may be called from a CPU other than the 396 + * one the task was running on. This can happen when a task 397 + * property is changed (i.e., affinity), since dequeue_task_scx(), 398 + * which triggers this callback, may run on a CPU different from 399 + * the task's assigned CPU. 400 + * 401 + * Therefore, always use scx_bpf_task_cpu(@p) to retrieve the CPU 402 + * the task was running on. 403 + * 404 + * See ->runnable() for explanation on the task state notifiers. If 405 + * !@runnable, ->quiescent() will be invoked after this operation 406 + * returns. 407 + */ 408 + void (*stopping)(struct task_struct *p, bool runnable); 409 + 410 + /** 411 + * @quiescent: A task is becoming not runnable on its associated CPU 412 + * @p: task becoming not runnable 413 + * @deq_flags: %SCX_DEQ_* 414 + * 415 + * See ->runnable() for explanation on the task state notifiers. 416 + * 417 + * @p is becoming quiescent on the CPU because it's 418 + * 419 + * - sleeping (%SCX_DEQ_SLEEP) 420 + * - being moved to another CPU 421 + * - being temporarily taken off the queue for an attribute change 422 + * (%SCX_DEQ_SAVE) 423 + * 424 + * This and ->dequeue() are related but not coupled. This operation 425 + * notifies @p's state transition and may not be preceded by ->dequeue() 426 + * e.g. when @p is being dispatched to a remote CPU. 427 + */ 428 + void (*quiescent)(struct task_struct *p, u64 deq_flags); 429 + 430 + /** 431 + * @yield: Yield CPU 432 + * @from: yielding task 433 + * @to: optional yield target task 434 + * 435 + * If @to is NULL, @from is yielding the CPU to other runnable tasks. 436 + * The BPF scheduler should ensure that other available tasks are 437 + * dispatched before the yielding task. Return value is ignored in this 438 + * case. 439 + * 440 + * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf 441 + * scheduler can implement the request, return %true; otherwise, %false. 442 + */ 443 + bool (*yield)(struct task_struct *from, struct task_struct *to); 444 + 445 + /** 446 + * @core_sched_before: Task ordering for core-sched 447 + * @a: task A 448 + * @b: task B 449 + * 450 + * Used by core-sched to determine the ordering between two tasks. See 451 + * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on 452 + * core-sched. 453 + * 454 + * Both @a and @b are runnable and may or may not currently be queued on 455 + * the BPF scheduler. Should return %true if @a should run before @b. 456 + * %false if there's no required ordering or @b should run before @a. 457 + * 458 + * If not specified, the default is ordering them according to when they 459 + * became runnable. 460 + */ 461 + bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); 462 + 463 + /** 464 + * @set_weight: Set task weight 465 + * @p: task to set weight for 466 + * @weight: new weight [1..10000] 467 + * 468 + * Update @p's weight to @weight. 469 + */ 470 + void (*set_weight)(struct task_struct *p, u32 weight); 471 + 472 + /** 473 + * @set_cpumask: Set CPU affinity 474 + * @p: task to set CPU affinity for 475 + * @cpumask: cpumask of cpus that @p can run on 476 + * 477 + * Update @p's CPU affinity to @cpumask. 478 + */ 479 + void (*set_cpumask)(struct task_struct *p, 480 + const struct cpumask *cpumask); 481 + 482 + /** 483 + * @update_idle: Update the idle state of a CPU 484 + * @cpu: CPU to update the idle state for 485 + * @idle: whether entering or exiting the idle state 486 + * 487 + * This operation is called when @rq's CPU goes or leaves the idle 488 + * state. By default, implementing this operation disables the built-in 489 + * idle CPU tracking and the following helpers become unavailable: 490 + * 491 + * - scx_bpf_select_cpu_dfl() 492 + * - scx_bpf_select_cpu_and() 493 + * - scx_bpf_test_and_clear_cpu_idle() 494 + * - scx_bpf_pick_idle_cpu() 495 + * 496 + * The user also must implement ops.select_cpu() as the default 497 + * implementation relies on scx_bpf_select_cpu_dfl(). 498 + * 499 + * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle 500 + * tracking. 501 + */ 502 + void (*update_idle)(s32 cpu, bool idle); 503 + 504 + /** 505 + * @cpu_acquire: A CPU is becoming available to the BPF scheduler 506 + * @cpu: The CPU being acquired by the BPF scheduler. 507 + * @args: Acquire arguments, see the struct definition. 508 + * 509 + * A CPU that was previously released from the BPF scheduler is now once 510 + * again under its control. 511 + */ 512 + void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); 513 + 514 + /** 515 + * @cpu_release: A CPU is taken away from the BPF scheduler 516 + * @cpu: The CPU being released by the BPF scheduler. 517 + * @args: Release arguments, see the struct definition. 518 + * 519 + * The specified CPU is no longer under the control of the BPF 520 + * scheduler. This could be because it was preempted by a higher 521 + * priority sched_class, though there may be other reasons as well. The 522 + * caller should consult @args->reason to determine the cause. 523 + */ 524 + void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); 525 + 526 + /** 527 + * @init_task: Initialize a task to run in a BPF scheduler 528 + * @p: task to initialize for BPF scheduling 529 + * @args: init arguments, see the struct definition 530 + * 531 + * Either we're loading a BPF scheduler or a new task is being forked. 532 + * Initialize @p for BPF scheduling. This operation may block and can 533 + * be used for allocations, and is called exactly once for a task. 534 + * 535 + * Return 0 for success, -errno for failure. An error return while 536 + * loading will abort loading of the BPF scheduler. During a fork, it 537 + * will abort that specific fork. 538 + */ 539 + s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); 540 + 541 + /** 542 + * @exit_task: Exit a previously-running task from the system 543 + * @p: task to exit 544 + * @args: exit arguments, see the struct definition 545 + * 546 + * @p is exiting or the BPF scheduler is being unloaded. Perform any 547 + * necessary cleanup for @p. 548 + */ 549 + void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); 550 + 551 + /** 552 + * @enable: Enable BPF scheduling for a task 553 + * @p: task to enable BPF scheduling for 554 + * 555 + * Enable @p for BPF scheduling. enable() is called on @p any time it 556 + * enters SCX, and is always paired with a matching disable(). 557 + */ 558 + void (*enable)(struct task_struct *p); 559 + 560 + /** 561 + * @disable: Disable BPF scheduling for a task 562 + * @p: task to disable BPF scheduling for 563 + * 564 + * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. 565 + * Disable BPF scheduling for @p. A disable() call is always matched 566 + * with a prior enable() call. 567 + */ 568 + void (*disable)(struct task_struct *p); 569 + 570 + /** 571 + * @dump: Dump BPF scheduler state on error 572 + * @ctx: debug dump context 573 + * 574 + * Use scx_bpf_dump() to generate BPF scheduler specific debug dump. 575 + */ 576 + void (*dump)(struct scx_dump_ctx *ctx); 577 + 578 + /** 579 + * @dump_cpu: Dump BPF scheduler state for a CPU on error 580 + * @ctx: debug dump context 581 + * @cpu: CPU to generate debug dump for 582 + * @idle: @cpu is currently idle without any runnable tasks 583 + * 584 + * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for 585 + * @cpu. If @idle is %true and this operation doesn't produce any 586 + * output, @cpu is skipped for dump. 587 + */ 588 + void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle); 589 + 590 + /** 591 + * @dump_task: Dump BPF scheduler state for a runnable task on error 592 + * @ctx: debug dump context 593 + * @p: runnable task to generate debug dump for 594 + * 595 + * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for 596 + * @p. 597 + */ 598 + void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); 599 + 600 + #ifdef CONFIG_EXT_GROUP_SCHED 601 + /** 602 + * @cgroup_init: Initialize a cgroup 603 + * @cgrp: cgroup being initialized 604 + * @args: init arguments, see the struct definition 605 + * 606 + * Either the BPF scheduler is being loaded or @cgrp created, initialize 607 + * @cgrp for sched_ext. This operation may block. 608 + * 609 + * Return 0 for success, -errno for failure. An error return while 610 + * loading will abort loading of the BPF scheduler. During cgroup 611 + * creation, it will abort the specific cgroup creation. 612 + */ 613 + s32 (*cgroup_init)(struct cgroup *cgrp, 614 + struct scx_cgroup_init_args *args); 615 + 616 + /** 617 + * @cgroup_exit: Exit a cgroup 618 + * @cgrp: cgroup being exited 619 + * 620 + * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit 621 + * @cgrp for sched_ext. This operation my block. 622 + */ 623 + void (*cgroup_exit)(struct cgroup *cgrp); 624 + 625 + /** 626 + * @cgroup_prep_move: Prepare a task to be moved to a different cgroup 627 + * @p: task being moved 628 + * @from: cgroup @p is being moved from 629 + * @to: cgroup @p is being moved to 630 + * 631 + * Prepare @p for move from cgroup @from to @to. This operation may 632 + * block and can be used for allocations. 633 + * 634 + * Return 0 for success, -errno for failure. An error return aborts the 635 + * migration. 636 + */ 637 + s32 (*cgroup_prep_move)(struct task_struct *p, 638 + struct cgroup *from, struct cgroup *to); 639 + 640 + /** 641 + * @cgroup_move: Commit cgroup move 642 + * @p: task being moved 643 + * @from: cgroup @p is being moved from 644 + * @to: cgroup @p is being moved to 645 + * 646 + * Commit the move. @p is dequeued during this operation. 647 + */ 648 + void (*cgroup_move)(struct task_struct *p, 649 + struct cgroup *from, struct cgroup *to); 650 + 651 + /** 652 + * @cgroup_cancel_move: Cancel cgroup move 653 + * @p: task whose cgroup move is being canceled 654 + * @from: cgroup @p was being moved from 655 + * @to: cgroup @p was being moved to 656 + * 657 + * @p was cgroup_prep_move()'d but failed before reaching cgroup_move(). 658 + * Undo the preparation. 659 + */ 660 + void (*cgroup_cancel_move)(struct task_struct *p, 661 + struct cgroup *from, struct cgroup *to); 662 + 663 + /** 664 + * @cgroup_set_weight: A cgroup's weight is being changed 665 + * @cgrp: cgroup whose weight is being updated 666 + * @weight: new weight [1..10000] 667 + * 668 + * Update @cgrp's weight to @weight. 669 + */ 670 + void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); 671 + 672 + /** 673 + * @cgroup_set_bandwidth: A cgroup's bandwidth is being changed 674 + * @cgrp: cgroup whose bandwidth is being updated 675 + * @period_us: bandwidth control period 676 + * @quota_us: bandwidth control quota 677 + * @burst_us: bandwidth control burst 678 + * 679 + * Update @cgrp's bandwidth control parameters. This is from the cpu.max 680 + * cgroup interface. 681 + * 682 + * @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled 683 + * to. For example, if @period_us is 1_000_000 and @quota_us is 684 + * 2_500_000. @cgrp is entitled to 2.5 CPUs. @burst_us can be 685 + * interpreted in the same fashion and specifies how much @cgrp can 686 + * burst temporarily. The specific control mechanism and thus the 687 + * interpretation of @period_us and burstiness is upto to the BPF 688 + * scheduler. 689 + */ 690 + void (*cgroup_set_bandwidth)(struct cgroup *cgrp, 691 + u64 period_us, u64 quota_us, u64 burst_us); 692 + 693 + #endif /* CONFIG_EXT_GROUP_SCHED */ 694 + 695 + /* 696 + * All online ops must come before ops.cpu_online(). 697 + */ 698 + 699 + /** 700 + * @cpu_online: A CPU became online 701 + * @cpu: CPU which just came up 702 + * 703 + * @cpu just came online. @cpu will not call ops.enqueue() or 704 + * ops.dispatch(), nor run tasks associated with other CPUs beforehand. 705 + */ 706 + void (*cpu_online)(s32 cpu); 707 + 708 + /** 709 + * @cpu_offline: A CPU is going offline 710 + * @cpu: CPU which is going offline 711 + * 712 + * @cpu is going offline. @cpu will not call ops.enqueue() or 713 + * ops.dispatch(), nor run tasks associated with other CPUs afterwards. 714 + */ 715 + void (*cpu_offline)(s32 cpu); 716 + 717 + /* 718 + * All CPU hotplug ops must come before ops.init(). 719 + */ 720 + 721 + /** 722 + * @init: Initialize the BPF scheduler 723 + */ 724 + s32 (*init)(void); 725 + 726 + /** 727 + * @exit: Clean up after the BPF scheduler 728 + * @info: Exit info 729 + * 730 + * ops.exit() is also called on ops.init() failure, which is a bit 731 + * unusual. This is to allow rich reporting through @info on how 732 + * ops.init() failed. 733 + */ 734 + void (*exit)(struct scx_exit_info *info); 735 + 736 + /** 737 + * @dispatch_max_batch: Max nr of tasks that dispatch() can dispatch 738 + */ 739 + u32 dispatch_max_batch; 740 + 741 + /** 742 + * @flags: %SCX_OPS_* flags 743 + */ 744 + u64 flags; 745 + 746 + /** 747 + * @timeout_ms: The maximum amount of time, in milliseconds, that a 748 + * runnable task should be able to wait before being scheduled. The 749 + * maximum timeout may not exceed the default timeout of 30 seconds. 750 + * 751 + * Defaults to the maximum allowed timeout value of 30 seconds. 752 + */ 753 + u32 timeout_ms; 754 + 755 + /** 756 + * @exit_dump_len: scx_exit_info.dump buffer length. If 0, the default 757 + * value of 32768 is used. 758 + */ 759 + u32 exit_dump_len; 760 + 761 + /** 762 + * @hotplug_seq: A sequence number that may be set by the scheduler to 763 + * detect when a hotplug event has occurred during the loading process. 764 + * If 0, no detection occurs. Otherwise, the scheduler will fail to 765 + * load if the sequence number does not match @scx_hotplug_seq on the 766 + * enable path. 767 + */ 768 + u64 hotplug_seq; 769 + 770 + /** 771 + * @name: BPF scheduler's name 772 + * 773 + * Must be a non-zero valid BPF object name including only isalnum(), 774 + * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the 775 + * BPF scheduler is enabled. 776 + */ 777 + char name[SCX_OPS_NAME_LEN]; 778 + 779 + /* internal use only, must be NULL */ 780 + void *priv; 781 + }; 782 + 783 + enum scx_opi { 784 + SCX_OPI_BEGIN = 0, 785 + SCX_OPI_NORMAL_BEGIN = 0, 786 + SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), 787 + SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), 788 + SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), 789 + SCX_OPI_END = SCX_OP_IDX(init), 790 + }; 791 + 792 + /* 793 + * Collection of event counters. Event types are placed in descending order. 794 + */ 795 + struct scx_event_stats { 796 + /* 797 + * If ops.select_cpu() returns a CPU which can't be used by the task, 798 + * the core scheduler code silently picks a fallback CPU. 799 + */ 800 + s64 SCX_EV_SELECT_CPU_FALLBACK; 801 + 802 + /* 803 + * When dispatching to a local DSQ, the CPU may have gone offline in 804 + * the meantime. In this case, the task is bounced to the global DSQ. 805 + */ 806 + s64 SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE; 807 + 808 + /* 809 + * If SCX_OPS_ENQ_LAST is not set, the number of times that a task 810 + * continued to run because there were no other tasks on the CPU. 811 + */ 812 + s64 SCX_EV_DISPATCH_KEEP_LAST; 813 + 814 + /* 815 + * If SCX_OPS_ENQ_EXITING is not set, the number of times that a task 816 + * is dispatched to a local DSQ when exiting. 817 + */ 818 + s64 SCX_EV_ENQ_SKIP_EXITING; 819 + 820 + /* 821 + * If SCX_OPS_ENQ_MIGRATION_DISABLED is not set, the number of times a 822 + * migration disabled task skips ops.enqueue() and is dispatched to its 823 + * local DSQ. 824 + */ 825 + s64 SCX_EV_ENQ_SKIP_MIGRATION_DISABLED; 826 + 827 + /* 828 + * Total number of times a task's time slice was refilled with the 829 + * default value (SCX_SLICE_DFL). 830 + */ 831 + s64 SCX_EV_REFILL_SLICE_DFL; 832 + 833 + /* 834 + * The total duration of bypass modes in nanoseconds. 835 + */ 836 + s64 SCX_EV_BYPASS_DURATION; 837 + 838 + /* 839 + * The number of tasks dispatched in the bypassing mode. 840 + */ 841 + s64 SCX_EV_BYPASS_DISPATCH; 842 + 843 + /* 844 + * The number of times the bypassing mode has been activated. 845 + */ 846 + s64 SCX_EV_BYPASS_ACTIVATE; 847 + }; 848 + 849 + struct scx_sched { 850 + struct sched_ext_ops ops; 851 + DECLARE_BITMAP(has_op, SCX_OPI_END); 852 + 853 + /* 854 + * Dispatch queues. 855 + * 856 + * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. 857 + * This is to avoid live-locking in bypass mode where all tasks are 858 + * dispatched to %SCX_DSQ_GLOBAL and all CPUs consume from it. If 859 + * per-node split isn't sufficient, it can be further split. 860 + */ 861 + struct rhashtable dsq_hash; 862 + struct scx_dispatch_q **global_dsqs; 863 + 864 + /* 865 + * The event counters are in a per-CPU variable to minimize the 866 + * accounting overhead. A system-wide view on the event counter is 867 + * constructed when requested by scx_bpf_events(). 868 + */ 869 + struct scx_event_stats __percpu *event_stats_cpu; 870 + 871 + bool warned_zero_slice; 872 + 873 + atomic_t exit_kind; 874 + struct scx_exit_info *exit_info; 875 + 876 + struct kobject kobj; 877 + 878 + struct kthread_worker *helper; 879 + struct irq_work error_irq_work; 880 + struct kthread_work disable_work; 881 + struct rcu_work rcu_work; 882 + }; 883 + 884 + enum scx_wake_flags { 885 + /* expose select WF_* flags as enums */ 886 + SCX_WAKE_FORK = WF_FORK, 887 + SCX_WAKE_TTWU = WF_TTWU, 888 + SCX_WAKE_SYNC = WF_SYNC, 889 + }; 890 + 891 + enum scx_enq_flags { 892 + /* expose select ENQUEUE_* flags as enums */ 893 + SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, 894 + SCX_ENQ_HEAD = ENQUEUE_HEAD, 895 + SCX_ENQ_CPU_SELECTED = ENQUEUE_RQ_SELECTED, 896 + 897 + /* high 32bits are SCX specific */ 898 + 899 + /* 900 + * Set the following to trigger preemption when calling 901 + * scx_bpf_dsq_insert() with a local dsq as the target. The slice of the 902 + * current task is cleared to zero and the CPU is kicked into the 903 + * scheduling path. Implies %SCX_ENQ_HEAD. 904 + */ 905 + SCX_ENQ_PREEMPT = 1LLU << 32, 906 + 907 + /* 908 + * The task being enqueued was previously enqueued on the current CPU's 909 + * %SCX_DSQ_LOCAL, but was removed from it in a call to the 910 + * scx_bpf_reenqueue_local() kfunc. If scx_bpf_reenqueue_local() was 911 + * invoked in a ->cpu_release() callback, and the task is again 912 + * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the 913 + * task will not be scheduled on the CPU until at least the next invocation 914 + * of the ->cpu_acquire() callback. 915 + */ 916 + SCX_ENQ_REENQ = 1LLU << 40, 917 + 918 + /* 919 + * The task being enqueued is the only task available for the cpu. By 920 + * default, ext core keeps executing such tasks but when 921 + * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the 922 + * %SCX_ENQ_LAST flag set. 923 + * 924 + * The BPF scheduler is responsible for triggering a follow-up 925 + * scheduling event. Otherwise, Execution may stall. 926 + */ 927 + SCX_ENQ_LAST = 1LLU << 41, 928 + 929 + /* high 8 bits are internal */ 930 + __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, 931 + 932 + SCX_ENQ_CLEAR_OPSS = 1LLU << 56, 933 + SCX_ENQ_DSQ_PRIQ = 1LLU << 57, 934 + }; 935 + 936 + enum scx_deq_flags { 937 + /* expose select DEQUEUE_* flags as enums */ 938 + SCX_DEQ_SLEEP = DEQUEUE_SLEEP, 939 + 940 + /* high 32bits are SCX specific */ 941 + 942 + /* 943 + * The generic core-sched layer decided to execute the task even though 944 + * it hasn't been dispatched yet. Dequeue from the BPF side. 945 + */ 946 + SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, 947 + }; 948 + 949 + enum scx_pick_idle_cpu_flags { 950 + SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */ 951 + SCX_PICK_IDLE_IN_NODE = 1LLU << 1, /* pick a CPU in the same target NUMA node */ 952 + }; 953 + 954 + enum scx_kick_flags { 955 + /* 956 + * Kick the target CPU if idle. Guarantees that the target CPU goes 957 + * through at least one full scheduling cycle before going idle. If the 958 + * target CPU can be determined to be currently not idle and going to go 959 + * through a scheduling cycle before going idle, noop. 960 + */ 961 + SCX_KICK_IDLE = 1LLU << 0, 962 + 963 + /* 964 + * Preempt the current task and execute the dispatch path. If the 965 + * current task of the target CPU is an SCX task, its ->scx.slice is 966 + * cleared to zero before the scheduling path is invoked so that the 967 + * task expires and the dispatch path is invoked. 968 + */ 969 + SCX_KICK_PREEMPT = 1LLU << 1, 970 + 971 + /* 972 + * Wait for the CPU to be rescheduled. The scx_bpf_kick_cpu() call will 973 + * return after the target CPU finishes picking the next task. 974 + */ 975 + SCX_KICK_WAIT = 1LLU << 2, 976 + }; 977 + 978 + enum scx_tg_flags { 979 + SCX_TG_ONLINE = 1U << 0, 980 + SCX_TG_INITED = 1U << 1, 981 + }; 982 + 983 + enum scx_enable_state { 984 + SCX_ENABLING, 985 + SCX_ENABLED, 986 + SCX_DISABLING, 987 + SCX_DISABLED, 988 + }; 989 + 990 + static const char *scx_enable_state_str[] = { 991 + [SCX_ENABLING] = "enabling", 992 + [SCX_ENABLED] = "enabled", 993 + [SCX_DISABLING] = "disabling", 994 + [SCX_DISABLED] = "disabled", 995 + }; 996 + 997 + /* 998 + * sched_ext_entity->ops_state 999 + * 1000 + * Used to track the task ownership between the SCX core and the BPF scheduler. 1001 + * State transitions look as follows: 1002 + * 1003 + * NONE -> QUEUEING -> QUEUED -> DISPATCHING 1004 + * ^ | | 1005 + * | v v 1006 + * \-------------------------------/ 1007 + * 1008 + * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call 1009 + * sites for explanations on the conditions being waited upon and why they are 1010 + * safe. Transitions out of them into NONE or QUEUED must store_release and the 1011 + * waiters should load_acquire. 1012 + * 1013 + * Tracking scx_ops_state enables sched_ext core to reliably determine whether 1014 + * any given task can be dispatched by the BPF scheduler at all times and thus 1015 + * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler 1016 + * to try to dispatch any task anytime regardless of its state as the SCX core 1017 + * can safely reject invalid dispatches. 1018 + */ 1019 + enum scx_ops_state { 1020 + SCX_OPSS_NONE, /* owned by the SCX core */ 1021 + SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ 1022 + SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ 1023 + SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ 1024 + 1025 + /* 1026 + * QSEQ brands each QUEUED instance so that, when dispatch races 1027 + * dequeue/requeue, the dispatcher can tell whether it still has a claim 1028 + * on the task being dispatched. 1029 + * 1030 + * As some 32bit archs can't do 64bit store_release/load_acquire, 1031 + * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on 1032 + * 32bit machines. The dispatch race window QSEQ protects is very narrow 1033 + * and runs with IRQ disabled. 30 bits should be sufficient. 1034 + */ 1035 + SCX_OPSS_QSEQ_SHIFT = 2, 1036 + }; 1037 + 1038 + /* Use macros to ensure that the type is unsigned long for the masks */ 1039 + #define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) 1040 + #define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) 1041 + 1042 + DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); 1043 + 1044 + /* 1045 + * Return the rq currently locked from an scx callback, or NULL if no rq is 1046 + * locked. 1047 + */ 1048 + static inline struct rq *scx_locked_rq(void) 1049 + { 1050 + return __this_cpu_read(scx_locked_rq_state); 1051 + } 1052 + 1053 + static inline bool scx_kf_allowed_if_unlocked(void) 1054 + { 1055 + return !current->scx.kf_mask; 1056 + } 1057 + 1058 + static inline bool scx_rq_bypassing(struct rq *rq) 1059 + { 1060 + return unlikely(rq->scx.flags & SCX_RQ_BYPASSING); 1061 + }

Configure Feed

Configure Feed