Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

sched_ext: idle: Introduce scx_bpf_select_cpu_and()

Provide a new kfunc, scx_bpf_select_cpu_and(), that can be used to apply
the built-in idle CPU selection policy to a subset of allowed CPU.

This new helper is basically an extension of scx_bpf_select_cpu_dfl().
However, when an idle CPU can't be found, it returns a negative value
instead of @prev_cpu, aligning its behavior more closely with
scx_bpf_pick_idle_cpu().

It also accepts %SCX_PICK_IDLE_* flags, which can be used to enforce
strict selection to @prev_cpu's node (%SCX_PICK_IDLE_IN_NODE), or to
request only a full-idle SMT core (%SCX_PICK_IDLE_CORE), while applying
the built-in selection logic.

With this helper, BPF schedulers can apply the built-in idle CPU
selection policy restricted to any arbitrary subset of CPUs.

Example usage
=============

Possible usage in ops.select_cpu():

s32 BPF_STRUCT_OPS(foo_select_cpu, struct task_struct *p,
s32 prev_cpu, u64 wake_flags)
{
const struct cpumask *cpus = task_allowed_cpus(p) ?: p->cpus_ptr;
s32 cpu;

cpu = scx_bpf_select_cpu_and(p, prev_cpu, wake_flags, cpus, 0);
if (cpu >= 0) {
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
return cpu;
}

return prev_cpu;
}

Results
=======

Load distribution on a 4 sockets, 4 cores per socket system, simulated
using virtme-ng, running a modified version of scx_bpfland that uses
scx_bpf_select_cpu_and() with 0xff00 as the allowed subset of CPUs:

$ vng --cpu 16,sockets=4,cores=4,threads=1
...
$ stress-ng -c 16
...
$ htop
...
0[ 0.0%] 8[||||||||||||||||||||||||100.0%]
1[ 0.0%] 9[||||||||||||||||||||||||100.0%]
2[ 0.0%] 10[||||||||||||||||||||||||100.0%]
3[ 0.0%] 11[||||||||||||||||||||||||100.0%]
4[ 0.0%] 12[||||||||||||||||||||||||100.0%]
5[ 0.0%] 13[||||||||||||||||||||||||100.0%]
6[ 0.0%] 14[||||||||||||||||||||||||100.0%]
7[ 0.0%] 15[||||||||||||||||||||||||100.0%]

With scx_bpf_select_cpu_dfl() tasks would be distributed evenly across
all the available CPUs.

Signed-off-by: Andrea Righi <arighi@nvidia.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

authored by

Andrea Righi and committed by
Tejun Heo
683d2d0f c2d8b2a5

+58
+1
kernel/sched/ext.c
··· 465 465 * idle CPU tracking and the following helpers become unavailable: 466 466 * 467 467 * - scx_bpf_select_cpu_dfl() 468 + * - scx_bpf_select_cpu_and() 468 469 * - scx_bpf_test_and_clear_cpu_idle() 469 470 * - scx_bpf_pick_idle_cpu() 470 471 *
+55
kernel/sched/ext_idle.c
··· 913 913 } 914 914 915 915 /** 916 + * scx_bpf_select_cpu_and - Pick an idle CPU usable by task @p, 917 + * prioritizing those in @cpus_allowed 918 + * @p: task_struct to select a CPU for 919 + * @prev_cpu: CPU @p was on previously 920 + * @wake_flags: %SCX_WAKE_* flags 921 + * @cpus_allowed: cpumask of allowed CPUs 922 + * @flags: %SCX_PICK_IDLE* flags 923 + * 924 + * Can only be called from ops.select_cpu() or ops.enqueue() if the 925 + * built-in CPU selection is enabled: ops.update_idle() is missing or 926 + * %SCX_OPS_KEEP_BUILTIN_IDLE is set. 927 + * 928 + * @p, @prev_cpu and @wake_flags match ops.select_cpu(). 929 + * 930 + * Returns the selected idle CPU, which will be automatically awakened upon 931 + * returning from ops.select_cpu() and can be used for direct dispatch, or 932 + * a negative value if no idle CPU is available. 933 + */ 934 + __bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags, 935 + const struct cpumask *cpus_allowed, u64 flags) 936 + { 937 + s32 cpu; 938 + 939 + if (!ops_cpu_valid(prev_cpu, NULL)) 940 + return -EINVAL; 941 + 942 + if (!check_builtin_idle_enabled()) 943 + return -EBUSY; 944 + 945 + if (!scx_kf_allowed(SCX_KF_SELECT_CPU | SCX_KF_ENQUEUE)) 946 + return -EPERM; 947 + 948 + #ifdef CONFIG_SMP 949 + /* 950 + * This may also be called from ops.enqueue(), so we need to handle 951 + * per-CPU tasks as well. For these tasks, we can skip all idle CPU 952 + * selection optimizations and simply check whether the previously 953 + * used CPU is idle and within the allowed cpumask. 954 + */ 955 + if (p->nr_cpus_allowed == 1) { 956 + if (cpumask_test_cpu(prev_cpu, cpus_allowed) && 957 + scx_idle_test_and_clear_cpu(prev_cpu)) 958 + return prev_cpu; 959 + return -EBUSY; 960 + } 961 + cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, cpus_allowed, flags); 962 + #else 963 + cpu = -EBUSY; 964 + #endif 965 + 966 + return cpu; 967 + } 968 + 969 + /** 916 970 * scx_bpf_get_idle_cpumask_node - Get a referenced kptr to the 917 971 * idle-tracking per-CPU cpumask of a target NUMA node. 918 972 * @node: target NUMA node ··· 1274 1220 1275 1221 BTF_KFUNCS_START(scx_kfunc_ids_select_cpu) 1276 1222 BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_RCU) 1223 + BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) 1277 1224 BTF_KFUNCS_END(scx_kfunc_ids_select_cpu) 1278 1225 1279 1226 static const struct btf_kfunc_id_set scx_kfunc_set_select_cpu = {
+2
tools/sched_ext/include/scx/common.bpf.h
··· 48 48 49 49 s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; 50 50 s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym; 51 + s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags, 52 + const struct cpumask *cpus_allowed, u64 flags) __ksym __weak; 51 53 void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak; 52 54 void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak; 53 55 u32 scx_bpf_dispatch_nr_slots(void) __ksym;