Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

scx: Allow calling sleepable kfuncs from BPF_PROG_TYPE_SYSCALL

We currently only allow calling sleepable scx kfuncs (i.e.
scx_bpf_create_dsq()) from BPF_PROG_TYPE_STRUCT_OPS progs. The idea here
was that we'd never have to call scx_bpf_create_dsq() outside of a
sched_ext struct_ops callback, but that might not actually be true. For
example, a scheduler could do something like the following:

1. Open and load (not yet attach) a scheduler skel

2. Synchronously call into a BPF_PROG_TYPE_SYSCALL prog from user space.
For example, to initialize an LLC domain, or some other global,
read-only state.

3. Attach the skel, which actually enables the scheduler

The advantage of doing this is that it can preclude having to do pretty
ugly boilerplate like initializing a read-only, statically sized array of
u64[]'s which the kernel consumes literally once at init time to then
create struct bpf_cpumask objects which are actually queried at runtime.

Doing the above is already possible given that we can invoke core BPF
kfuncs, such as bpf_cpumask_create(), from BPF_PROG_TYPE_SYSCALL progs. We
already allow many scx kfuncs to be called from BPF_PROG_TYPE_SYSCALL progs
(e.g. scx_bpf_kick_cpu()). Let's allow the sleepable kfuncs as well.

Signed-off-by: David Vernet <void@manifault.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

authored by

David Vernet and committed by
Tejun Heo
298dec19 c8faf11c

+17 -24
+6 -8
include/linux/sched/ext.h
··· 106 106 * mechanism. See scx_kf_allow(). 107 107 */ 108 108 enum scx_kf_mask { 109 - SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ 110 - /* all non-sleepables may be nested inside SLEEPABLE */ 111 - SCX_KF_SLEEPABLE = 1 << 0, /* sleepable init operations */ 109 + SCX_KF_UNLOCKED = 0, /* sleepable and not rq locked */ 112 110 /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ 113 - SCX_KF_CPU_RELEASE = 1 << 1, /* ops.cpu_release() */ 111 + SCX_KF_CPU_RELEASE = 1 << 0, /* ops.cpu_release() */ 114 112 /* ops.dequeue (in REST) may be nested inside DISPATCH */ 115 - SCX_KF_DISPATCH = 1 << 2, /* ops.dispatch() */ 116 - SCX_KF_ENQUEUE = 1 << 3, /* ops.enqueue() and ops.select_cpu() */ 117 - SCX_KF_SELECT_CPU = 1 << 4, /* ops.select_cpu() */ 118 - SCX_KF_REST = 1 << 5, /* other rq-locked operations */ 113 + SCX_KF_DISPATCH = 1 << 1, /* ops.dispatch() */ 114 + SCX_KF_ENQUEUE = 1 << 2, /* ops.enqueue() and ops.select_cpu() */ 115 + SCX_KF_SELECT_CPU = 1 << 3, /* ops.select_cpu() */ 116 + SCX_KF_REST = 1 << 4, /* other rq-locked operations */ 119 117 120 118 __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | 121 119 SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU | SCX_KF_REST,
+11 -16
kernel/sched/ext.c
··· 1029 1029 return false; 1030 1030 } 1031 1031 1032 - if (unlikely((mask & SCX_KF_SLEEPABLE) && in_interrupt())) { 1033 - scx_ops_error("sleepable kfunc called from non-sleepable context"); 1034 - return false; 1035 - } 1036 - 1037 1032 /* 1038 1033 * Enforce nesting boundaries. e.g. A kfunc which can be called from 1039 1034 * DISPATCH must not be called if we're running DEQUEUE which is nested 1040 - * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE 1041 - * boundary thanks to the above in_interrupt() check. 1035 + * inside ops.dispatch(). We don't need to check boundaries for any 1036 + * blocking kfuncs as the verifier ensures they're only called from 1037 + * sleepable progs. 1042 1038 */ 1043 1039 if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && 1044 1040 (current->scx.kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { ··· 3220 3224 atomic_long_inc(&scx_hotplug_seq); 3221 3225 3222 3226 if (online && SCX_HAS_OP(cpu_online)) 3223 - SCX_CALL_OP(SCX_KF_SLEEPABLE, cpu_online, cpu); 3227 + SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_online, cpu); 3224 3228 else if (!online && SCX_HAS_OP(cpu_offline)) 3225 - SCX_CALL_OP(SCX_KF_SLEEPABLE, cpu_offline, cpu); 3229 + SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_offline, cpu); 3226 3230 else 3227 3231 scx_ops_exit(SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, 3228 3232 "cpu %d going %s, exiting scheduler", cpu, ··· 3386 3390 .fork = fork, 3387 3391 }; 3388 3392 3389 - ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, init_task, p, &args); 3393 + ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, init_task, p, &args); 3390 3394 if (unlikely(ret)) { 3391 3395 ret = ops_sanitize_err("init_task", ret); 3392 3396 return ret; ··· 4644 4648 cpus_read_lock(); 4645 4649 4646 4650 if (scx_ops.init) { 4647 - ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, init); 4651 + ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, init); 4648 4652 if (ret) { 4649 4653 ret = ops_sanitize_err("init", ret); 4650 4654 goto err_disable_unlock_cpus; ··· 5420 5424 * @dsq_id: DSQ to create 5421 5425 * @node: NUMA node to allocate from 5422 5426 * 5423 - * Create a custom DSQ identified by @dsq_id. Can be called from ops.init() and 5424 - * ops.init_task(). 5427 + * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable 5428 + * scx callback, and any BPF_PROG_TYPE_SYSCALL prog. 5425 5429 */ 5426 5430 __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) 5427 5431 { 5428 - if (!scx_kf_allowed(SCX_KF_SLEEPABLE)) 5429 - return -EINVAL; 5430 - 5431 5432 if (unlikely(node >= (int)nr_node_ids || 5432 5433 (node < 0 && node != NUMA_NO_NODE))) 5433 5434 return -EINVAL; ··· 6482 6489 * check using scx_kf_allowed(). 6483 6490 */ 6484 6491 if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 6492 + &scx_kfunc_set_sleepable)) || 6493 + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, 6485 6494 &scx_kfunc_set_sleepable)) || 6486 6495 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 6487 6496 &scx_kfunc_set_select_cpu)) ||