Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'update-kf_rcu_protected'
Kumar Kartikeya Dwivedi says:
====================
Update KF_RCU_PROTECTED
Currently, KF_RCU_PROTECTED only applies to iterator APIs and that too
in a convoluted fashion: the presence of this flag on the kfunc is used
to set MEM_RCU in iterator type, and the lack of RCU protection results
in an error only later, once next() or destroy() methods are invoked on
the iterator. While there is no bug, this is certainly a bit unintuitive,
and makes the enforcement of the flag iterator specific.
In the interest of making this flag useful for other upcoming kfuncs,
e.g. scx_bpf_cpu_curr() [0][1], add enforcement for invoking the kfunc
in an RCU critical section in general.
In addition to this, the aforementioned kfunc also needs to return an
RCU protected pointer, which currently has no generic kfunc flag or
annotation. Add such a flag as well while we are at it.
[0]: https://lore.kernel.org/all/20250903212311.369697-3-christian.loehle@arm.com
[1]: https://lore.kernel.org/all/20250909195709.92669-1-arighi@nvidia.com
Changelog:
----------
v2 -> v3
v2: https://lore.kernel.org/bpf/20250917032014.4060112-1-memxor@gmail.com
* Add back lost hunk reworking documentation for KF_RCU_PROTECTED.
v1 -> v2
v1: https://lore.kernel.org/bpf/20250915024731.1494251-1-memxor@gmail.com
* Drop KF_RET_RCU and fold change into KF_RCU_PROTECTED. (Andrea, Alexei)
* Update tests for non-struct pointer return values with KF_RCU_PROTECTED.
====================
Link: https://patch.msgid.link/20250917032755.4068726-1-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+91
-4
+18
-1
Documentation/bpf/kfuncs.rst
+18
-1
Documentation/bpf/kfuncs.rst
···
335
335
pointer. Note as well that a KF_ACQUIRE kfunc that is KF_RCU should very likely
336
336
also be KF_RET_NULL.
337
337
338
+
2.4.8 KF_RCU_PROTECTED flag
339
+
---------------------------
340
+
341
+
The KF_RCU_PROTECTED flag is used to indicate that the kfunc must be invoked in
342
+
an RCU critical section. This is assumed by default in non-sleepable programs,
343
+
and must be explicitly ensured by calling ``bpf_rcu_read_lock`` for sleepable
344
+
ones.
345
+
346
+
If the kfunc returns a pointer value, this flag also enforces that the returned
347
+
pointer is RCU protected, and can only be used while the RCU critical section is
348
+
active.
349
+
350
+
The flag is distinct from the ``KF_RCU`` flag, which only ensures that its
351
+
arguments are at least RCU protected pointers. This may transitively imply that
352
+
RCU protection is ensured, but it does not work in cases of kfuncs which require
353
+
RCU protection but do not take RCU protected arguments.
354
+
338
355
.. _KF_deprecated_flag:
339
356
340
-
2.4.8 KF_DEPRECATED flag
357
+
2.4.9 KF_DEPRECATED flag
341
358
------------------------
342
359
343
360
The KF_DEPRECATED flag is used for kfuncs which are scheduled to be
+10
kernel/bpf/verifier.c
+10
kernel/bpf/verifier.c
···
13931
13931
return -EACCES;
13932
13932
}
13933
13933
13934
+
if (is_kfunc_rcu_protected(&meta) && !in_rcu_cs(env)) {
13935
+
verbose(env, "kernel func %s requires RCU critical section protection\n", func_name);
13936
+
return -EACCES;
13937
+
}
13938
+
13934
13939
/* In case of release function, we get register number of refcounted
13935
13940
* PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now.
13936
13941
*/
···
14049
14044
/* Ensures we don't access the memory after a release_reference() */
14050
14045
if (meta.ref_obj_id)
14051
14046
regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
14047
+
14048
+
if (is_kfunc_rcu_protected(&meta))
14049
+
regs[BPF_REG_0].type |= MEM_RCU;
14052
14050
} else {
14053
14051
mark_reg_known_zero(env, regs, BPF_REG_0);
14054
14052
regs[BPF_REG_0].btf = desc_btf;
···
14060
14052
14061
14053
if (meta.func_id == special_kfunc_list[KF_bpf_get_kmem_cache])
14062
14054
regs[BPF_REG_0].type |= PTR_UNTRUSTED;
14055
+
else if (is_kfunc_rcu_protected(&meta))
14056
+
regs[BPF_REG_0].type |= MEM_RCU;
14063
14057
14064
14058
if (is_iter_next_kfunc(&meta)) {
14065
14059
struct bpf_reg_state *cur_iter;
+1
-1
tools/testing/selftests/bpf/progs/cgroup_read_xattr.c
+1
-1
tools/testing/selftests/bpf/progs/cgroup_read_xattr.c
···
73
73
}
74
74
75
75
SEC("lsm.s/socket_connect")
76
-
__failure __msg("expected an RCU CS")
76
+
__failure __msg("kernel func bpf_iter_css_new requires RCU critical section protection")
77
77
int BPF_PROG(use_css_iter_sleepable_missing_rcu_lock)
78
78
{
79
79
u64 cgrp_id = bpf_get_current_cgroup_id();
+2
-2
tools/testing/selftests/bpf/progs/iters_task_failure.c
+2
-2
tools/testing/selftests/bpf/progs/iters_task_failure.c
···
15
15
void bpf_rcu_read_unlock(void) __ksym;
16
16
17
17
SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
18
-
__failure __msg("expected an RCU CS when using bpf_iter_task_next")
18
+
__failure __msg("kernel func bpf_iter_task_new requires RCU critical section protection")
19
19
int BPF_PROG(iter_tasks_without_lock)
20
20
{
21
21
struct task_struct *pos;
···
27
27
}
28
28
29
29
SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
30
-
__failure __msg("expected an RCU CS when using bpf_iter_css_next")
30
+
__failure __msg("kernel func bpf_iter_css_new requires RCU critical section protection")
31
31
int BPF_PROG(iter_css_without_lock)
32
32
{
33
33
u64 cg_id = bpf_get_current_cgroup_id();
+46
tools/testing/selftests/bpf/progs/iters_testmod.c
+46
tools/testing/selftests/bpf/progs/iters_testmod.c
···
123
123
bpf_iter_num_destroy(&num_it);
124
124
return 0;
125
125
}
126
+
127
+
SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
128
+
__failure __msg("kernel func bpf_kfunc_ret_rcu_test requires RCU critical section protection")
129
+
int iter_ret_rcu_test_protected(const void *ctx)
130
+
{
131
+
struct task_struct *p;
132
+
133
+
p = bpf_kfunc_ret_rcu_test();
134
+
return p->pid;
135
+
}
136
+
137
+
SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
138
+
__failure __msg("R1 type=rcu_ptr_or_null_ expected=")
139
+
int iter_ret_rcu_test_type(const void *ctx)
140
+
{
141
+
struct task_struct *p;
142
+
143
+
bpf_rcu_read_lock();
144
+
p = bpf_kfunc_ret_rcu_test();
145
+
bpf_this_cpu_ptr(p);
146
+
bpf_rcu_read_unlock();
147
+
return 0;
148
+
}
149
+
150
+
SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
151
+
__failure __msg("kernel func bpf_kfunc_ret_rcu_test_nostruct requires RCU critical section protection")
152
+
int iter_ret_rcu_test_protected_nostruct(const void *ctx)
153
+
{
154
+
void *p;
155
+
156
+
p = bpf_kfunc_ret_rcu_test_nostruct(4);
157
+
return *(int *)p;
158
+
}
159
+
160
+
SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
161
+
__failure __msg("R1 type=rdonly_rcu_mem_or_null expected=")
162
+
int iter_ret_rcu_test_type_nostruct(const void *ctx)
163
+
{
164
+
void *p;
165
+
166
+
bpf_rcu_read_lock();
167
+
p = bpf_kfunc_ret_rcu_test_nostruct(4);
168
+
bpf_this_cpu_ptr(p);
169
+
bpf_rcu_read_unlock();
170
+
return 0;
171
+
}
+12
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c
+12
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c
···
218
218
{
219
219
}
220
220
221
+
__bpf_kfunc struct task_struct *bpf_kfunc_ret_rcu_test(void)
222
+
{
223
+
return NULL;
224
+
}
225
+
226
+
__bpf_kfunc int *bpf_kfunc_ret_rcu_test_nostruct(int rdonly_buf_size)
227
+
{
228
+
return NULL;
229
+
}
230
+
221
231
__bpf_kfunc struct bpf_testmod_ctx *
222
232
bpf_testmod_ctx_create(int *err)
223
233
{
···
633
623
BTF_ID_FLAGS(func, bpf_kfunc_trusted_task_test, KF_TRUSTED_ARGS)
634
624
BTF_ID_FLAGS(func, bpf_kfunc_trusted_num_test, KF_TRUSTED_ARGS)
635
625
BTF_ID_FLAGS(func, bpf_kfunc_rcu_task_test, KF_RCU)
626
+
BTF_ID_FLAGS(func, bpf_kfunc_ret_rcu_test, KF_RET_NULL | KF_RCU_PROTECTED)
627
+
BTF_ID_FLAGS(func, bpf_kfunc_ret_rcu_test_nostruct, KF_RET_NULL | KF_RCU_PROTECTED)
636
628
BTF_ID_FLAGS(func, bpf_testmod_ctx_create, KF_ACQUIRE | KF_RET_NULL)
637
629
BTF_ID_FLAGS(func, bpf_testmod_ctx_release, KF_RELEASE)
638
630
BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_1)
+2
tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h
+2
tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h
···
158
158
void bpf_kfunc_trusted_task_test(struct task_struct *ptr) __ksym;
159
159
void bpf_kfunc_trusted_num_test(int *ptr) __ksym;
160
160
void bpf_kfunc_rcu_task_test(struct task_struct *ptr) __ksym;
161
+
struct task_struct *bpf_kfunc_ret_rcu_test(void) __ksym;
162
+
int *bpf_kfunc_ret_rcu_test_nostruct(int rdonly_buf_size) __ksym;
161
163
162
164
int bpf_kfunc_multi_st_ops_test_1(struct st_ops_args *args, u32 id) __ksym;
163
165