Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
bpf: Remove cgroup local storage percpu counter
The percpu counter in cgroup local storage is no longer needed as the
underlying bpf_local_storage can now handle deadlock with the help of
rqspinlock. Remove the percpu counter and related migrate_{disable,
enable}.
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Amery Hung <ameryhung@gmail.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://patch.msgid.link/20260205222916.1788211-8-ameryhung@gmail.com
authored by
Amery Hung
and committed by
Martin KaFai Lau
5254de7b
4a98c2ef
+8
-51
+8
-51
kernel/bpf/bpf_cgrp_storage.c
+8
-51
kernel/bpf/bpf_cgrp_storage.c
···
11
11
12
12
DEFINE_BPF_STORAGE_CACHE(cgroup_cache);
13
13
14
-
static DEFINE_PER_CPU(int, bpf_cgrp_storage_busy);
15
-
16
-
static void bpf_cgrp_storage_lock(void)
17
-
{
18
-
cant_migrate();
19
-
this_cpu_inc(bpf_cgrp_storage_busy);
20
-
}
21
-
22
-
static void bpf_cgrp_storage_unlock(void)
23
-
{
24
-
this_cpu_dec(bpf_cgrp_storage_busy);
25
-
}
26
-
27
-
static bool bpf_cgrp_storage_trylock(void)
28
-
{
29
-
cant_migrate();
30
-
if (unlikely(this_cpu_inc_return(bpf_cgrp_storage_busy) != 1)) {
31
-
this_cpu_dec(bpf_cgrp_storage_busy);
32
-
return false;
33
-
}
34
-
return true;
35
-
}
36
-
37
14
static struct bpf_local_storage __rcu **cgroup_storage_ptr(void *owner)
38
15
{
39
16
struct cgroup *cg = owner;
···
22
45
{
23
46
struct bpf_local_storage *local_storage;
24
47
25
-
rcu_read_lock_dont_migrate();
48
+
rcu_read_lock();
26
49
local_storage = rcu_dereference(cgroup->bpf_cgrp_storage);
27
50
if (!local_storage)
28
51
goto out;
29
52
30
-
bpf_cgrp_storage_lock();
31
53
bpf_local_storage_destroy(local_storage);
32
-
bpf_cgrp_storage_unlock();
33
54
out:
34
-
rcu_read_unlock_migrate();
55
+
rcu_read_unlock();
35
56
}
36
57
37
58
static struct bpf_local_storage_data *
···
58
83
if (IS_ERR(cgroup))
59
84
return ERR_CAST(cgroup);
60
85
61
-
bpf_cgrp_storage_lock();
62
86
sdata = cgroup_storage_lookup(cgroup, map, true);
63
-
bpf_cgrp_storage_unlock();
64
87
cgroup_put(cgroup);
65
88
return sdata ? sdata->data : NULL;
66
89
}
···
75
102
if (IS_ERR(cgroup))
76
103
return PTR_ERR(cgroup);
77
104
78
-
bpf_cgrp_storage_lock();
79
105
sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
80
106
value, map_flags, false, GFP_ATOMIC);
81
-
bpf_cgrp_storage_unlock();
82
107
cgroup_put(cgroup);
83
108
return PTR_ERR_OR_ZERO(sdata);
84
109
}
···
102
131
if (IS_ERR(cgroup))
103
132
return PTR_ERR(cgroup);
104
133
105
-
bpf_cgrp_storage_lock();
106
134
err = cgroup_storage_delete(cgroup, map);
107
-
bpf_cgrp_storage_unlock();
108
135
cgroup_put(cgroup);
109
136
return err;
110
137
}
···
119
150
120
151
static void cgroup_storage_map_free(struct bpf_map *map)
121
152
{
122
-
bpf_local_storage_map_free(map, &cgroup_cache, &bpf_cgrp_storage_busy);
153
+
bpf_local_storage_map_free(map, &cgroup_cache, NULL);
123
154
}
124
155
125
156
/* *gfp_flags* is a hidden argument provided by the verifier */
···
127
158
void *, value, u64, flags, gfp_t, gfp_flags)
128
159
{
129
160
struct bpf_local_storage_data *sdata;
130
-
bool nobusy;
131
161
132
162
WARN_ON_ONCE(!bpf_rcu_lock_held());
133
163
if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
···
135
167
if (!cgroup)
136
168
return (unsigned long)NULL;
137
169
138
-
nobusy = bpf_cgrp_storage_trylock();
139
-
140
-
sdata = cgroup_storage_lookup(cgroup, map, nobusy);
170
+
sdata = cgroup_storage_lookup(cgroup, map, true);
141
171
if (sdata)
142
-
goto unlock;
172
+
goto out;
143
173
144
174
/* only allocate new storage, when the cgroup is refcounted */
145
175
if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
146
-
(flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy)
176
+
(flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
147
177
sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
148
178
value, BPF_NOEXIST, false, gfp_flags);
149
179
150
-
unlock:
151
-
if (nobusy)
152
-
bpf_cgrp_storage_unlock();
180
+
out:
153
181
return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
154
182
}
155
183
156
184
BPF_CALL_2(bpf_cgrp_storage_delete, struct bpf_map *, map, struct cgroup *, cgroup)
157
185
{
158
-
int ret;
159
-
160
186
WARN_ON_ONCE(!bpf_rcu_lock_held());
161
187
if (!cgroup)
162
188
return -EINVAL;
163
189
164
-
if (!bpf_cgrp_storage_trylock())
165
-
return -EBUSY;
166
-
167
-
ret = cgroup_storage_delete(cgroup, map);
168
-
bpf_cgrp_storage_unlock();
169
-
return ret;
190
+
return cgroup_storage_delete(cgroup, map);
170
191
}
171
192
172
193
const struct bpf_map_ops cgrp_storage_map_ops = {