Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'fix-garbage-data-in-task-local-data'
Amery Hung says:
====================
Fix garbage data in task local data
Hi,
The patchset fixes two scenarios where BPF side task local data API may
see garbage data and adds corresponding selftests.
====================
Link: https://patch.msgid.link/20260413190259.358442-1-ameryhung@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+100
-14
+10
-3
tools/testing/selftests/bpf/prog_tests/task_local_data.h
+10
-3
tools/testing/selftests/bpf/prog_tests/task_local_data.h
···
99
99
struct tld_metadata metadata[];
100
100
};
101
101
102
+
/*
103
+
* The unused field ensures map_val.start > 0. On the BPF side, __tld_fetch_key()
104
+
* calculates off by summing map_val.start and tld_key_t.off and treats off == 0
105
+
* as key not cached.
106
+
*/
102
107
struct tld_data_u {
103
-
__u64 start; /* offset of tld_data_u->data in a page */
108
+
__u64 unused;
104
109
char data[] __attribute__((aligned(8)));
105
110
};
106
111
107
112
struct tld_map_value {
108
113
void *data;
109
114
struct tld_meta_u *meta;
115
+
__u16 start; /* offset of tld_data_u->data in a page */
110
116
};
111
117
112
118
struct tld_meta_u * _Atomic tld_meta_p __attribute__((weak));
···
188
182
* is a page in BTF.
189
183
*/
190
184
map_val.data = (void *)(TLD_PAGE_MASK & (intptr_t)data);
191
-
data->start = (~TLD_PAGE_MASK & (intptr_t)data) + sizeof(struct tld_data_u);
185
+
map_val.start = (~TLD_PAGE_MASK & (intptr_t)data) + sizeof(struct tld_data_u);
192
186
map_val.meta = tld_meta_p;
193
187
194
188
err = bpf_map_update_elem(map_fd, &tid_fd, &map_val, 0);
···
247
241
* TLD_DYN_DATA_SIZE is allocated for tld_create_key()
248
242
*/
249
243
if (dyn_data) {
250
-
if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size)
244
+
if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size ||
245
+
tld_meta_p->size > TLD_PAGE_SIZE - sizeof(struct tld_data_u))
251
246
return (tld_key_t){-E2BIG};
252
247
} else {
253
248
if (off + TLD_ROUND_UP(size, 8) > TLD_PAGE_SIZE - sizeof(struct tld_data_u))
+87
-9
tools/testing/selftests/bpf/prog_tests/test_task_local_data.c
+87
-9
tools/testing/selftests/bpf/prog_tests/test_task_local_data.c
···
3
3
#include <bpf/btf.h>
4
4
#include <test_progs.h>
5
5
6
+
/*
7
+
* Only a page is pinned to kernel, so the maximum amount of dynamic data
8
+
* allowed is page_size - sizeof(struct tld_data_u) - static TLD fields.
9
+
*/
10
+
#define TLD_DYN_DATA_SIZE_MAX (getpagesize() - sizeof(struct tld_data_u) - 8)
11
+
6
12
#define TLD_FREE_DATA_ON_THREAD_EXIT
7
-
#define TLD_DYN_DATA_SIZE (getpagesize() - 8)
13
+
#define TLD_DYN_DATA_SIZE TLD_DYN_DATA_SIZE_MAX
8
14
#include "task_local_data.h"
9
15
10
16
struct test_tld_struct {
···
30
24
* sequentially. Users of task local data library should not touch
31
25
* library internal.
32
26
*/
33
-
static void reset_tld(void)
27
+
static void reset_tld(__u16 dyn_data_size)
34
28
{
35
29
if (tld_meta_p) {
36
30
/* Remove TLDs created by tld_create_key() */
37
31
tld_meta_p->cnt = 1;
38
-
tld_meta_p->size = TLD_DYN_DATA_SIZE;
32
+
tld_meta_p->size = dyn_data_size + 8;
39
33
memset(&tld_meta_p->metadata[1], 0,
40
34
(TLD_MAX_DATA_CNT - 1) * sizeof(struct tld_metadata));
41
35
}
···
133
127
tld_key_t key;
134
128
int i, err;
135
129
136
-
reset_tld();
130
+
reset_tld(TLD_DYN_DATA_SIZE_MAX);
137
131
138
132
ASSERT_OK(pthread_mutex_init(&global_mutex, NULL), "pthread_mutex_init");
139
133
···
153
147
154
148
/*
155
149
* Shouldn't be able to store data exceed a page. Create a TLD just big
156
-
* enough to exceed a page. TLDs already created are int value0, int
157
-
* value1, and struct test_tld_struct value2.
150
+
* enough to exceed a page. Data already contains struct tld_data_u,
151
+
* value0 and value1 of int type, and value 2 of struct test_tld_struct.
158
152
*/
159
-
key = tld_create_key("value_not_exist",
160
-
TLD_PAGE_SIZE - 2 * sizeof(int) - sizeof(struct test_tld_struct) + 1);
153
+
key = tld_create_key("value_not_exist", TLD_PAGE_SIZE + 1 -
154
+
sizeof(struct tld_data_u) -
155
+
TLD_ROUND_UP(sizeof(int), 8) * 2 -
156
+
TLD_ROUND_UP(sizeof(struct test_tld_struct), 8));
161
157
ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key");
162
158
163
159
key = tld_create_key("value2", sizeof(struct test_tld_struct));
···
247
239
tld_keys[0] = value0_key;
248
240
249
241
for (j = 0; j < 100; j++) {
250
-
reset_tld();
242
+
reset_tld(TLD_DYN_DATA_SIZE_MAX);
251
243
252
244
for (i = 0; i < TEST_RACE_THREAD_NUM; i++) {
253
245
/*
···
296
288
test_task_local_data__destroy(skel);
297
289
}
298
290
291
+
static void test_task_local_data_dyn_size(__u16 dyn_data_size)
292
+
{
293
+
LIBBPF_OPTS(bpf_test_run_opts, opts);
294
+
struct test_task_local_data *skel;
295
+
int max_keys, i, err, fd, *data;
296
+
char name[TLD_NAME_LEN];
297
+
tld_key_t key;
298
+
299
+
reset_tld(dyn_data_size);
300
+
301
+
skel = test_task_local_data__open_and_load();
302
+
if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
303
+
return;
304
+
305
+
tld_keys = calloc(TLD_MAX_DATA_CNT, sizeof(tld_key_t));
306
+
if (!ASSERT_OK_PTR(tld_keys, "calloc tld_keys"))
307
+
goto out;
308
+
309
+
fd = bpf_map__fd(skel->maps.tld_data_map);
310
+
311
+
/* Create as many int-sized TLDs as the dynamic data size allows */
312
+
max_keys = dyn_data_size / TLD_ROUND_UP(sizeof(int), 8);
313
+
for (i = 0; i < max_keys; i++) {
314
+
snprintf(name, TLD_NAME_LEN, "value_%d", i);
315
+
tld_keys[i] = tld_create_key(name, sizeof(int));
316
+
if (!ASSERT_FALSE(tld_key_is_err(tld_keys[i]), "tld_create_key"))
317
+
goto out;
318
+
319
+
data = tld_get_data(fd, tld_keys[i]);
320
+
if (!ASSERT_OK_PTR(data, "tld_get_data"))
321
+
goto out;
322
+
*data = i;
323
+
}
324
+
325
+
/* The next key should fail with E2BIG */
326
+
key = tld_create_key("overflow", sizeof(int));
327
+
ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key overflow");
328
+
329
+
/* Verify data for value_i do not overlap */
330
+
for (i = 0; i < max_keys; i++) {
331
+
data = tld_get_data(fd, tld_keys[i]);
332
+
if (!ASSERT_OK_PTR(data, "tld_get_data"))
333
+
goto out;
334
+
335
+
ASSERT_EQ(*data, i, "tld_get_data value_i");
336
+
}
337
+
338
+
/* Verify BPF side can still read the static key */
339
+
data = tld_get_data(fd, value0_key);
340
+
if (!ASSERT_OK_PTR(data, "tld_get_data value0"))
341
+
goto out;
342
+
*data = 0xdeadbeef;
343
+
344
+
err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.task_main), &opts);
345
+
ASSERT_OK(err, "run task_main");
346
+
ASSERT_EQ(skel->bss->test_value0, 0xdeadbeef, "tld_get_data value0");
347
+
348
+
out:
349
+
if (tld_keys) {
350
+
free(tld_keys);
351
+
tld_keys = NULL;
352
+
}
353
+
tld_free();
354
+
test_task_local_data__destroy(skel);
355
+
}
356
+
299
357
void test_task_local_data(void)
300
358
{
301
359
if (test__start_subtest("task_local_data_basic"))
302
360
test_task_local_data_basic();
303
361
if (test__start_subtest("task_local_data_race"))
304
362
test_task_local_data_race();
363
+
if (test__start_subtest("task_local_data_dyn_size_small"))
364
+
test_task_local_data_dyn_size(64);
365
+
if (test__start_subtest("task_local_data_dyn_size_zero"))
366
+
test_task_local_data_dyn_size(0);
305
367
}
+3
-2
tools/testing/selftests/bpf/progs/task_local_data.bpf.h
+3
-2
tools/testing/selftests/bpf/progs/task_local_data.bpf.h
···
86
86
};
87
87
88
88
struct tld_data_u {
89
-
__u64 start; /* offset of tld_data_u->data in a page */
89
+
__u64 unused;
90
90
char data[__PAGE_SIZE - sizeof(__u64)] __attribute__((aligned(8)));
91
91
};
92
92
93
93
struct tld_map_value {
94
94
struct tld_data_u __uptr *data;
95
95
struct tld_meta_u __uptr *meta;
96
+
__u16 start; /* offset of tld_data_u->data in a page */
96
97
};
97
98
98
99
typedef struct tld_uptr_dummy {
···
177
176
if (!tld_obj->data_map || !tld_obj->data_map->data || !tld_obj->data_map->meta)
178
177
return 0;
179
178
180
-
start = tld_obj->data_map->data->start;
179
+
start = tld_obj->data_map->start;
181
180
cnt = tld_obj->data_map->meta->cnt;
182
181
metadata = tld_obj->data_map->meta->metadata;
183
182