Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'selftests-bpf-fix-for-bpf_signal-stalls-watchdog-for-test_progs'
Eduard Zingerman says:
====================
selftests/bpf: fix for bpf_signal stalls, watchdog for test_progs
Test case 'bpf_signal' had been recently reported to stall, both on
the mailing list [1] and CI [2]. The stall is caused by CPU cycles
perf event not being delivered within expected time frame, before test
process enters system call and waits indefinitely.
This patch-set addresses the issue in several ways:
- A watchdog timer is added to test_progs.c runner:
- it prints current sub-test name to stderr if sub-test takes longer
than 10 seconds to finish;
- it terminates process executing sub-test if sub-test takes longer
than 120 seconds to finish.
- The test case is updated to await perf event notification with a
timeout and a few retries, this serves two purposes:
- busy loops longer to increase the time frame for CPU cycles event
generation/delivery;
- makes a timeout, not stall, a worst case scenario.
- The test case is updated to lower frequency of perf events, as high
frequency of such events caused events generation throttling,
which in turn delayed events delivery by amount of time sufficient
to cause test case failure.
Note:
librt pthread-based timer API is used to implement watchdog timer.
I chose this API over SIGALRM because signal handler execution
within test process context was sufficient to trigger perf event
delivery for send_signal/send_signal_nmi_thread_remote test case,
w/o any additional changes. Thus I concluded that SIGALRM based
implementation interferes with tests execution.
[1] https://lore.kernel.org/bpf/CAP01T75OUeE8E-Lw9df84dm8ag2YmHW619f1DmPSVZ5_O89+Bg@mail.gmail.com/
[2] https://github.com/kernel-patches/bpf/actions/runs/11791485271/job/32843996871
====================
Link: https://lore.kernel.org/r/20241112110906.3045278-1-eddyz87@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+166
-20
+1
tools/testing/selftests/bpf/Makefile
+1
tools/testing/selftests/bpf/Makefile
+21
tools/testing/selftests/bpf/io_helpers.c
+21
tools/testing/selftests/bpf/io_helpers.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#include <sys/select.h>
3
+
#include <unistd.h>
4
+
#include <errno.h>
5
+
6
+
int read_with_timeout(int fd, char *buf, size_t count, long usec)
7
+
{
8
+
const long M = 1000 * 1000;
9
+
struct timeval tv = { usec / M, usec % M };
10
+
fd_set fds;
11
+
int err;
12
+
13
+
FD_ZERO(&fds);
14
+
FD_SET(fd, &fds);
15
+
err = select(fd + 1, &fds, NULL, NULL, &tv);
16
+
if (err < 0)
17
+
return err;
18
+
if (FD_ISSET(fd, &fds))
19
+
return read(fd, buf, count);
20
+
return -EAGAIN;
21
+
}
+7
tools/testing/selftests/bpf/io_helpers.h
+7
tools/testing/selftests/bpf/io_helpers.h
+4
-4
tools/testing/selftests/bpf/prog_tests/bpf_iter.c
+4
-4
tools/testing/selftests/bpf/prog_tests/bpf_iter.c
···
265
265
266
266
linfo.task.tid = 0;
267
267
linfo.task.pid = getpid();
268
-
/* This includes the parent thread, this thread,
268
+
/* This includes the parent thread, this thread, watchdog timer thread
269
269
* and the do_nothing_wait thread
270
270
*/
271
-
test_task_common(&opts, 2, 1);
271
+
test_task_common(&opts, 3, 1);
272
272
273
273
test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
274
274
ASSERT_GT(num_unknown_tid, 2, "check_num_unknown_tid");
···
297
297
opts.link_info = &linfo;
298
298
opts.link_info_len = sizeof(linfo);
299
299
300
-
test_task_common(&opts, 1, 1);
300
+
test_task_common(&opts, 2, 1);
301
301
}
302
302
303
303
static void test_task_pidfd(void)
···
315
315
opts.link_info = &linfo;
316
316
opts.link_info_len = sizeof(linfo);
317
317
318
-
test_task_common(&opts, 1, 1);
318
+
test_task_common(&opts, 2, 1);
319
319
320
320
close(pidfd);
321
321
}
+2
-2
tools/testing/selftests/bpf/prog_tests/iters.c
+2
-2
tools/testing/selftests/bpf/prog_tests/iters.c
···
192
192
syscall(SYS_getpgid);
193
193
iters_task__detach(skel);
194
194
ASSERT_EQ(skel->bss->procs_cnt, 1, "procs_cnt");
195
-
ASSERT_EQ(skel->bss->threads_cnt, thread_num + 1, "threads_cnt");
196
-
ASSERT_EQ(skel->bss->proc_threads_cnt, thread_num + 1, "proc_threads_cnt");
195
+
ASSERT_EQ(skel->bss->threads_cnt, thread_num + 2, "threads_cnt");
196
+
ASSERT_EQ(skel->bss->proc_threads_cnt, thread_num + 2, "proc_threads_cnt");
197
197
ASSERT_EQ(skel->bss->invalid_cnt, 0, "invalid_cnt");
198
198
pthread_mutex_unlock(&do_nothing_mutex);
199
199
for (int i = 0; i < thread_num; i++)
+21
-14
tools/testing/selftests/bpf/prog_tests/send_signal.c
+21
-14
tools/testing/selftests/bpf/prog_tests/send_signal.c
···
3
3
#include <sys/time.h>
4
4
#include <sys/resource.h>
5
5
#include "test_send_signal_kern.skel.h"
6
+
#include "io_helpers.h"
6
7
7
8
static int sigusr1_received;
8
9
···
25
24
int pipe_c2p[2], pipe_p2c[2];
26
25
int err = -1, pmu_fd = -1;
27
26
volatile int j = 0;
27
+
int retry_count;
28
28
char buf[256];
29
29
pid_t pid;
30
30
int old_prio;
···
165
163
/* notify child that bpf program can send_signal now */
166
164
ASSERT_EQ(write(pipe_p2c[1], buf, 1), 1, "pipe_write");
167
165
168
-
/* For the remote test, the BPF program is triggered from this
169
-
* process but the other process/thread is signaled.
170
-
*/
171
-
if (remote) {
172
-
if (!attr) {
173
-
for (int i = 0; i < 10; i++)
174
-
usleep(1);
175
-
} else {
176
-
for (int i = 0; i < 100000000; i++)
177
-
j /= i + 1;
166
+
for (retry_count = 0;;) {
167
+
/* For the remote test, the BPF program is triggered from this
168
+
* process but the other process/thread is signaled.
169
+
*/
170
+
if (remote) {
171
+
if (!attr) {
172
+
for (int i = 0; i < 10; i++)
173
+
usleep(1);
174
+
} else {
175
+
for (int i = 0; i < 100000000; i++)
176
+
j /= i + 1;
177
+
}
178
178
}
179
+
/* wait for result */
180
+
err = read_with_timeout(pipe_c2p[0], buf, 1, 100);
181
+
if (err == -EAGAIN && retry_count++ < 10000)
182
+
continue;
183
+
break;
179
184
}
180
-
181
-
/* wait for result */
182
-
err = read(pipe_c2p[0], buf, 1);
183
185
if (!ASSERT_GE(err, 0, "reading pipe"))
184
186
goto disable_pmu;
185
187
if (!ASSERT_GT(err, 0, "reading pipe error: size 0")) {
···
229
223
static void test_send_signal_nmi(bool signal_thread, bool remote)
230
224
{
231
225
struct perf_event_attr attr = {
232
-
.sample_period = 1,
226
+
.freq = 1,
227
+
.sample_freq = 1000,
233
228
.type = PERF_TYPE_HARDWARE,
234
229
.config = PERF_COUNT_HW_CPU_CYCLES,
235
230
};
+104
tools/testing/selftests/bpf/test_progs.c
+104
tools/testing/selftests/bpf/test_progs.c
···
16
16
#include <sys/socket.h>
17
17
#include <sys/un.h>
18
18
#include <bpf/btf.h>
19
+
#include <time.h>
19
20
#include "json_writer.h"
20
21
21
22
#include "network_helpers.h"
···
178
177
};
179
178
180
179
return syscall(__NR_nanosleep, &ts, NULL);
180
+
}
181
+
182
+
/* Watchdog timer is started by watchdog_start() and stopped by watchdog_stop().
183
+
* If timer is active for longer than env.secs_till_notify,
184
+
* it prints the name of the current test to the stderr.
185
+
* If timer is active for longer than env.secs_till_kill,
186
+
* it kills the thread executing the test by sending a SIGSEGV signal to it.
187
+
*/
188
+
static void watchdog_timer_func(union sigval sigval)
189
+
{
190
+
struct itimerspec timeout = {};
191
+
char test_name[256];
192
+
int err;
193
+
194
+
if (env.subtest_state)
195
+
snprintf(test_name, sizeof(test_name), "%s/%s",
196
+
env.test->test_name, env.subtest_state->name);
197
+
else
198
+
snprintf(test_name, sizeof(test_name), "%s",
199
+
env.test->test_name);
200
+
201
+
switch (env.watchdog_state) {
202
+
case WD_NOTIFY:
203
+
fprintf(env.stderr_saved, "WATCHDOG: test case %s executes for %d seconds...\n",
204
+
test_name, env.secs_till_notify);
205
+
timeout.it_value.tv_sec = env.secs_till_kill - env.secs_till_notify;
206
+
env.watchdog_state = WD_KILL;
207
+
err = timer_settime(env.watchdog, 0, &timeout, NULL);
208
+
if (err)
209
+
fprintf(env.stderr_saved, "Failed to arm watchdog timer\n");
210
+
break;
211
+
case WD_KILL:
212
+
fprintf(env.stderr_saved,
213
+
"WATCHDOG: test case %s executes for %d seconds, terminating with SIGSEGV\n",
214
+
test_name, env.secs_till_kill);
215
+
pthread_kill(env.main_thread, SIGSEGV);
216
+
break;
217
+
}
218
+
}
219
+
220
+
static void watchdog_start(void)
221
+
{
222
+
struct itimerspec timeout = {};
223
+
int err;
224
+
225
+
if (env.secs_till_kill == 0)
226
+
return;
227
+
if (env.secs_till_notify > 0) {
228
+
env.watchdog_state = WD_NOTIFY;
229
+
timeout.it_value.tv_sec = env.secs_till_notify;
230
+
} else {
231
+
env.watchdog_state = WD_KILL;
232
+
timeout.it_value.tv_sec = env.secs_till_kill;
233
+
}
234
+
err = timer_settime(env.watchdog, 0, &timeout, NULL);
235
+
if (err)
236
+
fprintf(env.stderr_saved, "Failed to start watchdog timer\n");
237
+
}
238
+
239
+
static void watchdog_stop(void)
240
+
{
241
+
struct itimerspec timeout = {};
242
+
int err;
243
+
244
+
env.watchdog_state = WD_NOTIFY;
245
+
err = timer_settime(env.watchdog, 0, &timeout, NULL);
246
+
if (err)
247
+
fprintf(env.stderr_saved, "Failed to stop watchdog timer\n");
248
+
}
249
+
250
+
static void watchdog_init(void)
251
+
{
252
+
struct sigevent watchdog_sev = {
253
+
.sigev_notify = SIGEV_THREAD,
254
+
.sigev_notify_function = watchdog_timer_func,
255
+
};
256
+
int err;
257
+
258
+
env.main_thread = pthread_self();
259
+
err = timer_create(CLOCK_MONOTONIC, &watchdog_sev, &env.watchdog);
260
+
if (err)
261
+
fprintf(stderr, "Failed to initialize watchdog timer\n");
181
262
}
182
263
183
264
static bool should_run(struct test_selector *sel, int num, const char *name)
···
598
515
599
516
env.subtest_state = subtest_state;
600
517
stdio_hijack_init(&subtest_state->log_buf, &subtest_state->log_cnt);
518
+
watchdog_start();
601
519
602
520
return true;
603
521
}
···
864
780
ARG_DEBUG = -1,
865
781
ARG_JSON_SUMMARY = 'J',
866
782
ARG_TRAFFIC_MONITOR = 'm',
783
+
ARG_WATCHDOG_TIMEOUT = 'w',
867
784
};
868
785
869
786
static const struct argp_option opts[] = {
···
895
810
{ "traffic-monitor", ARG_TRAFFIC_MONITOR, "NAMES", 0,
896
811
"Monitor network traffic of tests with name matching the pattern (supports '*' wildcard)." },
897
812
#endif
813
+
{ "watchdog-timeout", ARG_WATCHDOG_TIMEOUT, "SECONDS", 0,
814
+
"Kill the process if tests are not making progress for specified number of seconds." },
898
815
{},
899
816
};
900
817
···
1122
1035
true);
1123
1036
break;
1124
1037
#endif
1038
+
case ARG_WATCHDOG_TIMEOUT:
1039
+
env->secs_till_kill = atoi(arg);
1040
+
if (env->secs_till_kill < 0) {
1041
+
fprintf(stderr, "Invalid watchdog timeout: %s.\n", arg);
1042
+
return -EINVAL;
1043
+
}
1044
+
if (env->secs_till_kill < env->secs_till_notify) {
1045
+
env->secs_till_notify = 0;
1046
+
}
1047
+
break;
1125
1048
default:
1126
1049
return ARGP_ERR_UNKNOWN;
1127
1050
}
···
1360
1263
1361
1264
stdio_hijack(&state->log_buf, &state->log_cnt);
1362
1265
1266
+
watchdog_start();
1363
1267
if (test->run_test)
1364
1268
test->run_test();
1365
1269
else if (test->run_serial_test)
1366
1270
test->run_serial_test();
1271
+
watchdog_stop();
1367
1272
1368
1273
/* ensure last sub-test is finalized properly */
1369
1274
if (env.subtest_state)
···
1806
1707
static int worker_main(int sock)
1807
1708
{
1808
1709
save_netns();
1710
+
watchdog_init();
1809
1711
1810
1712
while (true) {
1811
1713
/* receive command */
···
1916
1816
1917
1817
sigaction(SIGSEGV, &sigact, NULL);
1918
1818
1819
+
env.secs_till_notify = 10;
1820
+
env.secs_till_kill = 120;
1919
1821
err = argp_parse(&argp, argc, argv, 0, NULL, &env);
1920
1822
if (err)
1921
1823
return err;
···
1925
1823
err = cd_flavor_subdir(argv[0]);
1926
1824
if (err)
1927
1825
return err;
1826
+
1827
+
watchdog_init();
1928
1828
1929
1829
/* Use libbpf 1.0 API mode */
1930
1830
libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
+6
tools/testing/selftests/bpf/test_progs.h
+6
tools/testing/selftests/bpf/test_progs.h
···
131
131
pid_t *worker_pids; /* array of worker pids */
132
132
int *worker_socks; /* array of worker socks */
133
133
int *worker_current_test; /* array of current running test for each worker */
134
+
135
+
pthread_t main_thread;
136
+
int secs_till_notify;
137
+
int secs_till_kill;
138
+
timer_t watchdog; /* watch for stalled tests/subtests */
139
+
enum { WD_NOTIFY, WD_KILL } watchdog_state;
134
140
};
135
141
136
142
#define MAX_LOG_TRUNK_SIZE 8192