Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'misc-rqspinlock-updates'
Kumar Kartikeya Dwivedi says:
====================
Misc rqspinlock updates
A couple of changes for rqspinlock, the first disables propagation of AA
and ABBA deadlocks to waiters succeeding the deadlocking waiter. A more
verbose rationale is available in the commit log. The second commit
expands the stress test to introduce a ABBCCA mode that will reliably
exercise the timeout fallback.
====================
Link: https://lore.kernel.org/r/20251029181828.231529-1-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+74
-27
+8
kernel/bpf/rqspinlock.c
+8
kernel/bpf/rqspinlock.c
···
572
572
val = res_atomic_cond_read_acquire(&lock->val, !(VAL & _Q_LOCKED_PENDING_MASK) ||
573
573
RES_CHECK_TIMEOUT(ts, ret, _Q_LOCKED_PENDING_MASK));
574
574
575
+
/* Disable queue destruction when we detect deadlocks. */
576
+
if (ret == -EDEADLK) {
577
+
if (!next)
578
+
next = smp_cond_load_relaxed(&node->next, (VAL));
579
+
arch_mcs_spin_unlock_contended(&next->locked);
580
+
goto err_release_node;
581
+
}
582
+
575
583
waitq_timeout:
576
584
if (ret) {
577
585
/*
+4
-4
tools/testing/selftests/bpf/prog_tests/res_spin_lock.c
+4
-4
tools/testing/selftests/bpf/prog_tests/res_spin_lock.c
···
110
110
ASSERT_OK(load_module("bpf_test_rqspinlock.ko", false), "load module AA");
111
111
sleep(5);
112
112
unload_module("bpf_test_rqspinlock", false);
113
-
114
-
ASSERT_OK(load_module_params("bpf_test_rqspinlock.ko", "test_ab=1", false), "load module ABBA");
115
-
sleep(5);
116
-
unload_module("bpf_test_rqspinlock", false);
113
+
/*
114
+
* Insert bpf_test_rqspinlock.ko manually with test_mode=[1|2] to test
115
+
* other cases (ABBA, ABBCCA).
116
+
*/
117
117
}
+62
-23
tools/testing/selftests/bpf/test_kmods/bpf_test_rqspinlock.c
+62
-23
tools/testing/selftests/bpf/test_kmods/bpf_test_rqspinlock.c
···
22
22
23
23
static rqspinlock_t lock_a;
24
24
static rqspinlock_t lock_b;
25
+
static rqspinlock_t lock_c;
26
+
27
+
enum rqsl_mode {
28
+
RQSL_MODE_AA = 0,
29
+
RQSL_MODE_ABBA,
30
+
RQSL_MODE_ABBCCA,
31
+
};
32
+
33
+
static int test_mode = RQSL_MODE_AA;
34
+
module_param(test_mode, int, 0644);
35
+
MODULE_PARM_DESC(test_mode,
36
+
"rqspinlock test mode: 0 = AA, 1 = ABBA, 2 = ABBCCA");
25
37
26
38
static struct perf_event **rqsl_evts;
27
39
static int rqsl_nevts;
28
-
29
-
static bool test_ab = false;
30
-
module_param(test_ab, bool, 0644);
31
-
MODULE_PARM_DESC(test_ab, "Test ABBA situations instead of AA situations");
32
40
33
41
static struct task_struct **rqsl_threads;
34
42
static int rqsl_nthreads;
···
44
36
45
37
static int pause = 0;
46
38
47
-
static bool nmi_locks_a(int cpu)
39
+
static const char *rqsl_mode_names[] = {
40
+
[RQSL_MODE_AA] = "AA",
41
+
[RQSL_MODE_ABBA] = "ABBA",
42
+
[RQSL_MODE_ABBCCA] = "ABBCCA",
43
+
};
44
+
45
+
struct rqsl_lock_pair {
46
+
rqspinlock_t *worker_lock;
47
+
rqspinlock_t *nmi_lock;
48
+
};
49
+
50
+
static struct rqsl_lock_pair rqsl_get_lock_pair(int cpu)
48
51
{
49
-
return (cpu & 1) && test_ab;
52
+
int mode = READ_ONCE(test_mode);
53
+
54
+
switch (mode) {
55
+
default:
56
+
case RQSL_MODE_AA:
57
+
return (struct rqsl_lock_pair){ &lock_a, &lock_a };
58
+
case RQSL_MODE_ABBA:
59
+
if (cpu & 1)
60
+
return (struct rqsl_lock_pair){ &lock_b, &lock_a };
61
+
return (struct rqsl_lock_pair){ &lock_a, &lock_b };
62
+
case RQSL_MODE_ABBCCA:
63
+
switch (cpu % 3) {
64
+
case 0:
65
+
return (struct rqsl_lock_pair){ &lock_a, &lock_b };
66
+
case 1:
67
+
return (struct rqsl_lock_pair){ &lock_b, &lock_c };
68
+
default:
69
+
return (struct rqsl_lock_pair){ &lock_c, &lock_a };
70
+
}
71
+
}
50
72
}
51
73
52
74
static int rqspinlock_worker_fn(void *arg)
···
89
51
atomic_inc(&rqsl_ready_cpus);
90
52
91
53
while (!kthread_should_stop()) {
54
+
struct rqsl_lock_pair locks = rqsl_get_lock_pair(cpu);
55
+
rqspinlock_t *worker_lock = locks.worker_lock;
56
+
92
57
if (READ_ONCE(pause)) {
93
58
msleep(1000);
94
59
continue;
95
60
}
96
-
if (nmi_locks_a(cpu))
97
-
ret = raw_res_spin_lock_irqsave(&lock_b, flags);
98
-
else
99
-
ret = raw_res_spin_lock_irqsave(&lock_a, flags);
61
+
ret = raw_res_spin_lock_irqsave(worker_lock, flags);
100
62
mdelay(20);
101
-
if (nmi_locks_a(cpu) && !ret)
102
-
raw_res_spin_unlock_irqrestore(&lock_b, flags);
103
-
else if (!ret)
104
-
raw_res_spin_unlock_irqrestore(&lock_a, flags);
63
+
if (!ret)
64
+
raw_res_spin_unlock_irqrestore(worker_lock, flags);
105
65
cpu_relax();
106
66
}
107
67
return 0;
···
127
91
static void nmi_cb(struct perf_event *event, struct perf_sample_data *data,
128
92
struct pt_regs *regs)
129
93
{
94
+
struct rqsl_lock_pair locks;
130
95
int cpu = smp_processor_id();
131
96
unsigned long flags;
132
97
int ret;
···
135
98
if (!cpu || READ_ONCE(pause))
136
99
return;
137
100
138
-
if (nmi_locks_a(cpu))
139
-
ret = raw_res_spin_lock_irqsave(&lock_a, flags);
140
-
else
141
-
ret = raw_res_spin_lock_irqsave(test_ab ? &lock_b : &lock_a, flags);
101
+
locks = rqsl_get_lock_pair(cpu);
102
+
ret = raw_res_spin_lock_irqsave(locks.nmi_lock, flags);
142
103
143
104
mdelay(10);
144
105
145
-
if (nmi_locks_a(cpu) && !ret)
146
-
raw_res_spin_unlock_irqrestore(&lock_a, flags);
147
-
else if (!ret)
148
-
raw_res_spin_unlock_irqrestore(test_ab ? &lock_b : &lock_a, flags);
106
+
if (!ret)
107
+
raw_res_spin_unlock_irqrestore(locks.nmi_lock, flags);
149
108
}
150
109
151
110
static void free_rqsl_threads(void)
···
175
142
int i, ret;
176
143
int ncpus = num_online_cpus();
177
144
178
-
pr_err("Mode = %s\n", test_ab ? "ABBA" : "AA");
145
+
if (test_mode < RQSL_MODE_AA || test_mode > RQSL_MODE_ABBCCA) {
146
+
pr_err("Invalid mode %d\n", test_mode);
147
+
return -EINVAL;
148
+
}
149
+
150
+
pr_err("Mode = %s\n", rqsl_mode_names[test_mode]);
179
151
180
152
if (ncpus < 3)
181
153
return -ENOTSUPP;
182
154
183
155
raw_res_spin_lock_init(&lock_a);
184
156
raw_res_spin_lock_init(&lock_b);
157
+
raw_res_spin_lock_init(&lock_c);
185
158
186
159
rqsl_evts = kcalloc(ncpus - 1, sizeof(*rqsl_evts), GFP_KERNEL);
187
160
if (!rqsl_evts)