Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking fix from Thomas Gleixner:
"A single fix to address a race in the static key logic"
* 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
locking/static_key: Fix concurrent static_key_slow_inc()
+46
-6
+13
-3
include/linux/jump_label.h
+13
-3
include/linux/jump_label.h
···
117
117
118
118
#include <linux/atomic.h>
119
119
120
+
#ifdef HAVE_JUMP_LABEL
121
+
120
122
static inline int static_key_count(struct static_key *key)
121
123
{
122
-
return atomic_read(&key->enabled);
124
+
/*
125
+
* -1 means the first static_key_slow_inc() is in progress.
126
+
* static_key_enabled() must return true, so return 1 here.
127
+
*/
128
+
int n = atomic_read(&key->enabled);
129
+
return n >= 0 ? n : 1;
123
130
}
124
-
125
-
#ifdef HAVE_JUMP_LABEL
126
131
127
132
#define JUMP_TYPE_FALSE 0UL
128
133
#define JUMP_TYPE_TRUE 1UL
···
166
161
.entries = (void *)JUMP_TYPE_FALSE }
167
162
168
163
#else /* !HAVE_JUMP_LABEL */
164
+
165
+
static inline int static_key_count(struct static_key *key)
166
+
{
167
+
return atomic_read(&key->enabled);
168
+
}
169
169
170
170
static __always_inline void jump_label_init(void)
171
171
{
+33
-3
kernel/jump_label.c
+33
-3
kernel/jump_label.c
···
58
58
59
59
void static_key_slow_inc(struct static_key *key)
60
60
{
61
+
int v, v1;
62
+
61
63
STATIC_KEY_CHECK_USE();
62
-
if (atomic_inc_not_zero(&key->enabled))
63
-
return;
64
+
65
+
/*
66
+
* Careful if we get concurrent static_key_slow_inc() calls;
67
+
* later calls must wait for the first one to _finish_ the
68
+
* jump_label_update() process. At the same time, however,
69
+
* the jump_label_update() call below wants to see
70
+
* static_key_enabled(&key) for jumps to be updated properly.
71
+
*
72
+
* So give a special meaning to negative key->enabled: it sends
73
+
* static_key_slow_inc() down the slow path, and it is non-zero
74
+
* so it counts as "enabled" in jump_label_update(). Note that
75
+
* atomic_inc_unless_negative() checks >= 0, so roll our own.
76
+
*/
77
+
for (v = atomic_read(&key->enabled); v > 0; v = v1) {
78
+
v1 = atomic_cmpxchg(&key->enabled, v, v + 1);
79
+
if (likely(v1 == v))
80
+
return;
81
+
}
64
82
65
83
jump_label_lock();
66
-
if (atomic_inc_return(&key->enabled) == 1)
84
+
if (atomic_read(&key->enabled) == 0) {
85
+
atomic_set(&key->enabled, -1);
67
86
jump_label_update(key);
87
+
atomic_set(&key->enabled, 1);
88
+
} else {
89
+
atomic_inc(&key->enabled);
90
+
}
68
91
jump_label_unlock();
69
92
}
70
93
EXPORT_SYMBOL_GPL(static_key_slow_inc);
···
95
72
static void __static_key_slow_dec(struct static_key *key,
96
73
unsigned long rate_limit, struct delayed_work *work)
97
74
{
75
+
/*
76
+
* The negative count check is valid even when a negative
77
+
* key->enabled is in use by static_key_slow_inc(); a
78
+
* __static_key_slow_dec() before the first static_key_slow_inc()
79
+
* returns is unbalanced, because all other static_key_slow_inc()
80
+
* instances block while the update is in progress.
81
+
*/
98
82
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
99
83
WARN(atomic_read(&key->enabled) < 0,
100
84
"jump label: negative count!\n");