Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
locking: Move common qspinlock helpers to a private header
Move qspinlock helper functions that encode, decode tail word, set and
clear the pending and locked bits, and other miscellaneous definitions
and macros to a private header. To this end, create a qspinlock.h header
file in kernel/locking. Subsequent commits will introduce a modified
qspinlock slow path function, thus moving shared code to a private
header will help minimize unnecessary code duplication.
Reviewed-by: Barret Rhoden <brho@google.com>
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/r/20250316040541.108729-3-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
authored by
Kumar Kartikeya Dwivedi
and committed by
Alexei Starovoitov
ac08f68f
8707d1ee
+206
-188
+5
-188
kernel/locking/qspinlock.c
+5
-188
kernel/locking/qspinlock.c
···
25
25
#include <trace/events/lock.h>
26
26
27
27
/*
28
-
* Include queued spinlock statistics code
28
+
* Include queued spinlock definitions and statistics code
29
29
*/
30
+
#include "qspinlock.h"
30
31
#include "qspinlock_stat.h"
31
32
32
33
/*
···
68
67
*/
69
68
70
69
#include "mcs_spinlock.h"
71
-
#define MAX_NODES 4
72
-
73
-
/*
74
-
* On 64-bit architectures, the mcs_spinlock structure will be 16 bytes in
75
-
* size and four of them will fit nicely in one 64-byte cacheline. For
76
-
* pvqspinlock, however, we need more space for extra data. To accommodate
77
-
* that, we insert two more long words to pad it up to 32 bytes. IOW, only
78
-
* two of them can fit in a cacheline in this case. That is OK as it is rare
79
-
* to have more than 2 levels of slowpath nesting in actual use. We don't
80
-
* want to penalize pvqspinlocks to optimize for a rare case in native
81
-
* qspinlocks.
82
-
*/
83
-
struct qnode {
84
-
struct mcs_spinlock mcs;
85
-
#ifdef CONFIG_PARAVIRT_SPINLOCKS
86
-
long reserved[2];
87
-
#endif
88
-
};
89
-
90
-
/*
91
-
* The pending bit spinning loop count.
92
-
* This heuristic is used to limit the number of lockword accesses
93
-
* made by atomic_cond_read_relaxed when waiting for the lock to
94
-
* transition out of the "== _Q_PENDING_VAL" state. We don't spin
95
-
* indefinitely because there's no guarantee that we'll make forward
96
-
* progress.
97
-
*/
98
-
#ifndef _Q_PENDING_LOOPS
99
-
#define _Q_PENDING_LOOPS 1
100
-
#endif
101
70
102
71
/*
103
72
* Per-CPU queue node structures; we can never have more than 4 nested
···
77
106
*
78
107
* PV doubles the storage and uses the second cacheline for PV state.
79
108
*/
80
-
static DEFINE_PER_CPU_ALIGNED(struct qnode, qnodes[MAX_NODES]);
81
-
82
-
/*
83
-
* We must be able to distinguish between no-tail and the tail at 0:0,
84
-
* therefore increment the cpu number by one.
85
-
*/
86
-
87
-
static inline __pure u32 encode_tail(int cpu, int idx)
88
-
{
89
-
u32 tail;
90
-
91
-
tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
92
-
tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
93
-
94
-
return tail;
95
-
}
96
-
97
-
static inline __pure struct mcs_spinlock *decode_tail(u32 tail)
98
-
{
99
-
int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
100
-
int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
101
-
102
-
return per_cpu_ptr(&qnodes[idx].mcs, cpu);
103
-
}
104
-
105
-
static inline __pure
106
-
struct mcs_spinlock *grab_mcs_node(struct mcs_spinlock *base, int idx)
107
-
{
108
-
return &((struct qnode *)base + idx)->mcs;
109
-
}
110
-
111
-
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
112
-
113
-
#if _Q_PENDING_BITS == 8
114
-
/**
115
-
* clear_pending - clear the pending bit.
116
-
* @lock: Pointer to queued spinlock structure
117
-
*
118
-
* *,1,* -> *,0,*
119
-
*/
120
-
static __always_inline void clear_pending(struct qspinlock *lock)
121
-
{
122
-
WRITE_ONCE(lock->pending, 0);
123
-
}
124
-
125
-
/**
126
-
* clear_pending_set_locked - take ownership and clear the pending bit.
127
-
* @lock: Pointer to queued spinlock structure
128
-
*
129
-
* *,1,0 -> *,0,1
130
-
*
131
-
* Lock stealing is not allowed if this function is used.
132
-
*/
133
-
static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
134
-
{
135
-
WRITE_ONCE(lock->locked_pending, _Q_LOCKED_VAL);
136
-
}
137
-
138
-
/*
139
-
* xchg_tail - Put in the new queue tail code word & retrieve previous one
140
-
* @lock : Pointer to queued spinlock structure
141
-
* @tail : The new queue tail code word
142
-
* Return: The previous queue tail code word
143
-
*
144
-
* xchg(lock, tail), which heads an address dependency
145
-
*
146
-
* p,*,* -> n,*,* ; prev = xchg(lock, node)
147
-
*/
148
-
static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
149
-
{
150
-
/*
151
-
* We can use relaxed semantics since the caller ensures that the
152
-
* MCS node is properly initialized before updating the tail.
153
-
*/
154
-
return (u32)xchg_relaxed(&lock->tail,
155
-
tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
156
-
}
157
-
158
-
#else /* _Q_PENDING_BITS == 8 */
159
-
160
-
/**
161
-
* clear_pending - clear the pending bit.
162
-
* @lock: Pointer to queued spinlock structure
163
-
*
164
-
* *,1,* -> *,0,*
165
-
*/
166
-
static __always_inline void clear_pending(struct qspinlock *lock)
167
-
{
168
-
atomic_andnot(_Q_PENDING_VAL, &lock->val);
169
-
}
170
-
171
-
/**
172
-
* clear_pending_set_locked - take ownership and clear the pending bit.
173
-
* @lock: Pointer to queued spinlock structure
174
-
*
175
-
* *,1,0 -> *,0,1
176
-
*/
177
-
static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
178
-
{
179
-
atomic_add(-_Q_PENDING_VAL + _Q_LOCKED_VAL, &lock->val);
180
-
}
181
-
182
-
/**
183
-
* xchg_tail - Put in the new queue tail code word & retrieve previous one
184
-
* @lock : Pointer to queued spinlock structure
185
-
* @tail : The new queue tail code word
186
-
* Return: The previous queue tail code word
187
-
*
188
-
* xchg(lock, tail)
189
-
*
190
-
* p,*,* -> n,*,* ; prev = xchg(lock, node)
191
-
*/
192
-
static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
193
-
{
194
-
u32 old, new;
195
-
196
-
old = atomic_read(&lock->val);
197
-
do {
198
-
new = (old & _Q_LOCKED_PENDING_MASK) | tail;
199
-
/*
200
-
* We can use relaxed semantics since the caller ensures that
201
-
* the MCS node is properly initialized before updating the
202
-
* tail.
203
-
*/
204
-
} while (!atomic_try_cmpxchg_relaxed(&lock->val, &old, new));
205
-
206
-
return old;
207
-
}
208
-
#endif /* _Q_PENDING_BITS == 8 */
209
-
210
-
/**
211
-
* queued_fetch_set_pending_acquire - fetch the whole lock value and set pending
212
-
* @lock : Pointer to queued spinlock structure
213
-
* Return: The previous lock value
214
-
*
215
-
* *,*,* -> *,1,*
216
-
*/
217
-
#ifndef queued_fetch_set_pending_acquire
218
-
static __always_inline u32 queued_fetch_set_pending_acquire(struct qspinlock *lock)
219
-
{
220
-
return atomic_fetch_or_acquire(_Q_PENDING_VAL, &lock->val);
221
-
}
222
-
#endif
223
-
224
-
/**
225
-
* set_locked - Set the lock bit and own the lock
226
-
* @lock: Pointer to queued spinlock structure
227
-
*
228
-
* *,*,0 -> *,0,1
229
-
*/
230
-
static __always_inline void set_locked(struct qspinlock *lock)
231
-
{
232
-
WRITE_ONCE(lock->locked, _Q_LOCKED_VAL);
233
-
}
234
-
109
+
static DEFINE_PER_CPU_ALIGNED(struct qnode, qnodes[_Q_MAX_NODES]);
235
110
236
111
/*
237
112
* Generate the native code for queued_spin_unlock_slowpath(); provide NOPs for
···
227
410
* any MCS node. This is not the most elegant solution, but is
228
411
* simple enough.
229
412
*/
230
-
if (unlikely(idx >= MAX_NODES)) {
413
+
if (unlikely(idx >= _Q_MAX_NODES)) {
231
414
lockevent_inc(lock_no_node);
232
415
while (!queued_spin_trylock(lock))
233
416
cpu_relax();
···
282
465
* head of the waitqueue.
283
466
*/
284
467
if (old & _Q_TAIL_MASK) {
285
-
prev = decode_tail(old);
468
+
prev = decode_tail(old, qnodes);
286
469
287
470
/* Link @node into the waitqueue. */
288
471
WRITE_ONCE(prev->next, node);
+201
kernel/locking/qspinlock.h
+201
kernel/locking/qspinlock.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2
+
/*
3
+
* Queued spinlock defines
4
+
*
5
+
* This file contains macro definitions and functions shared between different
6
+
* qspinlock slow path implementations.
7
+
*/
8
+
#ifndef __LINUX_QSPINLOCK_H
9
+
#define __LINUX_QSPINLOCK_H
10
+
11
+
#include <asm-generic/percpu.h>
12
+
#include <linux/percpu-defs.h>
13
+
#include <asm-generic/qspinlock.h>
14
+
#include <asm-generic/mcs_spinlock.h>
15
+
16
+
#define _Q_MAX_NODES 4
17
+
18
+
/*
19
+
* The pending bit spinning loop count.
20
+
* This heuristic is used to limit the number of lockword accesses
21
+
* made by atomic_cond_read_relaxed when waiting for the lock to
22
+
* transition out of the "== _Q_PENDING_VAL" state. We don't spin
23
+
* indefinitely because there's no guarantee that we'll make forward
24
+
* progress.
25
+
*/
26
+
#ifndef _Q_PENDING_LOOPS
27
+
#define _Q_PENDING_LOOPS 1
28
+
#endif
29
+
30
+
/*
31
+
* On 64-bit architectures, the mcs_spinlock structure will be 16 bytes in
32
+
* size and four of them will fit nicely in one 64-byte cacheline. For
33
+
* pvqspinlock, however, we need more space for extra data. To accommodate
34
+
* that, we insert two more long words to pad it up to 32 bytes. IOW, only
35
+
* two of them can fit in a cacheline in this case. That is OK as it is rare
36
+
* to have more than 2 levels of slowpath nesting in actual use. We don't
37
+
* want to penalize pvqspinlocks to optimize for a rare case in native
38
+
* qspinlocks.
39
+
*/
40
+
struct qnode {
41
+
struct mcs_spinlock mcs;
42
+
#ifdef CONFIG_PARAVIRT_SPINLOCKS
43
+
long reserved[2];
44
+
#endif
45
+
};
46
+
47
+
/*
48
+
* We must be able to distinguish between no-tail and the tail at 0:0,
49
+
* therefore increment the cpu number by one.
50
+
*/
51
+
52
+
static inline __pure u32 encode_tail(int cpu, int idx)
53
+
{
54
+
u32 tail;
55
+
56
+
tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
57
+
tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
58
+
59
+
return tail;
60
+
}
61
+
62
+
static inline __pure struct mcs_spinlock *decode_tail(u32 tail,
63
+
struct qnode __percpu *qnodes)
64
+
{
65
+
int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
66
+
int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
67
+
68
+
return per_cpu_ptr(&qnodes[idx].mcs, cpu);
69
+
}
70
+
71
+
static inline __pure
72
+
struct mcs_spinlock *grab_mcs_node(struct mcs_spinlock *base, int idx)
73
+
{
74
+
return &((struct qnode *)base + idx)->mcs;
75
+
}
76
+
77
+
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
78
+
79
+
#if _Q_PENDING_BITS == 8
80
+
/**
81
+
* clear_pending - clear the pending bit.
82
+
* @lock: Pointer to queued spinlock structure
83
+
*
84
+
* *,1,* -> *,0,*
85
+
*/
86
+
static __always_inline void clear_pending(struct qspinlock *lock)
87
+
{
88
+
WRITE_ONCE(lock->pending, 0);
89
+
}
90
+
91
+
/**
92
+
* clear_pending_set_locked - take ownership and clear the pending bit.
93
+
* @lock: Pointer to queued spinlock structure
94
+
*
95
+
* *,1,0 -> *,0,1
96
+
*
97
+
* Lock stealing is not allowed if this function is used.
98
+
*/
99
+
static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
100
+
{
101
+
WRITE_ONCE(lock->locked_pending, _Q_LOCKED_VAL);
102
+
}
103
+
104
+
/*
105
+
* xchg_tail - Put in the new queue tail code word & retrieve previous one
106
+
* @lock : Pointer to queued spinlock structure
107
+
* @tail : The new queue tail code word
108
+
* Return: The previous queue tail code word
109
+
*
110
+
* xchg(lock, tail), which heads an address dependency
111
+
*
112
+
* p,*,* -> n,*,* ; prev = xchg(lock, node)
113
+
*/
114
+
static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
115
+
{
116
+
/*
117
+
* We can use relaxed semantics since the caller ensures that the
118
+
* MCS node is properly initialized before updating the tail.
119
+
*/
120
+
return (u32)xchg_relaxed(&lock->tail,
121
+
tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
122
+
}
123
+
124
+
#else /* _Q_PENDING_BITS == 8 */
125
+
126
+
/**
127
+
* clear_pending - clear the pending bit.
128
+
* @lock: Pointer to queued spinlock structure
129
+
*
130
+
* *,1,* -> *,0,*
131
+
*/
132
+
static __always_inline void clear_pending(struct qspinlock *lock)
133
+
{
134
+
atomic_andnot(_Q_PENDING_VAL, &lock->val);
135
+
}
136
+
137
+
/**
138
+
* clear_pending_set_locked - take ownership and clear the pending bit.
139
+
* @lock: Pointer to queued spinlock structure
140
+
*
141
+
* *,1,0 -> *,0,1
142
+
*/
143
+
static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
144
+
{
145
+
atomic_add(-_Q_PENDING_VAL + _Q_LOCKED_VAL, &lock->val);
146
+
}
147
+
148
+
/**
149
+
* xchg_tail - Put in the new queue tail code word & retrieve previous one
150
+
* @lock : Pointer to queued spinlock structure
151
+
* @tail : The new queue tail code word
152
+
* Return: The previous queue tail code word
153
+
*
154
+
* xchg(lock, tail)
155
+
*
156
+
* p,*,* -> n,*,* ; prev = xchg(lock, node)
157
+
*/
158
+
static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
159
+
{
160
+
u32 old, new;
161
+
162
+
old = atomic_read(&lock->val);
163
+
do {
164
+
new = (old & _Q_LOCKED_PENDING_MASK) | tail;
165
+
/*
166
+
* We can use relaxed semantics since the caller ensures that
167
+
* the MCS node is properly initialized before updating the
168
+
* tail.
169
+
*/
170
+
} while (!atomic_try_cmpxchg_relaxed(&lock->val, &old, new));
171
+
172
+
return old;
173
+
}
174
+
#endif /* _Q_PENDING_BITS == 8 */
175
+
176
+
/**
177
+
* queued_fetch_set_pending_acquire - fetch the whole lock value and set pending
178
+
* @lock : Pointer to queued spinlock structure
179
+
* Return: The previous lock value
180
+
*
181
+
* *,*,* -> *,1,*
182
+
*/
183
+
#ifndef queued_fetch_set_pending_acquire
184
+
static __always_inline u32 queued_fetch_set_pending_acquire(struct qspinlock *lock)
185
+
{
186
+
return atomic_fetch_or_acquire(_Q_PENDING_VAL, &lock->val);
187
+
}
188
+
#endif
189
+
190
+
/**
191
+
* set_locked - Set the lock bit and own the lock
192
+
* @lock: Pointer to queued spinlock structure
193
+
*
194
+
* *,*,0 -> *,0,1
195
+
*/
196
+
static __always_inline void set_locked(struct qspinlock *lock)
197
+
{
198
+
WRITE_ONCE(lock->locked, _Q_LOCKED_VAL);
199
+
}
200
+
201
+
#endif /* __LINUX_QSPINLOCK_H */