Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'locking-core-2022-10-07' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking updates from Ingo Molnar:
- Disable preemption in rwsem_write_trylock()'s attempt to take the
rwsem, to avoid RT tasks hogging the CPU, which managed to preempt
this function after the owner has been cleared but before a new owner
is set. Also add debug checks to enforce this.
- Add __lockfunc to more slow path functions and add __sched to
semaphore functions.
- Mark spinlock APIs noinline when the respective CONFIG_INLINE_SPIN_*
toggles are disabled, to reduce LTO text size.
- Print more debug information when lockdep gets confused in
look_up_lock_class().
- Improve header file abuse checks.
- Misc cleanups
* tag 'locking-core-2022-10-07' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
locking/lockdep: Print more debug information - report name and key when look_up_lock_class() got confused
locking: Add __sched to semaphore functions
locking/rwsem: Disable preemption while trying for rwsem lock
locking: Detect includes rwlock.h outside of spinlock.h
locking: Add __lockfunc to slow path functions
locking/spinlocks: Mark spinlocks noinline when inline spinlocks are disabled
selftests: futex: Fix 'the the' typo in comment
+70
-55
+7
-6
arch/x86/include/asm/qspinlock_paravirt.h
+7
-6
arch/x86/include/asm/qspinlock_paravirt.h
···
12
12
*/
13
13
#ifdef CONFIG_64BIT
14
14
15
-
PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
15
+
__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
16
16
#define __pv_queued_spin_unlock __pv_queued_spin_unlock
17
17
#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"
18
18
#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
···
20
20
/*
21
21
* Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
22
22
* which combines the registers saving trunk and the body of the following
23
-
* C code:
23
+
* C code. Note that it puts the code in the .spinlock.text section which
24
+
* is equivalent to adding __lockfunc in the C code:
24
25
*
25
-
* void __pv_queued_spin_unlock(struct qspinlock *lock)
26
+
* void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
26
27
* {
27
28
* u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
28
29
*
···
37
36
* rsi = lockval (second argument)
38
37
* rdx = internal variable (set to 0)
39
38
*/
40
-
asm (".pushsection .text;"
39
+
asm (".pushsection .spinlock.text;"
41
40
".globl " PV_UNLOCK ";"
42
41
".type " PV_UNLOCK ", @function;"
43
42
".align 4,0x90;"
···
66
65
67
66
#else /* CONFIG_64BIT */
68
67
69
-
extern void __pv_queued_spin_unlock(struct qspinlock *lock);
70
-
PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
68
+
extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
69
+
__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
71
70
72
71
#endif /* CONFIG_64BIT */
73
72
#endif
+1
-1
include/linux/rwlock.h
+1
-1
include/linux/rwlock.h
+2
include/linux/spinlock.h
+2
include/linux/spinlock.h
···
1
1
/* SPDX-License-Identifier: GPL-2.0 */
2
2
#ifndef __LINUX_SPINLOCK_H
3
3
#define __LINUX_SPINLOCK_H
4
+
#define __LINUX_INSIDE_SPINLOCK_H
4
5
5
6
/*
6
7
* include/linux/spinlock.h - generic spinlock/rwlock declarations
···
493
492
494
493
void free_bucket_spinlocks(spinlock_t *locks);
495
494
495
+
#undef __LINUX_INSIDE_SPINLOCK_H
496
496
#endif /* __LINUX_SPINLOCK_H */
+1
-1
include/linux/spinlock_api_smp.h
+1
-1
include/linux/spinlock_api_smp.h
+1
-1
include/linux/spinlock_api_up.h
+1
-1
include/linux/spinlock_api_up.h
+1
-1
include/linux/spinlock_rt.h
+1
-1
include/linux/spinlock_rt.h
+1
-1
include/linux/spinlock_up.h
+1
-1
include/linux/spinlock_up.h
+4
-2
kernel/locking/lockdep.c
+4
-2
kernel/locking/lockdep.c
···
934
934
* Huh! same key, different name? Did someone trample
935
935
* on some memory? We're most confused.
936
936
*/
937
-
WARN_ON_ONCE(class->name != lock->name &&
938
-
lock->key != &__lockdep_no_validate__);
937
+
WARN_ONCE(class->name != lock->name &&
938
+
lock->key != &__lockdep_no_validate__,
939
+
"Looking for class \"%s\" with key %ps, but found a different class \"%s\" with the same key\n",
940
+
lock->name, lock->key, class->name);
939
941
return class;
940
942
}
941
943
}
+2
-2
kernel/locking/qrwlock.c
+2
-2
kernel/locking/qrwlock.c
···
18
18
* queued_read_lock_slowpath - acquire read lock of a queued rwlock
19
19
* @lock: Pointer to queued rwlock structure
20
20
*/
21
-
void queued_read_lock_slowpath(struct qrwlock *lock)
21
+
void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)
22
22
{
23
23
/*
24
24
* Readers come here when they cannot get the lock without waiting
···
63
63
* queued_write_lock_slowpath - acquire write lock of a queued rwlock
64
64
* @lock : Pointer to queued rwlock structure
65
65
*/
66
-
void queued_write_lock_slowpath(struct qrwlock *lock)
66
+
void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)
67
67
{
68
68
int cnts;
69
69
+1
-1
kernel/locking/qspinlock.c
+1
-1
kernel/locking/qspinlock.c
···
313
313
* contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
314
314
* queue : ^--' :
315
315
*/
316
-
void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
316
+
void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
317
317
{
318
318
struct mcs_spinlock *prev, *next, *node;
319
319
u32 old, tail;
+2
-2
kernel/locking/qspinlock_paravirt.h
+2
-2
kernel/locking/qspinlock_paravirt.h
···
489
489
* PV versions of the unlock fastpath and slowpath functions to be used
490
490
* instead of queued_spin_unlock().
491
491
*/
492
-
__visible void
492
+
__visible __lockfunc void
493
493
__pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
494
494
{
495
495
struct pv_node *node;
···
544
544
#include <asm/qspinlock_paravirt.h>
545
545
546
546
#ifndef __pv_queued_spin_unlock
547
-
__visible void __pv_queued_spin_unlock(struct qspinlock *lock)
547
+
__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock)
548
548
{
549
549
u8 locked;
550
550
+12
-2
kernel/locking/rwsem.c
+12
-2
kernel/locking/rwsem.c
···
133
133
* the owner value concurrently without lock. Read from owner, however,
134
134
* may not need READ_ONCE() as long as the pointer value is only used
135
135
* for comparison and isn't being dereferenced.
136
+
*
137
+
* Both rwsem_{set,clear}_owner() functions should be in the same
138
+
* preempt disable section as the atomic op that changes sem->count.
136
139
*/
137
140
static inline void rwsem_set_owner(struct rw_semaphore *sem)
138
141
{
142
+
lockdep_assert_preemption_disabled();
139
143
atomic_long_set(&sem->owner, (long)current);
140
144
}
141
145
142
146
static inline void rwsem_clear_owner(struct rw_semaphore *sem)
143
147
{
148
+
lockdep_assert_preemption_disabled();
144
149
atomic_long_set(&sem->owner, 0);
145
150
}
146
151
···
256
251
static inline bool rwsem_write_trylock(struct rw_semaphore *sem)
257
252
{
258
253
long tmp = RWSEM_UNLOCKED_VALUE;
254
+
bool ret = false;
259
255
256
+
preempt_disable();
260
257
if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) {
261
258
rwsem_set_owner(sem);
262
-
return true;
259
+
ret = true;
263
260
}
264
261
265
-
return false;
262
+
preempt_enable();
263
+
return ret;
266
264
}
267
265
268
266
/*
···
1360
1352
DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) &&
1361
1353
!rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem);
1362
1354
1355
+
preempt_disable();
1363
1356
rwsem_clear_owner(sem);
1364
1357
tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count);
1358
+
preempt_enable();
1365
1359
if (unlikely(tmp & RWSEM_FLAG_WAITERS))
1366
1360
rwsem_wake(sem);
1367
1361
}
+6
-6
kernel/locking/semaphore.c
+6
-6
kernel/locking/semaphore.c
···
51
51
* Use of this function is deprecated, please use down_interruptible() or
52
52
* down_killable() instead.
53
53
*/
54
-
void down(struct semaphore *sem)
54
+
void __sched down(struct semaphore *sem)
55
55
{
56
56
unsigned long flags;
57
57
···
74
74
* If the sleep is interrupted by a signal, this function will return -EINTR.
75
75
* If the semaphore is successfully acquired, this function returns 0.
76
76
*/
77
-
int down_interruptible(struct semaphore *sem)
77
+
int __sched down_interruptible(struct semaphore *sem)
78
78
{
79
79
unsigned long flags;
80
80
int result = 0;
···
101
101
* -EINTR. If the semaphore is successfully acquired, this function returns
102
102
* 0.
103
103
*/
104
-
int down_killable(struct semaphore *sem)
104
+
int __sched down_killable(struct semaphore *sem)
105
105
{
106
106
unsigned long flags;
107
107
int result = 0;
···
131
131
* Unlike mutex_trylock, this function can be used from interrupt context,
132
132
* and the semaphore can be released by any task or interrupt.
133
133
*/
134
-
int down_trylock(struct semaphore *sem)
134
+
int __sched down_trylock(struct semaphore *sem)
135
135
{
136
136
unsigned long flags;
137
137
int count;
···
156
156
* If the semaphore is not released within the specified number of jiffies,
157
157
* this function returns -ETIME. It returns 0 if the semaphore was acquired.
158
158
*/
159
-
int down_timeout(struct semaphore *sem, long timeout)
159
+
int __sched down_timeout(struct semaphore *sem, long timeout)
160
160
{
161
161
unsigned long flags;
162
162
int result = 0;
···
180
180
* Release the semaphore. Unlike mutexes, up() may be called from any
181
181
* context and even by tasks which have never called down().
182
182
*/
183
-
void up(struct semaphore *sem)
183
+
void __sched up(struct semaphore *sem)
184
184
{
185
185
unsigned long flags;
186
186
+28
-28
kernel/locking/spinlock.c
+28
-28
kernel/locking/spinlock.c
···
133
133
#endif
134
134
135
135
#ifndef CONFIG_INLINE_SPIN_TRYLOCK
136
-
int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock)
136
+
noinline int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock)
137
137
{
138
138
return __raw_spin_trylock(lock);
139
139
}
···
141
141
#endif
142
142
143
143
#ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH
144
-
int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock)
144
+
noinline int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock)
145
145
{
146
146
return __raw_spin_trylock_bh(lock);
147
147
}
···
149
149
#endif
150
150
151
151
#ifndef CONFIG_INLINE_SPIN_LOCK
152
-
void __lockfunc _raw_spin_lock(raw_spinlock_t *lock)
152
+
noinline void __lockfunc _raw_spin_lock(raw_spinlock_t *lock)
153
153
{
154
154
__raw_spin_lock(lock);
155
155
}
···
157
157
#endif
158
158
159
159
#ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE
160
-
unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock)
160
+
noinline unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock)
161
161
{
162
162
return __raw_spin_lock_irqsave(lock);
163
163
}
···
165
165
#endif
166
166
167
167
#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ
168
-
void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
168
+
noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
169
169
{
170
170
__raw_spin_lock_irq(lock);
171
171
}
···
173
173
#endif
174
174
175
175
#ifndef CONFIG_INLINE_SPIN_LOCK_BH
176
-
void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock)
176
+
noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock)
177
177
{
178
178
__raw_spin_lock_bh(lock);
179
179
}
···
181
181
#endif
182
182
183
183
#ifdef CONFIG_UNINLINE_SPIN_UNLOCK
184
-
void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock)
184
+
noinline void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock)
185
185
{
186
186
__raw_spin_unlock(lock);
187
187
}
···
189
189
#endif
190
190
191
191
#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE
192
-
void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
192
+
noinline void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
193
193
{
194
194
__raw_spin_unlock_irqrestore(lock, flags);
195
195
}
···
197
197
#endif
198
198
199
199
#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ
200
-
void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock)
200
+
noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock)
201
201
{
202
202
__raw_spin_unlock_irq(lock);
203
203
}
···
205
205
#endif
206
206
207
207
#ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
208
-
void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
208
+
noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
209
209
{
210
210
__raw_spin_unlock_bh(lock);
211
211
}
···
215
215
#ifndef CONFIG_PREEMPT_RT
216
216
217
217
#ifndef CONFIG_INLINE_READ_TRYLOCK
218
-
int __lockfunc _raw_read_trylock(rwlock_t *lock)
218
+
noinline int __lockfunc _raw_read_trylock(rwlock_t *lock)
219
219
{
220
220
return __raw_read_trylock(lock);
221
221
}
···
223
223
#endif
224
224
225
225
#ifndef CONFIG_INLINE_READ_LOCK
226
-
void __lockfunc _raw_read_lock(rwlock_t *lock)
226
+
noinline void __lockfunc _raw_read_lock(rwlock_t *lock)
227
227
{
228
228
__raw_read_lock(lock);
229
229
}
···
231
231
#endif
232
232
233
233
#ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE
234
-
unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock)
234
+
noinline unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock)
235
235
{
236
236
return __raw_read_lock_irqsave(lock);
237
237
}
···
239
239
#endif
240
240
241
241
#ifndef CONFIG_INLINE_READ_LOCK_IRQ
242
-
void __lockfunc _raw_read_lock_irq(rwlock_t *lock)
242
+
noinline void __lockfunc _raw_read_lock_irq(rwlock_t *lock)
243
243
{
244
244
__raw_read_lock_irq(lock);
245
245
}
···
247
247
#endif
248
248
249
249
#ifndef CONFIG_INLINE_READ_LOCK_BH
250
-
void __lockfunc _raw_read_lock_bh(rwlock_t *lock)
250
+
noinline void __lockfunc _raw_read_lock_bh(rwlock_t *lock)
251
251
{
252
252
__raw_read_lock_bh(lock);
253
253
}
···
255
255
#endif
256
256
257
257
#ifndef CONFIG_INLINE_READ_UNLOCK
258
-
void __lockfunc _raw_read_unlock(rwlock_t *lock)
258
+
noinline void __lockfunc _raw_read_unlock(rwlock_t *lock)
259
259
{
260
260
__raw_read_unlock(lock);
261
261
}
···
263
263
#endif
264
264
265
265
#ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE
266
-
void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
266
+
noinline void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
267
267
{
268
268
__raw_read_unlock_irqrestore(lock, flags);
269
269
}
···
271
271
#endif
272
272
273
273
#ifndef CONFIG_INLINE_READ_UNLOCK_IRQ
274
-
void __lockfunc _raw_read_unlock_irq(rwlock_t *lock)
274
+
noinline void __lockfunc _raw_read_unlock_irq(rwlock_t *lock)
275
275
{
276
276
__raw_read_unlock_irq(lock);
277
277
}
···
279
279
#endif
280
280
281
281
#ifndef CONFIG_INLINE_READ_UNLOCK_BH
282
-
void __lockfunc _raw_read_unlock_bh(rwlock_t *lock)
282
+
noinline void __lockfunc _raw_read_unlock_bh(rwlock_t *lock)
283
283
{
284
284
__raw_read_unlock_bh(lock);
285
285
}
···
287
287
#endif
288
288
289
289
#ifndef CONFIG_INLINE_WRITE_TRYLOCK
290
-
int __lockfunc _raw_write_trylock(rwlock_t *lock)
290
+
noinline int __lockfunc _raw_write_trylock(rwlock_t *lock)
291
291
{
292
292
return __raw_write_trylock(lock);
293
293
}
···
295
295
#endif
296
296
297
297
#ifndef CONFIG_INLINE_WRITE_LOCK
298
-
void __lockfunc _raw_write_lock(rwlock_t *lock)
298
+
noinline void __lockfunc _raw_write_lock(rwlock_t *lock)
299
299
{
300
300
__raw_write_lock(lock);
301
301
}
···
313
313
#endif
314
314
315
315
#ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE
316
-
unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock)
316
+
noinline unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock)
317
317
{
318
318
return __raw_write_lock_irqsave(lock);
319
319
}
···
321
321
#endif
322
322
323
323
#ifndef CONFIG_INLINE_WRITE_LOCK_IRQ
324
-
void __lockfunc _raw_write_lock_irq(rwlock_t *lock)
324
+
noinline void __lockfunc _raw_write_lock_irq(rwlock_t *lock)
325
325
{
326
326
__raw_write_lock_irq(lock);
327
327
}
···
329
329
#endif
330
330
331
331
#ifndef CONFIG_INLINE_WRITE_LOCK_BH
332
-
void __lockfunc _raw_write_lock_bh(rwlock_t *lock)
332
+
noinline void __lockfunc _raw_write_lock_bh(rwlock_t *lock)
333
333
{
334
334
__raw_write_lock_bh(lock);
335
335
}
···
337
337
#endif
338
338
339
339
#ifndef CONFIG_INLINE_WRITE_UNLOCK
340
-
void __lockfunc _raw_write_unlock(rwlock_t *lock)
340
+
noinline void __lockfunc _raw_write_unlock(rwlock_t *lock)
341
341
{
342
342
__raw_write_unlock(lock);
343
343
}
···
345
345
#endif
346
346
347
347
#ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE
348
-
void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
348
+
noinline void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
349
349
{
350
350
__raw_write_unlock_irqrestore(lock, flags);
351
351
}
···
353
353
#endif
354
354
355
355
#ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ
356
-
void __lockfunc _raw_write_unlock_irq(rwlock_t *lock)
356
+
noinline void __lockfunc _raw_write_unlock_irq(rwlock_t *lock)
357
357
{
358
358
__raw_write_unlock_irq(lock);
359
359
}
···
361
361
#endif
362
362
363
363
#ifndef CONFIG_INLINE_WRITE_UNLOCK_BH
364
-
void __lockfunc _raw_write_unlock_bh(rwlock_t *lock)
364
+
noinline void __lockfunc _raw_write_unlock_bh(rwlock_t *lock)
365
365
{
366
366
__raw_write_unlock_bh(lock);
367
367
}
+1
-1
tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c
+1
-1
tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c
···
184
184
/*
185
185
* If res is non-zero, we either requeued the waiter or hit an
186
186
* error, break out and handle it. If it is zero, then the
187
-
* signal may have hit before the the waiter was blocked on f1.
187
+
* signal may have hit before the waiter was blocked on f1.
188
188
* Try again.
189
189
*/
190
190
if (res > 0) {