Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'locking-core-2022-03-21' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking updates from Ingo Molnar:
"Changes in this cycle were:
Bitops & cpumask:
- Always inline various generic helpers, to improve code generation,
but also for instrumentation, found by noinstr validation.
- Add a x86-specific cpumask_clear_cpu() helper to improve code
generation
Atomics:
- Fix atomic64_{read_acquire,set_release} fallbacks
Lockdep:
- Fix /proc/lockdep output loop iteration for classes
- Fix /proc/lockdep potential access to invalid memory
- Add Mark Rutland as reviewer for atomic primitives
- Minor cleanups
Jump labels:
- Clean up the code a bit
Misc:
- Add __sched annotations to percpu rwsem primitives
- Enable RT_MUTEXES on PREEMPT_RT by default
- Stray v8086_mode() inlining fix, result of noinstr objtool
validation"
* tag 'locking-core-2022-03-21' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
jump_label: Refactor #ifdef of struct static_key
jump_label: Avoid unneeded casts in STATIC_KEY_INIT_{TRUE,FALSE}
locking/lockdep: Iterate lock_classes directly when reading lockdep files
x86/ptrace: Always inline v8086_mode() for instrumentation
cpumask: Add a x86-specific cpumask_clear_cpu() helper
locking: Enable RT_MUTEXES by default on PREEMPT_RT.
locking/local_lock: Make the empty local_lock_*() function a macro.
atomics: Fix atomic64_{read_acquire,set_release} fallbacks
locking: Add missing __sched attributes
cpumask: Always inline helpers which use bit manipulation functions
asm-generic/bitops: Always inline all bit manipulation helpers
locking/lockdep: Avoid potential access of invalid memory in lock_class
lockdep: Use memset_startat() helper in reinit_class()
MAINTAINERS: add myself as reviewer for atomics
+168
-74
+1
MAINTAINERS
+1
MAINTAINERS
+10
arch/x86/include/asm/cpumask.h
+10
arch/x86/include/asm/cpumask.h
···
20
20
{
21
21
return arch_test_bit(cpu, cpumask_bits(cpu_online_mask));
22
22
}
23
+
24
+
static __always_inline void arch_cpumask_clear_cpu(int cpu, struct cpumask *dstp)
25
+
{
26
+
arch_clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
27
+
}
23
28
#else
24
29
static __always_inline bool arch_cpu_online(int cpu)
25
30
{
26
31
return cpu == 0;
32
+
}
33
+
34
+
static __always_inline void arch_cpumask_clear_cpu(int cpu, struct cpumask *dstp)
35
+
{
36
+
return;
27
37
}
28
38
#endif
29
39
+1
-1
arch/x86/include/asm/ptrace.h
+1
-1
arch/x86/include/asm/ptrace.h
+6
-6
include/asm-generic/bitops/instrumented-atomic.h
+6
-6
include/asm-generic/bitops/instrumented-atomic.h
···
23
23
* Note that @nr may be almost arbitrarily large; this function is not
24
24
* restricted to acting on a single-word quantity.
25
25
*/
26
-
static inline void set_bit(long nr, volatile unsigned long *addr)
26
+
static __always_inline void set_bit(long nr, volatile unsigned long *addr)
27
27
{
28
28
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
29
29
arch_set_bit(nr, addr);
···
36
36
*
37
37
* This is a relaxed atomic operation (no implied memory barriers).
38
38
*/
39
-
static inline void clear_bit(long nr, volatile unsigned long *addr)
39
+
static __always_inline void clear_bit(long nr, volatile unsigned long *addr)
40
40
{
41
41
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
42
42
arch_clear_bit(nr, addr);
···
52
52
* Note that @nr may be almost arbitrarily large; this function is not
53
53
* restricted to acting on a single-word quantity.
54
54
*/
55
-
static inline void change_bit(long nr, volatile unsigned long *addr)
55
+
static __always_inline void change_bit(long nr, volatile unsigned long *addr)
56
56
{
57
57
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
58
58
arch_change_bit(nr, addr);
···
65
65
*
66
66
* This is an atomic fully-ordered operation (implied full memory barrier).
67
67
*/
68
-
static inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
68
+
static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
69
69
{
70
70
kcsan_mb();
71
71
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
···
79
79
*
80
80
* This is an atomic fully-ordered operation (implied full memory barrier).
81
81
*/
82
-
static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
82
+
static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
83
83
{
84
84
kcsan_mb();
85
85
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
···
93
93
*
94
94
* This is an atomic fully-ordered operation (implied full memory barrier).
95
95
*/
96
-
static inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
96
+
static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
97
97
{
98
98
kcsan_mb();
99
99
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
+8
-8
include/asm-generic/bitops/instrumented-non-atomic.h
+8
-8
include/asm-generic/bitops/instrumented-non-atomic.h
···
22
22
* region of memory concurrently, the effect may be that only one operation
23
23
* succeeds.
24
24
*/
25
-
static inline void __set_bit(long nr, volatile unsigned long *addr)
25
+
static __always_inline void __set_bit(long nr, volatile unsigned long *addr)
26
26
{
27
27
instrument_write(addr + BIT_WORD(nr), sizeof(long));
28
28
arch___set_bit(nr, addr);
···
37
37
* region of memory concurrently, the effect may be that only one operation
38
38
* succeeds.
39
39
*/
40
-
static inline void __clear_bit(long nr, volatile unsigned long *addr)
40
+
static __always_inline void __clear_bit(long nr, volatile unsigned long *addr)
41
41
{
42
42
instrument_write(addr + BIT_WORD(nr), sizeof(long));
43
43
arch___clear_bit(nr, addr);
···
52
52
* region of memory concurrently, the effect may be that only one operation
53
53
* succeeds.
54
54
*/
55
-
static inline void __change_bit(long nr, volatile unsigned long *addr)
55
+
static __always_inline void __change_bit(long nr, volatile unsigned long *addr)
56
56
{
57
57
instrument_write(addr + BIT_WORD(nr), sizeof(long));
58
58
arch___change_bit(nr, addr);
59
59
}
60
60
61
-
static inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
61
+
static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
62
62
{
63
63
if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
64
64
/*
···
90
90
* This operation is non-atomic. If two instances of this operation race, one
91
91
* can appear to succeed but actually fail.
92
92
*/
93
-
static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
93
+
static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
94
94
{
95
95
__instrument_read_write_bitop(nr, addr);
96
96
return arch___test_and_set_bit(nr, addr);
···
104
104
* This operation is non-atomic. If two instances of this operation race, one
105
105
* can appear to succeed but actually fail.
106
106
*/
107
-
static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
107
+
static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
108
108
{
109
109
__instrument_read_write_bitop(nr, addr);
110
110
return arch___test_and_clear_bit(nr, addr);
···
118
118
* This operation is non-atomic. If two instances of this operation race, one
119
119
* can appear to succeed but actually fail.
120
120
*/
121
-
static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
121
+
static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
122
122
{
123
123
__instrument_read_write_bitop(nr, addr);
124
124
return arch___test_and_change_bit(nr, addr);
···
129
129
* @nr: bit number to test
130
130
* @addr: Address to start counting from
131
131
*/
132
-
static inline bool test_bit(long nr, const volatile unsigned long *addr)
132
+
static __always_inline bool test_bit(long nr, const volatile unsigned long *addr)
133
133
{
134
134
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
135
135
return arch_test_bit(nr, addr);
+33
-5
include/linux/atomic/atomic-arch-fallback.h
+33
-5
include/linux/atomic/atomic-arch-fallback.h
···
151
151
static __always_inline int
152
152
arch_atomic_read_acquire(const atomic_t *v)
153
153
{
154
-
return smp_load_acquire(&(v)->counter);
154
+
int ret;
155
+
156
+
if (__native_word(atomic_t)) {
157
+
ret = smp_load_acquire(&(v)->counter);
158
+
} else {
159
+
ret = arch_atomic_read(v);
160
+
__atomic_acquire_fence();
161
+
}
162
+
163
+
return ret;
155
164
}
156
165
#define arch_atomic_read_acquire arch_atomic_read_acquire
157
166
#endif
···
169
160
static __always_inline void
170
161
arch_atomic_set_release(atomic_t *v, int i)
171
162
{
172
-
smp_store_release(&(v)->counter, i);
163
+
if (__native_word(atomic_t)) {
164
+
smp_store_release(&(v)->counter, i);
165
+
} else {
166
+
__atomic_release_fence();
167
+
arch_atomic_set(v, i);
168
+
}
173
169
}
174
170
#define arch_atomic_set_release arch_atomic_set_release
175
171
#endif
···
1272
1258
static __always_inline s64
1273
1259
arch_atomic64_read_acquire(const atomic64_t *v)
1274
1260
{
1275
-
return smp_load_acquire(&(v)->counter);
1261
+
s64 ret;
1262
+
1263
+
if (__native_word(atomic64_t)) {
1264
+
ret = smp_load_acquire(&(v)->counter);
1265
+
} else {
1266
+
ret = arch_atomic64_read(v);
1267
+
__atomic_acquire_fence();
1268
+
}
1269
+
1270
+
return ret;
1276
1271
}
1277
1272
#define arch_atomic64_read_acquire arch_atomic64_read_acquire
1278
1273
#endif
···
1290
1267
static __always_inline void
1291
1268
arch_atomic64_set_release(atomic64_t *v, s64 i)
1292
1269
{
1293
-
smp_store_release(&(v)->counter, i);
1270
+
if (__native_word(atomic64_t)) {
1271
+
smp_store_release(&(v)->counter, i);
1272
+
} else {
1273
+
__atomic_release_fence();
1274
+
arch_atomic64_set(v, i);
1275
+
}
1294
1276
}
1295
1277
#define arch_atomic64_set_release arch_atomic64_set_release
1296
1278
#endif
···
2386
2358
#endif
2387
2359
2388
2360
#endif /* _LINUX_ATOMIC_FALLBACK_H */
2389
-
// cca554917d7ea73d5e3e7397dd70c484cad9b2c4
2361
+
// 8e2cc06bc0d2c0967d2f8424762bd48555ee40ae
+9
-9
include/linux/cpumask.h
+9
-9
include/linux/cpumask.h
···
102
102
103
103
extern cpumask_t cpus_booted_once_mask;
104
104
105
-
static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
105
+
static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
106
106
{
107
107
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
108
108
WARN_ON_ONCE(cpu >= bits);
···
110
110
}
111
111
112
112
/* verify cpu argument to cpumask_* operators */
113
-
static inline unsigned int cpumask_check(unsigned int cpu)
113
+
static __always_inline unsigned int cpumask_check(unsigned int cpu)
114
114
{
115
115
cpu_max_bits_warn(cpu, nr_cpumask_bits);
116
116
return cpu;
···
341
341
* @cpu: cpu number (< nr_cpu_ids)
342
342
* @dstp: the cpumask pointer
343
343
*/
344
-
static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
344
+
static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
345
345
{
346
346
set_bit(cpumask_check(cpu), cpumask_bits(dstp));
347
347
}
348
348
349
-
static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
349
+
static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
350
350
{
351
351
__set_bit(cpumask_check(cpu), cpumask_bits(dstp));
352
352
}
···
357
357
* @cpu: cpu number (< nr_cpu_ids)
358
358
* @dstp: the cpumask pointer
359
359
*/
360
-
static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
360
+
static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
361
361
{
362
362
clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
363
363
}
364
364
365
-
static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
365
+
static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
366
366
{
367
367
__clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
368
368
}
···
374
374
*
375
375
* Returns 1 if @cpu is set in @cpumask, else returns 0
376
376
*/
377
-
static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
377
+
static __always_inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
378
378
{
379
379
return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
380
380
}
···
388
388
*
389
389
* test_and_set_bit wrapper for cpumasks.
390
390
*/
391
-
static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
391
+
static __always_inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
392
392
{
393
393
return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
394
394
}
···
402
402
*
403
403
* test_and_clear_bit wrapper for cpumasks.
404
404
*/
405
-
static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
405
+
static __always_inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
406
406
{
407
407
return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
408
408
}
+4
-9
include/linux/jump_label.h
+4
-9
include/linux/jump_label.h
···
82
82
"%s(): static key '%pS' used before call to jump_label_init()", \
83
83
__func__, (key))
84
84
85
-
#ifdef CONFIG_JUMP_LABEL
86
-
87
85
struct static_key {
88
86
atomic_t enabled;
87
+
#ifdef CONFIG_JUMP_LABEL
89
88
/*
90
89
* Note:
91
90
* To make anonymous unions work with old compilers, the static
···
103
104
struct jump_entry *entries;
104
105
struct static_key_mod *next;
105
106
};
107
+
#endif /* CONFIG_JUMP_LABEL */
106
108
};
107
109
108
-
#else
109
-
struct static_key {
110
-
atomic_t enabled;
111
-
};
112
-
#endif /* CONFIG_JUMP_LABEL */
113
110
#endif /* __ASSEMBLY__ */
114
111
115
112
#ifdef CONFIG_JUMP_LABEL
···
246
251
*/
247
252
#define STATIC_KEY_INIT_TRUE \
248
253
{ .enabled = { 1 }, \
249
-
{ .entries = (void *)JUMP_TYPE_TRUE } }
254
+
{ .type = JUMP_TYPE_TRUE } }
250
255
#define STATIC_KEY_INIT_FALSE \
251
256
{ .enabled = { 0 }, \
252
-
{ .entries = (void *)JUMP_TYPE_FALSE } }
257
+
{ .type = JUMP_TYPE_FALSE } }
253
258
254
259
#else /* !CONFIG_JUMP_LABEL */
255
260
+3
-3
include/linux/local_lock_internal.h
+3
-3
include/linux/local_lock_internal.h
···
44
44
}
45
45
#else /* CONFIG_DEBUG_LOCK_ALLOC */
46
46
# define LOCAL_LOCK_DEBUG_INIT(lockname)
47
-
static inline void local_lock_acquire(local_lock_t *l) { }
48
-
static inline void local_lock_release(local_lock_t *l) { }
49
-
static inline void local_lock_debug_init(local_lock_t *l) { }
47
+
# define local_lock_acquire(__ll) do { typecheck(local_lock_t *, __ll); } while (0)
48
+
# define local_lock_release(__ll) do { typecheck(local_lock_t *, __ll); } while (0)
49
+
# define local_lock_debug_init(__ll) do { typecheck(local_lock_t *, __ll); } while (0)
50
50
#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
51
51
52
52
#define INIT_LOCAL_LOCK(lockname) { LOCAL_LOCK_DEBUG_INIT(lockname) }
+1
init/Kconfig
+1
init/Kconfig
+25
-18
kernel/locking/lockdep.c
+25
-18
kernel/locking/lockdep.c
···
183
183
static struct hlist_head lock_keys_hash[KEYHASH_SIZE];
184
184
unsigned long nr_lock_classes;
185
185
unsigned long nr_zapped_classes;
186
-
#ifndef CONFIG_DEBUG_LOCKDEP
187
-
static
188
-
#endif
186
+
unsigned long max_lock_class_idx;
189
187
struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
190
-
static DECLARE_BITMAP(lock_classes_in_use, MAX_LOCKDEP_KEYS);
188
+
DECLARE_BITMAP(lock_classes_in_use, MAX_LOCKDEP_KEYS);
191
189
192
190
static inline struct lock_class *hlock_class(struct held_lock *hlock)
193
191
{
···
336
338
* elements. These elements are linked together by the lock_entry member in
337
339
* struct lock_class.
338
340
*/
339
-
LIST_HEAD(all_lock_classes);
341
+
static LIST_HEAD(all_lock_classes);
340
342
static LIST_HEAD(free_lock_classes);
341
343
342
344
/**
···
1250
1252
struct lockdep_subclass_key *key;
1251
1253
struct hlist_head *hash_head;
1252
1254
struct lock_class *class;
1255
+
int idx;
1253
1256
1254
1257
DEBUG_LOCKS_WARN_ON(!irqs_disabled());
1255
1258
···
1316
1317
* of classes.
1317
1318
*/
1318
1319
list_move_tail(&class->lock_entry, &all_lock_classes);
1320
+
idx = class - lock_classes;
1321
+
if (idx > max_lock_class_idx)
1322
+
max_lock_class_idx = idx;
1319
1323
1320
1324
if (verbose(class)) {
1321
1325
graph_unlock();
···
6002
6000
WRITE_ONCE(class->name, NULL);
6003
6001
nr_lock_classes--;
6004
6002
__clear_bit(class - lock_classes, lock_classes_in_use);
6003
+
if (class - lock_classes == max_lock_class_idx)
6004
+
max_lock_class_idx--;
6005
6005
} else {
6006
6006
WARN_ONCE(true, "%s() failed for class %s\n", __func__,
6007
6007
class->name);
···
6015
6011
6016
6012
static void reinit_class(struct lock_class *class)
6017
6013
{
6018
-
void *const p = class;
6019
-
const unsigned int offset = offsetof(struct lock_class, key);
6020
-
6021
6014
WARN_ON_ONCE(!class->lock_entry.next);
6022
6015
WARN_ON_ONCE(!list_empty(&class->locks_after));
6023
6016
WARN_ON_ONCE(!list_empty(&class->locks_before));
6024
-
memset(p + offset, 0, sizeof(*class) - offset);
6017
+
memset_startat(class, 0, key);
6025
6018
WARN_ON_ONCE(!class->lock_entry.next);
6026
6019
WARN_ON_ONCE(!list_empty(&class->locks_after));
6027
6020
WARN_ON_ONCE(!list_empty(&class->locks_before));
···
6291
6290
lockdep_reset_lock_reg(lock);
6292
6291
}
6293
6292
6294
-
/* Unregister a dynamically allocated key. */
6293
+
/*
6294
+
* Unregister a dynamically allocated key.
6295
+
*
6296
+
* Unlike lockdep_register_key(), a search is always done to find a matching
6297
+
* key irrespective of debug_locks to avoid potential invalid access to freed
6298
+
* memory in lock_class entry.
6299
+
*/
6295
6300
void lockdep_unregister_key(struct lock_class_key *key)
6296
6301
{
6297
6302
struct hlist_head *hash_head = keyhashentry(key);
···
6312
6305
return;
6313
6306
6314
6307
raw_local_irq_save(flags);
6315
-
if (!graph_lock())
6316
-
goto out_irq;
6308
+
lockdep_lock();
6317
6309
6318
-
pf = get_pending_free();
6319
6310
hlist_for_each_entry_rcu(k, hash_head, hash_entry) {
6320
6311
if (k == key) {
6321
6312
hlist_del_rcu(&k->hash_entry);
···
6321
6316
break;
6322
6317
}
6323
6318
}
6324
-
WARN_ON_ONCE(!found);
6325
-
__lockdep_free_key_range(pf, key, 1);
6326
-
call_rcu_zapped(pf);
6327
-
graph_unlock();
6328
-
out_irq:
6319
+
WARN_ON_ONCE(!found && debug_locks);
6320
+
if (found) {
6321
+
pf = get_pending_free();
6322
+
__lockdep_free_key_range(pf, key, 1);
6323
+
call_rcu_zapped(pf);
6324
+
}
6325
+
lockdep_unlock();
6329
6326
raw_local_irq_restore(flags);
6330
6327
6331
6328
/* Wait until is_dynamic_key() has finished accessing k->hash_entry. */
+4
-2
kernel/locking/lockdep_internals.h
+4
-2
kernel/locking/lockdep_internals.h
···
121
121
122
122
#define MAX_LOCKDEP_CHAIN_HLOCKS (MAX_LOCKDEP_CHAINS*5)
123
123
124
-
extern struct list_head all_lock_classes;
125
124
extern struct lock_chain lock_chains[];
126
125
127
126
#define LOCK_USAGE_CHARS (2*XXX_LOCK_USAGE_STATES + 1)
···
150
151
151
152
extern unsigned int max_lockdep_depth;
152
153
extern unsigned int max_bfs_queue_depth;
154
+
extern unsigned long max_lock_class_idx;
155
+
156
+
extern struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
157
+
extern unsigned long lock_classes_in_use[];
153
158
154
159
#ifdef CONFIG_PROVE_LOCKING
155
160
extern unsigned long lockdep_count_forward_deps(struct lock_class *);
···
208
205
};
209
206
210
207
DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats);
211
-
extern struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
212
208
213
209
#define __debug_atomic_inc(ptr) \
214
210
this_cpu_inc(lockdep_stats.ptr);
+43
-8
kernel/locking/lockdep_proc.c
+43
-8
kernel/locking/lockdep_proc.c
···
24
24
25
25
#include "lockdep_internals.h"
26
26
27
+
/*
28
+
* Since iteration of lock_classes is done without holding the lockdep lock,
29
+
* it is not safe to iterate all_lock_classes list directly as the iteration
30
+
* may branch off to free_lock_classes or the zapped list. Iteration is done
31
+
* directly on the lock_classes array by checking the lock_classes_in_use
32
+
* bitmap and max_lock_class_idx.
33
+
*/
34
+
#define iterate_lock_classes(idx, class) \
35
+
for (idx = 0, class = lock_classes; idx <= max_lock_class_idx; \
36
+
idx++, class++)
37
+
27
38
static void *l_next(struct seq_file *m, void *v, loff_t *pos)
28
39
{
29
-
return seq_list_next(v, &all_lock_classes, pos);
40
+
struct lock_class *class = v;
41
+
42
+
++class;
43
+
*pos = class - lock_classes;
44
+
return (*pos > max_lock_class_idx) ? NULL : class;
30
45
}
31
46
32
47
static void *l_start(struct seq_file *m, loff_t *pos)
33
48
{
34
-
return seq_list_start_head(&all_lock_classes, *pos);
49
+
unsigned long idx = *pos;
50
+
51
+
if (idx > max_lock_class_idx)
52
+
return NULL;
53
+
return lock_classes + idx;
35
54
}
36
55
37
56
static void l_stop(struct seq_file *m, void *v)
···
76
57
77
58
static int l_show(struct seq_file *m, void *v)
78
59
{
79
-
struct lock_class *class = list_entry(v, struct lock_class, lock_entry);
60
+
struct lock_class *class = v;
80
61
struct lock_list *entry;
81
62
char usage[LOCK_USAGE_CHARS];
63
+
int idx = class - lock_classes;
82
64
83
-
if (v == &all_lock_classes) {
65
+
if (v == lock_classes)
84
66
seq_printf(m, "all lock classes:\n");
67
+
68
+
if (!test_bit(idx, lock_classes_in_use))
85
69
return 0;
86
-
}
87
70
88
71
seq_printf(m, "%p", class->key);
89
72
#ifdef CONFIG_DEBUG_LOCKDEP
···
241
220
242
221
#ifdef CONFIG_PROVE_LOCKING
243
222
struct lock_class *class;
223
+
unsigned long idx;
244
224
245
-
list_for_each_entry(class, &all_lock_classes, lock_entry) {
225
+
iterate_lock_classes(idx, class) {
226
+
if (!test_bit(idx, lock_classes_in_use))
227
+
continue;
246
228
247
229
if (class->usage_mask == 0)
248
230
nr_unused++;
···
278
254
279
255
sum_forward_deps += lockdep_count_forward_deps(class);
280
256
}
257
+
281
258
#ifdef CONFIG_DEBUG_LOCKDEP
282
259
DEBUG_LOCKS_WARN_ON(debug_atomic_read(nr_unused_locks) != nr_unused);
283
260
#endif
···
370
345
seq_printf(m, " max bfs queue depth: %11u\n",
371
346
max_bfs_queue_depth);
372
347
#endif
348
+
seq_printf(m, " max lock class index: %11lu\n",
349
+
max_lock_class_idx);
373
350
lockdep_stats_debug_show(m);
374
351
seq_printf(m, " debug_locks: %11u\n",
375
352
debug_locks);
···
649
622
if (!res) {
650
623
struct lock_stat_data *iter = data->stats;
651
624
struct seq_file *m = file->private_data;
625
+
unsigned long idx;
652
626
653
-
list_for_each_entry(class, &all_lock_classes, lock_entry) {
627
+
iterate_lock_classes(idx, class) {
628
+
if (!test_bit(idx, lock_classes_in_use))
629
+
continue;
654
630
iter->class = class;
655
631
iter->stats = lock_stats(class);
656
632
iter++;
657
633
}
634
+
658
635
data->iter_end = iter;
659
636
660
637
sort(data->stats, data->iter_end - data->stats,
···
676
645
size_t count, loff_t *ppos)
677
646
{
678
647
struct lock_class *class;
648
+
unsigned long idx;
679
649
char c;
680
650
681
651
if (count) {
···
686
654
if (c != '0')
687
655
return count;
688
656
689
-
list_for_each_entry(class, &all_lock_classes, lock_entry)
657
+
iterate_lock_classes(idx, class) {
658
+
if (!test_bit(idx, lock_classes_in_use))
659
+
continue;
690
660
clear_lock_stats(class);
661
+
}
691
662
}
692
663
return count;
693
664
}
+3
-2
kernel/locking/percpu-rwsem.c
+3
-2
kernel/locking/percpu-rwsem.c
···
7
7
#include <linux/rcupdate.h>
8
8
#include <linux/sched.h>
9
9
#include <linux/sched/task.h>
10
+
#include <linux/sched/debug.h>
10
11
#include <linux/errno.h>
11
12
12
13
int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
···
163
162
__set_current_state(TASK_RUNNING);
164
163
}
165
164
166
-
bool __percpu_down_read(struct percpu_rw_semaphore *sem, bool try)
165
+
bool __sched __percpu_down_read(struct percpu_rw_semaphore *sem, bool try)
167
166
{
168
167
if (__percpu_down_read_trylock(sem))
169
168
return true;
···
212
211
return true;
213
212
}
214
213
215
-
void percpu_down_write(struct percpu_rw_semaphore *sem)
214
+
void __sched percpu_down_write(struct percpu_rw_semaphore *sem)
216
215
{
217
216
might_sleep();
218
217
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
+1
-1
kernel/locking/rwsem.c
+1
-1
kernel/locking/rwsem.c
+10
-1
scripts/atomic/fallbacks/read_acquire
+10
-1
scripts/atomic/fallbacks/read_acquire
···
2
2
static __always_inline ${ret}
3
3
arch_${atomic}_read_acquire(const ${atomic}_t *v)
4
4
{
5
-
return smp_load_acquire(&(v)->counter);
5
+
${int} ret;
6
+
7
+
if (__native_word(${atomic}_t)) {
8
+
ret = smp_load_acquire(&(v)->counter);
9
+
} else {
10
+
ret = arch_${atomic}_read(v);
11
+
__atomic_acquire_fence();
12
+
}
13
+
14
+
return ret;
6
15
}
7
16
EOF
+6
-1
scripts/atomic/fallbacks/set_release
+6
-1
scripts/atomic/fallbacks/set_release
···
2
2
static __always_inline void
3
3
arch_${atomic}_set_release(${atomic}_t *v, ${int} i)
4
4
{
5
-
smp_store_release(&(v)->counter, i);
5
+
if (__native_word(${atomic}_t)) {
6
+
smp_store_release(&(v)->counter, i);
7
+
} else {
8
+
__atomic_release_fence();
9
+
arch_${atomic}_set(v, i);
10
+
}
6
11
}
7
12
EOF