Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
locking/local_lock: Support Clang's context analysis
Add support for Clang's context analysis for local_lock_t and
local_trylock_t.
Signed-off-by: Marco Elver <elver@google.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20251219154418.3592607-20-elver@google.com
authored by
Marco Elver
and committed by
Peter Zijlstra
d3febf16
8c9c8566
+159
-34
+1
-1
Documentation/dev-tools/context-analysis.rst
+1
-1
Documentation/dev-tools/context-analysis.rst
···
80
80
81
81
Currently the following synchronization primitives are supported:
82
82
`raw_spinlock_t`, `spinlock_t`, `rwlock_t`, `mutex`, `seqlock_t`,
83
-
`bit_spinlock`, RCU, SRCU (`srcu_struct`), `rw_semaphore`.
83
+
`bit_spinlock`, RCU, SRCU (`srcu_struct`), `rw_semaphore`, `local_lock_t`.
84
84
85
85
For context locks with an initialization function (e.g., `spin_lock_init()`),
86
86
calling this function before initializing any guarded members or globals
+28
-19
include/linux/local_lock.h
+28
-19
include/linux/local_lock.h
···
14
14
* local_lock - Acquire a per CPU local lock
15
15
* @lock: The lock variable
16
16
*/
17
-
#define local_lock(lock) __local_lock(this_cpu_ptr(lock))
17
+
#define local_lock(lock) __local_lock(__this_cpu_local_lock(lock))
18
18
19
19
/**
20
20
* local_lock_irq - Acquire a per CPU local lock and disable interrupts
21
21
* @lock: The lock variable
22
22
*/
23
-
#define local_lock_irq(lock) __local_lock_irq(this_cpu_ptr(lock))
23
+
#define local_lock_irq(lock) __local_lock_irq(__this_cpu_local_lock(lock))
24
24
25
25
/**
26
26
* local_lock_irqsave - Acquire a per CPU local lock, save and disable
···
29
29
* @flags: Storage for interrupt flags
30
30
*/
31
31
#define local_lock_irqsave(lock, flags) \
32
-
__local_lock_irqsave(this_cpu_ptr(lock), flags)
32
+
__local_lock_irqsave(__this_cpu_local_lock(lock), flags)
33
33
34
34
/**
35
35
* local_unlock - Release a per CPU local lock
36
36
* @lock: The lock variable
37
37
*/
38
-
#define local_unlock(lock) __local_unlock(this_cpu_ptr(lock))
38
+
#define local_unlock(lock) __local_unlock(__this_cpu_local_lock(lock))
39
39
40
40
/**
41
41
* local_unlock_irq - Release a per CPU local lock and enable interrupts
42
42
* @lock: The lock variable
43
43
*/
44
-
#define local_unlock_irq(lock) __local_unlock_irq(this_cpu_ptr(lock))
44
+
#define local_unlock_irq(lock) __local_unlock_irq(__this_cpu_local_lock(lock))
45
45
46
46
/**
47
47
* local_unlock_irqrestore - Release a per CPU local lock and restore
···
50
50
* @flags: Interrupt flags to restore
51
51
*/
52
52
#define local_unlock_irqrestore(lock, flags) \
53
-
__local_unlock_irqrestore(this_cpu_ptr(lock), flags)
53
+
__local_unlock_irqrestore(__this_cpu_local_lock(lock), flags)
54
54
55
55
/**
56
56
* local_trylock_init - Runtime initialize a lock instance
···
66
66
* locking constrains it will _always_ fail to acquire the lock in NMI or
67
67
* HARDIRQ context on PREEMPT_RT.
68
68
*/
69
-
#define local_trylock(lock) __local_trylock(this_cpu_ptr(lock))
69
+
#define local_trylock(lock) __local_trylock(__this_cpu_local_lock(lock))
70
70
71
71
#define local_lock_is_locked(lock) __local_lock_is_locked(lock)
72
72
···
81
81
* HARDIRQ context on PREEMPT_RT.
82
82
*/
83
83
#define local_trylock_irqsave(lock, flags) \
84
-
__local_trylock_irqsave(this_cpu_ptr(lock), flags)
84
+
__local_trylock_irqsave(__this_cpu_local_lock(lock), flags)
85
85
86
-
DEFINE_GUARD(local_lock, local_lock_t __percpu*,
87
-
local_lock(_T),
88
-
local_unlock(_T))
89
-
DEFINE_GUARD(local_lock_irq, local_lock_t __percpu*,
90
-
local_lock_irq(_T),
91
-
local_unlock_irq(_T))
86
+
DEFINE_LOCK_GUARD_1(local_lock, local_lock_t __percpu,
87
+
local_lock(_T->lock),
88
+
local_unlock(_T->lock))
89
+
DEFINE_LOCK_GUARD_1(local_lock_irq, local_lock_t __percpu,
90
+
local_lock_irq(_T->lock),
91
+
local_unlock_irq(_T->lock))
92
92
DEFINE_LOCK_GUARD_1(local_lock_irqsave, local_lock_t __percpu,
93
93
local_lock_irqsave(_T->lock, _T->flags),
94
94
local_unlock_irqrestore(_T->lock, _T->flags),
95
95
unsigned long flags)
96
96
97
97
#define local_lock_nested_bh(_lock) \
98
-
__local_lock_nested_bh(this_cpu_ptr(_lock))
98
+
__local_lock_nested_bh(__this_cpu_local_lock(_lock))
99
99
100
100
#define local_unlock_nested_bh(_lock) \
101
-
__local_unlock_nested_bh(this_cpu_ptr(_lock))
101
+
__local_unlock_nested_bh(__this_cpu_local_lock(_lock))
102
102
103
-
DEFINE_GUARD(local_lock_nested_bh, local_lock_t __percpu*,
104
-
local_lock_nested_bh(_T),
105
-
local_unlock_nested_bh(_T))
103
+
DEFINE_LOCK_GUARD_1(local_lock_nested_bh, local_lock_t __percpu,
104
+
local_lock_nested_bh(_T->lock),
105
+
local_unlock_nested_bh(_T->lock))
106
+
107
+
DECLARE_LOCK_GUARD_1_ATTRS(local_lock, __acquires(_T), __releases(*(local_lock_t __percpu **)_T))
108
+
#define class_local_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock, _T)
109
+
DECLARE_LOCK_GUARD_1_ATTRS(local_lock_irq, __acquires(_T), __releases(*(local_lock_t __percpu **)_T))
110
+
#define class_local_lock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock_irq, _T)
111
+
DECLARE_LOCK_GUARD_1_ATTRS(local_lock_irqsave, __acquires(_T), __releases(*(local_lock_t __percpu **)_T))
112
+
#define class_local_lock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock_irqsave, _T)
113
+
DECLARE_LOCK_GUARD_1_ATTRS(local_lock_nested_bh, __acquires(_T), __releases(*(local_lock_t __percpu **)_T))
114
+
#define class_local_lock_nested_bh_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock_nested_bh, _T)
106
115
107
116
#endif
+57
-14
include/linux/local_lock_internal.h
+57
-14
include/linux/local_lock_internal.h
···
10
10
11
11
#ifndef CONFIG_PREEMPT_RT
12
12
13
-
typedef struct {
13
+
context_lock_struct(local_lock) {
14
14
#ifdef CONFIG_DEBUG_LOCK_ALLOC
15
15
struct lockdep_map dep_map;
16
16
struct task_struct *owner;
17
17
#endif
18
-
} local_lock_t;
18
+
};
19
+
typedef struct local_lock local_lock_t;
19
20
20
21
/* local_trylock() and local_trylock_irqsave() only work with local_trylock_t */
21
-
typedef struct {
22
+
context_lock_struct(local_trylock) {
22
23
#ifdef CONFIG_DEBUG_LOCK_ALLOC
23
24
struct lockdep_map dep_map;
24
25
struct task_struct *owner;
25
26
#endif
26
27
u8 acquired;
27
-
} local_trylock_t;
28
+
};
29
+
typedef struct local_trylock local_trylock_t;
28
30
29
31
#ifdef CONFIG_DEBUG_LOCK_ALLOC
30
32
# define LOCAL_LOCK_DEBUG_INIT(lockname) \
···
86
84
0, LD_WAIT_CONFIG, LD_WAIT_INV, \
87
85
LD_LOCK_PERCPU); \
88
86
local_lock_debug_init(lock); \
87
+
__assume_ctx_lock(lock); \
89
88
} while (0)
90
89
91
-
#define __local_trylock_init(lock) __local_lock_init((local_lock_t *)lock)
90
+
#define __local_trylock_init(lock) \
91
+
do { \
92
+
__local_lock_init((local_lock_t *)lock); \
93
+
__assume_ctx_lock(lock); \
94
+
} while (0)
92
95
93
96
#define __spinlock_nested_bh_init(lock) \
94
97
do { \
···
104
97
0, LD_WAIT_CONFIG, LD_WAIT_INV, \
105
98
LD_LOCK_NORMAL); \
106
99
local_lock_debug_init(lock); \
100
+
__assume_ctx_lock(lock); \
107
101
} while (0)
108
102
109
103
#define __local_lock_acquire(lock) \
···
127
119
do { \
128
120
preempt_disable(); \
129
121
__local_lock_acquire(lock); \
122
+
__acquire(lock); \
130
123
} while (0)
131
124
132
125
#define __local_lock_irq(lock) \
133
126
do { \
134
127
local_irq_disable(); \
135
128
__local_lock_acquire(lock); \
129
+
__acquire(lock); \
136
130
} while (0)
137
131
138
132
#define __local_lock_irqsave(lock, flags) \
139
133
do { \
140
134
local_irq_save(flags); \
141
135
__local_lock_acquire(lock); \
136
+
__acquire(lock); \
142
137
} while (0)
143
138
144
139
#define __local_trylock(lock) \
145
-
({ \
140
+
__try_acquire_ctx_lock(lock, ({ \
146
141
local_trylock_t *__tl; \
147
142
\
148
143
preempt_disable(); \
···
159
148
(local_lock_t *)__tl); \
160
149
} \
161
150
!!__tl; \
162
-
})
151
+
}))
163
152
164
153
#define __local_trylock_irqsave(lock, flags) \
165
-
({ \
154
+
__try_acquire_ctx_lock(lock, ({ \
166
155
local_trylock_t *__tl; \
167
156
\
168
157
local_irq_save(flags); \
···
176
165
(local_lock_t *)__tl); \
177
166
} \
178
167
!!__tl; \
179
-
})
168
+
}))
180
169
181
170
/* preemption or migration must be disabled before calling __local_lock_is_locked */
182
171
#define __local_lock_is_locked(lock) READ_ONCE(this_cpu_ptr(lock)->acquired)
···
199
188
200
189
#define __local_unlock(lock) \
201
190
do { \
191
+
__release(lock); \
202
192
__local_lock_release(lock); \
203
193
preempt_enable(); \
204
194
} while (0)
205
195
206
196
#define __local_unlock_irq(lock) \
207
197
do { \
198
+
__release(lock); \
208
199
__local_lock_release(lock); \
209
200
local_irq_enable(); \
210
201
} while (0)
211
202
212
203
#define __local_unlock_irqrestore(lock, flags) \
213
204
do { \
205
+
__release(lock); \
214
206
__local_lock_release(lock); \
215
207
local_irq_restore(flags); \
216
208
} while (0)
···
222
208
do { \
223
209
lockdep_assert_in_softirq(); \
224
210
local_lock_acquire((lock)); \
211
+
__acquire(lock); \
225
212
} while (0)
226
213
227
214
#define __local_unlock_nested_bh(lock) \
228
-
local_lock_release((lock))
215
+
do { \
216
+
__release(lock); \
217
+
local_lock_release((lock)); \
218
+
} while (0)
229
219
230
220
#else /* !CONFIG_PREEMPT_RT */
221
+
222
+
#include <linux/spinlock.h>
231
223
232
224
/*
233
225
* On PREEMPT_RT local_lock maps to a per CPU spinlock, which protects the
···
289
269
} while (0)
290
270
291
271
#define __local_trylock(lock) \
292
-
({ \
272
+
__try_acquire_ctx_lock(lock, context_unsafe(({ \
293
273
int __locked; \
294
274
\
295
275
if (in_nmi() | in_hardirq()) { \
···
301
281
migrate_enable(); \
302
282
} \
303
283
__locked; \
304
-
})
284
+
})))
305
285
306
286
#define __local_trylock_irqsave(lock, flags) \
307
-
({ \
287
+
__try_acquire_ctx_lock(lock, ({ \
308
288
typecheck(unsigned long, flags); \
309
289
flags = 0; \
310
290
__local_trylock(lock); \
311
-
})
291
+
}))
312
292
313
293
/* migration must be disabled before calling __local_lock_is_locked */
314
294
#define __local_lock_is_locked(__lock) \
315
295
(rt_mutex_owner(&this_cpu_ptr(__lock)->lock) == current)
316
296
317
297
#endif /* CONFIG_PREEMPT_RT */
298
+
299
+
#if defined(WARN_CONTEXT_ANALYSIS)
300
+
/*
301
+
* Because the compiler only knows about the base per-CPU variable, use this
302
+
* helper function to make the compiler think we lock/unlock the @base variable,
303
+
* and hide the fact we actually pass the per-CPU instance to lock/unlock
304
+
* functions.
305
+
*/
306
+
static __always_inline local_lock_t *__this_cpu_local_lock(local_lock_t __percpu *base)
307
+
__returns_ctx_lock(base) __attribute__((overloadable))
308
+
{
309
+
return this_cpu_ptr(base);
310
+
}
311
+
#ifndef CONFIG_PREEMPT_RT
312
+
static __always_inline local_trylock_t *__this_cpu_local_lock(local_trylock_t __percpu *base)
313
+
__returns_ctx_lock(base) __attribute__((overloadable))
314
+
{
315
+
return this_cpu_ptr(base);
316
+
}
317
+
#endif /* CONFIG_PREEMPT_RT */
318
+
#else /* WARN_CONTEXT_ANALYSIS */
319
+
#define __this_cpu_local_lock(base) this_cpu_ptr(base)
320
+
#endif /* WARN_CONTEXT_ANALYSIS */
+73
lib/test_context-analysis.c
+73
lib/test_context-analysis.c
···
6
6
7
7
#include <linux/bit_spinlock.h>
8
8
#include <linux/build_bug.h>
9
+
#include <linux/local_lock.h>
9
10
#include <linux/mutex.h>
11
+
#include <linux/percpu.h>
10
12
#include <linux/rcupdate.h>
11
13
#include <linux/rwsem.h>
12
14
#include <linux/seqlock.h>
···
459
457
{ guard(srcu)(&d->srcu); (void)srcu_dereference(d->data, &d->srcu); }
460
458
{ guard(srcu_fast)(&d->srcu); (void)srcu_dereference(d->data, &d->srcu); }
461
459
{ guard(srcu_fast_notrace)(&d->srcu); (void)srcu_dereference(d->data, &d->srcu); }
460
+
}
461
+
462
+
struct test_local_lock_data {
463
+
local_lock_t lock;
464
+
int counter __guarded_by(&lock);
465
+
};
466
+
467
+
static DEFINE_PER_CPU(struct test_local_lock_data, test_local_lock_data) = {
468
+
.lock = INIT_LOCAL_LOCK(lock),
469
+
};
470
+
471
+
static void __used test_local_lock_init(struct test_local_lock_data *d)
472
+
{
473
+
local_lock_init(&d->lock);
474
+
d->counter = 0;
475
+
}
476
+
477
+
static void __used test_local_lock(void)
478
+
{
479
+
unsigned long flags;
480
+
481
+
local_lock(&test_local_lock_data.lock);
482
+
this_cpu_add(test_local_lock_data.counter, 1);
483
+
local_unlock(&test_local_lock_data.lock);
484
+
485
+
local_lock_irq(&test_local_lock_data.lock);
486
+
this_cpu_add(test_local_lock_data.counter, 1);
487
+
local_unlock_irq(&test_local_lock_data.lock);
488
+
489
+
local_lock_irqsave(&test_local_lock_data.lock, flags);
490
+
this_cpu_add(test_local_lock_data.counter, 1);
491
+
local_unlock_irqrestore(&test_local_lock_data.lock, flags);
492
+
493
+
local_lock_nested_bh(&test_local_lock_data.lock);
494
+
this_cpu_add(test_local_lock_data.counter, 1);
495
+
local_unlock_nested_bh(&test_local_lock_data.lock);
496
+
}
497
+
498
+
static void __used test_local_lock_guard(void)
499
+
{
500
+
{ guard(local_lock)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); }
501
+
{ guard(local_lock_irq)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); }
502
+
{ guard(local_lock_irqsave)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); }
503
+
{ guard(local_lock_nested_bh)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); }
504
+
}
505
+
506
+
struct test_local_trylock_data {
507
+
local_trylock_t lock;
508
+
int counter __guarded_by(&lock);
509
+
};
510
+
511
+
static DEFINE_PER_CPU(struct test_local_trylock_data, test_local_trylock_data) = {
512
+
.lock = INIT_LOCAL_TRYLOCK(lock),
513
+
};
514
+
515
+
static void __used test_local_trylock_init(struct test_local_trylock_data *d)
516
+
{
517
+
local_trylock_init(&d->lock);
518
+
d->counter = 0;
519
+
}
520
+
521
+
static void __used test_local_trylock(void)
522
+
{
523
+
local_lock(&test_local_trylock_data.lock);
524
+
this_cpu_add(test_local_trylock_data.counter, 1);
525
+
local_unlock(&test_local_trylock_data.lock);
526
+
527
+
if (local_trylock(&test_local_trylock_data.lock)) {
528
+
this_cpu_add(test_local_trylock_data.counter, 1);
529
+
local_unlock(&test_local_trylock_data.lock);
530
+
}
462
531
}