Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'locking-core-2023-05-05' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking updates from Ingo Molnar:
- Introduce local{,64}_try_cmpxchg() - a slightly more optimal
primitive, which will be used in perf events ring-buffer code
- Simplify/modify rwsems on PREEMPT_RT, to address writer starvation
- Misc cleanups/fixes
* tag 'locking-core-2023-05-05' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
locking/atomic: Correct (cmp)xchg() instrumentation
locking/x86: Define arch_try_cmpxchg_local()
locking/arch: Wire up local_try_cmpxchg()
locking/generic: Wire up local{,64}_try_cmpxchg()
locking/atomic: Add generic try_cmpxchg{,64}_local() support
locking/rwbase: Mitigate indefinite writer starvation
locking/arch: Rename all internal __xchg() names to __arch_xchg()
+209
-111
+5
-5
arch/alpha/include/asm/cmpxchg.h
+5
-5
arch/alpha/include/asm/cmpxchg.h
···
6
6
* Atomic exchange routines.
7
7
*/
8
8
9
-
#define ____xchg(type, args...) __xchg ## type ## _local(args)
9
+
#define ____xchg(type, args...) __arch_xchg ## type ## _local(args)
10
10
#define ____cmpxchg(type, args...) __cmpxchg ## type ## _local(args)
11
11
#include <asm/xchg.h>
12
12
13
13
#define xchg_local(ptr, x) \
14
14
({ \
15
15
__typeof__(*(ptr)) _x_ = (x); \
16
-
(__typeof__(*(ptr))) __xchg_local((ptr), (unsigned long)_x_, \
17
-
sizeof(*(ptr))); \
16
+
(__typeof__(*(ptr))) __arch_xchg_local((ptr), (unsigned long)_x_,\
17
+
sizeof(*(ptr))); \
18
18
})
19
19
20
20
#define arch_cmpxchg_local(ptr, o, n) \
···
34
34
35
35
#undef ____xchg
36
36
#undef ____cmpxchg
37
-
#define ____xchg(type, args...) __xchg ##type(args)
37
+
#define ____xchg(type, args...) __arch_xchg ##type(args)
38
38
#define ____cmpxchg(type, args...) __cmpxchg ##type(args)
39
39
#include <asm/xchg.h>
40
40
···
48
48
__typeof__(*(ptr)) _x_ = (x); \
49
49
smp_mb(); \
50
50
__ret = (__typeof__(*(ptr))) \
51
-
__xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
51
+
__arch_xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
52
52
smp_mb(); \
53
53
__ret; \
54
54
})
+10
-2
arch/alpha/include/asm/local.h
+10
-2
arch/alpha/include/asm/local.h
···
52
52
return result;
53
53
}
54
54
55
-
#define local_cmpxchg(l, o, n) \
56
-
(cmpxchg_local(&((l)->a.counter), (o), (n)))
55
+
static __inline__ long local_cmpxchg(local_t *l, long old, long new)
56
+
{
57
+
return cmpxchg_local(&l->a.counter, old, new);
58
+
}
59
+
60
+
static __inline__ bool local_try_cmpxchg(local_t *l, long *old, long new)
61
+
{
62
+
return try_cmpxchg_local(&l->a.counter, (s64 *)old, new);
63
+
}
64
+
57
65
#define local_xchg(l, n) (xchg_local(&((l)->a.counter), (n)))
58
66
59
67
/**
+2
-2
arch/arc/include/asm/cmpxchg.h
+2
-2
arch/arc/include/asm/cmpxchg.h
···
85
85
*/
86
86
#ifdef CONFIG_ARC_HAS_LLSC
87
87
88
-
#define __xchg(ptr, val) \
88
+
#define __arch_xchg(ptr, val) \
89
89
({ \
90
90
__asm__ __volatile__( \
91
91
" ex %0, [%1] \n" /* set new value */ \
···
102
102
\
103
103
switch(sizeof(*(_p_))) { \
104
104
case 4: \
105
-
_val_ = __xchg(_p_, _val_); \
105
+
_val_ = __arch_xchg(_p_, _val_); \
106
106
break; \
107
107
default: \
108
108
BUILD_BUG(); \
+4
-3
arch/arm/include/asm/cmpxchg.h
+4
-3
arch/arm/include/asm/cmpxchg.h
···
25
25
#define swp_is_buggy
26
26
#endif
27
27
28
-
static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
28
+
static inline unsigned long
29
+
__arch_xchg(unsigned long x, volatile void *ptr, int size)
29
30
{
30
31
extern void __bad_xchg(volatile void *, int);
31
32
unsigned long ret;
···
116
115
}
117
116
118
117
#define arch_xchg_relaxed(ptr, x) ({ \
119
-
(__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), \
120
-
sizeof(*(ptr))); \
118
+
(__typeof__(*(ptr)))__arch_xchg((unsigned long)(x), (ptr), \
119
+
sizeof(*(ptr))); \
121
120
})
122
121
123
122
#include <asm-generic/cmpxchg-local.h>
+3
-4
arch/arm64/include/asm/cmpxchg.h
+3
-4
arch/arm64/include/asm/cmpxchg.h
···
62
62
#undef __XCHG_CASE
63
63
64
64
#define __XCHG_GEN(sfx) \
65
-
static __always_inline unsigned long __xchg##sfx(unsigned long x, \
66
-
volatile void *ptr, \
67
-
int size) \
65
+
static __always_inline unsigned long \
66
+
__arch_xchg##sfx(unsigned long x, volatile void *ptr, int size) \
68
67
{ \
69
68
switch (size) { \
70
69
case 1: \
···
92
93
({ \
93
94
__typeof__(*(ptr)) __ret; \
94
95
__ret = (__typeof__(*(ptr))) \
95
-
__xchg##sfx((unsigned long)(x), (ptr), sizeof(*(ptr))); \
96
+
__arch_xchg##sfx((unsigned long)(x), (ptr), sizeof(*(ptr))); \
96
97
__ret; \
97
98
})
98
99
+5
-5
arch/hexagon/include/asm/cmpxchg.h
+5
-5
arch/hexagon/include/asm/cmpxchg.h
···
9
9
#define _ASM_CMPXCHG_H
10
10
11
11
/*
12
-
* __xchg - atomically exchange a register and a memory location
12
+
* __arch_xchg - atomically exchange a register and a memory location
13
13
* @x: value to swap
14
14
* @ptr: pointer to memory
15
15
* @size: size of the value
···
19
19
* Note: there was an errata for V2 about .new's and memw_locked.
20
20
*
21
21
*/
22
-
static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
23
-
int size)
22
+
static inline unsigned long
23
+
__arch_xchg(unsigned long x, volatile void *ptr, int size)
24
24
{
25
25
unsigned long retval;
26
26
···
42
42
* Atomically swap the contents of a register with memory. Should be atomic
43
43
* between multiple CPU's and within interrupts on the same CPU.
44
44
*/
45
-
#define arch_xchg(ptr, v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v), (ptr), \
46
-
sizeof(*(ptr))))
45
+
#define arch_xchg(ptr, v) ((__typeof__(*(ptr)))__arch_xchg((unsigned long)(v), (ptr), \
46
+
sizeof(*(ptr))))
47
47
48
48
/*
49
49
* see rt-mutex-design.txt; cmpxchg supposedly checks if *ptr == A and swaps.
+1
-1
arch/ia64/include/asm/cmpxchg.h
+1
-1
arch/ia64/include/asm/cmpxchg.h
···
5
5
#include <uapi/asm/cmpxchg.h>
6
6
7
7
#define arch_xchg(ptr, x) \
8
-
({(__typeof__(*(ptr))) __xchg((unsigned long) (x), (ptr), sizeof(*(ptr)));})
8
+
({(__typeof__(*(ptr))) __arch_xchg((unsigned long) (x), (ptr), sizeof(*(ptr)));})
9
9
10
10
#define arch_cmpxchg(ptr, o, n) cmpxchg_acq((ptr), (o), (n))
11
11
#define arch_cmpxchg64(ptr, o, n) cmpxchg_acq((ptr), (o), (n))
+2
-2
arch/ia64/include/uapi/asm/cmpxchg.h
+2
-2
arch/ia64/include/uapi/asm/cmpxchg.h
···
23
23
*/
24
24
extern void ia64_xchg_called_with_bad_pointer(void);
25
25
26
-
#define __xchg(x, ptr, size) \
26
+
#define __arch_xchg(x, ptr, size) \
27
27
({ \
28
28
unsigned long __xchg_result; \
29
29
\
···
51
51
52
52
#ifndef __KERNEL__
53
53
#define xchg(ptr, x) \
54
-
({(__typeof__(*(ptr))) __xchg((unsigned long) (x), (ptr), sizeof(*(ptr)));})
54
+
({(__typeof__(*(ptr))) __arch_xchg((unsigned long) (x), (ptr), sizeof(*(ptr)));})
55
55
#endif
56
56
57
57
/*
+2
-2
arch/loongarch/include/asm/cmpxchg.h
+2
-2
arch/loongarch/include/asm/cmpxchg.h
···
62
62
}
63
63
64
64
static __always_inline unsigned long
65
-
__xchg(volatile void *ptr, unsigned long x, int size)
65
+
__arch_xchg(volatile void *ptr, unsigned long x, int size)
66
66
{
67
67
switch (size) {
68
68
case 1:
···
87
87
__typeof__(*(ptr)) __res; \
88
88
\
89
89
__res = (__typeof__(*(ptr))) \
90
-
__xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \
90
+
__arch_xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \
91
91
\
92
92
__res; \
93
93
})
+11
-2
arch/loongarch/include/asm/local.h
+11
-2
arch/loongarch/include/asm/local.h
···
56
56
return result;
57
57
}
58
58
59
-
#define local_cmpxchg(l, o, n) \
60
-
((long)cmpxchg_local(&((l)->a.counter), (o), (n)))
59
+
static inline long local_cmpxchg(local_t *l, long old, long new)
60
+
{
61
+
return cmpxchg_local(&l->a.counter, old, new);
62
+
}
63
+
64
+
static inline bool local_try_cmpxchg(local_t *l, long *old, long new)
65
+
{
66
+
typeof(l->a.counter) *__old = (typeof(l->a.counter) *) old;
67
+
return try_cmpxchg_local(&l->a.counter, __old, new);
68
+
}
69
+
61
70
#define local_xchg(l, n) (atomic_long_xchg((&(l)->a), (n)))
62
71
63
72
/**
+3
-3
arch/m68k/include/asm/cmpxchg.h
+3
-3
arch/m68k/include/asm/cmpxchg.h
···
9
9
extern unsigned long __invalid_xchg_size(unsigned long, volatile void *, int);
10
10
11
11
#ifndef CONFIG_RMW_INSNS
12
-
static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size)
12
+
static inline unsigned long __arch_xchg(unsigned long x, volatile void * ptr, int size)
13
13
{
14
14
unsigned long flags, tmp;
15
15
···
40
40
return x;
41
41
}
42
42
#else
43
-
static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size)
43
+
static inline unsigned long __arch_xchg(unsigned long x, volatile void * ptr, int size)
44
44
{
45
45
switch (size) {
46
46
case 1:
···
75
75
}
76
76
#endif
77
77
78
-
#define arch_xchg(ptr,x) ({(__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));})
78
+
#define arch_xchg(ptr,x) ({(__typeof__(*(ptr)))__arch_xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));})
79
79
80
80
#include <asm-generic/cmpxchg-local.h>
81
81
+2
-2
arch/mips/include/asm/cmpxchg.h
+2
-2
arch/mips/include/asm/cmpxchg.h
···
68
68
unsigned int size);
69
69
70
70
static __always_inline
71
-
unsigned long __xchg(volatile void *ptr, unsigned long x, int size)
71
+
unsigned long __arch_xchg(volatile void *ptr, unsigned long x, int size)
72
72
{
73
73
switch (size) {
74
74
case 1:
···
102
102
smp_mb__before_llsc(); \
103
103
\
104
104
__res = (__typeof__(*(ptr))) \
105
-
__xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \
105
+
__arch_xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \
106
106
\
107
107
smp_llsc_mb(); \
108
108
\
+11
-2
arch/mips/include/asm/local.h
+11
-2
arch/mips/include/asm/local.h
···
94
94
return result;
95
95
}
96
96
97
-
#define local_cmpxchg(l, o, n) \
98
-
((long)cmpxchg_local(&((l)->a.counter), (o), (n)))
97
+
static __inline__ long local_cmpxchg(local_t *l, long old, long new)
98
+
{
99
+
return cmpxchg_local(&l->a.counter, old, new);
100
+
}
101
+
102
+
static __inline__ bool local_try_cmpxchg(local_t *l, long *old, long new)
103
+
{
104
+
typeof(l->a.counter) *__old = (typeof(l->a.counter) *) old;
105
+
return try_cmpxchg_local(&l->a.counter, __old, new);
106
+
}
107
+
99
108
#define local_xchg(l, n) (atomic_long_xchg((&(l)->a), (n)))
100
109
101
110
/**
+5
-5
arch/openrisc/include/asm/cmpxchg.h
+5
-5
arch/openrisc/include/asm/cmpxchg.h
···
147
147
extern unsigned long __xchg_called_with_bad_pointer(void)
148
148
__compiletime_error("Bad argument size for xchg");
149
149
150
-
static inline unsigned long __xchg(volatile void *ptr, unsigned long with,
151
-
int size)
150
+
static inline unsigned long
151
+
__arch_xchg(volatile void *ptr, unsigned long with, int size)
152
152
{
153
153
switch (size) {
154
154
case 1:
···
163
163
164
164
#define arch_xchg(ptr, with) \
165
165
({ \
166
-
(__typeof__(*(ptr))) __xchg((ptr), \
167
-
(unsigned long)(with), \
168
-
sizeof(*(ptr))); \
166
+
(__typeof__(*(ptr))) __arch_xchg((ptr), \
167
+
(unsigned long)(with), \
168
+
sizeof(*(ptr))); \
169
169
})
170
170
171
171
#endif /* __ASM_OPENRISC_CMPXCHG_H */
+2
-2
arch/parisc/include/asm/cmpxchg.h
+2
-2
arch/parisc/include/asm/cmpxchg.h
···
22
22
23
23
/* optimizer better get rid of switch since size is a constant */
24
24
static inline unsigned long
25
-
__xchg(unsigned long x, volatile void *ptr, int size)
25
+
__arch_xchg(unsigned long x, volatile void *ptr, int size)
26
26
{
27
27
switch (size) {
28
28
#ifdef CONFIG_64BIT
···
49
49
__typeof__(*(ptr)) __ret; \
50
50
__typeof__(*(ptr)) _x_ = (x); \
51
51
__ret = (__typeof__(*(ptr))) \
52
-
__xchg((unsigned long)_x_, (ptr), sizeof(*(ptr))); \
52
+
__arch_xchg((unsigned long)_x_, (ptr), sizeof(*(ptr))); \
53
53
__ret; \
54
54
})
55
55
+2
-2
arch/powerpc/include/asm/cmpxchg.h
+2
-2
arch/powerpc/include/asm/cmpxchg.h
···
229
229
return __xchg_u64_local(ptr, x);
230
230
#endif
231
231
}
232
-
BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg");
232
+
BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
233
233
return x;
234
234
}
235
235
···
248
248
return __xchg_u64_relaxed(ptr, x);
249
249
#endif
250
250
}
251
-
BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
251
+
BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_relaxed");
252
252
return x;
253
253
}
254
254
#define arch_xchg_local(ptr,x) \
+11
arch/powerpc/include/asm/local.h
+11
arch/powerpc/include/asm/local.h
···
90
90
return t;
91
91
}
92
92
93
+
static __inline__ bool local_try_cmpxchg(local_t *l, long *po, long n)
94
+
{
95
+
long o = *po, r;
96
+
97
+
r = local_cmpxchg(l, o, n);
98
+
if (unlikely(r != o))
99
+
*po = r;
100
+
101
+
return likely(r == o);
102
+
}
103
+
93
104
static __inline__ long local_xchg(local_t *l, long n)
94
105
{
95
106
long t;
+1
-1
arch/riscv/include/asm/atomic.h
+1
-1
arch/riscv/include/asm/atomic.h
···
261
261
static __always_inline \
262
262
c_t arch_atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \
263
263
{ \
264
-
return __xchg(&(v->counter), n, size); \
264
+
return __arch_xchg(&(v->counter), n, size); \
265
265
} \
266
266
static __always_inline \
267
267
c_t arch_atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \
+2
-2
arch/riscv/include/asm/cmpxchg.h
+2
-2
arch/riscv/include/asm/cmpxchg.h
···
114
114
_x_, sizeof(*(ptr))); \
115
115
})
116
116
117
-
#define __xchg(ptr, new, size) \
117
+
#define __arch_xchg(ptr, new, size) \
118
118
({ \
119
119
__typeof__(ptr) __ptr = (ptr); \
120
120
__typeof__(new) __new = (new); \
···
143
143
#define arch_xchg(ptr, x) \
144
144
({ \
145
145
__typeof__(*(ptr)) _x_ = (x); \
146
-
(__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \
146
+
(__typeof__(*(ptr))) __arch_xchg((ptr), _x_, sizeof(*(ptr))); \
147
147
})
148
148
149
149
#define xchg32(ptr, x) \
+4
-4
arch/s390/include/asm/cmpxchg.h
+4
-4
arch/s390/include/asm/cmpxchg.h
···
14
14
15
15
void __xchg_called_with_bad_pointer(void);
16
16
17
-
static __always_inline unsigned long __xchg(unsigned long x,
18
-
unsigned long address, int size)
17
+
static __always_inline unsigned long
18
+
__arch_xchg(unsigned long x, unsigned long address, int size)
19
19
{
20
20
unsigned long old;
21
21
int shift;
···
77
77
__typeof__(*(ptr)) __ret; \
78
78
\
79
79
__ret = (__typeof__(*(ptr))) \
80
-
__xchg((unsigned long)(x), (unsigned long)(ptr), \
81
-
sizeof(*(ptr))); \
80
+
__arch_xchg((unsigned long)(x), (unsigned long)(ptr), \
81
+
sizeof(*(ptr))); \
82
82
__ret; \
83
83
})
84
84
+2
-2
arch/sh/include/asm/cmpxchg.h
+2
-2
arch/sh/include/asm/cmpxchg.h
···
22
22
23
23
extern void __xchg_called_with_bad_pointer(void);
24
24
25
-
#define __xchg(ptr, x, size) \
25
+
#define __arch_xchg(ptr, x, size) \
26
26
({ \
27
27
unsigned long __xchg__res; \
28
28
volatile void *__xchg_ptr = (ptr); \
···
46
46
})
47
47
48
48
#define arch_xchg(ptr,x) \
49
-
((__typeof__(*(ptr)))__xchg((ptr),(unsigned long)(x), sizeof(*(ptr))))
49
+
((__typeof__(*(ptr)))__arch_xchg((ptr),(unsigned long)(x), sizeof(*(ptr))))
50
50
51
51
/* This function doesn't exist, so you'll get a linker error
52
52
* if something tries to do an invalid cmpxchg(). */
+2
-2
arch/sparc/include/asm/cmpxchg_32.h
+2
-2
arch/sparc/include/asm/cmpxchg_32.h
···
15
15
unsigned long __xchg_u32(volatile u32 *m, u32 new);
16
16
void __xchg_called_with_bad_pointer(void);
17
17
18
-
static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, int size)
18
+
static inline unsigned long __arch_xchg(unsigned long x, __volatile__ void * ptr, int size)
19
19
{
20
20
switch (size) {
21
21
case 4:
···
25
25
return x;
26
26
}
27
27
28
-
#define arch_xchg(ptr,x) ({(__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));})
28
+
#define arch_xchg(ptr,x) ({(__typeof__(*(ptr)))__arch_xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));})
29
29
30
30
/* Emulate cmpxchg() the same way we emulate atomics,
31
31
* by hashing the object address and indexing into an array
+3
-3
arch/sparc/include/asm/cmpxchg_64.h
+3
-3
arch/sparc/include/asm/cmpxchg_64.h
···
55
55
#define arch_xchg(ptr,x) \
56
56
({ __typeof__(*(ptr)) __ret; \
57
57
__ret = (__typeof__(*(ptr))) \
58
-
__xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
58
+
__arch_xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
59
59
__ret; \
60
60
})
61
61
···
87
87
return (load32 & mask) >> bit_shift;
88
88
}
89
89
90
-
static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr,
91
-
int size)
90
+
static inline unsigned long
91
+
__arch_xchg(unsigned long x, __volatile__ void * ptr, int size)
92
92
{
93
93
switch (size) {
94
94
case 2:
+6
arch/x86/include/asm/cmpxchg.h
+6
arch/x86/include/asm/cmpxchg.h
···
221
221
#define __try_cmpxchg(ptr, pold, new, size) \
222
222
__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)
223
223
224
+
#define __try_cmpxchg_local(ptr, pold, new, size) \
225
+
__raw_try_cmpxchg((ptr), (pold), (new), (size), "")
226
+
224
227
#define arch_try_cmpxchg(ptr, pold, new) \
225
228
__try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))
229
+
230
+
#define arch_try_cmpxchg_local(ptr, pold, new) \
231
+
__try_cmpxchg_local((ptr), (pold), (new), sizeof(*(ptr)))
226
232
227
233
/*
228
234
* xadd() adds "inc" to "*ptr" and atomically returns the previous
+11
-2
arch/x86/include/asm/local.h
+11
-2
arch/x86/include/asm/local.h
···
120
120
#define local_inc_return(l) (local_add_return(1, l))
121
121
#define local_dec_return(l) (local_sub_return(1, l))
122
122
123
-
#define local_cmpxchg(l, o, n) \
124
-
(cmpxchg_local(&((l)->a.counter), (o), (n)))
123
+
static inline long local_cmpxchg(local_t *l, long old, long new)
124
+
{
125
+
return cmpxchg_local(&l->a.counter, old, new);
126
+
}
127
+
128
+
static inline bool local_try_cmpxchg(local_t *l, long *old, long new)
129
+
{
130
+
typeof(l->a.counter) *__old = (typeof(l->a.counter) *) old;
131
+
return try_cmpxchg_local(&l->a.counter, __old, new);
132
+
}
133
+
125
134
/* Always has a lock prefix */
126
135
#define local_xchg(l, n) (xchg(&((l)->a.counter), (n)))
127
136
+2
-2
arch/xtensa/include/asm/cmpxchg.h
+2
-2
arch/xtensa/include/asm/cmpxchg.h
···
170
170
}
171
171
172
172
#define arch_xchg(ptr,x) \
173
-
((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
173
+
((__typeof__(*(ptr)))__arch_xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
174
174
175
175
static inline u32 xchg_small(volatile void *ptr, u32 x, int size)
176
176
{
···
203
203
extern void __xchg_called_with_bad_pointer(void);
204
204
205
205
static __inline__ unsigned long
206
-
__xchg(unsigned long x, volatile void * ptr, int size)
206
+
__arch_xchg(unsigned long x, volatile void * ptr, int size)
207
207
{
208
208
switch (size) {
209
209
case 1:
+1
include/asm-generic/local.h
+1
include/asm-generic/local.h
···
42
42
#define local_inc_return(l) atomic_long_inc_return(&(l)->a)
43
43
44
44
#define local_cmpxchg(l, o, n) atomic_long_cmpxchg((&(l)->a), (o), (n))
45
+
#define local_try_cmpxchg(l, po, n) atomic_long_try_cmpxchg((&(l)->a), (po), (n))
45
46
#define local_xchg(l, n) atomic_long_xchg((&(l)->a), (n))
46
47
#define local_add_unless(l, _a, u) atomic_long_add_unless((&(l)->a), (_a), (u))
47
48
#define local_inc_not_zero(l) atomic_long_inc_not_zero(&(l)->a)
+11
-1
include/asm-generic/local64.h
+11
-1
include/asm-generic/local64.h
···
42
42
#define local64_sub_return(i, l) local_sub_return((i), (&(l)->a))
43
43
#define local64_inc_return(l) local_inc_return(&(l)->a)
44
44
45
-
#define local64_cmpxchg(l, o, n) local_cmpxchg((&(l)->a), (o), (n))
45
+
static inline s64 local64_cmpxchg(local64_t *l, s64 old, s64 new)
46
+
{
47
+
return local_cmpxchg(&l->a, old, new);
48
+
}
49
+
50
+
static inline bool local64_try_cmpxchg(local64_t *l, s64 *old, s64 new)
51
+
{
52
+
return local_try_cmpxchg(&l->a, (long *)old, new);
53
+
}
54
+
46
55
#define local64_xchg(l, n) local_xchg((&(l)->a), (n))
47
56
#define local64_add_unless(l, _a, u) local_add_unless((&(l)->a), (_a), (u))
48
57
#define local64_inc_not_zero(l) local_inc_not_zero(&(l)->a)
···
90
81
#define local64_inc_return(l) atomic64_inc_return(&(l)->a)
91
82
92
83
#define local64_cmpxchg(l, o, n) atomic64_cmpxchg((&(l)->a), (o), (n))
84
+
#define local64_try_cmpxchg(l, po, n) atomic64_try_cmpxchg((&(l)->a), (po), (n))
93
85
#define local64_xchg(l, n) atomic64_xchg((&(l)->a), (n))
94
86
#define local64_add_unless(l, _a, u) atomic64_add_unless((&(l)->a), (_a), (u))
95
87
#define local64_inc_not_zero(l) atomic64_inc_not_zero(&(l)->a)
+23
-1
include/linux/atomic/atomic-arch-fallback.h
+23
-1
include/linux/atomic/atomic-arch-fallback.h
···
217
217
218
218
#endif /* arch_try_cmpxchg64_relaxed */
219
219
220
+
#ifndef arch_try_cmpxchg_local
221
+
#define arch_try_cmpxchg_local(_ptr, _oldp, _new) \
222
+
({ \
223
+
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
224
+
___r = arch_cmpxchg_local((_ptr), ___o, (_new)); \
225
+
if (unlikely(___r != ___o)) \
226
+
*___op = ___r; \
227
+
likely(___r == ___o); \
228
+
})
229
+
#endif /* arch_try_cmpxchg_local */
230
+
231
+
#ifndef arch_try_cmpxchg64_local
232
+
#define arch_try_cmpxchg64_local(_ptr, _oldp, _new) \
233
+
({ \
234
+
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
235
+
___r = arch_cmpxchg64_local((_ptr), ___o, (_new)); \
236
+
if (unlikely(___r != ___o)) \
237
+
*___op = ___r; \
238
+
likely(___r == ___o); \
239
+
})
240
+
#endif /* arch_try_cmpxchg64_local */
241
+
220
242
#ifndef arch_atomic_read_acquire
221
243
static __always_inline int
222
244
arch_atomic_read_acquire(const atomic_t *v)
···
2668
2646
#endif
2669
2647
2670
2648
#endif /* _LINUX_ATOMIC_FALLBACK_H */
2671
-
// 00071fffa021cec66f6290d706d69c91df87bade
2649
+
// ad2e2b4d168dbc60a73922616047a9bfa446af36
+52
-34
include/linux/atomic/atomic-instrumented.h
+52
-34
include/linux/atomic/atomic-instrumented.h
···
1948
1948
({ \
1949
1949
typeof(ptr) __ai_ptr = (ptr); \
1950
1950
kcsan_mb(); \
1951
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1951
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1952
1952
arch_xchg(__ai_ptr, __VA_ARGS__); \
1953
1953
})
1954
1954
1955
1955
#define xchg_acquire(ptr, ...) \
1956
1956
({ \
1957
1957
typeof(ptr) __ai_ptr = (ptr); \
1958
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1958
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1959
1959
arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1960
1960
})
1961
1961
···
1963
1963
({ \
1964
1964
typeof(ptr) __ai_ptr = (ptr); \
1965
1965
kcsan_release(); \
1966
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1966
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1967
1967
arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1968
1968
})
1969
1969
1970
1970
#define xchg_relaxed(ptr, ...) \
1971
1971
({ \
1972
1972
typeof(ptr) __ai_ptr = (ptr); \
1973
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1973
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1974
1974
arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1975
1975
})
1976
1976
···
1978
1978
({ \
1979
1979
typeof(ptr) __ai_ptr = (ptr); \
1980
1980
kcsan_mb(); \
1981
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1981
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1982
1982
arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1983
1983
})
1984
1984
1985
1985
#define cmpxchg_acquire(ptr, ...) \
1986
1986
({ \
1987
1987
typeof(ptr) __ai_ptr = (ptr); \
1988
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1988
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1989
1989
arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1990
1990
})
1991
1991
···
1993
1993
({ \
1994
1994
typeof(ptr) __ai_ptr = (ptr); \
1995
1995
kcsan_release(); \
1996
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1996
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1997
1997
arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1998
1998
})
1999
1999
2000
2000
#define cmpxchg_relaxed(ptr, ...) \
2001
2001
({ \
2002
2002
typeof(ptr) __ai_ptr = (ptr); \
2003
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2003
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2004
2004
arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
2005
2005
})
2006
2006
···
2008
2008
({ \
2009
2009
typeof(ptr) __ai_ptr = (ptr); \
2010
2010
kcsan_mb(); \
2011
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2011
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2012
2012
arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
2013
2013
})
2014
2014
2015
2015
#define cmpxchg64_acquire(ptr, ...) \
2016
2016
({ \
2017
2017
typeof(ptr) __ai_ptr = (ptr); \
2018
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2018
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2019
2019
arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
2020
2020
})
2021
2021
···
2023
2023
({ \
2024
2024
typeof(ptr) __ai_ptr = (ptr); \
2025
2025
kcsan_release(); \
2026
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2026
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2027
2027
arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
2028
2028
})
2029
2029
2030
2030
#define cmpxchg64_relaxed(ptr, ...) \
2031
2031
({ \
2032
2032
typeof(ptr) __ai_ptr = (ptr); \
2033
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2033
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034
2034
arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
2035
2035
})
2036
2036
···
2039
2039
typeof(ptr) __ai_ptr = (ptr); \
2040
2040
typeof(oldp) __ai_oldp = (oldp); \
2041
2041
kcsan_mb(); \
2042
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043
-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2042
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043
+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044
2044
arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045
2045
})
2046
2046
···
2048
2048
({ \
2049
2049
typeof(ptr) __ai_ptr = (ptr); \
2050
2050
typeof(oldp) __ai_oldp = (oldp); \
2051
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2052
-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2051
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2052
+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2053
2053
arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2054
2054
})
2055
2055
···
2058
2058
typeof(ptr) __ai_ptr = (ptr); \
2059
2059
typeof(oldp) __ai_oldp = (oldp); \
2060
2060
kcsan_release(); \
2061
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2062
-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2061
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2062
+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2063
2063
arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2064
2064
})
2065
2065
···
2067
2067
({ \
2068
2068
typeof(ptr) __ai_ptr = (ptr); \
2069
2069
typeof(oldp) __ai_oldp = (oldp); \
2070
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2071
-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2070
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2071
+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2072
2072
arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2073
2073
})
2074
2074
···
2077
2077
typeof(ptr) __ai_ptr = (ptr); \
2078
2078
typeof(oldp) __ai_oldp = (oldp); \
2079
2079
kcsan_mb(); \
2080
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2081
-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2080
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2081
+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2082
2082
arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2083
2083
})
2084
2084
···
2086
2086
({ \
2087
2087
typeof(ptr) __ai_ptr = (ptr); \
2088
2088
typeof(oldp) __ai_oldp = (oldp); \
2089
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2090
-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2089
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2090
+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2091
2091
arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2092
2092
})
2093
2093
···
2096
2096
typeof(ptr) __ai_ptr = (ptr); \
2097
2097
typeof(oldp) __ai_oldp = (oldp); \
2098
2098
kcsan_release(); \
2099
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2100
-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2099
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2100
+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2101
2101
arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2102
2102
})
2103
2103
···
2105
2105
({ \
2106
2106
typeof(ptr) __ai_ptr = (ptr); \
2107
2107
typeof(oldp) __ai_oldp = (oldp); \
2108
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2109
-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2108
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2109
+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2110
2110
arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2111
2111
})
2112
2112
2113
2113
#define cmpxchg_local(ptr, ...) \
2114
2114
({ \
2115
2115
typeof(ptr) __ai_ptr = (ptr); \
2116
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2116
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2117
2117
arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2118
2118
})
2119
2119
2120
2120
#define cmpxchg64_local(ptr, ...) \
2121
2121
({ \
2122
2122
typeof(ptr) __ai_ptr = (ptr); \
2123
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2123
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2124
2124
arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2125
2125
})
2126
2126
···
2128
2128
({ \
2129
2129
typeof(ptr) __ai_ptr = (ptr); \
2130
2130
kcsan_mb(); \
2131
-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2131
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2132
2132
arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2133
+
})
2134
+
2135
+
#define try_cmpxchg_local(ptr, oldp, ...) \
2136
+
({ \
2137
+
typeof(ptr) __ai_ptr = (ptr); \
2138
+
typeof(oldp) __ai_oldp = (oldp); \
2139
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2140
+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2141
+
arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2142
+
})
2143
+
2144
+
#define try_cmpxchg64_local(ptr, oldp, ...) \
2145
+
({ \
2146
+
typeof(ptr) __ai_ptr = (ptr); \
2147
+
typeof(oldp) __ai_oldp = (oldp); \
2148
+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2149
+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2150
+
arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2133
2151
})
2134
2152
2135
2153
#define cmpxchg_double(ptr, ...) \
2136
2154
({ \
2137
2155
typeof(ptr) __ai_ptr = (ptr); \
2138
2156
kcsan_mb(); \
2139
-
instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2157
+
instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2140
2158
arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2141
2159
})
2142
2160
···
2162
2144
#define cmpxchg_double_local(ptr, ...) \
2163
2145
({ \
2164
2146
typeof(ptr) __ai_ptr = (ptr); \
2165
-
instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2147
+
instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2166
2148
arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2167
2149
})
2168
2150
2169
2151
#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2170
-
// 1b485de9cbaa4900de59e14ee2084357eaeb1c3a
2152
+
// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e
-9
kernel/locking/rwbase_rt.c
-9
kernel/locking/rwbase_rt.c
···
72
72
int ret;
73
73
74
74
raw_spin_lock_irq(&rtm->wait_lock);
75
-
/*
76
-
* Allow readers, as long as the writer has not completely
77
-
* acquired the semaphore for write.
78
-
*/
79
-
if (atomic_read(&rwb->readers) != WRITER_BIAS) {
80
-
atomic_inc(&rwb->readers);
81
-
raw_spin_unlock_irq(&rtm->wait_lock);
82
-
return 0;
83
-
}
84
75
85
76
/*
86
77
* Call into the slow lock path with the rtmutex->wait_lock
+4
scripts/atomic/gen-atomic-fallback.sh
+4
scripts/atomic/gen-atomic-fallback.sh
···
225
225
gen_try_cmpxchg_fallbacks "${cmpxchg}"
226
226
done
227
227
228
+
for cmpxchg in "cmpxchg_local" "cmpxchg64_local"; do
229
+
gen_try_cmpxchg_fallback "${cmpxchg}" ""
230
+
done
231
+
228
232
grep '^[a-z]' "$1" | while read name meta args; do
229
233
gen_proto "${meta}" "${name}" "atomic" "int" ${args}
230
234
done
+4
-4
scripts/atomic/gen-atomic-instrumented.sh
+4
-4
scripts/atomic/gen-atomic-instrumented.sh
···
104
104
EOF
105
105
[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
106
106
cat <<EOF
107
-
instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
108
-
instrument_atomic_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\
107
+
instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
108
+
instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\
109
109
arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\
110
110
})
111
111
EOF
···
119
119
EOF
120
120
[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
121
121
cat <<EOF
122
-
instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
122
+
instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
123
123
arch_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\
124
124
})
125
125
EOF
···
173
173
done
174
174
done
175
175
176
-
for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg"; do
176
+
for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" ; do
177
177
gen_xchg "${xchg}" "" ""
178
178
printf "\n"
179
179
done