Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

locking/atomic: scripts: restructure fallback ifdeffery

Currently the various ordering variants of an atomic operation are
defined in groups of full/acquire/release/relaxed ordering variants with
some shared ifdeffery and several potential definitions of each ordering
variant in different branches of the shared ifdeffery.

As an ordering variant can have several potential definitions down
different branches of the shared ifdeffery, it can be painful for a
human to find a relevant definition, and we don't have a good location
to place anything common to all definitions of an ordering variant (e.g.
kerneldoc).

Historically the grouping of full/acquire/release/relaxed ordering
variants was necessary as we filled in the missing atomics in the same
namespace as the architecture used. It would be easy to accidentally
define one ordering fallback in terms of another ordering fallback with
redundant barriers, and avoiding that would otherwise require a lot of
baroque ifdeffery.

With recent changes we no longer need to fill in the missing atomics in
the arch_atomic*_<op>() namespace, and only need to fill in the
raw_atomic*_<op>() namespace. Due to this, there's no risk of a
namespace collision, and we can define each raw_atomic*_<op> ordering
variant with its own ifdeffery checking for the arch_atomic*_<op>
ordering variants.

Restructure the fallbacks in this way, with each ordering variant having
its own ifdeffery of the form:

| #if defined(arch_atomic_fetch_andnot_acquire)
| #define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
| #elif defined(arch_atomic_fetch_andnot_relaxed)
| static __always_inline int
| raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
| {
| int ret = arch_atomic_fetch_andnot_relaxed(i, v);
| __atomic_acquire_fence();
| return ret;
| }
| #elif defined(arch_atomic_fetch_andnot)
| #define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
| #else
| static __always_inline int
| raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
| {
| return raw_atomic_fetch_and_acquire(~i, v);
| }
| #endif

Note that where there's no relevant arch_atomic*_<op>() ordering
variant, we'll define the operation in terms of a distinct
raw_atomic*_<otherop>(), as this itself might have been filled in with a
fallback.

As we now generate the raw_atomic*_<op>() implementations directly, we
no longer need the trivial wrappers, so they are removed.

This makes the ifdeffery easier to follow, and will allow for further
improvements in subsequent patches.

There should be no functional change as a result of this patch.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Kees Cook <keescook@chromium.org>
Link: https://lore.kernel.org/r/20230605070124.3741859-21-mark.rutland@arm.com

authored by

Mark Rutland and committed by
Peter Zijlstra
9257959a 1815da17

+2136 -3121
-1
include/linux/atomic.h
··· 78 78 }) 79 79 80 80 #include <linux/atomic/atomic-arch-fallback.h> 81 - #include <linux/atomic/atomic-raw.h> 82 81 #include <linux/atomic/atomic-long.h> 83 82 #include <linux/atomic/atomic-instrumented.h> 84 83
+1934 -1772
include/linux/atomic/atomic-arch-fallback.h
··· 8 8 9 9 #include <linux/compiler.h> 10 10 11 - #ifndef arch_xchg_relaxed 12 - #define arch_xchg_acquire arch_xchg 13 - #define arch_xchg_release arch_xchg 14 - #define arch_xchg_relaxed arch_xchg 15 - #else /* arch_xchg_relaxed */ 16 - 17 - #ifndef arch_xchg_acquire 18 - #define arch_xchg_acquire(...) \ 19 - __atomic_op_acquire(arch_xchg, __VA_ARGS__) 20 - #endif 21 - 22 - #ifndef arch_xchg_release 23 - #define arch_xchg_release(...) \ 24 - __atomic_op_release(arch_xchg, __VA_ARGS__) 25 - #endif 26 - 27 - #ifndef arch_xchg 28 - #define arch_xchg(...) \ 11 + #if defined(arch_xchg) 12 + #define raw_xchg arch_xchg 13 + #elif defined(arch_xchg_relaxed) 14 + #define raw_xchg(...) \ 29 15 __atomic_op_fence(arch_xchg, __VA_ARGS__) 16 + #else 17 + extern void raw_xchg_not_implemented(void); 18 + #define raw_xchg(...) raw_xchg_not_implemented() 30 19 #endif 31 20 32 - #endif /* arch_xchg_relaxed */ 33 - 34 - #ifndef arch_cmpxchg_relaxed 35 - #define arch_cmpxchg_acquire arch_cmpxchg 36 - #define arch_cmpxchg_release arch_cmpxchg 37 - #define arch_cmpxchg_relaxed arch_cmpxchg 38 - #else /* arch_cmpxchg_relaxed */ 39 - 40 - #ifndef arch_cmpxchg_acquire 41 - #define arch_cmpxchg_acquire(...) \ 42 - __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__) 21 + #if defined(arch_xchg_acquire) 22 + #define raw_xchg_acquire arch_xchg_acquire 23 + #elif defined(arch_xchg_relaxed) 24 + #define raw_xchg_acquire(...) \ 25 + __atomic_op_acquire(arch_xchg, __VA_ARGS__) 26 + #elif defined(arch_xchg) 27 + #define raw_xchg_acquire arch_xchg 28 + #else 29 + extern void raw_xchg_acquire_not_implemented(void); 30 + #define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented() 43 31 #endif 44 32 45 - #ifndef arch_cmpxchg_release 46 - #define arch_cmpxchg_release(...) \ 47 - __atomic_op_release(arch_cmpxchg, __VA_ARGS__) 33 + #if defined(arch_xchg_release) 34 + #define raw_xchg_release arch_xchg_release 35 + #elif defined(arch_xchg_relaxed) 36 + #define raw_xchg_release(...) \ 37 + __atomic_op_release(arch_xchg, __VA_ARGS__) 38 + #elif defined(arch_xchg) 39 + #define raw_xchg_release arch_xchg 40 + #else 41 + extern void raw_xchg_release_not_implemented(void); 42 + #define raw_xchg_release(...) raw_xchg_release_not_implemented() 48 43 #endif 49 44 50 - #ifndef arch_cmpxchg 51 - #define arch_cmpxchg(...) \ 45 + #if defined(arch_xchg_relaxed) 46 + #define raw_xchg_relaxed arch_xchg_relaxed 47 + #elif defined(arch_xchg) 48 + #define raw_xchg_relaxed arch_xchg 49 + #else 50 + extern void raw_xchg_relaxed_not_implemented(void); 51 + #define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented() 52 + #endif 53 + 54 + #if defined(arch_cmpxchg) 55 + #define raw_cmpxchg arch_cmpxchg 56 + #elif defined(arch_cmpxchg_relaxed) 57 + #define raw_cmpxchg(...) \ 52 58 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__) 59 + #else 60 + extern void raw_cmpxchg_not_implemented(void); 61 + #define raw_cmpxchg(...) raw_cmpxchg_not_implemented() 53 62 #endif 54 63 55 - #endif /* arch_cmpxchg_relaxed */ 56 - 57 - #ifndef arch_cmpxchg64_relaxed 58 - #define arch_cmpxchg64_acquire arch_cmpxchg64 59 - #define arch_cmpxchg64_release arch_cmpxchg64 60 - #define arch_cmpxchg64_relaxed arch_cmpxchg64 61 - #else /* arch_cmpxchg64_relaxed */ 62 - 63 - #ifndef arch_cmpxchg64_acquire 64 - #define arch_cmpxchg64_acquire(...) \ 65 - __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__) 64 + #if defined(arch_cmpxchg_acquire) 65 + #define raw_cmpxchg_acquire arch_cmpxchg_acquire 66 + #elif defined(arch_cmpxchg_relaxed) 67 + #define raw_cmpxchg_acquire(...) \ 68 + __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__) 69 + #elif defined(arch_cmpxchg) 70 + #define raw_cmpxchg_acquire arch_cmpxchg 71 + #else 72 + extern void raw_cmpxchg_acquire_not_implemented(void); 73 + #define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented() 66 74 #endif 67 75 68 - #ifndef arch_cmpxchg64_release 69 - #define arch_cmpxchg64_release(...) \ 70 - __atomic_op_release(arch_cmpxchg64, __VA_ARGS__) 76 + #if defined(arch_cmpxchg_release) 77 + #define raw_cmpxchg_release arch_cmpxchg_release 78 + #elif defined(arch_cmpxchg_relaxed) 79 + #define raw_cmpxchg_release(...) \ 80 + __atomic_op_release(arch_cmpxchg, __VA_ARGS__) 81 + #elif defined(arch_cmpxchg) 82 + #define raw_cmpxchg_release arch_cmpxchg 83 + #else 84 + extern void raw_cmpxchg_release_not_implemented(void); 85 + #define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented() 71 86 #endif 72 87 73 - #ifndef arch_cmpxchg64 74 - #define arch_cmpxchg64(...) \ 88 + #if defined(arch_cmpxchg_relaxed) 89 + #define raw_cmpxchg_relaxed arch_cmpxchg_relaxed 90 + #elif defined(arch_cmpxchg) 91 + #define raw_cmpxchg_relaxed arch_cmpxchg 92 + #else 93 + extern void raw_cmpxchg_relaxed_not_implemented(void); 94 + #define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented() 95 + #endif 96 + 97 + #if defined(arch_cmpxchg64) 98 + #define raw_cmpxchg64 arch_cmpxchg64 99 + #elif defined(arch_cmpxchg64_relaxed) 100 + #define raw_cmpxchg64(...) \ 75 101 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__) 102 + #else 103 + extern void raw_cmpxchg64_not_implemented(void); 104 + #define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented() 76 105 #endif 77 106 78 - #endif /* arch_cmpxchg64_relaxed */ 79 - 80 - #ifndef arch_cmpxchg128_relaxed 81 - #define arch_cmpxchg128_acquire arch_cmpxchg128 82 - #define arch_cmpxchg128_release arch_cmpxchg128 83 - #define arch_cmpxchg128_relaxed arch_cmpxchg128 84 - #else /* arch_cmpxchg128_relaxed */ 85 - 86 - #ifndef arch_cmpxchg128_acquire 87 - #define arch_cmpxchg128_acquire(...) \ 88 - __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__) 107 + #if defined(arch_cmpxchg64_acquire) 108 + #define raw_cmpxchg64_acquire arch_cmpxchg64_acquire 109 + #elif defined(arch_cmpxchg64_relaxed) 110 + #define raw_cmpxchg64_acquire(...) \ 111 + __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__) 112 + #elif defined(arch_cmpxchg64) 113 + #define raw_cmpxchg64_acquire arch_cmpxchg64 114 + #else 115 + extern void raw_cmpxchg64_acquire_not_implemented(void); 116 + #define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented() 89 117 #endif 90 118 91 - #ifndef arch_cmpxchg128_release 92 - #define arch_cmpxchg128_release(...) \ 93 - __atomic_op_release(arch_cmpxchg128, __VA_ARGS__) 119 + #if defined(arch_cmpxchg64_release) 120 + #define raw_cmpxchg64_release arch_cmpxchg64_release 121 + #elif defined(arch_cmpxchg64_relaxed) 122 + #define raw_cmpxchg64_release(...) \ 123 + __atomic_op_release(arch_cmpxchg64, __VA_ARGS__) 124 + #elif defined(arch_cmpxchg64) 125 + #define raw_cmpxchg64_release arch_cmpxchg64 126 + #else 127 + extern void raw_cmpxchg64_release_not_implemented(void); 128 + #define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented() 94 129 #endif 95 130 96 - #ifndef arch_cmpxchg128 97 - #define arch_cmpxchg128(...) \ 131 + #if defined(arch_cmpxchg64_relaxed) 132 + #define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed 133 + #elif defined(arch_cmpxchg64) 134 + #define raw_cmpxchg64_relaxed arch_cmpxchg64 135 + #else 136 + extern void raw_cmpxchg64_relaxed_not_implemented(void); 137 + #define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented() 138 + #endif 139 + 140 + #if defined(arch_cmpxchg128) 141 + #define raw_cmpxchg128 arch_cmpxchg128 142 + #elif defined(arch_cmpxchg128_relaxed) 143 + #define raw_cmpxchg128(...) \ 98 144 __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__) 145 + #else 146 + extern void raw_cmpxchg128_not_implemented(void); 147 + #define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented() 99 148 #endif 100 149 101 - #endif /* arch_cmpxchg128_relaxed */ 102 - 103 - #ifndef arch_try_cmpxchg_relaxed 104 - #ifdef arch_try_cmpxchg 105 - #define arch_try_cmpxchg_acquire arch_try_cmpxchg 106 - #define arch_try_cmpxchg_release arch_try_cmpxchg 107 - #define arch_try_cmpxchg_relaxed arch_try_cmpxchg 108 - #endif /* arch_try_cmpxchg */ 109 - 110 - #ifndef arch_try_cmpxchg 111 - #define arch_try_cmpxchg(_ptr, _oldp, _new) \ 112 - ({ \ 113 - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 114 - ___r = arch_cmpxchg((_ptr), ___o, (_new)); \ 115 - if (unlikely(___r != ___o)) \ 116 - *___op = ___r; \ 117 - likely(___r == ___o); \ 118 - }) 119 - #endif /* arch_try_cmpxchg */ 120 - 121 - #ifndef arch_try_cmpxchg_acquire 122 - #define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \ 123 - ({ \ 124 - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 125 - ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \ 126 - if (unlikely(___r != ___o)) \ 127 - *___op = ___r; \ 128 - likely(___r == ___o); \ 129 - }) 130 - #endif /* arch_try_cmpxchg_acquire */ 131 - 132 - #ifndef arch_try_cmpxchg_release 133 - #define arch_try_cmpxchg_release(_ptr, _oldp, _new) \ 134 - ({ \ 135 - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 136 - ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \ 137 - if (unlikely(___r != ___o)) \ 138 - *___op = ___r; \ 139 - likely(___r == ___o); \ 140 - }) 141 - #endif /* arch_try_cmpxchg_release */ 142 - 143 - #ifndef arch_try_cmpxchg_relaxed 144 - #define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \ 145 - ({ \ 146 - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 147 - ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \ 148 - if (unlikely(___r != ___o)) \ 149 - *___op = ___r; \ 150 - likely(___r == ___o); \ 151 - }) 152 - #endif /* arch_try_cmpxchg_relaxed */ 153 - 154 - #else /* arch_try_cmpxchg_relaxed */ 155 - 156 - #ifndef arch_try_cmpxchg_acquire 157 - #define arch_try_cmpxchg_acquire(...) \ 158 - __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__) 150 + #if defined(arch_cmpxchg128_acquire) 151 + #define raw_cmpxchg128_acquire arch_cmpxchg128_acquire 152 + #elif defined(arch_cmpxchg128_relaxed) 153 + #define raw_cmpxchg128_acquire(...) \ 154 + __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__) 155 + #elif defined(arch_cmpxchg128) 156 + #define raw_cmpxchg128_acquire arch_cmpxchg128 157 + #else 158 + extern void raw_cmpxchg128_acquire_not_implemented(void); 159 + #define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented() 159 160 #endif 160 161 161 - #ifndef arch_try_cmpxchg_release 162 - #define arch_try_cmpxchg_release(...) \ 163 - __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__) 162 + #if defined(arch_cmpxchg128_release) 163 + #define raw_cmpxchg128_release arch_cmpxchg128_release 164 + #elif defined(arch_cmpxchg128_relaxed) 165 + #define raw_cmpxchg128_release(...) \ 166 + __atomic_op_release(arch_cmpxchg128, __VA_ARGS__) 167 + #elif defined(arch_cmpxchg128) 168 + #define raw_cmpxchg128_release arch_cmpxchg128 169 + #else 170 + extern void raw_cmpxchg128_release_not_implemented(void); 171 + #define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented() 164 172 #endif 165 173 166 - #ifndef arch_try_cmpxchg 167 - #define arch_try_cmpxchg(...) \ 174 + #if defined(arch_cmpxchg128_relaxed) 175 + #define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed 176 + #elif defined(arch_cmpxchg128) 177 + #define raw_cmpxchg128_relaxed arch_cmpxchg128 178 + #else 179 + extern void raw_cmpxchg128_relaxed_not_implemented(void); 180 + #define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented() 181 + #endif 182 + 183 + #if defined(arch_try_cmpxchg) 184 + #define raw_try_cmpxchg arch_try_cmpxchg 185 + #elif defined(arch_try_cmpxchg_relaxed) 186 + #define raw_try_cmpxchg(...) \ 168 187 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__) 188 + #else 189 + #define raw_try_cmpxchg(_ptr, _oldp, _new) \ 190 + ({ \ 191 + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 192 + ___r = raw_cmpxchg((_ptr), ___o, (_new)); \ 193 + if (unlikely(___r != ___o)) \ 194 + *___op = ___r; \ 195 + likely(___r == ___o); \ 196 + }) 169 197 #endif 170 198 171 - #endif /* arch_try_cmpxchg_relaxed */ 172 - 173 - #ifndef arch_try_cmpxchg64_relaxed 174 - #ifdef arch_try_cmpxchg64 175 - #define arch_try_cmpxchg64_acquire arch_try_cmpxchg64 176 - #define arch_try_cmpxchg64_release arch_try_cmpxchg64 177 - #define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64 178 - #endif /* arch_try_cmpxchg64 */ 179 - 180 - #ifndef arch_try_cmpxchg64 181 - #define arch_try_cmpxchg64(_ptr, _oldp, _new) \ 199 + #if defined(arch_try_cmpxchg_acquire) 200 + #define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire 201 + #elif defined(arch_try_cmpxchg_relaxed) 202 + #define raw_try_cmpxchg_acquire(...) \ 203 + __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__) 204 + #elif defined(arch_try_cmpxchg) 205 + #define raw_try_cmpxchg_acquire arch_try_cmpxchg 206 + #else 207 + #define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \ 182 208 ({ \ 183 209 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 184 - ___r = arch_cmpxchg64((_ptr), ___o, (_new)); \ 210 + ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \ 185 211 if (unlikely(___r != ___o)) \ 186 212 *___op = ___r; \ 187 213 likely(___r == ___o); \ 188 214 }) 189 - #endif /* arch_try_cmpxchg64 */ 190 - 191 - #ifndef arch_try_cmpxchg64_acquire 192 - #define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \ 193 - ({ \ 194 - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 195 - ___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \ 196 - if (unlikely(___r != ___o)) \ 197 - *___op = ___r; \ 198 - likely(___r == ___o); \ 199 - }) 200 - #endif /* arch_try_cmpxchg64_acquire */ 201 - 202 - #ifndef arch_try_cmpxchg64_release 203 - #define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \ 204 - ({ \ 205 - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 206 - ___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \ 207 - if (unlikely(___r != ___o)) \ 208 - *___op = ___r; \ 209 - likely(___r == ___o); \ 210 - }) 211 - #endif /* arch_try_cmpxchg64_release */ 212 - 213 - #ifndef arch_try_cmpxchg64_relaxed 214 - #define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \ 215 - ({ \ 216 - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 217 - ___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \ 218 - if (unlikely(___r != ___o)) \ 219 - *___op = ___r; \ 220 - likely(___r == ___o); \ 221 - }) 222 - #endif /* arch_try_cmpxchg64_relaxed */ 223 - 224 - #else /* arch_try_cmpxchg64_relaxed */ 225 - 226 - #ifndef arch_try_cmpxchg64_acquire 227 - #define arch_try_cmpxchg64_acquire(...) \ 228 - __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__) 229 215 #endif 230 216 231 - #ifndef arch_try_cmpxchg64_release 232 - #define arch_try_cmpxchg64_release(...) \ 233 - __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__) 217 + #if defined(arch_try_cmpxchg_release) 218 + #define raw_try_cmpxchg_release arch_try_cmpxchg_release 219 + #elif defined(arch_try_cmpxchg_relaxed) 220 + #define raw_try_cmpxchg_release(...) \ 221 + __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__) 222 + #elif defined(arch_try_cmpxchg) 223 + #define raw_try_cmpxchg_release arch_try_cmpxchg 224 + #else 225 + #define raw_try_cmpxchg_release(_ptr, _oldp, _new) \ 226 + ({ \ 227 + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 228 + ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \ 229 + if (unlikely(___r != ___o)) \ 230 + *___op = ___r; \ 231 + likely(___r == ___o); \ 232 + }) 234 233 #endif 235 234 236 - #ifndef arch_try_cmpxchg64 237 - #define arch_try_cmpxchg64(...) \ 235 + #if defined(arch_try_cmpxchg_relaxed) 236 + #define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed 237 + #elif defined(arch_try_cmpxchg) 238 + #define raw_try_cmpxchg_relaxed arch_try_cmpxchg 239 + #else 240 + #define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \ 241 + ({ \ 242 + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 243 + ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \ 244 + if (unlikely(___r != ___o)) \ 245 + *___op = ___r; \ 246 + likely(___r == ___o); \ 247 + }) 248 + #endif 249 + 250 + #if defined(arch_try_cmpxchg64) 251 + #define raw_try_cmpxchg64 arch_try_cmpxchg64 252 + #elif defined(arch_try_cmpxchg64_relaxed) 253 + #define raw_try_cmpxchg64(...) \ 238 254 __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__) 255 + #else 256 + #define raw_try_cmpxchg64(_ptr, _oldp, _new) \ 257 + ({ \ 258 + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 259 + ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \ 260 + if (unlikely(___r != ___o)) \ 261 + *___op = ___r; \ 262 + likely(___r == ___o); \ 263 + }) 239 264 #endif 240 265 241 - #endif /* arch_try_cmpxchg64_relaxed */ 242 - 243 - #ifndef arch_try_cmpxchg128_relaxed 244 - #ifdef arch_try_cmpxchg128 245 - #define arch_try_cmpxchg128_acquire arch_try_cmpxchg128 246 - #define arch_try_cmpxchg128_release arch_try_cmpxchg128 247 - #define arch_try_cmpxchg128_relaxed arch_try_cmpxchg128 248 - #endif /* arch_try_cmpxchg128 */ 249 - 250 - #ifndef arch_try_cmpxchg128 251 - #define arch_try_cmpxchg128(_ptr, _oldp, _new) \ 266 + #if defined(arch_try_cmpxchg64_acquire) 267 + #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire 268 + #elif defined(arch_try_cmpxchg64_relaxed) 269 + #define raw_try_cmpxchg64_acquire(...) \ 270 + __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__) 271 + #elif defined(arch_try_cmpxchg64) 272 + #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64 273 + #else 274 + #define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \ 252 275 ({ \ 253 276 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 254 - ___r = arch_cmpxchg128((_ptr), ___o, (_new)); \ 277 + ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \ 255 278 if (unlikely(___r != ___o)) \ 256 279 *___op = ___r; \ 257 280 likely(___r == ___o); \ 258 281 }) 259 - #endif /* arch_try_cmpxchg128 */ 260 - 261 - #ifndef arch_try_cmpxchg128_acquire 262 - #define arch_try_cmpxchg128_acquire(_ptr, _oldp, _new) \ 263 - ({ \ 264 - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 265 - ___r = arch_cmpxchg128_acquire((_ptr), ___o, (_new)); \ 266 - if (unlikely(___r != ___o)) \ 267 - *___op = ___r; \ 268 - likely(___r == ___o); \ 269 - }) 270 - #endif /* arch_try_cmpxchg128_acquire */ 271 - 272 - #ifndef arch_try_cmpxchg128_release 273 - #define arch_try_cmpxchg128_release(_ptr, _oldp, _new) \ 274 - ({ \ 275 - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 276 - ___r = arch_cmpxchg128_release((_ptr), ___o, (_new)); \ 277 - if (unlikely(___r != ___o)) \ 278 - *___op = ___r; \ 279 - likely(___r == ___o); \ 280 - }) 281 - #endif /* arch_try_cmpxchg128_release */ 282 - 283 - #ifndef arch_try_cmpxchg128_relaxed 284 - #define arch_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \ 285 - ({ \ 286 - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 287 - ___r = arch_cmpxchg128_relaxed((_ptr), ___o, (_new)); \ 288 - if (unlikely(___r != ___o)) \ 289 - *___op = ___r; \ 290 - likely(___r == ___o); \ 291 - }) 292 - #endif /* arch_try_cmpxchg128_relaxed */ 293 - 294 - #else /* arch_try_cmpxchg128_relaxed */ 295 - 296 - #ifndef arch_try_cmpxchg128_acquire 297 - #define arch_try_cmpxchg128_acquire(...) \ 298 - __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__) 299 282 #endif 300 283 301 - #ifndef arch_try_cmpxchg128_release 302 - #define arch_try_cmpxchg128_release(...) \ 303 - __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__) 284 + #if defined(arch_try_cmpxchg64_release) 285 + #define raw_try_cmpxchg64_release arch_try_cmpxchg64_release 286 + #elif defined(arch_try_cmpxchg64_relaxed) 287 + #define raw_try_cmpxchg64_release(...) \ 288 + __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__) 289 + #elif defined(arch_try_cmpxchg64) 290 + #define raw_try_cmpxchg64_release arch_try_cmpxchg64 291 + #else 292 + #define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \ 293 + ({ \ 294 + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 295 + ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \ 296 + if (unlikely(___r != ___o)) \ 297 + *___op = ___r; \ 298 + likely(___r == ___o); \ 299 + }) 304 300 #endif 305 301 306 - #ifndef arch_try_cmpxchg128 307 - #define arch_try_cmpxchg128(...) \ 302 + #if defined(arch_try_cmpxchg64_relaxed) 303 + #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed 304 + #elif defined(arch_try_cmpxchg64) 305 + #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64 306 + #else 307 + #define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \ 308 + ({ \ 309 + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 310 + ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \ 311 + if (unlikely(___r != ___o)) \ 312 + *___op = ___r; \ 313 + likely(___r == ___o); \ 314 + }) 315 + #endif 316 + 317 + #if defined(arch_try_cmpxchg128) 318 + #define raw_try_cmpxchg128 arch_try_cmpxchg128 319 + #elif defined(arch_try_cmpxchg128_relaxed) 320 + #define raw_try_cmpxchg128(...) \ 308 321 __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__) 322 + #else 323 + #define raw_try_cmpxchg128(_ptr, _oldp, _new) \ 324 + ({ \ 325 + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 326 + ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \ 327 + if (unlikely(___r != ___o)) \ 328 + *___op = ___r; \ 329 + likely(___r == ___o); \ 330 + }) 309 331 #endif 310 332 311 - #endif /* arch_try_cmpxchg128_relaxed */ 312 - 313 - #ifndef arch_try_cmpxchg_local 314 - #define arch_try_cmpxchg_local(_ptr, _oldp, _new) \ 333 + #if defined(arch_try_cmpxchg128_acquire) 334 + #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire 335 + #elif defined(arch_try_cmpxchg128_relaxed) 336 + #define raw_try_cmpxchg128_acquire(...) \ 337 + __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__) 338 + #elif defined(arch_try_cmpxchg128) 339 + #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128 340 + #else 341 + #define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \ 315 342 ({ \ 316 343 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 317 - ___r = arch_cmpxchg_local((_ptr), ___o, (_new)); \ 344 + ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \ 318 345 if (unlikely(___r != ___o)) \ 319 346 *___op = ___r; \ 320 347 likely(___r == ___o); \ 321 348 }) 322 - #endif /* arch_try_cmpxchg_local */ 349 + #endif 323 350 324 - #ifndef arch_try_cmpxchg64_local 325 - #define arch_try_cmpxchg64_local(_ptr, _oldp, _new) \ 351 + #if defined(arch_try_cmpxchg128_release) 352 + #define raw_try_cmpxchg128_release arch_try_cmpxchg128_release 353 + #elif defined(arch_try_cmpxchg128_relaxed) 354 + #define raw_try_cmpxchg128_release(...) \ 355 + __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__) 356 + #elif defined(arch_try_cmpxchg128) 357 + #define raw_try_cmpxchg128_release arch_try_cmpxchg128 358 + #else 359 + #define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \ 326 360 ({ \ 327 361 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 328 - ___r = arch_cmpxchg64_local((_ptr), ___o, (_new)); \ 362 + ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \ 329 363 if (unlikely(___r != ___o)) \ 330 364 *___op = ___r; \ 331 365 likely(___r == ___o); \ 332 366 }) 333 - #endif /* arch_try_cmpxchg64_local */ 367 + #endif 334 368 335 - #ifndef arch_atomic_read_acquire 369 + #if defined(arch_try_cmpxchg128_relaxed) 370 + #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed 371 + #elif defined(arch_try_cmpxchg128) 372 + #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128 373 + #else 374 + #define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \ 375 + ({ \ 376 + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 377 + ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \ 378 + if (unlikely(___r != ___o)) \ 379 + *___op = ___r; \ 380 + likely(___r == ___o); \ 381 + }) 382 + #endif 383 + 384 + #define raw_cmpxchg_local arch_cmpxchg_local 385 + 386 + #ifdef arch_try_cmpxchg_local 387 + #define raw_try_cmpxchg_local arch_try_cmpxchg_local 388 + #else 389 + #define raw_try_cmpxchg_local(_ptr, _oldp, _new) \ 390 + ({ \ 391 + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 392 + ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \ 393 + if (unlikely(___r != ___o)) \ 394 + *___op = ___r; \ 395 + likely(___r == ___o); \ 396 + }) 397 + #endif 398 + 399 + #define raw_cmpxchg64_local arch_cmpxchg64_local 400 + 401 + #ifdef arch_try_cmpxchg64_local 402 + #define raw_try_cmpxchg64_local arch_try_cmpxchg64_local 403 + #else 404 + #define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \ 405 + ({ \ 406 + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 407 + ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \ 408 + if (unlikely(___r != ___o)) \ 409 + *___op = ___r; \ 410 + likely(___r == ___o); \ 411 + }) 412 + #endif 413 + 414 + #define raw_cmpxchg128_local arch_cmpxchg128_local 415 + 416 + #ifdef arch_try_cmpxchg128_local 417 + #define raw_try_cmpxchg128_local arch_try_cmpxchg128_local 418 + #else 419 + #define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \ 420 + ({ \ 421 + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 422 + ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \ 423 + if (unlikely(___r != ___o)) \ 424 + *___op = ___r; \ 425 + likely(___r == ___o); \ 426 + }) 427 + #endif 428 + 429 + #define raw_sync_cmpxchg arch_sync_cmpxchg 430 + 431 + #define raw_atomic_read arch_atomic_read 432 + 433 + #if defined(arch_atomic_read_acquire) 434 + #define raw_atomic_read_acquire arch_atomic_read_acquire 435 + #elif defined(arch_atomic_read) 436 + #define raw_atomic_read_acquire arch_atomic_read 437 + #else 336 438 static __always_inline int 337 - arch_atomic_read_acquire(const atomic_t *v) 439 + raw_atomic_read_acquire(const atomic_t *v) 338 440 { 339 441 int ret; 340 442 341 443 if (__native_word(atomic_t)) { 342 444 ret = smp_load_acquire(&(v)->counter); 343 445 } else { 344 - ret = arch_atomic_read(v); 446 + ret = raw_atomic_read(v); 345 447 __atomic_acquire_fence(); 346 448 } 347 449 348 450 return ret; 349 451 } 350 - #define arch_atomic_read_acquire arch_atomic_read_acquire 351 452 #endif 352 453 353 - #ifndef arch_atomic_set_release 454 + #define raw_atomic_set arch_atomic_set 455 + 456 + #if defined(arch_atomic_set_release) 457 + #define raw_atomic_set_release arch_atomic_set_release 458 + #elif defined(arch_atomic_set) 459 + #define raw_atomic_set_release arch_atomic_set 460 + #else 354 461 static __always_inline void 355 - arch_atomic_set_release(atomic_t *v, int i) 462 + raw_atomic_set_release(atomic_t *v, int i) 356 463 { 357 464 if (__native_word(atomic_t)) { 358 465 smp_store_release(&(v)->counter, i); 359 466 } else { 360 467 __atomic_release_fence(); 361 - arch_atomic_set(v, i); 468 + raw_atomic_set(v, i); 362 469 } 363 470 } 364 - #define arch_atomic_set_release arch_atomic_set_release 365 471 #endif 366 472 367 - #ifndef arch_atomic_add_return_relaxed 368 - #define arch_atomic_add_return_acquire arch_atomic_add_return 369 - #define arch_atomic_add_return_release arch_atomic_add_return 370 - #define arch_atomic_add_return_relaxed arch_atomic_add_return 371 - #else /* arch_atomic_add_return_relaxed */ 473 + #define raw_atomic_add arch_atomic_add 372 474 373 - #ifndef arch_atomic_add_return_acquire 475 + #if defined(arch_atomic_add_return) 476 + #define raw_atomic_add_return arch_atomic_add_return 477 + #elif defined(arch_atomic_add_return_relaxed) 374 478 static __always_inline int 375 - arch_atomic_add_return_acquire(int i, atomic_t *v) 376 - { 377 - int ret = arch_atomic_add_return_relaxed(i, v); 378 - __atomic_acquire_fence(); 379 - return ret; 380 - } 381 - #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire 382 - #endif 383 - 384 - #ifndef arch_atomic_add_return_release 385 - static __always_inline int 386 - arch_atomic_add_return_release(int i, atomic_t *v) 387 - { 388 - __atomic_release_fence(); 389 - return arch_atomic_add_return_relaxed(i, v); 390 - } 391 - #define arch_atomic_add_return_release arch_atomic_add_return_release 392 - #endif 393 - 394 - #ifndef arch_atomic_add_return 395 - static __always_inline int 396 - arch_atomic_add_return(int i, atomic_t *v) 479 + raw_atomic_add_return(int i, atomic_t *v) 397 480 { 398 481 int ret; 399 482 __atomic_pre_full_fence(); ··· 484 401 __atomic_post_full_fence(); 485 402 return ret; 486 403 } 487 - #define arch_atomic_add_return arch_atomic_add_return 404 + #else 405 + #error "Unable to define raw_atomic_add_return" 488 406 #endif 489 407 490 - #endif /* arch_atomic_add_return_relaxed */ 491 - 492 - #ifndef arch_atomic_fetch_add_relaxed 493 - #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add 494 - #define arch_atomic_fetch_add_release arch_atomic_fetch_add 495 - #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add 496 - #else /* arch_atomic_fetch_add_relaxed */ 497 - 498 - #ifndef arch_atomic_fetch_add_acquire 408 + #if defined(arch_atomic_add_return_acquire) 409 + #define raw_atomic_add_return_acquire arch_atomic_add_return_acquire 410 + #elif defined(arch_atomic_add_return_relaxed) 499 411 static __always_inline int 500 - arch_atomic_fetch_add_acquire(int i, atomic_t *v) 412 + raw_atomic_add_return_acquire(int i, atomic_t *v) 501 413 { 502 - int ret = arch_atomic_fetch_add_relaxed(i, v); 414 + int ret = arch_atomic_add_return_relaxed(i, v); 503 415 __atomic_acquire_fence(); 504 416 return ret; 505 417 } 506 - #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire 418 + #elif defined(arch_atomic_add_return) 419 + #define raw_atomic_add_return_acquire arch_atomic_add_return 420 + #else 421 + #error "Unable to define raw_atomic_add_return_acquire" 507 422 #endif 508 423 509 - #ifndef arch_atomic_fetch_add_release 424 + #if defined(arch_atomic_add_return_release) 425 + #define raw_atomic_add_return_release arch_atomic_add_return_release 426 + #elif defined(arch_atomic_add_return_relaxed) 510 427 static __always_inline int 511 - arch_atomic_fetch_add_release(int i, atomic_t *v) 428 + raw_atomic_add_return_release(int i, atomic_t *v) 512 429 { 513 430 __atomic_release_fence(); 514 - return arch_atomic_fetch_add_relaxed(i, v); 431 + return arch_atomic_add_return_relaxed(i, v); 515 432 } 516 - #define arch_atomic_fetch_add_release arch_atomic_fetch_add_release 433 + #elif defined(arch_atomic_add_return) 434 + #define raw_atomic_add_return_release arch_atomic_add_return 435 + #else 436 + #error "Unable to define raw_atomic_add_return_release" 517 437 #endif 518 438 519 - #ifndef arch_atomic_fetch_add 439 + #if defined(arch_atomic_add_return_relaxed) 440 + #define raw_atomic_add_return_relaxed arch_atomic_add_return_relaxed 441 + #elif defined(arch_atomic_add_return) 442 + #define raw_atomic_add_return_relaxed arch_atomic_add_return 443 + #else 444 + #error "Unable to define raw_atomic_add_return_relaxed" 445 + #endif 446 + 447 + #if defined(arch_atomic_fetch_add) 448 + #define raw_atomic_fetch_add arch_atomic_fetch_add 449 + #elif defined(arch_atomic_fetch_add_relaxed) 520 450 static __always_inline int 521 - arch_atomic_fetch_add(int i, atomic_t *v) 451 + raw_atomic_fetch_add(int i, atomic_t *v) 522 452 { 523 453 int ret; 524 454 __atomic_pre_full_fence(); ··· 539 443 __atomic_post_full_fence(); 540 444 return ret; 541 445 } 542 - #define arch_atomic_fetch_add arch_atomic_fetch_add 446 + #else 447 + #error "Unable to define raw_atomic_fetch_add" 543 448 #endif 544 449 545 - #endif /* arch_atomic_fetch_add_relaxed */ 546 - 547 - #ifndef arch_atomic_sub_return_relaxed 548 - #define arch_atomic_sub_return_acquire arch_atomic_sub_return 549 - #define arch_atomic_sub_return_release arch_atomic_sub_return 550 - #define arch_atomic_sub_return_relaxed arch_atomic_sub_return 551 - #else /* arch_atomic_sub_return_relaxed */ 552 - 553 - #ifndef arch_atomic_sub_return_acquire 450 + #if defined(arch_atomic_fetch_add_acquire) 451 + #define raw_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire 452 + #elif defined(arch_atomic_fetch_add_relaxed) 554 453 static __always_inline int 555 - arch_atomic_sub_return_acquire(int i, atomic_t *v) 454 + raw_atomic_fetch_add_acquire(int i, atomic_t *v) 556 455 { 557 - int ret = arch_atomic_sub_return_relaxed(i, v); 456 + int ret = arch_atomic_fetch_add_relaxed(i, v); 558 457 __atomic_acquire_fence(); 559 458 return ret; 560 459 } 561 - #define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire 460 + #elif defined(arch_atomic_fetch_add) 461 + #define raw_atomic_fetch_add_acquire arch_atomic_fetch_add 462 + #else 463 + #error "Unable to define raw_atomic_fetch_add_acquire" 562 464 #endif 563 465 564 - #ifndef arch_atomic_sub_return_release 466 + #if defined(arch_atomic_fetch_add_release) 467 + #define raw_atomic_fetch_add_release arch_atomic_fetch_add_release 468 + #elif defined(arch_atomic_fetch_add_relaxed) 565 469 static __always_inline int 566 - arch_atomic_sub_return_release(int i, atomic_t *v) 470 + raw_atomic_fetch_add_release(int i, atomic_t *v) 567 471 { 568 472 __atomic_release_fence(); 569 - return arch_atomic_sub_return_relaxed(i, v); 473 + return arch_atomic_fetch_add_relaxed(i, v); 570 474 } 571 - #define arch_atomic_sub_return_release arch_atomic_sub_return_release 475 + #elif defined(arch_atomic_fetch_add) 476 + #define raw_atomic_fetch_add_release arch_atomic_fetch_add 477 + #else 478 + #error "Unable to define raw_atomic_fetch_add_release" 572 479 #endif 573 480 574 - #ifndef arch_atomic_sub_return 481 + #if defined(arch_atomic_fetch_add_relaxed) 482 + #define raw_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed 483 + #elif defined(arch_atomic_fetch_add) 484 + #define raw_atomic_fetch_add_relaxed arch_atomic_fetch_add 485 + #else 486 + #error "Unable to define raw_atomic_fetch_add_relaxed" 487 + #endif 488 + 489 + #define raw_atomic_sub arch_atomic_sub 490 + 491 + #if defined(arch_atomic_sub_return) 492 + #define raw_atomic_sub_return arch_atomic_sub_return 493 + #elif defined(arch_atomic_sub_return_relaxed) 575 494 static __always_inline int 576 - arch_atomic_sub_return(int i, atomic_t *v) 495 + raw_atomic_sub_return(int i, atomic_t *v) 577 496 { 578 497 int ret; 579 498 __atomic_pre_full_fence(); ··· 596 485 __atomic_post_full_fence(); 597 486 return ret; 598 487 } 599 - #define arch_atomic_sub_return arch_atomic_sub_return 488 + #else 489 + #error "Unable to define raw_atomic_sub_return" 600 490 #endif 601 491 602 - #endif /* arch_atomic_sub_return_relaxed */ 603 - 604 - #ifndef arch_atomic_fetch_sub_relaxed 605 - #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub 606 - #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub 607 - #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub 608 - #else /* arch_atomic_fetch_sub_relaxed */ 609 - 610 - #ifndef arch_atomic_fetch_sub_acquire 492 + #if defined(arch_atomic_sub_return_acquire) 493 + #define raw_atomic_sub_return_acquire arch_atomic_sub_return_acquire 494 + #elif defined(arch_atomic_sub_return_relaxed) 611 495 static __always_inline int 612 - arch_atomic_fetch_sub_acquire(int i, atomic_t *v) 496 + raw_atomic_sub_return_acquire(int i, atomic_t *v) 613 497 { 614 - int ret = arch_atomic_fetch_sub_relaxed(i, v); 498 + int ret = arch_atomic_sub_return_relaxed(i, v); 615 499 __atomic_acquire_fence(); 616 500 return ret; 617 501 } 618 - #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire 502 + #elif defined(arch_atomic_sub_return) 503 + #define raw_atomic_sub_return_acquire arch_atomic_sub_return 504 + #else 505 + #error "Unable to define raw_atomic_sub_return_acquire" 619 506 #endif 620 507 621 - #ifndef arch_atomic_fetch_sub_release 508 + #if defined(arch_atomic_sub_return_release) 509 + #define raw_atomic_sub_return_release arch_atomic_sub_return_release 510 + #elif defined(arch_atomic_sub_return_relaxed) 622 511 static __always_inline int 623 - arch_atomic_fetch_sub_release(int i, atomic_t *v) 512 + raw_atomic_sub_return_release(int i, atomic_t *v) 624 513 { 625 514 __atomic_release_fence(); 626 - return arch_atomic_fetch_sub_relaxed(i, v); 515 + return arch_atomic_sub_return_relaxed(i, v); 627 516 } 628 - #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release 517 + #elif defined(arch_atomic_sub_return) 518 + #define raw_atomic_sub_return_release arch_atomic_sub_return 519 + #else 520 + #error "Unable to define raw_atomic_sub_return_release" 629 521 #endif 630 522 631 - #ifndef arch_atomic_fetch_sub 523 + #if defined(arch_atomic_sub_return_relaxed) 524 + #define raw_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed 525 + #elif defined(arch_atomic_sub_return) 526 + #define raw_atomic_sub_return_relaxed arch_atomic_sub_return 527 + #else 528 + #error "Unable to define raw_atomic_sub_return_relaxed" 529 + #endif 530 + 531 + #if defined(arch_atomic_fetch_sub) 532 + #define raw_atomic_fetch_sub arch_atomic_fetch_sub 533 + #elif defined(arch_atomic_fetch_sub_relaxed) 632 534 static __always_inline int 633 - arch_atomic_fetch_sub(int i, atomic_t *v) 535 + raw_atomic_fetch_sub(int i, atomic_t *v) 634 536 { 635 537 int ret; 636 538 __atomic_pre_full_fence(); ··· 651 527 __atomic_post_full_fence(); 652 528 return ret; 653 529 } 654 - #define arch_atomic_fetch_sub arch_atomic_fetch_sub 530 + #else 531 + #error "Unable to define raw_atomic_fetch_sub" 655 532 #endif 656 533 657 - #endif /* arch_atomic_fetch_sub_relaxed */ 658 - 659 - #ifndef arch_atomic_inc 660 - static __always_inline void 661 - arch_atomic_inc(atomic_t *v) 662 - { 663 - arch_atomic_add(1, v); 664 - } 665 - #define arch_atomic_inc arch_atomic_inc 666 - #endif 667 - 668 - #ifndef arch_atomic_inc_return_relaxed 669 - #ifdef arch_atomic_inc_return 670 - #define arch_atomic_inc_return_acquire arch_atomic_inc_return 671 - #define arch_atomic_inc_return_release arch_atomic_inc_return 672 - #define arch_atomic_inc_return_relaxed arch_atomic_inc_return 673 - #endif /* arch_atomic_inc_return */ 674 - 675 - #ifndef arch_atomic_inc_return 534 + #if defined(arch_atomic_fetch_sub_acquire) 535 + #define raw_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire 536 + #elif defined(arch_atomic_fetch_sub_relaxed) 676 537 static __always_inline int 677 - arch_atomic_inc_return(atomic_t *v) 538 + raw_atomic_fetch_sub_acquire(int i, atomic_t *v) 678 539 { 679 - return arch_atomic_add_return(1, v); 680 - } 681 - #define arch_atomic_inc_return arch_atomic_inc_return 682 - #endif 683 - 684 - #ifndef arch_atomic_inc_return_acquire 685 - static __always_inline int 686 - arch_atomic_inc_return_acquire(atomic_t *v) 687 - { 688 - return arch_atomic_add_return_acquire(1, v); 689 - } 690 - #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire 691 - #endif 692 - 693 - #ifndef arch_atomic_inc_return_release 694 - static __always_inline int 695 - arch_atomic_inc_return_release(atomic_t *v) 696 - { 697 - return arch_atomic_add_return_release(1, v); 698 - } 699 - #define arch_atomic_inc_return_release arch_atomic_inc_return_release 700 - #endif 701 - 702 - #ifndef arch_atomic_inc_return_relaxed 703 - static __always_inline int 704 - arch_atomic_inc_return_relaxed(atomic_t *v) 705 - { 706 - return arch_atomic_add_return_relaxed(1, v); 707 - } 708 - #define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed 709 - #endif 710 - 711 - #else /* arch_atomic_inc_return_relaxed */ 712 - 713 - #ifndef arch_atomic_inc_return_acquire 714 - static __always_inline int 715 - arch_atomic_inc_return_acquire(atomic_t *v) 716 - { 717 - int ret = arch_atomic_inc_return_relaxed(v); 540 + int ret = arch_atomic_fetch_sub_relaxed(i, v); 718 541 __atomic_acquire_fence(); 719 542 return ret; 720 543 } 721 - #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire 544 + #elif defined(arch_atomic_fetch_sub) 545 + #define raw_atomic_fetch_sub_acquire arch_atomic_fetch_sub 546 + #else 547 + #error "Unable to define raw_atomic_fetch_sub_acquire" 722 548 #endif 723 549 724 - #ifndef arch_atomic_inc_return_release 550 + #if defined(arch_atomic_fetch_sub_release) 551 + #define raw_atomic_fetch_sub_release arch_atomic_fetch_sub_release 552 + #elif defined(arch_atomic_fetch_sub_relaxed) 725 553 static __always_inline int 726 - arch_atomic_inc_return_release(atomic_t *v) 554 + raw_atomic_fetch_sub_release(int i, atomic_t *v) 727 555 { 728 556 __atomic_release_fence(); 729 - return arch_atomic_inc_return_relaxed(v); 557 + return arch_atomic_fetch_sub_relaxed(i, v); 730 558 } 731 - #define arch_atomic_inc_return_release arch_atomic_inc_return_release 559 + #elif defined(arch_atomic_fetch_sub) 560 + #define raw_atomic_fetch_sub_release arch_atomic_fetch_sub 561 + #else 562 + #error "Unable to define raw_atomic_fetch_sub_release" 732 563 #endif 733 564 734 - #ifndef arch_atomic_inc_return 565 + #if defined(arch_atomic_fetch_sub_relaxed) 566 + #define raw_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed 567 + #elif defined(arch_atomic_fetch_sub) 568 + #define raw_atomic_fetch_sub_relaxed arch_atomic_fetch_sub 569 + #else 570 + #error "Unable to define raw_atomic_fetch_sub_relaxed" 571 + #endif 572 + 573 + #if defined(arch_atomic_inc) 574 + #define raw_atomic_inc arch_atomic_inc 575 + #else 576 + static __always_inline void 577 + raw_atomic_inc(atomic_t *v) 578 + { 579 + raw_atomic_add(1, v); 580 + } 581 + #endif 582 + 583 + #if defined(arch_atomic_inc_return) 584 + #define raw_atomic_inc_return arch_atomic_inc_return 585 + #elif defined(arch_atomic_inc_return_relaxed) 735 586 static __always_inline int 736 - arch_atomic_inc_return(atomic_t *v) 587 + raw_atomic_inc_return(atomic_t *v) 737 588 { 738 589 int ret; 739 590 __atomic_pre_full_fence(); ··· 716 617 __atomic_post_full_fence(); 717 618 return ret; 718 619 } 719 - #define arch_atomic_inc_return arch_atomic_inc_return 720 - #endif 721 - 722 - #endif /* arch_atomic_inc_return_relaxed */ 723 - 724 - #ifndef arch_atomic_fetch_inc_relaxed 725 - #ifdef arch_atomic_fetch_inc 726 - #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc 727 - #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc 728 - #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc 729 - #endif /* arch_atomic_fetch_inc */ 730 - 731 - #ifndef arch_atomic_fetch_inc 620 + #else 732 621 static __always_inline int 733 - arch_atomic_fetch_inc(atomic_t *v) 622 + raw_atomic_inc_return(atomic_t *v) 734 623 { 735 - return arch_atomic_fetch_add(1, v); 624 + return raw_atomic_add_return(1, v); 736 625 } 737 - #define arch_atomic_fetch_inc arch_atomic_fetch_inc 738 626 #endif 739 627 740 - #ifndef arch_atomic_fetch_inc_acquire 628 + #if defined(arch_atomic_inc_return_acquire) 629 + #define raw_atomic_inc_return_acquire arch_atomic_inc_return_acquire 630 + #elif defined(arch_atomic_inc_return_relaxed) 741 631 static __always_inline int 742 - arch_atomic_fetch_inc_acquire(atomic_t *v) 632 + raw_atomic_inc_return_acquire(atomic_t *v) 743 633 { 744 - return arch_atomic_fetch_add_acquire(1, v); 745 - } 746 - #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire 747 - #endif 748 - 749 - #ifndef arch_atomic_fetch_inc_release 750 - static __always_inline int 751 - arch_atomic_fetch_inc_release(atomic_t *v) 752 - { 753 - return arch_atomic_fetch_add_release(1, v); 754 - } 755 - #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release 756 - #endif 757 - 758 - #ifndef arch_atomic_fetch_inc_relaxed 759 - static __always_inline int 760 - arch_atomic_fetch_inc_relaxed(atomic_t *v) 761 - { 762 - return arch_atomic_fetch_add_relaxed(1, v); 763 - } 764 - #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed 765 - #endif 766 - 767 - #else /* arch_atomic_fetch_inc_relaxed */ 768 - 769 - #ifndef arch_atomic_fetch_inc_acquire 770 - static __always_inline int 771 - arch_atomic_fetch_inc_acquire(atomic_t *v) 772 - { 773 - int ret = arch_atomic_fetch_inc_relaxed(v); 634 + int ret = arch_atomic_inc_return_relaxed(v); 774 635 __atomic_acquire_fence(); 775 636 return ret; 776 637 } 777 - #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire 638 + #elif defined(arch_atomic_inc_return) 639 + #define raw_atomic_inc_return_acquire arch_atomic_inc_return 640 + #else 641 + static __always_inline int 642 + raw_atomic_inc_return_acquire(atomic_t *v) 643 + { 644 + return raw_atomic_add_return_acquire(1, v); 645 + } 778 646 #endif 779 647 780 - #ifndef arch_atomic_fetch_inc_release 648 + #if defined(arch_atomic_inc_return_release) 649 + #define raw_atomic_inc_return_release arch_atomic_inc_return_release 650 + #elif defined(arch_atomic_inc_return_relaxed) 781 651 static __always_inline int 782 - arch_atomic_fetch_inc_release(atomic_t *v) 652 + raw_atomic_inc_return_release(atomic_t *v) 783 653 { 784 654 __atomic_release_fence(); 785 - return arch_atomic_fetch_inc_relaxed(v); 655 + return arch_atomic_inc_return_relaxed(v); 786 656 } 787 - #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release 657 + #elif defined(arch_atomic_inc_return) 658 + #define raw_atomic_inc_return_release arch_atomic_inc_return 659 + #else 660 + static __always_inline int 661 + raw_atomic_inc_return_release(atomic_t *v) 662 + { 663 + return raw_atomic_add_return_release(1, v); 664 + } 788 665 #endif 789 666 790 - #ifndef arch_atomic_fetch_inc 667 + #if defined(arch_atomic_inc_return_relaxed) 668 + #define raw_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed 669 + #elif defined(arch_atomic_inc_return) 670 + #define raw_atomic_inc_return_relaxed arch_atomic_inc_return 671 + #else 791 672 static __always_inline int 792 - arch_atomic_fetch_inc(atomic_t *v) 673 + raw_atomic_inc_return_relaxed(atomic_t *v) 674 + { 675 + return raw_atomic_add_return_relaxed(1, v); 676 + } 677 + #endif 678 + 679 + #if defined(arch_atomic_fetch_inc) 680 + #define raw_atomic_fetch_inc arch_atomic_fetch_inc 681 + #elif defined(arch_atomic_fetch_inc_relaxed) 682 + static __always_inline int 683 + raw_atomic_fetch_inc(atomic_t *v) 793 684 { 794 685 int ret; 795 686 __atomic_pre_full_fence(); ··· 787 698 __atomic_post_full_fence(); 788 699 return ret; 789 700 } 790 - #define arch_atomic_fetch_inc arch_atomic_fetch_inc 791 - #endif 792 - 793 - #endif /* arch_atomic_fetch_inc_relaxed */ 794 - 795 - #ifndef arch_atomic_dec 796 - static __always_inline void 797 - arch_atomic_dec(atomic_t *v) 798 - { 799 - arch_atomic_sub(1, v); 800 - } 801 - #define arch_atomic_dec arch_atomic_dec 802 - #endif 803 - 804 - #ifndef arch_atomic_dec_return_relaxed 805 - #ifdef arch_atomic_dec_return 806 - #define arch_atomic_dec_return_acquire arch_atomic_dec_return 807 - #define arch_atomic_dec_return_release arch_atomic_dec_return 808 - #define arch_atomic_dec_return_relaxed arch_atomic_dec_return 809 - #endif /* arch_atomic_dec_return */ 810 - 811 - #ifndef arch_atomic_dec_return 701 + #else 812 702 static __always_inline int 813 - arch_atomic_dec_return(atomic_t *v) 703 + raw_atomic_fetch_inc(atomic_t *v) 814 704 { 815 - return arch_atomic_sub_return(1, v); 705 + return raw_atomic_fetch_add(1, v); 816 706 } 817 - #define arch_atomic_dec_return arch_atomic_dec_return 818 707 #endif 819 708 820 - #ifndef arch_atomic_dec_return_acquire 709 + #if defined(arch_atomic_fetch_inc_acquire) 710 + #define raw_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire 711 + #elif defined(arch_atomic_fetch_inc_relaxed) 821 712 static __always_inline int 822 - arch_atomic_dec_return_acquire(atomic_t *v) 713 + raw_atomic_fetch_inc_acquire(atomic_t *v) 823 714 { 824 - return arch_atomic_sub_return_acquire(1, v); 825 - } 826 - #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire 827 - #endif 828 - 829 - #ifndef arch_atomic_dec_return_release 830 - static __always_inline int 831 - arch_atomic_dec_return_release(atomic_t *v) 832 - { 833 - return arch_atomic_sub_return_release(1, v); 834 - } 835 - #define arch_atomic_dec_return_release arch_atomic_dec_return_release 836 - #endif 837 - 838 - #ifndef arch_atomic_dec_return_relaxed 839 - static __always_inline int 840 - arch_atomic_dec_return_relaxed(atomic_t *v) 841 - { 842 - return arch_atomic_sub_return_relaxed(1, v); 843 - } 844 - #define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed 845 - #endif 846 - 847 - #else /* arch_atomic_dec_return_relaxed */ 848 - 849 - #ifndef arch_atomic_dec_return_acquire 850 - static __always_inline int 851 - arch_atomic_dec_return_acquire(atomic_t *v) 852 - { 853 - int ret = arch_atomic_dec_return_relaxed(v); 715 + int ret = arch_atomic_fetch_inc_relaxed(v); 854 716 __atomic_acquire_fence(); 855 717 return ret; 856 718 } 857 - #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire 719 + #elif defined(arch_atomic_fetch_inc) 720 + #define raw_atomic_fetch_inc_acquire arch_atomic_fetch_inc 721 + #else 722 + static __always_inline int 723 + raw_atomic_fetch_inc_acquire(atomic_t *v) 724 + { 725 + return raw_atomic_fetch_add_acquire(1, v); 726 + } 858 727 #endif 859 728 860 - #ifndef arch_atomic_dec_return_release 729 + #if defined(arch_atomic_fetch_inc_release) 730 + #define raw_atomic_fetch_inc_release arch_atomic_fetch_inc_release 731 + #elif defined(arch_atomic_fetch_inc_relaxed) 861 732 static __always_inline int 862 - arch_atomic_dec_return_release(atomic_t *v) 733 + raw_atomic_fetch_inc_release(atomic_t *v) 863 734 { 864 735 __atomic_release_fence(); 865 - return arch_atomic_dec_return_relaxed(v); 736 + return arch_atomic_fetch_inc_relaxed(v); 866 737 } 867 - #define arch_atomic_dec_return_release arch_atomic_dec_return_release 738 + #elif defined(arch_atomic_fetch_inc) 739 + #define raw_atomic_fetch_inc_release arch_atomic_fetch_inc 740 + #else 741 + static __always_inline int 742 + raw_atomic_fetch_inc_release(atomic_t *v) 743 + { 744 + return raw_atomic_fetch_add_release(1, v); 745 + } 868 746 #endif 869 747 870 - #ifndef arch_atomic_dec_return 748 + #if defined(arch_atomic_fetch_inc_relaxed) 749 + #define raw_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed 750 + #elif defined(arch_atomic_fetch_inc) 751 + #define raw_atomic_fetch_inc_relaxed arch_atomic_fetch_inc 752 + #else 871 753 static __always_inline int 872 - arch_atomic_dec_return(atomic_t *v) 754 + raw_atomic_fetch_inc_relaxed(atomic_t *v) 755 + { 756 + return raw_atomic_fetch_add_relaxed(1, v); 757 + } 758 + #endif 759 + 760 + #if defined(arch_atomic_dec) 761 + #define raw_atomic_dec arch_atomic_dec 762 + #else 763 + static __always_inline void 764 + raw_atomic_dec(atomic_t *v) 765 + { 766 + raw_atomic_sub(1, v); 767 + } 768 + #endif 769 + 770 + #if defined(arch_atomic_dec_return) 771 + #define raw_atomic_dec_return arch_atomic_dec_return 772 + #elif defined(arch_atomic_dec_return_relaxed) 773 + static __always_inline int 774 + raw_atomic_dec_return(atomic_t *v) 873 775 { 874 776 int ret; 875 777 __atomic_pre_full_fence(); ··· 868 788 __atomic_post_full_fence(); 869 789 return ret; 870 790 } 871 - #define arch_atomic_dec_return arch_atomic_dec_return 872 - #endif 873 - 874 - #endif /* arch_atomic_dec_return_relaxed */ 875 - 876 - #ifndef arch_atomic_fetch_dec_relaxed 877 - #ifdef arch_atomic_fetch_dec 878 - #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec 879 - #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec 880 - #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec 881 - #endif /* arch_atomic_fetch_dec */ 882 - 883 - #ifndef arch_atomic_fetch_dec 791 + #else 884 792 static __always_inline int 885 - arch_atomic_fetch_dec(atomic_t *v) 793 + raw_atomic_dec_return(atomic_t *v) 886 794 { 887 - return arch_atomic_fetch_sub(1, v); 795 + return raw_atomic_sub_return(1, v); 888 796 } 889 - #define arch_atomic_fetch_dec arch_atomic_fetch_dec 890 797 #endif 891 798 892 - #ifndef arch_atomic_fetch_dec_acquire 799 + #if defined(arch_atomic_dec_return_acquire) 800 + #define raw_atomic_dec_return_acquire arch_atomic_dec_return_acquire 801 + #elif defined(arch_atomic_dec_return_relaxed) 893 802 static __always_inline int 894 - arch_atomic_fetch_dec_acquire(atomic_t *v) 803 + raw_atomic_dec_return_acquire(atomic_t *v) 895 804 { 896 - return arch_atomic_fetch_sub_acquire(1, v); 897 - } 898 - #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire 899 - #endif 900 - 901 - #ifndef arch_atomic_fetch_dec_release 902 - static __always_inline int 903 - arch_atomic_fetch_dec_release(atomic_t *v) 904 - { 905 - return arch_atomic_fetch_sub_release(1, v); 906 - } 907 - #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release 908 - #endif 909 - 910 - #ifndef arch_atomic_fetch_dec_relaxed 911 - static __always_inline int 912 - arch_atomic_fetch_dec_relaxed(atomic_t *v) 913 - { 914 - return arch_atomic_fetch_sub_relaxed(1, v); 915 - } 916 - #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed 917 - #endif 918 - 919 - #else /* arch_atomic_fetch_dec_relaxed */ 920 - 921 - #ifndef arch_atomic_fetch_dec_acquire 922 - static __always_inline int 923 - arch_atomic_fetch_dec_acquire(atomic_t *v) 924 - { 925 - int ret = arch_atomic_fetch_dec_relaxed(v); 805 + int ret = arch_atomic_dec_return_relaxed(v); 926 806 __atomic_acquire_fence(); 927 807 return ret; 928 808 } 929 - #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire 809 + #elif defined(arch_atomic_dec_return) 810 + #define raw_atomic_dec_return_acquire arch_atomic_dec_return 811 + #else 812 + static __always_inline int 813 + raw_atomic_dec_return_acquire(atomic_t *v) 814 + { 815 + return raw_atomic_sub_return_acquire(1, v); 816 + } 930 817 #endif 931 818 932 - #ifndef arch_atomic_fetch_dec_release 819 + #if defined(arch_atomic_dec_return_release) 820 + #define raw_atomic_dec_return_release arch_atomic_dec_return_release 821 + #elif defined(arch_atomic_dec_return_relaxed) 933 822 static __always_inline int 934 - arch_atomic_fetch_dec_release(atomic_t *v) 823 + raw_atomic_dec_return_release(atomic_t *v) 935 824 { 936 825 __atomic_release_fence(); 937 - return arch_atomic_fetch_dec_relaxed(v); 826 + return arch_atomic_dec_return_relaxed(v); 938 827 } 939 - #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release 828 + #elif defined(arch_atomic_dec_return) 829 + #define raw_atomic_dec_return_release arch_atomic_dec_return 830 + #else 831 + static __always_inline int 832 + raw_atomic_dec_return_release(atomic_t *v) 833 + { 834 + return raw_atomic_sub_return_release(1, v); 835 + } 940 836 #endif 941 837 942 - #ifndef arch_atomic_fetch_dec 838 + #if defined(arch_atomic_dec_return_relaxed) 839 + #define raw_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed 840 + #elif defined(arch_atomic_dec_return) 841 + #define raw_atomic_dec_return_relaxed arch_atomic_dec_return 842 + #else 943 843 static __always_inline int 944 - arch_atomic_fetch_dec(atomic_t *v) 844 + raw_atomic_dec_return_relaxed(atomic_t *v) 845 + { 846 + return raw_atomic_sub_return_relaxed(1, v); 847 + } 848 + #endif 849 + 850 + #if defined(arch_atomic_fetch_dec) 851 + #define raw_atomic_fetch_dec arch_atomic_fetch_dec 852 + #elif defined(arch_atomic_fetch_dec_relaxed) 853 + static __always_inline int 854 + raw_atomic_fetch_dec(atomic_t *v) 945 855 { 946 856 int ret; 947 857 __atomic_pre_full_fence(); ··· 939 869 __atomic_post_full_fence(); 940 870 return ret; 941 871 } 942 - #define arch_atomic_fetch_dec arch_atomic_fetch_dec 872 + #else 873 + static __always_inline int 874 + raw_atomic_fetch_dec(atomic_t *v) 875 + { 876 + return raw_atomic_fetch_sub(1, v); 877 + } 943 878 #endif 944 879 945 - #endif /* arch_atomic_fetch_dec_relaxed */ 946 - 947 - #ifndef arch_atomic_fetch_and_relaxed 948 - #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and 949 - #define arch_atomic_fetch_and_release arch_atomic_fetch_and 950 - #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and 951 - #else /* arch_atomic_fetch_and_relaxed */ 952 - 953 - #ifndef arch_atomic_fetch_and_acquire 880 + #if defined(arch_atomic_fetch_dec_acquire) 881 + #define raw_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire 882 + #elif defined(arch_atomic_fetch_dec_relaxed) 954 883 static __always_inline int 955 - arch_atomic_fetch_and_acquire(int i, atomic_t *v) 884 + raw_atomic_fetch_dec_acquire(atomic_t *v) 956 885 { 957 - int ret = arch_atomic_fetch_and_relaxed(i, v); 886 + int ret = arch_atomic_fetch_dec_relaxed(v); 958 887 __atomic_acquire_fence(); 959 888 return ret; 960 889 } 961 - #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire 890 + #elif defined(arch_atomic_fetch_dec) 891 + #define raw_atomic_fetch_dec_acquire arch_atomic_fetch_dec 892 + #else 893 + static __always_inline int 894 + raw_atomic_fetch_dec_acquire(atomic_t *v) 895 + { 896 + return raw_atomic_fetch_sub_acquire(1, v); 897 + } 962 898 #endif 963 899 964 - #ifndef arch_atomic_fetch_and_release 900 + #if defined(arch_atomic_fetch_dec_release) 901 + #define raw_atomic_fetch_dec_release arch_atomic_fetch_dec_release 902 + #elif defined(arch_atomic_fetch_dec_relaxed) 965 903 static __always_inline int 966 - arch_atomic_fetch_and_release(int i, atomic_t *v) 904 + raw_atomic_fetch_dec_release(atomic_t *v) 967 905 { 968 906 __atomic_release_fence(); 969 - return arch_atomic_fetch_and_relaxed(i, v); 907 + return arch_atomic_fetch_dec_relaxed(v); 970 908 } 971 - #define arch_atomic_fetch_and_release arch_atomic_fetch_and_release 909 + #elif defined(arch_atomic_fetch_dec) 910 + #define raw_atomic_fetch_dec_release arch_atomic_fetch_dec 911 + #else 912 + static __always_inline int 913 + raw_atomic_fetch_dec_release(atomic_t *v) 914 + { 915 + return raw_atomic_fetch_sub_release(1, v); 916 + } 972 917 #endif 973 918 974 - #ifndef arch_atomic_fetch_and 919 + #if defined(arch_atomic_fetch_dec_relaxed) 920 + #define raw_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed 921 + #elif defined(arch_atomic_fetch_dec) 922 + #define raw_atomic_fetch_dec_relaxed arch_atomic_fetch_dec 923 + #else 975 924 static __always_inline int 976 - arch_atomic_fetch_and(int i, atomic_t *v) 925 + raw_atomic_fetch_dec_relaxed(atomic_t *v) 926 + { 927 + return raw_atomic_fetch_sub_relaxed(1, v); 928 + } 929 + #endif 930 + 931 + #define raw_atomic_and arch_atomic_and 932 + 933 + #if defined(arch_atomic_fetch_and) 934 + #define raw_atomic_fetch_and arch_atomic_fetch_and 935 + #elif defined(arch_atomic_fetch_and_relaxed) 936 + static __always_inline int 937 + raw_atomic_fetch_and(int i, atomic_t *v) 977 938 { 978 939 int ret; 979 940 __atomic_pre_full_fence(); ··· 1012 911 __atomic_post_full_fence(); 1013 912 return ret; 1014 913 } 1015 - #define arch_atomic_fetch_and arch_atomic_fetch_and 914 + #else 915 + #error "Unable to define raw_atomic_fetch_and" 1016 916 #endif 1017 917 1018 - #endif /* arch_atomic_fetch_and_relaxed */ 1019 - 1020 - #ifndef arch_atomic_andnot 1021 - static __always_inline void 1022 - arch_atomic_andnot(int i, atomic_t *v) 1023 - { 1024 - arch_atomic_and(~i, v); 1025 - } 1026 - #define arch_atomic_andnot arch_atomic_andnot 1027 - #endif 1028 - 1029 - #ifndef arch_atomic_fetch_andnot_relaxed 1030 - #ifdef arch_atomic_fetch_andnot 1031 - #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot 1032 - #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot 1033 - #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot 1034 - #endif /* arch_atomic_fetch_andnot */ 1035 - 1036 - #ifndef arch_atomic_fetch_andnot 918 + #if defined(arch_atomic_fetch_and_acquire) 919 + #define raw_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire 920 + #elif defined(arch_atomic_fetch_and_relaxed) 1037 921 static __always_inline int 1038 - arch_atomic_fetch_andnot(int i, atomic_t *v) 922 + raw_atomic_fetch_and_acquire(int i, atomic_t *v) 1039 923 { 1040 - return arch_atomic_fetch_and(~i, v); 1041 - } 1042 - #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot 1043 - #endif 1044 - 1045 - #ifndef arch_atomic_fetch_andnot_acquire 1046 - static __always_inline int 1047 - arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) 1048 - { 1049 - return arch_atomic_fetch_and_acquire(~i, v); 1050 - } 1051 - #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire 1052 - #endif 1053 - 1054 - #ifndef arch_atomic_fetch_andnot_release 1055 - static __always_inline int 1056 - arch_atomic_fetch_andnot_release(int i, atomic_t *v) 1057 - { 1058 - return arch_atomic_fetch_and_release(~i, v); 1059 - } 1060 - #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release 1061 - #endif 1062 - 1063 - #ifndef arch_atomic_fetch_andnot_relaxed 1064 - static __always_inline int 1065 - arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v) 1066 - { 1067 - return arch_atomic_fetch_and_relaxed(~i, v); 1068 - } 1069 - #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed 1070 - #endif 1071 - 1072 - #else /* arch_atomic_fetch_andnot_relaxed */ 1073 - 1074 - #ifndef arch_atomic_fetch_andnot_acquire 1075 - static __always_inline int 1076 - arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) 1077 - { 1078 - int ret = arch_atomic_fetch_andnot_relaxed(i, v); 924 + int ret = arch_atomic_fetch_and_relaxed(i, v); 1079 925 __atomic_acquire_fence(); 1080 926 return ret; 1081 927 } 1082 - #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire 928 + #elif defined(arch_atomic_fetch_and) 929 + #define raw_atomic_fetch_and_acquire arch_atomic_fetch_and 930 + #else 931 + #error "Unable to define raw_atomic_fetch_and_acquire" 1083 932 #endif 1084 933 1085 - #ifndef arch_atomic_fetch_andnot_release 934 + #if defined(arch_atomic_fetch_and_release) 935 + #define raw_atomic_fetch_and_release arch_atomic_fetch_and_release 936 + #elif defined(arch_atomic_fetch_and_relaxed) 1086 937 static __always_inline int 1087 - arch_atomic_fetch_andnot_release(int i, atomic_t *v) 938 + raw_atomic_fetch_and_release(int i, atomic_t *v) 1088 939 { 1089 940 __atomic_release_fence(); 1090 - return arch_atomic_fetch_andnot_relaxed(i, v); 941 + return arch_atomic_fetch_and_relaxed(i, v); 1091 942 } 1092 - #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release 943 + #elif defined(arch_atomic_fetch_and) 944 + #define raw_atomic_fetch_and_release arch_atomic_fetch_and 945 + #else 946 + #error "Unable to define raw_atomic_fetch_and_release" 1093 947 #endif 1094 948 1095 - #ifndef arch_atomic_fetch_andnot 949 + #if defined(arch_atomic_fetch_and_relaxed) 950 + #define raw_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed 951 + #elif defined(arch_atomic_fetch_and) 952 + #define raw_atomic_fetch_and_relaxed arch_atomic_fetch_and 953 + #else 954 + #error "Unable to define raw_atomic_fetch_and_relaxed" 955 + #endif 956 + 957 + #if defined(arch_atomic_andnot) 958 + #define raw_atomic_andnot arch_atomic_andnot 959 + #else 960 + static __always_inline void 961 + raw_atomic_andnot(int i, atomic_t *v) 962 + { 963 + raw_atomic_and(~i, v); 964 + } 965 + #endif 966 + 967 + #if defined(arch_atomic_fetch_andnot) 968 + #define raw_atomic_fetch_andnot arch_atomic_fetch_andnot 969 + #elif defined(arch_atomic_fetch_andnot_relaxed) 1096 970 static __always_inline int 1097 - arch_atomic_fetch_andnot(int i, atomic_t *v) 971 + raw_atomic_fetch_andnot(int i, atomic_t *v) 1098 972 { 1099 973 int ret; 1100 974 __atomic_pre_full_fence(); ··· 1077 1001 __atomic_post_full_fence(); 1078 1002 return ret; 1079 1003 } 1080 - #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot 1004 + #else 1005 + static __always_inline int 1006 + raw_atomic_fetch_andnot(int i, atomic_t *v) 1007 + { 1008 + return raw_atomic_fetch_and(~i, v); 1009 + } 1081 1010 #endif 1082 1011 1083 - #endif /* arch_atomic_fetch_andnot_relaxed */ 1084 - 1085 - #ifndef arch_atomic_fetch_or_relaxed 1086 - #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or 1087 - #define arch_atomic_fetch_or_release arch_atomic_fetch_or 1088 - #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or 1089 - #else /* arch_atomic_fetch_or_relaxed */ 1090 - 1091 - #ifndef arch_atomic_fetch_or_acquire 1012 + #if defined(arch_atomic_fetch_andnot_acquire) 1013 + #define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire 1014 + #elif defined(arch_atomic_fetch_andnot_relaxed) 1092 1015 static __always_inline int 1093 - arch_atomic_fetch_or_acquire(int i, atomic_t *v) 1016 + raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) 1094 1017 { 1095 - int ret = arch_atomic_fetch_or_relaxed(i, v); 1018 + int ret = arch_atomic_fetch_andnot_relaxed(i, v); 1096 1019 __atomic_acquire_fence(); 1097 1020 return ret; 1098 1021 } 1099 - #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire 1022 + #elif defined(arch_atomic_fetch_andnot) 1023 + #define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot 1024 + #else 1025 + static __always_inline int 1026 + raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) 1027 + { 1028 + return raw_atomic_fetch_and_acquire(~i, v); 1029 + } 1100 1030 #endif 1101 1031 1102 - #ifndef arch_atomic_fetch_or_release 1032 + #if defined(arch_atomic_fetch_andnot_release) 1033 + #define raw_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release 1034 + #elif defined(arch_atomic_fetch_andnot_relaxed) 1103 1035 static __always_inline int 1104 - arch_atomic_fetch_or_release(int i, atomic_t *v) 1036 + raw_atomic_fetch_andnot_release(int i, atomic_t *v) 1105 1037 { 1106 1038 __atomic_release_fence(); 1107 - return arch_atomic_fetch_or_relaxed(i, v); 1039 + return arch_atomic_fetch_andnot_relaxed(i, v); 1108 1040 } 1109 - #define arch_atomic_fetch_or_release arch_atomic_fetch_or_release 1041 + #elif defined(arch_atomic_fetch_andnot) 1042 + #define raw_atomic_fetch_andnot_release arch_atomic_fetch_andnot 1043 + #else 1044 + static __always_inline int 1045 + raw_atomic_fetch_andnot_release(int i, atomic_t *v) 1046 + { 1047 + return raw_atomic_fetch_and_release(~i, v); 1048 + } 1110 1049 #endif 1111 1050 1112 - #ifndef arch_atomic_fetch_or 1051 + #if defined(arch_atomic_fetch_andnot_relaxed) 1052 + #define raw_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed 1053 + #elif defined(arch_atomic_fetch_andnot) 1054 + #define raw_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot 1055 + #else 1113 1056 static __always_inline int 1114 - arch_atomic_fetch_or(int i, atomic_t *v) 1057 + raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v) 1058 + { 1059 + return raw_atomic_fetch_and_relaxed(~i, v); 1060 + } 1061 + #endif 1062 + 1063 + #define raw_atomic_or arch_atomic_or 1064 + 1065 + #if defined(arch_atomic_fetch_or) 1066 + #define raw_atomic_fetch_or arch_atomic_fetch_or 1067 + #elif defined(arch_atomic_fetch_or_relaxed) 1068 + static __always_inline int 1069 + raw_atomic_fetch_or(int i, atomic_t *v) 1115 1070 { 1116 1071 int ret; 1117 1072 __atomic_pre_full_fence(); ··· 1150 1043 __atomic_post_full_fence(); 1151 1044 return ret; 1152 1045 } 1153 - #define arch_atomic_fetch_or arch_atomic_fetch_or 1046 + #else 1047 + #error "Unable to define raw_atomic_fetch_or" 1154 1048 #endif 1155 1049 1156 - #endif /* arch_atomic_fetch_or_relaxed */ 1157 - 1158 - #ifndef arch_atomic_fetch_xor_relaxed 1159 - #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor 1160 - #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor 1161 - #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor 1162 - #else /* arch_atomic_fetch_xor_relaxed */ 1163 - 1164 - #ifndef arch_atomic_fetch_xor_acquire 1050 + #if defined(arch_atomic_fetch_or_acquire) 1051 + #define raw_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire 1052 + #elif defined(arch_atomic_fetch_or_relaxed) 1165 1053 static __always_inline int 1166 - arch_atomic_fetch_xor_acquire(int i, atomic_t *v) 1054 + raw_atomic_fetch_or_acquire(int i, atomic_t *v) 1167 1055 { 1168 - int ret = arch_atomic_fetch_xor_relaxed(i, v); 1056 + int ret = arch_atomic_fetch_or_relaxed(i, v); 1169 1057 __atomic_acquire_fence(); 1170 1058 return ret; 1171 1059 } 1172 - #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire 1060 + #elif defined(arch_atomic_fetch_or) 1061 + #define raw_atomic_fetch_or_acquire arch_atomic_fetch_or 1062 + #else 1063 + #error "Unable to define raw_atomic_fetch_or_acquire" 1173 1064 #endif 1174 1065 1175 - #ifndef arch_atomic_fetch_xor_release 1066 + #if defined(arch_atomic_fetch_or_release) 1067 + #define raw_atomic_fetch_or_release arch_atomic_fetch_or_release 1068 + #elif defined(arch_atomic_fetch_or_relaxed) 1176 1069 static __always_inline int 1177 - arch_atomic_fetch_xor_release(int i, atomic_t *v) 1070 + raw_atomic_fetch_or_release(int i, atomic_t *v) 1178 1071 { 1179 1072 __atomic_release_fence(); 1180 - return arch_atomic_fetch_xor_relaxed(i, v); 1073 + return arch_atomic_fetch_or_relaxed(i, v); 1181 1074 } 1182 - #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release 1075 + #elif defined(arch_atomic_fetch_or) 1076 + #define raw_atomic_fetch_or_release arch_atomic_fetch_or 1077 + #else 1078 + #error "Unable to define raw_atomic_fetch_or_release" 1183 1079 #endif 1184 1080 1185 - #ifndef arch_atomic_fetch_xor 1081 + #if defined(arch_atomic_fetch_or_relaxed) 1082 + #define raw_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed 1083 + #elif defined(arch_atomic_fetch_or) 1084 + #define raw_atomic_fetch_or_relaxed arch_atomic_fetch_or 1085 + #else 1086 + #error "Unable to define raw_atomic_fetch_or_relaxed" 1087 + #endif 1088 + 1089 + #define raw_atomic_xor arch_atomic_xor 1090 + 1091 + #if defined(arch_atomic_fetch_xor) 1092 + #define raw_atomic_fetch_xor arch_atomic_fetch_xor 1093 + #elif defined(arch_atomic_fetch_xor_relaxed) 1186 1094 static __always_inline int 1187 - arch_atomic_fetch_xor(int i, atomic_t *v) 1095 + raw_atomic_fetch_xor(int i, atomic_t *v) 1188 1096 { 1189 1097 int ret; 1190 1098 __atomic_pre_full_fence(); ··· 1207 1085 __atomic_post_full_fence(); 1208 1086 return ret; 1209 1087 } 1210 - #define arch_atomic_fetch_xor arch_atomic_fetch_xor 1088 + #else 1089 + #error "Unable to define raw_atomic_fetch_xor" 1211 1090 #endif 1212 1091 1213 - #endif /* arch_atomic_fetch_xor_relaxed */ 1214 - 1215 - #ifndef arch_atomic_xchg_relaxed 1216 - #ifdef arch_atomic_xchg 1217 - #define arch_atomic_xchg_acquire arch_atomic_xchg 1218 - #define arch_atomic_xchg_release arch_atomic_xchg 1219 - #define arch_atomic_xchg_relaxed arch_atomic_xchg 1220 - #endif /* arch_atomic_xchg */ 1221 - 1222 - #ifndef arch_atomic_xchg 1092 + #if defined(arch_atomic_fetch_xor_acquire) 1093 + #define raw_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire 1094 + #elif defined(arch_atomic_fetch_xor_relaxed) 1223 1095 static __always_inline int 1224 - arch_atomic_xchg(atomic_t *v, int new) 1096 + raw_atomic_fetch_xor_acquire(int i, atomic_t *v) 1225 1097 { 1226 - return arch_xchg(&v->counter, new); 1227 - } 1228 - #define arch_atomic_xchg arch_atomic_xchg 1229 - #endif 1230 - 1231 - #ifndef arch_atomic_xchg_acquire 1232 - static __always_inline int 1233 - arch_atomic_xchg_acquire(atomic_t *v, int new) 1234 - { 1235 - return arch_xchg_acquire(&v->counter, new); 1236 - } 1237 - #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire 1238 - #endif 1239 - 1240 - #ifndef arch_atomic_xchg_release 1241 - static __always_inline int 1242 - arch_atomic_xchg_release(atomic_t *v, int new) 1243 - { 1244 - return arch_xchg_release(&v->counter, new); 1245 - } 1246 - #define arch_atomic_xchg_release arch_atomic_xchg_release 1247 - #endif 1248 - 1249 - #ifndef arch_atomic_xchg_relaxed 1250 - static __always_inline int 1251 - arch_atomic_xchg_relaxed(atomic_t *v, int new) 1252 - { 1253 - return arch_xchg_relaxed(&v->counter, new); 1254 - } 1255 - #define arch_atomic_xchg_relaxed arch_atomic_xchg_relaxed 1256 - #endif 1257 - 1258 - #else /* arch_atomic_xchg_relaxed */ 1259 - 1260 - #ifndef arch_atomic_xchg_acquire 1261 - static __always_inline int 1262 - arch_atomic_xchg_acquire(atomic_t *v, int i) 1263 - { 1264 - int ret = arch_atomic_xchg_relaxed(v, i); 1098 + int ret = arch_atomic_fetch_xor_relaxed(i, v); 1265 1099 __atomic_acquire_fence(); 1266 1100 return ret; 1267 1101 } 1268 - #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire 1102 + #elif defined(arch_atomic_fetch_xor) 1103 + #define raw_atomic_fetch_xor_acquire arch_atomic_fetch_xor 1104 + #else 1105 + #error "Unable to define raw_atomic_fetch_xor_acquire" 1269 1106 #endif 1270 1107 1271 - #ifndef arch_atomic_xchg_release 1108 + #if defined(arch_atomic_fetch_xor_release) 1109 + #define raw_atomic_fetch_xor_release arch_atomic_fetch_xor_release 1110 + #elif defined(arch_atomic_fetch_xor_relaxed) 1272 1111 static __always_inline int 1273 - arch_atomic_xchg_release(atomic_t *v, int i) 1112 + raw_atomic_fetch_xor_release(int i, atomic_t *v) 1274 1113 { 1275 1114 __atomic_release_fence(); 1276 - return arch_atomic_xchg_relaxed(v, i); 1115 + return arch_atomic_fetch_xor_relaxed(i, v); 1277 1116 } 1278 - #define arch_atomic_xchg_release arch_atomic_xchg_release 1117 + #elif defined(arch_atomic_fetch_xor) 1118 + #define raw_atomic_fetch_xor_release arch_atomic_fetch_xor 1119 + #else 1120 + #error "Unable to define raw_atomic_fetch_xor_release" 1279 1121 #endif 1280 1122 1281 - #ifndef arch_atomic_xchg 1123 + #if defined(arch_atomic_fetch_xor_relaxed) 1124 + #define raw_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed 1125 + #elif defined(arch_atomic_fetch_xor) 1126 + #define raw_atomic_fetch_xor_relaxed arch_atomic_fetch_xor 1127 + #else 1128 + #error "Unable to define raw_atomic_fetch_xor_relaxed" 1129 + #endif 1130 + 1131 + #if defined(arch_atomic_xchg) 1132 + #define raw_atomic_xchg arch_atomic_xchg 1133 + #elif defined(arch_atomic_xchg_relaxed) 1282 1134 static __always_inline int 1283 - arch_atomic_xchg(atomic_t *v, int i) 1135 + raw_atomic_xchg(atomic_t *v, int i) 1284 1136 { 1285 1137 int ret; 1286 1138 __atomic_pre_full_fence(); ··· 1262 1166 __atomic_post_full_fence(); 1263 1167 return ret; 1264 1168 } 1265 - #define arch_atomic_xchg arch_atomic_xchg 1266 - #endif 1267 - 1268 - #endif /* arch_atomic_xchg_relaxed */ 1269 - 1270 - #ifndef arch_atomic_cmpxchg_relaxed 1271 - #ifdef arch_atomic_cmpxchg 1272 - #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg 1273 - #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg 1274 - #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg 1275 - #endif /* arch_atomic_cmpxchg */ 1276 - 1277 - #ifndef arch_atomic_cmpxchg 1169 + #else 1278 1170 static __always_inline int 1279 - arch_atomic_cmpxchg(atomic_t *v, int old, int new) 1171 + raw_atomic_xchg(atomic_t *v, int new) 1280 1172 { 1281 - return arch_cmpxchg(&v->counter, old, new); 1173 + return raw_xchg(&v->counter, new); 1282 1174 } 1283 - #define arch_atomic_cmpxchg arch_atomic_cmpxchg 1284 1175 #endif 1285 1176 1286 - #ifndef arch_atomic_cmpxchg_acquire 1177 + #if defined(arch_atomic_xchg_acquire) 1178 + #define raw_atomic_xchg_acquire arch_atomic_xchg_acquire 1179 + #elif defined(arch_atomic_xchg_relaxed) 1287 1180 static __always_inline int 1288 - arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 1181 + raw_atomic_xchg_acquire(atomic_t *v, int i) 1289 1182 { 1290 - return arch_cmpxchg_acquire(&v->counter, old, new); 1291 - } 1292 - #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire 1293 - #endif 1294 - 1295 - #ifndef arch_atomic_cmpxchg_release 1296 - static __always_inline int 1297 - arch_atomic_cmpxchg_release(atomic_t *v, int old, int new) 1298 - { 1299 - return arch_cmpxchg_release(&v->counter, old, new); 1300 - } 1301 - #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release 1302 - #endif 1303 - 1304 - #ifndef arch_atomic_cmpxchg_relaxed 1305 - static __always_inline int 1306 - arch_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) 1307 - { 1308 - return arch_cmpxchg_relaxed(&v->counter, old, new); 1309 - } 1310 - #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed 1311 - #endif 1312 - 1313 - #else /* arch_atomic_cmpxchg_relaxed */ 1314 - 1315 - #ifndef arch_atomic_cmpxchg_acquire 1316 - static __always_inline int 1317 - arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 1318 - { 1319 - int ret = arch_atomic_cmpxchg_relaxed(v, old, new); 1183 + int ret = arch_atomic_xchg_relaxed(v, i); 1320 1184 __atomic_acquire_fence(); 1321 1185 return ret; 1322 1186 } 1323 - #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire 1187 + #elif defined(arch_atomic_xchg) 1188 + #define raw_atomic_xchg_acquire arch_atomic_xchg 1189 + #else 1190 + static __always_inline int 1191 + raw_atomic_xchg_acquire(atomic_t *v, int new) 1192 + { 1193 + return raw_xchg_acquire(&v->counter, new); 1194 + } 1324 1195 #endif 1325 1196 1326 - #ifndef arch_atomic_cmpxchg_release 1197 + #if defined(arch_atomic_xchg_release) 1198 + #define raw_atomic_xchg_release arch_atomic_xchg_release 1199 + #elif defined(arch_atomic_xchg_relaxed) 1327 1200 static __always_inline int 1328 - arch_atomic_cmpxchg_release(atomic_t *v, int old, int new) 1201 + raw_atomic_xchg_release(atomic_t *v, int i) 1329 1202 { 1330 1203 __atomic_release_fence(); 1331 - return arch_atomic_cmpxchg_relaxed(v, old, new); 1204 + return arch_atomic_xchg_relaxed(v, i); 1332 1205 } 1333 - #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release 1206 + #elif defined(arch_atomic_xchg) 1207 + #define raw_atomic_xchg_release arch_atomic_xchg 1208 + #else 1209 + static __always_inline int 1210 + raw_atomic_xchg_release(atomic_t *v, int new) 1211 + { 1212 + return raw_xchg_release(&v->counter, new); 1213 + } 1334 1214 #endif 1335 1215 1336 - #ifndef arch_atomic_cmpxchg 1216 + #if defined(arch_atomic_xchg_relaxed) 1217 + #define raw_atomic_xchg_relaxed arch_atomic_xchg_relaxed 1218 + #elif defined(arch_atomic_xchg) 1219 + #define raw_atomic_xchg_relaxed arch_atomic_xchg 1220 + #else 1337 1221 static __always_inline int 1338 - arch_atomic_cmpxchg(atomic_t *v, int old, int new) 1222 + raw_atomic_xchg_relaxed(atomic_t *v, int new) 1223 + { 1224 + return raw_xchg_relaxed(&v->counter, new); 1225 + } 1226 + #endif 1227 + 1228 + #if defined(arch_atomic_cmpxchg) 1229 + #define raw_atomic_cmpxchg arch_atomic_cmpxchg 1230 + #elif defined(arch_atomic_cmpxchg_relaxed) 1231 + static __always_inline int 1232 + raw_atomic_cmpxchg(atomic_t *v, int old, int new) 1339 1233 { 1340 1234 int ret; 1341 1235 __atomic_pre_full_fence(); ··· 1333 1247 __atomic_post_full_fence(); 1334 1248 return ret; 1335 1249 } 1336 - #define arch_atomic_cmpxchg arch_atomic_cmpxchg 1337 - #endif 1338 - 1339 - #endif /* arch_atomic_cmpxchg_relaxed */ 1340 - 1341 - #ifndef arch_atomic_try_cmpxchg_relaxed 1342 - #ifdef arch_atomic_try_cmpxchg 1343 - #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg 1344 - #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg 1345 - #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg 1346 - #endif /* arch_atomic_try_cmpxchg */ 1347 - 1348 - #ifndef arch_atomic_try_cmpxchg 1349 - static __always_inline bool 1350 - arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 1250 + #else 1251 + static __always_inline int 1252 + raw_atomic_cmpxchg(atomic_t *v, int old, int new) 1351 1253 { 1352 - int r, o = *old; 1353 - r = arch_atomic_cmpxchg(v, o, new); 1354 - if (unlikely(r != o)) 1355 - *old = r; 1356 - return likely(r == o); 1254 + return raw_cmpxchg(&v->counter, old, new); 1357 1255 } 1358 - #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg 1359 1256 #endif 1360 1257 1361 - #ifndef arch_atomic_try_cmpxchg_acquire 1362 - static __always_inline bool 1363 - arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1258 + #if defined(arch_atomic_cmpxchg_acquire) 1259 + #define raw_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire 1260 + #elif defined(arch_atomic_cmpxchg_relaxed) 1261 + static __always_inline int 1262 + raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 1364 1263 { 1365 - int r, o = *old; 1366 - r = arch_atomic_cmpxchg_acquire(v, o, new); 1367 - if (unlikely(r != o)) 1368 - *old = r; 1369 - return likely(r == o); 1370 - } 1371 - #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire 1372 - #endif 1373 - 1374 - #ifndef arch_atomic_try_cmpxchg_release 1375 - static __always_inline bool 1376 - arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 1377 - { 1378 - int r, o = *old; 1379 - r = arch_atomic_cmpxchg_release(v, o, new); 1380 - if (unlikely(r != o)) 1381 - *old = r; 1382 - return likely(r == o); 1383 - } 1384 - #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release 1385 - #endif 1386 - 1387 - #ifndef arch_atomic_try_cmpxchg_relaxed 1388 - static __always_inline bool 1389 - arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 1390 - { 1391 - int r, o = *old; 1392 - r = arch_atomic_cmpxchg_relaxed(v, o, new); 1393 - if (unlikely(r != o)) 1394 - *old = r; 1395 - return likely(r == o); 1396 - } 1397 - #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed 1398 - #endif 1399 - 1400 - #else /* arch_atomic_try_cmpxchg_relaxed */ 1401 - 1402 - #ifndef arch_atomic_try_cmpxchg_acquire 1403 - static __always_inline bool 1404 - arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1405 - { 1406 - bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 1264 + int ret = arch_atomic_cmpxchg_relaxed(v, old, new); 1407 1265 __atomic_acquire_fence(); 1408 1266 return ret; 1409 1267 } 1410 - #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire 1268 + #elif defined(arch_atomic_cmpxchg) 1269 + #define raw_atomic_cmpxchg_acquire arch_atomic_cmpxchg 1270 + #else 1271 + static __always_inline int 1272 + raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 1273 + { 1274 + return raw_cmpxchg_acquire(&v->counter, old, new); 1275 + } 1411 1276 #endif 1412 1277 1413 - #ifndef arch_atomic_try_cmpxchg_release 1414 - static __always_inline bool 1415 - arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 1278 + #if defined(arch_atomic_cmpxchg_release) 1279 + #define raw_atomic_cmpxchg_release arch_atomic_cmpxchg_release 1280 + #elif defined(arch_atomic_cmpxchg_relaxed) 1281 + static __always_inline int 1282 + raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) 1416 1283 { 1417 1284 __atomic_release_fence(); 1418 - return arch_atomic_try_cmpxchg_relaxed(v, old, new); 1285 + return arch_atomic_cmpxchg_relaxed(v, old, new); 1419 1286 } 1420 - #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release 1287 + #elif defined(arch_atomic_cmpxchg) 1288 + #define raw_atomic_cmpxchg_release arch_atomic_cmpxchg 1289 + #else 1290 + static __always_inline int 1291 + raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) 1292 + { 1293 + return raw_cmpxchg_release(&v->counter, old, new); 1294 + } 1421 1295 #endif 1422 1296 1423 - #ifndef arch_atomic_try_cmpxchg 1297 + #if defined(arch_atomic_cmpxchg_relaxed) 1298 + #define raw_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed 1299 + #elif defined(arch_atomic_cmpxchg) 1300 + #define raw_atomic_cmpxchg_relaxed arch_atomic_cmpxchg 1301 + #else 1302 + static __always_inline int 1303 + raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) 1304 + { 1305 + return raw_cmpxchg_relaxed(&v->counter, old, new); 1306 + } 1307 + #endif 1308 + 1309 + #if defined(arch_atomic_try_cmpxchg) 1310 + #define raw_atomic_try_cmpxchg arch_atomic_try_cmpxchg 1311 + #elif defined(arch_atomic_try_cmpxchg_relaxed) 1424 1312 static __always_inline bool 1425 - arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 1313 + raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 1426 1314 { 1427 1315 bool ret; 1428 1316 __atomic_pre_full_fence(); ··· 1404 1344 __atomic_post_full_fence(); 1405 1345 return ret; 1406 1346 } 1407 - #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg 1408 - #endif 1409 - 1410 - #endif /* arch_atomic_try_cmpxchg_relaxed */ 1411 - 1412 - #ifndef arch_atomic_sub_and_test 1347 + #else 1413 1348 static __always_inline bool 1414 - arch_atomic_sub_and_test(int i, atomic_t *v) 1349 + raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 1415 1350 { 1416 - return arch_atomic_sub_return(i, v) == 0; 1351 + int r, o = *old; 1352 + r = raw_atomic_cmpxchg(v, o, new); 1353 + if (unlikely(r != o)) 1354 + *old = r; 1355 + return likely(r == o); 1417 1356 } 1418 - #define arch_atomic_sub_and_test arch_atomic_sub_and_test 1419 1357 #endif 1420 1358 1421 - #ifndef arch_atomic_dec_and_test 1359 + #if defined(arch_atomic_try_cmpxchg_acquire) 1360 + #define raw_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire 1361 + #elif defined(arch_atomic_try_cmpxchg_relaxed) 1422 1362 static __always_inline bool 1423 - arch_atomic_dec_and_test(atomic_t *v) 1363 + raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1424 1364 { 1425 - return arch_atomic_dec_return(v) == 0; 1426 - } 1427 - #define arch_atomic_dec_and_test arch_atomic_dec_and_test 1428 - #endif 1429 - 1430 - #ifndef arch_atomic_inc_and_test 1431 - static __always_inline bool 1432 - arch_atomic_inc_and_test(atomic_t *v) 1433 - { 1434 - return arch_atomic_inc_return(v) == 0; 1435 - } 1436 - #define arch_atomic_inc_and_test arch_atomic_inc_and_test 1437 - #endif 1438 - 1439 - #ifndef arch_atomic_add_negative_relaxed 1440 - #ifdef arch_atomic_add_negative 1441 - #define arch_atomic_add_negative_acquire arch_atomic_add_negative 1442 - #define arch_atomic_add_negative_release arch_atomic_add_negative 1443 - #define arch_atomic_add_negative_relaxed arch_atomic_add_negative 1444 - #endif /* arch_atomic_add_negative */ 1445 - 1446 - #ifndef arch_atomic_add_negative 1447 - static __always_inline bool 1448 - arch_atomic_add_negative(int i, atomic_t *v) 1449 - { 1450 - return arch_atomic_add_return(i, v) < 0; 1451 - } 1452 - #define arch_atomic_add_negative arch_atomic_add_negative 1453 - #endif 1454 - 1455 - #ifndef arch_atomic_add_negative_acquire 1456 - static __always_inline bool 1457 - arch_atomic_add_negative_acquire(int i, atomic_t *v) 1458 - { 1459 - return arch_atomic_add_return_acquire(i, v) < 0; 1460 - } 1461 - #define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire 1462 - #endif 1463 - 1464 - #ifndef arch_atomic_add_negative_release 1465 - static __always_inline bool 1466 - arch_atomic_add_negative_release(int i, atomic_t *v) 1467 - { 1468 - return arch_atomic_add_return_release(i, v) < 0; 1469 - } 1470 - #define arch_atomic_add_negative_release arch_atomic_add_negative_release 1471 - #endif 1472 - 1473 - #ifndef arch_atomic_add_negative_relaxed 1474 - static __always_inline bool 1475 - arch_atomic_add_negative_relaxed(int i, atomic_t *v) 1476 - { 1477 - return arch_atomic_add_return_relaxed(i, v) < 0; 1478 - } 1479 - #define arch_atomic_add_negative_relaxed arch_atomic_add_negative_relaxed 1480 - #endif 1481 - 1482 - #else /* arch_atomic_add_negative_relaxed */ 1483 - 1484 - #ifndef arch_atomic_add_negative_acquire 1485 - static __always_inline bool 1486 - arch_atomic_add_negative_acquire(int i, atomic_t *v) 1487 - { 1488 - bool ret = arch_atomic_add_negative_relaxed(i, v); 1365 + bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 1489 1366 __atomic_acquire_fence(); 1490 1367 return ret; 1491 1368 } 1492 - #define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire 1369 + #elif defined(arch_atomic_try_cmpxchg) 1370 + #define raw_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg 1371 + #else 1372 + static __always_inline bool 1373 + raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1374 + { 1375 + int r, o = *old; 1376 + r = raw_atomic_cmpxchg_acquire(v, o, new); 1377 + if (unlikely(r != o)) 1378 + *old = r; 1379 + return likely(r == o); 1380 + } 1493 1381 #endif 1494 1382 1495 - #ifndef arch_atomic_add_negative_release 1383 + #if defined(arch_atomic_try_cmpxchg_release) 1384 + #define raw_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release 1385 + #elif defined(arch_atomic_try_cmpxchg_relaxed) 1496 1386 static __always_inline bool 1497 - arch_atomic_add_negative_release(int i, atomic_t *v) 1387 + raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 1498 1388 { 1499 1389 __atomic_release_fence(); 1500 - return arch_atomic_add_negative_relaxed(i, v); 1390 + return arch_atomic_try_cmpxchg_relaxed(v, old, new); 1501 1391 } 1502 - #define arch_atomic_add_negative_release arch_atomic_add_negative_release 1392 + #elif defined(arch_atomic_try_cmpxchg) 1393 + #define raw_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg 1394 + #else 1395 + static __always_inline bool 1396 + raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 1397 + { 1398 + int r, o = *old; 1399 + r = raw_atomic_cmpxchg_release(v, o, new); 1400 + if (unlikely(r != o)) 1401 + *old = r; 1402 + return likely(r == o); 1403 + } 1503 1404 #endif 1504 1405 1505 - #ifndef arch_atomic_add_negative 1406 + #if defined(arch_atomic_try_cmpxchg_relaxed) 1407 + #define raw_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed 1408 + #elif defined(arch_atomic_try_cmpxchg) 1409 + #define raw_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg 1410 + #else 1506 1411 static __always_inline bool 1507 - arch_atomic_add_negative(int i, atomic_t *v) 1412 + raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 1413 + { 1414 + int r, o = *old; 1415 + r = raw_atomic_cmpxchg_relaxed(v, o, new); 1416 + if (unlikely(r != o)) 1417 + *old = r; 1418 + return likely(r == o); 1419 + } 1420 + #endif 1421 + 1422 + #if defined(arch_atomic_sub_and_test) 1423 + #define raw_atomic_sub_and_test arch_atomic_sub_and_test 1424 + #else 1425 + static __always_inline bool 1426 + raw_atomic_sub_and_test(int i, atomic_t *v) 1427 + { 1428 + return raw_atomic_sub_return(i, v) == 0; 1429 + } 1430 + #endif 1431 + 1432 + #if defined(arch_atomic_dec_and_test) 1433 + #define raw_atomic_dec_and_test arch_atomic_dec_and_test 1434 + #else 1435 + static __always_inline bool 1436 + raw_atomic_dec_and_test(atomic_t *v) 1437 + { 1438 + return raw_atomic_dec_return(v) == 0; 1439 + } 1440 + #endif 1441 + 1442 + #if defined(arch_atomic_inc_and_test) 1443 + #define raw_atomic_inc_and_test arch_atomic_inc_and_test 1444 + #else 1445 + static __always_inline bool 1446 + raw_atomic_inc_and_test(atomic_t *v) 1447 + { 1448 + return raw_atomic_inc_return(v) == 0; 1449 + } 1450 + #endif 1451 + 1452 + #if defined(arch_atomic_add_negative) 1453 + #define raw_atomic_add_negative arch_atomic_add_negative 1454 + #elif defined(arch_atomic_add_negative_relaxed) 1455 + static __always_inline bool 1456 + raw_atomic_add_negative(int i, atomic_t *v) 1508 1457 { 1509 1458 bool ret; 1510 1459 __atomic_pre_full_fence(); ··· 1521 1452 __atomic_post_full_fence(); 1522 1453 return ret; 1523 1454 } 1524 - #define arch_atomic_add_negative arch_atomic_add_negative 1455 + #else 1456 + static __always_inline bool 1457 + raw_atomic_add_negative(int i, atomic_t *v) 1458 + { 1459 + return raw_atomic_add_return(i, v) < 0; 1460 + } 1525 1461 #endif 1526 1462 1527 - #endif /* arch_atomic_add_negative_relaxed */ 1528 - 1529 - #ifndef arch_atomic_fetch_add_unless 1530 - static __always_inline int 1531 - arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 1463 + #if defined(arch_atomic_add_negative_acquire) 1464 + #define raw_atomic_add_negative_acquire arch_atomic_add_negative_acquire 1465 + #elif defined(arch_atomic_add_negative_relaxed) 1466 + static __always_inline bool 1467 + raw_atomic_add_negative_acquire(int i, atomic_t *v) 1532 1468 { 1533 - int c = arch_atomic_read(v); 1469 + bool ret = arch_atomic_add_negative_relaxed(i, v); 1470 + __atomic_acquire_fence(); 1471 + return ret; 1472 + } 1473 + #elif defined(arch_atomic_add_negative) 1474 + #define raw_atomic_add_negative_acquire arch_atomic_add_negative 1475 + #else 1476 + static __always_inline bool 1477 + raw_atomic_add_negative_acquire(int i, atomic_t *v) 1478 + { 1479 + return raw_atomic_add_return_acquire(i, v) < 0; 1480 + } 1481 + #endif 1482 + 1483 + #if defined(arch_atomic_add_negative_release) 1484 + #define raw_atomic_add_negative_release arch_atomic_add_negative_release 1485 + #elif defined(arch_atomic_add_negative_relaxed) 1486 + static __always_inline bool 1487 + raw_atomic_add_negative_release(int i, atomic_t *v) 1488 + { 1489 + __atomic_release_fence(); 1490 + return arch_atomic_add_negative_relaxed(i, v); 1491 + } 1492 + #elif defined(arch_atomic_add_negative) 1493 + #define raw_atomic_add_negative_release arch_atomic_add_negative 1494 + #else 1495 + static __always_inline bool 1496 + raw_atomic_add_negative_release(int i, atomic_t *v) 1497 + { 1498 + return raw_atomic_add_return_release(i, v) < 0; 1499 + } 1500 + #endif 1501 + 1502 + #if defined(arch_atomic_add_negative_relaxed) 1503 + #define raw_atomic_add_negative_relaxed arch_atomic_add_negative_relaxed 1504 + #elif defined(arch_atomic_add_negative) 1505 + #define raw_atomic_add_negative_relaxed arch_atomic_add_negative 1506 + #else 1507 + static __always_inline bool 1508 + raw_atomic_add_negative_relaxed(int i, atomic_t *v) 1509 + { 1510 + return raw_atomic_add_return_relaxed(i, v) < 0; 1511 + } 1512 + #endif 1513 + 1514 + #if defined(arch_atomic_fetch_add_unless) 1515 + #define raw_atomic_fetch_add_unless arch_atomic_fetch_add_unless 1516 + #else 1517 + static __always_inline int 1518 + raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) 1519 + { 1520 + int c = raw_atomic_read(v); 1534 1521 1535 1522 do { 1536 1523 if (unlikely(c == u)) 1537 1524 break; 1538 - } while (!arch_atomic_try_cmpxchg(v, &c, c + a)); 1525 + } while (!raw_atomic_try_cmpxchg(v, &c, c + a)); 1539 1526 1540 1527 return c; 1541 1528 } 1542 - #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 1543 1529 #endif 1544 1530 1545 - #ifndef arch_atomic_add_unless 1531 + #if defined(arch_atomic_add_unless) 1532 + #define raw_atomic_add_unless arch_atomic_add_unless 1533 + #else 1546 1534 static __always_inline bool 1547 - arch_atomic_add_unless(atomic_t *v, int a, int u) 1535 + raw_atomic_add_unless(atomic_t *v, int a, int u) 1548 1536 { 1549 - return arch_atomic_fetch_add_unless(v, a, u) != u; 1537 + return raw_atomic_fetch_add_unless(v, a, u) != u; 1550 1538 } 1551 - #define arch_atomic_add_unless arch_atomic_add_unless 1552 1539 #endif 1553 1540 1554 - #ifndef arch_atomic_inc_not_zero 1541 + #if defined(arch_atomic_inc_not_zero) 1542 + #define raw_atomic_inc_not_zero arch_atomic_inc_not_zero 1543 + #else 1555 1544 static __always_inline bool 1556 - arch_atomic_inc_not_zero(atomic_t *v) 1545 + raw_atomic_inc_not_zero(atomic_t *v) 1557 1546 { 1558 - return arch_atomic_add_unless(v, 1, 0); 1547 + return raw_atomic_add_unless(v, 1, 0); 1559 1548 } 1560 - #define arch_atomic_inc_not_zero arch_atomic_inc_not_zero 1561 1549 #endif 1562 1550 1563 - #ifndef arch_atomic_inc_unless_negative 1551 + #if defined(arch_atomic_inc_unless_negative) 1552 + #define raw_atomic_inc_unless_negative arch_atomic_inc_unless_negative 1553 + #else 1564 1554 static __always_inline bool 1565 - arch_atomic_inc_unless_negative(atomic_t *v) 1555 + raw_atomic_inc_unless_negative(atomic_t *v) 1566 1556 { 1567 - int c = arch_atomic_read(v); 1557 + int c = raw_atomic_read(v); 1568 1558 1569 1559 do { 1570 1560 if (unlikely(c < 0)) 1571 1561 return false; 1572 - } while (!arch_atomic_try_cmpxchg(v, &c, c + 1)); 1562 + } while (!raw_atomic_try_cmpxchg(v, &c, c + 1)); 1573 1563 1574 1564 return true; 1575 1565 } 1576 - #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative 1577 1566 #endif 1578 1567 1579 - #ifndef arch_atomic_dec_unless_positive 1568 + #if defined(arch_atomic_dec_unless_positive) 1569 + #define raw_atomic_dec_unless_positive arch_atomic_dec_unless_positive 1570 + #else 1580 1571 static __always_inline bool 1581 - arch_atomic_dec_unless_positive(atomic_t *v) 1572 + raw_atomic_dec_unless_positive(atomic_t *v) 1582 1573 { 1583 - int c = arch_atomic_read(v); 1574 + int c = raw_atomic_read(v); 1584 1575 1585 1576 do { 1586 1577 if (unlikely(c > 0)) 1587 1578 return false; 1588 - } while (!arch_atomic_try_cmpxchg(v, &c, c - 1)); 1579 + } while (!raw_atomic_try_cmpxchg(v, &c, c - 1)); 1589 1580 1590 1581 return true; 1591 1582 } 1592 - #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive 1593 1583 #endif 1594 1584 1595 - #ifndef arch_atomic_dec_if_positive 1585 + #if defined(arch_atomic_dec_if_positive) 1586 + #define raw_atomic_dec_if_positive arch_atomic_dec_if_positive 1587 + #else 1596 1588 static __always_inline int 1597 - arch_atomic_dec_if_positive(atomic_t *v) 1589 + raw_atomic_dec_if_positive(atomic_t *v) 1598 1590 { 1599 - int dec, c = arch_atomic_read(v); 1591 + int dec, c = raw_atomic_read(v); 1600 1592 1601 1593 do { 1602 1594 dec = c - 1; 1603 1595 if (unlikely(dec < 0)) 1604 1596 break; 1605 - } while (!arch_atomic_try_cmpxchg(v, &c, dec)); 1597 + } while (!raw_atomic_try_cmpxchg(v, &c, dec)); 1606 1598 1607 1599 return dec; 1608 1600 } 1609 - #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive 1610 1601 #endif 1611 1602 1612 1603 #ifdef CONFIG_GENERIC_ATOMIC64 1613 1604 #include <asm-generic/atomic64.h> 1614 1605 #endif 1615 1606 1616 - #ifndef arch_atomic64_read_acquire 1607 + #define raw_atomic64_read arch_atomic64_read 1608 + 1609 + #if defined(arch_atomic64_read_acquire) 1610 + #define raw_atomic64_read_acquire arch_atomic64_read_acquire 1611 + #elif defined(arch_atomic64_read) 1612 + #define raw_atomic64_read_acquire arch_atomic64_read 1613 + #else 1617 1614 static __always_inline s64 1618 - arch_atomic64_read_acquire(const atomic64_t *v) 1615 + raw_atomic64_read_acquire(const atomic64_t *v) 1619 1616 { 1620 1617 s64 ret; 1621 1618 1622 1619 if (__native_word(atomic64_t)) { 1623 1620 ret = smp_load_acquire(&(v)->counter); 1624 1621 } else { 1625 - ret = arch_atomic64_read(v); 1622 + ret = raw_atomic64_read(v); 1626 1623 __atomic_acquire_fence(); 1627 1624 } 1628 1625 1629 1626 return ret; 1630 1627 } 1631 - #define arch_atomic64_read_acquire arch_atomic64_read_acquire 1632 1628 #endif 1633 1629 1634 - #ifndef arch_atomic64_set_release 1630 + #define raw_atomic64_set arch_atomic64_set 1631 + 1632 + #if defined(arch_atomic64_set_release) 1633 + #define raw_atomic64_set_release arch_atomic64_set_release 1634 + #elif defined(arch_atomic64_set) 1635 + #define raw_atomic64_set_release arch_atomic64_set 1636 + #else 1635 1637 static __always_inline void 1636 - arch_atomic64_set_release(atomic64_t *v, s64 i) 1638 + raw_atomic64_set_release(atomic64_t *v, s64 i) 1637 1639 { 1638 1640 if (__native_word(atomic64_t)) { 1639 1641 smp_store_release(&(v)->counter, i); 1640 1642 } else { 1641 1643 __atomic_release_fence(); 1642 - arch_atomic64_set(v, i); 1644 + raw_atomic64_set(v, i); 1643 1645 } 1644 1646 } 1645 - #define arch_atomic64_set_release arch_atomic64_set_release 1646 1647 #endif 1647 1648 1648 - #ifndef arch_atomic64_add_return_relaxed 1649 - #define arch_atomic64_add_return_acquire arch_atomic64_add_return 1650 - #define arch_atomic64_add_return_release arch_atomic64_add_return 1651 - #define arch_atomic64_add_return_relaxed arch_atomic64_add_return 1652 - #else /* arch_atomic64_add_return_relaxed */ 1649 + #define raw_atomic64_add arch_atomic64_add 1653 1650 1654 - #ifndef arch_atomic64_add_return_acquire 1651 + #if defined(arch_atomic64_add_return) 1652 + #define raw_atomic64_add_return arch_atomic64_add_return 1653 + #elif defined(arch_atomic64_add_return_relaxed) 1655 1654 static __always_inline s64 1656 - arch_atomic64_add_return_acquire(s64 i, atomic64_t *v) 1657 - { 1658 - s64 ret = arch_atomic64_add_return_relaxed(i, v); 1659 - __atomic_acquire_fence(); 1660 - return ret; 1661 - } 1662 - #define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire 1663 - #endif 1664 - 1665 - #ifndef arch_atomic64_add_return_release 1666 - static __always_inline s64 1667 - arch_atomic64_add_return_release(s64 i, atomic64_t *v) 1668 - { 1669 - __atomic_release_fence(); 1670 - return arch_atomic64_add_return_relaxed(i, v); 1671 - } 1672 - #define arch_atomic64_add_return_release arch_atomic64_add_return_release 1673 - #endif 1674 - 1675 - #ifndef arch_atomic64_add_return 1676 - static __always_inline s64 1677 - arch_atomic64_add_return(s64 i, atomic64_t *v) 1655 + raw_atomic64_add_return(s64 i, atomic64_t *v) 1678 1656 { 1679 1657 s64 ret; 1680 1658 __atomic_pre_full_fence(); ··· 1729 1613 __atomic_post_full_fence(); 1730 1614 return ret; 1731 1615 } 1732 - #define arch_atomic64_add_return arch_atomic64_add_return 1616 + #else 1617 + #error "Unable to define raw_atomic64_add_return" 1733 1618 #endif 1734 1619 1735 - #endif /* arch_atomic64_add_return_relaxed */ 1736 - 1737 - #ifndef arch_atomic64_fetch_add_relaxed 1738 - #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add 1739 - #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add 1740 - #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add 1741 - #else /* arch_atomic64_fetch_add_relaxed */ 1742 - 1743 - #ifndef arch_atomic64_fetch_add_acquire 1620 + #if defined(arch_atomic64_add_return_acquire) 1621 + #define raw_atomic64_add_return_acquire arch_atomic64_add_return_acquire 1622 + #elif defined(arch_atomic64_add_return_relaxed) 1744 1623 static __always_inline s64 1745 - arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 1624 + raw_atomic64_add_return_acquire(s64 i, atomic64_t *v) 1746 1625 { 1747 - s64 ret = arch_atomic64_fetch_add_relaxed(i, v); 1626 + s64 ret = arch_atomic64_add_return_relaxed(i, v); 1748 1627 __atomic_acquire_fence(); 1749 1628 return ret; 1750 1629 } 1751 - #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire 1630 + #elif defined(arch_atomic64_add_return) 1631 + #define raw_atomic64_add_return_acquire arch_atomic64_add_return 1632 + #else 1633 + #error "Unable to define raw_atomic64_add_return_acquire" 1752 1634 #endif 1753 1635 1754 - #ifndef arch_atomic64_fetch_add_release 1636 + #if defined(arch_atomic64_add_return_release) 1637 + #define raw_atomic64_add_return_release arch_atomic64_add_return_release 1638 + #elif defined(arch_atomic64_add_return_relaxed) 1755 1639 static __always_inline s64 1756 - arch_atomic64_fetch_add_release(s64 i, atomic64_t *v) 1640 + raw_atomic64_add_return_release(s64 i, atomic64_t *v) 1757 1641 { 1758 1642 __atomic_release_fence(); 1759 - return arch_atomic64_fetch_add_relaxed(i, v); 1643 + return arch_atomic64_add_return_relaxed(i, v); 1760 1644 } 1761 - #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release 1645 + #elif defined(arch_atomic64_add_return) 1646 + #define raw_atomic64_add_return_release arch_atomic64_add_return 1647 + #else 1648 + #error "Unable to define raw_atomic64_add_return_release" 1762 1649 #endif 1763 1650 1764 - #ifndef arch_atomic64_fetch_add 1651 + #if defined(arch_atomic64_add_return_relaxed) 1652 + #define raw_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 1653 + #elif defined(arch_atomic64_add_return) 1654 + #define raw_atomic64_add_return_relaxed arch_atomic64_add_return 1655 + #else 1656 + #error "Unable to define raw_atomic64_add_return_relaxed" 1657 + #endif 1658 + 1659 + #if defined(arch_atomic64_fetch_add) 1660 + #define raw_atomic64_fetch_add arch_atomic64_fetch_add 1661 + #elif defined(arch_atomic64_fetch_add_relaxed) 1765 1662 static __always_inline s64 1766 - arch_atomic64_fetch_add(s64 i, atomic64_t *v) 1663 + raw_atomic64_fetch_add(s64 i, atomic64_t *v) 1767 1664 { 1768 1665 s64 ret; 1769 1666 __atomic_pre_full_fence(); ··· 1784 1655 __atomic_post_full_fence(); 1785 1656 return ret; 1786 1657 } 1787 - #define arch_atomic64_fetch_add arch_atomic64_fetch_add 1658 + #else 1659 + #error "Unable to define raw_atomic64_fetch_add" 1788 1660 #endif 1789 1661 1790 - #endif /* arch_atomic64_fetch_add_relaxed */ 1791 - 1792 - #ifndef arch_atomic64_sub_return_relaxed 1793 - #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return 1794 - #define arch_atomic64_sub_return_release arch_atomic64_sub_return 1795 - #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return 1796 - #else /* arch_atomic64_sub_return_relaxed */ 1797 - 1798 - #ifndef arch_atomic64_sub_return_acquire 1662 + #if defined(arch_atomic64_fetch_add_acquire) 1663 + #define raw_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire 1664 + #elif defined(arch_atomic64_fetch_add_relaxed) 1799 1665 static __always_inline s64 1800 - arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v) 1666 + raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 1801 1667 { 1802 - s64 ret = arch_atomic64_sub_return_relaxed(i, v); 1668 + s64 ret = arch_atomic64_fetch_add_relaxed(i, v); 1803 1669 __atomic_acquire_fence(); 1804 1670 return ret; 1805 1671 } 1806 - #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire 1672 + #elif defined(arch_atomic64_fetch_add) 1673 + #define raw_atomic64_fetch_add_acquire arch_atomic64_fetch_add 1674 + #else 1675 + #error "Unable to define raw_atomic64_fetch_add_acquire" 1807 1676 #endif 1808 1677 1809 - #ifndef arch_atomic64_sub_return_release 1678 + #if defined(arch_atomic64_fetch_add_release) 1679 + #define raw_atomic64_fetch_add_release arch_atomic64_fetch_add_release 1680 + #elif defined(arch_atomic64_fetch_add_relaxed) 1810 1681 static __always_inline s64 1811 - arch_atomic64_sub_return_release(s64 i, atomic64_t *v) 1682 + raw_atomic64_fetch_add_release(s64 i, atomic64_t *v) 1812 1683 { 1813 1684 __atomic_release_fence(); 1814 - return arch_atomic64_sub_return_relaxed(i, v); 1685 + return arch_atomic64_fetch_add_relaxed(i, v); 1815 1686 } 1816 - #define arch_atomic64_sub_return_release arch_atomic64_sub_return_release 1687 + #elif defined(arch_atomic64_fetch_add) 1688 + #define raw_atomic64_fetch_add_release arch_atomic64_fetch_add 1689 + #else 1690 + #error "Unable to define raw_atomic64_fetch_add_release" 1817 1691 #endif 1818 1692 1819 - #ifndef arch_atomic64_sub_return 1693 + #if defined(arch_atomic64_fetch_add_relaxed) 1694 + #define raw_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 1695 + #elif defined(arch_atomic64_fetch_add) 1696 + #define raw_atomic64_fetch_add_relaxed arch_atomic64_fetch_add 1697 + #else 1698 + #error "Unable to define raw_atomic64_fetch_add_relaxed" 1699 + #endif 1700 + 1701 + #define raw_atomic64_sub arch_atomic64_sub 1702 + 1703 + #if defined(arch_atomic64_sub_return) 1704 + #define raw_atomic64_sub_return arch_atomic64_sub_return 1705 + #elif defined(arch_atomic64_sub_return_relaxed) 1820 1706 static __always_inline s64 1821 - arch_atomic64_sub_return(s64 i, atomic64_t *v) 1707 + raw_atomic64_sub_return(s64 i, atomic64_t *v) 1822 1708 { 1823 1709 s64 ret; 1824 1710 __atomic_pre_full_fence(); ··· 1841 1697 __atomic_post_full_fence(); 1842 1698 return ret; 1843 1699 } 1844 - #define arch_atomic64_sub_return arch_atomic64_sub_return 1700 + #else 1701 + #error "Unable to define raw_atomic64_sub_return" 1845 1702 #endif 1846 1703 1847 - #endif /* arch_atomic64_sub_return_relaxed */ 1848 - 1849 - #ifndef arch_atomic64_fetch_sub_relaxed 1850 - #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub 1851 - #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub 1852 - #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub 1853 - #else /* arch_atomic64_fetch_sub_relaxed */ 1854 - 1855 - #ifndef arch_atomic64_fetch_sub_acquire 1704 + #if defined(arch_atomic64_sub_return_acquire) 1705 + #define raw_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire 1706 + #elif defined(arch_atomic64_sub_return_relaxed) 1856 1707 static __always_inline s64 1857 - arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 1708 + raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v) 1858 1709 { 1859 - s64 ret = arch_atomic64_fetch_sub_relaxed(i, v); 1710 + s64 ret = arch_atomic64_sub_return_relaxed(i, v); 1860 1711 __atomic_acquire_fence(); 1861 1712 return ret; 1862 1713 } 1863 - #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire 1714 + #elif defined(arch_atomic64_sub_return) 1715 + #define raw_atomic64_sub_return_acquire arch_atomic64_sub_return 1716 + #else 1717 + #error "Unable to define raw_atomic64_sub_return_acquire" 1864 1718 #endif 1865 1719 1866 - #ifndef arch_atomic64_fetch_sub_release 1720 + #if defined(arch_atomic64_sub_return_release) 1721 + #define raw_atomic64_sub_return_release arch_atomic64_sub_return_release 1722 + #elif defined(arch_atomic64_sub_return_relaxed) 1867 1723 static __always_inline s64 1868 - arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v) 1724 + raw_atomic64_sub_return_release(s64 i, atomic64_t *v) 1869 1725 { 1870 1726 __atomic_release_fence(); 1871 - return arch_atomic64_fetch_sub_relaxed(i, v); 1727 + return arch_atomic64_sub_return_relaxed(i, v); 1872 1728 } 1873 - #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release 1729 + #elif defined(arch_atomic64_sub_return) 1730 + #define raw_atomic64_sub_return_release arch_atomic64_sub_return 1731 + #else 1732 + #error "Unable to define raw_atomic64_sub_return_release" 1874 1733 #endif 1875 1734 1876 - #ifndef arch_atomic64_fetch_sub 1735 + #if defined(arch_atomic64_sub_return_relaxed) 1736 + #define raw_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 1737 + #elif defined(arch_atomic64_sub_return) 1738 + #define raw_atomic64_sub_return_relaxed arch_atomic64_sub_return 1739 + #else 1740 + #error "Unable to define raw_atomic64_sub_return_relaxed" 1741 + #endif 1742 + 1743 + #if defined(arch_atomic64_fetch_sub) 1744 + #define raw_atomic64_fetch_sub arch_atomic64_fetch_sub 1745 + #elif defined(arch_atomic64_fetch_sub_relaxed) 1877 1746 static __always_inline s64 1878 - arch_atomic64_fetch_sub(s64 i, atomic64_t *v) 1747 + raw_atomic64_fetch_sub(s64 i, atomic64_t *v) 1879 1748 { 1880 1749 s64 ret; 1881 1750 __atomic_pre_full_fence(); ··· 1896 1739 __atomic_post_full_fence(); 1897 1740 return ret; 1898 1741 } 1899 - #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub 1742 + #else 1743 + #error "Unable to define raw_atomic64_fetch_sub" 1900 1744 #endif 1901 1745 1902 - #endif /* arch_atomic64_fetch_sub_relaxed */ 1903 - 1904 - #ifndef arch_atomic64_inc 1905 - static __always_inline void 1906 - arch_atomic64_inc(atomic64_t *v) 1907 - { 1908 - arch_atomic64_add(1, v); 1909 - } 1910 - #define arch_atomic64_inc arch_atomic64_inc 1911 - #endif 1912 - 1913 - #ifndef arch_atomic64_inc_return_relaxed 1914 - #ifdef arch_atomic64_inc_return 1915 - #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return 1916 - #define arch_atomic64_inc_return_release arch_atomic64_inc_return 1917 - #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return 1918 - #endif /* arch_atomic64_inc_return */ 1919 - 1920 - #ifndef arch_atomic64_inc_return 1746 + #if defined(arch_atomic64_fetch_sub_acquire) 1747 + #define raw_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire 1748 + #elif defined(arch_atomic64_fetch_sub_relaxed) 1921 1749 static __always_inline s64 1922 - arch_atomic64_inc_return(atomic64_t *v) 1750 + raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 1923 1751 { 1924 - return arch_atomic64_add_return(1, v); 1925 - } 1926 - #define arch_atomic64_inc_return arch_atomic64_inc_return 1927 - #endif 1928 - 1929 - #ifndef arch_atomic64_inc_return_acquire 1930 - static __always_inline s64 1931 - arch_atomic64_inc_return_acquire(atomic64_t *v) 1932 - { 1933 - return arch_atomic64_add_return_acquire(1, v); 1934 - } 1935 - #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire 1936 - #endif 1937 - 1938 - #ifndef arch_atomic64_inc_return_release 1939 - static __always_inline s64 1940 - arch_atomic64_inc_return_release(atomic64_t *v) 1941 - { 1942 - return arch_atomic64_add_return_release(1, v); 1943 - } 1944 - #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release 1945 - #endif 1946 - 1947 - #ifndef arch_atomic64_inc_return_relaxed 1948 - static __always_inline s64 1949 - arch_atomic64_inc_return_relaxed(atomic64_t *v) 1950 - { 1951 - return arch_atomic64_add_return_relaxed(1, v); 1952 - } 1953 - #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed 1954 - #endif 1955 - 1956 - #else /* arch_atomic64_inc_return_relaxed */ 1957 - 1958 - #ifndef arch_atomic64_inc_return_acquire 1959 - static __always_inline s64 1960 - arch_atomic64_inc_return_acquire(atomic64_t *v) 1961 - { 1962 - s64 ret = arch_atomic64_inc_return_relaxed(v); 1752 + s64 ret = arch_atomic64_fetch_sub_relaxed(i, v); 1963 1753 __atomic_acquire_fence(); 1964 1754 return ret; 1965 1755 } 1966 - #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire 1756 + #elif defined(arch_atomic64_fetch_sub) 1757 + #define raw_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub 1758 + #else 1759 + #error "Unable to define raw_atomic64_fetch_sub_acquire" 1967 1760 #endif 1968 1761 1969 - #ifndef arch_atomic64_inc_return_release 1762 + #if defined(arch_atomic64_fetch_sub_release) 1763 + #define raw_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release 1764 + #elif defined(arch_atomic64_fetch_sub_relaxed) 1970 1765 static __always_inline s64 1971 - arch_atomic64_inc_return_release(atomic64_t *v) 1766 + raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v) 1972 1767 { 1973 1768 __atomic_release_fence(); 1974 - return arch_atomic64_inc_return_relaxed(v); 1769 + return arch_atomic64_fetch_sub_relaxed(i, v); 1975 1770 } 1976 - #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release 1771 + #elif defined(arch_atomic64_fetch_sub) 1772 + #define raw_atomic64_fetch_sub_release arch_atomic64_fetch_sub 1773 + #else 1774 + #error "Unable to define raw_atomic64_fetch_sub_release" 1977 1775 #endif 1978 1776 1979 - #ifndef arch_atomic64_inc_return 1777 + #if defined(arch_atomic64_fetch_sub_relaxed) 1778 + #define raw_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 1779 + #elif defined(arch_atomic64_fetch_sub) 1780 + #define raw_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub 1781 + #else 1782 + #error "Unable to define raw_atomic64_fetch_sub_relaxed" 1783 + #endif 1784 + 1785 + #if defined(arch_atomic64_inc) 1786 + #define raw_atomic64_inc arch_atomic64_inc 1787 + #else 1788 + static __always_inline void 1789 + raw_atomic64_inc(atomic64_t *v) 1790 + { 1791 + raw_atomic64_add(1, v); 1792 + } 1793 + #endif 1794 + 1795 + #if defined(arch_atomic64_inc_return) 1796 + #define raw_atomic64_inc_return arch_atomic64_inc_return 1797 + #elif defined(arch_atomic64_inc_return_relaxed) 1980 1798 static __always_inline s64 1981 - arch_atomic64_inc_return(atomic64_t *v) 1799 + raw_atomic64_inc_return(atomic64_t *v) 1982 1800 { 1983 1801 s64 ret; 1984 1802 __atomic_pre_full_fence(); ··· 1961 1829 __atomic_post_full_fence(); 1962 1830 return ret; 1963 1831 } 1964 - #define arch_atomic64_inc_return arch_atomic64_inc_return 1965 - #endif 1966 - 1967 - #endif /* arch_atomic64_inc_return_relaxed */ 1968 - 1969 - #ifndef arch_atomic64_fetch_inc_relaxed 1970 - #ifdef arch_atomic64_fetch_inc 1971 - #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc 1972 - #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc 1973 - #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc 1974 - #endif /* arch_atomic64_fetch_inc */ 1975 - 1976 - #ifndef arch_atomic64_fetch_inc 1832 + #else 1977 1833 static __always_inline s64 1978 - arch_atomic64_fetch_inc(atomic64_t *v) 1834 + raw_atomic64_inc_return(atomic64_t *v) 1979 1835 { 1980 - return arch_atomic64_fetch_add(1, v); 1836 + return raw_atomic64_add_return(1, v); 1981 1837 } 1982 - #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc 1983 1838 #endif 1984 1839 1985 - #ifndef arch_atomic64_fetch_inc_acquire 1840 + #if defined(arch_atomic64_inc_return_acquire) 1841 + #define raw_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire 1842 + #elif defined(arch_atomic64_inc_return_relaxed) 1986 1843 static __always_inline s64 1987 - arch_atomic64_fetch_inc_acquire(atomic64_t *v) 1844 + raw_atomic64_inc_return_acquire(atomic64_t *v) 1988 1845 { 1989 - return arch_atomic64_fetch_add_acquire(1, v); 1990 - } 1991 - #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire 1992 - #endif 1993 - 1994 - #ifndef arch_atomic64_fetch_inc_release 1995 - static __always_inline s64 1996 - arch_atomic64_fetch_inc_release(atomic64_t *v) 1997 - { 1998 - return arch_atomic64_fetch_add_release(1, v); 1999 - } 2000 - #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release 2001 - #endif 2002 - 2003 - #ifndef arch_atomic64_fetch_inc_relaxed 2004 - static __always_inline s64 2005 - arch_atomic64_fetch_inc_relaxed(atomic64_t *v) 2006 - { 2007 - return arch_atomic64_fetch_add_relaxed(1, v); 2008 - } 2009 - #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed 2010 - #endif 2011 - 2012 - #else /* arch_atomic64_fetch_inc_relaxed */ 2013 - 2014 - #ifndef arch_atomic64_fetch_inc_acquire 2015 - static __always_inline s64 2016 - arch_atomic64_fetch_inc_acquire(atomic64_t *v) 2017 - { 2018 - s64 ret = arch_atomic64_fetch_inc_relaxed(v); 1846 + s64 ret = arch_atomic64_inc_return_relaxed(v); 2019 1847 __atomic_acquire_fence(); 2020 1848 return ret; 2021 1849 } 2022 - #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire 1850 + #elif defined(arch_atomic64_inc_return) 1851 + #define raw_atomic64_inc_return_acquire arch_atomic64_inc_return 1852 + #else 1853 + static __always_inline s64 1854 + raw_atomic64_inc_return_acquire(atomic64_t *v) 1855 + { 1856 + return raw_atomic64_add_return_acquire(1, v); 1857 + } 2023 1858 #endif 2024 1859 2025 - #ifndef arch_atomic64_fetch_inc_release 1860 + #if defined(arch_atomic64_inc_return_release) 1861 + #define raw_atomic64_inc_return_release arch_atomic64_inc_return_release 1862 + #elif defined(arch_atomic64_inc_return_relaxed) 2026 1863 static __always_inline s64 2027 - arch_atomic64_fetch_inc_release(atomic64_t *v) 1864 + raw_atomic64_inc_return_release(atomic64_t *v) 2028 1865 { 2029 1866 __atomic_release_fence(); 2030 - return arch_atomic64_fetch_inc_relaxed(v); 1867 + return arch_atomic64_inc_return_relaxed(v); 2031 1868 } 2032 - #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release 1869 + #elif defined(arch_atomic64_inc_return) 1870 + #define raw_atomic64_inc_return_release arch_atomic64_inc_return 1871 + #else 1872 + static __always_inline s64 1873 + raw_atomic64_inc_return_release(atomic64_t *v) 1874 + { 1875 + return raw_atomic64_add_return_release(1, v); 1876 + } 2033 1877 #endif 2034 1878 2035 - #ifndef arch_atomic64_fetch_inc 1879 + #if defined(arch_atomic64_inc_return_relaxed) 1880 + #define raw_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed 1881 + #elif defined(arch_atomic64_inc_return) 1882 + #define raw_atomic64_inc_return_relaxed arch_atomic64_inc_return 1883 + #else 2036 1884 static __always_inline s64 2037 - arch_atomic64_fetch_inc(atomic64_t *v) 1885 + raw_atomic64_inc_return_relaxed(atomic64_t *v) 1886 + { 1887 + return raw_atomic64_add_return_relaxed(1, v); 1888 + } 1889 + #endif 1890 + 1891 + #if defined(arch_atomic64_fetch_inc) 1892 + #define raw_atomic64_fetch_inc arch_atomic64_fetch_inc 1893 + #elif defined(arch_atomic64_fetch_inc_relaxed) 1894 + static __always_inline s64 1895 + raw_atomic64_fetch_inc(atomic64_t *v) 2038 1896 { 2039 1897 s64 ret; 2040 1898 __atomic_pre_full_fence(); ··· 2032 1910 __atomic_post_full_fence(); 2033 1911 return ret; 2034 1912 } 2035 - #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc 2036 - #endif 2037 - 2038 - #endif /* arch_atomic64_fetch_inc_relaxed */ 2039 - 2040 - #ifndef arch_atomic64_dec 2041 - static __always_inline void 2042 - arch_atomic64_dec(atomic64_t *v) 2043 - { 2044 - arch_atomic64_sub(1, v); 2045 - } 2046 - #define arch_atomic64_dec arch_atomic64_dec 2047 - #endif 2048 - 2049 - #ifndef arch_atomic64_dec_return_relaxed 2050 - #ifdef arch_atomic64_dec_return 2051 - #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return 2052 - #define arch_atomic64_dec_return_release arch_atomic64_dec_return 2053 - #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return 2054 - #endif /* arch_atomic64_dec_return */ 2055 - 2056 - #ifndef arch_atomic64_dec_return 1913 + #else 2057 1914 static __always_inline s64 2058 - arch_atomic64_dec_return(atomic64_t *v) 1915 + raw_atomic64_fetch_inc(atomic64_t *v) 2059 1916 { 2060 - return arch_atomic64_sub_return(1, v); 1917 + return raw_atomic64_fetch_add(1, v); 2061 1918 } 2062 - #define arch_atomic64_dec_return arch_atomic64_dec_return 2063 1919 #endif 2064 1920 2065 - #ifndef arch_atomic64_dec_return_acquire 1921 + #if defined(arch_atomic64_fetch_inc_acquire) 1922 + #define raw_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire 1923 + #elif defined(arch_atomic64_fetch_inc_relaxed) 2066 1924 static __always_inline s64 2067 - arch_atomic64_dec_return_acquire(atomic64_t *v) 1925 + raw_atomic64_fetch_inc_acquire(atomic64_t *v) 2068 1926 { 2069 - return arch_atomic64_sub_return_acquire(1, v); 2070 - } 2071 - #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire 2072 - #endif 2073 - 2074 - #ifndef arch_atomic64_dec_return_release 2075 - static __always_inline s64 2076 - arch_atomic64_dec_return_release(atomic64_t *v) 2077 - { 2078 - return arch_atomic64_sub_return_release(1, v); 2079 - } 2080 - #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release 2081 - #endif 2082 - 2083 - #ifndef arch_atomic64_dec_return_relaxed 2084 - static __always_inline s64 2085 - arch_atomic64_dec_return_relaxed(atomic64_t *v) 2086 - { 2087 - return arch_atomic64_sub_return_relaxed(1, v); 2088 - } 2089 - #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed 2090 - #endif 2091 - 2092 - #else /* arch_atomic64_dec_return_relaxed */ 2093 - 2094 - #ifndef arch_atomic64_dec_return_acquire 2095 - static __always_inline s64 2096 - arch_atomic64_dec_return_acquire(atomic64_t *v) 2097 - { 2098 - s64 ret = arch_atomic64_dec_return_relaxed(v); 1927 + s64 ret = arch_atomic64_fetch_inc_relaxed(v); 2099 1928 __atomic_acquire_fence(); 2100 1929 return ret; 2101 1930 } 2102 - #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire 1931 + #elif defined(arch_atomic64_fetch_inc) 1932 + #define raw_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc 1933 + #else 1934 + static __always_inline s64 1935 + raw_atomic64_fetch_inc_acquire(atomic64_t *v) 1936 + { 1937 + return raw_atomic64_fetch_add_acquire(1, v); 1938 + } 2103 1939 #endif 2104 1940 2105 - #ifndef arch_atomic64_dec_return_release 1941 + #if defined(arch_atomic64_fetch_inc_release) 1942 + #define raw_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release 1943 + #elif defined(arch_atomic64_fetch_inc_relaxed) 2106 1944 static __always_inline s64 2107 - arch_atomic64_dec_return_release(atomic64_t *v) 1945 + raw_atomic64_fetch_inc_release(atomic64_t *v) 2108 1946 { 2109 1947 __atomic_release_fence(); 2110 - return arch_atomic64_dec_return_relaxed(v); 1948 + return arch_atomic64_fetch_inc_relaxed(v); 2111 1949 } 2112 - #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release 1950 + #elif defined(arch_atomic64_fetch_inc) 1951 + #define raw_atomic64_fetch_inc_release arch_atomic64_fetch_inc 1952 + #else 1953 + static __always_inline s64 1954 + raw_atomic64_fetch_inc_release(atomic64_t *v) 1955 + { 1956 + return raw_atomic64_fetch_add_release(1, v); 1957 + } 2113 1958 #endif 2114 1959 2115 - #ifndef arch_atomic64_dec_return 1960 + #if defined(arch_atomic64_fetch_inc_relaxed) 1961 + #define raw_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed 1962 + #elif defined(arch_atomic64_fetch_inc) 1963 + #define raw_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc 1964 + #else 2116 1965 static __always_inline s64 2117 - arch_atomic64_dec_return(atomic64_t *v) 1966 + raw_atomic64_fetch_inc_relaxed(atomic64_t *v) 1967 + { 1968 + return raw_atomic64_fetch_add_relaxed(1, v); 1969 + } 1970 + #endif 1971 + 1972 + #if defined(arch_atomic64_dec) 1973 + #define raw_atomic64_dec arch_atomic64_dec 1974 + #else 1975 + static __always_inline void 1976 + raw_atomic64_dec(atomic64_t *v) 1977 + { 1978 + raw_atomic64_sub(1, v); 1979 + } 1980 + #endif 1981 + 1982 + #if defined(arch_atomic64_dec_return) 1983 + #define raw_atomic64_dec_return arch_atomic64_dec_return 1984 + #elif defined(arch_atomic64_dec_return_relaxed) 1985 + static __always_inline s64 1986 + raw_atomic64_dec_return(atomic64_t *v) 2118 1987 { 2119 1988 s64 ret; 2120 1989 __atomic_pre_full_fence(); ··· 2113 2000 __atomic_post_full_fence(); 2114 2001 return ret; 2115 2002 } 2116 - #define arch_atomic64_dec_return arch_atomic64_dec_return 2117 - #endif 2118 - 2119 - #endif /* arch_atomic64_dec_return_relaxed */ 2120 - 2121 - #ifndef arch_atomic64_fetch_dec_relaxed 2122 - #ifdef arch_atomic64_fetch_dec 2123 - #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec 2124 - #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec 2125 - #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec 2126 - #endif /* arch_atomic64_fetch_dec */ 2127 - 2128 - #ifndef arch_atomic64_fetch_dec 2003 + #else 2129 2004 static __always_inline s64 2130 - arch_atomic64_fetch_dec(atomic64_t *v) 2005 + raw_atomic64_dec_return(atomic64_t *v) 2131 2006 { 2132 - return arch_atomic64_fetch_sub(1, v); 2007 + return raw_atomic64_sub_return(1, v); 2133 2008 } 2134 - #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec 2135 2009 #endif 2136 2010 2137 - #ifndef arch_atomic64_fetch_dec_acquire 2011 + #if defined(arch_atomic64_dec_return_acquire) 2012 + #define raw_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire 2013 + #elif defined(arch_atomic64_dec_return_relaxed) 2138 2014 static __always_inline s64 2139 - arch_atomic64_fetch_dec_acquire(atomic64_t *v) 2015 + raw_atomic64_dec_return_acquire(atomic64_t *v) 2140 2016 { 2141 - return arch_atomic64_fetch_sub_acquire(1, v); 2142 - } 2143 - #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire 2144 - #endif 2145 - 2146 - #ifndef arch_atomic64_fetch_dec_release 2147 - static __always_inline s64 2148 - arch_atomic64_fetch_dec_release(atomic64_t *v) 2149 - { 2150 - return arch_atomic64_fetch_sub_release(1, v); 2151 - } 2152 - #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release 2153 - #endif 2154 - 2155 - #ifndef arch_atomic64_fetch_dec_relaxed 2156 - static __always_inline s64 2157 - arch_atomic64_fetch_dec_relaxed(atomic64_t *v) 2158 - { 2159 - return arch_atomic64_fetch_sub_relaxed(1, v); 2160 - } 2161 - #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed 2162 - #endif 2163 - 2164 - #else /* arch_atomic64_fetch_dec_relaxed */ 2165 - 2166 - #ifndef arch_atomic64_fetch_dec_acquire 2167 - static __always_inline s64 2168 - arch_atomic64_fetch_dec_acquire(atomic64_t *v) 2169 - { 2170 - s64 ret = arch_atomic64_fetch_dec_relaxed(v); 2017 + s64 ret = arch_atomic64_dec_return_relaxed(v); 2171 2018 __atomic_acquire_fence(); 2172 2019 return ret; 2173 2020 } 2174 - #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire 2021 + #elif defined(arch_atomic64_dec_return) 2022 + #define raw_atomic64_dec_return_acquire arch_atomic64_dec_return 2023 + #else 2024 + static __always_inline s64 2025 + raw_atomic64_dec_return_acquire(atomic64_t *v) 2026 + { 2027 + return raw_atomic64_sub_return_acquire(1, v); 2028 + } 2175 2029 #endif 2176 2030 2177 - #ifndef arch_atomic64_fetch_dec_release 2031 + #if defined(arch_atomic64_dec_return_release) 2032 + #define raw_atomic64_dec_return_release arch_atomic64_dec_return_release 2033 + #elif defined(arch_atomic64_dec_return_relaxed) 2178 2034 static __always_inline s64 2179 - arch_atomic64_fetch_dec_release(atomic64_t *v) 2035 + raw_atomic64_dec_return_release(atomic64_t *v) 2180 2036 { 2181 2037 __atomic_release_fence(); 2182 - return arch_atomic64_fetch_dec_relaxed(v); 2038 + return arch_atomic64_dec_return_relaxed(v); 2183 2039 } 2184 - #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release 2040 + #elif defined(arch_atomic64_dec_return) 2041 + #define raw_atomic64_dec_return_release arch_atomic64_dec_return 2042 + #else 2043 + static __always_inline s64 2044 + raw_atomic64_dec_return_release(atomic64_t *v) 2045 + { 2046 + return raw_atomic64_sub_return_release(1, v); 2047 + } 2185 2048 #endif 2186 2049 2187 - #ifndef arch_atomic64_fetch_dec 2050 + #if defined(arch_atomic64_dec_return_relaxed) 2051 + #define raw_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed 2052 + #elif defined(arch_atomic64_dec_return) 2053 + #define raw_atomic64_dec_return_relaxed arch_atomic64_dec_return 2054 + #else 2188 2055 static __always_inline s64 2189 - arch_atomic64_fetch_dec(atomic64_t *v) 2056 + raw_atomic64_dec_return_relaxed(atomic64_t *v) 2057 + { 2058 + return raw_atomic64_sub_return_relaxed(1, v); 2059 + } 2060 + #endif 2061 + 2062 + #if defined(arch_atomic64_fetch_dec) 2063 + #define raw_atomic64_fetch_dec arch_atomic64_fetch_dec 2064 + #elif defined(arch_atomic64_fetch_dec_relaxed) 2065 + static __always_inline s64 2066 + raw_atomic64_fetch_dec(atomic64_t *v) 2190 2067 { 2191 2068 s64 ret; 2192 2069 __atomic_pre_full_fence(); ··· 2184 2081 __atomic_post_full_fence(); 2185 2082 return ret; 2186 2083 } 2187 - #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec 2084 + #else 2085 + static __always_inline s64 2086 + raw_atomic64_fetch_dec(atomic64_t *v) 2087 + { 2088 + return raw_atomic64_fetch_sub(1, v); 2089 + } 2188 2090 #endif 2189 2091 2190 - #endif /* arch_atomic64_fetch_dec_relaxed */ 2191 - 2192 - #ifndef arch_atomic64_fetch_and_relaxed 2193 - #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and 2194 - #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and 2195 - #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and 2196 - #else /* arch_atomic64_fetch_and_relaxed */ 2197 - 2198 - #ifndef arch_atomic64_fetch_and_acquire 2092 + #if defined(arch_atomic64_fetch_dec_acquire) 2093 + #define raw_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire 2094 + #elif defined(arch_atomic64_fetch_dec_relaxed) 2199 2095 static __always_inline s64 2200 - arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 2096 + raw_atomic64_fetch_dec_acquire(atomic64_t *v) 2201 2097 { 2202 - s64 ret = arch_atomic64_fetch_and_relaxed(i, v); 2098 + s64 ret = arch_atomic64_fetch_dec_relaxed(v); 2203 2099 __atomic_acquire_fence(); 2204 2100 return ret; 2205 2101 } 2206 - #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire 2102 + #elif defined(arch_atomic64_fetch_dec) 2103 + #define raw_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec 2104 + #else 2105 + static __always_inline s64 2106 + raw_atomic64_fetch_dec_acquire(atomic64_t *v) 2107 + { 2108 + return raw_atomic64_fetch_sub_acquire(1, v); 2109 + } 2207 2110 #endif 2208 2111 2209 - #ifndef arch_atomic64_fetch_and_release 2112 + #if defined(arch_atomic64_fetch_dec_release) 2113 + #define raw_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release 2114 + #elif defined(arch_atomic64_fetch_dec_relaxed) 2210 2115 static __always_inline s64 2211 - arch_atomic64_fetch_and_release(s64 i, atomic64_t *v) 2116 + raw_atomic64_fetch_dec_release(atomic64_t *v) 2212 2117 { 2213 2118 __atomic_release_fence(); 2214 - return arch_atomic64_fetch_and_relaxed(i, v); 2119 + return arch_atomic64_fetch_dec_relaxed(v); 2215 2120 } 2216 - #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release 2121 + #elif defined(arch_atomic64_fetch_dec) 2122 + #define raw_atomic64_fetch_dec_release arch_atomic64_fetch_dec 2123 + #else 2124 + static __always_inline s64 2125 + raw_atomic64_fetch_dec_release(atomic64_t *v) 2126 + { 2127 + return raw_atomic64_fetch_sub_release(1, v); 2128 + } 2217 2129 #endif 2218 2130 2219 - #ifndef arch_atomic64_fetch_and 2131 + #if defined(arch_atomic64_fetch_dec_relaxed) 2132 + #define raw_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed 2133 + #elif defined(arch_atomic64_fetch_dec) 2134 + #define raw_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec 2135 + #else 2220 2136 static __always_inline s64 2221 - arch_atomic64_fetch_and(s64 i, atomic64_t *v) 2137 + raw_atomic64_fetch_dec_relaxed(atomic64_t *v) 2138 + { 2139 + return raw_atomic64_fetch_sub_relaxed(1, v); 2140 + } 2141 + #endif 2142 + 2143 + #define raw_atomic64_and arch_atomic64_and 2144 + 2145 + #if defined(arch_atomic64_fetch_and) 2146 + #define raw_atomic64_fetch_and arch_atomic64_fetch_and 2147 + #elif defined(arch_atomic64_fetch_and_relaxed) 2148 + static __always_inline s64 2149 + raw_atomic64_fetch_and(s64 i, atomic64_t *v) 2222 2150 { 2223 2151 s64 ret; 2224 2152 __atomic_pre_full_fence(); ··· 2257 2123 __atomic_post_full_fence(); 2258 2124 return ret; 2259 2125 } 2260 - #define arch_atomic64_fetch_and arch_atomic64_fetch_and 2126 + #else 2127 + #error "Unable to define raw_atomic64_fetch_and" 2261 2128 #endif 2262 2129 2263 - #endif /* arch_atomic64_fetch_and_relaxed */ 2264 - 2265 - #ifndef arch_atomic64_andnot 2266 - static __always_inline void 2267 - arch_atomic64_andnot(s64 i, atomic64_t *v) 2268 - { 2269 - arch_atomic64_and(~i, v); 2270 - } 2271 - #define arch_atomic64_andnot arch_atomic64_andnot 2272 - #endif 2273 - 2274 - #ifndef arch_atomic64_fetch_andnot_relaxed 2275 - #ifdef arch_atomic64_fetch_andnot 2276 - #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot 2277 - #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot 2278 - #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot 2279 - #endif /* arch_atomic64_fetch_andnot */ 2280 - 2281 - #ifndef arch_atomic64_fetch_andnot 2130 + #if defined(arch_atomic64_fetch_and_acquire) 2131 + #define raw_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire 2132 + #elif defined(arch_atomic64_fetch_and_relaxed) 2282 2133 static __always_inline s64 2283 - arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) 2134 + raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 2284 2135 { 2285 - return arch_atomic64_fetch_and(~i, v); 2286 - } 2287 - #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot 2288 - #endif 2289 - 2290 - #ifndef arch_atomic64_fetch_andnot_acquire 2291 - static __always_inline s64 2292 - arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 2293 - { 2294 - return arch_atomic64_fetch_and_acquire(~i, v); 2295 - } 2296 - #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire 2297 - #endif 2298 - 2299 - #ifndef arch_atomic64_fetch_andnot_release 2300 - static __always_inline s64 2301 - arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 2302 - { 2303 - return arch_atomic64_fetch_and_release(~i, v); 2304 - } 2305 - #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release 2306 - #endif 2307 - 2308 - #ifndef arch_atomic64_fetch_andnot_relaxed 2309 - static __always_inline s64 2310 - arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 2311 - { 2312 - return arch_atomic64_fetch_and_relaxed(~i, v); 2313 - } 2314 - #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 2315 - #endif 2316 - 2317 - #else /* arch_atomic64_fetch_andnot_relaxed */ 2318 - 2319 - #ifndef arch_atomic64_fetch_andnot_acquire 2320 - static __always_inline s64 2321 - arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 2322 - { 2323 - s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 2136 + s64 ret = arch_atomic64_fetch_and_relaxed(i, v); 2324 2137 __atomic_acquire_fence(); 2325 2138 return ret; 2326 2139 } 2327 - #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire 2140 + #elif defined(arch_atomic64_fetch_and) 2141 + #define raw_atomic64_fetch_and_acquire arch_atomic64_fetch_and 2142 + #else 2143 + #error "Unable to define raw_atomic64_fetch_and_acquire" 2328 2144 #endif 2329 2145 2330 - #ifndef arch_atomic64_fetch_andnot_release 2146 + #if defined(arch_atomic64_fetch_and_release) 2147 + #define raw_atomic64_fetch_and_release arch_atomic64_fetch_and_release 2148 + #elif defined(arch_atomic64_fetch_and_relaxed) 2331 2149 static __always_inline s64 2332 - arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 2150 + raw_atomic64_fetch_and_release(s64 i, atomic64_t *v) 2333 2151 { 2334 2152 __atomic_release_fence(); 2335 - return arch_atomic64_fetch_andnot_relaxed(i, v); 2153 + return arch_atomic64_fetch_and_relaxed(i, v); 2336 2154 } 2337 - #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release 2155 + #elif defined(arch_atomic64_fetch_and) 2156 + #define raw_atomic64_fetch_and_release arch_atomic64_fetch_and 2157 + #else 2158 + #error "Unable to define raw_atomic64_fetch_and_release" 2338 2159 #endif 2339 2160 2340 - #ifndef arch_atomic64_fetch_andnot 2161 + #if defined(arch_atomic64_fetch_and_relaxed) 2162 + #define raw_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 2163 + #elif defined(arch_atomic64_fetch_and) 2164 + #define raw_atomic64_fetch_and_relaxed arch_atomic64_fetch_and 2165 + #else 2166 + #error "Unable to define raw_atomic64_fetch_and_relaxed" 2167 + #endif 2168 + 2169 + #if defined(arch_atomic64_andnot) 2170 + #define raw_atomic64_andnot arch_atomic64_andnot 2171 + #else 2172 + static __always_inline void 2173 + raw_atomic64_andnot(s64 i, atomic64_t *v) 2174 + { 2175 + raw_atomic64_and(~i, v); 2176 + } 2177 + #endif 2178 + 2179 + #if defined(arch_atomic64_fetch_andnot) 2180 + #define raw_atomic64_fetch_andnot arch_atomic64_fetch_andnot 2181 + #elif defined(arch_atomic64_fetch_andnot_relaxed) 2341 2182 static __always_inline s64 2342 - arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) 2183 + raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) 2343 2184 { 2344 2185 s64 ret; 2345 2186 __atomic_pre_full_fence(); ··· 2322 2213 __atomic_post_full_fence(); 2323 2214 return ret; 2324 2215 } 2325 - #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot 2216 + #else 2217 + static __always_inline s64 2218 + raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) 2219 + { 2220 + return raw_atomic64_fetch_and(~i, v); 2221 + } 2326 2222 #endif 2327 2223 2328 - #endif /* arch_atomic64_fetch_andnot_relaxed */ 2329 - 2330 - #ifndef arch_atomic64_fetch_or_relaxed 2331 - #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or 2332 - #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or 2333 - #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or 2334 - #else /* arch_atomic64_fetch_or_relaxed */ 2335 - 2336 - #ifndef arch_atomic64_fetch_or_acquire 2224 + #if defined(arch_atomic64_fetch_andnot_acquire) 2225 + #define raw_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire 2226 + #elif defined(arch_atomic64_fetch_andnot_relaxed) 2337 2227 static __always_inline s64 2338 - arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 2228 + raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 2339 2229 { 2340 - s64 ret = arch_atomic64_fetch_or_relaxed(i, v); 2230 + s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 2341 2231 __atomic_acquire_fence(); 2342 2232 return ret; 2343 2233 } 2344 - #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire 2234 + #elif defined(arch_atomic64_fetch_andnot) 2235 + #define raw_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot 2236 + #else 2237 + static __always_inline s64 2238 + raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 2239 + { 2240 + return raw_atomic64_fetch_and_acquire(~i, v); 2241 + } 2345 2242 #endif 2346 2243 2347 - #ifndef arch_atomic64_fetch_or_release 2244 + #if defined(arch_atomic64_fetch_andnot_release) 2245 + #define raw_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release 2246 + #elif defined(arch_atomic64_fetch_andnot_relaxed) 2348 2247 static __always_inline s64 2349 - arch_atomic64_fetch_or_release(s64 i, atomic64_t *v) 2248 + raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 2350 2249 { 2351 2250 __atomic_release_fence(); 2352 - return arch_atomic64_fetch_or_relaxed(i, v); 2251 + return arch_atomic64_fetch_andnot_relaxed(i, v); 2353 2252 } 2354 - #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release 2253 + #elif defined(arch_atomic64_fetch_andnot) 2254 + #define raw_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot 2255 + #else 2256 + static __always_inline s64 2257 + raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 2258 + { 2259 + return raw_atomic64_fetch_and_release(~i, v); 2260 + } 2355 2261 #endif 2356 2262 2357 - #ifndef arch_atomic64_fetch_or 2263 + #if defined(arch_atomic64_fetch_andnot_relaxed) 2264 + #define raw_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 2265 + #elif defined(arch_atomic64_fetch_andnot) 2266 + #define raw_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot 2267 + #else 2358 2268 static __always_inline s64 2359 - arch_atomic64_fetch_or(s64 i, atomic64_t *v) 2269 + raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 2270 + { 2271 + return raw_atomic64_fetch_and_relaxed(~i, v); 2272 + } 2273 + #endif 2274 + 2275 + #define raw_atomic64_or arch_atomic64_or 2276 + 2277 + #if defined(arch_atomic64_fetch_or) 2278 + #define raw_atomic64_fetch_or arch_atomic64_fetch_or 2279 + #elif defined(arch_atomic64_fetch_or_relaxed) 2280 + static __always_inline s64 2281 + raw_atomic64_fetch_or(s64 i, atomic64_t *v) 2360 2282 { 2361 2283 s64 ret; 2362 2284 __atomic_pre_full_fence(); ··· 2395 2255 __atomic_post_full_fence(); 2396 2256 return ret; 2397 2257 } 2398 - #define arch_atomic64_fetch_or arch_atomic64_fetch_or 2258 + #else 2259 + #error "Unable to define raw_atomic64_fetch_or" 2399 2260 #endif 2400 2261 2401 - #endif /* arch_atomic64_fetch_or_relaxed */ 2402 - 2403 - #ifndef arch_atomic64_fetch_xor_relaxed 2404 - #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor 2405 - #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor 2406 - #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor 2407 - #else /* arch_atomic64_fetch_xor_relaxed */ 2408 - 2409 - #ifndef arch_atomic64_fetch_xor_acquire 2262 + #if defined(arch_atomic64_fetch_or_acquire) 2263 + #define raw_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire 2264 + #elif defined(arch_atomic64_fetch_or_relaxed) 2410 2265 static __always_inline s64 2411 - arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 2266 + raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 2412 2267 { 2413 - s64 ret = arch_atomic64_fetch_xor_relaxed(i, v); 2268 + s64 ret = arch_atomic64_fetch_or_relaxed(i, v); 2414 2269 __atomic_acquire_fence(); 2415 2270 return ret; 2416 2271 } 2417 - #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire 2272 + #elif defined(arch_atomic64_fetch_or) 2273 + #define raw_atomic64_fetch_or_acquire arch_atomic64_fetch_or 2274 + #else 2275 + #error "Unable to define raw_atomic64_fetch_or_acquire" 2418 2276 #endif 2419 2277 2420 - #ifndef arch_atomic64_fetch_xor_release 2278 + #if defined(arch_atomic64_fetch_or_release) 2279 + #define raw_atomic64_fetch_or_release arch_atomic64_fetch_or_release 2280 + #elif defined(arch_atomic64_fetch_or_relaxed) 2421 2281 static __always_inline s64 2422 - arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v) 2282 + raw_atomic64_fetch_or_release(s64 i, atomic64_t *v) 2423 2283 { 2424 2284 __atomic_release_fence(); 2425 - return arch_atomic64_fetch_xor_relaxed(i, v); 2285 + return arch_atomic64_fetch_or_relaxed(i, v); 2426 2286 } 2427 - #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release 2287 + #elif defined(arch_atomic64_fetch_or) 2288 + #define raw_atomic64_fetch_or_release arch_atomic64_fetch_or 2289 + #else 2290 + #error "Unable to define raw_atomic64_fetch_or_release" 2428 2291 #endif 2429 2292 2430 - #ifndef arch_atomic64_fetch_xor 2293 + #if defined(arch_atomic64_fetch_or_relaxed) 2294 + #define raw_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 2295 + #elif defined(arch_atomic64_fetch_or) 2296 + #define raw_atomic64_fetch_or_relaxed arch_atomic64_fetch_or 2297 + #else 2298 + #error "Unable to define raw_atomic64_fetch_or_relaxed" 2299 + #endif 2300 + 2301 + #define raw_atomic64_xor arch_atomic64_xor 2302 + 2303 + #if defined(arch_atomic64_fetch_xor) 2304 + #define raw_atomic64_fetch_xor arch_atomic64_fetch_xor 2305 + #elif defined(arch_atomic64_fetch_xor_relaxed) 2431 2306 static __always_inline s64 2432 - arch_atomic64_fetch_xor(s64 i, atomic64_t *v) 2307 + raw_atomic64_fetch_xor(s64 i, atomic64_t *v) 2433 2308 { 2434 2309 s64 ret; 2435 2310 __atomic_pre_full_fence(); ··· 2452 2297 __atomic_post_full_fence(); 2453 2298 return ret; 2454 2299 } 2455 - #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor 2300 + #else 2301 + #error "Unable to define raw_atomic64_fetch_xor" 2456 2302 #endif 2457 2303 2458 - #endif /* arch_atomic64_fetch_xor_relaxed */ 2459 - 2460 - #ifndef arch_atomic64_xchg_relaxed 2461 - #ifdef arch_atomic64_xchg 2462 - #define arch_atomic64_xchg_acquire arch_atomic64_xchg 2463 - #define arch_atomic64_xchg_release arch_atomic64_xchg 2464 - #define arch_atomic64_xchg_relaxed arch_atomic64_xchg 2465 - #endif /* arch_atomic64_xchg */ 2466 - 2467 - #ifndef arch_atomic64_xchg 2304 + #if defined(arch_atomic64_fetch_xor_acquire) 2305 + #define raw_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire 2306 + #elif defined(arch_atomic64_fetch_xor_relaxed) 2468 2307 static __always_inline s64 2469 - arch_atomic64_xchg(atomic64_t *v, s64 new) 2308 + raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 2470 2309 { 2471 - return arch_xchg(&v->counter, new); 2472 - } 2473 - #define arch_atomic64_xchg arch_atomic64_xchg 2474 - #endif 2475 - 2476 - #ifndef arch_atomic64_xchg_acquire 2477 - static __always_inline s64 2478 - arch_atomic64_xchg_acquire(atomic64_t *v, s64 new) 2479 - { 2480 - return arch_xchg_acquire(&v->counter, new); 2481 - } 2482 - #define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire 2483 - #endif 2484 - 2485 - #ifndef arch_atomic64_xchg_release 2486 - static __always_inline s64 2487 - arch_atomic64_xchg_release(atomic64_t *v, s64 new) 2488 - { 2489 - return arch_xchg_release(&v->counter, new); 2490 - } 2491 - #define arch_atomic64_xchg_release arch_atomic64_xchg_release 2492 - #endif 2493 - 2494 - #ifndef arch_atomic64_xchg_relaxed 2495 - static __always_inline s64 2496 - arch_atomic64_xchg_relaxed(atomic64_t *v, s64 new) 2497 - { 2498 - return arch_xchg_relaxed(&v->counter, new); 2499 - } 2500 - #define arch_atomic64_xchg_relaxed arch_atomic64_xchg_relaxed 2501 - #endif 2502 - 2503 - #else /* arch_atomic64_xchg_relaxed */ 2504 - 2505 - #ifndef arch_atomic64_xchg_acquire 2506 - static __always_inline s64 2507 - arch_atomic64_xchg_acquire(atomic64_t *v, s64 i) 2508 - { 2509 - s64 ret = arch_atomic64_xchg_relaxed(v, i); 2310 + s64 ret = arch_atomic64_fetch_xor_relaxed(i, v); 2510 2311 __atomic_acquire_fence(); 2511 2312 return ret; 2512 2313 } 2513 - #define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire 2314 + #elif defined(arch_atomic64_fetch_xor) 2315 + #define raw_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor 2316 + #else 2317 + #error "Unable to define raw_atomic64_fetch_xor_acquire" 2514 2318 #endif 2515 2319 2516 - #ifndef arch_atomic64_xchg_release 2320 + #if defined(arch_atomic64_fetch_xor_release) 2321 + #define raw_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release 2322 + #elif defined(arch_atomic64_fetch_xor_relaxed) 2517 2323 static __always_inline s64 2518 - arch_atomic64_xchg_release(atomic64_t *v, s64 i) 2324 + raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v) 2519 2325 { 2520 2326 __atomic_release_fence(); 2521 - return arch_atomic64_xchg_relaxed(v, i); 2327 + return arch_atomic64_fetch_xor_relaxed(i, v); 2522 2328 } 2523 - #define arch_atomic64_xchg_release arch_atomic64_xchg_release 2329 + #elif defined(arch_atomic64_fetch_xor) 2330 + #define raw_atomic64_fetch_xor_release arch_atomic64_fetch_xor 2331 + #else 2332 + #error "Unable to define raw_atomic64_fetch_xor_release" 2524 2333 #endif 2525 2334 2526 - #ifndef arch_atomic64_xchg 2335 + #if defined(arch_atomic64_fetch_xor_relaxed) 2336 + #define raw_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 2337 + #elif defined(arch_atomic64_fetch_xor) 2338 + #define raw_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor 2339 + #else 2340 + #error "Unable to define raw_atomic64_fetch_xor_relaxed" 2341 + #endif 2342 + 2343 + #if defined(arch_atomic64_xchg) 2344 + #define raw_atomic64_xchg arch_atomic64_xchg 2345 + #elif defined(arch_atomic64_xchg_relaxed) 2527 2346 static __always_inline s64 2528 - arch_atomic64_xchg(atomic64_t *v, s64 i) 2347 + raw_atomic64_xchg(atomic64_t *v, s64 i) 2529 2348 { 2530 2349 s64 ret; 2531 2350 __atomic_pre_full_fence(); ··· 2507 2378 __atomic_post_full_fence(); 2508 2379 return ret; 2509 2380 } 2510 - #define arch_atomic64_xchg arch_atomic64_xchg 2511 - #endif 2512 - 2513 - #endif /* arch_atomic64_xchg_relaxed */ 2514 - 2515 - #ifndef arch_atomic64_cmpxchg_relaxed 2516 - #ifdef arch_atomic64_cmpxchg 2517 - #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg 2518 - #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg 2519 - #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg 2520 - #endif /* arch_atomic64_cmpxchg */ 2521 - 2522 - #ifndef arch_atomic64_cmpxchg 2381 + #else 2523 2382 static __always_inline s64 2524 - arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 2383 + raw_atomic64_xchg(atomic64_t *v, s64 new) 2525 2384 { 2526 - return arch_cmpxchg(&v->counter, old, new); 2385 + return raw_xchg(&v->counter, new); 2527 2386 } 2528 - #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg 2529 2387 #endif 2530 2388 2531 - #ifndef arch_atomic64_cmpxchg_acquire 2389 + #if defined(arch_atomic64_xchg_acquire) 2390 + #define raw_atomic64_xchg_acquire arch_atomic64_xchg_acquire 2391 + #elif defined(arch_atomic64_xchg_relaxed) 2532 2392 static __always_inline s64 2533 - arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 2393 + raw_atomic64_xchg_acquire(atomic64_t *v, s64 i) 2534 2394 { 2535 - return arch_cmpxchg_acquire(&v->counter, old, new); 2536 - } 2537 - #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire 2538 - #endif 2539 - 2540 - #ifndef arch_atomic64_cmpxchg_release 2541 - static __always_inline s64 2542 - arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 2543 - { 2544 - return arch_cmpxchg_release(&v->counter, old, new); 2545 - } 2546 - #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release 2547 - #endif 2548 - 2549 - #ifndef arch_atomic64_cmpxchg_relaxed 2550 - static __always_inline s64 2551 - arch_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) 2552 - { 2553 - return arch_cmpxchg_relaxed(&v->counter, old, new); 2554 - } 2555 - #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg_relaxed 2556 - #endif 2557 - 2558 - #else /* arch_atomic64_cmpxchg_relaxed */ 2559 - 2560 - #ifndef arch_atomic64_cmpxchg_acquire 2561 - static __always_inline s64 2562 - arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 2563 - { 2564 - s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 2395 + s64 ret = arch_atomic64_xchg_relaxed(v, i); 2565 2396 __atomic_acquire_fence(); 2566 2397 return ret; 2567 2398 } 2568 - #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire 2399 + #elif defined(arch_atomic64_xchg) 2400 + #define raw_atomic64_xchg_acquire arch_atomic64_xchg 2401 + #else 2402 + static __always_inline s64 2403 + raw_atomic64_xchg_acquire(atomic64_t *v, s64 new) 2404 + { 2405 + return raw_xchg_acquire(&v->counter, new); 2406 + } 2569 2407 #endif 2570 2408 2571 - #ifndef arch_atomic64_cmpxchg_release 2409 + #if defined(arch_atomic64_xchg_release) 2410 + #define raw_atomic64_xchg_release arch_atomic64_xchg_release 2411 + #elif defined(arch_atomic64_xchg_relaxed) 2572 2412 static __always_inline s64 2573 - arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 2413 + raw_atomic64_xchg_release(atomic64_t *v, s64 i) 2574 2414 { 2575 2415 __atomic_release_fence(); 2576 - return arch_atomic64_cmpxchg_relaxed(v, old, new); 2416 + return arch_atomic64_xchg_relaxed(v, i); 2577 2417 } 2578 - #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release 2418 + #elif defined(arch_atomic64_xchg) 2419 + #define raw_atomic64_xchg_release arch_atomic64_xchg 2420 + #else 2421 + static __always_inline s64 2422 + raw_atomic64_xchg_release(atomic64_t *v, s64 new) 2423 + { 2424 + return raw_xchg_release(&v->counter, new); 2425 + } 2579 2426 #endif 2580 2427 2581 - #ifndef arch_atomic64_cmpxchg 2428 + #if defined(arch_atomic64_xchg_relaxed) 2429 + #define raw_atomic64_xchg_relaxed arch_atomic64_xchg_relaxed 2430 + #elif defined(arch_atomic64_xchg) 2431 + #define raw_atomic64_xchg_relaxed arch_atomic64_xchg 2432 + #else 2582 2433 static __always_inline s64 2583 - arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 2434 + raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new) 2435 + { 2436 + return raw_xchg_relaxed(&v->counter, new); 2437 + } 2438 + #endif 2439 + 2440 + #if defined(arch_atomic64_cmpxchg) 2441 + #define raw_atomic64_cmpxchg arch_atomic64_cmpxchg 2442 + #elif defined(arch_atomic64_cmpxchg_relaxed) 2443 + static __always_inline s64 2444 + raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 2584 2445 { 2585 2446 s64 ret; 2586 2447 __atomic_pre_full_fence(); ··· 2578 2459 __atomic_post_full_fence(); 2579 2460 return ret; 2580 2461 } 2581 - #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg 2582 - #endif 2583 - 2584 - #endif /* arch_atomic64_cmpxchg_relaxed */ 2585 - 2586 - #ifndef arch_atomic64_try_cmpxchg_relaxed 2587 - #ifdef arch_atomic64_try_cmpxchg 2588 - #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg 2589 - #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg 2590 - #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg 2591 - #endif /* arch_atomic64_try_cmpxchg */ 2592 - 2593 - #ifndef arch_atomic64_try_cmpxchg 2594 - static __always_inline bool 2595 - arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 2462 + #else 2463 + static __always_inline s64 2464 + raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 2596 2465 { 2597 - s64 r, o = *old; 2598 - r = arch_atomic64_cmpxchg(v, o, new); 2599 - if (unlikely(r != o)) 2600 - *old = r; 2601 - return likely(r == o); 2466 + return raw_cmpxchg(&v->counter, old, new); 2602 2467 } 2603 - #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg 2604 2468 #endif 2605 2469 2606 - #ifndef arch_atomic64_try_cmpxchg_acquire 2607 - static __always_inline bool 2608 - arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2470 + #if defined(arch_atomic64_cmpxchg_acquire) 2471 + #define raw_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire 2472 + #elif defined(arch_atomic64_cmpxchg_relaxed) 2473 + static __always_inline s64 2474 + raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 2609 2475 { 2610 - s64 r, o = *old; 2611 - r = arch_atomic64_cmpxchg_acquire(v, o, new); 2612 - if (unlikely(r != o)) 2613 - *old = r; 2614 - return likely(r == o); 2615 - } 2616 - #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire 2617 - #endif 2618 - 2619 - #ifndef arch_atomic64_try_cmpxchg_release 2620 - static __always_inline bool 2621 - arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 2622 - { 2623 - s64 r, o = *old; 2624 - r = arch_atomic64_cmpxchg_release(v, o, new); 2625 - if (unlikely(r != o)) 2626 - *old = r; 2627 - return likely(r == o); 2628 - } 2629 - #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release 2630 - #endif 2631 - 2632 - #ifndef arch_atomic64_try_cmpxchg_relaxed 2633 - static __always_inline bool 2634 - arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 2635 - { 2636 - s64 r, o = *old; 2637 - r = arch_atomic64_cmpxchg_relaxed(v, o, new); 2638 - if (unlikely(r != o)) 2639 - *old = r; 2640 - return likely(r == o); 2641 - } 2642 - #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed 2643 - #endif 2644 - 2645 - #else /* arch_atomic64_try_cmpxchg_relaxed */ 2646 - 2647 - #ifndef arch_atomic64_try_cmpxchg_acquire 2648 - static __always_inline bool 2649 - arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2650 - { 2651 - bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2476 + s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 2652 2477 __atomic_acquire_fence(); 2653 2478 return ret; 2654 2479 } 2655 - #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire 2480 + #elif defined(arch_atomic64_cmpxchg) 2481 + #define raw_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg 2482 + #else 2483 + static __always_inline s64 2484 + raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 2485 + { 2486 + return raw_cmpxchg_acquire(&v->counter, old, new); 2487 + } 2656 2488 #endif 2657 2489 2658 - #ifndef arch_atomic64_try_cmpxchg_release 2659 - static __always_inline bool 2660 - arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 2490 + #if defined(arch_atomic64_cmpxchg_release) 2491 + #define raw_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release 2492 + #elif defined(arch_atomic64_cmpxchg_relaxed) 2493 + static __always_inline s64 2494 + raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 2661 2495 { 2662 2496 __atomic_release_fence(); 2663 - return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2497 + return arch_atomic64_cmpxchg_relaxed(v, old, new); 2664 2498 } 2665 - #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release 2499 + #elif defined(arch_atomic64_cmpxchg) 2500 + #define raw_atomic64_cmpxchg_release arch_atomic64_cmpxchg 2501 + #else 2502 + static __always_inline s64 2503 + raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 2504 + { 2505 + return raw_cmpxchg_release(&v->counter, old, new); 2506 + } 2666 2507 #endif 2667 2508 2668 - #ifndef arch_atomic64_try_cmpxchg 2509 + #if defined(arch_atomic64_cmpxchg_relaxed) 2510 + #define raw_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg_relaxed 2511 + #elif defined(arch_atomic64_cmpxchg) 2512 + #define raw_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg 2513 + #else 2514 + static __always_inline s64 2515 + raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) 2516 + { 2517 + return raw_cmpxchg_relaxed(&v->counter, old, new); 2518 + } 2519 + #endif 2520 + 2521 + #if defined(arch_atomic64_try_cmpxchg) 2522 + #define raw_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg 2523 + #elif defined(arch_atomic64_try_cmpxchg_relaxed) 2669 2524 static __always_inline bool 2670 - arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 2525 + raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 2671 2526 { 2672 2527 bool ret; 2673 2528 __atomic_pre_full_fence(); ··· 2649 2556 __atomic_post_full_fence(); 2650 2557 return ret; 2651 2558 } 2652 - #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg 2653 - #endif 2654 - 2655 - #endif /* arch_atomic64_try_cmpxchg_relaxed */ 2656 - 2657 - #ifndef arch_atomic64_sub_and_test 2559 + #else 2658 2560 static __always_inline bool 2659 - arch_atomic64_sub_and_test(s64 i, atomic64_t *v) 2561 + raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 2660 2562 { 2661 - return arch_atomic64_sub_return(i, v) == 0; 2563 + s64 r, o = *old; 2564 + r = raw_atomic64_cmpxchg(v, o, new); 2565 + if (unlikely(r != o)) 2566 + *old = r; 2567 + return likely(r == o); 2662 2568 } 2663 - #define arch_atomic64_sub_and_test arch_atomic64_sub_and_test 2664 2569 #endif 2665 2570 2666 - #ifndef arch_atomic64_dec_and_test 2571 + #if defined(arch_atomic64_try_cmpxchg_acquire) 2572 + #define raw_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire 2573 + #elif defined(arch_atomic64_try_cmpxchg_relaxed) 2667 2574 static __always_inline bool 2668 - arch_atomic64_dec_and_test(atomic64_t *v) 2575 + raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2669 2576 { 2670 - return arch_atomic64_dec_return(v) == 0; 2671 - } 2672 - #define arch_atomic64_dec_and_test arch_atomic64_dec_and_test 2673 - #endif 2674 - 2675 - #ifndef arch_atomic64_inc_and_test 2676 - static __always_inline bool 2677 - arch_atomic64_inc_and_test(atomic64_t *v) 2678 - { 2679 - return arch_atomic64_inc_return(v) == 0; 2680 - } 2681 - #define arch_atomic64_inc_and_test arch_atomic64_inc_and_test 2682 - #endif 2683 - 2684 - #ifndef arch_atomic64_add_negative_relaxed 2685 - #ifdef arch_atomic64_add_negative 2686 - #define arch_atomic64_add_negative_acquire arch_atomic64_add_negative 2687 - #define arch_atomic64_add_negative_release arch_atomic64_add_negative 2688 - #define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative 2689 - #endif /* arch_atomic64_add_negative */ 2690 - 2691 - #ifndef arch_atomic64_add_negative 2692 - static __always_inline bool 2693 - arch_atomic64_add_negative(s64 i, atomic64_t *v) 2694 - { 2695 - return arch_atomic64_add_return(i, v) < 0; 2696 - } 2697 - #define arch_atomic64_add_negative arch_atomic64_add_negative 2698 - #endif 2699 - 2700 - #ifndef arch_atomic64_add_negative_acquire 2701 - static __always_inline bool 2702 - arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v) 2703 - { 2704 - return arch_atomic64_add_return_acquire(i, v) < 0; 2705 - } 2706 - #define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire 2707 - #endif 2708 - 2709 - #ifndef arch_atomic64_add_negative_release 2710 - static __always_inline bool 2711 - arch_atomic64_add_negative_release(s64 i, atomic64_t *v) 2712 - { 2713 - return arch_atomic64_add_return_release(i, v) < 0; 2714 - } 2715 - #define arch_atomic64_add_negative_release arch_atomic64_add_negative_release 2716 - #endif 2717 - 2718 - #ifndef arch_atomic64_add_negative_relaxed 2719 - static __always_inline bool 2720 - arch_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) 2721 - { 2722 - return arch_atomic64_add_return_relaxed(i, v) < 0; 2723 - } 2724 - #define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative_relaxed 2725 - #endif 2726 - 2727 - #else /* arch_atomic64_add_negative_relaxed */ 2728 - 2729 - #ifndef arch_atomic64_add_negative_acquire 2730 - static __always_inline bool 2731 - arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v) 2732 - { 2733 - bool ret = arch_atomic64_add_negative_relaxed(i, v); 2577 + bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2734 2578 __atomic_acquire_fence(); 2735 2579 return ret; 2736 2580 } 2737 - #define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire 2581 + #elif defined(arch_atomic64_try_cmpxchg) 2582 + #define raw_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg 2583 + #else 2584 + static __always_inline bool 2585 + raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2586 + { 2587 + s64 r, o = *old; 2588 + r = raw_atomic64_cmpxchg_acquire(v, o, new); 2589 + if (unlikely(r != o)) 2590 + *old = r; 2591 + return likely(r == o); 2592 + } 2738 2593 #endif 2739 2594 2740 - #ifndef arch_atomic64_add_negative_release 2595 + #if defined(arch_atomic64_try_cmpxchg_release) 2596 + #define raw_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release 2597 + #elif defined(arch_atomic64_try_cmpxchg_relaxed) 2741 2598 static __always_inline bool 2742 - arch_atomic64_add_negative_release(s64 i, atomic64_t *v) 2599 + raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 2743 2600 { 2744 2601 __atomic_release_fence(); 2745 - return arch_atomic64_add_negative_relaxed(i, v); 2602 + return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2746 2603 } 2747 - #define arch_atomic64_add_negative_release arch_atomic64_add_negative_release 2604 + #elif defined(arch_atomic64_try_cmpxchg) 2605 + #define raw_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg 2606 + #else 2607 + static __always_inline bool 2608 + raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 2609 + { 2610 + s64 r, o = *old; 2611 + r = raw_atomic64_cmpxchg_release(v, o, new); 2612 + if (unlikely(r != o)) 2613 + *old = r; 2614 + return likely(r == o); 2615 + } 2748 2616 #endif 2749 2617 2750 - #ifndef arch_atomic64_add_negative 2618 + #if defined(arch_atomic64_try_cmpxchg_relaxed) 2619 + #define raw_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed 2620 + #elif defined(arch_atomic64_try_cmpxchg) 2621 + #define raw_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg 2622 + #else 2751 2623 static __always_inline bool 2752 - arch_atomic64_add_negative(s64 i, atomic64_t *v) 2624 + raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 2625 + { 2626 + s64 r, o = *old; 2627 + r = raw_atomic64_cmpxchg_relaxed(v, o, new); 2628 + if (unlikely(r != o)) 2629 + *old = r; 2630 + return likely(r == o); 2631 + } 2632 + #endif 2633 + 2634 + #if defined(arch_atomic64_sub_and_test) 2635 + #define raw_atomic64_sub_and_test arch_atomic64_sub_and_test 2636 + #else 2637 + static __always_inline bool 2638 + raw_atomic64_sub_and_test(s64 i, atomic64_t *v) 2639 + { 2640 + return raw_atomic64_sub_return(i, v) == 0; 2641 + } 2642 + #endif 2643 + 2644 + #if defined(arch_atomic64_dec_and_test) 2645 + #define raw_atomic64_dec_and_test arch_atomic64_dec_and_test 2646 + #else 2647 + static __always_inline bool 2648 + raw_atomic64_dec_and_test(atomic64_t *v) 2649 + { 2650 + return raw_atomic64_dec_return(v) == 0; 2651 + } 2652 + #endif 2653 + 2654 + #if defined(arch_atomic64_inc_and_test) 2655 + #define raw_atomic64_inc_and_test arch_atomic64_inc_and_test 2656 + #else 2657 + static __always_inline bool 2658 + raw_atomic64_inc_and_test(atomic64_t *v) 2659 + { 2660 + return raw_atomic64_inc_return(v) == 0; 2661 + } 2662 + #endif 2663 + 2664 + #if defined(arch_atomic64_add_negative) 2665 + #define raw_atomic64_add_negative arch_atomic64_add_negative 2666 + #elif defined(arch_atomic64_add_negative_relaxed) 2667 + static __always_inline bool 2668 + raw_atomic64_add_negative(s64 i, atomic64_t *v) 2753 2669 { 2754 2670 bool ret; 2755 2671 __atomic_pre_full_fence(); ··· 2766 2664 __atomic_post_full_fence(); 2767 2665 return ret; 2768 2666 } 2769 - #define arch_atomic64_add_negative arch_atomic64_add_negative 2667 + #else 2668 + static __always_inline bool 2669 + raw_atomic64_add_negative(s64 i, atomic64_t *v) 2670 + { 2671 + return raw_atomic64_add_return(i, v) < 0; 2672 + } 2770 2673 #endif 2771 2674 2772 - #endif /* arch_atomic64_add_negative_relaxed */ 2773 - 2774 - #ifndef arch_atomic64_fetch_add_unless 2775 - static __always_inline s64 2776 - arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 2675 + #if defined(arch_atomic64_add_negative_acquire) 2676 + #define raw_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire 2677 + #elif defined(arch_atomic64_add_negative_relaxed) 2678 + static __always_inline bool 2679 + raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) 2777 2680 { 2778 - s64 c = arch_atomic64_read(v); 2681 + bool ret = arch_atomic64_add_negative_relaxed(i, v); 2682 + __atomic_acquire_fence(); 2683 + return ret; 2684 + } 2685 + #elif defined(arch_atomic64_add_negative) 2686 + #define raw_atomic64_add_negative_acquire arch_atomic64_add_negative 2687 + #else 2688 + static __always_inline bool 2689 + raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) 2690 + { 2691 + return raw_atomic64_add_return_acquire(i, v) < 0; 2692 + } 2693 + #endif 2694 + 2695 + #if defined(arch_atomic64_add_negative_release) 2696 + #define raw_atomic64_add_negative_release arch_atomic64_add_negative_release 2697 + #elif defined(arch_atomic64_add_negative_relaxed) 2698 + static __always_inline bool 2699 + raw_atomic64_add_negative_release(s64 i, atomic64_t *v) 2700 + { 2701 + __atomic_release_fence(); 2702 + return arch_atomic64_add_negative_relaxed(i, v); 2703 + } 2704 + #elif defined(arch_atomic64_add_negative) 2705 + #define raw_atomic64_add_negative_release arch_atomic64_add_negative 2706 + #else 2707 + static __always_inline bool 2708 + raw_atomic64_add_negative_release(s64 i, atomic64_t *v) 2709 + { 2710 + return raw_atomic64_add_return_release(i, v) < 0; 2711 + } 2712 + #endif 2713 + 2714 + #if defined(arch_atomic64_add_negative_relaxed) 2715 + #define raw_atomic64_add_negative_relaxed arch_atomic64_add_negative_relaxed 2716 + #elif defined(arch_atomic64_add_negative) 2717 + #define raw_atomic64_add_negative_relaxed arch_atomic64_add_negative 2718 + #else 2719 + static __always_inline bool 2720 + raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) 2721 + { 2722 + return raw_atomic64_add_return_relaxed(i, v) < 0; 2723 + } 2724 + #endif 2725 + 2726 + #if defined(arch_atomic64_fetch_add_unless) 2727 + #define raw_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 2728 + #else 2729 + static __always_inline s64 2730 + raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 2731 + { 2732 + s64 c = raw_atomic64_read(v); 2779 2733 2780 2734 do { 2781 2735 if (unlikely(c == u)) 2782 2736 break; 2783 - } while (!arch_atomic64_try_cmpxchg(v, &c, c + a)); 2737 + } while (!raw_atomic64_try_cmpxchg(v, &c, c + a)); 2784 2738 2785 2739 return c; 2786 2740 } 2787 - #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 2788 2741 #endif 2789 2742 2790 - #ifndef arch_atomic64_add_unless 2743 + #if defined(arch_atomic64_add_unless) 2744 + #define raw_atomic64_add_unless arch_atomic64_add_unless 2745 + #else 2791 2746 static __always_inline bool 2792 - arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 2747 + raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 2793 2748 { 2794 - return arch_atomic64_fetch_add_unless(v, a, u) != u; 2749 + return raw_atomic64_fetch_add_unless(v, a, u) != u; 2795 2750 } 2796 - #define arch_atomic64_add_unless arch_atomic64_add_unless 2797 2751 #endif 2798 2752 2799 - #ifndef arch_atomic64_inc_not_zero 2753 + #if defined(arch_atomic64_inc_not_zero) 2754 + #define raw_atomic64_inc_not_zero arch_atomic64_inc_not_zero 2755 + #else 2800 2756 static __always_inline bool 2801 - arch_atomic64_inc_not_zero(atomic64_t *v) 2757 + raw_atomic64_inc_not_zero(atomic64_t *v) 2802 2758 { 2803 - return arch_atomic64_add_unless(v, 1, 0); 2759 + return raw_atomic64_add_unless(v, 1, 0); 2804 2760 } 2805 - #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero 2806 2761 #endif 2807 2762 2808 - #ifndef arch_atomic64_inc_unless_negative 2763 + #if defined(arch_atomic64_inc_unless_negative) 2764 + #define raw_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative 2765 + #else 2809 2766 static __always_inline bool 2810 - arch_atomic64_inc_unless_negative(atomic64_t *v) 2767 + raw_atomic64_inc_unless_negative(atomic64_t *v) 2811 2768 { 2812 - s64 c = arch_atomic64_read(v); 2769 + s64 c = raw_atomic64_read(v); 2813 2770 2814 2771 do { 2815 2772 if (unlikely(c < 0)) 2816 2773 return false; 2817 - } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1)); 2774 + } while (!raw_atomic64_try_cmpxchg(v, &c, c + 1)); 2818 2775 2819 2776 return true; 2820 2777 } 2821 - #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative 2822 2778 #endif 2823 2779 2824 - #ifndef arch_atomic64_dec_unless_positive 2780 + #if defined(arch_atomic64_dec_unless_positive) 2781 + #define raw_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive 2782 + #else 2825 2783 static __always_inline bool 2826 - arch_atomic64_dec_unless_positive(atomic64_t *v) 2784 + raw_atomic64_dec_unless_positive(atomic64_t *v) 2827 2785 { 2828 - s64 c = arch_atomic64_read(v); 2786 + s64 c = raw_atomic64_read(v); 2829 2787 2830 2788 do { 2831 2789 if (unlikely(c > 0)) 2832 2790 return false; 2833 - } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1)); 2791 + } while (!raw_atomic64_try_cmpxchg(v, &c, c - 1)); 2834 2792 2835 2793 return true; 2836 2794 } 2837 - #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive 2838 2795 #endif 2839 2796 2840 - #ifndef arch_atomic64_dec_if_positive 2797 + #if defined(arch_atomic64_dec_if_positive) 2798 + #define raw_atomic64_dec_if_positive arch_atomic64_dec_if_positive 2799 + #else 2841 2800 static __always_inline s64 2842 - arch_atomic64_dec_if_positive(atomic64_t *v) 2801 + raw_atomic64_dec_if_positive(atomic64_t *v) 2843 2802 { 2844 - s64 dec, c = arch_atomic64_read(v); 2803 + s64 dec, c = raw_atomic64_read(v); 2845 2804 2846 2805 do { 2847 2806 dec = c - 1; 2848 2807 if (unlikely(dec < 0)) 2849 2808 break; 2850 - } while (!arch_atomic64_try_cmpxchg(v, &c, dec)); 2809 + } while (!raw_atomic64_try_cmpxchg(v, &c, dec)); 2851 2810 2852 2811 return dec; 2853 2812 } 2854 - #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 2855 2813 #endif 2856 2814 2857 2815 #endif /* _LINUX_ATOMIC_FALLBACK_H */ 2858 - // e1cee558cc61cae887890db30fcdf93baca9f498 2816 + // c2048fccede6fac923252290e2b303949d5dec83
-1135
include/linux/atomic/atomic-raw.h
··· 1 - // SPDX-License-Identifier: GPL-2.0 2 - 3 - // Generated by scripts/atomic/gen-atomic-raw.sh 4 - // DO NOT MODIFY THIS FILE DIRECTLY 5 - 6 - #ifndef _LINUX_ATOMIC_RAW_H 7 - #define _LINUX_ATOMIC_RAW_H 8 - 9 - static __always_inline int 10 - raw_atomic_read(const atomic_t *v) 11 - { 12 - return arch_atomic_read(v); 13 - } 14 - 15 - static __always_inline int 16 - raw_atomic_read_acquire(const atomic_t *v) 17 - { 18 - return arch_atomic_read_acquire(v); 19 - } 20 - 21 - static __always_inline void 22 - raw_atomic_set(atomic_t *v, int i) 23 - { 24 - arch_atomic_set(v, i); 25 - } 26 - 27 - static __always_inline void 28 - raw_atomic_set_release(atomic_t *v, int i) 29 - { 30 - arch_atomic_set_release(v, i); 31 - } 32 - 33 - static __always_inline void 34 - raw_atomic_add(int i, atomic_t *v) 35 - { 36 - arch_atomic_add(i, v); 37 - } 38 - 39 - static __always_inline int 40 - raw_atomic_add_return(int i, atomic_t *v) 41 - { 42 - return arch_atomic_add_return(i, v); 43 - } 44 - 45 - static __always_inline int 46 - raw_atomic_add_return_acquire(int i, atomic_t *v) 47 - { 48 - return arch_atomic_add_return_acquire(i, v); 49 - } 50 - 51 - static __always_inline int 52 - raw_atomic_add_return_release(int i, atomic_t *v) 53 - { 54 - return arch_atomic_add_return_release(i, v); 55 - } 56 - 57 - static __always_inline int 58 - raw_atomic_add_return_relaxed(int i, atomic_t *v) 59 - { 60 - return arch_atomic_add_return_relaxed(i, v); 61 - } 62 - 63 - static __always_inline int 64 - raw_atomic_fetch_add(int i, atomic_t *v) 65 - { 66 - return arch_atomic_fetch_add(i, v); 67 - } 68 - 69 - static __always_inline int 70 - raw_atomic_fetch_add_acquire(int i, atomic_t *v) 71 - { 72 - return arch_atomic_fetch_add_acquire(i, v); 73 - } 74 - 75 - static __always_inline int 76 - raw_atomic_fetch_add_release(int i, atomic_t *v) 77 - { 78 - return arch_atomic_fetch_add_release(i, v); 79 - } 80 - 81 - static __always_inline int 82 - raw_atomic_fetch_add_relaxed(int i, atomic_t *v) 83 - { 84 - return arch_atomic_fetch_add_relaxed(i, v); 85 - } 86 - 87 - static __always_inline void 88 - raw_atomic_sub(int i, atomic_t *v) 89 - { 90 - arch_atomic_sub(i, v); 91 - } 92 - 93 - static __always_inline int 94 - raw_atomic_sub_return(int i, atomic_t *v) 95 - { 96 - return arch_atomic_sub_return(i, v); 97 - } 98 - 99 - static __always_inline int 100 - raw_atomic_sub_return_acquire(int i, atomic_t *v) 101 - { 102 - return arch_atomic_sub_return_acquire(i, v); 103 - } 104 - 105 - static __always_inline int 106 - raw_atomic_sub_return_release(int i, atomic_t *v) 107 - { 108 - return arch_atomic_sub_return_release(i, v); 109 - } 110 - 111 - static __always_inline int 112 - raw_atomic_sub_return_relaxed(int i, atomic_t *v) 113 - { 114 - return arch_atomic_sub_return_relaxed(i, v); 115 - } 116 - 117 - static __always_inline int 118 - raw_atomic_fetch_sub(int i, atomic_t *v) 119 - { 120 - return arch_atomic_fetch_sub(i, v); 121 - } 122 - 123 - static __always_inline int 124 - raw_atomic_fetch_sub_acquire(int i, atomic_t *v) 125 - { 126 - return arch_atomic_fetch_sub_acquire(i, v); 127 - } 128 - 129 - static __always_inline int 130 - raw_atomic_fetch_sub_release(int i, atomic_t *v) 131 - { 132 - return arch_atomic_fetch_sub_release(i, v); 133 - } 134 - 135 - static __always_inline int 136 - raw_atomic_fetch_sub_relaxed(int i, atomic_t *v) 137 - { 138 - return arch_atomic_fetch_sub_relaxed(i, v); 139 - } 140 - 141 - static __always_inline void 142 - raw_atomic_inc(atomic_t *v) 143 - { 144 - arch_atomic_inc(v); 145 - } 146 - 147 - static __always_inline int 148 - raw_atomic_inc_return(atomic_t *v) 149 - { 150 - return arch_atomic_inc_return(v); 151 - } 152 - 153 - static __always_inline int 154 - raw_atomic_inc_return_acquire(atomic_t *v) 155 - { 156 - return arch_atomic_inc_return_acquire(v); 157 - } 158 - 159 - static __always_inline int 160 - raw_atomic_inc_return_release(atomic_t *v) 161 - { 162 - return arch_atomic_inc_return_release(v); 163 - } 164 - 165 - static __always_inline int 166 - raw_atomic_inc_return_relaxed(atomic_t *v) 167 - { 168 - return arch_atomic_inc_return_relaxed(v); 169 - } 170 - 171 - static __always_inline int 172 - raw_atomic_fetch_inc(atomic_t *v) 173 - { 174 - return arch_atomic_fetch_inc(v); 175 - } 176 - 177 - static __always_inline int 178 - raw_atomic_fetch_inc_acquire(atomic_t *v) 179 - { 180 - return arch_atomic_fetch_inc_acquire(v); 181 - } 182 - 183 - static __always_inline int 184 - raw_atomic_fetch_inc_release(atomic_t *v) 185 - { 186 - return arch_atomic_fetch_inc_release(v); 187 - } 188 - 189 - static __always_inline int 190 - raw_atomic_fetch_inc_relaxed(atomic_t *v) 191 - { 192 - return arch_atomic_fetch_inc_relaxed(v); 193 - } 194 - 195 - static __always_inline void 196 - raw_atomic_dec(atomic_t *v) 197 - { 198 - arch_atomic_dec(v); 199 - } 200 - 201 - static __always_inline int 202 - raw_atomic_dec_return(atomic_t *v) 203 - { 204 - return arch_atomic_dec_return(v); 205 - } 206 - 207 - static __always_inline int 208 - raw_atomic_dec_return_acquire(atomic_t *v) 209 - { 210 - return arch_atomic_dec_return_acquire(v); 211 - } 212 - 213 - static __always_inline int 214 - raw_atomic_dec_return_release(atomic_t *v) 215 - { 216 - return arch_atomic_dec_return_release(v); 217 - } 218 - 219 - static __always_inline int 220 - raw_atomic_dec_return_relaxed(atomic_t *v) 221 - { 222 - return arch_atomic_dec_return_relaxed(v); 223 - } 224 - 225 - static __always_inline int 226 - raw_atomic_fetch_dec(atomic_t *v) 227 - { 228 - return arch_atomic_fetch_dec(v); 229 - } 230 - 231 - static __always_inline int 232 - raw_atomic_fetch_dec_acquire(atomic_t *v) 233 - { 234 - return arch_atomic_fetch_dec_acquire(v); 235 - } 236 - 237 - static __always_inline int 238 - raw_atomic_fetch_dec_release(atomic_t *v) 239 - { 240 - return arch_atomic_fetch_dec_release(v); 241 - } 242 - 243 - static __always_inline int 244 - raw_atomic_fetch_dec_relaxed(atomic_t *v) 245 - { 246 - return arch_atomic_fetch_dec_relaxed(v); 247 - } 248 - 249 - static __always_inline void 250 - raw_atomic_and(int i, atomic_t *v) 251 - { 252 - arch_atomic_and(i, v); 253 - } 254 - 255 - static __always_inline int 256 - raw_atomic_fetch_and(int i, atomic_t *v) 257 - { 258 - return arch_atomic_fetch_and(i, v); 259 - } 260 - 261 - static __always_inline int 262 - raw_atomic_fetch_and_acquire(int i, atomic_t *v) 263 - { 264 - return arch_atomic_fetch_and_acquire(i, v); 265 - } 266 - 267 - static __always_inline int 268 - raw_atomic_fetch_and_release(int i, atomic_t *v) 269 - { 270 - return arch_atomic_fetch_and_release(i, v); 271 - } 272 - 273 - static __always_inline int 274 - raw_atomic_fetch_and_relaxed(int i, atomic_t *v) 275 - { 276 - return arch_atomic_fetch_and_relaxed(i, v); 277 - } 278 - 279 - static __always_inline void 280 - raw_atomic_andnot(int i, atomic_t *v) 281 - { 282 - arch_atomic_andnot(i, v); 283 - } 284 - 285 - static __always_inline int 286 - raw_atomic_fetch_andnot(int i, atomic_t *v) 287 - { 288 - return arch_atomic_fetch_andnot(i, v); 289 - } 290 - 291 - static __always_inline int 292 - raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) 293 - { 294 - return arch_atomic_fetch_andnot_acquire(i, v); 295 - } 296 - 297 - static __always_inline int 298 - raw_atomic_fetch_andnot_release(int i, atomic_t *v) 299 - { 300 - return arch_atomic_fetch_andnot_release(i, v); 301 - } 302 - 303 - static __always_inline int 304 - raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v) 305 - { 306 - return arch_atomic_fetch_andnot_relaxed(i, v); 307 - } 308 - 309 - static __always_inline void 310 - raw_atomic_or(int i, atomic_t *v) 311 - { 312 - arch_atomic_or(i, v); 313 - } 314 - 315 - static __always_inline int 316 - raw_atomic_fetch_or(int i, atomic_t *v) 317 - { 318 - return arch_atomic_fetch_or(i, v); 319 - } 320 - 321 - static __always_inline int 322 - raw_atomic_fetch_or_acquire(int i, atomic_t *v) 323 - { 324 - return arch_atomic_fetch_or_acquire(i, v); 325 - } 326 - 327 - static __always_inline int 328 - raw_atomic_fetch_or_release(int i, atomic_t *v) 329 - { 330 - return arch_atomic_fetch_or_release(i, v); 331 - } 332 - 333 - static __always_inline int 334 - raw_atomic_fetch_or_relaxed(int i, atomic_t *v) 335 - { 336 - return arch_atomic_fetch_or_relaxed(i, v); 337 - } 338 - 339 - static __always_inline void 340 - raw_atomic_xor(int i, atomic_t *v) 341 - { 342 - arch_atomic_xor(i, v); 343 - } 344 - 345 - static __always_inline int 346 - raw_atomic_fetch_xor(int i, atomic_t *v) 347 - { 348 - return arch_atomic_fetch_xor(i, v); 349 - } 350 - 351 - static __always_inline int 352 - raw_atomic_fetch_xor_acquire(int i, atomic_t *v) 353 - { 354 - return arch_atomic_fetch_xor_acquire(i, v); 355 - } 356 - 357 - static __always_inline int 358 - raw_atomic_fetch_xor_release(int i, atomic_t *v) 359 - { 360 - return arch_atomic_fetch_xor_release(i, v); 361 - } 362 - 363 - static __always_inline int 364 - raw_atomic_fetch_xor_relaxed(int i, atomic_t *v) 365 - { 366 - return arch_atomic_fetch_xor_relaxed(i, v); 367 - } 368 - 369 - static __always_inline int 370 - raw_atomic_xchg(atomic_t *v, int i) 371 - { 372 - return arch_atomic_xchg(v, i); 373 - } 374 - 375 - static __always_inline int 376 - raw_atomic_xchg_acquire(atomic_t *v, int i) 377 - { 378 - return arch_atomic_xchg_acquire(v, i); 379 - } 380 - 381 - static __always_inline int 382 - raw_atomic_xchg_release(atomic_t *v, int i) 383 - { 384 - return arch_atomic_xchg_release(v, i); 385 - } 386 - 387 - static __always_inline int 388 - raw_atomic_xchg_relaxed(atomic_t *v, int i) 389 - { 390 - return arch_atomic_xchg_relaxed(v, i); 391 - } 392 - 393 - static __always_inline int 394 - raw_atomic_cmpxchg(atomic_t *v, int old, int new) 395 - { 396 - return arch_atomic_cmpxchg(v, old, new); 397 - } 398 - 399 - static __always_inline int 400 - raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 401 - { 402 - return arch_atomic_cmpxchg_acquire(v, old, new); 403 - } 404 - 405 - static __always_inline int 406 - raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) 407 - { 408 - return arch_atomic_cmpxchg_release(v, old, new); 409 - } 410 - 411 - static __always_inline int 412 - raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) 413 - { 414 - return arch_atomic_cmpxchg_relaxed(v, old, new); 415 - } 416 - 417 - static __always_inline bool 418 - raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 419 - { 420 - return arch_atomic_try_cmpxchg(v, old, new); 421 - } 422 - 423 - static __always_inline bool 424 - raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 425 - { 426 - return arch_atomic_try_cmpxchg_acquire(v, old, new); 427 - } 428 - 429 - static __always_inline bool 430 - raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 431 - { 432 - return arch_atomic_try_cmpxchg_release(v, old, new); 433 - } 434 - 435 - static __always_inline bool 436 - raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 437 - { 438 - return arch_atomic_try_cmpxchg_relaxed(v, old, new); 439 - } 440 - 441 - static __always_inline bool 442 - raw_atomic_sub_and_test(int i, atomic_t *v) 443 - { 444 - return arch_atomic_sub_and_test(i, v); 445 - } 446 - 447 - static __always_inline bool 448 - raw_atomic_dec_and_test(atomic_t *v) 449 - { 450 - return arch_atomic_dec_and_test(v); 451 - } 452 - 453 - static __always_inline bool 454 - raw_atomic_inc_and_test(atomic_t *v) 455 - { 456 - return arch_atomic_inc_and_test(v); 457 - } 458 - 459 - static __always_inline bool 460 - raw_atomic_add_negative(int i, atomic_t *v) 461 - { 462 - return arch_atomic_add_negative(i, v); 463 - } 464 - 465 - static __always_inline bool 466 - raw_atomic_add_negative_acquire(int i, atomic_t *v) 467 - { 468 - return arch_atomic_add_negative_acquire(i, v); 469 - } 470 - 471 - static __always_inline bool 472 - raw_atomic_add_negative_release(int i, atomic_t *v) 473 - { 474 - return arch_atomic_add_negative_release(i, v); 475 - } 476 - 477 - static __always_inline bool 478 - raw_atomic_add_negative_relaxed(int i, atomic_t *v) 479 - { 480 - return arch_atomic_add_negative_relaxed(i, v); 481 - } 482 - 483 - static __always_inline int 484 - raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) 485 - { 486 - return arch_atomic_fetch_add_unless(v, a, u); 487 - } 488 - 489 - static __always_inline bool 490 - raw_atomic_add_unless(atomic_t *v, int a, int u) 491 - { 492 - return arch_atomic_add_unless(v, a, u); 493 - } 494 - 495 - static __always_inline bool 496 - raw_atomic_inc_not_zero(atomic_t *v) 497 - { 498 - return arch_atomic_inc_not_zero(v); 499 - } 500 - 501 - static __always_inline bool 502 - raw_atomic_inc_unless_negative(atomic_t *v) 503 - { 504 - return arch_atomic_inc_unless_negative(v); 505 - } 506 - 507 - static __always_inline bool 508 - raw_atomic_dec_unless_positive(atomic_t *v) 509 - { 510 - return arch_atomic_dec_unless_positive(v); 511 - } 512 - 513 - static __always_inline int 514 - raw_atomic_dec_if_positive(atomic_t *v) 515 - { 516 - return arch_atomic_dec_if_positive(v); 517 - } 518 - 519 - static __always_inline s64 520 - raw_atomic64_read(const atomic64_t *v) 521 - { 522 - return arch_atomic64_read(v); 523 - } 524 - 525 - static __always_inline s64 526 - raw_atomic64_read_acquire(const atomic64_t *v) 527 - { 528 - return arch_atomic64_read_acquire(v); 529 - } 530 - 531 - static __always_inline void 532 - raw_atomic64_set(atomic64_t *v, s64 i) 533 - { 534 - arch_atomic64_set(v, i); 535 - } 536 - 537 - static __always_inline void 538 - raw_atomic64_set_release(atomic64_t *v, s64 i) 539 - { 540 - arch_atomic64_set_release(v, i); 541 - } 542 - 543 - static __always_inline void 544 - raw_atomic64_add(s64 i, atomic64_t *v) 545 - { 546 - arch_atomic64_add(i, v); 547 - } 548 - 549 - static __always_inline s64 550 - raw_atomic64_add_return(s64 i, atomic64_t *v) 551 - { 552 - return arch_atomic64_add_return(i, v); 553 - } 554 - 555 - static __always_inline s64 556 - raw_atomic64_add_return_acquire(s64 i, atomic64_t *v) 557 - { 558 - return arch_atomic64_add_return_acquire(i, v); 559 - } 560 - 561 - static __always_inline s64 562 - raw_atomic64_add_return_release(s64 i, atomic64_t *v) 563 - { 564 - return arch_atomic64_add_return_release(i, v); 565 - } 566 - 567 - static __always_inline s64 568 - raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v) 569 - { 570 - return arch_atomic64_add_return_relaxed(i, v); 571 - } 572 - 573 - static __always_inline s64 574 - raw_atomic64_fetch_add(s64 i, atomic64_t *v) 575 - { 576 - return arch_atomic64_fetch_add(i, v); 577 - } 578 - 579 - static __always_inline s64 580 - raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 581 - { 582 - return arch_atomic64_fetch_add_acquire(i, v); 583 - } 584 - 585 - static __always_inline s64 586 - raw_atomic64_fetch_add_release(s64 i, atomic64_t *v) 587 - { 588 - return arch_atomic64_fetch_add_release(i, v); 589 - } 590 - 591 - static __always_inline s64 592 - raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) 593 - { 594 - return arch_atomic64_fetch_add_relaxed(i, v); 595 - } 596 - 597 - static __always_inline void 598 - raw_atomic64_sub(s64 i, atomic64_t *v) 599 - { 600 - arch_atomic64_sub(i, v); 601 - } 602 - 603 - static __always_inline s64 604 - raw_atomic64_sub_return(s64 i, atomic64_t *v) 605 - { 606 - return arch_atomic64_sub_return(i, v); 607 - } 608 - 609 - static __always_inline s64 610 - raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v) 611 - { 612 - return arch_atomic64_sub_return_acquire(i, v); 613 - } 614 - 615 - static __always_inline s64 616 - raw_atomic64_sub_return_release(s64 i, atomic64_t *v) 617 - { 618 - return arch_atomic64_sub_return_release(i, v); 619 - } 620 - 621 - static __always_inline s64 622 - raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) 623 - { 624 - return arch_atomic64_sub_return_relaxed(i, v); 625 - } 626 - 627 - static __always_inline s64 628 - raw_atomic64_fetch_sub(s64 i, atomic64_t *v) 629 - { 630 - return arch_atomic64_fetch_sub(i, v); 631 - } 632 - 633 - static __always_inline s64 634 - raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 635 - { 636 - return arch_atomic64_fetch_sub_acquire(i, v); 637 - } 638 - 639 - static __always_inline s64 640 - raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v) 641 - { 642 - return arch_atomic64_fetch_sub_release(i, v); 643 - } 644 - 645 - static __always_inline s64 646 - raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) 647 - { 648 - return arch_atomic64_fetch_sub_relaxed(i, v); 649 - } 650 - 651 - static __always_inline void 652 - raw_atomic64_inc(atomic64_t *v) 653 - { 654 - arch_atomic64_inc(v); 655 - } 656 - 657 - static __always_inline s64 658 - raw_atomic64_inc_return(atomic64_t *v) 659 - { 660 - return arch_atomic64_inc_return(v); 661 - } 662 - 663 - static __always_inline s64 664 - raw_atomic64_inc_return_acquire(atomic64_t *v) 665 - { 666 - return arch_atomic64_inc_return_acquire(v); 667 - } 668 - 669 - static __always_inline s64 670 - raw_atomic64_inc_return_release(atomic64_t *v) 671 - { 672 - return arch_atomic64_inc_return_release(v); 673 - } 674 - 675 - static __always_inline s64 676 - raw_atomic64_inc_return_relaxed(atomic64_t *v) 677 - { 678 - return arch_atomic64_inc_return_relaxed(v); 679 - } 680 - 681 - static __always_inline s64 682 - raw_atomic64_fetch_inc(atomic64_t *v) 683 - { 684 - return arch_atomic64_fetch_inc(v); 685 - } 686 - 687 - static __always_inline s64 688 - raw_atomic64_fetch_inc_acquire(atomic64_t *v) 689 - { 690 - return arch_atomic64_fetch_inc_acquire(v); 691 - } 692 - 693 - static __always_inline s64 694 - raw_atomic64_fetch_inc_release(atomic64_t *v) 695 - { 696 - return arch_atomic64_fetch_inc_release(v); 697 - } 698 - 699 - static __always_inline s64 700 - raw_atomic64_fetch_inc_relaxed(atomic64_t *v) 701 - { 702 - return arch_atomic64_fetch_inc_relaxed(v); 703 - } 704 - 705 - static __always_inline void 706 - raw_atomic64_dec(atomic64_t *v) 707 - { 708 - arch_atomic64_dec(v); 709 - } 710 - 711 - static __always_inline s64 712 - raw_atomic64_dec_return(atomic64_t *v) 713 - { 714 - return arch_atomic64_dec_return(v); 715 - } 716 - 717 - static __always_inline s64 718 - raw_atomic64_dec_return_acquire(atomic64_t *v) 719 - { 720 - return arch_atomic64_dec_return_acquire(v); 721 - } 722 - 723 - static __always_inline s64 724 - raw_atomic64_dec_return_release(atomic64_t *v) 725 - { 726 - return arch_atomic64_dec_return_release(v); 727 - } 728 - 729 - static __always_inline s64 730 - raw_atomic64_dec_return_relaxed(atomic64_t *v) 731 - { 732 - return arch_atomic64_dec_return_relaxed(v); 733 - } 734 - 735 - static __always_inline s64 736 - raw_atomic64_fetch_dec(atomic64_t *v) 737 - { 738 - return arch_atomic64_fetch_dec(v); 739 - } 740 - 741 - static __always_inline s64 742 - raw_atomic64_fetch_dec_acquire(atomic64_t *v) 743 - { 744 - return arch_atomic64_fetch_dec_acquire(v); 745 - } 746 - 747 - static __always_inline s64 748 - raw_atomic64_fetch_dec_release(atomic64_t *v) 749 - { 750 - return arch_atomic64_fetch_dec_release(v); 751 - } 752 - 753 - static __always_inline s64 754 - raw_atomic64_fetch_dec_relaxed(atomic64_t *v) 755 - { 756 - return arch_atomic64_fetch_dec_relaxed(v); 757 - } 758 - 759 - static __always_inline void 760 - raw_atomic64_and(s64 i, atomic64_t *v) 761 - { 762 - arch_atomic64_and(i, v); 763 - } 764 - 765 - static __always_inline s64 766 - raw_atomic64_fetch_and(s64 i, atomic64_t *v) 767 - { 768 - return arch_atomic64_fetch_and(i, v); 769 - } 770 - 771 - static __always_inline s64 772 - raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 773 - { 774 - return arch_atomic64_fetch_and_acquire(i, v); 775 - } 776 - 777 - static __always_inline s64 778 - raw_atomic64_fetch_and_release(s64 i, atomic64_t *v) 779 - { 780 - return arch_atomic64_fetch_and_release(i, v); 781 - } 782 - 783 - static __always_inline s64 784 - raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) 785 - { 786 - return arch_atomic64_fetch_and_relaxed(i, v); 787 - } 788 - 789 - static __always_inline void 790 - raw_atomic64_andnot(s64 i, atomic64_t *v) 791 - { 792 - arch_atomic64_andnot(i, v); 793 - } 794 - 795 - static __always_inline s64 796 - raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) 797 - { 798 - return arch_atomic64_fetch_andnot(i, v); 799 - } 800 - 801 - static __always_inline s64 802 - raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 803 - { 804 - return arch_atomic64_fetch_andnot_acquire(i, v); 805 - } 806 - 807 - static __always_inline s64 808 - raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 809 - { 810 - return arch_atomic64_fetch_andnot_release(i, v); 811 - } 812 - 813 - static __always_inline s64 814 - raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 815 - { 816 - return arch_atomic64_fetch_andnot_relaxed(i, v); 817 - } 818 - 819 - static __always_inline void 820 - raw_atomic64_or(s64 i, atomic64_t *v) 821 - { 822 - arch_atomic64_or(i, v); 823 - } 824 - 825 - static __always_inline s64 826 - raw_atomic64_fetch_or(s64 i, atomic64_t *v) 827 - { 828 - return arch_atomic64_fetch_or(i, v); 829 - } 830 - 831 - static __always_inline s64 832 - raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 833 - { 834 - return arch_atomic64_fetch_or_acquire(i, v); 835 - } 836 - 837 - static __always_inline s64 838 - raw_atomic64_fetch_or_release(s64 i, atomic64_t *v) 839 - { 840 - return arch_atomic64_fetch_or_release(i, v); 841 - } 842 - 843 - static __always_inline s64 844 - raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) 845 - { 846 - return arch_atomic64_fetch_or_relaxed(i, v); 847 - } 848 - 849 - static __always_inline void 850 - raw_atomic64_xor(s64 i, atomic64_t *v) 851 - { 852 - arch_atomic64_xor(i, v); 853 - } 854 - 855 - static __always_inline s64 856 - raw_atomic64_fetch_xor(s64 i, atomic64_t *v) 857 - { 858 - return arch_atomic64_fetch_xor(i, v); 859 - } 860 - 861 - static __always_inline s64 862 - raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 863 - { 864 - return arch_atomic64_fetch_xor_acquire(i, v); 865 - } 866 - 867 - static __always_inline s64 868 - raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v) 869 - { 870 - return arch_atomic64_fetch_xor_release(i, v); 871 - } 872 - 873 - static __always_inline s64 874 - raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) 875 - { 876 - return arch_atomic64_fetch_xor_relaxed(i, v); 877 - } 878 - 879 - static __always_inline s64 880 - raw_atomic64_xchg(atomic64_t *v, s64 i) 881 - { 882 - return arch_atomic64_xchg(v, i); 883 - } 884 - 885 - static __always_inline s64 886 - raw_atomic64_xchg_acquire(atomic64_t *v, s64 i) 887 - { 888 - return arch_atomic64_xchg_acquire(v, i); 889 - } 890 - 891 - static __always_inline s64 892 - raw_atomic64_xchg_release(atomic64_t *v, s64 i) 893 - { 894 - return arch_atomic64_xchg_release(v, i); 895 - } 896 - 897 - static __always_inline s64 898 - raw_atomic64_xchg_relaxed(atomic64_t *v, s64 i) 899 - { 900 - return arch_atomic64_xchg_relaxed(v, i); 901 - } 902 - 903 - static __always_inline s64 904 - raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 905 - { 906 - return arch_atomic64_cmpxchg(v, old, new); 907 - } 908 - 909 - static __always_inline s64 910 - raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 911 - { 912 - return arch_atomic64_cmpxchg_acquire(v, old, new); 913 - } 914 - 915 - static __always_inline s64 916 - raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 917 - { 918 - return arch_atomic64_cmpxchg_release(v, old, new); 919 - } 920 - 921 - static __always_inline s64 922 - raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) 923 - { 924 - return arch_atomic64_cmpxchg_relaxed(v, old, new); 925 - } 926 - 927 - static __always_inline bool 928 - raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 929 - { 930 - return arch_atomic64_try_cmpxchg(v, old, new); 931 - } 932 - 933 - static __always_inline bool 934 - raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 935 - { 936 - return arch_atomic64_try_cmpxchg_acquire(v, old, new); 937 - } 938 - 939 - static __always_inline bool 940 - raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 941 - { 942 - return arch_atomic64_try_cmpxchg_release(v, old, new); 943 - } 944 - 945 - static __always_inline bool 946 - raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 947 - { 948 - return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 949 - } 950 - 951 - static __always_inline bool 952 - raw_atomic64_sub_and_test(s64 i, atomic64_t *v) 953 - { 954 - return arch_atomic64_sub_and_test(i, v); 955 - } 956 - 957 - static __always_inline bool 958 - raw_atomic64_dec_and_test(atomic64_t *v) 959 - { 960 - return arch_atomic64_dec_and_test(v); 961 - } 962 - 963 - static __always_inline bool 964 - raw_atomic64_inc_and_test(atomic64_t *v) 965 - { 966 - return arch_atomic64_inc_and_test(v); 967 - } 968 - 969 - static __always_inline bool 970 - raw_atomic64_add_negative(s64 i, atomic64_t *v) 971 - { 972 - return arch_atomic64_add_negative(i, v); 973 - } 974 - 975 - static __always_inline bool 976 - raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) 977 - { 978 - return arch_atomic64_add_negative_acquire(i, v); 979 - } 980 - 981 - static __always_inline bool 982 - raw_atomic64_add_negative_release(s64 i, atomic64_t *v) 983 - { 984 - return arch_atomic64_add_negative_release(i, v); 985 - } 986 - 987 - static __always_inline bool 988 - raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) 989 - { 990 - return arch_atomic64_add_negative_relaxed(i, v); 991 - } 992 - 993 - static __always_inline s64 994 - raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 995 - { 996 - return arch_atomic64_fetch_add_unless(v, a, u); 997 - } 998 - 999 - static __always_inline bool 1000 - raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 1001 - { 1002 - return arch_atomic64_add_unless(v, a, u); 1003 - } 1004 - 1005 - static __always_inline bool 1006 - raw_atomic64_inc_not_zero(atomic64_t *v) 1007 - { 1008 - return arch_atomic64_inc_not_zero(v); 1009 - } 1010 - 1011 - static __always_inline bool 1012 - raw_atomic64_inc_unless_negative(atomic64_t *v) 1013 - { 1014 - return arch_atomic64_inc_unless_negative(v); 1015 - } 1016 - 1017 - static __always_inline bool 1018 - raw_atomic64_dec_unless_positive(atomic64_t *v) 1019 - { 1020 - return arch_atomic64_dec_unless_positive(v); 1021 - } 1022 - 1023 - static __always_inline s64 1024 - raw_atomic64_dec_if_positive(atomic64_t *v) 1025 - { 1026 - return arch_atomic64_dec_if_positive(v); 1027 - } 1028 - 1029 - #define raw_xchg(...) \ 1030 - arch_xchg(__VA_ARGS__) 1031 - 1032 - #define raw_xchg_acquire(...) \ 1033 - arch_xchg_acquire(__VA_ARGS__) 1034 - 1035 - #define raw_xchg_release(...) \ 1036 - arch_xchg_release(__VA_ARGS__) 1037 - 1038 - #define raw_xchg_relaxed(...) \ 1039 - arch_xchg_relaxed(__VA_ARGS__) 1040 - 1041 - #define raw_cmpxchg(...) \ 1042 - arch_cmpxchg(__VA_ARGS__) 1043 - 1044 - #define raw_cmpxchg_acquire(...) \ 1045 - arch_cmpxchg_acquire(__VA_ARGS__) 1046 - 1047 - #define raw_cmpxchg_release(...) \ 1048 - arch_cmpxchg_release(__VA_ARGS__) 1049 - 1050 - #define raw_cmpxchg_relaxed(...) \ 1051 - arch_cmpxchg_relaxed(__VA_ARGS__) 1052 - 1053 - #define raw_cmpxchg64(...) \ 1054 - arch_cmpxchg64(__VA_ARGS__) 1055 - 1056 - #define raw_cmpxchg64_acquire(...) \ 1057 - arch_cmpxchg64_acquire(__VA_ARGS__) 1058 - 1059 - #define raw_cmpxchg64_release(...) \ 1060 - arch_cmpxchg64_release(__VA_ARGS__) 1061 - 1062 - #define raw_cmpxchg64_relaxed(...) \ 1063 - arch_cmpxchg64_relaxed(__VA_ARGS__) 1064 - 1065 - #define raw_cmpxchg128(...) \ 1066 - arch_cmpxchg128(__VA_ARGS__) 1067 - 1068 - #define raw_cmpxchg128_acquire(...) \ 1069 - arch_cmpxchg128_acquire(__VA_ARGS__) 1070 - 1071 - #define raw_cmpxchg128_release(...) \ 1072 - arch_cmpxchg128_release(__VA_ARGS__) 1073 - 1074 - #define raw_cmpxchg128_relaxed(...) \ 1075 - arch_cmpxchg128_relaxed(__VA_ARGS__) 1076 - 1077 - #define raw_try_cmpxchg(...) \ 1078 - arch_try_cmpxchg(__VA_ARGS__) 1079 - 1080 - #define raw_try_cmpxchg_acquire(...) \ 1081 - arch_try_cmpxchg_acquire(__VA_ARGS__) 1082 - 1083 - #define raw_try_cmpxchg_release(...) \ 1084 - arch_try_cmpxchg_release(__VA_ARGS__) 1085 - 1086 - #define raw_try_cmpxchg_relaxed(...) \ 1087 - arch_try_cmpxchg_relaxed(__VA_ARGS__) 1088 - 1089 - #define raw_try_cmpxchg64(...) \ 1090 - arch_try_cmpxchg64(__VA_ARGS__) 1091 - 1092 - #define raw_try_cmpxchg64_acquire(...) \ 1093 - arch_try_cmpxchg64_acquire(__VA_ARGS__) 1094 - 1095 - #define raw_try_cmpxchg64_release(...) \ 1096 - arch_try_cmpxchg64_release(__VA_ARGS__) 1097 - 1098 - #define raw_try_cmpxchg64_relaxed(...) \ 1099 - arch_try_cmpxchg64_relaxed(__VA_ARGS__) 1100 - 1101 - #define raw_try_cmpxchg128(...) \ 1102 - arch_try_cmpxchg128(__VA_ARGS__) 1103 - 1104 - #define raw_try_cmpxchg128_acquire(...) \ 1105 - arch_try_cmpxchg128_acquire(__VA_ARGS__) 1106 - 1107 - #define raw_try_cmpxchg128_release(...) \ 1108 - arch_try_cmpxchg128_release(__VA_ARGS__) 1109 - 1110 - #define raw_try_cmpxchg128_relaxed(...) \ 1111 - arch_try_cmpxchg128_relaxed(__VA_ARGS__) 1112 - 1113 - #define raw_cmpxchg_local(...) \ 1114 - arch_cmpxchg_local(__VA_ARGS__) 1115 - 1116 - #define raw_cmpxchg64_local(...) \ 1117 - arch_cmpxchg64_local(__VA_ARGS__) 1118 - 1119 - #define raw_cmpxchg128_local(...) \ 1120 - arch_cmpxchg128_local(__VA_ARGS__) 1121 - 1122 - #define raw_sync_cmpxchg(...) \ 1123 - arch_sync_cmpxchg(__VA_ARGS__) 1124 - 1125 - #define raw_try_cmpxchg_local(...) \ 1126 - arch_try_cmpxchg_local(__VA_ARGS__) 1127 - 1128 - #define raw_try_cmpxchg64_local(...) \ 1129 - arch_try_cmpxchg64_local(__VA_ARGS__) 1130 - 1131 - #define raw_try_cmpxchg128_local(...) \ 1132 - arch_try_cmpxchg128_local(__VA_ARGS__) 1133 - 1134 - #endif /* _LINUX_ATOMIC_RAW_H */ 1135 - // b23ed4424e85200e200ded094522e1d743b3a5b1
+1 -1
scripts/atomic/fallbacks/acquire
··· 1 1 cat <<EOF 2 2 static __always_inline ${ret} 3 - arch_${atomic}_${pfx}${name}${sfx}_acquire(${params}) 3 + raw_${atomic}_${pfx}${name}${sfx}_acquire(${params}) 4 4 { 5 5 ${ret} ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args}); 6 6 __atomic_acquire_fence();
+2 -2
scripts/atomic/fallbacks/add_negative
··· 1 1 cat <<EOF 2 2 static __always_inline bool 3 - arch_${atomic}_add_negative${order}(${int} i, ${atomic}_t *v) 3 + raw_${atomic}_add_negative${order}(${int} i, ${atomic}_t *v) 4 4 { 5 - return arch_${atomic}_add_return${order}(i, v) < 0; 5 + return raw_${atomic}_add_return${order}(i, v) < 0; 6 6 } 7 7 EOF
+2 -2
scripts/atomic/fallbacks/add_unless
··· 1 1 cat << EOF 2 2 static __always_inline bool 3 - arch_${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u) 3 + raw_${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u) 4 4 { 5 - return arch_${atomic}_fetch_add_unless(v, a, u) != u; 5 + return raw_${atomic}_fetch_add_unless(v, a, u) != u; 6 6 } 7 7 EOF
+2 -2
scripts/atomic/fallbacks/andnot
··· 1 1 cat <<EOF 2 2 static __always_inline ${ret} 3 - arch_${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v) 3 + raw_${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v) 4 4 { 5 - ${retstmt}arch_${atomic}_${pfx}and${sfx}${order}(~i, v); 5 + ${retstmt}raw_${atomic}_${pfx}and${sfx}${order}(~i, v); 6 6 } 7 7 EOF
+2 -2
scripts/atomic/fallbacks/cmpxchg
··· 1 1 cat <<EOF 2 2 static __always_inline ${int} 3 - arch_${atomic}_cmpxchg${order}(${atomic}_t *v, ${int} old, ${int} new) 3 + raw_${atomic}_cmpxchg${order}(${atomic}_t *v, ${int} old, ${int} new) 4 4 { 5 - return arch_cmpxchg${order}(&v->counter, old, new); 5 + return raw_cmpxchg${order}(&v->counter, old, new); 6 6 } 7 7 EOF
+2 -2
scripts/atomic/fallbacks/dec
··· 1 1 cat <<EOF 2 2 static __always_inline ${ret} 3 - arch_${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v) 3 + raw_${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v) 4 4 { 5 - ${retstmt}arch_${atomic}_${pfx}sub${sfx}${order}(1, v); 5 + ${retstmt}raw_${atomic}_${pfx}sub${sfx}${order}(1, v); 6 6 } 7 7 EOF
+2 -2
scripts/atomic/fallbacks/dec_and_test
··· 1 1 cat <<EOF 2 2 static __always_inline bool 3 - arch_${atomic}_dec_and_test(${atomic}_t *v) 3 + raw_${atomic}_dec_and_test(${atomic}_t *v) 4 4 { 5 - return arch_${atomic}_dec_return(v) == 0; 5 + return raw_${atomic}_dec_return(v) == 0; 6 6 } 7 7 EOF
+3 -3
scripts/atomic/fallbacks/dec_if_positive
··· 1 1 cat <<EOF 2 2 static __always_inline ${ret} 3 - arch_${atomic}_dec_if_positive(${atomic}_t *v) 3 + raw_${atomic}_dec_if_positive(${atomic}_t *v) 4 4 { 5 - ${int} dec, c = arch_${atomic}_read(v); 5 + ${int} dec, c = raw_${atomic}_read(v); 6 6 7 7 do { 8 8 dec = c - 1; 9 9 if (unlikely(dec < 0)) 10 10 break; 11 - } while (!arch_${atomic}_try_cmpxchg(v, &c, dec)); 11 + } while (!raw_${atomic}_try_cmpxchg(v, &c, dec)); 12 12 13 13 return dec; 14 14 }
+3 -3
scripts/atomic/fallbacks/dec_unless_positive
··· 1 1 cat <<EOF 2 2 static __always_inline bool 3 - arch_${atomic}_dec_unless_positive(${atomic}_t *v) 3 + raw_${atomic}_dec_unless_positive(${atomic}_t *v) 4 4 { 5 - ${int} c = arch_${atomic}_read(v); 5 + ${int} c = raw_${atomic}_read(v); 6 6 7 7 do { 8 8 if (unlikely(c > 0)) 9 9 return false; 10 - } while (!arch_${atomic}_try_cmpxchg(v, &c, c - 1)); 10 + } while (!raw_${atomic}_try_cmpxchg(v, &c, c - 1)); 11 11 12 12 return true; 13 13 }
+1 -1
scripts/atomic/fallbacks/fence
··· 1 1 cat <<EOF 2 2 static __always_inline ${ret} 3 - arch_${atomic}_${pfx}${name}${sfx}(${params}) 3 + raw_${atomic}_${pfx}${name}${sfx}(${params}) 4 4 { 5 5 ${ret} ret; 6 6 __atomic_pre_full_fence();
+3 -3
scripts/atomic/fallbacks/fetch_add_unless
··· 1 1 cat << EOF 2 2 static __always_inline ${int} 3 - arch_${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u) 3 + raw_${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u) 4 4 { 5 - ${int} c = arch_${atomic}_read(v); 5 + ${int} c = raw_${atomic}_read(v); 6 6 7 7 do { 8 8 if (unlikely(c == u)) 9 9 break; 10 - } while (!arch_${atomic}_try_cmpxchg(v, &c, c + a)); 10 + } while (!raw_${atomic}_try_cmpxchg(v, &c, c + a)); 11 11 12 12 return c; 13 13 }
+2 -2
scripts/atomic/fallbacks/inc
··· 1 1 cat <<EOF 2 2 static __always_inline ${ret} 3 - arch_${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v) 3 + raw_${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v) 4 4 { 5 - ${retstmt}arch_${atomic}_${pfx}add${sfx}${order}(1, v); 5 + ${retstmt}raw_${atomic}_${pfx}add${sfx}${order}(1, v); 6 6 } 7 7 EOF
+2 -2
scripts/atomic/fallbacks/inc_and_test
··· 1 1 cat <<EOF 2 2 static __always_inline bool 3 - arch_${atomic}_inc_and_test(${atomic}_t *v) 3 + raw_${atomic}_inc_and_test(${atomic}_t *v) 4 4 { 5 - return arch_${atomic}_inc_return(v) == 0; 5 + return raw_${atomic}_inc_return(v) == 0; 6 6 } 7 7 EOF
+2 -2
scripts/atomic/fallbacks/inc_not_zero
··· 1 1 cat <<EOF 2 2 static __always_inline bool 3 - arch_${atomic}_inc_not_zero(${atomic}_t *v) 3 + raw_${atomic}_inc_not_zero(${atomic}_t *v) 4 4 { 5 - return arch_${atomic}_add_unless(v, 1, 0); 5 + return raw_${atomic}_add_unless(v, 1, 0); 6 6 } 7 7 EOF
+3 -3
scripts/atomic/fallbacks/inc_unless_negative
··· 1 1 cat <<EOF 2 2 static __always_inline bool 3 - arch_${atomic}_inc_unless_negative(${atomic}_t *v) 3 + raw_${atomic}_inc_unless_negative(${atomic}_t *v) 4 4 { 5 - ${int} c = arch_${atomic}_read(v); 5 + ${int} c = raw_${atomic}_read(v); 6 6 7 7 do { 8 8 if (unlikely(c < 0)) 9 9 return false; 10 - } while (!arch_${atomic}_try_cmpxchg(v, &c, c + 1)); 10 + } while (!raw_${atomic}_try_cmpxchg(v, &c, c + 1)); 11 11 12 12 return true; 13 13 }
+2 -2
scripts/atomic/fallbacks/read_acquire
··· 1 1 cat <<EOF 2 2 static __always_inline ${ret} 3 - arch_${atomic}_read_acquire(const ${atomic}_t *v) 3 + raw_${atomic}_read_acquire(const ${atomic}_t *v) 4 4 { 5 5 ${int} ret; 6 6 7 7 if (__native_word(${atomic}_t)) { 8 8 ret = smp_load_acquire(&(v)->counter); 9 9 } else { 10 - ret = arch_${atomic}_read(v); 10 + ret = raw_${atomic}_read(v); 11 11 __atomic_acquire_fence(); 12 12 } 13 13
+1 -1
scripts/atomic/fallbacks/release
··· 1 1 cat <<EOF 2 2 static __always_inline ${ret} 3 - arch_${atomic}_${pfx}${name}${sfx}_release(${params}) 3 + raw_${atomic}_${pfx}${name}${sfx}_release(${params}) 4 4 { 5 5 __atomic_release_fence(); 6 6 ${retstmt}arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
+2 -2
scripts/atomic/fallbacks/set_release
··· 1 1 cat <<EOF 2 2 static __always_inline void 3 - arch_${atomic}_set_release(${atomic}_t *v, ${int} i) 3 + raw_${atomic}_set_release(${atomic}_t *v, ${int} i) 4 4 { 5 5 if (__native_word(${atomic}_t)) { 6 6 smp_store_release(&(v)->counter, i); 7 7 } else { 8 8 __atomic_release_fence(); 9 - arch_${atomic}_set(v, i); 9 + raw_${atomic}_set(v, i); 10 10 } 11 11 } 12 12 EOF
+2 -2
scripts/atomic/fallbacks/sub_and_test
··· 1 1 cat <<EOF 2 2 static __always_inline bool 3 - arch_${atomic}_sub_and_test(${int} i, ${atomic}_t *v) 3 + raw_${atomic}_sub_and_test(${int} i, ${atomic}_t *v) 4 4 { 5 - return arch_${atomic}_sub_return(i, v) == 0; 5 + return raw_${atomic}_sub_return(i, v) == 0; 6 6 } 7 7 EOF
+2 -2
scripts/atomic/fallbacks/try_cmpxchg
··· 1 1 cat <<EOF 2 2 static __always_inline bool 3 - arch_${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new) 3 + raw_${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new) 4 4 { 5 5 ${int} r, o = *old; 6 - r = arch_${atomic}_cmpxchg${order}(v, o, new); 6 + r = raw_${atomic}_cmpxchg${order}(v, o, new); 7 7 if (unlikely(r != o)) 8 8 *old = r; 9 9 return likely(r == o);
+2 -2
scripts/atomic/fallbacks/xchg
··· 1 1 cat <<EOF 2 2 static __always_inline ${int} 3 - arch_${atomic}_xchg${order}(${atomic}_t *v, ${int} new) 3 + raw_${atomic}_xchg${order}(${atomic}_t *v, ${int} new) 4 4 { 5 - return arch_xchg${order}(&v->counter, new); 5 + return raw_xchg${order}(&v->counter, new); 6 6 } 7 7 EOF
+159 -89
scripts/atomic/gen-atomic-fallback.sh
··· 17 17 local atomic="$1"; shift 18 18 local int="$1"; shift 19 19 20 - local atomicname="arch_${atomic}_${pfx}${name}${sfx}${order}" 21 - 22 20 local ret="$(gen_ret_type "${meta}" "${int}")" 23 21 local retstmt="$(gen_ret_stmt "${meta}")" 24 22 local params="$(gen_params "${int}" "${atomic}" "$@")" 25 23 local args="$(gen_args "$@")" 26 24 27 - if [ ! -z "${template}" ]; then 28 - printf "#ifndef ${atomicname}\n" 29 - . ${template} 30 - printf "#define ${atomicname} ${atomicname}\n" 31 - printf "#endif\n\n" 32 - fi 25 + . ${template} 33 26 } 34 27 35 28 #gen_order_fallback(meta, pfx, name, sfx, order, atomic, int, args...) ··· 52 59 gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@" 53 60 } 54 61 55 - #gen_basic_fallbacks(basename) 56 - gen_basic_fallbacks() 62 + #gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, args...) 63 + gen_proto_order_variant() 57 64 { 58 - local basename="$1"; shift 59 - cat << EOF 60 - #define ${basename}_acquire ${basename} 61 - #define ${basename}_release ${basename} 62 - #define ${basename}_relaxed ${basename} 63 - EOF 65 + local meta="$1"; shift 66 + local pfx="$1"; shift 67 + local name="$1"; shift 68 + local sfx="$1"; shift 69 + local order="$1"; shift 70 + local atomic="$1" 71 + 72 + local atomicname="${atomic}_${pfx}${name}${sfx}${order}" 73 + local basename="${atomic}_${pfx}${name}${sfx}" 74 + 75 + local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")" 76 + 77 + # Where there is no possible fallback, this order variant is mandatory 78 + # and must be provided by arch code. Add a comment to the header to 79 + # make this obvious. 80 + # 81 + # Ideally we'd error on a missing definition, but arch code might 82 + # define this order variant as a C function without a preprocessor 83 + # symbol. 84 + if [ -z ${template} ] && [ -z "${order}" ] && ! meta_has_relaxed "${meta}"; then 85 + printf "#define raw_${atomicname} arch_${atomicname}\n\n" 86 + return 87 + fi 88 + 89 + printf "#if defined(arch_${atomicname})\n" 90 + printf "#define raw_${atomicname} arch_${atomicname}\n" 91 + 92 + # Allow FULL/ACQUIRE/RELEASE ops to be defined in terms of RELAXED ops 93 + if [ "${order}" != "_relaxed" ] && meta_has_relaxed "${meta}"; then 94 + printf "#elif defined(arch_${basename}_relaxed)\n" 95 + gen_order_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@" 96 + fi 97 + 98 + # Allow ACQUIRE/RELEASE/RELAXED ops to be defined in terms of FULL ops 99 + if [ ! -z "${order}" ]; then 100 + printf "#elif defined(arch_${basename})\n" 101 + printf "#define raw_${atomicname} arch_${basename}\n" 102 + fi 103 + 104 + printf "#else\n" 105 + if [ ! -z "${template}" ]; then 106 + gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@" 107 + else 108 + printf "#error \"Unable to define raw_${atomicname}\"\n" 109 + fi 110 + 111 + printf "#endif\n\n" 64 112 } 113 + 65 114 66 115 #gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...) 67 116 gen_proto_order_variants() ··· 114 79 local sfx="$1"; shift 115 80 local atomic="$1" 116 81 117 - local basename="arch_${atomic}_${pfx}${name}${sfx}" 82 + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" 118 83 119 - local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "")" 120 - 121 - # If we don't have relaxed atomics, then we don't bother with ordering fallbacks 122 - # read_acquire and set_release need to be templated, though 123 - if ! meta_has_relaxed "${meta}"; then 124 - gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" 125 - 126 - if meta_has_acquire "${meta}"; then 127 - gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" 128 - fi 129 - 130 - if meta_has_release "${meta}"; then 131 - gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" 132 - fi 133 - 134 - return 84 + if meta_has_acquire "${meta}"; then 85 + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" 135 86 fi 136 87 137 - printf "#ifndef ${basename}_relaxed\n" 138 - 139 - if [ ! -z "${template}" ]; then 140 - printf "#ifdef ${basename}\n" 88 + if meta_has_release "${meta}"; then 89 + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" 141 90 fi 142 91 143 - gen_basic_fallbacks "${basename}" 144 - 145 - if [ ! -z "${template}" ]; then 146 - printf "#endif /* ${basename} */\n\n" 147 - gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" 148 - gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" 149 - gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" 150 - gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@" 92 + if meta_has_relaxed "${meta}"; then 93 + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@" 151 94 fi 95 + } 152 96 153 - printf "#else /* ${basename}_relaxed */\n\n" 154 - 155 - gen_order_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" 156 - gen_order_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" 157 - gen_order_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" 158 - 159 - printf "#endif /* ${basename}_relaxed */\n\n" 97 + #gen_basic_fallbacks(basename) 98 + gen_basic_fallbacks() 99 + { 100 + local basename="$1"; shift 101 + cat << EOF 102 + #define raw_${basename}_acquire arch_${basename} 103 + #define raw_${basename}_release arch_${basename} 104 + #define raw_${basename}_relaxed arch_${basename} 105 + EOF 160 106 } 161 107 162 108 gen_order_fallbacks() ··· 146 130 147 131 cat <<EOF 148 132 149 - #ifndef ${xchg}_acquire 150 - #define ${xchg}_acquire(...) \\ 151 - __atomic_op_acquire(${xchg}, __VA_ARGS__) 133 + #define raw_${xchg}_relaxed arch_${xchg}_relaxed 134 + 135 + #ifdef arch_${xchg}_acquire 136 + #define raw_${xchg}_acquire arch_${xchg}_acquire 137 + #else 138 + #define raw_${xchg}_acquire(...) \\ 139 + __atomic_op_acquire(arch_${xchg}, __VA_ARGS__) 152 140 #endif 153 141 154 - #ifndef ${xchg}_release 155 - #define ${xchg}_release(...) \\ 156 - __atomic_op_release(${xchg}, __VA_ARGS__) 142 + #ifdef arch_${xchg}_release 143 + #define raw_${xchg}_release arch_${xchg}_release 144 + #else 145 + #define raw_${xchg}_release(...) \\ 146 + __atomic_op_release(arch_${xchg}, __VA_ARGS__) 157 147 #endif 158 148 159 - #ifndef ${xchg} 160 - #define ${xchg}(...) \\ 161 - __atomic_op_fence(${xchg}, __VA_ARGS__) 149 + #ifdef arch_${xchg} 150 + #define raw_${xchg} arch_${xchg} 151 + #else 152 + #define raw_${xchg}(...) \\ 153 + __atomic_op_fence(arch_${xchg}, __VA_ARGS__) 162 154 #endif 163 155 164 156 EOF 165 157 } 166 158 159 + gen_xchg_order_fallback() 160 + { 161 + local xchg="$1"; shift 162 + local order="$1"; shift 163 + local forder="${order:-_fence}" 164 + 165 + printf "#if defined(arch_${xchg}${order})\n" 166 + printf "#define raw_${xchg}${order} arch_${xchg}${order}\n" 167 + 168 + if [ "${order}" != "_relaxed" ]; then 169 + printf "#elif defined(arch_${xchg}_relaxed)\n" 170 + printf "#define raw_${xchg}${order}(...) \\\\\n" 171 + printf " __atomic_op${forder}(arch_${xchg}, __VA_ARGS__)\n" 172 + fi 173 + 174 + if [ ! -z "${order}" ]; then 175 + printf "#elif defined(arch_${xchg})\n" 176 + printf "#define raw_${xchg}${order} arch_${xchg}\n" 177 + fi 178 + 179 + printf "#else\n" 180 + printf "extern void raw_${xchg}${order}_not_implemented(void);\n" 181 + printf "#define raw_${xchg}${order}(...) raw_${xchg}${order}_not_implemented()\n" 182 + printf "#endif\n\n" 183 + } 184 + 167 185 gen_xchg_fallbacks() 168 186 { 169 187 local xchg="$1"; shift 170 - printf "#ifndef ${xchg}_relaxed\n" 171 188 172 - gen_basic_fallbacks ${xchg} 173 - 174 - printf "#else /* ${xchg}_relaxed */\n" 175 - 176 - gen_order_fallbacks ${xchg} 177 - 178 - printf "#endif /* ${xchg}_relaxed */\n\n" 189 + for order in "" "_acquire" "_release" "_relaxed"; do 190 + gen_xchg_order_fallback "${xchg}" "${order}" 191 + done 179 192 } 180 193 181 194 gen_try_cmpxchg_fallback() ··· 213 168 local order="$1"; shift; 214 169 215 170 cat <<EOF 216 - #ifndef arch_try_${cmpxchg}${order} 217 - #define arch_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\ 171 + #define raw_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\ 218 172 ({ \\ 219 173 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\ 220 - ___r = arch_${cmpxchg}${order}((_ptr), ___o, (_new)); \\ 174 + ___r = raw_${cmpxchg}${order}((_ptr), ___o, (_new)); \\ 221 175 if (unlikely(___r != ___o)) \\ 222 176 *___op = ___r; \\ 223 177 likely(___r == ___o); \\ 224 178 }) 225 - #endif /* arch_try_${cmpxchg}${order} */ 226 - 227 179 EOF 180 + } 181 + 182 + gen_try_cmpxchg_order_fallback() 183 + { 184 + local cmpxchg="$1"; shift 185 + local order="$1"; shift 186 + local forder="${order:-_fence}" 187 + 188 + printf "#if defined(arch_try_${cmpxchg}${order})\n" 189 + printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}${order}\n" 190 + 191 + if [ "${order}" != "_relaxed" ]; then 192 + printf "#elif defined(arch_try_${cmpxchg}_relaxed)\n" 193 + printf "#define raw_try_${cmpxchg}${order}(...) \\\\\n" 194 + printf " __atomic_op${forder}(arch_try_${cmpxchg}, __VA_ARGS__)\n" 195 + fi 196 + 197 + if [ ! -z "${order}" ]; then 198 + printf "#elif defined(arch_try_${cmpxchg})\n" 199 + printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}\n" 200 + fi 201 + 202 + printf "#else\n" 203 + gen_try_cmpxchg_fallback "${cmpxchg}" "${order}" 204 + printf "#endif\n\n" 228 205 } 229 206 230 207 gen_try_cmpxchg_fallbacks() 231 208 { 232 209 local cmpxchg="$1"; shift; 233 210 234 - printf "#ifndef arch_try_${cmpxchg}_relaxed\n" 235 - printf "#ifdef arch_try_${cmpxchg}\n" 236 - 237 - gen_basic_fallbacks "arch_try_${cmpxchg}" 238 - 239 - printf "#endif /* arch_try_${cmpxchg} */\n\n" 240 - 241 211 for order in "" "_acquire" "_release" "_relaxed"; do 242 - gen_try_cmpxchg_fallback "${cmpxchg}" "${order}" 212 + gen_try_cmpxchg_order_fallback "${cmpxchg}" "${order}" 243 213 done 214 + } 244 215 245 - printf "#else /* arch_try_${cmpxchg}_relaxed */\n" 216 + gen_cmpxchg_local_fallbacks() 217 + { 218 + local cmpxchg="$1"; shift 246 219 247 - gen_order_fallbacks "arch_try_${cmpxchg}" 248 - 249 - printf "#endif /* arch_try_${cmpxchg}_relaxed */\n\n" 220 + printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n" 221 + printf "#ifdef arch_try_${cmpxchg}\n" 222 + printf "#define raw_try_${cmpxchg} arch_try_${cmpxchg}\n" 223 + printf "#else\n" 224 + gen_try_cmpxchg_fallback "${cmpxchg}" "" 225 + printf "#endif\n\n" 250 226 } 251 227 252 228 cat << EOF ··· 283 217 284 218 EOF 285 219 286 - for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64" "arch_cmpxchg128"; do 220 + for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128"; do 287 221 gen_xchg_fallbacks "${xchg}" 288 222 done 289 223 ··· 291 225 gen_try_cmpxchg_fallbacks "${cmpxchg}" 292 226 done 293 227 294 - for cmpxchg in "cmpxchg_local" "cmpxchg64_local"; do 295 - gen_try_cmpxchg_fallback "${cmpxchg}" "" 228 + for cmpxchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local"; do 229 + gen_cmpxchg_local_fallbacks "${cmpxchg}" "" 230 + done 231 + 232 + for cmpxchg in "sync_cmpxchg"; do 233 + printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n" 296 234 done 297 235 298 236 grep '^[a-z]' "$1" | while read name meta args; do
-80
scripts/atomic/gen-atomic-raw.sh
··· 1 - #!/bin/sh 2 - # SPDX-License-Identifier: GPL-2.0 3 - 4 - ATOMICDIR=$(dirname $0) 5 - 6 - . ${ATOMICDIR}/atomic-tbl.sh 7 - 8 - #gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...) 9 - gen_proto_order_variant() 10 - { 11 - local meta="$1"; shift 12 - local pfx="$1"; shift 13 - local name="$1"; shift 14 - local sfx="$1"; shift 15 - local order="$1"; shift 16 - local atomic="$1"; shift 17 - local int="$1"; shift 18 - 19 - local atomicname="${atomic}_${pfx}${name}${sfx}${order}" 20 - 21 - local ret="$(gen_ret_type "${meta}" "${int}")" 22 - local params="$(gen_params "${int}" "${atomic}" "$@")" 23 - local args="$(gen_args "$@")" 24 - local retstmt="$(gen_ret_stmt "${meta}")" 25 - 26 - cat <<EOF 27 - static __always_inline ${ret} 28 - raw_${atomicname}(${params}) 29 - { 30 - ${retstmt}arch_${atomicname}(${args}); 31 - } 32 - 33 - EOF 34 - } 35 - 36 - gen_xchg() 37 - { 38 - local xchg="$1"; shift 39 - local order="$1"; shift 40 - 41 - cat <<EOF 42 - #define raw_${xchg}${order}(...) \\ 43 - arch_${xchg}${order}(__VA_ARGS__) 44 - EOF 45 - } 46 - 47 - cat << EOF 48 - // SPDX-License-Identifier: GPL-2.0 49 - 50 - // Generated by $0 51 - // DO NOT MODIFY THIS FILE DIRECTLY 52 - 53 - #ifndef _LINUX_ATOMIC_RAW_H 54 - #define _LINUX_ATOMIC_RAW_H 55 - 56 - EOF 57 - 58 - grep '^[a-z]' "$1" | while read name meta args; do 59 - gen_proto "${meta}" "${name}" "atomic" "int" ${args} 60 - done 61 - 62 - grep '^[a-z]' "$1" | while read name meta args; do 63 - gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} 64 - done 65 - 66 - for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do 67 - for order in "" "_acquire" "_release" "_relaxed"; do 68 - gen_xchg "${xchg}" "${order}" 69 - printf "\n" 70 - done 71 - done 72 - 73 - for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do 74 - gen_xchg "${xchg}" "" 75 - printf "\n" 76 - done 77 - 78 - cat <<EOF 79 - #endif /* _LINUX_ATOMIC_RAW_H */ 80 - EOF
-1
scripts/atomic/gen-atomics.sh
··· 11 11 gen-atomic-instrumented.sh linux/atomic/atomic-instrumented.h 12 12 gen-atomic-long.sh linux/atomic/atomic-long.h 13 13 gen-atomic-fallback.sh linux/atomic/atomic-arch-fallback.h 14 - gen-atomic-raw.sh linux/atomic/atomic-raw.h 15 14 EOF 16 15 while read script header args; do 17 16 /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} ${args} > ${LINUXDIR}/include/${header}