Merge tag 'locking-core-2026-04-13' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

+19 -5

arch/arm64/include/asm/rwonce.h

··· 20 20 ARM64_HAS_LDAPR) 21 21 22 22 /* 23 + * Replace this with typeof_unqual() when minimum compiler versions are 24 + * increased to GCC 14 and Clang 19. For the time being, we need this 25 + * workaround, which relies on function return values dropping qualifiers. 26 + */ 27 + #define __rwonce_typeof_unqual(x) typeof(({ \ 28 + __diag_push() \ 29 + __diag_ignore_all("-Wignored-qualifiers", "") \ 30 + ((typeof(x)(*)(void))0)(); \ 31 + __diag_pop() })) 32 + 33 + /* 23 34 * When building with LTO, there is an increased risk of the compiler 24 35 * converting an address dependency headed by a READ_ONCE() invocation 25 36 * into a control dependency and consequently allowing for harmful ··· 42 31 */ 43 32 #define __READ_ONCE(x) \ 44 33 ({ \ 45 - typeof(&(x)) __x = &(x); \ 46 - int atomic = 1; \ 47 - union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u; \ 34 + auto __x = &(x); \ 35 + auto __ret = (__rwonce_typeof_unqual(*__x) *)__x; \ 36 + /* Hides alias reassignment from Clang's -Wthread-safety. */ \ 37 + auto __retp = &__ret; \ 38 + union { typeof(*__ret) __val; char __c[1]; } __u; \ 39 + *__retp = &__u.__val; \ 48 40 switch (sizeof(x)) { \ 49 41 case 1: \ 50 42 asm volatile(__LOAD_RCPC(b, %w0, %1) \ ··· 70 56 : "Q" (*__x) : "memory"); \ 71 57 break; \ 72 58 default: \ 73 - atomic = 0; \ 59 + __u.__val = *(volatile typeof(*__x) *)__x; \ 74 60 } \ 75 - atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(*__x) *)__x);\ 61 + *__ret; \ 76 62 }) 77 63 78 64 #endif /* !BUILD_VDSO */

+1 -1

drivers/acpi/osl.c

··· 1263 1263 1264 1264 ACPI_DEBUG_PRINT((ACPI_DB_MUTEX, "Deleting semaphore[%p].\n", handle)); 1265 1265 1266 - BUG_ON(!list_empty(&sem->wait_list)); 1266 + BUG_ON(sem->first_waiter); 1267 1267 kfree(sem); 1268 1268 sem = NULL; 1269 1269

+3 -1

include/asm-generic/futex.h

··· 25 25 * argument and comparison of the previous 26 26 * futex value with another constant. 27 27 * 28 - * @encoded_op: encoded operation to execute 28 + * @op: operation to execute 29 + * @oparg: argument of the operation 30 + * @oval: previous value at @uaddr on successful return 29 31 * @uaddr: pointer to user space address 30 32 * 31 33 * Return:

+11 -8

include/linux/cleanup.h

··· 286 286 __no_context_analysis \ 287 287 { _type t = _init; return t; } 288 288 289 - #define EXTEND_CLASS(_name, ext, _init, _init_args...) \ 290 - typedef lock_##_name##_t lock_##_name##ext##_t; \ 289 + #define EXTEND_CLASS_COND(_name, ext, _cond, _init, _init_args...) \ 290 + typedef lock_##_name##_t lock_##_name##ext##_t; \ 291 291 typedef class_##_name##_t class_##_name##ext##_t; \ 292 - static __always_inline void class_##_name##ext##_destructor(class_##_name##_t *p) \ 293 - { class_##_name##_destructor(p); } \ 292 + static __always_inline void class_##_name##ext##_destructor(class_##_name##_t *_T) \ 293 + { if (_cond) return; class_##_name##_destructor(_T); } \ 294 294 static __always_inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \ 295 295 __no_context_analysis \ 296 296 { class_##_name##_t t = _init; return t; } 297 + 298 + #define EXTEND_CLASS(_name, ext, _init, _init_args...) \ 299 + EXTEND_CLASS_COND(_name, ext, 0, _init, _init_args) 297 300 298 301 #define CLASS(_name, var) \ 299 302 class_##_name##_t var __cleanup(class_##_name##_destructor) = \ ··· 397 394 __DEFINE_GUARD_LOCK_PTR(_name, _T) 398 395 399 396 #define DEFINE_GUARD(_name, _type, _lock, _unlock) \ 400 - DEFINE_CLASS(_name, _type, if (!__GUARD_IS_ERR(_T)) { _unlock; }, ({ _lock; _T; }), _type _T); \ 397 + DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \ 401 398 DEFINE_CLASS_IS_GUARD(_name) 402 399 403 400 #define DEFINE_GUARD_COND_4(_name, _ext, _lock, _cond) \ 404 401 __DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true); \ 405 - EXTEND_CLASS(_name, _ext, \ 402 + EXTEND_CLASS_COND(_name, _ext, __GUARD_IS_ERR(*_T), \ 406 403 ({ void *_t = _T; int _RET = (_lock); if (_T && !(_cond)) _t = ERR_PTR(_RET); _t; }), \ 407 404 class_##_name##_t _T) \ 408 405 static __always_inline void * class_##_name##_ext##_lock_ptr(class_##_name##_t *_T) \ ··· 491 488 static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \ 492 489 __no_context_analysis \ 493 490 { \ 494 - if (!__GUARD_IS_ERR(_T->lock)) { _unlock; } \ 491 + if (_T->lock) { _unlock; } \ 495 492 } \ 496 493 \ 497 494 __DEFINE_GUARD_LOCK_PTR(_name, &_T->lock) ··· 568 565 569 566 #define DEFINE_LOCK_GUARD_1_COND_4(_name, _ext, _lock, _cond) \ 570 567 __DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true); \ 571 - EXTEND_CLASS(_name, _ext, \ 568 + EXTEND_CLASS_COND(_name, _ext, __GUARD_IS_ERR(_T->lock), \ 572 569 ({ class_##_name##_t _t = { .lock = l }, *_T = &_t;\ 573 570 int _RET = (_lock); \ 574 571 if (_T->lock && !(_cond)) _T->lock = ERR_PTR(_RET);\

+32

include/linux/compiler-context-analysis.h

··· 320 320 */ 321 321 #define __releases(...) __releases_ctx_lock(__VA_ARGS__) 322 322 323 + /* 324 + * Clang's analysis does not care precisely about the value, only that it is 325 + * either zero or non-zero. So the __cond_acquires() interface might be 326 + * misleading if we say that @ret is the value returned if acquired. Instead, 327 + * provide symbolic variants which we translate. 328 + */ 329 + #define __cond_acquires_impl_not_true(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(0, x) 330 + #define __cond_acquires_impl_not_false(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(1, x) 331 + #define __cond_acquires_impl_not_nonzero(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(0, x) 332 + #define __cond_acquires_impl_not_0(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(1, x) 333 + #define __cond_acquires_impl_not_nonnull(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(0, x) 334 + #define __cond_acquires_impl_not_NULL(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(1, x) 335 + 336 + /** 337 + * __cond_releases() - function attribute, function conditionally 338 + * releases a context lock exclusively 339 + * @ret: abstract value returned by function if context lock releases 340 + * @x: context lock instance pointer 341 + * 342 + * Function attribute declaring that the function conditionally releases the 343 + * given context lock instance @x exclusively. The associated context(s) must 344 + * be active on entry. The function return value @ret denotes when the context 345 + * lock is released. 346 + * 347 + * @ret may be one of: true, false, nonzero, 0, nonnull, NULL. 348 + * 349 + * NOTE: clang does not have a native attribute for this; instead implement 350 + * it as an unconditional release and a conditional acquire for the 351 + * inverted condition -- which is semantically equivalent. 352 + */ 353 + #define __cond_releases(ret, x) __releases(x) __cond_acquires_impl_not_##ret(x) 354 + 323 355 /** 324 356 * __acquire() - function to acquire context lock exclusively 325 357 * @x: context lock instance pointer

+1 -4

include/linux/compiler.h

··· 149 149 #endif 150 150 151 151 #ifndef RELOC_HIDE 152 - # define RELOC_HIDE(ptr, off) \ 153 - ({ unsigned long __ptr; \ 154 - __ptr = (unsigned long) (ptr); \ 155 - (typeof(ptr)) (__ptr + (off)); }) 152 + # define RELOC_HIDE(ptr, off) ((typeof(ptr))((unsigned long)(ptr) + (off))) 156 153 #endif 157 154 158 155 #define absolute_pointer(val) RELOC_HIDE((void *)(val), 0)

+4 -18

include/linux/jump_label.h

··· 87 87 atomic_t enabled; 88 88 #ifdef CONFIG_JUMP_LABEL 89 89 /* 90 - * Note: 91 - * To make anonymous unions work with old compilers, the static 92 - * initialization of them requires brackets. This creates a dependency 93 - * on the order of the struct with the initializers. If any fields 94 - * are added, STATIC_KEY_INIT_TRUE and STATIC_KEY_INIT_FALSE may need 95 - * to be modified. 96 - * 97 90 * bit 0 => 1 if key is initially true 98 91 * 0 if initially false 99 92 * bit 1 => 1 if points to struct static_key_mod ··· 231 238 extern void static_key_disable_cpuslocked(struct static_key *key); 232 239 extern enum jump_label_type jump_label_init_type(struct jump_entry *entry); 233 240 234 - /* 235 - * We should be using ATOMIC_INIT() for initializing .enabled, but 236 - * the inclusion of atomic.h is problematic for inclusion of jump_label.h 237 - * in 'low-level' headers. Thus, we are initializing .enabled with a 238 - * raw value, but have added a BUILD_BUG_ON() to catch any issues in 239 - * jump_label_init() see: kernel/jump_label.c. 240 - */ 241 241 #define STATIC_KEY_INIT_TRUE \ 242 - { .enabled = { 1 }, \ 243 - { .type = JUMP_TYPE_TRUE } } 242 + { .enabled = ATOMIC_INIT(1), \ 243 + .type = JUMP_TYPE_TRUE } 244 244 #define STATIC_KEY_INIT_FALSE \ 245 - { .enabled = { 0 }, \ 246 - { .type = JUMP_TYPE_FALSE } } 245 + { .enabled = ATOMIC_INIT(0), \ 246 + .type = JUMP_TYPE_FALSE } 247 247 248 248 #else /* !CONFIG_JUMP_LABEL */ 249 249

+8 -5

include/linux/mutex.h

··· 79 79 #define __MUTEX_INITIALIZER(lockname) \ 80 80 { .owner = ATOMIC_LONG_INIT(0) \ 81 81 , .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(lockname.wait_lock) \ 82 - , .wait_list = LIST_HEAD_INIT(lockname.wait_list) \ 82 + , .first_waiter = NULL \ 83 83 __DEBUG_MUTEX_INITIALIZER(lockname) \ 84 84 __DEP_MAP_MUTEX_INITIALIZER(lockname) } 85 85 ··· 87 87 struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) 88 88 89 89 #ifdef CONFIG_DEBUG_LOCK_ALLOC 90 - void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_key *key); 90 + void mutex_init_lockdep(struct mutex *lock, const char *name, struct lock_class_key *key); 91 91 92 92 static inline void __mutex_init(struct mutex *lock, const char *name, 93 93 struct lock_class_key *key) 94 94 { 95 - mutex_init_lockep(lock, name, key); 95 + mutex_init_lockdep(lock, name, key); 96 96 } 97 97 #else 98 98 extern void mutex_init_generic(struct mutex *lock); ··· 146 146 { 147 147 mutex_rt_init_generic(lock); 148 148 } 149 - #endif /* !CONFIG_LOCKDEP */ 149 + #endif /* !CONFIG_DEBUG_LOCK_ALLOC */ 150 150 #endif /* CONFIG_PREEMPT_RT */ 151 151 152 152 #ifdef CONFIG_DEBUG_MUTEXES ··· 183 183 */ 184 184 #ifdef CONFIG_DEBUG_LOCK_ALLOC 185 185 extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass) __acquires(lock); 186 - extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock); 186 + extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock) __acquires(lock); 187 187 extern int __must_check mutex_lock_interruptible_nested(struct mutex *lock, 188 188 unsigned int subclass) __cond_acquires(0, lock); 189 189 extern int __must_check _mutex_lock_killable(struct mutex *lock, ··· 253 253 DEFINE_LOCK_GUARD_1(mutex, struct mutex, mutex_lock(_T->lock), mutex_unlock(_T->lock)) 254 254 DEFINE_LOCK_GUARD_1_COND(mutex, _try, mutex_trylock(_T->lock)) 255 255 DEFINE_LOCK_GUARD_1_COND(mutex, _intr, mutex_lock_interruptible(_T->lock), _RET == 0) 256 + DEFINE_LOCK_GUARD_1_COND(mutex, _kill, mutex_lock_killable(_T->lock), _RET == 0) 256 257 DEFINE_LOCK_GUARD_1(mutex_init, struct mutex, mutex_init(_T->lock), /* */) 257 258 258 259 DECLARE_LOCK_GUARD_1_ATTRS(mutex, __acquires(_T), __releases(*(struct mutex **)_T)) ··· 262 261 #define class_mutex_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_try, _T) 263 262 DECLARE_LOCK_GUARD_1_ATTRS(mutex_intr, __acquires(_T), __releases(*(struct mutex **)_T)) 264 263 #define class_mutex_intr_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_intr, _T) 264 + DECLARE_LOCK_GUARD_1_ATTRS(mutex_kill, __acquires(_T), __releases(*(struct mutex **)_T)) 265 + #define class_mutex_kill_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_kill, _T) 265 266 DECLARE_LOCK_GUARD_1_ATTRS(mutex_init, __acquires(_T), __releases(*(struct mutex **)_T)) 266 267 #define class_mutex_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_init, _T) 267 268

+1 -1

include/linux/mutex_types.h

··· 44 44 #ifdef CONFIG_MUTEX_SPIN_ON_OWNER 45 45 struct optimistic_spin_queue osq; /* Spinner MCS lock */ 46 46 #endif 47 - struct list_head wait_list; 47 + struct mutex_waiter *first_waiter __guarded_by(&wait_lock); 48 48 #ifdef CONFIG_DEBUG_MUTEXES 49 49 void *magic; 50 50 #endif

+4 -4

include/linux/rtmutex.h

··· 22 22 23 23 struct rt_mutex_base { 24 24 raw_spinlock_t wait_lock; 25 - struct rb_root_cached waiters; 26 - struct task_struct *owner; 25 + struct rb_root_cached waiters __guarded_by(&wait_lock); 26 + struct task_struct *owner __guarded_by(&wait_lock); 27 27 }; 28 28 29 29 #define __RT_MUTEX_BASE_INITIALIZER(rtbasename) \ ··· 41 41 */ 42 42 static inline bool rt_mutex_base_is_locked(struct rt_mutex_base *lock) 43 43 { 44 - return READ_ONCE(lock->owner) != NULL; 44 + return data_race(READ_ONCE(lock->owner) != NULL); 45 45 } 46 46 47 47 #ifdef CONFIG_RT_MUTEXES ··· 49 49 50 50 static inline struct task_struct *rt_mutex_owner(struct rt_mutex_base *lock) 51 51 { 52 - unsigned long owner = (unsigned long) READ_ONCE(lock->owner); 52 + unsigned long owner = (unsigned long) data_race(READ_ONCE(lock->owner)); 53 53 54 54 return (struct task_struct *) (owner & ~RT_MUTEX_HAS_WAITERS); 55 55 }

+14 -4

include/linux/rwlock.h

··· 30 30 31 31 #ifdef CONFIG_DEBUG_SPINLOCK 32 32 extern void do_raw_read_lock(rwlock_t *lock) __acquires_shared(lock); 33 - extern int do_raw_read_trylock(rwlock_t *lock); 33 + extern int do_raw_read_trylock(rwlock_t *lock) __cond_acquires_shared(true, lock); 34 34 extern void do_raw_read_unlock(rwlock_t *lock) __releases_shared(lock); 35 35 extern void do_raw_write_lock(rwlock_t *lock) __acquires(lock); 36 - extern int do_raw_write_trylock(rwlock_t *lock); 36 + extern int do_raw_write_trylock(rwlock_t *lock) __cond_acquires(true, lock); 37 37 extern void do_raw_write_unlock(rwlock_t *lock) __releases(lock); 38 38 #else 39 39 # define do_raw_read_lock(rwlock) do {__acquire_shared(lock); arch_read_lock(&(rwlock)->raw_lock); } while (0) 40 - # define do_raw_read_trylock(rwlock) arch_read_trylock(&(rwlock)->raw_lock) 40 + static inline int do_raw_read_trylock(rwlock_t *rwlock) 41 + __cond_acquires_shared(true, rwlock) 42 + __no_context_analysis 43 + { 44 + return arch_read_trylock(&(rwlock)->raw_lock); 45 + } 41 46 # define do_raw_read_unlock(rwlock) do {arch_read_unlock(&(rwlock)->raw_lock); __release_shared(lock); } while (0) 42 47 # define do_raw_write_lock(rwlock) do {__acquire(lock); arch_write_lock(&(rwlock)->raw_lock); } while (0) 43 - # define do_raw_write_trylock(rwlock) arch_write_trylock(&(rwlock)->raw_lock) 48 + static inline int do_raw_write_trylock(rwlock_t *rwlock) 49 + __cond_acquires(true, rwlock) 50 + __no_context_analysis 51 + { 52 + return arch_write_trylock(&(rwlock)->raw_lock); 53 + } 44 54 # define do_raw_write_unlock(rwlock) do {arch_write_unlock(&(rwlock)->raw_lock); __release(lock); } while (0) 45 55 #endif 46 56

+4 -2

include/linux/rwlock_api_smp.h

··· 23 23 void __lockfunc _raw_read_lock_irq(rwlock_t *lock) __acquires_shared(lock); 24 24 void __lockfunc _raw_write_lock_irq(rwlock_t *lock) __acquires(lock); 25 25 unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) 26 - __acquires(lock); 26 + __acquires_shared(lock); 27 27 unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) 28 28 __acquires(lock); 29 29 int __lockfunc _raw_read_trylock(rwlock_t *lock) __cond_acquires_shared(true, lock); ··· 36 36 void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) __releases(lock); 37 37 void __lockfunc 38 38 _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) 39 - __releases(lock); 39 + __releases_shared(lock); 40 40 void __lockfunc 41 41 _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) 42 42 __releases(lock); ··· 116 116 #endif 117 117 118 118 static inline int __raw_read_trylock(rwlock_t *lock) 119 + __cond_acquires_shared(true, lock) 119 120 { 120 121 preempt_disable(); 121 122 if (do_raw_read_trylock(lock)) { ··· 128 127 } 129 128 130 129 static inline int __raw_write_trylock(rwlock_t *lock) 130 + __cond_acquires(true, lock) 131 131 { 132 132 preempt_disable(); 133 133 if (do_raw_write_trylock(lock)) {

+4 -4

include/linux/rwsem.h

··· 57 57 struct optimistic_spin_queue osq; /* spinner MCS lock */ 58 58 #endif 59 59 raw_spinlock_t wait_lock; 60 - struct list_head wait_list; 60 + struct rwsem_waiter *first_waiter __guarded_by(&wait_lock); 61 61 #ifdef CONFIG_DEBUG_RWSEMS 62 62 void *magic; 63 63 #endif ··· 106 106 .owner = ATOMIC_LONG_INIT(0), \ 107 107 __RWSEM_OPT_INIT(name) \ 108 108 .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock),\ 109 - .wait_list = LIST_HEAD_INIT((name).wait_list), \ 109 + .first_waiter = NULL, \ 110 110 __RWSEM_DEBUG_INIT(name) \ 111 111 __RWSEM_DEP_MAP_INIT(name) } 112 112 ··· 129 129 * rwsem to see if somebody from an incompatible type is wanting access to the 130 130 * lock. 131 131 */ 132 - static inline int rwsem_is_contended(struct rw_semaphore *sem) 132 + static inline bool rwsem_is_contended(struct rw_semaphore *sem) 133 133 { 134 - return !list_empty(&sem->wait_list); 134 + return data_race(sem->first_waiter != NULL); 135 135 } 136 136 137 137 #if defined(CONFIG_DEBUG_RWSEMS) || defined(CONFIG_DETECT_HUNG_TASK_BLOCKER)

+1 -1

include/linux/sched/signal.h

··· 740 740 741 741 extern struct sighand_struct *lock_task_sighand(struct task_struct *task, 742 742 unsigned long *flags) 743 - __acquires(&task->sighand->siglock); 743 + __cond_acquires(nonnull, &task->sighand->siglock); 744 744 745 745 static inline void unlock_task_sighand(struct task_struct *task, 746 746 unsigned long *flags)

+2 -2

include/linux/semaphore.h

··· 15 15 struct semaphore { 16 16 raw_spinlock_t lock; 17 17 unsigned int count; 18 - struct list_head wait_list; 18 + struct semaphore_waiter *first_waiter; 19 19 20 20 #ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER 21 21 unsigned long last_holder; ··· 33 33 { \ 34 34 .lock = __RAW_SPIN_LOCK_UNLOCKED((name).lock), \ 35 35 .count = n, \ 36 - .wait_list = LIST_HEAD_INIT((name).wait_list) \ 36 + .first_waiter = NULL \ 37 37 __LAST_HOLDER_SEMAPHORE_INITIALIZER \ 38 38 } 39 39

+2 -1

include/linux/spinlock.h

··· 178 178 179 179 #ifdef CONFIG_DEBUG_SPINLOCK 180 180 extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock); 181 - extern int do_raw_spin_trylock(raw_spinlock_t *lock); 181 + extern int do_raw_spin_trylock(raw_spinlock_t *lock) __cond_acquires(true, lock); 182 182 extern void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); 183 183 #else 184 184 static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock) ··· 189 189 } 190 190 191 191 static inline int do_raw_spin_trylock(raw_spinlock_t *lock) 192 + __cond_acquires(true, lock) 192 193 { 193 194 int ret = arch_spin_trylock(&(lock)->raw_lock); 194 195

+10 -10

include/linux/spinlock_up.h

··· 48 48 lock->slock = 1; 49 49 } 50 50 51 - /* 52 - * Read-write spinlocks. No debug version. 53 - */ 54 - #define arch_read_lock(lock) do { barrier(); (void)(lock); } while (0) 55 - #define arch_write_lock(lock) do { barrier(); (void)(lock); } while (0) 56 - #define arch_read_trylock(lock) ({ barrier(); (void)(lock); 1; }) 57 - #define arch_write_trylock(lock) ({ barrier(); (void)(lock); 1; }) 58 - #define arch_read_unlock(lock) do { barrier(); (void)(lock); } while (0) 59 - #define arch_write_unlock(lock) do { barrier(); (void)(lock); } while (0) 60 - 61 51 #else /* DEBUG_SPINLOCK */ 62 52 #define arch_spin_is_locked(lock) ((void)(lock), 0) 63 53 /* for sched/core.c and kernel_lock.c: */ ··· 57 67 #endif /* DEBUG_SPINLOCK */ 58 68 59 69 #define arch_spin_is_contended(lock) (((void)(lock), 0)) 70 + 71 + /* 72 + * Read-write spinlocks. No debug version. 73 + */ 74 + #define arch_read_lock(lock) do { barrier(); (void)(lock); } while (0) 75 + #define arch_write_lock(lock) do { barrier(); (void)(lock); } while (0) 76 + #define arch_read_trylock(lock) ({ barrier(); (void)(lock); 1; }) 77 + #define arch_write_trylock(lock) ({ barrier(); (void)(lock); 1; }) 78 + #define arch_read_unlock(lock) do { barrier(); (void)(lock); } while (0) 79 + #define arch_write_unlock(lock) do { barrier(); (void)(lock); } while (0) 60 80 61 81 #endif /* __LINUX_SPINLOCK_UP_H */

+2 -2

include/linux/ww_mutex.h

··· 181 181 * data structures. 182 182 */ 183 183 static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) 184 - __releases(ctx) __acquires_shared(ctx) __no_context_analysis 184 + __must_hold(ctx) 185 185 { 186 186 #ifdef DEBUG_WW_MUTEXES 187 187 lockdep_assert_held(ctx); ··· 199 199 * mutexes have been released with ww_mutex_unlock. 200 200 */ 201 201 static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) 202 - __releases_shared(ctx) __no_context_analysis 202 + __releases(ctx) __no_context_analysis 203 203 { 204 204 #ifdef CONFIG_DEBUG_LOCK_ALLOC 205 205 mutex_release(&ctx->first_lock_dep_map, _THIS_IP_);

+2

kernel/futex/Makefile

··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 2 3 + CONTEXT_ANALYSIS := y 4 + 3 5 obj-y += core.o syscalls.o pi.o requeue.o waitwake.o

+6 -3

kernel/futex/core.c

··· 864 864 865 865 /* The key must be already stored in q->key. */ 866 866 void futex_q_lock(struct futex_q *q, struct futex_hash_bucket *hb) 867 - __acquires(&hb->lock) 868 867 { 869 868 /* 870 869 * Increment the counter before taking the lock so that ··· 878 879 q->lock_ptr = &hb->lock; 879 880 880 881 spin_lock(&hb->lock); 882 + __acquire(q->lock_ptr); 881 883 } 882 884 883 885 void futex_q_unlock(struct futex_hash_bucket *hb) 884 - __releases(&hb->lock) 885 886 { 886 887 futex_hb_waiters_dec(hb); 887 888 spin_unlock(&hb->lock); ··· 1442 1443 void futex_exit_recursive(struct task_struct *tsk) 1443 1444 { 1444 1445 /* If the state is FUTEX_STATE_EXITING then futex_exit_mutex is held */ 1445 - if (tsk->futex_state == FUTEX_STATE_EXITING) 1446 + if (tsk->futex_state == FUTEX_STATE_EXITING) { 1447 + __assume_ctx_lock(&tsk->futex_exit_mutex); 1446 1448 mutex_unlock(&tsk->futex_exit_mutex); 1449 + } 1447 1450 tsk->futex_state = FUTEX_STATE_DEAD; 1448 1451 } 1449 1452 1450 1453 static void futex_cleanup_begin(struct task_struct *tsk) 1454 + __acquires(&tsk->futex_exit_mutex) 1451 1455 { 1452 1456 /* 1453 1457 * Prevent various race issues against a concurrent incoming waiter ··· 1477 1475 } 1478 1476 1479 1477 static void futex_cleanup_end(struct task_struct *tsk, int state) 1478 + __releases(&tsk->futex_exit_mutex) 1480 1479 { 1481 1480 /* 1482 1481 * Lockless store. The only side effect is that an observer might

+14 -3

kernel/futex/futex.h

··· 217 217 218 218 extern int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, 219 219 enum futex_access rw); 220 - extern void futex_q_lockptr_lock(struct futex_q *q); 220 + extern void futex_q_lockptr_lock(struct futex_q *q) __acquires(q->lock_ptr); 221 221 extern struct hrtimer_sleeper * 222 222 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, 223 223 int flags, u64 range_ns); ··· 311 311 static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb, 312 312 struct task_struct *task) 313 313 __releases(&hb->lock) 314 + __releases(q->lock_ptr) 314 315 { 315 316 __futex_queue(q, hb, task); 316 317 spin_unlock(&hb->lock); 318 + __release(q->lock_ptr); 317 319 } 318 320 319 321 extern void futex_unqueue_pi(struct futex_q *q); ··· 360 358 #endif 361 359 } 362 360 363 - extern void futex_q_lock(struct futex_q *q, struct futex_hash_bucket *hb); 364 - extern void futex_q_unlock(struct futex_hash_bucket *hb); 361 + extern void futex_q_lock(struct futex_q *q, struct futex_hash_bucket *hb) 362 + __acquires(&hb->lock) 363 + __acquires(q->lock_ptr); 365 364 365 + extern void futex_q_unlock(struct futex_hash_bucket *hb) 366 + __releases(&hb->lock); 366 367 367 368 extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb, 368 369 union futex_key *key, ··· 384 379 */ 385 380 static inline void 386 381 double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 382 + __acquires(&hb1->lock) 383 + __acquires(&hb2->lock) 384 + __no_context_analysis 387 385 { 388 386 if (hb1 > hb2) 389 387 swap(hb1, hb2); ··· 398 390 399 391 static inline void 400 392 double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 393 + __releases(&hb1->lock) 394 + __releases(&hb2->lock) 395 + __no_context_analysis 401 396 { 402 397 spin_unlock(&hb1->lock); 403 398 if (hb1 != hb2)

+9

kernel/futex/pi.c

··· 389 389 * Initialize the pi_mutex in locked state and make @p 390 390 * the owner of it: 391 391 */ 392 + __assume_ctx_lock(&pi_state->pi_mutex.wait_lock); 392 393 rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p); 393 394 394 395 /* Store the key for possible exit cleanups: */ ··· 615 614 static int wake_futex_pi(u32 __user *uaddr, u32 uval, 616 615 struct futex_pi_state *pi_state, 617 616 struct rt_mutex_waiter *top_waiter) 617 + __must_hold(&pi_state->pi_mutex.wait_lock) 618 + __releases(&pi_state->pi_mutex.wait_lock) 618 619 { 619 620 struct task_struct *new_owner; 620 621 bool postunlock = false; ··· 673 670 674 671 static int __fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q, 675 672 struct task_struct *argowner) 673 + __must_hold(&q->pi_state->pi_mutex.wait_lock) 674 + __must_hold(q->lock_ptr) 676 675 { 677 676 struct futex_pi_state *pi_state = q->pi_state; 678 677 struct task_struct *oldowner, *newowner; ··· 972 967 * - EAGAIN: The user space value changed. 973 968 */ 974 969 futex_q_unlock(hb); 970 + __release(q.lock_ptr); 975 971 /* 976 972 * Handle the case where the owner is in the middle of 977 973 * exiting. Wait for the exit to complete otherwise ··· 1097 1091 if (res) 1098 1092 ret = (res < 0) ? res : 0; 1099 1093 1094 + __release(&hb->lock); 1100 1095 futex_unqueue_pi(&q); 1101 1096 spin_unlock(q.lock_ptr); 1102 1097 if (q.drop_hb_ref) { ··· 1109 1102 1110 1103 out_unlock_put_key: 1111 1104 futex_q_unlock(hb); 1105 + __release(q.lock_ptr); 1112 1106 goto out; 1113 1107 1114 1108 uaddr_faulted: 1115 1109 futex_q_unlock(hb); 1110 + __release(q.lock_ptr); 1116 1111 1117 1112 ret = fault_in_user_writeable(uaddr); 1118 1113 if (ret)

+4

kernel/futex/waitwake.c

··· 462 462 } 463 463 464 464 futex_q_unlock(hb); 465 + __release(q->lock_ptr); 465 466 } 466 467 __set_current_state(TASK_RUNNING); 467 468 ··· 629 628 630 629 if (ret) { 631 630 futex_q_unlock(hb); 631 + __release(q->lock_ptr); 632 632 633 633 ret = get_user(uval, uaddr); 634 634 if (ret) ··· 643 641 644 642 if (uval != val) { 645 643 futex_q_unlock(hb); 644 + __release(q->lock_ptr); 646 645 return -EWOULDBLOCK; 647 646 } 648 647 649 648 if (key2 && futex_match(&q->key, key2)) { 650 649 futex_q_unlock(hb); 650 + __release(q->lock_ptr); 651 651 return -EINVAL; 652 652 } 653 653

-9

kernel/jump_label.c

··· 529 529 struct static_key *key = NULL; 530 530 struct jump_entry *iter; 531 531 532 - /* 533 - * Since we are initializing the static_key.enabled field with 534 - * with the 'raw' int values (to avoid pulling in atomic.h) in 535 - * jump_label.h, let's make sure that is safe. There are only two 536 - * cases to check since we initialize to 0 or 1. 537 - */ 538 - BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0); 539 - BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1); 540 - 541 532 if (static_key_initialized) 542 533 return; 543 534

+5

kernel/locking/Makefile

··· 3 3 # and is generally not a function of system call inputs. 4 4 KCOV_INSTRUMENT := n 5 5 6 + CONTEXT_ANALYSIS_mutex.o := y 7 + CONTEXT_ANALYSIS_rtmutex_api.o := y 8 + CONTEXT_ANALYSIS_ww_rt_mutex.o := y 9 + CONTEXT_ANALYSIS_rwsem.o := y 10 + 6 11 obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o 7 12 8 13 # Avoid recursion lockdep -> sanitizer -> ... -> lockdep & improve performance.

+1 -4

kernel/locking/mutex-debug.c

··· 37 37 void debug_mutex_wake_waiter(struct mutex *lock, struct mutex_waiter *waiter) 38 38 { 39 39 lockdep_assert_held(&lock->wait_lock); 40 - DEBUG_LOCKS_WARN_ON(list_empty(&lock->wait_list)); 40 + DEBUG_LOCKS_WARN_ON(!lock->first_waiter); 41 41 DEBUG_LOCKS_WARN_ON(waiter->magic != waiter); 42 - DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list)); 43 42 } 44 43 45 44 void debug_mutex_free_waiter(struct mutex_waiter *waiter) ··· 61 62 { 62 63 struct mutex *blocked_on = __get_task_blocked_on(task); 63 64 64 - DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list)); 65 65 DEBUG_LOCKS_WARN_ON(waiter->task != task); 66 66 DEBUG_LOCKS_WARN_ON(blocked_on && blocked_on != lock); 67 67 ··· 72 74 { 73 75 if (likely(debug_locks)) { 74 76 DEBUG_LOCKS_WARN_ON(lock->magic != lock); 75 - DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next); 76 77 } 77 78 } 78 79

+55 -27

kernel/locking/mutex.c

··· 46 46 static void __mutex_init_generic(struct mutex *lock) 47 47 { 48 48 atomic_long_set(&lock->owner, 0); 49 - raw_spin_lock_init(&lock->wait_lock); 50 - INIT_LIST_HEAD(&lock->wait_list); 49 + scoped_guard (raw_spinlock_init, &lock->wait_lock) { 50 + lock->first_waiter = NULL; 51 + } 51 52 #ifdef CONFIG_MUTEX_SPIN_ON_OWNER 52 53 osq_lock_init(&lock->osq); 53 54 #endif ··· 151 150 * follow with a __mutex_trylock() before failing. 152 151 */ 153 152 static __always_inline bool __mutex_trylock_fast(struct mutex *lock) 153 + __cond_acquires(true, lock) 154 154 { 155 155 unsigned long curr = (unsigned long)current; 156 156 unsigned long zero = 0UL; ··· 165 163 } 166 164 167 165 static __always_inline bool __mutex_unlock_fast(struct mutex *lock) 166 + __cond_releases(true, lock) 168 167 { 169 168 unsigned long curr = (unsigned long)current; 170 169 ··· 174 171 175 172 #else /* !CONFIG_DEBUG_LOCK_ALLOC */ 176 173 177 - void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_key *key) 174 + void mutex_init_lockdep(struct mutex *lock, const char *name, struct lock_class_key *key) 178 175 { 179 176 __mutex_init_generic(lock); 180 177 ··· 184 181 debug_check_no_locks_freed((void *)lock, sizeof(*lock)); 185 182 lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_SLEEP); 186 183 } 187 - EXPORT_SYMBOL(mutex_init_lockep); 184 + EXPORT_SYMBOL(mutex_init_lockdep); 188 185 #endif /* !CONFIG_DEBUG_LOCK_ALLOC */ 189 186 190 187 static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag) ··· 197 194 atomic_long_andnot(flag, &lock->owner); 198 195 } 199 196 200 - static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter) 201 - { 202 - return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter; 203 - } 204 - 205 197 /* 206 198 * Add @waiter to a given location in the lock wait_list and set the 207 199 * FLAG_WAITERS flag if it's the first waiter. 208 200 */ 209 201 static void 210 202 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter, 211 - struct list_head *list) 203 + struct mutex_waiter *first) 204 + __must_hold(&lock->wait_lock) 212 205 { 213 206 hung_task_set_blocker(lock, BLOCKER_TYPE_MUTEX); 214 207 debug_mutex_add_waiter(lock, waiter, current); 215 208 216 - list_add_tail(&waiter->list, list); 217 - if (__mutex_waiter_is_first(lock, waiter)) 209 + if (!first) 210 + first = lock->first_waiter; 211 + 212 + if (first) { 213 + list_add_tail(&waiter->list, &first->list); 214 + } else { 215 + INIT_LIST_HEAD(&waiter->list); 216 + lock->first_waiter = waiter; 218 217 __mutex_set_flag(lock, MUTEX_FLAG_WAITERS); 218 + } 219 219 } 220 220 221 221 static void 222 222 __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter) 223 + __must_hold(&lock->wait_lock) 223 224 { 224 - list_del(&waiter->list); 225 - if (likely(list_empty(&lock->wait_list))) 225 + if (list_empty(&waiter->list)) { 226 226 __mutex_clear_flag(lock, MUTEX_FLAGS); 227 + lock->first_waiter = NULL; 228 + } else { 229 + if (lock->first_waiter == waiter) { 230 + lock->first_waiter = list_first_entry(&waiter->list, 231 + struct mutex_waiter, list); 232 + } 233 + list_del(&waiter->list); 234 + } 227 235 228 236 debug_mutex_remove_waiter(lock, waiter, current); 229 237 hung_task_clear_blocker(); ··· 273 259 * We also put the fastpath first in the kernel image, to make sure the 274 260 * branch is predicted by the CPU as default-untaken. 275 261 */ 276 - static void __sched __mutex_lock_slowpath(struct mutex *lock); 262 + static void __sched __mutex_lock_slowpath(struct mutex *lock) 263 + __acquires(lock); 277 264 278 265 /** 279 266 * mutex_lock - acquire the mutex ··· 355 340 * Similarly, stop spinning if we are no longer the 356 341 * first waiter. 357 342 */ 358 - if (waiter && !__mutex_waiter_is_first(lock, waiter)) 343 + if (waiter && data_race(lock->first_waiter != waiter)) 359 344 return false; 360 345 361 346 return true; ··· 540 525 } 541 526 #endif 542 527 543 - static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip); 528 + static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip) 529 + __releases(lock); 544 530 545 531 /** 546 532 * mutex_unlock - release the mutex ··· 581 565 * of a unlocked mutex is not allowed. 582 566 */ 583 567 void __sched ww_mutex_unlock(struct ww_mutex *lock) 568 + __no_context_analysis 584 569 { 585 570 __ww_mutex_unlock(lock); 586 571 mutex_unlock(&lock->base); ··· 595 578 __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass, 596 579 struct lockdep_map *nest_lock, unsigned long ip, 597 580 struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx) 581 + __cond_acquires(0, lock) 598 582 { 599 583 DEFINE_WAKE_Q(wake_q); 600 584 struct mutex_waiter waiter; ··· 663 645 664 646 if (!use_ww_ctx) { 665 647 /* add waiting tasks to the end of the waitqueue (FIFO): */ 666 - __mutex_add_waiter(lock, &waiter, &lock->wait_list); 648 + __mutex_add_waiter(lock, &waiter, NULL); 667 649 } else { 668 650 /* 669 651 * Add in stamp order, waking up waiters that must kill ··· 709 691 710 692 schedule_preempt_disabled(); 711 693 712 - first = __mutex_waiter_is_first(lock, &waiter); 694 + first = lock->first_waiter == &waiter; 713 695 714 696 /* 715 697 * As we likely have been woken up by task ··· 752 734 * Wound-Wait; we stole the lock (!first_waiter), check the 753 735 * waiters as anyone might want to wound us. 754 736 */ 755 - if (!ww_ctx->is_wait_die && 756 - !__mutex_waiter_is_first(lock, &waiter)) 737 + if (!ww_ctx->is_wait_die && lock->first_waiter != &waiter) 757 738 __ww_mutex_check_waiters(lock, ww_ctx, &wake_q); 758 739 } 759 740 ··· 789 772 static int __sched 790 773 __mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass, 791 774 struct lockdep_map *nest_lock, unsigned long ip) 775 + __cond_acquires(0, lock) 792 776 { 793 777 return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, false); 794 778 } ··· 797 779 static int __sched 798 780 __ww_mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass, 799 781 unsigned long ip, struct ww_acquire_ctx *ww_ctx) 782 + __cond_acquires(0, lock) 800 783 { 801 784 return __mutex_lock_common(lock, state, subclass, NULL, ip, ww_ctx, true); 802 785 } ··· 845 826 mutex_lock_nested(struct mutex *lock, unsigned int subclass) 846 827 { 847 828 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_); 829 + __acquire(lock); 848 830 } 849 831 850 832 EXPORT_SYMBOL_GPL(mutex_lock_nested); ··· 854 834 _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest) 855 835 { 856 836 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_); 837 + __acquire(lock); 857 838 } 858 839 EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock); 859 840 ··· 883 862 token = io_schedule_prepare(); 884 863 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 885 864 subclass, NULL, _RET_IP_, NULL, 0); 865 + __acquire(lock); 886 866 io_schedule_finish(token); 887 867 } 888 868 EXPORT_SYMBOL_GPL(mutex_lock_io_nested); 889 869 890 870 static inline int 891 871 ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) 872 + __cond_releases(nonzero, lock) 892 873 { 893 874 #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH 894 875 unsigned tmp; ··· 952 929 * Release the lock, slowpath: 953 930 */ 954 931 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip) 932 + __releases(lock) 955 933 { 956 934 struct task_struct *next = NULL; 935 + struct mutex_waiter *waiter; 957 936 DEFINE_WAKE_Q(wake_q); 958 937 unsigned long owner; 959 938 unsigned long flags; 960 939 961 940 mutex_release(&lock->dep_map, ip); 941 + __release(lock); 962 942 963 943 /* 964 944 * Release the lock before (potentially) taking the spinlock such that ··· 988 962 989 963 raw_spin_lock_irqsave(&lock->wait_lock, flags); 990 964 debug_mutex_unlock(lock); 991 - if (!list_empty(&lock->wait_list)) { 992 - /* get the first entry from the wait-list: */ 993 - struct mutex_waiter *waiter = 994 - list_first_entry(&lock->wait_list, 995 - struct mutex_waiter, list); 996 - 965 + waiter = lock->first_waiter; 966 + if (waiter) { 997 967 next = waiter->task; 998 968 999 969 debug_mutex_wake_waiter(lock, waiter); ··· 1083 1061 1084 1062 static noinline void __sched 1085 1063 __mutex_lock_slowpath(struct mutex *lock) 1064 + __acquires(lock) 1086 1065 { 1087 1066 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_); 1067 + __acquire(lock); 1088 1068 } 1089 1069 1090 1070 static noinline int __sched 1091 1071 __mutex_lock_killable_slowpath(struct mutex *lock) 1072 + __cond_acquires(0, lock) 1092 1073 { 1093 1074 return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_); 1094 1075 } 1095 1076 1096 1077 static noinline int __sched 1097 1078 __mutex_lock_interruptible_slowpath(struct mutex *lock) 1079 + __cond_acquires(0, lock) 1098 1080 { 1099 1081 return __mutex_lock(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_); 1100 1082 } 1101 1083 1102 1084 static noinline int __sched 1103 1085 __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) 1086 + __cond_acquires(0, lock) 1104 1087 { 1105 1088 return __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 0, 1106 1089 _RET_IP_, ctx); ··· 1114 1087 static noinline int __sched 1115 1088 __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock, 1116 1089 struct ww_acquire_ctx *ctx) 1090 + __cond_acquires(0, lock) 1117 1091 { 1118 1092 return __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 0, 1119 1093 _RET_IP_, ctx);

+1

kernel/locking/mutex.h

··· 7 7 * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 8 8 */ 9 9 #ifndef CONFIG_PREEMPT_RT 10 + #include <linux/mutex.h> 10 11 /* 11 12 * This is the control structure for tasks blocked on mutex, which resides 12 13 * on the blocked task's kernel stack:

+17 -1

kernel/locking/rtmutex.c

··· 94 94 95 95 static __always_inline struct task_struct * 96 96 rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner) 97 + __must_hold(&lock->wait_lock) 97 98 { 98 99 unsigned long val = (unsigned long)owner; 99 100 ··· 106 105 107 106 static __always_inline void 108 107 rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner) 108 + __must_hold(&lock->wait_lock) 109 109 { 110 110 /* 111 111 * lock->wait_lock is held but explicit acquire semantics are needed ··· 116 114 } 117 115 118 116 static __always_inline void rt_mutex_clear_owner(struct rt_mutex_base *lock) 117 + __must_hold(&lock->wait_lock) 119 118 { 120 119 /* lock->wait_lock is held so the unlock provides release semantics. */ 121 120 WRITE_ONCE(lock->owner, rt_mutex_owner_encode(lock, NULL)); 122 121 } 123 122 124 123 static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock) 124 + __must_hold(&lock->wait_lock) 125 125 { 126 126 lock->owner = (struct task_struct *) 127 127 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS); ··· 131 127 132 128 static __always_inline void 133 129 fixup_rt_mutex_waiters(struct rt_mutex_base *lock, bool acquire_lock) 130 + __must_hold(&lock->wait_lock) 134 131 { 135 132 unsigned long owner, *p = (unsigned long *) &lock->owner; 136 133 ··· 333 328 } 334 329 335 330 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock) 331 + __must_hold(&lock->wait_lock) 336 332 { 337 333 lock->owner = (struct task_struct *) 338 334 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS); ··· 1212 1206 struct ww_acquire_ctx *ww_ctx, 1213 1207 enum rtmutex_chainwalk chwalk, 1214 1208 struct wake_q_head *wake_q) 1209 + __must_hold(&lock->wait_lock) 1215 1210 { 1216 1211 struct task_struct *owner = rt_mutex_owner(lock); 1217 1212 struct rt_mutex_waiter *top_waiter = waiter; ··· 1256 1249 1257 1250 /* Check whether the waiter should back out immediately */ 1258 1251 rtm = container_of(lock, struct rt_mutex, rtmutex); 1252 + __assume_ctx_lock(&rtm->rtmutex.wait_lock); 1259 1253 res = __ww_mutex_add_waiter(waiter, rtm, ww_ctx, wake_q); 1260 1254 if (res) { 1261 1255 raw_spin_lock(&task->pi_lock); ··· 1364 1356 } 1365 1357 1366 1358 static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock) 1359 + __must_hold(&lock->wait_lock) 1367 1360 { 1368 1361 int ret = try_to_take_rt_mutex(lock, current, NULL); 1369 1362 ··· 1514 1505 * - the VCPU on which owner runs is preempted 1515 1506 */ 1516 1507 if (!owner_on_cpu(owner) || need_resched() || 1517 - !rt_mutex_waiter_is_top_waiter(lock, waiter)) { 1508 + !data_race(rt_mutex_waiter_is_top_waiter(lock, waiter))) { 1518 1509 res = false; 1519 1510 break; 1520 1511 } ··· 1547 1538 */ 1548 1539 static void __sched remove_waiter(struct rt_mutex_base *lock, 1549 1540 struct rt_mutex_waiter *waiter) 1541 + __must_hold(&lock->wait_lock) 1550 1542 { 1551 1543 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); 1552 1544 struct task_struct *owner = rt_mutex_owner(lock); ··· 1623 1613 struct task_struct *owner; 1624 1614 int ret = 0; 1625 1615 1616 + __assume_ctx_lock(&rtm->rtmutex.wait_lock); 1617 + 1626 1618 lockevent_inc(rtmutex_slow_block); 1627 1619 for (;;) { 1628 1620 /* Try to acquire the lock: */ ··· 1670 1658 static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock, 1671 1659 struct rt_mutex_base *lock, 1672 1660 struct rt_mutex_waiter *w) 1661 + __must_hold(&lock->wait_lock) 1673 1662 { 1674 1663 /* 1675 1664 * If the result is not -EDEADLOCK or the caller requested ··· 1707 1694 enum rtmutex_chainwalk chwalk, 1708 1695 struct rt_mutex_waiter *waiter, 1709 1696 struct wake_q_head *wake_q) 1697 + __must_hold(&lock->wait_lock) 1710 1698 { 1711 1699 struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex); 1712 1700 struct ww_mutex *ww = ww_container_of(rtm); 1713 1701 int ret; 1714 1702 1703 + __assume_ctx_lock(&rtm->rtmutex.wait_lock); 1715 1704 lockdep_assert_held(&lock->wait_lock); 1716 1705 lockevent_inc(rtmutex_slowlock); 1717 1706 ··· 1765 1750 struct ww_acquire_ctx *ww_ctx, 1766 1751 unsigned int state, 1767 1752 struct wake_q_head *wake_q) 1753 + __must_hold(&lock->wait_lock) 1768 1754 { 1769 1755 struct rt_mutex_waiter waiter; 1770 1756 int ret;

+2

kernel/locking/rtmutex_api.c

··· 526 526 unsigned int subclass, 527 527 struct lockdep_map *nest_lock, 528 528 unsigned long ip) 529 + __acquires(lock) __no_context_analysis 529 530 { 530 531 int ret; 531 532 ··· 648 647 #endif /* !CONFIG_DEBUG_LOCK_ALLOC */ 649 648 650 649 void __sched mutex_unlock(struct mutex *lock) 650 + __releases(lock) __no_context_analysis 651 651 { 652 652 mutex_release(&lock->dep_map, _RET_IP_); 653 653 __rt_mutex_unlock(&lock->rtmutex);

+19 -8

kernel/locking/rtmutex_common.h

··· 79 79 * PI-futex support (proxy locking functions, etc.): 80 80 */ 81 81 extern void rt_mutex_init_proxy_locked(struct rt_mutex_base *lock, 82 - struct task_struct *proxy_owner); 83 - extern void rt_mutex_proxy_unlock(struct rt_mutex_base *lock); 82 + struct task_struct *proxy_owner) 83 + __must_hold(&lock->wait_lock); 84 + 85 + extern void rt_mutex_proxy_unlock(struct rt_mutex_base *lock) 86 + __must_hold(&lock->wait_lock); 87 + 84 88 extern int __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock, 85 89 struct rt_mutex_waiter *waiter, 86 90 struct task_struct *task, 87 - struct wake_q_head *); 91 + struct wake_q_head *) 92 + __must_hold(&lock->wait_lock); 93 + 88 94 extern int rt_mutex_start_proxy_lock(struct rt_mutex_base *lock, 89 95 struct rt_mutex_waiter *waiter, 90 96 struct task_struct *task); ··· 100 94 extern bool rt_mutex_cleanup_proxy_lock(struct rt_mutex_base *lock, 101 95 struct rt_mutex_waiter *waiter); 102 96 103 - extern int rt_mutex_futex_trylock(struct rt_mutex_base *l); 104 - extern int __rt_mutex_futex_trylock(struct rt_mutex_base *l); 97 + extern int rt_mutex_futex_trylock(struct rt_mutex_base *lock); 98 + extern int __rt_mutex_futex_trylock(struct rt_mutex_base *lock) 99 + __must_hold(&lock->wait_lock); 105 100 106 101 extern void rt_mutex_futex_unlock(struct rt_mutex_base *lock); 107 102 extern bool __rt_mutex_futex_unlock(struct rt_mutex_base *lock, ··· 116 109 */ 117 110 #ifdef CONFIG_RT_MUTEXES 118 111 static inline int rt_mutex_has_waiters(struct rt_mutex_base *lock) 112 + __must_hold(&lock->wait_lock) 119 113 { 120 114 return !RB_EMPTY_ROOT(&lock->waiters.rb_root); 121 115 } ··· 128 120 */ 129 121 static inline bool rt_mutex_waiter_is_top_waiter(struct rt_mutex_base *lock, 130 122 struct rt_mutex_waiter *waiter) 123 + __must_hold(&lock->wait_lock) 131 124 { 132 125 struct rb_node *leftmost = rb_first_cached(&lock->waiters); 133 126 ··· 136 127 } 137 128 138 129 static inline struct rt_mutex_waiter *rt_mutex_top_waiter(struct rt_mutex_base *lock) 130 + __must_hold(&lock->wait_lock) 139 131 { 140 132 struct rb_node *leftmost = rb_first_cached(&lock->waiters); 141 133 struct rt_mutex_waiter *w = NULL; ··· 180 170 181 171 static inline void __rt_mutex_base_init(struct rt_mutex_base *lock) 182 172 { 183 - raw_spin_lock_init(&lock->wait_lock); 184 - lock->waiters = RB_ROOT_CACHED; 185 - lock->owner = NULL; 173 + scoped_guard (raw_spinlock_init, &lock->wait_lock) { 174 + lock->waiters = RB_ROOT_CACHED; 175 + lock->owner = NULL; 176 + } 186 177 } 187 178 188 179 /* Debug functions */

+1

kernel/locking/rwbase_rt.c

··· 186 186 187 187 static inline void __rwbase_write_unlock(struct rwbase_rt *rwb, int bias, 188 188 unsigned long flags) 189 + __releases(&rwb->rtmutex.wait_lock) 189 190 { 190 191 struct rt_mutex_base *rtm = &rwb->rtmutex; 191 192

+80 -33

kernel/locking/rwsem.c

··· 72 72 #c, atomic_long_read(&(sem)->count), \ 73 73 (unsigned long) sem->magic, \ 74 74 atomic_long_read(&(sem)->owner), (long)current, \ 75 - list_empty(&(sem)->wait_list) ? "" : "not ")) \ 75 + rwsem_is_contended(sem) ? "" : "not ")) \ 76 76 debug_locks_off(); \ 77 77 } while (0) 78 78 #else ··· 320 320 sem->magic = sem; 321 321 #endif 322 322 atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE); 323 - raw_spin_lock_init(&sem->wait_lock); 324 - INIT_LIST_HEAD(&sem->wait_list); 325 323 atomic_long_set(&sem->owner, 0L); 324 + scoped_guard (raw_spinlock_init, &sem->wait_lock) { 325 + sem->first_waiter = NULL; 326 + } 326 327 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER 327 328 osq_lock_init(&sem->osq); 328 329 #endif ··· 342 341 unsigned long timeout; 343 342 bool handoff_set; 344 343 }; 345 - #define rwsem_first_waiter(sem) \ 346 - list_first_entry(&sem->wait_list, struct rwsem_waiter, list) 347 344 348 345 enum rwsem_wake_type { 349 346 RWSEM_WAKE_ANY, /* Wake whatever's at head of wait list */ ··· 364 365 */ 365 366 #define MAX_READERS_WAKEUP 0x100 366 367 367 - static inline void 368 - rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) 368 + static inline 369 + bool __rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) 370 + __must_hold(&sem->wait_lock) 369 371 { 370 - lockdep_assert_held(&sem->wait_lock); 371 - list_add_tail(&waiter->list, &sem->wait_list); 372 - /* caller will set RWSEM_FLAG_WAITERS */ 372 + if (list_empty(&waiter->list)) { 373 + sem->first_waiter = NULL; 374 + return false; 375 + } 376 + 377 + if (sem->first_waiter == waiter) { 378 + sem->first_waiter = list_first_entry(&waiter->list, 379 + struct rwsem_waiter, list); 380 + } 381 + list_del(&waiter->list); 382 + 383 + return true; 373 384 } 374 385 375 386 /* ··· 394 385 rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) 395 386 { 396 387 lockdep_assert_held(&sem->wait_lock); 397 - list_del(&waiter->list); 398 - if (likely(!list_empty(&sem->wait_list))) 388 + if (__rwsem_del_waiter(sem, waiter)) 399 389 return true; 400 - 401 390 atomic_long_andnot(RWSEM_FLAG_HANDOFF | RWSEM_FLAG_WAITERS, &sem->count); 402 391 return false; 392 + } 393 + 394 + static inline 395 + struct rwsem_waiter *next_waiter(const struct rw_semaphore *sem, 396 + const struct rwsem_waiter *waiter) 397 + __must_hold(&sem->wait_lock) 398 + { 399 + struct rwsem_waiter *next = list_first_entry(&waiter->list, 400 + struct rwsem_waiter, list); 401 + if (next == sem->first_waiter) 402 + return NULL; 403 + return next; 403 404 } 404 405 405 406 /* ··· 430 411 enum rwsem_wake_type wake_type, 431 412 struct wake_q_head *wake_q) 432 413 { 433 - struct rwsem_waiter *waiter, *tmp; 414 + struct rwsem_waiter *waiter, *next; 434 415 long oldcount, woken = 0, adjustment = 0; 435 416 struct list_head wlist; 436 417 ··· 440 421 * Take a peek at the queue head waiter such that we can determine 441 422 * the wakeup(s) to perform. 442 423 */ 443 - waiter = rwsem_first_waiter(sem); 424 + waiter = sem->first_waiter; 444 425 445 426 if (waiter->type == RWSEM_WAITING_FOR_WRITE) { 446 427 if (wake_type == RWSEM_WAKE_ANY) { ··· 525 506 * put them into wake_q to be woken up later. 526 507 */ 527 508 INIT_LIST_HEAD(&wlist); 528 - list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) { 509 + do { 510 + next = next_waiter(sem, waiter); 529 511 if (waiter->type == RWSEM_WAITING_FOR_WRITE) 530 512 continue; 531 513 532 514 woken++; 533 515 list_move_tail(&waiter->list, &wlist); 516 + if (sem->first_waiter == waiter) 517 + sem->first_waiter = next; 534 518 535 519 /* 536 520 * Limit # of readers that can be woken up per wakeup call. 537 521 */ 538 522 if (unlikely(woken >= MAX_READERS_WAKEUP)) 539 523 break; 540 - } 524 + } while ((waiter = next) != NULL); 541 525 542 526 adjustment = woken * RWSEM_READER_BIAS - adjustment; 543 527 lockevent_cond_inc(rwsem_wake_reader, woken); 544 528 545 529 oldcount = atomic_long_read(&sem->count); 546 - if (list_empty(&sem->wait_list)) { 530 + if (!sem->first_waiter) { 547 531 /* 548 532 * Combined with list_move_tail() above, this implies 549 533 * rwsem_del_waiter(). ··· 567 545 atomic_long_add(adjustment, &sem->count); 568 546 569 547 /* 2nd pass */ 570 - list_for_each_entry_safe(waiter, tmp, &wlist, list) { 548 + list_for_each_entry_safe(waiter, next, &wlist, list) { 571 549 struct task_struct *tsk; 572 550 573 551 tsk = waiter->task; ··· 599 577 struct wake_q_head *wake_q) 600 578 __releases(&sem->wait_lock) 601 579 { 602 - bool first = rwsem_first_waiter(sem) == waiter; 580 + bool first = sem->first_waiter == waiter; 603 581 604 582 wake_q_init(wake_q); 605 583 ··· 624 602 */ 625 603 static inline bool rwsem_try_write_lock(struct rw_semaphore *sem, 626 604 struct rwsem_waiter *waiter) 605 + __must_hold(&sem->wait_lock) 627 606 { 628 - struct rwsem_waiter *first = rwsem_first_waiter(sem); 607 + struct rwsem_waiter *first = sem->first_waiter; 629 608 long count, new; 630 609 631 610 lockdep_assert_held(&sem->wait_lock); ··· 662 639 new |= RWSEM_WRITER_LOCKED; 663 640 new &= ~RWSEM_FLAG_HANDOFF; 664 641 665 - if (list_is_singular(&sem->wait_list)) 642 + if (list_empty(&first->list)) 666 643 new &= ~RWSEM_FLAG_WAITERS; 667 644 } 668 645 } while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new)); ··· 682 659 * Have rwsem_try_write_lock() fully imply rwsem_del_waiter() on 683 660 * success. 684 661 */ 685 - list_del(&waiter->list); 662 + __rwsem_del_waiter(sem, waiter); 663 + 686 664 rwsem_set_owner(sem); 687 665 return true; 688 666 } ··· 1018 994 { 1019 995 long adjustment = -RWSEM_READER_BIAS; 1020 996 long rcnt = (count >> RWSEM_READER_SHIFT); 1021 - struct rwsem_waiter waiter; 997 + struct rwsem_waiter waiter, *first; 1022 998 DEFINE_WAKE_Q(wake_q); 1023 999 1024 1000 /* ··· 1043 1019 */ 1044 1020 if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) { 1045 1021 raw_spin_lock_irq(&sem->wait_lock); 1046 - if (!list_empty(&sem->wait_list)) 1022 + if (sem->first_waiter) 1047 1023 rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, 1048 1024 &wake_q); 1049 1025 raw_spin_unlock_irq(&sem->wait_lock); ··· 1059 1035 waiter.handoff_set = false; 1060 1036 1061 1037 raw_spin_lock_irq(&sem->wait_lock); 1062 - if (list_empty(&sem->wait_list)) { 1038 + first = sem->first_waiter; 1039 + if (!first) { 1063 1040 /* 1064 1041 * In case the wait queue is empty and the lock isn't owned 1065 1042 * by a writer, this reader can exit the slowpath and return ··· 1076 1051 return sem; 1077 1052 } 1078 1053 adjustment += RWSEM_FLAG_WAITERS; 1054 + INIT_LIST_HEAD(&waiter.list); 1055 + sem->first_waiter = &waiter; 1056 + } else { 1057 + list_add_tail(&waiter.list, &first->list); 1079 1058 } 1080 - rwsem_add_waiter(sem, &waiter); 1081 1059 1082 1060 /* we're now waiting on the lock, but no longer actively locking */ 1083 1061 count = atomic_long_add_return(adjustment, &sem->count); ··· 1138 1110 static struct rw_semaphore __sched * 1139 1111 rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) 1140 1112 { 1141 - struct rwsem_waiter waiter; 1113 + struct rwsem_waiter waiter, *first; 1142 1114 DEFINE_WAKE_Q(wake_q); 1143 1115 1144 1116 /* do optimistic spinning and steal lock if possible */ ··· 1157 1129 waiter.handoff_set = false; 1158 1130 1159 1131 raw_spin_lock_irq(&sem->wait_lock); 1160 - rwsem_add_waiter(sem, &waiter); 1161 1132 1162 - /* we're now waiting on the lock */ 1163 - if (rwsem_first_waiter(sem) != &waiter) { 1133 + first = sem->first_waiter; 1134 + if (first) { 1135 + list_add_tail(&waiter.list, &first->list); 1164 1136 rwsem_cond_wake_waiter(sem, atomic_long_read(&sem->count), 1165 1137 &wake_q); 1166 1138 if (!wake_q_empty(&wake_q)) { ··· 1173 1145 raw_spin_lock_irq(&sem->wait_lock); 1174 1146 } 1175 1147 } else { 1148 + INIT_LIST_HEAD(&waiter.list); 1149 + sem->first_waiter = &waiter; 1176 1150 atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count); 1177 1151 } 1178 1152 ··· 1248 1218 1249 1219 raw_spin_lock_irqsave(&sem->wait_lock, flags); 1250 1220 1251 - if (!list_empty(&sem->wait_list)) 1221 + if (sem->first_waiter) 1252 1222 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); 1253 1223 1254 1224 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); ··· 1269 1239 1270 1240 raw_spin_lock_irqsave(&sem->wait_lock, flags); 1271 1241 1272 - if (!list_empty(&sem->wait_list)) 1242 + if (sem->first_waiter) 1273 1243 rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q); 1274 1244 1275 1245 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); ··· 1562 1532 * lock for reading 1563 1533 */ 1564 1534 void __sched down_read(struct rw_semaphore *sem) 1535 + __no_context_analysis 1565 1536 { 1566 1537 might_sleep(); 1567 1538 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); ··· 1572 1541 EXPORT_SYMBOL(down_read); 1573 1542 1574 1543 int __sched down_read_interruptible(struct rw_semaphore *sem) 1544 + __no_context_analysis 1575 1545 { 1576 1546 might_sleep(); 1577 1547 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); ··· 1587 1555 EXPORT_SYMBOL(down_read_interruptible); 1588 1556 1589 1557 int __sched down_read_killable(struct rw_semaphore *sem) 1558 + __no_context_analysis 1590 1559 { 1591 1560 might_sleep(); 1592 1561 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); ··· 1605 1572 * trylock for reading -- returns 1 if successful, 0 if contention 1606 1573 */ 1607 1574 int down_read_trylock(struct rw_semaphore *sem) 1575 + __no_context_analysis 1608 1576 { 1609 1577 int ret = __down_read_trylock(sem); 1610 1578 ··· 1619 1585 * lock for writing 1620 1586 */ 1621 1587 void __sched down_write(struct rw_semaphore *sem) 1588 + __no_context_analysis 1622 1589 { 1623 1590 might_sleep(); 1624 1591 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); ··· 1631 1596 * lock for writing 1632 1597 */ 1633 1598 int __sched down_write_killable(struct rw_semaphore *sem) 1599 + __no_context_analysis 1634 1600 { 1635 1601 might_sleep(); 1636 1602 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); ··· 1650 1614 * trylock for writing -- returns 1 if successful, 0 if contention 1651 1615 */ 1652 1616 int down_write_trylock(struct rw_semaphore *sem) 1617 + __no_context_analysis 1653 1618 { 1654 1619 int ret = __down_write_trylock(sem); 1655 1620 ··· 1665 1628 * release a read lock 1666 1629 */ 1667 1630 void up_read(struct rw_semaphore *sem) 1631 + __no_context_analysis 1668 1632 { 1669 1633 rwsem_release(&sem->dep_map, _RET_IP_); 1670 1634 __up_read(sem); ··· 1676 1638 * release a write lock 1677 1639 */ 1678 1640 void up_write(struct rw_semaphore *sem) 1641 + __no_context_analysis 1679 1642 { 1680 1643 rwsem_release(&sem->dep_map, _RET_IP_); 1681 1644 __up_write(sem); ··· 1687 1648 * downgrade write lock to read lock 1688 1649 */ 1689 1650 void downgrade_write(struct rw_semaphore *sem) 1651 + __no_context_analysis 1690 1652 { 1691 1653 lock_downgrade(&sem->dep_map, _RET_IP_); 1692 1654 __downgrade_write(sem); ··· 1697 1657 #ifdef CONFIG_DEBUG_LOCK_ALLOC 1698 1658 1699 1659 void down_read_nested(struct rw_semaphore *sem, int subclass) 1660 + __no_context_analysis 1700 1661 { 1701 1662 might_sleep(); 1702 1663 rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); ··· 1706 1665 EXPORT_SYMBOL(down_read_nested); 1707 1666 1708 1667 int down_read_killable_nested(struct rw_semaphore *sem, int subclass) 1668 + __no_context_analysis 1709 1669 { 1710 1670 might_sleep(); 1711 1671 rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); ··· 1721 1679 EXPORT_SYMBOL(down_read_killable_nested); 1722 1680 1723 1681 void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest) 1682 + __no_context_analysis 1724 1683 { 1725 1684 might_sleep(); 1726 1685 rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_); ··· 1730 1687 EXPORT_SYMBOL(_down_write_nest_lock); 1731 1688 1732 1689 void down_read_non_owner(struct rw_semaphore *sem) 1690 + __no_context_analysis 1733 1691 { 1734 1692 might_sleep(); 1735 1693 __down_read(sem); ··· 1745 1701 EXPORT_SYMBOL(down_read_non_owner); 1746 1702 1747 1703 void down_write_nested(struct rw_semaphore *sem, int subclass) 1704 + __no_context_analysis 1748 1705 { 1749 1706 might_sleep(); 1750 1707 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); ··· 1754 1709 EXPORT_SYMBOL(down_write_nested); 1755 1710 1756 1711 int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass) 1712 + __no_context_analysis 1757 1713 { 1758 1714 might_sleep(); 1759 1715 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); ··· 1770 1724 EXPORT_SYMBOL(down_write_killable_nested); 1771 1725 1772 1726 void up_read_non_owner(struct rw_semaphore *sem) 1727 + __no_context_analysis 1773 1728 { 1774 1729 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); 1775 1730 __up_read(sem);

+30 -9

kernel/locking/semaphore.c

··· 21 21 * too. 22 22 * 23 23 * The ->count variable represents how many more tasks can acquire this 24 - * semaphore. If it's zero, there may be tasks waiting on the wait_list. 24 + * semaphore. If it's zero, there may be waiters. 25 25 */ 26 26 27 27 #include <linux/compiler.h> ··· 226 226 227 227 hung_task_sem_clear_if_holder(sem); 228 228 229 - if (likely(list_empty(&sem->wait_list))) 229 + if (likely(!sem->first_waiter)) 230 230 sem->count++; 231 231 else 232 232 __up(sem, &wake_q); ··· 244 244 bool up; 245 245 }; 246 246 247 + static inline 248 + void sem_del_waiter(struct semaphore *sem, struct semaphore_waiter *waiter) 249 + { 250 + if (list_empty(&waiter->list)) { 251 + sem->first_waiter = NULL; 252 + return; 253 + } 254 + 255 + if (sem->first_waiter == waiter) { 256 + sem->first_waiter = list_first_entry(&waiter->list, 257 + struct semaphore_waiter, list); 258 + } 259 + list_del(&waiter->list); 260 + } 261 + 247 262 /* 248 263 * Because this function is inlined, the 'state' parameter will be 249 264 * constant, and thus optimised away by the compiler. Likewise the ··· 267 252 static inline int __sched ___down_common(struct semaphore *sem, long state, 268 253 long timeout) 269 254 { 270 - struct semaphore_waiter waiter; 255 + struct semaphore_waiter waiter, *first; 271 256 272 - list_add_tail(&waiter.list, &sem->wait_list); 257 + first = sem->first_waiter; 258 + if (first) { 259 + list_add_tail(&waiter.list, &first->list); 260 + } else { 261 + INIT_LIST_HEAD(&waiter.list); 262 + sem->first_waiter = &waiter; 263 + } 273 264 waiter.task = current; 274 265 waiter.up = false; 275 266 ··· 295 274 } 296 275 297 276 timed_out: 298 - list_del(&waiter.list); 277 + sem_del_waiter(sem, &waiter); 299 278 return -ETIME; 300 279 301 280 interrupted: 302 - list_del(&waiter.list); 281 + sem_del_waiter(sem, &waiter); 303 282 return -EINTR; 304 283 } 305 284 ··· 342 321 static noinline void __sched __up(struct semaphore *sem, 343 322 struct wake_q_head *wake_q) 344 323 { 345 - struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list, 346 - struct semaphore_waiter, list); 347 - list_del(&waiter->list); 324 + struct semaphore_waiter *waiter = sem->first_waiter; 325 + 326 + sem_del_waiter(sem, waiter); 348 327 waiter->up = true; 349 328 wake_q_add(wake_q, waiter->task); 350 329 }

+8 -4

kernel/locking/spinlock.c

··· 64 64 * time (making _this_ CPU preemptible if possible), and we also signal 65 65 * towards that other CPU that it should break the lock ASAP. 66 66 */ 67 - #define BUILD_LOCK_OPS(op, locktype) \ 67 + #define BUILD_LOCK_OPS(op, locktype, lock_ctx_op) \ 68 68 static void __lockfunc __raw_##op##_lock(locktype##_t *lock) \ 69 + lock_ctx_op(lock) \ 69 70 { \ 70 71 for (;;) { \ 71 72 preempt_disable(); \ ··· 79 78 } \ 80 79 \ 81 80 static unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \ 81 + lock_ctx_op(lock) \ 82 82 { \ 83 83 unsigned long flags; \ 84 84 \ ··· 98 96 } \ 99 97 \ 100 98 static void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock) \ 99 + lock_ctx_op(lock) \ 101 100 { \ 102 101 _raw_##op##_lock_irqsave(lock); \ 103 102 } \ 104 103 \ 105 104 static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ 105 + lock_ctx_op(lock) \ 106 106 { \ 107 107 unsigned long flags; \ 108 108 \ ··· 127 123 * __[spin|read|write]_lock_irqsave() 128 124 * __[spin|read|write]_lock_bh() 129 125 */ 130 - BUILD_LOCK_OPS(spin, raw_spinlock); 126 + BUILD_LOCK_OPS(spin, raw_spinlock, __acquires); 131 127 132 128 #ifndef CONFIG_PREEMPT_RT 133 - BUILD_LOCK_OPS(read, rwlock); 134 - BUILD_LOCK_OPS(write, rwlock); 129 + BUILD_LOCK_OPS(read, rwlock, __acquires_shared); 130 + BUILD_LOCK_OPS(write, rwlock, __acquires); 135 131 #endif 136 132 137 133 #endif

+30 -19

kernel/locking/ww_mutex.h

··· 4 4 5 5 #define MUTEX mutex 6 6 #define MUTEX_WAITER mutex_waiter 7 + #define WAIT_LOCK wait_lock 7 8 8 9 static inline struct mutex_waiter * 9 10 __ww_waiter_first(struct mutex *lock) 11 + __must_hold(&lock->wait_lock) 10 12 { 11 - struct mutex_waiter *w; 12 - 13 - w = list_first_entry(&lock->wait_list, struct mutex_waiter, list); 14 - if (list_entry_is_head(w, &lock->wait_list, list)) 15 - return NULL; 16 - 17 - return w; 13 + return lock->first_waiter; 18 14 } 19 15 20 16 static inline struct mutex_waiter * 21 17 __ww_waiter_next(struct mutex *lock, struct mutex_waiter *w) 18 + __must_hold(&lock->wait_lock) 22 19 { 23 20 w = list_next_entry(w, list); 24 - if (list_entry_is_head(w, &lock->wait_list, list)) 21 + if (lock->first_waiter == w) 25 22 return NULL; 26 23 27 24 return w; ··· 26 29 27 30 static inline struct mutex_waiter * 28 31 __ww_waiter_prev(struct mutex *lock, struct mutex_waiter *w) 32 + __must_hold(&lock->wait_lock) 29 33 { 30 34 w = list_prev_entry(w, list); 31 - if (list_entry_is_head(w, &lock->wait_list, list)) 35 + if (lock->first_waiter == w) 32 36 return NULL; 33 37 34 38 return w; ··· 37 39 38 40 static inline struct mutex_waiter * 39 41 __ww_waiter_last(struct mutex *lock) 42 + __must_hold(&lock->wait_lock) 40 43 { 41 - struct mutex_waiter *w; 44 + struct mutex_waiter *w = lock->first_waiter; 42 45 43 - w = list_last_entry(&lock->wait_list, struct mutex_waiter, list); 44 - if (list_entry_is_head(w, &lock->wait_list, list)) 45 - return NULL; 46 - 46 + if (w) 47 + w = list_prev_entry(w, list); 47 48 return w; 48 49 } 49 50 50 51 static inline void 51 52 __ww_waiter_add(struct mutex *lock, struct mutex_waiter *waiter, struct mutex_waiter *pos) 53 + __must_hold(&lock->wait_lock) 52 54 { 53 - struct list_head *p = &lock->wait_list; 54 - if (pos) 55 - p = &pos->list; 56 - __mutex_add_waiter(lock, waiter, p); 55 + __mutex_add_waiter(lock, waiter, pos); 57 56 } 58 57 59 58 static inline struct task_struct * ··· 66 71 } 67 72 68 73 static inline void lock_wait_lock(struct mutex *lock, unsigned long *flags) 74 + __acquires(&lock->wait_lock) 69 75 { 70 76 raw_spin_lock_irqsave(&lock->wait_lock, *flags); 71 77 } 72 78 73 79 static inline void unlock_wait_lock(struct mutex *lock, unsigned long *flags) 80 + __releases(&lock->wait_lock) 74 81 { 75 82 raw_spin_unlock_irqrestore(&lock->wait_lock, *flags); 76 83 } 77 84 78 85 static inline void lockdep_assert_wait_lock_held(struct mutex *lock) 86 + __must_hold(&lock->wait_lock) 79 87 { 80 88 lockdep_assert_held(&lock->wait_lock); 81 89 } ··· 87 89 88 90 #define MUTEX rt_mutex 89 91 #define MUTEX_WAITER rt_mutex_waiter 92 + #define WAIT_LOCK rtmutex.wait_lock 90 93 91 94 static inline struct rt_mutex_waiter * 92 95 __ww_waiter_first(struct rt_mutex *lock) 96 + __must_hold(&lock->rtmutex.wait_lock) 93 97 { 94 98 struct rb_node *n = rb_first(&lock->rtmutex.waiters.rb_root); 95 99 if (!n) ··· 119 119 120 120 static inline struct rt_mutex_waiter * 121 121 __ww_waiter_last(struct rt_mutex *lock) 122 + __must_hold(&lock->rtmutex.wait_lock) 122 123 { 123 124 struct rb_node *n = rb_last(&lock->rtmutex.waiters.rb_root); 124 125 if (!n) ··· 141 140 142 141 static inline bool 143 142 __ww_mutex_has_waiters(struct rt_mutex *lock) 143 + __must_hold(&lock->rtmutex.wait_lock) 144 144 { 145 145 return rt_mutex_has_waiters(&lock->rtmutex); 146 146 } 147 147 148 148 static inline void lock_wait_lock(struct rt_mutex *lock, unsigned long *flags) 149 + __acquires(&lock->rtmutex.wait_lock) 149 150 { 150 151 raw_spin_lock_irqsave(&lock->rtmutex.wait_lock, *flags); 151 152 } 152 153 153 154 static inline void unlock_wait_lock(struct rt_mutex *lock, unsigned long *flags) 155 + __releases(&lock->rtmutex.wait_lock) 154 156 { 155 157 raw_spin_unlock_irqrestore(&lock->rtmutex.wait_lock, *flags); 156 158 } 157 159 158 160 static inline void lockdep_assert_wait_lock_held(struct rt_mutex *lock) 161 + __must_hold(&lock->rtmutex.wait_lock) 159 162 { 160 163 lockdep_assert_held(&lock->rtmutex.wait_lock); 161 164 } ··· 312 307 struct ww_acquire_ctx *ww_ctx, 313 308 struct ww_acquire_ctx *hold_ctx, 314 309 struct wake_q_head *wake_q) 310 + __must_hold(&lock->WAIT_LOCK) 315 311 { 316 312 struct task_struct *owner = __ww_mutex_owner(lock); 317 313 ··· 377 371 static void 378 372 __ww_mutex_check_waiters(struct MUTEX *lock, struct ww_acquire_ctx *ww_ctx, 379 373 struct wake_q_head *wake_q) 374 + __must_hold(&lock->WAIT_LOCK) 380 375 { 381 376 struct MUTEX_WAITER *cur; 382 377 ··· 404 397 { 405 398 DEFINE_WAKE_Q(wake_q); 406 399 unsigned long flags; 400 + bool has_waiters; 407 401 408 402 ww_mutex_lock_acquired(lock, ctx); 409 403 ··· 426 418 * __ww_mutex_add_waiter() and makes sure we either observe ww->ctx 427 419 * and/or !empty list. 428 420 */ 429 - if (likely(!__ww_mutex_has_waiters(&lock->base))) 421 + has_waiters = data_race(__ww_mutex_has_waiters(&lock->base)); 422 + if (likely(!has_waiters)) 430 423 return; 431 424 432 425 /* ··· 473 464 static inline int 474 465 __ww_mutex_check_kill(struct MUTEX *lock, struct MUTEX_WAITER *waiter, 475 466 struct ww_acquire_ctx *ctx) 467 + __must_hold(&lock->WAIT_LOCK) 476 468 { 477 469 struct ww_mutex *ww = container_of(lock, struct ww_mutex, base); 478 470 struct ww_acquire_ctx *hold_ctx = READ_ONCE(ww->ctx); ··· 524 514 struct MUTEX *lock, 525 515 struct ww_acquire_ctx *ww_ctx, 526 516 struct wake_q_head *wake_q) 517 + __must_hold(&lock->WAIT_LOCK) 527 518 { 528 519 struct MUTEX_WAITER *cur, *pos = NULL; 529 520 bool is_wait_die;

+1

kernel/locking/ww_rt_mutex.c

··· 90 90 EXPORT_SYMBOL(ww_mutex_lock_interruptible); 91 91 92 92 void __sched ww_mutex_unlock(struct ww_mutex *lock) 93 + __no_context_analysis 93 94 { 94 95 struct rt_mutex *rtm = &lock->base; 95 96

+8

lib/Kconfig.debug

··· 1618 1618 int "Size for MAX_STACK_TRACE_ENTRIES (as Nth power of 2)" 1619 1619 depends on LOCKDEP && !LOCKDEP_SMALL 1620 1620 range 10 26 1621 + default 21 if KASAN 1621 1622 default 19 1622 1623 help 1623 1624 Try increasing this value if you hit "BUG: MAX_STACK_TRACE_ENTRIES too low!" message. 1625 + 1626 + KASAN significantly increases stack trace consumption because its 1627 + slab tracking interacts with lockdep's dependency validation under 1628 + PREEMPT_FULL, creating a feedback loop. The higher default when 1629 + KASAN is enabled costs ~12MB extra, which is negligible compared to 1630 + KASAN's own shadow memory overhead. 1624 1631 1625 1632 config LOCKDEP_STACK_TRACE_HASH_BITS 1626 1633 int "Size for STACK_TRACE_HASH_SIZE (as Nth power of 2)" 1627 1634 depends on LOCKDEP && !LOCKDEP_SMALL 1628 1635 range 10 26 1636 + default 16 if KASAN 1629 1637 default 14 1630 1638 help 1631 1639 Try increasing this value if you need large STACK_TRACE_HASH_SIZE.

+11

lib/test_context-analysis.c

··· 596 596 597 597 ww_mutex_destroy(&d->mtx); 598 598 } 599 + 600 + static DEFINE_PER_CPU(raw_spinlock_t, test_per_cpu_lock); 601 + 602 + static void __used test_per_cpu(int cpu) 603 + { 604 + raw_spin_lock(&per_cpu(test_per_cpu_lock, cpu)); 605 + raw_spin_unlock(&per_cpu(test_per_cpu_lock, cpu)); 606 + 607 + raw_spin_lock(per_cpu_ptr(&test_per_cpu_lock, cpu)); 608 + raw_spin_unlock(per_cpu_ptr(&test_per_cpu_lock, cpu)); 609 + }

+50 -102

rust/helpers/atomic_ext.c

··· 4 4 #include <asm/rwonce.h> 5 5 #include <linux/atomic.h> 6 6 7 - __rust_helper s8 rust_helper_atomic_i8_read(s8 *ptr) 8 - { 9 - return READ_ONCE(*ptr); 7 + #define GEN_READ_HELPER(tname, type) \ 8 + __rust_helper type rust_helper_atomic_##tname##_read(type *ptr) \ 9 + { \ 10 + return READ_ONCE(*ptr); \ 10 11 } 11 12 12 - __rust_helper s8 rust_helper_atomic_i8_read_acquire(s8 *ptr) 13 - { 14 - return smp_load_acquire(ptr); 13 + #define GEN_SET_HELPER(tname, type) \ 14 + __rust_helper void rust_helper_atomic_##tname##_set(type *ptr, type val) \ 15 + { \ 16 + WRITE_ONCE(*ptr, val); \ 15 17 } 16 18 17 - __rust_helper s16 rust_helper_atomic_i16_read(s16 *ptr) 18 - { 19 - return READ_ONCE(*ptr); 19 + #define GEN_READ_ACQUIRE_HELPER(tname, type) \ 20 + __rust_helper type rust_helper_atomic_##tname##_read_acquire(type *ptr) \ 21 + { \ 22 + return smp_load_acquire(ptr); \ 20 23 } 21 24 22 - __rust_helper s16 rust_helper_atomic_i16_read_acquire(s16 *ptr) 23 - { 24 - return smp_load_acquire(ptr); 25 + #define GEN_SET_RELEASE_HELPER(tname, type) \ 26 + __rust_helper void rust_helper_atomic_##tname##_set_release(type *ptr, type val)\ 27 + { \ 28 + smp_store_release(ptr, val); \ 25 29 } 26 30 27 - __rust_helper void rust_helper_atomic_i8_set(s8 *ptr, s8 val) 28 - { 29 - WRITE_ONCE(*ptr, val); 30 - } 31 + #define GEN_READ_SET_HELPERS(tname, type) \ 32 + GEN_READ_HELPER(tname, type) \ 33 + GEN_SET_HELPER(tname, type) \ 34 + GEN_READ_ACQUIRE_HELPER(tname, type) \ 35 + GEN_SET_RELEASE_HELPER(tname, type) \ 31 36 32 - __rust_helper void rust_helper_atomic_i8_set_release(s8 *ptr, s8 val) 33 - { 34 - smp_store_release(ptr, val); 35 - } 36 - 37 - __rust_helper void rust_helper_atomic_i16_set(s16 *ptr, s16 val) 38 - { 39 - WRITE_ONCE(*ptr, val); 40 - } 41 - 42 - __rust_helper void rust_helper_atomic_i16_set_release(s16 *ptr, s16 val) 43 - { 44 - smp_store_release(ptr, val); 45 - } 37 + GEN_READ_SET_HELPERS(i8, s8) 38 + GEN_READ_SET_HELPERS(i16, s16) 39 + GEN_READ_SET_HELPERS(ptr, const void *) 46 40 47 41 /* 48 42 * xchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the ··· 45 51 * The architectures that currently support Rust (x86_64, armv7, 46 52 * arm64, riscv, and loongarch) satisfy these requirements. 47 53 */ 48 - __rust_helper s8 rust_helper_atomic_i8_xchg(s8 *ptr, s8 new) 49 - { 50 - return xchg(ptr, new); 54 + #define GEN_XCHG_HELPER(tname, type, suffix) \ 55 + __rust_helper type \ 56 + rust_helper_atomic_##tname##_xchg##suffix(type *ptr, type new) \ 57 + { \ 58 + return xchg##suffix(ptr, new); \ 51 59 } 52 60 53 - __rust_helper s16 rust_helper_atomic_i16_xchg(s16 *ptr, s16 new) 54 - { 55 - return xchg(ptr, new); 56 - } 61 + #define GEN_XCHG_HELPERS(tname, type) \ 62 + GEN_XCHG_HELPER(tname, type, ) \ 63 + GEN_XCHG_HELPER(tname, type, _acquire) \ 64 + GEN_XCHG_HELPER(tname, type, _release) \ 65 + GEN_XCHG_HELPER(tname, type, _relaxed) \ 57 66 58 - __rust_helper s8 rust_helper_atomic_i8_xchg_acquire(s8 *ptr, s8 new) 59 - { 60 - return xchg_acquire(ptr, new); 61 - } 62 - 63 - __rust_helper s16 rust_helper_atomic_i16_xchg_acquire(s16 *ptr, s16 new) 64 - { 65 - return xchg_acquire(ptr, new); 66 - } 67 - 68 - __rust_helper s8 rust_helper_atomic_i8_xchg_release(s8 *ptr, s8 new) 69 - { 70 - return xchg_release(ptr, new); 71 - } 72 - 73 - __rust_helper s16 rust_helper_atomic_i16_xchg_release(s16 *ptr, s16 new) 74 - { 75 - return xchg_release(ptr, new); 76 - } 77 - 78 - __rust_helper s8 rust_helper_atomic_i8_xchg_relaxed(s8 *ptr, s8 new) 79 - { 80 - return xchg_relaxed(ptr, new); 81 - } 82 - 83 - __rust_helper s16 rust_helper_atomic_i16_xchg_relaxed(s16 *ptr, s16 new) 84 - { 85 - return xchg_relaxed(ptr, new); 86 - } 67 + GEN_XCHG_HELPERS(i8, s8) 68 + GEN_XCHG_HELPERS(i16, s16) 69 + GEN_XCHG_HELPERS(ptr, const void *) 87 70 88 71 /* 89 72 * try_cmpxchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the ··· 69 98 * The architectures that currently support Rust (x86_64, armv7, 70 99 * arm64, riscv, and loongarch) satisfy these requirements. 71 100 */ 72 - __rust_helper bool rust_helper_atomic_i8_try_cmpxchg(s8 *ptr, s8 *old, s8 new) 73 - { 74 - return try_cmpxchg(ptr, old, new); 101 + #define GEN_TRY_CMPXCHG_HELPER(tname, type, suffix) \ 102 + __rust_helper bool \ 103 + rust_helper_atomic_##tname##_try_cmpxchg##suffix(type *ptr, type *old, type new)\ 104 + { \ 105 + return try_cmpxchg##suffix(ptr, old, new); \ 75 106 } 76 107 77 - __rust_helper bool rust_helper_atomic_i16_try_cmpxchg(s16 *ptr, s16 *old, s16 new) 78 - { 79 - return try_cmpxchg(ptr, old, new); 80 - } 108 + #define GEN_TRY_CMPXCHG_HELPERS(tname, type) \ 109 + GEN_TRY_CMPXCHG_HELPER(tname, type, ) \ 110 + GEN_TRY_CMPXCHG_HELPER(tname, type, _acquire) \ 111 + GEN_TRY_CMPXCHG_HELPER(tname, type, _release) \ 112 + GEN_TRY_CMPXCHG_HELPER(tname, type, _relaxed) \ 81 113 82 - __rust_helper bool rust_helper_atomic_i8_try_cmpxchg_acquire(s8 *ptr, s8 *old, s8 new) 83 - { 84 - return try_cmpxchg_acquire(ptr, old, new); 85 - } 86 - 87 - __rust_helper bool rust_helper_atomic_i16_try_cmpxchg_acquire(s16 *ptr, s16 *old, s16 new) 88 - { 89 - return try_cmpxchg_acquire(ptr, old, new); 90 - } 91 - 92 - __rust_helper bool rust_helper_atomic_i8_try_cmpxchg_release(s8 *ptr, s8 *old, s8 new) 93 - { 94 - return try_cmpxchg_release(ptr, old, new); 95 - } 96 - 97 - __rust_helper bool rust_helper_atomic_i16_try_cmpxchg_release(s16 *ptr, s16 *old, s16 new) 98 - { 99 - return try_cmpxchg_release(ptr, old, new); 100 - } 101 - 102 - __rust_helper bool rust_helper_atomic_i8_try_cmpxchg_relaxed(s8 *ptr, s8 *old, s8 new) 103 - { 104 - return try_cmpxchg_relaxed(ptr, old, new); 105 - } 106 - 107 - __rust_helper bool rust_helper_atomic_i16_try_cmpxchg_relaxed(s16 *ptr, s16 *old, s16 new) 108 - { 109 - return try_cmpxchg_relaxed(ptr, old, new); 110 - } 114 + GEN_TRY_CMPXCHG_HELPERS(i8, s8) 115 + GEN_TRY_CMPXCHG_HELPERS(i16, s16) 116 + GEN_TRY_CMPXCHG_HELPERS(ptr, const void *)

+4 -4

rust/kernel/list/arc.rs

··· 6 6 7 7 use crate::alloc::{AllocError, Flags}; 8 8 use crate::prelude::*; 9 - use crate::sync::atomic::{ordering, Atomic}; 9 + use crate::sync::atomic::{ordering, AtomicFlag}; 10 10 use crate::sync::{Arc, ArcBorrow, UniqueArc}; 11 11 use core::marker::PhantomPinned; 12 12 use core::ops::Deref; ··· 450 450 /// If the boolean is `false`, then there is no [`ListArc`] for this value. 451 451 #[repr(transparent)] 452 452 pub struct AtomicTracker<const ID: u64 = 0> { 453 - inner: Atomic<bool>, 453 + inner: AtomicFlag, 454 454 // This value needs to be pinned to justify the INVARIANT: comment in `AtomicTracker::new`. 455 455 _pin: PhantomPinned, 456 456 } ··· 461 461 // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will 462 462 // not be constructed in an `Arc` that already has a `ListArc`. 463 463 Self { 464 - inner: Atomic::new(false), 464 + inner: AtomicFlag::new(false), 465 465 _pin: PhantomPinned, 466 466 } 467 467 } 468 468 469 - fn project_inner(self: Pin<&mut Self>) -> &mut Atomic<bool> { 469 + fn project_inner(self: Pin<&mut Self>) -> &mut AtomicFlag { 470 470 // SAFETY: The `inner` field is not structurally pinned, so we may obtain a mutable 471 471 // reference to it even if we only have a pinned reference to `self`. 472 472 unsafe { &mut Pin::into_inner_unchecked(self).inner }

+299 -11

rust/kernel/sync/atomic.rs

··· 51 51 #[repr(transparent)] 52 52 pub struct Atomic<T: AtomicType>(AtomicRepr<T::Repr>); 53 53 54 + // SAFETY: `Atomic<T>` is safe to transfer between execution contexts because of the safety 55 + // requirement of `AtomicType`. 56 + unsafe impl<T: AtomicType> Send for Atomic<T> {} 57 + 54 58 // SAFETY: `Atomic<T>` is safe to share among execution contexts because all accesses are atomic. 55 59 unsafe impl<T: AtomicType> Sync for Atomic<T> {} 56 60 ··· 72 68 /// 73 69 /// - [`Self`] must have the same size and alignment as [`Self::Repr`]. 74 70 /// - [`Self`] must be [round-trip transmutable] to [`Self::Repr`]. 71 + /// - [`Self`] must be safe to transfer between execution contexts, if it's [`Send`], this is 72 + /// automatically satisfied. The exception is pointer types that are even though marked as 73 + /// `!Send` (e.g. raw pointers and [`NonNull<T>`]) but requiring `unsafe` to do anything 74 + /// meaningful on them. This is because transferring pointer values between execution contexts is 75 + /// safe as long as the actual `unsafe` dereferencing is justified. 75 76 /// 76 77 /// Note that this is more relaxed than requiring the bi-directional transmutability (i.e. 77 78 /// [`transmute()`] is always sound between `U` and `T`) because of the support for atomic ··· 117 108 /// [`transmute()`]: core::mem::transmute 118 109 /// [round-trip transmutable]: AtomicType#round-trip-transmutability 119 110 /// [Examples]: AtomicType#examples 120 - pub unsafe trait AtomicType: Sized + Send + Copy { 111 + /// [`NonNull<T>`]: core::ptr::NonNull 112 + pub unsafe trait AtomicType: Sized + Copy { 121 113 /// The backing atomic implementation type. 122 114 type Repr: AtomicImpl; 123 115 } ··· 214 204 /// // no data race. 215 205 /// unsafe { Atomic::from_ptr(foo_a_ptr) }.store(2, Release); 216 206 /// ``` 217 - pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self 218 - where 219 - T: Sync, 220 - { 207 + pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self { 221 208 // CAST: `T` and `Atomic<T>` have the same size, alignment and bit validity. 222 209 // SAFETY: Per function safety requirement, `ptr` is a valid pointer and the object will 223 210 // live long enough. It's safe to return a `&Atomic<T>` because function safety requirement ··· 242 235 /// Returns a mutable reference to the underlying atomic `T`. 243 236 /// 244 237 /// This is safe because the mutable reference of the atomic `T` guarantees exclusive access. 238 + /// 239 + /// # Examples 240 + /// 241 + /// ``` 242 + /// use kernel::sync::atomic::{Atomic, Relaxed}; 243 + /// 244 + /// let mut atomic_val = Atomic::new(0u32); 245 + /// let val_mut = atomic_val.get_mut(); 246 + /// *val_mut = 101; 247 + /// assert_eq!(101, atomic_val.load(Relaxed)); 248 + /// ``` 245 249 pub fn get_mut(&mut self) -> &mut T { 246 250 // CAST: `T` and `T::Repr` has the same size and alignment per the safety requirement of 247 251 // `AtomicType`, and per the type invariants `self.0` is a valid `T`, therefore the casting ··· 545 527 /// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed}; 546 528 /// 547 529 /// let x = Atomic::new(42); 548 - /// 549 530 /// assert_eq!(42, x.load(Relaxed)); 550 - /// 551 - /// assert_eq!(54, { x.fetch_add(12, Acquire); x.load(Relaxed) }); 531 + /// assert_eq!(42, x.fetch_add(12, Acquire)); 532 + /// assert_eq!(54, x.load(Relaxed)); 552 533 /// 553 534 /// let x = Atomic::new(42); 554 - /// 555 535 /// assert_eq!(42, x.load(Relaxed)); 556 - /// 557 - /// assert_eq!(54, { x.fetch_add(12, Full); x.load(Relaxed) } ); 536 + /// assert_eq!(42, x.fetch_add(12, Full)); 537 + /// assert_eq!(54, x.load(Relaxed)); 558 538 /// ``` 559 539 #[inline(always)] 560 540 pub fn fetch_add<Rhs, Ordering: ordering::Ordering>(&self, v: Rhs, _: Ordering) -> T ··· 575 559 // SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants. 576 560 unsafe { from_repr(ret) } 577 561 } 562 + 563 + /// Atomic fetch and subtract. 564 + /// 565 + /// Atomically updates `*self` to `(*self).wrapping_sub(v)`, and returns the value of `*self` 566 + /// before the update. 567 + /// 568 + /// # Examples 569 + /// 570 + /// ``` 571 + /// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed}; 572 + /// 573 + /// let x = Atomic::new(42); 574 + /// assert_eq!(42, x.load(Relaxed)); 575 + /// assert_eq!(42, x.fetch_sub(12, Acquire)); 576 + /// assert_eq!(30, x.load(Relaxed)); 577 + /// 578 + /// let x = Atomic::new(42); 579 + /// assert_eq!(42, x.load(Relaxed)); 580 + /// assert_eq!(42, x.fetch_sub(12, Full)); 581 + /// assert_eq!(30, x.load(Relaxed)); 582 + /// ``` 583 + #[inline(always)] 584 + pub fn fetch_sub<Rhs, Ordering: ordering::Ordering>(&self, v: Rhs, _: Ordering) -> T 585 + where 586 + // Types that support addition also support subtraction. 587 + T: AtomicAdd<Rhs>, 588 + { 589 + let v = T::rhs_into_delta(v); 590 + 591 + // INVARIANT: `self.0` is a valid `T` after `atomic_fetch_sub*()` due to safety requirement 592 + // of `AtomicAdd`. 593 + let ret = { 594 + match Ordering::TYPE { 595 + OrderingType::Full => T::Repr::atomic_fetch_sub(&self.0, v), 596 + OrderingType::Acquire => T::Repr::atomic_fetch_sub_acquire(&self.0, v), 597 + OrderingType::Release => T::Repr::atomic_fetch_sub_release(&self.0, v), 598 + OrderingType::Relaxed => T::Repr::atomic_fetch_sub_relaxed(&self.0, v), 599 + } 600 + }; 601 + 602 + // SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants. 603 + unsafe { from_repr(ret) } 604 + } 605 + } 606 + 607 + #[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] 608 + #[repr(C)] 609 + #[derive(Clone, Copy)] 610 + struct Flag { 611 + bool_field: bool, 612 + } 613 + 614 + /// # Invariants 615 + /// 616 + /// `padding` must be all zeroes. 617 + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] 618 + #[repr(C, align(4))] 619 + #[derive(Clone, Copy)] 620 + struct Flag { 621 + #[cfg(target_endian = "big")] 622 + padding: [u8; 3], 623 + bool_field: bool, 624 + #[cfg(target_endian = "little")] 625 + padding: [u8; 3], 626 + } 627 + 628 + impl Flag { 629 + #[inline(always)] 630 + const fn new(b: bool) -> Self { 631 + // INVARIANT: `padding` is all zeroes. 632 + Self { 633 + bool_field: b, 634 + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] 635 + padding: [0; 3], 636 + } 637 + } 638 + } 639 + 640 + // SAFETY: `Flag` and `Repr` have the same size and alignment, and `Flag` is round-trip 641 + // transmutable to the selected representation (`i8` or `i32`). 642 + unsafe impl AtomicType for Flag { 643 + #[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] 644 + type Repr = i8; 645 + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] 646 + type Repr = i32; 647 + } 648 + 649 + /// An atomic flag type intended to be backed by performance-optimal integer type. 650 + /// 651 + /// The backing integer type is an implementation detail; it may vary by architecture and change 652 + /// in the future. 653 + /// 654 + /// [`AtomicFlag`] is generally preferable to [`Atomic<bool>`] when you need read-modify-write 655 + /// (RMW) operations (e.g. [`Atomic::xchg()`]/[`Atomic::cmpxchg()`]) or when [`Atomic<bool>`] does 656 + /// not save memory due to padding. On some architectures that do not support byte-sized atomic 657 + /// RMW operations, RMW operations on [`Atomic<bool>`] are slower. 658 + /// 659 + /// If you only use [`Atomic::load()`]/[`Atomic::store()`], [`Atomic<bool>`] is fine. 660 + /// 661 + /// # Examples 662 + /// 663 + /// ``` 664 + /// use kernel::sync::atomic::{AtomicFlag, Relaxed}; 665 + /// 666 + /// let flag = AtomicFlag::new(false); 667 + /// assert_eq!(false, flag.load(Relaxed)); 668 + /// flag.store(true, Relaxed); 669 + /// assert_eq!(true, flag.load(Relaxed)); 670 + /// ``` 671 + pub struct AtomicFlag(Atomic<Flag>); 672 + 673 + impl AtomicFlag { 674 + /// Creates a new atomic flag. 675 + #[inline(always)] 676 + pub const fn new(b: bool) -> Self { 677 + Self(Atomic::new(Flag::new(b))) 678 + } 679 + 680 + /// Returns a mutable reference to the underlying flag as a [`bool`]. 681 + /// 682 + /// This is safe because the mutable reference of the atomic flag guarantees exclusive access. 683 + /// 684 + /// # Examples 685 + /// 686 + /// ``` 687 + /// use kernel::sync::atomic::{AtomicFlag, Relaxed}; 688 + /// 689 + /// let mut atomic_flag = AtomicFlag::new(false); 690 + /// assert_eq!(false, atomic_flag.load(Relaxed)); 691 + /// *atomic_flag.get_mut() = true; 692 + /// assert_eq!(true, atomic_flag.load(Relaxed)); 693 + /// ``` 694 + #[inline(always)] 695 + pub fn get_mut(&mut self) -> &mut bool { 696 + &mut self.0.get_mut().bool_field 697 + } 698 + 699 + /// Loads the value from the atomic flag. 700 + #[inline(always)] 701 + pub fn load<Ordering: ordering::AcquireOrRelaxed>(&self, o: Ordering) -> bool { 702 + self.0.load(o).bool_field 703 + } 704 + 705 + /// Stores a value to the atomic flag. 706 + #[inline(always)] 707 + pub fn store<Ordering: ordering::ReleaseOrRelaxed>(&self, v: bool, o: Ordering) { 708 + self.0.store(Flag::new(v), o); 709 + } 710 + 711 + /// Stores a value to the atomic flag and returns the previous value. 712 + #[inline(always)] 713 + pub fn xchg<Ordering: ordering::Ordering>(&self, new: bool, o: Ordering) -> bool { 714 + self.0.xchg(Flag::new(new), o).bool_field 715 + } 716 + 717 + /// Store a value to the atomic flag if the current value is equal to `old`. 718 + #[inline(always)] 719 + pub fn cmpxchg<Ordering: ordering::Ordering>( 720 + &self, 721 + old: bool, 722 + new: bool, 723 + o: Ordering, 724 + ) -> Result<bool, bool> { 725 + match self.0.cmpxchg(Flag::new(old), Flag::new(new), o) { 726 + Ok(_) => Ok(old), 727 + Err(f) => Err(f.bool_field), 728 + } 729 + } 730 + } 731 + 732 + /// Atomic load over raw pointers. 733 + /// 734 + /// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work 735 + /// with C side on synchronizations: 736 + /// 737 + /// - `atomic_load(.., Relaxed)` maps to `READ_ONCE()` when used for inter-thread communication. 738 + /// - `atomic_load(.., Acquire)` maps to `smp_load_acquire()`. 739 + /// 740 + /// # Safety 741 + /// 742 + /// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. 743 + /// - If there is a concurrent store from kernel (C or Rust), it has to be atomic. 744 + #[doc(alias("READ_ONCE", "smp_load_acquire"))] 745 + #[inline(always)] 746 + pub unsafe fn atomic_load<T: AtomicType, Ordering: ordering::AcquireOrRelaxed>( 747 + ptr: *mut T, 748 + o: Ordering, 749 + ) -> T 750 + where 751 + T::Repr: AtomicBasicOps, 752 + { 753 + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to 754 + // `align_of::<T>()`, and all concurrent stores from kernel are atomic, hence no data race per 755 + // LKMM. 756 + unsafe { Atomic::from_ptr(ptr) }.load(o) 757 + } 758 + 759 + /// Atomic store over raw pointers. 760 + /// 761 + /// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work 762 + /// with C side on synchronizations: 763 + /// 764 + /// - `atomic_store(.., Relaxed)` maps to `WRITE_ONCE()` when used for inter-thread communication. 765 + /// - `atomic_load(.., Release)` maps to `smp_store_release()`. 766 + /// 767 + /// # Safety 768 + /// 769 + /// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. 770 + /// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. 771 + #[doc(alias("WRITE_ONCE", "smp_store_release"))] 772 + #[inline(always)] 773 + pub unsafe fn atomic_store<T: AtomicType, Ordering: ordering::ReleaseOrRelaxed>( 774 + ptr: *mut T, 775 + v: T, 776 + o: Ordering, 777 + ) where 778 + T::Repr: AtomicBasicOps, 779 + { 780 + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to 781 + // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race 782 + // per LKMM. 783 + unsafe { Atomic::from_ptr(ptr) }.store(v, o); 784 + } 785 + 786 + /// Atomic exchange over raw pointers. 787 + /// 788 + /// This function provides a short-cut of `Atomic::from_ptr().xchg(..)`, and can be used to work 789 + /// with C side on synchronizations. 790 + /// 791 + /// # Safety 792 + /// 793 + /// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. 794 + /// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. 795 + #[inline(always)] 796 + pub unsafe fn xchg<T: AtomicType, Ordering: ordering::Ordering>( 797 + ptr: *mut T, 798 + new: T, 799 + o: Ordering, 800 + ) -> T 801 + where 802 + T::Repr: AtomicExchangeOps, 803 + { 804 + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to 805 + // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race 806 + // per LKMM. 807 + unsafe { Atomic::from_ptr(ptr) }.xchg(new, o) 808 + } 809 + 810 + /// Atomic compare and exchange over raw pointers. 811 + /// 812 + /// This function provides a short-cut of `Atomic::from_ptr().cmpxchg(..)`, and can be used to work 813 + /// with C side on synchronizations. 814 + /// 815 + /// # Safety 816 + /// 817 + /// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. 818 + /// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. 819 + #[doc(alias("try_cmpxchg"))] 820 + #[inline(always)] 821 + pub unsafe fn cmpxchg<T: AtomicType, Ordering: ordering::Ordering>( 822 + ptr: *mut T, 823 + old: T, 824 + new: T, 825 + o: Ordering, 826 + ) -> Result<T, T> 827 + where 828 + T::Repr: AtomicExchangeOps, 829 + { 830 + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to 831 + // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race 832 + // per LKMM. 833 + unsafe { Atomic::from_ptr(ptr) }.cmpxchg(old, new, o) 578 834 }

+31 -13

rust/kernel/sync/atomic/internal.rs

··· 7 7 use crate::bindings; 8 8 use crate::macros::paste; 9 9 use core::cell::UnsafeCell; 10 + use ffi::c_void; 10 11 11 12 mod private { 12 13 /// Sealed trait marker to disable customized impls on atomic implementation traits. ··· 15 14 } 16 15 17 16 // The C side supports atomic primitives only for `i32` and `i64` (`atomic_t` and `atomic64_t`), 18 - // while the Rust side also layers provides atomic support for `i8` and `i16` 19 - // on top of lower-level C primitives. 17 + // while the Rust side also provides atomic support for `i8`, `i16` and `*const c_void` on top of 18 + // lower-level C primitives. 20 19 impl private::Sealed for i8 {} 21 20 impl private::Sealed for i16 {} 21 + impl private::Sealed for *const c_void {} 22 22 impl private::Sealed for i32 {} 23 23 impl private::Sealed for i64 {} 24 24 ··· 28 26 /// This trait is sealed, and only types that map directly to the C side atomics 29 27 /// or can be implemented with lower-level C primitives are allowed to implement this: 30 28 /// 31 - /// - `i8` and `i16` are implemented with lower-level C primitives. 29 + /// - `i8`, `i16` and `*const c_void` are implemented with lower-level C primitives. 32 30 /// - `i32` map to `atomic_t` 33 31 /// - `i64` map to `atomic64_t` 34 - pub trait AtomicImpl: Sized + Send + Copy + private::Sealed { 32 + pub trait AtomicImpl: Sized + Copy + private::Sealed { 35 33 /// The type of the delta in arithmetic or logical operations. 36 34 /// 37 35 /// For example, in `atomic_add(ptr, v)`, it's the type of `v`. Usually it's the same type of ··· 39 37 type Delta; 40 38 } 41 39 42 - // The current helpers of load/store uses `{WRITE,READ}_ONCE()` hence the atomicity is only 43 - // guaranteed against read-modify-write operations if the architecture supports native atomic RmW. 44 - #[cfg(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW)] 40 + // The current helpers of load/store of atomic `i8`, `i16` and pointers use `{WRITE,READ}_ONCE()` 41 + // hence the atomicity is only guaranteed against read-modify-write operations if the architecture 42 + // supports native atomic RmW. 43 + // 44 + // In the future when a CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=n architecture plans to support Rust, the 45 + // load/store helpers that guarantee atomicity against RmW operations (usually via a lock) need to 46 + // be added. 47 + crate::static_assert!( 48 + cfg!(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW), 49 + "The current implementation of atomic i8/i16/ptr relies on the architecure being \ 50 + ARCH_SUPPORTS_ATOMIC_RMW" 51 + ); 52 + 45 53 impl AtomicImpl for i8 { 46 54 type Delta = Self; 47 55 } 48 56 49 - // The current helpers of load/store uses `{WRITE,READ}_ONCE()` hence the atomicity is only 50 - // guaranteed against read-modify-write operations if the architecture supports native atomic RmW. 51 - #[cfg(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW)] 52 57 impl AtomicImpl for i16 { 53 58 type Delta = Self; 59 + } 60 + 61 + impl AtomicImpl for *const c_void { 62 + type Delta = isize; 54 63 } 55 64 56 65 // `atomic_t` implements atomic operations on `i32`. ··· 275 262 } 276 263 277 264 declare_and_impl_atomic_methods!( 278 - [ i8 => atomic_i8, i16 => atomic_i16, i32 => atomic, i64 => atomic64 ] 265 + [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ] 279 266 /// Basic atomic operations 280 267 pub trait AtomicBasicOps { 281 268 /// Atomic read (load). ··· 293 280 ); 294 281 295 282 declare_and_impl_atomic_methods!( 296 - [ i8 => atomic_i8, i16 => atomic_i16, i32 => atomic, i64 => atomic64 ] 283 + [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ] 297 284 /// Exchange and compare-and-exchange atomic operations 298 285 pub trait AtomicExchangeOps { 299 286 /// Atomic exchange. ··· 337 324 /// Atomically updates `*a` to `(*a).wrapping_add(v)`, and returns the value of `*a` 338 325 /// before the update. 339 326 fn fetch_add[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self { 340 - // SAFETY: `a.as_ptr()` is valid and properly aligned. 327 + // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned. 328 + unsafe { bindings::#call(v, a.as_ptr().cast()) } 329 + } 330 + 331 + fn fetch_sub[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self { 332 + // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned. 341 333 unsafe { bindings::#call(v, a.as_ptr().cast()) } 342 334 } 343 335 }

+109

rust/kernel/sync/atomic/predefine.rs

··· 4 4 5 5 use crate::static_assert; 6 6 use core::mem::{align_of, size_of}; 7 + use ffi::c_void; 7 8 8 9 // Ensure size and alignment requirements are checked. 9 10 static_assert!(size_of::<bool>() == size_of::<i8>()); ··· 27 26 // itself. 28 27 unsafe impl super::AtomicType for i16 { 29 28 type Repr = i16; 29 + } 30 + 31 + // SAFETY: 32 + // 33 + // - `*mut T` has the same size and alignment with `*const c_void`, and is round-trip 34 + // transmutable to `*const c_void`. 35 + // - `*mut T` is safe to transfer between execution contexts. See the safety requirement of 36 + // [`AtomicType`]. 37 + unsafe impl<T: Sized> super::AtomicType for *mut T { 38 + type Repr = *const c_void; 39 + } 40 + 41 + // SAFETY: 42 + // 43 + // - `*const T` has the same size and alignment with `*const c_void`, and is round-trip 44 + // transmutable to `*const c_void`. 45 + // - `*const T` is safe to transfer between execution contexts. See the safety requirement of 46 + // [`AtomicType`]. 47 + unsafe impl<T: Sized> super::AtomicType for *const T { 48 + type Repr = *const c_void; 30 49 } 31 50 32 51 // SAFETY: `i32` has the same size and alignment with itself, and is round-trip transmutable to ··· 178 157 179 158 assert_eq!(v, x.load(Relaxed)); 180 159 }); 160 + 161 + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 162 + let x = Atomic::new(v); 163 + let ptr = x.as_ptr(); 164 + 165 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 166 + assert_eq!(v, unsafe { atomic_load(ptr, Relaxed) }); 167 + }); 181 168 } 182 169 183 170 #[test] ··· 195 166 196 167 x.store(v, Release); 197 168 assert_eq!(v, x.load(Acquire)); 169 + }); 170 + 171 + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 172 + let x = Atomic::new(0); 173 + let ptr = x.as_ptr(); 174 + 175 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 176 + unsafe { atomic_store(ptr, v, Release) }; 177 + 178 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 179 + assert_eq!(v, unsafe { atomic_load(ptr, Acquire) }); 198 180 }); 199 181 } 200 182 ··· 218 178 let new = v + 1; 219 179 220 180 assert_eq!(old, x.xchg(new, Full)); 181 + assert_eq!(new, x.load(Relaxed)); 182 + }); 183 + 184 + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 185 + let x = Atomic::new(v); 186 + let ptr = x.as_ptr(); 187 + 188 + let old = v; 189 + let new = v + 1; 190 + 191 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 192 + assert_eq!(old, unsafe { xchg(ptr, new, Full) }); 221 193 assert_eq!(new, x.load(Relaxed)); 222 194 }); 223 195 } ··· 245 193 assert_eq!(Err(old), x.cmpxchg(new, new, Full)); 246 194 assert_eq!(old, x.load(Relaxed)); 247 195 assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed)); 196 + assert_eq!(new, x.load(Relaxed)); 197 + }); 198 + 199 + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 200 + let x = Atomic::new(v); 201 + let ptr = x.as_ptr(); 202 + 203 + let old = v; 204 + let new = v + 1; 205 + 206 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 207 + assert_eq!(Err(old), unsafe { cmpxchg(ptr, new, new, Full) }); 208 + assert_eq!(old, x.load(Relaxed)); 209 + // SAFETY: `ptr` is a valid pointer and no concurrent access. 210 + assert_eq!(Ok(old), unsafe { cmpxchg(ptr, old, new, Relaxed) }); 248 211 assert_eq!(new, x.load(Relaxed)); 249 212 }); 250 213 } ··· 292 225 assert_eq!(Err(false), x.cmpxchg(true, true, Relaxed)); 293 226 assert_eq!(false, x.load(Relaxed)); 294 227 assert_eq!(Ok(false), x.cmpxchg(false, true, Full)); 228 + } 229 + 230 + #[test] 231 + fn atomic_ptr_tests() { 232 + let mut v = 42; 233 + let mut u = 43; 234 + let x = Atomic::new(&raw mut v); 235 + 236 + assert_eq!(x.load(Acquire), &raw mut v); 237 + assert_eq!(x.cmpxchg(&raw mut u, &raw mut u, Relaxed), Err(&raw mut v)); 238 + assert_eq!(x.cmpxchg(&raw mut v, &raw mut u, Relaxed), Ok(&raw mut v)); 239 + assert_eq!(x.load(Relaxed), &raw mut u); 240 + 241 + let x = Atomic::new(&raw const v); 242 + 243 + assert_eq!(x.load(Acquire), &raw const v); 244 + assert_eq!( 245 + x.cmpxchg(&raw const u, &raw const u, Relaxed), 246 + Err(&raw const v) 247 + ); 248 + assert_eq!( 249 + x.cmpxchg(&raw const v, &raw const u, Relaxed), 250 + Ok(&raw const v) 251 + ); 252 + assert_eq!(x.load(Relaxed), &raw const u); 253 + } 254 + 255 + #[test] 256 + fn atomic_flag_tests() { 257 + let mut flag = AtomicFlag::new(false); 258 + 259 + assert_eq!(false, flag.load(Relaxed)); 260 + 261 + *flag.get_mut() = true; 262 + assert_eq!(true, flag.load(Relaxed)); 263 + 264 + assert_eq!(true, flag.xchg(false, Relaxed)); 265 + assert_eq!(false, flag.load(Relaxed)); 266 + 267 + *flag.get_mut() = true; 268 + assert_eq!(Ok(true), flag.cmpxchg(true, false, Full)); 269 + assert_eq!(false, flag.load(Relaxed)); 295 270 } 296 271 }

+1

scripts/context-analysis-suppression.txt

··· 24 24 src:*include/linux/rcupdate.h=emit 25 25 src:*include/linux/refcount.h=emit 26 26 src:*include/linux/rhashtable.h=emit 27 + src:*include/linux/rtmutex*.h=emit 27 28 src:*include/linux/rwlock*.h=emit 28 29 src:*include/linux/rwsem.h=emit 29 30 src:*include/linux/sched*=emit

Configure Feed

Configure Feed