+4 -4

include/linux/rtmutex.h

reviewed

··· 22 22 23 23 struct rt_mutex_base { 24 24 raw_spinlock_t wait_lock; 25 25 - struct rb_root_cached waiters; 26 26 - struct task_struct *owner; 25 25 + struct rb_root_cached waiters __guarded_by(&wait_lock); 26 26 + struct task_struct *owner __guarded_by(&wait_lock); 27 27 }; 28 28 29 29 #define __RT_MUTEX_BASE_INITIALIZER(rtbasename) \ ··· 41 41 */ 42 42 static inline bool rt_mutex_base_is_locked(struct rt_mutex_base *lock) 43 43 { 44 44 - return READ_ONCE(lock->owner) != NULL; 44 44 + return data_race(READ_ONCE(lock->owner) != NULL); 45 45 } 46 46 47 47 #ifdef CONFIG_RT_MUTEXES ··· 49 49 50 50 static inline struct task_struct *rt_mutex_owner(struct rt_mutex_base *lock) 51 51 { 52 52 - unsigned long owner = (unsigned long) READ_ONCE(lock->owner); 52 52 + unsigned long owner = (unsigned long) data_race(READ_ONCE(lock->owner)); 53 53 54 54 return (struct task_struct *) (owner & ~RT_MUTEX_HAS_WAITERS); 55 55 }

+2

kernel/locking/Makefile

reviewed

··· 4 4 KCOV_INSTRUMENT := n 5 5 6 6 CONTEXT_ANALYSIS_mutex.o := y 7 7 + CONTEXT_ANALYSIS_rtmutex_api.o := y 8 8 + CONTEXT_ANALYSIS_ww_rt_mutex.o := y 7 9 8 10 obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o 9 11

+17 -1

kernel/locking/rtmutex.c

reviewed

··· 94 94 95 95 static __always_inline struct task_struct * 96 96 rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner) 97 97 + __must_hold(&lock->wait_lock) 97 98 { 98 99 unsigned long val = (unsigned long)owner; 99 100 ··· 106 105 107 106 static __always_inline void 108 107 rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner) 108 108 + __must_hold(&lock->wait_lock) 109 109 { 110 110 /* 111 111 * lock->wait_lock is held but explicit acquire semantics are needed ··· 116 114 } 117 115 118 116 static __always_inline void rt_mutex_clear_owner(struct rt_mutex_base *lock) 117 117 + __must_hold(&lock->wait_lock) 119 118 { 120 119 /* lock->wait_lock is held so the unlock provides release semantics. */ 121 120 WRITE_ONCE(lock->owner, rt_mutex_owner_encode(lock, NULL)); 122 121 } 123 122 124 123 static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock) 124 124 + __must_hold(&lock->wait_lock) 125 125 { 126 126 lock->owner = (struct task_struct *) 127 127 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS); ··· 131 127 132 128 static __always_inline void 133 129 fixup_rt_mutex_waiters(struct rt_mutex_base *lock, bool acquire_lock) 130 130 + __must_hold(&lock->wait_lock) 134 131 { 135 132 unsigned long owner, *p = (unsigned long *) &lock->owner; 136 133 ··· 333 328 } 334 329 335 330 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock) 331 331 + __must_hold(&lock->wait_lock) 336 332 { 337 333 lock->owner = (struct task_struct *) 338 334 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS); ··· 1212 1206 struct ww_acquire_ctx *ww_ctx, 1213 1207 enum rtmutex_chainwalk chwalk, 1214 1208 struct wake_q_head *wake_q) 1209 1209 + __must_hold(&lock->wait_lock) 1215 1210 { 1216 1211 struct task_struct *owner = rt_mutex_owner(lock); 1217 1212 struct rt_mutex_waiter *top_waiter = waiter; ··· 1256 1249 1257 1250 /* Check whether the waiter should back out immediately */ 1258 1251 rtm = container_of(lock, struct rt_mutex, rtmutex); 1252 1252 + __assume_ctx_lock(&rtm->rtmutex.wait_lock); 1259 1253 res = __ww_mutex_add_waiter(waiter, rtm, ww_ctx, wake_q); 1260 1254 if (res) { 1261 1255 raw_spin_lock(&task->pi_lock); ··· 1364 1356 } 1365 1357 1366 1358 static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock) 1359 1359 + __must_hold(&lock->wait_lock) 1367 1360 { 1368 1361 int ret = try_to_take_rt_mutex(lock, current, NULL); 1369 1362 ··· 1514 1505 * - the VCPU on which owner runs is preempted 1515 1506 */ 1516 1507 if (!owner_on_cpu(owner) || need_resched() || 1517 1517 - !rt_mutex_waiter_is_top_waiter(lock, waiter)) { 1508 1508 + !data_race(rt_mutex_waiter_is_top_waiter(lock, waiter))) { 1518 1509 res = false; 1519 1510 break; 1520 1511 } ··· 1547 1538 */ 1548 1539 static void __sched remove_waiter(struct rt_mutex_base *lock, 1549 1540 struct rt_mutex_waiter *waiter) 1541 1541 + __must_hold(&lock->wait_lock) 1550 1542 { 1551 1543 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); 1552 1544 struct task_struct *owner = rt_mutex_owner(lock); ··· 1623 1613 struct task_struct *owner; 1624 1614 int ret = 0; 1625 1615 1616 1616 + __assume_ctx_lock(&rtm->rtmutex.wait_lock); 1617 1617 + 1626 1618 lockevent_inc(rtmutex_slow_block); 1627 1619 for (;;) { 1628 1620 /* Try to acquire the lock: */ ··· 1670 1658 static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock, 1671 1659 struct rt_mutex_base *lock, 1672 1660 struct rt_mutex_waiter *w) 1661 1661 + __must_hold(&lock->wait_lock) 1673 1662 { 1674 1663 /* 1675 1664 * If the result is not -EDEADLOCK or the caller requested ··· 1707 1694 enum rtmutex_chainwalk chwalk, 1708 1695 struct rt_mutex_waiter *waiter, 1709 1696 struct wake_q_head *wake_q) 1697 1697 + __must_hold(&lock->wait_lock) 1710 1698 { 1711 1699 struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex); 1712 1700 struct ww_mutex *ww = ww_container_of(rtm); 1713 1701 int ret; 1714 1702 1703 1703 + __assume_ctx_lock(&rtm->rtmutex.wait_lock); 1715 1704 lockdep_assert_held(&lock->wait_lock); 1716 1705 lockevent_inc(rtmutex_slowlock); 1717 1706 ··· 1765 1750 struct ww_acquire_ctx *ww_ctx, 1766 1751 unsigned int state, 1767 1752 struct wake_q_head *wake_q) 1753 1753 + __must_hold(&lock->wait_lock) 1768 1754 { 1769 1755 struct rt_mutex_waiter waiter; 1770 1756 int ret;

+2

kernel/locking/rtmutex_api.c

reviewed

··· 526 526 unsigned int subclass, 527 527 struct lockdep_map *nest_lock, 528 528 unsigned long ip) 529 529 + __acquires(lock) __no_context_analysis 529 530 { 530 531 int ret; 531 532 ··· 648 647 #endif /* !CONFIG_DEBUG_LOCK_ALLOC */ 649 648 650 649 void __sched mutex_unlock(struct mutex *lock) 650 650 + __releases(lock) __no_context_analysis 651 651 { 652 652 mutex_release(&lock->dep_map, _RET_IP_); 653 653 __rt_mutex_unlock(&lock->rtmutex);

+19 -8

kernel/locking/rtmutex_common.h

reviewed

··· 79 79 * PI-futex support (proxy locking functions, etc.): 80 80 */ 81 81 extern void rt_mutex_init_proxy_locked(struct rt_mutex_base *lock, 82 82 - struct task_struct *proxy_owner); 83 83 - extern void rt_mutex_proxy_unlock(struct rt_mutex_base *lock); 82 82 + struct task_struct *proxy_owner) 83 83 + __must_hold(&lock->wait_lock); 84 84 + 85 85 + extern void rt_mutex_proxy_unlock(struct rt_mutex_base *lock) 86 86 + __must_hold(&lock->wait_lock); 87 87 + 84 88 extern int __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock, 85 89 struct rt_mutex_waiter *waiter, 86 90 struct task_struct *task, 87 87 - struct wake_q_head *); 91 91 + struct wake_q_head *) 92 92 + __must_hold(&lock->wait_lock); 93 93 + 88 94 extern int rt_mutex_start_proxy_lock(struct rt_mutex_base *lock, 89 95 struct rt_mutex_waiter *waiter, 90 96 struct task_struct *task); ··· 100 94 extern bool rt_mutex_cleanup_proxy_lock(struct rt_mutex_base *lock, 101 95 struct rt_mutex_waiter *waiter); 102 96 103 103 - extern int rt_mutex_futex_trylock(struct rt_mutex_base *l); 104 104 - extern int __rt_mutex_futex_trylock(struct rt_mutex_base *l); 97 97 + extern int rt_mutex_futex_trylock(struct rt_mutex_base *lock); 98 98 + extern int __rt_mutex_futex_trylock(struct rt_mutex_base *lock) 99 99 + __must_hold(&lock->wait_lock); 105 100 106 101 extern void rt_mutex_futex_unlock(struct rt_mutex_base *lock); 107 102 extern bool __rt_mutex_futex_unlock(struct rt_mutex_base *lock, ··· 116 109 */ 117 110 #ifdef CONFIG_RT_MUTEXES 118 111 static inline int rt_mutex_has_waiters(struct rt_mutex_base *lock) 112 112 + __must_hold(&lock->wait_lock) 119 113 { 120 114 return !RB_EMPTY_ROOT(&lock->waiters.rb_root); 121 115 } ··· 128 120 */ 129 121 static inline bool rt_mutex_waiter_is_top_waiter(struct rt_mutex_base *lock, 130 122 struct rt_mutex_waiter *waiter) 123 123 + __must_hold(&lock->wait_lock) 131 124 { 132 125 struct rb_node *leftmost = rb_first_cached(&lock->waiters); 133 126 ··· 136 127 } 137 128 138 129 static inline struct rt_mutex_waiter *rt_mutex_top_waiter(struct rt_mutex_base *lock) 130 130 + __must_hold(&lock->wait_lock) 139 131 { 140 132 struct rb_node *leftmost = rb_first_cached(&lock->waiters); 141 133 struct rt_mutex_waiter *w = NULL; ··· 180 170 181 171 static inline void __rt_mutex_base_init(struct rt_mutex_base *lock) 182 172 { 183 183 - raw_spin_lock_init(&lock->wait_lock); 184 184 - lock->waiters = RB_ROOT_CACHED; 185 185 - lock->owner = NULL; 173 173 + scoped_guard (raw_spinlock_init, &lock->wait_lock) { 174 174 + lock->waiters = RB_ROOT_CACHED; 175 175 + lock->owner = NULL; 176 176 + } 186 177 } 187 178 188 179 /* Debug functions */

+15 -5

kernel/locking/ww_mutex.h

reviewed

··· 4 4 5 5 #define MUTEX mutex 6 6 #define MUTEX_WAITER mutex_waiter 7 7 + #define WAIT_LOCK wait_lock 7 8 8 9 static inline struct mutex_waiter * 9 10 __ww_waiter_first(struct mutex *lock) ··· 87 86 88 87 #define MUTEX rt_mutex 89 88 #define MUTEX_WAITER rt_mutex_waiter 89 89 + #define WAIT_LOCK rtmutex.wait_lock 90 90 91 91 static inline struct rt_mutex_waiter * 92 92 __ww_waiter_first(struct rt_mutex *lock) 93 93 + __must_hold(&lock->rtmutex.wait_lock) 93 94 { 94 95 struct rb_node *n = rb_first(&lock->rtmutex.waiters.rb_root); 95 96 if (!n) ··· 119 116 120 117 static inline struct rt_mutex_waiter * 121 118 __ww_waiter_last(struct rt_mutex *lock) 119 119 + __must_hold(&lock->rtmutex.wait_lock) 122 120 { 123 121 struct rb_node *n = rb_last(&lock->rtmutex.waiters.rb_root); 124 122 if (!n) ··· 141 137 142 138 static inline bool 143 139 __ww_mutex_has_waiters(struct rt_mutex *lock) 140 140 + __must_hold(&lock->rtmutex.wait_lock) 144 141 { 145 142 return rt_mutex_has_waiters(&lock->rtmutex); 146 143 } 147 144 148 145 static inline void lock_wait_lock(struct rt_mutex *lock, unsigned long *flags) 146 146 + __acquires(&lock->rtmutex.wait_lock) 149 147 { 150 148 raw_spin_lock_irqsave(&lock->rtmutex.wait_lock, *flags); 151 149 } 152 150 153 151 static inline void unlock_wait_lock(struct rt_mutex *lock, unsigned long *flags) 152 152 + __releases(&lock->rtmutex.wait_lock) 154 153 { 155 154 raw_spin_unlock_irqrestore(&lock->rtmutex.wait_lock, *flags); 156 155 } 157 156 158 157 static inline void lockdep_assert_wait_lock_held(struct rt_mutex *lock) 158 158 + __must_hold(&lock->rtmutex.wait_lock) 159 159 { 160 160 lockdep_assert_held(&lock->rtmutex.wait_lock); 161 161 } ··· 312 304 struct ww_acquire_ctx *ww_ctx, 313 305 struct ww_acquire_ctx *hold_ctx, 314 306 struct wake_q_head *wake_q) 315 315 - __must_hold(&lock->wait_lock) 307 307 + __must_hold(&lock->WAIT_LOCK) 316 308 { 317 309 struct task_struct *owner = __ww_mutex_owner(lock); 318 310 ··· 377 369 static void 378 370 __ww_mutex_check_waiters(struct MUTEX *lock, struct ww_acquire_ctx *ww_ctx, 379 371 struct wake_q_head *wake_q) 380 380 - __must_hold(&lock->wait_lock) 372 372 + __must_hold(&lock->WAIT_LOCK) 381 373 { 382 374 struct MUTEX_WAITER *cur; 383 375 ··· 404 396 { 405 397 DEFINE_WAKE_Q(wake_q); 406 398 unsigned long flags; 399 399 + bool has_waiters; 407 400 408 401 ww_mutex_lock_acquired(lock, ctx); 409 402 ··· 426 417 * __ww_mutex_add_waiter() and makes sure we either observe ww->ctx 427 418 * and/or !empty list. 428 419 */ 429 429 - if (likely(!__ww_mutex_has_waiters(&lock->base))) 420 420 + has_waiters = data_race(__ww_mutex_has_waiters(&lock->base)); 421 421 + if (likely(!has_waiters)) 430 422 return; 431 423 432 424 /* ··· 473 463 static inline int 474 464 __ww_mutex_check_kill(struct MUTEX *lock, struct MUTEX_WAITER *waiter, 475 465 struct ww_acquire_ctx *ctx) 476 476 - __must_hold(&lock->wait_lock) 466 466 + __must_hold(&lock->WAIT_LOCK) 477 467 { 478 468 struct ww_mutex *ww = container_of(lock, struct ww_mutex, base); 479 469 struct ww_acquire_ctx *hold_ctx = READ_ONCE(ww->ctx); ··· 524 514 struct MUTEX *lock, 525 515 struct ww_acquire_ctx *ww_ctx, 526 516 struct wake_q_head *wake_q) 527 527 - __must_hold(&lock->wait_lock) 517 517 + __must_hold(&lock->WAIT_LOCK) 528 518 { 529 519 struct MUTEX_WAITER *cur, *pos = NULL; 530 520 bool is_wait_die;

+1

kernel/locking/ww_rt_mutex.c

reviewed

··· 90 90 EXPORT_SYMBOL(ww_mutex_lock_interruptible); 91 91 92 92 void __sched ww_mutex_unlock(struct ww_mutex *lock) 93 93 + __no_context_analysis 93 94 { 94 95 struct rt_mutex *rtm = &lock->base; 95 96

+1

scripts/context-analysis-suppression.txt

reviewed

··· 24 24 src:*include/linux/rcupdate.h=emit 25 25 src:*include/linux/refcount.h=emit 26 26 src:*include/linux/rhashtable.h=emit 27 27 + src:*include/linux/rtmutex*.h=emit 27 28 src:*include/linux/rwlock*.h=emit 28 29 src:*include/linux/rwsem.h=emit 29 30 src:*include/linux/sched*=emit