locking/mutex: Fix ww_mutex wait_list operations

Chaitanya, John and Mikhail reported commit 25500ba7e77c ("locking/mutex:
Remove the list_head from struct mutex") wrecked ww_mutex.

Specifically there were 2 issues:

- __ww_waiter_prev() had the termination condition wrong; it would terminate
when the previous entry was the first, which results in a truncated
iteration: W3, W2, (no W1).

- __mutex_add_waiter(@pos != NULL), as used by __ww_waiter_add() /
__ww_mutex_add_waiter(); this inserts @waiter before @pos (which is what
list_add_tail() does). But this should then also update lock->first_waiter.

Much thanks to Prateek for spotting the __mutex_add_waiter() issue!

Fixes: 25500ba7e77c ("locking/mutex: Remove the list_head from struct mutex")
Reported-by: "Borah, Chaitanya Kumar" <chaitanya.kumar.borah@intel.com>
Closes: https://lore.kernel.org/r/af005996-05e9-4336-8450-d14ca652ba5d%40intel.com
Reported-by: John Stultz <jstultz@google.com>
Closes: https://lore.kernel.org/r/CANDhNCq%3Doizzud3hH3oqGzTrcjB8OwGeineJ3mwZuGdDWG8fRQ%40mail.gmail.com
Reported-by: Mikhail Gavrilov <mikhail.v.gavrilov@gmail.com>
Closes: https://lore.kernel.org/r/CABXGCsO5fKq2nD9nO8yO1z50ZzgCPWqueNXHANjntaswoOh2Dg@mail.gmail.com
Debugged-by: K Prateek Nayak <kprateek.nayak@amd.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Tested-by: K Prateek Nayak <kprateek.nayak@amd.com>
Tested-by: Mikhail Gavrilov <mikhail.v.gavrilov@gmail.com>
Link: https://patch.msgid.link/20260422092335.GH3102924%40noisy.programming.kicks-ass.net

Peter Zijlstra 3 weeks ago 0adc92b9 3bfdc639

+59 -15

2 changed files

expand all

kernel

locking

mutex.c

ww_mutex.h

+27 -13

kernel/locking/mutex.c

··· 198 198 } 199 199 200 200 /* 201 - * Add @waiter to a given location in the lock wait_list and set the 202 - * FLAG_WAITERS flag if it's the first waiter. 201 + * Add @waiter to the @lock wait_list and set the FLAG_WAITERS flag if it's 202 + * the first waiter. 203 + * 204 + * When @pos, @waiter is added before the waiter indicated by @pos. Otherwise 205 + * @waiter will be added to the tail of the list. 203 206 */ 204 207 static void 205 208 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter, 206 - struct mutex_waiter *first) 209 + struct mutex_waiter *pos) 207 210 __must_hold(&lock->wait_lock) 208 211 { 212 + struct mutex_waiter *first = lock->first_waiter; 213 + 209 214 hung_task_set_blocker(lock, BLOCKER_TYPE_MUTEX); 210 215 debug_mutex_add_waiter(lock, waiter, current); 211 216 212 - if (!first) 213 - first = lock->first_waiter; 217 + if (pos) { 218 + /* 219 + * Insert @waiter before @pos. 220 + */ 221 + list_add_tail(&waiter->list, &pos->list); 222 + /* 223 + * If @pos == @first, then @waiter will be the new first. 224 + */ 225 + if (pos == first) 226 + lock->first_waiter = waiter; 227 + return; 228 + } 214 229 215 230 if (first) { 216 231 list_add_tail(&waiter->list, &first->list); 217 - } else { 218 - INIT_LIST_HEAD(&waiter->list); 219 - lock->first_waiter = waiter; 220 - __mutex_set_flag(lock, MUTEX_FLAG_WAITERS); 232 + return; 221 233 } 234 + 235 + INIT_LIST_HEAD(&waiter->list); 236 + lock->first_waiter = waiter; 237 + __mutex_set_flag(lock, MUTEX_FLAG_WAITERS); 222 238 } 223 239 224 240 static void ··· 245 229 __mutex_clear_flag(lock, MUTEX_FLAGS); 246 230 lock->first_waiter = NULL; 247 231 } else { 248 - if (lock->first_waiter == waiter) { 249 - lock->first_waiter = list_first_entry(&waiter->list, 250 - struct mutex_waiter, list); 251 - } 232 + if (lock->first_waiter == waiter) 233 + lock->first_waiter = list_next_entry(waiter, list); 252 234 list_del(&waiter->list); 253 235 } 254 236

+32 -2

kernel/locking/ww_mutex.h

··· 6 6 #define MUTEX_WAITER mutex_waiter 7 7 #define WAIT_LOCK wait_lock 8 8 9 + /* 10 + * +--------+ 11 + * | first | 12 + * +--------+ 13 + * | 14 + * v 15 + * +----+ +----+ +----+ 16 + * | W3 | <-> | W1 | <-> | W2 | 17 + * +----+ +----+ +----+ 18 + * ^ ^ 19 + * +---------------------+ 20 + */ 21 + 9 22 static inline struct mutex_waiter * 10 23 __ww_waiter_first(struct mutex *lock) 11 24 __must_hold(&lock->wait_lock) ··· 26 13 return lock->first_waiter; 27 14 } 28 15 16 + /* 17 + * for (cur = __ww_waiter_first(); cur; cur = __ww_waiter_next()) 18 + * 19 + * Should iterate like: W1, W2, W3 20 + */ 29 21 static inline struct mutex_waiter * 30 22 __ww_waiter_next(struct mutex *lock, struct mutex_waiter *w) 31 23 __must_hold(&lock->wait_lock) 32 24 { 33 25 w = list_next_entry(w, list); 26 + /* 27 + * Terminate if the next entry is the first again, that has already 28 + * been observed. 29 + */ 34 30 if (lock->first_waiter == w) 35 31 return NULL; 36 32 37 33 return w; 38 34 } 39 35 36 + /* 37 + * for (cur = __ww_waiter_last(); cur; cur = __ww_waiter_prev()) 38 + * 39 + * Should iterate like: W3, W2, W1 40 + */ 40 41 static inline struct mutex_waiter * 41 42 __ww_waiter_prev(struct mutex *lock, struct mutex_waiter *w) 42 43 __must_hold(&lock->wait_lock) 43 44 { 44 - w = list_prev_entry(w, list); 45 + /* 46 + * Terminate at the first entry, the previous entry of first is the 47 + * last and that has already been observed. 48 + */ 45 49 if (lock->first_waiter == w) 46 50 return NULL; 47 51 48 - return w; 52 + return list_prev_entry(w, list); 49 53 } 50 54 51 55 static inline struct mutex_waiter *

Configure Feed

Configure Feed