Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
locking/mutex: Introduce __mutex_trylock_or_handoff()
Yanfei reported that it is possible to loose HANDOFF when we race with
mutex_unlock() and end up setting HANDOFF on an unlocked mutex. At
that point anybody can steal it, losing HANDOFF in the process.
If this happens often enough, we can in fact starve the top waiter.
Solve this by folding the 'set HANDOFF' operation into the trylock
operation, such that either we acquire the lock, or it gets HANDOFF
set. This avoids having HANDOFF set on an unlocked mutex.
Reported-by: Yanfei Xu <yanfei.xu@windriver.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Waiman Long <longman@redhat.com>
Reviewed-by: Yanfei Xu <yanfei.xu@windriver.com>
Link: https://lore.kernel.org/r/20210630154114.958507900@infradead.org
+36
-24
+36
-24
kernel/locking/mutex.c
+36
-24
kernel/locking/mutex.c
···
91
91
return owner & MUTEX_FLAGS;
92
92
}
93
93
94
-
/*
95
-
* Trylock variant that returns the owning task on failure.
96
-
*/
97
-
static inline struct task_struct *__mutex_trylock_or_owner(struct mutex *lock)
94
+
static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, bool handoff)
98
95
{
99
96
unsigned long owner, curr = (unsigned long)current;
100
97
···
101
104
unsigned long task = owner & ~MUTEX_FLAGS;
102
105
103
106
if (task) {
104
-
if (likely(task != curr))
107
+
if (flags & MUTEX_FLAG_PICKUP) {
108
+
if (task != curr)
109
+
break;
110
+
flags &= ~MUTEX_FLAG_PICKUP;
111
+
} else if (handoff) {
112
+
if (flags & MUTEX_FLAG_HANDOFF)
113
+
break;
114
+
flags |= MUTEX_FLAG_HANDOFF;
115
+
} else {
105
116
break;
106
-
107
-
if (likely(!(flags & MUTEX_FLAG_PICKUP)))
108
-
break;
109
-
110
-
flags &= ~MUTEX_FLAG_PICKUP;
117
+
}
111
118
} else {
112
119
#ifdef CONFIG_DEBUG_MUTEXES
113
-
DEBUG_LOCKS_WARN_ON(flags & MUTEX_FLAG_PICKUP);
120
+
DEBUG_LOCKS_WARN_ON(flags & (MUTEX_FLAG_HANDOFF | MUTEX_FLAG_PICKUP));
114
121
#endif
122
+
task = curr;
115
123
}
116
124
117
-
/*
118
-
* We set the HANDOFF bit, we must make sure it doesn't live
119
-
* past the point where we acquire it. This would be possible
120
-
* if we (accidentally) set the bit on an unlocked mutex.
121
-
*/
122
-
flags &= ~MUTEX_FLAG_HANDOFF;
123
-
124
-
if (atomic_long_try_cmpxchg_acquire(&lock->owner, &owner, curr | flags))
125
-
return NULL;
125
+
if (atomic_long_try_cmpxchg_acquire(&lock->owner, &owner, task | flags)) {
126
+
if (task == curr)
127
+
return NULL;
128
+
break;
129
+
}
126
130
}
127
131
128
132
return __owner_task(owner);
133
+
}
134
+
135
+
/*
136
+
* Trylock or set HANDOFF
137
+
*/
138
+
static inline bool __mutex_trylock_or_handoff(struct mutex *lock, bool handoff)
139
+
{
140
+
return !__mutex_trylock_common(lock, handoff);
129
141
}
130
142
131
143
/*
···
142
136
*/
143
137
static inline bool __mutex_trylock(struct mutex *lock)
144
138
{
145
-
return !__mutex_trylock_or_owner(lock);
139
+
return !__mutex_trylock_common(lock, false);
146
140
}
147
141
148
142
#ifndef CONFIG_DEBUG_LOCK_ALLOC
···
484
478
}
485
479
486
480
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
481
+
482
+
/*
483
+
* Trylock variant that returns the owning task on failure.
484
+
*/
485
+
static inline struct task_struct *__mutex_trylock_or_owner(struct mutex *lock)
486
+
{
487
+
return __mutex_trylock_common(lock, false);
488
+
}
487
489
488
490
static inline
489
491
bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
···
1032
1018
schedule_preempt_disabled();
1033
1019
1034
1020
first = __mutex_waiter_is_first(lock, &waiter);
1035
-
if (first)
1036
-
__mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);
1037
1021
1038
1022
set_current_state(state);
1039
1023
/*
···
1039
1027
* state back to RUNNING and fall through the next schedule(),
1040
1028
* or we must see its unlock and acquire.
1041
1029
*/
1042
-
if (__mutex_trylock(lock) ||
1030
+
if (__mutex_trylock_or_handoff(lock, first) ||
1043
1031
(first && mutex_optimistic_spin(lock, ww_ctx, &waiter)))
1044
1032
break;
1045
1033