Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'locking-urgent-2021-09-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking fixes from Thomas Gleixner:
"A set of updates for the RT specific reader/writer locking base code:
- Make the fast path reader ordering guarantees correct.
- Code reshuffling to make the fix simpler"
[ This plays ugly games with atomic_add_return_release() because we
don't have a plain atomic_add_release(), and should really be cleaned
up, I think - Linus ]
* tag 'locking-urgent-2021-09-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
locking/rwbase: Take care of ordering guarantee for fastpath reader
locking/rwbase: Extract __rwbase_write_trylock()
locking/rwbase: Properly match set_and_save_state() to restore_state()
+46
-21
+46
-21
kernel/locking/rwbase_rt.c
+46
-21
kernel/locking/rwbase_rt.c
···
41
41
* The risk of writer starvation is there, but the pathological use cases
42
42
* which trigger it are not necessarily the typical RT workloads.
43
43
*
44
+
* Fast-path orderings:
45
+
* The lock/unlock of readers can run in fast paths: lock and unlock are only
46
+
* atomic ops, and there is no inner lock to provide ACQUIRE and RELEASE
47
+
* semantics of rwbase_rt. Atomic ops should thus provide _acquire()
48
+
* and _release() (or stronger).
49
+
*
44
50
* Common code shared between RT rw_semaphore and rwlock
45
51
*/
46
52
···
59
53
* set.
60
54
*/
61
55
for (r = atomic_read(&rwb->readers); r < 0;) {
56
+
/* Fully-ordered if cmpxchg() succeeds, provides ACQUIRE */
62
57
if (likely(atomic_try_cmpxchg(&rwb->readers, &r, r + 1)))
63
58
return 1;
64
59
}
···
169
162
/*
170
163
* rwb->readers can only hit 0 when a writer is waiting for the
171
164
* active readers to leave the critical section.
165
+
*
166
+
* dec_and_test() is fully ordered, provides RELEASE.
172
167
*/
173
168
if (unlikely(atomic_dec_and_test(&rwb->readers)))
174
169
__rwbase_read_unlock(rwb, state);
···
181
172
{
182
173
struct rt_mutex_base *rtm = &rwb->rtmutex;
183
174
184
-
atomic_add(READER_BIAS - bias, &rwb->readers);
175
+
/*
176
+
* _release() is needed in case that reader is in fast path, pairing
177
+
* with atomic_try_cmpxchg() in rwbase_read_trylock(), provides RELEASE
178
+
*/
179
+
(void)atomic_add_return_release(READER_BIAS - bias, &rwb->readers);
185
180
raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
186
181
rwbase_rtmutex_unlock(rtm);
187
182
}
···
209
196
__rwbase_write_unlock(rwb, WRITER_BIAS - 1, flags);
210
197
}
211
198
199
+
static inline bool __rwbase_write_trylock(struct rwbase_rt *rwb)
200
+
{
201
+
/* Can do without CAS because we're serialized by wait_lock. */
202
+
lockdep_assert_held(&rwb->rtmutex.wait_lock);
203
+
204
+
/*
205
+
* _acquire is needed in case the reader is in the fast path, pairing
206
+
* with rwbase_read_unlock(), provides ACQUIRE.
207
+
*/
208
+
if (!atomic_read_acquire(&rwb->readers)) {
209
+
atomic_set(&rwb->readers, WRITER_BIAS);
210
+
return 1;
211
+
}
212
+
213
+
return 0;
214
+
}
215
+
212
216
static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
213
217
unsigned int state)
214
218
{
···
240
210
atomic_sub(READER_BIAS, &rwb->readers);
241
211
242
212
raw_spin_lock_irqsave(&rtm->wait_lock, flags);
243
-
/*
244
-
* set_current_state() for rw_semaphore
245
-
* current_save_and_set_rtlock_wait_state() for rwlock
246
-
*/
247
-
rwbase_set_and_save_current_state(state);
213
+
if (__rwbase_write_trylock(rwb))
214
+
goto out_unlock;
248
215
249
-
/* Block until all readers have left the critical section. */
250
-
for (; atomic_read(&rwb->readers);) {
216
+
rwbase_set_and_save_current_state(state);
217
+
for (;;) {
251
218
/* Optimized out for rwlocks */
252
219
if (rwbase_signal_pending_state(state, current)) {
253
-
__set_current_state(TASK_RUNNING);
220
+
rwbase_restore_current_state();
254
221
__rwbase_write_unlock(rwb, 0, flags);
255
222
return -EINTR;
256
223
}
224
+
225
+
if (__rwbase_write_trylock(rwb))
226
+
break;
227
+
257
228
raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
258
-
259
-
/*
260
-
* Schedule and wait for the readers to leave the critical
261
-
* section. The last reader leaving it wakes the waiter.
262
-
*/
263
-
if (atomic_read(&rwb->readers) != 0)
264
-
rwbase_schedule();
265
-
set_current_state(state);
229
+
rwbase_schedule();
266
230
raw_spin_lock_irqsave(&rtm->wait_lock, flags);
267
-
}
268
231
269
-
atomic_set(&rwb->readers, WRITER_BIAS);
232
+
set_current_state(state);
233
+
}
270
234
rwbase_restore_current_state();
235
+
236
+
out_unlock:
271
237
raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
272
238
return 0;
273
239
}
···
279
253
atomic_sub(READER_BIAS, &rwb->readers);
280
254
281
255
raw_spin_lock_irqsave(&rtm->wait_lock, flags);
282
-
if (!atomic_read(&rwb->readers)) {
283
-
atomic_set(&rwb->readers, WRITER_BIAS);
256
+
if (__rwbase_write_trylock(rwb)) {
284
257
raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
285
258
return 1;
286
259
}