Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'locking-urgent-2021-05-09' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking fixes from Thomas Gleixner:
"A set of locking related fixes and updates:
- Two fixes for the futex syscall related to the timeout handling.
FUTEX_LOCK_PI does not support the FUTEX_CLOCK_REALTIME bit and
because it's not set the time namespace adjustment for clock
MONOTONIC is applied wrongly.
FUTEX_WAIT cannot support the FUTEX_CLOCK_REALTIME bit because its
always a relative timeout.
- Cleanups in the futex syscall entry points which became obvious
when the two timeout handling bugs were fixed.
- Cleanup of queued_write_lock_slowpath() as suggested by Linus
- Fixup of the smp_call_function_single_async() prototype"
* tag 'locking-urgent-2021-05-09' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
futex: Make syscall entry points less convoluted
futex: Get rid of the val2 conditional dance
futex: Do not apply time namespace adjustment on FUTEX_LOCK_PI
Revert 337f13046ff0 ("futex: Allow FUTEX_CLOCK_REALTIME with FUTEX_WAIT op")
locking/qrwlock: Cleanup queued_write_lock_slowpath()
smp: Fix smp_call_function_single_async prototype
+58
-60
+1
-1
include/linux/smp.h
+1
-1
include/linux/smp.h
···
53
53
void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
54
54
void *info, bool wait, const struct cpumask *mask);
55
55
56
-
int smp_call_function_single_async(int cpu, call_single_data_t *csd);
56
+
int smp_call_function_single_async(int cpu, struct __call_single_data *csd);
57
57
58
58
/*
59
59
* Cpus stopping functions in panic. All have default weak definitions.
+40
-42
kernel/futex.c
+40
-42
kernel/futex.c
···
3710
3710
3711
3711
if (op & FUTEX_CLOCK_REALTIME) {
3712
3712
flags |= FLAGS_CLOCKRT;
3713
-
if (cmd != FUTEX_WAIT && cmd != FUTEX_WAIT_BITSET && \
3714
-
cmd != FUTEX_WAIT_REQUEUE_PI)
3713
+
if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI)
3715
3714
return -ENOSYS;
3716
3715
}
3717
3716
···
3757
3758
return -ENOSYS;
3758
3759
}
3759
3760
3761
+
static __always_inline bool futex_cmd_has_timeout(u32 cmd)
3762
+
{
3763
+
switch (cmd) {
3764
+
case FUTEX_WAIT:
3765
+
case FUTEX_LOCK_PI:
3766
+
case FUTEX_WAIT_BITSET:
3767
+
case FUTEX_WAIT_REQUEUE_PI:
3768
+
return true;
3769
+
}
3770
+
return false;
3771
+
}
3772
+
3773
+
static __always_inline int
3774
+
futex_init_timeout(u32 cmd, u32 op, struct timespec64 *ts, ktime_t *t)
3775
+
{
3776
+
if (!timespec64_valid(ts))
3777
+
return -EINVAL;
3778
+
3779
+
*t = timespec64_to_ktime(*ts);
3780
+
if (cmd == FUTEX_WAIT)
3781
+
*t = ktime_add_safe(ktime_get(), *t);
3782
+
else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME))
3783
+
*t = timens_ktime_to_host(CLOCK_MONOTONIC, *t);
3784
+
return 0;
3785
+
}
3760
3786
3761
3787
SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
3762
3788
const struct __kernel_timespec __user *, utime,
3763
3789
u32 __user *, uaddr2, u32, val3)
3764
3790
{
3765
-
struct timespec64 ts;
3791
+
int ret, cmd = op & FUTEX_CMD_MASK;
3766
3792
ktime_t t, *tp = NULL;
3767
-
u32 val2 = 0;
3768
-
int cmd = op & FUTEX_CMD_MASK;
3793
+
struct timespec64 ts;
3769
3794
3770
-
if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
3771
-
cmd == FUTEX_WAIT_BITSET ||
3772
-
cmd == FUTEX_WAIT_REQUEUE_PI)) {
3795
+
if (utime && futex_cmd_has_timeout(cmd)) {
3773
3796
if (unlikely(should_fail_futex(!(op & FUTEX_PRIVATE_FLAG))))
3774
3797
return -EFAULT;
3775
3798
if (get_timespec64(&ts, utime))
3776
3799
return -EFAULT;
3777
-
if (!timespec64_valid(&ts))
3778
-
return -EINVAL;
3779
-
3780
-
t = timespec64_to_ktime(ts);
3781
-
if (cmd == FUTEX_WAIT)
3782
-
t = ktime_add_safe(ktime_get(), t);
3783
-
else if (!(op & FUTEX_CLOCK_REALTIME))
3784
-
t = timens_ktime_to_host(CLOCK_MONOTONIC, t);
3800
+
ret = futex_init_timeout(cmd, op, &ts, &t);
3801
+
if (ret)
3802
+
return ret;
3785
3803
tp = &t;
3786
3804
}
3787
-
/*
3788
-
* requeue parameter in 'utime' if cmd == FUTEX_*_REQUEUE_*.
3789
-
* number of waiters to wake in 'utime' if cmd == FUTEX_WAKE_OP.
3790
-
*/
3791
-
if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
3792
-
cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)
3793
-
val2 = (u32) (unsigned long) utime;
3794
3805
3795
-
return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);
3806
+
return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
3796
3807
}
3797
3808
3798
3809
#ifdef CONFIG_COMPAT
···
3968
3959
const struct old_timespec32 __user *, utime, u32 __user *, uaddr2,
3969
3960
u32, val3)
3970
3961
{
3971
-
struct timespec64 ts;
3962
+
int ret, cmd = op & FUTEX_CMD_MASK;
3972
3963
ktime_t t, *tp = NULL;
3973
-
int val2 = 0;
3974
-
int cmd = op & FUTEX_CMD_MASK;
3964
+
struct timespec64 ts;
3975
3965
3976
-
if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
3977
-
cmd == FUTEX_WAIT_BITSET ||
3978
-
cmd == FUTEX_WAIT_REQUEUE_PI)) {
3966
+
if (utime && futex_cmd_has_timeout(cmd)) {
3979
3967
if (get_old_timespec32(&ts, utime))
3980
3968
return -EFAULT;
3981
-
if (!timespec64_valid(&ts))
3982
-
return -EINVAL;
3983
-
3984
-
t = timespec64_to_ktime(ts);
3985
-
if (cmd == FUTEX_WAIT)
3986
-
t = ktime_add_safe(ktime_get(), t);
3987
-
else if (!(op & FUTEX_CLOCK_REALTIME))
3988
-
t = timens_ktime_to_host(CLOCK_MONOTONIC, t);
3969
+
ret = futex_init_timeout(cmd, op, &ts, &t);
3970
+
if (ret)
3971
+
return ret;
3989
3972
tp = &t;
3990
3973
}
3991
-
if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
3992
-
cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)
3993
-
val2 = (int) (unsigned long) utime;
3994
3974
3995
-
return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);
3975
+
return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
3996
3976
}
3997
3977
#endif /* CONFIG_COMPAT_32BIT_TIME */
3998
3978
+3
-3
kernel/locking/qrwlock.c
+3
-3
kernel/locking/qrwlock.c
···
66
66
arch_spin_lock(&lock->wait_lock);
67
67
68
68
/* Try to acquire the lock directly if no reader is present */
69
-
if (!atomic_read(&lock->cnts) &&
70
-
(atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0))
69
+
if (!(cnts = atomic_read(&lock->cnts)) &&
70
+
atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED))
71
71
goto unlock;
72
72
73
73
/* Set the waiting flag to notify readers that a writer is pending */
74
-
atomic_add(_QW_WAITING, &lock->cnts);
74
+
atomic_or(_QW_WAITING, &lock->cnts);
75
75
76
76
/* When no more readers or writers, set the locked flag */
77
77
do {
+13
-13
kernel/smp.c
+13
-13
kernel/smp.c
···
211
211
} while (0)
212
212
213
213
/* Record current CSD work for current CPU, NULL to erase. */
214
-
static void __csd_lock_record(call_single_data_t *csd)
214
+
static void __csd_lock_record(struct __call_single_data *csd)
215
215
{
216
216
if (!csd) {
217
217
smp_mb(); /* NULL cur_csd after unlock. */
···
226
226
/* Or before unlock, as the case may be. */
227
227
}
228
228
229
-
static __always_inline void csd_lock_record(call_single_data_t *csd)
229
+
static __always_inline void csd_lock_record(struct __call_single_data *csd)
230
230
{
231
231
if (static_branch_unlikely(&csdlock_debug_enabled))
232
232
__csd_lock_record(csd);
233
233
}
234
234
235
-
static int csd_lock_wait_getcpu(call_single_data_t *csd)
235
+
static int csd_lock_wait_getcpu(struct __call_single_data *csd)
236
236
{
237
237
unsigned int csd_type;
238
238
···
282
282
return (type >= ARRAY_SIZE(seq_type)) ? "?" : seq_type[type];
283
283
}
284
284
285
-
static void csd_lock_print_extended(call_single_data_t *csd, int cpu)
285
+
static void csd_lock_print_extended(struct __call_single_data *csd, int cpu)
286
286
{
287
287
struct cfd_seq_local *seq = &per_cpu(cfd_seq_local, cpu);
288
288
unsigned int srccpu = csd->node.src;
···
321
321
* the CSD_TYPE_SYNC/ASYNC types provide the destination CPU,
322
322
* so waiting on other types gets much less information.
323
323
*/
324
-
static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id)
324
+
static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *ts1, int *bug_id)
325
325
{
326
326
int cpu = -1;
327
327
int cpux;
···
387
387
* previous function call. For multi-cpu calls its even more interesting
388
388
* as we'll have to ensure no other cpu is observing our csd.
389
389
*/
390
-
static void __csd_lock_wait(call_single_data_t *csd)
390
+
static void __csd_lock_wait(struct __call_single_data *csd)
391
391
{
392
392
int bug_id = 0;
393
393
u64 ts0, ts1;
···
401
401
smp_acquire__after_ctrl_dep();
402
402
}
403
403
404
-
static __always_inline void csd_lock_wait(call_single_data_t *csd)
404
+
static __always_inline void csd_lock_wait(struct __call_single_data *csd)
405
405
{
406
406
if (static_branch_unlikely(&csdlock_debug_enabled)) {
407
407
__csd_lock_wait(csd);
···
431
431
#else
432
432
#define cfd_seq_store(var, src, dst, type)
433
433
434
-
static void csd_lock_record(call_single_data_t *csd)
434
+
static void csd_lock_record(struct __call_single_data *csd)
435
435
{
436
436
}
437
437
438
-
static __always_inline void csd_lock_wait(call_single_data_t *csd)
438
+
static __always_inline void csd_lock_wait(struct __call_single_data *csd)
439
439
{
440
440
smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
441
441
}
442
442
#endif
443
443
444
-
static __always_inline void csd_lock(call_single_data_t *csd)
444
+
static __always_inline void csd_lock(struct __call_single_data *csd)
445
445
{
446
446
csd_lock_wait(csd);
447
447
csd->node.u_flags |= CSD_FLAG_LOCK;
···
454
454
smp_wmb();
455
455
}
456
456
457
-
static __always_inline void csd_unlock(call_single_data_t *csd)
457
+
static __always_inline void csd_unlock(struct __call_single_data *csd)
458
458
{
459
459
WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));
460
460
···
501
501
* for execution on the given CPU. data must already have
502
502
* ->func, ->info, and ->flags set.
503
503
*/
504
-
static int generic_exec_single(int cpu, call_single_data_t *csd)
504
+
static int generic_exec_single(int cpu, struct __call_single_data *csd)
505
505
{
506
506
if (cpu == smp_processor_id()) {
507
507
smp_call_func_t func = csd->func;
···
784
784
* NOTE: Be careful, there is unfortunately no current debugging facility to
785
785
* validate the correctness of this serialization.
786
786
*/
787
-
int smp_call_function_single_async(int cpu, call_single_data_t *csd)
787
+
int smp_call_function_single_async(int cpu, struct __call_single_data *csd)
788
788
{
789
789
int err = 0;
790
790