futex: Introduce futex_q_lockptr_lock()

futex_lock_pi() and __fixup_pi_state_owner() acquire the
futex_q::lock_ptr without holding a reference assuming the previously
obtained hash bucket and the assigned lock_ptr are still valid. This
isn't the case once the private hash can be resized and becomes invalid
after the reference drop.

Introduce futex_q_lockptr_lock() to lock the hash bucket recorded in
futex_q::lock_ptr. The lock pointer is read in a RCU section to ensure
that it does not go away if the hash bucket has been replaced and the
old pointer has been observed. After locking the pointer needs to be
compared to check if it changed. If so then the hash bucket has been
replaced and the user has been moved to the new one and lock_ptr has
been updated. The lock operation needs to be redone in this case.

The locked hash bucket is not returned.

A special case is an early return in futex_lock_pi() (due to signal or
timeout) and a successful futex_wait_requeue_pi(). In both cases a valid
futex_q::lock_ptr is expected (and its matching hash bucket) but since
the waiter has been removed from the hash this can no longer be
guaranteed. Therefore before the waiter is removed and a reference is
acquired which is later dropped by the waiter to avoid a resize.

Add futex_q_lockptr_lock() and use it.
Acquire an additional reference in requeue_pi_wake_futex() and
futex_unlock_pi() while the futex_q is removed, denote this extra
reference in futex_q::drop_hb_ref and let the waiter drop the reference
in this case.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250416162921.513656-11-bigeasy@linutronix.de

authored by

Sebastian Andrzej Siewior and committed by

Peter Zijlstra 1 year ago b04b8f30 fe00e88d

+53 -6

4 changed files

expand all

kernel

futex

core.c

futex.h

pi.c

requeue.c

+25

kernel/futex/core.c

··· 134 134 return &futex_queues[hash & futex_hashmask]; 135 135 } 136 136 137 + /** 138 + * futex_hash_get - Get an additional reference for the local hash. 139 + * @hb: ptr to the private local hash. 140 + * 141 + * Obtain an additional reference for the already obtained hash bucket. The 142 + * caller must already own an reference. 143 + */ 137 144 void futex_hash_get(struct futex_hash_bucket *hb) { } 138 145 void futex_hash_put(struct futex_hash_bucket *hb) { } 139 146 ··· 620 613 } 621 614 622 615 return ret; 616 + } 617 + 618 + void futex_q_lockptr_lock(struct futex_q *q) 619 + { 620 + spinlock_t *lock_ptr; 621 + 622 + /* 623 + * See futex_unqueue() why lock_ptr can change. 624 + */ 625 + guard(rcu)(); 626 + retry: 627 + lock_ptr = READ_ONCE(q->lock_ptr); 628 + spin_lock(lock_ptr); 629 + 630 + if (unlikely(lock_ptr != q->lock_ptr)) { 631 + spin_unlock(lock_ptr); 632 + goto retry; 633 + } 623 634 } 624 635 625 636 /*

+2 -1

kernel/futex/futex.h

··· 183 183 union futex_key *requeue_pi_key; 184 184 u32 bitset; 185 185 atomic_t requeue_state; 186 + bool drop_hb_ref; 186 187 #ifdef CONFIG_PREEMPT_RT 187 188 struct rcuwait requeue_wait; 188 189 #endif ··· 198 197 199 198 extern int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, 200 199 enum futex_access rw); 201 - 200 + extern void futex_q_lockptr_lock(struct futex_q *q); 202 201 extern struct hrtimer_sleeper * 203 202 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, 204 203 int flags, u64 range_ns);

+13 -2

kernel/futex/pi.c

··· 806 806 break; 807 807 } 808 808 809 - spin_lock(q->lock_ptr); 809 + futex_q_lockptr_lock(q); 810 810 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); 811 811 812 812 /* ··· 1072 1072 * spinlock/rtlock (which might enqueue its own rt_waiter) and fix up 1073 1073 * the 1074 1074 */ 1075 - spin_lock(q.lock_ptr); 1075 + futex_q_lockptr_lock(&q); 1076 1076 /* 1077 1077 * Waiter is unqueued. 1078 1078 */ ··· 1092 1092 1093 1093 futex_unqueue_pi(&q); 1094 1094 spin_unlock(q.lock_ptr); 1095 + if (q.drop_hb_ref) { 1096 + CLASS(hb, hb)(&q.key); 1097 + /* Additional reference from futex_unlock_pi() */ 1098 + futex_hash_put(hb); 1099 + } 1095 1100 goto out; 1096 1101 1097 1102 out_unlock_put_key: ··· 1205 1200 */ 1206 1201 rt_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex); 1207 1202 if (!rt_waiter) { 1203 + /* 1204 + * Acquire a reference for the leaving waiter to ensure 1205 + * valid futex_q::lock_ptr. 1206 + */ 1207 + futex_hash_get(hb); 1208 + top_waiter->drop_hb_ref = true; 1208 1209 __futex_unqueue(top_waiter); 1209 1210 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); 1210 1211 goto retry_hb;

+13 -3

kernel/futex/requeue.c

··· 231 231 232 232 WARN_ON(!q->rt_waiter); 233 233 q->rt_waiter = NULL; 234 - 234 + /* 235 + * Acquire a reference for the waiter to ensure valid 236 + * futex_q::lock_ptr. 237 + */ 238 + futex_hash_get(hb); 239 + q->drop_hb_ref = true; 235 240 q->lock_ptr = &hb->lock; 236 241 237 242 /* Signal locked state to the waiter */ ··· 831 826 case Q_REQUEUE_PI_LOCKED: 832 827 /* The requeue acquired the lock */ 833 828 if (q.pi_state && (q.pi_state->owner != current)) { 834 - spin_lock(q.lock_ptr); 829 + futex_q_lockptr_lock(&q); 835 830 ret = fixup_pi_owner(uaddr2, &q, true); 836 831 /* 837 832 * Drop the reference to the pi state which the ··· 858 853 if (ret && !rt_mutex_cleanup_proxy_lock(pi_mutex, &rt_waiter)) 859 854 ret = 0; 860 855 861 - spin_lock(q.lock_ptr); 856 + futex_q_lockptr_lock(&q); 862 857 debug_rt_mutex_free_waiter(&rt_waiter); 863 858 /* 864 859 * Fixup the pi_state owner and possibly acquire the lock if we ··· 889 884 break; 890 885 default: 891 886 BUG(); 887 + } 888 + if (q.drop_hb_ref) { 889 + CLASS(hb, hb)(&q.key); 890 + /* Additional reference from requeue_pi_wake_futex() */ 891 + futex_hash_put(hb); 892 892 } 893 893 894 894 out:

Configure Feed

Configure Feed