futex: Move futex_queue() into futex_wait_setup()

+1 -3

io_uring/futex.c

··· 273 273 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 274 274 struct io_ring_ctx *ctx = req->ctx; 275 275 struct io_futex_data *ifd = NULL; 276 - struct futex_hash_bucket *hb; 277 276 int ret; 278 277 279 278 if (!iof->futex_mask) { ··· 294 295 ifd->req = req; 295 296 296 297 ret = futex_wait_setup(iof->uaddr, iof->futex_val, iof->futex_flags, 297 - &ifd->q, &hb); 298 + &ifd->q, NULL, NULL); 298 299 if (!ret) { 299 300 hlist_add_head(&req->hash_node, &ctx->futex_list); 300 301 io_ring_submit_unlock(ctx, issue_flags); 301 302 302 - futex_queue(&ifd->q, hb, NULL); 303 303 return IOU_ISSUE_SKIP_COMPLETE; 304 304 } 305 305

+3 -3

kernel/futex/futex.h

··· 219 219 } 220 220 221 221 extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, 222 - struct futex_q *q, struct futex_hash_bucket **hb); 223 - extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, 224 - struct hrtimer_sleeper *timeout); 222 + struct futex_q *q, union futex_key *key2, 223 + struct task_struct *task); 224 + extern void futex_do_wait(struct futex_q *q, struct hrtimer_sleeper *timeout); 225 225 extern bool __futex_wake_mark(struct futex_q *q); 226 226 extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q); 227 227

+11 -17

kernel/futex/requeue.c

··· 769 769 { 770 770 struct hrtimer_sleeper timeout, *to; 771 771 struct rt_mutex_waiter rt_waiter; 772 - struct futex_hash_bucket *hb; 773 772 union futex_key key2 = FUTEX_KEY_INIT; 774 773 struct futex_q q = futex_q_init; 775 774 struct rt_mutex_base *pi_mutex; ··· 804 805 * Prepare to wait on uaddr. On success, it holds hb->lock and q 805 806 * is initialized. 806 807 */ 807 - ret = futex_wait_setup(uaddr, val, flags, &q, &hb); 808 + ret = futex_wait_setup(uaddr, val, flags, &q, &key2, current); 808 809 if (ret) 809 810 goto out; 810 811 811 - /* 812 - * The check above which compares uaddrs is not sufficient for 813 - * shared futexes. We need to compare the keys: 814 - */ 815 - if (futex_match(&q.key, &key2)) { 816 - futex_q_unlock(hb); 817 - ret = -EINVAL; 818 - goto out; 819 - } 820 - 821 812 /* Queue the futex_q, drop the hb lock, wait for wakeup. */ 822 - futex_wait_queue(hb, &q, to); 813 + futex_do_wait(&q, to); 823 814 824 815 switch (futex_requeue_pi_wakeup_sync(&q)) { 825 816 case Q_REQUEUE_PI_IGNORE: 826 - /* The waiter is still on uaddr1 */ 827 - spin_lock(&hb->lock); 828 - ret = handle_early_requeue_pi_wakeup(hb, &q, to); 829 - spin_unlock(&hb->lock); 817 + { 818 + struct futex_hash_bucket *hb; 819 + 820 + hb = futex_hash(&q.key); 821 + /* The waiter is still on uaddr1 */ 822 + spin_lock(&hb->lock); 823 + ret = handle_early_requeue_pi_wakeup(hb, &q, to); 824 + spin_unlock(&hb->lock); 825 + } 830 826 break; 831 827 832 828 case Q_REQUEUE_PI_LOCKED:

+27 -20

kernel/futex/waitwake.c

··· 339 339 * @q: the futex_q to queue up on 340 340 * @timeout: the prepared hrtimer_sleeper, or null for no timeout 341 341 */ 342 - void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, 343 - struct hrtimer_sleeper *timeout) 342 + void futex_do_wait(struct futex_q *q, struct hrtimer_sleeper *timeout) 344 343 { 345 - /* 346 - * The task state is guaranteed to be set before another task can 347 - * wake it. set_current_state() is implemented using smp_store_mb() and 348 - * futex_queue() calls spin_unlock() upon completion, both serializing 349 - * access to the hash list and forcing another memory barrier. 350 - */ 351 - set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); 352 - futex_queue(q, hb, current); 353 - 354 344 /* Arm the timer */ 355 345 if (timeout) 356 346 hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS); ··· 568 578 * @val: the expected value 569 579 * @flags: futex flags (FLAGS_SHARED, etc.) 570 580 * @q: the associated futex_q 571 - * @hb: storage for hash_bucket pointer to be returned to caller 581 + * @key2: the second futex_key if used for requeue PI 582 + * task: Task queueing this futex 572 583 * 573 584 * Setup the futex_q and locate the hash_bucket. Get the futex value and 574 585 * compare it with the expected value. Handle atomic faults internally. ··· 580 589 * - <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlocked 581 590 */ 582 591 int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, 583 - struct futex_q *q, struct futex_hash_bucket **hb) 592 + struct futex_q *q, union futex_key *key2, 593 + struct task_struct *task) 584 594 { 595 + struct futex_hash_bucket *hb; 585 596 u32 uval; 586 597 int ret; 587 598 ··· 611 618 return ret; 612 619 613 620 retry_private: 614 - *hb = futex_q_lock(q); 621 + hb = futex_q_lock(q); 615 622 616 623 ret = futex_get_value_locked(&uval, uaddr); 617 624 618 625 if (ret) { 619 - futex_q_unlock(*hb); 626 + futex_q_unlock(hb); 620 627 621 628 ret = get_user(uval, uaddr); 622 629 if (ret) ··· 629 636 } 630 637 631 638 if (uval != val) { 632 - futex_q_unlock(*hb); 633 - ret = -EWOULDBLOCK; 639 + futex_q_unlock(hb); 640 + return -EWOULDBLOCK; 634 641 } 642 + 643 + if (key2 && futex_match(&q->key, key2)) { 644 + futex_q_unlock(hb); 645 + return -EINVAL; 646 + } 647 + 648 + /* 649 + * The task state is guaranteed to be set before another task can 650 + * wake it. set_current_state() is implemented using smp_store_mb() and 651 + * futex_queue() calls spin_unlock() upon completion, both serializing 652 + * access to the hash list and forcing another memory barrier. 653 + */ 654 + if (task == current) 655 + set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); 656 + futex_queue(q, hb, task); 635 657 636 658 return ret; 637 659 } ··· 655 647 struct hrtimer_sleeper *to, u32 bitset) 656 648 { 657 649 struct futex_q q = futex_q_init; 658 - struct futex_hash_bucket *hb; 659 650 int ret; 660 651 661 652 if (!bitset) ··· 667 660 * Prepare to wait on uaddr. On success, it holds hb->lock and q 668 661 * is initialized. 669 662 */ 670 - ret = futex_wait_setup(uaddr, val, flags, &q, &hb); 663 + ret = futex_wait_setup(uaddr, val, flags, &q, NULL, current); 671 664 if (ret) 672 665 return ret; 673 666 674 667 /* futex_queue and wait for wakeup, timeout, or a signal. */ 675 - futex_wait_queue(hb, &q, to); 668 + futex_do_wait(&q, to); 676 669 677 670 /* If we were woken (and unqueued), we succeeded, whatever. */ 678 671 if (!futex_unqueue(&q))

Configure Feed

Configure Feed