Merge tag 'io_uring-5.12-2021-03-19' of git://git.kernel.dk/linux-block

+5 -1

fs/io-wq.c

··· 488 488 set_task_comm(current, buf); 489 489 490 490 while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { 491 + long ret; 492 + 491 493 set_current_state(TASK_INTERRUPTIBLE); 492 494 loop: 493 495 raw_spin_lock_irq(&wqe->lock); ··· 500 498 __io_worker_idle(wqe, worker); 501 499 raw_spin_unlock_irq(&wqe->lock); 502 500 io_flush_signals(); 503 - if (schedule_timeout(WORKER_IDLE_TIMEOUT)) 501 + ret = schedule_timeout(WORKER_IDLE_TIMEOUT); 502 + if (try_to_freeze() || ret) 504 503 continue; 505 504 if (fatal_signal_pending(current)) 506 505 break; ··· 712 709 set_current_state(TASK_INTERRUPTIBLE); 713 710 io_wq_check_workers(wq); 714 711 schedule_timeout(HZ); 712 + try_to_freeze(); 715 713 if (fatal_signal_pending(current)) 716 714 set_bit(IO_WQ_BIT_EXIT, &wq->state); 717 715 } while (!test_bit(IO_WQ_BIT_EXIT, &wq->state));

+9 -1

fs/io-wq.h

··· 2 2 #define INTERNAL_IO_WQ_H 3 3 4 4 #include <linux/refcount.h> 5 - #include <linux/io_uring.h> 6 5 7 6 struct io_wq; 8 7 ··· 18 19 IO_WQ_CANCEL_OK, /* cancelled before started */ 19 20 IO_WQ_CANCEL_RUNNING, /* found, running, and attempted cancelled */ 20 21 IO_WQ_CANCEL_NOTFOUND, /* work not found */ 22 + }; 23 + 24 + struct io_wq_work_node { 25 + struct io_wq_work_node *next; 26 + }; 27 + 28 + struct io_wq_work_list { 29 + struct io_wq_work_node *first; 30 + struct io_wq_work_node *last; 21 31 }; 22 32 23 33 static inline void wq_list_add_after(struct io_wq_work_node *node,

+127 -101

fs/io_uring.c

··· 258 258 259 259 struct io_sq_data { 260 260 refcount_t refs; 261 - struct rw_semaphore rw_lock; 261 + atomic_t park_pending; 262 + struct mutex lock; 262 263 263 264 /* ctx's that are using this sqd */ 264 265 struct list_head ctx_list; ··· 274 273 275 274 unsigned long state; 276 275 struct completion exited; 276 + struct callback_head *park_task_work; 277 277 }; 278 278 279 279 #define IO_IOPOLL_BATCH 8 ··· 404 402 struct socket *ring_sock; 405 403 #endif 406 404 407 - struct idr io_buffer_idr; 405 + struct xarray io_buffers; 408 406 409 407 struct xarray personalities; 410 408 u32 pers_next; ··· 454 452 /* Keep this last, we don't need it for the fast path */ 455 453 struct work_struct exit_work; 456 454 struct list_head tctx_list; 455 + }; 456 + 457 + struct io_uring_task { 458 + /* submission side */ 459 + struct xarray xa; 460 + struct wait_queue_head wait; 461 + const struct io_ring_ctx *last; 462 + struct io_wq *io_wq; 463 + struct percpu_counter inflight; 464 + atomic_t in_idle; 465 + bool sqpoll; 466 + 467 + spinlock_t task_lock; 468 + struct io_wq_work_list task_list; 469 + unsigned long task_state; 470 + struct callback_head task_work; 457 471 }; 458 472 459 473 /* ··· 1153 1135 init_waitqueue_head(&ctx->cq_wait); 1154 1136 INIT_LIST_HEAD(&ctx->cq_overflow_list); 1155 1137 init_completion(&ctx->ref_comp); 1156 - idr_init(&ctx->io_buffer_idr); 1138 + xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1); 1157 1139 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); 1158 1140 mutex_init(&ctx->uring_lock); 1159 1141 init_waitqueue_head(&ctx->wait); ··· 1568 1550 io_put_task(req->task, 1); 1569 1551 list_add(&req->compl.list, &cs->locked_free_list); 1570 1552 cs->locked_free_nr++; 1571 - } else 1572 - req = NULL; 1553 + } else { 1554 + if (!percpu_ref_tryget(&ctx->refs)) 1555 + req = NULL; 1556 + } 1573 1557 io_commit_cqring(ctx); 1574 1558 spin_unlock_irqrestore(&ctx->completion_lock, flags); 1575 - io_cqring_ev_posted(ctx); 1576 1559 1577 - if (req) 1560 + if (req) { 1561 + io_cqring_ev_posted(ctx); 1578 1562 percpu_ref_put(&ctx->refs); 1563 + } 1579 1564 } 1580 1565 1581 1566 static void io_req_complete_state(struct io_kiocb *req, long res, ··· 1946 1925 return ret; 1947 1926 } 1948 1927 1928 + static bool io_run_task_work_head(struct callback_head **work_head) 1929 + { 1930 + struct callback_head *work, *next; 1931 + bool executed = false; 1932 + 1933 + do { 1934 + work = xchg(work_head, NULL); 1935 + if (!work) 1936 + break; 1937 + 1938 + do { 1939 + next = work->next; 1940 + work->func(work); 1941 + work = next; 1942 + cond_resched(); 1943 + } while (work); 1944 + executed = true; 1945 + } while (1); 1946 + 1947 + return executed; 1948 + } 1949 + 1950 + static void io_task_work_add_head(struct callback_head **work_head, 1951 + struct callback_head *task_work) 1952 + { 1953 + struct callback_head *head; 1954 + 1955 + do { 1956 + head = READ_ONCE(*work_head); 1957 + task_work->next = head; 1958 + } while (cmpxchg(work_head, head, task_work) != head); 1959 + } 1960 + 1949 1961 static void io_req_task_work_add_fallback(struct io_kiocb *req, 1950 1962 task_work_func_t cb) 1951 1963 { 1952 - struct io_ring_ctx *ctx = req->ctx; 1953 - struct callback_head *head; 1954 - 1955 1964 init_task_work(&req->task_work, cb); 1956 - do { 1957 - head = READ_ONCE(ctx->exit_task_work); 1958 - req->task_work.next = head; 1959 - } while (cmpxchg(&ctx->exit_task_work, head, &req->task_work) != head); 1965 + io_task_work_add_head(&req->ctx->exit_task_work, &req->task_work); 1960 1966 } 1961 1967 1962 1968 static void __io_req_task_cancel(struct io_kiocb *req, int error) ··· 2891 2843 2892 2844 lockdep_assert_held(&req->ctx->uring_lock); 2893 2845 2894 - head = idr_find(&req->ctx->io_buffer_idr, bgid); 2846 + head = xa_load(&req->ctx->io_buffers, bgid); 2895 2847 if (head) { 2896 2848 if (!list_empty(&head->list)) { 2897 2849 kbuf = list_last_entry(&head->list, struct io_buffer, ··· 2899 2851 list_del(&kbuf->list); 2900 2852 } else { 2901 2853 kbuf = head; 2902 - idr_remove(&req->ctx->io_buffer_idr, bgid); 2854 + xa_erase(&req->ctx->io_buffers, bgid); 2903 2855 } 2904 2856 if (*len > kbuf->len) 2905 2857 *len = kbuf->len; ··· 3940 3892 } 3941 3893 i++; 3942 3894 kfree(buf); 3943 - idr_remove(&ctx->io_buffer_idr, bgid); 3895 + xa_erase(&ctx->io_buffers, bgid); 3944 3896 3945 3897 return i; 3946 3898 } ··· 3958 3910 lockdep_assert_held(&ctx->uring_lock); 3959 3911 3960 3912 ret = -ENOENT; 3961 - head = idr_find(&ctx->io_buffer_idr, p->bgid); 3913 + head = xa_load(&ctx->io_buffers, p->bgid); 3962 3914 if (head) 3963 3915 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs); 3964 3916 if (ret < 0) ··· 4041 3993 4042 3994 lockdep_assert_held(&ctx->uring_lock); 4043 3995 4044 - list = head = idr_find(&ctx->io_buffer_idr, p->bgid); 3996 + list = head = xa_load(&ctx->io_buffers, p->bgid); 4045 3997 4046 3998 ret = io_add_buffers(p, &head); 4047 - if (ret < 0) 4048 - goto out; 4049 - 4050 - if (!list) { 4051 - ret = idr_alloc(&ctx->io_buffer_idr, head, p->bgid, p->bgid + 1, 4052 - GFP_KERNEL); 4053 - if (ret < 0) { 3999 + if (ret >= 0 && !list) { 4000 + ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL); 4001 + if (ret < 0) 4054 4002 __io_remove_buffers(ctx, head, p->bgid, -1U); 4055 - goto out; 4056 - } 4057 4003 } 4058 - out: 4059 4004 if (ret < 0) 4060 4005 req_set_fail_links(req); 4061 4006 ··· 4400 4359 kmsg = &iomsg; 4401 4360 } 4402 4361 4403 - flags = req->sr_msg.msg_flags; 4362 + flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; 4404 4363 if (flags & MSG_DONTWAIT) 4405 4364 req->flags |= REQ_F_NOWAIT; 4406 4365 else if (issue_flags & IO_URING_F_NONBLOCK) ··· 4444 4403 msg.msg_controllen = 0; 4445 4404 msg.msg_namelen = 0; 4446 4405 4447 - flags = req->sr_msg.msg_flags; 4406 + flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; 4448 4407 if (flags & MSG_DONTWAIT) 4449 4408 req->flags |= REQ_F_NOWAIT; 4450 4409 else if (issue_flags & IO_URING_F_NONBLOCK) ··· 4634 4593 1, req->sr_msg.len); 4635 4594 } 4636 4595 4637 - flags = req->sr_msg.msg_flags; 4596 + flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; 4638 4597 if (flags & MSG_DONTWAIT) 4639 4598 req->flags |= REQ_F_NOWAIT; 4640 4599 else if (force_nonblock) ··· 4693 4652 msg.msg_iocb = NULL; 4694 4653 msg.msg_flags = 0; 4695 4654 4696 - flags = req->sr_msg.msg_flags; 4655 + flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; 4697 4656 if (flags & MSG_DONTWAIT) 4698 4657 req->flags |= REQ_F_NOWAIT; 4699 4658 else if (force_nonblock) ··· 6245 6204 spin_unlock_irqrestore(&ctx->completion_lock, flags); 6246 6205 6247 6206 if (prev) { 6248 - req_set_fail_links(prev); 6249 6207 io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME); 6250 6208 io_put_req_deferred(prev, 1); 6251 6209 } else { ··· 6734 6694 set_cpus_allowed_ptr(current, cpu_online_mask); 6735 6695 current->flags |= PF_NO_SETAFFINITY; 6736 6696 6737 - down_read(&sqd->rw_lock); 6738 - 6697 + mutex_lock(&sqd->lock); 6739 6698 while (!test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) { 6740 6699 int ret; 6741 6700 bool cap_entries, sqt_spin, needs_sched; 6742 6701 6743 6702 if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)) { 6744 - up_read(&sqd->rw_lock); 6703 + mutex_unlock(&sqd->lock); 6745 6704 cond_resched(); 6746 - down_read(&sqd->rw_lock); 6705 + mutex_lock(&sqd->lock); 6747 6706 io_run_task_work(); 6707 + io_run_task_work_head(&sqd->park_task_work); 6748 6708 timeout = jiffies + sqd->sq_thread_idle; 6749 6709 continue; 6750 6710 } ··· 6790 6750 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) 6791 6751 io_ring_set_wakeup_flag(ctx); 6792 6752 6793 - up_read(&sqd->rw_lock); 6753 + mutex_unlock(&sqd->lock); 6794 6754 schedule(); 6795 - down_read(&sqd->rw_lock); 6755 + try_to_freeze(); 6756 + mutex_lock(&sqd->lock); 6796 6757 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) 6797 6758 io_ring_clear_wakeup_flag(ctx); 6798 6759 } 6799 6760 6800 6761 finish_wait(&sqd->wait, &wait); 6762 + io_run_task_work_head(&sqd->park_task_work); 6801 6763 timeout = jiffies + sqd->sq_thread_idle; 6802 6764 } 6803 - up_read(&sqd->rw_lock); 6804 - down_write(&sqd->rw_lock); 6805 - /* 6806 - * someone may have parked and added a cancellation task_work, run 6807 - * it first because we don't want it in io_uring_cancel_sqpoll() 6808 - */ 6809 - io_run_task_work(); 6810 6765 6811 6766 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) 6812 6767 io_uring_cancel_sqpoll(ctx); 6813 6768 sqd->thread = NULL; 6814 6769 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) 6815 6770 io_ring_set_wakeup_flag(ctx); 6816 - up_write(&sqd->rw_lock); 6771 + mutex_unlock(&sqd->lock); 6817 6772 6818 6773 io_run_task_work(); 6774 + io_run_task_work_head(&sqd->park_task_work); 6819 6775 complete(&sqd->exited); 6820 6776 do_exit(0); 6821 6777 } ··· 7111 7075 } 7112 7076 7113 7077 static void io_sq_thread_unpark(struct io_sq_data *sqd) 7114 - __releases(&sqd->rw_lock) 7078 + __releases(&sqd->lock) 7115 7079 { 7116 7080 WARN_ON_ONCE(sqd->thread == current); 7117 7081 7082 + /* 7083 + * Do the dance but not conditional clear_bit() because it'd race with 7084 + * other threads incrementing park_pending and setting the bit. 7085 + */ 7118 7086 clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); 7119 - up_write(&sqd->rw_lock); 7087 + if (atomic_dec_return(&sqd->park_pending)) 7088 + set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); 7089 + mutex_unlock(&sqd->lock); 7120 7090 } 7121 7091 7122 7092 static void io_sq_thread_park(struct io_sq_data *sqd) 7123 - __acquires(&sqd->rw_lock) 7093 + __acquires(&sqd->lock) 7124 7094 { 7125 7095 WARN_ON_ONCE(sqd->thread == current); 7126 7096 7097 + atomic_inc(&sqd->park_pending); 7127 7098 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); 7128 - down_write(&sqd->rw_lock); 7129 - /* set again for consistency, in case concurrent parks are happening */ 7130 - set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); 7099 + mutex_lock(&sqd->lock); 7131 7100 if (sqd->thread) 7132 7101 wake_up_process(sqd->thread); 7133 7102 } ··· 7141 7100 { 7142 7101 WARN_ON_ONCE(sqd->thread == current); 7143 7102 7144 - down_write(&sqd->rw_lock); 7103 + mutex_lock(&sqd->lock); 7145 7104 set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); 7146 7105 if (sqd->thread) 7147 7106 wake_up_process(sqd->thread); 7148 - up_write(&sqd->rw_lock); 7107 + mutex_unlock(&sqd->lock); 7149 7108 wait_for_completion(&sqd->exited); 7150 7109 } 7151 7110 7152 7111 static void io_put_sq_data(struct io_sq_data *sqd) 7153 7112 { 7154 7113 if (refcount_dec_and_test(&sqd->refs)) { 7114 + WARN_ON_ONCE(atomic_read(&sqd->park_pending)); 7115 + 7155 7116 io_sq_thread_stop(sqd); 7156 7117 kfree(sqd); 7157 7118 } ··· 7227 7184 if (!sqd) 7228 7185 return ERR_PTR(-ENOMEM); 7229 7186 7187 + atomic_set(&sqd->park_pending, 0); 7230 7188 refcount_set(&sqd->refs, 1); 7231 7189 INIT_LIST_HEAD(&sqd->ctx_list); 7232 - init_rwsem(&sqd->rw_lock); 7190 + mutex_init(&sqd->lock); 7233 7191 init_waitqueue_head(&sqd->wait); 7234 7192 init_completion(&sqd->exited); 7235 7193 return sqd; ··· 7910 7866 7911 7867 ret = 0; 7912 7868 io_sq_thread_park(sqd); 7869 + list_add(&ctx->sqd_list, &sqd->ctx_list); 7870 + io_sqd_update_thread_idle(sqd); 7913 7871 /* don't attach to a dying SQPOLL thread, would be racy */ 7914 - if (attached && !sqd->thread) { 7872 + if (attached && !sqd->thread) 7915 7873 ret = -ENXIO; 7916 - } else { 7917 - list_add(&ctx->sqd_list, &sqd->ctx_list); 7918 - io_sqd_update_thread_idle(sqd); 7919 - } 7920 7874 io_sq_thread_unpark(sqd); 7921 7875 7922 - if (ret < 0) { 7923 - io_put_sq_data(sqd); 7924 - ctx->sq_data = NULL; 7925 - return ret; 7926 - } else if (attached) { 7876 + if (ret < 0) 7877 + goto err; 7878 + if (attached) 7927 7879 return 0; 7928 - } 7929 7880 7930 7881 if (p->flags & IORING_SETUP_SQ_AFF) { 7931 7882 int cpu = p->sq_thread_cpu; ··· 8371 8332 return -ENXIO; 8372 8333 } 8373 8334 8374 - static int __io_destroy_buffers(int id, void *p, void *data) 8375 - { 8376 - struct io_ring_ctx *ctx = data; 8377 - struct io_buffer *buf = p; 8378 - 8379 - __io_remove_buffers(ctx, buf, id, -1U); 8380 - return 0; 8381 - } 8382 - 8383 8335 static void io_destroy_buffers(struct io_ring_ctx *ctx) 8384 8336 { 8385 - idr_for_each(&ctx->io_buffer_idr, __io_destroy_buffers, ctx); 8386 - idr_destroy(&ctx->io_buffer_idr); 8337 + struct io_buffer *buf; 8338 + unsigned long index; 8339 + 8340 + xa_for_each(&ctx->io_buffers, index, buf) 8341 + __io_remove_buffers(ctx, buf, index, -1U); 8387 8342 } 8388 8343 8389 8344 static void io_req_cache_free(struct list_head *list, struct task_struct *tsk) ··· 8419 8386 { 8420 8387 /* 8421 8388 * Some may use context even when all refs and requests have been put, 8422 - * and they are free to do so while still holding uring_lock, see 8423 - * __io_req_task_submit(). Wait for them to finish. 8389 + * and they are free to do so while still holding uring_lock or 8390 + * completion_lock, see __io_req_task_submit(). Wait for them to finish. 8424 8391 */ 8425 8392 mutex_lock(&ctx->uring_lock); 8426 8393 mutex_unlock(&ctx->uring_lock); 8394 + spin_lock_irq(&ctx->completion_lock); 8395 + spin_unlock_irq(&ctx->completion_lock); 8427 8396 8428 8397 io_sq_thread_finish(ctx); 8429 8398 io_sqe_buffers_unregister(ctx); ··· 8513 8478 return -EINVAL; 8514 8479 } 8515 8480 8516 - static bool io_run_ctx_fallback(struct io_ring_ctx *ctx) 8481 + static inline bool io_run_ctx_fallback(struct io_ring_ctx *ctx) 8517 8482 { 8518 - struct callback_head *work, *next; 8519 - bool executed = false; 8520 - 8521 - do { 8522 - work = xchg(&ctx->exit_task_work, NULL); 8523 - if (!work) 8524 - break; 8525 - 8526 - do { 8527 - next = work->next; 8528 - work->func(work); 8529 - work = next; 8530 - cond_resched(); 8531 - } while (work); 8532 - executed = true; 8533 - } while (1); 8534 - 8535 - return executed; 8483 + return io_run_task_work_head(&ctx->exit_task_work); 8536 8484 } 8537 8485 8538 8486 struct io_tctx_exit { ··· 8597 8579 xa_for_each(&ctx->personalities, index, creds) 8598 8580 io_unregister_personality(ctx, index); 8599 8581 mutex_unlock(&ctx->uring_lock); 8582 + 8583 + /* prevent SQPOLL from submitting new requests */ 8584 + if (ctx->sq_data) { 8585 + io_sq_thread_park(ctx->sq_data); 8586 + list_del_init(&ctx->sqd_list); 8587 + io_sqd_update_thread_idle(ctx->sq_data); 8588 + io_sq_thread_unpark(ctx->sq_data); 8589 + } 8600 8590 8601 8591 io_kill_timeouts(ctx, NULL, NULL); 8602 8592 io_poll_remove_all(ctx, NULL, NULL); ··· 8905 8879 if (task) { 8906 8880 init_completion(&work.completion); 8907 8881 init_task_work(&work.task_work, io_sqpoll_cancel_cb); 8908 - WARN_ON_ONCE(task_work_add(task, &work.task_work, TWA_SIGNAL)); 8882 + io_task_work_add_head(&sqd->park_task_work, &work.task_work); 8909 8883 wake_up_process(task); 8910 8884 } 8911 8885 io_sq_thread_unpark(sqd);

-25

include/linux/io_uring.h

··· 5 5 #include <linux/sched.h> 6 6 #include <linux/xarray.h> 7 7 8 - struct io_wq_work_node { 9 - struct io_wq_work_node *next; 10 - }; 11 - 12 - struct io_wq_work_list { 13 - struct io_wq_work_node *first; 14 - struct io_wq_work_node *last; 15 - }; 16 - 17 - struct io_uring_task { 18 - /* submission side */ 19 - struct xarray xa; 20 - struct wait_queue_head wait; 21 - void *last; 22 - void *io_wq; 23 - struct percpu_counter inflight; 24 - atomic_t in_idle; 25 - bool sqpoll; 26 - 27 - spinlock_t task_lock; 28 - struct io_wq_work_list task_list; 29 - unsigned long task_state; 30 - struct callback_head task_work; 31 - }; 32 - 33 8 #if defined(CONFIG_IO_URING) 34 9 struct sock *io_uring_get_socket(struct file *file); 35 10 void __io_uring_task_cancel(void);

-1

kernel/fork.c

··· 2444 2444 if (!IS_ERR(tsk)) { 2445 2445 sigfillset(&tsk->blocked); 2446 2446 sigdelsetmask(&tsk->blocked, sigmask(SIGKILL)); 2447 - tsk->flags |= PF_NOFREEZE; 2448 2447 } 2449 2448 return tsk; 2450 2449 }

+1 -1

kernel/freezer.c

··· 134 134 return false; 135 135 } 136 136 137 - if (!(p->flags & PF_KTHREAD)) 137 + if (!(p->flags & (PF_KTHREAD | PF_IO_WORKER))) 138 138 fake_signal_wake_up(p); 139 139 else 140 140 wake_up_state(p, TASK_INTERRUPTIBLE);

Configure Feed

Configure Feed