Merge tag 'io_uring-5.9-2020-08-28' of git://git.kernel.dk/linux-block

+76 -44

2 changed files

expand all

io-wq.c

io_uring.c

+19 -2

fs/io-wq.c

··· 925 925 return match->nr_running && !match->cancel_all; 926 926 } 927 927 928 + static inline void io_wqe_remove_pending(struct io_wqe *wqe, 929 + struct io_wq_work *work, 930 + struct io_wq_work_node *prev) 931 + { 932 + unsigned int hash = io_get_work_hash(work); 933 + struct io_wq_work *prev_work = NULL; 934 + 935 + if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) { 936 + if (prev) 937 + prev_work = container_of(prev, struct io_wq_work, list); 938 + if (prev_work && io_get_work_hash(prev_work) == hash) 939 + wqe->hash_tail[hash] = prev_work; 940 + else 941 + wqe->hash_tail[hash] = NULL; 942 + } 943 + wq_list_del(&wqe->work_list, &work->list, prev); 944 + } 945 + 928 946 static void io_wqe_cancel_pending_work(struct io_wqe *wqe, 929 947 struct io_cb_cancel_data *match) 930 948 { ··· 956 938 work = container_of(node, struct io_wq_work, list); 957 939 if (!match->fn(work, match->data)) 958 940 continue; 959 - 960 - wq_list_del(&wqe->work_list, node, prev); 941 + io_wqe_remove_pending(wqe, work, prev); 961 942 spin_unlock_irqrestore(&wqe->lock, flags); 962 943 io_run_cancel(work, wqe); 963 944 match->nr_pending++;

+57 -42

fs/io_uring.c

··· 1150 1150 io_req_init_async(req); 1151 1151 1152 1152 if (req->flags & REQ_F_ISREG) { 1153 - if (def->hash_reg_file) 1153 + if (def->hash_reg_file || (req->ctx->flags & IORING_SETUP_IOPOLL)) 1154 1154 io_wq_hash_work(&req->work, file_inode(req->file)); 1155 1155 } else { 1156 1156 if (def->unbound_nonreg_file) ··· 1746 1746 return __io_req_find_next(req); 1747 1747 } 1748 1748 1749 - static int io_req_task_work_add(struct io_kiocb *req, struct callback_head *cb) 1749 + static int io_req_task_work_add(struct io_kiocb *req, struct callback_head *cb, 1750 + bool twa_signal_ok) 1750 1751 { 1751 1752 struct task_struct *tsk = req->task; 1752 1753 struct io_ring_ctx *ctx = req->ctx; ··· 1760 1759 * will do the job. 1761 1760 */ 1762 1761 notify = 0; 1763 - if (!(ctx->flags & IORING_SETUP_SQPOLL)) 1762 + if (!(ctx->flags & IORING_SETUP_SQPOLL) && twa_signal_ok) 1764 1763 notify = TWA_SIGNAL; 1765 1764 1766 1765 ret = task_work_add(tsk, cb, notify); ··· 1820 1819 init_task_work(&req->task_work, io_req_task_submit); 1821 1820 percpu_ref_get(&req->ctx->refs); 1822 1821 1823 - ret = io_req_task_work_add(req, &req->task_work); 1822 + ret = io_req_task_work_add(req, &req->task_work, true); 1824 1823 if (unlikely(ret)) { 1825 1824 struct task_struct *tsk; 1826 1825 ··· 2049 2048 2050 2049 req = list_first_entry(done, struct io_kiocb, inflight_entry); 2051 2050 if (READ_ONCE(req->result) == -EAGAIN) { 2051 + req->result = 0; 2052 2052 req->iopoll_completed = 0; 2053 2053 list_move_tail(&req->inflight_entry, &again); 2054 2054 continue; ··· 2295 2293 io_req_complete(req, ret); 2296 2294 return false; 2297 2295 } 2298 - 2299 - static void io_rw_resubmit(struct callback_head *cb) 2300 - { 2301 - struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); 2302 - struct io_ring_ctx *ctx = req->ctx; 2303 - int err; 2304 - 2305 - err = io_sq_thread_acquire_mm(ctx, req); 2306 - 2307 - if (io_resubmit_prep(req, err)) { 2308 - refcount_inc(&req->refs); 2309 - io_queue_async_work(req); 2310 - } 2311 - 2312 - percpu_ref_put(&ctx->refs); 2313 - } 2314 2296 #endif 2315 2297 2316 2298 static bool io_rw_reissue(struct io_kiocb *req, long res) ··· 2305 2319 if ((res != -EAGAIN && res != -EOPNOTSUPP) || io_wq_current_is_worker()) 2306 2320 return false; 2307 2321 2308 - init_task_work(&req->task_work, io_rw_resubmit); 2309 - percpu_ref_get(&req->ctx->refs); 2322 + ret = io_sq_thread_acquire_mm(req->ctx, req); 2310 2323 2311 - ret = io_req_task_work_add(req, &req->task_work); 2312 - if (!ret) 2324 + if (io_resubmit_prep(req, ret)) { 2325 + refcount_inc(&req->refs); 2326 + io_queue_async_work(req); 2313 2327 return true; 2328 + } 2329 + 2314 2330 #endif 2315 2331 return false; 2316 2332 } ··· 2853 2865 return iov_iter_count(&req->io->rw.iter); 2854 2866 } 2855 2867 2868 + static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb) 2869 + { 2870 + return kiocb->ki_filp->f_mode & FMODE_STREAM ? NULL : &kiocb->ki_pos; 2871 + } 2872 + 2856 2873 /* 2857 2874 * For files that don't have ->read_iter() and ->write_iter(), handle them 2858 2875 * by looping over ->read() or ->write() manually. ··· 2893 2900 2894 2901 if (rw == READ) { 2895 2902 nr = file->f_op->read(file, iovec.iov_base, 2896 - iovec.iov_len, &kiocb->ki_pos); 2903 + iovec.iov_len, io_kiocb_ppos(kiocb)); 2897 2904 } else { 2898 2905 nr = file->f_op->write(file, iovec.iov_base, 2899 - iovec.iov_len, &kiocb->ki_pos); 2906 + iovec.iov_len, io_kiocb_ppos(kiocb)); 2900 2907 } 2901 2908 2902 2909 if (iov_iter_is_bvec(iter)) ··· 3037 3044 3038 3045 /* submit ref gets dropped, acquire a new one */ 3039 3046 refcount_inc(&req->refs); 3040 - ret = io_req_task_work_add(req, &req->task_work); 3047 + ret = io_req_task_work_add(req, &req->task_work, true); 3041 3048 if (unlikely(ret)) { 3042 3049 struct task_struct *tsk; 3043 3050 ··· 3118 3125 ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock); 3119 3126 if (ret < 0) 3120 3127 return ret; 3128 + iov_count = iov_iter_count(iter); 3121 3129 io_size = ret; 3122 3130 req->result = io_size; 3123 3131 ret = 0; ··· 3131 3137 if (force_nonblock && !io_file_supports_async(req->file, READ)) 3132 3138 goto copy_iov; 3133 3139 3134 - iov_count = iov_iter_count(iter); 3135 - ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count); 3140 + ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), iov_count); 3136 3141 if (unlikely(ret)) 3137 3142 goto out_free; 3138 3143 ··· 3143 3150 ret = 0; 3144 3151 goto out_free; 3145 3152 } else if (ret == -EAGAIN) { 3146 - if (!force_nonblock) 3153 + /* IOPOLL retry should happen for io-wq threads */ 3154 + if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) 3147 3155 goto done; 3156 + /* some cases will consume bytes even on error returns */ 3157 + iov_iter_revert(iter, iov_count - iov_iter_count(iter)); 3148 3158 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false); 3149 3159 if (ret) 3150 3160 goto out_free; 3151 3161 return -EAGAIN; 3152 3162 } else if (ret < 0) { 3153 - goto out_free; 3163 + /* make sure -ERESTARTSYS -> -EINTR is done */ 3164 + goto done; 3154 3165 } 3155 3166 3156 3167 /* read it all, or we did blocking attempt. no retry. */ ··· 3238 3241 ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock); 3239 3242 if (ret < 0) 3240 3243 return ret; 3244 + iov_count = iov_iter_count(iter); 3241 3245 io_size = ret; 3242 3246 req->result = io_size; 3243 3247 ··· 3255 3257 (req->flags & REQ_F_ISREG)) 3256 3258 goto copy_iov; 3257 3259 3258 - iov_count = iov_iter_count(iter); 3259 - ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count); 3260 + ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), iov_count); 3260 3261 if (unlikely(ret)) 3261 3262 goto out_free; 3262 3263 ··· 3288 3291 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) 3289 3292 ret2 = -EAGAIN; 3290 3293 if (!force_nonblock || ret2 != -EAGAIN) { 3294 + /* IOPOLL retry should happen for io-wq threads */ 3295 + if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN) 3296 + goto copy_iov; 3291 3297 kiocb_done(kiocb, ret2, cs); 3292 3298 } else { 3293 3299 copy_iov: 3300 + /* some cases will consume bytes even on error returns */ 3301 + iov_iter_revert(iter, iov_count - iov_iter_count(iter)); 3294 3302 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false); 3295 3303 if (!ret) 3296 3304 return -EAGAIN; ··· 4568 4566 static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, 4569 4567 __poll_t mask, task_work_func_t func) 4570 4568 { 4569 + bool twa_signal_ok; 4571 4570 int ret; 4572 4571 4573 4572 /* for instances that support it check for an event match first: */ ··· 4584 4581 percpu_ref_get(&req->ctx->refs); 4585 4582 4586 4583 /* 4584 + * If we using the signalfd wait_queue_head for this wakeup, then 4585 + * it's not safe to use TWA_SIGNAL as we could be recursing on the 4586 + * tsk->sighand->siglock on doing the wakeup. Should not be needed 4587 + * either, as the normal wakeup will suffice. 4588 + */ 4589 + twa_signal_ok = (poll->head != &req->task->sighand->signalfd_wqh); 4590 + 4591 + /* 4587 4592 * If this fails, then the task is exiting. When a task exits, the 4588 4593 * work gets canceled, so just cancel this request as well instead 4589 4594 * of executing it. We can't safely execute it anyway, as we may not 4590 4595 * have the needed state needed for it anyway. 4591 4596 */ 4592 - ret = io_req_task_work_add(req, &req->task_work); 4597 + ret = io_req_task_work_add(req, &req->task_work, twa_signal_ok); 4593 4598 if (unlikely(ret)) { 4594 4599 struct task_struct *tsk; 4595 4600 ··· 4886 4875 struct async_poll *apoll; 4887 4876 struct io_poll_table ipt; 4888 4877 __poll_t mask, ret; 4878 + int rw; 4889 4879 4890 4880 if (!req->file || !file_can_poll(req->file)) 4891 4881 return false; 4892 4882 if (req->flags & REQ_F_POLLED) 4893 4883 return false; 4894 - if (!def->pollin && !def->pollout) 4884 + if (def->pollin) 4885 + rw = READ; 4886 + else if (def->pollout) 4887 + rw = WRITE; 4888 + else 4889 + return false; 4890 + /* if we can't nonblock try, then no point in arming a poll handler */ 4891 + if (!io_file_supports_async(req->file, rw)) 4895 4892 return false; 4896 4893 4897 4894 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC); ··· 7452 7433 { 7453 7434 int ret; 7454 7435 7455 - mmgrab(current->mm); 7456 - ctx->sqo_mm = current->mm; 7457 - 7458 7436 if (ctx->flags & IORING_SETUP_SQPOLL) { 7459 7437 ret = -EPERM; 7460 7438 if (!capable(CAP_SYS_ADMIN)) ··· 7496 7480 return 0; 7497 7481 err: 7498 7482 io_finish_async(ctx); 7499 - if (ctx->sqo_mm) { 7500 - mmdrop(ctx->sqo_mm); 7501 - ctx->sqo_mm = NULL; 7502 - } 7503 7483 return ret; 7504 7484 } 7505 7485 ··· 8544 8532 ctx->compat = in_compat_syscall(); 8545 8533 ctx->user = user; 8546 8534 ctx->creds = get_current_cred(); 8535 + 8536 + mmgrab(current->mm); 8537 + ctx->sqo_mm = current->mm; 8547 8538 8548 8539 /* 8549 8540 * Account memory _before_ installing the file descriptor. Once

Configure Feed

Configure Feed