tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

io_uring: handle completions in the core

Normally request handlers complete requests themselves, if they don't
return an error. For the latter case, the core will complete it for
them.

This is unhandy for pushing opcode handlers further out, as we don't
want a bunch of inline completion code and we don't want to make the
completion path slower than it is now.

Let the core handle any completion, unless the handler explicitly
asks us not to.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

Jens Axboe 97b388d7 de23077e

+142 -134
+137 -134
io_uring/io_uring.c
··· 625 625 bool cancel_all); 626 626 static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd); 627 627 628 - static void __io_req_complete_post(struct io_kiocb *req, s32 res, u32 cflags); 629 628 static void io_dismantle_req(struct io_kiocb *req); 630 629 static void io_queue_linked_timeout(struct io_kiocb *req); 631 630 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, ··· 1125 1126 static inline void req_fail_link_node(struct io_kiocb *req, int res) 1126 1127 { 1127 1128 req_set_fail(req); 1128 - req->cqe.res = res; 1129 + io_req_set_res(req, res, 0); 1129 1130 } 1130 1131 1131 1132 static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx) ··· 1854 1855 } 1855 1856 } 1856 1857 1857 - static void __io_req_complete_post(struct io_kiocb *req, s32 res, 1858 - u32 cflags) 1858 + static void __io_req_complete_post(struct io_kiocb *req) 1859 1859 { 1860 - if (!(req->flags & REQ_F_CQE_SKIP)) { 1861 - req->cqe.res = res; 1862 - req->cqe.flags = cflags; 1860 + if (!(req->flags & REQ_F_CQE_SKIP)) 1863 1861 __io_fill_cqe_req(req->ctx, req); 1864 - } 1865 1862 __io_req_complete_put(req); 1866 1863 } 1867 1864 1868 - static void io_req_complete_post(struct io_kiocb *req, s32 res, u32 cflags) 1865 + static void io_req_complete_post(struct io_kiocb *req) 1869 1866 { 1870 1867 struct io_ring_ctx *ctx = req->ctx; 1871 1868 1872 1869 spin_lock(&ctx->completion_lock); 1873 - __io_req_complete_post(req, res, cflags); 1870 + __io_req_complete_post(req); 1874 1871 io_commit_cqring(ctx); 1875 1872 spin_unlock(&ctx->completion_lock); 1876 1873 io_cqring_ev_posted(ctx); 1877 1874 } 1878 1875 1879 - static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, 1880 - s32 res, u32 cflags) 1876 + static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags) 1881 1877 { 1882 - if (issue_flags & IO_URING_F_COMPLETE_DEFER) { 1883 - io_req_set_res(req, res, cflags); 1878 + if (issue_flags & IO_URING_F_COMPLETE_DEFER) 1884 1879 req->flags |= REQ_F_COMPLETE_INLINE; 1885 - } else { 1886 - io_req_complete_post(req, res, cflags); 1887 - } 1888 - } 1889 - 1890 - static inline void io_req_complete(struct io_kiocb *req, s32 res) 1891 - { 1892 - if (res < 0) 1893 - req_set_fail(req); 1894 - __io_req_complete(req, 0, res, 0); 1880 + else 1881 + io_req_complete_post(req); 1895 1882 } 1896 1883 1897 1884 static void io_req_complete_failed(struct io_kiocb *req, s32 res) 1898 1885 { 1899 1886 req_set_fail(req); 1900 - io_req_complete_post(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED)); 1887 + io_req_set_res(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED)); 1888 + io_req_complete_post(req); 1901 1889 } 1902 1890 1903 1891 /* ··· 2057 2071 link->flags |= REQ_F_CQE_SKIP; 2058 2072 else 2059 2073 link->flags &= ~REQ_F_CQE_SKIP; 2060 - __io_req_complete_post(link, res, 0); 2074 + io_req_set_res(link, res, 0); 2075 + __io_req_complete_post(link); 2061 2076 link = nxt; 2062 2077 } 2063 2078 } ··· 2172 2185 if (unlikely(!*uring_locked)) 2173 2186 spin_lock(&(*ctx)->completion_lock); 2174 2187 } 2175 - if (likely(*uring_locked)) 2188 + if (likely(*uring_locked)) { 2176 2189 req->io_task_work.func(req, uring_locked); 2177 - else 2178 - __io_req_complete_post(req, req->cqe.res, 2179 - io_put_kbuf_comp(req)); 2190 + } else { 2191 + req->cqe.flags = io_put_kbuf_comp(req); 2192 + __io_req_complete_post(req); 2193 + } 2180 2194 node = next; 2181 2195 } while (node); 2182 2196 ··· 2305 2317 2306 2318 static void io_req_tw_post(struct io_kiocb *req, bool *locked) 2307 2319 { 2308 - io_req_complete_post(req, req->cqe.res, req->cqe.flags); 2320 + io_req_complete_post(req); 2309 2321 } 2310 2322 2311 2323 static void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags) 2312 2324 { 2313 - req->cqe.res = res; 2314 - req->cqe.flags = cflags; 2325 + io_req_set_res(req, res, cflags); 2315 2326 req->io_task_work.func = io_req_tw_post; 2316 2327 io_req_task_work_add(req); 2317 2328 } ··· 2334 2347 2335 2348 static void io_req_task_queue_fail(struct io_kiocb *req, int ret) 2336 2349 { 2337 - req->cqe.res = ret; 2350 + io_req_set_res(req, ret, 0); 2338 2351 req->io_task_work.func = io_req_task_cancel; 2339 2352 io_req_task_work_add(req); 2340 2353 } ··· 2728 2741 2729 2742 static inline void io_req_task_complete(struct io_kiocb *req, bool *locked) 2730 2743 { 2731 - int res = req->cqe.res; 2732 - 2733 2744 if (*locked) { 2734 - io_req_set_res(req, res, io_put_kbuf(req, 0)); 2745 + req->cqe.flags |= io_put_kbuf(req, 0); 2735 2746 req->flags |= REQ_F_COMPLETE_INLINE; 2736 2747 io_req_add_compl_list(req); 2737 2748 } else { 2738 - io_req_complete_post(req, res, 2739 - io_put_kbuf(req, IO_URING_F_UNLOCKED)); 2749 + req->cqe.flags |= io_put_kbuf(req, IO_URING_F_UNLOCKED); 2750 + io_req_complete_post(req); 2740 2751 } 2741 2752 } 2742 2753 ··· 2743 2758 { 2744 2759 if (__io_complete_rw_common(req, res)) 2745 2760 return; 2746 - __io_req_complete(req, issue_flags, req->cqe.res, 2747 - io_put_kbuf(req, issue_flags)); 2761 + io_req_set_res(req, req->cqe.res, io_put_kbuf(req, issue_flags)); 2762 + __io_req_complete(req, issue_flags); 2748 2763 } 2749 2764 2750 2765 static void io_complete_rw(struct kiocb *kiocb, long res) ··· 2754 2769 2755 2770 if (__io_complete_rw_common(req, res)) 2756 2771 return; 2757 - req->cqe.res = res; 2772 + io_req_set_res(req, res, 0); 2758 2773 req->io_task_work.func = io_req_task_complete; 2759 2774 io_req_task_prio_work_add(req); 2760 2775 } ··· 3730 3745 */ 3731 3746 ret = io_iter_do_read(rw, &s->iter); 3732 3747 if (ret == -EIOCBQUEUED) 3733 - return 0; 3748 + return IOU_ISSUE_SKIP_COMPLETE; 3734 3749 /* we got some bytes, but not all. retry. */ 3735 3750 kiocb->ki_flags &= ~IOCB_WAITQ; 3736 3751 iov_iter_restore(&s->iter, &s->iter_state); ··· 3741 3756 /* it's faster to check here then delegate to kfree */ 3742 3757 if (iovec) 3743 3758 kfree(iovec); 3744 - return 0; 3759 + return IOU_ISSUE_SKIP_COMPLETE; 3745 3760 } 3746 3761 3747 3762 static int io_write(struct io_kiocb *req, unsigned int issue_flags) ··· 3835 3850 goto copy_iov; 3836 3851 done: 3837 3852 kiocb_done(req, ret2, issue_flags); 3853 + ret = IOU_ISSUE_SKIP_COMPLETE; 3838 3854 } else { 3839 3855 copy_iov: 3840 3856 iov_iter_restore(&s->iter, &s->iter_state); ··· 3892 3906 ren->newpath, ren->flags); 3893 3907 3894 3908 req->flags &= ~REQ_F_NEED_CLEANUP; 3895 - io_req_complete(req, ret); 3896 - return 0; 3909 + io_req_set_res(req, ret, 0); 3910 + return IOU_OK; 3897 3911 } 3898 3912 3899 3913 static void io_renameat_cleanup(struct io_kiocb *req) ··· 3920 3934 req->flags &= ~REQ_F_NEED_CLEANUP; 3921 3935 3922 3936 io_xattr_cleanup(req); 3923 - io_req_complete(req, ret); 3937 + io_req_set_res(req, ret, 0); 3924 3938 } 3925 3939 3926 3940 static int __io_getxattr_prep(struct io_kiocb *req, ··· 4001 4015 &ix->ctx); 4002 4016 4003 4017 io_xattr_finish(req, ret); 4004 - return 0; 4018 + return IOU_OK; 4005 4019 } 4006 4020 4007 4021 static int io_getxattr(struct io_kiocb *req, unsigned int issue_flags) ··· 4029 4043 } 4030 4044 4031 4045 io_xattr_finish(req, ret); 4032 - return 0; 4046 + return IOU_OK; 4033 4047 } 4034 4048 4035 4049 static int __io_setxattr_prep(struct io_kiocb *req, ··· 4115 4129 4116 4130 ret = __io_setxattr(req, issue_flags, &req->file->f_path); 4117 4131 io_xattr_finish(req, ret); 4118 - 4119 - return 0; 4132 + return IOU_OK; 4120 4133 } 4121 4134 4122 4135 static int io_setxattr(struct io_kiocb *req, unsigned int issue_flags) ··· 4140 4155 } 4141 4156 4142 4157 io_xattr_finish(req, ret); 4143 - return 0; 4158 + return IOU_OK; 4144 4159 } 4145 4160 4146 4161 static int io_unlinkat_prep(struct io_kiocb *req, ··· 4183 4198 ret = do_unlinkat(un->dfd, un->filename); 4184 4199 4185 4200 req->flags &= ~REQ_F_NEED_CLEANUP; 4186 - io_req_complete(req, ret); 4187 - return 0; 4201 + io_req_set_res(req, ret, 0); 4202 + return IOU_OK; 4188 4203 } 4189 4204 4190 4205 static void io_unlinkat_cleanup(struct io_kiocb *req) ··· 4228 4243 ret = do_mkdirat(mkd->dfd, mkd->filename, mkd->mode); 4229 4244 4230 4245 req->flags &= ~REQ_F_NEED_CLEANUP; 4231 - io_req_complete(req, ret); 4232 - return 0; 4246 + io_req_set_res(req, ret, 0); 4247 + return IOU_OK; 4233 4248 } 4234 4249 4235 4250 static void io_mkdirat_cleanup(struct io_kiocb *req) ··· 4279 4294 ret = do_symlinkat(sl->oldpath, sl->new_dfd, sl->newpath); 4280 4295 4281 4296 req->flags &= ~REQ_F_NEED_CLEANUP; 4282 - io_req_complete(req, ret); 4283 - return 0; 4297 + io_req_set_res(req, ret, 0); 4298 + return IOU_OK; 4284 4299 } 4285 4300 4286 4301 static int io_linkat_prep(struct io_kiocb *req, ··· 4326 4341 lnk->newpath, lnk->flags); 4327 4342 4328 4343 req->flags &= ~REQ_F_NEED_CLEANUP; 4329 - io_req_complete(req, ret); 4330 - return 0; 4344 + io_req_set_res(req, ret, 0); 4345 + return IOU_OK; 4331 4346 } 4332 4347 4333 4348 static void io_link_cleanup(struct io_kiocb *req) ··· 4378 4393 io_req_set_res(req, 0, ret); 4379 4394 if (req->ctx->flags & IORING_SETUP_CQE32) 4380 4395 io_req_set_cqe32_extra(req, res2, 0); 4381 - io_req_complete(req, ret); 4396 + __io_req_complete(req, 0); 4382 4397 } 4383 4398 EXPORT_SYMBOL_GPL(io_uring_cmd_done); 4384 4399 ··· 4435 4450 return -EAGAIN; 4436 4451 } 4437 4452 4438 - if (ret != -EIOCBQUEUED) 4453 + if (ret != -EIOCBQUEUED) { 4439 4454 io_uring_cmd_done(ioucmd, ret, 0); 4440 - return 0; 4455 + return IOU_OK; 4456 + } 4457 + 4458 + return IOU_ISSUE_SKIP_COMPLETE; 4441 4459 } 4442 4460 4443 4461 static int __io_splice_prep(struct io_kiocb *req, ··· 4493 4505 done: 4494 4506 if (ret != sp->len) 4495 4507 req_set_fail(req); 4496 - __io_req_complete(req, 0, ret, 0); 4497 - return 0; 4508 + io_req_set_res(req, ret, 0); 4509 + return IOU_OK; 4498 4510 } 4499 4511 4500 4512 static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ··· 4538 4550 done: 4539 4551 if (ret != sp->len) 4540 4552 req_set_fail(req); 4541 - __io_req_complete(req, 0, ret, 0); 4542 - return 0; 4553 + io_req_set_res(req, ret, 0); 4554 + return IOU_OK; 4543 4555 } 4544 4556 4545 4557 static int io_nop_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ··· 4552 4564 */ 4553 4565 static int io_nop(struct io_kiocb *req, unsigned int issue_flags) 4554 4566 { 4555 - __io_req_complete(req, issue_flags, 0, 0); 4556 - return 0; 4567 + io_req_set_res(req, 0, 0); 4568 + return IOU_OK; 4557 4569 } 4558 4570 4559 4571 static int io_msg_ring_prep(struct io_kiocb *req, ··· 4597 4609 done: 4598 4610 if (ret < 0) 4599 4611 req_set_fail(req); 4600 - __io_req_complete(req, issue_flags, ret, 0); 4612 + io_req_set_res(req, ret, 0); 4601 4613 /* put file to avoid an attempt to IOPOLL the req */ 4602 4614 io_put_file(req->file); 4603 4615 req->file = NULL; 4604 - return 0; 4616 + return IOU_OK; 4605 4617 } 4606 4618 4607 4619 static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ··· 4632 4644 4633 4645 ret = vfs_fsync_range(req->file, sync->off, end > 0 ? end : LLONG_MAX, 4634 4646 sync->flags & IORING_FSYNC_DATASYNC); 4635 - io_req_complete(req, ret); 4636 - return 0; 4647 + io_req_set_res(req, ret, 0); 4648 + return IOU_OK; 4637 4649 } 4638 4650 4639 4651 static int io_fallocate_prep(struct io_kiocb *req, ··· 4661 4673 ret = vfs_fallocate(req->file, sync->mode, sync->off, sync->len); 4662 4674 if (ret >= 0) 4663 4675 fsnotify_modify(req->file); 4664 - io_req_complete(req, ret); 4665 - return 0; 4676 + io_req_set_res(req, ret, 0); 4677 + return IOU_OK; 4666 4678 } 4667 4679 4668 4680 static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ··· 4843 4855 req->flags &= ~REQ_F_NEED_CLEANUP; 4844 4856 if (ret < 0) 4845 4857 req_set_fail(req); 4846 - __io_req_complete(req, issue_flags, ret, 0); 4847 - return 0; 4858 + io_req_set_res(req, ret, 0); 4859 + return IOU_OK; 4848 4860 } 4849 4861 4850 4862 static int io_openat(struct io_kiocb *req, unsigned int issue_flags) ··· 4939 4951 req_set_fail(req); 4940 4952 4941 4953 /* complete before unlock, IOPOLL may need the lock */ 4942 - __io_req_complete(req, issue_flags, ret, 0); 4954 + io_req_set_res(req, ret, 0); 4955 + __io_req_complete(req, issue_flags); 4943 4956 io_ring_submit_unlock(ctx, issue_flags); 4944 - return 0; 4957 + return IOU_ISSUE_SKIP_COMPLETE; 4945 4958 } 4946 4959 4947 4960 static int io_provide_buffers_prep(struct io_kiocb *req, ··· 5106 5117 if (ret < 0) 5107 5118 req_set_fail(req); 5108 5119 /* complete before unlock, IOPOLL may need the lock */ 5109 - __io_req_complete(req, issue_flags, ret, 0); 5120 + io_req_set_res(req, ret, 0); 5121 + __io_req_complete(req, issue_flags); 5110 5122 io_ring_submit_unlock(ctx, issue_flags); 5111 - return 0; 5123 + return IOU_ISSUE_SKIP_COMPLETE; 5112 5124 } 5113 5125 5114 5126 static int io_epoll_ctl_prep(struct io_kiocb *req, ··· 5152 5162 5153 5163 if (ret < 0) 5154 5164 req_set_fail(req); 5155 - __io_req_complete(req, issue_flags, ret, 0); 5156 - return 0; 5165 + io_req_set_res(req, ret, 0); 5166 + return IOU_OK; 5157 5167 #else 5158 5168 return -EOPNOTSUPP; 5159 5169 #endif ··· 5186 5196 return -EAGAIN; 5187 5197 5188 5198 ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice); 5189 - io_req_complete(req, ret); 5190 - return 0; 5199 + io_req_set_res(req, ret, 0); 5200 + return IOU_OK; 5191 5201 #else 5192 5202 return -EOPNOTSUPP; 5193 5203 #endif ··· 5225 5235 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice); 5226 5236 if (ret < 0) 5227 5237 req_set_fail(req); 5228 - __io_req_complete(req, issue_flags, ret, 0); 5229 - return 0; 5238 + io_req_set_res(req, ret, 0); 5239 + return IOU_OK; 5230 5240 } 5231 5241 5232 5242 static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ··· 5269 5279 return -EAGAIN; 5270 5280 5271 5281 ret = do_statx(sx->dfd, sx->filename, sx->flags, sx->mask, sx->buffer); 5272 - io_req_complete(req, ret); 5273 - return 0; 5282 + io_req_set_res(req, ret, 0); 5283 + return IOU_OK; 5274 5284 } 5275 5285 5276 5286 static void io_statx_cleanup(struct io_kiocb *req) ··· 5340 5350 err: 5341 5351 if (ret < 0) 5342 5352 req_set_fail(req); 5343 - __io_req_complete(req, issue_flags, ret, 0); 5344 - return 0; 5353 + io_req_set_res(req, ret, 0); 5354 + return IOU_OK; 5345 5355 } 5346 5356 5347 5357 static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ··· 5367 5377 return -EAGAIN; 5368 5378 5369 5379 ret = sync_file_range(req->file, sync->off, sync->len, sync->flags); 5370 - io_req_complete(req, ret); 5371 - return 0; 5380 + io_req_set_res(req, ret, 0); 5381 + return IOU_OK; 5372 5382 } 5373 5383 5374 5384 #if defined(CONFIG_NET) ··· 5399 5409 return -ENOTSOCK; 5400 5410 5401 5411 ret = __sys_shutdown_sock(sock, shutdown->how); 5402 - io_req_complete(req, ret); 5403 - return 0; 5412 + io_req_set_res(req, ret, 0); 5413 + return IOU_OK; 5404 5414 } 5405 5415 5406 5416 static bool io_net_retry(struct socket *sock, int flags) ··· 5538 5548 ret += sr->done_io; 5539 5549 else if (sr->done_io) 5540 5550 ret = sr->done_io; 5541 - __io_req_complete(req, issue_flags, ret, 0); 5542 - return 0; 5551 + io_req_set_res(req, ret, 0); 5552 + return IOU_OK; 5543 5553 } 5544 5554 5545 5555 static int io_send(struct io_kiocb *req, unsigned int issue_flags) ··· 5595 5605 ret += sr->done_io; 5596 5606 else if (sr->done_io) 5597 5607 ret = sr->done_io; 5598 - __io_req_complete(req, issue_flags, ret, 0); 5599 - return 0; 5608 + io_req_set_res(req, ret, 0); 5609 + return IOU_OK; 5600 5610 } 5601 5611 5602 5612 static int __io_recvmsg_copy_hdr(struct io_kiocb *req, ··· 5795 5805 cflags = io_put_kbuf(req, issue_flags); 5796 5806 if (kmsg->msg.msg_inq) 5797 5807 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 5798 - __io_req_complete(req, issue_flags, ret, cflags); 5799 - return 0; 5808 + io_req_set_res(req, ret, cflags); 5809 + return IOU_OK; 5800 5810 } 5801 5811 5802 5812 static int io_recv(struct io_kiocb *req, unsigned int issue_flags) ··· 5871 5881 cflags = io_put_kbuf(req, issue_flags); 5872 5882 if (msg.msg_inq) 5873 5883 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 5874 - __io_req_complete(req, issue_flags, ret, cflags); 5875 - return 0; 5884 + io_req_set_res(req, ret, cflags); 5885 + return IOU_OK; 5876 5886 } 5877 5887 5878 5888 static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ··· 5938 5948 */ 5939 5949 if ((req->flags & IO_APOLL_MULTI_POLLED) == 5940 5950 IO_APOLL_MULTI_POLLED) 5941 - ret = 0; 5951 + ret = IOU_ISSUE_SKIP_COMPLETE; 5942 5952 return ret; 5943 5953 } 5944 5954 if (ret == -ERESTARTSYS) ··· 5953 5963 } 5954 5964 5955 5965 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 5956 - __io_req_complete(req, issue_flags, ret, 0); 5957 - return 0; 5966 + io_req_set_res(req, ret, 0); 5967 + return IOU_OK; 5958 5968 } 5959 5969 if (ret >= 0) { 5960 5970 bool filled; ··· 6024 6034 ret = io_fixed_fd_install(req, issue_flags, file, 6025 6035 sock->file_slot); 6026 6036 } 6027 - __io_req_complete(req, issue_flags, ret, 0); 6028 - return 0; 6037 + io_req_set_res(req, ret, 0); 6038 + return IOU_OK; 6029 6039 } 6030 6040 6031 6041 static int io_connect_prep_async(struct io_kiocb *req) ··· 6086 6096 out: 6087 6097 if (ret < 0) 6088 6098 req_set_fail(req); 6089 - __io_req_complete(req, issue_flags, ret, 0); 6090 - return 0; 6099 + io_req_set_res(req, ret, 0); 6100 + return IOU_OK; 6091 6101 } 6092 6102 #else /* !CONFIG_NET */ 6093 6103 #define IO_NETOP_FN(op) \ ··· 6318 6328 io_poll_remove_entries(req); 6319 6329 spin_lock(&ctx->completion_lock); 6320 6330 hash_del(&req->hash_node); 6321 - __io_req_complete_post(req, req->cqe.res, 0); 6331 + req->cqe.flags = 0; 6332 + __io_req_complete_post(req); 6322 6333 io_commit_cqring(ctx); 6323 6334 spin_unlock(&ctx->completion_lock); 6324 6335 io_cqring_ev_posted(ctx); ··· 6348 6357 static void __io_poll_execute(struct io_kiocb *req, int mask, 6349 6358 __poll_t __maybe_unused events) 6350 6359 { 6351 - req->cqe.res = mask; 6360 + io_req_set_res(req, mask, 0); 6352 6361 /* 6353 6362 * This is useful for poll that is armed on behalf of another 6354 6363 * request, and where the wakeup path could be on a different ··· 6801 6810 ipt.pt._qproc = io_poll_queue_proc; 6802 6811 6803 6812 ret = __io_arm_poll_handler(req, poll, &ipt, poll->events); 6804 - if (!ret && ipt.error) 6813 + if (ret) { 6814 + io_req_set_res(req, ret, 0); 6815 + return IOU_OK; 6816 + } 6817 + if (ipt.error) { 6805 6818 req_set_fail(req); 6806 - ret = ret ?: ipt.error; 6807 - if (ret) 6808 - __io_req_complete(req, issue_flags, ret, 0); 6809 - return 0; 6819 + return ipt.error; 6820 + } 6821 + 6822 + return IOU_ISSUE_SKIP_COMPLETE; 6810 6823 } 6811 6824 6812 6825 static int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) ··· 6845 6850 6846 6851 ret2 = io_poll_add(preq, issue_flags); 6847 6852 /* successfully updated, don't complete poll request */ 6848 - if (!ret2) 6853 + if (!ret2 || ret2 == -EIOCBQUEUED) 6849 6854 goto out; 6850 6855 } 6851 6856 6852 6857 req_set_fail(preq); 6853 - preq->cqe.res = -ECANCELED; 6858 + io_req_set_res(preq, -ECANCELED, 0); 6854 6859 locked = !(issue_flags & IO_URING_F_UNLOCKED); 6855 6860 io_req_task_complete(preq, &locked); 6856 6861 out: 6857 - if (ret < 0) 6862 + if (ret < 0) { 6858 6863 req_set_fail(req); 6864 + return ret; 6865 + } 6859 6866 /* complete update request, we're done with it */ 6860 - __io_req_complete(req, issue_flags, ret, 0); 6861 - return 0; 6867 + io_req_set_res(req, ret, 0); 6868 + return IOU_OK; 6862 6869 } 6863 6870 6864 6871 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) ··· 6881 6884 if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS)) 6882 6885 req_set_fail(req); 6883 6886 6884 - req->cqe.res = -ETIME; 6887 + io_req_set_res(req, -ETIME, 0); 6885 6888 req->io_task_work.func = io_req_task_complete; 6886 6889 io_req_task_work_add(req); 6887 6890 return HRTIMER_NORESTART; ··· 7066 7069 7067 7070 if (ret < 0) 7068 7071 req_set_fail(req); 7069 - io_req_complete_post(req, ret, 0); 7070 - return 0; 7072 + io_req_set_res(req, ret, 0); 7073 + return IOU_OK; 7071 7074 } 7072 7075 7073 7076 static int __io_timeout_prep(struct io_kiocb *req, ··· 7188 7191 data->timer.function = io_timeout_fn; 7189 7192 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); 7190 7193 spin_unlock_irq(&ctx->timeout_lock); 7191 - return 0; 7194 + return IOU_ISSUE_SKIP_COMPLETE; 7192 7195 } 7193 7196 7194 7197 static bool io_cancel_cb(struct io_wq_work *work, void *data) ··· 7356 7359 done: 7357 7360 if (ret < 0) 7358 7361 req_set_fail(req); 7359 - io_req_complete_post(req, ret, 0); 7360 - return 0; 7362 + io_req_set_res(req, ret, 0); 7363 + return IOU_OK; 7361 7364 } 7362 7365 7363 7366 static int io_files_update_prep(struct io_kiocb *req, ··· 7442 7445 7443 7446 if (ret < 0) 7444 7447 req_set_fail(req); 7445 - __io_req_complete(req, issue_flags, ret, 0); 7446 - return 0; 7448 + io_req_set_res(req, ret, 0); 7449 + return IOU_OK; 7447 7450 } 7448 7451 7449 7452 static int io_req_prep_async(struct io_kiocb *req) ··· 7587 7590 7588 7591 if (creds) 7589 7592 revert_creds(creds); 7590 - if (ret) 7593 + 7594 + if (ret == IOU_OK) 7595 + __io_req_complete(req, issue_flags); 7596 + else if (ret != IOU_ISSUE_SKIP_COMPLETE) 7591 7597 return ret; 7598 + 7592 7599 /* If the op doesn't have a file, we're not polling for it */ 7593 7600 if ((req->ctx->flags & IORING_SETUP_IOPOLL) && req->file) 7594 7601 io_iopoll_req_issued(req, issue_flags); ··· 7669 7668 } while (1); 7670 7669 7671 7670 /* avoid locking problems by failing it from a clean context */ 7672 - if (ret) 7671 + if (ret < 0) 7673 7672 io_req_task_queue_fail(req, ret); 7674 7673 } 7675 7674 ··· 7746 7745 7747 7746 ret = io_try_cancel(req, &cd); 7748 7747 } 7749 - io_req_complete_post(req, ret ?: -ETIME, 0); 7748 + io_req_set_res(req, ret ?: -ETIME, 0); 7749 + io_req_complete_post(req); 7750 7750 io_put_req(prev); 7751 7751 } else { 7752 - io_req_complete_post(req, -ETIME, 0); 7752 + io_req_set_res(req, -ETIME, 0); 7753 + io_req_complete_post(req); 7753 7754 } 7754 7755 } 7755 7756
+5
io_uring/io_uring.h
··· 4 4 #include <linux/errno.h> 5 5 #include "io_uring_types.h" 6 6 7 + enum { 8 + IOU_OK = 0, 9 + IOU_ISSUE_SKIP_COMPLETE = -EIOCBQUEUED, 10 + }; 11 + 7 12 static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags) 8 13 { 9 14 req->cqe.res = res;