Merge tag 'io_uring-5.18-2022-04-08' of git://git.kernel.dk/linux-block

+198 -420

2 changed files

expand all

io-wq.h

io_uring.c

fs/io-wq.h

··· 155 155 struct io_wq_work { 156 156 struct io_wq_work_node list; 157 157 unsigned flags; 158 + int fd; 158 159 }; 159 160 160 161 static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)

+197 -420

fs/io_uring.c

··· 63 63 #include <net/sock.h> 64 64 #include <net/af_unix.h> 65 65 #include <net/scm.h> 66 - #include <net/busy_poll.h> 67 66 #include <linux/anon_inodes.h> 68 67 #include <linux/sched/mm.h> 69 68 #include <linux/uaccess.h> ··· 111 112 IOSQE_IO_DRAIN | IOSQE_CQE_SKIP_SUCCESS) 112 113 113 114 #define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \ 114 - REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS | \ 115 - REQ_F_ASYNC_DATA) 115 + REQ_F_POLLED | REQ_F_CREDS | REQ_F_ASYNC_DATA) 116 116 117 117 #define IO_TCTX_REFS_CACHE_NR (1U << 10) 118 118 ··· 410 412 struct list_head sqd_list; 411 413 412 414 unsigned long check_cq_overflow; 413 - #ifdef CONFIG_NET_RX_BUSY_POLL 414 - /* used to track busy poll napi_id */ 415 - struct list_head napi_list; 416 - spinlock_t napi_lock; /* napi_list lock */ 417 - #endif 418 415 419 416 struct { 420 417 unsigned cached_cq_tail; ··· 493 500 const struct io_ring_ctx *last; 494 501 struct io_wq *io_wq; 495 502 struct percpu_counter inflight; 496 - atomic_t inflight_tracked; 497 503 atomic_t in_idle; 498 504 499 505 spinlock_t task_lock; ··· 584 592 /* NOTE: kiocb has the file as the first member, so don't do it here */ 585 593 struct kiocb kiocb; 586 594 u64 addr; 587 - u64 len; 595 + u32 len; 596 + u32 flags; 588 597 }; 589 598 590 599 struct io_connect { ··· 647 654 648 655 struct io_splice { 649 656 struct file *file_out; 650 - struct file *file_in; 651 657 loff_t off_out; 652 658 loff_t off_in; 653 659 u64 len; 660 + int splice_fd_in; 654 661 unsigned int flags; 655 662 }; 656 663 ··· 1175 1182 struct io_uring_rsrc_update2 *up, 1176 1183 unsigned nr_args); 1177 1184 static void io_clean_op(struct io_kiocb *req); 1178 - static struct file *io_file_get(struct io_ring_ctx *ctx, 1179 - struct io_kiocb *req, int fd, bool fixed); 1185 + static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 1186 + unsigned issue_flags); 1187 + static inline struct file *io_file_get_normal(struct io_kiocb *req, int fd); 1188 + static void io_drop_inflight_file(struct io_kiocb *req); 1189 + static bool io_assign_file(struct io_kiocb *req, unsigned int issue_flags); 1180 1190 static void __io_queue_sqe(struct io_kiocb *req); 1181 1191 static void io_rsrc_put_work(struct work_struct *work); 1182 1192 ··· 1309 1313 } 1310 1314 1311 1315 static inline void io_req_set_rsrc_node(struct io_kiocb *req, 1312 - struct io_ring_ctx *ctx) 1316 + struct io_ring_ctx *ctx, 1317 + unsigned int issue_flags) 1313 1318 { 1314 1319 if (!req->fixed_rsrc_refs) { 1315 1320 req->fixed_rsrc_refs = &ctx->rsrc_node->refs; 1316 - ctx->rsrc_cached_refs--; 1317 - if (unlikely(ctx->rsrc_cached_refs < 0)) 1318 - io_rsrc_refs_refill(ctx); 1321 + 1322 + if (!(issue_flags & IO_URING_F_UNLOCKED)) { 1323 + lockdep_assert_held(&ctx->uring_lock); 1324 + ctx->rsrc_cached_refs--; 1325 + if (unlikely(ctx->rsrc_cached_refs < 0)) 1326 + io_rsrc_refs_refill(ctx); 1327 + } else { 1328 + percpu_ref_get(req->fixed_rsrc_refs); 1329 + } 1319 1330 } 1320 1331 } 1321 1332 ··· 1427 1424 bool cancel_all) 1428 1425 __must_hold(&req->ctx->timeout_lock) 1429 1426 { 1430 - struct io_kiocb *req; 1431 - 1432 1427 if (task && head->task != task) 1433 1428 return false; 1434 - if (cancel_all) 1435 - return true; 1436 - 1437 - io_for_each_link(req, head) { 1438 - if (req->flags & REQ_F_INFLIGHT) 1439 - return true; 1440 - } 1441 - return false; 1442 - } 1443 - 1444 - static bool io_match_linked(struct io_kiocb *head) 1445 - { 1446 - struct io_kiocb *req; 1447 - 1448 - io_for_each_link(req, head) { 1449 - if (req->flags & REQ_F_INFLIGHT) 1450 - return true; 1451 - } 1452 - return false; 1429 + return cancel_all; 1453 1430 } 1454 1431 1455 1432 /* ··· 1439 1456 static bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task, 1440 1457 bool cancel_all) 1441 1458 { 1442 - bool matched; 1443 - 1444 1459 if (task && head->task != task) 1445 1460 return false; 1446 - if (cancel_all) 1447 - return true; 1448 - 1449 - if (head->flags & REQ_F_LINK_TIMEOUT) { 1450 - struct io_ring_ctx *ctx = head->ctx; 1451 - 1452 - /* protect against races with linked timeouts */ 1453 - spin_lock_irq(&ctx->timeout_lock); 1454 - matched = io_match_linked(head); 1455 - spin_unlock_irq(&ctx->timeout_lock); 1456 - } else { 1457 - matched = io_match_linked(head); 1458 - } 1459 - return matched; 1461 + return cancel_all; 1460 1462 } 1461 1463 1462 1464 static inline bool req_has_async_data(struct io_kiocb *req) ··· 1563 1595 INIT_WQ_LIST(&ctx->locked_free_list); 1564 1596 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func); 1565 1597 INIT_WQ_LIST(&ctx->submit_state.compl_reqs); 1566 - #ifdef CONFIG_NET_RX_BUSY_POLL 1567 - INIT_LIST_HEAD(&ctx->napi_list); 1568 - spin_lock_init(&ctx->napi_lock); 1569 - #endif 1570 1598 return ctx; 1571 1599 err: 1572 1600 kfree(ctx->dummy_ubuf); ··· 1598 1634 static inline bool io_req_ffs_set(struct io_kiocb *req) 1599 1635 { 1600 1636 return req->flags & REQ_F_FIXED_FILE; 1601 - } 1602 - 1603 - static inline void io_req_track_inflight(struct io_kiocb *req) 1604 - { 1605 - if (!(req->flags & REQ_F_INFLIGHT)) { 1606 - req->flags |= REQ_F_INFLIGHT; 1607 - atomic_inc(&current->io_uring->inflight_tracked); 1608 - } 1609 1637 } 1610 1638 1611 1639 static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req) ··· 1642 1686 } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) { 1643 1687 if (def->unbound_nonreg_file) 1644 1688 req->work.flags |= IO_WQ_WORK_UNBOUND; 1645 - } 1646 - 1647 - switch (req->opcode) { 1648 - case IORING_OP_SPLICE: 1649 - case IORING_OP_TEE: 1650 - if (!S_ISREG(file_inode(req->splice.file_in)->i_mode)) 1651 - req->work.flags |= IO_WQ_WORK_UNBOUND; 1652 - break; 1653 1689 } 1654 1690 } 1655 1691 ··· 1736 1788 __must_hold(&ctx->completion_lock) 1737 1789 { 1738 1790 u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); 1791 + struct io_kiocb *req, *tmp; 1739 1792 1740 1793 spin_lock_irq(&ctx->timeout_lock); 1741 - while (!list_empty(&ctx->timeout_list)) { 1794 + list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { 1742 1795 u32 events_needed, events_got; 1743 - struct io_kiocb *req = list_first_entry(&ctx->timeout_list, 1744 - struct io_kiocb, timeout.list); 1745 1796 1746 1797 if (io_is_timeout_noseq(req)) 1747 1798 break; ··· 1757 1810 if (events_got < events_needed) 1758 1811 break; 1759 1812 1760 - list_del_init(&req->timeout.list); 1761 1813 io_kill_timeout(req, 0); 1762 1814 } 1763 1815 ctx->cq_last_tm_flush = seq; ··· 2508 2562 2509 2563 WARN_ON_ONCE(!tctx); 2510 2564 2565 + io_drop_inflight_file(req); 2566 + 2511 2567 spin_lock_irqsave(&tctx->task_lock, flags); 2512 2568 if (priority) 2513 2569 wq_list_add_tail(&req->io_task_work.node, &tctx->prior_task_list); ··· 3134 3186 3135 3187 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) 3136 3188 { 3137 - struct io_ring_ctx *ctx = req->ctx; 3138 3189 struct kiocb *kiocb = &req->rw.kiocb; 3139 - struct file *file = req->file; 3140 3190 unsigned ioprio; 3141 3191 int ret; 3142 3192 3143 - if (!io_req_ffs_set(req)) 3144 - req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT; 3145 - 3146 3193 kiocb->ki_pos = READ_ONCE(sqe->off); 3147 - kiocb->ki_flags = iocb_flags(file); 3148 - ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags)); 3149 - if (unlikely(ret)) 3150 - return ret; 3151 - 3152 - /* 3153 - * If the file is marked O_NONBLOCK, still allow retry for it if it 3154 - * supports async. Otherwise it's impossible to use O_NONBLOCK files 3155 - * reliably. If not, or it IOCB_NOWAIT is set, don't retry. 3156 - */ 3157 - if ((kiocb->ki_flags & IOCB_NOWAIT) || 3158 - ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req))) 3159 - req->flags |= REQ_F_NOWAIT; 3160 - 3161 - if (ctx->flags & IORING_SETUP_IOPOLL) { 3162 - if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll) 3163 - return -EOPNOTSUPP; 3164 - 3165 - kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE; 3166 - kiocb->ki_complete = io_complete_rw_iopoll; 3167 - req->iopoll_completed = 0; 3168 - } else { 3169 - if (kiocb->ki_flags & IOCB_HIPRI) 3170 - return -EINVAL; 3171 - kiocb->ki_complete = io_complete_rw; 3172 - } 3173 3194 3174 3195 ioprio = READ_ONCE(sqe->ioprio); 3175 3196 if (ioprio) { ··· 3154 3237 req->imu = NULL; 3155 3238 req->rw.addr = READ_ONCE(sqe->addr); 3156 3239 req->rw.len = READ_ONCE(sqe->len); 3240 + req->rw.flags = READ_ONCE(sqe->rw_flags); 3157 3241 req->buf_index = READ_ONCE(sqe->buf_index); 3158 3242 return 0; 3159 3243 } ··· 3285 3367 return 0; 3286 3368 } 3287 3369 3288 - static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter) 3370 + static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter, 3371 + unsigned int issue_flags) 3289 3372 { 3290 3373 struct io_mapped_ubuf *imu = req->imu; 3291 3374 u16 index, buf_index = req->buf_index; ··· 3296 3377 3297 3378 if (unlikely(buf_index >= ctx->nr_user_bufs)) 3298 3379 return -EFAULT; 3299 - io_req_set_rsrc_node(req, ctx); 3380 + io_req_set_rsrc_node(req, ctx, issue_flags); 3300 3381 index = array_index_nospec(buf_index, ctx->nr_user_bufs); 3301 3382 imu = READ_ONCE(ctx->user_bufs[index]); 3302 3383 req->imu = imu; ··· 3458 3539 ssize_t ret; 3459 3540 3460 3541 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) { 3461 - ret = io_import_fixed(req, rw, iter); 3542 + ret = io_import_fixed(req, rw, iter, issue_flags); 3462 3543 if (ret) 3463 3544 return ERR_PTR(ret); 3464 3545 return NULL; ··· 3659 3740 return 0; 3660 3741 } 3661 3742 3662 - static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 3663 - { 3664 - if (unlikely(!(req->file->f_mode & FMODE_READ))) 3665 - return -EBADF; 3666 - return io_prep_rw(req, sqe); 3667 - } 3668 - 3669 3743 /* 3670 3744 * This is our waitqueue callback handler, registered through __folio_lock_async() 3671 3745 * when we initially tried to do the IO with the iocb armed our waitqueue. ··· 3746 3834 S_ISBLK(file_inode(req->file)->i_mode); 3747 3835 } 3748 3836 3837 + static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) 3838 + { 3839 + struct kiocb *kiocb = &req->rw.kiocb; 3840 + struct io_ring_ctx *ctx = req->ctx; 3841 + struct file *file = req->file; 3842 + int ret; 3843 + 3844 + if (unlikely(!file || !(file->f_mode & mode))) 3845 + return -EBADF; 3846 + 3847 + if (!io_req_ffs_set(req)) 3848 + req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT; 3849 + 3850 + kiocb->ki_flags = iocb_flags(file); 3851 + ret = kiocb_set_rw_flags(kiocb, req->rw.flags); 3852 + if (unlikely(ret)) 3853 + return ret; 3854 + 3855 + /* 3856 + * If the file is marked O_NONBLOCK, still allow retry for it if it 3857 + * supports async. Otherwise it's impossible to use O_NONBLOCK files 3858 + * reliably. If not, or it IOCB_NOWAIT is set, don't retry. 3859 + */ 3860 + if ((kiocb->ki_flags & IOCB_NOWAIT) || 3861 + ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req))) 3862 + req->flags |= REQ_F_NOWAIT; 3863 + 3864 + if (ctx->flags & IORING_SETUP_IOPOLL) { 3865 + if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll) 3866 + return -EOPNOTSUPP; 3867 + 3868 + kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE; 3869 + kiocb->ki_complete = io_complete_rw_iopoll; 3870 + req->iopoll_completed = 0; 3871 + } else { 3872 + if (kiocb->ki_flags & IOCB_HIPRI) 3873 + return -EINVAL; 3874 + kiocb->ki_complete = io_complete_rw; 3875 + } 3876 + 3877 + return 0; 3878 + } 3879 + 3749 3880 static int io_read(struct io_kiocb *req, unsigned int issue_flags) 3750 3881 { 3751 3882 struct io_rw_state __s, *s = &__s; ··· 3824 3869 iov_iter_restore(&s->iter, &s->iter_state); 3825 3870 iovec = NULL; 3826 3871 } 3872 + ret = io_rw_init_file(req, FMODE_READ); 3873 + if (unlikely(ret)) 3874 + return ret; 3827 3875 req->result = iov_iter_count(&s->iter); 3828 3876 3829 3877 if (force_nonblock) { ··· 3930 3972 return 0; 3931 3973 } 3932 3974 3933 - static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 3934 - { 3935 - if (unlikely(!(req->file->f_mode & FMODE_WRITE))) 3936 - return -EBADF; 3937 - return io_prep_rw(req, sqe); 3938 - } 3939 - 3940 3975 static int io_write(struct io_kiocb *req, unsigned int issue_flags) 3941 3976 { 3942 3977 struct io_rw_state __s, *s = &__s; ··· 3950 3999 iov_iter_restore(&s->iter, &s->iter_state); 3951 4000 iovec = NULL; 3952 4001 } 4002 + ret = io_rw_init_file(req, FMODE_WRITE); 4003 + if (unlikely(ret)) 4004 + return ret; 3953 4005 req->result = iov_iter_count(&s->iter); 3954 4006 3955 4007 if (force_nonblock) { ··· 4323 4369 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) 4324 4370 return -EINVAL; 4325 4371 4326 - sp->file_in = NULL; 4327 4372 sp->len = READ_ONCE(sqe->len); 4328 4373 sp->flags = READ_ONCE(sqe->splice_flags); 4329 - 4330 4374 if (unlikely(sp->flags & ~valid_flags)) 4331 4375 return -EINVAL; 4332 - 4333 - sp->file_in = io_file_get(req->ctx, req, READ_ONCE(sqe->splice_fd_in), 4334 - (sp->flags & SPLICE_F_FD_IN_FIXED)); 4335 - if (!sp->file_in) 4336 - return -EBADF; 4337 - req->flags |= REQ_F_NEED_CLEANUP; 4376 + sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in); 4338 4377 return 0; 4339 4378 } 4340 4379 ··· 4342 4395 static int io_tee(struct io_kiocb *req, unsigned int issue_flags) 4343 4396 { 4344 4397 struct io_splice *sp = &req->splice; 4345 - struct file *in = sp->file_in; 4346 4398 struct file *out = sp->file_out; 4347 4399 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; 4400 + struct file *in; 4348 4401 long ret = 0; 4349 4402 4350 4403 if (issue_flags & IO_URING_F_NONBLOCK) 4351 4404 return -EAGAIN; 4405 + 4406 + if (sp->flags & SPLICE_F_FD_IN_FIXED) 4407 + in = io_file_get_fixed(req, sp->splice_fd_in, IO_URING_F_UNLOCKED); 4408 + else 4409 + in = io_file_get_normal(req, sp->splice_fd_in); 4410 + if (!in) { 4411 + ret = -EBADF; 4412 + goto done; 4413 + } 4414 + 4352 4415 if (sp->len) 4353 4416 ret = do_tee(in, out, sp->len, flags); 4354 4417 4355 4418 if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) 4356 4419 io_put_file(in); 4357 - req->flags &= ~REQ_F_NEED_CLEANUP; 4358 - 4420 + done: 4359 4421 if (ret != sp->len) 4360 4422 req_set_fail(req); 4361 4423 io_req_complete(req, ret); ··· 4383 4427 static int io_splice(struct io_kiocb *req, unsigned int issue_flags) 4384 4428 { 4385 4429 struct io_splice *sp = &req->splice; 4386 - struct file *in = sp->file_in; 4387 4430 struct file *out = sp->file_out; 4388 4431 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; 4389 4432 loff_t *poff_in, *poff_out; 4433 + struct file *in; 4390 4434 long ret = 0; 4391 4435 4392 4436 if (issue_flags & IO_URING_F_NONBLOCK) 4393 4437 return -EAGAIN; 4438 + 4439 + if (sp->flags & SPLICE_F_FD_IN_FIXED) 4440 + in = io_file_get_fixed(req, sp->splice_fd_in, IO_URING_F_UNLOCKED); 4441 + else 4442 + in = io_file_get_normal(req, sp->splice_fd_in); 4443 + if (!in) { 4444 + ret = -EBADF; 4445 + goto done; 4446 + } 4394 4447 4395 4448 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in; 4396 4449 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out; ··· 4409 4444 4410 4445 if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) 4411 4446 io_put_file(in); 4412 - req->flags &= ~REQ_F_NEED_CLEANUP; 4413 - 4447 + done: 4414 4448 if (ret != sp->len) 4415 4449 req_set_fail(req); 4416 4450 io_req_complete(req, ret); ··· 4476 4512 static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 4477 4513 { 4478 4514 struct io_ring_ctx *ctx = req->ctx; 4479 - 4480 - if (!req->file) 4481 - return -EBADF; 4482 4515 4483 4516 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) 4484 4517 return -EINVAL; ··· 5718 5757 IO_NETOP_FN(recv); 5719 5758 #endif /* CONFIG_NET */ 5720 5759 5721 - #ifdef CONFIG_NET_RX_BUSY_POLL 5722 - 5723 - #define NAPI_TIMEOUT (60 * SEC_CONVERSION) 5724 - 5725 - struct napi_entry { 5726 - struct list_head list; 5727 - unsigned int napi_id; 5728 - unsigned long timeout; 5729 - }; 5730 - 5731 - /* 5732 - * Add busy poll NAPI ID from sk. 5733 - */ 5734 - static void io_add_napi(struct file *file, struct io_ring_ctx *ctx) 5735 - { 5736 - unsigned int napi_id; 5737 - struct socket *sock; 5738 - struct sock *sk; 5739 - struct napi_entry *ne; 5740 - 5741 - if (!net_busy_loop_on()) 5742 - return; 5743 - 5744 - sock = sock_from_file(file); 5745 - if (!sock) 5746 - return; 5747 - 5748 - sk = sock->sk; 5749 - if (!sk) 5750 - return; 5751 - 5752 - napi_id = READ_ONCE(sk->sk_napi_id); 5753 - 5754 - /* Non-NAPI IDs can be rejected */ 5755 - if (napi_id < MIN_NAPI_ID) 5756 - return; 5757 - 5758 - spin_lock(&ctx->napi_lock); 5759 - list_for_each_entry(ne, &ctx->napi_list, list) { 5760 - if (ne->napi_id == napi_id) { 5761 - ne->timeout = jiffies + NAPI_TIMEOUT; 5762 - goto out; 5763 - } 5764 - } 5765 - 5766 - ne = kmalloc(sizeof(*ne), GFP_NOWAIT); 5767 - if (!ne) 5768 - goto out; 5769 - 5770 - ne->napi_id = napi_id; 5771 - ne->timeout = jiffies + NAPI_TIMEOUT; 5772 - list_add_tail(&ne->list, &ctx->napi_list); 5773 - out: 5774 - spin_unlock(&ctx->napi_lock); 5775 - } 5776 - 5777 - static inline void io_check_napi_entry_timeout(struct napi_entry *ne) 5778 - { 5779 - if (time_after(jiffies, ne->timeout)) { 5780 - list_del(&ne->list); 5781 - kfree(ne); 5782 - } 5783 - } 5784 - 5785 - /* 5786 - * Busy poll if globally on and supporting sockets found 5787 - */ 5788 - static bool io_napi_busy_loop(struct list_head *napi_list) 5789 - { 5790 - struct napi_entry *ne, *n; 5791 - 5792 - list_for_each_entry_safe(ne, n, napi_list, list) { 5793 - napi_busy_loop(ne->napi_id, NULL, NULL, true, 5794 - BUSY_POLL_BUDGET); 5795 - io_check_napi_entry_timeout(ne); 5796 - } 5797 - return !list_empty(napi_list); 5798 - } 5799 - 5800 - static void io_free_napi_list(struct io_ring_ctx *ctx) 5801 - { 5802 - spin_lock(&ctx->napi_lock); 5803 - while (!list_empty(&ctx->napi_list)) { 5804 - struct napi_entry *ne = 5805 - list_first_entry(&ctx->napi_list, struct napi_entry, 5806 - list); 5807 - 5808 - list_del(&ne->list); 5809 - kfree(ne); 5810 - } 5811 - spin_unlock(&ctx->napi_lock); 5812 - } 5813 - #else 5814 - static inline void io_add_napi(struct file *file, struct io_ring_ctx *ctx) 5815 - { 5816 - } 5817 - 5818 - static inline void io_free_napi_list(struct io_ring_ctx *ctx) 5819 - { 5820 - } 5821 - #endif /* CONFIG_NET_RX_BUSY_POLL */ 5822 - 5823 5760 struct io_poll_table { 5824 5761 struct poll_table_struct pt; 5825 5762 struct io_kiocb *req; ··· 5831 5972 * either spurious wakeup or multishot CQE is served. 0 when it's done with 5832 5973 * the request, then the mask is stored in req->result. 5833 5974 */ 5834 - static int io_poll_check_events(struct io_kiocb *req) 5975 + static int io_poll_check_events(struct io_kiocb *req, bool locked) 5835 5976 { 5836 5977 struct io_ring_ctx *ctx = req->ctx; 5837 5978 struct io_poll_iocb *poll = io_poll_get_single(req); ··· 5853 5994 if (!req->result) { 5854 5995 struct poll_table_struct pt = { ._key = req->cflags }; 5855 5996 5856 - req->result = vfs_poll(req->file, &pt) & req->cflags; 5997 + if (unlikely(!io_assign_file(req, IO_URING_F_UNLOCKED))) 5998 + req->result = -EBADF; 5999 + else 6000 + req->result = vfs_poll(req->file, &pt) & req->cflags; 5857 6001 } 5858 6002 5859 6003 /* multishot, just fill an CQE and proceed */ ··· 5872 6010 if (unlikely(!filled)) 5873 6011 return -ECANCELED; 5874 6012 io_cqring_ev_posted(ctx); 5875 - io_add_napi(req->file, ctx); 5876 6013 } else if (req->result) { 5877 6014 return 0; 5878 6015 } ··· 5890 6029 struct io_ring_ctx *ctx = req->ctx; 5891 6030 int ret; 5892 6031 5893 - ret = io_poll_check_events(req); 6032 + ret = io_poll_check_events(req, *locked); 5894 6033 if (ret > 0) 5895 6034 return; 5896 6035 ··· 5915 6054 struct io_ring_ctx *ctx = req->ctx; 5916 6055 int ret; 5917 6056 5918 - ret = io_poll_check_events(req); 6057 + ret = io_poll_check_events(req, *locked); 5919 6058 if (ret > 0) 5920 6059 return; 5921 6060 ··· 6122 6261 __io_poll_execute(req, mask, poll->events); 6123 6262 return 0; 6124 6263 } 6125 - io_add_napi(req->file, req->ctx); 6126 6264 6127 6265 /* 6128 6266 * Release ownership. If someone tried to queue a tw while it was ··· 6626 6766 if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0) 6627 6767 return -EINVAL; 6628 6768 6769 + INIT_LIST_HEAD(&req->timeout.list); 6629 6770 data->mode = io_translate_timeout_mode(flags); 6630 6771 hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode); 6631 6772 ··· 6853 6992 case IORING_OP_READV: 6854 6993 case IORING_OP_READ_FIXED: 6855 6994 case IORING_OP_READ: 6856 - return io_read_prep(req, sqe); 6857 6995 case IORING_OP_WRITEV: 6858 6996 case IORING_OP_WRITE_FIXED: 6859 6997 case IORING_OP_WRITE: 6860 - return io_write_prep(req, sqe); 6998 + return io_prep_rw(req, sqe); 6861 6999 case IORING_OP_POLL_ADD: 6862 7000 return io_poll_add_prep(req, sqe); 6863 7001 case IORING_OP_POLL_REMOVE: ··· 7039 7179 kfree(io->free_iov); 7040 7180 break; 7041 7181 } 7042 - case IORING_OP_SPLICE: 7043 - case IORING_OP_TEE: 7044 - if (!(req->splice.flags & SPLICE_F_FD_IN_FIXED)) 7045 - io_put_file(req->splice.file_in); 7046 - break; 7047 7182 case IORING_OP_OPENAT: 7048 7183 case IORING_OP_OPENAT2: 7049 7184 if (req->open.filename) ··· 7073 7218 kfree(req->apoll); 7074 7219 req->apoll = NULL; 7075 7220 } 7076 - if (req->flags & REQ_F_INFLIGHT) { 7077 - struct io_uring_task *tctx = req->task->io_uring; 7078 - 7079 - atomic_dec(&tctx->inflight_tracked); 7080 - } 7081 7221 if (req->flags & REQ_F_CREDS) 7082 7222 put_cred(req->creds); 7083 7223 if (req->flags & REQ_F_ASYNC_DATA) { ··· 7080 7230 req->async_data = NULL; 7081 7231 } 7082 7232 req->flags &= ~IO_REQ_CLEAN_FLAGS; 7233 + } 7234 + 7235 + static bool io_assign_file(struct io_kiocb *req, unsigned int issue_flags) 7236 + { 7237 + if (req->file || !io_op_defs[req->opcode].needs_file) 7238 + return true; 7239 + 7240 + if (req->flags & REQ_F_FIXED_FILE) 7241 + req->file = io_file_get_fixed(req, req->work.fd, issue_flags); 7242 + else 7243 + req->file = io_file_get_normal(req, req->work.fd); 7244 + if (req->file) 7245 + return true; 7246 + 7247 + req_set_fail(req); 7248 + req->result = -EBADF; 7249 + return false; 7083 7250 } 7084 7251 7085 7252 static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) ··· 7109 7242 7110 7243 if (!io_op_defs[req->opcode].audit_skip) 7111 7244 audit_uring_entry(req->opcode); 7245 + if (unlikely(!io_assign_file(req, issue_flags))) 7246 + return -EBADF; 7112 7247 7113 7248 switch (req->opcode) { 7114 7249 case IORING_OP_NOP: ··· 7255 7386 static void io_wq_submit_work(struct io_wq_work *work) 7256 7387 { 7257 7388 struct io_kiocb *req = container_of(work, struct io_kiocb, work); 7389 + const struct io_op_def *def = &io_op_defs[req->opcode]; 7258 7390 unsigned int issue_flags = IO_URING_F_UNLOCKED; 7259 7391 bool needs_poll = false; 7260 7392 struct io_kiocb *timeout; 7261 - int ret = 0; 7393 + int ret = 0, err = -ECANCELED; 7262 7394 7263 7395 /* one will be dropped by ->io_free_work() after returning to io-wq */ 7264 7396 if (!(req->flags & REQ_F_REFCOUNT)) ··· 7271 7401 if (timeout) 7272 7402 io_queue_linked_timeout(timeout); 7273 7403 7404 + if (!io_assign_file(req, issue_flags)) { 7405 + err = -EBADF; 7406 + work->flags |= IO_WQ_WORK_CANCEL; 7407 + } 7408 + 7274 7409 /* either cancelled or io-wq is dying, so don't touch tctx->iowq */ 7275 7410 if (work->flags & IO_WQ_WORK_CANCEL) { 7276 - io_req_task_queue_fail(req, -ECANCELED); 7411 + io_req_task_queue_fail(req, err); 7277 7412 return; 7278 7413 } 7279 7414 7280 7415 if (req->flags & REQ_F_FORCE_ASYNC) { 7281 - const struct io_op_def *def = &io_op_defs[req->opcode]; 7282 7416 bool opcode_poll = def->pollin || def->pollout; 7283 7417 7284 7418 if (opcode_poll && file_can_poll(req->file)) { ··· 7339 7465 file_slot->file_ptr = file_ptr; 7340 7466 } 7341 7467 7342 - static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx, 7343 - struct io_kiocb *req, int fd) 7468 + static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 7469 + unsigned int issue_flags) 7344 7470 { 7345 - struct file *file; 7471 + struct io_ring_ctx *ctx = req->ctx; 7472 + struct file *file = NULL; 7346 7473 unsigned long file_ptr; 7347 7474 7475 + if (issue_flags & IO_URING_F_UNLOCKED) 7476 + mutex_lock(&ctx->uring_lock); 7477 + 7348 7478 if (unlikely((unsigned int)fd >= ctx->nr_user_files)) 7349 - return NULL; 7479 + goto out; 7350 7480 fd = array_index_nospec(fd, ctx->nr_user_files); 7351 7481 file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr; 7352 7482 file = (struct file *) (file_ptr & FFS_MASK); 7353 7483 file_ptr &= ~FFS_MASK; 7354 7484 /* mask in overlapping REQ_F and FFS bits */ 7355 7485 req->flags |= (file_ptr << REQ_F_SUPPORT_NOWAIT_BIT); 7356 - io_req_set_rsrc_node(req, ctx); 7486 + io_req_set_rsrc_node(req, ctx, 0); 7487 + out: 7488 + if (issue_flags & IO_URING_F_UNLOCKED) 7489 + mutex_unlock(&ctx->uring_lock); 7357 7490 return file; 7358 7491 } 7359 7492 7360 - static struct file *io_file_get_normal(struct io_ring_ctx *ctx, 7361 - struct io_kiocb *req, int fd) 7493 + /* 7494 + * Drop the file for requeue operations. Only used of req->file is the 7495 + * io_uring descriptor itself. 7496 + */ 7497 + static void io_drop_inflight_file(struct io_kiocb *req) 7498 + { 7499 + if (unlikely(req->flags & REQ_F_INFLIGHT)) { 7500 + fput(req->file); 7501 + req->file = NULL; 7502 + req->flags &= ~REQ_F_INFLIGHT; 7503 + } 7504 + } 7505 + 7506 + static struct file *io_file_get_normal(struct io_kiocb *req, int fd) 7362 7507 { 7363 7508 struct file *file = fget(fd); 7364 7509 7365 - trace_io_uring_file_get(ctx, req, req->user_data, fd); 7510 + trace_io_uring_file_get(req->ctx, req, req->user_data, fd); 7366 7511 7367 7512 /* we don't allow fixed io_uring files */ 7368 - if (file && unlikely(file->f_op == &io_uring_fops)) 7369 - io_req_track_inflight(req); 7513 + if (file && file->f_op == &io_uring_fops) 7514 + req->flags |= REQ_F_INFLIGHT; 7370 7515 return file; 7371 - } 7372 - 7373 - static inline struct file *io_file_get(struct io_ring_ctx *ctx, 7374 - struct io_kiocb *req, int fd, bool fixed) 7375 - { 7376 - if (fixed) 7377 - return io_file_get_fixed(ctx, req, fd); 7378 - else 7379 - return io_file_get_normal(ctx, req, fd); 7380 7516 } 7381 7517 7382 7518 static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) ··· 7628 7744 if (io_op_defs[opcode].needs_file) { 7629 7745 struct io_submit_state *state = &ctx->submit_state; 7630 7746 7747 + req->work.fd = READ_ONCE(sqe->fd); 7748 + 7631 7749 /* 7632 7750 * Plug now if we have more than 2 IO left after this, and the 7633 7751 * target is potentially a read/write to block based storage. ··· 7639 7753 state->need_plug = false; 7640 7754 blk_start_plug_nr_ios(&state->plug, state->submit_nr); 7641 7755 } 7642 - 7643 - req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd), 7644 - (sqe_flags & IOSQE_FIXED_FILE)); 7645 - if (unlikely(!req->file)) 7646 - return -EBADF; 7647 7756 } 7648 7757 7649 7758 personality = READ_ONCE(sqe->personality); ··· 7913 8032 !(ctx->flags & IORING_SETUP_R_DISABLED)) 7914 8033 ret = io_submit_sqes(ctx, to_submit); 7915 8034 mutex_unlock(&ctx->uring_lock); 7916 - #ifdef CONFIG_NET_RX_BUSY_POLL 7917 - spin_lock(&ctx->napi_lock); 7918 - if (!list_empty(&ctx->napi_list) && 7919 - io_napi_busy_loop(&ctx->napi_list)) 7920 - ++ret; 7921 - spin_unlock(&ctx->napi_lock); 7922 - #endif 8035 + 7923 8036 if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait)) 7924 8037 wake_up(&ctx->sqo_sq_wait); 7925 8038 if (creds) ··· 8051 8176 struct io_ring_ctx *ctx; 8052 8177 unsigned cq_tail; 8053 8178 unsigned nr_timeouts; 8054 - #ifdef CONFIG_NET_RX_BUSY_POLL 8055 - unsigned busy_poll_to; 8056 - #endif 8057 8179 }; 8058 8180 8059 8181 static inline bool io_should_wake(struct io_wait_queue *iowq) ··· 8112 8240 return 1; 8113 8241 } 8114 8242 8115 - #ifdef CONFIG_NET_RX_BUSY_POLL 8116 - static void io_adjust_busy_loop_timeout(struct timespec64 *ts, 8117 - struct io_wait_queue *iowq) 8118 - { 8119 - unsigned busy_poll_to = READ_ONCE(sysctl_net_busy_poll); 8120 - struct timespec64 pollto = ns_to_timespec64(1000 * (s64)busy_poll_to); 8121 - 8122 - if (timespec64_compare(ts, &pollto) > 0) { 8123 - *ts = timespec64_sub(*ts, pollto); 8124 - iowq->busy_poll_to = busy_poll_to; 8125 - } else { 8126 - u64 to = timespec64_to_ns(ts); 8127 - 8128 - do_div(to, 1000); 8129 - iowq->busy_poll_to = to; 8130 - ts->tv_sec = 0; 8131 - ts->tv_nsec = 0; 8132 - } 8133 - } 8134 - 8135 - static inline bool io_busy_loop_timeout(unsigned long start_time, 8136 - unsigned long bp_usec) 8137 - { 8138 - if (bp_usec) { 8139 - unsigned long end_time = start_time + bp_usec; 8140 - unsigned long now = busy_loop_current_time(); 8141 - 8142 - return time_after(now, end_time); 8143 - } 8144 - return true; 8145 - } 8146 - 8147 - static bool io_busy_loop_end(void *p, unsigned long start_time) 8148 - { 8149 - struct io_wait_queue *iowq = p; 8150 - 8151 - return signal_pending(current) || 8152 - io_should_wake(iowq) || 8153 - io_busy_loop_timeout(start_time, iowq->busy_poll_to); 8154 - } 8155 - 8156 - static void io_blocking_napi_busy_loop(struct list_head *napi_list, 8157 - struct io_wait_queue *iowq) 8158 - { 8159 - unsigned long start_time = 8160 - list_is_singular(napi_list) ? 0 : 8161 - busy_loop_current_time(); 8162 - 8163 - do { 8164 - if (list_is_singular(napi_list)) { 8165 - struct napi_entry *ne = 8166 - list_first_entry(napi_list, 8167 - struct napi_entry, list); 8168 - 8169 - napi_busy_loop(ne->napi_id, io_busy_loop_end, iowq, 8170 - true, BUSY_POLL_BUDGET); 8171 - io_check_napi_entry_timeout(ne); 8172 - break; 8173 - } 8174 - } while (io_napi_busy_loop(napi_list) && 8175 - !io_busy_loop_end(iowq, start_time)); 8176 - } 8177 - 8178 - static void io_putback_napi_list(struct io_ring_ctx *ctx, 8179 - struct list_head *napi_list) 8180 - { 8181 - struct napi_entry *cne, *lne; 8182 - 8183 - spin_lock(&ctx->napi_lock); 8184 - list_for_each_entry(cne, &ctx->napi_list, list) 8185 - list_for_each_entry(lne, napi_list, list) 8186 - if (cne->napi_id == lne->napi_id) { 8187 - list_del(&lne->list); 8188 - kfree(lne); 8189 - break; 8190 - } 8191 - list_splice(napi_list, &ctx->napi_list); 8192 - spin_unlock(&ctx->napi_lock); 8193 - } 8194 - #endif /* CONFIG_NET_RX_BUSY_POLL */ 8195 - 8196 8243 /* 8197 8244 * Wait until events become available, if we don't already have some. The 8198 8245 * application must reap them itself, as they reside on the shared cq ring. ··· 8124 8333 struct io_rings *rings = ctx->rings; 8125 8334 ktime_t timeout = KTIME_MAX; 8126 8335 int ret; 8127 - #ifdef CONFIG_NET_RX_BUSY_POLL 8128 - LIST_HEAD(local_napi_list); 8129 - #endif 8130 8336 8131 8337 do { 8132 8338 io_cqring_overflow_flush(ctx); ··· 8146 8358 return ret; 8147 8359 } 8148 8360 8149 - #ifdef CONFIG_NET_RX_BUSY_POLL 8150 - iowq.busy_poll_to = 0; 8151 - if (!(ctx->flags & IORING_SETUP_SQPOLL)) { 8152 - spin_lock(&ctx->napi_lock); 8153 - list_splice_init(&ctx->napi_list, &local_napi_list); 8154 - spin_unlock(&ctx->napi_lock); 8155 - } 8156 - #endif 8157 8361 if (uts) { 8158 8362 struct timespec64 ts; 8159 8363 8160 8364 if (get_timespec64(&ts, uts)) 8161 8365 return -EFAULT; 8162 - #ifdef CONFIG_NET_RX_BUSY_POLL 8163 - if (!list_empty(&local_napi_list)) 8164 - io_adjust_busy_loop_timeout(&ts, &iowq); 8165 - #endif 8166 8366 timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns()); 8167 8367 } 8168 - #ifdef CONFIG_NET_RX_BUSY_POLL 8169 - else if (!list_empty(&local_napi_list)) 8170 - iowq.busy_poll_to = READ_ONCE(sysctl_net_busy_poll); 8171 - #endif 8172 8368 8173 8369 init_waitqueue_func_entry(&iowq.wq, io_wake_function); 8174 8370 iowq.wq.private = current; ··· 8162 8390 iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events; 8163 8391 8164 8392 trace_io_uring_cqring_wait(ctx, min_events); 8165 - #ifdef CONFIG_NET_RX_BUSY_POLL 8166 - if (iowq.busy_poll_to) 8167 - io_blocking_napi_busy_loop(&local_napi_list, &iowq); 8168 - if (!list_empty(&local_napi_list)) 8169 - io_putback_napi_list(ctx, &local_napi_list); 8170 - #endif 8171 8393 do { 8172 8394 /* if we can't even flush overflow, don't wait for more */ 8173 8395 if (!io_cqring_overflow_flush(ctx)) { ··· 8630 8864 refcount_add(skb->truesize, &sk->sk_wmem_alloc); 8631 8865 skb_queue_head(&sk->sk_receive_queue, skb); 8632 8866 8633 - for (i = 0; i < nr_files; i++) 8634 - fput(fpl->fp[i]); 8867 + for (i = 0; i < nr; i++) { 8868 + struct file *file = io_file_from_index(ctx, i + offset); 8869 + 8870 + if (file) 8871 + fput(file); 8872 + } 8635 8873 } else { 8636 8874 kfree_skb(skb); 8637 8875 free_uid(fpl->user); ··· 8926 9156 static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, 8927 9157 struct io_rsrc_node *node, void *rsrc) 8928 9158 { 9159 + u64 *tag_slot = io_get_tag_slot(data, idx); 8929 9160 struct io_rsrc_put *prsrc; 8930 9161 8931 9162 prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL); 8932 9163 if (!prsrc) 8933 9164 return -ENOMEM; 8934 9165 8935 - prsrc->tag = *io_get_tag_slot(data, idx); 9166 + prsrc->tag = *tag_slot; 9167 + *tag_slot = 0; 8936 9168 prsrc->rsrc = rsrc; 8937 9169 list_add(&prsrc->list, &node->rsrc_list); 8938 9170 return 0; ··· 9003 9231 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED; 9004 9232 struct io_fixed_file *file_slot; 9005 9233 struct file *file; 9006 - int ret, i; 9234 + int ret; 9007 9235 9008 9236 io_ring_submit_lock(ctx, needs_lock); 9009 9237 ret = -ENXIO; ··· 9016 9244 if (ret) 9017 9245 goto out; 9018 9246 9019 - i = array_index_nospec(offset, ctx->nr_user_files); 9020 - file_slot = io_fixed_file_slot(&ctx->file_table, i); 9247 + offset = array_index_nospec(offset, ctx->nr_user_files); 9248 + file_slot = io_fixed_file_slot(&ctx->file_table, offset); 9021 9249 ret = -EBADF; 9022 9250 if (!file_slot->file_ptr) 9023 9251 goto out; ··· 9073 9301 9074 9302 if (file_slot->file_ptr) { 9075 9303 file = (struct file *)(file_slot->file_ptr & FFS_MASK); 9076 - err = io_queue_rsrc_removal(data, up->offset + done, 9077 - ctx->rsrc_node, file); 9304 + err = io_queue_rsrc_removal(data, i, ctx->rsrc_node, file); 9078 9305 if (err) 9079 9306 break; 9080 9307 file_slot->file_ptr = 0; ··· 9098 9327 err = -EBADF; 9099 9328 break; 9100 9329 } 9101 - *io_get_tag_slot(data, up->offset + done) = tag; 9330 + *io_get_tag_slot(data, i) = tag; 9102 9331 io_fixed_file_set(file_slot, file); 9103 9332 err = io_sqe_file_register(ctx, file, i); 9104 9333 if (err) { ··· 9182 9411 xa_init(&tctx->xa); 9183 9412 init_waitqueue_head(&tctx->wait); 9184 9413 atomic_set(&tctx->in_idle, 0); 9185 - atomic_set(&tctx->inflight_tracked, 0); 9186 9414 task->io_uring = tctx; 9187 9415 spin_lock_init(&tctx->task_lock); 9188 9416 INIT_WQ_LIST(&tctx->task_list); ··· 9756 9986 9757 9987 i = array_index_nospec(offset, ctx->nr_user_bufs); 9758 9988 if (ctx->user_bufs[i] != ctx->dummy_ubuf) { 9759 - err = io_queue_rsrc_removal(ctx->buf_data, offset, 9989 + err = io_queue_rsrc_removal(ctx->buf_data, i, 9760 9990 ctx->rsrc_node, ctx->user_bufs[i]); 9761 9991 if (unlikely(err)) { 9762 9992 io_buffer_unmap(ctx, &imu); ··· 9951 10181 io_req_caches_free(ctx); 9952 10182 if (ctx->hash_map) 9953 10183 io_wq_put_hash(ctx->hash_map); 9954 - io_free_napi_list(ctx); 9955 10184 kfree(ctx->cancel_hash); 9956 10185 kfree(ctx->dummy_ubuf); 9957 10186 kfree(ctx->io_buffers); ··· 10373 10604 static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked) 10374 10605 { 10375 10606 if (tracked) 10376 - return atomic_read(&tctx->inflight_tracked); 10607 + return 0; 10377 10608 return percpu_counter_sum(&tctx->inflight); 10378 10609 } 10379 10610 ··· 11476 11707 if (len > cpumask_size()) 11477 11708 len = cpumask_size(); 11478 11709 11479 - if (copy_from_user(new_mask, arg, len)) { 11710 + if (in_compat_syscall()) { 11711 + ret = compat_get_bitmap(cpumask_bits(new_mask), 11712 + (const compat_ulong_t __user *)arg, 11713 + len * 8 /* CHAR_BIT */); 11714 + } else { 11715 + ret = copy_from_user(new_mask, arg, len); 11716 + } 11717 + 11718 + if (ret) { 11480 11719 free_cpumask_var(new_mask); 11481 11720 return -EFAULT; 11482 11721 }

Configure Feed

Configure Feed