Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

io_uring: split network related opcodes into its own file

While at it, convert the handlers to just use io_eopnotsupp_prep()
if CONFIG_NET isn't set.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

+884 -835
+1 -1
io_uring/Makefile
··· 5 5 obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \ 6 6 sync.o advise.o filetable.o \ 7 7 openclose.o uring_cmd.o epoll.o \ 8 - statx.o 8 + statx.o net.o 9 9 obj-$(CONFIG_IO_WQ) += io-wq.o
+39 -834
io_uring/io_uring.c
··· 102 102 #include "uring_cmd.h" 103 103 #include "epoll.h" 104 104 #include "statx.h" 105 + #include "net.h" 105 106 106 107 #define IORING_MAX_ENTRIES 32768 107 108 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) ··· 131 130 132 131 #define IO_REQ_CLEAN_SLOW_FLAGS (REQ_F_REFCOUNT | REQ_F_LINK | REQ_F_HARDLINK |\ 133 132 IO_REQ_CLEAN_FLAGS) 134 - 135 - #define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED) 136 133 137 134 #define IO_TCTX_REFS_CACHE_NR (1U << 10) 138 135 ··· 294 295 u32 flags; 295 296 }; 296 297 297 - struct io_accept { 298 - struct file *file; 299 - struct sockaddr __user *addr; 300 - int __user *addr_len; 301 - int flags; 302 - u32 file_slot; 303 - unsigned long nofile; 304 - }; 305 - 306 - struct io_socket { 307 - struct file *file; 308 - int domain; 309 - int type; 310 - int protocol; 311 - int flags; 312 - u32 file_slot; 313 - unsigned long nofile; 314 - }; 315 - 316 298 struct io_cancel { 317 299 struct file *file; 318 300 u64 addr; ··· 330 350 rwf_t flags; 331 351 }; 332 352 333 - struct io_connect { 334 - struct file *file; 335 - struct sockaddr __user *addr; 336 - int addr_len; 337 - }; 338 - 339 - struct io_sr_msg { 340 - struct file *file; 341 - union { 342 - struct compat_msghdr __user *umsg_compat; 343 - struct user_msghdr __user *umsg; 344 - void __user *buf; 345 - }; 346 - int msg_flags; 347 - size_t len; 348 - size_t done_io; 349 - unsigned int flags; 350 - }; 351 - 352 353 struct io_rsrc_update { 353 354 struct file *file; 354 355 u64 arg; ··· 346 385 __u16 bid; 347 386 }; 348 387 349 - struct io_shutdown { 350 - struct file *file; 351 - int how; 352 - }; 353 - 354 388 struct io_msg { 355 389 struct file *file; 356 390 u64 user_data; 357 391 u32 len; 358 - }; 359 - 360 - struct io_async_connect { 361 - struct sockaddr_storage address; 362 - }; 363 - 364 - struct io_async_msghdr { 365 - struct iovec fast_iov[UIO_FASTIOV]; 366 - /* points to an allocated iov, if NULL we use fast_iov instead */ 367 - struct iovec *free_iov; 368 - struct sockaddr __user *uaddr; 369 - struct msghdr msg; 370 - struct sockaddr_storage addr; 371 392 }; 372 393 373 394 struct io_rw_state { ··· 459 516 static void io_req_task_queue(struct io_kiocb *req); 460 517 static void __io_submit_flush_completions(struct io_ring_ctx *ctx); 461 518 static int io_req_prep_async(struct io_kiocb *req); 462 - 463 - static int io_install_fixed_file(struct io_kiocb *req, struct file *file, 464 - unsigned int issue_flags, u32 slot_index); 465 519 466 520 static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer); 467 521 static void io_eventfd_signal(struct io_ring_ctx *ctx); ··· 748 808 return __io_put_kbuf(req, &req->ctx->io_buffers_comp); 749 809 } 750 810 751 - static inline unsigned int io_put_kbuf(struct io_kiocb *req, 752 - unsigned issue_flags) 811 + inline unsigned int io_put_kbuf(struct io_kiocb *req, unsigned issue_flags) 753 812 { 754 813 unsigned int cflags; 755 814 ··· 1230 1291 spin_unlock_irq(&ctx->timeout_lock); 1231 1292 } 1232 1293 1233 - static inline void io_commit_cqring(struct io_ring_ctx *ctx) 1234 - { 1235 - /* order cqe stores with ring update */ 1236 - smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); 1237 - } 1238 - 1239 1294 static void __io_commit_cqring_flush(struct io_ring_ctx *ctx) 1240 1295 { 1241 1296 if (ctx->off_timeout_used || ctx->drain_active) { ··· 1351 1418 * 1:1 relationship between how many times this function is called (and 1352 1419 * hence the eventfd count) and number of CQEs posted to the CQ ring. 1353 1420 */ 1354 - static inline void io_cqring_ev_posted(struct io_ring_ctx *ctx) 1421 + void io_cqring_ev_posted(struct io_ring_ctx *ctx) 1355 1422 { 1356 1423 if (unlikely(ctx->off_timeout_used || ctx->drain_active || 1357 1424 ctx->has_evfd)) ··· 1572 1639 } 1573 1640 } 1574 1641 1575 - static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, 1576 - s32 res, u32 cflags) 1642 + bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, 1643 + u32 cflags) 1577 1644 { 1578 1645 struct io_uring_cqe *cqe; 1579 1646 ··· 2913 2980 return u64_to_user_ptr(buf->addr); 2914 2981 } 2915 2982 2916 - static void __user *io_buffer_select(struct io_kiocb *req, size_t *len, 2917 - unsigned int issue_flags) 2983 + void __user *io_buffer_select(struct io_kiocb *req, size_t *len, 2984 + unsigned int issue_flags) 2918 2985 { 2919 2986 struct io_ring_ctx *ctx = req->ctx; 2920 2987 struct io_buffer_list *bl; ··· 3004 3071 #endif 3005 3072 3006 3073 return __io_iov_buffer_select(req, iov, issue_flags); 3007 - } 3008 - 3009 - static inline bool io_do_buffer_select(struct io_kiocb *req) 3010 - { 3011 - if (!(req->flags & REQ_F_BUFFER_SELECT)) 3012 - return false; 3013 - return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)); 3014 3074 } 3015 3075 3016 3076 static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, ··· 3950 4024 { 3951 4025 return -EOPNOTSUPP; 3952 4026 } 3953 - 3954 - #if defined(CONFIG_NET) 3955 - static int io_shutdown_prep(struct io_kiocb *req, 3956 - const struct io_uring_sqe *sqe) 3957 - { 3958 - struct io_shutdown *shutdown = io_kiocb_to_cmd(req); 3959 - 3960 - if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || 3961 - sqe->buf_index || sqe->splice_fd_in)) 3962 - return -EINVAL; 3963 - 3964 - shutdown->how = READ_ONCE(sqe->len); 3965 - return 0; 3966 - } 3967 - 3968 - static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) 3969 - { 3970 - struct io_shutdown *shutdown = io_kiocb_to_cmd(req); 3971 - struct socket *sock; 3972 - int ret; 3973 - 3974 - if (issue_flags & IO_URING_F_NONBLOCK) 3975 - return -EAGAIN; 3976 - 3977 - sock = sock_from_file(req->file); 3978 - if (unlikely(!sock)) 3979 - return -ENOTSOCK; 3980 - 3981 - ret = __sys_shutdown_sock(sock, shutdown->how); 3982 - io_req_set_res(req, ret, 0); 3983 - return IOU_OK; 3984 - } 3985 - 3986 - static bool io_net_retry(struct socket *sock, int flags) 3987 - { 3988 - if (!(flags & MSG_WAITALL)) 3989 - return false; 3990 - return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; 3991 - } 3992 - 3993 - static int io_setup_async_msg(struct io_kiocb *req, 3994 - struct io_async_msghdr *kmsg) 3995 - { 3996 - struct io_async_msghdr *async_msg = req->async_data; 3997 - 3998 - if (async_msg) 3999 - return -EAGAIN; 4000 - if (io_alloc_async_data(req)) { 4001 - kfree(kmsg->free_iov); 4002 - return -ENOMEM; 4003 - } 4004 - async_msg = req->async_data; 4005 - req->flags |= REQ_F_NEED_CLEANUP; 4006 - memcpy(async_msg, kmsg, sizeof(*kmsg)); 4007 - async_msg->msg.msg_name = &async_msg->addr; 4008 - /* if were using fast_iov, set it to the new one */ 4009 - if (!async_msg->free_iov) 4010 - async_msg->msg.msg_iter.iov = async_msg->fast_iov; 4011 - 4012 - return -EAGAIN; 4013 - } 4014 - 4015 - static int io_sendmsg_copy_hdr(struct io_kiocb *req, 4016 - struct io_async_msghdr *iomsg) 4017 - { 4018 - struct io_sr_msg *sr = io_kiocb_to_cmd(req); 4019 - 4020 - iomsg->msg.msg_name = &iomsg->addr; 4021 - iomsg->free_iov = iomsg->fast_iov; 4022 - return sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags, 4023 - &iomsg->free_iov); 4024 - } 4025 - 4026 - static int io_sendmsg_prep_async(struct io_kiocb *req) 4027 - { 4028 - int ret; 4029 - 4030 - ret = io_sendmsg_copy_hdr(req, req->async_data); 4031 - if (!ret) 4032 - req->flags |= REQ_F_NEED_CLEANUP; 4033 - return ret; 4034 - } 4035 - 4036 - static void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) 4037 - { 4038 - struct io_async_msghdr *io = req->async_data; 4039 - 4040 - kfree(io->free_iov); 4041 - } 4042 - 4043 - static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 4044 - { 4045 - struct io_sr_msg *sr = io_kiocb_to_cmd(req); 4046 - 4047 - if (unlikely(sqe->file_index || sqe->addr2)) 4048 - return -EINVAL; 4049 - 4050 - sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 4051 - sr->len = READ_ONCE(sqe->len); 4052 - sr->flags = READ_ONCE(sqe->ioprio); 4053 - if (sr->flags & ~IORING_RECVSEND_POLL_FIRST) 4054 - return -EINVAL; 4055 - sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 4056 - if (sr->msg_flags & MSG_DONTWAIT) 4057 - req->flags |= REQ_F_NOWAIT; 4058 - 4059 - #ifdef CONFIG_COMPAT 4060 - if (req->ctx->compat) 4061 - sr->msg_flags |= MSG_CMSG_COMPAT; 4062 - #endif 4063 - sr->done_io = 0; 4064 - return 0; 4065 - } 4066 - 4067 - static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) 4068 - { 4069 - struct io_sr_msg *sr = io_kiocb_to_cmd(req); 4070 - struct io_async_msghdr iomsg, *kmsg; 4071 - struct socket *sock; 4072 - unsigned flags; 4073 - int min_ret = 0; 4074 - int ret; 4075 - 4076 - sock = sock_from_file(req->file); 4077 - if (unlikely(!sock)) 4078 - return -ENOTSOCK; 4079 - 4080 - if (req_has_async_data(req)) { 4081 - kmsg = req->async_data; 4082 - } else { 4083 - ret = io_sendmsg_copy_hdr(req, &iomsg); 4084 - if (ret) 4085 - return ret; 4086 - kmsg = &iomsg; 4087 - } 4088 - 4089 - if (!(req->flags & REQ_F_POLLED) && 4090 - (sr->flags & IORING_RECVSEND_POLL_FIRST)) 4091 - return io_setup_async_msg(req, kmsg); 4092 - 4093 - flags = sr->msg_flags; 4094 - if (issue_flags & IO_URING_F_NONBLOCK) 4095 - flags |= MSG_DONTWAIT; 4096 - if (flags & MSG_WAITALL) 4097 - min_ret = iov_iter_count(&kmsg->msg.msg_iter); 4098 - 4099 - ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 4100 - 4101 - if (ret < min_ret) { 4102 - if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 4103 - return io_setup_async_msg(req, kmsg); 4104 - if (ret == -ERESTARTSYS) 4105 - ret = -EINTR; 4106 - if (ret > 0 && io_net_retry(sock, flags)) { 4107 - sr->done_io += ret; 4108 - req->flags |= REQ_F_PARTIAL_IO; 4109 - return io_setup_async_msg(req, kmsg); 4110 - } 4111 - req_set_fail(req); 4112 - } 4113 - /* fast path, check for non-NULL to avoid function call */ 4114 - if (kmsg->free_iov) 4115 - kfree(kmsg->free_iov); 4116 - req->flags &= ~REQ_F_NEED_CLEANUP; 4117 - if (ret >= 0) 4118 - ret += sr->done_io; 4119 - else if (sr->done_io) 4120 - ret = sr->done_io; 4121 - io_req_set_res(req, ret, 0); 4122 - return IOU_OK; 4123 - } 4124 - 4125 - static int io_send(struct io_kiocb *req, unsigned int issue_flags) 4126 - { 4127 - struct io_sr_msg *sr = io_kiocb_to_cmd(req); 4128 - struct msghdr msg; 4129 - struct iovec iov; 4130 - struct socket *sock; 4131 - unsigned flags; 4132 - int min_ret = 0; 4133 - int ret; 4134 - 4135 - if (!(req->flags & REQ_F_POLLED) && 4136 - (sr->flags & IORING_RECVSEND_POLL_FIRST)) 4137 - return -EAGAIN; 4138 - 4139 - sock = sock_from_file(req->file); 4140 - if (unlikely(!sock)) 4141 - return -ENOTSOCK; 4142 - 4143 - ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter); 4144 - if (unlikely(ret)) 4145 - return ret; 4146 - 4147 - msg.msg_name = NULL; 4148 - msg.msg_control = NULL; 4149 - msg.msg_controllen = 0; 4150 - msg.msg_namelen = 0; 4151 - 4152 - flags = sr->msg_flags; 4153 - if (issue_flags & IO_URING_F_NONBLOCK) 4154 - flags |= MSG_DONTWAIT; 4155 - if (flags & MSG_WAITALL) 4156 - min_ret = iov_iter_count(&msg.msg_iter); 4157 - 4158 - msg.msg_flags = flags; 4159 - ret = sock_sendmsg(sock, &msg); 4160 - if (ret < min_ret) { 4161 - if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 4162 - return -EAGAIN; 4163 - if (ret == -ERESTARTSYS) 4164 - ret = -EINTR; 4165 - if (ret > 0 && io_net_retry(sock, flags)) { 4166 - sr->len -= ret; 4167 - sr->buf += ret; 4168 - sr->done_io += ret; 4169 - req->flags |= REQ_F_PARTIAL_IO; 4170 - return -EAGAIN; 4171 - } 4172 - req_set_fail(req); 4173 - } 4174 - if (ret >= 0) 4175 - ret += sr->done_io; 4176 - else if (sr->done_io) 4177 - ret = sr->done_io; 4178 - io_req_set_res(req, ret, 0); 4179 - return IOU_OK; 4180 - } 4181 - 4182 - static int __io_recvmsg_copy_hdr(struct io_kiocb *req, 4183 - struct io_async_msghdr *iomsg) 4184 - { 4185 - struct io_sr_msg *sr = io_kiocb_to_cmd(req); 4186 - struct iovec __user *uiov; 4187 - size_t iov_len; 4188 - int ret; 4189 - 4190 - ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg, 4191 - &iomsg->uaddr, &uiov, &iov_len); 4192 - if (ret) 4193 - return ret; 4194 - 4195 - if (req->flags & REQ_F_BUFFER_SELECT) { 4196 - if (iov_len > 1) 4197 - return -EINVAL; 4198 - if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov))) 4199 - return -EFAULT; 4200 - sr->len = iomsg->fast_iov[0].iov_len; 4201 - iomsg->free_iov = NULL; 4202 - } else { 4203 - iomsg->free_iov = iomsg->fast_iov; 4204 - ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV, 4205 - &iomsg->free_iov, &iomsg->msg.msg_iter, 4206 - false); 4207 - if (ret > 0) 4208 - ret = 0; 4209 - } 4210 - 4211 - return ret; 4212 - } 4213 - 4214 - #ifdef CONFIG_COMPAT 4215 - static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, 4216 - struct io_async_msghdr *iomsg) 4217 - { 4218 - struct io_sr_msg *sr = io_kiocb_to_cmd(req); 4219 - struct compat_iovec __user *uiov; 4220 - compat_uptr_t ptr; 4221 - compat_size_t len; 4222 - int ret; 4223 - 4224 - ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr, 4225 - &ptr, &len); 4226 - if (ret) 4227 - return ret; 4228 - 4229 - uiov = compat_ptr(ptr); 4230 - if (req->flags & REQ_F_BUFFER_SELECT) { 4231 - compat_ssize_t clen; 4232 - 4233 - if (len > 1) 4234 - return -EINVAL; 4235 - if (!access_ok(uiov, sizeof(*uiov))) 4236 - return -EFAULT; 4237 - if (__get_user(clen, &uiov->iov_len)) 4238 - return -EFAULT; 4239 - if (clen < 0) 4240 - return -EINVAL; 4241 - sr->len = clen; 4242 - iomsg->free_iov = NULL; 4243 - } else { 4244 - iomsg->free_iov = iomsg->fast_iov; 4245 - ret = __import_iovec(READ, (struct iovec __user *)uiov, len, 4246 - UIO_FASTIOV, &iomsg->free_iov, 4247 - &iomsg->msg.msg_iter, true); 4248 - if (ret < 0) 4249 - return ret; 4250 - } 4251 - 4252 - return 0; 4253 - } 4254 - #endif 4255 - 4256 - static int io_recvmsg_copy_hdr(struct io_kiocb *req, 4257 - struct io_async_msghdr *iomsg) 4258 - { 4259 - iomsg->msg.msg_name = &iomsg->addr; 4260 - 4261 - #ifdef CONFIG_COMPAT 4262 - if (req->ctx->compat) 4263 - return __io_compat_recvmsg_copy_hdr(req, iomsg); 4264 - #endif 4265 - 4266 - return __io_recvmsg_copy_hdr(req, iomsg); 4267 - } 4268 - 4269 - static int io_recvmsg_prep_async(struct io_kiocb *req) 4270 - { 4271 - int ret; 4272 - 4273 - ret = io_recvmsg_copy_hdr(req, req->async_data); 4274 - if (!ret) 4275 - req->flags |= REQ_F_NEED_CLEANUP; 4276 - return ret; 4277 - } 4278 - 4279 - static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 4280 - { 4281 - struct io_sr_msg *sr = io_kiocb_to_cmd(req); 4282 - 4283 - if (unlikely(sqe->file_index || sqe->addr2)) 4284 - return -EINVAL; 4285 - 4286 - sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 4287 - sr->len = READ_ONCE(sqe->len); 4288 - sr->flags = READ_ONCE(sqe->ioprio); 4289 - if (sr->flags & ~IORING_RECVSEND_POLL_FIRST) 4290 - return -EINVAL; 4291 - sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 4292 - if (sr->msg_flags & MSG_DONTWAIT) 4293 - req->flags |= REQ_F_NOWAIT; 4294 - if (sr->msg_flags & MSG_ERRQUEUE) 4295 - req->flags |= REQ_F_CLEAR_POLLIN; 4296 - 4297 - #ifdef CONFIG_COMPAT 4298 - if (req->ctx->compat) 4299 - sr->msg_flags |= MSG_CMSG_COMPAT; 4300 - #endif 4301 - sr->done_io = 0; 4302 - return 0; 4303 - } 4304 - 4305 - static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 4306 - { 4307 - struct io_sr_msg *sr = io_kiocb_to_cmd(req); 4308 - struct io_async_msghdr iomsg, *kmsg; 4309 - struct socket *sock; 4310 - unsigned int cflags; 4311 - unsigned flags; 4312 - int ret, min_ret = 0; 4313 - bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 4314 - 4315 - sock = sock_from_file(req->file); 4316 - if (unlikely(!sock)) 4317 - return -ENOTSOCK; 4318 - 4319 - if (req_has_async_data(req)) { 4320 - kmsg = req->async_data; 4321 - } else { 4322 - ret = io_recvmsg_copy_hdr(req, &iomsg); 4323 - if (ret) 4324 - return ret; 4325 - kmsg = &iomsg; 4326 - } 4327 - 4328 - if (!(req->flags & REQ_F_POLLED) && 4329 - (sr->flags & IORING_RECVSEND_POLL_FIRST)) 4330 - return io_setup_async_msg(req, kmsg); 4331 - 4332 - if (io_do_buffer_select(req)) { 4333 - void __user *buf; 4334 - 4335 - buf = io_buffer_select(req, &sr->len, issue_flags); 4336 - if (!buf) 4337 - return -ENOBUFS; 4338 - kmsg->fast_iov[0].iov_base = buf; 4339 - kmsg->fast_iov[0].iov_len = sr->len; 4340 - iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1, 4341 - sr->len); 4342 - } 4343 - 4344 - flags = sr->msg_flags; 4345 - if (force_nonblock) 4346 - flags |= MSG_DONTWAIT; 4347 - if (flags & MSG_WAITALL) 4348 - min_ret = iov_iter_count(&kmsg->msg.msg_iter); 4349 - 4350 - kmsg->msg.msg_get_inq = 1; 4351 - ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags); 4352 - if (ret < min_ret) { 4353 - if (ret == -EAGAIN && force_nonblock) 4354 - return io_setup_async_msg(req, kmsg); 4355 - if (ret == -ERESTARTSYS) 4356 - ret = -EINTR; 4357 - if (ret > 0 && io_net_retry(sock, flags)) { 4358 - sr->done_io += ret; 4359 - req->flags |= REQ_F_PARTIAL_IO; 4360 - return io_setup_async_msg(req, kmsg); 4361 - } 4362 - req_set_fail(req); 4363 - } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 4364 - req_set_fail(req); 4365 - } 4366 - 4367 - /* fast path, check for non-NULL to avoid function call */ 4368 - if (kmsg->free_iov) 4369 - kfree(kmsg->free_iov); 4370 - req->flags &= ~REQ_F_NEED_CLEANUP; 4371 - if (ret >= 0) 4372 - ret += sr->done_io; 4373 - else if (sr->done_io) 4374 - ret = sr->done_io; 4375 - cflags = io_put_kbuf(req, issue_flags); 4376 - if (kmsg->msg.msg_inq) 4377 - cflags |= IORING_CQE_F_SOCK_NONEMPTY; 4378 - io_req_set_res(req, ret, cflags); 4379 - return IOU_OK; 4380 - } 4381 - 4382 - static int io_recv(struct io_kiocb *req, unsigned int issue_flags) 4383 - { 4384 - struct io_sr_msg *sr = io_kiocb_to_cmd(req); 4385 - struct msghdr msg; 4386 - struct socket *sock; 4387 - struct iovec iov; 4388 - unsigned int cflags; 4389 - unsigned flags; 4390 - int ret, min_ret = 0; 4391 - bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 4392 - 4393 - if (!(req->flags & REQ_F_POLLED) && 4394 - (sr->flags & IORING_RECVSEND_POLL_FIRST)) 4395 - return -EAGAIN; 4396 - 4397 - sock = sock_from_file(req->file); 4398 - if (unlikely(!sock)) 4399 - return -ENOTSOCK; 4400 - 4401 - if (io_do_buffer_select(req)) { 4402 - void __user *buf; 4403 - 4404 - buf = io_buffer_select(req, &sr->len, issue_flags); 4405 - if (!buf) 4406 - return -ENOBUFS; 4407 - sr->buf = buf; 4408 - } 4409 - 4410 - ret = import_single_range(READ, sr->buf, sr->len, &iov, &msg.msg_iter); 4411 - if (unlikely(ret)) 4412 - goto out_free; 4413 - 4414 - msg.msg_name = NULL; 4415 - msg.msg_namelen = 0; 4416 - msg.msg_control = NULL; 4417 - msg.msg_get_inq = 1; 4418 - msg.msg_flags = 0; 4419 - msg.msg_controllen = 0; 4420 - msg.msg_iocb = NULL; 4421 - 4422 - flags = sr->msg_flags; 4423 - if (force_nonblock) 4424 - flags |= MSG_DONTWAIT; 4425 - if (flags & MSG_WAITALL) 4426 - min_ret = iov_iter_count(&msg.msg_iter); 4427 - 4428 - ret = sock_recvmsg(sock, &msg, flags); 4429 - if (ret < min_ret) { 4430 - if (ret == -EAGAIN && force_nonblock) 4431 - return -EAGAIN; 4432 - if (ret == -ERESTARTSYS) 4433 - ret = -EINTR; 4434 - if (ret > 0 && io_net_retry(sock, flags)) { 4435 - sr->len -= ret; 4436 - sr->buf += ret; 4437 - sr->done_io += ret; 4438 - req->flags |= REQ_F_PARTIAL_IO; 4439 - return -EAGAIN; 4440 - } 4441 - req_set_fail(req); 4442 - } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 4443 - out_free: 4444 - req_set_fail(req); 4445 - } 4446 - 4447 - if (ret >= 0) 4448 - ret += sr->done_io; 4449 - else if (sr->done_io) 4450 - ret = sr->done_io; 4451 - cflags = io_put_kbuf(req, issue_flags); 4452 - if (msg.msg_inq) 4453 - cflags |= IORING_CQE_F_SOCK_NONEMPTY; 4454 - io_req_set_res(req, ret, cflags); 4455 - return IOU_OK; 4456 - } 4457 - 4458 - static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 4459 - { 4460 - struct io_accept *accept = io_kiocb_to_cmd(req); 4461 - unsigned flags; 4462 - 4463 - if (sqe->len || sqe->buf_index) 4464 - return -EINVAL; 4465 - 4466 - accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 4467 - accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 4468 - accept->flags = READ_ONCE(sqe->accept_flags); 4469 - accept->nofile = rlimit(RLIMIT_NOFILE); 4470 - flags = READ_ONCE(sqe->ioprio); 4471 - if (flags & ~IORING_ACCEPT_MULTISHOT) 4472 - return -EINVAL; 4473 - 4474 - accept->file_slot = READ_ONCE(sqe->file_index); 4475 - if (accept->file_slot) { 4476 - if (accept->flags & SOCK_CLOEXEC) 4477 - return -EINVAL; 4478 - if (flags & IORING_ACCEPT_MULTISHOT && 4479 - accept->file_slot != IORING_FILE_INDEX_ALLOC) 4480 - return -EINVAL; 4481 - } 4482 - if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 4483 - return -EINVAL; 4484 - if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) 4485 - accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; 4486 - if (flags & IORING_ACCEPT_MULTISHOT) 4487 - req->flags |= REQ_F_APOLL_MULTISHOT; 4488 - return 0; 4489 - } 4490 - 4491 - static int io_accept(struct io_kiocb *req, unsigned int issue_flags) 4492 - { 4493 - struct io_ring_ctx *ctx = req->ctx; 4494 - struct io_accept *accept = io_kiocb_to_cmd(req); 4495 - bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 4496 - unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; 4497 - bool fixed = !!accept->file_slot; 4498 - struct file *file; 4499 - int ret, fd; 4500 - 4501 - retry: 4502 - if (!fixed) { 4503 - fd = __get_unused_fd_flags(accept->flags, accept->nofile); 4504 - if (unlikely(fd < 0)) 4505 - return fd; 4506 - } 4507 - file = do_accept(req->file, file_flags, accept->addr, accept->addr_len, 4508 - accept->flags); 4509 - if (IS_ERR(file)) { 4510 - if (!fixed) 4511 - put_unused_fd(fd); 4512 - ret = PTR_ERR(file); 4513 - if (ret == -EAGAIN && force_nonblock) { 4514 - /* 4515 - * if it's multishot and polled, we don't need to 4516 - * return EAGAIN to arm the poll infra since it 4517 - * has already been done 4518 - */ 4519 - if ((req->flags & IO_APOLL_MULTI_POLLED) == 4520 - IO_APOLL_MULTI_POLLED) 4521 - ret = IOU_ISSUE_SKIP_COMPLETE; 4522 - return ret; 4523 - } 4524 - if (ret == -ERESTARTSYS) 4525 - ret = -EINTR; 4526 - req_set_fail(req); 4527 - } else if (!fixed) { 4528 - fd_install(fd, file); 4529 - ret = fd; 4530 - } else { 4531 - ret = io_fixed_fd_install(req, issue_flags, file, 4532 - accept->file_slot); 4533 - } 4534 - 4535 - if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 4536 - io_req_set_res(req, ret, 0); 4537 - return IOU_OK; 4538 - } 4539 - if (ret >= 0) { 4540 - bool filled; 4541 - 4542 - spin_lock(&ctx->completion_lock); 4543 - filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret, 4544 - IORING_CQE_F_MORE); 4545 - io_commit_cqring(ctx); 4546 - spin_unlock(&ctx->completion_lock); 4547 - if (filled) { 4548 - io_cqring_ev_posted(ctx); 4549 - goto retry; 4550 - } 4551 - ret = -ECANCELED; 4552 - } 4553 - 4554 - return ret; 4555 - } 4556 - 4557 - static int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 4558 - { 4559 - struct io_socket *sock = io_kiocb_to_cmd(req); 4560 - 4561 - if (sqe->addr || sqe->rw_flags || sqe->buf_index) 4562 - return -EINVAL; 4563 - 4564 - sock->domain = READ_ONCE(sqe->fd); 4565 - sock->type = READ_ONCE(sqe->off); 4566 - sock->protocol = READ_ONCE(sqe->len); 4567 - sock->file_slot = READ_ONCE(sqe->file_index); 4568 - sock->nofile = rlimit(RLIMIT_NOFILE); 4569 - 4570 - sock->flags = sock->type & ~SOCK_TYPE_MASK; 4571 - if (sock->file_slot && (sock->flags & SOCK_CLOEXEC)) 4572 - return -EINVAL; 4573 - if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 4574 - return -EINVAL; 4575 - return 0; 4576 - } 4577 - 4578 - static int io_socket(struct io_kiocb *req, unsigned int issue_flags) 4579 - { 4580 - struct io_socket *sock = io_kiocb_to_cmd(req); 4581 - bool fixed = !!sock->file_slot; 4582 - struct file *file; 4583 - int ret, fd; 4584 - 4585 - if (!fixed) { 4586 - fd = __get_unused_fd_flags(sock->flags, sock->nofile); 4587 - if (unlikely(fd < 0)) 4588 - return fd; 4589 - } 4590 - file = __sys_socket_file(sock->domain, sock->type, sock->protocol); 4591 - if (IS_ERR(file)) { 4592 - if (!fixed) 4593 - put_unused_fd(fd); 4594 - ret = PTR_ERR(file); 4595 - if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 4596 - return -EAGAIN; 4597 - if (ret == -ERESTARTSYS) 4598 - ret = -EINTR; 4599 - req_set_fail(req); 4600 - } else if (!fixed) { 4601 - fd_install(fd, file); 4602 - ret = fd; 4603 - } else { 4604 - ret = io_fixed_fd_install(req, issue_flags, file, 4605 - sock->file_slot); 4606 - } 4607 - io_req_set_res(req, ret, 0); 4608 - return IOU_OK; 4609 - } 4610 - 4611 - static int io_connect_prep_async(struct io_kiocb *req) 4612 - { 4613 - struct io_async_connect *io = req->async_data; 4614 - struct io_connect *conn = io_kiocb_to_cmd(req); 4615 - 4616 - return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); 4617 - } 4618 - 4619 - static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 4620 - { 4621 - struct io_connect *conn = io_kiocb_to_cmd(req); 4622 - 4623 - if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 4624 - return -EINVAL; 4625 - 4626 - conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 4627 - conn->addr_len = READ_ONCE(sqe->addr2); 4628 - return 0; 4629 - } 4630 - 4631 - static int io_connect(struct io_kiocb *req, unsigned int issue_flags) 4632 - { 4633 - struct io_connect *connect = io_kiocb_to_cmd(req); 4634 - struct io_async_connect __io, *io; 4635 - unsigned file_flags; 4636 - int ret; 4637 - bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 4638 - 4639 - if (req_has_async_data(req)) { 4640 - io = req->async_data; 4641 - } else { 4642 - ret = move_addr_to_kernel(connect->addr, 4643 - connect->addr_len, 4644 - &__io.address); 4645 - if (ret) 4646 - goto out; 4647 - io = &__io; 4648 - } 4649 - 4650 - file_flags = force_nonblock ? O_NONBLOCK : 0; 4651 - 4652 - ret = __sys_connect_file(req->file, &io->address, 4653 - connect->addr_len, file_flags); 4654 - if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) { 4655 - if (req_has_async_data(req)) 4656 - return -EAGAIN; 4657 - if (io_alloc_async_data(req)) { 4658 - ret = -ENOMEM; 4659 - goto out; 4660 - } 4661 - memcpy(req->async_data, &__io, sizeof(__io)); 4662 - return -EAGAIN; 4663 - } 4664 - if (ret == -ERESTARTSYS) 4665 - ret = -EINTR; 4666 - out: 4667 - if (ret < 0) 4668 - req_set_fail(req); 4669 - io_req_set_res(req, ret, 0); 4670 - return IOU_OK; 4671 - } 4672 - #else /* !CONFIG_NET */ 4673 - #define IO_NETOP_FN(op) \ 4674 - static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \ 4675 - { \ 4676 - return -EOPNOTSUPP; \ 4677 - } 4678 - 4679 - #define IO_NETOP_PREP(op) \ 4680 - IO_NETOP_FN(op) \ 4681 - static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \ 4682 - { \ 4683 - return -EOPNOTSUPP; \ 4684 - } \ 4685 - 4686 - #define IO_NETOP_PREP_ASYNC(op) \ 4687 - IO_NETOP_PREP(op) \ 4688 - static int io_##op##_prep_async(struct io_kiocb *req) \ 4689 - { \ 4690 - return -EOPNOTSUPP; \ 4691 - } 4692 - 4693 - IO_NETOP_PREP_ASYNC(sendmsg); 4694 - IO_NETOP_PREP_ASYNC(recvmsg); 4695 - IO_NETOP_PREP_ASYNC(connect); 4696 - IO_NETOP_PREP(accept); 4697 - IO_NETOP_PREP(socket); 4698 - IO_NETOP_PREP(shutdown); 4699 - IO_NETOP_FN(send); 4700 - IO_NETOP_FN(recv); 4701 - #endif /* CONFIG_NET */ 4702 4027 4703 4028 struct io_poll_table { 4704 4029 struct poll_table_struct pt; ··· 7051 7874 return 0; 7052 7875 } 7053 7876 7054 - static int io_install_fixed_file(struct io_kiocb *req, struct file *file, 7055 - unsigned int issue_flags, u32 slot_index) 7877 + int io_install_fixed_file(struct io_kiocb *req, struct file *file, 7878 + unsigned int issue_flags, u32 slot_index) 7056 7879 __must_hold(&req->ctx->uring_lock) 7057 7880 { 7058 7881 struct io_ring_ctx *ctx = req->ctx; ··· 10163 10986 .unbound_nonreg_file = 1, 10164 10987 .pollout = 1, 10165 10988 .ioprio = 1, 10989 + #if defined(CONFIG_NET) 10166 10990 .async_size = sizeof(struct io_async_msghdr), 10167 10991 .prep = io_sendmsg_prep, 10168 10992 .issue = io_sendmsg, 10169 10993 .prep_async = io_sendmsg_prep_async, 10170 - #if defined(CONFIG_NET) 10171 10994 .cleanup = io_sendmsg_recvmsg_cleanup, 10995 + #else 10996 + .prep = io_eopnotsupp_prep, 10172 10997 #endif 10173 10998 }, 10174 10999 [IORING_OP_RECVMSG] = { ··· 10179 11000 .pollin = 1, 10180 11001 .buffer_select = 1, 10181 11002 .ioprio = 1, 11003 + #if defined(CONFIG_NET) 10182 11004 .async_size = sizeof(struct io_async_msghdr), 10183 11005 .prep = io_recvmsg_prep, 10184 11006 .issue = io_recvmsg, 10185 11007 .prep_async = io_recvmsg_prep_async, 10186 - #if defined(CONFIG_NET) 10187 11008 .cleanup = io_sendmsg_recvmsg_cleanup, 11009 + #else 11010 + .prep = io_eopnotsupp_prep, 10188 11011 #endif 10189 11012 }, 10190 11013 [IORING_OP_TIMEOUT] = { ··· 10207 11026 .pollin = 1, 10208 11027 .poll_exclusive = 1, 10209 11028 .ioprio = 1, /* used for flags */ 11029 + #if defined(CONFIG_NET) 10210 11030 .prep = io_accept_prep, 10211 11031 .issue = io_accept, 11032 + #else 11033 + .prep = io_eopnotsupp_prep, 11034 + #endif 10212 11035 }, 10213 11036 [IORING_OP_ASYNC_CANCEL] = { 10214 11037 .audit_skip = 1, ··· 10229 11044 .needs_file = 1, 10230 11045 .unbound_nonreg_file = 1, 10231 11046 .pollout = 1, 11047 + #if defined(CONFIG_NET) 10232 11048 .async_size = sizeof(struct io_async_connect), 10233 11049 .prep = io_connect_prep, 10234 11050 .issue = io_connect, 10235 11051 .prep_async = io_connect_prep_async, 11052 + #else 11053 + .prep = io_eopnotsupp_prep, 11054 + #endif 10236 11055 }, 10237 11056 [IORING_OP_FALLOCATE] = { 10238 11057 .needs_file = 1, ··· 10306 11117 .pollout = 1, 10307 11118 .audit_skip = 1, 10308 11119 .ioprio = 1, 11120 + #if defined(CONFIG_NET) 10309 11121 .prep = io_sendmsg_prep, 10310 11122 .issue = io_send, 11123 + #else 11124 + .prep = io_eopnotsupp_prep, 11125 + #endif 10311 11126 }, 10312 11127 [IORING_OP_RECV] = { 10313 11128 .needs_file = 1, ··· 10320 11127 .buffer_select = 1, 10321 11128 .audit_skip = 1, 10322 11129 .ioprio = 1, 11130 + #if defined(CONFIG_NET) 10323 11131 .prep = io_recvmsg_prep, 10324 11132 .issue = io_recv, 11133 + #else 11134 + .prep = io_eopnotsupp_prep, 11135 + #endif 10325 11136 }, 10326 11137 [IORING_OP_OPENAT2] = { 10327 11138 .prep = io_openat2_prep, ··· 10372 11175 }, 10373 11176 [IORING_OP_SHUTDOWN] = { 10374 11177 .needs_file = 1, 11178 + #if defined(CONFIG_NET) 10375 11179 .prep = io_shutdown_prep, 10376 11180 .issue = io_shutdown, 11181 + #else 11182 + .prep = io_eopnotsupp_prep, 11183 + #endif 10377 11184 }, 10378 11185 [IORING_OP_RENAMEAT] = { 10379 11186 .prep = io_renameat_prep, ··· 10434 11233 }, 10435 11234 [IORING_OP_SOCKET] = { 10436 11235 .audit_skip = 1, 11236 + #if defined(CONFIG_NET) 10437 11237 .prep = io_socket_prep, 10438 11238 .issue = io_socket, 11239 + #else 11240 + .prep = io_eopnotsupp_prep, 11241 + #endif 10439 11242 }, 10440 11243 [IORING_OP_URING_CMD] = { 10441 11244 .needs_file = 1,
+22
io_uring/io_uring.h
··· 58 58 lockdep_assert_held(&ctx->uring_lock); 59 59 } 60 60 61 + static inline void io_commit_cqring(struct io_ring_ctx *ctx) 62 + { 63 + /* order cqe stores with ring update */ 64 + smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); 65 + } 66 + 61 67 void __io_req_complete(struct io_kiocb *req, unsigned issue_flags); 68 + 69 + bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, 70 + u32 cflags); 71 + void io_cqring_ev_posted(struct io_ring_ctx *ctx); 72 + void __user *io_buffer_select(struct io_kiocb *req, size_t *len, 73 + unsigned int issue_flags); 74 + unsigned int io_put_kbuf(struct io_kiocb *req, unsigned issue_flags); 75 + 76 + static inline bool io_do_buffer_select(struct io_kiocb *req) 77 + { 78 + if (!(req->flags & REQ_F_BUFFER_SELECT)) 79 + return false; 80 + return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)); 81 + } 62 82 63 83 struct file *io_file_get_normal(struct io_kiocb *req, int fd); 64 84 struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 65 85 unsigned issue_flags); 66 86 int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags, 67 87 struct file *file, unsigned int file_slot); 88 + int io_install_fixed_file(struct io_kiocb *req, struct file *file, 89 + unsigned int issue_flags, u32 slot_index); 68 90 69 91 int io_rsrc_node_switch_start(struct io_ring_ctx *ctx); 70 92 int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
+779
io_uring/net.c
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + #include <linux/kernel.h> 3 + #include <linux/errno.h> 4 + #include <linux/file.h> 5 + #include <linux/slab.h> 6 + #include <linux/net.h> 7 + #include <linux/compat.h> 8 + #include <net/compat.h> 9 + #include <linux/io_uring.h> 10 + 11 + #include <uapi/linux/io_uring.h> 12 + 13 + #include "io_uring_types.h" 14 + #include "io_uring.h" 15 + #include "net.h" 16 + 17 + #if defined(CONFIG_NET) 18 + struct io_shutdown { 19 + struct file *file; 20 + int how; 21 + }; 22 + 23 + struct io_accept { 24 + struct file *file; 25 + struct sockaddr __user *addr; 26 + int __user *addr_len; 27 + int flags; 28 + u32 file_slot; 29 + unsigned long nofile; 30 + }; 31 + 32 + struct io_socket { 33 + struct file *file; 34 + int domain; 35 + int type; 36 + int protocol; 37 + int flags; 38 + u32 file_slot; 39 + unsigned long nofile; 40 + }; 41 + 42 + struct io_connect { 43 + struct file *file; 44 + struct sockaddr __user *addr; 45 + int addr_len; 46 + }; 47 + 48 + struct io_sr_msg { 49 + struct file *file; 50 + union { 51 + struct compat_msghdr __user *umsg_compat; 52 + struct user_msghdr __user *umsg; 53 + void __user *buf; 54 + }; 55 + int msg_flags; 56 + size_t len; 57 + size_t done_io; 58 + unsigned int flags; 59 + }; 60 + 61 + #define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED) 62 + 63 + int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 64 + { 65 + struct io_shutdown *shutdown = io_kiocb_to_cmd(req); 66 + 67 + if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || 68 + sqe->buf_index || sqe->splice_fd_in)) 69 + return -EINVAL; 70 + 71 + shutdown->how = READ_ONCE(sqe->len); 72 + return 0; 73 + } 74 + 75 + int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) 76 + { 77 + struct io_shutdown *shutdown = io_kiocb_to_cmd(req); 78 + struct socket *sock; 79 + int ret; 80 + 81 + if (issue_flags & IO_URING_F_NONBLOCK) 82 + return -EAGAIN; 83 + 84 + sock = sock_from_file(req->file); 85 + if (unlikely(!sock)) 86 + return -ENOTSOCK; 87 + 88 + ret = __sys_shutdown_sock(sock, shutdown->how); 89 + io_req_set_res(req, ret, 0); 90 + return IOU_OK; 91 + } 92 + 93 + static bool io_net_retry(struct socket *sock, int flags) 94 + { 95 + if (!(flags & MSG_WAITALL)) 96 + return false; 97 + return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; 98 + } 99 + 100 + static int io_setup_async_msg(struct io_kiocb *req, 101 + struct io_async_msghdr *kmsg) 102 + { 103 + struct io_async_msghdr *async_msg = req->async_data; 104 + 105 + if (async_msg) 106 + return -EAGAIN; 107 + if (io_alloc_async_data(req)) { 108 + kfree(kmsg->free_iov); 109 + return -ENOMEM; 110 + } 111 + async_msg = req->async_data; 112 + req->flags |= REQ_F_NEED_CLEANUP; 113 + memcpy(async_msg, kmsg, sizeof(*kmsg)); 114 + async_msg->msg.msg_name = &async_msg->addr; 115 + /* if were using fast_iov, set it to the new one */ 116 + if (!async_msg->free_iov) 117 + async_msg->msg.msg_iter.iov = async_msg->fast_iov; 118 + 119 + return -EAGAIN; 120 + } 121 + 122 + static int io_sendmsg_copy_hdr(struct io_kiocb *req, 123 + struct io_async_msghdr *iomsg) 124 + { 125 + struct io_sr_msg *sr = io_kiocb_to_cmd(req); 126 + 127 + iomsg->msg.msg_name = &iomsg->addr; 128 + iomsg->free_iov = iomsg->fast_iov; 129 + return sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags, 130 + &iomsg->free_iov); 131 + } 132 + 133 + int io_sendmsg_prep_async(struct io_kiocb *req) 134 + { 135 + int ret; 136 + 137 + ret = io_sendmsg_copy_hdr(req, req->async_data); 138 + if (!ret) 139 + req->flags |= REQ_F_NEED_CLEANUP; 140 + return ret; 141 + } 142 + 143 + void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) 144 + { 145 + struct io_async_msghdr *io = req->async_data; 146 + 147 + kfree(io->free_iov); 148 + } 149 + 150 + int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 151 + { 152 + struct io_sr_msg *sr = io_kiocb_to_cmd(req); 153 + 154 + if (unlikely(sqe->file_index || sqe->addr2)) 155 + return -EINVAL; 156 + 157 + sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 158 + sr->len = READ_ONCE(sqe->len); 159 + sr->flags = READ_ONCE(sqe->ioprio); 160 + if (sr->flags & ~IORING_RECVSEND_POLL_FIRST) 161 + return -EINVAL; 162 + sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 163 + if (sr->msg_flags & MSG_DONTWAIT) 164 + req->flags |= REQ_F_NOWAIT; 165 + 166 + #ifdef CONFIG_COMPAT 167 + if (req->ctx->compat) 168 + sr->msg_flags |= MSG_CMSG_COMPAT; 169 + #endif 170 + sr->done_io = 0; 171 + return 0; 172 + } 173 + 174 + int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) 175 + { 176 + struct io_sr_msg *sr = io_kiocb_to_cmd(req); 177 + struct io_async_msghdr iomsg, *kmsg; 178 + struct socket *sock; 179 + unsigned flags; 180 + int min_ret = 0; 181 + int ret; 182 + 183 + sock = sock_from_file(req->file); 184 + if (unlikely(!sock)) 185 + return -ENOTSOCK; 186 + 187 + if (req_has_async_data(req)) { 188 + kmsg = req->async_data; 189 + } else { 190 + ret = io_sendmsg_copy_hdr(req, &iomsg); 191 + if (ret) 192 + return ret; 193 + kmsg = &iomsg; 194 + } 195 + 196 + if (!(req->flags & REQ_F_POLLED) && 197 + (sr->flags & IORING_RECVSEND_POLL_FIRST)) 198 + return io_setup_async_msg(req, kmsg); 199 + 200 + flags = sr->msg_flags; 201 + if (issue_flags & IO_URING_F_NONBLOCK) 202 + flags |= MSG_DONTWAIT; 203 + if (flags & MSG_WAITALL) 204 + min_ret = iov_iter_count(&kmsg->msg.msg_iter); 205 + 206 + ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 207 + 208 + if (ret < min_ret) { 209 + if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 210 + return io_setup_async_msg(req, kmsg); 211 + if (ret == -ERESTARTSYS) 212 + ret = -EINTR; 213 + if (ret > 0 && io_net_retry(sock, flags)) { 214 + sr->done_io += ret; 215 + req->flags |= REQ_F_PARTIAL_IO; 216 + return io_setup_async_msg(req, kmsg); 217 + } 218 + req_set_fail(req); 219 + } 220 + /* fast path, check for non-NULL to avoid function call */ 221 + if (kmsg->free_iov) 222 + kfree(kmsg->free_iov); 223 + req->flags &= ~REQ_F_NEED_CLEANUP; 224 + if (ret >= 0) 225 + ret += sr->done_io; 226 + else if (sr->done_io) 227 + ret = sr->done_io; 228 + io_req_set_res(req, ret, 0); 229 + return IOU_OK; 230 + } 231 + 232 + int io_send(struct io_kiocb *req, unsigned int issue_flags) 233 + { 234 + struct io_sr_msg *sr = io_kiocb_to_cmd(req); 235 + struct msghdr msg; 236 + struct iovec iov; 237 + struct socket *sock; 238 + unsigned flags; 239 + int min_ret = 0; 240 + int ret; 241 + 242 + if (!(req->flags & REQ_F_POLLED) && 243 + (sr->flags & IORING_RECVSEND_POLL_FIRST)) 244 + return -EAGAIN; 245 + 246 + sock = sock_from_file(req->file); 247 + if (unlikely(!sock)) 248 + return -ENOTSOCK; 249 + 250 + ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter); 251 + if (unlikely(ret)) 252 + return ret; 253 + 254 + msg.msg_name = NULL; 255 + msg.msg_control = NULL; 256 + msg.msg_controllen = 0; 257 + msg.msg_namelen = 0; 258 + 259 + flags = sr->msg_flags; 260 + if (issue_flags & IO_URING_F_NONBLOCK) 261 + flags |= MSG_DONTWAIT; 262 + if (flags & MSG_WAITALL) 263 + min_ret = iov_iter_count(&msg.msg_iter); 264 + 265 + msg.msg_flags = flags; 266 + ret = sock_sendmsg(sock, &msg); 267 + if (ret < min_ret) { 268 + if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 269 + return -EAGAIN; 270 + if (ret == -ERESTARTSYS) 271 + ret = -EINTR; 272 + if (ret > 0 && io_net_retry(sock, flags)) { 273 + sr->len -= ret; 274 + sr->buf += ret; 275 + sr->done_io += ret; 276 + req->flags |= REQ_F_PARTIAL_IO; 277 + return -EAGAIN; 278 + } 279 + req_set_fail(req); 280 + } 281 + if (ret >= 0) 282 + ret += sr->done_io; 283 + else if (sr->done_io) 284 + ret = sr->done_io; 285 + io_req_set_res(req, ret, 0); 286 + return IOU_OK; 287 + } 288 + 289 + static int __io_recvmsg_copy_hdr(struct io_kiocb *req, 290 + struct io_async_msghdr *iomsg) 291 + { 292 + struct io_sr_msg *sr = io_kiocb_to_cmd(req); 293 + struct iovec __user *uiov; 294 + size_t iov_len; 295 + int ret; 296 + 297 + ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg, 298 + &iomsg->uaddr, &uiov, &iov_len); 299 + if (ret) 300 + return ret; 301 + 302 + if (req->flags & REQ_F_BUFFER_SELECT) { 303 + if (iov_len > 1) 304 + return -EINVAL; 305 + if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov))) 306 + return -EFAULT; 307 + sr->len = iomsg->fast_iov[0].iov_len; 308 + iomsg->free_iov = NULL; 309 + } else { 310 + iomsg->free_iov = iomsg->fast_iov; 311 + ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV, 312 + &iomsg->free_iov, &iomsg->msg.msg_iter, 313 + false); 314 + if (ret > 0) 315 + ret = 0; 316 + } 317 + 318 + return ret; 319 + } 320 + 321 + #ifdef CONFIG_COMPAT 322 + static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, 323 + struct io_async_msghdr *iomsg) 324 + { 325 + struct io_sr_msg *sr = io_kiocb_to_cmd(req); 326 + struct compat_iovec __user *uiov; 327 + compat_uptr_t ptr; 328 + compat_size_t len; 329 + int ret; 330 + 331 + ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr, 332 + &ptr, &len); 333 + if (ret) 334 + return ret; 335 + 336 + uiov = compat_ptr(ptr); 337 + if (req->flags & REQ_F_BUFFER_SELECT) { 338 + compat_ssize_t clen; 339 + 340 + if (len > 1) 341 + return -EINVAL; 342 + if (!access_ok(uiov, sizeof(*uiov))) 343 + return -EFAULT; 344 + if (__get_user(clen, &uiov->iov_len)) 345 + return -EFAULT; 346 + if (clen < 0) 347 + return -EINVAL; 348 + sr->len = clen; 349 + iomsg->free_iov = NULL; 350 + } else { 351 + iomsg->free_iov = iomsg->fast_iov; 352 + ret = __import_iovec(READ, (struct iovec __user *)uiov, len, 353 + UIO_FASTIOV, &iomsg->free_iov, 354 + &iomsg->msg.msg_iter, true); 355 + if (ret < 0) 356 + return ret; 357 + } 358 + 359 + return 0; 360 + } 361 + #endif 362 + 363 + static int io_recvmsg_copy_hdr(struct io_kiocb *req, 364 + struct io_async_msghdr *iomsg) 365 + { 366 + iomsg->msg.msg_name = &iomsg->addr; 367 + 368 + #ifdef CONFIG_COMPAT 369 + if (req->ctx->compat) 370 + return __io_compat_recvmsg_copy_hdr(req, iomsg); 371 + #endif 372 + 373 + return __io_recvmsg_copy_hdr(req, iomsg); 374 + } 375 + 376 + int io_recvmsg_prep_async(struct io_kiocb *req) 377 + { 378 + int ret; 379 + 380 + ret = io_recvmsg_copy_hdr(req, req->async_data); 381 + if (!ret) 382 + req->flags |= REQ_F_NEED_CLEANUP; 383 + return ret; 384 + } 385 + 386 + int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 387 + { 388 + struct io_sr_msg *sr = io_kiocb_to_cmd(req); 389 + 390 + if (unlikely(sqe->file_index || sqe->addr2)) 391 + return -EINVAL; 392 + 393 + sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 394 + sr->len = READ_ONCE(sqe->len); 395 + sr->flags = READ_ONCE(sqe->ioprio); 396 + if (sr->flags & ~IORING_RECVSEND_POLL_FIRST) 397 + return -EINVAL; 398 + sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 399 + if (sr->msg_flags & MSG_DONTWAIT) 400 + req->flags |= REQ_F_NOWAIT; 401 + if (sr->msg_flags & MSG_ERRQUEUE) 402 + req->flags |= REQ_F_CLEAR_POLLIN; 403 + 404 + #ifdef CONFIG_COMPAT 405 + if (req->ctx->compat) 406 + sr->msg_flags |= MSG_CMSG_COMPAT; 407 + #endif 408 + sr->done_io = 0; 409 + return 0; 410 + } 411 + 412 + int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 413 + { 414 + struct io_sr_msg *sr = io_kiocb_to_cmd(req); 415 + struct io_async_msghdr iomsg, *kmsg; 416 + struct socket *sock; 417 + unsigned int cflags; 418 + unsigned flags; 419 + int ret, min_ret = 0; 420 + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 421 + 422 + sock = sock_from_file(req->file); 423 + if (unlikely(!sock)) 424 + return -ENOTSOCK; 425 + 426 + if (req_has_async_data(req)) { 427 + kmsg = req->async_data; 428 + } else { 429 + ret = io_recvmsg_copy_hdr(req, &iomsg); 430 + if (ret) 431 + return ret; 432 + kmsg = &iomsg; 433 + } 434 + 435 + if (!(req->flags & REQ_F_POLLED) && 436 + (sr->flags & IORING_RECVSEND_POLL_FIRST)) 437 + return io_setup_async_msg(req, kmsg); 438 + 439 + if (io_do_buffer_select(req)) { 440 + void __user *buf; 441 + 442 + buf = io_buffer_select(req, &sr->len, issue_flags); 443 + if (!buf) 444 + return -ENOBUFS; 445 + kmsg->fast_iov[0].iov_base = buf; 446 + kmsg->fast_iov[0].iov_len = sr->len; 447 + iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1, 448 + sr->len); 449 + } 450 + 451 + flags = sr->msg_flags; 452 + if (force_nonblock) 453 + flags |= MSG_DONTWAIT; 454 + if (flags & MSG_WAITALL) 455 + min_ret = iov_iter_count(&kmsg->msg.msg_iter); 456 + 457 + kmsg->msg.msg_get_inq = 1; 458 + ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags); 459 + if (ret < min_ret) { 460 + if (ret == -EAGAIN && force_nonblock) 461 + return io_setup_async_msg(req, kmsg); 462 + if (ret == -ERESTARTSYS) 463 + ret = -EINTR; 464 + if (ret > 0 && io_net_retry(sock, flags)) { 465 + sr->done_io += ret; 466 + req->flags |= REQ_F_PARTIAL_IO; 467 + return io_setup_async_msg(req, kmsg); 468 + } 469 + req_set_fail(req); 470 + } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 471 + req_set_fail(req); 472 + } 473 + 474 + /* fast path, check for non-NULL to avoid function call */ 475 + if (kmsg->free_iov) 476 + kfree(kmsg->free_iov); 477 + req->flags &= ~REQ_F_NEED_CLEANUP; 478 + if (ret >= 0) 479 + ret += sr->done_io; 480 + else if (sr->done_io) 481 + ret = sr->done_io; 482 + cflags = io_put_kbuf(req, issue_flags); 483 + if (kmsg->msg.msg_inq) 484 + cflags |= IORING_CQE_F_SOCK_NONEMPTY; 485 + io_req_set_res(req, ret, cflags); 486 + return IOU_OK; 487 + } 488 + 489 + int io_recv(struct io_kiocb *req, unsigned int issue_flags) 490 + { 491 + struct io_sr_msg *sr = io_kiocb_to_cmd(req); 492 + struct msghdr msg; 493 + struct socket *sock; 494 + struct iovec iov; 495 + unsigned int cflags; 496 + unsigned flags; 497 + int ret, min_ret = 0; 498 + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 499 + 500 + if (!(req->flags & REQ_F_POLLED) && 501 + (sr->flags & IORING_RECVSEND_POLL_FIRST)) 502 + return -EAGAIN; 503 + 504 + sock = sock_from_file(req->file); 505 + if (unlikely(!sock)) 506 + return -ENOTSOCK; 507 + 508 + if (io_do_buffer_select(req)) { 509 + void __user *buf; 510 + 511 + buf = io_buffer_select(req, &sr->len, issue_flags); 512 + if (!buf) 513 + return -ENOBUFS; 514 + sr->buf = buf; 515 + } 516 + 517 + ret = import_single_range(READ, sr->buf, sr->len, &iov, &msg.msg_iter); 518 + if (unlikely(ret)) 519 + goto out_free; 520 + 521 + msg.msg_name = NULL; 522 + msg.msg_namelen = 0; 523 + msg.msg_control = NULL; 524 + msg.msg_get_inq = 1; 525 + msg.msg_flags = 0; 526 + msg.msg_controllen = 0; 527 + msg.msg_iocb = NULL; 528 + 529 + flags = sr->msg_flags; 530 + if (force_nonblock) 531 + flags |= MSG_DONTWAIT; 532 + if (flags & MSG_WAITALL) 533 + min_ret = iov_iter_count(&msg.msg_iter); 534 + 535 + ret = sock_recvmsg(sock, &msg, flags); 536 + if (ret < min_ret) { 537 + if (ret == -EAGAIN && force_nonblock) 538 + return -EAGAIN; 539 + if (ret == -ERESTARTSYS) 540 + ret = -EINTR; 541 + if (ret > 0 && io_net_retry(sock, flags)) { 542 + sr->len -= ret; 543 + sr->buf += ret; 544 + sr->done_io += ret; 545 + req->flags |= REQ_F_PARTIAL_IO; 546 + return -EAGAIN; 547 + } 548 + req_set_fail(req); 549 + } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 550 + out_free: 551 + req_set_fail(req); 552 + } 553 + 554 + if (ret >= 0) 555 + ret += sr->done_io; 556 + else if (sr->done_io) 557 + ret = sr->done_io; 558 + cflags = io_put_kbuf(req, issue_flags); 559 + if (msg.msg_inq) 560 + cflags |= IORING_CQE_F_SOCK_NONEMPTY; 561 + io_req_set_res(req, ret, cflags); 562 + return IOU_OK; 563 + } 564 + 565 + int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 566 + { 567 + struct io_accept *accept = io_kiocb_to_cmd(req); 568 + unsigned flags; 569 + 570 + if (sqe->len || sqe->buf_index) 571 + return -EINVAL; 572 + 573 + accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 574 + accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 575 + accept->flags = READ_ONCE(sqe->accept_flags); 576 + accept->nofile = rlimit(RLIMIT_NOFILE); 577 + flags = READ_ONCE(sqe->ioprio); 578 + if (flags & ~IORING_ACCEPT_MULTISHOT) 579 + return -EINVAL; 580 + 581 + accept->file_slot = READ_ONCE(sqe->file_index); 582 + if (accept->file_slot) { 583 + if (accept->flags & SOCK_CLOEXEC) 584 + return -EINVAL; 585 + if (flags & IORING_ACCEPT_MULTISHOT && 586 + accept->file_slot != IORING_FILE_INDEX_ALLOC) 587 + return -EINVAL; 588 + } 589 + if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 590 + return -EINVAL; 591 + if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) 592 + accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; 593 + if (flags & IORING_ACCEPT_MULTISHOT) 594 + req->flags |= REQ_F_APOLL_MULTISHOT; 595 + return 0; 596 + } 597 + 598 + int io_accept(struct io_kiocb *req, unsigned int issue_flags) 599 + { 600 + struct io_ring_ctx *ctx = req->ctx; 601 + struct io_accept *accept = io_kiocb_to_cmd(req); 602 + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 603 + unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; 604 + bool fixed = !!accept->file_slot; 605 + struct file *file; 606 + int ret, fd; 607 + 608 + retry: 609 + if (!fixed) { 610 + fd = __get_unused_fd_flags(accept->flags, accept->nofile); 611 + if (unlikely(fd < 0)) 612 + return fd; 613 + } 614 + file = do_accept(req->file, file_flags, accept->addr, accept->addr_len, 615 + accept->flags); 616 + if (IS_ERR(file)) { 617 + if (!fixed) 618 + put_unused_fd(fd); 619 + ret = PTR_ERR(file); 620 + if (ret == -EAGAIN && force_nonblock) { 621 + /* 622 + * if it's multishot and polled, we don't need to 623 + * return EAGAIN to arm the poll infra since it 624 + * has already been done 625 + */ 626 + if ((req->flags & IO_APOLL_MULTI_POLLED) == 627 + IO_APOLL_MULTI_POLLED) 628 + ret = IOU_ISSUE_SKIP_COMPLETE; 629 + return ret; 630 + } 631 + if (ret == -ERESTARTSYS) 632 + ret = -EINTR; 633 + req_set_fail(req); 634 + } else if (!fixed) { 635 + fd_install(fd, file); 636 + ret = fd; 637 + } else { 638 + ret = io_fixed_fd_install(req, issue_flags, file, 639 + accept->file_slot); 640 + } 641 + 642 + if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 643 + io_req_set_res(req, ret, 0); 644 + return IOU_OK; 645 + } 646 + if (ret >= 0) { 647 + bool filled; 648 + 649 + spin_lock(&ctx->completion_lock); 650 + filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret, 651 + IORING_CQE_F_MORE); 652 + io_commit_cqring(ctx); 653 + spin_unlock(&ctx->completion_lock); 654 + if (filled) { 655 + io_cqring_ev_posted(ctx); 656 + goto retry; 657 + } 658 + ret = -ECANCELED; 659 + } 660 + 661 + return ret; 662 + } 663 + 664 + int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 665 + { 666 + struct io_socket *sock = io_kiocb_to_cmd(req); 667 + 668 + if (sqe->addr || sqe->rw_flags || sqe->buf_index) 669 + return -EINVAL; 670 + 671 + sock->domain = READ_ONCE(sqe->fd); 672 + sock->type = READ_ONCE(sqe->off); 673 + sock->protocol = READ_ONCE(sqe->len); 674 + sock->file_slot = READ_ONCE(sqe->file_index); 675 + sock->nofile = rlimit(RLIMIT_NOFILE); 676 + 677 + sock->flags = sock->type & ~SOCK_TYPE_MASK; 678 + if (sock->file_slot && (sock->flags & SOCK_CLOEXEC)) 679 + return -EINVAL; 680 + if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 681 + return -EINVAL; 682 + return 0; 683 + } 684 + 685 + int io_socket(struct io_kiocb *req, unsigned int issue_flags) 686 + { 687 + struct io_socket *sock = io_kiocb_to_cmd(req); 688 + bool fixed = !!sock->file_slot; 689 + struct file *file; 690 + int ret, fd; 691 + 692 + if (!fixed) { 693 + fd = __get_unused_fd_flags(sock->flags, sock->nofile); 694 + if (unlikely(fd < 0)) 695 + return fd; 696 + } 697 + file = __sys_socket_file(sock->domain, sock->type, sock->protocol); 698 + if (IS_ERR(file)) { 699 + if (!fixed) 700 + put_unused_fd(fd); 701 + ret = PTR_ERR(file); 702 + if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 703 + return -EAGAIN; 704 + if (ret == -ERESTARTSYS) 705 + ret = -EINTR; 706 + req_set_fail(req); 707 + } else if (!fixed) { 708 + fd_install(fd, file); 709 + ret = fd; 710 + } else { 711 + ret = io_fixed_fd_install(req, issue_flags, file, 712 + sock->file_slot); 713 + } 714 + io_req_set_res(req, ret, 0); 715 + return IOU_OK; 716 + } 717 + 718 + int io_connect_prep_async(struct io_kiocb *req) 719 + { 720 + struct io_async_connect *io = req->async_data; 721 + struct io_connect *conn = io_kiocb_to_cmd(req); 722 + 723 + return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); 724 + } 725 + 726 + int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 727 + { 728 + struct io_connect *conn = io_kiocb_to_cmd(req); 729 + 730 + if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 731 + return -EINVAL; 732 + 733 + conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 734 + conn->addr_len = READ_ONCE(sqe->addr2); 735 + return 0; 736 + } 737 + 738 + int io_connect(struct io_kiocb *req, unsigned int issue_flags) 739 + { 740 + struct io_connect *connect = io_kiocb_to_cmd(req); 741 + struct io_async_connect __io, *io; 742 + unsigned file_flags; 743 + int ret; 744 + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 745 + 746 + if (req_has_async_data(req)) { 747 + io = req->async_data; 748 + } else { 749 + ret = move_addr_to_kernel(connect->addr, 750 + connect->addr_len, 751 + &__io.address); 752 + if (ret) 753 + goto out; 754 + io = &__io; 755 + } 756 + 757 + file_flags = force_nonblock ? O_NONBLOCK : 0; 758 + 759 + ret = __sys_connect_file(req->file, &io->address, 760 + connect->addr_len, file_flags); 761 + if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) { 762 + if (req_has_async_data(req)) 763 + return -EAGAIN; 764 + if (io_alloc_async_data(req)) { 765 + ret = -ENOMEM; 766 + goto out; 767 + } 768 + memcpy(req->async_data, &__io, sizeof(__io)); 769 + return -EAGAIN; 770 + } 771 + if (ret == -ERESTARTSYS) 772 + ret = -EINTR; 773 + out: 774 + if (ret < 0) 775 + req_set_fail(req); 776 + io_req_set_res(req, ret, 0); 777 + return IOU_OK; 778 + } 779 + #endif
+43
io_uring/net.h
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + #include <linux/net.h> 4 + #include <linux/uio.h> 5 + 6 + #if defined(CONFIG_NET) 7 + struct io_async_msghdr { 8 + struct iovec fast_iov[UIO_FASTIOV]; 9 + /* points to an allocated iov, if NULL we use fast_iov instead */ 10 + struct iovec *free_iov; 11 + struct sockaddr __user *uaddr; 12 + struct msghdr msg; 13 + struct sockaddr_storage addr; 14 + }; 15 + 16 + struct io_async_connect { 17 + struct sockaddr_storage address; 18 + }; 19 + 20 + int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 21 + int io_shutdown(struct io_kiocb *req, unsigned int issue_flags); 22 + 23 + int io_sendmsg_prep_async(struct io_kiocb *req); 24 + void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req); 25 + int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 26 + int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags); 27 + int io_send(struct io_kiocb *req, unsigned int issue_flags); 28 + 29 + int io_recvmsg_prep_async(struct io_kiocb *req); 30 + int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 31 + int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags); 32 + int io_recv(struct io_kiocb *req, unsigned int issue_flags); 33 + 34 + int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 35 + int io_accept(struct io_kiocb *req, unsigned int issue_flags); 36 + 37 + int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 38 + int io_socket(struct io_kiocb *req, unsigned int issue_flags); 39 + 40 + int io_connect_prep_async(struct io_kiocb *req); 41 + int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 42 + int io_connect(struct io_kiocb *req, unsigned int issue_flags); 43 + #endif