Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring: split network related opcodes into its own file
While at it, convert the handlers to just use io_eopnotsupp_prep()
if CONFIG_NET isn't set.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+884
-835
+1
-1
io_uring/Makefile
+1
-1
io_uring/Makefile
+39
-834
io_uring/io_uring.c
+39
-834
io_uring/io_uring.c
···
102
102
#include "uring_cmd.h"
103
103
#include "epoll.h"
104
104
#include "statx.h"
105
+
#include "net.h"
105
106
106
107
#define IORING_MAX_ENTRIES 32768
107
108
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
···
131
130
132
131
#define IO_REQ_CLEAN_SLOW_FLAGS (REQ_F_REFCOUNT | REQ_F_LINK | REQ_F_HARDLINK |\
133
132
IO_REQ_CLEAN_FLAGS)
134
-
135
-
#define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED)
136
133
137
134
#define IO_TCTX_REFS_CACHE_NR (1U << 10)
138
135
···
294
295
u32 flags;
295
296
};
296
297
297
-
struct io_accept {
298
-
struct file *file;
299
-
struct sockaddr __user *addr;
300
-
int __user *addr_len;
301
-
int flags;
302
-
u32 file_slot;
303
-
unsigned long nofile;
304
-
};
305
-
306
-
struct io_socket {
307
-
struct file *file;
308
-
int domain;
309
-
int type;
310
-
int protocol;
311
-
int flags;
312
-
u32 file_slot;
313
-
unsigned long nofile;
314
-
};
315
-
316
298
struct io_cancel {
317
299
struct file *file;
318
300
u64 addr;
···
330
350
rwf_t flags;
331
351
};
332
352
333
-
struct io_connect {
334
-
struct file *file;
335
-
struct sockaddr __user *addr;
336
-
int addr_len;
337
-
};
338
-
339
-
struct io_sr_msg {
340
-
struct file *file;
341
-
union {
342
-
struct compat_msghdr __user *umsg_compat;
343
-
struct user_msghdr __user *umsg;
344
-
void __user *buf;
345
-
};
346
-
int msg_flags;
347
-
size_t len;
348
-
size_t done_io;
349
-
unsigned int flags;
350
-
};
351
-
352
353
struct io_rsrc_update {
353
354
struct file *file;
354
355
u64 arg;
···
346
385
__u16 bid;
347
386
};
348
387
349
-
struct io_shutdown {
350
-
struct file *file;
351
-
int how;
352
-
};
353
-
354
388
struct io_msg {
355
389
struct file *file;
356
390
u64 user_data;
357
391
u32 len;
358
-
};
359
-
360
-
struct io_async_connect {
361
-
struct sockaddr_storage address;
362
-
};
363
-
364
-
struct io_async_msghdr {
365
-
struct iovec fast_iov[UIO_FASTIOV];
366
-
/* points to an allocated iov, if NULL we use fast_iov instead */
367
-
struct iovec *free_iov;
368
-
struct sockaddr __user *uaddr;
369
-
struct msghdr msg;
370
-
struct sockaddr_storage addr;
371
392
};
372
393
373
394
struct io_rw_state {
···
459
516
static void io_req_task_queue(struct io_kiocb *req);
460
517
static void __io_submit_flush_completions(struct io_ring_ctx *ctx);
461
518
static int io_req_prep_async(struct io_kiocb *req);
462
-
463
-
static int io_install_fixed_file(struct io_kiocb *req, struct file *file,
464
-
unsigned int issue_flags, u32 slot_index);
465
519
466
520
static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer);
467
521
static void io_eventfd_signal(struct io_ring_ctx *ctx);
···
748
808
return __io_put_kbuf(req, &req->ctx->io_buffers_comp);
749
809
}
750
810
751
-
static inline unsigned int io_put_kbuf(struct io_kiocb *req,
752
-
unsigned issue_flags)
811
+
inline unsigned int io_put_kbuf(struct io_kiocb *req, unsigned issue_flags)
753
812
{
754
813
unsigned int cflags;
755
814
···
1230
1291
spin_unlock_irq(&ctx->timeout_lock);
1231
1292
}
1232
1293
1233
-
static inline void io_commit_cqring(struct io_ring_ctx *ctx)
1234
-
{
1235
-
/* order cqe stores with ring update */
1236
-
smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
1237
-
}
1238
-
1239
1294
static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
1240
1295
{
1241
1296
if (ctx->off_timeout_used || ctx->drain_active) {
···
1351
1418
* 1:1 relationship between how many times this function is called (and
1352
1419
* hence the eventfd count) and number of CQEs posted to the CQ ring.
1353
1420
*/
1354
-
static inline void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1421
+
void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1355
1422
{
1356
1423
if (unlikely(ctx->off_timeout_used || ctx->drain_active ||
1357
1424
ctx->has_evfd))
···
1572
1639
}
1573
1640
}
1574
1641
1575
-
static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,
1576
-
s32 res, u32 cflags)
1642
+
bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
1643
+
u32 cflags)
1577
1644
{
1578
1645
struct io_uring_cqe *cqe;
1579
1646
···
2913
2980
return u64_to_user_ptr(buf->addr);
2914
2981
}
2915
2982
2916
-
static void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
2917
-
unsigned int issue_flags)
2983
+
void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
2984
+
unsigned int issue_flags)
2918
2985
{
2919
2986
struct io_ring_ctx *ctx = req->ctx;
2920
2987
struct io_buffer_list *bl;
···
3004
3071
#endif
3005
3072
3006
3073
return __io_iov_buffer_select(req, iov, issue_flags);
3007
-
}
3008
-
3009
-
static inline bool io_do_buffer_select(struct io_kiocb *req)
3010
-
{
3011
-
if (!(req->flags & REQ_F_BUFFER_SELECT))
3012
-
return false;
3013
-
return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING));
3014
3074
}
3015
3075
3016
3076
static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req,
···
3950
4024
{
3951
4025
return -EOPNOTSUPP;
3952
4026
}
3953
-
3954
-
#if defined(CONFIG_NET)
3955
-
static int io_shutdown_prep(struct io_kiocb *req,
3956
-
const struct io_uring_sqe *sqe)
3957
-
{
3958
-
struct io_shutdown *shutdown = io_kiocb_to_cmd(req);
3959
-
3960
-
if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
3961
-
sqe->buf_index || sqe->splice_fd_in))
3962
-
return -EINVAL;
3963
-
3964
-
shutdown->how = READ_ONCE(sqe->len);
3965
-
return 0;
3966
-
}
3967
-
3968
-
static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
3969
-
{
3970
-
struct io_shutdown *shutdown = io_kiocb_to_cmd(req);
3971
-
struct socket *sock;
3972
-
int ret;
3973
-
3974
-
if (issue_flags & IO_URING_F_NONBLOCK)
3975
-
return -EAGAIN;
3976
-
3977
-
sock = sock_from_file(req->file);
3978
-
if (unlikely(!sock))
3979
-
return -ENOTSOCK;
3980
-
3981
-
ret = __sys_shutdown_sock(sock, shutdown->how);
3982
-
io_req_set_res(req, ret, 0);
3983
-
return IOU_OK;
3984
-
}
3985
-
3986
-
static bool io_net_retry(struct socket *sock, int flags)
3987
-
{
3988
-
if (!(flags & MSG_WAITALL))
3989
-
return false;
3990
-
return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
3991
-
}
3992
-
3993
-
static int io_setup_async_msg(struct io_kiocb *req,
3994
-
struct io_async_msghdr *kmsg)
3995
-
{
3996
-
struct io_async_msghdr *async_msg = req->async_data;
3997
-
3998
-
if (async_msg)
3999
-
return -EAGAIN;
4000
-
if (io_alloc_async_data(req)) {
4001
-
kfree(kmsg->free_iov);
4002
-
return -ENOMEM;
4003
-
}
4004
-
async_msg = req->async_data;
4005
-
req->flags |= REQ_F_NEED_CLEANUP;
4006
-
memcpy(async_msg, kmsg, sizeof(*kmsg));
4007
-
async_msg->msg.msg_name = &async_msg->addr;
4008
-
/* if were using fast_iov, set it to the new one */
4009
-
if (!async_msg->free_iov)
4010
-
async_msg->msg.msg_iter.iov = async_msg->fast_iov;
4011
-
4012
-
return -EAGAIN;
4013
-
}
4014
-
4015
-
static int io_sendmsg_copy_hdr(struct io_kiocb *req,
4016
-
struct io_async_msghdr *iomsg)
4017
-
{
4018
-
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
4019
-
4020
-
iomsg->msg.msg_name = &iomsg->addr;
4021
-
iomsg->free_iov = iomsg->fast_iov;
4022
-
return sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags,
4023
-
&iomsg->free_iov);
4024
-
}
4025
-
4026
-
static int io_sendmsg_prep_async(struct io_kiocb *req)
4027
-
{
4028
-
int ret;
4029
-
4030
-
ret = io_sendmsg_copy_hdr(req, req->async_data);
4031
-
if (!ret)
4032
-
req->flags |= REQ_F_NEED_CLEANUP;
4033
-
return ret;
4034
-
}
4035
-
4036
-
static void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
4037
-
{
4038
-
struct io_async_msghdr *io = req->async_data;
4039
-
4040
-
kfree(io->free_iov);
4041
-
}
4042
-
4043
-
static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4044
-
{
4045
-
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
4046
-
4047
-
if (unlikely(sqe->file_index || sqe->addr2))
4048
-
return -EINVAL;
4049
-
4050
-
sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
4051
-
sr->len = READ_ONCE(sqe->len);
4052
-
sr->flags = READ_ONCE(sqe->ioprio);
4053
-
if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
4054
-
return -EINVAL;
4055
-
sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4056
-
if (sr->msg_flags & MSG_DONTWAIT)
4057
-
req->flags |= REQ_F_NOWAIT;
4058
-
4059
-
#ifdef CONFIG_COMPAT
4060
-
if (req->ctx->compat)
4061
-
sr->msg_flags |= MSG_CMSG_COMPAT;
4062
-
#endif
4063
-
sr->done_io = 0;
4064
-
return 0;
4065
-
}
4066
-
4067
-
static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
4068
-
{
4069
-
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
4070
-
struct io_async_msghdr iomsg, *kmsg;
4071
-
struct socket *sock;
4072
-
unsigned flags;
4073
-
int min_ret = 0;
4074
-
int ret;
4075
-
4076
-
sock = sock_from_file(req->file);
4077
-
if (unlikely(!sock))
4078
-
return -ENOTSOCK;
4079
-
4080
-
if (req_has_async_data(req)) {
4081
-
kmsg = req->async_data;
4082
-
} else {
4083
-
ret = io_sendmsg_copy_hdr(req, &iomsg);
4084
-
if (ret)
4085
-
return ret;
4086
-
kmsg = &iomsg;
4087
-
}
4088
-
4089
-
if (!(req->flags & REQ_F_POLLED) &&
4090
-
(sr->flags & IORING_RECVSEND_POLL_FIRST))
4091
-
return io_setup_async_msg(req, kmsg);
4092
-
4093
-
flags = sr->msg_flags;
4094
-
if (issue_flags & IO_URING_F_NONBLOCK)
4095
-
flags |= MSG_DONTWAIT;
4096
-
if (flags & MSG_WAITALL)
4097
-
min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4098
-
4099
-
ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
4100
-
4101
-
if (ret < min_ret) {
4102
-
if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
4103
-
return io_setup_async_msg(req, kmsg);
4104
-
if (ret == -ERESTARTSYS)
4105
-
ret = -EINTR;
4106
-
if (ret > 0 && io_net_retry(sock, flags)) {
4107
-
sr->done_io += ret;
4108
-
req->flags |= REQ_F_PARTIAL_IO;
4109
-
return io_setup_async_msg(req, kmsg);
4110
-
}
4111
-
req_set_fail(req);
4112
-
}
4113
-
/* fast path, check for non-NULL to avoid function call */
4114
-
if (kmsg->free_iov)
4115
-
kfree(kmsg->free_iov);
4116
-
req->flags &= ~REQ_F_NEED_CLEANUP;
4117
-
if (ret >= 0)
4118
-
ret += sr->done_io;
4119
-
else if (sr->done_io)
4120
-
ret = sr->done_io;
4121
-
io_req_set_res(req, ret, 0);
4122
-
return IOU_OK;
4123
-
}
4124
-
4125
-
static int io_send(struct io_kiocb *req, unsigned int issue_flags)
4126
-
{
4127
-
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
4128
-
struct msghdr msg;
4129
-
struct iovec iov;
4130
-
struct socket *sock;
4131
-
unsigned flags;
4132
-
int min_ret = 0;
4133
-
int ret;
4134
-
4135
-
if (!(req->flags & REQ_F_POLLED) &&
4136
-
(sr->flags & IORING_RECVSEND_POLL_FIRST))
4137
-
return -EAGAIN;
4138
-
4139
-
sock = sock_from_file(req->file);
4140
-
if (unlikely(!sock))
4141
-
return -ENOTSOCK;
4142
-
4143
-
ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
4144
-
if (unlikely(ret))
4145
-
return ret;
4146
-
4147
-
msg.msg_name = NULL;
4148
-
msg.msg_control = NULL;
4149
-
msg.msg_controllen = 0;
4150
-
msg.msg_namelen = 0;
4151
-
4152
-
flags = sr->msg_flags;
4153
-
if (issue_flags & IO_URING_F_NONBLOCK)
4154
-
flags |= MSG_DONTWAIT;
4155
-
if (flags & MSG_WAITALL)
4156
-
min_ret = iov_iter_count(&msg.msg_iter);
4157
-
4158
-
msg.msg_flags = flags;
4159
-
ret = sock_sendmsg(sock, &msg);
4160
-
if (ret < min_ret) {
4161
-
if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
4162
-
return -EAGAIN;
4163
-
if (ret == -ERESTARTSYS)
4164
-
ret = -EINTR;
4165
-
if (ret > 0 && io_net_retry(sock, flags)) {
4166
-
sr->len -= ret;
4167
-
sr->buf += ret;
4168
-
sr->done_io += ret;
4169
-
req->flags |= REQ_F_PARTIAL_IO;
4170
-
return -EAGAIN;
4171
-
}
4172
-
req_set_fail(req);
4173
-
}
4174
-
if (ret >= 0)
4175
-
ret += sr->done_io;
4176
-
else if (sr->done_io)
4177
-
ret = sr->done_io;
4178
-
io_req_set_res(req, ret, 0);
4179
-
return IOU_OK;
4180
-
}
4181
-
4182
-
static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
4183
-
struct io_async_msghdr *iomsg)
4184
-
{
4185
-
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
4186
-
struct iovec __user *uiov;
4187
-
size_t iov_len;
4188
-
int ret;
4189
-
4190
-
ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
4191
-
&iomsg->uaddr, &uiov, &iov_len);
4192
-
if (ret)
4193
-
return ret;
4194
-
4195
-
if (req->flags & REQ_F_BUFFER_SELECT) {
4196
-
if (iov_len > 1)
4197
-
return -EINVAL;
4198
-
if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
4199
-
return -EFAULT;
4200
-
sr->len = iomsg->fast_iov[0].iov_len;
4201
-
iomsg->free_iov = NULL;
4202
-
} else {
4203
-
iomsg->free_iov = iomsg->fast_iov;
4204
-
ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
4205
-
&iomsg->free_iov, &iomsg->msg.msg_iter,
4206
-
false);
4207
-
if (ret > 0)
4208
-
ret = 0;
4209
-
}
4210
-
4211
-
return ret;
4212
-
}
4213
-
4214
-
#ifdef CONFIG_COMPAT
4215
-
static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
4216
-
struct io_async_msghdr *iomsg)
4217
-
{
4218
-
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
4219
-
struct compat_iovec __user *uiov;
4220
-
compat_uptr_t ptr;
4221
-
compat_size_t len;
4222
-
int ret;
4223
-
4224
-
ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr,
4225
-
&ptr, &len);
4226
-
if (ret)
4227
-
return ret;
4228
-
4229
-
uiov = compat_ptr(ptr);
4230
-
if (req->flags & REQ_F_BUFFER_SELECT) {
4231
-
compat_ssize_t clen;
4232
-
4233
-
if (len > 1)
4234
-
return -EINVAL;
4235
-
if (!access_ok(uiov, sizeof(*uiov)))
4236
-
return -EFAULT;
4237
-
if (__get_user(clen, &uiov->iov_len))
4238
-
return -EFAULT;
4239
-
if (clen < 0)
4240
-
return -EINVAL;
4241
-
sr->len = clen;
4242
-
iomsg->free_iov = NULL;
4243
-
} else {
4244
-
iomsg->free_iov = iomsg->fast_iov;
4245
-
ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
4246
-
UIO_FASTIOV, &iomsg->free_iov,
4247
-
&iomsg->msg.msg_iter, true);
4248
-
if (ret < 0)
4249
-
return ret;
4250
-
}
4251
-
4252
-
return 0;
4253
-
}
4254
-
#endif
4255
-
4256
-
static int io_recvmsg_copy_hdr(struct io_kiocb *req,
4257
-
struct io_async_msghdr *iomsg)
4258
-
{
4259
-
iomsg->msg.msg_name = &iomsg->addr;
4260
-
4261
-
#ifdef CONFIG_COMPAT
4262
-
if (req->ctx->compat)
4263
-
return __io_compat_recvmsg_copy_hdr(req, iomsg);
4264
-
#endif
4265
-
4266
-
return __io_recvmsg_copy_hdr(req, iomsg);
4267
-
}
4268
-
4269
-
static int io_recvmsg_prep_async(struct io_kiocb *req)
4270
-
{
4271
-
int ret;
4272
-
4273
-
ret = io_recvmsg_copy_hdr(req, req->async_data);
4274
-
if (!ret)
4275
-
req->flags |= REQ_F_NEED_CLEANUP;
4276
-
return ret;
4277
-
}
4278
-
4279
-
static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4280
-
{
4281
-
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
4282
-
4283
-
if (unlikely(sqe->file_index || sqe->addr2))
4284
-
return -EINVAL;
4285
-
4286
-
sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
4287
-
sr->len = READ_ONCE(sqe->len);
4288
-
sr->flags = READ_ONCE(sqe->ioprio);
4289
-
if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
4290
-
return -EINVAL;
4291
-
sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4292
-
if (sr->msg_flags & MSG_DONTWAIT)
4293
-
req->flags |= REQ_F_NOWAIT;
4294
-
if (sr->msg_flags & MSG_ERRQUEUE)
4295
-
req->flags |= REQ_F_CLEAR_POLLIN;
4296
-
4297
-
#ifdef CONFIG_COMPAT
4298
-
if (req->ctx->compat)
4299
-
sr->msg_flags |= MSG_CMSG_COMPAT;
4300
-
#endif
4301
-
sr->done_io = 0;
4302
-
return 0;
4303
-
}
4304
-
4305
-
static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
4306
-
{
4307
-
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
4308
-
struct io_async_msghdr iomsg, *kmsg;
4309
-
struct socket *sock;
4310
-
unsigned int cflags;
4311
-
unsigned flags;
4312
-
int ret, min_ret = 0;
4313
-
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
4314
-
4315
-
sock = sock_from_file(req->file);
4316
-
if (unlikely(!sock))
4317
-
return -ENOTSOCK;
4318
-
4319
-
if (req_has_async_data(req)) {
4320
-
kmsg = req->async_data;
4321
-
} else {
4322
-
ret = io_recvmsg_copy_hdr(req, &iomsg);
4323
-
if (ret)
4324
-
return ret;
4325
-
kmsg = &iomsg;
4326
-
}
4327
-
4328
-
if (!(req->flags & REQ_F_POLLED) &&
4329
-
(sr->flags & IORING_RECVSEND_POLL_FIRST))
4330
-
return io_setup_async_msg(req, kmsg);
4331
-
4332
-
if (io_do_buffer_select(req)) {
4333
-
void __user *buf;
4334
-
4335
-
buf = io_buffer_select(req, &sr->len, issue_flags);
4336
-
if (!buf)
4337
-
return -ENOBUFS;
4338
-
kmsg->fast_iov[0].iov_base = buf;
4339
-
kmsg->fast_iov[0].iov_len = sr->len;
4340
-
iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1,
4341
-
sr->len);
4342
-
}
4343
-
4344
-
flags = sr->msg_flags;
4345
-
if (force_nonblock)
4346
-
flags |= MSG_DONTWAIT;
4347
-
if (flags & MSG_WAITALL)
4348
-
min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4349
-
4350
-
kmsg->msg.msg_get_inq = 1;
4351
-
ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags);
4352
-
if (ret < min_ret) {
4353
-
if (ret == -EAGAIN && force_nonblock)
4354
-
return io_setup_async_msg(req, kmsg);
4355
-
if (ret == -ERESTARTSYS)
4356
-
ret = -EINTR;
4357
-
if (ret > 0 && io_net_retry(sock, flags)) {
4358
-
sr->done_io += ret;
4359
-
req->flags |= REQ_F_PARTIAL_IO;
4360
-
return io_setup_async_msg(req, kmsg);
4361
-
}
4362
-
req_set_fail(req);
4363
-
} else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
4364
-
req_set_fail(req);
4365
-
}
4366
-
4367
-
/* fast path, check for non-NULL to avoid function call */
4368
-
if (kmsg->free_iov)
4369
-
kfree(kmsg->free_iov);
4370
-
req->flags &= ~REQ_F_NEED_CLEANUP;
4371
-
if (ret >= 0)
4372
-
ret += sr->done_io;
4373
-
else if (sr->done_io)
4374
-
ret = sr->done_io;
4375
-
cflags = io_put_kbuf(req, issue_flags);
4376
-
if (kmsg->msg.msg_inq)
4377
-
cflags |= IORING_CQE_F_SOCK_NONEMPTY;
4378
-
io_req_set_res(req, ret, cflags);
4379
-
return IOU_OK;
4380
-
}
4381
-
4382
-
static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
4383
-
{
4384
-
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
4385
-
struct msghdr msg;
4386
-
struct socket *sock;
4387
-
struct iovec iov;
4388
-
unsigned int cflags;
4389
-
unsigned flags;
4390
-
int ret, min_ret = 0;
4391
-
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
4392
-
4393
-
if (!(req->flags & REQ_F_POLLED) &&
4394
-
(sr->flags & IORING_RECVSEND_POLL_FIRST))
4395
-
return -EAGAIN;
4396
-
4397
-
sock = sock_from_file(req->file);
4398
-
if (unlikely(!sock))
4399
-
return -ENOTSOCK;
4400
-
4401
-
if (io_do_buffer_select(req)) {
4402
-
void __user *buf;
4403
-
4404
-
buf = io_buffer_select(req, &sr->len, issue_flags);
4405
-
if (!buf)
4406
-
return -ENOBUFS;
4407
-
sr->buf = buf;
4408
-
}
4409
-
4410
-
ret = import_single_range(READ, sr->buf, sr->len, &iov, &msg.msg_iter);
4411
-
if (unlikely(ret))
4412
-
goto out_free;
4413
-
4414
-
msg.msg_name = NULL;
4415
-
msg.msg_namelen = 0;
4416
-
msg.msg_control = NULL;
4417
-
msg.msg_get_inq = 1;
4418
-
msg.msg_flags = 0;
4419
-
msg.msg_controllen = 0;
4420
-
msg.msg_iocb = NULL;
4421
-
4422
-
flags = sr->msg_flags;
4423
-
if (force_nonblock)
4424
-
flags |= MSG_DONTWAIT;
4425
-
if (flags & MSG_WAITALL)
4426
-
min_ret = iov_iter_count(&msg.msg_iter);
4427
-
4428
-
ret = sock_recvmsg(sock, &msg, flags);
4429
-
if (ret < min_ret) {
4430
-
if (ret == -EAGAIN && force_nonblock)
4431
-
return -EAGAIN;
4432
-
if (ret == -ERESTARTSYS)
4433
-
ret = -EINTR;
4434
-
if (ret > 0 && io_net_retry(sock, flags)) {
4435
-
sr->len -= ret;
4436
-
sr->buf += ret;
4437
-
sr->done_io += ret;
4438
-
req->flags |= REQ_F_PARTIAL_IO;
4439
-
return -EAGAIN;
4440
-
}
4441
-
req_set_fail(req);
4442
-
} else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
4443
-
out_free:
4444
-
req_set_fail(req);
4445
-
}
4446
-
4447
-
if (ret >= 0)
4448
-
ret += sr->done_io;
4449
-
else if (sr->done_io)
4450
-
ret = sr->done_io;
4451
-
cflags = io_put_kbuf(req, issue_flags);
4452
-
if (msg.msg_inq)
4453
-
cflags |= IORING_CQE_F_SOCK_NONEMPTY;
4454
-
io_req_set_res(req, ret, cflags);
4455
-
return IOU_OK;
4456
-
}
4457
-
4458
-
static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4459
-
{
4460
-
struct io_accept *accept = io_kiocb_to_cmd(req);
4461
-
unsigned flags;
4462
-
4463
-
if (sqe->len || sqe->buf_index)
4464
-
return -EINVAL;
4465
-
4466
-
accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4467
-
accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4468
-
accept->flags = READ_ONCE(sqe->accept_flags);
4469
-
accept->nofile = rlimit(RLIMIT_NOFILE);
4470
-
flags = READ_ONCE(sqe->ioprio);
4471
-
if (flags & ~IORING_ACCEPT_MULTISHOT)
4472
-
return -EINVAL;
4473
-
4474
-
accept->file_slot = READ_ONCE(sqe->file_index);
4475
-
if (accept->file_slot) {
4476
-
if (accept->flags & SOCK_CLOEXEC)
4477
-
return -EINVAL;
4478
-
if (flags & IORING_ACCEPT_MULTISHOT &&
4479
-
accept->file_slot != IORING_FILE_INDEX_ALLOC)
4480
-
return -EINVAL;
4481
-
}
4482
-
if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
4483
-
return -EINVAL;
4484
-
if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
4485
-
accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
4486
-
if (flags & IORING_ACCEPT_MULTISHOT)
4487
-
req->flags |= REQ_F_APOLL_MULTISHOT;
4488
-
return 0;
4489
-
}
4490
-
4491
-
static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
4492
-
{
4493
-
struct io_ring_ctx *ctx = req->ctx;
4494
-
struct io_accept *accept = io_kiocb_to_cmd(req);
4495
-
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
4496
-
unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
4497
-
bool fixed = !!accept->file_slot;
4498
-
struct file *file;
4499
-
int ret, fd;
4500
-
4501
-
retry:
4502
-
if (!fixed) {
4503
-
fd = __get_unused_fd_flags(accept->flags, accept->nofile);
4504
-
if (unlikely(fd < 0))
4505
-
return fd;
4506
-
}
4507
-
file = do_accept(req->file, file_flags, accept->addr, accept->addr_len,
4508
-
accept->flags);
4509
-
if (IS_ERR(file)) {
4510
-
if (!fixed)
4511
-
put_unused_fd(fd);
4512
-
ret = PTR_ERR(file);
4513
-
if (ret == -EAGAIN && force_nonblock) {
4514
-
/*
4515
-
* if it's multishot and polled, we don't need to
4516
-
* return EAGAIN to arm the poll infra since it
4517
-
* has already been done
4518
-
*/
4519
-
if ((req->flags & IO_APOLL_MULTI_POLLED) ==
4520
-
IO_APOLL_MULTI_POLLED)
4521
-
ret = IOU_ISSUE_SKIP_COMPLETE;
4522
-
return ret;
4523
-
}
4524
-
if (ret == -ERESTARTSYS)
4525
-
ret = -EINTR;
4526
-
req_set_fail(req);
4527
-
} else if (!fixed) {
4528
-
fd_install(fd, file);
4529
-
ret = fd;
4530
-
} else {
4531
-
ret = io_fixed_fd_install(req, issue_flags, file,
4532
-
accept->file_slot);
4533
-
}
4534
-
4535
-
if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
4536
-
io_req_set_res(req, ret, 0);
4537
-
return IOU_OK;
4538
-
}
4539
-
if (ret >= 0) {
4540
-
bool filled;
4541
-
4542
-
spin_lock(&ctx->completion_lock);
4543
-
filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret,
4544
-
IORING_CQE_F_MORE);
4545
-
io_commit_cqring(ctx);
4546
-
spin_unlock(&ctx->completion_lock);
4547
-
if (filled) {
4548
-
io_cqring_ev_posted(ctx);
4549
-
goto retry;
4550
-
}
4551
-
ret = -ECANCELED;
4552
-
}
4553
-
4554
-
return ret;
4555
-
}
4556
-
4557
-
static int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4558
-
{
4559
-
struct io_socket *sock = io_kiocb_to_cmd(req);
4560
-
4561
-
if (sqe->addr || sqe->rw_flags || sqe->buf_index)
4562
-
return -EINVAL;
4563
-
4564
-
sock->domain = READ_ONCE(sqe->fd);
4565
-
sock->type = READ_ONCE(sqe->off);
4566
-
sock->protocol = READ_ONCE(sqe->len);
4567
-
sock->file_slot = READ_ONCE(sqe->file_index);
4568
-
sock->nofile = rlimit(RLIMIT_NOFILE);
4569
-
4570
-
sock->flags = sock->type & ~SOCK_TYPE_MASK;
4571
-
if (sock->file_slot && (sock->flags & SOCK_CLOEXEC))
4572
-
return -EINVAL;
4573
-
if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
4574
-
return -EINVAL;
4575
-
return 0;
4576
-
}
4577
-
4578
-
static int io_socket(struct io_kiocb *req, unsigned int issue_flags)
4579
-
{
4580
-
struct io_socket *sock = io_kiocb_to_cmd(req);
4581
-
bool fixed = !!sock->file_slot;
4582
-
struct file *file;
4583
-
int ret, fd;
4584
-
4585
-
if (!fixed) {
4586
-
fd = __get_unused_fd_flags(sock->flags, sock->nofile);
4587
-
if (unlikely(fd < 0))
4588
-
return fd;
4589
-
}
4590
-
file = __sys_socket_file(sock->domain, sock->type, sock->protocol);
4591
-
if (IS_ERR(file)) {
4592
-
if (!fixed)
4593
-
put_unused_fd(fd);
4594
-
ret = PTR_ERR(file);
4595
-
if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
4596
-
return -EAGAIN;
4597
-
if (ret == -ERESTARTSYS)
4598
-
ret = -EINTR;
4599
-
req_set_fail(req);
4600
-
} else if (!fixed) {
4601
-
fd_install(fd, file);
4602
-
ret = fd;
4603
-
} else {
4604
-
ret = io_fixed_fd_install(req, issue_flags, file,
4605
-
sock->file_slot);
4606
-
}
4607
-
io_req_set_res(req, ret, 0);
4608
-
return IOU_OK;
4609
-
}
4610
-
4611
-
static int io_connect_prep_async(struct io_kiocb *req)
4612
-
{
4613
-
struct io_async_connect *io = req->async_data;
4614
-
struct io_connect *conn = io_kiocb_to_cmd(req);
4615
-
4616
-
return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
4617
-
}
4618
-
4619
-
static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4620
-
{
4621
-
struct io_connect *conn = io_kiocb_to_cmd(req);
4622
-
4623
-
if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
4624
-
return -EINVAL;
4625
-
4626
-
conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4627
-
conn->addr_len = READ_ONCE(sqe->addr2);
4628
-
return 0;
4629
-
}
4630
-
4631
-
static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
4632
-
{
4633
-
struct io_connect *connect = io_kiocb_to_cmd(req);
4634
-
struct io_async_connect __io, *io;
4635
-
unsigned file_flags;
4636
-
int ret;
4637
-
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
4638
-
4639
-
if (req_has_async_data(req)) {
4640
-
io = req->async_data;
4641
-
} else {
4642
-
ret = move_addr_to_kernel(connect->addr,
4643
-
connect->addr_len,
4644
-
&__io.address);
4645
-
if (ret)
4646
-
goto out;
4647
-
io = &__io;
4648
-
}
4649
-
4650
-
file_flags = force_nonblock ? O_NONBLOCK : 0;
4651
-
4652
-
ret = __sys_connect_file(req->file, &io->address,
4653
-
connect->addr_len, file_flags);
4654
-
if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
4655
-
if (req_has_async_data(req))
4656
-
return -EAGAIN;
4657
-
if (io_alloc_async_data(req)) {
4658
-
ret = -ENOMEM;
4659
-
goto out;
4660
-
}
4661
-
memcpy(req->async_data, &__io, sizeof(__io));
4662
-
return -EAGAIN;
4663
-
}
4664
-
if (ret == -ERESTARTSYS)
4665
-
ret = -EINTR;
4666
-
out:
4667
-
if (ret < 0)
4668
-
req_set_fail(req);
4669
-
io_req_set_res(req, ret, 0);
4670
-
return IOU_OK;
4671
-
}
4672
-
#else /* !CONFIG_NET */
4673
-
#define IO_NETOP_FN(op) \
4674
-
static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \
4675
-
{ \
4676
-
return -EOPNOTSUPP; \
4677
-
}
4678
-
4679
-
#define IO_NETOP_PREP(op) \
4680
-
IO_NETOP_FN(op) \
4681
-
static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \
4682
-
{ \
4683
-
return -EOPNOTSUPP; \
4684
-
} \
4685
-
4686
-
#define IO_NETOP_PREP_ASYNC(op) \
4687
-
IO_NETOP_PREP(op) \
4688
-
static int io_##op##_prep_async(struct io_kiocb *req) \
4689
-
{ \
4690
-
return -EOPNOTSUPP; \
4691
-
}
4692
-
4693
-
IO_NETOP_PREP_ASYNC(sendmsg);
4694
-
IO_NETOP_PREP_ASYNC(recvmsg);
4695
-
IO_NETOP_PREP_ASYNC(connect);
4696
-
IO_NETOP_PREP(accept);
4697
-
IO_NETOP_PREP(socket);
4698
-
IO_NETOP_PREP(shutdown);
4699
-
IO_NETOP_FN(send);
4700
-
IO_NETOP_FN(recv);
4701
-
#endif /* CONFIG_NET */
4702
4027
4703
4028
struct io_poll_table {
4704
4029
struct poll_table_struct pt;
···
7051
7874
return 0;
7052
7875
}
7053
7876
7054
-
static int io_install_fixed_file(struct io_kiocb *req, struct file *file,
7055
-
unsigned int issue_flags, u32 slot_index)
7877
+
int io_install_fixed_file(struct io_kiocb *req, struct file *file,
7878
+
unsigned int issue_flags, u32 slot_index)
7056
7879
__must_hold(&req->ctx->uring_lock)
7057
7880
{
7058
7881
struct io_ring_ctx *ctx = req->ctx;
···
10163
10986
.unbound_nonreg_file = 1,
10164
10987
.pollout = 1,
10165
10988
.ioprio = 1,
10989
+
#if defined(CONFIG_NET)
10166
10990
.async_size = sizeof(struct io_async_msghdr),
10167
10991
.prep = io_sendmsg_prep,
10168
10992
.issue = io_sendmsg,
10169
10993
.prep_async = io_sendmsg_prep_async,
10170
-
#if defined(CONFIG_NET)
10171
10994
.cleanup = io_sendmsg_recvmsg_cleanup,
10995
+
#else
10996
+
.prep = io_eopnotsupp_prep,
10172
10997
#endif
10173
10998
},
10174
10999
[IORING_OP_RECVMSG] = {
···
10179
11000
.pollin = 1,
10180
11001
.buffer_select = 1,
10181
11002
.ioprio = 1,
11003
+
#if defined(CONFIG_NET)
10182
11004
.async_size = sizeof(struct io_async_msghdr),
10183
11005
.prep = io_recvmsg_prep,
10184
11006
.issue = io_recvmsg,
10185
11007
.prep_async = io_recvmsg_prep_async,
10186
-
#if defined(CONFIG_NET)
10187
11008
.cleanup = io_sendmsg_recvmsg_cleanup,
11009
+
#else
11010
+
.prep = io_eopnotsupp_prep,
10188
11011
#endif
10189
11012
},
10190
11013
[IORING_OP_TIMEOUT] = {
···
10207
11026
.pollin = 1,
10208
11027
.poll_exclusive = 1,
10209
11028
.ioprio = 1, /* used for flags */
11029
+
#if defined(CONFIG_NET)
10210
11030
.prep = io_accept_prep,
10211
11031
.issue = io_accept,
11032
+
#else
11033
+
.prep = io_eopnotsupp_prep,
11034
+
#endif
10212
11035
},
10213
11036
[IORING_OP_ASYNC_CANCEL] = {
10214
11037
.audit_skip = 1,
···
10229
11044
.needs_file = 1,
10230
11045
.unbound_nonreg_file = 1,
10231
11046
.pollout = 1,
11047
+
#if defined(CONFIG_NET)
10232
11048
.async_size = sizeof(struct io_async_connect),
10233
11049
.prep = io_connect_prep,
10234
11050
.issue = io_connect,
10235
11051
.prep_async = io_connect_prep_async,
11052
+
#else
11053
+
.prep = io_eopnotsupp_prep,
11054
+
#endif
10236
11055
},
10237
11056
[IORING_OP_FALLOCATE] = {
10238
11057
.needs_file = 1,
···
10306
11117
.pollout = 1,
10307
11118
.audit_skip = 1,
10308
11119
.ioprio = 1,
11120
+
#if defined(CONFIG_NET)
10309
11121
.prep = io_sendmsg_prep,
10310
11122
.issue = io_send,
11123
+
#else
11124
+
.prep = io_eopnotsupp_prep,
11125
+
#endif
10311
11126
},
10312
11127
[IORING_OP_RECV] = {
10313
11128
.needs_file = 1,
···
10320
11127
.buffer_select = 1,
10321
11128
.audit_skip = 1,
10322
11129
.ioprio = 1,
11130
+
#if defined(CONFIG_NET)
10323
11131
.prep = io_recvmsg_prep,
10324
11132
.issue = io_recv,
11133
+
#else
11134
+
.prep = io_eopnotsupp_prep,
11135
+
#endif
10325
11136
},
10326
11137
[IORING_OP_OPENAT2] = {
10327
11138
.prep = io_openat2_prep,
···
10372
11175
},
10373
11176
[IORING_OP_SHUTDOWN] = {
10374
11177
.needs_file = 1,
11178
+
#if defined(CONFIG_NET)
10375
11179
.prep = io_shutdown_prep,
10376
11180
.issue = io_shutdown,
11181
+
#else
11182
+
.prep = io_eopnotsupp_prep,
11183
+
#endif
10377
11184
},
10378
11185
[IORING_OP_RENAMEAT] = {
10379
11186
.prep = io_renameat_prep,
···
10434
11233
},
10435
11234
[IORING_OP_SOCKET] = {
10436
11235
.audit_skip = 1,
11236
+
#if defined(CONFIG_NET)
10437
11237
.prep = io_socket_prep,
10438
11238
.issue = io_socket,
11239
+
#else
11240
+
.prep = io_eopnotsupp_prep,
11241
+
#endif
10439
11242
},
10440
11243
[IORING_OP_URING_CMD] = {
10441
11244
.needs_file = 1,
+22
io_uring/io_uring.h
+22
io_uring/io_uring.h
···
58
58
lockdep_assert_held(&ctx->uring_lock);
59
59
}
60
60
61
+
static inline void io_commit_cqring(struct io_ring_ctx *ctx)
62
+
{
63
+
/* order cqe stores with ring update */
64
+
smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
65
+
}
66
+
61
67
void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
68
+
69
+
bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
70
+
u32 cflags);
71
+
void io_cqring_ev_posted(struct io_ring_ctx *ctx);
72
+
void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
73
+
unsigned int issue_flags);
74
+
unsigned int io_put_kbuf(struct io_kiocb *req, unsigned issue_flags);
75
+
76
+
static inline bool io_do_buffer_select(struct io_kiocb *req)
77
+
{
78
+
if (!(req->flags & REQ_F_BUFFER_SELECT))
79
+
return false;
80
+
return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING));
81
+
}
62
82
63
83
struct file *io_file_get_normal(struct io_kiocb *req, int fd);
64
84
struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
65
85
unsigned issue_flags);
66
86
int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,
67
87
struct file *file, unsigned int file_slot);
88
+
int io_install_fixed_file(struct io_kiocb *req, struct file *file,
89
+
unsigned int issue_flags, u32 slot_index);
68
90
69
91
int io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
70
92
int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
+779
io_uring/net.c
+779
io_uring/net.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
#include <linux/kernel.h>
3
+
#include <linux/errno.h>
4
+
#include <linux/file.h>
5
+
#include <linux/slab.h>
6
+
#include <linux/net.h>
7
+
#include <linux/compat.h>
8
+
#include <net/compat.h>
9
+
#include <linux/io_uring.h>
10
+
11
+
#include <uapi/linux/io_uring.h>
12
+
13
+
#include "io_uring_types.h"
14
+
#include "io_uring.h"
15
+
#include "net.h"
16
+
17
+
#if defined(CONFIG_NET)
18
+
struct io_shutdown {
19
+
struct file *file;
20
+
int how;
21
+
};
22
+
23
+
struct io_accept {
24
+
struct file *file;
25
+
struct sockaddr __user *addr;
26
+
int __user *addr_len;
27
+
int flags;
28
+
u32 file_slot;
29
+
unsigned long nofile;
30
+
};
31
+
32
+
struct io_socket {
33
+
struct file *file;
34
+
int domain;
35
+
int type;
36
+
int protocol;
37
+
int flags;
38
+
u32 file_slot;
39
+
unsigned long nofile;
40
+
};
41
+
42
+
struct io_connect {
43
+
struct file *file;
44
+
struct sockaddr __user *addr;
45
+
int addr_len;
46
+
};
47
+
48
+
struct io_sr_msg {
49
+
struct file *file;
50
+
union {
51
+
struct compat_msghdr __user *umsg_compat;
52
+
struct user_msghdr __user *umsg;
53
+
void __user *buf;
54
+
};
55
+
int msg_flags;
56
+
size_t len;
57
+
size_t done_io;
58
+
unsigned int flags;
59
+
};
60
+
61
+
#define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED)
62
+
63
+
int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
64
+
{
65
+
struct io_shutdown *shutdown = io_kiocb_to_cmd(req);
66
+
67
+
if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
68
+
sqe->buf_index || sqe->splice_fd_in))
69
+
return -EINVAL;
70
+
71
+
shutdown->how = READ_ONCE(sqe->len);
72
+
return 0;
73
+
}
74
+
75
+
int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
76
+
{
77
+
struct io_shutdown *shutdown = io_kiocb_to_cmd(req);
78
+
struct socket *sock;
79
+
int ret;
80
+
81
+
if (issue_flags & IO_URING_F_NONBLOCK)
82
+
return -EAGAIN;
83
+
84
+
sock = sock_from_file(req->file);
85
+
if (unlikely(!sock))
86
+
return -ENOTSOCK;
87
+
88
+
ret = __sys_shutdown_sock(sock, shutdown->how);
89
+
io_req_set_res(req, ret, 0);
90
+
return IOU_OK;
91
+
}
92
+
93
+
static bool io_net_retry(struct socket *sock, int flags)
94
+
{
95
+
if (!(flags & MSG_WAITALL))
96
+
return false;
97
+
return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
98
+
}
99
+
100
+
static int io_setup_async_msg(struct io_kiocb *req,
101
+
struct io_async_msghdr *kmsg)
102
+
{
103
+
struct io_async_msghdr *async_msg = req->async_data;
104
+
105
+
if (async_msg)
106
+
return -EAGAIN;
107
+
if (io_alloc_async_data(req)) {
108
+
kfree(kmsg->free_iov);
109
+
return -ENOMEM;
110
+
}
111
+
async_msg = req->async_data;
112
+
req->flags |= REQ_F_NEED_CLEANUP;
113
+
memcpy(async_msg, kmsg, sizeof(*kmsg));
114
+
async_msg->msg.msg_name = &async_msg->addr;
115
+
/* if were using fast_iov, set it to the new one */
116
+
if (!async_msg->free_iov)
117
+
async_msg->msg.msg_iter.iov = async_msg->fast_iov;
118
+
119
+
return -EAGAIN;
120
+
}
121
+
122
+
static int io_sendmsg_copy_hdr(struct io_kiocb *req,
123
+
struct io_async_msghdr *iomsg)
124
+
{
125
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
126
+
127
+
iomsg->msg.msg_name = &iomsg->addr;
128
+
iomsg->free_iov = iomsg->fast_iov;
129
+
return sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags,
130
+
&iomsg->free_iov);
131
+
}
132
+
133
+
int io_sendmsg_prep_async(struct io_kiocb *req)
134
+
{
135
+
int ret;
136
+
137
+
ret = io_sendmsg_copy_hdr(req, req->async_data);
138
+
if (!ret)
139
+
req->flags |= REQ_F_NEED_CLEANUP;
140
+
return ret;
141
+
}
142
+
143
+
void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
144
+
{
145
+
struct io_async_msghdr *io = req->async_data;
146
+
147
+
kfree(io->free_iov);
148
+
}
149
+
150
+
int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
151
+
{
152
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
153
+
154
+
if (unlikely(sqe->file_index || sqe->addr2))
155
+
return -EINVAL;
156
+
157
+
sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
158
+
sr->len = READ_ONCE(sqe->len);
159
+
sr->flags = READ_ONCE(sqe->ioprio);
160
+
if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
161
+
return -EINVAL;
162
+
sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
163
+
if (sr->msg_flags & MSG_DONTWAIT)
164
+
req->flags |= REQ_F_NOWAIT;
165
+
166
+
#ifdef CONFIG_COMPAT
167
+
if (req->ctx->compat)
168
+
sr->msg_flags |= MSG_CMSG_COMPAT;
169
+
#endif
170
+
sr->done_io = 0;
171
+
return 0;
172
+
}
173
+
174
+
int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
175
+
{
176
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
177
+
struct io_async_msghdr iomsg, *kmsg;
178
+
struct socket *sock;
179
+
unsigned flags;
180
+
int min_ret = 0;
181
+
int ret;
182
+
183
+
sock = sock_from_file(req->file);
184
+
if (unlikely(!sock))
185
+
return -ENOTSOCK;
186
+
187
+
if (req_has_async_data(req)) {
188
+
kmsg = req->async_data;
189
+
} else {
190
+
ret = io_sendmsg_copy_hdr(req, &iomsg);
191
+
if (ret)
192
+
return ret;
193
+
kmsg = &iomsg;
194
+
}
195
+
196
+
if (!(req->flags & REQ_F_POLLED) &&
197
+
(sr->flags & IORING_RECVSEND_POLL_FIRST))
198
+
return io_setup_async_msg(req, kmsg);
199
+
200
+
flags = sr->msg_flags;
201
+
if (issue_flags & IO_URING_F_NONBLOCK)
202
+
flags |= MSG_DONTWAIT;
203
+
if (flags & MSG_WAITALL)
204
+
min_ret = iov_iter_count(&kmsg->msg.msg_iter);
205
+
206
+
ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
207
+
208
+
if (ret < min_ret) {
209
+
if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
210
+
return io_setup_async_msg(req, kmsg);
211
+
if (ret == -ERESTARTSYS)
212
+
ret = -EINTR;
213
+
if (ret > 0 && io_net_retry(sock, flags)) {
214
+
sr->done_io += ret;
215
+
req->flags |= REQ_F_PARTIAL_IO;
216
+
return io_setup_async_msg(req, kmsg);
217
+
}
218
+
req_set_fail(req);
219
+
}
220
+
/* fast path, check for non-NULL to avoid function call */
221
+
if (kmsg->free_iov)
222
+
kfree(kmsg->free_iov);
223
+
req->flags &= ~REQ_F_NEED_CLEANUP;
224
+
if (ret >= 0)
225
+
ret += sr->done_io;
226
+
else if (sr->done_io)
227
+
ret = sr->done_io;
228
+
io_req_set_res(req, ret, 0);
229
+
return IOU_OK;
230
+
}
231
+
232
+
int io_send(struct io_kiocb *req, unsigned int issue_flags)
233
+
{
234
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
235
+
struct msghdr msg;
236
+
struct iovec iov;
237
+
struct socket *sock;
238
+
unsigned flags;
239
+
int min_ret = 0;
240
+
int ret;
241
+
242
+
if (!(req->flags & REQ_F_POLLED) &&
243
+
(sr->flags & IORING_RECVSEND_POLL_FIRST))
244
+
return -EAGAIN;
245
+
246
+
sock = sock_from_file(req->file);
247
+
if (unlikely(!sock))
248
+
return -ENOTSOCK;
249
+
250
+
ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
251
+
if (unlikely(ret))
252
+
return ret;
253
+
254
+
msg.msg_name = NULL;
255
+
msg.msg_control = NULL;
256
+
msg.msg_controllen = 0;
257
+
msg.msg_namelen = 0;
258
+
259
+
flags = sr->msg_flags;
260
+
if (issue_flags & IO_URING_F_NONBLOCK)
261
+
flags |= MSG_DONTWAIT;
262
+
if (flags & MSG_WAITALL)
263
+
min_ret = iov_iter_count(&msg.msg_iter);
264
+
265
+
msg.msg_flags = flags;
266
+
ret = sock_sendmsg(sock, &msg);
267
+
if (ret < min_ret) {
268
+
if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
269
+
return -EAGAIN;
270
+
if (ret == -ERESTARTSYS)
271
+
ret = -EINTR;
272
+
if (ret > 0 && io_net_retry(sock, flags)) {
273
+
sr->len -= ret;
274
+
sr->buf += ret;
275
+
sr->done_io += ret;
276
+
req->flags |= REQ_F_PARTIAL_IO;
277
+
return -EAGAIN;
278
+
}
279
+
req_set_fail(req);
280
+
}
281
+
if (ret >= 0)
282
+
ret += sr->done_io;
283
+
else if (sr->done_io)
284
+
ret = sr->done_io;
285
+
io_req_set_res(req, ret, 0);
286
+
return IOU_OK;
287
+
}
288
+
289
+
static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
290
+
struct io_async_msghdr *iomsg)
291
+
{
292
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
293
+
struct iovec __user *uiov;
294
+
size_t iov_len;
295
+
int ret;
296
+
297
+
ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
298
+
&iomsg->uaddr, &uiov, &iov_len);
299
+
if (ret)
300
+
return ret;
301
+
302
+
if (req->flags & REQ_F_BUFFER_SELECT) {
303
+
if (iov_len > 1)
304
+
return -EINVAL;
305
+
if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
306
+
return -EFAULT;
307
+
sr->len = iomsg->fast_iov[0].iov_len;
308
+
iomsg->free_iov = NULL;
309
+
} else {
310
+
iomsg->free_iov = iomsg->fast_iov;
311
+
ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
312
+
&iomsg->free_iov, &iomsg->msg.msg_iter,
313
+
false);
314
+
if (ret > 0)
315
+
ret = 0;
316
+
}
317
+
318
+
return ret;
319
+
}
320
+
321
+
#ifdef CONFIG_COMPAT
322
+
static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
323
+
struct io_async_msghdr *iomsg)
324
+
{
325
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
326
+
struct compat_iovec __user *uiov;
327
+
compat_uptr_t ptr;
328
+
compat_size_t len;
329
+
int ret;
330
+
331
+
ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr,
332
+
&ptr, &len);
333
+
if (ret)
334
+
return ret;
335
+
336
+
uiov = compat_ptr(ptr);
337
+
if (req->flags & REQ_F_BUFFER_SELECT) {
338
+
compat_ssize_t clen;
339
+
340
+
if (len > 1)
341
+
return -EINVAL;
342
+
if (!access_ok(uiov, sizeof(*uiov)))
343
+
return -EFAULT;
344
+
if (__get_user(clen, &uiov->iov_len))
345
+
return -EFAULT;
346
+
if (clen < 0)
347
+
return -EINVAL;
348
+
sr->len = clen;
349
+
iomsg->free_iov = NULL;
350
+
} else {
351
+
iomsg->free_iov = iomsg->fast_iov;
352
+
ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
353
+
UIO_FASTIOV, &iomsg->free_iov,
354
+
&iomsg->msg.msg_iter, true);
355
+
if (ret < 0)
356
+
return ret;
357
+
}
358
+
359
+
return 0;
360
+
}
361
+
#endif
362
+
363
+
static int io_recvmsg_copy_hdr(struct io_kiocb *req,
364
+
struct io_async_msghdr *iomsg)
365
+
{
366
+
iomsg->msg.msg_name = &iomsg->addr;
367
+
368
+
#ifdef CONFIG_COMPAT
369
+
if (req->ctx->compat)
370
+
return __io_compat_recvmsg_copy_hdr(req, iomsg);
371
+
#endif
372
+
373
+
return __io_recvmsg_copy_hdr(req, iomsg);
374
+
}
375
+
376
+
int io_recvmsg_prep_async(struct io_kiocb *req)
377
+
{
378
+
int ret;
379
+
380
+
ret = io_recvmsg_copy_hdr(req, req->async_data);
381
+
if (!ret)
382
+
req->flags |= REQ_F_NEED_CLEANUP;
383
+
return ret;
384
+
}
385
+
386
+
int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
387
+
{
388
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
389
+
390
+
if (unlikely(sqe->file_index || sqe->addr2))
391
+
return -EINVAL;
392
+
393
+
sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
394
+
sr->len = READ_ONCE(sqe->len);
395
+
sr->flags = READ_ONCE(sqe->ioprio);
396
+
if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
397
+
return -EINVAL;
398
+
sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
399
+
if (sr->msg_flags & MSG_DONTWAIT)
400
+
req->flags |= REQ_F_NOWAIT;
401
+
if (sr->msg_flags & MSG_ERRQUEUE)
402
+
req->flags |= REQ_F_CLEAR_POLLIN;
403
+
404
+
#ifdef CONFIG_COMPAT
405
+
if (req->ctx->compat)
406
+
sr->msg_flags |= MSG_CMSG_COMPAT;
407
+
#endif
408
+
sr->done_io = 0;
409
+
return 0;
410
+
}
411
+
412
+
int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
413
+
{
414
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
415
+
struct io_async_msghdr iomsg, *kmsg;
416
+
struct socket *sock;
417
+
unsigned int cflags;
418
+
unsigned flags;
419
+
int ret, min_ret = 0;
420
+
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
421
+
422
+
sock = sock_from_file(req->file);
423
+
if (unlikely(!sock))
424
+
return -ENOTSOCK;
425
+
426
+
if (req_has_async_data(req)) {
427
+
kmsg = req->async_data;
428
+
} else {
429
+
ret = io_recvmsg_copy_hdr(req, &iomsg);
430
+
if (ret)
431
+
return ret;
432
+
kmsg = &iomsg;
433
+
}
434
+
435
+
if (!(req->flags & REQ_F_POLLED) &&
436
+
(sr->flags & IORING_RECVSEND_POLL_FIRST))
437
+
return io_setup_async_msg(req, kmsg);
438
+
439
+
if (io_do_buffer_select(req)) {
440
+
void __user *buf;
441
+
442
+
buf = io_buffer_select(req, &sr->len, issue_flags);
443
+
if (!buf)
444
+
return -ENOBUFS;
445
+
kmsg->fast_iov[0].iov_base = buf;
446
+
kmsg->fast_iov[0].iov_len = sr->len;
447
+
iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1,
448
+
sr->len);
449
+
}
450
+
451
+
flags = sr->msg_flags;
452
+
if (force_nonblock)
453
+
flags |= MSG_DONTWAIT;
454
+
if (flags & MSG_WAITALL)
455
+
min_ret = iov_iter_count(&kmsg->msg.msg_iter);
456
+
457
+
kmsg->msg.msg_get_inq = 1;
458
+
ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags);
459
+
if (ret < min_ret) {
460
+
if (ret == -EAGAIN && force_nonblock)
461
+
return io_setup_async_msg(req, kmsg);
462
+
if (ret == -ERESTARTSYS)
463
+
ret = -EINTR;
464
+
if (ret > 0 && io_net_retry(sock, flags)) {
465
+
sr->done_io += ret;
466
+
req->flags |= REQ_F_PARTIAL_IO;
467
+
return io_setup_async_msg(req, kmsg);
468
+
}
469
+
req_set_fail(req);
470
+
} else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
471
+
req_set_fail(req);
472
+
}
473
+
474
+
/* fast path, check for non-NULL to avoid function call */
475
+
if (kmsg->free_iov)
476
+
kfree(kmsg->free_iov);
477
+
req->flags &= ~REQ_F_NEED_CLEANUP;
478
+
if (ret >= 0)
479
+
ret += sr->done_io;
480
+
else if (sr->done_io)
481
+
ret = sr->done_io;
482
+
cflags = io_put_kbuf(req, issue_flags);
483
+
if (kmsg->msg.msg_inq)
484
+
cflags |= IORING_CQE_F_SOCK_NONEMPTY;
485
+
io_req_set_res(req, ret, cflags);
486
+
return IOU_OK;
487
+
}
488
+
489
+
int io_recv(struct io_kiocb *req, unsigned int issue_flags)
490
+
{
491
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req);
492
+
struct msghdr msg;
493
+
struct socket *sock;
494
+
struct iovec iov;
495
+
unsigned int cflags;
496
+
unsigned flags;
497
+
int ret, min_ret = 0;
498
+
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
499
+
500
+
if (!(req->flags & REQ_F_POLLED) &&
501
+
(sr->flags & IORING_RECVSEND_POLL_FIRST))
502
+
return -EAGAIN;
503
+
504
+
sock = sock_from_file(req->file);
505
+
if (unlikely(!sock))
506
+
return -ENOTSOCK;
507
+
508
+
if (io_do_buffer_select(req)) {
509
+
void __user *buf;
510
+
511
+
buf = io_buffer_select(req, &sr->len, issue_flags);
512
+
if (!buf)
513
+
return -ENOBUFS;
514
+
sr->buf = buf;
515
+
}
516
+
517
+
ret = import_single_range(READ, sr->buf, sr->len, &iov, &msg.msg_iter);
518
+
if (unlikely(ret))
519
+
goto out_free;
520
+
521
+
msg.msg_name = NULL;
522
+
msg.msg_namelen = 0;
523
+
msg.msg_control = NULL;
524
+
msg.msg_get_inq = 1;
525
+
msg.msg_flags = 0;
526
+
msg.msg_controllen = 0;
527
+
msg.msg_iocb = NULL;
528
+
529
+
flags = sr->msg_flags;
530
+
if (force_nonblock)
531
+
flags |= MSG_DONTWAIT;
532
+
if (flags & MSG_WAITALL)
533
+
min_ret = iov_iter_count(&msg.msg_iter);
534
+
535
+
ret = sock_recvmsg(sock, &msg, flags);
536
+
if (ret < min_ret) {
537
+
if (ret == -EAGAIN && force_nonblock)
538
+
return -EAGAIN;
539
+
if (ret == -ERESTARTSYS)
540
+
ret = -EINTR;
541
+
if (ret > 0 && io_net_retry(sock, flags)) {
542
+
sr->len -= ret;
543
+
sr->buf += ret;
544
+
sr->done_io += ret;
545
+
req->flags |= REQ_F_PARTIAL_IO;
546
+
return -EAGAIN;
547
+
}
548
+
req_set_fail(req);
549
+
} else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
550
+
out_free:
551
+
req_set_fail(req);
552
+
}
553
+
554
+
if (ret >= 0)
555
+
ret += sr->done_io;
556
+
else if (sr->done_io)
557
+
ret = sr->done_io;
558
+
cflags = io_put_kbuf(req, issue_flags);
559
+
if (msg.msg_inq)
560
+
cflags |= IORING_CQE_F_SOCK_NONEMPTY;
561
+
io_req_set_res(req, ret, cflags);
562
+
return IOU_OK;
563
+
}
564
+
565
+
int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
566
+
{
567
+
struct io_accept *accept = io_kiocb_to_cmd(req);
568
+
unsigned flags;
569
+
570
+
if (sqe->len || sqe->buf_index)
571
+
return -EINVAL;
572
+
573
+
accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
574
+
accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
575
+
accept->flags = READ_ONCE(sqe->accept_flags);
576
+
accept->nofile = rlimit(RLIMIT_NOFILE);
577
+
flags = READ_ONCE(sqe->ioprio);
578
+
if (flags & ~IORING_ACCEPT_MULTISHOT)
579
+
return -EINVAL;
580
+
581
+
accept->file_slot = READ_ONCE(sqe->file_index);
582
+
if (accept->file_slot) {
583
+
if (accept->flags & SOCK_CLOEXEC)
584
+
return -EINVAL;
585
+
if (flags & IORING_ACCEPT_MULTISHOT &&
586
+
accept->file_slot != IORING_FILE_INDEX_ALLOC)
587
+
return -EINVAL;
588
+
}
589
+
if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
590
+
return -EINVAL;
591
+
if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
592
+
accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
593
+
if (flags & IORING_ACCEPT_MULTISHOT)
594
+
req->flags |= REQ_F_APOLL_MULTISHOT;
595
+
return 0;
596
+
}
597
+
598
+
int io_accept(struct io_kiocb *req, unsigned int issue_flags)
599
+
{
600
+
struct io_ring_ctx *ctx = req->ctx;
601
+
struct io_accept *accept = io_kiocb_to_cmd(req);
602
+
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
603
+
unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
604
+
bool fixed = !!accept->file_slot;
605
+
struct file *file;
606
+
int ret, fd;
607
+
608
+
retry:
609
+
if (!fixed) {
610
+
fd = __get_unused_fd_flags(accept->flags, accept->nofile);
611
+
if (unlikely(fd < 0))
612
+
return fd;
613
+
}
614
+
file = do_accept(req->file, file_flags, accept->addr, accept->addr_len,
615
+
accept->flags);
616
+
if (IS_ERR(file)) {
617
+
if (!fixed)
618
+
put_unused_fd(fd);
619
+
ret = PTR_ERR(file);
620
+
if (ret == -EAGAIN && force_nonblock) {
621
+
/*
622
+
* if it's multishot and polled, we don't need to
623
+
* return EAGAIN to arm the poll infra since it
624
+
* has already been done
625
+
*/
626
+
if ((req->flags & IO_APOLL_MULTI_POLLED) ==
627
+
IO_APOLL_MULTI_POLLED)
628
+
ret = IOU_ISSUE_SKIP_COMPLETE;
629
+
return ret;
630
+
}
631
+
if (ret == -ERESTARTSYS)
632
+
ret = -EINTR;
633
+
req_set_fail(req);
634
+
} else if (!fixed) {
635
+
fd_install(fd, file);
636
+
ret = fd;
637
+
} else {
638
+
ret = io_fixed_fd_install(req, issue_flags, file,
639
+
accept->file_slot);
640
+
}
641
+
642
+
if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
643
+
io_req_set_res(req, ret, 0);
644
+
return IOU_OK;
645
+
}
646
+
if (ret >= 0) {
647
+
bool filled;
648
+
649
+
spin_lock(&ctx->completion_lock);
650
+
filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret,
651
+
IORING_CQE_F_MORE);
652
+
io_commit_cqring(ctx);
653
+
spin_unlock(&ctx->completion_lock);
654
+
if (filled) {
655
+
io_cqring_ev_posted(ctx);
656
+
goto retry;
657
+
}
658
+
ret = -ECANCELED;
659
+
}
660
+
661
+
return ret;
662
+
}
663
+
664
+
int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
665
+
{
666
+
struct io_socket *sock = io_kiocb_to_cmd(req);
667
+
668
+
if (sqe->addr || sqe->rw_flags || sqe->buf_index)
669
+
return -EINVAL;
670
+
671
+
sock->domain = READ_ONCE(sqe->fd);
672
+
sock->type = READ_ONCE(sqe->off);
673
+
sock->protocol = READ_ONCE(sqe->len);
674
+
sock->file_slot = READ_ONCE(sqe->file_index);
675
+
sock->nofile = rlimit(RLIMIT_NOFILE);
676
+
677
+
sock->flags = sock->type & ~SOCK_TYPE_MASK;
678
+
if (sock->file_slot && (sock->flags & SOCK_CLOEXEC))
679
+
return -EINVAL;
680
+
if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
681
+
return -EINVAL;
682
+
return 0;
683
+
}
684
+
685
+
int io_socket(struct io_kiocb *req, unsigned int issue_flags)
686
+
{
687
+
struct io_socket *sock = io_kiocb_to_cmd(req);
688
+
bool fixed = !!sock->file_slot;
689
+
struct file *file;
690
+
int ret, fd;
691
+
692
+
if (!fixed) {
693
+
fd = __get_unused_fd_flags(sock->flags, sock->nofile);
694
+
if (unlikely(fd < 0))
695
+
return fd;
696
+
}
697
+
file = __sys_socket_file(sock->domain, sock->type, sock->protocol);
698
+
if (IS_ERR(file)) {
699
+
if (!fixed)
700
+
put_unused_fd(fd);
701
+
ret = PTR_ERR(file);
702
+
if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
703
+
return -EAGAIN;
704
+
if (ret == -ERESTARTSYS)
705
+
ret = -EINTR;
706
+
req_set_fail(req);
707
+
} else if (!fixed) {
708
+
fd_install(fd, file);
709
+
ret = fd;
710
+
} else {
711
+
ret = io_fixed_fd_install(req, issue_flags, file,
712
+
sock->file_slot);
713
+
}
714
+
io_req_set_res(req, ret, 0);
715
+
return IOU_OK;
716
+
}
717
+
718
+
int io_connect_prep_async(struct io_kiocb *req)
719
+
{
720
+
struct io_async_connect *io = req->async_data;
721
+
struct io_connect *conn = io_kiocb_to_cmd(req);
722
+
723
+
return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
724
+
}
725
+
726
+
int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
727
+
{
728
+
struct io_connect *conn = io_kiocb_to_cmd(req);
729
+
730
+
if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
731
+
return -EINVAL;
732
+
733
+
conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
734
+
conn->addr_len = READ_ONCE(sqe->addr2);
735
+
return 0;
736
+
}
737
+
738
+
int io_connect(struct io_kiocb *req, unsigned int issue_flags)
739
+
{
740
+
struct io_connect *connect = io_kiocb_to_cmd(req);
741
+
struct io_async_connect __io, *io;
742
+
unsigned file_flags;
743
+
int ret;
744
+
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
745
+
746
+
if (req_has_async_data(req)) {
747
+
io = req->async_data;
748
+
} else {
749
+
ret = move_addr_to_kernel(connect->addr,
750
+
connect->addr_len,
751
+
&__io.address);
752
+
if (ret)
753
+
goto out;
754
+
io = &__io;
755
+
}
756
+
757
+
file_flags = force_nonblock ? O_NONBLOCK : 0;
758
+
759
+
ret = __sys_connect_file(req->file, &io->address,
760
+
connect->addr_len, file_flags);
761
+
if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
762
+
if (req_has_async_data(req))
763
+
return -EAGAIN;
764
+
if (io_alloc_async_data(req)) {
765
+
ret = -ENOMEM;
766
+
goto out;
767
+
}
768
+
memcpy(req->async_data, &__io, sizeof(__io));
769
+
return -EAGAIN;
770
+
}
771
+
if (ret == -ERESTARTSYS)
772
+
ret = -EINTR;
773
+
out:
774
+
if (ret < 0)
775
+
req_set_fail(req);
776
+
io_req_set_res(req, ret, 0);
777
+
return IOU_OK;
778
+
}
779
+
#endif
+43
io_uring/net.h
+43
io_uring/net.h
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
3
+
#include <linux/net.h>
4
+
#include <linux/uio.h>
5
+
6
+
#if defined(CONFIG_NET)
7
+
struct io_async_msghdr {
8
+
struct iovec fast_iov[UIO_FASTIOV];
9
+
/* points to an allocated iov, if NULL we use fast_iov instead */
10
+
struct iovec *free_iov;
11
+
struct sockaddr __user *uaddr;
12
+
struct msghdr msg;
13
+
struct sockaddr_storage addr;
14
+
};
15
+
16
+
struct io_async_connect {
17
+
struct sockaddr_storage address;
18
+
};
19
+
20
+
int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
21
+
int io_shutdown(struct io_kiocb *req, unsigned int issue_flags);
22
+
23
+
int io_sendmsg_prep_async(struct io_kiocb *req);
24
+
void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req);
25
+
int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
26
+
int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags);
27
+
int io_send(struct io_kiocb *req, unsigned int issue_flags);
28
+
29
+
int io_recvmsg_prep_async(struct io_kiocb *req);
30
+
int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
31
+
int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags);
32
+
int io_recv(struct io_kiocb *req, unsigned int issue_flags);
33
+
34
+
int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
35
+
int io_accept(struct io_kiocb *req, unsigned int issue_flags);
36
+
37
+
int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
38
+
int io_socket(struct io_kiocb *req, unsigned int issue_flags);
39
+
40
+
int io_connect_prep_async(struct io_kiocb *req);
41
+
int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
42
+
int io_connect(struct io_kiocb *req, unsigned int issue_flags);
43
+
#endif