Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/net: support bundles for recv
If IORING_OP_RECV is used with provided buffers, the caller may also set
IORING_RECVSEND_BUNDLE to turn it into a multi-buffer recv. This grabs
buffers available and receives into them, posting a single completion for
all of it.
This can be used with multishot receive as well, or without it.
Now that both send and receive support bundles, add a feature flag for
it as well. If IORING_FEAT_RECVSEND_BUNDLE is set after registering the
ring, then the kernel supports bundles for recv and send.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+105
-29
+8
-7
include/uapi/linux/io_uring.h
+8
-7
include/uapi/linux/io_uring.h
···
352
352
* IORING_NOTIF_USAGE_ZC_COPIED if data was copied
353
353
* (at least partially).
354
354
*
355
-
* IORING_RECVSEND_BUNDLE Used with IOSQE_BUFFER_SELECT. If set, send will
356
-
* grab as many buffers from the buffer group ID
357
-
* given and send them all. The completion result
358
-
* will be the number of buffers send, with the
359
-
* starting buffer ID in cqe->flags as per usual
360
-
* for provided buffer usage. The buffers will be
361
-
* contigious from the starting buffer ID.
355
+
* IORING_RECVSEND_BUNDLE Used with IOSQE_BUFFER_SELECT. If set, send or
356
+
* recv will grab as many buffers from the buffer
357
+
* group ID given and send them all. The completion
358
+
* result will be the number of buffers send, with
359
+
* the starting buffer ID in cqe->flags as per
360
+
* usual for provided buffer usage. The buffers
361
+
* will be contigious from the starting buffer ID.
362
362
*/
363
363
#define IORING_RECVSEND_POLL_FIRST (1U << 0)
364
364
#define IORING_RECV_MULTISHOT (1U << 1)
···
529
529
#define IORING_FEAT_CQE_SKIP (1U << 11)
530
530
#define IORING_FEAT_LINKED_FILE (1U << 12)
531
531
#define IORING_FEAT_REG_REG_RING (1U << 13)
532
+
#define IORING_FEAT_RECVSEND_BUNDLE (1U << 14)
532
533
533
534
/*
534
535
* io_uring_register(2) opcodes and arguments
+2
-1
io_uring/io_uring.c
+2
-1
io_uring/io_uring.c
···
3583
3583
IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
3584
3584
IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
3585
3585
IORING_FEAT_RSRC_TAGS | IORING_FEAT_CQE_SKIP |
3586
-
IORING_FEAT_LINKED_FILE | IORING_FEAT_REG_REG_RING;
3586
+
IORING_FEAT_LINKED_FILE | IORING_FEAT_REG_REG_RING |
3587
+
IORING_FEAT_RECVSEND_BUNDLE;
3587
3588
3588
3589
if (copy_to_user(params, p, sizeof(*p))) {
3589
3590
ret = -EFAULT;
+95
-21
io_uring/net.c
+95
-21
io_uring/net.c
···
747
747
return ret;
748
748
}
749
749
750
-
#define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT)
750
+
#define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \
751
+
IORING_RECVSEND_BUNDLE)
751
752
752
753
int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
753
754
{
···
762
761
sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
763
762
sr->len = READ_ONCE(sqe->len);
764
763
sr->flags = READ_ONCE(sqe->ioprio);
765
-
if (sr->flags & ~(RECVMSG_FLAGS))
764
+
if (sr->flags & ~RECVMSG_FLAGS)
766
765
return -EINVAL;
767
766
sr->msg_flags = READ_ONCE(sqe->msg_flags);
768
767
if (sr->msg_flags & MSG_DONTWAIT)
769
768
req->flags |= REQ_F_NOWAIT;
770
769
if (sr->msg_flags & MSG_ERRQUEUE)
771
770
req->flags |= REQ_F_CLEAR_POLLIN;
772
-
if (sr->flags & IORING_RECV_MULTISHOT) {
773
-
if (!(req->flags & REQ_F_BUFFER_SELECT))
774
-
return -EINVAL;
775
-
if (sr->msg_flags & MSG_WAITALL)
776
-
return -EINVAL;
777
-
if (req->opcode == IORING_OP_RECV && sr->len)
778
-
return -EINVAL;
779
-
req->flags |= REQ_F_APOLL_MULTISHOT;
771
+
if (req->flags & REQ_F_BUFFER_SELECT) {
780
772
/*
781
773
* Store the buffer group for this multishot receive separately,
782
774
* as if we end up doing an io-wq based issue that selects a
···
779
785
* restore it.
780
786
*/
781
787
sr->buf_group = req->buf_index;
788
+
req->buf_list = NULL;
789
+
}
790
+
if (sr->flags & IORING_RECV_MULTISHOT) {
791
+
if (!(req->flags & REQ_F_BUFFER_SELECT))
792
+
return -EINVAL;
793
+
if (sr->msg_flags & MSG_WAITALL)
794
+
return -EINVAL;
795
+
if (req->opcode == IORING_OP_RECV && sr->len)
796
+
return -EINVAL;
797
+
req->flags |= REQ_F_APOLL_MULTISHOT;
798
+
}
799
+
if (sr->flags & IORING_RECVSEND_BUNDLE) {
800
+
if (req->opcode == IORING_OP_RECVMSG)
801
+
return -EINVAL;
782
802
}
783
803
784
804
#ifdef CONFIG_COMPAT
···
813
805
struct io_async_msghdr *kmsg,
814
806
bool mshot_finished, unsigned issue_flags)
815
807
{
808
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
816
809
unsigned int cflags;
817
810
818
-
cflags = io_put_kbuf(req, issue_flags);
811
+
if (sr->flags & IORING_RECVSEND_BUNDLE)
812
+
cflags = io_put_kbufs(req, io_bundle_nbufs(kmsg, *ret),
813
+
issue_flags);
814
+
else
815
+
cflags = io_put_kbuf(req, issue_flags);
816
+
819
817
if (kmsg->msg.msg_inq > 0)
820
818
cflags |= IORING_CQE_F_SOCK_NONEMPTY;
819
+
820
+
/* bundle with no more immediate buffers, we're done */
821
+
if (sr->flags & IORING_RECVSEND_BUNDLE && req->flags & REQ_F_BL_EMPTY)
822
+
goto finish;
821
823
822
824
/*
823
825
* Fill CQE for this receive and see if we should keep trying to
···
835
817
*/
836
818
if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished &&
837
819
io_req_post_cqe(req, *ret, cflags | IORING_CQE_F_MORE)) {
838
-
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
839
820
int mshot_retry_ret = IOU_ISSUE_SKIP_COMPLETE;
840
821
841
822
io_mshot_prep_retry(req, kmsg);
···
854
837
}
855
838
856
839
/* Finish the request / stop multishot. */
840
+
finish:
857
841
io_req_set_res(req, *ret, cflags);
858
842
859
843
if (issue_flags & IO_URING_F_MULTISHOT)
···
1038
1020
return ret;
1039
1021
}
1040
1022
1023
+
static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg,
1024
+
size_t *len, unsigned int issue_flags)
1025
+
{
1026
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1027
+
int ret;
1028
+
1029
+
/*
1030
+
* If the ring isn't locked, then don't use the peek interface
1031
+
* to grab multiple buffers as we will lock/unlock between
1032
+
* this selection and posting the buffers.
1033
+
*/
1034
+
if (!(issue_flags & IO_URING_F_UNLOCKED) &&
1035
+
sr->flags & IORING_RECVSEND_BUNDLE) {
1036
+
struct buf_sel_arg arg = {
1037
+
.iovs = &kmsg->fast_iov,
1038
+
.nr_iovs = 1,
1039
+
.mode = KBUF_MODE_EXPAND,
1040
+
};
1041
+
1042
+
if (kmsg->free_iov) {
1043
+
arg.nr_iovs = kmsg->free_iov_nr;
1044
+
arg.iovs = kmsg->free_iov;
1045
+
arg.mode |= KBUF_MODE_FREE;
1046
+
}
1047
+
1048
+
if (kmsg->msg.msg_inq > 0)
1049
+
arg.max_len = min_not_zero(sr->len, kmsg->msg.msg_inq);
1050
+
1051
+
ret = io_buffers_peek(req, &arg);
1052
+
if (unlikely(ret < 0))
1053
+
return ret;
1054
+
1055
+
/* special case 1 vec, can be a fast path */
1056
+
if (ret == 1) {
1057
+
sr->buf = arg.iovs[0].iov_base;
1058
+
sr->len = arg.iovs[0].iov_len;
1059
+
goto map_ubuf;
1060
+
}
1061
+
iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,
1062
+
arg.out_len);
1063
+
if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->free_iov) {
1064
+
kmsg->free_iov_nr = ret;
1065
+
kmsg->free_iov = arg.iovs;
1066
+
}
1067
+
} else {
1068
+
void __user *buf;
1069
+
1070
+
*len = sr->len;
1071
+
buf = io_buffer_select(req, len, issue_flags);
1072
+
if (!buf)
1073
+
return -ENOBUFS;
1074
+
sr->buf = buf;
1075
+
sr->len = *len;
1076
+
map_ubuf:
1077
+
ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
1078
+
&kmsg->msg.msg_iter);
1079
+
if (unlikely(ret))
1080
+
return ret;
1081
+
}
1082
+
1083
+
return 0;
1084
+
}
1085
+
1041
1086
int io_recv(struct io_kiocb *req, unsigned int issue_flags)
1042
1087
{
1043
1088
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
···
1125
1044
1126
1045
retry_multishot:
1127
1046
if (io_do_buffer_select(req)) {
1128
-
void __user *buf;
1129
-
1130
-
buf = io_buffer_select(req, &len, issue_flags);
1131
-
if (!buf)
1132
-
return -ENOBUFS;
1133
-
sr->buf = buf;
1134
-
sr->len = len;
1135
-
ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
1136
-
&kmsg->msg.msg_iter);
1047
+
ret = io_recv_buf_select(req, kmsg, &len, issue_flags);
1137
1048
if (unlikely(ret))
1138
1049
goto out_free;
1050
+
sr->buf = NULL;
1139
1051
}
1140
1052
1141
1053
kmsg->msg.msg_inq = -1;