Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-6.8-2024-02-01' of git://git.kernel.dk/linux
Pull io_uring fixes from Jens Axboe:
- Fix for missing retry for read multishot.
If we trigger the execution of it and there's more than one buffer to
be read, then we don't always read more than the first one. As it's
edge triggered, this can lead to stalls.
- Limit inline receive multishot retries for fairness reasons.
If we have a very bursty socket receiving data, we still need to
ensure we process other requests as well. This is really two minor
cleanups, then adding a way for poll reissue to trigger a requeue,
and then finally having multishot receive utilize that.
- Fix for a weird corner case for non-multishot receive with
MSG_WAITALL, using provided buffers, and setting the length to
zero (to let the buffer dictate the receive size).
* tag 'io_uring-6.8-2024-02-01' of git://git.kernel.dk/linux:
io_uring/net: fix sr->len for IORING_OP_RECV with MSG_WAITALL and buffers
io_uring/net: limit inline multishot retries
io_uring/poll: add requeue return code from poll multishot handling
io_uring/net: un-indent mshot retry path in io_recv_finish()
io_uring/poll: move poll execution helpers higher up
io_uring/rw: ensure poll based multishot read retries appropriately
+91
-39
+7
-1
io_uring/io_uring.h
+7
-1
io_uring/io_uring.h
···
15
15
#include <trace/events/io_uring.h>
16
16
#endif
17
17
18
-
19
18
enum {
20
19
IOU_OK = 0,
21
20
IOU_ISSUE_SKIP_COMPLETE = -EIOCBQUEUED,
21
+
22
+
/*
23
+
* Requeue the task_work to restart operations on this request. The
24
+
* actual value isn't important, should just be not an otherwise
25
+
* valid error code, yet less than -MAX_ERRNO and valid internally.
26
+
*/
27
+
IOU_REQUEUE = -3072,
22
28
23
29
/*
24
30
* Intended only when both IO_URING_F_MULTISHOT is passed
+38
-16
io_uring/net.c
+38
-16
io_uring/net.c
···
60
60
unsigned len;
61
61
unsigned done_io;
62
62
unsigned msg_flags;
63
+
unsigned nr_multishot_loops;
63
64
u16 flags;
64
65
/* initialised and used only by !msg send variants */
65
66
u16 addr_len;
···
70
69
/* used only for send zerocopy */
71
70
struct io_kiocb *notif;
72
71
};
72
+
73
+
/*
74
+
* Number of times we'll try and do receives if there's more data. If we
75
+
* exceed this limit, then add us to the back of the queue and retry from
76
+
* there. This helps fairness between flooding clients.
77
+
*/
78
+
#define MULTISHOT_MAX_RETRY 32
73
79
74
80
static inline bool io_check_multishot(struct io_kiocb *req,
75
81
unsigned int issue_flags)
···
619
611
sr->msg_flags |= MSG_CMSG_COMPAT;
620
612
#endif
621
613
sr->done_io = 0;
614
+
sr->nr_multishot_loops = 0;
622
615
return 0;
623
616
}
624
617
···
654
645
return true;
655
646
}
656
647
657
-
if (!mshot_finished) {
658
-
if (io_fill_cqe_req_aux(req, issue_flags & IO_URING_F_COMPLETE_DEFER,
659
-
*ret, cflags | IORING_CQE_F_MORE)) {
660
-
io_recv_prep_retry(req);
661
-
/* Known not-empty or unknown state, retry */
662
-
if (cflags & IORING_CQE_F_SOCK_NONEMPTY ||
663
-
msg->msg_inq == -1)
664
-
return false;
665
-
if (issue_flags & IO_URING_F_MULTISHOT)
666
-
*ret = IOU_ISSUE_SKIP_COMPLETE;
667
-
else
668
-
*ret = -EAGAIN;
669
-
return true;
670
-
}
671
-
/* Otherwise stop multishot but use the current result. */
672
-
}
648
+
if (mshot_finished)
649
+
goto finish;
673
650
651
+
/*
652
+
* Fill CQE for this receive and see if we should keep trying to
653
+
* receive from this socket.
654
+
*/
655
+
if (io_fill_cqe_req_aux(req, issue_flags & IO_URING_F_COMPLETE_DEFER,
656
+
*ret, cflags | IORING_CQE_F_MORE)) {
657
+
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
658
+
int mshot_retry_ret = IOU_ISSUE_SKIP_COMPLETE;
659
+
660
+
io_recv_prep_retry(req);
661
+
/* Known not-empty or unknown state, retry */
662
+
if (cflags & IORING_CQE_F_SOCK_NONEMPTY || msg->msg_inq == -1) {
663
+
if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY)
664
+
return false;
665
+
/* mshot retries exceeded, force a requeue */
666
+
sr->nr_multishot_loops = 0;
667
+
mshot_retry_ret = IOU_REQUEUE;
668
+
}
669
+
if (issue_flags & IO_URING_F_MULTISHOT)
670
+
*ret = mshot_retry_ret;
671
+
else
672
+
*ret = -EAGAIN;
673
+
return true;
674
+
}
675
+
/* Otherwise stop multishot but use the current result. */
676
+
finish:
674
677
io_req_set_res(req, *ret, cflags);
675
678
676
679
if (issue_flags & IO_URING_F_MULTISHOT)
···
923
902
if (!buf)
924
903
return -ENOBUFS;
925
904
sr->buf = buf;
905
+
sr->len = len;
926
906
}
927
907
928
908
ret = import_ubuf(ITER_DEST, sr->buf, len, &msg.msg_iter);
+28
-21
io_uring/poll.c
+28
-21
io_uring/poll.c
···
226
226
IOU_POLL_NO_ACTION = 1,
227
227
IOU_POLL_REMOVE_POLL_USE_RES = 2,
228
228
IOU_POLL_REISSUE = 3,
229
+
IOU_POLL_REQUEUE = 4,
229
230
};
231
+
232
+
static void __io_poll_execute(struct io_kiocb *req, int mask)
233
+
{
234
+
unsigned flags = 0;
235
+
236
+
io_req_set_res(req, mask, 0);
237
+
req->io_task_work.func = io_poll_task_func;
238
+
239
+
trace_io_uring_task_add(req, mask);
240
+
241
+
if (!(req->flags & REQ_F_POLL_NO_LAZY))
242
+
flags = IOU_F_TWQ_LAZY_WAKE;
243
+
__io_req_task_work_add(req, flags);
244
+
}
245
+
246
+
static inline void io_poll_execute(struct io_kiocb *req, int res)
247
+
{
248
+
if (io_poll_get_ownership(req))
249
+
__io_poll_execute(req, res);
250
+
}
230
251
231
252
/*
232
253
* All poll tw should go through this. Checks for poll events, manages
···
330
309
int ret = io_poll_issue(req, ts);
331
310
if (ret == IOU_STOP_MULTISHOT)
332
311
return IOU_POLL_REMOVE_POLL_USE_RES;
312
+
else if (ret == IOU_REQUEUE)
313
+
return IOU_POLL_REQUEUE;
333
314
if (ret < 0)
334
315
return ret;
335
316
}
···
354
331
int ret;
355
332
356
333
ret = io_poll_check_events(req, ts);
357
-
if (ret == IOU_POLL_NO_ACTION)
334
+
if (ret == IOU_POLL_NO_ACTION) {
358
335
return;
336
+
} else if (ret == IOU_POLL_REQUEUE) {
337
+
__io_poll_execute(req, 0);
338
+
return;
339
+
}
359
340
io_poll_remove_entries(req);
360
341
io_poll_tw_hash_eject(req, ts);
361
342
···
389
362
else
390
363
io_req_defer_failed(req, ret);
391
364
}
392
-
}
393
-
394
-
static void __io_poll_execute(struct io_kiocb *req, int mask)
395
-
{
396
-
unsigned flags = 0;
397
-
398
-
io_req_set_res(req, mask, 0);
399
-
req->io_task_work.func = io_poll_task_func;
400
-
401
-
trace_io_uring_task_add(req, mask);
402
-
403
-
if (!(req->flags & REQ_F_POLL_NO_LAZY))
404
-
flags = IOU_F_TWQ_LAZY_WAKE;
405
-
__io_req_task_work_add(req, flags);
406
-
}
407
-
408
-
static inline void io_poll_execute(struct io_kiocb *req, int res)
409
-
{
410
-
if (io_poll_get_ownership(req))
411
-
__io_poll_execute(req, res);
412
365
}
413
366
414
367
static void io_poll_cancel_req(struct io_kiocb *req)
+9
io_uring/poll.h
+9
io_uring/poll.h
···
24
24
struct io_poll *double_poll;
25
25
};
26
26
27
+
/*
28
+
* Must only be called inside issue_flags & IO_URING_F_MULTISHOT, or
29
+
* potentially other cases where we already "own" this poll request.
30
+
*/
31
+
static inline void io_poll_multishot_retry(struct io_kiocb *req)
32
+
{
33
+
atomic_inc(&req->poll_refs);
34
+
}
35
+
27
36
int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
28
37
int io_poll_add(struct io_kiocb *req, unsigned int issue_flags);
29
38
+9
-1
io_uring/rw.c
+9
-1
io_uring/rw.c
···
18
18
#include "opdef.h"
19
19
#include "kbuf.h"
20
20
#include "rsrc.h"
21
+
#include "poll.h"
21
22
#include "rw.h"
22
23
23
24
struct io_rw {
···
963
962
if (io_fill_cqe_req_aux(req,
964
963
issue_flags & IO_URING_F_COMPLETE_DEFER,
965
964
ret, cflags | IORING_CQE_F_MORE)) {
966
-
if (issue_flags & IO_URING_F_MULTISHOT)
965
+
if (issue_flags & IO_URING_F_MULTISHOT) {
966
+
/*
967
+
* Force retry, as we might have more data to
968
+
* be read and otherwise it won't get retried
969
+
* until (if ever) another poll is triggered.
970
+
*/
971
+
io_poll_multishot_retry(req);
967
972
return IOU_ISSUE_SKIP_COMPLETE;
973
+
}
968
974
return -EAGAIN;
969
975
}
970
976
}