Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'io_uring-6.7-2023-11-10' of git://git.kernel.dk/linux
Pull io_uring fixes from Jens Axboe:
"Mostly just a few fixes and cleanups caused by the read multishot
support.
Outside of that, a stable fix for how a connect retry is done"
* tag 'io_uring-6.7-2023-11-10' of git://git.kernel.dk/linux:
io_uring: do not clamp read length for multishot read
io_uring: do not allow multishot read to set addr or len
io_uring: indicate if io_kbuf_recycle did recycle anything
io_uring/rw: add separate prep handler for fixed read/write
io_uring/rw: add separate prep handler for readv/writev
io_uring/net: ensure socket is marked connected on connect retry
io_uring/rw: don't attempt to allocate async data if opcode doesn't need it
+80
-45
+3
-3
io_uring/kbuf.c
+3
-3
io_uring/kbuf.c
···
52
52
return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
53
53
}
54
54
55
-
void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)
55
+
bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)
56
56
{
57
57
struct io_ring_ctx *ctx = req->ctx;
58
58
struct io_buffer_list *bl;
···
65
65
* multiple use.
66
66
*/
67
67
if (req->flags & REQ_F_PARTIAL_IO)
68
-
return;
68
+
return false;
69
69
70
70
io_ring_submit_lock(ctx, issue_flags);
71
71
···
76
76
req->buf_index = buf->bgid;
77
77
78
78
io_ring_submit_unlock(ctx, issue_flags);
79
-
return;
79
+
return true;
80
80
}
81
81
82
82
unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags)
+8
-5
io_uring/kbuf.h
+8
-5
io_uring/kbuf.h
···
53
53
54
54
unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags);
55
55
56
-
void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
56
+
bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
57
57
58
58
void *io_pbuf_get_address(struct io_ring_ctx *ctx, unsigned long bgid);
59
59
60
-
static inline void io_kbuf_recycle_ring(struct io_kiocb *req)
60
+
static inline bool io_kbuf_recycle_ring(struct io_kiocb *req)
61
61
{
62
62
/*
63
63
* We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
···
80
80
} else {
81
81
req->buf_index = req->buf_list->bgid;
82
82
req->flags &= ~REQ_F_BUFFER_RING;
83
+
return true;
83
84
}
84
85
}
86
+
return false;
85
87
}
86
88
87
89
static inline bool io_do_buffer_select(struct io_kiocb *req)
···
93
91
return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING));
94
92
}
95
93
96
-
static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
94
+
static inline bool io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
97
95
{
98
96
if (req->flags & REQ_F_BUFFER_SELECTED)
99
-
io_kbuf_recycle_legacy(req, issue_flags);
97
+
return io_kbuf_recycle_legacy(req, issue_flags);
100
98
if (req->flags & REQ_F_BUFFER_RING)
101
-
io_kbuf_recycle_ring(req);
99
+
return io_kbuf_recycle_ring(req);
100
+
return false;
102
101
}
103
102
104
103
static inline unsigned int __io_put_kbuf_list(struct io_kiocb *req,
+11
-13
io_uring/net.c
+11
-13
io_uring/net.c
···
1461
1461
int ret;
1462
1462
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1463
1463
1464
-
if (connect->in_progress) {
1465
-
struct socket *socket;
1466
-
1467
-
ret = -ENOTSOCK;
1468
-
socket = sock_from_file(req->file);
1469
-
if (socket)
1470
-
ret = sock_error(socket->sk);
1471
-
goto out;
1472
-
}
1473
-
1474
1464
if (req_has_async_data(req)) {
1475
1465
io = req->async_data;
1476
1466
} else {
···
1480
1490
&& force_nonblock) {
1481
1491
if (ret == -EINPROGRESS) {
1482
1492
connect->in_progress = true;
1483
-
return -EAGAIN;
1484
-
}
1485
-
if (ret == -ECONNABORTED) {
1493
+
} else if (ret == -ECONNABORTED) {
1486
1494
if (connect->seen_econnaborted)
1487
1495
goto out;
1488
1496
connect->seen_econnaborted = true;
···
1493
1505
}
1494
1506
memcpy(req->async_data, &__io, sizeof(__io));
1495
1507
return -EAGAIN;
1508
+
}
1509
+
if (connect->in_progress) {
1510
+
/*
1511
+
* At least bluetooth will return -EBADFD on a re-connect
1512
+
* attempt, and it's (supposedly) also valid to get -EISCONN
1513
+
* which means the previous result is good. For both of these,
1514
+
* grab the sock_error() and use that for the completion.
1515
+
*/
1516
+
if (ret == -EBADFD || ret == -EISCONN)
1517
+
ret = sock_error(sock_from_file(req->file)->sk);
1496
1518
}
1497
1519
if (ret == -ERESTARTSYS)
1498
1520
ret = -EINTR;
+4
-4
io_uring/opdef.c
+4
-4
io_uring/opdef.c
···
66
66
.iopoll = 1,
67
67
.iopoll_queue = 1,
68
68
.vectored = 1,
69
-
.prep = io_prep_rw,
69
+
.prep = io_prep_rwv,
70
70
.issue = io_read,
71
71
},
72
72
[IORING_OP_WRITEV] = {
···
80
80
.iopoll = 1,
81
81
.iopoll_queue = 1,
82
82
.vectored = 1,
83
-
.prep = io_prep_rw,
83
+
.prep = io_prep_rwv,
84
84
.issue = io_write,
85
85
},
86
86
[IORING_OP_FSYNC] = {
···
98
98
.ioprio = 1,
99
99
.iopoll = 1,
100
100
.iopoll_queue = 1,
101
-
.prep = io_prep_rw,
101
+
.prep = io_prep_rw_fixed,
102
102
.issue = io_read,
103
103
},
104
104
[IORING_OP_WRITE_FIXED] = {
···
111
111
.ioprio = 1,
112
112
.iopoll = 1,
113
113
.iopoll_queue = 1,
114
-
.prep = io_prep_rw,
114
+
.prep = io_prep_rw_fixed,
115
115
.issue = io_write,
116
116
},
117
117
[IORING_OP_POLL_ADD] = {
+52
-20
io_uring/rw.c
+52
-20
io_uring/rw.c
···
83
83
/* used for fixed read/write too - just read unconditionally */
84
84
req->buf_index = READ_ONCE(sqe->buf_index);
85
85
86
-
if (req->opcode == IORING_OP_READ_FIXED ||
87
-
req->opcode == IORING_OP_WRITE_FIXED) {
88
-
struct io_ring_ctx *ctx = req->ctx;
89
-
u16 index;
90
-
91
-
if (unlikely(req->buf_index >= ctx->nr_user_bufs))
92
-
return -EFAULT;
93
-
index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);
94
-
req->imu = ctx->user_bufs[index];
95
-
io_req_set_rsrc_node(req, ctx, 0);
96
-
}
97
-
98
86
ioprio = READ_ONCE(sqe->ioprio);
99
87
if (ioprio) {
100
88
ret = ioprio_check_cap(ioprio);
···
98
110
rw->addr = READ_ONCE(sqe->addr);
99
111
rw->len = READ_ONCE(sqe->len);
100
112
rw->flags = READ_ONCE(sqe->rw_flags);
113
+
return 0;
114
+
}
101
115
102
-
/* Have to do this validation here, as this is in io_read() rw->len might
103
-
* have chanaged due to buffer selection
116
+
int io_prep_rwv(struct io_kiocb *req, const struct io_uring_sqe *sqe)
117
+
{
118
+
int ret;
119
+
120
+
ret = io_prep_rw(req, sqe);
121
+
if (unlikely(ret))
122
+
return ret;
123
+
124
+
/*
125
+
* Have to do this validation here, as this is in io_read() rw->len
126
+
* might have chanaged due to buffer selection
104
127
*/
105
-
if (req->opcode == IORING_OP_READV && req->flags & REQ_F_BUFFER_SELECT) {
106
-
ret = io_iov_buffer_select_prep(req);
107
-
if (ret)
108
-
return ret;
109
-
}
128
+
if (req->flags & REQ_F_BUFFER_SELECT)
129
+
return io_iov_buffer_select_prep(req);
110
130
131
+
return 0;
132
+
}
133
+
134
+
int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
135
+
{
136
+
struct io_ring_ctx *ctx = req->ctx;
137
+
u16 index;
138
+
int ret;
139
+
140
+
ret = io_prep_rw(req, sqe);
141
+
if (unlikely(ret))
142
+
return ret;
143
+
144
+
if (unlikely(req->buf_index >= ctx->nr_user_bufs))
145
+
return -EFAULT;
146
+
index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);
147
+
req->imu = ctx->user_bufs[index];
148
+
io_req_set_rsrc_node(req, ctx, 0);
111
149
return 0;
112
150
}
113
151
···
143
129
*/
144
130
int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
145
131
{
132
+
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
146
133
int ret;
134
+
135
+
/* must be used with provided buffers */
136
+
if (!(req->flags & REQ_F_BUFFER_SELECT))
137
+
return -EINVAL;
147
138
148
139
ret = io_prep_rw(req, sqe);
149
140
if (unlikely(ret))
150
141
return ret;
142
+
143
+
if (rw->addr || rw->len)
144
+
return -EINVAL;
151
145
152
146
req->flags |= REQ_F_APOLL_MULTISHOT;
153
147
return 0;
···
564
542
{
565
543
if (!force && !io_cold_defs[req->opcode].prep_async)
566
544
return 0;
545
+
/* opcode type doesn't need async data */
546
+
if (!io_cold_defs[req->opcode].async_size)
547
+
return 0;
567
548
if (!req_has_async_data(req)) {
568
549
struct io_async_rw *iorw;
569
550
···
912
887
913
888
int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
914
889
{
890
+
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
915
891
unsigned int cflags = 0;
916
892
int ret;
917
893
···
929
903
* handling arm it.
930
904
*/
931
905
if (ret == -EAGAIN) {
932
-
io_kbuf_recycle(req, issue_flags);
906
+
/*
907
+
* Reset rw->len to 0 again to avoid clamping future mshot
908
+
* reads, in case the buffer size varies.
909
+
*/
910
+
if (io_kbuf_recycle(req, issue_flags))
911
+
rw->len = 0;
933
912
return -EAGAIN;
934
913
}
935
914
···
947
916
* jump to the termination path. This request is then done.
948
917
*/
949
918
cflags = io_put_kbuf(req, issue_flags);
919
+
rw->len = 0; /* similarly to above, reset len to 0 */
950
920
951
921
if (io_fill_cqe_req_aux(req,
952
922
issue_flags & IO_URING_F_COMPLETE_DEFER,
+2
io_uring/rw.h
+2
io_uring/rw.h
···
16
16
};
17
17
18
18
int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe);
19
+
int io_prep_rwv(struct io_kiocb *req, const struct io_uring_sqe *sqe);
20
+
int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe);
19
21
int io_read(struct io_kiocb *req, unsigned int issue_flags);
20
22
int io_readv_prep_async(struct io_kiocb *req);
21
23
int io_write(struct io_kiocb *req, unsigned int issue_flags);