io_uring: fix nvme's 32b cqes on mixed cq

The nvme uring_cmd only uses 32b CQEs. If the ring uses a mixed CQ, then
we need to make sure we flag the completion as a 32b CQE.

On the other hand, if nvme uring_cmd was using a dedicated 32b CQE, the
posting was missing the extra memcpy because it only applied to bit CQEs
on a mixed CQ.

Fixes: e26dca67fde1943 ("io_uring: add support for IORING_SETUP_CQE_MIXED")
Signed-off-by: Keith Busch <kbusch@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Keith Busch and committed by

Jens Axboe 7 months ago 79525b51 7ea24326

+25 -10

4 changed files

expand all

drivers

nvme

host

ioctl.c

include

linux

io_uring

cmd.h

io_uring

io_uring.h

uring_cmd.c

+1 -1

drivers/nvme/host/ioctl.c

··· 410 410 411 411 if (pdu->bio) 412 412 blk_rq_unmap_user(pdu->bio); 413 - io_uring_cmd_done(ioucmd, pdu->status, pdu->result, issue_flags); 413 + io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, issue_flags); 414 414 } 415 415 416 416 static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,

+16 -4

include/linux/io_uring/cmd.h

··· 56 56 * Note: the caller should never hard code @issue_flags and is only allowed 57 57 * to pass the mask provided by the core io_uring code. 58 58 */ 59 - void io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2, 60 - unsigned issue_flags); 59 + void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2, 60 + unsigned issue_flags, bool is_cqe32); 61 61 62 62 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, 63 63 io_uring_cmd_tw_t task_work_cb, ··· 104 104 { 105 105 return -EOPNOTSUPP; 106 106 } 107 - static inline void io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, 108 - u64 ret2, unsigned issue_flags) 107 + static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, 108 + u64 ret2, unsigned issue_flags, bool is_cqe32) 109 109 { 110 110 } 111 111 static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, ··· 157 157 static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd) 158 158 { 159 159 return cmd_to_io_kiocb(cmd)->ctx; 160 + } 161 + 162 + static inline void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, 163 + u64 res2, unsigned issue_flags) 164 + { 165 + return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, false); 166 + } 167 + 168 + static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret, 169 + u64 res2, unsigned issue_flags) 170 + { 171 + return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true); 160 172 } 161 173 162 174 int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,

+1 -1

io_uring/io_uring.h

··· 275 275 return false; 276 276 277 277 memcpy(cqe, &req->cqe, sizeof(*cqe)); 278 - if (is_cqe32) { 278 + if (ctx->flags & IORING_SETUP_CQE32 || is_cqe32) { 279 279 memcpy(cqe->big_cqe, &req->big_cqe, sizeof(*cqe)); 280 280 memset(&req->big_cqe, 0, sizeof(req->big_cqe)); 281 281 }

+7 -4

io_uring/uring_cmd.c

··· 151 151 * Called by consumers of io_uring_cmd, if they originally returned 152 152 * -EIOCBQUEUED upon receiving the command. 153 153 */ 154 - void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2, 155 - unsigned issue_flags) 154 + void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2, 155 + unsigned issue_flags, bool is_cqe32) 156 156 { 157 157 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); 158 158 ··· 165 165 req_set_fail(req); 166 166 167 167 io_req_set_res(req, ret, 0); 168 - if (req->ctx->flags & IORING_SETUP_CQE32) 168 + if (is_cqe32) { 169 + if (req->ctx->flags & IORING_SETUP_CQE_MIXED) 170 + req->cqe.flags |= IORING_CQE_F_32; 169 171 io_req_set_cqe32_extra(req, res2, 0); 172 + } 170 173 io_req_uring_cleanup(req, issue_flags); 171 174 if (req->ctx->flags & IORING_SETUP_IOPOLL) { 172 175 /* order with io_iopoll_req_issued() checking ->iopoll_complete */ ··· 183 180 io_req_task_work_add(req); 184 181 } 185 182 } 186 - EXPORT_SYMBOL_GPL(io_uring_cmd_done); 183 + EXPORT_SYMBOL_GPL(__io_uring_cmd_done); 187 184 188 185 int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 189 186 {

Configure Feed

Configure Feed