io_uring: add mshot helper for posting CQE32

Add a helper for posting 32 byte CQEs in a multishot mode and add a cmd
helper on top. As it specifically works with requests, the helper ignore
the passed in cqe->user_data and sets it to the one stored in the
request.

The command helper is only valid with multishot requests.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/c29d7720c16e1f981cfaa903df187138baa3946b.1750065793.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Pavel Begunkov and committed by

Jens Axboe 1 year ago ac479eac b9557549

+56

4 changed files

expand all

io_uring

io_uring.c

io_uring.h

uring_cmd.c

uring_cmd.h

+40

io_uring/io_uring.c

··· 793 793 return true; 794 794 } 795 795 796 + static bool io_fill_cqe_aux32(struct io_ring_ctx *ctx, 797 + struct io_uring_cqe src_cqe[2]) 798 + { 799 + struct io_uring_cqe *cqe; 800 + 801 + if (WARN_ON_ONCE(!(ctx->flags & IORING_SETUP_CQE32))) 802 + return false; 803 + if (unlikely(!io_get_cqe(ctx, &cqe))) 804 + return false; 805 + 806 + memcpy(cqe, src_cqe, 2 * sizeof(*cqe)); 807 + trace_io_uring_complete(ctx, NULL, cqe); 808 + return true; 809 + } 810 + 796 811 static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, 797 812 u32 cflags) 798 813 { ··· 913 898 spin_unlock(&ctx->completion_lock); 914 899 } else { 915 900 posted = io_fill_cqe_aux(ctx, req->cqe.user_data, res, cflags); 901 + } 902 + 903 + ctx->submit_state.cq_flush = true; 904 + return posted; 905 + } 906 + 907 + /* 908 + * A helper for multishot requests posting additional CQEs. 909 + * Should only be used from a task_work including IO_URING_F_MULTISHOT. 910 + */ 911 + bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2]) 912 + { 913 + struct io_ring_ctx *ctx = req->ctx; 914 + bool posted; 915 + 916 + lockdep_assert(!io_wq_current_is_worker()); 917 + lockdep_assert_held(&ctx->uring_lock); 918 + 919 + cqe[0].user_data = req->cqe.user_data; 920 + if (!ctx->lockless_cq) { 921 + spin_lock(&ctx->completion_lock); 922 + posted = io_fill_cqe_aux32(ctx, cqe); 923 + spin_unlock(&ctx->completion_lock); 924 + } else { 925 + posted = io_fill_cqe_aux32(ctx, cqe); 916 926 } 917 927 918 928 ctx->submit_state.cq_flush = true;

io_uring/io_uring.h

··· 81 81 bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); 82 82 void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); 83 83 bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags); 84 + bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe src_cqe[2]); 84 85 void __io_commit_cqring_flush(struct io_ring_ctx *ctx); 85 86 86 87 void io_req_track_inflight(struct io_kiocb *req);

+11

io_uring/uring_cmd.c

··· 328 328 ret = io_arm_apoll(req, issue_flags, mask); 329 329 return ret == IO_APOLL_OK ? -EIOCBQUEUED : -ECANCELED; 330 330 } 331 + 332 + bool io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd *cmd, 333 + unsigned int issue_flags, 334 + struct io_uring_cqe cqe[2]) 335 + { 336 + struct io_kiocb *req = cmd_to_io_kiocb(cmd); 337 + 338 + if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_MULTISHOT))) 339 + return false; 340 + return io_req_post_cqe32(req, cqe); 341 + }

io_uring/uring_cmd.h

··· 17 17 bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx, 18 18 struct io_uring_task *tctx, bool cancel_all); 19 19 20 + bool io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd *cmd, 21 + unsigned int issue_flags, 22 + struct io_uring_cqe cqe[2]); 23 + 20 24 void io_cmd_cache_free(const void *entry); 21 25 22 26 int io_cmd_poll_multishot(struct io_uring_cmd *cmd,

Configure Feed

Configure Feed