io_uring: consolidate drain seq checking

We check sequences when queuing drained requests as well when flushing
them. Instead, always queue and immediately try to flush, so that all
seq handling can be kept contained in the flushing code.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/d4651f742e671af5b3216581e539ea5d31bc7125.1746788718.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Pavel Begunkov and committed by

Jens Axboe 1 year ago 19a94da4 e91e4f69

+17 -28

1 changed file

expand all

io_uring

io_uring.c

+17 -28

io_uring/io_uring.c

··· 389 389 ctx->cq_extra--; 390 390 } 391 391 392 - static bool req_need_defer(struct io_kiocb *req, u32 seq) 393 - { 394 - if (unlikely(req->flags & REQ_F_IO_DRAIN)) { 395 - struct io_ring_ctx *ctx = req->ctx; 396 - 397 - return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail; 398 - } 399 - 400 - return false; 401 - } 402 - 403 392 static void io_clean_op(struct io_kiocb *req) 404 393 { 405 394 if (unlikely(req->flags & REQ_F_BUFFER_SELECTED)) ··· 555 566 return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail; 556 567 } 557 568 558 - static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx) 569 + static __cold noinline void __io_queue_deferred(struct io_ring_ctx *ctx) 559 570 { 560 571 bool drain_seen = false, first = true; 561 572 562 - spin_lock(&ctx->completion_lock); 563 573 while (!list_empty(&ctx->defer_list)) { 564 574 struct io_defer_entry *de = list_first_entry(&ctx->defer_list, 565 575 struct io_defer_entry, list); ··· 572 584 kfree(de); 573 585 first = false; 574 586 } 575 - spin_unlock(&ctx->completion_lock); 587 + } 588 + 589 + static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx) 590 + { 591 + guard(spinlock)(&ctx->completion_lock); 592 + __io_queue_deferred(ctx); 576 593 } 577 594 578 595 void __io_commit_cqring_flush(struct io_ring_ctx *ctx) ··· 1664 1671 __must_hold(&ctx->uring_lock) 1665 1672 { 1666 1673 struct io_ring_ctx *ctx = req->ctx; 1674 + bool drain = req->flags & IOSQE_IO_DRAIN; 1667 1675 struct io_defer_entry *de; 1668 - u32 seq = io_get_sequence(req); 1669 1676 1670 - io_prep_async_link(req); 1671 1677 de = kmalloc(sizeof(*de), GFP_KERNEL_ACCOUNT); 1672 1678 if (!de) { 1673 1679 io_req_defer_failed(req, -ENOMEM); 1674 1680 return; 1675 1681 } 1676 1682 1677 - spin_lock(&ctx->completion_lock); 1678 - if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) { 1679 - spin_unlock(&ctx->completion_lock); 1680 - kfree(de); 1681 - ctx->drain_active = false; 1682 - io_req_task_queue(req); 1683 - return; 1684 - } 1685 - 1683 + io_prep_async_link(req); 1686 1684 trace_io_uring_defer(req); 1687 1685 de->req = req; 1688 - de->seq = seq; 1689 - list_add_tail(&de->list, &ctx->defer_list); 1690 - spin_unlock(&ctx->completion_lock); 1686 + de->seq = io_get_sequence(req); 1687 + 1688 + scoped_guard(spinlock, &ctx->completion_lock) { 1689 + list_add_tail(&de->list, &ctx->defer_list); 1690 + __io_queue_deferred(ctx); 1691 + if (!drain && list_empty(&ctx->defer_list)) 1692 + ctx->drain_active = false; 1693 + } 1691 1694 } 1692 1695 1693 1696 static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,

Configure Feed

Configure Feed