Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring: consolidate overflow flushing
Consolidate __io_cqring_overflow_flush and io_cqring_overflow_kill()
into a single function as it once was, it's easier to work with it this
way.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/986b42c35e76a6be7aa0cdcda0a236a2222da3a7.1712708261.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Pavel Begunkov
and committed by
Jens Axboe
6b231248
8d09a88e
+15
-25
+15
-25
io_uring/io_uring.c
+15
-25
io_uring/io_uring.c
···
668
668
io_commit_cqring_flush(ctx);
669
669
}
670
670
671
-
static void io_cqring_overflow_kill(struct io_ring_ctx *ctx)
672
-
{
673
-
struct io_overflow_cqe *ocqe;
674
-
LIST_HEAD(list);
675
-
676
-
lockdep_assert_held(&ctx->uring_lock);
677
-
678
-
spin_lock(&ctx->completion_lock);
679
-
list_splice_init(&ctx->cq_overflow_list, &list);
680
-
clear_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
681
-
spin_unlock(&ctx->completion_lock);
682
-
683
-
while (!list_empty(&list)) {
684
-
ocqe = list_first_entry(&list, struct io_overflow_cqe, list);
685
-
list_del(&ocqe->list);
686
-
kfree(ocqe);
687
-
}
688
-
}
689
-
690
-
static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx)
671
+
static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool dying)
691
672
{
692
673
size_t cqe_size = sizeof(struct io_uring_cqe);
693
674
···
685
704
struct io_uring_cqe *cqe;
686
705
struct io_overflow_cqe *ocqe;
687
706
688
-
if (!io_get_cqe_overflow(ctx, &cqe, true))
689
-
break;
690
707
ocqe = list_first_entry(&ctx->cq_overflow_list,
691
708
struct io_overflow_cqe, list);
692
-
memcpy(cqe, &ocqe->cqe, cqe_size);
709
+
710
+
if (!dying) {
711
+
if (!io_get_cqe_overflow(ctx, &cqe, true))
712
+
break;
713
+
memcpy(cqe, &ocqe->cqe, cqe_size);
714
+
}
693
715
list_del(&ocqe->list);
694
716
kfree(ocqe);
695
717
}
···
704
720
io_cq_unlock_post(ctx);
705
721
}
706
722
723
+
static void io_cqring_overflow_kill(struct io_ring_ctx *ctx)
724
+
{
725
+
if (ctx->rings)
726
+
__io_cqring_overflow_flush(ctx, true);
727
+
}
728
+
707
729
static void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx)
708
730
{
709
731
mutex_lock(&ctx->uring_lock);
710
-
__io_cqring_overflow_flush(ctx);
732
+
__io_cqring_overflow_flush(ctx, false);
711
733
mutex_unlock(&ctx->uring_lock);
712
734
}
713
735
···
1521
1531
check_cq = READ_ONCE(ctx->check_cq);
1522
1532
if (unlikely(check_cq)) {
1523
1533
if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
1524
-
__io_cqring_overflow_flush(ctx);
1534
+
__io_cqring_overflow_flush(ctx, false);
1525
1535
/*
1526
1536
* Similarly do not spin if we have not informed the user of any
1527
1537
* dropped CQE.