Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring: split alloc and add of overflow
Add a new helper, io_alloc_ocqe(), that simply allocates and fills an
overflow entry. Then it can get done outside of the locking section,
and hence use more appropriate gfp_t allocation flags rather than always
default to GFP_ATOMIC.
Inspired by a previous series from Pavel:
https://lore.kernel.org/io-uring/cover.1747209332.git.asml.silence@gmail.com/
Reviewed-by: Caleb Sander Mateos <csander@purestorage.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
+45
-29
+45
-29
io_uring/io_uring.c
+45
-29
io_uring/io_uring.c
···
697
697
}
698
698
}
699
699
700
-
static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
701
-
s32 res, u32 cflags, u64 extra1, u64 extra2)
700
+
static bool io_cqring_add_overflow(struct io_ring_ctx *ctx,
701
+
struct io_overflow_cqe *ocqe)
702
702
{
703
-
struct io_overflow_cqe *ocqe;
704
-
size_t ocq_size = sizeof(struct io_overflow_cqe);
705
-
bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
706
-
707
703
lockdep_assert_held(&ctx->completion_lock);
708
704
709
-
if (is_cqe32)
710
-
ocq_size += sizeof(struct io_uring_cqe);
711
-
712
-
ocqe = kmalloc(ocq_size, GFP_ATOMIC | __GFP_ACCOUNT);
713
-
trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
714
705
if (!ocqe) {
715
706
struct io_rings *r = ctx->rings;
716
707
···
719
728
atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
720
729
721
730
}
722
-
ocqe->cqe.user_data = user_data;
723
-
ocqe->cqe.res = res;
724
-
ocqe->cqe.flags = cflags;
725
-
if (is_cqe32) {
726
-
ocqe->cqe.big_cqe[0] = extra1;
727
-
ocqe->cqe.big_cqe[1] = extra2;
728
-
}
729
731
list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
730
732
return true;
733
+
}
734
+
735
+
static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,
736
+
u64 user_data, s32 res, u32 cflags,
737
+
u64 extra1, u64 extra2, gfp_t gfp)
738
+
{
739
+
struct io_overflow_cqe *ocqe;
740
+
size_t ocq_size = sizeof(struct io_overflow_cqe);
741
+
bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
742
+
743
+
if (is_cqe32)
744
+
ocq_size += sizeof(struct io_uring_cqe);
745
+
746
+
ocqe = kmalloc(ocq_size, gfp | __GFP_ACCOUNT);
747
+
trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
748
+
if (ocqe) {
749
+
ocqe->cqe.user_data = user_data;
750
+
ocqe->cqe.res = res;
751
+
ocqe->cqe.flags = cflags;
752
+
if (is_cqe32) {
753
+
ocqe->cqe.big_cqe[0] = extra1;
754
+
ocqe->cqe.big_cqe[1] = extra2;
755
+
}
756
+
}
757
+
return ocqe;
731
758
}
732
759
733
760
/*
···
812
803
813
804
io_cq_lock(ctx);
814
805
filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
815
-
if (!filled)
816
-
filled = io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
806
+
if (unlikely(!filled)) {
807
+
struct io_overflow_cqe *ocqe;
808
+
809
+
ocqe = io_alloc_ocqe(ctx, user_data, res, cflags, 0, 0, GFP_ATOMIC);
810
+
filled = io_cqring_add_overflow(ctx, ocqe);
811
+
}
817
812
io_cq_unlock_post(ctx);
818
813
return filled;
819
814
}
···
832
819
lockdep_assert(ctx->lockless_cq);
833
820
834
821
if (!io_fill_cqe_aux(ctx, user_data, res, cflags)) {
822
+
struct io_overflow_cqe *ocqe;
823
+
824
+
ocqe = io_alloc_ocqe(ctx, user_data, res, cflags, 0, 0, GFP_KERNEL);
835
825
spin_lock(&ctx->completion_lock);
836
-
io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
826
+
io_cqring_add_overflow(ctx, ocqe);
837
827
spin_unlock(&ctx->completion_lock);
838
828
}
839
829
ctx->submit_state.cq_flush = true;
···
1441
1425
*/
1442
1426
if (!(req->flags & (REQ_F_CQE_SKIP | REQ_F_REISSUE)) &&
1443
1427
unlikely(!io_fill_cqe_req(ctx, req))) {
1428
+
gfp_t gfp = ctx->lockless_cq ? GFP_KERNEL : GFP_ATOMIC;
1429
+
struct io_overflow_cqe *ocqe;
1430
+
1431
+
ocqe = io_alloc_ocqe(ctx, req->cqe.user_data, req->cqe.res,
1432
+
req->cqe.flags, req->big_cqe.extra1,
1433
+
req->big_cqe.extra2, gfp);
1444
1434
if (ctx->lockless_cq) {
1445
1435
spin_lock(&ctx->completion_lock);
1446
-
io_cqring_event_overflow(req->ctx, req->cqe.user_data,
1447
-
req->cqe.res, req->cqe.flags,
1448
-
req->big_cqe.extra1,
1449
-
req->big_cqe.extra2);
1436
+
io_cqring_add_overflow(ctx, ocqe);
1450
1437
spin_unlock(&ctx->completion_lock);
1451
1438
} else {
1452
-
io_cqring_event_overflow(req->ctx, req->cqe.user_data,
1453
-
req->cqe.res, req->cqe.flags,
1454
-
req->big_cqe.extra1,
1455
-
req->big_cqe.extra2);
1439
+
io_cqring_add_overflow(ctx, ocqe);
1456
1440
}
1457
1441
1458
1442
memset(&req->big_cqe, 0, sizeof(req->big_cqe));