io_uring/zcrx: return back two step unregistration

There are reports where io_uring instance removal takes too long and an
ifq reallocation by another zcrx instance fails. Split zcrx destruction
into two steps similarly how it was before, first close the queue early
but maintain zcrx alive, and then when all inflight requests are
completed, drop the main zcrx reference. For extra protection, mark
terminated zcrx instances in xarray and warn if we double put them.

Cc: stable@vger.kernel.org # 6.19+
Link: https://github.com/axboe/liburing/issues/1550
Reported-by: Youngmin Choi <youngminchoi94@gmail.com>
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://patch.msgid.link/0ce21f0565ab4358668922a28a8a36922dfebf76.1774261953.git.asml.silence@gmail.com
[axboe: NULL ifq before break inside scoped guard]
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Pavel Begunkov and committed by

Jens Axboe 2 months ago e5361d25 f41b0754

+51 -3

3 changed files

expand all

io_uring

io_uring.c

zcrx.c

zcrx.h

io_uring/io_uring.c

··· 2308 2308 struct io_tctx_node *node; 2309 2309 int ret; 2310 2310 2311 + mutex_lock(&ctx->uring_lock); 2312 + io_terminate_zcrx(ctx); 2313 + mutex_unlock(&ctx->uring_lock); 2314 + 2311 2315 /* 2312 2316 * If we're doing polled IO and end up having requests being 2313 2317 * submitted async (out-of-line), then completions can come in while

+43 -3

io_uring/zcrx.c

··· 624 624 } 625 625 } 626 626 627 - static void zcrx_unregister(struct io_zcrx_ifq *ifq) 627 + static void zcrx_unregister_user(struct io_zcrx_ifq *ifq) 628 628 { 629 629 if (refcount_dec_and_test(&ifq->user_refs)) { 630 630 io_close_queue(ifq); 631 631 io_zcrx_scrub(ifq); 632 632 } 633 + } 634 + 635 + static void zcrx_unregister(struct io_zcrx_ifq *ifq) 636 + { 637 + zcrx_unregister_user(ifq); 633 638 io_put_zcrx_ifq(ifq); 634 639 } 635 640 ··· 892 887 return &area->nia.niovs[niov_idx]; 893 888 } 894 889 890 + static inline bool is_zcrx_entry_marked(struct io_ring_ctx *ctx, unsigned long id) 891 + { 892 + return xa_get_mark(&ctx->zcrx_ctxs, id, XA_MARK_0); 893 + } 894 + 895 + static inline void set_zcrx_entry_mark(struct io_ring_ctx *ctx, unsigned long id) 896 + { 897 + xa_set_mark(&ctx->zcrx_ctxs, id, XA_MARK_0); 898 + } 899 + 900 + void io_terminate_zcrx(struct io_ring_ctx *ctx) 901 + { 902 + struct io_zcrx_ifq *ifq; 903 + unsigned long id = 0; 904 + 905 + lockdep_assert_held(&ctx->uring_lock); 906 + 907 + while (1) { 908 + scoped_guard(mutex, &ctx->mmap_lock) 909 + ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); 910 + if (!ifq) 911 + break; 912 + if (WARN_ON_ONCE(is_zcrx_entry_marked(ctx, id))) 913 + break; 914 + set_zcrx_entry_mark(ctx, id); 915 + id++; 916 + zcrx_unregister_user(ifq); 917 + } 918 + } 919 + 895 920 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) 896 921 { 897 922 struct io_zcrx_ifq *ifq; ··· 933 898 unsigned long id = 0; 934 899 935 900 ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); 936 - if (ifq) 901 + if (ifq) { 902 + if (WARN_ON_ONCE(!is_zcrx_entry_marked(ctx, id))) { 903 + ifq = NULL; 904 + break; 905 + } 937 906 xa_erase(&ctx->zcrx_ctxs, id); 907 + } 938 908 } 939 909 if (!ifq) 940 910 break; 941 - zcrx_unregister(ifq); 911 + io_put_zcrx_ifq(ifq); 942 912 } 943 913 944 914 xa_destroy(&ctx->zcrx_ctxs);

io_uring/zcrx.h

··· 74 74 int io_register_zcrx_ifq(struct io_ring_ctx *ctx, 75 75 struct io_uring_zcrx_ifq_reg __user *arg); 76 76 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx); 77 + void io_terminate_zcrx(struct io_ring_ctx *ctx); 77 78 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 78 79 struct socket *sock, unsigned int flags, 79 80 unsigned issue_flags, unsigned int *len); ··· 87 86 return -EOPNOTSUPP; 88 87 } 89 88 static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) 89 + { 90 + } 91 + static inline void io_terminate_zcrx(struct io_ring_ctx *ctx) 90 92 { 91 93 } 92 94 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,

Configure Feed

Configure Feed