io_uring/zcrx: add support for multiple ifqs

+2 -3

include/linux/io_uring_types.h

··· 40 40 IO_URING_F_TASK_DEAD = (1 << 13), 41 41 }; 42 42 43 - struct io_zcrx_ifq; 44 - 45 43 struct io_wq_work_node { 46 44 struct io_wq_work_node *next; 47 45 }; ··· 392 394 struct wait_queue_head poll_wq; 393 395 struct io_restriction restrictions; 394 396 395 - struct io_zcrx_ifq *ifq; 397 + /* Stores zcrx object pointers of type struct io_zcrx_ifq */ 398 + struct xarray zcrx_ctxs; 396 399 397 400 u32 pers_next; 398 401 struct xarray personalities;

+2 -1

io_uring/io_uring.c

··· 359 359 INIT_LIST_HEAD(&ctx->tctx_list); 360 360 ctx->submit_state.free_list.next = NULL; 361 361 INIT_HLIST_HEAD(&ctx->waitid_list); 362 + xa_init_flags(&ctx->zcrx_ctxs, XA_FLAGS_ALLOC); 362 363 #ifdef CONFIG_FUTEX 363 364 INIT_HLIST_HEAD(&ctx->futex_list); 364 365 #endif ··· 2890 2889 io_cqring_overflow_kill(ctx); 2891 2890 mutex_unlock(&ctx->uring_lock); 2892 2891 } 2893 - if (ctx->ifq) { 2892 + if (!xa_empty(&ctx->zcrx_ctxs)) { 2894 2893 mutex_lock(&ctx->uring_lock); 2895 2894 io_shutdown_zcrx_ifqs(ctx); 2896 2895 mutex_unlock(&ctx->uring_lock);

+2 -3

io_uring/net.c

··· 1189 1189 return -EINVAL; 1190 1190 1191 1191 ifq_idx = READ_ONCE(sqe->zcrx_ifq_idx); 1192 - if (ifq_idx != 0) 1193 - return -EINVAL; 1194 - zc->ifq = req->ctx->ifq; 1192 + zc->ifq = xa_load(&req->ctx->zcrx_ctxs, ifq_idx); 1195 1193 if (!zc->ifq) 1196 1194 return -EINVAL; 1195 + 1197 1196 zc->len = READ_ONCE(sqe->len); 1198 1197 zc->flags = READ_ONCE(sqe->ioprio); 1199 1198 zc->msg_flags = READ_ONCE(sqe->msg_flags);

+50 -23

io_uring/zcrx.c

··· 156 156 157 157 static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, 158 158 struct io_uring_zcrx_ifq_reg *reg, 159 - struct io_uring_region_desc *rd) 159 + struct io_uring_region_desc *rd, 160 + u32 id) 160 161 { 162 + u64 mmap_offset; 161 163 size_t off, size; 162 164 void *ptr; 163 165 int ret; ··· 169 167 if (size > rd->size) 170 168 return -EINVAL; 171 169 172 - ret = io_create_region(ifq->ctx, &ifq->region, rd, IORING_MAP_OFF_ZCRX_REGION); 170 + mmap_offset = IORING_MAP_OFF_ZCRX_REGION; 171 + mmap_offset += id << IORING_OFF_PBUF_SHIFT; 172 + 173 + ret = io_create_region(ifq->ctx, &ifq->region, rd, mmap_offset); 173 174 if (ret < 0) 174 175 return ret; 175 176 ··· 184 179 185 180 static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq) 186 181 { 187 - if (WARN_ON_ONCE(ifq->ctx->ifq)) 188 - return; 189 - 190 182 io_free_region(ifq->ctx, &ifq->region); 191 183 ifq->rq_ring = NULL; 192 184 ifq->rqes = NULL; ··· 345 343 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 346 344 unsigned int id) 347 345 { 346 + struct io_zcrx_ifq *ifq = xa_load(&ctx->zcrx_ctxs, id); 347 + 348 348 lockdep_assert_held(&ctx->mmap_lock); 349 349 350 - if (id != 0 || !ctx->ifq) 351 - return NULL; 352 - return &ctx->ifq->region; 350 + return ifq ? &ifq->region : NULL; 353 351 } 354 352 355 353 int io_register_zcrx_ifq(struct io_ring_ctx *ctx, ··· 361 359 struct io_uring_region_desc rd; 362 360 struct io_zcrx_ifq *ifq; 363 361 int ret; 362 + u32 id; 364 363 365 364 /* 366 365 * 1. Interface queue allocation. ··· 374 371 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN && 375 372 ctx->flags & IORING_SETUP_CQE32)) 376 373 return -EINVAL; 377 - if (ctx->ifq) 378 - return -EBUSY; 379 374 if (copy_from_user(&reg, arg, sizeof(reg))) 380 375 return -EFAULT; 381 376 if (copy_from_user(&rd, u64_to_user_ptr(reg.region_ptr), sizeof(rd))) ··· 397 396 if (!ifq) 398 397 return -ENOMEM; 399 398 400 - ret = io_allocate_rbuf_ring(ifq, &reg, &rd); 399 + scoped_guard(mutex, &ctx->mmap_lock) { 400 + /* preallocate id */ 401 + ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL); 402 + if (ret) 403 + goto ifq_free; 404 + } 405 + 406 + ret = io_allocate_rbuf_ring(ifq, &reg, &rd, id); 401 407 if (ret) 402 408 goto err; 403 409 ··· 436 428 reg.offsets.rqes = sizeof(struct io_uring); 437 429 reg.offsets.head = offsetof(struct io_uring, head); 438 430 reg.offsets.tail = offsetof(struct io_uring, tail); 431 + reg.zcrx_id = id; 432 + 433 + scoped_guard(mutex, &ctx->mmap_lock) { 434 + /* publish ifq */ 435 + ret = -ENOMEM; 436 + if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL)) 437 + goto err; 438 + } 439 439 440 440 if (copy_to_user(arg, &reg, sizeof(reg)) || 441 441 copy_to_user(u64_to_user_ptr(reg.region_ptr), &rd, sizeof(rd)) || ··· 451 435 ret = -EFAULT; 452 436 goto err; 453 437 } 454 - scoped_guard(mutex, &ctx->mmap_lock) 455 - ctx->ifq = ifq; 456 438 return 0; 457 439 err: 440 + scoped_guard(mutex, &ctx->mmap_lock) 441 + xa_erase(&ctx->zcrx_ctxs, id); 442 + ifq_free: 458 443 io_zcrx_ifq_free(ifq); 459 444 return ret; 460 445 } 461 446 462 447 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) 463 448 { 464 - struct io_zcrx_ifq *ifq = ctx->ifq; 449 + struct io_zcrx_ifq *ifq; 450 + unsigned long id; 465 451 466 452 lockdep_assert_held(&ctx->uring_lock); 467 453 468 - if (!ifq) 469 - return; 454 + while (1) { 455 + scoped_guard(mutex, &ctx->mmap_lock) { 456 + ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); 457 + if (ifq) 458 + xa_erase(&ctx->zcrx_ctxs, id); 459 + } 460 + if (!ifq) 461 + break; 462 + io_zcrx_ifq_free(ifq); 463 + } 470 464 471 - scoped_guard(mutex, &ctx->mmap_lock) 472 - ctx->ifq = NULL; 473 - io_zcrx_ifq_free(ifq); 465 + xa_destroy(&ctx->zcrx_ctxs); 474 466 } 475 467 476 468 static struct net_iov *__io_zcrx_get_free_niov(struct io_zcrx_area *area) ··· 535 511 536 512 void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx) 537 513 { 514 + struct io_zcrx_ifq *ifq; 515 + unsigned long index; 516 + 538 517 lockdep_assert_held(&ctx->uring_lock); 539 518 540 - if (!ctx->ifq) 541 - return; 542 - io_zcrx_scrub(ctx->ifq); 543 - io_close_queue(ctx->ifq); 519 + xa_for_each(&ctx->zcrx_ctxs, index, ifq) { 520 + io_zcrx_scrub(ifq); 521 + io_close_queue(ifq); 522 + } 544 523 } 545 524 546 525 static inline u32 io_zcrx_rqring_entries(struct io_zcrx_ifq *ifq)

Configure Feed

Configure Feed