tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

io_uring: keep ring laoyut in a structure

Add a structure keeping SQ/CQ sizes and offsets. For now it only records
data previously returned from rings_size and the SQ size.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Pavel Begunkov and committed by
Jens Axboe 001b76b7 0f4b5373

+45 -45
+28 -26
io_uring/io_uring.c
··· 2757 2757 ctx->sq_sqes = NULL; 2758 2758 } 2759 2759 2760 - unsigned long rings_size(unsigned int flags, unsigned int sq_entries, 2761 - unsigned int cq_entries, size_t *sq_offset) 2760 + int rings_size(unsigned int flags, unsigned int sq_entries, 2761 + unsigned int cq_entries, struct io_rings_layout *rl) 2762 2762 { 2763 2763 struct io_rings *rings; 2764 + size_t sqe_size; 2764 2765 size_t off; 2765 - 2766 - *sq_offset = SIZE_MAX; 2767 2766 2768 2767 if (flags & IORING_SETUP_CQE_MIXED) { 2769 2768 if (cq_entries < 2) 2770 - return SIZE_MAX; 2769 + return -EOVERFLOW; 2771 2770 } 2772 2771 if (flags & IORING_SETUP_SQE_MIXED) { 2773 2772 if (sq_entries < 2) 2774 - return SIZE_MAX; 2773 + return -EOVERFLOW; 2775 2774 } 2775 + 2776 + rl->sq_array_offset = SIZE_MAX; 2777 + 2778 + sqe_size = sizeof(struct io_uring_sqe); 2779 + if (flags & IORING_SETUP_SQE128) 2780 + sqe_size *= 2; 2781 + 2782 + rl->sq_size = array_size(sqe_size, sq_entries); 2783 + if (rl->sq_size == SIZE_MAX) 2784 + return -EOVERFLOW; 2776 2785 2777 2786 off = struct_size(rings, cqes, cq_entries); 2778 2787 if (flags & IORING_SETUP_CQE32) 2779 2788 off = size_mul(off, 2); 2780 2789 if (off == SIZE_MAX) 2781 - return SIZE_MAX; 2790 + return -EOVERFLOW; 2782 2791 2783 2792 #ifdef CONFIG_SMP 2784 2793 off = ALIGN(off, SMP_CACHE_BYTES); 2785 2794 if (off == 0) 2786 - return SIZE_MAX; 2795 + return -EOVERFLOW; 2787 2796 #endif 2788 2797 2789 2798 if (!(flags & IORING_SETUP_NO_SQARRAY)) { 2790 2799 size_t sq_array_size; 2791 2800 2792 - *sq_offset = off; 2801 + rl->sq_array_offset = off; 2793 2802 2794 2803 sq_array_size = array_size(sizeof(u32), sq_entries); 2795 2804 off = size_add(off, sq_array_size); 2796 2805 if (off == SIZE_MAX) 2797 - return SIZE_MAX; 2806 + return -EOVERFLOW; 2798 2807 } 2799 2808 2800 - return off; 2809 + rl->rings_size = off; 2810 + return 0; 2801 2811 } 2802 2812 2803 2813 static __cold void __io_req_caches_free(struct io_ring_ctx *ctx) ··· 3356 3346 struct io_uring_params *p) 3357 3347 { 3358 3348 struct io_uring_region_desc rd; 3349 + struct io_rings_layout __rl, *rl = &__rl; 3359 3350 struct io_rings *rings; 3360 - size_t sq_array_offset; 3361 - size_t sq_size, cq_size, sqe_size; 3362 3351 int ret; 3363 3352 3364 3353 /* make sure these are sane, as we already accounted them */ 3365 3354 ctx->sq_entries = p->sq_entries; 3366 3355 ctx->cq_entries = p->cq_entries; 3367 3356 3368 - sqe_size = sizeof(struct io_uring_sqe); 3369 - if (p->flags & IORING_SETUP_SQE128) 3370 - sqe_size *= 2; 3371 - sq_size = array_size(sqe_size, p->sq_entries); 3372 - if (sq_size == SIZE_MAX) 3373 - return -EOVERFLOW; 3374 - cq_size = rings_size(ctx->flags, p->sq_entries, p->cq_entries, 3375 - &sq_array_offset); 3376 - if (cq_size == SIZE_MAX) 3377 - return -EOVERFLOW; 3357 + ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl); 3358 + if (ret) 3359 + return ret; 3378 3360 3379 3361 memset(&rd, 0, sizeof(rd)); 3380 - rd.size = PAGE_ALIGN(cq_size); 3362 + rd.size = PAGE_ALIGN(rl->rings_size); 3381 3363 if (ctx->flags & IORING_SETUP_NO_MMAP) { 3382 3364 rd.user_addr = p->cq_off.user_addr; 3383 3365 rd.flags |= IORING_MEM_REGION_TYPE_USER; ··· 3380 3378 ctx->rings = rings = io_region_get_ptr(&ctx->ring_region); 3381 3379 3382 3380 if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) 3383 - ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); 3381 + ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset); 3384 3382 3385 3383 memset(&rd, 0, sizeof(rd)); 3386 - rd.size = PAGE_ALIGN(sq_size); 3384 + rd.size = PAGE_ALIGN(rl->sq_size); 3387 3385 if (ctx->flags & IORING_SETUP_NO_MMAP) { 3388 3386 rd.user_addr = p->sq_off.user_addr; 3389 3387 rd.flags |= IORING_MEM_REGION_TYPE_USER;
+10 -2
io_uring/io_uring.h
··· 17 17 #include <trace/events/io_uring.h> 18 18 #endif 19 19 20 + struct io_rings_layout { 21 + /* size of CQ + headers + SQ offset array */ 22 + size_t rings_size; 23 + size_t sq_size; 24 + 25 + size_t sq_array_offset; 26 + }; 27 + 20 28 struct io_ctx_config { 21 29 struct io_uring_params p; 22 30 struct io_uring_params __user *uptr; ··· 147 139 #define IORING_MAX_ENTRIES 32768 148 140 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) 149 141 150 - unsigned long rings_size(unsigned int flags, unsigned int sq_entries, 151 - unsigned int cq_entries, size_t *sq_offset); 142 + int rings_size(unsigned int flags, unsigned int sq_entries, 143 + unsigned int cq_entries, struct io_rings_layout *rl); 152 144 int io_prepare_config(struct io_ctx_config *config); 153 145 154 146 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
+7 -17
io_uring/register.c
··· 401 401 struct io_ctx_config config; 402 402 struct io_uring_region_desc rd; 403 403 struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL; 404 - size_t size, sq_array_offset; 405 404 unsigned i, tail, old_head; 406 405 struct io_uring_params *p = &config.p; 406 + struct io_rings_layout __rl, *rl = &__rl; 407 407 int ret; 408 408 409 409 memset(&config, 0, sizeof(config)); ··· 423 423 if (unlikely(ret)) 424 424 return ret; 425 425 426 - size = rings_size(p->flags, p->sq_entries, p->cq_entries, 427 - &sq_array_offset); 428 - if (size == SIZE_MAX) 429 - return -EOVERFLOW; 426 + ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl); 427 + if (ret) 428 + return ret; 430 429 431 430 memset(&rd, 0, sizeof(rd)); 432 - rd.size = PAGE_ALIGN(size); 431 + rd.size = PAGE_ALIGN(rl->rings_size); 433 432 if (p->flags & IORING_SETUP_NO_MMAP) { 434 433 rd.user_addr = p->cq_off.user_addr; 435 434 rd.flags |= IORING_MEM_REGION_TYPE_USER; ··· 457 458 return -EFAULT; 458 459 } 459 460 460 - if (p->flags & IORING_SETUP_SQE128) 461 - size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries); 462 - else 463 - size = array_size(sizeof(struct io_uring_sqe), p->sq_entries); 464 - if (size == SIZE_MAX) { 465 - io_register_free_rings(ctx, &n); 466 - return -EOVERFLOW; 467 - } 468 - 469 461 memset(&rd, 0, sizeof(rd)); 470 - rd.size = PAGE_ALIGN(size); 462 + rd.size = PAGE_ALIGN(rl->sq_size); 471 463 if (p->flags & IORING_SETUP_NO_MMAP) { 472 464 rd.user_addr = p->sq_off.user_addr; 473 465 rd.flags |= IORING_MEM_REGION_TYPE_USER; ··· 541 551 542 552 /* all done, store old pointers and assign new ones */ 543 553 if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) 544 - ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset); 554 + ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset); 545 555 546 556 ctx->sq_entries = p->sq_entries; 547 557 ctx->cq_entries = p->cq_entries;