io_uring: pre-calculate scq layout

Move ring layouts calculations into io_prepare_config(), so that more
misconfiguration checking can be done earlier before creating a ctx.
It also deduplicates some code with ring resizing. And as a bonus, now
it initialises params->sq_off.array, which is closer to all other user
offset init, and also applies it to ring resizing, which was previously
missing it.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Pavel Begunkov and committed by

Jens Axboe 7 months ago eb76ff6a 001b76b7

+15 -19

3 changed files

expand all

io_uring

io_uring.c

io_uring.h

+14 -13

io_uring/io_uring.c

··· 2757 2757 ctx->sq_sqes = NULL; 2758 2758 } 2759 2759 2760 - int rings_size(unsigned int flags, unsigned int sq_entries, 2761 - unsigned int cq_entries, struct io_rings_layout *rl) 2760 + static int rings_size(unsigned int flags, unsigned int sq_entries, 2761 + unsigned int cq_entries, struct io_rings_layout *rl) 2762 2762 { 2763 2763 struct io_rings *rings; 2764 2764 size_t sqe_size; ··· 3353 3353 } 3354 3354 3355 3355 static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, 3356 - struct io_uring_params *p) 3356 + struct io_ctx_config *config) 3357 3357 { 3358 + struct io_uring_params *p = &config->p; 3359 + struct io_rings_layout *rl = &config->layout; 3358 3360 struct io_uring_region_desc rd; 3359 - struct io_rings_layout __rl, *rl = &__rl; 3360 3361 struct io_rings *rings; 3361 3362 int ret; 3362 3363 3363 3364 /* make sure these are sane, as we already accounted them */ 3364 3365 ctx->sq_entries = p->sq_entries; 3365 3366 ctx->cq_entries = p->cq_entries; 3366 - 3367 - ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl); 3368 - if (ret) 3369 - return ret; 3370 3367 3371 3368 memset(&rd, 0, sizeof(rd)); 3372 3369 rd.size = PAGE_ALIGN(rl->rings_size); ··· 3375 3378 if (ret) 3376 3379 return ret; 3377 3380 ctx->rings = rings = io_region_get_ptr(&ctx->ring_region); 3378 - 3379 3381 if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) 3380 3382 ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset); 3381 3383 ··· 3556 3560 if (ret) 3557 3561 return ret; 3558 3562 3563 + ret = rings_size(p->flags, p->sq_entries, p->cq_entries, 3564 + &config->layout); 3565 + if (ret) 3566 + return ret; 3567 + 3568 + if (!(p->flags & IORING_SETUP_NO_SQARRAY)) 3569 + p->sq_off.array = config->layout.sq_array_offset; 3570 + 3559 3571 return 0; 3560 3572 } 3561 3573 ··· 3636 3632 mmgrab(current->mm); 3637 3633 ctx->mm_account = current->mm; 3638 3634 3639 - ret = io_allocate_scq_urings(ctx, p); 3635 + ret = io_allocate_scq_urings(ctx, config); 3640 3636 if (ret) 3641 3637 goto err; 3642 - 3643 - if (!(p->flags & IORING_SETUP_NO_SQARRAY)) 3644 - p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; 3645 3638 3646 3639 ret = io_sq_offload_create(ctx, p); 3647 3640 if (ret)

+1 -2

io_uring/io_uring.h

··· 27 27 28 28 struct io_ctx_config { 29 29 struct io_uring_params p; 30 + struct io_rings_layout layout; 30 31 struct io_uring_params __user *uptr; 31 32 }; 32 33 ··· 148 147 #define IORING_MAX_ENTRIES 32768 149 148 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) 150 149 151 - int rings_size(unsigned int flags, unsigned int sq_entries, 152 - unsigned int cq_entries, struct io_rings_layout *rl); 153 150 int io_prepare_config(struct io_ctx_config *config); 154 151 155 152 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);

-4

io_uring/register.c

··· 423 423 if (unlikely(ret)) 424 424 return ret; 425 425 426 - ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl); 427 - if (ret) 428 - return ret; 429 - 430 426 memset(&rd, 0, sizeof(rd)); 431 427 rd.size = PAGE_ALIGN(rl->rings_size); 432 428 if (p->flags & IORING_SETUP_NO_MMAP) {

Configure Feed

Configure Feed