Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/kbuf: unify legacy buf provision and removal
Combine IORING_OP_PROVIDE_BUFFERS and IORING_OP_REMOVE_BUFFERS
->issue(), so that we can deduplicate ring locking and list lookups.
This way we further reduce code for legacy provided buffers. Locking is
also separated from buffer related handling, which makes it a bit
simpler with label jumps.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/f61af131622ad4337c2fb9f7c453d5b0102c7b90.1747150490.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Pavel Begunkov
and committed by
Jens Axboe
2b61bb1d
c724e801
+34
-53
+31
-48
io_uring/kbuf.c
+31
-48
io_uring/kbuf.c
···
450
450
return 0;
451
451
}
452
452
453
-
int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
454
-
{
455
-
struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
456
-
struct io_ring_ctx *ctx = req->ctx;
457
-
struct io_buffer_list *bl;
458
-
int ret = 0;
459
-
460
-
io_ring_submit_lock(ctx, issue_flags);
461
-
462
-
ret = -ENOENT;
463
-
bl = io_buffer_get_list(ctx, p->bgid);
464
-
if (bl) {
465
-
ret = -EINVAL;
466
-
/* can't use provide/remove buffers command on mapped buffers */
467
-
if (!(bl->flags & IOBL_BUF_RING))
468
-
ret = io_remove_buffers_legacy(ctx, bl, p->nbufs);
469
-
}
470
-
io_ring_submit_unlock(ctx, issue_flags);
471
-
if (ret < 0)
472
-
req_set_fail(req);
473
-
io_req_set_res(req, ret, 0);
474
-
return IOU_OK;
475
-
}
476
-
477
453
int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
478
454
{
479
455
unsigned long size, tmp_check;
···
511
535
return i ? 0 : -ENOMEM;
512
536
}
513
537
514
-
int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
538
+
static int __io_manage_buffers_legacy(struct io_kiocb *req,
539
+
struct io_buffer_list *bl)
540
+
{
541
+
struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
542
+
int ret;
543
+
544
+
if (!bl) {
545
+
if (req->opcode != IORING_OP_PROVIDE_BUFFERS)
546
+
return -ENOENT;
547
+
bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT);
548
+
if (!bl)
549
+
return -ENOMEM;
550
+
551
+
INIT_LIST_HEAD(&bl->buf_list);
552
+
ret = io_buffer_add_list(req->ctx, bl, p->bgid);
553
+
if (ret) {
554
+
kfree(bl);
555
+
return ret;
556
+
}
557
+
}
558
+
/* can't use provide/remove buffers command on mapped buffers */
559
+
if (bl->flags & IOBL_BUF_RING)
560
+
return -EINVAL;
561
+
if (req->opcode == IORING_OP_PROVIDE_BUFFERS)
562
+
return io_add_buffers(req->ctx, p, bl);
563
+
return io_remove_buffers_legacy(req->ctx, bl, p->nbufs);
564
+
}
565
+
566
+
int io_manage_buffers_legacy(struct io_kiocb *req, unsigned int issue_flags)
515
567
{
516
568
struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
517
569
struct io_ring_ctx *ctx = req->ctx;
518
570
struct io_buffer_list *bl;
519
-
int ret = 0;
571
+
int ret;
520
572
521
573
io_ring_submit_lock(ctx, issue_flags);
522
-
523
574
bl = io_buffer_get_list(ctx, p->bgid);
524
-
if (unlikely(!bl)) {
525
-
bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT);
526
-
if (!bl) {
527
-
ret = -ENOMEM;
528
-
goto err;
529
-
}
530
-
INIT_LIST_HEAD(&bl->buf_list);
531
-
ret = io_buffer_add_list(ctx, bl, p->bgid);
532
-
if (ret) {
533
-
kfree(bl);
534
-
goto err;
535
-
}
536
-
}
537
-
/* can't add buffers via this command for a mapped buffer ring */
538
-
if (bl->flags & IOBL_BUF_RING) {
539
-
ret = -EINVAL;
540
-
goto err;
541
-
}
542
-
543
-
ret = io_add_buffers(ctx, p, bl);
544
-
err:
575
+
ret = __io_manage_buffers_legacy(req, bl);
545
576
io_ring_submit_unlock(ctx, issue_flags);
546
577
547
578
if (ret < 0)
+1
-3
io_uring/kbuf.h
+1
-3
io_uring/kbuf.h
···
66
66
void io_destroy_buffers(struct io_ring_ctx *ctx);
67
67
68
68
int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
69
-
int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags);
70
-
71
69
int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
72
-
int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags);
70
+
int io_manage_buffers_legacy(struct io_kiocb *req, unsigned int issue_flags);
73
71
74
72
int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
75
73
int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
+2
-2
io_uring/opdef.c
+2
-2
io_uring/opdef.c
···
333
333
.audit_skip = 1,
334
334
.iopoll = 1,
335
335
.prep = io_provide_buffers_prep,
336
-
.issue = io_provide_buffers,
336
+
.issue = io_manage_buffers_legacy,
337
337
},
338
338
[IORING_OP_REMOVE_BUFFERS] = {
339
339
.audit_skip = 1,
340
340
.iopoll = 1,
341
341
.prep = io_remove_buffers_prep,
342
-
.issue = io_remove_buffers,
342
+
.issue = io_manage_buffers_legacy,
343
343
},
344
344
[IORING_OP_TEE] = {
345
345
.needs_file = 1,