Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
io_uring/zctx: unify zerocopy issue variants
io_send_zc and io_sendmsg_zc started different but now the only real
difference between them is how registered buffers are imported and
which net helper we use. Avoid duplication and combine them into a
single function.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Pavel Begunkov
and committed by
Jens Axboe
403fec55
2f9965f5
+17
-78
+16
-76
io_uring/net.c
+16
-76
io_uring/net.c
···
1471
1471
return 0;
1472
1472
}
1473
1473
1474
-
int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
1475
-
{
1476
-
struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
1477
-
struct io_async_msghdr *kmsg = req->async_data;
1478
-
struct socket *sock;
1479
-
unsigned msg_flags;
1480
-
int ret, min_ret = 0;
1481
-
1482
-
sock = sock_from_file(req->file);
1483
-
if (unlikely(!sock))
1484
-
return -ENOTSOCK;
1485
-
if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags))
1486
-
return -EOPNOTSUPP;
1487
-
1488
-
if (!(req->flags & REQ_F_POLLED) &&
1489
-
(zc->flags & IORING_RECVSEND_POLL_FIRST))
1490
-
return -EAGAIN;
1491
-
1492
-
if (req->flags & REQ_F_IMPORT_BUFFER) {
1493
-
ret = io_send_zc_import(req, kmsg, issue_flags);
1494
-
if (unlikely(ret))
1495
-
return ret;
1496
-
}
1497
-
1498
-
msg_flags = zc->msg_flags;
1499
-
if (issue_flags & IO_URING_F_NONBLOCK)
1500
-
msg_flags |= MSG_DONTWAIT;
1501
-
if (msg_flags & MSG_WAITALL)
1502
-
min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1503
-
msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
1504
-
1505
-
kmsg->msg.msg_flags = msg_flags;
1506
-
kmsg->msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg;
1507
-
ret = sock_sendmsg(sock, &kmsg->msg);
1508
-
1509
-
if (unlikely(ret < min_ret)) {
1510
-
if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1511
-
return -EAGAIN;
1512
-
1513
-
if (ret > 0 && io_net_retry(sock, kmsg->msg.msg_flags)) {
1514
-
zc->done_io += ret;
1515
-
return -EAGAIN;
1516
-
}
1517
-
if (ret == -ERESTARTSYS)
1518
-
ret = -EINTR;
1519
-
req_set_fail(req);
1520
-
}
1521
-
1522
-
if (ret >= 0)
1523
-
ret += zc->done_io;
1524
-
else if (zc->done_io)
1525
-
ret = zc->done_io;
1526
-
1527
-
/*
1528
-
* If we're in io-wq we can't rely on tw ordering guarantees, defer
1529
-
* flushing notif to io_send_zc_cleanup()
1530
-
*/
1531
-
if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1532
-
io_notif_flush(zc->notif);
1533
-
zc->notif = NULL;
1534
-
io_req_msg_cleanup(req, 0);
1535
-
}
1536
-
io_req_set_res(req, ret, IORING_CQE_F_MORE);
1537
-
return IOU_COMPLETE;
1538
-
}
1539
-
1540
1474
int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
1541
1475
{
1542
1476
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
···
1479
1545
unsigned msg_flags;
1480
1546
int ret, min_ret = 0;
1481
1547
1482
-
if (req->flags & REQ_F_IMPORT_BUFFER) {
1483
-
ret = io_send_zc_import(req, kmsg, issue_flags);
1484
-
if (unlikely(ret))
1485
-
return ret;
1486
-
}
1487
-
1488
1548
sock = sock_from_file(req->file);
1489
1549
if (unlikely(!sock))
1490
1550
return -ENOTSOCK;
1491
1551
if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags))
1492
1552
return -EOPNOTSUPP;
1493
-
1494
1553
if (!(req->flags & REQ_F_POLLED) &&
1495
1554
(sr->flags & IORING_RECVSEND_POLL_FIRST))
1496
1555
return -EAGAIN;
1556
+
1557
+
if (req->flags & REQ_F_IMPORT_BUFFER) {
1558
+
ret = io_send_zc_import(req, kmsg, issue_flags);
1559
+
if (unlikely(ret))
1560
+
return ret;
1561
+
}
1497
1562
1498
1563
msg_flags = sr->msg_flags;
1499
1564
if (issue_flags & IO_URING_F_NONBLOCK)
···
1500
1567
if (msg_flags & MSG_WAITALL)
1501
1568
min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1502
1569
1503
-
kmsg->msg.msg_control_user = sr->msg_control;
1504
1570
kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;
1505
-
ret = __sys_sendmsg_sock(sock, &kmsg->msg, msg_flags);
1571
+
1572
+
if (req->opcode == IORING_OP_SEND_ZC) {
1573
+
msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
1574
+
kmsg->msg.msg_flags = msg_flags;
1575
+
ret = sock_sendmsg(sock, &kmsg->msg);
1576
+
} else {
1577
+
kmsg->msg.msg_control_user = sr->msg_control;
1578
+
ret = __sys_sendmsg_sock(sock, &kmsg->msg, msg_flags);
1579
+
}
1506
1580
1507
1581
if (unlikely(ret < min_ret)) {
1508
1582
if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1509
1583
return -EAGAIN;
1510
1584
1511
-
if (ret > 0 && io_net_retry(sock, msg_flags)) {
1585
+
if (ret > 0 && io_net_retry(sock, sr->msg_flags)) {
1512
1586
sr->done_io += ret;
1513
1587
return -EAGAIN;
1514
1588
}
-1
io_uring/net.h
-1
io_uring/net.h
···
50
50
int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
51
51
int io_connect(struct io_kiocb *req, unsigned int issue_flags);
52
52
53
-
int io_send_zc(struct io_kiocb *req, unsigned int issue_flags);
54
53
int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags);
55
54
int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
56
55
void io_send_zc_cleanup(struct io_kiocb *req);