Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'zerocopy-tx-cleanups'
Pavel Begunkov says:
====================
zerocopy tx cleanups
Assorted zerocopy send path cleanups, the main part of which is
moving some net stack specific accounting out of io_uring back
to net/ in Patch 4.
====================
Link: https://patch.msgid.link/cover.1719190216.git.asml.silence@gmail.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
+39
-35
+3
include/linux/skbuff.h
+3
include/linux/skbuff.h
···
1703
1703
struct sk_buff *skb, struct iov_iter *from,
1704
1704
size_t length);
1705
1705
1706
+
int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
1707
+
struct iov_iter *from, size_t length);
1708
+
1706
1709
static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb,
1707
1710
struct msghdr *msg, int len)
1708
1711
{
+1
-1
include/linux/socket.h
+1
-1
include/linux/socket.h
···
76
76
__kernel_size_t msg_controllen; /* ancillary data buffer length */
77
77
struct kiocb *msg_iocb; /* ptr to iocb for async requests */
78
78
struct ubuf_info *msg_ubuf;
79
-
int (*sg_from_iter)(struct sock *sk, struct sk_buff *skb,
79
+
int (*sg_from_iter)(struct sk_buff *skb,
80
80
struct iov_iter *from, size_t length);
81
81
};
82
82
+4
-12
io_uring/net.c
+4
-12
io_uring/net.c
···
1265
1265
return io_sendmsg_prep_setup(req, req->opcode == IORING_OP_SENDMSG_ZC);
1266
1266
}
1267
1267
1268
-
static int io_sg_from_iter_iovec(struct sock *sk, struct sk_buff *skb,
1268
+
static int io_sg_from_iter_iovec(struct sk_buff *skb,
1269
1269
struct iov_iter *from, size_t length)
1270
1270
{
1271
1271
skb_zcopy_downgrade_managed(skb);
1272
-
return __zerocopy_sg_from_iter(NULL, sk, skb, from, length);
1272
+
return zerocopy_fill_skb_from_iter(skb, from, length);
1273
1273
}
1274
1274
1275
-
static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb,
1275
+
static int io_sg_from_iter(struct sk_buff *skb,
1276
1276
struct iov_iter *from, size_t length)
1277
1277
{
1278
1278
struct skb_shared_info *shinfo = skb_shinfo(skb);
···
1285
1285
if (!frag)
1286
1286
shinfo->flags |= SKBFL_MANAGED_FRAG_REFS;
1287
1287
else if (unlikely(!skb_zcopy_managed(skb)))
1288
-
return __zerocopy_sg_from_iter(NULL, sk, skb, from, length);
1288
+
return zerocopy_fill_skb_from_iter(skb, from, length);
1289
1289
1290
1290
bi.bi_size = min(from->count, length);
1291
1291
bi.bi_bvec_done = from->iov_offset;
···
1312
1312
skb->data_len += copied;
1313
1313
skb->len += copied;
1314
1314
skb->truesize += truesize;
1315
-
1316
-
if (sk && sk->sk_type == SOCK_STREAM) {
1317
-
sk_wmem_queued_add(sk, truesize);
1318
-
if (!skb_zcopy_pure(skb))
1319
-
sk_mem_charge(sk, truesize);
1320
-
} else {
1321
-
refcount_add(truesize, &skb->sk->sk_wmem_alloc);
1322
-
}
1323
1315
return ret;
1324
1316
}
1325
1317
+28
-19
net/core/datagram.c
+28
-19
net/core/datagram.c
···
610
610
}
611
611
EXPORT_SYMBOL(skb_copy_datagram_from_iter);
612
612
613
-
int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
614
-
struct sk_buff *skb, struct iov_iter *from,
615
-
size_t length)
613
+
int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
614
+
struct iov_iter *from, size_t length)
616
615
{
617
-
int frag;
618
-
619
-
if (msg && msg->msg_ubuf && msg->sg_from_iter)
620
-
return msg->sg_from_iter(sk, skb, from, length);
621
-
622
-
frag = skb_shinfo(skb)->nr_frags;
616
+
int frag = skb_shinfo(skb)->nr_frags;
623
617
624
618
while (length && iov_iter_count(from)) {
625
619
struct page *head, *last_head = NULL;
···
621
627
int refs, order, n = 0;
622
628
size_t start;
623
629
ssize_t copied;
624
-
unsigned long truesize;
625
630
626
631
if (frag == MAX_SKB_FRAGS)
627
632
return -EMSGSIZE;
···
632
639
633
640
length -= copied;
634
641
635
-
truesize = PAGE_ALIGN(copied + start);
636
642
skb->data_len += copied;
637
643
skb->len += copied;
638
-
skb->truesize += truesize;
639
-
if (sk && sk->sk_type == SOCK_STREAM) {
640
-
sk_wmem_queued_add(sk, truesize);
641
-
if (!skb_zcopy_pure(skb))
642
-
sk_mem_charge(sk, truesize);
643
-
} else {
644
-
refcount_add(truesize, &skb->sk->sk_wmem_alloc);
645
-
}
644
+
skb->truesize += PAGE_ALIGN(copied + start);
646
645
647
646
head = compound_head(pages[n]);
648
647
order = compound_order(head);
···
676
691
page_ref_sub(last_head, refs);
677
692
}
678
693
return 0;
694
+
}
695
+
696
+
int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
697
+
struct sk_buff *skb, struct iov_iter *from,
698
+
size_t length)
699
+
{
700
+
unsigned long orig_size = skb->truesize;
701
+
unsigned long truesize;
702
+
int ret;
703
+
704
+
if (msg && msg->msg_ubuf && msg->sg_from_iter)
705
+
ret = msg->sg_from_iter(skb, from, length);
706
+
else
707
+
ret = zerocopy_fill_skb_from_iter(skb, from, length);
708
+
709
+
truesize = skb->truesize - orig_size;
710
+
if (sk && sk->sk_type == SOCK_STREAM) {
711
+
sk_wmem_queued_add(sk, truesize);
712
+
if (!skb_zcopy_pure(skb))
713
+
sk_mem_charge(sk, truesize);
714
+
} else {
715
+
refcount_add(truesize, &skb->sk->sk_wmem_alloc);
716
+
}
717
+
return ret;
679
718
}
680
719
EXPORT_SYMBOL(__zerocopy_sg_from_iter);
681
720
+3
-3
net/core/skbuff.c
+3
-3
net/core/skbuff.c
···
1871
1871
struct msghdr *msg, int len,
1872
1872
struct ubuf_info *uarg)
1873
1873
{
1874
-
struct ubuf_info *orig_uarg = skb_zcopy(skb);
1875
1874
int err, orig_len = skb->len;
1876
1875
1877
1876
if (uarg->ops->link_skb) {
···
1878
1879
if (err)
1879
1880
return err;
1880
1881
} else {
1882
+
struct ubuf_info *orig_uarg = skb_zcopy(skb);
1883
+
1881
1884
/* An skb can only point to one uarg. This edge case happens
1882
1885
* when TCP appends to an skb, but zerocopy_realloc triggered
1883
1886
* a new alloc.
···
1900
1899
return err;
1901
1900
}
1902
1901
1903
-
if (!uarg->ops->link_skb)
1904
-
skb_zcopy_set(skb, uarg, NULL);
1902
+
skb_zcopy_set(skb, uarg, NULL);
1905
1903
return skb->len - orig_len;
1906
1904
}
1907
1905
EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);