Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
vhost/net: remove vhost_net_page_frag_refill()
The page frag in vhost_net_page_frag_refill() uses the
'struct page_frag' from skb_page_frag_refill(), but it's
implementation is similar to page_frag_alloc_align() now.
This patch removes vhost_net_page_frag_refill() by using
'struct page_frag_cache' instead of 'struct page_frag',
and allocating frag using page_frag_alloc_align().
The added benefit is that not only unifying the page frag
implementation a little, but also having about 0.5% performance
boost testing by using the vhost_net_test introduced in the
last patch.
Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com>
Acked-by: Jason Wang <jasowang@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
authored by
Yunsheng Lin
and committed by
Paolo Abeni
4051bd81
a0727489
+27
-64
+27
-64
drivers/vhost/net.c
+27
-64
drivers/vhost/net.c
···
141
141
unsigned tx_zcopy_err;
142
142
/* Flush in progress. Protected by tx vq lock. */
143
143
bool tx_flush;
144
-
/* Private page frag */
145
-
struct page_frag page_frag;
146
-
/* Refcount bias of page frag */
147
-
int refcnt_bias;
144
+
/* Private page frag cache */
145
+
struct page_frag_cache pf_cache;
148
146
};
149
147
150
148
static unsigned vhost_net_zcopy_mask __read_mostly;
···
653
655
!vhost_vq_avail_empty(vq->dev, vq);
654
656
}
655
657
656
-
static bool vhost_net_page_frag_refill(struct vhost_net *net, unsigned int sz,
657
-
struct page_frag *pfrag, gfp_t gfp)
658
-
{
659
-
if (pfrag->page) {
660
-
if (pfrag->offset + sz <= pfrag->size)
661
-
return true;
662
-
__page_frag_cache_drain(pfrag->page, net->refcnt_bias);
663
-
}
664
-
665
-
pfrag->offset = 0;
666
-
net->refcnt_bias = 0;
667
-
if (SKB_FRAG_PAGE_ORDER) {
668
-
/* Avoid direct reclaim but allow kswapd to wake */
669
-
pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
670
-
__GFP_COMP | __GFP_NOWARN |
671
-
__GFP_NORETRY | __GFP_NOMEMALLOC,
672
-
SKB_FRAG_PAGE_ORDER);
673
-
if (likely(pfrag->page)) {
674
-
pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
675
-
goto done;
676
-
}
677
-
}
678
-
pfrag->page = alloc_page(gfp);
679
-
if (likely(pfrag->page)) {
680
-
pfrag->size = PAGE_SIZE;
681
-
goto done;
682
-
}
683
-
return false;
684
-
685
-
done:
686
-
net->refcnt_bias = USHRT_MAX;
687
-
page_ref_add(pfrag->page, USHRT_MAX - 1);
688
-
return true;
689
-
}
690
-
691
658
#define VHOST_NET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
692
659
693
660
static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
···
662
699
struct vhost_net *net = container_of(vq->dev, struct vhost_net,
663
700
dev);
664
701
struct socket *sock = vhost_vq_get_backend(vq);
665
-
struct page_frag *alloc_frag = &net->page_frag;
666
702
struct virtio_net_hdr *gso;
667
703
struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp];
668
704
struct tun_xdp_hdr *hdr;
···
672
710
int sock_hlen = nvq->sock_hlen;
673
711
void *buf;
674
712
int copied;
713
+
int ret;
675
714
676
715
if (unlikely(len < nvq->sock_hlen))
677
716
return -EFAULT;
···
682
719
return -ENOSPC;
683
720
684
721
buflen += SKB_DATA_ALIGN(len + pad);
685
-
alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES);
686
-
if (unlikely(!vhost_net_page_frag_refill(net, buflen,
687
-
alloc_frag, GFP_KERNEL)))
722
+
buf = page_frag_alloc_align(&net->pf_cache, buflen, GFP_KERNEL,
723
+
SMP_CACHE_BYTES);
724
+
if (unlikely(!buf))
688
725
return -ENOMEM;
689
726
690
-
buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
691
-
copied = copy_page_from_iter(alloc_frag->page,
692
-
alloc_frag->offset +
693
-
offsetof(struct tun_xdp_hdr, gso),
694
-
sock_hlen, from);
695
-
if (copied != sock_hlen)
696
-
return -EFAULT;
727
+
copied = copy_from_iter(buf + offsetof(struct tun_xdp_hdr, gso),
728
+
sock_hlen, from);
729
+
if (copied != sock_hlen) {
730
+
ret = -EFAULT;
731
+
goto err;
732
+
}
697
733
698
734
hdr = buf;
699
735
gso = &hdr->gso;
···
705
743
vhost16_to_cpu(vq, gso->csum_start) +
706
744
vhost16_to_cpu(vq, gso->csum_offset) + 2);
707
745
708
-
if (vhost16_to_cpu(vq, gso->hdr_len) > len)
709
-
return -EINVAL;
746
+
if (vhost16_to_cpu(vq, gso->hdr_len) > len) {
747
+
ret = -EINVAL;
748
+
goto err;
749
+
}
710
750
}
711
751
712
752
len -= sock_hlen;
713
-
copied = copy_page_from_iter(alloc_frag->page,
714
-
alloc_frag->offset + pad,
715
-
len, from);
716
-
if (copied != len)
717
-
return -EFAULT;
753
+
copied = copy_from_iter(buf + pad, len, from);
754
+
if (copied != len) {
755
+
ret = -EFAULT;
756
+
goto err;
757
+
}
718
758
719
759
xdp_init_buff(xdp, buflen, NULL);
720
760
xdp_prepare_buff(xdp, buf, pad, len, true);
721
761
hdr->buflen = buflen;
722
762
723
-
--net->refcnt_bias;
724
-
alloc_frag->offset += buflen;
725
-
726
763
++nvq->batched_xdp;
727
764
728
765
return 0;
766
+
767
+
err:
768
+
page_frag_free(buf);
769
+
return ret;
729
770
}
730
771
731
772
static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
···
1318
1353
vqs[VHOST_NET_VQ_RX]);
1319
1354
1320
1355
f->private_data = n;
1321
-
n->page_frag.page = NULL;
1322
-
n->refcnt_bias = 0;
1356
+
n->pf_cache.va = NULL;
1323
1357
1324
1358
return 0;
1325
1359
}
···
1386
1422
kfree(n->vqs[VHOST_NET_VQ_RX].rxq.queue);
1387
1423
kfree(n->vqs[VHOST_NET_VQ_TX].xdp);
1388
1424
kfree(n->dev.vqs);
1389
-
if (n->page_frag.page)
1390
-
__page_frag_cache_drain(n->page_frag.page, n->refcnt_bias);
1425
+
page_frag_cache_drain(&n->pf_cache);
1391
1426
kvfree(n);
1392
1427
return 0;
1393
1428
}