Merge branch 'xsk-refactors-around-generic-xmit-side'

+63 -50

1 changed file

expand all

net

xdp

xsk.c

+63 -50

net/xdp/xsk.c

··· 618 618 sock_wfree(skb); 619 619 } 620 620 621 - static void xsk_set_destructor_arg(struct sk_buff *skb, u64 addr) 621 + static void xsk_skb_init_misc(struct sk_buff *skb, struct xdp_sock *xs, 622 + u64 addr) 622 623 { 623 624 BUILD_BUG_ON(sizeof(struct xsk_addr_head) > sizeof(skb->cb)); 624 625 INIT_LIST_HEAD(&XSKCB(skb)->addrs_list); 626 + skb->dev = xs->dev; 627 + skb->priority = READ_ONCE(xs->sk.sk_priority); 628 + skb->mark = READ_ONCE(xs->sk.sk_mark); 625 629 XSKCB(skb)->num_descs = 0; 630 + skb->destructor = xsk_destruct_skb; 626 631 skb_shinfo(skb)->destructor_arg = (void *)(uintptr_t)addr; 627 632 } 628 633 ··· 657 652 xsk_consume_skb(skb); 658 653 } 659 654 655 + static int xsk_skb_metadata(struct sk_buff *skb, void *buffer, 656 + struct xdp_desc *desc, struct xsk_buff_pool *pool, 657 + u32 hr) 658 + { 659 + struct xsk_tx_metadata *meta = NULL; 660 + 661 + if (unlikely(pool->tx_metadata_len == 0)) 662 + return -EINVAL; 663 + 664 + meta = buffer - pool->tx_metadata_len; 665 + if (unlikely(!xsk_buff_valid_tx_metadata(meta))) 666 + return -EINVAL; 667 + 668 + if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) { 669 + if (unlikely(meta->request.csum_start + 670 + meta->request.csum_offset + 671 + sizeof(__sum16) > desc->len)) 672 + return -EINVAL; 673 + 674 + skb->csum_start = hr + meta->request.csum_start; 675 + skb->csum_offset = meta->request.csum_offset; 676 + skb->ip_summed = CHECKSUM_PARTIAL; 677 + 678 + if (unlikely(pool->tx_sw_csum)) { 679 + int err; 680 + 681 + err = skb_checksum_help(skb); 682 + if (err) 683 + return err; 684 + } 685 + } 686 + 687 + if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME) 688 + skb->skb_mstamp_ns = meta->request.launch_time; 689 + xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta); 690 + 691 + return 0; 692 + } 693 + 660 694 static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, 661 695 struct xdp_desc *desc) 662 696 { ··· 708 664 int err, i; 709 665 u64 addr; 710 666 667 + addr = desc->addr; 668 + buffer = xsk_buff_raw_get_data(pool, addr); 669 + 711 670 if (!skb) { 712 671 hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(xs->dev->needed_headroom)); 713 672 ··· 720 673 721 674 skb_reserve(skb, hr); 722 675 723 - xsk_set_destructor_arg(skb, desc->addr); 676 + xsk_skb_init_misc(skb, xs, desc->addr); 677 + if (desc->options & XDP_TX_METADATA) { 678 + err = xsk_skb_metadata(skb, buffer, desc, pool, hr); 679 + if (unlikely(err)) 680 + return ERR_PTR(err); 681 + } 724 682 } else { 725 683 xsk_addr = kmem_cache_zalloc(xsk_tx_generic_cache, GFP_KERNEL); 726 684 if (!xsk_addr) ··· 739 687 list_add_tail(&xsk_addr->addr_node, &XSKCB(skb)->addrs_list); 740 688 } 741 689 742 - addr = desc->addr; 743 690 len = desc->len; 744 691 ts = pool->unaligned ? len : pool->chunk_size; 745 692 746 - buffer = xsk_buff_raw_get_data(pool, addr); 747 693 offset = offset_in_page(buffer); 748 694 addr = buffer - pool->addrs; 749 695 ··· 772 722 static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, 773 723 struct xdp_desc *desc) 774 724 { 775 - struct xsk_tx_metadata *meta = NULL; 776 725 struct net_device *dev = xs->dev; 777 726 struct sk_buff *skb = xs->skb; 778 - bool first_frag = false; 779 727 int err; 780 728 781 729 if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) { 782 730 skb = xsk_build_skb_zerocopy(xs, desc); 783 731 if (IS_ERR(skb)) { 784 732 err = PTR_ERR(skb); 733 + skb = NULL; 785 734 goto free_err; 786 735 } 787 736 } else { ··· 791 742 len = desc->len; 792 743 793 744 if (!skb) { 794 - first_frag = true; 795 - 796 745 hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom)); 797 746 tr = dev->needed_tailroom; 798 747 skb = sock_alloc_send_skb(&xs->sk, hr + len + tr, 1, &err); ··· 804 757 if (unlikely(err)) 805 758 goto free_err; 806 759 807 - xsk_set_destructor_arg(skb, desc->addr); 760 + xsk_skb_init_misc(skb, xs, desc->addr); 761 + if (desc->options & XDP_TX_METADATA) { 762 + err = xsk_skb_metadata(skb, buffer, desc, 763 + xs->pool, hr); 764 + if (unlikely(err)) 765 + goto free_err; 766 + } 808 767 } else { 809 768 int nr_frags = skb_shinfo(skb)->nr_frags; 810 769 struct xsk_addr_node *xsk_addr; ··· 845 792 xsk_addr->addr = desc->addr; 846 793 list_add_tail(&xsk_addr->addr_node, &XSKCB(skb)->addrs_list); 847 794 } 848 - 849 - if (first_frag && desc->options & XDP_TX_METADATA) { 850 - if (unlikely(xs->pool->tx_metadata_len == 0)) { 851 - err = -EINVAL; 852 - goto free_err; 853 - } 854 - 855 - meta = buffer - xs->pool->tx_metadata_len; 856 - if (unlikely(!xsk_buff_valid_tx_metadata(meta))) { 857 - err = -EINVAL; 858 - goto free_err; 859 - } 860 - 861 - if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) { 862 - if (unlikely(meta->request.csum_start + 863 - meta->request.csum_offset + 864 - sizeof(__sum16) > len)) { 865 - err = -EINVAL; 866 - goto free_err; 867 - } 868 - 869 - skb->csum_start = hr + meta->request.csum_start; 870 - skb->csum_offset = meta->request.csum_offset; 871 - skb->ip_summed = CHECKSUM_PARTIAL; 872 - 873 - if (unlikely(xs->pool->tx_sw_csum)) { 874 - err = skb_checksum_help(skb); 875 - if (err) 876 - goto free_err; 877 - } 878 - } 879 - 880 - if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME) 881 - skb->skb_mstamp_ns = meta->request.launch_time; 882 - } 883 795 } 884 796 885 - skb->dev = dev; 886 - skb->priority = READ_ONCE(xs->sk.sk_priority); 887 - skb->mark = READ_ONCE(xs->sk.sk_mark); 888 - skb->destructor = xsk_destruct_skb; 889 - xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta); 890 797 xsk_inc_num_desc(skb); 891 798 892 799 return skb; 893 800 894 801 free_err: 895 - if (first_frag && skb) 802 + if (skb && !skb_shinfo(skb)->nr_frags) 896 803 kfree_skb(skb); 897 804 898 805 if (err == -EOVERFLOW) {

Configure Feed

Configure Feed