net/mlx5e: SHAMPO, Switch to header memcpy

+8 -12

drivers/net/ethernet/mellanox/mlx5/core/en.h

··· 82 82 83 83 #define MLX5E_RX_MAX_HEAD (256) 84 84 #define MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE (8) 85 - #define MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE (9) 86 - #define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE (PAGE_SIZE >> MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE) 87 - #define MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE (PAGE_SHIFT - MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE) 85 + #define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE \ 86 + (PAGE_SIZE >> MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE) 87 + #define MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE \ 88 + (PAGE_SHIFT - MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE) 88 89 #define MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT (6) 89 90 #define MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT (12) 90 91 #define MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE (16) ··· 639 638 }; 640 639 641 640 struct mlx5e_shampo_hd { 642 - struct mlx5e_frag_page *pages; 643 641 u32 hd_per_wq; 644 - u32 hd_per_page; 645 - u16 hd_per_wqe; 646 - u8 log_hd_per_page; 647 - u8 log_hd_entry_size; 648 - unsigned long *bitmap; 649 - u16 pi; 650 - u16 ci; 651 - __be32 mkey_be; 642 + u32 hd_buf_size; 643 + u32 mkey; 644 + u32 nentries; 645 + DECLARE_FLEX_ARRAY(struct mlx5e_dma_info, hd_buf_pages); 652 646 }; 653 647 654 648 struct mlx5e_hw_gro_data {

-23

drivers/net/ethernet/mellanox/mlx5/core/en/params.c

··· 1068 1068 return hd_per_wq; 1069 1069 } 1070 1070 1071 - static u32 mlx5e_shampo_icosq_sz(struct mlx5_core_dev *mdev, 1072 - struct mlx5e_params *params, 1073 - struct mlx5e_rq_param *rq_param) 1074 - { 1075 - int max_num_of_umr_per_wqe, max_hd_per_wqe, max_ksm_per_umr, rest; 1076 - void *wqc = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); 1077 - int wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz)); 1078 - u32 wqebbs; 1079 - 1080 - max_ksm_per_umr = MLX5E_MAX_KSM_PER_WQE(mdev); 1081 - max_hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rq_param); 1082 - max_num_of_umr_per_wqe = max_hd_per_wqe / max_ksm_per_umr; 1083 - rest = max_hd_per_wqe % max_ksm_per_umr; 1084 - wqebbs = MLX5E_KSM_UMR_WQEBBS(max_ksm_per_umr) * max_num_of_umr_per_wqe; 1085 - if (rest) 1086 - wqebbs += MLX5E_KSM_UMR_WQEBBS(rest); 1087 - wqebbs *= wq_size; 1088 - return wqebbs; 1089 - } 1090 - 1091 1071 #define MLX5E_LRO_TIMEOUT_ARR_SIZE 4 1092 1072 1093 1073 u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout) ··· 1152 1172 1153 1173 wqebbs += max_xsk_wqebbs; 1154 1174 } 1155 - 1156 - if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) 1157 - wqebbs += mlx5e_shampo_icosq_sz(mdev, params, rqp); 1158 1175 1159 1176 /* UMR WQEs don't cross the page boundary, they are padded with NOPs. 1160 1177 * This padding is always smaller than the max WQE size. That gives us

-1

drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h

··· 65 65 enum mlx5e_icosq_wqe_type { 66 66 MLX5E_ICOSQ_WQE_NOP, 67 67 MLX5E_ICOSQ_WQE_UMR_RX, 68 - MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR, 69 68 #ifdef CONFIG_MLX5_EN_TLS 70 69 MLX5E_ICOSQ_WQE_UMR_TLS, 71 70 MLX5E_ICOSQ_WQE_SET_PSV_TLS,

+121 -166

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

··· 492 492 return err; 493 493 } 494 494 495 - static int mlx5e_create_umr_ksm_mkey(struct mlx5_core_dev *mdev, 496 - u64 nentries, u8 log_entry_size, 497 - u32 *umr_mkey) 498 - { 499 - int inlen; 500 - void *mkc; 501 - u32 *in; 502 - int err; 503 - 504 - inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 505 - 506 - in = kvzalloc(inlen, GFP_KERNEL); 507 - if (!in) 508 - return -ENOMEM; 509 - 510 - mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 511 - 512 - MLX5_SET(mkc, mkc, free, 1); 513 - MLX5_SET(mkc, mkc, umr_en, 1); 514 - MLX5_SET(mkc, mkc, lw, 1); 515 - MLX5_SET(mkc, mkc, lr, 1); 516 - MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KSM); 517 - mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 518 - MLX5_SET(mkc, mkc, qpn, 0xffffff); 519 - MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 520 - MLX5_SET(mkc, mkc, translations_octword_size, nentries); 521 - MLX5_SET(mkc, mkc, log_page_size, log_entry_size); 522 - MLX5_SET64(mkc, mkc, len, nentries << log_entry_size); 523 - err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 524 - 525 - kvfree(in); 526 - return err; 527 - } 528 - 529 495 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) 530 496 { 531 497 u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0; ··· 515 549 rq->mpwqe.umr_mode, xsk_chunk_size); 516 550 rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey); 517 551 return err; 518 - } 519 - 520 - static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev, 521 - u16 hd_per_wq, __be32 *umr_mkey) 522 - { 523 - u32 max_ksm_size = BIT(MLX5_CAP_GEN(mdev, log_max_klm_list_size)); 524 - u32 mkey; 525 - int err; 526 - 527 - if (max_ksm_size < hd_per_wq) { 528 - mlx5_core_err(mdev, "max ksm list size 0x%x is smaller than shampo header buffer list size 0x%x\n", 529 - max_ksm_size, hd_per_wq); 530 - return -EINVAL; 531 - } 532 - 533 - err = mlx5e_create_umr_ksm_mkey(mdev, hd_per_wq, 534 - MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE, 535 - &mkey); 536 - if (err) 537 - return err; 538 - 539 - *umr_mkey = cpu_to_be32(mkey); 540 - return 0; 541 552 } 542 553 543 554 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) ··· 697 754 xdp_frag_size); 698 755 } 699 756 700 - static int mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq *rq, u16 hd_per_wq, 701 - int node) 757 + static void mlx5e_release_rq_hd_pages(struct mlx5e_rq *rq, 758 + struct mlx5e_shampo_hd *shampo) 759 + 702 760 { 703 - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 761 + for (int i = 0; i < shampo->nentries; i++) { 762 + struct mlx5e_dma_info *info = &shampo->hd_buf_pages[i]; 704 763 705 - shampo->hd_per_wq = hd_per_wq; 764 + if (!info->page) 765 + continue; 706 766 707 - shampo->bitmap = bitmap_zalloc_node(hd_per_wq, GFP_KERNEL, node); 708 - shampo->pages = kvzalloc_node(array_size(hd_per_wq, 709 - sizeof(*shampo->pages)), 710 - GFP_KERNEL, node); 711 - if (!shampo->bitmap || !shampo->pages) 712 - goto err_nomem; 767 + dma_unmap_page(rq->pdev, info->addr, PAGE_SIZE, 768 + rq->buff.map_dir); 769 + __free_page(info->page); 770 + } 771 + } 772 + 773 + static int mlx5e_alloc_rq_hd_pages(struct mlx5e_rq *rq, int node, 774 + struct mlx5e_shampo_hd *shampo) 775 + { 776 + int err, i; 777 + 778 + for (i = 0; i < shampo->nentries; i++) { 779 + struct page *page = alloc_pages_node(node, GFP_KERNEL, 0); 780 + dma_addr_t addr; 781 + 782 + if (!page) { 783 + err = -ENOMEM; 784 + goto err_free_pages; 785 + } 786 + 787 + addr = dma_map_page(rq->pdev, page, 0, PAGE_SIZE, 788 + rq->buff.map_dir); 789 + err = dma_mapping_error(rq->pdev, addr); 790 + if (err) { 791 + __free_page(page); 792 + goto err_free_pages; 793 + } 794 + 795 + shampo->hd_buf_pages[i].page = page; 796 + shampo->hd_buf_pages[i].addr = addr; 797 + } 713 798 714 799 return 0; 715 800 716 - err_nomem: 717 - kvfree(shampo->pages); 718 - bitmap_free(shampo->bitmap); 801 + err_free_pages: 802 + mlx5e_release_rq_hd_pages(rq, shampo); 719 803 720 - return -ENOMEM; 804 + return err; 721 805 } 722 806 723 - static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq) 807 + static int mlx5e_create_rq_hd_mkey(struct mlx5_core_dev *mdev, 808 + struct mlx5e_shampo_hd *shampo) 724 809 { 725 - kvfree(rq->mpwqe.shampo->pages); 726 - bitmap_free(rq->mpwqe.shampo->bitmap); 810 + enum mlx5e_mpwrq_umr_mode umr_mode = MLX5E_MPWRQ_UMR_MODE_ALIGNED; 811 + struct mlx5_mtt *mtt; 812 + void *mkc, *in; 813 + int inlen, err; 814 + u32 octwords; 815 + 816 + octwords = mlx5e_mpwrq_umr_octowords(shampo->nentries, umr_mode); 817 + inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in), 818 + MLX5_OCTWORD, octwords); 819 + if (inlen < 0) 820 + return inlen; 821 + 822 + in = kvzalloc(inlen, GFP_KERNEL); 823 + if (!in) 824 + return -ENOMEM; 825 + 826 + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 827 + 828 + MLX5_SET(mkc, mkc, lw, 1); 829 + MLX5_SET(mkc, mkc, lr, 1); 830 + MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 831 + mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 832 + MLX5_SET(mkc, mkc, qpn, 0xffffff); 833 + MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 834 + MLX5_SET64(mkc, mkc, len, shampo->hd_buf_size); 835 + MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); 836 + MLX5_SET(mkc, mkc, translations_octword_size, octwords); 837 + MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 838 + octwords); 839 + 840 + mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 841 + for (int i = 0; i < shampo->nentries; i++) 842 + mtt[i].ptag = cpu_to_be64(shampo->hd_buf_pages[i].addr); 843 + 844 + err = mlx5_core_create_mkey(mdev, &shampo->mkey, in, inlen); 845 + 846 + kvfree(in); 847 + return err; 727 848 } 728 849 729 850 static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, 730 851 struct mlx5e_params *params, 731 852 struct mlx5e_rq_param *rqp, 732 853 struct mlx5e_rq *rq, 733 - u32 *pool_size, 734 854 int node) 735 855 { 736 - void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq); 737 - u8 log_hd_per_page, log_hd_entry_size; 738 - u16 hd_per_wq, hd_per_wqe; 739 - u32 hd_pool_size; 740 - int wq_size; 741 - int err; 856 + struct mlx5e_shampo_hd *shampo; 857 + int nentries, err, shampo_sz; 858 + u32 hd_per_wq, hd_buf_size; 742 859 743 860 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 744 861 return 0; 745 862 746 - rq->mpwqe.shampo = kvzalloc_node(sizeof(*rq->mpwqe.shampo), 747 - GFP_KERNEL, node); 748 - if (!rq->mpwqe.shampo) 863 + hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rqp); 864 + hd_buf_size = hd_per_wq * BIT(MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE); 865 + nentries = hd_buf_size / PAGE_SIZE; 866 + if (!nentries) { 867 + mlx5_core_err(mdev, "SHAMPO header buffer size %u < %lu\n", 868 + hd_buf_size, PAGE_SIZE); 869 + return -EINVAL; 870 + } 871 + 872 + shampo_sz = struct_size(shampo, hd_buf_pages, nentries); 873 + shampo = kvzalloc_node(shampo_sz, GFP_KERNEL, node); 874 + if (!shampo) 749 875 return -ENOMEM; 750 876 751 - /* split headers data structures */ 752 - hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rqp); 753 - err = mlx5e_rq_shampo_hd_info_alloc(rq, hd_per_wq, node); 877 + shampo->hd_per_wq = hd_per_wq; 878 + shampo->hd_buf_size = hd_buf_size; 879 + shampo->nentries = nentries; 880 + err = mlx5e_alloc_rq_hd_pages(rq, node, shampo); 754 881 if (err) 755 - goto err_shampo_hd_info_alloc; 882 + goto err_free; 756 883 757 - err = mlx5e_create_rq_hd_umr_mkey(mdev, hd_per_wq, 758 - &rq->mpwqe.shampo->mkey_be); 884 + err = mlx5e_create_rq_hd_mkey(mdev, shampo); 759 885 if (err) 760 - goto err_umr_mkey; 761 - 762 - hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rqp); 763 - wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz)); 764 - 765 - BUILD_BUG_ON(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE > PAGE_SHIFT); 766 - if (hd_per_wqe >= MLX5E_SHAMPO_WQ_HEADER_PER_PAGE) { 767 - log_hd_per_page = MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE; 768 - log_hd_entry_size = MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE; 769 - } else { 770 - log_hd_per_page = order_base_2(hd_per_wqe); 771 - log_hd_entry_size = order_base_2(PAGE_SIZE / hd_per_wqe); 772 - } 773 - 774 - rq->mpwqe.shampo->hd_per_wqe = hd_per_wqe; 775 - rq->mpwqe.shampo->hd_per_page = BIT(log_hd_per_page); 776 - rq->mpwqe.shampo->log_hd_per_page = log_hd_per_page; 777 - rq->mpwqe.shampo->log_hd_entry_size = log_hd_entry_size; 778 - 779 - hd_pool_size = (hd_per_wqe * wq_size) >> log_hd_per_page; 780 - 781 - if (netif_rxq_has_unreadable_mp(rq->netdev, rq->ix)) { 782 - /* Separate page pool for shampo headers */ 783 - struct page_pool_params pp_params = { }; 784 - 785 - pp_params.order = 0; 786 - pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 787 - pp_params.pool_size = hd_pool_size; 788 - pp_params.nid = node; 789 - pp_params.dev = rq->pdev; 790 - pp_params.napi = rq->cq.napi; 791 - pp_params.netdev = rq->netdev; 792 - pp_params.dma_dir = rq->buff.map_dir; 793 - pp_params.max_len = PAGE_SIZE; 794 - 795 - rq->hd_page_pool = page_pool_create(&pp_params); 796 - if (IS_ERR(rq->hd_page_pool)) { 797 - err = PTR_ERR(rq->hd_page_pool); 798 - rq->hd_page_pool = NULL; 799 - goto err_hds_page_pool; 800 - } 801 - } else { 802 - /* Common page pool, reserve space for headers. */ 803 - *pool_size += hd_pool_size; 804 - rq->hd_page_pool = NULL; 805 - } 886 + goto err_release_pages; 806 887 807 888 /* gro only data structures */ 808 889 rq->hw_gro_data = kvzalloc_node(sizeof(*rq->hw_gro_data), GFP_KERNEL, node); 809 890 if (!rq->hw_gro_data) { 810 891 err = -ENOMEM; 811 - goto err_hw_gro_data; 892 + goto err_destroy_mkey; 812 893 } 894 + 895 + rq->mpwqe.shampo = shampo; 813 896 814 897 return 0; 815 898 816 - err_hw_gro_data: 817 - page_pool_destroy(rq->hd_page_pool); 818 - err_hds_page_pool: 819 - mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.shampo->mkey_be)); 820 - err_umr_mkey: 821 - mlx5e_rq_shampo_hd_info_free(rq); 822 - err_shampo_hd_info_alloc: 823 - kvfree(rq->mpwqe.shampo); 899 + err_destroy_mkey: 900 + mlx5_core_destroy_mkey(mdev, shampo->mkey); 901 + err_release_pages: 902 + mlx5e_release_rq_hd_pages(rq, shampo); 903 + err_free: 904 + kvfree(shampo); 905 + 824 906 return err; 825 907 } 826 908 827 909 static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) 828 910 { 829 - if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 911 + struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 912 + 913 + if (!shampo) 830 914 return; 831 915 832 916 kvfree(rq->hw_gro_data); 833 - if (rq->hd_page_pool != rq->page_pool) 834 - page_pool_destroy(rq->hd_page_pool); 835 - mlx5e_rq_shampo_hd_info_free(rq); 836 - mlx5_core_destroy_mkey(rq->mdev, 837 - be32_to_cpu(rq->mpwqe.shampo->mkey_be)); 838 - kvfree(rq->mpwqe.shampo); 917 + mlx5_core_destroy_mkey(rq->mdev, shampo->mkey); 918 + mlx5e_release_rq_hd_pages(rq, shampo); 919 + kvfree(shampo); 839 920 } 840 921 841 922 static int mlx5e_alloc_rq(struct mlx5e_params *params, ··· 937 970 if (err) 938 971 goto err_rq_mkey; 939 972 940 - err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node); 973 + err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, node); 941 974 if (err) 942 975 goto err_free_mpwqe_info; 943 976 ··· 1132 1165 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 1133 1166 MLX5_SET(wq, wq, log_headers_buffer_entry_num, 1134 1167 order_base_2(rq->mpwqe.shampo->hd_per_wq)); 1135 - MLX5_SET(wq, wq, headers_mkey, 1136 - be32_to_cpu(rq->mpwqe.shampo->mkey_be)); 1168 + MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey); 1137 1169 } 1138 1170 1139 1171 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf, ··· 1292 1326 rq->mpwqe.actual_wq_head = wq->head; 1293 1327 rq->mpwqe.umr_in_progress = 0; 1294 1328 rq->mpwqe.umr_completed = 0; 1295 - 1296 - if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 1297 - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 1298 - u16 len; 1299 - 1300 - len = (shampo->pi - shampo->ci) & shampo->hd_per_wq; 1301 - mlx5e_shampo_fill_umr(rq, len); 1302 - } 1303 1329 } 1304 1330 1305 1331 void mlx5e_free_rx_descs(struct mlx5e_rq *rq) ··· 1314 1356 mlx5_wq_ll_pop(wq, wqe_ix_be, 1315 1357 &wqe->next.next_wqe_index); 1316 1358 } 1317 - 1318 - if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 1319 - mlx5e_shampo_dealloc_hd(rq); 1320 1359 } else { 1321 1360 struct mlx5_wq_cyc *wq = &rq->wqe.wq; 1322 1361 u16 missing = mlx5_wq_cyc_missing(wq);

+60 -283

drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

··· 611 611 mlx5_wq_ll_update_db_record(wq); 612 612 } 613 613 614 - /* This function returns the size of the continuous free space inside a bitmap 615 - * that starts from first and no longer than len including circular ones. 616 - */ 617 - static int bitmap_find_window(unsigned long *bitmap, int len, 618 - int bitmap_size, int first) 619 - { 620 - int next_one, count; 621 - 622 - next_one = find_next_bit(bitmap, bitmap_size, first); 623 - if (next_one == bitmap_size) { 624 - if (bitmap_size - first >= len) 625 - return len; 626 - next_one = find_next_bit(bitmap, bitmap_size, 0); 627 - count = next_one + bitmap_size - first; 628 - } else { 629 - count = next_one - first; 630 - } 631 - 632 - return min(len, count); 633 - } 634 - 635 - static void build_ksm_umr(struct mlx5e_icosq *sq, struct mlx5e_umr_wqe *umr_wqe, 636 - __be32 key, u16 offset, u16 ksm_len) 637 - { 638 - memset(umr_wqe, 0, offsetof(struct mlx5e_umr_wqe, inline_ksms)); 639 - umr_wqe->hdr.ctrl.opmod_idx_opcode = 640 - cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | 641 - MLX5_OPCODE_UMR); 642 - umr_wqe->hdr.ctrl.umr_mkey = key; 643 - umr_wqe->hdr.ctrl.qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) 644 - | MLX5E_KSM_UMR_DS_CNT(ksm_len)); 645 - umr_wqe->hdr.uctrl.flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; 646 - umr_wqe->hdr.uctrl.xlt_offset = cpu_to_be16(offset); 647 - umr_wqe->hdr.uctrl.xlt_octowords = cpu_to_be16(ksm_len); 648 - umr_wqe->hdr.uctrl.mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 649 - } 650 - 651 - static struct mlx5e_frag_page *mlx5e_shampo_hd_to_frag_page(struct mlx5e_rq *rq, 652 - int header_index) 653 - { 654 - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 655 - 656 - return &shampo->pages[header_index >> shampo->log_hd_per_page]; 657 - } 658 - 659 - static u64 mlx5e_shampo_hd_offset(struct mlx5e_rq *rq, int header_index) 660 - { 661 - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 662 - u32 hd_per_page = shampo->hd_per_page; 663 - 664 - return (header_index & (hd_per_page - 1)) << shampo->log_hd_entry_size; 665 - } 666 - 667 - static void mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index); 668 - 669 - static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, 670 - struct mlx5e_icosq *sq, 671 - u16 ksm_entries, u16 index) 672 - { 673 - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 674 - u16 pi, header_offset, err, wqe_bbs; 675 - u32 lkey = rq->mdev->mlx5e_res.hw_objs.mkey; 676 - struct mlx5e_umr_wqe *umr_wqe; 677 - int headroom, i; 678 - 679 - headroom = rq->buff.headroom; 680 - wqe_bbs = MLX5E_KSM_UMR_WQEBBS(ksm_entries); 681 - pi = mlx5e_icosq_get_next_pi(sq, wqe_bbs); 682 - umr_wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 683 - build_ksm_umr(sq, umr_wqe, shampo->mkey_be, index, ksm_entries); 684 - 685 - for (i = 0; i < ksm_entries; i++, index++) { 686 - struct mlx5e_frag_page *frag_page; 687 - u64 addr; 688 - 689 - frag_page = mlx5e_shampo_hd_to_frag_page(rq, index); 690 - header_offset = mlx5e_shampo_hd_offset(rq, index); 691 - if (!header_offset) { 692 - err = mlx5e_page_alloc_fragmented(rq->hd_page_pool, 693 - frag_page); 694 - if (err) 695 - goto err_unmap; 696 - } 697 - 698 - addr = page_pool_get_dma_addr_netmem(frag_page->netmem); 699 - umr_wqe->inline_ksms[i] = (struct mlx5_ksm) { 700 - .key = cpu_to_be32(lkey), 701 - .va = cpu_to_be64(addr + header_offset + headroom), 702 - }; 703 - } 704 - 705 - sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) { 706 - .wqe_type = MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR, 707 - .num_wqebbs = wqe_bbs, 708 - .shampo.len = ksm_entries, 709 - }; 710 - 711 - shampo->pi = (shampo->pi + ksm_entries) & (shampo->hd_per_wq - 1); 712 - sq->pc += wqe_bbs; 713 - sq->doorbell_cseg = &umr_wqe->hdr.ctrl; 714 - 715 - return 0; 716 - 717 - err_unmap: 718 - while (--i >= 0) { 719 - --index; 720 - header_offset = mlx5e_shampo_hd_offset(rq, index); 721 - if (!header_offset) { 722 - struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, index); 723 - 724 - mlx5e_page_release_fragmented(rq->hd_page_pool, 725 - frag_page); 726 - } 727 - } 728 - 729 - rq->stats->buff_alloc_err++; 730 - return err; 731 - } 732 - 733 - static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq) 734 - { 735 - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 736 - u16 ksm_entries, num_wqe, index, entries_before; 737 - struct mlx5e_icosq *sq = rq->icosq; 738 - int i, err, max_ksm_entries, len; 739 - 740 - max_ksm_entries = MLX5E_MAX_KSM_PER_WQE(rq->mdev); 741 - ksm_entries = bitmap_find_window(shampo->bitmap, 742 - shampo->hd_per_wqe, 743 - shampo->hd_per_wq, shampo->pi); 744 - ksm_entries = ALIGN_DOWN(ksm_entries, shampo->hd_per_page); 745 - if (!ksm_entries) 746 - return 0; 747 - 748 - /* pi is aligned to MLX5E_SHAMPO_WQ_HEADER_PER_PAGE */ 749 - index = shampo->pi; 750 - entries_before = shampo->hd_per_wq - index; 751 - 752 - if (unlikely(entries_before < ksm_entries)) 753 - num_wqe = DIV_ROUND_UP(entries_before, max_ksm_entries) + 754 - DIV_ROUND_UP(ksm_entries - entries_before, max_ksm_entries); 755 - else 756 - num_wqe = DIV_ROUND_UP(ksm_entries, max_ksm_entries); 757 - 758 - for (i = 0; i < num_wqe; i++) { 759 - len = (ksm_entries > max_ksm_entries) ? max_ksm_entries : 760 - ksm_entries; 761 - if (unlikely(index + len > shampo->hd_per_wq)) 762 - len = shampo->hd_per_wq - index; 763 - err = mlx5e_build_shampo_hd_umr(rq, sq, len, index); 764 - if (unlikely(err)) 765 - return err; 766 - index = (index + len) & (rq->mpwqe.shampo->hd_per_wq - 1); 767 - ksm_entries -= len; 768 - } 769 - 770 - return 0; 771 - } 772 - 773 614 static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) 774 615 { 775 616 struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix); ··· 623 782 u16 pi; 624 783 int err; 625 784 int i; 626 - 627 - if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 628 - err = mlx5e_alloc_rx_hd_mpwqe(rq); 629 - if (unlikely(err)) 630 - goto err; 631 - } 632 785 633 786 sync_locked = mlx5e_icosq_sync_lock(sq); 634 787 pi = mlx5e_icosq_get_next_pi(sq, rq->mpwqe.umr_wqebbs); ··· 687 852 688 853 bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe); 689 854 690 - err: 691 855 rq->stats->buff_alloc_err++; 692 856 693 857 return err; 694 - } 695 - 696 - static void 697 - mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index) 698 - { 699 - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 700 - 701 - if (((header_index + 1) & (shampo->hd_per_page - 1)) == 0) { 702 - struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index); 703 - 704 - mlx5e_page_release_fragmented(rq->hd_page_pool, frag_page); 705 - } 706 - clear_bit(header_index, shampo->bitmap); 707 - } 708 - 709 - void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq) 710 - { 711 - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 712 - int i; 713 - 714 - for_each_set_bit(i, shampo->bitmap, rq->mpwqe.shampo->hd_per_wq) 715 - mlx5e_free_rx_shampo_hd_entry(rq, i); 716 858 } 717 859 718 860 static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) ··· 784 972 sq->cc = sqcc; 785 973 } 786 974 787 - void mlx5e_shampo_fill_umr(struct mlx5e_rq *rq, int len) 788 - { 789 - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 790 - int end, from, full_len = len; 791 - 792 - end = shampo->hd_per_wq; 793 - from = shampo->ci; 794 - if (from + len > end) { 795 - len -= end - from; 796 - bitmap_set(shampo->bitmap, from, end - from); 797 - from = 0; 798 - } 799 - 800 - bitmap_set(shampo->bitmap, from, len); 801 - shampo->ci = (shampo->ci + full_len) & (shampo->hd_per_wq - 1); 802 - } 803 - 804 - static void mlx5e_handle_shampo_hd_umr(struct mlx5e_shampo_umr umr, 805 - struct mlx5e_icosq *sq) 806 - { 807 - struct mlx5e_channel *c = container_of(sq, struct mlx5e_channel, icosq); 808 - /* assume 1:1 relationship between RQ and icosq */ 809 - struct mlx5e_rq *rq = &c->rq; 810 - 811 - mlx5e_shampo_fill_umr(rq, umr.len); 812 - } 813 - 814 975 int mlx5e_poll_ico_cq(struct mlx5e_cq *cq) 815 976 { 816 977 struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq); ··· 843 1058 wi->umr.rq->mpwqe.umr_completed++; 844 1059 break; 845 1060 case MLX5E_ICOSQ_WQE_NOP: 846 - break; 847 - case MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR: 848 - mlx5e_handle_shampo_hd_umr(wi->shampo, sq); 849 1061 break; 850 1062 #ifdef CONFIG_MLX5_EN_TLS 851 1063 case MLX5E_ICOSQ_WQE_UMR_TLS: ··· 1031 1249 return (unsigned int)((unsigned char *)tcp + tcp->doff * 4 - skb->data); 1032 1250 } 1033 1251 1034 - static void *mlx5e_shampo_get_packet_hd(struct mlx5e_rq *rq, u16 header_index) 1035 - { 1036 - struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index); 1037 - u16 head_offset = mlx5e_shampo_hd_offset(rq, header_index); 1038 - void *addr = netmem_address(frag_page->netmem); 1039 - 1040 - return addr + head_offset + rq->buff.headroom; 1041 - } 1042 - 1043 1252 static void mlx5e_shampo_update_ipv4_udp_hdr(struct mlx5e_rq *rq, struct iphdr *ipv4) 1044 1253 { 1045 1254 int udp_off = rq->hw_gro_data->fk.control.thoff; ··· 1069 1296 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4; 1070 1297 } 1071 1298 1299 + static void mlx5e_shampo_get_hd_buf_info(struct mlx5e_rq *rq, 1300 + struct mlx5_cqe64 *cqe, 1301 + struct mlx5e_dma_info **di, 1302 + u32 *head_offset) 1303 + { 1304 + u32 header_index = mlx5e_shampo_get_cqe_header_index(rq, cqe); 1305 + struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 1306 + u32 di_index; 1307 + 1308 + di_index = header_index >> MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE; 1309 + *di = &shampo->hd_buf_pages[di_index]; 1310 + *head_offset = (header_index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) * 1311 + BIT(MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE); 1312 + } 1313 + 1314 + static void *mlx5e_shampo_get_hdr(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, 1315 + int len) 1316 + { 1317 + struct mlx5e_dma_info *di; 1318 + u32 head_offset; 1319 + 1320 + mlx5e_shampo_get_hd_buf_info(rq, cqe, &di, &head_offset); 1321 + 1322 + dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset, 1323 + len, rq->buff.map_dir); 1324 + 1325 + return page_address(di->page) + head_offset; 1326 + } 1327 + 1072 1328 static void mlx5e_shampo_update_fin_psh_flags(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, 1073 1329 struct tcphdr *skb_tcp_hd) 1074 1330 { 1075 - u16 header_index = mlx5e_shampo_get_cqe_header_index(rq, cqe); 1331 + int nhoff = ETH_HLEN + rq->hw_gro_data->fk.control.thoff; 1332 + int len = nhoff + sizeof(struct tcphdr); 1076 1333 struct tcphdr *last_tcp_hd; 1077 1334 void *last_hd_addr; 1078 1335 1079 - last_hd_addr = mlx5e_shampo_get_packet_hd(rq, header_index); 1080 - last_tcp_hd = last_hd_addr + ETH_HLEN + rq->hw_gro_data->fk.control.thoff; 1336 + last_hd_addr = mlx5e_shampo_get_hdr(rq, cqe, len); 1337 + last_tcp_hd = (struct tcphdr *)(last_hd_addr + nhoff); 1338 + 1081 1339 tcp_flag_word(skb_tcp_hd) |= tcp_flag_word(last_tcp_hd) & (TCP_FLAG_FIN | TCP_FLAG_PSH); 1082 1340 } 1083 1341 ··· 2107 2303 mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, 2108 2304 struct mlx5_cqe64 *cqe, u16 header_index) 2109 2305 { 2110 - struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index); 2111 - u16 head_offset = mlx5e_shampo_hd_offset(rq, header_index); 2112 - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 2113 2306 u16 head_size = cqe->shampo.header_size; 2114 - u16 rx_headroom = rq->buff.headroom; 2115 - struct sk_buff *skb = NULL; 2116 - dma_addr_t page_dma_addr; 2117 - dma_addr_t dma_addr; 2118 - void *hdr, *data; 2119 - u32 frag_size; 2307 + struct mlx5e_dma_info *di; 2308 + struct sk_buff *skb; 2309 + u32 head_offset; 2310 + int len; 2120 2311 2121 - page_dma_addr = page_pool_get_dma_addr_netmem(frag_page->netmem); 2122 - dma_addr = page_dma_addr + head_offset; 2123 - 2124 - hdr = netmem_address(frag_page->netmem) + head_offset; 2125 - data = hdr + rx_headroom; 2126 - frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + head_size); 2127 - 2128 - if (likely(frag_size <= BIT(shampo->log_hd_entry_size))) { 2129 - /* build SKB around header */ 2130 - dma_sync_single_range_for_cpu(rq->pdev, dma_addr, 0, frag_size, rq->buff.map_dir); 2131 - net_prefetchw(hdr); 2132 - net_prefetch(data); 2133 - skb = mlx5e_build_linear_skb(rq, hdr, frag_size, rx_headroom, head_size, 0); 2134 - if (unlikely(!skb)) 2135 - return NULL; 2136 - 2137 - frag_page->frags++; 2138 - } else { 2139 - /* allocate SKB and copy header for large header */ 2140 - rq->stats->gro_large_hds++; 2141 - skb = napi_alloc_skb(rq->cq.napi, 2142 - ALIGN(head_size, sizeof(long))); 2143 - if (unlikely(!skb)) { 2144 - rq->stats->buff_alloc_err++; 2145 - return NULL; 2146 - } 2147 - 2148 - net_prefetchw(skb->data); 2149 - mlx5e_copy_skb_header(rq, skb, frag_page->netmem, dma_addr, 2150 - head_offset + rx_headroom, 2151 - rx_headroom, head_size); 2152 - /* skb linear part was allocated with headlen and aligned to long */ 2153 - skb->tail += head_size; 2154 - skb->len += head_size; 2312 + len = ALIGN(head_size, sizeof(long)); 2313 + skb = napi_alloc_skb(rq->cq.napi, len); 2314 + if (unlikely(!skb)) { 2315 + rq->stats->buff_alloc_err++; 2316 + return NULL; 2155 2317 } 2318 + 2319 + net_prefetchw(skb->data); 2320 + 2321 + mlx5e_shampo_get_hd_buf_info(rq, cqe, &di, &head_offset); 2322 + mlx5e_copy_skb_header(rq, skb, page_to_netmem(di->page), di->addr, 2323 + head_offset, head_offset, len); 2324 + __skb_put(skb, head_size); 2156 2325 2157 2326 /* queue up for recycling/reuse */ 2158 2327 skb_mark_for_recycle(skb); ··· 2226 2449 * prevent the kernel from touching it. 2227 2450 */ 2228 2451 if (unlikely(netmem_is_net_iov(frag_page->netmem))) 2229 - goto free_hd_entry; 2452 + goto mpwrq_cqe_out; 2230 2453 *skb = mlx5e_skb_from_cqe_mpwrq_nonlinear(rq, wi, cqe, 2231 2454 cqe_bcnt, 2232 2455 data_offset, ··· 2234 2457 } 2235 2458 2236 2459 if (unlikely(!*skb)) 2237 - goto free_hd_entry; 2460 + goto mpwrq_cqe_out; 2238 2461 2239 2462 NAPI_GRO_CB(*skb)->count = 1; 2240 2463 skb_shinfo(*skb)->gso_size = cqe_bcnt - head_size; 2241 2464 } else { 2242 2465 NAPI_GRO_CB(*skb)->count++; 2466 + 2243 2467 if (NAPI_GRO_CB(*skb)->count == 2 && 2244 2468 rq->hw_gro_data->fk.basic.n_proto == htons(ETH_P_IP)) { 2245 - void *hd_addr = mlx5e_shampo_get_packet_hd(rq, header_index); 2246 - int nhoff = ETH_HLEN + rq->hw_gro_data->fk.control.thoff - 2247 - sizeof(struct iphdr); 2248 - struct iphdr *iph = (struct iphdr *)(hd_addr + nhoff); 2469 + int len = ETH_HLEN + rq->hw_gro_data->fk.control.thoff; 2470 + int nhoff = len - sizeof(struct iphdr); 2471 + void *last_hd_addr; 2472 + struct iphdr *iph; 2249 2473 2474 + last_hd_addr = mlx5e_shampo_get_hdr(rq, cqe, len); 2475 + iph = (struct iphdr *)(last_hd_addr + nhoff); 2250 2476 rq->hw_gro_data->second_ip_id = ntohs(iph->id); 2251 2477 } 2252 2478 } ··· 2271 2491 2272 2492 if (mlx5e_shampo_complete_rx_cqe(rq, cqe, cqe_bcnt, *skb)) { 2273 2493 *skb = NULL; 2274 - goto free_hd_entry; 2494 + goto mpwrq_cqe_out; 2275 2495 } 2276 2496 if (flush && rq->hw_gro_data->skb) 2277 2497 mlx5e_shampo_flush_skb(rq, cqe, match); 2278 - free_hd_entry: 2279 - if (likely(head_size)) 2280 - mlx5e_free_rx_shampo_hd_entry(rq, header_index); 2281 2498 mpwrq_cqe_out: 2282 2499 if (likely(wi->consumed_strides < rq->mpwqe.num_strides)) 2283 2500 return;

Configure Feed

Configure Feed