tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge branch 'net-mlx5e-shampo-enable-hw-gro-once-more'

Tariq Toukan says:

====================
net/mlx5e: SHAMPO, Enable HW GRO once more

This series enables hardware GRO for ConnectX-7 and newer NICs.
SHAMPO stands for Split Header And Merge Payload Offload.

The first part of the series contains important fixes and improvements.

The second part reworks the HW GRO counters.

Lastly, HW GRO is perf optimized and enabled.

Here are the bandwidth numbers for a simple iperf3 test over a single rq
where the application and irq are pinned to the same CPU:

+---------+--------+--------+-----------+-------------+
| streams | SW GRO | HW GRO | Unit | Improvement |
+---------+--------+--------+-----------+-------------+
| 1 | 36 | 57 | Gbits/sec | 1.6 x |
| 4 | 34 | 50 | Gbits/sec | 1.5 x |
| 8 | 31 | 43 | Gbits/sec | 1.4 x |
+---------+--------+--------+-----------+-------------+

Benchmark details:
VM based setup
CPU: Intel(R) Xeon(R) Platinum 8380 CPU, 24 cores
NIC: ConnectX-7 100GbE
iperf3 and irq running on same CPU over a single receive queue
====================

Link: https://lore.kernel.org/r/20240603212219.1037656-1-tariqt@nvidia.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

Jakub Kicinski 7da375e2 ed20142e

+202 -178
+15 -9
Documentation/networking/device_drivers/ethernet/mellanox/mlx5/counters.rst
··· 189 189 190 190 * - `rx[i]_gro_packets` 191 191 - Number of received packets processed using hardware-accelerated GRO. The 192 - number of hardware GRO offloaded packets received on ring i. 192 + number of hardware GRO offloaded packets received on ring i. Only true GRO 193 + packets are counted: only packets that are in an SKB with a GRO count > 1. 193 194 - Acceleration 194 195 195 196 * - `rx[i]_gro_bytes` 196 197 - Number of received bytes processed using hardware-accelerated GRO. The 197 - number of hardware GRO offloaded bytes received on ring i. 198 + number of hardware GRO offloaded bytes received on ring i. Only true GRO 199 + packets are counted: only packets that are in an SKB with a GRO count > 1. 198 200 - Acceleration 199 201 200 202 * - `rx[i]_gro_skbs` 201 - - The number of receive SKBs constructed while performing 202 - hardware-accelerated GRO. 203 - - Informative 204 - 205 - * - `rx[i]_gro_match_packets` 206 - - Number of received packets processed using hardware-accelerated GRO that 207 - met the flow table match criteria. 203 + - The number of GRO SKBs constructed from hardware-accelerated GRO. Only SKBs 204 + with a GRO count > 1 are counted. 208 205 - Informative 209 206 210 207 * - `rx[i]_gro_large_hds` 211 208 - Number of receive packets using hardware-accelerated GRO that have large 212 209 headers that require additional memory to be allocated. 210 + - Informative 211 + 212 + * - `rx[i]_hds_nodata_packets` 213 + - Number of header only packets in header/data split mode [#accel]_. 214 + - Informative 215 + 216 + * - `rx[i]_hds_nodata_bytes` 217 + - Number of bytes for header only packets in header/data split mode 218 + [#accel]_. 213 219 - Informative 214 220 215 221 * - `rx[i]_lro_packets`
+2 -20
drivers/net/ethernet/mellanox/mlx5/core/en.h
··· 80 80 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 81 81 82 82 #define MLX5E_RX_MAX_HEAD (256) 83 + #define MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE (8) 83 84 #define MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE (9) 84 85 #define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE (PAGE_SIZE >> MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE) 85 86 #define MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE (64) ··· 146 145 #define MLX5E_TX_CQ_POLL_BUDGET 128 147 146 #define MLX5E_TX_XSK_POLL_BUDGET 64 148 147 #define MLX5E_SQ_RECOVER_MIN_INTERVAL 500 /* msecs */ 149 - 150 - #define MLX5E_KLM_UMR_WQE_SZ(sgl_len)\ 151 - (sizeof(struct mlx5e_umr_wqe) +\ 152 - (sizeof(struct mlx5_klm) * (sgl_len))) 153 - 154 - #define MLX5E_KLM_UMR_WQEBBS(klm_entries) \ 155 - (DIV_ROUND_UP(MLX5E_KLM_UMR_WQE_SZ(klm_entries), MLX5_SEND_WQE_BB)) 156 - 157 - #define MLX5E_KLM_UMR_DS_CNT(klm_entries)\ 158 - (DIV_ROUND_UP(MLX5E_KLM_UMR_WQE_SZ(klm_entries), MLX5_SEND_WQE_DS)) 159 - 160 - #define MLX5E_KLM_MAX_ENTRIES_PER_WQE(wqe_size)\ 161 - (((wqe_size) - sizeof(struct mlx5e_umr_wqe)) / sizeof(struct mlx5_klm)) 162 - 163 - #define MLX5E_KLM_ENTRIES_PER_WQE(wqe_size)\ 164 - ALIGN_DOWN(MLX5E_KLM_MAX_ENTRIES_PER_WQE(wqe_size), MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT) 165 - 166 - #define MLX5E_MAX_KLM_PER_WQE(mdev) \ 167 - MLX5E_KLM_ENTRIES_PER_WQE(MLX5_SEND_WQE_BB * mlx5e_get_max_sq_aligned_wqebbs(mdev)) 168 148 169 149 #define mlx5e_state_dereference(priv, p) \ 170 150 rcu_dereference_protected((p), lockdep_is_held(&(priv)->state_lock)) ··· 996 1014 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, 997 1015 enum mlx5e_mpwrq_umr_mode umr_mode); 998 1016 999 - void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close); 1017 + void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq); 1000 1018 void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); 1001 1019 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s); 1002 1020
+6 -6
drivers/net/ethernet/mellanox/mlx5/core/en/params.c
··· 1071 1071 struct mlx5e_params *params, 1072 1072 struct mlx5e_rq_param *rq_param) 1073 1073 { 1074 - int max_num_of_umr_per_wqe, max_hd_per_wqe, max_klm_per_umr, rest; 1074 + int max_num_of_umr_per_wqe, max_hd_per_wqe, max_ksm_per_umr, rest; 1075 1075 void *wqc = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); 1076 1076 int wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz)); 1077 1077 u32 wqebbs; 1078 1078 1079 - max_klm_per_umr = MLX5E_MAX_KLM_PER_WQE(mdev); 1079 + max_ksm_per_umr = MLX5E_MAX_KSM_PER_WQE(mdev); 1080 1080 max_hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rq_param); 1081 - max_num_of_umr_per_wqe = max_hd_per_wqe / max_klm_per_umr; 1082 - rest = max_hd_per_wqe % max_klm_per_umr; 1083 - wqebbs = MLX5E_KLM_UMR_WQEBBS(max_klm_per_umr) * max_num_of_umr_per_wqe; 1081 + max_num_of_umr_per_wqe = max_hd_per_wqe / max_ksm_per_umr; 1082 + rest = max_hd_per_wqe % max_ksm_per_umr; 1083 + wqebbs = MLX5E_KSM_UMR_WQEBBS(max_ksm_per_umr) * max_num_of_umr_per_wqe; 1084 1084 if (rest) 1085 - wqebbs += MLX5E_KLM_UMR_WQEBBS(rest); 1085 + wqebbs += MLX5E_KSM_UMR_WQEBBS(rest); 1086 1086 wqebbs *= wq_size; 1087 1087 return wqebbs; 1088 1088 }
+19
drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
··· 34 34 35 35 #define MLX5E_RX_ERR_CQE(cqe) (get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND) 36 36 37 + #define MLX5E_KSM_UMR_WQE_SZ(sgl_len)\ 38 + (sizeof(struct mlx5e_umr_wqe) +\ 39 + (sizeof(struct mlx5_ksm) * (sgl_len))) 40 + 41 + #define MLX5E_KSM_UMR_WQEBBS(ksm_entries) \ 42 + (DIV_ROUND_UP(MLX5E_KSM_UMR_WQE_SZ(ksm_entries), MLX5_SEND_WQE_BB)) 43 + 44 + #define MLX5E_KSM_UMR_DS_CNT(ksm_entries)\ 45 + (DIV_ROUND_UP(MLX5E_KSM_UMR_WQE_SZ(ksm_entries), MLX5_SEND_WQE_DS)) 46 + 47 + #define MLX5E_KSM_MAX_ENTRIES_PER_WQE(wqe_size)\ 48 + (((wqe_size) - sizeof(struct mlx5e_umr_wqe)) / sizeof(struct mlx5_ksm)) 49 + 50 + #define MLX5E_KSM_ENTRIES_PER_WQE(wqe_size)\ 51 + ALIGN_DOWN(MLX5E_KSM_MAX_ENTRIES_PER_WQE(wqe_size), MLX5_UMR_KSM_NUM_ENTRIES_ALIGNMENT) 52 + 53 + #define MLX5E_MAX_KSM_PER_WQE(mdev) \ 54 + MLX5E_KSM_ENTRIES_PER_WQE(MLX5_SEND_WQE_BB * mlx5e_get_max_sq_aligned_wqebbs(mdev)) 55 + 37 56 static inline 38 57 ktime_t mlx5e_cqe_ts_to_ns(cqe_ts_to_ns func, struct mlx5_clock *clock, u64 cqe_ts) 39 58 {
+48 -23
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
··· 74 74 #include "lib/devcom.h" 75 75 #include "lib/sd.h" 76 76 77 + static bool mlx5e_hw_gro_supported(struct mlx5_core_dev *mdev) 78 + { 79 + if (!MLX5_CAP_GEN(mdev, shampo)) 80 + return false; 81 + 82 + /* Our HW-GRO implementation relies on "KSM Mkey" for 83 + * SHAMPO headers buffer mapping 84 + */ 85 + if (!MLX5_CAP_GEN(mdev, fixed_buffer_size)) 86 + return false; 87 + 88 + if (!MLX5_CAP_GEN_2(mdev, min_mkey_log_entity_size_fixed_buffer_valid)) 89 + return false; 90 + 91 + if (MLX5_CAP_GEN_2(mdev, min_mkey_log_entity_size_fixed_buffer) > 92 + MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE) 93 + return false; 94 + 95 + return true; 96 + } 97 + 77 98 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, 78 99 enum mlx5e_mpwrq_umr_mode umr_mode) 79 100 { ··· 525 504 return err; 526 505 } 527 506 528 - static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev, 529 - u64 nentries, 507 + static int mlx5e_create_umr_ksm_mkey(struct mlx5_core_dev *mdev, 508 + u64 nentries, u8 log_entry_size, 530 509 u32 *umr_mkey) 531 510 { 532 511 int inlen; ··· 546 525 MLX5_SET(mkc, mkc, umr_en, 1); 547 526 MLX5_SET(mkc, mkc, lw, 1); 548 527 MLX5_SET(mkc, mkc, lr, 1); 549 - MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS); 528 + MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KSM); 550 529 mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 551 530 MLX5_SET(mkc, mkc, qpn, 0xffffff); 552 531 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 553 532 MLX5_SET(mkc, mkc, translations_octword_size, nentries); 554 - MLX5_SET(mkc, mkc, length64, 1); 533 + MLX5_SET(mkc, mkc, log_page_size, log_entry_size); 534 + MLX5_SET64(mkc, mkc, len, nentries << log_entry_size); 555 535 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 556 536 557 537 kvfree(in); ··· 587 565 static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev, 588 566 struct mlx5e_rq *rq) 589 567 { 590 - u32 max_klm_size = BIT(MLX5_CAP_GEN(mdev, log_max_klm_list_size)); 568 + u32 max_ksm_size = BIT(MLX5_CAP_GEN(mdev, log_max_klm_list_size)); 591 569 592 - if (max_klm_size < rq->mpwqe.shampo->hd_per_wq) { 593 - mlx5_core_err(mdev, "max klm list size 0x%x is smaller than shampo header buffer list size 0x%x\n", 594 - max_klm_size, rq->mpwqe.shampo->hd_per_wq); 570 + if (max_ksm_size < rq->mpwqe.shampo->hd_per_wq) { 571 + mlx5_core_err(mdev, "max ksm list size 0x%x is smaller than shampo header buffer list size 0x%x\n", 572 + max_ksm_size, rq->mpwqe.shampo->hd_per_wq); 595 573 return -EINVAL; 596 574 } 597 - return mlx5e_create_umr_klm_mkey(mdev, rq->mpwqe.shampo->hd_per_wq, 575 + 576 + return mlx5e_create_umr_ksm_mkey(mdev, rq->mpwqe.shampo->hd_per_wq, 577 + MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE, 598 578 &rq->mpwqe.shampo->mkey); 599 579 } 600 580 ··· 1232 1208 head = mlx5_wq_ll_get_wqe_next_ix(wq, head); 1233 1209 } 1234 1210 1235 - if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 1236 - u16 len; 1237 - 1238 - len = (rq->mpwqe.shampo->pi - rq->mpwqe.shampo->ci) & 1239 - (rq->mpwqe.shampo->hd_per_wq - 1); 1240 - mlx5e_shampo_dealloc_hd(rq, len, rq->mpwqe.shampo->ci, false); 1241 - rq->mpwqe.shampo->pi = rq->mpwqe.shampo->ci; 1242 - } 1243 - 1244 1211 rq->mpwqe.actual_wq_head = wq->head; 1245 1212 rq->mpwqe.umr_in_progress = 0; 1246 1213 rq->mpwqe.umr_completed = 0; ··· 1259 1244 } 1260 1245 1261 1246 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 1262 - mlx5e_shampo_dealloc_hd(rq, rq->mpwqe.shampo->hd_per_wq, 1263 - 0, true); 1247 + mlx5e_shampo_dealloc_hd(rq); 1264 1248 } else { 1265 1249 struct mlx5_wq_cyc *wq = &rq->wqe.wq; 1266 1250 u16 missing = mlx5_wq_cyc_missing(wq); ··· 4273 4259 #define MLX5E_HANDLE_FEATURE(feature, handler) \ 4274 4260 mlx5e_handle_feature(netdev, &oper_features, feature, handler) 4275 4261 4276 - err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 4277 - err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro); 4262 + if (features & (NETIF_F_GRO_HW | NETIF_F_LRO)) { 4263 + err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 4264 + err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 4265 + err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro); 4266 + } else { 4267 + err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 4268 + err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro); 4269 + err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 4270 + } 4278 4271 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER, 4279 4272 set_feature_cvlan_filter); 4280 4273 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc); 4281 4274 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all); 4282 - err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 4283 4275 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan); 4284 4276 #ifdef CONFIG_MLX5_EN_ARFS 4285 4277 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs); ··· 5351 5331 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 5352 5332 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 5353 5333 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 5334 + 5335 + if (mlx5e_hw_gro_supported(mdev) && 5336 + mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, 5337 + MLX5E_MPWRQ_UMR_MODE_ALIGNED)) 5338 + netdev->hw_features |= NETIF_F_GRO_HW; 5354 5339 5355 5340 if (mlx5e_tunnel_any_tx_proto_supported(mdev)) { 5356 5341 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
+92 -110
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
··· 523 523 524 524 static inline void 525 525 mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb, 526 - struct page *page, u32 frag_offset, u32 len, 526 + struct mlx5e_frag_page *frag_page, 527 + u32 frag_offset, u32 len, 527 528 unsigned int truesize) 528 529 { 529 - dma_addr_t addr = page_pool_get_dma_addr(page); 530 + dma_addr_t addr = page_pool_get_dma_addr(frag_page->page); 531 + u8 next_frag = skb_shinfo(skb)->nr_frags; 530 532 531 533 dma_sync_single_for_cpu(rq->pdev, addr + frag_offset, len, 532 534 rq->buff.map_dir); 533 - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 534 - page, frag_offset, len, truesize); 535 + 536 + if (skb_can_coalesce(skb, next_frag, frag_page->page, frag_offset)) { 537 + skb_coalesce_rx_frag(skb, next_frag - 1, len, truesize); 538 + } else { 539 + frag_page->frags++; 540 + skb_add_rx_frag(skb, next_frag, frag_page->page, 541 + frag_offset, len, truesize); 542 + } 535 543 } 536 544 537 545 static inline void ··· 627 619 return min(len, count); 628 620 } 629 621 630 - static void build_klm_umr(struct mlx5e_icosq *sq, struct mlx5e_umr_wqe *umr_wqe, 631 - __be32 key, u16 offset, u16 klm_len, u16 wqe_bbs) 622 + static void build_ksm_umr(struct mlx5e_icosq *sq, struct mlx5e_umr_wqe *umr_wqe, 623 + __be32 key, u16 offset, u16 ksm_len) 632 624 { 633 - memset(umr_wqe, 0, offsetof(struct mlx5e_umr_wqe, inline_klms)); 625 + memset(umr_wqe, 0, offsetof(struct mlx5e_umr_wqe, inline_ksms)); 634 626 umr_wqe->ctrl.opmod_idx_opcode = 635 627 cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | 636 628 MLX5_OPCODE_UMR); 637 629 umr_wqe->ctrl.umr_mkey = key; 638 630 umr_wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) 639 - | MLX5E_KLM_UMR_DS_CNT(klm_len)); 631 + | MLX5E_KSM_UMR_DS_CNT(ksm_len)); 640 632 umr_wqe->uctrl.flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; 641 633 umr_wqe->uctrl.xlt_offset = cpu_to_be16(offset); 642 - umr_wqe->uctrl.xlt_octowords = cpu_to_be16(klm_len); 634 + umr_wqe->uctrl.xlt_octowords = cpu_to_be16(ksm_len); 643 635 umr_wqe->uctrl.mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 644 636 } 645 637 646 638 static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, 647 639 struct mlx5e_icosq *sq, 648 - u16 klm_entries, u16 index) 640 + u16 ksm_entries, u16 index) 649 641 { 650 642 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 651 643 u16 entries, pi, header_offset, err, wqe_bbs, new_entries; ··· 658 650 int headroom, i; 659 651 660 652 headroom = rq->buff.headroom; 661 - new_entries = klm_entries - (shampo->pi & (MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT - 1)); 662 - entries = ALIGN(klm_entries, MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT); 663 - wqe_bbs = MLX5E_KLM_UMR_WQEBBS(entries); 653 + new_entries = ksm_entries - (shampo->pi & (MLX5_UMR_KSM_NUM_ENTRIES_ALIGNMENT - 1)); 654 + entries = ALIGN(ksm_entries, MLX5_UMR_KSM_NUM_ENTRIES_ALIGNMENT); 655 + wqe_bbs = MLX5E_KSM_UMR_WQEBBS(entries); 664 656 pi = mlx5e_icosq_get_next_pi(sq, wqe_bbs); 665 657 umr_wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 666 - build_klm_umr(sq, umr_wqe, shampo->key, index, entries, wqe_bbs); 658 + build_ksm_umr(sq, umr_wqe, shampo->key, index, entries); 667 659 668 660 frag_page = &shampo->pages[page_index]; 669 661 670 662 for (i = 0; i < entries; i++, index++) { 671 663 dma_info = &shampo->info[index]; 672 - if (i >= klm_entries || (index < shampo->pi && shampo->pi - index < 673 - MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT)) 674 - goto update_klm; 664 + if (i >= ksm_entries || (index < shampo->pi && shampo->pi - index < 665 + MLX5_UMR_KSM_NUM_ENTRIES_ALIGNMENT)) 666 + goto update_ksm; 675 667 header_offset = (index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) << 676 668 MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE; 677 669 if (!(header_offset & (PAGE_SIZE - 1))) { ··· 691 683 dma_info->frag_page = frag_page; 692 684 } 693 685 694 - update_klm: 695 - umr_wqe->inline_klms[i].bcount = 696 - cpu_to_be32(MLX5E_RX_MAX_HEAD); 697 - umr_wqe->inline_klms[i].key = cpu_to_be32(lkey); 698 - umr_wqe->inline_klms[i].va = 699 - cpu_to_be64(dma_info->addr + headroom); 686 + update_ksm: 687 + umr_wqe->inline_ksms[i] = (struct mlx5_ksm) { 688 + .key = cpu_to_be32(lkey), 689 + .va = cpu_to_be64(dma_info->addr + headroom), 690 + }; 700 691 } 701 692 702 693 sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) { ··· 727 720 static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq) 728 721 { 729 722 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 730 - u16 klm_entries, num_wqe, index, entries_before; 723 + u16 ksm_entries, num_wqe, index, entries_before; 731 724 struct mlx5e_icosq *sq = rq->icosq; 732 - int i, err, max_klm_entries, len; 725 + int i, err, max_ksm_entries, len; 733 726 734 - max_klm_entries = MLX5E_MAX_KLM_PER_WQE(rq->mdev); 735 - klm_entries = bitmap_find_window(shampo->bitmap, 727 + max_ksm_entries = MLX5E_MAX_KSM_PER_WQE(rq->mdev); 728 + ksm_entries = bitmap_find_window(shampo->bitmap, 736 729 shampo->hd_per_wqe, 737 730 shampo->hd_per_wq, shampo->pi); 738 - if (!klm_entries) 731 + if (!ksm_entries) 739 732 return 0; 740 733 741 - klm_entries += (shampo->pi & (MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT - 1)); 742 - index = ALIGN_DOWN(shampo->pi, MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT); 734 + ksm_entries += (shampo->pi & (MLX5_UMR_KSM_NUM_ENTRIES_ALIGNMENT - 1)); 735 + index = ALIGN_DOWN(shampo->pi, MLX5_UMR_KSM_NUM_ENTRIES_ALIGNMENT); 743 736 entries_before = shampo->hd_per_wq - index; 744 737 745 - if (unlikely(entries_before < klm_entries)) 746 - num_wqe = DIV_ROUND_UP(entries_before, max_klm_entries) + 747 - DIV_ROUND_UP(klm_entries - entries_before, max_klm_entries); 738 + if (unlikely(entries_before < ksm_entries)) 739 + num_wqe = DIV_ROUND_UP(entries_before, max_ksm_entries) + 740 + DIV_ROUND_UP(ksm_entries - entries_before, max_ksm_entries); 748 741 else 749 - num_wqe = DIV_ROUND_UP(klm_entries, max_klm_entries); 742 + num_wqe = DIV_ROUND_UP(ksm_entries, max_ksm_entries); 750 743 751 744 for (i = 0; i < num_wqe; i++) { 752 - len = (klm_entries > max_klm_entries) ? max_klm_entries : 753 - klm_entries; 745 + len = (ksm_entries > max_ksm_entries) ? max_ksm_entries : 746 + ksm_entries; 754 747 if (unlikely(index + len > shampo->hd_per_wq)) 755 748 len = shampo->hd_per_wq - index; 756 749 err = mlx5e_build_shampo_hd_umr(rq, sq, len, index); 757 750 if (unlikely(err)) 758 751 return err; 759 752 index = (index + len) & (rq->mpwqe.shampo->hd_per_wq - 1); 760 - klm_entries -= len; 753 + ksm_entries -= len; 761 754 } 762 755 763 756 return 0; ··· 846 839 return err; 847 840 } 848 841 849 - /* This function is responsible to dealloc SHAMPO header buffer. 850 - * close == true specifies that we are in the middle of closing RQ operation so 851 - * we go over all the entries and if they are not in use we free them, 852 - * otherwise we only go over a specific range inside the header buffer that are 853 - * not in use. 854 - */ 855 - void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close) 842 + static void 843 + mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index) 856 844 { 857 845 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 858 - struct mlx5e_frag_page *deleted_page = NULL; 859 - int hd_per_wq = shampo->hd_per_wq; 860 - struct mlx5e_dma_info *hd_info; 861 - int i, index = start; 846 + u64 addr = shampo->info[header_index].addr; 862 847 863 - for (i = 0; i < len; i++, index++) { 864 - if (index == hd_per_wq) 865 - index = 0; 848 + if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) { 849 + struct mlx5e_dma_info *dma_info = &shampo->info[header_index]; 866 850 867 - if (close && !test_bit(index, shampo->bitmap)) 868 - continue; 869 - 870 - hd_info = &shampo->info[index]; 871 - hd_info->addr = ALIGN_DOWN(hd_info->addr, PAGE_SIZE); 872 - if (hd_info->frag_page && hd_info->frag_page != deleted_page) { 873 - deleted_page = hd_info->frag_page; 874 - mlx5e_page_release_fragmented(rq, hd_info->frag_page); 875 - } 876 - 877 - hd_info->frag_page = NULL; 851 + dma_info->addr = ALIGN_DOWN(addr, PAGE_SIZE); 852 + mlx5e_page_release_fragmented(rq, dma_info->frag_page); 878 853 } 854 + clear_bit(header_index, shampo->bitmap); 855 + } 879 856 880 - if (start + len > hd_per_wq) { 881 - len -= hd_per_wq - start; 882 - bitmap_clear(shampo->bitmap, start, hd_per_wq - start); 883 - start = 0; 884 - } 857 + void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq) 858 + { 859 + struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 860 + int i; 885 861 886 - bitmap_clear(shampo->bitmap, start, len); 862 + for_each_set_bit(i, shampo->bitmap, rq->mpwqe.shampo->hd_per_wq) 863 + mlx5e_free_rx_shampo_hd_entry(rq, i); 887 864 } 888 865 889 866 static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) ··· 1603 1612 struct mlx5e_rq_stats *stats = rq->stats; 1604 1613 1605 1614 stats->packets++; 1606 - stats->gro_packets++; 1607 1615 stats->bytes += cqe_bcnt; 1608 - stats->gro_bytes += cqe_bcnt; 1609 1616 if (NAPI_GRO_CB(skb)->count != 1) 1610 1617 return; 1611 1618 mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb); ··· 1953 1964 #endif 1954 1965 1955 1966 static void 1956 - mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, 1957 - struct mlx5e_frag_page *frag_page, 1958 - u32 data_bcnt, u32 data_offset) 1967 + mlx5e_shampo_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, 1968 + struct mlx5e_frag_page *frag_page, 1969 + u32 data_bcnt, u32 data_offset) 1959 1970 { 1960 1971 net_prefetchw(skb->data); 1961 1972 1962 - while (data_bcnt) { 1973 + do { 1963 1974 /* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */ 1964 1975 u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - data_offset, data_bcnt); 1965 - unsigned int truesize; 1976 + unsigned int truesize = pg_consumed_bytes; 1966 1977 1967 - if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 1968 - truesize = pg_consumed_bytes; 1969 - else 1970 - truesize = ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz)); 1971 - 1972 - frag_page->frags++; 1973 - mlx5e_add_skb_frag(rq, skb, frag_page->page, data_offset, 1978 + mlx5e_add_skb_frag(rq, skb, frag_page, data_offset, 1974 1979 pg_consumed_bytes, truesize); 1975 1980 1976 1981 data_bcnt -= pg_consumed_bytes; 1977 1982 data_offset = 0; 1978 1983 frag_page++; 1979 - } 1984 + } while (data_bcnt); 1980 1985 } 1981 1986 1982 1987 static struct sk_buff * ··· 2195 2212 if (likely(frag_size <= BIT(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE))) { 2196 2213 /* build SKB around header */ 2197 2214 dma_sync_single_range_for_cpu(rq->pdev, head->addr, 0, frag_size, rq->buff.map_dir); 2198 - prefetchw(hdr); 2199 - prefetch(data); 2215 + net_prefetchw(hdr); 2216 + net_prefetch(data); 2200 2217 skb = mlx5e_build_linear_skb(rq, hdr, frag_size, rx_headroom, head_size, 0); 2201 2218 2202 2219 if (unlikely(!skb)) ··· 2213 2230 return NULL; 2214 2231 } 2215 2232 2216 - prefetchw(skb->data); 2233 + net_prefetchw(skb->data); 2217 2234 mlx5e_copy_skb_header(rq, skb, head->frag_page->page, head->addr, 2218 2235 head_offset + rx_headroom, 2219 2236 rx_headroom, head_size); ··· 2244 2261 { 2245 2262 struct sk_buff *skb = rq->hw_gro_data->skb; 2246 2263 struct mlx5e_rq_stats *stats = rq->stats; 2264 + u16 gro_count = NAPI_GRO_CB(skb)->count; 2247 2265 2248 - stats->gro_skbs++; 2249 2266 if (likely(skb_shinfo(skb)->nr_frags)) 2250 2267 mlx5e_shampo_align_fragment(skb, rq->mpwqe.log_stride_sz); 2251 - if (NAPI_GRO_CB(skb)->count > 1) 2268 + if (gro_count > 1) { 2269 + stats->gro_skbs++; 2270 + stats->gro_packets += gro_count; 2271 + stats->gro_bytes += skb->data_len + skb_headlen(skb) * gro_count; 2272 + 2252 2273 mlx5e_shampo_update_hdr(rq, cqe, match); 2274 + } else { 2275 + skb_shinfo(skb)->gso_size = 0; 2276 + } 2253 2277 napi_gro_receive(rq->cq.napi, skb); 2254 2278 rq->hw_gro_data->skb = NULL; 2255 2279 } ··· 2267 2277 int nr_frags = skb_shinfo(skb)->nr_frags; 2268 2278 2269 2279 return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE; 2270 - } 2271 - 2272 - static void 2273 - mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index) 2274 - { 2275 - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 2276 - u64 addr = shampo->info[header_index].addr; 2277 - 2278 - if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) { 2279 - struct mlx5e_dma_info *dma_info = &shampo->info[header_index]; 2280 - 2281 - dma_info->addr = ALIGN_DOWN(addr, PAGE_SIZE); 2282 - mlx5e_page_release_fragmented(rq, dma_info->frag_page); 2283 - } 2284 - bitmap_clear(shampo->bitmap, header_index, 1); 2285 2280 } 2286 2281 2287 2282 static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) ··· 2302 2327 goto mpwrq_cqe_out; 2303 2328 } 2304 2329 2305 - stats->gro_match_packets += match; 2306 - 2307 2330 if (*skb && (!match || !(mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt)))) { 2308 2331 match = false; 2309 2332 mlx5e_shampo_flush_skb(rq, cqe, match); ··· 2332 2359 } 2333 2360 2334 2361 if (likely(head_size)) { 2335 - struct mlx5e_frag_page *frag_page; 2362 + if (data_bcnt) { 2363 + struct mlx5e_frag_page *frag_page; 2336 2364 2337 - frag_page = &wi->alloc_units.frag_pages[page_idx]; 2338 - mlx5e_fill_skb_data(*skb, rq, frag_page, data_bcnt, data_offset); 2365 + frag_page = &wi->alloc_units.frag_pages[page_idx]; 2366 + mlx5e_shampo_fill_skb_data(*skb, rq, frag_page, data_bcnt, data_offset); 2367 + } else { 2368 + stats->hds_nodata_packets++; 2369 + stats->hds_nodata_bytes += head_size; 2370 + } 2339 2371 } 2340 2372 2341 2373 mlx5e_shampo_complete_rx_cqe(rq, cqe, cqe_bcnt, *skb); 2342 - if (flush) 2374 + if (flush && rq->hw_gro_data->skb) 2343 2375 mlx5e_shampo_flush_skb(rq, cqe, match); 2344 2376 free_hd_entry: 2345 - mlx5e_free_rx_shampo_hd_entry(rq, header_index); 2377 + if (likely(head_size)) 2378 + mlx5e_free_rx_shampo_hd_entry(rq, header_index); 2346 2379 mpwrq_cqe_out: 2347 2380 if (likely(wi->consumed_strides < rq->mpwqe.num_strides)) 2381 + return; 2382 + 2383 + if (unlikely(!cstrides)) 2348 2384 return; 2349 2385 2350 2386 wq = &rq->mpwqe.wq;
+4 -3
drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
··· 141 141 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_gro_packets) }, 142 142 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_gro_bytes) }, 143 143 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_gro_skbs) }, 144 - { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_gro_match_packets) }, 145 144 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_gro_large_hds) }, 146 145 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_ecn_mark) }, 147 146 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_removed_vlan_packets) }, ··· 342 343 s->rx_gro_packets += rq_stats->gro_packets; 343 344 s->rx_gro_bytes += rq_stats->gro_bytes; 344 345 s->rx_gro_skbs += rq_stats->gro_skbs; 345 - s->rx_gro_match_packets += rq_stats->gro_match_packets; 346 346 s->rx_gro_large_hds += rq_stats->gro_large_hds; 347 + s->rx_hds_nodata_packets += rq_stats->hds_nodata_packets; 348 + s->rx_hds_nodata_bytes += rq_stats->hds_nodata_bytes; 347 349 s->rx_ecn_mark += rq_stats->ecn_mark; 348 350 s->rx_removed_vlan_packets += rq_stats->removed_vlan_packets; 349 351 s->rx_csum_none += rq_stats->csum_none; ··· 2057 2057 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, gro_packets) }, 2058 2058 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, gro_bytes) }, 2059 2059 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, gro_skbs) }, 2060 - { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, gro_match_packets) }, 2061 2060 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, gro_large_hds) }, 2061 + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, hds_nodata_packets) }, 2062 + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, hds_nodata_bytes) }, 2062 2063 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, ecn_mark) }, 2063 2064 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, removed_vlan_packets) }, 2064 2065 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, wqe_err) },
+4 -2
drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
··· 153 153 u64 rx_gro_packets; 154 154 u64 rx_gro_bytes; 155 155 u64 rx_gro_skbs; 156 - u64 rx_gro_match_packets; 157 156 u64 rx_gro_large_hds; 157 + u64 rx_hds_nodata_packets; 158 + u64 rx_hds_nodata_bytes; 158 159 u64 rx_mcast_packets; 159 160 u64 rx_ecn_mark; 160 161 u64 rx_removed_vlan_packets; ··· 353 352 u64 gro_packets; 354 353 u64 gro_bytes; 355 354 u64 gro_skbs; 356 - u64 gro_match_packets; 357 355 u64 gro_large_hds; 356 + u64 hds_nodata_packets; 357 + u64 hds_nodata_bytes; 358 358 u64 mcast_packets; 359 359 u64 ecn_mark; 360 360 u64 removed_vlan_packets;
+1
include/linux/mlx5/device.h
··· 294 294 #define MLX5_UMR_FLEX_ALIGNMENT 0x40 295 295 #define MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT (MLX5_UMR_FLEX_ALIGNMENT / sizeof(struct mlx5_mtt)) 296 296 #define MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT (MLX5_UMR_FLEX_ALIGNMENT / sizeof(struct mlx5_klm)) 297 + #define MLX5_UMR_KSM_NUM_ENTRIES_ALIGNMENT (MLX5_UMR_FLEX_ALIGNMENT / sizeof(struct mlx5_ksm)) 297 298 298 299 #define MLX5_USER_INDEX_LEN (MLX5_FLD_SZ_BYTES(qpc, user_index) * 8) 299 300
+11 -5
include/linux/mlx5/mlx5_ifc.h
··· 1526 1526 u8 ts_cqe_to_dest_cqn[0x1]; 1527 1527 u8 reserved_at_b3[0x6]; 1528 1528 u8 go_back_n[0x1]; 1529 - u8 shampo[0x1]; 1530 - u8 reserved_at_bb[0x5]; 1529 + u8 reserved_at_ba[0x6]; 1531 1530 1532 1531 u8 max_sgl_for_optimized_performance[0x8]; 1533 1532 u8 log_max_cq_sz[0x8]; ··· 1743 1744 u8 reserved_at_280[0x10]; 1744 1745 u8 max_wqe_sz_sq[0x10]; 1745 1746 1746 - u8 reserved_at_2a0[0x10]; 1747 + u8 reserved_at_2a0[0xb]; 1748 + u8 shampo[0x1]; 1749 + u8 reserved_at_2ac[0x4]; 1747 1750 u8 max_wqe_sz_rq[0x10]; 1748 1751 1749 1752 u8 max_flow_counter_31_16[0x10]; ··· 2018 2017 u8 reserved_at_250[0x10]; 2019 2018 2020 2019 u8 reserved_at_260[0x120]; 2021 - u8 reserved_at_380[0x10]; 2020 + u8 reserved_at_380[0xb]; 2021 + u8 min_mkey_log_entity_size_fixed_buffer[0x5]; 2022 2022 u8 ec_vf_vport_base[0x10]; 2023 2023 2024 2024 u8 reserved_at_3a0[0x10]; ··· 2031 2029 u8 pcc_ifa2[0x1]; 2032 2030 u8 reserved_at_3f1[0xf]; 2033 2031 2034 - u8 reserved_at_400[0x400]; 2032 + u8 reserved_at_400[0x1]; 2033 + u8 min_mkey_log_entity_size_fixed_buffer_valid[0x1]; 2034 + u8 reserved_at_402[0x1e]; 2035 + 2036 + u8 reserved_at_420[0x3e0]; 2035 2037 }; 2036 2038 2037 2039 enum mlx5_ifc_flow_destination_type {