tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

dim: pass dim_sample to net_dim() by reference

net_dim() is currently passed a struct dim_sample argument by value.
struct dim_sample is 24 bytes. Since this is greater 16 bytes, x86-64
passes it on the stack. All callers have already initialized dim_sample
on the stack, so passing it by value requires pushing a duplicated copy
to the stack. Either witing to the stack and immediately reading it, or
perhaps dereferencing addresses relative to the stack pointer in a chain
of push instructions, seems to perform quite poorly.

In a heavy TCP workload, mlx5e_handle_rx_dim() consumes 3% of CPU time,
94% of which is attributed to the first push instruction to copy
dim_sample on the stack for the call to net_dim():
// Call ktime_get()
0.26 |4ead2: call 4ead7 <mlx5e_handle_rx_dim+0x47>
// Pass the address of struct dim in %rdi
|4ead7: lea 0x3d0(%rbx),%rdi
// Set dim_sample.pkt_ctr
|4eade: mov %r13d,0x8(%rsp)
// Set dim_sample.byte_ctr
|4eae3: mov %r12d,0xc(%rsp)
// Set dim_sample.event_ctr
0.15 |4eae8: mov %bp,0x10(%rsp)
// Duplicate dim_sample on the stack
94.16 |4eaed: push 0x10(%rsp)
2.79 |4eaf1: push 0x10(%rsp)
0.07 |4eaf5: push %rax
// Call net_dim()
0.21 |4eaf6: call 4eafb <mlx5e_handle_rx_dim+0x6b>

To allow the caller to reuse the struct dim_sample already on the stack,
pass the struct dim_sample by reference to net_dim().

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
Reviewed-by: Vladimir Oltean <vladimir.oltean@nxp.com>
Reviewed-by: Shannon Nelson <shannon.nelson@amd.com>
Reviewed-by: Florian Fainelli <florian.fainelli@broadcom.com>
Reviewed-by: Arthur Kiyanovski <akiyano@amazon.com>
Reviewed-by: Louis Peens <louis.peens@corigine.com>
Link: https://patch.msgid.link/20241031002326.3426181-2-csander@purestorage.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

authored by

Caleb Sander Mateos and committed by
Jakub Kicinski 61bf0009 a8652768

+31 -31
+1 -1
Documentation/networking/net_dim.rst
··· 156 156 my_entity->bytes, 157 157 &dim_sample); 158 158 /* Call net DIM */ 159 - net_dim(&my_entity->dim, dim_sample); 159 + net_dim(&my_entity->dim, &dim_sample); 160 160 ... 161 161 } 162 162
+1 -1
drivers/net/ethernet/amazon/ena/ena_netdev.c
··· 1383 1383 rx_ring->rx_stats.bytes, 1384 1384 &dim_sample); 1385 1385 1386 - net_dim(&ena_napi->dim, dim_sample); 1386 + net_dim(&ena_napi->dim, &dim_sample); 1387 1387 1388 1388 rx_ring->per_napi_packets = 0; 1389 1389 }
+1 -1
drivers/net/ethernet/broadcom/bcmsysport.c
··· 1029 1029 if (priv->dim.use_dim) { 1030 1030 dim_update_sample(priv->dim.event_ctr, priv->dim.packets, 1031 1031 priv->dim.bytes, &dim_sample); 1032 - net_dim(&priv->dim.dim, dim_sample); 1032 + net_dim(&priv->dim.dim, &dim_sample); 1033 1033 } 1034 1034 1035 1035 return work_done;
+2 -2
drivers/net/ethernet/broadcom/bnxt/bnxt.c
··· 3102 3102 cpr->rx_packets, 3103 3103 cpr->rx_bytes, 3104 3104 &dim_sample); 3105 - net_dim(&cpr->dim, dim_sample); 3105 + net_dim(&cpr->dim, &dim_sample); 3106 3106 } 3107 3107 return work_done; 3108 3108 } ··· 3233 3233 cpr_rx->rx_packets, 3234 3234 cpr_rx->rx_bytes, 3235 3235 &dim_sample); 3236 - net_dim(&cpr->dim, dim_sample); 3236 + net_dim(&cpr->dim, &dim_sample); 3237 3237 } 3238 3238 return work_done; 3239 3239 }
+1 -1
drivers/net/ethernet/broadcom/genet/bcmgenet.c
··· 2405 2405 if (ring->dim.use_dim) { 2406 2406 dim_update_sample(ring->dim.event_ctr, ring->dim.packets, 2407 2407 ring->dim.bytes, &dim_sample); 2408 - net_dim(&ring->dim.dim, dim_sample); 2408 + net_dim(&ring->dim.dim, &dim_sample); 2409 2409 } 2410 2410 2411 2411 return work_done;
+1 -1
drivers/net/ethernet/freescale/enetc/enetc.c
··· 718 718 v->rx_ring.stats.packets, 719 719 v->rx_ring.stats.bytes, 720 720 &dim_sample); 721 - net_dim(&v->rx_dim, dim_sample); 721 + net_dim(&v->rx_dim, &dim_sample); 722 722 } 723 723 724 724 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
+2 -2
drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
··· 4478 4478 4479 4479 dim_update_sample(tqp_vector->event_cnt, rx_group->total_packets, 4480 4480 rx_group->total_bytes, &sample); 4481 - net_dim(&rx_group->dim, sample); 4481 + net_dim(&rx_group->dim, &sample); 4482 4482 } 4483 4483 4484 4484 static void hns3_update_tx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector) ··· 4491 4491 4492 4492 dim_update_sample(tqp_vector->event_cnt, tx_group->total_packets, 4493 4493 tx_group->total_bytes, &sample); 4494 - net_dim(&tx_group->dim, sample); 4494 + net_dim(&tx_group->dim, &sample); 4495 4495 } 4496 4496 4497 4497 static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
+2 -2
drivers/net/ethernet/intel/ice/ice_txrx.c
··· 1352 1352 struct dim_sample dim_sample; 1353 1353 1354 1354 __ice_update_sample(q_vector, tx, &dim_sample, true); 1355 - net_dim(&tx->dim, dim_sample); 1355 + net_dim(&tx->dim, &dim_sample); 1356 1356 } 1357 1357 1358 1358 if (ITR_IS_DYNAMIC(rx)) { 1359 1359 struct dim_sample dim_sample; 1360 1360 1361 1361 __ice_update_sample(q_vector, rx, &dim_sample, false); 1362 - net_dim(&rx->dim, dim_sample); 1362 + net_dim(&rx->dim, &dim_sample); 1363 1363 } 1364 1364 } 1365 1365
+2 -2
drivers/net/ethernet/intel/idpf/idpf_txrx.c
··· 3679 3679 3680 3680 idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->tx_dim, 3681 3681 packets, bytes); 3682 - net_dim(&q_vector->tx_dim, dim_sample); 3682 + net_dim(&q_vector->tx_dim, &dim_sample); 3683 3683 3684 3684 check_rx_itr: 3685 3685 if (!IDPF_ITR_IS_DYNAMIC(q_vector->rx_intr_mode)) ··· 3698 3698 3699 3699 idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->rx_dim, 3700 3700 packets, bytes); 3701 - net_dim(&q_vector->rx_dim, dim_sample); 3701 + net_dim(&q_vector->rx_dim, &dim_sample); 3702 3702 } 3703 3703 3704 3704 /**
+1 -1
drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
··· 527 527 rx_frames + tx_frames, 528 528 rx_bytes + tx_bytes, 529 529 &dim_sample); 530 - net_dim(&cq_poll->dim, dim_sample); 530 + net_dim(&cq_poll->dim, &dim_sample); 531 531 } 532 532 533 533 int otx2_napi_handler(struct napi_struct *napi, int budget)
+2 -2
drivers/net/ethernet/mediatek/mtk_eth_soc.c
··· 2227 2227 eth->rx_bytes += bytes; 2228 2228 dim_update_sample(eth->rx_events, eth->rx_packets, eth->rx_bytes, 2229 2229 &dim_sample); 2230 - net_dim(&eth->rx_dim, dim_sample); 2230 + net_dim(&eth->rx_dim, &dim_sample); 2231 2231 2232 2232 if (xdp_flush) 2233 2233 xdp_do_flush(); ··· 2377 2377 2378 2378 dim_update_sample(eth->tx_events, eth->tx_packets, eth->tx_bytes, 2379 2379 &dim_sample); 2380 - net_dim(&eth->tx_dim, dim_sample); 2380 + net_dim(&eth->tx_dim, &dim_sample); 2381 2381 2382 2382 if (mtk_queue_stopped(eth) && 2383 2383 (atomic_read(&ring->free_count) > ring->thresh))
+2 -2
drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
··· 55 55 return; 56 56 57 57 dim_update_sample(sq->cq.event_ctr, stats->packets, stats->bytes, &dim_sample); 58 - net_dim(sq->dim, dim_sample); 58 + net_dim(sq->dim, &dim_sample); 59 59 } 60 60 61 61 static void mlx5e_handle_rx_dim(struct mlx5e_rq *rq) ··· 67 67 return; 68 68 69 69 dim_update_sample(rq->cq.event_ctr, stats->packets, stats->bytes, &dim_sample); 70 - net_dim(rq->dim, dim_sample); 70 + net_dim(rq->dim, &dim_sample); 71 71 } 72 72 73 73 void mlx5e_trigger_irq(struct mlx5e_icosq *sq)
+2 -2
drivers/net/ethernet/netronome/nfp/nfd3/dp.c
··· 1179 1179 } while (u64_stats_fetch_retry(&r_vec->rx_sync, start)); 1180 1180 1181 1181 dim_update_sample(r_vec->event_ctr, pkts, bytes, &dim_sample); 1182 - net_dim(&r_vec->rx_dim, dim_sample); 1182 + net_dim(&r_vec->rx_dim, &dim_sample); 1183 1183 } 1184 1184 1185 1185 if (r_vec->nfp_net->tx_coalesce_adapt_on && r_vec->tx_ring) { ··· 1194 1194 } while (u64_stats_fetch_retry(&r_vec->tx_sync, start)); 1195 1195 1196 1196 dim_update_sample(r_vec->event_ctr, pkts, bytes, &dim_sample); 1197 - net_dim(&r_vec->tx_dim, dim_sample); 1197 + net_dim(&r_vec->tx_dim, &dim_sample); 1198 1198 } 1199 1199 1200 1200 return pkts_polled;
+2 -2
drivers/net/ethernet/netronome/nfp/nfdk/dp.c
··· 1289 1289 } while (u64_stats_fetch_retry(&r_vec->rx_sync, start)); 1290 1290 1291 1291 dim_update_sample(r_vec->event_ctr, pkts, bytes, &dim_sample); 1292 - net_dim(&r_vec->rx_dim, dim_sample); 1292 + net_dim(&r_vec->rx_dim, &dim_sample); 1293 1293 } 1294 1294 1295 1295 if (r_vec->nfp_net->tx_coalesce_adapt_on && r_vec->tx_ring) { ··· 1304 1304 } while (u64_stats_fetch_retry(&r_vec->tx_sync, start)); 1305 1305 1306 1306 dim_update_sample(r_vec->event_ctr, pkts, bytes, &dim_sample); 1307 - net_dim(&r_vec->tx_dim, dim_sample); 1307 + net_dim(&r_vec->tx_dim, &dim_sample); 1308 1308 } 1309 1309 1310 1310 return pkts_polled;
+1 -1
drivers/net/ethernet/pensando/ionic/ionic_txrx.c
··· 928 928 dim_update_sample(qcq->cq.bound_intr->rearm_count, 929 929 pkts, bytes, &dim_sample); 930 930 931 - net_dim(&qcq->dim, dim_sample); 931 + net_dim(&qcq->dim, &dim_sample); 932 932 } 933 933 934 934 int ionic_tx_napi(struct napi_struct *napi, int budget)
+1 -1
drivers/net/virtio_net.c
··· 2804 2804 u64_stats_read(&rq->stats.bytes), 2805 2805 &cur_sample); 2806 2806 2807 - net_dim(&rq->dim, cur_sample); 2807 + net_dim(&rq->dim, &cur_sample); 2808 2808 rq->packets_in_napi = 0; 2809 2809 } 2810 2810
+1 -1
drivers/soc/fsl/dpio/dpio-service.c
··· 891 891 d->frames += frames; 892 892 893 893 dim_update_sample(d->event_ctr, d->frames, d->bytes, &dim_sample); 894 - net_dim(&d->rx_dim, dim_sample); 894 + net_dim(&d->rx_dim, &dim_sample); 895 895 896 896 spin_unlock(&d->dim_lock); 897 897 }
+1 -1
include/linux/dim.h
··· 425 425 * This is the main logic of the algorithm, where data is processed in order 426 426 * to decide on next required action. 427 427 */ 428 - void net_dim(struct dim *dim, struct dim_sample end_sample); 428 + void net_dim(struct dim *dim, const struct dim_sample *end_sample); 429 429 430 430 /* RDMA DIM */ 431 431
+5 -5
lib/dim/net_dim.c
··· 347 347 return dim->profile_ix != prev_ix; 348 348 } 349 349 350 - void net_dim(struct dim *dim, struct dim_sample end_sample) 350 + void net_dim(struct dim *dim, const struct dim_sample *end_sample) 351 351 { 352 352 struct dim_stats curr_stats; 353 353 u16 nevents; ··· 355 355 switch (dim->state) { 356 356 case DIM_MEASURE_IN_PROGRESS: 357 357 nevents = BIT_GAP(BITS_PER_TYPE(u16), 358 - end_sample.event_ctr, 358 + end_sample->event_ctr, 359 359 dim->start_sample.event_ctr); 360 360 if (nevents < DIM_NEVENTS) 361 361 break; 362 - if (!dim_calc_stats(&dim->start_sample, &end_sample, &curr_stats)) 362 + if (!dim_calc_stats(&dim->start_sample, end_sample, &curr_stats)) 363 363 break; 364 364 if (net_dim_decision(&curr_stats, dim)) { 365 365 dim->state = DIM_APPLY_NEW_PROFILE; ··· 368 368 } 369 369 fallthrough; 370 370 case DIM_START_MEASURE: 371 - dim_update_sample(end_sample.event_ctr, end_sample.pkt_ctr, 372 - end_sample.byte_ctr, &dim->start_sample); 371 + dim_update_sample(end_sample->event_ctr, end_sample->pkt_ctr, 372 + end_sample->byte_ctr, &dim->start_sample); 373 373 dim->state = DIM_MEASURE_IN_PROGRESS; 374 374 break; 375 375 case DIM_APPLY_NEW_PROFILE: