Merge branch 'net-mlx5e-save-per-channel-async-icosq-in-default'

+23 -3

drivers/net/ethernet/mellanox/mlx5/core/en.h

··· 388 388 MLX5E_SQ_STATE_DIM, 389 389 MLX5E_SQ_STATE_PENDING_XSK_TX, 390 390 MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, 391 + MLX5E_SQ_STATE_LOCK_NEEDED, 391 392 MLX5E_NUM_SQ_STATES, /* Must be kept last */ 392 393 }; 393 394 ··· 546 545 u32 sqn; 547 546 u16 reserved_room; 548 547 unsigned long state; 548 + /* icosq can be accessed from any CPU and from different contexts 549 + * (NAPI softirq or process/workqueue). Always use spin_lock_bh for 550 + * simplicity and correctness across all contexts. 551 + */ 552 + spinlock_t lock; 549 553 struct mlx5e_ktls_resync_resp *ktls_resync; 550 554 551 555 /* control path */ ··· 782 776 struct mlx5e_xdpsq xsksq; 783 777 784 778 /* Async ICOSQ */ 785 - struct mlx5e_icosq async_icosq; 786 - /* async_icosq can be accessed from any CPU - the spinlock protects it. */ 787 - spinlock_t async_icosq_lock; 779 + struct mlx5e_icosq *async_icosq; 788 780 789 781 /* data path - accessed per napi poll */ 790 782 const struct cpumask *aff_mask; ··· 804 800 struct dim_cq_moder rx_cq_moder; 805 801 struct dim_cq_moder tx_cq_moder; 806 802 }; 803 + 804 + static inline bool mlx5e_icosq_sync_lock(struct mlx5e_icosq *sq) 805 + { 806 + if (likely(!test_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &sq->state))) 807 + return false; 808 + 809 + spin_lock_bh(&sq->lock); 810 + return true; 811 + } 812 + 813 + static inline void mlx5e_icosq_sync_unlock(struct mlx5e_icosq *sq, bool locked) 814 + { 815 + if (unlikely(locked)) 816 + spin_unlock_bh(&sq->lock); 817 + } 807 818 808 819 struct mlx5e_ptp; 809 820 ··· 939 920 u8 max_opened_tc; 940 921 bool tx_ptp_opened; 941 922 bool rx_ptp_opened; 923 + bool ktls_rx_was_enabled; 942 924 struct kernel_hwtstamp_config hwtstamp_config; 943 925 u16 q_counter[MLX5_SD_MAX_GROUP_SZ]; 944 926 u16 drop_rq_q_counter;

+1

drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c

··· 15 15 [MLX5E_SQ_STATE_DIM] = "dim", 16 16 [MLX5E_SQ_STATE_PENDING_XSK_TX] = "pending_xsk_tx", 17 17 [MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC] = "pending_tls_rx_resync", 18 + [MLX5E_SQ_STATE_LOCK_NEEDED] = "lock_needed", 18 19 }; 19 20 20 21 static int mlx5e_wait_for_sq_flush(struct mlx5e_txqsq *sq)

+3

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c

··· 23 23 struct mlx5_wq_cyc *wq = &icosq->wq; 24 24 struct mlx5e_umr_wqe *umr_wqe; 25 25 struct xdp_buff **xsk_buffs; 26 + bool sync_locked; 26 27 int batch, i; 27 28 u32 offset; /* 17-bit value with MTT. */ 28 29 u16 pi; ··· 48 47 goto err_reuse_batch; 49 48 } 50 49 50 + sync_locked = mlx5e_icosq_sync_lock(icosq); 51 51 pi = mlx5e_icosq_get_next_pi(icosq, rq->mpwqe.umr_wqebbs); 52 52 umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi); 53 53 memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe)); ··· 145 143 }; 146 144 147 145 icosq->pc += rq->mpwqe.umr_wqebbs; 146 + mlx5e_icosq_sync_unlock(icosq, sync_locked); 148 147 149 148 icosq->doorbell_cseg = &umr_wqe->hdr.ctrl; 150 149

+4 -2

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c

··· 26 26 * active and not polled by NAPI. Return 0, because the upcoming 27 27 * activate will trigger the IRQ for us. 28 28 */ 29 - if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->async_icosq.state))) 29 + if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, 30 + &c->async_icosq->state))) 30 31 return 0; 31 32 32 - if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state)) 33 + if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, 34 + &c->async_icosq->state)) 33 35 return 0; 34 36 35 37 mlx5e_trigger_napi_icosq(c);

+8 -2

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c

··· 135 135 int err = 0; 136 136 137 137 mutex_lock(&priv->state_lock); 138 - if (enable) 138 + if (enable) { 139 139 err = mlx5e_accel_fs_tcp_create(priv->fs); 140 - else 140 + if (!err && !priv->ktls_rx_was_enabled) { 141 + priv->ktls_rx_was_enabled = true; 142 + mlx5e_safe_reopen_channels(priv); 143 + } 144 + } else { 141 145 mlx5e_accel_fs_tcp_destroy(priv->fs); 146 + } 142 147 mutex_unlock(&priv->state_lock); 143 148 144 149 return err; ··· 166 161 destroy_workqueue(priv->tls->rx_wq); 167 162 return err; 168 163 } 164 + priv->ktls_rx_was_enabled = true; 169 165 } 170 166 171 167 return 0;

+13 -13

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c

··· 202 202 int err; 203 203 204 204 err = 0; 205 - sq = &c->async_icosq; 206 - spin_lock_bh(&c->async_icosq_lock); 205 + sq = c->async_icosq; 206 + spin_lock_bh(&sq->lock); 207 207 208 208 cseg = post_static_params(sq, priv_rx); 209 209 if (IS_ERR(cseg)) ··· 214 214 215 215 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 216 216 unlock: 217 - spin_unlock_bh(&c->async_icosq_lock); 217 + spin_unlock_bh(&sq->lock); 218 218 219 219 return err; 220 220 ··· 277 277 278 278 buf->priv_rx = priv_rx; 279 279 280 - spin_lock_bh(&sq->channel->async_icosq_lock); 280 + spin_lock_bh(&sq->lock); 281 281 282 282 if (unlikely(!mlx5e_icosq_can_post_wqe(sq, MLX5E_KTLS_GET_PROGRESS_WQEBBS))) { 283 - spin_unlock_bh(&sq->channel->async_icosq_lock); 283 + spin_unlock_bh(&sq->lock); 284 284 err = -ENOSPC; 285 285 goto err_dma_unmap; 286 286 } ··· 311 311 icosq_fill_wi(sq, pi, &wi); 312 312 sq->pc++; 313 313 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 314 - spin_unlock_bh(&sq->channel->async_icosq_lock); 314 + spin_unlock_bh(&sq->lock); 315 315 316 316 return 0; 317 317 ··· 344 344 } 345 345 346 346 c = resync->priv->channels.c[priv_rx->rxq]; 347 - sq = &c->async_icosq; 347 + sq = c->async_icosq; 348 348 349 349 if (resync_post_get_progress_params(sq, priv_rx)) { 350 350 priv_rx->rq_stats->tls_resync_req_skip++; ··· 371 371 struct mlx5e_icosq *sq; 372 372 bool trigger_poll; 373 373 374 - sq = &c->async_icosq; 374 + sq = c->async_icosq; 375 375 ktls_resync = sq->ktls_resync; 376 376 trigger_poll = false; 377 377 ··· 413 413 return; 414 414 415 415 if (!napi_if_scheduled_mark_missed(&c->napi)) { 416 - spin_lock_bh(&c->async_icosq_lock); 416 + spin_lock_bh(&sq->lock); 417 417 mlx5e_trigger_irq(sq); 418 - spin_unlock_bh(&c->async_icosq_lock); 418 + spin_unlock_bh(&sq->lock); 419 419 } 420 420 } 421 421 ··· 753 753 LIST_HEAD(local_list); 754 754 int i, j; 755 755 756 - sq = &c->async_icosq; 756 + sq = c->async_icosq; 757 757 758 758 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) 759 759 return false; ··· 772 772 clear_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state); 773 773 spin_unlock(&ktls_resync->lock); 774 774 775 - spin_lock(&c->async_icosq_lock); 775 + spin_lock(&sq->lock); 776 776 for (j = 0; j < i; j++) { 777 777 struct mlx5_wqe_ctrl_seg *cseg; 778 778 ··· 791 791 } 792 792 if (db_cseg) 793 793 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, db_cseg); 794 - spin_unlock(&c->async_icosq_lock); 794 + spin_unlock(&sq->lock); 795 795 796 796 priv_rx->rq_stats->tls_resync_res_ok += j; 797 797

+2 -1

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h

··· 50 50 static inline bool 51 51 mlx5e_ktls_rx_pending_resync_list(struct mlx5e_channel *c, int budget) 52 52 { 53 - return budget && test_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &c->async_icosq.state); 53 + return budget && test_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, 54 + &c->async_icosq->state); 54 55 } 55 56 56 57 static inline void

+75 -25

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

··· 2075 2075 if (err) 2076 2076 goto err_free_icosq; 2077 2077 2078 + spin_lock_init(&sq->lock); 2079 + 2078 2080 if (param->is_tls) { 2079 2081 sq->ktls_resync = mlx5e_ktls_rx_resync_create_resp_list(); 2080 2082 if (IS_ERR(sq->ktls_resync)) { ··· 2589 2587 return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), q_counter, &c->rq); 2590 2588 } 2591 2589 2590 + static struct mlx5e_icosq * 2591 + mlx5e_open_async_icosq(struct mlx5e_channel *c, 2592 + struct mlx5e_params *params, 2593 + struct mlx5e_channel_param *cparam, 2594 + struct mlx5e_create_cq_param *ccp) 2595 + { 2596 + struct dim_cq_moder icocq_moder = {0, 0}; 2597 + struct mlx5e_icosq *async_icosq; 2598 + int err; 2599 + 2600 + async_icosq = kvzalloc_node(sizeof(*async_icosq), GFP_KERNEL, 2601 + cpu_to_node(c->cpu)); 2602 + if (!async_icosq) 2603 + return ERR_PTR(-ENOMEM); 2604 + 2605 + err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->async_icosq.cqp, ccp, 2606 + &async_icosq->cq); 2607 + if (err) 2608 + goto err_free_async_icosq; 2609 + 2610 + err = mlx5e_open_icosq(c, params, &cparam->async_icosq, async_icosq, 2611 + mlx5e_async_icosq_err_cqe_work); 2612 + if (err) 2613 + goto err_close_async_icosq_cq; 2614 + 2615 + return async_icosq; 2616 + 2617 + err_close_async_icosq_cq: 2618 + mlx5e_close_cq(&async_icosq->cq); 2619 + err_free_async_icosq: 2620 + kvfree(async_icosq); 2621 + return ERR_PTR(err); 2622 + } 2623 + 2624 + static void mlx5e_close_async_icosq(struct mlx5e_icosq *async_icosq) 2625 + { 2626 + mlx5e_close_icosq(async_icosq); 2627 + mlx5e_close_cq(&async_icosq->cq); 2628 + kvfree(async_icosq); 2629 + } 2630 + 2592 2631 static int mlx5e_open_queues(struct mlx5e_channel *c, 2593 2632 struct mlx5e_params *params, 2594 - struct mlx5e_channel_param *cparam) 2633 + struct mlx5e_channel_param *cparam, 2634 + bool async_icosq_needed) 2595 2635 { 2596 2636 const struct net_device_ops *netdev_ops = c->netdev->netdev_ops; 2597 2637 struct dim_cq_moder icocq_moder = {0, 0}; ··· 2642 2598 2643 2599 mlx5e_build_create_cq_param(&ccp, c); 2644 2600 2645 - err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->async_icosq.cqp, &ccp, 2646 - &c->async_icosq.cq); 2647 - if (err) 2648 - return err; 2649 - 2650 2601 err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->icosq.cqp, &ccp, 2651 2602 &c->icosq.cq); 2652 2603 if (err) 2653 - goto err_close_async_icosq_cq; 2604 + return err; 2654 2605 2655 2606 err = mlx5e_open_tx_cqs(c, params, &ccp, cparam); 2656 2607 if (err) ··· 2669 2630 if (err) 2670 2631 goto err_close_rx_cq; 2671 2632 2672 - spin_lock_init(&c->async_icosq_lock); 2673 - 2674 - err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq, 2675 - mlx5e_async_icosq_err_cqe_work); 2676 - if (err) 2677 - goto err_close_rq_xdpsq_cq; 2633 + if (async_icosq_needed) { 2634 + c->async_icosq = mlx5e_open_async_icosq(c, params, cparam, 2635 + &ccp); 2636 + if (IS_ERR(c->async_icosq)) { 2637 + err = PTR_ERR(c->async_icosq); 2638 + goto err_close_rq_xdpsq_cq; 2639 + } 2640 + } 2678 2641 2679 2642 mutex_init(&c->icosq_recovery_lock); 2680 2643 ··· 2712 2671 mlx5e_close_icosq(&c->icosq); 2713 2672 2714 2673 err_close_async_icosq: 2715 - mlx5e_close_icosq(&c->async_icosq); 2674 + if (c->async_icosq) 2675 + mlx5e_close_async_icosq(c->async_icosq); 2716 2676 2717 2677 err_close_rq_xdpsq_cq: 2718 2678 if (c->xdp) ··· 2732 2690 err_close_icosq_cq: 2733 2691 mlx5e_close_cq(&c->icosq.cq); 2734 2692 2735 - err_close_async_icosq_cq: 2736 - mlx5e_close_cq(&c->async_icosq.cq); 2737 - 2738 2693 return err; 2739 2694 } 2740 2695 ··· 2745 2706 mlx5e_close_sqs(c); 2746 2707 mlx5e_close_icosq(&c->icosq); 2747 2708 mutex_destroy(&c->icosq_recovery_lock); 2748 - mlx5e_close_icosq(&c->async_icosq); 2709 + if (c->async_icosq) 2710 + mlx5e_close_async_icosq(c->async_icosq); 2749 2711 if (c->xdp) 2750 2712 mlx5e_close_cq(&c->rq_xdpsq.cq); 2751 2713 mlx5e_close_cq(&c->rq.cq); ··· 2754 2714 mlx5e_close_xdpredirect_sq(c->xdpsq); 2755 2715 mlx5e_close_tx_cqs(c); 2756 2716 mlx5e_close_cq(&c->icosq.cq); 2757 - mlx5e_close_cq(&c->async_icosq.cq); 2758 2717 } 2759 2718 2760 2719 static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix) ··· 2789 2750 2790 2751 void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c) 2791 2752 { 2792 - spin_lock_bh(&c->async_icosq_lock); 2793 - mlx5e_trigger_irq(&c->async_icosq); 2794 - spin_unlock_bh(&c->async_icosq_lock); 2753 + bool locked; 2754 + 2755 + if (!test_and_set_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &c->icosq.state)) 2756 + synchronize_net(); 2757 + 2758 + locked = mlx5e_icosq_sync_lock(&c->icosq); 2759 + mlx5e_trigger_irq(&c->icosq); 2760 + mlx5e_icosq_sync_unlock(&c->icosq, locked); 2761 + 2762 + clear_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &c->icosq.state); 2795 2763 } 2796 2764 2797 2765 void mlx5e_trigger_napi_sched(struct napi_struct *napi) ··· 2831 2785 struct mlx5e_channel_param *cparam; 2832 2786 struct mlx5_core_dev *mdev; 2833 2787 struct mlx5e_xsk_param xsk; 2788 + bool async_icosq_needed; 2834 2789 struct mlx5e_channel *c; 2835 2790 unsigned int irq; 2836 2791 int vec_ix; ··· 2881 2834 netif_napi_add_config_locked(netdev, &c->napi, mlx5e_napi_poll, ix); 2882 2835 netif_napi_set_irq_locked(&c->napi, irq); 2883 2836 2884 - err = mlx5e_open_queues(c, params, cparam); 2837 + async_icosq_needed = !!xsk_pool || priv->ktls_rx_was_enabled; 2838 + err = mlx5e_open_queues(c, params, cparam, async_icosq_needed); 2885 2839 if (unlikely(err)) 2886 2840 goto err_napi_del; 2887 2841 ··· 2920 2872 for (tc = 0; tc < c->num_tc; tc++) 2921 2873 mlx5e_activate_txqsq(&c->sq[tc]); 2922 2874 mlx5e_activate_icosq(&c->icosq); 2923 - mlx5e_activate_icosq(&c->async_icosq); 2875 + if (c->async_icosq) 2876 + mlx5e_activate_icosq(c->async_icosq); 2924 2877 2925 2878 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2926 2879 mlx5e_activate_xsk(c); ··· 2942 2893 else 2943 2894 mlx5e_deactivate_rq(&c->rq); 2944 2895 2945 - mlx5e_deactivate_icosq(&c->async_icosq); 2896 + if (c->async_icosq) 2897 + mlx5e_deactivate_icosq(c->async_icosq); 2946 2898 mlx5e_deactivate_icosq(&c->icosq); 2947 2899 for (tc = 0; tc < c->num_tc; tc++) 2948 2900 mlx5e_deactivate_txqsq(&c->sq[tc]);

+4

drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

··· 778 778 struct mlx5_wq_cyc *wq = &sq->wq; 779 779 struct mlx5e_umr_wqe *umr_wqe; 780 780 u32 offset; /* 17-bit value with MTT. */ 781 + bool sync_locked; 781 782 u16 pi; 782 783 int err; 783 784 int i; ··· 789 788 goto err; 790 789 } 791 790 791 + sync_locked = mlx5e_icosq_sync_lock(sq); 792 792 pi = mlx5e_icosq_get_next_pi(sq, rq->mpwqe.umr_wqebbs); 793 793 umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi); 794 794 memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe)); ··· 837 835 }; 838 836 839 837 sq->pc += rq->mpwqe.umr_wqebbs; 838 + mlx5e_icosq_sync_unlock(sq, sync_locked); 840 839 841 840 sq->doorbell_cseg = &umr_wqe->hdr.ctrl; 842 841 843 842 return 0; 844 843 845 844 err_unmap: 845 + mlx5e_icosq_sync_unlock(sq, sync_locked); 846 846 while (--i >= 0) { 847 847 frag_page--; 848 848 mlx5e_page_release_fragmented(rq->page_pool, frag_page);

+20 -15

drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c

··· 125 125 { 126 126 struct mlx5e_channel *c = container_of(napi, struct mlx5e_channel, 127 127 napi); 128 + struct mlx5e_icosq *aicosq = c->async_icosq; 128 129 struct mlx5e_ch_stats *ch_stats = c->stats; 129 130 struct mlx5e_xdpsq *xsksq = &c->xsksq; 130 131 struct mlx5e_txqsq __rcu **qos_sqs; ··· 181 180 busy |= work_done == budget; 182 181 183 182 mlx5e_poll_ico_cq(&c->icosq.cq); 184 - if (mlx5e_poll_ico_cq(&c->async_icosq.cq)) 185 - /* Don't clear the flag if nothing was polled to prevent 186 - * queueing more WQEs and overflowing the async ICOSQ. 187 - */ 188 - clear_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state); 183 + if (aicosq) { 184 + if (mlx5e_poll_ico_cq(&aicosq->cq)) 185 + /* Don't clear the flag if nothing was polled to prevent 186 + * queueing more WQEs and overflowing the async ICOSQ. 187 + */ 188 + clear_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, 189 + &aicosq->state); 189 190 190 - /* Keep after async ICOSQ CQ poll */ 191 - if (unlikely(mlx5e_ktls_rx_pending_resync_list(c, budget))) 192 - busy |= mlx5e_ktls_rx_handle_resync_list(c, budget); 191 + /* Keep after async ICOSQ CQ poll */ 192 + if (unlikely(mlx5e_ktls_rx_pending_resync_list(c, budget))) 193 + busy |= mlx5e_ktls_rx_handle_resync_list(c, budget); 194 + } 193 195 194 196 busy |= INDIRECT_CALL_2(rq->post_wqes, 195 197 mlx5e_post_rx_mpwqes, ··· 240 236 241 237 mlx5e_cq_arm(&rq->cq); 242 238 mlx5e_cq_arm(&c->icosq.cq); 243 - mlx5e_cq_arm(&c->async_icosq.cq); 239 + if (aicosq) { 240 + mlx5e_cq_arm(&aicosq->cq); 241 + if (xsk_open) { 242 + mlx5e_handle_rx_dim(xskrq); 243 + mlx5e_cq_arm(&xsksq->cq); 244 + mlx5e_cq_arm(&xskrq->cq); 245 + } 246 + } 244 247 if (c->xdpsq) 245 248 mlx5e_cq_arm(&c->xdpsq->cq); 246 - 247 - if (xsk_open) { 248 - mlx5e_handle_rx_dim(xskrq); 249 - mlx5e_cq_arm(&xsksq->cq); 250 - mlx5e_cq_arm(&xskrq->cq); 251 - } 252 249 253 250 if (unlikely(aff_change && busy_xsk)) { 254 251 mlx5e_trigger_irq(&c->icosq);

Configure Feed

Configure Feed