Merge branch 'ionic-convert-rx-queue-buffers-to-use-page_pool'

+1

drivers/net/ethernet/pensando/Kconfig

··· 23 23 depends on PTP_1588_CLOCK_OPTIONAL 24 24 select NET_DEVLINK 25 25 select DIMLIB 26 + select PAGE_POOL 26 27 help 27 28 This enables the support for the Pensando family of Ethernet 28 29 adapters. More specific information on this driver can be

+12 -11

drivers/net/ethernet/pensando/ionic/ionic_dev.h

··· 181 181 struct ionic_qcq; 182 182 183 183 #define IONIC_MAX_BUF_LEN ((u16)-1) 184 - #define IONIC_PAGE_SIZE PAGE_SIZE 185 - #define IONIC_PAGE_SPLIT_SZ (PAGE_SIZE / 2) 186 - #define IONIC_PAGE_GFP_MASK (GFP_ATOMIC | __GFP_NOWARN |\ 187 - __GFP_COMP | __GFP_MEMALLOC) 184 + #define IONIC_PAGE_SIZE MIN(PAGE_SIZE, IONIC_MAX_BUF_LEN) 188 185 189 186 #define IONIC_XDP_MAX_LINEAR_MTU (IONIC_PAGE_SIZE - \ 190 187 (VLAN_ETH_HLEN + \ ··· 235 238 unsigned int index; 236 239 unsigned int num_descs; 237 240 unsigned int max_sg_elems; 241 + 238 242 u64 features; 239 - unsigned int type; 240 - unsigned int hw_index; 241 243 unsigned int hw_type; 242 244 bool xdp_flush; 243 245 union { ··· 246 250 struct ionic_admin_cmd *adminq; 247 251 }; 248 252 union { 249 - void __iomem *cmb_base; 250 - struct ionic_txq_desc __iomem *cmb_txq; 251 - struct ionic_rxq_desc __iomem *cmb_rxq; 252 - }; 253 - union { 254 253 void *sg_base; 255 254 struct ionic_txq_sg_desc *txq_sgl; 256 255 struct ionic_txq_sg_desc_v1 *txq_sgl_v1; 257 256 struct ionic_rxq_sg_desc *rxq_sgl; 258 257 }; 259 258 struct xdp_rxq_info *xdp_rxq_info; 259 + struct bpf_prog *xdp_prog; 260 + struct page_pool *page_pool; 260 261 struct ionic_queue *partner; 262 + 263 + union { 264 + void __iomem *cmb_base; 265 + struct ionic_txq_desc __iomem *cmb_txq; 266 + struct ionic_rxq_desc __iomem *cmb_rxq; 267 + }; 268 + unsigned int type; 269 + unsigned int hw_index; 261 270 dma_addr_t base_pa; 262 271 dma_addr_t cmb_base_pa; 263 272 dma_addr_t sg_base_pa;

+97 -66

drivers/net/ethernet/pensando/ionic/ionic_lif.c

··· 13 13 #include <linux/cpumask.h> 14 14 #include <linux/crash_dump.h> 15 15 #include <linux/vmalloc.h> 16 + #include <net/page_pool/helpers.h> 16 17 17 18 #include "ionic.h" 18 19 #include "ionic_bus.h" ··· 47 46 static void ionic_stop_queues(struct ionic_lif *lif); 48 47 static void ionic_lif_queue_identify(struct ionic_lif *lif); 49 48 50 - static int ionic_xdp_queues_config(struct ionic_lif *lif); 51 - static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q); 49 + static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif); 50 + static void ionic_unregister_rxq_info(struct ionic_queue *q); 51 + static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id); 52 52 53 53 static void ionic_dim_work(struct work_struct *work) 54 54 { ··· 382 380 if (!(qcq->flags & IONIC_QCQ_F_INITED)) 383 381 return; 384 382 383 + ionic_unregister_rxq_info(&qcq->q); 385 384 if (qcq->flags & IONIC_QCQ_F_INTR) { 386 385 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 387 386 IONIC_INTR_MASK_SET); ··· 440 437 qcq->sg_base_pa = 0; 441 438 } 442 439 443 - ionic_xdp_unregister_rxq_info(&qcq->q); 444 - ionic_qcq_intr_free(lif, qcq); 440 + page_pool_destroy(qcq->q.page_pool); 441 + qcq->q.page_pool = NULL; 445 442 443 + ionic_qcq_intr_free(lif, qcq); 446 444 vfree(qcq->q.info); 447 445 qcq->q.info = NULL; 448 446 } ··· 557 553 unsigned int cq_desc_size, 558 554 unsigned int sg_desc_size, 559 555 unsigned int desc_info_size, 560 - unsigned int pid, struct ionic_qcq **qcq) 556 + unsigned int pid, struct bpf_prog *xdp_prog, 557 + struct ionic_qcq **qcq) 561 558 { 562 559 struct ionic_dev *idev = &lif->ionic->idev; 563 560 struct device *dev = lif->ionic->dev; ··· 584 579 goto err_out_free_qcq; 585 580 } 586 581 582 + if (type == IONIC_QTYPE_RXQ) { 583 + struct page_pool_params pp_params = { 584 + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 585 + .order = 0, 586 + .pool_size = num_descs, 587 + .nid = NUMA_NO_NODE, 588 + .dev = lif->ionic->dev, 589 + .napi = &new->napi, 590 + .dma_dir = DMA_FROM_DEVICE, 591 + .max_len = PAGE_SIZE, 592 + .netdev = lif->netdev, 593 + }; 594 + 595 + if (xdp_prog) 596 + pp_params.dma_dir = DMA_BIDIRECTIONAL; 597 + 598 + new->q.page_pool = page_pool_create(&pp_params); 599 + if (IS_ERR(new->q.page_pool)) { 600 + netdev_err(lif->netdev, "Cannot create page_pool\n"); 601 + err = PTR_ERR(new->q.page_pool); 602 + new->q.page_pool = NULL; 603 + goto err_out_free_q_info; 604 + } 605 + } 606 + 587 607 new->q.type = type; 588 608 new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems; 589 609 ··· 616 586 desc_size, sg_desc_size, pid); 617 587 if (err) { 618 588 netdev_err(lif->netdev, "Cannot initialize queue\n"); 619 - goto err_out_free_q_info; 589 + goto err_out_free_page_pool; 620 590 } 621 591 622 592 err = ionic_alloc_qcq_interrupt(lif, new); 623 593 if (err) 624 - goto err_out_free_q_info; 594 + goto err_out_free_page_pool; 625 595 626 596 err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size); 627 597 if (err) { ··· 742 712 devm_free_irq(dev, new->intr.vector, &new->napi); 743 713 ionic_intr_free(lif->ionic, new->intr.index); 744 714 } 715 + err_out_free_page_pool: 716 + page_pool_destroy(new->q.page_pool); 745 717 err_out_free_q_info: 746 718 vfree(new->q.info); 747 719 err_out_free_qcq: ··· 766 734 sizeof(struct ionic_admin_comp), 767 735 0, 768 736 sizeof(struct ionic_admin_desc_info), 769 - lif->kern_pid, &lif->adminqcq); 737 + lif->kern_pid, NULL, &lif->adminqcq); 770 738 if (err) 771 739 return err; 772 740 ionic_debugfs_add_qcq(lif, lif->adminqcq); ··· 779 747 sizeof(union ionic_notifyq_comp), 780 748 0, 781 749 sizeof(struct ionic_admin_desc_info), 782 - lif->kern_pid, &lif->notifyqcq); 750 + lif->kern_pid, NULL, &lif->notifyqcq); 783 751 if (err) 784 752 goto err_out; 785 753 ionic_debugfs_add_qcq(lif, lif->notifyqcq); ··· 957 925 netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi); 958 926 else 959 927 netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi); 928 + err = ionic_register_rxq_info(q, qcq->napi.napi_id); 929 + if (err) { 930 + netif_napi_del(&qcq->napi); 931 + return err; 932 + } 960 933 961 934 qcq->flags |= IONIC_QCQ_F_INITED; 962 935 ··· 997 960 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags, 998 961 num_desc, desc_sz, comp_sz, sg_desc_sz, 999 962 sizeof(struct ionic_tx_desc_info), 1000 - lif->kern_pid, &txq); 963 + lif->kern_pid, NULL, &txq); 1001 964 if (err) 1002 965 goto err_qcq_alloc; 1003 966 ··· 1057 1020 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags, 1058 1021 num_desc, desc_sz, comp_sz, sg_desc_sz, 1059 1022 sizeof(struct ionic_rx_desc_info), 1060 - lif->kern_pid, &rxq); 1023 + lif->kern_pid, NULL, &rxq); 1061 1024 if (err) 1062 1025 goto err_qcq_alloc; 1063 1026 ··· 1074 1037 goto err_qcq_init; 1075 1038 1076 1039 if (test_bit(IONIC_LIF_F_UP, lif->state)) { 1077 - ionic_rx_fill(&rxq->q); 1040 + ionic_rx_fill(&rxq->q, NULL); 1078 1041 err = ionic_qcq_enable(rxq); 1079 1042 if (err) 1080 1043 goto err_qcq_enable; ··· 2083 2046 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags, 2084 2047 num_desc, desc_sz, comp_sz, sg_desc_sz, 2085 2048 sizeof(struct ionic_tx_desc_info), 2086 - lif->kern_pid, &lif->txqcqs[i]); 2049 + lif->kern_pid, NULL, &lif->txqcqs[i]); 2087 2050 if (err) 2088 2051 goto err_out; 2089 2052 ··· 2115 2078 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags, 2116 2079 num_desc, desc_sz, comp_sz, sg_desc_sz, 2117 2080 sizeof(struct ionic_rx_desc_info), 2118 - lif->kern_pid, &lif->rxqcqs[i]); 2081 + lif->kern_pid, lif->xdp_prog, 2082 + &lif->rxqcqs[i]); 2119 2083 if (err) 2120 2084 goto err_out; 2121 2085 ··· 2181 2143 int derr = 0; 2182 2144 int i, err; 2183 2145 2184 - err = ionic_xdp_queues_config(lif); 2185 - if (err) 2186 - return err; 2146 + ionic_xdp_rxqs_prog_update(lif); 2187 2147 2188 2148 for (i = 0; i < lif->nxqs; i++) { 2189 2149 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) { ··· 2190 2154 goto err_out; 2191 2155 } 2192 2156 2193 - ionic_rx_fill(&lif->rxqcqs[i]->q); 2157 + ionic_rx_fill(&lif->rxqcqs[i]->q, 2158 + READ_ONCE(lif->rxqcqs[i]->q.xdp_prog)); 2194 2159 err = ionic_qcq_enable(lif->rxqcqs[i]); 2195 2160 if (err) 2196 2161 goto err_out; ··· 2204 2167 } 2205 2168 2206 2169 if (lif->hwstamp_rxq) { 2207 - ionic_rx_fill(&lif->hwstamp_rxq->q); 2170 + ionic_rx_fill(&lif->hwstamp_rxq->q, NULL); 2208 2171 err = ionic_qcq_enable(lif->hwstamp_rxq); 2209 2172 if (err) 2210 2173 goto err_out_hwstamp_rx; ··· 2229 2192 derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr); 2230 2193 } 2231 2194 2232 - ionic_xdp_queues_config(lif); 2195 + ionic_xdp_rxqs_prog_update(lif); 2233 2196 2234 2197 return err; 2235 2198 } ··· 2688 2651 ionic_vf_start(ionic); 2689 2652 } 2690 2653 2691 - static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q) 2654 + static void ionic_unregister_rxq_info(struct ionic_queue *q) 2692 2655 { 2693 2656 struct xdp_rxq_info *xi; 2694 2657 ··· 2702 2665 kfree(xi); 2703 2666 } 2704 2667 2705 - static int ionic_xdp_register_rxq_info(struct ionic_queue *q, unsigned int napi_id) 2668 + static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id) 2706 2669 { 2707 2670 struct xdp_rxq_info *rxq_info; 2708 2671 int err; ··· 2713 2676 2714 2677 err = xdp_rxq_info_reg(rxq_info, q->lif->netdev, q->index, napi_id); 2715 2678 if (err) { 2716 - dev_err(q->dev, "Queue %d xdp_rxq_info_reg failed, err %d\n", 2717 - q->index, err); 2679 + netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg failed, err %d\n", 2680 + q->index, err); 2718 2681 goto err_out; 2719 2682 } 2720 2683 2721 - err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_ORDER0, NULL); 2684 + err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_POOL, q->page_pool); 2722 2685 if (err) { 2723 - dev_err(q->dev, "Queue %d xdp_rxq_info_reg_mem_model failed, err %d\n", 2724 - q->index, err); 2686 + netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg_mem_model failed, err %d\n", 2687 + q->index, err); 2725 2688 xdp_rxq_info_unreg(rxq_info); 2726 2689 goto err_out; 2727 2690 } ··· 2735 2698 return err; 2736 2699 } 2737 2700 2738 - static int ionic_xdp_queues_config(struct ionic_lif *lif) 2701 + static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif) 2739 2702 { 2703 + struct bpf_prog *xdp_prog; 2740 2704 unsigned int i; 2741 - int err; 2742 2705 2743 2706 if (!lif->rxqcqs) 2744 - return 0; 2707 + return; 2745 2708 2746 - /* There's no need to rework memory if not going to/from NULL program. 2747 - * If there is no lif->xdp_prog, there should also be no q.xdp_rxq_info 2748 - * This way we don't need to keep an *xdp_prog in every queue struct. 2749 - */ 2750 - if (!lif->xdp_prog == !lif->rxqcqs[0]->q.xdp_rxq_info) 2751 - return 0; 2752 - 2709 + xdp_prog = READ_ONCE(lif->xdp_prog); 2753 2710 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) { 2754 2711 struct ionic_queue *q = &lif->rxqcqs[i]->q; 2755 2712 2756 - if (q->xdp_rxq_info) { 2757 - ionic_xdp_unregister_rxq_info(q); 2758 - continue; 2759 - } 2760 - 2761 - err = ionic_xdp_register_rxq_info(q, lif->rxqcqs[i]->napi.napi_id); 2762 - if (err) { 2763 - dev_err(lif->ionic->dev, "failed to register RX queue %d info for XDP, err %d\n", 2764 - i, err); 2765 - goto err_out; 2766 - } 2713 + WRITE_ONCE(q->xdp_prog, xdp_prog); 2767 2714 } 2768 - 2769 - return 0; 2770 - 2771 - err_out: 2772 - for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) 2773 - ionic_xdp_unregister_rxq_info(&lif->rxqcqs[i]->q); 2774 - 2775 - return err; 2776 2715 } 2777 2716 2778 2717 static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf) ··· 2778 2765 2779 2766 if (!netif_running(netdev)) { 2780 2767 old_prog = xchg(&lif->xdp_prog, bpf->prog); 2781 - } else { 2782 - mutex_lock(&lif->queue_lock); 2783 - ionic_stop_queues_reconfig(lif); 2768 + } else if (lif->xdp_prog && bpf->prog) { 2784 2769 old_prog = xchg(&lif->xdp_prog, bpf->prog); 2785 - ionic_start_queues_reconfig(lif); 2770 + ionic_xdp_rxqs_prog_update(lif); 2771 + } else { 2772 + struct ionic_queue_params qparams; 2773 + 2774 + ionic_init_queue_params(lif, &qparams); 2775 + qparams.xdp_prog = bpf->prog; 2776 + mutex_lock(&lif->queue_lock); 2777 + ionic_reconfigure_queues(lif, &qparams); 2778 + old_prog = xchg(&lif->xdp_prog, bpf->prog); 2786 2779 mutex_unlock(&lif->queue_lock); 2787 2780 } 2788 2781 ··· 2890 2871 2891 2872 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b) 2892 2873 { 2893 - /* only swapping the queues, not the napi, flags, or other stuff */ 2874 + /* only swapping the queues and napi, not flags or other stuff */ 2875 + swap(a->napi, b->napi); 2876 + 2877 + if (a->q.type == IONIC_QTYPE_RXQ) { 2878 + swap(a->q.page_pool, b->q.page_pool); 2879 + a->q.page_pool->p.napi = &a->napi; 2880 + if (b->q.page_pool) /* is NULL when increasing queue count */ 2881 + b->q.page_pool->p.napi = &b->napi; 2882 + } 2883 + 2894 2884 swap(a->q.features, b->q.features); 2895 2885 swap(a->q.num_descs, b->q.num_descs); 2896 2886 swap(a->q.desc_size, b->q.desc_size); 2897 2887 swap(a->q.base, b->q.base); 2898 2888 swap(a->q.base_pa, b->q.base_pa); 2899 2889 swap(a->q.info, b->q.info); 2890 + swap(a->q.xdp_prog, b->q.xdp_prog); 2900 2891 swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info); 2901 2892 swap(a->q.partner, b->q.partner); 2902 2893 swap(a->q_base, b->q_base); ··· 2957 2928 } 2958 2929 if (qparam->nxqs != lif->nxqs || 2959 2930 qparam->nrxq_descs != lif->nrxq_descs || 2960 - qparam->rxq_features != lif->rxq_features) { 2931 + qparam->rxq_features != lif->rxq_features || 2932 + qparam->xdp_prog != lif->xdp_prog) { 2961 2933 rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif, 2962 2934 sizeof(struct ionic_qcq *), GFP_KERNEL); 2963 2935 if (!rx_qcqs) { ··· 2989 2959 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags, 2990 2960 4, desc_sz, comp_sz, sg_desc_sz, 2991 2961 sizeof(struct ionic_tx_desc_info), 2992 - lif->kern_pid, &lif->txqcqs[i]); 2962 + lif->kern_pid, NULL, &lif->txqcqs[i]); 2993 2963 if (err) 2994 2964 goto err_out; 2995 2965 } ··· 2998 2968 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags, 2999 2969 num_desc, desc_sz, comp_sz, sg_desc_sz, 3000 2970 sizeof(struct ionic_tx_desc_info), 3001 - lif->kern_pid, &tx_qcqs[i]); 2971 + lif->kern_pid, NULL, &tx_qcqs[i]); 3002 2972 if (err) 3003 2973 goto err_out; 3004 2974 } ··· 3020 2990 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags, 3021 2991 4, desc_sz, comp_sz, sg_desc_sz, 3022 2992 sizeof(struct ionic_rx_desc_info), 3023 - lif->kern_pid, &lif->rxqcqs[i]); 2993 + lif->kern_pid, NULL, &lif->rxqcqs[i]); 3024 2994 if (err) 3025 2995 goto err_out; 3026 2996 } ··· 3029 2999 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags, 3030 3000 num_desc, desc_sz, comp_sz, sg_desc_sz, 3031 3001 sizeof(struct ionic_rx_desc_info), 3032 - lif->kern_pid, &rx_qcqs[i]); 3002 + lif->kern_pid, qparam->xdp_prog, &rx_qcqs[i]); 3033 3003 if (err) 3034 3004 goto err_out; 3035 3005 3036 3006 rx_qcqs[i]->q.features = qparam->rxq_features; 3007 + rx_qcqs[i]->q.xdp_prog = qparam->xdp_prog; 3037 3008 } 3038 3009 } 3039 3010

+2

drivers/net/ethernet/pensando/ionic/ionic_lif.h

··· 268 268 unsigned int ntxq_descs; 269 269 unsigned int nrxq_descs; 270 270 u64 rxq_features; 271 + struct bpf_prog *xdp_prog; 271 272 bool intr_split; 272 273 bool cmb_tx; 273 274 bool cmb_rx; ··· 281 280 qparam->ntxq_descs = lif->ntxq_descs; 282 281 qparam->nrxq_descs = lif->nrxq_descs; 283 282 qparam->rxq_features = lif->rxq_features; 283 + qparam->xdp_prog = lif->xdp_prog; 284 284 qparam->intr_split = test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state); 285 285 qparam->cmb_tx = test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state); 286 286 qparam->cmb_rx = test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state);

+217 -203

drivers/net/ethernet/pensando/ionic/ionic_txrx.c

··· 6 6 #include <linux/if_vlan.h> 7 7 #include <net/ip6_checksum.h> 8 8 #include <net/netdev_queues.h> 9 + #include <net/page_pool/helpers.h> 9 10 10 11 #include "ionic.h" 11 12 #include "ionic_lif.h" ··· 119 118 120 119 static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info) 121 120 { 122 - return buf_info->dma_addr + buf_info->page_offset; 121 + return page_pool_get_dma_addr(buf_info->page) + buf_info->page_offset; 123 122 } 124 123 125 - static unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info) 124 + static void __ionic_rx_put_buf(struct ionic_queue *q, 125 + struct ionic_buf_info *buf_info, 126 + bool recycle_direct) 126 127 { 127 - return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset); 128 - } 129 - 130 - static int ionic_rx_page_alloc(struct ionic_queue *q, 131 - struct ionic_buf_info *buf_info) 132 - { 133 - struct device *dev = q->dev; 134 - dma_addr_t dma_addr; 135 - struct page *page; 136 - 137 - page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); 138 - if (unlikely(!page)) { 139 - net_err_ratelimited("%s: %s page alloc failed\n", 140 - dev_name(dev), q->name); 141 - q_to_rx_stats(q)->alloc_err++; 142 - return -ENOMEM; 143 - } 144 - 145 - dma_addr = dma_map_page(dev, page, 0, 146 - IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 147 - if (unlikely(dma_mapping_error(dev, dma_addr))) { 148 - __free_pages(page, 0); 149 - net_err_ratelimited("%s: %s dma map failed\n", 150 - dev_name(dev), q->name); 151 - q_to_rx_stats(q)->dma_map_err++; 152 - return -EIO; 153 - } 154 - 155 - buf_info->dma_addr = dma_addr; 156 - buf_info->page = page; 157 - buf_info->page_offset = 0; 158 - 159 - return 0; 160 - } 161 - 162 - static void ionic_rx_page_free(struct ionic_queue *q, 163 - struct ionic_buf_info *buf_info) 164 - { 165 - struct device *dev = q->dev; 166 - 167 - if (unlikely(!buf_info)) { 168 - net_err_ratelimited("%s: %s invalid buf_info in free\n", 169 - dev_name(dev), q->name); 170 - return; 171 - } 172 - 173 128 if (!buf_info->page) 174 129 return; 175 130 176 - dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 177 - __free_pages(buf_info->page, 0); 131 + page_pool_put_full_page(q->page_pool, buf_info->page, recycle_direct); 178 132 buf_info->page = NULL; 133 + buf_info->len = 0; 134 + buf_info->page_offset = 0; 179 135 } 180 136 181 - static bool ionic_rx_buf_recycle(struct ionic_queue *q, 182 - struct ionic_buf_info *buf_info, u32 len) 137 + 138 + static void ionic_rx_put_buf(struct ionic_queue *q, 139 + struct ionic_buf_info *buf_info) 183 140 { 184 - u32 size; 141 + __ionic_rx_put_buf(q, buf_info, false); 142 + } 185 143 186 - /* don't re-use pages allocated in low-mem condition */ 187 - if (page_is_pfmemalloc(buf_info->page)) 188 - return false; 189 - 190 - /* don't re-use buffers from non-local numa nodes */ 191 - if (page_to_nid(buf_info->page) != numa_mem_id()) 192 - return false; 193 - 194 - size = ALIGN(len, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ); 195 - buf_info->page_offset += size; 196 - if (buf_info->page_offset >= IONIC_PAGE_SIZE) 197 - return false; 198 - 199 - get_page(buf_info->page); 200 - 201 - return true; 144 + static void ionic_rx_put_buf_direct(struct ionic_queue *q, 145 + struct ionic_buf_info *buf_info) 146 + { 147 + __ionic_rx_put_buf(q, buf_info, true); 202 148 } 203 149 204 150 static void ionic_rx_add_skb_frag(struct ionic_queue *q, 205 151 struct sk_buff *skb, 206 152 struct ionic_buf_info *buf_info, 207 - u32 off, u32 len, 153 + u32 headroom, u32 len, 208 154 bool synced) 209 155 { 210 156 if (!synced) 211 - dma_sync_single_range_for_cpu(q->dev, ionic_rx_buf_pa(buf_info), 212 - off, len, DMA_FROM_DEVICE); 157 + page_pool_dma_sync_for_cpu(q->page_pool, 158 + buf_info->page, 159 + buf_info->page_offset + headroom, 160 + len); 213 161 214 162 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 215 - buf_info->page, buf_info->page_offset + off, 216 - len, 217 - IONIC_PAGE_SIZE); 163 + buf_info->page, buf_info->page_offset + headroom, 164 + len, buf_info->len); 218 165 219 - if (!ionic_rx_buf_recycle(q, buf_info, len)) { 220 - dma_unmap_page(q->dev, buf_info->dma_addr, 221 - IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 222 - buf_info->page = NULL; 223 - } 166 + /* napi_gro_frags() will release/recycle the 167 + * page_pool buffers from the frags list 168 + */ 169 + buf_info->page = NULL; 170 + buf_info->len = 0; 171 + buf_info->page_offset = 0; 224 172 } 225 173 226 174 static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q, ··· 194 244 q_to_rx_stats(q)->alloc_err++; 195 245 return NULL; 196 246 } 247 + skb_mark_for_recycle(skb); 197 248 198 249 if (headroom) 199 250 frag_len = min_t(u16, len, 200 251 IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 201 252 else 202 - frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); 253 + frag_len = min_t(u16, len, IONIC_PAGE_SIZE); 203 254 204 255 if (unlikely(!buf_info->page)) 205 256 goto err_bad_buf_page; ··· 211 260 for (i = 0; i < num_sg_elems; i++, buf_info++) { 212 261 if (unlikely(!buf_info->page)) 213 262 goto err_bad_buf_page; 214 - frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); 263 + frag_len = min_t(u16, len, buf_info->len); 215 264 ionic_rx_add_skb_frag(q, skb, buf_info, 0, frag_len, synced); 216 265 len -= frag_len; 217 266 } ··· 228 277 struct ionic_rx_desc_info *desc_info, 229 278 unsigned int headroom, 230 279 unsigned int len, 280 + unsigned int num_sg_elems, 231 281 bool synced) 232 282 { 233 283 struct ionic_buf_info *buf_info; 234 284 struct device *dev = q->dev; 235 285 struct sk_buff *skb; 286 + int i; 236 287 237 288 buf_info = &desc_info->bufs[0]; 238 289 ··· 245 292 q_to_rx_stats(q)->alloc_err++; 246 293 return NULL; 247 294 } 248 - 249 - if (unlikely(!buf_info->page)) { 250 - dev_kfree_skb(skb); 251 - return NULL; 252 - } 295 + skb_mark_for_recycle(skb); 253 296 254 297 if (!synced) 255 - dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), 256 - headroom, len, DMA_FROM_DEVICE); 298 + page_pool_dma_sync_for_cpu(q->page_pool, 299 + buf_info->page, 300 + buf_info->page_offset + headroom, 301 + len); 302 + 257 303 skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len); 258 - dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info), 259 - headroom, len, DMA_FROM_DEVICE); 260 304 261 305 skb_put(skb, len); 262 306 skb->protocol = eth_type_trans(skb, netdev); 307 + 308 + /* recycle the Rx buffer now that we're done with it */ 309 + ionic_rx_put_buf_direct(q, buf_info); 310 + buf_info++; 311 + for (i = 0; i < num_sg_elems; i++, buf_info++) 312 + ionic_rx_put_buf_direct(q, buf_info); 263 313 264 314 return skb; 265 315 } 266 316 267 317 static void ionic_xdp_tx_desc_clean(struct ionic_queue *q, 268 - struct ionic_tx_desc_info *desc_info) 318 + struct ionic_tx_desc_info *desc_info, 319 + bool in_napi) 269 320 { 270 - unsigned int nbufs = desc_info->nbufs; 271 - struct ionic_buf_info *buf_info; 272 - struct device *dev = q->dev; 273 - int i; 321 + struct xdp_frame_bulk bq; 274 322 275 - if (!nbufs) 323 + if (!desc_info->nbufs) 276 324 return; 277 325 278 - buf_info = desc_info->bufs; 279 - dma_unmap_single(dev, buf_info->dma_addr, 280 - buf_info->len, DMA_TO_DEVICE); 281 - if (desc_info->act == XDP_TX) 282 - __free_pages(buf_info->page, 0); 283 - buf_info->page = NULL; 326 + xdp_frame_bulk_init(&bq); 327 + rcu_read_lock(); /* need for xdp_return_frame_bulk */ 284 328 285 - buf_info++; 286 - for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) { 287 - dma_unmap_page(dev, buf_info->dma_addr, 288 - buf_info->len, DMA_TO_DEVICE); 289 - if (desc_info->act == XDP_TX) 290 - __free_pages(buf_info->page, 0); 291 - buf_info->page = NULL; 329 + if (desc_info->act == XDP_TX) { 330 + if (likely(in_napi)) 331 + xdp_return_frame_rx_napi(desc_info->xdpf); 332 + else 333 + xdp_return_frame(desc_info->xdpf); 334 + } else if (desc_info->act == XDP_REDIRECT) { 335 + ionic_tx_desc_unmap_bufs(q, desc_info); 336 + xdp_return_frame_bulk(desc_info->xdpf, &bq); 292 337 } 293 338 294 - if (desc_info->act == XDP_REDIRECT) 295 - xdp_return_frame(desc_info->xdpf); 339 + xdp_flush_frame_bulk(&bq); 340 + rcu_read_unlock(); 296 341 297 342 desc_info->nbufs = 0; 298 343 desc_info->xdpf = NULL; ··· 314 363 buf_info = desc_info->bufs; 315 364 stats = q_to_tx_stats(q); 316 365 317 - dma_addr = ionic_tx_map_single(q, frame->data, len); 318 - if (!dma_addr) 319 - return -EIO; 366 + if (act == XDP_TX) { 367 + dma_addr = page_pool_get_dma_addr(page) + 368 + off + XDP_PACKET_HEADROOM; 369 + dma_sync_single_for_device(q->dev, dma_addr, 370 + len, DMA_TO_DEVICE); 371 + } else /* XDP_REDIRECT */ { 372 + dma_addr = ionic_tx_map_single(q, frame->data, len); 373 + if (!dma_addr) 374 + return -EIO; 375 + } 376 + 320 377 buf_info->dma_addr = dma_addr; 321 378 buf_info->len = len; 322 379 buf_info->page = page; ··· 346 387 frag = sinfo->frags; 347 388 elem = ionic_tx_sg_elems(q); 348 389 for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) { 349 - dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 350 - if (!dma_addr) { 351 - ionic_tx_desc_unmap_bufs(q, desc_info); 352 - return -EIO; 390 + if (act == XDP_TX) { 391 + struct page *pg = skb_frag_page(frag); 392 + 393 + dma_addr = page_pool_get_dma_addr(pg) + 394 + skb_frag_off(frag); 395 + dma_sync_single_for_device(q->dev, dma_addr, 396 + skb_frag_size(frag), 397 + DMA_TO_DEVICE); 398 + } else { 399 + dma_addr = ionic_tx_map_frag(q, frag, 0, 400 + skb_frag_size(frag)); 401 + if (dma_mapping_error(q->dev, dma_addr)) { 402 + ionic_tx_desc_unmap_bufs(q, desc_info); 403 + return -EIO; 404 + } 353 405 } 354 406 bi->dma_addr = dma_addr; 355 407 bi->len = skb_frag_size(frag); ··· 451 481 return nxmit; 452 482 } 453 483 454 - static void ionic_xdp_rx_put_bufs(struct ionic_queue *q, 455 - struct ionic_buf_info *buf_info, 456 - int nbufs) 484 + static void ionic_xdp_rx_unlink_bufs(struct ionic_queue *q, 485 + struct ionic_buf_info *buf_info, 486 + int nbufs) 457 487 { 458 488 int i; 459 489 460 490 for (i = 0; i < nbufs; i++) { 461 - dma_unmap_page(q->dev, buf_info->dma_addr, 462 - IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 463 491 buf_info->page = NULL; 464 492 buf_info++; 465 493 } ··· 484 516 frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 485 517 xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info), 486 518 XDP_PACKET_HEADROOM, frag_len, false); 487 - 488 - dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info), 489 - XDP_PACKET_HEADROOM, frag_len, 490 - DMA_FROM_DEVICE); 491 - 519 + page_pool_dma_sync_for_cpu(rxq->page_pool, buf_info->page, 520 + buf_info->page_offset + XDP_PACKET_HEADROOM, 521 + frag_len); 492 522 prefetchw(&xdp_buf.data_hard_start); 493 523 494 524 /* We limit MTU size to one buffer if !xdp_has_frags, so ··· 508 542 do { 509 543 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) { 510 544 err = -ENOSPC; 511 - goto out_xdp_abort; 545 + break; 512 546 } 513 547 514 548 frag = &sinfo->frags[sinfo->nr_frags]; 515 549 sinfo->nr_frags++; 516 550 bi++; 517 - frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi)); 518 - dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi), 519 - 0, frag_len, DMA_FROM_DEVICE); 551 + frag_len = min_t(u16, remain_len, bi->len); 552 + page_pool_dma_sync_for_cpu(rxq->page_pool, bi->page, 553 + buf_info->page_offset, 554 + frag_len); 520 555 skb_frag_fill_page_desc(frag, bi->page, 0, frag_len); 521 556 sinfo->xdp_frags_size += frag_len; 522 557 remain_len -= frag_len; ··· 536 569 return false; /* false = we didn't consume the packet */ 537 570 538 571 case XDP_DROP: 539 - ionic_rx_page_free(rxq, buf_info); 572 + ionic_rx_put_buf_direct(rxq, buf_info); 540 573 stats->xdp_drop++; 541 574 break; 542 575 543 576 case XDP_TX: 544 577 xdpf = xdp_convert_buff_to_frame(&xdp_buf); 545 - if (!xdpf) 546 - goto out_xdp_abort; 578 + if (!xdpf) { 579 + err = -ENOSPC; 580 + break; 581 + } 547 582 548 583 txq = rxq->partner; 549 584 nq = netdev_get_tx_queue(netdev, txq->index); ··· 557 588 ionic_q_space_avail(txq), 558 589 1, 1)) { 559 590 __netif_tx_unlock(nq); 560 - goto out_xdp_abort; 591 + err = -EIO; 592 + break; 561 593 } 562 594 563 595 err = ionic_xdp_post_frame(txq, xdpf, XDP_TX, ··· 568 598 __netif_tx_unlock(nq); 569 599 if (unlikely(err)) { 570 600 netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err); 571 - goto out_xdp_abort; 601 + break; 572 602 } 573 - ionic_xdp_rx_put_bufs(rxq, buf_info, nbufs); 603 + ionic_xdp_rx_unlink_bufs(rxq, buf_info, nbufs); 574 604 stats->xdp_tx++; 575 - 576 - /* the Tx completion will free the buffers */ 577 605 break; 578 606 579 607 case XDP_REDIRECT: 580 608 err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog); 581 609 if (unlikely(err)) { 582 610 netdev_dbg(netdev, "xdp_do_redirect err %d\n", err); 583 - goto out_xdp_abort; 611 + break; 584 612 } 585 - ionic_xdp_rx_put_bufs(rxq, buf_info, nbufs); 613 + ionic_xdp_rx_unlink_bufs(rxq, buf_info, nbufs); 586 614 rxq->xdp_flush = true; 587 615 stats->xdp_redirect++; 588 616 break; 589 617 590 618 case XDP_ABORTED: 591 619 default: 592 - goto out_xdp_abort; 620 + err = -EIO; 621 + break; 593 622 } 594 623 595 - return true; 596 - 597 - out_xdp_abort: 598 - trace_xdp_exception(netdev, xdp_prog, xdp_action); 599 - ionic_rx_page_free(rxq, buf_info); 600 - stats->xdp_aborted++; 624 + if (err) { 625 + ionic_rx_put_buf_direct(rxq, buf_info); 626 + trace_xdp_exception(netdev, xdp_prog, xdp_action); 627 + stats->xdp_aborted++; 628 + } 601 629 602 630 return true; 603 631 } 604 632 605 633 static void ionic_rx_clean(struct ionic_queue *q, 606 634 struct ionic_rx_desc_info *desc_info, 607 - struct ionic_rxq_comp *comp) 635 + struct ionic_rxq_comp *comp, 636 + struct bpf_prog *xdp_prog) 608 637 { 609 638 struct net_device *netdev = q->lif->netdev; 610 639 struct ionic_qcq *qcq = q_to_qcq(q); 611 640 struct ionic_rx_stats *stats; 612 - struct bpf_prog *xdp_prog; 613 - unsigned int headroom; 641 + unsigned int headroom = 0; 614 642 struct sk_buff *skb; 615 643 bool synced = false; 616 644 bool use_copybreak; ··· 616 648 617 649 stats = q_to_rx_stats(q); 618 650 619 - if (comp->status) { 651 + if (unlikely(comp->status)) { 652 + /* Most likely status==2 and the pkt received was bigger 653 + * than the buffer available: comp->len will show the 654 + * pkt size received that didn't fit the advertised desc.len 655 + */ 656 + dev_dbg(q->dev, "q%d drop comp->status %d comp->len %d desc->len %d\n", 657 + q->index, comp->status, comp->len, q->rxq[q->head_idx].len); 658 + 620 659 stats->dropped++; 621 660 return; 622 661 } ··· 632 657 stats->pkts++; 633 658 stats->bytes += len; 634 659 635 - xdp_prog = READ_ONCE(q->lif->xdp_prog); 636 660 if (xdp_prog) { 637 661 if (ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len)) 638 662 return; 639 663 synced = true; 664 + headroom = XDP_PACKET_HEADROOM; 640 665 } 641 666 642 - headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 643 667 use_copybreak = len <= q->lif->rx_copybreak; 644 668 if (use_copybreak) 645 669 skb = ionic_rx_copybreak(netdev, q, desc_info, 646 - headroom, len, synced); 670 + headroom, len, 671 + comp->num_sg_elems, synced); 647 672 else 648 673 skb = ionic_rx_build_skb(q, desc_info, headroom, len, 649 674 comp->num_sg_elems, synced); ··· 719 744 napi_gro_frags(&qcq->napi); 720 745 } 721 746 722 - bool ionic_rx_service(struct ionic_cq *cq) 747 + static bool __ionic_rx_service(struct ionic_cq *cq, struct bpf_prog *xdp_prog) 723 748 { 724 749 struct ionic_rx_desc_info *desc_info; 725 750 struct ionic_queue *q = cq->bound_q; ··· 741 766 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 742 767 743 768 /* clean the related q entry, only one per qc completion */ 744 - ionic_rx_clean(q, desc_info, comp); 769 + ionic_rx_clean(q, desc_info, comp, xdp_prog); 745 770 746 771 return true; 772 + } 773 + 774 + bool ionic_rx_service(struct ionic_cq *cq) 775 + { 776 + return __ionic_rx_service(cq, NULL); 747 777 } 748 778 749 779 static inline void ionic_write_cmb_desc(struct ionic_queue *q, ··· 761 781 memcpy_toio(&q->cmb_txq[q->head_idx], desc, sizeof(q->cmb_txq[0])); 762 782 } 763 783 764 - void ionic_rx_fill(struct ionic_queue *q) 784 + void ionic_rx_fill(struct ionic_queue *q, struct bpf_prog *xdp_prog) 765 785 { 766 786 struct net_device *netdev = q->lif->netdev; 767 787 struct ionic_rx_desc_info *desc_info; ··· 769 789 struct ionic_buf_info *buf_info; 770 790 unsigned int fill_threshold; 771 791 struct ionic_rxq_desc *desc; 792 + unsigned int first_frag_len; 793 + unsigned int first_buf_len; 794 + unsigned int headroom = 0; 772 795 unsigned int remain_len; 773 796 unsigned int frag_len; 774 797 unsigned int nfrags; ··· 789 806 790 807 len = netdev->mtu + VLAN_ETH_HLEN; 791 808 792 - for (i = n_fill; i; i--) { 793 - unsigned int headroom; 794 - unsigned int buf_len; 809 + if (xdp_prog) { 810 + /* Always alloc the full size buffer, but only need 811 + * the actual frag_len in the descriptor 812 + * XDP uses space in the first buffer, so account for 813 + * head room, tail room, and ip header in the first frag size. 814 + */ 815 + headroom = XDP_PACKET_HEADROOM; 816 + first_buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN + headroom; 817 + first_frag_len = min_t(u16, len + headroom, first_buf_len); 818 + } else { 819 + /* Use MTU size if smaller than max buffer size */ 820 + first_frag_len = min_t(u16, len, IONIC_PAGE_SIZE); 821 + first_buf_len = first_frag_len; 822 + } 795 823 824 + for (i = n_fill; i; i--) { 825 + /* fill main descriptor - buf[0] */ 796 826 nfrags = 0; 797 827 remain_len = len; 798 828 desc = &q->rxq[q->head_idx]; 799 829 desc_info = &q->rx_info[q->head_idx]; 800 830 buf_info = &desc_info->bufs[0]; 801 831 802 - if (!buf_info->page) { /* alloc a new buffer? */ 803 - if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 804 - desc->addr = 0; 805 - desc->len = 0; 806 - return; 807 - } 808 - } 832 + buf_info->len = first_buf_len; 833 + frag_len = first_frag_len - headroom; 809 834 810 - /* fill main descriptor - buf[0] 811 - * XDP uses space in the first buffer, so account for 812 - * head room, tail room, and ip header in the first frag size. 813 - */ 814 - headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 815 - if (q->xdp_rxq_info) 816 - buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN; 817 - else 818 - buf_len = ionic_rx_buf_size(buf_info); 819 - frag_len = min_t(u16, len, buf_len); 835 + /* get a new buffer if we can't reuse one */ 836 + if (!buf_info->page) 837 + buf_info->page = page_pool_alloc(q->page_pool, 838 + &buf_info->page_offset, 839 + &buf_info->len, 840 + GFP_ATOMIC); 841 + if (unlikely(!buf_info->page)) { 842 + buf_info->len = 0; 843 + return; 844 + } 820 845 821 846 desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom); 822 847 desc->len = cpu_to_le16(frag_len); ··· 835 844 /* fill sg descriptors - buf[1..n] */ 836 845 sg_elem = q->rxq_sgl[q->head_idx].elems; 837 846 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++, sg_elem++) { 838 - if (!buf_info->page) { /* alloc a new sg buffer? */ 839 - if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 840 - sg_elem->addr = 0; 841 - sg_elem->len = 0; 847 + frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE); 848 + 849 + /* Recycle any leftover buffers that are too small to reuse */ 850 + if (unlikely(buf_info->page && buf_info->len < frag_len)) 851 + ionic_rx_put_buf_direct(q, buf_info); 852 + 853 + /* Get new buffer if needed */ 854 + if (!buf_info->page) { 855 + buf_info->len = frag_len; 856 + buf_info->page = page_pool_alloc(q->page_pool, 857 + &buf_info->page_offset, 858 + &buf_info->len, 859 + GFP_ATOMIC); 860 + if (unlikely(!buf_info->page)) { 861 + buf_info->len = 0; 842 862 return; 843 863 } 844 864 } 845 865 846 866 sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info)); 847 - frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info)); 848 867 sg_elem->len = cpu_to_le16(frag_len); 849 868 remain_len -= frag_len; 850 869 buf_info++; ··· 884 883 void ionic_rx_empty(struct ionic_queue *q) 885 884 { 886 885 struct ionic_rx_desc_info *desc_info; 887 - struct ionic_buf_info *buf_info; 888 886 unsigned int i, j; 889 887 890 888 for (i = 0; i < q->num_descs; i++) { 891 889 desc_info = &q->rx_info[i]; 892 - for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++) { 893 - buf_info = &desc_info->bufs[j]; 894 - if (buf_info->page) 895 - ionic_rx_page_free(q, buf_info); 896 - } 897 - 890 + for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++) 891 + ionic_rx_put_buf(q, &desc_info->bufs[j]); 898 892 desc_info->nbufs = 0; 899 893 } 900 894 ··· 970 974 } 971 975 } 972 976 977 + static unsigned int ionic_rx_cq_service(struct ionic_cq *cq, 978 + unsigned int work_to_do) 979 + { 980 + struct ionic_queue *q = cq->bound_q; 981 + unsigned int work_done = 0; 982 + struct bpf_prog *xdp_prog; 983 + 984 + if (work_to_do == 0) 985 + return 0; 986 + 987 + xdp_prog = READ_ONCE(q->xdp_prog); 988 + while (__ionic_rx_service(cq, xdp_prog)) { 989 + if (cq->tail_idx == cq->num_descs - 1) 990 + cq->done_color = !cq->done_color; 991 + 992 + cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 993 + 994 + if (++work_done >= work_to_do) 995 + break; 996 + } 997 + ionic_rx_fill(q, xdp_prog); 998 + ionic_xdp_do_flush(cq); 999 + 1000 + return work_done; 1001 + } 1002 + 973 1003 int ionic_rx_napi(struct napi_struct *napi, int budget) 974 1004 { 975 1005 struct ionic_qcq *qcq = napi_to_qcq(napi); ··· 1006 984 if (unlikely(!budget)) 1007 985 return budget; 1008 986 1009 - work_done = ionic_cq_service(cq, budget, 1010 - ionic_rx_service, NULL, NULL); 987 + work_done = ionic_rx_cq_service(cq, budget); 1011 988 1012 - ionic_rx_fill(cq->bound_q); 1013 - 1014 - ionic_xdp_do_flush(cq); 1015 989 if (work_done < budget && napi_complete_done(napi, work_done)) { 1016 990 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); 1017 991 flags |= IONIC_INTR_CRED_UNMASK; ··· 1048 1030 if (unlikely(!budget)) 1049 1031 return budget; 1050 1032 1051 - rx_work_done = ionic_cq_service(rxcq, budget, 1052 - ionic_rx_service, NULL, NULL); 1033 + rx_work_done = ionic_rx_cq_service(rxcq, budget); 1053 1034 1054 - ionic_rx_fill(rxcq->bound_q); 1055 - 1056 - ionic_xdp_do_flush(rxcq); 1057 1035 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 1058 1036 ionic_dim_update(rxqcq, 0); 1059 1037 flags |= IONIC_INTR_CRED_UNMASK; ··· 1180 1166 struct sk_buff *skb; 1181 1167 1182 1168 if (desc_info->xdpf) { 1183 - ionic_xdp_tx_desc_clean(q->partner, desc_info); 1169 + ionic_xdp_tx_desc_clean(q->partner, desc_info, in_napi); 1184 1170 stats->clean++; 1185 1171 1186 1172 if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index)))

+3 -1

drivers/net/ethernet/pensando/ionic/ionic_txrx.h

··· 4 4 #ifndef _IONIC_TXRX_H_ 5 5 #define _IONIC_TXRX_H_ 6 6 7 + struct bpf_prog; 8 + 7 9 void ionic_tx_flush(struct ionic_cq *cq); 8 10 9 - void ionic_rx_fill(struct ionic_queue *q); 11 + void ionic_rx_fill(struct ionic_queue *q, struct bpf_prog *xdp_prog); 10 12 void ionic_rx_empty(struct ionic_queue *q); 11 13 void ionic_tx_empty(struct ionic_queue *q); 12 14 int ionic_rx_napi(struct napi_struct *napi, int budget);

Configure Feed

Configure Feed