Merge branch 'ionic-xdp-support' · tjh.dev/kernel@d70a2a4

+11

drivers/net/ethernet/pensando/ionic/ionic_dev.h

··· 8 8 #include <linux/mutex.h> 9 9 #include <linux/workqueue.h> 10 10 #include <linux/skbuff.h> 11 + #include <linux/bpf_trace.h> 11 12 12 13 #include "ionic_if.h" 13 14 #include "ionic_regs.h" ··· 196 195 #define IONIC_PAGE_GFP_MASK (GFP_ATOMIC | __GFP_NOWARN |\ 197 196 __GFP_COMP | __GFP_MEMALLOC) 198 197 198 + #define IONIC_XDP_MAX_LINEAR_MTU (IONIC_PAGE_SIZE - \ 199 + (VLAN_ETH_HLEN + \ 200 + XDP_PACKET_HEADROOM + \ 201 + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))) 202 + 199 203 struct ionic_buf_info { 200 204 struct page *page; 201 205 dma_addr_t dma_addr; ··· 228 222 struct ionic_buf_info bufs[MAX_SKB_FRAGS + 1]; 229 223 ionic_desc_cb cb; 230 224 void *cb_arg; 225 + struct xdp_frame *xdpf; 226 + enum xdp_action act; 231 227 }; 232 228 233 229 #define IONIC_QUEUE_NAME_MAX_SZ 16 ··· 264 256 struct ionic_txq_sg_desc *txq_sgl; 265 257 struct ionic_rxq_sg_desc *rxq_sgl; 266 258 }; 259 + struct xdp_rxq_info *xdp_rxq_info; 260 + struct ionic_queue *partner; 261 + bool xdp_flush; 267 262 dma_addr_t base_pa; 268 263 dma_addr_t cmb_base_pa; 269 264 dma_addr_t sg_base_pa;

+5

drivers/net/ethernet/pensando/ionic/ionic_ethtool.c

··· 721 721 722 722 ionic_init_queue_params(lif, &qparam); 723 723 724 + if ((ch->rx_count || ch->tx_count) && lif->xdp_prog) { 725 + netdev_info(lif->netdev, "Split Tx/Rx interrupts not available when using XDP\n"); 726 + return -EOPNOTSUPP; 727 + } 728 + 724 729 if (ch->rx_count != ch->tx_count) { 725 730 netdev_info(netdev, "The rx and tx count must be equal\n"); 726 731 return -EINVAL;

+190 -3

drivers/net/ethernet/pensando/ionic/ionic_lif.c

··· 46 46 static void ionic_stop_queues(struct ionic_lif *lif); 47 47 static void ionic_lif_queue_identify(struct ionic_lif *lif); 48 48 49 + static int ionic_xdp_queues_config(struct ionic_lif *lif); 50 + static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q); 51 + 49 52 static void ionic_dim_work(struct work_struct *work) 50 53 { 51 54 struct dim *dim = container_of(work, struct dim, work); ··· 425 422 qcq->sg_base_pa = 0; 426 423 } 427 424 425 + ionic_xdp_unregister_rxq_info(&qcq->q); 428 426 ionic_qcq_intr_free(lif, qcq); 429 427 430 428 vfree(qcq->cq.info); ··· 866 862 .type = q->type, 867 863 .ver = lif->qtype_info[q->type].version, 868 864 .index = cpu_to_le32(q->index), 869 - .flags = cpu_to_le16(IONIC_QINIT_F_IRQ | 870 - IONIC_QINIT_F_SG), 865 + .flags = cpu_to_le16(IONIC_QINIT_F_IRQ), 871 866 .intr_index = cpu_to_le16(cq->bound_intr->index), 872 867 .pid = cpu_to_le16(q->pid), 873 868 .ring_size = ilog2(q->num_descs), ··· 877 874 }, 878 875 }; 879 876 int err; 877 + 878 + q->partner = &lif->txqcqs[q->index]->q; 879 + q->partner->partner = q; 880 + 881 + if (!lif->xdp_prog || 882 + (lif->xdp_prog->aux && lif->xdp_prog->aux->xdp_has_frags)) 883 + ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_SG); 880 884 881 885 if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) { 882 886 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB); ··· 1650 1640 netdev->priv_flags |= IFF_UNICAST_FLT | 1651 1641 IFF_LIVE_ADDR_CHANGE; 1652 1642 1643 + netdev->xdp_features = NETDEV_XDP_ACT_BASIC | 1644 + NETDEV_XDP_ACT_REDIRECT | 1645 + NETDEV_XDP_ACT_RX_SG | 1646 + NETDEV_XDP_ACT_NDO_XMIT | 1647 + NETDEV_XDP_ACT_NDO_XMIT_SG; 1648 + 1653 1649 return 0; 1654 1650 } 1655 1651 ··· 1793 1777 return err; 1794 1778 } 1795 1779 1780 + static bool ionic_xdp_is_valid_mtu(struct ionic_lif *lif, u32 mtu, 1781 + struct bpf_prog *xdp_prog) 1782 + { 1783 + if (!xdp_prog) 1784 + return true; 1785 + 1786 + if (mtu <= IONIC_XDP_MAX_LINEAR_MTU) 1787 + return true; 1788 + 1789 + if (xdp_prog->aux && xdp_prog->aux->xdp_has_frags) 1790 + return true; 1791 + 1792 + return false; 1793 + } 1794 + 1796 1795 static int ionic_change_mtu(struct net_device *netdev, int new_mtu) 1797 1796 { 1798 1797 struct ionic_lif *lif = netdev_priv(netdev); ··· 1820 1789 .mtu = cpu_to_le32(new_mtu), 1821 1790 }, 1822 1791 }; 1792 + struct bpf_prog *xdp_prog; 1823 1793 int err; 1794 + 1795 + xdp_prog = READ_ONCE(lif->xdp_prog); 1796 + if (!ionic_xdp_is_valid_mtu(lif, new_mtu, xdp_prog)) 1797 + return -EINVAL; 1824 1798 1825 1799 err = ionic_adminq_post_wait(lif, &ctx); 1826 1800 if (err) ··· 2202 2166 int derr = 0; 2203 2167 int i, err; 2204 2168 2169 + err = ionic_xdp_queues_config(lif); 2170 + if (err) 2171 + return err; 2172 + 2205 2173 for (i = 0; i < lif->nxqs; i++) { 2206 2174 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) { 2207 2175 dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i); ··· 2250 2210 derr = ionic_qcq_disable(lif, lif->txqcqs[i], derr); 2251 2211 derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr); 2252 2212 } 2213 + 2214 + ionic_xdp_queues_config(lif); 2253 2215 2254 2216 return err; 2255 2217 } ··· 2710 2668 ionic_vf_start(ionic); 2711 2669 } 2712 2670 2671 + static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q) 2672 + { 2673 + struct xdp_rxq_info *xi; 2674 + 2675 + if (!q->xdp_rxq_info) 2676 + return; 2677 + 2678 + xi = q->xdp_rxq_info; 2679 + q->xdp_rxq_info = NULL; 2680 + 2681 + xdp_rxq_info_unreg(xi); 2682 + kfree(xi); 2683 + } 2684 + 2685 + static int ionic_xdp_register_rxq_info(struct ionic_queue *q, unsigned int napi_id) 2686 + { 2687 + struct xdp_rxq_info *rxq_info; 2688 + int err; 2689 + 2690 + rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL); 2691 + if (!rxq_info) 2692 + return -ENOMEM; 2693 + 2694 + err = xdp_rxq_info_reg(rxq_info, q->lif->netdev, q->index, napi_id); 2695 + if (err) { 2696 + dev_err(q->dev, "Queue %d xdp_rxq_info_reg failed, err %d\n", 2697 + q->index, err); 2698 + goto err_out; 2699 + } 2700 + 2701 + err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_ORDER0, NULL); 2702 + if (err) { 2703 + dev_err(q->dev, "Queue %d xdp_rxq_info_reg_mem_model failed, err %d\n", 2704 + q->index, err); 2705 + xdp_rxq_info_unreg(rxq_info); 2706 + goto err_out; 2707 + } 2708 + 2709 + q->xdp_rxq_info = rxq_info; 2710 + 2711 + return 0; 2712 + 2713 + err_out: 2714 + kfree(rxq_info); 2715 + return err; 2716 + } 2717 + 2718 + static int ionic_xdp_queues_config(struct ionic_lif *lif) 2719 + { 2720 + unsigned int i; 2721 + int err; 2722 + 2723 + if (!lif->rxqcqs) 2724 + return 0; 2725 + 2726 + /* There's no need to rework memory if not going to/from NULL program. 2727 + * If there is no lif->xdp_prog, there should also be no q.xdp_rxq_info 2728 + * This way we don't need to keep an *xdp_prog in every queue struct. 2729 + */ 2730 + if (!lif->xdp_prog == !lif->rxqcqs[0]->q.xdp_rxq_info) 2731 + return 0; 2732 + 2733 + for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) { 2734 + struct ionic_queue *q = &lif->rxqcqs[i]->q; 2735 + 2736 + if (q->xdp_rxq_info) { 2737 + ionic_xdp_unregister_rxq_info(q); 2738 + continue; 2739 + } 2740 + 2741 + err = ionic_xdp_register_rxq_info(q, lif->rxqcqs[i]->napi.napi_id); 2742 + if (err) { 2743 + dev_err(lif->ionic->dev, "failed to register RX queue %d info for XDP, err %d\n", 2744 + i, err); 2745 + goto err_out; 2746 + } 2747 + } 2748 + 2749 + return 0; 2750 + 2751 + err_out: 2752 + for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) 2753 + ionic_xdp_unregister_rxq_info(&lif->rxqcqs[i]->q); 2754 + 2755 + return err; 2756 + } 2757 + 2758 + static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf) 2759 + { 2760 + struct ionic_lif *lif = netdev_priv(netdev); 2761 + struct bpf_prog *old_prog; 2762 + u32 maxfs; 2763 + 2764 + if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) { 2765 + #define XDP_ERR_SPLIT "XDP not available with split Tx/Rx interrupts" 2766 + NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_SPLIT); 2767 + netdev_info(lif->netdev, XDP_ERR_SPLIT); 2768 + return -EOPNOTSUPP; 2769 + } 2770 + 2771 + if (!ionic_xdp_is_valid_mtu(lif, netdev->mtu, bpf->prog)) { 2772 + #define XDP_ERR_MTU "MTU is too large for XDP without frags support" 2773 + NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_MTU); 2774 + netdev_info(lif->netdev, XDP_ERR_MTU); 2775 + return -EINVAL; 2776 + } 2777 + 2778 + maxfs = __le32_to_cpu(lif->identity->eth.max_frame_size) - VLAN_ETH_HLEN; 2779 + if (bpf->prog && !(bpf->prog->aux && bpf->prog->aux->xdp_has_frags)) 2780 + maxfs = min_t(u32, maxfs, IONIC_XDP_MAX_LINEAR_MTU); 2781 + netdev->max_mtu = maxfs; 2782 + 2783 + if (!netif_running(netdev)) { 2784 + old_prog = xchg(&lif->xdp_prog, bpf->prog); 2785 + } else { 2786 + mutex_lock(&lif->queue_lock); 2787 + ionic_stop_queues_reconfig(lif); 2788 + old_prog = xchg(&lif->xdp_prog, bpf->prog); 2789 + ionic_start_queues_reconfig(lif); 2790 + mutex_unlock(&lif->queue_lock); 2791 + } 2792 + 2793 + if (old_prog) 2794 + bpf_prog_put(old_prog); 2795 + 2796 + return 0; 2797 + } 2798 + 2799 + static int ionic_xdp(struct net_device *netdev, struct netdev_bpf *bpf) 2800 + { 2801 + switch (bpf->command) { 2802 + case XDP_SETUP_PROG: 2803 + return ionic_xdp_config(netdev, bpf); 2804 + default: 2805 + return -EINVAL; 2806 + } 2807 + } 2808 + 2713 2809 static const struct net_device_ops ionic_netdev_ops = { 2714 2810 .ndo_open = ionic_open, 2715 2811 .ndo_stop = ionic_stop, 2716 2812 .ndo_eth_ioctl = ionic_eth_ioctl, 2717 2813 .ndo_start_xmit = ionic_start_xmit, 2814 + .ndo_bpf = ionic_xdp, 2815 + .ndo_xdp_xmit = ionic_xdp_xmit, 2718 2816 .ndo_get_stats64 = ionic_get_stats64, 2719 2817 .ndo_set_rx_mode = ionic_ndo_set_rx_mode, 2720 2818 .ndo_set_features = ionic_set_features, ··· 2937 2755 swap(a->q.base, b->q.base); 2938 2756 swap(a->q.base_pa, b->q.base_pa); 2939 2757 swap(a->q.info, b->q.info); 2758 + swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info); 2759 + swap(a->q.partner, b->q.partner); 2940 2760 swap(a->q_base, b->q_base); 2941 2761 swap(a->q_base_pa, b->q_base_pa); 2942 2762 swap(a->q_size, b->q_size); ··· 3575 3391 3576 3392 napi_enable(&qcq->napi); 3577 3393 3578 - if (qcq->flags & IONIC_QCQ_F_INTR) 3394 + if (qcq->flags & IONIC_QCQ_F_INTR) { 3395 + irq_set_affinity_hint(qcq->intr.vector, 3396 + &qcq->intr.affinity_mask); 3579 3397 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 3580 3398 IONIC_INTR_MASK_CLEAR); 3399 + } 3581 3400 3582 3401 qcq->flags |= IONIC_QCQ_F_INITED; 3583 3402

+13

drivers/net/ethernet/pensando/ionic/ionic_lif.h

··· 37 37 u64 dma_map_err; 38 38 u64 hwstamp_valid; 39 39 u64 hwstamp_invalid; 40 + u64 xdp_frames; 40 41 }; 41 42 42 43 struct ionic_rx_stats { ··· 52 51 u64 alloc_err; 53 52 u64 hwstamp_valid; 54 53 u64 hwstamp_invalid; 54 + u64 xdp_drop; 55 + u64 xdp_aborted; 56 + u64 xdp_pass; 57 + u64 xdp_tx; 58 + u64 xdp_redirect; 55 59 }; 56 60 57 61 #define IONIC_QCQ_F_INITED BIT(0) ··· 141 135 u64 hw_rx_over_errors; 142 136 u64 hw_rx_missed_errors; 143 137 u64 hw_tx_aborted_errors; 138 + u64 xdp_drop; 139 + u64 xdp_aborted; 140 + u64 xdp_pass; 141 + u64 xdp_tx; 142 + u64 xdp_redirect; 143 + u64 xdp_frames; 144 144 }; 145 145 146 146 enum ionic_lif_state_flags { ··· 242 230 struct ionic_phc *phc; 243 231 244 232 struct dentry *dentry; 233 + struct bpf_prog *xdp_prog; 245 234 }; 246 235 247 236 struct ionic_phc {

+18

drivers/net/ethernet/pensando/ionic/ionic_stats.c

··· 27 27 IONIC_LIF_STAT_DESC(hw_rx_over_errors), 28 28 IONIC_LIF_STAT_DESC(hw_rx_missed_errors), 29 29 IONIC_LIF_STAT_DESC(hw_tx_aborted_errors), 30 + IONIC_LIF_STAT_DESC(xdp_drop), 31 + IONIC_LIF_STAT_DESC(xdp_aborted), 32 + IONIC_LIF_STAT_DESC(xdp_pass), 33 + IONIC_LIF_STAT_DESC(xdp_tx), 34 + IONIC_LIF_STAT_DESC(xdp_redirect), 35 + IONIC_LIF_STAT_DESC(xdp_frames), 30 36 }; 31 37 32 38 static const struct ionic_stat_desc ionic_port_stats_desc[] = { ··· 141 135 IONIC_TX_STAT_DESC(csum_none), 142 136 IONIC_TX_STAT_DESC(csum), 143 137 IONIC_TX_STAT_DESC(vlan_inserted), 138 + IONIC_TX_STAT_DESC(xdp_frames), 144 139 }; 145 140 146 141 static const struct ionic_stat_desc ionic_rx_stats_desc[] = { ··· 156 149 IONIC_RX_STAT_DESC(hwstamp_invalid), 157 150 IONIC_RX_STAT_DESC(dropped), 158 151 IONIC_RX_STAT_DESC(vlan_stripped), 152 + IONIC_RX_STAT_DESC(xdp_drop), 153 + IONIC_RX_STAT_DESC(xdp_aborted), 154 + IONIC_RX_STAT_DESC(xdp_pass), 155 + IONIC_RX_STAT_DESC(xdp_tx), 156 + IONIC_RX_STAT_DESC(xdp_redirect), 159 157 }; 160 158 161 159 #define IONIC_NUM_LIF_STATS ARRAY_SIZE(ionic_lif_stats_desc) ··· 183 171 stats->tx_csum += txstats->csum; 184 172 stats->tx_hwstamp_valid += txstats->hwstamp_valid; 185 173 stats->tx_hwstamp_invalid += txstats->hwstamp_invalid; 174 + stats->xdp_frames += txstats->xdp_frames; 186 175 } 187 176 188 177 static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num, ··· 198 185 stats->rx_csum_error += rxstats->csum_error; 199 186 stats->rx_hwstamp_valid += rxstats->hwstamp_valid; 200 187 stats->rx_hwstamp_invalid += rxstats->hwstamp_invalid; 188 + stats->xdp_drop += rxstats->xdp_drop; 189 + stats->xdp_aborted += rxstats->xdp_aborted; 190 + stats->xdp_pass += rxstats->xdp_pass; 191 + stats->xdp_tx += rxstats->xdp_tx; 192 + stats->xdp_redirect += rxstats->xdp_redirect; 201 193 } 202 194 203 195 static void ionic_get_lif_stats(struct ionic_lif *lif,

+425 -35

drivers/net/ethernet/pensando/ionic/ionic_txrx.c

··· 10 10 #include "ionic_lif.h" 11 11 #include "ionic_txrx.h" 12 12 13 + static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs); 14 + 15 + static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 16 + void *data, size_t len); 17 + 18 + static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 19 + const skb_frag_t *frag, 20 + size_t offset, size_t len); 21 + 22 + static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 23 + struct ionic_desc_info *desc_info); 24 + 25 + static void ionic_tx_clean(struct ionic_queue *q, 26 + struct ionic_desc_info *desc_info, 27 + struct ionic_cq_info *cq_info, 28 + void *cb_arg); 29 + 13 30 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell, 14 31 ionic_desc_cb cb_func, void *cb_arg) 15 32 { ··· 105 88 return netdev_get_tx_queue(q->lif->netdev, q->index); 106 89 } 107 90 91 + static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info) 92 + { 93 + return page_address(buf_info->page) + buf_info->page_offset; 94 + } 95 + 96 + static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info) 97 + { 98 + return buf_info->dma_addr + buf_info->page_offset; 99 + } 100 + 101 + static unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info) 102 + { 103 + return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset); 104 + } 105 + 108 106 static int ionic_rx_page_alloc(struct ionic_queue *q, 109 107 struct ionic_buf_info *buf_info) 110 108 { ··· 194 162 if (page_to_nid(buf_info->page) != numa_mem_id()) 195 163 return false; 196 164 197 - size = ALIGN(used, IONIC_PAGE_SPLIT_SZ); 165 + size = ALIGN(used, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ); 198 166 buf_info->page_offset += size; 199 167 if (buf_info->page_offset >= IONIC_PAGE_SIZE) 200 168 return false; ··· 206 174 207 175 static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, 208 176 struct ionic_desc_info *desc_info, 209 - struct ionic_rxq_comp *comp) 177 + unsigned int headroom, 178 + unsigned int len, 179 + unsigned int num_sg_elems, 180 + bool synced) 210 181 { 211 182 struct net_device *netdev = q->lif->netdev; 212 183 struct ionic_buf_info *buf_info; ··· 218 183 struct sk_buff *skb; 219 184 unsigned int i; 220 185 u16 frag_len; 221 - u16 len; 222 186 223 187 stats = q_to_rx_stats(q); 224 188 225 189 buf_info = &desc_info->bufs[0]; 226 - len = le16_to_cpu(comp->len); 227 190 228 191 prefetchw(buf_info->page); 229 192 ··· 233 200 return NULL; 234 201 } 235 202 236 - i = comp->num_sg_elems + 1; 203 + i = num_sg_elems + 1; 237 204 do { 238 205 if (unlikely(!buf_info->page)) { 239 206 dev_kfree_skb(skb); 240 207 return NULL; 241 208 } 242 209 243 - frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN, 244 - IONIC_PAGE_SIZE - buf_info->page_offset)); 210 + if (headroom) 211 + frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 212 + else 213 + frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); 245 214 len -= frag_len; 246 215 247 - dma_sync_single_for_cpu(dev, 248 - buf_info->dma_addr + buf_info->page_offset, 249 - frag_len, DMA_FROM_DEVICE); 216 + if (!synced) 217 + dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), 218 + headroom, frag_len, DMA_FROM_DEVICE); 250 219 251 220 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 252 - buf_info->page, buf_info->page_offset, frag_len, 253 - IONIC_PAGE_SIZE); 221 + buf_info->page, buf_info->page_offset + headroom, 222 + frag_len, IONIC_PAGE_SIZE); 254 223 255 224 if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) { 256 225 dma_unmap_page(dev, buf_info->dma_addr, 257 226 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 258 227 buf_info->page = NULL; 259 228 } 229 + 230 + /* only needed on the first buffer */ 231 + if (headroom) 232 + headroom = 0; 260 233 261 234 buf_info++; 262 235 ··· 274 235 275 236 static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, 276 237 struct ionic_desc_info *desc_info, 277 - struct ionic_rxq_comp *comp) 238 + unsigned int headroom, 239 + unsigned int len, 240 + bool synced) 278 241 { 279 242 struct net_device *netdev = q->lif->netdev; 280 243 struct ionic_buf_info *buf_info; 281 244 struct ionic_rx_stats *stats; 282 245 struct device *dev = q->dev; 283 246 struct sk_buff *skb; 284 - u16 len; 285 247 286 248 stats = q_to_rx_stats(q); 287 249 288 250 buf_info = &desc_info->bufs[0]; 289 - len = le16_to_cpu(comp->len); 290 251 291 252 skb = napi_alloc_skb(&q_to_qcq(q)->napi, len); 292 253 if (unlikely(!skb)) { ··· 301 262 return NULL; 302 263 } 303 264 304 - dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset, 305 - len, DMA_FROM_DEVICE); 306 - skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len); 307 - dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset, 308 - len, DMA_FROM_DEVICE); 265 + if (!synced) 266 + dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), 267 + headroom, len, DMA_FROM_DEVICE); 268 + skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len); 269 + dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info), 270 + headroom, len, DMA_FROM_DEVICE); 309 271 310 272 skb_put(skb, len); 311 273 skb->protocol = eth_type_trans(skb, q->lif->netdev); 312 274 313 275 return skb; 276 + } 277 + 278 + static void ionic_xdp_tx_desc_clean(struct ionic_queue *q, 279 + struct ionic_desc_info *desc_info) 280 + { 281 + unsigned int nbufs = desc_info->nbufs; 282 + struct ionic_buf_info *buf_info; 283 + struct device *dev = q->dev; 284 + int i; 285 + 286 + if (!nbufs) 287 + return; 288 + 289 + buf_info = desc_info->bufs; 290 + dma_unmap_single(dev, buf_info->dma_addr, 291 + buf_info->len, DMA_TO_DEVICE); 292 + if (desc_info->act == XDP_TX) 293 + __free_pages(buf_info->page, 0); 294 + buf_info->page = NULL; 295 + 296 + buf_info++; 297 + for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) { 298 + dma_unmap_page(dev, buf_info->dma_addr, 299 + buf_info->len, DMA_TO_DEVICE); 300 + if (desc_info->act == XDP_TX) 301 + __free_pages(buf_info->page, 0); 302 + buf_info->page = NULL; 303 + } 304 + 305 + if (desc_info->act == XDP_REDIRECT) 306 + xdp_return_frame(desc_info->xdpf); 307 + 308 + desc_info->nbufs = 0; 309 + desc_info->xdpf = NULL; 310 + desc_info->act = 0; 311 + } 312 + 313 + static int ionic_xdp_post_frame(struct net_device *netdev, 314 + struct ionic_queue *q, struct xdp_frame *frame, 315 + enum xdp_action act, struct page *page, int off, 316 + bool ring_doorbell) 317 + { 318 + struct ionic_desc_info *desc_info; 319 + struct ionic_buf_info *buf_info; 320 + struct ionic_tx_stats *stats; 321 + struct ionic_txq_desc *desc; 322 + size_t len = frame->len; 323 + dma_addr_t dma_addr; 324 + u64 cmd; 325 + 326 + desc_info = &q->info[q->head_idx]; 327 + desc = desc_info->txq_desc; 328 + buf_info = desc_info->bufs; 329 + stats = q_to_tx_stats(q); 330 + 331 + dma_addr = ionic_tx_map_single(q, frame->data, len); 332 + if (dma_mapping_error(q->dev, dma_addr)) { 333 + stats->dma_map_err++; 334 + return -EIO; 335 + } 336 + buf_info->dma_addr = dma_addr; 337 + buf_info->len = len; 338 + buf_info->page = page; 339 + buf_info->page_offset = off; 340 + 341 + desc_info->nbufs = 1; 342 + desc_info->xdpf = frame; 343 + desc_info->act = act; 344 + 345 + if (xdp_frame_has_frags(frame)) { 346 + struct ionic_txq_sg_elem *elem; 347 + struct skb_shared_info *sinfo; 348 + struct ionic_buf_info *bi; 349 + skb_frag_t *frag; 350 + int i; 351 + 352 + bi = &buf_info[1]; 353 + sinfo = xdp_get_shared_info_from_frame(frame); 354 + frag = sinfo->frags; 355 + elem = desc_info->txq_sg_desc->elems; 356 + for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) { 357 + dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 358 + if (dma_mapping_error(q->dev, dma_addr)) { 359 + stats->dma_map_err++; 360 + ionic_tx_desc_unmap_bufs(q, desc_info); 361 + return -EIO; 362 + } 363 + bi->dma_addr = dma_addr; 364 + bi->len = skb_frag_size(frag); 365 + bi->page = skb_frag_page(frag); 366 + 367 + elem->addr = cpu_to_le64(bi->dma_addr); 368 + elem->len = cpu_to_le16(bi->len); 369 + elem++; 370 + 371 + desc_info->nbufs++; 372 + } 373 + } 374 + 375 + cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 376 + 0, (desc_info->nbufs - 1), buf_info->dma_addr); 377 + desc->cmd = cpu_to_le64(cmd); 378 + desc->len = cpu_to_le16(len); 379 + desc->csum_start = 0; 380 + desc->csum_offset = 0; 381 + 382 + stats->xdp_frames++; 383 + stats->pkts++; 384 + stats->bytes += len; 385 + 386 + ionic_txq_post(q, ring_doorbell, ionic_tx_clean, NULL); 387 + 388 + return 0; 389 + } 390 + 391 + int ionic_xdp_xmit(struct net_device *netdev, int n, 392 + struct xdp_frame **xdp_frames, u32 flags) 393 + { 394 + struct ionic_lif *lif = netdev_priv(netdev); 395 + struct ionic_queue *txq; 396 + struct netdev_queue *nq; 397 + int nxmit; 398 + int space; 399 + int cpu; 400 + int qi; 401 + 402 + if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) 403 + return -ENETDOWN; 404 + 405 + if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 406 + return -EINVAL; 407 + 408 + /* AdminQ is assumed on cpu 0, while we attempt to affinitize the 409 + * TxRx queue pairs 0..n-1 on cpus 1..n. We try to keep with that 410 + * affinitization here, but of course irqbalance and friends might 411 + * have juggled things anyway, so we have to check for the 0 case. 412 + */ 413 + cpu = smp_processor_id(); 414 + qi = cpu ? (cpu - 1) % lif->nxqs : cpu; 415 + 416 + txq = &lif->txqcqs[qi]->q; 417 + nq = netdev_get_tx_queue(netdev, txq->index); 418 + __netif_tx_lock(nq, cpu); 419 + txq_trans_cond_update(nq); 420 + 421 + if (netif_tx_queue_stopped(nq) || 422 + unlikely(ionic_maybe_stop_tx(txq, 1))) { 423 + __netif_tx_unlock(nq); 424 + return -EIO; 425 + } 426 + 427 + space = min_t(int, n, ionic_q_space_avail(txq)); 428 + for (nxmit = 0; nxmit < space ; nxmit++) { 429 + if (ionic_xdp_post_frame(netdev, txq, xdp_frames[nxmit], 430 + XDP_REDIRECT, 431 + virt_to_page(xdp_frames[nxmit]->data), 432 + 0, false)) { 433 + nxmit--; 434 + break; 435 + } 436 + } 437 + 438 + if (flags & XDP_XMIT_FLUSH) 439 + ionic_dbell_ring(lif->kern_dbpage, txq->hw_type, 440 + txq->dbval | txq->head_idx); 441 + 442 + ionic_maybe_stop_tx(txq, 4); 443 + __netif_tx_unlock(nq); 444 + 445 + return nxmit; 446 + } 447 + 448 + static bool ionic_run_xdp(struct ionic_rx_stats *stats, 449 + struct net_device *netdev, 450 + struct bpf_prog *xdp_prog, 451 + struct ionic_queue *rxq, 452 + struct ionic_buf_info *buf_info, 453 + int len) 454 + { 455 + u32 xdp_action = XDP_ABORTED; 456 + struct xdp_buff xdp_buf; 457 + struct ionic_queue *txq; 458 + struct netdev_queue *nq; 459 + struct xdp_frame *xdpf; 460 + int remain_len; 461 + int frag_len; 462 + int err = 0; 463 + 464 + xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info); 465 + frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 466 + xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info), 467 + XDP_PACKET_HEADROOM, frag_len, false); 468 + 469 + dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info), 470 + XDP_PACKET_HEADROOM, len, 471 + DMA_FROM_DEVICE); 472 + 473 + prefetchw(&xdp_buf.data_hard_start); 474 + 475 + /* We limit MTU size to one buffer if !xdp_has_frags, so 476 + * if the recv len is bigger than one buffer 477 + * then we know we have frag info to gather 478 + */ 479 + remain_len = len - frag_len; 480 + if (remain_len) { 481 + struct skb_shared_info *sinfo; 482 + struct ionic_buf_info *bi; 483 + skb_frag_t *frag; 484 + 485 + bi = buf_info; 486 + sinfo = xdp_get_shared_info_from_buff(&xdp_buf); 487 + sinfo->nr_frags = 0; 488 + sinfo->xdp_frags_size = 0; 489 + xdp_buff_set_frags_flag(&xdp_buf); 490 + 491 + do { 492 + if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) { 493 + err = -ENOSPC; 494 + goto out_xdp_abort; 495 + } 496 + 497 + frag = &sinfo->frags[sinfo->nr_frags]; 498 + sinfo->nr_frags++; 499 + bi++; 500 + frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi)); 501 + dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi), 502 + 0, frag_len, DMA_FROM_DEVICE); 503 + skb_frag_fill_page_desc(frag, bi->page, 0, frag_len); 504 + sinfo->xdp_frags_size += frag_len; 505 + remain_len -= frag_len; 506 + 507 + if (page_is_pfmemalloc(bi->page)) 508 + xdp_buff_set_frag_pfmemalloc(&xdp_buf); 509 + } while (remain_len > 0); 510 + } 511 + 512 + xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf); 513 + 514 + switch (xdp_action) { 515 + case XDP_PASS: 516 + stats->xdp_pass++; 517 + return false; /* false = we didn't consume the packet */ 518 + 519 + case XDP_DROP: 520 + ionic_rx_page_free(rxq, buf_info); 521 + stats->xdp_drop++; 522 + break; 523 + 524 + case XDP_TX: 525 + xdpf = xdp_convert_buff_to_frame(&xdp_buf); 526 + if (!xdpf) 527 + goto out_xdp_abort; 528 + 529 + txq = rxq->partner; 530 + nq = netdev_get_tx_queue(netdev, txq->index); 531 + __netif_tx_lock(nq, smp_processor_id()); 532 + txq_trans_cond_update(nq); 533 + 534 + if (netif_tx_queue_stopped(nq) || 535 + unlikely(ionic_maybe_stop_tx(txq, 1))) { 536 + __netif_tx_unlock(nq); 537 + goto out_xdp_abort; 538 + } 539 + 540 + dma_unmap_page(rxq->dev, buf_info->dma_addr, 541 + IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 542 + 543 + err = ionic_xdp_post_frame(netdev, txq, xdpf, XDP_TX, 544 + buf_info->page, 545 + buf_info->page_offset, 546 + true); 547 + __netif_tx_unlock(nq); 548 + if (err) { 549 + netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err); 550 + goto out_xdp_abort; 551 + } 552 + stats->xdp_tx++; 553 + 554 + /* the Tx completion will free the buffers */ 555 + break; 556 + 557 + case XDP_REDIRECT: 558 + /* unmap the pages before handing them to a different device */ 559 + dma_unmap_page(rxq->dev, buf_info->dma_addr, 560 + IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 561 + 562 + err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog); 563 + if (err) { 564 + netdev_dbg(netdev, "xdp_do_redirect err %d\n", err); 565 + goto out_xdp_abort; 566 + } 567 + buf_info->page = NULL; 568 + rxq->xdp_flush = true; 569 + stats->xdp_redirect++; 570 + break; 571 + 572 + case XDP_ABORTED: 573 + default: 574 + goto out_xdp_abort; 575 + } 576 + 577 + return true; 578 + 579 + out_xdp_abort: 580 + trace_xdp_exception(netdev, xdp_prog, xdp_action); 581 + ionic_rx_page_free(rxq, buf_info); 582 + stats->xdp_aborted++; 583 + 584 + return true; 314 585 } 315 586 316 587 static void ionic_rx_clean(struct ionic_queue *q, ··· 632 283 struct ionic_qcq *qcq = q_to_qcq(q); 633 284 struct ionic_rx_stats *stats; 634 285 struct ionic_rxq_comp *comp; 286 + struct bpf_prog *xdp_prog; 287 + unsigned int headroom; 635 288 struct sk_buff *skb; 289 + u16 len; 636 290 637 291 comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp); 638 292 ··· 646 294 return; 647 295 } 648 296 297 + len = le16_to_cpu(comp->len); 649 298 stats->pkts++; 650 - stats->bytes += le16_to_cpu(comp->len); 299 + stats->bytes += len; 651 300 652 - if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 653 - skb = ionic_rx_copybreak(q, desc_info, comp); 301 + xdp_prog = READ_ONCE(q->lif->xdp_prog); 302 + if (xdp_prog && 303 + ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len)) 304 + return; 305 + 306 + headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 307 + if (len <= q->lif->rx_copybreak) 308 + skb = ionic_rx_copybreak(q, desc_info, headroom, len, !!xdp_prog); 654 309 else 655 - skb = ionic_rx_frags(q, desc_info, comp); 310 + skb = ionic_rx_frags(q, desc_info, headroom, len, 311 + comp->num_sg_elems, !!xdp_prog); 656 312 657 313 if (unlikely(!skb)) { 658 314 stats->dropped++; ··· 727 367 } 728 368 } 729 369 730 - if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 370 + if (len <= q->lif->rx_copybreak) 731 371 napi_gro_receive(&qcq->napi, skb); 732 372 else 733 373 napi_gro_frags(&qcq->napi); ··· 784 424 unsigned int frag_len; 785 425 unsigned int nfrags; 786 426 unsigned int n_fill; 787 - unsigned int i, j; 788 427 unsigned int len; 428 + unsigned int i; 429 + unsigned int j; 789 430 790 431 n_fill = ionic_q_space_avail(q); 791 432 ··· 795 434 if (n_fill < fill_threshold) 796 435 return; 797 436 798 - len = netdev->mtu + ETH_HLEN + VLAN_HLEN; 437 + len = netdev->mtu + VLAN_ETH_HLEN; 799 438 800 439 for (i = n_fill; i; i--) { 440 + unsigned int headroom; 441 + unsigned int buf_len; 442 + 801 443 nfrags = 0; 802 444 remain_len = len; 803 445 desc_info = &q->info[q->head_idx]; ··· 815 451 } 816 452 } 817 453 818 - /* fill main descriptor - buf[0] */ 819 - desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); 820 - frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN, 821 - IONIC_PAGE_SIZE - buf_info->page_offset)); 454 + /* fill main descriptor - buf[0] 455 + * XDP uses space in the first buffer, so account for 456 + * head room, tail room, and ip header in the first frag size. 457 + */ 458 + headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 459 + if (q->xdp_rxq_info) 460 + buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN; 461 + else 462 + buf_len = ionic_rx_buf_size(buf_info); 463 + frag_len = min_t(u16, len, buf_len); 464 + 465 + desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom); 822 466 desc->len = cpu_to_le16(frag_len); 823 467 remain_len -= frag_len; 824 468 buf_info++; ··· 844 472 } 845 473 } 846 474 847 - sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); 848 - frag_len = min_t(u16, remain_len, min_t(u32, IONIC_MAX_BUF_LEN, 849 - IONIC_PAGE_SIZE - 850 - buf_info->page_offset)); 475 + sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info)); 476 + frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info)); 851 477 sg_elem->len = cpu_to_le16(frag_len); 852 478 remain_len -= frag_len; 853 479 buf_info++; ··· 971 601 return work_done; 972 602 } 973 603 604 + static void ionic_xdp_do_flush(struct ionic_cq *cq) 605 + { 606 + if (cq->bound_q->xdp_flush) { 607 + xdp_do_flush(); 608 + cq->bound_q->xdp_flush = false; 609 + } 610 + } 611 + 974 612 int ionic_rx_napi(struct napi_struct *napi, int budget) 975 613 { 976 614 struct ionic_qcq *qcq = napi_to_qcq(napi); ··· 999 621 1000 622 ionic_rx_fill(cq->bound_q); 1001 623 624 + ionic_xdp_do_flush(cq); 1002 625 if (work_done < budget && napi_complete_done(napi, work_done)) { 1003 626 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); 1004 627 flags |= IONIC_INTR_CRED_UNMASK; ··· 1049 670 1050 671 ionic_rx_fill(rxcq->bound_q); 1051 672 673 + ionic_xdp_do_flush(rxcq); 1052 674 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 1053 675 ionic_dim_update(rxqcq, 0); 1054 676 flags |= IONIC_INTR_CRED_UNMASK; ··· 1184 804 struct ionic_qcq *qcq = q_to_qcq(q); 1185 805 struct sk_buff *skb = cb_arg; 1186 806 u16 qi; 807 + 808 + if (desc_info->xdpf) { 809 + ionic_xdp_tx_desc_clean(q->partner, desc_info); 810 + stats->clean++; 811 + 812 + if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index))) 813 + netif_wake_subqueue(q->lif->netdev, q->index); 814 + 815 + return; 816 + } 1187 817 1188 818 ionic_tx_desc_unmap_bufs(q, desc_info); 1189 819

+1

drivers/net/ethernet/pensando/ionic/ionic_txrx.h

··· 17 17 bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info); 18 18 bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info); 19 19 20 + int ionic_xdp_xmit(struct net_device *netdev, int n, struct xdp_frame **xdp, u32 flags); 20 21 #endif /* _IONIC_TXRX_H_ */

Configure Feed

Configure Feed