Merge branch 'net-ti-icssg-prueth-add-native-mode-xdp-support'

+1

drivers/net/ethernet/ti/Kconfig

··· 205 205 select PHYLIB 206 206 select TI_ICSS_IEP 207 207 select TI_K3_CPPI_DESC_POOL 208 + select PAGE_POOL 208 209 depends on PRU_REMOTEPROC 209 210 depends on NET_SWITCHDEV 210 211 depends on ARCH_K3 && OF && TI_K3_UDMA_GLUE_LAYER

+341 -80

drivers/net/ethernet/ti/icssg/icssg_common.c

··· 45 45 struct prueth_rx_chn *rx_chn, 46 46 int max_rflows) 47 47 { 48 + if (rx_chn->pg_pool) { 49 + page_pool_destroy(rx_chn->pg_pool); 50 + rx_chn->pg_pool = NULL; 51 + } 52 + 48 53 if (rx_chn->desc_pool) 49 54 k3_cppi_desc_pool_destroy(rx_chn->desc_pool); 50 55 ··· 98 93 { 99 94 struct cppi5_host_desc_t *first_desc, *next_desc; 100 95 dma_addr_t buf_dma, next_desc_dma; 96 + struct prueth_swdata *swdata; 97 + struct page *page; 101 98 u32 buf_dma_len; 102 99 103 100 first_desc = desc; 104 101 next_desc = first_desc; 102 + 103 + swdata = cppi5_hdesc_get_swdata(desc); 104 + if (swdata->type == PRUETH_SWDATA_PAGE) { 105 + page = swdata->data.page; 106 + page_pool_recycle_direct(page->pp, swdata->data.page); 107 + goto free_desc; 108 + } 105 109 106 110 cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len); 107 111 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma); ··· 135 121 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc); 136 122 } 137 123 124 + free_desc: 138 125 k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc); 139 126 } 140 127 EXPORT_SYMBOL_GPL(prueth_xmit_free); ··· 146 131 struct net_device *ndev = emac->ndev; 147 132 struct cppi5_host_desc_t *desc_tx; 148 133 struct netdev_queue *netif_txq; 134 + struct prueth_swdata *swdata; 149 135 struct prueth_tx_chn *tx_chn; 150 136 unsigned int total_bytes = 0; 137 + struct xdp_frame *xdpf; 151 138 struct sk_buff *skb; 152 139 dma_addr_t desc_dma; 153 140 int res, num_tx = 0; 154 - void **swdata; 155 141 156 142 tx_chn = &emac->tx_chns[chn]; 157 143 ··· 173 157 desc_dma); 174 158 swdata = cppi5_hdesc_get_swdata(desc_tx); 175 159 176 - /* was this command's TX complete? */ 177 - if (emac->is_sr1 && *(swdata) == emac->cmd_data) { 160 + switch (swdata->type) { 161 + case PRUETH_SWDATA_SKB: 162 + skb = swdata->data.skb; 163 + dev_sw_netstats_tx_add(skb->dev, 1, skb->len); 164 + total_bytes += skb->len; 165 + napi_consume_skb(skb, budget); 166 + break; 167 + case PRUETH_SWDATA_XDPF: 168 + xdpf = swdata->data.xdpf; 169 + dev_sw_netstats_tx_add(ndev, 1, xdpf->len); 170 + total_bytes += xdpf->len; 171 + xdp_return_frame(xdpf); 172 + break; 173 + default: 174 + netdev_err(ndev, "tx_complete: invalid swdata type %d\n", swdata->type); 178 175 prueth_xmit_free(tx_chn, desc_tx); 176 + ndev->stats.tx_dropped++; 179 177 continue; 180 178 } 181 179 182 - skb = *(swdata); 183 180 prueth_xmit_free(tx_chn, desc_tx); 184 - 185 - ndev = skb->dev; 186 - ndev->stats.tx_packets++; 187 - ndev->stats.tx_bytes += skb->len; 188 - total_bytes += skb->len; 189 - napi_consume_skb(skb, budget); 190 181 num_tx++; 191 182 } 192 183 ··· 484 461 } 485 462 EXPORT_SYMBOL_GPL(prueth_init_rx_chns); 486 463 487 - int prueth_dma_rx_push(struct prueth_emac *emac, 488 - struct sk_buff *skb, 489 - struct prueth_rx_chn *rx_chn) 464 + int prueth_dma_rx_push_mapped(struct prueth_emac *emac, 465 + struct prueth_rx_chn *rx_chn, 466 + struct page *page, u32 buf_len) 490 467 { 491 468 struct net_device *ndev = emac->ndev; 492 469 struct cppi5_host_desc_t *desc_rx; 493 - u32 pkt_len = skb_tailroom(skb); 470 + struct prueth_swdata *swdata; 494 471 dma_addr_t desc_dma; 495 472 dma_addr_t buf_dma; 496 - void **swdata; 497 473 474 + buf_dma = page_pool_get_dma_addr(page) + PRUETH_HEADROOM; 498 475 desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool); 499 476 if (!desc_rx) { 500 477 netdev_err(ndev, "rx push: failed to allocate descriptor\n"); ··· 502 479 } 503 480 desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx); 504 481 505 - buf_dma = dma_map_single(rx_chn->dma_dev, skb->data, pkt_len, DMA_FROM_DEVICE); 506 - if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) { 507 - k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); 508 - netdev_err(ndev, "rx push: failed to map rx pkt buffer\n"); 509 - return -EINVAL; 510 - } 511 - 512 482 cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT, 513 483 PRUETH_NAV_PS_DATA_SIZE); 514 484 k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma); 515 - cppi5_hdesc_attach_buf(desc_rx, buf_dma, skb_tailroom(skb), buf_dma, skb_tailroom(skb)); 485 + cppi5_hdesc_attach_buf(desc_rx, buf_dma, buf_len, buf_dma, buf_len); 516 486 517 487 swdata = cppi5_hdesc_get_swdata(desc_rx); 518 - *swdata = skb; 488 + swdata->type = PRUETH_SWDATA_PAGE; 489 + swdata->data.page = page; 519 490 520 - return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, 0, 491 + return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, PRUETH_RX_FLOW_DATA, 521 492 desc_rx, desc_dma); 522 493 } 523 - EXPORT_SYMBOL_GPL(prueth_dma_rx_push); 494 + EXPORT_SYMBOL_GPL(prueth_dma_rx_push_mapped); 524 495 525 496 u64 icssg_ts_to_ns(u32 hi_sw, u32 hi, u32 lo, u32 cycle_time_ns) 526 497 { ··· 552 535 ssh->hwtstamp = ns_to_ktime(ns); 553 536 } 554 537 555 - static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id) 538 + /** 539 + * emac_xmit_xdp_frame - transmits an XDP frame 540 + * @emac: emac device 541 + * @xdpf: data to transmit 542 + * @page: page from page pool if already DMA mapped 543 + * @q_idx: queue id 544 + * 545 + * Return: XDP state 546 + */ 547 + u32 emac_xmit_xdp_frame(struct prueth_emac *emac, 548 + struct xdp_frame *xdpf, 549 + struct page *page, 550 + unsigned int q_idx) 551 + { 552 + struct cppi5_host_desc_t *first_desc; 553 + struct net_device *ndev = emac->ndev; 554 + struct prueth_tx_chn *tx_chn; 555 + dma_addr_t desc_dma, buf_dma; 556 + struct prueth_swdata *swdata; 557 + u32 *epib; 558 + int ret; 559 + 560 + if (q_idx >= PRUETH_MAX_TX_QUEUES) { 561 + netdev_err(ndev, "xdp tx: invalid q_id %d\n", q_idx); 562 + return ICSSG_XDP_CONSUMED; /* drop */ 563 + } 564 + 565 + tx_chn = &emac->tx_chns[q_idx]; 566 + 567 + first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); 568 + if (!first_desc) { 569 + netdev_dbg(ndev, "xdp tx: failed to allocate descriptor\n"); 570 + goto drop_free_descs; /* drop */ 571 + } 572 + 573 + if (page) { /* already DMA mapped by page_pool */ 574 + buf_dma = page_pool_get_dma_addr(page); 575 + buf_dma += xdpf->headroom + sizeof(struct xdp_frame); 576 + } else { /* Map the linear buffer */ 577 + buf_dma = dma_map_single(tx_chn->dma_dev, xdpf->data, xdpf->len, DMA_TO_DEVICE); 578 + if (dma_mapping_error(tx_chn->dma_dev, buf_dma)) { 579 + netdev_err(ndev, "xdp tx: failed to map data buffer\n"); 580 + goto drop_free_descs; /* drop */ 581 + } 582 + } 583 + 584 + cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT, 585 + PRUETH_NAV_PS_DATA_SIZE); 586 + cppi5_hdesc_set_pkttype(first_desc, 0); 587 + epib = first_desc->epib; 588 + epib[0] = 0; 589 + epib[1] = 0; 590 + 591 + /* set dst tag to indicate internal qid at the firmware which is at 592 + * bit8..bit15. bit0..bit7 indicates port num for directed 593 + * packets in case of switch mode operation 594 + */ 595 + cppi5_desc_set_tags_ids(&first_desc->hdr, 0, (emac->port_id | (q_idx << 8))); 596 + k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); 597 + cppi5_hdesc_attach_buf(first_desc, buf_dma, xdpf->len, buf_dma, xdpf->len); 598 + swdata = cppi5_hdesc_get_swdata(first_desc); 599 + if (page) { 600 + swdata->type = PRUETH_SWDATA_PAGE; 601 + swdata->data.page = page; 602 + } else { 603 + swdata->type = PRUETH_SWDATA_XDPF; 604 + swdata->data.xdpf = xdpf; 605 + } 606 + 607 + cppi5_hdesc_set_pktlen(first_desc, xdpf->len); 608 + desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc); 609 + 610 + ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma); 611 + if (ret) { 612 + netdev_err(ndev, "xdp tx: push failed: %d\n", ret); 613 + goto drop_free_descs; 614 + } 615 + 616 + return ICSSG_XDP_TX; 617 + 618 + drop_free_descs: 619 + prueth_xmit_free(tx_chn, first_desc); 620 + return ICSSG_XDP_CONSUMED; 621 + } 622 + EXPORT_SYMBOL_GPL(emac_xmit_xdp_frame); 623 + 624 + /** 625 + * emac_run_xdp - run an XDP program 626 + * @emac: emac device 627 + * @xdp: XDP buffer containing the frame 628 + * @page: page with RX data if already DMA mapped 629 + * @len: Rx descriptor packet length 630 + * 631 + * Return: XDP state 632 + */ 633 + static u32 emac_run_xdp(struct prueth_emac *emac, struct xdp_buff *xdp, 634 + struct page *page, u32 *len) 635 + { 636 + struct net_device *ndev = emac->ndev; 637 + struct bpf_prog *xdp_prog; 638 + struct xdp_frame *xdpf; 639 + u32 pkt_len = *len; 640 + u32 act, result; 641 + int q_idx, err; 642 + 643 + xdp_prog = READ_ONCE(emac->xdp_prog); 644 + act = bpf_prog_run_xdp(xdp_prog, xdp); 645 + switch (act) { 646 + case XDP_PASS: 647 + return ICSSG_XDP_PASS; 648 + case XDP_TX: 649 + /* Send packet to TX ring for immediate transmission */ 650 + xdpf = xdp_convert_buff_to_frame(xdp); 651 + if (unlikely(!xdpf)) { 652 + ndev->stats.tx_dropped++; 653 + goto drop; 654 + } 655 + 656 + q_idx = smp_processor_id() % emac->tx_ch_num; 657 + result = emac_xmit_xdp_frame(emac, xdpf, page, q_idx); 658 + if (result == ICSSG_XDP_CONSUMED) 659 + goto drop; 660 + 661 + dev_sw_netstats_rx_add(ndev, xdpf->len); 662 + return result; 663 + case XDP_REDIRECT: 664 + err = xdp_do_redirect(emac->ndev, xdp, xdp_prog); 665 + if (err) 666 + goto drop; 667 + 668 + dev_sw_netstats_rx_add(ndev, pkt_len); 669 + return ICSSG_XDP_REDIR; 670 + default: 671 + bpf_warn_invalid_xdp_action(emac->ndev, xdp_prog, act); 672 + fallthrough; 673 + case XDP_ABORTED: 674 + drop: 675 + trace_xdp_exception(emac->ndev, xdp_prog, act); 676 + fallthrough; /* handle aborts by dropping packet */ 677 + case XDP_DROP: 678 + ndev->stats.rx_dropped++; 679 + page_pool_recycle_direct(emac->rx_chns.pg_pool, page); 680 + return ICSSG_XDP_CONSUMED; 681 + } 682 + } 683 + 684 + static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state) 556 685 { 557 686 struct prueth_rx_chn *rx_chn = &emac->rx_chns; 558 687 u32 buf_dma_len, pkt_len, port_id = 0; 559 688 struct net_device *ndev = emac->ndev; 560 689 struct cppi5_host_desc_t *desc_rx; 561 - struct sk_buff *skb, *new_skb; 690 + struct prueth_swdata *swdata; 562 691 dma_addr_t desc_dma, buf_dma; 563 - void **swdata; 692 + struct page *page, *new_page; 693 + struct page_pool *pool; 694 + struct sk_buff *skb; 695 + struct xdp_buff xdp; 564 696 u32 *psdata; 697 + void *pa; 565 698 int ret; 566 699 700 + *xdp_state = 0; 701 + pool = rx_chn->pg_pool; 567 702 ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_id, &desc_dma); 568 703 if (ret) { 569 704 if (ret != -ENODATA) ··· 727 558 return 0; 728 559 729 560 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma); 730 - 731 561 swdata = cppi5_hdesc_get_swdata(desc_rx); 732 - skb = *swdata; 562 + if (swdata->type != PRUETH_SWDATA_PAGE) { 563 + netdev_err(ndev, "rx_pkt: invalid swdata->type %d\n", swdata->type); 564 + k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); 565 + return 0; 566 + } 733 567 734 - psdata = cppi5_hdesc_get_psdata(desc_rx); 735 - /* RX HW timestamp */ 736 - if (emac->rx_ts_enabled) 737 - emac_rx_timestamp(emac, skb, psdata); 738 - 568 + page = swdata->data.page; 569 + page_pool_dma_sync_for_cpu(pool, page, 0, PAGE_SIZE); 739 570 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len); 740 571 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); 741 572 pkt_len = cppi5_hdesc_get_pktlen(desc_rx); ··· 743 574 pkt_len -= 4; 744 575 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL); 745 576 746 - dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE); 747 577 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); 748 578 749 - skb->dev = ndev; 750 - new_skb = netdev_alloc_skb_ip_align(ndev, PRUETH_MAX_PKT_SIZE); 751 579 /* if allocation fails we drop the packet but push the 752 - * descriptor back to the ring with old skb to prevent a stall 580 + * descriptor back to the ring with old page to prevent a stall 753 581 */ 754 - if (!new_skb) { 582 + new_page = page_pool_dev_alloc_pages(pool); 583 + if (unlikely(!new_page)) { 584 + new_page = page; 755 585 ndev->stats.rx_dropped++; 756 - new_skb = skb; 757 - } else { 758 - /* send the filled skb up the n/w stack */ 759 - skb_put(skb, pkt_len); 760 - if (emac->prueth->is_switch_mode) 761 - skb->offload_fwd_mark = emac->offload_fwd_mark; 762 - skb->protocol = eth_type_trans(skb, ndev); 763 - napi_gro_receive(&emac->napi_rx, skb); 764 - ndev->stats.rx_bytes += pkt_len; 765 - ndev->stats.rx_packets++; 586 + goto requeue; 766 587 } 767 588 589 + pa = page_address(page); 590 + if (emac->xdp_prog) { 591 + xdp_init_buff(&xdp, PAGE_SIZE, &rx_chn->xdp_rxq); 592 + xdp_prepare_buff(&xdp, pa, PRUETH_HEADROOM, pkt_len, false); 593 + 594 + *xdp_state = emac_run_xdp(emac, &xdp, page, &pkt_len); 595 + if (*xdp_state == ICSSG_XDP_PASS) 596 + skb = xdp_build_skb_from_buff(&xdp); 597 + else 598 + goto requeue; 599 + } else { 600 + /* prepare skb and send to n/w stack */ 601 + skb = napi_build_skb(pa, PAGE_SIZE); 602 + } 603 + 604 + if (!skb) { 605 + ndev->stats.rx_dropped++; 606 + page_pool_recycle_direct(pool, page); 607 + goto requeue; 608 + } 609 + 610 + skb_reserve(skb, PRUETH_HEADROOM); 611 + skb_put(skb, pkt_len); 612 + skb->dev = ndev; 613 + 614 + psdata = cppi5_hdesc_get_psdata(desc_rx); 615 + /* RX HW timestamp */ 616 + if (emac->rx_ts_enabled) 617 + emac_rx_timestamp(emac, skb, psdata); 618 + 619 + if (emac->prueth->is_switch_mode) 620 + skb->offload_fwd_mark = emac->offload_fwd_mark; 621 + skb->protocol = eth_type_trans(skb, ndev); 622 + 623 + skb_mark_for_recycle(skb); 624 + napi_gro_receive(&emac->napi_rx, skb); 625 + ndev->stats.rx_bytes += pkt_len; 626 + ndev->stats.rx_packets++; 627 + 628 + requeue: 768 629 /* queue another RX DMA */ 769 - ret = prueth_dma_rx_push(emac, new_skb, &emac->rx_chns); 630 + ret = prueth_dma_rx_push_mapped(emac, &emac->rx_chns, new_page, 631 + PRUETH_MAX_PKT_SIZE); 770 632 if (WARN_ON(ret < 0)) { 771 - dev_kfree_skb_any(new_skb); 633 + page_pool_recycle_direct(pool, new_page); 772 634 ndev->stats.rx_errors++; 773 635 ndev->stats.rx_dropped++; 774 636 } ··· 811 611 { 812 612 struct prueth_rx_chn *rx_chn = data; 813 613 struct cppi5_host_desc_t *desc_rx; 814 - struct sk_buff *skb; 815 - dma_addr_t buf_dma; 816 - u32 buf_dma_len; 817 - void **swdata; 614 + struct prueth_swdata *swdata; 615 + struct page_pool *pool; 616 + struct page *page; 818 617 618 + pool = rx_chn->pg_pool; 819 619 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma); 820 620 swdata = cppi5_hdesc_get_swdata(desc_rx); 821 - skb = *swdata; 822 - cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len); 823 - k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); 621 + if (swdata->type == PRUETH_SWDATA_PAGE) { 622 + page = swdata->data.page; 623 + page_pool_recycle_direct(pool, page); 624 + } 824 625 825 - dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, 826 - DMA_FROM_DEVICE); 827 626 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); 828 - 829 - dev_kfree_skb_any(skb); 830 627 } 831 628 832 629 static int prueth_tx_ts_cookie_get(struct prueth_emac *emac) ··· 859 662 struct prueth_emac *emac = netdev_priv(ndev); 860 663 struct prueth *prueth = emac->prueth; 861 664 struct netdev_queue *netif_txq; 665 + struct prueth_swdata *swdata; 862 666 struct prueth_tx_chn *tx_chn; 863 667 dma_addr_t desc_dma, buf_dma; 864 668 u32 pkt_len, dst_tag_id; 865 669 int i, ret = 0, q_idx; 866 670 bool in_tx_ts = 0; 867 671 int tx_ts_cookie; 868 - void **swdata; 869 672 u32 *epib; 870 673 871 674 pkt_len = skb_headlen(skb); ··· 927 730 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); 928 731 cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len); 929 732 swdata = cppi5_hdesc_get_swdata(first_desc); 930 - *swdata = skb; 733 + swdata->type = PRUETH_SWDATA_SKB; 734 + swdata->data.skb = skb; 931 735 932 736 /* Handle the case where skb is fragmented in pages */ 933 737 cur_desc = first_desc; ··· 1031 833 { 1032 834 struct prueth_tx_chn *tx_chn = data; 1033 835 struct cppi5_host_desc_t *desc_tx; 836 + struct prueth_swdata *swdata; 837 + struct xdp_frame *xdpf; 1034 838 struct sk_buff *skb; 1035 - void **swdata; 1036 839 1037 840 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma); 1038 841 swdata = cppi5_hdesc_get_swdata(desc_tx); 1039 - skb = *(swdata); 1040 - prueth_xmit_free(tx_chn, desc_tx); 1041 842 1042 - dev_kfree_skb_any(skb); 843 + switch (swdata->type) { 844 + case PRUETH_SWDATA_SKB: 845 + skb = swdata->data.skb; 846 + dev_kfree_skb_any(skb); 847 + break; 848 + case PRUETH_SWDATA_XDPF: 849 + xdpf = swdata->data.xdpf; 850 + xdp_return_frame(xdpf); 851 + break; 852 + default: 853 + break; 854 + } 855 + 856 + prueth_xmit_free(tx_chn, desc_tx); 1043 857 } 1044 858 1045 859 irqreturn_t prueth_rx_irq(int irq, void *dev_id) ··· 1085 875 PRUETH_RX_FLOW_DATA_SR1 : PRUETH_RX_FLOW_DATA; 1086 876 int flow = emac->is_sr1 ? 1087 877 PRUETH_MAX_RX_FLOWS_SR1 : PRUETH_MAX_RX_FLOWS; 878 + int xdp_state_or = 0; 1088 879 int num_rx = 0; 1089 880 int cur_budget; 881 + u32 xdp_state; 1090 882 int ret; 1091 883 1092 884 while (flow--) { 1093 885 cur_budget = budget - num_rx; 1094 886 1095 887 while (cur_budget--) { 1096 - ret = emac_rx_packet(emac, flow); 888 + ret = emac_rx_packet(emac, flow, &xdp_state); 889 + xdp_state_or |= xdp_state; 1097 890 if (ret) 1098 891 break; 1099 892 num_rx++; ··· 1105 892 if (num_rx >= budget) 1106 893 break; 1107 894 } 895 + 896 + if (xdp_state_or & ICSSG_XDP_REDIR) 897 + xdp_do_flush(); 1108 898 1109 899 if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) { 1110 900 if (unlikely(emac->rx_pace_timeout_ns)) { ··· 1123 907 } 1124 908 EXPORT_SYMBOL_GPL(icssg_napi_rx_poll); 1125 909 910 + static struct page_pool *prueth_create_page_pool(struct prueth_emac *emac, 911 + struct device *dma_dev, 912 + int size) 913 + { 914 + struct page_pool_params pp_params = { 0 }; 915 + struct page_pool *pool; 916 + 917 + pp_params.order = 0; 918 + pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 919 + pp_params.pool_size = size; 920 + pp_params.nid = dev_to_node(emac->prueth->dev); 921 + pp_params.dma_dir = DMA_BIDIRECTIONAL; 922 + pp_params.dev = dma_dev; 923 + pp_params.napi = &emac->napi_rx; 924 + pp_params.max_len = PAGE_SIZE; 925 + 926 + pool = page_pool_create(&pp_params); 927 + if (IS_ERR(pool)) 928 + netdev_err(emac->ndev, "cannot create rx page pool\n"); 929 + 930 + return pool; 931 + } 932 + 1126 933 int prueth_prepare_rx_chan(struct prueth_emac *emac, 1127 934 struct prueth_rx_chn *chn, 1128 935 int buf_size) 1129 936 { 1130 - struct sk_buff *skb; 937 + struct page_pool *pool; 938 + struct page *page; 1131 939 int i, ret; 1132 940 1133 - for (i = 0; i < chn->descs_num; i++) { 1134 - skb = __netdev_alloc_skb_ip_align(NULL, buf_size, GFP_KERNEL); 1135 - if (!skb) 1136 - return -ENOMEM; 941 + pool = prueth_create_page_pool(emac, chn->dma_dev, chn->descs_num); 942 + if (IS_ERR(pool)) 943 + return PTR_ERR(pool); 1137 944 1138 - ret = prueth_dma_rx_push(emac, skb, chn); 945 + chn->pg_pool = pool; 946 + 947 + for (i = 0; i < chn->descs_num; i++) { 948 + /* NOTE: we're not using memory efficiently here. 949 + * 1 full page (4KB?) used here instead of 950 + * PRUETH_MAX_PKT_SIZE (~1.5KB?) 951 + */ 952 + page = page_pool_dev_alloc_pages(pool); 953 + if (!page) { 954 + netdev_err(emac->ndev, "couldn't allocate rx page\n"); 955 + ret = -ENOMEM; 956 + goto recycle_alloc_pg; 957 + } 958 + 959 + ret = prueth_dma_rx_push_mapped(emac, chn, page, buf_size); 1139 960 if (ret < 0) { 1140 961 netdev_err(emac->ndev, 1141 - "cannot submit skb for rx chan %s ret %d\n", 962 + "cannot submit page for rx chan %s ret %d\n", 1142 963 chn->name, ret); 1143 - kfree_skb(skb); 1144 - return ret; 964 + page_pool_recycle_direct(pool, page); 965 + goto recycle_alloc_pg; 1145 966 } 1146 967 } 1147 968 1148 969 return 0; 970 + 971 + recycle_alloc_pg: 972 + prueth_reset_rx_chan(&emac->rx_chns, PRUETH_MAX_RX_FLOWS, false); 973 + 974 + return ret; 1149 975 } 1150 976 EXPORT_SYMBOL_GPL(prueth_prepare_rx_chan); 1151 977 ··· 1216 958 prueth_rx_cleanup, !!i); 1217 959 if (disable) 1218 960 k3_udma_glue_disable_rx_chn(chn->rx_chn); 961 + 962 + page_pool_destroy(chn->pg_pool); 963 + chn->pg_pool = NULL; 1219 964 } 1220 965 EXPORT_SYMBOL_GPL(prueth_reset_rx_chan); 1221 966

+129 -2

drivers/net/ethernet/ti/icssg/icssg_prueth.c

··· 559 559 .perout_enable = prueth_perout_enable, 560 560 }; 561 561 562 + static int prueth_create_xdp_rxqs(struct prueth_emac *emac) 563 + { 564 + struct xdp_rxq_info *rxq = &emac->rx_chns.xdp_rxq; 565 + struct page_pool *pool = emac->rx_chns.pg_pool; 566 + int ret; 567 + 568 + ret = xdp_rxq_info_reg(rxq, emac->ndev, 0, emac->napi_rx.napi_id); 569 + if (ret) 570 + return ret; 571 + 572 + ret = xdp_rxq_info_reg_mem_model(rxq, MEM_TYPE_PAGE_POOL, pool); 573 + if (ret) 574 + xdp_rxq_info_unreg(rxq); 575 + 576 + return ret; 577 + } 578 + 579 + static void prueth_destroy_xdp_rxqs(struct prueth_emac *emac) 580 + { 581 + struct xdp_rxq_info *rxq = &emac->rx_chns.xdp_rxq; 582 + 583 + if (!xdp_rxq_info_is_reg(rxq)) 584 + return; 585 + 586 + xdp_rxq_info_unreg(rxq); 587 + } 588 + 562 589 static int icssg_prueth_add_mcast(struct net_device *ndev, const u8 *addr) 563 590 { 564 591 struct net_device *real_dev; ··· 807 780 if (ret) 808 781 goto free_tx_ts_irq; 809 782 810 - ret = k3_udma_glue_enable_rx_chn(emac->rx_chns.rx_chn); 783 + ret = prueth_create_xdp_rxqs(emac); 811 784 if (ret) 812 785 goto reset_rx_chn; 786 + 787 + ret = k3_udma_glue_enable_rx_chn(emac->rx_chns.rx_chn); 788 + if (ret) 789 + goto destroy_xdp_rxqs; 813 790 814 791 for (i = 0; i < emac->tx_ch_num; i++) { 815 792 ret = k3_udma_glue_enable_tx_chn(emac->tx_chns[i].tx_chn); ··· 840 809 * any SKB for completion. So set false to free_skb 841 810 */ 842 811 prueth_reset_tx_chan(emac, i, false); 812 + destroy_xdp_rxqs: 813 + prueth_destroy_xdp_rxqs(emac); 843 814 reset_rx_chn: 844 815 prueth_reset_rx_chan(&emac->rx_chns, max_rx_flows, false); 845 816 free_tx_ts_irq: ··· 912 879 k3_udma_glue_tdown_rx_chn(emac->rx_chns.rx_chn, true); 913 880 914 881 prueth_reset_rx_chan(&emac->rx_chns, max_rx_flows, true); 915 - 882 + prueth_destroy_xdp_rxqs(emac); 916 883 napi_disable(&emac->napi_rx); 917 884 hrtimer_cancel(&emac->rx_hrtimer); 918 885 ··· 1057 1024 return 0; 1058 1025 } 1059 1026 1027 + /** 1028 + * emac_xdp_xmit - Implements ndo_xdp_xmit 1029 + * @dev: netdev 1030 + * @n: number of frames 1031 + * @frames: array of XDP buffer pointers 1032 + * @flags: XDP extra info 1033 + * 1034 + * Return: number of frames successfully sent. Failed frames 1035 + * will be free'ed by XDP core. 1036 + * 1037 + * For error cases, a negative errno code is returned and no-frames 1038 + * are transmitted (caller must handle freeing frames). 1039 + **/ 1040 + static int emac_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 1041 + u32 flags) 1042 + { 1043 + struct prueth_emac *emac = netdev_priv(dev); 1044 + struct net_device *ndev = emac->ndev; 1045 + struct xdp_frame *xdpf; 1046 + unsigned int q_idx; 1047 + int nxmit = 0; 1048 + u32 err; 1049 + int i; 1050 + 1051 + q_idx = smp_processor_id() % emac->tx_ch_num; 1052 + 1053 + if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1054 + return -EINVAL; 1055 + 1056 + for (i = 0; i < n; i++) { 1057 + xdpf = frames[i]; 1058 + err = emac_xmit_xdp_frame(emac, xdpf, NULL, q_idx); 1059 + if (err != ICSSG_XDP_TX) { 1060 + ndev->stats.tx_dropped++; 1061 + break; 1062 + } 1063 + nxmit++; 1064 + } 1065 + 1066 + return nxmit; 1067 + } 1068 + 1069 + /** 1070 + * emac_xdp_setup - add/remove an XDP program 1071 + * @emac: emac device 1072 + * @bpf: XDP program 1073 + * 1074 + * Return: Always 0 (Success) 1075 + **/ 1076 + static int emac_xdp_setup(struct prueth_emac *emac, struct netdev_bpf *bpf) 1077 + { 1078 + struct bpf_prog *prog = bpf->prog; 1079 + xdp_features_t val; 1080 + 1081 + val = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 1082 + NETDEV_XDP_ACT_NDO_XMIT; 1083 + xdp_set_features_flag(emac->ndev, val); 1084 + 1085 + if (!emac->xdpi.prog && !prog) 1086 + return 0; 1087 + 1088 + WRITE_ONCE(emac->xdp_prog, prog); 1089 + 1090 + xdp_attachment_setup(&emac->xdpi, bpf); 1091 + 1092 + return 0; 1093 + } 1094 + 1095 + /** 1096 + * emac_ndo_bpf - implements ndo_bpf for icssg_prueth 1097 + * @ndev: network adapter device 1098 + * @bpf: XDP program 1099 + * 1100 + * Return: 0 on success, error code on failure. 1101 + **/ 1102 + static int emac_ndo_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 1103 + { 1104 + struct prueth_emac *emac = netdev_priv(ndev); 1105 + 1106 + switch (bpf->command) { 1107 + case XDP_SETUP_PROG: 1108 + return emac_xdp_setup(emac, bpf); 1109 + default: 1110 + return -EINVAL; 1111 + } 1112 + } 1113 + 1060 1114 static const struct net_device_ops emac_netdev_ops = { 1061 1115 .ndo_open = emac_ndo_open, 1062 1116 .ndo_stop = emac_ndo_stop, ··· 1158 1038 .ndo_fix_features = emac_ndo_fix_features, 1159 1039 .ndo_vlan_rx_add_vid = emac_ndo_vlan_rx_add_vid, 1160 1040 .ndo_vlan_rx_kill_vid = emac_ndo_vlan_rx_del_vid, 1041 + .ndo_bpf = emac_ndo_bpf, 1042 + .ndo_xdp_xmit = emac_xdp_xmit, 1161 1043 }; 1162 1044 1163 1045 static int prueth_netdev_init(struct prueth *prueth, ··· 1188 1066 emac->prueth = prueth; 1189 1067 emac->ndev = ndev; 1190 1068 emac->port_id = port; 1069 + emac->xdp_prog = NULL; 1070 + emac->ndev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 1191 1071 emac->cmd_wq = create_singlethread_workqueue("icssg_cmd_wq"); 1192 1072 if (!emac->cmd_wq) { 1193 1073 ret = -ENOMEM; ··· 1645 1521 int i, ret; 1646 1522 1647 1523 np = dev->of_node; 1524 + 1525 + BUILD_BUG_ON_MSG((sizeof(struct prueth_swdata) > PRUETH_NAV_SW_DATA_SIZE), 1526 + "insufficient SW_DATA size"); 1648 1527 1649 1528 prueth = devm_kzalloc(dev, sizeof(*prueth), GFP_KERNEL); 1650 1529 if (!prueth)

+44 -3

drivers/net/ethernet/ti/icssg/icssg_prueth.h

··· 8 8 #ifndef __NET_TI_ICSSG_PRUETH_H 9 9 #define __NET_TI_ICSSG_PRUETH_H 10 10 11 + #include <linux/bpf.h> 12 + #include <linux/bpf_trace.h> 11 13 #include <linux/etherdevice.h> 12 14 #include <linux/genalloc.h> 13 15 #include <linux/if_vlan.h> ··· 35 33 #include <linux/dma/k3-udma-glue.h> 36 34 37 35 #include <net/devlink.h> 36 + #include <net/xdp.h> 37 + #include <net/page_pool/helpers.h> 38 38 39 39 #include "icssg_config.h" 40 40 #include "icss_iep.h" ··· 135 131 u32 descs_num; 136 132 unsigned int irq[ICSSG_MAX_RFLOWS]; /* separate irq per flow */ 137 133 char name[32]; 134 + struct page_pool *pg_pool; 135 + struct xdp_rxq_info xdp_rxq; 136 + }; 137 + 138 + enum prueth_swdata_type { 139 + PRUETH_SWDATA_INVALID = 0, 140 + PRUETH_SWDATA_SKB, 141 + PRUETH_SWDATA_PAGE, 142 + PRUETH_SWDATA_CMD, 143 + PRUETH_SWDATA_XDPF, 144 + }; 145 + 146 + struct prueth_swdata { 147 + enum prueth_swdata_type type; 148 + union prueth_data { 149 + struct sk_buff *skb; 150 + struct page *page; 151 + u32 cmd; 152 + struct xdp_frame *xdpf; 153 + } data; 138 154 }; 139 155 140 156 /* There are 4 Tx DMA channels, but the highest priority is CH3 (thread 3) ··· 163 139 #define PRUETH_MAX_TX_QUEUES 4 164 140 165 141 #define PRUETH_MAX_TX_TS_REQUESTS 50 /* Max simultaneous TX_TS requests */ 142 + 143 + /* XDP BPF state */ 144 + #define ICSSG_XDP_PASS 0 145 + #define ICSSG_XDP_CONSUMED BIT(0) 146 + #define ICSSG_XDP_TX BIT(1) 147 + #define ICSSG_XDP_REDIR BIT(2) 166 148 167 149 /* Minimum coalesce time in usecs for both Tx and Rx */ 168 150 #define ICSSG_MIN_COALESCE_USECS 20 ··· 238 208 unsigned long rx_pace_timeout_ns; 239 209 240 210 struct netdev_hw_addr_list vlan_mcast_list[MAX_VLAN_ID]; 211 + struct bpf_prog *xdp_prog; 212 + struct xdp_attachment_info xdpi; 241 213 }; 214 + 215 + /* The buf includes headroom compatible with both skb and xdpf */ 216 + #define PRUETH_HEADROOM_NA (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + NET_IP_ALIGN) 217 + #define PRUETH_HEADROOM ALIGN(PRUETH_HEADROOM_NA, sizeof(long)) 242 218 243 219 /** 244 220 * struct prueth_pdata - PRUeth platform data ··· 446 410 struct prueth_rx_chn *rx_chn, 447 411 char *name, u32 max_rflows, 448 412 u32 max_desc_num); 449 - int prueth_dma_rx_push(struct prueth_emac *emac, 450 - struct sk_buff *skb, 451 - struct prueth_rx_chn *rx_chn); 413 + int prueth_dma_rx_push_mapped(struct prueth_emac *emac, 414 + struct prueth_rx_chn *rx_chn, 415 + struct page *page, u32 buf_len); 416 + unsigned int prueth_rxbuf_total_len(unsigned int len); 452 417 void emac_rx_timestamp(struct prueth_emac *emac, 453 418 struct sk_buff *skb, u32 *psdata); 454 419 enum netdev_tx icssg_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev); ··· 478 441 479 442 /* Revision specific helper */ 480 443 u64 icssg_ts_to_ns(u32 hi_sw, u32 hi, u32 lo, u32 cycle_time_ns); 444 + u32 emac_xmit_xdp_frame(struct prueth_emac *emac, 445 + struct xdp_frame *xdpf, 446 + struct page *page, 447 + unsigned int q_idx); 481 448 482 449 #endif /* __NET_TI_ICSSG_PRUETH_H */

+28 -30

drivers/net/ethernet/ti/icssg/icssg_prueth_sr1.c

··· 84 84 __le32 *data = emac->cmd_data; 85 85 dma_addr_t desc_dma, buf_dma; 86 86 struct prueth_tx_chn *tx_chn; 87 - void **swdata; 87 + struct prueth_swdata *swdata; 88 88 int ret = 0; 89 89 u32 *epib; 90 90 ··· 122 122 123 123 cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len); 124 124 swdata = cppi5_hdesc_get_swdata(first_desc); 125 - *swdata = data; 125 + swdata->type = PRUETH_SWDATA_CMD; 126 + swdata->data.cmd = le32_to_cpu(data[0]); 126 127 127 128 cppi5_hdesc_set_pktlen(first_desc, pkt_len); 128 129 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc); ··· 269 268 * Returns skb pointer if packet found else NULL 270 269 * Caller must free the returned skb. 271 270 */ 272 - static struct sk_buff *prueth_process_rx_mgm(struct prueth_emac *emac, 273 - u32 flow_id) 271 + static struct page *prueth_process_rx_mgm(struct prueth_emac *emac, 272 + u32 flow_id) 274 273 { 275 274 struct prueth_rx_chn *rx_chn = &emac->rx_mgm_chn; 276 275 struct net_device *ndev = emac->ndev; 277 276 struct cppi5_host_desc_t *desc_rx; 278 - struct sk_buff *skb, *new_skb; 277 + struct page *page, *new_page; 278 + struct prueth_swdata *swdata; 279 279 dma_addr_t desc_dma, buf_dma; 280 - u32 buf_dma_len, pkt_len; 281 - void **swdata; 280 + u32 buf_dma_len; 282 281 int ret; 283 282 284 283 ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_id, &desc_dma); ··· 300 299 } 301 300 302 301 swdata = cppi5_hdesc_get_swdata(desc_rx); 303 - skb = *swdata; 302 + page = swdata->data.page; 304 303 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len); 305 - pkt_len = cppi5_hdesc_get_pktlen(desc_rx); 306 304 307 305 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE); 308 306 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); 309 307 310 - new_skb = netdev_alloc_skb_ip_align(ndev, PRUETH_MAX_PKT_SIZE); 308 + new_page = page_pool_dev_alloc_pages(rx_chn->pg_pool); 311 309 /* if allocation fails we drop the packet but push the 312 310 * descriptor back to the ring with old skb to prevent a stall 313 311 */ 314 - if (!new_skb) { 312 + if (!new_page) { 315 313 netdev_err(ndev, 316 - "skb alloc failed, dropped mgm pkt from flow %d\n", 314 + "page alloc failed, dropped mgm pkt from flow %d\n", 317 315 flow_id); 318 - new_skb = skb; 319 - skb = NULL; /* return NULL */ 320 - } else { 321 - /* return the filled skb */ 322 - skb_put(skb, pkt_len); 316 + new_page = page; 317 + page = NULL; /* return NULL */ 323 318 } 324 319 325 320 /* queue another DMA */ 326 - ret = prueth_dma_rx_push(emac, new_skb, &emac->rx_mgm_chn); 321 + ret = prueth_dma_rx_push_mapped(emac, &emac->rx_chns, new_page, 322 + PRUETH_MAX_PKT_SIZE); 327 323 if (WARN_ON(ret < 0)) 328 - dev_kfree_skb_any(new_skb); 324 + page_pool_recycle_direct(rx_chn->pg_pool, new_page); 329 325 330 - return skb; 326 + return page; 331 327 } 332 328 333 329 static void prueth_tx_ts_sr1(struct prueth_emac *emac, ··· 360 362 static irqreturn_t prueth_rx_mgm_ts_thread_sr1(int irq, void *dev_id) 361 363 { 362 364 struct prueth_emac *emac = dev_id; 363 - struct sk_buff *skb; 365 + struct page *page; 364 366 365 - skb = prueth_process_rx_mgm(emac, PRUETH_RX_MGM_FLOW_TIMESTAMP_SR1); 366 - if (!skb) 367 + page = prueth_process_rx_mgm(emac, PRUETH_RX_MGM_FLOW_TIMESTAMP_SR1); 368 + if (!page) 367 369 return IRQ_NONE; 368 370 369 - prueth_tx_ts_sr1(emac, (void *)skb->data); 370 - dev_kfree_skb_any(skb); 371 + prueth_tx_ts_sr1(emac, (void *)page_address(page)); 372 + page_pool_recycle_direct(page->pp, page); 371 373 372 374 return IRQ_HANDLED; 373 375 } ··· 375 377 static irqreturn_t prueth_rx_mgm_rsp_thread(int irq, void *dev_id) 376 378 { 377 379 struct prueth_emac *emac = dev_id; 378 - struct sk_buff *skb; 380 + struct page *page; 379 381 u32 rsp; 380 382 381 - skb = prueth_process_rx_mgm(emac, PRUETH_RX_MGM_FLOW_RESPONSE_SR1); 382 - if (!skb) 383 + page = prueth_process_rx_mgm(emac, PRUETH_RX_MGM_FLOW_RESPONSE_SR1); 384 + if (!page) 383 385 return IRQ_NONE; 384 386 385 387 /* Process command response */ 386 - rsp = le32_to_cpu(*(__le32 *)skb->data) & 0xffff0000; 388 + rsp = le32_to_cpu(*(__le32 *)page_address(page)) & 0xffff0000; 387 389 if (rsp == ICSSG_SHUTDOWN_CMD_SR1) { 388 390 netdev_dbg(emac->ndev, "f/w Shutdown cmd resp %x\n", rsp); 389 391 complete(&emac->cmd_complete); ··· 392 394 complete(&emac->cmd_complete); 393 395 } 394 396 395 - dev_kfree_skb_any(skb); 397 + page_pool_recycle_direct(page->pp, page); 396 398 397 399 return IRQ_HANDLED; 398 400 }

Configure Feed

Configure Feed