Merge branch 'eth-fbnic-support-queue-api-and-zero-copy-rx'

+1 -8

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

··· 780 780 bitmap_free(rq->mpwqe.shampo->bitmap); 781 781 } 782 782 783 - static bool mlx5_rq_needs_separate_hd_pool(struct mlx5e_rq *rq) 784 - { 785 - struct netdev_rx_queue *rxq = __netif_get_rx_queue(rq->netdev, rq->ix); 786 - 787 - return !!rxq->mp_params.mp_ops; 788 - } 789 - 790 783 static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, 791 784 struct mlx5e_params *params, 792 785 struct mlx5e_rq_param *rqp, ··· 818 825 hd_pool_size = (rq->mpwqe.shampo->hd_per_wqe * wq_size) / 819 826 MLX5E_SHAMPO_WQ_HEADER_PER_PAGE; 820 827 821 - if (mlx5_rq_needs_separate_hd_pool(rq)) { 828 + if (netif_rxq_has_unreadable_mp(rq->netdev, rq->ix)) { 822 829 /* Separate page pool for shampo headers */ 823 830 struct page_pool_params pp_params = { }; 824 831

+1

drivers/net/ethernet/meta/fbnic/fbnic_netdev.c

··· 711 711 712 712 netdev->netdev_ops = &fbnic_netdev_ops; 713 713 netdev->stat_ops = &fbnic_stat_ops; 714 + netdev->queue_mgmt_ops = &fbnic_queue_mgmt_ops; 714 715 715 716 fbnic_set_ethtool_ops(netdev); 716 717

+8 -1

drivers/net/ethernet/meta/fbnic/fbnic_pci.c

··· 206 206 207 207 fbnic_bmc_rpc_check(fbd); 208 208 209 - if (netif_carrier_ok(fbd->netdev)) 209 + if (netif_carrier_ok(fbd->netdev)) { 210 + netdev_lock(fbd->netdev); 210 211 fbnic_napi_depletion_check(fbd->netdev); 212 + netdev_unlock(fbd->netdev); 213 + } 211 214 212 215 if (netif_running(fbd->netdev)) 213 216 schedule_delayed_work(&fbd->service_task, HZ); ··· 394 391 goto null_uc_addr; 395 392 396 393 rtnl_lock(); 394 + netdev_lock(netdev); 397 395 398 396 netif_device_detach(netdev); 399 397 400 398 if (netif_running(netdev)) 401 399 netdev->netdev_ops->ndo_stop(netdev); 402 400 401 + netdev_unlock(netdev); 403 402 rtnl_unlock(); 404 403 405 404 null_uc_addr: ··· 466 461 fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues); 467 462 468 463 rtnl_lock(); 464 + netdev_lock(netdev); 469 465 470 466 if (netif_running(netdev)) 471 467 err = __fbnic_open(fbn); 472 468 469 + netdev_unlock(netdev); 473 470 rtnl_unlock(); 474 471 if (err) 475 472 goto err_free_mbx;

+408 -213

drivers/net/ethernet/meta/fbnic/fbnic_txrx.c

··· 640 640 FBNIC_TWD_TYPE_AL; 641 641 total_bytes += FIELD_GET(FBNIC_TWD_LEN_MASK, twd); 642 642 643 - page_pool_put_page(nv->page_pool, page, -1, pp_allow_direct); 643 + page_pool_put_page(page->pp, page, -1, pp_allow_direct); 644 644 next_desc: 645 645 head++; 646 646 head &= ring->size_mask; ··· 715 715 } 716 716 717 717 static void fbnic_page_pool_init(struct fbnic_ring *ring, unsigned int idx, 718 - struct page *page) 718 + netmem_ref netmem) 719 719 { 720 720 struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; 721 721 722 - page_pool_fragment_page(page, FBNIC_PAGECNT_BIAS_MAX); 722 + page_pool_fragment_netmem(netmem, FBNIC_PAGECNT_BIAS_MAX); 723 723 rx_buf->pagecnt_bias = FBNIC_PAGECNT_BIAS_MAX; 724 - rx_buf->page = page; 724 + rx_buf->netmem = netmem; 725 725 } 726 726 727 - static struct page *fbnic_page_pool_get(struct fbnic_ring *ring, 728 - unsigned int idx) 727 + static struct page * 728 + fbnic_page_pool_get_head(struct fbnic_q_triad *qt, unsigned int idx) 729 729 { 730 - struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; 730 + struct fbnic_rx_buf *rx_buf = &qt->sub0.rx_buf[idx]; 731 731 732 732 rx_buf->pagecnt_bias--; 733 733 734 - return rx_buf->page; 734 + /* sub0 is always fed system pages, from the NAPI-level page_pool */ 735 + return netmem_to_page(rx_buf->netmem); 736 + } 737 + 738 + static netmem_ref 739 + fbnic_page_pool_get_data(struct fbnic_q_triad *qt, unsigned int idx) 740 + { 741 + struct fbnic_rx_buf *rx_buf = &qt->sub1.rx_buf[idx]; 742 + 743 + rx_buf->pagecnt_bias--; 744 + 745 + return rx_buf->netmem; 735 746 } 736 747 737 748 static void fbnic_page_pool_drain(struct fbnic_ring *ring, unsigned int idx, 738 - struct fbnic_napi_vector *nv, int budget) 749 + int budget) 739 750 { 740 751 struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; 741 - struct page *page = rx_buf->page; 752 + netmem_ref netmem = rx_buf->netmem; 742 753 743 - if (!page_pool_unref_page(page, rx_buf->pagecnt_bias)) 744 - page_pool_put_unrefed_page(nv->page_pool, page, -1, !!budget); 754 + if (!page_pool_unref_netmem(netmem, rx_buf->pagecnt_bias)) 755 + page_pool_put_unrefed_netmem(ring->page_pool, netmem, -1, 756 + !!budget); 745 757 746 - rx_buf->page = NULL; 758 + rx_buf->netmem = 0; 747 759 } 748 760 749 761 static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget, ··· 838 826 fbnic_clean_twq(nv, napi_budget, qt, ts_head, head0, head1); 839 827 } 840 828 841 - static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget, 842 - struct fbnic_ring *ring, unsigned int hw_head) 829 + static void fbnic_clean_bdq(struct fbnic_ring *ring, unsigned int hw_head, 830 + int napi_budget) 843 831 { 844 832 unsigned int head = ring->head; 845 833 ··· 847 835 return; 848 836 849 837 do { 850 - fbnic_page_pool_drain(ring, head, nv, napi_budget); 838 + fbnic_page_pool_drain(ring, head, napi_budget); 851 839 852 840 head++; 853 841 head &= ring->size_mask; ··· 856 844 ring->head = head; 857 845 } 858 846 859 - static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, struct page *page) 847 + static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, netmem_ref netmem) 860 848 { 861 849 __le64 *bdq_desc = &bdq->desc[id * FBNIC_BD_FRAG_COUNT]; 862 - dma_addr_t dma = page_pool_get_dma_addr(page); 850 + dma_addr_t dma = page_pool_get_dma_addr_netmem(netmem); 863 851 u64 bd, i = FBNIC_BD_FRAG_COUNT; 864 852 865 853 bd = (FBNIC_BD_PAGE_ADDR_MASK & dma) | ··· 877 865 } while (--i); 878 866 } 879 867 880 - static void fbnic_fill_bdq(struct fbnic_napi_vector *nv, struct fbnic_ring *bdq) 868 + static void fbnic_fill_bdq(struct fbnic_ring *bdq) 881 869 { 882 870 unsigned int count = fbnic_desc_unused(bdq); 883 871 unsigned int i = bdq->tail; ··· 886 874 return; 887 875 888 876 do { 889 - struct page *page; 877 + netmem_ref netmem; 890 878 891 - page = page_pool_dev_alloc_pages(nv->page_pool); 892 - if (!page) { 879 + netmem = page_pool_dev_alloc_netmems(bdq->page_pool); 880 + if (!netmem) { 893 881 u64_stats_update_begin(&bdq->stats.syncp); 894 882 bdq->stats.rx.alloc_failed++; 895 883 u64_stats_update_end(&bdq->stats.syncp); ··· 897 885 break; 898 886 } 899 887 900 - fbnic_page_pool_init(bdq, i, page); 901 - fbnic_bd_prep(bdq, i, page); 888 + fbnic_page_pool_init(bdq, i, netmem); 889 + fbnic_bd_prep(bdq, i, netmem); 902 890 903 891 i++; 904 892 i &= bdq->size_mask; ··· 945 933 { 946 934 unsigned int hdr_pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); 947 935 unsigned int hdr_pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); 948 - struct page *page = fbnic_page_pool_get(&qt->sub0, hdr_pg_idx); 936 + struct page *page = fbnic_page_pool_get_head(qt, hdr_pg_idx); 949 937 unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); 950 938 unsigned int frame_sz, hdr_pg_start, hdr_pg_end, headroom; 951 939 unsigned char *hdr_start; ··· 986 974 unsigned int pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); 987 975 unsigned int pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); 988 976 unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); 989 - struct page *page = fbnic_page_pool_get(&qt->sub1, pg_idx); 977 + netmem_ref netmem = fbnic_page_pool_get_data(qt, pg_idx); 990 978 unsigned int truesize; 991 979 bool added; 992 980 ··· 997 985 FBNIC_BD_FRAG_SIZE; 998 986 999 987 /* Sync DMA buffer */ 1000 - dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page), 1001 - pg_off, truesize, DMA_BIDIRECTIONAL); 988 + page_pool_dma_sync_netmem_for_cpu(qt->sub1.page_pool, netmem, 989 + pg_off, truesize); 1002 990 1003 - added = xdp_buff_add_frag(&pkt->buff, page_to_netmem(page), pg_off, len, 1004 - truesize); 991 + added = xdp_buff_add_frag(&pkt->buff, netmem, pg_off, len, truesize); 1005 992 if (unlikely(!added)) { 1006 993 pkt->add_frag_failed = true; 1007 994 netdev_err_once(nv->napi.dev, ··· 1008 997 } 1009 998 } 1010 999 1011 - static void fbnic_put_pkt_buff(struct fbnic_napi_vector *nv, 1000 + static void fbnic_put_pkt_buff(struct fbnic_q_triad *qt, 1012 1001 struct fbnic_pkt_buff *pkt, int budget) 1013 1002 { 1014 1003 struct page *page; ··· 1018 1007 1019 1008 if (xdp_buff_has_frags(&pkt->buff)) { 1020 1009 struct skb_shared_info *shinfo; 1010 + netmem_ref netmem; 1021 1011 int nr_frags; 1022 1012 1023 1013 shinfo = xdp_get_shared_info_from_buff(&pkt->buff); 1024 1014 nr_frags = shinfo->nr_frags; 1025 1015 1026 1016 while (nr_frags--) { 1027 - page = skb_frag_page(&shinfo->frags[nr_frags]); 1028 - page_pool_put_full_page(nv->page_pool, page, !!budget); 1017 + netmem = skb_frag_netmem(&shinfo->frags[nr_frags]); 1018 + page_pool_put_full_netmem(qt->sub1.page_pool, netmem, 1019 + !!budget); 1029 1020 } 1030 1021 } 1031 1022 1032 1023 page = virt_to_page(pkt->buff.data_hard_start); 1033 - page_pool_put_full_page(nv->page_pool, page, !!budget); 1024 + page_pool_put_full_page(qt->sub0.page_pool, page, !!budget); 1034 1025 } 1035 1026 1036 1027 static struct sk_buff *fbnic_build_skb(struct fbnic_napi_vector *nv, ··· 1287 1274 dropped++; 1288 1275 } 1289 1276 1290 - fbnic_put_pkt_buff(nv, pkt, 1); 1277 + fbnic_put_pkt_buff(qt, pkt, 1); 1291 1278 } 1292 1279 1293 1280 pkt->buff.data_hard_start = NULL; ··· 1320 1307 1321 1308 /* Unmap and free processed buffers */ 1322 1309 if (head0 >= 0) 1323 - fbnic_clean_bdq(nv, budget, &qt->sub0, head0); 1324 - fbnic_fill_bdq(nv, &qt->sub0); 1310 + fbnic_clean_bdq(&qt->sub0, head0, budget); 1311 + fbnic_fill_bdq(&qt->sub0); 1325 1312 1326 1313 if (head1 >= 0) 1327 - fbnic_clean_bdq(nv, budget, &qt->sub1, head1); 1328 - fbnic_fill_bdq(nv, &qt->sub1); 1314 + fbnic_clean_bdq(&qt->sub1, head1, budget); 1315 + fbnic_fill_bdq(&qt->sub1); 1329 1316 1330 1317 /* Record the current head/tail of the queue */ 1331 1318 if (rcq->head != head) { ··· 1475 1462 fbn->rx[rxr->q_idx] = NULL; 1476 1463 } 1477 1464 1465 + static void fbnic_free_qt_page_pools(struct fbnic_q_triad *qt) 1466 + { 1467 + page_pool_destroy(qt->sub0.page_pool); 1468 + page_pool_destroy(qt->sub1.page_pool); 1469 + } 1470 + 1478 1471 static void fbnic_free_napi_vector(struct fbnic_net *fbn, 1479 1472 struct fbnic_napi_vector *nv) 1480 1473 { ··· 1494 1475 } 1495 1476 1496 1477 for (j = 0; j < nv->rxt_count; j++, i++) { 1497 - xdp_rxq_info_unreg(&nv->qt[i].xdp_rxq); 1498 1478 fbnic_remove_rx_ring(fbn, &nv->qt[i].sub0); 1499 1479 fbnic_remove_rx_ring(fbn, &nv->qt[i].sub1); 1500 1480 fbnic_remove_rx_ring(fbn, &nv->qt[i].cmpl); 1501 1481 } 1502 1482 1503 1483 fbnic_napi_free_irq(fbd, nv); 1504 - page_pool_destroy(nv->page_pool); 1505 - netif_napi_del(&nv->napi); 1484 + netif_napi_del_locked(&nv->napi); 1506 1485 fbn->napi[fbnic_napi_idx(nv)] = NULL; 1507 1486 kfree(nv); 1508 1487 } ··· 1514 1497 fbnic_free_napi_vector(fbn, fbn->napi[i]); 1515 1498 } 1516 1499 1517 - #define FBNIC_PAGE_POOL_FLAGS \ 1518 - (PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV) 1519 - 1520 - static int fbnic_alloc_nv_page_pool(struct fbnic_net *fbn, 1521 - struct fbnic_napi_vector *nv) 1500 + static int 1501 + fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt, 1502 + unsigned int rxq_idx) 1522 1503 { 1523 1504 struct page_pool_params pp_params = { 1524 1505 .order = 0, 1525 - .flags = FBNIC_PAGE_POOL_FLAGS, 1526 - .pool_size = (fbn->hpq_size + fbn->ppq_size) * nv->rxt_count, 1506 + .flags = PP_FLAG_DMA_MAP | 1507 + PP_FLAG_DMA_SYNC_DEV, 1508 + .pool_size = fbn->hpq_size + fbn->ppq_size, 1527 1509 .nid = NUMA_NO_NODE, 1528 - .dev = nv->dev, 1510 + .dev = fbn->netdev->dev.parent, 1529 1511 .dma_dir = DMA_BIDIRECTIONAL, 1530 1512 .offset = 0, 1531 1513 .max_len = PAGE_SIZE, 1532 - .napi = &nv->napi, 1533 1514 .netdev = fbn->netdev, 1515 + .queue_idx = rxq_idx, 1534 1516 }; 1535 1517 struct page_pool *pp; 1536 1518 ··· 1549 1533 if (IS_ERR(pp)) 1550 1534 return PTR_ERR(pp); 1551 1535 1552 - nv->page_pool = pp; 1536 + qt->sub0.page_pool = pp; 1537 + if (netif_rxq_has_unreadable_mp(fbn->netdev, rxq_idx)) { 1538 + pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM; 1539 + pp_params.dma_dir = DMA_FROM_DEVICE; 1540 + 1541 + pp = page_pool_create(&pp_params); 1542 + if (IS_ERR(pp)) 1543 + goto err_destroy_sub0; 1544 + } else { 1545 + page_pool_get(pp); 1546 + } 1547 + qt->sub1.page_pool = pp; 1553 1548 1554 1549 return 0; 1550 + 1551 + err_destroy_sub0: 1552 + page_pool_destroy(pp); 1553 + return PTR_ERR(pp); 1555 1554 } 1556 1555 1557 1556 static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell, ··· 1621 1590 1622 1591 /* Tie napi to netdev */ 1623 1592 fbn->napi[fbnic_napi_idx(nv)] = nv; 1624 - netif_napi_add(fbn->netdev, &nv->napi, fbnic_poll); 1593 + netif_napi_add_locked(fbn->netdev, &nv->napi, fbnic_poll); 1625 1594 1626 1595 /* Record IRQ to NAPI struct */ 1627 - netif_napi_set_irq(&nv->napi, 1628 - pci_irq_vector(to_pci_dev(fbd->dev), nv->v_idx)); 1596 + netif_napi_set_irq_locked(&nv->napi, 1597 + pci_irq_vector(to_pci_dev(fbd->dev), 1598 + nv->v_idx)); 1629 1599 1630 1600 /* Tie nv back to PCIe dev */ 1631 1601 nv->dev = fbd->dev; 1632 1602 1633 - /* Allocate page pool */ 1634 - if (rxq_count) { 1635 - err = fbnic_alloc_nv_page_pool(fbn, nv); 1636 - if (err) 1637 - goto napi_del; 1638 - } 1639 - 1640 1603 /* Request the IRQ for napi vector */ 1641 1604 err = fbnic_napi_request_irq(fbd, nv); 1642 1605 if (err) 1643 - goto pp_destroy; 1606 + goto napi_del; 1644 1607 1645 1608 /* Initialize queue triads */ 1646 1609 qt = nv->qt; ··· 1704 1679 fbnic_ring_init(&qt->cmpl, db, rxq_idx, FBNIC_RING_F_STATS); 1705 1680 fbn->rx[rxq_idx] = &qt->cmpl; 1706 1681 1707 - err = xdp_rxq_info_reg(&qt->xdp_rxq, fbn->netdev, rxq_idx, 1708 - nv->napi.napi_id); 1709 - if (err) 1710 - goto free_ring_cur_qt; 1711 - 1712 1682 /* Update Rx queue index */ 1713 1683 rxt_count--; 1714 1684 rxq_idx += v_count; ··· 1714 1694 1715 1695 return 0; 1716 1696 1717 - while (rxt_count < nv->rxt_count) { 1718 - qt--; 1719 - 1720 - xdp_rxq_info_unreg(&qt->xdp_rxq); 1721 - free_ring_cur_qt: 1722 - fbnic_remove_rx_ring(fbn, &qt->sub0); 1723 - fbnic_remove_rx_ring(fbn, &qt->sub1); 1724 - fbnic_remove_rx_ring(fbn, &qt->cmpl); 1725 - rxt_count++; 1726 - } 1727 - while (txt_count < nv->txt_count) { 1728 - qt--; 1729 - 1730 - fbnic_remove_tx_ring(fbn, &qt->sub0); 1731 - fbnic_remove_xdp_ring(fbn, &qt->sub1); 1732 - fbnic_remove_tx_ring(fbn, &qt->cmpl); 1733 - 1734 - txt_count++; 1735 - } 1736 - fbnic_napi_free_irq(fbd, nv); 1737 - pp_destroy: 1738 - page_pool_destroy(nv->page_pool); 1739 1697 napi_del: 1740 - netif_napi_del(&nv->napi); 1698 + netif_napi_del_locked(&nv->napi); 1741 1699 fbn->napi[fbnic_napi_idx(nv)] = NULL; 1742 1700 kfree(nv); 1743 1701 return err; ··· 1929 1931 fbnic_free_ring_resources(dev, &qt->cmpl); 1930 1932 fbnic_free_ring_resources(dev, &qt->sub1); 1931 1933 fbnic_free_ring_resources(dev, &qt->sub0); 1934 + 1935 + if (xdp_rxq_info_is_reg(&qt->xdp_rxq)) { 1936 + xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq); 1937 + xdp_rxq_info_unreg(&qt->xdp_rxq); 1938 + fbnic_free_qt_page_pools(qt); 1939 + } 1932 1940 } 1933 1941 1934 1942 static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn, ··· 1965 1961 } 1966 1962 1967 1963 static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn, 1964 + struct fbnic_napi_vector *nv, 1968 1965 struct fbnic_q_triad *qt) 1969 1966 { 1970 1967 struct device *dev = fbn->netdev->dev.parent; 1971 1968 int err; 1972 1969 1973 - err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub0); 1970 + err = fbnic_alloc_qt_page_pools(fbn, qt, qt->cmpl.q_idx); 1974 1971 if (err) 1975 1972 return err; 1973 + 1974 + err = xdp_rxq_info_reg(&qt->xdp_rxq, fbn->netdev, qt->sub0.q_idx, 1975 + nv->napi.napi_id); 1976 + if (err) 1977 + goto free_page_pools; 1978 + 1979 + err = xdp_rxq_info_reg_mem_model(&qt->xdp_rxq, MEM_TYPE_PAGE_POOL, 1980 + qt->sub0.page_pool); 1981 + if (err) 1982 + goto unreg_rxq; 1983 + 1984 + err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub0); 1985 + if (err) 1986 + goto unreg_mm; 1976 1987 1977 1988 err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub1); 1978 1989 if (err) ··· 2003 1984 fbnic_free_ring_resources(dev, &qt->sub1); 2004 1985 free_sub0: 2005 1986 fbnic_free_ring_resources(dev, &qt->sub0); 1987 + unreg_mm: 1988 + xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq); 1989 + unreg_rxq: 1990 + xdp_rxq_info_unreg(&qt->xdp_rxq); 1991 + free_page_pools: 1992 + fbnic_free_qt_page_pools(qt); 2006 1993 return err; 2007 1994 } 2008 1995 2009 1996 static void fbnic_free_nv_resources(struct fbnic_net *fbn, 2010 1997 struct fbnic_napi_vector *nv) 2011 1998 { 2012 - int i, j; 1999 + int i; 2013 2000 2014 - /* Free Tx Resources */ 2015 - for (i = 0; i < nv->txt_count; i++) 2001 + for (i = 0; i < nv->txt_count + nv->rxt_count; i++) 2016 2002 fbnic_free_qt_resources(fbn, &nv->qt[i]); 2017 - 2018 - for (j = 0; j < nv->rxt_count; j++, i++) { 2019 - fbnic_free_qt_resources(fbn, &nv->qt[i]); 2020 - xdp_rxq_info_unreg_mem_model(&nv->qt[i].xdp_rxq); 2021 - } 2022 2003 } 2023 2004 2024 2005 static int fbnic_alloc_nv_resources(struct fbnic_net *fbn, ··· 2035 2016 2036 2017 /* Allocate Rx Resources */ 2037 2018 for (j = 0; j < nv->rxt_count; j++, i++) { 2038 - /* Register XDP memory model for completion queue */ 2039 - err = xdp_reg_mem_model(&nv->qt[i].xdp_rxq.mem, 2040 - MEM_TYPE_PAGE_POOL, 2041 - nv->page_pool); 2019 + err = fbnic_alloc_rx_qt_resources(fbn, nv, &nv->qt[i]); 2042 2020 if (err) 2043 - goto xdp_unreg_mem_model; 2044 - 2045 - err = fbnic_alloc_rx_qt_resources(fbn, &nv->qt[i]); 2046 - if (err) 2047 - goto xdp_unreg_cur_model; 2021 + goto free_qt_resources; 2048 2022 } 2049 2023 2050 2024 return 0; 2051 2025 2052 - xdp_unreg_mem_model: 2053 - while (j-- && i--) { 2054 - fbnic_free_qt_resources(fbn, &nv->qt[i]); 2055 - xdp_unreg_cur_model: 2056 - xdp_rxq_info_unreg_mem_model(&nv->qt[i].xdp_rxq); 2057 - } 2058 2026 free_qt_resources: 2059 2027 while (i--) 2060 2028 fbnic_free_qt_resources(fbn, &nv->qt[i]); ··· 2184 2178 int i; 2185 2179 2186 2180 for (i = 0; i < fbn->num_napi; i++) { 2187 - napi_disable(&fbn->napi[i]->napi); 2181 + napi_disable_locked(&fbn->napi[i]->napi); 2188 2182 2189 2183 fbnic_nv_irq_disable(fbn->napi[i]); 2190 2184 } 2191 2185 } 2192 2186 2187 + static void __fbnic_nv_disable(struct fbnic_napi_vector *nv) 2188 + { 2189 + int i, t; 2190 + 2191 + /* Disable Tx queue triads */ 2192 + for (t = 0; t < nv->txt_count; t++) { 2193 + struct fbnic_q_triad *qt = &nv->qt[t]; 2194 + 2195 + fbnic_disable_twq0(&qt->sub0); 2196 + fbnic_disable_twq1(&qt->sub1); 2197 + fbnic_disable_tcq(&qt->cmpl); 2198 + } 2199 + 2200 + /* Disable Rx queue triads */ 2201 + for (i = 0; i < nv->rxt_count; i++, t++) { 2202 + struct fbnic_q_triad *qt = &nv->qt[t]; 2203 + 2204 + fbnic_disable_bdq(&qt->sub0, &qt->sub1); 2205 + fbnic_disable_rcq(&qt->cmpl); 2206 + } 2207 + } 2208 + 2209 + static void 2210 + fbnic_nv_disable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) 2211 + { 2212 + __fbnic_nv_disable(nv); 2213 + fbnic_wrfl(fbn->fbd); 2214 + } 2215 + 2193 2216 void fbnic_disable(struct fbnic_net *fbn) 2194 2217 { 2195 2218 struct fbnic_dev *fbd = fbn->fbd; 2196 - int i, j, t; 2219 + int i; 2197 2220 2198 - for (i = 0; i < fbn->num_napi; i++) { 2199 - struct fbnic_napi_vector *nv = fbn->napi[i]; 2200 - 2201 - /* Disable Tx queue triads */ 2202 - for (t = 0; t < nv->txt_count; t++) { 2203 - struct fbnic_q_triad *qt = &nv->qt[t]; 2204 - 2205 - fbnic_disable_twq0(&qt->sub0); 2206 - fbnic_disable_twq1(&qt->sub1); 2207 - fbnic_disable_tcq(&qt->cmpl); 2208 - } 2209 - 2210 - /* Disable Rx queue triads */ 2211 - for (j = 0; j < nv->rxt_count; j++, t++) { 2212 - struct fbnic_q_triad *qt = &nv->qt[t]; 2213 - 2214 - fbnic_disable_bdq(&qt->sub0, &qt->sub1); 2215 - fbnic_disable_rcq(&qt->cmpl); 2216 - } 2217 - } 2221 + for (i = 0; i < fbn->num_napi; i++) 2222 + __fbnic_nv_disable(fbn->napi[i]); 2218 2223 2219 2224 fbnic_wrfl(fbd); 2220 2225 } ··· 2314 2297 return err; 2315 2298 } 2316 2299 2300 + static int 2301 + fbnic_wait_queue_idle(struct fbnic_net *fbn, bool rx, unsigned int idx) 2302 + { 2303 + static const unsigned int tx_regs[] = { 2304 + FBNIC_QM_TWQ_IDLE(0), FBNIC_QM_TQS_IDLE(0), 2305 + FBNIC_QM_TDE_IDLE(0), FBNIC_QM_TCQ_IDLE(0), 2306 + }, rx_regs[] = { 2307 + FBNIC_QM_HPQ_IDLE(0), FBNIC_QM_PPQ_IDLE(0), 2308 + FBNIC_QM_RCQ_IDLE(0), 2309 + }; 2310 + struct fbnic_dev *fbd = fbn->fbd; 2311 + unsigned int val, mask, off; 2312 + const unsigned int *regs; 2313 + unsigned int reg_cnt; 2314 + int i, err; 2315 + 2316 + regs = rx ? rx_regs : tx_regs; 2317 + reg_cnt = rx ? ARRAY_SIZE(rx_regs) : ARRAY_SIZE(tx_regs); 2318 + 2319 + off = idx / 32; 2320 + mask = BIT(idx % 32); 2321 + 2322 + for (i = 0; i < reg_cnt; i++) { 2323 + err = read_poll_timeout_atomic(fbnic_rd32, val, val & mask, 2324 + 2, 500000, false, 2325 + fbd, regs[i] + off); 2326 + if (err) { 2327 + netdev_err(fbd->netdev, 2328 + "wait for queue %s%d idle failed 0x%04x(%d): %08x (mask: %08x)\n", 2329 + rx ? "Rx" : "Tx", idx, regs[i] + off, i, 2330 + val, mask); 2331 + return err; 2332 + } 2333 + } 2334 + 2335 + return 0; 2336 + } 2337 + 2338 + static void fbnic_nv_flush(struct fbnic_napi_vector *nv) 2339 + { 2340 + int j, t; 2341 + 2342 + /* Flush any processed Tx Queue Triads and drop the rest */ 2343 + for (t = 0; t < nv->txt_count; t++) { 2344 + struct fbnic_q_triad *qt = &nv->qt[t]; 2345 + struct netdev_queue *tx_queue; 2346 + 2347 + /* Clean the work queues of unprocessed work */ 2348 + fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail); 2349 + fbnic_clean_twq1(nv, false, &qt->sub1, true, 2350 + qt->sub1.tail); 2351 + 2352 + /* Reset completion queue descriptor ring */ 2353 + memset(qt->cmpl.desc, 0, qt->cmpl.size); 2354 + 2355 + /* Nothing else to do if Tx queue is disabled */ 2356 + if (qt->sub0.flags & FBNIC_RING_F_DISABLED) 2357 + continue; 2358 + 2359 + /* Reset BQL associated with Tx queue */ 2360 + tx_queue = netdev_get_tx_queue(nv->napi.dev, 2361 + qt->sub0.q_idx); 2362 + netdev_tx_reset_queue(tx_queue); 2363 + } 2364 + 2365 + /* Flush any processed Rx Queue Triads and drop the rest */ 2366 + for (j = 0; j < nv->rxt_count; j++, t++) { 2367 + struct fbnic_q_triad *qt = &nv->qt[t]; 2368 + 2369 + /* Clean the work queues of unprocessed work */ 2370 + fbnic_clean_bdq(&qt->sub0, qt->sub0.tail, 0); 2371 + fbnic_clean_bdq(&qt->sub1, qt->sub1.tail, 0); 2372 + 2373 + /* Reset completion queue descriptor ring */ 2374 + memset(qt->cmpl.desc, 0, qt->cmpl.size); 2375 + 2376 + fbnic_put_pkt_buff(qt, qt->cmpl.pkt, 0); 2377 + memset(qt->cmpl.pkt, 0, sizeof(struct fbnic_pkt_buff)); 2378 + } 2379 + } 2380 + 2317 2381 void fbnic_flush(struct fbnic_net *fbn) 2318 2382 { 2319 2383 int i; 2320 2384 2321 - for (i = 0; i < fbn->num_napi; i++) { 2322 - struct fbnic_napi_vector *nv = fbn->napi[i]; 2323 - int j, t; 2385 + for (i = 0; i < fbn->num_napi; i++) 2386 + fbnic_nv_flush(fbn->napi[i]); 2387 + } 2324 2388 2325 - /* Flush any processed Tx Queue Triads and drop the rest */ 2326 - for (t = 0; t < nv->txt_count; t++) { 2327 - struct fbnic_q_triad *qt = &nv->qt[t]; 2328 - struct netdev_queue *tx_queue; 2389 + static void fbnic_nv_fill(struct fbnic_napi_vector *nv) 2390 + { 2391 + int j, t; 2329 2392 2330 - /* Clean the work queues of unprocessed work */ 2331 - fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail); 2332 - fbnic_clean_twq1(nv, false, &qt->sub1, true, 2333 - qt->sub1.tail); 2393 + /* Configure NAPI mapping and populate pages 2394 + * in the BDQ rings to use for Rx 2395 + */ 2396 + for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) { 2397 + struct fbnic_q_triad *qt = &nv->qt[t]; 2334 2398 2335 - /* Reset completion queue descriptor ring */ 2336 - memset(qt->cmpl.desc, 0, qt->cmpl.size); 2337 - 2338 - /* Nothing else to do if Tx queue is disabled */ 2339 - if (qt->sub0.flags & FBNIC_RING_F_DISABLED) 2340 - continue; 2341 - 2342 - /* Reset BQL associated with Tx queue */ 2343 - tx_queue = netdev_get_tx_queue(nv->napi.dev, 2344 - qt->sub0.q_idx); 2345 - netdev_tx_reset_queue(tx_queue); 2346 - } 2347 - 2348 - /* Flush any processed Rx Queue Triads and drop the rest */ 2349 - for (j = 0; j < nv->rxt_count; j++, t++) { 2350 - struct fbnic_q_triad *qt = &nv->qt[t]; 2351 - 2352 - /* Clean the work queues of unprocessed work */ 2353 - fbnic_clean_bdq(nv, 0, &qt->sub0, qt->sub0.tail); 2354 - fbnic_clean_bdq(nv, 0, &qt->sub1, qt->sub1.tail); 2355 - 2356 - /* Reset completion queue descriptor ring */ 2357 - memset(qt->cmpl.desc, 0, qt->cmpl.size); 2358 - 2359 - fbnic_put_pkt_buff(nv, qt->cmpl.pkt, 0); 2360 - memset(qt->cmpl.pkt, 0, sizeof(struct fbnic_pkt_buff)); 2361 - } 2399 + /* Populate the header and payload BDQs */ 2400 + fbnic_fill_bdq(&qt->sub0); 2401 + fbnic_fill_bdq(&qt->sub1); 2362 2402 } 2363 2403 } 2364 2404 ··· 2423 2349 { 2424 2350 int i; 2425 2351 2426 - for (i = 0; i < fbn->num_napi; i++) { 2427 - struct fbnic_napi_vector *nv = fbn->napi[i]; 2428 - int j, t; 2429 - 2430 - /* Configure NAPI mapping and populate pages 2431 - * in the BDQ rings to use for Rx 2432 - */ 2433 - for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) { 2434 - struct fbnic_q_triad *qt = &nv->qt[t]; 2435 - 2436 - /* Populate the header and payload BDQs */ 2437 - fbnic_fill_bdq(nv, &qt->sub0); 2438 - fbnic_fill_bdq(nv, &qt->sub1); 2439 - } 2440 - } 2352 + for (i = 0; i < fbn->num_napi; i++) 2353 + fbnic_nv_fill(fbn->napi[i]); 2441 2354 } 2442 2355 2443 2356 static void fbnic_enable_twq0(struct fbnic_ring *twq) ··· 2642 2581 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_ENABLE); 2643 2582 } 2644 2583 2584 + static void __fbnic_nv_enable(struct fbnic_napi_vector *nv) 2585 + { 2586 + int j, t; 2587 + 2588 + /* Setup Tx Queue Triads */ 2589 + for (t = 0; t < nv->txt_count; t++) { 2590 + struct fbnic_q_triad *qt = &nv->qt[t]; 2591 + 2592 + fbnic_enable_twq0(&qt->sub0); 2593 + fbnic_enable_twq1(&qt->sub1); 2594 + fbnic_enable_tcq(nv, &qt->cmpl); 2595 + } 2596 + 2597 + /* Setup Rx Queue Triads */ 2598 + for (j = 0; j < nv->rxt_count; j++, t++) { 2599 + struct fbnic_q_triad *qt = &nv->qt[t]; 2600 + 2601 + page_pool_enable_direct_recycling(qt->sub0.page_pool, 2602 + &nv->napi); 2603 + page_pool_enable_direct_recycling(qt->sub1.page_pool, 2604 + &nv->napi); 2605 + 2606 + fbnic_enable_bdq(&qt->sub0, &qt->sub1); 2607 + fbnic_config_drop_mode_rcq(nv, &qt->cmpl); 2608 + fbnic_enable_rcq(nv, &qt->cmpl); 2609 + } 2610 + } 2611 + 2612 + static void fbnic_nv_enable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) 2613 + { 2614 + __fbnic_nv_enable(nv); 2615 + fbnic_wrfl(fbn->fbd); 2616 + } 2617 + 2645 2618 void fbnic_enable(struct fbnic_net *fbn) 2646 2619 { 2647 2620 struct fbnic_dev *fbd = fbn->fbd; 2648 2621 int i; 2649 2622 2650 - for (i = 0; i < fbn->num_napi; i++) { 2651 - struct fbnic_napi_vector *nv = fbn->napi[i]; 2652 - int j, t; 2653 - 2654 - /* Setup Tx Queue Triads */ 2655 - for (t = 0; t < nv->txt_count; t++) { 2656 - struct fbnic_q_triad *qt = &nv->qt[t]; 2657 - 2658 - fbnic_enable_twq0(&qt->sub0); 2659 - fbnic_enable_twq1(&qt->sub1); 2660 - fbnic_enable_tcq(nv, &qt->cmpl); 2661 - } 2662 - 2663 - /* Setup Rx Queue Triads */ 2664 - for (j = 0; j < nv->rxt_count; j++, t++) { 2665 - struct fbnic_q_triad *qt = &nv->qt[t]; 2666 - 2667 - fbnic_enable_bdq(&qt->sub0, &qt->sub1); 2668 - fbnic_config_drop_mode_rcq(nv, &qt->cmpl); 2669 - fbnic_enable_rcq(nv, &qt->cmpl); 2670 - } 2671 - } 2623 + for (i = 0; i < fbn->num_napi; i++) 2624 + __fbnic_nv_enable(fbn->napi[i]); 2672 2625 2673 2626 fbnic_wrfl(fbd); 2674 2627 } ··· 2701 2626 for (i = 0; i < fbn->num_napi; i++) { 2702 2627 struct fbnic_napi_vector *nv = fbn->napi[i]; 2703 2628 2704 - napi_enable(&nv->napi); 2629 + napi_enable_locked(&nv->napi); 2705 2630 2706 2631 fbnic_nv_irq_enable(nv); 2707 2632 ··· 2754 2679 2755 2680 fbnic_wrfl(fbd); 2756 2681 } 2682 + 2683 + static int fbnic_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) 2684 + { 2685 + struct fbnic_net *fbn = netdev_priv(dev); 2686 + const struct fbnic_q_triad *real; 2687 + struct fbnic_q_triad *qt = qmem; 2688 + struct fbnic_napi_vector *nv; 2689 + 2690 + if (!netif_running(dev)) 2691 + return fbnic_alloc_qt_page_pools(fbn, qt, idx); 2692 + 2693 + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); 2694 + nv = fbn->napi[idx % fbn->num_napi]; 2695 + 2696 + fbnic_ring_init(&qt->sub0, real->sub0.doorbell, real->sub0.q_idx, 2697 + real->sub0.flags); 2698 + fbnic_ring_init(&qt->sub1, real->sub1.doorbell, real->sub1.q_idx, 2699 + real->sub1.flags); 2700 + fbnic_ring_init(&qt->cmpl, real->cmpl.doorbell, real->cmpl.q_idx, 2701 + real->cmpl.flags); 2702 + 2703 + return fbnic_alloc_rx_qt_resources(fbn, nv, qt); 2704 + } 2705 + 2706 + static void fbnic_queue_mem_free(struct net_device *dev, void *qmem) 2707 + { 2708 + struct fbnic_net *fbn = netdev_priv(dev); 2709 + struct fbnic_q_triad *qt = qmem; 2710 + 2711 + if (!netif_running(dev)) 2712 + fbnic_free_qt_page_pools(qt); 2713 + else 2714 + fbnic_free_qt_resources(fbn, qt); 2715 + } 2716 + 2717 + static void __fbnic_nv_restart(struct fbnic_net *fbn, 2718 + struct fbnic_napi_vector *nv) 2719 + { 2720 + struct fbnic_dev *fbd = fbn->fbd; 2721 + int i; 2722 + 2723 + fbnic_nv_enable(fbn, nv); 2724 + fbnic_nv_fill(nv); 2725 + 2726 + napi_enable_locked(&nv->napi); 2727 + fbnic_nv_irq_enable(nv); 2728 + fbnic_wr32(fbd, FBNIC_INTR_SET(nv->v_idx / 32), BIT(nv->v_idx % 32)); 2729 + fbnic_wrfl(fbd); 2730 + 2731 + for (i = 0; i < nv->txt_count; i++) 2732 + netif_wake_subqueue(fbn->netdev, nv->qt[i].sub0.q_idx); 2733 + } 2734 + 2735 + static int fbnic_queue_start(struct net_device *dev, void *qmem, int idx) 2736 + { 2737 + struct fbnic_net *fbn = netdev_priv(dev); 2738 + struct fbnic_napi_vector *nv; 2739 + struct fbnic_q_triad *real; 2740 + 2741 + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); 2742 + nv = fbn->napi[idx % fbn->num_napi]; 2743 + 2744 + fbnic_aggregate_ring_rx_counters(fbn, &real->sub0); 2745 + fbnic_aggregate_ring_rx_counters(fbn, &real->sub1); 2746 + fbnic_aggregate_ring_rx_counters(fbn, &real->cmpl); 2747 + 2748 + memcpy(real, qmem, sizeof(*real)); 2749 + 2750 + __fbnic_nv_restart(fbn, nv); 2751 + 2752 + return 0; 2753 + } 2754 + 2755 + static int fbnic_queue_stop(struct net_device *dev, void *qmem, int idx) 2756 + { 2757 + struct fbnic_net *fbn = netdev_priv(dev); 2758 + const struct fbnic_q_triad *real; 2759 + struct fbnic_napi_vector *nv; 2760 + int i, t; 2761 + int err; 2762 + 2763 + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); 2764 + nv = fbn->napi[idx % fbn->num_napi]; 2765 + 2766 + napi_disable_locked(&nv->napi); 2767 + fbnic_nv_irq_disable(nv); 2768 + 2769 + for (i = 0; i < nv->txt_count; i++) 2770 + netif_stop_subqueue(dev, nv->qt[i].sub0.q_idx); 2771 + fbnic_nv_disable(fbn, nv); 2772 + 2773 + for (t = 0; t < nv->txt_count + nv->rxt_count; t++) { 2774 + err = fbnic_wait_queue_idle(fbn, t >= nv->txt_count, 2775 + nv->qt[t].sub0.q_idx); 2776 + if (err) 2777 + goto err_restart; 2778 + } 2779 + 2780 + fbnic_synchronize_irq(fbn->fbd, nv->v_idx); 2781 + fbnic_nv_flush(nv); 2782 + 2783 + page_pool_disable_direct_recycling(real->sub0.page_pool); 2784 + page_pool_disable_direct_recycling(real->sub1.page_pool); 2785 + 2786 + memcpy(qmem, real, sizeof(*real)); 2787 + 2788 + return 0; 2789 + 2790 + err_restart: 2791 + __fbnic_nv_restart(fbn, nv); 2792 + return err; 2793 + } 2794 + 2795 + const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops = { 2796 + .ndo_queue_mem_size = sizeof(struct fbnic_q_triad), 2797 + .ndo_queue_mem_alloc = fbnic_queue_mem_alloc, 2798 + .ndo_queue_mem_free = fbnic_queue_mem_free, 2799 + .ndo_queue_start = fbnic_queue_start, 2800 + .ndo_queue_stop = fbnic_queue_stop, 2801 + };

+13 -7

drivers/net/ethernet/meta/fbnic/fbnic_txrx.h

··· 100 100 #define FBNIC_PAGECNT_BIAS_MAX PAGE_SIZE 101 101 102 102 struct fbnic_rx_buf { 103 - struct page *page; 103 + netmem_ref netmem; 104 104 long pagecnt_bias; 105 105 }; 106 106 ··· 121 121 122 122 u32 head, tail; /* Head/Tail of ring */ 123 123 124 - /* Deferred_head is used to cache the head for TWQ1 if an attempt 125 - * is made to clean TWQ1 with zero napi_budget. We do not use it for 126 - * any other ring. 127 - */ 128 - s32 deferred_head; 124 + union { 125 + /* Rx BDQs only */ 126 + struct page_pool *page_pool; 127 + 128 + /* Deferred_head is used to cache the head for TWQ1 if 129 + * an attempt is made to clean TWQ1 with zero napi_budget. 130 + * We do not use it for any other ring. 131 + */ 132 + s32 deferred_head; 133 + }; 129 134 130 135 struct fbnic_queue_stats stats; 131 136 ··· 147 142 struct fbnic_napi_vector { 148 143 struct napi_struct napi; 149 144 struct device *dev; /* Device for DMA unmapping */ 150 - struct page_pool *page_pool; 151 145 struct fbnic_dev *fbd; 152 146 153 147 u16 v_idx; ··· 155 151 156 152 struct fbnic_q_triad qt[]; 157 153 }; 154 + 155 + extern const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops; 158 156 159 157 netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev); 160 158 netdev_features_t

+2

include/net/netdev_queues.h

··· 151 151 int idx); 152 152 }; 153 153 154 + bool netif_rxq_has_unreadable_mp(struct net_device *dev, int idx); 155 + 154 156 /** 155 157 * DOC: Lockless queue stopping / waking helpers. 156 158 *

+12

include/net/page_pool/helpers.h

··· 505 505 page_pool_update_nid(pool, new_nid); 506 506 } 507 507 508 + /** 509 + * page_pool_is_unreadable() - will allocated buffers be unreadable for the CPU 510 + * @pool: queried page pool 511 + * 512 + * Check if page pool will return buffers which are unreadable to the CPU / 513 + * kernel. This will only be the case if user space bound a memory provider (mp) 514 + * which returns unreadable memory to the queue served by the page pool. 515 + * If %PP_FLAG_ALLOW_UNREADABLE_NETMEM was set but there is no mp bound 516 + * this helper will return false. See also netif_rxq_has_unreadable_mp(). 517 + * 518 + * Return: true if memory allocated by the page pool may be unreadable 519 + */ 508 520 static inline bool page_pool_is_unreadable(struct page_pool *pool) 509 521 { 510 522 return !!pool->mp_ops;

+9

net/core/netdev_rx_queue.c

··· 9 9 10 10 #include "page_pool_priv.h" 11 11 12 + /* See also page_pool_is_unreadable() */ 13 + bool netif_rxq_has_unreadable_mp(struct net_device *dev, int idx) 14 + { 15 + struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, idx); 16 + 17 + return !!rxq->mp_params.mp_ops; 18 + } 19 + EXPORT_SYMBOL(netif_rxq_has_unreadable_mp); 20 + 12 21 int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx) 13 22 { 14 23 struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, rxq_idx);

Configure Feed

Configure Feed