Merge branch 'am65-cpsw-cleanup' · tjh.dev/kernel@de7a88b

+71 -148

2 changed files

expand all

drivers

net

ethernet

am65-cpsw-nuss.c

am65-cpsw-nuss.h

+63 -148

drivers/net/ethernet/ti/am65-cpsw-nuss.c

··· 164 164 #define AM65_CPSW_CPPI_TX_PKT_TYPE 0x7 165 165 166 166 /* XDP */ 167 + #define AM65_CPSW_XDP_TX BIT(2) 167 168 #define AM65_CPSW_XDP_CONSUMED BIT(1) 168 169 #define AM65_CPSW_XDP_REDIRECT BIT(0) 169 170 #define AM65_CPSW_XDP_PASS 0 ··· 830 829 { 831 830 struct am65_cpsw_tx_chn *tx_chn = data; 832 831 enum am65_cpsw_tx_buf_type buf_type; 832 + struct am65_cpsw_tx_swdata *swdata; 833 833 struct cppi5_host_desc_t *desc_tx; 834 834 struct xdp_frame *xdpf; 835 835 struct sk_buff *skb; 836 - void **swdata; 837 836 838 837 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma); 839 838 swdata = cppi5_hdesc_get_swdata(desc_tx); 840 839 buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma); 841 840 if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) { 842 - skb = *(swdata); 841 + skb = swdata->skb; 843 842 dev_kfree_skb_any(skb); 844 843 } else { 845 - xdpf = *(swdata); 844 + xdpf = swdata->xdpf; 846 845 xdp_return_frame(xdpf); 847 846 } 848 847 ··· 1099 1098 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 1100 1099 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 1101 1100 struct cppi5_host_desc_t *host_desc; 1101 + struct am65_cpsw_tx_swdata *swdata; 1102 1102 struct netdev_queue *netif_txq; 1103 1103 dma_addr_t dma_desc, dma_buf; 1104 1104 u32 pkt_len = xdpf->len; 1105 - void **swdata; 1106 1105 int ret; 1107 1106 1108 1107 host_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); ··· 1132 1131 cppi5_hdesc_attach_buf(host_desc, dma_buf, pkt_len, dma_buf, pkt_len); 1133 1132 1134 1133 swdata = cppi5_hdesc_get_swdata(host_desc); 1135 - *(swdata) = xdpf; 1134 + swdata->ndev = ndev; 1135 + swdata->xdpf = xdpf; 1136 1136 1137 1137 /* Report BQL before sending the packet */ 1138 1138 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id); ··· 1169 1167 1170 1168 static int am65_cpsw_run_xdp(struct am65_cpsw_rx_flow *flow, 1171 1169 struct am65_cpsw_port *port, 1172 - struct xdp_buff *xdp, 1173 - int cpu, int *len) 1170 + struct xdp_buff *xdp, int *len) 1174 1171 { 1175 1172 struct am65_cpsw_common *common = flow->common; 1176 1173 struct net_device *ndev = port->ndev; 1177 1174 int ret = AM65_CPSW_XDP_CONSUMED; 1178 1175 struct am65_cpsw_tx_chn *tx_chn; 1179 1176 struct netdev_queue *netif_txq; 1177 + int cpu = smp_processor_id(); 1180 1178 struct xdp_frame *xdpf; 1181 1179 struct bpf_prog *prog; 1182 - struct page *page; 1183 1180 int pkt_len; 1184 1181 u32 act; 1185 1182 int err; ··· 1194 1193 1195 1194 switch (act) { 1196 1195 case XDP_PASS: 1197 - ret = AM65_CPSW_XDP_PASS; 1198 - goto out; 1196 + return AM65_CPSW_XDP_PASS; 1199 1197 case XDP_TX: 1200 1198 tx_chn = &common->tx_chns[cpu % AM65_CPSW_MAX_QUEUES]; 1201 1199 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id); ··· 1213 1213 goto drop; 1214 1214 1215 1215 dev_sw_netstats_rx_add(ndev, pkt_len); 1216 - ret = AM65_CPSW_XDP_CONSUMED; 1217 - goto out; 1216 + return AM65_CPSW_XDP_TX; 1218 1217 case XDP_REDIRECT: 1219 1218 if (unlikely(xdp_do_redirect(ndev, xdp, prog))) 1220 1219 goto drop; 1221 1220 1222 1221 dev_sw_netstats_rx_add(ndev, pkt_len); 1223 - ret = AM65_CPSW_XDP_REDIRECT; 1224 - goto out; 1222 + return AM65_CPSW_XDP_REDIRECT; 1225 1223 default: 1226 1224 bpf_warn_invalid_xdp_action(ndev, prog, act); 1227 1225 fallthrough; ··· 1231 1233 ndev->stats.rx_dropped++; 1232 1234 } 1233 1235 1234 - page = virt_to_head_page(xdp->data); 1235 - am65_cpsw_put_page(flow, page, true); 1236 - 1237 - out: 1238 1236 return ret; 1239 1237 } 1240 1238 ··· 1268 1274 } 1269 1275 1270 1276 static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow *flow, 1271 - int cpu, int *xdp_state) 1277 + int *xdp_state) 1272 1278 { 1273 1279 struct am65_cpsw_rx_chn *rx_chn = &flow->common->rx_chns; 1274 1280 u32 buf_dma_len, pkt_len, port_id = 0, csum_info; ··· 1328 1334 xdp_init_buff(&xdp, PAGE_SIZE, &port->xdp_rxq[flow->id]); 1329 1335 xdp_prepare_buff(&xdp, page_addr, AM65_CPSW_HEADROOM, 1330 1336 pkt_len, false); 1331 - *xdp_state = am65_cpsw_run_xdp(flow, port, &xdp, 1332 - cpu, &pkt_len); 1337 + *xdp_state = am65_cpsw_run_xdp(flow, port, &xdp, &pkt_len); 1338 + if (*xdp_state == AM65_CPSW_XDP_CONSUMED) { 1339 + page = virt_to_head_page(xdp.data); 1340 + am65_cpsw_put_page(flow, page, true); 1341 + goto allocate; 1342 + } 1343 + 1333 1344 if (*xdp_state != AM65_CPSW_XDP_PASS) 1334 1345 goto allocate; 1335 1346 ··· 1400 1401 { 1401 1402 struct am65_cpsw_rx_flow *flow = am65_cpsw_napi_to_rx_flow(napi_rx); 1402 1403 struct am65_cpsw_common *common = flow->common; 1403 - int cpu = smp_processor_id(); 1404 1404 int xdp_state_or = 0; 1405 1405 int cur_budget, ret; 1406 1406 int xdp_state; ··· 1408 1410 /* process only this flow */ 1409 1411 cur_budget = budget; 1410 1412 while (cur_budget--) { 1411 - ret = am65_cpsw_nuss_rx_packets(flow, cpu, &xdp_state); 1413 + ret = am65_cpsw_nuss_rx_packets(flow, &xdp_state); 1412 1414 xdp_state_or |= xdp_state; 1413 1415 if (ret) 1414 1416 break; ··· 1436 1438 return num_rx; 1437 1439 } 1438 1440 1439 - static struct sk_buff * 1440 - am65_cpsw_nuss_tx_compl_packet_skb(struct am65_cpsw_tx_chn *tx_chn, 1441 - dma_addr_t desc_dma) 1442 - { 1443 - struct cppi5_host_desc_t *desc_tx; 1444 - struct sk_buff *skb; 1445 - void **swdata; 1446 - 1447 - desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, 1448 - desc_dma); 1449 - swdata = cppi5_hdesc_get_swdata(desc_tx); 1450 - skb = *(swdata); 1451 - am65_cpsw_nuss_xmit_free(tx_chn, desc_tx); 1452 - 1453 - am65_cpts_tx_timestamp(tx_chn->common->cpts, skb); 1454 - 1455 - dev_sw_netstats_tx_add(skb->dev, 1, skb->len); 1456 - 1457 - return skb; 1458 - } 1459 - 1460 - static struct xdp_frame * 1461 - am65_cpsw_nuss_tx_compl_packet_xdp(struct am65_cpsw_common *common, 1462 - struct am65_cpsw_tx_chn *tx_chn, 1463 - dma_addr_t desc_dma, 1464 - struct net_device **ndev) 1465 - { 1466 - struct cppi5_host_desc_t *desc_tx; 1467 - struct am65_cpsw_port *port; 1468 - struct xdp_frame *xdpf; 1469 - u32 port_id = 0; 1470 - void **swdata; 1471 - 1472 - desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma); 1473 - cppi5_desc_get_tags_ids(&desc_tx->hdr, NULL, &port_id); 1474 - swdata = cppi5_hdesc_get_swdata(desc_tx); 1475 - xdpf = *(swdata); 1476 - am65_cpsw_nuss_xmit_free(tx_chn, desc_tx); 1477 - 1478 - port = am65_common_get_port(common, port_id); 1479 - dev_sw_netstats_tx_add(port->ndev, 1, xdpf->len); 1480 - *ndev = port->ndev; 1481 - 1482 - return xdpf; 1483 - } 1484 - 1485 1441 static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev, 1486 1442 struct netdev_queue *netif_txq) 1487 1443 { ··· 1456 1504 static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common, 1457 1505 int chn, unsigned int budget, bool *tdown) 1458 1506 { 1507 + bool single_port = AM65_CPSW_IS_CPSW2G(common); 1459 1508 enum am65_cpsw_tx_buf_type buf_type; 1509 + struct am65_cpsw_tx_swdata *swdata; 1510 + struct cppi5_host_desc_t *desc_tx; 1460 1511 struct device *dev = common->dev; 1461 1512 struct am65_cpsw_tx_chn *tx_chn; 1462 1513 struct netdev_queue *netif_txq; 1463 1514 unsigned int total_bytes = 0; 1464 1515 struct net_device *ndev; 1465 1516 struct xdp_frame *xdpf; 1517 + unsigned int pkt_len; 1466 1518 struct sk_buff *skb; 1467 1519 dma_addr_t desc_dma; 1468 1520 int res, num_tx = 0; ··· 1474 1518 tx_chn = &common->tx_chns[chn]; 1475 1519 1476 1520 while (true) { 1477 - spin_lock(&tx_chn->lock); 1521 + if (!single_port) 1522 + spin_lock(&tx_chn->lock); 1478 1523 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma); 1479 - spin_unlock(&tx_chn->lock); 1524 + if (!single_port) 1525 + spin_unlock(&tx_chn->lock); 1526 + 1480 1527 if (res == -ENODATA) 1481 1528 break; 1482 1529 ··· 1490 1531 break; 1491 1532 } 1492 1533 1534 + desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, 1535 + desc_dma); 1536 + swdata = cppi5_hdesc_get_swdata(desc_tx); 1537 + ndev = swdata->ndev; 1493 1538 buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma); 1494 1539 if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) { 1495 - skb = am65_cpsw_nuss_tx_compl_packet_skb(tx_chn, desc_dma); 1496 - ndev = skb->dev; 1497 - total_bytes = skb->len; 1540 + skb = swdata->skb; 1541 + am65_cpts_tx_timestamp(tx_chn->common->cpts, skb); 1542 + pkt_len = skb->len; 1498 1543 napi_consume_skb(skb, budget); 1499 1544 } else { 1500 - xdpf = am65_cpsw_nuss_tx_compl_packet_xdp(common, tx_chn, 1501 - desc_dma, &ndev); 1502 - total_bytes = xdpf->len; 1545 + xdpf = swdata->xdpf; 1546 + pkt_len = xdpf->len; 1503 1547 if (buf_type == AM65_CPSW_TX_BUF_TYPE_XDP_TX) 1504 1548 xdp_return_frame_rx_napi(xdpf); 1505 1549 else 1506 1550 xdp_return_frame(xdpf); 1507 1551 } 1552 + 1553 + total_bytes += pkt_len; 1508 1554 num_tx++; 1555 + am65_cpsw_nuss_xmit_free(tx_chn, desc_tx); 1556 + dev_sw_netstats_tx_add(ndev, 1, pkt_len); 1557 + if (!single_port) { 1558 + /* as packets from multi ports can be interleaved 1559 + * on the same channel, we have to figure out the 1560 + * port/queue at every packet and report it/wake queue. 1561 + */ 1562 + netif_txq = netdev_get_tx_queue(ndev, chn); 1563 + netdev_tx_completed_queue(netif_txq, 1, pkt_len); 1564 + am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq); 1565 + } 1566 + } 1509 1567 1568 + if (single_port) { 1510 1569 netif_txq = netdev_get_tx_queue(ndev, chn); 1511 - 1512 1570 netdev_tx_completed_queue(netif_txq, num_tx, total_bytes); 1513 - 1514 1571 am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq); 1515 1572 } 1516 - 1517 - dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx); 1518 - 1519 - return num_tx; 1520 - } 1521 - 1522 - static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common, 1523 - int chn, unsigned int budget, bool *tdown) 1524 - { 1525 - enum am65_cpsw_tx_buf_type buf_type; 1526 - struct device *dev = common->dev; 1527 - struct am65_cpsw_tx_chn *tx_chn; 1528 - struct netdev_queue *netif_txq; 1529 - unsigned int total_bytes = 0; 1530 - struct net_device *ndev; 1531 - struct xdp_frame *xdpf; 1532 - struct sk_buff *skb; 1533 - dma_addr_t desc_dma; 1534 - int res, num_tx = 0; 1535 - 1536 - tx_chn = &common->tx_chns[chn]; 1537 - 1538 - while (true) { 1539 - res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma); 1540 - if (res == -ENODATA) 1541 - break; 1542 - 1543 - if (cppi5_desc_is_tdcm(desc_dma)) { 1544 - if (atomic_dec_and_test(&common->tdown_cnt)) 1545 - complete(&common->tdown_complete); 1546 - *tdown = true; 1547 - break; 1548 - } 1549 - 1550 - buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma); 1551 - if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) { 1552 - skb = am65_cpsw_nuss_tx_compl_packet_skb(tx_chn, desc_dma); 1553 - ndev = skb->dev; 1554 - total_bytes += skb->len; 1555 - napi_consume_skb(skb, budget); 1556 - } else { 1557 - xdpf = am65_cpsw_nuss_tx_compl_packet_xdp(common, tx_chn, 1558 - desc_dma, &ndev); 1559 - total_bytes += xdpf->len; 1560 - if (buf_type == AM65_CPSW_TX_BUF_TYPE_XDP_TX) 1561 - xdp_return_frame_rx_napi(xdpf); 1562 - else 1563 - xdp_return_frame(xdpf); 1564 - } 1565 - num_tx++; 1566 - } 1567 - 1568 - if (!num_tx) 1569 - return 0; 1570 - 1571 - netif_txq = netdev_get_tx_queue(ndev, chn); 1572 - 1573 - netdev_tx_completed_queue(netif_txq, num_tx, total_bytes); 1574 - 1575 - am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq); 1576 1573 1577 1574 dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx); 1578 1575 ··· 1550 1635 bool tdown = false; 1551 1636 int num_tx; 1552 1637 1553 - if (AM65_CPSW_IS_CPSW2G(tx_chn->common)) 1554 - num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id, 1555 - budget, &tdown); 1556 - else 1557 - num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common, 1558 - tx_chn->id, budget, &tdown); 1559 - 1638 + num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common, 1639 + tx_chn->id, budget, &tdown); 1560 1640 if (num_tx >= budget) 1561 1641 return budget; 1562 1642 ··· 1595 1685 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 1596 1686 struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc; 1597 1687 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 1688 + struct am65_cpsw_tx_swdata *swdata; 1598 1689 struct device *dev = common->dev; 1599 1690 struct am65_cpsw_tx_chn *tx_chn; 1600 1691 struct netdev_queue *netif_txq; 1601 1692 dma_addr_t desc_dma, buf_dma; 1602 1693 int ret, q_idx, i; 1603 - void **swdata; 1604 1694 u32 *psdata; 1605 1695 u32 pkt_len; 1606 1696 ··· 1646 1736 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); 1647 1737 cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len); 1648 1738 swdata = cppi5_hdesc_get_swdata(first_desc); 1649 - *(swdata) = skb; 1739 + swdata->ndev = ndev; 1740 + swdata->skb = skb; 1650 1741 psdata = cppi5_hdesc_get_psdata(first_desc); 1651 1742 1652 1743 /* HW csum offload if enabled */ ··· 3489 3578 __be64 id_temp; 3490 3579 int ret, i; 3491 3580 3581 + BUILD_BUG_ON_MSG(sizeof(struct am65_cpsw_tx_swdata) > AM65_CPSW_NAV_SW_DATA_SIZE, 3582 + "TX SW_DATA size exceeds AM65_CPSW_NAV_SW_DATA_SIZE"); 3583 + BUILD_BUG_ON_MSG(sizeof(struct am65_cpsw_swdata) > AM65_CPSW_NAV_SW_DATA_SIZE, 3584 + "SW_DATA size exceeds AM65_CPSW_NAV_SW_DATA_SIZE"); 3492 3585 common = devm_kzalloc(dev, sizeof(struct am65_cpsw_common), GFP_KERNEL); 3493 3586 if (!common) 3494 3587 return -ENOMEM;

drivers/net/ethernet/ti/am65-cpsw-nuss.h

··· 104 104 char name[32]; 105 105 }; 106 106 107 + struct am65_cpsw_tx_swdata { 108 + struct net_device *ndev; 109 + union { 110 + struct sk_buff *skb; 111 + struct xdp_frame *xdpf; 112 + }; 113 + }; 114 + 107 115 struct am65_cpsw_swdata { 108 116 u32 flow_id; 109 117 struct page *page;

Configure Feed

Configure Feed