tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

bpf, devmap: Move drop error path to devmap for XDP_REDIRECT

We want to change the current ndo_xdp_xmit drop semantics because it will
allow us to implement better queue overflow handling. This is working
towards the larger goal of a XDP TX queue-hook. Move XDP_REDIRECT error
path handling from each XDP ethernet driver to devmap code. According to
the new APIs, the driver running the ndo_xdp_xmit pointer, will break tx
loop whenever the hw reports a tx error and it will just return to devmap
caller the number of successfully transmitted frames. It will be devmap
responsibility to free dropped frames.

Move each XDP ndo_xdp_xmit capable driver to the new APIs:

- veth
- virtio-net
- mvneta
- mvpp2
- socionext
- amazon ena
- bnxt
- freescale (dpaa2, dpaa)
- xen-frontend
- qede
- ice
- igb
- ixgbe
- i40e
- mlx5
- ti (cpsw, cpsw-new)
- tun
- sfc

Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Reviewed-by: Ioana Ciornei <ioana.ciornei@nxp.com>
Reviewed-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
Reviewed-by: Camelia Groza <camelia.groza@nxp.com>
Acked-by: Edward Cree <ecree.xilinx@gmail.com>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Acked-by: Shay Agroskin <shayagr@amazon.com>
Link: https://lore.kernel.org/bpf/ed670de24f951cfd77590decf0229a0ad7fd12f6.1615201152.git.lorenzo@kernel.org

authored by

Lorenzo Bianconi and committed by
Daniel Borkmann fdc13979 6b282765

+153 -200
+9 -12
drivers/net/ethernet/amazon/ena/ena_netdev.c
··· 300 300 301 301 rc = ena_xdp_tx_map_frame(xdp_ring, tx_info, xdpf, &push_hdr, &push_len); 302 302 if (unlikely(rc)) 303 - goto error_drop_packet; 303 + return rc; 304 304 305 305 ena_tx_ctx.ena_bufs = tx_info->bufs; 306 306 ena_tx_ctx.push_header = push_hdr; ··· 330 330 error_unmap_dma: 331 331 ena_unmap_tx_buff(xdp_ring, tx_info); 332 332 tx_info->xdpf = NULL; 333 - error_drop_packet: 334 - xdp_return_frame(xdpf); 335 333 return rc; 336 334 } 337 335 ··· 337 339 struct xdp_frame **frames, u32 flags) 338 340 { 339 341 struct ena_adapter *adapter = netdev_priv(dev); 340 - int qid, i, err, drops = 0; 341 342 struct ena_ring *xdp_ring; 343 + int qid, i, nxmit = 0; 342 344 343 345 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 344 346 return -EINVAL; ··· 358 360 spin_lock(&xdp_ring->xdp_tx_lock); 359 361 360 362 for (i = 0; i < n; i++) { 361 - err = ena_xdp_xmit_frame(xdp_ring, dev, frames[i], 0); 362 - /* The descriptor is freed by ena_xdp_xmit_frame in case 363 - * of an error. 364 - */ 365 - if (err) 366 - drops++; 363 + if (ena_xdp_xmit_frame(xdp_ring, dev, frames[i], 0)) 364 + break; 365 + nxmit++; 367 366 } 368 367 369 368 /* Ring doorbell to make device aware of the packets */ ··· 373 378 spin_unlock(&xdp_ring->xdp_tx_lock); 374 379 375 380 /* Return number of packets sent */ 376 - return n - drops; 381 + return nxmit; 377 382 } 378 383 379 384 static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp) ··· 410 415 /* The XDP queues are shared between XDP_TX and XDP_REDIRECT */ 411 416 spin_lock(&xdp_ring->xdp_tx_lock); 412 417 413 - ena_xdp_xmit_frame(xdp_ring, rx_ring->netdev, xdpf, XDP_XMIT_FLUSH); 418 + if (ena_xdp_xmit_frame(xdp_ring, rx_ring->netdev, xdpf, 419 + XDP_XMIT_FLUSH)) 420 + xdp_return_frame(xdpf); 414 421 415 422 spin_unlock(&xdp_ring->xdp_tx_lock); 416 423 xdp_stat = &rx_ring->rx_stats.xdp_tx;
+8 -12
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
··· 217 217 struct pci_dev *pdev = bp->pdev; 218 218 struct bnxt_tx_ring_info *txr; 219 219 dma_addr_t mapping; 220 - int drops = 0; 220 + int nxmit = 0; 221 221 int ring; 222 222 int i; 223 223 ··· 233 233 struct xdp_frame *xdp = frames[i]; 234 234 235 235 if (!txr || !bnxt_tx_avail(bp, txr) || 236 - !(bp->bnapi[ring]->flags & BNXT_NAPI_FLAG_XDP)) { 237 - xdp_return_frame_rx_napi(xdp); 238 - drops++; 239 - continue; 240 - } 236 + !(bp->bnapi[ring]->flags & BNXT_NAPI_FLAG_XDP)) 237 + break; 241 238 242 239 mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len, 243 240 DMA_TO_DEVICE); 244 241 245 - if (dma_mapping_error(&pdev->dev, mapping)) { 246 - xdp_return_frame_rx_napi(xdp); 247 - drops++; 248 - continue; 249 - } 242 + if (dma_mapping_error(&pdev->dev, mapping)) 243 + break; 244 + 250 245 __bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp); 246 + nxmit++; 251 247 } 252 248 253 249 if (flags & XDP_XMIT_FLUSH) { ··· 252 256 bnxt_db_write(bp, &txr->tx_db, txr->tx_prod); 253 257 } 254 258 255 - return num_frames - drops; 259 + return nxmit; 256 260 } 257 261 258 262 /* Under rtnl_lock */
+5 -7
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
··· 3081 3081 struct xdp_frame **frames, u32 flags) 3082 3082 { 3083 3083 struct xdp_frame *xdpf; 3084 - int i, err, drops = 0; 3084 + int i, nxmit = 0; 3085 3085 3086 3086 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 3087 3087 return -EINVAL; ··· 3091 3091 3092 3092 for (i = 0; i < n; i++) { 3093 3093 xdpf = frames[i]; 3094 - err = dpaa_xdp_xmit_frame(net_dev, xdpf); 3095 - if (err) { 3096 - xdp_return_frame_rx_napi(xdpf); 3097 - drops++; 3098 - } 3094 + if (dpaa_xdp_xmit_frame(net_dev, xdpf)) 3095 + break; 3096 + nxmit++; 3099 3097 } 3100 3098 3101 - return n - drops; 3099 + return nxmit; 3102 3100 } 3103 3101 3104 3102 static int dpaa_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
-2
drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
··· 2431 2431 percpu_stats->tx_packets += enqueued; 2432 2432 for (i = 0; i < enqueued; i++) 2433 2433 percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 2434 - for (i = enqueued; i < n; i++) 2435 - xdp_return_frame_rx_napi(frames[i]); 2436 2434 2437 2435 return enqueued; 2438 2436 }
+7 -8
drivers/net/ethernet/intel/i40e/i40e_txrx.c
··· 3847 3847 * @frames: array of XDP buffer pointers 3848 3848 * @flags: XDP extra info 3849 3849 * 3850 - * Returns number of frames successfully sent. Frames that fail are 3851 - * free'ed via XDP return API. 3850 + * Returns number of frames successfully sent. Failed frames 3851 + * will be free'ed by XDP core. 3852 3852 * 3853 3853 * For error cases, a negative errno code is returned and no-frames 3854 3854 * are transmitted (caller must handle freeing frames). ··· 3861 3861 struct i40e_vsi *vsi = np->vsi; 3862 3862 struct i40e_pf *pf = vsi->back; 3863 3863 struct i40e_ring *xdp_ring; 3864 - int drops = 0; 3864 + int nxmit = 0; 3865 3865 int i; 3866 3866 3867 3867 if (test_bit(__I40E_VSI_DOWN, vsi->state)) ··· 3881 3881 int err; 3882 3882 3883 3883 err = i40e_xmit_xdp_ring(xdpf, xdp_ring); 3884 - if (err != I40E_XDP_TX) { 3885 - xdp_return_frame_rx_napi(xdpf); 3886 - drops++; 3887 - } 3884 + if (err != I40E_XDP_TX) 3885 + break; 3886 + nxmit++; 3888 3887 } 3889 3888 3890 3889 if (unlikely(flags & XDP_XMIT_FLUSH)) 3891 3890 i40e_xdp_ring_update_tail(xdp_ring); 3892 3891 3893 - return n - drops; 3892 + return nxmit; 3894 3893 }
+7 -8
drivers/net/ethernet/intel/ice/ice_txrx.c
··· 571 571 * @frames: XDP frames to be transmitted 572 572 * @flags: transmit flags 573 573 * 574 - * Returns number of frames successfully sent. Frames that fail are 575 - * free'ed via XDP return API. 574 + * Returns number of frames successfully sent. Failed frames 575 + * will be free'ed by XDP core. 576 576 * For error cases, a negative errno code is returned and no-frames 577 577 * are transmitted (caller must handle freeing frames). 578 578 */ ··· 584 584 unsigned int queue_index = smp_processor_id(); 585 585 struct ice_vsi *vsi = np->vsi; 586 586 struct ice_ring *xdp_ring; 587 - int drops = 0, i; 587 + int nxmit = 0, i; 588 588 589 589 if (test_bit(__ICE_DOWN, vsi->state)) 590 590 return -ENETDOWN; ··· 601 601 int err; 602 602 603 603 err = ice_xmit_xdp_ring(xdpf->data, xdpf->len, xdp_ring); 604 - if (err != ICE_XDP_TX) { 605 - xdp_return_frame_rx_napi(xdpf); 606 - drops++; 607 - } 604 + if (err != ICE_XDP_TX) 605 + break; 606 + nxmit++; 608 607 } 609 608 610 609 if (unlikely(flags & XDP_XMIT_FLUSH)) 611 610 ice_xdp_ring_update_tail(xdp_ring); 612 611 613 - return n - drops; 612 + return nxmit; 614 613 } 615 614 616 615 /**
+5 -6
drivers/net/ethernet/intel/igb/igb_main.c
··· 2934 2934 int cpu = smp_processor_id(); 2935 2935 struct igb_ring *tx_ring; 2936 2936 struct netdev_queue *nq; 2937 - int drops = 0; 2937 + int nxmit = 0; 2938 2938 int i; 2939 2939 2940 2940 if (unlikely(test_bit(__IGB_DOWN, &adapter->state))) ··· 2961 2961 int err; 2962 2962 2963 2963 err = igb_xmit_xdp_ring(adapter, tx_ring, xdpf); 2964 - if (err != IGB_XDP_TX) { 2965 - xdp_return_frame_rx_napi(xdpf); 2966 - drops++; 2967 - } 2964 + if (err != IGB_XDP_TX) 2965 + break; 2966 + nxmit++; 2968 2967 } 2969 2968 2970 2969 __netif_tx_unlock(nq); ··· 2971 2972 if (unlikely(flags & XDP_XMIT_FLUSH)) 2972 2973 igb_xdp_ring_update_tail(tx_ring); 2973 2974 2974 - return n - drops; 2975 + return nxmit; 2975 2976 } 2976 2977 2977 2978 static const struct net_device_ops igb_netdev_ops = {
+5 -6
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
··· 10188 10188 { 10189 10189 struct ixgbe_adapter *adapter = netdev_priv(dev); 10190 10190 struct ixgbe_ring *ring; 10191 - int drops = 0; 10191 + int nxmit = 0; 10192 10192 int i; 10193 10193 10194 10194 if (unlikely(test_bit(__IXGBE_DOWN, &adapter->state))) ··· 10212 10212 int err; 10213 10213 10214 10214 err = ixgbe_xmit_xdp_ring(adapter, xdpf); 10215 - if (err != IXGBE_XDP_TX) { 10216 - xdp_return_frame_rx_napi(xdpf); 10217 - drops++; 10218 - } 10215 + if (err != IXGBE_XDP_TX) 10216 + break; 10217 + nxmit++; 10219 10218 } 10220 10219 10221 10220 if (unlikely(flags & XDP_XMIT_FLUSH)) 10222 10221 ixgbe_xdp_ring_update_tail(ring); 10223 10222 10224 - return n - drops; 10223 + return nxmit; 10225 10224 } 10226 10225 10227 10226 static const struct net_device_ops ixgbe_netdev_ops = {
+6 -7
drivers/net/ethernet/marvell/mvneta.c
··· 2137 2137 { 2138 2138 struct mvneta_port *pp = netdev_priv(dev); 2139 2139 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); 2140 - int i, nxmit_byte = 0, nxmit = num_frame; 2140 + int i, nxmit_byte = 0, nxmit = 0; 2141 2141 int cpu = smp_processor_id(); 2142 2142 struct mvneta_tx_queue *txq; 2143 2143 struct netdev_queue *nq; ··· 2155 2155 __netif_tx_lock(nq, cpu); 2156 2156 for (i = 0; i < num_frame; i++) { 2157 2157 ret = mvneta_xdp_submit_frame(pp, txq, frames[i], true); 2158 - if (ret == MVNETA_XDP_TX) { 2159 - nxmit_byte += frames[i]->len; 2160 - } else { 2161 - xdp_return_frame_rx_napi(frames[i]); 2162 - nxmit--; 2163 - } 2158 + if (ret != MVNETA_XDP_TX) 2159 + break; 2160 + 2161 + nxmit_byte += frames[i]->len; 2162 + nxmit++; 2164 2163 } 2165 2164 2166 2165 if (unlikely(flags & XDP_XMIT_FLUSH))
+6 -7
drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
··· 3744 3744 struct xdp_frame **frames, u32 flags) 3745 3745 { 3746 3746 struct mvpp2_port *port = netdev_priv(dev); 3747 - int i, nxmit_byte = 0, nxmit = num_frame; 3747 + int i, nxmit_byte = 0, nxmit = 0; 3748 3748 struct mvpp2_pcpu_stats *stats; 3749 3749 u16 txq_id; 3750 3750 u32 ret; ··· 3762 3762 3763 3763 for (i = 0; i < num_frame; i++) { 3764 3764 ret = mvpp2_xdp_submit_frame(port, txq_id, frames[i], true); 3765 - if (ret == MVPP2_XDP_TX) { 3766 - nxmit_byte += frames[i]->len; 3767 - } else { 3768 - xdp_return_frame_rx_napi(frames[i]); 3769 - nxmit--; 3770 - } 3765 + if (ret != MVPP2_XDP_TX) 3766 + break; 3767 + 3768 + nxmit_byte += frames[i]->len; 3769 + nxmit++; 3771 3770 } 3772 3771 3773 3772 if (likely(nxmit > 0))
+6 -9
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
··· 500 500 { 501 501 struct mlx5e_priv *priv = netdev_priv(dev); 502 502 struct mlx5e_xdpsq *sq; 503 - int drops = 0; 503 + int nxmit = 0; 504 504 int sq_num; 505 505 int i; 506 506 ··· 529 529 xdptxd.dma_addr = dma_map_single(sq->pdev, xdptxd.data, 530 530 xdptxd.len, DMA_TO_DEVICE); 531 531 532 - if (unlikely(dma_mapping_error(sq->pdev, xdptxd.dma_addr))) { 533 - xdp_return_frame_rx_napi(xdpf); 534 - drops++; 535 - continue; 536 - } 532 + if (unlikely(dma_mapping_error(sq->pdev, xdptxd.dma_addr))) 533 + break; 537 534 538 535 xdpi.mode = MLX5E_XDP_XMIT_MODE_FRAME; 539 536 xdpi.frame.xdpf = xdpf; ··· 541 544 if (unlikely(!ret)) { 542 545 dma_unmap_single(sq->pdev, xdptxd.dma_addr, 543 546 xdptxd.len, DMA_TO_DEVICE); 544 - xdp_return_frame_rx_napi(xdpf); 545 - drops++; 547 + break; 546 548 } 549 + nxmit++; 547 550 } 548 551 549 552 if (flags & XDP_XMIT_FLUSH) { ··· 552 555 mlx5e_xmit_xdp_doorbell(sq); 553 556 } 554 557 555 - return n - drops; 558 + return nxmit; 556 559 } 557 560 558 561 void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq)
+7 -12
drivers/net/ethernet/qlogic/qede/qede_fp.c
··· 345 345 struct qede_tx_queue *xdp_tx; 346 346 struct xdp_frame *xdpf; 347 347 dma_addr_t mapping; 348 - int i, drops = 0; 348 + int i, nxmit = 0; 349 349 u16 xdp_prod; 350 350 351 351 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) ··· 364 364 365 365 mapping = dma_map_single(dmadev, xdpf->data, xdpf->len, 366 366 DMA_TO_DEVICE); 367 - if (unlikely(dma_mapping_error(dmadev, mapping))) { 368 - xdp_return_frame_rx_napi(xdpf); 369 - drops++; 370 - 371 - continue; 372 - } 367 + if (unlikely(dma_mapping_error(dmadev, mapping))) 368 + break; 373 369 374 370 if (unlikely(qede_xdp_xmit(xdp_tx, mapping, 0, xdpf->len, 375 - NULL, xdpf))) { 376 - xdp_return_frame_rx_napi(xdpf); 377 - drops++; 378 - } 371 + NULL, xdpf))) 372 + break; 373 + nxmit++; 379 374 } 380 375 381 376 if (flags & XDP_XMIT_FLUSH) { ··· 382 387 383 388 spin_unlock(&xdp_tx->xdp_tx_lock); 384 389 385 - return n_frames - drops; 390 + return nxmit; 386 391 } 387 392 388 393 int qede_txq_has_work(struct qede_tx_queue *txq)
+1 -14
drivers/net/ethernet/sfc/tx.c
··· 412 412 return NETDEV_TX_OK; 413 413 } 414 414 415 - static void efx_xdp_return_frames(int n, struct xdp_frame **xdpfs) 416 - { 417 - int i; 418 - 419 - for (i = 0; i < n; i++) 420 - xdp_return_frame_rx_napi(xdpfs[i]); 421 - } 422 - 423 415 /* Transmit a packet from an XDP buffer 424 416 * 425 417 * Returns number of packets sent on success, error code otherwise. ··· 484 492 if (flush && i > 0) 485 493 efx_nic_push_buffers(tx_queue); 486 494 487 - if (i == 0) 488 - return -EIO; 489 - 490 - efx_xdp_return_frames(n - i, xdpfs + i); 491 - 492 - return i; 495 + return i == 0 ? -EIO : i; 493 496 } 494 497 495 498 /* Initiate a packet transmission. We use one channel per CPU
+7 -9
drivers/net/ethernet/socionext/netsec.c
··· 1757 1757 { 1758 1758 struct netsec_priv *priv = netdev_priv(ndev); 1759 1759 struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX]; 1760 - int drops = 0; 1761 - int i; 1760 + int i, nxmit = 0; 1762 1761 1763 1762 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1764 1763 return -EINVAL; ··· 1768 1769 int err; 1769 1770 1770 1771 err = netsec_xdp_queue_one(priv, xdpf, true); 1771 - if (err != NETSEC_XDP_TX) { 1772 - xdp_return_frame_rx_napi(xdpf); 1773 - drops++; 1774 - } else { 1775 - tx_ring->xdp_xmit++; 1776 - } 1772 + if (err != NETSEC_XDP_TX) 1773 + break; 1774 + 1775 + tx_ring->xdp_xmit++; 1776 + nxmit++; 1777 1777 } 1778 1778 spin_unlock(&tx_ring->lock); 1779 1779 ··· 1781 1783 tx_ring->xdp_xmit = 0; 1782 1784 } 1783 1785 1784 - return n - drops; 1786 + return nxmit; 1785 1787 } 1786 1788 1787 1789 static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog,
+6 -8
drivers/net/ethernet/ti/cpsw.c
··· 1123 1123 struct cpsw_priv *priv = netdev_priv(ndev); 1124 1124 struct cpsw_common *cpsw = priv->cpsw; 1125 1125 struct xdp_frame *xdpf; 1126 - int i, drops = 0, port; 1126 + int i, nxmit = 0, port; 1127 1127 1128 1128 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1129 1129 return -EINVAL; 1130 1130 1131 1131 for (i = 0; i < n; i++) { 1132 1132 xdpf = frames[i]; 1133 - if (xdpf->len < CPSW_MIN_PACKET_SIZE) { 1134 - xdp_return_frame_rx_napi(xdpf); 1135 - drops++; 1136 - continue; 1137 - } 1133 + if (xdpf->len < CPSW_MIN_PACKET_SIZE) 1134 + break; 1138 1135 1139 1136 port = priv->emac_port + cpsw->data.dual_emac; 1140 1137 if (cpsw_xdp_tx_frame(priv, xdpf, NULL, port)) 1141 - drops++; 1138 + break; 1139 + nxmit++; 1142 1140 } 1143 1141 1144 - return n - drops; 1142 + return nxmit; 1145 1143 } 1146 1144 1147 1145 #ifdef CONFIG_NET_POLL_CONTROLLER
+6 -8
drivers/net/ethernet/ti/cpsw_new.c
··· 1093 1093 { 1094 1094 struct cpsw_priv *priv = netdev_priv(ndev); 1095 1095 struct xdp_frame *xdpf; 1096 - int i, drops = 0; 1096 + int i, nxmit = 0; 1097 1097 1098 1098 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1099 1099 return -EINVAL; 1100 1100 1101 1101 for (i = 0; i < n; i++) { 1102 1102 xdpf = frames[i]; 1103 - if (xdpf->len < CPSW_MIN_PACKET_SIZE) { 1104 - xdp_return_frame_rx_napi(xdpf); 1105 - drops++; 1106 - continue; 1107 - } 1103 + if (xdpf->len < CPSW_MIN_PACKET_SIZE) 1104 + break; 1108 1105 1109 1106 if (cpsw_xdp_tx_frame(priv, xdpf, NULL, priv->emac_port)) 1110 - drops++; 1107 + break; 1108 + nxmit++; 1111 1109 } 1112 1110 1113 - return n - drops; 1111 + return nxmit; 1114 1112 } 1115 1113 1116 1114 static int cpsw_get_port_parent_id(struct net_device *ndev,
+4 -7
drivers/net/ethernet/ti/cpsw_priv.c
··· 1305 1305 ret = cpdma_chan_submit_mapped(txch, cpsw_xdpf_to_handle(xdpf), 1306 1306 dma, xdpf->len, port); 1307 1307 } else { 1308 - if (sizeof(*xmeta) > xdpf->headroom) { 1309 - xdp_return_frame_rx_napi(xdpf); 1308 + if (sizeof(*xmeta) > xdpf->headroom) 1310 1309 return -EINVAL; 1311 - } 1312 1310 1313 1311 ret = cpdma_chan_submit(txch, cpsw_xdpf_to_handle(xdpf), 1314 1312 xdpf->data, xdpf->len, port); 1315 1313 } 1316 1314 1317 - if (ret) { 1315 + if (ret) 1318 1316 priv->ndev->stats.tx_dropped++; 1319 - xdp_return_frame_rx_napi(xdpf); 1320 - } 1321 1317 1322 1318 return ret; 1323 1319 } ··· 1349 1353 if (unlikely(!xdpf)) 1350 1354 goto drop; 1351 1355 1352 - cpsw_xdp_tx_frame(priv, xdpf, page, port); 1356 + if (cpsw_xdp_tx_frame(priv, xdpf, page, port)) 1357 + xdp_return_frame_rx_napi(xdpf); 1353 1358 break; 1354 1359 case XDP_REDIRECT: 1355 1360 if (xdp_do_redirect(ndev, xdp, prog))
+9 -6
drivers/net/tun.c
··· 1181 1181 struct tun_struct *tun = netdev_priv(dev); 1182 1182 struct tun_file *tfile; 1183 1183 u32 numqueues; 1184 - int drops = 0; 1185 - int cnt = n; 1184 + int nxmit = 0; 1186 1185 int i; 1187 1186 1188 1187 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) ··· 1211 1212 1212 1213 if (__ptr_ring_produce(&tfile->tx_ring, frame)) { 1213 1214 atomic_long_inc(&dev->tx_dropped); 1214 - xdp_return_frame_rx_napi(xdp); 1215 - drops++; 1215 + break; 1216 1216 } 1217 + nxmit++; 1217 1218 } 1218 1219 spin_unlock(&tfile->tx_ring.producer_lock); 1219 1220 ··· 1221 1222 __tun_xdp_flush_tfile(tfile); 1222 1223 1223 1224 rcu_read_unlock(); 1224 - return cnt - drops; 1225 + return nxmit; 1225 1226 } 1226 1227 1227 1228 static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp) 1228 1229 { 1229 1230 struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); 1231 + int nxmit; 1230 1232 1231 1233 if (unlikely(!frame)) 1232 1234 return -EOVERFLOW; 1233 1235 1234 - return tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH); 1236 + nxmit = tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH); 1237 + if (!nxmit) 1238 + xdp_return_frame_rx_napi(frame); 1239 + return nxmit; 1235 1240 } 1236 1241 1237 1242 static const struct net_device_ops tap_netdev_ops = {
+15 -13
drivers/net/veth.c
··· 434 434 u32 flags, bool ndo_xmit) 435 435 { 436 436 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 437 - int i, ret = -ENXIO, drops = 0; 437 + int i, ret = -ENXIO, nxmit = 0; 438 438 struct net_device *rcv; 439 439 unsigned int max_len; 440 440 struct veth_rq *rq; ··· 464 464 void *ptr = veth_xdp_to_ptr(frame); 465 465 466 466 if (unlikely(frame->len > max_len || 467 - __ptr_ring_produce(&rq->xdp_ring, ptr))) { 468 - xdp_return_frame_rx_napi(frame); 469 - drops++; 470 - } 467 + __ptr_ring_produce(&rq->xdp_ring, ptr))) 468 + break; 469 + nxmit++; 471 470 } 472 471 spin_unlock(&rq->xdp_ring.producer_lock); 473 472 474 473 if (flags & XDP_XMIT_FLUSH) 475 474 __veth_xdp_flush(rq); 476 475 477 - ret = n - drops; 476 + ret = nxmit; 478 477 if (ndo_xmit) { 479 478 u64_stats_update_begin(&rq->stats.syncp); 480 - rq->stats.vs.peer_tq_xdp_xmit += n - drops; 481 - rq->stats.vs.peer_tq_xdp_xmit_err += drops; 479 + rq->stats.vs.peer_tq_xdp_xmit += nxmit; 480 + rq->stats.vs.peer_tq_xdp_xmit_err += n - nxmit; 482 481 u64_stats_update_end(&rq->stats.syncp); 483 482 } 484 483 ··· 504 505 505 506 static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 506 507 { 507 - int sent, i, err = 0; 508 + int sent, i, err = 0, drops; 508 509 509 510 sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false); 510 511 if (sent < 0) { 511 512 err = sent; 512 513 sent = 0; 513 - for (i = 0; i < bq->count; i++) 514 - xdp_return_frame(bq->q[i]); 515 514 } 516 - trace_xdp_bulk_tx(rq->dev, sent, bq->count - sent, err); 515 + 516 + for (i = sent; unlikely(i < bq->count); i++) 517 + xdp_return_frame(bq->q[i]); 518 + 519 + drops = bq->count - sent; 520 + trace_xdp_bulk_tx(rq->dev, sent, drops, err); 517 521 518 522 u64_stats_update_begin(&rq->stats.syncp); 519 523 rq->stats.vs.xdp_tx += sent; 520 - rq->stats.vs.xdp_tx_err += bq->count - sent; 524 + rq->stats.vs.xdp_tx_err += drops; 521 525 u64_stats_update_end(&rq->stats.syncp); 522 526 523 527 bq->count = 0;
+13 -12
drivers/net/virtio_net.c
··· 499 499 unsigned int len; 500 500 int packets = 0; 501 501 int bytes = 0; 502 - int drops = 0; 502 + int nxmit = 0; 503 503 int kicks = 0; 504 - int ret, err; 505 504 void *ptr; 505 + int ret; 506 506 int i; 507 507 508 508 /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this ··· 516 516 517 517 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) { 518 518 ret = -EINVAL; 519 - drops = n; 520 519 goto out; 521 520 } 522 521 ··· 538 539 for (i = 0; i < n; i++) { 539 540 struct xdp_frame *xdpf = frames[i]; 540 541 541 - err = __virtnet_xdp_xmit_one(vi, sq, xdpf); 542 - if (err) { 543 - xdp_return_frame_rx_napi(xdpf); 544 - drops++; 545 - } 542 + if (__virtnet_xdp_xmit_one(vi, sq, xdpf)) 543 + break; 544 + nxmit++; 546 545 } 547 - ret = n - drops; 546 + ret = nxmit; 548 547 549 548 if (flags & XDP_XMIT_FLUSH) { 550 549 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) ··· 553 556 sq->stats.bytes += bytes; 554 557 sq->stats.packets += packets; 555 558 sq->stats.xdp_tx += n; 556 - sq->stats.xdp_tx_drops += drops; 559 + sq->stats.xdp_tx_drops += n - nxmit; 557 560 sq->stats.kicks += kicks; 558 561 u64_stats_update_end(&sq->stats.syncp); 559 562 ··· 706 709 if (unlikely(!xdpf)) 707 710 goto err_xdp; 708 711 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 709 - if (unlikely(err < 0)) { 712 + if (unlikely(!err)) { 713 + xdp_return_frame_rx_napi(xdpf); 714 + } else if (unlikely(err < 0)) { 710 715 trace_xdp_exception(vi->dev, xdp_prog, act); 711 716 goto err_xdp; 712 717 } ··· 895 896 if (unlikely(!xdpf)) 896 897 goto err_xdp; 897 898 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 898 - if (unlikely(err < 0)) { 899 + if (unlikely(!err)) { 900 + xdp_return_frame_rx_napi(xdpf); 901 + } else if (unlikely(err < 0)) { 899 902 trace_xdp_exception(vi->dev, xdp_prog, act); 900 903 if (unlikely(xdp_page != page)) 901 904 put_page(xdp_page);
+9 -9
drivers/net/xen-netfront.c
··· 608 608 struct netfront_info *np = netdev_priv(dev); 609 609 struct netfront_queue *queue = NULL; 610 610 unsigned long irq_flags; 611 - int drops = 0; 612 - int i, err; 611 + int nxmit = 0; 612 + int i; 613 613 614 614 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 615 615 return -EINVAL; ··· 622 622 623 623 if (!xdpf) 624 624 continue; 625 - err = xennet_xdp_xmit_one(dev, queue, xdpf); 626 - if (err) { 627 - xdp_return_frame_rx_napi(xdpf); 628 - drops++; 629 - } 625 + if (xennet_xdp_xmit_one(dev, queue, xdpf)) 626 + break; 627 + nxmit++; 630 628 } 631 629 spin_unlock_irqrestore(&queue->tx_lock, irq_flags); 632 630 633 - return n - drops; 631 + return nxmit; 634 632 } 635 633 636 634 ··· 873 875 get_page(pdata); 874 876 xdpf = xdp_convert_buff_to_frame(xdp); 875 877 err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0); 876 - if (unlikely(err < 0)) 878 + if (unlikely(!err)) 879 + xdp_return_frame_rx_napi(xdpf); 880 + else if (unlikely(err < 0)) 877 881 trace_xdp_exception(queue->info->netdev, prog, act); 878 882 break; 879 883 case XDP_REDIRECT:
+12 -18
kernel/bpf/devmap.c
··· 329 329 static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags) 330 330 { 331 331 struct net_device *dev = bq->dev; 332 - int sent = 0, drops = 0, err = 0; 332 + int sent = 0, err = 0; 333 333 int i; 334 334 335 335 if (unlikely(!bq->count)) ··· 343 343 344 344 sent = dev->netdev_ops->ndo_xdp_xmit(dev, bq->count, bq->q, flags); 345 345 if (sent < 0) { 346 + /* If ndo_xdp_xmit fails with an errno, no frames have 347 + * been xmit'ed. 348 + */ 346 349 err = sent; 347 350 sent = 0; 348 - goto error; 349 351 } 350 - drops = bq->count - sent; 351 - out: 352 - bq->count = 0; 353 352 354 - trace_xdp_devmap_xmit(bq->dev_rx, dev, sent, drops, err); 355 - bq->dev_rx = NULL; 356 - __list_del_clearprev(&bq->flush_node); 357 - return; 358 - error: 359 - /* If ndo_xdp_xmit fails with an errno, no frames have been 360 - * xmit'ed and it's our responsibility to them free all. 353 + /* If not all frames have been transmitted, it is our 354 + * responsibility to free them 361 355 */ 362 - for (i = 0; i < bq->count; i++) { 363 - struct xdp_frame *xdpf = bq->q[i]; 356 + for (i = sent; unlikely(i < bq->count); i++) 357 + xdp_return_frame_rx_napi(bq->q[i]); 364 358 365 - xdp_return_frame_rx_napi(xdpf); 366 - drops++; 367 - } 368 - goto out; 359 + trace_xdp_devmap_xmit(bq->dev_rx, dev, sent, bq->count - sent, err); 360 + bq->dev_rx = NULL; 361 + bq->count = 0; 362 + __list_del_clearprev(&bq->flush_node); 369 363 } 370 364 371 365 /* __dev_flush is called from xdp_do_flush() which _must_ be signaled