Merge branch '100GbE' of git://git.kernel.org/pub/scm/linux/kernel/git/tnguy/next-queue

+1

drivers/net/ethernet/intel/Kconfig

··· 296 296 depends on GNSS || GNSS = n 297 297 select AUXILIARY_BUS 298 298 select DIMLIB 299 + select LIBETH_XDP 299 300 select LIBIE 300 301 select LIBIE_ADMINQ 301 302 select LIBIE_FWLOG

+12

drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c

··· 4788 4788 unsigned long q_map; 4789 4789 struct i40e_vf *vf; 4790 4790 int abs_vf_id; 4791 + int old_link; 4791 4792 int ret = 0; 4792 4793 int tmp; 4793 4794 ··· 4806 4805 4807 4806 vf = &pf->vf[vf_id]; 4808 4807 abs_vf_id = vf->vf_id + hw->func_caps.vf_base_id; 4808 + 4809 + /* skip VF link state change if requested state is already set */ 4810 + if (!vf->link_forced) 4811 + old_link = IFLA_VF_LINK_STATE_AUTO; 4812 + else if (vf->link_up) 4813 + old_link = IFLA_VF_LINK_STATE_ENABLE; 4814 + else 4815 + old_link = IFLA_VF_LINK_STATE_DISABLE; 4816 + 4817 + if (link == old_link) 4818 + goto error_out; 4809 4819 4810 4820 pfe.event = VIRTCHNL_EVENT_LINK_CHANGE; 4811 4821 pfe.severity = PF_EVENT_SEVERITY_INFO;

+3 -1

drivers/net/ethernet/intel/ice/ice.h

··· 351 351 u16 num_q_vectors; 352 352 /* tell if only dynamic irq allocation is allowed */ 353 353 bool irq_dyn_alloc; 354 + bool hsplit:1; 354 355 355 356 u16 vsi_num; /* HW (absolute) index of this VSI */ 356 357 u16 idx; /* software index in pf->vsi[] */ ··· 374 373 struct ice_arfs_active_fltr_cntrs *arfs_fltr_cntrs; 375 374 spinlock_t arfs_lock; /* protects aRFS hash table and filter state */ 376 375 atomic_t *arfs_last_fltr_id; 376 + 377 + u16 max_frame; 377 378 378 379 struct ice_aqc_vsi_props info; /* VSI properties */ 379 380 struct ice_vsi_vlan_info vlan_info; /* vlan config to be restored */ ··· 512 509 ICE_FLAG_MOD_POWER_UNSUPPORTED, 513 510 ICE_FLAG_PHY_FW_LOAD_FAILED, 514 511 ICE_FLAG_ETHTOOL_CTXT, /* set when ethtool holds RTNL lock */ 515 - ICE_FLAG_LEGACY_RX, 516 512 ICE_FLAG_VF_TRUE_PROMISC_ENA, 517 513 ICE_FLAG_MDD_AUTO_RESET_VF, 518 514 ICE_FLAG_VF_VLAN_PRUNING,

+93 -75

drivers/net/ethernet/intel/ice/ice_base.c

··· 2 2 /* Copyright (c) 2019, Intel Corporation. */ 3 3 4 4 #include <net/xdp_sock_drv.h> 5 + #include <linux/net/intel/libie/rx.h> 5 6 #include "ice_base.h" 6 7 #include "ice_lib.h" 7 8 #include "ice_dcb_lib.h" ··· 463 462 } 464 463 465 464 /** 466 - * ice_rx_offset - Return expected offset into page to access data 467 - * @rx_ring: Ring we are requesting offset of 468 - * 469 - * Returns the offset value for ring into the data buffer. 470 - */ 471 - static unsigned int ice_rx_offset(struct ice_rx_ring *rx_ring) 472 - { 473 - if (ice_ring_uses_build_skb(rx_ring)) 474 - return ICE_SKB_PAD; 475 - return 0; 476 - } 477 - 478 - /** 479 465 * ice_setup_rx_ctx - Configure a receive ring context 480 466 * @ring: The Rx ring to configure 481 467 * ··· 524 536 else 525 537 rlan_ctx.l2tsel = 1; 526 538 527 - rlan_ctx.dtype = ICE_RX_DTYPE_NO_SPLIT; 528 - rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_NO_SPLIT; 539 + if (ring->hdr_pp) { 540 + rlan_ctx.hbuf = ring->rx_hdr_len >> ICE_RLAN_CTX_HBUF_S; 541 + rlan_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; 542 + 543 + /* 544 + * If the frame is TCP/UDP/SCTP, it will be split by the 545 + * payload. 546 + * If not, but it's an IPv4/IPv6 frame, it will be split by 547 + * the IP header. 548 + * If not IP, it will be split by the Ethernet header. 549 + * 550 + * In any case, the header buffer will never be left empty. 551 + */ 552 + rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_L2 | 553 + ICE_RLAN_RX_HSPLIT_0_SPLIT_IP | 554 + ICE_RLAN_RX_HSPLIT_0_SPLIT_TCP_UDP | 555 + ICE_RLAN_RX_HSPLIT_0_SPLIT_SCTP; 556 + } else { 557 + rlan_ctx.hbuf = 0; 558 + rlan_ctx.dtype = ICE_RX_DTYPE_NO_SPLIT; 559 + rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_NO_SPLIT; 560 + } 561 + 529 562 rlan_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_NO_SPLIT; 530 563 531 564 /* This controls whether VLAN is stripped from inner headers ··· 558 549 /* Max packet size for this queue - must not be set to a larger value 559 550 * than 5 x DBUF 560 551 */ 561 - rlan_ctx.rxmax = min_t(u32, ring->max_frame, 552 + rlan_ctx.rxmax = min_t(u32, vsi->max_frame, 562 553 ICE_MAX_CHAINED_RX_BUFS * ring->rx_buf_len); 563 554 564 555 /* Rx queue threshold in units of 64 */ ··· 595 586 if (vsi->type == ICE_VSI_VF) 596 587 return 0; 597 588 598 - /* configure Rx buffer alignment */ 599 - if (!vsi->netdev || test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags)) 600 - ice_clear_ring_build_skb_ena(ring); 601 - else 602 - ice_set_ring_build_skb_ena(ring); 603 - 604 - ring->rx_offset = ice_rx_offset(ring); 605 - 606 589 /* init queue specific tail register */ 607 590 ring->tail = hw->hw_addr + QRX_TAIL(pf_q); 608 591 writel(0, ring->tail); ··· 602 601 return 0; 603 602 } 604 603 605 - static void ice_xsk_pool_fill_cb(struct ice_rx_ring *ring) 604 + static int ice_rxq_pp_create(struct ice_rx_ring *rq) 606 605 { 607 - void *ctx_ptr = &ring->pkt_ctx; 608 - struct xsk_cb_desc desc = {}; 606 + struct libeth_fq fq = { 607 + .count = rq->count, 608 + .nid = NUMA_NO_NODE, 609 + .hsplit = rq->vsi->hsplit, 610 + .xdp = ice_is_xdp_ena_vsi(rq->vsi), 611 + .buf_len = LIBIE_MAX_RX_BUF_LEN, 612 + }; 613 + int err; 609 614 610 - XSK_CHECK_PRIV_TYPE(struct ice_xdp_buff); 611 - desc.src = &ctx_ptr; 612 - desc.off = offsetof(struct ice_xdp_buff, pkt_ctx) - 613 - sizeof(struct xdp_buff); 614 - desc.bytes = sizeof(ctx_ptr); 615 - xsk_pool_fill_cb(ring->xsk_pool, &desc); 616 - } 615 + err = libeth_rx_fq_create(&fq, &rq->q_vector->napi); 616 + if (err) 617 + return err; 617 618 618 - /** 619 - * ice_get_frame_sz - calculate xdp_buff::frame_sz 620 - * @rx_ring: the ring being configured 621 - * 622 - * Return frame size based on underlying PAGE_SIZE 623 - */ 624 - static unsigned int ice_get_frame_sz(struct ice_rx_ring *rx_ring) 625 - { 626 - unsigned int frame_sz; 619 + rq->pp = fq.pp; 620 + rq->rx_fqes = fq.fqes; 621 + rq->truesize = fq.truesize; 622 + rq->rx_buf_len = fq.buf_len; 627 623 628 - #if (PAGE_SIZE >= 8192) 629 - frame_sz = rx_ring->rx_buf_len; 630 - #else 631 - frame_sz = ice_rx_pg_size(rx_ring) / 2; 632 - #endif 624 + if (!fq.hsplit) 625 + return 0; 633 626 634 - return frame_sz; 627 + fq = (struct libeth_fq){ 628 + .count = rq->count, 629 + .type = LIBETH_FQE_HDR, 630 + .nid = NUMA_NO_NODE, 631 + .xdp = ice_is_xdp_ena_vsi(rq->vsi), 632 + }; 633 + 634 + err = libeth_rx_fq_create(&fq, &rq->q_vector->napi); 635 + if (err) 636 + goto destroy; 637 + 638 + rq->hdr_pp = fq.pp; 639 + rq->hdr_fqes = fq.fqes; 640 + rq->hdr_truesize = fq.truesize; 641 + rq->rx_hdr_len = fq.buf_len; 642 + 643 + return 0; 644 + 645 + destroy: 646 + ice_rxq_pp_destroy(rq); 647 + 648 + return err; 635 649 } 636 650 637 651 /** ··· 658 642 static int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) 659 643 { 660 644 struct device *dev = ice_pf_to_dev(ring->vsi->back); 661 - u32 num_bufs = ICE_RX_DESC_UNUSED(ring); 645 + u32 num_bufs = ICE_DESC_UNUSED(ring); 646 + u32 rx_buf_len; 662 647 int err; 663 648 664 649 if (ring->vsi->type == ICE_VSI_PF || ring->vsi->type == ICE_VSI_SF) { ··· 673 656 } 674 657 675 658 ice_rx_xsk_pool(ring); 659 + err = ice_realloc_rx_xdp_bufs(ring, ring->xsk_pool); 660 + if (err) 661 + return err; 662 + 676 663 if (ring->xsk_pool) { 677 664 xdp_rxq_info_unreg(&ring->xdp_rxq); 678 665 679 - ring->rx_buf_len = 666 + rx_buf_len = 680 667 xsk_pool_get_rx_frame_size(ring->xsk_pool); 681 668 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 682 669 ring->q_index, 683 670 ring->q_vector->napi.napi_id, 684 - ring->rx_buf_len); 671 + rx_buf_len); 685 672 if (err) 686 673 return err; 687 674 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, ··· 694 673 if (err) 695 674 return err; 696 675 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); 697 - ice_xsk_pool_fill_cb(ring); 698 676 699 677 dev_info(dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", 700 678 ring->q_index); 701 679 } else { 680 + err = ice_rxq_pp_create(ring); 681 + if (err) 682 + return err; 683 + 702 684 if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { 703 685 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 704 686 ring->q_index, 705 687 ring->q_vector->napi.napi_id, 706 688 ring->rx_buf_len); 707 689 if (err) 708 - return err; 690 + goto err_destroy_fq; 709 691 } 710 - 711 - err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 712 - MEM_TYPE_PAGE_SHARED, 713 - NULL); 714 - if (err) 715 - return err; 692 + xdp_rxq_info_attach_page_pool(&ring->xdp_rxq, 693 + ring->pp); 716 694 } 717 695 } 718 696 719 - xdp_init_buff(&ring->xdp, ice_get_frame_sz(ring), &ring->xdp_rxq); 720 697 ring->xdp.data = NULL; 721 - ring->xdp_ext.pkt_ctx = &ring->pkt_ctx; 722 698 err = ice_setup_rx_ctx(ring); 723 699 if (err) { 724 700 dev_err(dev, "ice_setup_rx_ctx failed for RxQ %d, err %d\n", 725 701 ring->q_index, err); 726 - return err; 702 + goto err_destroy_fq; 727 703 } 728 704 729 705 if (ring->xsk_pool) { ··· 748 730 if (ring->vsi->type == ICE_VSI_CTRL) 749 731 ice_init_ctrl_rx_descs(ring, num_bufs); 750 732 else 751 - ice_alloc_rx_bufs(ring, num_bufs); 733 + err = ice_alloc_rx_bufs(ring, num_bufs); 734 + 735 + if (err) 736 + goto err_destroy_fq; 752 737 753 738 return 0; 739 + 740 + err_destroy_fq: 741 + ice_rxq_pp_destroy(ring); 742 + 743 + return err; 754 744 } 755 745 756 746 int ice_vsi_cfg_single_rxq(struct ice_vsi *vsi, u16 q_idx) ··· 779 753 */ 780 754 static void ice_vsi_cfg_frame_size(struct ice_vsi *vsi, struct ice_rx_ring *ring) 781 755 { 782 - if (!vsi->netdev || test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags)) { 783 - ring->max_frame = ICE_MAX_FRAME_LEGACY_RX; 784 - ring->rx_buf_len = ICE_RXBUF_1664; 785 - #if (PAGE_SIZE < 8192) 786 - } else if (!ICE_2K_TOO_SMALL_WITH_PADDING && 787 - (vsi->netdev->mtu <= ETH_DATA_LEN)) { 788 - ring->max_frame = ICE_RXBUF_1536 - NET_IP_ALIGN; 789 - ring->rx_buf_len = ICE_RXBUF_1536 - NET_IP_ALIGN; 790 - #endif 756 + if (!vsi->netdev) { 757 + vsi->max_frame = ICE_MAX_FRAME_LEGACY_RX; 791 758 } else { 792 - ring->max_frame = ICE_AQ_SET_MAC_FRAME_SIZE_MAX; 793 - ring->rx_buf_len = ICE_RXBUF_3072; 759 + vsi->max_frame = ICE_AQ_SET_MAC_FRAME_SIZE_MAX; 794 760 } 795 761 } 796 762

+1

drivers/net/ethernet/intel/ice/ice_common.c

··· 3392 3392 case ICE_DEV_ID_E822L_SGMII: 3393 3393 case ICE_DEV_ID_E823L_1GBE: 3394 3394 case ICE_DEV_ID_E823C_SGMII: 3395 + case ICE_DEV_ID_E825C_SGMII: 3395 3396 return true; 3396 3397 default: 3397 3398 return false;

+23 -14

drivers/net/ethernet/intel/ice/ice_ethtool.c

··· 10 10 #include "ice_lib.h" 11 11 #include "ice_dcb_lib.h" 12 12 #include <net/dcbnl.h> 13 + #include <net/libeth/rx.h> 13 14 14 15 struct ice_stats { 15 16 char stat_string[ETH_GSTRING_LEN]; ··· 341 340 ICE_FLAG_VF_TRUE_PROMISC_ENA), 342 341 ICE_PRIV_FLAG("mdd-auto-reset-vf", ICE_FLAG_MDD_AUTO_RESET_VF), 343 342 ICE_PRIV_FLAG("vf-vlan-pruning", ICE_FLAG_VF_VLAN_PRUNING), 344 - ICE_PRIV_FLAG("legacy-rx", ICE_FLAG_LEGACY_RX), 345 343 }; 346 344 347 345 #define ICE_PRIV_FLAG_ARRAY_SIZE ARRAY_SIZE(ice_gstrings_priv_flags) ··· 1231 1231 */ 1232 1232 static int ice_lbtest_receive_frames(struct ice_rx_ring *rx_ring) 1233 1233 { 1234 - struct ice_rx_buf *rx_buf; 1234 + struct libeth_fqe *rx_buf; 1235 1235 int valid_frames, i; 1236 + struct page *page; 1236 1237 u8 *received_buf; 1237 1238 1238 1239 valid_frames = 0; ··· 1248 1247 cpu_to_le16(BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S))))) 1249 1248 continue; 1250 1249 1251 - rx_buf = &rx_ring->rx_buf[i]; 1252 - received_buf = page_address(rx_buf->page) + rx_buf->page_offset; 1250 + rx_buf = &rx_ring->rx_fqes[i]; 1251 + page = __netmem_to_page(rx_buf->netmem); 1252 + received_buf = page_address(page) + rx_buf->offset + 1253 + page->pp->p.offset; 1253 1254 1254 1255 if (ice_lbtest_check_frame(received_buf)) 1255 1256 valid_frames++; ··· 1858 1855 1859 1856 ice_nway_reset(netdev); 1860 1857 } 1861 - } 1862 - if (test_bit(ICE_FLAG_LEGACY_RX, change_flags)) { 1863 - /* down and up VSI so that changes of Rx cfg are reflected. */ 1864 - ice_down_up(vsi); 1865 1858 } 1866 1859 /* don't allow modification of this flag when a single VF is in 1867 1860 * promiscuous mode because it's not supported ··· 3151 3152 ring->rx_jumbo_max_pending = 0; 3152 3153 ring->rx_mini_pending = 0; 3153 3154 ring->rx_jumbo_pending = 0; 3155 + 3156 + kernel_ring->tcp_data_split = vsi->hsplit ? 3157 + ETHTOOL_TCP_DATA_SPLIT_ENABLED : 3158 + ETHTOOL_TCP_DATA_SPLIT_DISABLED; 3154 3159 } 3155 3160 3156 3161 static int ··· 3171 3168 int i, timeout = 50, err = 0; 3172 3169 struct ice_hw *hw = &pf->hw; 3173 3170 u16 new_rx_cnt, new_tx_cnt; 3171 + bool hsplit; 3174 3172 3175 3173 if (ring->tx_pending > ICE_MAX_NUM_DESC_BY_MAC(hw) || 3176 3174 ring->tx_pending < ICE_MIN_NUM_DESC || ··· 3197 3193 netdev_info(netdev, "Requested Rx descriptor count rounded up to %d\n", 3198 3194 new_rx_cnt); 3199 3195 3196 + hsplit = kernel_ring->tcp_data_split == ETHTOOL_TCP_DATA_SPLIT_ENABLED; 3197 + 3200 3198 /* if nothing to do return success */ 3201 3199 if (new_tx_cnt == vsi->tx_rings[0]->count && 3202 - new_rx_cnt == vsi->rx_rings[0]->count) { 3200 + new_rx_cnt == vsi->rx_rings[0]->count && 3201 + hsplit == vsi->hsplit) { 3203 3202 netdev_dbg(netdev, "Nothing to change, descriptor count is same as requested\n"); 3204 3203 return 0; 3205 3204 } ··· 3232 3225 vsi->xdp_rings[i]->count = new_tx_cnt; 3233 3226 vsi->num_tx_desc = (u16)new_tx_cnt; 3234 3227 vsi->num_rx_desc = (u16)new_rx_cnt; 3228 + vsi->hsplit = hsplit; 3229 + 3235 3230 netdev_dbg(netdev, "Link is down, descriptor count change happens when link is brought up\n"); 3236 3231 goto done; 3237 3232 } ··· 3317 3308 rx_rings[i].count = new_rx_cnt; 3318 3309 rx_rings[i].cached_phctime = pf->ptp.cached_phc_time; 3319 3310 rx_rings[i].desc = NULL; 3320 - rx_rings[i].rx_buf = NULL; 3311 + rx_rings[i].xdp_buf = NULL; 3312 + 3321 3313 /* this is to allow wr32 to have something to write to 3322 3314 * during early allocation of Rx buffers 3323 3315 */ ··· 3327 3317 err = ice_setup_rx_ring(&rx_rings[i]); 3328 3318 if (err) 3329 3319 goto rx_unwind; 3330 - 3331 - /* allocate Rx buffers */ 3332 - err = ice_alloc_rx_bufs(&rx_rings[i], 3333 - ICE_RX_DESC_UNUSED(&rx_rings[i])); 3334 3320 rx_unwind: 3335 3321 if (err) { 3336 3322 while (i) { ··· 3340 3334 } 3341 3335 3342 3336 process_link: 3337 + vsi->hsplit = hsplit; 3338 + 3343 3339 /* Bring interface down, copy in the new ring info, then restore the 3344 3340 * interface. if VSI is up, bring it down and then back up 3345 3341 */ ··· 4823 4815 ETHTOOL_COALESCE_USE_ADAPTIVE | 4824 4816 ETHTOOL_COALESCE_RX_USECS_HIGH, 4825 4817 .supported_input_xfrm = RXH_XFRM_SYM_XOR, 4818 + .supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT, 4826 4819 .get_link_ksettings = ice_get_link_ksettings, 4827 4820 .set_link_ksettings = ice_set_link_ksettings, 4828 4821 .get_fec_stats = ice_get_fec_stats,

+3

drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h

··· 342 342 /* for ice_32byte_rx_flex_desc.pkt_length member */ 343 343 #define ICE_RX_FLX_DESC_PKT_LEN_M (0x3FFF) /* 14-bits */ 344 344 345 + /* ice_32byte_rx_flex_desc::hdr_len_sph_flex_flags1 */ 346 + #define ICE_RX_FLEX_DESC_HDR_LEN_M GENMASK(10, 0) 347 + 345 348 enum ice_rx_flex_desc_status_error_0_bits { 346 349 /* Note: These are predefined bit offsets */ 347 350 ICE_RX_FLEX_DESC_STATUS0_DD_S = 0,

-1

drivers/net/ethernet/intel/ice/ice_lib.c

··· 1427 1427 ring->reg_idx = vsi->rxq_map[i]; 1428 1428 ring->vsi = vsi; 1429 1429 ring->netdev = vsi->netdev; 1430 - ring->dev = dev; 1431 1430 ring->count = vsi->num_rx_desc; 1432 1431 ring->cached_phctime = pf->ptp.cached_phc_time; 1433 1432

+3 -18

drivers/net/ethernet/intel/ice/ice_main.c

··· 37 37 #define ICE_DDP_PKG_FILE ICE_DDP_PKG_PATH "ice.pkg" 38 38 39 39 MODULE_DESCRIPTION(DRV_SUMMARY); 40 + MODULE_IMPORT_NS("LIBETH"); 41 + MODULE_IMPORT_NS("LIBETH_XDP"); 40 42 MODULE_IMPORT_NS("LIBIE"); 41 43 MODULE_IMPORT_NS("LIBIE_ADMINQ"); 42 44 MODULE_IMPORT_NS("LIBIE_FWLOG"); ··· 2959 2957 */ 2960 2958 static int ice_max_xdp_frame_size(struct ice_vsi *vsi) 2961 2959 { 2962 - if (test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags)) 2963 - return ICE_RXBUF_1664; 2964 - else 2965 - return ICE_RXBUF_3072; 2960 + return ICE_RXBUF_3072; 2966 2961 } 2967 2962 2968 2963 /** ··· 3017 3018 } 3018 3019 } 3019 3020 xdp_features_set_redirect_target(vsi->netdev, true); 3020 - /* reallocate Rx queues that are used for zero-copy */ 3021 - xdp_ring_err = ice_realloc_zc_buf(vsi, true); 3022 - if (xdp_ring_err) 3023 - NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Rx resources failed"); 3024 3021 } else if (ice_is_xdp_ena_vsi(vsi) && !prog) { 3025 3022 xdp_features_clear_redirect_target(vsi->netdev); 3026 3023 xdp_ring_err = ice_destroy_xdp_rings(vsi, ICE_XDP_CFG_FULL); 3027 3024 if (xdp_ring_err) 3028 3025 NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed"); 3029 - /* reallocate Rx queues that were used for zero-copy */ 3030 - xdp_ring_err = ice_realloc_zc_buf(vsi, false); 3031 - if (xdp_ring_err) 3032 - NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Rx resources failed"); 3033 3026 } 3034 3027 3035 3028 resume_if: ··· 7853 7862 if (new_mtu + ICE_ETH_PKT_HDR_PAD > frame_size) { 7854 7863 netdev_err(netdev, "max MTU for XDP usage is %d\n", 7855 7864 frame_size - ICE_ETH_PKT_HDR_PAD); 7856 - return -EINVAL; 7857 - } 7858 - } else if (test_bit(ICE_FLAG_LEGACY_RX, pf->flags)) { 7859 - if (new_mtu + ICE_ETH_PKT_HDR_PAD > ICE_MAX_FRAME_LEGACY_RX) { 7860 - netdev_err(netdev, "Too big MTU for legacy-rx; Max is %d\n", 7861 - ICE_MAX_FRAME_LEGACY_RX - ICE_ETH_PKT_HDR_PAD); 7862 7865 return -EINVAL; 7863 7866 } 7864 7867 }

+136 -574

drivers/net/ethernet/intel/ice/ice_txrx.c

··· 7 7 #include <linux/netdevice.h> 8 8 #include <linux/prefetch.h> 9 9 #include <linux/bpf_trace.h> 10 + #include <linux/net/intel/libie/rx.h> 11 + #include <net/libeth/xdp.h> 10 12 #include <net/dsfield.h> 11 13 #include <net/mpls.h> 12 14 #include <net/xdp.h> ··· 113 111 static void 114 112 ice_unmap_and_free_tx_buf(struct ice_tx_ring *ring, struct ice_tx_buf *tx_buf) 115 113 { 116 - if (dma_unmap_len(tx_buf, len)) 114 + if (tx_buf->type != ICE_TX_BUF_XDP_TX && dma_unmap_len(tx_buf, len)) 117 115 dma_unmap_page(ring->dev, 118 116 dma_unmap_addr(tx_buf, dma), 119 117 dma_unmap_len(tx_buf, len), ··· 127 125 dev_kfree_skb_any(tx_buf->skb); 128 126 break; 129 127 case ICE_TX_BUF_XDP_TX: 130 - page_frag_free(tx_buf->raw_buf); 128 + libeth_xdp_return_va(tx_buf->raw_buf, false); 131 129 break; 132 130 case ICE_TX_BUF_XDP_XMIT: 133 131 xdp_return_frame(tx_buf->xdpf); ··· 508 506 return -ENOMEM; 509 507 } 510 508 509 + void ice_rxq_pp_destroy(struct ice_rx_ring *rq) 510 + { 511 + struct libeth_fq fq = { 512 + .fqes = rq->rx_fqes, 513 + .pp = rq->pp, 514 + }; 515 + 516 + libeth_rx_fq_destroy(&fq); 517 + rq->rx_fqes = NULL; 518 + rq->pp = NULL; 519 + 520 + if (!rq->hdr_pp) 521 + return; 522 + 523 + fq.fqes = rq->hdr_fqes; 524 + fq.pp = rq->hdr_pp; 525 + 526 + libeth_rx_fq_destroy(&fq); 527 + rq->hdr_fqes = NULL; 528 + rq->hdr_pp = NULL; 529 + } 530 + 511 531 /** 512 532 * ice_clean_rx_ring - Free Rx buffers 513 533 * @rx_ring: ring to be cleaned 514 534 */ 515 535 void ice_clean_rx_ring(struct ice_rx_ring *rx_ring) 516 536 { 517 - struct xdp_buff *xdp = &rx_ring->xdp; 518 - struct device *dev = rx_ring->dev; 519 537 u32 size; 520 - u16 i; 521 - 522 - /* ring already cleared, nothing to do */ 523 - if (!rx_ring->rx_buf) 524 - return; 525 538 526 539 if (rx_ring->xsk_pool) { 527 540 ice_xsk_clean_rx_ring(rx_ring); 528 541 goto rx_skip_free; 529 542 } 530 543 531 - if (xdp->data) { 532 - xdp_return_buff(xdp); 533 - xdp->data = NULL; 534 - } 544 + /* ring already cleared, nothing to do */ 545 + if (!rx_ring->rx_fqes) 546 + return; 547 + 548 + libeth_xdp_return_stash(&rx_ring->xdp); 535 549 536 550 /* Free all the Rx ring sk_buffs */ 537 - for (i = 0; i < rx_ring->count; i++) { 538 - struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i]; 551 + for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { 552 + libeth_rx_recycle_slow(rx_ring->rx_fqes[i].netmem); 539 553 540 - if (!rx_buf->page) 541 - continue; 554 + if (rx_ring->hdr_pp) 555 + libeth_rx_recycle_slow(rx_ring->hdr_fqes[i].netmem); 542 556 543 - /* Invalidate cache lines that may have been written to by 544 - * device so that we avoid corrupting memory. 545 - */ 546 - dma_sync_single_range_for_cpu(dev, rx_buf->dma, 547 - rx_buf->page_offset, 548 - rx_ring->rx_buf_len, 549 - DMA_FROM_DEVICE); 550 - 551 - /* free resources associated with mapping */ 552 - dma_unmap_page_attrs(dev, rx_buf->dma, ice_rx_pg_size(rx_ring), 553 - DMA_FROM_DEVICE, ICE_RX_DMA_ATTR); 554 - __page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias); 555 - 556 - rx_buf->page = NULL; 557 - rx_buf->page_offset = 0; 557 + if (unlikely(++i == rx_ring->count)) 558 + i = 0; 558 559 } 559 560 560 - rx_skip_free: 561 - if (rx_ring->xsk_pool) 562 - memset(rx_ring->xdp_buf, 0, array_size(rx_ring->count, sizeof(*rx_ring->xdp_buf))); 563 - else 564 - memset(rx_ring->rx_buf, 0, array_size(rx_ring->count, sizeof(*rx_ring->rx_buf))); 561 + if (rx_ring->vsi->type == ICE_VSI_PF && 562 + xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) { 563 + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); 564 + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 565 + } 565 566 567 + ice_rxq_pp_destroy(rx_ring); 568 + 569 + rx_skip_free: 566 570 /* Zero out the descriptor ring */ 567 571 size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc), 568 572 PAGE_SIZE); ··· 576 568 577 569 rx_ring->next_to_alloc = 0; 578 570 rx_ring->next_to_clean = 0; 579 - rx_ring->first_desc = 0; 580 571 rx_ring->next_to_use = 0; 581 572 } 582 573 ··· 587 580 */ 588 581 void ice_free_rx_ring(struct ice_rx_ring *rx_ring) 589 582 { 583 + struct device *dev = ice_pf_to_dev(rx_ring->vsi->back); 590 584 u32 size; 591 585 592 586 ice_clean_rx_ring(rx_ring); 593 - if (rx_ring->vsi->type == ICE_VSI_PF) 594 - if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) 595 - xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 596 587 WRITE_ONCE(rx_ring->xdp_prog, NULL); 597 588 if (rx_ring->xsk_pool) { 598 589 kfree(rx_ring->xdp_buf); 599 590 rx_ring->xdp_buf = NULL; 600 - } else { 601 - kfree(rx_ring->rx_buf); 602 - rx_ring->rx_buf = NULL; 603 591 } 604 592 605 593 if (rx_ring->desc) { 606 594 size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc), 607 595 PAGE_SIZE); 608 - dmam_free_coherent(rx_ring->dev, size, 609 - rx_ring->desc, rx_ring->dma); 596 + dmam_free_coherent(dev, size, rx_ring->desc, rx_ring->dma); 610 597 rx_ring->desc = NULL; 611 598 } 612 599 } ··· 613 612 */ 614 613 int ice_setup_rx_ring(struct ice_rx_ring *rx_ring) 615 614 { 616 - struct device *dev = rx_ring->dev; 615 + struct device *dev = ice_pf_to_dev(rx_ring->vsi->back); 617 616 u32 size; 618 - 619 - if (!dev) 620 - return -ENOMEM; 621 - 622 - /* warn if we are about to overwrite the pointer */ 623 - WARN_ON(rx_ring->rx_buf); 624 - rx_ring->rx_buf = 625 - kcalloc(rx_ring->count, sizeof(*rx_ring->rx_buf), GFP_KERNEL); 626 - if (!rx_ring->rx_buf) 627 - return -ENOMEM; 628 617 629 618 /* round up to nearest page */ 630 619 size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc), ··· 624 633 if (!rx_ring->desc) { 625 634 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", 626 635 size); 627 - goto err; 636 + return -ENOMEM; 628 637 } 629 638 630 639 rx_ring->next_to_use = 0; 631 640 rx_ring->next_to_clean = 0; 632 - rx_ring->first_desc = 0; 633 641 634 642 if (ice_is_xdp_ena_vsi(rx_ring->vsi)) 635 643 WRITE_ONCE(rx_ring->xdp_prog, rx_ring->vsi->xdp_prog); 636 644 637 645 return 0; 638 - 639 - err: 640 - kfree(rx_ring->rx_buf); 641 - rx_ring->rx_buf = NULL; 642 - return -ENOMEM; 643 646 } 644 647 645 648 /** ··· 647 662 * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR} 648 663 */ 649 664 static u32 650 - ice_run_xdp(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, 665 + ice_run_xdp(struct ice_rx_ring *rx_ring, struct libeth_xdp_buff *xdp, 651 666 struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring, 652 667 union ice_32b_rx_flex_desc *eop_desc) 653 668 { ··· 657 672 if (!xdp_prog) 658 673 goto exit; 659 674 660 - ice_xdp_meta_set_desc(xdp, eop_desc); 675 + xdp->desc = eop_desc; 661 676 662 - act = bpf_prog_run_xdp(xdp_prog, xdp); 677 + act = bpf_prog_run_xdp(xdp_prog, &xdp->base); 663 678 switch (act) { 664 679 case XDP_PASS: 665 680 break; 666 681 case XDP_TX: 667 682 if (static_branch_unlikely(&ice_xdp_locking_key)) 668 683 spin_lock(&xdp_ring->tx_lock); 669 - ret = __ice_xmit_xdp_ring(xdp, xdp_ring, false); 684 + ret = __ice_xmit_xdp_ring(&xdp->base, xdp_ring, false); 670 685 if (static_branch_unlikely(&ice_xdp_locking_key)) 671 686 spin_unlock(&xdp_ring->tx_lock); 672 687 if (ret == ICE_XDP_CONSUMED) 673 688 goto out_failure; 674 689 break; 675 690 case XDP_REDIRECT: 676 - if (xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog)) 691 + if (xdp_do_redirect(rx_ring->netdev, &xdp->base, xdp_prog)) 677 692 goto out_failure; 678 693 ret = ICE_XDP_REDIR; 679 694 break; ··· 685 700 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); 686 701 fallthrough; 687 702 case XDP_DROP: 703 + libeth_xdp_return_buff(xdp); 688 704 ret = ICE_XDP_CONSUMED; 689 705 } 706 + 690 707 exit: 691 708 return ret; 692 709 } ··· 777 790 } 778 791 779 792 /** 780 - * ice_alloc_mapped_page - recycle or make a new page 781 - * @rx_ring: ring to use 782 - * @bi: rx_buf struct to modify 783 - * 784 - * Returns true if the page was successfully allocated or 785 - * reused. 786 - */ 787 - static bool 788 - ice_alloc_mapped_page(struct ice_rx_ring *rx_ring, struct ice_rx_buf *bi) 789 - { 790 - struct page *page = bi->page; 791 - dma_addr_t dma; 792 - 793 - /* since we are recycling buffers we should seldom need to alloc */ 794 - if (likely(page)) 795 - return true; 796 - 797 - /* alloc new page for storage */ 798 - page = dev_alloc_pages(ice_rx_pg_order(rx_ring)); 799 - if (unlikely(!page)) { 800 - rx_ring->ring_stats->rx_stats.alloc_page_failed++; 801 - return false; 802 - } 803 - 804 - /* map page for use */ 805 - dma = dma_map_page_attrs(rx_ring->dev, page, 0, ice_rx_pg_size(rx_ring), 806 - DMA_FROM_DEVICE, ICE_RX_DMA_ATTR); 807 - 808 - /* if mapping failed free memory back to system since 809 - * there isn't much point in holding memory we can't use 810 - */ 811 - if (dma_mapping_error(rx_ring->dev, dma)) { 812 - __free_pages(page, ice_rx_pg_order(rx_ring)); 813 - rx_ring->ring_stats->rx_stats.alloc_page_failed++; 814 - return false; 815 - } 816 - 817 - bi->dma = dma; 818 - bi->page = page; 819 - bi->page_offset = rx_ring->rx_offset; 820 - page_ref_add(page, USHRT_MAX - 1); 821 - bi->pagecnt_bias = USHRT_MAX; 822 - 823 - return true; 824 - } 825 - 826 - /** 827 793 * ice_init_ctrl_rx_descs - Initialize Rx descriptors for control vsi. 828 794 * @rx_ring: ring to init descriptors on 829 795 * @count: number of descriptors to initialize ··· 822 882 */ 823 883 bool ice_alloc_rx_bufs(struct ice_rx_ring *rx_ring, unsigned int cleaned_count) 824 884 { 885 + const struct libeth_fq_fp hdr_fq = { 886 + .pp = rx_ring->hdr_pp, 887 + .fqes = rx_ring->hdr_fqes, 888 + .truesize = rx_ring->hdr_truesize, 889 + .count = rx_ring->count, 890 + }; 891 + const struct libeth_fq_fp fq = { 892 + .pp = rx_ring->pp, 893 + .fqes = rx_ring->rx_fqes, 894 + .truesize = rx_ring->truesize, 895 + .count = rx_ring->count, 896 + }; 825 897 union ice_32b_rx_flex_desc *rx_desc; 826 898 u16 ntu = rx_ring->next_to_use; 827 - struct ice_rx_buf *bi; 828 899 829 900 /* do nothing if no valid netdev defined */ 830 901 if (!rx_ring->netdev || !cleaned_count) ··· 843 892 844 893 /* get the Rx descriptor and buffer based on next_to_use */ 845 894 rx_desc = ICE_RX_DESC(rx_ring, ntu); 846 - bi = &rx_ring->rx_buf[ntu]; 847 895 848 896 do { 849 - /* if we fail here, we have work remaining */ 850 - if (!ice_alloc_mapped_page(rx_ring, bi)) 851 - break; 897 + dma_addr_t addr; 852 898 853 - /* sync the buffer for use by the device */ 854 - dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 855 - bi->page_offset, 856 - rx_ring->rx_buf_len, 857 - DMA_FROM_DEVICE); 899 + addr = libeth_rx_alloc(&fq, ntu); 900 + if (addr == DMA_MAPPING_ERROR) { 901 + rx_ring->ring_stats->rx_stats.alloc_page_failed++; 902 + break; 903 + } 858 904 859 905 /* Refresh the desc even if buffer_addrs didn't change 860 906 * because each write-back erases this info. 861 907 */ 862 - rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 908 + rx_desc->read.pkt_addr = cpu_to_le64(addr); 863 909 910 + if (!hdr_fq.pp) 911 + goto next; 912 + 913 + addr = libeth_rx_alloc(&hdr_fq, ntu); 914 + if (addr == DMA_MAPPING_ERROR) { 915 + rx_ring->ring_stats->rx_stats.alloc_page_failed++; 916 + 917 + libeth_rx_recycle_slow(fq.fqes[ntu].netmem); 918 + break; 919 + } 920 + 921 + rx_desc->read.hdr_addr = cpu_to_le64(addr); 922 + 923 + next: 864 924 rx_desc++; 865 - bi++; 866 925 ntu++; 867 926 if (unlikely(ntu == rx_ring->count)) { 868 927 rx_desc = ICE_RX_DESC(rx_ring, 0); 869 - bi = rx_ring->rx_buf; 870 928 ntu = 0; 871 929 } 872 930 ··· 889 929 ice_release_rx_desc(rx_ring, ntu); 890 930 891 931 return !!cleaned_count; 892 - } 893 - 894 - /** 895 - * ice_rx_buf_adjust_pg_offset - Prepare Rx buffer for reuse 896 - * @rx_buf: Rx buffer to adjust 897 - * @size: Size of adjustment 898 - * 899 - * Update the offset within page so that Rx buf will be ready to be reused. 900 - * For systems with PAGE_SIZE < 8192 this function will flip the page offset 901 - * so the second half of page assigned to Rx buffer will be used, otherwise 902 - * the offset is moved by "size" bytes 903 - */ 904 - static void 905 - ice_rx_buf_adjust_pg_offset(struct ice_rx_buf *rx_buf, unsigned int size) 906 - { 907 - #if (PAGE_SIZE < 8192) 908 - /* flip page offset to other buffer */ 909 - rx_buf->page_offset ^= size; 910 - #else 911 - /* move offset up to the next cache line */ 912 - rx_buf->page_offset += size; 913 - #endif 914 - } 915 - 916 - /** 917 - * ice_can_reuse_rx_page - Determine if page can be reused for another Rx 918 - * @rx_buf: buffer containing the page 919 - * 920 - * If page is reusable, we have a green light for calling ice_reuse_rx_page, 921 - * which will assign the current buffer to the buffer that next_to_alloc is 922 - * pointing to; otherwise, the DMA mapping needs to be destroyed and 923 - * page freed 924 - */ 925 - static bool 926 - ice_can_reuse_rx_page(struct ice_rx_buf *rx_buf) 927 - { 928 - unsigned int pagecnt_bias = rx_buf->pagecnt_bias; 929 - struct page *page = rx_buf->page; 930 - 931 - /* avoid re-using remote and pfmemalloc pages */ 932 - if (!dev_page_is_reusable(page)) 933 - return false; 934 - 935 - /* if we are only owner of page we can reuse it */ 936 - if (unlikely(rx_buf->pgcnt - pagecnt_bias > 1)) 937 - return false; 938 - #if (PAGE_SIZE >= 8192) 939 - #define ICE_LAST_OFFSET \ 940 - (SKB_WITH_OVERHEAD(PAGE_SIZE) - ICE_RXBUF_3072) 941 - if (rx_buf->page_offset > ICE_LAST_OFFSET) 942 - return false; 943 - #endif /* PAGE_SIZE >= 8192) */ 944 - 945 - /* If we have drained the page fragment pool we need to update 946 - * the pagecnt_bias and page count so that we fully restock the 947 - * number of references the driver holds. 948 - */ 949 - if (unlikely(pagecnt_bias == 1)) { 950 - page_ref_add(page, USHRT_MAX - 1); 951 - rx_buf->pagecnt_bias = USHRT_MAX; 952 - } 953 - 954 - return true; 955 - } 956 - 957 - /** 958 - * ice_add_xdp_frag - Add contents of Rx buffer to xdp buf as a frag 959 - * @rx_ring: Rx descriptor ring to transact packets on 960 - * @xdp: xdp buff to place the data into 961 - * @rx_buf: buffer containing page to add 962 - * @size: packet length from rx_desc 963 - * 964 - * This function will add the data contained in rx_buf->page to the xdp buf. 965 - * It will just attach the page as a frag. 966 - */ 967 - static int 968 - ice_add_xdp_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, 969 - struct ice_rx_buf *rx_buf, const unsigned int size) 970 - { 971 - struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 972 - 973 - if (!size) 974 - return 0; 975 - 976 - if (!xdp_buff_has_frags(xdp)) { 977 - sinfo->nr_frags = 0; 978 - sinfo->xdp_frags_size = 0; 979 - xdp_buff_set_frags_flag(xdp); 980 - } 981 - 982 - if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) 983 - return -ENOMEM; 984 - 985 - __skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, rx_buf->page, 986 - rx_buf->page_offset, size); 987 - sinfo->xdp_frags_size += size; 988 - 989 - if (page_is_pfmemalloc(rx_buf->page)) 990 - xdp_buff_set_frag_pfmemalloc(xdp); 991 - 992 - return 0; 993 - } 994 - 995 - /** 996 - * ice_reuse_rx_page - page flip buffer and store it back on the ring 997 - * @rx_ring: Rx descriptor ring to store buffers on 998 - * @old_buf: donor buffer to have page reused 999 - * 1000 - * Synchronizes page for reuse by the adapter 1001 - */ 1002 - static void 1003 - ice_reuse_rx_page(struct ice_rx_ring *rx_ring, struct ice_rx_buf *old_buf) 1004 - { 1005 - u16 nta = rx_ring->next_to_alloc; 1006 - struct ice_rx_buf *new_buf; 1007 - 1008 - new_buf = &rx_ring->rx_buf[nta]; 1009 - 1010 - /* update, and store next to alloc */ 1011 - nta++; 1012 - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1013 - 1014 - /* Transfer page from old buffer to new buffer. 1015 - * Move each member individually to avoid possible store 1016 - * forwarding stalls and unnecessary copy of skb. 1017 - */ 1018 - new_buf->dma = old_buf->dma; 1019 - new_buf->page = old_buf->page; 1020 - new_buf->page_offset = old_buf->page_offset; 1021 - new_buf->pagecnt_bias = old_buf->pagecnt_bias; 1022 - } 1023 - 1024 - /** 1025 - * ice_get_rx_buf - Fetch Rx buffer and synchronize data for use 1026 - * @rx_ring: Rx descriptor ring to transact packets on 1027 - * @size: size of buffer to add to skb 1028 - * @ntc: index of next to clean element 1029 - * 1030 - * This function will pull an Rx buffer from the ring and synchronize it 1031 - * for use by the CPU. 1032 - */ 1033 - static struct ice_rx_buf * 1034 - ice_get_rx_buf(struct ice_rx_ring *rx_ring, const unsigned int size, 1035 - const unsigned int ntc) 1036 - { 1037 - struct ice_rx_buf *rx_buf; 1038 - 1039 - rx_buf = &rx_ring->rx_buf[ntc]; 1040 - prefetchw(rx_buf->page); 1041 - 1042 - if (!size) 1043 - return rx_buf; 1044 - /* we are reusing so sync this buffer for CPU use */ 1045 - dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma, 1046 - rx_buf->page_offset, size, 1047 - DMA_FROM_DEVICE); 1048 - 1049 - /* We have pulled a buffer for use, so decrement pagecnt_bias */ 1050 - rx_buf->pagecnt_bias--; 1051 - 1052 - return rx_buf; 1053 - } 1054 - 1055 - /** 1056 - * ice_get_pgcnts - grab page_count() for gathered fragments 1057 - * @rx_ring: Rx descriptor ring to store the page counts on 1058 - * @ntc: the next to clean element (not included in this frame!) 1059 - * 1060 - * This function is intended to be called right before running XDP 1061 - * program so that the page recycling mechanism will be able to take 1062 - * a correct decision regarding underlying pages; this is done in such 1063 - * way as XDP program can change the refcount of page 1064 - */ 1065 - static void ice_get_pgcnts(struct ice_rx_ring *rx_ring, unsigned int ntc) 1066 - { 1067 - u32 idx = rx_ring->first_desc; 1068 - struct ice_rx_buf *rx_buf; 1069 - u32 cnt = rx_ring->count; 1070 - 1071 - while (idx != ntc) { 1072 - rx_buf = &rx_ring->rx_buf[idx]; 1073 - rx_buf->pgcnt = page_count(rx_buf->page); 1074 - 1075 - if (++idx == cnt) 1076 - idx = 0; 1077 - } 1078 - } 1079 - 1080 - /** 1081 - * ice_build_skb - Build skb around an existing buffer 1082 - * @rx_ring: Rx descriptor ring to transact packets on 1083 - * @xdp: xdp_buff pointing to the data 1084 - * 1085 - * This function builds an skb around an existing XDP buffer, taking care 1086 - * to set up the skb correctly and avoid any memcpy overhead. Driver has 1087 - * already combined frags (if any) to skb_shared_info. 1088 - */ 1089 - static struct sk_buff * 1090 - ice_build_skb(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) 1091 - { 1092 - u8 metasize = xdp->data - xdp->data_meta; 1093 - struct skb_shared_info *sinfo = NULL; 1094 - unsigned int nr_frags; 1095 - struct sk_buff *skb; 1096 - 1097 - if (unlikely(xdp_buff_has_frags(xdp))) { 1098 - sinfo = xdp_get_shared_info_from_buff(xdp); 1099 - nr_frags = sinfo->nr_frags; 1100 - } 1101 - 1102 - /* Prefetch first cache line of first page. If xdp->data_meta 1103 - * is unused, this points exactly as xdp->data, otherwise we 1104 - * likely have a consumer accessing first few bytes of meta 1105 - * data, and then actual data. 1106 - */ 1107 - net_prefetch(xdp->data_meta); 1108 - /* build an skb around the page buffer */ 1109 - skb = napi_build_skb(xdp->data_hard_start, xdp->frame_sz); 1110 - if (unlikely(!skb)) 1111 - return NULL; 1112 - 1113 - /* must to record Rx queue, otherwise OS features such as 1114 - * symmetric queue won't work 1115 - */ 1116 - skb_record_rx_queue(skb, rx_ring->q_index); 1117 - 1118 - /* update pointers within the skb to store the data */ 1119 - skb_reserve(skb, xdp->data - xdp->data_hard_start); 1120 - __skb_put(skb, xdp->data_end - xdp->data); 1121 - if (metasize) 1122 - skb_metadata_set(skb, metasize); 1123 - 1124 - if (unlikely(xdp_buff_has_frags(xdp))) 1125 - xdp_update_skb_frags_info(skb, nr_frags, sinfo->xdp_frags_size, 1126 - nr_frags * xdp->frame_sz, 1127 - xdp_buff_get_skb_flags(xdp)); 1128 - 1129 - return skb; 1130 - } 1131 - 1132 - /** 1133 - * ice_construct_skb - Allocate skb and populate it 1134 - * @rx_ring: Rx descriptor ring to transact packets on 1135 - * @xdp: xdp_buff pointing to the data 1136 - * 1137 - * This function allocates an skb. It then populates it with the page 1138 - * data from the current receive descriptor, taking care to set up the 1139 - * skb correctly. 1140 - */ 1141 - static struct sk_buff * 1142 - ice_construct_skb(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) 1143 - { 1144 - unsigned int size = xdp->data_end - xdp->data; 1145 - struct skb_shared_info *sinfo = NULL; 1146 - struct ice_rx_buf *rx_buf; 1147 - unsigned int nr_frags = 0; 1148 - unsigned int headlen; 1149 - struct sk_buff *skb; 1150 - 1151 - /* prefetch first cache line of first page */ 1152 - net_prefetch(xdp->data); 1153 - 1154 - if (unlikely(xdp_buff_has_frags(xdp))) { 1155 - sinfo = xdp_get_shared_info_from_buff(xdp); 1156 - nr_frags = sinfo->nr_frags; 1157 - } 1158 - 1159 - /* allocate a skb to store the frags */ 1160 - skb = napi_alloc_skb(&rx_ring->q_vector->napi, ICE_RX_HDR_SIZE); 1161 - if (unlikely(!skb)) 1162 - return NULL; 1163 - 1164 - rx_buf = &rx_ring->rx_buf[rx_ring->first_desc]; 1165 - skb_record_rx_queue(skb, rx_ring->q_index); 1166 - /* Determine available headroom for copy */ 1167 - headlen = size; 1168 - if (headlen > ICE_RX_HDR_SIZE) 1169 - headlen = eth_get_headlen(skb->dev, xdp->data, ICE_RX_HDR_SIZE); 1170 - 1171 - /* align pull length to size of long to optimize memcpy performance */ 1172 - memcpy(__skb_put(skb, headlen), xdp->data, ALIGN(headlen, 1173 - sizeof(long))); 1174 - 1175 - /* if we exhaust the linear part then add what is left as a frag */ 1176 - size -= headlen; 1177 - if (size) { 1178 - /* besides adding here a partial frag, we are going to add 1179 - * frags from xdp_buff, make sure there is enough space for 1180 - * them 1181 - */ 1182 - if (unlikely(nr_frags >= MAX_SKB_FRAGS - 1)) { 1183 - dev_kfree_skb(skb); 1184 - return NULL; 1185 - } 1186 - skb_add_rx_frag(skb, 0, rx_buf->page, 1187 - rx_buf->page_offset + headlen, size, 1188 - xdp->frame_sz); 1189 - } else { 1190 - /* buffer is unused, restore biased page count in Rx buffer; 1191 - * data was copied onto skb's linear part so there's no 1192 - * need for adjusting page offset and we can reuse this buffer 1193 - * as-is 1194 - */ 1195 - rx_buf->pagecnt_bias++; 1196 - } 1197 - 1198 - if (unlikely(xdp_buff_has_frags(xdp))) { 1199 - struct skb_shared_info *skinfo = skb_shinfo(skb); 1200 - 1201 - memcpy(&skinfo->frags[skinfo->nr_frags], &sinfo->frags[0], 1202 - sizeof(skb_frag_t) * nr_frags); 1203 - 1204 - xdp_update_skb_frags_info(skb, skinfo->nr_frags + nr_frags, 1205 - sinfo->xdp_frags_size, 1206 - nr_frags * xdp->frame_sz, 1207 - xdp_buff_get_skb_flags(xdp)); 1208 - } 1209 - 1210 - return skb; 1211 - } 1212 - 1213 - /** 1214 - * ice_put_rx_buf - Clean up used buffer and either recycle or free 1215 - * @rx_ring: Rx descriptor ring to transact packets on 1216 - * @rx_buf: Rx buffer to pull data from 1217 - * 1218 - * This function will clean up the contents of the rx_buf. It will either 1219 - * recycle the buffer or unmap it and free the associated resources. 1220 - */ 1221 - static void 1222 - ice_put_rx_buf(struct ice_rx_ring *rx_ring, struct ice_rx_buf *rx_buf) 1223 - { 1224 - if (!rx_buf) 1225 - return; 1226 - 1227 - if (ice_can_reuse_rx_page(rx_buf)) { 1228 - /* hand second half of page back to the ring */ 1229 - ice_reuse_rx_page(rx_ring, rx_buf); 1230 - } else { 1231 - /* we are not reusing the buffer so unmap it */ 1232 - dma_unmap_page_attrs(rx_ring->dev, rx_buf->dma, 1233 - ice_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1234 - ICE_RX_DMA_ATTR); 1235 - __page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias); 1236 - } 1237 - 1238 - /* clear contents of buffer_info */ 1239 - rx_buf->page = NULL; 1240 - } 1241 - 1242 - /** 1243 - * ice_put_rx_mbuf - ice_put_rx_buf() caller, for all buffers in frame 1244 - * @rx_ring: Rx ring with all the auxiliary data 1245 - * @xdp: XDP buffer carrying linear + frags part 1246 - * @ntc: the next to clean element (not included in this frame!) 1247 - * @verdict: return code from XDP program execution 1248 - * 1249 - * Called after XDP program is completed, or on error with verdict set to 1250 - * ICE_XDP_CONSUMED. 1251 - * 1252 - * Walk through buffers from first_desc to the end of the frame, releasing 1253 - * buffers and satisfying internal page recycle mechanism. The action depends 1254 - * on verdict from XDP program. 1255 - */ 1256 - static void ice_put_rx_mbuf(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, 1257 - u32 ntc, u32 verdict) 1258 - { 1259 - u32 idx = rx_ring->first_desc; 1260 - u32 cnt = rx_ring->count; 1261 - struct ice_rx_buf *buf; 1262 - u32 xdp_frags = 0; 1263 - int i = 0; 1264 - 1265 - if (unlikely(xdp_buff_has_frags(xdp))) 1266 - xdp_frags = xdp_get_shared_info_from_buff(xdp)->nr_frags; 1267 - 1268 - while (idx != ntc) { 1269 - buf = &rx_ring->rx_buf[idx]; 1270 - if (++idx == cnt) 1271 - idx = 0; 1272 - 1273 - /* An XDP program could release fragments from the end of the 1274 - * buffer. For these, we need to keep the pagecnt_bias as-is. 1275 - * To do this, only adjust pagecnt_bias for fragments up to 1276 - * the total remaining after the XDP program has run. 1277 - */ 1278 - if (verdict != ICE_XDP_CONSUMED) 1279 - ice_rx_buf_adjust_pg_offset(buf, xdp->frame_sz); 1280 - else if (i++ <= xdp_frags) 1281 - buf->pagecnt_bias++; 1282 - 1283 - ice_put_rx_buf(rx_ring, buf); 1284 - } 1285 - 1286 - xdp->data = NULL; 1287 - rx_ring->first_desc = ntc; 1288 932 } 1289 933 1290 934 /** ··· 925 1361 total_rx_pkts++; 926 1362 } 927 1363 928 - rx_ring->first_desc = ntc; 929 1364 rx_ring->next_to_clean = ntc; 930 - ice_init_ctrl_rx_descs(rx_ring, ICE_RX_DESC_UNUSED(rx_ring)); 1365 + ice_init_ctrl_rx_descs(rx_ring, ICE_DESC_UNUSED(rx_ring)); 931 1366 } 932 1367 933 1368 /** ··· 944 1381 static int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget) 945 1382 { 946 1383 unsigned int total_rx_bytes = 0, total_rx_pkts = 0; 947 - unsigned int offset = rx_ring->rx_offset; 948 - struct xdp_buff *xdp = &rx_ring->xdp; 949 1384 struct ice_tx_ring *xdp_ring = NULL; 950 1385 struct bpf_prog *xdp_prog = NULL; 951 1386 u32 ntc = rx_ring->next_to_clean; 1387 + LIBETH_XDP_ONSTACK_BUFF(xdp); 952 1388 u32 cached_ntu, xdp_verdict; 953 1389 u32 cnt = rx_ring->count; 954 1390 u32 xdp_xmit = 0; 955 1391 bool failure; 1392 + 1393 + libeth_xdp_init_buff(xdp, &rx_ring->xdp, &rx_ring->xdp_rxq); 956 1394 957 1395 xdp_prog = READ_ONCE(rx_ring->xdp_prog); 958 1396 if (xdp_prog) { ··· 964 1400 /* start the loop to process Rx packets bounded by 'budget' */ 965 1401 while (likely(total_rx_pkts < (unsigned int)budget)) { 966 1402 union ice_32b_rx_flex_desc *rx_desc; 967 - struct ice_rx_buf *rx_buf; 1403 + struct libeth_fqe *rx_buf; 968 1404 struct sk_buff *skb; 969 1405 unsigned int size; 970 1406 u16 stat_err_bits; 971 1407 u16 vlan_tci; 1408 + bool rxe; 972 1409 973 1410 /* get the Rx desc from Rx ring based on 'next_to_clean' */ 974 1411 rx_desc = ICE_RX_DESC(rx_ring, ntc); 975 1412 976 - /* status_error_len will always be zero for unused descriptors 977 - * because it's cleared in cleanup, and overlaps with hdr_addr 978 - * which is always zero because packet split isn't used, if the 979 - * hardware wrote DD then it will be non-zero 1413 + /* 1414 + * The DD bit will always be zero for unused descriptors 1415 + * because it's cleared in cleanup or when setting the DMA 1416 + * address of the header buffer, which never uses the DD bit. 1417 + * If the hardware wrote the descriptor, it will be non-zero. 980 1418 */ 981 1419 stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S); 982 1420 if (!ice_test_staterr(rx_desc->wb.status_error0, stat_err_bits)) ··· 992 1426 993 1427 ice_trace(clean_rx_irq, rx_ring, rx_desc); 994 1428 1429 + stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_HBO_S) | 1430 + BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S); 1431 + rxe = ice_test_staterr(rx_desc->wb.status_error0, 1432 + stat_err_bits); 1433 + 1434 + if (!rx_ring->hdr_pp) 1435 + goto payload; 1436 + 1437 + size = le16_get_bits(rx_desc->wb.hdr_len_sph_flex_flags1, 1438 + ICE_RX_FLEX_DESC_HDR_LEN_M); 1439 + if (unlikely(rxe)) 1440 + size = 0; 1441 + 1442 + rx_buf = &rx_ring->hdr_fqes[ntc]; 1443 + libeth_xdp_process_buff(xdp, rx_buf, size); 1444 + rx_buf->netmem = 0; 1445 + 1446 + payload: 995 1447 size = le16_to_cpu(rx_desc->wb.pkt_len) & 996 1448 ICE_RX_FLX_DESC_PKT_LEN_M; 1449 + if (unlikely(rxe)) 1450 + size = 0; 997 1451 998 1452 /* retrieve a buffer from the ring */ 999 - rx_buf = ice_get_rx_buf(rx_ring, size, ntc); 1453 + rx_buf = &rx_ring->rx_fqes[ntc]; 1454 + libeth_xdp_process_buff(xdp, rx_buf, size); 1000 1455 1001 - /* Increment ntc before calls to ice_put_rx_mbuf() */ 1002 1456 if (++ntc == cnt) 1003 1457 ntc = 0; 1004 1458 1005 - if (!xdp->data) { 1006 - void *hard_start; 1007 - 1008 - hard_start = page_address(rx_buf->page) + rx_buf->page_offset - 1009 - offset; 1010 - xdp_prepare_buff(xdp, hard_start, offset, size, !!offset); 1011 - xdp_buff_clear_frags_flag(xdp); 1012 - } else if (ice_add_xdp_frag(rx_ring, xdp, rx_buf, size)) { 1013 - ice_put_rx_mbuf(rx_ring, xdp, ntc, ICE_XDP_CONSUMED); 1014 - break; 1015 - } 1016 - 1017 1459 /* skip if it is NOP desc */ 1018 - if (ice_is_non_eop(rx_ring, rx_desc)) 1460 + if (ice_is_non_eop(rx_ring, rx_desc) || unlikely(!xdp->data)) 1019 1461 continue; 1020 1462 1021 - ice_get_pgcnts(rx_ring, ntc); 1022 1463 xdp_verdict = ice_run_xdp(rx_ring, xdp, xdp_prog, xdp_ring, rx_desc); 1023 1464 if (xdp_verdict == ICE_XDP_PASS) 1024 1465 goto construct_skb; 1025 - total_rx_bytes += xdp_get_buff_len(xdp); 1466 + 1467 + if (xdp_verdict & (ICE_XDP_TX | ICE_XDP_REDIR)) 1468 + xdp_xmit |= xdp_verdict; 1469 + total_rx_bytes += xdp_get_buff_len(&xdp->base); 1026 1470 total_rx_pkts++; 1027 1471 1028 - ice_put_rx_mbuf(rx_ring, xdp, ntc, xdp_verdict); 1029 - xdp_xmit |= xdp_verdict & (ICE_XDP_TX | ICE_XDP_REDIR); 1030 - 1472 + xdp->data = NULL; 1031 1473 continue; 1474 + 1032 1475 construct_skb: 1033 - if (likely(ice_ring_uses_build_skb(rx_ring))) 1034 - skb = ice_build_skb(rx_ring, xdp); 1035 - else 1036 - skb = ice_construct_skb(rx_ring, xdp); 1476 + skb = xdp_build_skb_from_buff(&xdp->base); 1477 + xdp->data = NULL; 1478 + 1037 1479 /* exit if we failed to retrieve a buffer */ 1038 1480 if (!skb) { 1481 + libeth_xdp_return_buff_slow(xdp); 1039 1482 rx_ring->ring_stats->rx_stats.alloc_buf_failed++; 1040 - xdp_verdict = ICE_XDP_CONSUMED; 1041 - } 1042 - ice_put_rx_mbuf(rx_ring, xdp, ntc, xdp_verdict); 1043 - 1044 - if (!skb) 1045 - break; 1046 - 1047 - stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S); 1048 - if (unlikely(ice_test_staterr(rx_desc->wb.status_error0, 1049 - stat_err_bits))) { 1050 - dev_kfree_skb_any(skb); 1051 1483 continue; 1052 1484 } 1053 1485 1054 1486 vlan_tci = ice_get_vlan_tci(rx_desc); 1055 - 1056 - /* pad the skb if needed, to make a valid ethernet frame */ 1057 - if (eth_skb_pad(skb)) 1058 - continue; 1059 1487 1060 1488 /* probably a little skewed due to removing CRC */ 1061 1489 total_rx_bytes += skb->len; ··· 1067 1507 1068 1508 rx_ring->next_to_clean = ntc; 1069 1509 /* return up to cleaned_count buffers to hardware */ 1070 - failure = ice_alloc_rx_bufs(rx_ring, ICE_RX_DESC_UNUSED(rx_ring)); 1510 + failure = ice_alloc_rx_bufs(rx_ring, ICE_DESC_UNUSED(rx_ring)); 1071 1511 1072 1512 if (xdp_xmit) 1073 1513 ice_finalize_xdp_rx(xdp_ring, xdp_xmit, cached_ntu); 1514 + 1515 + libeth_xdp_save_buff(&rx_ring->xdp, xdp); 1074 1516 1075 1517 if (rx_ring->ring_stats) 1076 1518 ice_update_rx_ring_stats(rx_ring, total_rx_pkts,

+16 -116

drivers/net/ethernet/intel/ice/ice_txrx.h

··· 4 4 #ifndef _ICE_TXRX_H_ 5 5 #define _ICE_TXRX_H_ 6 6 7 + #include <net/libeth/types.h> 8 + 7 9 #include "ice_type.h" 8 10 9 11 #define ICE_DFLT_IRQ_WORK 256 ··· 29 27 30 28 #define ICE_MAX_TXQ_PER_TXQG 128 31 29 32 - /* Attempt to maximize the headroom available for incoming frames. We use a 2K 33 - * buffer for MTUs <= 1500 and need 1536/1534 to store the data for the frame. 34 - * This leaves us with 512 bytes of room. From that we need to deduct the 35 - * space needed for the shared info and the padding needed to IP align the 36 - * frame. 37 - * 38 - * Note: For cache line sizes 256 or larger this value is going to end 39 - * up negative. In these cases we should fall back to the legacy 40 - * receive path. 41 - */ 42 - #if (PAGE_SIZE < 8192) 43 - #define ICE_2K_TOO_SMALL_WITH_PADDING \ 44 - ((unsigned int)(NET_SKB_PAD + ICE_RXBUF_1536) > \ 45 - SKB_WITH_OVERHEAD(ICE_RXBUF_2048)) 46 - 47 - /** 48 - * ice_compute_pad - compute the padding 49 - * @rx_buf_len: buffer length 50 - * 51 - * Figure out the size of half page based on given buffer length and 52 - * then subtract the skb_shared_info followed by subtraction of the 53 - * actual buffer length; this in turn results in the actual space that 54 - * is left for padding usage 55 - */ 56 - static inline int ice_compute_pad(int rx_buf_len) 57 - { 58 - int half_page_size; 59 - 60 - half_page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2); 61 - return SKB_WITH_OVERHEAD(half_page_size) - rx_buf_len; 62 - } 63 - 64 - /** 65 - * ice_skb_pad - determine the padding that we can supply 66 - * 67 - * Figure out the right Rx buffer size and based on that calculate the 68 - * padding 69 - */ 70 - static inline int ice_skb_pad(void) 71 - { 72 - int rx_buf_len; 73 - 74 - /* If a 2K buffer cannot handle a standard Ethernet frame then 75 - * optimize padding for a 3K buffer instead of a 1.5K buffer. 76 - * 77 - * For a 3K buffer we need to add enough padding to allow for 78 - * tailroom due to NET_IP_ALIGN possibly shifting us out of 79 - * cache-line alignment. 80 - */ 81 - if (ICE_2K_TOO_SMALL_WITH_PADDING) 82 - rx_buf_len = ICE_RXBUF_3072 + SKB_DATA_ALIGN(NET_IP_ALIGN); 83 - else 84 - rx_buf_len = ICE_RXBUF_1536; 85 - 86 - /* if needed make room for NET_IP_ALIGN */ 87 - rx_buf_len -= NET_IP_ALIGN; 88 - 89 - return ice_compute_pad(rx_buf_len); 90 - } 91 - 92 - #define ICE_SKB_PAD ice_skb_pad() 93 - #else 94 - #define ICE_2K_TOO_SMALL_WITH_PADDING false 95 - #define ICE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) 96 - #endif 97 - 98 30 /* We are assuming that the cache line is always 64 Bytes here for ice. 99 31 * In order to make sure that is a correct assumption there is a check in probe 100 32 * to print a warning if the read from GLPCI_CNF2 tells us that the cache line ··· 47 111 #define ICE_DESC_UNUSED(R) \ 48 112 (u16)((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->count) + \ 49 113 (R)->next_to_clean - (R)->next_to_use - 1) 50 - 51 - #define ICE_RX_DESC_UNUSED(R) \ 52 - ((((R)->first_desc > (R)->next_to_use) ? 0 : (R)->count) + \ 53 - (R)->first_desc - (R)->next_to_use - 1) 54 114 55 115 #define ICE_RING_QUARTER(R) ((R)->count >> 2) 56 116 ··· 129 197 u8 header_len; 130 198 }; 131 199 132 - struct ice_rx_buf { 133 - dma_addr_t dma; 134 - struct page *page; 135 - unsigned int page_offset; 136 - unsigned int pgcnt; 137 - unsigned int pagecnt_bias; 138 - }; 139 - 140 200 struct ice_q_stats { 141 201 u64 pkts; 142 202 u64 bytes; ··· 186 262 __be16 vlan_proto; 187 263 }; 188 264 189 - struct ice_xdp_buff { 190 - struct xdp_buff xdp_buff; 191 - const union ice_32b_rx_flex_desc *eop_desc; 192 - const struct ice_pkt_ctx *pkt_ctx; 193 - }; 194 - 195 - /* Required for compatibility with xdp_buffs from xsk_pool */ 196 - static_assert(offsetof(struct ice_xdp_buff, xdp_buff) == 0); 197 - 198 265 /* indices into GLINT_ITR registers */ 199 266 #define ICE_RX_ITR ICE_IDX_ITR0 200 267 #define ICE_TX_ITR ICE_IDX_ITR1 ··· 238 323 struct ice_rx_ring { 239 324 /* CL1 - 1st cacheline starts here */ 240 325 void *desc; /* Descriptor ring memory */ 241 - struct device *dev; /* Used for DMA mapping */ 326 + struct page_pool *pp; 242 327 struct net_device *netdev; /* netdev ring maps to */ 243 328 struct ice_vsi *vsi; /* Backreference to associated VSI */ 244 329 struct ice_q_vector *q_vector; /* Backreference to associated vector */ ··· 250 335 u16 next_to_alloc; 251 336 252 337 union { 253 - struct ice_rx_buf *rx_buf; 338 + struct libeth_fqe *rx_fqes; 254 339 struct xdp_buff **xdp_buf; 255 340 }; 341 + 256 342 /* CL2 - 2nd cacheline starts here */ 343 + struct libeth_fqe *hdr_fqes; 344 + struct page_pool *hdr_pp; 345 + 257 346 union { 258 - struct ice_xdp_buff xdp_ext; 259 - struct xdp_buff xdp; 347 + struct libeth_xdp_buff_stash xdp; 348 + struct libeth_xdp_buff *xsk; 260 349 }; 350 + 261 351 /* CL3 - 3rd cacheline starts here */ 262 352 union { 263 353 struct ice_pkt_ctx pkt_ctx; ··· 272 352 }; 273 353 }; 274 354 struct bpf_prog *xdp_prog; 275 - u16 rx_offset; 276 355 277 356 /* used in interrupt processing */ 278 357 u16 next_to_use; 279 358 u16 next_to_clean; 280 - u16 first_desc; 359 + 360 + u32 hdr_truesize; 361 + u32 truesize; 281 362 282 363 /* stats structs */ 283 364 struct ice_ring_stats *ring_stats; ··· 289 368 struct ice_tx_ring *xdp_ring; 290 369 struct ice_rx_ring *next; /* pointer to next ring in q_vector */ 291 370 struct xsk_buff_pool *xsk_pool; 292 - u16 max_frame; 371 + u16 rx_hdr_len; 293 372 u16 rx_buf_len; 294 373 dma_addr_t dma; /* physical address of ring */ 295 374 u8 dcb_tc; /* Traffic class of ring */ 296 375 u8 ptp_rx; 297 - #define ICE_RX_FLAGS_RING_BUILD_SKB BIT(1) 298 376 #define ICE_RX_FLAGS_CRC_STRIP_DIS BIT(2) 299 377 #define ICE_RX_FLAGS_MULTIDEV BIT(3) 300 378 #define ICE_RX_FLAGS_RING_GCS BIT(4) ··· 341 421 u8 dcb_tc; /* Traffic class of ring */ 342 422 u16 quanta_prof_id; 343 423 } ____cacheline_internodealigned_in_smp; 344 - 345 - static inline bool ice_ring_uses_build_skb(struct ice_rx_ring *ring) 346 - { 347 - return !!(ring->flags & ICE_RX_FLAGS_RING_BUILD_SKB); 348 - } 349 - 350 - static inline void ice_set_ring_build_skb_ena(struct ice_rx_ring *ring) 351 - { 352 - ring->flags |= ICE_RX_FLAGS_RING_BUILD_SKB; 353 - } 354 - 355 - static inline void ice_clear_ring_build_skb_ena(struct ice_rx_ring *ring) 356 - { 357 - ring->flags &= ~ICE_RX_FLAGS_RING_BUILD_SKB; 358 - } 359 424 360 425 static inline bool ice_ring_ch_enabled(struct ice_tx_ring *ring) 361 426 { ··· 396 491 397 492 static inline unsigned int ice_rx_pg_order(struct ice_rx_ring *ring) 398 493 { 399 - #if (PAGE_SIZE < 8192) 400 - if (ring->rx_buf_len > (PAGE_SIZE / 2)) 401 - return 1; 402 - #endif 403 494 return 0; 404 495 } 405 - 406 - #define ice_rx_pg_size(_ring) (PAGE_SIZE << ice_rx_pg_order(_ring)) 407 496 408 497 union ice_32b_rx_flex_desc; 409 498 410 499 void ice_init_ctrl_rx_descs(struct ice_rx_ring *rx_ring, u32 num_descs); 500 + void ice_rxq_pp_destroy(struct ice_rx_ring *rq); 411 501 bool ice_alloc_rx_bufs(struct ice_rx_ring *rxr, unsigned int cleaned_count); 412 502 netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev); 413 503 u16

+43 -26

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

··· 3 3 4 4 #include <linux/filter.h> 5 5 #include <linux/net/intel/libie/rx.h> 6 + #include <net/libeth/xdp.h> 6 7 7 8 #include "ice_txrx_lib.h" 8 9 #include "ice_eswitch.h" ··· 231 230 232 231 if (ice_is_port_repr_netdev(netdev)) 233 232 ice_repr_inc_rx_stats(netdev, skb->len); 233 + 234 + /* __skb_push() is needed because xdp_build_skb_from_buff() 235 + * calls eth_type_trans() 236 + */ 237 + __skb_push(skb, ETH_HLEN); 234 238 skb->protocol = eth_type_trans(skb, netdev); 235 - } else { 236 - skb->protocol = eth_type_trans(skb, rx_ring->netdev); 237 239 } 238 240 239 241 ice_rx_csum(rx_ring, skb, rx_desc, ptype); ··· 274 270 ice_clean_xdp_tx_buf(struct device *dev, struct ice_tx_buf *tx_buf, 275 271 struct xdp_frame_bulk *bq) 276 272 { 277 - dma_unmap_single(dev, dma_unmap_addr(tx_buf, dma), 278 - dma_unmap_len(tx_buf, len), DMA_TO_DEVICE); 279 - dma_unmap_len_set(tx_buf, len, 0); 280 - 281 273 switch (tx_buf->type) { 282 274 case ICE_TX_BUF_XDP_TX: 283 - page_frag_free(tx_buf->raw_buf); 275 + libeth_xdp_return_va(tx_buf->raw_buf, true); 284 276 break; 285 277 case ICE_TX_BUF_XDP_XMIT: 278 + dma_unmap_single(dev, dma_unmap_addr(tx_buf, dma), 279 + dma_unmap_len(tx_buf, len), DMA_TO_DEVICE); 286 280 xdp_return_frame_bulk(tx_buf->xdpf, bq); 287 281 break; 288 282 } 289 283 284 + dma_unmap_len_set(tx_buf, len, 0); 290 285 tx_buf->type = ICE_TX_BUF_EMPTY; 291 286 } 292 287 ··· 380 377 struct ice_tx_buf *tx_buf; 381 378 u32 cnt = xdp_ring->count; 382 379 void *data = xdp->data; 380 + struct page *page; 383 381 u32 nr_frags = 0; 384 382 u32 free_space; 385 383 u32 frag = 0; 384 + u32 offset; 386 385 387 386 free_space = ICE_DESC_UNUSED(xdp_ring); 388 387 if (free_space < ICE_RING_QUARTER(xdp_ring)) ··· 404 399 tx_head = &xdp_ring->tx_buf[ntu]; 405 400 tx_buf = tx_head; 406 401 402 + page = virt_to_page(data); 403 + offset = offset_in_page(xdp->data); 404 + 407 405 for (;;) { 408 406 dma_addr_t dma; 409 407 410 - dma = dma_map_single(dev, data, size, DMA_TO_DEVICE); 411 - if (dma_mapping_error(dev, dma)) 412 - goto dma_unmap; 408 + if (frame) { 409 + dma = dma_map_single(dev, data, size, DMA_TO_DEVICE); 410 + if (dma_mapping_error(dev, dma)) 411 + goto dma_unmap; 412 + tx_buf->type = ICE_TX_BUF_FRAG; 413 + } else { 414 + dma = page_pool_get_dma_addr(page) + offset; 415 + dma_sync_single_for_device(dev, dma, size, DMA_BIDIRECTIONAL); 416 + tx_buf->type = ICE_TX_BUF_XDP_TX; 417 + tx_buf->raw_buf = data; 418 + } 413 419 414 420 /* record length, and DMA address */ 415 421 dma_unmap_len_set(tx_buf, len, size); 416 422 dma_unmap_addr_set(tx_buf, dma, dma); 417 - 418 - if (frame) { 419 - tx_buf->type = ICE_TX_BUF_FRAG; 420 - } else { 421 - tx_buf->type = ICE_TX_BUF_XDP_TX; 422 - tx_buf->raw_buf = data; 423 - } 424 423 425 424 tx_desc->buf_addr = cpu_to_le64(dma); 426 425 tx_desc->cmd_type_offset_bsz = ice_build_ctob(0, 0, size, 0); ··· 439 430 tx_desc = ICE_TX_DESC(xdp_ring, ntu); 440 431 tx_buf = &xdp_ring->tx_buf[ntu]; 441 432 433 + page = skb_frag_page(&sinfo->frags[frag]); 434 + offset = skb_frag_off(&sinfo->frags[frag]); 442 435 data = skb_frag_address(&sinfo->frags[frag]); 443 436 size = skb_frag_size(&sinfo->frags[frag]); 444 437 frag++; ··· 525 514 */ 526 515 static int ice_xdp_rx_hw_ts(const struct xdp_md *ctx, u64 *ts_ns) 527 516 { 528 - const struct ice_xdp_buff *xdp_ext = (void *)ctx; 517 + const struct libeth_xdp_buff *xdp_ext = (void *)ctx; 518 + struct ice_rx_ring *rx_ring; 529 519 530 - *ts_ns = ice_ptp_get_rx_hwts(xdp_ext->eop_desc, 531 - xdp_ext->pkt_ctx); 520 + rx_ring = libeth_xdp_buff_to_rq(xdp_ext, typeof(*rx_ring), xdp_rxq); 521 + 522 + *ts_ns = ice_ptp_get_rx_hwts(xdp_ext->desc, 523 + &rx_ring->pkt_ctx); 532 524 if (!*ts_ns) 533 525 return -ENODATA; 534 526 ··· 559 545 static int ice_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, 560 546 enum xdp_rss_hash_type *rss_type) 561 547 { 562 - const struct ice_xdp_buff *xdp_ext = (void *)ctx; 548 + const struct libeth_xdp_buff *xdp_ext = (void *)ctx; 563 549 564 - *hash = ice_get_rx_hash(xdp_ext->eop_desc); 565 - *rss_type = ice_xdp_rx_hash_type(xdp_ext->eop_desc); 550 + *hash = ice_get_rx_hash(xdp_ext->desc); 551 + *rss_type = ice_xdp_rx_hash_type(xdp_ext->desc); 566 552 if (!likely(*hash)) 567 553 return -ENODATA; 568 554 ··· 581 567 static int ice_xdp_rx_vlan_tag(const struct xdp_md *ctx, __be16 *vlan_proto, 582 568 u16 *vlan_tci) 583 569 { 584 - const struct ice_xdp_buff *xdp_ext = (void *)ctx; 570 + const struct libeth_xdp_buff *xdp_ext = (void *)ctx; 571 + struct ice_rx_ring *rx_ring; 585 572 586 - *vlan_proto = xdp_ext->pkt_ctx->vlan_proto; 573 + rx_ring = libeth_xdp_buff_to_rq(xdp_ext, typeof(*rx_ring), xdp_rxq); 574 + 575 + *vlan_proto = rx_ring->pkt_ctx.vlan_proto; 587 576 if (!*vlan_proto) 588 577 return -ENODATA; 589 578 590 - *vlan_tci = ice_get_vlan_tci(xdp_ext->eop_desc); 579 + *vlan_tci = ice_get_vlan_tci(xdp_ext->desc); 591 580 if (!*vlan_tci) 592 581 return -ENODATA; 593 582

-9

drivers/net/ethernet/intel/ice/ice_txrx_lib.h

··· 135 135 void 136 136 ice_receive_skb(struct ice_rx_ring *rx_ring, struct sk_buff *skb, u16 vlan_tci); 137 137 138 - static inline void 139 - ice_xdp_meta_set_desc(struct xdp_buff *xdp, 140 - union ice_32b_rx_flex_desc *eop_desc) 141 - { 142 - struct ice_xdp_buff *xdp_ext = container_of(xdp, struct ice_xdp_buff, 143 - xdp_buff); 144 - 145 - xdp_ext->eop_desc = eop_desc; 146 - } 147 138 #endif /* !_ICE_TXRX_LIB_H_ */

+21 -125

drivers/net/ethernet/intel/ice/ice_xsk.c

··· 3 3 4 4 #include <linux/bpf_trace.h> 5 5 #include <linux/unroll.h> 6 + #include <net/libeth/xdp.h> 6 7 #include <net/xdp_sock_drv.h> 7 8 #include <net/xdp.h> 8 9 #include "ice.h" ··· 170 169 * If allocation was successful, substitute buffer with allocated one. 171 170 * Returns 0 on success, negative on failure 172 171 */ 173 - static int 172 + int 174 173 ice_realloc_rx_xdp_bufs(struct ice_rx_ring *rx_ring, bool pool_present) 175 174 { 176 - size_t elem_size = pool_present ? sizeof(*rx_ring->xdp_buf) : 177 - sizeof(*rx_ring->rx_buf); 178 - void *sw_ring = kcalloc(rx_ring->count, elem_size, GFP_KERNEL); 179 - 180 - if (!sw_ring) 181 - return -ENOMEM; 182 - 183 175 if (pool_present) { 184 - kfree(rx_ring->rx_buf); 185 - rx_ring->rx_buf = NULL; 186 - rx_ring->xdp_buf = sw_ring; 176 + rx_ring->xdp_buf = kcalloc(rx_ring->count, 177 + sizeof(*rx_ring->xdp_buf), 178 + GFP_KERNEL); 179 + if (!rx_ring->xdp_buf) 180 + return -ENOMEM; 187 181 } else { 188 182 kfree(rx_ring->xdp_buf); 189 183 rx_ring->xdp_buf = NULL; 190 - rx_ring->rx_buf = sw_ring; 191 - } 192 - 193 - return 0; 194 - } 195 - 196 - /** 197 - * ice_realloc_zc_buf - reallocate XDP ZC queue pairs 198 - * @vsi: Current VSI 199 - * @zc: is zero copy set 200 - * 201 - * Reallocate buffer for rx_rings that might be used by XSK. 202 - * XDP requires more memory, than rx_buf provides. 203 - * Returns 0 on success, negative on failure 204 - */ 205 - int ice_realloc_zc_buf(struct ice_vsi *vsi, bool zc) 206 - { 207 - struct ice_rx_ring *rx_ring; 208 - uint i; 209 - 210 - ice_for_each_rxq(vsi, i) { 211 - rx_ring = vsi->rx_rings[i]; 212 - if (!rx_ring->xsk_pool) 213 - continue; 214 - 215 - if (ice_realloc_rx_xdp_bufs(rx_ring, zc)) 216 - return -ENOMEM; 217 184 } 218 185 219 186 return 0; ··· 197 228 */ 198 229 int ice_xsk_pool_setup(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) 199 230 { 231 + struct ice_rx_ring *rx_ring = vsi->rx_rings[qid]; 200 232 bool if_running, pool_present = !!pool; 201 233 int ret = 0, pool_failure = 0; 202 234 ··· 211 241 ice_is_xdp_ena_vsi(vsi); 212 242 213 243 if (if_running) { 214 - struct ice_rx_ring *rx_ring = vsi->rx_rings[qid]; 215 - 216 244 ret = ice_qp_dis(vsi, qid); 217 245 if (ret) { 218 246 netdev_err(vsi->netdev, "ice_qp_dis error = %d\n", ret); ··· 270 302 dma = xsk_buff_xdp_get_dma(*xdp); 271 303 rx_desc->read.pkt_addr = cpu_to_le64(dma); 272 304 rx_desc->wb.status_error0 = 0; 273 - 274 - /* Put private info that changes on a per-packet basis 275 - * into xdp_buff_xsk->cb. 276 - */ 277 - ice_xdp_meta_set_desc(*xdp, rx_desc); 278 305 279 306 rx_desc++; 280 307 xdp++; ··· 353 390 if (!__ice_alloc_rx_bufs_zc(rx_ring, xsk_pool, rx_thresh)) 354 391 return false; 355 392 return __ice_alloc_rx_bufs_zc(rx_ring, xsk_pool, leftover); 356 - } 357 - 358 - /** 359 - * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer 360 - * @rx_ring: Rx ring 361 - * @xdp: Pointer to XDP buffer 362 - * 363 - * This function allocates a new skb from a zero-copy Rx buffer. 364 - * 365 - * Returns the skb on success, NULL on failure. 366 - */ 367 - static struct sk_buff * 368 - ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) 369 - { 370 - unsigned int totalsize = xdp->data_end - xdp->data_meta; 371 - unsigned int metasize = xdp->data - xdp->data_meta; 372 - struct skb_shared_info *sinfo = NULL; 373 - struct sk_buff *skb; 374 - u32 nr_frags = 0; 375 - 376 - if (unlikely(xdp_buff_has_frags(xdp))) { 377 - sinfo = xdp_get_shared_info_from_buff(xdp); 378 - nr_frags = sinfo->nr_frags; 379 - } 380 - net_prefetch(xdp->data_meta); 381 - 382 - skb = napi_alloc_skb(&rx_ring->q_vector->napi, totalsize); 383 - if (unlikely(!skb)) 384 - return NULL; 385 - 386 - memcpy(__skb_put(skb, totalsize), xdp->data_meta, 387 - ALIGN(totalsize, sizeof(long))); 388 - 389 - if (metasize) { 390 - skb_metadata_set(skb, metasize); 391 - __skb_pull(skb, metasize); 392 - } 393 - 394 - if (likely(!xdp_buff_has_frags(xdp))) 395 - goto out; 396 - 397 - for (int i = 0; i < nr_frags; i++) { 398 - struct skb_shared_info *skinfo = skb_shinfo(skb); 399 - skb_frag_t *frag = &sinfo->frags[i]; 400 - struct page *page; 401 - void *addr; 402 - 403 - page = dev_alloc_page(); 404 - if (!page) { 405 - dev_kfree_skb(skb); 406 - return NULL; 407 - } 408 - addr = page_to_virt(page); 409 - 410 - memcpy(addr, skb_frag_page(frag), skb_frag_size(frag)); 411 - 412 - __skb_fill_page_desc_noacc(skinfo, skinfo->nr_frags++, 413 - addr, 0, skb_frag_size(frag)); 414 - } 415 - 416 - out: 417 - xsk_buff_free(xdp); 418 - return skb; 419 393 } 420 394 421 395 /** ··· 569 669 struct xsk_buff_pool *xsk_pool, 570 670 int budget) 571 671 { 672 + struct xdp_buff *first = (struct xdp_buff *)rx_ring->xsk; 572 673 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 573 674 u32 ntc = rx_ring->next_to_clean; 574 675 u32 ntu = rx_ring->next_to_use; 575 - struct xdp_buff *first = NULL; 576 676 struct ice_tx_ring *xdp_ring; 577 677 unsigned int xdp_xmit = 0; 578 678 struct bpf_prog *xdp_prog; ··· 585 685 */ 586 686 xdp_prog = READ_ONCE(rx_ring->xdp_prog); 587 687 xdp_ring = rx_ring->xdp_ring; 588 - 589 - if (ntc != rx_ring->first_desc) 590 - first = *ice_xdp_buf(rx_ring, rx_ring->first_desc); 591 688 592 689 while (likely(total_rx_packets < (unsigned int)budget)) { 593 690 union ice_32b_rx_flex_desc *rx_desc; ··· 621 724 first = xdp; 622 725 } else if (likely(size) && !xsk_buff_add_frag(first, xdp)) { 623 726 xsk_buff_free(first); 624 - break; 727 + first = NULL; 625 728 } 626 729 627 730 if (++ntc == cnt) 628 731 ntc = 0; 629 732 630 - if (ice_is_non_eop(rx_ring, rx_desc)) 733 + if (ice_is_non_eop(rx_ring, rx_desc) || unlikely(!first)) 631 734 continue; 735 + 736 + ((struct libeth_xdp_buff *)first)->desc = rx_desc; 632 737 633 738 xdp_res = ice_run_xdp_zc(rx_ring, first, xdp_prog, xdp_ring, 634 739 xsk_pool); ··· 639 740 } else if (xdp_res == ICE_XDP_EXIT) { 640 741 failure = true; 641 742 first = NULL; 642 - rx_ring->first_desc = ntc; 643 743 break; 644 744 } else if (xdp_res == ICE_XDP_CONSUMED) { 645 745 xsk_buff_free(first); ··· 650 752 total_rx_packets++; 651 753 652 754 first = NULL; 653 - rx_ring->first_desc = ntc; 654 755 continue; 655 756 656 757 construct_skb: 657 758 /* XDP_PASS path */ 658 - skb = ice_construct_skb_zc(rx_ring, first); 759 + skb = xdp_build_skb_from_zc(first); 659 760 if (!skb) { 761 + xsk_buff_free(first); 762 + first = NULL; 763 + 660 764 rx_ring->ring_stats->rx_stats.alloc_buf_failed++; 661 - break; 765 + continue; 662 766 } 663 767 664 768 first = NULL; 665 - rx_ring->first_desc = ntc; 666 - 667 - if (eth_skb_pad(skb)) { 668 - skb = NULL; 669 - continue; 670 - } 671 769 672 770 total_rx_bytes += skb->len; 673 771 total_rx_packets++; ··· 675 781 } 676 782 677 783 rx_ring->next_to_clean = ntc; 678 - entries_to_alloc = ICE_RX_DESC_UNUSED(rx_ring); 784 + rx_ring->xsk = (struct libeth_xdp_buff *)first; 785 + 786 + entries_to_alloc = ICE_DESC_UNUSED(rx_ring); 679 787 if (entries_to_alloc > ICE_RING_QUARTER(rx_ring)) 680 788 failure |= !ice_alloc_rx_bufs_zc(rx_ring, xsk_pool, 681 789 entries_to_alloc);

+3 -3

drivers/net/ethernet/intel/ice/ice_xsk.h

··· 22 22 void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring); 23 23 void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring); 24 24 bool ice_xmit_zc(struct ice_tx_ring *xdp_ring, struct xsk_buff_pool *xsk_pool); 25 - int ice_realloc_zc_buf(struct ice_vsi *vsi, bool zc); 25 + int ice_realloc_rx_xdp_bufs(struct ice_rx_ring *rx_ring, bool pool_present); 26 26 void ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector, 27 27 u16 qid); 28 28 void ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector, ··· 77 77 static inline void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring) { } 78 78 79 79 static inline int 80 - ice_realloc_zc_buf(struct ice_vsi __always_unused *vsi, 81 - bool __always_unused zc) 80 + ice_realloc_rx_xdp_bufs(struct ice_rx_ring *rx_ring, 81 + bool __always_unused pool_present) 82 82 { 83 83 return 0; 84 84 }

+2 -3

drivers/net/ethernet/intel/ice/virt/queues.c

··· 842 842 (qpi->rxq.databuffer_size > ((16 * 1024) - 128) || 843 843 qpi->rxq.databuffer_size < 1024)) 844 844 goto error_param; 845 - ring->rx_buf_len = qpi->rxq.databuffer_size; 846 845 if (qpi->rxq.max_pkt_size > max_frame_size || 847 846 qpi->rxq.max_pkt_size < 64) 848 847 goto error_param; 849 848 850 - ring->max_frame = qpi->rxq.max_pkt_size; 849 + vsi->max_frame = qpi->rxq.max_pkt_size; 851 850 /* add space for the port VLAN since the VF driver is 852 851 * not expected to account for it in the MTU 853 852 * calculation 854 853 */ 855 854 if (ice_vf_is_port_vlan_ena(vf)) 856 - ring->max_frame += VLAN_HLEN; 855 + vsi->max_frame += VLAN_HLEN; 857 856 858 857 if (ice_vsi_cfg_single_rxq(vsi, q_idx)) { 859 858 dev_warn(ice_pf_to_dev(pf), "VF-%d failed to configure RX queue %d\n",

-2

drivers/net/ethernet/intel/idpf/idpf.h

+1 -1

drivers/net/ethernet/intel/igbvf/netdev.c

··· 1235 1235 spin_lock_bh(&hw->mbx_lock); 1236 1236 1237 1237 if (hw->mac.ops.set_vfta(hw, vid, true)) { 1238 - dev_warn(&adapter->pdev->dev, "Vlan id %d\n is not added", vid); 1238 + dev_warn(&adapter->pdev->dev, "Vlan id %d is not added\n", vid); 1239 1239 spin_unlock_bh(&hw->mbx_lock); 1240 1240 return -EINVAL; 1241 1241 }

+2 -2

drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c

··· 198 198 * @hw: pointer to hardware structure 199 199 * @autoc: value to write to AUTOC 200 200 * @locked: bool to indicate whether the SW/FW lock was already taken by 201 - * previous proc_autoc_read_82599. 201 + * previous prot_autoc_read_82599. 202 202 * 203 203 * This part (82599) may need to hold a the SW/FW lock around all writes to 204 204 * AUTOC. Likewise after a write we need to do a pipeline reset. ··· 1622 1622 break; 1623 1623 } 1624 1624 1625 - /* store source and destination IP masks (big-enian) */ 1625 + /* store source and destination IP masks (big-endian) */ 1626 1626 IXGBE_WRITE_REG_BE32(hw, IXGBE_FDIRSIP4M, 1627 1627 ~input_mask->formatted.src_ip[0]); 1628 1628 IXGBE_WRITE_REG_BE32(hw, IXGBE_FDIRDIP4M,

+2 -2

drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c

··· 318 318 * ixgbe_set_dcb_sriov_queues: Allocate queues for SR-IOV devices w/ DCB 319 319 * @adapter: board private structure to initialize 320 320 * 321 - * When SR-IOV (Single Root IO Virtualiztion) is enabled, allocate queues 321 + * When SR-IOV (Single Root IO Virtualization) is enabled, allocate queues 322 322 * and VM pools where appropriate. Also assign queues based on DCB 323 323 * priorities and map accordingly.. 324 324 * ··· 492 492 * ixgbe_set_sriov_queues - Allocate queues for SR-IOV devices 493 493 * @adapter: board private structure to initialize 494 494 * 495 - * When SR-IOV (Single Root IO Virtualiztion) is enabled, allocate queues 495 + * When SR-IOV (Single Root IO Virtualization) is enabled, allocate queues 496 496 * and VM pools where appropriate. If RSS is available, then also try and 497 497 * enable RSS and map accordingly. 498 498 *

Configure Feed

Configure Feed