Merge branch '200GbE' of git://git.kernel.org/pub/scm/linux/kernel/git/tnguy/next-queue

+1 -1

drivers/net/ethernet/intel/idpf/Kconfig

··· 6 6 depends on PCI_MSI 7 7 depends on PTP_1588_CLOCK_OPTIONAL 8 8 select DIMLIB 9 - select LIBETH 9 + select LIBETH_XDP 10 10 help 11 11 This driver supports Intel(R) Infrastructure Data Path Function 12 12 devices.

+2

drivers/net/ethernet/intel/idpf/Makefile

··· 21 21 22 22 idpf-$(CONFIG_PTP_1588_CLOCK) += idpf_ptp.o 23 23 idpf-$(CONFIG_PTP_1588_CLOCK) += idpf_virtchnl_ptp.o 24 + 25 + idpf-y += xdp.o

+25 -6

drivers/net/ethernet/intel/idpf/idpf.h

··· 40 40 #define IDPF_NUM_CHUNKS_PER_MSG(struct_sz, chunk_sz) \ 41 41 ((IDPF_CTLQ_MAX_BUF_LEN - (struct_sz)) / (chunk_sz)) 42 42 43 + #define IDPF_WAIT_FOR_MARKER_TIMEO 500 43 44 #define IDPF_MAX_WAIT 500 44 45 45 46 /* available message levels */ ··· 249 248 /** 250 249 * enum idpf_vport_flags - Vport flags 251 250 * @IDPF_VPORT_DEL_QUEUES: To send delete queues message 252 - * @IDPF_VPORT_SW_MARKER: Indicate TX pipe drain software marker packets 253 - * processing is done 254 251 * @IDPF_VPORT_FLAGS_NBITS: Must be last 255 252 */ 256 253 enum idpf_vport_flags { 257 254 IDPF_VPORT_DEL_QUEUES, 258 - IDPF_VPORT_SW_MARKER, 259 255 IDPF_VPORT_FLAGS_NBITS, 260 256 }; 261 257 ··· 287 289 * @txq_model: Split queue or single queue queuing model 288 290 * @txqs: Used only in hotpath to get to the right queue very fast 289 291 * @crc_enable: Enable CRC insertion offload 292 + * @xdpsq_share: whether XDPSQ sharing is enabled 293 + * @num_xdp_txq: number of XDPSQs 294 + * @xdp_txq_offset: index of the first XDPSQ (== number of regular SQs) 295 + * @xdp_prog: installed XDP program 290 296 * @num_rxq: Number of allocated RX queues 291 297 * @num_bufq: Number of allocated buffer queues 292 298 * @rxq_desc_count: RX queue descriptor count. *MUST* have enough descriptors ··· 316 314 * @num_q_vectors: Number of IRQ vectors allocated 317 315 * @q_vectors: Array of queue vectors 318 316 * @q_vector_idxs: Starting index of queue vectors 317 + * @noirq_dyn_ctl: register to enable/disable the vector for NOIRQ queues 318 + * @noirq_dyn_ctl_ena: value to write to the above to enable it 319 + * @noirq_v_idx: ID of the NOIRQ vector 319 320 * @max_mtu: device given max possible MTU 320 321 * @default_mac_addr: device will give a default MAC to use 321 322 * @rx_itr_profile: RX profiles for Dynamic Interrupt Moderation 322 323 * @tx_itr_profile: TX profiles for Dynamic Interrupt Moderation 323 324 * @port_stats: per port csum, header split, and other offload stats 324 325 * @link_up: True if link is up 325 - * @sw_marker_wq: workqueue for marker packets 326 326 * @tx_tstamp_caps: Capabilities negotiated for Tx timestamping 327 327 * @tstamp_config: The Tx tstamp config 328 328 * @tstamp_task: Tx timestamping task ··· 340 336 u32 txq_model; 341 337 struct idpf_tx_queue **txqs; 342 338 bool crc_enable; 339 + 340 + bool xdpsq_share; 341 + u16 num_xdp_txq; 342 + u16 xdp_txq_offset; 343 + struct bpf_prog *xdp_prog; 343 344 344 345 u16 num_rxq; 345 346 u16 num_bufq; ··· 370 361 u16 num_q_vectors; 371 362 struct idpf_q_vector *q_vectors; 372 363 u16 *q_vector_idxs; 364 + 365 + void __iomem *noirq_dyn_ctl; 366 + u32 noirq_dyn_ctl_ena; 367 + u16 noirq_v_idx; 368 + 373 369 u16 max_mtu; 374 370 u8 default_mac_addr[ETH_ALEN]; 375 371 u16 rx_itr_profile[IDPF_DIM_PROFILE_SLOTS]; ··· 382 368 struct idpf_port_stats port_stats; 383 369 384 370 bool link_up; 385 - 386 - wait_queue_head_t sw_marker_wq; 387 371 388 372 struct idpf_ptp_vport_tx_tstamp_caps *tx_tstamp_caps; 389 373 struct kernel_hwtstamp_config tstamp_config; ··· 447 435 * ethtool 448 436 * @num_req_rxq_desc: Number of user requested RX queue descriptors through 449 437 * ethtool 438 + * @xdp_prog: requested XDP program to install 450 439 * @user_flags: User toggled config flags 451 440 * @mac_filter_list: List of MAC filters 452 441 * @num_fsteer_fltrs: number of flow steering filters ··· 462 449 u16 num_req_rx_qs; 463 450 u32 num_req_txq_desc; 464 451 u32 num_req_rxq_desc; 452 + struct bpf_prog *xdp_prog; 465 453 DECLARE_BITMAP(user_flags, __IDPF_USER_FLAGS_NBITS); 466 454 struct list_head mac_filter_list; 467 455 u32 num_fsteer_fltrs; ··· 690 676 { 691 677 return !IS_ENABLED(CONFIG_IDPF_SINGLEQ) || 692 678 q_model == VIRTCHNL2_QUEUE_MODEL_SPLIT; 679 + } 680 + 681 + static inline bool idpf_xdp_enabled(const struct idpf_vport *vport) 682 + { 683 + return vport->adapter && vport->xdp_prog; 693 684 } 694 685 695 686 #define idpf_is_cap_ena(adapter, field, flag) \

+10 -1

drivers/net/ethernet/intel/idpf/idpf_dev.c

··· 77 77 int num_vecs = vport->num_q_vectors; 78 78 struct idpf_vec_regs *reg_vals; 79 79 int num_regs, i, err = 0; 80 - u32 rx_itr, tx_itr; 80 + u32 rx_itr, tx_itr, val; 81 81 u16 total_vecs; 82 82 83 83 total_vecs = idpf_get_reserved_vecs(vport->adapter); ··· 120 120 intr->rx_itr = idpf_get_reg_addr(adapter, rx_itr); 121 121 intr->tx_itr = idpf_get_reg_addr(adapter, tx_itr); 122 122 } 123 + 124 + /* Data vector for NOIRQ queues */ 125 + 126 + val = reg_vals[vport->q_vector_idxs[i] - IDPF_MBX_Q_VEC].dyn_ctl_reg; 127 + vport->noirq_dyn_ctl = idpf_get_reg_addr(adapter, val); 128 + 129 + val = PF_GLINT_DYN_CTL_WB_ON_ITR_M | PF_GLINT_DYN_CTL_INTENA_MSK_M | 130 + FIELD_PREP(PF_GLINT_DYN_CTL_ITR_INDX_M, IDPF_NO_ITR_UPDATE_IDX); 131 + vport->noirq_dyn_ctl_ena = val; 123 132 124 133 free_reg_vals: 125 134 kfree(reg_vals);

+5 -1

drivers/net/ethernet/intel/idpf/idpf_lan_txrx.h

··· 186 186 __le64 qw1; /* type_cmd_offset_bsz_l2tag1 */ 187 187 }; /* read used with buffer queues */ 188 188 189 - struct idpf_splitq_tx_compl_desc { 189 + struct idpf_splitq_4b_tx_compl_desc { 190 190 /* qid=[10:0] comptype=[13:11] rsvd=[14] gen=[15] */ 191 191 __le16 qid_comptype_gen; 192 192 union { 193 193 __le16 q_head; /* Queue head */ 194 194 __le16 compl_tag; /* Completion tag */ 195 195 } q_head_compl_tag; 196 + }; /* writeback used with completion queues */ 197 + 198 + struct idpf_splitq_tx_compl_desc { 199 + struct idpf_splitq_4b_tx_compl_desc common; 196 200 u8 ts[3]; 197 201 u8 rsvd; /* Reserved */ 198 202 }; /* writeback used with completion queues */

+50 -17

drivers/net/ethernet/intel/idpf/idpf_lib.c

··· 4 4 #include "idpf.h" 5 5 #include "idpf_virtchnl.h" 6 6 #include "idpf_ptp.h" 7 + #include "xdp.h" 7 8 8 9 static const struct net_device_ops idpf_netdev_ops; 9 10 ··· 836 835 netdev->hw_features |= netdev->features | other_offloads; 837 836 netdev->vlan_features |= netdev->features | other_offloads; 838 837 netdev->hw_enc_features |= dflt_features | other_offloads; 838 + idpf_xdp_set_features(vport); 839 + 839 840 idpf_set_ethtool_ops(netdev); 840 841 netif_set_affinity_auto(netdev); 841 842 SET_NETDEV_DEV(netdev, &adapter->pdev->dev); ··· 887 884 /** 888 885 * idpf_vport_stop - Disable a vport 889 886 * @vport: vport to disable 887 + * @rtnl: whether to take RTNL lock 890 888 */ 891 - static void idpf_vport_stop(struct idpf_vport *vport) 889 + static void idpf_vport_stop(struct idpf_vport *vport, bool rtnl) 892 890 { 893 891 struct idpf_netdev_priv *np = netdev_priv(vport->netdev); 894 892 895 893 if (np->state <= __IDPF_VPORT_DOWN) 896 894 return; 895 + 896 + if (rtnl) 897 + rtnl_lock(); 897 898 898 899 netif_carrier_off(vport->netdev); 899 900 netif_tx_disable(vport->netdev); ··· 917 910 918 911 vport->link_up = false; 919 912 idpf_vport_intr_deinit(vport); 913 + idpf_xdp_rxq_info_deinit_all(vport); 920 914 idpf_vport_queues_rel(vport); 921 915 idpf_vport_intr_rel(vport); 922 916 np->state = __IDPF_VPORT_DOWN; 917 + 918 + if (rtnl) 919 + rtnl_unlock(); 923 920 } 924 921 925 922 /** ··· 947 936 idpf_vport_ctrl_lock(netdev); 948 937 vport = idpf_netdev_to_vport(netdev); 949 938 950 - idpf_vport_stop(vport); 939 + idpf_vport_stop(vport, false); 951 940 952 941 idpf_vport_ctrl_unlock(netdev); 953 942 ··· 1040 1029 idpf_idc_deinit_vport_aux_device(vport->vdev_info); 1041 1030 1042 1031 idpf_deinit_mac_addr(vport); 1043 - idpf_vport_stop(vport); 1032 + idpf_vport_stop(vport, true); 1044 1033 1045 1034 if (!test_bit(IDPF_HR_RESET_IN_PROG, adapter->flags)) 1046 1035 idpf_decfg_netdev(vport); ··· 1146 1135 if (!vport) 1147 1136 return vport; 1148 1137 1149 - num_max_q = max(max_q->max_txq, max_q->max_rxq); 1138 + num_max_q = max(max_q->max_txq, max_q->max_rxq) + IDPF_RESERVED_VECS; 1150 1139 if (!adapter->vport_config[idx]) { 1151 1140 struct idpf_vport_config *vport_config; 1152 1141 struct idpf_q_coalesce *q_coal; ··· 1320 1309 */ 1321 1310 static int idpf_set_real_num_queues(struct idpf_vport *vport) 1322 1311 { 1323 - int err; 1312 + int err, txq = vport->num_txq - vport->num_xdp_txq; 1324 1313 1325 1314 err = netif_set_real_num_rx_queues(vport->netdev, vport->num_rxq); 1326 1315 if (err) 1327 1316 return err; 1328 1317 1329 - return netif_set_real_num_tx_queues(vport->netdev, vport->num_txq); 1318 + return netif_set_real_num_tx_queues(vport->netdev, txq); 1330 1319 } 1331 1320 1332 1321 /** ··· 1381 1370 /** 1382 1371 * idpf_vport_open - Bring up a vport 1383 1372 * @vport: vport to bring up 1373 + * @rtnl: whether to take RTNL lock 1384 1374 */ 1385 - static int idpf_vport_open(struct idpf_vport *vport) 1375 + static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) 1386 1376 { 1387 1377 struct idpf_netdev_priv *np = netdev_priv(vport->netdev); 1388 1378 struct idpf_adapter *adapter = vport->adapter; ··· 1393 1381 if (np->state != __IDPF_VPORT_DOWN) 1394 1382 return -EBUSY; 1395 1383 1384 + if (rtnl) 1385 + rtnl_lock(); 1386 + 1396 1387 /* we do not allow interface up just yet */ 1397 1388 netif_carrier_off(vport->netdev); 1398 1389 ··· 1403 1388 if (err) { 1404 1389 dev_err(&adapter->pdev->dev, "Failed to allocate interrupts for vport %u: %d\n", 1405 1390 vport->vport_id, err); 1406 - return err; 1391 + goto err_rtnl_unlock; 1407 1392 } 1408 1393 1409 1394 err = idpf_vport_queues_alloc(vport); ··· 1439 1424 } 1440 1425 1441 1426 idpf_rx_init_buf_tail(vport); 1427 + 1428 + err = idpf_xdp_rxq_info_init_all(vport); 1429 + if (err) { 1430 + netdev_err(vport->netdev, 1431 + "Failed to initialize XDP RxQ info for vport %u: %pe\n", 1432 + vport->vport_id, ERR_PTR(err)); 1433 + goto intr_deinit; 1434 + } 1435 + 1442 1436 idpf_vport_intr_ena(vport); 1443 1437 1444 1438 err = idpf_send_config_queues_msg(vport); 1445 1439 if (err) { 1446 1440 dev_err(&adapter->pdev->dev, "Failed to configure queues for vport %u, %d\n", 1447 1441 vport->vport_id, err); 1448 - goto intr_deinit; 1442 + goto rxq_deinit; 1449 1443 } 1450 1444 1451 1445 err = idpf_send_map_unmap_queue_vector_msg(vport, true); 1452 1446 if (err) { 1453 1447 dev_err(&adapter->pdev->dev, "Failed to map queue vectors for vport %u: %d\n", 1454 1448 vport->vport_id, err); 1455 - goto intr_deinit; 1449 + goto rxq_deinit; 1456 1450 } 1457 1451 1458 1452 err = idpf_send_enable_queues_msg(vport); ··· 1499 1475 goto deinit_rss; 1500 1476 } 1501 1477 1478 + if (rtnl) 1479 + rtnl_unlock(); 1480 + 1502 1481 return 0; 1503 1482 1504 1483 deinit_rss: ··· 1512 1485 idpf_send_disable_queues_msg(vport); 1513 1486 unmap_queue_vectors: 1514 1487 idpf_send_map_unmap_queue_vector_msg(vport, false); 1488 + rxq_deinit: 1489 + idpf_xdp_rxq_info_deinit_all(vport); 1515 1490 intr_deinit: 1516 1491 idpf_vport_intr_deinit(vport); 1517 1492 queues_rel: 1518 1493 idpf_vport_queues_rel(vport); 1519 1494 intr_rel: 1520 1495 idpf_vport_intr_rel(vport); 1496 + 1497 + err_rtnl_unlock: 1498 + if (rtnl) 1499 + rtnl_unlock(); 1521 1500 1522 1501 return err; 1523 1502 } ··· 1581 1548 index = vport->idx; 1582 1549 vport_config = adapter->vport_config[index]; 1583 1550 1584 - init_waitqueue_head(&vport->sw_marker_wq); 1585 - 1586 1551 spin_lock_init(&vport_config->mac_filter_list_lock); 1587 1552 1588 1553 INIT_LIST_HEAD(&vport_config->user_config.mac_filter_list); ··· 1603 1572 np = netdev_priv(vport->netdev); 1604 1573 np->state = __IDPF_VPORT_DOWN; 1605 1574 if (test_and_clear_bit(IDPF_VPORT_UP_REQUESTED, vport_config->flags)) 1606 - idpf_vport_open(vport); 1575 + idpf_vport_open(vport, true); 1607 1576 1608 1577 /* Spawn and return 'idpf_init_task' work queue until all the 1609 1578 * default vports are created ··· 1993 1962 idpf_send_delete_queues_msg(vport); 1994 1963 } else { 1995 1964 set_bit(IDPF_VPORT_DEL_QUEUES, vport->flags); 1996 - idpf_vport_stop(vport); 1965 + idpf_vport_stop(vport, false); 1997 1966 } 1998 1967 1999 1968 idpf_deinit_rss(vport); ··· 2023 1992 goto err_open; 2024 1993 2025 1994 if (current_state == __IDPF_VPORT_UP) 2026 - err = idpf_vport_open(vport); 1995 + err = idpf_vport_open(vport, false); 2027 1996 2028 1997 goto free_vport; 2029 1998 ··· 2033 2002 2034 2003 err_open: 2035 2004 if (current_state == __IDPF_VPORT_UP) 2036 - idpf_vport_open(vport); 2005 + idpf_vport_open(vport, false); 2037 2006 2038 2007 free_vport: 2039 2008 kfree(new_vport); ··· 2271 2240 if (err) 2272 2241 goto unlock; 2273 2242 2274 - err = idpf_vport_open(vport); 2243 + err = idpf_vport_open(vport, false); 2275 2244 2276 2245 unlock: 2277 2246 idpf_vport_ctrl_unlock(netdev); ··· 2616 2585 .ndo_tx_timeout = idpf_tx_timeout, 2617 2586 .ndo_hwtstamp_get = idpf_hwtstamp_get, 2618 2587 .ndo_hwtstamp_set = idpf_hwtstamp_set, 2588 + .ndo_bpf = idpf_xdp, 2589 + .ndo_xdp_xmit = idpf_xdp_xmit, 2619 2590 };

+1

drivers/net/ethernet/intel/idpf/idpf_main.c

··· 9 9 10 10 MODULE_DESCRIPTION(DRV_SUMMARY); 11 11 MODULE_IMPORT_NS("LIBETH"); 12 + MODULE_IMPORT_NS("LIBETH_XDP"); 12 13 MODULE_LICENSE("GPL"); 13 14 14 15 /**

+50 -60

drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c

··· 1 1 // SPDX-License-Identifier: GPL-2.0-only 2 2 /* Copyright (C) 2023 Intel Corporation */ 3 3 4 - #include <net/libeth/rx.h> 5 - #include <net/libeth/tx.h> 4 + #include <net/libeth/xdp.h> 6 5 7 6 #include "idpf.h" 8 7 ··· 654 655 bool ipv4, ipv6; 655 656 656 657 /* check if Rx checksum is enabled */ 657 - if (!libeth_rx_pt_has_checksum(rxq->netdev, decoded)) 658 + if (!libeth_rx_pt_has_checksum(rxq->xdp_rxq.dev, decoded)) 658 659 return; 659 660 660 661 /* check if HW has decoded the packet and checksum */ ··· 793 794 { 794 795 u64 mask, qw1; 795 796 796 - if (!libeth_rx_pt_has_hash(rx_q->netdev, decoded)) 797 + if (!libeth_rx_pt_has_hash(rx_q->xdp_rxq.dev, decoded)) 797 798 return; 798 799 799 800 mask = VIRTCHNL2_RX_BASE_DESC_FLTSTAT_RSS_HASH_M; ··· 821 822 const union virtchnl2_rx_desc *rx_desc, 822 823 struct libeth_rx_pt decoded) 823 824 { 824 - if (!libeth_rx_pt_has_hash(rx_q->netdev, decoded)) 825 + if (!libeth_rx_pt_has_hash(rx_q->xdp_rxq.dev, decoded)) 825 826 return; 826 827 827 828 if (FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_STATUS0_RSS_VALID_M, ··· 833 834 } 834 835 835 836 /** 836 - * idpf_rx_singleq_process_skb_fields - Populate skb header fields from Rx 837 + * __idpf_rx_singleq_process_skb_fields - Populate skb header fields from Rx 837 838 * descriptor 838 839 * @rx_q: Rx ring being processed 839 840 * @skb: pointer to current skb being populated ··· 845 846 * other fields within the skb. 846 847 */ 847 848 static void 848 - idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q, 849 - struct sk_buff *skb, 850 - const union virtchnl2_rx_desc *rx_desc, 851 - u16 ptype) 849 + __idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q, 850 + struct sk_buff *skb, 851 + const union virtchnl2_rx_desc *rx_desc, 852 + u16 ptype) 852 853 { 853 854 struct libeth_rx_pt decoded = rx_q->rx_ptype_lkup[ptype]; 854 855 struct libeth_rx_csum csum_bits; 855 - 856 - /* modifies the skb - consumes the enet header */ 857 - skb->protocol = eth_type_trans(skb, rx_q->netdev); 858 856 859 857 /* Check if we're using base mode descriptor IDs */ 860 858 if (rx_q->rxdids == VIRTCHNL2_RXDID_1_32B_BASE_M) { ··· 863 867 } 864 868 865 869 idpf_rx_singleq_csum(rx_q, skb, csum_bits, decoded); 866 - skb_record_rx_queue(skb, rx_q->idx); 867 870 } 868 871 869 872 /** ··· 998 1003 idpf_rx_singleq_extract_flex_fields(rx_desc, fields); 999 1004 } 1000 1005 1006 + static bool 1007 + idpf_rx_singleq_process_skb_fields(struct sk_buff *skb, 1008 + const struct libeth_xdp_buff *xdp, 1009 + struct libeth_rq_napi_stats *rs) 1010 + { 1011 + struct libeth_rqe_info fields; 1012 + struct idpf_rx_queue *rxq; 1013 + 1014 + rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); 1015 + 1016 + idpf_rx_singleq_extract_fields(rxq, xdp->desc, &fields); 1017 + __idpf_rx_singleq_process_skb_fields(rxq, skb, xdp->desc, 1018 + fields.ptype); 1019 + 1020 + return true; 1021 + } 1022 + 1023 + static void idpf_xdp_run_pass(struct libeth_xdp_buff *xdp, 1024 + struct napi_struct *napi, 1025 + struct libeth_rq_napi_stats *rs, 1026 + const union virtchnl2_rx_desc *desc) 1027 + { 1028 + libeth_xdp_run_pass(xdp, NULL, napi, rs, desc, NULL, 1029 + idpf_rx_singleq_process_skb_fields); 1030 + } 1031 + 1001 1032 /** 1002 1033 * idpf_rx_singleq_clean - Reclaim resources after receive completes 1003 1034 * @rx_q: rx queue to clean ··· 1033 1012 */ 1034 1013 static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget) 1035 1014 { 1036 - unsigned int total_rx_bytes = 0, total_rx_pkts = 0; 1037 - struct sk_buff *skb = rx_q->skb; 1015 + struct libeth_rq_napi_stats rs = { }; 1038 1016 u16 ntc = rx_q->next_to_clean; 1017 + LIBETH_XDP_ONSTACK_BUFF(xdp); 1039 1018 u16 cleaned_count = 0; 1040 - bool failure = false; 1019 + 1020 + libeth_xdp_init_buff(xdp, &rx_q->xdp, &rx_q->xdp_rxq); 1041 1021 1042 1022 /* Process Rx packets bounded by budget */ 1043 - while (likely(total_rx_pkts < (unsigned int)budget)) { 1023 + while (likely(rs.packets < budget)) { 1044 1024 struct libeth_rqe_info fields = { }; 1045 1025 union virtchnl2_rx_desc *rx_desc; 1046 1026 struct idpf_rx_buf *rx_buf; ··· 1068 1046 idpf_rx_singleq_extract_fields(rx_q, rx_desc, &fields); 1069 1047 1070 1048 rx_buf = &rx_q->rx_buf[ntc]; 1071 - if (!libeth_rx_sync_for_cpu(rx_buf, fields.len)) 1072 - goto skip_data; 1073 - 1074 - if (skb) 1075 - idpf_rx_add_frag(rx_buf, skb, fields.len); 1076 - else 1077 - skb = idpf_rx_build_skb(rx_buf, fields.len); 1078 - 1079 - /* exit if we failed to retrieve a buffer */ 1080 - if (!skb) 1081 - break; 1082 - 1083 - skip_data: 1049 + libeth_xdp_process_buff(xdp, rx_buf, fields.len); 1084 1050 rx_buf->netmem = 0; 1085 1051 1086 1052 IDPF_SINGLEQ_BUMP_RING_IDX(rx_q, ntc); 1087 1053 cleaned_count++; 1088 1054 1089 1055 /* skip if it is non EOP desc */ 1090 - if (idpf_rx_singleq_is_non_eop(rx_desc) || unlikely(!skb)) 1056 + if (idpf_rx_singleq_is_non_eop(rx_desc) || 1057 + unlikely(!xdp->data)) 1091 1058 continue; 1092 1059 1093 1060 #define IDPF_RXD_ERR_S FIELD_PREP(VIRTCHNL2_RX_BASE_DESC_QW1_ERROR_M, \ 1094 1061 VIRTCHNL2_RX_BASE_DESC_ERROR_RXE_M) 1095 1062 if (unlikely(idpf_rx_singleq_test_staterr(rx_desc, 1096 1063 IDPF_RXD_ERR_S))) { 1097 - dev_kfree_skb_any(skb); 1098 - skb = NULL; 1064 + libeth_xdp_return_buff_slow(xdp); 1099 1065 continue; 1100 1066 } 1101 1067 1102 - /* pad skb if needed (to make valid ethernet frame) */ 1103 - if (eth_skb_pad(skb)) { 1104 - skb = NULL; 1105 - continue; 1106 - } 1107 - 1108 - /* probably a little skewed due to removing CRC */ 1109 - total_rx_bytes += skb->len; 1110 - 1111 - /* protocol */ 1112 - idpf_rx_singleq_process_skb_fields(rx_q, skb, rx_desc, 1113 - fields.ptype); 1114 - 1115 - /* send completed skb up the stack */ 1116 - napi_gro_receive(rx_q->pp->p.napi, skb); 1117 - skb = NULL; 1118 - 1119 - /* update budget accounting */ 1120 - total_rx_pkts++; 1068 + idpf_xdp_run_pass(xdp, rx_q->pp->p.napi, &rs, rx_desc); 1121 1069 } 1122 1070 1123 - rx_q->skb = skb; 1124 - 1125 1071 rx_q->next_to_clean = ntc; 1072 + libeth_xdp_save_buff(&rx_q->xdp, xdp); 1126 1073 1127 1074 page_pool_nid_changed(rx_q->pp, numa_mem_id()); 1128 1075 if (cleaned_count) 1129 - failure = idpf_rx_singleq_buf_hw_alloc_all(rx_q, cleaned_count); 1076 + idpf_rx_singleq_buf_hw_alloc_all(rx_q, cleaned_count); 1130 1077 1131 1078 u64_stats_update_begin(&rx_q->stats_sync); 1132 - u64_stats_add(&rx_q->q_stats.packets, total_rx_pkts); 1133 - u64_stats_add(&rx_q->q_stats.bytes, total_rx_bytes); 1079 + u64_stats_add(&rx_q->q_stats.packets, rs.packets); 1080 + u64_stats_add(&rx_q->q_stats.bytes, rs.bytes); 1134 1081 u64_stats_update_end(&rx_q->stats_sync); 1135 1082 1136 - /* guarantee a trip back through this routine if there was a failure */ 1137 - return failure ? budget : (int)total_rx_pkts; 1083 + return rs.packets; 1138 1084 } 1139 1085 1140 1086 /**

+228 -200

drivers/net/ethernet/intel/idpf/idpf_txrx.c

··· 1 1 // SPDX-License-Identifier: GPL-2.0-only 2 2 /* Copyright (C) 2023 Intel Corporation */ 3 3 4 - #include <net/libeth/rx.h> 5 - #include <net/libeth/tx.h> 6 - 7 4 #include "idpf.h" 8 5 #include "idpf_ptp.h" 9 6 #include "idpf_virtchnl.h" 7 + #include "xdp.h" 10 8 11 9 #define idpf_tx_buf_next(buf) (*(u32 *)&(buf)->priv) 12 10 LIBETH_SQE_CHECK_PRIV(u32); ··· 60 62 static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) 61 63 { 62 64 struct libeth_sq_napi_stats ss = { }; 65 + struct xdp_frame_bulk bq; 63 66 struct libeth_cq_pp cp = { 64 67 .dev = txq->dev, 68 + .bq = &bq, 65 69 .ss = &ss, 66 70 }; 67 71 u32 i; ··· 72 72 if (!txq->tx_buf) 73 73 return; 74 74 75 + xdp_frame_bulk_init(&bq); 76 + 75 77 /* Free all the Tx buffer sk_buffs */ 76 78 for (i = 0; i < txq->buf_pool_size; i++) 77 - libeth_tx_complete(&txq->tx_buf[i], &cp); 79 + libeth_tx_complete_any(&txq->tx_buf[i], &cp); 80 + 81 + xdp_flush_frame_bulk(&bq); 78 82 79 83 kfree(txq->tx_buf); 80 84 txq->tx_buf = NULL; ··· 92 88 */ 93 89 static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) 94 90 { 91 + bool xdp = idpf_queue_has(XDP, txq); 92 + 93 + if (xdp) 94 + libeth_xdpsq_deinit_timer(txq->timer); 95 + 95 96 idpf_tx_buf_rel_all(txq); 96 - netdev_tx_reset_subqueue(txq->netdev, txq->idx); 97 + 98 + if (!xdp) 99 + netdev_tx_reset_subqueue(txq->netdev, txq->idx); 97 100 98 101 if (!txq->desc_ring) 99 102 return; 100 103 101 - if (txq->refillq) 104 + if (!xdp && txq->refillq) 102 105 kfree(txq->refillq->ring); 103 106 104 107 dmam_free_coherent(txq->dev, txq->size, txq->desc_ring, txq->dma); ··· 126 115 return; 127 116 128 117 dma_free_coherent(complq->netdev->dev.parent, complq->size, 129 - complq->comp, complq->dma); 130 - complq->comp = NULL; 118 + complq->desc_ring, complq->dma); 119 + complq->desc_ring = NULL; 131 120 complq->next_to_use = 0; 132 121 complq->next_to_clean = 0; 133 122 } ··· 257 246 static int idpf_compl_desc_alloc(const struct idpf_vport *vport, 258 247 struct idpf_compl_queue *complq) 259 248 { 260 - complq->size = array_size(complq->desc_count, sizeof(*complq->comp)); 249 + u32 desc_size; 261 250 262 - complq->comp = dma_alloc_coherent(complq->netdev->dev.parent, 263 - complq->size, &complq->dma, 264 - GFP_KERNEL); 265 - if (!complq->comp) 251 + desc_size = idpf_queue_has(FLOW_SCH_EN, complq) ? 252 + sizeof(*complq->comp) : sizeof(*complq->comp_4b); 253 + complq->size = array_size(complq->desc_count, desc_size); 254 + 255 + complq->desc_ring = dma_alloc_coherent(complq->netdev->dev.parent, 256 + complq->size, &complq->dma, 257 + GFP_KERNEL); 258 + if (!complq->desc_ring) 266 259 return -ENOMEM; 267 260 268 261 complq->next_to_use = 0; ··· 420 405 if (!rxq) 421 406 return; 422 407 423 - if (rxq->skb) { 424 - dev_kfree_skb_any(rxq->skb); 425 - rxq->skb = NULL; 426 - } 408 + libeth_xdp_return_stash(&rxq->xdp); 427 409 428 410 if (!idpf_is_queue_model_split(model)) 429 411 idpf_rx_buf_rel_all(rxq); ··· 529 517 struct libeth_fq fq = { 530 518 .count = bufq->desc_count, 531 519 .type = LIBETH_FQE_HDR, 520 + .xdp = idpf_xdp_enabled(bufq->q_vector->vport), 532 521 .nid = idpf_q_vector_to_mem(bufq->q_vector), 533 522 }; 534 523 int ret; ··· 729 716 .count = bufq->desc_count, 730 717 .type = type, 731 718 .hsplit = idpf_queue_has(HSPLIT_EN, bufq), 719 + .xdp = idpf_xdp_enabled(bufq->q_vector->vport), 732 720 .nid = idpf_q_vector_to_mem(bufq->q_vector), 733 721 }; 734 722 int ret; ··· 756 742 { 757 743 bool split = idpf_is_queue_model_split(vport->rxq_model); 758 744 int i, j, err; 745 + 746 + idpf_xdp_copy_prog_to_rqs(vport, vport->xdp_prog); 759 747 760 748 for (i = 0; i < vport->num_rxq_grp; i++) { 761 749 struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; ··· 1037 1021 */ 1038 1022 void idpf_vport_queues_rel(struct idpf_vport *vport) 1039 1023 { 1024 + idpf_xdp_copy_prog_to_rqs(vport, NULL); 1025 + 1040 1026 idpf_tx_desc_rel_all(vport); 1041 1027 idpf_rx_desc_rel_all(vport); 1028 + 1029 + idpf_xdpsqs_put(vport); 1042 1030 idpf_vport_queue_grp_rel_all(vport); 1043 1031 1044 1032 kfree(vport->txqs); ··· 1116 1096 if (idpf_is_queue_model_split(vport->rxq_model)) 1117 1097 vport->num_bufq = le16_to_cpu(vport_msg->num_rx_bufq); 1118 1098 1099 + vport->xdp_prog = config_data->xdp_prog; 1100 + if (idpf_xdp_enabled(vport)) { 1101 + vport->xdp_txq_offset = config_data->num_req_tx_qs; 1102 + vport->num_xdp_txq = le16_to_cpu(vport_msg->num_tx_q) - 1103 + vport->xdp_txq_offset; 1104 + vport->xdpsq_share = libeth_xdpsq_shared(vport->num_xdp_txq); 1105 + } else { 1106 + vport->xdp_txq_offset = 0; 1107 + vport->num_xdp_txq = 0; 1108 + vport->xdpsq_share = false; 1109 + } 1110 + 1119 1111 /* Adjust number of buffer queues per Rx queue group. */ 1120 1112 if (!idpf_is_queue_model_split(vport->rxq_model)) { 1121 1113 vport->num_bufqs_per_qgrp = 0; ··· 1199 1167 int dflt_splitq_txq_grps = 0, dflt_singleq_txqs = 0; 1200 1168 int dflt_splitq_rxq_grps = 0, dflt_singleq_rxqs = 0; 1201 1169 u16 num_req_tx_qs = 0, num_req_rx_qs = 0; 1170 + struct idpf_vport_user_config_data *user; 1202 1171 struct idpf_vport_config *vport_config; 1203 1172 u16 num_txq_grps, num_rxq_grps; 1204 - u32 num_qs; 1173 + u32 num_qs, num_xdpsq; 1205 1174 1206 1175 vport_config = adapter->vport_config[vport_idx]; 1207 1176 if (vport_config) { 1208 1177 num_req_tx_qs = vport_config->user_config.num_req_tx_qs; 1209 1178 num_req_rx_qs = vport_config->user_config.num_req_rx_qs; 1210 1179 } else { 1211 - int num_cpus; 1212 - 1213 - /* Restrict num of queues to cpus online as a default 1214 - * configuration to give best performance. User can always 1215 - * override to a max number of queues via ethtool. 1216 - */ 1217 - num_cpus = num_online_cpus(); 1180 + u32 num_cpus = netif_get_num_default_rss_queues(); 1218 1181 1219 1182 dflt_splitq_txq_grps = min_t(int, max_q->max_txq, num_cpus); 1220 1183 dflt_singleq_txqs = min_t(int, max_q->max_txq, num_cpus); ··· 1243 1216 vport_msg->num_rx_q = cpu_to_le16(num_qs); 1244 1217 vport_msg->num_rx_bufq = 0; 1245 1218 } 1219 + 1220 + if (!vport_config) 1221 + return 0; 1222 + 1223 + user = &vport_config->user_config; 1224 + user->num_req_rx_qs = le16_to_cpu(vport_msg->num_rx_q); 1225 + user->num_req_tx_qs = le16_to_cpu(vport_msg->num_tx_q); 1226 + 1227 + if (vport_config->user_config.xdp_prog) 1228 + num_xdpsq = libeth_xdpsq_num(user->num_req_rx_qs, 1229 + user->num_req_tx_qs, 1230 + vport_config->max_q.max_txq); 1231 + else 1232 + num_xdpsq = 0; 1233 + 1234 + vport_msg->num_tx_q = cpu_to_le16(user->num_req_tx_qs + num_xdpsq); 1235 + if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->txq_model))) 1236 + vport_msg->num_tx_complq = vport_msg->num_tx_q; 1246 1237 1247 1238 return 0; 1248 1239 } ··· 1311 1266 static void idpf_rxq_set_descids(const struct idpf_vport *vport, 1312 1267 struct idpf_rx_queue *q) 1313 1268 { 1314 - if (idpf_is_queue_model_split(vport->rxq_model)) { 1315 - q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M; 1316 - } else { 1317 - if (vport->base_rxd) 1318 - q->rxdids = VIRTCHNL2_RXDID_1_32B_BASE_M; 1319 - else 1320 - q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M; 1321 - } 1269 + if (idpf_is_queue_model_split(vport->rxq_model)) 1270 + return; 1271 + 1272 + if (vport->base_rxd) 1273 + q->rxdids = VIRTCHNL2_RXDID_1_32B_BASE_M; 1274 + else 1275 + q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M; 1322 1276 } 1323 1277 1324 1278 /** ··· 1525 1481 setup_rxq: 1526 1482 q->desc_count = vport->rxq_desc_count; 1527 1483 q->rx_ptype_lkup = vport->rx_ptype_lkup; 1528 - q->netdev = vport->netdev; 1529 1484 q->bufq_sets = rx_qgrp->splitq.bufq_sets; 1530 1485 q->idx = (i * num_rxq) + j; 1531 1486 q->rx_buffer_low_watermark = IDPF_LOW_WATERMARK; ··· 1585 1542 if (err) 1586 1543 goto err_out; 1587 1544 1545 + err = idpf_vport_init_fast_path_txqs(vport); 1546 + if (err) 1547 + goto err_out; 1548 + 1549 + err = idpf_xdpsqs_get(vport); 1550 + if (err) 1551 + goto err_out; 1552 + 1588 1553 err = idpf_tx_desc_alloc_all(vport); 1589 1554 if (err) 1590 1555 goto err_out; 1591 1556 1592 1557 err = idpf_rx_desc_alloc_all(vport); 1593 - if (err) 1594 - goto err_out; 1595 - 1596 - err = idpf_vport_init_fast_path_txqs(vport); 1597 1558 if (err) 1598 1559 goto err_out; 1599 1560 ··· 1607 1560 idpf_vport_queues_rel(vport); 1608 1561 1609 1562 return err; 1610 - } 1611 - 1612 - /** 1613 - * idpf_tx_handle_sw_marker - Handle queue marker packet 1614 - * @tx_q: tx queue to handle software marker 1615 - */ 1616 - static void idpf_tx_handle_sw_marker(struct idpf_tx_queue *tx_q) 1617 - { 1618 - struct idpf_netdev_priv *priv = netdev_priv(tx_q->netdev); 1619 - struct idpf_vport *vport = priv->vport; 1620 - int i; 1621 - 1622 - idpf_queue_clear(SW_MARKER, tx_q); 1623 - /* Hardware must write marker packets to all queues associated with 1624 - * completion queues. So check if all queues received marker packets 1625 - */ 1626 - for (i = 0; i < vport->num_txq; i++) 1627 - /* If we're still waiting on any other TXQ marker completions, 1628 - * just return now since we cannot wake up the marker_wq yet. 1629 - */ 1630 - if (idpf_queue_has(SW_MARKER, vport->txqs[i])) 1631 - return; 1632 - 1633 - /* Drain complete */ 1634 - set_bit(IDPF_VPORT_SW_MARKER, vport->flags); 1635 - wake_up(&vport->sw_marker_wq); 1636 1563 } 1637 1564 1638 1565 /** ··· 1786 1765 /* RS completion contains queue head for queue based scheduling or 1787 1766 * completion tag for flow based scheduling. 1788 1767 */ 1789 - u16 rs_compl_val = le16_to_cpu(desc->q_head_compl_tag.q_head); 1768 + u16 rs_compl_val = le16_to_cpu(desc->common.q_head_compl_tag.q_head); 1790 1769 1791 1770 if (!idpf_queue_has(FLOW_SCH_EN, txq)) { 1792 1771 idpf_tx_splitq_clean(txq, rs_compl_val, budget, cleaned, false); ··· 1821 1800 do { 1822 1801 struct libeth_sq_napi_stats cleaned_stats = { }; 1823 1802 struct idpf_tx_queue *tx_q; 1803 + __le16 hw_head; 1824 1804 int rel_tx_qid; 1825 - u16 hw_head; 1826 1805 u8 ctype; /* completion type */ 1827 1806 u16 gen; 1828 1807 1829 1808 /* if the descriptor isn't done, no work yet to do */ 1830 - gen = le16_get_bits(tx_desc->qid_comptype_gen, 1809 + gen = le16_get_bits(tx_desc->common.qid_comptype_gen, 1831 1810 IDPF_TXD_COMPLQ_GEN_M); 1832 1811 if (idpf_queue_has(GEN_CHK, complq) != gen) 1833 1812 break; 1834 1813 1835 1814 /* Find necessary info of TX queue to clean buffers */ 1836 - rel_tx_qid = le16_get_bits(tx_desc->qid_comptype_gen, 1815 + rel_tx_qid = le16_get_bits(tx_desc->common.qid_comptype_gen, 1837 1816 IDPF_TXD_COMPLQ_QID_M); 1838 1817 if (rel_tx_qid >= complq->txq_grp->num_txq || 1839 1818 !complq->txq_grp->txqs[rel_tx_qid]) { ··· 1843 1822 tx_q = complq->txq_grp->txqs[rel_tx_qid]; 1844 1823 1845 1824 /* Determine completion type */ 1846 - ctype = le16_get_bits(tx_desc->qid_comptype_gen, 1825 + ctype = le16_get_bits(tx_desc->common.qid_comptype_gen, 1847 1826 IDPF_TXD_COMPLQ_COMPL_TYPE_M); 1848 1827 switch (ctype) { 1849 1828 case IDPF_TXD_COMPLT_RE: 1850 - hw_head = le16_to_cpu(tx_desc->q_head_compl_tag.q_head); 1829 + hw_head = tx_desc->common.q_head_compl_tag.q_head; 1851 1830 1852 - idpf_tx_splitq_clean(tx_q, hw_head, budget, 1853 - &cleaned_stats, true); 1831 + idpf_tx_splitq_clean(tx_q, le16_to_cpu(hw_head), 1832 + budget, &cleaned_stats, true); 1854 1833 break; 1855 1834 case IDPF_TXD_COMPLT_RS: 1856 1835 idpf_tx_handle_rs_completion(tx_q, tx_desc, 1857 1836 &cleaned_stats, budget); 1858 - break; 1859 - case IDPF_TXD_COMPLT_SW_MARKER: 1860 - idpf_tx_handle_sw_marker(tx_q); 1861 1837 break; 1862 1838 default: 1863 1839 netdev_err(tx_q->netdev, ··· 1925 1907 complq->next_to_clean = ntc; 1926 1908 1927 1909 return !!complq_budget; 1910 + } 1911 + 1912 + /** 1913 + * idpf_wait_for_sw_marker_completion - wait for SW marker of disabled Tx queue 1914 + * @txq: disabled Tx queue 1915 + * 1916 + * When Tx queue is requested for disabling, the CP sends a special completion 1917 + * descriptor called "SW marker", meaning the queue is ready to be destroyed. 1918 + * If, for some reason, the marker is not received within 500 ms, break the 1919 + * polling to not hang the driver. 1920 + */ 1921 + void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq) 1922 + { 1923 + struct idpf_compl_queue *complq; 1924 + unsigned long timeout; 1925 + bool flow, gen_flag; 1926 + u32 ntc; 1927 + 1928 + if (!idpf_queue_has(SW_MARKER, txq)) 1929 + return; 1930 + 1931 + complq = idpf_queue_has(XDP, txq) ? txq->complq : txq->txq_grp->complq; 1932 + ntc = complq->next_to_clean; 1933 + 1934 + flow = idpf_queue_has(FLOW_SCH_EN, complq); 1935 + gen_flag = idpf_queue_has(GEN_CHK, complq); 1936 + 1937 + timeout = jiffies + msecs_to_jiffies(IDPF_WAIT_FOR_MARKER_TIMEO); 1938 + 1939 + do { 1940 + struct idpf_splitq_4b_tx_compl_desc *tx_desc; 1941 + struct idpf_tx_queue *target; 1942 + u32 ctype_gen, id; 1943 + 1944 + tx_desc = flow ? &complq->comp[ntc].common : 1945 + &complq->comp_4b[ntc]; 1946 + ctype_gen = le16_to_cpu(tx_desc->qid_comptype_gen); 1947 + 1948 + if (!!(ctype_gen & IDPF_TXD_COMPLQ_GEN_M) != gen_flag) { 1949 + usleep_range(500, 1000); 1950 + continue; 1951 + } 1952 + 1953 + if (FIELD_GET(IDPF_TXD_COMPLQ_COMPL_TYPE_M, ctype_gen) != 1954 + IDPF_TXD_COMPLT_SW_MARKER) 1955 + goto next; 1956 + 1957 + id = FIELD_GET(IDPF_TXD_COMPLQ_QID_M, ctype_gen); 1958 + target = complq->txq_grp->txqs[id]; 1959 + 1960 + idpf_queue_clear(SW_MARKER, target); 1961 + if (target == txq) 1962 + break; 1963 + 1964 + next: 1965 + if (unlikely(++ntc == complq->desc_count)) { 1966 + ntc = 0; 1967 + gen_flag = !gen_flag; 1968 + } 1969 + } while (time_before(jiffies, timeout)); 1970 + 1971 + idpf_queue_assign(GEN_CHK, complq, gen_flag); 1972 + complq->next_to_clean = ntc; 1928 1973 } 1929 1974 1930 1975 /** ··· 2754 2673 */ 2755 2674 netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev) 2756 2675 { 2757 - struct idpf_vport *vport = idpf_netdev_to_vport(netdev); 2676 + const struct idpf_vport *vport = idpf_netdev_to_vport(netdev); 2758 2677 struct idpf_tx_queue *tx_q; 2759 2678 2760 - if (unlikely(skb_get_queue_mapping(skb) >= vport->num_txq)) { 2679 + if (unlikely(skb_get_queue_mapping(skb) >= 2680 + vport->num_txq - vport->num_xdp_txq)) { 2761 2681 dev_kfree_skb_any(skb); 2762 2682 2763 2683 return NETDEV_TX_OK; ··· 2795 2713 { 2796 2714 u32 hash; 2797 2715 2798 - if (!libeth_rx_pt_has_hash(rxq->netdev, decoded)) 2716 + if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, decoded)) 2799 2717 return; 2800 2718 2801 2719 hash = le16_to_cpu(rx_desc->hash1) | ··· 2821 2739 bool ipv4, ipv6; 2822 2740 2823 2741 /* check if Rx checksum is enabled */ 2824 - if (!libeth_rx_pt_has_checksum(rxq->netdev, decoded)) 2742 + if (!libeth_rx_pt_has_checksum(rxq->xdp_rxq.dev, decoded)) 2825 2743 return; 2826 2744 2827 2745 /* check if HW has decoded the packet and checksum */ ··· 2993 2911 } 2994 2912 2995 2913 /** 2996 - * idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor 2914 + * __idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor 2997 2915 * @rxq: Rx descriptor ring packet is being transacted on 2998 2916 * @skb: pointer to current skb being populated 2999 2917 * @rx_desc: Receive descriptor ··· 3003 2921 * other fields within the skb. 3004 2922 */ 3005 2923 static int 3006 - idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, 3007 - const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) 2924 + __idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, 2925 + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) 3008 2926 { 3009 2927 struct libeth_rx_csum csum_bits; 3010 2928 struct libeth_rx_pt decoded; ··· 3020 2938 if (idpf_queue_has(PTP, rxq)) 3021 2939 idpf_rx_hwtstamp(rxq, rx_desc, skb); 3022 2940 3023 - skb->protocol = eth_type_trans(skb, rxq->netdev); 3024 - skb_record_rx_queue(skb, rxq->idx); 3025 - 3026 2941 if (le16_get_bits(rx_desc->hdrlen_flags, 3027 2942 VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_M)) 3028 2943 return idpf_rx_rsc(rxq, skb, rx_desc, decoded); ··· 3030 2951 return 0; 3031 2952 } 3032 2953 3033 - /** 3034 - * idpf_rx_add_frag - Add contents of Rx buffer to sk_buff as a frag 3035 - * @rx_buf: buffer containing page to add 3036 - * @skb: sk_buff to place the data into 3037 - * @size: packet length from rx_desc 3038 - * 3039 - * This function will add the data contained in rx_buf->page to the skb. 3040 - * It will just attach the page as a frag to the skb. 3041 - * The function will then update the page offset. 3042 - */ 3043 - void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb, 3044 - unsigned int size) 2954 + static bool idpf_rx_process_skb_fields(struct sk_buff *skb, 2955 + const struct libeth_xdp_buff *xdp, 2956 + struct libeth_rq_napi_stats *rs) 3045 2957 { 3046 - u32 hr = netmem_get_pp(rx_buf->netmem)->p.offset; 2958 + struct idpf_rx_queue *rxq; 3047 2959 3048 - skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, rx_buf->netmem, 3049 - rx_buf->offset + hr, size, rx_buf->truesize); 2960 + rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); 2961 + 2962 + return !__idpf_rx_process_skb_fields(rxq, skb, xdp->desc); 3050 2963 } 2964 + 2965 + LIBETH_XDP_DEFINE_START(); 2966 + LIBETH_XDP_DEFINE_RUN(static idpf_xdp_run_pass, idpf_xdp_run_prog, 2967 + idpf_xdp_tx_flush_bulk, idpf_rx_process_skb_fields); 2968 + LIBETH_XDP_DEFINE_FINALIZE(static idpf_xdp_finalize_rx, idpf_xdp_tx_flush_bulk, 2969 + idpf_xdp_tx_finalize); 2970 + LIBETH_XDP_DEFINE_END(); 3051 2971 3052 2972 /** 3053 2973 * idpf_rx_hsplit_wa - handle header buffer overflows and split errors ··· 3090 3012 } 3091 3013 3092 3014 /** 3093 - * idpf_rx_build_skb - Allocate skb and populate it from header buffer 3094 - * @buf: Rx buffer to pull data from 3095 - * @size: the length of the packet 3096 - * 3097 - * This function allocates an skb. It then populates it with the page data from 3098 - * the current receive descriptor, taking care to set up the skb correctly. 3099 - */ 3100 - struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size) 3101 - { 3102 - struct page *buf_page = __netmem_to_page(buf->netmem); 3103 - u32 hr = pp_page_to_nmdesc(buf_page)->pp->p.offset; 3104 - struct sk_buff *skb; 3105 - void *va; 3106 - 3107 - va = page_address(buf_page) + buf->offset; 3108 - prefetch(va + hr); 3109 - 3110 - skb = napi_build_skb(va, buf->truesize); 3111 - if (unlikely(!skb)) 3112 - return NULL; 3113 - 3114 - skb_mark_for_recycle(skb); 3115 - 3116 - skb_reserve(skb, hr); 3117 - __skb_put(skb, size); 3118 - 3119 - return skb; 3120 - } 3121 - 3122 - /** 3123 3015 * idpf_rx_splitq_test_staterr - tests bits in Rx descriptor 3124 3016 * status and error fields 3125 3017 * @stat_err_field: field from descriptor to test bits in ··· 3130 3082 */ 3131 3083 static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) 3132 3084 { 3133 - int total_rx_bytes = 0, total_rx_pkts = 0; 3134 3085 struct idpf_buf_queue *rx_bufq = NULL; 3135 - struct sk_buff *skb = rxq->skb; 3086 + struct libeth_rq_napi_stats rs = { }; 3136 3087 u16 ntc = rxq->next_to_clean; 3088 + LIBETH_XDP_ONSTACK_BUFF(xdp); 3089 + LIBETH_XDP_ONSTACK_BULK(bq); 3090 + 3091 + libeth_xdp_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, 3092 + rxq->xdpsqs, rxq->num_xdp_txq); 3093 + libeth_xdp_init_buff(xdp, &rxq->xdp, &rxq->xdp_rxq); 3137 3094 3138 3095 /* Process Rx packets bounded by budget */ 3139 - while (likely(total_rx_pkts < budget)) { 3096 + while (likely(rs.packets < budget)) { 3140 3097 struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc; 3141 3098 struct libeth_fqe *hdr, *rx_buf = NULL; 3142 3099 struct idpf_sw_queue *refillq = NULL; ··· 3155 3102 /* get the Rx desc from Rx queue based on 'next_to_clean' */ 3156 3103 rx_desc = &rxq->rx[ntc].flex_adv_nic_3_wb; 3157 3104 3158 - /* This memory barrier is needed to keep us from reading 3159 - * any other fields out of the rx_desc 3160 - */ 3161 - dma_rmb(); 3162 - 3163 3105 /* if the descriptor isn't done, no work yet to do */ 3164 3106 gen_id = le16_get_bits(rx_desc->pktlen_gen_bufq_id, 3165 3107 VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_M); 3166 - 3167 3108 if (idpf_queue_has(GEN_CHK, rxq) != gen_id) 3168 3109 break; 3110 + 3111 + dma_rmb(); 3169 3112 3170 3113 rxdid = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_RXDID_M, 3171 3114 rx_desc->rxdid_ucast); ··· 3207 3158 3208 3159 hdr = &rx_bufq->hdr_buf[buf_id]; 3209 3160 3210 - if (unlikely(!hdr_len && !skb)) { 3161 + if (unlikely(!hdr_len && !xdp->data)) { 3211 3162 hdr_len = idpf_rx_hsplit_wa(hdr, rx_buf, pkt_len); 3212 3163 /* If failed, drop both buffers by setting len to 0 */ 3213 3164 pkt_len -= hdr_len ? : pkt_len; ··· 3217 3168 u64_stats_update_end(&rxq->stats_sync); 3218 3169 } 3219 3170 3220 - if (libeth_rx_sync_for_cpu(hdr, hdr_len)) { 3221 - skb = idpf_rx_build_skb(hdr, hdr_len); 3222 - if (!skb) 3223 - break; 3224 - 3225 - u64_stats_update_begin(&rxq->stats_sync); 3226 - u64_stats_inc(&rxq->q_stats.hsplit_pkts); 3227 - u64_stats_update_end(&rxq->stats_sync); 3228 - } 3171 + if (libeth_xdp_process_buff(xdp, hdr, hdr_len)) 3172 + rs.hsplit++; 3229 3173 3230 3174 hdr->netmem = 0; 3231 3175 3232 3176 payload: 3233 - if (!libeth_rx_sync_for_cpu(rx_buf, pkt_len)) 3234 - goto skip_data; 3235 - 3236 - if (skb) 3237 - idpf_rx_add_frag(rx_buf, skb, pkt_len); 3238 - else 3239 - skb = idpf_rx_build_skb(rx_buf, pkt_len); 3240 - 3241 - /* exit if we failed to retrieve a buffer */ 3242 - if (!skb) 3243 - break; 3244 - 3245 - skip_data: 3177 + libeth_xdp_process_buff(xdp, rx_buf, pkt_len); 3246 3178 rx_buf->netmem = 0; 3247 3179 3248 3180 idpf_post_buf_refill(refillq, buf_id); 3249 3181 IDPF_RX_BUMP_NTC(rxq, ntc); 3250 3182 3251 3183 /* skip if it is non EOP desc */ 3252 - if (!idpf_rx_splitq_is_eop(rx_desc) || unlikely(!skb)) 3184 + if (!idpf_rx_splitq_is_eop(rx_desc) || unlikely(!xdp->data)) 3253 3185 continue; 3254 3186 3255 - /* pad skb if needed (to make valid ethernet frame) */ 3256 - if (eth_skb_pad(skb)) { 3257 - skb = NULL; 3258 - continue; 3259 - } 3260 - 3261 - /* probably a little skewed due to removing CRC */ 3262 - total_rx_bytes += skb->len; 3263 - 3264 - /* protocol */ 3265 - if (unlikely(idpf_rx_process_skb_fields(rxq, skb, rx_desc))) { 3266 - dev_kfree_skb_any(skb); 3267 - skb = NULL; 3268 - continue; 3269 - } 3270 - 3271 - /* send completed skb up the stack */ 3272 - napi_gro_receive(rxq->napi, skb); 3273 - skb = NULL; 3274 - 3275 - /* update budget accounting */ 3276 - total_rx_pkts++; 3187 + idpf_xdp_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc); 3277 3188 } 3278 3189 3279 - rxq->next_to_clean = ntc; 3190 + idpf_xdp_finalize_rx(&bq); 3280 3191 3281 - rxq->skb = skb; 3192 + rxq->next_to_clean = ntc; 3193 + libeth_xdp_save_buff(&rxq->xdp, xdp); 3194 + 3282 3195 u64_stats_update_begin(&rxq->stats_sync); 3283 - u64_stats_add(&rxq->q_stats.packets, total_rx_pkts); 3284 - u64_stats_add(&rxq->q_stats.bytes, total_rx_bytes); 3196 + u64_stats_add(&rxq->q_stats.packets, rs.packets); 3197 + u64_stats_add(&rxq->q_stats.bytes, rs.bytes); 3198 + u64_stats_add(&rxq->q_stats.hsplit_pkts, rs.hsplit); 3285 3199 u64_stats_update_end(&rxq->stats_sync); 3286 3200 3287 - /* guarantee a trip back through this routine if there was a failure */ 3288 - return total_rx_pkts; 3201 + return rs.packets; 3289 3202 } 3290 3203 3291 3204 /** ··· 3445 3434 vport->q_vectors = NULL; 3446 3435 } 3447 3436 3437 + static void idpf_q_vector_set_napi(struct idpf_q_vector *q_vector, bool link) 3438 + { 3439 + struct napi_struct *napi = link ? &q_vector->napi : NULL; 3440 + struct net_device *dev = q_vector->vport->netdev; 3441 + 3442 + for (u32 i = 0; i < q_vector->num_rxq; i++) 3443 + netif_queue_set_napi(dev, q_vector->rx[i]->idx, 3444 + NETDEV_QUEUE_TYPE_RX, napi); 3445 + 3446 + for (u32 i = 0; i < q_vector->num_txq; i++) 3447 + netif_queue_set_napi(dev, q_vector->tx[i]->idx, 3448 + NETDEV_QUEUE_TYPE_TX, napi); 3449 + } 3450 + 3448 3451 /** 3449 3452 * idpf_vport_intr_rel_irq - Free the IRQ association with the OS 3450 3453 * @vport: main vport structure ··· 3479 3454 vidx = vport->q_vector_idxs[vector]; 3480 3455 irq_num = adapter->msix_entries[vidx].vector; 3481 3456 3457 + idpf_q_vector_set_napi(q_vector, false); 3482 3458 kfree(free_irq(irq_num, q_vector)); 3483 3459 } 3484 3460 } ··· 3492 3466 { 3493 3467 struct idpf_q_vector *q_vector = vport->q_vectors; 3494 3468 int q_idx; 3469 + 3470 + writel(0, vport->noirq_dyn_ctl); 3495 3471 3496 3472 for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) 3497 3473 writel(0, q_vector[q_idx].intr_reg.dyn_ctl); ··· 3669 3641 "Request_irq failed, error: %d\n", err); 3670 3642 goto free_q_irqs; 3671 3643 } 3644 + 3645 + idpf_q_vector_set_napi(q_vector, true); 3672 3646 } 3673 3647 3674 3648 return 0; ··· 3738 3708 if (qv->num_txq || qv->num_rxq) 3739 3709 idpf_vport_intr_update_itr_ena_irq(qv); 3740 3710 } 3711 + 3712 + writel(vport->noirq_dyn_ctl_ena, vport->noirq_dyn_ctl); 3741 3713 } 3742 3714 3743 3715 /** ··· 3933 3901 return budget; 3934 3902 } 3935 3903 3936 - /* Switch to poll mode in the tear-down path after sending disable 3937 - * queues virtchnl message, as the interrupts will be disabled after 3938 - * that. 3939 - */ 3940 - if (unlikely(q_vector->num_txq && idpf_queue_has(POLL_MODE, 3941 - q_vector->tx[0]))) 3942 - return budget; 3943 - 3944 3904 work_done = min_t(int, work_done, budget - 1); 3945 3905 3946 3906 /* Exit the polling mode, but don't re-enable interrupts if stack might ··· 3954 3930 */ 3955 3931 static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) 3956 3932 { 3933 + u16 num_txq_grp = vport->num_txq_grp - vport->num_xdp_txq; 3957 3934 bool split = idpf_is_queue_model_split(vport->rxq_model); 3958 - u16 num_txq_grp = vport->num_txq_grp; 3959 3935 struct idpf_rxq_group *rx_qgrp; 3960 3936 struct idpf_txq_group *tx_qgrp; 3961 3937 u32 i, qv_idx, q_index; ··· 4051 4027 for (i = 0; i < vport->num_q_vectors; i++) 4052 4028 vport->q_vectors[i].v_idx = vport->q_vector_idxs[i]; 4053 4029 4030 + vport->noirq_v_idx = vport->q_vector_idxs[i]; 4031 + 4054 4032 return 0; 4055 4033 } 4056 4034 ··· 4065 4039 4066 4040 for (i = 0; i < vport->num_q_vectors; i++) 4067 4041 vport->q_vectors[i].v_idx = vecids[vport->q_vector_idxs[i]]; 4042 + 4043 + vport->noirq_v_idx = vecids[vport->q_vector_idxs[i]]; 4068 4044 4069 4045 kfree(vecids); 4070 4046

+87 -53

drivers/net/ethernet/intel/idpf/idpf_txrx.h

··· 7 7 #include <linux/dim.h> 8 8 9 9 #include <net/libeth/cache.h> 10 - #include <net/tcp.h> 10 + #include <net/libeth/types.h> 11 11 #include <net/netdev_queues.h> 12 + #include <net/tcp.h> 13 + #include <net/xdp.h> 12 14 13 15 #include "idpf_lan_txrx.h" 14 16 #include "virtchnl2_lan_desc.h" ··· 60 58 #define IDPF_MBX_Q_VEC 1 61 59 #define IDPF_MIN_Q_VEC 1 62 60 #define IDPF_MIN_RDMA_VEC 2 61 + /* Data vector for NOIRQ queues */ 62 + #define IDPF_RESERVED_VECS 1 63 63 64 64 #define IDPF_DFLT_TX_Q_DESC_COUNT 512 65 65 #define IDPF_DFLT_TX_COMPLQ_DESC_COUNT 512 ··· 279 275 * bit and Q_RFL_GEN is the SW bit. 280 276 * @__IDPF_Q_FLOW_SCH_EN: Enable flow scheduling 281 277 * @__IDPF_Q_SW_MARKER: Used to indicate TX queue marker completions 282 - * @__IDPF_Q_POLL_MODE: Enable poll mode 283 278 * @__IDPF_Q_CRC_EN: enable CRC offload in singleq mode 284 279 * @__IDPF_Q_HSPLIT_EN: enable header split on Rx (splitq) 285 280 * @__IDPF_Q_PTP: indicates whether the Rx timestamping is enabled for the 286 281 * queue 282 + * @__IDPF_Q_NOIRQ: queue is polling-driven and has no interrupt 283 + * @__IDPF_Q_XDP: this is an XDP queue 287 284 * @__IDPF_Q_FLAGS_NBITS: Must be last 288 285 */ 289 286 enum idpf_queue_flags_t { ··· 292 287 __IDPF_Q_RFL_GEN_CHK, 293 288 __IDPF_Q_FLOW_SCH_EN, 294 289 __IDPF_Q_SW_MARKER, 295 - __IDPF_Q_POLL_MODE, 296 290 __IDPF_Q_CRC_EN, 297 291 __IDPF_Q_HSPLIT_EN, 298 292 __IDPF_Q_PTP, 293 + __IDPF_Q_NOIRQ, 294 + __IDPF_Q_XDP, 299 295 300 296 __IDPF_Q_FLAGS_NBITS, 301 297 }; ··· 467 461 * @desc_ring: virtual descriptor ring address 468 462 * @bufq_sets: Pointer to the array of buffer queues in splitq mode 469 463 * @napi: NAPI instance corresponding to this queue (splitq) 464 + * @xdp_prog: attached XDP program 470 465 * @rx_buf: See struct &libeth_fqe 471 466 * @pp: Page pool pointer in singleq mode 472 - * @netdev: &net_device corresponding to this queue 473 467 * @tail: Tail offset. Used for both queue models single and split. 474 468 * @flags: See enum idpf_queue_flags_t 475 469 * @idx: For RX queue, it is used to index to total RX queue across groups and 476 470 * used for skb reporting. 477 471 * @desc_count: Number of descriptors 472 + * @num_xdp_txq: total number of XDP Tx queues 473 + * @xdpsqs: shortcut for XDP Tx queues array 478 474 * @rxdids: Supported RX descriptor ids 475 + * @truesize: data buffer truesize in singleq 479 476 * @rx_ptype_lkup: LUT of Rx ptypes 477 + * @xdp_rxq: XDP queue info 480 478 * @next_to_use: Next descriptor to use 481 479 * @next_to_clean: Next descriptor to clean 482 480 * @next_to_alloc: RX buffer to allocate at 483 - * @skb: Pointer to the skb 484 - * @truesize: data buffer truesize in singleq 481 + * @xdp: XDP buffer with the current frame 485 482 * @cached_phc_time: Cached PHC time for the Rx queue 486 483 * @stats_sync: See struct u64_stats_sync 487 484 * @q_stats: See union idpf_rx_queue_stats ··· 509 500 struct { 510 501 struct idpf_bufq_set *bufq_sets; 511 502 struct napi_struct *napi; 503 + struct bpf_prog __rcu *xdp_prog; 512 504 }; 513 505 struct { 514 506 struct libeth_fqe *rx_buf; 515 507 struct page_pool *pp; 508 + void __iomem *tail; 516 509 }; 517 510 }; 518 - struct net_device *netdev; 519 - void __iomem *tail; 520 511 521 512 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 522 513 u16 idx; 523 514 u16 desc_count; 524 515 525 - u32 rxdids; 516 + u32 num_xdp_txq; 517 + union { 518 + struct idpf_tx_queue **xdpsqs; 519 + struct { 520 + u32 rxdids; 521 + u32 truesize; 522 + }; 523 + }; 526 524 const struct libeth_rx_pt *rx_ptype_lkup; 525 + 526 + struct xdp_rxq_info xdp_rxq; 527 527 __cacheline_group_end_aligned(read_mostly); 528 528 529 529 __cacheline_group_begin_aligned(read_write); 530 - u16 next_to_use; 531 - u16 next_to_clean; 532 - u16 next_to_alloc; 530 + u32 next_to_use; 531 + u32 next_to_clean; 532 + u32 next_to_alloc; 533 533 534 - struct sk_buff *skb; 535 - u32 truesize; 534 + struct libeth_xdp_buff_stash xdp; 536 535 u64 cached_phc_time; 537 536 538 537 struct u64_stats_sync stats_sync; ··· 560 543 u16 rx_max_pkt_size; 561 544 __cacheline_group_end_aligned(cold); 562 545 }; 563 - libeth_cacheline_set_assert(struct idpf_rx_queue, 64, 564 - 88 + sizeof(struct u64_stats_sync), 546 + libeth_cacheline_set_assert(struct idpf_rx_queue, 547 + ALIGN(64, __alignof(struct xdp_rxq_info)) + 548 + sizeof(struct xdp_rxq_info), 549 + 96 + offsetof(struct idpf_rx_queue, q_stats) - 550 + offsetofend(struct idpf_rx_queue, cached_phc_time), 565 551 32); 566 552 567 553 /** ··· 576 556 * @desc_ring: virtual descriptor ring address 577 557 * @tx_buf: See struct idpf_tx_buf 578 558 * @txq_grp: See struct idpf_txq_group 559 + * @complq: corresponding completion queue in XDP mode 579 560 * @dev: Device back pointer for DMA mapping 580 561 * @tail: Tail offset. Used for both queue models single and split 581 562 * @flags: See enum idpf_queue_flags_t ··· 584 563 * hot path TX pointers stored in vport. Used in both singleq/splitq. 585 564 * @desc_count: Number of descriptors 586 565 * @tx_min_pkt_len: Min supported packet length 587 - * @compl_tag_gen_s: Completion tag generation bit 588 - * The format of the completion tag will change based on the TXQ 589 - * descriptor ring size so that we can maintain roughly the same level 590 - * of "uniqueness" across all descriptor sizes. For example, if the 591 - * TXQ descriptor ring size is 64 (the minimum size supported), the 592 - * completion tag will be formatted as below: 593 - * 15 6 5 0 594 - * -------------------------------- 595 - * | GEN=0-1023 |IDX = 0-63| 596 - * -------------------------------- 597 - * 598 - * This gives us 64*1024 = 65536 possible unique values. Similarly, if 599 - * the TXQ descriptor ring size is 8160 (the maximum size supported), 600 - * the completion tag will be formatted as below: 601 - * 15 13 12 0 602 - * -------------------------------- 603 - * |GEN | IDX = 0-8159 | 604 - * -------------------------------- 605 - * 606 - * This gives us 8*8160 = 65280 possible unique values. 566 + * @thresh: XDP queue cleaning threshold 607 567 * @netdev: &net_device corresponding to this queue 608 568 * @next_to_use: Next descriptor to use 609 569 * @next_to_clean: Next descriptor to clean ··· 601 599 * @clean_budget: singleq only, queue cleaning budget 602 600 * @cleaned_pkts: Number of packets cleaned for the above said case 603 601 * @refillq: Pointer to refill queue 602 + * @pending: number of pending descriptors to send in QB 603 + * @xdp_tx: number of pending &xdp_buff or &xdp_frame buffers 604 + * @timer: timer for XDP Tx queue cleanup 605 + * @xdp_lock: lock for XDP Tx queues sharing 604 606 * @cached_tstamp_caps: Tx timestamp capabilities negotiated with the CP 605 607 * @tstamp_task: Work that handles Tx timestamp read 606 608 * @stats_sync: See struct u64_stats_sync ··· 626 620 void *desc_ring; 627 621 }; 628 622 struct libeth_sqe *tx_buf; 629 - struct idpf_txq_group *txq_grp; 623 + union { 624 + struct idpf_txq_group *txq_grp; 625 + struct idpf_compl_queue *complq; 626 + }; 630 627 struct device *dev; 631 628 void __iomem *tail; 632 629 ··· 637 628 u16 idx; 638 629 u16 desc_count; 639 630 640 - u16 tx_min_pkt_len; 631 + union { 632 + u16 tx_min_pkt_len; 633 + u32 thresh; 634 + }; 641 635 642 636 struct net_device *netdev; 643 637 __cacheline_group_end_aligned(read_mostly); 644 638 645 639 __cacheline_group_begin_aligned(read_write); 646 - u16 next_to_use; 647 - u16 next_to_clean; 648 - u16 last_re; 649 - u16 tx_max_bufs; 640 + u32 next_to_use; 641 + u32 next_to_clean; 650 642 651 643 union { 652 - u32 cleaned_bytes; 653 - u32 clean_budget; 654 - }; 655 - u16 cleaned_pkts; 644 + struct { 645 + u16 last_re; 646 + u16 tx_max_bufs; 656 647 657 - struct idpf_sw_queue *refillq; 648 + union { 649 + u32 cleaned_bytes; 650 + u32 clean_budget; 651 + }; 652 + u16 cleaned_pkts; 653 + 654 + struct idpf_sw_queue *refillq; 655 + }; 656 + struct { 657 + u32 pending; 658 + u32 xdp_tx; 659 + 660 + struct libeth_xdpsq_timer *timer; 661 + struct libeth_xdpsq_lock xdp_lock; 662 + }; 663 + }; 658 664 659 665 struct idpf_ptp_vport_tx_tstamp_caps *cached_tstamp_caps; 660 666 struct work_struct *tstamp_task; ··· 688 664 __cacheline_group_end_aligned(cold); 689 665 }; 690 666 libeth_cacheline_set_assert(struct idpf_tx_queue, 64, 691 - 104 + sizeof(struct u64_stats_sync), 667 + 104 + 668 + offsetof(struct idpf_tx_queue, cached_tstamp_caps) - 669 + offsetofend(struct idpf_tx_queue, timer) + 670 + offsetof(struct idpf_tx_queue, q_stats) - 671 + offsetofend(struct idpf_tx_queue, tstamp_task), 692 672 32); 693 673 694 674 /** ··· 756 728 757 729 /** 758 730 * struct idpf_compl_queue - software structure representing a completion queue 759 - * @comp: completion descriptor array 731 + * @comp: 8-byte completion descriptor array 732 + * @comp_4b: 4-byte completion descriptor array 733 + * @desc_ring: virtual descriptor ring address 760 734 * @txq_grp: See struct idpf_txq_group 761 735 * @flags: See enum idpf_queue_flags_t 762 736 * @desc_count: Number of descriptors ··· 778 748 */ 779 749 struct idpf_compl_queue { 780 750 __cacheline_group_begin_aligned(read_mostly); 781 - struct idpf_splitq_tx_compl_desc *comp; 751 + union { 752 + struct idpf_splitq_tx_compl_desc *comp; 753 + struct idpf_splitq_4b_tx_compl_desc *comp_4b; 754 + 755 + void *desc_ring; 756 + }; 782 757 struct idpf_txq_group *txq_grp; 783 758 784 759 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); ··· 1047 1012 int idpf_init_rss(struct idpf_vport *vport); 1048 1013 void idpf_deinit_rss(struct idpf_vport *vport); 1049 1014 int idpf_rx_bufs_init_all(struct idpf_vport *vport); 1050 - void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb, 1051 - unsigned int size); 1052 - struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size); 1053 1015 void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, 1054 1016 bool xmit_more); 1055 1017 unsigned int idpf_size_to_txd_count(unsigned int size); ··· 1060 1028 bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq, 1061 1029 u16 cleaned_count); 1062 1030 int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off); 1031 + 1032 + void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq); 1063 1033 1064 1034 #endif /* !_IDPF_TXRX_H_ */

+10 -1

drivers/net/ethernet/intel/idpf/idpf_vf_dev.c

··· 76 76 int num_vecs = vport->num_q_vectors; 77 77 struct idpf_vec_regs *reg_vals; 78 78 int num_regs, i, err = 0; 79 - u32 rx_itr, tx_itr; 79 + u32 rx_itr, tx_itr, val; 80 80 u16 total_vecs; 81 81 82 82 total_vecs = idpf_get_reserved_vecs(vport->adapter); ··· 119 119 intr->rx_itr = idpf_get_reg_addr(adapter, rx_itr); 120 120 intr->tx_itr = idpf_get_reg_addr(adapter, tx_itr); 121 121 } 122 + 123 + /* Data vector for NOIRQ queues */ 124 + 125 + val = reg_vals[vport->q_vector_idxs[i] - IDPF_MBX_Q_VEC].dyn_ctl_reg; 126 + vport->noirq_dyn_ctl = idpf_get_reg_addr(adapter, val); 127 + 128 + val = VF_INT_DYN_CTLN_WB_ON_ITR_M | VF_INT_DYN_CTLN_INTENA_MSK_M | 129 + FIELD_PREP(VF_INT_DYN_CTLN_ITR_INDX_M, IDPF_NO_ITR_UPDATE_IDX); 130 + vport->noirq_dyn_ctl_ena = val; 122 131 123 132 free_reg_vals: 124 133 kfree(reg_vals);

+98 -75

drivers/net/ethernet/intel/idpf/idpf_virtchnl.c

··· 724 724 **/ 725 725 static int idpf_wait_for_marker_event(struct idpf_vport *vport) 726 726 { 727 - int event; 728 - int i; 727 + bool markers_rcvd = true; 729 728 730 - for (i = 0; i < vport->num_txq; i++) 731 - idpf_queue_set(SW_MARKER, vport->txqs[i]); 729 + for (u32 i = 0; i < vport->num_txq; i++) { 730 + struct idpf_tx_queue *txq = vport->txqs[i]; 732 731 733 - event = wait_event_timeout(vport->sw_marker_wq, 734 - test_and_clear_bit(IDPF_VPORT_SW_MARKER, 735 - vport->flags), 736 - msecs_to_jiffies(500)); 732 + idpf_queue_set(SW_MARKER, txq); 733 + idpf_wait_for_sw_marker_completion(txq); 734 + markers_rcvd &= !idpf_queue_has(SW_MARKER, txq); 735 + } 737 736 738 - for (i = 0; i < vport->num_txq; i++) 739 - idpf_queue_clear(POLL_MODE, vport->txqs[i]); 740 - 741 - if (event) 737 + if (markers_rcvd) 742 738 return 0; 743 739 744 740 dev_warn(&vport->adapter->pdev->dev, "Failed to receive marker packets\n"); ··· 1057 1061 struct idpf_avail_queue_info *avail_queues = &adapter->avail_queues; 1058 1062 struct virtchnl2_get_capabilities *caps = &adapter->caps; 1059 1063 u16 default_vports = idpf_get_default_vports(adapter); 1060 - int max_rx_q, max_tx_q; 1064 + u32 max_rx_q, max_tx_q, max_buf_q, max_compl_q; 1061 1065 1062 1066 mutex_lock(&adapter->queue_lock); 1063 1067 1068 + /* Caps are device-wide. Give each vport an equal piece */ 1064 1069 max_rx_q = le16_to_cpu(caps->max_rx_q) / default_vports; 1065 1070 max_tx_q = le16_to_cpu(caps->max_tx_q) / default_vports; 1066 - if (adapter->num_alloc_vports < default_vports) { 1067 - max_q->max_rxq = min_t(u16, max_rx_q, IDPF_MAX_Q); 1068 - max_q->max_txq = min_t(u16, max_tx_q, IDPF_MAX_Q); 1069 - } else { 1070 - max_q->max_rxq = IDPF_MIN_Q; 1071 - max_q->max_txq = IDPF_MIN_Q; 1071 + max_buf_q = le16_to_cpu(caps->max_rx_bufq) / default_vports; 1072 + max_compl_q = le16_to_cpu(caps->max_tx_complq) / default_vports; 1073 + 1074 + if (adapter->num_alloc_vports >= default_vports) { 1075 + max_rx_q = IDPF_MIN_Q; 1076 + max_tx_q = IDPF_MIN_Q; 1072 1077 } 1073 - max_q->max_bufq = max_q->max_rxq * IDPF_MAX_BUFQS_PER_RXQ_GRP; 1074 - max_q->max_complq = max_q->max_txq; 1078 + 1079 + /* 1080 + * Harmonize the numbers. The current implementation always creates 1081 + * `IDPF_MAX_BUFQS_PER_RXQ_GRP` buffer queues for each Rx queue and 1082 + * one completion queue for each Tx queue for best performance. 1083 + * If less buffer or completion queues is available, cap the number 1084 + * of the corresponding Rx/Tx queues. 1085 + */ 1086 + max_rx_q = min(max_rx_q, max_buf_q / IDPF_MAX_BUFQS_PER_RXQ_GRP); 1087 + max_tx_q = min(max_tx_q, max_compl_q); 1088 + 1089 + max_q->max_rxq = max_rx_q; 1090 + max_q->max_txq = max_tx_q; 1091 + max_q->max_bufq = max_rx_q * IDPF_MAX_BUFQS_PER_RXQ_GRP; 1092 + max_q->max_complq = max_tx_q; 1075 1093 1076 1094 if (avail_queues->avail_rxq < max_q->max_rxq || 1077 1095 avail_queues->avail_txq < max_q->max_txq || ··· 1516 1506 xn_params.vc_op = VIRTCHNL2_OP_DESTROY_VPORT; 1517 1507 xn_params.send_buf.iov_base = &v_id; 1518 1508 xn_params.send_buf.iov_len = sizeof(v_id); 1519 - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; 1509 + xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; 1520 1510 reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); 1521 1511 1522 1512 return reply_sz < 0 ? reply_sz : 0; ··· 1564 1554 xn_params.vc_op = VIRTCHNL2_OP_DISABLE_VPORT; 1565 1555 xn_params.send_buf.iov_base = &v_id; 1566 1556 xn_params.send_buf.iov_len = sizeof(v_id); 1567 - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; 1557 + xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; 1568 1558 reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); 1569 1559 1570 1560 return reply_sz < 0 ? reply_sz : 0; ··· 1748 1738 for (j = 0; j < num_rxq; j++, k++) { 1749 1739 const struct idpf_bufq_set *sets; 1750 1740 struct idpf_rx_queue *rxq; 1741 + u32 rxdids; 1751 1742 1752 1743 if (!idpf_is_queue_model_split(vport->rxq_model)) { 1753 1744 rxq = rx_qgrp->singleq.rxqs[j]; 1745 + rxdids = rxq->rxdids; 1746 + 1754 1747 goto common_qi_fields; 1755 1748 } 1756 1749 ··· 1786 1773 cpu_to_le16(rxq->rx_hbuf_size); 1787 1774 } 1788 1775 1776 + rxdids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M; 1777 + 1789 1778 common_qi_fields: 1790 1779 qi[k].queue_id = cpu_to_le32(rxq->q_id); 1791 1780 qi[k].model = cpu_to_le16(vport->rxq_model); ··· 1798 1783 qi[k].data_buffer_size = cpu_to_le32(rxq->rx_buf_size); 1799 1784 qi[k].qflags |= 1800 1785 cpu_to_le16(VIRTCHNL2_RX_DESC_SIZE_32BYTE); 1801 - qi[k].desc_ids = cpu_to_le64(rxq->rxdids); 1786 + qi[k].desc_ids = cpu_to_le64(rxdids); 1802 1787 } 1803 1788 } 1804 1789 ··· 1860 1845 struct virtchnl2_del_ena_dis_queues *eq __free(kfree) = NULL; 1861 1846 struct virtchnl2_queue_chunk *qc __free(kfree) = NULL; 1862 1847 u32 num_msgs, num_chunks, num_txq, num_rxq, num_q; 1863 - struct idpf_vc_xn_params xn_params = {}; 1848 + struct idpf_vc_xn_params xn_params = { 1849 + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 1850 + }; 1864 1851 struct virtchnl2_queue_chunks *qcs; 1865 1852 u32 config_sz, chunk_sz, buf_sz; 1866 1853 ssize_t reply_sz; ··· 1963 1946 if (!eq) 1964 1947 return -ENOMEM; 1965 1948 1966 - if (ena) { 1949 + if (ena) 1967 1950 xn_params.vc_op = VIRTCHNL2_OP_ENABLE_QUEUES; 1968 - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; 1969 - } else { 1951 + else 1970 1952 xn_params.vc_op = VIRTCHNL2_OP_DISABLE_QUEUES; 1971 - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; 1972 - } 1973 1953 1974 1954 for (i = 0, k = 0; i < num_msgs; i++) { 1975 1955 memset(eq, 0, buf_sz); ··· 2004 1990 { 2005 1991 struct virtchnl2_queue_vector_maps *vqvm __free(kfree) = NULL; 2006 1992 struct virtchnl2_queue_vector *vqv __free(kfree) = NULL; 2007 - struct idpf_vc_xn_params xn_params = {}; 1993 + struct idpf_vc_xn_params xn_params = { 1994 + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 1995 + }; 2008 1996 u32 config_sz, chunk_sz, buf_sz; 2009 1997 u32 num_msgs, num_chunks, num_q; 2010 1998 ssize_t reply_sz; ··· 2023 2007 struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; 2024 2008 2025 2009 for (j = 0; j < tx_qgrp->num_txq; j++, k++) { 2010 + const struct idpf_tx_queue *txq = tx_qgrp->txqs[j]; 2011 + const struct idpf_q_vector *vec; 2012 + u32 v_idx, tx_itr_idx; 2013 + 2026 2014 vqv[k].queue_type = 2027 2015 cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); 2028 - vqv[k].queue_id = cpu_to_le32(tx_qgrp->txqs[j]->q_id); 2016 + vqv[k].queue_id = cpu_to_le32(txq->q_id); 2029 2017 2030 - if (idpf_is_queue_model_split(vport->txq_model)) { 2031 - vqv[k].vector_id = 2032 - cpu_to_le16(tx_qgrp->complq->q_vector->v_idx); 2033 - vqv[k].itr_idx = 2034 - cpu_to_le32(tx_qgrp->complq->q_vector->tx_itr_idx); 2018 + if (idpf_queue_has(NOIRQ, txq)) 2019 + vec = NULL; 2020 + else if (idpf_queue_has(XDP, txq)) 2021 + vec = txq->complq->q_vector; 2022 + else if (idpf_is_queue_model_split(vport->txq_model)) 2023 + vec = txq->txq_grp->complq->q_vector; 2024 + else 2025 + vec = txq->q_vector; 2026 + 2027 + if (vec) { 2028 + v_idx = vec->v_idx; 2029 + tx_itr_idx = vec->tx_itr_idx; 2035 2030 } else { 2036 - vqv[k].vector_id = 2037 - cpu_to_le16(tx_qgrp->txqs[j]->q_vector->v_idx); 2038 - vqv[k].itr_idx = 2039 - cpu_to_le32(tx_qgrp->txqs[j]->q_vector->tx_itr_idx); 2031 + v_idx = vport->noirq_v_idx; 2032 + tx_itr_idx = VIRTCHNL2_ITR_IDX_1; 2040 2033 } 2034 + 2035 + vqv[k].vector_id = cpu_to_le16(v_idx); 2036 + vqv[k].itr_idx = cpu_to_le32(tx_itr_idx); 2041 2037 } 2042 2038 } 2043 - 2044 - if (vport->num_txq != k) 2045 - return -EINVAL; 2046 2039 2047 2040 for (i = 0; i < vport->num_rxq_grp; i++) { 2048 2041 struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; ··· 2064 2039 2065 2040 for (j = 0; j < num_rxq; j++, k++) { 2066 2041 struct idpf_rx_queue *rxq; 2042 + u32 v_idx, rx_itr_idx; 2067 2043 2068 2044 if (idpf_is_queue_model_split(vport->rxq_model)) 2069 2045 rxq = &rx_qgrp->splitq.rxq_sets[j]->rxq; ··· 2074 2048 vqv[k].queue_type = 2075 2049 cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); 2076 2050 vqv[k].queue_id = cpu_to_le32(rxq->q_id); 2077 - vqv[k].vector_id = cpu_to_le16(rxq->q_vector->v_idx); 2078 - vqv[k].itr_idx = cpu_to_le32(rxq->q_vector->rx_itr_idx); 2051 + 2052 + if (idpf_queue_has(NOIRQ, rxq)) { 2053 + v_idx = vport->noirq_v_idx; 2054 + rx_itr_idx = VIRTCHNL2_ITR_IDX_0; 2055 + } else { 2056 + v_idx = rxq->q_vector->v_idx; 2057 + rx_itr_idx = rxq->q_vector->rx_itr_idx; 2058 + } 2059 + 2060 + vqv[k].vector_id = cpu_to_le16(v_idx); 2061 + vqv[k].itr_idx = cpu_to_le32(rx_itr_idx); 2079 2062 } 2080 2063 } 2081 2064 2082 - if (idpf_is_queue_model_split(vport->txq_model)) { 2083 - if (vport->num_rxq != k - vport->num_complq) 2084 - return -EINVAL; 2085 - } else { 2086 - if (vport->num_rxq != k - vport->num_txq) 2087 - return -EINVAL; 2088 - } 2065 + if (k != num_q) 2066 + return -EINVAL; 2089 2067 2090 2068 /* Chunk up the vector info into multiple messages */ 2091 2069 config_sz = sizeof(struct virtchnl2_queue_vector_maps); ··· 2104 2074 if (!vqvm) 2105 2075 return -ENOMEM; 2106 2076 2107 - if (map) { 2077 + if (map) 2108 2078 xn_params.vc_op = VIRTCHNL2_OP_MAP_QUEUE_VECTOR; 2109 - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; 2110 - } else { 2079 + else 2111 2080 xn_params.vc_op = VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR; 2112 - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; 2113 - } 2114 2081 2115 2082 for (i = 0, k = 0; i < num_msgs; i++) { 2116 2083 memset(vqvm, 0, buf_sz); ··· 2152 2125 */ 2153 2126 int idpf_send_disable_queues_msg(struct idpf_vport *vport) 2154 2127 { 2155 - int err, i; 2128 + int err; 2156 2129 2157 2130 err = idpf_send_ena_dis_queues_msg(vport, false); 2158 2131 if (err) 2159 2132 return err; 2160 - 2161 - /* switch to poll mode as interrupts will be disabled after disable 2162 - * queues virtchnl message is sent 2163 - */ 2164 - for (i = 0; i < vport->num_txq; i++) 2165 - idpf_queue_set(POLL_MODE, vport->txqs[i]); 2166 - 2167 - /* schedule the napi to receive all the marker packets */ 2168 - local_bh_disable(); 2169 - for (i = 0; i < vport->num_q_vectors; i++) 2170 - napi_schedule(&vport->q_vectors[i].napi); 2171 - local_bh_enable(); 2172 2133 2173 2134 return idpf_wait_for_marker_event(vport); 2174 2135 } ··· 2222 2207 num_chunks); 2223 2208 2224 2209 xn_params.vc_op = VIRTCHNL2_OP_DEL_QUEUES; 2225 - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; 2210 + xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; 2226 2211 xn_params.send_buf.iov_base = eq; 2227 2212 xn_params.send_buf.iov_len = buf_size; 2228 2213 reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); ··· 2386 2371 xn_params.vc_op = VIRTCHNL2_OP_DEALLOC_VECTORS; 2387 2372 xn_params.send_buf.iov_base = vcs; 2388 2373 xn_params.send_buf.iov_len = buf_size; 2389 - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; 2374 + xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; 2390 2375 reply_sz = idpf_vc_xn_exec(adapter, &xn_params); 2391 2376 if (reply_sz < 0) 2392 2377 return reply_sz; ··· 3300 3285 { 3301 3286 struct idpf_vector_info vec_info; 3302 3287 int num_alloc_vecs; 3288 + u32 req; 3303 3289 3304 3290 vec_info.num_curr_vecs = vport->num_q_vectors; 3305 - vec_info.num_req_vecs = max(vport->num_txq, vport->num_rxq); 3291 + if (vec_info.num_curr_vecs) 3292 + vec_info.num_curr_vecs += IDPF_RESERVED_VECS; 3293 + 3294 + /* XDPSQs are all bound to the NOIRQ vector from IDPF_RESERVED_VECS */ 3295 + req = max(vport->num_txq - vport->num_xdp_txq, vport->num_rxq) + 3296 + IDPF_RESERVED_VECS; 3297 + vec_info.num_req_vecs = req; 3298 + 3306 3299 vec_info.default_vport = vport->default_vport; 3307 3300 vec_info.index = vport->idx; 3308 3301 ··· 3323 3300 return -EINVAL; 3324 3301 } 3325 3302 3326 - vport->num_q_vectors = num_alloc_vecs; 3303 + vport->num_q_vectors = num_alloc_vecs - IDPF_RESERVED_VECS; 3327 3304 3328 3305 return 0; 3329 3306 }

-1

drivers/net/ethernet/intel/idpf/idpf_virtchnl.h

··· 4 4 #ifndef _IDPF_VIRTCHNL_H_ 5 5 #define _IDPF_VIRTCHNL_H_ 6 6 7 - #define IDPF_VC_XN_MIN_TIMEOUT_MSEC 2000 8 7 #define IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC (60 * 1000) 9 8 #define IDPF_VC_XN_IDX_M GENMASK(7, 0) 10 9 #define IDPF_VC_XN_SALT_M GENMASK(15, 8)

+454

drivers/net/ethernet/intel/idpf/xdp.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* Copyright (C) 2025 Intel Corporation */ 3 + 4 + #include "idpf.h" 5 + #include "idpf_virtchnl.h" 6 + #include "xdp.h" 7 + 8 + static int idpf_rxq_for_each(const struct idpf_vport *vport, 9 + int (*fn)(struct idpf_rx_queue *rxq, void *arg), 10 + void *arg) 11 + { 12 + bool splitq = idpf_is_queue_model_split(vport->rxq_model); 13 + 14 + if (!vport->rxq_grps) 15 + return -ENETDOWN; 16 + 17 + for (u32 i = 0; i < vport->num_rxq_grp; i++) { 18 + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 19 + u32 num_rxq; 20 + 21 + if (splitq) 22 + num_rxq = rx_qgrp->splitq.num_rxq_sets; 23 + else 24 + num_rxq = rx_qgrp->singleq.num_rxq; 25 + 26 + for (u32 j = 0; j < num_rxq; j++) { 27 + struct idpf_rx_queue *q; 28 + int err; 29 + 30 + if (splitq) 31 + q = &rx_qgrp->splitq.rxq_sets[j]->rxq; 32 + else 33 + q = rx_qgrp->singleq.rxqs[j]; 34 + 35 + err = fn(q, arg); 36 + if (err) 37 + return err; 38 + } 39 + } 40 + 41 + return 0; 42 + } 43 + 44 + static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg) 45 + { 46 + const struct idpf_vport *vport = rxq->q_vector->vport; 47 + bool split = idpf_is_queue_model_split(vport->rxq_model); 48 + const struct page_pool *pp; 49 + int err; 50 + 51 + err = __xdp_rxq_info_reg(&rxq->xdp_rxq, vport->netdev, rxq->idx, 52 + rxq->q_vector->napi.napi_id, 53 + rxq->rx_buf_size); 54 + if (err) 55 + return err; 56 + 57 + pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp; 58 + xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp); 59 + 60 + if (!split) 61 + return 0; 62 + 63 + rxq->xdpsqs = &vport->txqs[vport->xdp_txq_offset]; 64 + rxq->num_xdp_txq = vport->num_xdp_txq; 65 + 66 + return 0; 67 + } 68 + 69 + int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport) 70 + { 71 + return idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_init, NULL); 72 + } 73 + 74 + static int __idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, void *arg) 75 + { 76 + if (idpf_is_queue_model_split((size_t)arg)) { 77 + rxq->xdpsqs = NULL; 78 + rxq->num_xdp_txq = 0; 79 + } 80 + 81 + xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq); 82 + xdp_rxq_info_unreg(&rxq->xdp_rxq); 83 + 84 + return 0; 85 + } 86 + 87 + void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport) 88 + { 89 + idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_deinit, 90 + (void *)(size_t)vport->rxq_model); 91 + } 92 + 93 + static int idpf_xdp_rxq_assign_prog(struct idpf_rx_queue *rxq, void *arg) 94 + { 95 + struct bpf_prog *prog = arg; 96 + struct bpf_prog *old; 97 + 98 + if (prog) 99 + bpf_prog_inc(prog); 100 + 101 + old = rcu_replace_pointer(rxq->xdp_prog, prog, lockdep_rtnl_is_held()); 102 + if (old) 103 + bpf_prog_put(old); 104 + 105 + return 0; 106 + } 107 + 108 + void idpf_xdp_copy_prog_to_rqs(const struct idpf_vport *vport, 109 + struct bpf_prog *xdp_prog) 110 + { 111 + idpf_rxq_for_each(vport, idpf_xdp_rxq_assign_prog, xdp_prog); 112 + } 113 + 114 + static void idpf_xdp_tx_timer(struct work_struct *work); 115 + 116 + int idpf_xdpsqs_get(const struct idpf_vport *vport) 117 + { 118 + struct libeth_xdpsq_timer **timers __free(kvfree) = NULL; 119 + struct net_device *dev; 120 + u32 sqs; 121 + 122 + if (!idpf_xdp_enabled(vport)) 123 + return 0; 124 + 125 + timers = kvcalloc(vport->num_xdp_txq, sizeof(*timers), GFP_KERNEL); 126 + if (!timers) 127 + return -ENOMEM; 128 + 129 + for (u32 i = 0; i < vport->num_xdp_txq; i++) { 130 + timers[i] = kzalloc_node(sizeof(*timers[i]), GFP_KERNEL, 131 + cpu_to_mem(i)); 132 + if (!timers[i]) { 133 + for (int j = i - 1; j >= 0; j--) 134 + kfree(timers[j]); 135 + 136 + return -ENOMEM; 137 + } 138 + } 139 + 140 + dev = vport->netdev; 141 + sqs = vport->xdp_txq_offset; 142 + 143 + for (u32 i = sqs; i < vport->num_txq; i++) { 144 + struct idpf_tx_queue *xdpsq = vport->txqs[i]; 145 + 146 + xdpsq->complq = xdpsq->txq_grp->complq; 147 + kfree(xdpsq->refillq); 148 + xdpsq->refillq = NULL; 149 + 150 + idpf_queue_clear(FLOW_SCH_EN, xdpsq); 151 + idpf_queue_clear(FLOW_SCH_EN, xdpsq->complq); 152 + idpf_queue_set(NOIRQ, xdpsq); 153 + idpf_queue_set(XDP, xdpsq); 154 + idpf_queue_set(XDP, xdpsq->complq); 155 + 156 + xdpsq->timer = timers[i - sqs]; 157 + libeth_xdpsq_get(&xdpsq->xdp_lock, dev, vport->xdpsq_share); 158 + libeth_xdpsq_init_timer(xdpsq->timer, xdpsq, &xdpsq->xdp_lock, 159 + idpf_xdp_tx_timer); 160 + 161 + xdpsq->pending = 0; 162 + xdpsq->xdp_tx = 0; 163 + xdpsq->thresh = libeth_xdp_queue_threshold(xdpsq->desc_count); 164 + } 165 + 166 + return 0; 167 + } 168 + 169 + void idpf_xdpsqs_put(const struct idpf_vport *vport) 170 + { 171 + struct net_device *dev; 172 + u32 sqs; 173 + 174 + if (!idpf_xdp_enabled(vport)) 175 + return; 176 + 177 + dev = vport->netdev; 178 + sqs = vport->xdp_txq_offset; 179 + 180 + for (u32 i = sqs; i < vport->num_txq; i++) { 181 + struct idpf_tx_queue *xdpsq = vport->txqs[i]; 182 + 183 + if (!idpf_queue_has_clear(XDP, xdpsq)) 184 + continue; 185 + 186 + libeth_xdpsq_deinit_timer(xdpsq->timer); 187 + libeth_xdpsq_put(&xdpsq->xdp_lock, dev); 188 + 189 + kfree(xdpsq->timer); 190 + xdpsq->refillq = NULL; 191 + idpf_queue_clear(NOIRQ, xdpsq); 192 + } 193 + } 194 + 195 + static int idpf_xdp_parse_cqe(const struct idpf_splitq_4b_tx_compl_desc *desc, 196 + bool gen) 197 + { 198 + u32 val; 199 + 200 + #ifdef __LIBETH_WORD_ACCESS 201 + val = *(const u32 *)desc; 202 + #else 203 + val = ((u32)le16_to_cpu(desc->q_head_compl_tag.q_head) << 16) | 204 + le16_to_cpu(desc->qid_comptype_gen); 205 + #endif 206 + if (!!(val & IDPF_TXD_COMPLQ_GEN_M) != gen) 207 + return -ENODATA; 208 + 209 + if (unlikely((val & GENMASK(IDPF_TXD_COMPLQ_GEN_S - 1, 0)) != 210 + FIELD_PREP(IDPF_TXD_COMPLQ_COMPL_TYPE_M, 211 + IDPF_TXD_COMPLT_RS))) 212 + return -EINVAL; 213 + 214 + return upper_16_bits(val); 215 + } 216 + 217 + static u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget) 218 + { 219 + struct idpf_compl_queue *cq = xdpsq->complq; 220 + u32 tx_ntc = xdpsq->next_to_clean; 221 + u32 tx_cnt = xdpsq->desc_count; 222 + u32 ntc = cq->next_to_clean; 223 + u32 cnt = cq->desc_count; 224 + u32 done_frames; 225 + bool gen; 226 + 227 + gen = idpf_queue_has(GEN_CHK, cq); 228 + 229 + for (done_frames = 0; done_frames < budget; ) { 230 + int ret; 231 + 232 + ret = idpf_xdp_parse_cqe(&cq->comp_4b[ntc], gen); 233 + if (ret >= 0) { 234 + done_frames = ret > tx_ntc ? ret - tx_ntc : 235 + ret + tx_cnt - tx_ntc; 236 + goto next; 237 + } 238 + 239 + switch (ret) { 240 + case -ENODATA: 241 + goto out; 242 + case -EINVAL: 243 + break; 244 + } 245 + 246 + next: 247 + if (unlikely(++ntc == cnt)) { 248 + ntc = 0; 249 + gen = !gen; 250 + idpf_queue_change(GEN_CHK, cq); 251 + } 252 + } 253 + 254 + out: 255 + cq->next_to_clean = ntc; 256 + 257 + return done_frames; 258 + } 259 + 260 + static u32 idpf_xdpsq_complete(void *_xdpsq, u32 budget) 261 + { 262 + struct libeth_xdpsq_napi_stats ss = { }; 263 + struct idpf_tx_queue *xdpsq = _xdpsq; 264 + u32 tx_ntc = xdpsq->next_to_clean; 265 + u32 tx_cnt = xdpsq->desc_count; 266 + struct xdp_frame_bulk bq; 267 + struct libeth_cq_pp cp = { 268 + .dev = xdpsq->dev, 269 + .bq = &bq, 270 + .xss = &ss, 271 + .napi = true, 272 + }; 273 + u32 done_frames; 274 + 275 + done_frames = idpf_xdpsq_poll(xdpsq, budget); 276 + if (unlikely(!done_frames)) 277 + return 0; 278 + 279 + xdp_frame_bulk_init(&bq); 280 + 281 + for (u32 i = 0; likely(i < done_frames); i++) { 282 + libeth_xdp_complete_tx(&xdpsq->tx_buf[tx_ntc], &cp); 283 + 284 + if (unlikely(++tx_ntc == tx_cnt)) 285 + tx_ntc = 0; 286 + } 287 + 288 + xdp_flush_frame_bulk(&bq); 289 + 290 + xdpsq->next_to_clean = tx_ntc; 291 + xdpsq->pending -= done_frames; 292 + xdpsq->xdp_tx -= cp.xdp_tx; 293 + 294 + return done_frames; 295 + } 296 + 297 + static u32 idpf_xdp_tx_prep(void *_xdpsq, struct libeth_xdpsq *sq) 298 + { 299 + struct idpf_tx_queue *xdpsq = _xdpsq; 300 + u32 free; 301 + 302 + libeth_xdpsq_lock(&xdpsq->xdp_lock); 303 + 304 + free = xdpsq->desc_count - xdpsq->pending; 305 + if (free < xdpsq->thresh) 306 + free += idpf_xdpsq_complete(xdpsq, xdpsq->thresh); 307 + 308 + *sq = (struct libeth_xdpsq){ 309 + .sqes = xdpsq->tx_buf, 310 + .descs = xdpsq->desc_ring, 311 + .count = xdpsq->desc_count, 312 + .lock = &xdpsq->xdp_lock, 313 + .ntu = &xdpsq->next_to_use, 314 + .pending = &xdpsq->pending, 315 + .xdp_tx = &xdpsq->xdp_tx, 316 + }; 317 + 318 + return free; 319 + } 320 + 321 + LIBETH_XDP_DEFINE_START(); 322 + LIBETH_XDP_DEFINE_TIMER(static idpf_xdp_tx_timer, idpf_xdpsq_complete); 323 + LIBETH_XDP_DEFINE_FLUSH_TX(idpf_xdp_tx_flush_bulk, idpf_xdp_tx_prep, 324 + idpf_xdp_tx_xmit); 325 + LIBETH_XDP_DEFINE_FLUSH_XMIT(static idpf_xdp_xmit_flush_bulk, idpf_xdp_tx_prep, 326 + idpf_xdp_tx_xmit); 327 + LIBETH_XDP_DEFINE_END(); 328 + 329 + int idpf_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 330 + u32 flags) 331 + { 332 + const struct idpf_netdev_priv *np = netdev_priv(dev); 333 + const struct idpf_vport *vport = np->vport; 334 + 335 + if (unlikely(!netif_carrier_ok(dev) || !vport->link_up)) 336 + return -ENETDOWN; 337 + 338 + return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, 339 + &vport->txqs[vport->xdp_txq_offset], 340 + vport->num_xdp_txq, 341 + idpf_xdp_xmit_flush_bulk, 342 + idpf_xdp_tx_finalize); 343 + } 344 + 345 + static int idpf_xdpmo_rx_hash(const struct xdp_md *ctx, u32 *hash, 346 + enum xdp_rss_hash_type *rss_type) 347 + { 348 + const struct libeth_xdp_buff *xdp = (typeof(xdp))ctx; 349 + struct idpf_xdp_rx_desc desc __uninitialized; 350 + const struct idpf_rx_queue *rxq; 351 + struct libeth_rx_pt pt; 352 + 353 + rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); 354 + 355 + idpf_xdp_get_qw0(&desc, xdp->desc); 356 + 357 + pt = rxq->rx_ptype_lkup[idpf_xdp_rx_pt(&desc)]; 358 + if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, pt)) 359 + return -ENODATA; 360 + 361 + idpf_xdp_get_qw2(&desc, xdp->desc); 362 + 363 + return libeth_xdpmo_rx_hash(hash, rss_type, idpf_xdp_rx_hash(&desc), 364 + pt); 365 + } 366 + 367 + static const struct xdp_metadata_ops idpf_xdpmo = { 368 + .xmo_rx_hash = idpf_xdpmo_rx_hash, 369 + }; 370 + 371 + void idpf_xdp_set_features(const struct idpf_vport *vport) 372 + { 373 + if (!idpf_is_queue_model_split(vport->rxq_model)) 374 + return; 375 + 376 + libeth_xdp_set_features_noredir(vport->netdev, &idpf_xdpmo); 377 + } 378 + 379 + static int idpf_xdp_setup_prog(struct idpf_vport *vport, 380 + const struct netdev_bpf *xdp) 381 + { 382 + const struct idpf_netdev_priv *np = netdev_priv(vport->netdev); 383 + struct bpf_prog *old, *prog = xdp->prog; 384 + struct idpf_vport_config *cfg; 385 + int ret; 386 + 387 + cfg = vport->adapter->vport_config[vport->idx]; 388 + 389 + if (test_bit(IDPF_REMOVE_IN_PROG, vport->adapter->flags) || 390 + !test_bit(IDPF_VPORT_REG_NETDEV, cfg->flags) || 391 + !!vport->xdp_prog == !!prog) { 392 + if (np->state == __IDPF_VPORT_UP) 393 + idpf_xdp_copy_prog_to_rqs(vport, prog); 394 + 395 + old = xchg(&vport->xdp_prog, prog); 396 + if (old) 397 + bpf_prog_put(old); 398 + 399 + cfg->user_config.xdp_prog = prog; 400 + 401 + return 0; 402 + } 403 + 404 + if (!vport->num_xdp_txq && vport->num_txq == cfg->max_q.max_txq) { 405 + NL_SET_ERR_MSG_MOD(xdp->extack, 406 + "No Tx queues available for XDP, please decrease the number of regular SQs"); 407 + return -ENOSPC; 408 + } 409 + 410 + old = cfg->user_config.xdp_prog; 411 + cfg->user_config.xdp_prog = prog; 412 + 413 + ret = idpf_initiate_soft_reset(vport, IDPF_SR_Q_CHANGE); 414 + if (ret) { 415 + NL_SET_ERR_MSG_MOD(xdp->extack, 416 + "Could not reopen the vport after XDP setup"); 417 + 418 + cfg->user_config.xdp_prog = old; 419 + old = prog; 420 + } 421 + 422 + if (old) 423 + bpf_prog_put(old); 424 + 425 + libeth_xdp_set_redirect(vport->netdev, vport->xdp_prog); 426 + 427 + return ret; 428 + } 429 + 430 + int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp) 431 + { 432 + struct idpf_vport *vport; 433 + int ret; 434 + 435 + idpf_vport_ctrl_lock(dev); 436 + vport = idpf_netdev_to_vport(dev); 437 + 438 + if (!idpf_is_queue_model_split(vport->txq_model)) 439 + goto notsupp; 440 + 441 + switch (xdp->command) { 442 + case XDP_SETUP_PROG: 443 + ret = idpf_xdp_setup_prog(vport, xdp); 444 + break; 445 + default: 446 + notsupp: 447 + ret = -EOPNOTSUPP; 448 + break; 449 + } 450 + 451 + idpf_vport_ctrl_unlock(dev); 452 + 453 + return ret; 454 + }

+172

drivers/net/ethernet/intel/idpf/xdp.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + /* Copyright (C) 2025 Intel Corporation */ 3 + 4 + #ifndef _IDPF_XDP_H_ 5 + #define _IDPF_XDP_H_ 6 + 7 + #include <net/libeth/xdp.h> 8 + 9 + #include "idpf_txrx.h" 10 + 11 + int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport); 12 + void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport); 13 + void idpf_xdp_copy_prog_to_rqs(const struct idpf_vport *vport, 14 + struct bpf_prog *xdp_prog); 15 + 16 + int idpf_xdpsqs_get(const struct idpf_vport *vport); 17 + void idpf_xdpsqs_put(const struct idpf_vport *vport); 18 + 19 + bool idpf_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags); 20 + 21 + /** 22 + * idpf_xdp_tx_xmit - produce a single HW Tx descriptor out of XDP desc 23 + * @desc: XDP descriptor to pull the DMA address and length from 24 + * @i: descriptor index on the queue to fill 25 + * @sq: XDP queue to produce the HW Tx descriptor on 26 + * @priv: &xsk_tx_metadata_ops on XSk xmit or %NULL 27 + */ 28 + static inline void idpf_xdp_tx_xmit(struct libeth_xdp_tx_desc desc, u32 i, 29 + const struct libeth_xdpsq *sq, u64 priv) 30 + { 31 + struct idpf_flex_tx_desc *tx_desc = sq->descs; 32 + u32 cmd; 33 + 34 + cmd = FIELD_PREP(IDPF_FLEX_TXD_QW1_DTYPE_M, 35 + IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2); 36 + if (desc.flags & LIBETH_XDP_TX_LAST) 37 + cmd |= FIELD_PREP(IDPF_FLEX_TXD_QW1_CMD_M, 38 + IDPF_TX_DESC_CMD_EOP); 39 + if (priv && (desc.flags & LIBETH_XDP_TX_CSUM)) 40 + cmd |= FIELD_PREP(IDPF_FLEX_TXD_QW1_CMD_M, 41 + IDPF_TX_FLEX_DESC_CMD_CS_EN); 42 + 43 + tx_desc = &tx_desc[i]; 44 + tx_desc->buf_addr = cpu_to_le64(desc.addr); 45 + #ifdef __LIBETH_WORD_ACCESS 46 + *(u64 *)&tx_desc->qw1 = ((u64)desc.len << 48) | cmd; 47 + #else 48 + tx_desc->qw1.buf_size = cpu_to_le16(desc.len); 49 + tx_desc->qw1.cmd_dtype = cpu_to_le16(cmd); 50 + #endif 51 + } 52 + 53 + static inline void idpf_xdpsq_set_rs(const struct idpf_tx_queue *xdpsq) 54 + { 55 + u32 ntu, cmd; 56 + 57 + ntu = xdpsq->next_to_use; 58 + if (unlikely(!ntu)) 59 + ntu = xdpsq->desc_count; 60 + 61 + cmd = FIELD_PREP(IDPF_FLEX_TXD_QW1_CMD_M, IDPF_TX_DESC_CMD_RS); 62 + #ifdef __LIBETH_WORD_ACCESS 63 + *(u64 *)&xdpsq->flex_tx[ntu - 1].q.qw1 |= cmd; 64 + #else 65 + xdpsq->flex_tx[ntu - 1].q.qw1.cmd_dtype |= cpu_to_le16(cmd); 66 + #endif 67 + } 68 + 69 + static inline void idpf_xdpsq_update_tail(const struct idpf_tx_queue *xdpsq) 70 + { 71 + dma_wmb(); 72 + writel_relaxed(xdpsq->next_to_use, xdpsq->tail); 73 + } 74 + 75 + /** 76 + * idpf_xdp_tx_finalize - finalize sending over XDPSQ 77 + * @_xdpsq: XDP Tx queue 78 + * @sent: whether any frames were sent 79 + * @flush: whether to update RS bit and the tail register 80 + * 81 + * Set the RS bit ("end of batch"), bump the tail, and queue the cleanup timer. 82 + * To be called after a NAPI polling loop, at the end of .ndo_xdp_xmit() etc. 83 + */ 84 + static inline void idpf_xdp_tx_finalize(void *_xdpsq, bool sent, bool flush) 85 + { 86 + struct idpf_tx_queue *xdpsq = _xdpsq; 87 + 88 + if ((!flush || unlikely(!sent)) && 89 + likely(xdpsq->desc_count - 1 != xdpsq->pending)) 90 + return; 91 + 92 + libeth_xdpsq_lock(&xdpsq->xdp_lock); 93 + 94 + idpf_xdpsq_set_rs(xdpsq); 95 + idpf_xdpsq_update_tail(xdpsq); 96 + 97 + libeth_xdpsq_queue_timer(xdpsq->timer); 98 + 99 + libeth_xdpsq_unlock(&xdpsq->xdp_lock); 100 + } 101 + 102 + struct idpf_xdp_rx_desc { 103 + aligned_u64 qw0; 104 + #define IDPF_XDP_RX_BUFQ BIT_ULL(47) 105 + #define IDPF_XDP_RX_GEN BIT_ULL(46) 106 + #define IDPF_XDP_RX_LEN GENMASK_ULL(45, 32) 107 + #define IDPF_XDP_RX_PT GENMASK_ULL(25, 16) 108 + 109 + aligned_u64 qw1; 110 + #define IDPF_XDP_RX_BUF GENMASK_ULL(47, 32) 111 + #define IDPF_XDP_RX_EOP BIT_ULL(1) 112 + 113 + aligned_u64 qw2; 114 + #define IDPF_XDP_RX_HASH GENMASK_ULL(31, 0) 115 + 116 + aligned_u64 qw3; 117 + } __aligned(4 * sizeof(u64)); 118 + static_assert(sizeof(struct idpf_xdp_rx_desc) == 119 + sizeof(struct virtchnl2_rx_flex_desc_adv_nic_3)); 120 + 121 + #define idpf_xdp_rx_bufq(desc) !!((desc)->qw0 & IDPF_XDP_RX_BUFQ) 122 + #define idpf_xdp_rx_gen(desc) !!((desc)->qw0 & IDPF_XDP_RX_GEN) 123 + #define idpf_xdp_rx_len(desc) FIELD_GET(IDPF_XDP_RX_LEN, (desc)->qw0) 124 + #define idpf_xdp_rx_pt(desc) FIELD_GET(IDPF_XDP_RX_PT, (desc)->qw0) 125 + #define idpf_xdp_rx_buf(desc) FIELD_GET(IDPF_XDP_RX_BUF, (desc)->qw1) 126 + #define idpf_xdp_rx_eop(desc) !!((desc)->qw1 & IDPF_XDP_RX_EOP) 127 + #define idpf_xdp_rx_hash(desc) FIELD_GET(IDPF_XDP_RX_HASH, (desc)->qw2) 128 + 129 + static inline void 130 + idpf_xdp_get_qw0(struct idpf_xdp_rx_desc *desc, 131 + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rxd) 132 + { 133 + #ifdef __LIBETH_WORD_ACCESS 134 + desc->qw0 = ((const typeof(desc))rxd)->qw0; 135 + #else 136 + desc->qw0 = ((u64)le16_to_cpu(rxd->pktlen_gen_bufq_id) << 32) | 137 + ((u64)le16_to_cpu(rxd->ptype_err_fflags0) << 16); 138 + #endif 139 + } 140 + 141 + static inline void 142 + idpf_xdp_get_qw1(struct idpf_xdp_rx_desc *desc, 143 + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rxd) 144 + { 145 + #ifdef __LIBETH_WORD_ACCESS 146 + desc->qw1 = ((const typeof(desc))rxd)->qw1; 147 + #else 148 + desc->qw1 = ((u64)le16_to_cpu(rxd->buf_id) << 32) | 149 + rxd->status_err0_qw1; 150 + #endif 151 + } 152 + 153 + static inline void 154 + idpf_xdp_get_qw2(struct idpf_xdp_rx_desc *desc, 155 + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rxd) 156 + { 157 + #ifdef __LIBETH_WORD_ACCESS 158 + desc->qw2 = ((const typeof(desc))rxd)->qw2; 159 + #else 160 + desc->qw2 = ((u64)rxd->hash3 << 24) | 161 + ((u64)rxd->ff2_mirrid_hash2.hash2 << 16) | 162 + le16_to_cpu(rxd->hash1); 163 + #endif 164 + } 165 + 166 + void idpf_xdp_set_features(const struct idpf_vport *vport); 167 + 168 + int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp); 169 + int idpf_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 170 + u32 flags); 171 + 172 + #endif /* _IDPF_XDP_H_ */

+1 -10

include/net/libeth/xdp.h

··· 1274 1274 * Internal, use libeth_xdp_process_buff() instead. Initializes XDP buffer 1275 1275 * head with the Rx buffer data: data pointer, length, headroom, and 1276 1276 * truesize/tailroom. Zeroes the flags. 1277 - * Uses faster single u64 write instead of per-field access. 1278 1277 */ 1279 1278 static inline void libeth_xdp_prepare_buff(struct libeth_xdp_buff *xdp, 1280 1279 const struct libeth_fqe *fqe, ··· 1281 1282 { 1282 1283 const struct page *page = __netmem_to_page(fqe->netmem); 1283 1284 1284 - #ifdef __LIBETH_WORD_ACCESS 1285 - static_assert(offsetofend(typeof(xdp->base), flags) - 1286 - offsetof(typeof(xdp->base), frame_sz) == 1287 - sizeof(u64)); 1288 - 1289 - *(u64 *)&xdp->base.frame_sz = fqe->truesize; 1290 - #else 1291 - xdp_init_buff(&xdp->base, fqe->truesize, xdp->base.rxq); 1292 - #endif 1293 1285 xdp_prepare_buff(&xdp->base, page_address(page) + fqe->offset, 1294 1286 pp_page_to_nmdesc(page)->pp->p.offset, len, true); 1287 + xdp_init_buff(&xdp->base, fqe->truesize, xdp->base.rxq); 1295 1288 } 1296 1289 1297 1290 /**

+25 -3

include/net/xdp.h

··· 85 85 void *data_hard_start; 86 86 struct xdp_rxq_info *rxq; 87 87 struct xdp_txq_info *txq; 88 - u32 frame_sz; /* frame size to deduce data_hard_end/reserved tailroom*/ 89 - u32 flags; /* supported values defined in xdp_buff_flags */ 88 + 89 + union { 90 + struct { 91 + /* frame size to deduce data_hard_end/tailroom */ 92 + u32 frame_sz; 93 + /* supported values defined in xdp_buff_flags */ 94 + u32 flags; 95 + }; 96 + 97 + #ifdef __LITTLE_ENDIAN 98 + /* Used to micro-optimize xdp_init_buff(), don't use directly */ 99 + u64 frame_sz_flags_init; 100 + #endif 101 + }; 90 102 }; 91 103 92 104 static __always_inline bool xdp_buff_has_frags(const struct xdp_buff *xdp) ··· 130 118 static __always_inline void 131 119 xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq) 132 120 { 133 - xdp->frame_sz = frame_sz; 134 121 xdp->rxq = rxq; 122 + 123 + #ifdef __LITTLE_ENDIAN 124 + /* 125 + * Force the compilers to initialize ::flags and assign ::frame_sz with 126 + * one write on 64-bit LE architectures as they're often unable to do 127 + * it themselves. 128 + */ 129 + xdp->frame_sz_flags_init = frame_sz; 130 + #else 131 + xdp->frame_sz = frame_sz; 135 132 xdp->flags = 0; 133 + #endif 136 134 } 137 135 138 136 static __always_inline void

Configure Feed

Configure Feed