Merge branch 'bnxt_en-add-npar-1-2-and-tph-support'

+417 -147

2 changed files

expand all

drivers

net

ethernet

broadcom

bnxt

bnxt.c

bnxt.h

+409 -147

drivers/net/ethernet/broadcom/bnxt/bnxt.c

··· 55 55 #include <net/page_pool/helpers.h> 56 56 #include <linux/align.h> 57 57 #include <net/netdev_queues.h> 58 + #include <net/netdev_rx_queue.h> 59 + #include <linux/pci-tph.h> 58 60 59 61 #include "bnxt_hsi.h" 60 62 #include "bnxt.h" ··· 78 76 #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \ 79 77 NETIF_MSG_TX_ERR) 80 78 79 + MODULE_IMPORT_NS("NETDEV_INTERNAL"); 81 80 MODULE_LICENSE("GPL"); 82 81 MODULE_DESCRIPTION("Broadcom NetXtreme network driver"); 83 82 ··· 3317 3314 return work_done; 3318 3315 } 3319 3316 3320 - static void bnxt_free_tx_skbs(struct bnxt *bp) 3317 + static void bnxt_free_one_tx_ring_skbs(struct bnxt *bp, 3318 + struct bnxt_tx_ring_info *txr, int idx) 3321 3319 { 3322 3320 int i, max_idx; 3323 3321 struct pci_dev *pdev = bp->pdev; 3324 3322 3323 + max_idx = bp->tx_nr_pages * TX_DESC_CNT; 3324 + 3325 + for (i = 0; i < max_idx;) { 3326 + struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[i]; 3327 + struct sk_buff *skb; 3328 + int j, last; 3329 + 3330 + if (idx < bp->tx_nr_rings_xdp && 3331 + tx_buf->action == XDP_REDIRECT) { 3332 + dma_unmap_single(&pdev->dev, 3333 + dma_unmap_addr(tx_buf, mapping), 3334 + dma_unmap_len(tx_buf, len), 3335 + DMA_TO_DEVICE); 3336 + xdp_return_frame(tx_buf->xdpf); 3337 + tx_buf->action = 0; 3338 + tx_buf->xdpf = NULL; 3339 + i++; 3340 + continue; 3341 + } 3342 + 3343 + skb = tx_buf->skb; 3344 + if (!skb) { 3345 + i++; 3346 + continue; 3347 + } 3348 + 3349 + tx_buf->skb = NULL; 3350 + 3351 + if (tx_buf->is_push) { 3352 + dev_kfree_skb(skb); 3353 + i += 2; 3354 + continue; 3355 + } 3356 + 3357 + dma_unmap_single(&pdev->dev, 3358 + dma_unmap_addr(tx_buf, mapping), 3359 + skb_headlen(skb), 3360 + DMA_TO_DEVICE); 3361 + 3362 + last = tx_buf->nr_frags; 3363 + i += 2; 3364 + for (j = 0; j < last; j++, i++) { 3365 + int ring_idx = i & bp->tx_ring_mask; 3366 + skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 3367 + 3368 + tx_buf = &txr->tx_buf_ring[ring_idx]; 3369 + dma_unmap_page(&pdev->dev, 3370 + dma_unmap_addr(tx_buf, mapping), 3371 + skb_frag_size(frag), DMA_TO_DEVICE); 3372 + } 3373 + dev_kfree_skb(skb); 3374 + } 3375 + netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, idx)); 3376 + } 3377 + 3378 + static void bnxt_free_tx_skbs(struct bnxt *bp) 3379 + { 3380 + int i; 3381 + 3325 3382 if (!bp->tx_ring) 3326 3383 return; 3327 3384 3328 - max_idx = bp->tx_nr_pages * TX_DESC_CNT; 3329 3385 for (i = 0; i < bp->tx_nr_rings; i++) { 3330 3386 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3331 - int j; 3332 3387 3333 3388 if (!txr->tx_buf_ring) 3334 3389 continue; 3335 3390 3336 - for (j = 0; j < max_idx;) { 3337 - struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 3338 - struct sk_buff *skb; 3339 - int k, last; 3340 - 3341 - if (i < bp->tx_nr_rings_xdp && 3342 - tx_buf->action == XDP_REDIRECT) { 3343 - dma_unmap_single(&pdev->dev, 3344 - dma_unmap_addr(tx_buf, mapping), 3345 - dma_unmap_len(tx_buf, len), 3346 - DMA_TO_DEVICE); 3347 - xdp_return_frame(tx_buf->xdpf); 3348 - tx_buf->action = 0; 3349 - tx_buf->xdpf = NULL; 3350 - j++; 3351 - continue; 3352 - } 3353 - 3354 - skb = tx_buf->skb; 3355 - if (!skb) { 3356 - j++; 3357 - continue; 3358 - } 3359 - 3360 - tx_buf->skb = NULL; 3361 - 3362 - if (tx_buf->is_push) { 3363 - dev_kfree_skb(skb); 3364 - j += 2; 3365 - continue; 3366 - } 3367 - 3368 - dma_unmap_single(&pdev->dev, 3369 - dma_unmap_addr(tx_buf, mapping), 3370 - skb_headlen(skb), 3371 - DMA_TO_DEVICE); 3372 - 3373 - last = tx_buf->nr_frags; 3374 - j += 2; 3375 - for (k = 0; k < last; k++, j++) { 3376 - int ring_idx = j & bp->tx_ring_mask; 3377 - skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 3378 - 3379 - tx_buf = &txr->tx_buf_ring[ring_idx]; 3380 - dma_unmap_page( 3381 - &pdev->dev, 3382 - dma_unmap_addr(tx_buf, mapping), 3383 - skb_frag_size(frag), DMA_TO_DEVICE); 3384 - } 3385 - dev_kfree_skb(skb); 3386 - } 3387 - netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 3391 + bnxt_free_one_tx_ring_skbs(bp, txr, i); 3388 3392 } 3389 3393 } 3390 3394 ··· 5575 5565 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 5576 5566 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT | 5577 5567 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT; 5568 + if (bp->fw_cap & BNXT_FW_CAP_NPAR_1_2) 5569 + flags |= FUNC_DRV_RGTR_REQ_FLAGS_NPAR_1_2_SUPPORT; 5578 5570 req->flags = cpu_to_le32(flags); 5579 5571 req->ver_maj_8b = DRV_VER_MAJ; 5580 5572 req->ver_min_8b = DRV_VER_MIN; ··· 6947 6935 hwrm_req_drop(bp, req); 6948 6936 } 6949 6937 6938 + static void bnxt_set_rx_ring_params_p5(struct bnxt *bp, u32 ring_type, 6939 + struct hwrm_ring_alloc_input *req, 6940 + struct bnxt_ring_struct *ring) 6941 + { 6942 + struct bnxt_ring_grp_info *grp_info = &bp->grp_info[ring->grp_idx]; 6943 + u32 enables = RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID | 6944 + RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID; 6945 + 6946 + if (ring_type == HWRM_RING_ALLOC_AGG) { 6947 + req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 6948 + req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 6949 + req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 6950 + enables |= RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID; 6951 + } else { 6952 + req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 6953 + if (NET_IP_ALIGN == 2) 6954 + req->flags = 6955 + cpu_to_le16(RING_ALLOC_REQ_FLAGS_RX_SOP_PAD); 6956 + } 6957 + req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 6958 + req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 6959 + req->enables |= cpu_to_le32(enables); 6960 + } 6961 + 6950 6962 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 6951 6963 struct bnxt_ring_struct *ring, 6952 6964 u32 ring_type, u32 map_index) ··· 7022 6986 break; 7023 6987 } 7024 6988 case HWRM_RING_ALLOC_RX: 7025 - req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 7026 - req->length = cpu_to_le32(bp->rx_ring_mask + 1); 7027 - if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7028 - u16 flags = 0; 7029 - 7030 - /* Association of rx ring with stats context */ 7031 - grp_info = &bp->grp_info[ring->grp_idx]; 7032 - req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 7033 - req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 7034 - req->enables |= cpu_to_le32( 7035 - RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 7036 - if (NET_IP_ALIGN == 2) 7037 - flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD; 7038 - req->flags = cpu_to_le16(flags); 7039 - } 7040 - break; 7041 6989 case HWRM_RING_ALLOC_AGG: 7042 - if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7043 - req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 7044 - /* Association of agg ring with rx ring */ 7045 - grp_info = &bp->grp_info[ring->grp_idx]; 7046 - req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 7047 - req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 7048 - req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 7049 - req->enables |= cpu_to_le32( 7050 - RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID | 7051 - RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 7052 - } else { 7053 - req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 7054 - } 7055 - req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 6990 + req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 6991 + req->length = (ring_type == HWRM_RING_ALLOC_RX) ? 6992 + cpu_to_le32(bp->rx_ring_mask + 1) : 6993 + cpu_to_le32(bp->rx_agg_ring_mask + 1); 6994 + if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6995 + bnxt_set_rx_ring_params_p5(bp, ring_type, req, ring); 7056 6996 break; 7057 6997 case HWRM_RING_ALLOC_CMPL: 7058 6998 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; ··· 7209 7197 return 0; 7210 7198 } 7211 7199 7200 + static int bnxt_hwrm_cp_ring_alloc_p5(struct bnxt *bp, 7201 + struct bnxt_cp_ring_info *cpr) 7202 + { 7203 + const u32 type = HWRM_RING_ALLOC_CMPL; 7204 + struct bnxt_napi *bnapi = cpr->bnapi; 7205 + struct bnxt_ring_struct *ring; 7206 + u32 map_idx = bnapi->index; 7207 + int rc; 7208 + 7209 + ring = &cpr->cp_ring_struct; 7210 + ring->handle = BNXT_SET_NQ_HDL(cpr); 7211 + rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 7212 + if (rc) 7213 + return rc; 7214 + bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 7215 + bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 7216 + return 0; 7217 + } 7218 + 7219 + static int bnxt_hwrm_tx_ring_alloc(struct bnxt *bp, 7220 + struct bnxt_tx_ring_info *txr, u32 tx_idx) 7221 + { 7222 + struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 7223 + const u32 type = HWRM_RING_ALLOC_TX; 7224 + int rc; 7225 + 7226 + rc = hwrm_ring_alloc_send_msg(bp, ring, type, tx_idx); 7227 + if (rc) 7228 + return rc; 7229 + bnxt_set_db(bp, &txr->tx_db, type, tx_idx, ring->fw_ring_id); 7230 + return 0; 7231 + } 7232 + 7212 7233 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 7213 7234 { 7214 7235 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); ··· 7278 7233 } 7279 7234 } 7280 7235 7281 - type = HWRM_RING_ALLOC_TX; 7282 7236 for (i = 0; i < bp->tx_nr_rings; i++) { 7283 7237 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 7284 - struct bnxt_ring_struct *ring; 7285 - u32 map_idx; 7286 7238 7287 7239 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7288 - struct bnxt_cp_ring_info *cpr2 = txr->tx_cpr; 7289 - struct bnxt_napi *bnapi = txr->bnapi; 7290 - u32 type2 = HWRM_RING_ALLOC_CMPL; 7291 - 7292 - ring = &cpr2->cp_ring_struct; 7293 - ring->handle = BNXT_SET_NQ_HDL(cpr2); 7294 - map_idx = bnapi->index; 7295 - rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 7240 + rc = bnxt_hwrm_cp_ring_alloc_p5(bp, txr->tx_cpr); 7296 7241 if (rc) 7297 7242 goto err_out; 7298 - bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 7299 - ring->fw_ring_id); 7300 - bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 7301 7243 } 7302 - ring = &txr->tx_ring_struct; 7303 - map_idx = i; 7304 - rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 7244 + rc = bnxt_hwrm_tx_ring_alloc(bp, txr, i); 7305 7245 if (rc) 7306 7246 goto err_out; 7307 - bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id); 7308 7247 } 7309 7248 7310 7249 for (i = 0; i < bp->rx_nr_rings; i++) { ··· 7301 7272 if (!agg_rings) 7302 7273 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 7303 7274 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7304 - struct bnxt_cp_ring_info *cpr2 = rxr->rx_cpr; 7305 - struct bnxt_napi *bnapi = rxr->bnapi; 7306 - u32 type2 = HWRM_RING_ALLOC_CMPL; 7307 - struct bnxt_ring_struct *ring; 7308 - u32 map_idx = bnapi->index; 7309 - 7310 - ring = &cpr2->cp_ring_struct; 7311 - ring->handle = BNXT_SET_NQ_HDL(cpr2); 7312 - rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 7275 + rc = bnxt_hwrm_cp_ring_alloc_p5(bp, rxr->rx_cpr); 7313 7276 if (rc) 7314 7277 goto err_out; 7315 - bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 7316 - ring->fw_ring_id); 7317 - bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 7318 7278 } 7319 7279 } 7320 7280 ··· 7371 7353 return 0; 7372 7354 } 7373 7355 7356 + static void bnxt_hwrm_tx_ring_free(struct bnxt *bp, 7357 + struct bnxt_tx_ring_info *txr, 7358 + bool close_path) 7359 + { 7360 + struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 7361 + u32 cmpl_ring_id; 7362 + 7363 + if (ring->fw_ring_id == INVALID_HW_RING_ID) 7364 + return; 7365 + 7366 + cmpl_ring_id = close_path ? bnxt_cp_ring_for_tx(bp, txr) : 7367 + INVALID_HW_RING_ID; 7368 + hwrm_ring_free_send_msg(bp, ring, RING_FREE_REQ_RING_TYPE_TX, 7369 + cmpl_ring_id); 7370 + ring->fw_ring_id = INVALID_HW_RING_ID; 7371 + } 7372 + 7374 7373 static void bnxt_hwrm_rx_ring_free(struct bnxt *bp, 7375 7374 struct bnxt_rx_ring_info *rxr, 7376 7375 bool close_path) ··· 7432 7397 bp->grp_info[grp_idx].agg_fw_ring_id = INVALID_HW_RING_ID; 7433 7398 } 7434 7399 7400 + static void bnxt_hwrm_cp_ring_free(struct bnxt *bp, 7401 + struct bnxt_cp_ring_info *cpr) 7402 + { 7403 + struct bnxt_ring_struct *ring; 7404 + 7405 + ring = &cpr->cp_ring_struct; 7406 + if (ring->fw_ring_id == INVALID_HW_RING_ID) 7407 + return; 7408 + 7409 + hwrm_ring_free_send_msg(bp, ring, RING_FREE_REQ_RING_TYPE_L2_CMPL, 7410 + INVALID_HW_RING_ID); 7411 + ring->fw_ring_id = INVALID_HW_RING_ID; 7412 + } 7413 + 7414 + static void bnxt_clear_one_cp_ring(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 7415 + { 7416 + struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 7417 + int i, size = ring->ring_mem.page_size; 7418 + 7419 + cpr->cp_raw_cons = 0; 7420 + cpr->toggle = 0; 7421 + 7422 + for (i = 0; i < bp->cp_nr_pages; i++) 7423 + if (cpr->cp_desc_ring[i]) 7424 + memset(cpr->cp_desc_ring[i], 0, size); 7425 + } 7426 + 7435 7427 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 7436 7428 { 7437 7429 u32 type; ··· 7467 7405 if (!bp->bnapi) 7468 7406 return; 7469 7407 7470 - for (i = 0; i < bp->tx_nr_rings; i++) { 7471 - struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 7472 - struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 7473 - 7474 - if (ring->fw_ring_id != INVALID_HW_RING_ID) { 7475 - u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr); 7476 - 7477 - hwrm_ring_free_send_msg(bp, ring, 7478 - RING_FREE_REQ_RING_TYPE_TX, 7479 - close_path ? cmpl_ring_id : 7480 - INVALID_HW_RING_ID); 7481 - ring->fw_ring_id = INVALID_HW_RING_ID; 7482 - } 7483 - } 7408 + for (i = 0; i < bp->tx_nr_rings; i++) 7409 + bnxt_hwrm_tx_ring_free(bp, &bp->tx_ring[i], close_path); 7484 7410 7485 7411 bnxt_cancel_dim(bp); 7486 7412 for (i = 0; i < bp->rx_nr_rings; i++) { ··· 7492 7442 struct bnxt_ring_struct *ring; 7493 7443 int j; 7494 7444 7495 - for (j = 0; j < cpr->cp_ring_count && cpr->cp_ring_arr; j++) { 7496 - struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j]; 7445 + for (j = 0; j < cpr->cp_ring_count && cpr->cp_ring_arr; j++) 7446 + bnxt_hwrm_cp_ring_free(bp, &cpr->cp_ring_arr[j]); 7497 7447 7498 - ring = &cpr2->cp_ring_struct; 7499 - if (ring->fw_ring_id == INVALID_HW_RING_ID) 7500 - continue; 7501 - hwrm_ring_free_send_msg(bp, ring, 7502 - RING_FREE_REQ_RING_TYPE_L2_CMPL, 7503 - INVALID_HW_RING_ID); 7504 - ring->fw_ring_id = INVALID_HW_RING_ID; 7505 - } 7506 7448 ring = &cpr->cp_ring_struct; 7507 7449 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 7508 7450 hwrm_ring_free_send_msg(bp, ring, type, ··· 8407 8365 8408 8366 switch (resp->port_partition_type) { 8409 8367 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 8368 + case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_2: 8410 8369 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 8411 8370 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 8412 8371 bp->port_partition_type = resp->port_partition_type; ··· 9572 9529 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF; 9573 9530 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED)) 9574 9531 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH; 9532 + if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_NPAR_1_2_SUPPORTED) 9533 + bp->fw_cap |= BNXT_FW_CAP_NPAR_1_2; 9575 9534 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_DFLT_VLAN_TPID_PCP_SUPPORTED)) 9576 9535 bp->fw_cap |= BNXT_FW_CAP_DFLT_VLAN_TPID_PCP; 9577 9536 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_SUPPORTED) ··· 11282 11237 return 0; 11283 11238 } 11284 11239 11240 + static void bnxt_tx_queue_stop(struct bnxt *bp, int idx) 11241 + { 11242 + struct bnxt_tx_ring_info *txr; 11243 + struct netdev_queue *txq; 11244 + struct bnxt_napi *bnapi; 11245 + int i; 11246 + 11247 + bnapi = bp->bnapi[idx]; 11248 + bnxt_for_each_napi_tx(i, bnapi, txr) { 11249 + WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING); 11250 + synchronize_net(); 11251 + 11252 + if (!(bnapi->flags & BNXT_NAPI_FLAG_XDP)) { 11253 + txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 11254 + if (txq) { 11255 + __netif_tx_lock_bh(txq); 11256 + netif_tx_stop_queue(txq); 11257 + __netif_tx_unlock_bh(txq); 11258 + } 11259 + } 11260 + 11261 + if (!bp->tph_mode) 11262 + continue; 11263 + 11264 + bnxt_hwrm_tx_ring_free(bp, txr, true); 11265 + bnxt_hwrm_cp_ring_free(bp, txr->tx_cpr); 11266 + bnxt_free_one_tx_ring_skbs(bp, txr, txr->txq_index); 11267 + bnxt_clear_one_cp_ring(bp, txr->tx_cpr); 11268 + } 11269 + } 11270 + 11271 + static int bnxt_tx_queue_start(struct bnxt *bp, int idx) 11272 + { 11273 + struct bnxt_tx_ring_info *txr; 11274 + struct netdev_queue *txq; 11275 + struct bnxt_napi *bnapi; 11276 + int rc, i; 11277 + 11278 + bnapi = bp->bnapi[idx]; 11279 + /* All rings have been reserved and previously allocated. 11280 + * Reallocating with the same parameters should never fail. 11281 + */ 11282 + bnxt_for_each_napi_tx(i, bnapi, txr) { 11283 + if (!bp->tph_mode) 11284 + goto start_tx; 11285 + 11286 + rc = bnxt_hwrm_cp_ring_alloc_p5(bp, txr->tx_cpr); 11287 + if (rc) 11288 + return rc; 11289 + 11290 + rc = bnxt_hwrm_tx_ring_alloc(bp, txr, false); 11291 + if (rc) 11292 + return rc; 11293 + 11294 + txr->tx_prod = 0; 11295 + txr->tx_cons = 0; 11296 + txr->tx_hw_cons = 0; 11297 + start_tx: 11298 + WRITE_ONCE(txr->dev_state, 0); 11299 + synchronize_net(); 11300 + 11301 + if (bnapi->flags & BNXT_NAPI_FLAG_XDP) 11302 + continue; 11303 + 11304 + txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 11305 + if (txq) 11306 + netif_tx_start_queue(txq); 11307 + } 11308 + 11309 + return 0; 11310 + } 11311 + 11312 + static void bnxt_irq_affinity_notify(struct irq_affinity_notify *notify, 11313 + const cpumask_t *mask) 11314 + { 11315 + struct bnxt_irq *irq; 11316 + u16 tag; 11317 + int err; 11318 + 11319 + irq = container_of(notify, struct bnxt_irq, affinity_notify); 11320 + 11321 + if (!irq->bp->tph_mode) 11322 + return; 11323 + 11324 + cpumask_copy(irq->cpu_mask, mask); 11325 + 11326 + if (irq->ring_nr >= irq->bp->rx_nr_rings) 11327 + return; 11328 + 11329 + if (pcie_tph_get_cpu_st(irq->bp->pdev, TPH_MEM_TYPE_VM, 11330 + cpumask_first(irq->cpu_mask), &tag)) 11331 + return; 11332 + 11333 + if (pcie_tph_set_st_entry(irq->bp->pdev, irq->msix_nr, tag)) 11334 + return; 11335 + 11336 + rtnl_lock(); 11337 + if (netif_running(irq->bp->dev)) { 11338 + err = netdev_rx_queue_restart(irq->bp->dev, irq->ring_nr); 11339 + if (err) 11340 + netdev_err(irq->bp->dev, 11341 + "RX queue restart failed: err=%d\n", err); 11342 + } 11343 + rtnl_unlock(); 11344 + } 11345 + 11346 + static void bnxt_irq_affinity_release(struct kref *ref) 11347 + { 11348 + struct irq_affinity_notify *notify = 11349 + container_of(ref, struct irq_affinity_notify, kref); 11350 + struct bnxt_irq *irq; 11351 + 11352 + irq = container_of(notify, struct bnxt_irq, affinity_notify); 11353 + 11354 + if (!irq->bp->tph_mode) 11355 + return; 11356 + 11357 + if (pcie_tph_set_st_entry(irq->bp->pdev, irq->msix_nr, 0)) { 11358 + netdev_err(irq->bp->dev, 11359 + "Setting ST=0 for MSIX entry %d failed\n", 11360 + irq->msix_nr); 11361 + return; 11362 + } 11363 + } 11364 + 11365 + static void bnxt_release_irq_notifier(struct bnxt_irq *irq) 11366 + { 11367 + irq_set_affinity_notifier(irq->vector, NULL); 11368 + } 11369 + 11370 + static void bnxt_register_irq_notifier(struct bnxt *bp, struct bnxt_irq *irq) 11371 + { 11372 + struct irq_affinity_notify *notify; 11373 + 11374 + irq->bp = bp; 11375 + 11376 + /* Nothing to do if TPH is not enabled */ 11377 + if (!bp->tph_mode) 11378 + return; 11379 + 11380 + /* Register IRQ affinity notifier */ 11381 + notify = &irq->affinity_notify; 11382 + notify->irq = irq->vector; 11383 + notify->notify = bnxt_irq_affinity_notify; 11384 + notify->release = bnxt_irq_affinity_release; 11385 + 11386 + irq_set_affinity_notifier(irq->vector, notify); 11387 + } 11388 + 11285 11389 static void bnxt_free_irq(struct bnxt *bp) 11286 11390 { 11287 11391 struct bnxt_irq *irq; ··· 11453 11259 free_cpumask_var(irq->cpu_mask); 11454 11260 irq->have_cpumask = 0; 11455 11261 } 11262 + 11263 + bnxt_release_irq_notifier(irq); 11264 + 11456 11265 free_irq(irq->vector, bp->bnapi[i]); 11457 11266 } 11458 11267 11459 11268 irq->requested = 0; 11460 11269 } 11270 + 11271 + /* Disable TPH support */ 11272 + pcie_disable_tph(bp->pdev); 11273 + bp->tph_mode = 0; 11461 11274 } 11462 11275 11463 11276 static int bnxt_request_irq(struct bnxt *bp) ··· 11484 11283 #ifdef CONFIG_RFS_ACCEL 11485 11284 rmap = bp->dev->rx_cpu_rmap; 11486 11285 #endif 11286 + 11287 + /* Enable TPH support as part of IRQ request */ 11288 + rc = pcie_enable_tph(bp->pdev, PCI_TPH_ST_IV_MODE); 11289 + if (!rc) 11290 + bp->tph_mode = PCI_TPH_ST_IV_MODE; 11291 + 11487 11292 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 11488 11293 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 11489 11294 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; ··· 11513 11306 11514 11307 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 11515 11308 int numa_node = dev_to_node(&bp->pdev->dev); 11309 + u16 tag; 11516 11310 11517 11311 irq->have_cpumask = 1; 11312 + irq->msix_nr = map_idx; 11313 + irq->ring_nr = i; 11518 11314 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 11519 11315 irq->cpu_mask); 11520 11316 rc = irq_update_affinity_hint(irq->vector, irq->cpu_mask); ··· 11527 11317 irq->vector); 11528 11318 break; 11529 11319 } 11320 + 11321 + bnxt_register_irq_notifier(bp, irq); 11322 + 11323 + /* Init ST table entry */ 11324 + if (pcie_tph_get_cpu_st(irq->bp->pdev, TPH_MEM_TYPE_VM, 11325 + cpumask_first(irq->cpu_mask), 11326 + &tag)) 11327 + continue; 11328 + 11329 + pcie_tph_set_st_entry(irq->bp->pdev, irq->msix_nr, tag); 11530 11330 } 11531 11331 } 11532 11332 return rc; ··· 15821 15601 struct bnxt_rx_ring_info *rxr, *clone; 15822 15602 struct bnxt_cp_ring_info *cpr; 15823 15603 struct bnxt_vnic_info *vnic; 15604 + struct bnxt_napi *bnapi; 15824 15605 int i, rc; 15825 15606 15826 15607 rxr = &bp->rx_ring[idx]; ··· 15839 15618 15840 15619 bnxt_copy_rx_ring(bp, rxr, clone); 15841 15620 15621 + bnapi = rxr->bnapi; 15622 + cpr = &bnapi->cp_ring; 15623 + 15624 + /* All rings have been reserved and previously allocated. 15625 + * Reallocating with the same parameters should never fail. 15626 + */ 15842 15627 rc = bnxt_hwrm_rx_ring_alloc(bp, rxr); 15843 15628 if (rc) 15844 - return rc; 15629 + goto err_reset; 15630 + 15631 + if (bp->tph_mode) { 15632 + rc = bnxt_hwrm_cp_ring_alloc_p5(bp, rxr->rx_cpr); 15633 + if (rc) 15634 + goto err_reset; 15635 + } 15636 + 15845 15637 rc = bnxt_hwrm_rx_agg_ring_alloc(bp, rxr); 15846 15638 if (rc) 15847 - goto err_free_hwrm_rx_ring; 15639 + goto err_reset; 15848 15640 15849 15641 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 15850 15642 if (bp->flags & BNXT_FLAG_AGG_RINGS) 15851 15643 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 15852 15644 15853 - cpr = &rxr->bnapi->cp_ring; 15854 - cpr->sw_stats->rx.rx_resets++; 15645 + if (bp->flags & BNXT_FLAG_SHARED_RINGS) { 15646 + rc = bnxt_tx_queue_start(bp, idx); 15647 + if (rc) 15648 + goto err_reset; 15649 + } 15650 + 15651 + napi_enable(&bnapi->napi); 15652 + bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 15855 15653 15856 15654 for (i = 0; i <= BNXT_VNIC_NTUPLE; i++) { 15857 15655 vnic = &bp->vnic_info[i]; ··· 15888 15648 15889 15649 return 0; 15890 15650 15891 - err_free_hwrm_rx_ring: 15892 - bnxt_hwrm_rx_ring_free(bp, rxr, false); 15651 + err_reset: 15652 + netdev_err(bp->dev, "Unexpected HWRM error during queue start rc: %d\n", 15653 + rc); 15654 + napi_enable(&bnapi->napi); 15655 + bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 15656 + bnxt_reset_task(bp, true); 15893 15657 return rc; 15894 15658 } 15895 15659 ··· 15901 15657 { 15902 15658 struct bnxt *bp = netdev_priv(dev); 15903 15659 struct bnxt_rx_ring_info *rxr; 15660 + struct bnxt_cp_ring_info *cpr; 15904 15661 struct bnxt_vnic_info *vnic; 15662 + struct bnxt_napi *bnapi; 15905 15663 int i; 15906 15664 15907 15665 for (i = 0; i <= BNXT_VNIC_NTUPLE; i++) { ··· 15915 15669 /* Make sure NAPI sees that the VNIC is disabled */ 15916 15670 synchronize_net(); 15917 15671 rxr = &bp->rx_ring[idx]; 15918 - cancel_work_sync(&rxr->bnapi->cp_ring.dim.work); 15672 + bnapi = rxr->bnapi; 15673 + cpr = &bnapi->cp_ring; 15674 + cancel_work_sync(&cpr->dim.work); 15919 15675 bnxt_hwrm_rx_ring_free(bp, rxr, false); 15920 15676 bnxt_hwrm_rx_agg_ring_free(bp, rxr, false); 15921 - rxr->rx_next_cons = 0; 15922 15677 page_pool_disable_direct_recycling(rxr->page_pool); 15923 15678 if (bnxt_separate_head_pool()) 15924 15679 page_pool_disable_direct_recycling(rxr->head_pool); 15680 + 15681 + if (bp->flags & BNXT_FLAG_SHARED_RINGS) 15682 + bnxt_tx_queue_stop(bp, idx); 15683 + 15684 + /* Disable NAPI now after freeing the rings because HWRM_RING_FREE 15685 + * completion is handled in NAPI to guarantee no more DMA on that ring 15686 + * after seeing the completion. 15687 + */ 15688 + napi_disable(&bnapi->napi); 15689 + 15690 + if (bp->tph_mode) { 15691 + bnxt_hwrm_cp_ring_free(bp, rxr->rx_cpr); 15692 + bnxt_clear_one_cp_ring(bp, rxr->rx_cpr); 15693 + } 15694 + bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 15925 15695 15926 15696 memcpy(qmem, rxr, sizeof(*rxr)); 15927 15697 bnxt_init_rx_ring_struct(bp, qmem);

drivers/net/ethernet/broadcom/bnxt/bnxt.h

··· 1234 1234 u8 have_cpumask:1; 1235 1235 char name[IFNAMSIZ + BNXT_IRQ_NAME_EXTRA]; 1236 1236 cpumask_var_t cpu_mask; 1237 + 1238 + struct bnxt *bp; 1239 + int msix_nr; 1240 + int ring_nr; 1241 + struct irq_affinity_notify affinity_notify; 1237 1242 }; 1238 1243 1239 1244 #define HWRM_RING_ALLOC_TX 0x1 ··· 2415 2410 u8 max_q; 2416 2411 u8 num_tc; 2417 2412 2413 + u8 tph_mode; 2414 + 2418 2415 unsigned int current_interval; 2419 2416 #define BNXT_TIMER_INTERVAL HZ 2420 2417 ··· 2499 2492 #define BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V3 BIT_ULL(39) 2500 2493 #define BNXT_FW_CAP_VNIC_RE_FLUSH BIT_ULL(40) 2501 2494 #define BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS BIT_ULL(41) 2495 + #define BNXT_FW_CAP_NPAR_1_2 BIT_ULL(42) 2502 2496 2503 2497 u32 fw_dbg_cap; 2504 2498

Configure Feed

Configure Feed