Merge branch 'bnxt_en-netdev_queue_mgmt_ops'

+471 -110

1 changed file

expand all

drivers

net

ethernet

broadcom

bnxt

bnxt.c

+471 -110

drivers/net/ethernet/broadcom/bnxt/bnxt.c

··· 3316 3316 } 3317 3317 } 3318 3318 3319 + static void bnxt_free_one_rx_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 3320 + { 3321 + struct pci_dev *pdev = bp->pdev; 3322 + int i, max_idx; 3323 + 3324 + max_idx = bp->rx_nr_pages * RX_DESC_CNT; 3325 + 3326 + for (i = 0; i < max_idx; i++) { 3327 + struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 3328 + dma_addr_t mapping = rx_buf->mapping; 3329 + void *data = rx_buf->data; 3330 + 3331 + if (!data) 3332 + continue; 3333 + 3334 + rx_buf->data = NULL; 3335 + if (BNXT_RX_PAGE_MODE(bp)) { 3336 + page_pool_recycle_direct(rxr->page_pool, data); 3337 + } else { 3338 + dma_unmap_single_attrs(&pdev->dev, mapping, 3339 + bp->rx_buf_use_size, bp->rx_dir, 3340 + DMA_ATTR_WEAK_ORDERING); 3341 + skb_free_frag(data); 3342 + } 3343 + } 3344 + } 3345 + 3346 + static void bnxt_free_one_rx_agg_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 3347 + { 3348 + int i, max_idx; 3349 + 3350 + max_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 3351 + 3352 + for (i = 0; i < max_idx; i++) { 3353 + struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; 3354 + struct page *page = rx_agg_buf->page; 3355 + 3356 + if (!page) 3357 + continue; 3358 + 3359 + rx_agg_buf->page = NULL; 3360 + __clear_bit(i, rxr->rx_agg_bmap); 3361 + 3362 + page_pool_recycle_direct(rxr->page_pool, page); 3363 + } 3364 + } 3365 + 3319 3366 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr) 3320 3367 { 3321 3368 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 3322 3369 struct pci_dev *pdev = bp->pdev; 3323 3370 struct bnxt_tpa_idx_map *map; 3324 - int i, max_idx, max_agg_idx; 3371 + int i; 3325 3372 3326 - max_idx = bp->rx_nr_pages * RX_DESC_CNT; 3327 - max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 3328 3373 if (!rxr->rx_tpa) 3329 3374 goto skip_rx_tpa_free; 3330 3375 ··· 3393 3348 if (!rxr->rx_buf_ring) 3394 3349 goto skip_rx_buf_free; 3395 3350 3396 - for (i = 0; i < max_idx; i++) { 3397 - struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 3398 - dma_addr_t mapping = rx_buf->mapping; 3399 - void *data = rx_buf->data; 3400 - 3401 - if (!data) 3402 - continue; 3403 - 3404 - rx_buf->data = NULL; 3405 - if (BNXT_RX_PAGE_MODE(bp)) { 3406 - page_pool_recycle_direct(rxr->page_pool, data); 3407 - } else { 3408 - dma_unmap_single_attrs(&pdev->dev, mapping, 3409 - bp->rx_buf_use_size, bp->rx_dir, 3410 - DMA_ATTR_WEAK_ORDERING); 3411 - skb_free_frag(data); 3412 - } 3413 - } 3351 + bnxt_free_one_rx_ring(bp, rxr); 3414 3352 3415 3353 skip_rx_buf_free: 3416 3354 if (!rxr->rx_agg_ring) 3417 3355 goto skip_rx_agg_free; 3418 3356 3419 - for (i = 0; i < max_agg_idx; i++) { 3420 - struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; 3421 - struct page *page = rx_agg_buf->page; 3422 - 3423 - if (!page) 3424 - continue; 3425 - 3426 - rx_agg_buf->page = NULL; 3427 - __clear_bit(i, rxr->rx_agg_bmap); 3428 - 3429 - page_pool_recycle_direct(rxr->page_pool, page); 3430 - } 3357 + bnxt_free_one_rx_agg_ring(bp, rxr); 3431 3358 3432 3359 skip_rx_agg_free: 3433 3360 map = rxr->rx_tpa_idx_map; ··· 3996 3979 return 0; 3997 3980 } 3998 3981 3982 + static void bnxt_init_rx_ring_struct(struct bnxt *bp, 3983 + struct bnxt_rx_ring_info *rxr) 3984 + { 3985 + struct bnxt_ring_mem_info *rmem; 3986 + struct bnxt_ring_struct *ring; 3987 + 3988 + ring = &rxr->rx_ring_struct; 3989 + rmem = &ring->ring_mem; 3990 + rmem->nr_pages = bp->rx_nr_pages; 3991 + rmem->page_size = HW_RXBD_RING_SIZE; 3992 + rmem->pg_arr = (void **)rxr->rx_desc_ring; 3993 + rmem->dma_arr = rxr->rx_desc_mapping; 3994 + rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 3995 + rmem->vmem = (void **)&rxr->rx_buf_ring; 3996 + 3997 + ring = &rxr->rx_agg_ring_struct; 3998 + rmem = &ring->ring_mem; 3999 + rmem->nr_pages = bp->rx_agg_nr_pages; 4000 + rmem->page_size = HW_RXBD_RING_SIZE; 4001 + rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 4002 + rmem->dma_arr = rxr->rx_agg_desc_mapping; 4003 + rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 4004 + rmem->vmem = (void **)&rxr->rx_agg_ring; 4005 + } 4006 + 4007 + static void bnxt_reset_rx_ring_struct(struct bnxt *bp, 4008 + struct bnxt_rx_ring_info *rxr) 4009 + { 4010 + struct bnxt_ring_mem_info *rmem; 4011 + struct bnxt_ring_struct *ring; 4012 + int i; 4013 + 4014 + rxr->page_pool->p.napi = NULL; 4015 + rxr->page_pool = NULL; 4016 + 4017 + ring = &rxr->rx_ring_struct; 4018 + rmem = &ring->ring_mem; 4019 + rmem->pg_tbl = NULL; 4020 + rmem->pg_tbl_map = 0; 4021 + for (i = 0; i < rmem->nr_pages; i++) { 4022 + rmem->pg_arr[i] = NULL; 4023 + rmem->dma_arr[i] = 0; 4024 + } 4025 + *rmem->vmem = NULL; 4026 + 4027 + ring = &rxr->rx_agg_ring_struct; 4028 + rmem = &ring->ring_mem; 4029 + rmem->pg_tbl = NULL; 4030 + rmem->pg_tbl_map = 0; 4031 + for (i = 0; i < rmem->nr_pages; i++) { 4032 + rmem->pg_arr[i] = NULL; 4033 + rmem->dma_arr[i] = 0; 4034 + } 4035 + *rmem->vmem = NULL; 4036 + } 4037 + 3999 4038 static void bnxt_init_ring_struct(struct bnxt *bp) 4000 4039 { 4001 4040 int i, j; ··· 4134 4061 } 4135 4062 } 4136 4063 4137 - static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) 4064 + static void bnxt_alloc_one_rx_ring_skb(struct bnxt *bp, 4065 + struct bnxt_rx_ring_info *rxr, 4066 + int ring_nr) 4138 4067 { 4139 - struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 4140 - struct net_device *dev = bp->dev; 4141 4068 u32 prod; 4142 4069 int i; 4143 4070 4144 4071 prod = rxr->rx_prod; 4145 4072 for (i = 0; i < bp->rx_ring_size; i++) { 4146 4073 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) { 4147 - netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 4074 + netdev_warn(bp->dev, "init'ed rx ring %d with %d/%d skbs only\n", 4148 4075 ring_nr, i, bp->rx_ring_size); 4149 4076 break; 4150 4077 } 4151 4078 prod = NEXT_RX(prod); 4152 4079 } 4153 4080 rxr->rx_prod = prod; 4081 + } 4154 4082 4155 - if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 4156 - return 0; 4083 + static void bnxt_alloc_one_rx_ring_page(struct bnxt *bp, 4084 + struct bnxt_rx_ring_info *rxr, 4085 + int ring_nr) 4086 + { 4087 + u32 prod; 4088 + int i; 4157 4089 4158 4090 prod = rxr->rx_agg_prod; 4159 4091 for (i = 0; i < bp->rx_agg_ring_size; i++) { 4160 4092 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) { 4161 - netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 4093 + netdev_warn(bp->dev, "init'ed rx ring %d with %d/%d pages only\n", 4162 4094 ring_nr, i, bp->rx_ring_size); 4163 4095 break; 4164 4096 } 4165 4097 prod = NEXT_RX_AGG(prod); 4166 4098 } 4167 4099 rxr->rx_agg_prod = prod; 4100 + } 4101 + 4102 + static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) 4103 + { 4104 + struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 4105 + int i; 4106 + 4107 + bnxt_alloc_one_rx_ring_skb(bp, rxr, ring_nr); 4108 + 4109 + if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 4110 + return 0; 4111 + 4112 + bnxt_alloc_one_rx_ring_page(bp, rxr, ring_nr); 4168 4113 4169 4114 if (rxr->rx_tpa) { 4170 4115 dma_addr_t mapping; ··· 4201 4110 return 0; 4202 4111 } 4203 4112 4204 - static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 4113 + static void bnxt_init_one_rx_ring_rxbd(struct bnxt *bp, 4114 + struct bnxt_rx_ring_info *rxr) 4205 4115 { 4206 - struct bnxt_rx_ring_info *rxr; 4207 4116 struct bnxt_ring_struct *ring; 4208 4117 u32 type; 4209 4118 ··· 4213 4122 if (NET_IP_ALIGN == 2) 4214 4123 type |= RX_BD_FLAGS_SOP; 4215 4124 4216 - rxr = &bp->rx_ring[ring_nr]; 4217 4125 ring = &rxr->rx_ring_struct; 4218 4126 bnxt_init_rxbd_pages(ring, type); 4127 + ring->fw_ring_id = INVALID_HW_RING_ID; 4128 + } 4129 + 4130 + static void bnxt_init_one_rx_agg_ring_rxbd(struct bnxt *bp, 4131 + struct bnxt_rx_ring_info *rxr) 4132 + { 4133 + struct bnxt_ring_struct *ring; 4134 + u32 type; 4135 + 4136 + ring = &rxr->rx_agg_ring_struct; 4137 + ring->fw_ring_id = INVALID_HW_RING_ID; 4138 + if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { 4139 + type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 4140 + RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 4141 + 4142 + bnxt_init_rxbd_pages(ring, type); 4143 + } 4144 + } 4145 + 4146 + static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 4147 + { 4148 + struct bnxt_rx_ring_info *rxr; 4149 + 4150 + rxr = &bp->rx_ring[ring_nr]; 4151 + bnxt_init_one_rx_ring_rxbd(bp, rxr); 4219 4152 4220 4153 netif_queue_set_napi(bp->dev, ring_nr, NETDEV_QUEUE_TYPE_RX, 4221 4154 &rxr->bnapi->napi); ··· 4248 4133 bpf_prog_add(bp->xdp_prog, 1); 4249 4134 rxr->xdp_prog = bp->xdp_prog; 4250 4135 } 4251 - ring->fw_ring_id = INVALID_HW_RING_ID; 4252 4136 4253 - ring = &rxr->rx_agg_ring_struct; 4254 - ring->fw_ring_id = INVALID_HW_RING_ID; 4255 - 4256 - if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { 4257 - type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 4258 - RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 4259 - 4260 - bnxt_init_rxbd_pages(ring, type); 4261 - } 4137 + bnxt_init_one_rx_agg_ring_rxbd(bp, rxr); 4262 4138 4263 4139 return bnxt_alloc_one_rx_ring(bp, ring_nr); 4264 4140 } ··· 6974 6868 bnxt_set_db_mask(bp, db, ring_type); 6975 6869 } 6976 6870 6871 + static int bnxt_hwrm_rx_ring_alloc(struct bnxt *bp, 6872 + struct bnxt_rx_ring_info *rxr) 6873 + { 6874 + struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 6875 + struct bnxt_napi *bnapi = rxr->bnapi; 6876 + u32 type = HWRM_RING_ALLOC_RX; 6877 + u32 map_idx = bnapi->index; 6878 + int rc; 6879 + 6880 + rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 6881 + if (rc) 6882 + return rc; 6883 + 6884 + bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 6885 + bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 6886 + 6887 + return 0; 6888 + } 6889 + 6890 + static int bnxt_hwrm_rx_agg_ring_alloc(struct bnxt *bp, 6891 + struct bnxt_rx_ring_info *rxr) 6892 + { 6893 + struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 6894 + u32 type = HWRM_RING_ALLOC_AGG; 6895 + u32 grp_idx = ring->grp_idx; 6896 + u32 map_idx; 6897 + int rc; 6898 + 6899 + map_idx = grp_idx + bp->rx_nr_rings; 6900 + rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 6901 + if (rc) 6902 + return rc; 6903 + 6904 + bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 6905 + ring->fw_ring_id); 6906 + bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 6907 + bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 6908 + bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 6909 + 6910 + return 0; 6911 + } 6912 + 6977 6913 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 6978 6914 { 6979 6915 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); ··· 7081 6933 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id); 7082 6934 } 7083 6935 7084 - type = HWRM_RING_ALLOC_RX; 7085 6936 for (i = 0; i < bp->rx_nr_rings; i++) { 7086 6937 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 7087 - struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 7088 - struct bnxt_napi *bnapi = rxr->bnapi; 7089 - u32 map_idx = bnapi->index; 7090 6938 7091 - rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 6939 + rc = bnxt_hwrm_rx_ring_alloc(bp, rxr); 7092 6940 if (rc) 7093 6941 goto err_out; 7094 - bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 7095 6942 /* If we have agg rings, post agg buffers first. */ 7096 6943 if (!agg_rings) 7097 6944 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 7098 - bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 7099 6945 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7100 6946 struct bnxt_cp_ring_info *cpr2 = rxr->rx_cpr; 6947 + struct bnxt_napi *bnapi = rxr->bnapi; 7101 6948 u32 type2 = HWRM_RING_ALLOC_CMPL; 6949 + struct bnxt_ring_struct *ring; 6950 + u32 map_idx = bnapi->index; 7102 6951 7103 6952 ring = &cpr2->cp_ring_struct; 7104 6953 ring->handle = BNXT_SET_NQ_HDL(cpr2); ··· 7109 6964 } 7110 6965 7111 6966 if (agg_rings) { 7112 - type = HWRM_RING_ALLOC_AGG; 7113 6967 for (i = 0; i < bp->rx_nr_rings; i++) { 7114 - struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 7115 - struct bnxt_ring_struct *ring = 7116 - &rxr->rx_agg_ring_struct; 7117 - u32 grp_idx = ring->grp_idx; 7118 - u32 map_idx = grp_idx + bp->rx_nr_rings; 7119 - 7120 - rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 6968 + rc = bnxt_hwrm_rx_agg_ring_alloc(bp, &bp->rx_ring[i]); 7121 6969 if (rc) 7122 6970 goto err_out; 7123 - 7124 - bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 7125 - ring->fw_ring_id); 7126 - bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 7127 - bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 7128 - bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 7129 6971 } 7130 6972 } 7131 6973 err_out: ··· 7152 7020 return 0; 7153 7021 } 7154 7022 7023 + static void bnxt_hwrm_rx_ring_free(struct bnxt *bp, 7024 + struct bnxt_rx_ring_info *rxr, 7025 + bool close_path) 7026 + { 7027 + struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 7028 + u32 grp_idx = rxr->bnapi->index; 7029 + u32 cmpl_ring_id; 7030 + 7031 + if (ring->fw_ring_id == INVALID_HW_RING_ID) 7032 + return; 7033 + 7034 + cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 7035 + hwrm_ring_free_send_msg(bp, ring, 7036 + RING_FREE_REQ_RING_TYPE_RX, 7037 + close_path ? cmpl_ring_id : 7038 + INVALID_HW_RING_ID); 7039 + ring->fw_ring_id = INVALID_HW_RING_ID; 7040 + bp->grp_info[grp_idx].rx_fw_ring_id = INVALID_HW_RING_ID; 7041 + } 7042 + 7043 + static void bnxt_hwrm_rx_agg_ring_free(struct bnxt *bp, 7044 + struct bnxt_rx_ring_info *rxr, 7045 + bool close_path) 7046 + { 7047 + struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 7048 + u32 grp_idx = rxr->bnapi->index; 7049 + u32 type, cmpl_ring_id; 7050 + 7051 + if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7052 + type = RING_FREE_REQ_RING_TYPE_RX_AGG; 7053 + else 7054 + type = RING_FREE_REQ_RING_TYPE_RX; 7055 + 7056 + if (ring->fw_ring_id == INVALID_HW_RING_ID) 7057 + return; 7058 + 7059 + cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 7060 + hwrm_ring_free_send_msg(bp, ring, type, 7061 + close_path ? cmpl_ring_id : 7062 + INVALID_HW_RING_ID); 7063 + ring->fw_ring_id = INVALID_HW_RING_ID; 7064 + bp->grp_info[grp_idx].agg_fw_ring_id = INVALID_HW_RING_ID; 7065 + } 7066 + 7155 7067 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 7156 7068 { 7157 7069 u32 type; ··· 7220 7044 } 7221 7045 7222 7046 for (i = 0; i < bp->rx_nr_rings; i++) { 7223 - struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 7224 - struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 7225 - u32 grp_idx = rxr->bnapi->index; 7226 - 7227 - if (ring->fw_ring_id != INVALID_HW_RING_ID) { 7228 - u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 7229 - 7230 - hwrm_ring_free_send_msg(bp, ring, 7231 - RING_FREE_REQ_RING_TYPE_RX, 7232 - close_path ? cmpl_ring_id : 7233 - INVALID_HW_RING_ID); 7234 - ring->fw_ring_id = INVALID_HW_RING_ID; 7235 - bp->grp_info[grp_idx].rx_fw_ring_id = 7236 - INVALID_HW_RING_ID; 7237 - } 7238 - } 7239 - 7240 - if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7241 - type = RING_FREE_REQ_RING_TYPE_RX_AGG; 7242 - else 7243 - type = RING_FREE_REQ_RING_TYPE_RX; 7244 - for (i = 0; i < bp->rx_nr_rings; i++) { 7245 - struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 7246 - struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 7247 - u32 grp_idx = rxr->bnapi->index; 7248 - 7249 - if (ring->fw_ring_id != INVALID_HW_RING_ID) { 7250 - u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 7251 - 7252 - hwrm_ring_free_send_msg(bp, ring, type, 7253 - close_path ? cmpl_ring_id : 7254 - INVALID_HW_RING_ID); 7255 - ring->fw_ring_id = INVALID_HW_RING_ID; 7256 - bp->grp_info[grp_idx].agg_fw_ring_id = 7257 - INVALID_HW_RING_ID; 7258 - } 7047 + bnxt_hwrm_rx_ring_free(bp, &bp->rx_ring[i], close_path); 7048 + bnxt_hwrm_rx_agg_ring_free(bp, &bp->rx_ring[i], close_path); 7259 7049 } 7260 7050 7261 7051 /* The completion rings are about to be freed. After that the ··· 14970 14828 .get_base_stats = bnxt_get_base_stats, 14971 14829 }; 14972 14830 14831 + static int bnxt_alloc_rx_agg_bmap(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 14832 + { 14833 + u16 mem_size; 14834 + 14835 + rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 14836 + mem_size = rxr->rx_agg_bmap_size / 8; 14837 + rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 14838 + if (!rxr->rx_agg_bmap) 14839 + return -ENOMEM; 14840 + 14841 + return 0; 14842 + } 14843 + 14844 + static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) 14845 + { 14846 + struct bnxt_rx_ring_info *rxr, *clone; 14847 + struct bnxt *bp = netdev_priv(dev); 14848 + struct bnxt_ring_struct *ring; 14849 + int rc; 14850 + 14851 + rxr = &bp->rx_ring[idx]; 14852 + clone = qmem; 14853 + memcpy(clone, rxr, sizeof(*rxr)); 14854 + bnxt_init_rx_ring_struct(bp, clone); 14855 + bnxt_reset_rx_ring_struct(bp, clone); 14856 + 14857 + clone->rx_prod = 0; 14858 + clone->rx_agg_prod = 0; 14859 + clone->rx_sw_agg_prod = 0; 14860 + clone->rx_next_cons = 0; 14861 + 14862 + rc = bnxt_alloc_rx_page_pool(bp, clone, rxr->page_pool->p.nid); 14863 + if (rc) 14864 + return rc; 14865 + 14866 + ring = &clone->rx_ring_struct; 14867 + rc = bnxt_alloc_ring(bp, &ring->ring_mem); 14868 + if (rc) 14869 + goto err_free_rx_ring; 14870 + 14871 + if (bp->flags & BNXT_FLAG_AGG_RINGS) { 14872 + ring = &clone->rx_agg_ring_struct; 14873 + rc = bnxt_alloc_ring(bp, &ring->ring_mem); 14874 + if (rc) 14875 + goto err_free_rx_agg_ring; 14876 + 14877 + rc = bnxt_alloc_rx_agg_bmap(bp, clone); 14878 + if (rc) 14879 + goto err_free_rx_agg_ring; 14880 + } 14881 + 14882 + bnxt_init_one_rx_ring_rxbd(bp, clone); 14883 + bnxt_init_one_rx_agg_ring_rxbd(bp, clone); 14884 + 14885 + bnxt_alloc_one_rx_ring_skb(bp, clone, idx); 14886 + if (bp->flags & BNXT_FLAG_AGG_RINGS) 14887 + bnxt_alloc_one_rx_ring_page(bp, clone, idx); 14888 + 14889 + return 0; 14890 + 14891 + err_free_rx_agg_ring: 14892 + bnxt_free_ring(bp, &clone->rx_agg_ring_struct.ring_mem); 14893 + err_free_rx_ring: 14894 + bnxt_free_ring(bp, &clone->rx_ring_struct.ring_mem); 14895 + clone->page_pool->p.napi = NULL; 14896 + page_pool_destroy(clone->page_pool); 14897 + clone->page_pool = NULL; 14898 + return rc; 14899 + } 14900 + 14901 + static void bnxt_queue_mem_free(struct net_device *dev, void *qmem) 14902 + { 14903 + struct bnxt_rx_ring_info *rxr = qmem; 14904 + struct bnxt *bp = netdev_priv(dev); 14905 + struct bnxt_ring_struct *ring; 14906 + 14907 + bnxt_free_one_rx_ring(bp, rxr); 14908 + bnxt_free_one_rx_agg_ring(bp, rxr); 14909 + 14910 + /* At this point, this NAPI instance has another page pool associated 14911 + * with it. Disconnect here before freeing the old page pool to avoid 14912 + * warnings. 14913 + */ 14914 + rxr->page_pool->p.napi = NULL; 14915 + page_pool_destroy(rxr->page_pool); 14916 + rxr->page_pool = NULL; 14917 + 14918 + ring = &rxr->rx_ring_struct; 14919 + bnxt_free_ring(bp, &ring->ring_mem); 14920 + 14921 + ring = &rxr->rx_agg_ring_struct; 14922 + bnxt_free_ring(bp, &ring->ring_mem); 14923 + 14924 + kfree(rxr->rx_agg_bmap); 14925 + rxr->rx_agg_bmap = NULL; 14926 + } 14927 + 14928 + static void bnxt_copy_rx_ring(struct bnxt *bp, 14929 + struct bnxt_rx_ring_info *dst, 14930 + struct bnxt_rx_ring_info *src) 14931 + { 14932 + struct bnxt_ring_mem_info *dst_rmem, *src_rmem; 14933 + struct bnxt_ring_struct *dst_ring, *src_ring; 14934 + int i; 14935 + 14936 + dst_ring = &dst->rx_ring_struct; 14937 + dst_rmem = &dst_ring->ring_mem; 14938 + src_ring = &src->rx_ring_struct; 14939 + src_rmem = &src_ring->ring_mem; 14940 + 14941 + WARN_ON(dst_rmem->nr_pages != src_rmem->nr_pages); 14942 + WARN_ON(dst_rmem->page_size != src_rmem->page_size); 14943 + WARN_ON(dst_rmem->flags != src_rmem->flags); 14944 + WARN_ON(dst_rmem->depth != src_rmem->depth); 14945 + WARN_ON(dst_rmem->vmem_size != src_rmem->vmem_size); 14946 + WARN_ON(dst_rmem->ctx_mem != src_rmem->ctx_mem); 14947 + 14948 + dst_rmem->pg_tbl = src_rmem->pg_tbl; 14949 + dst_rmem->pg_tbl_map = src_rmem->pg_tbl_map; 14950 + *dst_rmem->vmem = *src_rmem->vmem; 14951 + for (i = 0; i < dst_rmem->nr_pages; i++) { 14952 + dst_rmem->pg_arr[i] = src_rmem->pg_arr[i]; 14953 + dst_rmem->dma_arr[i] = src_rmem->dma_arr[i]; 14954 + } 14955 + 14956 + if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 14957 + return; 14958 + 14959 + dst_ring = &dst->rx_agg_ring_struct; 14960 + dst_rmem = &dst_ring->ring_mem; 14961 + src_ring = &src->rx_agg_ring_struct; 14962 + src_rmem = &src_ring->ring_mem; 14963 + 14964 + WARN_ON(dst_rmem->nr_pages != src_rmem->nr_pages); 14965 + WARN_ON(dst_rmem->page_size != src_rmem->page_size); 14966 + WARN_ON(dst_rmem->flags != src_rmem->flags); 14967 + WARN_ON(dst_rmem->depth != src_rmem->depth); 14968 + WARN_ON(dst_rmem->vmem_size != src_rmem->vmem_size); 14969 + WARN_ON(dst_rmem->ctx_mem != src_rmem->ctx_mem); 14970 + WARN_ON(dst->rx_agg_bmap_size != src->rx_agg_bmap_size); 14971 + 14972 + dst_rmem->pg_tbl = src_rmem->pg_tbl; 14973 + dst_rmem->pg_tbl_map = src_rmem->pg_tbl_map; 14974 + *dst_rmem->vmem = *src_rmem->vmem; 14975 + for (i = 0; i < dst_rmem->nr_pages; i++) { 14976 + dst_rmem->pg_arr[i] = src_rmem->pg_arr[i]; 14977 + dst_rmem->dma_arr[i] = src_rmem->dma_arr[i]; 14978 + } 14979 + 14980 + dst->rx_agg_bmap = src->rx_agg_bmap; 14981 + } 14982 + 14983 + static int bnxt_queue_start(struct net_device *dev, void *qmem, int idx) 14984 + { 14985 + struct bnxt *bp = netdev_priv(dev); 14986 + struct bnxt_rx_ring_info *rxr, *clone; 14987 + struct bnxt_cp_ring_info *cpr; 14988 + int rc; 14989 + 14990 + rxr = &bp->rx_ring[idx]; 14991 + clone = qmem; 14992 + 14993 + rxr->rx_prod = clone->rx_prod; 14994 + rxr->rx_agg_prod = clone->rx_agg_prod; 14995 + rxr->rx_sw_agg_prod = clone->rx_sw_agg_prod; 14996 + rxr->rx_next_cons = clone->rx_next_cons; 14997 + rxr->page_pool = clone->page_pool; 14998 + 14999 + bnxt_copy_rx_ring(bp, rxr, clone); 15000 + 15001 + rc = bnxt_hwrm_rx_ring_alloc(bp, rxr); 15002 + if (rc) 15003 + return rc; 15004 + rc = bnxt_hwrm_rx_agg_ring_alloc(bp, rxr); 15005 + if (rc) 15006 + goto err_free_hwrm_rx_ring; 15007 + 15008 + bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 15009 + if (bp->flags & BNXT_FLAG_AGG_RINGS) 15010 + bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 15011 + 15012 + napi_enable(&rxr->bnapi->napi); 15013 + 15014 + cpr = &rxr->bnapi->cp_ring; 15015 + cpr->sw_stats->rx.rx_resets++; 15016 + 15017 + return 0; 15018 + 15019 + err_free_hwrm_rx_ring: 15020 + bnxt_hwrm_rx_ring_free(bp, rxr, false); 15021 + return rc; 15022 + } 15023 + 15024 + static int bnxt_queue_stop(struct net_device *dev, void *qmem, int idx) 15025 + { 15026 + struct bnxt *bp = netdev_priv(dev); 15027 + struct bnxt_rx_ring_info *rxr; 15028 + 15029 + rxr = &bp->rx_ring[idx]; 15030 + napi_disable(&rxr->bnapi->napi); 15031 + bnxt_hwrm_rx_ring_free(bp, rxr, false); 15032 + bnxt_hwrm_rx_agg_ring_free(bp, rxr, false); 15033 + rxr->rx_next_cons = 0; 15034 + 15035 + memcpy(qmem, rxr, sizeof(*rxr)); 15036 + bnxt_init_rx_ring_struct(bp, qmem); 15037 + 15038 + return 0; 15039 + } 15040 + 15041 + static const struct netdev_queue_mgmt_ops bnxt_queue_mgmt_ops = { 15042 + .ndo_queue_mem_size = sizeof(struct bnxt_rx_ring_info), 15043 + .ndo_queue_mem_alloc = bnxt_queue_mem_alloc, 15044 + .ndo_queue_mem_free = bnxt_queue_mem_free, 15045 + .ndo_queue_start = bnxt_queue_start, 15046 + .ndo_queue_stop = bnxt_queue_stop, 15047 + }; 15048 + 14973 15049 static void bnxt_remove_one(struct pci_dev *pdev) 14974 15050 { 14975 15051 struct net_device *dev = pci_get_drvdata(pdev); ··· 15653 15293 dev->stat_ops = &bnxt_stat_ops; 15654 15294 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 15655 15295 dev->ethtool_ops = &bnxt_ethtool_ops; 15296 + dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops; 15656 15297 pci_set_drvdata(pdev, dev); 15657 15298 15658 15299 rc = bnxt_alloc_hwrm_resources(bp);

Configure Feed

Configure Feed