RDMA/irdma: Add GEN3 support for AEQ and CEQ

+64 -12

drivers/infiniband/hw/irdma/ctrl.c

··· 2566 2566 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) | 2567 2567 FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, check_overflow) | 2568 2568 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) | 2569 + FIELD_PREP(IRDMA_CQPSQ_CQ_CQID_HIGH, cq->cq_uk.cq_id >> 22) | 2570 + FIELD_PREP(IRDMA_CQPSQ_CQ_CEQID_HIGH, 2571 + (cq->ceq_id_valid ? cq->ceq_id : 0) >> 10) | 2569 2572 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | 2570 2573 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) | 2571 2574 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | ··· 3931 3928 ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL); 3932 3929 ceq->tph_en = info->tph_en; 3933 3930 ceq->tph_val = info->tph_val; 3934 - ceq->vsi = info->vsi; 3931 + ceq->vsi_idx = info->vsi_idx; 3935 3932 ceq->polarity = 1; 3936 3933 IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt); 3937 3934 ceq->dev->ceq[info->ceq_id] = ceq; ··· 3964 3961 (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0)); 3965 3962 set_64bit_val(wqe, 56, 3966 3963 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) | 3967 - FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx)); 3964 + FIELD_PREP(IRDMA_CQPSQ_PASID, ceq->pasid) | 3965 + FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi_idx)); 3968 3966 hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) | 3967 + FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID_HIGH, ceq->ceq_id >> 10) | 3969 3968 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CEQ) | 3970 3969 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) | 3971 3970 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) | 3972 3971 FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) | 3973 3972 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) | 3973 + FIELD_PREP(IRDMA_CQPSQ_PASID_VALID, ceq->pasid_valid) | 3974 3974 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 3975 3975 dma_wmb(); /* make sure WQE is written before valid bit is set */ 3976 3976 ··· 4028 4022 int ret_code; 4029 4023 struct irdma_sc_dev *dev = ceq->dev; 4030 4024 4031 - dev->ccq->vsi = ceq->vsi; 4025 + dev->ccq->vsi_idx = ceq->vsi_idx; 4032 4026 if (ceq->reg_cq) { 4033 4027 ret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq); 4034 4028 if (ret_code) ··· 4061 4055 4062 4056 set_64bit_val(wqe, 16, ceq->elem_cnt); 4063 4057 set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx); 4058 + set_64bit_val(wqe, 56, 4059 + FIELD_PREP(IRDMA_CQPSQ_PASID, ceq->pasid)); 4064 4060 hdr = ceq->ceq_id | 4065 4061 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CEQ) | 4066 4062 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) | 4067 4063 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) | 4068 4064 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) | 4065 + FIELD_PREP(IRDMA_CQPSQ_PASID_VALID, ceq->pasid_valid) | 4069 4066 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 4070 4067 dma_wmb(); /* make sure WQE is written before valid bit is set */ 4071 4068 ··· 4232 4223 (aeq->virtual_map ? 0 : aeq->aeq_elem_pa)); 4233 4224 set_64bit_val(wqe, 48, 4234 4225 (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0)); 4226 + set_64bit_val(wqe, 56, 4227 + FIELD_PREP(IRDMA_CQPSQ_PASID, aeq->pasid)); 4235 4228 4236 4229 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_AEQ) | 4237 4230 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) | 4238 4231 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) | 4232 + FIELD_PREP(IRDMA_CQPSQ_PASID_VALID, aeq->pasid_valid) | 4239 4233 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 4240 4234 dma_wmb(); /* make sure WQE is written before valid bit is set */ 4241 4235 ··· 4267 4255 u64 hdr; 4268 4256 4269 4257 dev = aeq->dev; 4270 - writel(0, dev->hw_regs[IRDMA_PFINT_AEQCTL]); 4258 + if (dev->privileged) 4259 + writel(0, dev->hw_regs[IRDMA_PFINT_AEQCTL]); 4271 4260 4272 4261 cqp = dev->cqp; 4273 4262 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); ··· 4276 4263 return -ENOMEM; 4277 4264 set_64bit_val(wqe, 16, aeq->elem_cnt); 4278 4265 set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx); 4266 + set_64bit_val(wqe, 56, 4267 + FIELD_PREP(IRDMA_CQPSQ_PASID, aeq->pasid)); 4279 4268 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_AEQ) | 4280 4269 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) | 4281 4270 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) | 4271 + FIELD_PREP(IRDMA_CQPSQ_PASID_VALID, aeq->pasid_valid) | 4282 4272 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 4283 4273 dma_wmb(); /* make sure WQE is written before valid bit is set */ 4284 4274 ··· 4322 4306 print_hex_dump_debug("WQE: AEQ_ENTRY WQE", DUMP_PREFIX_OFFSET, 16, 8, 4323 4307 aeqe, 16, false); 4324 4308 4325 - ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp); 4326 - info->wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX, temp); 4327 - info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) | 4309 + if (aeq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) { 4310 + ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC_GEN_3, temp); 4311 + info->wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX_GEN_3, 4312 + temp); 4313 + info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_GEN_3, temp); 4314 + info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE_GEN_3, temp); 4315 + info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE_GEN_3, compl_ctx); 4316 + info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE_GEN_3, temp); 4317 + info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA_GEN_3, compl_ctx); 4318 + info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW_GEN_3, temp); 4319 + info->compl_ctx = FIELD_GET(IRDMA_AEQE_CMPL_CTXT, compl_ctx); 4320 + compl_ctx = FIELD_GET(IRDMA_AEQE_CMPL_CTXT, compl_ctx) << IRDMA_AEQE_CMPL_CTXT_S; 4321 + } else { 4322 + ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp); 4323 + info->wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX, temp); 4324 + info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) | 4328 4325 ((u32)FIELD_GET(IRDMA_AEQE_QPCQID_HI, temp) << 18); 4329 - info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp); 4330 - info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp); 4331 - info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp); 4332 - info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp); 4333 - info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW, temp); 4326 + info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp); 4327 + info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp); 4328 + info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp); 4329 + info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp); 4330 + info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW, 4331 + temp); 4332 + } 4334 4333 4335 4334 info->ae_src = ae_src; 4336 4335 switch (info->ae_id) { 4336 + case IRDMA_AE_SRQ_LIMIT: 4337 + info->srq = true; 4338 + /* [63:6] from CMPL_CTXT, [5:0] from WQDESCIDX. */ 4339 + info->compl_ctx = compl_ctx | info->wqe_idx; 4340 + ae_src = IRDMA_AE_SOURCE_RSVD; 4341 + break; 4337 4342 case IRDMA_AE_PRIV_OPERATION_DENIED: 4338 4343 case IRDMA_AE_AMP_INVALIDATE_TYPE1_MW: 4339 4344 case IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW: ··· 4387 4350 case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: 4388 4351 case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: 4389 4352 case IRDMA_AE_LLP_TOO_MANY_RETRIES: 4353 + case IRDMA_AE_LLP_TOO_MANY_RNRS: 4354 + case IRDMA_AE_REMOTE_QP_CATASTROPHIC: 4355 + case IRDMA_AE_LOCAL_QP_CATASTROPHIC: 4356 + case IRDMA_AE_RCE_QP_CATASTROPHIC: 4390 4357 case IRDMA_AE_LLP_DOUBT_REACHABILITY: 4391 4358 case IRDMA_AE_LLP_CONNECTION_ESTABLISHED: 4392 4359 case IRDMA_AE_RESET_SENT: ··· 4436 4395 info->qp = true; 4437 4396 info->rq = true; 4438 4397 info->compl_ctx = compl_ctx; 4398 + info->err_rq_idx_valid = true; 4439 4399 break; 4440 4400 case IRDMA_AE_SOURCE_CQ: 4441 4401 case IRDMA_AE_SOURCE_CQ_0110: ··· 4452 4410 info->compl_ctx = compl_ctx; 4453 4411 break; 4454 4412 case IRDMA_AE_SOURCE_IN_RR_WR: 4413 + info->qp = true; 4414 + if (aeq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) 4415 + info->err_rq_idx_valid = true; 4416 + info->compl_ctx = compl_ctx; 4417 + info->in_rdrsp_wr = true; 4418 + break; 4455 4419 case IRDMA_AE_SOURCE_IN_RR_WR_1011: 4456 4420 info->qp = true; 4421 + if (aeq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) { 4422 + info->sq = true; 4423 + info->err_rq_idx_valid = true; 4424 + } 4457 4425 info->compl_ctx = compl_ctx; 4458 4426 info->in_rdrsp_wr = true; 4459 4427 break;

+28 -1

drivers/infiniband/hw/irdma/defs.h

··· 319 319 #define IRDMA_AE_STAG_ZERO_INVALID 0x0206 320 320 #define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207 321 321 #define IRDMA_AE_IB_INVALID_REQUEST 0x0208 322 + #define IRDMA_AE_SRQ_LIMIT 0x0209 322 323 #define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a 323 324 #define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b 324 325 #define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c 325 326 #define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d 326 327 #define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e 328 + #define IRDMA_AE_SRQ_CATASTROPHIC_ERROR 0x020f 327 329 #define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220 330 + #define IRDMA_AE_ATOMIC_ALIGNMENT 0x0221 331 + #define IRDMA_AE_ATOMIC_MASK 0x0222 328 332 #define IRDMA_AE_INVALID_REQUEST 0x0223 333 + #define IRDMA_AE_PCIE_ATOMIC_DISABLE 0x0224 329 334 #define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301 330 335 #define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303 331 336 #define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304 ··· 371 366 #define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700 372 367 #define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701 373 368 #define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702 369 + #define IRDMA_AE_REMOTE_QP_CATASTROPHIC 0x0703 370 + #define IRDMA_AE_LOCAL_QP_CATASTROPHIC 0x0704 371 + #define IRDMA_AE_RCE_QP_CATASTROPHIC 0x0705 374 372 #define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900 375 373 #define IRDMA_AE_CQP_DEFERRED_COMPLETE 0x0901 374 + #define IRDMA_AE_ADAPTER_CATASTROPHIC 0x0B0B 376 375 377 376 #define FLD_LS_64(dev, val, field) \ 378 377 (((u64)(val) << (dev)->hw_shifts[field ## _S]) & (dev)->hw_masks[field ## _M]) ··· 544 535 #define IRDMA_AEQE_Q2DATA GENMASK_ULL(62, 61) 545 536 #define IRDMA_AEQE_VALID BIT_ULL(63) 546 537 538 + #define IRDMA_AEQE_Q2DATA_GEN_3 GENMASK_ULL(5, 4) 539 + #define IRDMA_AEQE_TCPSTATE_GEN_3 GENMASK_ULL(3, 0) 540 + #define IRDMA_AEQE_QPCQID_GEN_3 GENMASK_ULL(24, 0) 541 + #define IRDMA_AEQE_AECODE_GEN_3 GENMASK_ULL(61, 50) 542 + #define IRDMA_AEQE_OVERFLOW_GEN_3 BIT_ULL(62) 543 + #define IRDMA_AEQE_WQDESCIDX_GEN_3 GENMASK_ULL(49, 32) 544 + #define IRDMA_AEQE_IWSTATE_GEN_3 GENMASK_ULL(31, 29) 545 + #define IRDMA_AEQE_AESRC_GEN_3 GENMASK_ULL(28, 25) 546 + #define IRDMA_AEQE_CMPL_CTXT_S 6 547 + #define IRDMA_AEQE_CMPL_CTXT GENMASK_ULL(63, 6) 548 + 547 549 #define IRDMA_UDA_QPSQ_NEXT_HDR GENMASK_ULL(23, 16) 548 550 #define IRDMA_UDA_QPSQ_OPCODE GENMASK_ULL(37, 32) 549 551 #define IRDMA_UDA_QPSQ_L4LEN GENMASK_ULL(45, 42) ··· 577 557 #define IRDMA_CQPSQ_WQEVALID BIT_ULL(63) 578 558 #define IRDMA_CQPSQ_TPHVAL GENMASK_ULL(7, 0) 579 559 580 - #define IRDMA_CQPSQ_VSIIDX GENMASK_ULL(17, 8) 560 + #define IRDMA_CQPSQ_VSIIDX GENMASK_ULL(23, 8) 581 561 #define IRDMA_CQPSQ_TPHEN BIT_ULL(60) 582 562 583 563 #define IRDMA_CQPSQ_PBUFADDR IRDMA_CQPHC_QPCTX 564 + 565 + #define IRDMA_CQPSQ_PASID GENMASK_ULL(51, 32) 566 + #define IRDMA_CQPSQ_PASID_VALID BIT_ULL(62) 584 567 585 568 /* Create/Modify/Destroy QP */ 586 569 ··· 620 597 #define IRDMA_CQPSQ_CQ_CQCTX GENMASK_ULL(62, 0) 621 598 #define IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD GENMASK(17, 0) 622 599 600 + #define IRDMA_CQPSQ_CQ_CQID_HIGH GENMASK_ULL(52, 50) 601 + #define IRDMA_CQPSQ_CQ_CEQID_HIGH GENMASK_ULL(59, 54) 623 602 #define IRDMA_CQPSQ_CQ_OP GENMASK_ULL(37, 32) 624 603 #define IRDMA_CQPSQ_CQ_CQRESIZE BIT_ULL(43) 625 604 #define IRDMA_CQPSQ_CQ_LPBLSIZE GENMASK_ULL(45, 44) ··· 700 675 #define IRDMA_CQPSQ_SHMCRP_VFNUM GENMASK_ULL(37, 32) 701 676 #define IRDMA_CQPSQ_CEQ_CEQSIZE GENMASK_ULL(21, 0) 702 677 #define IRDMA_CQPSQ_CEQ_CEQID GENMASK_ULL(9, 0) 678 + 679 + #define IRDMA_CQPSQ_CEQ_CEQID_HIGH GENMASK_ULL(15, 10) 703 680 704 681 #define IRDMA_CQPSQ_CEQ_LPBLSIZE IRDMA_CQPSQ_CQ_LPBLSIZE 705 682 #define IRDMA_CQPSQ_CEQ_VMAP BIT_ULL(47)

+78 -56

drivers/infiniband/hw/irdma/hw.c

··· 282 282 if (ret) 283 283 break; 284 284 285 + if (info->aeqe_overflow) { 286 + ibdev_err(&iwdev->ibdev, "AEQ has overflowed\n"); 287 + rf->reset = true; 288 + rf->gen_ops.request_reset(rf); 289 + return; 290 + } 291 + 285 292 aeqcnt++; 286 293 ibdev_dbg(&iwdev->ibdev, 287 294 "AEQ: ae_id = 0x%x bool qp=%d qp_id = %d tcp_state=%d iwarp_state=%d ae_src=%d\n", ··· 449 442 case IRDMA_AE_LCE_FUNCTION_CATASTROPHIC: 450 443 case IRDMA_AE_LLP_TOO_MANY_RNRS: 451 444 case IRDMA_AE_LCE_CQ_CATASTROPHIC: 445 + case IRDMA_AE_REMOTE_QP_CATASTROPHIC: 446 + case IRDMA_AE_LOCAL_QP_CATASTROPHIC: 447 + case IRDMA_AE_RCE_QP_CATASTROPHIC: 452 448 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG: 453 449 default: 454 450 ibdev_err(&iwdev->ibdev, "abnormal ae_id = 0x%x bool qp=%d qp_id = %d, ae_src=%d\n", ··· 696 686 int status = -EBUSY; 697 687 698 688 if (!rf->msix_shared) { 699 - rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, rf->iw_msixtbl->idx, false); 689 + if (rf->sc_dev.privileged) 690 + rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, 691 + rf->iw_msixtbl->idx, false); 700 692 irdma_destroy_irq(rf, rf->iw_msixtbl, rf); 701 693 } 702 694 if (rf->reset) ··· 764 752 765 753 if (rf->msix_shared) { 766 754 msix_vec = &rf->iw_msixtbl[0]; 767 - rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, 768 - msix_vec->ceq_id, 769 - msix_vec->idx, false); 755 + if (rf->sc_dev.privileged) 756 + rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, 757 + msix_vec->ceq_id, 758 + msix_vec->idx, false); 770 759 irdma_destroy_irq(rf, msix_vec, rf); 771 760 } else { 772 761 msix_vec = &rf->iw_msixtbl[1]; ··· 798 785 msix_vec = &rf->iw_msixtbl[2]; 799 786 800 787 for (i = 1; i < rf->ceqs_count; i++, msix_vec++, iwceq++) { 801 - rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, msix_vec->ceq_id, 802 - msix_vec->idx, false); 788 + if (rf->sc_dev.privileged) 789 + rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, 790 + msix_vec->ceq_id, 791 + msix_vec->idx, false); 803 792 irdma_destroy_irq(rf, msix_vec, iwceq); 804 793 irdma_cqp_ceq_cmd(&rf->sc_dev, &iwceq->sc_ceq, 805 794 IRDMA_OP_CEQ_DESTROY); ··· 1224 1209 } 1225 1210 1226 1211 msix_vec->ceq_id = ceq_id; 1227 - rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id, msix_vec->idx, true); 1228 - 1229 - return 0; 1212 + if (rf->sc_dev.privileged) 1213 + rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id, 1214 + msix_vec->idx, true); 1215 + else 1216 + status = irdma_vchnl_req_ceq_vec_map(&rf->sc_dev, ceq_id, 1217 + msix_vec->idx); 1218 + return status; 1230 1219 } 1231 1220 1232 1221 /** ··· 1243 1224 static int irdma_cfg_aeq_vector(struct irdma_pci_f *rf) 1244 1225 { 1245 1226 struct irdma_msix_vector *msix_vec = rf->iw_msixtbl; 1246 - u32 ret = 0; 1227 + int ret = 0; 1247 1228 1248 1229 if (!rf->msix_shared) { 1249 1230 snprintf(msix_vec->name, sizeof(msix_vec->name) - 1, ··· 1254 1235 } 1255 1236 if (ret) { 1256 1237 ibdev_dbg(&rf->iwdev->ibdev, "ERR: aeq irq config fail\n"); 1257 - return -EINVAL; 1238 + return ret; 1258 1239 } 1259 1240 1260 - rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, msix_vec->idx, true); 1241 + if (rf->sc_dev.privileged) 1242 + rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, msix_vec->idx, 1243 + true); 1244 + else 1245 + ret = irdma_vchnl_req_aeq_vec_map(&rf->sc_dev, msix_vec->idx); 1261 1246 1262 - return 0; 1247 + return ret; 1263 1248 } 1264 1249 1265 1250 /** ··· 1271 1248 * @rf: RDMA PCI function 1272 1249 * @iwceq: pointer to the ceq resources to be created 1273 1250 * @ceq_id: the id number of the iwceq 1274 - * @vsi: SC vsi struct 1251 + * @vsi_idx: vsi idx 1275 1252 * 1276 1253 * Return 0, if the ceq and the resources associated with it 1277 1254 * are successfully created, otherwise return error 1278 1255 */ 1279 1256 static int irdma_create_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq, 1280 - u32 ceq_id, struct irdma_sc_vsi *vsi) 1257 + u32 ceq_id, u16 vsi_idx) 1281 1258 { 1282 1259 int status; 1283 1260 struct irdma_ceq_init_info info = {}; ··· 1301 1278 info.elem_cnt = ceq_size; 1302 1279 iwceq->sc_ceq.ceq_id = ceq_id; 1303 1280 info.dev = dev; 1304 - info.vsi = vsi; 1281 + info.vsi_idx = vsi_idx; 1305 1282 status = irdma_sc_ceq_init(&iwceq->sc_ceq, &info); 1306 1283 if (!status) { 1307 1284 if (dev->ceq_valid) ··· 1344 1321 } 1345 1322 1346 1323 iwceq = &rf->ceqlist[0]; 1347 - status = irdma_create_ceq(rf, iwceq, 0, &rf->default_vsi); 1324 + status = irdma_create_ceq(rf, iwceq, 0, rf->default_vsi.vsi_idx); 1348 1325 if (status) { 1349 1326 ibdev_dbg(&rf->iwdev->ibdev, "ERR: create ceq status = %d\n", 1350 1327 status); ··· 1379 1356 /** 1380 1357 * irdma_setup_ceqs - manage the device ceq's and their interrupt resources 1381 1358 * @rf: RDMA PCI function 1382 - * @vsi: VSI structure for this CEQ 1359 + * @vsi_idx: vsi_idx for this CEQ 1383 1360 * 1384 1361 * Allocate a list for all device completion event queues 1385 1362 * Create the ceq's and configure their msix interrupt vectors 1386 1363 * Return 0, if ceqs are successfully set up, otherwise return error 1387 1364 */ 1388 - static int irdma_setup_ceqs(struct irdma_pci_f *rf, struct irdma_sc_vsi *vsi) 1365 + static int irdma_setup_ceqs(struct irdma_pci_f *rf, u16 vsi_idx) 1389 1366 { 1390 1367 u32 i; 1391 1368 u32 ceq_id; ··· 1398 1375 i = (rf->msix_shared) ? 1 : 2; 1399 1376 for (ceq_id = 1; i < num_ceqs; i++, ceq_id++) { 1400 1377 iwceq = &rf->ceqlist[ceq_id]; 1401 - status = irdma_create_ceq(rf, iwceq, ceq_id, vsi); 1378 + status = irdma_create_ceq(rf, iwceq, ceq_id, vsi_idx); 1402 1379 if (status) { 1403 1380 ibdev_dbg(&rf->iwdev->ibdev, 1404 1381 "ERR: create ceq status = %d\n", status); ··· 1479 1456 aeq_size = multiplier * hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt + 1480 1457 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt; 1481 1458 aeq_size = min(aeq_size, dev->hw_attrs.max_hw_aeq_size); 1482 - 1459 + /* GEN_3 does not support virtual AEQ. Cap at max Kernel alloc size */ 1460 + if (rf->rdma_ver == IRDMA_GEN_3) 1461 + aeq_size = min(aeq_size, (u32)((PAGE_SIZE << MAX_PAGE_ORDER) / 1462 + sizeof(struct irdma_sc_aeqe))); 1483 1463 aeq->mem.size = ALIGN(sizeof(struct irdma_sc_aeqe) * aeq_size, 1484 1464 IRDMA_AEQ_ALIGNMENT); 1485 1465 aeq->mem.va = dma_alloc_coherent(dev->hw->device, aeq->mem.size, ··· 1490 1464 GFP_KERNEL | __GFP_NOWARN); 1491 1465 if (aeq->mem.va) 1492 1466 goto skip_virt_aeq; 1467 + else if (rf->rdma_ver == IRDMA_GEN_3) 1468 + return -ENOMEM; 1493 1469 1494 1470 /* physically mapped aeq failed. setup virtual aeq */ 1495 1471 status = irdma_create_virt_aeq(rf, aeq_size); ··· 1765 1737 irdma_del_local_mac_entry(iwdev->rf, 1766 1738 (u8)iwdev->mac_ip_table_idx); 1767 1739 fallthrough; 1768 - case AEQ_CREATED: 1769 - case PBLE_CHUNK_MEM: 1770 - case CEQS_CREATED: 1771 1740 case IEQ_CREATED: 1772 1741 if (!iwdev->roce_mode) 1773 1742 irdma_puda_dele_rsrc(&iwdev->vsi, IRDMA_PUDA_RSRC_TYPE_IEQ, ··· 1847 1822 enum init_completion_state state = rf->init_state; 1848 1823 1849 1824 rf->init_state = INVALID_STATE; 1850 - if (rf->rsrc_created) { 1851 - irdma_destroy_aeq(rf); 1852 - irdma_destroy_pble_prm(rf->pble_rsrc); 1853 - irdma_del_ceqs(rf); 1854 - rf->rsrc_created = false; 1855 - } 1825 + 1856 1826 switch (state) { 1827 + case AEQ_CREATED: 1828 + irdma_destroy_aeq(rf); 1829 + fallthrough; 1830 + case PBLE_CHUNK_MEM: 1831 + irdma_destroy_pble_prm(rf->pble_rsrc); 1832 + fallthrough; 1833 + case CEQS_CREATED: 1834 + irdma_del_ceqs(rf); 1835 + fallthrough; 1857 1836 case CEQ0_CREATED: 1858 1837 irdma_del_ceq_0(rf); 1859 1838 fallthrough; ··· 1936 1907 break; 1937 1908 iwdev->init_state = IEQ_CREATED; 1938 1909 } 1939 - if (!rf->rsrc_created) { 1940 - status = irdma_setup_ceqs(rf, &iwdev->vsi); 1941 - if (status) 1942 - break; 1943 - 1944 - iwdev->init_state = CEQS_CREATED; 1945 - 1946 - status = irdma_hmc_init_pble(&rf->sc_dev, 1947 - rf->pble_rsrc); 1948 - if (status) { 1949 - irdma_del_ceqs(rf); 1950 - break; 1951 - } 1952 - 1953 - iwdev->init_state = PBLE_CHUNK_MEM; 1954 - 1955 - status = irdma_setup_aeq(rf); 1956 - if (status) { 1957 - irdma_destroy_pble_prm(rf->pble_rsrc); 1958 - irdma_del_ceqs(rf); 1959 - break; 1960 - } 1961 - iwdev->init_state = AEQ_CREATED; 1962 - rf->rsrc_created = true; 1963 - } 1964 - 1965 1910 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1966 1911 irdma_alloc_set_mac(iwdev); 1967 1912 irdma_add_ip(iwdev); ··· 2017 2014 } 2018 2015 INIT_WORK(&rf->cqp_cmpl_work, cqp_compl_worker); 2019 2016 irdma_sc_ccq_arm(dev->ccq); 2017 + 2018 + status = irdma_setup_ceqs(rf, rf->iwdev ? rf->iwdev->vsi_num : 0); 2019 + if (status) 2020 + break; 2021 + 2022 + rf->init_state = CEQS_CREATED; 2023 + 2024 + status = irdma_hmc_init_pble(&rf->sc_dev, 2025 + rf->pble_rsrc); 2026 + if (status) 2027 + break; 2028 + 2029 + rf->init_state = PBLE_CHUNK_MEM; 2030 + 2031 + status = irdma_setup_aeq(rf); 2032 + if (status) 2033 + break; 2034 + rf->init_state = AEQ_CREATED; 2035 + 2020 2036 return 0; 2021 2037 } while (0); 2022 2038

+45

drivers/infiniband/hw/irdma/ig3rdma_hw.c

··· 5 5 #include "protos.h" 6 6 #include "ig3rdma_hw.h" 7 7 8 + /** 9 + * ig3rdma_ena_irq - Enable interrupt 10 + * @dev: pointer to the device structure 11 + * @idx: vector index 12 + */ 13 + static void ig3rdma_ena_irq(struct irdma_sc_dev *dev, u32 idx) 14 + { 15 + u32 val; 16 + u32 int_stride = 1; /* one u32 per register */ 17 + 18 + if (dev->is_pf) 19 + int_stride = 0x400; 20 + else 21 + idx--; /* VFs use DYN_CTL_N */ 22 + 23 + val = FIELD_PREP(IRDMA_GLINT_DYN_CTL_INTENA, 1) | 24 + FIELD_PREP(IRDMA_GLINT_DYN_CTL_CLEARPBA, 1); 25 + 26 + writel(val, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + (idx * int_stride)); 27 + } 28 + 29 + /** 30 + * ig3rdma_disable_irq - Disable interrupt 31 + * @dev: pointer to the device structure 32 + * @idx: vector index 33 + */ 34 + static void ig3rdma_disable_irq(struct irdma_sc_dev *dev, u32 idx) 35 + { 36 + u32 int_stride = 1; /* one u32 per register */ 37 + 38 + if (dev->is_pf) 39 + int_stride = 0x400; 40 + else 41 + idx--; /* VFs use DYN_CTL_N */ 42 + 43 + writel(0, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + (idx * int_stride)); 44 + } 45 + 46 + static const struct irdma_irq_ops ig3rdma_irq_ops = { 47 + .irdma_dis_irq = ig3rdma_disable_irq, 48 + .irdma_en_irq = ig3rdma_ena_irq, 49 + }; 50 + 8 51 void ig3rdma_init_hw(struct irdma_sc_dev *dev) 9 52 { 53 + dev->irq_ops = &ig3rdma_irq_ops; 54 + 10 55 dev->hw_attrs.uk_attrs.hw_rev = IRDMA_GEN_3; 11 56 dev->hw_attrs.uk_attrs.max_hw_wq_frags = IG3RDMA_MAX_WQ_FRAGMENT_COUNT; 12 57 dev->hw_attrs.uk_attrs.max_hw_read_sges = IG3RDMA_MAX_SGE_RD;

+10 -1

drivers/infiniband/hw/irdma/irdma.h

··· 32 32 #define IRDMA_PFHMC_SDDATALOW_PMSDDATALOW GENMASK(31, 12) 33 33 #define IRDMA_PFHMC_SDCMD_PMSDWR BIT(31) 34 34 35 - #define IRDMA_INVALID_CQ_IDX 0xffffffff 35 + #define IRDMA_INVALID_CQ_IDX 0xffffffff 36 + #define IRDMA_Q_INVALID_IDX 0xffff 37 + 38 + enum irdma_dyn_idx_t { 39 + IRDMA_IDX_ITR0 = 0, 40 + IRDMA_IDX_ITR1 = 1, 41 + IRDMA_IDX_ITR2 = 2, 42 + IRDMA_IDX_NOITR = 3, 43 + }; 44 + 36 45 enum irdma_registers { 37 46 IRDMA_CQPTAIL, 38 47 IRDMA_CQPDB,

+3 -3

drivers/infiniband/hw/irdma/main.h

··· 128 128 HMC_OBJS_CREATED, 129 129 HW_RSRC_INITIALIZED, 130 130 CCQ_CREATED, 131 - CEQ0_CREATED, /* Last state of probe */ 132 - ILQ_CREATED, 133 - IEQ_CREATED, 131 + CEQ0_CREATED, 134 132 CEQS_CREATED, 135 133 PBLE_CHUNK_MEM, 136 134 AEQ_CREATED, 135 + ILQ_CREATED, 136 + IEQ_CREATED, /* Last state of probe */ 137 137 IP_ADDR_REGISTERED, /* Last state of open */ 138 138 }; 139 139

+9 -2

drivers/infiniband/hw/irdma/type.h

··· 472 472 u32 msix_idx; 473 473 u8 polarity; 474 474 bool virtual_map:1; 475 + bool pasid_valid:1; 476 + u32 pasid; 475 477 }; 476 478 477 479 struct irdma_sc_ceq { ··· 489 487 u8 tph_val; 490 488 u32 first_pm_pbl_idx; 491 489 u8 polarity; 492 - struct irdma_sc_vsi *vsi; 490 + u16 vsi_idx; 493 491 struct irdma_sc_cq **reg_cq; 494 492 u32 reg_cq_size; 495 493 spinlock_t req_cq_lock; /* protect access to reg_cq array */ 496 494 bool virtual_map:1; 497 495 bool tph_en:1; 498 496 bool itr_no_expire:1; 497 + bool pasid_valid:1; 498 + u32 pasid; 499 499 }; 500 500 501 501 struct irdma_sc_cq { ··· 505 501 u64 cq_pa; 506 502 u64 shadow_area_pa; 507 503 struct irdma_sc_dev *dev; 504 + u16 vsi_idx; 508 505 struct irdma_sc_vsi *vsi; 509 506 void *pbl_list; 510 507 void *back_cq; ··· 839 834 bool itr_no_expire:1; 840 835 u8 pbl_chunk_size; 841 836 u8 tph_val; 837 + u16 vsi_idx; 842 838 u32 first_pm_pbl_idx; 843 - struct irdma_sc_vsi *vsi; 844 839 struct irdma_sc_cq **reg_cq; 845 840 u32 reg_cq_idx; 846 841 }; ··· 1047 1042 bool cq:1; 1048 1043 bool sq:1; 1049 1044 bool rq:1; 1045 + bool srq:1; 1050 1046 bool in_rdrsp_wr:1; 1051 1047 bool out_rdrsp:1; 1052 1048 bool aeqe_overflow:1; 1049 + bool err_rq_idx_valid:1; 1053 1050 u8 q2_data_written; 1054 1051 u8 ae_src; 1055 1052 };

+84

drivers/infiniband/hw/irdma/virtchnl.c

··· 110 110 return -EBADMSG; 111 111 break; 112 112 case IRDMA_VCHNL_OP_GET_REG_LAYOUT: 113 + case IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP: 114 + case IRDMA_VCHNL_OP_QUEUE_VECTOR_UNMAP: 113 115 break; 114 116 default: 115 117 return -EOPNOTSUPP; ··· 315 313 } 316 314 317 315 return 0; 316 + } 317 + 318 + /** 319 + * irdma_vchnl_req_aeq_vec_map - Map AEQ to vector on this function 320 + * @dev: RDMA device pointer 321 + * @v_idx: vector index 322 + */ 323 + int irdma_vchnl_req_aeq_vec_map(struct irdma_sc_dev *dev, u32 v_idx) 324 + { 325 + struct irdma_vchnl_req_init_info info = {}; 326 + struct irdma_vchnl_qvlist_info *qvl; 327 + struct irdma_vchnl_qv_info *qv; 328 + u16 qvl_size, num_vectors = 1; 329 + int ret; 330 + 331 + if (!dev->vchnl_up) 332 + return -EBUSY; 333 + 334 + qvl_size = struct_size(qvl, qv_info, num_vectors); 335 + 336 + qvl = kzalloc(qvl_size, GFP_KERNEL); 337 + if (!qvl) 338 + return -ENOMEM; 339 + 340 + qvl->num_vectors = 1; 341 + qv = qvl->qv_info; 342 + 343 + qv->ceq_idx = IRDMA_Q_INVALID_IDX; 344 + qv->v_idx = v_idx; 345 + qv->itr_idx = IRDMA_IDX_ITR0; 346 + 347 + info.op_code = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP; 348 + info.op_ver = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP_V0; 349 + info.req_parm = qvl; 350 + info.req_parm_len = qvl_size; 351 + 352 + ret = irdma_vchnl_req_send_sync(dev, &info); 353 + kfree(qvl); 354 + 355 + return ret; 356 + } 357 + 358 + /** 359 + * irdma_vchnl_req_ceq_vec_map - Map CEQ to vector on this function 360 + * @dev: RDMA device pointer 361 + * @ceq_id: CEQ index 362 + * @v_idx: vector index 363 + */ 364 + int irdma_vchnl_req_ceq_vec_map(struct irdma_sc_dev *dev, u16 ceq_id, u32 v_idx) 365 + { 366 + struct irdma_vchnl_req_init_info info = {}; 367 + struct irdma_vchnl_qvlist_info *qvl; 368 + struct irdma_vchnl_qv_info *qv; 369 + u16 qvl_size, num_vectors = 1; 370 + int ret; 371 + 372 + if (!dev->vchnl_up) 373 + return -EBUSY; 374 + 375 + qvl_size = struct_size(qvl, qv_info, num_vectors); 376 + 377 + qvl = kzalloc(qvl_size, GFP_KERNEL); 378 + if (!qvl) 379 + return -ENOMEM; 380 + 381 + qvl->num_vectors = num_vectors; 382 + qv = qvl->qv_info; 383 + 384 + qv->aeq_idx = IRDMA_Q_INVALID_IDX; 385 + qv->ceq_idx = ceq_id; 386 + qv->v_idx = v_idx; 387 + qv->itr_idx = IRDMA_IDX_ITR0; 388 + 389 + info.op_code = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP; 390 + info.op_ver = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP_V0; 391 + info.req_parm = qvl; 392 + info.req_parm_len = qvl_size; 393 + 394 + ret = irdma_vchnl_req_send_sync(dev, &info); 395 + kfree(qvl); 396 + 397 + return ret; 318 398 } 319 399 320 400 /**

+19

drivers/infiniband/hw/irdma/virtchnl.h

··· 15 15 #define IRDMA_VCHNL_OP_GET_HMC_FCN_V2 2 16 16 #define IRDMA_VCHNL_OP_PUT_HMC_FCN_V0 0 17 17 #define IRDMA_VCHNL_OP_GET_REG_LAYOUT_V0 0 18 + #define IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP_V0 0 19 + #define IRDMA_VCHNL_OP_QUEUE_VECTOR_UNMAP_V0 0 18 20 #define IRDMA_VCHNL_OP_GET_RDMA_CAPS_V0 0 19 21 #define IRDMA_VCHNL_OP_GET_RDMA_CAPS_MIN_SIZE 1 20 22 ··· 55 53 IRDMA_VCHNL_OP_PUT_HMC_FCN = 2, 56 54 IRDMA_VCHNL_OP_GET_REG_LAYOUT = 11, 57 55 IRDMA_VCHNL_OP_GET_RDMA_CAPS = 13, 56 + IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP = 14, 57 + IRDMA_VCHNL_OP_QUEUE_VECTOR_UNMAP = 15, 58 58 }; 59 59 60 60 struct irdma_vchnl_req_hmc_info { ··· 68 64 u16 hmc_func; 69 65 u16 qs_handle[IRDMA_MAX_USER_PRIORITY]; 70 66 } __packed; 67 + 68 + struct irdma_vchnl_qv_info { 69 + u32 v_idx; 70 + u16 ceq_idx; 71 + u16 aeq_idx; 72 + u8 itr_idx; 73 + }; 74 + 75 + struct irdma_vchnl_qvlist_info { 76 + u32 num_vectors; 77 + struct irdma_vchnl_qv_info qv_info[]; 78 + }; 71 79 72 80 struct irdma_vchnl_op_buf { 73 81 u16 op_code; ··· 151 135 int irdma_vchnl_req_get_resp(struct irdma_sc_dev *dev, 152 136 struct irdma_vchnl_req *vc_req); 153 137 int irdma_vchnl_req_get_reg_layout(struct irdma_sc_dev *dev); 138 + int irdma_vchnl_req_aeq_vec_map(struct irdma_sc_dev *dev, u32 v_idx); 139 + int irdma_vchnl_req_ceq_vec_map(struct irdma_sc_dev *dev, u16 ceq_id, 140 + u32 v_idx); 154 141 #endif /* IRDMA_VIRTCHNL_H */

Configure Feed

Configure Feed