RDMA/bnxt_re: introduce a function to allocate swq

+177 -171

drivers/infiniband/hw/bnxt_re/qplib_fp.c

··· 178 178 179 179 if (qp->rq_hdr_buf) 180 180 dma_free_coherent(&res->pdev->dev, 181 - rq->hwq.max_elements * qp->rq_hdr_buf_size, 181 + rq->max_wqe * qp->rq_hdr_buf_size, 182 182 qp->rq_hdr_buf, qp->rq_hdr_buf_map); 183 183 if (qp->sq_hdr_buf) 184 184 dma_free_coherent(&res->pdev->dev, 185 - sq->hwq.max_elements * qp->sq_hdr_buf_size, 185 + sq->max_wqe * qp->sq_hdr_buf_size, 186 186 qp->sq_hdr_buf, qp->sq_hdr_buf_map); 187 187 qp->rq_hdr_buf = NULL; 188 188 qp->sq_hdr_buf = NULL; ··· 199 199 struct bnxt_qplib_q *sq = &qp->sq; 200 200 int rc = 0; 201 201 202 - if (qp->sq_hdr_buf_size && sq->hwq.max_elements) { 202 + if (qp->sq_hdr_buf_size && sq->max_wqe) { 203 203 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, 204 - sq->hwq.max_elements * 205 - qp->sq_hdr_buf_size, 204 + sq->max_wqe * qp->sq_hdr_buf_size, 206 205 &qp->sq_hdr_buf_map, GFP_KERNEL); 207 206 if (!qp->sq_hdr_buf) { 208 207 rc = -ENOMEM; ··· 211 212 } 212 213 } 213 214 214 - if (qp->rq_hdr_buf_size && rq->hwq.max_elements) { 215 + if (qp->rq_hdr_buf_size && rq->max_wqe) { 215 216 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, 216 - rq->hwq.max_elements * 217 + rq->max_wqe * 217 218 qp->rq_hdr_buf_size, 218 219 &qp->rq_hdr_buf_map, 219 220 GFP_KERNEL); ··· 783 784 } 784 785 785 786 /* QP */ 787 + 788 + static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que) 789 + { 790 + int rc = 0; 791 + int indx; 792 + 793 + que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL); 794 + if (!que->swq) { 795 + rc = -ENOMEM; 796 + goto out; 797 + } 798 + 799 + que->swq_start = 0; 800 + que->swq_last = que->max_wqe - 1; 801 + for (indx = 0; indx < que->max_wqe; indx++) { 802 + que->swq[indx].slots = 1; 803 + que->swq[indx].next_idx = indx + 1; 804 + } 805 + que->swq[que->swq_last].next_idx = 0; /* Make it circular */ 806 + que->swq_last = 0; 807 + out: 808 + return rc; 809 + } 810 + 786 811 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 787 812 { 788 813 struct bnxt_qplib_hwq_attr hwq_attr = {}; ··· 838 815 if (rc) 839 816 goto exit; 840 817 841 - sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); 842 - if (!sq->swq) { 843 - rc = -ENOMEM; 818 + rc = bnxt_qplib_alloc_init_swq(sq); 819 + if (rc) 844 820 goto fail_sq; 845 - } 821 + 822 + req.sq_size = cpu_to_le32(sq->max_wqe); 846 823 pbl = &sq->hwq.pbl[PBL_LVL_0]; 847 824 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 848 825 pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << 849 826 CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT); 850 827 pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK); 851 828 req.sq_pg_size_sq_lvl = pg_sz_lvl; 829 + req.sq_fwo_sq_sge = 830 + cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << 831 + CMDQ_CREATE_QP1_SQ_SGE_SFT); 832 + req.scq_cid = cpu_to_le32(qp->scq->id); 852 833 853 - if (qp->scq) 854 - req.scq_cid = cpu_to_le32(qp->scq->id); 855 834 /* RQ */ 856 835 if (rq->max_wqe) { 857 836 hwq_attr.res = res; ··· 863 838 hwq_attr.type = HWQ_TYPE_QUEUE; 864 839 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); 865 840 if (rc) 866 - goto fail_sq; 867 - 868 - rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), 869 - GFP_KERNEL); 870 - if (!rq->swq) { 871 - rc = -ENOMEM; 841 + goto sq_swq; 842 + rc = bnxt_qplib_alloc_init_swq(rq); 843 + if (rc) 872 844 goto fail_rq; 873 - } 845 + req.rq_size = cpu_to_le32(rq->max_wqe); 874 846 pbl = &rq->hwq.pbl[PBL_LVL_0]; 875 847 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 876 848 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << 877 849 CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT); 878 850 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK); 879 851 req.rq_pg_size_rq_lvl = pg_sz_lvl; 880 - if (qp->rcq) 881 - req.rcq_cid = cpu_to_le32(qp->rcq->id); 852 + req.rq_fwo_rq_sge = 853 + cpu_to_le16((rq->max_sge & 854 + CMDQ_CREATE_QP1_RQ_SGE_MASK) << 855 + CMDQ_CREATE_QP1_RQ_SGE_SFT); 882 856 } 857 + req.rcq_cid = cpu_to_le32(qp->rcq->id); 883 858 /* Header buffer - allow hdr_buf pass in */ 884 859 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); 885 860 if (rc) { 886 861 rc = -ENOMEM; 887 - goto fail; 862 + goto rq_rwq; 888 863 } 889 864 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; 890 865 req.qp_flags = cpu_to_le32(qp_flags); 891 - req.sq_size = cpu_to_le32(sq->hwq.max_elements); 892 - req.rq_size = cpu_to_le32(rq->hwq.max_elements); 893 - 894 - req.sq_fwo_sq_sge = 895 - cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << 896 - CMDQ_CREATE_QP1_SQ_SGE_SFT); 897 - req.rq_fwo_rq_sge = 898 - cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) << 899 - CMDQ_CREATE_QP1_RQ_SGE_SFT); 900 - 901 866 req.pd_id = cpu_to_le32(qp->pd->id); 902 867 903 868 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, ··· 913 898 914 899 fail: 915 900 bnxt_qplib_free_qp_hdr_buf(res, qp); 901 + rq_rwq: 902 + kfree(rq->swq); 916 903 fail_rq: 917 904 bnxt_qplib_free_hwq(res, &rq->hwq); 918 - kfree(rq->swq); 905 + sq_swq: 906 + kfree(sq->swq); 919 907 fail_sq: 920 908 bnxt_qplib_free_hwq(res, &sq->hwq); 921 - kfree(sq->swq); 922 909 exit: 923 910 return rc; 924 911 } ··· 961 944 struct creq_create_qp_resp resp; 962 945 int rc, req_size, psn_sz = 0; 963 946 struct bnxt_qplib_hwq *xrrq; 964 - u16 cmd_flags = 0, max_ssge; 965 947 struct bnxt_qplib_pbl *pbl; 966 948 struct cmdq_create_qp req; 949 + u16 cmd_flags = 0; 967 950 u32 qp_flags = 0; 968 951 u8 pg_sz_lvl; 969 - u16 max_rsge; 952 + u16 nsge; 970 953 971 954 RCFW_CMD_PREP(req, CREATE_QP, cmd_flags); 972 955 ··· 993 976 if (rc) 994 977 goto exit; 995 978 996 - sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); 997 - if (!sq->swq) { 998 - rc = -ENOMEM; 979 + rc = bnxt_qplib_alloc_init_swq(sq); 980 + if (rc) 999 981 goto fail_sq; 1000 - } 1001 982 1002 983 if (psn_sz) 1003 984 bnxt_qplib_init_psn_ptr(qp, psn_sz); 1004 985 986 + req.sq_size = cpu_to_le32(sq->max_wqe); 1005 987 pbl = &sq->hwq.pbl[PBL_LVL_0]; 1006 988 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 1007 989 pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << 1008 990 CMDQ_CREATE_QP_SQ_PG_SIZE_SFT); 1009 991 pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK); 1010 992 req.sq_pg_size_sq_lvl = pg_sz_lvl; 1011 - 1012 - if (qp->scq) 1013 - req.scq_cid = cpu_to_le32(qp->scq->id); 993 + req.sq_fwo_sq_sge = 994 + cpu_to_le16(((sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK) << 995 + CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); 996 + req.scq_cid = cpu_to_le32(qp->scq->id); 1014 997 1015 998 /* RQ */ 1016 - if (rq->max_wqe) { 999 + if (!qp->srq) { 1017 1000 hwq_attr.res = res; 1018 1001 hwq_attr.sginfo = &rq->sg_info; 1019 1002 hwq_attr.stride = rq->wqe_size; ··· 1023 1006 hwq_attr.type = HWQ_TYPE_QUEUE; 1024 1007 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); 1025 1008 if (rc) 1026 - goto fail_sq; 1027 - 1028 - rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), 1029 - GFP_KERNEL); 1030 - if (!rq->swq) { 1031 - rc = -ENOMEM; 1009 + goto sq_swq; 1010 + rc = bnxt_qplib_alloc_init_swq(rq); 1011 + if (rc) 1032 1012 goto fail_rq; 1033 - } 1013 + 1014 + req.rq_size = cpu_to_le32(rq->max_wqe); 1034 1015 pbl = &rq->hwq.pbl[PBL_LVL_0]; 1035 1016 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 1036 1017 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << 1037 1018 CMDQ_CREATE_QP_RQ_PG_SIZE_SFT); 1038 1019 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK); 1039 1020 req.rq_pg_size_rq_lvl = pg_sz_lvl; 1021 + nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? 1022 + 6 : rq->max_sge; 1023 + req.rq_fwo_rq_sge = 1024 + cpu_to_le16(((nsge & 1025 + CMDQ_CREATE_QP_RQ_SGE_MASK) << 1026 + CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); 1040 1027 } else { 1041 1028 /* SRQ */ 1042 - if (qp->srq) { 1043 - qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED; 1044 - req.srq_cid = cpu_to_le32(qp->srq->id); 1045 - } 1029 + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED; 1030 + req.srq_cid = cpu_to_le32(qp->srq->id); 1046 1031 } 1047 - 1048 - if (qp->rcq) 1049 - req.rcq_cid = cpu_to_le32(qp->rcq->id); 1032 + req.rcq_cid = cpu_to_le32(qp->rcq->id); 1050 1033 1051 1034 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; 1052 1035 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; ··· 1054 1037 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; 1055 1038 req.qp_flags = cpu_to_le32(qp_flags); 1056 1039 1057 - req.sq_size = cpu_to_le32(sq->hwq.max_elements); 1058 - req.rq_size = cpu_to_le32(rq->hwq.max_elements); 1059 - qp->sq_hdr_buf = NULL; 1060 - qp->rq_hdr_buf = NULL; 1061 - 1062 - rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); 1063 - if (rc) 1064 - goto fail_rq; 1065 - 1066 - /* CTRL-22434: Irrespective of the requested SGE count on the SQ 1067 - * always create the QP with max send sges possible if the requested 1068 - * inline size is greater than 0. 1069 - */ 1070 - max_ssge = qp->max_inline_data ? 6 : sq->max_sge; 1071 - req.sq_fwo_sq_sge = cpu_to_le16( 1072 - ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK) 1073 - << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); 1074 - max_rsge = bnxt_qplib_is_chip_gen_p5(res->cctx) ? 6 : rq->max_sge; 1075 - req.rq_fwo_rq_sge = cpu_to_le16( 1076 - ((max_rsge & CMDQ_CREATE_QP_RQ_SGE_MASK) 1077 - << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); 1078 1040 /* ORRQ and IRRQ */ 1079 1041 if (psn_sz) { 1080 1042 xrrq = &qp->orrq; ··· 1074 1078 hwq_attr.type = HWQ_TYPE_CTX; 1075 1079 rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); 1076 1080 if (rc) 1077 - goto fail_buf_free; 1081 + goto rq_swq; 1078 1082 pbl = &xrrq->pbl[PBL_LVL_0]; 1079 1083 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); 1080 1084 ··· 1118 1122 rcfw->qp_tbl[qp->id].qp_handle = (void *)qp; 1119 1123 1120 1124 return 0; 1121 - 1122 1125 fail: 1123 - if (qp->irrq.max_elements) 1124 - bnxt_qplib_free_hwq(res, &qp->irrq); 1126 + bnxt_qplib_free_hwq(res, &qp->irrq); 1125 1127 fail_orrq: 1126 - if (qp->orrq.max_elements) 1127 - bnxt_qplib_free_hwq(res, &qp->orrq); 1128 - fail_buf_free: 1129 - bnxt_qplib_free_qp_hdr_buf(res, qp); 1128 + bnxt_qplib_free_hwq(res, &qp->orrq); 1129 + rq_swq: 1130 + kfree(rq->swq); 1130 1131 fail_rq: 1131 1132 bnxt_qplib_free_hwq(res, &rq->hwq); 1132 - kfree(rq->swq); 1133 + sq_swq: 1134 + kfree(sq->swq); 1133 1135 fail_sq: 1134 1136 bnxt_qplib_free_hwq(res, &sq->hwq); 1135 - kfree(sq->swq); 1136 1137 exit: 1137 1138 return rc; 1138 1139 } ··· 1505 1512 memset(sge, 0, sizeof(*sge)); 1506 1513 1507 1514 if (qp->sq_hdr_buf) { 1508 - sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1515 + sw_prod = sq->swq_start; 1509 1516 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map + 1510 1517 sw_prod * qp->sq_hdr_buf_size); 1511 1518 sge->lkey = 0xFFFFFFFF; ··· 1519 1526 { 1520 1527 struct bnxt_qplib_q *rq = &qp->rq; 1521 1528 1522 - return HWQ_CMP(rq->hwq.prod, &rq->hwq); 1529 + return rq->swq_start; 1523 1530 } 1524 1531 1525 1532 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) ··· 1536 1543 memset(sge, 0, sizeof(*sge)); 1537 1544 1538 1545 if (qp->rq_hdr_buf) { 1539 - sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1546 + sw_prod = rq->swq_start; 1540 1547 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map + 1541 1548 sw_prod * qp->rq_hdr_buf_size); 1542 1549 sge->lkey = 0xFFFFFFFF; ··· 1613 1620 rc = -ENOMEM; 1614 1621 goto done; 1615 1622 } 1616 - sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 1617 - swq = &sq->swq[sw_prod]; 1623 + sw_prod = sq->hwq.prod; 1624 + swq = bnxt_qplib_get_swqe(sq, NULL); 1618 1625 swq->wr_id = wqe->wr_id; 1619 1626 swq->type = wqe->type; 1620 1627 swq->flags = wqe->flags; ··· 1824 1831 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; 1825 1832 swq->start_psn = sq->psn & BTH_PSN_MASK; 1826 1833 } 1827 - sq->hwq.prod++; 1834 + bnxt_qplib_swq_mod_start(sq, sw_prod); 1835 + bnxt_qplib_hwq_incr_prod(&sq->hwq, 1); 1828 1836 qp->wqe_cnt++; 1829 1837 1830 1838 done: ··· 1857 1863 { 1858 1864 struct bnxt_qplib_nq_work *nq_work = NULL; 1859 1865 struct bnxt_qplib_q *rq = &qp->rq; 1866 + struct bnxt_qplib_swq *swq; 1860 1867 bool sch_handler = false; 1861 1868 struct sq_sge *hw_sge; 1862 1869 struct rq_wqe *rqe; ··· 1876 1881 rc = -EINVAL; 1877 1882 goto done; 1878 1883 } 1879 - sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1880 - rq->swq[sw_prod].wr_id = wqe->wr_id; 1884 + sw_prod = rq->hwq.prod; 1885 + swq = bnxt_qplib_get_swqe(rq, NULL); 1886 + swq->wr_id = wqe->wr_id; 1881 1887 1882 1888 rqe = bnxt_qplib_get_qe(&rq->hwq, sw_prod, NULL); 1883 1889 memset(rqe, 0, rq->wqe_size); ··· 1907 1911 if (sch_handler) { 1908 1912 /* Store the ULP info in the software structures */ 1909 1913 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 1910 - rq->swq[sw_prod].wr_id = wqe->wr_id; 1914 + swq = bnxt_qplib_get_swqe(rq, NULL); 1915 + swq->wr_id = wqe->wr_id; 1911 1916 } 1912 1917 1913 - rq->hwq.prod++; 1918 + bnxt_qplib_swq_mod_start(rq, sw_prod); 1919 + bnxt_qplib_hwq_incr_prod(&rq->hwq, 1); 1914 1920 if (sch_handler) { 1915 1921 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); 1916 1922 if (nq_work) { ··· 2024 2026 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, 2025 2027 struct bnxt_qplib_cqe **pcqe, int *budget) 2026 2028 { 2027 - u32 sw_prod, sw_cons; 2028 2029 struct bnxt_qplib_cqe *cqe; 2030 + u32 start, last; 2029 2031 int rc = 0; 2030 2032 2031 2033 /* Now complete all outstanding SQEs with FLUSHED_ERR */ 2032 - sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); 2034 + start = sq->swq_start; 2033 2035 cqe = *pcqe; 2034 2036 while (*budget) { 2035 - sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 2036 - if (sw_cons == sw_prod) { 2037 + last = sq->swq_last; 2038 + if (start == last) 2037 2039 break; 2038 - } 2039 2040 /* Skip the FENCE WQE completions */ 2040 - if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) { 2041 + if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) { 2041 2042 bnxt_qplib_cancel_phantom_processing(qp); 2042 2043 goto skip_compl; 2043 2044 } ··· 2044 2047 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; 2045 2048 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 2046 2049 cqe->qp_handle = (u64)(unsigned long)qp; 2047 - cqe->wr_id = sq->swq[sw_cons].wr_id; 2050 + cqe->wr_id = sq->swq[last].wr_id; 2048 2051 cqe->src_qp = qp->id; 2049 - cqe->type = sq->swq[sw_cons].type; 2052 + cqe->type = sq->swq[last].type; 2050 2053 cqe++; 2051 2054 (*budget)--; 2052 2055 skip_compl: 2053 - sq->hwq.cons++; 2056 + bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[last].slots); 2057 + sq->swq_last = sq->swq[last].next_idx; 2054 2058 } 2055 2059 *pcqe = cqe; 2056 - if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod) 2060 + if (!(*budget) && sq->swq_last != start) 2057 2061 /* Out of budget */ 2058 2062 rc = -EAGAIN; 2059 2063 ··· 2065 2067 struct bnxt_qplib_cqe **pcqe, int *budget) 2066 2068 { 2067 2069 struct bnxt_qplib_cqe *cqe; 2068 - u32 sw_prod, sw_cons; 2069 - int rc = 0; 2070 + u32 start, last; 2070 2071 int opcode = 0; 2072 + int rc = 0; 2071 2073 2072 2074 switch (qp->type) { 2073 2075 case CMDQ_CREATE_QP1_TYPE_GSI: ··· 2083 2085 } 2084 2086 2085 2087 /* Flush the rest of the RQ */ 2086 - sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); 2088 + start = rq->swq_start; 2087 2089 cqe = *pcqe; 2088 2090 while (*budget) { 2089 - sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq); 2090 - if (sw_cons == sw_prod) 2091 + last = rq->swq_last; 2092 + if (last == start) 2091 2093 break; 2092 2094 memset(cqe, 0, sizeof(*cqe)); 2093 2095 cqe->status = 2094 2096 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; 2095 2097 cqe->opcode = opcode; 2096 2098 cqe->qp_handle = (unsigned long)qp; 2097 - cqe->wr_id = rq->swq[sw_cons].wr_id; 2099 + cqe->wr_id = rq->swq[last].wr_id; 2098 2100 cqe++; 2099 2101 (*budget)--; 2100 - rq->hwq.cons++; 2102 + bnxt_qplib_hwq_incr_cons(&rq->hwq, rq->swq[last].slots); 2103 + rq->swq_last = rq->swq[last].next_idx; 2101 2104 } 2102 2105 *pcqe = cqe; 2103 - if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod) 2106 + if (!*budget && rq->swq_last != start) 2104 2107 /* Out of budget */ 2105 2108 rc = -EAGAIN; 2106 2109 ··· 2124 2125 * CQE is track from sw_cq_cons to max_element but valid only if VALID=1 2125 2126 */ 2126 2127 static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, 2127 - u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons) 2128 + u32 cq_cons, u32 swq_last, u32 cqe_sq_cons) 2128 2129 { 2129 2130 u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx; 2130 2131 struct bnxt_qplib_q *sq = &qp->sq; ··· 2137 2138 2138 2139 /* Normal mode */ 2139 2140 /* Check for the psn_search marking before completing */ 2140 - swq = &sq->swq[sw_sq_cons]; 2141 + swq = &sq->swq[swq_last]; 2141 2142 if (swq->psn_search && 2142 2143 le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { 2143 2144 /* Unmark */ ··· 2146 2147 & ~0x80000000); 2147 2148 dev_dbg(&cq->hwq.pdev->dev, 2148 2149 "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n", 2149 - cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); 2150 + cq_cons, qp->id, swq_last, cqe_sq_cons); 2150 2151 sq->condition = true; 2151 2152 sq->send_phantom = true; 2152 2153 ··· 2183 2184 le64_to_cpu 2184 2185 (peek_req_hwcqe->qp_handle)); 2185 2186 peek_sq = &peek_qp->sq; 2186 - peek_sq_cons_idx = HWQ_CMP(le16_to_cpu( 2187 - peek_req_hwcqe->sq_cons_idx) - 1 2188 - , &sq->hwq); 2187 + peek_sq_cons_idx = 2188 + ((le16_to_cpu( 2189 + peek_req_hwcqe->sq_cons_idx) 2190 + - 1) % sq->max_wqe); 2189 2191 /* If the hwcqe's sq's wr_id matches */ 2190 2192 if (peek_sq == sq && 2191 2193 sq->swq[peek_sq_cons_idx].wr_id == ··· 2214 2214 } 2215 2215 dev_err(&cq->hwq.pdev->dev, 2216 2216 "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n", 2217 - cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); 2217 + cq_cons, qp->id, swq_last, cqe_sq_cons); 2218 2218 rc = -EINVAL; 2219 2219 } 2220 2220 out: ··· 2226 2226 struct bnxt_qplib_cqe **pcqe, int *budget, 2227 2227 u32 cq_cons, struct bnxt_qplib_qp **lib_qp) 2228 2228 { 2229 - u32 sw_sq_cons, cqe_sq_cons; 2230 2229 struct bnxt_qplib_swq *swq; 2231 2230 struct bnxt_qplib_cqe *cqe; 2232 2231 struct bnxt_qplib_qp *qp; 2233 2232 struct bnxt_qplib_q *sq; 2233 + u32 cqe_sq_cons; 2234 2234 int rc = 0; 2235 2235 2236 2236 qp = (struct bnxt_qplib_qp *)((unsigned long) ··· 2242 2242 } 2243 2243 sq = &qp->sq; 2244 2244 2245 - cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq); 2246 - if (cqe_sq_cons > sq->hwq.max_elements) { 2247 - dev_err(&cq->hwq.pdev->dev, 2248 - "FP: CQ Process req reported sq_cons_idx 0x%x which exceeded max 0x%x\n", 2249 - cqe_sq_cons, sq->hwq.max_elements); 2250 - return -EINVAL; 2251 - } 2252 - 2245 + cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe; 2253 2246 if (qp->sq.flushed) { 2254 2247 dev_dbg(&cq->hwq.pdev->dev, 2255 2248 "%s: QP in Flush QP = %p\n", __func__, qp); ··· 2254 2261 */ 2255 2262 cqe = *pcqe; 2256 2263 while (*budget) { 2257 - sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 2258 - if (sw_sq_cons == cqe_sq_cons) 2264 + if (sq->swq_last == cqe_sq_cons) 2259 2265 /* Done */ 2260 2266 break; 2261 2267 2262 - swq = &sq->swq[sw_sq_cons]; 2268 + swq = &sq->swq[sq->swq_last]; 2263 2269 memset(cqe, 0, sizeof(*cqe)); 2264 2270 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 2265 2271 cqe->qp_handle = (u64)(unsigned long)qp; ··· 2272 2280 * of the request being signaled or not, it must complete with 2273 2281 * the hwcqe error status 2274 2282 */ 2275 - if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons && 2283 + if (swq->next_idx == cqe_sq_cons && 2276 2284 hwcqe->status != CQ_REQ_STATUS_OK) { 2277 2285 cqe->status = hwcqe->status; 2278 2286 dev_err(&cq->hwq.pdev->dev, 2279 2287 "FP: CQ Processed Req wr_id[%d] = 0x%llx with status 0x%x\n", 2280 - sw_sq_cons, cqe->wr_id, cqe->status); 2288 + sq->swq_last, cqe->wr_id, cqe->status); 2281 2289 cqe++; 2282 2290 (*budget)--; 2283 2291 bnxt_qplib_mark_qp_error(qp); ··· 2285 2293 bnxt_qplib_add_flush_qp(qp); 2286 2294 } else { 2287 2295 /* Before we complete, do WA 9060 */ 2288 - if (do_wa9060(qp, cq, cq_cons, sw_sq_cons, 2296 + if (do_wa9060(qp, cq, cq_cons, sq->swq_last, 2289 2297 cqe_sq_cons)) { 2290 2298 *lib_qp = qp; 2291 2299 goto out; ··· 2297 2305 } 2298 2306 } 2299 2307 skip: 2300 - sq->hwq.cons++; 2308 + bnxt_qplib_hwq_incr_cons(&sq->hwq, swq->slots); 2309 + sq->swq_last = swq->next_idx; 2301 2310 if (sq->single) 2302 2311 break; 2303 2312 } 2304 2313 out: 2305 2314 *pcqe = cqe; 2306 - if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) { 2315 + if (sq->swq_last != cqe_sq_cons) { 2307 2316 /* Out of budget */ 2308 2317 rc = -EAGAIN; 2309 2318 goto done; ··· 2379 2386 (*budget)--; 2380 2387 *pcqe = cqe; 2381 2388 } else { 2389 + struct bnxt_qplib_swq *swq; 2390 + 2382 2391 rq = &qp->rq; 2383 - if (wr_id_idx >= rq->hwq.max_elements) { 2392 + if (wr_id_idx > (rq->max_wqe - 1)) { 2384 2393 dev_err(&cq->hwq.pdev->dev, 2385 2394 "FP: CQ Process RC wr_id idx 0x%x exceeded RQ max 0x%x\n", 2386 - wr_id_idx, rq->hwq.max_elements); 2395 + wr_id_idx, rq->max_wqe); 2387 2396 return -EINVAL; 2388 2397 } 2389 - cqe->wr_id = rq->swq[wr_id_idx].wr_id; 2398 + if (wr_id_idx != rq->swq_last) 2399 + return -EINVAL; 2400 + swq = &rq->swq[rq->swq_last]; 2401 + cqe->wr_id = swq->wr_id; 2390 2402 cqe++; 2391 2403 (*budget)--; 2392 - rq->hwq.cons++; 2404 + bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); 2405 + rq->swq_last = swq->next_idx; 2393 2406 *pcqe = cqe; 2394 2407 2395 2408 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { ··· 2466 2467 (*budget)--; 2467 2468 *pcqe = cqe; 2468 2469 } else { 2470 + struct bnxt_qplib_swq *swq; 2471 + 2469 2472 rq = &qp->rq; 2470 - if (wr_id_idx >= rq->hwq.max_elements) { 2473 + if (wr_id_idx > (rq->max_wqe - 1)) { 2471 2474 dev_err(&cq->hwq.pdev->dev, 2472 2475 "FP: CQ Process UD wr_id idx 0x%x exceeded RQ max 0x%x\n", 2473 - wr_id_idx, rq->hwq.max_elements); 2476 + wr_id_idx, rq->max_wqe); 2474 2477 return -EINVAL; 2475 2478 } 2476 2479 2477 - cqe->wr_id = rq->swq[wr_id_idx].wr_id; 2480 + if (rq->swq_last != wr_id_idx) 2481 + return -EINVAL; 2482 + swq = &rq->swq[rq->swq_last]; 2483 + cqe->wr_id = swq->wr_id; 2478 2484 cqe++; 2479 2485 (*budget)--; 2480 - rq->hwq.cons++; 2486 + bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); 2487 + rq->swq_last = swq->next_idx; 2481 2488 *pcqe = cqe; 2482 2489 2483 2490 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { ··· 2574 2569 (*budget)--; 2575 2570 *pcqe = cqe; 2576 2571 } else { 2572 + struct bnxt_qplib_swq *swq; 2573 + 2577 2574 rq = &qp->rq; 2578 - if (wr_id_idx >= rq->hwq.max_elements) { 2575 + if (wr_id_idx > (rq->max_wqe - 1)) { 2579 2576 dev_err(&cq->hwq.pdev->dev, 2580 2577 "FP: CQ Process Raw/QP1 RQ wr_id idx 0x%x exceeded RQ max 0x%x\n", 2581 - wr_id_idx, rq->hwq.max_elements); 2578 + wr_id_idx, rq->max_wqe); 2582 2579 return -EINVAL; 2583 2580 } 2584 - cqe->wr_id = rq->swq[wr_id_idx].wr_id; 2581 + if (rq->swq_last != wr_id_idx) 2582 + return -EINVAL; 2583 + swq = &rq->swq[rq->swq_last]; 2584 + cqe->wr_id = swq->wr_id; 2585 2585 cqe++; 2586 2586 (*budget)--; 2587 - rq->hwq.cons++; 2587 + bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); 2588 + rq->swq_last = swq->next_idx; 2588 2589 *pcqe = cqe; 2589 2590 2590 2591 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { ··· 2612 2601 struct bnxt_qplib_qp *qp; 2613 2602 struct bnxt_qplib_q *sq, *rq; 2614 2603 struct bnxt_qplib_cqe *cqe; 2615 - u32 sw_cons = 0, cqe_cons; 2604 + u32 swq_last = 0, cqe_cons; 2616 2605 int rc = 0; 2617 2606 2618 2607 /* Check the Status */ ··· 2638 2627 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); 2639 2628 if (cqe_cons == 0xFFFF) 2640 2629 goto do_rq; 2641 - 2642 - if (cqe_cons > sq->hwq.max_elements) { 2643 - dev_err(&cq->hwq.pdev->dev, 2644 - "FP: CQ Process terminal reported sq_cons_idx 0x%x which exceeded max 0x%x\n", 2645 - cqe_cons, sq->hwq.max_elements); 2646 - goto do_rq; 2647 - } 2630 + cqe_cons %= sq->max_wqe; 2648 2631 2649 2632 if (qp->sq.flushed) { 2650 2633 dev_dbg(&cq->hwq.pdev->dev, ··· 2652 2647 */ 2653 2648 cqe = *pcqe; 2654 2649 while (*budget) { 2655 - sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); 2656 - if (sw_cons == cqe_cons) 2650 + swq_last = sq->swq_last; 2651 + if (swq_last == cqe_cons) 2657 2652 break; 2658 - if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { 2653 + if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { 2659 2654 memset(cqe, 0, sizeof(*cqe)); 2660 2655 cqe->status = CQ_REQ_STATUS_OK; 2661 2656 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 2662 2657 cqe->qp_handle = (u64)(unsigned long)qp; 2663 2658 cqe->src_qp = qp->id; 2664 - cqe->wr_id = sq->swq[sw_cons].wr_id; 2665 - cqe->type = sq->swq[sw_cons].type; 2659 + cqe->wr_id = sq->swq[swq_last].wr_id; 2660 + cqe->type = sq->swq[swq_last].type; 2666 2661 cqe++; 2667 2662 (*budget)--; 2668 2663 } 2669 - sq->hwq.cons++; 2664 + bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[swq_last].slots); 2665 + sq->swq_last = sq->swq[swq_last].next_idx; 2670 2666 } 2671 2667 *pcqe = cqe; 2672 - if (!(*budget) && sw_cons != cqe_cons) { 2668 + if (!(*budget) && swq_last != cqe_cons) { 2673 2669 /* Out of budget */ 2674 2670 rc = -EAGAIN; 2675 2671 goto sq_done; ··· 2682 2676 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); 2683 2677 if (cqe_cons == 0xFFFF) { 2684 2678 goto done; 2685 - } else if (cqe_cons > rq->hwq.max_elements) { 2679 + } else if (cqe_cons > rq->max_wqe - 1) { 2686 2680 dev_err(&cq->hwq.pdev->dev, 2687 2681 "FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n", 2688 - cqe_cons, rq->hwq.max_elements); 2682 + cqe_cons, rq->max_wqe); 2689 2683 goto done; 2690 2684 } 2691 2685

+19

drivers/infiniband/hw/bnxt_re/qplib_fp.h

··· 74 74 u8 flags; 75 75 u32 start_psn; 76 76 u32 next_psn; 77 + u8 slots; 77 78 struct sq_psn_search *psn_search; 78 79 struct sq_psn_search_ext *psn_ext; 79 80 }; ··· 214 213 u32 phantom_cqe_cnt; 215 214 u32 next_cq_cons; 216 215 bool flushed; 216 + u32 swq_start; 217 + u32 swq_last; 217 218 }; 218 219 219 220 struct bnxt_qplib_qp { ··· 493 490 struct bnxt_qplib_cqe *cqe, 494 491 int num_cqes); 495 492 void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp); 493 + 494 + static inline void *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que, u32 *swq_idx) 495 + { 496 + u32 idx; 497 + 498 + idx = que->swq_start; 499 + if (swq_idx) 500 + *swq_idx = idx; 501 + return &que->swq[idx]; 502 + } 503 + 504 + static inline void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx) 505 + { 506 + que->swq_start = que->swq[idx].next_idx; 507 + } 508 + 496 509 #endif /* __BNXT_QPLIB_FP_H__ */

+11

drivers/infiniband/hw/bnxt_re/qplib_res.h

··· 363 363 struct bnxt_qplib_ctx *ctx, 364 364 bool virt_fn, bool is_p5); 365 365 366 + static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_hwq *hwq, u32 cnt) 367 + { 368 + hwq->prod = (hwq->prod + cnt) % hwq->depth; 369 + } 370 + 371 + static inline void bnxt_qplib_hwq_incr_cons(struct bnxt_qplib_hwq *hwq, 372 + u32 cnt) 373 + { 374 + hwq->cons = (hwq->cons + cnt) % hwq->depth; 375 + } 376 + 366 377 static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info, 367 378 bool arm) 368 379 {

Configure Feed

Configure Feed