+1

MAINTAINERS

reviewed

··· 5112 5112 5113 5113 ISCSI EXTENSIONS FOR RDMA (ISER) INITIATOR 5114 5114 M: Or Gerlitz <ogerlitz@mellanox.com> 5115 5115 + M: Sagi Grimberg <sagig@mellanox.com> 5115 5116 M: Roi Dayan <roid@mellanox.com> 5116 5117 L: linux-rdma@vger.kernel.org 5117 5118 S: Supported

+2

drivers/infiniband/core/uverbs_cmd.c

reviewed

··· 2518 2518 attr.grh.sgid_index = cmd.attr.grh.sgid_index; 2519 2519 attr.grh.hop_limit = cmd.attr.grh.hop_limit; 2520 2520 attr.grh.traffic_class = cmd.attr.grh.traffic_class; 2521 2521 + attr.vlan_id = 0; 2522 2522 + memset(&attr.dmac, 0, sizeof(attr.dmac)); 2521 2523 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16); 2522 2524 2523 2525 ah = ib_create_ah(pd, &attr);

+5

drivers/infiniband/core/uverbs_main.c

reviewed

··· 477 477 478 478 entry->desc.async.element = element; 479 479 entry->desc.async.event_type = event; 480 480 + entry->desc.async.reserved = 0; 480 481 entry->counter = counter; 481 482 482 483 list_add_tail(&entry->list, &file->async_file->event_list); ··· 502 501 void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr) 503 502 { 504 503 struct ib_uevent_object *uobj; 504 504 + 505 505 + /* for XRC target qp's, check that qp is live */ 506 506 + if (!event->element.qp->uobject || !event->element.qp->uobject->live) 507 507 + return; 505 508 506 509 uobj = container_of(event->element.qp->uobject, 507 510 struct ib_uevent_object, uobject);

+20 -12

drivers/infiniband/hw/cxgb4/cm.c

reviewed

··· 236 236 static void set_emss(struct c4iw_ep *ep, u16 opt) 237 237 { 238 238 ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] - 239 239 - sizeof(struct iphdr) - sizeof(struct tcphdr); 239 239 + ((AF_INET == ep->com.remote_addr.ss_family) ? 240 240 + sizeof(struct iphdr) : sizeof(struct ipv6hdr)) - 241 241 + sizeof(struct tcphdr); 240 242 ep->mss = ep->emss; 241 243 if (GET_TCPOPT_TSTAMP(opt)) 242 242 - ep->emss -= 12; 244 244 + ep->emss -= round_up(TCPOLEN_TIMESTAMP, 4); 243 245 if (ep->emss < 128) 244 246 ep->emss = 128; 245 247 if (ep->emss & 7) ··· 417 415 return NULL; 418 416 if (!our_interface(dev, n->dev) && 419 417 !(n->dev->flags & IFF_LOOPBACK)) { 418 418 + neigh_release(n); 420 419 dst_release(&rt->dst); 421 420 return NULL; 422 421 } ··· 584 581 } 585 582 586 583 static void best_mtu(const unsigned short *mtus, unsigned short mtu, 587 587 - unsigned int *idx, int use_ts) 584 584 + unsigned int *idx, int use_ts, int ipv6) 588 585 { 589 589 - unsigned short hdr_size = sizeof(struct iphdr) + 586 586 + unsigned short hdr_size = (ipv6 ? 587 587 + sizeof(struct ipv6hdr) : 588 588 + sizeof(struct iphdr)) + 590 589 sizeof(struct tcphdr) + 591 591 - (use_ts ? 12 : 0); 590 590 + (use_ts ? 591 591 + round_up(TCPOLEN_TIMESTAMP, 4) : 0); 592 592 unsigned short data_size = mtu - hdr_size; 593 593 594 594 cxgb4_best_aligned_mtu(mtus, hdr_size, data_size, 8, idx); ··· 640 634 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 641 635 642 636 best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx, 643 643 - enable_tcp_timestamps); 637 637 + enable_tcp_timestamps, 638 638 + (AF_INET == ep->com.remote_addr.ss_family) ? 0 : 1); 644 639 wscale = compute_wscale(rcv_win); 645 640 646 641 /* ··· 675 668 if (is_t5(ep->com.dev->rdev.lldi.adapter_type)) { 676 669 opt2 |= T5_OPT_2_VALID; 677 670 opt2 |= V_CONG_CNTRL(CONG_ALG_TAHOE); 671 671 + opt2 |= CONG_CNTRL_VALID; /* OPT_2_ISS for T5 */ 678 672 } 679 673 t4_set_arp_err_handler(skb, ep, act_open_req_arp_failure); 680 674 ··· 721 713 } else { 722 714 u32 isn = (prandom_u32() & ~7UL) - 1; 723 715 724 724 - opt2 |= T5_OPT_2_VALID; 725 725 - opt2 |= CONG_CNTRL_VALID; /* OPT_2_ISS for T5 */ 726 716 if (peer2peer) 727 717 isn += 4; 728 718 ··· 762 756 t5_req6->peer_ip_lo = *((__be64 *) 763 757 (ra6->sin6_addr.s6_addr + 8)); 764 758 t5_req6->opt0 = cpu_to_be64(opt0); 765 765 - t5_req6->params = (__force __be64)cpu_to_be32( 759 759 + t5_req6->params = cpu_to_be64(V_FILTER_TUPLE( 766 760 cxgb4_select_ntuple( 767 761 ep->com.dev->rdev.lldi.ports[0], 768 768 - ep->l2t)); 762 762 + ep->l2t))); 769 763 t5_req6->rsvd = cpu_to_be32(isn); 770 764 PDBG("%s snd_isn %u\n", __func__, 771 765 be32_to_cpu(t5_req6->rsvd)); ··· 1769 1763 req->tcb.tx_max = (__force __be32) jiffies; 1770 1764 req->tcb.rcv_adv = htons(1); 1771 1765 best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx, 1772 1772 - enable_tcp_timestamps); 1766 1766 + enable_tcp_timestamps, 1767 1767 + (AF_INET == ep->com.remote_addr.ss_family) ? 0 : 1); 1773 1768 wscale = compute_wscale(rcv_win); 1774 1769 1775 1770 /* ··· 2169 2162 ep->hwtid)); 2170 2163 2171 2164 best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx, 2172 2172 - enable_tcp_timestamps && req->tcpopt.tstamp); 2165 2165 + enable_tcp_timestamps && req->tcpopt.tstamp, 2166 2166 + (AF_INET == ep->com.remote_addr.ss_family) ? 0 : 1); 2173 2167 wscale = compute_wscale(rcv_win); 2174 2168 2175 2169 /*

+1 -1

drivers/infiniband/hw/cxgb4/device.c

reviewed

··· 60 60 module_param(c4iw_wr_log, int, 0444); 61 61 MODULE_PARM_DESC(c4iw_wr_log, "Enables logging of work request timing data."); 62 62 63 63 - int c4iw_wr_log_size_order = 12; 63 63 + static int c4iw_wr_log_size_order = 12; 64 64 module_param(c4iw_wr_log_size_order, int, 0444); 65 65 MODULE_PARM_DESC(c4iw_wr_log_size_order, 66 66 "Number of entries (log2) in the work request timing log.");

+4 -4

drivers/infiniband/hw/mlx5/main.c

reviewed

··· 657 657 return -EINVAL; 658 658 659 659 idx = get_index(vma->vm_pgoff); 660 660 + if (idx >= uuari->num_uars) 661 661 + return -EINVAL; 662 662 + 660 663 pfn = uar_index2pfn(dev, uuari->uars[idx].index); 661 664 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn 0x%llx\n", idx, 662 665 (unsigned long long)pfn); 663 663 - 664 664 - if (idx >= uuari->num_uars) 665 665 - return -EINVAL; 666 666 667 667 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 668 668 if (io_remap_pfn_range(vma, vma->vm_start, pfn, ··· 1425 1425 static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context) 1426 1426 { 1427 1427 struct mlx5_ib_dev *dev = context; 1428 1428 - destroy_umrc_res(dev); 1429 1428 ib_unregister_device(&dev->ib_dev); 1429 1429 + destroy_umrc_res(dev); 1430 1430 destroy_dev_resources(&dev->devr); 1431 1431 free_comp_eqs(dev); 1432 1432 ib_dealloc_device(&dev->ib_dev);

+10 -8

drivers/infiniband/hw/mlx5/mem.c

reviewed

··· 55 55 u64 pfn; 56 56 struct scatterlist *sg; 57 57 int entry; 58 58 + unsigned long page_shift = ilog2(umem->page_size); 58 59 59 59 - addr = addr >> PAGE_SHIFT; 60 60 + addr = addr >> page_shift; 60 61 tmp = (unsigned long)addr; 61 62 m = find_first_bit(&tmp, sizeof(tmp)); 62 63 skip = 1 << m; 63 64 mask = skip - 1; 64 65 i = 0; 65 66 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 66 66 - len = sg_dma_len(sg) >> PAGE_SHIFT; 67 67 - pfn = sg_dma_address(sg) >> PAGE_SHIFT; 67 67 + len = sg_dma_len(sg) >> page_shift; 68 68 + pfn = sg_dma_address(sg) >> page_shift; 68 69 for (k = 0; k < len; k++) { 69 70 if (!(i & mask)) { 70 71 tmp = (unsigned long)pfn; ··· 104 103 105 104 *ncont = 0; 106 105 } 107 107 - *shift = PAGE_SHIFT + m; 106 106 + *shift = page_shift + m; 108 107 *count = i; 109 108 } 110 109 111 110 void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, 112 111 int page_shift, __be64 *pas, int umr) 113 112 { 114 114 - int shift = page_shift - PAGE_SHIFT; 113 113 + unsigned long umem_page_shift = ilog2(umem->page_size); 114 114 + int shift = page_shift - umem_page_shift; 115 115 int mask = (1 << shift) - 1; 116 116 int i, k; 117 117 u64 cur = 0; ··· 123 121 124 122 i = 0; 125 123 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 126 126 - len = sg_dma_len(sg) >> PAGE_SHIFT; 124 124 + len = sg_dma_len(sg) >> umem_page_shift; 127 125 base = sg_dma_address(sg); 128 126 for (k = 0; k < len; k++) { 129 127 if (!(i & mask)) { 130 130 - cur = base + (k << PAGE_SHIFT); 128 128 + cur = base + (k << umem_page_shift); 131 129 if (umr) 132 130 cur |= 3; 133 131 ··· 136 134 i >> shift, be64_to_cpu(pas[i >> shift])); 137 135 } else 138 136 mlx5_ib_dbg(dev, "=====> 0x%llx\n", 139 139 - base + (k << PAGE_SHIFT)); 137 137 + base + (k << umem_page_shift)); 140 138 i++; 141 139 } 142 140 }

+3 -3

drivers/infiniband/hw/mlx5/mr.c

reviewed

··· 881 881 int order; 882 882 int err; 883 883 884 884 - mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx\n", 885 885 - start, virt_addr, length); 884 884 + mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", 885 885 + start, virt_addr, length, access_flags); 886 886 umem = ib_umem_get(pd->uobject->context, start, length, access_flags, 887 887 0); 888 888 if (IS_ERR(umem)) { 889 889 - mlx5_ib_dbg(dev, "umem get failed\n"); 889 889 + mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(umem)); 890 890 return (void *)umem; 891 891 } 892 892

+61 -92

drivers/infiniband/hw/mlx5/qp.c

reviewed

··· 1317 1317 path->rlid = cpu_to_be16(ah->dlid); 1318 1318 1319 1319 if (ah->ah_flags & IB_AH_GRH) { 1320 1320 + if (ah->grh.sgid_index >= gen->port[port - 1].gid_table_len) { 1321 1321 + pr_err(KERN_ERR "sgid_index (%u) too large. max is %d\n", 1322 1322 + ah->grh.sgid_index, gen->port[port - 1].gid_table_len); 1323 1323 + return -EINVAL; 1324 1324 + } 1320 1325 path->grh_mlid |= 1 << 7; 1321 1326 path->mgid_index = ah->grh.sgid_index; 1322 1327 path->hop_limit = ah->grh.hop_limit; ··· 1336 1331 return err; 1337 1332 path->static_rate = err; 1338 1333 path->port = port; 1339 1339 - 1340 1340 - if (ah->ah_flags & IB_AH_GRH) { 1341 1341 - if (ah->grh.sgid_index >= gen->port[port - 1].gid_table_len) { 1342 1342 - pr_err(KERN_ERR "sgid_index (%u) too large. max is %d\n", 1343 1343 - ah->grh.sgid_index, gen->port[port - 1].gid_table_len); 1344 1344 - return -EINVAL; 1345 1345 - } 1346 1346 - 1347 1347 - path->grh_mlid |= 1 << 7; 1348 1348 - path->mgid_index = ah->grh.sgid_index; 1349 1349 - path->hop_limit = ah->grh.hop_limit; 1350 1350 - path->tclass_flowlabel = 1351 1351 - cpu_to_be32((ah->grh.traffic_class << 20) | 1352 1352 - (ah->grh.flow_label)); 1353 1353 - memcpy(path->rgid, ah->grh.dgid.raw, 16); 1354 1354 - } 1355 1334 1356 1335 if (attr_mask & IB_QP_TIMEOUT) 1357 1336 path->ackto_lt = attr->timeout << 3; ··· 2028 2039 } 2029 2040 } 2030 2041 2031 2031 - static int format_selector(struct ib_sig_attrs *attr, 2032 2032 - struct ib_sig_domain *domain, 2033 2033 - int *selector) 2042 2042 + static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain, 2043 2043 + struct mlx5_bsf_inl *inl) 2034 2044 { 2045 2045 + /* Valid inline section and allow BSF refresh */ 2046 2046 + inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID | 2047 2047 + MLX5_BSF_REFRESH_DIF); 2048 2048 + inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag); 2049 2049 + inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag); 2050 2050 + /* repeating block */ 2051 2051 + inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK; 2052 2052 + inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ? 2053 2053 + MLX5_DIF_CRC : MLX5_DIF_IPCS; 2035 2054 2036 2036 - #define FORMAT_DIF_NONE 0 2037 2037 - #define FORMAT_DIF_CRC_INC 8 2038 2038 - #define FORMAT_DIF_CRC_NO_INC 12 2039 2039 - #define FORMAT_DIF_CSUM_INC 13 2040 2040 - #define FORMAT_DIF_CSUM_NO_INC 14 2055 2055 + if (domain->sig.dif.ref_remap) 2056 2056 + inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG; 2041 2057 2042 2042 - switch (domain->sig.dif.type) { 2043 2043 - case IB_T10DIF_NONE: 2044 2044 - /* No DIF */ 2045 2045 - *selector = FORMAT_DIF_NONE; 2046 2046 - break; 2047 2047 - case IB_T10DIF_TYPE1: /* Fall through */ 2048 2048 - case IB_T10DIF_TYPE2: 2049 2049 - switch (domain->sig.dif.bg_type) { 2050 2050 - case IB_T10DIF_CRC: 2051 2051 - *selector = FORMAT_DIF_CRC_INC; 2052 2052 - break; 2053 2053 - case IB_T10DIF_CSUM: 2054 2054 - *selector = FORMAT_DIF_CSUM_INC; 2055 2055 - break; 2056 2056 - default: 2057 2057 - return 1; 2058 2058 - } 2059 2059 - break; 2060 2060 - case IB_T10DIF_TYPE3: 2061 2061 - switch (domain->sig.dif.bg_type) { 2062 2062 - case IB_T10DIF_CRC: 2063 2063 - *selector = domain->sig.dif.type3_inc_reftag ? 2064 2064 - FORMAT_DIF_CRC_INC : 2065 2065 - FORMAT_DIF_CRC_NO_INC; 2066 2066 - break; 2067 2067 - case IB_T10DIF_CSUM: 2068 2068 - *selector = domain->sig.dif.type3_inc_reftag ? 2069 2069 - FORMAT_DIF_CSUM_INC : 2070 2070 - FORMAT_DIF_CSUM_NO_INC; 2071 2071 - break; 2072 2072 - default: 2073 2073 - return 1; 2074 2074 - } 2075 2075 - break; 2076 2076 - default: 2077 2077 - return 1; 2058 2058 + if (domain->sig.dif.app_escape) { 2059 2059 + if (domain->sig.dif.ref_escape) 2060 2060 + inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE; 2061 2061 + else 2062 2062 + inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE; 2078 2063 } 2079 2064 2080 2080 - return 0; 2065 2065 + inl->dif_app_bitmask_check = 2066 2066 + cpu_to_be16(domain->sig.dif.apptag_check_mask); 2081 2067 } 2082 2068 2083 2069 static int mlx5_set_bsf(struct ib_mr *sig_mr, ··· 2063 2099 struct mlx5_bsf_basic *basic = &bsf->basic; 2064 2100 struct ib_sig_domain *mem = &sig_attrs->mem; 2065 2101 struct ib_sig_domain *wire = &sig_attrs->wire; 2066 2066 - int ret, selector; 2067 2102 2068 2103 memset(bsf, 0, sizeof(*bsf)); 2069 2069 - switch (sig_attrs->mem.sig_type) { 2070 2070 - case IB_SIG_TYPE_T10_DIF: 2071 2071 - if (sig_attrs->wire.sig_type != IB_SIG_TYPE_T10_DIF) 2072 2072 - return -EINVAL; 2073 2104 2074 2074 - /* Input domain check byte mask */ 2075 2075 - basic->check_byte_mask = sig_attrs->check_mask; 2105 2105 + /* Basic + Extended + Inline */ 2106 2106 + basic->bsf_size_sbs = 1 << 7; 2107 2107 + /* Input domain check byte mask */ 2108 2108 + basic->check_byte_mask = sig_attrs->check_mask; 2109 2109 + basic->raw_data_size = cpu_to_be32(data_size); 2110 2110 + 2111 2111 + /* Memory domain */ 2112 2112 + switch (sig_attrs->mem.sig_type) { 2113 2113 + case IB_SIG_TYPE_NONE: 2114 2114 + break; 2115 2115 + case IB_SIG_TYPE_T10_DIF: 2116 2116 + basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval); 2117 2117 + basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx); 2118 2118 + mlx5_fill_inl_bsf(mem, &bsf->m_inl); 2119 2119 + break; 2120 2120 + default: 2121 2121 + return -EINVAL; 2122 2122 + } 2123 2123 + 2124 2124 + /* Wire domain */ 2125 2125 + switch (sig_attrs->wire.sig_type) { 2126 2126 + case IB_SIG_TYPE_NONE: 2127 2127 + break; 2128 2128 + case IB_SIG_TYPE_T10_DIF: 2076 2129 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval && 2077 2077 - mem->sig.dif.type == wire->sig.dif.type) { 2130 2130 + mem->sig_type == wire->sig_type) { 2078 2131 /* Same block structure */ 2079 2079 - basic->bsf_size_sbs = 1 << 4; 2132 2132 + basic->bsf_size_sbs |= 1 << 4; 2080 2133 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type) 2081 2081 - basic->wire.copy_byte_mask |= 0xc0; 2134 2134 + basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK; 2082 2135 if (mem->sig.dif.app_tag == wire->sig.dif.app_tag) 2083 2083 - basic->wire.copy_byte_mask |= 0x30; 2136 2136 + basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK; 2084 2137 if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag) 2085 2085 - basic->wire.copy_byte_mask |= 0x0f; 2138 2138 + basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK; 2086 2139 } else 2087 2140 basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval); 2088 2141 2089 2089 - basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval); 2090 2090 - basic->raw_data_size = cpu_to_be32(data_size); 2091 2091 - 2092 2092 - ret = format_selector(sig_attrs, mem, &selector); 2093 2093 - if (ret) 2094 2094 - return -EINVAL; 2095 2095 - basic->m_bfs_psv = cpu_to_be32(selector << 24 | 2096 2096 - msig->psv_memory.psv_idx); 2097 2097 - 2098 2098 - ret = format_selector(sig_attrs, wire, &selector); 2099 2099 - if (ret) 2100 2100 - return -EINVAL; 2101 2101 - basic->w_bfs_psv = cpu_to_be32(selector << 24 | 2102 2102 - msig->psv_wire.psv_idx); 2142 2142 + basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx); 2143 2143 + mlx5_fill_inl_bsf(wire, &bsf->w_inl); 2103 2144 break; 2104 2104 - 2105 2145 default: 2106 2146 return -EINVAL; 2107 2147 } ··· 2304 2336 memset(psv_seg, 0, sizeof(*psv_seg)); 2305 2337 psv_seg->psv_num = cpu_to_be32(psv_idx); 2306 2338 switch (domain->sig_type) { 2339 2339 + case IB_SIG_TYPE_NONE: 2340 2340 + break; 2307 2341 case IB_SIG_TYPE_T10_DIF: 2308 2342 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 | 2309 2343 domain->sig.dif.app_tag); 2310 2344 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag); 2311 2311 - 2312 2312 - *seg += sizeof(*psv_seg); 2313 2313 - *size += sizeof(*psv_seg) / 16; 2314 2345 break; 2315 2315 - 2316 2346 default: 2317 2347 pr_err("Bad signature type given.\n"); 2318 2348 return 1; 2319 2349 } 2350 2350 + 2351 2351 + *seg += sizeof(*psv_seg); 2352 2352 + *size += sizeof(*psv_seg) / 16; 2320 2353 2321 2354 return 0; 2322 2355 }

+10 -15

drivers/infiniband/hw/ocrdma/ocrdma_hw.c

reviewed

··· 348 348 return mqe; 349 349 } 350 350 351 351 - static void *ocrdma_alloc_mqe(void) 352 352 - { 353 353 - return kzalloc(sizeof(struct ocrdma_mqe), GFP_KERNEL); 354 354 - } 355 355 - 356 351 static void ocrdma_free_q(struct ocrdma_dev *dev, struct ocrdma_queue_info *q) 357 352 { 358 353 dma_free_coherent(&dev->nic_info.pdev->dev, q->size, q->va, q->dma); ··· 561 566 cmd->cqid_pages |= (cq->id << OCRDMA_CREATE_MQ_CQ_ID_SHIFT); 562 567 cmd->async_cqid_valid = OCRDMA_CREATE_MQ_ASYNC_CQ_VALID; 563 568 564 564 - cmd->async_event_bitmap = Bit(OCRDMA_ASYNC_GRP5_EVE_CODE); 565 565 - cmd->async_event_bitmap |= Bit(OCRDMA_ASYNC_RDMA_EVE_CODE); 569 569 + cmd->async_event_bitmap = BIT(OCRDMA_ASYNC_GRP5_EVE_CODE); 570 570 + cmd->async_event_bitmap |= BIT(OCRDMA_ASYNC_RDMA_EVE_CODE); 566 571 567 572 cmd->async_cqid_ringsize = cq->id; 568 573 cmd->async_cqid_ringsize |= (ocrdma_encoded_q_len(mq->len) << ··· 1184 1189 { 1185 1190 struct ocrdma_rdma_stats_req *req = dev->stats_mem.va; 1186 1191 struct ocrdma_mqe *mqe = &dev->stats_mem.mqe; 1187 1187 - struct ocrdma_rdma_stats_resp *old_stats = NULL; 1192 1192 + struct ocrdma_rdma_stats_resp *old_stats; 1188 1193 int status; 1189 1194 1190 1190 - old_stats = kzalloc(sizeof(*old_stats), GFP_KERNEL); 1195 1195 + old_stats = kmalloc(sizeof(*old_stats), GFP_KERNEL); 1191 1196 if (old_stats == NULL) 1192 1197 return -ENOMEM; 1193 1198 ··· 1230 1235 struct ocrdma_get_ctrl_attribs_rsp *ctrl_attr_rsp; 1231 1236 struct mgmt_hba_attribs *hba_attribs; 1232 1237 1233 1233 - mqe = ocrdma_alloc_mqe(); 1238 1238 + mqe = kzalloc(sizeof(struct ocrdma_mqe), GFP_KERNEL); 1234 1239 if (!mqe) 1235 1240 return status; 1236 1236 - memset(mqe, 0, sizeof(*mqe)); 1237 1241 1238 1242 dma.size = sizeof(struct ocrdma_get_ctrl_attribs_rsp); 1239 1243 dma.va = dma_alloc_coherent(&dev->nic_info.pdev->dev, ··· 2273 2279 2274 2280 static int ocrdma_set_av_params(struct ocrdma_qp *qp, 2275 2281 struct ocrdma_modify_qp *cmd, 2276 2276 - struct ib_qp_attr *attrs) 2282 2282 + struct ib_qp_attr *attrs, 2283 2283 + int attr_mask) 2277 2284 { 2278 2285 int status; 2279 2286 struct ib_ah_attr *ah_attr = &attrs->ah_attr; ··· 2314 2319 ocrdma_cpu_to_le32(&cmd->params.dgid[0], sizeof(cmd->params.dgid)); 2315 2320 ocrdma_cpu_to_le32(&cmd->params.sgid[0], sizeof(cmd->params.sgid)); 2316 2321 cmd->params.vlan_dmac_b4_to_b5 = mac_addr[4] | (mac_addr[5] << 8); 2317 2317 - vlan_id = ah_attr->vlan_id; 2318 2318 - if (vlan_id && (vlan_id < 0x1000)) { 2322 2322 + if (attr_mask & IB_QP_VID) { 2323 2323 + vlan_id = attrs->vlan_id; 2319 2324 cmd->params.vlan_dmac_b4_to_b5 |= 2320 2325 vlan_id << OCRDMA_QP_PARAMS_VLAN_SHIFT; 2321 2326 cmd->flags |= OCRDMA_QP_PARA_VLAN_EN_VALID; ··· 2342 2347 cmd->flags |= OCRDMA_QP_PARA_QKEY_VALID; 2343 2348 } 2344 2349 if (attr_mask & IB_QP_AV) { 2345 2345 - status = ocrdma_set_av_params(qp, cmd, attrs); 2350 2350 + status = ocrdma_set_av_params(qp, cmd, attrs, attr_mask); 2346 2351 if (status) 2347 2352 return status; 2348 2353 } else if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_UD) {

+12

drivers/infiniband/hw/ocrdma/ocrdma_main.c

reviewed

··· 388 388 device_remove_file(&dev->ibdev.dev, ocrdma_attributes[i]); 389 389 } 390 390 391 391 + static void ocrdma_add_default_sgid(struct ocrdma_dev *dev) 392 392 + { 393 393 + /* GID Index 0 - Invariant manufacturer-assigned EUI-64 */ 394 394 + union ib_gid *sgid = &dev->sgid_tbl[0]; 395 395 + 396 396 + sgid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL); 397 397 + ocrdma_get_guid(dev, &sgid->raw[8]); 398 398 + } 399 399 + 391 400 static void ocrdma_init_ipv4_gids(struct ocrdma_dev *dev, 392 401 struct net_device *net) 393 402 { ··· 443 434 rdma_vlan_dev_real_dev(net_dev) : net_dev; 444 435 445 436 if (real_dev == dev->nic_info.netdev) { 437 437 + ocrdma_add_default_sgid(dev); 446 438 ocrdma_init_ipv4_gids(dev, net_dev); 447 439 ocrdma_init_ipv6_gids(dev, net_dev); 448 440 } ··· 656 646 return 0; 657 647 658 648 err_be_reg: 649 649 + #if IS_ENABLED(CONFIG_IPV6) 659 650 ocrdma_unregister_inet6addr_notifier(); 660 651 err_notifier6: 652 652 + #endif 661 653 ocrdma_unregister_inetaddr_notifier(); 662 654 return status; 663 655 }

+117 -119

drivers/infiniband/hw/ocrdma/ocrdma_sli.h

reviewed

··· 28 28 #ifndef __OCRDMA_SLI_H__ 29 29 #define __OCRDMA_SLI_H__ 30 30 31 31 - #define Bit(_b) (1 << (_b)) 32 32 - 33 31 enum { 34 32 OCRDMA_ASIC_GEN_SKH_R = 0x04, 35 33 OCRDMA_ASIC_GEN_LANCER = 0x0B ··· 101 103 QTYPE_MCCQ = 3 102 104 }; 103 105 104 104 - #define OCRDMA_MAX_SGID (8) 106 106 + #define OCRDMA_MAX_SGID 8 105 107 106 108 #define OCRDMA_MAX_QP 2048 107 109 #define OCRDMA_MAX_CQ 2048 ··· 126 128 #define OCRDMA_DB_CQ_RING_ID_EXT_MASK 0x0C00 /* bits 10-11 of qid at 12-11 */ 127 129 /* qid #2 msbits at 12-11 */ 128 130 #define OCRDMA_DB_CQ_RING_ID_EXT_MASK_SHIFT 0x1 129 129 - #define OCRDMA_DB_CQ_NUM_POPPED_SHIFT (16) /* bits 16 - 28 */ 131 131 + #define OCRDMA_DB_CQ_NUM_POPPED_SHIFT 16 /* bits 16 - 28 */ 130 132 /* Rearm bit */ 131 131 - #define OCRDMA_DB_CQ_REARM_SHIFT (29) /* bit 29 */ 133 133 + #define OCRDMA_DB_CQ_REARM_SHIFT 29 /* bit 29 */ 132 134 /* solicited bit */ 133 133 - #define OCRDMA_DB_CQ_SOLICIT_SHIFT (31) /* bit 31 */ 135 135 + #define OCRDMA_DB_CQ_SOLICIT_SHIFT 31 /* bit 31 */ 134 136 135 137 #define OCRDMA_EQ_ID_MASK 0x1FF /* bits 0 - 8 */ 136 138 #define OCRDMA_EQ_ID_EXT_MASK 0x3e00 /* bits 9-13 */ 137 137 - #define OCRDMA_EQ_ID_EXT_MASK_SHIFT (2) /* qid bits 9-13 at 11-15 */ 139 139 + #define OCRDMA_EQ_ID_EXT_MASK_SHIFT 2 /* qid bits 9-13 at 11-15 */ 138 140 139 141 /* Clear the interrupt for this eq */ 140 140 - #define OCRDMA_EQ_CLR_SHIFT (9) /* bit 9 */ 142 142 + #define OCRDMA_EQ_CLR_SHIFT 9 /* bit 9 */ 141 143 /* Must be 1 */ 142 142 - #define OCRDMA_EQ_TYPE_SHIFT (10) /* bit 10 */ 144 144 + #define OCRDMA_EQ_TYPE_SHIFT 10 /* bit 10 */ 143 145 /* Number of event entries processed */ 144 144 - #define OCRDMA_NUM_EQE_SHIFT (16) /* bits 16 - 28 */ 146 146 + #define OCRDMA_NUM_EQE_SHIFT 16 /* bits 16 - 28 */ 145 147 /* Rearm bit */ 146 146 - #define OCRDMA_REARM_SHIFT (29) /* bit 29 */ 148 148 + #define OCRDMA_REARM_SHIFT 29 /* bit 29 */ 147 149 148 150 #define OCRDMA_MQ_ID_MASK 0x7FF /* bits 0 - 10 */ 149 151 /* Number of entries posted */ 150 150 - #define OCRDMA_MQ_NUM_MQE_SHIFT (16) /* bits 16 - 29 */ 152 152 + #define OCRDMA_MQ_NUM_MQE_SHIFT 16 /* bits 16 - 29 */ 151 153 152 152 - #define OCRDMA_MIN_HPAGE_SIZE (4096) 154 154 + #define OCRDMA_MIN_HPAGE_SIZE 4096 153 155 154 154 - #define OCRDMA_MIN_Q_PAGE_SIZE (4096) 155 155 - #define OCRDMA_MAX_Q_PAGES (8) 156 156 + #define OCRDMA_MIN_Q_PAGE_SIZE 4096 157 157 + #define OCRDMA_MAX_Q_PAGES 8 156 158 157 159 #define OCRDMA_SLI_ASIC_ID_OFFSET 0x9C 158 160 #define OCRDMA_SLI_ASIC_REV_MASK 0x000000FF ··· 168 170 # 6: 256K Bytes 169 171 # 7: 512K Bytes 170 172 */ 171 171 - #define OCRDMA_MAX_Q_PAGE_SIZE_CNT (8) 173 173 + #define OCRDMA_MAX_Q_PAGE_SIZE_CNT 8 172 174 #define OCRDMA_Q_PAGE_BASE_SIZE (OCRDMA_MIN_Q_PAGE_SIZE * OCRDMA_MAX_Q_PAGES) 173 175 174 174 - #define MAX_OCRDMA_QP_PAGES (8) 176 176 + #define MAX_OCRDMA_QP_PAGES 8 175 177 #define OCRDMA_MAX_WQE_MEM_SIZE (MAX_OCRDMA_QP_PAGES * OCRDMA_MIN_HQ_PAGE_SIZE) 176 178 177 177 - #define OCRDMA_CREATE_CQ_MAX_PAGES (4) 178 178 - #define OCRDMA_DPP_CQE_SIZE (4) 179 179 + #define OCRDMA_CREATE_CQ_MAX_PAGES 4 180 180 + #define OCRDMA_DPP_CQE_SIZE 4 179 181 180 182 #define OCRDMA_GEN2_MAX_CQE 1024 181 183 #define OCRDMA_GEN2_CQ_PAGE_SIZE 4096 ··· 236 238 237 239 enum { 238 240 OCRDMA_MQE_HDR_EMB_SHIFT = 0, 239 239 - OCRDMA_MQE_HDR_EMB_MASK = Bit(0), 241 241 + OCRDMA_MQE_HDR_EMB_MASK = BIT(0), 240 242 OCRDMA_MQE_HDR_SGE_CNT_SHIFT = 3, 241 243 OCRDMA_MQE_HDR_SGE_CNT_MASK = 0x1F << OCRDMA_MQE_HDR_SGE_CNT_SHIFT, 242 244 OCRDMA_MQE_HDR_SPECIAL_SHIFT = 24, ··· 290 292 u32 hi; 291 293 }; 292 294 293 293 - #define MAX_OCRDMA_EQ_PAGES (8) 295 295 + #define MAX_OCRDMA_EQ_PAGES 8 294 296 struct ocrdma_create_eq_req { 295 297 struct ocrdma_mbx_hdr req; 296 298 u32 num_pages; ··· 302 304 }; 303 305 304 306 enum { 305 305 - OCRDMA_CREATE_EQ_VALID = Bit(29), 307 307 + OCRDMA_CREATE_EQ_VALID = BIT(29), 306 308 OCRDMA_CREATE_EQ_CNT_SHIFT = 26, 307 309 OCRDMA_CREATE_CQ_DELAY_SHIFT = 13, 308 310 }; ··· 312 314 u32 vector_eqid; 313 315 }; 314 316 315 315 - #define OCRDMA_EQ_MINOR_OTHER (0x1) 317 317 + #define OCRDMA_EQ_MINOR_OTHER 0x1 316 318 317 319 enum { 318 320 OCRDMA_MCQE_STATUS_SHIFT = 0, ··· 320 322 OCRDMA_MCQE_ESTATUS_SHIFT = 16, 321 323 OCRDMA_MCQE_ESTATUS_MASK = 0xFFFF << OCRDMA_MCQE_ESTATUS_SHIFT, 322 324 OCRDMA_MCQE_CONS_SHIFT = 27, 323 323 - OCRDMA_MCQE_CONS_MASK = Bit(27), 325 325 + OCRDMA_MCQE_CONS_MASK = BIT(27), 324 326 OCRDMA_MCQE_CMPL_SHIFT = 28, 325 325 - OCRDMA_MCQE_CMPL_MASK = Bit(28), 327 327 + OCRDMA_MCQE_CMPL_MASK = BIT(28), 326 328 OCRDMA_MCQE_AE_SHIFT = 30, 327 327 - OCRDMA_MCQE_AE_MASK = Bit(30), 329 329 + OCRDMA_MCQE_AE_MASK = BIT(30), 328 330 OCRDMA_MCQE_VALID_SHIFT = 31, 329 329 - OCRDMA_MCQE_VALID_MASK = Bit(31) 331 331 + OCRDMA_MCQE_VALID_MASK = BIT(31) 330 332 }; 331 333 332 334 struct ocrdma_mcqe { ··· 337 339 }; 338 340 339 341 enum { 340 340 - OCRDMA_AE_MCQE_QPVALID = Bit(31), 342 342 + OCRDMA_AE_MCQE_QPVALID = BIT(31), 341 343 OCRDMA_AE_MCQE_QPID_MASK = 0xFFFF, 342 344 343 343 - OCRDMA_AE_MCQE_CQVALID = Bit(31), 345 345 + OCRDMA_AE_MCQE_CQVALID = BIT(31), 344 346 OCRDMA_AE_MCQE_CQID_MASK = 0xFFFF, 345 345 - OCRDMA_AE_MCQE_VALID = Bit(31), 346 346 - OCRDMA_AE_MCQE_AE = Bit(30), 347 347 + OCRDMA_AE_MCQE_VALID = BIT(31), 348 348 + OCRDMA_AE_MCQE_AE = BIT(30), 347 349 OCRDMA_AE_MCQE_EVENT_TYPE_SHIFT = 16, 348 350 OCRDMA_AE_MCQE_EVENT_TYPE_MASK = 349 351 0xFF << OCRDMA_AE_MCQE_EVENT_TYPE_SHIFT, ··· 384 386 OCRDMA_AE_MPA_MCQE_EVENT_TYPE_MASK = 0xFF << 385 387 OCRDMA_AE_MPA_MCQE_EVENT_TYPE_SHIFT, 386 388 OCRDMA_AE_MPA_MCQE_EVENT_AE_SHIFT = 30, 387 387 - OCRDMA_AE_MPA_MCQE_EVENT_AE_MASK = Bit(30), 389 389 + OCRDMA_AE_MPA_MCQE_EVENT_AE_MASK = BIT(30), 388 390 OCRDMA_AE_MPA_MCQE_EVENT_VALID_SHIFT = 31, 389 389 - OCRDMA_AE_MPA_MCQE_EVENT_VALID_MASK = Bit(31) 391 391 + OCRDMA_AE_MPA_MCQE_EVENT_VALID_MASK = BIT(31) 390 392 }; 391 393 392 394 struct ocrdma_ae_mpa_mcqe { ··· 410 412 OCRDMA_AE_QP_MCQE_EVENT_TYPE_MASK = 0xFF << 411 413 OCRDMA_AE_QP_MCQE_EVENT_TYPE_SHIFT, 412 414 OCRDMA_AE_QP_MCQE_EVENT_AE_SHIFT = 30, 413 413 - OCRDMA_AE_QP_MCQE_EVENT_AE_MASK = Bit(30), 415 415 + OCRDMA_AE_QP_MCQE_EVENT_AE_MASK = BIT(30), 414 416 OCRDMA_AE_QP_MCQE_EVENT_VALID_SHIFT = 31, 415 415 - OCRDMA_AE_QP_MCQE_EVENT_VALID_MASK = Bit(31) 417 417 + OCRDMA_AE_QP_MCQE_EVENT_VALID_MASK = BIT(31) 416 418 }; 417 419 418 420 struct ocrdma_ae_qp_mcqe { ··· 447 449 /* mailbox command request and responses */ 448 450 enum { 449 451 OCRDMA_MBX_QUERY_CFG_CQ_OVERFLOW_SHIFT = 2, 450 450 - OCRDMA_MBX_QUERY_CFG_CQ_OVERFLOW_MASK = Bit(2), 452 452 + OCRDMA_MBX_QUERY_CFG_CQ_OVERFLOW_MASK = BIT(2), 451 453 OCRDMA_MBX_QUERY_CFG_SRQ_SUPPORTED_SHIFT = 3, 452 452 - OCRDMA_MBX_QUERY_CFG_SRQ_SUPPORTED_MASK = Bit(3), 454 454 + OCRDMA_MBX_QUERY_CFG_SRQ_SUPPORTED_MASK = BIT(3), 453 455 OCRDMA_MBX_QUERY_CFG_MAX_QP_SHIFT = 8, 454 456 OCRDMA_MBX_QUERY_CFG_MAX_QP_MASK = 0xFFFFFF << 455 457 OCRDMA_MBX_QUERY_CFG_MAX_QP_SHIFT, ··· 670 672 OCRDMA_CREATE_CQ_PAGE_SIZE_MASK = 0xFF, 671 673 672 674 OCRDMA_CREATE_CQ_COALESCWM_SHIFT = 12, 673 673 - OCRDMA_CREATE_CQ_COALESCWM_MASK = Bit(13) | Bit(12), 674 674 - OCRDMA_CREATE_CQ_FLAGS_NODELAY = Bit(14), 675 675 - OCRDMA_CREATE_CQ_FLAGS_AUTO_VALID = Bit(15), 675 675 + OCRDMA_CREATE_CQ_COALESCWM_MASK = BIT(13) | BIT(12), 676 676 + OCRDMA_CREATE_CQ_FLAGS_NODELAY = BIT(14), 677 677 + OCRDMA_CREATE_CQ_FLAGS_AUTO_VALID = BIT(15), 676 678 677 679 OCRDMA_CREATE_CQ_EQ_ID_MASK = 0xFFFF, 678 680 OCRDMA_CREATE_CQ_CQE_COUNT_MASK = 0xFFFF ··· 685 687 OCRDMA_CREATE_CQ_EQID_SHIFT = 22, 686 688 687 689 OCRDMA_CREATE_CQ_CNT_SHIFT = 27, 688 688 - OCRDMA_CREATE_CQ_FLAGS_VALID = Bit(29), 689 689 - OCRDMA_CREATE_CQ_FLAGS_EVENTABLE = Bit(31), 690 690 + OCRDMA_CREATE_CQ_FLAGS_VALID = BIT(29), 691 691 + OCRDMA_CREATE_CQ_FLAGS_EVENTABLE = BIT(31), 690 692 OCRDMA_CREATE_CQ_DEF_FLAGS = OCRDMA_CREATE_CQ_FLAGS_VALID | 691 693 OCRDMA_CREATE_CQ_FLAGS_EVENTABLE | 692 694 OCRDMA_CREATE_CQ_FLAGS_NODELAY ··· 729 731 OCRDMA_CREATE_MQ_V0_CQ_ID_SHIFT = 22, 730 732 OCRDMA_CREATE_MQ_CQ_ID_SHIFT = 16, 731 733 OCRDMA_CREATE_MQ_RING_SIZE_SHIFT = 16, 732 732 - OCRDMA_CREATE_MQ_VALID = Bit(31), 733 733 - OCRDMA_CREATE_MQ_ASYNC_CQ_VALID = Bit(0) 734 734 + OCRDMA_CREATE_MQ_VALID = BIT(31), 735 735 + OCRDMA_CREATE_MQ_ASYNC_CQ_VALID = BIT(0) 734 736 }; 735 737 736 738 struct ocrdma_create_mq_req { ··· 781 783 OCRDMA_CREATE_QP_REQ_SQ_PAGE_SIZE_SHIFT = 16, 782 784 OCRDMA_CREATE_QP_REQ_RQ_PAGE_SIZE_SHIFT = 19, 783 785 OCRDMA_CREATE_QP_REQ_QPT_SHIFT = 29, 784 784 - OCRDMA_CREATE_QP_REQ_QPT_MASK = Bit(31) | Bit(30) | Bit(29), 786 786 + OCRDMA_CREATE_QP_REQ_QPT_MASK = BIT(31) | BIT(30) | BIT(29), 785 787 786 788 OCRDMA_CREATE_QP_REQ_MAX_RQE_SHIFT = 0, 787 789 OCRDMA_CREATE_QP_REQ_MAX_RQE_MASK = 0xFFFF, ··· 796 798 OCRDMA_CREATE_QP_REQ_MAX_SGE_SEND_SHIFT, 797 799 798 800 OCRDMA_CREATE_QP_REQ_FMR_EN_SHIFT = 0, 799 799 - OCRDMA_CREATE_QP_REQ_FMR_EN_MASK = Bit(0), 801 801 + OCRDMA_CREATE_QP_REQ_FMR_EN_MASK = BIT(0), 800 802 OCRDMA_CREATE_QP_REQ_ZERO_LKEYEN_SHIFT = 1, 801 801 - OCRDMA_CREATE_QP_REQ_ZERO_LKEYEN_MASK = Bit(1), 803 803 + OCRDMA_CREATE_QP_REQ_ZERO_LKEYEN_MASK = BIT(1), 802 804 OCRDMA_CREATE_QP_REQ_BIND_MEMWIN_SHIFT = 2, 803 803 - OCRDMA_CREATE_QP_REQ_BIND_MEMWIN_MASK = Bit(2), 805 805 + OCRDMA_CREATE_QP_REQ_BIND_MEMWIN_MASK = BIT(2), 804 806 OCRDMA_CREATE_QP_REQ_INB_WREN_SHIFT = 3, 805 805 - OCRDMA_CREATE_QP_REQ_INB_WREN_MASK = Bit(3), 807 807 + OCRDMA_CREATE_QP_REQ_INB_WREN_MASK = BIT(3), 806 808 OCRDMA_CREATE_QP_REQ_INB_RDEN_SHIFT = 4, 807 807 - OCRDMA_CREATE_QP_REQ_INB_RDEN_MASK = Bit(4), 809 809 + OCRDMA_CREATE_QP_REQ_INB_RDEN_MASK = BIT(4), 808 810 OCRDMA_CREATE_QP_REQ_USE_SRQ_SHIFT = 5, 809 809 - OCRDMA_CREATE_QP_REQ_USE_SRQ_MASK = Bit(5), 811 811 + OCRDMA_CREATE_QP_REQ_USE_SRQ_MASK = BIT(5), 810 812 OCRDMA_CREATE_QP_REQ_ENABLE_RPIR_SHIFT = 6, 811 811 - OCRDMA_CREATE_QP_REQ_ENABLE_RPIR_MASK = Bit(6), 813 813 + OCRDMA_CREATE_QP_REQ_ENABLE_RPIR_MASK = BIT(6), 812 814 OCRDMA_CREATE_QP_REQ_ENABLE_DPP_SHIFT = 7, 813 813 - OCRDMA_CREATE_QP_REQ_ENABLE_DPP_MASK = Bit(7), 815 815 + OCRDMA_CREATE_QP_REQ_ENABLE_DPP_MASK = BIT(7), 814 816 OCRDMA_CREATE_QP_REQ_ENABLE_DPP_CQ_SHIFT = 8, 815 815 - OCRDMA_CREATE_QP_REQ_ENABLE_DPP_CQ_MASK = Bit(8), 817 817 + OCRDMA_CREATE_QP_REQ_ENABLE_DPP_CQ_MASK = BIT(8), 816 818 OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_SHIFT = 16, 817 819 OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_MASK = 0xFFFF << 818 820 OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_SHIFT, ··· 925 927 OCRDMA_CREATE_QP_RSP_SQ_ID_MASK = 0xFFFF << 926 928 OCRDMA_CREATE_QP_RSP_SQ_ID_SHIFT, 927 929 928 928 - OCRDMA_CREATE_QP_RSP_DPP_ENABLED_MASK = Bit(0), 930 930 + OCRDMA_CREATE_QP_RSP_DPP_ENABLED_MASK = BIT(0), 929 931 OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_SHIFT = 1, 930 932 OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_MASK = 0x7FFF << 931 933 OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_SHIFT, ··· 962 964 OCRDMA_MODIFY_QP_ID_SHIFT = 0, 963 965 OCRDMA_MODIFY_QP_ID_MASK = 0xFFFF, 964 966 965 965 - OCRDMA_QP_PARA_QPS_VALID = Bit(0), 966 966 - OCRDMA_QP_PARA_SQD_ASYNC_VALID = Bit(1), 967 967 - OCRDMA_QP_PARA_PKEY_VALID = Bit(2), 968 968 - OCRDMA_QP_PARA_QKEY_VALID = Bit(3), 969 969 - OCRDMA_QP_PARA_PMTU_VALID = Bit(4), 970 970 - OCRDMA_QP_PARA_ACK_TO_VALID = Bit(5), 971 971 - OCRDMA_QP_PARA_RETRY_CNT_VALID = Bit(6), 972 972 - OCRDMA_QP_PARA_RRC_VALID = Bit(7), 973 973 - OCRDMA_QP_PARA_RQPSN_VALID = Bit(8), 974 974 - OCRDMA_QP_PARA_MAX_IRD_VALID = Bit(9), 975 975 - OCRDMA_QP_PARA_MAX_ORD_VALID = Bit(10), 976 976 - OCRDMA_QP_PARA_RNT_VALID = Bit(11), 977 977 - OCRDMA_QP_PARA_SQPSN_VALID = Bit(12), 978 978 - OCRDMA_QP_PARA_DST_QPN_VALID = Bit(13), 979 979 - OCRDMA_QP_PARA_MAX_WQE_VALID = Bit(14), 980 980 - OCRDMA_QP_PARA_MAX_RQE_VALID = Bit(15), 981 981 - OCRDMA_QP_PARA_SGE_SEND_VALID = Bit(16), 982 982 - OCRDMA_QP_PARA_SGE_RECV_VALID = Bit(17), 983 983 - OCRDMA_QP_PARA_SGE_WR_VALID = Bit(18), 984 984 - OCRDMA_QP_PARA_INB_RDEN_VALID = Bit(19), 985 985 - OCRDMA_QP_PARA_INB_WREN_VALID = Bit(20), 986 986 - OCRDMA_QP_PARA_FLOW_LBL_VALID = Bit(21), 987 987 - OCRDMA_QP_PARA_BIND_EN_VALID = Bit(22), 988 988 - OCRDMA_QP_PARA_ZLKEY_EN_VALID = Bit(23), 989 989 - OCRDMA_QP_PARA_FMR_EN_VALID = Bit(24), 990 990 - OCRDMA_QP_PARA_INBAT_EN_VALID = Bit(25), 991 991 - OCRDMA_QP_PARA_VLAN_EN_VALID = Bit(26), 967 967 + OCRDMA_QP_PARA_QPS_VALID = BIT(0), 968 968 + OCRDMA_QP_PARA_SQD_ASYNC_VALID = BIT(1), 969 969 + OCRDMA_QP_PARA_PKEY_VALID = BIT(2), 970 970 + OCRDMA_QP_PARA_QKEY_VALID = BIT(3), 971 971 + OCRDMA_QP_PARA_PMTU_VALID = BIT(4), 972 972 + OCRDMA_QP_PARA_ACK_TO_VALID = BIT(5), 973 973 + OCRDMA_QP_PARA_RETRY_CNT_VALID = BIT(6), 974 974 + OCRDMA_QP_PARA_RRC_VALID = BIT(7), 975 975 + OCRDMA_QP_PARA_RQPSN_VALID = BIT(8), 976 976 + OCRDMA_QP_PARA_MAX_IRD_VALID = BIT(9), 977 977 + OCRDMA_QP_PARA_MAX_ORD_VALID = BIT(10), 978 978 + OCRDMA_QP_PARA_RNT_VALID = BIT(11), 979 979 + OCRDMA_QP_PARA_SQPSN_VALID = BIT(12), 980 980 + OCRDMA_QP_PARA_DST_QPN_VALID = BIT(13), 981 981 + OCRDMA_QP_PARA_MAX_WQE_VALID = BIT(14), 982 982 + OCRDMA_QP_PARA_MAX_RQE_VALID = BIT(15), 983 983 + OCRDMA_QP_PARA_SGE_SEND_VALID = BIT(16), 984 984 + OCRDMA_QP_PARA_SGE_RECV_VALID = BIT(17), 985 985 + OCRDMA_QP_PARA_SGE_WR_VALID = BIT(18), 986 986 + OCRDMA_QP_PARA_INB_RDEN_VALID = BIT(19), 987 987 + OCRDMA_QP_PARA_INB_WREN_VALID = BIT(20), 988 988 + OCRDMA_QP_PARA_FLOW_LBL_VALID = BIT(21), 989 989 + OCRDMA_QP_PARA_BIND_EN_VALID = BIT(22), 990 990 + OCRDMA_QP_PARA_ZLKEY_EN_VALID = BIT(23), 991 991 + OCRDMA_QP_PARA_FMR_EN_VALID = BIT(24), 992 992 + OCRDMA_QP_PARA_INBAT_EN_VALID = BIT(25), 993 993 + OCRDMA_QP_PARA_VLAN_EN_VALID = BIT(26), 992 994 993 993 - OCRDMA_MODIFY_QP_FLAGS_RD = Bit(0), 994 994 - OCRDMA_MODIFY_QP_FLAGS_WR = Bit(1), 995 995 - OCRDMA_MODIFY_QP_FLAGS_SEND = Bit(2), 996 996 - OCRDMA_MODIFY_QP_FLAGS_ATOMIC = Bit(3) 995 995 + OCRDMA_MODIFY_QP_FLAGS_RD = BIT(0), 996 996 + OCRDMA_MODIFY_QP_FLAGS_WR = BIT(1), 997 997 + OCRDMA_MODIFY_QP_FLAGS_SEND = BIT(2), 998 998 + OCRDMA_MODIFY_QP_FLAGS_ATOMIC = BIT(3) 997 999 }; 998 1000 999 1001 enum { ··· 1012 1014 OCRDMA_QP_PARAMS_MAX_SGE_SEND_MASK = 0xFFFF << 1013 1015 OCRDMA_QP_PARAMS_MAX_SGE_SEND_SHIFT, 1014 1016 1015 1015 - OCRDMA_QP_PARAMS_FLAGS_FMR_EN = Bit(0), 1016 1016 - OCRDMA_QP_PARAMS_FLAGS_LKEY_0_EN = Bit(1), 1017 1017 - OCRDMA_QP_PARAMS_FLAGS_BIND_MW_EN = Bit(2), 1018 1018 - OCRDMA_QP_PARAMS_FLAGS_INBWR_EN = Bit(3), 1019 1019 - OCRDMA_QP_PARAMS_FLAGS_INBRD_EN = Bit(4), 1017 1017 + OCRDMA_QP_PARAMS_FLAGS_FMR_EN = BIT(0), 1018 1018 + OCRDMA_QP_PARAMS_FLAGS_LKEY_0_EN = BIT(1), 1019 1019 + OCRDMA_QP_PARAMS_FLAGS_BIND_MW_EN = BIT(2), 1020 1020 + OCRDMA_QP_PARAMS_FLAGS_INBWR_EN = BIT(3), 1021 1021 + OCRDMA_QP_PARAMS_FLAGS_INBRD_EN = BIT(4), 1020 1022 OCRDMA_QP_PARAMS_STATE_SHIFT = 5, 1021 1021 - OCRDMA_QP_PARAMS_STATE_MASK = Bit(5) | Bit(6) | Bit(7), 1022 1022 - OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC = Bit(8), 1023 1023 - OCRDMA_QP_PARAMS_FLAGS_INB_ATEN = Bit(9), 1023 1023 + OCRDMA_QP_PARAMS_STATE_MASK = BIT(5) | BIT(6) | BIT(7), 1024 1024 + OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC = BIT(8), 1025 1025 + OCRDMA_QP_PARAMS_FLAGS_INB_ATEN = BIT(9), 1024 1026 OCRDMA_QP_PARAMS_MAX_SGE_RECV_SHIFT = 16, 1025 1027 OCRDMA_QP_PARAMS_MAX_SGE_RECV_MASK = 0xFFFF << 1026 1028 OCRDMA_QP_PARAMS_MAX_SGE_RECV_SHIFT, ··· 1275 1277 }; 1276 1278 1277 1279 enum { 1278 1278 - OCRDMA_ALLOC_PD_RSP_DPP = Bit(16), 1280 1280 + OCRDMA_ALLOC_PD_RSP_DPP = BIT(16), 1279 1281 OCRDMA_ALLOC_PD_RSP_DPP_PAGE_SHIFT = 20, 1280 1282 OCRDMA_ALLOC_PD_RSP_PDID_MASK = 0xFFFF, 1281 1283 }; ··· 1307 1309 OCRDMA_ALLOC_LKEY_PD_ID_MASK = 0xFFFF, 1308 1310 1309 1311 OCRDMA_ALLOC_LKEY_ADDR_CHECK_SHIFT = 0, 1310 1310 - OCRDMA_ALLOC_LKEY_ADDR_CHECK_MASK = Bit(0), 1312 1312 + OCRDMA_ALLOC_LKEY_ADDR_CHECK_MASK = BIT(0), 1311 1313 OCRDMA_ALLOC_LKEY_FMR_SHIFT = 1, 1312 1312 - OCRDMA_ALLOC_LKEY_FMR_MASK = Bit(1), 1314 1314 + OCRDMA_ALLOC_LKEY_FMR_MASK = BIT(1), 1313 1315 OCRDMA_ALLOC_LKEY_REMOTE_INV_SHIFT = 2, 1314 1314 - OCRDMA_ALLOC_LKEY_REMOTE_INV_MASK = Bit(2), 1316 1316 + OCRDMA_ALLOC_LKEY_REMOTE_INV_MASK = BIT(2), 1315 1317 OCRDMA_ALLOC_LKEY_REMOTE_WR_SHIFT = 3, 1316 1316 - OCRDMA_ALLOC_LKEY_REMOTE_WR_MASK = Bit(3), 1318 1318 + OCRDMA_ALLOC_LKEY_REMOTE_WR_MASK = BIT(3), 1317 1319 OCRDMA_ALLOC_LKEY_REMOTE_RD_SHIFT = 4, 1318 1318 - OCRDMA_ALLOC_LKEY_REMOTE_RD_MASK = Bit(4), 1320 1320 + OCRDMA_ALLOC_LKEY_REMOTE_RD_MASK = BIT(4), 1319 1321 OCRDMA_ALLOC_LKEY_LOCAL_WR_SHIFT = 5, 1320 1320 - OCRDMA_ALLOC_LKEY_LOCAL_WR_MASK = Bit(5), 1321 1321 - OCRDMA_ALLOC_LKEY_REMOTE_ATOMIC_MASK = Bit(6), 1322 1322 + OCRDMA_ALLOC_LKEY_LOCAL_WR_MASK = BIT(5), 1323 1323 + OCRDMA_ALLOC_LKEY_REMOTE_ATOMIC_MASK = BIT(6), 1322 1324 OCRDMA_ALLOC_LKEY_REMOTE_ATOMIC_SHIFT = 6, 1323 1325 OCRDMA_ALLOC_LKEY_PBL_SIZE_SHIFT = 16, 1324 1326 OCRDMA_ALLOC_LKEY_PBL_SIZE_MASK = 0xFFFF << ··· 1377 1379 OCRDMA_REG_NSMR_HPAGE_SIZE_MASK = 0xFF << 1378 1380 OCRDMA_REG_NSMR_HPAGE_SIZE_SHIFT, 1379 1381 OCRDMA_REG_NSMR_BIND_MEMWIN_SHIFT = 24, 1380 1380 - OCRDMA_REG_NSMR_BIND_MEMWIN_MASK = Bit(24), 1382 1382 + OCRDMA_REG_NSMR_BIND_MEMWIN_MASK = BIT(24), 1381 1383 OCRDMA_REG_NSMR_ZB_SHIFT = 25, 1382 1382 - OCRDMA_REG_NSMR_ZB_SHIFT_MASK = Bit(25), 1384 1384 + OCRDMA_REG_NSMR_ZB_SHIFT_MASK = BIT(25), 1383 1385 OCRDMA_REG_NSMR_REMOTE_INV_SHIFT = 26, 1384 1384 - OCRDMA_REG_NSMR_REMOTE_INV_MASK = Bit(26), 1386 1386 + OCRDMA_REG_NSMR_REMOTE_INV_MASK = BIT(26), 1385 1387 OCRDMA_REG_NSMR_REMOTE_WR_SHIFT = 27, 1386 1386 - OCRDMA_REG_NSMR_REMOTE_WR_MASK = Bit(27), 1388 1388 + OCRDMA_REG_NSMR_REMOTE_WR_MASK = BIT(27), 1387 1389 OCRDMA_REG_NSMR_REMOTE_RD_SHIFT = 28, 1388 1388 - OCRDMA_REG_NSMR_REMOTE_RD_MASK = Bit(28), 1390 1390 + OCRDMA_REG_NSMR_REMOTE_RD_MASK = BIT(28), 1389 1391 OCRDMA_REG_NSMR_LOCAL_WR_SHIFT = 29, 1390 1390 - OCRDMA_REG_NSMR_LOCAL_WR_MASK = Bit(29), 1392 1392 + OCRDMA_REG_NSMR_LOCAL_WR_MASK = BIT(29), 1391 1393 OCRDMA_REG_NSMR_REMOTE_ATOMIC_SHIFT = 30, 1392 1392 - OCRDMA_REG_NSMR_REMOTE_ATOMIC_MASK = Bit(30), 1394 1394 + OCRDMA_REG_NSMR_REMOTE_ATOMIC_MASK = BIT(30), 1393 1395 OCRDMA_REG_NSMR_LAST_SHIFT = 31, 1394 1394 - OCRDMA_REG_NSMR_LAST_MASK = Bit(31) 1396 1396 + OCRDMA_REG_NSMR_LAST_MASK = BIT(31) 1395 1397 }; 1396 1398 1397 1399 struct ocrdma_reg_nsmr { ··· 1418 1420 OCRDMA_REG_NSMR_CONT_NUM_PBL_SHIFT, 1419 1421 1420 1422 OCRDMA_REG_NSMR_CONT_LAST_SHIFT = 31, 1421 1421 - OCRDMA_REG_NSMR_CONT_LAST_MASK = Bit(31) 1423 1423 + OCRDMA_REG_NSMR_CONT_LAST_MASK = BIT(31) 1422 1424 }; 1423 1425 1424 1426 struct ocrdma_reg_nsmr_cont { ··· 1564 1566 1565 1567 enum { 1566 1568 OCRDMA_EQE_VALID_SHIFT = 0, 1567 1567 - OCRDMA_EQE_VALID_MASK = Bit(0), 1569 1569 + OCRDMA_EQE_VALID_MASK = BIT(0), 1568 1570 OCRDMA_EQE_FOR_CQE_MASK = 0xFFFE, 1569 1571 OCRDMA_EQE_RESOURCE_ID_SHIFT = 16, 1570 1572 OCRDMA_EQE_RESOURCE_ID_MASK = 0xFFFF << ··· 1622 1624 OCRDMA_CQE_UD_STATUS_MASK = 0x7 << OCRDMA_CQE_UD_STATUS_SHIFT, 1623 1625 OCRDMA_CQE_STATUS_SHIFT = 16, 1624 1626 OCRDMA_CQE_STATUS_MASK = 0xFF << OCRDMA_CQE_STATUS_SHIFT, 1625 1625 - OCRDMA_CQE_VALID = Bit(31), 1626 1626 - OCRDMA_CQE_INVALIDATE = Bit(30), 1627 1627 - OCRDMA_CQE_QTYPE = Bit(29), 1628 1628 - OCRDMA_CQE_IMM = Bit(28), 1629 1629 - OCRDMA_CQE_WRITE_IMM = Bit(27), 1627 1627 + OCRDMA_CQE_VALID = BIT(31), 1628 1628 + OCRDMA_CQE_INVALIDATE = BIT(30), 1629 1629 + OCRDMA_CQE_QTYPE = BIT(29), 1630 1630 + OCRDMA_CQE_IMM = BIT(28), 1631 1631 + OCRDMA_CQE_WRITE_IMM = BIT(27), 1630 1632 OCRDMA_CQE_QTYPE_SQ = 0, 1631 1633 OCRDMA_CQE_QTYPE_RQ = 1, 1632 1634 OCRDMA_CQE_SRCQP_MASK = 0xFFFFFF ··· 1770 1772 u16 rsvd; 1771 1773 } __packed; 1772 1774 1773 1773 - #define OCRDMA_AV_VALID Bit(7) 1774 1774 - #define OCRDMA_AV_VLAN_VALID Bit(1) 1775 1775 + #define OCRDMA_AV_VALID BIT(7) 1776 1776 + #define OCRDMA_AV_VLAN_VALID BIT(1) 1775 1777 1776 1778 struct ocrdma_av { 1777 1779 struct ocrdma_eth_vlan eth_hdr;

+5 -5

drivers/infiniband/hw/ocrdma/ocrdma_verbs.c

reviewed

··· 388 388 389 389 memset(&resp, 0, sizeof(resp)); 390 390 resp.ah_tbl_len = ctx->ah_tbl.len; 391 391 - resp.ah_tbl_page = ctx->ah_tbl.pa; 391 391 + resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va); 392 392 393 393 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len); 394 394 if (status) ··· 870 870 uresp.page_size = PAGE_ALIGN(cq->len); 871 871 uresp.num_pages = 1; 872 872 uresp.max_hw_cqe = cq->max_hw_cqe; 873 873 - uresp.page_addr[0] = cq->pa; 873 873 + uresp.page_addr[0] = virt_to_phys(cq->va); 874 874 uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id); 875 875 uresp.db_page_size = dev->nic_info.db_page_size; 876 876 uresp.phase_change = cq->phase_change ? 1 : 0; ··· 1123 1123 uresp.sq_dbid = qp->sq.dbid; 1124 1124 uresp.num_sq_pages = 1; 1125 1125 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len); 1126 1126 - uresp.sq_page_addr[0] = qp->sq.pa; 1126 1126 + uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va); 1127 1127 uresp.num_wqe_allocated = qp->sq.max_cnt; 1128 1128 if (!srq) { 1129 1129 uresp.rq_dbid = qp->rq.dbid; 1130 1130 uresp.num_rq_pages = 1; 1131 1131 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len); 1132 1132 - uresp.rq_page_addr[0] = qp->rq.pa; 1132 1132 + uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va); 1133 1133 uresp.num_rqe_allocated = qp->rq.max_cnt; 1134 1134 } 1135 1135 uresp.db_page_addr = usr_db; ··· 1680 1680 memset(&uresp, 0, sizeof(uresp)); 1681 1681 uresp.rq_dbid = srq->rq.dbid; 1682 1682 uresp.num_rq_pages = 1; 1683 1683 - uresp.rq_page_addr[0] = srq->rq.pa; 1683 1683 + uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va); 1684 1684 uresp.rq_page_size = srq->rq.len; 1685 1685 uresp.db_page_addr = dev->nic_info.unmapped_db + 1686 1686 (srq->pd->id * dev->nic_info.db_page_size);

+241 -72

drivers/infiniband/ulp/iser/iscsi_iser.c

reviewed

··· 83 83 84 84 int iser_debug_level = 0; 85 85 bool iser_pi_enable = false; 86 86 - int iser_pi_guard = 0; 86 86 + int iser_pi_guard = 1; 87 87 88 88 MODULE_DESCRIPTION("iSER (iSCSI Extensions for RDMA) Datamover"); 89 89 MODULE_LICENSE("Dual BSD/GPL"); ··· 97 97 MODULE_PARM_DESC(pi_enable, "Enable T10-PI offload support (default:disabled)"); 98 98 99 99 module_param_named(pi_guard, iser_pi_guard, int, 0644); 100 100 - MODULE_PARM_DESC(pi_guard, "T10-PI guard_type, 0:CRC|1:IP_CSUM (default:CRC)"); 100 100 + MODULE_PARM_DESC(pi_guard, "T10-PI guard_type, 0:CRC|1:IP_CSUM (default:IP_CSUM)"); 101 101 102 102 static struct workqueue_struct *release_wq; 103 103 struct iser_global ig; 104 104 105 105 + /* 106 106 + * iscsi_iser_recv() - Process a successfull recv completion 107 107 + * @conn: iscsi connection 108 108 + * @hdr: iscsi header 109 109 + * @rx_data: buffer containing receive data payload 110 110 + * @rx_data_len: length of rx_data 111 111 + * 112 112 + * Notes: In case of data length errors or iscsi PDU completion failures 113 113 + * this routine will signal iscsi layer of connection failure. 114 114 + */ 105 115 void 106 106 - iscsi_iser_recv(struct iscsi_conn *conn, 107 107 - struct iscsi_hdr *hdr, char *rx_data, int rx_data_len) 116 116 + iscsi_iser_recv(struct iscsi_conn *conn, struct iscsi_hdr *hdr, 117 117 + char *rx_data, int rx_data_len) 108 118 { 109 119 int rc = 0; 110 120 int datalen; ··· 145 135 iscsi_conn_failure(conn, rc); 146 136 } 147 137 148 148 - static int iscsi_iser_pdu_alloc(struct iscsi_task *task, uint8_t opcode) 138 138 + /** 139 139 + * iscsi_iser_pdu_alloc() - allocate an iscsi-iser PDU 140 140 + * @task: iscsi task 141 141 + * @opcode: iscsi command opcode 142 142 + * 143 143 + * Netes: This routine can't fail, just assign iscsi task 144 144 + * hdr and max hdr size. 145 145 + */ 146 146 + static int 147 147 + iscsi_iser_pdu_alloc(struct iscsi_task *task, uint8_t opcode) 149 148 { 150 149 struct iscsi_iser_task *iser_task = task->dd_data; 151 150 152 151 task->hdr = (struct iscsi_hdr *)&iser_task->desc.iscsi_header; 153 152 task->hdr_max = sizeof(iser_task->desc.iscsi_header); 153 153 + 154 154 return 0; 155 155 } 156 156 157 157 int iser_initialize_task_headers(struct iscsi_task *task, 158 158 struct iser_tx_desc *tx_desc) 159 159 { 160 160 - struct iser_conn *ib_conn = task->conn->dd_data; 161 161 - struct iser_device *device = ib_conn->device; 160 160 + struct iser_conn *iser_conn = task->conn->dd_data; 161 161 + struct iser_device *device = iser_conn->ib_conn.device; 162 162 struct iscsi_iser_task *iser_task = task->dd_data; 163 163 u64 dma_addr; 164 164 ··· 182 162 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 183 163 tx_desc->tx_sg[0].lkey = device->mr->lkey; 184 164 185 185 - iser_task->ib_conn = ib_conn; 165 165 + iser_task->iser_conn = iser_conn; 186 166 return 0; 187 167 } 168 168 + 188 169 /** 189 189 - * iscsi_iser_task_init - Initialize task 170 170 + * iscsi_iser_task_init() - Initialize iscsi-iser task 190 171 * @task: iscsi task 191 172 * 192 173 * Initialize the task for the scsi command or mgmt command. 174 174 + * 175 175 + * Return: Returns zero on success or -ENOMEM when failing 176 176 + * to init task headers (dma mapping error). 193 177 */ 194 178 static int 195 179 iscsi_iser_task_init(struct iscsi_task *task) ··· 215 191 } 216 192 217 193 /** 218 218 - * iscsi_iser_mtask_xmit - xmit management(immediate) task 194 194 + * iscsi_iser_mtask_xmit() - xmit management (immediate) task 219 195 * @conn: iscsi connection 220 196 * @task: task management task 221 197 * ··· 273 249 return error; 274 250 } 275 251 252 252 + /** 253 253 + * iscsi_iser_task_xmit() - xmit iscsi-iser task 254 254 + * @task: iscsi task 255 255 + * 256 256 + * Return: zero on success or escalates $error on failure. 257 257 + */ 276 258 static int 277 259 iscsi_iser_task_xmit(struct iscsi_task *task) 278 260 { ··· 316 286 return error; 317 287 } 318 288 289 289 + /** 290 290 + * iscsi_iser_cleanup_task() - cleanup an iscsi-iser task 291 291 + * @task: iscsi task 292 292 + * 293 293 + * Notes: In case the RDMA device is already NULL (might have 294 294 + * been removed in DEVICE_REMOVAL CM event it will bail-out 295 295 + * without doing dma unmapping. 296 296 + */ 319 297 static void iscsi_iser_cleanup_task(struct iscsi_task *task) 320 298 { 321 299 struct iscsi_iser_task *iser_task = task->dd_data; 322 300 struct iser_tx_desc *tx_desc = &iser_task->desc; 323 323 - struct iser_conn *ib_conn = task->conn->dd_data; 324 324 - struct iser_device *device = ib_conn->device; 301 301 + struct iser_conn *iser_conn = task->conn->dd_data; 302 302 + struct iser_device *device = iser_conn->ib_conn.device; 303 303 + 304 304 + /* DEVICE_REMOVAL event might have already released the device */ 305 305 + if (!device) 306 306 + return; 325 307 326 308 ib_dma_unmap_single(device->ib_device, 327 309 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); ··· 348 306 } 349 307 } 350 308 351 351 - static u8 iscsi_iser_check_protection(struct iscsi_task *task, sector_t *sector) 309 309 + /** 310 310 + * iscsi_iser_check_protection() - check protection information status of task. 311 311 + * @task: iscsi task 312 312 + * @sector: error sector if exsists (output) 313 313 + * 314 314 + * Return: zero if no data-integrity errors have occured 315 315 + * 0x1: data-integrity error occured in the guard-block 316 316 + * 0x2: data-integrity error occured in the reference tag 317 317 + * 0x3: data-integrity error occured in the application tag 318 318 + * 319 319 + * In addition the error sector is marked. 320 320 + */ 321 321 + static u8 322 322 + iscsi_iser_check_protection(struct iscsi_task *task, sector_t *sector) 352 323 { 353 324 struct iscsi_iser_task *iser_task = task->dd_data; 354 325 ··· 373 318 sector); 374 319 } 375 320 321 321 + /** 322 322 + * iscsi_iser_conn_create() - create a new iscsi-iser connection 323 323 + * @cls_session: iscsi class connection 324 324 + * @conn_idx: connection index within the session (for MCS) 325 325 + * 326 326 + * Return: iscsi_cls_conn when iscsi_conn_setup succeeds or NULL 327 327 + * otherwise. 328 328 + */ 376 329 static struct iscsi_cls_conn * 377 377 - iscsi_iser_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx) 330 330 + iscsi_iser_conn_create(struct iscsi_cls_session *cls_session, 331 331 + uint32_t conn_idx) 378 332 { 379 333 struct iscsi_conn *conn; 380 334 struct iscsi_cls_conn *cls_conn; ··· 402 338 return cls_conn; 403 339 } 404 340 341 341 + /** 342 342 + * iscsi_iser_conn_bind() - bind iscsi and iser connection structures 343 343 + * @cls_session: iscsi class session 344 344 + * @cls_conn: iscsi class connection 345 345 + * @transport_eph: transport end-point handle 346 346 + * @is_leading: indicate if this is the session leading connection (MCS) 347 347 + * 348 348 + * Return: zero on success, $error if iscsi_conn_bind fails and 349 349 + * -EINVAL in case end-point doesn't exsits anymore or iser connection 350 350 + * state is not UP (teardown already started). 351 351 + */ 405 352 static int 406 353 iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session, 407 407 - struct iscsi_cls_conn *cls_conn, uint64_t transport_eph, 354 354 + struct iscsi_cls_conn *cls_conn, 355 355 + uint64_t transport_eph, 408 356 int is_leading) 409 357 { 410 358 struct iscsi_conn *conn = cls_conn->dd_data; 411 411 - struct iser_conn *ib_conn; 359 359 + struct iser_conn *iser_conn; 412 360 struct iscsi_endpoint *ep; 413 361 int error; 414 362 ··· 436 360 (unsigned long long)transport_eph); 437 361 return -EINVAL; 438 362 } 439 439 - ib_conn = ep->dd_data; 363 363 + iser_conn = ep->dd_data; 440 364 441 441 - mutex_lock(&ib_conn->state_mutex); 442 442 - if (ib_conn->state != ISER_CONN_UP) { 365 365 + mutex_lock(&iser_conn->state_mutex); 366 366 + if (iser_conn->state != ISER_CONN_UP) { 443 367 error = -EINVAL; 444 368 iser_err("iser_conn %p state is %d, teardown started\n", 445 445 - ib_conn, ib_conn->state); 369 369 + iser_conn, iser_conn->state); 446 370 goto out; 447 371 } 448 372 449 449 - error = iser_alloc_rx_descriptors(ib_conn, conn->session); 373 373 + error = iser_alloc_rx_descriptors(iser_conn, conn->session); 450 374 if (error) 451 375 goto out; 452 376 453 377 /* binds the iSER connection retrieved from the previously 454 378 * connected ep_handle to the iSCSI layer connection. exchanges 455 379 * connection pointers */ 456 456 - iser_info("binding iscsi conn %p to ib_conn %p\n", conn, ib_conn); 380 380 + iser_info("binding iscsi conn %p to iser_conn %p\n", conn, iser_conn); 457 381 458 458 - conn->dd_data = ib_conn; 459 459 - ib_conn->iscsi_conn = conn; 382 382 + conn->dd_data = iser_conn; 383 383 + iser_conn->iscsi_conn = conn; 460 384 461 385 out: 462 462 - mutex_unlock(&ib_conn->state_mutex); 386 386 + mutex_unlock(&iser_conn->state_mutex); 463 387 return error; 464 388 } 465 389 390 390 + /** 391 391 + * iscsi_iser_conn_start() - start iscsi-iser connection 392 392 + * @cls_conn: iscsi class connection 393 393 + * 394 394 + * Notes: Here iser intialize (or re-initialize) stop_completion as 395 395 + * from this point iscsi must call conn_stop in session/connection 396 396 + * teardown so iser transport must wait for it. 397 397 + */ 466 398 static int 467 399 iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn) 468 400 { 469 401 struct iscsi_conn *iscsi_conn; 470 470 - struct iser_conn *ib_conn; 402 402 + struct iser_conn *iser_conn; 471 403 472 404 iscsi_conn = cls_conn->dd_data; 473 473 - ib_conn = iscsi_conn->dd_data; 474 474 - reinit_completion(&ib_conn->stop_completion); 405 405 + iser_conn = iscsi_conn->dd_data; 406 406 + reinit_completion(&iser_conn->stop_completion); 475 407 476 408 return iscsi_conn_start(cls_conn); 477 409 } 478 410 411 411 + /** 412 412 + * iscsi_iser_conn_stop() - stop iscsi-iser connection 413 413 + * @cls_conn: iscsi class connection 414 414 + * @flag: indicate if recover or terminate (passed as is) 415 415 + * 416 416 + * Notes: Calling iscsi_conn_stop might theoretically race with 417 417 + * DEVICE_REMOVAL event and dereference a previously freed RDMA device 418 418 + * handle, so we call it under iser the state lock to protect against 419 419 + * this kind of race. 420 420 + */ 479 421 static void 480 422 iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) 481 423 { 482 424 struct iscsi_conn *conn = cls_conn->dd_data; 483 483 - struct iser_conn *ib_conn = conn->dd_data; 425 425 + struct iser_conn *iser_conn = conn->dd_data; 484 426 485 485 - iser_dbg("stopping iscsi_conn: %p, ib_conn: %p\n", conn, ib_conn); 486 486 - iscsi_conn_stop(cls_conn, flag); 427 427 + iser_info("stopping iscsi_conn: %p, iser_conn: %p\n", conn, iser_conn); 487 428 488 429 /* 489 430 * Userspace may have goofed up and not bound the connection or 490 431 * might have only partially setup the connection. 491 432 */ 492 492 - if (ib_conn) { 433 433 + if (iser_conn) { 434 434 + mutex_lock(&iser_conn->state_mutex); 435 435 + iscsi_conn_stop(cls_conn, flag); 436 436 + iser_conn_terminate(iser_conn); 437 437 + 438 438 + /* unbind */ 439 439 + iser_conn->iscsi_conn = NULL; 493 440 conn->dd_data = NULL; 494 494 - complete(&ib_conn->stop_completion); 441 441 + 442 442 + complete(&iser_conn->stop_completion); 443 443 + mutex_unlock(&iser_conn->state_mutex); 444 444 + } else { 445 445 + iscsi_conn_stop(cls_conn, flag); 495 446 } 496 447 } 497 448 498 498 - static void iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session) 449 449 + /** 450 450 + * iscsi_iser_session_destroy() - destroy iscsi-iser session 451 451 + * @cls_session: iscsi class session 452 452 + * 453 453 + * Removes and free iscsi host. 454 454 + */ 455 455 + static void 456 456 + iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session) 499 457 { 500 458 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 501 459 ··· 549 439 SHOST_DIX_TYPE3_PROTECTION : 0); 550 440 } 551 441 442 442 + /** 443 443 + * iscsi_iser_session_create() - create an iscsi-iser session 444 444 + * @ep: iscsi end-point handle 445 445 + * @cmds_max: maximum commands in this session 446 446 + * @qdepth: session command queue depth 447 447 + * @initial_cmdsn: initiator command sequnce number 448 448 + * 449 449 + * Allocates and adds a scsi host, expose DIF supprot if 450 450 + * exists, and sets up an iscsi session. 451 451 + */ 552 452 static struct iscsi_cls_session * 553 453 iscsi_iser_session_create(struct iscsi_endpoint *ep, 554 454 uint16_t cmds_max, uint16_t qdepth, ··· 567 447 struct iscsi_cls_session *cls_session; 568 448 struct iscsi_session *session; 569 449 struct Scsi_Host *shost; 570 570 - struct iser_conn *ib_conn = NULL; 450 450 + struct iser_conn *iser_conn = NULL; 451 451 + struct ib_conn *ib_conn; 571 452 572 453 shost = iscsi_host_alloc(&iscsi_iser_sht, 0, 0); 573 454 if (!shost) ··· 585 464 * the leading conn's ep so this will be NULL; 586 465 */ 587 466 if (ep) { 588 588 - ib_conn = ep->dd_data; 467 467 + iser_conn = ep->dd_data; 468 468 + ib_conn = &iser_conn->ib_conn; 589 469 if (ib_conn->pi_support) { 590 470 u32 sig_caps = ib_conn->device->dev_attr.sig_prot_cap; 591 471 ··· 598 476 } 599 477 } 600 478 601 601 - if (iscsi_host_add(shost, 602 602 - ep ? ib_conn->device->ib_device->dma_device : NULL)) 479 479 + if (iscsi_host_add(shost, ep ? 480 480 + ib_conn->device->ib_device->dma_device : NULL)) 603 481 goto free_host; 604 482 605 483 if (cmds_max > ISER_DEF_XMIT_CMDS_MAX) { ··· 671 549 return 0; 672 550 } 673 551 552 552 + /** 553 553 + * iscsi_iser_set_param() - set class connection parameter 554 554 + * @cls_conn: iscsi class connection 555 555 + * @stats: iscsi stats to output 556 556 + * 557 557 + * Output connection statistics. 558 558 + */ 674 559 static void 675 560 iscsi_iser_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats) 676 561 { ··· 706 577 static int iscsi_iser_get_ep_param(struct iscsi_endpoint *ep, 707 578 enum iscsi_param param, char *buf) 708 579 { 709 709 - struct iser_conn *ib_conn = ep->dd_data; 580 580 + struct iser_conn *iser_conn = ep->dd_data; 710 581 int len; 711 582 712 583 switch (param) { 713 584 case ISCSI_PARAM_CONN_PORT: 714 585 case ISCSI_PARAM_CONN_ADDRESS: 715 715 - if (!ib_conn || !ib_conn->cma_id) 586 586 + if (!iser_conn || !iser_conn->ib_conn.cma_id) 716 587 return -ENOTCONN; 717 588 718 589 return iscsi_conn_get_addr_param((struct sockaddr_storage *) 719 719 - &ib_conn->cma_id->route.addr.dst_addr, 720 720 - param, buf); 590 590 + &iser_conn->ib_conn.cma_id->route.addr.dst_addr, 591 591 + param, buf); 721 592 break; 722 593 default: 723 594 return -ENOSYS; ··· 726 597 return len; 727 598 } 728 599 600 600 + /** 601 601 + * iscsi_iser_ep_connect() - Initiate iSER connection establishment 602 602 + * @shost: scsi_host 603 603 + * @dst_addr: destination address 604 604 + * @non-blocking: indicate if routine can block 605 605 + * 606 606 + * Allocate an iscsi endpoint, an iser_conn structure and bind them. 607 607 + * After that start RDMA connection establishment via rdma_cm. We 608 608 + * don't allocate iser_conn embedded in iscsi_endpoint since in teardown 609 609 + * the endpoint will be destroyed at ep_disconnect while iser_conn will 610 610 + * cleanup its resources asynchronuously. 611 611 + * 612 612 + * Return: iscsi_endpoint created by iscsi layer or ERR_PTR(error) 613 613 + * if fails. 614 614 + */ 729 615 static struct iscsi_endpoint * 730 616 iscsi_iser_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr, 731 617 int non_blocking) 732 618 { 733 619 int err; 734 734 - struct iser_conn *ib_conn; 620 620 + struct iser_conn *iser_conn; 735 621 struct iscsi_endpoint *ep; 736 622 737 623 ep = iscsi_create_endpoint(0); 738 624 if (!ep) 739 625 return ERR_PTR(-ENOMEM); 740 626 741 741 - ib_conn = kzalloc(sizeof(*ib_conn), GFP_KERNEL); 742 742 - if (!ib_conn) { 627 627 + iser_conn = kzalloc(sizeof(*iser_conn), GFP_KERNEL); 628 628 + if (!iser_conn) { 743 629 err = -ENOMEM; 744 630 goto failure; 745 631 } 746 632 747 747 - ep->dd_data = ib_conn; 748 748 - ib_conn->ep = ep; 749 749 - iser_conn_init(ib_conn); 633 633 + ep->dd_data = iser_conn; 634 634 + iser_conn->ep = ep; 635 635 + iser_conn_init(iser_conn); 750 636 751 751 - err = iser_connect(ib_conn, NULL, dst_addr, non_blocking); 637 637 + err = iser_connect(iser_conn, NULL, dst_addr, non_blocking); 752 638 if (err) 753 639 goto failure; 754 640 ··· 773 629 return ERR_PTR(err); 774 630 } 775 631 632 632 + /** 633 633 + * iscsi_iser_ep_poll() - poll for iser connection establishment to complete 634 634 + * @ep: iscsi endpoint (created at ep_connect) 635 635 + * @timeout_ms: polling timeout allowed in ms. 636 636 + * 637 637 + * This routine boils down to waiting for up_completion signaling 638 638 + * that cma_id got CONNECTED event. 639 639 + * 640 640 + * Return: 1 if succeeded in connection establishment, 0 if timeout expired 641 641 + * (libiscsi will retry will kick in) or -1 if interrupted by signal 642 642 + * or more likely iser connection state transitioned to TEMINATING or 643 643 + * DOWN during the wait period. 644 644 + */ 776 645 static int 777 646 iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) 778 647 { 779 779 - struct iser_conn *ib_conn; 648 648 + struct iser_conn *iser_conn; 780 649 int rc; 781 650 782 782 - ib_conn = ep->dd_data; 783 783 - rc = wait_for_completion_interruptible_timeout(&ib_conn->up_completion, 651 651 + iser_conn = ep->dd_data; 652 652 + rc = wait_for_completion_interruptible_timeout(&iser_conn->up_completion, 784 653 msecs_to_jiffies(timeout_ms)); 785 654 /* if conn establishment failed, return error code to iscsi */ 786 655 if (rc == 0) { 787 787 - mutex_lock(&ib_conn->state_mutex); 788 788 - if (ib_conn->state == ISER_CONN_TERMINATING || 789 789 - ib_conn->state == ISER_CONN_DOWN) 656 656 + mutex_lock(&iser_conn->state_mutex); 657 657 + if (iser_conn->state == ISER_CONN_TERMINATING || 658 658 + iser_conn->state == ISER_CONN_DOWN) 790 659 rc = -1; 791 791 - mutex_unlock(&ib_conn->state_mutex); 660 660 + mutex_unlock(&iser_conn->state_mutex); 792 661 } 793 662 794 794 - iser_info("ib conn %p rc = %d\n", ib_conn, rc); 663 663 + iser_info("ib conn %p rc = %d\n", iser_conn, rc); 795 664 796 665 if (rc > 0) 797 666 return 1; /* success, this is the equivalent of POLLOUT */ ··· 814 657 return rc; /* signal */ 815 658 } 816 659 660 660 + /** 661 661 + * iscsi_iser_ep_disconnect() - Initiate connection teardown process 662 662 + * @ep: iscsi endpoint handle 663 663 + * 664 664 + * This routine is not blocked by iser and RDMA termination process 665 665 + * completion as we queue a deffered work for iser/RDMA destruction 666 666 + * and cleanup or actually call it immediately in case we didn't pass 667 667 + * iscsi conn bind/start stage, thus it is safe. 668 668 + */ 817 669 static void 818 670 iscsi_iser_ep_disconnect(struct iscsi_endpoint *ep) 819 671 { 820 820 - struct iser_conn *ib_conn; 672 672 + struct iser_conn *iser_conn; 821 673 822 822 - ib_conn = ep->dd_data; 823 823 - iser_info("ep %p ib conn %p state %d\n", ep, ib_conn, ib_conn->state); 824 824 - mutex_lock(&ib_conn->state_mutex); 825 825 - iser_conn_terminate(ib_conn); 674 674 + iser_conn = ep->dd_data; 675 675 + iser_info("ep %p iser conn %p state %d\n", 676 676 + ep, iser_conn, iser_conn->state); 677 677 + 678 678 + mutex_lock(&iser_conn->state_mutex); 679 679 + iser_conn_terminate(iser_conn); 826 680 827 681 /* 828 682 * if iser_conn and iscsi_conn are bound, we must wait for ··· 841 673 * the iser resources. Otherwise we are safe to free resources 842 674 * immediately. 843 675 */ 844 844 - if (ib_conn->iscsi_conn) { 845 845 - INIT_WORK(&ib_conn->release_work, iser_release_work); 846 846 - queue_work(release_wq, &ib_conn->release_work); 847 847 - mutex_unlock(&ib_conn->state_mutex); 676 676 + if (iser_conn->iscsi_conn) { 677 677 + INIT_WORK(&iser_conn->release_work, iser_release_work); 678 678 + queue_work(release_wq, &iser_conn->release_work); 679 679 + mutex_unlock(&iser_conn->state_mutex); 848 680 } else { 849 849 - ib_conn->state = ISER_CONN_DOWN; 850 850 - mutex_unlock(&ib_conn->state_mutex); 851 851 - iser_conn_release(ib_conn); 681 681 + iser_conn->state = ISER_CONN_DOWN; 682 682 + mutex_unlock(&iser_conn->state_mutex); 683 683 + iser_conn_release(iser_conn); 852 684 } 853 685 iscsi_destroy_endpoint(ep); 854 686 } ··· 1011 843 1012 844 static void __exit iser_exit(void) 1013 845 { 1014 1014 - struct iser_conn *ib_conn, *n; 846 846 + struct iser_conn *iser_conn, *n; 1015 847 int connlist_empty; 1016 848 1017 849 iser_dbg("Removing iSER datamover...\n"); ··· 1024 856 if (!connlist_empty) { 1025 857 iser_err("Error cleanup stage completed but we still have iser " 1026 858 "connections, destroying them anyway.\n"); 1027 1027 - list_for_each_entry_safe(ib_conn, n, &ig.connlist, conn_list) { 1028 1028 - iser_conn_release(ib_conn); 859 859 + list_for_each_entry_safe(iser_conn, n, &ig.connlist, 860 860 + conn_list) { 861 861 + iser_conn_release(iser_conn); 1029 862 } 1030 863 } 1031 864

+294 -114

drivers/infiniband/ulp/iser/iscsi_iser.h

reviewed

··· 69 69 70 70 #define DRV_NAME "iser" 71 71 #define PFX DRV_NAME ": " 72 72 - #define DRV_VER "1.4.1" 72 72 + #define DRV_VER "1.4.8" 73 73 74 74 - #define iser_dbg(fmt, arg...) \ 75 75 - do { \ 76 76 - if (iser_debug_level > 2) \ 77 77 - printk(KERN_DEBUG PFX "%s:" fmt,\ 78 78 - __func__ , ## arg); \ 74 74 + #define iser_dbg(fmt, arg...) \ 75 75 + do { \ 76 76 + if (iser_debug_level > 2) \ 77 77 + printk(KERN_DEBUG PFX "%s: " fmt,\ 78 78 + __func__ , ## arg); \ 79 79 } while (0) 80 80 81 81 #define iser_warn(fmt, arg...) \ 82 82 do { \ 83 83 if (iser_debug_level > 0) \ 84 84 - pr_warn(PFX "%s:" fmt, \ 84 84 + pr_warn(PFX "%s: " fmt, \ 85 85 __func__ , ## arg); \ 86 86 } while (0) 87 87 88 88 #define iser_info(fmt, arg...) \ 89 89 do { \ 90 90 if (iser_debug_level > 1) \ 91 91 - pr_info(PFX "%s:" fmt, \ 91 91 + pr_info(PFX "%s: " fmt, \ 92 92 __func__ , ## arg); \ 93 93 } while (0) 94 94 95 95 #define iser_err(fmt, arg...) \ 96 96 do { \ 97 97 - printk(KERN_ERR PFX "%s:" fmt, \ 97 97 + printk(KERN_ERR PFX "%s: " fmt, \ 98 98 __func__ , ## arg); \ 99 99 } while (0) 100 100 101 101 #define SHIFT_4K 12 102 102 #define SIZE_4K (1ULL << SHIFT_4K) 103 103 #define MASK_4K (~(SIZE_4K-1)) 104 104 - 105 104 /* support up to 512KB in one RDMA */ 106 105 #define ISCSI_ISER_SG_TABLESIZE (0x80000 >> SHIFT_4K) 107 106 #define ISER_DEF_XMIT_CMDS_DEFAULT 512 ··· 144 145 ISER_MAX_TX_MISC_PDUS + \ 145 146 ISER_MAX_RX_MISC_PDUS) 146 147 148 148 + #define ISER_WC_BATCH_COUNT 16 149 149 + #define ISER_SIGNAL_CMD_COUNT 32 150 150 + 147 151 #define ISER_VER 0x10 148 152 #define ISER_WSV 0x08 149 153 #define ISER_RSV 0x04 150 154 151 155 #define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL 156 156 + #define ISER_BEACON_WRID 0xfffffffffffffffeULL 152 157 158 158 + /** 159 159 + * struct iser_hdr - iSER header 160 160 + * 161 161 + * @flags: flags support (zbva, remote_inv) 162 162 + * @rsvd: reserved 163 163 + * @write_stag: write rkey 164 164 + * @write_va: write virtual address 165 165 + * @reaf_stag: read rkey 166 166 + * @read_va: read virtual address 167 167 + */ 153 168 struct iser_hdr { 154 169 u8 flags; 155 170 u8 rsvd[3]; 156 156 - __be32 write_stag; /* write rkey */ 171 171 + __be32 write_stag; 157 172 __be64 write_va; 158 158 - __be32 read_stag; /* read rkey */ 173 173 + __be32 read_stag; 159 174 __be64 read_va; 160 175 } __attribute__((packed)); 161 176 ··· 192 179 /* Length of an object name string */ 193 180 #define ISER_OBJECT_NAME_SIZE 64 194 181 195 195 - enum iser_ib_conn_state { 182 182 + enum iser_conn_state { 196 183 ISER_CONN_INIT, /* descriptor allocd, no conn */ 197 184 ISER_CONN_PENDING, /* in the process of being established */ 198 185 ISER_CONN_UP, /* up and running */ ··· 213 200 ISER_DIRS_NUM 214 201 }; 215 202 203 203 + /** 204 204 + * struct iser_data_buf - iSER data buffer 205 205 + * 206 206 + * @buf: pointer to the sg list 207 207 + * @size: num entries of this sg 208 208 + * @data_len: total beffer byte len 209 209 + * @dma_nents: returned by dma_map_sg 210 210 + * @copy_buf: allocated copy buf for SGs unaligned 211 211 + * for rdma which are copied 212 212 + * @sg_single: SG-ified clone of a non SG SC or 213 213 + * unaligned SG 214 214 + */ 216 215 struct iser_data_buf { 217 217 - void *buf; /* pointer to the sg list */ 218 218 - unsigned int size; /* num entries of this sg */ 219 219 - unsigned long data_len; /* total data len */ 220 220 - unsigned int dma_nents; /* returned by dma_map_sg */ 221 221 - char *copy_buf; /* allocated copy buf for SGs unaligned * 222 222 - * for rdma which are copied */ 223 223 - struct scatterlist sg_single; /* SG-ified clone of a non SG SC or * 224 224 - * unaligned SG */ 216 216 + void *buf; 217 217 + unsigned int size; 218 218 + unsigned long data_len; 219 219 + unsigned int dma_nents; 220 220 + char *copy_buf; 221 221 + struct scatterlist sg_single; 225 222 }; 226 223 227 224 /* fwd declarations */ 228 225 struct iser_device; 229 229 - struct iser_cq_desc; 230 226 struct iscsi_iser_task; 231 227 struct iscsi_endpoint; 232 228 229 229 + /** 230 230 + * struct iser_mem_reg - iSER memory registration info 231 231 + * 232 232 + * @lkey: MR local key 233 233 + * @rkey: MR remote key 234 234 + * @va: MR start address (buffer va) 235 235 + * @len: MR length 236 236 + * @mem_h: pointer to registration context (FMR/Fastreg) 237 237 + * @is_mr: indicates weather we registered the buffer 238 238 + */ 233 239 struct iser_mem_reg { 234 240 u32 lkey; 235 241 u32 rkey; ··· 258 226 int is_mr; 259 227 }; 260 228 229 229 + /** 230 230 + * struct iser_regd_buf - iSER buffer registration desc 231 231 + * 232 232 + * @reg: memory registration info 233 233 + * @virt_addr: virtual address of buffer 234 234 + * @device: reference to iser device 235 235 + * @direction: dma direction (for dma_unmap) 236 236 + * @data_size: data buffer size in bytes 237 237 + */ 261 238 struct iser_regd_buf { 262 262 - struct iser_mem_reg reg; /* memory registration info */ 239 239 + struct iser_mem_reg reg; 263 240 void *virt_addr; 264 264 - struct iser_device *device; /* device->device for dma_unmap */ 265 265 - enum dma_data_direction direction; /* direction for dma_unmap */ 241 241 + struct iser_device *device; 242 242 + enum dma_data_direction direction; 266 243 unsigned int data_size; 267 244 }; 268 245 ··· 281 240 ISCSI_TX_DATAOUT 282 241 }; 283 242 243 243 + /** 244 244 + * struct iser_tx_desc - iSER TX descriptor (for send wr_id) 245 245 + * 246 246 + * @iser_header: iser header 247 247 + * @iscsi_header: iscsi header 248 248 + * @type: command/control/dataout 249 249 + * @dam_addr: header buffer dma_address 250 250 + * @tx_sg: sg[0] points to iser/iscsi headers 251 251 + * sg[1] optionally points to either of immediate data 252 252 + * unsolicited data-out or control 253 253 + * @num_sge: number sges used on this TX task 254 254 + */ 284 255 struct iser_tx_desc { 285 256 struct iser_hdr iser_header; 286 257 struct iscsi_hdr iscsi_header; 287 258 enum iser_desc_type type; 288 259 u64 dma_addr; 289 289 - /* sg[0] points to iser/iscsi headers, sg[1] optionally points to either 290 290 - of immediate data, unsolicited data-out or control (login,text) */ 291 260 struct ib_sge tx_sg[2]; 292 261 int num_sge; 293 262 }; 294 263 295 264 #define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \ 296 265 sizeof(u64) + sizeof(struct ib_sge))) 266 266 + /** 267 267 + * struct iser_rx_desc - iSER RX descriptor (for recv wr_id) 268 268 + * 269 269 + * @iser_header: iser header 270 270 + * @iscsi_header: iscsi header 271 271 + * @data: received data segment 272 272 + * @dma_addr: receive buffer dma address 273 273 + * @rx_sg: ib_sge of receive buffer 274 274 + * @pad: for sense data TODO: Modify to maximum sense length supported 275 275 + */ 297 276 struct iser_rx_desc { 298 277 struct iser_hdr iser_header; 299 278 struct iscsi_hdr iscsi_header; ··· 326 265 #define ISER_MAX_CQ 4 327 266 328 267 struct iser_conn; 268 268 + struct ib_conn; 329 269 struct iscsi_iser_task; 330 270 271 271 + /** 272 272 + * struct iser_comp - iSER completion context 273 273 + * 274 274 + * @device: pointer to device handle 275 275 + * @cq: completion queue 276 276 + * @wcs: work completion array 277 277 + * @tasklet: Tasklet handle 278 278 + * @active_qps: Number of active QPs attached 279 279 + * to completion context 280 280 + */ 281 281 + struct iser_comp { 282 282 + struct iser_device *device; 283 283 + struct ib_cq *cq; 284 284 + struct ib_wc wcs[ISER_WC_BATCH_COUNT]; 285 285 + struct tasklet_struct tasklet; 286 286 + int active_qps; 287 287 + }; 288 288 + 289 289 + /** 290 290 + * struct iser_device - iSER device handle 291 291 + * 292 292 + * @ib_device: RDMA device 293 293 + * @pd: Protection Domain for this device 294 294 + * @dev_attr: Device attributes container 295 295 + * @mr: Global DMA memory region 296 296 + * @event_handler: IB events handle routine 297 297 + * @ig_list: entry in devices list 298 298 + * @refcount: Reference counter, dominated by open iser connections 299 299 + * @comps_used: Number of completion contexts used, Min between online 300 300 + * cpus and device max completion vectors 301 301 + * @comps: Dinamically allocated array of completion handlers 302 302 + * Memory registration pool Function pointers (FMR or Fastreg): 303 303 + * @iser_alloc_rdma_reg_res: Allocation of memory regions pool 304 304 + * @iser_free_rdma_reg_res: Free of memory regions pool 305 305 + * @iser_reg_rdma_mem: Memory registration routine 306 306 + * @iser_unreg_rdma_mem: Memory deregistration routine 307 307 + */ 331 308 struct iser_device { 332 309 struct ib_device *ib_device; 333 310 struct ib_pd *pd; 334 311 struct ib_device_attr dev_attr; 335 335 - struct ib_cq *rx_cq[ISER_MAX_CQ]; 336 336 - struct ib_cq *tx_cq[ISER_MAX_CQ]; 337 312 struct ib_mr *mr; 338 338 - struct tasklet_struct cq_tasklet[ISER_MAX_CQ]; 339 313 struct ib_event_handler event_handler; 340 340 - struct list_head ig_list; /* entry in ig devices list */ 314 314 + struct list_head ig_list; 341 315 int refcount; 342 342 - int cq_active_qps[ISER_MAX_CQ]; 343 343 - int cqs_used; 344 344 - struct iser_cq_desc *cq_desc; 345 345 - int (*iser_alloc_rdma_reg_res)(struct iser_conn *ib_conn, 316 316 + int comps_used; 317 317 + struct iser_comp comps[ISER_MAX_CQ]; 318 318 + int (*iser_alloc_rdma_reg_res)(struct ib_conn *ib_conn, 346 319 unsigned cmds_max); 347 347 - void (*iser_free_rdma_reg_res)(struct iser_conn *ib_conn); 320 320 + void (*iser_free_rdma_reg_res)(struct ib_conn *ib_conn); 348 321 int (*iser_reg_rdma_mem)(struct iscsi_iser_task *iser_task, 349 322 enum iser_data_dir cmd_dir); 350 323 void (*iser_unreg_rdma_mem)(struct iscsi_iser_task *iser_task, ··· 396 301 ISER_FASTREG_PROTECTED = 1 << 3, 397 302 }; 398 303 304 304 + /** 305 305 + * struct iser_pi_context - Protection information context 306 306 + * 307 307 + * @prot_mr: protection memory region 308 308 + * @prot_frpl: protection fastreg page list 309 309 + * @sig_mr: signature feature enabled memory region 310 310 + */ 399 311 struct iser_pi_context { 400 312 struct ib_mr *prot_mr; 401 313 struct ib_fast_reg_page_list *prot_frpl; 402 314 struct ib_mr *sig_mr; 403 315 }; 404 316 317 317 + /** 318 318 + * struct fast_reg_descriptor - Fast registration descriptor 319 319 + * 320 320 + * @list: entry in connection fastreg pool 321 321 + * @data_mr: data memory region 322 322 + * @data_frpl: data fastreg page list 323 323 + * @pi_ctx: protection information context 324 324 + * @reg_indicators: fast registration indicators 325 325 + */ 405 326 struct fast_reg_descriptor { 406 327 struct list_head list; 407 407 - /* For fast registration - FRWR */ 408 328 struct ib_mr *data_mr; 409 329 struct ib_fast_reg_page_list *data_frpl; 410 330 struct iser_pi_context *pi_ctx; 411 411 - /* registration indicators container */ 412 331 u8 reg_indicators; 413 332 }; 414 333 334 334 + /** 335 335 + * struct ib_conn - Infiniband related objects 336 336 + * 337 337 + * @cma_id: rdma_cm connection maneger handle 338 338 + * @qp: Connection Queue-pair 339 339 + * @post_recv_buf_count: post receive counter 340 340 + * @rx_wr: receive work request for batch posts 341 341 + * @device: reference to iser device 342 342 + * @comp: iser completion context 343 343 + * @pi_support: Indicate device T10-PI support 344 344 + * @beacon: beacon send wr to signal all flush errors were drained 345 345 + * @flush_comp: completes when all connection completions consumed 346 346 + * @lock: protects fmr/fastreg pool 347 347 + * @union.fmr: 348 348 + * @pool: FMR pool for fast registrations 349 349 + * @page_vec: page vector to hold mapped commands pages 350 350 + * used for registration 351 351 + * @union.fastreg: 352 352 + * @pool: Fast registration descriptors pool for fast 353 353 + * registrations 354 354 + * @pool_size: Size of pool 355 355 + */ 356 356 + struct ib_conn { 357 357 + struct rdma_cm_id *cma_id; 358 358 + struct ib_qp *qp; 359 359 + int post_recv_buf_count; 360 360 + struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX]; 361 361 + struct iser_device *device; 362 362 + struct iser_comp *comp; 363 363 + bool pi_support; 364 364 + struct ib_send_wr beacon; 365 365 + struct completion flush_comp; 366 366 + spinlock_t lock; 367 367 + union { 368 368 + struct { 369 369 + struct ib_fmr_pool *pool; 370 370 + struct iser_page_vec *page_vec; 371 371 + } fmr; 372 372 + struct { 373 373 + struct list_head pool; 374 374 + int pool_size; 375 375 + } fastreg; 376 376 + }; 377 377 + }; 378 378 + 379 379 + /** 380 380 + * struct iser_conn - iSER connection context 381 381 + * 382 382 + * @ib_conn: connection RDMA resources 383 383 + * @iscsi_conn: link to matching iscsi connection 384 384 + * @ep: transport handle 385 385 + * @state: connection logical state 386 386 + * @qp_max_recv_dtos: maximum number of data outs, corresponds 387 387 + * to max number of post recvs 388 388 + * @qp_max_recv_dtos_mask: (qp_max_recv_dtos - 1) 389 389 + * @min_posted_rx: (qp_max_recv_dtos >> 2) 390 390 + * @name: connection peer portal 391 391 + * @release_work: deffered work for release job 392 392 + * @state_mutex: protects iser onnection state 393 393 + * @stop_completion: conn_stop completion 394 394 + * @ib_completion: RDMA cleanup completion 395 395 + * @up_completion: connection establishment completed 396 396 + * (state is ISER_CONN_UP) 397 397 + * @conn_list: entry in ig conn list 398 398 + * @login_buf: login data buffer (stores login parameters) 399 399 + * @login_req_buf: login request buffer 400 400 + * @login_req_dma: login request buffer dma address 401 401 + * @login_resp_buf: login response buffer 402 402 + * @login_resp_dma: login response buffer dma address 403 403 + * @rx_desc_head: head of rx_descs cyclic buffer 404 404 + * @rx_descs: rx buffers array (cyclic buffer) 405 405 + * @num_rx_descs: number of rx descriptors 406 406 + */ 415 407 struct iser_conn { 408 408 + struct ib_conn ib_conn; 416 409 struct iscsi_conn *iscsi_conn; 417 410 struct iscsi_endpoint *ep; 418 418 - enum iser_ib_conn_state state; /* rdma connection state */ 419 419 - atomic_t refcount; 420 420 - spinlock_t lock; /* used for state changes */ 421 421 - struct iser_device *device; /* device context */ 422 422 - struct rdma_cm_id *cma_id; /* CMA ID */ 423 423 - struct ib_qp *qp; /* QP */ 424 424 - unsigned qp_max_recv_dtos; /* num of rx buffers */ 425 425 - unsigned qp_max_recv_dtos_mask; /* above minus 1 */ 426 426 - unsigned min_posted_rx; /* qp_max_recv_dtos >> 2 */ 427 427 - int post_recv_buf_count; /* posted rx count */ 428 428 - atomic_t post_send_buf_count; /* posted tx count */ 411 411 + enum iser_conn_state state; 412 412 + unsigned qp_max_recv_dtos; 413 413 + unsigned qp_max_recv_dtos_mask; 414 414 + unsigned min_posted_rx; 429 415 char name[ISER_OBJECT_NAME_SIZE]; 430 416 struct work_struct release_work; 431 431 - struct completion stop_completion; 432 417 struct mutex state_mutex; 433 433 - struct completion flush_completion; 418 418 + struct completion stop_completion; 419 419 + struct completion ib_completion; 434 420 struct completion up_completion; 435 435 - struct list_head conn_list; /* entry in ig conn list */ 421 421 + struct list_head conn_list; 436 422 437 423 char *login_buf; 438 424 char *login_req_buf, *login_resp_buf; 439 425 u64 login_req_dma, login_resp_dma; 440 426 unsigned int rx_desc_head; 441 427 struct iser_rx_desc *rx_descs; 442 442 - struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX]; 443 443 - bool pi_support; 444 444 - 445 445 - /* Connection memory registration pool */ 446 446 - union { 447 447 - struct { 448 448 - struct ib_fmr_pool *pool; /* pool of IB FMRs */ 449 449 - struct iser_page_vec *page_vec; /* represents SG to fmr maps* 450 450 - * maps serialized as tx is*/ 451 451 - } fmr; 452 452 - struct { 453 453 - struct list_head pool; 454 454 - int pool_size; 455 455 - } fastreg; 456 456 - }; 428 428 + u32 num_rx_descs; 457 429 }; 458 430 431 431 + /** 432 432 + * struct iscsi_iser_task - iser task context 433 433 + * 434 434 + * @desc: TX descriptor 435 435 + * @iser_conn: link to iser connection 436 436 + * @status: current task status 437 437 + * @sc: link to scsi command 438 438 + * @command_sent: indicate if command was sent 439 439 + * @dir: iser data direction 440 440 + * @rdma_regd: task rdma registration desc 441 441 + * @data: iser data buffer desc 442 442 + * @data_copy: iser data copy buffer desc (bounce buffer) 443 443 + * @prot: iser protection buffer desc 444 444 + * @prot_copy: iser protection copy buffer desc (bounce buffer) 445 445 + */ 459 446 struct iscsi_iser_task { 460 447 struct iser_tx_desc desc; 461 461 - struct iser_conn *ib_conn; 448 448 + struct iser_conn *iser_conn; 462 449 enum iser_task_status status; 463 450 struct scsi_cmnd *sc; 464 464 - int command_sent; /* set if command sent */ 465 465 - int dir[ISER_DIRS_NUM]; /* set if dir use*/ 466 466 - struct iser_regd_buf rdma_regd[ISER_DIRS_NUM];/* regd rdma buf */ 467 467 - struct iser_data_buf data[ISER_DIRS_NUM]; /* orig. data des*/ 468 468 - struct iser_data_buf data_copy[ISER_DIRS_NUM];/* contig. copy */ 469 469 - struct iser_data_buf prot[ISER_DIRS_NUM]; /* prot desc */ 470 470 - struct iser_data_buf prot_copy[ISER_DIRS_NUM];/* prot copy */ 451 451 + int command_sent; 452 452 + int dir[ISER_DIRS_NUM]; 453 453 + struct iser_regd_buf rdma_regd[ISER_DIRS_NUM]; 454 454 + struct iser_data_buf data[ISER_DIRS_NUM]; 455 455 + struct iser_data_buf data_copy[ISER_DIRS_NUM]; 456 456 + struct iser_data_buf prot[ISER_DIRS_NUM]; 457 457 + struct iser_data_buf prot_copy[ISER_DIRS_NUM]; 471 458 }; 472 459 473 460 struct iser_page_vec { ··· 559 382 int data_size; 560 383 }; 561 384 562 562 - struct iser_cq_desc { 563 563 - struct iser_device *device; 564 564 - int cq_index; 565 565 - }; 566 566 - 385 385 + /** 386 386 + * struct iser_global: iSER global context 387 387 + * 388 388 + * @device_list_mutex: protects device_list 389 389 + * @device_list: iser devices global list 390 390 + * @connlist_mutex: protects connlist 391 391 + * @connlist: iser connections global list 392 392 + * @desc_cache: kmem cache for tx dataout 393 393 + */ 567 394 struct iser_global { 568 568 - struct mutex device_list_mutex;/* */ 569 569 - struct list_head device_list; /* all iSER devices */ 395 395 + struct mutex device_list_mutex; 396 396 + struct list_head device_list; 570 397 struct mutex connlist_mutex; 571 571 - struct list_head connlist; /* all iSER IB connections */ 572 572 - 398 398 + struct list_head connlist; 573 399 struct kmem_cache *desc_cache; 574 400 }; 575 401 ··· 580 400 extern int iser_debug_level; 581 401 extern bool iser_pi_enable; 582 402 extern int iser_pi_guard; 583 583 - 584 584 - /* allocate connection resources needed for rdma functionality */ 585 585 - int iser_conn_set_full_featured_mode(struct iscsi_conn *conn); 586 403 587 404 int iser_send_control(struct iscsi_conn *conn, 588 405 struct iscsi_task *task); ··· 592 415 struct iscsi_data *hdr); 593 416 594 417 void iscsi_iser_recv(struct iscsi_conn *conn, 595 595 - struct iscsi_hdr *hdr, 596 596 - char *rx_data, 597 597 - int rx_data_len); 418 418 + struct iscsi_hdr *hdr, 419 419 + char *rx_data, 420 420 + int rx_data_len); 598 421 599 599 - void iser_conn_init(struct iser_conn *ib_conn); 422 422 + void iser_conn_init(struct iser_conn *iser_conn); 600 423 601 601 - void iser_conn_release(struct iser_conn *ib_conn); 424 424 + void iser_conn_release(struct iser_conn *iser_conn); 602 425 603 603 - void iser_conn_terminate(struct iser_conn *ib_conn); 426 426 + int iser_conn_terminate(struct iser_conn *iser_conn); 604 427 605 428 void iser_release_work(struct work_struct *work); 606 429 607 430 void iser_rcv_completion(struct iser_rx_desc *desc, 608 608 - unsigned long dto_xfer_len, 609 609 - struct iser_conn *ib_conn); 431 431 + unsigned long dto_xfer_len, 432 432 + struct ib_conn *ib_conn); 610 433 611 611 - void iser_snd_completion(struct iser_tx_desc *desc, struct iser_conn *ib_conn); 434 434 + void iser_snd_completion(struct iser_tx_desc *desc, 435 435 + struct ib_conn *ib_conn); 612 436 613 437 void iser_task_rdma_init(struct iscsi_iser_task *task); 614 438 615 439 void iser_task_rdma_finalize(struct iscsi_iser_task *task); 616 440 617 617 - void iser_free_rx_descriptors(struct iser_conn *ib_conn); 441 441 + void iser_free_rx_descriptors(struct iser_conn *iser_conn); 618 442 619 443 void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, 620 444 struct iser_data_buf *mem, ··· 627 449 int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *task, 628 450 enum iser_data_dir cmd_dir); 629 451 630 630 - int iser_connect(struct iser_conn *ib_conn, 631 631 - struct sockaddr *src_addr, 632 632 - struct sockaddr *dst_addr, 633 633 - int non_blocking); 452 452 + int iser_connect(struct iser_conn *iser_conn, 453 453 + struct sockaddr *src_addr, 454 454 + struct sockaddr *dst_addr, 455 455 + int non_blocking); 634 456 635 635 - int iser_reg_page_vec(struct iser_conn *ib_conn, 457 457 + int iser_reg_page_vec(struct ib_conn *ib_conn, 636 458 struct iser_page_vec *page_vec, 637 637 - struct iser_mem_reg *mem_reg); 459 459 + struct iser_mem_reg *mem_reg); 638 460 639 461 void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task, 640 462 enum iser_data_dir cmd_dir); 641 463 void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task, 642 464 enum iser_data_dir cmd_dir); 643 465 644 644 - int iser_post_recvl(struct iser_conn *ib_conn); 645 645 - int iser_post_recvm(struct iser_conn *ib_conn, int count); 646 646 - int iser_post_send(struct iser_conn *ib_conn, struct iser_tx_desc *tx_desc); 466 466 + int iser_post_recvl(struct iser_conn *iser_conn); 467 467 + int iser_post_recvm(struct iser_conn *iser_conn, int count); 468 468 + int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, 469 469 + bool signal); 647 470 648 471 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, 649 649 - struct iser_data_buf *data, 650 650 - enum iser_data_dir iser_dir, 651 651 - enum dma_data_direction dma_dir); 472 472 + struct iser_data_buf *data, 473 473 + enum iser_data_dir iser_dir, 474 474 + enum dma_data_direction dma_dir); 652 475 653 476 void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, 654 477 struct iser_data_buf *data); 655 478 int iser_initialize_task_headers(struct iscsi_task *task, 656 479 struct iser_tx_desc *tx_desc); 657 657 - int iser_alloc_rx_descriptors(struct iser_conn *ib_conn, struct iscsi_session *session); 658 658 - int iser_create_fmr_pool(struct iser_conn *ib_conn, unsigned cmds_max); 659 659 - void iser_free_fmr_pool(struct iser_conn *ib_conn); 660 660 - int iser_create_fastreg_pool(struct iser_conn *ib_conn, unsigned cmds_max); 661 661 - void iser_free_fastreg_pool(struct iser_conn *ib_conn); 480 480 + int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, 481 481 + struct iscsi_session *session); 482 482 + int iser_create_fmr_pool(struct ib_conn *ib_conn, unsigned cmds_max); 483 483 + void iser_free_fmr_pool(struct ib_conn *ib_conn); 484 484 + int iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max); 485 485 + void iser_free_fastreg_pool(struct ib_conn *ib_conn); 662 486 u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task, 663 487 enum iser_data_dir cmd_dir, sector_t *sector); 664 488 #endif

+105 -93

drivers/infiniband/ulp/iser/iser_initiator.c

reviewed

··· 49 49 50 50 { 51 51 struct iscsi_iser_task *iser_task = task->dd_data; 52 52 - struct iser_device *device = iser_task->ib_conn->device; 52 52 + struct iser_device *device = iser_task->iser_conn->ib_conn.device; 53 53 struct iser_regd_buf *regd_buf; 54 54 int err; 55 55 struct iser_hdr *hdr = &iser_task->desc.iser_header; ··· 103 103 unsigned int edtl) 104 104 { 105 105 struct iscsi_iser_task *iser_task = task->dd_data; 106 106 - struct iser_device *device = iser_task->ib_conn->device; 106 106 + struct iser_device *device = iser_task->iser_conn->ib_conn.device; 107 107 struct iser_regd_buf *regd_buf; 108 108 int err; 109 109 struct iser_hdr *hdr = &iser_task->desc.iser_header; ··· 160 160 } 161 161 162 162 /* creates a new tx descriptor and adds header regd buffer */ 163 163 - static void iser_create_send_desc(struct iser_conn *ib_conn, 163 163 + static void iser_create_send_desc(struct iser_conn *iser_conn, 164 164 struct iser_tx_desc *tx_desc) 165 165 { 166 166 - struct iser_device *device = ib_conn->device; 166 166 + struct iser_device *device = iser_conn->ib_conn.device; 167 167 168 168 ib_dma_sync_single_for_cpu(device->ib_device, 169 169 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); ··· 179 179 } 180 180 } 181 181 182 182 - static void iser_free_login_buf(struct iser_conn *ib_conn) 182 182 + static void iser_free_login_buf(struct iser_conn *iser_conn) 183 183 { 184 184 - if (!ib_conn->login_buf) 184 184 + struct iser_device *device = iser_conn->ib_conn.device; 185 185 + 186 186 + if (!iser_conn->login_buf) 185 187 return; 186 188 187 187 - if (ib_conn->login_req_dma) 188 188 - ib_dma_unmap_single(ib_conn->device->ib_device, 189 189 - ib_conn->login_req_dma, 189 189 + if (iser_conn->login_req_dma) 190 190 + ib_dma_unmap_single(device->ib_device, 191 191 + iser_conn->login_req_dma, 190 192 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE); 191 193 192 192 - if (ib_conn->login_resp_dma) 193 193 - ib_dma_unmap_single(ib_conn->device->ib_device, 194 194 - ib_conn->login_resp_dma, 194 194 + if (iser_conn->login_resp_dma) 195 195 + ib_dma_unmap_single(device->ib_device, 196 196 + iser_conn->login_resp_dma, 195 197 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE); 196 198 197 197 - kfree(ib_conn->login_buf); 199 199 + kfree(iser_conn->login_buf); 198 200 199 201 /* make sure we never redo any unmapping */ 200 200 - ib_conn->login_req_dma = 0; 201 201 - ib_conn->login_resp_dma = 0; 202 202 - ib_conn->login_buf = NULL; 202 202 + iser_conn->login_req_dma = 0; 203 203 + iser_conn->login_resp_dma = 0; 204 204 + iser_conn->login_buf = NULL; 203 205 } 204 206 205 205 - static int iser_alloc_login_buf(struct iser_conn *ib_conn) 207 207 + static int iser_alloc_login_buf(struct iser_conn *iser_conn) 206 208 { 207 207 - struct iser_device *device; 209 209 + struct iser_device *device = iser_conn->ib_conn.device; 208 210 int req_err, resp_err; 209 211 210 210 - BUG_ON(ib_conn->device == NULL); 212 212 + BUG_ON(device == NULL); 211 213 212 212 - device = ib_conn->device; 213 213 - 214 214 - ib_conn->login_buf = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 214 214 + iser_conn->login_buf = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 215 215 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 216 216 - if (!ib_conn->login_buf) 216 216 + if (!iser_conn->login_buf) 217 217 goto out_err; 218 218 219 219 - ib_conn->login_req_buf = ib_conn->login_buf; 220 220 - ib_conn->login_resp_buf = ib_conn->login_buf + 219 219 + iser_conn->login_req_buf = iser_conn->login_buf; 220 220 + iser_conn->login_resp_buf = iser_conn->login_buf + 221 221 ISCSI_DEF_MAX_RECV_SEG_LEN; 222 222 223 223 - ib_conn->login_req_dma = ib_dma_map_single(ib_conn->device->ib_device, 224 224 - (void *)ib_conn->login_req_buf, 225 225 - ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE); 223 223 + iser_conn->login_req_dma = ib_dma_map_single(device->ib_device, 224 224 + iser_conn->login_req_buf, 225 225 + ISCSI_DEF_MAX_RECV_SEG_LEN, 226 226 + DMA_TO_DEVICE); 226 227 227 227 - ib_conn->login_resp_dma = ib_dma_map_single(ib_conn->device->ib_device, 228 228 - (void *)ib_conn->login_resp_buf, 229 229 - ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE); 228 228 + iser_conn->login_resp_dma = ib_dma_map_single(device->ib_device, 229 229 + iser_conn->login_resp_buf, 230 230 + ISER_RX_LOGIN_SIZE, 231 231 + DMA_FROM_DEVICE); 230 232 231 233 req_err = ib_dma_mapping_error(device->ib_device, 232 232 - ib_conn->login_req_dma); 234 234 + iser_conn->login_req_dma); 233 235 resp_err = ib_dma_mapping_error(device->ib_device, 234 234 - ib_conn->login_resp_dma); 236 236 + iser_conn->login_resp_dma); 235 237 236 238 if (req_err || resp_err) { 237 239 if (req_err) 238 238 - ib_conn->login_req_dma = 0; 240 240 + iser_conn->login_req_dma = 0; 239 241 if (resp_err) 240 240 - ib_conn->login_resp_dma = 0; 242 242 + iser_conn->login_resp_dma = 0; 241 243 goto free_login_buf; 242 244 } 243 245 return 0; 244 246 245 247 free_login_buf: 246 246 - iser_free_login_buf(ib_conn); 248 248 + iser_free_login_buf(iser_conn); 247 249 248 250 out_err: 249 251 iser_err("unable to alloc or map login buf\n"); 250 252 return -ENOMEM; 251 253 } 252 254 253 253 - int iser_alloc_rx_descriptors(struct iser_conn *ib_conn, struct iscsi_session *session) 255 255 + int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, 256 256 + struct iscsi_session *session) 254 257 { 255 258 int i, j; 256 259 u64 dma_addr; 257 260 struct iser_rx_desc *rx_desc; 258 261 struct ib_sge *rx_sg; 259 259 - struct iser_device *device = ib_conn->device; 262 262 + struct ib_conn *ib_conn = &iser_conn->ib_conn; 263 263 + struct iser_device *device = ib_conn->device; 260 264 261 261 - ib_conn->qp_max_recv_dtos = session->cmds_max; 262 262 - ib_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */ 263 263 - ib_conn->min_posted_rx = ib_conn->qp_max_recv_dtos >> 2; 265 265 + iser_conn->qp_max_recv_dtos = session->cmds_max; 266 266 + iser_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */ 267 267 + iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2; 264 268 265 269 if (device->iser_alloc_rdma_reg_res(ib_conn, session->scsi_cmds_max)) 266 270 goto create_rdma_reg_res_failed; 267 271 268 268 - if (iser_alloc_login_buf(ib_conn)) 272 272 + if (iser_alloc_login_buf(iser_conn)) 269 273 goto alloc_login_buf_fail; 270 274 271 271 - ib_conn->rx_descs = kmalloc(session->cmds_max * 275 275 + iser_conn->num_rx_descs = session->cmds_max; 276 276 + iser_conn->rx_descs = kmalloc(iser_conn->num_rx_descs * 272 277 sizeof(struct iser_rx_desc), GFP_KERNEL); 273 273 - if (!ib_conn->rx_descs) 278 278 + if (!iser_conn->rx_descs) 274 279 goto rx_desc_alloc_fail; 275 280 276 276 - rx_desc = ib_conn->rx_descs; 281 281 + rx_desc = iser_conn->rx_descs; 277 282 278 278 - for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++) { 283 283 + for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) { 279 284 dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc, 280 285 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 281 286 if (ib_dma_mapping_error(device->ib_device, dma_addr)) ··· 294 289 rx_sg->lkey = device->mr->lkey; 295 290 } 296 291 297 297 - ib_conn->rx_desc_head = 0; 292 292 + iser_conn->rx_desc_head = 0; 298 293 return 0; 299 294 300 295 rx_desc_dma_map_failed: 301 301 - rx_desc = ib_conn->rx_descs; 296 296 + rx_desc = iser_conn->rx_descs; 302 297 for (j = 0; j < i; j++, rx_desc++) 303 298 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 304 299 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 305 305 - kfree(ib_conn->rx_descs); 306 306 - ib_conn->rx_descs = NULL; 300 300 + kfree(iser_conn->rx_descs); 301 301 + iser_conn->rx_descs = NULL; 307 302 rx_desc_alloc_fail: 308 308 - iser_free_login_buf(ib_conn); 303 303 + iser_free_login_buf(iser_conn); 309 304 alloc_login_buf_fail: 310 305 device->iser_free_rdma_reg_res(ib_conn); 311 306 create_rdma_reg_res_failed: ··· 313 308 return -ENOMEM; 314 309 } 315 310 316 316 - void iser_free_rx_descriptors(struct iser_conn *ib_conn) 311 311 + void iser_free_rx_descriptors(struct iser_conn *iser_conn) 317 312 { 318 313 int i; 319 314 struct iser_rx_desc *rx_desc; 315 315 + struct ib_conn *ib_conn = &iser_conn->ib_conn; 320 316 struct iser_device *device = ib_conn->device; 321 317 322 322 - if (!ib_conn->rx_descs) 318 318 + if (!iser_conn->rx_descs) 323 319 goto free_login_buf; 324 320 325 321 if (device->iser_free_rdma_reg_res) 326 322 device->iser_free_rdma_reg_res(ib_conn); 327 323 328 328 - rx_desc = ib_conn->rx_descs; 329 329 - for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++) 324 324 + rx_desc = iser_conn->rx_descs; 325 325 + for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) 330 326 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 331 327 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 332 332 - kfree(ib_conn->rx_descs); 328 328 + kfree(iser_conn->rx_descs); 333 329 /* make sure we never redo any unmapping */ 334 334 - ib_conn->rx_descs = NULL; 330 330 + iser_conn->rx_descs = NULL; 335 331 336 332 free_login_buf: 337 337 - iser_free_login_buf(ib_conn); 333 333 + iser_free_login_buf(iser_conn); 338 334 } 339 335 340 336 static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req) 341 337 { 342 342 - struct iser_conn *ib_conn = conn->dd_data; 338 338 + struct iser_conn *iser_conn = conn->dd_data; 339 339 + struct ib_conn *ib_conn = &iser_conn->ib_conn; 343 340 struct iscsi_session *session = conn->session; 344 341 345 342 iser_dbg("req op %x flags %x\n", req->opcode, req->flags); ··· 350 343 return 0; 351 344 352 345 /* 353 353 - * Check that there is one posted recv buffer (for the last login 354 354 - * response) and no posted send buffers left - they must have been 355 355 - * consumed during previous login phases. 346 346 + * Check that there is one posted recv buffer 347 347 + * (for the last login response). 356 348 */ 357 349 WARN_ON(ib_conn->post_recv_buf_count != 1); 358 358 - WARN_ON(atomic_read(&ib_conn->post_send_buf_count) != 0); 359 350 360 351 if (session->discovery_sess) { 361 352 iser_info("Discovery session, re-using login RX buffer\n"); 362 353 return 0; 363 354 } else 364 355 iser_info("Normal session, posting batch of RX %d buffers\n", 365 365 - ib_conn->min_posted_rx); 356 356 + iser_conn->min_posted_rx); 366 357 367 358 /* Initial post receive buffers */ 368 368 - if (iser_post_recvm(ib_conn, ib_conn->min_posted_rx)) 359 359 + if (iser_post_recvm(iser_conn, iser_conn->min_posted_rx)) 369 360 return -ENOMEM; 370 361 371 362 return 0; 363 363 + } 364 364 + 365 365 + static inline bool iser_signal_comp(int sig_count) 366 366 + { 367 367 + return ((sig_count % ISER_SIGNAL_CMD_COUNT) == 0); 372 368 } 373 369 374 370 /** ··· 380 370 int iser_send_command(struct iscsi_conn *conn, 381 371 struct iscsi_task *task) 382 372 { 383 383 - struct iser_conn *ib_conn = conn->dd_data; 373 373 + struct iser_conn *iser_conn = conn->dd_data; 384 374 struct iscsi_iser_task *iser_task = task->dd_data; 385 375 unsigned long edtl; 386 376 int err; ··· 388 378 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr; 389 379 struct scsi_cmnd *sc = task->sc; 390 380 struct iser_tx_desc *tx_desc = &iser_task->desc; 381 381 + static unsigned sig_count; 391 382 392 383 edtl = ntohl(hdr->data_length); 393 384 394 385 /* build the tx desc regd header and add it to the tx desc dto */ 395 386 tx_desc->type = ISCSI_TX_SCSI_COMMAND; 396 396 - iser_create_send_desc(ib_conn, tx_desc); 387 387 + iser_create_send_desc(iser_conn, tx_desc); 397 388 398 389 if (hdr->flags & ISCSI_FLAG_CMD_READ) { 399 390 data_buf = &iser_task->data[ISER_DIR_IN]; ··· 434 423 435 424 iser_task->status = ISER_TASK_STATUS_STARTED; 436 425 437 437 - err = iser_post_send(ib_conn, tx_desc); 426 426 + err = iser_post_send(&iser_conn->ib_conn, tx_desc, 427 427 + iser_signal_comp(++sig_count)); 438 428 if (!err) 439 429 return 0; 440 430 ··· 451 439 struct iscsi_task *task, 452 440 struct iscsi_data *hdr) 453 441 { 454 454 - struct iser_conn *ib_conn = conn->dd_data; 442 442 + struct iser_conn *iser_conn = conn->dd_data; 455 443 struct iscsi_iser_task *iser_task = task->dd_data; 456 444 struct iser_tx_desc *tx_desc = NULL; 457 445 struct iser_regd_buf *regd_buf; ··· 500 488 itt, buf_offset, data_seg_len); 501 489 502 490 503 503 - err = iser_post_send(ib_conn, tx_desc); 491 491 + err = iser_post_send(&iser_conn->ib_conn, tx_desc, true); 504 492 if (!err) 505 493 return 0; 506 494 ··· 513 501 int iser_send_control(struct iscsi_conn *conn, 514 502 struct iscsi_task *task) 515 503 { 516 516 - struct iser_conn *ib_conn = conn->dd_data; 504 504 + struct iser_conn *iser_conn = conn->dd_data; 517 505 struct iscsi_iser_task *iser_task = task->dd_data; 518 506 struct iser_tx_desc *mdesc = &iser_task->desc; 519 507 unsigned long data_seg_len; ··· 522 510 523 511 /* build the tx desc regd header and add it to the tx desc dto */ 524 512 mdesc->type = ISCSI_TX_CONTROL; 525 525 - iser_create_send_desc(ib_conn, mdesc); 513 513 + iser_create_send_desc(iser_conn, mdesc); 526 514 527 527 - device = ib_conn->device; 515 515 + device = iser_conn->ib_conn.device; 528 516 529 517 data_seg_len = ntoh24(task->hdr->dlength); 530 518 ··· 536 524 } 537 525 538 526 ib_dma_sync_single_for_cpu(device->ib_device, 539 539 - ib_conn->login_req_dma, task->data_count, 527 527 + iser_conn->login_req_dma, task->data_count, 540 528 DMA_TO_DEVICE); 541 529 542 542 - memcpy(ib_conn->login_req_buf, task->data, task->data_count); 530 530 + memcpy(iser_conn->login_req_buf, task->data, task->data_count); 543 531 544 532 ib_dma_sync_single_for_device(device->ib_device, 545 545 - ib_conn->login_req_dma, task->data_count, 533 533 + iser_conn->login_req_dma, task->data_count, 546 534 DMA_TO_DEVICE); 547 535 548 548 - tx_dsg->addr = ib_conn->login_req_dma; 536 536 + tx_dsg->addr = iser_conn->login_req_dma; 549 537 tx_dsg->length = task->data_count; 550 538 tx_dsg->lkey = device->mr->lkey; 551 539 mdesc->num_sge = 2; ··· 554 542 if (task == conn->login_task) { 555 543 iser_dbg("op %x dsl %lx, posting login rx buffer\n", 556 544 task->hdr->opcode, data_seg_len); 557 557 - err = iser_post_recvl(ib_conn); 545 545 + err = iser_post_recvl(iser_conn); 558 546 if (err) 559 547 goto send_control_error; 560 548 err = iser_post_rx_bufs(conn, task->hdr); ··· 562 550 goto send_control_error; 563 551 } 564 552 565 565 - err = iser_post_send(ib_conn, mdesc); 553 553 + err = iser_post_send(&iser_conn->ib_conn, mdesc, true); 566 554 if (!err) 567 555 return 0; 568 556 ··· 576 564 */ 577 565 void iser_rcv_completion(struct iser_rx_desc *rx_desc, 578 566 unsigned long rx_xfer_len, 579 579 - struct iser_conn *ib_conn) 567 567 + struct ib_conn *ib_conn) 580 568 { 569 569 + struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn, 570 570 + ib_conn); 581 571 struct iscsi_hdr *hdr; 582 572 u64 rx_dma; 583 573 int rx_buflen, outstanding, count, err; 584 574 585 575 /* differentiate between login to all other PDUs */ 586 586 - if ((char *)rx_desc == ib_conn->login_resp_buf) { 587 587 - rx_dma = ib_conn->login_resp_dma; 576 576 + if ((char *)rx_desc == iser_conn->login_resp_buf) { 577 577 + rx_dma = iser_conn->login_resp_dma; 588 578 rx_buflen = ISER_RX_LOGIN_SIZE; 589 579 } else { 590 580 rx_dma = rx_desc->dma_addr; ··· 594 580 } 595 581 596 582 ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma, 597 597 - rx_buflen, DMA_FROM_DEVICE); 583 583 + rx_buflen, DMA_FROM_DEVICE); 598 584 599 585 hdr = &rx_desc->iscsi_header; 600 586 601 587 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, 602 588 hdr->itt, (int)(rx_xfer_len - ISER_HEADERS_LEN)); 603 589 604 604 - iscsi_iser_recv(ib_conn->iscsi_conn, hdr, rx_desc->data, 590 590 + iscsi_iser_recv(iser_conn->iscsi_conn, hdr, rx_desc->data, 605 591 rx_xfer_len - ISER_HEADERS_LEN); 606 592 607 593 ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma, ··· 613 599 * for the posted rx bufs refcount to become zero handles everything */ 614 600 ib_conn->post_recv_buf_count--; 615 601 616 616 - if (rx_dma == ib_conn->login_resp_dma) 602 602 + if (rx_dma == iser_conn->login_resp_dma) 617 603 return; 618 604 619 605 outstanding = ib_conn->post_recv_buf_count; 620 620 - if (outstanding + ib_conn->min_posted_rx <= ib_conn->qp_max_recv_dtos) { 621 621 - count = min(ib_conn->qp_max_recv_dtos - outstanding, 622 622 - ib_conn->min_posted_rx); 623 623 - err = iser_post_recvm(ib_conn, count); 606 606 + if (outstanding + iser_conn->min_posted_rx <= iser_conn->qp_max_recv_dtos) { 607 607 + count = min(iser_conn->qp_max_recv_dtos - outstanding, 608 608 + iser_conn->min_posted_rx); 609 609 + err = iser_post_recvm(iser_conn, count); 624 610 if (err) 625 611 iser_err("posting %d rx bufs err %d\n", count, err); 626 612 } 627 613 } 628 614 629 615 void iser_snd_completion(struct iser_tx_desc *tx_desc, 630 630 - struct iser_conn *ib_conn) 616 616 + struct ib_conn *ib_conn) 631 617 { 632 618 struct iscsi_task *task; 633 619 struct iser_device *device = ib_conn->device; ··· 638 624 kmem_cache_free(ig.desc_cache, tx_desc); 639 625 tx_desc = NULL; 640 626 } 641 641 - 642 642 - atomic_dec(&ib_conn->post_send_buf_count); 643 627 644 628 if (tx_desc && tx_desc->type == ISCSI_TX_CONTROL) { 645 629 /* this arithmetic is legal by libiscsi dd_data allocation */ ··· 670 658 671 659 void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task) 672 660 { 673 673 - struct iser_device *device = iser_task->ib_conn->device; 661 661 + struct iser_device *device = iser_task->iser_conn->ib_conn.device; 674 662 int is_rdma_data_aligned = 1; 675 663 int is_rdma_prot_aligned = 1; 676 664 int prot_count = scsi_prot_sg_count(iser_task->sc);

+48 -51

drivers/infiniband/ulp/iser/iser_memory.c

reviewed

··· 49 49 struct iser_data_buf *data_copy, 50 50 enum iser_data_dir cmd_dir) 51 51 { 52 52 - struct ib_device *dev = iser_task->ib_conn->device->ib_device; 52 52 + struct ib_device *dev = iser_task->iser_conn->ib_conn.device->ib_device; 53 53 struct scatterlist *sgl = (struct scatterlist *)data->buf; 54 54 struct scatterlist *sg; 55 55 char *mem = NULL; ··· 116 116 struct ib_device *dev; 117 117 unsigned long cmd_data_len; 118 118 119 119 - dev = iser_task->ib_conn->device->ib_device; 119 119 + dev = iser_task->iser_conn->ib_conn.device->ib_device; 120 120 121 121 ib_dma_unmap_sg(dev, &data_copy->sg_single, 1, 122 122 (cmd_dir == ISER_DIR_OUT) ? ··· 322 322 struct ib_device *dev; 323 323 324 324 iser_task->dir[iser_dir] = 1; 325 325 - dev = iser_task->ib_conn->device->ib_device; 325 325 + dev = iser_task->iser_conn->ib_conn.device->ib_device; 326 326 327 327 data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir); 328 328 if (data->dma_nents == 0) { ··· 337 337 { 338 338 struct ib_device *dev; 339 339 340 340 - dev = iser_task->ib_conn->device->ib_device; 340 340 + dev = iser_task->iser_conn->ib_conn.device->ib_device; 341 341 ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE); 342 342 } 343 343 ··· 348 348 enum iser_data_dir cmd_dir, 349 349 int aligned_len) 350 350 { 351 351 - struct iscsi_conn *iscsi_conn = iser_task->ib_conn->iscsi_conn; 351 351 + struct iscsi_conn *iscsi_conn = iser_task->iser_conn->iscsi_conn; 352 352 353 353 iscsi_conn->fmr_unalign_cnt++; 354 354 iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n", ··· 377 377 int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task, 378 378 enum iser_data_dir cmd_dir) 379 379 { 380 380 - struct iser_conn *ib_conn = iser_task->ib_conn; 380 380 + struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 381 381 struct iser_device *device = ib_conn->device; 382 382 struct ib_device *ibdev = device->ib_device; 383 383 struct iser_data_buf *mem = &iser_task->data[cmd_dir]; ··· 432 432 ib_conn->fmr.page_vec->offset); 433 433 for (i = 0; i < ib_conn->fmr.page_vec->length; i++) 434 434 iser_err("page_vec[%d] = 0x%llx\n", i, 435 435 - (unsigned long long) ib_conn->fmr.page_vec->pages[i]); 435 435 + (unsigned long long)ib_conn->fmr.page_vec->pages[i]); 436 436 } 437 437 if (err) 438 438 return err; ··· 440 440 return 0; 441 441 } 442 442 443 443 - static inline enum ib_t10_dif_type 444 444 - scsi2ib_prot_type(unsigned char prot_type) 443 443 + static inline void 444 444 + iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs, 445 445 + struct ib_sig_domain *domain) 445 446 { 446 446 - switch (prot_type) { 447 447 - case SCSI_PROT_DIF_TYPE0: 448 448 - return IB_T10DIF_NONE; 449 449 - case SCSI_PROT_DIF_TYPE1: 450 450 - return IB_T10DIF_TYPE1; 451 451 - case SCSI_PROT_DIF_TYPE2: 452 452 - return IB_T10DIF_TYPE2; 453 453 - case SCSI_PROT_DIF_TYPE3: 454 454 - return IB_T10DIF_TYPE3; 455 455 - default: 456 456 - return IB_T10DIF_NONE; 457 457 - } 458 458 - } 459 459 - 447 447 + domain->sig_type = IB_SIG_TYPE_T10_DIF; 448 448 + domain->sig.dif.pi_interval = sc->device->sector_size; 449 449 + domain->sig.dif.ref_tag = scsi_get_lba(sc) & 0xffffffff; 450 450 + /* 451 451 + * At the moment we hard code those, but in the future 452 452 + * we will take them from sc. 453 453 + */ 454 454 + domain->sig.dif.apptag_check_mask = 0xffff; 455 455 + domain->sig.dif.app_escape = true; 456 456 + domain->sig.dif.ref_escape = true; 457 457 + if (scsi_get_prot_type(sc) == SCSI_PROT_DIF_TYPE1 || 458 458 + scsi_get_prot_type(sc) == SCSI_PROT_DIF_TYPE2) 459 459 + domain->sig.dif.ref_remap = true; 460 460 + }; 460 461 461 462 static int 462 463 iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) 463 464 { 464 464 - unsigned char scsi_ptype = scsi_get_prot_type(sc); 465 465 - 466 466 - sig_attrs->mem.sig_type = IB_SIG_TYPE_T10_DIF; 467 467 - sig_attrs->wire.sig_type = IB_SIG_TYPE_T10_DIF; 468 468 - sig_attrs->mem.sig.dif.pi_interval = sc->device->sector_size; 469 469 - sig_attrs->wire.sig.dif.pi_interval = sc->device->sector_size; 470 470 - 471 465 switch (scsi_get_prot_op(sc)) { 472 466 case SCSI_PROT_WRITE_INSERT: 473 467 case SCSI_PROT_READ_STRIP: 474 474 - sig_attrs->mem.sig.dif.type = IB_T10DIF_NONE; 475 475 - sig_attrs->wire.sig.dif.type = scsi2ib_prot_type(scsi_ptype); 468 468 + sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 469 469 + iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); 476 470 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 477 477 - sig_attrs->wire.sig.dif.ref_tag = scsi_get_lba(sc) & 478 478 - 0xffffffff; 479 471 break; 480 472 case SCSI_PROT_READ_INSERT: 481 473 case SCSI_PROT_WRITE_STRIP: 482 482 - sig_attrs->mem.sig.dif.type = scsi2ib_prot_type(scsi_ptype); 483 483 - sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC; 484 484 - sig_attrs->mem.sig.dif.ref_tag = scsi_get_lba(sc) & 485 485 - 0xffffffff; 486 486 - sig_attrs->wire.sig.dif.type = IB_T10DIF_NONE; 474 474 + sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 475 475 + iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); 476 476 + /* 477 477 + * At the moment we use this modparam to tell what is 478 478 + * the memory bg_type, in the future we will take it 479 479 + * from sc. 480 480 + */ 481 481 + sig_attrs->mem.sig.dif.bg_type = iser_pi_guard ? IB_T10DIF_CSUM : 482 482 + IB_T10DIF_CRC; 487 483 break; 488 484 case SCSI_PROT_READ_PASS: 489 485 case SCSI_PROT_WRITE_PASS: 490 490 - sig_attrs->mem.sig.dif.type = scsi2ib_prot_type(scsi_ptype); 491 491 - sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC; 492 492 - sig_attrs->mem.sig.dif.ref_tag = scsi_get_lba(sc) & 493 493 - 0xffffffff; 494 494 - sig_attrs->wire.sig.dif.type = scsi2ib_prot_type(scsi_ptype); 486 486 + iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); 495 487 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 496 496 - sig_attrs->wire.sig.dif.ref_tag = scsi_get_lba(sc) & 497 497 - 0xffffffff; 488 488 + iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); 489 489 + /* 490 490 + * At the moment we use this modparam to tell what is 491 491 + * the memory bg_type, in the future we will take it 492 492 + * from sc. 493 493 + */ 494 494 + sig_attrs->mem.sig.dif.bg_type = iser_pi_guard ? IB_T10DIF_CSUM : 495 495 + IB_T10DIF_CRC; 498 496 break; 499 497 default: 500 498 iser_err("Unsupported PI operation %d\n", 501 499 scsi_get_prot_op(sc)); 502 500 return -EINVAL; 503 501 } 502 502 + 504 503 return 0; 505 504 } 506 506 - 507 505 508 506 static int 509 507 iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) 510 508 { 511 509 switch (scsi_get_prot_type(sc)) { 512 510 case SCSI_PROT_DIF_TYPE0: 513 513 - *mask = 0x0; 514 511 break; 515 512 case SCSI_PROT_DIF_TYPE1: 516 513 case SCSI_PROT_DIF_TYPE2: ··· 530 533 struct fast_reg_descriptor *desc, struct ib_sge *data_sge, 531 534 struct ib_sge *prot_sge, struct ib_sge *sig_sge) 532 535 { 533 533 - struct iser_conn *ib_conn = iser_task->ib_conn; 536 536 + struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 534 537 struct iser_pi_context *pi_ctx = desc->pi_ctx; 535 538 struct ib_send_wr sig_wr, inv_wr; 536 539 struct ib_send_wr *bad_wr, *wr = NULL; ··· 606 609 struct ib_sge *sge) 607 610 { 608 611 struct fast_reg_descriptor *desc = regd_buf->reg.mem_h; 609 609 - struct iser_conn *ib_conn = iser_task->ib_conn; 612 612 + struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 610 613 struct iser_device *device = ib_conn->device; 611 614 struct ib_device *ibdev = device->ib_device; 612 615 struct ib_mr *mr; ··· 697 700 int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task, 698 701 enum iser_data_dir cmd_dir) 699 702 { 700 700 - struct iser_conn *ib_conn = iser_task->ib_conn; 703 703 + struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 701 704 struct iser_device *device = ib_conn->device; 702 705 struct ib_device *ibdev = device->ib_device; 703 706 struct iser_data_buf *mem = &iser_task->data[cmd_dir];

+364 -297

drivers/infiniband/ulp/iser/iser_verbs.c

reviewed

··· 39 39 #include "iscsi_iser.h" 40 40 41 41 #define ISCSI_ISER_MAX_CONN 8 42 42 - #define ISER_MAX_RX_CQ_LEN (ISER_QP_MAX_RECV_DTOS * ISCSI_ISER_MAX_CONN) 43 43 - #define ISER_MAX_TX_CQ_LEN (ISER_QP_MAX_REQ_DTOS * ISCSI_ISER_MAX_CONN) 42 42 + #define ISER_MAX_RX_LEN (ISER_QP_MAX_RECV_DTOS * ISCSI_ISER_MAX_CONN) 43 43 + #define ISER_MAX_TX_LEN (ISER_QP_MAX_REQ_DTOS * ISCSI_ISER_MAX_CONN) 44 44 + #define ISER_MAX_CQ_LEN (ISER_MAX_RX_LEN + ISER_MAX_TX_LEN + \ 45 45 + ISCSI_ISER_MAX_CONN) 46 46 + 47 47 + static int iser_cq_poll_limit = 512; 44 48 45 49 static void iser_cq_tasklet_fn(unsigned long data); 46 50 static void iser_cq_callback(struct ib_cq *cq, void *cq_context); ··· 75 71 */ 76 72 static int iser_create_device_ib_res(struct iser_device *device) 77 73 { 78 78 - struct iser_cq_desc *cq_desc; 79 74 struct ib_device_attr *dev_attr = &device->dev_attr; 80 75 int ret, i; 81 76 ··· 104 101 return -1; 105 102 } 106 103 107 107 - device->cqs_used = min(ISER_MAX_CQ, device->ib_device->num_comp_vectors); 104 104 + device->comps_used = min(ISER_MAX_CQ, 105 105 + device->ib_device->num_comp_vectors); 108 106 iser_info("using %d CQs, device %s supports %d vectors\n", 109 109 - device->cqs_used, device->ib_device->name, 107 107 + device->comps_used, device->ib_device->name, 110 108 device->ib_device->num_comp_vectors); 111 111 - 112 112 - device->cq_desc = kmalloc(sizeof(struct iser_cq_desc) * device->cqs_used, 113 113 - GFP_KERNEL); 114 114 - if (device->cq_desc == NULL) 115 115 - goto cq_desc_err; 116 116 - cq_desc = device->cq_desc; 117 109 118 110 device->pd = ib_alloc_pd(device->ib_device); 119 111 if (IS_ERR(device->pd)) 120 112 goto pd_err; 121 113 122 122 - for (i = 0; i < device->cqs_used; i++) { 123 123 - cq_desc[i].device = device; 124 124 - cq_desc[i].cq_index = i; 114 114 + for (i = 0; i < device->comps_used; i++) { 115 115 + struct iser_comp *comp = &device->comps[i]; 125 116 126 126 - device->rx_cq[i] = ib_create_cq(device->ib_device, 127 127 - iser_cq_callback, 128 128 - iser_cq_event_callback, 129 129 - (void *)&cq_desc[i], 130 130 - ISER_MAX_RX_CQ_LEN, i); 131 131 - if (IS_ERR(device->rx_cq[i])) { 132 132 - device->rx_cq[i] = NULL; 117 117 + comp->device = device; 118 118 + comp->cq = ib_create_cq(device->ib_device, 119 119 + iser_cq_callback, 120 120 + iser_cq_event_callback, 121 121 + (void *)comp, 122 122 + ISER_MAX_CQ_LEN, i); 123 123 + if (IS_ERR(comp->cq)) { 124 124 + comp->cq = NULL; 133 125 goto cq_err; 134 126 } 135 127 136 136 - device->tx_cq[i] = ib_create_cq(device->ib_device, 137 137 - NULL, iser_cq_event_callback, 138 138 - (void *)&cq_desc[i], 139 139 - ISER_MAX_TX_CQ_LEN, i); 140 140 - 141 141 - if (IS_ERR(device->tx_cq[i])) { 142 142 - device->tx_cq[i] = NULL; 143 143 - goto cq_err; 144 144 - } 145 145 - 146 146 - if (ib_req_notify_cq(device->rx_cq[i], IB_CQ_NEXT_COMP)) 128 128 + if (ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP)) 147 129 goto cq_err; 148 130 149 149 - tasklet_init(&device->cq_tasklet[i], 150 150 - iser_cq_tasklet_fn, 151 151 - (unsigned long)&cq_desc[i]); 131 131 + tasklet_init(&comp->tasklet, iser_cq_tasklet_fn, 132 132 + (unsigned long)comp); 152 133 } 153 134 154 135 device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE | ··· 151 164 handler_err: 152 165 ib_dereg_mr(device->mr); 153 166 dma_mr_err: 154 154 - for (i = 0; i < device->cqs_used; i++) 155 155 - tasklet_kill(&device->cq_tasklet[i]); 167 167 + for (i = 0; i < device->comps_used; i++) 168 168 + tasklet_kill(&device->comps[i].tasklet); 156 169 cq_err: 157 157 - for (i = 0; i < device->cqs_used; i++) { 158 158 - if (device->tx_cq[i]) 159 159 - ib_destroy_cq(device->tx_cq[i]); 160 160 - if (device->rx_cq[i]) 161 161 - ib_destroy_cq(device->rx_cq[i]); 170 170 + for (i = 0; i < device->comps_used; i++) { 171 171 + struct iser_comp *comp = &device->comps[i]; 172 172 + 173 173 + if (comp->cq) 174 174 + ib_destroy_cq(comp->cq); 162 175 } 163 176 ib_dealloc_pd(device->pd); 164 177 pd_err: 165 165 - kfree(device->cq_desc); 166 166 - cq_desc_err: 167 178 iser_err("failed to allocate an IB resource\n"); 168 179 return -1; 169 180 } ··· 175 190 int i; 176 191 BUG_ON(device->mr == NULL); 177 192 178 178 - for (i = 0; i < device->cqs_used; i++) { 179 179 - tasklet_kill(&device->cq_tasklet[i]); 180 180 - (void)ib_destroy_cq(device->tx_cq[i]); 181 181 - (void)ib_destroy_cq(device->rx_cq[i]); 182 182 - device->tx_cq[i] = NULL; 183 183 - device->rx_cq[i] = NULL; 193 193 + for (i = 0; i < device->comps_used; i++) { 194 194 + struct iser_comp *comp = &device->comps[i]; 195 195 + 196 196 + tasklet_kill(&comp->tasklet); 197 197 + ib_destroy_cq(comp->cq); 198 198 + comp->cq = NULL; 184 199 } 185 200 186 201 (void)ib_unregister_event_handler(&device->event_handler); 187 202 (void)ib_dereg_mr(device->mr); 188 203 (void)ib_dealloc_pd(device->pd); 189 189 - 190 190 - kfree(device->cq_desc); 191 204 192 205 device->mr = NULL; 193 206 device->pd = NULL; ··· 196 213 * 197 214 * returns 0 on success, or errno code on failure 198 215 */ 199 199 - int iser_create_fmr_pool(struct iser_conn *ib_conn, unsigned cmds_max) 216 216 + int iser_create_fmr_pool(struct ib_conn *ib_conn, unsigned cmds_max) 200 217 { 201 218 struct iser_device *device = ib_conn->device; 202 219 struct ib_fmr_pool_param params; ··· 246 263 /** 247 264 * iser_free_fmr_pool - releases the FMR pool and page vec 248 265 */ 249 249 - void iser_free_fmr_pool(struct iser_conn *ib_conn) 266 266 + void iser_free_fmr_pool(struct ib_conn *ib_conn) 250 267 { 251 268 iser_info("freeing conn %p fmr pool %p\n", 252 269 ib_conn, ib_conn->fmr.pool); ··· 350 367 * for fast registration work requests. 351 368 * returns 0 on success, or errno code on failure 352 369 */ 353 353 - int iser_create_fastreg_pool(struct iser_conn *ib_conn, unsigned cmds_max) 370 370 + int iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max) 354 371 { 355 355 - struct iser_device *device = ib_conn->device; 356 356 - struct fast_reg_descriptor *desc; 372 372 + struct iser_device *device = ib_conn->device; 373 373 + struct fast_reg_descriptor *desc; 357 374 int i, ret; 358 375 359 376 INIT_LIST_HEAD(&ib_conn->fastreg.pool); ··· 389 406 /** 390 407 * iser_free_fastreg_pool - releases the pool of fast_reg descriptors 391 408 */ 392 392 - void iser_free_fastreg_pool(struct iser_conn *ib_conn) 409 409 + void iser_free_fastreg_pool(struct ib_conn *ib_conn) 393 410 { 394 411 struct fast_reg_descriptor *desc, *tmp; 395 412 int i = 0; ··· 423 440 * 424 441 * returns 0 on success, -1 on failure 425 442 */ 426 426 - static int iser_create_ib_conn_res(struct iser_conn *ib_conn) 443 443 + static int iser_create_ib_conn_res(struct ib_conn *ib_conn) 427 444 { 428 445 struct iser_device *device; 429 446 struct ib_qp_init_attr init_attr; ··· 438 455 439 456 mutex_lock(&ig.connlist_mutex); 440 457 /* select the CQ with the minimal number of usages */ 441 441 - for (index = 0; index < device->cqs_used; index++) 442 442 - if (device->cq_active_qps[index] < 443 443 - device->cq_active_qps[min_index]) 458 458 + for (index = 0; index < device->comps_used; index++) { 459 459 + if (device->comps[index].active_qps < 460 460 + device->comps[min_index].active_qps) 444 461 min_index = index; 445 445 - device->cq_active_qps[min_index]++; 462 462 + } 463 463 + ib_conn->comp = &device->comps[min_index]; 464 464 + ib_conn->comp->active_qps++; 446 465 mutex_unlock(&ig.connlist_mutex); 447 466 iser_info("cq index %d used for ib_conn %p\n", min_index, ib_conn); 448 467 449 468 init_attr.event_handler = iser_qp_event_callback; 450 469 init_attr.qp_context = (void *)ib_conn; 451 451 - init_attr.send_cq = device->tx_cq[min_index]; 452 452 - init_attr.recv_cq = device->rx_cq[min_index]; 470 470 + init_attr.send_cq = ib_conn->comp->cq; 471 471 + init_attr.recv_cq = ib_conn->comp->cq; 453 472 init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS; 454 473 init_attr.cap.max_send_sge = 2; 455 474 init_attr.cap.max_recv_sge = 1; 456 475 init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 457 476 init_attr.qp_type = IB_QPT_RC; 458 477 if (ib_conn->pi_support) { 459 459 - init_attr.cap.max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS; 478 478 + init_attr.cap.max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1; 460 479 init_attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 461 480 } else { 462 462 - init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS; 481 481 + init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS + 1; 463 482 } 464 483 465 484 ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr); ··· 477 492 out_err: 478 493 iser_err("unable to alloc mem or create resource, err %d\n", ret); 479 494 return ret; 480 480 - } 481 481 - 482 482 - /** 483 483 - * releases the QP object 484 484 - */ 485 485 - static void iser_free_ib_conn_res(struct iser_conn *ib_conn) 486 486 - { 487 487 - int cq_index; 488 488 - BUG_ON(ib_conn == NULL); 489 489 - 490 490 - iser_info("freeing conn %p cma_id %p qp %p\n", 491 491 - ib_conn, ib_conn->cma_id, 492 492 - ib_conn->qp); 493 493 - 494 494 - /* qp is created only once both addr & route are resolved */ 495 495 - 496 496 - if (ib_conn->qp != NULL) { 497 497 - cq_index = ((struct iser_cq_desc *)ib_conn->qp->recv_cq->cq_context)->cq_index; 498 498 - ib_conn->device->cq_active_qps[cq_index]--; 499 499 - 500 500 - rdma_destroy_qp(ib_conn->cma_id); 501 501 - } 502 502 - 503 503 - ib_conn->qp = NULL; 504 495 } 505 496 506 497 /** ··· 533 572 /** 534 573 * Called with state mutex held 535 574 **/ 536 536 - static int iser_conn_state_comp_exch(struct iser_conn *ib_conn, 537 537 - enum iser_ib_conn_state comp, 538 538 - enum iser_ib_conn_state exch) 575 575 + static int iser_conn_state_comp_exch(struct iser_conn *iser_conn, 576 576 + enum iser_conn_state comp, 577 577 + enum iser_conn_state exch) 539 578 { 540 579 int ret; 541 580 542 542 - if ((ret = (ib_conn->state == comp))) 543 543 - ib_conn->state = exch; 581 581 + ret = (iser_conn->state == comp); 582 582 + if (ret) 583 583 + iser_conn->state = exch; 584 584 + 544 585 return ret; 545 586 } 546 587 547 588 void iser_release_work(struct work_struct *work) 548 589 { 549 549 - struct iser_conn *ib_conn; 550 550 - int rc; 590 590 + struct iser_conn *iser_conn; 551 591 552 552 - ib_conn = container_of(work, struct iser_conn, release_work); 592 592 + iser_conn = container_of(work, struct iser_conn, release_work); 553 593 554 554 - /* wait for .conn_stop callback */ 555 555 - rc = wait_for_completion_timeout(&ib_conn->stop_completion, 30 * HZ); 556 556 - WARN_ON(rc == 0); 594 594 + /* Wait for conn_stop to complete */ 595 595 + wait_for_completion(&iser_conn->stop_completion); 596 596 + /* Wait for IB resouces cleanup to complete */ 597 597 + wait_for_completion(&iser_conn->ib_completion); 557 598 558 558 - /* wait for the qp`s post send and post receive buffers to empty */ 559 559 - rc = wait_for_completion_timeout(&ib_conn->flush_completion, 30 * HZ); 560 560 - WARN_ON(rc == 0); 599 599 + mutex_lock(&iser_conn->state_mutex); 600 600 + iser_conn->state = ISER_CONN_DOWN; 601 601 + mutex_unlock(&iser_conn->state_mutex); 561 602 562 562 - ib_conn->state = ISER_CONN_DOWN; 603 603 + iser_conn_release(iser_conn); 604 604 + } 563 605 564 564 - mutex_lock(&ib_conn->state_mutex); 565 565 - ib_conn->state = ISER_CONN_DOWN; 566 566 - mutex_unlock(&ib_conn->state_mutex); 606 606 + /** 607 607 + * iser_free_ib_conn_res - release IB related resources 608 608 + * @iser_conn: iser connection struct 609 609 + * @destroy_device: indicator if we need to try to release 610 610 + * the iser device (only iscsi shutdown and DEVICE_REMOVAL 611 611 + * will use this. 612 612 + * 613 613 + * This routine is called with the iser state mutex held 614 614 + * so the cm_id removal is out of here. It is Safe to 615 615 + * be invoked multiple times. 616 616 + */ 617 617 + static void iser_free_ib_conn_res(struct iser_conn *iser_conn, 618 618 + bool destroy_device) 619 619 + { 620 620 + struct ib_conn *ib_conn = &iser_conn->ib_conn; 621 621 + struct iser_device *device = ib_conn->device; 567 622 568 568 - iser_conn_release(ib_conn); 623 623 + iser_info("freeing conn %p cma_id %p qp %p\n", 624 624 + iser_conn, ib_conn->cma_id, ib_conn->qp); 625 625 + 626 626 + iser_free_rx_descriptors(iser_conn); 627 627 + 628 628 + if (ib_conn->qp != NULL) { 629 629 + ib_conn->comp->active_qps--; 630 630 + rdma_destroy_qp(ib_conn->cma_id); 631 631 + ib_conn->qp = NULL; 632 632 + } 633 633 + 634 634 + if (destroy_device && device != NULL) { 635 635 + iser_device_try_release(device); 636 636 + ib_conn->device = NULL; 637 637 + } 569 638 } 570 639 571 640 /** 572 641 * Frees all conn objects and deallocs conn descriptor 573 642 */ 574 574 - void iser_conn_release(struct iser_conn *ib_conn) 643 643 + void iser_conn_release(struct iser_conn *iser_conn) 575 644 { 576 576 - struct iser_device *device = ib_conn->device; 645 645 + struct ib_conn *ib_conn = &iser_conn->ib_conn; 577 646 578 647 mutex_lock(&ig.connlist_mutex); 579 579 - list_del(&ib_conn->conn_list); 648 648 + list_del(&iser_conn->conn_list); 580 649 mutex_unlock(&ig.connlist_mutex); 581 650 582 582 - mutex_lock(&ib_conn->state_mutex); 583 583 - BUG_ON(ib_conn->state != ISER_CONN_DOWN); 651 651 + mutex_lock(&iser_conn->state_mutex); 652 652 + if (iser_conn->state != ISER_CONN_DOWN) 653 653 + iser_warn("iser conn %p state %d, expected state down.\n", 654 654 + iser_conn, iser_conn->state); 655 655 + /* 656 656 + * In case we never got to bind stage, we still need to 657 657 + * release IB resources (which is safe to call more than once). 658 658 + */ 659 659 + iser_free_ib_conn_res(iser_conn, true); 660 660 + mutex_unlock(&iser_conn->state_mutex); 584 661 585 585 - iser_free_rx_descriptors(ib_conn); 586 586 - iser_free_ib_conn_res(ib_conn); 587 587 - ib_conn->device = NULL; 588 588 - /* on EVENT_ADDR_ERROR there's no device yet for this conn */ 589 589 - if (device != NULL) 590 590 - iser_device_try_release(device); 591 591 - mutex_unlock(&ib_conn->state_mutex); 592 592 - 593 593 - /* if cma handler context, the caller actually destroy the id */ 594 662 if (ib_conn->cma_id != NULL) { 595 663 rdma_destroy_id(ib_conn->cma_id); 596 664 ib_conn->cma_id = NULL; 597 665 } 598 598 - kfree(ib_conn); 666 666 + 667 667 + kfree(iser_conn); 599 668 } 600 669 601 670 /** 602 671 * triggers start of the disconnect procedures and wait for them to be done 672 672 + * Called with state mutex held 603 673 */ 604 604 - void iser_conn_terminate(struct iser_conn *ib_conn) 674 674 + int iser_conn_terminate(struct iser_conn *iser_conn) 605 675 { 676 676 + struct ib_conn *ib_conn = &iser_conn->ib_conn; 677 677 + struct ib_send_wr *bad_wr; 606 678 int err = 0; 607 679 608 608 - /* change the ib conn state only if the conn is UP, however always call 609 609 - * rdma_disconnect since this is the only way to cause the CMA to change 610 610 - * the QP state to ERROR 611 611 - */ 680 680 + /* terminate the iser conn only if the conn state is UP */ 681 681 + if (!iser_conn_state_comp_exch(iser_conn, ISER_CONN_UP, 682 682 + ISER_CONN_TERMINATING)) 683 683 + return 0; 612 684 613 613 - iser_conn_state_comp_exch(ib_conn, ISER_CONN_UP, ISER_CONN_TERMINATING); 614 614 - err = rdma_disconnect(ib_conn->cma_id); 615 615 - if (err) 616 616 - iser_err("Failed to disconnect, conn: 0x%p err %d\n", 617 617 - ib_conn,err); 685 685 + iser_info("iser_conn %p state %d\n", iser_conn, iser_conn->state); 686 686 + 687 687 + /* suspend queuing of new iscsi commands */ 688 688 + if (iser_conn->iscsi_conn) 689 689 + iscsi_suspend_queue(iser_conn->iscsi_conn); 690 690 + 691 691 + /* 692 692 + * In case we didn't already clean up the cma_id (peer initiated 693 693 + * a disconnection), we need to Cause the CMA to change the QP 694 694 + * state to ERROR. 695 695 + */ 696 696 + if (ib_conn->cma_id) { 697 697 + err = rdma_disconnect(ib_conn->cma_id); 698 698 + if (err) 699 699 + iser_err("Failed to disconnect, conn: 0x%p err %d\n", 700 700 + iser_conn, err); 701 701 + 702 702 + /* post an indication that all flush errors were consumed */ 703 703 + err = ib_post_send(ib_conn->qp, &ib_conn->beacon, &bad_wr); 704 704 + if (err) 705 705 + iser_err("conn %p failed to post beacon", ib_conn); 706 706 + 707 707 + wait_for_completion(&ib_conn->flush_comp); 708 708 + } 709 709 + 710 710 + return 1; 618 711 } 619 712 620 713 /** ··· 676 661 **/ 677 662 static void iser_connect_error(struct rdma_cm_id *cma_id) 678 663 { 679 679 - struct iser_conn *ib_conn; 664 664 + struct iser_conn *iser_conn; 680 665 681 681 - ib_conn = (struct iser_conn *)cma_id->context; 682 682 - ib_conn->state = ISER_CONN_DOWN; 666 666 + iser_conn = (struct iser_conn *)cma_id->context; 667 667 + iser_conn->state = ISER_CONN_DOWN; 683 668 } 684 669 685 670 /** ··· 688 673 static void iser_addr_handler(struct rdma_cm_id *cma_id) 689 674 { 690 675 struct iser_device *device; 691 691 - struct iser_conn *ib_conn; 676 676 + struct iser_conn *iser_conn; 677 677 + struct ib_conn *ib_conn; 692 678 int ret; 693 679 694 694 - ib_conn = (struct iser_conn *)cma_id->context; 695 695 - if (ib_conn->state != ISER_CONN_PENDING) 680 680 + iser_conn = (struct iser_conn *)cma_id->context; 681 681 + if (iser_conn->state != ISER_CONN_PENDING) 696 682 /* bailout */ 697 683 return; 698 684 685 685 + ib_conn = &iser_conn->ib_conn; 699 686 device = iser_device_find_by_ib_device(cma_id); 700 687 if (!device) { 701 688 iser_err("device lookup/creation failed\n"); ··· 736 719 struct rdma_conn_param conn_param; 737 720 int ret; 738 721 struct iser_cm_hdr req_hdr; 739 739 - struct iser_conn *ib_conn = (struct iser_conn *)cma_id->context; 722 722 + struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context; 723 723 + struct ib_conn *ib_conn = &iser_conn->ib_conn; 740 724 struct iser_device *device = ib_conn->device; 741 725 742 742 - if (ib_conn->state != ISER_CONN_PENDING) 726 726 + if (iser_conn->state != ISER_CONN_PENDING) 743 727 /* bailout */ 744 728 return; 745 729 746 746 - ret = iser_create_ib_conn_res((struct iser_conn *)cma_id->context); 730 730 + ret = iser_create_ib_conn_res(ib_conn); 747 731 if (ret) 748 732 goto failure; 749 733 ··· 773 755 774 756 static void iser_connected_handler(struct rdma_cm_id *cma_id) 775 757 { 776 776 - struct iser_conn *ib_conn; 758 758 + struct iser_conn *iser_conn; 777 759 struct ib_qp_attr attr; 778 760 struct ib_qp_init_attr init_attr; 779 761 780 780 - ib_conn = (struct iser_conn *)cma_id->context; 781 781 - if (ib_conn->state != ISER_CONN_PENDING) 762 762 + iser_conn = (struct iser_conn *)cma_id->context; 763 763 + if (iser_conn->state != ISER_CONN_PENDING) 782 764 /* bailout */ 783 765 return; 784 766 785 767 (void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr); 786 768 iser_info("remote qpn:%x my qpn:%x\n", attr.dest_qp_num, cma_id->qp->qp_num); 787 769 788 788 - ib_conn->state = ISER_CONN_UP; 789 789 - complete(&ib_conn->up_completion); 770 770 + iser_conn->state = ISER_CONN_UP; 771 771 + complete(&iser_conn->up_completion); 790 772 } 791 773 792 774 static void iser_disconnected_handler(struct rdma_cm_id *cma_id) 793 775 { 794 794 - struct iser_conn *ib_conn; 776 776 + struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context; 795 777 796 796 - ib_conn = (struct iser_conn *)cma_id->context; 797 797 - 798 798 - /* getting here when the state is UP means that the conn is being * 799 799 - * terminated asynchronously from the iSCSI layer's perspective. */ 800 800 - if (iser_conn_state_comp_exch(ib_conn, ISER_CONN_UP, 801 801 - ISER_CONN_TERMINATING)){ 802 802 - if (ib_conn->iscsi_conn) 803 803 - iscsi_conn_failure(ib_conn->iscsi_conn, ISCSI_ERR_CONN_FAILED); 778 778 + if (iser_conn_terminate(iser_conn)) { 779 779 + if (iser_conn->iscsi_conn) 780 780 + iscsi_conn_failure(iser_conn->iscsi_conn, 781 781 + ISCSI_ERR_CONN_FAILED); 804 782 else 805 783 iser_err("iscsi_iser connection isn't bound\n"); 806 784 } 807 807 - 808 808 - /* Complete the termination process if no posts are pending. This code 809 809 - * block also exists in iser_handle_comp_error(), but it is needed here 810 810 - * for cases of no flushes at all, e.g. discovery over rdma. 811 811 - */ 812 812 - if (ib_conn->post_recv_buf_count == 0 && 813 813 - (atomic_read(&ib_conn->post_send_buf_count) == 0)) { 814 814 - complete(&ib_conn->flush_completion); 815 815 - } 816 785 } 786 786 + 787 787 + static void iser_cleanup_handler(struct rdma_cm_id *cma_id, 788 788 + bool destroy_device) 789 789 + { 790 790 + struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context; 791 791 + 792 792 + /* 793 793 + * We are not guaranteed that we visited disconnected_handler 794 794 + * by now, call it here to be safe that we handle CM drep 795 795 + * and flush errors. 796 796 + */ 797 797 + iser_disconnected_handler(cma_id); 798 798 + iser_free_ib_conn_res(iser_conn, destroy_device); 799 799 + complete(&iser_conn->ib_completion); 800 800 + }; 817 801 818 802 static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 819 803 { 820 820 - struct iser_conn *ib_conn; 804 804 + struct iser_conn *iser_conn; 805 805 + int ret = 0; 821 806 822 822 - ib_conn = (struct iser_conn *)cma_id->context; 807 807 + iser_conn = (struct iser_conn *)cma_id->context; 823 808 iser_info("event %d status %d conn %p id %p\n", 824 809 event->event, event->status, cma_id->context, cma_id); 825 810 826 826 - mutex_lock(&ib_conn->state_mutex); 811 811 + mutex_lock(&iser_conn->state_mutex); 827 812 switch (event->event) { 828 813 case RDMA_CM_EVENT_ADDR_RESOLVED: 829 814 iser_addr_handler(cma_id); ··· 845 824 iser_connect_error(cma_id); 846 825 break; 847 826 case RDMA_CM_EVENT_DISCONNECTED: 848 848 - case RDMA_CM_EVENT_DEVICE_REMOVAL: 849 827 case RDMA_CM_EVENT_ADDR_CHANGE: 850 850 - case RDMA_CM_EVENT_TIMEWAIT_EXIT: 851 828 iser_disconnected_handler(cma_id); 829 829 + break; 830 830 + case RDMA_CM_EVENT_DEVICE_REMOVAL: 831 831 + /* 832 832 + * we *must* destroy the device as we cannot rely 833 833 + * on iscsid to be around to initiate error handling. 834 834 + * also implicitly destroy the cma_id. 835 835 + */ 836 836 + iser_cleanup_handler(cma_id, true); 837 837 + iser_conn->ib_conn.cma_id = NULL; 838 838 + ret = 1; 839 839 + break; 840 840 + case RDMA_CM_EVENT_TIMEWAIT_EXIT: 841 841 + iser_cleanup_handler(cma_id, false); 852 842 break; 853 843 default: 854 844 iser_err("Unexpected RDMA CM event (%d)\n", event->event); 855 845 break; 856 846 } 857 857 - mutex_unlock(&ib_conn->state_mutex); 858 858 - return 0; 847 847 + mutex_unlock(&iser_conn->state_mutex); 848 848 + 849 849 + return ret; 859 850 } 860 851 861 861 - void iser_conn_init(struct iser_conn *ib_conn) 852 852 + void iser_conn_init(struct iser_conn *iser_conn) 862 853 { 863 863 - ib_conn->state = ISER_CONN_INIT; 864 864 - ib_conn->post_recv_buf_count = 0; 865 865 - atomic_set(&ib_conn->post_send_buf_count, 0); 866 866 - init_completion(&ib_conn->stop_completion); 867 867 - init_completion(&ib_conn->flush_completion); 868 868 - init_completion(&ib_conn->up_completion); 869 869 - INIT_LIST_HEAD(&ib_conn->conn_list); 870 870 - spin_lock_init(&ib_conn->lock); 871 871 - mutex_init(&ib_conn->state_mutex); 854 854 + iser_conn->state = ISER_CONN_INIT; 855 855 + iser_conn->ib_conn.post_recv_buf_count = 0; 856 856 + init_completion(&iser_conn->ib_conn.flush_comp); 857 857 + init_completion(&iser_conn->stop_completion); 858 858 + init_completion(&iser_conn->ib_completion); 859 859 + init_completion(&iser_conn->up_completion); 860 860 + INIT_LIST_HEAD(&iser_conn->conn_list); 861 861 + spin_lock_init(&iser_conn->ib_conn.lock); 862 862 + mutex_init(&iser_conn->state_mutex); 872 863 } 873 864 874 865 /** 875 866 * starts the process of connecting to the target 876 867 * sleeps until the connection is established or rejected 877 868 */ 878 878 - int iser_connect(struct iser_conn *ib_conn, 869 869 + int iser_connect(struct iser_conn *iser_conn, 879 870 struct sockaddr *src_addr, 880 871 struct sockaddr *dst_addr, 881 872 int non_blocking) 882 873 { 874 874 + struct ib_conn *ib_conn = &iser_conn->ib_conn; 883 875 int err = 0; 884 876 885 885 - mutex_lock(&ib_conn->state_mutex); 877 877 + mutex_lock(&iser_conn->state_mutex); 886 878 887 887 - sprintf(ib_conn->name, "%pISp", dst_addr); 879 879 + sprintf(iser_conn->name, "%pISp", dst_addr); 888 880 889 889 - iser_info("connecting to: %s\n", ib_conn->name); 881 881 + iser_info("connecting to: %s\n", iser_conn->name); 890 882 891 883 /* the device is known only --after-- address resolution */ 892 884 ib_conn->device = NULL; 893 885 894 894 - ib_conn->state = ISER_CONN_PENDING; 886 886 + iser_conn->state = ISER_CONN_PENDING; 887 887 + 888 888 + ib_conn->beacon.wr_id = ISER_BEACON_WRID; 889 889 + ib_conn->beacon.opcode = IB_WR_SEND; 895 890 896 891 ib_conn->cma_id = rdma_create_id(iser_cma_handler, 897 897 - (void *)ib_conn, 898 898 - RDMA_PS_TCP, IB_QPT_RC); 892 892 + (void *)iser_conn, 893 893 + RDMA_PS_TCP, IB_QPT_RC); 899 894 if (IS_ERR(ib_conn->cma_id)) { 900 895 err = PTR_ERR(ib_conn->cma_id); 901 896 iser_err("rdma_create_id failed: %d\n", err); ··· 925 888 } 926 889 927 890 if (!non_blocking) { 928 928 - wait_for_completion_interruptible(&ib_conn->up_completion); 891 891 + wait_for_completion_interruptible(&iser_conn->up_completion); 929 892 930 930 - if (ib_conn->state != ISER_CONN_UP) { 893 893 + if (iser_conn->state != ISER_CONN_UP) { 931 894 err = -EIO; 932 895 goto connect_failure; 933 896 } 934 897 } 935 935 - mutex_unlock(&ib_conn->state_mutex); 898 898 + mutex_unlock(&iser_conn->state_mutex); 936 899 937 900 mutex_lock(&ig.connlist_mutex); 938 938 - list_add(&ib_conn->conn_list, &ig.connlist); 901 901 + list_add(&iser_conn->conn_list, &ig.connlist); 939 902 mutex_unlock(&ig.connlist_mutex); 940 903 return 0; 941 904 942 905 id_failure: 943 906 ib_conn->cma_id = NULL; 944 907 addr_failure: 945 945 - ib_conn->state = ISER_CONN_DOWN; 908 908 + iser_conn->state = ISER_CONN_DOWN; 946 909 connect_failure: 947 947 - mutex_unlock(&ib_conn->state_mutex); 948 948 - iser_conn_release(ib_conn); 910 910 + mutex_unlock(&iser_conn->state_mutex); 911 911 + iser_conn_release(iser_conn); 949 912 return err; 950 913 } 951 914 ··· 954 917 * 955 918 * returns: 0 on success, errno code on failure 956 919 */ 957 957 - int iser_reg_page_vec(struct iser_conn *ib_conn, 920 920 + int iser_reg_page_vec(struct ib_conn *ib_conn, 958 921 struct iser_page_vec *page_vec, 959 922 struct iser_mem_reg *mem_reg) 960 923 { ··· 1024 987 enum iser_data_dir cmd_dir) 1025 988 { 1026 989 struct iser_mem_reg *reg = &iser_task->rdma_regd[cmd_dir].reg; 1027 1027 - struct iser_conn *ib_conn = iser_task->ib_conn; 990 990 + struct iser_conn *iser_conn = iser_task->iser_conn; 991 991 + struct ib_conn *ib_conn = &iser_conn->ib_conn; 1028 992 struct fast_reg_descriptor *desc = reg->mem_h; 1029 993 1030 994 if (!reg->is_mr) ··· 1038 1000 spin_unlock_bh(&ib_conn->lock); 1039 1001 } 1040 1002 1041 1041 - int iser_post_recvl(struct iser_conn *ib_conn) 1003 1003 + int iser_post_recvl(struct iser_conn *iser_conn) 1042 1004 { 1043 1005 struct ib_recv_wr rx_wr, *rx_wr_failed; 1006 1006 + struct ib_conn *ib_conn = &iser_conn->ib_conn; 1044 1007 struct ib_sge sge; 1045 1008 int ib_ret; 1046 1009 1047 1047 - sge.addr = ib_conn->login_resp_dma; 1010 1010 + sge.addr = iser_conn->login_resp_dma; 1048 1011 sge.length = ISER_RX_LOGIN_SIZE; 1049 1012 sge.lkey = ib_conn->device->mr->lkey; 1050 1013 1051 1051 - rx_wr.wr_id = (unsigned long)ib_conn->login_resp_buf; 1014 1014 + rx_wr.wr_id = (unsigned long)iser_conn->login_resp_buf; 1052 1015 rx_wr.sg_list = &sge; 1053 1016 rx_wr.num_sge = 1; 1054 1017 rx_wr.next = NULL; ··· 1063 1024 return ib_ret; 1064 1025 } 1065 1026 1066 1066 - int iser_post_recvm(struct iser_conn *ib_conn, int count) 1027 1027 + int iser_post_recvm(struct iser_conn *iser_conn, int count) 1067 1028 { 1068 1029 struct ib_recv_wr *rx_wr, *rx_wr_failed; 1069 1030 int i, ib_ret; 1070 1070 - unsigned int my_rx_head = ib_conn->rx_desc_head; 1031 1031 + struct ib_conn *ib_conn = &iser_conn->ib_conn; 1032 1032 + unsigned int my_rx_head = iser_conn->rx_desc_head; 1071 1033 struct iser_rx_desc *rx_desc; 1072 1034 1073 1035 for (rx_wr = ib_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 1074 1074 - rx_desc = &ib_conn->rx_descs[my_rx_head]; 1036 1036 + rx_desc = &iser_conn->rx_descs[my_rx_head]; 1075 1037 rx_wr->wr_id = (unsigned long)rx_desc; 1076 1038 rx_wr->sg_list = &rx_desc->rx_sg; 1077 1039 rx_wr->num_sge = 1; 1078 1040 rx_wr->next = rx_wr + 1; 1079 1079 - my_rx_head = (my_rx_head + 1) & ib_conn->qp_max_recv_dtos_mask; 1041 1041 + my_rx_head = (my_rx_head + 1) & iser_conn->qp_max_recv_dtos_mask; 1080 1042 } 1081 1043 1082 1044 rx_wr--; ··· 1089 1049 iser_err("ib_post_recv failed ret=%d\n", ib_ret); 1090 1050 ib_conn->post_recv_buf_count -= count; 1091 1051 } else 1092 1092 - ib_conn->rx_desc_head = my_rx_head; 1052 1052 + iser_conn->rx_desc_head = my_rx_head; 1093 1053 return ib_ret; 1094 1054 } 1095 1055 ··· 1099 1059 * 1100 1060 * returns 0 on success, -1 on failure 1101 1061 */ 1102 1102 - int iser_post_send(struct iser_conn *ib_conn, struct iser_tx_desc *tx_desc) 1062 1062 + int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, 1063 1063 + bool signal) 1103 1064 { 1104 1065 int ib_ret; 1105 1066 struct ib_send_wr send_wr, *send_wr_failed; 1106 1067 1107 1068 ib_dma_sync_single_for_device(ib_conn->device->ib_device, 1108 1108 - tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); 1069 1069 + tx_desc->dma_addr, ISER_HEADERS_LEN, 1070 1070 + DMA_TO_DEVICE); 1109 1071 1110 1072 send_wr.next = NULL; 1111 1073 send_wr.wr_id = (unsigned long)tx_desc; 1112 1074 send_wr.sg_list = tx_desc->tx_sg; 1113 1075 send_wr.num_sge = tx_desc->num_sge; 1114 1076 send_wr.opcode = IB_WR_SEND; 1115 1115 - send_wr.send_flags = IB_SEND_SIGNALED; 1116 1116 - 1117 1117 - atomic_inc(&ib_conn->post_send_buf_count); 1077 1077 + send_wr.send_flags = signal ? IB_SEND_SIGNALED : 0; 1118 1078 1119 1079 ib_ret = ib_post_send(ib_conn->qp, &send_wr, &send_wr_failed); 1120 1120 - if (ib_ret) { 1080 1080 + if (ib_ret) 1121 1081 iser_err("ib_post_send failed, ret:%d\n", ib_ret); 1122 1122 - atomic_dec(&ib_conn->post_send_buf_count); 1123 1123 - } 1082 1082 + 1124 1083 return ib_ret; 1125 1084 } 1126 1085 1127 1127 - static void iser_handle_comp_error(struct iser_tx_desc *desc, 1128 1128 - struct iser_conn *ib_conn) 1086 1086 + /** 1087 1087 + * is_iser_tx_desc - Indicate if the completion wr_id 1088 1088 + * is a TX descriptor or not. 1089 1089 + * @iser_conn: iser connection 1090 1090 + * @wr_id: completion WR identifier 1091 1091 + * 1092 1092 + * Since we cannot rely on wc opcode in FLUSH errors 1093 1093 + * we must work around it by checking if the wr_id address 1094 1094 + * falls in the iser connection rx_descs buffer. If so 1095 1095 + * it is an RX descriptor, otherwize it is a TX. 1096 1096 + */ 1097 1097 + static inline bool 1098 1098 + is_iser_tx_desc(struct iser_conn *iser_conn, void *wr_id) 1129 1099 { 1130 1130 - if (desc && desc->type == ISCSI_TX_DATAOUT) 1131 1131 - kmem_cache_free(ig.desc_cache, desc); 1100 1100 + void *start = iser_conn->rx_descs; 1101 1101 + int len = iser_conn->num_rx_descs * sizeof(*iser_conn->rx_descs); 1132 1102 1133 1133 - if (ib_conn->post_recv_buf_count == 0 && 1134 1134 - atomic_read(&ib_conn->post_send_buf_count) == 0) { 1135 1135 - /** 1136 1136 - * getting here when the state is UP means that the conn is 1137 1137 - * being terminated asynchronously from the iSCSI layer's 1138 1138 - * perspective. It is safe to peek at the connection state 1139 1139 - * since iscsi_conn_failure is allowed to be called twice. 1140 1140 - **/ 1141 1141 - if (ib_conn->state == ISER_CONN_UP) 1142 1142 - iscsi_conn_failure(ib_conn->iscsi_conn, 1103 1103 + if (wr_id >= start && wr_id < start + len) 1104 1104 + return false; 1105 1105 + 1106 1106 + return true; 1107 1107 + } 1108 1108 + 1109 1109 + /** 1110 1110 + * iser_handle_comp_error() - Handle error completion 1111 1111 + * @ib_conn: connection RDMA resources 1112 1112 + * @wc: work completion 1113 1113 + * 1114 1114 + * Notes: We may handle a FLUSH error completion and in this case 1115 1115 + * we only cleanup in case TX type was DATAOUT. For non-FLUSH 1116 1116 + * error completion we should also notify iscsi layer that 1117 1117 + * connection is failed (in case we passed bind stage). 1118 1118 + */ 1119 1119 + static void 1120 1120 + iser_handle_comp_error(struct ib_conn *ib_conn, 1121 1121 + struct ib_wc *wc) 1122 1122 + { 1123 1123 + struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn, 1124 1124 + ib_conn); 1125 1125 + 1126 1126 + if (wc->status != IB_WC_WR_FLUSH_ERR) 1127 1127 + if (iser_conn->iscsi_conn) 1128 1128 + iscsi_conn_failure(iser_conn->iscsi_conn, 1143 1129 ISCSI_ERR_CONN_FAILED); 1144 1130 1145 1145 - /* no more non completed posts to the QP, complete the 1146 1146 - * termination process w.o worrying on disconnect event */ 1147 1147 - complete(&ib_conn->flush_completion); 1131 1131 + if (is_iser_tx_desc(iser_conn, (void *)wc->wr_id)) { 1132 1132 + struct iser_tx_desc *desc = (struct iser_tx_desc *)wc->wr_id; 1133 1133 + 1134 1134 + if (desc->type == ISCSI_TX_DATAOUT) 1135 1135 + kmem_cache_free(ig.desc_cache, desc); 1136 1136 + } else { 1137 1137 + ib_conn->post_recv_buf_count--; 1148 1138 } 1149 1139 } 1150 1140 1151 1151 - static int iser_drain_tx_cq(struct iser_device *device, int cq_index) 1141 1141 + /** 1142 1142 + * iser_handle_wc - handle a single work completion 1143 1143 + * @wc: work completion 1144 1144 + * 1145 1145 + * Soft-IRQ context, work completion can be either 1146 1146 + * SEND or RECV, and can turn out successful or 1147 1147 + * with error (or flush error). 1148 1148 + */ 1149 1149 + static void iser_handle_wc(struct ib_wc *wc) 1152 1150 { 1153 1153 - struct ib_cq *cq = device->tx_cq[cq_index]; 1154 1154 - struct ib_wc wc; 1151 1151 + struct ib_conn *ib_conn; 1155 1152 struct iser_tx_desc *tx_desc; 1156 1156 - struct iser_conn *ib_conn; 1157 1157 - int completed_tx = 0; 1153 1153 + struct iser_rx_desc *rx_desc; 1158 1154 1159 1159 - while (ib_poll_cq(cq, 1, &wc) == 1) { 1160 1160 - tx_desc = (struct iser_tx_desc *) (unsigned long) wc.wr_id; 1161 1161 - ib_conn = wc.qp->qp_context; 1162 1162 - if (wc.status == IB_WC_SUCCESS) { 1163 1163 - if (wc.opcode == IB_WC_SEND) 1164 1164 - iser_snd_completion(tx_desc, ib_conn); 1165 1165 - else 1166 1166 - iser_err("expected opcode %d got %d\n", 1167 1167 - IB_WC_SEND, wc.opcode); 1155 1155 + ib_conn = wc->qp->qp_context; 1156 1156 + if (wc->status == IB_WC_SUCCESS) { 1157 1157 + if (wc->opcode == IB_WC_RECV) { 1158 1158 + rx_desc = (struct iser_rx_desc *)wc->wr_id; 1159 1159 + iser_rcv_completion(rx_desc, wc->byte_len, 1160 1160 + ib_conn); 1161 1161 + } else 1162 1162 + if (wc->opcode == IB_WC_SEND) { 1163 1163 + tx_desc = (struct iser_tx_desc *)wc->wr_id; 1164 1164 + iser_snd_completion(tx_desc, ib_conn); 1168 1165 } else { 1169 1169 - iser_err("tx id %llx status %d vend_err %x\n", 1170 1170 - wc.wr_id, wc.status, wc.vendor_err); 1171 1171 - if (wc.wr_id != ISER_FASTREG_LI_WRID) { 1172 1172 - atomic_dec(&ib_conn->post_send_buf_count); 1173 1173 - iser_handle_comp_error(tx_desc, ib_conn); 1174 1174 - } 1166 1166 + iser_err("Unknown wc opcode %d\n", wc->opcode); 1175 1167 } 1176 1176 - completed_tx++; 1168 1168 + } else { 1169 1169 + if (wc->status != IB_WC_WR_FLUSH_ERR) 1170 1170 + iser_err("wr id %llx status %d vend_err %x\n", 1171 1171 + wc->wr_id, wc->status, wc->vendor_err); 1172 1172 + else 1173 1173 + iser_dbg("flush error: wr id %llx\n", wc->wr_id); 1174 1174 + 1175 1175 + if (wc->wr_id != ISER_FASTREG_LI_WRID && 1176 1176 + wc->wr_id != ISER_BEACON_WRID) 1177 1177 + iser_handle_comp_error(ib_conn, wc); 1178 1178 + 1179 1179 + /* complete in case all flush errors were consumed */ 1180 1180 + if (wc->wr_id == ISER_BEACON_WRID) 1181 1181 + complete(&ib_conn->flush_comp); 1177 1182 } 1178 1178 - return completed_tx; 1179 1183 } 1180 1184 1181 1181 - 1185 1185 + /** 1186 1186 + * iser_cq_tasklet_fn - iSER completion polling loop 1187 1187 + * @data: iSER completion context 1188 1188 + * 1189 1189 + * Soft-IRQ context, polling connection CQ until 1190 1190 + * either CQ was empty or we exausted polling budget 1191 1191 + */ 1182 1192 static void iser_cq_tasklet_fn(unsigned long data) 1183 1193 { 1184 1184 - struct iser_cq_desc *cq_desc = (struct iser_cq_desc *)data; 1185 1185 - struct iser_device *device = cq_desc->device; 1186 1186 - int cq_index = cq_desc->cq_index; 1187 1187 - struct ib_cq *cq = device->rx_cq[cq_index]; 1188 1188 - struct ib_wc wc; 1189 1189 - struct iser_rx_desc *desc; 1190 1190 - unsigned long xfer_len; 1191 1191 - struct iser_conn *ib_conn; 1192 1192 - int completed_tx, completed_rx = 0; 1194 1194 + struct iser_comp *comp = (struct iser_comp *)data; 1195 1195 + struct ib_cq *cq = comp->cq; 1196 1196 + struct ib_wc *const wcs = comp->wcs; 1197 1197 + int i, n, completed = 0; 1193 1198 1194 1194 - /* First do tx drain, so in a case where we have rx flushes and a successful 1195 1195 - * tx completion we will still go through completion error handling. 1196 1196 - */ 1197 1197 - completed_tx = iser_drain_tx_cq(device, cq_index); 1199 1199 + while ((n = ib_poll_cq(cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 1200 1200 + for (i = 0; i < n; i++) 1201 1201 + iser_handle_wc(&wcs[i]); 1198 1202 1199 1199 - while (ib_poll_cq(cq, 1, &wc) == 1) { 1200 1200 - desc = (struct iser_rx_desc *) (unsigned long) wc.wr_id; 1201 1201 - BUG_ON(desc == NULL); 1202 1202 - ib_conn = wc.qp->qp_context; 1203 1203 - if (wc.status == IB_WC_SUCCESS) { 1204 1204 - if (wc.opcode == IB_WC_RECV) { 1205 1205 - xfer_len = (unsigned long)wc.byte_len; 1206 1206 - iser_rcv_completion(desc, xfer_len, ib_conn); 1207 1207 - } else 1208 1208 - iser_err("expected opcode %d got %d\n", 1209 1209 - IB_WC_RECV, wc.opcode); 1210 1210 - } else { 1211 1211 - if (wc.status != IB_WC_WR_FLUSH_ERR) 1212 1212 - iser_err("rx id %llx status %d vend_err %x\n", 1213 1213 - wc.wr_id, wc.status, wc.vendor_err); 1214 1214 - ib_conn->post_recv_buf_count--; 1215 1215 - iser_handle_comp_error(NULL, ib_conn); 1216 1216 - } 1217 1217 - completed_rx++; 1218 1218 - if (!(completed_rx & 63)) 1219 1219 - completed_tx += iser_drain_tx_cq(device, cq_index); 1203 1203 + completed += n; 1204 1204 + if (completed >= iser_cq_poll_limit) 1205 1205 + break; 1220 1206 } 1221 1221 - /* #warning "it is assumed here that arming CQ only once its empty" * 1222 1222 - * " would not cause interrupts to be missed" */ 1207 1207 + 1208 1208 + /* 1209 1209 + * It is assumed here that arming CQ only once its empty 1210 1210 + * would not cause interrupts to be missed. 1211 1211 + */ 1223 1212 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 1224 1213 1225 1225 - iser_dbg("got %d rx %d tx completions\n", completed_rx, completed_tx); 1214 1214 + iser_dbg("got %d completions\n", completed); 1226 1215 } 1227 1216 1228 1217 static void iser_cq_callback(struct ib_cq *cq, void *cq_context) 1229 1218 { 1230 1230 - struct iser_cq_desc *cq_desc = (struct iser_cq_desc *)cq_context; 1231 1231 - struct iser_device *device = cq_desc->device; 1232 1232 - int cq_index = cq_desc->cq_index; 1219 1219 + struct iser_comp *comp = cq_context; 1233 1220 1234 1234 - tasklet_schedule(&device->cq_tasklet[cq_index]); 1221 1221 + tasklet_schedule(&comp->tasklet); 1235 1222 } 1236 1223 1237 1224 u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,

+25 -38

drivers/infiniband/ulp/isert/ib_isert.c

reviewed

··· 2609 2609 return ret; 2610 2610 } 2611 2611 2612 2612 - static inline enum ib_t10_dif_type 2613 2613 - se2ib_prot_type(enum target_prot_type prot_type) 2612 2612 + static inline void 2613 2613 + isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 2614 2614 + struct ib_sig_domain *domain) 2614 2615 { 2615 2615 - switch (prot_type) { 2616 2616 - case TARGET_DIF_TYPE0_PROT: 2617 2617 - return IB_T10DIF_NONE; 2618 2618 - case TARGET_DIF_TYPE1_PROT: 2619 2619 - return IB_T10DIF_TYPE1; 2620 2620 - case TARGET_DIF_TYPE2_PROT: 2621 2621 - return IB_T10DIF_TYPE2; 2622 2622 - case TARGET_DIF_TYPE3_PROT: 2623 2623 - return IB_T10DIF_TYPE3; 2624 2624 - default: 2625 2625 - return IB_T10DIF_NONE; 2626 2626 - } 2627 2627 - } 2616 2616 + domain->sig_type = IB_SIG_TYPE_T10_DIF; 2617 2617 + domain->sig.dif.bg_type = IB_T10DIF_CRC; 2618 2618 + domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2619 2619 + domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2620 2620 + /* 2621 2621 + * At the moment we hard code those, but if in the future 2622 2622 + * the target core would like to use it, we will take it 2623 2623 + * from se_cmd. 2624 2624 + */ 2625 2625 + domain->sig.dif.apptag_check_mask = 0xffff; 2626 2626 + domain->sig.dif.app_escape = true; 2627 2627 + domain->sig.dif.ref_escape = true; 2628 2628 + if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2629 2629 + se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2630 2630 + domain->sig.dif.ref_remap = true; 2631 2631 + }; 2628 2632 2629 2633 static int 2630 2634 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2631 2635 { 2632 2632 - enum ib_t10_dif_type ib_prot_type = se2ib_prot_type(se_cmd->prot_type); 2633 2633 - 2634 2634 - sig_attrs->mem.sig_type = IB_SIG_TYPE_T10_DIF; 2635 2635 - sig_attrs->wire.sig_type = IB_SIG_TYPE_T10_DIF; 2636 2636 - sig_attrs->mem.sig.dif.pi_interval = 2637 2637 - se_cmd->se_dev->dev_attrib.block_size; 2638 2638 - sig_attrs->wire.sig.dif.pi_interval = 2639 2639 - se_cmd->se_dev->dev_attrib.block_size; 2640 2640 - 2641 2636 switch (se_cmd->prot_op) { 2642 2637 case TARGET_PROT_DIN_INSERT: 2643 2638 case TARGET_PROT_DOUT_STRIP: 2644 2644 - sig_attrs->mem.sig.dif.type = IB_T10DIF_NONE; 2645 2645 - sig_attrs->wire.sig.dif.type = ib_prot_type; 2646 2646 - sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 2647 2647 - sig_attrs->wire.sig.dif.ref_tag = se_cmd->reftag_seed; 2639 2639 + sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2640 2640 + isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2648 2641 break; 2649 2642 case TARGET_PROT_DOUT_INSERT: 2650 2643 case TARGET_PROT_DIN_STRIP: 2651 2651 - sig_attrs->mem.sig.dif.type = ib_prot_type; 2652 2652 - sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC; 2653 2653 - sig_attrs->mem.sig.dif.ref_tag = se_cmd->reftag_seed; 2654 2654 - sig_attrs->wire.sig.dif.type = IB_T10DIF_NONE; 2644 2644 + sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2645 2645 + isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2655 2646 break; 2656 2647 case TARGET_PROT_DIN_PASS: 2657 2648 case TARGET_PROT_DOUT_PASS: 2658 2658 - sig_attrs->mem.sig.dif.type = ib_prot_type; 2659 2659 - sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC; 2660 2660 - sig_attrs->mem.sig.dif.ref_tag = se_cmd->reftag_seed; 2661 2661 - sig_attrs->wire.sig.dif.type = ib_prot_type; 2662 2662 - sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 2663 2663 - sig_attrs->wire.sig.dif.ref_tag = se_cmd->reftag_seed; 2649 2649 + isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2650 2650 + isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2664 2651 break; 2665 2652 default: 2666 2653 pr_err("Unsupported PI operation %d\n", se_cmd->prot_op);

+27 -8

include/linux/mlx5/qp.h

reviewed

··· 40 40 #define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 5) 41 41 #define MLX5_DIF_SIZE 8 42 42 #define MLX5_STRIDE_BLOCK_OP 0x400 43 43 + #define MLX5_CPY_GRD_MASK 0xc0 44 44 + #define MLX5_CPY_APP_MASK 0x30 45 45 + #define MLX5_CPY_REF_MASK 0x0f 46 46 + #define MLX5_BSF_INC_REFTAG (1 << 6) 47 47 + #define MLX5_BSF_INL_VALID (1 << 15) 48 48 + #define MLX5_BSF_REFRESH_DIF (1 << 14) 49 49 + #define MLX5_BSF_REPEAT_BLOCK (1 << 7) 50 50 + #define MLX5_BSF_APPTAG_ESCAPE 0x1 51 51 + #define MLX5_BSF_APPREF_ESCAPE 0x2 43 52 44 53 enum mlx5_qp_optpar { 45 54 MLX5_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0, ··· 296 287 __be32 byte_count; 297 288 }; 298 289 290 290 + enum mlx5_sig_type { 291 291 + MLX5_DIF_CRC = 0x1, 292 292 + MLX5_DIF_IPCS = 0x2, 293 293 + }; 294 294 + 295 295 + struct mlx5_bsf_inl { 296 296 + __be16 vld_refresh; 297 297 + __be16 dif_apptag; 298 298 + __be32 dif_reftag; 299 299 + u8 sig_type; 300 300 + u8 rp_inv_seed; 301 301 + u8 rsvd[3]; 302 302 + u8 dif_inc_ref_guard_check; 303 303 + __be16 dif_app_bitmask_check; 304 304 + }; 305 305 + 299 306 struct mlx5_bsf { 300 307 struct mlx5_bsf_basic { 301 308 u8 bsf_size_sbs; ··· 335 310 __be32 w_tfs_psv; 336 311 __be32 m_tfs_psv; 337 312 } ext; 338 338 - struct mlx5_bsf_inl { 339 339 - __be32 w_inl_vld; 340 340 - __be32 w_rsvd; 341 341 - __be64 w_block_format; 342 342 - __be32 m_inl_vld; 343 343 - __be32 m_rsvd; 344 344 - __be64 m_block_format; 345 345 - } inl; 313 313 + struct mlx5_bsf_inl w_inl; 314 314 + struct mlx5_bsf_inl m_inl; 346 315 }; 347 316 348 317 struct mlx5_klm {

+14 -18

include/rdma/ib_verbs.h

reviewed

··· 491 491 u32 flags; 492 492 }; 493 493 494 494 - enum ib_signature_type { 495 495 - IB_SIG_TYPE_T10_DIF, 496 496 - }; 497 497 - 498 494 /** 499 499 - * T10-DIF Signature types 500 500 - * T10-DIF types are defined by SCSI 501 501 - * specifications. 495 495 + * Signature types 496 496 + * IB_SIG_TYPE_NONE: Unprotected. 497 497 + * IB_SIG_TYPE_T10_DIF: Type T10-DIF 502 498 */ 503 503 - enum ib_t10_dif_type { 504 504 - IB_T10DIF_NONE, 505 505 - IB_T10DIF_TYPE1, 506 506 - IB_T10DIF_TYPE2, 507 507 - IB_T10DIF_TYPE3 499 499 + enum ib_signature_type { 500 500 + IB_SIG_TYPE_NONE, 501 501 + IB_SIG_TYPE_T10_DIF, 508 502 }; 509 503 510 504 /** ··· 514 520 /** 515 521 * struct ib_t10_dif_domain - Parameters specific for T10-DIF 516 522 * domain. 517 517 - * @type: T10-DIF type (0|1|2|3) 518 523 * @bg_type: T10-DIF block guard type (CRC|CSUM) 519 524 * @pi_interval: protection information interval. 520 525 * @bg: seed of guard computation. 521 526 * @app_tag: application tag of guard block 522 527 * @ref_tag: initial guard block reference tag. 523 523 - * @type3_inc_reftag: T10-DIF type 3 does not state 524 524 - * about the reference tag, it is the user 525 525 - * choice to increment it or not. 528 528 + * @ref_remap: Indicate wethear the reftag increments each block 529 529 + * @app_escape: Indicate to skip block check if apptag=0xffff 530 530 + * @ref_escape: Indicate to skip block check if reftag=0xffffffff 531 531 + * @apptag_check_mask: check bitmask of application tag. 526 532 */ 527 533 struct ib_t10_dif_domain { 528 528 - enum ib_t10_dif_type type; 529 534 enum ib_t10_dif_bg_type bg_type; 530 535 u16 pi_interval; 531 536 u16 bg; 532 537 u16 app_tag; 533 538 u32 ref_tag; 534 534 - bool type3_inc_reftag; 539 539 + bool ref_remap; 540 540 + bool app_escape; 541 541 + bool ref_escape; 542 542 + u16 apptag_check_mask; 535 543 }; 536 544 537 545 /**