tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/roland/infiniband

* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/roland/infiniband:
IB/ehca: Support small QP queues
IB/ehca: Make internal_create/destroy_qp() static
IB/ehca: Move ehca2ib_return_code() out of line
IB/ehca: Generate async event when SRQ limit reached
IB/ehca: Support large page MRs
IB/mlx4: Fix error path in create_qp_common()
mlx4_core: Change command token on timeout
IB/mthca: Change command token on timeout
IB/ipath: Remove ipath_layer dead code
IB/mlx4: Fix leaks in __mlx4_ib_modify_qp

Linus Torvalds a5b4860b e9ed7e72

+806 -719
+35 -15
drivers/infiniband/hw/ehca/ehca_classes.h
··· 43 43 #ifndef __EHCA_CLASSES_H__ 44 44 #define __EHCA_CLASSES_H__ 45 45 46 - 47 46 struct ehca_module; 48 47 struct ehca_qp; 49 48 struct ehca_cq; ··· 99 100 struct ehca_sma_attr saved_attr; 100 101 }; 101 102 103 + #define HCA_CAP_MR_PGSIZE_4K 1 104 + #define HCA_CAP_MR_PGSIZE_64K 2 105 + #define HCA_CAP_MR_PGSIZE_1M 4 106 + #define HCA_CAP_MR_PGSIZE_16M 8 107 + 102 108 struct ehca_shca { 103 109 struct ib_device ib_device; 104 110 struct ibmebus_dev *ibmebus_dev; ··· 119 115 struct h_galpas galpas; 120 116 struct mutex modify_mutex; 121 117 u64 hca_cap; 118 + /* MR pgsize: bit 0-3 means 4K, 64K, 1M, 16M respectively */ 119 + u32 hca_cap_mr_pgsize; 122 120 int max_mtu; 123 121 }; 124 122 ··· 128 122 struct ib_pd ib_pd; 129 123 struct ipz_pd fw_pd; 130 124 u32 ownpid; 125 + /* small queue mgmt */ 126 + struct mutex lock; 127 + struct list_head free[2]; 128 + struct list_head full[2]; 131 129 }; 132 130 133 131 enum ehca_ext_qp_type { ··· 216 206 enum ehca_mr_flag flags; 217 207 u32 num_kpages; /* number of kernel pages */ 218 208 u32 num_hwpages; /* number of hw pages to form MR */ 209 + u64 hwpage_size; /* hw page size used for this MR */ 219 210 int acl; /* ACL (stored here for usage in reregister) */ 220 211 u64 *start; /* virtual start address (stored here for */ 221 212 /* usage in reregister) */ ··· 251 240 enum ehca_mr_pgi_type type; 252 241 u64 num_kpages; 253 242 u64 kpage_cnt; 243 + u64 hwpage_size; /* hw page size used for this MR */ 254 244 u64 num_hwpages; /* number of hw pages */ 255 245 u64 hwpage_cnt; /* counter for hw pages */ 256 246 u64 next_hwpage; /* next hw page in buffer/chunk/listelem */ ··· 310 298 void ehca_cleanup_av_cache(void); 311 299 int ehca_init_mrmw_cache(void); 312 300 void ehca_cleanup_mrmw_cache(void); 301 + int ehca_init_small_qp_cache(void); 302 + void ehca_cleanup_small_qp_cache(void); 313 303 314 304 extern rwlock_t ehca_qp_idr_lock; 315 305 extern rwlock_t ehca_cq_idr_lock; ··· 329 315 u32 queue_length; /* queue length allocated in bytes */ 330 316 u32 pagesize; 331 317 u32 toggle_state; 332 - u32 dummy; /* padding for 8 byte alignment */ 318 + u32 offset; /* save offset within a page for small_qp */ 333 319 }; 334 320 335 321 struct ehca_create_cq_resp { ··· 371 357 LLQP_COMP_MASK = 0x60, 372 358 }; 373 359 360 + struct ehca_alloc_queue_parms { 361 + /* input parameters */ 362 + int max_wr; 363 + int max_sge; 364 + int page_size; 365 + int is_small; 366 + 367 + /* output parameters */ 368 + u16 act_nr_wqes; 369 + u8 act_nr_sges; 370 + u32 queue_size; /* bytes for small queues, pages otherwise */ 371 + }; 372 + 374 373 struct ehca_alloc_qp_parms { 375 - /* input parameters */ 374 + struct ehca_alloc_queue_parms squeue; 375 + struct ehca_alloc_queue_parms rqueue; 376 + 377 + /* input parameters */ 376 378 enum ehca_service_type servicetype; 379 + int qp_storage; 377 380 int sigtype; 378 381 enum ehca_ext_qp_type ext_type; 379 382 enum ehca_ll_comp_flags ll_comp_flags; 380 - 381 - int max_send_wr, max_recv_wr; 382 - int max_send_sge, max_recv_sge; 383 383 int ud_av_l_key_ctl; 384 384 385 385 u32 token; ··· 403 375 404 376 u32 srq_qpn, srq_token, srq_limit; 405 377 406 - /* output parameters */ 378 + /* output parameters */ 407 379 u32 real_qp_num; 408 380 struct ipz_qp_handle qp_handle; 409 381 struct h_galpas galpas; 410 - 411 - u16 act_nr_send_wqes; 412 - u16 act_nr_recv_wqes; 413 - u8 act_nr_recv_sges; 414 - u8 act_nr_send_sges; 415 - 416 - u32 nr_rq_pages; 417 - u32 nr_sq_pages; 418 382 }; 419 383 420 384 int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp);
+4 -4
drivers/infiniband/hw/ehca/ehca_cq.c
··· 190 190 goto create_cq_exit2; 191 191 } 192 192 193 - ipz_rc = ipz_queue_ctor(&my_cq->ipz_queue, param.act_pages, 194 - EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0); 193 + ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages, 194 + EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0); 195 195 if (!ipz_rc) { 196 196 ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%x device=%p", 197 197 ipz_rc, device); ··· 285 285 return cq; 286 286 287 287 create_cq_exit4: 288 - ipz_queue_dtor(&my_cq->ipz_queue); 288 + ipz_queue_dtor(NULL, &my_cq->ipz_queue); 289 289 290 290 create_cq_exit3: 291 291 h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1); ··· 359 359 "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num); 360 360 return ehca2ib_return_code(h_ret); 361 361 } 362 - ipz_queue_dtor(&my_cq->ipz_queue); 362 + ipz_queue_dtor(NULL, &my_cq->ipz_queue); 363 363 kmem_cache_free(cq_cache, my_cq); 364 364 365 365 return 0;
+4 -4
drivers/infiniband/hw/ehca/ehca_eq.c
··· 86 86 return -EINVAL; 87 87 } 88 88 89 - ret = ipz_queue_ctor(&eq->ipz_queue, nr_pages, 90 - EHCA_PAGESIZE, sizeof(struct ehca_eqe), 0); 89 + ret = ipz_queue_ctor(NULL, &eq->ipz_queue, nr_pages, 90 + EHCA_PAGESIZE, sizeof(struct ehca_eqe), 0, 0); 91 91 if (!ret) { 92 92 ehca_err(ib_dev, "Can't allocate EQ pages eq=%p", eq); 93 93 goto create_eq_exit1; ··· 145 145 return 0; 146 146 147 147 create_eq_exit2: 148 - ipz_queue_dtor(&eq->ipz_queue); 148 + ipz_queue_dtor(NULL, &eq->ipz_queue); 149 149 150 150 create_eq_exit1: 151 151 hipz_h_destroy_eq(shca->ipz_hca_handle, eq); ··· 181 181 ehca_err(&shca->ib_device, "Can't free EQ resources."); 182 182 return -EINVAL; 183 183 } 184 - ipz_queue_dtor(&eq->ipz_queue); 184 + ipz_queue_dtor(NULL, &eq->ipz_queue); 185 185 186 186 return 0; 187 187 }
+28 -14
drivers/infiniband/hw/ehca/ehca_irq.c
··· 175 175 176 176 } 177 177 178 - static void qp_event_callback(struct ehca_shca *shca, 179 - u64 eqe, 180 - enum ib_event_type event_type) 178 + static void qp_event_callback(struct ehca_shca *shca, u64 eqe, 179 + enum ib_event_type event_type, int fatal) 181 180 { 182 181 struct ib_event event; 183 182 struct ehca_qp *qp; ··· 190 191 if (!qp) 191 192 return; 192 193 193 - ehca_error_data(shca, qp, qp->ipz_qp_handle.handle); 194 + if (fatal) 195 + ehca_error_data(shca, qp, qp->ipz_qp_handle.handle); 194 196 195 - if (!qp->ib_qp.event_handler) 196 - return; 197 + event.device = &shca->ib_device; 197 198 198 - event.device = &shca->ib_device; 199 - event.event = event_type; 200 - event.element.qp = &qp->ib_qp; 199 + if (qp->ext_type == EQPT_SRQ) { 200 + if (!qp->ib_srq.event_handler) 201 + return; 201 202 202 - qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); 203 + event.event = fatal ? IB_EVENT_SRQ_ERR : event_type; 204 + event.element.srq = &qp->ib_srq; 205 + qp->ib_srq.event_handler(&event, qp->ib_srq.srq_context); 206 + } else { 207 + if (!qp->ib_qp.event_handler) 208 + return; 209 + 210 + event.event = event_type; 211 + event.element.qp = &qp->ib_qp; 212 + qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); 213 + } 203 214 204 215 return; 205 216 } ··· 243 234 244 235 switch (identifier) { 245 236 case 0x02: /* path migrated */ 246 - qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG); 237 + qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG, 0); 247 238 break; 248 239 case 0x03: /* communication established */ 249 - qp_event_callback(shca, eqe, IB_EVENT_COMM_EST); 240 + qp_event_callback(shca, eqe, IB_EVENT_COMM_EST, 0); 250 241 break; 251 242 case 0x04: /* send queue drained */ 252 - qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED); 243 + qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED, 0); 253 244 break; 254 245 case 0x05: /* QP error */ 255 246 case 0x06: /* QP error */ 256 - qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL); 247 + qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL, 1); 257 248 break; 258 249 case 0x07: /* CQ error */ 259 250 case 0x08: /* CQ error */ ··· 287 278 ehca_err(&shca->ib_device, "Interface trace stopped."); 288 279 break; 289 280 case 0x14: /* first error capture info available */ 281 + ehca_info(&shca->ib_device, "First error capture available"); 282 + break; 283 + case 0x15: /* SRQ limit reached */ 284 + qp_event_callback(shca, eqe, IB_EVENT_SRQ_LIMIT_REACHED, 0); 285 + break; 290 286 default: 291 287 ehca_err(&shca->ib_device, "Unknown identifier: %x on %s.", 292 288 identifier, shca->ib_device.name);
+46 -3
drivers/infiniband/hw/ehca/ehca_main.c
··· 63 63 int ehca_poll_all_eqs = 1; 64 64 int ehca_static_rate = -1; 65 65 int ehca_scaling_code = 0; 66 + int ehca_mr_largepage = 0; 66 67 67 68 module_param_named(open_aqp1, ehca_open_aqp1, int, 0); 68 69 module_param_named(debug_level, ehca_debug_level, int, 0); ··· 73 72 module_param_named(port_act_time, ehca_port_act_time, int, 0); 74 73 module_param_named(poll_all_eqs, ehca_poll_all_eqs, int, 0); 75 74 module_param_named(static_rate, ehca_static_rate, int, 0); 76 - module_param_named(scaling_code, ehca_scaling_code, int, 0); 75 + module_param_named(scaling_code, ehca_scaling_code, int, 0); 76 + module_param_named(mr_largepage, ehca_mr_largepage, int, 0); 77 77 78 78 MODULE_PARM_DESC(open_aqp1, 79 79 "AQP1 on startup (0: no (default), 1: yes)"); ··· 97 95 "set permanent static rate (default: disabled)"); 98 96 MODULE_PARM_DESC(scaling_code, 99 97 "set scaling code (0: disabled/default, 1: enabled)"); 98 + MODULE_PARM_DESC(mr_largepage, 99 + "use large page for MR (0: use PAGE_SIZE (default), " 100 + "1: use large page depending on MR size"); 100 101 101 102 DEFINE_RWLOCK(ehca_qp_idr_lock); 102 103 DEFINE_RWLOCK(ehca_cq_idr_lock); ··· 129 124 130 125 } 131 126 #endif 127 + 128 + int ehca2ib_return_code(u64 ehca_rc) 129 + { 130 + switch (ehca_rc) { 131 + case H_SUCCESS: 132 + return 0; 133 + case H_RESOURCE: /* Resource in use */ 134 + case H_BUSY: 135 + return -EBUSY; 136 + case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */ 137 + case H_CONSTRAINED: /* resource constraint */ 138 + case H_NO_MEM: 139 + return -ENOMEM; 140 + default: 141 + return -EINVAL; 142 + } 143 + } 132 144 133 145 static int ehca_create_slab_caches(void) 134 146 { ··· 181 159 goto create_slab_caches5; 182 160 } 183 161 162 + ret = ehca_init_small_qp_cache(); 163 + if (ret) { 164 + ehca_gen_err("Cannot create small queue SLAB cache."); 165 + goto create_slab_caches6; 166 + } 167 + 184 168 #ifdef CONFIG_PPC_64K_PAGES 185 169 ctblk_cache = kmem_cache_create("ehca_cache_ctblk", 186 170 EHCA_PAGESIZE, H_CB_ALIGNMENT, ··· 194 166 NULL); 195 167 if (!ctblk_cache) { 196 168 ehca_gen_err("Cannot create ctblk SLAB cache."); 197 - ehca_cleanup_mrmw_cache(); 198 - goto create_slab_caches5; 169 + ehca_cleanup_small_qp_cache(); 170 + goto create_slab_caches6; 199 171 } 200 172 #endif 201 173 return 0; 174 + 175 + create_slab_caches6: 176 + ehca_cleanup_mrmw_cache(); 202 177 203 178 create_slab_caches5: 204 179 ehca_cleanup_av_cache(); ··· 220 189 221 190 static void ehca_destroy_slab_caches(void) 222 191 { 192 + ehca_cleanup_small_qp_cache(); 223 193 ehca_cleanup_mrmw_cache(); 224 194 ehca_cleanup_av_cache(); 225 195 ehca_cleanup_qp_cache(); ··· 326 294 for (i = 0; i < ARRAY_SIZE(hca_cap_descr); i++) 327 295 if (EHCA_BMASK_GET(hca_cap_descr[i].mask, shca->hca_cap)) 328 296 ehca_gen_dbg(" %s", hca_cap_descr[i].descr); 297 + 298 + shca->hca_cap_mr_pgsize = rblock->memory_page_size_supported; 329 299 330 300 port = (struct hipz_query_port *)rblock; 331 301 h_ret = hipz_h_query_port(shca->ipz_hca_handle, 1, port); ··· 624 590 } 625 591 static DEVICE_ATTR(adapter_handle, S_IRUGO, ehca_show_adapter_handle, NULL); 626 592 593 + static ssize_t ehca_show_mr_largepage(struct device *dev, 594 + struct device_attribute *attr, 595 + char *buf) 596 + { 597 + return sprintf(buf, "%d\n", ehca_mr_largepage); 598 + } 599 + static DEVICE_ATTR(mr_largepage, S_IRUGO, ehca_show_mr_largepage, NULL); 600 + 627 601 static struct attribute *ehca_dev_attrs[] = { 628 602 &dev_attr_adapter_handle.attr, 629 603 &dev_attr_num_ports.attr, ··· 648 606 &dev_attr_cur_mw.attr, 649 607 &dev_attr_max_pd.attr, 650 608 &dev_attr_max_ah.attr, 609 + &dev_attr_mr_largepage.attr, 651 610 NULL 652 611 }; 653 612
+311 -60
drivers/infiniband/hw/ehca/ehca_mrmw.c
··· 5 5 * 6 6 * Authors: Dietmar Decker <ddecker@de.ibm.com> 7 7 * Christoph Raisch <raisch@de.ibm.com> 8 + * Hoang-Nam Nguyen <hnguyen@de.ibm.com> 8 9 * 9 10 * Copyright (c) 2005 IBM Corporation 10 11 * ··· 56 55 57 56 static struct kmem_cache *mr_cache; 58 57 static struct kmem_cache *mw_cache; 58 + 59 + enum ehca_mr_pgsize { 60 + EHCA_MR_PGSIZE4K = 0x1000L, 61 + EHCA_MR_PGSIZE64K = 0x10000L, 62 + EHCA_MR_PGSIZE1M = 0x100000L, 63 + EHCA_MR_PGSIZE16M = 0x1000000L 64 + }; 65 + 66 + extern int ehca_mr_largepage; 67 + 68 + static u32 ehca_encode_hwpage_size(u32 pgsize) 69 + { 70 + u32 idx = 0; 71 + pgsize >>= 12; 72 + /* 73 + * map mr page size into hw code: 74 + * 0, 1, 2, 3 for 4K, 64K, 1M, 64M 75 + */ 76 + while (!(pgsize & 1)) { 77 + idx++; 78 + pgsize >>= 4; 79 + } 80 + return idx; 81 + } 82 + 83 + static u64 ehca_get_max_hwpage_size(struct ehca_shca *shca) 84 + { 85 + if (shca->hca_cap_mr_pgsize & HCA_CAP_MR_PGSIZE_16M) 86 + return EHCA_MR_PGSIZE16M; 87 + return EHCA_MR_PGSIZE4K; 88 + } 59 89 60 90 static struct ehca_mr *ehca_mr_new(void) 61 91 { ··· 239 207 struct ehca_mr_pginfo pginfo; 240 208 u32 num_kpages; 241 209 u32 num_hwpages; 210 + u64 hw_pgsize; 242 211 243 212 num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size, 244 213 PAGE_SIZE); 245 - num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) + 246 - size, EHCA_PAGESIZE); 214 + /* for kernel space we try most possible pgsize */ 215 + hw_pgsize = ehca_get_max_hwpage_size(shca); 216 + num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size, 217 + hw_pgsize); 247 218 memset(&pginfo, 0, sizeof(pginfo)); 248 219 pginfo.type = EHCA_MR_PGI_PHYS; 249 220 pginfo.num_kpages = num_kpages; 221 + pginfo.hwpage_size = hw_pgsize; 250 222 pginfo.num_hwpages = num_hwpages; 251 223 pginfo.u.phy.num_phys_buf = num_phys_buf; 252 224 pginfo.u.phy.phys_buf_array = phys_buf_array; 253 - pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) / 254 - EHCA_PAGESIZE); 225 + pginfo.next_hwpage = 226 + ((u64)iova_start & ~(hw_pgsize - 1)) / hw_pgsize; 255 227 256 228 ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags, 257 229 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey, ··· 295 259 int ret; 296 260 u32 num_kpages; 297 261 u32 num_hwpages; 262 + u64 hwpage_size; 298 263 299 264 if (!pd) { 300 265 ehca_gen_err("bad pd=%p", pd); ··· 346 309 347 310 /* determine number of MR pages */ 348 311 num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE); 349 - num_hwpages = NUM_CHUNKS((virt % EHCA_PAGESIZE) + length, 350 - EHCA_PAGESIZE); 312 + /* select proper hw_pgsize */ 313 + if (ehca_mr_largepage && 314 + (shca->hca_cap_mr_pgsize & HCA_CAP_MR_PGSIZE_16M)) { 315 + if (length <= EHCA_MR_PGSIZE4K 316 + && PAGE_SIZE == EHCA_MR_PGSIZE4K) 317 + hwpage_size = EHCA_MR_PGSIZE4K; 318 + else if (length <= EHCA_MR_PGSIZE64K) 319 + hwpage_size = EHCA_MR_PGSIZE64K; 320 + else if (length <= EHCA_MR_PGSIZE1M) 321 + hwpage_size = EHCA_MR_PGSIZE1M; 322 + else 323 + hwpage_size = EHCA_MR_PGSIZE16M; 324 + } else 325 + hwpage_size = EHCA_MR_PGSIZE4K; 326 + ehca_dbg(pd->device, "hwpage_size=%lx", hwpage_size); 351 327 328 + reg_user_mr_fallback: 329 + num_hwpages = NUM_CHUNKS((virt % hwpage_size) + length, hwpage_size); 352 330 /* register MR on HCA */ 353 331 memset(&pginfo, 0, sizeof(pginfo)); 354 332 pginfo.type = EHCA_MR_PGI_USER; 333 + pginfo.hwpage_size = hwpage_size; 355 334 pginfo.num_kpages = num_kpages; 356 335 pginfo.num_hwpages = num_hwpages; 357 336 pginfo.u.usr.region = e_mr->umem; 358 - pginfo.next_hwpage = e_mr->umem->offset / EHCA_PAGESIZE; 337 + pginfo.next_hwpage = e_mr->umem->offset / hwpage_size; 359 338 pginfo.u.usr.next_chunk = list_prepare_entry(pginfo.u.usr.next_chunk, 360 339 (&e_mr->umem->chunk_list), 361 340 list); ··· 379 326 ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags, 380 327 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey, 381 328 &e_mr->ib.ib_mr.rkey); 329 + if (ret == -EINVAL && pginfo.hwpage_size > PAGE_SIZE) { 330 + ehca_warn(pd->device, "failed to register mr " 331 + "with hwpage_size=%lx", hwpage_size); 332 + ehca_info(pd->device, "try to register mr with " 333 + "kpage_size=%lx", PAGE_SIZE); 334 + /* 335 + * this means kpages are not contiguous for a hw page 336 + * try kernel page size as fallback solution 337 + */ 338 + hwpage_size = PAGE_SIZE; 339 + goto reg_user_mr_fallback; 340 + } 382 341 if (ret) { 383 342 ib_mr = ERR_PTR(ret); 384 343 goto reg_user_mr_exit2; ··· 517 452 new_pd = container_of(mr->pd, struct ehca_pd, ib_pd); 518 453 519 454 if (mr_rereg_mask & IB_MR_REREG_TRANS) { 455 + u64 hw_pgsize = ehca_get_max_hwpage_size(shca); 456 + 520 457 new_start = iova_start; /* change address */ 521 458 /* check physical buffer list and calculate size */ 522 459 ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, ··· 535 468 } 536 469 num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) + 537 470 new_size, PAGE_SIZE); 538 - num_hwpages = NUM_CHUNKS(((u64)new_start % EHCA_PAGESIZE) + 539 - new_size, EHCA_PAGESIZE); 471 + num_hwpages = NUM_CHUNKS(((u64)new_start % hw_pgsize) + 472 + new_size, hw_pgsize); 540 473 memset(&pginfo, 0, sizeof(pginfo)); 541 474 pginfo.type = EHCA_MR_PGI_PHYS; 542 475 pginfo.num_kpages = num_kpages; 476 + pginfo.hwpage_size = hw_pgsize; 543 477 pginfo.num_hwpages = num_hwpages; 544 478 pginfo.u.phy.num_phys_buf = num_phys_buf; 545 479 pginfo.u.phy.phys_buf_array = phys_buf_array; 546 - pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) / 547 - EHCA_PAGESIZE); 480 + pginfo.next_hwpage = 481 + ((u64)iova_start & ~(hw_pgsize - 1)) / hw_pgsize; 548 482 } 549 483 if (mr_rereg_mask & IB_MR_REREG_ACCESS) 550 484 new_acl = mr_access_flags; ··· 777 709 int ret; 778 710 u32 tmp_lkey, tmp_rkey; 779 711 struct ehca_mr_pginfo pginfo; 712 + u64 hw_pgsize; 780 713 781 714 /* check other parameters */ 782 715 if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) && ··· 807 738 ib_fmr = ERR_PTR(-EINVAL); 808 739 goto alloc_fmr_exit0; 809 740 } 810 - if (((1 << fmr_attr->page_shift) != EHCA_PAGESIZE) && 811 - ((1 << fmr_attr->page_shift) != PAGE_SIZE)) { 741 + hw_pgsize = ehca_get_max_hwpage_size(shca); 742 + if ((1 << fmr_attr->page_shift) != hw_pgsize) { 812 743 ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x", 813 744 fmr_attr->page_shift); 814 745 ib_fmr = ERR_PTR(-EINVAL); ··· 824 755 825 756 /* register MR on HCA */ 826 757 memset(&pginfo, 0, sizeof(pginfo)); 758 + /* 759 + * pginfo.num_hwpages==0, ie register_rpages() will not be called 760 + * but deferred to map_phys_fmr() 761 + */ 827 762 ret = ehca_reg_mr(shca, e_fmr, NULL, 828 763 fmr_attr->max_pages * (1 << fmr_attr->page_shift), 829 764 mr_access_flags, e_pd, &pginfo, ··· 838 765 } 839 766 840 767 /* successful */ 768 + e_fmr->hwpage_size = hw_pgsize; 841 769 e_fmr->fmr_page_size = 1 << fmr_attr->page_shift; 842 770 e_fmr->fmr_max_pages = fmr_attr->max_pages; 843 771 e_fmr->fmr_max_maps = fmr_attr->max_maps; ··· 896 822 memset(&pginfo, 0, sizeof(pginfo)); 897 823 pginfo.type = EHCA_MR_PGI_FMR; 898 824 pginfo.num_kpages = list_len; 899 - pginfo.num_hwpages = list_len * (e_fmr->fmr_page_size / EHCA_PAGESIZE); 825 + pginfo.hwpage_size = e_fmr->hwpage_size; 826 + pginfo.num_hwpages = 827 + list_len * e_fmr->fmr_page_size / pginfo.hwpage_size; 900 828 pginfo.u.fmr.page_list = page_list; 901 - pginfo.next_hwpage = ((iova & (e_fmr->fmr_page_size-1)) / 902 - EHCA_PAGESIZE); 829 + pginfo.next_hwpage = 830 + (iova & (e_fmr->fmr_page_size-1)) / pginfo.hwpage_size; 903 831 pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size; 904 832 905 833 ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova, ··· 1040 964 struct ehca_mr_hipzout_parms hipzout; 1041 965 1042 966 ehca_mrmw_map_acl(acl, &hipz_acl); 1043 - ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 967 + ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl); 1044 968 if (ehca_use_hp_mr == 1) 1045 969 hipz_acl |= 0x00000001; 1046 970 ··· 1063 987 /* successful registration */ 1064 988 e_mr->num_kpages = pginfo->num_kpages; 1065 989 e_mr->num_hwpages = pginfo->num_hwpages; 990 + e_mr->hwpage_size = pginfo->hwpage_size; 1066 991 e_mr->start = iova_start; 1067 992 e_mr->size = size; 1068 993 e_mr->acl = acl; ··· 1106 1029 u32 i; 1107 1030 u64 *kpage; 1108 1031 1032 + if (!pginfo->num_hwpages) /* in case of fmr */ 1033 + return 0; 1034 + 1109 1035 kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL); 1110 1036 if (!kpage) { 1111 1037 ehca_err(&shca->ib_device, "kpage alloc failed"); ··· 1116 1036 goto ehca_reg_mr_rpages_exit0; 1117 1037 } 1118 1038 1119 - /* max 512 pages per shot */ 1039 + /* max MAX_RPAGES ehca mr pages per register call */ 1120 1040 for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) { 1121 1041 1122 1042 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) { ··· 1129 1049 ret = ehca_set_pagebuf(pginfo, rnum, kpage); 1130 1050 if (ret) { 1131 1051 ehca_err(&shca->ib_device, "ehca_set_pagebuf " 1132 - "bad rc, ret=%x rnum=%x kpage=%p", 1133 - ret, rnum, kpage); 1052 + "bad rc, ret=%x rnum=%x kpage=%p", 1053 + ret, rnum, kpage); 1134 1054 goto ehca_reg_mr_rpages_exit1; 1135 1055 } 1136 1056 ··· 1145 1065 } else 1146 1066 rpage = *kpage; 1147 1067 1148 - h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, e_mr, 1149 - 0, /* pagesize 4k */ 1150 - 0, rpage, rnum); 1068 + h_ret = hipz_h_register_rpage_mr( 1069 + shca->ipz_hca_handle, e_mr, 1070 + ehca_encode_hwpage_size(pginfo->hwpage_size), 1071 + 0, rpage, rnum); 1151 1072 1152 1073 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) { 1153 1074 /* ··· 1212 1131 struct ehca_mr_hipzout_parms hipzout; 1213 1132 1214 1133 ehca_mrmw_map_acl(acl, &hipz_acl); 1215 - ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 1134 + ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl); 1216 1135 1217 1136 kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL); 1218 1137 if (!kpage) { ··· 1263 1182 */ 1264 1183 e_mr->num_kpages = pginfo->num_kpages; 1265 1184 e_mr->num_hwpages = pginfo->num_hwpages; 1185 + e_mr->hwpage_size = pginfo->hwpage_size; 1266 1186 e_mr->start = iova_start; 1267 1187 e_mr->size = size; 1268 1188 e_mr->acl = acl; ··· 1350 1268 1351 1269 /* set some MR values */ 1352 1270 e_mr->flags = save_mr.flags; 1271 + e_mr->hwpage_size = save_mr.hwpage_size; 1353 1272 e_mr->fmr_page_size = save_mr.fmr_page_size; 1354 1273 e_mr->fmr_max_pages = save_mr.fmr_max_pages; 1355 1274 e_mr->fmr_max_maps = save_mr.fmr_max_maps; 1356 1275 e_mr->fmr_map_cnt = save_mr.fmr_map_cnt; 1357 1276 1358 1277 ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl, 1359 - e_pd, pginfo, lkey, rkey); 1278 + e_pd, pginfo, lkey, rkey); 1360 1279 if (ret) { 1361 1280 u32 offset = (u64)(&e_mr->flags) - (u64)e_mr; 1362 1281 memcpy(&e_mr->flags, &(save_mr.flags), ··· 1438 1355 1439 1356 /* set some MR values */ 1440 1357 e_fmr->flags = save_fmr.flags; 1358 + e_fmr->hwpage_size = save_fmr.hwpage_size; 1441 1359 e_fmr->fmr_page_size = save_fmr.fmr_page_size; 1442 1360 e_fmr->fmr_max_pages = save_fmr.fmr_max_pages; 1443 1361 e_fmr->fmr_max_maps = save_fmr.fmr_max_maps; ··· 1447 1363 1448 1364 memset(&pginfo, 0, sizeof(pginfo)); 1449 1365 pginfo.type = EHCA_MR_PGI_FMR; 1450 - pginfo.num_kpages = 0; 1451 - pginfo.num_hwpages = 0; 1452 1366 ret = ehca_reg_mr(shca, e_fmr, NULL, 1453 1367 (e_fmr->fmr_max_pages * e_fmr->fmr_page_size), 1454 1368 e_fmr->acl, e_pd, &pginfo, &tmp_lkey, ··· 1455 1373 u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr; 1456 1374 memcpy(&e_fmr->flags, &(save_mr.flags), 1457 1375 sizeof(struct ehca_mr) - offset); 1458 - goto ehca_unmap_one_fmr_exit0; 1459 1376 } 1460 1377 1461 1378 ehca_unmap_one_fmr_exit0: ··· 1482 1401 struct ehca_mr_hipzout_parms hipzout; 1483 1402 1484 1403 ehca_mrmw_map_acl(acl, &hipz_acl); 1485 - ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 1404 + ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl); 1486 1405 1487 1406 h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr, 1488 1407 (u64)iova_start, hipz_acl, e_pd->fw_pd, ··· 1501 1420 /* successful registration */ 1502 1421 e_newmr->num_kpages = e_origmr->num_kpages; 1503 1422 e_newmr->num_hwpages = e_origmr->num_hwpages; 1423 + e_newmr->hwpage_size = e_origmr->hwpage_size; 1504 1424 e_newmr->start = iova_start; 1505 1425 e_newmr->size = e_origmr->size; 1506 1426 e_newmr->acl = acl; ··· 1534 1452 struct ib_phys_buf ib_pbuf; 1535 1453 u32 num_kpages; 1536 1454 u32 num_hwpages; 1455 + u64 hw_pgsize; 1537 1456 1538 1457 e_mr = ehca_mr_new(); 1539 1458 if (!e_mr) { ··· 1551 1468 ib_pbuf.size = size_maxmr; 1552 1469 num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr, 1553 1470 PAGE_SIZE); 1554 - num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) + size_maxmr, 1555 - EHCA_PAGESIZE); 1471 + hw_pgsize = ehca_get_max_hwpage_size(shca); 1472 + num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size_maxmr, 1473 + hw_pgsize); 1556 1474 1557 1475 memset(&pginfo, 0, sizeof(pginfo)); 1558 1476 pginfo.type = EHCA_MR_PGI_PHYS; 1559 1477 pginfo.num_kpages = num_kpages; 1560 1478 pginfo.num_hwpages = num_hwpages; 1479 + pginfo.hwpage_size = hw_pgsize; 1561 1480 pginfo.u.phy.num_phys_buf = 1; 1562 1481 pginfo.u.phy.phys_buf_array = &ib_pbuf; 1563 1482 ··· 1608 1523 struct ehca_mr_hipzout_parms hipzout; 1609 1524 1610 1525 ehca_mrmw_map_acl(acl, &hipz_acl); 1611 - ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 1526 + ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl); 1612 1527 1613 1528 h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr, 1614 1529 (u64)iova_start, hipz_acl, e_pd->fw_pd, ··· 1624 1539 /* successful registration */ 1625 1540 e_newmr->num_kpages = e_origmr->num_kpages; 1626 1541 e_newmr->num_hwpages = e_origmr->num_hwpages; 1542 + e_newmr->hwpage_size = e_origmr->hwpage_size; 1627 1543 e_newmr->start = iova_start; 1628 1544 e_newmr->size = e_origmr->size; 1629 1545 e_newmr->acl = acl; ··· 1770 1684 u64 pgaddr; 1771 1685 u32 i = 0; 1772 1686 u32 j = 0; 1687 + int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size; 1773 1688 1774 1689 /* loop over desired chunk entries */ 1775 1690 chunk = pginfo->u.usr.next_chunk; ··· 1782 1695 << PAGE_SHIFT ; 1783 1696 *kpage = phys_to_abs(pgaddr + 1784 1697 (pginfo->next_hwpage * 1785 - EHCA_PAGESIZE)); 1698 + pginfo->hwpage_size)); 1786 1699 if ( !(*kpage) ) { 1787 1700 ehca_gen_err("pgaddr=%lx " 1788 1701 "chunk->page_list[i]=%lx " ··· 1795 1708 (pginfo->hwpage_cnt)++; 1796 1709 (pginfo->next_hwpage)++; 1797 1710 kpage++; 1798 - if (pginfo->next_hwpage % 1799 - (PAGE_SIZE / EHCA_PAGESIZE) == 0) { 1711 + if (pginfo->next_hwpage % hwpages_per_kpage == 0) { 1800 1712 (pginfo->kpage_cnt)++; 1801 1713 (pginfo->u.usr.next_nmap)++; 1802 1714 pginfo->next_hwpage = 0; 1803 1715 i++; 1804 1716 } 1717 + j++; 1718 + if (j >= number) break; 1719 + } 1720 + if ((pginfo->u.usr.next_nmap >= chunk->nmap) && 1721 + (j >= number)) { 1722 + pginfo->u.usr.next_nmap = 0; 1723 + prev_chunk = chunk; 1724 + break; 1725 + } else if (pginfo->u.usr.next_nmap >= chunk->nmap) { 1726 + pginfo->u.usr.next_nmap = 0; 1727 + prev_chunk = chunk; 1728 + } else if (j >= number) 1729 + break; 1730 + else 1731 + prev_chunk = chunk; 1732 + } 1733 + pginfo->u.usr.next_chunk = 1734 + list_prepare_entry(prev_chunk, 1735 + (&(pginfo->u.usr.region->chunk_list)), 1736 + list); 1737 + return ret; 1738 + } 1739 + 1740 + /* 1741 + * check given pages for contiguous layout 1742 + * last page addr is returned in prev_pgaddr for further check 1743 + */ 1744 + static int ehca_check_kpages_per_ate(struct scatterlist *page_list, 1745 + int start_idx, int end_idx, 1746 + u64 *prev_pgaddr) 1747 + { 1748 + int t; 1749 + for (t = start_idx; t <= end_idx; t++) { 1750 + u64 pgaddr = page_to_pfn(page_list[t].page) << PAGE_SHIFT; 1751 + ehca_gen_dbg("chunk_page=%lx value=%016lx", pgaddr, 1752 + *(u64 *)abs_to_virt(phys_to_abs(pgaddr))); 1753 + if (pgaddr - PAGE_SIZE != *prev_pgaddr) { 1754 + ehca_gen_err("uncontiguous page found pgaddr=%lx " 1755 + "prev_pgaddr=%lx page_list_i=%x", 1756 + pgaddr, *prev_pgaddr, t); 1757 + return -EINVAL; 1758 + } 1759 + *prev_pgaddr = pgaddr; 1760 + } 1761 + return 0; 1762 + } 1763 + 1764 + /* PAGE_SIZE < pginfo->hwpage_size */ 1765 + static int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo, 1766 + u32 number, 1767 + u64 *kpage) 1768 + { 1769 + int ret = 0; 1770 + struct ib_umem_chunk *prev_chunk; 1771 + struct ib_umem_chunk *chunk; 1772 + u64 pgaddr, prev_pgaddr; 1773 + u32 i = 0; 1774 + u32 j = 0; 1775 + int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE; 1776 + int nr_kpages = kpages_per_hwpage; 1777 + 1778 + /* loop over desired chunk entries */ 1779 + chunk = pginfo->u.usr.next_chunk; 1780 + prev_chunk = pginfo->u.usr.next_chunk; 1781 + list_for_each_entry_continue( 1782 + chunk, (&(pginfo->u.usr.region->chunk_list)), list) { 1783 + for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) { 1784 + if (nr_kpages == kpages_per_hwpage) { 1785 + pgaddr = ( page_to_pfn(chunk->page_list[i].page) 1786 + << PAGE_SHIFT ); 1787 + *kpage = phys_to_abs(pgaddr); 1788 + if ( !(*kpage) ) { 1789 + ehca_gen_err("pgaddr=%lx i=%x", 1790 + pgaddr, i); 1791 + ret = -EFAULT; 1792 + return ret; 1793 + } 1794 + /* 1795 + * The first page in a hwpage must be aligned; 1796 + * the first MR page is exempt from this rule. 1797 + */ 1798 + if (pgaddr & (pginfo->hwpage_size - 1)) { 1799 + if (pginfo->hwpage_cnt) { 1800 + ehca_gen_err( 1801 + "invalid alignment " 1802 + "pgaddr=%lx i=%x " 1803 + "mr_pgsize=%lx", 1804 + pgaddr, i, 1805 + pginfo->hwpage_size); 1806 + ret = -EFAULT; 1807 + return ret; 1808 + } 1809 + /* first MR page */ 1810 + pginfo->kpage_cnt = 1811 + (pgaddr & 1812 + (pginfo->hwpage_size - 1)) >> 1813 + PAGE_SHIFT; 1814 + nr_kpages -= pginfo->kpage_cnt; 1815 + *kpage = phys_to_abs( 1816 + pgaddr & 1817 + ~(pginfo->hwpage_size - 1)); 1818 + } 1819 + ehca_gen_dbg("kpage=%lx chunk_page=%lx " 1820 + "value=%016lx", *kpage, pgaddr, 1821 + *(u64 *)abs_to_virt( 1822 + phys_to_abs(pgaddr))); 1823 + prev_pgaddr = pgaddr; 1824 + i++; 1825 + pginfo->kpage_cnt++; 1826 + pginfo->u.usr.next_nmap++; 1827 + nr_kpages--; 1828 + if (!nr_kpages) 1829 + goto next_kpage; 1830 + continue; 1831 + } 1832 + if (i + nr_kpages > chunk->nmap) { 1833 + ret = ehca_check_kpages_per_ate( 1834 + chunk->page_list, i, 1835 + chunk->nmap - 1, &prev_pgaddr); 1836 + if (ret) return ret; 1837 + pginfo->kpage_cnt += chunk->nmap - i; 1838 + pginfo->u.usr.next_nmap += chunk->nmap - i; 1839 + nr_kpages -= chunk->nmap - i; 1840 + break; 1841 + } 1842 + 1843 + ret = ehca_check_kpages_per_ate(chunk->page_list, i, 1844 + i + nr_kpages - 1, 1845 + &prev_pgaddr); 1846 + if (ret) return ret; 1847 + i += nr_kpages; 1848 + pginfo->kpage_cnt += nr_kpages; 1849 + pginfo->u.usr.next_nmap += nr_kpages; 1850 + next_kpage: 1851 + nr_kpages = kpages_per_hwpage; 1852 + (pginfo->hwpage_cnt)++; 1853 + kpage++; 1805 1854 j++; 1806 1855 if (j >= number) break; 1807 1856 } ··· 1973 1750 /* loop over desired phys_buf_array entries */ 1974 1751 while (i < number) { 1975 1752 pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf; 1976 - num_hw = NUM_CHUNKS((pbuf->addr % EHCA_PAGESIZE) + 1977 - pbuf->size, EHCA_PAGESIZE); 1978 - offs_hw = (pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE; 1753 + num_hw = NUM_CHUNKS((pbuf->addr % pginfo->hwpage_size) + 1754 + pbuf->size, pginfo->hwpage_size); 1755 + offs_hw = (pbuf->addr & ~(pginfo->hwpage_size - 1)) / 1756 + pginfo->hwpage_size; 1979 1757 while (pginfo->next_hwpage < offs_hw + num_hw) { 1980 1758 /* sanity check */ 1981 1759 if ((pginfo->kpage_cnt >= pginfo->num_kpages) || ··· 1992 1768 return -EFAULT; 1993 1769 } 1994 1770 *kpage = phys_to_abs( 1995 - (pbuf->addr & EHCA_PAGEMASK) 1996 - + (pginfo->next_hwpage * EHCA_PAGESIZE)); 1771 + (pbuf->addr & ~(pginfo->hwpage_size - 1)) + 1772 + (pginfo->next_hwpage * pginfo->hwpage_size)); 1997 1773 if ( !(*kpage) && pbuf->addr ) { 1998 - ehca_gen_err("pbuf->addr=%lx " 1999 - "pbuf->size=%lx " 1774 + ehca_gen_err("pbuf->addr=%lx pbuf->size=%lx " 2000 1775 "next_hwpage=%lx", pbuf->addr, 2001 - pbuf->size, 2002 - pginfo->next_hwpage); 1776 + pbuf->size, pginfo->next_hwpage); 2003 1777 return -EFAULT; 2004 1778 } 2005 1779 (pginfo->hwpage_cnt)++; 2006 1780 (pginfo->next_hwpage)++; 2007 - if (pginfo->next_hwpage % 2008 - (PAGE_SIZE / EHCA_PAGESIZE) == 0) 2009 - (pginfo->kpage_cnt)++; 1781 + if (PAGE_SIZE >= pginfo->hwpage_size) { 1782 + if (pginfo->next_hwpage % 1783 + (PAGE_SIZE / pginfo->hwpage_size) == 0) 1784 + (pginfo->kpage_cnt)++; 1785 + } else 1786 + pginfo->kpage_cnt += pginfo->hwpage_size / 1787 + PAGE_SIZE; 2010 1788 kpage++; 2011 1789 i++; 2012 1790 if (i >= number) break; ··· 2032 1806 /* loop over desired page_list entries */ 2033 1807 fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem; 2034 1808 for (i = 0; i < number; i++) { 2035 - *kpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) + 2036 - pginfo->next_hwpage * EHCA_PAGESIZE); 1809 + *kpage = phys_to_abs((*fmrlist & ~(pginfo->hwpage_size - 1)) + 1810 + pginfo->next_hwpage * pginfo->hwpage_size); 2037 1811 if ( !(*kpage) ) { 2038 1812 ehca_gen_err("*fmrlist=%lx fmrlist=%p " 2039 1813 "next_listelem=%lx next_hwpage=%lx", ··· 2043 1817 return -EFAULT; 2044 1818 } 2045 1819 (pginfo->hwpage_cnt)++; 2046 - (pginfo->next_hwpage)++; 2047 - kpage++; 2048 - if (pginfo->next_hwpage % 2049 - (pginfo->u.fmr.fmr_pgsize / EHCA_PAGESIZE) == 0) { 2050 - (pginfo->kpage_cnt)++; 2051 - (pginfo->u.fmr.next_listelem)++; 2052 - fmrlist++; 2053 - pginfo->next_hwpage = 0; 1820 + if (pginfo->u.fmr.fmr_pgsize >= pginfo->hwpage_size) { 1821 + if (pginfo->next_hwpage % 1822 + (pginfo->u.fmr.fmr_pgsize / 1823 + pginfo->hwpage_size) == 0) { 1824 + (pginfo->kpage_cnt)++; 1825 + (pginfo->u.fmr.next_listelem)++; 1826 + fmrlist++; 1827 + pginfo->next_hwpage = 0; 1828 + } else 1829 + (pginfo->next_hwpage)++; 1830 + } else { 1831 + unsigned int cnt_per_hwpage = pginfo->hwpage_size / 1832 + pginfo->u.fmr.fmr_pgsize; 1833 + unsigned int j; 1834 + u64 prev = *kpage; 1835 + /* check if adrs are contiguous */ 1836 + for (j = 1; j < cnt_per_hwpage; j++) { 1837 + u64 p = phys_to_abs(fmrlist[j] & 1838 + ~(pginfo->hwpage_size - 1)); 1839 + if (prev + pginfo->u.fmr.fmr_pgsize != p) { 1840 + ehca_gen_err("uncontiguous fmr pages " 1841 + "found prev=%lx p=%lx " 1842 + "idx=%x", prev, p, i + j); 1843 + return -EINVAL; 1844 + } 1845 + prev = p; 1846 + } 1847 + pginfo->kpage_cnt += cnt_per_hwpage; 1848 + pginfo->u.fmr.next_listelem += cnt_per_hwpage; 1849 + fmrlist += cnt_per_hwpage; 2054 1850 } 1851 + kpage++; 2055 1852 } 2056 1853 return ret; 2057 1854 } ··· 2091 1842 ret = ehca_set_pagebuf_phys(pginfo, number, kpage); 2092 1843 break; 2093 1844 case EHCA_MR_PGI_USER: 2094 - ret = ehca_set_pagebuf_user1(pginfo, number, kpage); 1845 + ret = PAGE_SIZE >= pginfo->hwpage_size ? 1846 + ehca_set_pagebuf_user1(pginfo, number, kpage) : 1847 + ehca_set_pagebuf_user2(pginfo, number, kpage); 2095 1848 break; 2096 1849 case EHCA_MR_PGI_FMR: 2097 1850 ret = ehca_set_pagebuf_fmr(pginfo, number, kpage); ··· 2146 1895 /*----------------------------------------------------------------------*/ 2147 1896 2148 1897 /* sets page size in hipz access control for MR/MW. */ 2149 - void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl) /*INOUT*/ 1898 + void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl) /*INOUT*/ 2150 1899 { 2151 - return; /* HCA supports only 4k */ 1900 + *hipz_acl |= (ehca_encode_hwpage_size(pgsize) << 24); 2152 1901 } /* end ehca_mrmw_set_pgsize_hipz_acl() */ 2153 1902 2154 1903 /*----------------------------------------------------------------------*/
+1 -1
drivers/infiniband/hw/ehca/ehca_mrmw.h
··· 111 111 void ehca_mrmw_map_acl(int ib_acl, 112 112 u32 *hipz_acl); 113 113 114 - void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl); 114 + void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl); 115 115 116 116 void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl, 117 117 int *ib_acl);
+23 -2
drivers/infiniband/hw/ehca/ehca_pd.c
··· 49 49 struct ib_ucontext *context, struct ib_udata *udata) 50 50 { 51 51 struct ehca_pd *pd; 52 + int i; 52 53 53 54 pd = kmem_cache_zalloc(pd_cache, GFP_KERNEL); 54 55 if (!pd) { ··· 59 58 } 60 59 61 60 pd->ownpid = current->tgid; 61 + for (i = 0; i < 2; i++) { 62 + INIT_LIST_HEAD(&pd->free[i]); 63 + INIT_LIST_HEAD(&pd->full[i]); 64 + } 65 + mutex_init(&pd->lock); 62 66 63 67 /* 64 68 * Kernel PD: when device = -1, 0 ··· 87 81 { 88 82 u32 cur_pid = current->tgid; 89 83 struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd); 84 + int i, leftovers = 0; 85 + extern struct kmem_cache *small_qp_cache; 86 + struct ipz_small_queue_page *page, *tmp; 90 87 91 88 if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context && 92 89 my_pd->ownpid != cur_pid) { ··· 98 89 return -EINVAL; 99 90 } 100 91 101 - kmem_cache_free(pd_cache, 102 - container_of(pd, struct ehca_pd, ib_pd)); 92 + for (i = 0; i < 2; i++) { 93 + list_splice(&my_pd->full[i], &my_pd->free[i]); 94 + list_for_each_entry_safe(page, tmp, &my_pd->free[i], list) { 95 + leftovers = 1; 96 + free_page(page->page); 97 + kmem_cache_free(small_qp_cache, page); 98 + } 99 + } 100 + 101 + if (leftovers) 102 + ehca_warn(pd->device, 103 + "Some small queue pages were not freed"); 104 + 105 + kmem_cache_free(pd_cache, my_pd); 103 106 104 107 return 0; 105 108 }
+106 -72
drivers/infiniband/hw/ehca/ehca_qp.c
··· 275 275 resp->toggle_state = queue->toggle_state; 276 276 } 277 277 278 - static inline int ll_qp_msg_size(int nr_sge) 279 - { 280 - return 128 << nr_sge; 281 - } 282 - 283 278 /* 284 279 * init_qp_queue initializes/constructs r/squeue and registers queue pages. 285 280 */ 286 281 static inline int init_qp_queue(struct ehca_shca *shca, 282 + struct ehca_pd *pd, 287 283 struct ehca_qp *my_qp, 288 284 struct ipz_queue *queue, 289 285 int q_type, 290 286 u64 expected_hret, 291 - int nr_q_pages, 292 - int wqe_size, 293 - int nr_sges) 287 + struct ehca_alloc_queue_parms *parms, 288 + int wqe_size) 294 289 { 295 - int ret, cnt, ipz_rc; 290 + int ret, cnt, ipz_rc, nr_q_pages; 296 291 void *vpage; 297 292 u64 rpage, h_ret; 298 293 struct ib_device *ib_dev = &shca->ib_device; 299 294 struct ipz_adapter_handle ipz_hca_handle = shca->ipz_hca_handle; 300 295 301 - if (!nr_q_pages) 296 + if (!parms->queue_size) 302 297 return 0; 303 298 304 - ipz_rc = ipz_queue_ctor(queue, nr_q_pages, EHCA_PAGESIZE, 305 - wqe_size, nr_sges); 299 + if (parms->is_small) { 300 + nr_q_pages = 1; 301 + ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages, 302 + 128 << parms->page_size, 303 + wqe_size, parms->act_nr_sges, 1); 304 + } else { 305 + nr_q_pages = parms->queue_size; 306 + ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages, 307 + EHCA_PAGESIZE, wqe_size, 308 + parms->act_nr_sges, 0); 309 + } 310 + 306 311 if (!ipz_rc) { 307 312 ehca_err(ib_dev, "Cannot allocate page for queue. ipz_rc=%x", 308 313 ipz_rc); ··· 328 323 h_ret = hipz_h_register_rpage_qp(ipz_hca_handle, 329 324 my_qp->ipz_qp_handle, 330 325 NULL, 0, q_type, 331 - rpage, 1, 326 + rpage, parms->is_small ? 0 : 1, 332 327 my_qp->galpas.kernel); 333 328 if (cnt == (nr_q_pages - 1)) { /* last page! */ 334 329 if (h_ret != expected_hret) { ··· 359 354 return 0; 360 355 361 356 init_qp_queue1: 362 - ipz_queue_dtor(queue); 357 + ipz_queue_dtor(pd, queue); 363 358 return ret; 359 + } 360 + 361 + static inline int ehca_calc_wqe_size(int act_nr_sge, int is_llqp) 362 + { 363 + if (is_llqp) 364 + return 128 << act_nr_sge; 365 + else 366 + return offsetof(struct ehca_wqe, 367 + u.nud.sg_list[act_nr_sge]); 368 + } 369 + 370 + static void ehca_determine_small_queue(struct ehca_alloc_queue_parms *queue, 371 + int req_nr_sge, int is_llqp) 372 + { 373 + u32 wqe_size, q_size; 374 + int act_nr_sge = req_nr_sge; 375 + 376 + if (!is_llqp) 377 + /* round up #SGEs so WQE size is a power of 2 */ 378 + for (act_nr_sge = 4; act_nr_sge <= 252; 379 + act_nr_sge = 4 + 2 * act_nr_sge) 380 + if (act_nr_sge >= req_nr_sge) 381 + break; 382 + 383 + wqe_size = ehca_calc_wqe_size(act_nr_sge, is_llqp); 384 + q_size = wqe_size * (queue->max_wr + 1); 385 + 386 + if (q_size <= 512) 387 + queue->page_size = 2; 388 + else if (q_size <= 1024) 389 + queue->page_size = 3; 390 + else 391 + queue->page_size = 0; 392 + 393 + queue->is_small = (queue->page_size != 0); 364 394 } 365 395 366 396 /* ··· 403 363 * the value of the is_srq parameter. If init_attr and srq_init_attr share 404 364 * fields, the field out of init_attr is used. 405 365 */ 406 - struct ehca_qp *internal_create_qp(struct ib_pd *pd, 407 - struct ib_qp_init_attr *init_attr, 408 - struct ib_srq_init_attr *srq_init_attr, 409 - struct ib_udata *udata, int is_srq) 366 + static struct ehca_qp *internal_create_qp( 367 + struct ib_pd *pd, 368 + struct ib_qp_init_attr *init_attr, 369 + struct ib_srq_init_attr *srq_init_attr, 370 + struct ib_udata *udata, int is_srq) 410 371 { 411 372 struct ehca_qp *my_qp; 412 373 struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd); ··· 593 552 if (my_qp->recv_cq) 594 553 parms.recv_cq_handle = my_qp->recv_cq->ipz_cq_handle; 595 554 596 - parms.max_send_wr = init_attr->cap.max_send_wr; 597 - parms.max_recv_wr = init_attr->cap.max_recv_wr; 598 - parms.max_send_sge = max_send_sge; 599 - parms.max_recv_sge = max_recv_sge; 555 + parms.squeue.max_wr = init_attr->cap.max_send_wr; 556 + parms.rqueue.max_wr = init_attr->cap.max_recv_wr; 557 + parms.squeue.max_sge = max_send_sge; 558 + parms.rqueue.max_sge = max_recv_sge; 559 + 560 + if (EHCA_BMASK_GET(HCA_CAP_MINI_QP, shca->hca_cap) 561 + && !(context && udata)) { /* no small QP support in userspace ATM */ 562 + ehca_determine_small_queue( 563 + &parms.squeue, max_send_sge, is_llqp); 564 + ehca_determine_small_queue( 565 + &parms.rqueue, max_recv_sge, is_llqp); 566 + parms.qp_storage = 567 + (parms.squeue.is_small || parms.rqueue.is_small); 568 + } 600 569 601 570 h_ret = hipz_h_alloc_resource_qp(shca->ipz_hca_handle, &parms); 602 571 if (h_ret != H_SUCCESS) { ··· 620 569 my_qp->ipz_qp_handle = parms.qp_handle; 621 570 my_qp->galpas = parms.galpas; 622 571 572 + swqe_size = ehca_calc_wqe_size(parms.squeue.act_nr_sges, is_llqp); 573 + rwqe_size = ehca_calc_wqe_size(parms.rqueue.act_nr_sges, is_llqp); 574 + 623 575 switch (qp_type) { 624 576 case IB_QPT_RC: 625 - if (!is_llqp) { 626 - swqe_size = offsetof(struct ehca_wqe, u.nud.sg_list[ 627 - (parms.act_nr_send_sges)]); 628 - rwqe_size = offsetof(struct ehca_wqe, u.nud.sg_list[ 629 - (parms.act_nr_recv_sges)]); 630 - } else { /* for LLQP we need to use msg size, not wqe size */ 631 - swqe_size = ll_qp_msg_size(max_send_sge); 632 - rwqe_size = ll_qp_msg_size(max_recv_sge); 633 - parms.act_nr_send_sges = 1; 634 - parms.act_nr_recv_sges = 1; 577 + if (is_llqp) { 578 + parms.squeue.act_nr_sges = 1; 579 + parms.rqueue.act_nr_sges = 1; 635 580 } 636 581 break; 637 - case IB_QPT_UC: 638 - swqe_size = offsetof(struct ehca_wqe, 639 - u.nud.sg_list[parms.act_nr_send_sges]); 640 - rwqe_size = offsetof(struct ehca_wqe, 641 - u.nud.sg_list[parms.act_nr_recv_sges]); 642 - break; 643 - 644 582 case IB_QPT_UD: 645 583 case IB_QPT_GSI: 646 584 case IB_QPT_SMI: 585 + /* UD circumvention */ 647 586 if (is_llqp) { 648 - swqe_size = ll_qp_msg_size(parms.act_nr_send_sges); 649 - rwqe_size = ll_qp_msg_size(parms.act_nr_recv_sges); 650 - parms.act_nr_send_sges = 1; 651 - parms.act_nr_recv_sges = 1; 587 + parms.squeue.act_nr_sges = 1; 588 + parms.rqueue.act_nr_sges = 1; 652 589 } else { 653 - /* UD circumvention */ 654 - parms.act_nr_send_sges -= 2; 655 - parms.act_nr_recv_sges -= 2; 656 - swqe_size = offsetof(struct ehca_wqe, u.ud_av.sg_list[ 657 - parms.act_nr_send_sges]); 658 - rwqe_size = offsetof(struct ehca_wqe, u.ud_av.sg_list[ 659 - parms.act_nr_recv_sges]); 590 + parms.squeue.act_nr_sges -= 2; 591 + parms.rqueue.act_nr_sges -= 2; 660 592 } 661 593 662 594 if (IB_QPT_GSI == qp_type || IB_QPT_SMI == qp_type) { 663 - parms.act_nr_send_wqes = init_attr->cap.max_send_wr; 664 - parms.act_nr_recv_wqes = init_attr->cap.max_recv_wr; 665 - parms.act_nr_send_sges = init_attr->cap.max_send_sge; 666 - parms.act_nr_recv_sges = init_attr->cap.max_recv_sge; 595 + parms.squeue.act_nr_wqes = init_attr->cap.max_send_wr; 596 + parms.rqueue.act_nr_wqes = init_attr->cap.max_recv_wr; 597 + parms.squeue.act_nr_sges = init_attr->cap.max_send_sge; 598 + parms.rqueue.act_nr_sges = init_attr->cap.max_recv_sge; 667 599 ib_qp_num = (qp_type == IB_QPT_SMI) ? 0 : 1; 668 600 } 669 601 ··· 659 625 /* initialize r/squeue and register queue pages */ 660 626 if (HAS_SQ(my_qp)) { 661 627 ret = init_qp_queue( 662 - shca, my_qp, &my_qp->ipz_squeue, 0, 628 + shca, my_pd, my_qp, &my_qp->ipz_squeue, 0, 663 629 HAS_RQ(my_qp) ? H_PAGE_REGISTERED : H_SUCCESS, 664 - parms.nr_sq_pages, swqe_size, 665 - parms.act_nr_send_sges); 630 + &parms.squeue, swqe_size); 666 631 if (ret) { 667 632 ehca_err(pd->device, "Couldn't initialize squeue " 668 633 "and pages ret=%x", ret); ··· 671 638 672 639 if (HAS_RQ(my_qp)) { 673 640 ret = init_qp_queue( 674 - shca, my_qp, &my_qp->ipz_rqueue, 1, 675 - H_SUCCESS, parms.nr_rq_pages, rwqe_size, 676 - parms.act_nr_recv_sges); 641 + shca, my_pd, my_qp, &my_qp->ipz_rqueue, 1, 642 + H_SUCCESS, &parms.rqueue, rwqe_size); 677 643 if (ret) { 678 644 ehca_err(pd->device, "Couldn't initialize rqueue " 679 645 "and pages ret=%x", ret); ··· 702 670 } 703 671 704 672 init_attr->cap.max_inline_data = 0; /* not supported yet */ 705 - init_attr->cap.max_recv_sge = parms.act_nr_recv_sges; 706 - init_attr->cap.max_recv_wr = parms.act_nr_recv_wqes; 707 - init_attr->cap.max_send_sge = parms.act_nr_send_sges; 708 - init_attr->cap.max_send_wr = parms.act_nr_send_wqes; 673 + init_attr->cap.max_recv_sge = parms.rqueue.act_nr_sges; 674 + init_attr->cap.max_recv_wr = parms.rqueue.act_nr_wqes; 675 + init_attr->cap.max_send_sge = parms.squeue.act_nr_sges; 676 + init_attr->cap.max_send_wr = parms.squeue.act_nr_wqes; 709 677 my_qp->init_attr = *init_attr; 710 678 711 679 /* NOTE: define_apq0() not supported yet */ ··· 739 707 resp.ext_type = my_qp->ext_type; 740 708 resp.qkey = my_qp->qkey; 741 709 resp.real_qp_num = my_qp->real_qp_num; 710 + resp.ipz_rqueue.offset = my_qp->ipz_rqueue.offset; 711 + resp.ipz_squeue.offset = my_qp->ipz_squeue.offset; 742 712 if (HAS_SQ(my_qp)) 743 713 queue2resp(&resp.ipz_squeue, &my_qp->ipz_squeue); 744 714 if (HAS_RQ(my_qp)) ··· 757 723 758 724 create_qp_exit4: 759 725 if (HAS_RQ(my_qp)) 760 - ipz_queue_dtor(&my_qp->ipz_rqueue); 726 + ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue); 761 727 762 728 create_qp_exit3: 763 729 if (HAS_SQ(my_qp)) 764 - ipz_queue_dtor(&my_qp->ipz_squeue); 730 + ipz_queue_dtor(my_pd, &my_qp->ipz_squeue); 765 731 766 732 create_qp_exit2: 767 733 hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp); ··· 786 752 return IS_ERR(ret) ? (struct ib_qp *)ret : &ret->ib_qp; 787 753 } 788 754 789 - int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp, 790 - struct ib_uobject *uobject); 755 + static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp, 756 + struct ib_uobject *uobject); 791 757 792 758 struct ib_srq *ehca_create_srq(struct ib_pd *pd, 793 759 struct ib_srq_init_attr *srq_init_attr, ··· 1703 1669 return ret; 1704 1670 } 1705 1671 1706 - int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp, 1707 - struct ib_uobject *uobject) 1672 + static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp, 1673 + struct ib_uobject *uobject) 1708 1674 { 1709 1675 struct ehca_shca *shca = container_of(dev, struct ehca_shca, ib_device); 1710 1676 struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd, ··· 1768 1734 } 1769 1735 1770 1736 if (HAS_RQ(my_qp)) 1771 - ipz_queue_dtor(&my_qp->ipz_rqueue); 1737 + ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue); 1772 1738 if (HAS_SQ(my_qp)) 1773 - ipz_queue_dtor(&my_qp->ipz_squeue); 1739 + ipz_queue_dtor(my_pd, &my_qp->ipz_squeue); 1774 1740 kmem_cache_free(qp_cache, my_qp); 1775 1741 return 0; 1776 1742 }
+1 -18
drivers/infiniband/hw/ehca/ehca_tools.h
··· 154 154 #define EHCA_BMASK_GET(mask, value) \ 155 155 (EHCA_BMASK_MASK(mask) & (((u64)(value)) >> EHCA_BMASK_SHIFTPOS(mask))) 156 156 157 - 158 157 /* Converts ehca to ib return code */ 159 - static inline int ehca2ib_return_code(u64 ehca_rc) 160 - { 161 - switch (ehca_rc) { 162 - case H_SUCCESS: 163 - return 0; 164 - case H_RESOURCE: /* Resource in use */ 165 - case H_BUSY: 166 - return -EBUSY; 167 - case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */ 168 - case H_CONSTRAINED: /* resource constraint */ 169 - case H_NO_MEM: 170 - return -ENOMEM; 171 - default: 172 - return -EINVAL; 173 - } 174 - } 175 - 158 + int ehca2ib_return_code(u64 ehca_rc); 176 159 177 160 #endif /* EHCA_TOOLS_H */
+1 -1
drivers/infiniband/hw/ehca/ehca_uverbs.c
··· 149 149 ehca_gen_err("vm_insert_page() failed rc=%x", ret); 150 150 return ret; 151 151 } 152 - start += PAGE_SIZE; 152 + start += PAGE_SIZE; 153 153 } 154 154 vma->vm_private_data = mm_count; 155 155 (*mm_count)++;
+38 -12
drivers/infiniband/hw/ehca/hcp_if.c
··· 52 52 #define H_ALL_RES_QP_ENHANCED_OPS EHCA_BMASK_IBM(9, 11) 53 53 #define H_ALL_RES_QP_PTE_PIN EHCA_BMASK_IBM(12, 12) 54 54 #define H_ALL_RES_QP_SERVICE_TYPE EHCA_BMASK_IBM(13, 15) 55 + #define H_ALL_RES_QP_STORAGE EHCA_BMASK_IBM(16, 17) 55 56 #define H_ALL_RES_QP_LL_RQ_CQE_POSTING EHCA_BMASK_IBM(18, 18) 56 57 #define H_ALL_RES_QP_LL_SQ_CQE_POSTING EHCA_BMASK_IBM(19, 21) 57 58 #define H_ALL_RES_QP_SIGNALING_TYPE EHCA_BMASK_IBM(22, 23) 58 59 #define H_ALL_RES_QP_UD_AV_LKEY_CTRL EHCA_BMASK_IBM(31, 31) 60 + #define H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE EHCA_BMASK_IBM(32, 35) 61 + #define H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE EHCA_BMASK_IBM(36, 39) 59 62 #define H_ALL_RES_QP_RESOURCE_TYPE EHCA_BMASK_IBM(56, 63) 60 63 61 64 #define H_ALL_RES_QP_MAX_OUTST_SEND_WR EHCA_BMASK_IBM(0, 15) ··· 302 299 | EHCA_BMASK_SET(H_ALL_RES_QP_PTE_PIN, 0) 303 300 | EHCA_BMASK_SET(H_ALL_RES_QP_SERVICE_TYPE, parms->servicetype) 304 301 | EHCA_BMASK_SET(H_ALL_RES_QP_SIGNALING_TYPE, parms->sigtype) 302 + | EHCA_BMASK_SET(H_ALL_RES_QP_STORAGE, parms->qp_storage) 303 + | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE, 304 + parms->squeue.page_size) 305 + | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE, 306 + parms->rqueue.page_size) 305 307 | EHCA_BMASK_SET(H_ALL_RES_QP_LL_RQ_CQE_POSTING, 306 308 !!(parms->ll_comp_flags & LLQP_RECV_COMP)) 307 309 | EHCA_BMASK_SET(H_ALL_RES_QP_LL_SQ_CQE_POSTING, ··· 317 309 318 310 max_r10_reg = 319 311 EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_SEND_WR, 320 - parms->max_send_wr + 1) 312 + parms->squeue.max_wr + 1) 321 313 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_RECV_WR, 322 - parms->max_recv_wr + 1) 314 + parms->rqueue.max_wr + 1) 323 315 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_SEND_SGE, 324 - parms->max_send_sge) 316 + parms->squeue.max_sge) 325 317 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_RECV_SGE, 326 - parms->max_recv_sge); 318 + parms->rqueue.max_sge); 327 319 328 320 r11 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QP_TOKEN, parms->srq_token); 329 321 ··· 343 335 344 336 parms->qp_handle.handle = outs[0]; 345 337 parms->real_qp_num = (u32)outs[1]; 346 - parms->act_nr_send_wqes = 338 + parms->squeue.act_nr_wqes = 347 339 (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_SEND_WR, outs[2]); 348 - parms->act_nr_recv_wqes = 340 + parms->rqueue.act_nr_wqes = 349 341 (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_RECV_WR, outs[2]); 350 - parms->act_nr_send_sges = 342 + parms->squeue.act_nr_sges = 351 343 (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_SEND_SGE, outs[3]); 352 - parms->act_nr_recv_sges = 344 + parms->rqueue.act_nr_sges = 353 345 (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_RECV_SGE, outs[3]); 354 - parms->nr_sq_pages = 346 + parms->squeue.queue_size = 355 347 (u32)EHCA_BMASK_GET(H_ALL_RES_QP_SQUEUE_SIZE_PAGES, outs[4]); 356 - parms->nr_rq_pages = 348 + parms->rqueue.queue_size = 357 349 (u32)EHCA_BMASK_GET(H_ALL_RES_QP_RQUEUE_SIZE_PAGES, outs[4]); 358 350 359 351 if (ret == H_SUCCESS) ··· 435 427 { 436 428 return ehca_plpar_hcall_norets(H_REGISTER_RPAGES, 437 429 adapter_handle.handle, /* r4 */ 438 - queue_type | pagesize << 8, /* r5 */ 430 + (u64)queue_type | ((u64)pagesize) << 8, 431 + /* r5 */ 439 432 resource_handle, /* r6 */ 440 433 logical_address_of_page, /* r7 */ 441 434 count, /* r8 */ ··· 505 496 const u64 count, 506 497 const struct h_galpa galpa) 507 498 { 508 - if (count != 1) { 499 + if (count > 1) { 509 500 ehca_gen_err("Page counter=%lx", count); 510 501 return H_PARAMETER; 511 502 } ··· 733 724 u64 ret; 734 725 u64 outs[PLPAR_HCALL9_BUFSIZE]; 735 726 727 + ehca_gen_dbg("kernel PAGE_SIZE=%x access_ctrl=%016x " 728 + "vaddr=%lx length=%lx", 729 + (u32)PAGE_SIZE, access_ctrl, vaddr, length); 736 730 ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs, 737 731 adapter_handle.handle, /* r4 */ 738 732 5, /* r5 */ ··· 758 746 const u64 logical_address_of_page, 759 747 const u64 count) 760 748 { 749 + extern int ehca_debug_level; 761 750 u64 ret; 751 + 752 + if (unlikely(ehca_debug_level >= 2)) { 753 + if (count > 1) { 754 + u64 *kpage; 755 + int i; 756 + kpage = (u64 *)abs_to_virt(logical_address_of_page); 757 + for (i = 0; i < count; i++) 758 + ehca_gen_dbg("kpage[%d]=%p", 759 + i, (void *)kpage[i]); 760 + } else 761 + ehca_gen_dbg("kpage=%p", 762 + (void *)logical_address_of_page); 763 + } 762 764 763 765 if ((count > 1) && (logical_address_of_page & (EHCA_PAGESIZE-1))) { 764 766 ehca_gen_err("logical_address_of_page not on a 4k boundary "
+175 -55
drivers/infiniband/hw/ehca/ipz_pt_fn.c
··· 40 40 41 41 #include "ehca_tools.h" 42 42 #include "ipz_pt_fn.h" 43 + #include "ehca_classes.h" 44 + 45 + #define PAGES_PER_KPAGE (PAGE_SIZE >> EHCA_PAGESHIFT) 46 + 47 + struct kmem_cache *small_qp_cache; 43 48 44 49 void *ipz_qpageit_get_inc(struct ipz_queue *queue) 45 50 { ··· 54 49 queue->current_q_offset -= queue->pagesize; 55 50 ret = NULL; 56 51 } 57 - if (((u64)ret) % EHCA_PAGESIZE) { 52 + if (((u64)ret) % queue->pagesize) { 58 53 ehca_gen_err("ERROR!! not at PAGE-Boundary"); 59 54 return NULL; 60 55 } ··· 88 83 return -EINVAL; 89 84 } 90 85 91 - int ipz_queue_ctor(struct ipz_queue *queue, 92 - const u32 nr_of_pages, 93 - const u32 pagesize, const u32 qe_size, const u32 nr_of_sg) 94 - { 95 - int pages_per_kpage = PAGE_SIZE >> EHCA_PAGESHIFT; 96 - int f; 86 + #if PAGE_SHIFT < EHCA_PAGESHIFT 87 + #error Kernel pages must be at least as large than eHCA pages (4K) ! 88 + #endif 97 89 98 - if (pagesize > PAGE_SIZE) { 99 - ehca_gen_err("FATAL ERROR: pagesize=%x is greater " 100 - "than kernel page size", pagesize); 101 - return 0; 102 - } 103 - if (!pages_per_kpage) { 104 - ehca_gen_err("FATAL ERROR: invalid kernel page size. " 105 - "pages_per_kpage=%x", pages_per_kpage); 106 - return 0; 107 - } 108 - queue->queue_length = nr_of_pages * pagesize; 109 - queue->queue_pages = vmalloc(nr_of_pages * sizeof(void *)); 110 - if (!queue->queue_pages) { 111 - ehca_gen_err("ERROR!! didn't get the memory"); 112 - return 0; 113 - } 114 - memset(queue->queue_pages, 0, nr_of_pages * sizeof(void *)); 115 - /* 116 - * allocate pages for queue: 117 - * outer loop allocates whole kernel pages (page aligned) and 118 - * inner loop divides a kernel page into smaller hca queue pages 119 - */ 120 - f = 0; 90 + /* 91 + * allocate pages for queue: 92 + * outer loop allocates whole kernel pages (page aligned) and 93 + * inner loop divides a kernel page into smaller hca queue pages 94 + */ 95 + static int alloc_queue_pages(struct ipz_queue *queue, const u32 nr_of_pages) 96 + { 97 + int k, f = 0; 98 + u8 *kpage; 99 + 121 100 while (f < nr_of_pages) { 122 - u8 *kpage = (u8 *)get_zeroed_page(GFP_KERNEL); 123 - int k; 101 + kpage = (u8 *)get_zeroed_page(GFP_KERNEL); 124 102 if (!kpage) 125 - goto ipz_queue_ctor_exit0; /*NOMEM*/ 126 - for (k = 0; k < pages_per_kpage && f < nr_of_pages; k++) { 127 - (queue->queue_pages)[f] = (struct ipz_page *)kpage; 103 + goto out; 104 + 105 + for (k = 0; k < PAGES_PER_KPAGE && f < nr_of_pages; k++) { 106 + queue->queue_pages[f] = (struct ipz_page *)kpage; 128 107 kpage += EHCA_PAGESIZE; 129 108 f++; 130 109 } 131 110 } 132 - 133 - queue->current_q_offset = 0; 134 - queue->qe_size = qe_size; 135 - queue->act_nr_of_sg = nr_of_sg; 136 - queue->pagesize = pagesize; 137 - queue->toggle_state = 1; 138 111 return 1; 139 112 140 - ipz_queue_ctor_exit0: 141 - ehca_gen_err("Couldn't get alloc pages queue=%p f=%x nr_of_pages=%x", 142 - queue, f, nr_of_pages); 143 - for (f = 0; f < nr_of_pages; f += pages_per_kpage) { 144 - if (!(queue->queue_pages)[f]) 145 - break; 113 + out: 114 + for (f = 0; f < nr_of_pages && queue->queue_pages[f]; 115 + f += PAGES_PER_KPAGE) 146 116 free_page((unsigned long)(queue->queue_pages)[f]); 147 - } 148 117 return 0; 149 118 } 150 119 151 - int ipz_queue_dtor(struct ipz_queue *queue) 120 + static int alloc_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd) 152 121 { 153 - int pages_per_kpage = PAGE_SIZE >> EHCA_PAGESHIFT; 154 - int g; 155 - int nr_pages; 122 + int order = ilog2(queue->pagesize) - 9; 123 + struct ipz_small_queue_page *page; 124 + unsigned long bit; 125 + 126 + mutex_lock(&pd->lock); 127 + 128 + if (!list_empty(&pd->free[order])) 129 + page = list_entry(pd->free[order].next, 130 + struct ipz_small_queue_page, list); 131 + else { 132 + page = kmem_cache_zalloc(small_qp_cache, GFP_KERNEL); 133 + if (!page) 134 + goto out; 135 + 136 + page->page = get_zeroed_page(GFP_KERNEL); 137 + if (!page->page) { 138 + kmem_cache_free(small_qp_cache, page); 139 + goto out; 140 + } 141 + 142 + list_add(&page->list, &pd->free[order]); 143 + } 144 + 145 + bit = find_first_zero_bit(page->bitmap, IPZ_SPAGE_PER_KPAGE >> order); 146 + __set_bit(bit, page->bitmap); 147 + page->fill++; 148 + 149 + if (page->fill == IPZ_SPAGE_PER_KPAGE >> order) 150 + list_move(&page->list, &pd->full[order]); 151 + 152 + mutex_unlock(&pd->lock); 153 + 154 + queue->queue_pages[0] = (void *)(page->page | (bit << (order + 9))); 155 + queue->small_page = page; 156 + return 1; 157 + 158 + out: 159 + ehca_err(pd->ib_pd.device, "failed to allocate small queue page"); 160 + return 0; 161 + } 162 + 163 + static void free_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd) 164 + { 165 + int order = ilog2(queue->pagesize) - 9; 166 + struct ipz_small_queue_page *page = queue->small_page; 167 + unsigned long bit; 168 + int free_page = 0; 169 + 170 + bit = ((unsigned long)queue->queue_pages[0] & PAGE_MASK) 171 + >> (order + 9); 172 + 173 + mutex_lock(&pd->lock); 174 + 175 + __clear_bit(bit, page->bitmap); 176 + page->fill--; 177 + 178 + if (page->fill == 0) { 179 + list_del(&page->list); 180 + free_page = 1; 181 + } 182 + 183 + if (page->fill == (IPZ_SPAGE_PER_KPAGE >> order) - 1) 184 + /* the page was full until we freed the chunk */ 185 + list_move_tail(&page->list, &pd->free[order]); 186 + 187 + mutex_unlock(&pd->lock); 188 + 189 + if (free_page) { 190 + free_page(page->page); 191 + kmem_cache_free(small_qp_cache, page); 192 + } 193 + } 194 + 195 + int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue, 196 + const u32 nr_of_pages, const u32 pagesize, 197 + const u32 qe_size, const u32 nr_of_sg, 198 + int is_small) 199 + { 200 + if (pagesize > PAGE_SIZE) { 201 + ehca_gen_err("FATAL ERROR: pagesize=%x " 202 + "is greater than kernel page size", pagesize); 203 + return 0; 204 + } 205 + 206 + /* init queue fields */ 207 + queue->queue_length = nr_of_pages * pagesize; 208 + queue->pagesize = pagesize; 209 + queue->qe_size = qe_size; 210 + queue->act_nr_of_sg = nr_of_sg; 211 + queue->current_q_offset = 0; 212 + queue->toggle_state = 1; 213 + queue->small_page = NULL; 214 + 215 + /* allocate queue page pointers */ 216 + queue->queue_pages = vmalloc(nr_of_pages * sizeof(void *)); 217 + if (!queue->queue_pages) { 218 + ehca_gen_err("Couldn't allocate queue page list"); 219 + return 0; 220 + } 221 + memset(queue->queue_pages, 0, nr_of_pages * sizeof(void *)); 222 + 223 + /* allocate actual queue pages */ 224 + if (is_small) { 225 + if (!alloc_small_queue_page(queue, pd)) 226 + goto ipz_queue_ctor_exit0; 227 + } else 228 + if (!alloc_queue_pages(queue, nr_of_pages)) 229 + goto ipz_queue_ctor_exit0; 230 + 231 + return 1; 232 + 233 + ipz_queue_ctor_exit0: 234 + ehca_gen_err("Couldn't alloc pages queue=%p " 235 + "nr_of_pages=%x", queue, nr_of_pages); 236 + vfree(queue->queue_pages); 237 + 238 + return 0; 239 + } 240 + 241 + int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue) 242 + { 243 + int i, nr_pages; 156 244 157 245 if (!queue || !queue->queue_pages) { 158 246 ehca_gen_dbg("queue or queue_pages is NULL"); 159 247 return 0; 160 248 } 161 - nr_pages = queue->queue_length / queue->pagesize; 162 - for (g = 0; g < nr_pages; g += pages_per_kpage) 163 - free_page((unsigned long)(queue->queue_pages)[g]); 249 + 250 + if (queue->small_page) 251 + free_small_queue_page(queue, pd); 252 + else { 253 + nr_pages = queue->queue_length / queue->pagesize; 254 + for (i = 0; i < nr_pages; i += PAGES_PER_KPAGE) 255 + free_page((unsigned long)queue->queue_pages[i]); 256 + } 257 + 164 258 vfree(queue->queue_pages); 165 259 166 260 return 1; 261 + } 262 + 263 + int ehca_init_small_qp_cache(void) 264 + { 265 + small_qp_cache = kmem_cache_create("ehca_cache_small_qp", 266 + sizeof(struct ipz_small_queue_page), 267 + 0, SLAB_HWCACHE_ALIGN, NULL); 268 + if (!small_qp_cache) 269 + return -ENOMEM; 270 + 271 + return 0; 272 + } 273 + 274 + void ehca_cleanup_small_qp_cache(void) 275 + { 276 + kmem_cache_destroy(small_qp_cache); 167 277 }
+21 -5
drivers/infiniband/hw/ehca/ipz_pt_fn.h
··· 51 51 #include "ehca_tools.h" 52 52 #include "ehca_qes.h" 53 53 54 + struct ehca_pd; 55 + struct ipz_small_queue_page; 56 + 54 57 /* struct generic ehca page */ 55 58 struct ipz_page { 56 59 u8 entries[EHCA_PAGESIZE]; 60 + }; 61 + 62 + #define IPZ_SPAGE_PER_KPAGE (PAGE_SIZE / 512) 63 + 64 + struct ipz_small_queue_page { 65 + unsigned long page; 66 + unsigned long bitmap[IPZ_SPAGE_PER_KPAGE / BITS_PER_LONG]; 67 + int fill; 68 + void *mapped_addr; 69 + u32 mmap_count; 70 + struct list_head list; 57 71 }; 58 72 59 73 /* struct generic queue in linux kernel virtual memory (kv) */ ··· 80 66 u32 queue_length; /* queue length allocated in bytes */ 81 67 u32 pagesize; 82 68 u32 toggle_state; /* toggle flag - per page */ 83 - u32 dummy3; /* 64 bit alignment */ 69 + u32 offset; /* save offset within page for small_qp */ 70 + struct ipz_small_queue_page *small_page; 84 71 }; 85 72 86 73 /* ··· 203 188 * see ipz_qpt_ctor() 204 189 * returns true if ok, false if out of memory 205 190 */ 206 - int ipz_queue_ctor(struct ipz_queue *queue, const u32 nr_of_pages, 207 - const u32 pagesize, const u32 qe_size, 208 - const u32 nr_of_sg); 191 + int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue, 192 + const u32 nr_of_pages, const u32 pagesize, 193 + const u32 qe_size, const u32 nr_of_sg, 194 + int is_small); 209 195 210 196 /* 211 197 * destructor for a ipz_queue_t ··· 214 198 * see ipz_queue_ctor() 215 199 * returns true if ok, false if queue was NULL-ptr of free failed 216 200 */ 217 - int ipz_queue_dtor(struct ipz_queue *queue); 201 + int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue); 218 202 219 203 /* 220 204 * constructor for a ipz_qpt_t,
-1
drivers/infiniband/hw/ipath/Makefile
··· 14 14 ipath_init_chip.o \ 15 15 ipath_intr.o \ 16 16 ipath_keys.o \ 17 - ipath_layer.o \ 18 17 ipath_mad.o \ 19 18 ipath_mmap.o \ 20 19 ipath_mr.o \
-365
drivers/infiniband/hw/ipath/ipath_layer.c
··· 1 - /* 2 - * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved. 3 - * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved. 4 - * 5 - * This software is available to you under a choice of one of two 6 - * licenses. You may choose to be licensed under the terms of the GNU 7 - * General Public License (GPL) Version 2, available from the file 8 - * COPYING in the main directory of this source tree, or the 9 - * OpenIB.org BSD license below: 10 - * 11 - * Redistribution and use in source and binary forms, with or 12 - * without modification, are permitted provided that the following 13 - * conditions are met: 14 - * 15 - * - Redistributions of source code must retain the above 16 - * copyright notice, this list of conditions and the following 17 - * disclaimer. 18 - * 19 - * - Redistributions in binary form must reproduce the above 20 - * copyright notice, this list of conditions and the following 21 - * disclaimer in the documentation and/or other materials 22 - * provided with the distribution. 23 - * 24 - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 - * SOFTWARE. 32 - */ 33 - 34 - /* 35 - * These are the routines used by layered drivers, currently just the 36 - * layered ethernet driver and verbs layer. 37 - */ 38 - 39 - #include <linux/io.h> 40 - #include <asm/byteorder.h> 41 - 42 - #include "ipath_kernel.h" 43 - #include "ipath_layer.h" 44 - #include "ipath_verbs.h" 45 - #include "ipath_common.h" 46 - 47 - /* Acquire before ipath_devs_lock. */ 48 - static DEFINE_MUTEX(ipath_layer_mutex); 49 - 50 - u16 ipath_layer_rcv_opcode; 51 - 52 - static int (*layer_intr)(void *, u32); 53 - static int (*layer_rcv)(void *, void *, struct sk_buff *); 54 - static int (*layer_rcv_lid)(void *, void *); 55 - 56 - static void *(*layer_add_one)(int, struct ipath_devdata *); 57 - static void (*layer_remove_one)(void *); 58 - 59 - int __ipath_layer_intr(struct ipath_devdata *dd, u32 arg) 60 - { 61 - int ret = -ENODEV; 62 - 63 - if (dd->ipath_layer.l_arg && layer_intr) 64 - ret = layer_intr(dd->ipath_layer.l_arg, arg); 65 - 66 - return ret; 67 - } 68 - 69 - int ipath_layer_intr(struct ipath_devdata *dd, u32 arg) 70 - { 71 - int ret; 72 - 73 - mutex_lock(&ipath_layer_mutex); 74 - 75 - ret = __ipath_layer_intr(dd, arg); 76 - 77 - mutex_unlock(&ipath_layer_mutex); 78 - 79 - return ret; 80 - } 81 - 82 - int __ipath_layer_rcv(struct ipath_devdata *dd, void *hdr, 83 - struct sk_buff *skb) 84 - { 85 - int ret = -ENODEV; 86 - 87 - if (dd->ipath_layer.l_arg && layer_rcv) 88 - ret = layer_rcv(dd->ipath_layer.l_arg, hdr, skb); 89 - 90 - return ret; 91 - } 92 - 93 - int __ipath_layer_rcv_lid(struct ipath_devdata *dd, void *hdr) 94 - { 95 - int ret = -ENODEV; 96 - 97 - if (dd->ipath_layer.l_arg && layer_rcv_lid) 98 - ret = layer_rcv_lid(dd->ipath_layer.l_arg, hdr); 99 - 100 - return ret; 101 - } 102 - 103 - void ipath_layer_lid_changed(struct ipath_devdata *dd) 104 - { 105 - mutex_lock(&ipath_layer_mutex); 106 - 107 - if (dd->ipath_layer.l_arg && layer_intr) 108 - layer_intr(dd->ipath_layer.l_arg, IPATH_LAYER_INT_LID); 109 - 110 - mutex_unlock(&ipath_layer_mutex); 111 - } 112 - 113 - void ipath_layer_add(struct ipath_devdata *dd) 114 - { 115 - mutex_lock(&ipath_layer_mutex); 116 - 117 - if (layer_add_one) 118 - dd->ipath_layer.l_arg = 119 - layer_add_one(dd->ipath_unit, dd); 120 - 121 - mutex_unlock(&ipath_layer_mutex); 122 - } 123 - 124 - void ipath_layer_remove(struct ipath_devdata *dd) 125 - { 126 - mutex_lock(&ipath_layer_mutex); 127 - 128 - if (dd->ipath_layer.l_arg && layer_remove_one) { 129 - layer_remove_one(dd->ipath_layer.l_arg); 130 - dd->ipath_layer.l_arg = NULL; 131 - } 132 - 133 - mutex_unlock(&ipath_layer_mutex); 134 - } 135 - 136 - int ipath_layer_register(void *(*l_add)(int, struct ipath_devdata *), 137 - void (*l_remove)(void *), 138 - int (*l_intr)(void *, u32), 139 - int (*l_rcv)(void *, void *, struct sk_buff *), 140 - u16 l_rcv_opcode, 141 - int (*l_rcv_lid)(void *, void *)) 142 - { 143 - struct ipath_devdata *dd, *tmp; 144 - unsigned long flags; 145 - 146 - mutex_lock(&ipath_layer_mutex); 147 - 148 - layer_add_one = l_add; 149 - layer_remove_one = l_remove; 150 - layer_intr = l_intr; 151 - layer_rcv = l_rcv; 152 - layer_rcv_lid = l_rcv_lid; 153 - ipath_layer_rcv_opcode = l_rcv_opcode; 154 - 155 - spin_lock_irqsave(&ipath_devs_lock, flags); 156 - 157 - list_for_each_entry_safe(dd, tmp, &ipath_dev_list, ipath_list) { 158 - if (!(dd->ipath_flags & IPATH_INITTED)) 159 - continue; 160 - 161 - if (dd->ipath_layer.l_arg) 162 - continue; 163 - 164 - spin_unlock_irqrestore(&ipath_devs_lock, flags); 165 - dd->ipath_layer.l_arg = l_add(dd->ipath_unit, dd); 166 - spin_lock_irqsave(&ipath_devs_lock, flags); 167 - } 168 - 169 - spin_unlock_irqrestore(&ipath_devs_lock, flags); 170 - mutex_unlock(&ipath_layer_mutex); 171 - 172 - return 0; 173 - } 174 - 175 - EXPORT_SYMBOL_GPL(ipath_layer_register); 176 - 177 - void ipath_layer_unregister(void) 178 - { 179 - struct ipath_devdata *dd, *tmp; 180 - unsigned long flags; 181 - 182 - mutex_lock(&ipath_layer_mutex); 183 - spin_lock_irqsave(&ipath_devs_lock, flags); 184 - 185 - list_for_each_entry_safe(dd, tmp, &ipath_dev_list, ipath_list) { 186 - if (dd->ipath_layer.l_arg && layer_remove_one) { 187 - spin_unlock_irqrestore(&ipath_devs_lock, flags); 188 - layer_remove_one(dd->ipath_layer.l_arg); 189 - spin_lock_irqsave(&ipath_devs_lock, flags); 190 - dd->ipath_layer.l_arg = NULL; 191 - } 192 - } 193 - 194 - spin_unlock_irqrestore(&ipath_devs_lock, flags); 195 - 196 - layer_add_one = NULL; 197 - layer_remove_one = NULL; 198 - layer_intr = NULL; 199 - layer_rcv = NULL; 200 - layer_rcv_lid = NULL; 201 - 202 - mutex_unlock(&ipath_layer_mutex); 203 - } 204 - 205 - EXPORT_SYMBOL_GPL(ipath_layer_unregister); 206 - 207 - int ipath_layer_open(struct ipath_devdata *dd, u32 * pktmax) 208 - { 209 - int ret; 210 - u32 intval = 0; 211 - 212 - mutex_lock(&ipath_layer_mutex); 213 - 214 - if (!dd->ipath_layer.l_arg) { 215 - ret = -EINVAL; 216 - goto bail; 217 - } 218 - 219 - ret = ipath_setrcvhdrsize(dd, IPATH_HEADER_QUEUE_WORDS); 220 - 221 - if (ret < 0) 222 - goto bail; 223 - 224 - *pktmax = dd->ipath_ibmaxlen; 225 - 226 - if (*dd->ipath_statusp & IPATH_STATUS_IB_READY) 227 - intval |= IPATH_LAYER_INT_IF_UP; 228 - if (dd->ipath_lid) 229 - intval |= IPATH_LAYER_INT_LID; 230 - if (dd->ipath_mlid) 231 - intval |= IPATH_LAYER_INT_BCAST; 232 - /* 233 - * do this on open, in case low level is already up and 234 - * just layered driver was reloaded, etc. 235 - */ 236 - if (intval) 237 - layer_intr(dd->ipath_layer.l_arg, intval); 238 - 239 - ret = 0; 240 - bail: 241 - mutex_unlock(&ipath_layer_mutex); 242 - 243 - return ret; 244 - } 245 - 246 - EXPORT_SYMBOL_GPL(ipath_layer_open); 247 - 248 - u16 ipath_layer_get_lid(struct ipath_devdata *dd) 249 - { 250 - return dd->ipath_lid; 251 - } 252 - 253 - EXPORT_SYMBOL_GPL(ipath_layer_get_lid); 254 - 255 - /** 256 - * ipath_layer_get_mac - get the MAC address 257 - * @dd: the infinipath device 258 - * @mac: the MAC is put here 259 - * 260 - * This is the EUID-64 OUI octets (top 3), then 261 - * skip the next 2 (which should both be zero or 0xff). 262 - * The returned MAC is in network order 263 - * mac points to at least 6 bytes of buffer 264 - * We assume that by the time the LID is set, that the GUID is as valid 265 - * as it's ever going to be, rather than adding yet another status bit. 266 - */ 267 - 268 - int ipath_layer_get_mac(struct ipath_devdata *dd, u8 * mac) 269 - { 270 - u8 *guid; 271 - 272 - guid = (u8 *) &dd->ipath_guid; 273 - 274 - mac[0] = guid[0]; 275 - mac[1] = guid[1]; 276 - mac[2] = guid[2]; 277 - mac[3] = guid[5]; 278 - mac[4] = guid[6]; 279 - mac[5] = guid[7]; 280 - if ((guid[3] || guid[4]) && !(guid[3] == 0xff && guid[4] == 0xff)) 281 - ipath_dbg("Warning, guid bytes 3 and 4 not 0 or 0xffff: " 282 - "%x %x\n", guid[3], guid[4]); 283 - return 0; 284 - } 285 - 286 - EXPORT_SYMBOL_GPL(ipath_layer_get_mac); 287 - 288 - u16 ipath_layer_get_bcast(struct ipath_devdata *dd) 289 - { 290 - return dd->ipath_mlid; 291 - } 292 - 293 - EXPORT_SYMBOL_GPL(ipath_layer_get_bcast); 294 - 295 - int ipath_layer_send_hdr(struct ipath_devdata *dd, struct ether_header *hdr) 296 - { 297 - int ret = 0; 298 - u32 __iomem *piobuf; 299 - u32 plen, *uhdr; 300 - size_t count; 301 - __be16 vlsllnh; 302 - 303 - if (!(dd->ipath_flags & IPATH_RCVHDRSZ_SET)) { 304 - ipath_dbg("send while not open\n"); 305 - ret = -EINVAL; 306 - } else 307 - if ((dd->ipath_flags & (IPATH_LINKUNK | IPATH_LINKDOWN)) || 308 - dd->ipath_lid == 0) { 309 - /* 310 - * lid check is for when sma hasn't yet configured 311 - */ 312 - ret = -ENETDOWN; 313 - ipath_cdbg(VERBOSE, "send while not ready, " 314 - "mylid=%u, flags=0x%x\n", 315 - dd->ipath_lid, dd->ipath_flags); 316 - } 317 - 318 - vlsllnh = *((__be16 *) hdr); 319 - if (vlsllnh != htons(IPATH_LRH_BTH)) { 320 - ipath_dbg("Warning: lrh[0] wrong (%x, not %x); " 321 - "not sending\n", be16_to_cpu(vlsllnh), 322 - IPATH_LRH_BTH); 323 - ret = -EINVAL; 324 - } 325 - if (ret) 326 - goto done; 327 - 328 - /* Get a PIO buffer to use. */ 329 - piobuf = ipath_getpiobuf(dd, NULL); 330 - if (piobuf == NULL) { 331 - ret = -EBUSY; 332 - goto done; 333 - } 334 - 335 - plen = (sizeof(*hdr) >> 2); /* actual length */ 336 - ipath_cdbg(EPKT, "0x%x+1w pio %p\n", plen, piobuf); 337 - 338 - writeq(plen+1, piobuf); /* len (+1 for pad) to pbc, no flags */ 339 - ipath_flush_wc(); 340 - piobuf += 2; 341 - uhdr = (u32 *)hdr; 342 - count = plen-1; /* amount we can copy before trigger word */ 343 - __iowrite32_copy(piobuf, uhdr, count); 344 - ipath_flush_wc(); 345 - __raw_writel(uhdr[count], piobuf + count); 346 - ipath_flush_wc(); /* ensure it's sent, now */ 347 - 348 - ipath_stats.sps_ether_spkts++; /* ether packet sent */ 349 - 350 - done: 351 - return ret; 352 - } 353 - 354 - EXPORT_SYMBOL_GPL(ipath_layer_send_hdr); 355 - 356 - int ipath_layer_set_piointbufavail_int(struct ipath_devdata *dd) 357 - { 358 - set_bit(IPATH_S_PIOINTBUFAVAIL, &dd->ipath_sendctrl); 359 - 360 - ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, 361 - dd->ipath_sendctrl); 362 - return 0; 363 - } 364 - 365 - EXPORT_SYMBOL_GPL(ipath_layer_set_piointbufavail_int);
-71
drivers/infiniband/hw/ipath/ipath_layer.h
··· 1 - /* 2 - * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved. 3 - * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved. 4 - * 5 - * This software is available to you under a choice of one of two 6 - * licenses. You may choose to be licensed under the terms of the GNU 7 - * General Public License (GPL) Version 2, available from the file 8 - * COPYING in the main directory of this source tree, or the 9 - * OpenIB.org BSD license below: 10 - * 11 - * Redistribution and use in source and binary forms, with or 12 - * without modification, are permitted provided that the following 13 - * conditions are met: 14 - * 15 - * - Redistributions of source code must retain the above 16 - * copyright notice, this list of conditions and the following 17 - * disclaimer. 18 - * 19 - * - Redistributions in binary form must reproduce the above 20 - * copyright notice, this list of conditions and the following 21 - * disclaimer in the documentation and/or other materials 22 - * provided with the distribution. 23 - * 24 - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 - * SOFTWARE. 32 - */ 33 - 34 - #ifndef _IPATH_LAYER_H 35 - #define _IPATH_LAYER_H 36 - 37 - /* 38 - * This header file is for symbols shared between the infinipath driver 39 - * and drivers layered upon it (such as ipath). 40 - */ 41 - 42 - struct sk_buff; 43 - struct ipath_devdata; 44 - struct ether_header; 45 - 46 - int ipath_layer_register(void *(*l_add)(int, struct ipath_devdata *), 47 - void (*l_remove)(void *), 48 - int (*l_intr)(void *, u32), 49 - int (*l_rcv)(void *, void *, 50 - struct sk_buff *), 51 - u16 rcv_opcode, 52 - int (*l_rcv_lid)(void *, void *)); 53 - void ipath_layer_unregister(void); 54 - int ipath_layer_open(struct ipath_devdata *, u32 * pktmax); 55 - u16 ipath_layer_get_lid(struct ipath_devdata *dd); 56 - int ipath_layer_get_mac(struct ipath_devdata *dd, u8 *); 57 - u16 ipath_layer_get_bcast(struct ipath_devdata *dd); 58 - int ipath_layer_send_hdr(struct ipath_devdata *dd, 59 - struct ether_header *hdr); 60 - int ipath_layer_set_piointbufavail_int(struct ipath_devdata *dd); 61 - 62 - /* ipath_ether interrupt values */ 63 - #define IPATH_LAYER_INT_IF_UP 0x2 64 - #define IPATH_LAYER_INT_IF_DOWN 0x4 65 - #define IPATH_LAYER_INT_LID 0x8 66 - #define IPATH_LAYER_INT_SEND_CONTINUE 0x10 67 - #define IPATH_LAYER_INT_BCAST 0x40 68 - 69 - extern unsigned ipath_debug; /* debugging bit mask */ 70 - 71 - #endif /* _IPATH_LAYER_H */
-2
drivers/infiniband/hw/ipath/ipath_verbs.h
··· 42 42 #include <rdma/ib_pack.h> 43 43 #include <rdma/ib_user_verbs.h> 44 44 45 - #include "ipath_layer.h" 46 - 47 45 #define IPATH_MAX_RDMA_ATOMIC 4 48 46 49 47 #define QPN_MAX (1 << 24)
+10 -10
drivers/infiniband/hw/mlx4/qp.c
··· 415 415 return 0; 416 416 417 417 err_wrid: 418 - if (pd->uobject && !init_attr->srq) 419 - mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db); 420 - else { 418 + if (pd->uobject) { 419 + if (!init_attr->srq) 420 + mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), 421 + &qp->db); 422 + } else { 421 423 kfree(qp->sq.wrid); 422 424 kfree(qp->rq.wrid); 423 425 } ··· 744 742 if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) { 745 743 printk(KERN_ERR "path MTU (%u) is invalid\n", 746 744 attr->path_mtu); 747 - return -EINVAL; 745 + goto out; 748 746 } 749 747 context->mtu_msgmax = (attr->path_mtu << 5) | 31; 750 748 } ··· 783 781 784 782 if (attr_mask & IB_QP_AV) { 785 783 if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path, 786 - attr_mask & IB_QP_PORT ? attr->port_num : qp->port)) { 787 - err = -EINVAL; 784 + attr_mask & IB_QP_PORT ? attr->port_num : qp->port)) 788 785 goto out; 789 - } 790 786 791 787 optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH | 792 788 MLX4_QP_OPTPAR_SCHED_QUEUE); ··· 798 798 if (attr_mask & IB_QP_ALT_PATH) { 799 799 if (attr->alt_port_num == 0 || 800 800 attr->alt_port_num > dev->dev->caps.num_ports) 801 - return -EINVAL; 801 + goto out; 802 802 803 803 if (attr->alt_pkey_index >= 804 804 dev->dev->caps.pkey_table_len[attr->alt_port_num]) 805 - return -EINVAL; 805 + goto out; 806 806 807 807 if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path, 808 808 attr->alt_port_num)) 809 - return -EINVAL; 809 + goto out; 810 810 811 811 context->alt_path.pkey_index = attr->alt_pkey_index; 812 812 context->alt_path.ackto = attr->alt_timeout << 3;
+1 -2
drivers/infiniband/hw/mthca/mthca_cmd.c
··· 357 357 context->status = status; 358 358 context->out_param = out_param; 359 359 360 - context->token += dev->cmd.token_mask + 1; 361 - 362 360 complete(&context->done); 363 361 } 364 362 ··· 378 380 spin_lock(&dev->cmd.context_lock); 379 381 BUG_ON(dev->cmd.free_head < 0); 380 382 context = &dev->cmd.context[dev->cmd.free_head]; 383 + context->token += dev->cmd.token_mask + 1; 381 384 dev->cmd.free_head = context->next; 382 385 spin_unlock(&dev->cmd.context_lock); 383 386
+1 -2
drivers/net/mlx4/cmd.c
··· 246 246 context->result = mlx4_status_to_errno(status); 247 247 context->out_param = out_param; 248 248 249 - context->token += priv->cmd.token_mask + 1; 250 - 251 249 complete(&context->done); 252 250 } 253 251 ··· 262 264 spin_lock(&cmd->context_lock); 263 265 BUG_ON(cmd->free_head < 0); 264 266 context = &cmd->context[cmd->free_head]; 267 + context->token += cmd->token_mask + 1; 265 268 cmd->free_head = context->next; 266 269 spin_unlock(&cmd->context_lock); 267 270