+20 -58

drivers/infiniband/core/cm.c

reviewed

··· 36 36 37 37 #define CM_DESTROY_ID_WAIT_TIMEOUT 10000 /* msecs */ 38 38 #define CM_DIRECT_RETRY_CTX ((void *) 1UL) 39 39 + #define CM_MRA_SETTING 24 /* 4.096us * 2^24 = ~68.7 seconds */ 39 40 40 41 static const char * const ibcm_rej_reason_strs[] = { 41 42 [IB_CM_REJ_NO_QP] = "no QP", ··· 168 167 struct cm_device { 169 168 struct kref kref; 170 169 struct list_head list; 171 171 - spinlock_t mad_agent_lock; 170 170 + rwlock_t mad_agent_lock; 172 171 struct ib_device *ib_device; 173 172 u8 ack_delay; 174 173 int going_down; ··· 242 241 u8 initiator_depth; 243 242 u8 retry_count; 244 243 u8 rnr_retry_count; 245 245 - u8 service_timeout; 246 244 u8 target_ack_delay; 247 245 248 246 struct list_head work_list; ··· 285 285 if (!cm_id_priv->av.port) 286 286 return ERR_PTR(-EINVAL); 287 287 288 288 - spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); 288 288 + read_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); 289 289 mad_agent = cm_id_priv->av.port->mad_agent; 290 290 if (!mad_agent) { 291 291 m = ERR_PTR(-EINVAL); ··· 311 311 m->ah = ah; 312 312 313 313 out: 314 314 - spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); 314 314 + read_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); 315 315 return m; 316 316 } 317 317 ··· 1297 1297 if (!cm_id_priv->av.port) 1298 1298 return cpu_to_be64(low_tid); 1299 1299 1300 1300 - spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); 1300 1300 + read_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); 1301 1301 if (cm_id_priv->av.port->mad_agent) 1302 1302 hi_tid = ((u64)cm_id_priv->av.port->mad_agent->hi_tid) << 32; 1303 1303 - spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); 1303 1303 + read_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); 1304 1304 return cpu_to_be64(hi_tid | low_tid); 1305 1305 } 1306 1306 ··· 1872 1872 1873 1873 static void cm_format_mra(struct cm_mra_msg *mra_msg, 1874 1874 struct cm_id_private *cm_id_priv, 1875 1875 - enum cm_msg_response msg_mraed, u8 service_timeout, 1875 1875 + enum cm_msg_response msg_mraed, 1876 1876 const void *private_data, u8 private_data_len) 1877 1877 { 1878 1878 cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid); ··· 1881 1881 be32_to_cpu(cm_id_priv->id.local_id)); 1882 1882 IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg, 1883 1883 be32_to_cpu(cm_id_priv->id.remote_id)); 1884 1884 - IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout); 1884 1884 + IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, CM_MRA_SETTING); 1885 1885 1886 1886 if (private_data && private_data_len) 1887 1887 IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data, ··· 1960 1960 switch (cm_id_priv->id.state) { 1961 1961 case IB_CM_MRA_REQ_SENT: 1962 1962 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 1963 1963 - CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout, 1963 1963 + CM_MSG_RESPONSE_REQ, 1964 1964 cm_id_priv->private_data, 1965 1965 cm_id_priv->private_data_len); 1966 1966 break; ··· 2454 2454 cm_id_priv->private_data_len); 2455 2455 else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT) 2456 2456 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 2457 2457 - CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout, 2457 2457 + CM_MSG_RESPONSE_REP, 2458 2458 cm_id_priv->private_data, 2459 2459 cm_id_priv->private_data_len); 2460 2460 else ··· 3094 3094 return -EINVAL; 3095 3095 } 3096 3096 3097 3097 - int ib_send_cm_mra(struct ib_cm_id *cm_id, 3098 3098 - u8 service_timeout, 3099 3099 - const void *private_data, 3100 3100 - u8 private_data_len) 3097 3097 + int ib_prepare_cm_mra(struct ib_cm_id *cm_id) 3101 3098 { 3102 3099 struct cm_id_private *cm_id_priv; 3103 3103 - struct ib_mad_send_buf *msg; 3104 3100 enum ib_cm_state cm_state; 3105 3101 enum ib_cm_lap_state lap_state; 3106 3106 - enum cm_msg_response msg_response; 3107 3107 - void *data; 3108 3102 unsigned long flags; 3109 3109 - int ret; 3110 3110 - 3111 3111 - if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE) 3112 3112 - return -EINVAL; 3113 3113 - 3114 3114 - data = cm_copy_private_data(private_data, private_data_len); 3115 3115 - if (IS_ERR(data)) 3116 3116 - return PTR_ERR(data); 3103 3103 + int ret = 0; 3117 3104 3118 3105 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 3119 3106 ··· 3109 3122 case IB_CM_REQ_RCVD: 3110 3123 cm_state = IB_CM_MRA_REQ_SENT; 3111 3124 lap_state = cm_id->lap_state; 3112 3112 - msg_response = CM_MSG_RESPONSE_REQ; 3113 3125 break; 3114 3126 case IB_CM_REP_RCVD: 3115 3127 cm_state = IB_CM_MRA_REP_SENT; 3116 3128 lap_state = cm_id->lap_state; 3117 3117 - msg_response = CM_MSG_RESPONSE_REP; 3118 3129 break; 3119 3130 case IB_CM_ESTABLISHED: 3120 3131 if (cm_id->lap_state == IB_CM_LAP_RCVD) { 3121 3132 cm_state = cm_id->state; 3122 3133 lap_state = IB_CM_MRA_LAP_SENT; 3123 3123 - msg_response = CM_MSG_RESPONSE_OTHER; 3124 3134 break; 3125 3135 } 3126 3136 fallthrough; 3127 3137 default: 3128 3128 - trace_icm_send_mra_unknown_err(&cm_id_priv->id); 3138 3138 + trace_icm_prepare_mra_unknown_err(&cm_id_priv->id); 3129 3139 ret = -EINVAL; 3130 3140 goto error_unlock; 3131 3141 } 3132 3142 3133 3133 - if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) { 3134 3134 - msg = cm_alloc_msg(cm_id_priv); 3135 3135 - if (IS_ERR(msg)) { 3136 3136 - ret = PTR_ERR(msg); 3137 3137 - goto error_unlock; 3138 3138 - } 3139 3139 - 3140 3140 - cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 3141 3141 - msg_response, service_timeout, 3142 3142 - private_data, private_data_len); 3143 3143 - trace_icm_send_mra(cm_id); 3144 3144 - ret = ib_post_send_mad(msg, NULL); 3145 3145 - if (ret) 3146 3146 - goto error_free_msg; 3147 3147 - } 3148 3148 - 3149 3143 cm_id->state = cm_state; 3150 3144 cm_id->lap_state = lap_state; 3151 3151 - cm_id_priv->service_timeout = service_timeout; 3152 3152 - cm_set_private_data(cm_id_priv, data, private_data_len); 3153 3153 - spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3154 3154 - return 0; 3145 3145 + cm_set_private_data(cm_id_priv, NULL, 0); 3155 3146 3156 3156 - error_free_msg: 3157 3157 - cm_free_msg(msg); 3158 3147 error_unlock: 3159 3148 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3160 3160 - kfree(data); 3161 3149 return ret; 3162 3150 } 3163 3163 - EXPORT_SYMBOL(ib_send_cm_mra); 3151 3151 + EXPORT_SYMBOL(ib_prepare_cm_mra); 3164 3152 3165 3153 static struct cm_id_private *cm_acquire_mraed_id(struct cm_mra_msg *mra_msg) 3166 3154 { ··· 3339 3377 3340 3378 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 3341 3379 CM_MSG_RESPONSE_OTHER, 3342 3342 - cm_id_priv->service_timeout, 3343 3380 cm_id_priv->private_data, 3344 3381 cm_id_priv->private_data_len); 3345 3382 spin_unlock_irq(&cm_id_priv->lock); ··· 3747 3786 spin_lock_irq(&cm_id_priv->lock); 3748 3787 if (msg != cm_id_priv->msg) { 3749 3788 spin_unlock_irq(&cm_id_priv->lock); 3750 3750 - cm_free_priv_msg(msg); 3789 3789 + cm_free_msg(msg); 3790 3790 + cm_deref_id(cm_id_priv); 3751 3791 return; 3752 3792 } 3753 3793 cm_free_priv_msg(msg); ··· 4340 4378 return -ENOMEM; 4341 4379 4342 4380 kref_init(&cm_dev->kref); 4343 4343 - spin_lock_init(&cm_dev->mad_agent_lock); 4381 4381 + rwlock_init(&cm_dev->mad_agent_lock); 4344 4382 cm_dev->ib_device = ib_device; 4345 4383 cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay; 4346 4384 cm_dev->going_down = 0; ··· 4456 4494 * The above ensures no call paths from the work are running, 4457 4495 * the remaining paths all take the mad_agent_lock. 4458 4496 */ 4459 4459 - spin_lock(&cm_dev->mad_agent_lock); 4497 4497 + write_lock(&cm_dev->mad_agent_lock); 4460 4498 port->mad_agent = NULL; 4461 4461 - spin_unlock(&cm_dev->mad_agent_lock); 4499 4499 + write_unlock(&cm_dev->mad_agent_lock); 4462 4500 ib_unregister_mad_agent(mad_agent); 4463 4501 ib_port_unregister_client_groups(ib_device, i, 4464 4502 cm_counter_groups);

+1 -1

drivers/infiniband/core/cm_trace.h

reviewed

··· 229 229 DEFINE_CM_ERR_EVENT(dreq_unknown); 230 230 DEFINE_CM_ERR_EVENT(send_unknown_rej); 231 231 DEFINE_CM_ERR_EVENT(rej_unknown); 232 232 - DEFINE_CM_ERR_EVENT(send_mra_unknown); 232 232 + DEFINE_CM_ERR_EVENT(prepare_mra_unknown); 233 233 DEFINE_CM_ERR_EVENT(mra_unknown); 234 234 DEFINE_CM_ERR_EVENT(qp_init); 235 235 DEFINE_CM_ERR_EVENT(qp_rtr);

+6 -19

drivers/infiniband/core/cma.c

reviewed

··· 46 46 47 47 #define CMA_CM_RESPONSE_TIMEOUT 20 48 48 #define CMA_MAX_CM_RETRIES 15 49 49 - #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24) 50 49 #define CMA_IBOE_PACKET_LIFETIME 16 51 50 #define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP 52 51 ··· 144 145 return NULL; 145 146 } 146 147 EXPORT_SYMBOL(rdma_iw_cm_id); 147 147 - 148 148 - /** 149 149 - * rdma_res_to_id() - return the rdma_cm_id pointer for this restrack. 150 150 - * @res: rdma resource tracking entry pointer 151 151 - */ 152 152 - struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res) 153 153 - { 154 154 - struct rdma_id_private *id_priv = 155 155 - container_of(res, struct rdma_id_private, res); 156 156 - 157 157 - return &id_priv->id; 158 158 - } 159 159 - EXPORT_SYMBOL(rdma_res_to_id); 160 148 161 149 static int cma_add_one(struct ib_device *device); 162 150 static void cma_remove_one(struct ib_device *device, void *client_data); ··· 2200 2214 case IB_CM_REP_RECEIVED: 2201 2215 if (state == RDMA_CM_CONNECT && 2202 2216 (id_priv->id.qp_type != IB_QPT_UD)) { 2203 2203 - trace_cm_send_mra(id_priv); 2204 2204 - ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); 2217 2217 + trace_cm_prepare_mra(id_priv); 2218 2218 + ib_prepare_cm_mra(cm_id); 2205 2219 } 2206 2220 if (id_priv->id.qp) { 2207 2221 event.status = cma_rep_recv(id_priv); ··· 2462 2476 2463 2477 if (READ_ONCE(conn_id->state) == RDMA_CM_CONNECT && 2464 2478 conn_id->id.qp_type != IB_QPT_UD) { 2465 2465 - trace_cm_send_mra(cm_id->context); 2466 2466 - ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); 2479 2479 + trace_cm_prepare_mra(cm_id->context); 2480 2480 + ib_prepare_cm_mra(cm_id); 2467 2481 } 2468 2482 mutex_unlock(&conn_id->handler_mutex); 2469 2483 ··· 5231 5245 neigh->ha, ETH_ALEN)) 5232 5246 continue; 5233 5247 cma_id_get(current_id); 5234 5234 - queue_work(cma_wq, &current_id->id.net_work); 5248 5248 + if (!queue_work(cma_wq, &current_id->id.net_work)) 5249 5249 + cma_id_put(current_id); 5235 5250 } 5236 5251 out: 5237 5252 spin_unlock_irqrestore(&id_table_lock, flags);

+1 -1

drivers/infiniband/core/cma_trace.h

reviewed

··· 55 55 56 56 DEFINE_CMA_FSM_EVENT(send_rtu); 57 57 DEFINE_CMA_FSM_EVENT(send_rej); 58 58 - DEFINE_CMA_FSM_EVENT(send_mra); 58 58 + DEFINE_CMA_FSM_EVENT(prepare_mra); 59 59 DEFINE_CMA_FSM_EVENT(send_sidr_req); 60 60 DEFINE_CMA_FSM_EVENT(send_sidr_rep); 61 61 DEFINE_CMA_FSM_EVENT(disconnect);

+15 -14

drivers/infiniband/core/iwcm.c

reviewed

··· 368 368 /* 369 369 * CM_ID <-- DESTROYING 370 370 * 371 371 - * Clean up all resources associated with the connection and release 372 372 - * the initial reference taken by iw_create_cm_id. 373 373 - * 374 374 - * Returns true if and only if the last cm_id_priv reference has been dropped. 371 371 + * Clean up all resources associated with the connection. 375 372 */ 376 376 - static bool destroy_cm_id(struct iw_cm_id *cm_id) 373 373 + static void destroy_cm_id(struct iw_cm_id *cm_id) 377 374 { 378 375 struct iwcm_id_private *cm_id_priv; 379 376 struct ib_qp *qp; ··· 439 442 iwpm_remove_mapinfo(&cm_id->local_addr, &cm_id->m_local_addr); 440 443 iwpm_remove_mapping(&cm_id->local_addr, RDMA_NL_IWCM); 441 444 } 442 442 - 443 443 - return iwcm_deref_id(cm_id_priv); 444 445 } 445 446 446 447 /* 447 447 - * This function is only called by the application thread and cannot 448 448 - * be called by the event thread. The function will wait for all 449 449 - * references to be released on the cm_id and then kfree the cm_id 450 450 - * object. 448 448 + * Destroy cm_id. If the cm_id still has other references, wait for all 449 449 + * references to be released on the cm_id and then release the initial 450 450 + * reference taken by iw_create_cm_id. 451 451 */ 452 452 void iw_destroy_cm_id(struct iw_cm_id *cm_id) 453 453 { 454 454 - if (!destroy_cm_id(cm_id)) 454 454 + struct iwcm_id_private *cm_id_priv; 455 455 + 456 456 + cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); 457 457 + destroy_cm_id(cm_id); 458 458 + if (refcount_read(&cm_id_priv->refcount) > 1) 455 459 flush_workqueue(iwcm_wq); 460 460 + iwcm_deref_id(cm_id_priv); 456 461 } 457 462 EXPORT_SYMBOL(iw_destroy_cm_id); 458 463 ··· 1034 1035 1035 1036 if (!test_bit(IWCM_F_DROP_EVENTS, &cm_id_priv->flags)) { 1036 1037 ret = process_event(cm_id_priv, &levent); 1037 1037 - if (ret) 1038 1038 - WARN_ON_ONCE(destroy_cm_id(&cm_id_priv->id)); 1038 1038 + if (ret) { 1039 1039 + destroy_cm_id(&cm_id_priv->id); 1040 1040 + WARN_ON_ONCE(iwcm_deref_id(cm_id_priv)); 1041 1041 + } 1039 1042 } else 1040 1043 pr_debug("dropping event %d\n", levent.event); 1041 1044 if (iwcm_deref_id(cm_id_priv))

+1 -1

drivers/infiniband/core/mad_rmpp.c

reviewed

··· 158 158 ah = ib_create_ah_from_wc(agent->qp->pd, recv_wc->wc, 159 159 recv_wc->recv_buf.grh, agent->port_num); 160 160 if (IS_ERR(ah)) 161 161 - return (void *) ah; 161 161 + return ERR_CAST(ah); 162 162 163 163 hdr_len = ib_get_mad_data_offset(recv_wc->recv_buf.mad->mad_hdr.mgmt_class); 164 164 msg = ib_create_send_mad(agent, recv_wc->wc->src_qp,

+103 -162

drivers/infiniband/core/umem_odp.c

reviewed

··· 41 41 #include <linux/hugetlb.h> 42 42 #include <linux/interval_tree.h> 43 43 #include <linux/hmm.h> 44 44 + #include <linux/hmm-dma.h> 44 45 #include <linux/pagemap.h> 45 46 46 47 #include <rdma/ib_umem_odp.h> 47 48 48 49 #include "uverbs.h" 49 50 50 50 - static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp, 51 51 - const struct mmu_interval_notifier_ops *ops) 51 51 + static void ib_init_umem_implicit_odp(struct ib_umem_odp *umem_odp) 52 52 { 53 53 - int ret; 53 53 + umem_odp->is_implicit_odp = 1; 54 54 + umem_odp->umem.is_odp = 1; 55 55 + mutex_init(&umem_odp->umem_mutex); 56 56 + } 57 57 + 58 58 + static int ib_init_umem_odp(struct ib_umem_odp *umem_odp, 59 59 + const struct mmu_interval_notifier_ops *ops) 60 60 + { 61 61 + struct ib_device *dev = umem_odp->umem.ibdev; 62 62 + size_t page_size = 1UL << umem_odp->page_shift; 63 63 + struct hmm_dma_map *map; 64 64 + unsigned long start; 65 65 + unsigned long end; 66 66 + size_t nr_entries; 67 67 + int ret = 0; 54 68 55 69 umem_odp->umem.is_odp = 1; 56 70 mutex_init(&umem_odp->umem_mutex); 57 71 58 58 - if (!umem_odp->is_implicit_odp) { 59 59 - size_t page_size = 1UL << umem_odp->page_shift; 60 60 - unsigned long start; 61 61 - unsigned long end; 62 62 - size_t ndmas, npfns; 72 72 + start = ALIGN_DOWN(umem_odp->umem.address, page_size); 73 73 + if (check_add_overflow(umem_odp->umem.address, 74 74 + (unsigned long)umem_odp->umem.length, &end)) 75 75 + return -EOVERFLOW; 76 76 + end = ALIGN(end, page_size); 77 77 + if (unlikely(end < page_size)) 78 78 + return -EOVERFLOW; 63 79 64 64 - start = ALIGN_DOWN(umem_odp->umem.address, page_size); 65 65 - if (check_add_overflow(umem_odp->umem.address, 66 66 - (unsigned long)umem_odp->umem.length, 67 67 - &end)) 68 68 - return -EOVERFLOW; 69 69 - end = ALIGN(end, page_size); 70 70 - if (unlikely(end < page_size)) 71 71 - return -EOVERFLOW; 80 80 + nr_entries = (end - start) >> PAGE_SHIFT; 81 81 + if (!(nr_entries * PAGE_SIZE / page_size)) 82 82 + return -EINVAL; 72 83 73 73 - ndmas = (end - start) >> umem_odp->page_shift; 74 74 - if (!ndmas) 75 75 - return -EINVAL; 76 76 - 77 77 - npfns = (end - start) >> PAGE_SHIFT; 78 78 - umem_odp->pfn_list = kvcalloc( 79 79 - npfns, sizeof(*umem_odp->pfn_list), 80 80 - GFP_KERNEL | __GFP_NOWARN); 81 81 - if (!umem_odp->pfn_list) 82 82 - return -ENOMEM; 83 83 - 84 84 - umem_odp->dma_list = kvcalloc( 85 85 - ndmas, sizeof(*umem_odp->dma_list), 86 86 - GFP_KERNEL | __GFP_NOWARN); 87 87 - if (!umem_odp->dma_list) { 84 84 + map = &umem_odp->map; 85 85 + if (ib_uses_virt_dma(dev)) { 86 86 + map->pfn_list = kvcalloc(nr_entries, sizeof(*map->pfn_list), 87 87 + GFP_KERNEL | __GFP_NOWARN); 88 88 + if (!map->pfn_list) 88 89 ret = -ENOMEM; 89 89 - goto out_pfn_list; 90 90 - } 90 90 + } else 91 91 + ret = hmm_dma_map_alloc(dev->dma_device, map, 92 92 + (end - start) >> PAGE_SHIFT, 93 93 + 1 << umem_odp->page_shift); 94 94 + if (ret) 95 95 + return ret; 91 96 92 92 - ret = mmu_interval_notifier_insert(&umem_odp->notifier, 93 93 - umem_odp->umem.owning_mm, 94 94 - start, end - start, ops); 95 95 - if (ret) 96 96 - goto out_dma_list; 97 97 - } 97 97 + ret = mmu_interval_notifier_insert(&umem_odp->notifier, 98 98 + umem_odp->umem.owning_mm, start, 99 99 + end - start, ops); 100 100 + if (ret) 101 101 + goto out_free_map; 98 102 99 103 return 0; 100 104 101 101 - out_dma_list: 102 102 - kvfree(umem_odp->dma_list); 103 103 - out_pfn_list: 104 104 - kvfree(umem_odp->pfn_list); 105 105 + out_free_map: 106 106 + if (ib_uses_virt_dma(dev)) 107 107 + kfree(map->pfn_list); 108 108 + else 109 109 + hmm_dma_map_free(dev->dma_device, map); 105 110 return ret; 106 111 } 107 112 ··· 125 120 { 126 121 struct ib_umem *umem; 127 122 struct ib_umem_odp *umem_odp; 128 128 - int ret; 129 123 130 124 if (access & IB_ACCESS_HUGETLB) 131 125 return ERR_PTR(-EINVAL); ··· 136 132 umem->ibdev = device; 137 133 umem->writable = ib_access_writable(access); 138 134 umem->owning_mm = current->mm; 139 139 - umem_odp->is_implicit_odp = 1; 140 135 umem_odp->page_shift = PAGE_SHIFT; 141 136 142 137 umem_odp->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); 143 143 - ret = ib_init_umem_odp(umem_odp, NULL); 144 144 - if (ret) { 145 145 - put_pid(umem_odp->tgid); 146 146 - kfree(umem_odp); 147 147 - return ERR_PTR(ret); 148 148 - } 138 138 + ib_init_umem_implicit_odp(umem_odp); 149 139 return umem_odp; 150 140 } 151 141 EXPORT_SYMBOL(ib_umem_odp_alloc_implicit); ··· 260 262 } 261 263 EXPORT_SYMBOL(ib_umem_odp_get); 262 264 263 263 - void ib_umem_odp_release(struct ib_umem_odp *umem_odp) 265 265 + static void ib_umem_odp_free(struct ib_umem_odp *umem_odp) 264 266 { 267 267 + struct ib_device *dev = umem_odp->umem.ibdev; 268 268 + 265 269 /* 266 270 * Ensure that no more pages are mapped in the umem. 267 271 * 268 272 * It is the driver's responsibility to ensure, before calling us, 269 273 * that the hardware will not attempt to access the MR any more. 270 274 */ 271 271 - if (!umem_odp->is_implicit_odp) { 272 272 - mutex_lock(&umem_odp->umem_mutex); 273 273 - ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp), 274 274 - ib_umem_end(umem_odp)); 275 275 - mutex_unlock(&umem_odp->umem_mutex); 276 276 - mmu_interval_notifier_remove(&umem_odp->notifier); 277 277 - kvfree(umem_odp->dma_list); 278 278 - kvfree(umem_odp->pfn_list); 279 279 - } 275 275 + mutex_lock(&umem_odp->umem_mutex); 276 276 + ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp), 277 277 + ib_umem_end(umem_odp)); 278 278 + mutex_unlock(&umem_odp->umem_mutex); 279 279 + mmu_interval_notifier_remove(&umem_odp->notifier); 280 280 + if (ib_uses_virt_dma(dev)) 281 281 + kfree(umem_odp->map.pfn_list); 282 282 + else 283 283 + hmm_dma_map_free(dev->dma_device, &umem_odp->map); 284 284 + } 285 285 + 286 286 + void ib_umem_odp_release(struct ib_umem_odp *umem_odp) 287 287 + { 288 288 + if (!umem_odp->is_implicit_odp) 289 289 + ib_umem_odp_free(umem_odp); 290 290 + 280 291 put_pid(umem_odp->tgid); 281 292 kfree(umem_odp); 282 293 } 283 294 EXPORT_SYMBOL(ib_umem_odp_release); 284 295 285 285 - /* 286 286 - * Map for DMA and insert a single page into the on-demand paging page tables. 287 287 - * 288 288 - * @umem: the umem to insert the page to. 289 289 - * @dma_index: index in the umem to add the dma to. 290 290 - * @page: the page struct to map and add. 291 291 - * @access_mask: access permissions needed for this page. 292 292 - * 293 293 - * The function returns -EFAULT if the DMA mapping operation fails. 294 294 - * 295 295 - */ 296 296 - static int ib_umem_odp_map_dma_single_page( 297 297 - struct ib_umem_odp *umem_odp, 298 298 - unsigned int dma_index, 299 299 - struct page *page, 300 300 - u64 access_mask) 301 301 - { 302 302 - struct ib_device *dev = umem_odp->umem.ibdev; 303 303 - dma_addr_t *dma_addr = &umem_odp->dma_list[dma_index]; 304 304 - 305 305 - if (*dma_addr) { 306 306 - /* 307 307 - * If the page is already dma mapped it means it went through 308 308 - * a non-invalidating trasition, like read-only to writable. 309 309 - * Resync the flags. 310 310 - */ 311 311 - *dma_addr = (*dma_addr & ODP_DMA_ADDR_MASK) | access_mask; 312 312 - return 0; 313 313 - } 314 314 - 315 315 - *dma_addr = ib_dma_map_page(dev, page, 0, 1 << umem_odp->page_shift, 316 316 - DMA_BIDIRECTIONAL); 317 317 - if (ib_dma_mapping_error(dev, *dma_addr)) { 318 318 - *dma_addr = 0; 319 319 - return -EFAULT; 320 320 - } 321 321 - umem_odp->npages++; 322 322 - *dma_addr |= access_mask; 323 323 - return 0; 324 324 - } 325 325 - 326 296 /** 327 297 * ib_umem_odp_map_dma_and_lock - DMA map userspace memory in an ODP MR and lock it. 328 298 * 329 299 * Maps the range passed in the argument to DMA addresses. 330 330 - * The DMA addresses of the mapped pages is updated in umem_odp->dma_list. 331 300 * Upon success the ODP MR will be locked to let caller complete its device 332 301 * page table update. 333 302 * ··· 322 357 struct hmm_range range = {}; 323 358 unsigned long timeout; 324 359 325 325 - if (access_mask == 0) 326 326 - return -EINVAL; 327 327 - 328 360 if (user_virt < ib_umem_start(umem_odp) || 329 361 user_virt + bcnt > ib_umem_end(umem_odp)) 330 362 return -EFAULT; ··· 347 385 if (fault) { 348 386 range.default_flags = HMM_PFN_REQ_FAULT; 349 387 350 350 - if (access_mask & ODP_WRITE_ALLOWED_BIT) 388 388 + if (access_mask & HMM_PFN_WRITE) 351 389 range.default_flags |= HMM_PFN_REQ_WRITE; 352 390 } 353 391 354 354 - range.hmm_pfns = &(umem_odp->pfn_list[pfn_start_idx]); 392 392 + range.hmm_pfns = &(umem_odp->map.pfn_list[pfn_start_idx]); 355 393 timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 356 394 357 395 retry: ··· 379 417 for (pfn_index = 0; pfn_index < num_pfns; 380 418 pfn_index += 1 << (page_shift - PAGE_SHIFT), dma_index++) { 381 419 382 382 - if (fault) { 383 383 - /* 384 384 - * Since we asked for hmm_range_fault() to populate 385 385 - * pages it shouldn't return an error entry on success. 386 386 - */ 387 387 - WARN_ON(range.hmm_pfns[pfn_index] & HMM_PFN_ERROR); 388 388 - WARN_ON(!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)); 389 389 - } else { 390 390 - if (!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)) { 391 391 - WARN_ON(umem_odp->dma_list[dma_index]); 392 392 - continue; 393 393 - } 394 394 - access_mask = ODP_READ_ALLOWED_BIT; 395 395 - if (range.hmm_pfns[pfn_index] & HMM_PFN_WRITE) 396 396 - access_mask |= ODP_WRITE_ALLOWED_BIT; 397 397 - } 420 420 + /* 421 421 + * Since we asked for hmm_range_fault() to populate 422 422 + * pages it shouldn't return an error entry on success. 423 423 + */ 424 424 + WARN_ON(fault && range.hmm_pfns[pfn_index] & HMM_PFN_ERROR); 425 425 + WARN_ON(fault && !(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)); 426 426 + if (!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)) 427 427 + continue; 428 428 + 429 429 + if (range.hmm_pfns[pfn_index] & HMM_PFN_DMA_MAPPED) 430 430 + continue; 398 431 399 432 hmm_order = hmm_pfn_to_map_order(range.hmm_pfns[pfn_index]); 400 433 /* If a hugepage was detected and ODP wasn't set for, the umem ··· 400 443 ibdev_dbg(umem_odp->umem.ibdev, 401 444 "%s: un-expected hmm_order %u, page_shift %u\n", 402 445 __func__, hmm_order, page_shift); 403 403 - break; 404 404 - } 405 405 - 406 406 - ret = ib_umem_odp_map_dma_single_page( 407 407 - umem_odp, dma_index, hmm_pfn_to_page(range.hmm_pfns[pfn_index]), 408 408 - access_mask); 409 409 - if (ret < 0) { 410 410 - ibdev_dbg(umem_odp->umem.ibdev, 411 411 - "ib_umem_odp_map_dma_single_page failed with error %d\n", ret); 412 446 break; 413 447 } 414 448 } ··· 421 473 void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt, 422 474 u64 bound) 423 475 { 424 424 - dma_addr_t dma_addr; 425 425 - dma_addr_t dma; 426 426 - int idx; 427 427 - u64 addr; 428 476 struct ib_device *dev = umem_odp->umem.ibdev; 477 477 + u64 addr; 429 478 430 479 lockdep_assert_held(&umem_odp->umem_mutex); 431 480 432 481 virt = max_t(u64, virt, ib_umem_start(umem_odp)); 433 482 bound = min_t(u64, bound, ib_umem_end(umem_odp)); 434 483 for (addr = virt; addr < bound; addr += BIT(umem_odp->page_shift)) { 435 435 - idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; 436 436 - dma = umem_odp->dma_list[idx]; 484 484 + u64 offset = addr - ib_umem_start(umem_odp); 485 485 + size_t idx = offset >> umem_odp->page_shift; 486 486 + unsigned long pfn = umem_odp->map.pfn_list[idx]; 437 487 438 438 - /* The access flags guaranteed a valid DMA address in case was NULL */ 439 439 - if (dma) { 440 440 - unsigned long pfn_idx = (addr - ib_umem_start(umem_odp)) >> PAGE_SHIFT; 441 441 - struct page *page = hmm_pfn_to_page(umem_odp->pfn_list[pfn_idx]); 488 488 + if (!hmm_dma_unmap_pfn(dev->dma_device, &umem_odp->map, idx)) 489 489 + goto clear; 442 490 443 443 - dma_addr = dma & ODP_DMA_ADDR_MASK; 444 444 - ib_dma_unmap_page(dev, dma_addr, 445 445 - BIT(umem_odp->page_shift), 446 446 - DMA_BIDIRECTIONAL); 447 447 - if (dma & ODP_WRITE_ALLOWED_BIT) { 448 448 - struct page *head_page = compound_head(page); 449 449 - /* 450 450 - * set_page_dirty prefers being called with 451 451 - * the page lock. However, MMU notifiers are 452 452 - * called sometimes with and sometimes without 453 453 - * the lock. We rely on the umem_mutex instead 454 454 - * to prevent other mmu notifiers from 455 455 - * continuing and allowing the page mapping to 456 456 - * be removed. 457 457 - */ 458 458 - set_page_dirty(head_page); 459 459 - } 460 460 - umem_odp->dma_list[idx] = 0; 461 461 - umem_odp->npages--; 491 491 + if (pfn & HMM_PFN_WRITE) { 492 492 + struct page *page = hmm_pfn_to_page(pfn); 493 493 + struct page *head_page = compound_head(page); 494 494 + /* 495 495 + * set_page_dirty prefers being called with 496 496 + * the page lock. However, MMU notifiers are 497 497 + * called sometimes with and sometimes without 498 498 + * the lock. We rely on the umem_mutex instead 499 499 + * to prevent other mmu notifiers from 500 500 + * continuing and allowing the page mapping to 501 501 + * be removed. 502 502 + */ 503 503 + set_page_dirty(head_page); 462 504 } 505 505 + umem_odp->npages--; 506 506 + clear: 507 507 + umem_odp->map.pfn_list[idx] &= ~HMM_PFN_FLAGS; 463 508 } 464 509 } 465 510 EXPORT_SYMBOL(ib_umem_odp_unmap_dma_pages);

+1 -1

drivers/infiniband/core/uverbs_cmd.c

reviewed

··· 193 193 fd, attrs); 194 194 195 195 if (IS_ERR(uobj)) 196 196 - return (void *)uobj; 196 196 + return ERR_CAST(uobj); 197 197 198 198 uverbs_uobject_get(uobj); 199 199 uobj_put_read(uobj);

+1 -1

drivers/infiniband/core/verbs.c

reviewed

··· 572 572 GFP_KERNEL : GFP_ATOMIC); 573 573 if (IS_ERR(slave)) { 574 574 rdma_unfill_sgid_attr(ah_attr, old_sgid_attr); 575 575 - return (void *)slave; 575 575 + return ERR_CAST(slave); 576 576 } 577 577 ah = _rdma_create_ah(pd, ah_attr, flags, NULL, slave); 578 578 rdma_lag_put_ah_roce_slave(slave);

+14 -6

drivers/infiniband/hw/bnxt_re/debugfs.c

reviewed

··· 170 170 case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TCP_CP: 171 171 *val = ccparam->tcp_cp; 172 172 break; 173 173 + case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INACTIVITY_CP: 174 174 + *val = ccparam->inact_th; 175 175 + break; 173 176 default: 174 177 return -EINVAL; 175 178 } ··· 206 203 return simple_read_from_buffer(buffer, usr_buf_len, ppos, (u8 *)(buf), rc); 207 204 } 208 205 209 209 - static void bnxt_re_fill_gen0_ext0(struct bnxt_qplib_cc_param *ccparam, u32 offset, u32 val) 206 206 + static int bnxt_re_fill_gen0_ext0(struct bnxt_qplib_cc_param *ccparam, u32 offset, u32 val) 210 207 { 211 208 u32 modify_mask; 212 209 ··· 250 247 ccparam->tcp_cp = val; 251 248 break; 252 249 case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TX_QUEUE: 250 250 + return -EOPNOTSUPP; 253 251 case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INACTIVITY_CP: 252 252 + ccparam->inact_th = val; 254 253 break; 255 254 case CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TIME_PER_PHASE: 256 255 ccparam->time_pph = val; ··· 263 258 } 264 259 265 260 ccparam->mask = modify_mask; 261 261 + return 0; 266 262 } 267 263 268 264 static int bnxt_re_configure_cc(struct bnxt_re_dev *rdev, u32 gen_ext, u32 offset, u32 val) 269 265 { 270 266 struct bnxt_qplib_cc_param ccparam = { }; 267 267 + int rc; 271 268 272 272 - /* Supporting only Gen 0 now */ 273 273 - if (gen_ext == CC_CONFIG_GEN0_EXT0) 274 274 - bnxt_re_fill_gen0_ext0(&ccparam, offset, val); 275 275 - else 276 276 - return -EINVAL; 269 269 + if (gen_ext != CC_CONFIG_GEN0_EXT0) 270 270 + return -EOPNOTSUPP; 271 271 + 272 272 + rc = bnxt_re_fill_gen0_ext0(&ccparam, offset, val); 273 273 + if (rc) 274 274 + return rc; 277 275 278 276 bnxt_qplib_modify_cc(&rdev->qplib_res, &ccparam); 279 277 return 0;

+1 -1

drivers/infiniband/hw/bnxt_re/qplib_fp.c

reviewed

··· 1113 1113 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; 1114 1114 if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) 1115 1115 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED; 1116 1116 - if (_is_ext_stats_supported(res->dattr->dev_cap_flags) && !res->is_vf) 1116 1116 + if (bnxt_ext_stats_supported(res->cctx, res->dattr->dev_cap_flags, res->is_vf)) 1117 1117 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED; 1118 1118 1119 1119 req.qp_flags = cpu_to_le32(qp_flags);

+6 -1

drivers/infiniband/hw/bnxt_re/qplib_sp.c

reviewed

··· 846 846 847 847 req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; 848 848 req.resp_addr = cpu_to_le64(sbuf.dma_addr); 849 849 - req.function_id = cpu_to_le32(fid); 849 849 + if (bnxt_qplib_is_chip_gen_p7(rcfw->res->cctx) && rcfw->res->is_vf) 850 850 + req.function_id = 851 851 + cpu_to_le32(CMDQ_QUERY_ROCE_STATS_EXT_VF_VALID | 852 852 + (fid << CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_SFT)); 853 853 + else 854 854 + req.function_id = cpu_to_le32(fid); 850 855 req.flags = cpu_to_le16(CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID); 851 856 852 857 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),

-1

drivers/infiniband/hw/hfi1/mad.h

reviewed

··· 124 124 } __packed ntc_2048; 125 125 126 126 }; 127 127 - u8 class_data[]; 128 127 }; 129 128 130 129 #define IB_VLARB_LOWPRI_0_31 1

-10

drivers/infiniband/hw/hfi1/pio.c

reviewed

··· 1361 1361 sc_wait_for_packet_egress(sc, 1); 1362 1362 } 1363 1363 1364 1364 - /* drop all packets on the context, no waiting until they are sent */ 1365 1365 - void sc_drop(struct send_context *sc) 1366 1366 - { 1367 1367 - if (!sc) 1368 1368 - return; 1369 1369 - 1370 1370 - dd_dev_info(sc->dd, "%s: context %u(%u) - not implemented\n", 1371 1371 - __func__, sc->sw_index, sc->hw_context); 1372 1372 - } 1373 1373 - 1374 1364 /* 1375 1365 * Start the software reaction to a context halt or SPC freeze: 1376 1366 * - mark the context as halted or frozen

-1

drivers/infiniband/hw/hfi1/pio.h

reviewed

··· 246 246 int sc_restart(struct send_context *sc); 247 247 void sc_return_credits(struct send_context *sc); 248 248 void sc_flush(struct send_context *sc); 249 249 - void sc_drop(struct send_context *sc); 250 249 void sc_stop(struct send_context *sc, int bit); 251 250 struct pio_buf *sc_buffer_alloc(struct send_context *sc, u32 dw_len, 252 251 pio_release_cb cb, void *arg);

-18

drivers/infiniband/hw/hfi1/sdma.c

reviewed

··· 1521 1521 } 1522 1522 1523 1523 /** 1524 1524 - * sdma_all_idle() - called when the link goes down 1525 1525 - * @dd: hfi1_devdata 1526 1526 - * 1527 1527 - * This routine moves all engines to the idle state. 1528 1528 - */ 1529 1529 - void sdma_all_idle(struct hfi1_devdata *dd) 1530 1530 - { 1531 1531 - struct sdma_engine *sde; 1532 1532 - unsigned int i; 1533 1533 - 1534 1534 - /* idle all engines */ 1535 1535 - for (i = 0; i < dd->num_sdma; ++i) { 1536 1536 - sde = &dd->per_sdma[i]; 1537 1537 - sdma_process_event(sde, sdma_event_e70_go_idle); 1538 1538 - } 1539 1539 - } 1540 1540 - 1541 1541 - /** 1542 1524 * sdma_start() - called to kick off state processing for all engines 1543 1525 * @dd: hfi1_devdata 1544 1526 *

-1

drivers/infiniband/hw/hfi1/sdma.h

reviewed

··· 373 373 void sdma_exit(struct hfi1_devdata *dd); 374 374 void sdma_clean(struct hfi1_devdata *dd, size_t num_engines); 375 375 void sdma_all_running(struct hfi1_devdata *dd); 376 376 - void sdma_all_idle(struct hfi1_devdata *dd); 377 376 void sdma_freeze_notify(struct hfi1_devdata *dd, int go_idle); 378 377 void sdma_freeze(struct hfi1_devdata *dd); 379 378 void sdma_unfreeze(struct hfi1_devdata *dd);

+1 -1

drivers/infiniband/hw/hfi1/user_exp_rcv.c

reviewed

··· 53 53 int ret = 0; 54 54 55 55 fd->entry_to_rb = kcalloc(uctxt->expected_count, 56 56 - sizeof(struct rb_node *), 56 56 + sizeof(*fd->entry_to_rb), 57 57 GFP_KERNEL); 58 58 if (!fd->entry_to_rb) 59 59 return -ENOMEM;

+1

drivers/infiniband/hw/hns/Makefile

reviewed

··· 4 4 # 5 5 6 6 ccflags-y := -I $(srctree)/drivers/net/ethernet/hisilicon/hns3 7 7 + ccflags-y += -I $(src) 7 8 8 9 hns-roce-hw-v2-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ 9 10 hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \

-1

drivers/infiniband/hw/hns/hns_roce_ah.c

reviewed

··· 33 33 #include <linux/pci.h> 34 34 #include <rdma/ib_addr.h> 35 35 #include <rdma/ib_cache.h> 36 36 - #include "hnae3.h" 37 36 #include "hns_roce_device.h" 38 37 #include "hns_roce_hw_v2.h" 39 38

+20

drivers/infiniband/hw/hns/hns_roce_device.h

reviewed

··· 1027 1027 atomic64_t *dfx_cnt; 1028 1028 }; 1029 1029 1030 1030 + enum hns_roce_trace_type { 1031 1031 + TRACE_SQ, 1032 1032 + TRACE_RQ, 1033 1033 + TRACE_SRQ, 1034 1034 + }; 1035 1035 + 1036 1036 + static inline const char *trace_type_to_str(enum hns_roce_trace_type type) 1037 1037 + { 1038 1038 + switch (type) { 1039 1039 + case TRACE_SQ: 1040 1040 + return "SQ"; 1041 1041 + case TRACE_RQ: 1042 1042 + return "RQ"; 1043 1043 + case TRACE_SRQ: 1044 1044 + return "SRQ"; 1045 1045 + default: 1046 1046 + return "UNKNOWN"; 1047 1047 + } 1048 1048 + } 1049 1049 + 1030 1050 static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev) 1031 1051 { 1032 1052 return container_of(ib_dev, struct hns_roce_dev, ib_dev);

+21 -5

drivers/infiniband/hw/hns/hns_roce_hw_v2.c

reviewed

··· 43 43 #include <rdma/ib_umem.h> 44 44 #include <rdma/uverbs_ioctl.h> 45 45 46 46 - #include "hnae3.h" 47 46 #include "hns_roce_common.h" 48 47 #include "hns_roce_device.h" 49 48 #include "hns_roce_cmd.h" 50 49 #include "hns_roce_hem.h" 51 50 #include "hns_roce_hw_v2.h" 51 51 + 52 52 + #define CREATE_TRACE_POINTS 53 53 + #include "hns_roce_trace.h" 52 54 53 55 enum { 54 56 CMD_RST_PRC_OTHERS, ··· 740 738 else 741 739 ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit); 742 740 741 741 + trace_hns_sq_wqe(qp->qpn, wqe_idx, wqe, 1 << qp->sq.wqe_shift, 742 742 + wr->wr_id, TRACE_SQ); 743 743 if (unlikely(ret)) { 744 744 *bad_wr = wr; 745 745 goto out; ··· 811 807 812 808 wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx); 813 809 fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp->rq.rsv_sge); 810 810 + 811 811 + trace_hns_rq_wqe(hr_qp->qpn, wqe_idx, wqe, 1 << hr_qp->rq.wqe_shift, 812 812 + wr->wr_id, TRACE_RQ); 814 813 } 815 814 816 815 static int hns_roce_v2_post_recv(struct ib_qp *ibqp, ··· 950 943 static void update_srq_db(struct hns_roce_srq *srq) 951 944 { 952 945 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device); 953 953 - struct hns_roce_v2_db db; 946 946 + struct hns_roce_v2_db db = {}; 954 947 955 948 hr_reg_write(&db, DB_TAG, srq->srqn); 956 949 hr_reg_write(&db, DB_CMD, HNS_ROCE_V2_SRQ_DB); ··· 991 984 fill_recv_sge_to_wqe(wr, wqe, max_sge, srq->rsv_sge); 992 985 fill_wqe_idx(srq, wqe_idx); 993 986 srq->wrid[wqe_idx] = wr->wr_id; 987 987 + 988 988 + trace_hns_srq_wqe(srq->srqn, wqe_idx, wqe, 1 << srq->wqe_shift, 989 989 + wr->wr_id, TRACE_SRQ); 994 990 } 995 991 996 992 if (likely(nreq)) { ··· 1321 1311 tail = csq->head; 1322 1312 1323 1313 for (i = 0; i < num; i++) { 1314 1314 + trace_hns_cmdq_req(hr_dev, &desc[i]); 1315 1315 + 1324 1316 csq->desc[csq->head++] = desc[i]; 1325 1317 if (csq->head == csq->desc_num) 1326 1318 csq->head = 0; ··· 1337 1325 if (hns_roce_cmq_csq_done(hr_dev)) { 1338 1326 ret = 0; 1339 1327 for (i = 0; i < num; i++) { 1328 1328 + trace_hns_cmdq_resp(hr_dev, &csq->desc[tail]); 1329 1329 + 1340 1330 /* check the result of hardware write back */ 1341 1331 desc_ret = le16_to_cpu(csq->desc[tail++].retval); 1342 1332 if (tail == csq->desc_num) ··· 4316 4302 } 4317 4303 4318 4304 static void modify_qp_reset_to_init(struct ib_qp *ibqp, 4319 4319 - struct hns_roce_v2_qp_context *context, 4320 4320 - struct hns_roce_v2_qp_context *qpc_mask) 4305 4305 + struct hns_roce_v2_qp_context *context) 4321 4306 { 4322 4307 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4323 4308 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); ··· 5135 5122 5136 5123 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { 5137 5124 memset(qpc_mask, 0, hr_dev->caps.qpc_sz); 5138 5138 - modify_qp_reset_to_init(ibqp, context, qpc_mask); 5125 5125 + modify_qp_reset_to_init(ibqp, context); 5139 5126 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { 5140 5127 modify_qp_init_to_init(ibqp, context, qpc_mask); 5141 5128 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { ··· 5326 5313 return; 5327 5314 5328 5315 spin_lock_irqsave(&hr_qp->sq.lock, sq_flag); 5316 5316 + trace_hns_sq_flush_cqe(hr_qp->qpn, hr_qp->sq.head, TRACE_SQ); 5329 5317 hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head); 5330 5318 hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX); 5331 5319 hr_qp->state = IB_QPS_ERR; ··· 5336 5322 return; 5337 5323 5338 5324 spin_lock_irqsave(&hr_qp->rq.lock, rq_flag); 5325 5325 + trace_hns_rq_flush_cqe(hr_qp->qpn, hr_qp->rq.head, TRACE_RQ); 5339 5326 hr_reg_write(context, QPC_RQ_PRODUCER_IDX, hr_qp->rq.head); 5340 5327 hr_reg_clear(qpc_mask, QPC_RQ_PRODUCER_IDX); 5341 5328 spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag); ··· 6263 6248 eq->sub_type = sub_type; 6264 6249 ++eq->cons_index; 6265 6250 aeqe_found = IRQ_HANDLED; 6251 6251 + trace_hns_ae_info(event_type, aeqe, eq->eqe_size); 6266 6252 6267 6253 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_AEQE_CNT]); 6268 6254

+1

drivers/infiniband/hw/hns/hns_roce_hw_v2.h

reviewed

··· 34 34 #define _HNS_ROCE_HW_V2_H 35 35 36 36 #include <linux/bitops.h> 37 37 + #include "hnae3.h" 37 38 38 39 #define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32 39 40 #define HNS_ROCE_V2_MTT_ENTRY_SZ 64

-1

drivers/infiniband/hw/hns/hns_roce_main.c

reviewed

··· 37 37 #include <rdma/ib_smi.h> 38 38 #include <rdma/ib_user_verbs.h> 39 39 #include <rdma/ib_cache.h> 40 40 - #include "hnae3.h" 41 40 #include "hns_roce_common.h" 42 41 #include "hns_roce_device.h" 43 42 #include "hns_roce_hem.h"

+3

drivers/infiniband/hw/hns/hns_roce_mr.c

reviewed

··· 38 38 #include "hns_roce_device.h" 39 39 #include "hns_roce_cmd.h" 40 40 #include "hns_roce_hem.h" 41 41 + #include "hns_roce_trace.h" 41 42 42 43 static u32 hw_index_to_key(int ind) 43 44 { ··· 160 159 if (IS_ERR(mailbox)) 161 160 return PTR_ERR(mailbox); 162 161 162 162 + trace_hns_mr(mr); 163 163 if (mr->type != MR_TYPE_FRMR) 164 164 ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr); 165 165 else ··· 1148 1146 struct ib_device *ibdev = &hr_dev->ib_dev; 1149 1147 int ret; 1150 1148 1149 1149 + trace_hns_buf_attr(buf_attr); 1151 1150 /* The caller has its own buffer list and invokes the hns_roce_mtr_map() 1152 1151 * to finish the MTT configuration. 1153 1152 */

-1

drivers/infiniband/hw/hns/hns_roce_restrack.c

reviewed

··· 4 4 #include <rdma/rdma_cm.h> 5 5 #include <rdma/restrack.h> 6 6 #include <uapi/rdma/rdma_netlink.h> 7 7 - #include "hnae3.h" 8 7 #include "hns_roce_common.h" 9 8 #include "hns_roce_device.h" 10 9 #include "hns_roce_hw_v2.h"

+216

drivers/infiniband/hw/hns/hns_roce_trace.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0+ */ 2 2 + /* 3 3 + * Copyright (c) 2025 Hisilicon Limited. 4 4 + */ 5 5 + 6 6 + #undef TRACE_SYSTEM 7 7 + #define TRACE_SYSTEM hns_roce 8 8 + 9 9 + #if !defined(__HNS_ROCE_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) 10 10 + #define __HNS_ROCE_TRACE_H 11 11 + 12 12 + #include <linux/tracepoint.h> 13 13 + #include <linux/string_choices.h> 14 14 + #include "hns_roce_device.h" 15 15 + #include "hns_roce_hw_v2.h" 16 16 + 17 17 + DECLARE_EVENT_CLASS(flush_head_template, 18 18 + TP_PROTO(unsigned long qpn, u32 pi, 19 19 + enum hns_roce_trace_type type), 20 20 + TP_ARGS(qpn, pi, type), 21 21 + 22 22 + TP_STRUCT__entry(__field(unsigned long, qpn) 23 23 + __field(u32, pi) 24 24 + __field(enum hns_roce_trace_type, type) 25 25 + ), 26 26 + 27 27 + TP_fast_assign(__entry->qpn = qpn; 28 28 + __entry->pi = pi; 29 29 + __entry->type = type; 30 30 + ), 31 31 + 32 32 + TP_printk("%s 0x%lx flush head 0x%x.", 33 33 + trace_type_to_str(__entry->type), 34 34 + __entry->qpn, __entry->pi) 35 35 + ); 36 36 + 37 37 + DEFINE_EVENT(flush_head_template, hns_sq_flush_cqe, 38 38 + TP_PROTO(unsigned long qpn, u32 pi, 39 39 + enum hns_roce_trace_type type), 40 40 + TP_ARGS(qpn, pi, type)); 41 41 + DEFINE_EVENT(flush_head_template, hns_rq_flush_cqe, 42 42 + TP_PROTO(unsigned long qpn, u32 pi, 43 43 + enum hns_roce_trace_type type), 44 44 + TP_ARGS(qpn, pi, type)); 45 45 + 46 46 + #define MAX_SGE_PER_WQE 64 47 47 + #define MAX_WQE_SIZE (MAX_SGE_PER_WQE * HNS_ROCE_SGE_SIZE) 48 48 + DECLARE_EVENT_CLASS(wqe_template, 49 49 + TP_PROTO(unsigned long qpn, u32 idx, void *wqe, u32 len, 50 50 + u64 id, enum hns_roce_trace_type type), 51 51 + TP_ARGS(qpn, idx, wqe, len, id, type), 52 52 + 53 53 + TP_STRUCT__entry(__field(unsigned long, qpn) 54 54 + __field(u32, idx) 55 55 + __array(u32, wqe, 56 56 + MAX_WQE_SIZE / sizeof(__le32)) 57 57 + __field(u32, len) 58 58 + __field(u64, id) 59 59 + __field(enum hns_roce_trace_type, type) 60 60 + ), 61 61 + 62 62 + TP_fast_assign(__entry->qpn = qpn; 63 63 + __entry->idx = idx; 64 64 + __entry->id = id; 65 65 + __entry->len = len / sizeof(__le32); 66 66 + __entry->type = type; 67 67 + for (int i = 0; i < __entry->len; i++) 68 68 + __entry->wqe[i] = le32_to_cpu(((__le32 *)wqe)[i]); 69 69 + ), 70 70 + 71 71 + TP_printk("%s 0x%lx wqe(0x%x/0x%llx): %s", 72 72 + trace_type_to_str(__entry->type), 73 73 + __entry->qpn, __entry->idx, __entry->id, 74 74 + __print_array(__entry->wqe, __entry->len, 75 75 + sizeof(__le32))) 76 76 + ); 77 77 + 78 78 + DEFINE_EVENT(wqe_template, hns_sq_wqe, 79 79 + TP_PROTO(unsigned long qpn, u32 idx, void *wqe, u32 len, u64 id, 80 80 + enum hns_roce_trace_type type), 81 81 + TP_ARGS(qpn, idx, wqe, len, id, type)); 82 82 + DEFINE_EVENT(wqe_template, hns_rq_wqe, 83 83 + TP_PROTO(unsigned long qpn, u32 idx, void *wqe, u32 len, u64 id, 84 84 + enum hns_roce_trace_type type), 85 85 + TP_ARGS(qpn, idx, wqe, len, id, type)); 86 86 + DEFINE_EVENT(wqe_template, hns_srq_wqe, 87 87 + TP_PROTO(unsigned long qpn, u32 idx, void *wqe, u32 len, u64 id, 88 88 + enum hns_roce_trace_type type), 89 89 + TP_ARGS(qpn, idx, wqe, len, id, type)); 90 90 + 91 91 + TRACE_EVENT(hns_ae_info, 92 92 + TP_PROTO(int event_type, void *aeqe, unsigned int len), 93 93 + TP_ARGS(event_type, aeqe, len), 94 94 + 95 95 + TP_STRUCT__entry(__field(int, event_type) 96 96 + __array(u32, aeqe, 97 97 + HNS_ROCE_V3_EQE_SIZE / sizeof(__le32)) 98 98 + __field(u32, len) 99 99 + ), 100 100 + 101 101 + TP_fast_assign(__entry->event_type = event_type; 102 102 + __entry->len = len / sizeof(__le32); 103 103 + for (int i = 0; i < __entry->len; i++) 104 104 + __entry->aeqe[i] = le32_to_cpu(((__le32 *)aeqe)[i]); 105 105 + ), 106 106 + 107 107 + TP_printk("event %2d aeqe: %s", __entry->event_type, 108 108 + __print_array(__entry->aeqe, __entry->len, sizeof(__le32))) 109 109 + ); 110 110 + 111 111 + TRACE_EVENT(hns_mr, 112 112 + TP_PROTO(struct hns_roce_mr *mr), 113 113 + TP_ARGS(mr), 114 114 + 115 115 + TP_STRUCT__entry(__field(u64, iova) 116 116 + __field(u64, size) 117 117 + __field(u32, key) 118 118 + __field(u32, pd) 119 119 + __field(u32, pbl_hop_num) 120 120 + __field(u32, npages) 121 121 + __field(int, type) 122 122 + __field(int, enabled) 123 123 + ), 124 124 + 125 125 + TP_fast_assign(__entry->iova = mr->iova; 126 126 + __entry->size = mr->size; 127 127 + __entry->key = mr->key; 128 128 + __entry->pd = mr->pd; 129 129 + __entry->pbl_hop_num = mr->pbl_hop_num; 130 130 + __entry->npages = mr->npages; 131 131 + __entry->type = mr->type; 132 132 + __entry->enabled = mr->enabled; 133 133 + ), 134 134 + 135 135 + TP_printk("iova:0x%llx, size:%llu, key:%u, pd:%u, pbl_hop:%u, npages:%u, type:%d, status:%d", 136 136 + __entry->iova, __entry->size, __entry->key, 137 137 + __entry->pd, __entry->pbl_hop_num, __entry->npages, 138 138 + __entry->type, __entry->enabled) 139 139 + ); 140 140 + 141 141 + TRACE_EVENT(hns_buf_attr, 142 142 + TP_PROTO(struct hns_roce_buf_attr *attr), 143 143 + TP_ARGS(attr), 144 144 + 145 145 + TP_STRUCT__entry(__field(unsigned int, region_count) 146 146 + __field(unsigned int, region0_size) 147 147 + __field(int, region0_hopnum) 148 148 + __field(unsigned int, region1_size) 149 149 + __field(int, region1_hopnum) 150 150 + __field(unsigned int, region2_size) 151 151 + __field(int, region2_hopnum) 152 152 + __field(unsigned int, page_shift) 153 153 + __field(bool, mtt_only) 154 154 + ), 155 155 + 156 156 + TP_fast_assign(__entry->region_count = attr->region_count; 157 157 + __entry->region0_size = attr->region[0].size; 158 158 + __entry->region0_hopnum = attr->region[0].hopnum; 159 159 + __entry->region1_size = attr->region[1].size; 160 160 + __entry->region1_hopnum = attr->region[1].hopnum; 161 161 + __entry->region2_size = attr->region[2].size; 162 162 + __entry->region2_hopnum = attr->region[2].hopnum; 163 163 + __entry->page_shift = attr->page_shift; 164 164 + __entry->mtt_only = attr->mtt_only; 165 165 + ), 166 166 + 167 167 + TP_printk("rg cnt:%u, pg_sft:0x%x, mtt_only:%s, rg 0 (sz:%u, hop:%u), rg 1 (sz:%u, hop:%u), rg 2 (sz:%u, hop:%u)\n", 168 168 + __entry->region_count, __entry->page_shift, 169 169 + str_yes_no(__entry->mtt_only), 170 170 + __entry->region0_size, __entry->region0_hopnum, 171 171 + __entry->region1_size, __entry->region1_hopnum, 172 172 + __entry->region2_size, __entry->region2_hopnum) 173 173 + ); 174 174 + 175 175 + DECLARE_EVENT_CLASS(cmdq, 176 176 + TP_PROTO(struct hns_roce_dev *hr_dev, 177 177 + struct hns_roce_cmq_desc *desc), 178 178 + TP_ARGS(hr_dev, desc), 179 179 + 180 180 + TP_STRUCT__entry(__string(dev_name, dev_name(hr_dev->dev)) 181 181 + __field(u16, opcode) 182 182 + __field(u16, flag) 183 183 + __field(u16, retval) 184 184 + __array(u32, data, 6) 185 185 + ), 186 186 + 187 187 + TP_fast_assign(__assign_str(dev_name); 188 188 + __entry->opcode = le16_to_cpu(desc->opcode); 189 189 + __entry->flag = le16_to_cpu(desc->flag); 190 190 + __entry->retval = le16_to_cpu(desc->retval); 191 191 + for (int i = 0; i < 6; i++) 192 192 + __entry->data[i] = le32_to_cpu(desc->data[i]); 193 193 + ), 194 194 + 195 195 + TP_printk("%s cmdq opcode:0x%x, flag:0x%x, retval:0x%x, data:%s\n", 196 196 + __get_str(dev_name), __entry->opcode, 197 197 + __entry->flag, __entry->retval, 198 198 + __print_array(__entry->data, 6, sizeof(__le32))) 199 199 + ); 200 200 + 201 201 + DEFINE_EVENT(cmdq, hns_cmdq_req, 202 202 + TP_PROTO(struct hns_roce_dev *hr_dev, 203 203 + struct hns_roce_cmq_desc *desc), 204 204 + TP_ARGS(hr_dev, desc)); 205 205 + DEFINE_EVENT(cmdq, hns_cmdq_resp, 206 206 + TP_PROTO(struct hns_roce_dev *hr_dev, 207 207 + struct hns_roce_cmq_desc *desc), 208 208 + TP_ARGS(hr_dev, desc)); 209 209 + 210 210 + #endif /* __HNS_ROCE_TRACE_H */ 211 211 + 212 212 + #undef TRACE_INCLUDE_FILE 213 213 + #define TRACE_INCLUDE_FILE hns_roce_trace 214 214 + #undef TRACE_INCLUDE_PATH 215 215 + #define TRACE_INCLUDE_PATH . 216 216 + #include <trace/define_trace.h>

+1 -1

drivers/infiniband/hw/irdma/ctrl.c

reviewed

··· 3131 3131 writel(0, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); 3132 3132 3133 3133 ibdev_dbg(to_ibdev(cqp->dev), 3134 3134 - "WQE: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%pK] cqp[%p] polarity[x%04x]\n", 3134 3134 + "WQE: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%p] cqp[%p] polarity[x%04x]\n", 3135 3135 cqp->sq_size, cqp->hw_sq_size, cqp->sq_base, 3136 3136 (u64 *)(uintptr_t)cqp->sq_pa, cqp, cqp->polarity); 3137 3137 return 0;

+1 -1

drivers/infiniband/hw/irdma/pble.c

reviewed

··· 108 108 chunk->vaddr = sd_entry->u.bp.addr.va + offset; 109 109 chunk->fpm_addr = pble_rsrc->next_fpm_addr; 110 110 ibdev_dbg(to_ibdev(dev), 111 111 - "PBLE: chunk_size[%lld] = 0x%llx vaddr=0x%pK fpm_addr = %llx\n", 111 111 + "PBLE: chunk_size[%lld] = 0x%llx vaddr=0x%p fpm_addr = %llx\n", 112 112 chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr); 113 113 114 114 return 0;

+1 -3

drivers/infiniband/hw/mana/cq.c

reviewed

··· 15 15 struct ib_device *ibdev = ibcq->device; 16 16 struct mana_ib_create_cq ucmd = {}; 17 17 struct mana_ib_dev *mdev; 18 18 - struct gdma_context *gc; 19 18 bool is_rnic_cq; 20 19 u32 doorbell; 21 20 u32 buf_size; 22 21 int err; 23 22 24 23 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 25 25 - gc = mdev_to_gc(mdev); 26 24 27 25 cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors; 28 26 cq->cq_handle = INVALID_MANA_HANDLE; ··· 63 65 ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err); 64 66 return err; 65 67 } 66 66 - doorbell = gc->mana_ib.doorbell; 68 68 + doorbell = mdev->gdma_dev->doorbell; 67 69 } 68 70 69 71 if (is_rnic_cq) {

+83 -91

drivers/infiniband/hw/mana/device.c

reviewed

··· 101 101 const struct auxiliary_device_id *id) 102 102 { 103 103 struct mana_adev *madev = container_of(adev, struct mana_adev, adev); 104 104 + struct gdma_context *gc = madev->mdev->gdma_context; 105 105 + struct mana_context *mc = gc->mana.driver_data; 104 106 struct gdma_dev *mdev = madev->mdev; 105 107 struct net_device *ndev; 106 106 - struct mana_context *mc; 107 108 struct mana_ib_dev *dev; 108 109 u8 mac_addr[ETH_ALEN]; 109 110 int ret; 110 110 - 111 111 - mc = mdev->driver_data; 112 111 113 112 dev = ib_alloc_device(mana_ib_dev, ib_dev); 114 113 if (!dev) 115 114 return -ENOMEM; 116 115 117 116 ib_set_device_ops(&dev->ib_dev, &mana_ib_dev_ops); 118 118 - 119 119 - dev->ib_dev.phys_port_cnt = mc->num_ports; 120 120 - 121 121 - ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev, 122 122 - mdev->dev_id.as_uint32, dev->ib_dev.phys_port_cnt); 123 123 - 124 117 dev->ib_dev.node_type = RDMA_NODE_IB_CA; 125 125 - 126 126 - /* 127 127 - * num_comp_vectors needs to set to the max MSIX index 128 128 - * when interrupts and event queues are implemented 129 129 - */ 130 130 - dev->ib_dev.num_comp_vectors = mdev->gdma_context->max_num_queues; 131 131 - dev->ib_dev.dev.parent = mdev->gdma_context->dev; 132 132 - 133 133 - ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker); 134 134 - if (!ndev) { 135 135 - ret = -ENODEV; 136 136 - ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1"); 137 137 - goto free_ib_device; 138 138 - } 139 139 - ether_addr_copy(mac_addr, ndev->dev_addr); 140 140 - addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr); 141 141 - ret = ib_device_set_netdev(&dev->ib_dev, ndev, 1); 142 142 - /* mana_get_primary_netdev() returns ndev with refcount held */ 143 143 - netdev_put(ndev, &dev->dev_tracker); 144 144 - if (ret) { 145 145 - ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret); 146 146 - goto free_ib_device; 147 147 - } 148 148 - 149 149 - ret = mana_gd_register_device(&mdev->gdma_context->mana_ib); 150 150 - if (ret) { 151 151 - ibdev_err(&dev->ib_dev, "Failed to register device, ret %d", 152 152 - ret); 153 153 - goto free_ib_device; 154 154 - } 155 155 - dev->gdma_dev = &mdev->gdma_context->mana_ib; 156 156 - 157 157 - dev->nb.notifier_call = mana_ib_netdev_event; 158 158 - ret = register_netdevice_notifier(&dev->nb); 159 159 - if (ret) { 160 160 - ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", 161 161 - ret); 162 162 - goto deregister_device; 163 163 - } 164 164 - 165 165 - ret = mana_ib_gd_query_adapter_caps(dev); 166 166 - if (ret) { 167 167 - ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", 168 168 - ret); 169 169 - goto deregister_net_notifier; 170 170 - } 171 171 - 172 172 - ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops); 173 173 - 174 174 - ret = mana_ib_create_eqs(dev); 175 175 - if (ret) { 176 176 - ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret); 177 177 - goto deregister_net_notifier; 178 178 - } 179 179 - 180 180 - ret = mana_ib_gd_create_rnic_adapter(dev); 181 181 - if (ret) 182 182 - goto destroy_eqs; 183 183 - 118 118 + dev->ib_dev.num_comp_vectors = gc->max_num_queues; 119 119 + dev->ib_dev.dev.parent = gc->dev; 120 120 + dev->gdma_dev = mdev; 184 121 xa_init_flags(&dev->qp_table_wq, XA_FLAGS_LOCK_IRQ); 185 185 - ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr); 186 186 - if (ret) { 187 187 - ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", 188 188 - ret); 189 189 - goto destroy_rnic; 122 122 + 123 123 + if (mana_ib_is_rnic(dev)) { 124 124 + dev->ib_dev.phys_port_cnt = 1; 125 125 + ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker); 126 126 + if (!ndev) { 127 127 + ret = -ENODEV; 128 128 + ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1"); 129 129 + goto free_ib_device; 130 130 + } 131 131 + ether_addr_copy(mac_addr, ndev->dev_addr); 132 132 + addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr); 133 133 + ret = ib_device_set_netdev(&dev->ib_dev, ndev, 1); 134 134 + /* mana_get_primary_netdev() returns ndev with refcount held */ 135 135 + netdev_put(ndev, &dev->dev_tracker); 136 136 + if (ret) { 137 137 + ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret); 138 138 + goto free_ib_device; 139 139 + } 140 140 + 141 141 + dev->nb.notifier_call = mana_ib_netdev_event; 142 142 + ret = register_netdevice_notifier(&dev->nb); 143 143 + if (ret) { 144 144 + ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", 145 145 + ret); 146 146 + goto free_ib_device; 147 147 + } 148 148 + 149 149 + ret = mana_ib_gd_query_adapter_caps(dev); 150 150 + if (ret) { 151 151 + ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret); 152 152 + goto deregister_net_notifier; 153 153 + } 154 154 + 155 155 + ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops); 156 156 + 157 157 + ret = mana_ib_create_eqs(dev); 158 158 + if (ret) { 159 159 + ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret); 160 160 + goto deregister_net_notifier; 161 161 + } 162 162 + 163 163 + ret = mana_ib_gd_create_rnic_adapter(dev); 164 164 + if (ret) 165 165 + goto destroy_eqs; 166 166 + 167 167 + ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr); 168 168 + if (ret) { 169 169 + ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret); 170 170 + goto destroy_rnic; 171 171 + } 172 172 + } else { 173 173 + dev->ib_dev.phys_port_cnt = mc->num_ports; 174 174 + ret = mana_eth_query_adapter_caps(dev); 175 175 + if (ret) { 176 176 + ibdev_err(&dev->ib_dev, "Failed to query ETH device caps, ret %d", ret); 177 177 + goto free_ib_device; 178 178 + } 190 179 } 191 180 192 192 - dev->av_pool = dma_pool_create("mana_ib_av", mdev->gdma_context->dev, 193 193 - MANA_AV_BUFFER_SIZE, MANA_AV_BUFFER_SIZE, 0); 181 181 + dev->av_pool = dma_pool_create("mana_ib_av", gc->dev, MANA_AV_BUFFER_SIZE, 182 182 + MANA_AV_BUFFER_SIZE, 0); 194 183 if (!dev->av_pool) { 195 184 ret = -ENOMEM; 196 185 goto destroy_rnic; 197 186 } 198 187 199 199 - ret = ib_register_device(&dev->ib_dev, "mana_%d", 200 200 - mdev->gdma_context->dev); 188 188 + ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev, 189 189 + mdev->dev_id.as_uint32, dev->ib_dev.phys_port_cnt); 190 190 + 191 191 + ret = ib_register_device(&dev->ib_dev, mana_ib_is_rnic(dev) ? "mana_%d" : "manae_%d", 192 192 + gc->dev); 201 193 if (ret) 202 194 goto deallocate_pool; 203 195 ··· 200 208 deallocate_pool: 201 209 dma_pool_destroy(dev->av_pool); 202 210 destroy_rnic: 203 203 - xa_destroy(&dev->qp_table_wq); 204 204 - mana_ib_gd_destroy_rnic_adapter(dev); 211 211 + if (mana_ib_is_rnic(dev)) 212 212 + mana_ib_gd_destroy_rnic_adapter(dev); 205 213 destroy_eqs: 206 206 - mana_ib_destroy_eqs(dev); 214 214 + if (mana_ib_is_rnic(dev)) 215 215 + mana_ib_destroy_eqs(dev); 207 216 deregister_net_notifier: 208 208 - unregister_netdevice_notifier(&dev->nb); 209 209 - deregister_device: 210 210 - mana_gd_deregister_device(dev->gdma_dev); 217 217 + if (mana_ib_is_rnic(dev)) 218 218 + unregister_netdevice_notifier(&dev->nb); 211 219 free_ib_device: 220 220 + xa_destroy(&dev->qp_table_wq); 212 221 ib_dealloc_device(&dev->ib_dev); 213 222 return ret; 214 223 } ··· 220 227 221 228 ib_unregister_device(&dev->ib_dev); 222 229 dma_pool_destroy(dev->av_pool); 230 230 + if (mana_ib_is_rnic(dev)) { 231 231 + mana_ib_gd_destroy_rnic_adapter(dev); 232 232 + mana_ib_destroy_eqs(dev); 233 233 + unregister_netdevice_notifier(&dev->nb); 234 234 + } 223 235 xa_destroy(&dev->qp_table_wq); 224 224 - mana_ib_gd_destroy_rnic_adapter(dev); 225 225 - mana_ib_destroy_eqs(dev); 226 226 - unregister_netdevice_notifier(&dev->nb); 227 227 - mana_gd_deregister_device(dev->gdma_dev); 228 236 ib_dealloc_device(&dev->ib_dev); 229 237 } 230 238 231 239 static const struct auxiliary_device_id mana_id_table[] = { 232 232 - { 233 233 - .name = "mana.rdma", 234 234 - }, 240 240 + { .name = "mana.rdma", }, 241 241 + { .name = "mana.eth", }, 235 242 {}, 236 243 }; 237 244 238 245 MODULE_DEVICE_TABLE(auxiliary, mana_id_table); 239 246 240 247 static struct auxiliary_driver mana_driver = { 241 241 - .name = "rdma", 242 248 .probe = mana_ib_probe, 243 249 .remove = mana_ib_remove, 244 250 .id_table = mana_id_table,

+69 -23

drivers/infiniband/hw/mana/main.c

reviewed

··· 4 4 */ 5 5 6 6 #include "mana_ib.h" 7 7 + #include "linux/pci.h" 7 8 8 9 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd, 9 10 u32 port) ··· 244 243 int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type, 245 244 struct mana_ib_queue *queue) 246 245 { 247 247 - struct gdma_context *gc = mdev_to_gc(mdev); 248 246 struct gdma_queue_spec spec = {}; 249 247 int err; 250 248 ··· 252 252 spec.type = type; 253 253 spec.monitor_avl_buf = false; 254 254 spec.queue_size = size; 255 255 - err = mana_gd_create_mana_wq_cq(&gc->mana_ib, &spec, &queue->kmem); 255 255 + err = mana_gd_create_mana_wq_cq(mdev->gdma_dev, &spec, &queue->kmem); 256 256 if (err) 257 257 return err; 258 258 /* take ownership into mana_ib from mana */ ··· 479 479 { 480 480 unsigned long page_sz; 481 481 482 482 - page_sz = ib_umem_find_best_pgsz(umem, PAGE_SZ_BM, virt); 482 482 + page_sz = ib_umem_find_best_pgsz(umem, dev->adapter_caps.page_size_cap, virt); 483 483 if (!page_sz) { 484 484 ibdev_dbg(&dev->ib_dev, "Failed to find page size.\n"); 485 485 return -EINVAL; ··· 494 494 unsigned long page_sz; 495 495 496 496 /* Hardware requires dma region to align to chosen page size */ 497 497 - page_sz = ib_umem_find_best_pgoff(umem, PAGE_SZ_BM, 0); 497 497 + page_sz = ib_umem_find_best_pgoff(umem, dev->adapter_caps.page_size_cap, 0); 498 498 if (!page_sz) { 499 499 ibdev_dbg(&dev->ib_dev, "Failed to find page size.\n"); 500 500 return -EINVAL; ··· 551 551 int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, 552 552 struct ib_port_immutable *immutable) 553 553 { 554 554 + struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 554 555 struct ib_port_attr attr; 555 556 int err; 556 557 ··· 561 560 562 561 immutable->pkey_tbl_len = attr.pkey_tbl_len; 563 562 immutable->gid_tbl_len = attr.gid_tbl_len; 564 564 - immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; 565 565 - if (port_num == 1) { 566 566 - immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 563 563 + 564 564 + if (mana_ib_is_rnic(dev)) { 565 565 + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 567 566 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 567 567 + } else { 568 568 + immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; 568 569 } 569 570 570 571 return 0; ··· 575 572 int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, 576 573 struct ib_udata *uhw) 577 574 { 578 578 - struct mana_ib_dev *dev = container_of(ibdev, 579 579 - struct mana_ib_dev, ib_dev); 575 575 + struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 576 576 + struct pci_dev *pdev = to_pci_dev(mdev_to_gc(dev)->dev); 580 577 581 578 memset(props, 0, sizeof(*props)); 579 579 + props->vendor_id = pdev->vendor; 580 580 + props->vendor_part_id = dev->gdma_dev->dev_id.type; 582 581 props->max_mr_size = MANA_IB_MAX_MR_SIZE; 583 583 - props->page_size_cap = PAGE_SZ_BM; 582 582 + props->page_size_cap = dev->adapter_caps.page_size_cap; 584 583 props->max_qp = dev->adapter_caps.max_qp_count; 585 584 props->max_qp_wr = dev->adapter_caps.max_qp_wr; 586 585 props->device_cap_flags = IB_DEVICE_RC_RNR_NAK_GEN; ··· 601 596 props->max_ah = INT_MAX; 602 597 props->max_pkeys = 1; 603 598 props->local_ca_ack_delay = MANA_CA_ACK_DELAY; 599 599 + if (!mana_ib_is_rnic(dev)) 600 600 + props->raw_packet_caps = IB_RAW_PACKET_CAP_IP_CSUM; 604 601 605 602 return 0; 606 603 } ··· 610 603 int mana_ib_query_port(struct ib_device *ibdev, u32 port, 611 604 struct ib_port_attr *props) 612 605 { 606 606 + struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 613 607 struct net_device *ndev = mana_ib_get_netdev(ibdev, port); 614 608 615 609 if (!ndev) ··· 631 623 props->active_width = IB_WIDTH_4X; 632 624 props->active_speed = IB_SPEED_EDR; 633 625 props->pkey_tbl_len = 1; 634 634 - if (port == 1) { 626 626 + if (mana_ib_is_rnic(dev)) { 635 627 props->gid_tbl_len = 16; 636 628 props->port_cap_flags = IB_PORT_CM_SUP; 637 629 props->ip_gids = true; ··· 704 696 caps->max_recv_sge_count = resp.max_recv_sge_count; 705 697 caps->feature_flags = resp.feature_flags; 706 698 699 699 + caps->page_size_cap = PAGE_SZ_BM; 700 700 + if (mdev_to_gc(dev)->pf_cap_flags1 & GDMA_DRV_CAP_FLAG_1_GDMA_PAGES_4MB_1GB_2GB) 701 701 + caps->page_size_cap |= (SZ_4M | SZ_1G | SZ_2G); 702 702 + 703 703 + return 0; 704 704 + } 705 705 + 706 706 + int mana_eth_query_adapter_caps(struct mana_ib_dev *dev) 707 707 + { 708 708 + struct mana_ib_adapter_caps *caps = &dev->adapter_caps; 709 709 + struct gdma_query_max_resources_resp resp = {}; 710 710 + struct gdma_general_req req = {}; 711 711 + int err; 712 712 + 713 713 + mana_gd_init_req_hdr(&req.hdr, GDMA_QUERY_MAX_RESOURCES, 714 714 + sizeof(req), sizeof(resp)); 715 715 + 716 716 + err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req), &req, sizeof(resp), &resp); 717 717 + if (err) { 718 718 + ibdev_err(&dev->ib_dev, 719 719 + "Failed to query adapter caps err %d", err); 720 720 + return err; 721 721 + } 722 722 + 723 723 + caps->max_qp_count = min_t(u32, resp.max_sq, resp.max_rq); 724 724 + caps->max_cq_count = resp.max_cq; 725 725 + caps->max_mr_count = resp.max_mst; 726 726 + caps->max_pd_count = 0x6000; 727 727 + caps->max_qp_wr = min_t(u32, 728 728 + 0x100000 / GDMA_MAX_SQE_SIZE, 729 729 + 0x100000 / GDMA_MAX_RQE_SIZE); 730 730 + caps->max_send_sge_count = 30; 731 731 + caps->max_recv_sge_count = 15; 732 732 + caps->page_size_cap = PAGE_SZ_BM; 733 733 + 707 734 return 0; 708 735 } 709 736 ··· 783 740 spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; 784 741 spec.eq.msix_index = 0; 785 742 786 786 - err = mana_gd_create_mana_eq(&gc->mana_ib, &spec, &mdev->fatal_err_eq); 743 743 + err = mana_gd_create_mana_eq(mdev->gdma_dev, &spec, &mdev->fatal_err_eq); 787 744 if (err) 788 745 return err; 789 746 ··· 834 791 835 792 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER, sizeof(req), sizeof(resp)); 836 793 req.hdr.req.msg_version = GDMA_MESSAGE_V2; 837 837 - req.hdr.dev_id = gc->mana_ib.dev_id; 794 794 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 838 795 req.notify_eq_id = mdev->fatal_err_eq->id; 839 796 840 797 if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT) ··· 859 816 860 817 gc = mdev_to_gc(mdev); 861 818 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_ADAPTER, sizeof(req), sizeof(resp)); 862 862 - req.hdr.dev_id = gc->mana_ib.dev_id; 819 819 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 863 820 req.adapter = mdev->adapter_handle; 864 821 865 822 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); ··· 886 843 } 887 844 888 845 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp)); 889 889 - req.hdr.dev_id = gc->mana_ib.dev_id; 846 846 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 890 847 req.adapter = mdev->adapter_handle; 891 848 req.op = ADDR_OP_ADD; 892 849 req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4; ··· 916 873 } 917 874 918 875 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp)); 919 919 - req.hdr.dev_id = gc->mana_ib.dev_id; 876 876 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 920 877 req.adapter = mdev->adapter_handle; 921 878 req.op = ADDR_OP_REMOVE; 922 879 req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4; ··· 939 896 int err; 940 897 941 898 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_MAC_ADDR, sizeof(req), sizeof(resp)); 942 942 - req.hdr.dev_id = gc->mana_ib.dev_id; 899 899 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 943 900 req.adapter = mdev->adapter_handle; 944 901 req.op = op; 945 902 copy_in_reverse(req.mac_addr, mac, ETH_ALEN); ··· 960 917 struct mana_rnic_create_cq_req req = {}; 961 918 int err; 962 919 920 920 + if (!mdev->eqs) 921 921 + return -EINVAL; 922 922 + 963 923 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_CQ, sizeof(req), sizeof(resp)); 964 964 - req.hdr.dev_id = gc->mana_ib.dev_id; 924 924 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 965 925 req.adapter = mdev->adapter_handle; 966 926 req.gdma_region = cq->queue.gdma_region; 967 927 req.eq_id = mdev->eqs[cq->comp_vector]->id; ··· 996 950 return 0; 997 951 998 952 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_CQ, sizeof(req), sizeof(resp)); 999 999 - req.hdr.dev_id = gc->mana_ib.dev_id; 953 953 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 1000 954 req.adapter = mdev->adapter_handle; 1001 955 req.cq_handle = cq->cq_handle; 1002 956 ··· 1022 976 int err, i; 1023 977 1024 978 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_RC_QP, sizeof(req), sizeof(resp)); 1025 1025 - req.hdr.dev_id = gc->mana_ib.dev_id; 979 979 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 1026 980 req.adapter = mdev->adapter_handle; 1027 981 req.pd_handle = pd->pd_handle; 1028 982 req.send_cq_handle = send_cq->cq_handle; ··· 1058 1012 int err; 1059 1013 1060 1014 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_RC_QP, sizeof(req), sizeof(resp)); 1061 1061 - req.hdr.dev_id = gc->mana_ib.dev_id; 1015 1015 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 1062 1016 req.adapter = mdev->adapter_handle; 1063 1017 req.rc_qp_handle = qp->qp_handle; 1064 1018 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); ··· 1081 1035 int err, i; 1082 1036 1083 1037 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_UD_QP, sizeof(req), sizeof(resp)); 1084 1084 - req.hdr.dev_id = gc->mana_ib.dev_id; 1038 1038 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 1085 1039 req.adapter = mdev->adapter_handle; 1086 1040 req.pd_handle = pd->pd_handle; 1087 1041 req.send_cq_handle = send_cq->cq_handle; ··· 1116 1070 int err; 1117 1071 1118 1072 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_UD_QP, sizeof(req), sizeof(resp)); 1119 1119 - req.hdr.dev_id = gc->mana_ib.dev_id; 1073 1073 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 1120 1074 req.adapter = mdev->adapter_handle; 1121 1075 req.qp_handle = qp->qp_handle; 1122 1076 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);

+7

drivers/infiniband/hw/mana/mana_ib.h

reviewed

··· 60 60 u32 max_recv_sge_count; 61 61 u32 max_inline_data_size; 62 62 u64 feature_flags; 63 63 + u64 page_size_cap; 63 64 }; 64 65 65 66 struct mana_ib_queue { ··· 544 543 complete(&qp->free); 545 544 } 546 545 546 546 + static inline bool mana_ib_is_rnic(struct mana_ib_dev *mdev) 547 547 + { 548 548 + return mdev->gdma_dev->dev_id.type == GDMA_DEVICE_MANA_IB; 549 549 + } 550 550 + 547 551 static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port) 548 552 { 549 553 struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); ··· 648 642 void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext); 649 643 650 644 int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev); 645 645 + int mana_eth_query_adapter_caps(struct mana_ib_dev *mdev); 651 646 652 647 int mana_ib_create_eqs(struct mana_ib_dev *mdev); 653 648

+21 -8

drivers/infiniband/hw/mana/mr.c

reviewed

··· 5 5 6 6 #include "mana_ib.h" 7 7 8 8 - #define VALID_MR_FLAGS \ 9 9 - (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ) 8 8 + #define VALID_MR_FLAGS (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |\ 9 9 + IB_ACCESS_REMOTE_ATOMIC | IB_ZERO_BASED) 10 10 11 11 #define VALID_DMA_MR_FLAGS (IB_ACCESS_LOCAL_WRITE) 12 12 ··· 23 23 24 24 if (access_flags & IB_ACCESS_REMOTE_READ) 25 25 flags |= GDMA_ACCESS_FLAG_REMOTE_READ; 26 26 + 27 27 + if (access_flags & IB_ACCESS_REMOTE_ATOMIC) 28 28 + flags |= GDMA_ACCESS_FLAG_REMOTE_ATOMIC; 26 29 27 30 return flags; 28 31 } ··· 51 48 req.gva.virtual_address = mr_params->gva.virtual_address; 52 49 req.gva.access_flags = mr_params->gva.access_flags; 53 50 break; 54 54 - 51 51 + case GDMA_MR_TYPE_ZBVA: 52 52 + req.zbva.dma_region_handle = mr_params->zbva.dma_region_handle; 53 53 + req.zbva.access_flags = mr_params->zbva.access_flags; 54 54 + break; 55 55 default: 56 56 ibdev_dbg(&dev->ib_dev, 57 57 "invalid param (GDMA_MR_TYPE) passed, type %d\n", ··· 150 144 dma_region_handle); 151 145 152 146 mr_params.pd_handle = pd->pd_handle; 153 153 - mr_params.mr_type = GDMA_MR_TYPE_GVA; 154 154 - mr_params.gva.dma_region_handle = dma_region_handle; 155 155 - mr_params.gva.virtual_address = iova; 156 156 - mr_params.gva.access_flags = 157 157 - mana_ib_verbs_to_gdma_access_flags(access_flags); 147 147 + if (access_flags & IB_ZERO_BASED) { 148 148 + mr_params.mr_type = GDMA_MR_TYPE_ZBVA; 149 149 + mr_params.zbva.dma_region_handle = dma_region_handle; 150 150 + mr_params.zbva.access_flags = 151 151 + mana_ib_verbs_to_gdma_access_flags(access_flags); 152 152 + } else { 153 153 + mr_params.mr_type = GDMA_MR_TYPE_GVA; 154 154 + mr_params.gva.dma_region_handle = dma_region_handle; 155 155 + mr_params.gva.virtual_address = iova; 156 156 + mr_params.gva.access_flags = 157 157 + mana_ib_verbs_to_gdma_access_flags(access_flags); 158 158 + } 158 159 159 160 err = mana_ib_gd_create_mr(dev, mr, &mr_params); 160 161 if (err)

+2 -3

drivers/infiniband/hw/mana/qp.c

reviewed

··· 635 635 { 636 636 struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev); 637 637 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 638 638 - struct gdma_context *gc = mdev_to_gc(mdev); 639 638 u32 doorbell, queue_size; 640 639 int i, err; 641 640 ··· 653 654 goto destroy_queues; 654 655 } 655 656 } 656 656 - doorbell = gc->mana_ib.doorbell; 657 657 + doorbell = mdev->gdma_dev->doorbell; 657 658 658 659 err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr, 659 660 sizeof(struct ud_rq_shadow_wqe)); ··· 735 736 int err; 736 737 737 738 mana_gd_init_req_hdr(&req.hdr, MANA_IB_SET_QP_STATE, sizeof(req), sizeof(resp)); 738 738 - req.hdr.dev_id = gc->mana_ib.dev_id; 739 739 + req.hdr.dev_id = mdev->gdma_dev->dev_id; 739 740 req.adapter = mdev->adapter_handle; 740 741 req.qp_handle = qp->qp_handle; 741 742 req.qp_state = attr->qp_state;

+4 -4

drivers/infiniband/hw/mlx4/mcg.c

reviewed

··· 43 43 44 44 #define MAX_VFS 80 45 45 #define MAX_PEND_REQS_PER_FUNC 4 46 46 - #define MAD_TIMEOUT_MS 2000 46 46 + #define MAD_TIMEOUT_SEC 2 47 47 48 48 #define mcg_warn(fmt, arg...) pr_warn("MCG WARNING: " fmt, ##arg) 49 49 #define mcg_error(fmt, arg...) pr_err(fmt, ##arg) ··· 270 270 if (!ret) { 271 271 /* calls mlx4_ib_mcg_timeout_handler */ 272 272 queue_delayed_work(group->demux->mcg_wq, &group->timeout_work, 273 273 - msecs_to_jiffies(MAD_TIMEOUT_MS)); 273 273 + secs_to_jiffies(MAD_TIMEOUT_SEC)); 274 274 } 275 275 276 276 return ret; ··· 309 309 if (!ret) { 310 310 /* calls mlx4_ib_mcg_timeout_handler */ 311 311 queue_delayed_work(group->demux->mcg_wq, &group->timeout_work, 312 312 - msecs_to_jiffies(MAD_TIMEOUT_MS)); 312 312 + secs_to_jiffies(MAD_TIMEOUT_SEC)); 313 313 } 314 314 315 315 return ret; ··· 1091 1091 for (i = 0; i < MAX_VFS; ++i) 1092 1092 clean_vf_mcast(ctx, i); 1093 1093 1094 1094 - end = jiffies + msecs_to_jiffies(MAD_TIMEOUT_MS + 3000); 1094 1094 + end = jiffies + secs_to_jiffies(MAD_TIMEOUT_SEC + 3); 1095 1095 do { 1096 1096 count = 0; 1097 1097 mutex_lock(&ctx->mcg_table_lock);

+21 -37

drivers/infiniband/hw/mlx5/fs.c

reviewed

··· 1645 1645 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); 1646 1646 } 1647 1647 1648 1648 - enum { 1649 1649 - LEFTOVERS_MC, 1650 1650 - LEFTOVERS_UC, 1651 1651 - }; 1652 1652 - 1653 1648 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, 1654 1649 struct mlx5_ib_flow_prio *ft_prio, 1655 1650 struct ib_flow_attr *flow_attr, ··· 1654 1659 struct mlx5_ib_flow_handler *handler = NULL; 1655 1660 1656 1661 static struct { 1657 1657 - struct ib_flow_attr flow_attr; 1658 1662 struct ib_flow_spec_eth eth_flow; 1659 1659 - } leftovers_specs[] = { 1660 1660 - [LEFTOVERS_MC] = { 1661 1661 - .flow_attr = { 1662 1662 - .num_of_specs = 1, 1663 1663 - .size = sizeof(leftovers_specs[0]) 1664 1664 - }, 1665 1665 - .eth_flow = { 1666 1666 - .type = IB_FLOW_SPEC_ETH, 1667 1667 - .size = sizeof(struct ib_flow_spec_eth), 1668 1668 - .mask = {.dst_mac = {0x1} }, 1669 1669 - .val = {.dst_mac = {0x1} } 1670 1670 - } 1671 1671 - }, 1672 1672 - [LEFTOVERS_UC] = { 1673 1673 - .flow_attr = { 1674 1674 - .num_of_specs = 1, 1675 1675 - .size = sizeof(leftovers_specs[0]) 1676 1676 - }, 1677 1677 - .eth_flow = { 1678 1678 - .type = IB_FLOW_SPEC_ETH, 1679 1679 - .size = sizeof(struct ib_flow_spec_eth), 1680 1680 - .mask = {.dst_mac = {0x1} }, 1681 1681 - .val = {.dst_mac = {} } 1682 1682 - } 1683 1683 - } 1684 1684 - }; 1663 1663 + struct ib_flow_attr flow_attr; 1664 1664 + } leftovers_wc = { .flow_attr = { .num_of_specs = 1, 1665 1665 + .size = sizeof(leftovers_wc) }, 1666 1666 + .eth_flow = { 1667 1667 + .type = IB_FLOW_SPEC_ETH, 1668 1668 + .size = sizeof(struct ib_flow_spec_eth), 1669 1669 + .mask = { .dst_mac = { 0x1 } }, 1670 1670 + .val = { .dst_mac = { 0x1 } } } }; 1685 1671 1686 1686 - handler = create_flow_rule(dev, ft_prio, 1687 1687 - &leftovers_specs[LEFTOVERS_MC].flow_attr, 1688 1688 - dst); 1672 1672 + static struct { 1673 1673 + struct ib_flow_spec_eth eth_flow; 1674 1674 + struct ib_flow_attr flow_attr; 1675 1675 + } leftovers_uc = { .flow_attr = { .num_of_specs = 1, 1676 1676 + .size = sizeof(leftovers_uc) }, 1677 1677 + .eth_flow = { 1678 1678 + .type = IB_FLOW_SPEC_ETH, 1679 1679 + .size = sizeof(struct ib_flow_spec_eth), 1680 1680 + .mask = { .dst_mac = { 0x1 } }, 1681 1681 + .val = { .dst_mac = {} } } }; 1682 1682 + 1683 1683 + handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst); 1689 1684 if (!IS_ERR(handler) && 1690 1685 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { 1691 1686 handler_ucast = create_flow_rule(dev, ft_prio, 1692 1692 - &leftovers_specs[LEFTOVERS_UC].flow_attr, 1693 1693 - dst); 1687 1687 + &leftovers_uc.flow_attr, dst); 1694 1688 if (IS_ERR(handler_ucast)) { 1695 1689 mlx5_del_flow_rules(handler->rule); 1696 1690 ft_prio->refcount--;

+12 -17

drivers/infiniband/hw/mlx5/main.c

reviewed

··· 485 485 *active_width = IB_WIDTH_2X; 486 486 *active_speed = IB_SPEED_NDR; 487 487 break; 488 488 + case MLX5E_PROT_MASK(MLX5E_200GAUI_1_200GBASE_CR1_KR1): 489 489 + *active_width = IB_WIDTH_1X; 490 490 + *active_speed = IB_SPEED_XDR; 491 491 + break; 488 492 case MLX5E_PROT_MASK(MLX5E_400GAUI_8_400GBASE_CR8): 489 493 *active_width = IB_WIDTH_8X; 490 494 *active_speed = IB_SPEED_HDR; ··· 497 493 *active_width = IB_WIDTH_4X; 498 494 *active_speed = IB_SPEED_NDR; 499 495 break; 496 496 + case MLX5E_PROT_MASK(MLX5E_400GAUI_2_400GBASE_CR2_KR2): 497 497 + *active_width = IB_WIDTH_2X; 498 498 + *active_speed = IB_SPEED_XDR; 499 499 + break; 500 500 case MLX5E_PROT_MASK(MLX5E_800GAUI_8_800GBASE_CR8_KR8): 501 501 *active_width = IB_WIDTH_8X; 502 502 *active_speed = IB_SPEED_NDR; 503 503 + break; 504 504 + case MLX5E_PROT_MASK(MLX5E_800GAUI_4_800GBASE_CR4_KR4): 505 505 + *active_width = IB_WIDTH_4X; 506 506 + *active_speed = IB_SPEED_XDR; 503 507 break; 504 508 default: 505 509 return -EINVAL; ··· 4434 4422 mlx5_core_native_port_num(dev->mdev) - 1); 4435 4423 } 4436 4424 4437 4437 - static int mlx5_ib_stage_uar_init(struct mlx5_ib_dev *dev) 4438 4438 - { 4439 4439 - dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev); 4440 4440 - return PTR_ERR_OR_ZERO(dev->mdev->priv.uar); 4441 4441 - } 4442 4442 - 4443 4443 - static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev) 4444 4444 - { 4445 4445 - mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar); 4446 4446 - } 4447 4447 - 4448 4425 static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev) 4449 4426 { 4450 4427 int err; ··· 4663 4662 STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS, 4664 4663 mlx5_ib_stage_cong_debugfs_init, 4665 4664 mlx5_ib_stage_cong_debugfs_cleanup), 4666 4666 - STAGE_CREATE(MLX5_IB_STAGE_UAR, 4667 4667 - mlx5_ib_stage_uar_init, 4668 4668 - mlx5_ib_stage_uar_cleanup), 4669 4665 STAGE_CREATE(MLX5_IB_STAGE_BFREG, 4670 4666 mlx5_ib_stage_bfrag_init, 4671 4667 mlx5_ib_stage_bfrag_cleanup), ··· 4720 4722 STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS, 4721 4723 mlx5_ib_stage_cong_debugfs_init, 4722 4724 mlx5_ib_stage_cong_debugfs_cleanup), 4723 4723 - STAGE_CREATE(MLX5_IB_STAGE_UAR, 4724 4724 - mlx5_ib_stage_uar_init, 4725 4725 - mlx5_ib_stage_uar_cleanup), 4726 4725 STAGE_CREATE(MLX5_IB_STAGE_BFREG, 4727 4726 mlx5_ib_stage_bfrag_init, 4728 4727 mlx5_ib_stage_bfrag_cleanup),

+8 -5

drivers/infiniband/hw/mlx5/mlx5_ib.h

reviewed

··· 351 351 #define MLX5_IB_UPD_XLT_PD BIT(4) 352 352 #define MLX5_IB_UPD_XLT_ACCESS BIT(5) 353 353 #define MLX5_IB_UPD_XLT_INDIRECT BIT(6) 354 354 + #define MLX5_IB_UPD_XLT_DOWNGRADE BIT(7) 354 355 355 356 /* Private QP creation flags to be passed in ib_qp_init_attr.create_flags. 356 357 * ··· 1006 1005 MLX5_IB_STAGE_ODP, 1007 1006 MLX5_IB_STAGE_COUNTERS, 1008 1007 MLX5_IB_STAGE_CONG_DEBUGFS, 1009 1009 - MLX5_IB_STAGE_UAR, 1010 1008 MLX5_IB_STAGE_BFREG, 1011 1009 MLX5_IB_STAGE_PRE_IB_REG_UMR, 1012 1010 MLX5_IB_STAGE_WHITELIST_UID, ··· 1473 1473 int __init mlx5_ib_odp_init(void); 1474 1474 void mlx5_ib_odp_cleanup(void); 1475 1475 int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev); 1476 1476 - void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 1477 1477 - struct mlx5_ib_mr *mr, int flags); 1476 1476 + int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 1477 1477 + struct mlx5_ib_mr *mr, int flags); 1478 1478 1479 1479 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, 1480 1480 enum ib_uverbs_advise_mr_advice advice, ··· 1495 1495 { 1496 1496 return 0; 1497 1497 } 1498 1498 - static inline void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 1499 1499 - struct mlx5_ib_mr *mr, int flags) {} 1498 1498 + static inline int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 1499 1499 + struct mlx5_ib_mr *mr, int flags) 1500 1500 + { 1501 1501 + return -EOPNOTSUPP; 1502 1502 + } 1500 1503 1501 1504 static inline int 1502 1505 mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,

+3 -3

drivers/infiniband/hw/mlx5/mr.c

reviewed

··· 525 525 ent->fill_to_high_water = false; 526 526 if (ent->pending) 527 527 queue_delayed_work(ent->dev->cache.wq, &ent->dwork, 528 528 - msecs_to_jiffies(1000)); 528 528 + secs_to_jiffies(1)); 529 529 else 530 530 mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0); 531 531 } ··· 576 576 "add keys command failed, err %d\n", 577 577 err); 578 578 queue_delayed_work(cache->wq, &ent->dwork, 579 579 - msecs_to_jiffies(1000)); 579 579 + secs_to_jiffies(1)); 580 580 } 581 581 } 582 582 } else if (ent->mkeys_queue.ci > 2 * ent->limit) { ··· 2051 2051 ent->in_use--; 2052 2052 if (ent->is_tmp && !ent->tmp_cleanup_scheduled) { 2053 2053 mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 2054 2054 - msecs_to_jiffies(30 * 1000)); 2054 2054 + secs_to_jiffies(30)); 2055 2055 ent->tmp_cleanup_scheduled = true; 2056 2056 } 2057 2057 spin_unlock_irq(&ent->mkeys_queue.lock);

+39 -26

drivers/infiniband/hw/mlx5/odp.c

reviewed

··· 34 34 #include <linux/kernel.h> 35 35 #include <linux/dma-buf.h> 36 36 #include <linux/dma-resv.h> 37 37 + #include <linux/hmm.h> 38 38 + #include <linux/hmm-dma.h> 39 39 + #include <linux/pci-p2pdma.h> 37 40 38 41 #include "mlx5_ib.h" 39 42 #include "cmd.h" ··· 161 158 } 162 159 } 163 160 164 164 - static u64 umem_dma_to_mtt(dma_addr_t umem_dma) 165 165 - { 166 166 - u64 mtt_entry = umem_dma & ODP_DMA_ADDR_MASK; 167 167 - 168 168 - if (umem_dma & ODP_READ_ALLOWED_BIT) 169 169 - mtt_entry |= MLX5_IB_MTT_READ; 170 170 - if (umem_dma & ODP_WRITE_ALLOWED_BIT) 171 171 - mtt_entry |= MLX5_IB_MTT_WRITE; 172 172 - 173 173 - return mtt_entry; 174 174 - } 175 175 - 176 176 - static void populate_mtt(__be64 *pas, size_t idx, size_t nentries, 177 177 - struct mlx5_ib_mr *mr, int flags) 161 161 + static int populate_mtt(__be64 *pas, size_t start, size_t nentries, 162 162 + struct mlx5_ib_mr *mr, int flags) 178 163 { 179 164 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 180 180 - dma_addr_t pa; 165 165 + bool downgrade = flags & MLX5_IB_UPD_XLT_DOWNGRADE; 166 166 + struct pci_p2pdma_map_state p2pdma_state = {}; 167 167 + struct ib_device *dev = odp->umem.ibdev; 181 168 size_t i; 182 169 183 170 if (flags & MLX5_IB_UPD_XLT_ZAP) 184 184 - return; 171 171 + return 0; 185 172 186 173 for (i = 0; i < nentries; i++) { 187 187 - pa = odp->dma_list[idx + i]; 188 188 - pas[i] = cpu_to_be64(umem_dma_to_mtt(pa)); 174 174 + unsigned long pfn = odp->map.pfn_list[start + i]; 175 175 + dma_addr_t dma_addr; 176 176 + 177 177 + pfn = odp->map.pfn_list[start + i]; 178 178 + if (!(pfn & HMM_PFN_VALID)) 179 179 + /* ODP initialization */ 180 180 + continue; 181 181 + 182 182 + dma_addr = hmm_dma_map_pfn(dev->dma_device, &odp->map, 183 183 + start + i, &p2pdma_state); 184 184 + if (ib_dma_mapping_error(dev, dma_addr)) 185 185 + return -EFAULT; 186 186 + 187 187 + dma_addr |= MLX5_IB_MTT_READ; 188 188 + if ((pfn & HMM_PFN_WRITE) && !downgrade) 189 189 + dma_addr |= MLX5_IB_MTT_WRITE; 190 190 + 191 191 + pas[i] = cpu_to_be64(dma_addr); 192 192 + odp->npages++; 189 193 } 194 194 + return 0; 190 195 } 191 196 192 192 - void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 193 193 - struct mlx5_ib_mr *mr, int flags) 197 197 + int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 198 198 + struct mlx5_ib_mr *mr, int flags) 194 199 { 195 200 if (flags & MLX5_IB_UPD_XLT_INDIRECT) { 196 201 populate_klm(xlt, idx, nentries, mr, flags); 202 202 + return 0; 197 203 } else { 198 198 - populate_mtt(xlt, idx, nentries, mr, flags); 204 204 + return populate_mtt(xlt, idx, nentries, mr, flags); 199 205 } 200 206 } 201 207 ··· 315 303 * estimate the cost of another UMR vs. the cost of bigger 316 304 * UMR. 317 305 */ 318 318 - if (umem_odp->dma_list[idx] & 319 319 - (ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) { 306 306 + if (umem_odp->map.pfn_list[idx] & HMM_PFN_VALID) { 320 307 if (!in_block) { 321 308 blk_start_idx = idx; 322 309 in_block = 1; ··· 698 687 { 699 688 int page_shift, ret, np; 700 689 bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; 701 701 - u64 access_mask; 690 690 + u64 access_mask = 0; 702 691 u64 start_idx; 703 692 bool fault = !(flags & MLX5_PF_FLAGS_SNAPSHOT); 704 693 u32 xlt_flags = MLX5_IB_UPD_XLT_ATOMIC; ··· 706 695 if (flags & MLX5_PF_FLAGS_ENABLE) 707 696 xlt_flags |= MLX5_IB_UPD_XLT_ENABLE; 708 697 698 698 + if (flags & MLX5_PF_FLAGS_DOWNGRADE) 699 699 + xlt_flags |= MLX5_IB_UPD_XLT_DOWNGRADE; 700 700 + 709 701 page_shift = odp->page_shift; 710 702 start_idx = (user_va - ib_umem_start(odp)) >> page_shift; 711 711 - access_mask = ODP_READ_ALLOWED_BIT; 712 703 713 704 if (odp->umem.writable && !downgrade) 714 714 - access_mask |= ODP_WRITE_ALLOWED_BIT; 705 705 + access_mask |= HMM_PFN_WRITE; 715 706 716 707 np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, fault); 717 708 if (np < 0)

+28 -2

drivers/infiniband/hw/mlx5/qpc.c

reviewed

··· 21 21 spin_lock_irqsave(&table->lock, flags); 22 22 23 23 common = radix_tree_lookup(&table->tree, rsn); 24 24 - if (common) 24 24 + if (common && !common->invalid) 25 25 refcount_inc(&common->refcount); 26 26 + else 27 27 + common = NULL; 26 28 27 29 spin_unlock_irqrestore(&table->lock, flags); 28 30 ··· 178 176 qp->pid = current->pid; 179 177 180 178 return 0; 179 179 + } 180 180 + 181 181 + static void modify_resource_common_state(struct mlx5_ib_dev *dev, 182 182 + struct mlx5_core_qp *qp, 183 183 + bool invalid) 184 184 + { 185 185 + struct mlx5_qp_table *table = &dev->qp_table; 186 186 + unsigned long flags; 187 187 + 188 188 + spin_lock_irqsave(&table->lock, flags); 189 189 + qp->common.invalid = invalid; 190 190 + spin_unlock_irqrestore(&table->lock, flags); 181 191 } 182 192 183 193 static void destroy_resource_common(struct mlx5_ib_dev *dev, ··· 623 609 int mlx5_core_destroy_rq_tracked(struct mlx5_ib_dev *dev, 624 610 struct mlx5_core_qp *rq) 625 611 { 612 612 + int ret; 613 613 + 614 614 + /* The rq destruction can be called again in case it fails, hence we 615 615 + * mark the common resource as invalid and only once FW destruction 616 616 + * is completed successfully we actually destroy the resources. 617 617 + */ 618 618 + modify_resource_common_state(dev, rq, true); 619 619 + ret = destroy_rq_tracked(dev, rq->qpn, rq->uid); 620 620 + if (ret) { 621 621 + modify_resource_common_state(dev, rq, false); 622 622 + return ret; 623 623 + } 626 624 destroy_resource_common(dev, rq); 627 627 - return destroy_rq_tracked(dev, rq->qpn, rq->uid); 625 625 + return 0; 628 626 } 629 627 630 628 static void destroy_sq_tracked(struct mlx5_ib_dev *dev, u32 sqn, u16 uid)

+11 -1

drivers/infiniband/hw/mlx5/umr.c

reviewed

··· 840 840 size_to_map = npages * desc_size; 841 841 dma_sync_single_for_cpu(ddev, sg.addr, sg.length, 842 842 DMA_TO_DEVICE); 843 843 - mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags); 843 843 + /* 844 844 + * npages is the maximum number of pages to map, but we 845 845 + * can't guarantee that all pages are actually mapped. 846 846 + * 847 847 + * For example, if page is p2p of type which is not supported 848 848 + * for mapping, the number of pages mapped will be less than 849 849 + * requested. 850 850 + */ 851 851 + err = mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags); 852 852 + if (err) 853 853 + return err; 844 854 dma_sync_single_for_device(ddev, sg.addr, sg.length, 845 855 DMA_TO_DEVICE); 846 856 sg.length = ALIGN(size_to_map, MLX5_UMR_FLEX_ALIGNMENT);

+1 -1

drivers/infiniband/hw/mthca/mthca_mr.c

reviewed

··· 144 144 buddy->max_order = max_order; 145 145 spin_lock_init(&buddy->lock); 146 146 147 147 - buddy->bits = kcalloc(buddy->max_order + 1, sizeof(long *), 147 147 + buddy->bits = kcalloc(buddy->max_order + 1, sizeof(*buddy->bits), 148 148 GFP_KERNEL); 149 149 buddy->num_free = kcalloc((buddy->max_order + 1), sizeof *buddy->num_free, 150 150 GFP_KERNEL);

+1 -1

drivers/infiniband/hw/usnic/usnic_uiom.c

reviewed

··· 56 56 unsigned long iova, int flags, 57 57 void *token) 58 58 { 59 59 - usnic_err("Device %s iommu fault domain 0x%pK va 0x%lx flags 0x%x\n", 59 59 + usnic_err("Device %s iommu fault domain 0x%p va 0x%lx flags 0x%x\n", 60 60 dev_name(dev), 61 61 domain, iova, flags); 62 62 return -ENOSYS;

+1 -1

drivers/infiniband/sw/rxe/Kconfig

reviewed

··· 1 1 # SPDX-License-Identifier: GPL-2.0-only 2 2 config RDMA_RXE 3 3 tristate "Software RDMA over Ethernet (RoCE) driver" 4 4 - depends on INET && PCI && INFINIBAND 4 4 + depends on INET && PCI && INFINIBAND && 64BIT 5 5 depends on INFINIBAND_VIRT_DMA 6 6 select NET_UDP_TUNNEL 7 7 select CRC32

+2

drivers/infiniband/sw/rxe/rxe.c

reviewed

··· 101 101 rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ; 102 102 rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 103 103 rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 104 104 + rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_FLUSH; 105 105 + rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC_WRITE; 104 106 } 105 107 } 106 108

+21 -8

drivers/infiniband/sw/rxe/rxe_loc.h

reviewed

··· 70 70 void *addr, int length, enum rxe_mr_copy_dir dir); 71 71 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 72 72 int sg_nents, unsigned int *sg_offset); 73 73 - int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 74 74 - u64 compare, u64 swap_add, u64 *orig_val); 75 75 - int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value); 73 73 + enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 74 74 + u64 compare, u64 swap_add, u64 *orig_val); 75 75 + enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value); 76 76 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key, 77 77 enum rxe_mr_lookup_type type); 78 78 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length); ··· 193 193 /* rxe_odp.c */ 194 194 extern const struct mmu_interval_notifier_ops rxe_mn_ops; 195 195 196 196 - #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 196 196 + #if defined CONFIG_INFINIBAND_ON_DEMAND_PAGING 197 197 int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, 198 198 u64 iova, int access_flags, struct rxe_mr *mr); 199 199 int rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length, 200 200 enum rxe_mr_copy_dir dir); 201 201 - int rxe_odp_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 202 202 - u64 compare, u64 swap_add, u64 *orig_val); 201 201 + enum resp_states rxe_odp_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 202 202 + u64 compare, u64 swap_add, u64 *orig_val); 203 203 + int rxe_odp_flush_pmem_iova(struct rxe_mr *mr, u64 iova, 204 204 + unsigned int length); 205 205 + enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value); 203 206 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ 204 207 static inline int 205 208 rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, ··· 215 212 { 216 213 return -EOPNOTSUPP; 217 214 } 218 218 - static inline int 215 215 + static inline enum resp_states 219 216 rxe_odp_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 220 220 - u64 compare, u64 swap_add, u64 *orig_val) 217 217 + u64 compare, u64 swap_add, u64 *orig_val) 218 218 + { 219 219 + return RESPST_ERR_UNSUPPORTED_OPCODE; 220 220 + } 221 221 + static inline int rxe_odp_flush_pmem_iova(struct rxe_mr *mr, u64 iova, 222 222 + unsigned int length) 223 223 + { 224 224 + return -EOPNOTSUPP; 225 225 + } 226 226 + static inline enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, 227 227 + u64 iova, u64 value) 221 228 { 222 229 return RESPST_ERR_UNSUPPORTED_OPCODE; 223 230 }

+29 -37

drivers/infiniband/sw/rxe/rxe_mr.c

reviewed

··· 424 424 return err; 425 425 } 426 426 427 427 - int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) 427 427 + static int rxe_mr_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) 428 428 { 429 429 unsigned int page_offset; 430 430 unsigned long index; ··· 432 432 unsigned int bytes; 433 433 int err; 434 434 u8 *va; 435 435 - 436 436 - /* mr must be valid even if length is zero */ 437 437 - if (WARN_ON(!mr)) 438 438 - return -EINVAL; 439 439 - 440 440 - if (length == 0) 441 441 - return 0; 442 442 - 443 443 - if (mr->ibmr.type == IB_MR_TYPE_DMA) 444 444 - return -EFAULT; 445 435 446 436 err = mr_check_range(mr, iova, length); 447 437 if (err) ··· 444 454 if (!page) 445 455 return -EFAULT; 446 456 bytes = min_t(unsigned int, length, 447 447 - mr_page_size(mr) - page_offset); 457 457 + mr_page_size(mr) - page_offset); 448 458 449 459 va = kmap_local_page(page); 450 460 arch_wb_cache_pmem(va + page_offset, bytes); ··· 458 468 return 0; 459 469 } 460 470 471 471 + int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 start, unsigned int length) 472 472 + { 473 473 + int err; 474 474 + 475 475 + /* mr must be valid even if length is zero */ 476 476 + if (WARN_ON(!mr)) 477 477 + return -EINVAL; 478 478 + 479 479 + if (length == 0) 480 480 + return 0; 481 481 + 482 482 + if (mr->ibmr.type == IB_MR_TYPE_DMA) 483 483 + return -EFAULT; 484 484 + 485 485 + if (is_odp_mr(mr)) 486 486 + err = rxe_odp_flush_pmem_iova(mr, start, length); 487 487 + else 488 488 + err = rxe_mr_flush_pmem_iova(mr, start, length); 489 489 + 490 490 + return err; 491 491 + } 492 492 + 461 493 /* Guarantee atomicity of atomic operations at the machine level. */ 462 494 DEFINE_SPINLOCK(atomic_ops_lock); 463 495 464 464 - int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 465 465 - u64 compare, u64 swap_add, u64 *orig_val) 496 496 + enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 497 497 + u64 compare, u64 swap_add, u64 *orig_val) 466 498 { 467 499 unsigned int page_offset; 468 500 struct page *page; ··· 536 524 537 525 kunmap_local(va); 538 526 539 539 - return 0; 527 527 + return RESPST_NONE; 540 528 } 541 529 542 542 - #if defined CONFIG_64BIT 543 543 - /* only implemented or called for 64 bit architectures */ 544 544 - int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) 530 530 + enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) 545 531 { 546 532 unsigned int page_offset; 547 533 struct page *page; 548 534 u64 *va; 549 549 - 550 550 - /* ODP is not supported right now. WIP. */ 551 551 - if (is_odp_mr(mr)) 552 552 - return RESPST_ERR_UNSUPPORTED_OPCODE; 553 553 - 554 554 - /* See IBA oA19-28 */ 555 555 - if (unlikely(mr->state != RXE_MR_STATE_VALID)) { 556 556 - rxe_dbg_mr(mr, "mr not in valid state\n"); 557 557 - return RESPST_ERR_RKEY_VIOLATION; 558 558 - } 559 535 560 536 if (mr->ibmr.type == IB_MR_TYPE_DMA) { 561 537 page_offset = iova & (PAGE_SIZE - 1); ··· 572 572 } 573 573 574 574 va = kmap_local_page(page); 575 575 - 576 575 /* Do atomic write after all prior operations have completed */ 577 576 smp_store_release(&va[page_offset >> 3], value); 578 578 - 579 577 kunmap_local(va); 580 578 581 581 - return 0; 579 579 + return RESPST_NONE; 582 580 } 583 583 - #else 584 584 - int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) 585 585 - { 586 586 - return RESPST_ERR_UNSUPPORTED_OPCODE; 587 587 - } 588 588 - #endif 589 581 590 582 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length) 591 583 {

+119 -25

drivers/infiniband/sw/rxe/rxe_odp.c

reviewed

··· 4 4 */ 5 5 6 6 #include <linux/hmm.h> 7 7 + #include <linux/libnvdimm.h> 7 8 8 9 #include <rdma/ib_umem_odp.h> 9 10 ··· 27 26 start = max_t(u64, ib_umem_start(umem_odp), range->start); 28 27 end = min_t(u64, ib_umem_end(umem_odp), range->end); 29 28 30 30 - /* update umem_odp->dma_list */ 29 29 + /* update umem_odp->map.pfn_list */ 31 30 ib_umem_odp_unmap_dma_pages(umem_odp, start, end); 32 31 33 32 mutex_unlock(&umem_odp->umem_mutex); ··· 45 44 { 46 45 struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); 47 46 bool fault = !(flags & RXE_PAGEFAULT_SNAPSHOT); 48 48 - u64 access_mask; 47 47 + u64 access_mask = 0; 49 48 int np; 50 49 51 51 - access_mask = ODP_READ_ALLOWED_BIT; 52 50 if (umem_odp->umem.writable && !(flags & RXE_PAGEFAULT_RDONLY)) 53 53 - access_mask |= ODP_WRITE_ALLOWED_BIT; 51 51 + access_mask |= HMM_PFN_WRITE; 54 52 55 53 /* 56 54 * ib_umem_odp_map_dma_and_lock() locks umem_mutex on success. ··· 124 124 return err; 125 125 } 126 126 127 127 - static inline bool rxe_check_pagefault(struct ib_umem_odp *umem_odp, 128 128 - u64 iova, int length, u32 perm) 127 127 + static inline bool rxe_check_pagefault(struct ib_umem_odp *umem_odp, u64 iova, 128 128 + int length) 129 129 { 130 130 bool need_fault = false; 131 131 u64 addr; ··· 137 137 while (addr < iova + length) { 138 138 idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; 139 139 140 140 - if (!(umem_odp->dma_list[idx] & perm)) { 140 140 + if (!(umem_odp->map.pfn_list[idx] & HMM_PFN_VALID)) { 141 141 need_fault = true; 142 142 break; 143 143 } ··· 147 147 return need_fault; 148 148 } 149 149 150 150 + static unsigned long rxe_odp_iova_to_index(struct ib_umem_odp *umem_odp, u64 iova) 151 151 + { 152 152 + return (iova - ib_umem_start(umem_odp)) >> umem_odp->page_shift; 153 153 + } 154 154 + 155 155 + static unsigned long rxe_odp_iova_to_page_offset(struct ib_umem_odp *umem_odp, u64 iova) 156 156 + { 157 157 + return iova & (BIT(umem_odp->page_shift) - 1); 158 158 + } 159 159 + 150 160 static int rxe_odp_map_range_and_lock(struct rxe_mr *mr, u64 iova, int length, u32 flags) 151 161 { 152 162 struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); 153 163 bool need_fault; 154 154 - u64 perm; 155 164 int err; 156 165 157 166 if (unlikely(length < 1)) 158 167 return -EINVAL; 159 168 160 160 - perm = ODP_READ_ALLOWED_BIT; 161 161 - if (!(flags & RXE_PAGEFAULT_RDONLY)) 162 162 - perm |= ODP_WRITE_ALLOWED_BIT; 163 163 - 164 169 mutex_lock(&umem_odp->umem_mutex); 165 170 166 166 - need_fault = rxe_check_pagefault(umem_odp, iova, length, perm); 171 171 + need_fault = rxe_check_pagefault(umem_odp, iova, length); 167 172 if (need_fault) { 168 173 mutex_unlock(&umem_odp->umem_mutex); 169 174 ··· 178 173 if (err < 0) 179 174 return err; 180 175 181 181 - need_fault = rxe_check_pagefault(umem_odp, iova, length, perm); 176 176 + need_fault = rxe_check_pagefault(umem_odp, iova, length); 182 177 if (need_fault) 183 178 return -EFAULT; 184 179 } ··· 195 190 size_t offset; 196 191 u8 *user_va; 197 192 198 198 - idx = (iova - ib_umem_start(umem_odp)) >> umem_odp->page_shift; 199 199 - offset = iova & (BIT(umem_odp->page_shift) - 1); 193 193 + idx = rxe_odp_iova_to_index(umem_odp, iova); 194 194 + offset = rxe_odp_iova_to_page_offset(umem_odp, iova); 200 195 201 196 while (length > 0) { 202 197 u8 *src, *dest; 203 198 204 204 - page = hmm_pfn_to_page(umem_odp->pfn_list[idx]); 199 199 + page = hmm_pfn_to_page(umem_odp->map.pfn_list[idx]); 205 200 user_va = kmap_local_page(page); 206 201 if (!user_va) 207 202 return -EFAULT; ··· 260 255 return err; 261 256 } 262 257 263 263 - static int rxe_odp_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 264 264 - u64 compare, u64 swap_add, u64 *orig_val) 258 258 + static enum resp_states rxe_odp_do_atomic_op(struct rxe_mr *mr, u64 iova, 259 259 + int opcode, u64 compare, 260 260 + u64 swap_add, u64 *orig_val) 265 261 { 266 262 struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); 267 263 unsigned int page_offset; ··· 283 277 return RESPST_ERR_RKEY_VIOLATION; 284 278 } 285 279 286 286 - idx = (iova - ib_umem_start(umem_odp)) >> umem_odp->page_shift; 287 287 - page_offset = iova & (BIT(umem_odp->page_shift) - 1); 288 288 - page = hmm_pfn_to_page(umem_odp->pfn_list[idx]); 280 280 + idx = rxe_odp_iova_to_index(umem_odp, iova); 281 281 + page_offset = rxe_odp_iova_to_page_offset(umem_odp, iova); 282 282 + page = hmm_pfn_to_page(umem_odp->map.pfn_list[idx]); 289 283 if (!page) 290 284 return RESPST_ERR_RKEY_VIOLATION; 291 285 ··· 310 304 311 305 kunmap_local(va); 312 306 313 313 - return 0; 307 307 + return RESPST_NONE; 314 308 } 315 309 316 316 - int rxe_odp_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 317 317 - u64 compare, u64 swap_add, u64 *orig_val) 310 310 + enum resp_states rxe_odp_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 311 311 + u64 compare, u64 swap_add, u64 *orig_val) 318 312 { 319 313 struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); 320 314 int err; ··· 329 323 mutex_unlock(&umem_odp->umem_mutex); 330 324 331 325 return err; 326 326 + } 327 327 + 328 328 + int rxe_odp_flush_pmem_iova(struct rxe_mr *mr, u64 iova, 329 329 + unsigned int length) 330 330 + { 331 331 + struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); 332 332 + unsigned int page_offset; 333 333 + unsigned long index; 334 334 + struct page *page; 335 335 + unsigned int bytes; 336 336 + int err; 337 337 + u8 *va; 338 338 + 339 339 + err = rxe_odp_map_range_and_lock(mr, iova, length, 340 340 + RXE_PAGEFAULT_DEFAULT); 341 341 + if (err) 342 342 + return err; 343 343 + 344 344 + while (length > 0) { 345 345 + index = rxe_odp_iova_to_index(umem_odp, iova); 346 346 + page_offset = rxe_odp_iova_to_page_offset(umem_odp, iova); 347 347 + 348 348 + page = hmm_pfn_to_page(umem_odp->map.pfn_list[index]); 349 349 + if (!page) { 350 350 + mutex_unlock(&umem_odp->umem_mutex); 351 351 + return -EFAULT; 352 352 + } 353 353 + 354 354 + bytes = min_t(unsigned int, length, 355 355 + mr_page_size(mr) - page_offset); 356 356 + 357 357 + va = kmap_local_page(page); 358 358 + arch_wb_cache_pmem(va + page_offset, bytes); 359 359 + kunmap_local(va); 360 360 + 361 361 + length -= bytes; 362 362 + iova += bytes; 363 363 + page_offset = 0; 364 364 + } 365 365 + 366 366 + mutex_unlock(&umem_odp->umem_mutex); 367 367 + 368 368 + return 0; 369 369 + } 370 370 + 371 371 + enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) 372 372 + { 373 373 + struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); 374 374 + unsigned int page_offset; 375 375 + unsigned long index; 376 376 + struct page *page; 377 377 + int err; 378 378 + u64 *va; 379 379 + 380 380 + /* See IBA oA19-28 */ 381 381 + err = mr_check_range(mr, iova, sizeof(value)); 382 382 + if (unlikely(err)) { 383 383 + rxe_dbg_mr(mr, "iova out of range\n"); 384 384 + return RESPST_ERR_RKEY_VIOLATION; 385 385 + } 386 386 + 387 387 + err = rxe_odp_map_range_and_lock(mr, iova, sizeof(value), 388 388 + RXE_PAGEFAULT_DEFAULT); 389 389 + if (err) 390 390 + return RESPST_ERR_RKEY_VIOLATION; 391 391 + 392 392 + page_offset = rxe_odp_iova_to_page_offset(umem_odp, iova); 393 393 + index = rxe_odp_iova_to_index(umem_odp, iova); 394 394 + page = hmm_pfn_to_page(umem_odp->map.pfn_list[index]); 395 395 + if (!page) { 396 396 + mutex_unlock(&umem_odp->umem_mutex); 397 397 + return RESPST_ERR_RKEY_VIOLATION; 398 398 + } 399 399 + /* See IBA A19.4.2 */ 400 400 + if (unlikely(page_offset & 0x7)) { 401 401 + mutex_unlock(&umem_odp->umem_mutex); 402 402 + rxe_dbg_mr(mr, "misaligned address\n"); 403 403 + return RESPST_ERR_MISALIGNED_ATOMIC; 404 404 + } 405 405 + 406 406 + va = kmap_local_page(page); 407 407 + /* Do atomic write after all prior operations have completed */ 408 408 + smp_store_release(&va[page_offset >> 3], value); 409 409 + kunmap_local(va); 410 410 + 411 411 + mutex_unlock(&umem_odp->umem_mutex); 412 412 + 413 413 + return RESPST_NONE; 332 414 }

+1 -4

drivers/infiniband/sw/rxe/rxe_param.h

reviewed

··· 53 53 | IB_DEVICE_MEM_WINDOW 54 54 | IB_DEVICE_FLUSH_GLOBAL 55 55 | IB_DEVICE_FLUSH_PERSISTENT 56 56 - #ifdef CONFIG_64BIT 57 56 | IB_DEVICE_MEM_WINDOW_TYPE_2B 58 57 | IB_DEVICE_ATOMIC_WRITE, 59 59 - #else 60 60 - | IB_DEVICE_MEM_WINDOW_TYPE_2B, 61 61 - #endif /* CONFIG_64BIT */ 58 58 + 62 59 RXE_MAX_SGE = 32, 63 60 RXE_MAX_WQE_SIZE = sizeof(struct rxe_send_wqe) + 64 61 sizeof(struct ib_sge) * RXE_MAX_SGE,

+6 -1

drivers/infiniband/sw/rxe/rxe_qp.c

reviewed

··· 811 811 spin_unlock_irqrestore(&qp->state_lock, flags); 812 812 qp->qp_timeout_jiffies = 0; 813 813 814 814 - if (qp_type(qp) == IB_QPT_RC) { 814 814 + /* In the function timer_setup, .function is initialized. If .function 815 815 + * is NULL, it indicates the function timer_setup is not called, the 816 816 + * timer is not initialized. Or else, the timer is initialized. 817 817 + */ 818 818 + if (qp_type(qp) == IB_QPT_RC && qp->retrans_timer.function && 819 819 + qp->rnr_nak_timer.function) { 815 820 timer_delete_sync(&qp->retrans_timer); 816 821 timer_delete_sync(&qp->rnr_nak_timer); 817 822 }

+10 -5

drivers/infiniband/sw/rxe/rxe_resp.c

reviewed

··· 649 649 struct rxe_mr *mr = qp->resp.mr; 650 650 struct resp_res *res = qp->resp.res; 651 651 652 652 - /* ODP is not supported right now. WIP. */ 653 653 - if (is_odp_mr(mr)) 654 654 - return RESPST_ERR_UNSUPPORTED_OPCODE; 655 655 - 656 652 /* oA19-14, oA19-15 */ 657 653 if (res && res->replay) 658 654 return RESPST_ACKNOWLEDGE; ··· 749 753 value = *(u64 *)payload_addr(pkt); 750 754 iova = qp->resp.va + qp->resp.offset; 751 755 752 752 - err = rxe_mr_do_atomic_write(mr, iova, value); 756 756 + /* See IBA oA19-28 */ 757 757 + if (unlikely(mr->state != RXE_MR_STATE_VALID)) { 758 758 + rxe_dbg_mr(mr, "mr not in valid state\n"); 759 759 + return RESPST_ERR_RKEY_VIOLATION; 760 760 + } 761 761 + 762 762 + if (is_odp_mr(mr)) 763 763 + err = rxe_odp_do_atomic_write(mr, iova, value); 764 764 + else 765 765 + err = rxe_mr_do_atomic_write(mr, iova, value); 753 766 if (err) 754 767 return err; 755 768

+11 -29

drivers/infiniband/sw/rxe/rxe_task.c

reviewed

··· 85 85 86 86 /* do_task is a wrapper for the three tasks (requester, 87 87 * completer, responder) and calls them in a loop until 88 88 - * they return a non-zero value. It is called either 89 89 - * directly by rxe_run_task or indirectly if rxe_sched_task 90 90 - * schedules the task. They must call __reserve_if_idle to 91 91 - * move the task to busy before calling or scheduling. 92 92 - * The task can also be moved to drained or invalid 93 93 - * by calls to rxe_cleanup_task or rxe_disable_task. 94 94 - * In that case tasks which get here are not executed but 95 95 - * just flushed. The tasks are designed to look to see if 96 96 - * there is work to do and then do part of it before returning 97 97 - * here with a return value of zero until all the work 98 98 - * has been consumed then it returns a non-zero value. 88 88 + * they return a non-zero value. It is called indirectly 89 89 + * when rxe_sched_task schedules the task. They must 90 90 + * call __reserve_if_idle to move the task to busy before 91 91 + * calling or scheduling. The task can also be moved to 92 92 + * drained or invalid by calls to rxe_cleanup_task or 93 93 + * rxe_disable_task. In that case tasks which get here 94 94 + * are not executed but just flushed. The tasks are 95 95 + * designed to look to see if there is work to do and 96 96 + * then do part of it before returning here with a return 97 97 + * value of zero until all the work has been consumed then 98 98 + * it returns a non-zero value. 99 99 * The number of times the task can be run is limited by 100 100 * max iterations so one task cannot hold the cpu forever. 101 101 * If the limit is hit and work remains the task is rescheduled. ··· 232 232 spin_lock_irqsave(&task->lock, flags); 233 233 task->state = TASK_STATE_INVALID; 234 234 spin_unlock_irqrestore(&task->lock, flags); 235 235 - } 236 236 - 237 237 - /* run the task inline if it is currently idle 238 238 - * cannot call do_task holding the lock 239 239 - */ 240 240 - void rxe_run_task(struct rxe_task *task) 241 241 - { 242 242 - unsigned long flags; 243 243 - bool run; 244 244 - 245 245 - WARN_ON(rxe_read(task->qp) <= 0); 246 246 - 247 247 - spin_lock_irqsave(&task->lock, flags); 248 248 - run = __reserve_if_idle(task); 249 249 - spin_unlock_irqrestore(&task->lock, flags); 250 250 - 251 251 - if (run) 252 252 - do_task(task); 253 235 } 254 236 255 237 /* schedule the task to run later as a work queue entry.

-2

drivers/infiniband/sw/rxe/rxe_task.h

reviewed

··· 47 47 /* cleanup task */ 48 48 void rxe_cleanup_task(struct rxe_task *task); 49 49 50 50 - void rxe_run_task(struct rxe_task *task); 51 51 - 52 50 void rxe_sched_task(struct rxe_task *task); 53 51 54 52 /* keep a task from scheduling */

+1 -1

drivers/infiniband/sw/siw/siw.h

reviewed

··· 718 718 "MEM[0x%08x] %s: " fmt, mem->stag, __func__, ##__VA_ARGS__) 719 719 720 720 #define siw_dbg_cep(cep, fmt, ...) \ 721 721 - ibdev_dbg(&cep->sdev->base_dev, "CEP[0x%pK] %s: " fmt, \ 721 721 + ibdev_dbg(&cep->sdev->base_dev, "CEP[0x%p] %s: " fmt, \ 722 722 cep, __func__, ##__VA_ARGS__) 723 723 724 724 void siw_cq_flush(struct siw_cq *cq);

+1 -1

drivers/infiniband/sw/siw/siw_cq.c

reviewed

··· 72 72 wc->opcode = map_wc_opcode[cqe->opcode]; 73 73 wc->status = map_cqe_status[cqe->status].ib; 74 74 siw_dbg_cq(cq, 75 75 - "idx %u, type %d, flags %2x, id 0x%pK\n", 75 75 + "idx %u, type %d, flags %2x, id 0x%p\n", 76 76 cq->cq_get % cq->num_cqe, cqe->opcode, 77 77 cqe->flags, (void *)(uintptr_t)cqe->id); 78 78 } else {

+2 -26

drivers/infiniband/sw/siw/siw_mem.c

reviewed

··· 18 18 #define SIW_STAG_MAX_INDEX 0x00ffffff 19 19 20 20 /* 21 21 - * The code avoids special Stag of zero and tries to randomize 22 22 - * STag values between 1 and SIW_STAG_MAX_INDEX. 23 23 - */ 24 24 - int siw_mem_add(struct siw_device *sdev, struct siw_mem *m) 25 25 - { 26 26 - struct xa_limit limit = XA_LIMIT(1, SIW_STAG_MAX_INDEX); 27 27 - u32 id, next; 28 28 - 29 29 - get_random_bytes(&next, 4); 30 30 - next &= SIW_STAG_MAX_INDEX; 31 31 - 32 32 - if (xa_alloc_cyclic(&sdev->mem_xa, &id, m, limit, &next, 33 33 - GFP_KERNEL) < 0) 34 34 - return -ENOMEM; 35 35 - 36 36 - /* Set the STag index part */ 37 37 - m->stag = id << 8; 38 38 - 39 39 - siw_dbg_mem(m, "new MEM object\n"); 40 40 - 41 41 - return 0; 42 42 - } 43 43 - 44 44 - /* 45 21 * siw_mem_id2obj() 46 22 * 47 23 * resolves memory from stag given by id. might be called from: ··· 157 181 */ 158 182 if (addr < mem->va || addr + len > mem->va + mem->len) { 159 183 siw_dbg_pd(pd, "MEM interval len %d\n", len); 160 160 - siw_dbg_pd(pd, "[0x%pK, 0x%pK] out of bounds\n", 184 184 + siw_dbg_pd(pd, "[0x%p, 0x%p] out of bounds\n", 161 185 (void *)(uintptr_t)addr, 162 186 (void *)(uintptr_t)(addr + len)); 163 163 - siw_dbg_pd(pd, "[0x%pK, 0x%pK] STag=0x%08x\n", 187 187 + siw_dbg_pd(pd, "[0x%p, 0x%p] STag=0x%08x\n", 164 188 (void *)(uintptr_t)mem->va, 165 189 (void *)(uintptr_t)(mem->va + mem->len), 166 190 mem->stag);

-1

drivers/infiniband/sw/siw/siw_mem.h

reviewed

··· 12 12 struct siw_pbl *siw_pbl_alloc(u32 num_buf); 13 13 dma_addr_t siw_pbl_get_buffer(struct siw_pbl *pbl, u64 off, int *len, int *idx); 14 14 struct siw_mem *siw_mem_id2obj(struct siw_device *sdev, int stag_index); 15 15 - int siw_mem_add(struct siw_device *sdev, struct siw_mem *m); 16 15 int siw_invalidate_stag(struct ib_pd *pd, u32 stag); 17 16 int siw_check_mem(struct ib_pd *pd, struct siw_mem *mem, u64 addr, 18 17 enum ib_access_flags perms, int len);

+4 -4

drivers/infiniband/sw/siw/siw_qp_rx.c

reviewed

··· 38 38 39 39 p = siw_get_upage(umem, dest_addr); 40 40 if (unlikely(!p)) { 41 41 - pr_warn("siw: %s: [QP %u]: bogus addr: %pK, %pK\n", 41 41 + pr_warn("siw: %s: [QP %u]: bogus addr: %p, %p\n", 42 42 __func__, qp_id(rx_qp(srx)), 43 43 (void *)(uintptr_t)dest_addr, 44 44 (void *)(uintptr_t)umem->fp_addr); ··· 51 51 pg_off = dest_addr & ~PAGE_MASK; 52 52 bytes = min(len, (int)PAGE_SIZE - pg_off); 53 53 54 54 - siw_dbg_qp(rx_qp(srx), "page %pK, bytes=%u\n", p, bytes); 54 54 + siw_dbg_qp(rx_qp(srx), "page %p, bytes=%u\n", p, bytes); 55 55 56 56 dest = kmap_atomic(p); 57 57 rv = skb_copy_bits(srx->skb, srx->skb_offset, dest + pg_off, ··· 105 105 { 106 106 int rv; 107 107 108 108 - siw_dbg_qp(rx_qp(srx), "kva: 0x%pK, len: %u\n", kva, len); 108 108 + siw_dbg_qp(rx_qp(srx), "kva: 0x%p, len: %u\n", kva, len); 109 109 110 110 rv = skb_copy_bits(srx->skb, srx->skb_offset, kva, len); 111 111 if (unlikely(rv)) { 112 112 - pr_warn("siw: [QP %u]: %s, len %d, kva 0x%pK, rv %d\n", 112 112 + pr_warn("siw: [QP %u]: %s, len %d, kva 0x%p, rv %d\n", 113 113 qp_id(rx_qp(srx)), __func__, len, kva, rv); 114 114 115 115 return rv;

+4 -4

drivers/infiniband/sw/siw/siw_verbs.c

reviewed

··· 936 936 rv = -EINVAL; 937 937 break; 938 938 } 939 939 - siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%pK\n", 939 939 + siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%p\n", 940 940 sqe->opcode, sqe->flags, 941 941 (void *)(uintptr_t)sqe->id); 942 942 ··· 1102 1102 siw_dbg_qp(qp, "error %d\n", rv); 1103 1103 *bad_wr = wr; 1104 1104 } 1105 1105 - return rv > 0 ? 0 : rv; 1105 1105 + return rv; 1106 1106 } 1107 1107 1108 1108 int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata) ··· 1332 1332 struct siw_device *sdev = to_siw_dev(pd->device); 1333 1333 int rv; 1334 1334 1335 1335 - siw_dbg_pd(pd, "start: 0x%pK, va: 0x%pK, len: %llu\n", 1335 1335 + siw_dbg_pd(pd, "start: 0x%p, va: 0x%p, len: %llu\n", 1336 1336 (void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va, 1337 1337 (unsigned long long)len); 1338 1338 ··· 1525 1525 mem->len = base_mr->length; 1526 1526 mem->va = base_mr->iova; 1527 1527 siw_dbg_mem(mem, 1528 1528 - "%llu bytes, start 0x%pK, %u SLE to %u entries\n", 1528 1528 + "%llu bytes, start 0x%p, %u SLE to %u entries\n", 1529 1529 mem->len, (void *)(uintptr_t)mem->va, num_sle, 1530 1530 pbl->num_buf); 1531 1531 }

+24 -3

drivers/net/ethernet/microsoft/mana/gdma_main.c

reviewed

··· 391 391 case GDMA_EQE_HWC_INIT_EQ_ID_DB: 392 392 case GDMA_EQE_HWC_INIT_DATA: 393 393 case GDMA_EQE_HWC_INIT_DONE: 394 394 + case GDMA_EQE_HWC_SOC_SERVICE: 394 395 case GDMA_EQE_RNIC_QP_FATAL: 395 396 if (!eq->eq.callback) 396 397 break; ··· 965 964 err, resp.hdr.status); 966 965 return err ? err : -EPROTO; 967 966 } 967 967 + gc->pf_cap_flags1 = resp.pf_cap_flags1; 968 968 if (resp.pf_cap_flags1 & GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG) { 969 969 err = mana_gd_query_hwc_timeout(pdev, &hwc->hwc_timeout); 970 970 if (err) { ··· 1006 1004 1007 1005 return 0; 1008 1006 } 1009 1009 - EXPORT_SYMBOL_NS(mana_gd_register_device, "NET_MANA"); 1010 1007 1011 1008 int mana_gd_deregister_device(struct gdma_dev *gd) 1012 1009 { ··· 1036 1035 1037 1036 return err; 1038 1037 } 1039 1039 - EXPORT_SYMBOL_NS(mana_gd_deregister_device, "NET_MANA"); 1040 1038 1041 1039 u32 mana_gd_wq_avail_space(struct gdma_queue *wq) 1042 1040 { ··· 1469 1469 mana_gd_init_registers(pdev); 1470 1470 mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base); 1471 1471 1472 1472 + gc->service_wq = alloc_ordered_workqueue("gdma_service_wq", 0); 1473 1473 + if (!gc->service_wq) 1474 1474 + return -ENOMEM; 1475 1475 + 1472 1476 err = mana_gd_setup_irqs(pdev); 1473 1477 if (err) { 1474 1478 dev_err(gc->dev, "Failed to setup IRQs: %d\n", err); 1475 1475 - return err; 1479 1479 + goto free_workqueue; 1476 1480 } 1477 1481 1478 1482 err = mana_hwc_create_channel(gc); ··· 1502 1498 mana_hwc_destroy_channel(gc); 1503 1499 remove_irq: 1504 1500 mana_gd_remove_irqs(pdev); 1501 1501 + free_workqueue: 1502 1502 + destroy_workqueue(gc->service_wq); 1505 1503 dev_err(&pdev->dev, "%s failed (error %d)\n", __func__, err); 1506 1504 return err; 1507 1505 } ··· 1515 1509 mana_hwc_destroy_channel(gc); 1516 1510 1517 1511 mana_gd_remove_irqs(pdev); 1512 1512 + 1513 1513 + destroy_workqueue(gc->service_wq); 1518 1514 dev_dbg(&pdev->dev, "mana gdma cleanup successful\n"); 1519 1515 } 1520 1516 ··· 1586 1578 if (err) 1587 1579 goto cleanup_gd; 1588 1580 1581 1581 + err = mana_rdma_probe(&gc->mana_ib); 1582 1582 + if (err) 1583 1583 + goto cleanup_mana; 1584 1584 + 1589 1585 return 0; 1590 1586 1587 1587 + cleanup_mana: 1588 1588 + mana_remove(&gc->mana, false); 1591 1589 cleanup_gd: 1592 1590 mana_gd_cleanup(pdev); 1593 1591 unmap_bar: ··· 1621 1607 { 1622 1608 struct gdma_context *gc = pci_get_drvdata(pdev); 1623 1609 1610 1610 + mana_rdma_remove(&gc->mana_ib); 1624 1611 mana_remove(&gc->mana, false); 1625 1612 1626 1613 mana_gd_cleanup(pdev); ··· 1645 1630 { 1646 1631 struct gdma_context *gc = pci_get_drvdata(pdev); 1647 1632 1633 1633 + mana_rdma_remove(&gc->mana_ib); 1648 1634 mana_remove(&gc->mana, true); 1649 1635 1650 1636 mana_gd_cleanup(pdev); ··· 1670 1654 if (err) 1671 1655 return err; 1672 1656 1657 1657 + err = mana_rdma_probe(&gc->mana_ib); 1658 1658 + if (err) 1659 1659 + return err; 1660 1660 + 1673 1661 return 0; 1674 1662 } 1675 1663 ··· 1684 1664 1685 1665 dev_info(&pdev->dev, "Shutdown was called\n"); 1686 1666 1667 1667 + mana_rdma_remove(&gc->mana_ib); 1687 1668 mana_remove(&gc->mana, true); 1688 1669 1689 1670 mana_gd_cleanup(pdev);

+19

drivers/net/ethernet/microsoft/mana/hw_channel.c

reviewed

··· 112 112 static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self, 113 113 struct gdma_event *event) 114 114 { 115 115 + union hwc_init_soc_service_type service_data; 115 116 struct hw_channel_context *hwc = ctx; 116 117 struct gdma_dev *gd = hwc->gdma_dev; 117 118 union hwc_init_type_data type_data; 118 119 union hwc_init_eq_id_db eq_db; 119 120 u32 type, val; 121 121 + int ret; 120 122 121 123 switch (event->type) { 122 124 case GDMA_EQE_HWC_INIT_EQ_ID_DB: ··· 201 199 } 202 200 203 201 break; 202 202 + case GDMA_EQE_HWC_SOC_SERVICE: 203 203 + service_data.as_uint32 = event->details[0]; 204 204 + type = service_data.type; 204 205 206 206 + switch (type) { 207 207 + case GDMA_SERVICE_TYPE_RDMA_SUSPEND: 208 208 + case GDMA_SERVICE_TYPE_RDMA_RESUME: 209 209 + ret = mana_rdma_service_event(gd->gdma_context, type); 210 210 + if (ret) 211 211 + dev_err(hwc->dev, "Failed to schedule adev service event: %d\n", 212 212 + ret); 213 213 + break; 214 214 + default: 215 215 + dev_warn(hwc->dev, "Received unknown SOC service type %u\n", type); 216 216 + break; 217 217 + } 218 218 + 219 219 + break; 205 220 default: 206 221 dev_warn(hwc->dev, "Received unknown gdma event %u\n", event->type); 207 222 /* Ignore unknown events, which should never happen. */

+105 -3

drivers/net/ethernet/microsoft/mana/mana_en.c

reviewed

··· 2950 2950 gd->adev = NULL; 2951 2951 } 2952 2952 2953 2953 - static int add_adev(struct gdma_dev *gd) 2953 2953 + static int add_adev(struct gdma_dev *gd, const char *name) 2954 2954 { 2955 2955 struct auxiliary_device *adev; 2956 2956 struct mana_adev *madev; ··· 2966 2966 goto idx_fail; 2967 2967 adev->id = ret; 2968 2968 2969 2969 - adev->name = "rdma"; 2969 2969 + adev->name = name; 2970 2970 adev->dev.parent = gd->gdma_context->dev; 2971 2971 adev->dev.release = adev_release; 2972 2972 madev->mdev = gd; ··· 2996 2996 kfree(madev); 2997 2997 2998 2998 return ret; 2999 2999 + } 3000 3000 + 3001 3001 + static void mana_rdma_service_handle(struct work_struct *work) 3002 3002 + { 3003 3003 + struct mana_service_work *serv_work = 3004 3004 + container_of(work, struct mana_service_work, work); 3005 3005 + struct gdma_dev *gd = serv_work->gdma_dev; 3006 3006 + struct device *dev = gd->gdma_context->dev; 3007 3007 + int ret; 3008 3008 + 3009 3009 + if (READ_ONCE(gd->rdma_teardown)) 3010 3010 + goto out; 3011 3011 + 3012 3012 + switch (serv_work->event) { 3013 3013 + case GDMA_SERVICE_TYPE_RDMA_SUSPEND: 3014 3014 + if (!gd->adev || gd->is_suspended) 3015 3015 + break; 3016 3016 + 3017 3017 + remove_adev(gd); 3018 3018 + gd->is_suspended = true; 3019 3019 + break; 3020 3020 + 3021 3021 + case GDMA_SERVICE_TYPE_RDMA_RESUME: 3022 3022 + if (!gd->is_suspended) 3023 3023 + break; 3024 3024 + 3025 3025 + ret = add_adev(gd, "rdma"); 3026 3026 + if (ret) 3027 3027 + dev_err(dev, "Failed to add adev on resume: %d\n", ret); 3028 3028 + else 3029 3029 + gd->is_suspended = false; 3030 3030 + break; 3031 3031 + 3032 3032 + default: 3033 3033 + dev_warn(dev, "unknown adev service event %u\n", 3034 3034 + serv_work->event); 3035 3035 + break; 3036 3036 + } 3037 3037 + 3038 3038 + out: 3039 3039 + kfree(serv_work); 3040 3040 + } 3041 3041 + 3042 3042 + int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event) 3043 3043 + { 3044 3044 + struct gdma_dev *gd = &gc->mana_ib; 3045 3045 + struct mana_service_work *serv_work; 3046 3046 + 3047 3047 + if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 3048 3048 + /* RDMA device is not detected on pci */ 3049 3049 + return 0; 3050 3050 + } 3051 3051 + 3052 3052 + serv_work = kzalloc(sizeof(*serv_work), GFP_ATOMIC); 3053 3053 + if (!serv_work) 3054 3054 + return -ENOMEM; 3055 3055 + 3056 3056 + serv_work->event = event; 3057 3057 + serv_work->gdma_dev = gd; 3058 3058 + 3059 3059 + INIT_WORK(&serv_work->work, mana_rdma_service_handle); 3060 3060 + queue_work(gc->service_wq, &serv_work->work); 3061 3061 + 3062 3062 + return 0; 2999 3063 } 3000 3064 3001 3065 int mana_probe(struct gdma_dev *gd, bool resuming) ··· 3149 3085 } 3150 3086 } 3151 3087 3152 3152 - err = add_adev(gd); 3088 3088 + err = add_adev(gd, "eth"); 3153 3089 out: 3154 3090 if (err) { 3155 3091 mana_remove(gd, false); ··· 3221 3157 gd->gdma_context = NULL; 3222 3158 kfree(ac); 3223 3159 dev_dbg(dev, "%s succeeded\n", __func__); 3160 3160 + } 3161 3161 + 3162 3162 + int mana_rdma_probe(struct gdma_dev *gd) 3163 3163 + { 3164 3164 + int err = 0; 3165 3165 + 3166 3166 + if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 3167 3167 + /* RDMA device is not detected on pci */ 3168 3168 + return err; 3169 3169 + } 3170 3170 + 3171 3171 + err = mana_gd_register_device(gd); 3172 3172 + if (err) 3173 3173 + return err; 3174 3174 + 3175 3175 + err = add_adev(gd, "rdma"); 3176 3176 + if (err) 3177 3177 + mana_gd_deregister_device(gd); 3178 3178 + 3179 3179 + return err; 3180 3180 + } 3181 3181 + 3182 3182 + void mana_rdma_remove(struct gdma_dev *gd) 3183 3183 + { 3184 3184 + struct gdma_context *gc = gd->gdma_context; 3185 3185 + 3186 3186 + if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 3187 3187 + /* RDMA device is not detected on pci */ 3188 3188 + return; 3189 3189 + } 3190 3190 + 3191 3191 + WRITE_ONCE(gd->rdma_teardown, true); 3192 3192 + flush_workqueue(gc->service_wq); 3193 3193 + 3194 3194 + if (gd->adev) 3195 3195 + remove_adev(gd); 3196 3196 + 3197 3197 + mana_gd_deregister_device(gd); 3224 3198 } 3225 3199 3226 3200 struct net_device *mana_get_primary_netdev(struct mana_context *ac,

+33

include/linux/hmm-dma.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 + /* Copyright (c) 2024 NVIDIA Corporation & Affiliates */ 3 3 + #ifndef LINUX_HMM_DMA_H 4 4 + #define LINUX_HMM_DMA_H 5 5 + 6 6 + #include <linux/dma-mapping.h> 7 7 + 8 8 + struct dma_iova_state; 9 9 + struct pci_p2pdma_map_state; 10 10 + 11 11 + /* 12 12 + * struct hmm_dma_map - array of PFNs and DMA addresses 13 13 + * 14 14 + * @state: DMA IOVA state 15 15 + * @pfns: array of PFNs 16 16 + * @dma_list: array of DMA addresses 17 17 + * @dma_entry_size: size of each DMA entry in the array 18 18 + */ 19 19 + struct hmm_dma_map { 20 20 + struct dma_iova_state state; 21 21 + unsigned long *pfn_list; 22 22 + dma_addr_t *dma_list; 23 23 + size_t dma_entry_size; 24 24 + }; 25 25 + 26 26 + int hmm_dma_map_alloc(struct device *dev, struct hmm_dma_map *map, 27 27 + size_t nr_entries, size_t dma_entry_size); 28 28 + void hmm_dma_map_free(struct device *dev, struct hmm_dma_map *map); 29 29 + dma_addr_t hmm_dma_map_pfn(struct device *dev, struct hmm_dma_map *map, 30 30 + size_t idx, 31 31 + struct pci_p2pdma_map_state *p2pdma_state); 32 32 + bool hmm_dma_unmap_pfn(struct device *dev, struct hmm_dma_map *map, size_t idx); 33 33 + #endif /* LINUX_HMM_DMA_H */

+22 -2

include/linux/hmm.h

reviewed

··· 23 23 * HMM_PFN_WRITE - if the page memory can be written to (requires HMM_PFN_VALID) 24 24 * HMM_PFN_ERROR - accessing the pfn is impossible and the device should 25 25 * fail. ie poisoned memory, special pages, no vma, etc 26 26 + * HMM_PFN_P2PDMA - P2P page 27 27 + * HMM_PFN_P2PDMA_BUS - Bus mapped P2P transfer 28 28 + * HMM_PFN_DMA_MAPPED - Flag preserved on input-to-output transformation 29 29 + * to mark that page is already DMA mapped 26 30 * 27 31 * On input: 28 32 * 0 - Return the current state of the page, do not fault it. ··· 40 36 HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1), 41 37 HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2), 42 38 HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3), 43 43 - HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8), 39 39 + /* 40 40 + * Sticky flags, carried from input to output, 41 41 + * don't forget to update HMM_PFN_INOUT_FLAGS 42 42 + */ 43 43 + HMM_PFN_DMA_MAPPED = 1UL << (BITS_PER_LONG - 4), 44 44 + HMM_PFN_P2PDMA = 1UL << (BITS_PER_LONG - 5), 45 45 + HMM_PFN_P2PDMA_BUS = 1UL << (BITS_PER_LONG - 6), 46 46 + 47 47 + HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 11), 44 48 45 49 /* Input flags */ 46 50 HMM_PFN_REQ_FAULT = HMM_PFN_VALID, 47 51 HMM_PFN_REQ_WRITE = HMM_PFN_WRITE, 48 52 49 49 - HMM_PFN_FLAGS = 0xFFUL << HMM_PFN_ORDER_SHIFT, 53 53 + HMM_PFN_FLAGS = ~((1UL << HMM_PFN_ORDER_SHIFT) - 1), 50 54 }; 51 55 52 56 /* ··· 67 55 static inline struct page *hmm_pfn_to_page(unsigned long hmm_pfn) 68 56 { 69 57 return pfn_to_page(hmm_pfn & ~HMM_PFN_FLAGS); 58 58 + } 59 59 + 60 60 + /* 61 61 + * hmm_pfn_to_phys() - return physical address pointed to by a device entry 62 62 + */ 63 63 + static inline phys_addr_t hmm_pfn_to_phys(unsigned long hmm_pfn) 64 64 + { 65 65 + return __pfn_to_phys(hmm_pfn & ~HMM_PFN_FLAGS); 70 66 } 71 67 72 68 /*

+1

include/linux/mlx5/driver.h

reviewed

··· 398 398 enum mlx5_res_type res; 399 399 refcount_t refcount; 400 400 struct completion free; 401 401 + bool invalid; 401 402 }; 402 403 403 404 struct mlx5_uars_page {

+32 -15

include/net/mana/gdma.h

reviewed

··· 60 60 GDMA_EQE_HWC_INIT_DONE = 131, 61 61 GDMA_EQE_HWC_SOC_RECONFIG = 132, 62 62 GDMA_EQE_HWC_SOC_RECONFIG_DATA = 133, 63 63 + GDMA_EQE_HWC_SOC_SERVICE = 134, 63 64 GDMA_EQE_RNIC_QP_FATAL = 176, 64 65 }; 65 66 ··· 69 68 GDMA_DEVICE_HWC = 1, 70 69 GDMA_DEVICE_MANA = 2, 71 70 GDMA_DEVICE_MANA_IB = 3, 71 71 + }; 72 72 + 73 73 + enum gdma_service_type { 74 74 + GDMA_SERVICE_TYPE_NONE = 0, 75 75 + GDMA_SERVICE_TYPE_RDMA_SUSPEND = 1, 76 76 + GDMA_SERVICE_TYPE_RDMA_RESUME = 2, 77 77 + }; 78 78 + 79 79 + struct mana_service_work { 80 80 + struct work_struct work; 81 81 + struct gdma_dev *gdma_dev; 82 82 + enum gdma_service_type event; 72 83 }; 73 84 74 85 struct gdma_resource { ··· 237 224 void *driver_data; 238 225 239 226 struct auxiliary_device *adev; 227 227 + bool is_suspended; 228 228 + bool rdma_teardown; 240 229 }; 241 230 242 231 /* MANA_PAGE_SIZE is the DMA unit */ ··· 422 407 423 408 /* Azure RDMA adapter */ 424 409 struct gdma_dev mana_ib; 410 410 + 411 411 + u64 pf_cap_flags1; 412 412 + 413 413 + struct workqueue_struct *service_wq; 425 414 }; 426 415 427 416 static inline bool mana_gd_is_mana(struct gdma_dev *gd) ··· 572 553 */ 573 554 #define GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX BIT(2) 574 555 #define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG BIT(3) 556 556 + #define GDMA_DRV_CAP_FLAG_1_GDMA_PAGES_4MB_1GB_2GB BIT(4) 575 557 #define GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT BIT(5) 576 558 577 559 /* Driver can handle holes (zeros) in the device list */ ··· 727 707 u32 reserved; 728 708 }; 729 709 730 730 - enum atb_page_size { 731 731 - ATB_PAGE_SIZE_4K, 732 732 - ATB_PAGE_SIZE_8K, 733 733 - ATB_PAGE_SIZE_16K, 734 734 - ATB_PAGE_SIZE_32K, 735 735 - ATB_PAGE_SIZE_64K, 736 736 - ATB_PAGE_SIZE_128K, 737 737 - ATB_PAGE_SIZE_256K, 738 738 - ATB_PAGE_SIZE_512K, 739 739 - ATB_PAGE_SIZE_1M, 740 740 - ATB_PAGE_SIZE_2M, 741 741 - ATB_PAGE_SIZE_MAX, 742 742 - }; 743 743 - 744 710 enum gdma_mr_access_flags { 745 711 GDMA_ACCESS_FLAG_LOCAL_READ = BIT_ULL(0), 746 712 GDMA_ACCESS_FLAG_LOCAL_WRITE = BIT_ULL(1), ··· 821 815 * address that is set up in the MST 822 816 */ 823 817 GDMA_MR_TYPE_GVA = 2, 818 818 + /* Guest zero-based address MRs */ 819 819 + GDMA_MR_TYPE_ZBVA = 4, 824 820 }; 825 821 826 822 struct gdma_create_mr_params { ··· 834 826 u64 virtual_address; 835 827 enum gdma_mr_access_flags access_flags; 836 828 } gva; 829 829 + struct { 830 830 + u64 dma_region_handle; 831 831 + enum gdma_mr_access_flags access_flags; 832 832 + } zbva; 837 833 }; 838 834 }; 839 835 ··· 853 841 u64 virtual_address; 854 842 enum gdma_mr_access_flags access_flags; 855 843 } gva; 856 856 - 844 844 + struct { 845 845 + u64 dma_region_handle; 846 846 + enum gdma_mr_access_flags access_flags; 847 847 + } zbva; 857 848 }; 858 849 u32 reserved_2; 859 850 };/* HW DATA */ ··· 907 892 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle); 908 893 void mana_register_debugfs(void); 909 894 void mana_unregister_debugfs(void); 895 895 + 896 896 + int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event); 910 897 911 898 #endif /* _GDMA_H */

+9

include/net/mana/hw_channel.h

reviewed

··· 49 49 }; 50 50 }; /* HW DATA */ 51 51 52 52 + union hwc_init_soc_service_type { 53 53 + u32 as_uint32; 54 54 + 55 55 + struct { 56 56 + u32 value : 28; 57 57 + u32 type : 4; 58 58 + }; 59 59 + }; /* HW DATA */ 60 60 + 52 61 struct hwc_rx_oob { 53 62 u32 type : 6; 54 63 u32 eom : 1;

+3

include/net/mana/mana.h

reviewed

··· 489 489 int mana_probe(struct gdma_dev *gd, bool resuming); 490 490 void mana_remove(struct gdma_dev *gd, bool suspending); 491 491 492 492 + int mana_rdma_probe(struct gdma_dev *gd); 493 493 + void mana_rdma_remove(struct gdma_dev *gd); 494 494 + 492 495 void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev); 493 496 int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames, 494 497 u32 flags);

+3 -14

include/rdma/ib_cm.h

reviewed

··· 480 480 const void *private_data, 481 481 u8 private_data_len); 482 482 483 483 - #define IB_CM_MRA_FLAG_DELAY 0x80 /* Send MRA only after a duplicate msg */ 484 484 - 485 483 /** 486 486 - * ib_send_cm_mra - Sends a message receipt acknowledgement to a connection 487 487 - * message. 484 484 + * ib_prepare_cm_mra - Prepares to send a message receipt acknowledgment to a 485 485 + connection message in case duplicates are received. 488 486 * @cm_id: Connection identifier associated with the connection message. 489 489 - * @service_timeout: The lower 5-bits specify the maximum time required for 490 490 - * the sender to reply to the connection message. The upper 3-bits 491 491 - * specify additional control flags. 492 492 - * @private_data: Optional user-defined private data sent with the 493 493 - * message receipt acknowledgement. 494 494 - * @private_data_len: Size of the private data buffer, in bytes. 495 487 */ 496 496 - int ib_send_cm_mra(struct ib_cm_id *cm_id, 497 497 - u8 service_timeout, 498 498 - const void *private_data, 499 499 - u8 private_data_len); 488 488 + int ib_prepare_cm_mra(struct ib_cm_id *cm_id); 500 489 501 490 /** 502 491 * ib_cm_init_qp_attr - Initializes the QP attributes for use in transitioning

+3 -22

include/rdma/ib_umem_odp.h

reviewed

··· 8 8 9 9 #include <rdma/ib_umem.h> 10 10 #include <rdma/ib_verbs.h> 11 11 + #include <linux/hmm-dma.h> 11 12 12 13 struct ib_umem_odp { 13 14 struct ib_umem umem; 14 15 struct mmu_interval_notifier notifier; 15 16 struct pid *tgid; 16 17 17 17 - /* An array of the pfns included in the on-demand paging umem. */ 18 18 - unsigned long *pfn_list; 18 18 + struct hmm_dma_map map; 19 19 20 20 /* 21 21 - * An array with DMA addresses mapped for pfns in pfn_list. 22 22 - * The lower two bits designate access permissions. 23 23 - * See ODP_READ_ALLOWED_BIT and ODP_WRITE_ALLOWED_BIT. 24 24 - */ 25 25 - dma_addr_t *dma_list; 26 26 - /* 27 27 - * The umem_mutex protects the page_list and dma_list fields of an ODP 21 21 + * The umem_mutex protects the page_list field of an ODP 28 22 * umem, allowing only a single thread to map/unmap pages. The mutex 29 23 * also protects access to the mmu notifier counters. 30 24 */ ··· 60 66 return (ib_umem_end(umem_odp) - ib_umem_start(umem_odp)) >> 61 67 umem_odp->page_shift; 62 68 } 63 63 - 64 64 - /* 65 65 - * The lower 2 bits of the DMA address signal the R/W permissions for 66 66 - * the entry. To upgrade the permissions, provide the appropriate 67 67 - * bitmask to the map_dma_pages function. 68 68 - * 69 69 - * Be aware that upgrading a mapped address might result in change of 70 70 - * the DMA address for the page. 71 71 - */ 72 72 - #define ODP_READ_ALLOWED_BIT (1<<0ULL) 73 73 - #define ODP_WRITE_ALLOWED_BIT (1<<1ULL) 74 74 - 75 75 - #define ODP_DMA_ADDR_MASK (~(ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) 76 69 77 70 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 78 71

+10 -8

include/rdma/ib_verbs.h

reviewed

··· 314 314 }; 315 315 316 316 enum ib_odp_general_cap_bits { 317 317 - IB_ODP_SUPPORT = 1 << 0, 318 318 - IB_ODP_SUPPORT_IMPLICIT = 1 << 1, 317 317 + IB_ODP_SUPPORT = IB_UVERBS_ODP_SUPPORT, 318 318 + IB_ODP_SUPPORT_IMPLICIT = IB_UVERBS_ODP_SUPPORT_IMPLICIT, 319 319 }; 320 320 321 321 enum ib_odp_transport_cap_bits { 322 322 - IB_ODP_SUPPORT_SEND = 1 << 0, 323 323 - IB_ODP_SUPPORT_RECV = 1 << 1, 324 324 - IB_ODP_SUPPORT_WRITE = 1 << 2, 325 325 - IB_ODP_SUPPORT_READ = 1 << 3, 326 326 - IB_ODP_SUPPORT_ATOMIC = 1 << 4, 327 327 - IB_ODP_SUPPORT_SRQ_RECV = 1 << 5, 322 322 + IB_ODP_SUPPORT_SEND = IB_UVERBS_ODP_SUPPORT_SEND, 323 323 + IB_ODP_SUPPORT_RECV = IB_UVERBS_ODP_SUPPORT_RECV, 324 324 + IB_ODP_SUPPORT_WRITE = IB_UVERBS_ODP_SUPPORT_WRITE, 325 325 + IB_ODP_SUPPORT_READ = IB_UVERBS_ODP_SUPPORT_READ, 326 326 + IB_ODP_SUPPORT_ATOMIC = IB_UVERBS_ODP_SUPPORT_ATOMIC, 327 327 + IB_ODP_SUPPORT_SRQ_RECV = IB_UVERBS_ODP_SUPPORT_SRQ_RECV, 328 328 + IB_ODP_SUPPORT_FLUSH = IB_UVERBS_ODP_SUPPORT_FLUSH, 329 329 + IB_ODP_SUPPORT_ATOMIC_WRITE = IB_UVERBS_ODP_SUPPORT_ATOMIC_WRITE, 328 330 }; 329 331 330 332 struct ib_odp_caps {

-1

include/rdma/rdma_cm.h

reviewed

··· 388 388 union ib_gid *dgid); 389 389 390 390 struct iw_cm_id *rdma_iw_cm_id(struct rdma_cm_id *cm_id); 391 391 - struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res); 392 391 393 392 #endif /* RDMA_CM_H */

+16

include/uapi/rdma/ib_user_verbs.h

reviewed

··· 233 233 __u32 reserved; 234 234 }; 235 235 236 236 + enum ib_uverbs_odp_general_cap_bits { 237 237 + IB_UVERBS_ODP_SUPPORT = 1 << 0, 238 238 + IB_UVERBS_ODP_SUPPORT_IMPLICIT = 1 << 1, 239 239 + }; 240 240 + 241 241 + enum ib_uverbs_odp_transport_cap_bits { 242 242 + IB_UVERBS_ODP_SUPPORT_SEND = 1 << 0, 243 243 + IB_UVERBS_ODP_SUPPORT_RECV = 1 << 1, 244 244 + IB_UVERBS_ODP_SUPPORT_WRITE = 1 << 2, 245 245 + IB_UVERBS_ODP_SUPPORT_READ = 1 << 3, 246 246 + IB_UVERBS_ODP_SUPPORT_ATOMIC = 1 << 4, 247 247 + IB_UVERBS_ODP_SUPPORT_SRQ_RECV = 1 << 5, 248 248 + IB_UVERBS_ODP_SUPPORT_FLUSH = 1 << 6, 249 249 + IB_UVERBS_ODP_SUPPORT_ATOMIC_WRITE = 1 << 7, 250 250 + }; 251 251 + 236 252 struct ib_uverbs_odp_caps { 237 253 __aligned_u64 general_caps; 238 254 struct {

+243 -19

mm/hmm.c

reviewed

··· 10 10 */ 11 11 #include <linux/pagewalk.h> 12 12 #include <linux/hmm.h> 13 13 + #include <linux/hmm-dma.h> 13 14 #include <linux/init.h> 14 15 #include <linux/rmap.h> 15 16 #include <linux/swap.h> ··· 24 23 #include <linux/sched/mm.h> 25 24 #include <linux/jump_label.h> 26 25 #include <linux/dma-mapping.h> 26 26 + #include <linux/pci-p2pdma.h> 27 27 #include <linux/mmu_notifier.h> 28 28 #include <linux/memory_hotplug.h> 29 29 ··· 41 39 HMM_NEED_ALL_BITS = HMM_NEED_FAULT | HMM_NEED_WRITE_FAULT, 42 40 }; 43 41 42 42 + enum { 43 43 + /* These flags are carried from input-to-output */ 44 44 + HMM_PFN_INOUT_FLAGS = HMM_PFN_DMA_MAPPED | HMM_PFN_P2PDMA | 45 45 + HMM_PFN_P2PDMA_BUS, 46 46 + }; 47 47 + 44 48 static int hmm_pfns_fill(unsigned long addr, unsigned long end, 45 49 struct hmm_range *range, unsigned long cpu_flags) 46 50 { 47 51 unsigned long i = (addr - range->start) >> PAGE_SHIFT; 48 52 49 49 - for (; addr < end; addr += PAGE_SIZE, i++) 50 50 - range->hmm_pfns[i] = cpu_flags; 53 53 + for (; addr < end; addr += PAGE_SIZE, i++) { 54 54 + range->hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS; 55 55 + range->hmm_pfns[i] |= cpu_flags; 56 56 + } 51 57 return 0; 52 58 } 53 59 ··· 212 202 return hmm_vma_fault(addr, end, required_fault, walk); 213 203 214 204 pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 215 215 - for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) 216 216 - hmm_pfns[i] = pfn | cpu_flags; 205 205 + for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) { 206 206 + hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS; 207 207 + hmm_pfns[i] |= pfn | cpu_flags; 208 208 + } 217 209 return 0; 218 210 } 219 211 #else /* CONFIG_TRANSPARENT_HUGEPAGE */ ··· 242 230 unsigned long cpu_flags; 243 231 pte_t pte = ptep_get(ptep); 244 232 uint64_t pfn_req_flags = *hmm_pfn; 233 233 + uint64_t new_pfn_flags = 0; 245 234 246 235 if (pte_none_mostly(pte)) { 247 236 required_fault = 248 237 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0); 249 238 if (required_fault) 250 239 goto fault; 251 251 - *hmm_pfn = 0; 252 252 - return 0; 240 240 + goto out; 253 241 } 254 242 255 243 if (!pte_present(pte)) { ··· 265 253 cpu_flags = HMM_PFN_VALID; 266 254 if (is_writable_device_private_entry(entry)) 267 255 cpu_flags |= HMM_PFN_WRITE; 268 268 - *hmm_pfn = swp_offset_pfn(entry) | cpu_flags; 269 269 - return 0; 256 256 + new_pfn_flags = swp_offset_pfn(entry) | cpu_flags; 257 257 + goto out; 270 258 } 271 259 272 260 required_fault = 273 261 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0); 274 274 - if (!required_fault) { 275 275 - *hmm_pfn = 0; 276 276 - return 0; 277 277 - } 262 262 + if (!required_fault) 263 263 + goto out; 278 264 279 265 if (!non_swap_entry(entry)) 280 266 goto fault; ··· 314 304 pte_unmap(ptep); 315 305 return -EFAULT; 316 306 } 317 317 - *hmm_pfn = HMM_PFN_ERROR; 318 318 - return 0; 307 307 + new_pfn_flags = HMM_PFN_ERROR; 308 308 + goto out; 319 309 } 320 310 321 321 - *hmm_pfn = pte_pfn(pte) | cpu_flags; 311 311 + new_pfn_flags = pte_pfn(pte) | cpu_flags; 312 312 + out: 313 313 + *hmm_pfn = (*hmm_pfn & HMM_PFN_INOUT_FLAGS) | new_pfn_flags; 322 314 return 0; 323 315 324 316 fault: ··· 460 448 } 461 449 462 450 pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 463 463 - for (i = 0; i < npages; ++i, ++pfn) 464 464 - hmm_pfns[i] = pfn | cpu_flags; 451 451 + for (i = 0; i < npages; ++i, ++pfn) { 452 452 + hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS; 453 453 + hmm_pfns[i] |= pfn | cpu_flags; 454 454 + } 465 455 goto out_unlock; 466 456 } 467 457 ··· 521 507 } 522 508 523 509 pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT); 524 524 - for (; addr < end; addr += PAGE_SIZE, i++, pfn++) 525 525 - range->hmm_pfns[i] = pfn | cpu_flags; 510 510 + for (; addr < end; addr += PAGE_SIZE, i++, pfn++) { 511 511 + range->hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS; 512 512 + range->hmm_pfns[i] |= pfn | cpu_flags; 513 513 + } 526 514 527 515 spin_unlock(ptl); 528 516 return 0; ··· 623 607 return ret; 624 608 } 625 609 EXPORT_SYMBOL(hmm_range_fault); 610 610 + 611 611 + /** 612 612 + * hmm_dma_map_alloc - Allocate HMM map structure 613 613 + * @dev: device to allocate structure for 614 614 + * @map: HMM map to allocate 615 615 + * @nr_entries: number of entries in the map 616 616 + * @dma_entry_size: size of the DMA entry in the map 617 617 + * 618 618 + * Allocate the HMM map structure and all the lists it contains. 619 619 + * Return 0 on success, -ENOMEM on failure. 620 620 + */ 621 621 + int hmm_dma_map_alloc(struct device *dev, struct hmm_dma_map *map, 622 622 + size_t nr_entries, size_t dma_entry_size) 623 623 + { 624 624 + bool dma_need_sync = false; 625 625 + bool use_iova; 626 626 + 627 627 + WARN_ON_ONCE(!(nr_entries * PAGE_SIZE / dma_entry_size)); 628 628 + 629 629 + /* 630 630 + * The HMM API violates our normal DMA buffer ownership rules and can't 631 631 + * transfer buffer ownership. The dma_addressing_limited() check is a 632 632 + * best approximation to ensure no swiotlb buffering happens. 633 633 + */ 634 634 + #ifdef CONFIG_DMA_NEED_SYNC 635 635 + dma_need_sync = !dev->dma_skip_sync; 636 636 + #endif /* CONFIG_DMA_NEED_SYNC */ 637 637 + if (dma_need_sync || dma_addressing_limited(dev)) 638 638 + return -EOPNOTSUPP; 639 639 + 640 640 + map->dma_entry_size = dma_entry_size; 641 641 + map->pfn_list = kvcalloc(nr_entries, sizeof(*map->pfn_list), 642 642 + GFP_KERNEL | __GFP_NOWARN); 643 643 + if (!map->pfn_list) 644 644 + return -ENOMEM; 645 645 + 646 646 + use_iova = dma_iova_try_alloc(dev, &map->state, 0, 647 647 + nr_entries * PAGE_SIZE); 648 648 + if (!use_iova && dma_need_unmap(dev)) { 649 649 + map->dma_list = kvcalloc(nr_entries, sizeof(*map->dma_list), 650 650 + GFP_KERNEL | __GFP_NOWARN); 651 651 + if (!map->dma_list) 652 652 + goto err_dma; 653 653 + } 654 654 + return 0; 655 655 + 656 656 + err_dma: 657 657 + kvfree(map->pfn_list); 658 658 + return -ENOMEM; 659 659 + } 660 660 + EXPORT_SYMBOL_GPL(hmm_dma_map_alloc); 661 661 + 662 662 + /** 663 663 + * hmm_dma_map_free - iFree HMM map structure 664 664 + * @dev: device to free structure from 665 665 + * @map: HMM map containing the various lists and state 666 666 + * 667 667 + * Free the HMM map structure and all the lists it contains. 668 668 + */ 669 669 + void hmm_dma_map_free(struct device *dev, struct hmm_dma_map *map) 670 670 + { 671 671 + if (dma_use_iova(&map->state)) 672 672 + dma_iova_free(dev, &map->state); 673 673 + kvfree(map->pfn_list); 674 674 + kvfree(map->dma_list); 675 675 + } 676 676 + EXPORT_SYMBOL_GPL(hmm_dma_map_free); 677 677 + 678 678 + /** 679 679 + * hmm_dma_map_pfn - Map a physical HMM page to DMA address 680 680 + * @dev: Device to map the page for 681 681 + * @map: HMM map 682 682 + * @idx: Index into the PFN and dma address arrays 683 683 + * @p2pdma_state: PCI P2P state. 684 684 + * 685 685 + * dma_alloc_iova() allocates IOVA based on the size specified by their use in 686 686 + * iova->size. Call this function after IOVA allocation to link whole @page 687 687 + * to get the DMA address. Note that very first call to this function 688 688 + * will have @offset set to 0 in the IOVA space allocated from 689 689 + * dma_alloc_iova(). For subsequent calls to this function on same @iova, 690 690 + * @offset needs to be advanced by the caller with the size of previous 691 691 + * page that was linked + DMA address returned for the previous page that was 692 692 + * linked by this function. 693 693 + */ 694 694 + dma_addr_t hmm_dma_map_pfn(struct device *dev, struct hmm_dma_map *map, 695 695 + size_t idx, 696 696 + struct pci_p2pdma_map_state *p2pdma_state) 697 697 + { 698 698 + struct dma_iova_state *state = &map->state; 699 699 + dma_addr_t *dma_addrs = map->dma_list; 700 700 + unsigned long *pfns = map->pfn_list; 701 701 + struct page *page = hmm_pfn_to_page(pfns[idx]); 702 702 + phys_addr_t paddr = hmm_pfn_to_phys(pfns[idx]); 703 703 + size_t offset = idx * map->dma_entry_size; 704 704 + unsigned long attrs = 0; 705 705 + dma_addr_t dma_addr; 706 706 + int ret; 707 707 + 708 708 + if ((pfns[idx] & HMM_PFN_DMA_MAPPED) && 709 709 + !(pfns[idx] & HMM_PFN_P2PDMA_BUS)) { 710 710 + /* 711 711 + * We are in this flow when there is a need to resync flags, 712 712 + * for example when page was already linked in prefetch call 713 713 + * with READ flag and now we need to add WRITE flag 714 714 + * 715 715 + * This page was already programmed to HW and we don't want/need 716 716 + * to unlink and link it again just to resync flags. 717 717 + */ 718 718 + if (dma_use_iova(state)) 719 719 + return state->addr + offset; 720 720 + 721 721 + /* 722 722 + * Without dma_need_unmap, the dma_addrs array is NULL, thus we 723 723 + * need to regenerate the address below even if there already 724 724 + * was a mapping. But !dma_need_unmap implies that the 725 725 + * mapping stateless, so this is fine. 726 726 + */ 727 727 + if (dma_need_unmap(dev)) 728 728 + return dma_addrs[idx]; 729 729 + 730 730 + /* Continue to remapping */ 731 731 + } 732 732 + 733 733 + switch (pci_p2pdma_state(p2pdma_state, dev, page)) { 734 734 + case PCI_P2PDMA_MAP_NONE: 735 735 + break; 736 736 + case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: 737 737 + attrs |= DMA_ATTR_SKIP_CPU_SYNC; 738 738 + pfns[idx] |= HMM_PFN_P2PDMA; 739 739 + break; 740 740 + case PCI_P2PDMA_MAP_BUS_ADDR: 741 741 + pfns[idx] |= HMM_PFN_P2PDMA_BUS | HMM_PFN_DMA_MAPPED; 742 742 + return pci_p2pdma_bus_addr_map(p2pdma_state, paddr); 743 743 + default: 744 744 + return DMA_MAPPING_ERROR; 745 745 + } 746 746 + 747 747 + if (dma_use_iova(state)) { 748 748 + ret = dma_iova_link(dev, state, paddr, offset, 749 749 + map->dma_entry_size, DMA_BIDIRECTIONAL, 750 750 + attrs); 751 751 + if (ret) 752 752 + goto error; 753 753 + 754 754 + ret = dma_iova_sync(dev, state, offset, map->dma_entry_size); 755 755 + if (ret) { 756 756 + dma_iova_unlink(dev, state, offset, map->dma_entry_size, 757 757 + DMA_BIDIRECTIONAL, attrs); 758 758 + goto error; 759 759 + } 760 760 + 761 761 + dma_addr = state->addr + offset; 762 762 + } else { 763 763 + if (WARN_ON_ONCE(dma_need_unmap(dev) && !dma_addrs)) 764 764 + goto error; 765 765 + 766 766 + dma_addr = dma_map_page(dev, page, 0, map->dma_entry_size, 767 767 + DMA_BIDIRECTIONAL); 768 768 + if (dma_mapping_error(dev, dma_addr)) 769 769 + goto error; 770 770 + 771 771 + if (dma_need_unmap(dev)) 772 772 + dma_addrs[idx] = dma_addr; 773 773 + } 774 774 + pfns[idx] |= HMM_PFN_DMA_MAPPED; 775 775 + return dma_addr; 776 776 + error: 777 777 + pfns[idx] &= ~HMM_PFN_P2PDMA; 778 778 + return DMA_MAPPING_ERROR; 779 779 + 780 780 + } 781 781 + EXPORT_SYMBOL_GPL(hmm_dma_map_pfn); 782 782 + 783 783 + /** 784 784 + * hmm_dma_unmap_pfn - Unmap a physical HMM page from DMA address 785 785 + * @dev: Device to unmap the page from 786 786 + * @map: HMM map 787 787 + * @idx: Index of the PFN to unmap 788 788 + * 789 789 + * Returns true if the PFN was mapped and has been unmapped, false otherwise. 790 790 + */ 791 791 + bool hmm_dma_unmap_pfn(struct device *dev, struct hmm_dma_map *map, size_t idx) 792 792 + { 793 793 + const unsigned long valid_dma = HMM_PFN_VALID | HMM_PFN_DMA_MAPPED; 794 794 + struct dma_iova_state *state = &map->state; 795 795 + dma_addr_t *dma_addrs = map->dma_list; 796 796 + unsigned long *pfns = map->pfn_list; 797 797 + unsigned long attrs = 0; 798 798 + 799 799 + if ((pfns[idx] & valid_dma) != valid_dma) 800 800 + return false; 801 801 + 802 802 + if (pfns[idx] & HMM_PFN_P2PDMA_BUS) 803 803 + ; /* no need to unmap bus address P2P mappings */ 804 804 + else if (dma_use_iova(state)) { 805 805 + if (pfns[idx] & HMM_PFN_P2PDMA) 806 806 + attrs |= DMA_ATTR_SKIP_CPU_SYNC; 807 807 + dma_iova_unlink(dev, state, idx * map->dma_entry_size, 808 808 + map->dma_entry_size, DMA_BIDIRECTIONAL, attrs); 809 809 + } else if (dma_need_unmap(dev)) 810 810 + dma_unmap_page(dev, dma_addrs[idx], map->dma_entry_size, 811 811 + DMA_BIDIRECTIONAL); 812 812 + 813 813 + pfns[idx] &= 814 814 + ~(HMM_PFN_DMA_MAPPED | HMM_PFN_P2PDMA | HMM_PFN_P2PDMA_BUS); 815 815 + return true; 816 816 + } 817 817 + EXPORT_SYMBOL_GPL(hmm_dma_unmap_pfn);