+159 -141

drivers/infiniband/core/cma.c

reviewed

··· 479 479 if (sa->sa_family != sb->sa_family) 480 480 return sa->sa_family - sb->sa_family; 481 481 482 482 - if (sa->sa_family == AF_INET) 483 483 - return memcmp((char *)&((struct sockaddr_in *)sa)->sin_addr, 484 484 - (char *)&((struct sockaddr_in *)sb)->sin_addr, 482 482 + if (sa->sa_family == AF_INET && 483 483 + __builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in)) { 484 484 + return memcmp(&((struct sockaddr_in *)sa)->sin_addr, 485 485 + &((struct sockaddr_in *)sb)->sin_addr, 485 486 sizeof(((struct sockaddr_in *)sa)->sin_addr)); 487 487 + } 486 488 487 487 - return ipv6_addr_cmp(&((struct sockaddr_in6 *)sa)->sin6_addr, 488 488 - &((struct sockaddr_in6 *)sb)->sin6_addr); 489 489 + if (sa->sa_family == AF_INET6 && 490 490 + __builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in6)) { 491 491 + return ipv6_addr_cmp(&((struct sockaddr_in6 *)sa)->sin6_addr, 492 492 + &((struct sockaddr_in6 *)sb)->sin6_addr); 493 493 + } 494 494 + 495 495 + return -1; 489 496 } 490 497 491 498 static int cma_add_id_to_tree(struct rdma_id_private *node_id_priv) ··· 2826 2819 } 2827 2820 EXPORT_SYMBOL(rdma_set_min_rnr_timer); 2828 2821 2829 2829 - static void route_set_path_rec_inbound(struct cma_work *work, 2830 2830 - struct sa_path_rec *path_rec) 2822 2822 + static int route_set_path_rec_inbound(struct cma_work *work, 2823 2823 + struct sa_path_rec *path_rec) 2831 2824 { 2832 2825 struct rdma_route *route = &work->id->id.route; 2833 2826 ··· 2835 2828 route->path_rec_inbound = 2836 2829 kzalloc(sizeof(*route->path_rec_inbound), GFP_KERNEL); 2837 2830 if (!route->path_rec_inbound) 2838 2838 - return; 2831 2831 + return -ENOMEM; 2839 2832 } 2840 2833 2841 2834 *route->path_rec_inbound = *path_rec; 2835 2835 + return 0; 2842 2836 } 2843 2837 2844 2844 - static void route_set_path_rec_outbound(struct cma_work *work, 2845 2845 - struct sa_path_rec *path_rec) 2838 2838 + static int route_set_path_rec_outbound(struct cma_work *work, 2839 2839 + struct sa_path_rec *path_rec) 2846 2840 { 2847 2841 struct rdma_route *route = &work->id->id.route; 2848 2842 ··· 2851 2843 route->path_rec_outbound = 2852 2844 kzalloc(sizeof(*route->path_rec_outbound), GFP_KERNEL); 2853 2845 if (!route->path_rec_outbound) 2854 2854 - return; 2846 2846 + return -ENOMEM; 2855 2847 } 2856 2848 2857 2849 *route->path_rec_outbound = *path_rec; 2850 2850 + return 0; 2858 2851 } 2859 2852 2860 2853 static void cma_query_handler(int status, struct sa_path_rec *path_rec, 2861 2861 - int num_prs, void *context) 2854 2854 + unsigned int num_prs, void *context) 2862 2855 { 2863 2856 struct cma_work *work = context; 2864 2857 struct rdma_route *route; ··· 2874 2865 if (!path_rec[i].flags || (path_rec[i].flags & IB_PATH_GMP)) 2875 2866 *route->path_rec = path_rec[i]; 2876 2867 else if (path_rec[i].flags & IB_PATH_INBOUND) 2877 2877 - route_set_path_rec_inbound(work, &path_rec[i]); 2868 2868 + status = route_set_path_rec_inbound(work, &path_rec[i]); 2878 2869 else if (path_rec[i].flags & IB_PATH_OUTBOUND) 2879 2879 - route_set_path_rec_outbound(work, &path_rec[i]); 2880 2880 - } 2881 2881 - if (!route->path_rec) { 2882 2882 - status = -EINVAL; 2883 2883 - goto fail; 2870 2870 + status = route_set_path_rec_outbound(work, 2871 2871 + &path_rec[i]); 2872 2872 + else 2873 2873 + status = -EINVAL; 2874 2874 + 2875 2875 + if (status) 2876 2876 + goto fail; 2884 2877 } 2885 2878 2886 2879 route->num_pri_alt_paths = 1; ··· 3552 3541 return ret; 3553 3542 } 3554 3543 3555 3555 - static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 3556 3556 - const struct sockaddr *dst_addr) 3557 3557 - { 3558 3558 - struct sockaddr_storage zero_sock = {}; 3559 3559 - 3560 3560 - if (src_addr && src_addr->sa_family) 3561 3561 - return rdma_bind_addr(id, src_addr); 3562 3562 - 3563 3563 - /* 3564 3564 - * When the src_addr is not specified, automatically supply an any addr 3565 3565 - */ 3566 3566 - zero_sock.ss_family = dst_addr->sa_family; 3567 3567 - if (IS_ENABLED(CONFIG_IPV6) && dst_addr->sa_family == AF_INET6) { 3568 3568 - struct sockaddr_in6 *src_addr6 = 3569 3569 - (struct sockaddr_in6 *)&zero_sock; 3570 3570 - struct sockaddr_in6 *dst_addr6 = 3571 3571 - (struct sockaddr_in6 *)dst_addr; 3572 3572 - 3573 3573 - src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id; 3574 3574 - if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL) 3575 3575 - id->route.addr.dev_addr.bound_dev_if = 3576 3576 - dst_addr6->sin6_scope_id; 3577 3577 - } else if (dst_addr->sa_family == AF_IB) { 3578 3578 - ((struct sockaddr_ib *)&zero_sock)->sib_pkey = 3579 3579 - ((struct sockaddr_ib *)dst_addr)->sib_pkey; 3580 3580 - } 3581 3581 - return rdma_bind_addr(id, (struct sockaddr *)&zero_sock); 3582 3582 - } 3583 3583 - 3584 3584 - /* 3585 3585 - * If required, resolve the source address for bind and leave the id_priv in 3586 3586 - * state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior 3587 3587 - * calls made by ULP, a previously bound ID will not be re-bound and src_addr is 3588 3588 - * ignored. 3589 3589 - */ 3590 3590 - static int resolve_prepare_src(struct rdma_id_private *id_priv, 3591 3591 - struct sockaddr *src_addr, 3592 3592 - const struct sockaddr *dst_addr) 3593 3593 - { 3594 3594 - int ret; 3595 3595 - 3596 3596 - memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr)); 3597 3597 - if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) { 3598 3598 - /* For a well behaved ULP state will be RDMA_CM_IDLE */ 3599 3599 - ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr); 3600 3600 - if (ret) 3601 3601 - goto err_dst; 3602 3602 - if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, 3603 3603 - RDMA_CM_ADDR_QUERY))) { 3604 3604 - ret = -EINVAL; 3605 3605 - goto err_dst; 3606 3606 - } 3607 3607 - } 3608 3608 - 3609 3609 - if (cma_family(id_priv) != dst_addr->sa_family) { 3610 3610 - ret = -EINVAL; 3611 3611 - goto err_state; 3612 3612 - } 3613 3613 - return 0; 3614 3614 - 3615 3615 - err_state: 3616 3616 - cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND); 3617 3617 - err_dst: 3618 3618 - memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr)); 3619 3619 - return ret; 3620 3620 - } 3621 3621 - 3622 3622 - int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 3623 3623 - const struct sockaddr *dst_addr, unsigned long timeout_ms) 3624 3624 - { 3625 3625 - struct rdma_id_private *id_priv = 3626 3626 - container_of(id, struct rdma_id_private, id); 3627 3627 - int ret; 3628 3628 - 3629 3629 - ret = resolve_prepare_src(id_priv, src_addr, dst_addr); 3630 3630 - if (ret) 3631 3631 - return ret; 3632 3632 - 3633 3633 - if (cma_any_addr(dst_addr)) { 3634 3634 - ret = cma_resolve_loopback(id_priv); 3635 3635 - } else { 3636 3636 - if (dst_addr->sa_family == AF_IB) { 3637 3637 - ret = cma_resolve_ib_addr(id_priv); 3638 3638 - } else { 3639 3639 - /* 3640 3640 - * The FSM can return back to RDMA_CM_ADDR_BOUND after 3641 3641 - * rdma_resolve_ip() is called, eg through the error 3642 3642 - * path in addr_handler(). If this happens the existing 3643 3643 - * request must be canceled before issuing a new one. 3644 3644 - * Since canceling a request is a bit slow and this 3645 3645 - * oddball path is rare, keep track once a request has 3646 3646 - * been issued. The track turns out to be a permanent 3647 3647 - * state since this is the only cancel as it is 3648 3648 - * immediately before rdma_resolve_ip(). 3649 3649 - */ 3650 3650 - if (id_priv->used_resolve_ip) 3651 3651 - rdma_addr_cancel(&id->route.addr.dev_addr); 3652 3652 - else 3653 3653 - id_priv->used_resolve_ip = 1; 3654 3654 - ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr, 3655 3655 - &id->route.addr.dev_addr, 3656 3656 - timeout_ms, addr_handler, 3657 3657 - false, id_priv); 3658 3658 - } 3659 3659 - } 3660 3660 - if (ret) 3661 3661 - goto err; 3662 3662 - 3663 3663 - return 0; 3664 3664 - err: 3665 3665 - cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND); 3666 3666 - return ret; 3667 3667 - } 3668 3668 - EXPORT_SYMBOL(rdma_resolve_addr); 3669 3669 - 3670 3544 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse) 3671 3545 { 3672 3546 struct rdma_id_private *id_priv; ··· 3954 4058 } 3955 4059 EXPORT_SYMBOL(rdma_listen); 3956 4060 3957 3957 - int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) 4061 4061 + static int rdma_bind_addr_dst(struct rdma_id_private *id_priv, 4062 4062 + struct sockaddr *addr, const struct sockaddr *daddr) 3958 4063 { 3959 3959 - struct rdma_id_private *id_priv; 4064 4064 + struct sockaddr *id_daddr; 3960 4065 int ret; 3961 3961 - struct sockaddr *daddr; 3962 4066 3963 4067 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 && 3964 4068 addr->sa_family != AF_IB) 3965 4069 return -EAFNOSUPPORT; 3966 4070 3967 3967 - id_priv = container_of(id, struct rdma_id_private, id); 3968 4071 if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND)) 3969 4072 return -EINVAL; 3970 4073 3971 3971 - ret = cma_check_linklocal(&id->route.addr.dev_addr, addr); 4074 4074 + ret = cma_check_linklocal(&id_priv->id.route.addr.dev_addr, addr); 3972 4075 if (ret) 3973 4076 goto err1; 3974 4077 3975 4078 memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr)); 3976 4079 if (!cma_any_addr(addr)) { 3977 3977 - ret = cma_translate_addr(addr, &id->route.addr.dev_addr); 4080 4080 + ret = cma_translate_addr(addr, &id_priv->id.route.addr.dev_addr); 3978 4081 if (ret) 3979 4082 goto err1; 3980 4083 ··· 3993 4098 } 3994 4099 #endif 3995 4100 } 3996 3996 - daddr = cma_dst_addr(id_priv); 3997 3997 - daddr->sa_family = addr->sa_family; 4101 4101 + id_daddr = cma_dst_addr(id_priv); 4102 4102 + if (daddr != id_daddr) 4103 4103 + memcpy(id_daddr, daddr, rdma_addr_size(addr)); 4104 4104 + id_daddr->sa_family = addr->sa_family; 3998 4105 3999 4106 ret = cma_get_port(id_priv); 4000 4107 if (ret) ··· 4011 4114 err1: 4012 4115 cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE); 4013 4116 return ret; 4117 4117 + } 4118 4118 + 4119 4119 + static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 4120 4120 + const struct sockaddr *dst_addr) 4121 4121 + { 4122 4122 + struct rdma_id_private *id_priv = 4123 4123 + container_of(id, struct rdma_id_private, id); 4124 4124 + struct sockaddr_storage zero_sock = {}; 4125 4125 + 4126 4126 + if (src_addr && src_addr->sa_family) 4127 4127 + return rdma_bind_addr_dst(id_priv, src_addr, dst_addr); 4128 4128 + 4129 4129 + /* 4130 4130 + * When the src_addr is not specified, automatically supply an any addr 4131 4131 + */ 4132 4132 + zero_sock.ss_family = dst_addr->sa_family; 4133 4133 + if (IS_ENABLED(CONFIG_IPV6) && dst_addr->sa_family == AF_INET6) { 4134 4134 + struct sockaddr_in6 *src_addr6 = 4135 4135 + (struct sockaddr_in6 *)&zero_sock; 4136 4136 + struct sockaddr_in6 *dst_addr6 = 4137 4137 + (struct sockaddr_in6 *)dst_addr; 4138 4138 + 4139 4139 + src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id; 4140 4140 + if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL) 4141 4141 + id->route.addr.dev_addr.bound_dev_if = 4142 4142 + dst_addr6->sin6_scope_id; 4143 4143 + } else if (dst_addr->sa_family == AF_IB) { 4144 4144 + ((struct sockaddr_ib *)&zero_sock)->sib_pkey = 4145 4145 + ((struct sockaddr_ib *)dst_addr)->sib_pkey; 4146 4146 + } 4147 4147 + return rdma_bind_addr_dst(id_priv, (struct sockaddr *)&zero_sock, dst_addr); 4148 4148 + } 4149 4149 + 4150 4150 + /* 4151 4151 + * If required, resolve the source address for bind and leave the id_priv in 4152 4152 + * state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior 4153 4153 + * calls made by ULP, a previously bound ID will not be re-bound and src_addr is 4154 4154 + * ignored. 4155 4155 + */ 4156 4156 + static int resolve_prepare_src(struct rdma_id_private *id_priv, 4157 4157 + struct sockaddr *src_addr, 4158 4158 + const struct sockaddr *dst_addr) 4159 4159 + { 4160 4160 + int ret; 4161 4161 + 4162 4162 + if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) { 4163 4163 + /* For a well behaved ULP state will be RDMA_CM_IDLE */ 4164 4164 + ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr); 4165 4165 + if (ret) 4166 4166 + return ret; 4167 4167 + if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, 4168 4168 + RDMA_CM_ADDR_QUERY))) 4169 4169 + return -EINVAL; 4170 4170 + 4171 4171 + } 4172 4172 + 4173 4173 + if (cma_family(id_priv) != dst_addr->sa_family) { 4174 4174 + ret = -EINVAL; 4175 4175 + goto err_state; 4176 4176 + } 4177 4177 + return 0; 4178 4178 + 4179 4179 + err_state: 4180 4180 + cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND); 4181 4181 + return ret; 4182 4182 + } 4183 4183 + 4184 4184 + int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 4185 4185 + const struct sockaddr *dst_addr, unsigned long timeout_ms) 4186 4186 + { 4187 4187 + struct rdma_id_private *id_priv = 4188 4188 + container_of(id, struct rdma_id_private, id); 4189 4189 + int ret; 4190 4190 + 4191 4191 + ret = resolve_prepare_src(id_priv, src_addr, dst_addr); 4192 4192 + if (ret) 4193 4193 + return ret; 4194 4194 + 4195 4195 + if (cma_any_addr(dst_addr)) { 4196 4196 + ret = cma_resolve_loopback(id_priv); 4197 4197 + } else { 4198 4198 + if (dst_addr->sa_family == AF_IB) { 4199 4199 + ret = cma_resolve_ib_addr(id_priv); 4200 4200 + } else { 4201 4201 + /* 4202 4202 + * The FSM can return back to RDMA_CM_ADDR_BOUND after 4203 4203 + * rdma_resolve_ip() is called, eg through the error 4204 4204 + * path in addr_handler(). If this happens the existing 4205 4205 + * request must be canceled before issuing a new one. 4206 4206 + * Since canceling a request is a bit slow and this 4207 4207 + * oddball path is rare, keep track once a request has 4208 4208 + * been issued. The track turns out to be a permanent 4209 4209 + * state since this is the only cancel as it is 4210 4210 + * immediately before rdma_resolve_ip(). 4211 4211 + */ 4212 4212 + if (id_priv->used_resolve_ip) 4213 4213 + rdma_addr_cancel(&id->route.addr.dev_addr); 4214 4214 + else 4215 4215 + id_priv->used_resolve_ip = 1; 4216 4216 + ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr, 4217 4217 + &id->route.addr.dev_addr, 4218 4218 + timeout_ms, addr_handler, 4219 4219 + false, id_priv); 4220 4220 + } 4221 4221 + } 4222 4222 + if (ret) 4223 4223 + goto err; 4224 4224 + 4225 4225 + return 0; 4226 4226 + err: 4227 4227 + cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND); 4228 4228 + return ret; 4229 4229 + } 4230 4230 + EXPORT_SYMBOL(rdma_resolve_addr); 4231 4231 + 4232 4232 + int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) 4233 4233 + { 4234 4234 + struct rdma_id_private *id_priv = 4235 4235 + container_of(id, struct rdma_id_private, id); 4236 4236 + 4237 4237 + return rdma_bind_addr_dst(id_priv, addr, cma_dst_addr(id_priv)); 4014 4238 } 4015 4239 EXPORT_SYMBOL(rdma_bind_addr); 4016 4240

+55 -116

drivers/infiniband/core/sa_query.c

reviewed

··· 106 106 107 107 struct ib_sa_query { 108 108 void (*callback)(struct ib_sa_query *sa_query, int status, 109 109 - int num_prs, struct ib_sa_mad *mad); 109 109 + struct ib_sa_mad *mad); 110 110 void (*release)(struct ib_sa_query *); 111 111 struct ib_sa_client *client; 112 112 struct ib_sa_port *port; ··· 118 118 u32 seq; /* Local svc request sequence number */ 119 119 unsigned long timeout; /* Local svc timeout */ 120 120 u8 path_use; /* How will the pathrecord be used */ 121 121 - 122 122 - /* A separate buffer to save pathrecords of a response, as in cases 123 123 - * like IB/netlink, mulptiple pathrecords are supported, so that 124 124 - * mad->data is not large enough to hold them 125 125 - */ 126 126 - void *resp_pr_data; 127 121 }; 128 122 129 123 #define IB_SA_ENABLE_LOCAL_SERVICE 0x00000001 ··· 126 132 127 133 struct ib_sa_path_query { 128 134 void (*callback)(int status, struct sa_path_rec *rec, 129 129 - int num_paths, void *context); 135 135 + unsigned int num_paths, void *context); 130 136 void *context; 131 137 struct ib_sa_query sa_query; 132 138 struct sa_path_rec *conv_pr; ··· 684 690 .size_bits = 512 }, 685 691 }; 686 692 693 693 + #define RDMA_PRIMARY_PATH_MAX_REC_NUM 3 694 694 + 687 695 static inline void ib_sa_disable_local_svc(struct ib_sa_query *query) 688 696 { 689 697 query->flags &= ~IB_SA_ENABLE_LOCAL_SERVICE; ··· 870 874 static void ib_nl_process_good_resolve_rsp(struct ib_sa_query *query, 871 875 const struct nlmsghdr *nlh) 872 876 { 873 873 - struct ib_path_rec_data *srec, *drec; 877 877 + struct sa_path_rec recs[RDMA_PRIMARY_PATH_MAX_REC_NUM]; 874 878 struct ib_sa_path_query *path_query; 879 879 + struct ib_path_rec_data *rec_data; 875 880 struct ib_mad_send_wc mad_send_wc; 876 881 const struct nlattr *head, *curr; 877 882 struct ib_sa_mad *mad = NULL; 878 878 - int len, rem, num_prs = 0; 883 883 + int len, rem, status = -EIO; 884 884 + unsigned int num_prs = 0; 879 885 u32 mask = 0; 880 880 - int status = -EIO; 881 886 882 887 if (!query->callback) 883 888 goto out; 884 889 885 890 path_query = container_of(query, struct ib_sa_path_query, sa_query); 886 891 mad = query->mad_buf->mad; 887 887 - if (!path_query->conv_pr && 888 888 - (be16_to_cpu(mad->mad_hdr.attr_id) == IB_SA_ATTR_PATH_REC)) { 889 889 - /* Need a larger buffer for possible multiple PRs */ 890 890 - query->resp_pr_data = kvcalloc(RDMA_PRIMARY_PATH_MAX_REC_NUM, 891 891 - sizeof(*drec), GFP_KERNEL); 892 892 - if (!query->resp_pr_data) { 893 893 - query->callback(query, -ENOMEM, 0, NULL); 894 894 - return; 895 895 - } 896 896 - } 897 892 898 893 head = (const struct nlattr *) nlmsg_data(nlh); 899 894 len = nlmsg_len(nlh); ··· 904 917 break; 905 918 } 906 919 907 907 - drec = (struct ib_path_rec_data *)query->resp_pr_data; 908 920 nla_for_each_attr(curr, head, len, rem) { 909 921 if (curr->nla_type != LS_NLA_TYPE_PATH_RECORD) 910 922 continue; 911 923 912 912 - srec = nla_data(curr); 913 913 - if ((srec->flags & mask) != mask) 924 924 + rec_data = nla_data(curr); 925 925 + if ((rec_data->flags & mask) != mask) 914 926 continue; 915 927 916 916 - status = 0; 917 917 - if (!drec) { 918 918 - memcpy(mad->data, srec->path_rec, 919 919 - sizeof(srec->path_rec)); 920 920 - num_prs = 1; 921 921 - break; 928 928 + if ((query->flags & IB_SA_QUERY_OPA) || 929 929 + path_query->conv_pr) { 930 930 + mad->mad_hdr.method |= IB_MGMT_METHOD_RESP; 931 931 + memcpy(mad->data, rec_data->path_rec, 932 932 + sizeof(rec_data->path_rec)); 933 933 + query->callback(query, 0, mad); 934 934 + goto out; 922 935 } 923 936 924 924 - memcpy(drec, srec, sizeof(*drec)); 925 925 - drec++; 937 937 + status = 0; 938 938 + ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table), 939 939 + rec_data->path_rec, &recs[num_prs]); 940 940 + recs[num_prs].flags = rec_data->flags; 941 941 + recs[num_prs].rec_type = SA_PATH_REC_TYPE_IB; 942 942 + sa_path_set_dmac_zero(&recs[num_prs]); 943 943 + 926 944 num_prs++; 927 945 if (num_prs >= RDMA_PRIMARY_PATH_MAX_REC_NUM) 928 946 break; 929 947 } 930 948 931 931 - if (!status) 949 949 + if (!status) { 932 950 mad->mad_hdr.method |= IB_MGMT_METHOD_RESP; 933 933 - 934 934 - query->callback(query, status, num_prs, mad); 935 935 - kvfree(query->resp_pr_data); 936 936 - query->resp_pr_data = NULL; 951 951 + path_query->callback(status, recs, num_prs, 952 952 + path_query->context); 953 953 + } else 954 954 + query->callback(query, status, mad); 937 955 938 956 out: 939 957 mad_send_wc.send_buf = query->mad_buf; ··· 1443 1451 return PR_IB_SUPPORTED; 1444 1452 } 1445 1453 1446 1446 - static void ib_sa_pr_callback_single(struct ib_sa_path_query *query, 1447 1447 - int status, struct ib_sa_mad *mad) 1454 1454 + static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query, 1455 1455 + int status, struct ib_sa_mad *mad) 1448 1456 { 1457 1457 + struct ib_sa_path_query *query = 1458 1458 + container_of(sa_query, struct ib_sa_path_query, sa_query); 1449 1459 struct sa_path_rec rec = {}; 1460 1460 + 1461 1461 + if (!mad) { 1462 1462 + query->callback(status, NULL, 0, query->context); 1463 1463 + return; 1464 1464 + } 1465 1465 + 1466 1466 + if (sa_query->flags & IB_SA_QUERY_OPA) { 1467 1467 + ib_unpack(opa_path_rec_table, ARRAY_SIZE(opa_path_rec_table), 1468 1468 + mad->data, &rec); 1469 1469 + rec.rec_type = SA_PATH_REC_TYPE_OPA; 1470 1470 + query->callback(status, &rec, 1, query->context); 1471 1471 + return; 1472 1472 + } 1450 1473 1451 1474 ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table), 1452 1475 mad->data, &rec); ··· 1476 1469 query->callback(status, &opa, 1, query->context); 1477 1470 } else { 1478 1471 query->callback(status, &rec, 1, query->context); 1479 1479 - } 1480 1480 - } 1481 1481 - 1482 1482 - /** 1483 1483 - * ib_sa_pr_callback_multiple() - Parse path records then do callback. 1484 1484 - * 1485 1485 - * In a multiple-PR case the PRs are saved in "query->resp_pr_data" 1486 1486 - * (instead of"mad->data") and with "ib_path_rec_data" structure format, 1487 1487 - * so that rec->flags can be set to indicate the type of PR. 1488 1488 - * This is valid only in IB fabric. 1489 1489 - */ 1490 1490 - static void ib_sa_pr_callback_multiple(struct ib_sa_path_query *query, 1491 1491 - int status, int num_prs, 1492 1492 - struct ib_path_rec_data *rec_data) 1493 1493 - { 1494 1494 - struct sa_path_rec *rec; 1495 1495 - int i; 1496 1496 - 1497 1497 - rec = kvcalloc(num_prs, sizeof(*rec), GFP_KERNEL); 1498 1498 - if (!rec) { 1499 1499 - query->callback(-ENOMEM, NULL, 0, query->context); 1500 1500 - return; 1501 1501 - } 1502 1502 - 1503 1503 - for (i = 0; i < num_prs; i++) { 1504 1504 - ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table), 1505 1505 - rec_data[i].path_rec, rec + i); 1506 1506 - rec[i].rec_type = SA_PATH_REC_TYPE_IB; 1507 1507 - sa_path_set_dmac_zero(rec + i); 1508 1508 - rec[i].flags = rec_data[i].flags; 1509 1509 - } 1510 1510 - 1511 1511 - query->callback(status, rec, num_prs, query->context); 1512 1512 - kvfree(rec); 1513 1513 - } 1514 1514 - 1515 1515 - static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query, 1516 1516 - int status, int num_prs, 1517 1517 - struct ib_sa_mad *mad) 1518 1518 - { 1519 1519 - struct ib_sa_path_query *query = 1520 1520 - container_of(sa_query, struct ib_sa_path_query, sa_query); 1521 1521 - struct sa_path_rec rec; 1522 1522 - 1523 1523 - if (!mad || !num_prs) { 1524 1524 - query->callback(status, NULL, 0, query->context); 1525 1525 - return; 1526 1526 - } 1527 1527 - 1528 1528 - if (sa_query->flags & IB_SA_QUERY_OPA) { 1529 1529 - if (num_prs != 1) { 1530 1530 - query->callback(-EINVAL, NULL, 0, query->context); 1531 1531 - return; 1532 1532 - } 1533 1533 - 1534 1534 - ib_unpack(opa_path_rec_table, ARRAY_SIZE(opa_path_rec_table), 1535 1535 - mad->data, &rec); 1536 1536 - rec.rec_type = SA_PATH_REC_TYPE_OPA; 1537 1537 - query->callback(status, &rec, num_prs, query->context); 1538 1538 - } else { 1539 1539 - if (!sa_query->resp_pr_data) 1540 1540 - ib_sa_pr_callback_single(query, status, mad); 1541 1541 - else 1542 1542 - ib_sa_pr_callback_multiple(query, status, num_prs, 1543 1543 - sa_query->resp_pr_data); 1544 1472 } 1545 1473 } 1546 1474 ··· 1520 1578 unsigned long timeout_ms, gfp_t gfp_mask, 1521 1579 void (*callback)(int status, 1522 1580 struct sa_path_rec *resp, 1523 1523 - int num_paths, void *context), 1581 1581 + unsigned int num_paths, void *context), 1524 1582 void *context, 1525 1583 struct ib_sa_query **sa_query) 1526 1584 { ··· 1619 1677 EXPORT_SYMBOL(ib_sa_path_rec_get); 1620 1678 1621 1679 static void ib_sa_mcmember_rec_callback(struct ib_sa_query *sa_query, 1622 1622 - int status, int num_prs, 1623 1623 - struct ib_sa_mad *mad) 1680 1680 + int status, struct ib_sa_mad *mad) 1624 1681 { 1625 1682 struct ib_sa_mcmember_query *query = 1626 1683 container_of(sa_query, struct ib_sa_mcmember_query, sa_query); ··· 1710 1769 1711 1770 /* Support GuidInfoRecord */ 1712 1771 static void ib_sa_guidinfo_rec_callback(struct ib_sa_query *sa_query, 1713 1713 - int status, int num_paths, 1714 1714 - struct ib_sa_mad *mad) 1772 1772 + int status, struct ib_sa_mad *mad) 1715 1773 { 1716 1774 struct ib_sa_guidinfo_query *query = 1717 1775 container_of(sa_query, struct ib_sa_guidinfo_query, sa_query); ··· 1819 1879 } 1820 1880 1821 1881 static void ib_sa_classport_info_rec_callback(struct ib_sa_query *sa_query, 1822 1822 - int status, int num_prs, 1823 1823 - struct ib_sa_mad *mad) 1882 1882 + int status, struct ib_sa_mad *mad) 1824 1883 { 1825 1884 unsigned long flags; 1826 1885 struct ib_sa_classport_info_query *query = ··· 1994 2055 /* No callback -- already got recv */ 1995 2056 break; 1996 2057 case IB_WC_RESP_TIMEOUT_ERR: 1997 1997 - query->callback(query, -ETIMEDOUT, 0, NULL); 2058 2058 + query->callback(query, -ETIMEDOUT, NULL); 1998 2059 break; 1999 2060 case IB_WC_WR_FLUSH_ERR: 2000 2000 - query->callback(query, -EINTR, 0, NULL); 2061 2061 + query->callback(query, -EINTR, NULL); 2001 2062 break; 2002 2063 default: 2003 2003 - query->callback(query, -EIO, 0, NULL); 2064 2064 + query->callback(query, -EIO, NULL); 2004 2065 break; 2005 2066 } 2006 2067 ··· 2028 2089 if (mad_recv_wc->wc->status == IB_WC_SUCCESS) 2029 2090 query->callback(query, 2030 2091 mad_recv_wc->recv_buf.mad->mad_hdr.status ? 2031 2031 - -EINVAL : 0, 1, 2092 2092 + -EINVAL : 0, 2032 2093 (struct ib_sa_mad *) mad_recv_wc->recv_buf.mad); 2033 2094 else 2034 2034 - query->callback(query, -EIO, 0, NULL); 2095 2095 + query->callback(query, -EIO, NULL); 2035 2096 } 2036 2097 2037 2098 ib_free_recv_mad(mad_recv_wc);

+7

drivers/infiniband/hw/cxgb4/cm.c

reviewed

··· 2676 2676 u16 tcp_opt = ntohs(req->tcp_opt); 2677 2677 2678 2678 ep = get_ep_from_tid(dev, tid); 2679 2679 + if (!ep) 2680 2680 + return 0; 2681 2681 + 2679 2682 pr_debug("ep %p tid %u\n", ep, ep->hwtid); 2680 2683 ep->snd_seq = be32_to_cpu(req->snd_isn); 2681 2684 ep->rcv_seq = be32_to_cpu(req->rcv_isn); ··· 4147 4144 4148 4145 if (neigh->dev->flags & IFF_LOOPBACK) { 4149 4146 pdev = ip_dev_find(&init_net, iph->daddr); 4147 4147 + if (!pdev) { 4148 4148 + pr_err("%s - failed to find device!\n", __func__); 4149 4149 + goto free_dst; 4150 4150 + } 4150 4151 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 4151 4152 pdev, 0); 4152 4153 pi = (struct port_info *)netdev_priv(pdev);

+1 -1

drivers/infiniband/hw/cxgb4/cq.c

reviewed

··· 767 767 goto out; 768 768 769 769 wc->wr_id = cookie; 770 770 - wc->qp = qhp ? &qhp->ibqp : NULL; 770 770 + wc->qp = &qhp->ibqp; 771 771 wc->vendor_err = CQE_STATUS(&cqe); 772 772 wc->wc_flags = 0; 773 773

+1 -1

drivers/infiniband/hw/cxgb4/restrack.c

reviewed

··· 238 238 if (rdma_nl_put_driver_u64_hex(msg, "history", epcp->history)) 239 239 goto err_cancel_table; 240 240 241 241 - if (epcp->state == LISTEN) { 241 241 + if (listen_ep) { 242 242 if (rdma_nl_put_driver_u32(msg, "stid", listen_ep->stid)) 243 243 goto err_cancel_table; 244 244 if (rdma_nl_put_driver_u32(msg, "backlog", listen_ep->backlog))

+6 -20

drivers/infiniband/hw/cxgb4/t4fw_ri_api.h

reviewed

··· 122 122 __be16 nsge; 123 123 __be32 len0; 124 124 __be64 addr0; 125 125 - #ifndef C99_NOT_SUPPORTED 126 125 struct fw_ri_dsge_pair sge[]; 127 127 - #endif 128 126 }; 129 127 130 128 struct fw_ri_sge { ··· 136 138 __u8 r1; 137 139 __be16 nsge; 138 140 __be32 r2; 139 139 - #ifndef C99_NOT_SUPPORTED 140 141 struct fw_ri_sge sge[]; 141 141 - #endif 142 142 }; 143 143 144 144 struct fw_ri_immd { ··· 144 148 __u8 r1; 145 149 __be16 r2; 146 150 __be32 immdlen; 147 147 - #ifndef C99_NOT_SUPPORTED 148 151 __u8 data[]; 149 149 - #endif 150 152 }; 151 153 152 154 struct fw_ri_tpte { ··· 314 320 __be32 op_nres; 315 321 __be32 len16_pkd; 316 322 __u64 cookie; 317 317 - #ifndef C99_NOT_SUPPORTED 318 323 struct fw_ri_res res[]; 319 319 - #endif 320 324 }; 321 325 322 326 #define FW_RI_RES_WR_NRES_S 0 ··· 554 562 __be32 plen; 555 563 __be32 stag_sink; 556 564 __be64 to_sink; 557 557 - #ifndef C99_NOT_SUPPORTED 558 565 union { 559 559 - struct fw_ri_immd immd_src[0]; 560 560 - struct fw_ri_isgl isgl_src[0]; 566 566 + DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src); 567 567 + DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src); 561 568 } u; 562 562 - #endif 563 569 }; 564 570 565 571 struct fw_ri_send_wr { ··· 571 581 __be32 plen; 572 582 __be32 r3; 573 583 __be64 r4; 574 574 - #ifndef C99_NOT_SUPPORTED 575 584 union { 576 576 - struct fw_ri_immd immd_src[0]; 577 577 - struct fw_ri_isgl isgl_src[0]; 585 585 + DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src); 586 586 + DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src); 578 587 } u; 579 579 - #endif 580 588 }; 581 589 582 590 #define FW_RI_SEND_WR_SENDOP_S 0 ··· 606 618 struct fw_ri_isgl isgl_src; 607 619 } u_cmpl; 608 620 __be64 r3; 609 609 - #ifndef C99_NOT_SUPPORTED 610 621 union fw_ri_write { 611 611 - struct fw_ri_immd immd_src[0]; 612 612 - struct fw_ri_isgl isgl_src[0]; 622 622 + DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src); 623 623 + DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src); 613 624 } u; 614 614 - #endif 615 625 }; 616 626 617 627 struct fw_ri_rdma_read_wr {

+2 -2

drivers/infiniband/hw/erdma/erdma_hw.h

reviewed

··· 397 397 398 398 __le32 rsvd; 399 399 400 400 - struct erdma_sge sgl[0]; 400 400 + struct erdma_sge sgl[]; 401 401 }; 402 402 403 403 struct erdma_send_sqe { ··· 408 408 }; 409 409 410 410 __le32 length; 411 411 - struct erdma_sge sgl[0]; 411 411 + struct erdma_sge sgl[]; 412 412 }; 413 413 414 414 struct erdma_readreq_sqe {

+3 -1

drivers/infiniband/hw/erdma/erdma_verbs.c

reviewed

··· 1110 1110 prot = pgprot_device(vma->vm_page_prot); 1111 1111 break; 1112 1112 default: 1113 1113 - return -EINVAL; 1113 1113 + err = -EINVAL; 1114 1114 + goto put_entry; 1114 1115 } 1115 1116 1116 1117 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE, 1117 1118 prot, rdma_entry); 1118 1119 1120 1120 + put_entry: 1119 1121 rdma_user_mmap_entry_put(rdma_entry); 1120 1122 return err; 1121 1123 }

+32 -27

drivers/infiniband/hw/hfi1/chip.c

reviewed

··· 1056 1056 static void handle_temp_err(struct hfi1_devdata *dd); 1057 1057 static void dc_shutdown(struct hfi1_devdata *dd); 1058 1058 static void dc_start(struct hfi1_devdata *dd); 1059 1059 - static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, 1059 1059 + static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp, 1060 1060 unsigned int *np); 1061 1061 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd); 1062 1062 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms); ··· 13362 13362 int ret; 13363 13363 unsigned ngroups; 13364 13364 int rmt_count; 13365 13365 - int user_rmt_reduced; 13366 13365 u32 n_usr_ctxts; 13367 13366 u32 send_contexts = chip_send_contexts(dd); 13368 13367 u32 rcv_contexts = chip_rcv_contexts(dd); ··· 13420 13421 (num_kernel_contexts + n_usr_ctxts), 13421 13422 &node_affinity.real_cpu_mask); 13422 13423 /* 13423 13423 - * The RMT entries are currently allocated as shown below: 13424 13424 - * 1. QOS (0 to 128 entries); 13425 13425 - * 2. FECN (num_kernel_context - 1 + num_user_contexts + 13426 13426 - * num_netdev_contexts); 13427 13427 - * 3. netdev (num_netdev_contexts). 13428 13428 - * It should be noted that FECN oversubscribe num_netdev_contexts 13429 13429 - * entries of RMT because both netdev and PSM could allocate any receive 13430 13430 - * context between dd->first_dyn_alloc_text and dd->num_rcv_contexts, 13431 13431 - * and PSM FECN must reserve an RMT entry for each possible PSM receive 13432 13432 - * context. 13424 13424 + * RMT entries are allocated as follows: 13425 13425 + * 1. QOS (0 to 128 entries) 13426 13426 + * 2. FECN (num_kernel_context - 1 [a] + num_user_contexts + 13427 13427 + * num_netdev_contexts [b]) 13428 13428 + * 3. netdev (NUM_NETDEV_MAP_ENTRIES) 13429 13429 + * 13430 13430 + * Notes: 13431 13431 + * [a] Kernel contexts (except control) are included in FECN if kernel 13432 13432 + * TID_RDMA is active. 13433 13433 + * [b] Netdev and user contexts are randomly allocated from the same 13434 13434 + * context pool, so FECN must cover all contexts in the pool. 13433 13435 */ 13434 13434 - rmt_count = qos_rmt_entries(dd, NULL, NULL) + (num_netdev_contexts * 2); 13435 13435 - if (HFI1_CAP_IS_KSET(TID_RDMA)) 13436 13436 - rmt_count += num_kernel_contexts - 1; 13437 13437 - if (rmt_count + n_usr_ctxts > NUM_MAP_ENTRIES) { 13438 13438 - user_rmt_reduced = NUM_MAP_ENTRIES - rmt_count; 13439 13439 - dd_dev_err(dd, 13440 13440 - "RMT size is reducing the number of user receive contexts from %u to %d\n", 13441 13441 - n_usr_ctxts, 13442 13442 - user_rmt_reduced); 13443 13443 - /* recalculate */ 13444 13444 - n_usr_ctxts = user_rmt_reduced; 13436 13436 + rmt_count = qos_rmt_entries(num_kernel_contexts - 1, NULL, NULL) 13437 13437 + + (HFI1_CAP_IS_KSET(TID_RDMA) ? num_kernel_contexts - 1 13438 13438 + : 0) 13439 13439 + + n_usr_ctxts 13440 13440 + + num_netdev_contexts 13441 13441 + + NUM_NETDEV_MAP_ENTRIES; 13442 13442 + if (rmt_count > NUM_MAP_ENTRIES) { 13443 13443 + int over = rmt_count - NUM_MAP_ENTRIES; 13444 13444 + /* try to squish user contexts, minimum of 1 */ 13445 13445 + if (over >= n_usr_ctxts) { 13446 13446 + dd_dev_err(dd, "RMT overflow: reduce the requested number of contexts\n"); 13447 13447 + return -EINVAL; 13448 13448 + } 13449 13449 + dd_dev_err(dd, "RMT overflow: reducing # user contexts from %u to %u\n", 13450 13450 + n_usr_ctxts, n_usr_ctxts - over); 13451 13451 + n_usr_ctxts -= over; 13445 13452 } 13446 13453 13447 13454 /* the first N are kernel contexts, the rest are user/netdev contexts */ ··· 14304 14299 } 14305 14300 14306 14301 /* return the number of RSM map table entries that will be used for QOS */ 14307 14307 - static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, 14302 14302 + static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp, 14308 14303 unsigned int *np) 14309 14304 { 14310 14305 int i; 14311 14306 unsigned int m, n; 14312 14312 - u8 max_by_vl = 0; 14307 14307 + uint max_by_vl = 0; 14313 14308 14314 14309 /* is QOS active at all? */ 14315 14315 - if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS || 14310 14310 + if (n_krcv_queues < MIN_KERNEL_KCTXTS || 14316 14311 num_vls == 1 || 14317 14312 krcvqsset <= 1) 14318 14313 goto no_qos; ··· 14370 14365 14371 14366 if (!rmt) 14372 14367 goto bail; 14373 14373 - rmt_entries = qos_rmt_entries(dd, &m, &n); 14368 14368 + rmt_entries = qos_rmt_entries(dd->n_krcv_queues - 1, &m, &n); 14374 14369 if (rmt_entries == 0) 14375 14370 goto bail; 14376 14371 qpns_per_vl = 1 << m;

+3 -2

drivers/infiniband/hw/hfi1/exp_rcv.h

reviewed

··· 133 133 return grp; 134 134 } 135 135 136 136 - static inline u32 rcventry2tidinfo(u32 rcventry) 136 136 + static inline u32 create_tid(u32 rcventry, u32 npages) 137 137 { 138 138 u32 pair = rcventry & ~0x1; 139 139 140 140 return EXP_TID_SET(IDX, pair >> 1) | 141 141 - EXP_TID_SET(CTRL, 1 << (rcventry - pair)); 141 141 + EXP_TID_SET(CTRL, 1 << (rcventry - pair)) | 142 142 + EXP_TID_SET(LEN, npages); 142 143 } 143 144 144 145 /**

+57 -24

drivers/infiniband/hw/hfi1/file_ops.c

reviewed

··· 306 306 return reqs; 307 307 } 308 308 309 309 + static inline void mmap_cdbg(u16 ctxt, u8 subctxt, u8 type, u8 mapio, u8 vmf, 310 310 + u64 memaddr, void *memvirt, dma_addr_t memdma, 311 311 + ssize_t memlen, struct vm_area_struct *vma) 312 312 + { 313 313 + hfi1_cdbg(PROC, 314 314 + "%u:%u type:%u io/vf/dma:%d/%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx", 315 315 + ctxt, subctxt, type, mapio, vmf, !!memdma, 316 316 + memaddr ?: (u64)memvirt, memlen, 317 317 + vma->vm_end - vma->vm_start, vma->vm_flags); 318 318 + } 319 319 + 309 320 static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) 310 321 { 311 322 struct hfi1_filedata *fd = fp->private_data; ··· 326 315 u64 token = vma->vm_pgoff << PAGE_SHIFT, 327 316 memaddr = 0; 328 317 void *memvirt = NULL; 318 318 + dma_addr_t memdma = 0; 329 319 u8 subctxt, mapio = 0, vmf = 0, type; 330 320 ssize_t memlen = 0; 331 321 int ret = 0; ··· 346 334 goto done; 347 335 } 348 336 337 337 + /* 338 338 + * vm_pgoff is used as a buffer selector cookie. Always mmap from 339 339 + * the beginning. 340 340 + */ 341 341 + vma->vm_pgoff = 0; 349 342 flags = vma->vm_flags; 350 343 351 344 switch (type) { ··· 372 355 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 373 356 mapio = 1; 374 357 break; 375 375 - case PIO_CRED: 358 358 + case PIO_CRED: { 359 359 + u64 cr_page_offset; 376 360 if (flags & VM_WRITE) { 377 361 ret = -EPERM; 378 362 goto done; ··· 383 365 * second or third page allocated for credit returns (if number 384 366 * of enabled contexts > 64 and 128 respectively). 385 367 */ 386 386 - memvirt = dd->cr_base[uctxt->numa_id].va; 387 387 - memaddr = virt_to_phys(memvirt) + 388 388 - (((u64)uctxt->sc->hw_free - 389 389 - (u64)dd->cr_base[uctxt->numa_id].va) & PAGE_MASK); 368 368 + cr_page_offset = ((u64)uctxt->sc->hw_free - 369 369 + (u64)dd->cr_base[uctxt->numa_id].va) & 370 370 + PAGE_MASK; 371 371 + memvirt = dd->cr_base[uctxt->numa_id].va + cr_page_offset; 372 372 + memdma = dd->cr_base[uctxt->numa_id].dma + cr_page_offset; 390 373 memlen = PAGE_SIZE; 391 374 flags &= ~VM_MAYWRITE; 392 375 flags |= VM_DONTCOPY | VM_DONTEXPAND; ··· 397 378 * memory been flagged as non-cached? 398 379 */ 399 380 /* vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); */ 400 400 - mapio = 1; 401 381 break; 382 382 + } 402 383 case RCV_HDRQ: 403 384 memlen = rcvhdrq_size(uctxt); 404 385 memvirt = uctxt->rcvhdrq; 386 386 + memdma = uctxt->rcvhdrq_dma; 405 387 break; 406 388 case RCV_EGRBUF: { 407 407 - unsigned long addr; 389 389 + unsigned long vm_start_save; 390 390 + unsigned long vm_end_save; 408 391 int i; 409 392 /* 410 393 * The RcvEgr buffer need to be handled differently ··· 425 404 goto done; 426 405 } 427 406 vm_flags_clear(vma, VM_MAYWRITE); 428 428 - addr = vma->vm_start; 407 407 + /* 408 408 + * Mmap multiple separate allocations into a single vma. From 409 409 + * here, dma_mmap_coherent() calls dma_direct_mmap(), which 410 410 + * requires the mmap to exactly fill the vma starting at 411 411 + * vma_start. Adjust the vma start and end for each eager 412 412 + * buffer segment mapped. Restore the originals when done. 413 413 + */ 414 414 + vm_start_save = vma->vm_start; 415 415 + vm_end_save = vma->vm_end; 416 416 + vma->vm_end = vma->vm_start; 429 417 for (i = 0 ; i < uctxt->egrbufs.numbufs; i++) { 430 418 memlen = uctxt->egrbufs.buffers[i].len; 431 419 memvirt = uctxt->egrbufs.buffers[i].addr; 432 432 - ret = remap_pfn_range( 433 433 - vma, addr, 434 434 - /* 435 435 - * virt_to_pfn() does the same, but 436 436 - * it's not available on x86_64 437 437 - * when CONFIG_MMU is enabled. 438 438 - */ 439 439 - PFN_DOWN(__pa(memvirt)), 440 440 - memlen, 441 441 - vma->vm_page_prot); 442 442 - if (ret < 0) 420 420 + memdma = uctxt->egrbufs.buffers[i].dma; 421 421 + vma->vm_end += memlen; 422 422 + mmap_cdbg(ctxt, subctxt, type, mapio, vmf, memaddr, 423 423 + memvirt, memdma, memlen, vma); 424 424 + ret = dma_mmap_coherent(&dd->pcidev->dev, vma, 425 425 + memvirt, memdma, memlen); 426 426 + if (ret < 0) { 427 427 + vma->vm_start = vm_start_save; 428 428 + vma->vm_end = vm_end_save; 443 429 goto done; 444 444 - addr += memlen; 430 430 + } 431 431 + vma->vm_start += memlen; 445 432 } 433 433 + vma->vm_start = vm_start_save; 434 434 + vma->vm_end = vm_end_save; 446 435 ret = 0; 447 436 goto done; 448 437 } ··· 512 481 } 513 482 memlen = PAGE_SIZE; 514 483 memvirt = (void *)hfi1_rcvhdrtail_kvaddr(uctxt); 484 484 + memdma = uctxt->rcvhdrqtailaddr_dma; 515 485 flags &= ~VM_MAYWRITE; 516 486 break; 517 487 case SUBCTXT_UREGS: ··· 561 529 } 562 530 563 531 vm_flags_reset(vma, flags); 564 564 - hfi1_cdbg(PROC, 565 565 - "%u:%u type:%u io/vf:%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx\n", 566 566 - ctxt, subctxt, type, mapio, vmf, memaddr, memlen, 567 567 - vma->vm_end - vma->vm_start, vma->vm_flags); 532 532 + mmap_cdbg(ctxt, subctxt, type, mapio, vmf, memaddr, memvirt, memdma, 533 533 + memlen, vma); 568 534 if (vmf) { 569 535 vma->vm_pgoff = PFN_DOWN(memaddr); 570 536 vma->vm_ops = &vm_ops; 571 537 ret = 0; 538 538 + } else if (memdma) { 539 539 + ret = dma_mmap_coherent(&dd->pcidev->dev, vma, 540 540 + memvirt, memdma, memlen); 572 541 } else if (mapio) { 573 542 ret = io_remap_pfn_range(vma, vma->vm_start, 574 543 PFN_DOWN(memaddr),

+1 -1

drivers/infiniband/hw/hfi1/init.c

reviewed

··· 464 464 * 465 465 * This wrapper is the free function that matches hfi1_create_ctxtdata(). 466 466 * When a context is done being used (kernel or user), this function is called 467 467 - * for the "final" put to match the kref init from hf1i_create_ctxtdata(). 467 467 + * for the "final" put to match the kref init from hfi1_create_ctxtdata(). 468 468 * Other users of the context do a get/put sequence to make sure that the 469 469 * structure isn't removed while in use. 470 470 */

+2 -2

drivers/infiniband/hw/hfi1/sdma.c

reviewed

··· 3160 3160 { 3161 3161 int rval = 0; 3162 3162 3163 3163 - tx->num_desc++; 3164 3164 - if ((unlikely(tx->num_desc == tx->desc_limit))) { 3163 3163 + if ((unlikely(tx->num_desc + 1 == tx->desc_limit))) { 3165 3164 rval = _extend_sdma_tx_descs(dd, tx); 3166 3165 if (rval) { 3167 3166 __sdma_txclean(dd, tx); ··· 3173 3174 SDMA_MAP_NONE, 3174 3175 dd->sdma_pad_phys, 3175 3176 sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1))); 3177 3177 + tx->num_desc++; 3176 3178 _sdma_close_tx(dd, tx); 3177 3179 return rval; 3178 3180 }

+7 -8

drivers/infiniband/hw/hfi1/sdma.h

reviewed

··· 631 631 static inline void _sdma_close_tx(struct hfi1_devdata *dd, 632 632 struct sdma_txreq *tx) 633 633 { 634 634 - tx->descp[tx->num_desc].qw[0] |= 635 635 - SDMA_DESC0_LAST_DESC_FLAG; 636 636 - tx->descp[tx->num_desc].qw[1] |= 637 637 - dd->default_desc1; 634 634 + u16 last_desc = tx->num_desc - 1; 635 635 + 636 636 + tx->descp[last_desc].qw[0] |= SDMA_DESC0_LAST_DESC_FLAG; 637 637 + tx->descp[last_desc].qw[1] |= dd->default_desc1; 638 638 if (tx->flags & SDMA_TXREQ_F_URGENT) 639 639 - tx->descp[tx->num_desc].qw[1] |= 640 640 - (SDMA_DESC1_HEAD_TO_HOST_FLAG | 641 641 - SDMA_DESC1_INT_REQ_FLAG); 639 639 + tx->descp[last_desc].qw[1] |= (SDMA_DESC1_HEAD_TO_HOST_FLAG | 640 640 + SDMA_DESC1_INT_REQ_FLAG); 642 641 } 643 642 644 643 static inline int _sdma_txadd_daddr( ··· 654 655 type, 655 656 addr, len); 656 657 WARN_ON(len > tx->tlen); 658 658 + tx->num_desc++; 657 659 tx->tlen -= len; 658 660 /* special cases for last */ 659 661 if (!tx->tlen) { ··· 666 666 _sdma_close_tx(dd, tx); 667 667 } 668 668 } 669 669 - tx->num_desc++; 670 669 return rval; 671 670 } 672 671

+20 -26

drivers/infiniband/hw/hfi1/user_exp_rcv.c

reviewed

··· 27 27 const struct mmu_notifier_range *range, 28 28 unsigned long cur_seq); 29 29 static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *, 30 30 - struct tid_group *grp, 31 31 - unsigned int start, u16 count, 30 30 + struct tid_group *grp, u16 count, 32 31 u32 *tidlist, unsigned int *tididx, 33 32 unsigned int *pmapped); 34 33 static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo); ··· 249 250 int ret = 0, need_group = 0, pinned; 250 251 struct hfi1_ctxtdata *uctxt = fd->uctxt; 251 252 struct hfi1_devdata *dd = uctxt->dd; 252 252 - unsigned int ngroups, pageidx = 0, pageset_count, 253 253 + unsigned int ngroups, pageset_count, 253 254 tididx = 0, mapped, mapped_pages = 0; 254 255 u32 *tidlist = NULL; 255 256 struct tid_user_buf *tidbuf; ··· 331 332 tid_group_pop(&uctxt->tid_group_list); 332 333 333 334 ret = program_rcvarray(fd, tidbuf, grp, 334 334 - pageidx, dd->rcv_entries.group_size, 335 335 + dd->rcv_entries.group_size, 335 336 tidlist, &tididx, &mapped); 336 337 /* 337 338 * If there was a failure to program the RcvArray ··· 347 348 348 349 tid_group_add_tail(grp, &uctxt->tid_full_list); 349 350 ngroups--; 350 350 - pageidx += ret; 351 351 mapped_pages += mapped; 352 352 } 353 353 354 354 - while (pageidx < pageset_count) { 354 354 + while (tididx < pageset_count) { 355 355 struct tid_group *grp, *ptr; 356 356 /* 357 357 * If we don't have any partially used tid groups, check ··· 372 374 */ 373 375 list_for_each_entry_safe(grp, ptr, &uctxt->tid_used_list.list, 374 376 list) { 375 375 - unsigned use = min_t(unsigned, pageset_count - pageidx, 377 377 + unsigned use = min_t(unsigned, pageset_count - tididx, 376 378 grp->size - grp->used); 377 379 378 380 ret = program_rcvarray(fd, tidbuf, grp, 379 379 - pageidx, use, tidlist, 381 381 + use, tidlist, 380 382 &tididx, &mapped); 381 383 if (ret < 0) { 382 384 hfi1_cdbg(TID, ··· 388 390 tid_group_move(grp, 389 391 &uctxt->tid_used_list, 390 392 &uctxt->tid_full_list); 391 391 - pageidx += ret; 392 393 mapped_pages += mapped; 393 394 need_group = 0; 394 395 /* Check if we are done so we break out early */ 395 395 - if (pageidx >= pageset_count) 396 396 + if (tididx >= pageset_count) 396 397 break; 397 398 } else if (WARN_ON(ret == 0)) { 398 399 /* ··· 635 638 * struct tid_pageset holding information on physically contiguous 636 639 * chunks from the user buffer), and other fields. 637 640 * @grp: RcvArray group 638 638 - * @start: starting index into sets array 639 641 * @count: number of struct tid_pageset's to program 640 642 * @tidlist: the array of u32 elements when the information about the 641 643 * programmed RcvArray entries is to be encoded. ··· 654 658 * number of RcvArray entries programmed. 655 659 */ 656 660 static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *tbuf, 657 657 - struct tid_group *grp, 658 658 - unsigned int start, u16 count, 661 661 + struct tid_group *grp, u16 count, 659 662 u32 *tidlist, unsigned int *tididx, 660 663 unsigned int *pmapped) 661 664 { 662 665 struct hfi1_ctxtdata *uctxt = fd->uctxt; 663 666 struct hfi1_devdata *dd = uctxt->dd; 664 667 u16 idx; 668 668 + unsigned int start = *tididx; 665 669 u32 tidinfo = 0, rcventry, useidx = 0; 666 670 int mapped = 0; 667 671 ··· 706 710 return ret; 707 711 mapped += npages; 708 712 709 709 - tidinfo = rcventry2tidinfo(rcventry - uctxt->expected_base) | 710 710 - EXP_TID_SET(LEN, npages); 713 713 + tidinfo = create_tid(rcventry - uctxt->expected_base, npages); 711 714 tidlist[(*tididx)++] = tidinfo; 712 715 grp->used++; 713 716 grp->map |= 1 << useidx++; ··· 790 795 struct hfi1_ctxtdata *uctxt = fd->uctxt; 791 796 struct hfi1_devdata *dd = uctxt->dd; 792 797 struct tid_rb_node *node; 793 793 - u8 tidctrl = EXP_TID_GET(tidinfo, CTRL); 798 798 + u32 tidctrl = EXP_TID_GET(tidinfo, CTRL); 794 799 u32 tididx = EXP_TID_GET(tidinfo, IDX) << 1, rcventry; 795 800 796 796 - if (tididx >= uctxt->expected_count) { 797 797 - dd_dev_err(dd, "Invalid RcvArray entry (%u) index for ctxt %u\n", 798 798 - tididx, uctxt->ctxt); 799 799 - return -EINVAL; 800 800 - } 801 801 - 802 802 - if (tidctrl == 0x3) 801 801 + if (tidctrl == 0x3 || tidctrl == 0x0) 803 802 return -EINVAL; 804 803 805 804 rcventry = tididx + (tidctrl - 1); 805 805 + 806 806 + if (rcventry >= uctxt->expected_count) { 807 807 + dd_dev_err(dd, "Invalid RcvArray entry (%u) index for ctxt %u\n", 808 808 + rcventry, uctxt->ctxt); 809 809 + return -EINVAL; 810 810 + } 806 811 807 812 node = fd->entry_to_rb[rcventry]; 808 813 if (!node || node->rcventry != (uctxt->expected_base + rcventry)) ··· 915 920 spin_lock(&fdata->invalid_lock); 916 921 if (fdata->invalid_tid_idx < uctxt->expected_count) { 917 922 fdata->invalid_tids[fdata->invalid_tid_idx] = 918 918 - rcventry2tidinfo(node->rcventry - uctxt->expected_base); 919 919 - fdata->invalid_tids[fdata->invalid_tid_idx] |= 920 920 - EXP_TID_SET(LEN, node->npages); 923 923 + create_tid(node->rcventry - uctxt->expected_base, 924 924 + node->npages); 921 925 if (!fdata->invalid_tid_idx) { 922 926 unsigned long *ev; 923 927

+38 -19

drivers/infiniband/hw/hfi1/user_pages.c

reviewed

··· 29 29 bool hfi1_can_pin_pages(struct hfi1_devdata *dd, struct mm_struct *mm, 30 30 u32 nlocked, u32 npages) 31 31 { 32 32 - unsigned long ulimit = rlimit(RLIMIT_MEMLOCK), pinned, cache_limit, 33 33 - size = (cache_size * (1UL << 20)); /* convert to bytes */ 34 34 - unsigned int usr_ctxts = 35 35 - dd->num_rcv_contexts - dd->first_dyn_alloc_ctxt; 36 36 - bool can_lock = capable(CAP_IPC_LOCK); 32 32 + unsigned long ulimit_pages; 33 33 + unsigned long cache_limit_pages; 34 34 + unsigned int usr_ctxts; 37 35 38 36 /* 39 39 - * Calculate per-cache size. The calculation below uses only a quarter 40 40 - * of the available per-context limit. This leaves space for other 41 41 - * pinning. Should we worry about shared ctxts? 37 37 + * Perform RLIMIT_MEMLOCK based checks unless CAP_IPC_LOCK is present. 42 38 */ 43 43 - cache_limit = (ulimit / usr_ctxts) / 4; 39 39 + if (!capable(CAP_IPC_LOCK)) { 40 40 + ulimit_pages = 41 41 + DIV_ROUND_DOWN_ULL(rlimit(RLIMIT_MEMLOCK), PAGE_SIZE); 44 42 45 45 - /* If ulimit isn't set to "unlimited" and is smaller than cache_size. */ 46 46 - if (ulimit != (-1UL) && size > cache_limit) 47 47 - size = cache_limit; 43 43 + /* 44 44 + * Pinning these pages would exceed this process's locked memory 45 45 + * limit. 46 46 + */ 47 47 + if (atomic64_read(&mm->pinned_vm) + npages > ulimit_pages) 48 48 + return false; 48 49 49 49 - /* Convert to number of pages */ 50 50 - size = DIV_ROUND_UP(size, PAGE_SIZE); 50 50 + /* 51 51 + * Only allow 1/4 of the user's RLIMIT_MEMLOCK to be used for HFI 52 52 + * caches. This fraction is then equally distributed among all 53 53 + * existing user contexts. Note that if RLIMIT_MEMLOCK is 54 54 + * 'unlimited' (-1), the value of this limit will be > 2^42 pages 55 55 + * (2^64 / 2^12 / 2^8 / 2^2). 56 56 + * 57 57 + * The effectiveness of this check may be reduced if I/O occurs on 58 58 + * some user contexts before all user contexts are created. This 59 59 + * check assumes that this process is the only one using this 60 60 + * context (e.g., the corresponding fd was not passed to another 61 61 + * process for concurrent access) as there is no per-context, 62 62 + * per-process tracking of pinned pages. It also assumes that each 63 63 + * user context has only one cache to limit. 64 64 + */ 65 65 + usr_ctxts = dd->num_rcv_contexts - dd->first_dyn_alloc_ctxt; 66 66 + if (nlocked + npages > (ulimit_pages / usr_ctxts / 4)) 67 67 + return false; 68 68 + } 51 69 52 52 - pinned = atomic64_read(&mm->pinned_vm); 53 53 - 54 54 - /* First, check the absolute limit against all pinned pages. */ 55 55 - if (pinned + npages >= ulimit && !can_lock) 70 70 + /* 71 71 + * Pinning these pages would exceed the size limit for this cache. 72 72 + */ 73 73 + cache_limit_pages = cache_size * (1024 * 1024) / PAGE_SIZE; 74 74 + if (nlocked + npages > cache_limit_pages) 56 75 return false; 57 76 58 58 - return ((nlocked + npages) <= size) || can_lock; 77 77 + return true; 59 78 } 60 79 61 80 int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t npages,

+32 -49

drivers/infiniband/hw/hfi1/verbs.c

reviewed

··· 1598 1598 "DRIVER_EgrHdrFull" 1599 1599 }; 1600 1600 1601 1601 - static DEFINE_MUTEX(cntr_names_lock); /* protects the *_cntr_names bufers */ 1602 1601 static struct rdma_stat_desc *dev_cntr_descs; 1603 1602 static struct rdma_stat_desc *port_cntr_descs; 1604 1603 int num_driver_cntrs = ARRAY_SIZE(driver_cntr_names); 1605 1604 static int num_dev_cntrs; 1606 1605 static int num_port_cntrs; 1607 1607 - static int cntr_names_initialized; 1608 1606 1609 1607 /* 1610 1608 * Convert a list of names separated by '\n' into an array of NULL terminated ··· 1613 1615 int num_extra_names, int *num_cntrs, 1614 1616 struct rdma_stat_desc **cntr_descs) 1615 1617 { 1616 1616 - struct rdma_stat_desc *q; 1617 1617 - char *names_out, *p; 1618 1618 + struct rdma_stat_desc *names_out; 1619 1619 + char *p; 1618 1620 int i, n; 1619 1621 1620 1622 n = 0; ··· 1622 1624 if (names_in[i] == '\n') 1623 1625 n++; 1624 1626 1625 1625 - names_out = 1626 1626 - kzalloc((n + num_extra_names) * sizeof(*q) + names_len, 1627 1627 - GFP_KERNEL); 1627 1627 + names_out = kzalloc((n + num_extra_names) * sizeof(*names_out) 1628 1628 + + names_len, 1629 1629 + GFP_KERNEL); 1628 1630 if (!names_out) { 1629 1631 *num_cntrs = 0; 1630 1632 *cntr_descs = NULL; 1631 1633 return -ENOMEM; 1632 1634 } 1633 1635 1634 1634 - p = names_out + (n + num_extra_names) * sizeof(*q); 1636 1636 + p = (char *)&names_out[n + num_extra_names]; 1635 1637 memcpy(p, names_in, names_len); 1636 1638 1637 1637 - q = (struct rdma_stat_desc *)names_out; 1638 1639 for (i = 0; i < n; i++) { 1639 1639 - q[i].name = p; 1640 1640 + names_out[i].name = p; 1640 1641 p = strchr(p, '\n'); 1641 1642 *p++ = '\0'; 1642 1643 } 1643 1644 1644 1645 *num_cntrs = n; 1645 1645 - *cntr_descs = (struct rdma_stat_desc *)names_out; 1646 1646 + *cntr_descs = names_out; 1646 1647 return 0; 1647 1647 - } 1648 1648 - 1649 1649 - static int init_counters(struct ib_device *ibdev) 1650 1650 - { 1651 1651 - struct hfi1_devdata *dd = dd_from_ibdev(ibdev); 1652 1652 - int i, err = 0; 1653 1653 - 1654 1654 - mutex_lock(&cntr_names_lock); 1655 1655 - if (cntr_names_initialized) 1656 1656 - goto out_unlock; 1657 1657 - 1658 1658 - err = init_cntr_names(dd->cntrnames, dd->cntrnameslen, num_driver_cntrs, 1659 1659 - &num_dev_cntrs, &dev_cntr_descs); 1660 1660 - if (err) 1661 1661 - goto out_unlock; 1662 1662 - 1663 1663 - for (i = 0; i < num_driver_cntrs; i++) 1664 1664 - dev_cntr_descs[num_dev_cntrs + i].name = driver_cntr_names[i]; 1665 1665 - 1666 1666 - err = init_cntr_names(dd->portcntrnames, dd->portcntrnameslen, 0, 1667 1667 - &num_port_cntrs, &port_cntr_descs); 1668 1668 - if (err) { 1669 1669 - kfree(dev_cntr_descs); 1670 1670 - dev_cntr_descs = NULL; 1671 1671 - goto out_unlock; 1672 1672 - } 1673 1673 - cntr_names_initialized = 1; 1674 1674 - 1675 1675 - out_unlock: 1676 1676 - mutex_unlock(&cntr_names_lock); 1677 1677 - return err; 1678 1648 } 1679 1649 1680 1650 static struct rdma_hw_stats *hfi1_alloc_hw_device_stats(struct ib_device *ibdev) 1681 1651 { 1682 1682 - if (init_counters(ibdev)) 1683 1683 - return NULL; 1652 1652 + if (!dev_cntr_descs) { 1653 1653 + struct hfi1_devdata *dd = dd_from_ibdev(ibdev); 1654 1654 + int i, err; 1655 1655 + 1656 1656 + err = init_cntr_names(dd->cntrnames, dd->cntrnameslen, 1657 1657 + num_driver_cntrs, 1658 1658 + &num_dev_cntrs, &dev_cntr_descs); 1659 1659 + if (err) 1660 1660 + return NULL; 1661 1661 + 1662 1662 + for (i = 0; i < num_driver_cntrs; i++) 1663 1663 + dev_cntr_descs[num_dev_cntrs + i].name = 1664 1664 + driver_cntr_names[i]; 1665 1665 + } 1684 1666 return rdma_alloc_hw_stats_struct(dev_cntr_descs, 1685 1667 num_dev_cntrs + num_driver_cntrs, 1686 1668 RDMA_HW_STATS_DEFAULT_LIFESPAN); ··· 1669 1691 static struct rdma_hw_stats *hfi_alloc_hw_port_stats(struct ib_device *ibdev, 1670 1692 u32 port_num) 1671 1693 { 1672 1672 - if (init_counters(ibdev)) 1673 1673 - return NULL; 1694 1694 + if (!port_cntr_descs) { 1695 1695 + struct hfi1_devdata *dd = dd_from_ibdev(ibdev); 1696 1696 + int err; 1697 1697 + 1698 1698 + err = init_cntr_names(dd->portcntrnames, dd->portcntrnameslen, 1699 1699 + 0, 1700 1700 + &num_port_cntrs, &port_cntr_descs); 1701 1701 + if (err) 1702 1702 + return NULL; 1703 1703 + } 1674 1704 return rdma_alloc_hw_stats_struct(port_cntr_descs, num_port_cntrs, 1675 1705 RDMA_HW_STATS_DEFAULT_LIFESPAN); 1676 1706 } ··· 1903 1917 del_timer_sync(&dev->mem_timer); 1904 1918 verbs_txreq_exit(dev); 1905 1919 1906 1906 - mutex_lock(&cntr_names_lock); 1907 1920 kfree(dev_cntr_descs); 1908 1921 kfree(port_cntr_descs); 1909 1922 dev_cntr_descs = NULL; 1910 1923 port_cntr_descs = NULL; 1911 1911 - cntr_names_initialized = 0; 1912 1912 - mutex_unlock(&cntr_names_lock); 1913 1924 } 1914 1925 1915 1926 void hfi1_cnp_rcv(struct hfi1_packet *packet)

+2 -17

drivers/infiniband/hw/hns/hns_roce_device.h

reviewed

··· 144 144 HNS_ROCE_CAP_FLAG_DIRECT_WQE = BIT(12), 145 145 HNS_ROCE_CAP_FLAG_SDI_MODE = BIT(14), 146 146 HNS_ROCE_CAP_FLAG_STASH = BIT(17), 147 147 + HNS_ROCE_CAP_FLAG_CQE_INLINE = BIT(19), 147 148 }; 148 149 149 150 #define HNS_ROCE_DB_TYPE_COUNT 2 ··· 568 567 569 568 struct hns_roce_dev; 570 569 571 571 - struct hns_roce_rinl_sge { 572 572 - void *addr; 573 573 - u32 len; 574 574 - }; 575 575 - 576 576 - struct hns_roce_rinl_wqe { 577 577 - struct hns_roce_rinl_sge *sg_list; 578 578 - u32 sge_cnt; 579 579 - }; 580 580 - 581 581 - struct hns_roce_rinl_buf { 582 582 - struct hns_roce_rinl_wqe *wqe_list; 583 583 - u32 wqe_cnt; 584 584 - }; 585 585 - 586 570 enum { 587 571 HNS_ROCE_FLUSH_FLAG = 0, 588 572 }; ··· 618 632 /* 0: flush needed, 1: unneeded */ 619 633 unsigned long flush_flag; 620 634 struct hns_roce_work flush_work; 621 621 - struct hns_roce_rinl_buf rq_inl_buf; 622 635 struct list_head node; /* all qps are on a list */ 623 636 struct list_head rq_node; /* all recv qps are on a list */ 624 637 struct list_head sq_node; /* all send qps are on a list */ ··· 872 887 u32 step_idx); 873 888 int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr, 874 889 int attr_mask, enum ib_qp_state cur_state, 875 875 - enum ib_qp_state new_state); 890 890 + enum ib_qp_state new_state, struct ib_udata *udata); 876 891 int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev, 877 892 struct hns_roce_qp *hr_qp); 878 893 void (*dereg_mr)(struct hns_roce_dev *hr_dev);

+33 -76

drivers/infiniband/hw/hns/hns_roce_hw_v2.c

reviewed

··· 821 821 static void fill_rq_wqe(struct hns_roce_qp *hr_qp, const struct ib_recv_wr *wr, 822 822 u32 wqe_idx, u32 max_sge) 823 823 { 824 824 - struct hns_roce_rinl_sge *sge_list; 825 824 void *wqe = NULL; 826 826 - u32 i; 827 825 828 826 wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx); 829 827 fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp->rq.rsv_sge); 830 830 - 831 831 - /* rq support inline data */ 832 832 - if (hr_qp->rq_inl_buf.wqe_cnt) { 833 833 - sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list; 834 834 - hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt = (u32)wr->num_sge; 835 835 - for (i = 0; i < wr->num_sge; i++) { 836 836 - sge_list[i].addr = (void *)(u64)wr->sg_list[i].addr; 837 837 - sge_list[i].len = wr->sg_list[i].length; 838 838 - } 839 839 - } 840 828 } 841 829 842 830 static int hns_roce_v2_post_recv(struct ib_qp *ibqp, ··· 2837 2849 attr->port_num = 1; 2838 2850 attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE; 2839 2851 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT, 2840 2840 - IB_QPS_INIT); 2852 2852 + IB_QPS_INIT, NULL); 2841 2853 if (ret) { 2842 2854 ibdev_err(ibdev, "failed to modify qp to init, ret = %d.\n", 2843 2855 ret); ··· 2859 2871 rdma_ah_set_sl(&attr->ah_attr, (u8)sl_num); 2860 2872 2861 2873 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT, 2862 2862 - IB_QPS_RTR); 2874 2874 + IB_QPS_RTR, NULL); 2863 2875 hr_dev->loop_idc = loopback; 2864 2876 if (ret) { 2865 2877 ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n", ··· 2874 2886 attr->retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT; 2875 2887 attr->timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT; 2876 2888 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_RTR, 2877 2877 - IB_QPS_RTS); 2889 2889 + IB_QPS_RTS, NULL); 2878 2890 if (ret) 2879 2891 ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n", 2880 2892 ret); ··· 3718 3730 return 0; 3719 3731 } 3720 3732 3721 3721 - static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, 3722 3722 - struct hns_roce_qp *qp, 3723 3723 - struct ib_wc *wc) 3724 3724 - { 3725 3725 - struct hns_roce_rinl_sge *sge_list; 3726 3726 - u32 wr_num, wr_cnt, sge_num; 3727 3727 - u32 sge_cnt, data_len, size; 3728 3728 - void *wqe_buf; 3729 3729 - 3730 3730 - wr_num = hr_reg_read(cqe, CQE_WQE_IDX); 3731 3731 - wr_cnt = wr_num & (qp->rq.wqe_cnt - 1); 3732 3732 - 3733 3733 - sge_list = qp->rq_inl_buf.wqe_list[wr_cnt].sg_list; 3734 3734 - sge_num = qp->rq_inl_buf.wqe_list[wr_cnt].sge_cnt; 3735 3735 - wqe_buf = hns_roce_get_recv_wqe(qp, wr_cnt); 3736 3736 - data_len = wc->byte_len; 3737 3737 - 3738 3738 - for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) { 3739 3739 - size = min(sge_list[sge_cnt].len, data_len); 3740 3740 - memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size); 3741 3741 - 3742 3742 - data_len -= size; 3743 3743 - wqe_buf += size; 3744 3744 - } 3745 3745 - 3746 3746 - if (unlikely(data_len)) { 3747 3747 - wc->status = IB_WC_LOC_LEN_ERR; 3748 3748 - return -EAGAIN; 3749 3749 - } 3750 3750 - 3751 3751 - return 0; 3752 3752 - } 3753 3753 - 3754 3733 static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq, 3755 3734 int num_entries, struct ib_wc *wc) 3756 3735 { ··· 3929 3974 wc->opcode = ib_opcode; 3930 3975 } 3931 3976 3932 3932 - static inline bool is_rq_inl_enabled(struct ib_wc *wc, u32 hr_opcode, 3933 3933 - struct hns_roce_v2_cqe *cqe) 3934 3934 - { 3935 3935 - return wc->qp->qp_type != IB_QPT_UD && wc->qp->qp_type != IB_QPT_GSI && 3936 3936 - (hr_opcode == HNS_ROCE_V2_OPCODE_SEND || 3937 3937 - hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM || 3938 3938 - hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) && 3939 3939 - hr_reg_read(cqe, CQE_RQ_INLINE); 3940 3940 - } 3941 3941 - 3942 3977 static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe) 3943 3978 { 3944 3944 - struct hns_roce_qp *qp = to_hr_qp(wc->qp); 3945 3979 u32 hr_opcode; 3946 3980 int ib_opcode; 3947 3947 - int ret; 3948 3981 3949 3982 wc->byte_len = le32_to_cpu(cqe->byte_cnt); 3950 3983 ··· 3956 4013 wc->status = IB_WC_GENERAL_ERR; 3957 4014 else 3958 4015 wc->opcode = ib_opcode; 3959 3959 - 3960 3960 - if (is_rq_inl_enabled(wc, hr_opcode, cqe)) { 3961 3961 - ret = hns_roce_handle_recv_inl_wqe(cqe, qp, wc); 3962 3962 - if (unlikely(ret)) 3963 3963 - return ret; 3964 3964 - } 3965 4016 3966 4017 wc->sl = hr_reg_read(cqe, CQE_SL); 3967 4018 wc->src_qp = hr_reg_read(cqe, CQE_RMT_QPN); ··· 4382 4445 hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_H, 4383 4446 upper_32_bits(hr_qp->rdb.dma)); 4384 4447 4385 4385 - if (ibqp->qp_type != IB_QPT_UD && ibqp->qp_type != IB_QPT_GSI) 4386 4386 - hr_reg_write_bool(context, QPC_RQIE, 4387 4387 - hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE); 4388 4388 - 4389 4448 hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq)); 4390 4449 4391 4450 if (ibqp->srq) { ··· 4572 4639 static int modify_qp_init_to_rtr(struct ib_qp *ibqp, 4573 4640 const struct ib_qp_attr *attr, int attr_mask, 4574 4641 struct hns_roce_v2_qp_context *context, 4575 4575 - struct hns_roce_v2_qp_context *qpc_mask) 4642 4642 + struct hns_roce_v2_qp_context *qpc_mask, 4643 4643 + struct ib_udata *udata) 4576 4644 { 4645 4645 + struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata, 4646 4646 + struct hns_roce_ucontext, ibucontext); 4577 4647 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4578 4648 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4579 4649 struct ib_device *ibdev = &hr_dev->ib_dev; ··· 4695 4759 /* rocee send 2^lp_sgen_ini segs every time */ 4696 4760 hr_reg_write(context, QPC_LP_SGEN_INI, 3); 4697 4761 hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI); 4762 4762 + 4763 4763 + if (udata && ibqp->qp_type == IB_QPT_RC && 4764 4764 + (uctx->config & HNS_ROCE_RQ_INLINE_FLAGS)) { 4765 4765 + hr_reg_write_bool(context, QPC_RQIE, 4766 4766 + hr_dev->caps.flags & 4767 4767 + HNS_ROCE_CAP_FLAG_RQ_INLINE); 4768 4768 + hr_reg_clear(qpc_mask, QPC_RQIE); 4769 4769 + } 4770 4770 + 4771 4771 + if (udata && 4772 4772 + (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_XRC_TGT) && 4773 4773 + (uctx->config & HNS_ROCE_CQE_INLINE_FLAGS)) { 4774 4774 + hr_reg_write_bool(context, QPC_CQEIE, 4775 4775 + hr_dev->caps.flags & 4776 4776 + HNS_ROCE_CAP_FLAG_CQE_INLINE); 4777 4777 + hr_reg_clear(qpc_mask, QPC_CQEIE); 4778 4778 + 4779 4779 + hr_reg_write(context, QPC_CQEIS, 0); 4780 4780 + hr_reg_clear(qpc_mask, QPC_CQEIS); 4781 4781 + } 4698 4782 4699 4783 return 0; 4700 4784 } ··· 5063 5107 enum ib_qp_state cur_state, 5064 5108 enum ib_qp_state new_state, 5065 5109 struct hns_roce_v2_qp_context *context, 5066 5066 - struct hns_roce_v2_qp_context *qpc_mask) 5110 5110 + struct hns_roce_v2_qp_context *qpc_mask, 5111 5111 + struct ib_udata *udata) 5067 5112 { 5068 5113 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5069 5114 int ret = 0; ··· 5081 5124 modify_qp_init_to_init(ibqp, attr, context, qpc_mask); 5082 5125 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { 5083 5126 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context, 5084 5084 - qpc_mask); 5127 5127 + qpc_mask, udata); 5085 5128 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) { 5086 5129 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context, 5087 5130 qpc_mask); ··· 5286 5329 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, 5287 5330 const struct ib_qp_attr *attr, 5288 5331 int attr_mask, enum ib_qp_state cur_state, 5289 5289 - enum ib_qp_state new_state) 5332 5332 + enum ib_qp_state new_state, struct ib_udata *udata) 5290 5333 { 5291 5334 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5292 5335 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); ··· 5309 5352 memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz); 5310 5353 5311 5354 ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state, 5312 5312 - new_state, context, qpc_mask); 5355 5355 + new_state, context, qpc_mask, udata); 5313 5356 if (ret) 5314 5357 goto out; 5315 5358 ··· 5512 5555 if (modify_qp_is_ok(hr_qp)) { 5513 5556 /* Modify qp to reset before destroying qp */ 5514 5557 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0, 5515 5515 - hr_qp->state, IB_QPS_RESET); 5558 5558 + hr_qp->state, IB_QPS_RESET, udata); 5516 5559 if (ret) 5517 5560 ibdev_err(ibdev, 5518 5561 "failed to modify QP to RST, ret = %d.\n",

+2 -1

drivers/infiniband/hw/hns/hns_roce_hw_v2.h

reviewed

··· 531 531 #define QPC_RQ_RTY_TX_ERR QPC_FIELD_LOC(607, 607) 532 532 #define QPC_RX_CQN QPC_FIELD_LOC(631, 608) 533 533 #define QPC_XRC_QP_TYPE QPC_FIELD_LOC(632, 632) 534 534 - #define QPC_RSV3 QPC_FIELD_LOC(634, 633) 534 534 + #define QPC_CQEIE QPC_FIELD_LOC(633, 633) 535 535 + #define QPC_CQEIS QPC_FIELD_LOC(634, 634) 535 536 #define QPC_MIN_RNR_TIME QPC_FIELD_LOC(639, 635) 536 537 #define QPC_RQ_PRODUCER_IDX QPC_FIELD_LOC(655, 640) 537 538 #define QPC_RQ_CONSUMER_IDX QPC_FIELD_LOC(671, 656)

+15 -2

drivers/infiniband/hw/hns/hns_roce_main.c

reviewed

··· 379 379 resp.max_inline_data = hr_dev->caps.max_sq_inline; 380 380 } 381 381 382 382 + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) { 383 383 + context->config |= ucmd.config & HNS_ROCE_RQ_INLINE_FLAGS; 384 384 + if (context->config & HNS_ROCE_RQ_INLINE_FLAGS) 385 385 + resp.config |= HNS_ROCE_RSP_RQ_INLINE_FLAGS; 386 386 + } 387 387 + 388 388 + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQE_INLINE) { 389 389 + context->config |= ucmd.config & HNS_ROCE_CQE_INLINE_FLAGS; 390 390 + if (context->config & HNS_ROCE_CQE_INLINE_FLAGS) 391 391 + resp.config |= HNS_ROCE_RSP_CQE_INLINE_FLAGS; 392 392 + } 393 393 + 382 394 ret = hns_roce_uar_alloc(hr_dev, &context->uar); 383 395 if (ret) 384 396 goto error_fail_uar_alloc; ··· 455 443 prot = pgprot_device(vma->vm_page_prot); 456 444 break; 457 445 default: 458 458 - return -EINVAL; 446 446 + ret = -EINVAL; 447 447 + goto out; 459 448 } 460 449 461 450 ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE, 462 451 prot, rdma_entry); 463 452 453 453 + out: 464 454 rdma_user_mmap_entry_put(rdma_entry); 465 465 - 466 455 return ret; 467 456 } 468 457

+1 -65

drivers/infiniband/hw/hns/hns_roce_qp.c

reviewed

··· 433 433 if (!has_rq) { 434 434 hr_qp->rq.wqe_cnt = 0; 435 435 hr_qp->rq.max_gs = 0; 436 436 - hr_qp->rq_inl_buf.wqe_cnt = 0; 437 436 cap->max_recv_wr = 0; 438 437 cap->max_recv_sge = 0; 439 438 ··· 462 463 hr_qp->rq.max_gs); 463 464 464 465 hr_qp->rq.wqe_cnt = cnt; 465 465 - if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE && 466 466 - hr_qp->ibqp.qp_type != IB_QPT_UD && 467 467 - hr_qp->ibqp.qp_type != IB_QPT_GSI) 468 468 - hr_qp->rq_inl_buf.wqe_cnt = cnt; 469 469 - else 470 470 - hr_qp->rq_inl_buf.wqe_cnt = 0; 471 466 472 467 cap->max_recv_wr = cnt; 473 468 cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; ··· 725 732 return 1; 726 733 } 727 734 728 728 - static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp, 729 729 - struct ib_qp_init_attr *init_attr) 730 730 - { 731 731 - u32 max_recv_sge = init_attr->cap.max_recv_sge; 732 732 - u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt; 733 733 - struct hns_roce_rinl_wqe *wqe_list; 734 734 - int i; 735 735 - 736 736 - /* allocate recv inline buf */ 737 737 - wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe), 738 738 - GFP_KERNEL); 739 739 - if (!wqe_list) 740 740 - goto err; 741 741 - 742 742 - /* Allocate a continuous buffer for all inline sge we need */ 743 743 - wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge * 744 744 - sizeof(struct hns_roce_rinl_sge)), 745 745 - GFP_KERNEL); 746 746 - if (!wqe_list[0].sg_list) 747 747 - goto err_wqe_list; 748 748 - 749 749 - /* Assign buffers of sg_list to each inline wqe */ 750 750 - for (i = 1; i < wqe_cnt; i++) 751 751 - wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge]; 752 752 - 753 753 - hr_qp->rq_inl_buf.wqe_list = wqe_list; 754 754 - 755 755 - return 0; 756 756 - 757 757 - err_wqe_list: 758 758 - kfree(wqe_list); 759 759 - 760 760 - err: 761 761 - return -ENOMEM; 762 762 - } 763 763 - 764 764 - static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) 765 765 - { 766 766 - if (hr_qp->rq_inl_buf.wqe_list) 767 767 - kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); 768 768 - kfree(hr_qp->rq_inl_buf.wqe_list); 769 769 - } 770 770 - 771 735 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 772 736 struct ib_qp_init_attr *init_attr, 773 737 struct ib_udata *udata, unsigned long addr) ··· 732 782 struct ib_device *ibdev = &hr_dev->ib_dev; 733 783 struct hns_roce_buf_attr buf_attr = {}; 734 784 int ret; 735 735 - 736 736 - if (!udata && hr_qp->rq_inl_buf.wqe_cnt) { 737 737 - ret = alloc_rq_inline_buf(hr_qp, init_attr); 738 738 - if (ret) { 739 739 - ibdev_err(ibdev, 740 740 - "failed to alloc inline buf, ret = %d.\n", 741 741 - ret); 742 742 - return ret; 743 743 - } 744 744 - } else { 745 745 - hr_qp->rq_inl_buf.wqe_list = NULL; 746 746 - } 747 785 748 786 ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr); 749 787 if (ret) { ··· 752 814 return 0; 753 815 754 816 err_inline: 755 755 - free_rq_inline_buf(hr_qp); 756 817 757 818 return ret; 758 819 } ··· 759 822 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 760 823 { 761 824 hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr); 762 762 - free_rq_inline_buf(hr_qp); 763 825 } 764 826 765 827 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev, ··· 1346 1410 goto out; 1347 1411 1348 1412 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, 1349 1349 - new_state); 1413 1413 + new_state, udata); 1350 1414 1351 1415 out: 1352 1416 mutex_unlock(&hr_qp->mutex);

+2

drivers/infiniband/hw/irdma/hw.c

reviewed

··· 483 483 iw_qvlist->num_vectors = rf->msix_count; 484 484 if (rf->msix_count <= num_online_cpus()) 485 485 rf->msix_shared = true; 486 486 + else if (rf->msix_count > num_online_cpus() + 1) 487 487 + rf->msix_count = num_online_cpus() + 1; 486 488 487 489 pmsix = rf->msix_entries; 488 490 for (i = 0, ceq_idx = 0; i < rf->msix_count; i++, iw_qvinfo++) {

+210 -103

drivers/infiniband/hw/irdma/verbs.c

reviewed

··· 2745 2745 return ret; 2746 2746 } 2747 2747 2748 2748 + static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access) 2749 2749 + { 2750 2750 + struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 2751 2751 + struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2752 2752 + bool use_pbles; 2753 2753 + u32 stag; 2754 2754 + int err; 2755 2755 + 2756 2756 + use_pbles = iwmr->page_cnt != 1; 2757 2757 + 2758 2758 + err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false); 2759 2759 + if (err) 2760 2760 + return err; 2761 2761 + 2762 2762 + if (use_pbles) { 2763 2763 + err = irdma_check_mr_contiguous(&iwpbl->pble_alloc, 2764 2764 + iwmr->page_size); 2765 2765 + if (err) { 2766 2766 + irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 2767 2767 + iwpbl->pbl_allocated = false; 2768 2768 + } 2769 2769 + } 2770 2770 + 2771 2771 + stag = irdma_create_stag(iwdev); 2772 2772 + if (!stag) { 2773 2773 + err = -ENOMEM; 2774 2774 + goto free_pble; 2775 2775 + } 2776 2776 + 2777 2777 + iwmr->stag = stag; 2778 2778 + iwmr->ibmr.rkey = stag; 2779 2779 + iwmr->ibmr.lkey = stag; 2780 2780 + err = irdma_hwreg_mr(iwdev, iwmr, access); 2781 2781 + if (err) 2782 2782 + goto err_hwreg; 2783 2783 + 2784 2784 + return 0; 2785 2785 + 2786 2786 + err_hwreg: 2787 2787 + irdma_free_stag(iwdev, stag); 2788 2788 + 2789 2789 + free_pble: 2790 2790 + if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) 2791 2791 + irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 2792 2792 + 2793 2793 + return err; 2794 2794 + } 2795 2795 + 2796 2796 + static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, 2797 2797 + struct ib_pd *pd, u64 virt, 2798 2798 + enum irdma_memreg_type reg_type) 2799 2799 + { 2800 2800 + struct irdma_device *iwdev = to_iwdev(pd->device); 2801 2801 + struct irdma_pbl *iwpbl = NULL; 2802 2802 + struct irdma_mr *iwmr = NULL; 2803 2803 + unsigned long pgsz_bitmap; 2804 2804 + 2805 2805 + iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2806 2806 + if (!iwmr) 2807 2807 + return ERR_PTR(-ENOMEM); 2808 2808 + 2809 2809 + iwpbl = &iwmr->iwpbl; 2810 2810 + iwpbl->iwmr = iwmr; 2811 2811 + iwmr->region = region; 2812 2812 + iwmr->ibmr.pd = pd; 2813 2813 + iwmr->ibmr.device = pd->device; 2814 2814 + iwmr->ibmr.iova = virt; 2815 2815 + iwmr->type = reg_type; 2816 2816 + 2817 2817 + pgsz_bitmap = (reg_type == IRDMA_MEMREG_TYPE_MEM) ? 2818 2818 + iwdev->rf->sc_dev.hw_attrs.page_size_cap : PAGE_SIZE; 2819 2819 + 2820 2820 + iwmr->page_size = ib_umem_find_best_pgsz(region, pgsz_bitmap, virt); 2821 2821 + if (unlikely(!iwmr->page_size)) { 2822 2822 + kfree(iwmr); 2823 2823 + return ERR_PTR(-EOPNOTSUPP); 2824 2824 + } 2825 2825 + 2826 2826 + iwmr->len = region->length; 2827 2827 + iwpbl->user_base = virt; 2828 2828 + iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); 2829 2829 + 2830 2830 + return iwmr; 2831 2831 + } 2832 2832 + 2833 2833 + static void irdma_free_iwmr(struct irdma_mr *iwmr) 2834 2834 + { 2835 2835 + kfree(iwmr); 2836 2836 + } 2837 2837 + 2838 2838 + static int irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req, 2839 2839 + struct ib_udata *udata, 2840 2840 + struct irdma_mr *iwmr) 2841 2841 + { 2842 2842 + struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 2843 2843 + struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2844 2844 + struct irdma_ucontext *ucontext = NULL; 2845 2845 + unsigned long flags; 2846 2846 + bool use_pbles; 2847 2847 + u32 total; 2848 2848 + int err; 2849 2849 + 2850 2850 + total = req.sq_pages + req.rq_pages + 1; 2851 2851 + if (total > iwmr->page_cnt) 2852 2852 + return -EINVAL; 2853 2853 + 2854 2854 + total = req.sq_pages + req.rq_pages; 2855 2855 + use_pbles = total > 2; 2856 2856 + err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2857 2857 + if (err) 2858 2858 + return err; 2859 2859 + 2860 2860 + ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2861 2861 + ibucontext); 2862 2862 + spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 2863 2863 + list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); 2864 2864 + iwpbl->on_list = true; 2865 2865 + spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 2866 2866 + 2867 2867 + return 0; 2868 2868 + } 2869 2869 + 2870 2870 + static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req, 2871 2871 + struct ib_udata *udata, 2872 2872 + struct irdma_mr *iwmr) 2873 2873 + { 2874 2874 + struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 2875 2875 + struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2876 2876 + struct irdma_ucontext *ucontext = NULL; 2877 2877 + u8 shadow_pgcnt = 1; 2878 2878 + unsigned long flags; 2879 2879 + bool use_pbles; 2880 2880 + u32 total; 2881 2881 + int err; 2882 2882 + 2883 2883 + if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) 2884 2884 + shadow_pgcnt = 0; 2885 2885 + total = req.cq_pages + shadow_pgcnt; 2886 2886 + if (total > iwmr->page_cnt) 2887 2887 + return -EINVAL; 2888 2888 + 2889 2889 + use_pbles = req.cq_pages > 1; 2890 2890 + err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2891 2891 + if (err) 2892 2892 + return err; 2893 2893 + 2894 2894 + ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2895 2895 + ibucontext); 2896 2896 + spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2897 2897 + list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); 2898 2898 + iwpbl->on_list = true; 2899 2899 + spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2900 2900 + 2901 2901 + return 0; 2902 2902 + } 2903 2903 + 2748 2904 /** 2749 2905 * irdma_reg_user_mr - Register a user memory region 2750 2906 * @pd: ptr of pd ··· 2916 2760 { 2917 2761 #define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages) 2918 2762 struct irdma_device *iwdev = to_iwdev(pd->device); 2919 2919 - struct irdma_ucontext *ucontext; 2920 2920 - struct irdma_pble_alloc *palloc; 2921 2921 - struct irdma_pbl *iwpbl; 2922 2922 - struct irdma_mr *iwmr; 2923 2923 - struct ib_umem *region; 2924 2924 - struct irdma_mem_reg_req req; 2925 2925 - u32 total, stag = 0; 2926 2926 - u8 shadow_pgcnt = 1; 2927 2927 - bool use_pbles = false; 2928 2928 - unsigned long flags; 2929 2929 - int err = -EINVAL; 2930 2930 - int ret; 2763 2763 + struct irdma_mem_reg_req req = {}; 2764 2764 + struct ib_umem *region = NULL; 2765 2765 + struct irdma_mr *iwmr = NULL; 2766 2766 + int err; 2931 2767 2932 2768 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 2933 2769 return ERR_PTR(-EINVAL); ··· 2940 2792 return ERR_PTR(-EFAULT); 2941 2793 } 2942 2794 2943 2943 - iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2944 2944 - if (!iwmr) { 2795 2795 + iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type); 2796 2796 + if (IS_ERR(iwmr)) { 2945 2797 ib_umem_release(region); 2946 2946 - return ERR_PTR(-ENOMEM); 2798 2798 + return (struct ib_mr *)iwmr; 2947 2799 } 2948 2948 - 2949 2949 - iwpbl = &iwmr->iwpbl; 2950 2950 - iwpbl->iwmr = iwmr; 2951 2951 - iwmr->region = region; 2952 2952 - iwmr->ibmr.pd = pd; 2953 2953 - iwmr->ibmr.device = pd->device; 2954 2954 - iwmr->ibmr.iova = virt; 2955 2955 - iwmr->page_size = PAGE_SIZE; 2956 2956 - 2957 2957 - if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) { 2958 2958 - iwmr->page_size = ib_umem_find_best_pgsz(region, 2959 2959 - iwdev->rf->sc_dev.hw_attrs.page_size_cap, 2960 2960 - virt); 2961 2961 - if (unlikely(!iwmr->page_size)) { 2962 2962 - kfree(iwmr); 2963 2963 - ib_umem_release(region); 2964 2964 - return ERR_PTR(-EOPNOTSUPP); 2965 2965 - } 2966 2966 - } 2967 2967 - iwmr->len = region->length; 2968 2968 - iwpbl->user_base = virt; 2969 2969 - palloc = &iwpbl->pble_alloc; 2970 2970 - iwmr->type = req.reg_type; 2971 2971 - iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); 2972 2800 2973 2801 switch (req.reg_type) { 2974 2802 case IRDMA_MEMREG_TYPE_QP: 2975 2975 - total = req.sq_pages + req.rq_pages + shadow_pgcnt; 2976 2976 - if (total > iwmr->page_cnt) { 2977 2977 - err = -EINVAL; 2978 2978 - goto error; 2979 2979 - } 2980 2980 - total = req.sq_pages + req.rq_pages; 2981 2981 - use_pbles = (total > 2); 2982 2982 - err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2803 2803 + err = irdma_reg_user_mr_type_qp(req, udata, iwmr); 2983 2804 if (err) 2984 2805 goto error; 2985 2806 2986 2986 - ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2987 2987 - ibucontext); 2988 2988 - spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 2989 2989 - list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); 2990 2990 - iwpbl->on_list = true; 2991 2991 - spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 2992 2807 break; 2993 2808 case IRDMA_MEMREG_TYPE_CQ: 2994 2994 - if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) 2995 2995 - shadow_pgcnt = 0; 2996 2996 - total = req.cq_pages + shadow_pgcnt; 2997 2997 - if (total > iwmr->page_cnt) { 2998 2998 - err = -EINVAL; 2999 2999 - goto error; 3000 3000 - } 3001 3001 - 3002 3002 - use_pbles = (req.cq_pages > 1); 3003 3003 - err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2809 2809 + err = irdma_reg_user_mr_type_cq(req, udata, iwmr); 3004 2810 if (err) 3005 2811 goto error; 3006 3006 - 3007 3007 - ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 3008 3008 - ibucontext); 3009 3009 - spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 3010 3010 - list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); 3011 3011 - iwpbl->on_list = true; 3012 3012 - spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 3013 2812 break; 3014 2813 case IRDMA_MEMREG_TYPE_MEM: 3015 3015 - use_pbles = (iwmr->page_cnt != 1); 3016 3016 - 3017 3017 - err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false); 2814 2814 + err = irdma_reg_user_mr_type_mem(iwmr, access); 3018 2815 if (err) 3019 2816 goto error; 3020 3020 - 3021 3021 - if (use_pbles) { 3022 3022 - ret = irdma_check_mr_contiguous(palloc, 3023 3023 - iwmr->page_size); 3024 3024 - if (ret) { 3025 3025 - irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 3026 3026 - iwpbl->pbl_allocated = false; 3027 3027 - } 3028 3028 - } 3029 3029 - 3030 3030 - stag = irdma_create_stag(iwdev); 3031 3031 - if (!stag) { 3032 3032 - err = -ENOMEM; 3033 3033 - goto error; 3034 3034 - } 3035 3035 - 3036 3036 - iwmr->stag = stag; 3037 3037 - iwmr->ibmr.rkey = stag; 3038 3038 - iwmr->ibmr.lkey = stag; 3039 3039 - err = irdma_hwreg_mr(iwdev, iwmr, access); 3040 3040 - if (err) { 3041 3041 - irdma_free_stag(iwdev, stag); 3042 3042 - goto error; 3043 3043 - } 3044 2817 3045 2818 break; 3046 2819 default: 2820 2820 + err = -EINVAL; 3047 2821 goto error; 3048 2822 } 3049 2823 3050 3050 - iwmr->type = req.reg_type; 2824 2824 + return &iwmr->ibmr; 2825 2825 + error: 2826 2826 + ib_umem_release(region); 2827 2827 + irdma_free_iwmr(iwmr); 2828 2828 + 2829 2829 + return ERR_PTR(err); 2830 2830 + } 2831 2831 + 2832 2832 + static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start, 2833 2833 + u64 len, u64 virt, 2834 2834 + int fd, int access, 2835 2835 + struct ib_udata *udata) 2836 2836 + { 2837 2837 + struct irdma_device *iwdev = to_iwdev(pd->device); 2838 2838 + struct ib_umem_dmabuf *umem_dmabuf; 2839 2839 + struct irdma_mr *iwmr; 2840 2840 + int err; 2841 2841 + 2842 2842 + if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 2843 2843 + return ERR_PTR(-EINVAL); 2844 2844 + 2845 2845 + umem_dmabuf = ib_umem_dmabuf_get_pinned(pd->device, start, len, fd, access); 2846 2846 + if (IS_ERR(umem_dmabuf)) { 2847 2847 + err = PTR_ERR(umem_dmabuf); 2848 2848 + ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%d]\n", err); 2849 2849 + return ERR_PTR(err); 2850 2850 + } 2851 2851 + 2852 2852 + iwmr = irdma_alloc_iwmr(&umem_dmabuf->umem, pd, virt, IRDMA_MEMREG_TYPE_MEM); 2853 2853 + if (IS_ERR(iwmr)) { 2854 2854 + err = PTR_ERR(iwmr); 2855 2855 + goto err_release; 2856 2856 + } 2857 2857 + 2858 2858 + err = irdma_reg_user_mr_type_mem(iwmr, access); 2859 2859 + if (err) 2860 2860 + goto err_iwmr; 3051 2861 3052 2862 return &iwmr->ibmr; 3053 2863 3054 3054 - error: 3055 3055 - if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) 3056 3056 - irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 3057 3057 - ib_umem_release(region); 3058 3058 - kfree(iwmr); 2864 2864 + err_iwmr: 2865 2865 + irdma_free_iwmr(iwmr); 2866 2866 + 2867 2867 + err_release: 2868 2868 + ib_umem_release(&umem_dmabuf->umem); 3059 2869 3060 2870 return ERR_PTR(err); 3061 2871 } ··· 4524 4418 .query_port = irdma_query_port, 4525 4419 .query_qp = irdma_query_qp, 4526 4420 .reg_user_mr = irdma_reg_user_mr, 4421 4421 + .reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf, 4527 4422 .req_notify_cq = irdma_req_notify_cq, 4528 4423 .resize_cq = irdma_resize_cq, 4529 4424 INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd),

+12 -10

drivers/infiniband/hw/mana/main.c

reviewed

··· 249 249 mana_ib_gd_first_dma_region(struct mana_ib_dev *dev, 250 250 struct gdma_context *gc, 251 251 struct gdma_create_dma_region_req *create_req, 252 252 - size_t num_pages, mana_handle_t *gdma_region) 252 252 + size_t num_pages, mana_handle_t *gdma_region, 253 253 + u32 expected_status) 253 254 { 254 255 struct gdma_create_dma_region_resp create_resp = {}; 255 256 unsigned int create_req_msg_size; ··· 262 261 263 262 err = mana_gd_send_request(gc, create_req_msg_size, create_req, 264 263 sizeof(create_resp), &create_resp); 265 265 - if (err || create_resp.hdr.status) { 264 264 + if (err || create_resp.hdr.status != expected_status) { 266 265 ibdev_dbg(&dev->ib_dev, 267 266 "Failed to create DMA region: %d, 0x%x\n", 268 267 err, create_resp.hdr.status); ··· 373 372 374 373 page_addr_list = create_req->page_addr_list; 375 374 rdma_umem_for_each_dma_block(umem, &biter, page_sz) { 375 375 + u32 expected_status = 0; 376 376 + 376 377 page_addr_list[tail++] = rdma_block_iter_dma_address(&biter); 377 378 if (tail < num_pages_to_handle) 378 379 continue; 379 380 381 381 + if (num_pages_processed + num_pages_to_handle < 382 382 + num_pages_total) 383 383 + expected_status = GDMA_STATUS_MORE_ENTRIES; 384 384 + 380 385 if (!num_pages_processed) { 381 386 /* First create message */ 382 387 err = mana_ib_gd_first_dma_region(dev, gc, create_req, 383 383 - tail, gdma_region); 388 388 + tail, gdma_region, 389 389 + expected_status); 384 390 if (err) 385 391 goto out; 386 392 ··· 400 392 page_addr_list = add_req->page_addr_list; 401 393 } else { 402 394 /* Subsequent create messages */ 403 403 - u32 expected_s = 0; 404 404 - 405 405 - if (num_pages_processed + num_pages_to_handle < 406 406 - num_pages_total) 407 407 - expected_s = GDMA_STATUS_MORE_ENTRIES; 408 408 - 409 395 err = mana_ib_gd_add_dma_region(dev, gc, add_req, tail, 410 410 - expected_s); 396 396 + expected_status); 411 397 if (err) 412 398 break; 413 399 }

+8

drivers/infiniband/hw/mlx4/main.c

reviewed

··· 3303 3303 if (!wq) 3304 3304 return -ENOMEM; 3305 3305 3306 3306 + err = mlx4_ib_qp_event_init(); 3307 3307 + if (err) 3308 3308 + goto clean_qp_event; 3309 3309 + 3306 3310 err = mlx4_ib_cm_init(); 3307 3311 if (err) 3308 3312 goto clean_wq; ··· 3328 3324 mlx4_ib_cm_destroy(); 3329 3325 3330 3326 clean_wq: 3327 3327 + mlx4_ib_qp_event_cleanup(); 3328 3328 + 3329 3329 + clean_qp_event: 3331 3330 destroy_workqueue(wq); 3332 3331 return err; 3333 3332 } ··· 3340 3333 mlx4_unregister_interface(&mlx4_ib_interface); 3341 3334 mlx4_ib_mcg_destroy(); 3342 3335 mlx4_ib_cm_destroy(); 3336 3336 + mlx4_ib_qp_event_cleanup(); 3343 3337 destroy_workqueue(wq); 3344 3338 } 3345 3339

+3

drivers/infiniband/hw/mlx4/mlx4_ib.h

reviewed

··· 940 940 int mlx4_ib_cm_init(void); 941 941 void mlx4_ib_cm_destroy(void); 942 942 943 943 + int mlx4_ib_qp_event_init(void); 944 944 + void mlx4_ib_qp_event_cleanup(void); 945 945 + 943 946 #endif /* MLX4_IB_H */

+85 -36

drivers/infiniband/hw/mlx4/qp.c

reviewed

··· 102 102 MLX4_IB_RWQ_SRC = 1, 103 103 }; 104 104 105 105 + struct mlx4_ib_qp_event_work { 106 106 + struct work_struct work; 107 107 + struct mlx4_qp *qp; 108 108 + enum mlx4_event type; 109 109 + }; 110 110 + 111 111 + static struct workqueue_struct *mlx4_ib_qp_event_wq; 112 112 + 105 113 static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) 106 114 { 107 115 if (!mlx4_is_master(dev->dev)) ··· 208 200 } 209 201 } 210 202 203 203 + static void mlx4_ib_handle_qp_event(struct work_struct *_work) 204 204 + { 205 205 + struct mlx4_ib_qp_event_work *qpe_work = 206 206 + container_of(_work, struct mlx4_ib_qp_event_work, work); 207 207 + struct ib_qp *ibqp = &to_mibqp(qpe_work->qp)->ibqp; 208 208 + struct ib_event event = {}; 209 209 + 210 210 + event.device = ibqp->device; 211 211 + event.element.qp = ibqp; 212 212 + 213 213 + switch (qpe_work->type) { 214 214 + case MLX4_EVENT_TYPE_PATH_MIG: 215 215 + event.event = IB_EVENT_PATH_MIG; 216 216 + break; 217 217 + case MLX4_EVENT_TYPE_COMM_EST: 218 218 + event.event = IB_EVENT_COMM_EST; 219 219 + break; 220 220 + case MLX4_EVENT_TYPE_SQ_DRAINED: 221 221 + event.event = IB_EVENT_SQ_DRAINED; 222 222 + break; 223 223 + case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: 224 224 + event.event = IB_EVENT_QP_LAST_WQE_REACHED; 225 225 + break; 226 226 + case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: 227 227 + event.event = IB_EVENT_QP_FATAL; 228 228 + break; 229 229 + case MLX4_EVENT_TYPE_PATH_MIG_FAILED: 230 230 + event.event = IB_EVENT_PATH_MIG_ERR; 231 231 + break; 232 232 + case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: 233 233 + event.event = IB_EVENT_QP_REQ_ERR; 234 234 + break; 235 235 + case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: 236 236 + event.event = IB_EVENT_QP_ACCESS_ERR; 237 237 + break; 238 238 + default: 239 239 + pr_warn("Unexpected event type %d on QP %06x\n", 240 240 + qpe_work->type, qpe_work->qp->qpn); 241 241 + goto out; 242 242 + } 243 243 + 244 244 + ibqp->event_handler(&event, ibqp->qp_context); 245 245 + 246 246 + out: 247 247 + mlx4_put_qp(qpe_work->qp); 248 248 + kfree(qpe_work); 249 249 + } 250 250 + 211 251 static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type) 212 252 { 213 213 - struct ib_event event; 214 253 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp; 254 254 + struct mlx4_ib_qp_event_work *qpe_work; 215 255 216 256 if (type == MLX4_EVENT_TYPE_PATH_MIG) 217 257 to_mibqp(qp)->port = to_mibqp(qp)->alt_port; 218 258 219 219 - if (ibqp->event_handler) { 220 220 - event.device = ibqp->device; 221 221 - event.element.qp = ibqp; 222 222 - switch (type) { 223 223 - case MLX4_EVENT_TYPE_PATH_MIG: 224 224 - event.event = IB_EVENT_PATH_MIG; 225 225 - break; 226 226 - case MLX4_EVENT_TYPE_COMM_EST: 227 227 - event.event = IB_EVENT_COMM_EST; 228 228 - break; 229 229 - case MLX4_EVENT_TYPE_SQ_DRAINED: 230 230 - event.event = IB_EVENT_SQ_DRAINED; 231 231 - break; 232 232 - case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: 233 233 - event.event = IB_EVENT_QP_LAST_WQE_REACHED; 234 234 - break; 235 235 - case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: 236 236 - event.event = IB_EVENT_QP_FATAL; 237 237 - break; 238 238 - case MLX4_EVENT_TYPE_PATH_MIG_FAILED: 239 239 - event.event = IB_EVENT_PATH_MIG_ERR; 240 240 - break; 241 241 - case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: 242 242 - event.event = IB_EVENT_QP_REQ_ERR; 243 243 - break; 244 244 - case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: 245 245 - event.event = IB_EVENT_QP_ACCESS_ERR; 246 246 - break; 247 247 - default: 248 248 - pr_warn("Unexpected event type %d " 249 249 - "on QP %06x\n", type, qp->qpn); 250 250 - return; 251 251 - } 259 259 + if (!ibqp->event_handler) 260 260 + goto out_no_handler; 252 261 253 253 - ibqp->event_handler(&event, ibqp->qp_context); 254 254 - } 262 262 + qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC); 263 263 + if (!qpe_work) 264 264 + goto out_no_handler; 265 265 + 266 266 + qpe_work->qp = qp; 267 267 + qpe_work->type = type; 268 268 + INIT_WORK(&qpe_work->work, mlx4_ib_handle_qp_event); 269 269 + queue_work(mlx4_ib_qp_event_wq, &qpe_work->work); 270 270 + return; 271 271 + 272 272 + out_no_handler: 273 273 + mlx4_put_qp(qp); 255 274 } 256 275 257 276 static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type) ··· 4502 4467 } 4503 4468 4504 4469 handle_drain_completion(cq, &rdrain, dev); 4470 4470 + } 4471 4471 + 4472 4472 + int mlx4_ib_qp_event_init(void) 4473 4473 + { 4474 4474 + mlx4_ib_qp_event_wq = alloc_ordered_workqueue("mlx4_ib_qp_event_wq", 0); 4475 4475 + if (!mlx4_ib_qp_event_wq) 4476 4476 + return -ENOMEM; 4477 4477 + 4478 4478 + return 0; 4479 4479 + } 4480 4480 + 4481 4481 + void mlx4_ib_qp_event_cleanup(void) 4482 4482 + { 4483 4483 + destroy_workqueue(mlx4_ib_qp_event_wq); 4505 4484 }

+26 -19

drivers/infiniband/hw/mlx5/cmd.c

reviewed

··· 5 5 6 6 #include "cmd.h" 7 7 8 8 - int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey) 8 8 + int mlx5r_cmd_query_special_mkeys(struct mlx5_ib_dev *dev) 9 9 { 10 10 u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {}; 11 11 u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {}; 12 12 + bool is_terminate, is_dump, is_null; 12 13 int err; 14 14 + 15 15 + is_terminate = MLX5_CAP_GEN(dev->mdev, terminate_scatter_list_mkey); 16 16 + is_dump = MLX5_CAP_GEN(dev->mdev, dump_fill_mkey); 17 17 + is_null = MLX5_CAP_GEN(dev->mdev, null_mkey); 18 18 + 19 19 + dev->mkeys.terminate_scatter_list_mkey = MLX5_TERMINATE_SCATTER_LIST_LKEY; 20 20 + if (!is_terminate && !is_dump && !is_null) 21 21 + return 0; 13 22 14 23 MLX5_SET(query_special_contexts_in, in, opcode, 15 24 MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS); 16 16 - err = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out); 17 17 - if (!err) 18 18 - *mkey = MLX5_GET(query_special_contexts_out, out, 19 19 - dump_fill_mkey); 20 20 - return err; 21 21 - } 25 25 + err = mlx5_cmd_exec_inout(dev->mdev, query_special_contexts, in, out); 26 26 + if (err) 27 27 + return err; 22 28 23 23 - int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey) 24 24 - { 25 25 - u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {}; 26 26 - u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {}; 27 27 - int err; 29 29 + if (is_dump) 30 30 + dev->mkeys.dump_fill_mkey = MLX5_GET(query_special_contexts_out, 31 31 + out, dump_fill_mkey); 28 32 29 29 - MLX5_SET(query_special_contexts_in, in, opcode, 30 30 - MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS); 31 31 - err = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out); 32 32 - if (!err) 33 33 - *null_mkey = MLX5_GET(query_special_contexts_out, out, 34 34 - null_mkey); 35 35 - return err; 33 33 + if (is_null) 34 34 + dev->mkeys.null_mkey = cpu_to_be32( 35 35 + MLX5_GET(query_special_contexts_out, out, null_mkey)); 36 36 + 37 37 + if (is_terminate) 38 38 + dev->mkeys.terminate_scatter_list_mkey = 39 39 + cpu_to_be32(MLX5_GET(query_special_contexts_out, out, 40 40 + terminate_scatter_list_mkey)); 41 41 + 42 42 + return 0; 36 43 } 37 44 38 45 int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,

+1 -2

drivers/infiniband/hw/mlx5/cmd.h

reviewed

··· 37 37 #include <linux/kernel.h> 38 38 #include <linux/mlx5/driver.h> 39 39 40 40 - int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey); 41 41 - int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey); 40 40 + int mlx5r_cmd_query_special_mkeys(struct mlx5_ib_dev *dev); 42 41 int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point, 43 42 void *out); 44 43 int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);

+25 -3

drivers/infiniband/hw/mlx5/cong.c

reviewed

··· 38 38 enum mlx5_ib_cong_node_type { 39 39 MLX5_IB_RROCE_ECN_RP = 1, 40 40 MLX5_IB_RROCE_ECN_NP = 2, 41 41 + MLX5_IB_RROCE_GENERAL = 3, 41 42 }; 42 43 43 44 static const char * const mlx5_ib_dbg_cc_name[] = { ··· 62 61 "np_cnp_dscp", 63 62 "np_cnp_prio_mode", 64 63 "np_cnp_prio", 64 64 + "rtt_resp_dscp_valid", 65 65 + "rtt_resp_dscp", 65 66 }; 66 67 67 68 #define MLX5_IB_RP_CLAMP_TGT_RATE_ATTR BIT(1) ··· 87 84 #define MLX5_IB_NP_CNP_DSCP_ATTR BIT(3) 88 85 #define MLX5_IB_NP_CNP_PRIO_MODE_ATTR BIT(4) 89 86 87 87 + #define MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR BIT(0) 88 88 + 90 89 static enum mlx5_ib_cong_node_type 91 90 mlx5_ib_param_to_node(enum mlx5_ib_dbg_cc_types param_offset) 92 91 { 93 93 - if (param_offset >= MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE && 94 94 - param_offset <= MLX5_IB_DBG_CC_RP_GD) 92 92 + if (param_offset <= MLX5_IB_DBG_CC_RP_GD) 95 93 return MLX5_IB_RROCE_ECN_RP; 96 96 - else 94 94 + 95 95 + if (param_offset <= MLX5_IB_DBG_CC_NP_CNP_PRIO) 97 96 return MLX5_IB_RROCE_ECN_NP; 97 97 + 98 98 + return MLX5_IB_RROCE_GENERAL; 98 99 } 99 100 100 101 static u32 mlx5_get_cc_param_val(void *field, int offset) ··· 164 157 case MLX5_IB_DBG_CC_NP_CNP_PRIO: 165 158 return MLX5_GET(cong_control_r_roce_ecn_np, field, 166 159 cnp_802p_prio); 160 160 + case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID: 161 161 + return MLX5_GET(cong_control_r_roce_general, field, 162 162 + rtt_resp_dscp_valid); 163 163 + case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP: 164 164 + return MLX5_GET(cong_control_r_roce_general, field, 165 165 + rtt_resp_dscp); 167 166 default: 168 167 return 0; 169 168 } ··· 276 263 *attr_mask |= MLX5_IB_NP_CNP_PRIO_MODE_ATTR; 277 264 MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_prio_mode, 0); 278 265 MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_802p_prio, var); 266 266 + break; 267 267 + case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID: 268 268 + *attr_mask |= MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR; 269 269 + MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp_valid, var); 270 270 + break; 271 271 + case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP: 272 272 + *attr_mask |= MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR; 273 273 + MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp_valid, 1); 274 274 + MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp, var); 279 275 break; 280 276 } 281 277 }

+13 -11

drivers/infiniband/hw/mlx5/main.c

reviewed

··· 1756 1756 struct mlx5_ib_dev *dev = to_mdev(ibdev); 1757 1757 struct mlx5_ib_ucontext *context = to_mucontext(uctx); 1758 1758 struct mlx5_bfreg_info *bfregi = &context->bfregi; 1759 1759 - int err; 1760 1759 1761 1760 if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) { 1762 1762 - err = mlx5_cmd_dump_fill_mkey(dev->mdev, 1763 1763 - &resp->dump_fill_mkey); 1764 1764 - if (err) 1765 1765 - return err; 1761 1761 + resp->dump_fill_mkey = dev->mkeys.dump_fill_mkey; 1766 1762 resp->comp_mask |= 1767 1763 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY; 1768 1764 } ··· 3662 3666 dev->port[i].roce.last_port_state = IB_PORT_DOWN; 3663 3667 } 3664 3668 3669 3669 + err = mlx5r_cmd_query_special_mkeys(dev); 3670 3670 + if (err) 3671 3671 + return err; 3672 3672 + 3665 3673 err = mlx5_ib_init_multiport_master(dev); 3666 3674 if (err) 3667 3675 return err; ··· 4030 4030 4031 4031 static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev) 4032 4032 { 4033 4033 - int err; 4034 4034 - 4035 4035 - err = mlx5_mkey_cache_cleanup(dev); 4036 4036 - if (err) 4037 4037 - mlx5_ib_warn(dev, "mr cache cleanup failed\n"); 4038 4038 - 4033 4033 + mlx5_mkey_cache_cleanup(dev); 4039 4034 mlx5r_umr_resource_cleanup(dev); 4040 4035 } 4041 4036 ··· 4428 4433 return -ENOMEM; 4429 4434 } 4430 4435 4436 4436 + ret = mlx5_ib_qp_event_init(); 4437 4437 + if (ret) 4438 4438 + goto qp_event_err; 4439 4439 + 4431 4440 mlx5_ib_odp_init(); 4432 4441 ret = mlx5r_rep_init(); 4433 4442 if (ret) ··· 4449 4450 mp_err: 4450 4451 mlx5r_rep_cleanup(); 4451 4452 rep_err: 4453 4453 + mlx5_ib_qp_event_cleanup(); 4454 4454 + qp_event_err: 4452 4455 destroy_workqueue(mlx5_ib_event_wq); 4453 4456 free_page((unsigned long)xlt_emergency_page); 4454 4457 return ret; ··· 4462 4461 auxiliary_driver_unregister(&mlx5r_mp_driver); 4463 4462 mlx5r_rep_cleanup(); 4464 4463 4464 4464 + mlx5_ib_qp_event_cleanup(); 4465 4465 destroy_workqueue(mlx5_ib_event_wq); 4466 4466 free_page((unsigned long)xlt_emergency_page); 4467 4467 }

+39 -12

drivers/infiniband/hw/mlx5/mlx5_ib.h

reviewed

··· 617 617 MLX5_MKEY_INDIRECT_DEVX, 618 618 }; 619 619 620 620 + struct mlx5r_cache_rb_key { 621 621 + u8 ats:1; 622 622 + unsigned int access_mode; 623 623 + unsigned int access_flags; 624 624 + unsigned int ndescs; 625 625 + }; 626 626 + 620 627 struct mlx5_ib_mkey { 621 628 u32 key; 622 629 enum mlx5_mkey_type type; 623 630 unsigned int ndescs; 624 631 struct wait_queue_head wait; 625 632 refcount_t usecount; 633 633 + /* User Mkey must hold either a rb_key or a cache_ent. */ 634 634 + struct mlx5r_cache_rb_key rb_key; 626 635 struct mlx5_cache_ent *cache_ent; 627 636 }; 628 637 ··· 746 737 unsigned long reserved; 747 738 748 739 char name[4]; 749 749 - u32 order; 750 750 - u32 access_mode; 751 751 - u32 page; 752 752 - unsigned int ndescs; 753 740 741 741 + struct rb_node node; 742 742 + struct mlx5r_cache_rb_key rb_key; 743 743 + 744 744 + u8 is_tmp:1; 754 745 u8 disabled:1; 755 746 u8 fill_to_high_water:1; 756 747 ··· 780 771 781 772 struct mlx5_mkey_cache { 782 773 struct workqueue_struct *wq; 783 783 - struct mlx5_cache_ent ent[MAX_MKEY_CACHE_ENTRIES]; 784 784 - struct dentry *root; 774 774 + struct rb_root rb_root; 775 775 + struct mutex rb_lock; 776 776 + struct dentry *fs_root; 785 777 unsigned long last_add; 778 778 + struct delayed_work remove_ent_dwork; 786 779 }; 787 780 788 781 struct mlx5_ib_port_resources { ··· 888 877 MLX5_IB_DBG_CC_NP_CNP_DSCP, 889 878 MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE, 890 879 MLX5_IB_DBG_CC_NP_CNP_PRIO, 880 880 + MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID, 881 881 + MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP, 891 882 MLX5_IB_DBG_CC_MAX, 892 883 }; 893 884 ··· 1067 1054 u8 ext_port_cap; 1068 1055 }; 1069 1056 1057 1057 + 1058 1058 + struct mlx5_special_mkeys { 1059 1059 + u32 dump_fill_mkey; 1060 1060 + __be32 null_mkey; 1061 1061 + __be32 terminate_scatter_list_mkey; 1062 1062 + }; 1063 1063 + 1070 1064 struct mlx5_ib_dev { 1071 1065 struct ib_device ib_dev; 1072 1066 struct mlx5_core_dev *mdev; ··· 1104 1084 1105 1085 struct xarray odp_mkeys; 1106 1086 1107 1107 - u32 null_mkey; 1108 1087 struct mlx5_ib_flow_db *flow_db; 1109 1088 /* protect resources needed as part of reset flow */ 1110 1089 spinlock_t reset_flow_resource_lock; ··· 1132 1113 struct mlx5_port_caps port_caps[MLX5_MAX_PORTS]; 1133 1114 u16 pkey_table_len; 1134 1115 u8 lag_ports; 1116 1116 + struct mlx5_special_mkeys mkeys; 1135 1117 }; 1136 1118 1137 1119 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq) ··· 1339 1319 void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num); 1340 1320 int mlx5_ib_get_cqe_size(struct ib_cq *ibcq); 1341 1321 int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev); 1342 1342 - int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev); 1322 1322 + void mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev); 1323 1323 + struct mlx5_cache_ent * 1324 1324 + mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev, 1325 1325 + struct mlx5r_cache_rb_key rb_key, 1326 1326 + bool persistent_entry); 1343 1327 1344 1328 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, 1345 1345 - struct mlx5_cache_ent *ent, 1346 1346 - int access_flags); 1329 1329 + int access_flags, int access_mode, 1330 1330 + int ndescs); 1347 1331 1348 1332 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, 1349 1333 struct ib_mr_status *mr_status); ··· 1371 1347 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev); 1372 1348 int __init mlx5_ib_odp_init(void); 1373 1349 void mlx5_ib_odp_cleanup(void); 1374 1374 - void mlx5_odp_init_mkey_cache_entry(struct mlx5_cache_ent *ent); 1350 1350 + int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev); 1375 1351 void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 1376 1352 struct mlx5_ib_mr *mr, int flags); 1377 1353 ··· 1390 1366 static inline void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev) {} 1391 1367 static inline int mlx5_ib_odp_init(void) { return 0; } 1392 1368 static inline void mlx5_ib_odp_cleanup(void) {} 1393 1393 - static inline void mlx5_odp_init_mkey_cache_entry(struct mlx5_cache_ent *ent) {} 1369 1369 + static inline int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev) 1370 1370 + { 1371 1371 + return 0; 1372 1372 + } 1394 1373 static inline void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 1395 1374 struct mlx5_ib_mr *mr, int flags) {} 1396 1375

+379 -111

drivers/infiniband/hw/mlx5/mr.c

reviewed

··· 140 140 mlx5_cmd_out_err(dev->mdev, MLX5_CMD_OP_CREATE_MKEY, 0, out); 141 141 } 142 142 143 143 - 144 144 - static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings, 145 145 - void *to_store) 143 143 + static int push_mkey_locked(struct mlx5_cache_ent *ent, bool limit_pendings, 144 144 + void *to_store) 146 145 { 147 146 XA_STATE(xas, &ent->mkeys, 0); 148 147 void *curr; 149 148 150 150 - xa_lock_irq(&ent->mkeys); 151 149 if (limit_pendings && 152 152 - (ent->reserved - ent->stored) > MAX_PENDING_REG_MR) { 153 153 - xa_unlock_irq(&ent->mkeys); 150 150 + (ent->reserved - ent->stored) > MAX_PENDING_REG_MR) 154 151 return -EAGAIN; 155 155 - } 152 152 + 156 153 while (1) { 157 154 /* 158 155 * This is cmpxchg (NULL, XA_ZERO_ENTRY) however this version ··· 188 191 break; 189 192 xa_lock_irq(&ent->mkeys); 190 193 } 194 194 + xa_lock_irq(&ent->mkeys); 191 195 if (xas_error(&xas)) 192 196 return xas_error(&xas); 193 197 if (WARN_ON(curr)) 194 198 return -EINVAL; 195 199 return 0; 200 200 + } 201 201 + 202 202 + static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings, 203 203 + void *to_store) 204 204 + { 205 205 + int ret; 206 206 + 207 207 + xa_lock_irq(&ent->mkeys); 208 208 + ret = push_mkey_locked(ent, limit_pendings, to_store); 209 209 + xa_unlock_irq(&ent->mkeys); 210 210 + return ret; 196 211 } 197 212 198 213 static void undo_push_reserve_mkey(struct mlx5_cache_ent *ent) ··· 301 292 set_mkc_access_pd_addr_fields(mkc, 0, 0, ent->dev->umrc.pd); 302 293 MLX5_SET(mkc, mkc, free, 1); 303 294 MLX5_SET(mkc, mkc, umr_en, 1); 304 304 - MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3); 305 305 - MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7); 295 295 + MLX5_SET(mkc, mkc, access_mode_1_0, ent->rb_key.access_mode & 0x3); 296 296 + MLX5_SET(mkc, mkc, access_mode_4_2, 297 297 + (ent->rb_key.access_mode >> 2) & 0x7); 306 298 307 299 MLX5_SET(mkc, mkc, translations_octword_size, 308 308 - get_mkc_octo_size(ent->access_mode, ent->ndescs)); 309 309 - MLX5_SET(mkc, mkc, log_page_size, ent->page); 300 300 + get_mkc_octo_size(ent->rb_key.access_mode, 301 301 + ent->rb_key.ndescs)); 302 302 + MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); 310 303 } 311 304 312 305 /* Asynchronously schedule new MRs to be populated in the cache. */ ··· 526 515 527 516 static bool someone_adding(struct mlx5_mkey_cache *cache) 528 517 { 529 529 - unsigned int i; 518 518 + struct mlx5_cache_ent *ent; 519 519 + struct rb_node *node; 520 520 + bool ret; 530 521 531 531 - for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) { 532 532 - struct mlx5_cache_ent *ent = &cache->ent[i]; 533 533 - bool ret; 534 534 - 522 522 + mutex_lock(&cache->rb_lock); 523 523 + for (node = rb_first(&cache->rb_root); node; node = rb_next(node)) { 524 524 + ent = rb_entry(node, struct mlx5_cache_ent, node); 535 525 xa_lock_irq(&ent->mkeys); 536 526 ret = ent->stored < ent->limit; 537 527 xa_unlock_irq(&ent->mkeys); 538 538 - if (ret) 528 528 + if (ret) { 529 529 + mutex_unlock(&cache->rb_lock); 539 530 return true; 531 531 + } 540 532 } 533 533 + mutex_unlock(&cache->rb_lock); 541 534 return false; 542 535 } 543 536 ··· 554 539 { 555 540 lockdep_assert_held(&ent->mkeys.xa_lock); 556 541 557 557 - if (ent->disabled || READ_ONCE(ent->dev->fill_delay)) 542 542 + if (ent->disabled || READ_ONCE(ent->dev->fill_delay) || ent->is_tmp) 558 543 return; 559 544 if (ent->stored < ent->limit) { 560 545 ent->fill_to_high_water = true; ··· 605 590 if (err != -EAGAIN) { 606 591 mlx5_ib_warn( 607 592 dev, 608 608 - "command failed order %d, err %d\n", 609 609 - ent->order, err); 593 593 + "add keys command failed, err %d\n", 594 594 + err); 610 595 queue_delayed_work(cache->wq, &ent->dwork, 611 596 msecs_to_jiffies(1000)); 612 597 } ··· 652 637 __cache_work_func(ent); 653 638 } 654 639 655 655 - struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, 656 656 - struct mlx5_cache_ent *ent, 657 657 - int access_flags) 640 640 + static int cache_ent_key_cmp(struct mlx5r_cache_rb_key key1, 641 641 + struct mlx5r_cache_rb_key key2) 642 642 + { 643 643 + int res; 644 644 + 645 645 + res = key1.ats - key2.ats; 646 646 + if (res) 647 647 + return res; 648 648 + 649 649 + res = key1.access_mode - key2.access_mode; 650 650 + if (res) 651 651 + return res; 652 652 + 653 653 + res = key1.access_flags - key2.access_flags; 654 654 + if (res) 655 655 + return res; 656 656 + 657 657 + /* 658 658 + * keep ndescs the last in the compare table since the find function 659 659 + * searches for an exact match on all properties and only closest 660 660 + * match in size. 661 661 + */ 662 662 + return key1.ndescs - key2.ndescs; 663 663 + } 664 664 + 665 665 + static int mlx5_cache_ent_insert(struct mlx5_mkey_cache *cache, 666 666 + struct mlx5_cache_ent *ent) 667 667 + { 668 668 + struct rb_node **new = &cache->rb_root.rb_node, *parent = NULL; 669 669 + struct mlx5_cache_ent *cur; 670 670 + int cmp; 671 671 + 672 672 + /* Figure out where to put new node */ 673 673 + while (*new) { 674 674 + cur = rb_entry(*new, struct mlx5_cache_ent, node); 675 675 + parent = *new; 676 676 + cmp = cache_ent_key_cmp(cur->rb_key, ent->rb_key); 677 677 + if (cmp > 0) 678 678 + new = &((*new)->rb_left); 679 679 + if (cmp < 0) 680 680 + new = &((*new)->rb_right); 681 681 + if (cmp == 0) { 682 682 + mutex_unlock(&cache->rb_lock); 683 683 + return -EEXIST; 684 684 + } 685 685 + } 686 686 + 687 687 + /* Add new node and rebalance tree. */ 688 688 + rb_link_node(&ent->node, parent, new); 689 689 + rb_insert_color(&ent->node, &cache->rb_root); 690 690 + 691 691 + return 0; 692 692 + } 693 693 + 694 694 + static struct mlx5_cache_ent * 695 695 + mkey_cache_ent_from_rb_key(struct mlx5_ib_dev *dev, 696 696 + struct mlx5r_cache_rb_key rb_key) 697 697 + { 698 698 + struct rb_node *node = dev->cache.rb_root.rb_node; 699 699 + struct mlx5_cache_ent *cur, *smallest = NULL; 700 700 + int cmp; 701 701 + 702 702 + /* 703 703 + * Find the smallest ent with order >= requested_order. 704 704 + */ 705 705 + while (node) { 706 706 + cur = rb_entry(node, struct mlx5_cache_ent, node); 707 707 + cmp = cache_ent_key_cmp(cur->rb_key, rb_key); 708 708 + if (cmp > 0) { 709 709 + smallest = cur; 710 710 + node = node->rb_left; 711 711 + } 712 712 + if (cmp < 0) 713 713 + node = node->rb_right; 714 714 + if (cmp == 0) 715 715 + return cur; 716 716 + } 717 717 + 718 718 + return (smallest && 719 719 + smallest->rb_key.access_mode == rb_key.access_mode && 720 720 + smallest->rb_key.access_flags == rb_key.access_flags && 721 721 + smallest->rb_key.ats == rb_key.ats) ? 722 722 + smallest : 723 723 + NULL; 724 724 + } 725 725 + 726 726 + static struct mlx5_ib_mr *_mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, 727 727 + struct mlx5_cache_ent *ent, 728 728 + int access_flags) 658 729 { 659 730 struct mlx5_ib_mr *mr; 660 731 int err; 661 661 - 662 662 - if (!mlx5r_umr_can_reconfig(dev, 0, access_flags)) 663 663 - return ERR_PTR(-EOPNOTSUPP); 664 732 665 733 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 666 734 if (!mr) ··· 775 677 return mr; 776 678 } 777 679 778 778 - static void clean_keys(struct mlx5_ib_dev *dev, int c) 680 680 + static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev, 681 681 + int access_flags) 779 682 { 780 780 - struct mlx5_mkey_cache *cache = &dev->cache; 781 781 - struct mlx5_cache_ent *ent = &cache->ent[c]; 683 683 + int ret = 0; 684 684 + 685 685 + if ((access_flags & IB_ACCESS_REMOTE_ATOMIC) && 686 686 + MLX5_CAP_GEN(dev->mdev, atomic) && 687 687 + MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)) 688 688 + ret |= IB_ACCESS_REMOTE_ATOMIC; 689 689 + 690 690 + if ((access_flags & IB_ACCESS_RELAXED_ORDERING) && 691 691 + MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) && 692 692 + !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)) 693 693 + ret |= IB_ACCESS_RELAXED_ORDERING; 694 694 + 695 695 + if ((access_flags & IB_ACCESS_RELAXED_ORDERING) && 696 696 + MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) && 697 697 + !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) 698 698 + ret |= IB_ACCESS_RELAXED_ORDERING; 699 699 + 700 700 + return ret; 701 701 + } 702 702 + 703 703 + struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, 704 704 + int access_flags, int access_mode, 705 705 + int ndescs) 706 706 + { 707 707 + struct mlx5r_cache_rb_key rb_key = { 708 708 + .ndescs = ndescs, 709 709 + .access_mode = access_mode, 710 710 + .access_flags = get_unchangeable_access_flags(dev, access_flags) 711 711 + }; 712 712 + struct mlx5_cache_ent *ent = mkey_cache_ent_from_rb_key(dev, rb_key); 713 713 + 714 714 + if (!ent) 715 715 + return ERR_PTR(-EOPNOTSUPP); 716 716 + 717 717 + return _mlx5_mr_cache_alloc(dev, ent, access_flags); 718 718 + } 719 719 + 720 720 + static void clean_keys(struct mlx5_ib_dev *dev, struct mlx5_cache_ent *ent) 721 721 + { 782 722 u32 mkey; 783 723 784 724 cancel_delayed_work(&ent->dwork); ··· 835 699 if (!mlx5_debugfs_root || dev->is_rep) 836 700 return; 837 701 838 838 - debugfs_remove_recursive(dev->cache.root); 839 839 - dev->cache.root = NULL; 702 702 + debugfs_remove_recursive(dev->cache.fs_root); 703 703 + dev->cache.fs_root = NULL; 840 704 } 841 705 842 842 - static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev) 706 706 + static void mlx5_mkey_cache_debugfs_add_ent(struct mlx5_ib_dev *dev, 707 707 + struct mlx5_cache_ent *ent) 843 708 { 844 844 - struct mlx5_mkey_cache *cache = &dev->cache; 845 845 - struct mlx5_cache_ent *ent; 709 709 + int order = order_base_2(ent->rb_key.ndescs); 846 710 struct dentry *dir; 847 847 - int i; 848 711 849 712 if (!mlx5_debugfs_root || dev->is_rep) 850 713 return; 851 714 852 852 - cache->root = debugfs_create_dir("mr_cache", mlx5_debugfs_get_dev_root(dev->mdev)); 715 715 + if (ent->rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM) 716 716 + order = MLX5_IMR_KSM_CACHE_ENTRY + 2; 853 717 854 854 - for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) { 855 855 - ent = &cache->ent[i]; 856 856 - sprintf(ent->name, "%d", ent->order); 857 857 - dir = debugfs_create_dir(ent->name, cache->root); 858 858 - debugfs_create_file("size", 0600, dir, ent, &size_fops); 859 859 - debugfs_create_file("limit", 0600, dir, ent, &limit_fops); 860 860 - debugfs_create_ulong("cur", 0400, dir, &ent->stored); 861 861 - debugfs_create_u32("miss", 0600, dir, &ent->miss); 862 862 - } 718 718 + sprintf(ent->name, "%d", order); 719 719 + dir = debugfs_create_dir(ent->name, dev->cache.fs_root); 720 720 + debugfs_create_file("size", 0600, dir, ent, &size_fops); 721 721 + debugfs_create_file("limit", 0600, dir, ent, &limit_fops); 722 722 + debugfs_create_ulong("cur", 0400, dir, &ent->stored); 723 723 + debugfs_create_u32("miss", 0600, dir, &ent->miss); 724 724 + } 725 725 + 726 726 + static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev) 727 727 + { 728 728 + struct dentry *dbg_root = mlx5_debugfs_get_dev_root(dev->mdev); 729 729 + struct mlx5_mkey_cache *cache = &dev->cache; 730 730 + 731 731 + if (!mlx5_debugfs_root || dev->is_rep) 732 732 + return; 733 733 + 734 734 + cache->fs_root = debugfs_create_dir("mr_cache", dbg_root); 863 735 } 864 736 865 737 static void delay_time_func(struct timer_list *t) ··· 877 733 WRITE_ONCE(dev->fill_delay, 0); 878 734 } 879 735 736 736 + struct mlx5_cache_ent * 737 737 + mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev, 738 738 + struct mlx5r_cache_rb_key rb_key, 739 739 + bool persistent_entry) 740 740 + { 741 741 + struct mlx5_cache_ent *ent; 742 742 + int order; 743 743 + int ret; 744 744 + 745 745 + ent = kzalloc(sizeof(*ent), GFP_KERNEL); 746 746 + if (!ent) 747 747 + return ERR_PTR(-ENOMEM); 748 748 + 749 749 + xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ); 750 750 + ent->rb_key = rb_key; 751 751 + ent->dev = dev; 752 752 + ent->is_tmp = !persistent_entry; 753 753 + 754 754 + INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); 755 755 + 756 756 + ret = mlx5_cache_ent_insert(&dev->cache, ent); 757 757 + if (ret) { 758 758 + kfree(ent); 759 759 + return ERR_PTR(ret); 760 760 + } 761 761 + 762 762 + if (persistent_entry) { 763 763 + if (rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM) 764 764 + order = MLX5_IMR_KSM_CACHE_ENTRY; 765 765 + else 766 766 + order = order_base_2(rb_key.ndescs) - 2; 767 767 + 768 768 + if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) && 769 769 + !dev->is_rep && mlx5_core_is_pf(dev->mdev) && 770 770 + mlx5r_umr_can_load_pas(dev, 0)) 771 771 + ent->limit = dev->mdev->profile.mr_cache[order].limit; 772 772 + else 773 773 + ent->limit = 0; 774 774 + 775 775 + mlx5_mkey_cache_debugfs_add_ent(dev, ent); 776 776 + } else { 777 777 + mod_delayed_work(ent->dev->cache.wq, 778 778 + &ent->dev->cache.remove_ent_dwork, 779 779 + msecs_to_jiffies(30 * 1000)); 780 780 + } 781 781 + 782 782 + return ent; 783 783 + } 784 784 + 785 785 + static void remove_ent_work_func(struct work_struct *work) 786 786 + { 787 787 + struct mlx5_mkey_cache *cache; 788 788 + struct mlx5_cache_ent *ent; 789 789 + struct rb_node *cur; 790 790 + 791 791 + cache = container_of(work, struct mlx5_mkey_cache, 792 792 + remove_ent_dwork.work); 793 793 + mutex_lock(&cache->rb_lock); 794 794 + cur = rb_last(&cache->rb_root); 795 795 + while (cur) { 796 796 + ent = rb_entry(cur, struct mlx5_cache_ent, node); 797 797 + cur = rb_prev(cur); 798 798 + mutex_unlock(&cache->rb_lock); 799 799 + 800 800 + xa_lock_irq(&ent->mkeys); 801 801 + if (!ent->is_tmp) { 802 802 + xa_unlock_irq(&ent->mkeys); 803 803 + mutex_lock(&cache->rb_lock); 804 804 + continue; 805 805 + } 806 806 + xa_unlock_irq(&ent->mkeys); 807 807 + 808 808 + clean_keys(ent->dev, ent); 809 809 + mutex_lock(&cache->rb_lock); 810 810 + } 811 811 + mutex_unlock(&cache->rb_lock); 812 812 + } 813 813 + 880 814 int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev) 881 815 { 882 816 struct mlx5_mkey_cache *cache = &dev->cache; 817 817 + struct rb_root *root = &dev->cache.rb_root; 818 818 + struct mlx5r_cache_rb_key rb_key = { 819 819 + .access_mode = MLX5_MKC_ACCESS_MODE_MTT, 820 820 + }; 883 821 struct mlx5_cache_ent *ent; 822 822 + struct rb_node *node; 823 823 + int ret; 884 824 int i; 885 825 886 826 mutex_init(&dev->slow_path_mutex); 827 827 + mutex_init(&dev->cache.rb_lock); 828 828 + dev->cache.rb_root = RB_ROOT; 829 829 + INIT_DELAYED_WORK(&dev->cache.remove_ent_dwork, remove_ent_work_func); 887 830 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM); 888 831 if (!cache->wq) { 889 832 mlx5_ib_warn(dev, "failed to create work queue\n"); ··· 979 748 980 749 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx); 981 750 timer_setup(&dev->delay_timer, delay_time_func, 0); 982 982 - for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) { 983 983 - ent = &cache->ent[i]; 984 984 - xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ); 985 985 - ent->order = i + 2; 986 986 - ent->dev = dev; 987 987 - ent->limit = 0; 988 988 - 989 989 - INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); 990 990 - 991 991 - if (i > MKEY_CACHE_LAST_STD_ENTRY) { 992 992 - mlx5_odp_init_mkey_cache_entry(ent); 993 993 - continue; 751 751 + mlx5_mkey_cache_debugfs_init(dev); 752 752 + mutex_lock(&cache->rb_lock); 753 753 + for (i = 0; i <= mkey_cache_max_order(dev); i++) { 754 754 + rb_key.ndescs = 1 << (i + 2); 755 755 + ent = mlx5r_cache_create_ent_locked(dev, rb_key, true); 756 756 + if (IS_ERR(ent)) { 757 757 + ret = PTR_ERR(ent); 758 758 + goto err; 994 759 } 760 760 + } 995 761 996 996 - if (ent->order > mkey_cache_max_order(dev)) 997 997 - continue; 762 762 + ret = mlx5_odp_init_mkey_cache(dev); 763 763 + if (ret) 764 764 + goto err; 998 765 999 999 - ent->page = PAGE_SHIFT; 1000 1000 - ent->ndescs = 1 << ent->order; 1001 1001 - ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 1002 1002 - if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) && 1003 1003 - !dev->is_rep && mlx5_core_is_pf(dev->mdev) && 1004 1004 - mlx5r_umr_can_load_pas(dev, 0)) 1005 1005 - ent->limit = dev->mdev->profile.mr_cache[i].limit; 1006 1006 - else 1007 1007 - ent->limit = 0; 766 766 + mutex_unlock(&cache->rb_lock); 767 767 + for (node = rb_first(root); node; node = rb_next(node)) { 768 768 + ent = rb_entry(node, struct mlx5_cache_ent, node); 1008 769 xa_lock_irq(&ent->mkeys); 1009 770 queue_adjust_cache_locked(ent); 1010 771 xa_unlock_irq(&ent->mkeys); 1011 772 } 1012 773 1013 1013 - mlx5_mkey_cache_debugfs_init(dev); 1014 1014 - 1015 774 return 0; 775 775 + 776 776 + err: 777 777 + mutex_unlock(&cache->rb_lock); 778 778 + mlx5_mkey_cache_debugfs_cleanup(dev); 779 779 + mlx5_ib_warn(dev, "failed to create mkey cache entry\n"); 780 780 + return ret; 1016 781 } 1017 782 1018 1018 - int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev) 783 783 + void mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev) 1019 784 { 1020 1020 - unsigned int i; 785 785 + struct rb_root *root = &dev->cache.rb_root; 786 786 + struct mlx5_cache_ent *ent; 787 787 + struct rb_node *node; 1021 788 1022 789 if (!dev->cache.wq) 1023 1023 - return 0; 790 790 + return; 1024 791 1025 1025 - for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) { 1026 1026 - struct mlx5_cache_ent *ent = &dev->cache.ent[i]; 1027 1027 - 792 792 + cancel_delayed_work_sync(&dev->cache.remove_ent_dwork); 793 793 + mutex_lock(&dev->cache.rb_lock); 794 794 + for (node = rb_first(root); node; node = rb_next(node)) { 795 795 + ent = rb_entry(node, struct mlx5_cache_ent, node); 1028 796 xa_lock_irq(&ent->mkeys); 1029 797 ent->disabled = true; 1030 798 xa_unlock_irq(&ent->mkeys); ··· 1033 803 mlx5_mkey_cache_debugfs_cleanup(dev); 1034 804 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx); 1035 805 1036 1036 - for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) 1037 1037 - clean_keys(dev, i); 806 806 + node = rb_first(root); 807 807 + while (node) { 808 808 + ent = rb_entry(node, struct mlx5_cache_ent, node); 809 809 + node = rb_next(node); 810 810 + clean_keys(dev, ent); 811 811 + rb_erase(&ent->node, root); 812 812 + kfree(ent); 813 813 + } 814 814 + mutex_unlock(&dev->cache.rb_lock); 1038 815 1039 816 destroy_workqueue(dev->cache.wq); 1040 817 del_timer_sync(&dev->delay_timer); 1041 1041 - 1042 1042 - return 0; 1043 818 } 1044 819 1045 820 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc) ··· 1108 873 static int mkey_cache_max_order(struct mlx5_ib_dev *dev) 1109 874 { 1110 875 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 1111 1111 - return MKEY_CACHE_LAST_STD_ENTRY + 2; 876 876 + return MKEY_CACHE_LAST_STD_ENTRY; 1112 877 return MLX5_MAX_UMR_SHIFT; 1113 1113 - } 1114 1114 - 1115 1115 - static struct mlx5_cache_ent *mkey_cache_ent_from_order(struct mlx5_ib_dev *dev, 1116 1116 - unsigned int order) 1117 1117 - { 1118 1118 - struct mlx5_mkey_cache *cache = &dev->cache; 1119 1119 - 1120 1120 - if (order < cache->ent[0].order) 1121 1121 - return &cache->ent[0]; 1122 1122 - order = order - cache->ent[0].order; 1123 1123 - if (order > MKEY_CACHE_LAST_STD_ENTRY) 1124 1124 - return NULL; 1125 1125 - return &cache->ent[order]; 1126 878 } 1127 879 1128 880 static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, ··· 1138 916 struct ib_umem *umem, u64 iova, 1139 917 int access_flags) 1140 918 { 919 919 + struct mlx5r_cache_rb_key rb_key = { 920 920 + .access_mode = MLX5_MKC_ACCESS_MODE_MTT, 921 921 + }; 1141 922 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1142 923 struct mlx5_cache_ent *ent; 1143 924 struct mlx5_ib_mr *mr; ··· 1153 928 0, iova); 1154 929 if (WARN_ON(!page_size)) 1155 930 return ERR_PTR(-EINVAL); 1156 1156 - ent = mkey_cache_ent_from_order( 1157 1157 - dev, order_base_2(ib_umem_num_dma_blocks(umem, page_size))); 931 931 + 932 932 + rb_key.ndescs = ib_umem_num_dma_blocks(umem, page_size); 933 933 + rb_key.ats = mlx5_umem_needs_ats(dev, umem, access_flags); 934 934 + rb_key.access_flags = get_unchangeable_access_flags(dev, access_flags); 935 935 + ent = mkey_cache_ent_from_rb_key(dev, rb_key); 1158 936 /* 1159 1159 - * Matches access in alloc_cache_mr(). If the MR can't come from the 1160 1160 - * cache then synchronously create an uncached one. 937 937 + * If the MR can't come from the cache then synchronously create an uncached 938 938 + * one. 1161 939 */ 1162 1162 - if (!ent || ent->limit == 0 || 1163 1163 - !mlx5r_umr_can_reconfig(dev, 0, access_flags) || 1164 1164 - mlx5_umem_needs_ats(dev, umem, access_flags)) { 940 940 + if (!ent) { 1165 941 mutex_lock(&dev->slow_path_mutex); 1166 942 mr = reg_create(pd, umem, iova, access_flags, page_size, false); 1167 943 mutex_unlock(&dev->slow_path_mutex); 944 944 + if (IS_ERR(mr)) 945 945 + return mr; 946 946 + mr->mmkey.rb_key = rb_key; 1168 947 return mr; 1169 948 } 1170 949 1171 1171 - mr = mlx5_mr_cache_alloc(dev, ent, access_flags); 950 950 + mr = _mlx5_mr_cache_alloc(dev, ent, access_flags); 1172 951 if (IS_ERR(mr)) 1173 952 return mr; 1174 953 ··· 1259 1030 goto err_2; 1260 1031 } 1261 1032 mr->mmkey.type = MLX5_MKEY_MR; 1033 1033 + mr->mmkey.ndescs = get_octo_len(iova, umem->length, mr->page_shift); 1262 1034 mr->umem = umem; 1263 1035 set_mr_fields(dev, mr, umem->length, access_flags, iova); 1264 1036 kvfree(in); ··· 1602 1372 mlx5_umem_find_best_pgsz(new_umem, mkc, log_page_size, 0, iova); 1603 1373 if (WARN_ON(!*page_size)) 1604 1374 return false; 1605 1605 - return (1ULL << mr->mmkey.cache_ent->order) >= 1375 1375 + return (mr->mmkey.cache_ent->rb_key.ndescs) >= 1606 1376 ib_umem_num_dma_blocks(new_umem, *page_size); 1607 1377 } 1608 1378 ··· 1797 1567 } 1798 1568 } 1799 1569 1570 1570 + static int cache_ent_find_and_store(struct mlx5_ib_dev *dev, 1571 1571 + struct mlx5_ib_mr *mr) 1572 1572 + { 1573 1573 + struct mlx5_mkey_cache *cache = &dev->cache; 1574 1574 + struct mlx5_cache_ent *ent; 1575 1575 + int ret; 1576 1576 + 1577 1577 + if (mr->mmkey.cache_ent) { 1578 1578 + xa_lock_irq(&mr->mmkey.cache_ent->mkeys); 1579 1579 + mr->mmkey.cache_ent->in_use--; 1580 1580 + goto end; 1581 1581 + } 1582 1582 + 1583 1583 + mutex_lock(&cache->rb_lock); 1584 1584 + ent = mkey_cache_ent_from_rb_key(dev, mr->mmkey.rb_key); 1585 1585 + if (ent) { 1586 1586 + if (ent->rb_key.ndescs == mr->mmkey.rb_key.ndescs) { 1587 1587 + if (ent->disabled) { 1588 1588 + mutex_unlock(&cache->rb_lock); 1589 1589 + return -EOPNOTSUPP; 1590 1590 + } 1591 1591 + mr->mmkey.cache_ent = ent; 1592 1592 + xa_lock_irq(&mr->mmkey.cache_ent->mkeys); 1593 1593 + mutex_unlock(&cache->rb_lock); 1594 1594 + goto end; 1595 1595 + } 1596 1596 + } 1597 1597 + 1598 1598 + ent = mlx5r_cache_create_ent_locked(dev, mr->mmkey.rb_key, false); 1599 1599 + mutex_unlock(&cache->rb_lock); 1600 1600 + if (IS_ERR(ent)) 1601 1601 + return PTR_ERR(ent); 1602 1602 + 1603 1603 + mr->mmkey.cache_ent = ent; 1604 1604 + xa_lock_irq(&mr->mmkey.cache_ent->mkeys); 1605 1605 + 1606 1606 + end: 1607 1607 + ret = push_mkey_locked(mr->mmkey.cache_ent, false, 1608 1608 + xa_mk_value(mr->mmkey.key)); 1609 1609 + xa_unlock_irq(&mr->mmkey.cache_ent->mkeys); 1610 1610 + return ret; 1611 1611 + } 1612 1612 + 1800 1613 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1801 1614 { 1802 1615 struct mlx5_ib_mr *mr = to_mmr(ibmr); ··· 1885 1612 } 1886 1613 1887 1614 /* Stop DMA */ 1888 1888 - if (mr->mmkey.cache_ent) { 1889 1889 - xa_lock_irq(&mr->mmkey.cache_ent->mkeys); 1890 1890 - mr->mmkey.cache_ent->in_use--; 1891 1891 - xa_unlock_irq(&mr->mmkey.cache_ent->mkeys); 1892 1892 - 1615 1615 + if (mr->umem && mlx5r_umr_can_load_pas(dev, mr->umem->length)) 1893 1616 if (mlx5r_umr_revoke_mr(mr) || 1894 1894 - push_mkey(mr->mmkey.cache_ent, false, 1895 1895 - xa_mk_value(mr->mmkey.key))) 1617 1617 + cache_ent_find_and_store(dev, mr)) 1896 1618 mr->mmkey.cache_ent = NULL; 1897 1897 - } 1619 1619 + 1898 1620 if (!mr->mmkey.cache_ent) { 1899 1621 rc = destroy_mkey(to_mdev(mr->ibmr.device), mr); 1900 1622 if (rc)

+27 -40

drivers/infiniband/hw/mlx5/odp.c

reviewed

··· 104 104 if (flags & MLX5_IB_UPD_XLT_ZAP) { 105 105 for (; pklm != end; pklm++, idx++) { 106 106 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 107 107 - pklm->key = cpu_to_be32(mr_to_mdev(imr)->null_mkey); 107 107 + pklm->key = mr_to_mdev(imr)->mkeys.null_mkey; 108 108 pklm->va = 0; 109 109 } 110 110 return; ··· 137 137 pklm->key = cpu_to_be32(mtt->ibmr.lkey); 138 138 pklm->va = cpu_to_be64(idx * MLX5_IMR_MTT_SIZE); 139 139 } else { 140 140 - pklm->key = cpu_to_be32(mr_to_mdev(imr)->null_mkey); 140 140 + pklm->key = mr_to_mdev(imr)->mkeys.null_mkey; 141 141 pklm->va = 0; 142 142 } 143 143 } ··· 417 417 if (IS_ERR(odp)) 418 418 return ERR_CAST(odp); 419 419 420 420 - mr = mlx5_mr_cache_alloc(dev, &dev->cache.ent[MLX5_IMR_MTT_CACHE_ENTRY], 421 421 - imr->access_flags); 420 420 + mr = mlx5_mr_cache_alloc(dev, imr->access_flags, 421 421 + MLX5_MKC_ACCESS_MODE_MTT, 422 422 + MLX5_IMR_MTT_ENTRIES); 422 423 if (IS_ERR(mr)) { 423 424 ib_umem_odp_release(odp); 424 425 return mr; ··· 493 492 if (IS_ERR(umem_odp)) 494 493 return ERR_CAST(umem_odp); 495 494 496 496 - imr = mlx5_mr_cache_alloc(dev, 497 497 - &dev->cache.ent[MLX5_IMR_KSM_CACHE_ENTRY], 498 498 - access_flags); 495 495 + imr = mlx5_mr_cache_alloc(dev, access_flags, MLX5_MKC_ACCESS_MODE_KSM, 496 496 + mlx5_imr_ksm_entries); 499 497 if (IS_ERR(imr)) { 500 498 ib_umem_odp_release(umem_odp); 501 499 return imr; ··· 986 986 { 987 987 int ret = 0, npages = 0; 988 988 u64 io_virt; 989 989 - u32 key; 989 989 + __be32 key; 990 990 u32 byte_count; 991 991 size_t bcnt; 992 992 int inline_segment; ··· 1000 1000 struct mlx5_wqe_data_seg *dseg = wqe; 1001 1001 1002 1002 io_virt = be64_to_cpu(dseg->addr); 1003 1003 - key = be32_to_cpu(dseg->lkey); 1003 1003 + key = dseg->lkey; 1004 1004 byte_count = be32_to_cpu(dseg->byte_count); 1005 1005 inline_segment = !!(byte_count & MLX5_INLINE_SEG); 1006 1006 bcnt = byte_count & ~MLX5_INLINE_SEG; ··· 1014 1014 } 1015 1015 1016 1016 /* receive WQE end of sg list. */ 1017 1017 - if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY && 1017 1017 + if (receive_queue && bcnt == 0 && 1018 1018 + key == dev->mkeys.terminate_scatter_list_mkey && 1018 1019 io_virt == 0) 1019 1020 break; 1020 1021 ··· 1035 1034 continue; 1036 1035 } 1037 1036 1038 1038 - ret = pagefault_single_data_segment(dev, NULL, key, 1037 1037 + ret = pagefault_single_data_segment(dev, NULL, be32_to_cpu(key), 1039 1038 io_virt, bcnt, 1040 1039 &pfault->bytes_committed, 1041 1040 bytes_mapped); ··· 1588 1587 return err; 1589 1588 } 1590 1589 1591 1591 - void mlx5_odp_init_mkey_cache_entry(struct mlx5_cache_ent *ent) 1590 1590 + int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev) 1592 1591 { 1593 1593 - if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1594 1594 - return; 1592 1592 + struct mlx5r_cache_rb_key rb_key = { 1593 1593 + .access_mode = MLX5_MKC_ACCESS_MODE_KSM, 1594 1594 + .ndescs = mlx5_imr_ksm_entries, 1595 1595 + }; 1596 1596 + struct mlx5_cache_ent *ent; 1595 1597 1596 1596 - switch (ent->order - 2) { 1597 1597 - case MLX5_IMR_MTT_CACHE_ENTRY: 1598 1598 - ent->page = PAGE_SHIFT; 1599 1599 - ent->ndescs = MLX5_IMR_MTT_ENTRIES; 1600 1600 - ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 1601 1601 - ent->limit = 0; 1602 1602 - break; 1598 1598 + if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1599 1599 + return 0; 1603 1600 1604 1604 - case MLX5_IMR_KSM_CACHE_ENTRY: 1605 1605 - ent->page = MLX5_KSM_PAGE_SHIFT; 1606 1606 - ent->ndescs = mlx5_imr_ksm_entries; 1607 1607 - ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM; 1608 1608 - ent->limit = 0; 1609 1609 - break; 1610 1610 - } 1601 1601 + ent = mlx5r_cache_create_ent_locked(dev, rb_key, true); 1602 1602 + if (IS_ERR(ent)) 1603 1603 + return PTR_ERR(ent); 1604 1604 + 1605 1605 + return 0; 1611 1606 } 1612 1607 1613 1608 static const struct ib_device_ops mlx5_ib_dev_odp_ops = { ··· 1612 1615 1613 1616 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) 1614 1617 { 1615 1615 - int ret = 0; 1616 1616 - 1617 1618 internal_fill_odp_caps(dev); 1618 1619 1619 1620 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1620 1620 - return ret; 1621 1621 + return 0; 1621 1622 1622 1623 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops); 1623 1624 1624 1624 - if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) { 1625 1625 - ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey); 1626 1626 - if (ret) { 1627 1627 - mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret); 1628 1628 - return ret; 1629 1629 - } 1630 1630 - } 1631 1631 - 1632 1625 mutex_init(&dev->odp_eq_mutex); 1633 1633 - return ret; 1626 1626 + return 0; 1634 1627 } 1635 1628 1636 1629 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev)

+128 -36

drivers/infiniband/hw/mlx5/qp.c

reviewed

··· 71 71 u32 port; 72 72 }; 73 73 74 74 + struct mlx5_ib_qp_event_work { 75 75 + struct work_struct work; 76 76 + struct mlx5_core_qp *qp; 77 77 + int type; 78 78 + }; 79 79 + 80 80 + static struct workqueue_struct *mlx5_ib_qp_event_wq; 81 81 + 74 82 static void get_cqs(enum ib_qp_type qp_type, 75 83 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq, 76 84 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq); ··· 310 302 return mlx5_ib_read_user_wqe_srq(srq, wqe_index, buffer, buflen, bc); 311 303 } 312 304 305 305 + static void mlx5_ib_qp_err_syndrome(struct ib_qp *ibqp) 306 306 + { 307 307 + struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 308 308 + int outlen = MLX5_ST_SZ_BYTES(query_qp_out); 309 309 + struct mlx5_ib_qp *qp = to_mqp(ibqp); 310 310 + void *pas_ext_union, *err_syn; 311 311 + u32 *outb; 312 312 + int err; 313 313 + 314 314 + if (!MLX5_CAP_GEN(dev->mdev, qpc_extension) || 315 315 + !MLX5_CAP_GEN(dev->mdev, qp_error_syndrome)) 316 316 + return; 317 317 + 318 318 + outb = kzalloc(outlen, GFP_KERNEL); 319 319 + if (!outb) 320 320 + return; 321 321 + 322 322 + err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen, 323 323 + true); 324 324 + if (err) 325 325 + goto out; 326 326 + 327 327 + pas_ext_union = 328 328 + MLX5_ADDR_OF(query_qp_out, outb, qp_pas_or_qpc_ext_and_pas); 329 329 + err_syn = MLX5_ADDR_OF(qpc_extension_and_pas_list_in, pas_ext_union, 330 330 + qpc_data_extension.error_syndrome); 331 331 + 332 332 + pr_err("%s/%d: QP %d error: %s (0x%x 0x%x 0x%x)\n", 333 333 + ibqp->device->name, ibqp->port, ibqp->qp_num, 334 334 + ib_wc_status_msg( 335 335 + MLX5_GET(cqe_error_syndrome, err_syn, syndrome)), 336 336 + MLX5_GET(cqe_error_syndrome, err_syn, vendor_error_syndrome), 337 337 + MLX5_GET(cqe_error_syndrome, err_syn, hw_syndrome_type), 338 338 + MLX5_GET(cqe_error_syndrome, err_syn, hw_error_syndrome)); 339 339 + out: 340 340 + kfree(outb); 341 341 + } 342 342 + 343 343 + static void mlx5_ib_handle_qp_event(struct work_struct *_work) 344 344 + { 345 345 + struct mlx5_ib_qp_event_work *qpe_work = 346 346 + container_of(_work, struct mlx5_ib_qp_event_work, work); 347 347 + struct ib_qp *ibqp = &to_mibqp(qpe_work->qp)->ibqp; 348 348 + struct ib_event event = {}; 349 349 + 350 350 + event.device = ibqp->device; 351 351 + event.element.qp = ibqp; 352 352 + switch (qpe_work->type) { 353 353 + case MLX5_EVENT_TYPE_PATH_MIG: 354 354 + event.event = IB_EVENT_PATH_MIG; 355 355 + break; 356 356 + case MLX5_EVENT_TYPE_COMM_EST: 357 357 + event.event = IB_EVENT_COMM_EST; 358 358 + break; 359 359 + case MLX5_EVENT_TYPE_SQ_DRAINED: 360 360 + event.event = IB_EVENT_SQ_DRAINED; 361 361 + break; 362 362 + case MLX5_EVENT_TYPE_SRQ_LAST_WQE: 363 363 + event.event = IB_EVENT_QP_LAST_WQE_REACHED; 364 364 + break; 365 365 + case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: 366 366 + event.event = IB_EVENT_QP_FATAL; 367 367 + break; 368 368 + case MLX5_EVENT_TYPE_PATH_MIG_FAILED: 369 369 + event.event = IB_EVENT_PATH_MIG_ERR; 370 370 + break; 371 371 + case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: 372 372 + event.event = IB_EVENT_QP_REQ_ERR; 373 373 + break; 374 374 + case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: 375 375 + event.event = IB_EVENT_QP_ACCESS_ERR; 376 376 + break; 377 377 + default: 378 378 + pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", 379 379 + qpe_work->type, qpe_work->qp->qpn); 380 380 + goto out; 381 381 + } 382 382 + 383 383 + if ((event.event == IB_EVENT_QP_FATAL) || 384 384 + (event.event == IB_EVENT_QP_ACCESS_ERR)) 385 385 + mlx5_ib_qp_err_syndrome(ibqp); 386 386 + 387 387 + ibqp->event_handler(&event, ibqp->qp_context); 388 388 + 389 389 + out: 390 390 + mlx5_core_res_put(&qpe_work->qp->common); 391 391 + kfree(qpe_work); 392 392 + } 393 393 + 313 394 static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type) 314 395 { 315 396 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp; 316 316 - struct ib_event event; 397 397 + struct mlx5_ib_qp_event_work *qpe_work; 317 398 318 399 if (type == MLX5_EVENT_TYPE_PATH_MIG) { 319 400 /* This event is only valid for trans_qps */ 320 401 to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port; 321 402 } 322 403 323 323 - if (ibqp->event_handler) { 324 324 - event.device = ibqp->device; 325 325 - event.element.qp = ibqp; 326 326 - switch (type) { 327 327 - case MLX5_EVENT_TYPE_PATH_MIG: 328 328 - event.event = IB_EVENT_PATH_MIG; 329 329 - break; 330 330 - case MLX5_EVENT_TYPE_COMM_EST: 331 331 - event.event = IB_EVENT_COMM_EST; 332 332 - break; 333 333 - case MLX5_EVENT_TYPE_SQ_DRAINED: 334 334 - event.event = IB_EVENT_SQ_DRAINED; 335 335 - break; 336 336 - case MLX5_EVENT_TYPE_SRQ_LAST_WQE: 337 337 - event.event = IB_EVENT_QP_LAST_WQE_REACHED; 338 338 - break; 339 339 - case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: 340 340 - event.event = IB_EVENT_QP_FATAL; 341 341 - break; 342 342 - case MLX5_EVENT_TYPE_PATH_MIG_FAILED: 343 343 - event.event = IB_EVENT_PATH_MIG_ERR; 344 344 - break; 345 345 - case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: 346 346 - event.event = IB_EVENT_QP_REQ_ERR; 347 347 - break; 348 348 - case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: 349 349 - event.event = IB_EVENT_QP_ACCESS_ERR; 350 350 - break; 351 351 - default: 352 352 - pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn); 353 353 - return; 354 354 - } 404 404 + if (!ibqp->event_handler) 405 405 + goto out_no_handler; 355 406 356 356 - ibqp->event_handler(&event, ibqp->qp_context); 357 357 - } 407 407 + qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC); 408 408 + if (!qpe_work) 409 409 + goto out_no_handler; 410 410 + 411 411 + qpe_work->qp = qp; 412 412 + qpe_work->type = type; 413 413 + INIT_WORK(&qpe_work->work, mlx5_ib_handle_qp_event); 414 414 + queue_work(mlx5_ib_qp_event_wq, &qpe_work->work); 415 415 + return; 416 416 + 417 417 + out_no_handler: 418 418 + mlx5_core_res_put(&qp->common); 358 419 } 359 420 360 421 static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap, ··· 4904 4827 if (!outb) 4905 4828 return -ENOMEM; 4906 4829 4907 4907 - err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen); 4830 4830 + err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen, 4831 4831 + false); 4908 4832 if (err) 4909 4833 goto out; 4910 4834 ··· 5797 5719 out: 5798 5720 mutex_unlock(&mqp->mutex); 5799 5721 return err; 5722 5722 + } 5723 5723 + 5724 5724 + int mlx5_ib_qp_event_init(void) 5725 5725 + { 5726 5726 + mlx5_ib_qp_event_wq = alloc_ordered_workqueue("mlx5_ib_qp_event_wq", 0); 5727 5727 + if (!mlx5_ib_qp_event_wq) 5728 5728 + return -ENOMEM; 5729 5729 + 5730 5730 + return 0; 5731 5731 + } 5732 5732 + 5733 5733 + void mlx5_ib_qp_event_cleanup(void) 5734 5734 + { 5735 5735 + destroy_workqueue(mlx5_ib_qp_event_wq); 5800 5736 }

+3 -1

drivers/infiniband/hw/mlx5/qp.h

reviewed

··· 20 20 int mlx5_core_destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp); 21 21 int mlx5_core_destroy_dct(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct); 22 22 int mlx5_core_qp_query(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp, 23 23 - u32 *out, int outlen); 23 23 + u32 *out, int outlen, bool qpc_ext); 24 24 int mlx5_core_dct_query(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct, 25 25 u32 *out, int outlen); 26 26 ··· 44 44 int mlx5_core_xrcd_alloc(struct mlx5_ib_dev *dev, u32 *xrcdn); 45 45 int mlx5_core_xrcd_dealloc(struct mlx5_ib_dev *dev, u32 xrcdn); 46 46 int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter); 47 47 + int mlx5_ib_qp_event_init(void); 48 48 + void mlx5_ib_qp_event_cleanup(void); 47 49 #endif /* _MLX5_IB_QP_H */

+5 -2

drivers/infiniband/hw/mlx5/qpc.c

reviewed

··· 135 135 case MLX5_RES_SQ: 136 136 qp = (struct mlx5_core_qp *)common; 137 137 qp->event(qp, event_type); 138 138 - break; 138 138 + /* Need to put resource in event handler */ 139 139 + return NOTIFY_OK; 139 140 case MLX5_RES_DCT: 140 141 dct = (struct mlx5_core_dct *)common; 141 142 if (event_type == MLX5_EVENT_TYPE_DCT_DRAINED) ··· 505 504 } 506 505 507 506 int mlx5_core_qp_query(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp, 508 508 - u32 *out, int outlen) 507 507 + u32 *out, int outlen, bool qpc_ext) 509 508 { 510 509 u32 in[MLX5_ST_SZ_DW(query_qp_in)] = {}; 511 510 512 511 MLX5_SET(query_qp_in, in, opcode, MLX5_CMD_OP_QUERY_QP); 513 512 MLX5_SET(query_qp_in, in, qpn, qp->qpn); 513 513 + MLX5_SET(query_qp_in, in, qpc_ext, qpc_ext); 514 514 + 514 515 return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, outlen); 515 516 } 516 517

+1 -1

drivers/infiniband/hw/mlx5/srq.c

reviewed

··· 447 447 448 448 if (i < srq->msrq.max_avail_gather) { 449 449 scat[i].byte_count = 0; 450 450 - scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 450 450 + scat[i].lkey = dev->mkeys.terminate_scatter_list_mkey; 451 451 scat[i].addr = 0; 452 452 } 453 453 }

+1 -3

drivers/infiniband/hw/mlx5/umr.c

reviewed

··· 636 636 mlx5r_umr_set_update_xlt_data_seg(&wqe.data_seg, &sg); 637 637 638 638 cur_mtt = mtt; 639 639 - rdma_for_each_block(mr->umem->sgt_append.sgt.sgl, &biter, 640 640 - mr->umem->sgt_append.sgt.nents, 641 641 - BIT(mr->page_shift)) { 639 639 + rdma_umem_for_each_dma_block(mr->umem, &biter, BIT(mr->page_shift)) { 642 640 if (cur_mtt == (void *)mtt + sg.length) { 643 641 dma_sync_single_for_device(ddev, sg.addr, sg.length, 644 642 DMA_TO_DEVICE);

+1 -1

drivers/infiniband/hw/mlx5/wr.c

reviewed

··· 1252 1252 1253 1253 if (i < qp->rq.max_gs) { 1254 1254 scat[i].byte_count = 0; 1255 1255 - scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 1255 1255 + scat[i].lkey = dev->mkeys.terminate_scatter_list_mkey; 1256 1256 scat[i].addr = 0; 1257 1257 } 1258 1258

+38

drivers/infiniband/sw/rxe/rxe.h

reviewed

··· 57 57 #define rxe_dbg_mw(mw, fmt, ...) ibdev_dbg((mw)->ibmw.device, \ 58 58 "mw#%d %s: " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__) 59 59 60 60 + /* responder states */ 61 61 + enum resp_states { 62 62 + RESPST_NONE, 63 63 + RESPST_GET_REQ, 64 64 + RESPST_CHK_PSN, 65 65 + RESPST_CHK_OP_SEQ, 66 66 + RESPST_CHK_OP_VALID, 67 67 + RESPST_CHK_RESOURCE, 68 68 + RESPST_CHK_LENGTH, 69 69 + RESPST_CHK_RKEY, 70 70 + RESPST_EXECUTE, 71 71 + RESPST_READ_REPLY, 72 72 + RESPST_ATOMIC_REPLY, 73 73 + RESPST_ATOMIC_WRITE_REPLY, 74 74 + RESPST_PROCESS_FLUSH, 75 75 + RESPST_COMPLETE, 76 76 + RESPST_ACKNOWLEDGE, 77 77 + RESPST_CLEANUP, 78 78 + RESPST_DUPLICATE_REQUEST, 79 79 + RESPST_ERR_MALFORMED_WQE, 80 80 + RESPST_ERR_UNSUPPORTED_OPCODE, 81 81 + RESPST_ERR_MISALIGNED_ATOMIC, 82 82 + RESPST_ERR_PSN_OUT_OF_SEQ, 83 83 + RESPST_ERR_MISSING_OPCODE_FIRST, 84 84 + RESPST_ERR_MISSING_OPCODE_LAST_C, 85 85 + RESPST_ERR_MISSING_OPCODE_LAST_D1E, 86 86 + RESPST_ERR_TOO_MANY_RDMA_ATM_REQ, 87 87 + RESPST_ERR_RNR, 88 88 + RESPST_ERR_RKEY_VIOLATION, 89 89 + RESPST_ERR_INVALIDATE_RKEY, 90 90 + RESPST_ERR_LENGTH, 91 91 + RESPST_ERR_CQ_OVERFLOW, 92 92 + RESPST_ERROR, 93 93 + RESPST_RESET, 94 94 + RESPST_DONE, 95 95 + RESPST_EXIT, 96 96 + }; 97 97 + 60 98 void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu); 61 99 62 100 int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name);

+8 -4

drivers/infiniband/sw/rxe/rxe_loc.h

reviewed

··· 64 64 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, 65 65 int access, struct rxe_mr *mr); 66 66 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr); 67 67 - int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length); 68 68 - int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length, 69 69 - enum rxe_mr_copy_dir dir); 67 67 + int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length); 68 68 + int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, 69 69 + unsigned int length, enum rxe_mr_copy_dir dir); 70 70 int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma, 71 71 void *addr, int length, enum rxe_mr_copy_dir dir); 72 72 - void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length); 72 72 + int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 73 73 + int sg_nents, unsigned int *sg_offset); 74 74 + int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 75 75 + u64 compare, u64 swap_add, u64 *orig_val); 76 76 + int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value); 73 77 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key, 74 78 enum rxe_mr_lookup_type type); 75 79 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);

+342 -272

drivers/infiniband/sw/rxe/rxe_mr.c

reviewed

··· 26 26 27 27 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length) 28 28 { 29 29 - 30 30 - 31 29 switch (mr->ibmr.type) { 32 30 case IB_MR_TYPE_DMA: 33 31 return 0; 34 32 35 33 case IB_MR_TYPE_USER: 36 34 case IB_MR_TYPE_MEM_REG: 37 37 - if (iova < mr->ibmr.iova || length > mr->ibmr.length || 38 38 - iova > mr->ibmr.iova + mr->ibmr.length - length) 39 39 - return -EFAULT; 35 35 + if (iova < mr->ibmr.iova || 36 36 + iova + length > mr->ibmr.iova + mr->ibmr.length) { 37 37 + rxe_dbg_mr(mr, "iova/length out of range"); 38 38 + return -EINVAL; 39 39 + } 40 40 return 0; 41 41 42 42 default: 43 43 - rxe_dbg_mr(mr, "type (%d) not supported\n", mr->ibmr.type); 44 44 - return -EFAULT; 43 43 + rxe_dbg_mr(mr, "mr type not supported\n"); 44 44 + return -EINVAL; 45 45 } 46 46 } 47 47 ··· 62 62 mr->lkey = mr->ibmr.lkey = lkey; 63 63 mr->rkey = mr->ibmr.rkey = rkey; 64 64 65 65 + mr->access = access; 66 66 + mr->ibmr.page_size = PAGE_SIZE; 67 67 + mr->page_mask = PAGE_MASK; 68 68 + mr->page_shift = PAGE_SHIFT; 65 69 mr->state = RXE_MR_STATE_INVALID; 66 66 - } 67 67 - 68 68 - static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf) 69 69 - { 70 70 - int i; 71 71 - int num_map; 72 72 - struct rxe_map **map = mr->map; 73 73 - 74 74 - num_map = (num_buf + RXE_BUF_PER_MAP - 1) / RXE_BUF_PER_MAP; 75 75 - 76 76 - mr->map = kmalloc_array(num_map, sizeof(*map), GFP_KERNEL); 77 77 - if (!mr->map) 78 78 - goto err1; 79 79 - 80 80 - for (i = 0; i < num_map; i++) { 81 81 - mr->map[i] = kmalloc(sizeof(**map), GFP_KERNEL); 82 82 - if (!mr->map[i]) 83 83 - goto err2; 84 84 - } 85 85 - 86 86 - BUILD_BUG_ON(!is_power_of_2(RXE_BUF_PER_MAP)); 87 87 - 88 88 - mr->map_shift = ilog2(RXE_BUF_PER_MAP); 89 89 - mr->map_mask = RXE_BUF_PER_MAP - 1; 90 90 - 91 91 - mr->num_buf = num_buf; 92 92 - mr->num_map = num_map; 93 93 - mr->max_buf = num_map * RXE_BUF_PER_MAP; 94 94 - 95 95 - return 0; 96 96 - 97 97 - err2: 98 98 - for (i--; i >= 0; i--) 99 99 - kfree(mr->map[i]); 100 100 - 101 101 - kfree(mr->map); 102 102 - mr->map = NULL; 103 103 - err1: 104 104 - return -ENOMEM; 105 70 } 106 71 107 72 void rxe_mr_init_dma(int access, struct rxe_mr *mr) 108 73 { 109 74 rxe_mr_init(access, mr); 110 75 111 111 - mr->access = access; 112 76 mr->state = RXE_MR_STATE_VALID; 113 77 mr->ibmr.type = IB_MR_TYPE_DMA; 78 78 + } 79 79 + 80 80 + static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova) 81 81 + { 82 82 + return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift); 83 83 + } 84 84 + 85 85 + static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova) 86 86 + { 87 87 + return iova & (mr_page_size(mr) - 1); 114 88 } 115 89 116 90 static bool is_pmem_page(struct page *pg) ··· 96 122 IORES_DESC_PERSISTENT_MEMORY); 97 123 } 98 124 125 125 + static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt) 126 126 + { 127 127 + XA_STATE(xas, &mr->page_list, 0); 128 128 + struct sg_page_iter sg_iter; 129 129 + struct page *page; 130 130 + bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); 131 131 + 132 132 + __sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0); 133 133 + if (!__sg_page_iter_next(&sg_iter)) 134 134 + return 0; 135 135 + 136 136 + do { 137 137 + xas_lock(&xas); 138 138 + while (true) { 139 139 + page = sg_page_iter_page(&sg_iter); 140 140 + 141 141 + if (persistent && !is_pmem_page(page)) { 142 142 + rxe_dbg_mr(mr, "Page can't be persistent\n"); 143 143 + xas_set_err(&xas, -EINVAL); 144 144 + break; 145 145 + } 146 146 + 147 147 + xas_store(&xas, page); 148 148 + if (xas_error(&xas)) 149 149 + break; 150 150 + xas_next(&xas); 151 151 + if (!__sg_page_iter_next(&sg_iter)) 152 152 + break; 153 153 + } 154 154 + xas_unlock(&xas); 155 155 + } while (xas_nomem(&xas, GFP_KERNEL)); 156 156 + 157 157 + return xas_error(&xas); 158 158 + } 159 159 + 99 160 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, 100 161 int access, struct rxe_mr *mr) 101 162 { 102 102 - struct rxe_map **map; 103 103 - struct rxe_phys_buf *buf = NULL; 104 104 - struct ib_umem *umem; 105 105 - struct sg_page_iter sg_iter; 106 106 - int num_buf; 107 107 - void *vaddr; 163 163 + struct ib_umem *umem; 108 164 int err; 165 165 + 166 166 + rxe_mr_init(access, mr); 167 167 + 168 168 + xa_init(&mr->page_list); 109 169 110 170 umem = ib_umem_get(&rxe->ib_dev, start, length, access); 111 171 if (IS_ERR(umem)) { 112 172 rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n", 113 173 (int)PTR_ERR(umem)); 114 114 - err = PTR_ERR(umem); 115 115 - goto err_out; 174 174 + return PTR_ERR(umem); 116 175 } 117 176 118 118 - num_buf = ib_umem_num_pages(umem); 119 119 - 120 120 - rxe_mr_init(access, mr); 121 121 - 122 122 - err = rxe_mr_alloc(mr, num_buf); 177 177 + err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt); 123 178 if (err) { 124 124 - rxe_dbg_mr(mr, "Unable to allocate memory for map\n"); 125 125 - goto err_release_umem; 126 126 - } 127 127 - 128 128 - mr->page_shift = PAGE_SHIFT; 129 129 - mr->page_mask = PAGE_SIZE - 1; 130 130 - 131 131 - num_buf = 0; 132 132 - map = mr->map; 133 133 - if (length > 0) { 134 134 - bool persistent_access = access & IB_ACCESS_FLUSH_PERSISTENT; 135 135 - 136 136 - buf = map[0]->buf; 137 137 - for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) { 138 138 - struct page *pg = sg_page_iter_page(&sg_iter); 139 139 - 140 140 - if (persistent_access && !is_pmem_page(pg)) { 141 141 - rxe_dbg_mr(mr, "Unable to register persistent access to non-pmem device\n"); 142 142 - err = -EINVAL; 143 143 - goto err_release_umem; 144 144 - } 145 145 - 146 146 - if (num_buf >= RXE_BUF_PER_MAP) { 147 147 - map++; 148 148 - buf = map[0]->buf; 149 149 - num_buf = 0; 150 150 - } 151 151 - 152 152 - vaddr = page_address(pg); 153 153 - if (!vaddr) { 154 154 - rxe_dbg_mr(mr, "Unable to get virtual address\n"); 155 155 - err = -ENOMEM; 156 156 - goto err_release_umem; 157 157 - } 158 158 - buf->addr = (uintptr_t)vaddr; 159 159 - buf->size = PAGE_SIZE; 160 160 - num_buf++; 161 161 - buf++; 162 162 - 163 163 - } 179 179 + ib_umem_release(umem); 180 180 + return err; 164 181 } 165 182 166 183 mr->umem = umem; 167 167 - mr->access = access; 168 168 - mr->offset = ib_umem_offset(umem); 169 169 - mr->state = RXE_MR_STATE_VALID; 170 184 mr->ibmr.type = IB_MR_TYPE_USER; 171 171 - mr->ibmr.page_size = PAGE_SIZE; 185 185 + mr->state = RXE_MR_STATE_VALID; 172 186 173 187 return 0; 188 188 + } 174 189 175 175 - err_release_umem: 176 176 - ib_umem_release(umem); 177 177 - err_out: 178 178 - return err; 190 190 + static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf) 191 191 + { 192 192 + XA_STATE(xas, &mr->page_list, 0); 193 193 + int i = 0; 194 194 + int err; 195 195 + 196 196 + xa_init(&mr->page_list); 197 197 + 198 198 + do { 199 199 + xas_lock(&xas); 200 200 + while (i != num_buf) { 201 201 + xas_store(&xas, XA_ZERO_ENTRY); 202 202 + if (xas_error(&xas)) 203 203 + break; 204 204 + xas_next(&xas); 205 205 + i++; 206 206 + } 207 207 + xas_unlock(&xas); 208 208 + } while (xas_nomem(&xas, GFP_KERNEL)); 209 209 + 210 210 + err = xas_error(&xas); 211 211 + if (err) 212 212 + return err; 213 213 + 214 214 + mr->num_buf = num_buf; 215 215 + 216 216 + return 0; 179 217 } 180 218 181 219 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) ··· 201 215 if (err) 202 216 goto err1; 203 217 204 204 - mr->max_buf = max_pages; 205 218 mr->state = RXE_MR_STATE_FREE; 206 219 mr->ibmr.type = IB_MR_TYPE_MEM_REG; 207 220 ··· 210 225 return err; 211 226 } 212 227 213 213 - static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out, 214 214 - size_t *offset_out) 228 228 + static int rxe_set_page(struct ib_mr *ibmr, u64 iova) 215 229 { 216 216 - size_t offset = iova - mr->ibmr.iova + mr->offset; 217 217 - int map_index; 218 218 - int buf_index; 219 219 - u64 length; 230 230 + struct rxe_mr *mr = to_rmr(ibmr); 231 231 + struct page *page = virt_to_page(iova & mr->page_mask); 232 232 + bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); 233 233 + int err; 220 234 221 221 - if (likely(mr->page_shift)) { 222 222 - *offset_out = offset & mr->page_mask; 223 223 - offset >>= mr->page_shift; 224 224 - *n_out = offset & mr->map_mask; 225 225 - *m_out = offset >> mr->map_shift; 226 226 - } else { 227 227 - map_index = 0; 228 228 - buf_index = 0; 229 229 - 230 230 - length = mr->map[map_index]->buf[buf_index].size; 231 231 - 232 232 - while (offset >= length) { 233 233 - offset -= length; 234 234 - buf_index++; 235 235 - 236 236 - if (buf_index == RXE_BUF_PER_MAP) { 237 237 - map_index++; 238 238 - buf_index = 0; 239 239 - } 240 240 - length = mr->map[map_index]->buf[buf_index].size; 241 241 - } 242 242 - 243 243 - *m_out = map_index; 244 244 - *n_out = buf_index; 245 245 - *offset_out = offset; 235 235 + if (persistent && !is_pmem_page(page)) { 236 236 + rxe_dbg_mr(mr, "Page cannot be persistent\n"); 237 237 + return -EINVAL; 246 238 } 239 239 + 240 240 + if (unlikely(mr->nbuf == mr->num_buf)) 241 241 + return -ENOMEM; 242 242 + 243 243 + err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL)); 244 244 + if (err) 245 245 + return err; 246 246 + 247 247 + mr->nbuf++; 248 248 + return 0; 247 249 } 248 250 249 249 - void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length) 251 251 + int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl, 252 252 + int sg_nents, unsigned int *sg_offset) 250 253 { 251 251 - size_t offset; 252 252 - int m, n; 253 253 - void *addr; 254 254 + struct rxe_mr *mr = to_rmr(ibmr); 255 255 + unsigned int page_size = mr_page_size(mr); 254 256 255 255 - if (mr->state != RXE_MR_STATE_VALID) { 256 256 - rxe_dbg_mr(mr, "Not in valid state\n"); 257 257 - addr = NULL; 258 258 - goto out; 259 259 - } 257 257 + mr->nbuf = 0; 258 258 + mr->page_shift = ilog2(page_size); 259 259 + mr->page_mask = ~((u64)page_size - 1); 260 260 + mr->page_offset = mr->ibmr.iova & (page_size - 1); 260 261 261 261 - if (!mr->map) { 262 262 - addr = (void *)(uintptr_t)iova; 263 263 - goto out; 264 264 - } 265 265 - 266 266 - if (mr_check_range(mr, iova, length)) { 267 267 - rxe_dbg_mr(mr, "Range violation\n"); 268 268 - addr = NULL; 269 269 - goto out; 270 270 - } 271 271 - 272 272 - lookup_iova(mr, iova, &m, &n, &offset); 273 273 - 274 274 - if (offset + length > mr->map[m]->buf[n].size) { 275 275 - rxe_dbg_mr(mr, "Crosses page boundary\n"); 276 276 - addr = NULL; 277 277 - goto out; 278 278 - } 279 279 - 280 280 - addr = (void *)(uintptr_t)mr->map[m]->buf[n].addr + offset; 281 281 - 282 282 - out: 283 283 - return addr; 262 262 + return ib_sg_to_pages(ibmr, sgl, sg_nents, sg_offset, rxe_set_page); 284 263 } 285 264 286 286 - int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length) 265 265 + static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr, 266 266 + unsigned int length, enum rxe_mr_copy_dir dir) 287 267 { 288 288 - size_t offset; 268 268 + unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); 269 269 + unsigned long index = rxe_mr_iova_to_index(mr, iova); 270 270 + unsigned int bytes; 271 271 + struct page *page; 272 272 + void *va; 289 273 290 290 - if (length == 0) 291 291 - return 0; 292 292 - 293 293 - if (mr->ibmr.type == IB_MR_TYPE_DMA) 294 294 - return -EFAULT; 295 295 - 296 296 - offset = (iova - mr->ibmr.iova + mr->offset) & mr->page_mask; 297 297 - while (length > 0) { 298 298 - u8 *va; 299 299 - int bytes; 300 300 - 301 301 - bytes = mr->ibmr.page_size - offset; 302 302 - if (bytes > length) 303 303 - bytes = length; 304 304 - 305 305 - va = iova_to_vaddr(mr, iova, length); 306 306 - if (!va) 274 274 + while (length) { 275 275 + page = xa_load(&mr->page_list, index); 276 276 + if (!page) 307 277 return -EFAULT; 308 278 309 309 - arch_wb_cache_pmem(va, bytes); 279 279 + bytes = min_t(unsigned int, length, 280 280 + mr_page_size(mr) - page_offset); 281 281 + va = kmap_local_page(page); 282 282 + if (dir == RXE_FROM_MR_OBJ) 283 283 + memcpy(addr, va + page_offset, bytes); 284 284 + else 285 285 + memcpy(va + page_offset, addr, bytes); 286 286 + kunmap_local(va); 310 287 288 288 + page_offset = 0; 289 289 + addr += bytes; 311 290 length -= bytes; 312 312 - iova += bytes; 313 313 - offset = 0; 291 291 + index++; 314 292 } 315 293 316 294 return 0; 317 295 } 318 296 319 319 - /* copy data from a range (vaddr, vaddr+length-1) to or from 320 320 - * a mr object starting at iova. 321 321 - */ 322 322 - int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length, 323 323 - enum rxe_mr_copy_dir dir) 297 297 + static void rxe_mr_copy_dma(struct rxe_mr *mr, u64 iova, void *addr, 298 298 + unsigned int length, enum rxe_mr_copy_dir dir) 324 299 { 325 325 - int err; 326 326 - int bytes; 327 327 - u8 *va; 328 328 - struct rxe_map **map; 329 329 - struct rxe_phys_buf *buf; 330 330 - int m; 331 331 - int i; 332 332 - size_t offset; 300 300 + unsigned int page_offset = iova & (PAGE_SIZE - 1); 301 301 + unsigned int bytes; 302 302 + struct page *page; 303 303 + u8 *va; 304 304 + 305 305 + while (length) { 306 306 + page = virt_to_page(iova & mr->page_mask); 307 307 + bytes = min_t(unsigned int, length, 308 308 + PAGE_SIZE - page_offset); 309 309 + va = kmap_local_page(page); 310 310 + 311 311 + if (dir == RXE_TO_MR_OBJ) 312 312 + memcpy(va + page_offset, addr, bytes); 313 313 + else 314 314 + memcpy(addr, va + page_offset, bytes); 315 315 + 316 316 + kunmap_local(va); 317 317 + page_offset = 0; 318 318 + iova += bytes; 319 319 + addr += bytes; 320 320 + length -= bytes; 321 321 + } 322 322 + } 323 323 + 324 324 + int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, 325 325 + unsigned int length, enum rxe_mr_copy_dir dir) 326 326 + { 327 327 + int err; 333 328 334 329 if (length == 0) 335 330 return 0; 336 331 332 332 + if (WARN_ON(!mr)) 333 333 + return -EINVAL; 334 334 + 337 335 if (mr->ibmr.type == IB_MR_TYPE_DMA) { 338 338 - u8 *src, *dest; 339 339 - 340 340 - src = (dir == RXE_TO_MR_OBJ) ? addr : ((void *)(uintptr_t)iova); 341 341 - 342 342 - dest = (dir == RXE_TO_MR_OBJ) ? ((void *)(uintptr_t)iova) : addr; 343 343 - 344 344 - memcpy(dest, src, length); 345 345 - 336 336 + rxe_mr_copy_dma(mr, iova, addr, length, dir); 346 337 return 0; 347 338 } 348 339 349 349 - WARN_ON_ONCE(!mr->map); 350 350 - 351 340 err = mr_check_range(mr, iova, length); 352 352 - if (err) { 353 353 - err = -EFAULT; 354 354 - goto err1; 341 341 + if (unlikely(err)) { 342 342 + rxe_dbg_mr(mr, "iova out of range"); 343 343 + return err; 355 344 } 356 345 357 357 - lookup_iova(mr, iova, &m, &i, &offset); 358 358 - 359 359 - map = mr->map + m; 360 360 - buf = map[0]->buf + i; 361 361 - 362 362 - while (length > 0) { 363 363 - u8 *src, *dest; 364 364 - 365 365 - va = (u8 *)(uintptr_t)buf->addr + offset; 366 366 - src = (dir == RXE_TO_MR_OBJ) ? addr : va; 367 367 - dest = (dir == RXE_TO_MR_OBJ) ? va : addr; 368 368 - 369 369 - bytes = buf->size - offset; 370 370 - 371 371 - if (bytes > length) 372 372 - bytes = length; 373 373 - 374 374 - memcpy(dest, src, bytes); 375 375 - 376 376 - length -= bytes; 377 377 - addr += bytes; 378 378 - 379 379 - offset = 0; 380 380 - buf++; 381 381 - i++; 382 382 - 383 383 - if (i == RXE_BUF_PER_MAP) { 384 384 - i = 0; 385 385 - map++; 386 386 - buf = map[0]->buf; 387 387 - } 388 388 - } 389 389 - 390 390 - return 0; 391 391 - 392 392 - err1: 393 393 - return err; 346 346 + return rxe_mr_copy_xarray(mr, iova, addr, length, dir); 394 347 } 395 348 396 349 /* copy data in or out of a wqe, i.e. sg list ··· 400 477 401 478 if (bytes > 0) { 402 479 iova = sge->addr + offset; 403 403 - 404 480 err = rxe_mr_copy(mr, iova, addr, bytes, dir); 405 481 if (err) 406 482 goto err2; ··· 425 503 err1: 426 504 return err; 427 505 } 506 506 + 507 507 + int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) 508 508 + { 509 509 + unsigned int page_offset; 510 510 + unsigned long index; 511 511 + struct page *page; 512 512 + unsigned int bytes; 513 513 + int err; 514 514 + u8 *va; 515 515 + 516 516 + /* mr must be valid even if length is zero */ 517 517 + if (WARN_ON(!mr)) 518 518 + return -EINVAL; 519 519 + 520 520 + if (length == 0) 521 521 + return 0; 522 522 + 523 523 + if (mr->ibmr.type == IB_MR_TYPE_DMA) 524 524 + return -EFAULT; 525 525 + 526 526 + err = mr_check_range(mr, iova, length); 527 527 + if (err) 528 528 + return err; 529 529 + 530 530 + while (length > 0) { 531 531 + index = rxe_mr_iova_to_index(mr, iova); 532 532 + page = xa_load(&mr->page_list, index); 533 533 + page_offset = rxe_mr_iova_to_page_offset(mr, iova); 534 534 + if (!page) 535 535 + return -EFAULT; 536 536 + bytes = min_t(unsigned int, length, 537 537 + mr_page_size(mr) - page_offset); 538 538 + 539 539 + va = kmap_local_page(page); 540 540 + arch_wb_cache_pmem(va + page_offset, bytes); 541 541 + kunmap_local(va); 542 542 + 543 543 + length -= bytes; 544 544 + iova += bytes; 545 545 + page_offset = 0; 546 546 + } 547 547 + 548 548 + return 0; 549 549 + } 550 550 + 551 551 + /* Guarantee atomicity of atomic operations at the machine level. */ 552 552 + static DEFINE_SPINLOCK(atomic_ops_lock); 553 553 + 554 554 + int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, 555 555 + u64 compare, u64 swap_add, u64 *orig_val) 556 556 + { 557 557 + unsigned int page_offset; 558 558 + struct page *page; 559 559 + u64 value; 560 560 + u64 *va; 561 561 + 562 562 + if (unlikely(mr->state != RXE_MR_STATE_VALID)) { 563 563 + rxe_dbg_mr(mr, "mr not in valid state"); 564 564 + return RESPST_ERR_RKEY_VIOLATION; 565 565 + } 566 566 + 567 567 + if (mr->ibmr.type == IB_MR_TYPE_DMA) { 568 568 + page_offset = iova & (PAGE_SIZE - 1); 569 569 + page = virt_to_page(iova & PAGE_MASK); 570 570 + } else { 571 571 + unsigned long index; 572 572 + int err; 573 573 + 574 574 + err = mr_check_range(mr, iova, sizeof(value)); 575 575 + if (err) { 576 576 + rxe_dbg_mr(mr, "iova out of range"); 577 577 + return RESPST_ERR_RKEY_VIOLATION; 578 578 + } 579 579 + page_offset = rxe_mr_iova_to_page_offset(mr, iova); 580 580 + index = rxe_mr_iova_to_index(mr, iova); 581 581 + page = xa_load(&mr->page_list, index); 582 582 + if (!page) 583 583 + return RESPST_ERR_RKEY_VIOLATION; 584 584 + } 585 585 + 586 586 + if (unlikely(page_offset & 0x7)) { 587 587 + rxe_dbg_mr(mr, "iova not aligned"); 588 588 + return RESPST_ERR_MISALIGNED_ATOMIC; 589 589 + } 590 590 + 591 591 + va = kmap_local_page(page); 592 592 + 593 593 + spin_lock_bh(&atomic_ops_lock); 594 594 + value = *orig_val = va[page_offset >> 3]; 595 595 + 596 596 + if (opcode == IB_OPCODE_RC_COMPARE_SWAP) { 597 597 + if (value == compare) 598 598 + va[page_offset >> 3] = swap_add; 599 599 + } else { 600 600 + value += swap_add; 601 601 + va[page_offset >> 3] = value; 602 602 + } 603 603 + spin_unlock_bh(&atomic_ops_lock); 604 604 + 605 605 + kunmap_local(va); 606 606 + 607 607 + return 0; 608 608 + } 609 609 + 610 610 + #if defined CONFIG_64BIT 611 611 + /* only implemented or called for 64 bit architectures */ 612 612 + int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) 613 613 + { 614 614 + unsigned int page_offset; 615 615 + struct page *page; 616 616 + u64 *va; 617 617 + 618 618 + /* See IBA oA19-28 */ 619 619 + if (unlikely(mr->state != RXE_MR_STATE_VALID)) { 620 620 + rxe_dbg_mr(mr, "mr not in valid state"); 621 621 + return RESPST_ERR_RKEY_VIOLATION; 622 622 + } 623 623 + 624 624 + if (mr->ibmr.type == IB_MR_TYPE_DMA) { 625 625 + page_offset = iova & (PAGE_SIZE - 1); 626 626 + page = virt_to_page(iova & PAGE_MASK); 627 627 + } else { 628 628 + unsigned long index; 629 629 + int err; 630 630 + 631 631 + /* See IBA oA19-28 */ 632 632 + err = mr_check_range(mr, iova, sizeof(value)); 633 633 + if (unlikely(err)) { 634 634 + rxe_dbg_mr(mr, "iova out of range"); 635 635 + return RESPST_ERR_RKEY_VIOLATION; 636 636 + } 637 637 + page_offset = rxe_mr_iova_to_page_offset(mr, iova); 638 638 + index = rxe_mr_iova_to_index(mr, iova); 639 639 + page = xa_load(&mr->page_list, index); 640 640 + if (!page) 641 641 + return RESPST_ERR_RKEY_VIOLATION; 642 642 + } 643 643 + 644 644 + /* See IBA A19.4.2 */ 645 645 + if (unlikely(page_offset & 0x7)) { 646 646 + rxe_dbg_mr(mr, "misaligned address"); 647 647 + return RESPST_ERR_MISALIGNED_ATOMIC; 648 648 + } 649 649 + 650 650 + va = kmap_local_page(page); 651 651 + 652 652 + /* Do atomic write after all prior operations have completed */ 653 653 + smp_store_release(&va[page_offset >> 3], value); 654 654 + 655 655 + kunmap_local(va); 656 656 + 657 657 + return 0; 658 658 + } 659 659 + #else 660 660 + int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) 661 661 + { 662 662 + return RESPST_ERR_UNSUPPORTED_OPCODE; 663 663 + } 664 664 + #endif 428 665 429 666 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length) 430 667 { ··· 618 537 return 0; 619 538 } 620 539 621 621 - /* (1) find the mr corresponding to lkey/rkey 622 622 - * depending on lookup_type 623 623 - * (2) verify that the (qp) pd matches the mr pd 624 624 - * (3) verify that the mr can support the requested access 625 625 - * (4) verify that mr state is valid 626 626 - */ 627 540 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key, 628 541 enum rxe_mr_lookup_type type) 629 542 { ··· 731 656 return -EINVAL; 732 657 733 658 rxe_cleanup(mr); 734 734 - 659 659 + kfree_rcu(mr); 735 660 return 0; 736 661 } 737 662 738 663 void rxe_mr_cleanup(struct rxe_pool_elem *elem) 739 664 { 740 665 struct rxe_mr *mr = container_of(elem, typeof(*mr), elem); 741 741 - int i; 742 666 743 667 rxe_put(mr_pd(mr)); 744 668 ib_umem_release(mr->umem); 745 669 746 746 - if (mr->map) { 747 747 - for (i = 0; i < mr->num_map; i++) 748 748 - kfree(mr->map[i]); 749 749 - 750 750 - kfree(mr->map); 751 751 - } 670 670 + if (mr->ibmr.type != IB_MR_TYPE_DMA) 671 671 + xa_destroy(&mr->page_list); 752 672 }

-46

drivers/infiniband/sw/rxe/rxe_pool.c

reviewed

··· 116 116 WARN_ON(!xa_empty(&pool->xa)); 117 117 } 118 118 119 119 - void *rxe_alloc(struct rxe_pool *pool) 120 120 - { 121 121 - struct rxe_pool_elem *elem; 122 122 - void *obj; 123 123 - int err; 124 124 - 125 125 - if (WARN_ON(!(pool->type == RXE_TYPE_MR))) 126 126 - return NULL; 127 127 - 128 128 - if (atomic_inc_return(&pool->num_elem) > pool->max_elem) 129 129 - goto err_cnt; 130 130 - 131 131 - obj = kzalloc(pool->elem_size, GFP_KERNEL); 132 132 - if (!obj) 133 133 - goto err_cnt; 134 134 - 135 135 - elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset); 136 136 - 137 137 - elem->pool = pool; 138 138 - elem->obj = obj; 139 139 - kref_init(&elem->ref_cnt); 140 140 - init_completion(&elem->complete); 141 141 - 142 142 - /* allocate index in array but leave pointer as NULL so it 143 143 - * can't be looked up until rxe_finalize() is called 144 144 - */ 145 145 - err = xa_alloc_cyclic(&pool->xa, &elem->index, NULL, pool->limit, 146 146 - &pool->next, GFP_KERNEL); 147 147 - if (err < 0) 148 148 - goto err_free; 149 149 - 150 150 - return obj; 151 151 - 152 152 - err_free: 153 153 - kfree(obj); 154 154 - err_cnt: 155 155 - atomic_dec(&pool->num_elem); 156 156 - return NULL; 157 157 - } 158 158 - 159 119 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem, 160 120 bool sleepable) 161 121 { 162 122 int err; 163 123 gfp_t gfp_flags; 164 164 - 165 165 - if (WARN_ON(pool->type == RXE_TYPE_MR)) 166 166 - return -EINVAL; 167 124 168 125 if (atomic_inc_return(&pool->num_elem) > pool->max_elem) 169 126 goto err_cnt; ··· 231 274 232 275 if (pool->cleanup) 233 276 pool->cleanup(elem); 234 234 - 235 235 - if (pool->type == RXE_TYPE_MR) 236 236 - kfree_rcu(elem->obj); 237 277 238 278 atomic_dec(&pool->num_elem); 239 279

-3

drivers/infiniband/sw/rxe/rxe_pool.h

reviewed

··· 54 54 /* free resources from object pool */ 55 55 void rxe_pool_cleanup(struct rxe_pool *pool); 56 56 57 57 - /* allocate an object from pool */ 58 58 - void *rxe_alloc(struct rxe_pool *pool); 59 59 - 60 57 /* connect already allocated object to pool */ 61 58 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem, 62 59 bool sleepable);

+66 -42

drivers/infiniband/sw/rxe/rxe_queue.h

reviewed

··· 35 35 /** 36 36 * enum queue_type - type of queue 37 37 * @QUEUE_TYPE_TO_CLIENT: Queue is written by rxe driver and 38 38 - * read by client. Used by rxe driver only. 38 38 + * read by client which may be a user space 39 39 + * application or a kernel ulp. 40 40 + * Used by rxe internals only. 39 41 * @QUEUE_TYPE_FROM_CLIENT: Queue is written by client and 40 40 - * read by rxe driver. Used by rxe driver only. 41 41 - * @QUEUE_TYPE_TO_DRIVER: Queue is written by client and 42 42 - * read by rxe driver. Used by kernel client only. 43 43 - * @QUEUE_TYPE_FROM_DRIVER: Queue is written by rxe driver and 44 44 - * read by client. Used by kernel client only. 42 42 + * read by rxe driver. 43 43 + * Used by rxe internals only. 44 44 + * @QUEUE_TYPE_FROM_ULP: Queue is written by kernel ulp and 45 45 + * read by rxe driver. 46 46 + * Used by kernel verbs APIs only on 47 47 + * behalf of ulps. 48 48 + * @QUEUE_TYPE_TO_ULP: Queue is written by rxe driver and 49 49 + * read by kernel ulp. 50 50 + * Used by kernel verbs APIs only on 51 51 + * behalf of ulps. 45 52 */ 46 53 enum queue_type { 47 54 QUEUE_TYPE_TO_CLIENT, 48 55 QUEUE_TYPE_FROM_CLIENT, 49 49 - QUEUE_TYPE_TO_DRIVER, 50 50 - QUEUE_TYPE_FROM_DRIVER, 56 56 + QUEUE_TYPE_FROM_ULP, 57 57 + QUEUE_TYPE_TO_ULP, 51 58 }; 52 59 53 60 struct rxe_queue_buf; ··· 69 62 u32 index_mask; 70 63 enum queue_type type; 71 64 /* private copy of index for shared queues between 72 72 - * kernel space and user space. Kernel reads and writes 65 65 + * driver and clients. Driver reads and writes 73 66 * this copy and then replicates to rxe_queue_buf 74 74 - * for read access by user space. 67 67 + * for read access by clients. 75 68 */ 76 69 u32 index; 77 70 }; ··· 104 97 105 98 switch (type) { 106 99 case QUEUE_TYPE_FROM_CLIENT: 107 107 - /* protect user index */ 100 100 + /* used by rxe, client owns the index */ 108 101 prod = smp_load_acquire(&q->buf->producer_index); 109 102 break; 110 103 case QUEUE_TYPE_TO_CLIENT: 104 104 + /* used by rxe which owns the index */ 111 105 prod = q->index; 112 106 break; 113 113 - case QUEUE_TYPE_FROM_DRIVER: 114 114 - /* protect driver index */ 115 115 - prod = smp_load_acquire(&q->buf->producer_index); 116 116 - break; 117 117 - case QUEUE_TYPE_TO_DRIVER: 107 107 + case QUEUE_TYPE_FROM_ULP: 108 108 + /* used by ulp which owns the index */ 118 109 prod = q->buf->producer_index; 110 110 + break; 111 111 + case QUEUE_TYPE_TO_ULP: 112 112 + /* used by ulp, rxe owns the index */ 113 113 + prod = smp_load_acquire(&q->buf->producer_index); 119 114 break; 120 115 } 121 116 ··· 131 122 132 123 switch (type) { 133 124 case QUEUE_TYPE_FROM_CLIENT: 125 125 + /* used by rxe which owns the index */ 134 126 cons = q->index; 135 127 break; 136 128 case QUEUE_TYPE_TO_CLIENT: 137 137 - /* protect user index */ 129 129 + /* used by rxe, client owns the index */ 138 130 cons = smp_load_acquire(&q->buf->consumer_index); 139 131 break; 140 140 - case QUEUE_TYPE_FROM_DRIVER: 132 132 + case QUEUE_TYPE_FROM_ULP: 133 133 + /* used by ulp, rxe owns the index */ 134 134 + cons = smp_load_acquire(&q->buf->consumer_index); 135 135 + break; 136 136 + case QUEUE_TYPE_TO_ULP: 137 137 + /* used by ulp which owns the index */ 141 138 cons = q->buf->consumer_index; 142 142 - break; 143 143 - case QUEUE_TYPE_TO_DRIVER: 144 144 - /* protect driver index */ 145 145 - cons = smp_load_acquire(&q->buf->consumer_index); 146 139 break; 147 140 } 148 141 ··· 183 172 184 173 switch (type) { 185 174 case QUEUE_TYPE_FROM_CLIENT: 186 186 - pr_warn("%s: attempt to advance client index\n", 187 187 - __func__); 175 175 + /* used by rxe, client owns the index */ 176 176 + if (WARN_ON(1)) 177 177 + pr_warn("%s: attempt to advance client index\n", 178 178 + __func__); 188 179 break; 189 180 case QUEUE_TYPE_TO_CLIENT: 181 181 + /* used by rxe which owns the index */ 190 182 prod = q->index; 191 183 prod = (prod + 1) & q->index_mask; 192 184 q->index = prod; 193 193 - /* protect user index */ 185 185 + /* release so client can read it safely */ 194 186 smp_store_release(&q->buf->producer_index, prod); 195 187 break; 196 196 - case QUEUE_TYPE_FROM_DRIVER: 197 197 - pr_warn("%s: attempt to advance driver index\n", 198 198 - __func__); 199 199 - break; 200 200 - case QUEUE_TYPE_TO_DRIVER: 188 188 + case QUEUE_TYPE_FROM_ULP: 189 189 + /* used by ulp which owns the index */ 201 190 prod = q->buf->producer_index; 202 191 prod = (prod + 1) & q->index_mask; 203 203 - q->buf->producer_index = prod; 192 192 + /* release so rxe can read it safely */ 193 193 + smp_store_release(&q->buf->producer_index, prod); 194 194 + break; 195 195 + case QUEUE_TYPE_TO_ULP: 196 196 + /* used by ulp, rxe owns the index */ 197 197 + if (WARN_ON(1)) 198 198 + pr_warn("%s: attempt to advance driver index\n", 199 199 + __func__); 204 200 break; 205 201 } 206 202 } ··· 219 201 220 202 switch (type) { 221 203 case QUEUE_TYPE_FROM_CLIENT: 222 222 - cons = q->index; 223 223 - cons = (cons + 1) & q->index_mask; 204 204 + /* used by rxe which owns the index */ 205 205 + cons = (q->index + 1) & q->index_mask; 224 206 q->index = cons; 225 225 - /* protect user index */ 207 207 + /* release so client can read it safely */ 226 208 smp_store_release(&q->buf->consumer_index, cons); 227 209 break; 228 210 case QUEUE_TYPE_TO_CLIENT: 229 229 - pr_warn("%s: attempt to advance client index\n", 230 230 - __func__); 211 211 + /* used by rxe, client owns the index */ 212 212 + if (WARN_ON(1)) 213 213 + pr_warn("%s: attempt to advance client index\n", 214 214 + __func__); 231 215 break; 232 232 - case QUEUE_TYPE_FROM_DRIVER: 216 216 + case QUEUE_TYPE_FROM_ULP: 217 217 + /* used by ulp, rxe owns the index */ 218 218 + if (WARN_ON(1)) 219 219 + pr_warn("%s: attempt to advance driver index\n", 220 220 + __func__); 221 221 + break; 222 222 + case QUEUE_TYPE_TO_ULP: 223 223 + /* used by ulp which owns the index */ 233 224 cons = q->buf->consumer_index; 234 225 cons = (cons + 1) & q->index_mask; 235 235 - q->buf->consumer_index = cons; 236 236 - break; 237 237 - case QUEUE_TYPE_TO_DRIVER: 238 238 - pr_warn("%s: attempt to advance driver index\n", 239 239 - __func__); 226 226 + /* release so rxe can read it safely */ 227 227 + smp_store_release(&q->buf->consumer_index, cons); 240 228 break; 241 229 } 242 230 }

+77 -129

drivers/infiniband/sw/rxe/rxe_resp.c

reviewed

··· 10 10 #include "rxe_loc.h" 11 11 #include "rxe_queue.h" 12 12 13 13 - enum resp_states { 14 14 - RESPST_NONE, 15 15 - RESPST_GET_REQ, 16 16 - RESPST_CHK_PSN, 17 17 - RESPST_CHK_OP_SEQ, 18 18 - RESPST_CHK_OP_VALID, 19 19 - RESPST_CHK_RESOURCE, 20 20 - RESPST_CHK_LENGTH, 21 21 - RESPST_CHK_RKEY, 22 22 - RESPST_EXECUTE, 23 23 - RESPST_READ_REPLY, 24 24 - RESPST_ATOMIC_REPLY, 25 25 - RESPST_ATOMIC_WRITE_REPLY, 26 26 - RESPST_PROCESS_FLUSH, 27 27 - RESPST_COMPLETE, 28 28 - RESPST_ACKNOWLEDGE, 29 29 - RESPST_CLEANUP, 30 30 - RESPST_DUPLICATE_REQUEST, 31 31 - RESPST_ERR_MALFORMED_WQE, 32 32 - RESPST_ERR_UNSUPPORTED_OPCODE, 33 33 - RESPST_ERR_MISALIGNED_ATOMIC, 34 34 - RESPST_ERR_PSN_OUT_OF_SEQ, 35 35 - RESPST_ERR_MISSING_OPCODE_FIRST, 36 36 - RESPST_ERR_MISSING_OPCODE_LAST_C, 37 37 - RESPST_ERR_MISSING_OPCODE_LAST_D1E, 38 38 - RESPST_ERR_TOO_MANY_RDMA_ATM_REQ, 39 39 - RESPST_ERR_RNR, 40 40 - RESPST_ERR_RKEY_VIOLATION, 41 41 - RESPST_ERR_INVALIDATE_RKEY, 42 42 - RESPST_ERR_LENGTH, 43 43 - RESPST_ERR_CQ_OVERFLOW, 44 44 - RESPST_ERROR, 45 45 - RESPST_RESET, 46 46 - RESPST_DONE, 47 47 - RESPST_EXIT, 48 48 - }; 49 49 - 50 13 static char *resp_state_name[] = { 51 14 [RESPST_NONE] = "NONE", 52 15 [RESPST_GET_REQ] = "GET_REQ", ··· 420 457 return RESPST_CHK_RKEY; 421 458 } 422 459 460 460 + /* if the reth length field is zero we can assume nothing 461 461 + * about the rkey value and should not validate or use it. 462 462 + * Instead set qp->resp.rkey to 0 which is an invalid rkey 463 463 + * value since the minimum index part is 1. 464 464 + */ 423 465 static void qp_resp_from_reth(struct rxe_qp *qp, struct rxe_pkt_info *pkt) 424 466 { 467 467 + unsigned int length = reth_len(pkt); 468 468 + 425 469 qp->resp.va = reth_va(pkt); 426 470 qp->resp.offset = 0; 427 427 - qp->resp.rkey = reth_rkey(pkt); 428 428 - qp->resp.resid = reth_len(pkt); 429 429 - qp->resp.length = reth_len(pkt); 471 471 + qp->resp.resid = length; 472 472 + qp->resp.length = length; 473 473 + if (pkt->mask & RXE_READ_OR_WRITE_MASK && length == 0) 474 474 + qp->resp.rkey = 0; 475 475 + else 476 476 + qp->resp.rkey = reth_rkey(pkt); 430 477 } 431 478 432 479 static void qp_resp_from_atmeth(struct rxe_qp *qp, struct rxe_pkt_info *pkt) ··· 447 474 qp->resp.resid = sizeof(u64); 448 475 } 449 476 477 477 + /* resolve the packet rkey to qp->resp.mr or set qp->resp.mr to NULL 478 478 + * if an invalid rkey is received or the rdma length is zero. For middle 479 479 + * or last packets use the stored value of mr. 480 480 + */ 450 481 static enum resp_states check_rkey(struct rxe_qp *qp, 451 482 struct rxe_pkt_info *pkt) 452 483 { ··· 487 510 return RESPST_EXECUTE; 488 511 } 489 512 490 490 - /* A zero-byte op is not required to set an addr or rkey. See C9-88 */ 513 513 + /* A zero-byte read or write op is not required to 514 514 + * set an addr or rkey. See C9-88 515 515 + */ 491 516 if ((pkt->mask & RXE_READ_OR_WRITE_MASK) && 492 492 - (pkt->mask & RXE_RETH_MASK) && 493 493 - reth_len(pkt) == 0) { 517 517 + (pkt->mask & RXE_RETH_MASK) && reth_len(pkt) == 0) { 518 518 + qp->resp.mr = NULL; 494 519 return RESPST_EXECUTE; 495 520 } 496 521 ··· 571 592 return RESPST_EXECUTE; 572 593 573 594 err: 595 595 + qp->resp.mr = NULL; 574 596 if (mr) 575 597 rxe_put(mr); 576 598 if (mw) ··· 705 725 return RESPST_ACKNOWLEDGE; 706 726 } 707 727 708 708 - /* Guarantee atomicity of atomic operations at the machine level. */ 709 709 - static DEFINE_SPINLOCK(atomic_ops_lock); 710 710 - 711 728 static enum resp_states atomic_reply(struct rxe_qp *qp, 712 712 - struct rxe_pkt_info *pkt) 729 729 + struct rxe_pkt_info *pkt) 713 730 { 714 714 - u64 *vaddr; 715 715 - enum resp_states ret; 716 731 struct rxe_mr *mr = qp->resp.mr; 717 732 struct resp_res *res = qp->resp.res; 718 718 - u64 value; 733 733 + int err; 719 734 720 735 if (!res) { 721 736 res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_MASK); ··· 718 743 } 719 744 720 745 if (!res->replay) { 721 721 - if (mr->state != RXE_MR_STATE_VALID) { 722 722 - ret = RESPST_ERR_RKEY_VIOLATION; 723 723 - goto out; 724 724 - } 746 746 + u64 iova = qp->resp.va + qp->resp.offset; 725 747 726 726 - vaddr = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, 727 727 - sizeof(u64)); 728 728 - 729 729 - /* check vaddr is 8 bytes aligned. */ 730 730 - if (!vaddr || (uintptr_t)vaddr & 7) { 731 731 - ret = RESPST_ERR_MISALIGNED_ATOMIC; 732 732 - goto out; 733 733 - } 734 734 - 735 735 - spin_lock_bh(&atomic_ops_lock); 736 736 - res->atomic.orig_val = value = *vaddr; 737 737 - 738 738 - if (pkt->opcode == IB_OPCODE_RC_COMPARE_SWAP) { 739 739 - if (value == atmeth_comp(pkt)) 740 740 - value = atmeth_swap_add(pkt); 741 741 - } else { 742 742 - value += atmeth_swap_add(pkt); 743 743 - } 744 744 - 745 745 - *vaddr = value; 746 746 - spin_unlock_bh(&atomic_ops_lock); 748 748 + err = rxe_mr_do_atomic_op(mr, iova, pkt->opcode, 749 749 + atmeth_comp(pkt), 750 750 + atmeth_swap_add(pkt), 751 751 + &res->atomic.orig_val); 752 752 + if (err) 753 753 + return err; 747 754 748 755 qp->resp.msn++; 749 756 ··· 737 780 qp->resp.status = IB_WC_SUCCESS; 738 781 } 739 782 740 740 - ret = RESPST_ACKNOWLEDGE; 741 741 - out: 742 742 - return ret; 743 743 - } 744 744 - 745 745 - #ifdef CONFIG_64BIT 746 746 - static enum resp_states do_atomic_write(struct rxe_qp *qp, 747 747 - struct rxe_pkt_info *pkt) 748 748 - { 749 749 - struct rxe_mr *mr = qp->resp.mr; 750 750 - int payload = payload_size(pkt); 751 751 - u64 src, *dst; 752 752 - 753 753 - if (mr->state != RXE_MR_STATE_VALID) 754 754 - return RESPST_ERR_RKEY_VIOLATION; 755 755 - 756 756 - memcpy(&src, payload_addr(pkt), payload); 757 757 - 758 758 - dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload); 759 759 - /* check vaddr is 8 bytes aligned. */ 760 760 - if (!dst || (uintptr_t)dst & 7) 761 761 - return RESPST_ERR_MISALIGNED_ATOMIC; 762 762 - 763 763 - /* Do atomic write after all prior operations have completed */ 764 764 - smp_store_release(dst, src); 765 765 - 766 766 - /* decrease resp.resid to zero */ 767 767 - qp->resp.resid -= sizeof(payload); 768 768 - 769 769 - qp->resp.msn++; 770 770 - 771 771 - /* next expected psn, read handles this separately */ 772 772 - qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK; 773 773 - qp->resp.ack_psn = qp->resp.psn; 774 774 - 775 775 - qp->resp.opcode = pkt->opcode; 776 776 - qp->resp.status = IB_WC_SUCCESS; 777 783 return RESPST_ACKNOWLEDGE; 778 784 } 779 779 - #else 780 780 - static enum resp_states do_atomic_write(struct rxe_qp *qp, 781 781 - struct rxe_pkt_info *pkt) 782 782 - { 783 783 - return RESPST_ERR_UNSUPPORTED_OPCODE; 784 784 - } 785 785 - #endif /* CONFIG_64BIT */ 786 785 787 786 static enum resp_states atomic_write_reply(struct rxe_qp *qp, 788 787 struct rxe_pkt_info *pkt) 789 788 { 790 789 struct resp_res *res = qp->resp.res; 790 790 + struct rxe_mr *mr; 791 791 + u64 value; 792 792 + u64 iova; 793 793 + int err; 791 794 792 795 if (!res) { 793 796 res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK); ··· 756 839 757 840 if (res->replay) 758 841 return RESPST_ACKNOWLEDGE; 759 759 - return do_atomic_write(qp, pkt); 842 842 + 843 843 + mr = qp->resp.mr; 844 844 + value = *(u64 *)payload_addr(pkt); 845 845 + iova = qp->resp.va + qp->resp.offset; 846 846 + 847 847 + err = rxe_mr_do_atomic_write(mr, iova, value); 848 848 + if (err) 849 849 + return err; 850 850 + 851 851 + qp->resp.resid = 0; 852 852 + qp->resp.msn++; 853 853 + 854 854 + /* next expected psn, read handles this separately */ 855 855 + qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK; 856 856 + qp->resp.ack_psn = qp->resp.psn; 857 857 + 858 858 + qp->resp.opcode = pkt->opcode; 859 859 + qp->resp.status = IB_WC_SUCCESS; 860 860 + 861 861 + return RESPST_ACKNOWLEDGE; 760 862 } 761 863 762 864 static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp, ··· 902 966 } 903 967 904 968 if (res->state == rdatm_res_state_new) { 905 905 - if (!res->replay) { 969 969 + if (!res->replay || qp->resp.length == 0) { 970 970 + /* if length == 0 mr will be NULL (is ok) 971 971 + * otherwise qp->resp.mr holds a ref on mr 972 972 + * which we transfer to mr and drop below. 973 973 + */ 906 974 mr = qp->resp.mr; 907 975 qp->resp.mr = NULL; 908 976 } else { ··· 920 980 else 921 981 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST; 922 982 } else { 983 983 + /* re-lookup mr from rkey on all later packets. 984 984 + * length will be non-zero. This can fail if someone 985 985 + * modifies or destroys the mr since the first packet. 986 986 + */ 923 987 mr = rxe_recheck_mr(qp, res->read.rkey); 924 988 if (!mr) 925 989 return RESPST_ERR_RKEY_VIOLATION; ··· 941 997 skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload, 942 998 res->cur_psn, AETH_ACK_UNLIMITED); 943 999 if (!skb) { 944 944 - if (mr) 945 945 - rxe_put(mr); 946 946 - return RESPST_ERR_RNR; 1000 1000 + state = RESPST_ERR_RNR; 1001 1001 + goto err_out; 947 1002 } 948 1003 949 1004 err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt), 950 1005 payload, RXE_FROM_MR_OBJ); 951 951 - if (mr) 952 952 - rxe_put(mr); 953 1006 if (err) { 954 1007 kfree_skb(skb); 955 955 - return RESPST_ERR_RKEY_VIOLATION; 1008 1008 + state = RESPST_ERR_RKEY_VIOLATION; 1009 1009 + goto err_out; 956 1010 } 957 1011 958 1012 if (bth_pad(&ack_pkt)) { ··· 959 1017 memset(pad, 0, bth_pad(&ack_pkt)); 960 1018 } 961 1019 1020 1020 + /* rxe_xmit_packet always consumes the skb */ 962 1021 err = rxe_xmit_packet(qp, &ack_pkt, skb); 963 963 - if (err) 964 964 - return RESPST_ERR_RNR; 1022 1022 + if (err) { 1023 1023 + state = RESPST_ERR_RNR; 1024 1024 + goto err_out; 1025 1025 + } 965 1026 966 1027 res->read.va += payload; 967 1028 res->read.resid -= payload; ··· 981 1036 state = RESPST_CLEANUP; 982 1037 } 983 1038 1039 1039 + err_out: 1040 1040 + if (mr) 1041 1041 + rxe_put(mr); 984 1042 return state; 985 1043 } 986 1044

+53 -62

drivers/infiniband/sw/rxe/rxe_verbs.c

reviewed

··· 245 245 int num_sge = ibwr->num_sge; 246 246 int full; 247 247 248 248 - full = queue_full(rq->queue, QUEUE_TYPE_TO_DRIVER); 248 248 + full = queue_full(rq->queue, QUEUE_TYPE_FROM_ULP); 249 249 if (unlikely(full)) 250 250 return -ENOMEM; 251 251 ··· 256 256 for (i = 0; i < num_sge; i++) 257 257 length += ibwr->sg_list[i].length; 258 258 259 259 - recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_TO_DRIVER); 259 259 + recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_FROM_ULP); 260 260 recv_wqe->wr_id = ibwr->wr_id; 261 261 262 262 memcpy(recv_wqe->dma.sge, ibwr->sg_list, ··· 268 268 recv_wqe->dma.cur_sge = 0; 269 269 recv_wqe->dma.sge_offset = 0; 270 270 271 271 - queue_advance_producer(rq->queue, QUEUE_TYPE_TO_DRIVER); 271 271 + queue_advance_producer(rq->queue, QUEUE_TYPE_FROM_ULP); 272 272 273 273 return 0; 274 274 } ··· 623 623 624 624 spin_lock_irqsave(&qp->sq.sq_lock, flags); 625 625 626 626 - full = queue_full(sq->queue, QUEUE_TYPE_TO_DRIVER); 626 626 + full = queue_full(sq->queue, QUEUE_TYPE_FROM_ULP); 627 627 628 628 if (unlikely(full)) { 629 629 spin_unlock_irqrestore(&qp->sq.sq_lock, flags); 630 630 return -ENOMEM; 631 631 } 632 632 633 633 - send_wqe = queue_producer_addr(sq->queue, QUEUE_TYPE_TO_DRIVER); 633 633 + send_wqe = queue_producer_addr(sq->queue, QUEUE_TYPE_FROM_ULP); 634 634 init_send_wqe(qp, ibwr, mask, length, send_wqe); 635 635 636 636 - queue_advance_producer(sq->queue, QUEUE_TYPE_TO_DRIVER); 636 636 + queue_advance_producer(sq->queue, QUEUE_TYPE_FROM_ULP); 637 637 638 638 spin_unlock_irqrestore(&qp->sq.sq_lock, flags); 639 639 ··· 821 821 822 822 spin_lock_irqsave(&cq->cq_lock, flags); 823 823 for (i = 0; i < num_entries; i++) { 824 824 - cqe = queue_head(cq->queue, QUEUE_TYPE_FROM_DRIVER); 824 824 + cqe = queue_head(cq->queue, QUEUE_TYPE_TO_ULP); 825 825 if (!cqe) 826 826 break; 827 827 828 828 memcpy(wc++, &cqe->ibwc, sizeof(*wc)); 829 829 - queue_advance_consumer(cq->queue, QUEUE_TYPE_FROM_DRIVER); 829 829 + queue_advance_consumer(cq->queue, QUEUE_TYPE_TO_ULP); 830 830 } 831 831 spin_unlock_irqrestore(&cq->cq_lock, flags); 832 832 ··· 838 838 struct rxe_cq *cq = to_rcq(ibcq); 839 839 int count; 840 840 841 841 - count = queue_count(cq->queue, QUEUE_TYPE_FROM_DRIVER); 841 841 + count = queue_count(cq->queue, QUEUE_TYPE_TO_ULP); 842 842 843 843 return (count > wc_cnt) ? wc_cnt : count; 844 844 } ··· 854 854 if (cq->notify != IB_CQ_NEXT_COMP) 855 855 cq->notify = flags & IB_CQ_SOLICITED_MASK; 856 856 857 857 - empty = queue_empty(cq->queue, QUEUE_TYPE_FROM_DRIVER); 857 857 + empty = queue_empty(cq->queue, QUEUE_TYPE_TO_ULP); 858 858 859 859 if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !empty) 860 860 ret = 1; ··· 869 869 struct rxe_dev *rxe = to_rdev(ibpd->device); 870 870 struct rxe_pd *pd = to_rpd(ibpd); 871 871 struct rxe_mr *mr; 872 872 + int err; 872 873 873 873 - mr = rxe_alloc(&rxe->mr_pool); 874 874 - if (!mr) 875 875 - return ERR_PTR(-ENOMEM); 874 874 + mr = kzalloc(sizeof(*mr), GFP_KERNEL); 875 875 + if (!mr) { 876 876 + err = -ENOMEM; 877 877 + goto err_out; 878 878 + } 879 879 + 880 880 + err = rxe_add_to_pool(&rxe->mr_pool, mr); 881 881 + if (err) 882 882 + goto err_free; 876 883 877 884 rxe_get(pd); 878 885 mr->ibmr.pd = ibpd; ··· 887 880 888 881 rxe_mr_init_dma(access, mr); 889 882 rxe_finalize(mr); 890 890 - 891 883 return &mr->ibmr; 884 884 + 885 885 + err_free: 886 886 + kfree(mr); 887 887 + err_out: 888 888 + return ERR_PTR(err); 892 889 } 893 890 894 891 static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, ··· 906 895 struct rxe_pd *pd = to_rpd(ibpd); 907 896 struct rxe_mr *mr; 908 897 909 909 - mr = rxe_alloc(&rxe->mr_pool); 910 910 - if (!mr) 911 911 - return ERR_PTR(-ENOMEM); 898 898 + mr = kzalloc(sizeof(*mr), GFP_KERNEL); 899 899 + if (!mr) { 900 900 + err = -ENOMEM; 901 901 + goto err_out; 902 902 + } 903 903 + 904 904 + err = rxe_add_to_pool(&rxe->mr_pool, mr); 905 905 + if (err) 906 906 + goto err_free; 912 907 913 908 rxe_get(pd); 914 909 mr->ibmr.pd = ibpd; ··· 922 905 923 906 err = rxe_mr_init_user(rxe, start, length, iova, access, mr); 924 907 if (err) 925 925 - goto err1; 908 908 + goto err_cleanup; 926 909 927 910 rxe_finalize(mr); 928 928 - 929 911 return &mr->ibmr; 930 912 931 931 - err1: 913 913 + err_cleanup: 932 914 rxe_cleanup(mr); 915 915 + err_free: 916 916 + kfree(mr); 917 917 + err_out: 933 918 return ERR_PTR(err); 934 919 } 935 920 ··· 946 927 if (mr_type != IB_MR_TYPE_MEM_REG) 947 928 return ERR_PTR(-EINVAL); 948 929 949 949 - mr = rxe_alloc(&rxe->mr_pool); 950 950 - if (!mr) 951 951 - return ERR_PTR(-ENOMEM); 930 930 + mr = kzalloc(sizeof(*mr), GFP_KERNEL); 931 931 + if (!mr) { 932 932 + err = -ENOMEM; 933 933 + goto err_out; 934 934 + } 935 935 + 936 936 + err = rxe_add_to_pool(&rxe->mr_pool, mr); 937 937 + if (err) 938 938 + goto err_free; 952 939 953 940 rxe_get(pd); 954 941 mr->ibmr.pd = ibpd; ··· 962 937 963 938 err = rxe_mr_init_fast(max_num_sg, mr); 964 939 if (err) 965 965 - goto err1; 940 940 + goto err_cleanup; 966 941 967 942 rxe_finalize(mr); 968 968 - 969 943 return &mr->ibmr; 970 944 971 971 - err1: 945 945 + err_cleanup: 972 946 rxe_cleanup(mr); 947 947 + err_free: 948 948 + kfree(mr); 949 949 + err_out: 973 950 return ERR_PTR(err); 974 974 - } 975 975 - 976 976 - static int rxe_set_page(struct ib_mr *ibmr, u64 addr) 977 977 - { 978 978 - struct rxe_mr *mr = to_rmr(ibmr); 979 979 - struct rxe_map *map; 980 980 - struct rxe_phys_buf *buf; 981 981 - 982 982 - if (unlikely(mr->nbuf == mr->num_buf)) 983 983 - return -ENOMEM; 984 984 - 985 985 - map = mr->map[mr->nbuf / RXE_BUF_PER_MAP]; 986 986 - buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP]; 987 987 - 988 988 - buf->addr = addr; 989 989 - buf->size = ibmr->page_size; 990 990 - mr->nbuf++; 991 991 - 992 992 - return 0; 993 993 - } 994 994 - 995 995 - static int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 996 996 - int sg_nents, unsigned int *sg_offset) 997 997 - { 998 998 - struct rxe_mr *mr = to_rmr(ibmr); 999 999 - int n; 1000 1000 - 1001 1001 - mr->nbuf = 0; 1002 1002 - 1003 1003 - n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page); 1004 1004 - 1005 1005 - mr->page_shift = ilog2(ibmr->page_size); 1006 1006 - mr->page_mask = ibmr->page_size - 1; 1007 1007 - mr->offset = ibmr->iova & mr->page_mask; 1008 1008 - 1009 1009 - return n; 1010 951 } 1011 952 1012 953 static ssize_t parent_show(struct device *device,

+10 -22

drivers/infiniband/sw/rxe/rxe_verbs.h

reviewed

··· 283 283 RXE_LOOKUP_REMOTE, 284 284 }; 285 285 286 286 - #define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf)) 287 287 - 288 288 - struct rxe_phys_buf { 289 289 - u64 addr; 290 290 - u64 size; 291 291 - }; 292 292 - 293 293 - struct rxe_map { 294 294 - struct rxe_phys_buf buf[RXE_BUF_PER_MAP]; 295 295 - }; 296 296 - 297 286 static inline int rkey_is_mw(u32 rkey) 298 287 { 299 288 u32 index = rkey >> 8; ··· 299 310 u32 lkey; 300 311 u32 rkey; 301 312 enum rxe_mr_state state; 302 302 - u32 offset; 303 313 int access; 314 314 + atomic_t num_mw; 304 315 305 305 - int page_shift; 306 306 - int page_mask; 307 307 - int map_shift; 308 308 - int map_mask; 316 316 + unsigned int page_offset; 317 317 + unsigned int page_shift; 318 318 + u64 page_mask; 309 319 310 320 u32 num_buf; 311 321 u32 nbuf; 312 322 313 313 - u32 max_buf; 314 314 - u32 num_map; 315 315 - 316 316 - atomic_t num_mw; 317 317 - 318 318 - struct rxe_map **map; 323 323 + struct xarray page_list; 319 324 }; 325 325 + 326 326 + static inline unsigned int mr_page_size(struct rxe_mr *mr) 327 327 + { 328 328 + return mr ? mr->ibmr.page_size : PAGE_SIZE; 329 329 + } 320 330 321 331 enum rxe_mw_state { 322 332 RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID,

+12 -11

drivers/infiniband/sw/siw/siw_mem.c

reviewed

··· 398 398 399 399 mlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 400 400 401 401 - if (num_pages + atomic64_read(&mm_s->pinned_vm) > mlock_limit) { 401 401 + if (atomic64_add_return(num_pages, &mm_s->pinned_vm) > mlock_limit) { 402 402 rv = -ENOMEM; 403 403 goto out_sem_up; 404 404 } ··· 411 411 goto out_sem_up; 412 412 } 413 413 for (i = 0; num_pages; i++) { 414 414 - int got, nents = min_t(int, num_pages, PAGES_PER_CHUNK); 415 415 - 416 416 - umem->page_chunk[i].plist = 414 414 + int nents = min_t(int, num_pages, PAGES_PER_CHUNK); 415 415 + struct page **plist = 417 416 kcalloc(nents, sizeof(struct page *), GFP_KERNEL); 418 418 - if (!umem->page_chunk[i].plist) { 417 417 + 418 418 + if (!plist) { 419 419 rv = -ENOMEM; 420 420 goto out_sem_up; 421 421 } 422 422 - got = 0; 422 422 + umem->page_chunk[i].plist = plist; 423 423 while (nents) { 424 424 - struct page **plist = &umem->page_chunk[i].plist[got]; 425 425 - 426 424 rv = pin_user_pages(first_page_va, nents, foll_flags, 427 425 plist, NULL); 428 426 if (rv < 0) 429 427 goto out_sem_up; 430 428 431 429 umem->num_pages += rv; 432 432 - atomic64_add(rv, &mm_s->pinned_vm); 433 430 first_page_va += rv * PAGE_SIZE; 431 431 + plist += rv; 434 432 nents -= rv; 435 435 - got += rv; 433 433 + num_pages -= rv; 436 434 } 437 437 - num_pages -= got; 438 435 } 439 436 out_sem_up: 440 437 mmap_read_unlock(mm_s); 441 438 442 439 if (rv > 0) 443 440 return umem; 441 441 + 442 442 + /* Adjust accounting for pages not pinned */ 443 443 + if (num_pages) 444 444 + atomic64_sub(num_pages, &mm_s->pinned_vm); 444 445 445 446 siw_umem_release(umem, false); 446 447

+1 -1

drivers/infiniband/ulp/ipoib/ipoib_main.c

reviewed

··· 742 742 743 743 static void path_rec_completion(int status, 744 744 struct sa_path_rec *pathrec, 745 745 - int num_prs, void *path_ptr) 745 745 + unsigned int num_prs, void *path_ptr) 746 746 { 747 747 struct ipoib_path *path = path_ptr; 748 748 struct net_device *dev = path->dev;

+1 -1

drivers/infiniband/ulp/srp/ib_srp.c

reviewed

··· 699 699 700 700 static void srp_path_rec_completion(int status, 701 701 struct sa_path_rec *pathrec, 702 702 - int num_paths, void *ch_ptr) 702 702 + unsigned int num_paths, void *ch_ptr) 703 703 { 704 704 struct srp_rdma_ch *ch = ch_ptr; 705 705 struct srp_target_port *target = ch->target;

+9 -5

drivers/net/ethernet/mellanox/mlx4/qp.c

reviewed

··· 46 46 #define MLX4_BF_QP_SKIP_MASK 0xc0 47 47 #define MLX4_MAX_BF_QP_RANGE 0x40 48 48 49 49 + void mlx4_put_qp(struct mlx4_qp *qp) 50 50 + { 51 51 + if (refcount_dec_and_test(&qp->refcount)) 52 52 + complete(&qp->free); 53 53 + } 54 54 + EXPORT_SYMBOL_GPL(mlx4_put_qp); 55 55 + 49 56 void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type) 50 57 { 51 58 struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; ··· 71 64 return; 72 65 } 73 66 67 67 + /* Need to call mlx4_put_qp() in event handler */ 74 68 qp->event(qp, event_type); 75 75 - 76 76 - if (refcount_dec_and_test(&qp->refcount)) 77 77 - complete(&qp->free); 78 69 } 79 70 80 71 /* used for INIT/CLOSE port logic */ ··· 528 523 529 524 void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp) 530 525 { 531 531 - if (refcount_dec_and_test(&qp->refcount)) 532 532 - complete(&qp->free); 526 526 + mlx4_put_qp(qp); 533 527 wait_for_completion(&qp->free); 534 528 535 529 mlx4_qp_free_icm(dev, qp->qpn);

+21 -1

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

reviewed

··· 668 668 mlx5e_rq_shampo_hd_free(rq); 669 669 } 670 670 671 671 + static __be32 mlx5e_get_terminate_scatter_list_mkey(struct mlx5_core_dev *dev) 672 672 + { 673 673 + u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {}; 674 674 + u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {}; 675 675 + int res; 676 676 + 677 677 + if (!MLX5_CAP_GEN(dev, terminate_scatter_list_mkey)) 678 678 + return MLX5_TERMINATE_SCATTER_LIST_LKEY; 679 679 + 680 680 + MLX5_SET(query_special_contexts_in, in, opcode, 681 681 + MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS); 682 682 + res = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out); 683 683 + if (res) 684 684 + return MLX5_TERMINATE_SCATTER_LIST_LKEY; 685 685 + 686 686 + res = MLX5_GET(query_special_contexts_out, out, 687 687 + terminate_scatter_list_mkey); 688 688 + return cpu_to_be32(res); 689 689 + } 690 690 + 671 691 static int mlx5e_alloc_rq(struct mlx5e_params *params, 672 692 struct mlx5e_xsk_param *xsk, 673 693 struct mlx5e_rq_param *rqp, ··· 852 832 /* check if num_frags is not a pow of two */ 853 833 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { 854 834 wqe->data[f].byte_count = 0; 855 855 - wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 835 835 + wqe->data[f].lkey = mlx5e_get_terminate_scatter_list_mkey(mdev); 856 836 wqe->data[f].addr = 0; 857 837 } 858 838 }

+1

include/linux/mlx4/qp.h

reviewed

··· 504 504 505 505 u16 mlx4_qp_roce_entropy(struct mlx4_dev *dev, u32 qpn); 506 506 507 507 + void mlx4_put_qp(struct mlx4_qp *qp); 507 508 #endif /* MLX4_QP_H */

-1

include/linux/mlx5/driver.h

reviewed

··· 742 742 743 743 enum { 744 744 MKEY_CACHE_LAST_STD_ENTRY = 20, 745 745 - MLX5_IMR_MTT_CACHE_ENTRY, 746 745 MLX5_IMR_KSM_CACHE_ENTRY, 747 746 MAX_MKEY_CACHE_ENTRIES 748 747 };

+20 -2

include/linux/mlx5/mlx5_ifc.h

reviewed

··· 1510 1510 u8 relaxed_ordering_write[0x1]; 1511 1511 u8 relaxed_ordering_read[0x1]; 1512 1512 u8 log_max_mkey[0x6]; 1513 1513 - u8 reserved_at_f0[0x8]; 1513 1513 + u8 reserved_at_f0[0x6]; 1514 1514 + u8 terminate_scatter_list_mkey[0x1]; 1515 1515 + u8 repeated_mkey[0x1]; 1514 1516 u8 dump_fill_mkey[0x1]; 1515 1517 u8 reserved_at_f9[0x2]; 1516 1518 u8 fast_teardown[0x1]; ··· 2183 2181 u8 initial_alpha_value[0x20]; 2184 2182 2185 2183 u8 reserved_at_360[0x4a0]; 2184 2184 + }; 2185 2185 + 2186 2186 + struct mlx5_ifc_cong_control_r_roce_general_bits { 2187 2187 + u8 reserved_at_0[0x80]; 2188 2188 + 2189 2189 + u8 reserved_at_80[0x10]; 2190 2190 + u8 rtt_resp_dscp_valid[0x1]; 2191 2191 + u8 reserved_at_91[0x9]; 2192 2192 + u8 rtt_resp_dscp[0x6]; 2193 2193 + 2194 2194 + u8 reserved_at_a0[0x760]; 2186 2195 }; 2187 2196 2188 2197 struct mlx5_ifc_cong_control_802_1qau_rp_bits { ··· 4366 4353 struct mlx5_ifc_cong_control_802_1qau_rp_bits cong_control_802_1qau_rp; 4367 4354 struct mlx5_ifc_cong_control_r_roce_ecn_rp_bits cong_control_r_roce_ecn_rp; 4368 4355 struct mlx5_ifc_cong_control_r_roce_ecn_np_bits cong_control_r_roce_ecn_np; 4356 4356 + struct mlx5_ifc_cong_control_r_roce_general_bits cong_control_r_roce_general; 4369 4357 u8 reserved_at_0[0x800]; 4370 4358 }; 4371 4359 ··· 5275 5261 5276 5262 u8 null_mkey[0x20]; 5277 5263 5278 5278 - u8 reserved_at_a0[0x60]; 5264 5264 + u8 terminate_scatter_list_mkey[0x20]; 5265 5265 + 5266 5266 + u8 repeated_mkey[0x20]; 5267 5267 + 5268 5268 + u8 reserved_at_a0[0x20]; 5279 5269 }; 5280 5270 5281 5271 struct mlx5_ifc_query_special_contexts_in_bits {

+1 -1

include/linux/mlx5/qp.h

reviewed

··· 36 36 #include <linux/mlx5/device.h> 37 37 #include <linux/mlx5/driver.h> 38 38 39 39 - #define MLX5_INVALID_LKEY 0x100 39 39 + #define MLX5_TERMINATE_SCATTER_LIST_LKEY cpu_to_be32(0x100) 40 40 /* UMR (3 WQE_BB's) + SIG (3 WQE_BB's) + PSV (mem) + PSV (wire) */ 41 41 #define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 8) 42 42 #define MLX5_DIF_SIZE 8

+1 -1

include/rdma/ib_sa.h

reviewed

··· 414 414 ib_sa_comp_mask comp_mask, unsigned long timeout_ms, 415 415 gfp_t gfp_mask, 416 416 void (*callback)(int status, struct sa_path_rec *resp, 417 417 - int num_prs, void *context), 417 417 + unsigned int num_prs, void *context), 418 418 void *context, struct ib_sa_query **query); 419 419 420 420 struct ib_sa_multicast {

-1

include/rdma/ib_umem.h

reviewed

··· 25 25 u32 writable : 1; 26 26 u32 is_odp : 1; 27 27 u32 is_dmabuf : 1; 28 28 - struct work_struct work; 29 28 struct sg_append_table sgt_append; 30 29 }; 31 30

+1 -1

include/rdma/ib_verbs.h

reviewed

··· 1168 1168 */ 1169 1169 1170 1170 struct ib_qp_init_attr { 1171 1171 - /* Consumer's event_handler callback must not block */ 1171 1171 + /* This callback occurs in workqueue context */ 1172 1172 void (*event_handler)(struct ib_event *, void *); 1173 1173 1174 1174 void *qp_context;

-1

include/rdma/rdma_cm.h

reviewed

··· 49 49 struct rdma_dev_addr dev_addr; 50 50 }; 51 51 52 52 - #define RDMA_PRIMARY_PATH_MAX_REC_NUM 3 53 52 struct rdma_route { 54 53 struct rdma_addr addr; 55 54 struct sa_path_rec *path_rec;

+2 -2

include/rdma/restrack.h

reviewed

··· 162 162 * rdma_restrack_no_track() - don't add resource to the DB 163 163 * @res: resource entry 164 164 * 165 165 - * Every user of thie API should be cross examined. 166 166 - * Probaby you don't need to use this function. 165 165 + * Every user of this API should be cross examined. 166 166 + * Probably you don't need to use this function. 167 167 */ 168 168 static inline void rdma_restrack_no_track(struct rdma_restrack_entry *res) 169 169 {

+4

include/uapi/rdma/hns-abi.h

reviewed

··· 87 87 88 88 enum { 89 89 HNS_ROCE_EXSGE_FLAGS = 1 << 0, 90 90 + HNS_ROCE_RQ_INLINE_FLAGS = 1 << 1, 91 91 + HNS_ROCE_CQE_INLINE_FLAGS = 1 << 2, 90 92 }; 91 93 92 94 enum { 93 95 HNS_ROCE_RSP_EXSGE_FLAGS = 1 << 0, 96 96 + HNS_ROCE_RSP_RQ_INLINE_FLAGS = 1 << 1, 97 97 + HNS_ROCE_RSP_CQE_INLINE_FLAGS = 1 << 2, 94 98 }; 95 99 96 100 struct hns_roce_ib_alloc_ucontext_resp {