Merge tag 'v6.18-rc-part1-smb3-common' of git://git.samba.org/ksmbd

+40 -41

fs/smb/client/cifs_debug.c

··· 24 24 #endif 25 25 #ifdef CONFIG_CIFS_SMB_DIRECT 26 26 #include "smbdirect.h" 27 + #include "../common/smbdirect/smbdirect_pdu.h" 27 28 #endif 28 29 #include "cifs_swn.h" 29 30 #include "cached_dir.h" ··· 457 456 sc = &server->smbd_conn->socket; 458 457 sp = &sc->parameters; 459 458 460 - seq_printf(m, "\nSMBDirect (in hex) protocol version: %x " 461 - "transport status: %x", 462 - server->smbd_conn->protocol, 463 - server->smbd_conn->socket.status); 464 - seq_printf(m, "\nConn receive_credit_max: %x " 465 - "send_credit_target: %x max_send_size: %x", 459 + seq_printf(m, "\nSMBDirect protocol version: 0x%x " 460 + "transport status: %s (%u)", 461 + SMBDIRECT_V1, 462 + smbdirect_socket_status_string(sc->status), 463 + sc->status); 464 + seq_printf(m, "\nConn receive_credit_max: %u " 465 + "send_credit_target: %u max_send_size: %u", 466 466 sp->recv_credit_max, 467 467 sp->send_credit_target, 468 468 sp->max_send_size); 469 - seq_printf(m, "\nConn max_fragmented_recv_size: %x " 470 - "max_fragmented_send_size: %x max_receive_size:%x", 469 + seq_printf(m, "\nConn max_fragmented_recv_size: %u " 470 + "max_fragmented_send_size: %u max_receive_size:%u", 471 471 sp->max_fragmented_recv_size, 472 472 sp->max_fragmented_send_size, 473 473 sp->max_recv_size); 474 - seq_printf(m, "\nConn keep_alive_interval: %x " 475 - "max_readwrite_size: %x rdma_readwrite_threshold: %x", 474 + seq_printf(m, "\nConn keep_alive_interval: %u " 475 + "max_readwrite_size: %u rdma_readwrite_threshold: %u", 476 476 sp->keepalive_interval_msec * 1000, 477 477 sp->max_read_write_size, 478 - server->smbd_conn->rdma_readwrite_threshold); 479 - seq_printf(m, "\nDebug count_get_receive_buffer: %x " 480 - "count_put_receive_buffer: %x count_send_empty: %x", 481 - server->smbd_conn->count_get_receive_buffer, 482 - server->smbd_conn->count_put_receive_buffer, 483 - server->smbd_conn->count_send_empty); 484 - seq_printf(m, "\nRead Queue count_reassembly_queue: %x " 485 - "count_enqueue_reassembly_queue: %x " 486 - "count_dequeue_reassembly_queue: %x " 487 - "reassembly_data_length: %x " 488 - "reassembly_queue_length: %x", 489 - server->smbd_conn->count_reassembly_queue, 490 - server->smbd_conn->count_enqueue_reassembly_queue, 491 - server->smbd_conn->count_dequeue_reassembly_queue, 478 + server->rdma_readwrite_threshold); 479 + seq_printf(m, "\nDebug count_get_receive_buffer: %llu " 480 + "count_put_receive_buffer: %llu count_send_empty: %llu", 481 + sc->statistics.get_receive_buffer, 482 + sc->statistics.put_receive_buffer, 483 + sc->statistics.send_empty); 484 + seq_printf(m, "\nRead Queue " 485 + "count_enqueue_reassembly_queue: %llu " 486 + "count_dequeue_reassembly_queue: %llu " 487 + "reassembly_data_length: %u " 488 + "reassembly_queue_length: %u", 489 + sc->statistics.enqueue_reassembly_queue, 490 + sc->statistics.dequeue_reassembly_queue, 492 491 sc->recv_io.reassembly.data_length, 493 492 sc->recv_io.reassembly.queue_length); 494 - seq_printf(m, "\nCurrent Credits send_credits: %x " 495 - "receive_credits: %x receive_credit_target: %x", 496 - atomic_read(&server->smbd_conn->send_credits), 497 - atomic_read(&server->smbd_conn->receive_credits), 498 - server->smbd_conn->receive_credit_target); 499 - seq_printf(m, "\nPending send_pending: %x ", 500 - atomic_read(&server->smbd_conn->send_pending)); 501 - seq_printf(m, "\nReceive buffers count_receive_queue: %x ", 502 - server->smbd_conn->count_receive_queue); 503 - seq_printf(m, "\nMR responder_resources: %x " 504 - "max_frmr_depth: %x mr_type: %x", 505 - server->smbd_conn->responder_resources, 506 - server->smbd_conn->max_frmr_depth, 507 - server->smbd_conn->mr_type); 508 - seq_printf(m, "\nMR mr_ready_count: %x mr_used_count: %x", 509 - atomic_read(&server->smbd_conn->mr_ready_count), 510 - atomic_read(&server->smbd_conn->mr_used_count)); 493 + seq_printf(m, "\nCurrent Credits send_credits: %u " 494 + "receive_credits: %u receive_credit_target: %u", 495 + atomic_read(&sc->send_io.credits.count), 496 + atomic_read(&sc->recv_io.credits.count), 497 + sc->recv_io.credits.target); 498 + seq_printf(m, "\nPending send_pending: %u ", 499 + atomic_read(&sc->send_io.pending.count)); 500 + seq_printf(m, "\nMR responder_resources: %u " 501 + "max_frmr_depth: %u mr_type: 0x%x", 502 + sp->responder_resources, 503 + sp->max_frmr_depth, 504 + sc->mr_io.type); 505 + seq_printf(m, "\nMR mr_ready_count: %u mr_used_count: %u", 506 + atomic_read(&sc->mr_io.ready.count), 507 + atomic_read(&sc->mr_io.used.count)); 511 508 skip_rdma: 512 509 #endif 513 510 seq_printf(m, "\nNumber of credits: %d,%d,%d Dialect 0x%x",

+8 -1

fs/smb/client/cifsglob.h

··· 814 814 unsigned int max_read; 815 815 unsigned int max_write; 816 816 unsigned int min_offload; 817 + /* 818 + * If payload is less than or equal to the threshold, 819 + * use RDMA send/recv to send upper layer I/O. 820 + * If payload is more than the threshold, 821 + * use RDMA read/write through memory registration for I/O. 822 + */ 823 + unsigned int rdma_readwrite_threshold; 817 824 unsigned int retrans; 818 825 struct { 819 826 bool requested; /* "compress" mount option set*/ ··· 1547 1540 struct kvec iov[2]; 1548 1541 struct TCP_Server_Info *server; 1549 1542 #ifdef CONFIG_CIFS_SMB_DIRECT 1550 - struct smbd_mr *mr; 1543 + struct smbdirect_mr_io *mr; 1551 1544 #endif 1552 1545 struct cifs_credits credits; 1553 1546 };

+12 -4

fs/smb/client/file.c

··· 97 97 cifs_trace_rw_credits_write_prepare); 98 98 99 99 #ifdef CONFIG_CIFS_SMB_DIRECT 100 - if (server->smbd_conn) 101 - stream->sreq_max_segs = server->smbd_conn->max_frmr_depth; 100 + if (server->smbd_conn) { 101 + const struct smbdirect_socket_parameters *sp = 102 + smbd_get_parameters(server->smbd_conn); 103 + 104 + stream->sreq_max_segs = sp->max_frmr_depth; 105 + } 102 106 #endif 103 107 } 104 108 ··· 191 187 cifs_trace_rw_credits_read_submit); 192 188 193 189 #ifdef CONFIG_CIFS_SMB_DIRECT 194 - if (server->smbd_conn) 195 - rreq->io_streams[0].sreq_max_segs = server->smbd_conn->max_frmr_depth; 190 + if (server->smbd_conn) { 191 + const struct smbdirect_socket_parameters *sp = 192 + smbd_get_parameters(server->smbd_conn); 193 + 194 + rreq->io_streams[0].sreq_max_segs = sp->max_frmr_depth; 195 + } 196 196 #endif 197 197 return 0; 198 198 }

+4 -4

fs/smb/client/smb2ops.c

··· 504 504 wsize = min_t(unsigned int, wsize, server->max_write); 505 505 #ifdef CONFIG_CIFS_SMB_DIRECT 506 506 if (server->rdma) { 507 - struct smbdirect_socket_parameters *sp = 508 - &server->smbd_conn->socket.parameters; 507 + const struct smbdirect_socket_parameters *sp = 508 + smbd_get_parameters(server->smbd_conn); 509 509 510 510 if (server->sign) 511 511 /* ··· 555 555 rsize = min_t(unsigned int, rsize, server->max_read); 556 556 #ifdef CONFIG_CIFS_SMB_DIRECT 557 557 if (server->rdma) { 558 - struct smbdirect_socket_parameters *sp = 559 - &server->smbd_conn->socket.parameters; 558 + const struct smbdirect_socket_parameters *sp = 559 + smbd_get_parameters(server->smbd_conn); 560 560 561 561 if (server->sign) 562 562 /*

+1 -1

fs/smb/client/smb2pdu.c

··· 4411 4411 return false; 4412 4412 4413 4413 /* offload also has its overhead, so only do it if desired */ 4414 - if (io_parms->length < server->smbd_conn->rdma_readwrite_threshold) 4414 + if (io_parms->length < server->rdma_readwrite_threshold) 4415 4415 return false; 4416 4416 4417 4417 return true;

+745 -480

fs/smb/client/smbdirect.c

··· 13 13 #include "cifsproto.h" 14 14 #include "smb2proto.h" 15 15 16 + const struct smbdirect_socket_parameters *smbd_get_parameters(struct smbd_connection *conn) 17 + { 18 + struct smbdirect_socket *sc = &conn->socket; 19 + 20 + return &sc->parameters; 21 + } 22 + 16 23 static struct smbdirect_recv_io *get_receive_buffer( 17 - struct smbd_connection *info); 24 + struct smbdirect_socket *sc); 18 25 static void put_receive_buffer( 19 - struct smbd_connection *info, 26 + struct smbdirect_socket *sc, 20 27 struct smbdirect_recv_io *response); 21 - static int allocate_receive_buffers(struct smbd_connection *info, int num_buf); 22 - static void destroy_receive_buffers(struct smbd_connection *info); 28 + static int allocate_receive_buffers(struct smbdirect_socket *sc, int num_buf); 29 + static void destroy_receive_buffers(struct smbdirect_socket *sc); 23 30 24 31 static void enqueue_reassembly( 25 - struct smbd_connection *info, 32 + struct smbdirect_socket *sc, 26 33 struct smbdirect_recv_io *response, int data_length); 27 34 static struct smbdirect_recv_io *_get_first_reassembly( 28 - struct smbd_connection *info); 35 + struct smbdirect_socket *sc); 29 36 30 37 static int smbd_post_recv( 31 - struct smbd_connection *info, 38 + struct smbdirect_socket *sc, 32 39 struct smbdirect_recv_io *response); 33 40 34 - static int smbd_post_send_empty(struct smbd_connection *info); 41 + static int smbd_post_send_empty(struct smbdirect_socket *sc); 35 42 36 - static void destroy_mr_list(struct smbd_connection *info); 37 - static int allocate_mr_list(struct smbd_connection *info); 43 + static void destroy_mr_list(struct smbdirect_socket *sc); 44 + static int allocate_mr_list(struct smbdirect_socket *sc); 38 45 39 46 struct smb_extract_to_rdma { 40 47 struct ib_sge *sge; ··· 63 56 64 57 /* SMBD negotiation timeout in seconds */ 65 58 #define SMBD_NEGOTIATE_TIMEOUT 120 59 + 60 + /* The timeout to wait for a keepalive message from peer in seconds */ 61 + #define KEEPALIVE_RECV_TIMEOUT 5 66 62 67 63 /* SMBD minimum receive size and fragmented sized defined in [MS-SMBD] */ 68 64 #define SMBD_MIN_RECEIVE_SIZE 128 ··· 165 155 #define log_rdma_mr(level, fmt, args...) \ 166 156 log_rdma(level, LOG_RDMA_MR, fmt, ##args) 167 157 168 - static void smbd_disconnect_rdma_work(struct work_struct *work) 158 + static void smbd_disconnect_wake_up_all(struct smbdirect_socket *sc) 169 159 { 170 - struct smbd_connection *info = 171 - container_of(work, struct smbd_connection, disconnect_work); 172 - struct smbdirect_socket *sc = &info->socket; 173 - 174 - if (sc->status == SMBDIRECT_SOCKET_CONNECTED) { 175 - sc->status = SMBDIRECT_SOCKET_DISCONNECTING; 176 - rdma_disconnect(sc->rdma.cm_id); 177 - } 160 + /* 161 + * Wake up all waiters in all wait queues 162 + * in order to notice the broken connection. 163 + */ 164 + wake_up_all(&sc->status_wait); 165 + wake_up_all(&sc->send_io.credits.wait_queue); 166 + wake_up_all(&sc->send_io.pending.dec_wait_queue); 167 + wake_up_all(&sc->send_io.pending.zero_wait_queue); 168 + wake_up_all(&sc->recv_io.reassembly.wait_queue); 169 + wake_up_all(&sc->mr_io.ready.wait_queue); 170 + wake_up_all(&sc->mr_io.cleanup.wait_queue); 178 171 } 179 172 180 - static void smbd_disconnect_rdma_connection(struct smbd_connection *info) 173 + static void smbd_disconnect_rdma_work(struct work_struct *work) 181 174 { 182 - queue_work(info->workqueue, &info->disconnect_work); 175 + struct smbdirect_socket *sc = 176 + container_of(work, struct smbdirect_socket, disconnect_work); 177 + 178 + /* 179 + * make sure this and other work is not queued again 180 + * but here we don't block and avoid 181 + * disable[_delayed]_work_sync() 182 + */ 183 + disable_work(&sc->disconnect_work); 184 + disable_work(&sc->recv_io.posted.refill_work); 185 + disable_work(&sc->mr_io.recovery_work); 186 + disable_work(&sc->idle.immediate_work); 187 + disable_delayed_work(&sc->idle.timer_work); 188 + 189 + if (sc->first_error == 0) 190 + sc->first_error = -ECONNABORTED; 191 + 192 + switch (sc->status) { 193 + case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: 194 + case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: 195 + case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: 196 + case SMBDIRECT_SOCKET_CONNECTED: 197 + case SMBDIRECT_SOCKET_ERROR: 198 + sc->status = SMBDIRECT_SOCKET_DISCONNECTING; 199 + rdma_disconnect(sc->rdma.cm_id); 200 + break; 201 + 202 + case SMBDIRECT_SOCKET_CREATED: 203 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: 204 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: 205 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: 206 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: 207 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: 208 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: 209 + case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: 210 + case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: 211 + case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: 212 + /* 213 + * rdma_connect() never reached 214 + * RDMA_CM_EVENT_ESTABLISHED 215 + */ 216 + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; 217 + break; 218 + 219 + case SMBDIRECT_SOCKET_DISCONNECTING: 220 + case SMBDIRECT_SOCKET_DISCONNECTED: 221 + case SMBDIRECT_SOCKET_DESTROYED: 222 + break; 223 + } 224 + 225 + /* 226 + * Wake up all waiters in all wait queues 227 + * in order to notice the broken connection. 228 + */ 229 + smbd_disconnect_wake_up_all(sc); 230 + } 231 + 232 + static void smbd_disconnect_rdma_connection(struct smbdirect_socket *sc) 233 + { 234 + /* 235 + * make sure other work (than disconnect_work) is 236 + * not queued again but here we don't block and avoid 237 + * disable[_delayed]_work_sync() 238 + */ 239 + disable_work(&sc->recv_io.posted.refill_work); 240 + disable_work(&sc->mr_io.recovery_work); 241 + disable_work(&sc->idle.immediate_work); 242 + disable_delayed_work(&sc->idle.timer_work); 243 + 244 + if (sc->first_error == 0) 245 + sc->first_error = -ECONNABORTED; 246 + 247 + switch (sc->status) { 248 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: 249 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: 250 + case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: 251 + case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: 252 + case SMBDIRECT_SOCKET_ERROR: 253 + case SMBDIRECT_SOCKET_DISCONNECTING: 254 + case SMBDIRECT_SOCKET_DISCONNECTED: 255 + case SMBDIRECT_SOCKET_DESTROYED: 256 + /* 257 + * Keep the current error status 258 + */ 259 + break; 260 + 261 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: 262 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: 263 + sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; 264 + break; 265 + 266 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: 267 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: 268 + sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; 269 + break; 270 + 271 + case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: 272 + case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: 273 + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; 274 + break; 275 + 276 + case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: 277 + case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: 278 + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; 279 + break; 280 + 281 + case SMBDIRECT_SOCKET_CREATED: 282 + case SMBDIRECT_SOCKET_CONNECTED: 283 + sc->status = SMBDIRECT_SOCKET_ERROR; 284 + break; 285 + } 286 + 287 + /* 288 + * Wake up all waiters in all wait queues 289 + * in order to notice the broken connection. 290 + */ 291 + smbd_disconnect_wake_up_all(sc); 292 + 293 + queue_work(sc->workqueue, &sc->disconnect_work); 183 294 } 184 295 185 296 /* Upcall from RDMA CM */ 186 297 static int smbd_conn_upcall( 187 298 struct rdma_cm_id *id, struct rdma_cm_event *event) 188 299 { 189 - struct smbd_connection *info = id->context; 190 - struct smbdirect_socket *sc = &info->socket; 300 + struct smbdirect_socket *sc = id->context; 301 + struct smbdirect_socket_parameters *sp = &sc->parameters; 191 302 const char *event_name = rdma_event_msg(event->event); 303 + u8 peer_initiator_depth; 304 + u8 peer_responder_resources; 192 305 193 306 log_rdma_event(INFO, "event=%s status=%d\n", 194 307 event_name, event->status); 195 308 196 309 switch (event->event) { 197 310 case RDMA_CM_EVENT_ADDR_RESOLVED: 311 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING); 312 + sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED; 313 + wake_up(&sc->status_wait); 314 + break; 315 + 198 316 case RDMA_CM_EVENT_ROUTE_RESOLVED: 199 - info->ri_rc = 0; 200 - complete(&info->ri_done); 317 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING); 318 + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; 319 + wake_up(&sc->status_wait); 201 320 break; 202 321 203 322 case RDMA_CM_EVENT_ADDR_ERROR: 204 323 log_rdma_event(ERR, "connecting failed event=%s\n", event_name); 205 - info->ri_rc = -EHOSTUNREACH; 206 - complete(&info->ri_done); 324 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING); 325 + sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; 326 + smbd_disconnect_rdma_work(&sc->disconnect_work); 207 327 break; 208 328 209 329 case RDMA_CM_EVENT_ROUTE_ERROR: 210 330 log_rdma_event(ERR, "connecting failed event=%s\n", event_name); 211 - info->ri_rc = -ENETUNREACH; 212 - complete(&info->ri_done); 331 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING); 332 + sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; 333 + smbd_disconnect_rdma_work(&sc->disconnect_work); 213 334 break; 214 335 215 336 case RDMA_CM_EVENT_ESTABLISHED: 216 337 log_rdma_event(INFO, "connected event=%s\n", event_name); 217 - sc->status = SMBDIRECT_SOCKET_CONNECTED; 218 - wake_up_interruptible(&info->status_wait); 338 + 339 + /* 340 + * Here we work around an inconsistency between 341 + * iWarp and other devices (at least rxe and irdma using RoCEv2) 342 + */ 343 + if (rdma_protocol_iwarp(id->device, id->port_num)) { 344 + /* 345 + * iWarp devices report the peer's values 346 + * with the perspective of the peer here. 347 + * Tested with siw and irdma (in iwarp mode) 348 + * We need to change to our perspective here, 349 + * so we need to switch the values. 350 + */ 351 + peer_initiator_depth = event->param.conn.responder_resources; 352 + peer_responder_resources = event->param.conn.initiator_depth; 353 + } else { 354 + /* 355 + * Non iWarp devices report the peer's values 356 + * already changed to our perspective here. 357 + * Tested with rxe and irdma (in roce mode). 358 + */ 359 + peer_initiator_depth = event->param.conn.initiator_depth; 360 + peer_responder_resources = event->param.conn.responder_resources; 361 + } 362 + if (rdma_protocol_iwarp(id->device, id->port_num) && 363 + event->param.conn.private_data_len == 8) { 364 + /* 365 + * Legacy clients with only iWarp MPA v1 support 366 + * need a private blob in order to negotiate 367 + * the IRD/ORD values. 368 + */ 369 + const __be32 *ird_ord_hdr = event->param.conn.private_data; 370 + u32 ird32 = be32_to_cpu(ird_ord_hdr[0]); 371 + u32 ord32 = be32_to_cpu(ird_ord_hdr[1]); 372 + 373 + /* 374 + * cifs.ko sends the legacy IRD/ORD negotiation 375 + * event if iWarp MPA v2 was used. 376 + * 377 + * Here we check that the values match and only 378 + * mark the client as legacy if they don't match. 379 + */ 380 + if ((u32)event->param.conn.initiator_depth != ird32 || 381 + (u32)event->param.conn.responder_resources != ord32) { 382 + /* 383 + * There are broken clients (old cifs.ko) 384 + * using little endian and also 385 + * struct rdma_conn_param only uses u8 386 + * for initiator_depth and responder_resources, 387 + * so we truncate the value to U8_MAX. 388 + * 389 + * smb_direct_accept_client() will then 390 + * do the real negotiation in order to 391 + * select the minimum between client and 392 + * server. 393 + */ 394 + ird32 = min_t(u32, ird32, U8_MAX); 395 + ord32 = min_t(u32, ord32, U8_MAX); 396 + 397 + sc->rdma.legacy_iwarp = true; 398 + peer_initiator_depth = (u8)ird32; 399 + peer_responder_resources = (u8)ord32; 400 + } 401 + } 402 + 403 + /* 404 + * negotiate the value by using the minimum 405 + * between client and server if the client provided 406 + * non 0 values. 407 + */ 408 + if (peer_initiator_depth != 0) 409 + sp->initiator_depth = 410 + min_t(u8, sp->initiator_depth, 411 + peer_initiator_depth); 412 + if (peer_responder_resources != 0) 413 + sp->responder_resources = 414 + min_t(u8, sp->responder_resources, 415 + peer_responder_resources); 416 + 417 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING); 418 + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; 419 + wake_up(&sc->status_wait); 219 420 break; 220 421 221 422 case RDMA_CM_EVENT_CONNECT_ERROR: 222 423 case RDMA_CM_EVENT_UNREACHABLE: 223 424 case RDMA_CM_EVENT_REJECTED: 224 425 log_rdma_event(ERR, "connecting failed event=%s\n", event_name); 225 - sc->status = SMBDIRECT_SOCKET_DISCONNECTED; 226 - wake_up_interruptible(&info->status_wait); 426 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING); 427 + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; 428 + smbd_disconnect_rdma_work(&sc->disconnect_work); 227 429 break; 228 430 229 431 case RDMA_CM_EVENT_DEVICE_REMOVAL: ··· 443 221 /* This happens when we fail the negotiation */ 444 222 if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_FAILED) { 445 223 log_rdma_event(ERR, "event=%s during negotiation\n", event_name); 446 - sc->status = SMBDIRECT_SOCKET_DISCONNECTED; 447 - wake_up(&info->status_wait); 448 - break; 449 224 } 450 225 451 226 sc->status = SMBDIRECT_SOCKET_DISCONNECTED; 452 - wake_up_interruptible(&info->status_wait); 453 - wake_up_interruptible(&sc->recv_io.reassembly.wait_queue); 454 - wake_up_interruptible_all(&info->wait_send_queue); 227 + smbd_disconnect_rdma_work(&sc->disconnect_work); 455 228 break; 456 229 457 230 default: ··· 462 245 static void 463 246 smbd_qp_async_error_upcall(struct ib_event *event, void *context) 464 247 { 465 - struct smbd_connection *info = context; 248 + struct smbdirect_socket *sc = context; 466 249 467 - log_rdma_event(ERR, "%s on device %s info %p\n", 468 - ib_event_msg(event->event), event->device->name, info); 250 + log_rdma_event(ERR, "%s on device %s socket %p\n", 251 + ib_event_msg(event->event), event->device->name, sc); 469 252 470 253 switch (event->event) { 471 254 case IB_EVENT_CQ_ERR: 472 255 case IB_EVENT_QP_FATAL: 473 - smbd_disconnect_rdma_connection(info); 256 + smbd_disconnect_rdma_connection(sc); 474 257 break; 475 258 476 259 default: ··· 495 278 struct smbdirect_send_io *request = 496 279 container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); 497 280 struct smbdirect_socket *sc = request->socket; 498 - struct smbd_connection *info = 499 - container_of(sc, struct smbd_connection, socket); 500 281 501 - log_rdma_send(INFO, "smbdirect_send_io 0x%p completed wc->status=%d\n", 502 - request, wc->status); 282 + log_rdma_send(INFO, "smbdirect_send_io 0x%p completed wc->status=%s\n", 283 + request, ib_wc_status_msg(wc->status)); 503 284 504 285 for (i = 0; i < request->num_sge; i++) 505 286 ib_dma_unmap_single(sc->ib.dev, ··· 506 291 DMA_TO_DEVICE); 507 292 508 293 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { 509 - log_rdma_send(ERR, "wc->status=%d wc->opcode=%d\n", 510 - wc->status, wc->opcode); 294 + if (wc->status != IB_WC_WR_FLUSH_ERR) 295 + log_rdma_send(ERR, "wc->status=%s wc->opcode=%d\n", 296 + ib_wc_status_msg(wc->status), wc->opcode); 511 297 mempool_free(request, sc->send_io.mem.pool); 512 - smbd_disconnect_rdma_connection(info); 298 + smbd_disconnect_rdma_connection(sc); 513 299 return; 514 300 } 515 301 516 - if (atomic_dec_and_test(&info->send_pending)) 517 - wake_up(&info->wait_send_pending); 302 + if (atomic_dec_and_test(&sc->send_io.pending.count)) 303 + wake_up(&sc->send_io.pending.zero_wait_queue); 518 304 519 - wake_up(&info->wait_post_send); 305 + wake_up(&sc->send_io.pending.dec_wait_queue); 520 306 521 307 mempool_free(request, sc->send_io.mem.pool); 522 308 } ··· 541 325 struct smbdirect_recv_io *response, int packet_length) 542 326 { 543 327 struct smbdirect_socket *sc = response->socket; 544 - struct smbd_connection *info = 545 - container_of(sc, struct smbd_connection, socket); 546 328 struct smbdirect_socket_parameters *sp = &sc->parameters; 547 329 struct smbdirect_negotiate_resp *packet = smbdirect_recv_io_payload(response); 548 330 ··· 555 341 le16_to_cpu(packet->negotiated_version)); 556 342 return false; 557 343 } 558 - info->protocol = le16_to_cpu(packet->negotiated_version); 559 344 560 345 if (packet->credits_requested == 0) { 561 346 log_rdma_event(ERR, "error: credits_requested==0\n"); 562 347 return false; 563 348 } 564 - info->receive_credit_target = le16_to_cpu(packet->credits_requested); 349 + sc->recv_io.credits.target = le16_to_cpu(packet->credits_requested); 350 + sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max); 565 351 566 352 if (packet->credits_granted == 0) { 567 353 log_rdma_event(ERR, "error: credits_granted==0\n"); 568 354 return false; 569 355 } 570 - atomic_set(&info->send_credits, le16_to_cpu(packet->credits_granted)); 571 - 572 - atomic_set(&info->receive_credits, 0); 356 + atomic_set(&sc->send_io.credits.count, le16_to_cpu(packet->credits_granted)); 573 357 574 358 if (le32_to_cpu(packet->preferred_send_size) > sp->max_recv_size) { 575 359 log_rdma_event(ERR, "error: preferred_send_size=%d\n", ··· 592 380 } 593 381 sp->max_fragmented_send_size = 594 382 le32_to_cpu(packet->max_fragmented_size); 595 - info->rdma_readwrite_threshold = 596 - rdma_readwrite_threshold > sp->max_fragmented_send_size ? 597 - sp->max_fragmented_send_size : 598 - rdma_readwrite_threshold; 599 383 600 384 601 385 sp->max_read_write_size = min_t(u32, 602 386 le32_to_cpu(packet->max_readwrite_size), 603 - info->max_frmr_depth * PAGE_SIZE); 604 - info->max_frmr_depth = sp->max_read_write_size / PAGE_SIZE; 387 + sp->max_frmr_depth * PAGE_SIZE); 388 + sp->max_frmr_depth = sp->max_read_write_size / PAGE_SIZE; 605 389 606 390 sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; 607 391 return true; ··· 605 397 606 398 static void smbd_post_send_credits(struct work_struct *work) 607 399 { 608 - int ret = 0; 609 400 int rc; 610 401 struct smbdirect_recv_io *response; 611 - struct smbd_connection *info = 612 - container_of(work, struct smbd_connection, 613 - post_send_credits_work); 614 - struct smbdirect_socket *sc = &info->socket; 402 + struct smbdirect_socket *sc = 403 + container_of(work, struct smbdirect_socket, recv_io.posted.refill_work); 615 404 616 405 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { 617 - wake_up(&info->wait_receive_queues); 618 406 return; 619 407 } 620 408 621 - if (info->receive_credit_target > 622 - atomic_read(&info->receive_credits)) { 409 + if (sc->recv_io.credits.target > 410 + atomic_read(&sc->recv_io.credits.count)) { 623 411 while (true) { 624 - response = get_receive_buffer(info); 412 + response = get_receive_buffer(sc); 625 413 if (!response) 626 414 break; 627 415 628 416 response->first_segment = false; 629 - rc = smbd_post_recv(info, response); 417 + rc = smbd_post_recv(sc, response); 630 418 if (rc) { 631 419 log_rdma_recv(ERR, 632 420 "post_recv failed rc=%d\n", rc); 633 - put_receive_buffer(info, response); 421 + put_receive_buffer(sc, response); 634 422 break; 635 423 } 636 424 637 - ret++; 425 + atomic_inc(&sc->recv_io.posted.count); 638 426 } 639 427 } 640 428 641 - spin_lock(&info->lock_new_credits_offered); 642 - info->new_credits_offered += ret; 643 - spin_unlock(&info->lock_new_credits_offered); 644 - 645 429 /* Promptly send an immediate packet as defined in [MS-SMBD] 3.1.1.1 */ 646 - info->send_immediate = true; 647 - if (atomic_read(&info->receive_credits) < 648 - info->receive_credit_target - 1) { 649 - if (info->keep_alive_requested == KEEP_ALIVE_PENDING || 650 - info->send_immediate) { 651 - log_keep_alive(INFO, "send an empty message\n"); 652 - smbd_post_send_empty(info); 653 - } 430 + if (atomic_read(&sc->recv_io.credits.count) < 431 + sc->recv_io.credits.target - 1) { 432 + log_keep_alive(INFO, "schedule send of an empty message\n"); 433 + queue_work(sc->workqueue, &sc->idle.immediate_work); 654 434 } 655 435 } 656 436 ··· 650 454 container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); 651 455 struct smbdirect_socket *sc = response->socket; 652 456 struct smbdirect_socket_parameters *sp = &sc->parameters; 653 - struct smbd_connection *info = 654 - container_of(sc, struct smbd_connection, socket); 457 + u16 old_recv_credit_target; 655 458 u32 data_offset = 0; 656 459 u32 data_length = 0; 657 460 u32 remaining_data_length = 0; 461 + bool negotiate_done = false; 658 462 659 - log_rdma_recv(INFO, "response=0x%p type=%d wc status=%d wc opcode %d byte_len=%d pkey_index=%u\n", 660 - response, sc->recv_io.expected, wc->status, wc->opcode, 463 + log_rdma_recv(INFO, 464 + "response=0x%p type=%d wc status=%s wc opcode %d byte_len=%d pkey_index=%u\n", 465 + response, sc->recv_io.expected, 466 + ib_wc_status_msg(wc->status), wc->opcode, 661 467 wc->byte_len, wc->pkey_index); 662 468 663 469 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { 664 - log_rdma_recv(INFO, "wc->status=%d opcode=%d\n", 665 - wc->status, wc->opcode); 470 + if (wc->status != IB_WC_WR_FLUSH_ERR) 471 + log_rdma_recv(ERR, "wc->status=%s opcode=%d\n", 472 + ib_wc_status_msg(wc->status), wc->opcode); 666 473 goto error; 667 474 } 668 475 ··· 675 476 response->sge.length, 676 477 DMA_FROM_DEVICE); 677 478 479 + /* 480 + * Reset timer to the keepalive interval in 481 + * order to trigger our next keepalive message. 482 + */ 483 + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; 484 + mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 485 + msecs_to_jiffies(sp->keepalive_interval_msec)); 486 + 678 487 switch (sc->recv_io.expected) { 679 488 /* SMBD negotiation response */ 680 489 case SMBDIRECT_EXPECT_NEGOTIATE_REP: 681 490 dump_smbdirect_negotiate_resp(smbdirect_recv_io_payload(response)); 682 491 sc->recv_io.reassembly.full_packet_received = true; 683 - info->negotiate_done = 492 + negotiate_done = 684 493 process_negotiation_response(response, wc->byte_len); 685 - put_receive_buffer(info, response); 686 - complete(&info->negotiate_completion); 494 + put_receive_buffer(sc, response); 495 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING); 496 + if (!negotiate_done) { 497 + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; 498 + smbd_disconnect_rdma_connection(sc); 499 + } else { 500 + sc->status = SMBDIRECT_SOCKET_CONNECTED; 501 + wake_up(&sc->status_wait); 502 + } 503 + 687 504 return; 688 505 689 506 /* SMBD data transfer packet */ ··· 732 517 sc->recv_io.reassembly.full_packet_received = true; 733 518 } 734 519 735 - atomic_dec(&info->receive_credits); 736 - info->receive_credit_target = 520 + atomic_dec(&sc->recv_io.posted.count); 521 + atomic_dec(&sc->recv_io.credits.count); 522 + old_recv_credit_target = sc->recv_io.credits.target; 523 + sc->recv_io.credits.target = 737 524 le16_to_cpu(data_transfer->credits_requested); 525 + sc->recv_io.credits.target = 526 + min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max); 527 + sc->recv_io.credits.target = 528 + max_t(u16, sc->recv_io.credits.target, 1); 738 529 if (le16_to_cpu(data_transfer->credits_granted)) { 739 530 atomic_add(le16_to_cpu(data_transfer->credits_granted), 740 - &info->send_credits); 531 + &sc->send_io.credits.count); 741 532 /* 742 533 * We have new send credits granted from remote peer 743 534 * If any sender is waiting for credits, unblock it 744 535 */ 745 - wake_up_interruptible(&info->wait_send_queue); 536 + wake_up(&sc->send_io.credits.wait_queue); 746 537 } 747 538 748 539 log_incoming(INFO, "data flags %d data_offset %d data_length %d remaining_data_length %d\n", ··· 757 536 le32_to_cpu(data_transfer->data_length), 758 537 le32_to_cpu(data_transfer->remaining_data_length)); 759 538 760 - /* Send a KEEP_ALIVE response right away if requested */ 761 - info->keep_alive_requested = KEEP_ALIVE_NONE; 539 + /* Send an immediate response right away if requested */ 762 540 if (le16_to_cpu(data_transfer->flags) & 763 541 SMBDIRECT_FLAG_RESPONSE_REQUESTED) { 764 - info->keep_alive_requested = KEEP_ALIVE_PENDING; 542 + log_keep_alive(INFO, "schedule send of immediate response\n"); 543 + queue_work(sc->workqueue, &sc->idle.immediate_work); 765 544 } 766 545 767 546 /* ··· 769 548 * reassembly queue and wake up the reading thread 770 549 */ 771 550 if (data_length) { 772 - enqueue_reassembly(info, response, data_length); 773 - wake_up_interruptible(&sc->recv_io.reassembly.wait_queue); 551 + if (sc->recv_io.credits.target > old_recv_credit_target) 552 + queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); 553 + 554 + enqueue_reassembly(sc, response, data_length); 555 + wake_up(&sc->recv_io.reassembly.wait_queue); 774 556 } else 775 - put_receive_buffer(info, response); 557 + put_receive_buffer(sc, response); 776 558 777 559 return; 778 560 ··· 790 566 log_rdma_recv(ERR, "unexpected response type=%d\n", sc->recv_io.expected); 791 567 WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_DATA_TRANSFER); 792 568 error: 793 - put_receive_buffer(info, response); 794 - smbd_disconnect_rdma_connection(info); 569 + put_receive_buffer(sc, response); 570 + smbd_disconnect_rdma_connection(sc); 795 571 } 796 572 797 573 static struct rdma_cm_id *smbd_create_id( 798 - struct smbd_connection *info, 574 + struct smbdirect_socket *sc, 799 575 struct sockaddr *dstaddr, int port) 800 576 { 577 + struct smbdirect_socket_parameters *sp = &sc->parameters; 801 578 struct rdma_cm_id *id; 802 579 int rc; 803 580 __be16 *sport; 804 581 805 - id = rdma_create_id(&init_net, smbd_conn_upcall, info, 582 + id = rdma_create_id(&init_net, smbd_conn_upcall, sc, 806 583 RDMA_PS_TCP, IB_QPT_RC); 807 584 if (IS_ERR(id)) { 808 585 rc = PTR_ERR(id); ··· 818 593 819 594 *sport = htons(port); 820 595 821 - init_completion(&info->ri_done); 822 - info->ri_rc = -ETIMEDOUT; 823 - 596 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED); 597 + sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING; 824 598 rc = rdma_resolve_addr(id, NULL, (struct sockaddr *)dstaddr, 825 - RDMA_RESOLVE_TIMEOUT); 599 + sp->resolve_addr_timeout_msec); 826 600 if (rc) { 827 601 log_rdma_event(ERR, "rdma_resolve_addr() failed %i\n", rc); 828 602 goto out; 829 603 } 830 - rc = wait_for_completion_interruptible_timeout( 831 - &info->ri_done, msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT)); 604 + rc = wait_event_interruptible_timeout( 605 + sc->status_wait, 606 + sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING, 607 + msecs_to_jiffies(sp->resolve_addr_timeout_msec)); 832 608 /* e.g. if interrupted returns -ERESTARTSYS */ 833 609 if (rc < 0) { 834 610 log_rdma_event(ERR, "rdma_resolve_addr timeout rc: %i\n", rc); 835 611 goto out; 836 612 } 837 - rc = info->ri_rc; 838 - if (rc) { 613 + if (sc->status == SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING) { 614 + rc = -ETIMEDOUT; 615 + log_rdma_event(ERR, "rdma_resolve_addr() completed %i\n", rc); 616 + goto out; 617 + } 618 + if (sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED) { 619 + rc = -EHOSTUNREACH; 839 620 log_rdma_event(ERR, "rdma_resolve_addr() completed %i\n", rc); 840 621 goto out; 841 622 } 842 623 843 - info->ri_rc = -ETIMEDOUT; 844 - rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT); 624 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED); 625 + sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING; 626 + rc = rdma_resolve_route(id, sp->resolve_route_timeout_msec); 845 627 if (rc) { 846 628 log_rdma_event(ERR, "rdma_resolve_route() failed %i\n", rc); 847 629 goto out; 848 630 } 849 - rc = wait_for_completion_interruptible_timeout( 850 - &info->ri_done, msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT)); 631 + rc = wait_event_interruptible_timeout( 632 + sc->status_wait, 633 + sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING, 634 + msecs_to_jiffies(sp->resolve_route_timeout_msec)); 851 635 /* e.g. if interrupted returns -ERESTARTSYS */ 852 636 if (rc < 0) { 853 637 log_rdma_event(ERR, "rdma_resolve_addr timeout rc: %i\n", rc); 854 638 goto out; 855 639 } 856 - rc = info->ri_rc; 857 - if (rc) { 640 + if (sc->status == SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING) { 641 + rc = -ETIMEDOUT; 642 + log_rdma_event(ERR, "rdma_resolve_route() completed %i\n", rc); 643 + goto out; 644 + } 645 + if (sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED) { 646 + rc = -ENETUNREACH; 858 647 log_rdma_event(ERR, "rdma_resolve_route() completed %i\n", rc); 859 648 goto out; 860 649 } ··· 895 656 } 896 657 897 658 static int smbd_ia_open( 898 - struct smbd_connection *info, 659 + struct smbdirect_socket *sc, 899 660 struct sockaddr *dstaddr, int port) 900 661 { 901 - struct smbdirect_socket *sc = &info->socket; 662 + struct smbdirect_socket_parameters *sp = &sc->parameters; 902 663 int rc; 903 664 904 - sc->rdma.cm_id = smbd_create_id(info, dstaddr, port); 665 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED); 666 + sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED; 667 + 668 + sc->rdma.cm_id = smbd_create_id(sc, dstaddr, port); 905 669 if (IS_ERR(sc->rdma.cm_id)) { 906 670 rc = PTR_ERR(sc->rdma.cm_id); 907 671 goto out1; ··· 919 677 rc = -EPROTONOSUPPORT; 920 678 goto out2; 921 679 } 922 - info->max_frmr_depth = min_t(int, 923 - smbd_max_frmr_depth, 680 + sp->max_frmr_depth = min_t(u32, 681 + sp->max_frmr_depth, 924 682 sc->ib.dev->attrs.max_fast_reg_page_list_len); 925 - info->mr_type = IB_MR_TYPE_MEM_REG; 683 + sc->mr_io.type = IB_MR_TYPE_MEM_REG; 926 684 if (sc->ib.dev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG) 927 - info->mr_type = IB_MR_TYPE_SG_GAPS; 928 - 929 - sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0); 930 - if (IS_ERR(sc->ib.pd)) { 931 - rc = PTR_ERR(sc->ib.pd); 932 - log_rdma_event(ERR, "ib_alloc_pd() returned %d\n", rc); 933 - goto out2; 934 - } 685 + sc->mr_io.type = IB_MR_TYPE_SG_GAPS; 935 686 936 687 return 0; 937 688 ··· 942 707 * After negotiation, the transport is connected and ready for 943 708 * carrying upper layer SMB payload 944 709 */ 945 - static int smbd_post_send_negotiate_req(struct smbd_connection *info) 710 + static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) 946 711 { 947 - struct smbdirect_socket *sc = &info->socket; 948 712 struct smbdirect_socket_parameters *sp = &sc->parameters; 949 713 struct ib_send_wr send_wr; 950 714 int rc = -ENOMEM; ··· 995 761 request->sge[0].addr, 996 762 request->sge[0].length, request->sge[0].lkey); 997 763 998 - atomic_inc(&info->send_pending); 764 + atomic_inc(&sc->send_io.pending.count); 999 765 rc = ib_post_send(sc->ib.qp, &send_wr, NULL); 1000 766 if (!rc) 1001 767 return 0; 1002 768 1003 769 /* if we reach here, post send failed */ 1004 770 log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); 1005 - atomic_dec(&info->send_pending); 771 + atomic_dec(&sc->send_io.pending.count); 1006 772 ib_dma_unmap_single(sc->ib.dev, request->sge[0].addr, 1007 773 request->sge[0].length, DMA_TO_DEVICE); 1008 774 1009 - smbd_disconnect_rdma_connection(info); 775 + smbd_disconnect_rdma_connection(sc); 1010 776 1011 777 dma_mapping_failed: 1012 778 mempool_free(request, sc->send_io.mem.pool); ··· 1021 787 * buffer as possible, and extend the receive credits to remote peer 1022 788 * return value: the new credtis being granted. 1023 789 */ 1024 - static int manage_credits_prior_sending(struct smbd_connection *info) 790 + static int manage_credits_prior_sending(struct smbdirect_socket *sc) 1025 791 { 1026 792 int new_credits; 1027 793 1028 - spin_lock(&info->lock_new_credits_offered); 1029 - new_credits = info->new_credits_offered; 1030 - info->new_credits_offered = 0; 1031 - spin_unlock(&info->lock_new_credits_offered); 794 + if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) 795 + return 0; 796 + 797 + new_credits = atomic_read(&sc->recv_io.posted.count); 798 + if (new_credits == 0) 799 + return 0; 800 + 801 + new_credits -= atomic_read(&sc->recv_io.credits.count); 802 + if (new_credits <= 0) 803 + return 0; 1032 804 1033 805 return new_credits; 1034 806 } ··· 1048 808 * 1 if SMBDIRECT_FLAG_RESPONSE_REQUESTED needs to be set 1049 809 * 0: otherwise 1050 810 */ 1051 - static int manage_keep_alive_before_sending(struct smbd_connection *info) 811 + static int manage_keep_alive_before_sending(struct smbdirect_socket *sc) 1052 812 { 1053 - if (info->keep_alive_requested == KEEP_ALIVE_PENDING) { 1054 - info->keep_alive_requested = KEEP_ALIVE_SENT; 813 + struct smbdirect_socket_parameters *sp = &sc->parameters; 814 + 815 + if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) { 816 + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT; 817 + /* 818 + * Now use the keepalive timeout (instead of keepalive interval) 819 + * in order to wait for a response 820 + */ 821 + mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 822 + msecs_to_jiffies(sp->keepalive_timeout_msec)); 1055 823 return 1; 1056 824 } 1057 825 return 0; 1058 826 } 1059 827 1060 828 /* Post the send request */ 1061 - static int smbd_post_send(struct smbd_connection *info, 829 + static int smbd_post_send(struct smbdirect_socket *sc, 1062 830 struct smbdirect_send_io *request) 1063 831 { 1064 - struct smbdirect_socket *sc = &info->socket; 1065 - struct smbdirect_socket_parameters *sp = &sc->parameters; 1066 832 struct ib_send_wr send_wr; 1067 833 int rc, i; 1068 834 ··· 1095 849 rc = ib_post_send(sc->ib.qp, &send_wr, NULL); 1096 850 if (rc) { 1097 851 log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); 1098 - smbd_disconnect_rdma_connection(info); 852 + smbd_disconnect_rdma_connection(sc); 1099 853 rc = -EAGAIN; 1100 - } else 1101 - /* Reset timer for idle connection after packet is sent */ 1102 - mod_delayed_work(info->workqueue, &info->idle_timer_work, 1103 - msecs_to_jiffies(sp->keepalive_interval_msec)); 854 + } 1104 855 1105 856 return rc; 1106 857 } 1107 858 1108 - static int smbd_post_send_iter(struct smbd_connection *info, 859 + static int smbd_post_send_iter(struct smbdirect_socket *sc, 1109 860 struct iov_iter *iter, 1110 861 int *_remaining_data_length) 1111 862 { 1112 - struct smbdirect_socket *sc = &info->socket; 1113 863 struct smbdirect_socket_parameters *sp = &sc->parameters; 1114 864 int i, rc; 1115 865 int header_length; ··· 1116 874 1117 875 wait_credit: 1118 876 /* Wait for send credits. A SMBD packet needs one credit */ 1119 - rc = wait_event_interruptible(info->wait_send_queue, 1120 - atomic_read(&info->send_credits) > 0 || 877 + rc = wait_event_interruptible(sc->send_io.credits.wait_queue, 878 + atomic_read(&sc->send_io.credits.count) > 0 || 1121 879 sc->status != SMBDIRECT_SOCKET_CONNECTED); 1122 880 if (rc) 1123 881 goto err_wait_credit; ··· 1127 885 rc = -EAGAIN; 1128 886 goto err_wait_credit; 1129 887 } 1130 - if (unlikely(atomic_dec_return(&info->send_credits) < 0)) { 1131 - atomic_inc(&info->send_credits); 888 + if (unlikely(atomic_dec_return(&sc->send_io.credits.count) < 0)) { 889 + atomic_inc(&sc->send_io.credits.count); 1132 890 goto wait_credit; 1133 891 } 1134 892 1135 893 wait_send_queue: 1136 - wait_event(info->wait_post_send, 1137 - atomic_read(&info->send_pending) < sp->send_credit_target || 894 + wait_event(sc->send_io.pending.dec_wait_queue, 895 + atomic_read(&sc->send_io.pending.count) < sp->send_credit_target || 1138 896 sc->status != SMBDIRECT_SOCKET_CONNECTED); 1139 897 1140 898 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { ··· 1143 901 goto err_wait_send_queue; 1144 902 } 1145 903 1146 - if (unlikely(atomic_inc_return(&info->send_pending) > 904 + if (unlikely(atomic_inc_return(&sc->send_io.pending.count) > 1147 905 sp->send_credit_target)) { 1148 - atomic_dec(&info->send_pending); 906 + atomic_dec(&sc->send_io.pending.count); 1149 907 goto wait_send_queue; 1150 908 } 1151 909 ··· 1158 916 request->socket = sc; 1159 917 memset(request->sge, 0, sizeof(request->sge)); 1160 918 919 + /* Map the packet to DMA */ 920 + header_length = sizeof(struct smbdirect_data_transfer); 921 + /* If this is a packet without payload, don't send padding */ 922 + if (!iter) 923 + header_length = offsetof(struct smbdirect_data_transfer, padding); 924 + 925 + packet = smbdirect_send_io_payload(request); 926 + request->sge[0].addr = ib_dma_map_single(sc->ib.dev, 927 + (void *)packet, 928 + header_length, 929 + DMA_TO_DEVICE); 930 + if (ib_dma_mapping_error(sc->ib.dev, request->sge[0].addr)) { 931 + rc = -EIO; 932 + goto err_dma; 933 + } 934 + 935 + request->sge[0].length = header_length; 936 + request->sge[0].lkey = sc->ib.pd->local_dma_lkey; 937 + request->num_sge = 1; 938 + 1161 939 /* Fill in the data payload to find out how much data we can add */ 1162 940 if (iter) { 1163 941 struct smb_extract_to_rdma extract = { 1164 - .nr_sge = 1, 942 + .nr_sge = request->num_sge, 1165 943 .max_sge = SMBDIRECT_SEND_IO_MAX_SGE, 1166 944 .sge = request->sge, 1167 945 .device = sc->ib.dev, ··· 1200 938 *_remaining_data_length -= data_length; 1201 939 } else { 1202 940 data_length = 0; 1203 - request->num_sge = 1; 1204 941 } 1205 942 1206 943 /* Fill in the packet header */ 1207 - packet = smbdirect_send_io_payload(request); 1208 944 packet->credits_requested = cpu_to_le16(sp->send_credit_target); 1209 945 1210 - new_credits = manage_credits_prior_sending(info); 1211 - atomic_add(new_credits, &info->receive_credits); 946 + new_credits = manage_credits_prior_sending(sc); 947 + atomic_add(new_credits, &sc->recv_io.credits.count); 1212 948 packet->credits_granted = cpu_to_le16(new_credits); 1213 949 1214 - info->send_immediate = false; 1215 - 1216 950 packet->flags = 0; 1217 - if (manage_keep_alive_before_sending(info)) 951 + if (manage_keep_alive_before_sending(sc)) 1218 952 packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED); 1219 953 1220 954 packet->reserved = 0; ··· 1229 971 le32_to_cpu(packet->data_length), 1230 972 le32_to_cpu(packet->remaining_data_length)); 1231 973 1232 - /* Map the packet to DMA */ 1233 - header_length = sizeof(struct smbdirect_data_transfer); 1234 - /* If this is a packet without payload, don't send padding */ 1235 - if (!data_length) 1236 - header_length = offsetof(struct smbdirect_data_transfer, padding); 1237 - 1238 - request->sge[0].addr = ib_dma_map_single(sc->ib.dev, 1239 - (void *)packet, 1240 - header_length, 1241 - DMA_TO_DEVICE); 1242 - if (ib_dma_mapping_error(sc->ib.dev, request->sge[0].addr)) { 1243 - rc = -EIO; 1244 - request->sge[0].addr = 0; 1245 - goto err_dma; 1246 - } 1247 - 1248 - request->sge[0].length = header_length; 1249 - request->sge[0].lkey = sc->ib.pd->local_dma_lkey; 1250 - 1251 - rc = smbd_post_send(info, request); 974 + rc = smbd_post_send(sc, request); 1252 975 if (!rc) 1253 976 return 0; 1254 977 ··· 1242 1003 DMA_TO_DEVICE); 1243 1004 mempool_free(request, sc->send_io.mem.pool); 1244 1005 1245 - /* roll back receive credits and credits to be offered */ 1246 - spin_lock(&info->lock_new_credits_offered); 1247 - info->new_credits_offered += new_credits; 1248 - spin_unlock(&info->lock_new_credits_offered); 1249 - atomic_sub(new_credits, &info->receive_credits); 1006 + /* roll back the granted receive credits */ 1007 + atomic_sub(new_credits, &sc->recv_io.credits.count); 1250 1008 1251 1009 err_alloc: 1252 - if (atomic_dec_and_test(&info->send_pending)) 1253 - wake_up(&info->wait_send_pending); 1010 + if (atomic_dec_and_test(&sc->send_io.pending.count)) 1011 + wake_up(&sc->send_io.pending.zero_wait_queue); 1254 1012 1255 1013 err_wait_send_queue: 1256 1014 /* roll back send credits and pending */ 1257 - atomic_inc(&info->send_credits); 1015 + atomic_inc(&sc->send_io.credits.count); 1258 1016 1259 1017 err_wait_credit: 1260 1018 return rc; ··· 1262 1026 * Empty message is used to extend credits to peer to for keep live 1263 1027 * while there is no upper layer payload to send at the time 1264 1028 */ 1265 - static int smbd_post_send_empty(struct smbd_connection *info) 1029 + static int smbd_post_send_empty(struct smbdirect_socket *sc) 1266 1030 { 1267 1031 int remaining_data_length = 0; 1268 1032 1269 - info->count_send_empty++; 1270 - return smbd_post_send_iter(info, NULL, &remaining_data_length); 1033 + sc->statistics.send_empty++; 1034 + return smbd_post_send_iter(sc, NULL, &remaining_data_length); 1271 1035 } 1272 1036 1273 - static int smbd_post_send_full_iter(struct smbd_connection *info, 1037 + static int smbd_post_send_full_iter(struct smbdirect_socket *sc, 1274 1038 struct iov_iter *iter, 1275 1039 int *_remaining_data_length) 1276 1040 { ··· 1283 1047 */ 1284 1048 1285 1049 while (iov_iter_count(iter) > 0) { 1286 - rc = smbd_post_send_iter(info, iter, _remaining_data_length); 1050 + rc = smbd_post_send_iter(sc, iter, _remaining_data_length); 1287 1051 if (rc < 0) 1288 1052 break; 1289 1053 } ··· 1297 1061 * The interaction is controlled by send/receive credit system 1298 1062 */ 1299 1063 static int smbd_post_recv( 1300 - struct smbd_connection *info, struct smbdirect_recv_io *response) 1064 + struct smbdirect_socket *sc, struct smbdirect_recv_io *response) 1301 1065 { 1302 - struct smbdirect_socket *sc = &info->socket; 1303 1066 struct smbdirect_socket_parameters *sp = &sc->parameters; 1304 1067 struct ib_recv_wr recv_wr; 1305 1068 int rc = -EIO; ··· 1324 1089 ib_dma_unmap_single(sc->ib.dev, response->sge.addr, 1325 1090 response->sge.length, DMA_FROM_DEVICE); 1326 1091 response->sge.length = 0; 1327 - smbd_disconnect_rdma_connection(info); 1092 + smbd_disconnect_rdma_connection(sc); 1328 1093 log_rdma_recv(ERR, "ib_post_recv failed rc=%d\n", rc); 1329 1094 } 1330 1095 ··· 1332 1097 } 1333 1098 1334 1099 /* Perform SMBD negotiate according to [MS-SMBD] 3.1.5.2 */ 1335 - static int smbd_negotiate(struct smbd_connection *info) 1100 + static int smbd_negotiate(struct smbdirect_socket *sc) 1336 1101 { 1337 - struct smbdirect_socket *sc = &info->socket; 1102 + struct smbdirect_socket_parameters *sp = &sc->parameters; 1338 1103 int rc; 1339 - struct smbdirect_recv_io *response = get_receive_buffer(info); 1104 + struct smbdirect_recv_io *response = get_receive_buffer(sc); 1105 + 1106 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED); 1107 + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; 1340 1108 1341 1109 sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REP; 1342 - rc = smbd_post_recv(info, response); 1110 + rc = smbd_post_recv(sc, response); 1343 1111 log_rdma_event(INFO, "smbd_post_recv rc=%d iov.addr=0x%llx iov.length=%u iov.lkey=0x%x\n", 1344 1112 rc, response->sge.addr, 1345 1113 response->sge.length, response->sge.lkey); 1346 1114 if (rc) { 1347 - put_receive_buffer(info, response); 1115 + put_receive_buffer(sc, response); 1348 1116 return rc; 1349 1117 } 1350 1118 1351 - init_completion(&info->negotiate_completion); 1352 - info->negotiate_done = false; 1353 - rc = smbd_post_send_negotiate_req(info); 1119 + rc = smbd_post_send_negotiate_req(sc); 1354 1120 if (rc) 1355 1121 return rc; 1356 1122 1357 - rc = wait_for_completion_interruptible_timeout( 1358 - &info->negotiate_completion, SMBD_NEGOTIATE_TIMEOUT * HZ); 1359 - log_rdma_event(INFO, "wait_for_completion_timeout rc=%d\n", rc); 1123 + rc = wait_event_interruptible_timeout( 1124 + sc->status_wait, 1125 + sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING, 1126 + msecs_to_jiffies(sp->negotiate_timeout_msec)); 1127 + log_rdma_event(INFO, "wait_event_interruptible_timeout rc=%d\n", rc); 1360 1128 1361 - if (info->negotiate_done) 1129 + if (sc->status == SMBDIRECT_SOCKET_CONNECTED) 1362 1130 return 0; 1363 1131 1364 1132 if (rc == 0) ··· 1385 1147 * data_length: the size of payload in this packet 1386 1148 */ 1387 1149 static void enqueue_reassembly( 1388 - struct smbd_connection *info, 1150 + struct smbdirect_socket *sc, 1389 1151 struct smbdirect_recv_io *response, 1390 1152 int data_length) 1391 1153 { 1392 - struct smbdirect_socket *sc = &info->socket; 1154 + unsigned long flags; 1393 1155 1394 - spin_lock(&sc->recv_io.reassembly.lock); 1156 + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 1395 1157 list_add_tail(&response->list, &sc->recv_io.reassembly.list); 1396 1158 sc->recv_io.reassembly.queue_length++; 1397 1159 /* ··· 1402 1164 */ 1403 1165 virt_wmb(); 1404 1166 sc->recv_io.reassembly.data_length += data_length; 1405 - spin_unlock(&sc->recv_io.reassembly.lock); 1406 - info->count_reassembly_queue++; 1407 - info->count_enqueue_reassembly_queue++; 1167 + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 1168 + sc->statistics.enqueue_reassembly_queue++; 1408 1169 } 1409 1170 1410 1171 /* ··· 1411 1174 * Caller is responsible for locking 1412 1175 * return value: the first entry if any, NULL if queue is empty 1413 1176 */ 1414 - static struct smbdirect_recv_io *_get_first_reassembly(struct smbd_connection *info) 1177 + static struct smbdirect_recv_io *_get_first_reassembly(struct smbdirect_socket *sc) 1415 1178 { 1416 - struct smbdirect_socket *sc = &info->socket; 1417 1179 struct smbdirect_recv_io *ret = NULL; 1418 1180 1419 1181 if (!list_empty(&sc->recv_io.reassembly.list)) { ··· 1429 1193 * pre-allocated in advance. 1430 1194 * return value: the receive buffer, NULL if none is available 1431 1195 */ 1432 - static struct smbdirect_recv_io *get_receive_buffer(struct smbd_connection *info) 1196 + static struct smbdirect_recv_io *get_receive_buffer(struct smbdirect_socket *sc) 1433 1197 { 1434 - struct smbdirect_socket *sc = &info->socket; 1435 1198 struct smbdirect_recv_io *ret = NULL; 1436 1199 unsigned long flags; 1437 1200 ··· 1440 1205 &sc->recv_io.free.list, 1441 1206 struct smbdirect_recv_io, list); 1442 1207 list_del(&ret->list); 1443 - info->count_receive_queue--; 1444 - info->count_get_receive_buffer++; 1208 + sc->statistics.get_receive_buffer++; 1445 1209 } 1446 1210 spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); 1447 1211 ··· 1454 1220 * receive buffer is returned. 1455 1221 */ 1456 1222 static void put_receive_buffer( 1457 - struct smbd_connection *info, struct smbdirect_recv_io *response) 1223 + struct smbdirect_socket *sc, struct smbdirect_recv_io *response) 1458 1224 { 1459 - struct smbdirect_socket *sc = &info->socket; 1460 1225 unsigned long flags; 1461 1226 1462 1227 if (likely(response->sge.length != 0)) { ··· 1468 1235 1469 1236 spin_lock_irqsave(&sc->recv_io.free.lock, flags); 1470 1237 list_add_tail(&response->list, &sc->recv_io.free.list); 1471 - info->count_receive_queue++; 1472 - info->count_put_receive_buffer++; 1238 + sc->statistics.put_receive_buffer++; 1473 1239 spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); 1474 1240 1475 - queue_work(info->workqueue, &info->post_send_credits_work); 1241 + queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); 1476 1242 } 1477 1243 1478 1244 /* Preallocate all receive buffer on transport establishment */ 1479 - static int allocate_receive_buffers(struct smbd_connection *info, int num_buf) 1245 + static int allocate_receive_buffers(struct smbdirect_socket *sc, int num_buf) 1480 1246 { 1481 - struct smbdirect_socket *sc = &info->socket; 1482 1247 struct smbdirect_recv_io *response; 1483 1248 int i; 1484 - 1485 - INIT_LIST_HEAD(&sc->recv_io.reassembly.list); 1486 - spin_lock_init(&sc->recv_io.reassembly.lock); 1487 - sc->recv_io.reassembly.data_length = 0; 1488 - sc->recv_io.reassembly.queue_length = 0; 1489 - 1490 - INIT_LIST_HEAD(&sc->recv_io.free.list); 1491 - spin_lock_init(&sc->recv_io.free.lock); 1492 - info->count_receive_queue = 0; 1493 - 1494 - init_waitqueue_head(&info->wait_receive_queues); 1495 1249 1496 1250 for (i = 0; i < num_buf; i++) { 1497 1251 response = mempool_alloc(sc->recv_io.mem.pool, GFP_KERNEL); ··· 1488 1268 response->socket = sc; 1489 1269 response->sge.length = 0; 1490 1270 list_add_tail(&response->list, &sc->recv_io.free.list); 1491 - info->count_receive_queue++; 1492 1271 } 1493 1272 1494 1273 return 0; ··· 1498 1279 &sc->recv_io.free.list, 1499 1280 struct smbdirect_recv_io, list); 1500 1281 list_del(&response->list); 1501 - info->count_receive_queue--; 1502 1282 1503 1283 mempool_free(response, sc->recv_io.mem.pool); 1504 1284 } 1505 1285 return -ENOMEM; 1506 1286 } 1507 1287 1508 - static void destroy_receive_buffers(struct smbd_connection *info) 1288 + static void destroy_receive_buffers(struct smbdirect_socket *sc) 1509 1289 { 1510 - struct smbdirect_socket *sc = &info->socket; 1511 1290 struct smbdirect_recv_io *response; 1512 1291 1513 - while ((response = get_receive_buffer(info))) 1292 + while ((response = get_receive_buffer(sc))) 1514 1293 mempool_free(response, sc->recv_io.mem.pool); 1294 + } 1295 + 1296 + static void send_immediate_empty_message(struct work_struct *work) 1297 + { 1298 + struct smbdirect_socket *sc = 1299 + container_of(work, struct smbdirect_socket, idle.immediate_work); 1300 + 1301 + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 1302 + return; 1303 + 1304 + log_keep_alive(INFO, "send an empty message\n"); 1305 + smbd_post_send_empty(sc); 1515 1306 } 1516 1307 1517 1308 /* Implement idle connection timer [MS-SMBD] 3.1.6.2 */ 1518 1309 static void idle_connection_timer(struct work_struct *work) 1519 1310 { 1520 - struct smbd_connection *info = container_of( 1521 - work, struct smbd_connection, 1522 - idle_timer_work.work); 1523 - struct smbdirect_socket *sc = &info->socket; 1311 + struct smbdirect_socket *sc = 1312 + container_of(work, struct smbdirect_socket, idle.timer_work.work); 1524 1313 struct smbdirect_socket_parameters *sp = &sc->parameters; 1525 1314 1526 - if (info->keep_alive_requested != KEEP_ALIVE_NONE) { 1315 + if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) { 1527 1316 log_keep_alive(ERR, 1528 - "error status info->keep_alive_requested=%d\n", 1529 - info->keep_alive_requested); 1530 - smbd_disconnect_rdma_connection(info); 1317 + "error status sc->idle.keepalive=%d\n", 1318 + sc->idle.keepalive); 1319 + smbd_disconnect_rdma_connection(sc); 1531 1320 return; 1532 1321 } 1533 1322 1534 - log_keep_alive(INFO, "about to send an empty idle message\n"); 1535 - smbd_post_send_empty(info); 1323 + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 1324 + return; 1536 1325 1537 - /* Setup the next idle timeout work */ 1538 - queue_delayed_work(info->workqueue, &info->idle_timer_work, 1539 - msecs_to_jiffies(sp->keepalive_interval_msec)); 1326 + /* 1327 + * Now use the keepalive timeout (instead of keepalive interval) 1328 + * in order to wait for a response 1329 + */ 1330 + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; 1331 + mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 1332 + msecs_to_jiffies(sp->keepalive_timeout_msec)); 1333 + log_keep_alive(INFO, "schedule send of empty idle message\n"); 1334 + queue_work(sc->workqueue, &sc->idle.immediate_work); 1540 1335 } 1541 1336 1542 1337 /* ··· 1562 1329 { 1563 1330 struct smbd_connection *info = server->smbd_conn; 1564 1331 struct smbdirect_socket *sc; 1565 - struct smbdirect_socket_parameters *sp; 1566 1332 struct smbdirect_recv_io *response; 1567 1333 unsigned long flags; 1568 1334 ··· 1570 1338 return; 1571 1339 } 1572 1340 sc = &info->socket; 1573 - sp = &sc->parameters; 1341 + 1342 + log_rdma_event(INFO, "cancelling and disable disconnect_work\n"); 1343 + disable_work_sync(&sc->disconnect_work); 1574 1344 1575 1345 log_rdma_event(INFO, "destroying rdma session\n"); 1576 - if (sc->status != SMBDIRECT_SOCKET_DISCONNECTED) { 1577 - rdma_disconnect(sc->rdma.cm_id); 1346 + if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING) { 1347 + smbd_disconnect_rdma_work(&sc->disconnect_work); 1578 1348 log_rdma_event(INFO, "wait for transport being disconnected\n"); 1579 1349 wait_event_interruptible( 1580 - info->status_wait, 1350 + sc->status_wait, 1581 1351 sc->status == SMBDIRECT_SOCKET_DISCONNECTED); 1582 1352 } 1583 1353 1584 - log_rdma_event(INFO, "cancelling post_send_credits_work\n"); 1585 - disable_work_sync(&info->post_send_credits_work); 1354 + /* 1355 + * Wake up all waiters in all wait queues 1356 + * in order to notice the broken connection. 1357 + * 1358 + * Most likely this was already called via 1359 + * smbd_disconnect_rdma_work(), but call it again... 1360 + */ 1361 + smbd_disconnect_wake_up_all(sc); 1362 + 1363 + log_rdma_event(INFO, "cancelling recv_io.posted.refill_work\n"); 1364 + disable_work_sync(&sc->recv_io.posted.refill_work); 1586 1365 1587 1366 log_rdma_event(INFO, "destroying qp\n"); 1588 1367 ib_drain_qp(sc->ib.qp); ··· 1601 1358 sc->ib.qp = NULL; 1602 1359 1603 1360 log_rdma_event(INFO, "cancelling idle timer\n"); 1604 - disable_delayed_work_sync(&info->idle_timer_work); 1361 + disable_delayed_work_sync(&sc->idle.timer_work); 1362 + log_rdma_event(INFO, "cancelling send immediate work\n"); 1363 + disable_work_sync(&sc->idle.immediate_work); 1605 1364 1606 1365 /* It's not possible for upper layer to get to reassembly */ 1607 1366 log_rdma_event(INFO, "drain the reassembly queue\n"); 1608 1367 do { 1609 1368 spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 1610 - response = _get_first_reassembly(info); 1369 + response = _get_first_reassembly(sc); 1611 1370 if (response) { 1612 1371 list_del(&response->list); 1613 1372 spin_unlock_irqrestore( 1614 1373 &sc->recv_io.reassembly.lock, flags); 1615 - put_receive_buffer(info, response); 1374 + put_receive_buffer(sc, response); 1616 1375 } else 1617 1376 spin_unlock_irqrestore( 1618 1377 &sc->recv_io.reassembly.lock, flags); ··· 1622 1377 sc->recv_io.reassembly.data_length = 0; 1623 1378 1624 1379 log_rdma_event(INFO, "free receive buffers\n"); 1625 - wait_event(info->wait_receive_queues, 1626 - info->count_receive_queue == sp->recv_credit_max); 1627 - destroy_receive_buffers(info); 1380 + destroy_receive_buffers(sc); 1628 1381 1629 1382 /* 1630 1383 * For performance reasons, memory registration and deregistration ··· 1632 1389 * path when sending data, and then release memory registrations. 1633 1390 */ 1634 1391 log_rdma_event(INFO, "freeing mr list\n"); 1635 - wake_up_interruptible_all(&info->wait_mr); 1636 - while (atomic_read(&info->mr_used_count)) { 1392 + while (atomic_read(&sc->mr_io.used.count)) { 1637 1393 cifs_server_unlock(server); 1638 1394 msleep(1000); 1639 1395 cifs_server_lock(server); 1640 1396 } 1641 - destroy_mr_list(info); 1397 + destroy_mr_list(sc); 1642 1398 1643 1399 ib_free_cq(sc->ib.send_cq); 1644 1400 ib_free_cq(sc->ib.recv_cq); ··· 1653 1411 1654 1412 sc->status = SMBDIRECT_SOCKET_DESTROYED; 1655 1413 1656 - destroy_workqueue(info->workqueue); 1414 + destroy_workqueue(sc->workqueue); 1657 1415 log_rdma_event(INFO, "rdma session destroyed\n"); 1658 1416 kfree(info); 1659 1417 server->smbd_conn = NULL; ··· 1695 1453 return -ENOENT; 1696 1454 } 1697 1455 1698 - static void destroy_caches_and_workqueue(struct smbd_connection *info) 1456 + static void destroy_caches(struct smbdirect_socket *sc) 1699 1457 { 1700 - struct smbdirect_socket *sc = &info->socket; 1701 - 1702 - destroy_receive_buffers(info); 1703 - destroy_workqueue(info->workqueue); 1458 + destroy_receive_buffers(sc); 1704 1459 mempool_destroy(sc->recv_io.mem.pool); 1705 1460 kmem_cache_destroy(sc->recv_io.mem.cache); 1706 1461 mempool_destroy(sc->send_io.mem.pool); ··· 1705 1466 } 1706 1467 1707 1468 #define MAX_NAME_LEN 80 1708 - static int allocate_caches_and_workqueue(struct smbd_connection *info) 1469 + static int allocate_caches(struct smbdirect_socket *sc) 1709 1470 { 1710 - struct smbdirect_socket *sc = &info->socket; 1711 1471 struct smbdirect_socket_parameters *sp = &sc->parameters; 1712 1472 char name[MAX_NAME_LEN]; 1713 1473 int rc; ··· 1714 1476 if (WARN_ON_ONCE(sp->max_recv_size < sizeof(struct smbdirect_data_transfer))) 1715 1477 return -ENOMEM; 1716 1478 1717 - scnprintf(name, MAX_NAME_LEN, "smbdirect_send_io_%p", info); 1479 + scnprintf(name, MAX_NAME_LEN, "smbdirect_send_io_%p", sc); 1718 1480 sc->send_io.mem.cache = 1719 1481 kmem_cache_create( 1720 1482 name, ··· 1730 1492 if (!sc->send_io.mem.pool) 1731 1493 goto out1; 1732 1494 1733 - scnprintf(name, MAX_NAME_LEN, "smbdirect_recv_io_%p", info); 1495 + scnprintf(name, MAX_NAME_LEN, "smbdirect_recv_io_%p", sc); 1734 1496 1735 1497 struct kmem_cache_args response_args = { 1736 1498 .align = __alignof__(struct smbdirect_recv_io), ··· 1751 1513 if (!sc->recv_io.mem.pool) 1752 1514 goto out3; 1753 1515 1754 - scnprintf(name, MAX_NAME_LEN, "smbd_%p", info); 1755 - info->workqueue = create_workqueue(name); 1756 - if (!info->workqueue) 1757 - goto out4; 1758 - 1759 - rc = allocate_receive_buffers(info, sp->recv_credit_max); 1516 + rc = allocate_receive_buffers(sc, sp->recv_credit_max); 1760 1517 if (rc) { 1761 1518 log_rdma_event(ERR, "failed to allocate receive buffers\n"); 1762 - goto out5; 1519 + goto out4; 1763 1520 } 1764 1521 1765 1522 return 0; 1766 1523 1767 - out5: 1768 - destroy_workqueue(info->workqueue); 1769 1524 out4: 1770 1525 mempool_destroy(sc->recv_io.mem.pool); 1771 1526 out3: ··· 1782 1551 struct ib_qp_init_attr qp_attr; 1783 1552 struct sockaddr_in *addr_in = (struct sockaddr_in *) dstaddr; 1784 1553 struct ib_port_immutable port_immutable; 1785 - u32 ird_ord_hdr[2]; 1554 + __be32 ird_ord_hdr[2]; 1555 + char wq_name[80]; 1556 + struct workqueue_struct *workqueue; 1786 1557 1787 1558 info = kzalloc(sizeof(struct smbd_connection), GFP_KERNEL); 1788 1559 if (!info) 1789 1560 return NULL; 1790 1561 sc = &info->socket; 1562 + scnprintf(wq_name, ARRAY_SIZE(wq_name), "smbd_%p", sc); 1563 + workqueue = create_workqueue(wq_name); 1564 + if (!workqueue) 1565 + goto create_wq_failed; 1566 + smbdirect_socket_init(sc); 1567 + sc->workqueue = workqueue; 1791 1568 sp = &sc->parameters; 1792 1569 1793 - sc->status = SMBDIRECT_SOCKET_CONNECTING; 1794 - rc = smbd_ia_open(info, dstaddr, port); 1795 - if (rc) { 1796 - log_rdma_event(INFO, "smbd_ia_open rc=%d\n", rc); 1797 - goto create_id_failed; 1798 - } 1570 + INIT_WORK(&sc->disconnect_work, smbd_disconnect_rdma_work); 1799 1571 1800 - if (smbd_send_credit_target > sc->ib.dev->attrs.max_cqe || 1801 - smbd_send_credit_target > sc->ib.dev->attrs.max_qp_wr) { 1802 - log_rdma_event(ERR, "consider lowering send_credit_target = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", 1803 - smbd_send_credit_target, 1804 - sc->ib.dev->attrs.max_cqe, 1805 - sc->ib.dev->attrs.max_qp_wr); 1806 - goto config_failed; 1807 - } 1808 - 1809 - if (smbd_receive_credit_max > sc->ib.dev->attrs.max_cqe || 1810 - smbd_receive_credit_max > sc->ib.dev->attrs.max_qp_wr) { 1811 - log_rdma_event(ERR, "consider lowering receive_credit_max = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", 1812 - smbd_receive_credit_max, 1813 - sc->ib.dev->attrs.max_cqe, 1814 - sc->ib.dev->attrs.max_qp_wr); 1815 - goto config_failed; 1816 - } 1817 - 1572 + sp->resolve_addr_timeout_msec = RDMA_RESOLVE_TIMEOUT; 1573 + sp->resolve_route_timeout_msec = RDMA_RESOLVE_TIMEOUT; 1574 + sp->rdma_connect_timeout_msec = RDMA_RESOLVE_TIMEOUT; 1575 + sp->negotiate_timeout_msec = SMBD_NEGOTIATE_TIMEOUT * 1000; 1576 + sp->initiator_depth = 1; 1577 + sp->responder_resources = SMBD_CM_RESPONDER_RESOURCES; 1818 1578 sp->recv_credit_max = smbd_receive_credit_max; 1819 1579 sp->send_credit_target = smbd_send_credit_target; 1820 1580 sp->max_send_size = smbd_max_send_size; 1821 1581 sp->max_fragmented_recv_size = smbd_max_fragmented_recv_size; 1822 1582 sp->max_recv_size = smbd_max_receive_size; 1583 + sp->max_frmr_depth = smbd_max_frmr_depth; 1823 1584 sp->keepalive_interval_msec = smbd_keep_alive_interval * 1000; 1585 + sp->keepalive_timeout_msec = KEEPALIVE_RECV_TIMEOUT * 1000; 1586 + 1587 + rc = smbd_ia_open(sc, dstaddr, port); 1588 + if (rc) { 1589 + log_rdma_event(INFO, "smbd_ia_open rc=%d\n", rc); 1590 + goto create_id_failed; 1591 + } 1592 + 1593 + if (sp->send_credit_target > sc->ib.dev->attrs.max_cqe || 1594 + sp->send_credit_target > sc->ib.dev->attrs.max_qp_wr) { 1595 + log_rdma_event(ERR, "consider lowering send_credit_target = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", 1596 + sp->send_credit_target, 1597 + sc->ib.dev->attrs.max_cqe, 1598 + sc->ib.dev->attrs.max_qp_wr); 1599 + goto config_failed; 1600 + } 1601 + 1602 + if (sp->recv_credit_max > sc->ib.dev->attrs.max_cqe || 1603 + sp->recv_credit_max > sc->ib.dev->attrs.max_qp_wr) { 1604 + log_rdma_event(ERR, "consider lowering receive_credit_max = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", 1605 + sp->recv_credit_max, 1606 + sc->ib.dev->attrs.max_cqe, 1607 + sc->ib.dev->attrs.max_qp_wr); 1608 + goto config_failed; 1609 + } 1824 1610 1825 1611 if (sc->ib.dev->attrs.max_send_sge < SMBDIRECT_SEND_IO_MAX_SGE || 1826 1612 sc->ib.dev->attrs.max_recv_sge < SMBDIRECT_RECV_IO_MAX_SGE) { ··· 1850 1602 goto config_failed; 1851 1603 } 1852 1604 1605 + sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0); 1606 + if (IS_ERR(sc->ib.pd)) { 1607 + rc = PTR_ERR(sc->ib.pd); 1608 + sc->ib.pd = NULL; 1609 + log_rdma_event(ERR, "ib_alloc_pd() returned %d\n", rc); 1610 + goto alloc_pd_failed; 1611 + } 1612 + 1853 1613 sc->ib.send_cq = 1854 - ib_alloc_cq_any(sc->ib.dev, info, 1614 + ib_alloc_cq_any(sc->ib.dev, sc, 1855 1615 sp->send_credit_target, IB_POLL_SOFTIRQ); 1856 1616 if (IS_ERR(sc->ib.send_cq)) { 1857 1617 sc->ib.send_cq = NULL; ··· 1867 1611 } 1868 1612 1869 1613 sc->ib.recv_cq = 1870 - ib_alloc_cq_any(sc->ib.dev, info, 1614 + ib_alloc_cq_any(sc->ib.dev, sc, 1871 1615 sp->recv_credit_max, IB_POLL_SOFTIRQ); 1872 1616 if (IS_ERR(sc->ib.recv_cq)) { 1873 1617 sc->ib.recv_cq = NULL; ··· 1876 1620 1877 1621 memset(&qp_attr, 0, sizeof(qp_attr)); 1878 1622 qp_attr.event_handler = smbd_qp_async_error_upcall; 1879 - qp_attr.qp_context = info; 1623 + qp_attr.qp_context = sc; 1880 1624 qp_attr.cap.max_send_wr = sp->send_credit_target; 1881 1625 qp_attr.cap.max_recv_wr = sp->recv_credit_max; 1882 1626 qp_attr.cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE; ··· 1895 1639 } 1896 1640 sc->ib.qp = sc->rdma.cm_id->qp; 1897 1641 1898 - memset(&conn_param, 0, sizeof(conn_param)); 1899 - conn_param.initiator_depth = 0; 1900 - 1901 - conn_param.responder_resources = 1902 - min(sc->ib.dev->attrs.max_qp_rd_atom, 1903 - SMBD_CM_RESPONDER_RESOURCES); 1904 - info->responder_resources = conn_param.responder_resources; 1642 + sp->responder_resources = 1643 + min_t(u8, sp->responder_resources, 1644 + sc->ib.dev->attrs.max_qp_rd_atom); 1905 1645 log_rdma_mr(INFO, "responder_resources=%d\n", 1906 - info->responder_resources); 1646 + sp->responder_resources); 1647 + 1648 + memset(&conn_param, 0, sizeof(conn_param)); 1649 + conn_param.initiator_depth = sp->initiator_depth; 1650 + conn_param.responder_resources = sp->responder_resources; 1907 1651 1908 1652 /* Need to send IRD/ORD in private data for iWARP */ 1909 1653 sc->ib.dev->ops.get_port_immutable( 1910 1654 sc->ib.dev, sc->rdma.cm_id->port_num, &port_immutable); 1911 1655 if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) { 1912 - ird_ord_hdr[0] = info->responder_resources; 1913 - ird_ord_hdr[1] = 1; 1656 + ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources); 1657 + ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth); 1914 1658 conn_param.private_data = ird_ord_hdr; 1915 1659 conn_param.private_data_len = sizeof(ird_ord_hdr); 1916 1660 } else { ··· 1925 1669 log_rdma_event(INFO, "connecting to IP %pI4 port %d\n", 1926 1670 &addr_in->sin_addr, port); 1927 1671 1928 - init_waitqueue_head(&info->status_wait); 1929 - init_waitqueue_head(&sc->recv_io.reassembly.wait_queue); 1672 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED); 1673 + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING; 1930 1674 rc = rdma_connect(sc->rdma.cm_id, &conn_param); 1931 1675 if (rc) { 1932 1676 log_rdma_event(ERR, "rdma_connect() failed with %i\n", rc); ··· 1934 1678 } 1935 1679 1936 1680 wait_event_interruptible_timeout( 1937 - info->status_wait, 1938 - sc->status != SMBDIRECT_SOCKET_CONNECTING, 1939 - msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT)); 1681 + sc->status_wait, 1682 + sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING, 1683 + msecs_to_jiffies(sp->rdma_connect_timeout_msec)); 1940 1684 1941 - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { 1685 + if (sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED) { 1942 1686 log_rdma_event(ERR, "rdma_connect failed port=%d\n", port); 1943 1687 goto rdma_connect_failed; 1944 1688 } 1945 1689 1946 1690 log_rdma_event(INFO, "rdma_connect connected\n"); 1947 1691 1948 - rc = allocate_caches_and_workqueue(info); 1692 + rc = allocate_caches(sc); 1949 1693 if (rc) { 1950 1694 log_rdma_event(ERR, "cache allocation failed\n"); 1951 1695 goto allocate_cache_failed; 1952 1696 } 1953 1697 1954 - init_waitqueue_head(&info->wait_send_queue); 1955 - INIT_DELAYED_WORK(&info->idle_timer_work, idle_connection_timer); 1956 - queue_delayed_work(info->workqueue, &info->idle_timer_work, 1957 - msecs_to_jiffies(sp->keepalive_interval_msec)); 1698 + INIT_WORK(&sc->idle.immediate_work, send_immediate_empty_message); 1699 + INIT_DELAYED_WORK(&sc->idle.timer_work, idle_connection_timer); 1700 + /* 1701 + * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING 1702 + * so that the timer will cause a disconnect. 1703 + */ 1704 + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; 1705 + mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 1706 + msecs_to_jiffies(sp->negotiate_timeout_msec)); 1958 1707 1959 - init_waitqueue_head(&info->wait_send_pending); 1960 - atomic_set(&info->send_pending, 0); 1708 + INIT_WORK(&sc->recv_io.posted.refill_work, smbd_post_send_credits); 1961 1709 1962 - init_waitqueue_head(&info->wait_post_send); 1963 - 1964 - INIT_WORK(&info->disconnect_work, smbd_disconnect_rdma_work); 1965 - INIT_WORK(&info->post_send_credits_work, smbd_post_send_credits); 1966 - info->new_credits_offered = 0; 1967 - spin_lock_init(&info->lock_new_credits_offered); 1968 - 1969 - rc = smbd_negotiate(info); 1710 + rc = smbd_negotiate(sc); 1970 1711 if (rc) { 1971 1712 log_rdma_event(ERR, "smbd_negotiate rc=%d\n", rc); 1972 1713 goto negotiation_failed; 1973 1714 } 1974 1715 1975 - rc = allocate_mr_list(info); 1716 + rc = allocate_mr_list(sc); 1976 1717 if (rc) { 1977 1718 log_rdma_mr(ERR, "memory registration allocation failed\n"); 1978 1719 goto allocate_mr_failed; ··· 1984 1731 return NULL; 1985 1732 1986 1733 negotiation_failed: 1987 - disable_delayed_work_sync(&info->idle_timer_work); 1988 - destroy_caches_and_workqueue(info); 1734 + disable_delayed_work_sync(&sc->idle.timer_work); 1735 + destroy_caches(sc); 1989 1736 sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; 1990 1737 rdma_disconnect(sc->rdma.cm_id); 1991 - wait_event(info->status_wait, 1738 + wait_event(sc->status_wait, 1992 1739 sc->status == SMBDIRECT_SOCKET_DISCONNECTED); 1993 1740 1994 1741 allocate_cache_failed: ··· 2002 1749 if (sc->ib.recv_cq) 2003 1750 ib_free_cq(sc->ib.recv_cq); 2004 1751 2005 - config_failed: 2006 1752 ib_dealloc_pd(sc->ib.pd); 1753 + 1754 + alloc_pd_failed: 1755 + config_failed: 2007 1756 rdma_destroy_id(sc->rdma.cm_id); 2008 1757 2009 1758 create_id_failed: 1759 + destroy_workqueue(sc->workqueue); 1760 + create_wq_failed: 2010 1761 kfree(info); 2011 1762 return NULL; 2012 1763 } ··· 2019 1762 struct TCP_Server_Info *server, struct sockaddr *dstaddr) 2020 1763 { 2021 1764 struct smbd_connection *ret; 1765 + const struct smbdirect_socket_parameters *sp; 2022 1766 int port = SMBD_PORT; 2023 1767 2024 1768 try_again: ··· 2030 1772 port = SMB_PORT; 2031 1773 goto try_again; 2032 1774 } 1775 + if (!ret) 1776 + return NULL; 1777 + 1778 + sp = &ret->socket.parameters; 1779 + 1780 + server->rdma_readwrite_threshold = 1781 + rdma_readwrite_threshold > sp->max_fragmented_send_size ? 1782 + sp->max_fragmented_send_size : 1783 + rdma_readwrite_threshold; 1784 + 2033 1785 return ret; 2034 1786 } 2035 1787 ··· 2081 1813 if (sc->recv_io.reassembly.data_length >= size) { 2082 1814 int queue_length; 2083 1815 int queue_removed = 0; 1816 + unsigned long flags; 2084 1817 2085 1818 /* 2086 1819 * Need to make sure reassembly_data_length is read before ··· 2096 1827 to_read = size; 2097 1828 offset = sc->recv_io.reassembly.first_entry_offset; 2098 1829 while (data_read < size) { 2099 - response = _get_first_reassembly(info); 1830 + response = _get_first_reassembly(sc); 2100 1831 data_transfer = smbdirect_recv_io_payload(response); 2101 1832 data_length = le32_to_cpu(data_transfer->data_length); 2102 1833 remaining_data_length = ··· 2141 1872 if (queue_length) 2142 1873 list_del(&response->list); 2143 1874 else { 2144 - spin_lock_irq( 2145 - &sc->recv_io.reassembly.lock); 1875 + spin_lock_irqsave( 1876 + &sc->recv_io.reassembly.lock, flags); 2146 1877 list_del(&response->list); 2147 - spin_unlock_irq( 2148 - &sc->recv_io.reassembly.lock); 1878 + spin_unlock_irqrestore( 1879 + &sc->recv_io.reassembly.lock, flags); 2149 1880 } 2150 1881 queue_removed++; 2151 - info->count_reassembly_queue--; 2152 - info->count_dequeue_reassembly_queue++; 2153 - put_receive_buffer(info, response); 1882 + sc->statistics.dequeue_reassembly_queue++; 1883 + put_receive_buffer(sc, response); 2154 1884 offset = 0; 2155 1885 log_read(INFO, "put_receive_buffer offset=0\n"); 2156 1886 } else ··· 2163 1895 to_read, data_read, offset); 2164 1896 } 2165 1897 2166 - spin_lock_irq(&sc->recv_io.reassembly.lock); 1898 + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 2167 1899 sc->recv_io.reassembly.data_length -= data_read; 2168 1900 sc->recv_io.reassembly.queue_length -= queue_removed; 2169 - spin_unlock_irq(&sc->recv_io.reassembly.lock); 1901 + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 2170 1902 2171 1903 sc->recv_io.reassembly.first_entry_offset = offset; 2172 1904 log_read(INFO, "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n", ··· 2251 1983 klen += rqst->rq_iov[i].iov_len; 2252 1984 iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, klen); 2253 1985 2254 - rc = smbd_post_send_full_iter(info, &iter, &remaining_data_length); 1986 + rc = smbd_post_send_full_iter(sc, &iter, &remaining_data_length); 2255 1987 if (rc < 0) 2256 1988 break; 2257 1989 2258 1990 if (iov_iter_count(&rqst->rq_iter) > 0) { 2259 1991 /* And then the data pages if there are any */ 2260 - rc = smbd_post_send_full_iter(info, &rqst->rq_iter, 1992 + rc = smbd_post_send_full_iter(sc, &rqst->rq_iter, 2261 1993 &remaining_data_length); 2262 1994 if (rc < 0) 2263 1995 break; ··· 2272 2004 * that means all the I/Os have been out and we are good to return 2273 2005 */ 2274 2006 2275 - wait_event(info->wait_send_pending, 2276 - atomic_read(&info->send_pending) == 0 || 2007 + wait_event(sc->send_io.pending.zero_wait_queue, 2008 + atomic_read(&sc->send_io.pending.count) == 0 || 2277 2009 sc->status != SMBDIRECT_SOCKET_CONNECTED); 2278 2010 2279 2011 if (sc->status != SMBDIRECT_SOCKET_CONNECTED && rc == 0) ··· 2284 2016 2285 2017 static void register_mr_done(struct ib_cq *cq, struct ib_wc *wc) 2286 2018 { 2287 - struct smbd_mr *mr; 2288 - struct ib_cqe *cqe; 2019 + struct smbdirect_mr_io *mr = 2020 + container_of(wc->wr_cqe, struct smbdirect_mr_io, cqe); 2021 + struct smbdirect_socket *sc = mr->socket; 2289 2022 2290 2023 if (wc->status) { 2291 2024 log_rdma_mr(ERR, "status=%d\n", wc->status); 2292 - cqe = wc->wr_cqe; 2293 - mr = container_of(cqe, struct smbd_mr, cqe); 2294 - smbd_disconnect_rdma_connection(mr->conn); 2025 + smbd_disconnect_rdma_connection(sc); 2295 2026 } 2296 2027 } 2297 2028 ··· 2305 2038 */ 2306 2039 static void smbd_mr_recovery_work(struct work_struct *work) 2307 2040 { 2308 - struct smbd_connection *info = 2309 - container_of(work, struct smbd_connection, mr_recovery_work); 2310 - struct smbdirect_socket *sc = &info->socket; 2311 - struct smbd_mr *smbdirect_mr; 2041 + struct smbdirect_socket *sc = 2042 + container_of(work, struct smbdirect_socket, mr_io.recovery_work); 2043 + struct smbdirect_socket_parameters *sp = &sc->parameters; 2044 + struct smbdirect_mr_io *smbdirect_mr; 2312 2045 int rc; 2313 2046 2314 - list_for_each_entry(smbdirect_mr, &info->mr_list, list) { 2315 - if (smbdirect_mr->state == MR_ERROR) { 2047 + list_for_each_entry(smbdirect_mr, &sc->mr_io.all.list, list) { 2048 + if (smbdirect_mr->state == SMBDIRECT_MR_ERROR) { 2316 2049 2317 2050 /* recover this MR entry */ 2318 2051 rc = ib_dereg_mr(smbdirect_mr->mr); ··· 2320 2053 log_rdma_mr(ERR, 2321 2054 "ib_dereg_mr failed rc=%x\n", 2322 2055 rc); 2323 - smbd_disconnect_rdma_connection(info); 2056 + smbd_disconnect_rdma_connection(sc); 2324 2057 continue; 2325 2058 } 2326 2059 2327 2060 smbdirect_mr->mr = ib_alloc_mr( 2328 - sc->ib.pd, info->mr_type, 2329 - info->max_frmr_depth); 2061 + sc->ib.pd, sc->mr_io.type, 2062 + sp->max_frmr_depth); 2330 2063 if (IS_ERR(smbdirect_mr->mr)) { 2331 2064 log_rdma_mr(ERR, "ib_alloc_mr failed mr_type=%x max_frmr_depth=%x\n", 2332 - info->mr_type, 2333 - info->max_frmr_depth); 2334 - smbd_disconnect_rdma_connection(info); 2065 + sc->mr_io.type, 2066 + sp->max_frmr_depth); 2067 + smbd_disconnect_rdma_connection(sc); 2335 2068 continue; 2336 2069 } 2337 2070 } else 2338 2071 /* This MR is being used, don't recover it */ 2339 2072 continue; 2340 2073 2341 - smbdirect_mr->state = MR_READY; 2074 + smbdirect_mr->state = SMBDIRECT_MR_READY; 2342 2075 2343 2076 /* smbdirect_mr->state is updated by this function 2344 2077 * and is read and updated by I/O issuing CPUs trying ··· 2347 2080 * value is updated before waking up any calls to 2348 2081 * get_mr() from the I/O issuing CPUs 2349 2082 */ 2350 - if (atomic_inc_return(&info->mr_ready_count) == 1) 2351 - wake_up_interruptible(&info->wait_mr); 2083 + if (atomic_inc_return(&sc->mr_io.ready.count) == 1) 2084 + wake_up(&sc->mr_io.ready.wait_queue); 2352 2085 } 2353 2086 } 2354 2087 2355 - static void destroy_mr_list(struct smbd_connection *info) 2088 + static void destroy_mr_list(struct smbdirect_socket *sc) 2356 2089 { 2357 - struct smbdirect_socket *sc = &info->socket; 2358 - struct smbd_mr *mr, *tmp; 2090 + struct smbdirect_mr_io *mr, *tmp; 2359 2091 2360 - disable_work_sync(&info->mr_recovery_work); 2361 - list_for_each_entry_safe(mr, tmp, &info->mr_list, list) { 2362 - if (mr->state == MR_INVALIDATED) 2092 + disable_work_sync(&sc->mr_io.recovery_work); 2093 + list_for_each_entry_safe(mr, tmp, &sc->mr_io.all.list, list) { 2094 + if (mr->state == SMBDIRECT_MR_INVALIDATED) 2363 2095 ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, 2364 2096 mr->sgt.nents, mr->dir); 2365 2097 ib_dereg_mr(mr->mr); ··· 2374 2108 * Recovery is done in smbd_mr_recovery_work. The content of list entry changes 2375 2109 * as MRs are used and recovered for I/O, but the list links will not change 2376 2110 */ 2377 - static int allocate_mr_list(struct smbd_connection *info) 2111 + static int allocate_mr_list(struct smbdirect_socket *sc) 2378 2112 { 2379 - struct smbdirect_socket *sc = &info->socket; 2113 + struct smbdirect_socket_parameters *sp = &sc->parameters; 2380 2114 int i; 2381 - struct smbd_mr *smbdirect_mr, *tmp; 2115 + struct smbdirect_mr_io *smbdirect_mr, *tmp; 2382 2116 2383 - INIT_LIST_HEAD(&info->mr_list); 2384 - init_waitqueue_head(&info->wait_mr); 2385 - spin_lock_init(&info->mr_list_lock); 2386 - atomic_set(&info->mr_ready_count, 0); 2387 - atomic_set(&info->mr_used_count, 0); 2388 - init_waitqueue_head(&info->wait_for_mr_cleanup); 2389 - INIT_WORK(&info->mr_recovery_work, smbd_mr_recovery_work); 2117 + INIT_WORK(&sc->mr_io.recovery_work, smbd_mr_recovery_work); 2118 + 2119 + if (sp->responder_resources == 0) { 2120 + log_rdma_mr(ERR, "responder_resources negotiated as 0\n"); 2121 + return -EINVAL; 2122 + } 2123 + 2390 2124 /* Allocate more MRs (2x) than hardware responder_resources */ 2391 - for (i = 0; i < info->responder_resources * 2; i++) { 2125 + for (i = 0; i < sp->responder_resources * 2; i++) { 2392 2126 smbdirect_mr = kzalloc(sizeof(*smbdirect_mr), GFP_KERNEL); 2393 2127 if (!smbdirect_mr) 2394 2128 goto cleanup_entries; 2395 - smbdirect_mr->mr = ib_alloc_mr(sc->ib.pd, info->mr_type, 2396 - info->max_frmr_depth); 2129 + smbdirect_mr->mr = ib_alloc_mr(sc->ib.pd, sc->mr_io.type, 2130 + sp->max_frmr_depth); 2397 2131 if (IS_ERR(smbdirect_mr->mr)) { 2398 2132 log_rdma_mr(ERR, "ib_alloc_mr failed mr_type=%x max_frmr_depth=%x\n", 2399 - info->mr_type, info->max_frmr_depth); 2133 + sc->mr_io.type, sp->max_frmr_depth); 2400 2134 goto out; 2401 2135 } 2402 - smbdirect_mr->sgt.sgl = kcalloc(info->max_frmr_depth, 2136 + smbdirect_mr->sgt.sgl = kcalloc(sp->max_frmr_depth, 2403 2137 sizeof(struct scatterlist), 2404 2138 GFP_KERNEL); 2405 2139 if (!smbdirect_mr->sgt.sgl) { ··· 2407 2141 ib_dereg_mr(smbdirect_mr->mr); 2408 2142 goto out; 2409 2143 } 2410 - smbdirect_mr->state = MR_READY; 2411 - smbdirect_mr->conn = info; 2144 + smbdirect_mr->state = SMBDIRECT_MR_READY; 2145 + smbdirect_mr->socket = sc; 2412 2146 2413 - list_add_tail(&smbdirect_mr->list, &info->mr_list); 2414 - atomic_inc(&info->mr_ready_count); 2147 + list_add_tail(&smbdirect_mr->list, &sc->mr_io.all.list); 2148 + atomic_inc(&sc->mr_io.ready.count); 2415 2149 } 2416 2150 return 0; 2417 2151 2418 2152 out: 2419 2153 kfree(smbdirect_mr); 2420 2154 cleanup_entries: 2421 - list_for_each_entry_safe(smbdirect_mr, tmp, &info->mr_list, list) { 2155 + list_for_each_entry_safe(smbdirect_mr, tmp, &sc->mr_io.all.list, list) { 2422 2156 list_del(&smbdirect_mr->list); 2423 2157 ib_dereg_mr(smbdirect_mr->mr); 2424 2158 kfree(smbdirect_mr->sgt.sgl); ··· 2435 2169 * issuing I/O trying to get MR at the same time, mr_list_lock is used to 2436 2170 * protect this situation. 2437 2171 */ 2438 - static struct smbd_mr *get_mr(struct smbd_connection *info) 2172 + static struct smbdirect_mr_io *get_mr(struct smbdirect_socket *sc) 2439 2173 { 2440 - struct smbdirect_socket *sc = &info->socket; 2441 - struct smbd_mr *ret; 2174 + struct smbdirect_mr_io *ret; 2175 + unsigned long flags; 2442 2176 int rc; 2443 2177 again: 2444 - rc = wait_event_interruptible(info->wait_mr, 2445 - atomic_read(&info->mr_ready_count) || 2178 + rc = wait_event_interruptible(sc->mr_io.ready.wait_queue, 2179 + atomic_read(&sc->mr_io.ready.count) || 2446 2180 sc->status != SMBDIRECT_SOCKET_CONNECTED); 2447 2181 if (rc) { 2448 2182 log_rdma_mr(ERR, "wait_event_interruptible rc=%x\n", rc); ··· 2454 2188 return NULL; 2455 2189 } 2456 2190 2457 - spin_lock(&info->mr_list_lock); 2458 - list_for_each_entry(ret, &info->mr_list, list) { 2459 - if (ret->state == MR_READY) { 2460 - ret->state = MR_REGISTERED; 2461 - spin_unlock(&info->mr_list_lock); 2462 - atomic_dec(&info->mr_ready_count); 2463 - atomic_inc(&info->mr_used_count); 2191 + spin_lock_irqsave(&sc->mr_io.all.lock, flags); 2192 + list_for_each_entry(ret, &sc->mr_io.all.list, list) { 2193 + if (ret->state == SMBDIRECT_MR_READY) { 2194 + ret->state = SMBDIRECT_MR_REGISTERED; 2195 + spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); 2196 + atomic_dec(&sc->mr_io.ready.count); 2197 + atomic_inc(&sc->mr_io.used.count); 2464 2198 return ret; 2465 2199 } 2466 2200 } 2467 2201 2468 - spin_unlock(&info->mr_list_lock); 2202 + spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); 2469 2203 /* 2470 2204 * It is possible that we could fail to get MR because other processes may 2471 2205 * try to acquire a MR at the same time. If this is the case, retry it. ··· 2476 2210 /* 2477 2211 * Transcribe the pages from an iterator into an MR scatterlist. 2478 2212 */ 2479 - static int smbd_iter_to_mr(struct smbd_connection *info, 2480 - struct iov_iter *iter, 2213 + static int smbd_iter_to_mr(struct iov_iter *iter, 2481 2214 struct sg_table *sgt, 2482 2215 unsigned int max_sg) 2483 2216 { ··· 2498 2233 * need_invalidate: true if this MR needs to be locally invalidated after I/O 2499 2234 * return value: the MR registered, NULL if failed. 2500 2235 */ 2501 - struct smbd_mr *smbd_register_mr(struct smbd_connection *info, 2236 + struct smbdirect_mr_io *smbd_register_mr(struct smbd_connection *info, 2502 2237 struct iov_iter *iter, 2503 2238 bool writing, bool need_invalidate) 2504 2239 { 2505 2240 struct smbdirect_socket *sc = &info->socket; 2506 - struct smbd_mr *smbdirect_mr; 2241 + struct smbdirect_socket_parameters *sp = &sc->parameters; 2242 + struct smbdirect_mr_io *smbdirect_mr; 2507 2243 int rc, num_pages; 2508 2244 enum dma_data_direction dir; 2509 2245 struct ib_reg_wr *reg_wr; 2510 2246 2511 - num_pages = iov_iter_npages(iter, info->max_frmr_depth + 1); 2512 - if (num_pages > info->max_frmr_depth) { 2247 + num_pages = iov_iter_npages(iter, sp->max_frmr_depth + 1); 2248 + if (num_pages > sp->max_frmr_depth) { 2513 2249 log_rdma_mr(ERR, "num_pages=%d max_frmr_depth=%d\n", 2514 - num_pages, info->max_frmr_depth); 2250 + num_pages, sp->max_frmr_depth); 2515 2251 WARN_ON_ONCE(1); 2516 2252 return NULL; 2517 2253 } 2518 2254 2519 - smbdirect_mr = get_mr(info); 2255 + smbdirect_mr = get_mr(sc); 2520 2256 if (!smbdirect_mr) { 2521 2257 log_rdma_mr(ERR, "get_mr returning NULL\n"); 2522 2258 return NULL; ··· 2530 2264 smbdirect_mr->sgt.orig_nents = 0; 2531 2265 2532 2266 log_rdma_mr(INFO, "num_pages=0x%x count=0x%zx depth=%u\n", 2533 - num_pages, iov_iter_count(iter), info->max_frmr_depth); 2534 - smbd_iter_to_mr(info, iter, &smbdirect_mr->sgt, info->max_frmr_depth); 2267 + num_pages, iov_iter_count(iter), sp->max_frmr_depth); 2268 + smbd_iter_to_mr(iter, &smbdirect_mr->sgt, sp->max_frmr_depth); 2535 2269 2536 2270 rc = ib_dma_map_sg(sc->ib.dev, smbdirect_mr->sgt.sgl, 2537 2271 smbdirect_mr->sgt.nents, dir); ··· 2576 2310 log_rdma_mr(ERR, "ib_post_send failed rc=%x reg_wr->key=%x\n", 2577 2311 rc, reg_wr->key); 2578 2312 2579 - /* If all failed, attempt to recover this MR by setting it MR_ERROR*/ 2313 + /* If all failed, attempt to recover this MR by setting it SMBDIRECT_MR_ERROR*/ 2580 2314 map_mr_error: 2581 2315 ib_dma_unmap_sg(sc->ib.dev, smbdirect_mr->sgt.sgl, 2582 2316 smbdirect_mr->sgt.nents, smbdirect_mr->dir); 2583 2317 2584 2318 dma_map_error: 2585 - smbdirect_mr->state = MR_ERROR; 2586 - if (atomic_dec_and_test(&info->mr_used_count)) 2587 - wake_up(&info->wait_for_mr_cleanup); 2319 + smbdirect_mr->state = SMBDIRECT_MR_ERROR; 2320 + if (atomic_dec_and_test(&sc->mr_io.used.count)) 2321 + wake_up(&sc->mr_io.cleanup.wait_queue); 2588 2322 2589 - smbd_disconnect_rdma_connection(info); 2323 + smbd_disconnect_rdma_connection(sc); 2590 2324 2591 2325 return NULL; 2592 2326 } 2593 2327 2594 2328 static void local_inv_done(struct ib_cq *cq, struct ib_wc *wc) 2595 2329 { 2596 - struct smbd_mr *smbdirect_mr; 2330 + struct smbdirect_mr_io *smbdirect_mr; 2597 2331 struct ib_cqe *cqe; 2598 2332 2599 2333 cqe = wc->wr_cqe; 2600 - smbdirect_mr = container_of(cqe, struct smbd_mr, cqe); 2601 - smbdirect_mr->state = MR_INVALIDATED; 2334 + smbdirect_mr = container_of(cqe, struct smbdirect_mr_io, cqe); 2335 + smbdirect_mr->state = SMBDIRECT_MR_INVALIDATED; 2602 2336 if (wc->status != IB_WC_SUCCESS) { 2603 2337 log_rdma_mr(ERR, "invalidate failed status=%x\n", wc->status); 2604 - smbdirect_mr->state = MR_ERROR; 2338 + smbdirect_mr->state = SMBDIRECT_MR_ERROR; 2605 2339 } 2606 2340 complete(&smbdirect_mr->invalidate_done); 2607 2341 } ··· 2612 2346 * and we have to locally invalidate the buffer to prevent data is being 2613 2347 * modified by remote peer after upper layer consumes it 2614 2348 */ 2615 - int smbd_deregister_mr(struct smbd_mr *smbdirect_mr) 2349 + int smbd_deregister_mr(struct smbdirect_mr_io *smbdirect_mr) 2616 2350 { 2617 2351 struct ib_send_wr *wr; 2618 - struct smbd_connection *info = smbdirect_mr->conn; 2619 - struct smbdirect_socket *sc = &info->socket; 2352 + struct smbdirect_socket *sc = smbdirect_mr->socket; 2620 2353 int rc = 0; 2621 2354 2622 2355 if (smbdirect_mr->need_invalidate) { ··· 2632 2367 rc = ib_post_send(sc->ib.qp, wr, NULL); 2633 2368 if (rc) { 2634 2369 log_rdma_mr(ERR, "ib_post_send failed rc=%x\n", rc); 2635 - smbd_disconnect_rdma_connection(info); 2370 + smbd_disconnect_rdma_connection(sc); 2636 2371 goto done; 2637 2372 } 2638 2373 wait_for_completion(&smbdirect_mr->invalidate_done); 2639 2374 smbdirect_mr->need_invalidate = false; 2640 2375 } else 2641 2376 /* 2642 - * For remote invalidation, just set it to MR_INVALIDATED 2377 + * For remote invalidation, just set it to SMBDIRECT_MR_INVALIDATED 2643 2378 * and defer to mr_recovery_work to recover the MR for next use 2644 2379 */ 2645 - smbdirect_mr->state = MR_INVALIDATED; 2380 + smbdirect_mr->state = SMBDIRECT_MR_INVALIDATED; 2646 2381 2647 - if (smbdirect_mr->state == MR_INVALIDATED) { 2382 + if (smbdirect_mr->state == SMBDIRECT_MR_INVALIDATED) { 2648 2383 ib_dma_unmap_sg( 2649 2384 sc->ib.dev, smbdirect_mr->sgt.sgl, 2650 2385 smbdirect_mr->sgt.nents, 2651 2386 smbdirect_mr->dir); 2652 - smbdirect_mr->state = MR_READY; 2653 - if (atomic_inc_return(&info->mr_ready_count) == 1) 2654 - wake_up_interruptible(&info->wait_mr); 2387 + smbdirect_mr->state = SMBDIRECT_MR_READY; 2388 + if (atomic_inc_return(&sc->mr_io.ready.count) == 1) 2389 + wake_up(&sc->mr_io.ready.wait_queue); 2655 2390 } else 2656 2391 /* 2657 2392 * Schedule the work to do MR recovery for future I/Os MR 2658 2393 * recovery is slow and don't want it to block current I/O 2659 2394 */ 2660 - queue_work(info->workqueue, &info->mr_recovery_work); 2395 + queue_work(sc->workqueue, &sc->mr_io.recovery_work); 2661 2396 2662 2397 done: 2663 - if (atomic_dec_and_test(&info->mr_used_count)) 2664 - wake_up(&info->wait_for_mr_cleanup); 2398 + if (atomic_dec_and_test(&sc->mr_io.used.count)) 2399 + wake_up(&sc->mr_io.cleanup.wait_queue); 2665 2400 2666 2401 return rc; 2667 2402 }

+4 -98

fs/smb/client/smbdirect.h

··· 27 27 extern int smbd_send_credit_target; 28 28 extern int smbd_receive_credit_max; 29 29 30 - enum keep_alive_status { 31 - KEEP_ALIVE_NONE, 32 - KEEP_ALIVE_PENDING, 33 - KEEP_ALIVE_SENT, 34 - }; 35 - 36 30 /* 37 31 * The context for the SMBDirect transport 38 32 * Everything related to the transport is here. It has several logical parts ··· 38 44 */ 39 45 struct smbd_connection { 40 46 struct smbdirect_socket socket; 41 - 42 - int ri_rc; 43 - struct completion ri_done; 44 - wait_queue_head_t status_wait; 45 - 46 - struct completion negotiate_completion; 47 - bool negotiate_done; 48 - 49 - struct work_struct disconnect_work; 50 - struct work_struct post_send_credits_work; 51 - 52 - spinlock_t lock_new_credits_offered; 53 - int new_credits_offered; 54 - 55 - /* dynamic connection parameters defined in [MS-SMBD] 3.1.1.1 */ 56 - enum keep_alive_status keep_alive_requested; 57 - int protocol; 58 - atomic_t send_credits; 59 - atomic_t receive_credits; 60 - int receive_credit_target; 61 - 62 - /* Memory registrations */ 63 - /* Maximum number of RDMA read/write outstanding on this connection */ 64 - int responder_resources; 65 - /* Maximum number of pages in a single RDMA write/read on this connection */ 66 - int max_frmr_depth; 67 - /* 68 - * If payload is less than or equal to the threshold, 69 - * use RDMA send/recv to send upper layer I/O. 70 - * If payload is more than the threshold, 71 - * use RDMA read/write through memory registration for I/O. 72 - */ 73 - int rdma_readwrite_threshold; 74 - enum ib_mr_type mr_type; 75 - struct list_head mr_list; 76 - spinlock_t mr_list_lock; 77 - /* The number of available MRs ready for memory registration */ 78 - atomic_t mr_ready_count; 79 - atomic_t mr_used_count; 80 - wait_queue_head_t wait_mr; 81 - struct work_struct mr_recovery_work; 82 - /* Used by transport to wait until all MRs are returned */ 83 - wait_queue_head_t wait_for_mr_cleanup; 84 - 85 - /* Activity accounting */ 86 - atomic_t send_pending; 87 - wait_queue_head_t wait_send_pending; 88 - wait_queue_head_t wait_post_send; 89 - 90 - /* Receive queue */ 91 - int count_receive_queue; 92 - wait_queue_head_t wait_receive_queues; 93 - 94 - bool send_immediate; 95 - 96 - wait_queue_head_t wait_send_queue; 97 - 98 - struct workqueue_struct *workqueue; 99 - struct delayed_work idle_timer_work; 100 - 101 - /* for debug purposes */ 102 - unsigned int count_get_receive_buffer; 103 - unsigned int count_put_receive_buffer; 104 - unsigned int count_reassembly_queue; 105 - unsigned int count_enqueue_reassembly_queue; 106 - unsigned int count_dequeue_reassembly_queue; 107 - unsigned int count_send_empty; 108 47 }; 109 48 110 49 /* Create a SMBDirect session */ 111 50 struct smbd_connection *smbd_get_connection( 112 51 struct TCP_Server_Info *server, struct sockaddr *dstaddr); 52 + 53 + const struct smbdirect_socket_parameters *smbd_get_parameters(struct smbd_connection *conn); 113 54 114 55 /* Reconnect SMBDirect session */ 115 56 int smbd_reconnect(struct TCP_Server_Info *server); ··· 56 127 int smbd_send(struct TCP_Server_Info *server, 57 128 int num_rqst, struct smb_rqst *rqst); 58 129 59 - enum mr_state { 60 - MR_READY, 61 - MR_REGISTERED, 62 - MR_INVALIDATED, 63 - MR_ERROR 64 - }; 65 - 66 - struct smbd_mr { 67 - struct smbd_connection *conn; 68 - struct list_head list; 69 - enum mr_state state; 70 - struct ib_mr *mr; 71 - struct sg_table sgt; 72 - enum dma_data_direction dir; 73 - union { 74 - struct ib_reg_wr wr; 75 - struct ib_send_wr inv_wr; 76 - }; 77 - struct ib_cqe cqe; 78 - bool need_invalidate; 79 - struct completion invalidate_done; 80 - }; 81 - 82 130 /* Interfaces to register and deregister MR for RDMA read/write */ 83 - struct smbd_mr *smbd_register_mr( 131 + struct smbdirect_mr_io *smbd_register_mr( 84 132 struct smbd_connection *info, struct iov_iter *iter, 85 133 bool writing, bool need_invalidate); 86 - int smbd_deregister_mr(struct smbd_mr *mr); 134 + int smbd_deregister_mr(struct smbdirect_mr_io *mr); 87 135 88 136 #else 89 137 #define cifs_rdma_enabled(server) 0

+7

fs/smb/common/smbdirect/smbdirect.h

··· 23 23 * Some values are important for the upper layer. 24 24 */ 25 25 struct smbdirect_socket_parameters { 26 + __u32 resolve_addr_timeout_msec; 27 + __u32 resolve_route_timeout_msec; 28 + __u32 rdma_connect_timeout_msec; 29 + __u32 negotiate_timeout_msec; 30 + __u8 initiator_depth; 31 + __u8 responder_resources; 26 32 __u16 recv_credit_max; 27 33 __u16 send_credit_target; 28 34 __u32 max_send_size; ··· 36 30 __u32 max_recv_size; 37 31 __u32 max_fragmented_recv_size; 38 32 __u32 max_read_write_size; 33 + __u32 max_frmr_depth; 39 34 __u32 keepalive_interval_msec; 40 35 __u32 keepalive_timeout_msec; 41 36 } __packed;

+317 -2

fs/smb/common/smbdirect/smbdirect_socket.h

··· 6 6 #ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ 7 7 #define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ 8 8 9 + #include <rdma/rw.h> 10 + 9 11 enum smbdirect_socket_status { 10 12 SMBDIRECT_SOCKET_CREATED, 11 - SMBDIRECT_SOCKET_CONNECTING, 12 - SMBDIRECT_SOCKET_CONNECTED, 13 + SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED, 14 + SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING, 15 + SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED, 16 + SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED, 17 + SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING, 18 + SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED, 19 + SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED, 20 + SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING, 21 + SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED, 22 + SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, 23 + SMBDIRECT_SOCKET_NEGOTIATE_RUNNING, 13 24 SMBDIRECT_SOCKET_NEGOTIATE_FAILED, 25 + SMBDIRECT_SOCKET_CONNECTED, 26 + SMBDIRECT_SOCKET_ERROR, 14 27 SMBDIRECT_SOCKET_DISCONNECTING, 15 28 SMBDIRECT_SOCKET_DISCONNECTED, 16 29 SMBDIRECT_SOCKET_DESTROYED 17 30 }; 18 31 32 + static __always_inline 33 + const char *smbdirect_socket_status_string(enum smbdirect_socket_status status) 34 + { 35 + switch (status) { 36 + case SMBDIRECT_SOCKET_CREATED: 37 + return "CREATED"; 38 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: 39 + return "RESOLVE_ADDR_NEEDED"; 40 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: 41 + return "RESOLVE_ADDR_RUNNING"; 42 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: 43 + return "RESOLVE_ADDR_FAILED"; 44 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: 45 + return "RESOLVE_ROUTE_NEEDED"; 46 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: 47 + return "RESOLVE_ROUTE_RUNNING"; 48 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: 49 + return "RESOLVE_ROUTE_FAILED"; 50 + case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: 51 + return "RDMA_CONNECT_NEEDED"; 52 + case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: 53 + return "RDMA_CONNECT_RUNNING"; 54 + case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: 55 + return "RDMA_CONNECT_FAILED"; 56 + case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: 57 + return "NEGOTIATE_NEEDED"; 58 + case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: 59 + return "NEGOTIATE_RUNNING"; 60 + case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: 61 + return "NEGOTIATE_FAILED"; 62 + case SMBDIRECT_SOCKET_CONNECTED: 63 + return "CONNECTED"; 64 + case SMBDIRECT_SOCKET_ERROR: 65 + return "ERROR"; 66 + case SMBDIRECT_SOCKET_DISCONNECTING: 67 + return "DISCONNECTING"; 68 + case SMBDIRECT_SOCKET_DISCONNECTED: 69 + return "DISCONNECTED"; 70 + case SMBDIRECT_SOCKET_DESTROYED: 71 + return "DESTROYED"; 72 + } 73 + 74 + return "<unknown>"; 75 + } 76 + 77 + enum smbdirect_keepalive_status { 78 + SMBDIRECT_KEEPALIVE_NONE, 79 + SMBDIRECT_KEEPALIVE_PENDING, 80 + SMBDIRECT_KEEPALIVE_SENT 81 + }; 82 + 19 83 struct smbdirect_socket { 20 84 enum smbdirect_socket_status status; 85 + wait_queue_head_t status_wait; 86 + int first_error; 87 + 88 + /* 89 + * This points to the workqueue to 90 + * be used for this socket. 91 + * It can be per socket (on the client) 92 + * or point to a global workqueue (on the server) 93 + */ 94 + struct workqueue_struct *workqueue; 95 + 96 + struct work_struct disconnect_work; 21 97 22 98 /* RDMA related */ 23 99 struct { 24 100 struct rdma_cm_id *cm_id; 101 + /* 102 + * This is for iWarp MPA v1 103 + */ 104 + bool legacy_iwarp; 25 105 } rdma; 26 106 27 107 /* IB verbs related */ ··· 120 40 struct smbdirect_socket_parameters parameters; 121 41 122 42 /* 43 + * The state for keepalive and timeout handling 44 + */ 45 + struct { 46 + enum smbdirect_keepalive_status keepalive; 47 + struct work_struct immediate_work; 48 + struct delayed_work timer_work; 49 + } idle; 50 + 51 + /* 123 52 * The state for posted send buffers 124 53 */ 125 54 struct { ··· 140 51 struct kmem_cache *cache; 141 52 mempool_t *pool; 142 53 } mem; 54 + 55 + /* 56 + * The credit state for the send side 57 + */ 58 + struct { 59 + atomic_t count; 60 + wait_queue_head_t wait_queue; 61 + } credits; 62 + 63 + /* 64 + * The state about posted/pending sends 65 + */ 66 + struct { 67 + atomic_t count; 68 + /* 69 + * woken when count is decremented 70 + */ 71 + wait_queue_head_t dec_wait_queue; 72 + /* 73 + * woken when count reached zero 74 + */ 75 + wait_queue_head_t zero_wait_queue; 76 + } pending; 143 77 } send_io; 144 78 145 79 /* ··· 197 85 } free; 198 86 199 87 /* 88 + * The state for posted recv_io messages 89 + * and the refill work struct. 90 + */ 91 + struct { 92 + atomic_t count; 93 + struct work_struct refill_work; 94 + } posted; 95 + 96 + /* 97 + * The credit state for the recv side 98 + */ 99 + struct { 100 + u16 target; 101 + atomic_t count; 102 + } credits; 103 + 104 + /* 200 105 * The list of arrived non-empty smbdirect_recv_io 201 106 * structures 202 107 * ··· 239 110 bool full_packet_received; 240 111 } reassembly; 241 112 } recv_io; 113 + 114 + /* 115 + * The state for Memory registrations on the client 116 + */ 117 + struct { 118 + enum ib_mr_type type; 119 + 120 + /* 121 + * The list of free smbdirect_mr_io 122 + * structures 123 + */ 124 + struct { 125 + struct list_head list; 126 + spinlock_t lock; 127 + } all; 128 + 129 + /* 130 + * The number of available MRs ready for memory registration 131 + */ 132 + struct { 133 + atomic_t count; 134 + wait_queue_head_t wait_queue; 135 + } ready; 136 + 137 + /* 138 + * The number of used MRs 139 + */ 140 + struct { 141 + atomic_t count; 142 + } used; 143 + 144 + struct work_struct recovery_work; 145 + 146 + /* Used by transport to wait until all MRs are returned */ 147 + struct { 148 + wait_queue_head_t wait_queue; 149 + } cleanup; 150 + } mr_io; 151 + 152 + /* 153 + * The state for RDMA read/write requests on the server 154 + */ 155 + struct { 156 + /* 157 + * The credit state for the send side 158 + */ 159 + struct { 160 + /* 161 + * The maximum number of rw credits 162 + */ 163 + size_t max; 164 + /* 165 + * The number of pages per credit 166 + */ 167 + size_t num_pages; 168 + atomic_t count; 169 + wait_queue_head_t wait_queue; 170 + } credits; 171 + } rw_io; 172 + 173 + /* 174 + * For debug purposes 175 + */ 176 + struct { 177 + u64 get_receive_buffer; 178 + u64 put_receive_buffer; 179 + u64 enqueue_reassembly_queue; 180 + u64 dequeue_reassembly_queue; 181 + u64 send_empty; 182 + } statistics; 242 183 }; 184 + 185 + static void __smbdirect_socket_disabled_work(struct work_struct *work) 186 + { 187 + /* 188 + * Should never be called as disable_[delayed_]work_sync() was used. 189 + */ 190 + WARN_ON_ONCE(1); 191 + } 192 + 193 + static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) 194 + { 195 + /* 196 + * This also sets status = SMBDIRECT_SOCKET_CREATED 197 + */ 198 + BUILD_BUG_ON(SMBDIRECT_SOCKET_CREATED != 0); 199 + memset(sc, 0, sizeof(*sc)); 200 + 201 + init_waitqueue_head(&sc->status_wait); 202 + 203 + INIT_WORK(&sc->disconnect_work, __smbdirect_socket_disabled_work); 204 + disable_work_sync(&sc->disconnect_work); 205 + 206 + INIT_WORK(&sc->idle.immediate_work, __smbdirect_socket_disabled_work); 207 + disable_work_sync(&sc->idle.immediate_work); 208 + INIT_DELAYED_WORK(&sc->idle.timer_work, __smbdirect_socket_disabled_work); 209 + disable_delayed_work_sync(&sc->idle.timer_work); 210 + 211 + atomic_set(&sc->send_io.credits.count, 0); 212 + init_waitqueue_head(&sc->send_io.credits.wait_queue); 213 + 214 + atomic_set(&sc->send_io.pending.count, 0); 215 + init_waitqueue_head(&sc->send_io.pending.dec_wait_queue); 216 + init_waitqueue_head(&sc->send_io.pending.zero_wait_queue); 217 + 218 + INIT_LIST_HEAD(&sc->recv_io.free.list); 219 + spin_lock_init(&sc->recv_io.free.lock); 220 + 221 + atomic_set(&sc->recv_io.posted.count, 0); 222 + INIT_WORK(&sc->recv_io.posted.refill_work, __smbdirect_socket_disabled_work); 223 + disable_work_sync(&sc->recv_io.posted.refill_work); 224 + 225 + atomic_set(&sc->recv_io.credits.count, 0); 226 + 227 + INIT_LIST_HEAD(&sc->recv_io.reassembly.list); 228 + spin_lock_init(&sc->recv_io.reassembly.lock); 229 + init_waitqueue_head(&sc->recv_io.reassembly.wait_queue); 230 + 231 + atomic_set(&sc->rw_io.credits.count, 0); 232 + init_waitqueue_head(&sc->rw_io.credits.wait_queue); 233 + 234 + spin_lock_init(&sc->mr_io.all.lock); 235 + INIT_LIST_HEAD(&sc->mr_io.all.list); 236 + atomic_set(&sc->mr_io.ready.count, 0); 237 + init_waitqueue_head(&sc->mr_io.ready.wait_queue); 238 + atomic_set(&sc->mr_io.used.count, 0); 239 + INIT_WORK(&sc->mr_io.recovery_work, __smbdirect_socket_disabled_work); 240 + disable_work_sync(&sc->mr_io.recovery_work); 241 + init_waitqueue_head(&sc->mr_io.cleanup.wait_queue); 242 + } 243 243 244 244 struct smbdirect_send_io { 245 245 struct smbdirect_socket *socket; ··· 394 136 u8 packet[]; 395 137 }; 396 138 139 + struct smbdirect_send_batch { 140 + /* 141 + * List of smbdirect_send_io messages 142 + */ 143 + struct list_head msg_list; 144 + /* 145 + * Number of list entries 146 + */ 147 + size_t wr_cnt; 148 + 149 + /* 150 + * Possible remote key invalidation state 151 + */ 152 + bool need_invalidate_rkey; 153 + u32 remote_key; 154 + }; 155 + 397 156 struct smbdirect_recv_io { 398 157 struct smbdirect_socket *socket; 399 158 struct ib_cqe cqe; ··· 431 156 432 157 /* SMBD packet header and payload follows this structure */ 433 158 u8 packet[]; 159 + }; 160 + 161 + enum smbdirect_mr_state { 162 + SMBDIRECT_MR_READY, 163 + SMBDIRECT_MR_REGISTERED, 164 + SMBDIRECT_MR_INVALIDATED, 165 + SMBDIRECT_MR_ERROR 166 + }; 167 + 168 + struct smbdirect_mr_io { 169 + struct smbdirect_socket *socket; 170 + struct ib_cqe cqe; 171 + 172 + struct list_head list; 173 + 174 + enum smbdirect_mr_state state; 175 + struct ib_mr *mr; 176 + struct sg_table sgt; 177 + enum dma_data_direction dir; 178 + union { 179 + struct ib_reg_wr wr; 180 + struct ib_send_wr inv_wr; 181 + }; 182 + 183 + bool need_invalidate; 184 + struct completion invalidate_done; 185 + }; 186 + 187 + struct smbdirect_rw_io { 188 + struct smbdirect_socket *socket; 189 + struct ib_cqe cqe; 190 + 191 + struct list_head list; 192 + 193 + int error; 194 + struct completion *completion; 195 + 196 + struct rdma_rw_ctx rdma_ctx; 197 + struct sg_table sgt; 198 + struct scatterlist sg_list[]; 434 199 }; 435 200 436 201 #endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ */

+2 -2

fs/smb/server/connection.c

··· 243 243 244 244 int ksmbd_conn_rdma_read(struct ksmbd_conn *conn, 245 245 void *buf, unsigned int buflen, 246 - struct smb2_buffer_desc_v1 *desc, 246 + struct smbdirect_buffer_descriptor_v1 *desc, 247 247 unsigned int desc_len) 248 248 { 249 249 int ret = -EINVAL; ··· 257 257 258 258 int ksmbd_conn_rdma_write(struct ksmbd_conn *conn, 259 259 void *buf, unsigned int buflen, 260 - struct smb2_buffer_desc_v1 *desc, 260 + struct smbdirect_buffer_descriptor_v1 *desc, 261 261 unsigned int desc_len) 262 262 { 263 263 int ret = -EINVAL;

+6 -4

fs/smb/server/connection.h

··· 19 19 #include "smb_common.h" 20 20 #include "ksmbd_work.h" 21 21 22 + struct smbdirect_buffer_descriptor_v1; 23 + 22 24 #define KSMBD_SOCKET_BACKLOG 16 23 25 24 26 enum { ··· 135 133 unsigned int remote_key); 136 134 int (*rdma_read)(struct ksmbd_transport *t, 137 135 void *buf, unsigned int len, 138 - struct smb2_buffer_desc_v1 *desc, 136 + struct smbdirect_buffer_descriptor_v1 *desc, 139 137 unsigned int desc_len); 140 138 int (*rdma_write)(struct ksmbd_transport *t, 141 139 void *buf, unsigned int len, 142 - struct smb2_buffer_desc_v1 *desc, 140 + struct smbdirect_buffer_descriptor_v1 *desc, 143 141 unsigned int desc_len); 144 142 void (*free_transport)(struct ksmbd_transport *kt); 145 143 }; ··· 165 163 int ksmbd_conn_write(struct ksmbd_work *work); 166 164 int ksmbd_conn_rdma_read(struct ksmbd_conn *conn, 167 165 void *buf, unsigned int buflen, 168 - struct smb2_buffer_desc_v1 *desc, 166 + struct smbdirect_buffer_descriptor_v1 *desc, 169 167 unsigned int desc_len); 170 168 int ksmbd_conn_rdma_write(struct ksmbd_conn *conn, 171 169 void *buf, unsigned int buflen, 172 - struct smb2_buffer_desc_v1 *desc, 170 + struct smbdirect_buffer_descriptor_v1 *desc, 173 171 unsigned int desc_len); 174 172 void ksmbd_conn_enqueue_request(struct ksmbd_work *work); 175 173 void ksmbd_conn_try_dequeue_request(struct ksmbd_work *work);

+1

fs/smb/server/server.c

··· 365 365 return; 366 366 } 367 367 368 + pr_info("running\n"); 368 369 WRITE_ONCE(server_conf.state, SERVER_STATE_RUNNING); 369 370 } 370 371

+16 -7

fs/smb/server/smb2pdu.c

··· 23 23 #include "asn1.h" 24 24 #include "connection.h" 25 25 #include "transport_ipc.h" 26 + #include "../common/smbdirect/smbdirect.h" 26 27 #include "transport_rdma.h" 27 28 #include "vfs.h" 28 29 #include "vfs_cache.h" ··· 6666 6665 } 6667 6666 6668 6667 static int smb2_set_remote_key_for_rdma(struct ksmbd_work *work, 6669 - struct smb2_buffer_desc_v1 *desc, 6668 + struct smbdirect_buffer_descriptor_v1 *desc, 6670 6669 __le32 Channel, 6671 6670 __le16 ChannelInfoLength) 6672 6671 { ··· 6702 6701 int err; 6703 6702 6704 6703 err = ksmbd_conn_rdma_write(work->conn, data_buf, length, 6705 - (struct smb2_buffer_desc_v1 *) 6704 + (struct smbdirect_buffer_descriptor_v1 *) 6706 6705 ((char *)req + le16_to_cpu(req->ReadChannelInfoOffset)), 6707 6706 le16_to_cpu(req->ReadChannelInfoLength)); 6708 6707 if (err) ··· 6762 6761 if (req->Channel == SMB2_CHANNEL_RDMA_V1_INVALIDATE || 6763 6762 req->Channel == SMB2_CHANNEL_RDMA_V1) { 6764 6763 is_rdma_channel = true; 6765 - max_read_size = get_smbd_max_read_write_size(); 6764 + max_read_size = get_smbd_max_read_write_size(work->conn->transport); 6765 + if (max_read_size == 0) { 6766 + err = -EINVAL; 6767 + goto out; 6768 + } 6766 6769 } 6767 6770 6768 6771 if (is_rdma_channel == true) { ··· 6777 6772 goto out; 6778 6773 } 6779 6774 err = smb2_set_remote_key_for_rdma(work, 6780 - (struct smb2_buffer_desc_v1 *) 6775 + (struct smbdirect_buffer_descriptor_v1 *) 6781 6776 ((char *)req + ch_offset), 6782 6777 req->Channel, 6783 6778 req->ReadChannelInfoLength); ··· 6972 6967 return -ENOMEM; 6973 6968 6974 6969 ret = ksmbd_conn_rdma_read(work->conn, data_buf, length, 6975 - (struct smb2_buffer_desc_v1 *) 6970 + (struct smbdirect_buffer_descriptor_v1 *) 6976 6971 ((char *)req + le16_to_cpu(req->WriteChannelInfoOffset)), 6977 6972 le16_to_cpu(req->WriteChannelInfoLength)); 6978 6973 if (ret < 0) { ··· 7024 7019 if (req->Channel == SMB2_CHANNEL_RDMA_V1 || 7025 7020 req->Channel == SMB2_CHANNEL_RDMA_V1_INVALIDATE) { 7026 7021 is_rdma_channel = true; 7027 - max_write_size = get_smbd_max_read_write_size(); 7022 + max_write_size = get_smbd_max_read_write_size(work->conn->transport); 7023 + if (max_write_size == 0) { 7024 + err = -EINVAL; 7025 + goto out; 7026 + } 7028 7027 length = le32_to_cpu(req->RemainingBytes); 7029 7028 } 7030 7029 ··· 7041 7032 goto out; 7042 7033 } 7043 7034 err = smb2_set_remote_key_for_rdma(work, 7044 - (struct smb2_buffer_desc_v1 *) 7035 + (struct smbdirect_buffer_descriptor_v1 *) 7045 7036 ((char *)req + ch_offset), 7046 7037 req->Channel, 7047 7038 req->WriteChannelInfoLength);

-6

fs/smb/server/smb2pdu.h

··· 136 136 u8 SidBuffer[44]; 137 137 } __packed; 138 138 139 - struct smb2_buffer_desc_v1 { 140 - __le64 offset; 141 - __le32 token; 142 - __le32 length; 143 - } __packed; 144 - 145 139 #define SMB2_0_IOCTL_IS_FSCTL 0x00000001 146 140 147 141 struct smb_sockaddr_in {

+906 -703

fs/smb/server/transport_rdma.c

··· 23 23 #include "connection.h" 24 24 #include "smb_common.h" 25 25 #include "../common/smb2status.h" 26 + #include "../common/smbdirect/smbdirect.h" 27 + #include "../common/smbdirect/smbdirect_pdu.h" 28 + #include "../common/smbdirect/smbdirect_socket.h" 26 29 #include "transport_rdma.h" 27 30 28 31 #define SMB_DIRECT_PORT_IWARP 5445 29 32 #define SMB_DIRECT_PORT_INFINIBAND 445 30 33 31 - #define SMB_DIRECT_VERSION_LE cpu_to_le16(0x0100) 34 + #define SMB_DIRECT_VERSION_LE cpu_to_le16(SMBDIRECT_V1) 32 35 33 - /* SMB_DIRECT negotiation timeout in seconds */ 34 - #define SMB_DIRECT_NEGOTIATE_TIMEOUT 120 36 + /* SMB_DIRECT negotiation timeout (for the server) in seconds */ 37 + #define SMB_DIRECT_NEGOTIATE_TIMEOUT 5 35 38 36 - #define SMB_DIRECT_MAX_SEND_SGES 6 37 - #define SMB_DIRECT_MAX_RECV_SGES 1 39 + /* The timeout to wait for a keepalive message from peer in seconds */ 40 + #define SMB_DIRECT_KEEPALIVE_SEND_INTERVAL 120 41 + 42 + /* The timeout to wait for a keepalive message from peer in seconds */ 43 + #define SMB_DIRECT_KEEPALIVE_RECV_TIMEOUT 5 38 44 39 45 /* 40 46 * Default maximum number of RDMA read/write outstanding on this connection ··· 93 87 94 88 static struct workqueue_struct *smb_direct_wq; 95 89 96 - enum smb_direct_status { 97 - SMB_DIRECT_CS_NEW = 0, 98 - SMB_DIRECT_CS_CONNECTED, 99 - SMB_DIRECT_CS_DISCONNECTING, 100 - SMB_DIRECT_CS_DISCONNECTED, 101 - }; 102 - 103 90 struct smb_direct_transport { 104 91 struct ksmbd_transport transport; 105 92 106 - enum smb_direct_status status; 107 - bool full_packet_received; 108 - wait_queue_head_t wait_status; 109 - 110 - struct rdma_cm_id *cm_id; 111 - struct ib_cq *send_cq; 112 - struct ib_cq *recv_cq; 113 - struct ib_pd *pd; 114 - struct ib_qp *qp; 115 - 116 - int max_send_size; 117 - int max_recv_size; 118 - int max_fragmented_send_size; 119 - int max_fragmented_recv_size; 120 - int max_rdma_rw_size; 121 - 122 - spinlock_t reassembly_queue_lock; 123 - struct list_head reassembly_queue; 124 - int reassembly_data_length; 125 - int reassembly_queue_length; 126 - int first_entry_offset; 127 - wait_queue_head_t wait_reassembly_queue; 128 - 129 - spinlock_t receive_credit_lock; 130 - int recv_credits; 131 - int count_avail_recvmsg; 132 - int recv_credit_max; 133 - int recv_credit_target; 134 - 135 - spinlock_t recvmsg_queue_lock; 136 - struct list_head recvmsg_queue; 137 - 138 - int send_credit_target; 139 - atomic_t send_credits; 140 - spinlock_t lock_new_recv_credits; 141 - int new_recv_credits; 142 - int max_rw_credits; 143 - int pages_per_rw_credit; 144 - atomic_t rw_credits; 145 - 146 - wait_queue_head_t wait_send_credits; 147 - wait_queue_head_t wait_rw_credits; 148 - 149 - mempool_t *sendmsg_mempool; 150 - struct kmem_cache *sendmsg_cache; 151 - mempool_t *recvmsg_mempool; 152 - struct kmem_cache *recvmsg_cache; 153 - 154 - wait_queue_head_t wait_send_pending; 155 - atomic_t send_pending; 156 - 157 - struct work_struct post_recv_credits_work; 158 - struct work_struct send_immediate_work; 159 - struct work_struct disconnect_work; 160 - 161 - bool negotiation_requested; 93 + struct smbdirect_socket socket; 162 94 }; 163 95 164 - #define KSMBD_TRANS(t) ((struct ksmbd_transport *)&((t)->transport)) 165 - #define SMBD_TRANS(t) ((struct smb_direct_transport *)container_of(t, \ 96 + #define KSMBD_TRANS(t) (&(t)->transport) 97 + #define SMBD_TRANS(t) (container_of(t, \ 166 98 struct smb_direct_transport, transport)) 167 - enum { 168 - SMB_DIRECT_MSG_NEGOTIATE_REQ = 0, 169 - SMB_DIRECT_MSG_DATA_TRANSFER 170 - }; 171 99 172 100 static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops; 173 - 174 - struct smb_direct_send_ctx { 175 - struct list_head msg_list; 176 - int wr_cnt; 177 - bool need_invalidate_rkey; 178 - unsigned int remote_key; 179 - }; 180 - 181 - struct smb_direct_sendmsg { 182 - struct smb_direct_transport *transport; 183 - struct ib_send_wr wr; 184 - struct list_head list; 185 - int num_sge; 186 - struct ib_sge sge[SMB_DIRECT_MAX_SEND_SGES]; 187 - struct ib_cqe cqe; 188 - u8 packet[]; 189 - }; 190 - 191 - struct smb_direct_recvmsg { 192 - struct smb_direct_transport *transport; 193 - struct list_head list; 194 - int type; 195 - struct ib_sge sge; 196 - struct ib_cqe cqe; 197 - bool first_segment; 198 - u8 packet[]; 199 - }; 200 - 201 - struct smb_direct_rdma_rw_msg { 202 - struct smb_direct_transport *t; 203 - struct ib_cqe cqe; 204 - int status; 205 - struct completion *completion; 206 - struct list_head list; 207 - struct rdma_rw_ctx rw_ctx; 208 - struct sg_table sgt; 209 - struct scatterlist sg_list[]; 210 - }; 211 101 212 102 void init_smbd_max_io_size(unsigned int sz) 213 103 { ··· 111 209 smb_direct_max_read_write_size = sz; 112 210 } 113 211 114 - unsigned int get_smbd_max_read_write_size(void) 212 + unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt) 115 213 { 116 - return smb_direct_max_read_write_size; 214 + struct smb_direct_transport *t; 215 + struct smbdirect_socket *sc; 216 + struct smbdirect_socket_parameters *sp; 217 + 218 + if (kt->ops != &ksmbd_smb_direct_transport_ops) 219 + return 0; 220 + 221 + t = SMBD_TRANS(kt); 222 + sc = &t->socket; 223 + sp = &sc->parameters; 224 + 225 + return sp->max_read_write_size; 117 226 } 118 227 119 228 static inline int get_buf_page_count(void *buf, int size) ··· 133 220 (uintptr_t)buf / PAGE_SIZE; 134 221 } 135 222 136 - static void smb_direct_destroy_pools(struct smb_direct_transport *transport); 223 + static void smb_direct_destroy_pools(struct smbdirect_socket *sc); 137 224 static void smb_direct_post_recv_credits(struct work_struct *work); 138 - static int smb_direct_post_send_data(struct smb_direct_transport *t, 139 - struct smb_direct_send_ctx *send_ctx, 225 + static int smb_direct_post_send_data(struct smbdirect_socket *sc, 226 + struct smbdirect_send_batch *send_ctx, 140 227 struct kvec *iov, int niov, 141 228 int remaining_data_length); 142 229 143 - static inline struct smb_direct_transport * 144 - smb_trans_direct_transfort(struct ksmbd_transport *t) 145 - { 146 - return container_of(t, struct smb_direct_transport, transport); 147 - } 148 - 149 230 static inline void 150 - *smb_direct_recvmsg_payload(struct smb_direct_recvmsg *recvmsg) 231 + *smbdirect_recv_io_payload(struct smbdirect_recv_io *recvmsg) 151 232 { 152 233 return (void *)recvmsg->packet; 153 234 } 154 235 155 - static inline bool is_receive_credit_post_required(int receive_credits, 156 - int avail_recvmsg_count) 157 - { 158 - return receive_credits <= (smb_direct_receive_credit_max >> 3) && 159 - avail_recvmsg_count >= (receive_credits >> 2); 160 - } 161 - 162 236 static struct 163 - smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t) 237 + smbdirect_recv_io *get_free_recvmsg(struct smbdirect_socket *sc) 164 238 { 165 - struct smb_direct_recvmsg *recvmsg = NULL; 239 + struct smbdirect_recv_io *recvmsg = NULL; 240 + unsigned long flags; 166 241 167 - spin_lock(&t->recvmsg_queue_lock); 168 - if (!list_empty(&t->recvmsg_queue)) { 169 - recvmsg = list_first_entry(&t->recvmsg_queue, 170 - struct smb_direct_recvmsg, 242 + spin_lock_irqsave(&sc->recv_io.free.lock, flags); 243 + if (!list_empty(&sc->recv_io.free.list)) { 244 + recvmsg = list_first_entry(&sc->recv_io.free.list, 245 + struct smbdirect_recv_io, 171 246 list); 172 247 list_del(&recvmsg->list); 173 248 } 174 - spin_unlock(&t->recvmsg_queue_lock); 249 + spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); 175 250 return recvmsg; 176 251 } 177 252 178 - static void put_recvmsg(struct smb_direct_transport *t, 179 - struct smb_direct_recvmsg *recvmsg) 253 + static void put_recvmsg(struct smbdirect_socket *sc, 254 + struct smbdirect_recv_io *recvmsg) 180 255 { 256 + unsigned long flags; 257 + 181 258 if (likely(recvmsg->sge.length != 0)) { 182 - ib_dma_unmap_single(t->cm_id->device, 259 + ib_dma_unmap_single(sc->ib.dev, 183 260 recvmsg->sge.addr, 184 261 recvmsg->sge.length, 185 262 DMA_FROM_DEVICE); 186 263 recvmsg->sge.length = 0; 187 264 } 188 265 189 - spin_lock(&t->recvmsg_queue_lock); 190 - list_add(&recvmsg->list, &t->recvmsg_queue); 191 - spin_unlock(&t->recvmsg_queue_lock); 266 + spin_lock_irqsave(&sc->recv_io.free.lock, flags); 267 + list_add(&recvmsg->list, &sc->recv_io.free.list); 268 + spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); 269 + 270 + queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); 192 271 } 193 272 194 - static void enqueue_reassembly(struct smb_direct_transport *t, 195 - struct smb_direct_recvmsg *recvmsg, 273 + static void enqueue_reassembly(struct smbdirect_socket *sc, 274 + struct smbdirect_recv_io *recvmsg, 196 275 int data_length) 197 276 { 198 - spin_lock(&t->reassembly_queue_lock); 199 - list_add_tail(&recvmsg->list, &t->reassembly_queue); 200 - t->reassembly_queue_length++; 277 + unsigned long flags; 278 + 279 + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 280 + list_add_tail(&recvmsg->list, &sc->recv_io.reassembly.list); 281 + sc->recv_io.reassembly.queue_length++; 201 282 /* 202 283 * Make sure reassembly_data_length is updated after list and 203 284 * reassembly_queue_length are updated. On the dequeue side ··· 199 292 * if reassembly_queue_length and list is up to date 200 293 */ 201 294 virt_wmb(); 202 - t->reassembly_data_length += data_length; 203 - spin_unlock(&t->reassembly_queue_lock); 295 + sc->recv_io.reassembly.data_length += data_length; 296 + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 204 297 } 205 298 206 - static struct smb_direct_recvmsg *get_first_reassembly(struct smb_direct_transport *t) 299 + static struct smbdirect_recv_io *get_first_reassembly(struct smbdirect_socket *sc) 207 300 { 208 - if (!list_empty(&t->reassembly_queue)) 209 - return list_first_entry(&t->reassembly_queue, 210 - struct smb_direct_recvmsg, list); 301 + if (!list_empty(&sc->recv_io.reassembly.list)) 302 + return list_first_entry(&sc->recv_io.reassembly.list, 303 + struct smbdirect_recv_io, list); 211 304 else 212 305 return NULL; 213 306 } 214 307 308 + static void smb_direct_disconnect_wake_up_all(struct smbdirect_socket *sc) 309 + { 310 + /* 311 + * Wake up all waiters in all wait queues 312 + * in order to notice the broken connection. 313 + */ 314 + wake_up_all(&sc->status_wait); 315 + wake_up_all(&sc->send_io.credits.wait_queue); 316 + wake_up_all(&sc->send_io.pending.zero_wait_queue); 317 + wake_up_all(&sc->recv_io.reassembly.wait_queue); 318 + wake_up_all(&sc->rw_io.credits.wait_queue); 319 + } 320 + 215 321 static void smb_direct_disconnect_rdma_work(struct work_struct *work) 216 322 { 217 - struct smb_direct_transport *t = 218 - container_of(work, struct smb_direct_transport, 219 - disconnect_work); 323 + struct smbdirect_socket *sc = 324 + container_of(work, struct smbdirect_socket, disconnect_work); 220 325 221 - if (t->status == SMB_DIRECT_CS_CONNECTED) { 222 - t->status = SMB_DIRECT_CS_DISCONNECTING; 223 - rdma_disconnect(t->cm_id); 326 + /* 327 + * make sure this and other work is not queued again 328 + * but here we don't block and avoid 329 + * disable[_delayed]_work_sync() 330 + */ 331 + disable_work(&sc->disconnect_work); 332 + disable_work(&sc->recv_io.posted.refill_work); 333 + disable_delayed_work(&sc->idle.timer_work); 334 + disable_work(&sc->idle.immediate_work); 335 + 336 + if (sc->first_error == 0) 337 + sc->first_error = -ECONNABORTED; 338 + 339 + switch (sc->status) { 340 + case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: 341 + case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: 342 + case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: 343 + case SMBDIRECT_SOCKET_CONNECTED: 344 + case SMBDIRECT_SOCKET_ERROR: 345 + sc->status = SMBDIRECT_SOCKET_DISCONNECTING; 346 + rdma_disconnect(sc->rdma.cm_id); 347 + break; 348 + 349 + case SMBDIRECT_SOCKET_CREATED: 350 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: 351 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: 352 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: 353 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: 354 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: 355 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: 356 + case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: 357 + case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: 358 + case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: 359 + /* 360 + * rdma_accept() never reached 361 + * RDMA_CM_EVENT_ESTABLISHED 362 + */ 363 + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; 364 + break; 365 + 366 + case SMBDIRECT_SOCKET_DISCONNECTING: 367 + case SMBDIRECT_SOCKET_DISCONNECTED: 368 + case SMBDIRECT_SOCKET_DESTROYED: 369 + break; 224 370 } 371 + 372 + /* 373 + * Wake up all waiters in all wait queues 374 + * in order to notice the broken connection. 375 + */ 376 + smb_direct_disconnect_wake_up_all(sc); 225 377 } 226 378 227 379 static void 228 - smb_direct_disconnect_rdma_connection(struct smb_direct_transport *t) 380 + smb_direct_disconnect_rdma_connection(struct smbdirect_socket *sc) 229 381 { 230 - if (t->status == SMB_DIRECT_CS_CONNECTED) 231 - queue_work(smb_direct_wq, &t->disconnect_work); 382 + /* 383 + * make sure other work (than disconnect_work) is 384 + * not queued again but here we don't block and avoid 385 + * disable[_delayed]_work_sync() 386 + */ 387 + disable_work(&sc->recv_io.posted.refill_work); 388 + disable_work(&sc->idle.immediate_work); 389 + disable_delayed_work(&sc->idle.timer_work); 390 + 391 + if (sc->first_error == 0) 392 + sc->first_error = -ECONNABORTED; 393 + 394 + switch (sc->status) { 395 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: 396 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: 397 + case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: 398 + case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: 399 + case SMBDIRECT_SOCKET_ERROR: 400 + case SMBDIRECT_SOCKET_DISCONNECTING: 401 + case SMBDIRECT_SOCKET_DISCONNECTED: 402 + case SMBDIRECT_SOCKET_DESTROYED: 403 + /* 404 + * Keep the current error status 405 + */ 406 + break; 407 + 408 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: 409 + case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: 410 + sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; 411 + break; 412 + 413 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: 414 + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: 415 + sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; 416 + break; 417 + 418 + case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: 419 + case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: 420 + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; 421 + break; 422 + 423 + case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: 424 + case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: 425 + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; 426 + break; 427 + 428 + case SMBDIRECT_SOCKET_CREATED: 429 + case SMBDIRECT_SOCKET_CONNECTED: 430 + sc->status = SMBDIRECT_SOCKET_ERROR; 431 + break; 432 + } 433 + 434 + /* 435 + * Wake up all waiters in all wait queues 436 + * in order to notice the broken connection. 437 + */ 438 + smb_direct_disconnect_wake_up_all(sc); 439 + 440 + queue_work(sc->workqueue, &sc->disconnect_work); 232 441 } 233 442 234 443 static void smb_direct_send_immediate_work(struct work_struct *work) 235 444 { 236 - struct smb_direct_transport *t = container_of(work, 237 - struct smb_direct_transport, send_immediate_work); 445 + struct smbdirect_socket *sc = 446 + container_of(work, struct smbdirect_socket, idle.immediate_work); 238 447 239 - if (t->status != SMB_DIRECT_CS_CONNECTED) 448 + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 240 449 return; 241 450 242 - smb_direct_post_send_data(t, NULL, NULL, 0, 0); 451 + smb_direct_post_send_data(sc, NULL, NULL, 0, 0); 452 + } 453 + 454 + static void smb_direct_idle_connection_timer(struct work_struct *work) 455 + { 456 + struct smbdirect_socket *sc = 457 + container_of(work, struct smbdirect_socket, idle.timer_work.work); 458 + struct smbdirect_socket_parameters *sp = &sc->parameters; 459 + 460 + if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) { 461 + smb_direct_disconnect_rdma_connection(sc); 462 + return; 463 + } 464 + 465 + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 466 + return; 467 + 468 + /* 469 + * Now use the keepalive timeout (instead of keepalive interval) 470 + * in order to wait for a response 471 + */ 472 + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; 473 + mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 474 + msecs_to_jiffies(sp->keepalive_timeout_msec)); 475 + queue_work(sc->workqueue, &sc->idle.immediate_work); 243 476 } 244 477 245 478 static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id) 246 479 { 247 480 struct smb_direct_transport *t; 481 + struct smbdirect_socket *sc; 482 + struct smbdirect_socket_parameters *sp; 248 483 struct ksmbd_conn *conn; 249 484 250 485 t = kzalloc(sizeof(*t), KSMBD_DEFAULT_GFP); 251 486 if (!t) 252 487 return NULL; 488 + sc = &t->socket; 489 + smbdirect_socket_init(sc); 490 + sp = &sc->parameters; 253 491 254 - t->cm_id = cm_id; 255 - cm_id->context = t; 492 + sc->workqueue = smb_direct_wq; 256 493 257 - t->status = SMB_DIRECT_CS_NEW; 258 - init_waitqueue_head(&t->wait_status); 494 + INIT_WORK(&sc->disconnect_work, smb_direct_disconnect_rdma_work); 259 495 260 - spin_lock_init(&t->reassembly_queue_lock); 261 - INIT_LIST_HEAD(&t->reassembly_queue); 262 - t->reassembly_data_length = 0; 263 - t->reassembly_queue_length = 0; 264 - init_waitqueue_head(&t->wait_reassembly_queue); 265 - init_waitqueue_head(&t->wait_send_credits); 266 - init_waitqueue_head(&t->wait_rw_credits); 496 + sp->negotiate_timeout_msec = SMB_DIRECT_NEGOTIATE_TIMEOUT * 1000; 497 + sp->initiator_depth = SMB_DIRECT_CM_INITIATOR_DEPTH; 498 + sp->responder_resources = 1; 499 + sp->recv_credit_max = smb_direct_receive_credit_max; 500 + sp->send_credit_target = smb_direct_send_credit_target; 501 + sp->max_send_size = smb_direct_max_send_size; 502 + sp->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size; 503 + sp->max_recv_size = smb_direct_max_receive_size; 504 + sp->max_read_write_size = smb_direct_max_read_write_size; 505 + sp->keepalive_interval_msec = SMB_DIRECT_KEEPALIVE_SEND_INTERVAL * 1000; 506 + sp->keepalive_timeout_msec = SMB_DIRECT_KEEPALIVE_RECV_TIMEOUT * 1000; 267 507 268 - spin_lock_init(&t->receive_credit_lock); 269 - spin_lock_init(&t->recvmsg_queue_lock); 270 - INIT_LIST_HEAD(&t->recvmsg_queue); 508 + sc->rdma.cm_id = cm_id; 509 + cm_id->context = sc; 271 510 272 - init_waitqueue_head(&t->wait_send_pending); 273 - atomic_set(&t->send_pending, 0); 511 + sc->ib.dev = sc->rdma.cm_id->device; 274 512 275 - spin_lock_init(&t->lock_new_recv_credits); 276 - 277 - INIT_WORK(&t->post_recv_credits_work, 513 + INIT_WORK(&sc->recv_io.posted.refill_work, 278 514 smb_direct_post_recv_credits); 279 - INIT_WORK(&t->send_immediate_work, smb_direct_send_immediate_work); 280 - INIT_WORK(&t->disconnect_work, smb_direct_disconnect_rdma_work); 515 + INIT_WORK(&sc->idle.immediate_work, smb_direct_send_immediate_work); 516 + INIT_DELAYED_WORK(&sc->idle.timer_work, smb_direct_idle_connection_timer); 281 517 282 518 conn = ksmbd_conn_alloc(); 283 519 if (!conn) ··· 441 391 442 392 static void free_transport(struct smb_direct_transport *t) 443 393 { 444 - struct smb_direct_recvmsg *recvmsg; 394 + struct smbdirect_socket *sc = &t->socket; 395 + struct smbdirect_recv_io *recvmsg; 445 396 446 - wake_up_interruptible(&t->wait_send_credits); 397 + disable_work_sync(&sc->disconnect_work); 398 + if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING) { 399 + smb_direct_disconnect_rdma_work(&sc->disconnect_work); 400 + wait_event_interruptible(sc->status_wait, 401 + sc->status == SMBDIRECT_SOCKET_DISCONNECTED); 402 + } 447 403 448 - ksmbd_debug(RDMA, "wait for all send posted to IB to finish\n"); 449 - wait_event(t->wait_send_pending, 450 - atomic_read(&t->send_pending) == 0); 404 + /* 405 + * Wake up all waiters in all wait queues 406 + * in order to notice the broken connection. 407 + * 408 + * Most likely this was already called via 409 + * smb_direct_disconnect_rdma_work(), but call it again... 410 + */ 411 + smb_direct_disconnect_wake_up_all(sc); 451 412 452 - disable_work_sync(&t->disconnect_work); 453 - disable_work_sync(&t->post_recv_credits_work); 454 - disable_work_sync(&t->send_immediate_work); 413 + disable_work_sync(&sc->recv_io.posted.refill_work); 414 + disable_delayed_work_sync(&sc->idle.timer_work); 415 + disable_work_sync(&sc->idle.immediate_work); 455 416 456 - if (t->qp) { 457 - ib_drain_qp(t->qp); 458 - ib_mr_pool_destroy(t->qp, &t->qp->rdma_mrs); 459 - t->qp = NULL; 460 - rdma_destroy_qp(t->cm_id); 417 + if (sc->ib.qp) { 418 + ib_drain_qp(sc->ib.qp); 419 + ib_mr_pool_destroy(sc->ib.qp, &sc->ib.qp->rdma_mrs); 420 + sc->ib.qp = NULL; 421 + rdma_destroy_qp(sc->rdma.cm_id); 461 422 } 462 423 463 424 ksmbd_debug(RDMA, "drain the reassembly queue\n"); 464 425 do { 465 - spin_lock(&t->reassembly_queue_lock); 466 - recvmsg = get_first_reassembly(t); 426 + unsigned long flags; 427 + 428 + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 429 + recvmsg = get_first_reassembly(sc); 467 430 if (recvmsg) { 468 431 list_del(&recvmsg->list); 469 - spin_unlock(&t->reassembly_queue_lock); 470 - put_recvmsg(t, recvmsg); 432 + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 433 + put_recvmsg(sc, recvmsg); 471 434 } else { 472 - spin_unlock(&t->reassembly_queue_lock); 435 + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 473 436 } 474 437 } while (recvmsg); 475 - t->reassembly_data_length = 0; 438 + sc->recv_io.reassembly.data_length = 0; 476 439 477 - if (t->send_cq) 478 - ib_free_cq(t->send_cq); 479 - if (t->recv_cq) 480 - ib_free_cq(t->recv_cq); 481 - if (t->pd) 482 - ib_dealloc_pd(t->pd); 483 - if (t->cm_id) 484 - rdma_destroy_id(t->cm_id); 440 + if (sc->ib.send_cq) 441 + ib_free_cq(sc->ib.send_cq); 442 + if (sc->ib.recv_cq) 443 + ib_free_cq(sc->ib.recv_cq); 444 + if (sc->ib.pd) 445 + ib_dealloc_pd(sc->ib.pd); 446 + if (sc->rdma.cm_id) 447 + rdma_destroy_id(sc->rdma.cm_id); 485 448 486 - smb_direct_destroy_pools(t); 449 + smb_direct_destroy_pools(sc); 487 450 ksmbd_conn_free(KSMBD_TRANS(t)->conn); 488 451 } 489 452 490 - static struct smb_direct_sendmsg 491 - *smb_direct_alloc_sendmsg(struct smb_direct_transport *t) 453 + static struct smbdirect_send_io 454 + *smb_direct_alloc_sendmsg(struct smbdirect_socket *sc) 492 455 { 493 - struct smb_direct_sendmsg *msg; 456 + struct smbdirect_send_io *msg; 494 457 495 - msg = mempool_alloc(t->sendmsg_mempool, KSMBD_DEFAULT_GFP); 458 + msg = mempool_alloc(sc->send_io.mem.pool, KSMBD_DEFAULT_GFP); 496 459 if (!msg) 497 460 return ERR_PTR(-ENOMEM); 498 - msg->transport = t; 499 - INIT_LIST_HEAD(&msg->list); 461 + msg->socket = sc; 462 + INIT_LIST_HEAD(&msg->sibling_list); 500 463 msg->num_sge = 0; 501 464 return msg; 502 465 } 503 466 504 - static void smb_direct_free_sendmsg(struct smb_direct_transport *t, 505 - struct smb_direct_sendmsg *msg) 467 + static void smb_direct_free_sendmsg(struct smbdirect_socket *sc, 468 + struct smbdirect_send_io *msg) 506 469 { 507 470 int i; 508 471 509 472 if (msg->num_sge > 0) { 510 - ib_dma_unmap_single(t->cm_id->device, 473 + ib_dma_unmap_single(sc->ib.dev, 511 474 msg->sge[0].addr, msg->sge[0].length, 512 475 DMA_TO_DEVICE); 513 476 for (i = 1; i < msg->num_sge; i++) 514 - ib_dma_unmap_page(t->cm_id->device, 477 + ib_dma_unmap_page(sc->ib.dev, 515 478 msg->sge[i].addr, msg->sge[i].length, 516 479 DMA_TO_DEVICE); 517 480 } 518 - mempool_free(msg, t->sendmsg_mempool); 481 + mempool_free(msg, sc->send_io.mem.pool); 519 482 } 520 483 521 - static int smb_direct_check_recvmsg(struct smb_direct_recvmsg *recvmsg) 484 + static int smb_direct_check_recvmsg(struct smbdirect_recv_io *recvmsg) 522 485 { 523 - switch (recvmsg->type) { 524 - case SMB_DIRECT_MSG_DATA_TRANSFER: { 525 - struct smb_direct_data_transfer *req = 526 - (struct smb_direct_data_transfer *)recvmsg->packet; 486 + struct smbdirect_socket *sc = recvmsg->socket; 487 + 488 + switch (sc->recv_io.expected) { 489 + case SMBDIRECT_EXPECT_DATA_TRANSFER: { 490 + struct smbdirect_data_transfer *req = 491 + (struct smbdirect_data_transfer *)recvmsg->packet; 527 492 struct smb2_hdr *hdr = (struct smb2_hdr *)(recvmsg->packet 528 493 + le32_to_cpu(req->data_offset)); 529 494 ksmbd_debug(RDMA, ··· 547 482 le16_to_cpu(req->credits_requested), 548 483 req->data_length, req->remaining_data_length, 549 484 hdr->ProtocolId, hdr->Command); 550 - break; 485 + return 0; 551 486 } 552 - case SMB_DIRECT_MSG_NEGOTIATE_REQ: { 553 - struct smb_direct_negotiate_req *req = 554 - (struct smb_direct_negotiate_req *)recvmsg->packet; 487 + case SMBDIRECT_EXPECT_NEGOTIATE_REQ: { 488 + struct smbdirect_negotiate_req *req = 489 + (struct smbdirect_negotiate_req *)recvmsg->packet; 555 490 ksmbd_debug(RDMA, 556 491 "MinVersion: %u, MaxVersion: %u, CreditRequested: %u, MaxSendSize: %u, MaxRecvSize: %u, MaxFragmentedSize: %u\n", 557 492 le16_to_cpu(req->min_version), ··· 569 504 128 * 1024) 570 505 return -ECONNABORTED; 571 506 507 + return 0; 508 + } 509 + case SMBDIRECT_EXPECT_NEGOTIATE_REP: 510 + /* client only */ 572 511 break; 573 512 } 574 - default: 575 - return -EINVAL; 576 - } 577 - return 0; 513 + 514 + /* This is an internal error */ 515 + return -EINVAL; 578 516 } 579 517 580 518 static void recv_done(struct ib_cq *cq, struct ib_wc *wc) 581 519 { 582 - struct smb_direct_recvmsg *recvmsg; 583 - struct smb_direct_transport *t; 520 + struct smbdirect_recv_io *recvmsg; 521 + struct smbdirect_socket *sc; 522 + struct smbdirect_socket_parameters *sp; 584 523 585 - recvmsg = container_of(wc->wr_cqe, struct smb_direct_recvmsg, cqe); 586 - t = recvmsg->transport; 524 + recvmsg = container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); 525 + sc = recvmsg->socket; 526 + sp = &sc->parameters; 587 527 588 528 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { 589 - put_recvmsg(t, recvmsg); 529 + put_recvmsg(sc, recvmsg); 590 530 if (wc->status != IB_WC_WR_FLUSH_ERR) { 591 531 pr_err("Recv error. status='%s (%d)' opcode=%d\n", 592 532 ib_wc_status_msg(wc->status), wc->status, 593 533 wc->opcode); 594 - smb_direct_disconnect_rdma_connection(t); 534 + smb_direct_disconnect_rdma_connection(sc); 595 535 } 596 536 return; 597 537 } ··· 608 538 ib_dma_sync_single_for_cpu(wc->qp->device, recvmsg->sge.addr, 609 539 recvmsg->sge.length, DMA_FROM_DEVICE); 610 540 611 - switch (recvmsg->type) { 612 - case SMB_DIRECT_MSG_NEGOTIATE_REQ: 613 - if (wc->byte_len < sizeof(struct smb_direct_negotiate_req)) { 614 - put_recvmsg(t, recvmsg); 615 - smb_direct_disconnect_rdma_connection(t); 541 + /* 542 + * Reset timer to the keepalive interval in 543 + * order to trigger our next keepalive message. 544 + */ 545 + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; 546 + mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 547 + msecs_to_jiffies(sp->keepalive_interval_msec)); 548 + 549 + switch (sc->recv_io.expected) { 550 + case SMBDIRECT_EXPECT_NEGOTIATE_REQ: 551 + if (wc->byte_len < sizeof(struct smbdirect_negotiate_req)) { 552 + put_recvmsg(sc, recvmsg); 553 + smb_direct_disconnect_rdma_connection(sc); 616 554 return; 617 555 } 618 - t->negotiation_requested = true; 619 - t->full_packet_received = true; 620 - t->status = SMB_DIRECT_CS_CONNECTED; 621 - enqueue_reassembly(t, recvmsg, 0); 622 - wake_up_interruptible(&t->wait_status); 556 + sc->recv_io.reassembly.full_packet_received = true; 557 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED); 558 + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; 559 + enqueue_reassembly(sc, recvmsg, 0); 560 + wake_up(&sc->status_wait); 623 561 return; 624 - case SMB_DIRECT_MSG_DATA_TRANSFER: { 625 - struct smb_direct_data_transfer *data_transfer = 626 - (struct smb_direct_data_transfer *)recvmsg->packet; 562 + case SMBDIRECT_EXPECT_DATA_TRANSFER: { 563 + struct smbdirect_data_transfer *data_transfer = 564 + (struct smbdirect_data_transfer *)recvmsg->packet; 627 565 u32 remaining_data_length, data_offset, data_length; 628 - int avail_recvmsg_count, receive_credits; 566 + u16 old_recv_credit_target; 629 567 630 568 if (wc->byte_len < 631 - offsetof(struct smb_direct_data_transfer, padding)) { 632 - put_recvmsg(t, recvmsg); 633 - smb_direct_disconnect_rdma_connection(t); 569 + offsetof(struct smbdirect_data_transfer, padding)) { 570 + put_recvmsg(sc, recvmsg); 571 + smb_direct_disconnect_rdma_connection(sc); 634 572 return; 635 573 } 636 574 ··· 647 569 data_offset = le32_to_cpu(data_transfer->data_offset); 648 570 if (wc->byte_len < data_offset || 649 571 wc->byte_len < (u64)data_offset + data_length) { 650 - put_recvmsg(t, recvmsg); 651 - smb_direct_disconnect_rdma_connection(t); 572 + put_recvmsg(sc, recvmsg); 573 + smb_direct_disconnect_rdma_connection(sc); 652 574 return; 653 575 } 654 - if (remaining_data_length > t->max_fragmented_recv_size || 655 - data_length > t->max_fragmented_recv_size || 576 + if (remaining_data_length > sp->max_fragmented_recv_size || 577 + data_length > sp->max_fragmented_recv_size || 656 578 (u64)remaining_data_length + (u64)data_length > 657 - (u64)t->max_fragmented_recv_size) { 658 - put_recvmsg(t, recvmsg); 659 - smb_direct_disconnect_rdma_connection(t); 579 + (u64)sp->max_fragmented_recv_size) { 580 + put_recvmsg(sc, recvmsg); 581 + smb_direct_disconnect_rdma_connection(sc); 660 582 return; 661 583 } 662 584 663 585 if (data_length) { 664 - if (t->full_packet_received) 586 + if (sc->recv_io.reassembly.full_packet_received) 665 587 recvmsg->first_segment = true; 666 588 667 589 if (le32_to_cpu(data_transfer->remaining_data_length)) 668 - t->full_packet_received = false; 590 + sc->recv_io.reassembly.full_packet_received = false; 669 591 else 670 - t->full_packet_received = true; 671 - 672 - spin_lock(&t->receive_credit_lock); 673 - receive_credits = --(t->recv_credits); 674 - avail_recvmsg_count = t->count_avail_recvmsg; 675 - spin_unlock(&t->receive_credit_lock); 676 - } else { 677 - spin_lock(&t->receive_credit_lock); 678 - receive_credits = --(t->recv_credits); 679 - avail_recvmsg_count = ++(t->count_avail_recvmsg); 680 - spin_unlock(&t->receive_credit_lock); 592 + sc->recv_io.reassembly.full_packet_received = true; 681 593 } 682 594 683 - t->recv_credit_target = 595 + atomic_dec(&sc->recv_io.posted.count); 596 + atomic_dec(&sc->recv_io.credits.count); 597 + 598 + old_recv_credit_target = sc->recv_io.credits.target; 599 + sc->recv_io.credits.target = 684 600 le16_to_cpu(data_transfer->credits_requested); 601 + sc->recv_io.credits.target = 602 + min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max); 603 + sc->recv_io.credits.target = 604 + max_t(u16, sc->recv_io.credits.target, 1); 685 605 atomic_add(le16_to_cpu(data_transfer->credits_granted), 686 - &t->send_credits); 606 + &sc->send_io.credits.count); 687 607 688 608 if (le16_to_cpu(data_transfer->flags) & 689 - SMB_DIRECT_RESPONSE_REQUESTED) 690 - queue_work(smb_direct_wq, &t->send_immediate_work); 609 + SMBDIRECT_FLAG_RESPONSE_REQUESTED) 610 + queue_work(sc->workqueue, &sc->idle.immediate_work); 691 611 692 - if (atomic_read(&t->send_credits) > 0) 693 - wake_up_interruptible(&t->wait_send_credits); 694 - 695 - if (is_receive_credit_post_required(receive_credits, avail_recvmsg_count)) 696 - queue_work(smb_direct_wq, &t->post_recv_credits_work); 612 + if (atomic_read(&sc->send_io.credits.count) > 0) 613 + wake_up(&sc->send_io.credits.wait_queue); 697 614 698 615 if (data_length) { 699 - enqueue_reassembly(t, recvmsg, (int)data_length); 700 - wake_up_interruptible(&t->wait_reassembly_queue); 616 + if (sc->recv_io.credits.target > old_recv_credit_target) 617 + queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); 618 + 619 + enqueue_reassembly(sc, recvmsg, (int)data_length); 620 + wake_up(&sc->recv_io.reassembly.wait_queue); 701 621 } else 702 - put_recvmsg(t, recvmsg); 622 + put_recvmsg(sc, recvmsg); 703 623 704 624 return; 705 625 } 626 + case SMBDIRECT_EXPECT_NEGOTIATE_REP: 627 + /* client only */ 628 + break; 706 629 } 707 630 708 631 /* 709 632 * This is an internal error! 710 633 */ 711 - WARN_ON_ONCE(recvmsg->type != SMB_DIRECT_MSG_DATA_TRANSFER); 712 - put_recvmsg(t, recvmsg); 713 - smb_direct_disconnect_rdma_connection(t); 634 + WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_DATA_TRANSFER); 635 + put_recvmsg(sc, recvmsg); 636 + smb_direct_disconnect_rdma_connection(sc); 714 637 } 715 638 716 - static int smb_direct_post_recv(struct smb_direct_transport *t, 717 - struct smb_direct_recvmsg *recvmsg) 639 + static int smb_direct_post_recv(struct smbdirect_socket *sc, 640 + struct smbdirect_recv_io *recvmsg) 718 641 { 642 + struct smbdirect_socket_parameters *sp = &sc->parameters; 719 643 struct ib_recv_wr wr; 720 644 int ret; 721 645 722 - recvmsg->sge.addr = ib_dma_map_single(t->cm_id->device, 723 - recvmsg->packet, t->max_recv_size, 646 + recvmsg->sge.addr = ib_dma_map_single(sc->ib.dev, 647 + recvmsg->packet, 648 + sp->max_recv_size, 724 649 DMA_FROM_DEVICE); 725 - ret = ib_dma_mapping_error(t->cm_id->device, recvmsg->sge.addr); 650 + ret = ib_dma_mapping_error(sc->ib.dev, recvmsg->sge.addr); 726 651 if (ret) 727 652 return ret; 728 - recvmsg->sge.length = t->max_recv_size; 729 - recvmsg->sge.lkey = t->pd->local_dma_lkey; 653 + recvmsg->sge.length = sp->max_recv_size; 654 + recvmsg->sge.lkey = sc->ib.pd->local_dma_lkey; 730 655 recvmsg->cqe.done = recv_done; 731 656 732 657 wr.wr_cqe = &recvmsg->cqe; ··· 737 656 wr.sg_list = &recvmsg->sge; 738 657 wr.num_sge = 1; 739 658 740 - ret = ib_post_recv(t->qp, &wr, NULL); 659 + ret = ib_post_recv(sc->ib.qp, &wr, NULL); 741 660 if (ret) { 742 661 pr_err("Can't post recv: %d\n", ret); 743 - ib_dma_unmap_single(t->cm_id->device, 662 + ib_dma_unmap_single(sc->ib.dev, 744 663 recvmsg->sge.addr, recvmsg->sge.length, 745 664 DMA_FROM_DEVICE); 746 665 recvmsg->sge.length = 0; 747 - smb_direct_disconnect_rdma_connection(t); 666 + smb_direct_disconnect_rdma_connection(sc); 748 667 return ret; 749 668 } 750 669 return ret; ··· 753 672 static int smb_direct_read(struct ksmbd_transport *t, char *buf, 754 673 unsigned int size, int unused) 755 674 { 756 - struct smb_direct_recvmsg *recvmsg; 757 - struct smb_direct_data_transfer *data_transfer; 675 + struct smbdirect_recv_io *recvmsg; 676 + struct smbdirect_data_transfer *data_transfer; 758 677 int to_copy, to_read, data_read, offset; 759 678 u32 data_length, remaining_data_length, data_offset; 760 679 int rc; 761 - struct smb_direct_transport *st = smb_trans_direct_transfort(t); 680 + struct smb_direct_transport *st = SMBD_TRANS(t); 681 + struct smbdirect_socket *sc = &st->socket; 762 682 763 683 again: 764 - if (st->status != SMB_DIRECT_CS_CONNECTED) { 684 + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { 765 685 pr_err("disconnected\n"); 766 686 return -ENOTCONN; 767 687 } ··· 772 690 * the only one reading from the front of the queue. The transport 773 691 * may add more entries to the back of the queue at the same time 774 692 */ 775 - if (st->reassembly_data_length >= size) { 693 + if (sc->recv_io.reassembly.data_length >= size) { 776 694 int queue_length; 777 695 int queue_removed = 0; 696 + unsigned long flags; 778 697 779 698 /* 780 699 * Need to make sure reassembly_data_length is read before ··· 785 702 * updated in SOFTIRQ as more data is received 786 703 */ 787 704 virt_rmb(); 788 - queue_length = st->reassembly_queue_length; 705 + queue_length = sc->recv_io.reassembly.queue_length; 789 706 data_read = 0; 790 707 to_read = size; 791 - offset = st->first_entry_offset; 708 + offset = sc->recv_io.reassembly.first_entry_offset; 792 709 while (data_read < size) { 793 - recvmsg = get_first_reassembly(st); 794 - data_transfer = smb_direct_recvmsg_payload(recvmsg); 710 + recvmsg = get_first_reassembly(sc); 711 + data_transfer = smbdirect_recv_io_payload(recvmsg); 795 712 data_length = le32_to_cpu(data_transfer->data_length); 796 713 remaining_data_length = 797 714 le32_to_cpu(data_transfer->remaining_data_length); ··· 831 748 if (queue_length) { 832 749 list_del(&recvmsg->list); 833 750 } else { 834 - spin_lock_irq(&st->reassembly_queue_lock); 751 + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 835 752 list_del(&recvmsg->list); 836 - spin_unlock_irq(&st->reassembly_queue_lock); 753 + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 837 754 } 838 755 queue_removed++; 839 - put_recvmsg(st, recvmsg); 756 + put_recvmsg(sc, recvmsg); 840 757 offset = 0; 841 758 } else { 842 759 offset += to_copy; ··· 846 763 data_read += to_copy; 847 764 } 848 765 849 - spin_lock_irq(&st->reassembly_queue_lock); 850 - st->reassembly_data_length -= data_read; 851 - st->reassembly_queue_length -= queue_removed; 852 - spin_unlock_irq(&st->reassembly_queue_lock); 766 + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 767 + sc->recv_io.reassembly.data_length -= data_read; 768 + sc->recv_io.reassembly.queue_length -= queue_removed; 769 + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 853 770 854 - spin_lock(&st->receive_credit_lock); 855 - st->count_avail_recvmsg += queue_removed; 856 - if (is_receive_credit_post_required(st->recv_credits, st->count_avail_recvmsg)) { 857 - spin_unlock(&st->receive_credit_lock); 858 - queue_work(smb_direct_wq, &st->post_recv_credits_work); 859 - } else { 860 - spin_unlock(&st->receive_credit_lock); 861 - } 862 - 863 - st->first_entry_offset = offset; 771 + sc->recv_io.reassembly.first_entry_offset = offset; 864 772 ksmbd_debug(RDMA, 865 773 "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n", 866 - data_read, st->reassembly_data_length, 867 - st->first_entry_offset); 774 + data_read, sc->recv_io.reassembly.data_length, 775 + sc->recv_io.reassembly.first_entry_offset); 868 776 read_rfc1002_done: 869 777 return data_read; 870 778 } 871 779 872 780 ksmbd_debug(RDMA, "wait_event on more data\n"); 873 - rc = wait_event_interruptible(st->wait_reassembly_queue, 874 - st->reassembly_data_length >= size || 875 - st->status != SMB_DIRECT_CS_CONNECTED); 781 + rc = wait_event_interruptible(sc->recv_io.reassembly.wait_queue, 782 + sc->recv_io.reassembly.data_length >= size || 783 + sc->status != SMBDIRECT_SOCKET_CONNECTED); 876 784 if (rc) 877 785 return -EINTR; 878 786 ··· 872 798 873 799 static void smb_direct_post_recv_credits(struct work_struct *work) 874 800 { 875 - struct smb_direct_transport *t = container_of(work, 876 - struct smb_direct_transport, post_recv_credits_work); 877 - struct smb_direct_recvmsg *recvmsg; 878 - int receive_credits, credits = 0; 801 + struct smbdirect_socket *sc = 802 + container_of(work, struct smbdirect_socket, recv_io.posted.refill_work); 803 + struct smbdirect_recv_io *recvmsg; 804 + int credits = 0; 879 805 int ret; 880 806 881 - spin_lock(&t->receive_credit_lock); 882 - receive_credits = t->recv_credits; 883 - spin_unlock(&t->receive_credit_lock); 884 - 885 - if (receive_credits < t->recv_credit_target) { 807 + if (atomic_read(&sc->recv_io.credits.count) < sc->recv_io.credits.target) { 886 808 while (true) { 887 - recvmsg = get_free_recvmsg(t); 809 + recvmsg = get_free_recvmsg(sc); 888 810 if (!recvmsg) 889 811 break; 890 812 891 - recvmsg->type = SMB_DIRECT_MSG_DATA_TRANSFER; 892 813 recvmsg->first_segment = false; 893 814 894 - ret = smb_direct_post_recv(t, recvmsg); 815 + ret = smb_direct_post_recv(sc, recvmsg); 895 816 if (ret) { 896 817 pr_err("Can't post recv: %d\n", ret); 897 - put_recvmsg(t, recvmsg); 818 + put_recvmsg(sc, recvmsg); 898 819 break; 899 820 } 900 821 credits++; 822 + 823 + atomic_inc(&sc->recv_io.posted.count); 901 824 } 902 825 } 903 826 904 - spin_lock(&t->receive_credit_lock); 905 - t->recv_credits += credits; 906 - t->count_avail_recvmsg -= credits; 907 - spin_unlock(&t->receive_credit_lock); 908 - 909 - spin_lock(&t->lock_new_recv_credits); 910 - t->new_recv_credits += credits; 911 - spin_unlock(&t->lock_new_recv_credits); 912 - 913 827 if (credits) 914 - queue_work(smb_direct_wq, &t->send_immediate_work); 828 + queue_work(sc->workqueue, &sc->idle.immediate_work); 915 829 } 916 830 917 831 static void send_done(struct ib_cq *cq, struct ib_wc *wc) 918 832 { 919 - struct smb_direct_sendmsg *sendmsg, *sibling; 920 - struct smb_direct_transport *t; 833 + struct smbdirect_send_io *sendmsg, *sibling; 834 + struct smbdirect_socket *sc; 921 835 struct list_head *pos, *prev, *end; 922 836 923 - sendmsg = container_of(wc->wr_cqe, struct smb_direct_sendmsg, cqe); 924 - t = sendmsg->transport; 837 + sendmsg = container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); 838 + sc = sendmsg->socket; 925 839 926 840 ksmbd_debug(RDMA, "Send completed. status='%s (%d)', opcode=%d\n", 927 841 ib_wc_status_msg(wc->status), wc->status, ··· 919 857 pr_err("Send error. status='%s (%d)', opcode=%d\n", 920 858 ib_wc_status_msg(wc->status), wc->status, 921 859 wc->opcode); 922 - smb_direct_disconnect_rdma_connection(t); 860 + smb_direct_disconnect_rdma_connection(sc); 923 861 } 924 862 925 - if (atomic_dec_and_test(&t->send_pending)) 926 - wake_up(&t->wait_send_pending); 863 + if (atomic_dec_and_test(&sc->send_io.pending.count)) 864 + wake_up(&sc->send_io.pending.zero_wait_queue); 927 865 928 866 /* iterate and free the list of messages in reverse. the list's head 929 867 * is invalid. 930 868 */ 931 - for (pos = &sendmsg->list, prev = pos->prev, end = sendmsg->list.next; 869 + for (pos = &sendmsg->sibling_list, prev = pos->prev, end = sendmsg->sibling_list.next; 932 870 prev != end; pos = prev, prev = prev->prev) { 933 - sibling = container_of(pos, struct smb_direct_sendmsg, list); 934 - smb_direct_free_sendmsg(t, sibling); 871 + sibling = container_of(pos, struct smbdirect_send_io, sibling_list); 872 + smb_direct_free_sendmsg(sc, sibling); 935 873 } 936 874 937 - sibling = container_of(pos, struct smb_direct_sendmsg, list); 938 - smb_direct_free_sendmsg(t, sibling); 875 + sibling = container_of(pos, struct smbdirect_send_io, sibling_list); 876 + smb_direct_free_sendmsg(sc, sibling); 939 877 } 940 878 941 - static int manage_credits_prior_sending(struct smb_direct_transport *t) 879 + static int manage_credits_prior_sending(struct smbdirect_socket *sc) 942 880 { 943 881 int new_credits; 944 882 945 - spin_lock(&t->lock_new_recv_credits); 946 - new_credits = t->new_recv_credits; 947 - t->new_recv_credits = 0; 948 - spin_unlock(&t->lock_new_recv_credits); 883 + if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) 884 + return 0; 949 885 886 + new_credits = atomic_read(&sc->recv_io.posted.count); 887 + if (new_credits == 0) 888 + return 0; 889 + 890 + new_credits -= atomic_read(&sc->recv_io.credits.count); 891 + if (new_credits <= 0) 892 + return 0; 893 + 894 + atomic_add(new_credits, &sc->recv_io.credits.count); 950 895 return new_credits; 951 896 } 952 897 953 - static int smb_direct_post_send(struct smb_direct_transport *t, 898 + static int manage_keep_alive_before_sending(struct smbdirect_socket *sc) 899 + { 900 + struct smbdirect_socket_parameters *sp = &sc->parameters; 901 + 902 + if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) { 903 + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT; 904 + /* 905 + * Now use the keepalive timeout (instead of keepalive interval) 906 + * in order to wait for a response 907 + */ 908 + mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 909 + msecs_to_jiffies(sp->keepalive_timeout_msec)); 910 + return 1; 911 + } 912 + return 0; 913 + } 914 + 915 + static int smb_direct_post_send(struct smbdirect_socket *sc, 954 916 struct ib_send_wr *wr) 955 917 { 956 918 int ret; 957 919 958 - atomic_inc(&t->send_pending); 959 - ret = ib_post_send(t->qp, wr, NULL); 920 + atomic_inc(&sc->send_io.pending.count); 921 + ret = ib_post_send(sc->ib.qp, wr, NULL); 960 922 if (ret) { 961 923 pr_err("failed to post send: %d\n", ret); 962 - if (atomic_dec_and_test(&t->send_pending)) 963 - wake_up(&t->wait_send_pending); 964 - smb_direct_disconnect_rdma_connection(t); 924 + if (atomic_dec_and_test(&sc->send_io.pending.count)) 925 + wake_up(&sc->send_io.pending.zero_wait_queue); 926 + smb_direct_disconnect_rdma_connection(sc); 965 927 } 966 928 return ret; 967 929 } 968 930 969 - static void smb_direct_send_ctx_init(struct smb_direct_transport *t, 970 - struct smb_direct_send_ctx *send_ctx, 931 + static void smb_direct_send_ctx_init(struct smbdirect_send_batch *send_ctx, 971 932 bool need_invalidate_rkey, 972 933 unsigned int remote_key) 973 934 { ··· 1000 915 send_ctx->remote_key = remote_key; 1001 916 } 1002 917 1003 - static int smb_direct_flush_send_list(struct smb_direct_transport *t, 1004 - struct smb_direct_send_ctx *send_ctx, 918 + static int smb_direct_flush_send_list(struct smbdirect_socket *sc, 919 + struct smbdirect_send_batch *send_ctx, 1005 920 bool is_last) 1006 921 { 1007 - struct smb_direct_sendmsg *first, *last; 922 + struct smbdirect_send_io *first, *last; 1008 923 int ret; 1009 924 1010 925 if (list_empty(&send_ctx->msg_list)) 1011 926 return 0; 1012 927 1013 928 first = list_first_entry(&send_ctx->msg_list, 1014 - struct smb_direct_sendmsg, 1015 - list); 929 + struct smbdirect_send_io, 930 + sibling_list); 1016 931 last = list_last_entry(&send_ctx->msg_list, 1017 - struct smb_direct_sendmsg, 1018 - list); 932 + struct smbdirect_send_io, 933 + sibling_list); 934 + 935 + if (send_ctx->need_invalidate_rkey) { 936 + first->wr.opcode = IB_WR_SEND_WITH_INV; 937 + first->wr.ex.invalidate_rkey = send_ctx->remote_key; 938 + send_ctx->need_invalidate_rkey = false; 939 + send_ctx->remote_key = 0; 940 + } 1019 941 1020 942 last->wr.send_flags = IB_SEND_SIGNALED; 1021 943 last->wr.wr_cqe = &last->cqe; 1022 - if (is_last && send_ctx->need_invalidate_rkey) { 1023 - last->wr.opcode = IB_WR_SEND_WITH_INV; 1024 - last->wr.ex.invalidate_rkey = send_ctx->remote_key; 1025 - } 1026 944 1027 - ret = smb_direct_post_send(t, &first->wr); 945 + ret = smb_direct_post_send(sc, &first->wr); 1028 946 if (!ret) { 1029 - smb_direct_send_ctx_init(t, send_ctx, 947 + smb_direct_send_ctx_init(send_ctx, 1030 948 send_ctx->need_invalidate_rkey, 1031 949 send_ctx->remote_key); 1032 950 } else { 1033 - atomic_add(send_ctx->wr_cnt, &t->send_credits); 1034 - wake_up(&t->wait_send_credits); 951 + atomic_add(send_ctx->wr_cnt, &sc->send_io.credits.count); 952 + wake_up(&sc->send_io.credits.wait_queue); 1035 953 list_for_each_entry_safe(first, last, &send_ctx->msg_list, 1036 - list) { 1037 - smb_direct_free_sendmsg(t, first); 954 + sibling_list) { 955 + smb_direct_free_sendmsg(sc, first); 1038 956 } 1039 957 } 1040 958 return ret; 1041 959 } 1042 960 1043 - static int wait_for_credits(struct smb_direct_transport *t, 961 + static int wait_for_credits(struct smbdirect_socket *sc, 1044 962 wait_queue_head_t *waitq, atomic_t *total_credits, 1045 963 int needed) 1046 964 { ··· 1056 968 atomic_add(needed, total_credits); 1057 969 ret = wait_event_interruptible(*waitq, 1058 970 atomic_read(total_credits) >= needed || 1059 - t->status != SMB_DIRECT_CS_CONNECTED); 971 + sc->status != SMBDIRECT_SOCKET_CONNECTED); 1060 972 1061 - if (t->status != SMB_DIRECT_CS_CONNECTED) 973 + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 1062 974 return -ENOTCONN; 1063 975 else if (ret < 0) 1064 976 return ret; 1065 977 } while (true); 1066 978 } 1067 979 1068 - static int wait_for_send_credits(struct smb_direct_transport *t, 1069 - struct smb_direct_send_ctx *send_ctx) 980 + static int wait_for_send_credits(struct smbdirect_socket *sc, 981 + struct smbdirect_send_batch *send_ctx) 1070 982 { 1071 983 int ret; 1072 984 1073 985 if (send_ctx && 1074 - (send_ctx->wr_cnt >= 16 || atomic_read(&t->send_credits) <= 1)) { 1075 - ret = smb_direct_flush_send_list(t, send_ctx, false); 986 + (send_ctx->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) { 987 + ret = smb_direct_flush_send_list(sc, send_ctx, false); 1076 988 if (ret) 1077 989 return ret; 1078 990 } 1079 991 1080 - return wait_for_credits(t, &t->wait_send_credits, &t->send_credits, 1); 992 + return wait_for_credits(sc, &sc->send_io.credits.wait_queue, &sc->send_io.credits.count, 1); 1081 993 } 1082 994 1083 - static int wait_for_rw_credits(struct smb_direct_transport *t, int credits) 995 + static int wait_for_rw_credits(struct smbdirect_socket *sc, int credits) 1084 996 { 1085 - return wait_for_credits(t, &t->wait_rw_credits, &t->rw_credits, credits); 997 + return wait_for_credits(sc, 998 + &sc->rw_io.credits.wait_queue, 999 + &sc->rw_io.credits.count, 1000 + credits); 1086 1001 } 1087 1002 1088 - static int calc_rw_credits(struct smb_direct_transport *t, 1003 + static int calc_rw_credits(struct smbdirect_socket *sc, 1089 1004 char *buf, unsigned int len) 1090 1005 { 1091 1006 return DIV_ROUND_UP(get_buf_page_count(buf, len), 1092 - t->pages_per_rw_credit); 1007 + sc->rw_io.credits.num_pages); 1093 1008 } 1094 1009 1095 - static int smb_direct_create_header(struct smb_direct_transport *t, 1010 + static int smb_direct_create_header(struct smbdirect_socket *sc, 1096 1011 int size, int remaining_data_length, 1097 - struct smb_direct_sendmsg **sendmsg_out) 1012 + struct smbdirect_send_io **sendmsg_out) 1098 1013 { 1099 - struct smb_direct_sendmsg *sendmsg; 1100 - struct smb_direct_data_transfer *packet; 1014 + struct smbdirect_socket_parameters *sp = &sc->parameters; 1015 + struct smbdirect_send_io *sendmsg; 1016 + struct smbdirect_data_transfer *packet; 1101 1017 int header_length; 1102 1018 int ret; 1103 1019 1104 - sendmsg = smb_direct_alloc_sendmsg(t); 1020 + sendmsg = smb_direct_alloc_sendmsg(sc); 1105 1021 if (IS_ERR(sendmsg)) 1106 1022 return PTR_ERR(sendmsg); 1107 1023 1108 1024 /* Fill in the packet header */ 1109 - packet = (struct smb_direct_data_transfer *)sendmsg->packet; 1110 - packet->credits_requested = cpu_to_le16(t->send_credit_target); 1111 - packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(t)); 1025 + packet = (struct smbdirect_data_transfer *)sendmsg->packet; 1026 + packet->credits_requested = cpu_to_le16(sp->send_credit_target); 1027 + packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc)); 1112 1028 1113 1029 packet->flags = 0; 1030 + if (manage_keep_alive_before_sending(sc)) 1031 + packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED); 1032 + 1114 1033 packet->reserved = 0; 1115 1034 if (!size) 1116 1035 packet->data_offset = 0; ··· 1136 1041 le32_to_cpu(packet->remaining_data_length)); 1137 1042 1138 1043 /* Map the packet to DMA */ 1139 - header_length = sizeof(struct smb_direct_data_transfer); 1044 + header_length = sizeof(struct smbdirect_data_transfer); 1140 1045 /* If this is a packet without payload, don't send padding */ 1141 1046 if (!size) 1142 1047 header_length = 1143 - offsetof(struct smb_direct_data_transfer, padding); 1048 + offsetof(struct smbdirect_data_transfer, padding); 1144 1049 1145 - sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device, 1050 + sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev, 1146 1051 (void *)packet, 1147 1052 header_length, 1148 1053 DMA_TO_DEVICE); 1149 - ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr); 1054 + ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr); 1150 1055 if (ret) { 1151 - smb_direct_free_sendmsg(t, sendmsg); 1056 + smb_direct_free_sendmsg(sc, sendmsg); 1152 1057 return ret; 1153 1058 } 1154 1059 1155 1060 sendmsg->num_sge = 1; 1156 1061 sendmsg->sge[0].length = header_length; 1157 - sendmsg->sge[0].lkey = t->pd->local_dma_lkey; 1062 + sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey; 1158 1063 1159 1064 *sendmsg_out = sendmsg; 1160 1065 return 0; ··· 1204 1109 return ib_dma_map_sg(device, sg_list, npages, dir); 1205 1110 } 1206 1111 1207 - static int post_sendmsg(struct smb_direct_transport *t, 1208 - struct smb_direct_send_ctx *send_ctx, 1209 - struct smb_direct_sendmsg *msg) 1112 + static int post_sendmsg(struct smbdirect_socket *sc, 1113 + struct smbdirect_send_batch *send_ctx, 1114 + struct smbdirect_send_io *msg) 1210 1115 { 1211 1116 int i; 1212 1117 1213 1118 for (i = 0; i < msg->num_sge; i++) 1214 - ib_dma_sync_single_for_device(t->cm_id->device, 1119 + ib_dma_sync_single_for_device(sc->ib.dev, 1215 1120 msg->sge[i].addr, msg->sge[i].length, 1216 1121 DMA_TO_DEVICE); 1217 1122 ··· 1225 1130 msg->wr.wr_cqe = NULL; 1226 1131 msg->wr.send_flags = 0; 1227 1132 if (!list_empty(&send_ctx->msg_list)) { 1228 - struct smb_direct_sendmsg *last; 1133 + struct smbdirect_send_io *last; 1229 1134 1230 1135 last = list_last_entry(&send_ctx->msg_list, 1231 - struct smb_direct_sendmsg, 1232 - list); 1136 + struct smbdirect_send_io, 1137 + sibling_list); 1233 1138 last->wr.next = &msg->wr; 1234 1139 } 1235 - list_add_tail(&msg->list, &send_ctx->msg_list); 1140 + list_add_tail(&msg->sibling_list, &send_ctx->msg_list); 1236 1141 send_ctx->wr_cnt++; 1237 1142 return 0; 1238 1143 } 1239 1144 1240 1145 msg->wr.wr_cqe = &msg->cqe; 1241 1146 msg->wr.send_flags = IB_SEND_SIGNALED; 1242 - return smb_direct_post_send(t, &msg->wr); 1147 + return smb_direct_post_send(sc, &msg->wr); 1243 1148 } 1244 1149 1245 - static int smb_direct_post_send_data(struct smb_direct_transport *t, 1246 - struct smb_direct_send_ctx *send_ctx, 1150 + static int smb_direct_post_send_data(struct smbdirect_socket *sc, 1151 + struct smbdirect_send_batch *send_ctx, 1247 1152 struct kvec *iov, int niov, 1248 1153 int remaining_data_length) 1249 1154 { 1250 1155 int i, j, ret; 1251 - struct smb_direct_sendmsg *msg; 1156 + struct smbdirect_send_io *msg; 1252 1157 int data_length; 1253 - struct scatterlist sg[SMB_DIRECT_MAX_SEND_SGES - 1]; 1158 + struct scatterlist sg[SMBDIRECT_SEND_IO_MAX_SGE - 1]; 1254 1159 1255 - ret = wait_for_send_credits(t, send_ctx); 1160 + ret = wait_for_send_credits(sc, send_ctx); 1256 1161 if (ret) 1257 1162 return ret; 1258 1163 ··· 1260 1165 for (i = 0; i < niov; i++) 1261 1166 data_length += iov[i].iov_len; 1262 1167 1263 - ret = smb_direct_create_header(t, data_length, remaining_data_length, 1168 + ret = smb_direct_create_header(sc, data_length, remaining_data_length, 1264 1169 &msg); 1265 1170 if (ret) { 1266 - atomic_inc(&t->send_credits); 1171 + atomic_inc(&sc->send_io.credits.count); 1267 1172 return ret; 1268 1173 } 1269 1174 ··· 1271 1176 struct ib_sge *sge; 1272 1177 int sg_cnt; 1273 1178 1274 - sg_init_table(sg, SMB_DIRECT_MAX_SEND_SGES - 1); 1275 - sg_cnt = get_mapped_sg_list(t->cm_id->device, 1179 + sg_init_table(sg, SMBDIRECT_SEND_IO_MAX_SGE - 1); 1180 + sg_cnt = get_mapped_sg_list(sc->ib.dev, 1276 1181 iov[i].iov_base, iov[i].iov_len, 1277 - sg, SMB_DIRECT_MAX_SEND_SGES - 1, 1182 + sg, SMBDIRECT_SEND_IO_MAX_SGE - 1, 1278 1183 DMA_TO_DEVICE); 1279 1184 if (sg_cnt <= 0) { 1280 1185 pr_err("failed to map buffer\n"); 1281 1186 ret = -ENOMEM; 1282 1187 goto err; 1283 - } else if (sg_cnt + msg->num_sge > SMB_DIRECT_MAX_SEND_SGES) { 1188 + } else if (sg_cnt + msg->num_sge > SMBDIRECT_SEND_IO_MAX_SGE) { 1284 1189 pr_err("buffer not fitted into sges\n"); 1285 1190 ret = -E2BIG; 1286 - ib_dma_unmap_sg(t->cm_id->device, sg, sg_cnt, 1191 + ib_dma_unmap_sg(sc->ib.dev, sg, sg_cnt, 1287 1192 DMA_TO_DEVICE); 1288 1193 goto err; 1289 1194 } ··· 1292 1197 sge = &msg->sge[msg->num_sge]; 1293 1198 sge->addr = sg_dma_address(&sg[j]); 1294 1199 sge->length = sg_dma_len(&sg[j]); 1295 - sge->lkey = t->pd->local_dma_lkey; 1200 + sge->lkey = sc->ib.pd->local_dma_lkey; 1296 1201 msg->num_sge++; 1297 1202 } 1298 1203 } 1299 1204 1300 - ret = post_sendmsg(t, send_ctx, msg); 1205 + ret = post_sendmsg(sc, send_ctx, msg); 1301 1206 if (ret) 1302 1207 goto err; 1303 1208 return 0; 1304 1209 err: 1305 - smb_direct_free_sendmsg(t, msg); 1306 - atomic_inc(&t->send_credits); 1210 + smb_direct_free_sendmsg(sc, msg); 1211 + atomic_inc(&sc->send_io.credits.count); 1307 1212 return ret; 1308 1213 } 1309 1214 ··· 1311 1216 struct kvec *iov, int niovs, int buflen, 1312 1217 bool need_invalidate, unsigned int remote_key) 1313 1218 { 1314 - struct smb_direct_transport *st = smb_trans_direct_transfort(t); 1219 + struct smb_direct_transport *st = SMBD_TRANS(t); 1220 + struct smbdirect_socket *sc = &st->socket; 1221 + struct smbdirect_socket_parameters *sp = &sc->parameters; 1315 1222 size_t remaining_data_length; 1316 1223 size_t iov_idx; 1317 1224 size_t iov_ofs; 1318 - size_t max_iov_size = st->max_send_size - 1319 - sizeof(struct smb_direct_data_transfer); 1225 + size_t max_iov_size = sp->max_send_size - 1226 + sizeof(struct smbdirect_data_transfer); 1320 1227 int ret; 1321 - struct smb_direct_send_ctx send_ctx; 1228 + struct smbdirect_send_batch send_ctx; 1322 1229 int error = 0; 1323 1230 1324 - if (st->status != SMB_DIRECT_CS_CONNECTED) 1231 + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 1325 1232 return -ENOTCONN; 1326 1233 1327 1234 //FIXME: skip RFC1002 header.. ··· 1336 1239 remaining_data_length = buflen; 1337 1240 ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen); 1338 1241 1339 - smb_direct_send_ctx_init(st, &send_ctx, need_invalidate, remote_key); 1242 + smb_direct_send_ctx_init(&send_ctx, need_invalidate, remote_key); 1340 1243 while (remaining_data_length) { 1341 - struct kvec vecs[SMB_DIRECT_MAX_SEND_SGES - 1]; /* minus smbdirect hdr */ 1244 + struct kvec vecs[SMBDIRECT_SEND_IO_MAX_SGE - 1]; /* minus smbdirect hdr */ 1342 1245 size_t possible_bytes = max_iov_size; 1343 1246 size_t possible_vecs; 1344 1247 size_t bytes = 0; ··· 1425 1328 1426 1329 remaining_data_length -= bytes; 1427 1330 1428 - ret = smb_direct_post_send_data(st, &send_ctx, 1331 + ret = smb_direct_post_send_data(sc, &send_ctx, 1429 1332 vecs, nvecs, 1430 1333 remaining_data_length); 1431 1334 if (unlikely(ret)) { ··· 1435 1338 } 1436 1339 1437 1340 done: 1438 - ret = smb_direct_flush_send_list(st, &send_ctx, true); 1341 + ret = smb_direct_flush_send_list(sc, &send_ctx, true); 1439 1342 if (unlikely(!ret && error)) 1440 1343 ret = error; 1441 1344 ··· 1446 1349 * that means all the I/Os have been out and we are good to return 1447 1350 */ 1448 1351 1449 - wait_event(st->wait_send_pending, 1450 - atomic_read(&st->send_pending) == 0); 1352 + wait_event(sc->send_io.pending.zero_wait_queue, 1353 + atomic_read(&sc->send_io.pending.count) == 0 || 1354 + sc->status != SMBDIRECT_SOCKET_CONNECTED); 1355 + if (sc->status != SMBDIRECT_SOCKET_CONNECTED && ret == 0) 1356 + ret = -ENOTCONN; 1357 + 1451 1358 return ret; 1452 1359 } 1453 1360 1454 1361 static void smb_direct_free_rdma_rw_msg(struct smb_direct_transport *t, 1455 - struct smb_direct_rdma_rw_msg *msg, 1362 + struct smbdirect_rw_io *msg, 1456 1363 enum dma_data_direction dir) 1457 1364 { 1458 - rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port, 1365 + struct smbdirect_socket *sc = &t->socket; 1366 + 1367 + rdma_rw_ctx_destroy(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port, 1459 1368 msg->sgt.sgl, msg->sgt.nents, dir); 1460 1369 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); 1461 1370 kfree(msg); ··· 1470 1367 static void read_write_done(struct ib_cq *cq, struct ib_wc *wc, 1471 1368 enum dma_data_direction dir) 1472 1369 { 1473 - struct smb_direct_rdma_rw_msg *msg = container_of(wc->wr_cqe, 1474 - struct smb_direct_rdma_rw_msg, cqe); 1475 - struct smb_direct_transport *t = msg->t; 1370 + struct smbdirect_rw_io *msg = 1371 + container_of(wc->wr_cqe, struct smbdirect_rw_io, cqe); 1372 + struct smbdirect_socket *sc = msg->socket; 1476 1373 1477 1374 if (wc->status != IB_WC_SUCCESS) { 1478 - msg->status = -EIO; 1375 + msg->error = -EIO; 1479 1376 pr_err("read/write error. opcode = %d, status = %s(%d)\n", 1480 1377 wc->opcode, ib_wc_status_msg(wc->status), wc->status); 1481 1378 if (wc->status != IB_WC_WR_FLUSH_ERR) 1482 - smb_direct_disconnect_rdma_connection(t); 1379 + smb_direct_disconnect_rdma_connection(sc); 1483 1380 } 1484 1381 1485 1382 complete(msg->completion); ··· 1497 1394 1498 1395 static int smb_direct_rdma_xmit(struct smb_direct_transport *t, 1499 1396 void *buf, int buf_len, 1500 - struct smb2_buffer_desc_v1 *desc, 1397 + struct smbdirect_buffer_descriptor_v1 *desc, 1501 1398 unsigned int desc_len, 1502 1399 bool is_read) 1503 1400 { 1504 - struct smb_direct_rdma_rw_msg *msg, *next_msg; 1401 + struct smbdirect_socket *sc = &t->socket; 1402 + struct smbdirect_socket_parameters *sp = &sc->parameters; 1403 + struct smbdirect_rw_io *msg, *next_msg; 1505 1404 int i, ret; 1506 1405 DECLARE_COMPLETION_ONSTACK(completion); 1507 1406 struct ib_send_wr *first_wr; ··· 1512 1407 int credits_needed; 1513 1408 unsigned int desc_buf_len, desc_num = 0; 1514 1409 1515 - if (t->status != SMB_DIRECT_CS_CONNECTED) 1410 + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 1516 1411 return -ENOTCONN; 1517 1412 1518 - if (buf_len > t->max_rdma_rw_size) 1413 + if (buf_len > sp->max_read_write_size) 1519 1414 return -EINVAL; 1520 1415 1521 1416 /* calculate needed credits */ ··· 1535 1430 buf_len = 0; 1536 1431 } 1537 1432 1538 - credits_needed += calc_rw_credits(t, desc_buf, desc_buf_len); 1433 + credits_needed += calc_rw_credits(sc, desc_buf, desc_buf_len); 1539 1434 desc_buf += desc_buf_len; 1540 1435 buf_len -= desc_buf_len; 1541 1436 desc_num++; ··· 1544 1439 ksmbd_debug(RDMA, "RDMA %s, len %#x, needed credits %#x\n", 1545 1440 str_read_write(is_read), buf_len, credits_needed); 1546 1441 1547 - ret = wait_for_rw_credits(t, credits_needed); 1442 + ret = wait_for_rw_credits(sc, credits_needed); 1548 1443 if (ret < 0) 1549 1444 return ret; 1550 1445 ··· 1560 1455 1561 1456 desc_buf_len = le32_to_cpu(desc[i].length); 1562 1457 1563 - msg->t = t; 1458 + msg->socket = sc; 1564 1459 msg->cqe.done = is_read ? read_done : write_done; 1565 1460 msg->completion = &completion; 1566 1461 ··· 1582 1477 goto out; 1583 1478 } 1584 1479 1585 - ret = rdma_rw_ctx_init(&msg->rw_ctx, t->qp, t->qp->port, 1480 + ret = rdma_rw_ctx_init(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port, 1586 1481 msg->sgt.sgl, 1587 1482 get_buf_page_count(desc_buf, desc_buf_len), 1588 1483 0, ··· 1603 1498 /* concatenate work requests of rdma_rw_ctxs */ 1604 1499 first_wr = NULL; 1605 1500 list_for_each_entry_reverse(msg, &msg_list, list) { 1606 - first_wr = rdma_rw_ctx_wrs(&msg->rw_ctx, t->qp, t->qp->port, 1501 + first_wr = rdma_rw_ctx_wrs(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port, 1607 1502 &msg->cqe, first_wr); 1608 1503 } 1609 1504 1610 - ret = ib_post_send(t->qp, first_wr, NULL); 1505 + ret = ib_post_send(sc->ib.qp, first_wr, NULL); 1611 1506 if (ret) { 1612 1507 pr_err("failed to post send wr for RDMA R/W: %d\n", ret); 1613 1508 goto out; 1614 1509 } 1615 1510 1616 - msg = list_last_entry(&msg_list, struct smb_direct_rdma_rw_msg, list); 1511 + msg = list_last_entry(&msg_list, struct smbdirect_rw_io, list); 1617 1512 wait_for_completion(&completion); 1618 - ret = msg->status; 1513 + ret = msg->error; 1619 1514 out: 1620 1515 list_for_each_entry_safe(msg, next_msg, &msg_list, list) { 1621 1516 list_del(&msg->list); 1622 1517 smb_direct_free_rdma_rw_msg(t, msg, 1623 1518 is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE); 1624 1519 } 1625 - atomic_add(credits_needed, &t->rw_credits); 1626 - wake_up(&t->wait_rw_credits); 1520 + atomic_add(credits_needed, &sc->rw_io.credits.count); 1521 + wake_up(&sc->rw_io.credits.wait_queue); 1627 1522 return ret; 1628 1523 } 1629 1524 1630 1525 static int smb_direct_rdma_write(struct ksmbd_transport *t, 1631 1526 void *buf, unsigned int buflen, 1632 - struct smb2_buffer_desc_v1 *desc, 1527 + struct smbdirect_buffer_descriptor_v1 *desc, 1633 1528 unsigned int desc_len) 1634 1529 { 1635 - return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen, 1530 + return smb_direct_rdma_xmit(SMBD_TRANS(t), buf, buflen, 1636 1531 desc, desc_len, false); 1637 1532 } 1638 1533 1639 1534 static int smb_direct_rdma_read(struct ksmbd_transport *t, 1640 1535 void *buf, unsigned int buflen, 1641 - struct smb2_buffer_desc_v1 *desc, 1536 + struct smbdirect_buffer_descriptor_v1 *desc, 1642 1537 unsigned int desc_len) 1643 1538 { 1644 - return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen, 1539 + return smb_direct_rdma_xmit(SMBD_TRANS(t), buf, buflen, 1645 1540 desc, desc_len, true); 1646 1541 } 1647 1542 1648 1543 static void smb_direct_disconnect(struct ksmbd_transport *t) 1649 1544 { 1650 - struct smb_direct_transport *st = smb_trans_direct_transfort(t); 1545 + struct smb_direct_transport *st = SMBD_TRANS(t); 1546 + struct smbdirect_socket *sc = &st->socket; 1651 1547 1652 - ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", st->cm_id); 1548 + ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", sc->rdma.cm_id); 1653 1549 1654 - smb_direct_disconnect_rdma_work(&st->disconnect_work); 1655 - wait_event_interruptible(st->wait_status, 1656 - st->status == SMB_DIRECT_CS_DISCONNECTED); 1657 1550 free_transport(st); 1658 1551 } 1659 1552 1660 1553 static void smb_direct_shutdown(struct ksmbd_transport *t) 1661 1554 { 1662 - struct smb_direct_transport *st = smb_trans_direct_transfort(t); 1555 + struct smb_direct_transport *st = SMBD_TRANS(t); 1556 + struct smbdirect_socket *sc = &st->socket; 1663 1557 1664 - ksmbd_debug(RDMA, "smb-direct shutdown cm_id=%p\n", st->cm_id); 1558 + ksmbd_debug(RDMA, "smb-direct shutdown cm_id=%p\n", sc->rdma.cm_id); 1665 1559 1666 - smb_direct_disconnect_rdma_work(&st->disconnect_work); 1560 + smb_direct_disconnect_rdma_work(&sc->disconnect_work); 1667 1561 } 1668 1562 1669 1563 static int smb_direct_cm_handler(struct rdma_cm_id *cm_id, 1670 1564 struct rdma_cm_event *event) 1671 1565 { 1672 - struct smb_direct_transport *t = cm_id->context; 1566 + struct smbdirect_socket *sc = cm_id->context; 1673 1567 1674 1568 ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n", 1675 1569 cm_id, rdma_event_msg(event->event), event->event); 1676 1570 1677 1571 switch (event->event) { 1678 1572 case RDMA_CM_EVENT_ESTABLISHED: { 1679 - t->status = SMB_DIRECT_CS_CONNECTED; 1680 - wake_up_interruptible(&t->wait_status); 1573 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING); 1574 + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; 1575 + wake_up(&sc->status_wait); 1681 1576 break; 1682 1577 } 1683 1578 case RDMA_CM_EVENT_DEVICE_REMOVAL: 1684 1579 case RDMA_CM_EVENT_DISCONNECTED: { 1685 - ib_drain_qp(t->qp); 1580 + ib_drain_qp(sc->ib.qp); 1686 1581 1687 - t->status = SMB_DIRECT_CS_DISCONNECTED; 1688 - wake_up_interruptible(&t->wait_status); 1689 - wake_up_interruptible(&t->wait_reassembly_queue); 1690 - wake_up(&t->wait_send_credits); 1582 + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; 1583 + smb_direct_disconnect_rdma_work(&sc->disconnect_work); 1691 1584 break; 1692 1585 } 1693 1586 case RDMA_CM_EVENT_CONNECT_ERROR: { 1694 - t->status = SMB_DIRECT_CS_DISCONNECTED; 1695 - wake_up_interruptible(&t->wait_status); 1587 + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; 1588 + smb_direct_disconnect_rdma_work(&sc->disconnect_work); 1696 1589 break; 1697 1590 } 1698 1591 default: ··· 1704 1601 1705 1602 static void smb_direct_qpair_handler(struct ib_event *event, void *context) 1706 1603 { 1707 - struct smb_direct_transport *t = context; 1604 + struct smbdirect_socket *sc = context; 1708 1605 1709 1606 ksmbd_debug(RDMA, "Received QP event. cm_id=%p, event=%s (%d)\n", 1710 - t->cm_id, ib_event_msg(event->event), event->event); 1607 + sc->rdma.cm_id, ib_event_msg(event->event), event->event); 1711 1608 1712 1609 switch (event->event) { 1713 1610 case IB_EVENT_CQ_ERR: 1714 1611 case IB_EVENT_QP_FATAL: 1715 - smb_direct_disconnect_rdma_connection(t); 1612 + smb_direct_disconnect_rdma_connection(sc); 1716 1613 break; 1717 1614 default: 1718 1615 break; 1719 1616 } 1720 1617 } 1721 1618 1722 - static int smb_direct_send_negotiate_response(struct smb_direct_transport *t, 1619 + static int smb_direct_send_negotiate_response(struct smbdirect_socket *sc, 1723 1620 int failed) 1724 1621 { 1725 - struct smb_direct_sendmsg *sendmsg; 1726 - struct smb_direct_negotiate_resp *resp; 1622 + struct smbdirect_socket_parameters *sp = &sc->parameters; 1623 + struct smbdirect_send_io *sendmsg; 1624 + struct smbdirect_negotiate_resp *resp; 1727 1625 int ret; 1728 1626 1729 - sendmsg = smb_direct_alloc_sendmsg(t); 1627 + sendmsg = smb_direct_alloc_sendmsg(sc); 1730 1628 if (IS_ERR(sendmsg)) 1731 1629 return -ENOMEM; 1732 1630 1733 - resp = (struct smb_direct_negotiate_resp *)sendmsg->packet; 1631 + resp = (struct smbdirect_negotiate_resp *)sendmsg->packet; 1734 1632 if (failed) { 1735 1633 memset(resp, 0, sizeof(*resp)); 1736 - resp->min_version = cpu_to_le16(0x0100); 1737 - resp->max_version = cpu_to_le16(0x0100); 1634 + resp->min_version = SMB_DIRECT_VERSION_LE; 1635 + resp->max_version = SMB_DIRECT_VERSION_LE; 1738 1636 resp->status = STATUS_NOT_SUPPORTED; 1637 + 1638 + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; 1739 1639 } else { 1740 1640 resp->status = STATUS_SUCCESS; 1741 1641 resp->min_version = SMB_DIRECT_VERSION_LE; ··· 1746 1640 resp->negotiated_version = SMB_DIRECT_VERSION_LE; 1747 1641 resp->reserved = 0; 1748 1642 resp->credits_requested = 1749 - cpu_to_le16(t->send_credit_target); 1750 - resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(t)); 1751 - resp->max_readwrite_size = cpu_to_le32(t->max_rdma_rw_size); 1752 - resp->preferred_send_size = cpu_to_le32(t->max_send_size); 1753 - resp->max_receive_size = cpu_to_le32(t->max_recv_size); 1643 + cpu_to_le16(sp->send_credit_target); 1644 + resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc)); 1645 + resp->max_readwrite_size = cpu_to_le32(sp->max_read_write_size); 1646 + resp->preferred_send_size = cpu_to_le32(sp->max_send_size); 1647 + resp->max_receive_size = cpu_to_le32(sp->max_recv_size); 1754 1648 resp->max_fragmented_size = 1755 - cpu_to_le32(t->max_fragmented_recv_size); 1649 + cpu_to_le32(sp->max_fragmented_recv_size); 1650 + 1651 + sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; 1652 + sc->status = SMBDIRECT_SOCKET_CONNECTED; 1756 1653 } 1757 1654 1758 - sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device, 1655 + sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev, 1759 1656 (void *)resp, sizeof(*resp), 1760 1657 DMA_TO_DEVICE); 1761 - ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr); 1658 + ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr); 1762 1659 if (ret) { 1763 - smb_direct_free_sendmsg(t, sendmsg); 1660 + smb_direct_free_sendmsg(sc, sendmsg); 1764 1661 return ret; 1765 1662 } 1766 1663 1767 1664 sendmsg->num_sge = 1; 1768 1665 sendmsg->sge[0].length = sizeof(*resp); 1769 - sendmsg->sge[0].lkey = t->pd->local_dma_lkey; 1666 + sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey; 1770 1667 1771 - ret = post_sendmsg(t, NULL, sendmsg); 1668 + ret = post_sendmsg(sc, NULL, sendmsg); 1772 1669 if (ret) { 1773 - smb_direct_free_sendmsg(t, sendmsg); 1670 + smb_direct_free_sendmsg(sc, sendmsg); 1774 1671 return ret; 1775 1672 } 1776 1673 1777 - wait_event(t->wait_send_pending, 1778 - atomic_read(&t->send_pending) == 0); 1674 + wait_event(sc->send_io.pending.zero_wait_queue, 1675 + atomic_read(&sc->send_io.pending.count) == 0 || 1676 + sc->status != SMBDIRECT_SOCKET_CONNECTED); 1677 + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 1678 + return -ENOTCONN; 1679 + 1779 1680 return 0; 1780 1681 } 1781 1682 1782 - static int smb_direct_accept_client(struct smb_direct_transport *t) 1683 + static int smb_direct_accept_client(struct smbdirect_socket *sc) 1783 1684 { 1685 + struct smbdirect_socket_parameters *sp = &sc->parameters; 1784 1686 struct rdma_conn_param conn_param; 1785 - struct ib_port_immutable port_immutable; 1786 - u32 ird_ord_hdr[2]; 1687 + __be32 ird_ord_hdr[2]; 1787 1688 int ret; 1788 1689 1690 + /* 1691 + * smb_direct_handle_connect_request() 1692 + * already negotiated sp->initiator_depth 1693 + * and sp->responder_resources 1694 + */ 1789 1695 memset(&conn_param, 0, sizeof(conn_param)); 1790 - conn_param.initiator_depth = min_t(u8, t->cm_id->device->attrs.max_qp_rd_atom, 1791 - SMB_DIRECT_CM_INITIATOR_DEPTH); 1792 - conn_param.responder_resources = 0; 1696 + conn_param.initiator_depth = sp->initiator_depth; 1697 + conn_param.responder_resources = sp->responder_resources; 1793 1698 1794 - t->cm_id->device->ops.get_port_immutable(t->cm_id->device, 1795 - t->cm_id->port_num, 1796 - &port_immutable); 1797 - if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) { 1798 - ird_ord_hdr[0] = conn_param.responder_resources; 1799 - ird_ord_hdr[1] = 1; 1699 + if (sc->rdma.legacy_iwarp) { 1700 + ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources); 1701 + ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth); 1800 1702 conn_param.private_data = ird_ord_hdr; 1801 1703 conn_param.private_data_len = sizeof(ird_ord_hdr); 1802 1704 } else { ··· 1815 1701 conn_param.rnr_retry_count = SMB_DIRECT_CM_RNR_RETRY; 1816 1702 conn_param.flow_control = 0; 1817 1703 1818 - ret = rdma_accept(t->cm_id, &conn_param); 1704 + /* 1705 + * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING 1706 + * so that the timer will cause a disconnect. 1707 + */ 1708 + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; 1709 + mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 1710 + msecs_to_jiffies(sp->negotiate_timeout_msec)); 1711 + 1712 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED); 1713 + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING; 1714 + ret = rdma_accept(sc->rdma.cm_id, &conn_param); 1819 1715 if (ret) { 1820 1716 pr_err("error at rdma_accept: %d\n", ret); 1821 1717 return ret; ··· 1833 1709 return 0; 1834 1710 } 1835 1711 1836 - static int smb_direct_prepare_negotiation(struct smb_direct_transport *t) 1712 + static int smb_direct_prepare_negotiation(struct smbdirect_socket *sc) 1837 1713 { 1714 + struct smbdirect_recv_io *recvmsg; 1838 1715 int ret; 1839 - struct smb_direct_recvmsg *recvmsg; 1840 1716 1841 - recvmsg = get_free_recvmsg(t); 1717 + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED); 1718 + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; 1719 + 1720 + sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REQ; 1721 + 1722 + recvmsg = get_free_recvmsg(sc); 1842 1723 if (!recvmsg) 1843 1724 return -ENOMEM; 1844 - recvmsg->type = SMB_DIRECT_MSG_NEGOTIATE_REQ; 1845 1725 1846 - ret = smb_direct_post_recv(t, recvmsg); 1726 + ret = smb_direct_post_recv(sc, recvmsg); 1847 1727 if (ret) { 1848 1728 pr_err("Can't post recv: %d\n", ret); 1849 1729 goto out_err; 1850 1730 } 1851 1731 1852 - t->negotiation_requested = false; 1853 - ret = smb_direct_accept_client(t); 1732 + ret = smb_direct_accept_client(sc); 1854 1733 if (ret) { 1855 1734 pr_err("Can't accept client\n"); 1856 1735 goto out_err; 1857 1736 } 1858 1737 1859 - smb_direct_post_recv_credits(&t->post_recv_credits_work); 1738 + smb_direct_post_recv_credits(&sc->recv_io.posted.refill_work); 1860 1739 return 0; 1861 1740 out_err: 1862 - put_recvmsg(t, recvmsg); 1741 + put_recvmsg(sc, recvmsg); 1863 1742 return ret; 1864 1743 } 1865 1744 1866 - static unsigned int smb_direct_get_max_fr_pages(struct smb_direct_transport *t) 1745 + static unsigned int smb_direct_get_max_fr_pages(struct smbdirect_socket *sc) 1867 1746 { 1868 1747 return min_t(unsigned int, 1869 - t->cm_id->device->attrs.max_fast_reg_page_list_len, 1748 + sc->ib.dev->attrs.max_fast_reg_page_list_len, 1870 1749 256); 1871 1750 } 1872 1751 1873 - static int smb_direct_init_params(struct smb_direct_transport *t, 1752 + static int smb_direct_init_params(struct smbdirect_socket *sc, 1874 1753 struct ib_qp_cap *cap) 1875 1754 { 1876 - struct ib_device *device = t->cm_id->device; 1755 + struct smbdirect_socket_parameters *sp = &sc->parameters; 1756 + struct ib_device *device = sc->ib.dev; 1877 1757 int max_send_sges, max_rw_wrs, max_send_wrs; 1878 1758 unsigned int max_sge_per_wr, wrs_per_credit; 1879 1759 1880 1760 /* need 3 more sge. because a SMB_DIRECT header, SMB2 header, 1881 1761 * SMB2 response could be mapped. 1882 1762 */ 1883 - t->max_send_size = smb_direct_max_send_size; 1884 - max_send_sges = DIV_ROUND_UP(t->max_send_size, PAGE_SIZE) + 3; 1885 - if (max_send_sges > SMB_DIRECT_MAX_SEND_SGES) { 1886 - pr_err("max_send_size %d is too large\n", t->max_send_size); 1763 + max_send_sges = DIV_ROUND_UP(sp->max_send_size, PAGE_SIZE) + 3; 1764 + if (max_send_sges > SMBDIRECT_SEND_IO_MAX_SGE) { 1765 + pr_err("max_send_size %d is too large\n", sp->max_send_size); 1887 1766 return -EINVAL; 1888 1767 } 1889 1768 ··· 1897 1770 * are needed for MR registration, RDMA R/W, local & remote 1898 1771 * MR invalidation. 1899 1772 */ 1900 - t->max_rdma_rw_size = smb_direct_max_read_write_size; 1901 - t->pages_per_rw_credit = smb_direct_get_max_fr_pages(t); 1902 - t->max_rw_credits = DIV_ROUND_UP(t->max_rdma_rw_size, 1903 - (t->pages_per_rw_credit - 1) * 1773 + sc->rw_io.credits.num_pages = smb_direct_get_max_fr_pages(sc); 1774 + sc->rw_io.credits.max = DIV_ROUND_UP(sp->max_read_write_size, 1775 + (sc->rw_io.credits.num_pages - 1) * 1904 1776 PAGE_SIZE); 1905 1777 1906 1778 max_sge_per_wr = min_t(unsigned int, device->attrs.max_send_sge, ··· 1907 1781 max_sge_per_wr = max_t(unsigned int, max_sge_per_wr, 1908 1782 max_send_sges); 1909 1783 wrs_per_credit = max_t(unsigned int, 4, 1910 - DIV_ROUND_UP(t->pages_per_rw_credit, 1784 + DIV_ROUND_UP(sc->rw_io.credits.num_pages, 1911 1785 max_sge_per_wr) + 1); 1912 - max_rw_wrs = t->max_rw_credits * wrs_per_credit; 1786 + max_rw_wrs = sc->rw_io.credits.max * wrs_per_credit; 1913 1787 1914 - max_send_wrs = smb_direct_send_credit_target + max_rw_wrs; 1788 + max_send_wrs = sp->send_credit_target + max_rw_wrs; 1915 1789 if (max_send_wrs > device->attrs.max_cqe || 1916 1790 max_send_wrs > device->attrs.max_qp_wr) { 1917 1791 pr_err("consider lowering send_credit_target = %d\n", 1918 - smb_direct_send_credit_target); 1792 + sp->send_credit_target); 1919 1793 pr_err("Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", 1920 1794 device->attrs.max_cqe, device->attrs.max_qp_wr); 1921 1795 return -EINVAL; 1922 1796 } 1923 1797 1924 - if (smb_direct_receive_credit_max > device->attrs.max_cqe || 1925 - smb_direct_receive_credit_max > device->attrs.max_qp_wr) { 1798 + if (sp->recv_credit_max > device->attrs.max_cqe || 1799 + sp->recv_credit_max > device->attrs.max_qp_wr) { 1926 1800 pr_err("consider lowering receive_credit_max = %d\n", 1927 - smb_direct_receive_credit_max); 1801 + sp->recv_credit_max); 1928 1802 pr_err("Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n", 1929 1803 device->attrs.max_cqe, device->attrs.max_qp_wr); 1930 1804 return -EINVAL; 1931 1805 } 1932 1806 1933 - if (device->attrs.max_send_sge < SMB_DIRECT_MAX_SEND_SGES) { 1807 + if (device->attrs.max_send_sge < SMBDIRECT_SEND_IO_MAX_SGE) { 1934 1808 pr_err("warning: device max_send_sge = %d too small\n", 1935 1809 device->attrs.max_send_sge); 1936 1810 return -EINVAL; 1937 1811 } 1938 - if (device->attrs.max_recv_sge < SMB_DIRECT_MAX_RECV_SGES) { 1812 + if (device->attrs.max_recv_sge < SMBDIRECT_RECV_IO_MAX_SGE) { 1939 1813 pr_err("warning: device max_recv_sge = %d too small\n", 1940 1814 device->attrs.max_recv_sge); 1941 1815 return -EINVAL; 1942 1816 } 1943 1817 1944 - t->recv_credits = 0; 1945 - t->count_avail_recvmsg = 0; 1818 + sc->recv_io.credits.target = 1; 1946 1819 1947 - t->recv_credit_max = smb_direct_receive_credit_max; 1948 - t->recv_credit_target = 10; 1949 - t->new_recv_credits = 0; 1950 - 1951 - t->send_credit_target = smb_direct_send_credit_target; 1952 - atomic_set(&t->send_credits, 0); 1953 - atomic_set(&t->rw_credits, t->max_rw_credits); 1954 - 1955 - t->max_send_size = smb_direct_max_send_size; 1956 - t->max_recv_size = smb_direct_max_receive_size; 1957 - t->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size; 1820 + atomic_set(&sc->rw_io.credits.count, sc->rw_io.credits.max); 1958 1821 1959 1822 cap->max_send_wr = max_send_wrs; 1960 - cap->max_recv_wr = t->recv_credit_max; 1961 - cap->max_send_sge = SMB_DIRECT_MAX_SEND_SGES; 1962 - cap->max_recv_sge = SMB_DIRECT_MAX_RECV_SGES; 1823 + cap->max_recv_wr = sp->recv_credit_max; 1824 + cap->max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE; 1825 + cap->max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE; 1963 1826 cap->max_inline_data = 0; 1964 - cap->max_rdma_ctxs = t->max_rw_credits; 1827 + cap->max_rdma_ctxs = sc->rw_io.credits.max; 1965 1828 return 0; 1966 1829 } 1967 1830 1968 - static void smb_direct_destroy_pools(struct smb_direct_transport *t) 1831 + static void smb_direct_destroy_pools(struct smbdirect_socket *sc) 1969 1832 { 1970 - struct smb_direct_recvmsg *recvmsg; 1833 + struct smbdirect_recv_io *recvmsg; 1971 1834 1972 - while ((recvmsg = get_free_recvmsg(t))) 1973 - mempool_free(recvmsg, t->recvmsg_mempool); 1835 + while ((recvmsg = get_free_recvmsg(sc))) 1836 + mempool_free(recvmsg, sc->recv_io.mem.pool); 1974 1837 1975 - mempool_destroy(t->recvmsg_mempool); 1976 - t->recvmsg_mempool = NULL; 1838 + mempool_destroy(sc->recv_io.mem.pool); 1839 + sc->recv_io.mem.pool = NULL; 1977 1840 1978 - kmem_cache_destroy(t->recvmsg_cache); 1979 - t->recvmsg_cache = NULL; 1841 + kmem_cache_destroy(sc->recv_io.mem.cache); 1842 + sc->recv_io.mem.cache = NULL; 1980 1843 1981 - mempool_destroy(t->sendmsg_mempool); 1982 - t->sendmsg_mempool = NULL; 1844 + mempool_destroy(sc->send_io.mem.pool); 1845 + sc->send_io.mem.pool = NULL; 1983 1846 1984 - kmem_cache_destroy(t->sendmsg_cache); 1985 - t->sendmsg_cache = NULL; 1847 + kmem_cache_destroy(sc->send_io.mem.cache); 1848 + sc->send_io.mem.cache = NULL; 1986 1849 } 1987 1850 1988 - static int smb_direct_create_pools(struct smb_direct_transport *t) 1851 + static int smb_direct_create_pools(struct smbdirect_socket *sc) 1989 1852 { 1853 + struct smbdirect_socket_parameters *sp = &sc->parameters; 1990 1854 char name[80]; 1991 1855 int i; 1992 - struct smb_direct_recvmsg *recvmsg; 1856 + struct smbdirect_recv_io *recvmsg; 1993 1857 1994 - snprintf(name, sizeof(name), "smb_direct_rqst_pool_%p", t); 1995 - t->sendmsg_cache = kmem_cache_create(name, 1996 - sizeof(struct smb_direct_sendmsg) + 1997 - sizeof(struct smb_direct_negotiate_resp), 1858 + snprintf(name, sizeof(name), "smbdirect_send_io_pool_%p", sc); 1859 + sc->send_io.mem.cache = kmem_cache_create(name, 1860 + sizeof(struct smbdirect_send_io) + 1861 + sizeof(struct smbdirect_negotiate_resp), 1998 1862 0, SLAB_HWCACHE_ALIGN, NULL); 1999 - if (!t->sendmsg_cache) 1863 + if (!sc->send_io.mem.cache) 2000 1864 return -ENOMEM; 2001 1865 2002 - t->sendmsg_mempool = mempool_create(t->send_credit_target, 1866 + sc->send_io.mem.pool = mempool_create(sp->send_credit_target, 2003 1867 mempool_alloc_slab, mempool_free_slab, 2004 - t->sendmsg_cache); 2005 - if (!t->sendmsg_mempool) 1868 + sc->send_io.mem.cache); 1869 + if (!sc->send_io.mem.pool) 2006 1870 goto err; 2007 1871 2008 - snprintf(name, sizeof(name), "smb_direct_resp_%p", t); 2009 - t->recvmsg_cache = kmem_cache_create(name, 2010 - sizeof(struct smb_direct_recvmsg) + 2011 - t->max_recv_size, 1872 + snprintf(name, sizeof(name), "smbdirect_recv_io_pool_%p", sc); 1873 + sc->recv_io.mem.cache = kmem_cache_create(name, 1874 + sizeof(struct smbdirect_recv_io) + 1875 + sp->max_recv_size, 2012 1876 0, SLAB_HWCACHE_ALIGN, NULL); 2013 - if (!t->recvmsg_cache) 1877 + if (!sc->recv_io.mem.cache) 2014 1878 goto err; 2015 1879 2016 - t->recvmsg_mempool = 2017 - mempool_create(t->recv_credit_max, mempool_alloc_slab, 2018 - mempool_free_slab, t->recvmsg_cache); 2019 - if (!t->recvmsg_mempool) 1880 + sc->recv_io.mem.pool = 1881 + mempool_create(sp->recv_credit_max, mempool_alloc_slab, 1882 + mempool_free_slab, sc->recv_io.mem.cache); 1883 + if (!sc->recv_io.mem.pool) 2020 1884 goto err; 2021 1885 2022 - INIT_LIST_HEAD(&t->recvmsg_queue); 2023 - 2024 - for (i = 0; i < t->recv_credit_max; i++) { 2025 - recvmsg = mempool_alloc(t->recvmsg_mempool, KSMBD_DEFAULT_GFP); 1886 + for (i = 0; i < sp->recv_credit_max; i++) { 1887 + recvmsg = mempool_alloc(sc->recv_io.mem.pool, KSMBD_DEFAULT_GFP); 2026 1888 if (!recvmsg) 2027 1889 goto err; 2028 - recvmsg->transport = t; 1890 + recvmsg->socket = sc; 2029 1891 recvmsg->sge.length = 0; 2030 - list_add(&recvmsg->list, &t->recvmsg_queue); 1892 + list_add(&recvmsg->list, &sc->recv_io.free.list); 2031 1893 } 2032 - t->count_avail_recvmsg = t->recv_credit_max; 2033 1894 2034 1895 return 0; 2035 1896 err: 2036 - smb_direct_destroy_pools(t); 1897 + smb_direct_destroy_pools(sc); 2037 1898 return -ENOMEM; 2038 1899 } 2039 1900 2040 - static int smb_direct_create_qpair(struct smb_direct_transport *t, 1901 + static int smb_direct_create_qpair(struct smbdirect_socket *sc, 2041 1902 struct ib_qp_cap *cap) 2042 1903 { 1904 + struct smbdirect_socket_parameters *sp = &sc->parameters; 2043 1905 int ret; 2044 1906 struct ib_qp_init_attr qp_attr; 2045 1907 int pages_per_rw; 2046 1908 2047 - t->pd = ib_alloc_pd(t->cm_id->device, 0); 2048 - if (IS_ERR(t->pd)) { 1909 + sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0); 1910 + if (IS_ERR(sc->ib.pd)) { 2049 1911 pr_err("Can't create RDMA PD\n"); 2050 - ret = PTR_ERR(t->pd); 2051 - t->pd = NULL; 1912 + ret = PTR_ERR(sc->ib.pd); 1913 + sc->ib.pd = NULL; 2052 1914 return ret; 2053 1915 } 2054 1916 2055 - t->send_cq = ib_alloc_cq(t->cm_id->device, t, 2056 - smb_direct_send_credit_target + cap->max_rdma_ctxs, 2057 - 0, IB_POLL_WORKQUEUE); 2058 - if (IS_ERR(t->send_cq)) { 1917 + sc->ib.send_cq = ib_alloc_cq_any(sc->ib.dev, sc, 1918 + sp->send_credit_target + 1919 + cap->max_rdma_ctxs, 1920 + IB_POLL_WORKQUEUE); 1921 + if (IS_ERR(sc->ib.send_cq)) { 2059 1922 pr_err("Can't create RDMA send CQ\n"); 2060 - ret = PTR_ERR(t->send_cq); 2061 - t->send_cq = NULL; 1923 + ret = PTR_ERR(sc->ib.send_cq); 1924 + sc->ib.send_cq = NULL; 2062 1925 goto err; 2063 1926 } 2064 1927 2065 - t->recv_cq = ib_alloc_cq(t->cm_id->device, t, 2066 - t->recv_credit_max, 0, IB_POLL_WORKQUEUE); 2067 - if (IS_ERR(t->recv_cq)) { 1928 + sc->ib.recv_cq = ib_alloc_cq_any(sc->ib.dev, sc, 1929 + sp->recv_credit_max, 1930 + IB_POLL_WORKQUEUE); 1931 + if (IS_ERR(sc->ib.recv_cq)) { 2068 1932 pr_err("Can't create RDMA recv CQ\n"); 2069 - ret = PTR_ERR(t->recv_cq); 2070 - t->recv_cq = NULL; 1933 + ret = PTR_ERR(sc->ib.recv_cq); 1934 + sc->ib.recv_cq = NULL; 2071 1935 goto err; 2072 1936 } 2073 1937 2074 1938 memset(&qp_attr, 0, sizeof(qp_attr)); 2075 1939 qp_attr.event_handler = smb_direct_qpair_handler; 2076 - qp_attr.qp_context = t; 1940 + qp_attr.qp_context = sc; 2077 1941 qp_attr.cap = *cap; 2078 1942 qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 2079 1943 qp_attr.qp_type = IB_QPT_RC; 2080 - qp_attr.send_cq = t->send_cq; 2081 - qp_attr.recv_cq = t->recv_cq; 1944 + qp_attr.send_cq = sc->ib.send_cq; 1945 + qp_attr.recv_cq = sc->ib.recv_cq; 2082 1946 qp_attr.port_num = ~0; 2083 1947 2084 - ret = rdma_create_qp(t->cm_id, t->pd, &qp_attr); 1948 + ret = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr); 2085 1949 if (ret) { 2086 1950 pr_err("Can't create RDMA QP: %d\n", ret); 2087 1951 goto err; 2088 1952 } 2089 1953 2090 - t->qp = t->cm_id->qp; 2091 - t->cm_id->event_handler = smb_direct_cm_handler; 1954 + sc->ib.qp = sc->rdma.cm_id->qp; 1955 + sc->rdma.cm_id->event_handler = smb_direct_cm_handler; 2092 1956 2093 - pages_per_rw = DIV_ROUND_UP(t->max_rdma_rw_size, PAGE_SIZE) + 1; 2094 - if (pages_per_rw > t->cm_id->device->attrs.max_sgl_rd) { 2095 - ret = ib_mr_pool_init(t->qp, &t->qp->rdma_mrs, 2096 - t->max_rw_credits, IB_MR_TYPE_MEM_REG, 2097 - t->pages_per_rw_credit, 0); 1957 + pages_per_rw = DIV_ROUND_UP(sp->max_read_write_size, PAGE_SIZE) + 1; 1958 + if (pages_per_rw > sc->ib.dev->attrs.max_sgl_rd) { 1959 + ret = ib_mr_pool_init(sc->ib.qp, &sc->ib.qp->rdma_mrs, 1960 + sc->rw_io.credits.max, IB_MR_TYPE_MEM_REG, 1961 + sc->rw_io.credits.num_pages, 0); 2098 1962 if (ret) { 2099 - pr_err("failed to init mr pool count %d pages %d\n", 2100 - t->max_rw_credits, t->pages_per_rw_credit); 1963 + pr_err("failed to init mr pool count %zu pages %zu\n", 1964 + sc->rw_io.credits.max, sc->rw_io.credits.num_pages); 2101 1965 goto err; 2102 1966 } 2103 1967 } 2104 1968 2105 1969 return 0; 2106 1970 err: 2107 - if (t->qp) { 2108 - t->qp = NULL; 2109 - rdma_destroy_qp(t->cm_id); 1971 + if (sc->ib.qp) { 1972 + sc->ib.qp = NULL; 1973 + rdma_destroy_qp(sc->rdma.cm_id); 2110 1974 } 2111 - if (t->recv_cq) { 2112 - ib_destroy_cq(t->recv_cq); 2113 - t->recv_cq = NULL; 1975 + if (sc->ib.recv_cq) { 1976 + ib_destroy_cq(sc->ib.recv_cq); 1977 + sc->ib.recv_cq = NULL; 2114 1978 } 2115 - if (t->send_cq) { 2116 - ib_destroy_cq(t->send_cq); 2117 - t->send_cq = NULL; 1979 + if (sc->ib.send_cq) { 1980 + ib_destroy_cq(sc->ib.send_cq); 1981 + sc->ib.send_cq = NULL; 2118 1982 } 2119 - if (t->pd) { 2120 - ib_dealloc_pd(t->pd); 2121 - t->pd = NULL; 1983 + if (sc->ib.pd) { 1984 + ib_dealloc_pd(sc->ib.pd); 1985 + sc->ib.pd = NULL; 2122 1986 } 2123 1987 return ret; 2124 1988 } 2125 1989 2126 1990 static int smb_direct_prepare(struct ksmbd_transport *t) 2127 1991 { 2128 - struct smb_direct_transport *st = smb_trans_direct_transfort(t); 2129 - struct smb_direct_recvmsg *recvmsg; 2130 - struct smb_direct_negotiate_req *req; 1992 + struct smb_direct_transport *st = SMBD_TRANS(t); 1993 + struct smbdirect_socket *sc = &st->socket; 1994 + struct smbdirect_socket_parameters *sp = &sc->parameters; 1995 + struct smbdirect_recv_io *recvmsg; 1996 + struct smbdirect_negotiate_req *req; 1997 + unsigned long flags; 2131 1998 int ret; 2132 1999 2000 + /* 2001 + * We are waiting to pass the following states: 2002 + * 2003 + * SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED 2004 + * SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING 2005 + * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED 2006 + * 2007 + * To finally get to SMBDIRECT_SOCKET_NEGOTIATE_RUNNING 2008 + * in order to continue below. 2009 + * 2010 + * Everything else is unexpected and an error. 2011 + */ 2133 2012 ksmbd_debug(RDMA, "Waiting for SMB_DIRECT negotiate request\n"); 2134 - ret = wait_event_interruptible_timeout(st->wait_status, 2135 - st->negotiation_requested || 2136 - st->status == SMB_DIRECT_CS_DISCONNECTED, 2137 - SMB_DIRECT_NEGOTIATE_TIMEOUT * HZ); 2138 - if (ret <= 0 || st->status == SMB_DIRECT_CS_DISCONNECTED) 2013 + ret = wait_event_interruptible_timeout(sc->status_wait, 2014 + sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED && 2015 + sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING && 2016 + sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, 2017 + msecs_to_jiffies(sp->negotiate_timeout_msec)); 2018 + if (ret <= 0 || sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING) 2139 2019 return ret < 0 ? ret : -ETIMEDOUT; 2140 2020 2141 - recvmsg = get_first_reassembly(st); 2021 + recvmsg = get_first_reassembly(sc); 2142 2022 if (!recvmsg) 2143 2023 return -ECONNABORTED; 2144 2024 ··· 2152 2020 if (ret == -ECONNABORTED) 2153 2021 goto out; 2154 2022 2155 - req = (struct smb_direct_negotiate_req *)recvmsg->packet; 2156 - st->max_recv_size = min_t(int, st->max_recv_size, 2023 + req = (struct smbdirect_negotiate_req *)recvmsg->packet; 2024 + sp->max_recv_size = min_t(int, sp->max_recv_size, 2157 2025 le32_to_cpu(req->preferred_send_size)); 2158 - st->max_send_size = min_t(int, st->max_send_size, 2026 + sp->max_send_size = min_t(int, sp->max_send_size, 2159 2027 le32_to_cpu(req->max_receive_size)); 2160 - st->max_fragmented_send_size = 2028 + sp->max_fragmented_send_size = 2161 2029 le32_to_cpu(req->max_fragmented_size); 2162 - st->max_fragmented_recv_size = 2163 - (st->recv_credit_max * st->max_recv_size) / 2; 2030 + sp->max_fragmented_recv_size = 2031 + (sp->recv_credit_max * sp->max_recv_size) / 2; 2032 + sc->recv_io.credits.target = le16_to_cpu(req->credits_requested); 2033 + sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max); 2034 + sc->recv_io.credits.target = max_t(u16, sc->recv_io.credits.target, 1); 2164 2035 2165 - ret = smb_direct_send_negotiate_response(st, ret); 2036 + ret = smb_direct_send_negotiate_response(sc, ret); 2166 2037 out: 2167 - spin_lock_irq(&st->reassembly_queue_lock); 2168 - st->reassembly_queue_length--; 2038 + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 2039 + sc->recv_io.reassembly.queue_length--; 2169 2040 list_del(&recvmsg->list); 2170 - spin_unlock_irq(&st->reassembly_queue_lock); 2171 - put_recvmsg(st, recvmsg); 2041 + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 2042 + put_recvmsg(sc, recvmsg); 2172 2043 2173 2044 return ret; 2174 2045 } 2175 2046 2176 - static int smb_direct_connect(struct smb_direct_transport *st) 2047 + static int smb_direct_connect(struct smbdirect_socket *sc) 2177 2048 { 2178 - int ret; 2179 2049 struct ib_qp_cap qp_cap; 2050 + int ret; 2180 2051 2181 - ret = smb_direct_init_params(st, &qp_cap); 2052 + ret = smb_direct_init_params(sc, &qp_cap); 2182 2053 if (ret) { 2183 2054 pr_err("Can't configure RDMA parameters\n"); 2184 2055 return ret; 2185 2056 } 2186 2057 2187 - ret = smb_direct_create_pools(st); 2058 + ret = smb_direct_create_pools(sc); 2188 2059 if (ret) { 2189 2060 pr_err("Can't init RDMA pool: %d\n", ret); 2190 2061 return ret; 2191 2062 } 2192 2063 2193 - ret = smb_direct_create_qpair(st, &qp_cap); 2064 + ret = smb_direct_create_qpair(sc, &qp_cap); 2194 2065 if (ret) { 2195 2066 pr_err("Can't accept RDMA client: %d\n", ret); 2196 2067 return ret; 2197 2068 } 2198 2069 2199 - ret = smb_direct_prepare_negotiation(st); 2070 + ret = smb_direct_prepare_negotiation(sc); 2200 2071 if (ret) { 2201 2072 pr_err("Can't negotiate: %d\n", ret); 2202 2073 return ret; ··· 2216 2081 return true; 2217 2082 } 2218 2083 2219 - static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id) 2084 + static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id, 2085 + struct rdma_cm_event *event) 2220 2086 { 2221 2087 struct smb_direct_transport *t; 2088 + struct smbdirect_socket *sc; 2089 + struct smbdirect_socket_parameters *sp; 2222 2090 struct task_struct *handler; 2091 + u8 peer_initiator_depth; 2092 + u8 peer_responder_resources; 2223 2093 int ret; 2224 2094 2225 2095 if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) { ··· 2237 2097 t = alloc_transport(new_cm_id); 2238 2098 if (!t) 2239 2099 return -ENOMEM; 2100 + sc = &t->socket; 2101 + sp = &sc->parameters; 2240 2102 2241 - ret = smb_direct_connect(t); 2103 + peer_initiator_depth = event->param.conn.initiator_depth; 2104 + peer_responder_resources = event->param.conn.responder_resources; 2105 + if (rdma_protocol_iwarp(new_cm_id->device, new_cm_id->port_num) && 2106 + event->param.conn.private_data_len == 8) { 2107 + /* 2108 + * Legacy clients with only iWarp MPA v1 support 2109 + * need a private blob in order to negotiate 2110 + * the IRD/ORD values. 2111 + */ 2112 + const __be32 *ird_ord_hdr = event->param.conn.private_data; 2113 + u32 ird32 = be32_to_cpu(ird_ord_hdr[0]); 2114 + u32 ord32 = be32_to_cpu(ird_ord_hdr[1]); 2115 + 2116 + /* 2117 + * cifs.ko sends the legacy IRD/ORD negotiation 2118 + * event if iWarp MPA v2 was used. 2119 + * 2120 + * Here we check that the values match and only 2121 + * mark the client as legacy if they don't match. 2122 + */ 2123 + if ((u32)event->param.conn.initiator_depth != ird32 || 2124 + (u32)event->param.conn.responder_resources != ord32) { 2125 + /* 2126 + * There are broken clients (old cifs.ko) 2127 + * using little endian and also 2128 + * struct rdma_conn_param only uses u8 2129 + * for initiator_depth and responder_resources, 2130 + * so we truncate the value to U8_MAX. 2131 + * 2132 + * smb_direct_accept_client() will then 2133 + * do the real negotiation in order to 2134 + * select the minimum between client and 2135 + * server. 2136 + */ 2137 + ird32 = min_t(u32, ird32, U8_MAX); 2138 + ord32 = min_t(u32, ord32, U8_MAX); 2139 + 2140 + sc->rdma.legacy_iwarp = true; 2141 + peer_initiator_depth = (u8)ird32; 2142 + peer_responder_resources = (u8)ord32; 2143 + } 2144 + } 2145 + 2146 + /* 2147 + * First set what the we as server are able to support 2148 + */ 2149 + sp->initiator_depth = min_t(u8, sp->initiator_depth, 2150 + new_cm_id->device->attrs.max_qp_rd_atom); 2151 + 2152 + /* 2153 + * negotiate the value by using the minimum 2154 + * between client and server if the client provided 2155 + * non 0 values. 2156 + */ 2157 + if (peer_initiator_depth != 0) 2158 + sp->initiator_depth = min_t(u8, sp->initiator_depth, 2159 + peer_initiator_depth); 2160 + if (peer_responder_resources != 0) 2161 + sp->responder_resources = min_t(u8, sp->responder_resources, 2162 + peer_responder_resources); 2163 + 2164 + ret = smb_direct_connect(sc); 2242 2165 if (ret) 2243 2166 goto out_err; 2244 2167 ··· 2325 2122 { 2326 2123 switch (event->event) { 2327 2124 case RDMA_CM_EVENT_CONNECT_REQUEST: { 2328 - int ret = smb_direct_handle_connect_request(cm_id); 2125 + int ret = smb_direct_handle_connect_request(cm_id, event); 2329 2126 2330 2127 if (ret) { 2331 2128 pr_err("Can't create transport: %d\n", ret);

+2 -43

fs/smb/server/transport_rdma.h

··· 11 11 #define SMBD_MIN_IOSIZE (512 * 1024) 12 12 #define SMBD_MAX_IOSIZE (16 * 1024 * 1024) 13 13 14 - /* SMB DIRECT negotiation request packet [MS-SMBD] 2.2.1 */ 15 - struct smb_direct_negotiate_req { 16 - __le16 min_version; 17 - __le16 max_version; 18 - __le16 reserved; 19 - __le16 credits_requested; 20 - __le32 preferred_send_size; 21 - __le32 max_receive_size; 22 - __le32 max_fragmented_size; 23 - } __packed; 24 - 25 - /* SMB DIRECT negotiation response packet [MS-SMBD] 2.2.2 */ 26 - struct smb_direct_negotiate_resp { 27 - __le16 min_version; 28 - __le16 max_version; 29 - __le16 negotiated_version; 30 - __le16 reserved; 31 - __le16 credits_requested; 32 - __le16 credits_granted; 33 - __le32 status; 34 - __le32 max_readwrite_size; 35 - __le32 preferred_send_size; 36 - __le32 max_receive_size; 37 - __le32 max_fragmented_size; 38 - } __packed; 39 - 40 - #define SMB_DIRECT_RESPONSE_REQUESTED 0x0001 41 - 42 - /* SMB DIRECT data transfer packet with payload [MS-SMBD] 2.2.3 */ 43 - struct smb_direct_data_transfer { 44 - __le16 credits_requested; 45 - __le16 credits_granted; 46 - __le16 flags; 47 - __le16 reserved; 48 - __le32 remaining_data_length; 49 - __le32 data_offset; 50 - __le32 data_length; 51 - __le32 padding; 52 - __u8 buffer[]; 53 - } __packed; 54 - 55 14 #ifdef CONFIG_SMB_SERVER_SMBDIRECT 56 15 int ksmbd_rdma_init(void); 57 16 void ksmbd_rdma_stop_listening(void); 58 17 void ksmbd_rdma_destroy(void); 59 18 bool ksmbd_rdma_capable_netdev(struct net_device *netdev); 60 19 void init_smbd_max_io_size(unsigned int sz); 61 - unsigned int get_smbd_max_read_write_size(void); 20 + unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt); 62 21 #else 63 22 static inline int ksmbd_rdma_init(void) { return 0; } 64 23 static inline void ksmbd_rdma_stop_listening(void) { } 65 24 static inline void ksmbd_rdma_destroy(void) { } 66 25 static inline bool ksmbd_rdma_capable_netdev(struct net_device *netdev) { return false; } 67 26 static inline void init_smbd_max_io_size(unsigned int sz) { } 68 - static inline unsigned int get_smbd_max_read_write_size(void) { return 0; } 27 + static inline unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt) { return 0; } 69 28 #endif 70 29 71 30 #endif /* __KSMBD_TRANSPORT_RDMA_H__ */

Configure Feed

Configure Feed