smb: server: make use of smbdirect_connection_send_iter() and related functions

+4 -427

1 changed file

expand all

smb

server

+4 -427

fs/smb/server/transport_rdma.c

··· 209 209 return sp->max_read_write_size; 210 210 } 211 211 212 - static int smb_direct_post_send_data(struct smbdirect_socket *sc, 213 - struct smbdirect_send_batch *send_ctx, 214 - struct iov_iter *iter, 215 - u32 remaining_data_length); 216 - 217 - static void smb_direct_send_immediate_work(struct work_struct *work) 218 - { 219 - struct smbdirect_socket *sc = 220 - container_of(work, struct smbdirect_socket, idle.immediate_work); 221 - 222 - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 223 - return; 224 - 225 - smb_direct_post_send_data(sc, NULL, NULL, 0); 226 - } 227 - 228 212 static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id) 229 213 { 230 214 struct smb_direct_transport *t; ··· 602 618 return ret; 603 619 } 604 620 605 - static int smb_direct_post_send(struct smbdirect_socket *sc, 606 - struct ib_send_wr *wr) 607 - { 608 - int ret; 609 - 610 - atomic_inc(&sc->send_io.pending.count); 611 - ret = ib_post_send(sc->ib.qp, wr, NULL); 612 - if (ret) { 613 - pr_err("failed to post send: %d\n", ret); 614 - smbdirect_socket_schedule_cleanup(sc, ret); 615 - } 616 - return ret; 617 - } 618 - 619 - static void smb_direct_send_ctx_init(struct smbdirect_send_batch *send_ctx, 620 - bool need_invalidate_rkey, 621 - unsigned int remote_key) 622 - { 623 - INIT_LIST_HEAD(&send_ctx->msg_list); 624 - send_ctx->wr_cnt = 0; 625 - send_ctx->need_invalidate_rkey = need_invalidate_rkey; 626 - send_ctx->remote_key = remote_key; 627 - send_ctx->credit = 0; 628 - } 629 - 630 - static int smb_direct_flush_send_list(struct smbdirect_socket *sc, 631 - struct smbdirect_send_batch *send_ctx, 632 - bool is_last) 633 - { 634 - struct smbdirect_send_io *first, *last; 635 - int ret = 0; 636 - 637 - if (list_empty(&send_ctx->msg_list)) 638 - goto release_credit; 639 - 640 - first = list_first_entry(&send_ctx->msg_list, 641 - struct smbdirect_send_io, 642 - sibling_list); 643 - last = list_last_entry(&send_ctx->msg_list, 644 - struct smbdirect_send_io, 645 - sibling_list); 646 - 647 - if (send_ctx->need_invalidate_rkey) { 648 - first->wr.opcode = IB_WR_SEND_WITH_INV; 649 - first->wr.ex.invalidate_rkey = send_ctx->remote_key; 650 - send_ctx->need_invalidate_rkey = false; 651 - send_ctx->remote_key = 0; 652 - } 653 - 654 - last->wr.send_flags = IB_SEND_SIGNALED; 655 - last->wr.wr_cqe = &last->cqe; 656 - 657 - /* 658 - * Remove last from send_ctx->msg_list 659 - * and splice the rest of send_ctx->msg_list 660 - * to last->sibling_list. 661 - * 662 - * send_ctx->msg_list is a valid empty list 663 - * at the end. 664 - */ 665 - list_del_init(&last->sibling_list); 666 - list_splice_tail_init(&send_ctx->msg_list, &last->sibling_list); 667 - send_ctx->wr_cnt = 0; 668 - 669 - ret = smb_direct_post_send(sc, &first->wr); 670 - if (ret) { 671 - struct smbdirect_send_io *sibling, *next; 672 - 673 - list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) { 674 - list_del_init(&sibling->sibling_list); 675 - smbdirect_connection_free_send_io(sibling); 676 - } 677 - smbdirect_connection_free_send_io(last); 678 - } 679 - 680 - release_credit: 681 - if (is_last && !ret && send_ctx->credit) { 682 - atomic_add(send_ctx->credit, &sc->send_io.bcredits.count); 683 - send_ctx->credit = 0; 684 - wake_up(&sc->send_io.bcredits.wait_queue); 685 - } 686 - 687 - return ret; 688 - } 689 - 690 - 691 - static int wait_for_send_bcredit(struct smbdirect_socket *sc, 692 - struct smbdirect_send_batch *send_ctx) 693 - { 694 - int ret; 695 - 696 - if (send_ctx->credit) 697 - return 0; 698 - 699 - ret = smbdirect_socket_wait_for_credits(sc, 700 - SMBDIRECT_SOCKET_CONNECTED, 701 - -ENOTCONN, 702 - &sc->send_io.bcredits.wait_queue, 703 - &sc->send_io.bcredits.count, 704 - 1); 705 - if (ret) 706 - return ret; 707 - 708 - send_ctx->credit = 1; 709 - return 0; 710 - } 711 - 712 - static int wait_for_send_lcredit(struct smbdirect_socket *sc, 713 - struct smbdirect_send_batch *send_ctx) 714 - { 715 - if (send_ctx && (atomic_read(&sc->send_io.lcredits.count) <= 1)) { 716 - int ret; 717 - 718 - ret = smb_direct_flush_send_list(sc, send_ctx, false); 719 - if (ret) 720 - return ret; 721 - } 722 - 723 - return smbdirect_socket_wait_for_credits(sc, 724 - SMBDIRECT_SOCKET_CONNECTED, 725 - -ENOTCONN, 726 - &sc->send_io.lcredits.wait_queue, 727 - &sc->send_io.lcredits.count, 728 - 1); 729 - } 730 - 731 - static int wait_for_send_credits(struct smbdirect_socket *sc, 732 - struct smbdirect_send_batch *send_ctx) 733 - { 734 - int ret; 735 - 736 - if (send_ctx && 737 - (send_ctx->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) { 738 - ret = smb_direct_flush_send_list(sc, send_ctx, false); 739 - if (ret) 740 - return ret; 741 - } 742 - 743 - return smbdirect_socket_wait_for_credits(sc, 744 - SMBDIRECT_SOCKET_CONNECTED, 745 - -ENOTCONN, 746 - &sc->send_io.credits.wait_queue, 747 - &sc->send_io.credits.count, 748 - 1); 749 - } 750 - 751 - static int post_sendmsg(struct smbdirect_socket *sc, 752 - struct smbdirect_send_batch *send_ctx, 753 - struct smbdirect_send_io *msg) 754 - { 755 - int i; 756 - 757 - for (i = 0; i < msg->num_sge; i++) 758 - ib_dma_sync_single_for_device(sc->ib.dev, 759 - msg->sge[i].addr, msg->sge[i].length, 760 - DMA_TO_DEVICE); 761 - 762 - msg->cqe.done = smbdirect_connection_send_io_done; 763 - msg->wr.opcode = IB_WR_SEND; 764 - msg->wr.sg_list = &msg->sge[0]; 765 - msg->wr.num_sge = msg->num_sge; 766 - msg->wr.next = NULL; 767 - 768 - if (send_ctx) { 769 - msg->wr.wr_cqe = NULL; 770 - msg->wr.send_flags = 0; 771 - if (!list_empty(&send_ctx->msg_list)) { 772 - struct smbdirect_send_io *last; 773 - 774 - last = list_last_entry(&send_ctx->msg_list, 775 - struct smbdirect_send_io, 776 - sibling_list); 777 - last->wr.next = &msg->wr; 778 - } 779 - list_add_tail(&msg->sibling_list, &send_ctx->msg_list); 780 - send_ctx->wr_cnt++; 781 - return 0; 782 - } 783 - 784 - msg->wr.wr_cqe = &msg->cqe; 785 - msg->wr.send_flags = IB_SEND_SIGNALED; 786 - return smb_direct_post_send(sc, &msg->wr); 787 - } 788 - 789 - static int smb_direct_post_send_data(struct smbdirect_socket *sc, 790 - struct smbdirect_send_batch *send_ctx, 791 - struct iov_iter *iter, 792 - u32 remaining_data_length) 793 - { 794 - const struct smbdirect_socket_parameters *sp = &sc->parameters; 795 - int ret; 796 - struct smbdirect_send_io *msg; 797 - struct smbdirect_data_transfer *packet; 798 - size_t header_length; 799 - u32 data_length = 0; 800 - struct smbdirect_send_batch _send_ctx; 801 - u16 new_credits; 802 - 803 - if (iter) { 804 - header_length = sizeof(struct smbdirect_data_transfer); 805 - if (WARN_ON_ONCE(remaining_data_length == 0 || 806 - iov_iter_count(iter) > remaining_data_length)) 807 - return -EINVAL; 808 - } else { 809 - /* If this is a packet without payload, don't send padding */ 810 - header_length = offsetof(struct smbdirect_data_transfer, padding); 811 - if (WARN_ON_ONCE(remaining_data_length)) 812 - return -EINVAL; 813 - } 814 - 815 - if (!send_ctx) { 816 - smb_direct_send_ctx_init(&_send_ctx, false, 0); 817 - send_ctx = &_send_ctx; 818 - } 819 - 820 - ret = wait_for_send_bcredit(sc, send_ctx); 821 - if (ret) 822 - goto bcredit_failed; 823 - 824 - ret = wait_for_send_lcredit(sc, send_ctx); 825 - if (ret) 826 - goto lcredit_failed; 827 - 828 - ret = wait_for_send_credits(sc, send_ctx); 829 - if (ret) 830 - goto credit_failed; 831 - 832 - new_credits = smbdirect_connection_grant_recv_credits(sc); 833 - if (new_credits == 0 && 834 - atomic_read(&sc->send_io.credits.count) == 0 && 835 - atomic_read(&sc->recv_io.credits.count) == 0) { 836 - queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); 837 - ret = wait_event_interruptible(sc->send_io.credits.wait_queue, 838 - atomic_read(&sc->send_io.credits.count) >= 1 || 839 - atomic_read(&sc->recv_io.credits.available) >= 1 || 840 - sc->status != SMBDIRECT_SOCKET_CONNECTED); 841 - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 842 - ret = -ENOTCONN; 843 - if (ret < 0) 844 - goto credit_failed; 845 - 846 - new_credits = smbdirect_connection_grant_recv_credits(sc); 847 - } 848 - 849 - msg = smbdirect_connection_alloc_send_io(sc); 850 - if (IS_ERR(msg)) { 851 - ret = PTR_ERR(msg); 852 - goto alloc_failed; 853 - } 854 - 855 - /* Map the packet to DMA */ 856 - msg->sge[0].addr = ib_dma_map_single(sc->ib.dev, 857 - msg->packet, 858 - header_length, 859 - DMA_TO_DEVICE); 860 - ret = ib_dma_mapping_error(sc->ib.dev, msg->sge[0].addr); 861 - if (ret) 862 - goto err; 863 - 864 - msg->sge[0].length = header_length; 865 - msg->sge[0].lkey = sc->ib.pd->local_dma_lkey; 866 - msg->num_sge = 1; 867 - 868 - if (iter) { 869 - struct smbdirect_map_sges extract = { 870 - .num_sge = msg->num_sge, 871 - .max_sge = ARRAY_SIZE(msg->sge), 872 - .sge = msg->sge, 873 - .device = sc->ib.dev, 874 - .local_dma_lkey = sc->ib.pd->local_dma_lkey, 875 - .direction = DMA_TO_DEVICE, 876 - }; 877 - size_t payload_len = umin(iov_iter_count(iter), 878 - sp->max_send_size - sizeof(*packet)); 879 - 880 - ret = smbdirect_map_sges_from_iter(iter, payload_len, &extract); 881 - if (ret < 0) 882 - goto err; 883 - data_length = ret; 884 - remaining_data_length -= data_length; 885 - msg->num_sge = extract.num_sge; 886 - } 887 - 888 - /* Fill in the packet header */ 889 - packet = (struct smbdirect_data_transfer *)msg->packet; 890 - packet->credits_requested = cpu_to_le16(sp->send_credit_target); 891 - new_credits = smbdirect_connection_grant_recv_credits(sc); 892 - packet->credits_granted = cpu_to_le16(new_credits); 893 - 894 - packet->flags = 0; 895 - if (smbdirect_connection_request_keep_alive(sc)) 896 - packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED); 897 - 898 - packet->reserved = 0; 899 - if (!data_length) 900 - packet->data_offset = 0; 901 - else 902 - packet->data_offset = cpu_to_le32(24); 903 - packet->data_length = cpu_to_le32(data_length); 904 - packet->remaining_data_length = cpu_to_le32(remaining_data_length); 905 - packet->padding = 0; 906 - 907 - ksmbd_debug(RDMA, 908 - "credits_req=%u credits_granted=%u flags=0x%x ofs=%u len=%u remaining=%u\n", 909 - le16_to_cpu(packet->credits_requested), 910 - le16_to_cpu(packet->credits_granted), 911 - le16_to_cpu(packet->flags), 912 - le32_to_cpu(packet->data_offset), 913 - le32_to_cpu(packet->data_length), 914 - le32_to_cpu(packet->remaining_data_length)); 915 - 916 - ret = post_sendmsg(sc, send_ctx, msg); 917 - if (ret) 918 - goto err; 919 - 920 - /* 921 - * From here msg is moved to send_ctx 922 - * and we should not free it explicitly. 923 - */ 924 - 925 - if (send_ctx == &_send_ctx) { 926 - ret = smb_direct_flush_send_list(sc, send_ctx, true); 927 - if (ret) 928 - goto flush_failed; 929 - } 930 - 931 - return data_length; 932 - err: 933 - smbdirect_connection_free_send_io(msg); 934 - flush_failed: 935 - alloc_failed: 936 - atomic_inc(&sc->send_io.credits.count); 937 - credit_failed: 938 - atomic_inc(&sc->send_io.lcredits.count); 939 - lcredit_failed: 940 - atomic_add(send_ctx->credit, &sc->send_io.bcredits.count); 941 - send_ctx->credit = 0; 942 - bcredit_failed: 943 - return ret; 944 - } 945 - 946 - static int smb_direct_send_iter(struct smbdirect_socket *sc, 947 - struct iov_iter *iter, 948 - bool need_invalidate, 949 - unsigned int remote_key) 950 - { 951 - struct smbdirect_socket_parameters *sp = &sc->parameters; 952 - int ret; 953 - struct smbdirect_send_batch send_ctx; 954 - int error = 0; 955 - __be32 hdr; 956 - 957 - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 958 - return -ENOTCONN; 959 - 960 - /* 961 - * For now we expect the iter to have the full 962 - * message, including a 4 byte length header. 963 - */ 964 - if (iov_iter_count(iter) <= 4) 965 - return -EINVAL; 966 - if (!copy_from_iter_full(&hdr, sizeof(hdr), iter)) 967 - return -EFAULT; 968 - if (iov_iter_count(iter) != be32_to_cpu(hdr)) 969 - return -EINVAL; 970 - 971 - /* 972 - * The size must fit into the negotiated 973 - * fragmented send size. 974 - */ 975 - if (iov_iter_count(iter) > sp->max_fragmented_send_size) 976 - return -EMSGSIZE; 977 - 978 - ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%zu\n", 979 - iov_iter_count(iter)); 980 - 981 - smb_direct_send_ctx_init(&send_ctx, need_invalidate, remote_key); 982 - while (iov_iter_count(iter)) { 983 - ret = smb_direct_post_send_data(sc, 984 - &send_ctx, 985 - iter, 986 - iov_iter_count(iter)); 987 - if (unlikely(ret < 0)) { 988 - error = ret; 989 - break; 990 - } 991 - } 992 - 993 - ret = smb_direct_flush_send_list(sc, &send_ctx, true); 994 - if (unlikely(!ret && error)) 995 - ret = error; 996 - 997 - /* 998 - * As an optimization, we don't wait for individual I/O to finish 999 - * before sending the next one. 1000 - * Send them all and wait for pending send count to get to 0 1001 - * that means all the I/Os have been out and we are good to return 1002 - */ 1003 - 1004 - wait_event(sc->send_io.pending.zero_wait_queue, 1005 - atomic_read(&sc->send_io.pending.count) == 0 || 1006 - sc->status != SMBDIRECT_SOCKET_CONNECTED); 1007 - if (sc->status != SMBDIRECT_SOCKET_CONNECTED && ret == 0) 1008 - ret = -ENOTCONN; 1009 - 1010 - return ret; 1011 - } 1012 - 1013 621 static int smb_direct_writev(struct ksmbd_transport *t, 1014 622 struct kvec *iov, int niovs, int buflen, 1015 623 bool need_invalidate, unsigned int remote_key) ··· 612 1036 613 1037 iov_iter_kvec(&iter, ITER_SOURCE, iov, niovs, buflen); 614 1038 615 - return smb_direct_send_iter(sc, &iter, need_invalidate, remote_key); 1039 + return smbdirect_connection_send_iter(sc, &iter, 0, 1040 + need_invalidate, remote_key); 616 1041 } 617 1042 618 1043 static int smb_direct_rdma_write(struct ksmbd_transport *t, ··· 771 1194 sendmsg->sge[0].length = sizeof(*resp); 772 1195 sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey; 773 1196 774 - ret = post_sendmsg(sc, NULL, sendmsg); 1197 + ret = smbdirect_connection_post_send_io(sc, NULL, sendmsg); 775 1198 if (ret) { 776 1199 smbdirect_connection_free_send_io(sendmsg); 777 1200 return ret; ··· 1012 1435 return ret; 1013 1436 1014 1437 INIT_WORK(&sc->recv_io.posted.refill_work, smbdirect_connection_recv_io_refill_work); 1015 - INIT_WORK(&sc->idle.immediate_work, smb_direct_send_immediate_work); 1438 + INIT_WORK(&sc->idle.immediate_work, smbdirect_connection_send_immediate_work); 1016 1439 1017 1440 return 0; 1018 1441 }

Configure Feed

Configure Feed