+22

Documentation/scsi/tcm_qla2xxx.txt

reviewed

··· 1 1 + tcm_qla2xxx jam_host attribute 2 2 + ------------------------------ 3 3 + There is now a new module endpoint atribute called jam_host 4 4 + attribute: jam_host: boolean=0/1 5 5 + This attribute and accompanying code is only included if the 6 6 + Kconfig parameter TCM_QLA2XXX_DEBUG is set to Y 7 7 + By default this jammer code and functionality is disabled 8 8 + 9 9 + Use this attribute to control the discarding of SCSI commands to a 10 10 + selected host. 11 11 + This may be useful for testing error handling and simulating slow drain 12 12 + and other fabric issues. 13 13 + 14 14 + Setting a boolean of 1 for the jam_host attribute for a particular host 15 15 + will discard the commands for that host. 16 16 + Reset back to 0 to stop the jamming. 17 17 + 18 18 + Enable host 4 to be jammed 19 19 + echo 1 > /sys/kernel/config/target/qla2xxx/21:00:00:24:ff:27:8f:ae/tpgt_1/attrib/jam_host 20 20 + 21 21 + Disable jamming on host 4 22 22 + echo 0 > /sys/kernel/config/target/qla2xxx/21:00:00:24:ff:27:8f:ae/tpgt_1/attrib/jam_host

-16

Documentation/target/tcm_mod_builder.py

reviewed

··· 294 294 buf += " .tpg_check_prod_mode_write_protect = " + fabric_mod_name + "_check_false,\n" 295 295 buf += " .tpg_get_inst_index = " + fabric_mod_name + "_tpg_get_inst_index,\n" 296 296 buf += " .release_cmd = " + fabric_mod_name + "_release_cmd,\n" 297 297 - buf += " .shutdown_session = " + fabric_mod_name + "_shutdown_session,\n" 298 298 - buf += " .close_session = " + fabric_mod_name + "_close_session,\n" 299 297 buf += " .sess_get_index = " + fabric_mod_name + "_sess_get_index,\n" 300 298 buf += " .sess_get_initiator_sid = NULL,\n" 301 299 buf += " .write_pending = " + fabric_mod_name + "_write_pending,\n" ··· 464 466 buf += " return;\n" 465 467 buf += "}\n\n" 466 468 bufi += "void " + fabric_mod_name + "_release_cmd(struct se_cmd *);\n" 467 467 - 468 468 - if re.search('shutdown_session\)\(', fo): 469 469 - buf += "int " + fabric_mod_name + "_shutdown_session(struct se_session *se_sess)\n" 470 470 - buf += "{\n" 471 471 - buf += " return 0;\n" 472 472 - buf += "}\n\n" 473 473 - bufi += "int " + fabric_mod_name + "_shutdown_session(struct se_session *);\n" 474 474 - 475 475 - if re.search('close_session\)\(', fo): 476 476 - buf += "void " + fabric_mod_name + "_close_session(struct se_session *se_sess)\n" 477 477 - buf += "{\n" 478 478 - buf += " return;\n" 479 479 - buf += "}\n\n" 480 480 - bufi += "void " + fabric_mod_name + "_close_session(struct se_session *);\n" 481 469 482 470 if re.search('sess_get_index\)\(', fo): 483 471 buf += "u32 " + fabric_mod_name + "_sess_get_index(struct se_session *se_sess)\n"

+11

drivers/infiniband/ulp/isert/ib_isert.c

reviewed

··· 2596 2596 isert_put_conn(isert_conn); 2597 2597 } 2598 2598 2599 2599 + static void isert_get_rx_pdu(struct iscsi_conn *conn) 2600 2600 + { 2601 2601 + struct completion comp; 2602 2602 + 2603 2603 + init_completion(&comp); 2604 2604 + 2605 2605 + wait_for_completion_interruptible(&comp); 2606 2606 + } 2607 2607 + 2599 2608 static struct iscsit_transport iser_target_transport = { 2600 2609 .name = "IB/iSER", 2601 2610 .transport_type = ISCSI_INFINIBAND, 2611 2611 + .rdma_shutdown = true, 2602 2612 .priv_size = sizeof(struct isert_cmd), 2603 2613 .owner = THIS_MODULE, 2604 2614 .iscsit_setup_np = isert_setup_np, ··· 2624 2614 .iscsit_queue_data_in = isert_put_datain, 2625 2615 .iscsit_queue_status = isert_put_response, 2626 2616 .iscsit_aborted_task = isert_aborted_task, 2617 2617 + .iscsit_get_rx_pdu = isert_get_rx_pdu, 2627 2618 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2628 2619 }; 2629 2620

-9

drivers/infiniband/ulp/srpt/ib_srpt.c

reviewed

··· 1767 1767 } 1768 1768 } 1769 1769 1770 1770 - /** 1771 1771 - * srpt_shutdown_session() - Whether or not a session may be shut down. 1772 1772 - */ 1773 1773 - static int srpt_shutdown_session(struct se_session *se_sess) 1774 1774 - { 1775 1775 - return 1; 1776 1776 - } 1777 1777 - 1778 1770 static void srpt_free_ch(struct kref *kref) 1779 1771 { 1780 1772 struct srpt_rdma_ch *ch = container_of(kref, struct srpt_rdma_ch, kref); ··· 3056 3064 .tpg_get_inst_index = srpt_tpg_get_inst_index, 3057 3065 .release_cmd = srpt_release_cmd, 3058 3066 .check_stop_free = srpt_check_stop_free, 3059 3059 - .shutdown_session = srpt_shutdown_session, 3060 3067 .close_session = srpt_close_session, 3061 3068 .sess_get_index = srpt_sess_get_index, 3062 3069 .sess_get_initiator_sid = NULL,

+9

drivers/scsi/qla2xxx/Kconfig

reviewed

··· 36 36 default n 37 37 ---help--- 38 38 Say Y here to enable the TCM_QLA2XXX fabric module for QLogic 24xx+ series target mode HBAs 39 39 + 40 40 + if TCM_QLA2XXX 41 41 + config TCM_QLA2XXX_DEBUG 42 42 + bool "TCM_QLA2XXX fabric module DEBUG mode for QLogic 24xx+ series target mode HBAs" 43 43 + default n 44 44 + ---help--- 45 45 + Say Y here to enable the TCM_QLA2XXX fabric module DEBUG for QLogic 24xx+ series target mode HBAs 46 46 + This will include code to enable the SCSI command jammer 47 47 + endif

+31 -25

drivers/scsi/qla2xxx/qla_target.c

reviewed

··· 637 637 } 638 638 639 639 /* ha->tgt.sess_lock supposed to be held on entry */ 640 640 - void qlt_unreg_sess(struct qla_tgt_sess *sess) 640 640 + static void qlt_release_session(struct kref *kref) 641 641 { 642 642 + struct qla_tgt_sess *sess = 643 643 + container_of(kref, struct qla_tgt_sess, sess_kref); 642 644 struct scsi_qla_host *vha = sess->vha; 643 645 644 646 if (sess->se_sess) ··· 653 651 INIT_WORK(&sess->free_work, qlt_free_session_done); 654 652 schedule_work(&sess->free_work); 655 653 } 656 656 - EXPORT_SYMBOL(qlt_unreg_sess); 657 654 655 655 + void qlt_put_sess(struct qla_tgt_sess *sess) 656 656 + { 657 657 + if (!sess) 658 658 + return; 659 659 + 660 660 + assert_spin_locked(&sess->vha->hw->tgt.sess_lock); 661 661 + kref_put(&sess->sess_kref, qlt_release_session); 662 662 + } 663 663 + EXPORT_SYMBOL(qlt_put_sess); 658 664 659 665 static int qlt_reset(struct scsi_qla_host *vha, void *iocb, int mcmd) 660 666 { ··· 867 857 ql_dbg(ql_dbg_tgt_mgt, vha, 0xf004, 868 858 "Timeout: sess %p about to be deleted\n", 869 859 sess); 870 870 - if (sess->se_sess) { 860 860 + if (sess->se_sess) 871 861 ha->tgt.tgt_ops->shutdown_sess(sess); 872 872 - ha->tgt.tgt_ops->put_sess(sess); 873 873 - } else { 874 874 - qlt_unreg_sess(sess); 875 875 - } 862 862 + qlt_put_sess(sess); 876 863 } else { 877 864 schedule_delayed_work(&tgt->sess_del_work, 878 865 sess->expires - elapsed); ··· 924 917 } 925 918 } 926 919 927 927 - kref_get(&sess->se_sess->sess_kref); 920 920 + kref_get(&sess->sess_kref); 928 921 ha->tgt.tgt_ops->update_sess(sess, fcport->d_id, fcport->loop_id, 929 922 (fcport->flags & FCF_CONF_COMP_SUPPORTED)); 930 923 ··· 954 947 sess->s_id = fcport->d_id; 955 948 sess->loop_id = fcport->loop_id; 956 949 sess->local = local; 950 950 + kref_init(&sess->sess_kref); 957 951 INIT_LIST_HEAD(&sess->del_list_entry); 958 952 959 953 /* Under normal circumstances we want to logout from firmware when ··· 999 991 * Take an extra reference to ->sess_kref here to handle qla_tgt_sess 1000 992 * access across ->tgt.sess_lock reaquire. 1001 993 */ 1002 1002 - kref_get(&sess->se_sess->sess_kref); 994 994 + kref_get(&sess->sess_kref); 1003 995 } 1004 996 1005 997 return sess; ··· 1043 1035 spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); 1044 1036 return; 1045 1037 } else { 1046 1046 - kref_get(&sess->se_sess->sess_kref); 1038 1038 + kref_get(&sess->sess_kref); 1047 1039 1048 1040 if (sess->deleted) { 1049 1041 qlt_undelete_sess(sess); ··· 1068 1060 fcport->port_name, sess->loop_id); 1069 1061 sess->local = 0; 1070 1062 } 1071 1071 - ha->tgt.tgt_ops->put_sess(sess); 1063 1063 + qlt_put_sess(sess); 1072 1064 spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); 1073 1065 } 1074 1066 ··· 3825 3817 * Drop extra session reference from qla_tgt_handle_cmd_for_atio*( 3826 3818 */ 3827 3819 spin_lock_irqsave(&ha->tgt.sess_lock, flags); 3828 3828 - ha->tgt.tgt_ops->put_sess(sess); 3820 3820 + qlt_put_sess(sess); 3829 3821 spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); 3830 3822 return; 3831 3823 ··· 3844 3836 spin_unlock_irqrestore(&ha->hardware_lock, flags); 3845 3837 3846 3838 spin_lock_irqsave(&ha->tgt.sess_lock, flags); 3847 3847 - ha->tgt.tgt_ops->put_sess(sess); 3839 3839 + qlt_put_sess(sess); 3848 3840 spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); 3849 3841 } 3850 3842 ··· 3944 3936 if (!cmd) { 3945 3937 spin_lock_irqsave(&ha->hardware_lock, flags); 3946 3938 qlt_send_busy(vha, &op->atio, SAM_STAT_BUSY); 3947 3947 - ha->tgt.tgt_ops->put_sess(sess); 3939 3939 + qlt_put_sess(sess); 3948 3940 spin_unlock_irqrestore(&ha->hardware_lock, flags); 3949 3941 kfree(op); 3950 3942 return; 3951 3943 } 3952 3944 /* 3953 3953 - * __qlt_do_work() will call ha->tgt.tgt_ops->put_sess() to release 3945 3945 + * __qlt_do_work() will call qlt_put_sess() to release 3954 3946 * the extra reference taken above by qlt_make_local_sess() 3955 3947 */ 3956 3948 __qlt_do_work(cmd); ··· 4011 4003 /* 4012 4004 * Do kref_get() before returning + dropping qla_hw_data->hardware_lock. 4013 4005 */ 4014 4014 - kref_get(&sess->se_sess->sess_kref); 4006 4006 + kref_get(&sess->sess_kref); 4015 4007 4016 4008 cmd = qlt_get_tag(vha, sess, atio); 4017 4009 if (!cmd) { 4018 4010 ql_dbg(ql_dbg_io, vha, 0x3062, 4019 4011 "qla_target(%d): Allocation of cmd failed\n", vha->vp_idx); 4020 4020 - ha->tgt.tgt_ops->put_sess(sess); 4012 4012 + qlt_put_sess(sess); 4021 4013 return -ENOMEM; 4022 4014 } 4023 4015 ··· 5919 5911 goto out_term2; 5920 5912 } 5921 5913 5922 5922 - kref_get(&sess->se_sess->sess_kref); 5914 5914 + kref_get(&sess->sess_kref); 5923 5915 } 5924 5916 5925 5917 spin_lock_irqsave(&ha->hardware_lock, flags); ··· 5932 5924 goto out_term; 5933 5925 spin_unlock_irqrestore(&ha->hardware_lock, flags); 5934 5926 5935 5935 - ha->tgt.tgt_ops->put_sess(sess); 5927 5927 + qlt_put_sess(sess); 5936 5928 spin_unlock_irqrestore(&ha->tgt.sess_lock, flags2); 5937 5929 return; 5938 5930 ··· 5943 5935 qlt_24xx_send_abts_resp(vha, &prm->abts, FCP_TMF_REJECTED, false); 5944 5936 spin_unlock_irqrestore(&ha->hardware_lock, flags); 5945 5937 5946 5946 - if (sess) 5947 5947 - ha->tgt.tgt_ops->put_sess(sess); 5938 5938 + qlt_put_sess(sess); 5948 5939 spin_unlock_irqrestore(&ha->tgt.sess_lock, flags2); 5949 5940 } 5950 5941 ··· 5983 5976 goto out_term; 5984 5977 } 5985 5978 5986 5986 - kref_get(&sess->se_sess->sess_kref); 5979 5979 + kref_get(&sess->sess_kref); 5987 5980 } 5988 5981 5989 5982 iocb = a; ··· 5995 5988 if (rc != 0) 5996 5989 goto out_term; 5997 5990 5998 5998 - ha->tgt.tgt_ops->put_sess(sess); 5991 5991 + qlt_put_sess(sess); 5999 5992 spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); 6000 5993 return; 6001 5994 6002 5995 out_term: 6003 5996 qlt_send_term_exchange(vha, NULL, &prm->tm_iocb2, 1, 0); 6004 6004 - if (sess) 6005 6005 - ha->tgt.tgt_ops->put_sess(sess); 5997 5997 + qlt_put_sess(sess); 6006 5998 spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); 6007 5999 } 6008 6000

+2 -2

drivers/scsi/qla2xxx/qla_target.h

reviewed

··· 738 738 struct qla_tgt_sess *(*find_sess_by_s_id)(struct scsi_qla_host *, 739 739 const uint8_t *); 740 740 void (*clear_nacl_from_fcport_map)(struct qla_tgt_sess *); 741 741 - void (*put_sess)(struct qla_tgt_sess *); 742 741 void (*shutdown_sess)(struct qla_tgt_sess *); 743 742 }; 744 743 ··· 929 930 int generation; 930 931 931 932 struct se_session *se_sess; 933 933 + struct kref sess_kref; 932 934 struct scsi_qla_host *vha; 933 935 struct qla_tgt *tgt; 934 936 ··· 1101 1101 extern int qlt_lport_register(void *, u64, u64, u64, 1102 1102 int (*callback)(struct scsi_qla_host *, void *, u64, u64)); 1103 1103 extern void qlt_lport_deregister(struct scsi_qla_host *); 1104 1104 - extern void qlt_unreg_sess(struct qla_tgt_sess *); 1104 1104 + void qlt_put_sess(struct qla_tgt_sess *sess); 1105 1105 extern void qlt_fc_port_added(struct scsi_qla_host *, fc_port_t *); 1106 1106 extern void qlt_fc_port_deleted(struct scsi_qla_host *, fc_port_t *, int); 1107 1107 extern int __init qlt_init(void);

+22 -37

drivers/scsi/qla2xxx/tcm_qla2xxx.c

reviewed

··· 339 339 qlt_free_cmd(cmd); 340 340 } 341 341 342 342 - static int tcm_qla2xxx_shutdown_session(struct se_session *se_sess) 343 343 - { 344 344 - struct qla_tgt_sess *sess = se_sess->fabric_sess_ptr; 345 345 - struct scsi_qla_host *vha; 346 346 - unsigned long flags; 347 347 - 348 348 - BUG_ON(!sess); 349 349 - vha = sess->vha; 350 350 - 351 351 - spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags); 352 352 - target_sess_cmd_list_set_waiting(se_sess); 353 353 - spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags); 354 354 - 355 355 - return 1; 356 356 - } 357 357 - 358 342 static void tcm_qla2xxx_close_session(struct se_session *se_sess) 359 343 { 360 344 struct qla_tgt_sess *sess = se_sess->fabric_sess_ptr; ··· 349 365 vha = sess->vha; 350 366 351 367 spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags); 352 352 - qlt_unreg_sess(sess); 368 368 + target_sess_cmd_list_set_waiting(se_sess); 369 369 + qlt_put_sess(sess); 353 370 spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags); 354 371 } 355 372 ··· 442 457 struct se_cmd *se_cmd = &cmd->se_cmd; 443 458 struct se_session *se_sess; 444 459 struct qla_tgt_sess *sess; 460 460 + #ifdef CONFIG_TCM_QLA2XXX_DEBUG 461 461 + struct se_portal_group *se_tpg; 462 462 + struct tcm_qla2xxx_tpg *tpg; 463 463 + #endif 445 464 int flags = TARGET_SCF_ACK_KREF; 446 465 447 466 if (bidi) ··· 465 476 pr_err("Unable to locate active struct se_session\n"); 466 477 return -EINVAL; 467 478 } 479 479 + 480 480 + #ifdef CONFIG_TCM_QLA2XXX_DEBUG 481 481 + se_tpg = se_sess->se_tpg; 482 482 + tpg = container_of(se_tpg, struct tcm_qla2xxx_tpg, se_tpg); 483 483 + if (unlikely(tpg->tpg_attrib.jam_host)) { 484 484 + /* return, and dont run target_submit_cmd,discarding command */ 485 485 + return 0; 486 486 + } 487 487 + #endif 468 488 469 489 cmd->vha->tgt_counters.qla_core_sbt_cmd++; 470 490 return target_submit_cmd(se_cmd, se_sess, cdb, &cmd->sense_buffer[0], ··· 756 758 tcm_qla2xxx_clear_sess_lookup(lport, nacl, sess); 757 759 } 758 760 759 759 - static void tcm_qla2xxx_release_session(struct kref *kref) 760 760 - { 761 761 - struct se_session *se_sess = container_of(kref, 762 762 - struct se_session, sess_kref); 763 763 - 764 764 - qlt_unreg_sess(se_sess->fabric_sess_ptr); 765 765 - } 766 766 - 767 767 - static void tcm_qla2xxx_put_sess(struct qla_tgt_sess *sess) 768 768 - { 769 769 - if (!sess) 770 770 - return; 771 771 - 772 772 - assert_spin_locked(&sess->vha->hw->tgt.sess_lock); 773 773 - kref_put(&sess->se_sess->sess_kref, tcm_qla2xxx_release_session); 774 774 - } 775 775 - 776 761 static void tcm_qla2xxx_shutdown_sess(struct qla_tgt_sess *sess) 777 762 { 778 763 assert_spin_locked(&sess->vha->hw->tgt.sess_lock); ··· 825 844 DEF_QLA_TPG_ATTRIB(demo_mode_write_protect); 826 845 DEF_QLA_TPG_ATTRIB(prod_mode_write_protect); 827 846 DEF_QLA_TPG_ATTRIB(demo_mode_login_only); 847 847 + #ifdef CONFIG_TCM_QLA2XXX_DEBUG 848 848 + DEF_QLA_TPG_ATTRIB(jam_host); 849 849 + #endif 828 850 829 851 static struct configfs_attribute *tcm_qla2xxx_tpg_attrib_attrs[] = { 830 852 &tcm_qla2xxx_tpg_attrib_attr_generate_node_acls, ··· 835 851 &tcm_qla2xxx_tpg_attrib_attr_demo_mode_write_protect, 836 852 &tcm_qla2xxx_tpg_attrib_attr_prod_mode_write_protect, 837 853 &tcm_qla2xxx_tpg_attrib_attr_demo_mode_login_only, 854 854 + #ifdef CONFIG_TCM_QLA2XXX_DEBUG 855 855 + &tcm_qla2xxx_tpg_attrib_attr_jam_host, 856 856 + #endif 838 857 NULL, 839 858 }; 840 859 ··· 1010 1023 tpg->tpg_attrib.demo_mode_write_protect = 1; 1011 1024 tpg->tpg_attrib.cache_dynamic_acls = 1; 1012 1025 tpg->tpg_attrib.demo_mode_login_only = 1; 1026 1026 + tpg->tpg_attrib.jam_host = 0; 1013 1027 1014 1028 ret = core_tpg_register(wwn, &tpg->se_tpg, SCSI_PROTOCOL_FCP); 1015 1029 if (ret < 0) { ··· 1567 1579 .find_sess_by_s_id = tcm_qla2xxx_find_sess_by_s_id, 1568 1580 .find_sess_by_loop_id = tcm_qla2xxx_find_sess_by_loop_id, 1569 1581 .clear_nacl_from_fcport_map = tcm_qla2xxx_clear_nacl_from_fcport_map, 1570 1570 - .put_sess = tcm_qla2xxx_put_sess, 1571 1582 .shutdown_sess = tcm_qla2xxx_shutdown_sess, 1572 1583 }; 1573 1584 ··· 1834 1847 .tpg_get_inst_index = tcm_qla2xxx_tpg_get_inst_index, 1835 1848 .check_stop_free = tcm_qla2xxx_check_stop_free, 1836 1849 .release_cmd = tcm_qla2xxx_release_cmd, 1837 1837 - .shutdown_session = tcm_qla2xxx_shutdown_session, 1838 1850 .close_session = tcm_qla2xxx_close_session, 1839 1851 .sess_get_index = tcm_qla2xxx_sess_get_index, 1840 1852 .sess_get_initiator_sid = NULL, ··· 1876 1890 .tpg_get_inst_index = tcm_qla2xxx_tpg_get_inst_index, 1877 1891 .check_stop_free = tcm_qla2xxx_check_stop_free, 1878 1892 .release_cmd = tcm_qla2xxx_release_cmd, 1879 1879 - .shutdown_session = tcm_qla2xxx_shutdown_session, 1880 1893 .close_session = tcm_qla2xxx_close_session, 1881 1894 .sess_get_index = tcm_qla2xxx_sess_get_index, 1882 1895 .sess_get_initiator_sid = NULL,

+1

drivers/scsi/qla2xxx/tcm_qla2xxx.h

reviewed

··· 34 34 int prod_mode_write_protect; 35 35 int demo_mode_login_only; 36 36 int fabric_prot_type; 37 37 + int jam_host; 37 38 }; 38 39 39 40 struct tcm_qla2xxx_tpg {

+2

drivers/target/iscsi/Kconfig

reviewed

··· 7 7 help 8 8 Say M here to enable the ConfigFS enabled Linux-iSCSI.org iSCSI 9 9 Target Mode Stack. 10 10 + 11 11 + source "drivers/target/iscsi/cxgbit/Kconfig"

+1

drivers/target/iscsi/Makefile

reviewed

··· 18 18 iscsi_target_transport.o 19 19 20 20 obj-$(CONFIG_ISCSI_TARGET) += iscsi_target_mod.o 21 21 + obj-$(CONFIG_ISCSI_TARGET_CXGB4) += cxgbit/

+7

drivers/target/iscsi/cxgbit/Kconfig

reviewed

··· 1 1 + config ISCSI_TARGET_CXGB4 2 2 + tristate "Chelsio iSCSI target offload driver" 3 3 + depends on ISCSI_TARGET && CHELSIO_T4 && INET 4 4 + select CHELSIO_T4_UWIRE 5 5 + ---help--- 6 6 + To compile this driver as module, choose M here: the module 7 7 + will be called cxgbit.

+6

drivers/target/iscsi/cxgbit/Makefile

reviewed

··· 1 1 + ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 2 2 + ccflags-y += -Idrivers/target/iscsi 3 3 + 4 4 + obj-$(CONFIG_ISCSI_TARGET_CXGB4) += cxgbit.o 5 5 + 6 6 + cxgbit-y := cxgbit_main.o cxgbit_cm.o cxgbit_target.o cxgbit_ddp.o

+353

drivers/target/iscsi/cxgbit/cxgbit.h

reviewed

··· 1 1 + /* 2 2 + * Copyright (c) 2016 Chelsio Communications, Inc. 3 3 + * 4 4 + * This program is free software; you can redistribute it and/or modify 5 5 + * it under the terms of the GNU General Public License version 2 as 6 6 + * published by the Free Software Foundation. 7 7 + */ 8 8 + 9 9 + #ifndef __CXGBIT_H__ 10 10 + #define __CXGBIT_H__ 11 11 + 12 12 + #include <linux/mutex.h> 13 13 + #include <linux/list.h> 14 14 + #include <linux/spinlock.h> 15 15 + #include <linux/idr.h> 16 16 + #include <linux/completion.h> 17 17 + #include <linux/netdevice.h> 18 18 + #include <linux/sched.h> 19 19 + #include <linux/pci.h> 20 20 + #include <linux/dma-mapping.h> 21 21 + #include <linux/inet.h> 22 22 + #include <linux/wait.h> 23 23 + #include <linux/kref.h> 24 24 + #include <linux/timer.h> 25 25 + #include <linux/io.h> 26 26 + 27 27 + #include <asm/byteorder.h> 28 28 + 29 29 + #include <net/net_namespace.h> 30 30 + 31 31 + #include <target/iscsi/iscsi_transport.h> 32 32 + #include <iscsi_target_parameters.h> 33 33 + #include <iscsi_target_login.h> 34 34 + 35 35 + #include "t4_regs.h" 36 36 + #include "t4_msg.h" 37 37 + #include "cxgb4.h" 38 38 + #include "cxgb4_uld.h" 39 39 + #include "l2t.h" 40 40 + #include "cxgb4_ppm.h" 41 41 + #include "cxgbit_lro.h" 42 42 + 43 43 + extern struct mutex cdev_list_lock; 44 44 + extern struct list_head cdev_list_head; 45 45 + struct cxgbit_np; 46 46 + 47 47 + struct cxgbit_sock; 48 48 + 49 49 + struct cxgbit_cmd { 50 50 + struct scatterlist sg; 51 51 + struct cxgbi_task_tag_info ttinfo; 52 52 + bool setup_ddp; 53 53 + bool release; 54 54 + }; 55 55 + 56 56 + #define CXGBIT_MAX_ISO_PAYLOAD \ 57 57 + min_t(u32, MAX_SKB_FRAGS * PAGE_SIZE, 65535) 58 58 + 59 59 + struct cxgbit_iso_info { 60 60 + u8 flags; 61 61 + u32 mpdu; 62 62 + u32 len; 63 63 + u32 burst_len; 64 64 + }; 65 65 + 66 66 + enum cxgbit_skcb_flags { 67 67 + SKCBF_TX_NEED_HDR = (1 << 0), /* packet needs a header */ 68 68 + SKCBF_TX_FLAG_COMPL = (1 << 1), /* wr completion flag */ 69 69 + SKCBF_TX_ISO = (1 << 2), /* iso cpl in tx skb */ 70 70 + SKCBF_RX_LRO = (1 << 3), /* lro skb */ 71 71 + }; 72 72 + 73 73 + struct cxgbit_skb_rx_cb { 74 74 + u8 opcode; 75 75 + void *pdu_cb; 76 76 + void (*backlog_fn)(struct cxgbit_sock *, struct sk_buff *); 77 77 + }; 78 78 + 79 79 + struct cxgbit_skb_tx_cb { 80 80 + u8 submode; 81 81 + u32 extra_len; 82 82 + }; 83 83 + 84 84 + union cxgbit_skb_cb { 85 85 + struct { 86 86 + u8 flags; 87 87 + union { 88 88 + struct cxgbit_skb_tx_cb tx; 89 89 + struct cxgbit_skb_rx_cb rx; 90 90 + }; 91 91 + }; 92 92 + 93 93 + struct { 94 94 + /* This member must be first. */ 95 95 + struct l2t_skb_cb l2t; 96 96 + struct sk_buff *wr_next; 97 97 + }; 98 98 + }; 99 99 + 100 100 + #define CXGBIT_SKB_CB(skb) ((union cxgbit_skb_cb *)&((skb)->cb[0])) 101 101 + #define cxgbit_skcb_flags(skb) (CXGBIT_SKB_CB(skb)->flags) 102 102 + #define cxgbit_skcb_submode(skb) (CXGBIT_SKB_CB(skb)->tx.submode) 103 103 + #define cxgbit_skcb_tx_wr_next(skb) (CXGBIT_SKB_CB(skb)->wr_next) 104 104 + #define cxgbit_skcb_tx_extralen(skb) (CXGBIT_SKB_CB(skb)->tx.extra_len) 105 105 + #define cxgbit_skcb_rx_opcode(skb) (CXGBIT_SKB_CB(skb)->rx.opcode) 106 106 + #define cxgbit_skcb_rx_backlog_fn(skb) (CXGBIT_SKB_CB(skb)->rx.backlog_fn) 107 107 + #define cxgbit_rx_pdu_cb(skb) (CXGBIT_SKB_CB(skb)->rx.pdu_cb) 108 108 + 109 109 + static inline void *cplhdr(struct sk_buff *skb) 110 110 + { 111 111 + return skb->data; 112 112 + } 113 113 + 114 114 + enum cxgbit_cdev_flags { 115 115 + CDEV_STATE_UP = 0, 116 116 + CDEV_ISO_ENABLE, 117 117 + CDEV_DDP_ENABLE, 118 118 + }; 119 119 + 120 120 + #define NP_INFO_HASH_SIZE 32 121 121 + 122 122 + struct np_info { 123 123 + struct np_info *next; 124 124 + struct cxgbit_np *cnp; 125 125 + unsigned int stid; 126 126 + }; 127 127 + 128 128 + struct cxgbit_list_head { 129 129 + struct list_head list; 130 130 + /* device lock */ 131 131 + spinlock_t lock; 132 132 + }; 133 133 + 134 134 + struct cxgbit_device { 135 135 + struct list_head list; 136 136 + struct cxgb4_lld_info lldi; 137 137 + struct np_info *np_hash_tab[NP_INFO_HASH_SIZE]; 138 138 + /* np lock */ 139 139 + spinlock_t np_lock; 140 140 + u8 selectq[MAX_NPORTS][2]; 141 141 + struct cxgbit_list_head cskq; 142 142 + u32 mdsl; 143 143 + struct kref kref; 144 144 + unsigned long flags; 145 145 + }; 146 146 + 147 147 + struct cxgbit_wr_wait { 148 148 + struct completion completion; 149 149 + int ret; 150 150 + }; 151 151 + 152 152 + enum cxgbit_csk_state { 153 153 + CSK_STATE_IDLE = 0, 154 154 + CSK_STATE_LISTEN, 155 155 + CSK_STATE_CONNECTING, 156 156 + CSK_STATE_ESTABLISHED, 157 157 + CSK_STATE_ABORTING, 158 158 + CSK_STATE_CLOSING, 159 159 + CSK_STATE_MORIBUND, 160 160 + CSK_STATE_DEAD, 161 161 + }; 162 162 + 163 163 + enum cxgbit_csk_flags { 164 164 + CSK_TX_DATA_SENT = 0, 165 165 + CSK_LOGIN_PDU_DONE, 166 166 + CSK_LOGIN_DONE, 167 167 + CSK_DDP_ENABLE, 168 168 + }; 169 169 + 170 170 + struct cxgbit_sock_common { 171 171 + struct cxgbit_device *cdev; 172 172 + struct sockaddr_storage local_addr; 173 173 + struct sockaddr_storage remote_addr; 174 174 + struct cxgbit_wr_wait wr_wait; 175 175 + enum cxgbit_csk_state state; 176 176 + unsigned long flags; 177 177 + }; 178 178 + 179 179 + struct cxgbit_np { 180 180 + struct cxgbit_sock_common com; 181 181 + wait_queue_head_t accept_wait; 182 182 + struct iscsi_np *np; 183 183 + struct completion accept_comp; 184 184 + struct list_head np_accept_list; 185 185 + /* np accept lock */ 186 186 + spinlock_t np_accept_lock; 187 187 + struct kref kref; 188 188 + unsigned int stid; 189 189 + }; 190 190 + 191 191 + struct cxgbit_sock { 192 192 + struct cxgbit_sock_common com; 193 193 + struct cxgbit_np *cnp; 194 194 + struct iscsi_conn *conn; 195 195 + struct l2t_entry *l2t; 196 196 + struct dst_entry *dst; 197 197 + struct list_head list; 198 198 + struct sk_buff_head rxq; 199 199 + struct sk_buff_head txq; 200 200 + struct sk_buff_head ppodq; 201 201 + struct sk_buff_head backlogq; 202 202 + struct sk_buff_head skbq; 203 203 + struct sk_buff *wr_pending_head; 204 204 + struct sk_buff *wr_pending_tail; 205 205 + struct sk_buff *skb; 206 206 + struct sk_buff *lro_skb; 207 207 + struct sk_buff *lro_hskb; 208 208 + struct list_head accept_node; 209 209 + /* socket lock */ 210 210 + spinlock_t lock; 211 211 + wait_queue_head_t waitq; 212 212 + wait_queue_head_t ack_waitq; 213 213 + bool lock_owner; 214 214 + struct kref kref; 215 215 + u32 max_iso_npdu; 216 216 + u32 wr_cred; 217 217 + u32 wr_una_cred; 218 218 + u32 wr_max_cred; 219 219 + u32 snd_una; 220 220 + u32 tid; 221 221 + u32 snd_nxt; 222 222 + u32 rcv_nxt; 223 223 + u32 smac_idx; 224 224 + u32 tx_chan; 225 225 + u32 mtu; 226 226 + u32 write_seq; 227 227 + u32 rx_credits; 228 228 + u32 snd_win; 229 229 + u32 rcv_win; 230 230 + u16 mss; 231 231 + u16 emss; 232 232 + u16 plen; 233 233 + u16 rss_qid; 234 234 + u16 txq_idx; 235 235 + u16 ctrlq_idx; 236 236 + u8 tos; 237 237 + u8 port_id; 238 238 + #define CXGBIT_SUBMODE_HCRC 0x1 239 239 + #define CXGBIT_SUBMODE_DCRC 0x2 240 240 + u8 submode; 241 241 + #ifdef CONFIG_CHELSIO_T4_DCB 242 242 + u8 dcb_priority; 243 243 + #endif 244 244 + u8 snd_wscale; 245 245 + }; 246 246 + 247 247 + void _cxgbit_free_cdev(struct kref *kref); 248 248 + void _cxgbit_free_csk(struct kref *kref); 249 249 + void _cxgbit_free_cnp(struct kref *kref); 250 250 + 251 251 + static inline void cxgbit_get_cdev(struct cxgbit_device *cdev) 252 252 + { 253 253 + kref_get(&cdev->kref); 254 254 + } 255 255 + 256 256 + static inline void cxgbit_put_cdev(struct cxgbit_device *cdev) 257 257 + { 258 258 + kref_put(&cdev->kref, _cxgbit_free_cdev); 259 259 + } 260 260 + 261 261 + static inline void cxgbit_get_csk(struct cxgbit_sock *csk) 262 262 + { 263 263 + kref_get(&csk->kref); 264 264 + } 265 265 + 266 266 + static inline void cxgbit_put_csk(struct cxgbit_sock *csk) 267 267 + { 268 268 + kref_put(&csk->kref, _cxgbit_free_csk); 269 269 + } 270 270 + 271 271 + static inline void cxgbit_get_cnp(struct cxgbit_np *cnp) 272 272 + { 273 273 + kref_get(&cnp->kref); 274 274 + } 275 275 + 276 276 + static inline void cxgbit_put_cnp(struct cxgbit_np *cnp) 277 277 + { 278 278 + kref_put(&cnp->kref, _cxgbit_free_cnp); 279 279 + } 280 280 + 281 281 + static inline void cxgbit_sock_reset_wr_list(struct cxgbit_sock *csk) 282 282 + { 283 283 + csk->wr_pending_tail = NULL; 284 284 + csk->wr_pending_head = NULL; 285 285 + } 286 286 + 287 287 + static inline struct sk_buff *cxgbit_sock_peek_wr(const struct cxgbit_sock *csk) 288 288 + { 289 289 + return csk->wr_pending_head; 290 290 + } 291 291 + 292 292 + static inline void 293 293 + cxgbit_sock_enqueue_wr(struct cxgbit_sock *csk, struct sk_buff *skb) 294 294 + { 295 295 + cxgbit_skcb_tx_wr_next(skb) = NULL; 296 296 + 297 297 + skb_get(skb); 298 298 + 299 299 + if (!csk->wr_pending_head) 300 300 + csk->wr_pending_head = skb; 301 301 + else 302 302 + cxgbit_skcb_tx_wr_next(csk->wr_pending_tail) = skb; 303 303 + csk->wr_pending_tail = skb; 304 304 + } 305 305 + 306 306 + static inline struct sk_buff *cxgbit_sock_dequeue_wr(struct cxgbit_sock *csk) 307 307 + { 308 308 + struct sk_buff *skb = csk->wr_pending_head; 309 309 + 310 310 + if (likely(skb)) { 311 311 + csk->wr_pending_head = cxgbit_skcb_tx_wr_next(skb); 312 312 + cxgbit_skcb_tx_wr_next(skb) = NULL; 313 313 + } 314 314 + return skb; 315 315 + } 316 316 + 317 317 + typedef void (*cxgbit_cplhandler_func)(struct cxgbit_device *, 318 318 + struct sk_buff *); 319 319 + 320 320 + int cxgbit_setup_np(struct iscsi_np *, struct sockaddr_storage *); 321 321 + int cxgbit_setup_conn_digest(struct cxgbit_sock *); 322 322 + int cxgbit_accept_np(struct iscsi_np *, struct iscsi_conn *); 323 323 + void cxgbit_free_np(struct iscsi_np *); 324 324 + void cxgbit_free_conn(struct iscsi_conn *); 325 325 + extern cxgbit_cplhandler_func cxgbit_cplhandlers[NUM_CPL_CMDS]; 326 326 + int cxgbit_get_login_rx(struct iscsi_conn *, struct iscsi_login *); 327 327 + int cxgbit_rx_data_ack(struct cxgbit_sock *); 328 328 + int cxgbit_l2t_send(struct cxgbit_device *, struct sk_buff *, 329 329 + struct l2t_entry *); 330 330 + void cxgbit_push_tx_frames(struct cxgbit_sock *); 331 331 + int cxgbit_put_login_tx(struct iscsi_conn *, struct iscsi_login *, u32); 332 332 + int cxgbit_xmit_pdu(struct iscsi_conn *, struct iscsi_cmd *, 333 333 + struct iscsi_datain_req *, const void *, u32); 334 334 + void cxgbit_get_r2t_ttt(struct iscsi_conn *, struct iscsi_cmd *, 335 335 + struct iscsi_r2t *); 336 336 + u32 cxgbit_send_tx_flowc_wr(struct cxgbit_sock *); 337 337 + int cxgbit_ofld_send(struct cxgbit_device *, struct sk_buff *); 338 338 + void cxgbit_get_rx_pdu(struct iscsi_conn *); 339 339 + int cxgbit_validate_params(struct iscsi_conn *); 340 340 + struct cxgbit_device *cxgbit_find_device(struct net_device *, u8 *); 341 341 + 342 342 + /* DDP */ 343 343 + int cxgbit_ddp_init(struct cxgbit_device *); 344 344 + int cxgbit_setup_conn_pgidx(struct cxgbit_sock *, u32); 345 345 + int cxgbit_reserve_ttt(struct cxgbit_sock *, struct iscsi_cmd *); 346 346 + void cxgbit_release_cmd(struct iscsi_conn *, struct iscsi_cmd *); 347 347 + 348 348 + static inline 349 349 + struct cxgbi_ppm *cdev2ppm(struct cxgbit_device *cdev) 350 350 + { 351 351 + return (struct cxgbi_ppm *)(*cdev->lldi.iscsi_ppm); 352 352 + } 353 353 + #endif /* __CXGBIT_H__ */

+2086

drivers/target/iscsi/cxgbit/cxgbit_cm.c

reviewed

··· 1 1 + /* 2 2 + * Copyright (c) 2016 Chelsio Communications, Inc. 3 3 + * 4 4 + * This program is free software; you can redistribute it and/or modify 5 5 + * it under the terms of the GNU General Public License version 2 as 6 6 + * published by the Free Software Foundation. 7 7 + */ 8 8 + 9 9 + #include <linux/module.h> 10 10 + #include <linux/list.h> 11 11 + #include <linux/workqueue.h> 12 12 + #include <linux/skbuff.h> 13 13 + #include <linux/timer.h> 14 14 + #include <linux/notifier.h> 15 15 + #include <linux/inetdevice.h> 16 16 + #include <linux/ip.h> 17 17 + #include <linux/tcp.h> 18 18 + #include <linux/if_vlan.h> 19 19 + 20 20 + #include <net/neighbour.h> 21 21 + #include <net/netevent.h> 22 22 + #include <net/route.h> 23 23 + #include <net/tcp.h> 24 24 + #include <net/ip6_route.h> 25 25 + #include <net/addrconf.h> 26 26 + 27 27 + #include "cxgbit.h" 28 28 + #include "clip_tbl.h" 29 29 + 30 30 + static void cxgbit_init_wr_wait(struct cxgbit_wr_wait *wr_waitp) 31 31 + { 32 32 + wr_waitp->ret = 0; 33 33 + reinit_completion(&wr_waitp->completion); 34 34 + } 35 35 + 36 36 + static void 37 37 + cxgbit_wake_up(struct cxgbit_wr_wait *wr_waitp, const char *func, u8 ret) 38 38 + { 39 39 + if (ret == CPL_ERR_NONE) 40 40 + wr_waitp->ret = 0; 41 41 + else 42 42 + wr_waitp->ret = -EIO; 43 43 + 44 44 + if (wr_waitp->ret) 45 45 + pr_err("%s: err:%u", func, ret); 46 46 + 47 47 + complete(&wr_waitp->completion); 48 48 + } 49 49 + 50 50 + static int 51 51 + cxgbit_wait_for_reply(struct cxgbit_device *cdev, 52 52 + struct cxgbit_wr_wait *wr_waitp, u32 tid, u32 timeout, 53 53 + const char *func) 54 54 + { 55 55 + int ret; 56 56 + 57 57 + if (!test_bit(CDEV_STATE_UP, &cdev->flags)) { 58 58 + wr_waitp->ret = -EIO; 59 59 + goto out; 60 60 + } 61 61 + 62 62 + ret = wait_for_completion_timeout(&wr_waitp->completion, timeout * HZ); 63 63 + if (!ret) { 64 64 + pr_info("%s - Device %s not responding tid %u\n", 65 65 + func, pci_name(cdev->lldi.pdev), tid); 66 66 + wr_waitp->ret = -ETIMEDOUT; 67 67 + } 68 68 + out: 69 69 + if (wr_waitp->ret) 70 70 + pr_info("%s: FW reply %d tid %u\n", 71 71 + pci_name(cdev->lldi.pdev), wr_waitp->ret, tid); 72 72 + return wr_waitp->ret; 73 73 + } 74 74 + 75 75 + /* Returns whether a CPL status conveys negative advice. 76 76 + */ 77 77 + static int cxgbit_is_neg_adv(unsigned int status) 78 78 + { 79 79 + return status == CPL_ERR_RTX_NEG_ADVICE || 80 80 + status == CPL_ERR_PERSIST_NEG_ADVICE || 81 81 + status == CPL_ERR_KEEPALV_NEG_ADVICE; 82 82 + } 83 83 + 84 84 + static int cxgbit_np_hashfn(const struct cxgbit_np *cnp) 85 85 + { 86 86 + return ((unsigned long)cnp >> 10) & (NP_INFO_HASH_SIZE - 1); 87 87 + } 88 88 + 89 89 + static struct np_info * 90 90 + cxgbit_np_hash_add(struct cxgbit_device *cdev, struct cxgbit_np *cnp, 91 91 + unsigned int stid) 92 92 + { 93 93 + struct np_info *p = kzalloc(sizeof(*p), GFP_KERNEL); 94 94 + 95 95 + if (p) { 96 96 + int bucket = cxgbit_np_hashfn(cnp); 97 97 + 98 98 + p->cnp = cnp; 99 99 + p->stid = stid; 100 100 + spin_lock(&cdev->np_lock); 101 101 + p->next = cdev->np_hash_tab[bucket]; 102 102 + cdev->np_hash_tab[bucket] = p; 103 103 + spin_unlock(&cdev->np_lock); 104 104 + } 105 105 + 106 106 + return p; 107 107 + } 108 108 + 109 109 + static int 110 110 + cxgbit_np_hash_find(struct cxgbit_device *cdev, struct cxgbit_np *cnp) 111 111 + { 112 112 + int stid = -1, bucket = cxgbit_np_hashfn(cnp); 113 113 + struct np_info *p; 114 114 + 115 115 + spin_lock(&cdev->np_lock); 116 116 + for (p = cdev->np_hash_tab[bucket]; p; p = p->next) { 117 117 + if (p->cnp == cnp) { 118 118 + stid = p->stid; 119 119 + break; 120 120 + } 121 121 + } 122 122 + spin_unlock(&cdev->np_lock); 123 123 + 124 124 + return stid; 125 125 + } 126 126 + 127 127 + static int cxgbit_np_hash_del(struct cxgbit_device *cdev, struct cxgbit_np *cnp) 128 128 + { 129 129 + int stid = -1, bucket = cxgbit_np_hashfn(cnp); 130 130 + struct np_info *p, **prev = &cdev->np_hash_tab[bucket]; 131 131 + 132 132 + spin_lock(&cdev->np_lock); 133 133 + for (p = *prev; p; prev = &p->next, p = p->next) { 134 134 + if (p->cnp == cnp) { 135 135 + stid = p->stid; 136 136 + *prev = p->next; 137 137 + kfree(p); 138 138 + break; 139 139 + } 140 140 + } 141 141 + spin_unlock(&cdev->np_lock); 142 142 + 143 143 + return stid; 144 144 + } 145 145 + 146 146 + void _cxgbit_free_cnp(struct kref *kref) 147 147 + { 148 148 + struct cxgbit_np *cnp; 149 149 + 150 150 + cnp = container_of(kref, struct cxgbit_np, kref); 151 151 + kfree(cnp); 152 152 + } 153 153 + 154 154 + static int 155 155 + cxgbit_create_server6(struct cxgbit_device *cdev, unsigned int stid, 156 156 + struct cxgbit_np *cnp) 157 157 + { 158 158 + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) 159 159 + &cnp->com.local_addr; 160 160 + int addr_type; 161 161 + int ret; 162 162 + 163 163 + pr_debug("%s: dev = %s; stid = %u; sin6_port = %u\n", 164 164 + __func__, cdev->lldi.ports[0]->name, stid, sin6->sin6_port); 165 165 + 166 166 + addr_type = ipv6_addr_type((const struct in6_addr *) 167 167 + &sin6->sin6_addr); 168 168 + if (addr_type != IPV6_ADDR_ANY) { 169 169 + ret = cxgb4_clip_get(cdev->lldi.ports[0], 170 170 + (const u32 *)&sin6->sin6_addr.s6_addr, 1); 171 171 + if (ret) { 172 172 + pr_err("Unable to find clip table entry. laddr %pI6. Error:%d.\n", 173 173 + sin6->sin6_addr.s6_addr, ret); 174 174 + return -ENOMEM; 175 175 + } 176 176 + } 177 177 + 178 178 + cxgbit_get_cnp(cnp); 179 179 + cxgbit_init_wr_wait(&cnp->com.wr_wait); 180 180 + 181 181 + ret = cxgb4_create_server6(cdev->lldi.ports[0], 182 182 + stid, &sin6->sin6_addr, 183 183 + sin6->sin6_port, 184 184 + cdev->lldi.rxq_ids[0]); 185 185 + if (!ret) 186 186 + ret = cxgbit_wait_for_reply(cdev, &cnp->com.wr_wait, 187 187 + 0, 10, __func__); 188 188 + else if (ret > 0) 189 189 + ret = net_xmit_errno(ret); 190 190 + else 191 191 + cxgbit_put_cnp(cnp); 192 192 + 193 193 + if (ret) { 194 194 + if (ret != -ETIMEDOUT) 195 195 + cxgb4_clip_release(cdev->lldi.ports[0], 196 196 + (const u32 *)&sin6->sin6_addr.s6_addr, 1); 197 197 + 198 198 + pr_err("create server6 err %d stid %d laddr %pI6 lport %d\n", 199 199 + ret, stid, sin6->sin6_addr.s6_addr, 200 200 + ntohs(sin6->sin6_port)); 201 201 + } 202 202 + 203 203 + return ret; 204 204 + } 205 205 + 206 206 + static int 207 207 + cxgbit_create_server4(struct cxgbit_device *cdev, unsigned int stid, 208 208 + struct cxgbit_np *cnp) 209 209 + { 210 210 + struct sockaddr_in *sin = (struct sockaddr_in *) 211 211 + &cnp->com.local_addr; 212 212 + int ret; 213 213 + 214 214 + pr_debug("%s: dev = %s; stid = %u; sin_port = %u\n", 215 215 + __func__, cdev->lldi.ports[0]->name, stid, sin->sin_port); 216 216 + 217 217 + cxgbit_get_cnp(cnp); 218 218 + cxgbit_init_wr_wait(&cnp->com.wr_wait); 219 219 + 220 220 + ret = cxgb4_create_server(cdev->lldi.ports[0], 221 221 + stid, sin->sin_addr.s_addr, 222 222 + sin->sin_port, 0, 223 223 + cdev->lldi.rxq_ids[0]); 224 224 + if (!ret) 225 225 + ret = cxgbit_wait_for_reply(cdev, 226 226 + &cnp->com.wr_wait, 227 227 + 0, 10, __func__); 228 228 + else if (ret > 0) 229 229 + ret = net_xmit_errno(ret); 230 230 + else 231 231 + cxgbit_put_cnp(cnp); 232 232 + 233 233 + if (ret) 234 234 + pr_err("create server failed err %d stid %d laddr %pI4 lport %d\n", 235 235 + ret, stid, &sin->sin_addr, ntohs(sin->sin_port)); 236 236 + return ret; 237 237 + } 238 238 + 239 239 + struct cxgbit_device *cxgbit_find_device(struct net_device *ndev, u8 *port_id) 240 240 + { 241 241 + struct cxgbit_device *cdev; 242 242 + u8 i; 243 243 + 244 244 + list_for_each_entry(cdev, &cdev_list_head, list) { 245 245 + struct cxgb4_lld_info *lldi = &cdev->lldi; 246 246 + 247 247 + for (i = 0; i < lldi->nports; i++) { 248 248 + if (lldi->ports[i] == ndev) { 249 249 + if (port_id) 250 250 + *port_id = i; 251 251 + return cdev; 252 252 + } 253 253 + } 254 254 + } 255 255 + 256 256 + return NULL; 257 257 + } 258 258 + 259 259 + static struct net_device *cxgbit_get_real_dev(struct net_device *ndev) 260 260 + { 261 261 + if (ndev->priv_flags & IFF_BONDING) { 262 262 + pr_err("Bond devices are not supported. Interface:%s\n", 263 263 + ndev->name); 264 264 + return NULL; 265 265 + } 266 266 + 267 267 + if (is_vlan_dev(ndev)) 268 268 + return vlan_dev_real_dev(ndev); 269 269 + 270 270 + return ndev; 271 271 + } 272 272 + 273 273 + static struct net_device *cxgbit_ipv4_netdev(__be32 saddr) 274 274 + { 275 275 + struct net_device *ndev; 276 276 + 277 277 + ndev = __ip_dev_find(&init_net, saddr, false); 278 278 + if (!ndev) 279 279 + return NULL; 280 280 + 281 281 + return cxgbit_get_real_dev(ndev); 282 282 + } 283 283 + 284 284 + static struct net_device *cxgbit_ipv6_netdev(struct in6_addr *addr6) 285 285 + { 286 286 + struct net_device *ndev = NULL; 287 287 + bool found = false; 288 288 + 289 289 + if (IS_ENABLED(CONFIG_IPV6)) { 290 290 + for_each_netdev_rcu(&init_net, ndev) 291 291 + if (ipv6_chk_addr(&init_net, addr6, ndev, 1)) { 292 292 + found = true; 293 293 + break; 294 294 + } 295 295 + } 296 296 + if (!found) 297 297 + return NULL; 298 298 + return cxgbit_get_real_dev(ndev); 299 299 + } 300 300 + 301 301 + static struct cxgbit_device *cxgbit_find_np_cdev(struct cxgbit_np *cnp) 302 302 + { 303 303 + struct sockaddr_storage *sockaddr = &cnp->com.local_addr; 304 304 + int ss_family = sockaddr->ss_family; 305 305 + struct net_device *ndev = NULL; 306 306 + struct cxgbit_device *cdev = NULL; 307 307 + 308 308 + rcu_read_lock(); 309 309 + if (ss_family == AF_INET) { 310 310 + struct sockaddr_in *sin; 311 311 + 312 312 + sin = (struct sockaddr_in *)sockaddr; 313 313 + ndev = cxgbit_ipv4_netdev(sin->sin_addr.s_addr); 314 314 + } else if (ss_family == AF_INET6) { 315 315 + struct sockaddr_in6 *sin6; 316 316 + 317 317 + sin6 = (struct sockaddr_in6 *)sockaddr; 318 318 + ndev = cxgbit_ipv6_netdev(&sin6->sin6_addr); 319 319 + } 320 320 + if (!ndev) 321 321 + goto out; 322 322 + 323 323 + cdev = cxgbit_find_device(ndev, NULL); 324 324 + out: 325 325 + rcu_read_unlock(); 326 326 + return cdev; 327 327 + } 328 328 + 329 329 + static bool cxgbit_inaddr_any(struct cxgbit_np *cnp) 330 330 + { 331 331 + struct sockaddr_storage *sockaddr = &cnp->com.local_addr; 332 332 + int ss_family = sockaddr->ss_family; 333 333 + int addr_type; 334 334 + 335 335 + if (ss_family == AF_INET) { 336 336 + struct sockaddr_in *sin; 337 337 + 338 338 + sin = (struct sockaddr_in *)sockaddr; 339 339 + if (sin->sin_addr.s_addr == htonl(INADDR_ANY)) 340 340 + return true; 341 341 + } else if (ss_family == AF_INET6) { 342 342 + struct sockaddr_in6 *sin6; 343 343 + 344 344 + sin6 = (struct sockaddr_in6 *)sockaddr; 345 345 + addr_type = ipv6_addr_type((const struct in6_addr *) 346 346 + &sin6->sin6_addr); 347 347 + if (addr_type == IPV6_ADDR_ANY) 348 348 + return true; 349 349 + } 350 350 + return false; 351 351 + } 352 352 + 353 353 + static int 354 354 + __cxgbit_setup_cdev_np(struct cxgbit_device *cdev, struct cxgbit_np *cnp) 355 355 + { 356 356 + int stid, ret; 357 357 + int ss_family = cnp->com.local_addr.ss_family; 358 358 + 359 359 + if (!test_bit(CDEV_STATE_UP, &cdev->flags)) 360 360 + return -EINVAL; 361 361 + 362 362 + stid = cxgb4_alloc_stid(cdev->lldi.tids, ss_family, cnp); 363 363 + if (stid < 0) 364 364 + return -EINVAL; 365 365 + 366 366 + if (!cxgbit_np_hash_add(cdev, cnp, stid)) { 367 367 + cxgb4_free_stid(cdev->lldi.tids, stid, ss_family); 368 368 + return -EINVAL; 369 369 + } 370 370 + 371 371 + if (ss_family == AF_INET) 372 372 + ret = cxgbit_create_server4(cdev, stid, cnp); 373 373 + else 374 374 + ret = cxgbit_create_server6(cdev, stid, cnp); 375 375 + 376 376 + if (ret) { 377 377 + if (ret != -ETIMEDOUT) 378 378 + cxgb4_free_stid(cdev->lldi.tids, stid, 379 379 + ss_family); 380 380 + cxgbit_np_hash_del(cdev, cnp); 381 381 + return ret; 382 382 + } 383 383 + return ret; 384 384 + } 385 385 + 386 386 + static int cxgbit_setup_cdev_np(struct cxgbit_np *cnp) 387 387 + { 388 388 + struct cxgbit_device *cdev; 389 389 + int ret = -1; 390 390 + 391 391 + mutex_lock(&cdev_list_lock); 392 392 + cdev = cxgbit_find_np_cdev(cnp); 393 393 + if (!cdev) 394 394 + goto out; 395 395 + 396 396 + if (cxgbit_np_hash_find(cdev, cnp) >= 0) 397 397 + goto out; 398 398 + 399 399 + if (__cxgbit_setup_cdev_np(cdev, cnp)) 400 400 + goto out; 401 401 + 402 402 + cnp->com.cdev = cdev; 403 403 + ret = 0; 404 404 + out: 405 405 + mutex_unlock(&cdev_list_lock); 406 406 + return ret; 407 407 + } 408 408 + 409 409 + static int cxgbit_setup_all_np(struct cxgbit_np *cnp) 410 410 + { 411 411 + struct cxgbit_device *cdev; 412 412 + int ret; 413 413 + u32 count = 0; 414 414 + 415 415 + mutex_lock(&cdev_list_lock); 416 416 + list_for_each_entry(cdev, &cdev_list_head, list) { 417 417 + if (cxgbit_np_hash_find(cdev, cnp) >= 0) { 418 418 + mutex_unlock(&cdev_list_lock); 419 419 + return -1; 420 420 + } 421 421 + } 422 422 + 423 423 + list_for_each_entry(cdev, &cdev_list_head, list) { 424 424 + ret = __cxgbit_setup_cdev_np(cdev, cnp); 425 425 + if (ret == -ETIMEDOUT) 426 426 + break; 427 427 + if (ret != 0) 428 428 + continue; 429 429 + count++; 430 430 + } 431 431 + mutex_unlock(&cdev_list_lock); 432 432 + 433 433 + return count ? 0 : -1; 434 434 + } 435 435 + 436 436 + int cxgbit_setup_np(struct iscsi_np *np, struct sockaddr_storage *ksockaddr) 437 437 + { 438 438 + struct cxgbit_np *cnp; 439 439 + int ret; 440 440 + 441 441 + if ((ksockaddr->ss_family != AF_INET) && 442 442 + (ksockaddr->ss_family != AF_INET6)) 443 443 + return -EINVAL; 444 444 + 445 445 + cnp = kzalloc(sizeof(*cnp), GFP_KERNEL); 446 446 + if (!cnp) 447 447 + return -ENOMEM; 448 448 + 449 449 + init_waitqueue_head(&cnp->accept_wait); 450 450 + init_completion(&cnp->com.wr_wait.completion); 451 451 + init_completion(&cnp->accept_comp); 452 452 + INIT_LIST_HEAD(&cnp->np_accept_list); 453 453 + spin_lock_init(&cnp->np_accept_lock); 454 454 + kref_init(&cnp->kref); 455 455 + memcpy(&np->np_sockaddr, ksockaddr, 456 456 + sizeof(struct sockaddr_storage)); 457 457 + memcpy(&cnp->com.local_addr, &np->np_sockaddr, 458 458 + sizeof(cnp->com.local_addr)); 459 459 + 460 460 + cnp->np = np; 461 461 + cnp->com.cdev = NULL; 462 462 + 463 463 + if (cxgbit_inaddr_any(cnp)) 464 464 + ret = cxgbit_setup_all_np(cnp); 465 465 + else 466 466 + ret = cxgbit_setup_cdev_np(cnp); 467 467 + 468 468 + if (ret) { 469 469 + cxgbit_put_cnp(cnp); 470 470 + return -EINVAL; 471 471 + } 472 472 + 473 473 + np->np_context = cnp; 474 474 + cnp->com.state = CSK_STATE_LISTEN; 475 475 + return 0; 476 476 + } 477 477 + 478 478 + static void 479 479 + cxgbit_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 480 480 + struct cxgbit_sock *csk) 481 481 + { 482 482 + conn->login_family = np->np_sockaddr.ss_family; 483 483 + conn->login_sockaddr = csk->com.remote_addr; 484 484 + conn->local_sockaddr = csk->com.local_addr; 485 485 + } 486 486 + 487 487 + int cxgbit_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 488 488 + { 489 489 + struct cxgbit_np *cnp = np->np_context; 490 490 + struct cxgbit_sock *csk; 491 491 + int ret = 0; 492 492 + 493 493 + accept_wait: 494 494 + ret = wait_for_completion_interruptible(&cnp->accept_comp); 495 495 + if (ret) 496 496 + return -ENODEV; 497 497 + 498 498 + spin_lock_bh(&np->np_thread_lock); 499 499 + if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 500 500 + spin_unlock_bh(&np->np_thread_lock); 501 501 + /** 502 502 + * No point in stalling here when np_thread 503 503 + * is in state RESET/SHUTDOWN/EXIT - bail 504 504 + **/ 505 505 + return -ENODEV; 506 506 + } 507 507 + spin_unlock_bh(&np->np_thread_lock); 508 508 + 509 509 + spin_lock_bh(&cnp->np_accept_lock); 510 510 + if (list_empty(&cnp->np_accept_list)) { 511 511 + spin_unlock_bh(&cnp->np_accept_lock); 512 512 + goto accept_wait; 513 513 + } 514 514 + 515 515 + csk = list_first_entry(&cnp->np_accept_list, 516 516 + struct cxgbit_sock, 517 517 + accept_node); 518 518 + 519 519 + list_del_init(&csk->accept_node); 520 520 + spin_unlock_bh(&cnp->np_accept_lock); 521 521 + conn->context = csk; 522 522 + csk->conn = conn; 523 523 + 524 524 + cxgbit_set_conn_info(np, conn, csk); 525 525 + return 0; 526 526 + } 527 527 + 528 528 + static int 529 529 + __cxgbit_free_cdev_np(struct cxgbit_device *cdev, struct cxgbit_np *cnp) 530 530 + { 531 531 + int stid, ret; 532 532 + bool ipv6 = false; 533 533 + 534 534 + stid = cxgbit_np_hash_del(cdev, cnp); 535 535 + if (stid < 0) 536 536 + return -EINVAL; 537 537 + if (!test_bit(CDEV_STATE_UP, &cdev->flags)) 538 538 + return -EINVAL; 539 539 + 540 540 + if (cnp->np->np_sockaddr.ss_family == AF_INET6) 541 541 + ipv6 = true; 542 542 + 543 543 + cxgbit_get_cnp(cnp); 544 544 + cxgbit_init_wr_wait(&cnp->com.wr_wait); 545 545 + ret = cxgb4_remove_server(cdev->lldi.ports[0], stid, 546 546 + cdev->lldi.rxq_ids[0], ipv6); 547 547 + 548 548 + if (ret > 0) 549 549 + ret = net_xmit_errno(ret); 550 550 + 551 551 + if (ret) { 552 552 + cxgbit_put_cnp(cnp); 553 553 + return ret; 554 554 + } 555 555 + 556 556 + ret = cxgbit_wait_for_reply(cdev, &cnp->com.wr_wait, 557 557 + 0, 10, __func__); 558 558 + if (ret == -ETIMEDOUT) 559 559 + return ret; 560 560 + 561 561 + if (ipv6 && cnp->com.cdev) { 562 562 + struct sockaddr_in6 *sin6; 563 563 + 564 564 + sin6 = (struct sockaddr_in6 *)&cnp->com.local_addr; 565 565 + cxgb4_clip_release(cdev->lldi.ports[0], 566 566 + (const u32 *)&sin6->sin6_addr.s6_addr, 567 567 + 1); 568 568 + } 569 569 + 570 570 + cxgb4_free_stid(cdev->lldi.tids, stid, 571 571 + cnp->com.local_addr.ss_family); 572 572 + return 0; 573 573 + } 574 574 + 575 575 + static void cxgbit_free_all_np(struct cxgbit_np *cnp) 576 576 + { 577 577 + struct cxgbit_device *cdev; 578 578 + int ret; 579 579 + 580 580 + mutex_lock(&cdev_list_lock); 581 581 + list_for_each_entry(cdev, &cdev_list_head, list) { 582 582 + ret = __cxgbit_free_cdev_np(cdev, cnp); 583 583 + if (ret == -ETIMEDOUT) 584 584 + break; 585 585 + } 586 586 + mutex_unlock(&cdev_list_lock); 587 587 + } 588 588 + 589 589 + static void cxgbit_free_cdev_np(struct cxgbit_np *cnp) 590 590 + { 591 591 + struct cxgbit_device *cdev; 592 592 + bool found = false; 593 593 + 594 594 + mutex_lock(&cdev_list_lock); 595 595 + list_for_each_entry(cdev, &cdev_list_head, list) { 596 596 + if (cdev == cnp->com.cdev) { 597 597 + found = true; 598 598 + break; 599 599 + } 600 600 + } 601 601 + if (!found) 602 602 + goto out; 603 603 + 604 604 + __cxgbit_free_cdev_np(cdev, cnp); 605 605 + out: 606 606 + mutex_unlock(&cdev_list_lock); 607 607 + } 608 608 + 609 609 + void cxgbit_free_np(struct iscsi_np *np) 610 610 + { 611 611 + struct cxgbit_np *cnp = np->np_context; 612 612 + 613 613 + cnp->com.state = CSK_STATE_DEAD; 614 614 + if (cnp->com.cdev) 615 615 + cxgbit_free_cdev_np(cnp); 616 616 + else 617 617 + cxgbit_free_all_np(cnp); 618 618 + 619 619 + np->np_context = NULL; 620 620 + cxgbit_put_cnp(cnp); 621 621 + } 622 622 + 623 623 + static void cxgbit_send_halfclose(struct cxgbit_sock *csk) 624 624 + { 625 625 + struct sk_buff *skb; 626 626 + struct cpl_close_con_req *req; 627 627 + unsigned int len = roundup(sizeof(struct cpl_close_con_req), 16); 628 628 + 629 629 + skb = alloc_skb(len, GFP_ATOMIC); 630 630 + if (!skb) 631 631 + return; 632 632 + 633 633 + req = (struct cpl_close_con_req *)__skb_put(skb, len); 634 634 + memset(req, 0, len); 635 635 + 636 636 + set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 637 637 + INIT_TP_WR(req, csk->tid); 638 638 + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, 639 639 + csk->tid)); 640 640 + req->rsvd = 0; 641 641 + 642 642 + cxgbit_skcb_flags(skb) |= SKCBF_TX_FLAG_COMPL; 643 643 + __skb_queue_tail(&csk->txq, skb); 644 644 + cxgbit_push_tx_frames(csk); 645 645 + } 646 646 + 647 647 + static void cxgbit_arp_failure_discard(void *handle, struct sk_buff *skb) 648 648 + { 649 649 + pr_debug("%s cxgbit_device %p\n", __func__, handle); 650 650 + kfree_skb(skb); 651 651 + } 652 652 + 653 653 + static void cxgbit_abort_arp_failure(void *handle, struct sk_buff *skb) 654 654 + { 655 655 + struct cxgbit_device *cdev = handle; 656 656 + struct cpl_abort_req *req = cplhdr(skb); 657 657 + 658 658 + pr_debug("%s cdev %p\n", __func__, cdev); 659 659 + req->cmd = CPL_ABORT_NO_RST; 660 660 + cxgbit_ofld_send(cdev, skb); 661 661 + } 662 662 + 663 663 + static int cxgbit_send_abort_req(struct cxgbit_sock *csk) 664 664 + { 665 665 + struct cpl_abort_req *req; 666 666 + unsigned int len = roundup(sizeof(*req), 16); 667 667 + struct sk_buff *skb; 668 668 + 669 669 + pr_debug("%s: csk %p tid %u; state %d\n", 670 670 + __func__, csk, csk->tid, csk->com.state); 671 671 + 672 672 + __skb_queue_purge(&csk->txq); 673 673 + 674 674 + if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) 675 675 + cxgbit_send_tx_flowc_wr(csk); 676 676 + 677 677 + skb = __skb_dequeue(&csk->skbq); 678 678 + req = (struct cpl_abort_req *)__skb_put(skb, len); 679 679 + memset(req, 0, len); 680 680 + 681 681 + set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 682 682 + t4_set_arp_err_handler(skb, csk->com.cdev, cxgbit_abort_arp_failure); 683 683 + INIT_TP_WR(req, csk->tid); 684 684 + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, 685 685 + csk->tid)); 686 686 + req->cmd = CPL_ABORT_SEND_RST; 687 687 + return cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 688 688 + } 689 689 + 690 690 + void cxgbit_free_conn(struct iscsi_conn *conn) 691 691 + { 692 692 + struct cxgbit_sock *csk = conn->context; 693 693 + bool release = false; 694 694 + 695 695 + pr_debug("%s: state %d\n", 696 696 + __func__, csk->com.state); 697 697 + 698 698 + spin_lock_bh(&csk->lock); 699 699 + switch (csk->com.state) { 700 700 + case CSK_STATE_ESTABLISHED: 701 701 + if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) { 702 702 + csk->com.state = CSK_STATE_CLOSING; 703 703 + cxgbit_send_halfclose(csk); 704 704 + } else { 705 705 + csk->com.state = CSK_STATE_ABORTING; 706 706 + cxgbit_send_abort_req(csk); 707 707 + } 708 708 + break; 709 709 + case CSK_STATE_CLOSING: 710 710 + csk->com.state = CSK_STATE_MORIBUND; 711 711 + cxgbit_send_halfclose(csk); 712 712 + break; 713 713 + case CSK_STATE_DEAD: 714 714 + release = true; 715 715 + break; 716 716 + default: 717 717 + pr_err("%s: csk %p; state %d\n", 718 718 + __func__, csk, csk->com.state); 719 719 + } 720 720 + spin_unlock_bh(&csk->lock); 721 721 + 722 722 + if (release) 723 723 + cxgbit_put_csk(csk); 724 724 + } 725 725 + 726 726 + static void cxgbit_set_emss(struct cxgbit_sock *csk, u16 opt) 727 727 + { 728 728 + csk->emss = csk->com.cdev->lldi.mtus[TCPOPT_MSS_G(opt)] - 729 729 + ((csk->com.remote_addr.ss_family == AF_INET) ? 730 730 + sizeof(struct iphdr) : sizeof(struct ipv6hdr)) - 731 731 + sizeof(struct tcphdr); 732 732 + csk->mss = csk->emss; 733 733 + if (TCPOPT_TSTAMP_G(opt)) 734 734 + csk->emss -= round_up(TCPOLEN_TIMESTAMP, 4); 735 735 + if (csk->emss < 128) 736 736 + csk->emss = 128; 737 737 + if (csk->emss & 7) 738 738 + pr_info("Warning: misaligned mtu idx %u mss %u emss=%u\n", 739 739 + TCPOPT_MSS_G(opt), csk->mss, csk->emss); 740 740 + pr_debug("%s mss_idx %u mss %u emss=%u\n", __func__, TCPOPT_MSS_G(opt), 741 741 + csk->mss, csk->emss); 742 742 + } 743 743 + 744 744 + static void cxgbit_free_skb(struct cxgbit_sock *csk) 745 745 + { 746 746 + struct sk_buff *skb; 747 747 + 748 748 + __skb_queue_purge(&csk->txq); 749 749 + __skb_queue_purge(&csk->rxq); 750 750 + __skb_queue_purge(&csk->backlogq); 751 751 + __skb_queue_purge(&csk->ppodq); 752 752 + __skb_queue_purge(&csk->skbq); 753 753 + 754 754 + while ((skb = cxgbit_sock_dequeue_wr(csk))) 755 755 + kfree_skb(skb); 756 756 + 757 757 + __kfree_skb(csk->lro_hskb); 758 758 + } 759 759 + 760 760 + void _cxgbit_free_csk(struct kref *kref) 761 761 + { 762 762 + struct cxgbit_sock *csk; 763 763 + struct cxgbit_device *cdev; 764 764 + 765 765 + csk = container_of(kref, struct cxgbit_sock, kref); 766 766 + 767 767 + pr_debug("%s csk %p state %d\n", __func__, csk, csk->com.state); 768 768 + 769 769 + if (csk->com.local_addr.ss_family == AF_INET6) { 770 770 + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) 771 771 + &csk->com.local_addr; 772 772 + cxgb4_clip_release(csk->com.cdev->lldi.ports[0], 773 773 + (const u32 *) 774 774 + &sin6->sin6_addr.s6_addr, 1); 775 775 + } 776 776 + 777 777 + cxgb4_remove_tid(csk->com.cdev->lldi.tids, 0, csk->tid); 778 778 + dst_release(csk->dst); 779 779 + cxgb4_l2t_release(csk->l2t); 780 780 + 781 781 + cdev = csk->com.cdev; 782 782 + spin_lock_bh(&cdev->cskq.lock); 783 783 + list_del(&csk->list); 784 784 + spin_unlock_bh(&cdev->cskq.lock); 785 785 + 786 786 + cxgbit_free_skb(csk); 787 787 + cxgbit_put_cdev(cdev); 788 788 + 789 789 + kfree(csk); 790 790 + } 791 791 + 792 792 + static void 793 793 + cxgbit_get_tuple_info(struct cpl_pass_accept_req *req, int *iptype, 794 794 + __u8 *local_ip, __u8 *peer_ip, __be16 *local_port, 795 795 + __be16 *peer_port) 796 796 + { 797 797 + u32 eth_len = ETH_HDR_LEN_G(be32_to_cpu(req->hdr_len)); 798 798 + u32 ip_len = IP_HDR_LEN_G(be32_to_cpu(req->hdr_len)); 799 799 + struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len); 800 800 + struct ipv6hdr *ip6 = (struct ipv6hdr *)((u8 *)(req + 1) + eth_len); 801 801 + struct tcphdr *tcp = (struct tcphdr *) 802 802 + ((u8 *)(req + 1) + eth_len + ip_len); 803 803 + 804 804 + if (ip->version == 4) { 805 805 + pr_debug("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", 806 806 + __func__, 807 807 + ntohl(ip->saddr), ntohl(ip->daddr), 808 808 + ntohs(tcp->source), 809 809 + ntohs(tcp->dest)); 810 810 + *iptype = 4; 811 811 + memcpy(peer_ip, &ip->saddr, 4); 812 812 + memcpy(local_ip, &ip->daddr, 4); 813 813 + } else { 814 814 + pr_debug("%s saddr %pI6 daddr %pI6 sport %u dport %u\n", 815 815 + __func__, 816 816 + ip6->saddr.s6_addr, ip6->daddr.s6_addr, 817 817 + ntohs(tcp->source), 818 818 + ntohs(tcp->dest)); 819 819 + *iptype = 6; 820 820 + memcpy(peer_ip, ip6->saddr.s6_addr, 16); 821 821 + memcpy(local_ip, ip6->daddr.s6_addr, 16); 822 822 + } 823 823 + 824 824 + *peer_port = tcp->source; 825 825 + *local_port = tcp->dest; 826 826 + } 827 827 + 828 828 + static int 829 829 + cxgbit_our_interface(struct cxgbit_device *cdev, struct net_device *egress_dev) 830 830 + { 831 831 + u8 i; 832 832 + 833 833 + egress_dev = cxgbit_get_real_dev(egress_dev); 834 834 + for (i = 0; i < cdev->lldi.nports; i++) 835 835 + if (cdev->lldi.ports[i] == egress_dev) 836 836 + return 1; 837 837 + return 0; 838 838 + } 839 839 + 840 840 + static struct dst_entry * 841 841 + cxgbit_find_route6(struct cxgbit_device *cdev, __u8 *local_ip, __u8 *peer_ip, 842 842 + __be16 local_port, __be16 peer_port, u8 tos, 843 843 + __u32 sin6_scope_id) 844 844 + { 845 845 + struct dst_entry *dst = NULL; 846 846 + 847 847 + if (IS_ENABLED(CONFIG_IPV6)) { 848 848 + struct flowi6 fl6; 849 849 + 850 850 + memset(&fl6, 0, sizeof(fl6)); 851 851 + memcpy(&fl6.daddr, peer_ip, 16); 852 852 + memcpy(&fl6.saddr, local_ip, 16); 853 853 + if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) 854 854 + fl6.flowi6_oif = sin6_scope_id; 855 855 + dst = ip6_route_output(&init_net, NULL, &fl6); 856 856 + if (!dst) 857 857 + goto out; 858 858 + if (!cxgbit_our_interface(cdev, ip6_dst_idev(dst)->dev) && 859 859 + !(ip6_dst_idev(dst)->dev->flags & IFF_LOOPBACK)) { 860 860 + dst_release(dst); 861 861 + dst = NULL; 862 862 + } 863 863 + } 864 864 + out: 865 865 + return dst; 866 866 + } 867 867 + 868 868 + static struct dst_entry * 869 869 + cxgbit_find_route(struct cxgbit_device *cdev, __be32 local_ip, __be32 peer_ip, 870 870 + __be16 local_port, __be16 peer_port, u8 tos) 871 871 + { 872 872 + struct rtable *rt; 873 873 + struct flowi4 fl4; 874 874 + struct neighbour *n; 875 875 + 876 876 + rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, 877 877 + local_ip, 878 878 + peer_port, local_port, IPPROTO_TCP, 879 879 + tos, 0); 880 880 + if (IS_ERR(rt)) 881 881 + return NULL; 882 882 + n = dst_neigh_lookup(&rt->dst, &peer_ip); 883 883 + if (!n) 884 884 + return NULL; 885 885 + if (!cxgbit_our_interface(cdev, n->dev) && 886 886 + !(n->dev->flags & IFF_LOOPBACK)) { 887 887 + neigh_release(n); 888 888 + dst_release(&rt->dst); 889 889 + return NULL; 890 890 + } 891 891 + neigh_release(n); 892 892 + return &rt->dst; 893 893 + } 894 894 + 895 895 + static void cxgbit_set_tcp_window(struct cxgbit_sock *csk, struct port_info *pi) 896 896 + { 897 897 + unsigned int linkspeed; 898 898 + u8 scale; 899 899 + 900 900 + linkspeed = pi->link_cfg.speed; 901 901 + scale = linkspeed / SPEED_10000; 902 902 + 903 903 + #define CXGBIT_10G_RCV_WIN (256 * 1024) 904 904 + csk->rcv_win = CXGBIT_10G_RCV_WIN; 905 905 + if (scale) 906 906 + csk->rcv_win *= scale; 907 907 + 908 908 + #define CXGBIT_10G_SND_WIN (256 * 1024) 909 909 + csk->snd_win = CXGBIT_10G_SND_WIN; 910 910 + if (scale) 911 911 + csk->snd_win *= scale; 912 912 + 913 913 + pr_debug("%s snd_win %d rcv_win %d\n", 914 914 + __func__, csk->snd_win, csk->rcv_win); 915 915 + } 916 916 + 917 917 + #ifdef CONFIG_CHELSIO_T4_DCB 918 918 + static u8 cxgbit_get_iscsi_dcb_state(struct net_device *ndev) 919 919 + { 920 920 + return ndev->dcbnl_ops->getstate(ndev); 921 921 + } 922 922 + 923 923 + static int cxgbit_select_priority(int pri_mask) 924 924 + { 925 925 + if (!pri_mask) 926 926 + return 0; 927 927 + 928 928 + return (ffs(pri_mask) - 1); 929 929 + } 930 930 + 931 931 + static u8 cxgbit_get_iscsi_dcb_priority(struct net_device *ndev, u16 local_port) 932 932 + { 933 933 + int ret; 934 934 + u8 caps; 935 935 + 936 936 + struct dcb_app iscsi_dcb_app = { 937 937 + .protocol = local_port 938 938 + }; 939 939 + 940 940 + ret = (int)ndev->dcbnl_ops->getcap(ndev, DCB_CAP_ATTR_DCBX, &caps); 941 941 + 942 942 + if (ret) 943 943 + return 0; 944 944 + 945 945 + if (caps & DCB_CAP_DCBX_VER_IEEE) { 946 946 + iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_ANY; 947 947 + 948 948 + ret = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app); 949 949 + 950 950 + } else if (caps & DCB_CAP_DCBX_VER_CEE) { 951 951 + iscsi_dcb_app.selector = DCB_APP_IDTYPE_PORTNUM; 952 952 + 953 953 + ret = dcb_getapp(ndev, &iscsi_dcb_app); 954 954 + } 955 955 + 956 956 + pr_info("iSCSI priority is set to %u\n", cxgbit_select_priority(ret)); 957 957 + 958 958 + return cxgbit_select_priority(ret); 959 959 + } 960 960 + #endif 961 961 + 962 962 + static int 963 963 + cxgbit_offload_init(struct cxgbit_sock *csk, int iptype, __u8 *peer_ip, 964 964 + u16 local_port, struct dst_entry *dst, 965 965 + struct cxgbit_device *cdev) 966 966 + { 967 967 + struct neighbour *n; 968 968 + int ret, step; 969 969 + struct net_device *ndev; 970 970 + u16 rxq_idx, port_id; 971 971 + #ifdef CONFIG_CHELSIO_T4_DCB 972 972 + u8 priority = 0; 973 973 + #endif 974 974 + 975 975 + n = dst_neigh_lookup(dst, peer_ip); 976 976 + if (!n) 977 977 + return -ENODEV; 978 978 + 979 979 + rcu_read_lock(); 980 980 + ret = -ENOMEM; 981 981 + if (n->dev->flags & IFF_LOOPBACK) { 982 982 + if (iptype == 4) 983 983 + ndev = cxgbit_ipv4_netdev(*(__be32 *)peer_ip); 984 984 + else if (IS_ENABLED(CONFIG_IPV6)) 985 985 + ndev = cxgbit_ipv6_netdev((struct in6_addr *)peer_ip); 986 986 + else 987 987 + ndev = NULL; 988 988 + 989 989 + if (!ndev) { 990 990 + ret = -ENODEV; 991 991 + goto out; 992 992 + } 993 993 + 994 994 + csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, 995 995 + n, ndev, 0); 996 996 + if (!csk->l2t) 997 997 + goto out; 998 998 + csk->mtu = ndev->mtu; 999 999 + csk->tx_chan = cxgb4_port_chan(ndev); 1000 1000 + csk->smac_idx = (cxgb4_port_viid(ndev) & 0x7F) << 1; 1001 1001 + step = cdev->lldi.ntxq / 1002 1002 + cdev->lldi.nchan; 1003 1003 + csk->txq_idx = cxgb4_port_idx(ndev) * step; 1004 1004 + step = cdev->lldi.nrxq / 1005 1005 + cdev->lldi.nchan; 1006 1006 + csk->ctrlq_idx = cxgb4_port_idx(ndev); 1007 1007 + csk->rss_qid = cdev->lldi.rxq_ids[ 1008 1008 + cxgb4_port_idx(ndev) * step]; 1009 1009 + csk->port_id = cxgb4_port_idx(ndev); 1010 1010 + cxgbit_set_tcp_window(csk, 1011 1011 + (struct port_info *)netdev_priv(ndev)); 1012 1012 + } else { 1013 1013 + ndev = cxgbit_get_real_dev(n->dev); 1014 1014 + if (!ndev) { 1015 1015 + ret = -ENODEV; 1016 1016 + goto out; 1017 1017 + } 1018 1018 + 1019 1019 + #ifdef CONFIG_CHELSIO_T4_DCB 1020 1020 + if (cxgbit_get_iscsi_dcb_state(ndev)) 1021 1021 + priority = cxgbit_get_iscsi_dcb_priority(ndev, 1022 1022 + local_port); 1023 1023 + 1024 1024 + csk->dcb_priority = priority; 1025 1025 + 1026 1026 + csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, n, ndev, priority); 1027 1027 + #else 1028 1028 + csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, n, ndev, 0); 1029 1029 + #endif 1030 1030 + if (!csk->l2t) 1031 1031 + goto out; 1032 1032 + port_id = cxgb4_port_idx(ndev); 1033 1033 + csk->mtu = dst_mtu(dst); 1034 1034 + csk->tx_chan = cxgb4_port_chan(ndev); 1035 1035 + csk->smac_idx = (cxgb4_port_viid(ndev) & 0x7F) << 1; 1036 1036 + step = cdev->lldi.ntxq / 1037 1037 + cdev->lldi.nports; 1038 1038 + csk->txq_idx = (port_id * step) + 1039 1039 + (cdev->selectq[port_id][0]++ % step); 1040 1040 + csk->ctrlq_idx = cxgb4_port_idx(ndev); 1041 1041 + step = cdev->lldi.nrxq / 1042 1042 + cdev->lldi.nports; 1043 1043 + rxq_idx = (port_id * step) + 1044 1044 + (cdev->selectq[port_id][1]++ % step); 1045 1045 + csk->rss_qid = cdev->lldi.rxq_ids[rxq_idx]; 1046 1046 + csk->port_id = port_id; 1047 1047 + cxgbit_set_tcp_window(csk, 1048 1048 + (struct port_info *)netdev_priv(ndev)); 1049 1049 + } 1050 1050 + ret = 0; 1051 1051 + out: 1052 1052 + rcu_read_unlock(); 1053 1053 + neigh_release(n); 1054 1054 + return ret; 1055 1055 + } 1056 1056 + 1057 1057 + int cxgbit_ofld_send(struct cxgbit_device *cdev, struct sk_buff *skb) 1058 1058 + { 1059 1059 + int ret = 0; 1060 1060 + 1061 1061 + if (!test_bit(CDEV_STATE_UP, &cdev->flags)) { 1062 1062 + kfree_skb(skb); 1063 1063 + pr_err("%s - device not up - dropping\n", __func__); 1064 1064 + return -EIO; 1065 1065 + } 1066 1066 + 1067 1067 + ret = cxgb4_ofld_send(cdev->lldi.ports[0], skb); 1068 1068 + if (ret < 0) 1069 1069 + kfree_skb(skb); 1070 1070 + return ret < 0 ? ret : 0; 1071 1071 + } 1072 1072 + 1073 1073 + static void cxgbit_release_tid(struct cxgbit_device *cdev, u32 tid) 1074 1074 + { 1075 1075 + struct cpl_tid_release *req; 1076 1076 + unsigned int len = roundup(sizeof(*req), 16); 1077 1077 + struct sk_buff *skb; 1078 1078 + 1079 1079 + skb = alloc_skb(len, GFP_ATOMIC); 1080 1080 + if (!skb) 1081 1081 + return; 1082 1082 + 1083 1083 + req = (struct cpl_tid_release *)__skb_put(skb, len); 1084 1084 + memset(req, 0, len); 1085 1085 + 1086 1086 + INIT_TP_WR(req, tid); 1087 1087 + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID( 1088 1088 + CPL_TID_RELEASE, tid)); 1089 1089 + set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 1090 1090 + cxgbit_ofld_send(cdev, skb); 1091 1091 + } 1092 1092 + 1093 1093 + int 1094 1094 + cxgbit_l2t_send(struct cxgbit_device *cdev, struct sk_buff *skb, 1095 1095 + struct l2t_entry *l2e) 1096 1096 + { 1097 1097 + int ret = 0; 1098 1098 + 1099 1099 + if (!test_bit(CDEV_STATE_UP, &cdev->flags)) { 1100 1100 + kfree_skb(skb); 1101 1101 + pr_err("%s - device not up - dropping\n", __func__); 1102 1102 + return -EIO; 1103 1103 + } 1104 1104 + 1105 1105 + ret = cxgb4_l2t_send(cdev->lldi.ports[0], skb, l2e); 1106 1106 + if (ret < 0) 1107 1107 + kfree_skb(skb); 1108 1108 + return ret < 0 ? ret : 0; 1109 1109 + } 1110 1110 + 1111 1111 + static void 1112 1112 + cxgbit_best_mtu(const unsigned short *mtus, unsigned short mtu, 1113 1113 + unsigned int *idx, int use_ts, int ipv6) 1114 1114 + { 1115 1115 + unsigned short hdr_size = (ipv6 ? sizeof(struct ipv6hdr) : 1116 1116 + sizeof(struct iphdr)) + 1117 1117 + sizeof(struct tcphdr) + 1118 1118 + (use_ts ? round_up(TCPOLEN_TIMESTAMP, 1119 1119 + 4) : 0); 1120 1120 + unsigned short data_size = mtu - hdr_size; 1121 1121 + 1122 1122 + cxgb4_best_aligned_mtu(mtus, hdr_size, data_size, 8, idx); 1123 1123 + } 1124 1124 + 1125 1125 + static void cxgbit_send_rx_credits(struct cxgbit_sock *csk, struct sk_buff *skb) 1126 1126 + { 1127 1127 + if (csk->com.state != CSK_STATE_ESTABLISHED) { 1128 1128 + __kfree_skb(skb); 1129 1129 + return; 1130 1130 + } 1131 1131 + 1132 1132 + cxgbit_ofld_send(csk->com.cdev, skb); 1133 1133 + } 1134 1134 + 1135 1135 + /* 1136 1136 + * CPL connection rx data ack: host -> 1137 1137 + * Send RX credits through an RX_DATA_ACK CPL message. 1138 1138 + * Returns the number of credits sent. 1139 1139 + */ 1140 1140 + int cxgbit_rx_data_ack(struct cxgbit_sock *csk) 1141 1141 + { 1142 1142 + struct sk_buff *skb; 1143 1143 + struct cpl_rx_data_ack *req; 1144 1144 + unsigned int len = roundup(sizeof(*req), 16); 1145 1145 + 1146 1146 + skb = alloc_skb(len, GFP_KERNEL); 1147 1147 + if (!skb) 1148 1148 + return -1; 1149 1149 + 1150 1150 + req = (struct cpl_rx_data_ack *)__skb_put(skb, len); 1151 1151 + memset(req, 0, len); 1152 1152 + 1153 1153 + set_wr_txq(skb, CPL_PRIORITY_ACK, csk->ctrlq_idx); 1154 1154 + INIT_TP_WR(req, csk->tid); 1155 1155 + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, 1156 1156 + csk->tid)); 1157 1157 + req->credit_dack = cpu_to_be32(RX_DACK_CHANGE_F | RX_DACK_MODE_V(1) | 1158 1158 + RX_CREDITS_V(csk->rx_credits)); 1159 1159 + 1160 1160 + csk->rx_credits = 0; 1161 1161 + 1162 1162 + spin_lock_bh(&csk->lock); 1163 1163 + if (csk->lock_owner) { 1164 1164 + cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_send_rx_credits; 1165 1165 + __skb_queue_tail(&csk->backlogq, skb); 1166 1166 + spin_unlock_bh(&csk->lock); 1167 1167 + return 0; 1168 1168 + } 1169 1169 + 1170 1170 + cxgbit_send_rx_credits(csk, skb); 1171 1171 + spin_unlock_bh(&csk->lock); 1172 1172 + 1173 1173 + return 0; 1174 1174 + } 1175 1175 + 1176 1176 + #define FLOWC_WR_NPARAMS_MIN 9 1177 1177 + #define FLOWC_WR_NPARAMS_MAX 11 1178 1178 + static int cxgbit_alloc_csk_skb(struct cxgbit_sock *csk) 1179 1179 + { 1180 1180 + struct sk_buff *skb; 1181 1181 + u32 len, flowclen; 1182 1182 + u8 i; 1183 1183 + 1184 1184 + flowclen = offsetof(struct fw_flowc_wr, 1185 1185 + mnemval[FLOWC_WR_NPARAMS_MAX]); 1186 1186 + 1187 1187 + len = max_t(u32, sizeof(struct cpl_abort_req), 1188 1188 + sizeof(struct cpl_abort_rpl)); 1189 1189 + 1190 1190 + len = max(len, flowclen); 1191 1191 + len = roundup(len, 16); 1192 1192 + 1193 1193 + for (i = 0; i < 3; i++) { 1194 1194 + skb = alloc_skb(len, GFP_ATOMIC); 1195 1195 + if (!skb) 1196 1196 + goto out; 1197 1197 + __skb_queue_tail(&csk->skbq, skb); 1198 1198 + } 1199 1199 + 1200 1200 + skb = alloc_skb(LRO_SKB_MIN_HEADROOM, GFP_ATOMIC); 1201 1201 + if (!skb) 1202 1202 + goto out; 1203 1203 + 1204 1204 + memset(skb->data, 0, LRO_SKB_MIN_HEADROOM); 1205 1205 + csk->lro_hskb = skb; 1206 1206 + 1207 1207 + return 0; 1208 1208 + out: 1209 1209 + __skb_queue_purge(&csk->skbq); 1210 1210 + return -ENOMEM; 1211 1211 + } 1212 1212 + 1213 1213 + static u32 cxgbit_compute_wscale(u32 win) 1214 1214 + { 1215 1215 + u32 wscale = 0; 1216 1216 + 1217 1217 + while (wscale < 14 && (65535 << wscale) < win) 1218 1218 + wscale++; 1219 1219 + return wscale; 1220 1220 + } 1221 1221 + 1222 1222 + static void 1223 1223 + cxgbit_pass_accept_rpl(struct cxgbit_sock *csk, struct cpl_pass_accept_req *req) 1224 1224 + { 1225 1225 + struct sk_buff *skb; 1226 1226 + const struct tcphdr *tcph; 1227 1227 + struct cpl_t5_pass_accept_rpl *rpl5; 1228 1228 + unsigned int len = roundup(sizeof(*rpl5), 16); 1229 1229 + unsigned int mtu_idx; 1230 1230 + u64 opt0; 1231 1231 + u32 opt2, hlen; 1232 1232 + u32 wscale; 1233 1233 + u32 win; 1234 1234 + 1235 1235 + pr_debug("%s csk %p tid %u\n", __func__, csk, csk->tid); 1236 1236 + 1237 1237 + skb = alloc_skb(len, GFP_ATOMIC); 1238 1238 + if (!skb) { 1239 1239 + cxgbit_put_csk(csk); 1240 1240 + return; 1241 1241 + } 1242 1242 + 1243 1243 + rpl5 = (struct cpl_t5_pass_accept_rpl *)__skb_put(skb, len); 1244 1244 + memset(rpl5, 0, len); 1245 1245 + 1246 1246 + INIT_TP_WR(rpl5, csk->tid); 1247 1247 + OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, 1248 1248 + csk->tid)); 1249 1249 + cxgbit_best_mtu(csk->com.cdev->lldi.mtus, csk->mtu, &mtu_idx, 1250 1250 + req->tcpopt.tstamp, 1251 1251 + (csk->com.remote_addr.ss_family == AF_INET) ? 0 : 1); 1252 1252 + wscale = cxgbit_compute_wscale(csk->rcv_win); 1253 1253 + /* 1254 1254 + * Specify the largest window that will fit in opt0. The 1255 1255 + * remainder will be specified in the rx_data_ack. 1256 1256 + */ 1257 1257 + win = csk->rcv_win >> 10; 1258 1258 + if (win > RCV_BUFSIZ_M) 1259 1259 + win = RCV_BUFSIZ_M; 1260 1260 + opt0 = TCAM_BYPASS_F | 1261 1261 + WND_SCALE_V(wscale) | 1262 1262 + MSS_IDX_V(mtu_idx) | 1263 1263 + L2T_IDX_V(csk->l2t->idx) | 1264 1264 + TX_CHAN_V(csk->tx_chan) | 1265 1265 + SMAC_SEL_V(csk->smac_idx) | 1266 1266 + DSCP_V(csk->tos >> 2) | 1267 1267 + ULP_MODE_V(ULP_MODE_ISCSI) | 1268 1268 + RCV_BUFSIZ_V(win); 1269 1269 + 1270 1270 + opt2 = RX_CHANNEL_V(0) | 1271 1271 + RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid); 1272 1272 + 1273 1273 + if (req->tcpopt.tstamp) 1274 1274 + opt2 |= TSTAMPS_EN_F; 1275 1275 + if (req->tcpopt.sack) 1276 1276 + opt2 |= SACK_EN_F; 1277 1277 + if (wscale) 1278 1278 + opt2 |= WND_SCALE_EN_F; 1279 1279 + 1280 1280 + hlen = ntohl(req->hdr_len); 1281 1281 + tcph = (const void *)(req + 1) + ETH_HDR_LEN_G(hlen) + 1282 1282 + IP_HDR_LEN_G(hlen); 1283 1283 + 1284 1284 + if (tcph->ece && tcph->cwr) 1285 1285 + opt2 |= CCTRL_ECN_V(1); 1286 1286 + 1287 1287 + opt2 |= RX_COALESCE_V(3); 1288 1288 + opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO); 1289 1289 + 1290 1290 + opt2 |= T5_ISS_F; 1291 1291 + rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1); 1292 1292 + 1293 1293 + opt2 |= T5_OPT_2_VALID_F; 1294 1294 + 1295 1295 + rpl5->opt0 = cpu_to_be64(opt0); 1296 1296 + rpl5->opt2 = cpu_to_be32(opt2); 1297 1297 + set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->ctrlq_idx); 1298 1298 + t4_set_arp_err_handler(skb, NULL, cxgbit_arp_failure_discard); 1299 1299 + cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 1300 1300 + } 1301 1301 + 1302 1302 + static void 1303 1303 + cxgbit_pass_accept_req(struct cxgbit_device *cdev, struct sk_buff *skb) 1304 1304 + { 1305 1305 + struct cxgbit_sock *csk = NULL; 1306 1306 + struct cxgbit_np *cnp; 1307 1307 + struct cpl_pass_accept_req *req = cplhdr(skb); 1308 1308 + unsigned int stid = PASS_OPEN_TID_G(ntohl(req->tos_stid)); 1309 1309 + struct tid_info *t = cdev->lldi.tids; 1310 1310 + unsigned int tid = GET_TID(req); 1311 1311 + u16 peer_mss = ntohs(req->tcpopt.mss); 1312 1312 + unsigned short hdrs; 1313 1313 + 1314 1314 + struct dst_entry *dst; 1315 1315 + __u8 local_ip[16], peer_ip[16]; 1316 1316 + __be16 local_port, peer_port; 1317 1317 + int ret; 1318 1318 + int iptype; 1319 1319 + 1320 1320 + pr_debug("%s: cdev = %p; stid = %u; tid = %u\n", 1321 1321 + __func__, cdev, stid, tid); 1322 1322 + 1323 1323 + cnp = lookup_stid(t, stid); 1324 1324 + if (!cnp) { 1325 1325 + pr_err("%s connect request on invalid stid %d\n", 1326 1326 + __func__, stid); 1327 1327 + goto rel_skb; 1328 1328 + } 1329 1329 + 1330 1330 + if (cnp->com.state != CSK_STATE_LISTEN) { 1331 1331 + pr_err("%s - listening parent not in CSK_STATE_LISTEN\n", 1332 1332 + __func__); 1333 1333 + goto reject; 1334 1334 + } 1335 1335 + 1336 1336 + csk = lookup_tid(t, tid); 1337 1337 + if (csk) { 1338 1338 + pr_err("%s csk not null tid %u\n", 1339 1339 + __func__, tid); 1340 1340 + goto rel_skb; 1341 1341 + } 1342 1342 + 1343 1343 + cxgbit_get_tuple_info(req, &iptype, local_ip, peer_ip, 1344 1344 + &local_port, &peer_port); 1345 1345 + 1346 1346 + /* Find output route */ 1347 1347 + if (iptype == 4) { 1348 1348 + pr_debug("%s parent sock %p tid %u laddr %pI4 raddr %pI4 " 1349 1349 + "lport %d rport %d peer_mss %d\n" 1350 1350 + , __func__, cnp, tid, 1351 1351 + local_ip, peer_ip, ntohs(local_port), 1352 1352 + ntohs(peer_port), peer_mss); 1353 1353 + dst = cxgbit_find_route(cdev, *(__be32 *)local_ip, 1354 1354 + *(__be32 *)peer_ip, 1355 1355 + local_port, peer_port, 1356 1356 + PASS_OPEN_TOS_G(ntohl(req->tos_stid))); 1357 1357 + } else { 1358 1358 + pr_debug("%s parent sock %p tid %u laddr %pI6 raddr %pI6 " 1359 1359 + "lport %d rport %d peer_mss %d\n" 1360 1360 + , __func__, cnp, tid, 1361 1361 + local_ip, peer_ip, ntohs(local_port), 1362 1362 + ntohs(peer_port), peer_mss); 1363 1363 + dst = cxgbit_find_route6(cdev, local_ip, peer_ip, 1364 1364 + local_port, peer_port, 1365 1365 + PASS_OPEN_TOS_G(ntohl(req->tos_stid)), 1366 1366 + ((struct sockaddr_in6 *) 1367 1367 + &cnp->com.local_addr)->sin6_scope_id); 1368 1368 + } 1369 1369 + if (!dst) { 1370 1370 + pr_err("%s - failed to find dst entry!\n", 1371 1371 + __func__); 1372 1372 + goto reject; 1373 1373 + } 1374 1374 + 1375 1375 + csk = kzalloc(sizeof(*csk), GFP_ATOMIC); 1376 1376 + if (!csk) { 1377 1377 + dst_release(dst); 1378 1378 + goto rel_skb; 1379 1379 + } 1380 1380 + 1381 1381 + ret = cxgbit_offload_init(csk, iptype, peer_ip, ntohs(local_port), 1382 1382 + dst, cdev); 1383 1383 + if (ret) { 1384 1384 + pr_err("%s - failed to allocate l2t entry!\n", 1385 1385 + __func__); 1386 1386 + dst_release(dst); 1387 1387 + kfree(csk); 1388 1388 + goto reject; 1389 1389 + } 1390 1390 + 1391 1391 + kref_init(&csk->kref); 1392 1392 + init_completion(&csk->com.wr_wait.completion); 1393 1393 + 1394 1394 + INIT_LIST_HEAD(&csk->accept_node); 1395 1395 + 1396 1396 + hdrs = (iptype == 4 ? sizeof(struct iphdr) : sizeof(struct ipv6hdr)) + 1397 1397 + sizeof(struct tcphdr) + (req->tcpopt.tstamp ? 12 : 0); 1398 1398 + if (peer_mss && csk->mtu > (peer_mss + hdrs)) 1399 1399 + csk->mtu = peer_mss + hdrs; 1400 1400 + 1401 1401 + csk->com.state = CSK_STATE_CONNECTING; 1402 1402 + csk->com.cdev = cdev; 1403 1403 + csk->cnp = cnp; 1404 1404 + csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid)); 1405 1405 + csk->dst = dst; 1406 1406 + csk->tid = tid; 1407 1407 + csk->wr_cred = cdev->lldi.wr_cred - 1408 1408 + DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16); 1409 1409 + csk->wr_max_cred = csk->wr_cred; 1410 1410 + csk->wr_una_cred = 0; 1411 1411 + 1412 1412 + if (iptype == 4) { 1413 1413 + struct sockaddr_in *sin = (struct sockaddr_in *) 1414 1414 + &csk->com.local_addr; 1415 1415 + sin->sin_family = AF_INET; 1416 1416 + sin->sin_port = local_port; 1417 1417 + sin->sin_addr.s_addr = *(__be32 *)local_ip; 1418 1418 + 1419 1419 + sin = (struct sockaddr_in *)&csk->com.remote_addr; 1420 1420 + sin->sin_family = AF_INET; 1421 1421 + sin->sin_port = peer_port; 1422 1422 + sin->sin_addr.s_addr = *(__be32 *)peer_ip; 1423 1423 + } else { 1424 1424 + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) 1425 1425 + &csk->com.local_addr; 1426 1426 + 1427 1427 + sin6->sin6_family = PF_INET6; 1428 1428 + sin6->sin6_port = local_port; 1429 1429 + memcpy(sin6->sin6_addr.s6_addr, local_ip, 16); 1430 1430 + cxgb4_clip_get(cdev->lldi.ports[0], 1431 1431 + (const u32 *)&sin6->sin6_addr.s6_addr, 1432 1432 + 1); 1433 1433 + 1434 1434 + sin6 = (struct sockaddr_in6 *)&csk->com.remote_addr; 1435 1435 + sin6->sin6_family = PF_INET6; 1436 1436 + sin6->sin6_port = peer_port; 1437 1437 + memcpy(sin6->sin6_addr.s6_addr, peer_ip, 16); 1438 1438 + } 1439 1439 + 1440 1440 + skb_queue_head_init(&csk->rxq); 1441 1441 + skb_queue_head_init(&csk->txq); 1442 1442 + skb_queue_head_init(&csk->ppodq); 1443 1443 + skb_queue_head_init(&csk->backlogq); 1444 1444 + skb_queue_head_init(&csk->skbq); 1445 1445 + cxgbit_sock_reset_wr_list(csk); 1446 1446 + spin_lock_init(&csk->lock); 1447 1447 + init_waitqueue_head(&csk->waitq); 1448 1448 + init_waitqueue_head(&csk->ack_waitq); 1449 1449 + csk->lock_owner = false; 1450 1450 + 1451 1451 + if (cxgbit_alloc_csk_skb(csk)) { 1452 1452 + dst_release(dst); 1453 1453 + kfree(csk); 1454 1454 + goto rel_skb; 1455 1455 + } 1456 1456 + 1457 1457 + cxgbit_get_cdev(cdev); 1458 1458 + 1459 1459 + spin_lock(&cdev->cskq.lock); 1460 1460 + list_add_tail(&csk->list, &cdev->cskq.list); 1461 1461 + spin_unlock(&cdev->cskq.lock); 1462 1462 + 1463 1463 + cxgb4_insert_tid(t, csk, tid); 1464 1464 + cxgbit_pass_accept_rpl(csk, req); 1465 1465 + goto rel_skb; 1466 1466 + 1467 1467 + reject: 1468 1468 + cxgbit_release_tid(cdev, tid); 1469 1469 + rel_skb: 1470 1470 + __kfree_skb(skb); 1471 1471 + } 1472 1472 + 1473 1473 + static u32 1474 1474 + cxgbit_tx_flowc_wr_credits(struct cxgbit_sock *csk, u32 *nparamsp, 1475 1475 + u32 *flowclenp) 1476 1476 + { 1477 1477 + u32 nparams, flowclen16, flowclen; 1478 1478 + 1479 1479 + nparams = FLOWC_WR_NPARAMS_MIN; 1480 1480 + 1481 1481 + if (csk->snd_wscale) 1482 1482 + nparams++; 1483 1483 + 1484 1484 + #ifdef CONFIG_CHELSIO_T4_DCB 1485 1485 + nparams++; 1486 1486 + #endif 1487 1487 + flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]); 1488 1488 + flowclen16 = DIV_ROUND_UP(flowclen, 16); 1489 1489 + flowclen = flowclen16 * 16; 1490 1490 + /* 1491 1491 + * Return the number of 16-byte credits used by the flowc request. 1492 1492 + * Pass back the nparams and actual flowc length if requested. 1493 1493 + */ 1494 1494 + if (nparamsp) 1495 1495 + *nparamsp = nparams; 1496 1496 + if (flowclenp) 1497 1497 + *flowclenp = flowclen; 1498 1498 + return flowclen16; 1499 1499 + } 1500 1500 + 1501 1501 + u32 cxgbit_send_tx_flowc_wr(struct cxgbit_sock *csk) 1502 1502 + { 1503 1503 + struct cxgbit_device *cdev = csk->com.cdev; 1504 1504 + struct fw_flowc_wr *flowc; 1505 1505 + u32 nparams, flowclen16, flowclen; 1506 1506 + struct sk_buff *skb; 1507 1507 + u8 index; 1508 1508 + 1509 1509 + #ifdef CONFIG_CHELSIO_T4_DCB 1510 1510 + u16 vlan = ((struct l2t_entry *)csk->l2t)->vlan; 1511 1511 + #endif 1512 1512 + 1513 1513 + flowclen16 = cxgbit_tx_flowc_wr_credits(csk, &nparams, &flowclen); 1514 1514 + 1515 1515 + skb = __skb_dequeue(&csk->skbq); 1516 1516 + flowc = (struct fw_flowc_wr *)__skb_put(skb, flowclen); 1517 1517 + memset(flowc, 0, flowclen); 1518 1518 + 1519 1519 + flowc->op_to_nparams = cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) | 1520 1520 + FW_FLOWC_WR_NPARAMS_V(nparams)); 1521 1521 + flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(flowclen16) | 1522 1522 + FW_WR_FLOWID_V(csk->tid)); 1523 1523 + flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 1524 1524 + flowc->mnemval[0].val = cpu_to_be32(FW_PFVF_CMD_PFN_V 1525 1525 + (csk->com.cdev->lldi.pf)); 1526 1526 + flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 1527 1527 + flowc->mnemval[1].val = cpu_to_be32(csk->tx_chan); 1528 1528 + flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 1529 1529 + flowc->mnemval[2].val = cpu_to_be32(csk->tx_chan); 1530 1530 + flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 1531 1531 + flowc->mnemval[3].val = cpu_to_be32(csk->rss_qid); 1532 1532 + flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 1533 1533 + flowc->mnemval[4].val = cpu_to_be32(csk->snd_nxt); 1534 1534 + flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 1535 1535 + flowc->mnemval[5].val = cpu_to_be32(csk->rcv_nxt); 1536 1536 + flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 1537 1537 + flowc->mnemval[6].val = cpu_to_be32(csk->snd_win); 1538 1538 + flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 1539 1539 + flowc->mnemval[7].val = cpu_to_be32(csk->emss); 1540 1540 + 1541 1541 + flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX; 1542 1542 + if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) 1543 1543 + flowc->mnemval[8].val = cpu_to_be32(CXGBIT_MAX_ISO_PAYLOAD); 1544 1544 + else 1545 1545 + flowc->mnemval[8].val = cpu_to_be32(16384); 1546 1546 + 1547 1547 + index = 9; 1548 1548 + 1549 1549 + if (csk->snd_wscale) { 1550 1550 + flowc->mnemval[index].mnemonic = FW_FLOWC_MNEM_RCV_SCALE; 1551 1551 + flowc->mnemval[index].val = cpu_to_be32(csk->snd_wscale); 1552 1552 + index++; 1553 1553 + } 1554 1554 + 1555 1555 + #ifdef CONFIG_CHELSIO_T4_DCB 1556 1556 + flowc->mnemval[index].mnemonic = FW_FLOWC_MNEM_DCBPRIO; 1557 1557 + if (vlan == VLAN_NONE) { 1558 1558 + pr_warn("csk %u without VLAN Tag on DCB Link\n", csk->tid); 1559 1559 + flowc->mnemval[index].val = cpu_to_be32(0); 1560 1560 + } else 1561 1561 + flowc->mnemval[index].val = cpu_to_be32( 1562 1562 + (vlan & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT); 1563 1563 + #endif 1564 1564 + 1565 1565 + pr_debug("%s: csk %p; tx_chan = %u; rss_qid = %u; snd_seq = %u;" 1566 1566 + " rcv_seq = %u; snd_win = %u; emss = %u\n", 1567 1567 + __func__, csk, csk->tx_chan, csk->rss_qid, csk->snd_nxt, 1568 1568 + csk->rcv_nxt, csk->snd_win, csk->emss); 1569 1569 + set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 1570 1570 + cxgbit_ofld_send(csk->com.cdev, skb); 1571 1571 + return flowclen16; 1572 1572 + } 1573 1573 + 1574 1574 + int cxgbit_setup_conn_digest(struct cxgbit_sock *csk) 1575 1575 + { 1576 1576 + struct sk_buff *skb; 1577 1577 + struct cpl_set_tcb_field *req; 1578 1578 + u8 hcrc = csk->submode & CXGBIT_SUBMODE_HCRC; 1579 1579 + u8 dcrc = csk->submode & CXGBIT_SUBMODE_DCRC; 1580 1580 + unsigned int len = roundup(sizeof(*req), 16); 1581 1581 + int ret; 1582 1582 + 1583 1583 + skb = alloc_skb(len, GFP_KERNEL); 1584 1584 + if (!skb) 1585 1585 + return -ENOMEM; 1586 1586 + 1587 1587 + /* set up ulp submode */ 1588 1588 + req = (struct cpl_set_tcb_field *)__skb_put(skb, len); 1589 1589 + memset(req, 0, len); 1590 1590 + 1591 1591 + INIT_TP_WR(req, csk->tid); 1592 1592 + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid)); 1593 1593 + req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid)); 1594 1594 + req->word_cookie = htons(0); 1595 1595 + req->mask = cpu_to_be64(0x3 << 4); 1596 1596 + req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) | 1597 1597 + (dcrc ? ULP_CRC_DATA : 0)) << 4); 1598 1598 + set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx); 1599 1599 + 1600 1600 + cxgbit_get_csk(csk); 1601 1601 + cxgbit_init_wr_wait(&csk->com.wr_wait); 1602 1602 + 1603 1603 + cxgbit_ofld_send(csk->com.cdev, skb); 1604 1604 + 1605 1605 + ret = cxgbit_wait_for_reply(csk->com.cdev, 1606 1606 + &csk->com.wr_wait, 1607 1607 + csk->tid, 5, __func__); 1608 1608 + if (ret) 1609 1609 + return -1; 1610 1610 + 1611 1611 + return 0; 1612 1612 + } 1613 1613 + 1614 1614 + int cxgbit_setup_conn_pgidx(struct cxgbit_sock *csk, u32 pg_idx) 1615 1615 + { 1616 1616 + struct sk_buff *skb; 1617 1617 + struct cpl_set_tcb_field *req; 1618 1618 + unsigned int len = roundup(sizeof(*req), 16); 1619 1619 + int ret; 1620 1620 + 1621 1621 + skb = alloc_skb(len, GFP_KERNEL); 1622 1622 + if (!skb) 1623 1623 + return -ENOMEM; 1624 1624 + 1625 1625 + req = (struct cpl_set_tcb_field *)__skb_put(skb, len); 1626 1626 + memset(req, 0, len); 1627 1627 + 1628 1628 + INIT_TP_WR(req, csk->tid); 1629 1629 + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid)); 1630 1630 + req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid)); 1631 1631 + req->word_cookie = htons(0); 1632 1632 + req->mask = cpu_to_be64(0x3 << 8); 1633 1633 + req->val = cpu_to_be64(pg_idx << 8); 1634 1634 + set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx); 1635 1635 + 1636 1636 + cxgbit_get_csk(csk); 1637 1637 + cxgbit_init_wr_wait(&csk->com.wr_wait); 1638 1638 + 1639 1639 + cxgbit_ofld_send(csk->com.cdev, skb); 1640 1640 + 1641 1641 + ret = cxgbit_wait_for_reply(csk->com.cdev, 1642 1642 + &csk->com.wr_wait, 1643 1643 + csk->tid, 5, __func__); 1644 1644 + if (ret) 1645 1645 + return -1; 1646 1646 + 1647 1647 + return 0; 1648 1648 + } 1649 1649 + 1650 1650 + static void 1651 1651 + cxgbit_pass_open_rpl(struct cxgbit_device *cdev, struct sk_buff *skb) 1652 1652 + { 1653 1653 + struct cpl_pass_open_rpl *rpl = cplhdr(skb); 1654 1654 + struct tid_info *t = cdev->lldi.tids; 1655 1655 + unsigned int stid = GET_TID(rpl); 1656 1656 + struct cxgbit_np *cnp = lookup_stid(t, stid); 1657 1657 + 1658 1658 + pr_debug("%s: cnp = %p; stid = %u; status = %d\n", 1659 1659 + __func__, cnp, stid, rpl->status); 1660 1660 + 1661 1661 + if (!cnp) { 1662 1662 + pr_info("%s stid %d lookup failure\n", __func__, stid); 1663 1663 + return; 1664 1664 + } 1665 1665 + 1666 1666 + cxgbit_wake_up(&cnp->com.wr_wait, __func__, rpl->status); 1667 1667 + cxgbit_put_cnp(cnp); 1668 1668 + } 1669 1669 + 1670 1670 + static void 1671 1671 + cxgbit_close_listsrv_rpl(struct cxgbit_device *cdev, struct sk_buff *skb) 1672 1672 + { 1673 1673 + struct cpl_close_listsvr_rpl *rpl = cplhdr(skb); 1674 1674 + struct tid_info *t = cdev->lldi.tids; 1675 1675 + unsigned int stid = GET_TID(rpl); 1676 1676 + struct cxgbit_np *cnp = lookup_stid(t, stid); 1677 1677 + 1678 1678 + pr_debug("%s: cnp = %p; stid = %u; status = %d\n", 1679 1679 + __func__, cnp, stid, rpl->status); 1680 1680 + 1681 1681 + if (!cnp) { 1682 1682 + pr_info("%s stid %d lookup failure\n", __func__, stid); 1683 1683 + return; 1684 1684 + } 1685 1685 + 1686 1686 + cxgbit_wake_up(&cnp->com.wr_wait, __func__, rpl->status); 1687 1687 + cxgbit_put_cnp(cnp); 1688 1688 + } 1689 1689 + 1690 1690 + static void 1691 1691 + cxgbit_pass_establish(struct cxgbit_device *cdev, struct sk_buff *skb) 1692 1692 + { 1693 1693 + struct cpl_pass_establish *req = cplhdr(skb); 1694 1694 + struct tid_info *t = cdev->lldi.tids; 1695 1695 + unsigned int tid = GET_TID(req); 1696 1696 + struct cxgbit_sock *csk; 1697 1697 + struct cxgbit_np *cnp; 1698 1698 + u16 tcp_opt = be16_to_cpu(req->tcp_opt); 1699 1699 + u32 snd_isn = be32_to_cpu(req->snd_isn); 1700 1700 + u32 rcv_isn = be32_to_cpu(req->rcv_isn); 1701 1701 + 1702 1702 + csk = lookup_tid(t, tid); 1703 1703 + if (unlikely(!csk)) { 1704 1704 + pr_err("can't find connection for tid %u.\n", tid); 1705 1705 + goto rel_skb; 1706 1706 + } 1707 1707 + cnp = csk->cnp; 1708 1708 + 1709 1709 + pr_debug("%s: csk %p; tid %u; cnp %p\n", 1710 1710 + __func__, csk, tid, cnp); 1711 1711 + 1712 1712 + csk->write_seq = snd_isn; 1713 1713 + csk->snd_una = snd_isn; 1714 1714 + csk->snd_nxt = snd_isn; 1715 1715 + 1716 1716 + csk->rcv_nxt = rcv_isn; 1717 1717 + 1718 1718 + if (csk->rcv_win > (RCV_BUFSIZ_M << 10)) 1719 1719 + csk->rx_credits = (csk->rcv_win - (RCV_BUFSIZ_M << 10)); 1720 1720 + 1721 1721 + csk->snd_wscale = TCPOPT_SND_WSCALE_G(tcp_opt); 1722 1722 + cxgbit_set_emss(csk, tcp_opt); 1723 1723 + dst_confirm(csk->dst); 1724 1724 + csk->com.state = CSK_STATE_ESTABLISHED; 1725 1725 + spin_lock_bh(&cnp->np_accept_lock); 1726 1726 + list_add_tail(&csk->accept_node, &cnp->np_accept_list); 1727 1727 + spin_unlock_bh(&cnp->np_accept_lock); 1728 1728 + complete(&cnp->accept_comp); 1729 1729 + rel_skb: 1730 1730 + __kfree_skb(skb); 1731 1731 + } 1732 1732 + 1733 1733 + static void cxgbit_queue_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1734 1734 + { 1735 1735 + cxgbit_skcb_flags(skb) = 0; 1736 1736 + spin_lock_bh(&csk->rxq.lock); 1737 1737 + __skb_queue_tail(&csk->rxq, skb); 1738 1738 + spin_unlock_bh(&csk->rxq.lock); 1739 1739 + wake_up(&csk->waitq); 1740 1740 + } 1741 1741 + 1742 1742 + static void cxgbit_peer_close(struct cxgbit_sock *csk, struct sk_buff *skb) 1743 1743 + { 1744 1744 + pr_debug("%s: csk %p; tid %u; state %d\n", 1745 1745 + __func__, csk, csk->tid, csk->com.state); 1746 1746 + 1747 1747 + switch (csk->com.state) { 1748 1748 + case CSK_STATE_ESTABLISHED: 1749 1749 + csk->com.state = CSK_STATE_CLOSING; 1750 1750 + cxgbit_queue_rx_skb(csk, skb); 1751 1751 + return; 1752 1752 + case CSK_STATE_CLOSING: 1753 1753 + /* simultaneous close */ 1754 1754 + csk->com.state = CSK_STATE_MORIBUND; 1755 1755 + break; 1756 1756 + case CSK_STATE_MORIBUND: 1757 1757 + csk->com.state = CSK_STATE_DEAD; 1758 1758 + cxgbit_put_csk(csk); 1759 1759 + break; 1760 1760 + case CSK_STATE_ABORTING: 1761 1761 + break; 1762 1762 + default: 1763 1763 + pr_info("%s: cpl_peer_close in bad state %d\n", 1764 1764 + __func__, csk->com.state); 1765 1765 + } 1766 1766 + 1767 1767 + __kfree_skb(skb); 1768 1768 + } 1769 1769 + 1770 1770 + static void cxgbit_close_con_rpl(struct cxgbit_sock *csk, struct sk_buff *skb) 1771 1771 + { 1772 1772 + pr_debug("%s: csk %p; tid %u; state %d\n", 1773 1773 + __func__, csk, csk->tid, csk->com.state); 1774 1774 + 1775 1775 + switch (csk->com.state) { 1776 1776 + case CSK_STATE_CLOSING: 1777 1777 + csk->com.state = CSK_STATE_MORIBUND; 1778 1778 + break; 1779 1779 + case CSK_STATE_MORIBUND: 1780 1780 + csk->com.state = CSK_STATE_DEAD; 1781 1781 + cxgbit_put_csk(csk); 1782 1782 + break; 1783 1783 + case CSK_STATE_ABORTING: 1784 1784 + case CSK_STATE_DEAD: 1785 1785 + break; 1786 1786 + default: 1787 1787 + pr_info("%s: cpl_close_con_rpl in bad state %d\n", 1788 1788 + __func__, csk->com.state); 1789 1789 + } 1790 1790 + 1791 1791 + __kfree_skb(skb); 1792 1792 + } 1793 1793 + 1794 1794 + static void cxgbit_abort_req_rss(struct cxgbit_sock *csk, struct sk_buff *skb) 1795 1795 + { 1796 1796 + struct cpl_abort_req_rss *hdr = cplhdr(skb); 1797 1797 + unsigned int tid = GET_TID(hdr); 1798 1798 + struct cpl_abort_rpl *rpl; 1799 1799 + struct sk_buff *rpl_skb; 1800 1800 + bool release = false; 1801 1801 + bool wakeup_thread = false; 1802 1802 + unsigned int len = roundup(sizeof(*rpl), 16); 1803 1803 + 1804 1804 + pr_debug("%s: csk %p; tid %u; state %d\n", 1805 1805 + __func__, csk, tid, csk->com.state); 1806 1806 + 1807 1807 + if (cxgbit_is_neg_adv(hdr->status)) { 1808 1808 + pr_err("%s: got neg advise %d on tid %u\n", 1809 1809 + __func__, hdr->status, tid); 1810 1810 + goto rel_skb; 1811 1811 + } 1812 1812 + 1813 1813 + switch (csk->com.state) { 1814 1814 + case CSK_STATE_CONNECTING: 1815 1815 + case CSK_STATE_MORIBUND: 1816 1816 + csk->com.state = CSK_STATE_DEAD; 1817 1817 + release = true; 1818 1818 + break; 1819 1819 + case CSK_STATE_ESTABLISHED: 1820 1820 + csk->com.state = CSK_STATE_DEAD; 1821 1821 + wakeup_thread = true; 1822 1822 + break; 1823 1823 + case CSK_STATE_CLOSING: 1824 1824 + csk->com.state = CSK_STATE_DEAD; 1825 1825 + if (!csk->conn) 1826 1826 + release = true; 1827 1827 + break; 1828 1828 + case CSK_STATE_ABORTING: 1829 1829 + break; 1830 1830 + default: 1831 1831 + pr_info("%s: cpl_abort_req_rss in bad state %d\n", 1832 1832 + __func__, csk->com.state); 1833 1833 + csk->com.state = CSK_STATE_DEAD; 1834 1834 + } 1835 1835 + 1836 1836 + __skb_queue_purge(&csk->txq); 1837 1837 + 1838 1838 + if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) 1839 1839 + cxgbit_send_tx_flowc_wr(csk); 1840 1840 + 1841 1841 + rpl_skb = __skb_dequeue(&csk->skbq); 1842 1842 + set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 1843 1843 + 1844 1844 + rpl = (struct cpl_abort_rpl *)__skb_put(rpl_skb, len); 1845 1845 + memset(rpl, 0, len); 1846 1846 + 1847 1847 + INIT_TP_WR(rpl, csk->tid); 1848 1848 + OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, tid)); 1849 1849 + rpl->cmd = CPL_ABORT_NO_RST; 1850 1850 + cxgbit_ofld_send(csk->com.cdev, rpl_skb); 1851 1851 + 1852 1852 + if (wakeup_thread) { 1853 1853 + cxgbit_queue_rx_skb(csk, skb); 1854 1854 + return; 1855 1855 + } 1856 1856 + 1857 1857 + if (release) 1858 1858 + cxgbit_put_csk(csk); 1859 1859 + rel_skb: 1860 1860 + __kfree_skb(skb); 1861 1861 + } 1862 1862 + 1863 1863 + static void cxgbit_abort_rpl_rss(struct cxgbit_sock *csk, struct sk_buff *skb) 1864 1864 + { 1865 1865 + pr_debug("%s: csk %p; tid %u; state %d\n", 1866 1866 + __func__, csk, csk->tid, csk->com.state); 1867 1867 + 1868 1868 + switch (csk->com.state) { 1869 1869 + case CSK_STATE_ABORTING: 1870 1870 + csk->com.state = CSK_STATE_DEAD; 1871 1871 + cxgbit_put_csk(csk); 1872 1872 + break; 1873 1873 + default: 1874 1874 + pr_info("%s: cpl_abort_rpl_rss in state %d\n", 1875 1875 + __func__, csk->com.state); 1876 1876 + } 1877 1877 + 1878 1878 + __kfree_skb(skb); 1879 1879 + } 1880 1880 + 1881 1881 + static bool cxgbit_credit_err(const struct cxgbit_sock *csk) 1882 1882 + { 1883 1883 + const struct sk_buff *skb = csk->wr_pending_head; 1884 1884 + u32 credit = 0; 1885 1885 + 1886 1886 + if (unlikely(csk->wr_cred > csk->wr_max_cred)) { 1887 1887 + pr_err("csk 0x%p, tid %u, credit %u > %u\n", 1888 1888 + csk, csk->tid, csk->wr_cred, csk->wr_max_cred); 1889 1889 + return true; 1890 1890 + } 1891 1891 + 1892 1892 + while (skb) { 1893 1893 + credit += skb->csum; 1894 1894 + skb = cxgbit_skcb_tx_wr_next(skb); 1895 1895 + } 1896 1896 + 1897 1897 + if (unlikely((csk->wr_cred + credit) != csk->wr_max_cred)) { 1898 1898 + pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n", 1899 1899 + csk, csk->tid, csk->wr_cred, 1900 1900 + credit, csk->wr_max_cred); 1901 1901 + 1902 1902 + return true; 1903 1903 + } 1904 1904 + 1905 1905 + return false; 1906 1906 + } 1907 1907 + 1908 1908 + static void cxgbit_fw4_ack(struct cxgbit_sock *csk, struct sk_buff *skb) 1909 1909 + { 1910 1910 + struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)cplhdr(skb); 1911 1911 + u32 credits = rpl->credits; 1912 1912 + u32 snd_una = ntohl(rpl->snd_una); 1913 1913 + 1914 1914 + csk->wr_cred += credits; 1915 1915 + if (csk->wr_una_cred > (csk->wr_max_cred - csk->wr_cred)) 1916 1916 + csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred; 1917 1917 + 1918 1918 + while (credits) { 1919 1919 + struct sk_buff *p = cxgbit_sock_peek_wr(csk); 1920 1920 + 1921 1921 + if (unlikely(!p)) { 1922 1922 + pr_err("csk 0x%p,%u, cr %u,%u+%u, empty.\n", 1923 1923 + csk, csk->tid, credits, 1924 1924 + csk->wr_cred, csk->wr_una_cred); 1925 1925 + break; 1926 1926 + } 1927 1927 + 1928 1928 + if (unlikely(credits < p->csum)) { 1929 1929 + pr_warn("csk 0x%p,%u, cr %u,%u+%u, < %u.\n", 1930 1930 + csk, csk->tid, 1931 1931 + credits, csk->wr_cred, csk->wr_una_cred, 1932 1932 + p->csum); 1933 1933 + p->csum -= credits; 1934 1934 + break; 1935 1935 + } 1936 1936 + 1937 1937 + cxgbit_sock_dequeue_wr(csk); 1938 1938 + credits -= p->csum; 1939 1939 + kfree_skb(p); 1940 1940 + } 1941 1941 + 1942 1942 + if (unlikely(cxgbit_credit_err(csk))) { 1943 1943 + cxgbit_queue_rx_skb(csk, skb); 1944 1944 + return; 1945 1945 + } 1946 1946 + 1947 1947 + if (rpl->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) { 1948 1948 + if (unlikely(before(snd_una, csk->snd_una))) { 1949 1949 + pr_warn("csk 0x%p,%u, snd_una %u/%u.", 1950 1950 + csk, csk->tid, snd_una, 1951 1951 + csk->snd_una); 1952 1952 + goto rel_skb; 1953 1953 + } 1954 1954 + 1955 1955 + if (csk->snd_una != snd_una) { 1956 1956 + csk->snd_una = snd_una; 1957 1957 + dst_confirm(csk->dst); 1958 1958 + wake_up(&csk->ack_waitq); 1959 1959 + } 1960 1960 + } 1961 1961 + 1962 1962 + if (skb_queue_len(&csk->txq)) 1963 1963 + cxgbit_push_tx_frames(csk); 1964 1964 + 1965 1965 + rel_skb: 1966 1966 + __kfree_skb(skb); 1967 1967 + } 1968 1968 + 1969 1969 + static void cxgbit_set_tcb_rpl(struct cxgbit_device *cdev, struct sk_buff *skb) 1970 1970 + { 1971 1971 + struct cxgbit_sock *csk; 1972 1972 + struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data; 1973 1973 + unsigned int tid = GET_TID(rpl); 1974 1974 + struct cxgb4_lld_info *lldi = &cdev->lldi; 1975 1975 + struct tid_info *t = lldi->tids; 1976 1976 + 1977 1977 + csk = lookup_tid(t, tid); 1978 1978 + if (unlikely(!csk)) 1979 1979 + pr_err("can't find connection for tid %u.\n", tid); 1980 1980 + else 1981 1981 + cxgbit_wake_up(&csk->com.wr_wait, __func__, rpl->status); 1982 1982 + 1983 1983 + cxgbit_put_csk(csk); 1984 1984 + } 1985 1985 + 1986 1986 + static void cxgbit_rx_data(struct cxgbit_device *cdev, struct sk_buff *skb) 1987 1987 + { 1988 1988 + struct cxgbit_sock *csk; 1989 1989 + struct cpl_rx_data *cpl = cplhdr(skb); 1990 1990 + unsigned int tid = GET_TID(cpl); 1991 1991 + struct cxgb4_lld_info *lldi = &cdev->lldi; 1992 1992 + struct tid_info *t = lldi->tids; 1993 1993 + 1994 1994 + csk = lookup_tid(t, tid); 1995 1995 + if (unlikely(!csk)) { 1996 1996 + pr_err("can't find conn. for tid %u.\n", tid); 1997 1997 + goto rel_skb; 1998 1998 + } 1999 1999 + 2000 2000 + cxgbit_queue_rx_skb(csk, skb); 2001 2001 + return; 2002 2002 + rel_skb: 2003 2003 + __kfree_skb(skb); 2004 2004 + } 2005 2005 + 2006 2006 + static void 2007 2007 + __cxgbit_process_rx_cpl(struct cxgbit_sock *csk, struct sk_buff *skb) 2008 2008 + { 2009 2009 + spin_lock(&csk->lock); 2010 2010 + if (csk->lock_owner) { 2011 2011 + __skb_queue_tail(&csk->backlogq, skb); 2012 2012 + spin_unlock(&csk->lock); 2013 2013 + return; 2014 2014 + } 2015 2015 + 2016 2016 + cxgbit_skcb_rx_backlog_fn(skb)(csk, skb); 2017 2017 + spin_unlock(&csk->lock); 2018 2018 + } 2019 2019 + 2020 2020 + static void cxgbit_process_rx_cpl(struct cxgbit_sock *csk, struct sk_buff *skb) 2021 2021 + { 2022 2022 + cxgbit_get_csk(csk); 2023 2023 + __cxgbit_process_rx_cpl(csk, skb); 2024 2024 + cxgbit_put_csk(csk); 2025 2025 + } 2026 2026 + 2027 2027 + static void cxgbit_rx_cpl(struct cxgbit_device *cdev, struct sk_buff *skb) 2028 2028 + { 2029 2029 + struct cxgbit_sock *csk; 2030 2030 + struct cpl_tx_data *cpl = cplhdr(skb); 2031 2031 + struct cxgb4_lld_info *lldi = &cdev->lldi; 2032 2032 + struct tid_info *t = lldi->tids; 2033 2033 + unsigned int tid = GET_TID(cpl); 2034 2034 + u8 opcode = cxgbit_skcb_rx_opcode(skb); 2035 2035 + bool ref = true; 2036 2036 + 2037 2037 + switch (opcode) { 2038 2038 + case CPL_FW4_ACK: 2039 2039 + cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_fw4_ack; 2040 2040 + ref = false; 2041 2041 + break; 2042 2042 + case CPL_PEER_CLOSE: 2043 2043 + cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_peer_close; 2044 2044 + break; 2045 2045 + case CPL_CLOSE_CON_RPL: 2046 2046 + cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_close_con_rpl; 2047 2047 + break; 2048 2048 + case CPL_ABORT_REQ_RSS: 2049 2049 + cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_abort_req_rss; 2050 2050 + break; 2051 2051 + case CPL_ABORT_RPL_RSS: 2052 2052 + cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_abort_rpl_rss; 2053 2053 + break; 2054 2054 + default: 2055 2055 + goto rel_skb; 2056 2056 + } 2057 2057 + 2058 2058 + csk = lookup_tid(t, tid); 2059 2059 + if (unlikely(!csk)) { 2060 2060 + pr_err("can't find conn. for tid %u.\n", tid); 2061 2061 + goto rel_skb; 2062 2062 + } 2063 2063 + 2064 2064 + if (ref) 2065 2065 + cxgbit_process_rx_cpl(csk, skb); 2066 2066 + else 2067 2067 + __cxgbit_process_rx_cpl(csk, skb); 2068 2068 + 2069 2069 + return; 2070 2070 + rel_skb: 2071 2071 + __kfree_skb(skb); 2072 2072 + } 2073 2073 + 2074 2074 + cxgbit_cplhandler_func cxgbit_cplhandlers[NUM_CPL_CMDS] = { 2075 2075 + [CPL_PASS_OPEN_RPL] = cxgbit_pass_open_rpl, 2076 2076 + [CPL_CLOSE_LISTSRV_RPL] = cxgbit_close_listsrv_rpl, 2077 2077 + [CPL_PASS_ACCEPT_REQ] = cxgbit_pass_accept_req, 2078 2078 + [CPL_PASS_ESTABLISH] = cxgbit_pass_establish, 2079 2079 + [CPL_SET_TCB_RPL] = cxgbit_set_tcb_rpl, 2080 2080 + [CPL_RX_DATA] = cxgbit_rx_data, 2081 2081 + [CPL_FW4_ACK] = cxgbit_rx_cpl, 2082 2082 + [CPL_PEER_CLOSE] = cxgbit_rx_cpl, 2083 2083 + [CPL_CLOSE_CON_RPL] = cxgbit_rx_cpl, 2084 2084 + [CPL_ABORT_REQ_RSS] = cxgbit_rx_cpl, 2085 2085 + [CPL_ABORT_RPL_RSS] = cxgbit_rx_cpl, 2086 2086 + };

+325

drivers/target/iscsi/cxgbit/cxgbit_ddp.c

reviewed

··· 1 1 + /* 2 2 + * Copyright (c) 2016 Chelsio Communications, Inc. 3 3 + * 4 4 + * This program is free software; you can redistribute it and/or modify 5 5 + * it under the terms of the GNU General Public License version 2 as 6 6 + * published by the Free Software Foundation. 7 7 + */ 8 8 + 9 9 + #include "cxgbit.h" 10 10 + 11 11 + static void 12 12 + cxgbit_set_one_ppod(struct cxgbi_pagepod *ppod, 13 13 + struct cxgbi_task_tag_info *ttinfo, 14 14 + struct scatterlist **sg_pp, unsigned int *sg_off) 15 15 + { 16 16 + struct scatterlist *sg = sg_pp ? *sg_pp : NULL; 17 17 + unsigned int offset = sg_off ? *sg_off : 0; 18 18 + dma_addr_t addr = 0UL; 19 19 + unsigned int len = 0; 20 20 + int i; 21 21 + 22 22 + memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr)); 23 23 + 24 24 + if (sg) { 25 25 + addr = sg_dma_address(sg); 26 26 + len = sg_dma_len(sg); 27 27 + } 28 28 + 29 29 + for (i = 0; i < PPOD_PAGES_MAX; i++) { 30 30 + if (sg) { 31 31 + ppod->addr[i] = cpu_to_be64(addr + offset); 32 32 + offset += PAGE_SIZE; 33 33 + if (offset == (len + sg->offset)) { 34 34 + offset = 0; 35 35 + sg = sg_next(sg); 36 36 + if (sg) { 37 37 + addr = sg_dma_address(sg); 38 38 + len = sg_dma_len(sg); 39 39 + } 40 40 + } 41 41 + } else { 42 42 + ppod->addr[i] = 0ULL; 43 43 + } 44 44 + } 45 45 + 46 46 + /* 47 47 + * the fifth address needs to be repeated in the next ppod, so do 48 48 + * not move sg 49 49 + */ 50 50 + if (sg_pp) { 51 51 + *sg_pp = sg; 52 52 + *sg_off = offset; 53 53 + } 54 54 + 55 55 + if (offset == len) { 56 56 + offset = 0; 57 57 + if (sg) { 58 58 + sg = sg_next(sg); 59 59 + if (sg) 60 60 + addr = sg_dma_address(sg); 61 61 + } 62 62 + } 63 63 + ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL; 64 64 + } 65 65 + 66 66 + static struct sk_buff * 67 67 + cxgbit_ppod_init_idata(struct cxgbit_device *cdev, struct cxgbi_ppm *ppm, 68 68 + unsigned int idx, unsigned int npods, unsigned int tid) 69 69 + { 70 70 + struct ulp_mem_io *req; 71 71 + struct ulptx_idata *idata; 72 72 + unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit; 73 73 + unsigned int dlen = npods << PPOD_SIZE_SHIFT; 74 74 + unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) + 75 75 + sizeof(struct ulptx_idata) + dlen, 16); 76 76 + struct sk_buff *skb; 77 77 + 78 78 + skb = alloc_skb(wr_len, GFP_KERNEL); 79 79 + if (!skb) 80 80 + return NULL; 81 81 + 82 82 + req = (struct ulp_mem_io *)__skb_put(skb, wr_len); 83 83 + INIT_ULPTX_WR(req, wr_len, 0, tid); 84 84 + req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) | 85 85 + FW_WR_ATOMIC_V(0)); 86 86 + req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) | 87 87 + ULP_MEMIO_ORDER_V(0) | 88 88 + T5_ULP_MEMIO_IMM_V(1)); 89 89 + req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5)); 90 90 + req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5)); 91 91 + req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16)); 92 92 + 93 93 + idata = (struct ulptx_idata *)(req + 1); 94 94 + idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); 95 95 + idata->len = htonl(dlen); 96 96 + 97 97 + return skb; 98 98 + } 99 99 + 100 100 + static int 101 101 + cxgbit_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk, 102 102 + struct cxgbi_task_tag_info *ttinfo, unsigned int idx, 103 103 + unsigned int npods, struct scatterlist **sg_pp, 104 104 + unsigned int *sg_off) 105 105 + { 106 106 + struct cxgbit_device *cdev = csk->com.cdev; 107 107 + struct sk_buff *skb; 108 108 + struct ulp_mem_io *req; 109 109 + struct ulptx_idata *idata; 110 110 + struct cxgbi_pagepod *ppod; 111 111 + unsigned int i; 112 112 + 113 113 + skb = cxgbit_ppod_init_idata(cdev, ppm, idx, npods, csk->tid); 114 114 + if (!skb) 115 115 + return -ENOMEM; 116 116 + 117 117 + req = (struct ulp_mem_io *)skb->data; 118 118 + idata = (struct ulptx_idata *)(req + 1); 119 119 + ppod = (struct cxgbi_pagepod *)(idata + 1); 120 120 + 121 121 + for (i = 0; i < npods; i++, ppod++) 122 122 + cxgbit_set_one_ppod(ppod, ttinfo, sg_pp, sg_off); 123 123 + 124 124 + __skb_queue_tail(&csk->ppodq, skb); 125 125 + 126 126 + return 0; 127 127 + } 128 128 + 129 129 + static int 130 130 + cxgbit_ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk, 131 131 + struct cxgbi_task_tag_info *ttinfo) 132 132 + { 133 133 + unsigned int pidx = ttinfo->idx; 134 134 + unsigned int npods = ttinfo->npods; 135 135 + unsigned int i, cnt; 136 136 + struct scatterlist *sg = ttinfo->sgl; 137 137 + unsigned int offset = 0; 138 138 + int ret = 0; 139 139 + 140 140 + for (i = 0; i < npods; i += cnt, pidx += cnt) { 141 141 + cnt = npods - i; 142 142 + 143 143 + if (cnt > ULPMEM_IDATA_MAX_NPPODS) 144 144 + cnt = ULPMEM_IDATA_MAX_NPPODS; 145 145 + 146 146 + ret = cxgbit_ppod_write_idata(ppm, csk, ttinfo, pidx, cnt, 147 147 + &sg, &offset); 148 148 + if (ret < 0) 149 149 + break; 150 150 + } 151 151 + 152 152 + return ret; 153 153 + } 154 154 + 155 155 + static int cxgbit_ddp_sgl_check(struct scatterlist *sg, 156 156 + unsigned int nents) 157 157 + { 158 158 + unsigned int last_sgidx = nents - 1; 159 159 + unsigned int i; 160 160 + 161 161 + for (i = 0; i < nents; i++, sg = sg_next(sg)) { 162 162 + unsigned int len = sg->length + sg->offset; 163 163 + 164 164 + if ((sg->offset & 0x3) || (i && sg->offset) || 165 165 + ((i != last_sgidx) && (len != PAGE_SIZE))) { 166 166 + return -EINVAL; 167 167 + } 168 168 + } 169 169 + 170 170 + return 0; 171 171 + } 172 172 + 173 173 + static int 174 174 + cxgbit_ddp_reserve(struct cxgbit_sock *csk, struct cxgbi_task_tag_info *ttinfo, 175 175 + unsigned int xferlen) 176 176 + { 177 177 + struct cxgbit_device *cdev = csk->com.cdev; 178 178 + struct cxgbi_ppm *ppm = cdev2ppm(cdev); 179 179 + struct scatterlist *sgl = ttinfo->sgl; 180 180 + unsigned int sgcnt = ttinfo->nents; 181 181 + unsigned int sg_offset = sgl->offset; 182 182 + int ret; 183 183 + 184 184 + if ((xferlen < DDP_THRESHOLD) || (!sgcnt)) { 185 185 + pr_debug("ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n", 186 186 + ppm, ppm->tformat.pgsz_idx_dflt, 187 187 + xferlen, ttinfo->nents); 188 188 + return -EINVAL; 189 189 + } 190 190 + 191 191 + if (cxgbit_ddp_sgl_check(sgl, sgcnt) < 0) 192 192 + return -EINVAL; 193 193 + 194 194 + ttinfo->nr_pages = (xferlen + sgl->offset + 195 195 + (1 << PAGE_SHIFT) - 1) >> PAGE_SHIFT; 196 196 + 197 197 + /* 198 198 + * the ddp tag will be used for the ttt in the outgoing r2t pdu 199 199 + */ 200 200 + ret = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx, 201 201 + &ttinfo->tag, 0); 202 202 + if (ret < 0) 203 203 + return ret; 204 204 + ttinfo->npods = ret; 205 205 + 206 206 + sgl->offset = 0; 207 207 + ret = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE); 208 208 + sgl->offset = sg_offset; 209 209 + if (!ret) { 210 210 + pr_info("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n", 211 211 + __func__, 0, xferlen, sgcnt); 212 212 + goto rel_ppods; 213 213 + } 214 214 + 215 215 + cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset, 216 216 + xferlen, &ttinfo->hdr); 217 217 + 218 218 + ret = cxgbit_ddp_set_map(ppm, csk, ttinfo); 219 219 + if (ret < 0) { 220 220 + __skb_queue_purge(&csk->ppodq); 221 221 + dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE); 222 222 + goto rel_ppods; 223 223 + } 224 224 + 225 225 + return 0; 226 226 + 227 227 + rel_ppods: 228 228 + cxgbi_ppm_ppod_release(ppm, ttinfo->idx); 229 229 + return -EINVAL; 230 230 + } 231 231 + 232 232 + void 233 233 + cxgbit_get_r2t_ttt(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 234 234 + struct iscsi_r2t *r2t) 235 235 + { 236 236 + struct cxgbit_sock *csk = conn->context; 237 237 + struct cxgbit_device *cdev = csk->com.cdev; 238 238 + struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 239 239 + struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo; 240 240 + int ret = -EINVAL; 241 241 + 242 242 + if ((!ccmd->setup_ddp) || 243 243 + (!test_bit(CSK_DDP_ENABLE, &csk->com.flags))) 244 244 + goto out; 245 245 + 246 246 + ccmd->setup_ddp = false; 247 247 + 248 248 + ttinfo->sgl = cmd->se_cmd.t_data_sg; 249 249 + ttinfo->nents = cmd->se_cmd.t_data_nents; 250 250 + 251 251 + ret = cxgbit_ddp_reserve(csk, ttinfo, cmd->se_cmd.data_length); 252 252 + if (ret < 0) { 253 253 + pr_info("csk 0x%p, cmd 0x%p, xfer len %u, sgcnt %u no ddp.\n", 254 254 + csk, cmd, cmd->se_cmd.data_length, ttinfo->nents); 255 255 + 256 256 + ttinfo->sgl = NULL; 257 257 + ttinfo->nents = 0; 258 258 + } else { 259 259 + ccmd->release = true; 260 260 + } 261 261 + out: 262 262 + pr_debug("cdev 0x%p, cmd 0x%p, tag 0x%x\n", cdev, cmd, ttinfo->tag); 263 263 + r2t->targ_xfer_tag = ttinfo->tag; 264 264 + } 265 265 + 266 266 + void cxgbit_release_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 267 267 + { 268 268 + struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 269 269 + 270 270 + if (ccmd->release) { 271 271 + struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo; 272 272 + 273 273 + if (ttinfo->sgl) { 274 274 + struct cxgbit_sock *csk = conn->context; 275 275 + struct cxgbit_device *cdev = csk->com.cdev; 276 276 + struct cxgbi_ppm *ppm = cdev2ppm(cdev); 277 277 + 278 278 + cxgbi_ppm_ppod_release(ppm, ttinfo->idx); 279 279 + 280 280 + dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl, 281 281 + ttinfo->nents, DMA_FROM_DEVICE); 282 282 + } else { 283 283 + put_page(sg_page(&ccmd->sg)); 284 284 + } 285 285 + 286 286 + ccmd->release = false; 287 287 + } 288 288 + } 289 289 + 290 290 + int cxgbit_ddp_init(struct cxgbit_device *cdev) 291 291 + { 292 292 + struct cxgb4_lld_info *lldi = &cdev->lldi; 293 293 + struct net_device *ndev = cdev->lldi.ports[0]; 294 294 + struct cxgbi_tag_format tformat; 295 295 + unsigned int ppmax; 296 296 + int ret, i; 297 297 + 298 298 + if (!lldi->vr->iscsi.size) { 299 299 + pr_warn("%s, iscsi NOT enabled, check config!\n", ndev->name); 300 300 + return -EACCES; 301 301 + } 302 302 + 303 303 + ppmax = lldi->vr->iscsi.size >> PPOD_SIZE_SHIFT; 304 304 + 305 305 + memset(&tformat, 0, sizeof(struct cxgbi_tag_format)); 306 306 + for (i = 0; i < 4; i++) 307 307 + tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3)) 308 308 + & 0xF; 309 309 + cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat); 310 310 + 311 311 + ret = cxgbi_ppm_init(lldi->iscsi_ppm, cdev->lldi.ports[0], 312 312 + cdev->lldi.pdev, &cdev->lldi, &tformat, 313 313 + ppmax, lldi->iscsi_llimit, 314 314 + lldi->vr->iscsi.start, 2); 315 315 + if (ret >= 0) { 316 316 + struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*lldi->iscsi_ppm); 317 317 + 318 318 + if ((ppm->tformat.pgsz_idx_dflt < DDP_PGIDX_MAX) && 319 319 + (ppm->ppmax >= 1024)) 320 320 + set_bit(CDEV_DDP_ENABLE, &cdev->flags); 321 321 + ret = 0; 322 322 + } 323 323 + 324 324 + return ret; 325 325 + }

+72

drivers/target/iscsi/cxgbit/cxgbit_lro.h

reviewed

··· 1 1 + /* 2 2 + * Copyright (c) 2016 Chelsio Communications, Inc. 3 3 + * 4 4 + * This program is free software; you can redistribute it and/or modify 5 5 + * it under the terms of the GNU General Public License as published by 6 6 + * the Free Software Foundation. 7 7 + * 8 8 + */ 9 9 + 10 10 + #ifndef __CXGBIT_LRO_H__ 11 11 + #define __CXGBIT_LRO_H__ 12 12 + 13 13 + #include <linux/kernel.h> 14 14 + #include <linux/module.h> 15 15 + #include <linux/errno.h> 16 16 + #include <linux/types.h> 17 17 + #include <linux/skbuff.h> 18 18 + 19 19 + #define LRO_FLUSH_LEN_MAX 65535 20 20 + 21 21 + struct cxgbit_lro_cb { 22 22 + struct cxgbit_sock *csk; 23 23 + u32 pdu_totallen; 24 24 + u32 offset; 25 25 + u8 pdu_idx; 26 26 + bool complete; 27 27 + }; 28 28 + 29 29 + enum cxgbit_pducb_flags { 30 30 + PDUCBF_RX_HDR = (1 << 0), /* received pdu header */ 31 31 + PDUCBF_RX_DATA = (1 << 1), /* received pdu payload */ 32 32 + PDUCBF_RX_STATUS = (1 << 2), /* received ddp status */ 33 33 + PDUCBF_RX_DATA_DDPD = (1 << 3), /* pdu payload ddp'd */ 34 34 + PDUCBF_RX_HCRC_ERR = (1 << 4), /* header digest error */ 35 35 + PDUCBF_RX_DCRC_ERR = (1 << 5), /* data digest error */ 36 36 + }; 37 37 + 38 38 + struct cxgbit_lro_pdu_cb { 39 39 + u8 flags; 40 40 + u8 frags; 41 41 + u8 hfrag_idx; 42 42 + u8 nr_dfrags; 43 43 + u8 dfrag_idx; 44 44 + bool complete; 45 45 + u32 seq; 46 46 + u32 pdulen; 47 47 + u32 hlen; 48 48 + u32 dlen; 49 49 + u32 doffset; 50 50 + u32 ddigest; 51 51 + void *hdr; 52 52 + }; 53 53 + 54 54 + #define LRO_SKB_MAX_HEADROOM \ 55 55 + (sizeof(struct cxgbit_lro_cb) + \ 56 56 + (MAX_SKB_FRAGS * sizeof(struct cxgbit_lro_pdu_cb))) 57 57 + 58 58 + #define LRO_SKB_MIN_HEADROOM \ 59 59 + (sizeof(struct cxgbit_lro_cb) + \ 60 60 + sizeof(struct cxgbit_lro_pdu_cb)) 61 61 + 62 62 + #define cxgbit_skb_lro_cb(skb) ((struct cxgbit_lro_cb *)skb->data) 63 63 + #define cxgbit_skb_lro_pdu_cb(skb, i) \ 64 64 + ((struct cxgbit_lro_pdu_cb *)(skb->data + sizeof(struct cxgbit_lro_cb) \ 65 65 + + (i * sizeof(struct cxgbit_lro_pdu_cb)))) 66 66 + 67 67 + #define CPL_RX_ISCSI_DDP_STATUS_DDP_SHIFT 16 /* ddp'able */ 68 68 + #define CPL_RX_ISCSI_DDP_STATUS_PAD_SHIFT 19 /* pad error */ 69 69 + #define CPL_RX_ISCSI_DDP_STATUS_HCRC_SHIFT 20 /* hcrc error */ 70 70 + #define CPL_RX_ISCSI_DDP_STATUS_DCRC_SHIFT 21 /* dcrc error */ 71 71 + 72 72 + #endif /*__CXGBIT_LRO_H_*/

+702

drivers/target/iscsi/cxgbit/cxgbit_main.c

reviewed

··· 1 1 + /* 2 2 + * Copyright (c) 2016 Chelsio Communications, Inc. 3 3 + * 4 4 + * This program is free software; you can redistribute it and/or modify 5 5 + * it under the terms of the GNU General Public License version 2 as 6 6 + * published by the Free Software Foundation. 7 7 + */ 8 8 + 9 9 + #define DRV_NAME "cxgbit" 10 10 + #define DRV_VERSION "1.0.0-ko" 11 11 + #define pr_fmt(fmt) DRV_NAME ": " fmt 12 12 + 13 13 + #include "cxgbit.h" 14 14 + 15 15 + #ifdef CONFIG_CHELSIO_T4_DCB 16 16 + #include <net/dcbevent.h> 17 17 + #include "cxgb4_dcb.h" 18 18 + #endif 19 19 + 20 20 + LIST_HEAD(cdev_list_head); 21 21 + /* cdev list lock */ 22 22 + DEFINE_MUTEX(cdev_list_lock); 23 23 + 24 24 + void _cxgbit_free_cdev(struct kref *kref) 25 25 + { 26 26 + struct cxgbit_device *cdev; 27 27 + 28 28 + cdev = container_of(kref, struct cxgbit_device, kref); 29 29 + kfree(cdev); 30 30 + } 31 31 + 32 32 + static void cxgbit_set_mdsl(struct cxgbit_device *cdev) 33 33 + { 34 34 + struct cxgb4_lld_info *lldi = &cdev->lldi; 35 35 + u32 mdsl; 36 36 + 37 37 + #define ULP2_MAX_PKT_LEN 16224 38 38 + #define ISCSI_PDU_NONPAYLOAD_LEN 312 39 39 + mdsl = min_t(u32, lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN, 40 40 + ULP2_MAX_PKT_LEN - ISCSI_PDU_NONPAYLOAD_LEN); 41 41 + mdsl = min_t(u32, mdsl, 8192); 42 42 + mdsl = min_t(u32, mdsl, (MAX_SKB_FRAGS - 1) * PAGE_SIZE); 43 43 + 44 44 + cdev->mdsl = mdsl; 45 45 + } 46 46 + 47 47 + static void *cxgbit_uld_add(const struct cxgb4_lld_info *lldi) 48 48 + { 49 49 + struct cxgbit_device *cdev; 50 50 + 51 51 + if (is_t4(lldi->adapter_type)) 52 52 + return ERR_PTR(-ENODEV); 53 53 + 54 54 + cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); 55 55 + if (!cdev) 56 56 + return ERR_PTR(-ENOMEM); 57 57 + 58 58 + kref_init(&cdev->kref); 59 59 + 60 60 + cdev->lldi = *lldi; 61 61 + 62 62 + cxgbit_set_mdsl(cdev); 63 63 + 64 64 + if (cxgbit_ddp_init(cdev) < 0) { 65 65 + kfree(cdev); 66 66 + return ERR_PTR(-EINVAL); 67 67 + } 68 68 + 69 69 + if (!test_bit(CDEV_DDP_ENABLE, &cdev->flags)) 70 70 + pr_info("cdev %s ddp init failed\n", 71 71 + pci_name(lldi->pdev)); 72 72 + 73 73 + if (lldi->fw_vers >= 0x10d2b00) 74 74 + set_bit(CDEV_ISO_ENABLE, &cdev->flags); 75 75 + 76 76 + spin_lock_init(&cdev->cskq.lock); 77 77 + INIT_LIST_HEAD(&cdev->cskq.list); 78 78 + 79 79 + mutex_lock(&cdev_list_lock); 80 80 + list_add_tail(&cdev->list, &cdev_list_head); 81 81 + mutex_unlock(&cdev_list_lock); 82 82 + 83 83 + pr_info("cdev %s added for iSCSI target transport\n", 84 84 + pci_name(lldi->pdev)); 85 85 + 86 86 + return cdev; 87 87 + } 88 88 + 89 89 + static void cxgbit_close_conn(struct cxgbit_device *cdev) 90 90 + { 91 91 + struct cxgbit_sock *csk; 92 92 + struct sk_buff *skb; 93 93 + bool wakeup_thread = false; 94 94 + 95 95 + spin_lock_bh(&cdev->cskq.lock); 96 96 + list_for_each_entry(csk, &cdev->cskq.list, list) { 97 97 + skb = alloc_skb(0, GFP_ATOMIC); 98 98 + if (!skb) 99 99 + continue; 100 100 + 101 101 + spin_lock_bh(&csk->rxq.lock); 102 102 + __skb_queue_tail(&csk->rxq, skb); 103 103 + if (skb_queue_len(&csk->rxq) == 1) 104 104 + wakeup_thread = true; 105 105 + spin_unlock_bh(&csk->rxq.lock); 106 106 + 107 107 + if (wakeup_thread) { 108 108 + wake_up(&csk->waitq); 109 109 + wakeup_thread = false; 110 110 + } 111 111 + } 112 112 + spin_unlock_bh(&cdev->cskq.lock); 113 113 + } 114 114 + 115 115 + static void cxgbit_detach_cdev(struct cxgbit_device *cdev) 116 116 + { 117 117 + bool free_cdev = false; 118 118 + 119 119 + spin_lock_bh(&cdev->cskq.lock); 120 120 + if (list_empty(&cdev->cskq.list)) 121 121 + free_cdev = true; 122 122 + spin_unlock_bh(&cdev->cskq.lock); 123 123 + 124 124 + if (free_cdev) { 125 125 + mutex_lock(&cdev_list_lock); 126 126 + list_del(&cdev->list); 127 127 + mutex_unlock(&cdev_list_lock); 128 128 + 129 129 + cxgbit_put_cdev(cdev); 130 130 + } else { 131 131 + cxgbit_close_conn(cdev); 132 132 + } 133 133 + } 134 134 + 135 135 + static int cxgbit_uld_state_change(void *handle, enum cxgb4_state state) 136 136 + { 137 137 + struct cxgbit_device *cdev = handle; 138 138 + 139 139 + switch (state) { 140 140 + case CXGB4_STATE_UP: 141 141 + set_bit(CDEV_STATE_UP, &cdev->flags); 142 142 + pr_info("cdev %s state UP.\n", pci_name(cdev->lldi.pdev)); 143 143 + break; 144 144 + case CXGB4_STATE_START_RECOVERY: 145 145 + clear_bit(CDEV_STATE_UP, &cdev->flags); 146 146 + cxgbit_close_conn(cdev); 147 147 + pr_info("cdev %s state RECOVERY.\n", pci_name(cdev->lldi.pdev)); 148 148 + break; 149 149 + case CXGB4_STATE_DOWN: 150 150 + pr_info("cdev %s state DOWN.\n", pci_name(cdev->lldi.pdev)); 151 151 + break; 152 152 + case CXGB4_STATE_DETACH: 153 153 + clear_bit(CDEV_STATE_UP, &cdev->flags); 154 154 + pr_info("cdev %s state DETACH.\n", pci_name(cdev->lldi.pdev)); 155 155 + cxgbit_detach_cdev(cdev); 156 156 + break; 157 157 + default: 158 158 + pr_info("cdev %s unknown state %d.\n", 159 159 + pci_name(cdev->lldi.pdev), state); 160 160 + break; 161 161 + } 162 162 + return 0; 163 163 + } 164 164 + 165 165 + static void 166 166 + cxgbit_proc_ddp_status(unsigned int tid, struct cpl_rx_data_ddp *cpl, 167 167 + struct cxgbit_lro_pdu_cb *pdu_cb) 168 168 + { 169 169 + unsigned int status = ntohl(cpl->ddpvld); 170 170 + 171 171 + pdu_cb->flags |= PDUCBF_RX_STATUS; 172 172 + pdu_cb->ddigest = ntohl(cpl->ulp_crc); 173 173 + pdu_cb->pdulen = ntohs(cpl->len); 174 174 + 175 175 + if (status & (1 << CPL_RX_ISCSI_DDP_STATUS_HCRC_SHIFT)) { 176 176 + pr_info("tid 0x%x, status 0x%x, hcrc bad.\n", tid, status); 177 177 + pdu_cb->flags |= PDUCBF_RX_HCRC_ERR; 178 178 + } 179 179 + 180 180 + if (status & (1 << CPL_RX_ISCSI_DDP_STATUS_DCRC_SHIFT)) { 181 181 + pr_info("tid 0x%x, status 0x%x, dcrc bad.\n", tid, status); 182 182 + pdu_cb->flags |= PDUCBF_RX_DCRC_ERR; 183 183 + } 184 184 + 185 185 + if (status & (1 << CPL_RX_ISCSI_DDP_STATUS_PAD_SHIFT)) 186 186 + pr_info("tid 0x%x, status 0x%x, pad bad.\n", tid, status); 187 187 + 188 188 + if ((status & (1 << CPL_RX_ISCSI_DDP_STATUS_DDP_SHIFT)) && 189 189 + (!(pdu_cb->flags & PDUCBF_RX_DATA))) { 190 190 + pdu_cb->flags |= PDUCBF_RX_DATA_DDPD; 191 191 + } 192 192 + } 193 193 + 194 194 + static void 195 195 + cxgbit_lro_add_packet_rsp(struct sk_buff *skb, u8 op, const __be64 *rsp) 196 196 + { 197 197 + struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 198 198 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 199 199 + lro_cb->pdu_idx); 200 200 + struct cpl_rx_iscsi_ddp *cpl = (struct cpl_rx_iscsi_ddp *)(rsp + 1); 201 201 + 202 202 + cxgbit_proc_ddp_status(lro_cb->csk->tid, cpl, pdu_cb); 203 203 + 204 204 + if (pdu_cb->flags & PDUCBF_RX_HDR) 205 205 + pdu_cb->complete = true; 206 206 + 207 207 + lro_cb->complete = true; 208 208 + lro_cb->pdu_totallen += pdu_cb->pdulen; 209 209 + lro_cb->pdu_idx++; 210 210 + } 211 211 + 212 212 + static void 213 213 + cxgbit_copy_frags(struct sk_buff *skb, const struct pkt_gl *gl, 214 214 + unsigned int offset) 215 215 + { 216 216 + u8 skb_frag_idx = skb_shinfo(skb)->nr_frags; 217 217 + u8 i; 218 218 + 219 219 + /* usually there's just one frag */ 220 220 + __skb_fill_page_desc(skb, skb_frag_idx, gl->frags[0].page, 221 221 + gl->frags[0].offset + offset, 222 222 + gl->frags[0].size - offset); 223 223 + for (i = 1; i < gl->nfrags; i++) 224 224 + __skb_fill_page_desc(skb, skb_frag_idx + i, 225 225 + gl->frags[i].page, 226 226 + gl->frags[i].offset, 227 227 + gl->frags[i].size); 228 228 + 229 229 + skb_shinfo(skb)->nr_frags += gl->nfrags; 230 230 + 231 231 + /* get a reference to the last page, we don't own it */ 232 232 + get_page(gl->frags[gl->nfrags - 1].page); 233 233 + } 234 234 + 235 235 + static void 236 236 + cxgbit_lro_add_packet_gl(struct sk_buff *skb, u8 op, const struct pkt_gl *gl) 237 237 + { 238 238 + struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 239 239 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 240 240 + lro_cb->pdu_idx); 241 241 + u32 len, offset; 242 242 + 243 243 + if (op == CPL_ISCSI_HDR) { 244 244 + struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)gl->va; 245 245 + 246 246 + offset = sizeof(struct cpl_iscsi_hdr); 247 247 + pdu_cb->flags |= PDUCBF_RX_HDR; 248 248 + pdu_cb->seq = ntohl(cpl->seq); 249 249 + len = ntohs(cpl->len); 250 250 + pdu_cb->hdr = gl->va + offset; 251 251 + pdu_cb->hlen = len; 252 252 + pdu_cb->hfrag_idx = skb_shinfo(skb)->nr_frags; 253 253 + 254 254 + if (unlikely(gl->nfrags > 1)) 255 255 + cxgbit_skcb_flags(skb) = 0; 256 256 + 257 257 + lro_cb->complete = false; 258 258 + } else { 259 259 + struct cpl_iscsi_data *cpl = (struct cpl_iscsi_data *)gl->va; 260 260 + 261 261 + offset = sizeof(struct cpl_iscsi_data); 262 262 + pdu_cb->flags |= PDUCBF_RX_DATA; 263 263 + len = ntohs(cpl->len); 264 264 + pdu_cb->dlen = len; 265 265 + pdu_cb->doffset = lro_cb->offset; 266 266 + pdu_cb->nr_dfrags = gl->nfrags; 267 267 + pdu_cb->dfrag_idx = skb_shinfo(skb)->nr_frags; 268 268 + } 269 269 + 270 270 + cxgbit_copy_frags(skb, gl, offset); 271 271 + 272 272 + pdu_cb->frags += gl->nfrags; 273 273 + lro_cb->offset += len; 274 274 + skb->len += len; 275 275 + skb->data_len += len; 276 276 + skb->truesize += len; 277 277 + } 278 278 + 279 279 + static struct sk_buff * 280 280 + cxgbit_lro_init_skb(struct cxgbit_sock *csk, u8 op, const struct pkt_gl *gl, 281 281 + const __be64 *rsp, struct napi_struct *napi) 282 282 + { 283 283 + struct sk_buff *skb; 284 284 + struct cxgbit_lro_cb *lro_cb; 285 285 + 286 286 + skb = napi_alloc_skb(napi, LRO_SKB_MAX_HEADROOM); 287 287 + 288 288 + if (unlikely(!skb)) 289 289 + return NULL; 290 290 + 291 291 + memset(skb->data, 0, LRO_SKB_MAX_HEADROOM); 292 292 + 293 293 + cxgbit_skcb_flags(skb) |= SKCBF_RX_LRO; 294 294 + 295 295 + lro_cb = cxgbit_skb_lro_cb(skb); 296 296 + 297 297 + cxgbit_get_csk(csk); 298 298 + 299 299 + lro_cb->csk = csk; 300 300 + 301 301 + return skb; 302 302 + } 303 303 + 304 304 + static void cxgbit_queue_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 305 305 + { 306 306 + bool wakeup_thread = false; 307 307 + 308 308 + spin_lock(&csk->rxq.lock); 309 309 + __skb_queue_tail(&csk->rxq, skb); 310 310 + if (skb_queue_len(&csk->rxq) == 1) 311 311 + wakeup_thread = true; 312 312 + spin_unlock(&csk->rxq.lock); 313 313 + 314 314 + if (wakeup_thread) 315 315 + wake_up(&csk->waitq); 316 316 + } 317 317 + 318 318 + static void cxgbit_lro_flush(struct t4_lro_mgr *lro_mgr, struct sk_buff *skb) 319 319 + { 320 320 + struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 321 321 + struct cxgbit_sock *csk = lro_cb->csk; 322 322 + 323 323 + csk->lro_skb = NULL; 324 324 + 325 325 + __skb_unlink(skb, &lro_mgr->lroq); 326 326 + cxgbit_queue_lro_skb(csk, skb); 327 327 + 328 328 + cxgbit_put_csk(csk); 329 329 + 330 330 + lro_mgr->lro_pkts++; 331 331 + lro_mgr->lro_session_cnt--; 332 332 + } 333 333 + 334 334 + static void cxgbit_uld_lro_flush(struct t4_lro_mgr *lro_mgr) 335 335 + { 336 336 + struct sk_buff *skb; 337 337 + 338 338 + while ((skb = skb_peek(&lro_mgr->lroq))) 339 339 + cxgbit_lro_flush(lro_mgr, skb); 340 340 + } 341 341 + 342 342 + static int 343 343 + cxgbit_lro_receive(struct cxgbit_sock *csk, u8 op, const __be64 *rsp, 344 344 + const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr, 345 345 + struct napi_struct *napi) 346 346 + { 347 347 + struct sk_buff *skb; 348 348 + struct cxgbit_lro_cb *lro_cb; 349 349 + 350 350 + if (!csk) { 351 351 + pr_err("%s: csk NULL, op 0x%x.\n", __func__, op); 352 352 + goto out; 353 353 + } 354 354 + 355 355 + if (csk->lro_skb) 356 356 + goto add_packet; 357 357 + 358 358 + start_lro: 359 359 + if (lro_mgr->lro_session_cnt >= MAX_LRO_SESSIONS) { 360 360 + cxgbit_uld_lro_flush(lro_mgr); 361 361 + goto start_lro; 362 362 + } 363 363 + 364 364 + skb = cxgbit_lro_init_skb(csk, op, gl, rsp, napi); 365 365 + if (unlikely(!skb)) 366 366 + goto out; 367 367 + 368 368 + csk->lro_skb = skb; 369 369 + 370 370 + __skb_queue_tail(&lro_mgr->lroq, skb); 371 371 + lro_mgr->lro_session_cnt++; 372 372 + 373 373 + add_packet: 374 374 + skb = csk->lro_skb; 375 375 + lro_cb = cxgbit_skb_lro_cb(skb); 376 376 + 377 377 + if ((gl && (((skb_shinfo(skb)->nr_frags + gl->nfrags) > 378 378 + MAX_SKB_FRAGS) || (lro_cb->pdu_totallen >= LRO_FLUSH_LEN_MAX))) || 379 379 + (lro_cb->pdu_idx >= MAX_SKB_FRAGS)) { 380 380 + cxgbit_lro_flush(lro_mgr, skb); 381 381 + goto start_lro; 382 382 + } 383 383 + 384 384 + if (gl) 385 385 + cxgbit_lro_add_packet_gl(skb, op, gl); 386 386 + else 387 387 + cxgbit_lro_add_packet_rsp(skb, op, rsp); 388 388 + 389 389 + lro_mgr->lro_merged++; 390 390 + 391 391 + return 0; 392 392 + 393 393 + out: 394 394 + return -1; 395 395 + } 396 396 + 397 397 + static int 398 398 + cxgbit_uld_lro_rx_handler(void *hndl, const __be64 *rsp, 399 399 + const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr, 400 400 + struct napi_struct *napi) 401 401 + { 402 402 + struct cxgbit_device *cdev = hndl; 403 403 + struct cxgb4_lld_info *lldi = &cdev->lldi; 404 404 + struct cpl_tx_data *rpl = NULL; 405 405 + struct cxgbit_sock *csk = NULL; 406 406 + unsigned int tid = 0; 407 407 + struct sk_buff *skb; 408 408 + unsigned int op = *(u8 *)rsp; 409 409 + bool lro_flush = true; 410 410 + 411 411 + switch (op) { 412 412 + case CPL_ISCSI_HDR: 413 413 + case CPL_ISCSI_DATA: 414 414 + case CPL_RX_ISCSI_DDP: 415 415 + case CPL_FW4_ACK: 416 416 + lro_flush = false; 417 417 + case CPL_ABORT_RPL_RSS: 418 418 + case CPL_PASS_ESTABLISH: 419 419 + case CPL_PEER_CLOSE: 420 420 + case CPL_CLOSE_CON_RPL: 421 421 + case CPL_ABORT_REQ_RSS: 422 422 + case CPL_SET_TCB_RPL: 423 423 + case CPL_RX_DATA: 424 424 + rpl = gl ? (struct cpl_tx_data *)gl->va : 425 425 + (struct cpl_tx_data *)(rsp + 1); 426 426 + tid = GET_TID(rpl); 427 427 + csk = lookup_tid(lldi->tids, tid); 428 428 + break; 429 429 + default: 430 430 + break; 431 431 + } 432 432 + 433 433 + if (csk && csk->lro_skb && lro_flush) 434 434 + cxgbit_lro_flush(lro_mgr, csk->lro_skb); 435 435 + 436 436 + if (!gl) { 437 437 + unsigned int len; 438 438 + 439 439 + if (op == CPL_RX_ISCSI_DDP) { 440 440 + if (!cxgbit_lro_receive(csk, op, rsp, NULL, lro_mgr, 441 441 + napi)) 442 442 + return 0; 443 443 + } 444 444 + 445 445 + len = 64 - sizeof(struct rsp_ctrl) - 8; 446 446 + skb = napi_alloc_skb(napi, len); 447 447 + if (!skb) 448 448 + goto nomem; 449 449 + __skb_put(skb, len); 450 450 + skb_copy_to_linear_data(skb, &rsp[1], len); 451 451 + } else { 452 452 + if (unlikely(op != *(u8 *)gl->va)) { 453 453 + pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n", 454 454 + gl->va, be64_to_cpu(*rsp), 455 455 + be64_to_cpu(*(u64 *)gl->va), 456 456 + gl->tot_len); 457 457 + return 0; 458 458 + } 459 459 + 460 460 + if (op == CPL_ISCSI_HDR || op == CPL_ISCSI_DATA) { 461 461 + if (!cxgbit_lro_receive(csk, op, rsp, gl, lro_mgr, 462 462 + napi)) 463 463 + return 0; 464 464 + } 465 465 + 466 466 + #define RX_PULL_LEN 128 467 467 + skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN); 468 468 + if (unlikely(!skb)) 469 469 + goto nomem; 470 470 + } 471 471 + 472 472 + rpl = (struct cpl_tx_data *)skb->data; 473 473 + op = rpl->ot.opcode; 474 474 + cxgbit_skcb_rx_opcode(skb) = op; 475 475 + 476 476 + pr_debug("cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n", 477 477 + cdev, op, rpl->ot.opcode_tid, 478 478 + ntohl(rpl->ot.opcode_tid), skb); 479 479 + 480 480 + if (op < NUM_CPL_CMDS && cxgbit_cplhandlers[op]) { 481 481 + cxgbit_cplhandlers[op](cdev, skb); 482 482 + } else { 483 483 + pr_err("No handler for opcode 0x%x.\n", op); 484 484 + __kfree_skb(skb); 485 485 + } 486 486 + return 0; 487 487 + nomem: 488 488 + pr_err("%s OOM bailing out.\n", __func__); 489 489 + return 1; 490 490 + } 491 491 + 492 492 + #ifdef CONFIG_CHELSIO_T4_DCB 493 493 + struct cxgbit_dcb_work { 494 494 + struct dcb_app_type dcb_app; 495 495 + struct work_struct work; 496 496 + }; 497 497 + 498 498 + static void 499 499 + cxgbit_update_dcb_priority(struct cxgbit_device *cdev, u8 port_id, 500 500 + u8 dcb_priority, u16 port_num) 501 501 + { 502 502 + struct cxgbit_sock *csk; 503 503 + struct sk_buff *skb; 504 504 + u16 local_port; 505 505 + bool wakeup_thread = false; 506 506 + 507 507 + spin_lock_bh(&cdev->cskq.lock); 508 508 + list_for_each_entry(csk, &cdev->cskq.list, list) { 509 509 + if (csk->port_id != port_id) 510 510 + continue; 511 511 + 512 512 + if (csk->com.local_addr.ss_family == AF_INET6) { 513 513 + struct sockaddr_in6 *sock_in6; 514 514 + 515 515 + sock_in6 = (struct sockaddr_in6 *)&csk->com.local_addr; 516 516 + local_port = ntohs(sock_in6->sin6_port); 517 517 + } else { 518 518 + struct sockaddr_in *sock_in; 519 519 + 520 520 + sock_in = (struct sockaddr_in *)&csk->com.local_addr; 521 521 + local_port = ntohs(sock_in->sin_port); 522 522 + } 523 523 + 524 524 + if (local_port != port_num) 525 525 + continue; 526 526 + 527 527 + if (csk->dcb_priority == dcb_priority) 528 528 + continue; 529 529 + 530 530 + skb = alloc_skb(0, GFP_ATOMIC); 531 531 + if (!skb) 532 532 + continue; 533 533 + 534 534 + spin_lock(&csk->rxq.lock); 535 535 + __skb_queue_tail(&csk->rxq, skb); 536 536 + if (skb_queue_len(&csk->rxq) == 1) 537 537 + wakeup_thread = true; 538 538 + spin_unlock(&csk->rxq.lock); 539 539 + 540 540 + if (wakeup_thread) { 541 541 + wake_up(&csk->waitq); 542 542 + wakeup_thread = false; 543 543 + } 544 544 + } 545 545 + spin_unlock_bh(&cdev->cskq.lock); 546 546 + } 547 547 + 548 548 + static void cxgbit_dcb_workfn(struct work_struct *work) 549 549 + { 550 550 + struct cxgbit_dcb_work *dcb_work; 551 551 + struct net_device *ndev; 552 552 + struct cxgbit_device *cdev = NULL; 553 553 + struct dcb_app_type *iscsi_app; 554 554 + u8 priority, port_id = 0xff; 555 555 + 556 556 + dcb_work = container_of(work, struct cxgbit_dcb_work, work); 557 557 + iscsi_app = &dcb_work->dcb_app; 558 558 + 559 559 + if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_IEEE) { 560 560 + if (iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_ANY) 561 561 + goto out; 562 562 + 563 563 + priority = iscsi_app->app.priority; 564 564 + 565 565 + } else if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_CEE) { 566 566 + if (iscsi_app->app.selector != DCB_APP_IDTYPE_PORTNUM) 567 567 + goto out; 568 568 + 569 569 + if (!iscsi_app->app.priority) 570 570 + goto out; 571 571 + 572 572 + priority = ffs(iscsi_app->app.priority) - 1; 573 573 + } else { 574 574 + goto out; 575 575 + } 576 576 + 577 577 + pr_debug("priority for ifid %d is %u\n", 578 578 + iscsi_app->ifindex, priority); 579 579 + 580 580 + ndev = dev_get_by_index(&init_net, iscsi_app->ifindex); 581 581 + 582 582 + if (!ndev) 583 583 + goto out; 584 584 + 585 585 + mutex_lock(&cdev_list_lock); 586 586 + cdev = cxgbit_find_device(ndev, &port_id); 587 587 + 588 588 + dev_put(ndev); 589 589 + 590 590 + if (!cdev) { 591 591 + mutex_unlock(&cdev_list_lock); 592 592 + goto out; 593 593 + } 594 594 + 595 595 + cxgbit_update_dcb_priority(cdev, port_id, priority, 596 596 + iscsi_app->app.protocol); 597 597 + mutex_unlock(&cdev_list_lock); 598 598 + out: 599 599 + kfree(dcb_work); 600 600 + } 601 601 + 602 602 + static int 603 603 + cxgbit_dcbevent_notify(struct notifier_block *nb, unsigned long action, 604 604 + void *data) 605 605 + { 606 606 + struct cxgbit_dcb_work *dcb_work; 607 607 + struct dcb_app_type *dcb_app = data; 608 608 + 609 609 + dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC); 610 610 + if (!dcb_work) 611 611 + return NOTIFY_DONE; 612 612 + 613 613 + dcb_work->dcb_app = *dcb_app; 614 614 + INIT_WORK(&dcb_work->work, cxgbit_dcb_workfn); 615 615 + schedule_work(&dcb_work->work); 616 616 + return NOTIFY_OK; 617 617 + } 618 618 + #endif 619 619 + 620 620 + static enum target_prot_op cxgbit_get_sup_prot_ops(struct iscsi_conn *conn) 621 621 + { 622 622 + return TARGET_PROT_NORMAL; 623 623 + } 624 624 + 625 625 + static struct iscsit_transport cxgbit_transport = { 626 626 + .name = DRV_NAME, 627 627 + .transport_type = ISCSI_CXGBIT, 628 628 + .rdma_shutdown = false, 629 629 + .priv_size = sizeof(struct cxgbit_cmd), 630 630 + .owner = THIS_MODULE, 631 631 + .iscsit_setup_np = cxgbit_setup_np, 632 632 + .iscsit_accept_np = cxgbit_accept_np, 633 633 + .iscsit_free_np = cxgbit_free_np, 634 634 + .iscsit_free_conn = cxgbit_free_conn, 635 635 + .iscsit_get_login_rx = cxgbit_get_login_rx, 636 636 + .iscsit_put_login_tx = cxgbit_put_login_tx, 637 637 + .iscsit_immediate_queue = iscsit_immediate_queue, 638 638 + .iscsit_response_queue = iscsit_response_queue, 639 639 + .iscsit_get_dataout = iscsit_build_r2ts_for_cmd, 640 640 + .iscsit_queue_data_in = iscsit_queue_rsp, 641 641 + .iscsit_queue_status = iscsit_queue_rsp, 642 642 + .iscsit_xmit_pdu = cxgbit_xmit_pdu, 643 643 + .iscsit_get_r2t_ttt = cxgbit_get_r2t_ttt, 644 644 + .iscsit_get_rx_pdu = cxgbit_get_rx_pdu, 645 645 + .iscsit_validate_params = cxgbit_validate_params, 646 646 + .iscsit_release_cmd = cxgbit_release_cmd, 647 647 + .iscsit_aborted_task = iscsit_aborted_task, 648 648 + .iscsit_get_sup_prot_ops = cxgbit_get_sup_prot_ops, 649 649 + }; 650 650 + 651 651 + static struct cxgb4_uld_info cxgbit_uld_info = { 652 652 + .name = DRV_NAME, 653 653 + .add = cxgbit_uld_add, 654 654 + .state_change = cxgbit_uld_state_change, 655 655 + .lro_rx_handler = cxgbit_uld_lro_rx_handler, 656 656 + .lro_flush = cxgbit_uld_lro_flush, 657 657 + }; 658 658 + 659 659 + #ifdef CONFIG_CHELSIO_T4_DCB 660 660 + static struct notifier_block cxgbit_dcbevent_nb = { 661 661 + .notifier_call = cxgbit_dcbevent_notify, 662 662 + }; 663 663 + #endif 664 664 + 665 665 + static int __init cxgbit_init(void) 666 666 + { 667 667 + cxgb4_register_uld(CXGB4_ULD_ISCSIT, &cxgbit_uld_info); 668 668 + iscsit_register_transport(&cxgbit_transport); 669 669 + 670 670 + #ifdef CONFIG_CHELSIO_T4_DCB 671 671 + pr_info("%s dcb enabled.\n", DRV_NAME); 672 672 + register_dcbevent_notifier(&cxgbit_dcbevent_nb); 673 673 + #endif 674 674 + BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, cb) < 675 675 + sizeof(union cxgbit_skb_cb)); 676 676 + return 0; 677 677 + } 678 678 + 679 679 + static void __exit cxgbit_exit(void) 680 680 + { 681 681 + struct cxgbit_device *cdev, *tmp; 682 682 + 683 683 + #ifdef CONFIG_CHELSIO_T4_DCB 684 684 + unregister_dcbevent_notifier(&cxgbit_dcbevent_nb); 685 685 + #endif 686 686 + mutex_lock(&cdev_list_lock); 687 687 + list_for_each_entry_safe(cdev, tmp, &cdev_list_head, list) { 688 688 + list_del(&cdev->list); 689 689 + cxgbit_put_cdev(cdev); 690 690 + } 691 691 + mutex_unlock(&cdev_list_lock); 692 692 + iscsit_unregister_transport(&cxgbit_transport); 693 693 + cxgb4_unregister_uld(CXGB4_ULD_ISCSIT); 694 694 + } 695 695 + 696 696 + module_init(cxgbit_init); 697 697 + module_exit(cxgbit_exit); 698 698 + 699 699 + MODULE_DESCRIPTION("Chelsio iSCSI target offload driver"); 700 700 + MODULE_AUTHOR("Chelsio Communications"); 701 701 + MODULE_VERSION(DRV_VERSION); 702 702 + MODULE_LICENSE("GPL");

+1561

drivers/target/iscsi/cxgbit/cxgbit_target.c

reviewed

··· 1 1 + /* 2 2 + * Copyright (c) 2016 Chelsio Communications, Inc. 3 3 + * 4 4 + * This program is free software; you can redistribute it and/or modify 5 5 + * it under the terms of the GNU General Public License version 2 as 6 6 + * published by the Free Software Foundation. 7 7 + */ 8 8 + 9 9 + #include <linux/workqueue.h> 10 10 + #include <linux/kthread.h> 11 11 + #include <asm/unaligned.h> 12 12 + #include <target/target_core_base.h> 13 13 + #include <target/target_core_fabric.h> 14 14 + #include "cxgbit.h" 15 15 + 16 16 + struct sge_opaque_hdr { 17 17 + void *dev; 18 18 + dma_addr_t addr[MAX_SKB_FRAGS + 1]; 19 19 + }; 20 20 + 21 21 + static const u8 cxgbit_digest_len[] = {0, 4, 4, 8}; 22 22 + 23 23 + #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \ 24 24 + sizeof(struct fw_ofld_tx_data_wr)) 25 25 + 26 26 + static struct sk_buff * 27 27 + __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso) 28 28 + { 29 29 + struct sk_buff *skb = NULL; 30 30 + u8 submode = 0; 31 31 + int errcode; 32 32 + static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN; 33 33 + 34 34 + if (len) { 35 35 + skb = alloc_skb_with_frags(hdr_len, len, 36 36 + 0, &errcode, 37 37 + GFP_KERNEL); 38 38 + if (!skb) 39 39 + return NULL; 40 40 + 41 41 + skb_reserve(skb, TX_HDR_LEN); 42 42 + skb_reset_transport_header(skb); 43 43 + __skb_put(skb, ISCSI_HDR_LEN); 44 44 + skb->data_len = len; 45 45 + skb->len += len; 46 46 + submode |= (csk->submode & CXGBIT_SUBMODE_DCRC); 47 47 + 48 48 + } else { 49 49 + u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0; 50 50 + 51 51 + skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL); 52 52 + if (!skb) 53 53 + return NULL; 54 54 + 55 55 + skb_reserve(skb, TX_HDR_LEN + iso_len); 56 56 + skb_reset_transport_header(skb); 57 57 + __skb_put(skb, ISCSI_HDR_LEN); 58 58 + } 59 59 + 60 60 + submode |= (csk->submode & CXGBIT_SUBMODE_HCRC); 61 61 + cxgbit_skcb_submode(skb) = submode; 62 62 + cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode]; 63 63 + cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR; 64 64 + return skb; 65 65 + } 66 66 + 67 67 + static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len) 68 68 + { 69 69 + return __cxgbit_alloc_skb(csk, len, false); 70 70 + } 71 71 + 72 72 + /* 73 73 + * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data 74 74 + * @skb: the packet 75 75 + * 76 76 + * Returns true if a packet can be sent as an offload WR with immediate 77 77 + * data. We currently use the same limit as for Ethernet packets. 78 78 + */ 79 79 + static int cxgbit_is_ofld_imm(const struct sk_buff *skb) 80 80 + { 81 81 + int length = skb->len; 82 82 + 83 83 + if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 84 84 + length += sizeof(struct fw_ofld_tx_data_wr); 85 85 + 86 86 + if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)) 87 87 + length += sizeof(struct cpl_tx_data_iso); 88 88 + 89 89 + #define MAX_IMM_TX_PKT_LEN 256 90 90 + return length <= MAX_IMM_TX_PKT_LEN; 91 91 + } 92 92 + 93 93 + /* 94 94 + * cxgbit_sgl_len - calculates the size of an SGL of the given capacity 95 95 + * @n: the number of SGL entries 96 96 + * Calculates the number of flits needed for a scatter/gather list that 97 97 + * can hold the given number of entries. 98 98 + */ 99 99 + static inline unsigned int cxgbit_sgl_len(unsigned int n) 100 100 + { 101 101 + n--; 102 102 + return (3 * n) / 2 + (n & 1) + 2; 103 103 + } 104 104 + 105 105 + /* 106 106 + * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet 107 107 + * @skb: the packet 108 108 + * 109 109 + * Returns the number of flits needed for the given offload packet. 110 110 + * These packets are already fully constructed and no additional headers 111 111 + * will be added. 112 112 + */ 113 113 + static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb) 114 114 + { 115 115 + unsigned int flits, cnt; 116 116 + 117 117 + if (cxgbit_is_ofld_imm(skb)) 118 118 + return DIV_ROUND_UP(skb->len, 8); 119 119 + flits = skb_transport_offset(skb) / 8; 120 120 + cnt = skb_shinfo(skb)->nr_frags; 121 121 + if (skb_tail_pointer(skb) != skb_transport_header(skb)) 122 122 + cnt++; 123 123 + return flits + cxgbit_sgl_len(cnt); 124 124 + } 125 125 + 126 126 + #define CXGBIT_ISO_FSLICE 0x1 127 127 + #define CXGBIT_ISO_LSLICE 0x2 128 128 + static void 129 129 + cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info) 130 130 + { 131 131 + struct cpl_tx_data_iso *cpl; 132 132 + unsigned int submode = cxgbit_skcb_submode(skb); 133 133 + unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE); 134 134 + unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE); 135 135 + 136 136 + cpl = (struct cpl_tx_data_iso *)__skb_push(skb, sizeof(*cpl)); 137 137 + 138 138 + cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) | 139 139 + CPL_TX_DATA_ISO_FIRST_V(fslice) | 140 140 + CPL_TX_DATA_ISO_LAST_V(lslice) | 141 141 + CPL_TX_DATA_ISO_CPLHDRLEN_V(0) | 142 142 + CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) | 143 143 + CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) | 144 144 + CPL_TX_DATA_ISO_IMMEDIATE_V(0) | 145 145 + CPL_TX_DATA_ISO_SCSI_V(2)); 146 146 + 147 147 + cpl->ahs_len = 0; 148 148 + cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4)); 149 149 + cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4)); 150 150 + cpl->len = htonl(iso_info->len); 151 151 + cpl->reserved2_seglen_offset = htonl(0); 152 152 + cpl->datasn_offset = htonl(0); 153 153 + cpl->buffer_offset = htonl(0); 154 154 + cpl->reserved3 = 0; 155 155 + 156 156 + __skb_pull(skb, sizeof(*cpl)); 157 157 + } 158 158 + 159 159 + static void 160 160 + cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen, 161 161 + u32 len, u32 credits, u32 compl) 162 162 + { 163 163 + struct fw_ofld_tx_data_wr *req; 164 164 + u32 submode = cxgbit_skcb_submode(skb); 165 165 + u32 wr_ulp_mode = 0; 166 166 + u32 hdr_size = sizeof(*req); 167 167 + u32 opcode = FW_OFLD_TX_DATA_WR; 168 168 + u32 immlen = 0; 169 169 + u32 force = TX_FORCE_V(!submode); 170 170 + 171 171 + if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) { 172 172 + opcode = FW_ISCSI_TX_DATA_WR; 173 173 + immlen += sizeof(struct cpl_tx_data_iso); 174 174 + hdr_size += sizeof(struct cpl_tx_data_iso); 175 175 + submode |= 8; 176 176 + } 177 177 + 178 178 + if (cxgbit_is_ofld_imm(skb)) 179 179 + immlen += dlen; 180 180 + 181 181 + req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, 182 182 + hdr_size); 183 183 + req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) | 184 184 + FW_WR_COMPL_V(compl) | 185 185 + FW_WR_IMMDLEN_V(immlen)); 186 186 + req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | 187 187 + FW_WR_LEN16_V(credits)); 188 188 + req->plen = htonl(len); 189 189 + wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) | 190 190 + FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); 191 191 + 192 192 + req->tunnel_to_proxy = htonl((wr_ulp_mode) | force | 193 193 + FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1)); 194 194 + } 195 195 + 196 196 + static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb) 197 197 + { 198 198 + kfree_skb(skb); 199 199 + } 200 200 + 201 201 + void cxgbit_push_tx_frames(struct cxgbit_sock *csk) 202 202 + { 203 203 + struct sk_buff *skb; 204 204 + 205 205 + while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) { 206 206 + u32 dlen = skb->len; 207 207 + u32 len = skb->len; 208 208 + u32 credits_needed; 209 209 + u32 compl = 0; 210 210 + u32 flowclen16 = 0; 211 211 + u32 iso_cpl_len = 0; 212 212 + 213 213 + if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) 214 214 + iso_cpl_len = sizeof(struct cpl_tx_data_iso); 215 215 + 216 216 + if (cxgbit_is_ofld_imm(skb)) 217 217 + credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16); 218 218 + else 219 219 + credits_needed = DIV_ROUND_UP((8 * 220 220 + cxgbit_calc_tx_flits_ofld(skb)) + 221 221 + iso_cpl_len, 16); 222 222 + 223 223 + if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 224 224 + credits_needed += DIV_ROUND_UP( 225 225 + sizeof(struct fw_ofld_tx_data_wr), 16); 226 226 + /* 227 227 + * Assumes the initial credits is large enough to support 228 228 + * fw_flowc_wr plus largest possible first payload 229 229 + */ 230 230 + 231 231 + if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) { 232 232 + flowclen16 = cxgbit_send_tx_flowc_wr(csk); 233 233 + csk->wr_cred -= flowclen16; 234 234 + csk->wr_una_cred += flowclen16; 235 235 + } 236 236 + 237 237 + if (csk->wr_cred < credits_needed) { 238 238 + pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n", 239 239 + csk, skb->len, skb->data_len, 240 240 + credits_needed, csk->wr_cred); 241 241 + break; 242 242 + } 243 243 + __skb_unlink(skb, &csk->txq); 244 244 + set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 245 245 + skb->csum = credits_needed + flowclen16; 246 246 + csk->wr_cred -= credits_needed; 247 247 + csk->wr_una_cred += credits_needed; 248 248 + 249 249 + pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", 250 250 + csk, skb->len, skb->data_len, credits_needed, 251 251 + csk->wr_cred, csk->wr_una_cred); 252 252 + 253 253 + if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) { 254 254 + len += cxgbit_skcb_tx_extralen(skb); 255 255 + 256 256 + if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) || 257 257 + (!before(csk->write_seq, 258 258 + csk->snd_una + csk->snd_win))) { 259 259 + compl = 1; 260 260 + csk->wr_una_cred = 0; 261 261 + } 262 262 + 263 263 + cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed, 264 264 + compl); 265 265 + csk->snd_nxt += len; 266 266 + 267 267 + } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) || 268 268 + (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { 269 269 + struct cpl_close_con_req *req = 270 270 + (struct cpl_close_con_req *)skb->data; 271 271 + req->wr.wr_hi |= htonl(FW_WR_COMPL_F); 272 272 + csk->wr_una_cred = 0; 273 273 + } 274 274 + 275 275 + cxgbit_sock_enqueue_wr(csk, skb); 276 276 + t4_set_arp_err_handler(skb, csk, 277 277 + cxgbit_arp_failure_skb_discard); 278 278 + 279 279 + pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n", 280 280 + csk, csk->tid, skb, len); 281 281 + 282 282 + cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 283 283 + } 284 284 + } 285 285 + 286 286 + static bool cxgbit_lock_sock(struct cxgbit_sock *csk) 287 287 + { 288 288 + spin_lock_bh(&csk->lock); 289 289 + 290 290 + if (before(csk->write_seq, csk->snd_una + csk->snd_win)) 291 291 + csk->lock_owner = true; 292 292 + 293 293 + spin_unlock_bh(&csk->lock); 294 294 + 295 295 + return csk->lock_owner; 296 296 + } 297 297 + 298 298 + static void cxgbit_unlock_sock(struct cxgbit_sock *csk) 299 299 + { 300 300 + struct sk_buff_head backlogq; 301 301 + struct sk_buff *skb; 302 302 + void (*fn)(struct cxgbit_sock *, struct sk_buff *); 303 303 + 304 304 + skb_queue_head_init(&backlogq); 305 305 + 306 306 + spin_lock_bh(&csk->lock); 307 307 + while (skb_queue_len(&csk->backlogq)) { 308 308 + skb_queue_splice_init(&csk->backlogq, &backlogq); 309 309 + spin_unlock_bh(&csk->lock); 310 310 + 311 311 + while ((skb = __skb_dequeue(&backlogq))) { 312 312 + fn = cxgbit_skcb_rx_backlog_fn(skb); 313 313 + fn(csk, skb); 314 314 + } 315 315 + 316 316 + spin_lock_bh(&csk->lock); 317 317 + } 318 318 + 319 319 + csk->lock_owner = false; 320 320 + spin_unlock_bh(&csk->lock); 321 321 + } 322 322 + 323 323 + static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 324 324 + { 325 325 + int ret = 0; 326 326 + 327 327 + wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk)); 328 328 + 329 329 + if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) || 330 330 + signal_pending(current))) { 331 331 + __kfree_skb(skb); 332 332 + __skb_queue_purge(&csk->ppodq); 333 333 + ret = -1; 334 334 + spin_lock_bh(&csk->lock); 335 335 + if (csk->lock_owner) { 336 336 + spin_unlock_bh(&csk->lock); 337 337 + goto unlock; 338 338 + } 339 339 + spin_unlock_bh(&csk->lock); 340 340 + return ret; 341 341 + } 342 342 + 343 343 + csk->write_seq += skb->len + 344 344 + cxgbit_skcb_tx_extralen(skb); 345 345 + 346 346 + skb_queue_splice_tail_init(&csk->ppodq, &csk->txq); 347 347 + __skb_queue_tail(&csk->txq, skb); 348 348 + cxgbit_push_tx_frames(csk); 349 349 + 350 350 + unlock: 351 351 + cxgbit_unlock_sock(csk); 352 352 + return ret; 353 353 + } 354 354 + 355 355 + static int 356 356 + cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset, 357 357 + u32 data_length) 358 358 + { 359 359 + u32 i = 0, nr_frags = MAX_SKB_FRAGS; 360 360 + u32 padding = ((-data_length) & 3); 361 361 + struct scatterlist *sg; 362 362 + struct page *page; 363 363 + unsigned int page_off; 364 364 + 365 365 + if (padding) 366 366 + nr_frags--; 367 367 + 368 368 + /* 369 369 + * We know each entry in t_data_sg contains a page. 370 370 + */ 371 371 + sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE]; 372 372 + page_off = (data_offset % PAGE_SIZE); 373 373 + 374 374 + while (data_length && (i < nr_frags)) { 375 375 + u32 cur_len = min_t(u32, data_length, sg->length - page_off); 376 376 + 377 377 + page = sg_page(sg); 378 378 + 379 379 + get_page(page); 380 380 + skb_fill_page_desc(skb, i, page, sg->offset + page_off, 381 381 + cur_len); 382 382 + skb->data_len += cur_len; 383 383 + skb->len += cur_len; 384 384 + skb->truesize += cur_len; 385 385 + 386 386 + data_length -= cur_len; 387 387 + page_off = 0; 388 388 + sg = sg_next(sg); 389 389 + i++; 390 390 + } 391 391 + 392 392 + if (data_length) 393 393 + return -1; 394 394 + 395 395 + if (padding) { 396 396 + page = alloc_page(GFP_KERNEL | __GFP_ZERO); 397 397 + if (!page) 398 398 + return -1; 399 399 + skb_fill_page_desc(skb, i, page, 0, padding); 400 400 + skb->data_len += padding; 401 401 + skb->len += padding; 402 402 + skb->truesize += padding; 403 403 + } 404 404 + 405 405 + return 0; 406 406 + } 407 407 + 408 408 + static int 409 409 + cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 410 410 + struct iscsi_datain_req *dr) 411 411 + { 412 412 + struct iscsi_conn *conn = csk->conn; 413 413 + struct sk_buff *skb; 414 414 + struct iscsi_datain datain; 415 415 + struct cxgbit_iso_info iso_info; 416 416 + u32 data_length = cmd->se_cmd.data_length; 417 417 + u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 418 418 + u32 num_pdu, plen, tx_data = 0; 419 419 + bool task_sense = !!(cmd->se_cmd.se_cmd_flags & 420 420 + SCF_TRANSPORT_TASK_SENSE); 421 421 + bool set_statsn = false; 422 422 + int ret = -1; 423 423 + 424 424 + while (data_length) { 425 425 + num_pdu = (data_length + mrdsl - 1) / mrdsl; 426 426 + if (num_pdu > csk->max_iso_npdu) 427 427 + num_pdu = csk->max_iso_npdu; 428 428 + 429 429 + plen = num_pdu * mrdsl; 430 430 + if (plen > data_length) 431 431 + plen = data_length; 432 432 + 433 433 + skb = __cxgbit_alloc_skb(csk, 0, true); 434 434 + if (unlikely(!skb)) 435 435 + return -ENOMEM; 436 436 + 437 437 + memset(skb->data, 0, ISCSI_HDR_LEN); 438 438 + cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO; 439 439 + cxgbit_skcb_submode(skb) |= (csk->submode & 440 440 + CXGBIT_SUBMODE_DCRC); 441 441 + cxgbit_skcb_tx_extralen(skb) = (num_pdu * 442 442 + cxgbit_digest_len[cxgbit_skcb_submode(skb)]) + 443 443 + ((num_pdu - 1) * ISCSI_HDR_LEN); 444 444 + 445 445 + memset(&datain, 0, sizeof(struct iscsi_datain)); 446 446 + memset(&iso_info, 0, sizeof(iso_info)); 447 447 + 448 448 + if (!tx_data) 449 449 + iso_info.flags |= CXGBIT_ISO_FSLICE; 450 450 + 451 451 + if (!(data_length - plen)) { 452 452 + iso_info.flags |= CXGBIT_ISO_LSLICE; 453 453 + if (!task_sense) { 454 454 + datain.flags = ISCSI_FLAG_DATA_STATUS; 455 455 + iscsit_increment_maxcmdsn(cmd, conn->sess); 456 456 + cmd->stat_sn = conn->stat_sn++; 457 457 + set_statsn = true; 458 458 + } 459 459 + } 460 460 + 461 461 + iso_info.burst_len = num_pdu * mrdsl; 462 462 + iso_info.mpdu = mrdsl; 463 463 + iso_info.len = ISCSI_HDR_LEN + plen; 464 464 + 465 465 + cxgbit_cpl_tx_data_iso(skb, &iso_info); 466 466 + 467 467 + datain.offset = tx_data; 468 468 + datain.data_sn = cmd->data_sn - 1; 469 469 + 470 470 + iscsit_build_datain_pdu(cmd, conn, &datain, 471 471 + (struct iscsi_data_rsp *)skb->data, 472 472 + set_statsn); 473 473 + 474 474 + ret = cxgbit_map_skb(cmd, skb, tx_data, plen); 475 475 + if (unlikely(ret)) { 476 476 + __kfree_skb(skb); 477 477 + goto out; 478 478 + } 479 479 + 480 480 + ret = cxgbit_queue_skb(csk, skb); 481 481 + if (unlikely(ret)) 482 482 + goto out; 483 483 + 484 484 + tx_data += plen; 485 485 + data_length -= plen; 486 486 + 487 487 + cmd->read_data_done += plen; 488 488 + cmd->data_sn += num_pdu; 489 489 + } 490 490 + 491 491 + dr->dr_complete = DATAIN_COMPLETE_NORMAL; 492 492 + 493 493 + return 0; 494 494 + 495 495 + out: 496 496 + return ret; 497 497 + } 498 498 + 499 499 + static int 500 500 + cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 501 501 + const struct iscsi_datain *datain) 502 502 + { 503 503 + struct sk_buff *skb; 504 504 + int ret = 0; 505 505 + 506 506 + skb = cxgbit_alloc_skb(csk, 0); 507 507 + if (unlikely(!skb)) 508 508 + return -ENOMEM; 509 509 + 510 510 + memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 511 511 + 512 512 + if (datain->length) { 513 513 + cxgbit_skcb_submode(skb) |= (csk->submode & 514 514 + CXGBIT_SUBMODE_DCRC); 515 515 + cxgbit_skcb_tx_extralen(skb) = 516 516 + cxgbit_digest_len[cxgbit_skcb_submode(skb)]; 517 517 + } 518 518 + 519 519 + ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length); 520 520 + if (ret < 0) { 521 521 + __kfree_skb(skb); 522 522 + return ret; 523 523 + } 524 524 + 525 525 + return cxgbit_queue_skb(csk, skb); 526 526 + } 527 527 + 528 528 + static int 529 529 + cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 530 530 + struct iscsi_datain_req *dr, 531 531 + const struct iscsi_datain *datain) 532 532 + { 533 533 + struct cxgbit_sock *csk = conn->context; 534 534 + u32 data_length = cmd->se_cmd.data_length; 535 535 + u32 padding = ((-data_length) & 3); 536 536 + u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 537 537 + 538 538 + if ((data_length > mrdsl) && (!dr->recovery) && 539 539 + (!padding) && (!datain->offset) && csk->max_iso_npdu) { 540 540 + atomic_long_add(data_length - datain->length, 541 541 + &conn->sess->tx_data_octets); 542 542 + return cxgbit_tx_datain_iso(csk, cmd, dr); 543 543 + } 544 544 + 545 545 + return cxgbit_tx_datain(csk, cmd, datain); 546 546 + } 547 547 + 548 548 + static int 549 549 + cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 550 550 + const void *data_buf, u32 data_buf_len) 551 551 + { 552 552 + struct cxgbit_sock *csk = conn->context; 553 553 + struct sk_buff *skb; 554 554 + u32 padding = ((-data_buf_len) & 3); 555 555 + 556 556 + skb = cxgbit_alloc_skb(csk, data_buf_len + padding); 557 557 + if (unlikely(!skb)) 558 558 + return -ENOMEM; 559 559 + 560 560 + memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 561 561 + 562 562 + if (data_buf_len) { 563 563 + u32 pad_bytes = 0; 564 564 + 565 565 + skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len); 566 566 + 567 567 + if (padding) 568 568 + skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len, 569 569 + &pad_bytes, padding); 570 570 + } 571 571 + 572 572 + cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[ 573 573 + cxgbit_skcb_submode(skb)]; 574 574 + 575 575 + return cxgbit_queue_skb(csk, skb); 576 576 + } 577 577 + 578 578 + int 579 579 + cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 580 580 + struct iscsi_datain_req *dr, const void *buf, u32 buf_len) 581 581 + { 582 582 + if (dr) 583 583 + return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf); 584 584 + else 585 585 + return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 586 586 + } 587 587 + 588 588 + int cxgbit_validate_params(struct iscsi_conn *conn) 589 589 + { 590 590 + struct cxgbit_sock *csk = conn->context; 591 591 + struct cxgbit_device *cdev = csk->com.cdev; 592 592 + struct iscsi_param *param; 593 593 + u32 max_xmitdsl; 594 594 + 595 595 + param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH, 596 596 + conn->param_list); 597 597 + if (!param) 598 598 + return -1; 599 599 + 600 600 + if (kstrtou32(param->value, 0, &max_xmitdsl) < 0) 601 601 + return -1; 602 602 + 603 603 + if (max_xmitdsl > cdev->mdsl) { 604 604 + if (iscsi_change_param_sprintf( 605 605 + conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl)) 606 606 + return -1; 607 607 + } 608 608 + 609 609 + return 0; 610 610 + } 611 611 + 612 612 + static int cxgbit_set_digest(struct cxgbit_sock *csk) 613 613 + { 614 614 + struct iscsi_conn *conn = csk->conn; 615 615 + struct iscsi_param *param; 616 616 + 617 617 + param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list); 618 618 + if (!param) { 619 619 + pr_err("param not found key %s\n", HEADERDIGEST); 620 620 + return -1; 621 621 + } 622 622 + 623 623 + if (!strcmp(param->value, CRC32C)) 624 624 + csk->submode |= CXGBIT_SUBMODE_HCRC; 625 625 + 626 626 + param = iscsi_find_param_from_key(DATADIGEST, conn->param_list); 627 627 + if (!param) { 628 628 + csk->submode = 0; 629 629 + pr_err("param not found key %s\n", DATADIGEST); 630 630 + return -1; 631 631 + } 632 632 + 633 633 + if (!strcmp(param->value, CRC32C)) 634 634 + csk->submode |= CXGBIT_SUBMODE_DCRC; 635 635 + 636 636 + if (cxgbit_setup_conn_digest(csk)) { 637 637 + csk->submode = 0; 638 638 + return -1; 639 639 + } 640 640 + 641 641 + return 0; 642 642 + } 643 643 + 644 644 + static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk) 645 645 + { 646 646 + struct iscsi_conn *conn = csk->conn; 647 647 + struct iscsi_conn_ops *conn_ops = conn->conn_ops; 648 648 + struct iscsi_param *param; 649 649 + u32 mrdsl, mbl; 650 650 + u32 max_npdu, max_iso_npdu; 651 651 + 652 652 + if (conn->login->leading_connection) { 653 653 + param = iscsi_find_param_from_key(DATASEQUENCEINORDER, 654 654 + conn->param_list); 655 655 + if (!param) { 656 656 + pr_err("param not found key %s\n", DATASEQUENCEINORDER); 657 657 + return -1; 658 658 + } 659 659 + 660 660 + if (strcmp(param->value, YES)) 661 661 + return 0; 662 662 + 663 663 + param = iscsi_find_param_from_key(DATAPDUINORDER, 664 664 + conn->param_list); 665 665 + if (!param) { 666 666 + pr_err("param not found key %s\n", DATAPDUINORDER); 667 667 + return -1; 668 668 + } 669 669 + 670 670 + if (strcmp(param->value, YES)) 671 671 + return 0; 672 672 + 673 673 + param = iscsi_find_param_from_key(MAXBURSTLENGTH, 674 674 + conn->param_list); 675 675 + if (!param) { 676 676 + pr_err("param not found key %s\n", MAXBURSTLENGTH); 677 677 + return -1; 678 678 + } 679 679 + 680 680 + if (kstrtou32(param->value, 0, &mbl) < 0) 681 681 + return -1; 682 682 + } else { 683 683 + if (!conn->sess->sess_ops->DataSequenceInOrder) 684 684 + return 0; 685 685 + if (!conn->sess->sess_ops->DataPDUInOrder) 686 686 + return 0; 687 687 + 688 688 + mbl = conn->sess->sess_ops->MaxBurstLength; 689 689 + } 690 690 + 691 691 + mrdsl = conn_ops->MaxRecvDataSegmentLength; 692 692 + max_npdu = mbl / mrdsl; 693 693 + 694 694 + max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD / 695 695 + (ISCSI_HDR_LEN + mrdsl + 696 696 + cxgbit_digest_len[csk->submode]); 697 697 + 698 698 + csk->max_iso_npdu = min(max_npdu, max_iso_npdu); 699 699 + 700 700 + if (csk->max_iso_npdu <= 1) 701 701 + csk->max_iso_npdu = 0; 702 702 + 703 703 + return 0; 704 704 + } 705 705 + 706 706 + static int cxgbit_set_params(struct iscsi_conn *conn) 707 707 + { 708 708 + struct cxgbit_sock *csk = conn->context; 709 709 + struct cxgbit_device *cdev = csk->com.cdev; 710 710 + struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm; 711 711 + struct iscsi_conn_ops *conn_ops = conn->conn_ops; 712 712 + struct iscsi_param *param; 713 713 + u8 erl; 714 714 + 715 715 + if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl) 716 716 + conn_ops->MaxRecvDataSegmentLength = cdev->mdsl; 717 717 + 718 718 + if (conn->login->leading_connection) { 719 719 + param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL, 720 720 + conn->param_list); 721 721 + if (!param) { 722 722 + pr_err("param not found key %s\n", ERRORRECOVERYLEVEL); 723 723 + return -1; 724 724 + } 725 725 + if (kstrtou8(param->value, 0, &erl) < 0) 726 726 + return -1; 727 727 + } else { 728 728 + erl = conn->sess->sess_ops->ErrorRecoveryLevel; 729 729 + } 730 730 + 731 731 + if (!erl) { 732 732 + if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) { 733 733 + if (cxgbit_set_iso_npdu(csk)) 734 734 + return -1; 735 735 + } 736 736 + 737 737 + if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) { 738 738 + if (cxgbit_setup_conn_pgidx(csk, 739 739 + ppm->tformat.pgsz_idx_dflt)) 740 740 + return -1; 741 741 + set_bit(CSK_DDP_ENABLE, &csk->com.flags); 742 742 + } 743 743 + } 744 744 + 745 745 + if (cxgbit_set_digest(csk)) 746 746 + return -1; 747 747 + 748 748 + return 0; 749 749 + } 750 750 + 751 751 + int 752 752 + cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 753 753 + u32 length) 754 754 + { 755 755 + struct cxgbit_sock *csk = conn->context; 756 756 + struct sk_buff *skb; 757 757 + u32 padding_buf = 0; 758 758 + u8 padding = ((-length) & 3); 759 759 + 760 760 + skb = cxgbit_alloc_skb(csk, length + padding); 761 761 + if (!skb) 762 762 + return -ENOMEM; 763 763 + skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN); 764 764 + skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length); 765 765 + 766 766 + if (padding) 767 767 + skb_store_bits(skb, ISCSI_HDR_LEN + length, 768 768 + &padding_buf, padding); 769 769 + 770 770 + if (login->login_complete) { 771 771 + if (cxgbit_set_params(conn)) { 772 772 + kfree_skb(skb); 773 773 + return -1; 774 774 + } 775 775 + 776 776 + set_bit(CSK_LOGIN_DONE, &csk->com.flags); 777 777 + } 778 778 + 779 779 + if (cxgbit_queue_skb(csk, skb)) 780 780 + return -1; 781 781 + 782 782 + if ((!login->login_complete) && (!login->login_failed)) 783 783 + schedule_delayed_work(&conn->login_work, 0); 784 784 + 785 785 + return 0; 786 786 + } 787 787 + 788 788 + static void 789 789 + cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg, 790 790 + unsigned int nents) 791 791 + { 792 792 + struct skb_seq_state st; 793 793 + const u8 *buf; 794 794 + unsigned int consumed = 0, buf_len; 795 795 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb); 796 796 + 797 797 + skb_prepare_seq_read(skb, pdu_cb->doffset, 798 798 + pdu_cb->doffset + pdu_cb->dlen, 799 799 + &st); 800 800 + 801 801 + while (true) { 802 802 + buf_len = skb_seq_read(consumed, &buf, &st); 803 803 + if (!buf_len) { 804 804 + skb_abort_seq_read(&st); 805 805 + break; 806 806 + } 807 807 + 808 808 + consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf, 809 809 + buf_len, consumed); 810 810 + } 811 811 + } 812 812 + 813 813 + static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk) 814 814 + { 815 815 + struct iscsi_conn *conn = csk->conn; 816 816 + struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev); 817 817 + struct cxgbit_cmd *ccmd; 818 818 + struct iscsi_cmd *cmd; 819 819 + 820 820 + cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 821 821 + if (!cmd) { 822 822 + pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n"); 823 823 + return NULL; 824 824 + } 825 825 + 826 826 + ccmd = iscsit_priv_cmd(cmd); 827 827 + ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask; 828 828 + ccmd->setup_ddp = true; 829 829 + 830 830 + return cmd; 831 831 + } 832 832 + 833 833 + static int 834 834 + cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 835 835 + u32 length) 836 836 + { 837 837 + struct iscsi_conn *conn = cmd->conn; 838 838 + struct cxgbit_sock *csk = conn->context; 839 839 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 840 840 + 841 841 + if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 842 842 + pr_err("ImmediateData CRC32C DataDigest error\n"); 843 843 + if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 844 844 + pr_err("Unable to recover from" 845 845 + " Immediate Data digest failure while" 846 846 + " in ERL=0.\n"); 847 847 + iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 848 848 + (unsigned char *)hdr); 849 849 + return IMMEDIATE_DATA_CANNOT_RECOVER; 850 850 + } 851 851 + 852 852 + iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 853 853 + (unsigned char *)hdr); 854 854 + return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 855 855 + } 856 856 + 857 857 + if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 858 858 + struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 859 859 + struct skb_shared_info *ssi = skb_shinfo(csk->skb); 860 860 + skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx]; 861 861 + 862 862 + sg_init_table(&ccmd->sg, 1); 863 863 + sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag), 864 864 + dfrag->page_offset); 865 865 + get_page(dfrag->page.p); 866 866 + 867 867 + cmd->se_cmd.t_data_sg = &ccmd->sg; 868 868 + cmd->se_cmd.t_data_nents = 1; 869 869 + 870 870 + ccmd->release = true; 871 871 + } else { 872 872 + struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0]; 873 873 + u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE)); 874 874 + 875 875 + cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents); 876 876 + } 877 877 + 878 878 + cmd->write_data_done += pdu_cb->dlen; 879 879 + 880 880 + if (cmd->write_data_done == cmd->se_cmd.data_length) { 881 881 + spin_lock_bh(&cmd->istate_lock); 882 882 + cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 883 883 + cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 884 884 + spin_unlock_bh(&cmd->istate_lock); 885 885 + } 886 886 + 887 887 + return IMMEDIATE_DATA_NORMAL_OPERATION; 888 888 + } 889 889 + 890 890 + static int 891 891 + cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 892 892 + bool dump_payload) 893 893 + { 894 894 + struct iscsi_conn *conn = cmd->conn; 895 895 + int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 896 896 + /* 897 897 + * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. 898 898 + */ 899 899 + if (dump_payload) 900 900 + goto after_immediate_data; 901 901 + 902 902 + immed_ret = cxgbit_handle_immediate_data(cmd, hdr, 903 903 + cmd->first_burst_len); 904 904 + after_immediate_data: 905 905 + if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 906 906 + /* 907 907 + * A PDU/CmdSN carrying Immediate Data passed 908 908 + * DataCRC, check against ExpCmdSN/MaxCmdSN if 909 909 + * Immediate Bit is not set. 910 910 + */ 911 911 + cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 912 912 + (unsigned char *)hdr, 913 913 + hdr->cmdsn); 914 914 + if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 915 915 + return -1; 916 916 + 917 917 + if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 918 918 + target_put_sess_cmd(&cmd->se_cmd); 919 919 + return 0; 920 920 + } else if (cmd->unsolicited_data) { 921 921 + iscsit_set_unsoliticed_dataout(cmd); 922 922 + } 923 923 + 924 924 + } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 925 925 + /* 926 926 + * Immediate Data failed DataCRC and ERL>=1, 927 927 + * silently drop this PDU and let the initiator 928 928 + * plug the CmdSN gap. 929 929 + * 930 930 + * FIXME: Send Unsolicited NOPIN with reserved 931 931 + * TTT here to help the initiator figure out 932 932 + * the missing CmdSN, although they should be 933 933 + * intelligent enough to determine the missing 934 934 + * CmdSN and issue a retry to plug the sequence. 935 935 + */ 936 936 + cmd->i_state = ISTATE_REMOVE; 937 937 + iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 938 938 + } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 939 939 + return -1; 940 940 + 941 941 + return 0; 942 942 + } 943 943 + 944 944 + static int 945 945 + cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 946 946 + { 947 947 + struct iscsi_conn *conn = csk->conn; 948 948 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 949 949 + struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr; 950 950 + int rc; 951 951 + bool dump_payload = false; 952 952 + 953 953 + rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr); 954 954 + if (rc < 0) 955 955 + return rc; 956 956 + 957 957 + if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) && 958 958 + (pdu_cb->nr_dfrags == 1)) 959 959 + cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 960 960 + 961 961 + rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 962 962 + if (rc < 0) 963 963 + return 0; 964 964 + else if (rc > 0) 965 965 + dump_payload = true; 966 966 + 967 967 + if (!pdu_cb->dlen) 968 968 + return 0; 969 969 + 970 970 + return cxgbit_get_immediate_data(cmd, hdr, dump_payload); 971 971 + } 972 972 + 973 973 + static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk) 974 974 + { 975 975 + struct scatterlist *sg_start; 976 976 + struct iscsi_conn *conn = csk->conn; 977 977 + struct iscsi_cmd *cmd = NULL; 978 978 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 979 979 + struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr; 980 980 + u32 data_offset = be32_to_cpu(hdr->offset); 981 981 + u32 data_len = pdu_cb->dlen; 982 982 + int rc, sg_nents, sg_off; 983 983 + bool dcrc_err = false; 984 984 + 985 985 + rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd); 986 986 + if (rc < 0) 987 987 + return rc; 988 988 + else if (!cmd) 989 989 + return 0; 990 990 + 991 991 + if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 992 992 + pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 993 993 + " DataSN: 0x%08x\n", 994 994 + hdr->itt, hdr->offset, data_len, 995 995 + hdr->datasn); 996 996 + 997 997 + dcrc_err = true; 998 998 + goto check_payload; 999 999 + } 1000 1000 + 1001 1001 + pr_debug("DataOut data_len: %u, " 1002 1002 + "write_data_done: %u, data_length: %u\n", 1003 1003 + data_len, cmd->write_data_done, 1004 1004 + cmd->se_cmd.data_length); 1005 1005 + 1006 1006 + if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) { 1007 1007 + sg_off = data_offset / PAGE_SIZE; 1008 1008 + sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1009 1009 + sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE)); 1010 1010 + 1011 1011 + cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents); 1012 1012 + } 1013 1013 + 1014 1014 + check_payload: 1015 1015 + 1016 1016 + rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err); 1017 1017 + if (rc < 0) 1018 1018 + return rc; 1019 1019 + 1020 1020 + return 0; 1021 1021 + } 1022 1022 + 1023 1023 + static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1024 1024 + { 1025 1025 + struct iscsi_conn *conn = csk->conn; 1026 1026 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1027 1027 + struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr; 1028 1028 + unsigned char *ping_data = NULL; 1029 1029 + u32 payload_length = pdu_cb->dlen; 1030 1030 + int ret; 1031 1031 + 1032 1032 + ret = iscsit_setup_nop_out(conn, cmd, hdr); 1033 1033 + if (ret < 0) 1034 1034 + return 0; 1035 1035 + 1036 1036 + if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1037 1037 + if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1038 1038 + pr_err("Unable to recover from" 1039 1039 + " NOPOUT Ping DataCRC failure while in" 1040 1040 + " ERL=0.\n"); 1041 1041 + ret = -1; 1042 1042 + goto out; 1043 1043 + } else { 1044 1044 + /* 1045 1045 + * drop this PDU and let the 1046 1046 + * initiator plug the CmdSN gap. 1047 1047 + */ 1048 1048 + pr_info("Dropping NOPOUT" 1049 1049 + " Command CmdSN: 0x%08x due to" 1050 1050 + " DataCRC error.\n", hdr->cmdsn); 1051 1051 + ret = 0; 1052 1052 + goto out; 1053 1053 + } 1054 1054 + } 1055 1055 + 1056 1056 + /* 1057 1057 + * Handle NOP-OUT payload for traditional iSCSI sockets 1058 1058 + */ 1059 1059 + if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1060 1060 + ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1061 1061 + if (!ping_data) { 1062 1062 + pr_err("Unable to allocate memory for" 1063 1063 + " NOPOUT ping data.\n"); 1064 1064 + ret = -1; 1065 1065 + goto out; 1066 1066 + } 1067 1067 + 1068 1068 + skb_copy_bits(csk->skb, pdu_cb->doffset, 1069 1069 + ping_data, payload_length); 1070 1070 + 1071 1071 + ping_data[payload_length] = '\0'; 1072 1072 + /* 1073 1073 + * Attach ping data to struct iscsi_cmd->buf_ptr. 1074 1074 + */ 1075 1075 + cmd->buf_ptr = ping_data; 1076 1076 + cmd->buf_ptr_size = payload_length; 1077 1077 + 1078 1078 + pr_debug("Got %u bytes of NOPOUT ping" 1079 1079 + " data.\n", payload_length); 1080 1080 + pr_debug("Ping Data: \"%s\"\n", ping_data); 1081 1081 + } 1082 1082 + 1083 1083 + return iscsit_process_nop_out(conn, cmd, hdr); 1084 1084 + out: 1085 1085 + if (cmd) 1086 1086 + iscsit_free_cmd(cmd, false); 1087 1087 + return ret; 1088 1088 + } 1089 1089 + 1090 1090 + static int 1091 1091 + cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1092 1092 + { 1093 1093 + struct iscsi_conn *conn = csk->conn; 1094 1094 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1095 1095 + struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr; 1096 1096 + u32 payload_length = pdu_cb->dlen; 1097 1097 + int rc; 1098 1098 + unsigned char *text_in = NULL; 1099 1099 + 1100 1100 + rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1101 1101 + if (rc < 0) 1102 1102 + return rc; 1103 1103 + 1104 1104 + if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1105 1105 + if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1106 1106 + pr_err("Unable to recover from" 1107 1107 + " Text Data digest failure while in" 1108 1108 + " ERL=0.\n"); 1109 1109 + goto reject; 1110 1110 + } else { 1111 1111 + /* 1112 1112 + * drop this PDU and let the 1113 1113 + * initiator plug the CmdSN gap. 1114 1114 + */ 1115 1115 + pr_info("Dropping Text" 1116 1116 + " Command CmdSN: 0x%08x due to" 1117 1117 + " DataCRC error.\n", hdr->cmdsn); 1118 1118 + return 0; 1119 1119 + } 1120 1120 + } 1121 1121 + 1122 1122 + if (payload_length) { 1123 1123 + text_in = kzalloc(payload_length, GFP_KERNEL); 1124 1124 + if (!text_in) { 1125 1125 + pr_err("Unable to allocate text_in of payload_length: %u\n", 1126 1126 + payload_length); 1127 1127 + return -ENOMEM; 1128 1128 + } 1129 1129 + skb_copy_bits(csk->skb, pdu_cb->doffset, 1130 1130 + text_in, payload_length); 1131 1131 + 1132 1132 + text_in[payload_length - 1] = '\0'; 1133 1133 + 1134 1134 + cmd->text_in_ptr = text_in; 1135 1135 + } 1136 1136 + 1137 1137 + return iscsit_process_text_cmd(conn, cmd, hdr); 1138 1138 + 1139 1139 + reject: 1140 1140 + return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1141 1141 + pdu_cb->hdr); 1142 1142 + } 1143 1143 + 1144 1144 + static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk) 1145 1145 + { 1146 1146 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1147 1147 + struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr; 1148 1148 + struct iscsi_conn *conn = csk->conn; 1149 1149 + struct iscsi_cmd *cmd = NULL; 1150 1150 + u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1151 1151 + int ret = -EINVAL; 1152 1152 + 1153 1153 + switch (opcode) { 1154 1154 + case ISCSI_OP_SCSI_CMD: 1155 1155 + cmd = cxgbit_allocate_cmd(csk); 1156 1156 + if (!cmd) 1157 1157 + goto reject; 1158 1158 + 1159 1159 + ret = cxgbit_handle_scsi_cmd(csk, cmd); 1160 1160 + break; 1161 1161 + case ISCSI_OP_SCSI_DATA_OUT: 1162 1162 + ret = cxgbit_handle_iscsi_dataout(csk); 1163 1163 + break; 1164 1164 + case ISCSI_OP_NOOP_OUT: 1165 1165 + if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1166 1166 + cmd = cxgbit_allocate_cmd(csk); 1167 1167 + if (!cmd) 1168 1168 + goto reject; 1169 1169 + } 1170 1170 + 1171 1171 + ret = cxgbit_handle_nop_out(csk, cmd); 1172 1172 + break; 1173 1173 + case ISCSI_OP_SCSI_TMFUNC: 1174 1174 + cmd = cxgbit_allocate_cmd(csk); 1175 1175 + if (!cmd) 1176 1176 + goto reject; 1177 1177 + 1178 1178 + ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1179 1179 + (unsigned char *)hdr); 1180 1180 + break; 1181 1181 + case ISCSI_OP_TEXT: 1182 1182 + if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 1183 1183 + cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1184 1184 + if (!cmd) 1185 1185 + goto reject; 1186 1186 + } else { 1187 1187 + cmd = cxgbit_allocate_cmd(csk); 1188 1188 + if (!cmd) 1189 1189 + goto reject; 1190 1190 + } 1191 1191 + 1192 1192 + ret = cxgbit_handle_text_cmd(csk, cmd); 1193 1193 + break; 1194 1194 + case ISCSI_OP_LOGOUT: 1195 1195 + cmd = cxgbit_allocate_cmd(csk); 1196 1196 + if (!cmd) 1197 1197 + goto reject; 1198 1198 + 1199 1199 + ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1200 1200 + if (ret > 0) 1201 1201 + wait_for_completion_timeout(&conn->conn_logout_comp, 1202 1202 + SECONDS_FOR_LOGOUT_COMP 1203 1203 + * HZ); 1204 1204 + break; 1205 1205 + case ISCSI_OP_SNACK: 1206 1206 + ret = iscsit_handle_snack(conn, (unsigned char *)hdr); 1207 1207 + break; 1208 1208 + default: 1209 1209 + pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1210 1210 + dump_stack(); 1211 1211 + break; 1212 1212 + } 1213 1213 + 1214 1214 + return ret; 1215 1215 + 1216 1216 + reject: 1217 1217 + return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1218 1218 + (unsigned char *)hdr); 1219 1219 + return ret; 1220 1220 + } 1221 1221 + 1222 1222 + static int cxgbit_rx_opcode(struct cxgbit_sock *csk) 1223 1223 + { 1224 1224 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1225 1225 + struct iscsi_conn *conn = csk->conn; 1226 1226 + struct iscsi_hdr *hdr = pdu_cb->hdr; 1227 1227 + u8 opcode; 1228 1228 + 1229 1229 + if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) { 1230 1230 + atomic_long_inc(&conn->sess->conn_digest_errors); 1231 1231 + goto transport_err; 1232 1232 + } 1233 1233 + 1234 1234 + if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 1235 1235 + goto transport_err; 1236 1236 + 1237 1237 + opcode = hdr->opcode & ISCSI_OPCODE_MASK; 1238 1238 + 1239 1239 + if (conn->sess->sess_ops->SessionType && 1240 1240 + ((!(opcode & ISCSI_OP_TEXT)) || 1241 1241 + (!(opcode & ISCSI_OP_LOGOUT)))) { 1242 1242 + pr_err("Received illegal iSCSI Opcode: 0x%02x" 1243 1243 + " while in Discovery Session, rejecting.\n", opcode); 1244 1244 + iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1245 1245 + (unsigned char *)hdr); 1246 1246 + goto transport_err; 1247 1247 + } 1248 1248 + 1249 1249 + if (cxgbit_target_rx_opcode(csk) < 0) 1250 1250 + goto transport_err; 1251 1251 + 1252 1252 + return 0; 1253 1253 + 1254 1254 + transport_err: 1255 1255 + return -1; 1256 1256 + } 1257 1257 + 1258 1258 + static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk) 1259 1259 + { 1260 1260 + struct iscsi_conn *conn = csk->conn; 1261 1261 + struct iscsi_login *login = conn->login; 1262 1262 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1263 1263 + struct iscsi_login_req *login_req; 1264 1264 + 1265 1265 + login_req = (struct iscsi_login_req *)login->req; 1266 1266 + memcpy(login_req, pdu_cb->hdr, sizeof(*login_req)); 1267 1267 + 1268 1268 + pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," 1269 1269 + " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n", 1270 1270 + login_req->flags, login_req->itt, login_req->cmdsn, 1271 1271 + login_req->exp_statsn, login_req->cid, pdu_cb->dlen); 1272 1272 + /* 1273 1273 + * Setup the initial iscsi_login values from the leading 1274 1274 + * login request PDU. 1275 1275 + */ 1276 1276 + if (login->first_request) { 1277 1277 + login_req = (struct iscsi_login_req *)login->req; 1278 1278 + login->leading_connection = (!login_req->tsih) ? 1 : 0; 1279 1279 + login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 1280 1280 + login_req->flags); 1281 1281 + login->version_min = login_req->min_version; 1282 1282 + login->version_max = login_req->max_version; 1283 1283 + memcpy(login->isid, login_req->isid, 6); 1284 1284 + login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1285 1285 + login->init_task_tag = login_req->itt; 1286 1286 + login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1287 1287 + login->cid = be16_to_cpu(login_req->cid); 1288 1288 + login->tsih = be16_to_cpu(login_req->tsih); 1289 1289 + } 1290 1290 + 1291 1291 + if (iscsi_target_check_login_request(conn, login) < 0) 1292 1292 + return -1; 1293 1293 + 1294 1294 + memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS); 1295 1295 + skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen); 1296 1296 + 1297 1297 + return 0; 1298 1298 + } 1299 1299 + 1300 1300 + static int 1301 1301 + cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx) 1302 1302 + { 1303 1303 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx); 1304 1304 + int ret; 1305 1305 + 1306 1306 + cxgbit_rx_pdu_cb(skb) = pdu_cb; 1307 1307 + 1308 1308 + csk->skb = skb; 1309 1309 + 1310 1310 + if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) { 1311 1311 + ret = cxgbit_rx_login_pdu(csk); 1312 1312 + set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1313 1313 + } else { 1314 1314 + ret = cxgbit_rx_opcode(csk); 1315 1315 + } 1316 1316 + 1317 1317 + return ret; 1318 1318 + } 1319 1319 + 1320 1320 + static void cxgbit_lro_skb_dump(struct sk_buff *skb) 1321 1321 + { 1322 1322 + struct skb_shared_info *ssi = skb_shinfo(skb); 1323 1323 + struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1324 1324 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1325 1325 + u8 i; 1326 1326 + 1327 1327 + pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n", 1328 1328 + skb, skb->head, skb->data, skb->len, skb->data_len, 1329 1329 + ssi->nr_frags); 1330 1330 + pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n", 1331 1331 + skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen); 1332 1332 + 1333 1333 + for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++) 1334 1334 + pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, " 1335 1335 + "frags %u.\n", 1336 1336 + skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq, 1337 1337 + pdu_cb->ddigest, pdu_cb->frags); 1338 1338 + for (i = 0; i < ssi->nr_frags; i++) 1339 1339 + pr_info("skb 0x%p, frag %d, off %u, sz %u.\n", 1340 1340 + skb, i, ssi->frags[i].page_offset, ssi->frags[i].size); 1341 1341 + } 1342 1342 + 1343 1343 + static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk) 1344 1344 + { 1345 1345 + struct sk_buff *skb = csk->lro_hskb; 1346 1346 + struct skb_shared_info *ssi = skb_shinfo(skb); 1347 1347 + u8 i; 1348 1348 + 1349 1349 + memset(skb->data, 0, LRO_SKB_MIN_HEADROOM); 1350 1350 + for (i = 0; i < ssi->nr_frags; i++) 1351 1351 + put_page(skb_frag_page(&ssi->frags[i])); 1352 1352 + ssi->nr_frags = 0; 1353 1353 + } 1354 1354 + 1355 1355 + static void 1356 1356 + cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx) 1357 1357 + { 1358 1358 + struct sk_buff *hskb = csk->lro_hskb; 1359 1359 + struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0); 1360 1360 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx); 1361 1361 + struct skb_shared_info *hssi = skb_shinfo(hskb); 1362 1362 + struct skb_shared_info *ssi = skb_shinfo(skb); 1363 1363 + unsigned int len = 0; 1364 1364 + 1365 1365 + if (pdu_cb->flags & PDUCBF_RX_HDR) { 1366 1366 + hpdu_cb->flags = pdu_cb->flags; 1367 1367 + hpdu_cb->seq = pdu_cb->seq; 1368 1368 + hpdu_cb->hdr = pdu_cb->hdr; 1369 1369 + hpdu_cb->hlen = pdu_cb->hlen; 1370 1370 + 1371 1371 + memcpy(&hssi->frags[0], &ssi->frags[pdu_cb->hfrag_idx], 1372 1372 + sizeof(skb_frag_t)); 1373 1373 + 1374 1374 + get_page(skb_frag_page(&hssi->frags[0])); 1375 1375 + hssi->nr_frags = 1; 1376 1376 + hpdu_cb->frags = 1; 1377 1377 + hpdu_cb->hfrag_idx = 0; 1378 1378 + 1379 1379 + len = hssi->frags[0].size; 1380 1380 + hskb->len = len; 1381 1381 + hskb->data_len = len; 1382 1382 + hskb->truesize = len; 1383 1383 + } 1384 1384 + 1385 1385 + if (pdu_cb->flags & PDUCBF_RX_DATA) { 1386 1386 + u8 hfrag_idx = 1, i; 1387 1387 + 1388 1388 + hpdu_cb->flags |= pdu_cb->flags; 1389 1389 + 1390 1390 + len = 0; 1391 1391 + for (i = 0; i < pdu_cb->nr_dfrags; hfrag_idx++, i++) { 1392 1392 + memcpy(&hssi->frags[hfrag_idx], 1393 1393 + &ssi->frags[pdu_cb->dfrag_idx + i], 1394 1394 + sizeof(skb_frag_t)); 1395 1395 + 1396 1396 + get_page(skb_frag_page(&hssi->frags[hfrag_idx])); 1397 1397 + 1398 1398 + len += hssi->frags[hfrag_idx].size; 1399 1399 + 1400 1400 + hssi->nr_frags++; 1401 1401 + hpdu_cb->frags++; 1402 1402 + } 1403 1403 + 1404 1404 + hpdu_cb->dlen = pdu_cb->dlen; 1405 1405 + hpdu_cb->doffset = hpdu_cb->hlen; 1406 1406 + hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags; 1407 1407 + hpdu_cb->dfrag_idx = 1; 1408 1408 + hskb->len += len; 1409 1409 + hskb->data_len += len; 1410 1410 + hskb->truesize += len; 1411 1411 + } 1412 1412 + 1413 1413 + if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1414 1414 + hpdu_cb->flags |= pdu_cb->flags; 1415 1415 + 1416 1416 + if (hpdu_cb->flags & PDUCBF_RX_DATA) 1417 1417 + hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD; 1418 1418 + 1419 1419 + hpdu_cb->ddigest = pdu_cb->ddigest; 1420 1420 + hpdu_cb->pdulen = pdu_cb->pdulen; 1421 1421 + } 1422 1422 + } 1423 1423 + 1424 1424 + static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1425 1425 + { 1426 1426 + struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1427 1427 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1428 1428 + u8 pdu_idx = 0, last_idx = 0; 1429 1429 + int ret = 0; 1430 1430 + 1431 1431 + if (!pdu_cb->complete) { 1432 1432 + cxgbit_lro_skb_merge(csk, skb, 0); 1433 1433 + 1434 1434 + if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1435 1435 + struct sk_buff *hskb = csk->lro_hskb; 1436 1436 + 1437 1437 + ret = cxgbit_process_iscsi_pdu(csk, hskb, 0); 1438 1438 + 1439 1439 + cxgbit_lro_hskb_reset(csk); 1440 1440 + 1441 1441 + if (ret < 0) 1442 1442 + goto out; 1443 1443 + } 1444 1444 + 1445 1445 + pdu_idx = 1; 1446 1446 + } 1447 1447 + 1448 1448 + if (lro_cb->pdu_idx) 1449 1449 + last_idx = lro_cb->pdu_idx - 1; 1450 1450 + 1451 1451 + for (; pdu_idx <= last_idx; pdu_idx++) { 1452 1452 + ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx); 1453 1453 + if (ret < 0) 1454 1454 + goto out; 1455 1455 + } 1456 1456 + 1457 1457 + if ((!lro_cb->complete) && lro_cb->pdu_idx) 1458 1458 + cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx); 1459 1459 + 1460 1460 + out: 1461 1461 + return ret; 1462 1462 + } 1463 1463 + 1464 1464 + static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1465 1465 + { 1466 1466 + struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1467 1467 + struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1468 1468 + int ret = -1; 1469 1469 + 1470 1470 + if ((pdu_cb->flags & PDUCBF_RX_HDR) && 1471 1471 + (pdu_cb->seq != csk->rcv_nxt)) { 1472 1472 + pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n", 1473 1473 + csk, csk->tid, pdu_cb->seq, csk->rcv_nxt); 1474 1474 + cxgbit_lro_skb_dump(skb); 1475 1475 + return ret; 1476 1476 + } 1477 1477 + 1478 1478 + csk->rcv_nxt += lro_cb->pdu_totallen; 1479 1479 + 1480 1480 + ret = cxgbit_process_lro_skb(csk, skb); 1481 1481 + 1482 1482 + csk->rx_credits += lro_cb->pdu_totallen; 1483 1483 + 1484 1484 + if (csk->rx_credits >= (csk->rcv_win / 4)) 1485 1485 + cxgbit_rx_data_ack(csk); 1486 1486 + 1487 1487 + return ret; 1488 1488 + } 1489 1489 + 1490 1490 + static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1491 1491 + { 1492 1492 + int ret = -1; 1493 1493 + 1494 1494 + if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) 1495 1495 + ret = cxgbit_rx_lro_skb(csk, skb); 1496 1496 + 1497 1497 + __kfree_skb(skb); 1498 1498 + return ret; 1499 1499 + } 1500 1500 + 1501 1501 + static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq) 1502 1502 + { 1503 1503 + spin_lock_bh(&csk->rxq.lock); 1504 1504 + if (skb_queue_len(&csk->rxq)) { 1505 1505 + skb_queue_splice_init(&csk->rxq, rxq); 1506 1506 + spin_unlock_bh(&csk->rxq.lock); 1507 1507 + return true; 1508 1508 + } 1509 1509 + spin_unlock_bh(&csk->rxq.lock); 1510 1510 + return false; 1511 1511 + } 1512 1512 + 1513 1513 + static int cxgbit_wait_rxq(struct cxgbit_sock *csk) 1514 1514 + { 1515 1515 + struct sk_buff *skb; 1516 1516 + struct sk_buff_head rxq; 1517 1517 + 1518 1518 + skb_queue_head_init(&rxq); 1519 1519 + 1520 1520 + wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq)); 1521 1521 + 1522 1522 + if (signal_pending(current)) 1523 1523 + goto out; 1524 1524 + 1525 1525 + while ((skb = __skb_dequeue(&rxq))) { 1526 1526 + if (cxgbit_rx_skb(csk, skb)) 1527 1527 + goto out; 1528 1528 + } 1529 1529 + 1530 1530 + return 0; 1531 1531 + out: 1532 1532 + __skb_queue_purge(&rxq); 1533 1533 + return -1; 1534 1534 + } 1535 1535 + 1536 1536 + int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 1537 1537 + { 1538 1538 + struct cxgbit_sock *csk = conn->context; 1539 1539 + int ret = -1; 1540 1540 + 1541 1541 + while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) { 1542 1542 + ret = cxgbit_wait_rxq(csk); 1543 1543 + if (ret) { 1544 1544 + clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1545 1545 + break; 1546 1546 + } 1547 1547 + } 1548 1548 + 1549 1549 + return ret; 1550 1550 + } 1551 1551 + 1552 1552 + void cxgbit_get_rx_pdu(struct iscsi_conn *conn) 1553 1553 + { 1554 1554 + struct cxgbit_sock *csk = conn->context; 1555 1555 + 1556 1556 + while (!kthread_should_stop()) { 1557 1557 + iscsit_thread_check_cpumask(conn, current, 0); 1558 1558 + if (cxgbit_wait_rxq(csk)) 1559 1559 + return; 1560 1560 + } 1561 1561 + }

+272 -429

drivers/target/iscsi/iscsi_target.c

reviewed

··· 478 478 return 0; 479 479 } 480 480 481 481 - static int iscsit_immediate_queue(struct iscsi_conn *, struct iscsi_cmd *, int); 482 482 - static int iscsit_response_queue(struct iscsi_conn *, struct iscsi_cmd *, int); 481 481 + static void iscsit_get_rx_pdu(struct iscsi_conn *); 483 482 484 484 - static int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 483 483 + int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 485 484 { 486 485 iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state); 487 486 return 0; 488 487 } 488 488 + EXPORT_SYMBOL(iscsit_queue_rsp); 489 489 490 490 - static void iscsit_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 490 490 + void iscsit_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 491 491 { 492 492 bool scsi_cmd = (cmd->iscsi_opcode == ISCSI_OP_SCSI_CMD); 493 493 ··· 498 498 499 499 __iscsit_free_cmd(cmd, scsi_cmd, true); 500 500 } 501 501 + EXPORT_SYMBOL(iscsit_aborted_task); 502 502 + 503 503 + static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *, 504 504 + u32, u32, u8 *, u8 *); 505 505 + static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *); 506 506 + 507 507 + static int 508 508 + iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 509 509 + const void *data_buf, u32 data_buf_len) 510 510 + { 511 511 + struct iscsi_hdr *hdr = (struct iscsi_hdr *)cmd->pdu; 512 512 + struct kvec *iov; 513 513 + u32 niov = 0, tx_size = ISCSI_HDR_LEN; 514 514 + int ret; 515 515 + 516 516 + iov = &cmd->iov_misc[0]; 517 517 + iov[niov].iov_base = cmd->pdu; 518 518 + iov[niov++].iov_len = ISCSI_HDR_LEN; 519 519 + 520 520 + if (conn->conn_ops->HeaderDigest) { 521 521 + u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 522 522 + 523 523 + iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, 524 524 + ISCSI_HDR_LEN, 0, NULL, 525 525 + (u8 *)header_digest); 526 526 + 527 527 + iov[0].iov_len += ISCSI_CRC_LEN; 528 528 + tx_size += ISCSI_CRC_LEN; 529 529 + pr_debug("Attaching CRC32C HeaderDigest" 530 530 + " to opcode 0x%x 0x%08x\n", 531 531 + hdr->opcode, *header_digest); 532 532 + } 533 533 + 534 534 + if (data_buf_len) { 535 535 + u32 padding = ((-data_buf_len) & 3); 536 536 + 537 537 + iov[niov].iov_base = (void *)data_buf; 538 538 + iov[niov++].iov_len = data_buf_len; 539 539 + tx_size += data_buf_len; 540 540 + 541 541 + if (padding != 0) { 542 542 + iov[niov].iov_base = &cmd->pad_bytes; 543 543 + iov[niov++].iov_len = padding; 544 544 + tx_size += padding; 545 545 + pr_debug("Attaching %u additional" 546 546 + " padding bytes.\n", padding); 547 547 + } 548 548 + 549 549 + if (conn->conn_ops->DataDigest) { 550 550 + iscsit_do_crypto_hash_buf(conn->conn_tx_hash, 551 551 + data_buf, data_buf_len, 552 552 + padding, 553 553 + (u8 *)&cmd->pad_bytes, 554 554 + (u8 *)&cmd->data_crc); 555 555 + 556 556 + iov[niov].iov_base = &cmd->data_crc; 557 557 + iov[niov++].iov_len = ISCSI_CRC_LEN; 558 558 + tx_size += ISCSI_CRC_LEN; 559 559 + pr_debug("Attached DataDigest for %u" 560 560 + " bytes opcode 0x%x, CRC 0x%08x\n", 561 561 + data_buf_len, hdr->opcode, cmd->data_crc); 562 562 + } 563 563 + } 564 564 + 565 565 + cmd->iov_misc_count = niov; 566 566 + cmd->tx_size = tx_size; 567 567 + 568 568 + ret = iscsit_send_tx_data(cmd, conn, 1); 569 569 + if (ret < 0) { 570 570 + iscsit_tx_thread_wait_for_tcp(conn); 571 571 + return ret; 572 572 + } 573 573 + 574 574 + return 0; 575 575 + } 576 576 + 577 577 + static int iscsit_map_iovec(struct iscsi_cmd *, struct kvec *, u32, u32); 578 578 + static void iscsit_unmap_iovec(struct iscsi_cmd *); 579 579 + static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsi_cmd *, 580 580 + u32, u32, u32, u8 *); 581 581 + static int 582 582 + iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 583 583 + const struct iscsi_datain *datain) 584 584 + { 585 585 + struct kvec *iov; 586 586 + u32 iov_count = 0, tx_size = 0; 587 587 + int ret, iov_ret; 588 588 + 589 589 + iov = &cmd->iov_data[0]; 590 590 + iov[iov_count].iov_base = cmd->pdu; 591 591 + iov[iov_count++].iov_len = ISCSI_HDR_LEN; 592 592 + tx_size += ISCSI_HDR_LEN; 593 593 + 594 594 + if (conn->conn_ops->HeaderDigest) { 595 595 + u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 596 596 + 597 597 + iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu, 598 598 + ISCSI_HDR_LEN, 0, NULL, 599 599 + (u8 *)header_digest); 600 600 + 601 601 + iov[0].iov_len += ISCSI_CRC_LEN; 602 602 + tx_size += ISCSI_CRC_LEN; 603 603 + 604 604 + pr_debug("Attaching CRC32 HeaderDigest for DataIN PDU 0x%08x\n", 605 605 + *header_digest); 606 606 + } 607 607 + 608 608 + iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], 609 609 + datain->offset, datain->length); 610 610 + if (iov_ret < 0) 611 611 + return -1; 612 612 + 613 613 + iov_count += iov_ret; 614 614 + tx_size += datain->length; 615 615 + 616 616 + cmd->padding = ((-datain->length) & 3); 617 617 + if (cmd->padding) { 618 618 + iov[iov_count].iov_base = cmd->pad_bytes; 619 619 + iov[iov_count++].iov_len = cmd->padding; 620 620 + tx_size += cmd->padding; 621 621 + 622 622 + pr_debug("Attaching %u padding bytes\n", cmd->padding); 623 623 + } 624 624 + 625 625 + if (conn->conn_ops->DataDigest) { 626 626 + cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash, 627 627 + cmd, datain->offset, 628 628 + datain->length, 629 629 + cmd->padding, 630 630 + cmd->pad_bytes); 631 631 + 632 632 + iov[iov_count].iov_base = &cmd->data_crc; 633 633 + iov[iov_count++].iov_len = ISCSI_CRC_LEN; 634 634 + tx_size += ISCSI_CRC_LEN; 635 635 + 636 636 + pr_debug("Attached CRC32C DataDigest %d bytes, crc 0x%08x\n", 637 637 + datain->length + cmd->padding, cmd->data_crc); 638 638 + } 639 639 + 640 640 + cmd->iov_data_count = iov_count; 641 641 + cmd->tx_size = tx_size; 642 642 + 643 643 + ret = iscsit_fe_sendpage_sg(cmd, conn); 644 644 + 645 645 + iscsit_unmap_iovec(cmd); 646 646 + 647 647 + if (ret < 0) { 648 648 + iscsit_tx_thread_wait_for_tcp(conn); 649 649 + return ret; 650 650 + } 651 651 + 652 652 + return 0; 653 653 + } 654 654 + 655 655 + static int iscsit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 656 656 + struct iscsi_datain_req *dr, const void *buf, 657 657 + u32 buf_len) 658 658 + { 659 659 + if (dr) 660 660 + return iscsit_xmit_datain_pdu(conn, cmd, buf); 661 661 + else 662 662 + return iscsit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 663 663 + } 501 664 502 665 static enum target_prot_op iscsit_get_sup_prot_ops(struct iscsi_conn *conn) 503 666 { ··· 670 507 static struct iscsit_transport iscsi_target_transport = { 671 508 .name = "iSCSI/TCP", 672 509 .transport_type = ISCSI_TCP, 510 510 + .rdma_shutdown = false, 673 511 .owner = NULL, 674 512 .iscsit_setup_np = iscsit_setup_np, 675 513 .iscsit_accept_np = iscsit_accept_np, ··· 683 519 .iscsit_queue_data_in = iscsit_queue_rsp, 684 520 .iscsit_queue_status = iscsit_queue_rsp, 685 521 .iscsit_aborted_task = iscsit_aborted_task, 522 522 + .iscsit_xmit_pdu = iscsit_xmit_pdu, 523 523 + .iscsit_get_rx_pdu = iscsit_get_rx_pdu, 686 524 .iscsit_get_sup_prot_ops = iscsit_get_sup_prot_ops, 687 525 }; 688 526 ··· 800 634 kfree(iscsit_global); 801 635 } 802 636 803 803 - static int iscsit_add_reject( 637 637 + int iscsit_add_reject( 804 638 struct iscsi_conn *conn, 805 639 u8 reason, 806 640 unsigned char *buf) ··· 830 664 831 665 return -1; 832 666 } 667 667 + EXPORT_SYMBOL(iscsit_add_reject); 833 668 834 669 static int iscsit_add_reject_from_cmd( 835 670 struct iscsi_cmd *cmd, ··· 886 719 { 887 720 return iscsit_add_reject_from_cmd(cmd, reason, false, buf); 888 721 } 722 722 + EXPORT_SYMBOL(iscsit_reject_cmd); 889 723 890 724 /* 891 725 * Map some portion of the allocated scatterlist to an iovec, suitable for ··· 905 737 /* 906 738 * We know each entry in t_data_sg contains a page. 907 739 */ 908 908 - sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE]; 740 740 + u32 ent = data_offset / PAGE_SIZE; 741 741 + 742 742 + if (ent >= cmd->se_cmd.t_data_nents) { 743 743 + pr_err("Initial page entry out-of-bounds\n"); 744 744 + return -1; 745 745 + } 746 746 + 747 747 + sg = &cmd->se_cmd.t_data_sg[ent]; 909 748 page_off = (data_offset % PAGE_SIZE); 910 749 911 750 cmd->first_data_sg = sg; ··· 2510 2335 } 2511 2336 EXPORT_SYMBOL(iscsit_handle_logout_cmd); 2512 2337 2513 2513 - static int iscsit_handle_snack( 2338 2338 + int iscsit_handle_snack( 2514 2339 struct iscsi_conn *conn, 2515 2340 unsigned char *buf) 2516 2341 { ··· 2563 2388 2564 2389 return 0; 2565 2390 } 2391 2391 + EXPORT_SYMBOL(iscsit_handle_snack); 2566 2392 2567 2393 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn) 2568 2394 { ··· 2710 2534 { 2711 2535 struct iscsi_async *hdr; 2712 2536 2713 2713 - cmd->tx_size = ISCSI_HDR_LEN; 2714 2537 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT; 2715 2538 2716 2539 hdr = (struct iscsi_async *) cmd->pdu; ··· 2727 2552 hdr->param2 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait); 2728 2553 hdr->param3 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain); 2729 2554 2730 2730 - if (conn->conn_ops->HeaderDigest) { 2731 2731 - u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2732 2732 - 2733 2733 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, 2734 2734 - ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); 2735 2735 - 2736 2736 - cmd->tx_size += ISCSI_CRC_LEN; 2737 2737 - pr_debug("Attaching CRC32C HeaderDigest to" 2738 2738 - " Async Message 0x%08x\n", *header_digest); 2739 2739 - } 2740 2740 - 2741 2741 - cmd->iov_misc[0].iov_base = cmd->pdu; 2742 2742 - cmd->iov_misc[0].iov_len = cmd->tx_size; 2743 2743 - cmd->iov_misc_count = 1; 2744 2744 - 2745 2555 pr_debug("Sending Connection Dropped Async Message StatSN:" 2746 2556 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn, 2747 2557 cmd->logout_cid, conn->cid); 2748 2748 - return 0; 2558 2558 + 2559 2559 + return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 2749 2560 } 2750 2561 2751 2562 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn) ··· 2744 2583 } 2745 2584 } 2746 2585 2747 2747 - static void 2586 2586 + void 2748 2587 iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2749 2588 struct iscsi_datain *datain, struct iscsi_data_rsp *hdr, 2750 2589 bool set_statsn) ··· 2788 2627 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn), 2789 2628 ntohl(hdr->offset), datain->length, conn->cid); 2790 2629 } 2630 2630 + EXPORT_SYMBOL(iscsit_build_datain_pdu); 2791 2631 2792 2632 static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2793 2633 { 2794 2634 struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0]; 2795 2635 struct iscsi_datain datain; 2796 2636 struct iscsi_datain_req *dr; 2797 2797 - struct kvec *iov; 2798 2798 - u32 iov_count = 0, tx_size = 0; 2799 2799 - int eodr = 0, ret, iov_ret; 2637 2637 + int eodr = 0, ret; 2800 2638 bool set_statsn = false; 2801 2639 2802 2640 memset(&datain, 0, sizeof(struct iscsi_datain)); ··· 2837 2677 2838 2678 iscsit_build_datain_pdu(cmd, conn, &datain, hdr, set_statsn); 2839 2679 2840 2840 - iov = &cmd->iov_data[0]; 2841 2841 - iov[iov_count].iov_base = cmd->pdu; 2842 2842 - iov[iov_count++].iov_len = ISCSI_HDR_LEN; 2843 2843 - tx_size += ISCSI_HDR_LEN; 2844 2844 - 2845 2845 - if (conn->conn_ops->HeaderDigest) { 2846 2846 - u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2847 2847 - 2848 2848 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu, 2849 2849 - ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); 2850 2850 - 2851 2851 - iov[0].iov_len += ISCSI_CRC_LEN; 2852 2852 - tx_size += ISCSI_CRC_LEN; 2853 2853 - 2854 2854 - pr_debug("Attaching CRC32 HeaderDigest" 2855 2855 - " for DataIN PDU 0x%08x\n", *header_digest); 2856 2856 - } 2857 2857 - 2858 2858 - iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], 2859 2859 - datain.offset, datain.length); 2860 2860 - if (iov_ret < 0) 2861 2861 - return -1; 2862 2862 - 2863 2863 - iov_count += iov_ret; 2864 2864 - tx_size += datain.length; 2865 2865 - 2866 2866 - cmd->padding = ((-datain.length) & 3); 2867 2867 - if (cmd->padding) { 2868 2868 - iov[iov_count].iov_base = cmd->pad_bytes; 2869 2869 - iov[iov_count++].iov_len = cmd->padding; 2870 2870 - tx_size += cmd->padding; 2871 2871 - 2872 2872 - pr_debug("Attaching %u padding bytes\n", 2873 2873 - cmd->padding); 2874 2874 - } 2875 2875 - if (conn->conn_ops->DataDigest) { 2876 2876 - cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash, cmd, 2877 2877 - datain.offset, datain.length, cmd->padding, cmd->pad_bytes); 2878 2878 - 2879 2879 - iov[iov_count].iov_base = &cmd->data_crc; 2880 2880 - iov[iov_count++].iov_len = ISCSI_CRC_LEN; 2881 2881 - tx_size += ISCSI_CRC_LEN; 2882 2882 - 2883 2883 - pr_debug("Attached CRC32C DataDigest %d bytes, crc" 2884 2884 - " 0x%08x\n", datain.length+cmd->padding, cmd->data_crc); 2885 2885 - } 2886 2886 - 2887 2887 - cmd->iov_data_count = iov_count; 2888 2888 - cmd->tx_size = tx_size; 2889 2889 - 2890 2890 - ret = iscsit_fe_sendpage_sg(cmd, conn); 2891 2891 - 2892 2892 - iscsit_unmap_iovec(cmd); 2893 2893 - 2894 2894 - if (ret < 0) { 2895 2895 - iscsit_tx_thread_wait_for_tcp(conn); 2680 2680 + ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, dr, &datain, 0); 2681 2681 + if (ret < 0) 2896 2682 return ret; 2897 2897 - } 2898 2683 2899 2684 if (dr->dr_complete) { 2900 2685 eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ? ··· 2948 2843 static int 2949 2844 iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2950 2845 { 2951 2951 - struct kvec *iov; 2952 2952 - int niov = 0, tx_size, rc; 2846 2846 + int rc; 2953 2847 2954 2848 rc = iscsit_build_logout_rsp(cmd, conn, 2955 2849 (struct iscsi_logout_rsp *)&cmd->pdu[0]); 2956 2850 if (rc < 0) 2957 2851 return rc; 2958 2852 2959 2959 - tx_size = ISCSI_HDR_LEN; 2960 2960 - iov = &cmd->iov_misc[0]; 2961 2961 - iov[niov].iov_base = cmd->pdu; 2962 2962 - iov[niov++].iov_len = ISCSI_HDR_LEN; 2963 2963 - 2964 2964 - if (conn->conn_ops->HeaderDigest) { 2965 2965 - u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2966 2966 - 2967 2967 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, &cmd->pdu[0], 2968 2968 - ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); 2969 2969 - 2970 2970 - iov[0].iov_len += ISCSI_CRC_LEN; 2971 2971 - tx_size += ISCSI_CRC_LEN; 2972 2972 - pr_debug("Attaching CRC32C HeaderDigest to" 2973 2973 - " Logout Response 0x%08x\n", *header_digest); 2974 2974 - } 2975 2975 - cmd->iov_misc_count = niov; 2976 2976 - cmd->tx_size = tx_size; 2977 2977 - 2978 2978 - return 0; 2853 2853 + return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 2979 2854 } 2980 2855 2981 2856 void ··· 2995 2910 int want_response) 2996 2911 { 2997 2912 struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; 2998 2998 - int tx_size = ISCSI_HDR_LEN, ret; 2913 2913 + int ret; 2999 2914 3000 2915 iscsit_build_nopin_rsp(cmd, conn, hdr, false); 3001 3001 - 3002 3002 - if (conn->conn_ops->HeaderDigest) { 3003 3003 - u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3004 3004 - 3005 3005 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, 3006 3006 - ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); 3007 3007 - 3008 3008 - tx_size += ISCSI_CRC_LEN; 3009 3009 - pr_debug("Attaching CRC32C HeaderDigest to" 3010 3010 - " NopIN 0x%08x\n", *header_digest); 3011 3011 - } 3012 3012 - 3013 3013 - cmd->iov_misc[0].iov_base = cmd->pdu; 3014 3014 - cmd->iov_misc[0].iov_len = tx_size; 3015 3015 - cmd->iov_misc_count = 1; 3016 3016 - cmd->tx_size = tx_size; 3017 2916 3018 2917 pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:" 3019 2918 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid); 3020 2919 3021 3021 - ret = iscsit_send_tx_data(cmd, conn, 1); 3022 3022 - if (ret < 0) { 3023 3023 - iscsit_tx_thread_wait_for_tcp(conn); 2920 2920 + ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 2921 2921 + if (ret < 0) 3024 2922 return ret; 3025 3025 - } 3026 2923 3027 2924 spin_lock_bh(&cmd->istate_lock); 3028 2925 cmd->i_state = want_response ? ··· 3018 2951 iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3019 2952 { 3020 2953 struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; 3021 3021 - struct kvec *iov; 3022 3022 - u32 padding = 0; 3023 3023 - int niov = 0, tx_size; 3024 2954 3025 2955 iscsit_build_nopin_rsp(cmd, conn, hdr, true); 3026 3026 - 3027 3027 - tx_size = ISCSI_HDR_LEN; 3028 3028 - iov = &cmd->iov_misc[0]; 3029 3029 - iov[niov].iov_base = cmd->pdu; 3030 3030 - iov[niov++].iov_len = ISCSI_HDR_LEN; 3031 3031 - 3032 3032 - if (conn->conn_ops->HeaderDigest) { 3033 3033 - u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3034 3034 - 3035 3035 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, 3036 3036 - ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); 3037 3037 - 3038 3038 - iov[0].iov_len += ISCSI_CRC_LEN; 3039 3039 - tx_size += ISCSI_CRC_LEN; 3040 3040 - pr_debug("Attaching CRC32C HeaderDigest" 3041 3041 - " to NopIn 0x%08x\n", *header_digest); 3042 3042 - } 3043 2956 3044 2957 /* 3045 2958 * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr. 3046 2959 * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size. 3047 2960 */ 3048 3048 - if (cmd->buf_ptr_size) { 3049 3049 - iov[niov].iov_base = cmd->buf_ptr; 3050 3050 - iov[niov++].iov_len = cmd->buf_ptr_size; 3051 3051 - tx_size += cmd->buf_ptr_size; 2961 2961 + pr_debug("Echoing back %u bytes of ping data.\n", cmd->buf_ptr_size); 3052 2962 3053 3053 - pr_debug("Echoing back %u bytes of ping" 3054 3054 - " data.\n", cmd->buf_ptr_size); 3055 3055 - 3056 3056 - padding = ((-cmd->buf_ptr_size) & 3); 3057 3057 - if (padding != 0) { 3058 3058 - iov[niov].iov_base = &cmd->pad_bytes; 3059 3059 - iov[niov++].iov_len = padding; 3060 3060 - tx_size += padding; 3061 3061 - pr_debug("Attaching %u additional" 3062 3062 - " padding bytes.\n", padding); 3063 3063 - } 3064 3064 - if (conn->conn_ops->DataDigest) { 3065 3065 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, 3066 3066 - cmd->buf_ptr, cmd->buf_ptr_size, 3067 3067 - padding, (u8 *)&cmd->pad_bytes, 3068 3068 - (u8 *)&cmd->data_crc); 3069 3069 - 3070 3070 - iov[niov].iov_base = &cmd->data_crc; 3071 3071 - iov[niov++].iov_len = ISCSI_CRC_LEN; 3072 3072 - tx_size += ISCSI_CRC_LEN; 3073 3073 - pr_debug("Attached DataDigest for %u" 3074 3074 - " bytes of ping data, CRC 0x%08x\n", 3075 3075 - cmd->buf_ptr_size, cmd->data_crc); 3076 3076 - } 3077 3077 - } 3078 3078 - 3079 3079 - cmd->iov_misc_count = niov; 3080 3080 - cmd->tx_size = tx_size; 3081 3081 - 3082 3082 - return 0; 2963 2963 + return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 2964 2964 + cmd->buf_ptr, 2965 2965 + cmd->buf_ptr_size); 3083 2966 } 3084 2967 3085 2968 static int iscsit_send_r2t( 3086 2969 struct iscsi_cmd *cmd, 3087 2970 struct iscsi_conn *conn) 3088 2971 { 3089 3089 - int tx_size = 0; 3090 2972 struct iscsi_r2t *r2t; 3091 2973 struct iscsi_r2t_rsp *hdr; 3092 2974 int ret; ··· 3051 3035 int_to_scsilun(cmd->se_cmd.orig_fe_lun, 3052 3036 (struct scsi_lun *)&hdr->lun); 3053 3037 hdr->itt = cmd->init_task_tag; 3054 3054 - r2t->targ_xfer_tag = session_get_next_ttt(conn->sess); 3038 3038 + if (conn->conn_transport->iscsit_get_r2t_ttt) 3039 3039 + conn->conn_transport->iscsit_get_r2t_ttt(conn, cmd, r2t); 3040 3040 + else 3041 3041 + r2t->targ_xfer_tag = session_get_next_ttt(conn->sess); 3055 3042 hdr->ttt = cpu_to_be32(r2t->targ_xfer_tag); 3056 3043 hdr->statsn = cpu_to_be32(conn->stat_sn); 3057 3044 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); ··· 3063 3044 hdr->data_offset = cpu_to_be32(r2t->offset); 3064 3045 hdr->data_length = cpu_to_be32(r2t->xfer_len); 3065 3046 3066 3066 - cmd->iov_misc[0].iov_base = cmd->pdu; 3067 3067 - cmd->iov_misc[0].iov_len = ISCSI_HDR_LEN; 3068 3068 - tx_size += ISCSI_HDR_LEN; 3069 3069 - 3070 3070 - if (conn->conn_ops->HeaderDigest) { 3071 3071 - u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3072 3072 - 3073 3073 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, 3074 3074 - ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); 3075 3075 - 3076 3076 - cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN; 3077 3077 - tx_size += ISCSI_CRC_LEN; 3078 3078 - pr_debug("Attaching CRC32 HeaderDigest for R2T" 3079 3079 - " PDU 0x%08x\n", *header_digest); 3080 3080 - } 3081 3081 - 3082 3047 pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:" 3083 3048 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n", 3084 3049 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag, 3085 3050 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn, 3086 3051 r2t->offset, r2t->xfer_len, conn->cid); 3087 3052 3088 3088 - cmd->iov_misc_count = 1; 3089 3089 - cmd->tx_size = tx_size; 3090 3090 - 3091 3053 spin_lock_bh(&cmd->r2t_lock); 3092 3054 r2t->sent_r2t = 1; 3093 3055 spin_unlock_bh(&cmd->r2t_lock); 3094 3056 3095 3095 - ret = iscsit_send_tx_data(cmd, conn, 1); 3057 3057 + ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 3096 3058 if (ret < 0) { 3097 3097 - iscsit_tx_thread_wait_for_tcp(conn); 3098 3059 return ret; 3099 3060 } 3100 3061 ··· 3165 3166 3166 3167 return 0; 3167 3168 } 3169 3169 + EXPORT_SYMBOL(iscsit_build_r2ts_for_cmd); 3168 3170 3169 3171 void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3170 3172 bool inc_stat_sn, struct iscsi_scsi_rsp *hdr) ··· 3204 3204 static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3205 3205 { 3206 3206 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0]; 3207 3207 - struct kvec *iov; 3208 3208 - u32 padding = 0, tx_size = 0; 3209 3209 - int iov_count = 0; 3210 3207 bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS); 3208 3208 + void *data_buf = NULL; 3209 3209 + u32 padding = 0, data_buf_len = 0; 3211 3210 3212 3211 iscsit_build_rsp_pdu(cmd, conn, inc_stat_sn, hdr); 3213 3213 - 3214 3214 - iov = &cmd->iov_misc[0]; 3215 3215 - iov[iov_count].iov_base = cmd->pdu; 3216 3216 - iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3217 3217 - tx_size += ISCSI_HDR_LEN; 3218 3212 3219 3213 /* 3220 3214 * Attach SENSE DATA payload to iSCSI Response PDU ··· 3221 3227 3222 3228 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 3223 3229 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 3224 3224 - iov[iov_count].iov_base = cmd->sense_buffer; 3225 3225 - iov[iov_count++].iov_len = 3226 3226 - (cmd->se_cmd.scsi_sense_length + padding); 3227 3227 - tx_size += cmd->se_cmd.scsi_sense_length; 3230 3230 + data_buf = cmd->sense_buffer; 3231 3231 + data_buf_len = cmd->se_cmd.scsi_sense_length + padding; 3228 3232 3229 3233 if (padding) { 3230 3234 memset(cmd->sense_buffer + 3231 3235 cmd->se_cmd.scsi_sense_length, 0, padding); 3232 3232 - tx_size += padding; 3233 3236 pr_debug("Adding %u bytes of padding to" 3234 3237 " SENSE.\n", padding); 3235 3235 - } 3236 3236 - 3237 3237 - if (conn->conn_ops->DataDigest) { 3238 3238 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, 3239 3239 - cmd->sense_buffer, 3240 3240 - (cmd->se_cmd.scsi_sense_length + padding), 3241 3241 - 0, NULL, (u8 *)&cmd->data_crc); 3242 3242 - 3243 3243 - iov[iov_count].iov_base = &cmd->data_crc; 3244 3244 - iov[iov_count++].iov_len = ISCSI_CRC_LEN; 3245 3245 - tx_size += ISCSI_CRC_LEN; 3246 3246 - 3247 3247 - pr_debug("Attaching CRC32 DataDigest for" 3248 3248 - " SENSE, %u bytes CRC 0x%08x\n", 3249 3249 - (cmd->se_cmd.scsi_sense_length + padding), 3250 3250 - cmd->data_crc); 3251 3238 } 3252 3239 3253 3240 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI" ··· 3236 3261 cmd->se_cmd.scsi_sense_length); 3237 3262 } 3238 3263 3239 3239 - if (conn->conn_ops->HeaderDigest) { 3240 3240 - u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3241 3241 - 3242 3242 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu, 3243 3243 - ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); 3244 3244 - 3245 3245 - iov[0].iov_len += ISCSI_CRC_LEN; 3246 3246 - tx_size += ISCSI_CRC_LEN; 3247 3247 - pr_debug("Attaching CRC32 HeaderDigest for Response" 3248 3248 - " PDU 0x%08x\n", *header_digest); 3249 3249 - } 3250 3250 - 3251 3251 - cmd->iov_misc_count = iov_count; 3252 3252 - cmd->tx_size = tx_size; 3253 3253 - 3254 3254 - return 0; 3264 3264 + return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, data_buf, 3265 3265 + data_buf_len); 3255 3266 } 3256 3267 3257 3268 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr) ··· 3284 3323 iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3285 3324 { 3286 3325 struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0]; 3287 3287 - u32 tx_size = 0; 3288 3326 3289 3327 iscsit_build_task_mgt_rsp(cmd, conn, hdr); 3290 3328 3291 3291 - cmd->iov_misc[0].iov_base = cmd->pdu; 3292 3292 - cmd->iov_misc[0].iov_len = ISCSI_HDR_LEN; 3293 3293 - tx_size += ISCSI_HDR_LEN; 3294 3294 - 3295 3295 - if (conn->conn_ops->HeaderDigest) { 3296 3296 - u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3297 3297 - 3298 3298 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, 3299 3299 - ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); 3300 3300 - 3301 3301 - cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN; 3302 3302 - tx_size += ISCSI_CRC_LEN; 3303 3303 - pr_debug("Attaching CRC32 HeaderDigest for Task" 3304 3304 - " Mgmt Response PDU 0x%08x\n", *header_digest); 3305 3305 - } 3306 3306 - 3307 3307 - cmd->iov_misc_count = 1; 3308 3308 - cmd->tx_size = tx_size; 3309 3309 - 3310 3310 - return 0; 3329 3329 + return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 3311 3330 } 3312 3331 3313 3332 static bool iscsit_check_inaddr_any(struct iscsi_np *np) ··· 3524 3583 struct iscsi_conn *conn) 3525 3584 { 3526 3585 struct iscsi_text_rsp *hdr = (struct iscsi_text_rsp *)cmd->pdu; 3527 3527 - struct kvec *iov; 3528 3528 - u32 tx_size = 0; 3529 3529 - int text_length, iov_count = 0, rc; 3586 3586 + int text_length; 3530 3587 3531 3531 - rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_TCP); 3532 3532 - if (rc < 0) 3533 3533 - return rc; 3588 3588 + text_length = iscsit_build_text_rsp(cmd, conn, hdr, 3589 3589 + conn->conn_transport->transport_type); 3590 3590 + if (text_length < 0) 3591 3591 + return text_length; 3534 3592 3535 3535 - text_length = rc; 3536 3536 - iov = &cmd->iov_misc[0]; 3537 3537 - iov[iov_count].iov_base = cmd->pdu; 3538 3538 - iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3539 3539 - iov[iov_count].iov_base = cmd->buf_ptr; 3540 3540 - iov[iov_count++].iov_len = text_length; 3541 3541 - 3542 3542 - tx_size += (ISCSI_HDR_LEN + text_length); 3543 3543 - 3544 3544 - if (conn->conn_ops->HeaderDigest) { 3545 3545 - u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3546 3546 - 3547 3547 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, 3548 3548 - ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); 3549 3549 - 3550 3550 - iov[0].iov_len += ISCSI_CRC_LEN; 3551 3551 - tx_size += ISCSI_CRC_LEN; 3552 3552 - pr_debug("Attaching CRC32 HeaderDigest for" 3553 3553 - " Text Response PDU 0x%08x\n", *header_digest); 3554 3554 - } 3555 3555 - 3556 3556 - if (conn->conn_ops->DataDigest) { 3557 3557 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, 3558 3558 - cmd->buf_ptr, text_length, 3559 3559 - 0, NULL, (u8 *)&cmd->data_crc); 3560 3560 - 3561 3561 - iov[iov_count].iov_base = &cmd->data_crc; 3562 3562 - iov[iov_count++].iov_len = ISCSI_CRC_LEN; 3563 3563 - tx_size += ISCSI_CRC_LEN; 3564 3564 - 3565 3565 - pr_debug("Attaching DataDigest for %u bytes of text" 3566 3566 - " data, CRC 0x%08x\n", text_length, 3567 3567 - cmd->data_crc); 3568 3568 - } 3569 3569 - 3570 3570 - cmd->iov_misc_count = iov_count; 3571 3571 - cmd->tx_size = tx_size; 3572 3572 - 3573 3573 - return 0; 3593 3593 + return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 3594 3594 + cmd->buf_ptr, 3595 3595 + text_length); 3574 3596 } 3575 3597 3576 3598 void ··· 3558 3654 struct iscsi_conn *conn) 3559 3655 { 3560 3656 struct iscsi_reject *hdr = (struct iscsi_reject *)&cmd->pdu[0]; 3561 3561 - struct kvec *iov; 3562 3562 - u32 iov_count = 0, tx_size; 3563 3657 3564 3658 iscsit_build_reject(cmd, conn, hdr); 3565 3565 - 3566 3566 - iov = &cmd->iov_misc[0]; 3567 3567 - iov[iov_count].iov_base = cmd->pdu; 3568 3568 - iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3569 3569 - iov[iov_count].iov_base = cmd->buf_ptr; 3570 3570 - iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3571 3571 - 3572 3572 - tx_size = (ISCSI_HDR_LEN + ISCSI_HDR_LEN); 3573 3573 - 3574 3574 - if (conn->conn_ops->HeaderDigest) { 3575 3575 - u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3576 3576 - 3577 3577 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, 3578 3578 - ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); 3579 3579 - 3580 3580 - iov[0].iov_len += ISCSI_CRC_LEN; 3581 3581 - tx_size += ISCSI_CRC_LEN; 3582 3582 - pr_debug("Attaching CRC32 HeaderDigest for" 3583 3583 - " REJECT PDU 0x%08x\n", *header_digest); 3584 3584 - } 3585 3585 - 3586 3586 - if (conn->conn_ops->DataDigest) { 3587 3587 - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->buf_ptr, 3588 3588 - ISCSI_HDR_LEN, 0, NULL, (u8 *)&cmd->data_crc); 3589 3589 - 3590 3590 - iov[iov_count].iov_base = &cmd->data_crc; 3591 3591 - iov[iov_count++].iov_len = ISCSI_CRC_LEN; 3592 3592 - tx_size += ISCSI_CRC_LEN; 3593 3593 - pr_debug("Attaching CRC32 DataDigest for REJECT" 3594 3594 - " PDU 0x%08x\n", cmd->data_crc); 3595 3595 - } 3596 3596 - 3597 3597 - cmd->iov_misc_count = iov_count; 3598 3598 - cmd->tx_size = tx_size; 3599 3659 3600 3660 pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x," 3601 3661 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid); 3602 3662 3603 3603 - return 0; 3663 3663 + return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 3664 3664 + cmd->buf_ptr, 3665 3665 + ISCSI_HDR_LEN); 3604 3666 } 3605 3667 3606 3668 void iscsit_thread_get_cpumask(struct iscsi_conn *conn) ··· 3594 3724 cpumask_setall(conn->conn_cpumask); 3595 3725 } 3596 3726 3597 3597 - static inline void iscsit_thread_check_cpumask( 3598 3598 - struct iscsi_conn *conn, 3599 3599 - struct task_struct *p, 3600 3600 - int mode) 3601 3601 - { 3602 3602 - /* 3603 3603 - * mode == 1 signals iscsi_target_tx_thread() usage. 3604 3604 - * mode == 0 signals iscsi_target_rx_thread() usage. 3605 3605 - */ 3606 3606 - if (mode == 1) { 3607 3607 - if (!conn->conn_tx_reset_cpumask) 3608 3608 - return; 3609 3609 - conn->conn_tx_reset_cpumask = 0; 3610 3610 - } else { 3611 3611 - if (!conn->conn_rx_reset_cpumask) 3612 3612 - return; 3613 3613 - conn->conn_rx_reset_cpumask = 0; 3614 3614 - } 3615 3615 - /* 3616 3616 - * Update the CPU mask for this single kthread so that 3617 3617 - * both TX and RX kthreads are scheduled to run on the 3618 3618 - * same CPU. 3619 3619 - */ 3620 3620 - set_cpus_allowed_ptr(p, conn->conn_cpumask); 3621 3621 - } 3622 3622 - 3623 3623 - static int 3727 3727 + int 3624 3728 iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3625 3729 { 3626 3730 int ret; ··· 3636 3792 err: 3637 3793 return -1; 3638 3794 } 3795 3795 + EXPORT_SYMBOL(iscsit_immediate_queue); 3639 3796 3640 3797 static int 3641 3798 iscsit_handle_immediate_queue(struct iscsi_conn *conn) ··· 3661 3816 return 0; 3662 3817 } 3663 3818 3664 3664 - static int 3819 3819 + int 3665 3820 iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3666 3821 { 3667 3822 int ret; ··· 3734 3889 if (ret < 0) 3735 3890 goto err; 3736 3891 3737 3737 - if (iscsit_send_tx_data(cmd, conn, 1) < 0) { 3738 3738 - iscsit_tx_thread_wait_for_tcp(conn); 3739 3739 - iscsit_unmap_iovec(cmd); 3740 3740 - goto err; 3741 3741 - } 3742 3742 - iscsit_unmap_iovec(cmd); 3743 3743 - 3744 3892 switch (state) { 3745 3893 case ISTATE_SEND_LOGOUTRSP: 3746 3894 if (!iscsit_logout_post_handler(cmd, conn)) ··· 3766 3928 err: 3767 3929 return -1; 3768 3930 } 3931 3931 + EXPORT_SYMBOL(iscsit_response_queue); 3769 3932 3770 3933 static int iscsit_handle_response_queue(struct iscsi_conn *conn) 3771 3934 { ··· 3926 4087 return ret; 3927 4088 } 3928 4089 3929 3929 - int iscsi_target_rx_thread(void *arg) 4090 4090 + static void iscsit_get_rx_pdu(struct iscsi_conn *conn) 3930 4091 { 3931 3931 - int ret, rc; 4092 4092 + int ret; 3932 4093 u8 buffer[ISCSI_HDR_LEN], opcode; 3933 4094 u32 checksum = 0, digest = 0; 3934 3934 - struct iscsi_conn *conn = arg; 3935 4095 struct kvec iov; 3936 3936 - /* 3937 3937 - * Allow ourselves to be interrupted by SIGINT so that a 3938 3938 - * connection recovery / failure event can be triggered externally. 3939 3939 - */ 3940 3940 - allow_signal(SIGINT); 3941 3941 - /* 3942 3942 - * Wait for iscsi_post_login_handler() to complete before allowing 3943 3943 - * incoming iscsi/tcp socket I/O, and/or failing the connection. 3944 3944 - */ 3945 3945 - rc = wait_for_completion_interruptible(&conn->rx_login_comp); 3946 3946 - if (rc < 0 || iscsi_target_check_conn_state(conn)) 3947 3947 - return 0; 3948 3948 - 3949 3949 - if (conn->conn_transport->transport_type == ISCSI_INFINIBAND) { 3950 3950 - struct completion comp; 3951 3951 - 3952 3952 - init_completion(&comp); 3953 3953 - rc = wait_for_completion_interruptible(&comp); 3954 3954 - if (rc < 0) 3955 3955 - goto transport_err; 3956 3956 - 3957 3957 - goto transport_err; 3958 3958 - } 3959 4096 3960 4097 while (!kthread_should_stop()) { 3961 4098 /* ··· 3949 4134 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN); 3950 4135 if (ret != ISCSI_HDR_LEN) { 3951 4136 iscsit_rx_thread_wait_for_tcp(conn); 3952 3952 - goto transport_err; 4137 4137 + return; 3953 4138 } 3954 4139 3955 4140 if (conn->conn_ops->HeaderDigest) { ··· 3959 4144 ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN); 3960 4145 if (ret != ISCSI_CRC_LEN) { 3961 4146 iscsit_rx_thread_wait_for_tcp(conn); 3962 3962 - goto transport_err; 4147 4147 + return; 3963 4148 } 3964 4149 3965 4150 iscsit_do_crypto_hash_buf(conn->conn_rx_hash, ··· 3983 4168 } 3984 4169 3985 4170 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 3986 3986 - goto transport_err; 4171 4171 + return; 3987 4172 3988 4173 opcode = buffer[0] & ISCSI_OPCODE_MASK; 3989 4174 ··· 3994 4179 " while in Discovery Session, rejecting.\n", opcode); 3995 4180 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 3996 4181 buffer); 3997 3997 - goto transport_err; 4182 4182 + return; 3998 4183 } 3999 4184 4000 4185 ret = iscsi_target_rx_opcode(conn, buffer); 4001 4186 if (ret < 0) 4002 4002 - goto transport_err; 4187 4187 + return; 4003 4188 } 4189 4189 + } 4004 4190 4005 4005 - transport_err: 4191 4191 + int iscsi_target_rx_thread(void *arg) 4192 4192 + { 4193 4193 + int rc; 4194 4194 + struct iscsi_conn *conn = arg; 4195 4195 + 4196 4196 + /* 4197 4197 + * Allow ourselves to be interrupted by SIGINT so that a 4198 4198 + * connection recovery / failure event can be triggered externally. 4199 4199 + */ 4200 4200 + allow_signal(SIGINT); 4201 4201 + /* 4202 4202 + * Wait for iscsi_post_login_handler() to complete before allowing 4203 4203 + * incoming iscsi/tcp socket I/O, and/or failing the connection. 4204 4204 + */ 4205 4205 + rc = wait_for_completion_interruptible(&conn->rx_login_comp); 4206 4206 + if (rc < 0 || iscsi_target_check_conn_state(conn)) 4207 4207 + return 0; 4208 4208 + 4209 4209 + if (!conn->conn_transport->iscsit_get_rx_pdu) 4210 4210 + return 0; 4211 4211 + 4212 4212 + conn->conn_transport->iscsit_get_rx_pdu(conn); 4213 4213 + 4006 4214 if (!signal_pending(current)) 4007 4215 atomic_set(&conn->transport_failed, 1); 4008 4216 iscsit_take_action_for_connection_exit(conn); ··· 4078 4240 pr_debug("Closing iSCSI connection CID %hu on SID:" 4079 4241 " %u\n", conn->cid, sess->sid); 4080 4242 /* 4081 4081 - * Always up conn_logout_comp for the traditional TCP case just in case 4082 4082 - * the RX Thread in iscsi_target_rx_opcode() is sleeping and the logout 4083 4083 - * response never got sent because the connection failed. 4243 4243 + * Always up conn_logout_comp for the traditional TCP and HW_OFFLOAD 4244 4244 + * case just in case the RX Thread in iscsi_target_rx_opcode() is 4245 4245 + * sleeping and the logout response never got sent because the 4246 4246 + * connection failed. 4084 4247 * 4085 4248 * However for iser-target, isert_wait4logout() is using conn_logout_comp 4086 4249 * to signal logout response TX interrupt completion. Go ahead and skip 4087 4250 * this for iser since isert_rx_opcode() does not wait on logout failure, 4088 4251 * and to avoid iscsi_conn pointer dereference in iser-target code. 4089 4252 */ 4090 4090 - if (conn->conn_transport->transport_type == ISCSI_TCP) 4253 4253 + if (!conn->conn_transport->rdma_shutdown) 4091 4254 complete(&conn->conn_logout_comp); 4092 4255 4093 4256 if (!strcmp(current->comm, ISCSI_RX_THREAD_NAME)) { ··· 4277 4438 if (!atomic_read(&sess->session_reinstatement) && 4278 4439 atomic_read(&sess->session_fall_back_to_erl0)) { 4279 4440 spin_unlock_bh(&sess->conn_lock); 4280 4280 - target_put_session(sess->se_sess); 4441 4441 + iscsit_close_session(sess); 4281 4442 4282 4443 return 0; 4283 4444 } else if (atomic_read(&sess->session_logout)) { ··· 4306 4467 } 4307 4468 } 4308 4469 4470 4470 + /* 4471 4471 + * If the iSCSI Session for the iSCSI Initiator Node exists, 4472 4472 + * forcefully shutdown the iSCSI NEXUS. 4473 4473 + */ 4309 4474 int iscsit_close_session(struct iscsi_session *sess) 4310 4475 { 4311 4476 struct iscsi_portal_group *tpg = sess->tpg; ··· 4399 4556 * always sleep waiting for RX/TX thread shutdown to complete 4400 4557 * within iscsit_close_connection(). 4401 4558 */ 4402 4402 - if (conn->conn_transport->transport_type == ISCSI_TCP) 4559 4559 + if (!conn->conn_transport->rdma_shutdown) 4403 4560 sleep = cmpxchg(&conn->tx_thread_active, true, false); 4404 4561 4405 4562 atomic_set(&conn->conn_logout_remove, 0); ··· 4408 4565 iscsit_dec_conn_usage_count(conn); 4409 4566 iscsit_stop_session(sess, sleep, sleep); 4410 4567 iscsit_dec_session_usage_count(sess); 4411 4411 - target_put_session(sess->se_sess); 4568 4568 + iscsit_close_session(sess); 4412 4569 } 4413 4570 4414 4571 static void iscsit_logout_post_handler_samecid( ··· 4416 4573 { 4417 4574 int sleep = 1; 4418 4575 4419 4419 - if (conn->conn_transport->transport_type == ISCSI_TCP) 4576 4576 + if (!conn->conn_transport->rdma_shutdown) 4420 4577 sleep = cmpxchg(&conn->tx_thread_active, true, false); 4421 4578 4422 4579 atomic_set(&conn->conn_logout_remove, 0); ··· 4579 4736 } else 4580 4737 spin_unlock_bh(&sess->conn_lock); 4581 4738 4582 4582 - target_put_session(sess->se_sess); 4739 4739 + iscsit_close_session(sess); 4583 4740 return 0; 4584 4741 } 4585 4742

+53 -109

drivers/target/iscsi/iscsi_target_configfs.c

reviewed

··· 43 43 return container_of(to_tpg_np(item), struct iscsi_tpg_np, se_tpg_np); 44 44 } 45 45 46 46 - static ssize_t lio_target_np_sctp_show(struct config_item *item, char *page) 46 46 + static ssize_t lio_target_np_driver_show(struct config_item *item, char *page, 47 47 + enum iscsit_transport_type type) 47 48 { 48 49 struct iscsi_tpg_np *tpg_np = to_iscsi_tpg_np(item); 49 49 - struct iscsi_tpg_np *tpg_np_sctp; 50 50 + struct iscsi_tpg_np *tpg_np_new; 50 51 ssize_t rb; 51 52 52 52 - tpg_np_sctp = iscsit_tpg_locate_child_np(tpg_np, ISCSI_SCTP_TCP); 53 53 - if (tpg_np_sctp) 53 53 + tpg_np_new = iscsit_tpg_locate_child_np(tpg_np, type); 54 54 + if (tpg_np_new) 54 55 rb = sprintf(page, "1\n"); 55 56 else 56 57 rb = sprintf(page, "0\n"); ··· 59 58 return rb; 60 59 } 61 60 62 62 - static ssize_t lio_target_np_sctp_store(struct config_item *item, 63 63 - const char *page, size_t count) 61 61 + static ssize_t lio_target_np_driver_store(struct config_item *item, 62 62 + const char *page, size_t count, enum iscsit_transport_type type, 63 63 + const char *mod_name) 64 64 { 65 65 struct iscsi_tpg_np *tpg_np = to_iscsi_tpg_np(item); 66 66 struct iscsi_np *np; 67 67 struct iscsi_portal_group *tpg; 68 68 - struct iscsi_tpg_np *tpg_np_sctp = NULL; 68 68 + struct iscsi_tpg_np *tpg_np_new = NULL; 69 69 u32 op; 70 70 - int ret; 70 70 + int rc; 71 71 72 72 - ret = kstrtou32(page, 0, &op); 73 73 - if (ret) 74 74 - return ret; 72 72 + rc = kstrtou32(page, 0, &op); 73 73 + if (rc) 74 74 + return rc; 75 75 if ((op != 1) && (op != 0)) { 76 76 pr_err("Illegal value for tpg_enable: %u\n", op); 77 77 return -EINVAL; ··· 89 87 return -EINVAL; 90 88 91 89 if (op) { 92 92 - /* 93 93 - * Use existing np->np_sockaddr for SCTP network portal reference 94 94 - */ 95 95 - tpg_np_sctp = iscsit_tpg_add_network_portal(tpg, &np->np_sockaddr, 96 96 - tpg_np, ISCSI_SCTP_TCP); 97 97 - if (!tpg_np_sctp || IS_ERR(tpg_np_sctp)) 98 98 - goto out; 99 99 - } else { 100 100 - tpg_np_sctp = iscsit_tpg_locate_child_np(tpg_np, ISCSI_SCTP_TCP); 101 101 - if (!tpg_np_sctp) 102 102 - goto out; 103 103 - 104 104 - ret = iscsit_tpg_del_network_portal(tpg, tpg_np_sctp); 105 105 - if (ret < 0) 106 106 - goto out; 107 107 - } 108 108 - 109 109 - iscsit_put_tpg(tpg); 110 110 - return count; 111 111 - out: 112 112 - iscsit_put_tpg(tpg); 113 113 - return -EINVAL; 114 114 - } 115 115 - 116 116 - static ssize_t lio_target_np_iser_show(struct config_item *item, char *page) 117 117 - { 118 118 - struct iscsi_tpg_np *tpg_np = to_iscsi_tpg_np(item); 119 119 - struct iscsi_tpg_np *tpg_np_iser; 120 120 - ssize_t rb; 121 121 - 122 122 - tpg_np_iser = iscsit_tpg_locate_child_np(tpg_np, ISCSI_INFINIBAND); 123 123 - if (tpg_np_iser) 124 124 - rb = sprintf(page, "1\n"); 125 125 - else 126 126 - rb = sprintf(page, "0\n"); 127 127 - 128 128 - return rb; 129 129 - } 130 130 - 131 131 - static ssize_t lio_target_np_iser_store(struct config_item *item, 132 132 - const char *page, size_t count) 133 133 - { 134 134 - struct iscsi_tpg_np *tpg_np = to_iscsi_tpg_np(item); 135 135 - struct iscsi_np *np; 136 136 - struct iscsi_portal_group *tpg; 137 137 - struct iscsi_tpg_np *tpg_np_iser = NULL; 138 138 - char *endptr; 139 139 - u32 op; 140 140 - int rc = 0; 141 141 - 142 142 - op = simple_strtoul(page, &endptr, 0); 143 143 - if ((op != 1) && (op != 0)) { 144 144 - pr_err("Illegal value for tpg_enable: %u\n", op); 145 145 - return -EINVAL; 146 146 - } 147 147 - np = tpg_np->tpg_np; 148 148 - if (!np) { 149 149 - pr_err("Unable to locate struct iscsi_np from" 150 150 - " struct iscsi_tpg_np\n"); 151 151 - return -EINVAL; 152 152 - } 153 153 - 154 154 - tpg = tpg_np->tpg; 155 155 - if (iscsit_get_tpg(tpg) < 0) 156 156 - return -EINVAL; 157 157 - 158 158 - if (op) { 159 159 - rc = request_module("ib_isert"); 160 160 - if (rc != 0) { 161 161 - pr_warn("Unable to request_module for ib_isert\n"); 162 162 - rc = 0; 90 90 + if (strlen(mod_name)) { 91 91 + rc = request_module(mod_name); 92 92 + if (rc != 0) { 93 93 + pr_warn("Unable to request_module for %s\n", 94 94 + mod_name); 95 95 + rc = 0; 96 96 + } 163 97 } 164 98 165 165 - tpg_np_iser = iscsit_tpg_add_network_portal(tpg, &np->np_sockaddr, 166 166 - tpg_np, ISCSI_INFINIBAND); 167 167 - if (IS_ERR(tpg_np_iser)) { 168 168 - rc = PTR_ERR(tpg_np_iser); 99 99 + tpg_np_new = iscsit_tpg_add_network_portal(tpg, 100 100 + &np->np_sockaddr, tpg_np, type); 101 101 + if (IS_ERR(tpg_np_new)) 169 102 goto out; 170 170 - } 171 103 } else { 172 172 - tpg_np_iser = iscsit_tpg_locate_child_np(tpg_np, ISCSI_INFINIBAND); 173 173 - if (tpg_np_iser) { 174 174 - rc = iscsit_tpg_del_network_portal(tpg, tpg_np_iser); 104 104 + tpg_np_new = iscsit_tpg_locate_child_np(tpg_np, type); 105 105 + if (tpg_np_new) { 106 106 + rc = iscsit_tpg_del_network_portal(tpg, tpg_np_new); 175 107 if (rc < 0) 176 108 goto out; 177 109 } ··· 118 182 return rc; 119 183 } 120 184 121 121 - CONFIGFS_ATTR(lio_target_np_, sctp); 185 185 + static ssize_t lio_target_np_iser_show(struct config_item *item, char *page) 186 186 + { 187 187 + return lio_target_np_driver_show(item, page, ISCSI_INFINIBAND); 188 188 + } 189 189 + 190 190 + static ssize_t lio_target_np_iser_store(struct config_item *item, 191 191 + const char *page, size_t count) 192 192 + { 193 193 + return lio_target_np_driver_store(item, page, count, 194 194 + ISCSI_INFINIBAND, "ib_isert"); 195 195 + } 122 196 CONFIGFS_ATTR(lio_target_np_, iser); 123 197 198 198 + static ssize_t lio_target_np_cxgbit_show(struct config_item *item, char *page) 199 199 + { 200 200 + return lio_target_np_driver_show(item, page, ISCSI_CXGBIT); 201 201 + } 202 202 + 203 203 + static ssize_t lio_target_np_cxgbit_store(struct config_item *item, 204 204 + const char *page, size_t count) 205 205 + { 206 206 + return lio_target_np_driver_store(item, page, count, 207 207 + ISCSI_CXGBIT, "cxgbit"); 208 208 + } 209 209 + CONFIGFS_ATTR(lio_target_np_, cxgbit); 210 210 + 124 211 static struct configfs_attribute *lio_target_portal_attrs[] = { 125 125 - &lio_target_np_attr_sctp, 126 212 &lio_target_np_attr_iser, 213 213 + &lio_target_np_attr_cxgbit, 127 214 NULL, 128 215 }; 129 216 ··· 1513 1554 * This function calls iscsit_inc_session_usage_count() on the 1514 1555 * struct iscsi_session in question. 1515 1556 */ 1516 1516 - static int lio_tpg_shutdown_session(struct se_session *se_sess) 1557 1557 + static void lio_tpg_close_session(struct se_session *se_sess) 1517 1558 { 1518 1559 struct iscsi_session *sess = se_sess->fabric_sess_ptr; 1519 1560 struct se_portal_group *se_tpg = &sess->tpg->tpg_se_tpg; ··· 1525 1566 (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) { 1526 1567 spin_unlock(&sess->conn_lock); 1527 1568 spin_unlock_bh(&se_tpg->session_lock); 1528 1528 - return 0; 1569 1569 + return; 1529 1570 } 1530 1571 atomic_set(&sess->session_reinstatement, 1); 1531 1572 spin_unlock(&sess->conn_lock); ··· 1534 1575 spin_unlock_bh(&se_tpg->session_lock); 1535 1576 1536 1577 iscsit_stop_session(sess, 1, 1); 1537 1537 - return 1; 1538 1538 - } 1539 1539 - 1540 1540 - /* 1541 1541 - * Calls iscsit_dec_session_usage_count() as inverse of 1542 1542 - * lio_tpg_shutdown_session() 1543 1543 - */ 1544 1544 - static void lio_tpg_close_session(struct se_session *se_sess) 1545 1545 - { 1546 1546 - struct iscsi_session *sess = se_sess->fabric_sess_ptr; 1547 1547 - /* 1548 1548 - * If the iSCSI Session for the iSCSI Initiator Node exists, 1549 1549 - * forcefully shutdown the iSCSI NEXUS. 1550 1550 - */ 1551 1578 iscsit_close_session(sess); 1552 1579 } 1553 1580 ··· 1585 1640 .tpg_get_inst_index = lio_tpg_get_inst_index, 1586 1641 .check_stop_free = lio_check_stop_free, 1587 1642 .release_cmd = lio_release_cmd, 1588 1588 - .shutdown_session = lio_tpg_shutdown_session, 1589 1643 .close_session = lio_tpg_close_session, 1590 1644 .sess_get_index = lio_sess_get_index, 1591 1645 .sess_get_initiator_sid = lio_sess_get_initiator_sid,

+1

drivers/target/iscsi/iscsi_target_datain_values.c

reviewed

··· 524 524 525 525 return NULL; 526 526 } 527 527 + EXPORT_SYMBOL(iscsit_get_datain_values);

+1 -1

drivers/target/iscsi/iscsi_target_erl0.c

reviewed

··· 786 786 } 787 787 788 788 spin_unlock_bh(&se_tpg->session_lock); 789 789 - target_put_session(sess->se_sess); 789 789 + iscsit_close_session(sess); 790 790 } 791 791 792 792 void iscsit_start_time2retain_handler(struct iscsi_session *sess)

+14 -3

drivers/target/iscsi/iscsi_target_login.c

reviewed

··· 228 228 if (sess->session_state == TARG_SESS_STATE_FAILED) { 229 229 spin_unlock_bh(&sess->conn_lock); 230 230 iscsit_dec_session_usage_count(sess); 231 231 - target_put_session(sess->se_sess); 231 231 + iscsit_close_session(sess); 232 232 return 0; 233 233 } 234 234 spin_unlock_bh(&sess->conn_lock); ··· 236 236 iscsit_stop_session(sess, 1, 1); 237 237 iscsit_dec_session_usage_count(sess); 238 238 239 239 - target_put_session(sess->se_sess); 239 239 + iscsit_close_session(sess); 240 240 return 0; 241 241 } 242 242 ··· 258 258 mutex_unlock(&auth_id_lock); 259 259 } 260 260 261 261 - static __printf(2, 3) int iscsi_change_param_sprintf( 261 261 + __printf(2, 3) int iscsi_change_param_sprintf( 262 262 struct iscsi_conn *conn, 263 263 const char *fmt, ...) 264 264 { ··· 279 279 280 280 return 0; 281 281 } 282 282 + EXPORT_SYMBOL(iscsi_change_param_sprintf); 282 283 283 284 /* 284 285 * This is the leading connection of a new session, ··· 1386 1385 } else { 1387 1386 if (iscsi_login_non_zero_tsih_s2(conn, buffer) < 0) 1388 1387 goto old_sess_out; 1388 1388 + } 1389 1389 + 1390 1390 + if (conn->conn_transport->iscsit_validate_params) { 1391 1391 + ret = conn->conn_transport->iscsit_validate_params(conn); 1392 1392 + if (ret < 0) { 1393 1393 + if (zero_tsih) 1394 1394 + goto new_sess_out; 1395 1395 + else 1396 1396 + goto old_sess_out; 1397 1397 + } 1389 1398 } 1390 1399 1391 1400 ret = iscsi_target_start_negotiation(login, conn);

+10 -9

drivers/target/iscsi/iscsi_target_nego.c

reviewed

··· 269 269 270 270 return 0; 271 271 } 272 272 + EXPORT_SYMBOL(iscsi_target_check_login_request); 272 273 273 274 static int iscsi_target_check_first_request( 274 275 struct iscsi_conn *conn, ··· 1247 1246 { 1248 1247 int ret; 1249 1248 1250 1250 - ret = iscsi_target_do_login(conn, login); 1251 1251 - if (!ret) { 1252 1252 - if (conn->sock) { 1253 1253 - struct sock *sk = conn->sock->sk; 1249 1249 + if (conn->sock) { 1250 1250 + struct sock *sk = conn->sock->sk; 1254 1251 1255 1255 - write_lock_bh(&sk->sk_callback_lock); 1256 1256 - set_bit(LOGIN_FLAGS_READY, &conn->login_flags); 1257 1257 - write_unlock_bh(&sk->sk_callback_lock); 1258 1258 - } 1259 1259 - } else if (ret < 0) { 1252 1252 + write_lock_bh(&sk->sk_callback_lock); 1253 1253 + set_bit(LOGIN_FLAGS_READY, &conn->login_flags); 1254 1254 + write_unlock_bh(&sk->sk_callback_lock); 1255 1255 + } 1256 1256 + 1257 1257 + ret = iscsi_target_do_login(conn, login); 1258 1258 + if (ret < 0) { 1260 1259 cancel_delayed_work_sync(&conn->login_work); 1261 1260 cancel_delayed_work_sync(&conn->login_cleanup_work); 1262 1261 iscsi_target_restore_sock_callbacks(conn);

+1

drivers/target/iscsi/iscsi_target_parameters.c

reviewed

··· 680 680 pr_err("Unable to locate key \"%s\".\n", key); 681 681 return NULL; 682 682 } 683 683 + EXPORT_SYMBOL(iscsi_find_param_from_key); 683 684 684 685 int iscsi_extract_key_value(char *textbuf, char **key, char **value) 685 686 {

+5

drivers/target/iscsi/iscsi_target_util.c

reviewed

··· 514 514 515 515 wake_up(&conn->queues_wq); 516 516 } 517 517 + EXPORT_SYMBOL(iscsit_add_cmd_to_immediate_queue); 517 518 518 519 struct iscsi_queue_req *iscsit_get_cmd_from_immediate_queue(struct iscsi_conn *conn) 519 520 { ··· 726 725 iscsit_remove_cmd_from_immediate_queue(cmd, conn); 727 726 iscsit_remove_cmd_from_response_queue(cmd, conn); 728 727 } 728 728 + 729 729 + if (conn && conn->conn_transport->iscsit_release_cmd) 730 730 + conn->conn_transport->iscsit_release_cmd(conn, cmd); 729 731 } 730 732 731 733 void iscsit_free_cmd(struct iscsi_cmd *cmd, bool shutdown) ··· 777 773 break; 778 774 } 779 775 } 776 776 + EXPORT_SYMBOL(iscsit_free_cmd); 780 777 781 778 int iscsit_check_session_usage_count(struct iscsi_session *sess) 782 779 {

-12

drivers/target/loopback/tcm_loop.c

reviewed

··· 601 601 return tl_cmd->sc_cmd_state; 602 602 } 603 603 604 604 - static int tcm_loop_shutdown_session(struct se_session *se_sess) 605 605 - { 606 606 - return 0; 607 607 - } 608 608 - 609 609 - static void tcm_loop_close_session(struct se_session *se_sess) 610 610 - { 611 611 - return; 612 612 - }; 613 613 - 614 604 static int tcm_loop_write_pending(struct se_cmd *se_cmd) 615 605 { 616 606 /* ··· 1233 1243 .tpg_get_inst_index = tcm_loop_get_inst_index, 1234 1244 .check_stop_free = tcm_loop_check_stop_free, 1235 1245 .release_cmd = tcm_loop_release_cmd, 1236 1236 - .shutdown_session = tcm_loop_shutdown_session, 1237 1237 - .close_session = tcm_loop_close_session, 1238 1246 .sess_get_index = tcm_loop_sess_get_index, 1239 1247 .write_pending = tcm_loop_write_pending, 1240 1248 .write_pending_status = tcm_loop_write_pending_status,

-12

drivers/target/sbp/sbp_target.c

reviewed

··· 1726 1726 sbp_free_request(req); 1727 1727 } 1728 1728 1729 1729 - static int sbp_shutdown_session(struct se_session *se_sess) 1730 1730 - { 1731 1731 - return 0; 1732 1732 - } 1733 1733 - 1734 1734 - static void sbp_close_session(struct se_session *se_sess) 1735 1735 - { 1736 1736 - return; 1737 1737 - } 1738 1738 - 1739 1729 static u32 sbp_sess_get_index(struct se_session *se_sess) 1740 1730 { 1741 1731 return 0; ··· 2339 2349 .tpg_check_prod_mode_write_protect = sbp_check_false, 2340 2350 .tpg_get_inst_index = sbp_tpg_get_inst_index, 2341 2351 .release_cmd = sbp_release_cmd, 2342 2342 - .shutdown_session = sbp_shutdown_session, 2343 2343 - .close_session = sbp_close_session, 2344 2352 .sess_get_index = sbp_sess_get_index, 2345 2353 .write_pending = sbp_write_pending, 2346 2354 .write_pending_status = sbp_write_pending_status,

+3 -3

drivers/target/target_core_alua.c

reviewed

··· 932 932 tg_pt_gp->tg_pt_gp_alua_access_status); 933 933 934 934 snprintf(path, ALUA_METADATA_PATH_LEN, 935 935 - "/var/target/alua/tpgs_%s/%s", &wwn->unit_serial[0], 935 935 + "%s/alua/tpgs_%s/%s", db_root, &wwn->unit_serial[0], 936 936 config_item_name(&tg_pt_gp->tg_pt_gp_group.cg_item)); 937 937 938 938 rc = core_alua_write_tpg_metadata(path, md_buf, len); ··· 1275 1275 atomic_read(&lun->lun_tg_pt_secondary_offline), 1276 1276 lun->lun_tg_pt_secondary_stat); 1277 1277 1278 1278 - snprintf(path, ALUA_METADATA_PATH_LEN, "/var/target/alua/%s/%s/lun_%llu", 1279 1279 - se_tpg->se_tpg_tfo->get_fabric_name(), wwn, 1278 1278 + snprintf(path, ALUA_METADATA_PATH_LEN, "%s/alua/%s/%s/lun_%llu", 1279 1279 + db_root, se_tpg->se_tpg_tfo->get_fabric_name(), wwn, 1280 1280 lun->unpacked_lun); 1281 1281 1282 1282 rc = core_alua_write_tpg_metadata(path, md_buf, len);

+62 -8

drivers/target/target_core_configfs.c

reviewed

··· 99 99 100 100 CONFIGFS_ATTR_RO(target_core_item_, version); 101 101 102 102 + char db_root[DB_ROOT_LEN] = DB_ROOT_DEFAULT; 103 103 + static char db_root_stage[DB_ROOT_LEN]; 104 104 + 105 105 + static ssize_t target_core_item_dbroot_show(struct config_item *item, 106 106 + char *page) 107 107 + { 108 108 + return sprintf(page, "%s\n", db_root); 109 109 + } 110 110 + 111 111 + static ssize_t target_core_item_dbroot_store(struct config_item *item, 112 112 + const char *page, size_t count) 113 113 + { 114 114 + ssize_t read_bytes; 115 115 + struct file *fp; 116 116 + 117 117 + mutex_lock(&g_tf_lock); 118 118 + if (!list_empty(&g_tf_list)) { 119 119 + mutex_unlock(&g_tf_lock); 120 120 + pr_err("db_root: cannot be changed: target drivers registered"); 121 121 + return -EINVAL; 122 122 + } 123 123 + 124 124 + if (count > (DB_ROOT_LEN - 1)) { 125 125 + mutex_unlock(&g_tf_lock); 126 126 + pr_err("db_root: count %d exceeds DB_ROOT_LEN-1: %u\n", 127 127 + (int)count, DB_ROOT_LEN - 1); 128 128 + return -EINVAL; 129 129 + } 130 130 + 131 131 + read_bytes = snprintf(db_root_stage, DB_ROOT_LEN, "%s", page); 132 132 + if (!read_bytes) { 133 133 + mutex_unlock(&g_tf_lock); 134 134 + return -EINVAL; 135 135 + } 136 136 + if (db_root_stage[read_bytes - 1] == '\n') 137 137 + db_root_stage[read_bytes - 1] = '\0'; 138 138 + 139 139 + /* validate new db root before accepting it */ 140 140 + fp = filp_open(db_root_stage, O_RDONLY, 0); 141 141 + if (IS_ERR(fp)) { 142 142 + mutex_unlock(&g_tf_lock); 143 143 + pr_err("db_root: cannot open: %s\n", db_root_stage); 144 144 + return -EINVAL; 145 145 + } 146 146 + if (!S_ISDIR(fp->f_inode->i_mode)) { 147 147 + filp_close(fp, 0); 148 148 + mutex_unlock(&g_tf_lock); 149 149 + pr_err("db_root: not a directory: %s\n", db_root_stage); 150 150 + return -EINVAL; 151 151 + } 152 152 + filp_close(fp, 0); 153 153 + 154 154 + strncpy(db_root, db_root_stage, read_bytes); 155 155 + 156 156 + mutex_unlock(&g_tf_lock); 157 157 + 158 158 + return read_bytes; 159 159 + } 160 160 + 161 161 + CONFIGFS_ATTR(target_core_item_, dbroot); 162 162 + 102 163 static struct target_fabric_configfs *target_core_get_fabric( 103 164 const char *name) 104 165 { ··· 300 239 */ 301 240 static struct configfs_attribute *target_core_fabric_item_attrs[] = { 302 241 &target_core_item_attr_version, 242 242 + &target_core_item_attr_dbroot, 303 243 NULL, 304 244 }; 305 245 ··· 383 321 } 384 322 if (!tfo->release_cmd) { 385 323 pr_err("Missing tfo->release_cmd()\n"); 386 386 - return -EINVAL; 387 387 - } 388 388 - if (!tfo->shutdown_session) { 389 389 - pr_err("Missing tfo->shutdown_session()\n"); 390 390 - return -EINVAL; 391 391 - } 392 392 - if (!tfo->close_session) { 393 393 - pr_err("Missing tfo->close_session()\n"); 394 324 return -EINVAL; 395 325 } 396 326 if (!tfo->sess_get_index) {

+6

drivers/target/target_core_internal.h

reviewed

··· 155 155 /* target_core_xcopy.c */ 156 156 extern struct se_portal_group xcopy_pt_tpg; 157 157 158 158 + /* target_core_configfs.c */ 159 159 + #define DB_ROOT_LEN 4096 160 160 + #define DB_ROOT_DEFAULT "/var/target" 161 161 + 162 162 + extern char db_root[]; 163 163 + 158 164 #endif /* TARGET_CORE_INTERNAL_H */

+1 -1

drivers/target/target_core_pr.c

reviewed

··· 1985 1985 return -EMSGSIZE; 1986 1986 } 1987 1987 1988 1988 - snprintf(path, 512, "/var/target/pr/aptpl_%s", &wwn->unit_serial[0]); 1988 1988 + snprintf(path, 512, "%s/pr/aptpl_%s", db_root, &wwn->unit_serial[0]); 1989 1989 file = filp_open(path, flags, 0600); 1990 1990 if (IS_ERR(file)) { 1991 1991 pr_err("filp_open(%s) for APTPL metadata"

-4

drivers/target/target_core_rd.c

reviewed

··· 403 403 struct se_device *se_dev = cmd->se_dev; 404 404 struct rd_dev *dev = RD_DEV(se_dev); 405 405 struct rd_dev_sg_table *prot_table; 406 406 - bool need_to_release = false; 407 406 struct scatterlist *prot_sg; 408 407 u32 sectors = cmd->data_length / se_dev->dev_attrib.block_size; 409 408 u32 prot_offset, prot_page; ··· 430 431 431 432 if (!rc) 432 433 sbc_dif_copy_prot(cmd, sectors, is_read, prot_sg, prot_offset); 433 433 - 434 434 - if (need_to_release) 435 435 - kfree(prot_sg); 436 434 437 435 return rc; 438 436 }

+27 -56

drivers/target/target_core_tpg.c

reviewed

··· 336 336 return acl; 337 337 } 338 338 339 339 + static void target_shutdown_sessions(struct se_node_acl *acl) 340 340 + { 341 341 + struct se_session *sess; 342 342 + unsigned long flags; 343 343 + 344 344 + restart: 345 345 + spin_lock_irqsave(&acl->nacl_sess_lock, flags); 346 346 + list_for_each_entry(sess, &acl->acl_sess_list, sess_acl_list) { 347 347 + if (sess->sess_tearing_down) 348 348 + continue; 349 349 + 350 350 + list_del_init(&sess->sess_acl_list); 351 351 + spin_unlock_irqrestore(&acl->nacl_sess_lock, flags); 352 352 + 353 353 + if (acl->se_tpg->se_tpg_tfo->close_session) 354 354 + acl->se_tpg->se_tpg_tfo->close_session(sess); 355 355 + goto restart; 356 356 + } 357 357 + spin_unlock_irqrestore(&acl->nacl_sess_lock, flags); 358 358 + } 359 359 + 339 360 void core_tpg_del_initiator_node_acl(struct se_node_acl *acl) 340 361 { 341 362 struct se_portal_group *tpg = acl->se_tpg; 342 342 - LIST_HEAD(sess_list); 343 343 - struct se_session *sess, *sess_tmp; 344 344 - unsigned long flags; 345 345 - int rc; 346 363 347 364 mutex_lock(&tpg->acl_node_mutex); 348 348 - if (acl->dynamic_node_acl) { 365 365 + if (acl->dynamic_node_acl) 349 366 acl->dynamic_node_acl = 0; 350 350 - } 351 367 list_del(&acl->acl_list); 352 368 mutex_unlock(&tpg->acl_node_mutex); 353 369 354 354 - spin_lock_irqsave(&acl->nacl_sess_lock, flags); 355 355 - acl->acl_stop = 1; 370 370 + target_shutdown_sessions(acl); 356 371 357 357 - list_for_each_entry_safe(sess, sess_tmp, &acl->acl_sess_list, 358 358 - sess_acl_list) { 359 359 - if (sess->sess_tearing_down != 0) 360 360 - continue; 361 361 - 362 362 - if (!target_get_session(sess)) 363 363 - continue; 364 364 - list_move(&sess->sess_acl_list, &sess_list); 365 365 - } 366 366 - spin_unlock_irqrestore(&acl->nacl_sess_lock, flags); 367 367 - 368 368 - list_for_each_entry_safe(sess, sess_tmp, &sess_list, sess_acl_list) { 369 369 - list_del(&sess->sess_acl_list); 370 370 - 371 371 - rc = tpg->se_tpg_tfo->shutdown_session(sess); 372 372 - target_put_session(sess); 373 373 - if (!rc) 374 374 - continue; 375 375 - target_put_session(sess); 376 376 - } 377 372 target_put_nacl(acl); 378 373 /* 379 374 * Wait for last target_put_nacl() to complete in target_complete_nacl() ··· 395 400 struct se_node_acl *acl, 396 401 u32 queue_depth) 397 402 { 398 398 - LIST_HEAD(sess_list); 399 403 struct se_portal_group *tpg = acl->se_tpg; 400 400 - struct se_session *sess, *sess_tmp; 401 401 - unsigned long flags; 402 402 - int rc; 403 404 404 405 /* 405 406 * User has requested to change the queue depth for a Initiator Node. ··· 404 413 */ 405 414 target_set_nacl_queue_depth(tpg, acl, queue_depth); 406 415 407 407 - spin_lock_irqsave(&acl->nacl_sess_lock, flags); 408 408 - list_for_each_entry_safe(sess, sess_tmp, &acl->acl_sess_list, 409 409 - sess_acl_list) { 410 410 - if (sess->sess_tearing_down != 0) 411 411 - continue; 412 412 - if (!target_get_session(sess)) 413 413 - continue; 414 414 - spin_unlock_irqrestore(&acl->nacl_sess_lock, flags); 415 415 - 416 416 - /* 417 417 - * Finally call tpg->se_tpg_tfo->close_session() to force session 418 418 - * reinstatement to occur if there is an active session for the 419 419 - * $FABRIC_MOD Initiator Node in question. 420 420 - */ 421 421 - rc = tpg->se_tpg_tfo->shutdown_session(sess); 422 422 - target_put_session(sess); 423 423 - if (!rc) { 424 424 - spin_lock_irqsave(&acl->nacl_sess_lock, flags); 425 425 - continue; 426 426 - } 427 427 - target_put_session(sess); 428 428 - spin_lock_irqsave(&acl->nacl_sess_lock, flags); 429 429 - } 430 430 - spin_unlock_irqrestore(&acl->nacl_sess_lock, flags); 416 416 + /* 417 417 + * Shutdown all pending sessions to force session reinstatement. 418 418 + */ 419 419 + target_shutdown_sessions(acl); 431 420 432 421 pr_debug("Successfully changed queue depth to: %d for Initiator" 433 422 " Node: %s on %s Target Portal Group: %u\n", acl->queue_depth,

+2 -24

drivers/target/target_core_transport.c

reviewed

··· 239 239 INIT_LIST_HEAD(&se_sess->sess_cmd_list); 240 240 INIT_LIST_HEAD(&se_sess->sess_wait_list); 241 241 spin_lock_init(&se_sess->sess_cmd_lock); 242 242 - kref_init(&se_sess->sess_kref); 243 242 se_sess->sup_prot_ops = sup_prot_ops; 244 243 245 244 return se_sess; ··· 429 430 } 430 431 EXPORT_SYMBOL(target_alloc_session); 431 432 432 432 - static void target_release_session(struct kref *kref) 433 433 - { 434 434 - struct se_session *se_sess = container_of(kref, 435 435 - struct se_session, sess_kref); 436 436 - struct se_portal_group *se_tpg = se_sess->se_tpg; 437 437 - 438 438 - se_tpg->se_tpg_tfo->close_session(se_sess); 439 439 - } 440 440 - 441 441 - int target_get_session(struct se_session *se_sess) 442 442 - { 443 443 - return kref_get_unless_zero(&se_sess->sess_kref); 444 444 - } 445 445 - EXPORT_SYMBOL(target_get_session); 446 446 - 447 447 - void target_put_session(struct se_session *se_sess) 448 448 - { 449 449 - kref_put(&se_sess->sess_kref, target_release_session); 450 450 - } 451 451 - EXPORT_SYMBOL(target_put_session); 452 452 - 453 433 ssize_t target_show_dynamic_sessions(struct se_portal_group *se_tpg, char *page) 454 434 { 455 435 struct se_session *se_sess; ··· 477 499 se_nacl = se_sess->se_node_acl; 478 500 if (se_nacl) { 479 501 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags); 480 480 - if (se_nacl->acl_stop == 0) 481 481 - list_del(&se_sess->sess_acl_list); 502 502 + if (!list_empty(&se_sess->sess_acl_list)) 503 503 + list_del_init(&se_sess->sess_acl_list); 482 504 /* 483 505 * If the session list is empty, then clear the pointer. 484 506 * Otherwise, set the struct se_session pointer from the tail

-1

drivers/target/tcm_fc/tcm_fc.h

reviewed

··· 139 139 * Session ops. 140 140 */ 141 141 void ft_sess_put(struct ft_sess *); 142 142 - int ft_sess_shutdown(struct se_session *); 143 142 void ft_sess_close(struct se_session *); 144 143 u32 ft_sess_get_index(struct se_session *); 145 144 u32 ft_sess_get_port_name(struct se_session *, unsigned char *, u32);

-1

drivers/target/tcm_fc/tfc_conf.c

reviewed

··· 442 442 .tpg_get_inst_index = ft_tpg_get_inst_index, 443 443 .check_stop_free = ft_check_stop_free, 444 444 .release_cmd = ft_release_cmd, 445 445 - .shutdown_session = ft_sess_shutdown, 446 445 .close_session = ft_sess_close, 447 446 .sess_get_index = ft_sess_get_index, 448 447 .sess_get_initiator_sid = NULL,

-12

drivers/target/tcm_fc/tfc_sess.c

reviewed

··· 303 303 */ 304 304 305 305 /* 306 306 - * Determine whether session is allowed to be shutdown in the current context. 307 307 - * Returns non-zero if the session should be shutdown. 308 308 - */ 309 309 - int ft_sess_shutdown(struct se_session *se_sess) 310 310 - { 311 311 - struct ft_sess *sess = se_sess->fabric_sess_ptr; 312 312 - 313 313 - pr_debug("port_id %x\n", sess->port_id); 314 314 - return 1; 315 315 - } 316 316 - 317 317 - /* 318 306 * Remove session and send PRLO. 319 307 * This is called when the ACL is being deleted or queue depth is changing. 320 308 */

-11

drivers/usb/gadget/function/f_tcm.c

reviewed

··· 1290 1290 percpu_ida_free(&se_sess->sess_tag_pool, se_cmd->map_tag); 1291 1291 } 1292 1292 1293 1293 - static int usbg_shutdown_session(struct se_session *se_sess) 1294 1294 - { 1295 1295 - return 0; 1296 1296 - } 1297 1297 - 1298 1298 - static void usbg_close_session(struct se_session *se_sess) 1299 1299 - { 1300 1300 - } 1301 1301 - 1302 1293 static u32 usbg_sess_get_index(struct se_session *se_sess) 1303 1294 { 1304 1295 return 0; ··· 1726 1735 .tpg_check_prod_mode_write_protect = usbg_check_false, 1727 1736 .tpg_get_inst_index = usbg_tpg_get_inst_index, 1728 1737 .release_cmd = usbg_release_cmd, 1729 1729 - .shutdown_session = usbg_shutdown_session, 1730 1730 - .close_session = usbg_close_session, 1731 1738 .sess_get_index = usbg_sess_get_index, 1732 1739 .sess_get_initiator_sid = NULL, 1733 1740 .write_pending = usbg_send_write_request,

-12

drivers/vhost/scsi.c

reviewed

··· 333 333 percpu_ida_free(&se_sess->sess_tag_pool, se_cmd->map_tag); 334 334 } 335 335 336 336 - static int vhost_scsi_shutdown_session(struct se_session *se_sess) 337 337 - { 338 338 - return 0; 339 339 - } 340 340 - 341 341 - static void vhost_scsi_close_session(struct se_session *se_sess) 342 342 - { 343 343 - return; 344 344 - } 345 345 - 346 336 static u32 vhost_scsi_sess_get_index(struct se_session *se_sess) 347 337 { 348 338 return 0; ··· 2104 2114 .tpg_get_inst_index = vhost_scsi_tpg_get_inst_index, 2105 2115 .release_cmd = vhost_scsi_release_cmd, 2106 2116 .check_stop_free = vhost_scsi_check_stop_free, 2107 2107 - .shutdown_session = vhost_scsi_shutdown_session, 2108 2108 - .close_session = vhost_scsi_close_session, 2109 2117 .sess_get_index = vhost_scsi_sess_get_index, 2110 2118 .sess_get_initiator_sid = NULL, 2111 2119 .write_pending = vhost_scsi_write_pending,

-11

drivers/xen/xen-scsiback.c

reviewed

··· 1399 1399 percpu_ida_free(&se_sess->sess_tag_pool, se_cmd->map_tag); 1400 1400 } 1401 1401 1402 1402 - static int scsiback_shutdown_session(struct se_session *se_sess) 1403 1403 - { 1404 1404 - return 0; 1405 1405 - } 1406 1406 - 1407 1407 - static void scsiback_close_session(struct se_session *se_sess) 1408 1408 - { 1409 1409 - } 1410 1410 - 1411 1402 static u32 scsiback_sess_get_index(struct se_session *se_sess) 1412 1403 { 1413 1404 return 0; ··· 1832 1841 .tpg_get_inst_index = scsiback_tpg_get_inst_index, 1833 1842 .check_stop_free = scsiback_check_stop_free, 1834 1843 .release_cmd = scsiback_release_cmd, 1835 1835 - .shutdown_session = scsiback_shutdown_session, 1836 1836 - .close_session = scsiback_close_session, 1837 1844 .sess_get_index = scsiback_sess_get_index, 1838 1845 .sess_get_initiator_sid = NULL, 1839 1846 .write_pending = scsiback_write_pending,

+27

include/target/iscsi/iscsi_target_core.h

reviewed

··· 74 74 ISCSI_IWARP_TCP = 3, 75 75 ISCSI_IWARP_SCTP = 4, 76 76 ISCSI_INFINIBAND = 5, 77 77 + ISCSI_CXGBIT = 6, 77 78 }; 78 79 79 80 /* RFC-3720 7.1.4 Standard Connection State Diagram for a Target */ ··· 891 890 } 892 891 893 892 extern struct iscsi_cmd *iscsit_find_cmd_from_itt(struct iscsi_conn *, itt_t); 893 893 + 894 894 + static inline void iscsit_thread_check_cpumask( 895 895 + struct iscsi_conn *conn, 896 896 + struct task_struct *p, 897 897 + int mode) 898 898 + { 899 899 + /* 900 900 + * mode == 1 signals iscsi_target_tx_thread() usage. 901 901 + * mode == 0 signals iscsi_target_rx_thread() usage. 902 902 + */ 903 903 + if (mode == 1) { 904 904 + if (!conn->conn_tx_reset_cpumask) 905 905 + return; 906 906 + conn->conn_tx_reset_cpumask = 0; 907 907 + } else { 908 908 + if (!conn->conn_rx_reset_cpumask) 909 909 + return; 910 910 + conn->conn_rx_reset_cpumask = 0; 911 911 + } 912 912 + /* 913 913 + * Update the CPU mask for this single kthread so that 914 914 + * both TX and RX kthreads are scheduled to run on the 915 915 + * same CPU. 916 916 + */ 917 917 + set_cpus_allowed_ptr(p, conn->conn_cpumask); 918 918 + } 894 919 #endif /* ISCSI_TARGET_CORE_H */

+41

include/target/iscsi/iscsi_transport.h

reviewed

··· 6 6 #define ISCSIT_TRANSPORT_NAME 16 7 7 char name[ISCSIT_TRANSPORT_NAME]; 8 8 int transport_type; 9 9 + bool rdma_shutdown; 9 10 int priv_size; 10 11 struct module *owner; 11 12 struct list_head t_node; ··· 23 22 int (*iscsit_queue_data_in)(struct iscsi_conn *, struct iscsi_cmd *); 24 23 int (*iscsit_queue_status)(struct iscsi_conn *, struct iscsi_cmd *); 25 24 void (*iscsit_aborted_task)(struct iscsi_conn *, struct iscsi_cmd *); 25 25 + int (*iscsit_xmit_pdu)(struct iscsi_conn *, struct iscsi_cmd *, 26 26 + struct iscsi_datain_req *, const void *, u32); 27 27 + void (*iscsit_release_cmd)(struct iscsi_conn *, struct iscsi_cmd *); 28 28 + void (*iscsit_get_rx_pdu)(struct iscsi_conn *); 29 29 + int (*iscsit_validate_params)(struct iscsi_conn *); 30 30 + void (*iscsit_get_r2t_ttt)(struct iscsi_conn *, struct iscsi_cmd *, 31 31 + struct iscsi_r2t *); 26 32 enum target_prot_op (*iscsit_get_sup_prot_ops)(struct iscsi_conn *); 27 33 }; 28 34 ··· 85 77 extern int iscsit_build_logout_rsp(struct iscsi_cmd *, struct iscsi_conn *, 86 78 struct iscsi_logout_rsp *); 87 79 extern int iscsit_logout_post_handler(struct iscsi_cmd *, struct iscsi_conn *); 80 80 + extern int iscsit_queue_rsp(struct iscsi_conn *, struct iscsi_cmd *); 81 81 + extern void iscsit_aborted_task(struct iscsi_conn *, struct iscsi_cmd *); 82 82 + extern int iscsit_add_reject(struct iscsi_conn *, u8, unsigned char *); 83 83 + extern int iscsit_reject_cmd(struct iscsi_cmd *, u8, unsigned char *); 84 84 + extern int iscsit_handle_snack(struct iscsi_conn *, unsigned char *); 85 85 + extern void iscsit_build_datain_pdu(struct iscsi_cmd *, struct iscsi_conn *, 86 86 + struct iscsi_datain *, 87 87 + struct iscsi_data_rsp *, bool); 88 88 + extern int iscsit_build_r2ts_for_cmd(struct iscsi_conn *, struct iscsi_cmd *, 89 89 + bool); 90 90 + extern int iscsit_immediate_queue(struct iscsi_conn *, struct iscsi_cmd *, int); 91 91 + extern int iscsit_response_queue(struct iscsi_conn *, struct iscsi_cmd *, int); 88 92 /* 89 93 * From iscsi_target_device.c 90 94 */ ··· 122 102 extern int iscsit_sequence_cmd(struct iscsi_conn *, struct iscsi_cmd *, 123 103 unsigned char *, __be32); 124 104 extern void iscsit_release_cmd(struct iscsi_cmd *); 105 105 + extern void iscsit_free_cmd(struct iscsi_cmd *, bool); 106 106 + extern void iscsit_add_cmd_to_immediate_queue(struct iscsi_cmd *, 107 107 + struct iscsi_conn *, u8); 108 108 + 109 109 + /* 110 110 + * From iscsi_target_nego.c 111 111 + */ 112 112 + extern int iscsi_target_check_login_request(struct iscsi_conn *, 113 113 + struct iscsi_login *); 114 114 + 115 115 + /* 116 116 + * From iscsi_target_login.c 117 117 + */ 118 118 + extern __printf(2, 3) int iscsi_change_param_sprintf( 119 119 + struct iscsi_conn *, const char *, ...); 120 120 + 121 121 + /* 122 122 + * From iscsi_target_parameters.c 123 123 + */ 124 124 + extern struct iscsi_param *iscsi_find_param_from_key( 125 125 + char *, struct iscsi_param_list *);

-2

include/target/target_core_base.h

reviewed

··· 536 536 char initiatorname[TRANSPORT_IQN_LEN]; 537 537 /* Used to signal demo mode created ACL, disabled by default */ 538 538 bool dynamic_node_acl; 539 539 - bool acl_stop:1; 540 539 u32 queue_depth; 541 540 u32 acl_index; 542 541 enum target_prot_type saved_prot_type; ··· 602 603 struct list_head sess_cmd_list; 603 604 struct list_head sess_wait_list; 604 605 spinlock_t sess_cmd_lock; 605 605 - struct kref sess_kref; 606 606 void *sess_cmd_map; 607 607 struct percpu_ida sess_tag_pool; 608 608 };

-6

include/target/target_core_fabric.h

reviewed

··· 50 50 */ 51 51 int (*check_stop_free)(struct se_cmd *); 52 52 void (*release_cmd)(struct se_cmd *); 53 53 - /* 54 54 - * Called with spin_lock_bh(struct se_portal_group->session_lock held. 55 55 - */ 56 56 - int (*shutdown_session)(struct se_session *); 57 53 void (*close_session)(struct se_session *); 58 54 u32 (*sess_get_index)(struct se_session *); 59 55 /* ··· 119 123 struct se_node_acl *, struct se_session *, void *); 120 124 void transport_register_session(struct se_portal_group *, 121 125 struct se_node_acl *, struct se_session *, void *); 122 122 - int target_get_session(struct se_session *); 123 123 - void target_put_session(struct se_session *); 124 126 ssize_t target_show_dynamic_sessions(struct se_portal_group *, char *); 125 127 void transport_free_session(struct se_session *); 126 128 void target_put_nacl(struct se_node_acl *);