340 cxgbit_map_skb(struct iscsit_cmd *cmd, struct sk_buff *skb, u32 data_offset,  in cxgbit_map_skb()  argument
355 	sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];  in cxgbit_map_skb()
393 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsit_cmd *cmd,  in cxgbit_tx_datain_iso()  argument
400 	u32 data_length = cmd->se_cmd.data_length;  in cxgbit_tx_datain_iso()
403 	bool task_sense = !!(cmd->se_cmd.se_cmd_flags &  in cxgbit_tx_datain_iso()
439 				iscsit_increment_maxcmdsn(cmd, conn->sess);  in cxgbit_tx_datain_iso()
440 				cmd->stat_sn = conn->stat_sn++;  in cxgbit_tx_datain_iso()
452 		datain.data_sn = cmd->data_sn - 1;  in cxgbit_tx_datain_iso()
454 		iscsit_build_datain_pdu(cmd, conn, &datain,  in cxgbit_tx_datain_iso()
458 		ret = cxgbit_map_skb(cmd, skb, tx_data, plen);  in cxgbit_tx_datain_iso()
471 		cmd->read_data_done += plen;  in cxgbit_tx_datain_iso()
472 		cmd->data_sn += num_pdu;  in cxgbit_tx_datain_iso()
484 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsit_cmd *cmd,  in cxgbit_tx_datain()  argument
494 	memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);  in cxgbit_tx_datain()
503 	ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);  in cxgbit_tx_datain()
513 cxgbit_xmit_datain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,  in cxgbit_xmit_datain_pdu()  argument
518 	u32 data_length = cmd->se_cmd.data_length;  in cxgbit_xmit_datain_pdu()
526 		return cxgbit_tx_datain_iso(csk, cmd, dr);  in cxgbit_xmit_datain_pdu()
529 	return cxgbit_tx_datain(csk, cmd, datain);  in cxgbit_xmit_datain_pdu()
533 cxgbit_xmit_nondatain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,  in cxgbit_xmit_nondatain_pdu()  argument
544 	memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);  in cxgbit_xmit_nondatain_pdu()
563 cxgbit_xmit_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,  in cxgbit_xmit_pdu()  argument
567 		return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);  in cxgbit_xmit_pdu()
569 		return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);  in cxgbit_xmit_pdu()
840 	struct iscsit_cmd *cmd;  in cxgbit_allocate_cmd()  local
842 	cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);  in cxgbit_allocate_cmd()
843 	if (!cmd) {  in cxgbit_allocate_cmd()
848 	ccmd = iscsit_priv_cmd(cmd);  in cxgbit_allocate_cmd()
852 	return cmd;  in cxgbit_allocate_cmd()
856 cxgbit_handle_immediate_data(struct iscsit_cmd *cmd, struct iscsi_scsi_req *hdr,  in cxgbit_handle_immediate_data()  argument
859 	struct iscsit_conn *conn = cmd->conn;  in cxgbit_handle_immediate_data()
869 			iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,  in cxgbit_handle_immediate_data()
874 		iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,  in cxgbit_handle_immediate_data()
879 	if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {  in cxgbit_handle_immediate_data()
880 		struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);  in cxgbit_handle_immediate_data()
889 		cmd->se_cmd.t_data_sg = &ccmd->sg;  in cxgbit_handle_immediate_data()
890 		cmd->se_cmd.t_data_nents = 1;  in cxgbit_handle_immediate_data()
894 		struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];  in cxgbit_handle_immediate_data()
900 	cmd->write_data_done += pdu_cb->dlen;  in cxgbit_handle_immediate_data()
902 	if (cmd->write_data_done == cmd->se_cmd.data_length) {  in cxgbit_handle_immediate_data()
903 		spin_lock_bh(&cmd->istate_lock);  in cxgbit_handle_immediate_data()
904 		cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;  in cxgbit_handle_immediate_data()
905 		cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;  in cxgbit_handle_immediate_data()
906 		spin_unlock_bh(&cmd->istate_lock);  in cxgbit_handle_immediate_data()
913 cxgbit_get_immediate_data(struct iscsit_cmd *cmd, struct iscsi_scsi_req *hdr,  in cxgbit_get_immediate_data()  argument
916 	struct iscsit_conn *conn = cmd->conn;  in cxgbit_get_immediate_data()
924 	immed_ret = cxgbit_handle_immediate_data(cmd, hdr,  in cxgbit_get_immediate_data()
925 						 cmd->first_burst_len);  in cxgbit_get_immediate_data()
933 		cmdsn_ret = iscsit_sequence_cmd(conn, cmd,  in cxgbit_get_immediate_data()
939 		if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {  in cxgbit_get_immediate_data()
940 			target_put_sess_cmd(&cmd->se_cmd);  in cxgbit_get_immediate_data()
942 		} else if (cmd->unsolicited_data) {  in cxgbit_get_immediate_data()
943 			iscsit_set_unsolicited_dataout(cmd);  in cxgbit_get_immediate_data()
958 		cmd->i_state = ISTATE_REMOVE;  in cxgbit_get_immediate_data()
959 		iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);  in cxgbit_get_immediate_data()
967 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsit_cmd *cmd)  in cxgbit_handle_scsi_cmd()  argument
975 	rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);  in cxgbit_handle_scsi_cmd()
979 	if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&  in cxgbit_handle_scsi_cmd()
981 		cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;  in cxgbit_handle_scsi_cmd()
983 	rc = iscsit_process_scsi_cmd(conn, cmd, hdr);  in cxgbit_handle_scsi_cmd()
992 	return cxgbit_get_immediate_data(cmd, hdr, dump_payload);  in cxgbit_handle_scsi_cmd()
999 	struct iscsit_cmd *cmd = NULL;  in cxgbit_handle_iscsi_dataout()  local
1014 		cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0);  in cxgbit_handle_iscsi_dataout()
1015 		if (!cmd)  in cxgbit_handle_iscsi_dataout()
1018 		ddp_data_len = offset - cmd->write_data_done;  in cxgbit_handle_iscsi_dataout()
1021 		cmd->write_data_done = offset;  in cxgbit_handle_iscsi_dataout()
1022 		cmd->next_burst_len = ddp_data_len;  in cxgbit_handle_iscsi_dataout()
1023 		cmd->data_sn = be32_to_cpu(hdr->datasn);  in cxgbit_handle_iscsi_dataout()
1026 						cmd, data_len, &success);  in cxgbit_handle_iscsi_dataout()
1032 		rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);  in cxgbit_handle_iscsi_dataout()
1035 		else if (!cmd)  in cxgbit_handle_iscsi_dataout()
1051 		  data_len,  cmd->write_data_done,  in cxgbit_handle_iscsi_dataout()
1052 		  cmd->se_cmd.data_length);  in cxgbit_handle_iscsi_dataout()
1058 		sg_start = &cmd->se_cmd.t_data_sg[sg_off];  in cxgbit_handle_iscsi_dataout()
1064 	ccmd = iscsit_priv_cmd(cmd);  in cxgbit_handle_iscsi_dataout()
1068 	    (cmd->se_cmd.data_length ==	(cmd->write_data_done + data_len))) {  in cxgbit_handle_iscsi_dataout()
1080 	rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);  in cxgbit_handle_iscsi_dataout()
1087 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsit_cmd *cmd)  in cxgbit_handle_nop_out()  argument
1096 	ret = iscsit_setup_nop_out(conn, cmd, hdr);  in cxgbit_handle_nop_out()
1139 		cmd->buf_ptr = ping_data;  in cxgbit_handle_nop_out()
1140 		cmd->buf_ptr_size = payload_length;  in cxgbit_handle_nop_out()
1147 	return iscsit_process_nop_out(conn, cmd, hdr);  in cxgbit_handle_nop_out()
1149 	if (cmd)  in cxgbit_handle_nop_out()
1150 		iscsit_free_cmd(cmd, false);  in cxgbit_handle_nop_out()
1155 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsit_cmd *cmd)  in cxgbit_handle_text_cmd()  argument
1164 	rc = iscsit_setup_text_cmd(conn, cmd, hdr);  in cxgbit_handle_text_cmd()
1198 		cmd->text_in_ptr = text_in;  in cxgbit_handle_text_cmd()
1201 	return iscsit_process_text_cmd(conn, cmd, hdr);  in cxgbit_handle_text_cmd()
1204 	return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,  in cxgbit_handle_text_cmd()
1213 	struct iscsit_cmd *cmd = NULL;  in cxgbit_target_rx_opcode()  local
1219 		cmd = cxgbit_allocate_cmd(csk);  in cxgbit_target_rx_opcode()
1220 		if (!cmd)  in cxgbit_target_rx_opcode()
1223 		ret = cxgbit_handle_scsi_cmd(csk, cmd);  in cxgbit_target_rx_opcode()
1230 			cmd = cxgbit_allocate_cmd(csk);  in cxgbit_target_rx_opcode()
1231 			if (!cmd)  in cxgbit_target_rx_opcode()
1235 		ret = cxgbit_handle_nop_out(csk, cmd);  in cxgbit_target_rx_opcode()
1238 		cmd = cxgbit_allocate_cmd(csk);  in cxgbit_target_rx_opcode()
1239 		if (!cmd)  in cxgbit_target_rx_opcode()
1242 		ret = iscsit_handle_task_mgt_cmd(conn, cmd,  in cxgbit_target_rx_opcode()
1247 			cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);  in cxgbit_target_rx_opcode()
1248 			if (!cmd)  in cxgbit_target_rx_opcode()
1251 			cmd = cxgbit_allocate_cmd(csk);  in cxgbit_target_rx_opcode()
1252 			if (!cmd)  in cxgbit_target_rx_opcode()
1256 		ret = cxgbit_handle_text_cmd(csk, cmd);  in cxgbit_target_rx_opcode()
1259 		cmd = cxgbit_allocate_cmd(csk);  in cxgbit_target_rx_opcode()
1260 		if (!cmd)  in cxgbit_target_rx_opcode()
1263 		ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);  in cxgbit_target_rx_opcode()