1e48354ceSNicholas Bellinger /*******************************************************************************
2e48354ceSNicholas Bellinger  * This file contains the iSCSI Target specific utility functions.
3e48354ceSNicholas Bellinger  *
44c76251eSNicholas Bellinger  * (c) Copyright 2007-2013 Datera, Inc.
5e48354ceSNicholas Bellinger  *
6e48354ceSNicholas Bellinger  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
7e48354ceSNicholas Bellinger  *
8e48354ceSNicholas Bellinger  * This program is free software; you can redistribute it and/or modify
9e48354ceSNicholas Bellinger  * it under the terms of the GNU General Public License as published by
10e48354ceSNicholas Bellinger  * the Free Software Foundation; either version 2 of the License, or
11e48354ceSNicholas Bellinger  * (at your option) any later version.
12e48354ceSNicholas Bellinger  *
13e48354ceSNicholas Bellinger  * This program is distributed in the hope that it will be useful,
14e48354ceSNicholas Bellinger  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15e48354ceSNicholas Bellinger  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16e48354ceSNicholas Bellinger  * GNU General Public License for more details.
17e48354ceSNicholas Bellinger  ******************************************************************************/
18e48354ceSNicholas Bellinger 
19e48354ceSNicholas Bellinger #include <linux/list.h>
20988e3a85SNicholas Bellinger #include <linux/percpu_ida.h>
21e48354ceSNicholas Bellinger #include <scsi/scsi_tcq.h>
22e48354ceSNicholas Bellinger #include <scsi/iscsi_proto.h>
23e48354ceSNicholas Bellinger #include <target/target_core_base.h>
24c4795fb2SChristoph Hellwig #include <target/target_core_fabric.h>
25e48354ceSNicholas Bellinger #include <target/target_core_configfs.h>
26baa4d64bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h>
27e48354ceSNicholas Bellinger 
2867f091f2SSagi Grimberg #include <target/iscsi/iscsi_target_core.h>
29e48354ceSNicholas Bellinger #include "iscsi_target_parameters.h"
30e48354ceSNicholas Bellinger #include "iscsi_target_seq_pdu_list.h"
31e48354ceSNicholas Bellinger #include "iscsi_target_datain_values.h"
32e48354ceSNicholas Bellinger #include "iscsi_target_erl0.h"
33e48354ceSNicholas Bellinger #include "iscsi_target_erl1.h"
34e48354ceSNicholas Bellinger #include "iscsi_target_erl2.h"
35e48354ceSNicholas Bellinger #include "iscsi_target_tpg.h"
36e48354ceSNicholas Bellinger #include "iscsi_target_tq.h"
37e48354ceSNicholas Bellinger #include "iscsi_target_util.h"
38e48354ceSNicholas Bellinger #include "iscsi_target.h"
39e48354ceSNicholas Bellinger 
40e48354ceSNicholas Bellinger #define PRINT_BUFF(buff, len)					\
41e48354ceSNicholas Bellinger {								\
42e48354ceSNicholas Bellinger 	int zzz;						\
43e48354ceSNicholas Bellinger 								\
44e48354ceSNicholas Bellinger 	pr_debug("%d:\n", __LINE__);				\
45e48354ceSNicholas Bellinger 	for (zzz = 0; zzz < len; zzz++) {			\
46e48354ceSNicholas Bellinger 		if (zzz % 16 == 0) {				\
47e48354ceSNicholas Bellinger 			if (zzz)				\
48e48354ceSNicholas Bellinger 				pr_debug("\n");			\
49e48354ceSNicholas Bellinger 			pr_debug("%4i: ", zzz);			\
50e48354ceSNicholas Bellinger 		}						\
51e48354ceSNicholas Bellinger 		pr_debug("%02x ", (unsigned char) (buff)[zzz]);	\
52e48354ceSNicholas Bellinger 	}							\
53e48354ceSNicholas Bellinger 	if ((len + 1) % 16)					\
54e48354ceSNicholas Bellinger 		pr_debug("\n");					\
55e48354ceSNicholas Bellinger }
56e48354ceSNicholas Bellinger 
57e48354ceSNicholas Bellinger extern struct list_head g_tiqn_list;
58e48354ceSNicholas Bellinger extern spinlock_t tiqn_lock;
59e48354ceSNicholas Bellinger 
60e48354ceSNicholas Bellinger /*
61e48354ceSNicholas Bellinger  *	Called with cmd->r2t_lock held.
62e48354ceSNicholas Bellinger  */
63e48354ceSNicholas Bellinger int iscsit_add_r2t_to_list(
64e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd,
65e48354ceSNicholas Bellinger 	u32 offset,
66e48354ceSNicholas Bellinger 	u32 xfer_len,
67e48354ceSNicholas Bellinger 	int recovery,
68e48354ceSNicholas Bellinger 	u32 r2t_sn)
69e48354ceSNicholas Bellinger {
70e48354ceSNicholas Bellinger 	struct iscsi_r2t *r2t;
71e48354ceSNicholas Bellinger 
72e48354ceSNicholas Bellinger 	r2t = kmem_cache_zalloc(lio_r2t_cache, GFP_ATOMIC);
73e48354ceSNicholas Bellinger 	if (!r2t) {
74e48354ceSNicholas Bellinger 		pr_err("Unable to allocate memory for struct iscsi_r2t.\n");
75e48354ceSNicholas Bellinger 		return -1;
76e48354ceSNicholas Bellinger 	}
77e48354ceSNicholas Bellinger 	INIT_LIST_HEAD(&r2t->r2t_list);
78e48354ceSNicholas Bellinger 
79e48354ceSNicholas Bellinger 	r2t->recovery_r2t = recovery;
80e48354ceSNicholas Bellinger 	r2t->r2t_sn = (!r2t_sn) ? cmd->r2t_sn++ : r2t_sn;
81e48354ceSNicholas Bellinger 	r2t->offset = offset;
82e48354ceSNicholas Bellinger 	r2t->xfer_len = xfer_len;
83e48354ceSNicholas Bellinger 	list_add_tail(&r2t->r2t_list, &cmd->cmd_r2t_list);
84e48354ceSNicholas Bellinger 	spin_unlock_bh(&cmd->r2t_lock);
85e48354ceSNicholas Bellinger 
86e48354ceSNicholas Bellinger 	iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, ISTATE_SEND_R2T);
87e48354ceSNicholas Bellinger 
88e48354ceSNicholas Bellinger 	spin_lock_bh(&cmd->r2t_lock);
89e48354ceSNicholas Bellinger 	return 0;
90e48354ceSNicholas Bellinger }
91e48354ceSNicholas Bellinger 
92e48354ceSNicholas Bellinger struct iscsi_r2t *iscsit_get_r2t_for_eos(
93e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd,
94e48354ceSNicholas Bellinger 	u32 offset,
95e48354ceSNicholas Bellinger 	u32 length)
96e48354ceSNicholas Bellinger {
97e48354ceSNicholas Bellinger 	struct iscsi_r2t *r2t;
98e48354ceSNicholas Bellinger 
99e48354ceSNicholas Bellinger 	spin_lock_bh(&cmd->r2t_lock);
100e48354ceSNicholas Bellinger 	list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
101e48354ceSNicholas Bellinger 		if ((r2t->offset <= offset) &&
102e48354ceSNicholas Bellinger 		    (r2t->offset + r2t->xfer_len) >= (offset + length)) {
103e48354ceSNicholas Bellinger 			spin_unlock_bh(&cmd->r2t_lock);
104e48354ceSNicholas Bellinger 			return r2t;
105e48354ceSNicholas Bellinger 		}
106e48354ceSNicholas Bellinger 	}
107e48354ceSNicholas Bellinger 	spin_unlock_bh(&cmd->r2t_lock);
108e48354ceSNicholas Bellinger 
109e48354ceSNicholas Bellinger 	pr_err("Unable to locate R2T for Offset: %u, Length:"
110e48354ceSNicholas Bellinger 			" %u\n", offset, length);
111e48354ceSNicholas Bellinger 	return NULL;
112e48354ceSNicholas Bellinger }
113e48354ceSNicholas Bellinger 
114e48354ceSNicholas Bellinger struct iscsi_r2t *iscsit_get_r2t_from_list(struct iscsi_cmd *cmd)
115e48354ceSNicholas Bellinger {
116e48354ceSNicholas Bellinger 	struct iscsi_r2t *r2t;
117e48354ceSNicholas Bellinger 
118e48354ceSNicholas Bellinger 	spin_lock_bh(&cmd->r2t_lock);
119e48354ceSNicholas Bellinger 	list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
120e48354ceSNicholas Bellinger 		if (!r2t->sent_r2t) {
121e48354ceSNicholas Bellinger 			spin_unlock_bh(&cmd->r2t_lock);
122e48354ceSNicholas Bellinger 			return r2t;
123e48354ceSNicholas Bellinger 		}
124e48354ceSNicholas Bellinger 	}
125e48354ceSNicholas Bellinger 	spin_unlock_bh(&cmd->r2t_lock);
126e48354ceSNicholas Bellinger 
127e48354ceSNicholas Bellinger 	pr_err("Unable to locate next R2T to send for ITT:"
128e48354ceSNicholas Bellinger 			" 0x%08x.\n", cmd->init_task_tag);
129e48354ceSNicholas Bellinger 	return NULL;
130e48354ceSNicholas Bellinger }
131e48354ceSNicholas Bellinger 
132e48354ceSNicholas Bellinger /*
133e48354ceSNicholas Bellinger  *	Called with cmd->r2t_lock held.
134e48354ceSNicholas Bellinger  */
135e48354ceSNicholas Bellinger void iscsit_free_r2t(struct iscsi_r2t *r2t, struct iscsi_cmd *cmd)
136e48354ceSNicholas Bellinger {
137e48354ceSNicholas Bellinger 	list_del(&r2t->r2t_list);
138e48354ceSNicholas Bellinger 	kmem_cache_free(lio_r2t_cache, r2t);
139e48354ceSNicholas Bellinger }
140e48354ceSNicholas Bellinger 
141e48354ceSNicholas Bellinger void iscsit_free_r2ts_from_list(struct iscsi_cmd *cmd)
142e48354ceSNicholas Bellinger {
143e48354ceSNicholas Bellinger 	struct iscsi_r2t *r2t, *r2t_tmp;
144e48354ceSNicholas Bellinger 
145e48354ceSNicholas Bellinger 	spin_lock_bh(&cmd->r2t_lock);
146e48354ceSNicholas Bellinger 	list_for_each_entry_safe(r2t, r2t_tmp, &cmd->cmd_r2t_list, r2t_list)
147e48354ceSNicholas Bellinger 		iscsit_free_r2t(r2t, cmd);
148e48354ceSNicholas Bellinger 	spin_unlock_bh(&cmd->r2t_lock);
149e48354ceSNicholas Bellinger }
150e48354ceSNicholas Bellinger 
151e48354ceSNicholas Bellinger /*
152e48354ceSNicholas Bellinger  * May be called from software interrupt (timer) context for allocating
153e48354ceSNicholas Bellinger  * iSCSI NopINs.
154e48354ceSNicholas Bellinger  */
155676687c6SNicholas Bellinger struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *conn, int state)
156e48354ceSNicholas Bellinger {
157e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd;
158988e3a85SNicholas Bellinger 	struct se_session *se_sess = conn->sess->se_sess;
159676687c6SNicholas Bellinger 	int size, tag;
160e48354ceSNicholas Bellinger 
1616f6b5d1eSKent Overstreet 	tag = percpu_ida_alloc(&se_sess->sess_tag_pool, state);
1626f6b5d1eSKent Overstreet 	if (tag < 0)
1636f6b5d1eSKent Overstreet 		return NULL;
1646f6b5d1eSKent Overstreet 
165988e3a85SNicholas Bellinger 	size = sizeof(struct iscsi_cmd) + conn->conn_transport->priv_size;
166988e3a85SNicholas Bellinger 	cmd = (struct iscsi_cmd *)(se_sess->sess_cmd_map + (tag * size));
167988e3a85SNicholas Bellinger 	memset(cmd, 0, size);
168988e3a85SNicholas Bellinger 
169988e3a85SNicholas Bellinger 	cmd->se_cmd.map_tag = tag;
170e48354ceSNicholas Bellinger 	cmd->conn = conn;
1712fbb471eSAndy Grover 	INIT_LIST_HEAD(&cmd->i_conn_node);
172e48354ceSNicholas Bellinger 	INIT_LIST_HEAD(&cmd->datain_list);
173e48354ceSNicholas Bellinger 	INIT_LIST_HEAD(&cmd->cmd_r2t_list);
174e48354ceSNicholas Bellinger 	spin_lock_init(&cmd->datain_lock);
175e48354ceSNicholas Bellinger 	spin_lock_init(&cmd->dataout_timeout_lock);
176e48354ceSNicholas Bellinger 	spin_lock_init(&cmd->istate_lock);
177e48354ceSNicholas Bellinger 	spin_lock_init(&cmd->error_lock);
178e48354ceSNicholas Bellinger 	spin_lock_init(&cmd->r2t_lock);
179e48354ceSNicholas Bellinger 
180e48354ceSNicholas Bellinger 	return cmd;
181e48354ceSNicholas Bellinger }
182cdb72665SNicholas Bellinger EXPORT_SYMBOL(iscsit_allocate_cmd);
183e48354ceSNicholas Bellinger 
184e48354ceSNicholas Bellinger struct iscsi_seq *iscsit_get_seq_holder_for_datain(
185e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd,
186e48354ceSNicholas Bellinger 	u32 seq_send_order)
187e48354ceSNicholas Bellinger {
188e48354ceSNicholas Bellinger 	u32 i;
189e48354ceSNicholas Bellinger 
190e48354ceSNicholas Bellinger 	for (i = 0; i < cmd->seq_count; i++)
191e48354ceSNicholas Bellinger 		if (cmd->seq_list[i].seq_send_order == seq_send_order)
192e48354ceSNicholas Bellinger 			return &cmd->seq_list[i];
193e48354ceSNicholas Bellinger 
194e48354ceSNicholas Bellinger 	return NULL;
195e48354ceSNicholas Bellinger }
196e48354ceSNicholas Bellinger 
197e48354ceSNicholas Bellinger struct iscsi_seq *iscsit_get_seq_holder_for_r2t(struct iscsi_cmd *cmd)
198e48354ceSNicholas Bellinger {
199e48354ceSNicholas Bellinger 	u32 i;
200e48354ceSNicholas Bellinger 
201e48354ceSNicholas Bellinger 	if (!cmd->seq_list) {
202e48354ceSNicholas Bellinger 		pr_err("struct iscsi_cmd->seq_list is NULL!\n");
203e48354ceSNicholas Bellinger 		return NULL;
204e48354ceSNicholas Bellinger 	}
205e48354ceSNicholas Bellinger 
206e48354ceSNicholas Bellinger 	for (i = 0; i < cmd->seq_count; i++) {
207e48354ceSNicholas Bellinger 		if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
208e48354ceSNicholas Bellinger 			continue;
209e48354ceSNicholas Bellinger 		if (cmd->seq_list[i].seq_send_order == cmd->seq_send_order) {
210e48354ceSNicholas Bellinger 			cmd->seq_send_order++;
211e48354ceSNicholas Bellinger 			return &cmd->seq_list[i];
212e48354ceSNicholas Bellinger 		}
213e48354ceSNicholas Bellinger 	}
214e48354ceSNicholas Bellinger 
215e48354ceSNicholas Bellinger 	return NULL;
216e48354ceSNicholas Bellinger }
217e48354ceSNicholas Bellinger 
218e48354ceSNicholas Bellinger struct iscsi_r2t *iscsit_get_holder_for_r2tsn(
219e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd,
220e48354ceSNicholas Bellinger 	u32 r2t_sn)
221e48354ceSNicholas Bellinger {
222e48354ceSNicholas Bellinger 	struct iscsi_r2t *r2t;
223e48354ceSNicholas Bellinger 
224e48354ceSNicholas Bellinger 	spin_lock_bh(&cmd->r2t_lock);
225e48354ceSNicholas Bellinger 	list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
226e48354ceSNicholas Bellinger 		if (r2t->r2t_sn == r2t_sn) {
227e48354ceSNicholas Bellinger 			spin_unlock_bh(&cmd->r2t_lock);
228e48354ceSNicholas Bellinger 			return r2t;
229e48354ceSNicholas Bellinger 		}
230e48354ceSNicholas Bellinger 	}
231e48354ceSNicholas Bellinger 	spin_unlock_bh(&cmd->r2t_lock);
232e48354ceSNicholas Bellinger 
233e48354ceSNicholas Bellinger 	return NULL;
234e48354ceSNicholas Bellinger }
235e48354ceSNicholas Bellinger 
236e48354ceSNicholas Bellinger static inline int iscsit_check_received_cmdsn(struct iscsi_session *sess, u32 cmdsn)
237e48354ceSNicholas Bellinger {
238e48354ceSNicholas Bellinger 	int ret;
239e48354ceSNicholas Bellinger 
240e48354ceSNicholas Bellinger 	/*
241e48354ceSNicholas Bellinger 	 * This is the proper method of checking received CmdSN against
242e48354ceSNicholas Bellinger 	 * ExpCmdSN and MaxCmdSN values, as well as accounting for out
243e48354ceSNicholas Bellinger 	 * or order CmdSNs due to multiple connection sessions and/or
244e48354ceSNicholas Bellinger 	 * CRC failures.
245e48354ceSNicholas Bellinger 	 */
246e48354ceSNicholas Bellinger 	if (iscsi_sna_gt(cmdsn, sess->max_cmd_sn)) {
247e48354ceSNicholas Bellinger 		pr_err("Received CmdSN: 0x%08x is greater than"
248ea7e32beSNicholas Bellinger 		       " MaxCmdSN: 0x%08x, ignoring.\n", cmdsn,
249e48354ceSNicholas Bellinger 		       sess->max_cmd_sn);
250ea7e32beSNicholas Bellinger 		ret = CMDSN_MAXCMDSN_OVERRUN;
251e48354ceSNicholas Bellinger 
252e48354ceSNicholas Bellinger 	} else if (cmdsn == sess->exp_cmd_sn) {
253e48354ceSNicholas Bellinger 		sess->exp_cmd_sn++;
254e48354ceSNicholas Bellinger 		pr_debug("Received CmdSN matches ExpCmdSN,"
255e48354ceSNicholas Bellinger 		      " incremented ExpCmdSN to: 0x%08x\n",
256e48354ceSNicholas Bellinger 		      sess->exp_cmd_sn);
257e48354ceSNicholas Bellinger 		ret = CMDSN_NORMAL_OPERATION;
258e48354ceSNicholas Bellinger 
259e48354ceSNicholas Bellinger 	} else if (iscsi_sna_gt(cmdsn, sess->exp_cmd_sn)) {
260e48354ceSNicholas Bellinger 		pr_debug("Received CmdSN: 0x%08x is greater"
261e48354ceSNicholas Bellinger 		      " than ExpCmdSN: 0x%08x, not acknowledging.\n",
262e48354ceSNicholas Bellinger 		      cmdsn, sess->exp_cmd_sn);
263e48354ceSNicholas Bellinger 		ret = CMDSN_HIGHER_THAN_EXP;
264e48354ceSNicholas Bellinger 
265e48354ceSNicholas Bellinger 	} else {
266e48354ceSNicholas Bellinger 		pr_err("Received CmdSN: 0x%08x is less than"
267e48354ceSNicholas Bellinger 		       " ExpCmdSN: 0x%08x, ignoring.\n", cmdsn,
268e48354ceSNicholas Bellinger 		       sess->exp_cmd_sn);
269e48354ceSNicholas Bellinger 		ret = CMDSN_LOWER_THAN_EXP;
270e48354ceSNicholas Bellinger 	}
271e48354ceSNicholas Bellinger 
272e48354ceSNicholas Bellinger 	return ret;
273e48354ceSNicholas Bellinger }
274e48354ceSNicholas Bellinger 
275e48354ceSNicholas Bellinger /*
276e48354ceSNicholas Bellinger  * Commands may be received out of order if MC/S is in use.
277e48354ceSNicholas Bellinger  * Ensure they are executed in CmdSN order.
278e48354ceSNicholas Bellinger  */
279561bf158SNicholas Bellinger int iscsit_sequence_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
280561bf158SNicholas Bellinger 			unsigned char *buf, __be32 cmdsn)
281e48354ceSNicholas Bellinger {
282561bf158SNicholas Bellinger 	int ret, cmdsn_ret;
283561bf158SNicholas Bellinger 	bool reject = false;
284561bf158SNicholas Bellinger 	u8 reason = ISCSI_REASON_BOOKMARK_NO_RESOURCES;
285e48354ceSNicholas Bellinger 
286e48354ceSNicholas Bellinger 	mutex_lock(&conn->sess->cmdsn_mutex);
287e48354ceSNicholas Bellinger 
28850e5c87dSChristoph Hellwig 	cmdsn_ret = iscsit_check_received_cmdsn(conn->sess, be32_to_cpu(cmdsn));
289e48354ceSNicholas Bellinger 	switch (cmdsn_ret) {
290e48354ceSNicholas Bellinger 	case CMDSN_NORMAL_OPERATION:
291e48354ceSNicholas Bellinger 		ret = iscsit_execute_cmd(cmd, 0);
292e48354ceSNicholas Bellinger 		if ((ret >= 0) && !list_empty(&conn->sess->sess_ooo_cmdsn_list))
293e48354ceSNicholas Bellinger 			iscsit_execute_ooo_cmdsns(conn->sess);
294561bf158SNicholas Bellinger 		else if (ret < 0) {
295561bf158SNicholas Bellinger 			reject = true;
296561bf158SNicholas Bellinger 			ret = CMDSN_ERROR_CANNOT_RECOVER;
297561bf158SNicholas Bellinger 		}
298e48354ceSNicholas Bellinger 		break;
299e48354ceSNicholas Bellinger 	case CMDSN_HIGHER_THAN_EXP:
30050e5c87dSChristoph Hellwig 		ret = iscsit_handle_ooo_cmdsn(conn->sess, cmd, be32_to_cpu(cmdsn));
301561bf158SNicholas Bellinger 		if (ret < 0) {
302561bf158SNicholas Bellinger 			reject = true;
303561bf158SNicholas Bellinger 			ret = CMDSN_ERROR_CANNOT_RECOVER;
304561bf158SNicholas Bellinger 			break;
305561bf158SNicholas Bellinger 		}
306561bf158SNicholas Bellinger 		ret = CMDSN_HIGHER_THAN_EXP;
307e48354ceSNicholas Bellinger 		break;
308e48354ceSNicholas Bellinger 	case CMDSN_LOWER_THAN_EXP:
309ea7e32beSNicholas Bellinger 	case CMDSN_MAXCMDSN_OVERRUN:
310ea7e32beSNicholas Bellinger 	default:
311e48354ceSNicholas Bellinger 		cmd->i_state = ISTATE_REMOVE;
312e48354ceSNicholas Bellinger 		iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
313ea7e32beSNicholas Bellinger 		/*
314ea7e32beSNicholas Bellinger 		 * Existing callers for iscsit_sequence_cmd() will silently
315ea7e32beSNicholas Bellinger 		 * ignore commands with CMDSN_LOWER_THAN_EXP, so force this
316ea7e32beSNicholas Bellinger 		 * return for CMDSN_MAXCMDSN_OVERRUN as well..
317ea7e32beSNicholas Bellinger 		 */
318ea7e32beSNicholas Bellinger 		ret = CMDSN_LOWER_THAN_EXP;
319e48354ceSNicholas Bellinger 		break;
320e48354ceSNicholas Bellinger 	}
321e48354ceSNicholas Bellinger 	mutex_unlock(&conn->sess->cmdsn_mutex);
322e48354ceSNicholas Bellinger 
323561bf158SNicholas Bellinger 	if (reject)
324561bf158SNicholas Bellinger 		iscsit_reject_cmd(cmd, reason, buf);
325561bf158SNicholas Bellinger 
326e48354ceSNicholas Bellinger 	return ret;
327e48354ceSNicholas Bellinger }
3283e1c81a9SNicholas Bellinger EXPORT_SYMBOL(iscsit_sequence_cmd);
329e48354ceSNicholas Bellinger 
330e48354ceSNicholas Bellinger int iscsit_check_unsolicited_dataout(struct iscsi_cmd *cmd, unsigned char *buf)
331e48354ceSNicholas Bellinger {
332e48354ceSNicholas Bellinger 	struct iscsi_conn *conn = cmd->conn;
333e48354ceSNicholas Bellinger 	struct se_cmd *se_cmd = &cmd->se_cmd;
334e48354ceSNicholas Bellinger 	struct iscsi_data *hdr = (struct iscsi_data *) buf;
335e48354ceSNicholas Bellinger 	u32 payload_length = ntoh24(hdr->dlength);
336e48354ceSNicholas Bellinger 
337e48354ceSNicholas Bellinger 	if (conn->sess->sess_ops->InitialR2T) {
338e48354ceSNicholas Bellinger 		pr_err("Received unexpected unsolicited data"
339e48354ceSNicholas Bellinger 			" while InitialR2T=Yes, protocol error.\n");
340e48354ceSNicholas Bellinger 		transport_send_check_condition_and_sense(se_cmd,
341e48354ceSNicholas Bellinger 				TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
342e48354ceSNicholas Bellinger 		return -1;
343e48354ceSNicholas Bellinger 	}
344e48354ceSNicholas Bellinger 
345e48354ceSNicholas Bellinger 	if ((cmd->first_burst_len + payload_length) >
346e48354ceSNicholas Bellinger 	     conn->sess->sess_ops->FirstBurstLength) {
347e48354ceSNicholas Bellinger 		pr_err("Total %u bytes exceeds FirstBurstLength: %u"
348e48354ceSNicholas Bellinger 			" for this Unsolicited DataOut Burst.\n",
349e48354ceSNicholas Bellinger 			(cmd->first_burst_len + payload_length),
350e48354ceSNicholas Bellinger 				conn->sess->sess_ops->FirstBurstLength);
351e48354ceSNicholas Bellinger 		transport_send_check_condition_and_sense(se_cmd,
352e48354ceSNicholas Bellinger 				TCM_INCORRECT_AMOUNT_OF_DATA, 0);
353e48354ceSNicholas Bellinger 		return -1;
354e48354ceSNicholas Bellinger 	}
355e48354ceSNicholas Bellinger 
356e48354ceSNicholas Bellinger 	if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))
357e48354ceSNicholas Bellinger 		return 0;
358e48354ceSNicholas Bellinger 
359ebf1d95cSAndy Grover 	if (((cmd->first_burst_len + payload_length) != cmd->se_cmd.data_length) &&
360e48354ceSNicholas Bellinger 	    ((cmd->first_burst_len + payload_length) !=
361e48354ceSNicholas Bellinger 	      conn->sess->sess_ops->FirstBurstLength)) {
362e48354ceSNicholas Bellinger 		pr_err("Unsolicited non-immediate data received %u"
363e48354ceSNicholas Bellinger 			" does not equal FirstBurstLength: %u, and does"
364e48354ceSNicholas Bellinger 			" not equal ExpXferLen %u.\n",
365e48354ceSNicholas Bellinger 			(cmd->first_burst_len + payload_length),
366ebf1d95cSAndy Grover 			conn->sess->sess_ops->FirstBurstLength, cmd->se_cmd.data_length);
367e48354ceSNicholas Bellinger 		transport_send_check_condition_and_sense(se_cmd,
368e48354ceSNicholas Bellinger 				TCM_INCORRECT_AMOUNT_OF_DATA, 0);
369e48354ceSNicholas Bellinger 		return -1;
370e48354ceSNicholas Bellinger 	}
371e48354ceSNicholas Bellinger 	return 0;
372e48354ceSNicholas Bellinger }
373e48354ceSNicholas Bellinger 
374e48354ceSNicholas Bellinger struct iscsi_cmd *iscsit_find_cmd_from_itt(
375e48354ceSNicholas Bellinger 	struct iscsi_conn *conn,
37666c7db68SChristoph Hellwig 	itt_t init_task_tag)
377e48354ceSNicholas Bellinger {
378e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd;
379e48354ceSNicholas Bellinger 
380e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->cmd_lock);
3812fbb471eSAndy Grover 	list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
382e48354ceSNicholas Bellinger 		if (cmd->init_task_tag == init_task_tag) {
383e48354ceSNicholas Bellinger 			spin_unlock_bh(&conn->cmd_lock);
384e48354ceSNicholas Bellinger 			return cmd;
385e48354ceSNicholas Bellinger 		}
386e48354ceSNicholas Bellinger 	}
387e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->cmd_lock);
388e48354ceSNicholas Bellinger 
389e48354ceSNicholas Bellinger 	pr_err("Unable to locate ITT: 0x%08x on CID: %hu",
390e48354ceSNicholas Bellinger 			init_task_tag, conn->cid);
391e48354ceSNicholas Bellinger 	return NULL;
392e48354ceSNicholas Bellinger }
393e48354ceSNicholas Bellinger 
394e48354ceSNicholas Bellinger struct iscsi_cmd *iscsit_find_cmd_from_itt_or_dump(
395e48354ceSNicholas Bellinger 	struct iscsi_conn *conn,
39666c7db68SChristoph Hellwig 	itt_t init_task_tag,
397e48354ceSNicholas Bellinger 	u32 length)
398e48354ceSNicholas Bellinger {
399e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd;
400e48354ceSNicholas Bellinger 
401e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->cmd_lock);
4022fbb471eSAndy Grover 	list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
4033687db88SNicholas Bellinger 		if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT)
4043687db88SNicholas Bellinger 			continue;
405e48354ceSNicholas Bellinger 		if (cmd->init_task_tag == init_task_tag) {
406e48354ceSNicholas Bellinger 			spin_unlock_bh(&conn->cmd_lock);
407e48354ceSNicholas Bellinger 			return cmd;
408e48354ceSNicholas Bellinger 		}
409e48354ceSNicholas Bellinger 	}
410e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->cmd_lock);
411e48354ceSNicholas Bellinger 
412e48354ceSNicholas Bellinger 	pr_err("Unable to locate ITT: 0x%08x on CID: %hu,"
413e48354ceSNicholas Bellinger 			" dumping payload\n", init_task_tag, conn->cid);
414e48354ceSNicholas Bellinger 	if (length)
415e48354ceSNicholas Bellinger 		iscsit_dump_data_payload(conn, length, 1);
416e48354ceSNicholas Bellinger 
417e48354ceSNicholas Bellinger 	return NULL;
418e48354ceSNicholas Bellinger }
419e48354ceSNicholas Bellinger 
420e48354ceSNicholas Bellinger struct iscsi_cmd *iscsit_find_cmd_from_ttt(
421e48354ceSNicholas Bellinger 	struct iscsi_conn *conn,
422e48354ceSNicholas Bellinger 	u32 targ_xfer_tag)
423e48354ceSNicholas Bellinger {
424e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd = NULL;
425e48354ceSNicholas Bellinger 
426e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->cmd_lock);
4272fbb471eSAndy Grover 	list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
428e48354ceSNicholas Bellinger 		if (cmd->targ_xfer_tag == targ_xfer_tag) {
429e48354ceSNicholas Bellinger 			spin_unlock_bh(&conn->cmd_lock);
430e48354ceSNicholas Bellinger 			return cmd;
431e48354ceSNicholas Bellinger 		}
432e48354ceSNicholas Bellinger 	}
433e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->cmd_lock);
434e48354ceSNicholas Bellinger 
435e48354ceSNicholas Bellinger 	pr_err("Unable to locate TTT: 0x%08x on CID: %hu\n",
436e48354ceSNicholas Bellinger 			targ_xfer_tag, conn->cid);
437e48354ceSNicholas Bellinger 	return NULL;
438e48354ceSNicholas Bellinger }
439e48354ceSNicholas Bellinger 
440e48354ceSNicholas Bellinger int iscsit_find_cmd_for_recovery(
441e48354ceSNicholas Bellinger 	struct iscsi_session *sess,
442e48354ceSNicholas Bellinger 	struct iscsi_cmd **cmd_ptr,
443e48354ceSNicholas Bellinger 	struct iscsi_conn_recovery **cr_ptr,
44466c7db68SChristoph Hellwig 	itt_t init_task_tag)
445e48354ceSNicholas Bellinger {
446e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd = NULL;
447e48354ceSNicholas Bellinger 	struct iscsi_conn_recovery *cr;
448e48354ceSNicholas Bellinger 	/*
449e48354ceSNicholas Bellinger 	 * Scan through the inactive connection recovery list's command list.
450e48354ceSNicholas Bellinger 	 * If init_task_tag matches the command is still alligent.
451e48354ceSNicholas Bellinger 	 */
452e48354ceSNicholas Bellinger 	spin_lock(&sess->cr_i_lock);
453e48354ceSNicholas Bellinger 	list_for_each_entry(cr, &sess->cr_inactive_list, cr_list) {
454e48354ceSNicholas Bellinger 		spin_lock(&cr->conn_recovery_cmd_lock);
4552fbb471eSAndy Grover 		list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {
456e48354ceSNicholas Bellinger 			if (cmd->init_task_tag == init_task_tag) {
457e48354ceSNicholas Bellinger 				spin_unlock(&cr->conn_recovery_cmd_lock);
458e48354ceSNicholas Bellinger 				spin_unlock(&sess->cr_i_lock);
459e48354ceSNicholas Bellinger 
460e48354ceSNicholas Bellinger 				*cr_ptr = cr;
461e48354ceSNicholas Bellinger 				*cmd_ptr = cmd;
462e48354ceSNicholas Bellinger 				return -2;
463e48354ceSNicholas Bellinger 			}
464e48354ceSNicholas Bellinger 		}
465e48354ceSNicholas Bellinger 		spin_unlock(&cr->conn_recovery_cmd_lock);
466e48354ceSNicholas Bellinger 	}
467e48354ceSNicholas Bellinger 	spin_unlock(&sess->cr_i_lock);
468e48354ceSNicholas Bellinger 	/*
469e48354ceSNicholas Bellinger 	 * Scan through the active connection recovery list's command list.
470e48354ceSNicholas Bellinger 	 * If init_task_tag matches the command is ready to be reassigned.
471e48354ceSNicholas Bellinger 	 */
472e48354ceSNicholas Bellinger 	spin_lock(&sess->cr_a_lock);
473e48354ceSNicholas Bellinger 	list_for_each_entry(cr, &sess->cr_active_list, cr_list) {
474e48354ceSNicholas Bellinger 		spin_lock(&cr->conn_recovery_cmd_lock);
4752fbb471eSAndy Grover 		list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {
476e48354ceSNicholas Bellinger 			if (cmd->init_task_tag == init_task_tag) {
477e48354ceSNicholas Bellinger 				spin_unlock(&cr->conn_recovery_cmd_lock);
478e48354ceSNicholas Bellinger 				spin_unlock(&sess->cr_a_lock);
479e48354ceSNicholas Bellinger 
480e48354ceSNicholas Bellinger 				*cr_ptr = cr;
481e48354ceSNicholas Bellinger 				*cmd_ptr = cmd;
482e48354ceSNicholas Bellinger 				return 0;
483e48354ceSNicholas Bellinger 			}
484e48354ceSNicholas Bellinger 		}
485e48354ceSNicholas Bellinger 		spin_unlock(&cr->conn_recovery_cmd_lock);
486e48354ceSNicholas Bellinger 	}
487e48354ceSNicholas Bellinger 	spin_unlock(&sess->cr_a_lock);
488e48354ceSNicholas Bellinger 
489e48354ceSNicholas Bellinger 	return -1;
490e48354ceSNicholas Bellinger }
491e48354ceSNicholas Bellinger 
492e48354ceSNicholas Bellinger void iscsit_add_cmd_to_immediate_queue(
493e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd,
494e48354ceSNicholas Bellinger 	struct iscsi_conn *conn,
495e48354ceSNicholas Bellinger 	u8 state)
496e48354ceSNicholas Bellinger {
497e48354ceSNicholas Bellinger 	struct iscsi_queue_req *qr;
498e48354ceSNicholas Bellinger 
499e48354ceSNicholas Bellinger 	qr = kmem_cache_zalloc(lio_qr_cache, GFP_ATOMIC);
500e48354ceSNicholas Bellinger 	if (!qr) {
501e48354ceSNicholas Bellinger 		pr_err("Unable to allocate memory for"
502e48354ceSNicholas Bellinger 				" struct iscsi_queue_req\n");
503e48354ceSNicholas Bellinger 		return;
504e48354ceSNicholas Bellinger 	}
505e48354ceSNicholas Bellinger 	INIT_LIST_HEAD(&qr->qr_list);
506e48354ceSNicholas Bellinger 	qr->cmd = cmd;
507e48354ceSNicholas Bellinger 	qr->state = state;
508e48354ceSNicholas Bellinger 
509e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->immed_queue_lock);
510e48354ceSNicholas Bellinger 	list_add_tail(&qr->qr_list, &conn->immed_queue_list);
511e48354ceSNicholas Bellinger 	atomic_inc(&cmd->immed_queue_count);
512e48354ceSNicholas Bellinger 	atomic_set(&conn->check_immediate_queue, 1);
513e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->immed_queue_lock);
514e48354ceSNicholas Bellinger 
515d5627acbSRoland Dreier 	wake_up(&conn->queues_wq);
516e48354ceSNicholas Bellinger }
517e48354ceSNicholas Bellinger 
518e48354ceSNicholas Bellinger struct iscsi_queue_req *iscsit_get_cmd_from_immediate_queue(struct iscsi_conn *conn)
519e48354ceSNicholas Bellinger {
520e48354ceSNicholas Bellinger 	struct iscsi_queue_req *qr;
521e48354ceSNicholas Bellinger 
522e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->immed_queue_lock);
523e48354ceSNicholas Bellinger 	if (list_empty(&conn->immed_queue_list)) {
524e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->immed_queue_lock);
525e48354ceSNicholas Bellinger 		return NULL;
526e48354ceSNicholas Bellinger 	}
5271f981de5SRoland Dreier 	qr = list_first_entry(&conn->immed_queue_list,
5281f981de5SRoland Dreier 			      struct iscsi_queue_req, qr_list);
529e48354ceSNicholas Bellinger 
530e48354ceSNicholas Bellinger 	list_del(&qr->qr_list);
531e48354ceSNicholas Bellinger 	if (qr->cmd)
532e48354ceSNicholas Bellinger 		atomic_dec(&qr->cmd->immed_queue_count);
533e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->immed_queue_lock);
534e48354ceSNicholas Bellinger 
535e48354ceSNicholas Bellinger 	return qr;
536e48354ceSNicholas Bellinger }
537e48354ceSNicholas Bellinger 
538e48354ceSNicholas Bellinger static void iscsit_remove_cmd_from_immediate_queue(
539e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd,
540e48354ceSNicholas Bellinger 	struct iscsi_conn *conn)
541e48354ceSNicholas Bellinger {
542e48354ceSNicholas Bellinger 	struct iscsi_queue_req *qr, *qr_tmp;
543e48354ceSNicholas Bellinger 
544e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->immed_queue_lock);
545e48354ceSNicholas Bellinger 	if (!atomic_read(&cmd->immed_queue_count)) {
546e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->immed_queue_lock);
547e48354ceSNicholas Bellinger 		return;
548e48354ceSNicholas Bellinger 	}
549e48354ceSNicholas Bellinger 
550e48354ceSNicholas Bellinger 	list_for_each_entry_safe(qr, qr_tmp, &conn->immed_queue_list, qr_list) {
551e48354ceSNicholas Bellinger 		if (qr->cmd != cmd)
552e48354ceSNicholas Bellinger 			continue;
553e48354ceSNicholas Bellinger 
554e48354ceSNicholas Bellinger 		atomic_dec(&qr->cmd->immed_queue_count);
555e48354ceSNicholas Bellinger 		list_del(&qr->qr_list);
556e48354ceSNicholas Bellinger 		kmem_cache_free(lio_qr_cache, qr);
557e48354ceSNicholas Bellinger 	}
558e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->immed_queue_lock);
559e48354ceSNicholas Bellinger 
560e48354ceSNicholas Bellinger 	if (atomic_read(&cmd->immed_queue_count)) {
561e48354ceSNicholas Bellinger 		pr_err("ITT: 0x%08x immed_queue_count: %d\n",
562e48354ceSNicholas Bellinger 			cmd->init_task_tag,
563e48354ceSNicholas Bellinger 			atomic_read(&cmd->immed_queue_count));
564e48354ceSNicholas Bellinger 	}
565e48354ceSNicholas Bellinger }
566e48354ceSNicholas Bellinger 
567e48354ceSNicholas Bellinger void iscsit_add_cmd_to_response_queue(
568e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd,
569e48354ceSNicholas Bellinger 	struct iscsi_conn *conn,
570e48354ceSNicholas Bellinger 	u8 state)
571e48354ceSNicholas Bellinger {
572e48354ceSNicholas Bellinger 	struct iscsi_queue_req *qr;
573e48354ceSNicholas Bellinger 
574e48354ceSNicholas Bellinger 	qr = kmem_cache_zalloc(lio_qr_cache, GFP_ATOMIC);
575e48354ceSNicholas Bellinger 	if (!qr) {
576e48354ceSNicholas Bellinger 		pr_err("Unable to allocate memory for"
577e48354ceSNicholas Bellinger 			" struct iscsi_queue_req\n");
578e48354ceSNicholas Bellinger 		return;
579e48354ceSNicholas Bellinger 	}
580e48354ceSNicholas Bellinger 	INIT_LIST_HEAD(&qr->qr_list);
581e48354ceSNicholas Bellinger 	qr->cmd = cmd;
582e48354ceSNicholas Bellinger 	qr->state = state;
583e48354ceSNicholas Bellinger 
584e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->response_queue_lock);
585e48354ceSNicholas Bellinger 	list_add_tail(&qr->qr_list, &conn->response_queue_list);
586e48354ceSNicholas Bellinger 	atomic_inc(&cmd->response_queue_count);
587e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->response_queue_lock);
588e48354ceSNicholas Bellinger 
589d5627acbSRoland Dreier 	wake_up(&conn->queues_wq);
590e48354ceSNicholas Bellinger }
591e48354ceSNicholas Bellinger 
592e48354ceSNicholas Bellinger struct iscsi_queue_req *iscsit_get_cmd_from_response_queue(struct iscsi_conn *conn)
593e48354ceSNicholas Bellinger {
594e48354ceSNicholas Bellinger 	struct iscsi_queue_req *qr;
595e48354ceSNicholas Bellinger 
596e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->response_queue_lock);
597e48354ceSNicholas Bellinger 	if (list_empty(&conn->response_queue_list)) {
598e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->response_queue_lock);
599e48354ceSNicholas Bellinger 		return NULL;
600e48354ceSNicholas Bellinger 	}
601e48354ceSNicholas Bellinger 
6021f981de5SRoland Dreier 	qr = list_first_entry(&conn->response_queue_list,
6031f981de5SRoland Dreier 			      struct iscsi_queue_req, qr_list);
604e48354ceSNicholas Bellinger 
605e48354ceSNicholas Bellinger 	list_del(&qr->qr_list);
606e48354ceSNicholas Bellinger 	if (qr->cmd)
607e48354ceSNicholas Bellinger 		atomic_dec(&qr->cmd->response_queue_count);
608e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->response_queue_lock);
609e48354ceSNicholas Bellinger 
610e48354ceSNicholas Bellinger 	return qr;
611e48354ceSNicholas Bellinger }
612e48354ceSNicholas Bellinger 
613e48354ceSNicholas Bellinger static void iscsit_remove_cmd_from_response_queue(
614e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd,
615e48354ceSNicholas Bellinger 	struct iscsi_conn *conn)
616e48354ceSNicholas Bellinger {
617e48354ceSNicholas Bellinger 	struct iscsi_queue_req *qr, *qr_tmp;
618e48354ceSNicholas Bellinger 
619e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->response_queue_lock);
620e48354ceSNicholas Bellinger 	if (!atomic_read(&cmd->response_queue_count)) {
621e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->response_queue_lock);
622e48354ceSNicholas Bellinger 		return;
623e48354ceSNicholas Bellinger 	}
624e48354ceSNicholas Bellinger 
625e48354ceSNicholas Bellinger 	list_for_each_entry_safe(qr, qr_tmp, &conn->response_queue_list,
626e48354ceSNicholas Bellinger 				qr_list) {
627e48354ceSNicholas Bellinger 		if (qr->cmd != cmd)
628e48354ceSNicholas Bellinger 			continue;
629e48354ceSNicholas Bellinger 
630e48354ceSNicholas Bellinger 		atomic_dec(&qr->cmd->response_queue_count);
631e48354ceSNicholas Bellinger 		list_del(&qr->qr_list);
632e48354ceSNicholas Bellinger 		kmem_cache_free(lio_qr_cache, qr);
633e48354ceSNicholas Bellinger 	}
634e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->response_queue_lock);
635e48354ceSNicholas Bellinger 
636e48354ceSNicholas Bellinger 	if (atomic_read(&cmd->response_queue_count)) {
637e48354ceSNicholas Bellinger 		pr_err("ITT: 0x%08x response_queue_count: %d\n",
638e48354ceSNicholas Bellinger 			cmd->init_task_tag,
639e48354ceSNicholas Bellinger 			atomic_read(&cmd->response_queue_count));
640e48354ceSNicholas Bellinger 	}
641e48354ceSNicholas Bellinger }
642e48354ceSNicholas Bellinger 
643d5627acbSRoland Dreier bool iscsit_conn_all_queues_empty(struct iscsi_conn *conn)
644d5627acbSRoland Dreier {
645d5627acbSRoland Dreier 	bool empty;
646d5627acbSRoland Dreier 
647d5627acbSRoland Dreier 	spin_lock_bh(&conn->immed_queue_lock);
648d5627acbSRoland Dreier 	empty = list_empty(&conn->immed_queue_list);
649d5627acbSRoland Dreier 	spin_unlock_bh(&conn->immed_queue_lock);
650d5627acbSRoland Dreier 
651d5627acbSRoland Dreier 	if (!empty)
652d5627acbSRoland Dreier 		return empty;
653d5627acbSRoland Dreier 
654d5627acbSRoland Dreier 	spin_lock_bh(&conn->response_queue_lock);
655d5627acbSRoland Dreier 	empty = list_empty(&conn->response_queue_list);
656d5627acbSRoland Dreier 	spin_unlock_bh(&conn->response_queue_lock);
657d5627acbSRoland Dreier 
658d5627acbSRoland Dreier 	return empty;
659d5627acbSRoland Dreier }
660d5627acbSRoland Dreier 
661e48354ceSNicholas Bellinger void iscsit_free_queue_reqs_for_conn(struct iscsi_conn *conn)
662e48354ceSNicholas Bellinger {
663e48354ceSNicholas Bellinger 	struct iscsi_queue_req *qr, *qr_tmp;
664e48354ceSNicholas Bellinger 
665e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->immed_queue_lock);
666e48354ceSNicholas Bellinger 	list_for_each_entry_safe(qr, qr_tmp, &conn->immed_queue_list, qr_list) {
667e48354ceSNicholas Bellinger 		list_del(&qr->qr_list);
668e48354ceSNicholas Bellinger 		if (qr->cmd)
669e48354ceSNicholas Bellinger 			atomic_dec(&qr->cmd->immed_queue_count);
670e48354ceSNicholas Bellinger 
671e48354ceSNicholas Bellinger 		kmem_cache_free(lio_qr_cache, qr);
672e48354ceSNicholas Bellinger 	}
673e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->immed_queue_lock);
674e48354ceSNicholas Bellinger 
675e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->response_queue_lock);
676e48354ceSNicholas Bellinger 	list_for_each_entry_safe(qr, qr_tmp, &conn->response_queue_list,
677e48354ceSNicholas Bellinger 			qr_list) {
678e48354ceSNicholas Bellinger 		list_del(&qr->qr_list);
679e48354ceSNicholas Bellinger 		if (qr->cmd)
680e48354ceSNicholas Bellinger 			atomic_dec(&qr->cmd->response_queue_count);
681e48354ceSNicholas Bellinger 
682e48354ceSNicholas Bellinger 		kmem_cache_free(lio_qr_cache, qr);
683e48354ceSNicholas Bellinger 	}
684e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->response_queue_lock);
685e48354ceSNicholas Bellinger }
686e48354ceSNicholas Bellinger 
687e48354ceSNicholas Bellinger void iscsit_release_cmd(struct iscsi_cmd *cmd)
688e48354ceSNicholas Bellinger {
689988e3a85SNicholas Bellinger 	struct iscsi_session *sess;
690988e3a85SNicholas Bellinger 	struct se_cmd *se_cmd = &cmd->se_cmd;
691988e3a85SNicholas Bellinger 
692988e3a85SNicholas Bellinger 	if (cmd->conn)
693988e3a85SNicholas Bellinger 		sess = cmd->conn->sess;
694988e3a85SNicholas Bellinger 	else
695988e3a85SNicholas Bellinger 		sess = cmd->sess;
696988e3a85SNicholas Bellinger 
697988e3a85SNicholas Bellinger 	BUG_ON(!sess || !sess->se_sess);
698988e3a85SNicholas Bellinger 
699e48354ceSNicholas Bellinger 	kfree(cmd->buf_ptr);
700e48354ceSNicholas Bellinger 	kfree(cmd->pdu_list);
701e48354ceSNicholas Bellinger 	kfree(cmd->seq_list);
702e48354ceSNicholas Bellinger 	kfree(cmd->tmr_req);
703e48354ceSNicholas Bellinger 	kfree(cmd->iov_data);
7049864ca9dSNicholas Bellinger 	kfree(cmd->text_in_ptr);
705e48354ceSNicholas Bellinger 
706988e3a85SNicholas Bellinger 	percpu_ida_free(&sess->se_sess->sess_tag_pool, se_cmd->map_tag);
707e48354ceSNicholas Bellinger }
708d703ce2fSNicholas Bellinger EXPORT_SYMBOL(iscsit_release_cmd);
709e48354ceSNicholas Bellinger 
710131e6abcSNicholas Bellinger void __iscsit_free_cmd(struct iscsi_cmd *cmd, bool scsi_cmd,
711aafc9d15SNicholas Bellinger 		       bool check_queues)
712d270190aSNicholas Bellinger {
713aafc9d15SNicholas Bellinger 	struct iscsi_conn *conn = cmd->conn;
714aafc9d15SNicholas Bellinger 
715aafc9d15SNicholas Bellinger 	if (scsi_cmd) {
716aafc9d15SNicholas Bellinger 		if (cmd->data_direction == DMA_TO_DEVICE) {
717aafc9d15SNicholas Bellinger 			iscsit_stop_dataout_timer(cmd);
718aafc9d15SNicholas Bellinger 			iscsit_free_r2ts_from_list(cmd);
719aafc9d15SNicholas Bellinger 		}
720aafc9d15SNicholas Bellinger 		if (cmd->data_direction == DMA_FROM_DEVICE)
721aafc9d15SNicholas Bellinger 			iscsit_free_all_datain_reqs(cmd);
722aafc9d15SNicholas Bellinger 	}
723aafc9d15SNicholas Bellinger 
724aafc9d15SNicholas Bellinger 	if (conn && check_queues) {
725aafc9d15SNicholas Bellinger 		iscsit_remove_cmd_from_immediate_queue(cmd, conn);
726aafc9d15SNicholas Bellinger 		iscsit_remove_cmd_from_response_queue(cmd, conn);
727aafc9d15SNicholas Bellinger 	}
728aafc9d15SNicholas Bellinger }
729aafc9d15SNicholas Bellinger 
730aafc9d15SNicholas Bellinger void iscsit_free_cmd(struct iscsi_cmd *cmd, bool shutdown)
731aafc9d15SNicholas Bellinger {
732aafc9d15SNicholas Bellinger 	struct se_cmd *se_cmd = NULL;
733aafc9d15SNicholas Bellinger 	int rc;
734d270190aSNicholas Bellinger 	/*
73520879696SMasanari Iida 	 * Determine if a struct se_cmd is associated with
736d270190aSNicholas Bellinger 	 * this struct iscsi_cmd.
737d270190aSNicholas Bellinger 	 */
738d270190aSNicholas Bellinger 	switch (cmd->iscsi_opcode) {
739d270190aSNicholas Bellinger 	case ISCSI_OP_SCSI_CMD:
740aafc9d15SNicholas Bellinger 		se_cmd = &cmd->se_cmd;
741aafc9d15SNicholas Bellinger 		__iscsit_free_cmd(cmd, true, shutdown);
742cdb72665SNicholas Bellinger 		/*
743cdb72665SNicholas Bellinger 		 * Fallthrough
744cdb72665SNicholas Bellinger 		 */
745d270190aSNicholas Bellinger 	case ISCSI_OP_SCSI_TMFUNC:
746e255a285SNicholas Bellinger 		rc = transport_generic_free_cmd(&cmd->se_cmd, shutdown);
747aafc9d15SNicholas Bellinger 		if (!rc && shutdown && se_cmd && se_cmd->se_sess) {
748aafc9d15SNicholas Bellinger 			__iscsit_free_cmd(cmd, true, shutdown);
749aafc9d15SNicholas Bellinger 			target_put_sess_cmd(se_cmd->se_sess, se_cmd);
750aafc9d15SNicholas Bellinger 		}
751d270190aSNicholas Bellinger 		break;
752c1ce4bd5SNicholas Bellinger 	case ISCSI_OP_REJECT:
753c1ce4bd5SNicholas Bellinger 		/*
754c1ce4bd5SNicholas Bellinger 		 * Handle special case for REJECT when iscsi_add_reject*() has
755c1ce4bd5SNicholas Bellinger 		 * overwritten the original iscsi_opcode assignment, and the
756c1ce4bd5SNicholas Bellinger 		 * associated cmd->se_cmd needs to be released.
757c1ce4bd5SNicholas Bellinger 		 */
758c1ce4bd5SNicholas Bellinger 		if (cmd->se_cmd.se_tfo != NULL) {
759aafc9d15SNicholas Bellinger 			se_cmd = &cmd->se_cmd;
760aafc9d15SNicholas Bellinger 			__iscsit_free_cmd(cmd, true, shutdown);
761aafc9d15SNicholas Bellinger 
762e255a285SNicholas Bellinger 			rc = transport_generic_free_cmd(&cmd->se_cmd, shutdown);
763aafc9d15SNicholas Bellinger 			if (!rc && shutdown && se_cmd->se_sess) {
764aafc9d15SNicholas Bellinger 				__iscsit_free_cmd(cmd, true, shutdown);
765aafc9d15SNicholas Bellinger 				target_put_sess_cmd(se_cmd->se_sess, se_cmd);
766aafc9d15SNicholas Bellinger 			}
767c1ce4bd5SNicholas Bellinger 			break;
768c1ce4bd5SNicholas Bellinger 		}
769c1ce4bd5SNicholas Bellinger 		/* Fall-through */
770d270190aSNicholas Bellinger 	default:
771aafc9d15SNicholas Bellinger 		__iscsit_free_cmd(cmd, false, shutdown);
772d703ce2fSNicholas Bellinger 		iscsit_release_cmd(cmd);
773d270190aSNicholas Bellinger 		break;
774d270190aSNicholas Bellinger 	}
775d270190aSNicholas Bellinger }
776d270190aSNicholas Bellinger 
777e48354ceSNicholas Bellinger int iscsit_check_session_usage_count(struct iscsi_session *sess)
778e48354ceSNicholas Bellinger {
779e48354ceSNicholas Bellinger 	spin_lock_bh(&sess->session_usage_lock);
780e48354ceSNicholas Bellinger 	if (sess->session_usage_count != 0) {
781e48354ceSNicholas Bellinger 		sess->session_waiting_on_uc = 1;
782e48354ceSNicholas Bellinger 		spin_unlock_bh(&sess->session_usage_lock);
783e48354ceSNicholas Bellinger 		if (in_interrupt())
784e48354ceSNicholas Bellinger 			return 2;
785e48354ceSNicholas Bellinger 
786e48354ceSNicholas Bellinger 		wait_for_completion(&sess->session_waiting_on_uc_comp);
787e48354ceSNicholas Bellinger 		return 1;
788e48354ceSNicholas Bellinger 	}
789e48354ceSNicholas Bellinger 	spin_unlock_bh(&sess->session_usage_lock);
790e48354ceSNicholas Bellinger 
791e48354ceSNicholas Bellinger 	return 0;
792e48354ceSNicholas Bellinger }
793e48354ceSNicholas Bellinger 
794e48354ceSNicholas Bellinger void iscsit_dec_session_usage_count(struct iscsi_session *sess)
795e48354ceSNicholas Bellinger {
796e48354ceSNicholas Bellinger 	spin_lock_bh(&sess->session_usage_lock);
797e48354ceSNicholas Bellinger 	sess->session_usage_count--;
798e48354ceSNicholas Bellinger 
799e48354ceSNicholas Bellinger 	if (!sess->session_usage_count && sess->session_waiting_on_uc)
800e48354ceSNicholas Bellinger 		complete(&sess->session_waiting_on_uc_comp);
801e48354ceSNicholas Bellinger 
802e48354ceSNicholas Bellinger 	spin_unlock_bh(&sess->session_usage_lock);
803e48354ceSNicholas Bellinger }
804e48354ceSNicholas Bellinger 
805e48354ceSNicholas Bellinger void iscsit_inc_session_usage_count(struct iscsi_session *sess)
806e48354ceSNicholas Bellinger {
807e48354ceSNicholas Bellinger 	spin_lock_bh(&sess->session_usage_lock);
808e48354ceSNicholas Bellinger 	sess->session_usage_count++;
809e48354ceSNicholas Bellinger 	spin_unlock_bh(&sess->session_usage_lock);
810e48354ceSNicholas Bellinger }
811e48354ceSNicholas Bellinger 
812e48354ceSNicholas Bellinger /*
813e48354ceSNicholas Bellinger  *	Setup conn->if_marker and conn->of_marker values based upon
814e48354ceSNicholas Bellinger  *	the initial marker-less interval. (see iSCSI v19 A.2)
815e48354ceSNicholas Bellinger  */
816e48354ceSNicholas Bellinger int iscsit_set_sync_and_steering_values(struct iscsi_conn *conn)
817e48354ceSNicholas Bellinger {
818e48354ceSNicholas Bellinger 	int login_ifmarker_count = 0, login_ofmarker_count = 0, next_marker = 0;
819e48354ceSNicholas Bellinger 	/*
820e48354ceSNicholas Bellinger 	 * IFMarkInt and OFMarkInt are negotiated as 32-bit words.
821e48354ceSNicholas Bellinger 	 */
822e48354ceSNicholas Bellinger 	u32 IFMarkInt = (conn->conn_ops->IFMarkInt * 4);
823e48354ceSNicholas Bellinger 	u32 OFMarkInt = (conn->conn_ops->OFMarkInt * 4);
824e48354ceSNicholas Bellinger 
825e48354ceSNicholas Bellinger 	if (conn->conn_ops->OFMarker) {
826e48354ceSNicholas Bellinger 		/*
827e48354ceSNicholas Bellinger 		 * Account for the first Login Command received not
828e48354ceSNicholas Bellinger 		 * via iscsi_recv_msg().
829e48354ceSNicholas Bellinger 		 */
830e48354ceSNicholas Bellinger 		conn->of_marker += ISCSI_HDR_LEN;
831e48354ceSNicholas Bellinger 		if (conn->of_marker <= OFMarkInt) {
832e48354ceSNicholas Bellinger 			conn->of_marker = (OFMarkInt - conn->of_marker);
833e48354ceSNicholas Bellinger 		} else {
834e48354ceSNicholas Bellinger 			login_ofmarker_count = (conn->of_marker / OFMarkInt);
835e48354ceSNicholas Bellinger 			next_marker = (OFMarkInt * (login_ofmarker_count + 1)) +
836e48354ceSNicholas Bellinger 					(login_ofmarker_count * MARKER_SIZE);
837e48354ceSNicholas Bellinger 			conn->of_marker = (next_marker - conn->of_marker);
838e48354ceSNicholas Bellinger 		}
839e48354ceSNicholas Bellinger 		conn->of_marker_offset = 0;
840e48354ceSNicholas Bellinger 		pr_debug("Setting OFMarker value to %u based on Initial"
841e48354ceSNicholas Bellinger 			" Markerless Interval.\n", conn->of_marker);
842e48354ceSNicholas Bellinger 	}
843e48354ceSNicholas Bellinger 
844e48354ceSNicholas Bellinger 	if (conn->conn_ops->IFMarker) {
845e48354ceSNicholas Bellinger 		if (conn->if_marker <= IFMarkInt) {
846e48354ceSNicholas Bellinger 			conn->if_marker = (IFMarkInt - conn->if_marker);
847e48354ceSNicholas Bellinger 		} else {
848e48354ceSNicholas Bellinger 			login_ifmarker_count = (conn->if_marker / IFMarkInt);
849e48354ceSNicholas Bellinger 			next_marker = (IFMarkInt * (login_ifmarker_count + 1)) +
850e48354ceSNicholas Bellinger 					(login_ifmarker_count * MARKER_SIZE);
851e48354ceSNicholas Bellinger 			conn->if_marker = (next_marker - conn->if_marker);
852e48354ceSNicholas Bellinger 		}
853e48354ceSNicholas Bellinger 		pr_debug("Setting IFMarker value to %u based on Initial"
854e48354ceSNicholas Bellinger 			" Markerless Interval.\n", conn->if_marker);
855e48354ceSNicholas Bellinger 	}
856e48354ceSNicholas Bellinger 
857e48354ceSNicholas Bellinger 	return 0;
858e48354ceSNicholas Bellinger }
859e48354ceSNicholas Bellinger 
860e48354ceSNicholas Bellinger struct iscsi_conn *iscsit_get_conn_from_cid(struct iscsi_session *sess, u16 cid)
861e48354ceSNicholas Bellinger {
862e48354ceSNicholas Bellinger 	struct iscsi_conn *conn;
863e48354ceSNicholas Bellinger 
864e48354ceSNicholas Bellinger 	spin_lock_bh(&sess->conn_lock);
865e48354ceSNicholas Bellinger 	list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
866e48354ceSNicholas Bellinger 		if ((conn->cid == cid) &&
867e48354ceSNicholas Bellinger 		    (conn->conn_state == TARG_CONN_STATE_LOGGED_IN)) {
868e48354ceSNicholas Bellinger 			iscsit_inc_conn_usage_count(conn);
869e48354ceSNicholas Bellinger 			spin_unlock_bh(&sess->conn_lock);
870e48354ceSNicholas Bellinger 			return conn;
871e48354ceSNicholas Bellinger 		}
872e48354ceSNicholas Bellinger 	}
873e48354ceSNicholas Bellinger 	spin_unlock_bh(&sess->conn_lock);
874e48354ceSNicholas Bellinger 
875e48354ceSNicholas Bellinger 	return NULL;
876e48354ceSNicholas Bellinger }
877e48354ceSNicholas Bellinger 
878e48354ceSNicholas Bellinger struct iscsi_conn *iscsit_get_conn_from_cid_rcfr(struct iscsi_session *sess, u16 cid)
879e48354ceSNicholas Bellinger {
880e48354ceSNicholas Bellinger 	struct iscsi_conn *conn;
881e48354ceSNicholas Bellinger 
882e48354ceSNicholas Bellinger 	spin_lock_bh(&sess->conn_lock);
883e48354ceSNicholas Bellinger 	list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
884e48354ceSNicholas Bellinger 		if (conn->cid == cid) {
885e48354ceSNicholas Bellinger 			iscsit_inc_conn_usage_count(conn);
886e48354ceSNicholas Bellinger 			spin_lock(&conn->state_lock);
887e48354ceSNicholas Bellinger 			atomic_set(&conn->connection_wait_rcfr, 1);
888e48354ceSNicholas Bellinger 			spin_unlock(&conn->state_lock);
889e48354ceSNicholas Bellinger 			spin_unlock_bh(&sess->conn_lock);
890e48354ceSNicholas Bellinger 			return conn;
891e48354ceSNicholas Bellinger 		}
892e48354ceSNicholas Bellinger 	}
893e48354ceSNicholas Bellinger 	spin_unlock_bh(&sess->conn_lock);
894e48354ceSNicholas Bellinger 
895e48354ceSNicholas Bellinger 	return NULL;
896e48354ceSNicholas Bellinger }
897e48354ceSNicholas Bellinger 
898e48354ceSNicholas Bellinger void iscsit_check_conn_usage_count(struct iscsi_conn *conn)
899e48354ceSNicholas Bellinger {
900e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->conn_usage_lock);
901e48354ceSNicholas Bellinger 	if (conn->conn_usage_count != 0) {
902e48354ceSNicholas Bellinger 		conn->conn_waiting_on_uc = 1;
903e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->conn_usage_lock);
904e48354ceSNicholas Bellinger 
905e48354ceSNicholas Bellinger 		wait_for_completion(&conn->conn_waiting_on_uc_comp);
906e48354ceSNicholas Bellinger 		return;
907e48354ceSNicholas Bellinger 	}
908e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->conn_usage_lock);
909e48354ceSNicholas Bellinger }
910e48354ceSNicholas Bellinger 
911e48354ceSNicholas Bellinger void iscsit_dec_conn_usage_count(struct iscsi_conn *conn)
912e48354ceSNicholas Bellinger {
913e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->conn_usage_lock);
914e48354ceSNicholas Bellinger 	conn->conn_usage_count--;
915e48354ceSNicholas Bellinger 
916e48354ceSNicholas Bellinger 	if (!conn->conn_usage_count && conn->conn_waiting_on_uc)
917e48354ceSNicholas Bellinger 		complete(&conn->conn_waiting_on_uc_comp);
918e48354ceSNicholas Bellinger 
919e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->conn_usage_lock);
920e48354ceSNicholas Bellinger }
921e48354ceSNicholas Bellinger 
922e48354ceSNicholas Bellinger void iscsit_inc_conn_usage_count(struct iscsi_conn *conn)
923e48354ceSNicholas Bellinger {
924e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->conn_usage_lock);
925e48354ceSNicholas Bellinger 	conn->conn_usage_count++;
926e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->conn_usage_lock);
927e48354ceSNicholas Bellinger }
928e48354ceSNicholas Bellinger 
929e48354ceSNicholas Bellinger static int iscsit_add_nopin(struct iscsi_conn *conn, int want_response)
930e48354ceSNicholas Bellinger {
931e48354ceSNicholas Bellinger 	u8 state;
932e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd;
933e48354ceSNicholas Bellinger 
934676687c6SNicholas Bellinger 	cmd = iscsit_allocate_cmd(conn, TASK_RUNNING);
935e48354ceSNicholas Bellinger 	if (!cmd)
936e48354ceSNicholas Bellinger 		return -1;
937e48354ceSNicholas Bellinger 
938e48354ceSNicholas Bellinger 	cmd->iscsi_opcode = ISCSI_OP_NOOP_IN;
939e48354ceSNicholas Bellinger 	state = (want_response) ? ISTATE_SEND_NOPIN_WANT_RESPONSE :
940e48354ceSNicholas Bellinger 				ISTATE_SEND_NOPIN_NO_RESPONSE;
94166c7db68SChristoph Hellwig 	cmd->init_task_tag = RESERVED_ITT;
942e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->sess->ttt_lock);
943e48354ceSNicholas Bellinger 	cmd->targ_xfer_tag = (want_response) ? conn->sess->targ_xfer_tag++ :
944e48354ceSNicholas Bellinger 			0xFFFFFFFF;
945e48354ceSNicholas Bellinger 	if (want_response && (cmd->targ_xfer_tag == 0xFFFFFFFF))
946e48354ceSNicholas Bellinger 		cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
947e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->sess->ttt_lock);
948e48354ceSNicholas Bellinger 
949e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->cmd_lock);
9502fbb471eSAndy Grover 	list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
951e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->cmd_lock);
952e48354ceSNicholas Bellinger 
953e48354ceSNicholas Bellinger 	if (want_response)
954e48354ceSNicholas Bellinger 		iscsit_start_nopin_response_timer(conn);
955e48354ceSNicholas Bellinger 	iscsit_add_cmd_to_immediate_queue(cmd, conn, state);
956e48354ceSNicholas Bellinger 
957e48354ceSNicholas Bellinger 	return 0;
958e48354ceSNicholas Bellinger }
959e48354ceSNicholas Bellinger 
960e48354ceSNicholas Bellinger static void iscsit_handle_nopin_response_timeout(unsigned long data)
961e48354ceSNicholas Bellinger {
962e48354ceSNicholas Bellinger 	struct iscsi_conn *conn = (struct iscsi_conn *) data;
963e48354ceSNicholas Bellinger 
964e48354ceSNicholas Bellinger 	iscsit_inc_conn_usage_count(conn);
965e48354ceSNicholas Bellinger 
966e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->nopin_timer_lock);
967e48354ceSNicholas Bellinger 	if (conn->nopin_response_timer_flags & ISCSI_TF_STOP) {
968e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->nopin_timer_lock);
969e48354ceSNicholas Bellinger 		iscsit_dec_conn_usage_count(conn);
970e48354ceSNicholas Bellinger 		return;
971e48354ceSNicholas Bellinger 	}
972e48354ceSNicholas Bellinger 
973e48354ceSNicholas Bellinger 	pr_debug("Did not receive response to NOPIN on CID: %hu on"
974e48354ceSNicholas Bellinger 		" SID: %u, failing connection.\n", conn->cid,
975e48354ceSNicholas Bellinger 			conn->sess->sid);
976e48354ceSNicholas Bellinger 	conn->nopin_response_timer_flags &= ~ISCSI_TF_RUNNING;
977e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->nopin_timer_lock);
978e48354ceSNicholas Bellinger 
979e48354ceSNicholas Bellinger 	{
980e48354ceSNicholas Bellinger 	struct iscsi_portal_group *tpg = conn->sess->tpg;
981e48354ceSNicholas Bellinger 	struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
982e48354ceSNicholas Bellinger 
983e48354ceSNicholas Bellinger 	if (tiqn) {
984e48354ceSNicholas Bellinger 		spin_lock_bh(&tiqn->sess_err_stats.lock);
985e48354ceSNicholas Bellinger 		strcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
9868359cf43SJörn Engel 				conn->sess->sess_ops->InitiatorName);
987e48354ceSNicholas Bellinger 		tiqn->sess_err_stats.last_sess_failure_type =
988e48354ceSNicholas Bellinger 				ISCSI_SESS_ERR_CXN_TIMEOUT;
989e48354ceSNicholas Bellinger 		tiqn->sess_err_stats.cxn_timeout_errors++;
99004f3b31bSNicholas Bellinger 		atomic_long_inc(&conn->sess->conn_timeout_errors);
991e48354ceSNicholas Bellinger 		spin_unlock_bh(&tiqn->sess_err_stats.lock);
992e48354ceSNicholas Bellinger 	}
993e48354ceSNicholas Bellinger 	}
994e48354ceSNicholas Bellinger 
995e48354ceSNicholas Bellinger 	iscsit_cause_connection_reinstatement(conn, 0);
996e48354ceSNicholas Bellinger 	iscsit_dec_conn_usage_count(conn);
997e48354ceSNicholas Bellinger }
998e48354ceSNicholas Bellinger 
999e48354ceSNicholas Bellinger void iscsit_mod_nopin_response_timer(struct iscsi_conn *conn)
1000e48354ceSNicholas Bellinger {
1001e48354ceSNicholas Bellinger 	struct iscsi_session *sess = conn->sess;
1002e48354ceSNicholas Bellinger 	struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1003e48354ceSNicholas Bellinger 
1004e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->nopin_timer_lock);
1005e48354ceSNicholas Bellinger 	if (!(conn->nopin_response_timer_flags & ISCSI_TF_RUNNING)) {
1006e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->nopin_timer_lock);
1007e48354ceSNicholas Bellinger 		return;
1008e48354ceSNicholas Bellinger 	}
1009e48354ceSNicholas Bellinger 
1010e48354ceSNicholas Bellinger 	mod_timer(&conn->nopin_response_timer,
1011e48354ceSNicholas Bellinger 		(get_jiffies_64() + na->nopin_response_timeout * HZ));
1012e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->nopin_timer_lock);
1013e48354ceSNicholas Bellinger }
1014e48354ceSNicholas Bellinger 
1015e48354ceSNicholas Bellinger /*
1016e48354ceSNicholas Bellinger  *	Called with conn->nopin_timer_lock held.
1017e48354ceSNicholas Bellinger  */
1018e48354ceSNicholas Bellinger void iscsit_start_nopin_response_timer(struct iscsi_conn *conn)
1019e48354ceSNicholas Bellinger {
1020e48354ceSNicholas Bellinger 	struct iscsi_session *sess = conn->sess;
1021e48354ceSNicholas Bellinger 	struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1022e48354ceSNicholas Bellinger 
1023e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->nopin_timer_lock);
1024e48354ceSNicholas Bellinger 	if (conn->nopin_response_timer_flags & ISCSI_TF_RUNNING) {
1025e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->nopin_timer_lock);
1026e48354ceSNicholas Bellinger 		return;
1027e48354ceSNicholas Bellinger 	}
1028e48354ceSNicholas Bellinger 
1029e48354ceSNicholas Bellinger 	init_timer(&conn->nopin_response_timer);
1030e48354ceSNicholas Bellinger 	conn->nopin_response_timer.expires =
1031e48354ceSNicholas Bellinger 		(get_jiffies_64() + na->nopin_response_timeout * HZ);
1032e48354ceSNicholas Bellinger 	conn->nopin_response_timer.data = (unsigned long)conn;
1033e48354ceSNicholas Bellinger 	conn->nopin_response_timer.function = iscsit_handle_nopin_response_timeout;
1034e48354ceSNicholas Bellinger 	conn->nopin_response_timer_flags &= ~ISCSI_TF_STOP;
1035e48354ceSNicholas Bellinger 	conn->nopin_response_timer_flags |= ISCSI_TF_RUNNING;
1036e48354ceSNicholas Bellinger 	add_timer(&conn->nopin_response_timer);
1037e48354ceSNicholas Bellinger 
1038e48354ceSNicholas Bellinger 	pr_debug("Started NOPIN Response Timer on CID: %d to %u"
1039e48354ceSNicholas Bellinger 		" seconds\n", conn->cid, na->nopin_response_timeout);
1040e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->nopin_timer_lock);
1041e48354ceSNicholas Bellinger }
1042e48354ceSNicholas Bellinger 
1043e48354ceSNicholas Bellinger void iscsit_stop_nopin_response_timer(struct iscsi_conn *conn)
1044e48354ceSNicholas Bellinger {
1045e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->nopin_timer_lock);
1046e48354ceSNicholas Bellinger 	if (!(conn->nopin_response_timer_flags & ISCSI_TF_RUNNING)) {
1047e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->nopin_timer_lock);
1048e48354ceSNicholas Bellinger 		return;
1049e48354ceSNicholas Bellinger 	}
1050e48354ceSNicholas Bellinger 	conn->nopin_response_timer_flags |= ISCSI_TF_STOP;
1051e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->nopin_timer_lock);
1052e48354ceSNicholas Bellinger 
1053e48354ceSNicholas Bellinger 	del_timer_sync(&conn->nopin_response_timer);
1054e48354ceSNicholas Bellinger 
1055e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->nopin_timer_lock);
1056e48354ceSNicholas Bellinger 	conn->nopin_response_timer_flags &= ~ISCSI_TF_RUNNING;
1057e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->nopin_timer_lock);
1058e48354ceSNicholas Bellinger }
1059e48354ceSNicholas Bellinger 
1060e48354ceSNicholas Bellinger static void iscsit_handle_nopin_timeout(unsigned long data)
1061e48354ceSNicholas Bellinger {
1062e48354ceSNicholas Bellinger 	struct iscsi_conn *conn = (struct iscsi_conn *) data;
1063e48354ceSNicholas Bellinger 
1064e48354ceSNicholas Bellinger 	iscsit_inc_conn_usage_count(conn);
1065e48354ceSNicholas Bellinger 
1066e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->nopin_timer_lock);
1067e48354ceSNicholas Bellinger 	if (conn->nopin_timer_flags & ISCSI_TF_STOP) {
1068e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->nopin_timer_lock);
1069e48354ceSNicholas Bellinger 		iscsit_dec_conn_usage_count(conn);
1070e48354ceSNicholas Bellinger 		return;
1071e48354ceSNicholas Bellinger 	}
1072e48354ceSNicholas Bellinger 	conn->nopin_timer_flags &= ~ISCSI_TF_RUNNING;
1073e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->nopin_timer_lock);
1074e48354ceSNicholas Bellinger 
1075e48354ceSNicholas Bellinger 	iscsit_add_nopin(conn, 1);
1076e48354ceSNicholas Bellinger 	iscsit_dec_conn_usage_count(conn);
1077e48354ceSNicholas Bellinger }
1078e48354ceSNicholas Bellinger 
1079e48354ceSNicholas Bellinger /*
1080e48354ceSNicholas Bellinger  * Called with conn->nopin_timer_lock held.
1081e48354ceSNicholas Bellinger  */
1082e48354ceSNicholas Bellinger void __iscsit_start_nopin_timer(struct iscsi_conn *conn)
1083e48354ceSNicholas Bellinger {
1084e48354ceSNicholas Bellinger 	struct iscsi_session *sess = conn->sess;
1085e48354ceSNicholas Bellinger 	struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1086e48354ceSNicholas Bellinger 	/*
1087e48354ceSNicholas Bellinger 	* NOPIN timeout is disabled.
1088e48354ceSNicholas Bellinger 	 */
1089e48354ceSNicholas Bellinger 	if (!na->nopin_timeout)
1090e48354ceSNicholas Bellinger 		return;
1091e48354ceSNicholas Bellinger 
1092e48354ceSNicholas Bellinger 	if (conn->nopin_timer_flags & ISCSI_TF_RUNNING)
1093e48354ceSNicholas Bellinger 		return;
1094e48354ceSNicholas Bellinger 
1095e48354ceSNicholas Bellinger 	init_timer(&conn->nopin_timer);
1096e48354ceSNicholas Bellinger 	conn->nopin_timer.expires = (get_jiffies_64() + na->nopin_timeout * HZ);
1097e48354ceSNicholas Bellinger 	conn->nopin_timer.data = (unsigned long)conn;
1098e48354ceSNicholas Bellinger 	conn->nopin_timer.function = iscsit_handle_nopin_timeout;
1099e48354ceSNicholas Bellinger 	conn->nopin_timer_flags &= ~ISCSI_TF_STOP;
1100e48354ceSNicholas Bellinger 	conn->nopin_timer_flags |= ISCSI_TF_RUNNING;
1101e48354ceSNicholas Bellinger 	add_timer(&conn->nopin_timer);
1102e48354ceSNicholas Bellinger 
1103e48354ceSNicholas Bellinger 	pr_debug("Started NOPIN Timer on CID: %d at %u second"
1104e48354ceSNicholas Bellinger 		" interval\n", conn->cid, na->nopin_timeout);
1105e48354ceSNicholas Bellinger }
1106e48354ceSNicholas Bellinger 
1107e48354ceSNicholas Bellinger void iscsit_start_nopin_timer(struct iscsi_conn *conn)
1108e48354ceSNicholas Bellinger {
1109e48354ceSNicholas Bellinger 	struct iscsi_session *sess = conn->sess;
1110e48354ceSNicholas Bellinger 	struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1111e48354ceSNicholas Bellinger 	/*
1112e48354ceSNicholas Bellinger 	 * NOPIN timeout is disabled..
1113e48354ceSNicholas Bellinger 	 */
1114e48354ceSNicholas Bellinger 	if (!na->nopin_timeout)
1115e48354ceSNicholas Bellinger 		return;
1116e48354ceSNicholas Bellinger 
1117e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->nopin_timer_lock);
1118e48354ceSNicholas Bellinger 	if (conn->nopin_timer_flags & ISCSI_TF_RUNNING) {
1119e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->nopin_timer_lock);
1120e48354ceSNicholas Bellinger 		return;
1121e48354ceSNicholas Bellinger 	}
1122e48354ceSNicholas Bellinger 
1123e48354ceSNicholas Bellinger 	init_timer(&conn->nopin_timer);
1124e48354ceSNicholas Bellinger 	conn->nopin_timer.expires = (get_jiffies_64() + na->nopin_timeout * HZ);
1125e48354ceSNicholas Bellinger 	conn->nopin_timer.data = (unsigned long)conn;
1126e48354ceSNicholas Bellinger 	conn->nopin_timer.function = iscsit_handle_nopin_timeout;
1127e48354ceSNicholas Bellinger 	conn->nopin_timer_flags &= ~ISCSI_TF_STOP;
1128e48354ceSNicholas Bellinger 	conn->nopin_timer_flags |= ISCSI_TF_RUNNING;
1129e48354ceSNicholas Bellinger 	add_timer(&conn->nopin_timer);
1130e48354ceSNicholas Bellinger 
1131e48354ceSNicholas Bellinger 	pr_debug("Started NOPIN Timer on CID: %d at %u second"
1132e48354ceSNicholas Bellinger 			" interval\n", conn->cid, na->nopin_timeout);
1133e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->nopin_timer_lock);
1134e48354ceSNicholas Bellinger }
1135e48354ceSNicholas Bellinger 
1136e48354ceSNicholas Bellinger void iscsit_stop_nopin_timer(struct iscsi_conn *conn)
1137e48354ceSNicholas Bellinger {
1138e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->nopin_timer_lock);
1139e48354ceSNicholas Bellinger 	if (!(conn->nopin_timer_flags & ISCSI_TF_RUNNING)) {
1140e48354ceSNicholas Bellinger 		spin_unlock_bh(&conn->nopin_timer_lock);
1141e48354ceSNicholas Bellinger 		return;
1142e48354ceSNicholas Bellinger 	}
1143e48354ceSNicholas Bellinger 	conn->nopin_timer_flags |= ISCSI_TF_STOP;
1144e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->nopin_timer_lock);
1145e48354ceSNicholas Bellinger 
1146e48354ceSNicholas Bellinger 	del_timer_sync(&conn->nopin_timer);
1147e48354ceSNicholas Bellinger 
1148e48354ceSNicholas Bellinger 	spin_lock_bh(&conn->nopin_timer_lock);
1149e48354ceSNicholas Bellinger 	conn->nopin_timer_flags &= ~ISCSI_TF_RUNNING;
1150e48354ceSNicholas Bellinger 	spin_unlock_bh(&conn->nopin_timer_lock);
1151e48354ceSNicholas Bellinger }
1152e48354ceSNicholas Bellinger 
1153e48354ceSNicholas Bellinger int iscsit_send_tx_data(
1154e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd,
1155e48354ceSNicholas Bellinger 	struct iscsi_conn *conn,
1156e48354ceSNicholas Bellinger 	int use_misc)
1157e48354ceSNicholas Bellinger {
1158e48354ceSNicholas Bellinger 	int tx_sent, tx_size;
1159e48354ceSNicholas Bellinger 	u32 iov_count;
1160e48354ceSNicholas Bellinger 	struct kvec *iov;
1161e48354ceSNicholas Bellinger 
1162e48354ceSNicholas Bellinger send_data:
1163e48354ceSNicholas Bellinger 	tx_size = cmd->tx_size;
1164e48354ceSNicholas Bellinger 
1165e48354ceSNicholas Bellinger 	if (!use_misc) {
1166e48354ceSNicholas Bellinger 		iov = &cmd->iov_data[0];
1167e48354ceSNicholas Bellinger 		iov_count = cmd->iov_data_count;
1168e48354ceSNicholas Bellinger 	} else {
1169e48354ceSNicholas Bellinger 		iov = &cmd->iov_misc[0];
1170e48354ceSNicholas Bellinger 		iov_count = cmd->iov_misc_count;
1171e48354ceSNicholas Bellinger 	}
1172e48354ceSNicholas Bellinger 
1173e48354ceSNicholas Bellinger 	tx_sent = tx_data(conn, &iov[0], iov_count, tx_size);
1174e48354ceSNicholas Bellinger 	if (tx_size != tx_sent) {
1175e48354ceSNicholas Bellinger 		if (tx_sent == -EAGAIN) {
1176e48354ceSNicholas Bellinger 			pr_err("tx_data() returned -EAGAIN\n");
1177e48354ceSNicholas Bellinger 			goto send_data;
1178e48354ceSNicholas Bellinger 		} else
1179e48354ceSNicholas Bellinger 			return -1;
1180e48354ceSNicholas Bellinger 	}
1181e48354ceSNicholas Bellinger 	cmd->tx_size = 0;
1182e48354ceSNicholas Bellinger 
1183e48354ceSNicholas Bellinger 	return 0;
1184e48354ceSNicholas Bellinger }
1185e48354ceSNicholas Bellinger 
1186e48354ceSNicholas Bellinger int iscsit_fe_sendpage_sg(
1187e48354ceSNicholas Bellinger 	struct iscsi_cmd *cmd,
1188e48354ceSNicholas Bellinger 	struct iscsi_conn *conn)
1189e48354ceSNicholas Bellinger {
1190e48354ceSNicholas Bellinger 	struct scatterlist *sg = cmd->first_data_sg;
1191e48354ceSNicholas Bellinger 	struct kvec iov;
1192e48354ceSNicholas Bellinger 	u32 tx_hdr_size, data_len;
1193e48354ceSNicholas Bellinger 	u32 offset = cmd->first_data_sg_off;
119440b05497SNicholas Bellinger 	int tx_sent, iov_off;
1195e48354ceSNicholas Bellinger 
1196e48354ceSNicholas Bellinger send_hdr:
1197e48354ceSNicholas Bellinger 	tx_hdr_size = ISCSI_HDR_LEN;
1198e48354ceSNicholas Bellinger 	if (conn->conn_ops->HeaderDigest)
1199e48354ceSNicholas Bellinger 		tx_hdr_size += ISCSI_CRC_LEN;
1200e48354ceSNicholas Bellinger 
1201e48354ceSNicholas Bellinger 	iov.iov_base = cmd->pdu;
1202e48354ceSNicholas Bellinger 	iov.iov_len = tx_hdr_size;
1203e48354ceSNicholas Bellinger 
1204e48354ceSNicholas Bellinger 	tx_sent = tx_data(conn, &iov, 1, tx_hdr_size);
1205e48354ceSNicholas Bellinger 	if (tx_hdr_size != tx_sent) {
1206e48354ceSNicholas Bellinger 		if (tx_sent == -EAGAIN) {
1207e48354ceSNicholas Bellinger 			pr_err("tx_data() returned -EAGAIN\n");
1208e48354ceSNicholas Bellinger 			goto send_hdr;
1209e48354ceSNicholas Bellinger 		}
1210e48354ceSNicholas Bellinger 		return -1;
1211e48354ceSNicholas Bellinger 	}
1212e48354ceSNicholas Bellinger 
1213e48354ceSNicholas Bellinger 	data_len = cmd->tx_size - tx_hdr_size - cmd->padding;
121440b05497SNicholas Bellinger 	/*
121540b05497SNicholas Bellinger 	 * Set iov_off used by padding and data digest tx_data() calls below
121640b05497SNicholas Bellinger 	 * in order to determine proper offset into cmd->iov_data[]
121740b05497SNicholas Bellinger 	 */
121840b05497SNicholas Bellinger 	if (conn->conn_ops->DataDigest) {
1219e48354ceSNicholas Bellinger 		data_len -= ISCSI_CRC_LEN;
122040b05497SNicholas Bellinger 		if (cmd->padding)
122140b05497SNicholas Bellinger 			iov_off = (cmd->iov_data_count - 2);
122240b05497SNicholas Bellinger 		else
122340b05497SNicholas Bellinger 			iov_off = (cmd->iov_data_count - 1);
122440b05497SNicholas Bellinger 	} else {
122540b05497SNicholas Bellinger 		iov_off = (cmd->iov_data_count - 1);
122640b05497SNicholas Bellinger 	}
1227e48354ceSNicholas Bellinger 	/*
1228e48354ceSNicholas Bellinger 	 * Perform sendpage() for each page in the scatterlist
1229e48354ceSNicholas Bellinger 	 */
1230e48354ceSNicholas Bellinger 	while (data_len) {
1231e48354ceSNicholas Bellinger 		u32 space = (sg->length - offset);
1232e48354ceSNicholas Bellinger 		u32 sub_len = min_t(u32, data_len, space);
1233e48354ceSNicholas Bellinger send_pg:
1234e48354ceSNicholas Bellinger 		tx_sent = conn->sock->ops->sendpage(conn->sock,
1235e48354ceSNicholas Bellinger 					sg_page(sg), sg->offset + offset, sub_len, 0);
1236e48354ceSNicholas Bellinger 		if (tx_sent != sub_len) {
1237e48354ceSNicholas Bellinger 			if (tx_sent == -EAGAIN) {
1238e48354ceSNicholas Bellinger 				pr_err("tcp_sendpage() returned"
1239e48354ceSNicholas Bellinger 						" -EAGAIN\n");
1240e48354ceSNicholas Bellinger 				goto send_pg;
1241e48354ceSNicholas Bellinger 			}
1242e48354ceSNicholas Bellinger 
1243e48354ceSNicholas Bellinger 			pr_err("tcp_sendpage() failure: %d\n",
1244e48354ceSNicholas Bellinger 					tx_sent);
1245e48354ceSNicholas Bellinger 			return -1;
1246e48354ceSNicholas Bellinger 		}
1247e48354ceSNicholas Bellinger 
1248e48354ceSNicholas Bellinger 		data_len -= sub_len;
1249e48354ceSNicholas Bellinger 		offset = 0;
1250e48354ceSNicholas Bellinger 		sg = sg_next(sg);
1251e48354ceSNicholas Bellinger 	}
1252e48354ceSNicholas Bellinger 
1253e48354ceSNicholas Bellinger send_padding:
1254e48354ceSNicholas Bellinger 	if (cmd->padding) {
125540b05497SNicholas Bellinger 		struct kvec *iov_p = &cmd->iov_data[iov_off++];
1256e48354ceSNicholas Bellinger 
1257e48354ceSNicholas Bellinger 		tx_sent = tx_data(conn, iov_p, 1, cmd->padding);
1258e48354ceSNicholas Bellinger 		if (cmd->padding != tx_sent) {
1259e48354ceSNicholas Bellinger 			if (tx_sent == -EAGAIN) {
1260e48354ceSNicholas Bellinger 				pr_err("tx_data() returned -EAGAIN\n");
1261e48354ceSNicholas Bellinger 				goto send_padding;
1262e48354ceSNicholas Bellinger 			}
1263e48354ceSNicholas Bellinger 			return -1;
1264e48354ceSNicholas Bellinger 		}
1265e48354ceSNicholas Bellinger 	}
1266e48354ceSNicholas Bellinger 
1267e48354ceSNicholas Bellinger send_datacrc:
1268e48354ceSNicholas Bellinger 	if (conn->conn_ops->DataDigest) {
126940b05497SNicholas Bellinger 		struct kvec *iov_d = &cmd->iov_data[iov_off];
1270e48354ceSNicholas Bellinger 
1271e48354ceSNicholas Bellinger 		tx_sent = tx_data(conn, iov_d, 1, ISCSI_CRC_LEN);
1272e48354ceSNicholas Bellinger 		if (ISCSI_CRC_LEN != tx_sent) {
1273e48354ceSNicholas Bellinger 			if (tx_sent == -EAGAIN) {
1274e48354ceSNicholas Bellinger 				pr_err("tx_data() returned -EAGAIN\n");
1275e48354ceSNicholas Bellinger 				goto send_datacrc;
1276e48354ceSNicholas Bellinger 			}
1277e48354ceSNicholas Bellinger 			return -1;
1278e48354ceSNicholas Bellinger 		}
1279e48354ceSNicholas Bellinger 	}
1280e48354ceSNicholas Bellinger 
1281e48354ceSNicholas Bellinger 	return 0;
1282e48354ceSNicholas Bellinger }
1283e48354ceSNicholas Bellinger 
1284e48354ceSNicholas Bellinger /*
1285e48354ceSNicholas Bellinger  *      This function is used for mainly sending a ISCSI_TARG_LOGIN_RSP PDU
1286e48354ceSNicholas Bellinger  *      back to the Initiator when an expection condition occurs with the
1287e48354ceSNicholas Bellinger  *      errors set in status_class and status_detail.
1288e48354ceSNicholas Bellinger  *
1289e48354ceSNicholas Bellinger  *      Parameters:     iSCSI Connection, Status Class, Status Detail.
1290e48354ceSNicholas Bellinger  *      Returns:        0 on success, -1 on error.
1291e48354ceSNicholas Bellinger  */
1292e48354ceSNicholas Bellinger int iscsit_tx_login_rsp(struct iscsi_conn *conn, u8 status_class, u8 status_detail)
1293e48354ceSNicholas Bellinger {
1294e48354ceSNicholas Bellinger 	struct iscsi_login_rsp *hdr;
1295baa4d64bSNicholas Bellinger 	struct iscsi_login *login = conn->conn_login;
1296e48354ceSNicholas Bellinger 
1297baa4d64bSNicholas Bellinger 	login->login_failed = 1;
1298e48354ceSNicholas Bellinger 	iscsit_collect_login_stats(conn, status_class, status_detail);
1299e48354ceSNicholas Bellinger 
130068349756SNicholas Bellinger 	memset(&login->rsp[0], 0, ISCSI_HDR_LEN);
130168349756SNicholas Bellinger 
1302baa4d64bSNicholas Bellinger 	hdr	= (struct iscsi_login_rsp *)&login->rsp[0];
1303e48354ceSNicholas Bellinger 	hdr->opcode		= ISCSI_OP_LOGIN_RSP;
1304e48354ceSNicholas Bellinger 	hdr->status_class	= status_class;
1305e48354ceSNicholas Bellinger 	hdr->status_detail	= status_detail;
130666c7db68SChristoph Hellwig 	hdr->itt		= conn->login_itt;
1307e48354ceSNicholas Bellinger 
1308baa4d64bSNicholas Bellinger 	return conn->conn_transport->iscsit_put_login_tx(conn, login, 0);
1309e48354ceSNicholas Bellinger }
1310e48354ceSNicholas Bellinger 
1311e48354ceSNicholas Bellinger void iscsit_print_session_params(struct iscsi_session *sess)
1312e48354ceSNicholas Bellinger {
1313e48354ceSNicholas Bellinger 	struct iscsi_conn *conn;
1314e48354ceSNicholas Bellinger 
1315e48354ceSNicholas Bellinger 	pr_debug("-----------------------------[Session Params for"
1316e48354ceSNicholas Bellinger 		" SID: %u]-----------------------------\n", sess->sid);
1317e48354ceSNicholas Bellinger 	spin_lock_bh(&sess->conn_lock);
1318e48354ceSNicholas Bellinger 	list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
1319e48354ceSNicholas Bellinger 		iscsi_dump_conn_ops(conn->conn_ops);
1320e48354ceSNicholas Bellinger 	spin_unlock_bh(&sess->conn_lock);
1321e48354ceSNicholas Bellinger 
1322e48354ceSNicholas Bellinger 	iscsi_dump_sess_ops(sess->sess_ops);
1323e48354ceSNicholas Bellinger }
1324e48354ceSNicholas Bellinger 
1325e48354ceSNicholas Bellinger static int iscsit_do_rx_data(
1326e48354ceSNicholas Bellinger 	struct iscsi_conn *conn,
1327e48354ceSNicholas Bellinger 	struct iscsi_data_count *count)
1328e48354ceSNicholas Bellinger {
1329e5a4b0bbSAl Viro 	int data = count->data_length, rx_loop = 0, total_rx = 0;
1330e48354ceSNicholas Bellinger 	struct msghdr msg;
1331e48354ceSNicholas Bellinger 
1332e48354ceSNicholas Bellinger 	if (!conn || !conn->sock || !conn->conn_ops)
1333e48354ceSNicholas Bellinger 		return -1;
1334e48354ceSNicholas Bellinger 
1335e48354ceSNicholas Bellinger 	memset(&msg, 0, sizeof(struct msghdr));
1336e5a4b0bbSAl Viro 	iov_iter_kvec(&msg.msg_iter, READ | ITER_KVEC,
1337e5a4b0bbSAl Viro 		      count->iov, count->iov_count, data);
1338e48354ceSNicholas Bellinger 
1339e48354ceSNicholas Bellinger 	while (total_rx < data) {
1340e5a4b0bbSAl Viro 		rx_loop = sock_recvmsg(conn->sock, &msg,
1341e48354ceSNicholas Bellinger 				      (data - total_rx), MSG_WAITALL);
1342e48354ceSNicholas Bellinger 		if (rx_loop <= 0) {
1343e48354ceSNicholas Bellinger 			pr_debug("rx_loop: %d total_rx: %d\n",
1344e48354ceSNicholas Bellinger 				rx_loop, total_rx);
1345e48354ceSNicholas Bellinger 			return rx_loop;
1346e48354ceSNicholas Bellinger 		}
1347e48354ceSNicholas Bellinger 		total_rx += rx_loop;
1348e48354ceSNicholas Bellinger 		pr_debug("rx_loop: %d, total_rx: %d, data: %d\n",
1349e48354ceSNicholas Bellinger 				rx_loop, total_rx, data);
1350e48354ceSNicholas Bellinger 	}
1351e48354ceSNicholas Bellinger 
1352e48354ceSNicholas Bellinger 	return total_rx;
1353e48354ceSNicholas Bellinger }
1354e48354ceSNicholas Bellinger 
1355e48354ceSNicholas Bellinger static int iscsit_do_tx_data(
1356e48354ceSNicholas Bellinger 	struct iscsi_conn *conn,
1357e48354ceSNicholas Bellinger 	struct iscsi_data_count *count)
1358e48354ceSNicholas Bellinger {
13596bf6ca75SNicholas Bellinger 	int ret, iov_len;
13602ff017f5SNicholas Bellinger 	struct kvec *iov_p;
1361e48354ceSNicholas Bellinger 	struct msghdr msg;
1362e48354ceSNicholas Bellinger 
1363e48354ceSNicholas Bellinger 	if (!conn || !conn->sock || !conn->conn_ops)
1364e48354ceSNicholas Bellinger 		return -1;
1365e48354ceSNicholas Bellinger 
13666bf6ca75SNicholas Bellinger 	if (count->data_length <= 0) {
13676bf6ca75SNicholas Bellinger 		pr_err("Data length is: %d\n", count->data_length);
1368e48354ceSNicholas Bellinger 		return -1;
1369e48354ceSNicholas Bellinger 	}
1370e48354ceSNicholas Bellinger 
1371e48354ceSNicholas Bellinger 	memset(&msg, 0, sizeof(struct msghdr));
1372e48354ceSNicholas Bellinger 
1373e48354ceSNicholas Bellinger 	iov_p = count->iov;
1374e48354ceSNicholas Bellinger 	iov_len = count->iov_count;
1375e48354ceSNicholas Bellinger 
13766bf6ca75SNicholas Bellinger 	ret = kernel_sendmsg(conn->sock, &msg, iov_p, iov_len,
13776bf6ca75SNicholas Bellinger 			     count->data_length);
13786bf6ca75SNicholas Bellinger 	if (ret != count->data_length) {
13796bf6ca75SNicholas Bellinger 		pr_err("Unexpected ret: %d send data %d\n",
13806bf6ca75SNicholas Bellinger 		       ret, count->data_length);
13816bf6ca75SNicholas Bellinger 		return -EPIPE;
1382e48354ceSNicholas Bellinger 	}
13836bf6ca75SNicholas Bellinger 	pr_debug("ret: %d, sent data: %d\n", ret, count->data_length);
1384e48354ceSNicholas Bellinger 
13856bf6ca75SNicholas Bellinger 	return ret;
1386e48354ceSNicholas Bellinger }
1387e48354ceSNicholas Bellinger 
1388e48354ceSNicholas Bellinger int rx_data(
1389e48354ceSNicholas Bellinger 	struct iscsi_conn *conn,
1390e48354ceSNicholas Bellinger 	struct kvec *iov,
1391e48354ceSNicholas Bellinger 	int iov_count,
1392e48354ceSNicholas Bellinger 	int data)
1393e48354ceSNicholas Bellinger {
1394e48354ceSNicholas Bellinger 	struct iscsi_data_count c;
1395e48354ceSNicholas Bellinger 
1396e48354ceSNicholas Bellinger 	if (!conn || !conn->sock || !conn->conn_ops)
1397e48354ceSNicholas Bellinger 		return -1;
1398e48354ceSNicholas Bellinger 
1399e48354ceSNicholas Bellinger 	memset(&c, 0, sizeof(struct iscsi_data_count));
1400e48354ceSNicholas Bellinger 	c.iov = iov;
1401e48354ceSNicholas Bellinger 	c.iov_count = iov_count;
1402e48354ceSNicholas Bellinger 	c.data_length = data;
1403e48354ceSNicholas Bellinger 	c.type = ISCSI_RX_DATA;
1404e48354ceSNicholas Bellinger 
1405e48354ceSNicholas Bellinger 	return iscsit_do_rx_data(conn, &c);
1406e48354ceSNicholas Bellinger }
1407e48354ceSNicholas Bellinger 
1408e48354ceSNicholas Bellinger int tx_data(
1409e48354ceSNicholas Bellinger 	struct iscsi_conn *conn,
1410e48354ceSNicholas Bellinger 	struct kvec *iov,
1411e48354ceSNicholas Bellinger 	int iov_count,
1412e48354ceSNicholas Bellinger 	int data)
1413e48354ceSNicholas Bellinger {
1414e48354ceSNicholas Bellinger 	struct iscsi_data_count c;
1415e48354ceSNicholas Bellinger 
1416e48354ceSNicholas Bellinger 	if (!conn || !conn->sock || !conn->conn_ops)
1417e48354ceSNicholas Bellinger 		return -1;
1418e48354ceSNicholas Bellinger 
1419e48354ceSNicholas Bellinger 	memset(&c, 0, sizeof(struct iscsi_data_count));
1420e48354ceSNicholas Bellinger 	c.iov = iov;
1421e48354ceSNicholas Bellinger 	c.iov_count = iov_count;
1422e48354ceSNicholas Bellinger 	c.data_length = data;
1423e48354ceSNicholas Bellinger 	c.type = ISCSI_TX_DATA;
1424e48354ceSNicholas Bellinger 
1425e48354ceSNicholas Bellinger 	return iscsit_do_tx_data(conn, &c);
1426e48354ceSNicholas Bellinger }
1427e48354ceSNicholas Bellinger 
1428e48354ceSNicholas Bellinger void iscsit_collect_login_stats(
1429e48354ceSNicholas Bellinger 	struct iscsi_conn *conn,
1430e48354ceSNicholas Bellinger 	u8 status_class,
1431e48354ceSNicholas Bellinger 	u8 status_detail)
1432e48354ceSNicholas Bellinger {
1433e48354ceSNicholas Bellinger 	struct iscsi_param *intrname = NULL;
1434e48354ceSNicholas Bellinger 	struct iscsi_tiqn *tiqn;
1435e48354ceSNicholas Bellinger 	struct iscsi_login_stats *ls;
1436e48354ceSNicholas Bellinger 
1437e48354ceSNicholas Bellinger 	tiqn = iscsit_snmp_get_tiqn(conn);
1438e48354ceSNicholas Bellinger 	if (!tiqn)
1439e48354ceSNicholas Bellinger 		return;
1440e48354ceSNicholas Bellinger 
1441e48354ceSNicholas Bellinger 	ls = &tiqn->login_stats;
1442e48354ceSNicholas Bellinger 
1443e48354ceSNicholas Bellinger 	spin_lock(&ls->lock);
1444e48354ceSNicholas Bellinger 	if (!strcmp(conn->login_ip, ls->last_intr_fail_ip_addr) &&
1445e48354ceSNicholas Bellinger 	    ((get_jiffies_64() - ls->last_fail_time) < 10)) {
1446e48354ceSNicholas Bellinger 		/* We already have the failure info for this login */
1447e48354ceSNicholas Bellinger 		spin_unlock(&ls->lock);
1448e48354ceSNicholas Bellinger 		return;
1449e48354ceSNicholas Bellinger 	}
1450e48354ceSNicholas Bellinger 
1451e48354ceSNicholas Bellinger 	if (status_class == ISCSI_STATUS_CLS_SUCCESS)
1452e48354ceSNicholas Bellinger 		ls->accepts++;
1453e48354ceSNicholas Bellinger 	else if (status_class == ISCSI_STATUS_CLS_REDIRECT) {
1454e48354ceSNicholas Bellinger 		ls->redirects++;
1455e48354ceSNicholas Bellinger 		ls->last_fail_type = ISCSI_LOGIN_FAIL_REDIRECT;
1456e48354ceSNicholas Bellinger 	} else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR)  &&
1457e48354ceSNicholas Bellinger 		 (status_detail == ISCSI_LOGIN_STATUS_AUTH_FAILED)) {
1458e48354ceSNicholas Bellinger 		ls->authenticate_fails++;
1459e48354ceSNicholas Bellinger 		ls->last_fail_type =  ISCSI_LOGIN_FAIL_AUTHENTICATE;
1460e48354ceSNicholas Bellinger 	} else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR)  &&
1461e48354ceSNicholas Bellinger 		 (status_detail == ISCSI_LOGIN_STATUS_TGT_FORBIDDEN)) {
1462e48354ceSNicholas Bellinger 		ls->authorize_fails++;
1463e48354ceSNicholas Bellinger 		ls->last_fail_type = ISCSI_LOGIN_FAIL_AUTHORIZE;
1464e48354ceSNicholas Bellinger 	} else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR) &&
1465e48354ceSNicholas Bellinger 		 (status_detail == ISCSI_LOGIN_STATUS_INIT_ERR)) {
1466e48354ceSNicholas Bellinger 		ls->negotiate_fails++;
1467e48354ceSNicholas Bellinger 		ls->last_fail_type = ISCSI_LOGIN_FAIL_NEGOTIATE;
1468e48354ceSNicholas Bellinger 	} else {
1469e48354ceSNicholas Bellinger 		ls->other_fails++;
1470e48354ceSNicholas Bellinger 		ls->last_fail_type = ISCSI_LOGIN_FAIL_OTHER;
1471e48354ceSNicholas Bellinger 	}
1472e48354ceSNicholas Bellinger 
1473e48354ceSNicholas Bellinger 	/* Save initiator name, ip address and time, if it is a failed login */
1474e48354ceSNicholas Bellinger 	if (status_class != ISCSI_STATUS_CLS_SUCCESS) {
1475e48354ceSNicholas Bellinger 		if (conn->param_list)
1476e48354ceSNicholas Bellinger 			intrname = iscsi_find_param_from_key(INITIATORNAME,
1477e48354ceSNicholas Bellinger 							     conn->param_list);
1478fdc84d11SJoern Engel 		strlcpy(ls->last_intr_fail_name,
1479fdc84d11SJoern Engel 		       (intrname ? intrname->value : "Unknown"),
1480fdc84d11SJoern Engel 		       sizeof(ls->last_intr_fail_name));
1481e48354ceSNicholas Bellinger 
1482baa4d64bSNicholas Bellinger 		ls->last_intr_fail_ip_family = conn->login_family;
1483baa4d64bSNicholas Bellinger 
1484e48354ceSNicholas Bellinger 		snprintf(ls->last_intr_fail_ip_addr, IPV6_ADDRESS_SPACE,
1485e48354ceSNicholas Bellinger 				"%s", conn->login_ip);
1486e48354ceSNicholas Bellinger 		ls->last_fail_time = get_jiffies_64();
1487e48354ceSNicholas Bellinger 	}
1488e48354ceSNicholas Bellinger 
1489e48354ceSNicholas Bellinger 	spin_unlock(&ls->lock);
1490e48354ceSNicholas Bellinger }
1491e48354ceSNicholas Bellinger 
1492e48354ceSNicholas Bellinger struct iscsi_tiqn *iscsit_snmp_get_tiqn(struct iscsi_conn *conn)
1493e48354ceSNicholas Bellinger {
1494e48354ceSNicholas Bellinger 	struct iscsi_portal_group *tpg;
1495e48354ceSNicholas Bellinger 
1496e48354ceSNicholas Bellinger 	if (!conn || !conn->sess)
1497e48354ceSNicholas Bellinger 		return NULL;
1498e48354ceSNicholas Bellinger 
1499e48354ceSNicholas Bellinger 	tpg = conn->sess->tpg;
1500e48354ceSNicholas Bellinger 	if (!tpg)
1501e48354ceSNicholas Bellinger 		return NULL;
1502e48354ceSNicholas Bellinger 
1503e48354ceSNicholas Bellinger 	if (!tpg->tpg_tiqn)
1504e48354ceSNicholas Bellinger 		return NULL;
1505e48354ceSNicholas Bellinger 
1506e48354ceSNicholas Bellinger 	return tpg->tpg_tiqn;
1507e48354ceSNicholas Bellinger }
1508