1 /******************************************************************************
2  * This file contains error recovery level zero functions used by
3  * the iSCSI Target driver.
4  *
5  * (c) Copyright 2007-2013 Datera, Inc.
6  *
7  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  ******************************************************************************/
19 
20 #include <scsi/iscsi_proto.h>
21 #include <target/target_core_base.h>
22 #include <target/target_core_fabric.h>
23 
24 #include "iscsi_target_core.h"
25 #include "iscsi_target_seq_pdu_list.h"
26 #include "iscsi_target_tq.h"
27 #include "iscsi_target_erl0.h"
28 #include "iscsi_target_erl1.h"
29 #include "iscsi_target_erl2.h"
30 #include "iscsi_target_util.h"
31 #include "iscsi_target.h"
32 
33 /*
34  *	Used to set values in struct iscsi_cmd that iscsit_dataout_check_sequence()
35  *	checks against to determine a PDU's Offset+Length is within the current
36  *	DataOUT Sequence.  Used for DataSequenceInOrder=Yes only.
37  */
38 void iscsit_set_dataout_sequence_values(
39 	struct iscsi_cmd *cmd)
40 {
41 	struct iscsi_conn *conn = cmd->conn;
42 	/*
43 	 * Still set seq_start_offset and seq_end_offset for Unsolicited
44 	 * DataOUT, even if DataSequenceInOrder=No.
45 	 */
46 	if (cmd->unsolicited_data) {
47 		cmd->seq_start_offset = cmd->write_data_done;
48 		cmd->seq_end_offset = (cmd->write_data_done +
49 			((cmd->se_cmd.data_length >
50 			  conn->sess->sess_ops->FirstBurstLength) ?
51 			 conn->sess->sess_ops->FirstBurstLength : cmd->se_cmd.data_length));
52 		return;
53 	}
54 
55 	if (!conn->sess->sess_ops->DataSequenceInOrder)
56 		return;
57 
58 	if (!cmd->seq_start_offset && !cmd->seq_end_offset) {
59 		cmd->seq_start_offset = cmd->write_data_done;
60 		cmd->seq_end_offset = (cmd->se_cmd.data_length >
61 			conn->sess->sess_ops->MaxBurstLength) ?
62 			(cmd->write_data_done +
63 			conn->sess->sess_ops->MaxBurstLength) : cmd->se_cmd.data_length;
64 	} else {
65 		cmd->seq_start_offset = cmd->seq_end_offset;
66 		cmd->seq_end_offset = ((cmd->seq_end_offset +
67 			conn->sess->sess_ops->MaxBurstLength) >=
68 			cmd->se_cmd.data_length) ? cmd->se_cmd.data_length :
69 			(cmd->seq_end_offset +
70 			 conn->sess->sess_ops->MaxBurstLength);
71 	}
72 }
73 
74 static int iscsit_dataout_within_command_recovery_check(
75 	struct iscsi_cmd *cmd,
76 	unsigned char *buf)
77 {
78 	struct iscsi_conn *conn = cmd->conn;
79 	struct iscsi_data *hdr = (struct iscsi_data *) buf;
80 	u32 payload_length = ntoh24(hdr->dlength);
81 
82 	/*
83 	 * We do the within-command recovery checks here as it is
84 	 * the first function called in iscsi_check_pre_dataout().
85 	 * Basically, if we are in within-command recovery and
86 	 * the PDU does not contain the offset the sequence needs,
87 	 * dump the payload.
88 	 *
89 	 * This only applies to DataPDUInOrder=Yes, for
90 	 * DataPDUInOrder=No we only re-request the failed PDU
91 	 * and check that all PDUs in a sequence are received
92 	 * upon end of sequence.
93 	 */
94 	if (conn->sess->sess_ops->DataSequenceInOrder) {
95 		if ((cmd->cmd_flags & ICF_WITHIN_COMMAND_RECOVERY) &&
96 		    cmd->write_data_done != be32_to_cpu(hdr->offset))
97 			goto dump;
98 
99 		cmd->cmd_flags &= ~ICF_WITHIN_COMMAND_RECOVERY;
100 	} else {
101 		struct iscsi_seq *seq;
102 
103 		seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
104 					    payload_length);
105 		if (!seq)
106 			return DATAOUT_CANNOT_RECOVER;
107 		/*
108 		 * Set the struct iscsi_seq pointer to reuse later.
109 		 */
110 		cmd->seq_ptr = seq;
111 
112 		if (conn->sess->sess_ops->DataPDUInOrder) {
113 			if (seq->status ==
114 			    DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
115 			   (seq->offset != be32_to_cpu(hdr->offset) ||
116 			    seq->data_sn != be32_to_cpu(hdr->datasn)))
117 				goto dump;
118 		} else {
119 			if (seq->status ==
120 			     DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
121 			    seq->data_sn != be32_to_cpu(hdr->datasn))
122 				goto dump;
123 		}
124 
125 		if (seq->status == DATAOUT_SEQUENCE_COMPLETE)
126 			goto dump;
127 
128 		if (seq->status != DATAOUT_SEQUENCE_COMPLETE)
129 			seq->status = 0;
130 	}
131 
132 	return DATAOUT_NORMAL;
133 
134 dump:
135 	pr_err("Dumping DataOUT PDU Offset: %u Length: %d DataSN:"
136 		" 0x%08x\n", hdr->offset, payload_length, hdr->datasn);
137 	return iscsit_dump_data_payload(conn, payload_length, 1);
138 }
139 
140 static int iscsit_dataout_check_unsolicited_sequence(
141 	struct iscsi_cmd *cmd,
142 	unsigned char *buf)
143 {
144 	u32 first_burst_len;
145 	struct iscsi_conn *conn = cmd->conn;
146 	struct iscsi_data *hdr = (struct iscsi_data *) buf;
147 	u32 payload_length = ntoh24(hdr->dlength);
148 
149 
150 	if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
151 	   ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
152 		pr_err("Command ITT: 0x%08x with Offset: %u,"
153 		" Length: %u outside of Unsolicited Sequence %u:%u while"
154 		" DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
155 		be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
156 			cmd->seq_end_offset);
157 		return DATAOUT_CANNOT_RECOVER;
158 	}
159 
160 	first_burst_len = (cmd->first_burst_len + payload_length);
161 
162 	if (first_burst_len > conn->sess->sess_ops->FirstBurstLength) {
163 		pr_err("Total %u bytes exceeds FirstBurstLength: %u"
164 			" for this Unsolicited DataOut Burst.\n",
165 			first_burst_len, conn->sess->sess_ops->FirstBurstLength);
166 		transport_send_check_condition_and_sense(&cmd->se_cmd,
167 				TCM_INCORRECT_AMOUNT_OF_DATA, 0);
168 		return DATAOUT_CANNOT_RECOVER;
169 	}
170 
171 	/*
172 	 * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
173 	 * checks for the current Unsolicited DataOUT Sequence.
174 	 */
175 	if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
176 		/*
177 		 * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
178 		 * sequence checks are handled in
179 		 * iscsit_dataout_datapduinorder_no_fbit().
180 		 */
181 		if (!conn->sess->sess_ops->DataPDUInOrder)
182 			goto out;
183 
184 		if ((first_burst_len != cmd->se_cmd.data_length) &&
185 		    (first_burst_len != conn->sess->sess_ops->FirstBurstLength)) {
186 			pr_err("Unsolicited non-immediate data"
187 			" received %u does not equal FirstBurstLength: %u, and"
188 			" does not equal ExpXferLen %u.\n", first_burst_len,
189 				conn->sess->sess_ops->FirstBurstLength,
190 				cmd->se_cmd.data_length);
191 			transport_send_check_condition_and_sense(&cmd->se_cmd,
192 					TCM_INCORRECT_AMOUNT_OF_DATA, 0);
193 			return DATAOUT_CANNOT_RECOVER;
194 		}
195 	} else {
196 		if (first_burst_len == conn->sess->sess_ops->FirstBurstLength) {
197 			pr_err("Command ITT: 0x%08x reached"
198 			" FirstBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
199 				" error.\n", cmd->init_task_tag,
200 				conn->sess->sess_ops->FirstBurstLength);
201 			return DATAOUT_CANNOT_RECOVER;
202 		}
203 		if (first_burst_len == cmd->se_cmd.data_length) {
204 			pr_err("Command ITT: 0x%08x reached"
205 			" ExpXferLen: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
206 			" error.\n", cmd->init_task_tag, cmd->se_cmd.data_length);
207 			return DATAOUT_CANNOT_RECOVER;
208 		}
209 	}
210 
211 out:
212 	return DATAOUT_NORMAL;
213 }
214 
215 static int iscsit_dataout_check_sequence(
216 	struct iscsi_cmd *cmd,
217 	unsigned char *buf)
218 {
219 	u32 next_burst_len;
220 	struct iscsi_conn *conn = cmd->conn;
221 	struct iscsi_seq *seq = NULL;
222 	struct iscsi_data *hdr = (struct iscsi_data *) buf;
223 	u32 payload_length = ntoh24(hdr->dlength);
224 
225 	/*
226 	 * For DataSequenceInOrder=Yes: Check that the offset and offset+length
227 	 * is within range as defined by iscsi_set_dataout_sequence_values().
228 	 *
229 	 * For DataSequenceInOrder=No: Check that an struct iscsi_seq exists for
230 	 * offset+length tuple.
231 	 */
232 	if (conn->sess->sess_ops->DataSequenceInOrder) {
233 		/*
234 		 * Due to possibility of recovery DataOUT sent by the initiator
235 		 * fullfilling an Recovery R2T, it's best to just dump the
236 		 * payload here, instead of erroring out.
237 		 */
238 		if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
239 		   ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
240 			pr_err("Command ITT: 0x%08x with Offset: %u,"
241 			" Length: %u outside of Sequence %u:%u while"
242 			" DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
243 			be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
244 				cmd->seq_end_offset);
245 
246 			if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
247 				return DATAOUT_CANNOT_RECOVER;
248 			return DATAOUT_WITHIN_COMMAND_RECOVERY;
249 		}
250 
251 		next_burst_len = (cmd->next_burst_len + payload_length);
252 	} else {
253 		seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
254 					    payload_length);
255 		if (!seq)
256 			return DATAOUT_CANNOT_RECOVER;
257 		/*
258 		 * Set the struct iscsi_seq pointer to reuse later.
259 		 */
260 		cmd->seq_ptr = seq;
261 
262 		if (seq->status == DATAOUT_SEQUENCE_COMPLETE) {
263 			if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
264 				return DATAOUT_CANNOT_RECOVER;
265 			return DATAOUT_WITHIN_COMMAND_RECOVERY;
266 		}
267 
268 		next_burst_len = (seq->next_burst_len + payload_length);
269 	}
270 
271 	if (next_burst_len > conn->sess->sess_ops->MaxBurstLength) {
272 		pr_err("Command ITT: 0x%08x, NextBurstLength: %u and"
273 			" Length: %u exceeds MaxBurstLength: %u. protocol"
274 			" error.\n", cmd->init_task_tag,
275 			(next_burst_len - payload_length),
276 			payload_length, conn->sess->sess_ops->MaxBurstLength);
277 		return DATAOUT_CANNOT_RECOVER;
278 	}
279 
280 	/*
281 	 * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
282 	 * checks for the current DataOUT Sequence.
283 	 */
284 	if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
285 		/*
286 		 * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
287 		 * sequence checks are handled in
288 		 * iscsit_dataout_datapduinorder_no_fbit().
289 		 */
290 		if (!conn->sess->sess_ops->DataPDUInOrder)
291 			goto out;
292 
293 		if (conn->sess->sess_ops->DataSequenceInOrder) {
294 			if ((next_burst_len <
295 			     conn->sess->sess_ops->MaxBurstLength) &&
296 			   ((cmd->write_data_done + payload_length) <
297 			     cmd->se_cmd.data_length)) {
298 				pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
299 				" before end of DataOUT sequence, protocol"
300 				" error.\n", cmd->init_task_tag);
301 				return DATAOUT_CANNOT_RECOVER;
302 			}
303 		} else {
304 			if (next_burst_len < seq->xfer_len) {
305 				pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
306 				" before end of DataOUT sequence, protocol"
307 				" error.\n", cmd->init_task_tag);
308 				return DATAOUT_CANNOT_RECOVER;
309 			}
310 		}
311 	} else {
312 		if (conn->sess->sess_ops->DataSequenceInOrder) {
313 			if (next_burst_len ==
314 					conn->sess->sess_ops->MaxBurstLength) {
315 				pr_err("Command ITT: 0x%08x reached"
316 				" MaxBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is"
317 				" not set, protocol error.", cmd->init_task_tag,
318 					conn->sess->sess_ops->MaxBurstLength);
319 				return DATAOUT_CANNOT_RECOVER;
320 			}
321 			if ((cmd->write_data_done + payload_length) ==
322 					cmd->se_cmd.data_length) {
323 				pr_err("Command ITT: 0x%08x reached"
324 				" last DataOUT PDU in sequence but ISCSI_FLAG_"
325 				"CMD_FINAL is not set, protocol error.\n",
326 					cmd->init_task_tag);
327 				return DATAOUT_CANNOT_RECOVER;
328 			}
329 		} else {
330 			if (next_burst_len == seq->xfer_len) {
331 				pr_err("Command ITT: 0x%08x reached"
332 				" last DataOUT PDU in sequence but ISCSI_FLAG_"
333 				"CMD_FINAL is not set, protocol error.\n",
334 					cmd->init_task_tag);
335 				return DATAOUT_CANNOT_RECOVER;
336 			}
337 		}
338 	}
339 
340 out:
341 	return DATAOUT_NORMAL;
342 }
343 
344 static int iscsit_dataout_check_datasn(
345 	struct iscsi_cmd *cmd,
346 	unsigned char *buf)
347 {
348 	int dump = 0, recovery = 0;
349 	u32 data_sn = 0;
350 	struct iscsi_conn *conn = cmd->conn;
351 	struct iscsi_data *hdr = (struct iscsi_data *) buf;
352 	u32 payload_length = ntoh24(hdr->dlength);
353 
354 	/*
355 	 * Considering the target has no method of re-requesting DataOUT
356 	 * by DataSN, if we receieve a greater DataSN than expected we
357 	 * assume the functions for DataPDUInOrder=[Yes,No] below will
358 	 * handle it.
359 	 *
360 	 * If the DataSN is less than expected, dump the payload.
361 	 */
362 	if (conn->sess->sess_ops->DataSequenceInOrder)
363 		data_sn = cmd->data_sn;
364 	else {
365 		struct iscsi_seq *seq = cmd->seq_ptr;
366 		data_sn = seq->data_sn;
367 	}
368 
369 	if (be32_to_cpu(hdr->datasn) > data_sn) {
370 		pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
371 			" higher than expected 0x%08x.\n", cmd->init_task_tag,
372 				be32_to_cpu(hdr->datasn), data_sn);
373 		recovery = 1;
374 		goto recover;
375 	} else if (be32_to_cpu(hdr->datasn) < data_sn) {
376 		pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
377 			" lower than expected 0x%08x, discarding payload.\n",
378 			cmd->init_task_tag, be32_to_cpu(hdr->datasn), data_sn);
379 		dump = 1;
380 		goto dump;
381 	}
382 
383 	return DATAOUT_NORMAL;
384 
385 recover:
386 	if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
387 		pr_err("Unable to perform within-command recovery"
388 				" while ERL=0.\n");
389 		return DATAOUT_CANNOT_RECOVER;
390 	}
391 dump:
392 	if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
393 		return DATAOUT_CANNOT_RECOVER;
394 
395 	return (recovery || dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY :
396 				DATAOUT_NORMAL;
397 }
398 
399 static int iscsit_dataout_pre_datapduinorder_yes(
400 	struct iscsi_cmd *cmd,
401 	unsigned char *buf)
402 {
403 	int dump = 0, recovery = 0;
404 	struct iscsi_conn *conn = cmd->conn;
405 	struct iscsi_data *hdr = (struct iscsi_data *) buf;
406 	u32 payload_length = ntoh24(hdr->dlength);
407 
408 	/*
409 	 * For DataSequenceInOrder=Yes: If the offset is greater than the global
410 	 * DataPDUInOrder=Yes offset counter in struct iscsi_cmd a protcol error has
411 	 * occurred and fail the connection.
412 	 *
413 	 * For DataSequenceInOrder=No: If the offset is greater than the per
414 	 * sequence DataPDUInOrder=Yes offset counter in struct iscsi_seq a protocol
415 	 * error has occurred and fail the connection.
416 	 */
417 	if (conn->sess->sess_ops->DataSequenceInOrder) {
418 		if (be32_to_cpu(hdr->offset) != cmd->write_data_done) {
419 			pr_err("Command ITT: 0x%08x, received offset"
420 			" %u different than expected %u.\n", cmd->init_task_tag,
421 				be32_to_cpu(hdr->offset), cmd->write_data_done);
422 			recovery = 1;
423 			goto recover;
424 		}
425 	} else {
426 		struct iscsi_seq *seq = cmd->seq_ptr;
427 
428 		if (be32_to_cpu(hdr->offset) > seq->offset) {
429 			pr_err("Command ITT: 0x%08x, received offset"
430 			" %u greater than expected %u.\n", cmd->init_task_tag,
431 				be32_to_cpu(hdr->offset), seq->offset);
432 			recovery = 1;
433 			goto recover;
434 		} else if (be32_to_cpu(hdr->offset) < seq->offset) {
435 			pr_err("Command ITT: 0x%08x, received offset"
436 			" %u less than expected %u, discarding payload.\n",
437 				cmd->init_task_tag, be32_to_cpu(hdr->offset),
438 				seq->offset);
439 			dump = 1;
440 			goto dump;
441 		}
442 	}
443 
444 	return DATAOUT_NORMAL;
445 
446 recover:
447 	if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
448 		pr_err("Unable to perform within-command recovery"
449 				" while ERL=0.\n");
450 		return DATAOUT_CANNOT_RECOVER;
451 	}
452 dump:
453 	if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
454 		return DATAOUT_CANNOT_RECOVER;
455 
456 	return (recovery) ? iscsit_recover_dataout_sequence(cmd,
457 		be32_to_cpu(hdr->offset), payload_length) :
458 	       (dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY : DATAOUT_NORMAL;
459 }
460 
461 static int iscsit_dataout_pre_datapduinorder_no(
462 	struct iscsi_cmd *cmd,
463 	unsigned char *buf)
464 {
465 	struct iscsi_pdu *pdu;
466 	struct iscsi_data *hdr = (struct iscsi_data *) buf;
467 	u32 payload_length = ntoh24(hdr->dlength);
468 
469 	pdu = iscsit_get_pdu_holder(cmd, be32_to_cpu(hdr->offset),
470 				    payload_length);
471 	if (!pdu)
472 		return DATAOUT_CANNOT_RECOVER;
473 
474 	cmd->pdu_ptr = pdu;
475 
476 	switch (pdu->status) {
477 	case ISCSI_PDU_NOT_RECEIVED:
478 	case ISCSI_PDU_CRC_FAILED:
479 	case ISCSI_PDU_TIMED_OUT:
480 		break;
481 	case ISCSI_PDU_RECEIVED_OK:
482 		pr_err("Command ITT: 0x%08x received already gotten"
483 			" Offset: %u, Length: %u\n", cmd->init_task_tag,
484 				be32_to_cpu(hdr->offset), payload_length);
485 		return iscsit_dump_data_payload(cmd->conn, payload_length, 1);
486 	default:
487 		return DATAOUT_CANNOT_RECOVER;
488 	}
489 
490 	return DATAOUT_NORMAL;
491 }
492 
493 static int iscsit_dataout_update_r2t(struct iscsi_cmd *cmd, u32 offset, u32 length)
494 {
495 	struct iscsi_r2t *r2t;
496 
497 	if (cmd->unsolicited_data)
498 		return 0;
499 
500 	r2t = iscsit_get_r2t_for_eos(cmd, offset, length);
501 	if (!r2t)
502 		return -1;
503 
504 	spin_lock_bh(&cmd->r2t_lock);
505 	r2t->seq_complete = 1;
506 	cmd->outstanding_r2ts--;
507 	spin_unlock_bh(&cmd->r2t_lock);
508 
509 	return 0;
510 }
511 
512 static int iscsit_dataout_update_datapduinorder_no(
513 	struct iscsi_cmd *cmd,
514 	u32 data_sn,
515 	int f_bit)
516 {
517 	int ret = 0;
518 	struct iscsi_pdu *pdu = cmd->pdu_ptr;
519 
520 	pdu->data_sn = data_sn;
521 
522 	switch (pdu->status) {
523 	case ISCSI_PDU_NOT_RECEIVED:
524 		pdu->status = ISCSI_PDU_RECEIVED_OK;
525 		break;
526 	case ISCSI_PDU_CRC_FAILED:
527 		pdu->status = ISCSI_PDU_RECEIVED_OK;
528 		break;
529 	case ISCSI_PDU_TIMED_OUT:
530 		pdu->status = ISCSI_PDU_RECEIVED_OK;
531 		break;
532 	default:
533 		return DATAOUT_CANNOT_RECOVER;
534 	}
535 
536 	if (f_bit) {
537 		ret = iscsit_dataout_datapduinorder_no_fbit(cmd, pdu);
538 		if (ret == DATAOUT_CANNOT_RECOVER)
539 			return ret;
540 	}
541 
542 	return DATAOUT_NORMAL;
543 }
544 
545 static int iscsit_dataout_post_crc_passed(
546 	struct iscsi_cmd *cmd,
547 	unsigned char *buf)
548 {
549 	int ret, send_r2t = 0;
550 	struct iscsi_conn *conn = cmd->conn;
551 	struct iscsi_seq *seq = NULL;
552 	struct iscsi_data *hdr = (struct iscsi_data *) buf;
553 	u32 payload_length = ntoh24(hdr->dlength);
554 
555 	if (cmd->unsolicited_data) {
556 		if ((cmd->first_burst_len + payload_length) ==
557 		     conn->sess->sess_ops->FirstBurstLength) {
558 			if (iscsit_dataout_update_r2t(cmd, be32_to_cpu(hdr->offset),
559 					payload_length) < 0)
560 				return DATAOUT_CANNOT_RECOVER;
561 			send_r2t = 1;
562 		}
563 
564 		if (!conn->sess->sess_ops->DataPDUInOrder) {
565 			ret = iscsit_dataout_update_datapduinorder_no(cmd,
566 				be32_to_cpu(hdr->datasn),
567 				(hdr->flags & ISCSI_FLAG_CMD_FINAL));
568 			if (ret == DATAOUT_CANNOT_RECOVER)
569 				return ret;
570 		}
571 
572 		cmd->first_burst_len += payload_length;
573 
574 		if (conn->sess->sess_ops->DataSequenceInOrder)
575 			cmd->data_sn++;
576 		else {
577 			seq = cmd->seq_ptr;
578 			seq->data_sn++;
579 			seq->offset += payload_length;
580 		}
581 
582 		if (send_r2t) {
583 			if (seq)
584 				seq->status = DATAOUT_SEQUENCE_COMPLETE;
585 			cmd->first_burst_len = 0;
586 			cmd->unsolicited_data = 0;
587 		}
588 	} else {
589 		if (conn->sess->sess_ops->DataSequenceInOrder) {
590 			if ((cmd->next_burst_len + payload_length) ==
591 			     conn->sess->sess_ops->MaxBurstLength) {
592 				if (iscsit_dataout_update_r2t(cmd,
593 						be32_to_cpu(hdr->offset),
594 						payload_length) < 0)
595 					return DATAOUT_CANNOT_RECOVER;
596 				send_r2t = 1;
597 			}
598 
599 			if (!conn->sess->sess_ops->DataPDUInOrder) {
600 				ret = iscsit_dataout_update_datapduinorder_no(
601 						cmd, be32_to_cpu(hdr->datasn),
602 						(hdr->flags & ISCSI_FLAG_CMD_FINAL));
603 				if (ret == DATAOUT_CANNOT_RECOVER)
604 					return ret;
605 			}
606 
607 			cmd->next_burst_len += payload_length;
608 			cmd->data_sn++;
609 
610 			if (send_r2t)
611 				cmd->next_burst_len = 0;
612 		} else {
613 			seq = cmd->seq_ptr;
614 
615 			if ((seq->next_burst_len + payload_length) ==
616 			     seq->xfer_len) {
617 				if (iscsit_dataout_update_r2t(cmd,
618 						be32_to_cpu(hdr->offset),
619 						payload_length) < 0)
620 					return DATAOUT_CANNOT_RECOVER;
621 				send_r2t = 1;
622 			}
623 
624 			if (!conn->sess->sess_ops->DataPDUInOrder) {
625 				ret = iscsit_dataout_update_datapduinorder_no(
626 						cmd, be32_to_cpu(hdr->datasn),
627 						(hdr->flags & ISCSI_FLAG_CMD_FINAL));
628 				if (ret == DATAOUT_CANNOT_RECOVER)
629 					return ret;
630 			}
631 
632 			seq->data_sn++;
633 			seq->offset += payload_length;
634 			seq->next_burst_len += payload_length;
635 
636 			if (send_r2t) {
637 				seq->next_burst_len = 0;
638 				seq->status = DATAOUT_SEQUENCE_COMPLETE;
639 			}
640 		}
641 	}
642 
643 	if (send_r2t && conn->sess->sess_ops->DataSequenceInOrder)
644 		cmd->data_sn = 0;
645 
646 	cmd->write_data_done += payload_length;
647 
648 	if (cmd->write_data_done == cmd->se_cmd.data_length)
649 		return DATAOUT_SEND_TO_TRANSPORT;
650 	else if (send_r2t)
651 		return DATAOUT_SEND_R2T;
652 	else
653 		return DATAOUT_NORMAL;
654 }
655 
656 static int iscsit_dataout_post_crc_failed(
657 	struct iscsi_cmd *cmd,
658 	unsigned char *buf)
659 {
660 	struct iscsi_conn *conn = cmd->conn;
661 	struct iscsi_pdu *pdu;
662 	struct iscsi_data *hdr = (struct iscsi_data *) buf;
663 	u32 payload_length = ntoh24(hdr->dlength);
664 
665 	if (conn->sess->sess_ops->DataPDUInOrder)
666 		goto recover;
667 	/*
668 	 * The rest of this function is only called when DataPDUInOrder=No.
669 	 */
670 	pdu = cmd->pdu_ptr;
671 
672 	switch (pdu->status) {
673 	case ISCSI_PDU_NOT_RECEIVED:
674 		pdu->status = ISCSI_PDU_CRC_FAILED;
675 		break;
676 	case ISCSI_PDU_CRC_FAILED:
677 		break;
678 	case ISCSI_PDU_TIMED_OUT:
679 		pdu->status = ISCSI_PDU_CRC_FAILED;
680 		break;
681 	default:
682 		return DATAOUT_CANNOT_RECOVER;
683 	}
684 
685 recover:
686 	return iscsit_recover_dataout_sequence(cmd, be32_to_cpu(hdr->offset),
687 						payload_length);
688 }
689 
690 /*
691  *	Called from iscsit_handle_data_out() before DataOUT Payload is received
692  *	and CRC computed.
693  */
694 int iscsit_check_pre_dataout(
695 	struct iscsi_cmd *cmd,
696 	unsigned char *buf)
697 {
698 	int ret;
699 	struct iscsi_conn *conn = cmd->conn;
700 
701 	ret = iscsit_dataout_within_command_recovery_check(cmd, buf);
702 	if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
703 	    (ret == DATAOUT_CANNOT_RECOVER))
704 		return ret;
705 
706 	ret = iscsit_dataout_check_datasn(cmd, buf);
707 	if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
708 	    (ret == DATAOUT_CANNOT_RECOVER))
709 		return ret;
710 
711 	if (cmd->unsolicited_data) {
712 		ret = iscsit_dataout_check_unsolicited_sequence(cmd, buf);
713 		if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
714 		    (ret == DATAOUT_CANNOT_RECOVER))
715 			return ret;
716 	} else {
717 		ret = iscsit_dataout_check_sequence(cmd, buf);
718 		if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
719 		    (ret == DATAOUT_CANNOT_RECOVER))
720 			return ret;
721 	}
722 
723 	return (conn->sess->sess_ops->DataPDUInOrder) ?
724 		iscsit_dataout_pre_datapduinorder_yes(cmd, buf) :
725 		iscsit_dataout_pre_datapduinorder_no(cmd, buf);
726 }
727 
728 /*
729  *	Called from iscsit_handle_data_out() after DataOUT Payload is received
730  *	and CRC computed.
731  */
732 int iscsit_check_post_dataout(
733 	struct iscsi_cmd *cmd,
734 	unsigned char *buf,
735 	u8 data_crc_failed)
736 {
737 	struct iscsi_conn *conn = cmd->conn;
738 
739 	cmd->dataout_timeout_retries = 0;
740 
741 	if (!data_crc_failed)
742 		return iscsit_dataout_post_crc_passed(cmd, buf);
743 	else {
744 		if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
745 			pr_err("Unable to recover from DataOUT CRC"
746 				" failure while ERL=0, closing session.\n");
747 			iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
748 					  buf);
749 			return DATAOUT_CANNOT_RECOVER;
750 		}
751 
752 		iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, buf);
753 		return iscsit_dataout_post_crc_failed(cmd, buf);
754 	}
755 }
756 
757 static void iscsit_handle_time2retain_timeout(unsigned long data)
758 {
759 	struct iscsi_session *sess = (struct iscsi_session *) data;
760 	struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
761 	struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
762 
763 	spin_lock_bh(&se_tpg->session_lock);
764 	if (sess->time2retain_timer_flags & ISCSI_TF_STOP) {
765 		spin_unlock_bh(&se_tpg->session_lock);
766 		return;
767 	}
768 	if (atomic_read(&sess->session_reinstatement)) {
769 		pr_err("Exiting Time2Retain handler because"
770 				" session_reinstatement=1\n");
771 		spin_unlock_bh(&se_tpg->session_lock);
772 		return;
773 	}
774 	sess->time2retain_timer_flags |= ISCSI_TF_EXPIRED;
775 
776 	pr_err("Time2Retain timer expired for SID: %u, cleaning up"
777 			" iSCSI session.\n", sess->sid);
778 	{
779 	struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
780 
781 	if (tiqn) {
782 		spin_lock(&tiqn->sess_err_stats.lock);
783 		strcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
784 			(void *)sess->sess_ops->InitiatorName);
785 		tiqn->sess_err_stats.last_sess_failure_type =
786 				ISCSI_SESS_ERR_CXN_TIMEOUT;
787 		tiqn->sess_err_stats.cxn_timeout_errors++;
788 		sess->conn_timeout_errors++;
789 		spin_unlock(&tiqn->sess_err_stats.lock);
790 	}
791 	}
792 
793 	spin_unlock_bh(&se_tpg->session_lock);
794 	target_put_session(sess->se_sess);
795 }
796 
797 void iscsit_start_time2retain_handler(struct iscsi_session *sess)
798 {
799 	int tpg_active;
800 	/*
801 	 * Only start Time2Retain timer when the associated TPG is still in
802 	 * an ACTIVE (eg: not disabled or shutdown) state.
803 	 */
804 	spin_lock(&ISCSI_TPG_S(sess)->tpg_state_lock);
805 	tpg_active = (ISCSI_TPG_S(sess)->tpg_state == TPG_STATE_ACTIVE);
806 	spin_unlock(&ISCSI_TPG_S(sess)->tpg_state_lock);
807 
808 	if (!tpg_active)
809 		return;
810 
811 	if (sess->time2retain_timer_flags & ISCSI_TF_RUNNING)
812 		return;
813 
814 	pr_debug("Starting Time2Retain timer for %u seconds on"
815 		" SID: %u\n", sess->sess_ops->DefaultTime2Retain, sess->sid);
816 
817 	init_timer(&sess->time2retain_timer);
818 	sess->time2retain_timer.expires =
819 		(get_jiffies_64() + sess->sess_ops->DefaultTime2Retain * HZ);
820 	sess->time2retain_timer.data = (unsigned long)sess;
821 	sess->time2retain_timer.function = iscsit_handle_time2retain_timeout;
822 	sess->time2retain_timer_flags &= ~ISCSI_TF_STOP;
823 	sess->time2retain_timer_flags |= ISCSI_TF_RUNNING;
824 	add_timer(&sess->time2retain_timer);
825 }
826 
827 /*
828  *	Called with spin_lock_bh(&struct se_portal_group->session_lock) held
829  */
830 int iscsit_stop_time2retain_timer(struct iscsi_session *sess)
831 {
832 	struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
833 	struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
834 
835 	if (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)
836 		return -1;
837 
838 	if (!(sess->time2retain_timer_flags & ISCSI_TF_RUNNING))
839 		return 0;
840 
841 	sess->time2retain_timer_flags |= ISCSI_TF_STOP;
842 	spin_unlock(&se_tpg->session_lock);
843 
844 	del_timer_sync(&sess->time2retain_timer);
845 
846 	spin_lock(&se_tpg->session_lock);
847 	sess->time2retain_timer_flags &= ~ISCSI_TF_RUNNING;
848 	pr_debug("Stopped Time2Retain Timer for SID: %u\n",
849 			sess->sid);
850 	return 0;
851 }
852 
853 void iscsit_connection_reinstatement_rcfr(struct iscsi_conn *conn)
854 {
855 	spin_lock_bh(&conn->state_lock);
856 	if (atomic_read(&conn->connection_exit)) {
857 		spin_unlock_bh(&conn->state_lock);
858 		goto sleep;
859 	}
860 
861 	if (atomic_read(&conn->transport_failed)) {
862 		spin_unlock_bh(&conn->state_lock);
863 		goto sleep;
864 	}
865 	spin_unlock_bh(&conn->state_lock);
866 
867 	iscsi_thread_set_force_reinstatement(conn);
868 
869 sleep:
870 	wait_for_completion(&conn->conn_wait_rcfr_comp);
871 	complete(&conn->conn_post_wait_comp);
872 }
873 
874 void iscsit_cause_connection_reinstatement(struct iscsi_conn *conn, int sleep)
875 {
876 	spin_lock_bh(&conn->state_lock);
877 	if (atomic_read(&conn->connection_exit)) {
878 		spin_unlock_bh(&conn->state_lock);
879 		return;
880 	}
881 
882 	if (atomic_read(&conn->transport_failed)) {
883 		spin_unlock_bh(&conn->state_lock);
884 		return;
885 	}
886 
887 	if (atomic_read(&conn->connection_reinstatement)) {
888 		spin_unlock_bh(&conn->state_lock);
889 		return;
890 	}
891 
892 	if (iscsi_thread_set_force_reinstatement(conn) < 0) {
893 		spin_unlock_bh(&conn->state_lock);
894 		return;
895 	}
896 
897 	atomic_set(&conn->connection_reinstatement, 1);
898 	if (!sleep) {
899 		spin_unlock_bh(&conn->state_lock);
900 		return;
901 	}
902 
903 	atomic_set(&conn->sleep_on_conn_wait_comp, 1);
904 	spin_unlock_bh(&conn->state_lock);
905 
906 	wait_for_completion(&conn->conn_wait_comp);
907 	complete(&conn->conn_post_wait_comp);
908 }
909 EXPORT_SYMBOL(iscsit_cause_connection_reinstatement);
910 
911 void iscsit_fall_back_to_erl0(struct iscsi_session *sess)
912 {
913 	pr_debug("Falling back to ErrorRecoveryLevel=0 for SID:"
914 			" %u\n", sess->sid);
915 
916 	atomic_set(&sess->session_fall_back_to_erl0, 1);
917 }
918 
919 static void iscsit_handle_connection_cleanup(struct iscsi_conn *conn)
920 {
921 	struct iscsi_session *sess = conn->sess;
922 
923 	if ((sess->sess_ops->ErrorRecoveryLevel == 2) &&
924 	    !atomic_read(&sess->session_reinstatement) &&
925 	    !atomic_read(&sess->session_fall_back_to_erl0))
926 		iscsit_connection_recovery_transport_reset(conn);
927 	else {
928 		pr_debug("Performing cleanup for failed iSCSI"
929 			" Connection ID: %hu from %s\n", conn->cid,
930 			sess->sess_ops->InitiatorName);
931 		iscsit_close_connection(conn);
932 	}
933 }
934 
935 void iscsit_take_action_for_connection_exit(struct iscsi_conn *conn)
936 {
937 	spin_lock_bh(&conn->state_lock);
938 	if (atomic_read(&conn->connection_exit)) {
939 		spin_unlock_bh(&conn->state_lock);
940 		return;
941 	}
942 	atomic_set(&conn->connection_exit, 1);
943 
944 	if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) {
945 		spin_unlock_bh(&conn->state_lock);
946 		iscsit_close_connection(conn);
947 		return;
948 	}
949 
950 	if (conn->conn_state == TARG_CONN_STATE_CLEANUP_WAIT) {
951 		spin_unlock_bh(&conn->state_lock);
952 		return;
953 	}
954 
955 	pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
956 	conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
957 	spin_unlock_bh(&conn->state_lock);
958 
959 	iscsit_handle_connection_cleanup(conn);
960 }
961 
962 /*
963  *	This is the simple function that makes the magic of
964  *	sync and steering happen in the follow paradoxical order:
965  *
966  *	0) Receive conn->of_marker (bytes left until next OFMarker)
967  *	   bytes into an offload buffer.  When we pass the exact number
968  *	   of bytes in conn->of_marker, iscsit_dump_data_payload() and hence
969  *	   rx_data() will automatically receive the identical u32 marker
970  *	   values and store it in conn->of_marker_offset;
971  *	1) Now conn->of_marker_offset will contain the offset to the start
972  *	   of the next iSCSI PDU.  Dump these remaining bytes into another
973  *	   offload buffer.
974  *	2) We are done!
975  *	   Next byte in the TCP stream will contain the next iSCSI PDU!
976  *	   Cool Huh?!
977  */
978 int iscsit_recover_from_unknown_opcode(struct iscsi_conn *conn)
979 {
980 	/*
981 	 * Make sure the remaining bytes to next maker is a sane value.
982 	 */
983 	if (conn->of_marker > (conn->conn_ops->OFMarkInt * 4)) {
984 		pr_err("Remaining bytes to OFMarker: %u exceeds"
985 			" OFMarkInt bytes: %u.\n", conn->of_marker,
986 				conn->conn_ops->OFMarkInt * 4);
987 		return -1;
988 	}
989 
990 	pr_debug("Advancing %u bytes in TCP stream to get to the"
991 			" next OFMarker.\n", conn->of_marker);
992 
993 	if (iscsit_dump_data_payload(conn, conn->of_marker, 0) < 0)
994 		return -1;
995 
996 	/*
997 	 * Make sure the offset marker we retrived is a valid value.
998 	 */
999 	if (conn->of_marker_offset > (ISCSI_HDR_LEN + (ISCSI_CRC_LEN * 2) +
1000 	    conn->conn_ops->MaxRecvDataSegmentLength)) {
1001 		pr_err("OfMarker offset value: %u exceeds limit.\n",
1002 			conn->of_marker_offset);
1003 		return -1;
1004 	}
1005 
1006 	pr_debug("Discarding %u bytes of TCP stream to get to the"
1007 			" next iSCSI Opcode.\n", conn->of_marker_offset);
1008 
1009 	if (iscsit_dump_data_payload(conn, conn->of_marker_offset, 0) < 0)
1010 		return -1;
1011 
1012 	return 0;
1013 }
1014