1 /*******************************************************************************
2  * This file contains the iSCSI Target specific Task Management functions.
3  *
4  * (c) Copyright 2007-2013 Datera, Inc.
5  *
6  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16  * GNU General Public License for more details.
17  ******************************************************************************/
18 
19 #include <asm/unaligned.h>
20 #include <scsi/scsi_proto.h>
21 #include <scsi/iscsi_proto.h>
22 #include <target/target_core_base.h>
23 #include <target/target_core_fabric.h>
24 #include <target/iscsi/iscsi_transport.h>
25 
26 #include <target/iscsi/iscsi_target_core.h>
27 #include "iscsi_target_seq_pdu_list.h"
28 #include "iscsi_target_datain_values.h"
29 #include "iscsi_target_device.h"
30 #include "iscsi_target_erl0.h"
31 #include "iscsi_target_erl1.h"
32 #include "iscsi_target_erl2.h"
33 #include "iscsi_target_tmr.h"
34 #include "iscsi_target_tpg.h"
35 #include "iscsi_target_util.h"
36 #include "iscsi_target.h"
37 
38 u8 iscsit_tmr_abort_task(
39 	struct iscsi_cmd *cmd,
40 	unsigned char *buf)
41 {
42 	struct iscsi_cmd *ref_cmd;
43 	struct iscsi_conn *conn = cmd->conn;
44 	struct iscsi_tmr_req *tmr_req = cmd->tmr_req;
45 	struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
46 	struct iscsi_tm *hdr = (struct iscsi_tm *) buf;
47 
48 	ref_cmd = iscsit_find_cmd_from_itt(conn, hdr->rtt);
49 	if (!ref_cmd) {
50 		pr_err("Unable to locate RefTaskTag: 0x%08x on CID:"
51 			" %hu.\n", hdr->rtt, conn->cid);
52 		return (iscsi_sna_gte(be32_to_cpu(hdr->refcmdsn), conn->sess->exp_cmd_sn) &&
53 			iscsi_sna_lte(be32_to_cpu(hdr->refcmdsn), (u32) atomic_read(&conn->sess->max_cmd_sn))) ?
54 			ISCSI_TMF_RSP_COMPLETE : ISCSI_TMF_RSP_NO_TASK;
55 	}
56 	if (ref_cmd->cmd_sn != be32_to_cpu(hdr->refcmdsn)) {
57 		pr_err("RefCmdSN 0x%08x does not equal"
58 			" task's CmdSN 0x%08x. Rejecting ABORT_TASK.\n",
59 			hdr->refcmdsn, ref_cmd->cmd_sn);
60 		return ISCSI_TMF_RSP_REJECTED;
61 	}
62 
63 	se_tmr->ref_task_tag		= (__force u32)hdr->rtt;
64 	tmr_req->ref_cmd		= ref_cmd;
65 	tmr_req->exp_data_sn		= be32_to_cpu(hdr->exp_datasn);
66 
67 	return ISCSI_TMF_RSP_COMPLETE;
68 }
69 
70 /*
71  *	Called from iscsit_handle_task_mgt_cmd().
72  */
73 int iscsit_tmr_task_warm_reset(
74 	struct iscsi_conn *conn,
75 	struct iscsi_tmr_req *tmr_req,
76 	unsigned char *buf)
77 {
78 	struct iscsi_session *sess = conn->sess;
79 	struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
80 
81 	if (!na->tmr_warm_reset) {
82 		pr_err("TMR Opcode TARGET_WARM_RESET authorization"
83 			" failed for Initiator Node: %s\n",
84 			sess->se_sess->se_node_acl->initiatorname);
85 		return -1;
86 	}
87 	/*
88 	 * Do the real work in transport_generic_do_tmr().
89 	 */
90 	return 0;
91 }
92 
93 int iscsit_tmr_task_cold_reset(
94 	struct iscsi_conn *conn,
95 	struct iscsi_tmr_req *tmr_req,
96 	unsigned char *buf)
97 {
98 	struct iscsi_session *sess = conn->sess;
99 	struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
100 
101 	if (!na->tmr_cold_reset) {
102 		pr_err("TMR Opcode TARGET_COLD_RESET authorization"
103 			" failed for Initiator Node: %s\n",
104 			sess->se_sess->se_node_acl->initiatorname);
105 		return -1;
106 	}
107 	/*
108 	 * Do the real work in transport_generic_do_tmr().
109 	 */
110 	return 0;
111 }
112 
113 u8 iscsit_tmr_task_reassign(
114 	struct iscsi_cmd *cmd,
115 	unsigned char *buf)
116 {
117 	struct iscsi_cmd *ref_cmd = NULL;
118 	struct iscsi_conn *conn = cmd->conn;
119 	struct iscsi_conn_recovery *cr = NULL;
120 	struct iscsi_tmr_req *tmr_req = cmd->tmr_req;
121 	struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
122 	struct iscsi_tm *hdr = (struct iscsi_tm *) buf;
123 	u64 ret, ref_lun;
124 
125 	pr_debug("Got TASK_REASSIGN TMR ITT: 0x%08x,"
126 		" RefTaskTag: 0x%08x, ExpDataSN: 0x%08x, CID: %hu\n",
127 		hdr->itt, hdr->rtt, hdr->exp_datasn, conn->cid);
128 
129 	if (conn->sess->sess_ops->ErrorRecoveryLevel != 2) {
130 		pr_err("TMR TASK_REASSIGN not supported in ERL<2,"
131 				" ignoring request.\n");
132 		return ISCSI_TMF_RSP_NOT_SUPPORTED;
133 	}
134 
135 	ret = iscsit_find_cmd_for_recovery(conn->sess, &ref_cmd, &cr, hdr->rtt);
136 	if (ret == -2) {
137 		pr_err("Command ITT: 0x%08x is still alligent to CID:"
138 			" %hu\n", ref_cmd->init_task_tag, cr->cid);
139 		return ISCSI_TMF_RSP_TASK_ALLEGIANT;
140 	} else if (ret == -1) {
141 		pr_err("Unable to locate RefTaskTag: 0x%08x in"
142 			" connection recovery command list.\n", hdr->rtt);
143 		return ISCSI_TMF_RSP_NO_TASK;
144 	}
145 	/*
146 	 * Temporary check to prevent connection recovery for
147 	 * connections with a differing Max*DataSegmentLength.
148 	 */
149 	if (cr->maxrecvdatasegmentlength !=
150 	    conn->conn_ops->MaxRecvDataSegmentLength) {
151 		pr_err("Unable to perform connection recovery for"
152 			" differing MaxRecvDataSegmentLength, rejecting"
153 			" TMR TASK_REASSIGN.\n");
154 		return ISCSI_TMF_RSP_REJECTED;
155 	}
156 	if (cr->maxxmitdatasegmentlength !=
157 	    conn->conn_ops->MaxXmitDataSegmentLength) {
158 		pr_err("Unable to perform connection recovery for"
159 			" differing MaxXmitDataSegmentLength, rejecting"
160 			" TMR TASK_REASSIGN.\n");
161 		return ISCSI_TMF_RSP_REJECTED;
162 	}
163 
164 	ref_lun = scsilun_to_int(&hdr->lun);
165 	if (ref_lun != ref_cmd->se_cmd.orig_fe_lun) {
166 		pr_err("Unable to perform connection recovery for"
167 			" differing ref_lun: %llu ref_cmd orig_fe_lun: %llu\n",
168 			ref_lun, ref_cmd->se_cmd.orig_fe_lun);
169 		return ISCSI_TMF_RSP_REJECTED;
170 	}
171 
172 	se_tmr->ref_task_tag		= (__force u32)hdr->rtt;
173 	tmr_req->ref_cmd		= ref_cmd;
174 	tmr_req->exp_data_sn		= be32_to_cpu(hdr->exp_datasn);
175 	tmr_req->conn_recovery		= cr;
176 	tmr_req->task_reassign		= 1;
177 	/*
178 	 * Command can now be reassigned to a new connection.
179 	 * The task management response must be sent before the
180 	 * reassignment actually happens.  See iscsi_tmr_post_handler().
181 	 */
182 	return ISCSI_TMF_RSP_COMPLETE;
183 }
184 
185 static void iscsit_task_reassign_remove_cmd(
186 	struct iscsi_cmd *cmd,
187 	struct iscsi_conn_recovery *cr,
188 	struct iscsi_session *sess)
189 {
190 	int ret;
191 
192 	spin_lock(&cr->conn_recovery_cmd_lock);
193 	ret = iscsit_remove_cmd_from_connection_recovery(cmd, sess);
194 	spin_unlock(&cr->conn_recovery_cmd_lock);
195 	if (!ret) {
196 		pr_debug("iSCSI connection recovery successful for CID:"
197 			" %hu on SID: %u\n", cr->cid, sess->sid);
198 		iscsit_remove_active_connection_recovery_entry(cr, sess);
199 	}
200 }
201 
202 static int iscsit_task_reassign_complete_nop_out(
203 	struct iscsi_tmr_req *tmr_req,
204 	struct iscsi_conn *conn)
205 {
206 	struct iscsi_cmd *cmd = tmr_req->ref_cmd;
207 	struct iscsi_conn_recovery *cr;
208 
209 	if (!cmd->cr) {
210 		pr_err("struct iscsi_conn_recovery pointer for ITT: 0x%08x"
211 			" is NULL!\n", cmd->init_task_tag);
212 		return -1;
213 	}
214 	cr = cmd->cr;
215 
216 	/*
217 	 * Reset the StatSN so a new one for this commands new connection
218 	 * will be assigned.
219 	 * Reset the ExpStatSN as well so we may receive Status SNACKs.
220 	 */
221 	cmd->stat_sn = cmd->exp_stat_sn = 0;
222 
223 	iscsit_task_reassign_remove_cmd(cmd, cr, conn->sess);
224 
225 	spin_lock_bh(&conn->cmd_lock);
226 	list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
227 	spin_unlock_bh(&conn->cmd_lock);
228 
229 	cmd->i_state = ISTATE_SEND_NOPIN;
230 	iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
231 	return 0;
232 }
233 
234 static int iscsit_task_reassign_complete_write(
235 	struct iscsi_cmd *cmd,
236 	struct iscsi_tmr_req *tmr_req)
237 {
238 	int no_build_r2ts = 0;
239 	u32 length = 0, offset = 0;
240 	struct iscsi_conn *conn = cmd->conn;
241 	struct se_cmd *se_cmd = &cmd->se_cmd;
242 	/*
243 	 * The Initiator must not send a R2T SNACK with a Begrun less than
244 	 * the TMR TASK_REASSIGN's ExpDataSN.
245 	 */
246 	if (!tmr_req->exp_data_sn) {
247 		cmd->cmd_flags &= ~ICF_GOT_DATACK_SNACK;
248 		cmd->acked_data_sn = 0;
249 	} else {
250 		cmd->cmd_flags |= ICF_GOT_DATACK_SNACK;
251 		cmd->acked_data_sn = (tmr_req->exp_data_sn - 1);
252 	}
253 
254 	/*
255 	 * The TMR TASK_REASSIGN's ExpDataSN contains the next R2TSN the
256 	 * Initiator is expecting.  The Target controls all WRITE operations
257 	 * so if we have received all DataOUT we can safety ignore Initiator.
258 	 */
259 	if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
260 		if (!(cmd->se_cmd.transport_state & CMD_T_SENT)) {
261 			pr_debug("WRITE ITT: 0x%08x: t_state: %d"
262 				" never sent to transport\n",
263 				cmd->init_task_tag, cmd->se_cmd.t_state);
264 			target_execute_cmd(se_cmd);
265 			return 0;
266 		}
267 
268 		cmd->i_state = ISTATE_SEND_STATUS;
269 		iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
270 		return 0;
271 	}
272 
273 	/*
274 	 * Special case to deal with DataSequenceInOrder=No and Non-Immeidate
275 	 * Unsolicited DataOut.
276 	 */
277 	if (cmd->unsolicited_data) {
278 		cmd->unsolicited_data = 0;
279 
280 		offset = cmd->next_burst_len = cmd->write_data_done;
281 
282 		if ((conn->sess->sess_ops->FirstBurstLength - offset) >=
283 		     cmd->se_cmd.data_length) {
284 			no_build_r2ts = 1;
285 			length = (cmd->se_cmd.data_length - offset);
286 		} else
287 			length = (conn->sess->sess_ops->FirstBurstLength - offset);
288 
289 		spin_lock_bh(&cmd->r2t_lock);
290 		if (iscsit_add_r2t_to_list(cmd, offset, length, 0, 0) < 0) {
291 			spin_unlock_bh(&cmd->r2t_lock);
292 			return -1;
293 		}
294 		cmd->outstanding_r2ts++;
295 		spin_unlock_bh(&cmd->r2t_lock);
296 
297 		if (no_build_r2ts)
298 			return 0;
299 	}
300 	/*
301 	 * iscsit_build_r2ts_for_cmd() can handle the rest from here.
302 	 */
303 	return conn->conn_transport->iscsit_get_dataout(conn, cmd, true);
304 }
305 
306 static int iscsit_task_reassign_complete_read(
307 	struct iscsi_cmd *cmd,
308 	struct iscsi_tmr_req *tmr_req)
309 {
310 	struct iscsi_conn *conn = cmd->conn;
311 	struct iscsi_datain_req *dr;
312 	struct se_cmd *se_cmd = &cmd->se_cmd;
313 	/*
314 	 * The Initiator must not send a Data SNACK with a BegRun less than
315 	 * the TMR TASK_REASSIGN's ExpDataSN.
316 	 */
317 	if (!tmr_req->exp_data_sn) {
318 		cmd->cmd_flags &= ~ICF_GOT_DATACK_SNACK;
319 		cmd->acked_data_sn = 0;
320 	} else {
321 		cmd->cmd_flags |= ICF_GOT_DATACK_SNACK;
322 		cmd->acked_data_sn = (tmr_req->exp_data_sn - 1);
323 	}
324 
325 	if (!(cmd->se_cmd.transport_state & CMD_T_SENT)) {
326 		pr_debug("READ ITT: 0x%08x: t_state: %d never sent to"
327 			" transport\n", cmd->init_task_tag,
328 			cmd->se_cmd.t_state);
329 		transport_handle_cdb_direct(se_cmd);
330 		return 0;
331 	}
332 
333 	if (!(se_cmd->transport_state & CMD_T_COMPLETE)) {
334 		pr_err("READ ITT: 0x%08x: t_state: %d, never returned"
335 			" from transport\n", cmd->init_task_tag,
336 			cmd->se_cmd.t_state);
337 		return -1;
338 	}
339 
340 	dr = iscsit_allocate_datain_req();
341 	if (!dr)
342 		return -1;
343 	/*
344 	 * The TMR TASK_REASSIGN's ExpDataSN contains the next DataSN the
345 	 * Initiator is expecting.
346 	 */
347 	dr->data_sn = dr->begrun = tmr_req->exp_data_sn;
348 	dr->runlength = 0;
349 	dr->generate_recovery_values = 1;
350 	dr->recovery = DATAIN_CONNECTION_RECOVERY;
351 
352 	iscsit_attach_datain_req(cmd, dr);
353 
354 	cmd->i_state = ISTATE_SEND_DATAIN;
355 	iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
356 	return 0;
357 }
358 
359 static int iscsit_task_reassign_complete_none(
360 	struct iscsi_cmd *cmd,
361 	struct iscsi_tmr_req *tmr_req)
362 {
363 	struct iscsi_conn *conn = cmd->conn;
364 
365 	cmd->i_state = ISTATE_SEND_STATUS;
366 	iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
367 	return 0;
368 }
369 
370 static int iscsit_task_reassign_complete_scsi_cmnd(
371 	struct iscsi_tmr_req *tmr_req,
372 	struct iscsi_conn *conn)
373 {
374 	struct iscsi_cmd *cmd = tmr_req->ref_cmd;
375 	struct iscsi_conn_recovery *cr;
376 
377 	if (!cmd->cr) {
378 		pr_err("struct iscsi_conn_recovery pointer for ITT: 0x%08x"
379 			" is NULL!\n", cmd->init_task_tag);
380 		return -1;
381 	}
382 	cr = cmd->cr;
383 
384 	/*
385 	 * Reset the StatSN so a new one for this commands new connection
386 	 * will be assigned.
387 	 * Reset the ExpStatSN as well so we may receive Status SNACKs.
388 	 */
389 	cmd->stat_sn = cmd->exp_stat_sn = 0;
390 
391 	iscsit_task_reassign_remove_cmd(cmd, cr, conn->sess);
392 
393 	spin_lock_bh(&conn->cmd_lock);
394 	list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
395 	spin_unlock_bh(&conn->cmd_lock);
396 
397 	if (cmd->se_cmd.se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
398 		cmd->i_state = ISTATE_SEND_STATUS;
399 		iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
400 		return 0;
401 	}
402 
403 	switch (cmd->data_direction) {
404 	case DMA_TO_DEVICE:
405 		return iscsit_task_reassign_complete_write(cmd, tmr_req);
406 	case DMA_FROM_DEVICE:
407 		return iscsit_task_reassign_complete_read(cmd, tmr_req);
408 	case DMA_NONE:
409 		return iscsit_task_reassign_complete_none(cmd, tmr_req);
410 	default:
411 		pr_err("Unknown cmd->data_direction: 0x%02x\n",
412 				cmd->data_direction);
413 		return -1;
414 	}
415 
416 	return 0;
417 }
418 
419 static int iscsit_task_reassign_complete(
420 	struct iscsi_tmr_req *tmr_req,
421 	struct iscsi_conn *conn)
422 {
423 	struct iscsi_cmd *cmd;
424 	int ret = 0;
425 
426 	if (!tmr_req->ref_cmd) {
427 		pr_err("TMR Request is missing a RefCmd struct iscsi_cmd.\n");
428 		return -1;
429 	}
430 	cmd = tmr_req->ref_cmd;
431 
432 	cmd->conn = conn;
433 
434 	switch (cmd->iscsi_opcode) {
435 	case ISCSI_OP_NOOP_OUT:
436 		ret = iscsit_task_reassign_complete_nop_out(tmr_req, conn);
437 		break;
438 	case ISCSI_OP_SCSI_CMD:
439 		ret = iscsit_task_reassign_complete_scsi_cmnd(tmr_req, conn);
440 		break;
441 	default:
442 		 pr_err("Illegal iSCSI Opcode 0x%02x during"
443 			" command reallegiance\n", cmd->iscsi_opcode);
444 		return -1;
445 	}
446 
447 	if (ret != 0)
448 		return ret;
449 
450 	pr_debug("Completed connection reallegiance for Opcode: 0x%02x,"
451 		" ITT: 0x%08x to CID: %hu.\n", cmd->iscsi_opcode,
452 			cmd->init_task_tag, conn->cid);
453 
454 	return 0;
455 }
456 
457 /*
458  *	Handles special after-the-fact actions related to TMRs.
459  *	Right now the only one that its really needed for is
460  *	connection recovery releated TASK_REASSIGN.
461  */
462 int iscsit_tmr_post_handler(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
463 {
464 	struct iscsi_tmr_req *tmr_req = cmd->tmr_req;
465 	struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
466 
467 	if (tmr_req->task_reassign &&
468 	   (se_tmr->response == ISCSI_TMF_RSP_COMPLETE))
469 		return iscsit_task_reassign_complete(tmr_req, conn);
470 
471 	return 0;
472 }
473 EXPORT_SYMBOL(iscsit_tmr_post_handler);
474 
475 /*
476  *	Nothing to do here, but leave it for good measure. :-)
477  */
478 static int iscsit_task_reassign_prepare_read(
479 	struct iscsi_tmr_req *tmr_req,
480 	struct iscsi_conn *conn)
481 {
482 	return 0;
483 }
484 
485 static void iscsit_task_reassign_prepare_unsolicited_dataout(
486 	struct iscsi_cmd *cmd,
487 	struct iscsi_conn *conn)
488 {
489 	int i, j;
490 	struct iscsi_pdu *pdu = NULL;
491 	struct iscsi_seq *seq = NULL;
492 
493 	if (conn->sess->sess_ops->DataSequenceInOrder) {
494 		cmd->data_sn = 0;
495 
496 		if (cmd->immediate_data)
497 			cmd->r2t_offset += (cmd->first_burst_len -
498 				cmd->seq_start_offset);
499 
500 		if (conn->sess->sess_ops->DataPDUInOrder) {
501 			cmd->write_data_done -= (cmd->immediate_data) ?
502 						(cmd->first_burst_len -
503 						 cmd->seq_start_offset) :
504 						 cmd->first_burst_len;
505 			cmd->first_burst_len = 0;
506 			return;
507 		}
508 
509 		for (i = 0; i < cmd->pdu_count; i++) {
510 			pdu = &cmd->pdu_list[i];
511 
512 			if (pdu->status != ISCSI_PDU_RECEIVED_OK)
513 				continue;
514 
515 			if ((pdu->offset >= cmd->seq_start_offset) &&
516 			   ((pdu->offset + pdu->length) <=
517 			     cmd->seq_end_offset)) {
518 				cmd->first_burst_len -= pdu->length;
519 				cmd->write_data_done -= pdu->length;
520 				pdu->status = ISCSI_PDU_NOT_RECEIVED;
521 			}
522 		}
523 	} else {
524 		for (i = 0; i < cmd->seq_count; i++) {
525 			seq = &cmd->seq_list[i];
526 
527 			if (seq->type != SEQTYPE_UNSOLICITED)
528 				continue;
529 
530 			cmd->write_data_done -=
531 					(seq->offset - seq->orig_offset);
532 			cmd->first_burst_len = 0;
533 			seq->data_sn = 0;
534 			seq->offset = seq->orig_offset;
535 			seq->next_burst_len = 0;
536 			seq->status = DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY;
537 
538 			if (conn->sess->sess_ops->DataPDUInOrder)
539 				continue;
540 
541 			for (j = 0; j < seq->pdu_count; j++) {
542 				pdu = &cmd->pdu_list[j+seq->pdu_start];
543 
544 				if (pdu->status != ISCSI_PDU_RECEIVED_OK)
545 					continue;
546 
547 				pdu->status = ISCSI_PDU_NOT_RECEIVED;
548 			}
549 		}
550 	}
551 }
552 
553 static int iscsit_task_reassign_prepare_write(
554 	struct iscsi_tmr_req *tmr_req,
555 	struct iscsi_conn *conn)
556 {
557 	struct iscsi_cmd *cmd = tmr_req->ref_cmd;
558 	struct iscsi_pdu *pdu = NULL;
559 	struct iscsi_r2t *r2t = NULL, *r2t_tmp;
560 	int first_incomplete_r2t = 1, i = 0;
561 
562 	/*
563 	 * The command was in the process of receiving Unsolicited DataOUT when
564 	 * the connection failed.
565 	 */
566 	if (cmd->unsolicited_data)
567 		iscsit_task_reassign_prepare_unsolicited_dataout(cmd, conn);
568 
569 	/*
570 	 * The Initiator is requesting R2Ts starting from zero,  skip
571 	 * checking acknowledged R2Ts and start checking struct iscsi_r2ts
572 	 * greater than zero.
573 	 */
574 	if (!tmr_req->exp_data_sn)
575 		goto drop_unacknowledged_r2ts;
576 
577 	/*
578 	 * We now check that the PDUs in DataOUT sequences below
579 	 * the TMR TASK_REASSIGN ExpDataSN (R2TSN the Initiator is
580 	 * expecting next) have all the DataOUT they require to complete
581 	 * the DataOUT sequence.  First scan from R2TSN 0 to TMR
582 	 * TASK_REASSIGN ExpDataSN-1.
583 	 *
584 	 * If we have not received all DataOUT in question,  we must
585 	 * make sure to make the appropriate changes to values in
586 	 * struct iscsi_cmd (and elsewhere depending on session parameters)
587 	 * so iscsit_build_r2ts_for_cmd() in iscsit_task_reassign_complete_write()
588 	 * will resend a new R2T for the DataOUT sequences in question.
589 	 */
590 	spin_lock_bh(&cmd->r2t_lock);
591 	if (list_empty(&cmd->cmd_r2t_list)) {
592 		spin_unlock_bh(&cmd->r2t_lock);
593 		return -1;
594 	}
595 
596 	list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
597 
598 		if (r2t->r2t_sn >= tmr_req->exp_data_sn)
599 			continue;
600 		/*
601 		 * Safely ignore Recovery R2Ts and R2Ts that have completed
602 		 * DataOUT sequences.
603 		 */
604 		if (r2t->seq_complete)
605 			continue;
606 
607 		if (r2t->recovery_r2t)
608 			continue;
609 
610 		/*
611 		 *                 DataSequenceInOrder=Yes:
612 		 *
613 		 * Taking into account the iSCSI implementation requirement of
614 		 * MaxOutstandingR2T=1 while ErrorRecoveryLevel>0 and
615 		 * DataSequenceInOrder=Yes, we must take into consideration
616 		 * the following:
617 		 *
618 		 *                  DataSequenceInOrder=No:
619 		 *
620 		 * Taking into account that the Initiator controls the (possibly
621 		 * random) PDU Order in (possibly random) Sequence Order of
622 		 * DataOUT the target requests with R2Ts,  we must take into
623 		 * consideration the following:
624 		 *
625 		 *      DataPDUInOrder=Yes for DataSequenceInOrder=[Yes,No]:
626 		 *
627 		 * While processing non-complete R2T DataOUT sequence requests
628 		 * the Target will re-request only the total sequence length
629 		 * minus current received offset.  This is because we must
630 		 * assume the initiator will continue sending DataOUT from the
631 		 * last PDU before the connection failed.
632 		 *
633 		 *      DataPDUInOrder=No for DataSequenceInOrder=[Yes,No]:
634 		 *
635 		 * While processing non-complete R2T DataOUT sequence requests
636 		 * the Target will re-request the entire DataOUT sequence if
637 		 * any single PDU is missing from the sequence.  This is because
638 		 * we have no logical method to determine the next PDU offset,
639 		 * and we must assume the Initiator will be sending any random
640 		 * PDU offset in the current sequence after TASK_REASSIGN
641 		 * has completed.
642 		 */
643 		if (conn->sess->sess_ops->DataSequenceInOrder) {
644 			if (!first_incomplete_r2t) {
645 				cmd->r2t_offset -= r2t->xfer_len;
646 				goto next;
647 			}
648 
649 			if (conn->sess->sess_ops->DataPDUInOrder) {
650 				cmd->data_sn = 0;
651 				cmd->r2t_offset -= (r2t->xfer_len -
652 					cmd->next_burst_len);
653 				first_incomplete_r2t = 0;
654 				goto next;
655 			}
656 
657 			cmd->data_sn = 0;
658 			cmd->r2t_offset -= r2t->xfer_len;
659 
660 			for (i = 0; i < cmd->pdu_count; i++) {
661 				pdu = &cmd->pdu_list[i];
662 
663 				if (pdu->status != ISCSI_PDU_RECEIVED_OK)
664 					continue;
665 
666 				if ((pdu->offset >= r2t->offset) &&
667 				    (pdu->offset < (r2t->offset +
668 						r2t->xfer_len))) {
669 					cmd->next_burst_len -= pdu->length;
670 					cmd->write_data_done -= pdu->length;
671 					pdu->status = ISCSI_PDU_NOT_RECEIVED;
672 				}
673 			}
674 
675 			first_incomplete_r2t = 0;
676 		} else {
677 			struct iscsi_seq *seq;
678 
679 			seq = iscsit_get_seq_holder(cmd, r2t->offset,
680 					r2t->xfer_len);
681 			if (!seq) {
682 				spin_unlock_bh(&cmd->r2t_lock);
683 				return -1;
684 			}
685 
686 			cmd->write_data_done -=
687 					(seq->offset - seq->orig_offset);
688 			seq->data_sn = 0;
689 			seq->offset = seq->orig_offset;
690 			seq->next_burst_len = 0;
691 			seq->status = DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY;
692 
693 			cmd->seq_send_order--;
694 
695 			if (conn->sess->sess_ops->DataPDUInOrder)
696 				goto next;
697 
698 			for (i = 0; i < seq->pdu_count; i++) {
699 				pdu = &cmd->pdu_list[i+seq->pdu_start];
700 
701 				if (pdu->status != ISCSI_PDU_RECEIVED_OK)
702 					continue;
703 
704 				pdu->status = ISCSI_PDU_NOT_RECEIVED;
705 			}
706 		}
707 
708 next:
709 		cmd->outstanding_r2ts--;
710 	}
711 	spin_unlock_bh(&cmd->r2t_lock);
712 
713 	/*
714 	 * We now drop all unacknowledged R2Ts, ie: ExpDataSN from TMR
715 	 * TASK_REASSIGN to the last R2T in the list..  We are also careful
716 	 * to check that the Initiator is not requesting R2Ts for DataOUT
717 	 * sequences it has already completed.
718 	 *
719 	 * Free each R2T in question and adjust values in struct iscsi_cmd
720 	 * accordingly so iscsit_build_r2ts_for_cmd() do the rest of
721 	 * the work after the TMR TASK_REASSIGN Response is sent.
722 	 */
723 drop_unacknowledged_r2ts:
724 
725 	cmd->cmd_flags &= ~ICF_SENT_LAST_R2T;
726 	cmd->r2t_sn = tmr_req->exp_data_sn;
727 
728 	spin_lock_bh(&cmd->r2t_lock);
729 	list_for_each_entry_safe(r2t, r2t_tmp, &cmd->cmd_r2t_list, r2t_list) {
730 		/*
731 		 * Skip up to the R2T Sequence number provided by the
732 		 * iSCSI TASK_REASSIGN TMR
733 		 */
734 		if (r2t->r2t_sn < tmr_req->exp_data_sn)
735 			continue;
736 
737 		if (r2t->seq_complete) {
738 			pr_err("Initiator is requesting R2Ts from"
739 				" R2TSN: 0x%08x, but R2TSN: 0x%08x, Offset: %u,"
740 				" Length: %u is already complete."
741 				"   BAD INITIATOR ERL=2 IMPLEMENTATION!\n",
742 				tmr_req->exp_data_sn, r2t->r2t_sn,
743 				r2t->offset, r2t->xfer_len);
744 			spin_unlock_bh(&cmd->r2t_lock);
745 			return -1;
746 		}
747 
748 		if (r2t->recovery_r2t) {
749 			iscsit_free_r2t(r2t, cmd);
750 			continue;
751 		}
752 
753 		/*		   DataSequenceInOrder=Yes:
754 		 *
755 		 * Taking into account the iSCSI implementation requirement of
756 		 * MaxOutstandingR2T=1 while ErrorRecoveryLevel>0 and
757 		 * DataSequenceInOrder=Yes, it's safe to subtract the R2Ts
758 		 * entire transfer length from the commands R2T offset marker.
759 		 *
760 		 *		   DataSequenceInOrder=No:
761 		 *
762 		 * We subtract the difference from struct iscsi_seq between the
763 		 * current offset and original offset from cmd->write_data_done
764 		 * for account for DataOUT PDUs already received.  Then reset
765 		 * the current offset to the original and zero out the current
766 		 * burst length,  to make sure we re-request the entire DataOUT
767 		 * sequence.
768 		 */
769 		if (conn->sess->sess_ops->DataSequenceInOrder)
770 			cmd->r2t_offset -= r2t->xfer_len;
771 		else
772 			cmd->seq_send_order--;
773 
774 		cmd->outstanding_r2ts--;
775 		iscsit_free_r2t(r2t, cmd);
776 	}
777 	spin_unlock_bh(&cmd->r2t_lock);
778 
779 	return 0;
780 }
781 
782 /*
783  *	Performs sanity checks TMR TASK_REASSIGN's ExpDataSN for
784  *	a given struct iscsi_cmd.
785  */
786 int iscsit_check_task_reassign_expdatasn(
787 	struct iscsi_tmr_req *tmr_req,
788 	struct iscsi_conn *conn)
789 {
790 	struct iscsi_cmd *ref_cmd = tmr_req->ref_cmd;
791 
792 	if (ref_cmd->iscsi_opcode != ISCSI_OP_SCSI_CMD)
793 		return 0;
794 
795 	if (ref_cmd->se_cmd.se_cmd_flags & SCF_SENT_CHECK_CONDITION)
796 		return 0;
797 
798 	if (ref_cmd->data_direction == DMA_NONE)
799 		return 0;
800 
801 	/*
802 	 * For READs the TMR TASK_REASSIGNs ExpDataSN contains the next DataSN
803 	 * of DataIN the Initiator is expecting.
804 	 *
805 	 * Also check that the Initiator is not re-requesting DataIN that has
806 	 * already been acknowledged with a DataAck SNACK.
807 	 */
808 	if (ref_cmd->data_direction == DMA_FROM_DEVICE) {
809 		if (tmr_req->exp_data_sn > ref_cmd->data_sn) {
810 			pr_err("Received ExpDataSN: 0x%08x for READ"
811 				" in TMR TASK_REASSIGN greater than command's"
812 				" DataSN: 0x%08x.\n", tmr_req->exp_data_sn,
813 				ref_cmd->data_sn);
814 			return -1;
815 		}
816 		if ((ref_cmd->cmd_flags & ICF_GOT_DATACK_SNACK) &&
817 		    (tmr_req->exp_data_sn <= ref_cmd->acked_data_sn)) {
818 			pr_err("Received ExpDataSN: 0x%08x for READ"
819 				" in TMR TASK_REASSIGN for previously"
820 				" acknowledged DataIN: 0x%08x,"
821 				" protocol error\n", tmr_req->exp_data_sn,
822 				ref_cmd->acked_data_sn);
823 			return -1;
824 		}
825 		return iscsit_task_reassign_prepare_read(tmr_req, conn);
826 	}
827 
828 	/*
829 	 * For WRITEs the TMR TASK_REASSIGNs ExpDataSN contains the next R2TSN
830 	 * for R2Ts the Initiator is expecting.
831 	 *
832 	 * Do the magic in iscsit_task_reassign_prepare_write().
833 	 */
834 	if (ref_cmd->data_direction == DMA_TO_DEVICE) {
835 		if (tmr_req->exp_data_sn > ref_cmd->r2t_sn) {
836 			pr_err("Received ExpDataSN: 0x%08x for WRITE"
837 				" in TMR TASK_REASSIGN greater than command's"
838 				" R2TSN: 0x%08x.\n", tmr_req->exp_data_sn,
839 					ref_cmd->r2t_sn);
840 			return -1;
841 		}
842 		return iscsit_task_reassign_prepare_write(tmr_req, conn);
843 	}
844 
845 	pr_err("Unknown iSCSI data_direction: 0x%02x\n",
846 			ref_cmd->data_direction);
847 
848 	return -1;
849 }
850