1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*******************************************************************************
3  * This file contains error recovery level two functions used by
4  * the iSCSI Target driver.
5  *
6  * (c) Copyright 2007-2013 Datera, Inc.
7  *
8  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
9  *
10  ******************************************************************************/
11 
12 #include <linux/slab.h>
13 #include <scsi/iscsi_proto.h>
14 #include <target/target_core_base.h>
15 #include <target/target_core_fabric.h>
16 
17 #include <target/iscsi/iscsi_target_core.h>
18 #include "iscsi_target_datain_values.h"
19 #include "iscsi_target_util.h"
20 #include "iscsi_target_erl0.h"
21 #include "iscsi_target_erl1.h"
22 #include "iscsi_target_erl2.h"
23 #include "iscsi_target.h"
24 
25 /*
26  *	FIXME: Does RData SNACK apply here as well?
27  */
28 void iscsit_create_conn_recovery_datain_values(
29 	struct iscsi_cmd *cmd,
30 	__be32 exp_data_sn)
31 {
32 	u32 data_sn = 0;
33 	struct iscsi_conn *conn = cmd->conn;
34 
35 	cmd->next_burst_len = 0;
36 	cmd->read_data_done = 0;
37 
38 	while (be32_to_cpu(exp_data_sn) > data_sn) {
39 		if ((cmd->next_burst_len +
40 		     conn->conn_ops->MaxRecvDataSegmentLength) <
41 		     conn->sess->sess_ops->MaxBurstLength) {
42 			cmd->read_data_done +=
43 			       conn->conn_ops->MaxRecvDataSegmentLength;
44 			cmd->next_burst_len +=
45 			       conn->conn_ops->MaxRecvDataSegmentLength;
46 		} else {
47 			cmd->read_data_done +=
48 				(conn->sess->sess_ops->MaxBurstLength -
49 				cmd->next_burst_len);
50 			cmd->next_burst_len = 0;
51 		}
52 		data_sn++;
53 	}
54 }
55 
56 void iscsit_create_conn_recovery_dataout_values(
57 	struct iscsi_cmd *cmd)
58 {
59 	u32 write_data_done = 0;
60 	struct iscsi_conn *conn = cmd->conn;
61 
62 	cmd->data_sn = 0;
63 	cmd->next_burst_len = 0;
64 
65 	while (cmd->write_data_done > write_data_done) {
66 		if ((write_data_done + conn->sess->sess_ops->MaxBurstLength) <=
67 		     cmd->write_data_done)
68 			write_data_done += conn->sess->sess_ops->MaxBurstLength;
69 		else
70 			break;
71 	}
72 
73 	cmd->write_data_done = write_data_done;
74 }
75 
76 static int iscsit_attach_active_connection_recovery_entry(
77 	struct iscsi_session *sess,
78 	struct iscsi_conn_recovery *cr)
79 {
80 	spin_lock(&sess->cr_a_lock);
81 	list_add_tail(&cr->cr_list, &sess->cr_active_list);
82 	spin_unlock(&sess->cr_a_lock);
83 
84 	return 0;
85 }
86 
87 static int iscsit_attach_inactive_connection_recovery_entry(
88 	struct iscsi_session *sess,
89 	struct iscsi_conn_recovery *cr)
90 {
91 	spin_lock(&sess->cr_i_lock);
92 	list_add_tail(&cr->cr_list, &sess->cr_inactive_list);
93 
94 	sess->conn_recovery_count++;
95 	pr_debug("Incremented connection recovery count to %u for"
96 		" SID: %u\n", sess->conn_recovery_count, sess->sid);
97 	spin_unlock(&sess->cr_i_lock);
98 
99 	return 0;
100 }
101 
102 struct iscsi_conn_recovery *iscsit_get_inactive_connection_recovery_entry(
103 	struct iscsi_session *sess,
104 	u16 cid)
105 {
106 	struct iscsi_conn_recovery *cr;
107 
108 	spin_lock(&sess->cr_i_lock);
109 	list_for_each_entry(cr, &sess->cr_inactive_list, cr_list) {
110 		if (cr->cid == cid) {
111 			spin_unlock(&sess->cr_i_lock);
112 			return cr;
113 		}
114 	}
115 	spin_unlock(&sess->cr_i_lock);
116 
117 	return NULL;
118 }
119 
120 void iscsit_free_connection_recovery_entries(struct iscsi_session *sess)
121 {
122 	struct iscsi_cmd *cmd, *cmd_tmp;
123 	struct iscsi_conn_recovery *cr, *cr_tmp;
124 
125 	spin_lock(&sess->cr_a_lock);
126 	list_for_each_entry_safe(cr, cr_tmp, &sess->cr_active_list, cr_list) {
127 		list_del(&cr->cr_list);
128 		spin_unlock(&sess->cr_a_lock);
129 
130 		spin_lock(&cr->conn_recovery_cmd_lock);
131 		list_for_each_entry_safe(cmd, cmd_tmp,
132 				&cr->conn_recovery_cmd_list, i_conn_node) {
133 
134 			list_del_init(&cmd->i_conn_node);
135 			cmd->conn = NULL;
136 			spin_unlock(&cr->conn_recovery_cmd_lock);
137 			iscsit_free_cmd(cmd, true);
138 			spin_lock(&cr->conn_recovery_cmd_lock);
139 		}
140 		spin_unlock(&cr->conn_recovery_cmd_lock);
141 		spin_lock(&sess->cr_a_lock);
142 
143 		kfree(cr);
144 	}
145 	spin_unlock(&sess->cr_a_lock);
146 
147 	spin_lock(&sess->cr_i_lock);
148 	list_for_each_entry_safe(cr, cr_tmp, &sess->cr_inactive_list, cr_list) {
149 		list_del(&cr->cr_list);
150 		spin_unlock(&sess->cr_i_lock);
151 
152 		spin_lock(&cr->conn_recovery_cmd_lock);
153 		list_for_each_entry_safe(cmd, cmd_tmp,
154 				&cr->conn_recovery_cmd_list, i_conn_node) {
155 
156 			list_del_init(&cmd->i_conn_node);
157 			cmd->conn = NULL;
158 			spin_unlock(&cr->conn_recovery_cmd_lock);
159 			iscsit_free_cmd(cmd, true);
160 			spin_lock(&cr->conn_recovery_cmd_lock);
161 		}
162 		spin_unlock(&cr->conn_recovery_cmd_lock);
163 		spin_lock(&sess->cr_i_lock);
164 
165 		kfree(cr);
166 	}
167 	spin_unlock(&sess->cr_i_lock);
168 }
169 
170 int iscsit_remove_active_connection_recovery_entry(
171 	struct iscsi_conn_recovery *cr,
172 	struct iscsi_session *sess)
173 {
174 	spin_lock(&sess->cr_a_lock);
175 	list_del(&cr->cr_list);
176 
177 	sess->conn_recovery_count--;
178 	pr_debug("Decremented connection recovery count to %u for"
179 		" SID: %u\n", sess->conn_recovery_count, sess->sid);
180 	spin_unlock(&sess->cr_a_lock);
181 
182 	kfree(cr);
183 
184 	return 0;
185 }
186 
187 static void iscsit_remove_inactive_connection_recovery_entry(
188 	struct iscsi_conn_recovery *cr,
189 	struct iscsi_session *sess)
190 {
191 	spin_lock(&sess->cr_i_lock);
192 	list_del(&cr->cr_list);
193 	spin_unlock(&sess->cr_i_lock);
194 }
195 
196 /*
197  *	Called with cr->conn_recovery_cmd_lock help.
198  */
199 int iscsit_remove_cmd_from_connection_recovery(
200 	struct iscsi_cmd *cmd,
201 	struct iscsi_session *sess)
202 {
203 	struct iscsi_conn_recovery *cr;
204 
205 	if (!cmd->cr) {
206 		pr_err("struct iscsi_conn_recovery pointer for ITT: 0x%08x"
207 			" is NULL!\n", cmd->init_task_tag);
208 		BUG();
209 	}
210 	cr = cmd->cr;
211 
212 	list_del_init(&cmd->i_conn_node);
213 	return --cr->cmd_count;
214 }
215 
216 void iscsit_discard_cr_cmds_by_expstatsn(
217 	struct iscsi_conn_recovery *cr,
218 	u32 exp_statsn)
219 {
220 	u32 dropped_count = 0;
221 	struct iscsi_cmd *cmd, *cmd_tmp;
222 	struct iscsi_session *sess = cr->sess;
223 
224 	spin_lock(&cr->conn_recovery_cmd_lock);
225 	list_for_each_entry_safe(cmd, cmd_tmp,
226 			&cr->conn_recovery_cmd_list, i_conn_node) {
227 
228 		if (((cmd->deferred_i_state != ISTATE_SENT_STATUS) &&
229 		     (cmd->deferred_i_state != ISTATE_REMOVE)) ||
230 		     (cmd->stat_sn >= exp_statsn)) {
231 			continue;
232 		}
233 
234 		dropped_count++;
235 		pr_debug("Dropping Acknowledged ITT: 0x%08x, StatSN:"
236 			" 0x%08x, CID: %hu.\n", cmd->init_task_tag,
237 				cmd->stat_sn, cr->cid);
238 
239 		iscsit_remove_cmd_from_connection_recovery(cmd, sess);
240 
241 		spin_unlock(&cr->conn_recovery_cmd_lock);
242 		iscsit_free_cmd(cmd, true);
243 		spin_lock(&cr->conn_recovery_cmd_lock);
244 	}
245 	spin_unlock(&cr->conn_recovery_cmd_lock);
246 
247 	pr_debug("Dropped %u total acknowledged commands on"
248 		" CID: %hu less than old ExpStatSN: 0x%08x\n",
249 			dropped_count, cr->cid, exp_statsn);
250 
251 	if (!cr->cmd_count) {
252 		pr_debug("No commands to be reassigned for failed"
253 			" connection CID: %hu on SID: %u\n",
254 			cr->cid, sess->sid);
255 		iscsit_remove_inactive_connection_recovery_entry(cr, sess);
256 		iscsit_attach_active_connection_recovery_entry(sess, cr);
257 		pr_debug("iSCSI connection recovery successful for CID:"
258 			" %hu on SID: %u\n", cr->cid, sess->sid);
259 		iscsit_remove_active_connection_recovery_entry(cr, sess);
260 	} else {
261 		iscsit_remove_inactive_connection_recovery_entry(cr, sess);
262 		iscsit_attach_active_connection_recovery_entry(sess, cr);
263 	}
264 }
265 
266 int iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(struct iscsi_conn *conn)
267 {
268 	u32 dropped_count = 0;
269 	struct iscsi_cmd *cmd, *cmd_tmp;
270 	struct iscsi_ooo_cmdsn *ooo_cmdsn, *ooo_cmdsn_tmp;
271 	struct iscsi_session *sess = conn->sess;
272 
273 	mutex_lock(&sess->cmdsn_mutex);
274 	list_for_each_entry_safe(ooo_cmdsn, ooo_cmdsn_tmp,
275 			&sess->sess_ooo_cmdsn_list, ooo_list) {
276 
277 		if (ooo_cmdsn->cid != conn->cid)
278 			continue;
279 
280 		dropped_count++;
281 		pr_debug("Dropping unacknowledged CmdSN:"
282 		" 0x%08x during connection recovery on CID: %hu\n",
283 			ooo_cmdsn->cmdsn, conn->cid);
284 		iscsit_remove_ooo_cmdsn(sess, ooo_cmdsn);
285 	}
286 	mutex_unlock(&sess->cmdsn_mutex);
287 
288 	spin_lock_bh(&conn->cmd_lock);
289 	list_for_each_entry_safe(cmd, cmd_tmp, &conn->conn_cmd_list, i_conn_node) {
290 		if (!(cmd->cmd_flags & ICF_OOO_CMDSN))
291 			continue;
292 
293 		list_del_init(&cmd->i_conn_node);
294 
295 		spin_unlock_bh(&conn->cmd_lock);
296 		iscsit_free_cmd(cmd, true);
297 		spin_lock_bh(&conn->cmd_lock);
298 	}
299 	spin_unlock_bh(&conn->cmd_lock);
300 
301 	pr_debug("Dropped %u total unacknowledged commands on CID:"
302 		" %hu for ExpCmdSN: 0x%08x.\n", dropped_count, conn->cid,
303 				sess->exp_cmd_sn);
304 	return 0;
305 }
306 
307 int iscsit_prepare_cmds_for_reallegiance(struct iscsi_conn *conn)
308 {
309 	u32 cmd_count = 0;
310 	struct iscsi_cmd *cmd, *cmd_tmp;
311 	struct iscsi_conn_recovery *cr;
312 
313 	/*
314 	 * Allocate an struct iscsi_conn_recovery for this connection.
315 	 * Each struct iscsi_cmd contains an struct iscsi_conn_recovery pointer
316 	 * (struct iscsi_cmd->cr) so we need to allocate this before preparing the
317 	 * connection's command list for connection recovery.
318 	 */
319 	cr = kzalloc(sizeof(struct iscsi_conn_recovery), GFP_KERNEL);
320 	if (!cr) {
321 		pr_err("Unable to allocate memory for"
322 			" struct iscsi_conn_recovery.\n");
323 		return -1;
324 	}
325 	INIT_LIST_HEAD(&cr->cr_list);
326 	INIT_LIST_HEAD(&cr->conn_recovery_cmd_list);
327 	spin_lock_init(&cr->conn_recovery_cmd_lock);
328 	/*
329 	 * Only perform connection recovery on ISCSI_OP_SCSI_CMD or
330 	 * ISCSI_OP_NOOP_OUT opcodes.  For all other opcodes call
331 	 * list_del_init(&cmd->i_conn_node); to release the command to the
332 	 * session pool and remove it from the connection's list.
333 	 *
334 	 * Also stop the DataOUT timer, which will be restarted after
335 	 * sending the TMR response.
336 	 */
337 	spin_lock_bh(&conn->cmd_lock);
338 	list_for_each_entry_safe(cmd, cmd_tmp, &conn->conn_cmd_list, i_conn_node) {
339 
340 		if ((cmd->iscsi_opcode != ISCSI_OP_SCSI_CMD) &&
341 		    (cmd->iscsi_opcode != ISCSI_OP_NOOP_OUT)) {
342 			pr_debug("Not performing reallegiance on"
343 				" Opcode: 0x%02x, ITT: 0x%08x, CmdSN: 0x%08x,"
344 				" CID: %hu\n", cmd->iscsi_opcode,
345 				cmd->init_task_tag, cmd->cmd_sn, conn->cid);
346 
347 			list_del_init(&cmd->i_conn_node);
348 			spin_unlock_bh(&conn->cmd_lock);
349 			iscsit_free_cmd(cmd, true);
350 			spin_lock_bh(&conn->cmd_lock);
351 			continue;
352 		}
353 
354 		/*
355 		 * Special case where commands greater than or equal to
356 		 * the session's ExpCmdSN are attached to the connection
357 		 * list but not to the out of order CmdSN list.  The one
358 		 * obvious case is when a command with immediate data
359 		 * attached must only check the CmdSN against ExpCmdSN
360 		 * after the data is received.  The special case below
361 		 * is when the connection fails before data is received,
362 		 * but also may apply to other PDUs, so it has been
363 		 * made generic here.
364 		 */
365 		if (!(cmd->cmd_flags & ICF_OOO_CMDSN) && !cmd->immediate_cmd &&
366 		     iscsi_sna_gte(cmd->cmd_sn, conn->sess->exp_cmd_sn)) {
367 			list_del_init(&cmd->i_conn_node);
368 			spin_unlock_bh(&conn->cmd_lock);
369 			iscsit_free_cmd(cmd, true);
370 			spin_lock_bh(&conn->cmd_lock);
371 			continue;
372 		}
373 
374 		cmd_count++;
375 		pr_debug("Preparing Opcode: 0x%02x, ITT: 0x%08x,"
376 			" CmdSN: 0x%08x, StatSN: 0x%08x, CID: %hu for"
377 			" reallegiance.\n", cmd->iscsi_opcode,
378 			cmd->init_task_tag, cmd->cmd_sn, cmd->stat_sn,
379 			conn->cid);
380 
381 		cmd->deferred_i_state = cmd->i_state;
382 		cmd->i_state = ISTATE_IN_CONNECTION_RECOVERY;
383 
384 		if (cmd->data_direction == DMA_TO_DEVICE)
385 			iscsit_stop_dataout_timer(cmd);
386 
387 		cmd->sess = conn->sess;
388 
389 		list_del_init(&cmd->i_conn_node);
390 		spin_unlock_bh(&conn->cmd_lock);
391 
392 		iscsit_free_all_datain_reqs(cmd);
393 
394 		transport_wait_for_tasks(&cmd->se_cmd);
395 		/*
396 		 * Add the struct iscsi_cmd to the connection recovery cmd list
397 		 */
398 		spin_lock(&cr->conn_recovery_cmd_lock);
399 		list_add_tail(&cmd->i_conn_node, &cr->conn_recovery_cmd_list);
400 		spin_unlock(&cr->conn_recovery_cmd_lock);
401 
402 		spin_lock_bh(&conn->cmd_lock);
403 		cmd->cr = cr;
404 		cmd->conn = NULL;
405 	}
406 	spin_unlock_bh(&conn->cmd_lock);
407 	/*
408 	 * Fill in the various values in the preallocated struct iscsi_conn_recovery.
409 	 */
410 	cr->cid = conn->cid;
411 	cr->cmd_count = cmd_count;
412 	cr->maxrecvdatasegmentlength = conn->conn_ops->MaxRecvDataSegmentLength;
413 	cr->maxxmitdatasegmentlength = conn->conn_ops->MaxXmitDataSegmentLength;
414 	cr->sess = conn->sess;
415 
416 	iscsit_attach_inactive_connection_recovery_entry(conn->sess, cr);
417 
418 	return 0;
419 }
420 
421 int iscsit_connection_recovery_transport_reset(struct iscsi_conn *conn)
422 {
423 	atomic_set(&conn->connection_recovery, 1);
424 
425 	if (iscsit_close_connection(conn) < 0)
426 		return -1;
427 
428 	return 0;
429 }
430