1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*******************************************************************************
3  * This file contains the iSCSI Target DataIN value generation functions.
4  *
5  * (c) Copyright 2007-2013 Datera, Inc.
6  *
7  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
8  *
9  ******************************************************************************/
10 
11 #include <linux/slab.h>
12 #include <scsi/iscsi_proto.h>
13 #include <target/iscsi/iscsi_target_core.h>
14 #include "iscsi_target_seq_pdu_list.h"
15 #include "iscsi_target_erl1.h"
16 #include "iscsi_target_util.h"
17 #include "iscsi_target.h"
18 #include "iscsi_target_datain_values.h"
19 
20 struct iscsi_datain_req *iscsit_allocate_datain_req(void)
21 {
22 	struct iscsi_datain_req *dr;
23 
24 	dr = kmem_cache_zalloc(lio_dr_cache, GFP_ATOMIC);
25 	if (!dr) {
26 		pr_err("Unable to allocate memory for"
27 				" struct iscsi_datain_req\n");
28 		return NULL;
29 	}
30 	INIT_LIST_HEAD(&dr->cmd_datain_node);
31 
32 	return dr;
33 }
34 
35 void iscsit_attach_datain_req(struct iscsit_cmd *cmd, struct iscsi_datain_req *dr)
36 {
37 	spin_lock(&cmd->datain_lock);
38 	list_add_tail(&dr->cmd_datain_node, &cmd->datain_list);
39 	spin_unlock(&cmd->datain_lock);
40 }
41 
42 void iscsit_free_datain_req(struct iscsit_cmd *cmd, struct iscsi_datain_req *dr)
43 {
44 	spin_lock(&cmd->datain_lock);
45 	list_del(&dr->cmd_datain_node);
46 	spin_unlock(&cmd->datain_lock);
47 
48 	kmem_cache_free(lio_dr_cache, dr);
49 }
50 
51 void iscsit_free_all_datain_reqs(struct iscsit_cmd *cmd)
52 {
53 	struct iscsi_datain_req *dr, *dr_tmp;
54 
55 	spin_lock(&cmd->datain_lock);
56 	list_for_each_entry_safe(dr, dr_tmp, &cmd->datain_list, cmd_datain_node) {
57 		list_del(&dr->cmd_datain_node);
58 		kmem_cache_free(lio_dr_cache, dr);
59 	}
60 	spin_unlock(&cmd->datain_lock);
61 }
62 
63 struct iscsi_datain_req *iscsit_get_datain_req(struct iscsit_cmd *cmd)
64 {
65 	if (list_empty(&cmd->datain_list)) {
66 		pr_err("cmd->datain_list is empty for ITT:"
67 			" 0x%08x\n", cmd->init_task_tag);
68 		return NULL;
69 	}
70 
71 	return list_first_entry(&cmd->datain_list, struct iscsi_datain_req,
72 				cmd_datain_node);
73 }
74 
75 /*
76  *	For Normal and Recovery DataSequenceInOrder=Yes and DataPDUInOrder=Yes.
77  */
78 static struct iscsi_datain_req *iscsit_set_datain_values_yes_and_yes(
79 	struct iscsit_cmd *cmd,
80 	struct iscsi_datain *datain)
81 {
82 	u32 next_burst_len, read_data_done, read_data_left;
83 	struct iscsit_conn *conn = cmd->conn;
84 	struct iscsi_datain_req *dr;
85 
86 	dr = iscsit_get_datain_req(cmd);
87 	if (!dr)
88 		return NULL;
89 
90 	if (dr->recovery && dr->generate_recovery_values) {
91 		if (iscsit_create_recovery_datain_values_datasequenceinorder_yes(
92 					cmd, dr) < 0)
93 			return NULL;
94 
95 		dr->generate_recovery_values = 0;
96 	}
97 
98 	next_burst_len = (!dr->recovery) ?
99 			cmd->next_burst_len : dr->next_burst_len;
100 	read_data_done = (!dr->recovery) ?
101 			cmd->read_data_done : dr->read_data_done;
102 
103 	read_data_left = (cmd->se_cmd.data_length - read_data_done);
104 	if (!read_data_left) {
105 		pr_err("ITT: 0x%08x read_data_left is zero!\n",
106 				cmd->init_task_tag);
107 		return NULL;
108 	}
109 
110 	if ((read_data_left <= conn->conn_ops->MaxRecvDataSegmentLength) &&
111 	    (read_data_left <= (conn->sess->sess_ops->MaxBurstLength -
112 	     next_burst_len))) {
113 		datain->length = read_data_left;
114 
115 		datain->flags |= (ISCSI_FLAG_CMD_FINAL | ISCSI_FLAG_DATA_STATUS);
116 		if (conn->sess->sess_ops->ErrorRecoveryLevel > 0)
117 			datain->flags |= ISCSI_FLAG_DATA_ACK;
118 	} else {
119 		if ((next_burst_len +
120 		     conn->conn_ops->MaxRecvDataSegmentLength) <
121 		     conn->sess->sess_ops->MaxBurstLength) {
122 			datain->length =
123 				conn->conn_ops->MaxRecvDataSegmentLength;
124 			next_burst_len += datain->length;
125 		} else {
126 			datain->length = (conn->sess->sess_ops->MaxBurstLength -
127 					  next_burst_len);
128 			next_burst_len = 0;
129 
130 			datain->flags |= ISCSI_FLAG_CMD_FINAL;
131 			if (conn->sess->sess_ops->ErrorRecoveryLevel > 0)
132 				datain->flags |= ISCSI_FLAG_DATA_ACK;
133 		}
134 	}
135 
136 	datain->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++;
137 	datain->offset = read_data_done;
138 
139 	if (!dr->recovery) {
140 		cmd->next_burst_len = next_burst_len;
141 		cmd->read_data_done += datain->length;
142 	} else {
143 		dr->next_burst_len = next_burst_len;
144 		dr->read_data_done += datain->length;
145 	}
146 
147 	if (!dr->recovery) {
148 		if (datain->flags & ISCSI_FLAG_DATA_STATUS)
149 			dr->dr_complete = DATAIN_COMPLETE_NORMAL;
150 
151 		return dr;
152 	}
153 
154 	if (!dr->runlength) {
155 		if (datain->flags & ISCSI_FLAG_DATA_STATUS) {
156 			dr->dr_complete =
157 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?
158 				DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY :
159 				DATAIN_COMPLETE_CONNECTION_RECOVERY;
160 		}
161 	} else {
162 		if ((dr->begrun + dr->runlength) == dr->data_sn) {
163 			dr->dr_complete =
164 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?
165 				DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY :
166 				DATAIN_COMPLETE_CONNECTION_RECOVERY;
167 		}
168 	}
169 
170 	return dr;
171 }
172 
173 /*
174  *	For Normal and Recovery DataSequenceInOrder=No and DataPDUInOrder=Yes.
175  */
176 static struct iscsi_datain_req *iscsit_set_datain_values_no_and_yes(
177 	struct iscsit_cmd *cmd,
178 	struct iscsi_datain *datain)
179 {
180 	u32 offset, read_data_done, read_data_left, seq_send_order;
181 	struct iscsit_conn *conn = cmd->conn;
182 	struct iscsi_datain_req *dr;
183 	struct iscsi_seq *seq;
184 
185 	dr = iscsit_get_datain_req(cmd);
186 	if (!dr)
187 		return NULL;
188 
189 	if (dr->recovery && dr->generate_recovery_values) {
190 		if (iscsit_create_recovery_datain_values_datasequenceinorder_no(
191 					cmd, dr) < 0)
192 			return NULL;
193 
194 		dr->generate_recovery_values = 0;
195 	}
196 
197 	read_data_done = (!dr->recovery) ?
198 			cmd->read_data_done : dr->read_data_done;
199 	seq_send_order = (!dr->recovery) ?
200 			cmd->seq_send_order : dr->seq_send_order;
201 
202 	read_data_left = (cmd->se_cmd.data_length - read_data_done);
203 	if (!read_data_left) {
204 		pr_err("ITT: 0x%08x read_data_left is zero!\n",
205 				cmd->init_task_tag);
206 		return NULL;
207 	}
208 
209 	seq = iscsit_get_seq_holder_for_datain(cmd, seq_send_order);
210 	if (!seq)
211 		return NULL;
212 
213 	seq->sent = 1;
214 
215 	if (!dr->recovery && !seq->next_burst_len)
216 		seq->first_datasn = cmd->data_sn;
217 
218 	offset = (seq->offset + seq->next_burst_len);
219 
220 	if ((offset + conn->conn_ops->MaxRecvDataSegmentLength) >=
221 	     cmd->se_cmd.data_length) {
222 		datain->length = (cmd->se_cmd.data_length - offset);
223 		datain->offset = offset;
224 
225 		datain->flags |= ISCSI_FLAG_CMD_FINAL;
226 		if (conn->sess->sess_ops->ErrorRecoveryLevel > 0)
227 			datain->flags |= ISCSI_FLAG_DATA_ACK;
228 
229 		seq->next_burst_len = 0;
230 		seq_send_order++;
231 	} else {
232 		if ((seq->next_burst_len +
233 		     conn->conn_ops->MaxRecvDataSegmentLength) <
234 		     conn->sess->sess_ops->MaxBurstLength) {
235 			datain->length =
236 				conn->conn_ops->MaxRecvDataSegmentLength;
237 			datain->offset = (seq->offset + seq->next_burst_len);
238 
239 			seq->next_burst_len += datain->length;
240 		} else {
241 			datain->length = (conn->sess->sess_ops->MaxBurstLength -
242 					  seq->next_burst_len);
243 			datain->offset = (seq->offset + seq->next_burst_len);
244 
245 			datain->flags |= ISCSI_FLAG_CMD_FINAL;
246 			if (conn->sess->sess_ops->ErrorRecoveryLevel > 0)
247 				datain->flags |= ISCSI_FLAG_DATA_ACK;
248 
249 			seq->next_burst_len = 0;
250 			seq_send_order++;
251 		}
252 	}
253 
254 	if ((read_data_done + datain->length) == cmd->se_cmd.data_length)
255 		datain->flags |= ISCSI_FLAG_DATA_STATUS;
256 
257 	datain->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++;
258 	if (!dr->recovery) {
259 		cmd->seq_send_order = seq_send_order;
260 		cmd->read_data_done += datain->length;
261 	} else {
262 		dr->seq_send_order = seq_send_order;
263 		dr->read_data_done += datain->length;
264 	}
265 
266 	if (!dr->recovery) {
267 		if (datain->flags & ISCSI_FLAG_CMD_FINAL)
268 			seq->last_datasn = datain->data_sn;
269 		if (datain->flags & ISCSI_FLAG_DATA_STATUS)
270 			dr->dr_complete = DATAIN_COMPLETE_NORMAL;
271 
272 		return dr;
273 	}
274 
275 	if (!dr->runlength) {
276 		if (datain->flags & ISCSI_FLAG_DATA_STATUS) {
277 			dr->dr_complete =
278 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?
279 				DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY :
280 				DATAIN_COMPLETE_CONNECTION_RECOVERY;
281 		}
282 	} else {
283 		if ((dr->begrun + dr->runlength) == dr->data_sn) {
284 			dr->dr_complete =
285 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?
286 				DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY :
287 				DATAIN_COMPLETE_CONNECTION_RECOVERY;
288 		}
289 	}
290 
291 	return dr;
292 }
293 
294 /*
295  *	For Normal and Recovery DataSequenceInOrder=Yes and DataPDUInOrder=No.
296  */
297 static struct iscsi_datain_req *iscsit_set_datain_values_yes_and_no(
298 	struct iscsit_cmd *cmd,
299 	struct iscsi_datain *datain)
300 {
301 	u32 next_burst_len, read_data_done, read_data_left;
302 	struct iscsit_conn *conn = cmd->conn;
303 	struct iscsi_datain_req *dr;
304 	struct iscsi_pdu *pdu;
305 
306 	dr = iscsit_get_datain_req(cmd);
307 	if (!dr)
308 		return NULL;
309 
310 	if (dr->recovery && dr->generate_recovery_values) {
311 		if (iscsit_create_recovery_datain_values_datasequenceinorder_yes(
312 					cmd, dr) < 0)
313 			return NULL;
314 
315 		dr->generate_recovery_values = 0;
316 	}
317 
318 	next_burst_len = (!dr->recovery) ?
319 			cmd->next_burst_len : dr->next_burst_len;
320 	read_data_done = (!dr->recovery) ?
321 			cmd->read_data_done : dr->read_data_done;
322 
323 	read_data_left = (cmd->se_cmd.data_length - read_data_done);
324 	if (!read_data_left) {
325 		pr_err("ITT: 0x%08x read_data_left is zero!\n",
326 				cmd->init_task_tag);
327 		return dr;
328 	}
329 
330 	pdu = iscsit_get_pdu_holder_for_seq(cmd, NULL);
331 	if (!pdu)
332 		return dr;
333 
334 	if ((read_data_done + pdu->length) == cmd->se_cmd.data_length) {
335 		pdu->flags |= (ISCSI_FLAG_CMD_FINAL | ISCSI_FLAG_DATA_STATUS);
336 		if (conn->sess->sess_ops->ErrorRecoveryLevel > 0)
337 			pdu->flags |= ISCSI_FLAG_DATA_ACK;
338 
339 		next_burst_len = 0;
340 	} else {
341 		if ((next_burst_len + conn->conn_ops->MaxRecvDataSegmentLength) <
342 		     conn->sess->sess_ops->MaxBurstLength)
343 			next_burst_len += pdu->length;
344 		else {
345 			pdu->flags |= ISCSI_FLAG_CMD_FINAL;
346 			if (conn->sess->sess_ops->ErrorRecoveryLevel > 0)
347 				pdu->flags |= ISCSI_FLAG_DATA_ACK;
348 
349 			next_burst_len = 0;
350 		}
351 	}
352 
353 	pdu->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++;
354 	if (!dr->recovery) {
355 		cmd->next_burst_len = next_burst_len;
356 		cmd->read_data_done += pdu->length;
357 	} else {
358 		dr->next_burst_len = next_burst_len;
359 		dr->read_data_done += pdu->length;
360 	}
361 
362 	datain->flags = pdu->flags;
363 	datain->length = pdu->length;
364 	datain->offset = pdu->offset;
365 	datain->data_sn = pdu->data_sn;
366 
367 	if (!dr->recovery) {
368 		if (datain->flags & ISCSI_FLAG_DATA_STATUS)
369 			dr->dr_complete = DATAIN_COMPLETE_NORMAL;
370 
371 		return dr;
372 	}
373 
374 	if (!dr->runlength) {
375 		if (datain->flags & ISCSI_FLAG_DATA_STATUS) {
376 			dr->dr_complete =
377 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?
378 				DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY :
379 				DATAIN_COMPLETE_CONNECTION_RECOVERY;
380 		}
381 	} else {
382 		if ((dr->begrun + dr->runlength) == dr->data_sn) {
383 			dr->dr_complete =
384 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?
385 				DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY :
386 				DATAIN_COMPLETE_CONNECTION_RECOVERY;
387 		}
388 	}
389 
390 	return dr;
391 }
392 
393 /*
394  *	For Normal and Recovery DataSequenceInOrder=No and DataPDUInOrder=No.
395  */
396 static struct iscsi_datain_req *iscsit_set_datain_values_no_and_no(
397 	struct iscsit_cmd *cmd,
398 	struct iscsi_datain *datain)
399 {
400 	u32 read_data_done, read_data_left, seq_send_order;
401 	struct iscsit_conn *conn = cmd->conn;
402 	struct iscsi_datain_req *dr;
403 	struct iscsi_pdu *pdu;
404 	struct iscsi_seq *seq = NULL;
405 
406 	dr = iscsit_get_datain_req(cmd);
407 	if (!dr)
408 		return NULL;
409 
410 	if (dr->recovery && dr->generate_recovery_values) {
411 		if (iscsit_create_recovery_datain_values_datasequenceinorder_no(
412 					cmd, dr) < 0)
413 			return NULL;
414 
415 		dr->generate_recovery_values = 0;
416 	}
417 
418 	read_data_done = (!dr->recovery) ?
419 			cmd->read_data_done : dr->read_data_done;
420 	seq_send_order = (!dr->recovery) ?
421 			cmd->seq_send_order : dr->seq_send_order;
422 
423 	read_data_left = (cmd->se_cmd.data_length - read_data_done);
424 	if (!read_data_left) {
425 		pr_err("ITT: 0x%08x read_data_left is zero!\n",
426 				cmd->init_task_tag);
427 		return NULL;
428 	}
429 
430 	seq = iscsit_get_seq_holder_for_datain(cmd, seq_send_order);
431 	if (!seq)
432 		return NULL;
433 
434 	seq->sent = 1;
435 
436 	if (!dr->recovery && !seq->next_burst_len)
437 		seq->first_datasn = cmd->data_sn;
438 
439 	pdu = iscsit_get_pdu_holder_for_seq(cmd, seq);
440 	if (!pdu)
441 		return NULL;
442 
443 	if (seq->pdu_send_order == seq->pdu_count) {
444 		pdu->flags |= ISCSI_FLAG_CMD_FINAL;
445 		if (conn->sess->sess_ops->ErrorRecoveryLevel > 0)
446 			pdu->flags |= ISCSI_FLAG_DATA_ACK;
447 
448 		seq->next_burst_len = 0;
449 		seq_send_order++;
450 	} else
451 		seq->next_burst_len += pdu->length;
452 
453 	if ((read_data_done + pdu->length) == cmd->se_cmd.data_length)
454 		pdu->flags |= ISCSI_FLAG_DATA_STATUS;
455 
456 	pdu->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++;
457 	if (!dr->recovery) {
458 		cmd->seq_send_order = seq_send_order;
459 		cmd->read_data_done += pdu->length;
460 	} else {
461 		dr->seq_send_order = seq_send_order;
462 		dr->read_data_done += pdu->length;
463 	}
464 
465 	datain->flags = pdu->flags;
466 	datain->length = pdu->length;
467 	datain->offset = pdu->offset;
468 	datain->data_sn = pdu->data_sn;
469 
470 	if (!dr->recovery) {
471 		if (datain->flags & ISCSI_FLAG_CMD_FINAL)
472 			seq->last_datasn = datain->data_sn;
473 		if (datain->flags & ISCSI_FLAG_DATA_STATUS)
474 			dr->dr_complete = DATAIN_COMPLETE_NORMAL;
475 
476 		return dr;
477 	}
478 
479 	if (!dr->runlength) {
480 		if (datain->flags & ISCSI_FLAG_DATA_STATUS) {
481 			dr->dr_complete =
482 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?
483 				DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY :
484 				DATAIN_COMPLETE_CONNECTION_RECOVERY;
485 		}
486 	} else {
487 		if ((dr->begrun + dr->runlength) == dr->data_sn) {
488 			dr->dr_complete =
489 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?
490 				DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY :
491 				DATAIN_COMPLETE_CONNECTION_RECOVERY;
492 		}
493 	}
494 
495 	return dr;
496 }
497 
498 struct iscsi_datain_req *iscsit_get_datain_values(
499 	struct iscsit_cmd *cmd,
500 	struct iscsi_datain *datain)
501 {
502 	struct iscsit_conn *conn = cmd->conn;
503 
504 	if (conn->sess->sess_ops->DataSequenceInOrder &&
505 	    conn->sess->sess_ops->DataPDUInOrder)
506 		return iscsit_set_datain_values_yes_and_yes(cmd, datain);
507 	else if (!conn->sess->sess_ops->DataSequenceInOrder &&
508 		  conn->sess->sess_ops->DataPDUInOrder)
509 		return iscsit_set_datain_values_no_and_yes(cmd, datain);
510 	else if (conn->sess->sess_ops->DataSequenceInOrder &&
511 		 !conn->sess->sess_ops->DataPDUInOrder)
512 		return iscsit_set_datain_values_yes_and_no(cmd, datain);
513 	else if (!conn->sess->sess_ops->DataSequenceInOrder &&
514 		   !conn->sess->sess_ops->DataPDUInOrder)
515 		return iscsit_set_datain_values_no_and_no(cmd, datain);
516 
517 	return NULL;
518 }
519 EXPORT_SYMBOL(iscsit_get_datain_values);
520