1 /******************************************************************************* 2 * This file contains error recovery level one used by the iSCSI Target driver. 3 * 4 * (c) Copyright 2007-2013 Datera, Inc. 5 * 6 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ******************************************************************************/ 18 19 #include <linux/list.h> 20 #include <linux/slab.h> 21 #include <scsi/iscsi_proto.h> 22 #include <target/target_core_base.h> 23 #include <target/target_core_fabric.h> 24 #include <target/iscsi/iscsi_transport.h> 25 26 #include <target/iscsi/iscsi_target_core.h> 27 #include "iscsi_target_seq_pdu_list.h" 28 #include "iscsi_target_datain_values.h" 29 #include "iscsi_target_device.h" 30 #include "iscsi_target_tpg.h" 31 #include "iscsi_target_util.h" 32 #include "iscsi_target_erl0.h" 33 #include "iscsi_target_erl1.h" 34 #include "iscsi_target_erl2.h" 35 #include "iscsi_target.h" 36 37 #define OFFLOAD_BUF_SIZE 32768U 38 39 /* 40 * Used to dump excess datain payload for certain error recovery 41 * situations. Receive in OFFLOAD_BUF_SIZE max of datain per rx_data(). 42 * 43 * dump_padding_digest denotes if padding and data digests need 44 * to be dumped. 45 */ 46 int iscsit_dump_data_payload( 47 struct iscsi_conn *conn, 48 u32 buf_len, 49 int dump_padding_digest) 50 { 51 char *buf; 52 int ret = DATAOUT_WITHIN_COMMAND_RECOVERY, rx_got; 53 u32 length, offset = 0, size; 54 struct kvec iov; 55 56 if (conn->sess->sess_ops->RDMAExtensions) 57 return 0; 58 59 if (dump_padding_digest) { 60 buf_len = ALIGN(buf_len, 4); 61 if (conn->conn_ops->DataDigest) 62 buf_len += ISCSI_CRC_LEN; 63 } 64 65 length = min(buf_len, OFFLOAD_BUF_SIZE); 66 67 buf = kzalloc(length, GFP_ATOMIC); 68 if (!buf) { 69 pr_err("Unable to allocate %u bytes for offload" 70 " buffer.\n", length); 71 return -1; 72 } 73 memset(&iov, 0, sizeof(struct kvec)); 74 75 while (offset < buf_len) { 76 size = min(buf_len - offset, length); 77 78 iov.iov_len = size; 79 iov.iov_base = buf; 80 81 rx_got = rx_data(conn, &iov, 1, size); 82 if (rx_got != size) { 83 ret = DATAOUT_CANNOT_RECOVER; 84 break; 85 } 86 87 offset += size; 88 } 89 90 kfree(buf); 91 return ret; 92 } 93 94 /* 95 * Used for retransmitting R2Ts from a R2T SNACK request. 96 */ 97 static int iscsit_send_recovery_r2t_for_snack( 98 struct iscsi_cmd *cmd, 99 struct iscsi_r2t *r2t) 100 { 101 /* 102 * If the struct iscsi_r2t has not been sent yet, we can safely 103 * ignore retransmission 104 * of the R2TSN in question. 105 */ 106 spin_lock_bh(&cmd->r2t_lock); 107 if (!r2t->sent_r2t) { 108 spin_unlock_bh(&cmd->r2t_lock); 109 return 0; 110 } 111 r2t->sent_r2t = 0; 112 spin_unlock_bh(&cmd->r2t_lock); 113 114 iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, ISTATE_SEND_R2T); 115 116 return 0; 117 } 118 119 static int iscsit_handle_r2t_snack( 120 struct iscsi_cmd *cmd, 121 unsigned char *buf, 122 u32 begrun, 123 u32 runlength) 124 { 125 u32 last_r2tsn; 126 struct iscsi_r2t *r2t; 127 128 /* 129 * Make sure the initiator is not requesting retransmission 130 * of R2TSNs already acknowledged by a TMR TASK_REASSIGN. 131 */ 132 if ((cmd->cmd_flags & ICF_GOT_DATACK_SNACK) && 133 (begrun <= cmd->acked_data_sn)) { 134 pr_err("ITT: 0x%08x, R2T SNACK requesting" 135 " retransmission of R2TSN: 0x%08x to 0x%08x but already" 136 " acked to R2TSN: 0x%08x by TMR TASK_REASSIGN," 137 " protocol error.\n", cmd->init_task_tag, begrun, 138 (begrun + runlength), cmd->acked_data_sn); 139 140 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf); 141 } 142 143 if (runlength) { 144 if ((begrun + runlength) > cmd->r2t_sn) { 145 pr_err("Command ITT: 0x%08x received R2T SNACK" 146 " with BegRun: 0x%08x, RunLength: 0x%08x, exceeds" 147 " current R2TSN: 0x%08x, protocol error.\n", 148 cmd->init_task_tag, begrun, runlength, cmd->r2t_sn); 149 return iscsit_reject_cmd(cmd, 150 ISCSI_REASON_BOOKMARK_INVALID, buf); 151 } 152 last_r2tsn = (begrun + runlength); 153 } else 154 last_r2tsn = cmd->r2t_sn; 155 156 while (begrun < last_r2tsn) { 157 r2t = iscsit_get_holder_for_r2tsn(cmd, begrun); 158 if (!r2t) 159 return -1; 160 if (iscsit_send_recovery_r2t_for_snack(cmd, r2t) < 0) 161 return -1; 162 163 begrun++; 164 } 165 166 return 0; 167 } 168 169 /* 170 * Generates Offsets and NextBurstLength based on Begrun and Runlength 171 * carried in a Data SNACK or ExpDataSN in TMR TASK_REASSIGN. 172 * 173 * For DataSequenceInOrder=Yes and DataPDUInOrder=[Yes,No] only. 174 * 175 * FIXME: How is this handled for a RData SNACK? 176 */ 177 int iscsit_create_recovery_datain_values_datasequenceinorder_yes( 178 struct iscsi_cmd *cmd, 179 struct iscsi_datain_req *dr) 180 { 181 u32 data_sn = 0, data_sn_count = 0; 182 u32 pdu_start = 0, seq_no = 0; 183 u32 begrun = dr->begrun; 184 struct iscsi_conn *conn = cmd->conn; 185 186 while (begrun > data_sn++) { 187 data_sn_count++; 188 if ((dr->next_burst_len + 189 conn->conn_ops->MaxRecvDataSegmentLength) < 190 conn->sess->sess_ops->MaxBurstLength) { 191 dr->read_data_done += 192 conn->conn_ops->MaxRecvDataSegmentLength; 193 dr->next_burst_len += 194 conn->conn_ops->MaxRecvDataSegmentLength; 195 } else { 196 dr->read_data_done += 197 (conn->sess->sess_ops->MaxBurstLength - 198 dr->next_burst_len); 199 dr->next_burst_len = 0; 200 pdu_start += data_sn_count; 201 data_sn_count = 0; 202 seq_no++; 203 } 204 } 205 206 if (!conn->sess->sess_ops->DataPDUInOrder) { 207 cmd->seq_no = seq_no; 208 cmd->pdu_start = pdu_start; 209 cmd->pdu_send_order = data_sn_count; 210 } 211 212 return 0; 213 } 214 215 /* 216 * Generates Offsets and NextBurstLength based on Begrun and Runlength 217 * carried in a Data SNACK or ExpDataSN in TMR TASK_REASSIGN. 218 * 219 * For DataSequenceInOrder=No and DataPDUInOrder=[Yes,No] only. 220 * 221 * FIXME: How is this handled for a RData SNACK? 222 */ 223 int iscsit_create_recovery_datain_values_datasequenceinorder_no( 224 struct iscsi_cmd *cmd, 225 struct iscsi_datain_req *dr) 226 { 227 int found_seq = 0, i; 228 u32 data_sn, read_data_done = 0, seq_send_order = 0; 229 u32 begrun = dr->begrun; 230 u32 runlength = dr->runlength; 231 struct iscsi_conn *conn = cmd->conn; 232 struct iscsi_seq *first_seq = NULL, *seq = NULL; 233 234 if (!cmd->seq_list) { 235 pr_err("struct iscsi_cmd->seq_list is NULL!\n"); 236 return -1; 237 } 238 239 /* 240 * Calculate read_data_done for all sequences containing a 241 * first_datasn and last_datasn less than the BegRun. 242 * 243 * Locate the struct iscsi_seq the BegRun lies within and calculate 244 * NextBurstLenghth up to the DataSN based on MaxRecvDataSegmentLength. 245 * 246 * Also use struct iscsi_seq->seq_send_order to determine where to start. 247 */ 248 for (i = 0; i < cmd->seq_count; i++) { 249 seq = &cmd->seq_list[i]; 250 251 if (!seq->seq_send_order) 252 first_seq = seq; 253 254 /* 255 * No data has been transferred for this DataIN sequence, so the 256 * seq->first_datasn and seq->last_datasn have not been set. 257 */ 258 if (!seq->sent) { 259 pr_err("Ignoring non-sent sequence 0x%08x ->" 260 " 0x%08x\n\n", seq->first_datasn, 261 seq->last_datasn); 262 continue; 263 } 264 265 /* 266 * This DataIN sequence is precedes the received BegRun, add the 267 * total xfer_len of the sequence to read_data_done and reset 268 * seq->pdu_send_order. 269 */ 270 if ((seq->first_datasn < begrun) && 271 (seq->last_datasn < begrun)) { 272 pr_err("Pre BegRun sequence 0x%08x ->" 273 " 0x%08x\n", seq->first_datasn, 274 seq->last_datasn); 275 276 read_data_done += cmd->seq_list[i].xfer_len; 277 seq->next_burst_len = seq->pdu_send_order = 0; 278 continue; 279 } 280 281 /* 282 * The BegRun lies within this DataIN sequence. 283 */ 284 if ((seq->first_datasn <= begrun) && 285 (seq->last_datasn >= begrun)) { 286 pr_err("Found sequence begrun: 0x%08x in" 287 " 0x%08x -> 0x%08x\n", begrun, 288 seq->first_datasn, seq->last_datasn); 289 290 seq_send_order = seq->seq_send_order; 291 data_sn = seq->first_datasn; 292 seq->next_burst_len = seq->pdu_send_order = 0; 293 found_seq = 1; 294 295 /* 296 * For DataPDUInOrder=Yes, while the first DataSN of 297 * the sequence is less than the received BegRun, add 298 * the MaxRecvDataSegmentLength to read_data_done and 299 * to the sequence's next_burst_len; 300 * 301 * For DataPDUInOrder=No, while the first DataSN of the 302 * sequence is less than the received BegRun, find the 303 * struct iscsi_pdu of the DataSN in question and add the 304 * MaxRecvDataSegmentLength to read_data_done and to the 305 * sequence's next_burst_len; 306 */ 307 if (conn->sess->sess_ops->DataPDUInOrder) { 308 while (data_sn < begrun) { 309 seq->pdu_send_order++; 310 read_data_done += 311 conn->conn_ops->MaxRecvDataSegmentLength; 312 seq->next_burst_len += 313 conn->conn_ops->MaxRecvDataSegmentLength; 314 data_sn++; 315 } 316 } else { 317 int j; 318 struct iscsi_pdu *pdu; 319 320 while (data_sn < begrun) { 321 seq->pdu_send_order++; 322 323 for (j = 0; j < seq->pdu_count; j++) { 324 pdu = &cmd->pdu_list[ 325 seq->pdu_start + j]; 326 if (pdu->data_sn == data_sn) { 327 read_data_done += 328 pdu->length; 329 seq->next_burst_len += 330 pdu->length; 331 } 332 } 333 data_sn++; 334 } 335 } 336 continue; 337 } 338 339 /* 340 * This DataIN sequence is larger than the received BegRun, 341 * reset seq->pdu_send_order and continue. 342 */ 343 if ((seq->first_datasn > begrun) || 344 (seq->last_datasn > begrun)) { 345 pr_err("Post BegRun sequence 0x%08x -> 0x%08x\n", 346 seq->first_datasn, seq->last_datasn); 347 348 seq->next_burst_len = seq->pdu_send_order = 0; 349 continue; 350 } 351 } 352 353 if (!found_seq) { 354 if (!begrun) { 355 if (!first_seq) { 356 pr_err("ITT: 0x%08x, Begrun: 0x%08x" 357 " but first_seq is NULL\n", 358 cmd->init_task_tag, begrun); 359 return -1; 360 } 361 seq_send_order = first_seq->seq_send_order; 362 seq->next_burst_len = seq->pdu_send_order = 0; 363 goto done; 364 } 365 366 pr_err("Unable to locate struct iscsi_seq for ITT: 0x%08x," 367 " BegRun: 0x%08x, RunLength: 0x%08x while" 368 " DataSequenceInOrder=No and DataPDUInOrder=%s.\n", 369 cmd->init_task_tag, begrun, runlength, 370 (conn->sess->sess_ops->DataPDUInOrder) ? "Yes" : "No"); 371 return -1; 372 } 373 374 done: 375 dr->read_data_done = read_data_done; 376 dr->seq_send_order = seq_send_order; 377 378 return 0; 379 } 380 381 static int iscsit_handle_recovery_datain( 382 struct iscsi_cmd *cmd, 383 unsigned char *buf, 384 u32 begrun, 385 u32 runlength) 386 { 387 struct iscsi_conn *conn = cmd->conn; 388 struct iscsi_datain_req *dr; 389 struct se_cmd *se_cmd = &cmd->se_cmd; 390 391 if (!(se_cmd->transport_state & CMD_T_COMPLETE)) { 392 pr_err("Ignoring ITT: 0x%08x Data SNACK\n", 393 cmd->init_task_tag); 394 return 0; 395 } 396 397 /* 398 * Make sure the initiator is not requesting retransmission 399 * of DataSNs already acknowledged by a Data ACK SNACK. 400 */ 401 if ((cmd->cmd_flags & ICF_GOT_DATACK_SNACK) && 402 (begrun <= cmd->acked_data_sn)) { 403 pr_err("ITT: 0x%08x, Data SNACK requesting" 404 " retransmission of DataSN: 0x%08x to 0x%08x but" 405 " already acked to DataSN: 0x%08x by Data ACK SNACK," 406 " protocol error.\n", cmd->init_task_tag, begrun, 407 (begrun + runlength), cmd->acked_data_sn); 408 409 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf); 410 } 411 412 /* 413 * Make sure BegRun and RunLength in the Data SNACK are sane. 414 * Note: (cmd->data_sn - 1) will carry the maximum DataSN sent. 415 */ 416 if ((begrun + runlength) > (cmd->data_sn - 1)) { 417 pr_err("Initiator requesting BegRun: 0x%08x, RunLength" 418 ": 0x%08x greater than maximum DataSN: 0x%08x.\n", 419 begrun, runlength, (cmd->data_sn - 1)); 420 return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID, 421 buf); 422 } 423 424 dr = iscsit_allocate_datain_req(); 425 if (!dr) 426 return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_NO_RESOURCES, 427 buf); 428 429 dr->data_sn = dr->begrun = begrun; 430 dr->runlength = runlength; 431 dr->generate_recovery_values = 1; 432 dr->recovery = DATAIN_WITHIN_COMMAND_RECOVERY; 433 434 iscsit_attach_datain_req(cmd, dr); 435 436 cmd->i_state = ISTATE_SEND_DATAIN; 437 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 438 439 return 0; 440 } 441 442 int iscsit_handle_recovery_datain_or_r2t( 443 struct iscsi_conn *conn, 444 unsigned char *buf, 445 itt_t init_task_tag, 446 u32 targ_xfer_tag, 447 u32 begrun, 448 u32 runlength) 449 { 450 struct iscsi_cmd *cmd; 451 452 cmd = iscsit_find_cmd_from_itt(conn, init_task_tag); 453 if (!cmd) 454 return 0; 455 456 /* 457 * FIXME: This will not work for bidi commands. 458 */ 459 switch (cmd->data_direction) { 460 case DMA_TO_DEVICE: 461 return iscsit_handle_r2t_snack(cmd, buf, begrun, runlength); 462 case DMA_FROM_DEVICE: 463 return iscsit_handle_recovery_datain(cmd, buf, begrun, 464 runlength); 465 default: 466 pr_err("Unknown cmd->data_direction: 0x%02x\n", 467 cmd->data_direction); 468 return -1; 469 } 470 471 return 0; 472 } 473 474 /* #warning FIXME: Status SNACK needs to be dependent on OPCODE!!! */ 475 int iscsit_handle_status_snack( 476 struct iscsi_conn *conn, 477 itt_t init_task_tag, 478 u32 targ_xfer_tag, 479 u32 begrun, 480 u32 runlength) 481 { 482 struct iscsi_cmd *cmd = NULL; 483 u32 last_statsn; 484 int found_cmd; 485 486 if (!begrun) { 487 begrun = conn->exp_statsn; 488 } else if (conn->exp_statsn > begrun) { 489 pr_err("Got Status SNACK Begrun: 0x%08x, RunLength:" 490 " 0x%08x but already got ExpStatSN: 0x%08x on CID:" 491 " %hu.\n", begrun, runlength, conn->exp_statsn, 492 conn->cid); 493 return 0; 494 } 495 496 last_statsn = (!runlength) ? conn->stat_sn : (begrun + runlength); 497 498 while (begrun < last_statsn) { 499 found_cmd = 0; 500 501 spin_lock_bh(&conn->cmd_lock); 502 list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) { 503 if (cmd->stat_sn == begrun) { 504 found_cmd = 1; 505 break; 506 } 507 } 508 spin_unlock_bh(&conn->cmd_lock); 509 510 if (!found_cmd) { 511 pr_err("Unable to find StatSN: 0x%08x for" 512 " a Status SNACK, assuming this was a" 513 " protactic SNACK for an untransmitted" 514 " StatSN, ignoring.\n", begrun); 515 begrun++; 516 continue; 517 } 518 519 spin_lock_bh(&cmd->istate_lock); 520 if (cmd->i_state == ISTATE_SEND_DATAIN) { 521 spin_unlock_bh(&cmd->istate_lock); 522 pr_err("Ignoring Status SNACK for BegRun:" 523 " 0x%08x, RunLength: 0x%08x, assuming this was" 524 " a protactic SNACK for an untransmitted" 525 " StatSN\n", begrun, runlength); 526 begrun++; 527 continue; 528 } 529 spin_unlock_bh(&cmd->istate_lock); 530 531 cmd->i_state = ISTATE_SEND_STATUS_RECOVERY; 532 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 533 begrun++; 534 } 535 536 return 0; 537 } 538 539 int iscsit_handle_data_ack( 540 struct iscsi_conn *conn, 541 u32 targ_xfer_tag, 542 u32 begrun, 543 u32 runlength) 544 { 545 struct iscsi_cmd *cmd = NULL; 546 547 cmd = iscsit_find_cmd_from_ttt(conn, targ_xfer_tag); 548 if (!cmd) { 549 pr_err("Data ACK SNACK for TTT: 0x%08x is" 550 " invalid.\n", targ_xfer_tag); 551 return -1; 552 } 553 554 if (begrun <= cmd->acked_data_sn) { 555 pr_err("ITT: 0x%08x Data ACK SNACK BegRUN: 0x%08x is" 556 " less than the already acked DataSN: 0x%08x.\n", 557 cmd->init_task_tag, begrun, cmd->acked_data_sn); 558 return -1; 559 } 560 561 /* 562 * For Data ACK SNACK, BegRun is the next expected DataSN. 563 * (see iSCSI v19: 10.16.6) 564 */ 565 cmd->cmd_flags |= ICF_GOT_DATACK_SNACK; 566 cmd->acked_data_sn = (begrun - 1); 567 568 pr_debug("Received Data ACK SNACK for ITT: 0x%08x," 569 " updated acked DataSN to 0x%08x.\n", 570 cmd->init_task_tag, cmd->acked_data_sn); 571 572 return 0; 573 } 574 575 static int iscsit_send_recovery_r2t( 576 struct iscsi_cmd *cmd, 577 u32 offset, 578 u32 xfer_len) 579 { 580 int ret; 581 582 spin_lock_bh(&cmd->r2t_lock); 583 ret = iscsit_add_r2t_to_list(cmd, offset, xfer_len, 1, 0); 584 spin_unlock_bh(&cmd->r2t_lock); 585 586 return ret; 587 } 588 589 int iscsit_dataout_datapduinorder_no_fbit( 590 struct iscsi_cmd *cmd, 591 struct iscsi_pdu *pdu) 592 { 593 int i, send_recovery_r2t = 0, recovery = 0; 594 u32 length = 0, offset = 0, pdu_count = 0, xfer_len = 0; 595 struct iscsi_conn *conn = cmd->conn; 596 struct iscsi_pdu *first_pdu = NULL; 597 598 /* 599 * Get an struct iscsi_pdu pointer to the first PDU, and total PDU count 600 * of the DataOUT sequence. 601 */ 602 if (conn->sess->sess_ops->DataSequenceInOrder) { 603 for (i = 0; i < cmd->pdu_count; i++) { 604 if (cmd->pdu_list[i].seq_no == pdu->seq_no) { 605 if (!first_pdu) 606 first_pdu = &cmd->pdu_list[i]; 607 xfer_len += cmd->pdu_list[i].length; 608 pdu_count++; 609 } else if (pdu_count) 610 break; 611 } 612 } else { 613 struct iscsi_seq *seq = cmd->seq_ptr; 614 615 first_pdu = &cmd->pdu_list[seq->pdu_start]; 616 pdu_count = seq->pdu_count; 617 } 618 619 if (!first_pdu || !pdu_count) 620 return DATAOUT_CANNOT_RECOVER; 621 622 /* 623 * Loop through the ending DataOUT Sequence checking each struct iscsi_pdu. 624 * The following ugly logic does batching of not received PDUs. 625 */ 626 for (i = 0; i < pdu_count; i++) { 627 if (first_pdu[i].status == ISCSI_PDU_RECEIVED_OK) { 628 if (!send_recovery_r2t) 629 continue; 630 631 if (iscsit_send_recovery_r2t(cmd, offset, length) < 0) 632 return DATAOUT_CANNOT_RECOVER; 633 634 send_recovery_r2t = length = offset = 0; 635 continue; 636 } 637 /* 638 * Set recovery = 1 for any missing, CRC failed, or timed 639 * out PDUs to let the DataOUT logic know that this sequence 640 * has not been completed yet. 641 * 642 * Also, only send a Recovery R2T for ISCSI_PDU_NOT_RECEIVED. 643 * We assume if the PDU either failed CRC or timed out 644 * that a Recovery R2T has already been sent. 645 */ 646 recovery = 1; 647 648 if (first_pdu[i].status != ISCSI_PDU_NOT_RECEIVED) 649 continue; 650 651 if (!offset) 652 offset = first_pdu[i].offset; 653 length += first_pdu[i].length; 654 655 send_recovery_r2t = 1; 656 } 657 658 if (send_recovery_r2t) 659 if (iscsit_send_recovery_r2t(cmd, offset, length) < 0) 660 return DATAOUT_CANNOT_RECOVER; 661 662 return (!recovery) ? DATAOUT_NORMAL : DATAOUT_WITHIN_COMMAND_RECOVERY; 663 } 664 665 static int iscsit_recalculate_dataout_values( 666 struct iscsi_cmd *cmd, 667 u32 pdu_offset, 668 u32 pdu_length, 669 u32 *r2t_offset, 670 u32 *r2t_length) 671 { 672 int i; 673 struct iscsi_conn *conn = cmd->conn; 674 struct iscsi_pdu *pdu = NULL; 675 676 if (conn->sess->sess_ops->DataSequenceInOrder) { 677 cmd->data_sn = 0; 678 679 if (conn->sess->sess_ops->DataPDUInOrder) { 680 *r2t_offset = cmd->write_data_done; 681 *r2t_length = (cmd->seq_end_offset - 682 cmd->write_data_done); 683 return 0; 684 } 685 686 *r2t_offset = cmd->seq_start_offset; 687 *r2t_length = (cmd->seq_end_offset - cmd->seq_start_offset); 688 689 for (i = 0; i < cmd->pdu_count; i++) { 690 pdu = &cmd->pdu_list[i]; 691 692 if (pdu->status != ISCSI_PDU_RECEIVED_OK) 693 continue; 694 695 if ((pdu->offset >= cmd->seq_start_offset) && 696 ((pdu->offset + pdu->length) <= 697 cmd->seq_end_offset)) { 698 if (!cmd->unsolicited_data) 699 cmd->next_burst_len -= pdu->length; 700 else 701 cmd->first_burst_len -= pdu->length; 702 703 cmd->write_data_done -= pdu->length; 704 pdu->status = ISCSI_PDU_NOT_RECEIVED; 705 } 706 } 707 } else { 708 struct iscsi_seq *seq = NULL; 709 710 seq = iscsit_get_seq_holder(cmd, pdu_offset, pdu_length); 711 if (!seq) 712 return -1; 713 714 *r2t_offset = seq->orig_offset; 715 *r2t_length = seq->xfer_len; 716 717 cmd->write_data_done -= (seq->offset - seq->orig_offset); 718 if (cmd->immediate_data) 719 cmd->first_burst_len = cmd->write_data_done; 720 721 seq->data_sn = 0; 722 seq->offset = seq->orig_offset; 723 seq->next_burst_len = 0; 724 seq->status = DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY; 725 726 if (conn->sess->sess_ops->DataPDUInOrder) 727 return 0; 728 729 for (i = 0; i < seq->pdu_count; i++) { 730 pdu = &cmd->pdu_list[i+seq->pdu_start]; 731 732 if (pdu->status != ISCSI_PDU_RECEIVED_OK) 733 continue; 734 735 pdu->status = ISCSI_PDU_NOT_RECEIVED; 736 } 737 } 738 739 return 0; 740 } 741 742 int iscsit_recover_dataout_sequence( 743 struct iscsi_cmd *cmd, 744 u32 pdu_offset, 745 u32 pdu_length) 746 { 747 u32 r2t_length = 0, r2t_offset = 0; 748 749 spin_lock_bh(&cmd->istate_lock); 750 cmd->cmd_flags |= ICF_WITHIN_COMMAND_RECOVERY; 751 spin_unlock_bh(&cmd->istate_lock); 752 753 if (iscsit_recalculate_dataout_values(cmd, pdu_offset, pdu_length, 754 &r2t_offset, &r2t_length) < 0) 755 return DATAOUT_CANNOT_RECOVER; 756 757 iscsit_send_recovery_r2t(cmd, r2t_offset, r2t_length); 758 759 return DATAOUT_WITHIN_COMMAND_RECOVERY; 760 } 761 762 static struct iscsi_ooo_cmdsn *iscsit_allocate_ooo_cmdsn(void) 763 { 764 struct iscsi_ooo_cmdsn *ooo_cmdsn = NULL; 765 766 ooo_cmdsn = kmem_cache_zalloc(lio_ooo_cache, GFP_ATOMIC); 767 if (!ooo_cmdsn) { 768 pr_err("Unable to allocate memory for" 769 " struct iscsi_ooo_cmdsn.\n"); 770 return NULL; 771 } 772 INIT_LIST_HEAD(&ooo_cmdsn->ooo_list); 773 774 return ooo_cmdsn; 775 } 776 777 static int iscsit_attach_ooo_cmdsn( 778 struct iscsi_session *sess, 779 struct iscsi_ooo_cmdsn *ooo_cmdsn) 780 { 781 struct iscsi_ooo_cmdsn *ooo_tail, *ooo_tmp; 782 783 lockdep_assert_held(&sess->cmdsn_mutex); 784 785 /* 786 * We attach the struct iscsi_ooo_cmdsn entry to the out of order 787 * list in increasing CmdSN order. 788 * This allows iscsi_execute_ooo_cmdsns() to detect any 789 * additional CmdSN holes while performing delayed execution. 790 */ 791 if (list_empty(&sess->sess_ooo_cmdsn_list)) 792 list_add_tail(&ooo_cmdsn->ooo_list, 793 &sess->sess_ooo_cmdsn_list); 794 else { 795 ooo_tail = list_entry(sess->sess_ooo_cmdsn_list.prev, 796 typeof(*ooo_tail), ooo_list); 797 /* 798 * CmdSN is greater than the tail of the list. 799 */ 800 if (iscsi_sna_lt(ooo_tail->cmdsn, ooo_cmdsn->cmdsn)) 801 list_add_tail(&ooo_cmdsn->ooo_list, 802 &sess->sess_ooo_cmdsn_list); 803 else { 804 /* 805 * CmdSN is either lower than the head, or somewhere 806 * in the middle. 807 */ 808 list_for_each_entry(ooo_tmp, &sess->sess_ooo_cmdsn_list, 809 ooo_list) { 810 if (iscsi_sna_lt(ooo_tmp->cmdsn, ooo_cmdsn->cmdsn)) 811 continue; 812 813 /* Insert before this entry */ 814 list_add(&ooo_cmdsn->ooo_list, 815 ooo_tmp->ooo_list.prev); 816 break; 817 } 818 } 819 } 820 821 return 0; 822 } 823 824 /* 825 * Removes an struct iscsi_ooo_cmdsn from a session's list, 826 * called with struct iscsi_session->cmdsn_mutex held. 827 */ 828 void iscsit_remove_ooo_cmdsn( 829 struct iscsi_session *sess, 830 struct iscsi_ooo_cmdsn *ooo_cmdsn) 831 { 832 list_del(&ooo_cmdsn->ooo_list); 833 kmem_cache_free(lio_ooo_cache, ooo_cmdsn); 834 } 835 836 void iscsit_clear_ooo_cmdsns_for_conn(struct iscsi_conn *conn) 837 { 838 struct iscsi_ooo_cmdsn *ooo_cmdsn; 839 struct iscsi_session *sess = conn->sess; 840 841 mutex_lock(&sess->cmdsn_mutex); 842 list_for_each_entry(ooo_cmdsn, &sess->sess_ooo_cmdsn_list, ooo_list) { 843 if (ooo_cmdsn->cid != conn->cid) 844 continue; 845 846 ooo_cmdsn->cmd = NULL; 847 } 848 mutex_unlock(&sess->cmdsn_mutex); 849 } 850 851 int iscsit_execute_ooo_cmdsns(struct iscsi_session *sess) 852 { 853 int ooo_count = 0; 854 struct iscsi_cmd *cmd = NULL; 855 struct iscsi_ooo_cmdsn *ooo_cmdsn, *ooo_cmdsn_tmp; 856 857 lockdep_assert_held(&sess->cmdsn_mutex); 858 859 list_for_each_entry_safe(ooo_cmdsn, ooo_cmdsn_tmp, 860 &sess->sess_ooo_cmdsn_list, ooo_list) { 861 if (ooo_cmdsn->cmdsn != sess->exp_cmd_sn) 862 continue; 863 864 if (!ooo_cmdsn->cmd) { 865 sess->exp_cmd_sn++; 866 iscsit_remove_ooo_cmdsn(sess, ooo_cmdsn); 867 continue; 868 } 869 870 cmd = ooo_cmdsn->cmd; 871 cmd->i_state = cmd->deferred_i_state; 872 ooo_count++; 873 sess->exp_cmd_sn++; 874 pr_debug("Executing out of order CmdSN: 0x%08x," 875 " incremented ExpCmdSN to 0x%08x.\n", 876 cmd->cmd_sn, sess->exp_cmd_sn); 877 878 iscsit_remove_ooo_cmdsn(sess, ooo_cmdsn); 879 880 if (iscsit_execute_cmd(cmd, 1) < 0) 881 return -1; 882 883 continue; 884 } 885 886 return ooo_count; 887 } 888 889 /* 890 * Called either: 891 * 892 * 1. With sess->cmdsn_mutex held from iscsi_execute_ooo_cmdsns() 893 * or iscsi_check_received_cmdsn(). 894 * 2. With no locks held directly from iscsi_handle_XXX_pdu() functions 895 * for immediate commands. 896 */ 897 int iscsit_execute_cmd(struct iscsi_cmd *cmd, int ooo) 898 { 899 struct se_cmd *se_cmd = &cmd->se_cmd; 900 struct iscsi_conn *conn = cmd->conn; 901 int lr = 0; 902 903 spin_lock_bh(&cmd->istate_lock); 904 if (ooo) 905 cmd->cmd_flags &= ~ICF_OOO_CMDSN; 906 907 switch (cmd->iscsi_opcode) { 908 case ISCSI_OP_SCSI_CMD: 909 /* 910 * Go ahead and send the CHECK_CONDITION status for 911 * any SCSI CDB exceptions that may have occurred. 912 */ 913 if (cmd->sense_reason) { 914 if (cmd->sense_reason == TCM_RESERVATION_CONFLICT) { 915 cmd->i_state = ISTATE_SEND_STATUS; 916 spin_unlock_bh(&cmd->istate_lock); 917 iscsit_add_cmd_to_response_queue(cmd, cmd->conn, 918 cmd->i_state); 919 return 0; 920 } 921 spin_unlock_bh(&cmd->istate_lock); 922 if (cmd->se_cmd.transport_state & CMD_T_ABORTED) 923 return 0; 924 return transport_send_check_condition_and_sense(se_cmd, 925 cmd->sense_reason, 0); 926 } 927 /* 928 * Special case for delayed CmdSN with Immediate 929 * Data and/or Unsolicited Data Out attached. 930 */ 931 if (cmd->immediate_data) { 932 if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) { 933 spin_unlock_bh(&cmd->istate_lock); 934 target_execute_cmd(&cmd->se_cmd); 935 return 0; 936 } 937 spin_unlock_bh(&cmd->istate_lock); 938 939 if (!(cmd->cmd_flags & 940 ICF_NON_IMMEDIATE_UNSOLICITED_DATA)) { 941 if (cmd->se_cmd.transport_state & CMD_T_ABORTED) 942 return 0; 943 944 iscsit_set_dataout_sequence_values(cmd); 945 conn->conn_transport->iscsit_get_dataout(conn, cmd, false); 946 } 947 return 0; 948 } 949 /* 950 * The default handler. 951 */ 952 spin_unlock_bh(&cmd->istate_lock); 953 954 if ((cmd->data_direction == DMA_TO_DEVICE) && 955 !(cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA)) { 956 if (cmd->se_cmd.transport_state & CMD_T_ABORTED) 957 return 0; 958 959 iscsit_set_unsolicited_dataout(cmd); 960 } 961 return transport_handle_cdb_direct(&cmd->se_cmd); 962 963 case ISCSI_OP_NOOP_OUT: 964 case ISCSI_OP_TEXT: 965 spin_unlock_bh(&cmd->istate_lock); 966 iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state); 967 break; 968 case ISCSI_OP_SCSI_TMFUNC: 969 if (cmd->se_cmd.se_tmr_req->response) { 970 spin_unlock_bh(&cmd->istate_lock); 971 iscsit_add_cmd_to_response_queue(cmd, cmd->conn, 972 cmd->i_state); 973 return 0; 974 } 975 spin_unlock_bh(&cmd->istate_lock); 976 977 return transport_generic_handle_tmr(&cmd->se_cmd); 978 case ISCSI_OP_LOGOUT: 979 spin_unlock_bh(&cmd->istate_lock); 980 switch (cmd->logout_reason) { 981 case ISCSI_LOGOUT_REASON_CLOSE_SESSION: 982 lr = iscsit_logout_closesession(cmd, cmd->conn); 983 break; 984 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: 985 lr = iscsit_logout_closeconnection(cmd, cmd->conn); 986 break; 987 case ISCSI_LOGOUT_REASON_RECOVERY: 988 lr = iscsit_logout_removeconnforrecovery(cmd, cmd->conn); 989 break; 990 default: 991 pr_err("Unknown iSCSI Logout Request Code:" 992 " 0x%02x\n", cmd->logout_reason); 993 return -1; 994 } 995 996 return lr; 997 default: 998 spin_unlock_bh(&cmd->istate_lock); 999 pr_err("Cannot perform out of order execution for" 1000 " unknown iSCSI Opcode: 0x%02x\n", cmd->iscsi_opcode); 1001 return -1; 1002 } 1003 1004 return 0; 1005 } 1006 1007 void iscsit_free_all_ooo_cmdsns(struct iscsi_session *sess) 1008 { 1009 struct iscsi_ooo_cmdsn *ooo_cmdsn, *ooo_cmdsn_tmp; 1010 1011 mutex_lock(&sess->cmdsn_mutex); 1012 list_for_each_entry_safe(ooo_cmdsn, ooo_cmdsn_tmp, 1013 &sess->sess_ooo_cmdsn_list, ooo_list) { 1014 1015 list_del(&ooo_cmdsn->ooo_list); 1016 kmem_cache_free(lio_ooo_cache, ooo_cmdsn); 1017 } 1018 mutex_unlock(&sess->cmdsn_mutex); 1019 } 1020 1021 int iscsit_handle_ooo_cmdsn( 1022 struct iscsi_session *sess, 1023 struct iscsi_cmd *cmd, 1024 u32 cmdsn) 1025 { 1026 int batch = 0; 1027 struct iscsi_ooo_cmdsn *ooo_cmdsn = NULL, *ooo_tail = NULL; 1028 1029 cmd->deferred_i_state = cmd->i_state; 1030 cmd->i_state = ISTATE_DEFERRED_CMD; 1031 cmd->cmd_flags |= ICF_OOO_CMDSN; 1032 1033 if (list_empty(&sess->sess_ooo_cmdsn_list)) 1034 batch = 1; 1035 else { 1036 ooo_tail = list_entry(sess->sess_ooo_cmdsn_list.prev, 1037 typeof(*ooo_tail), ooo_list); 1038 if (ooo_tail->cmdsn != (cmdsn - 1)) 1039 batch = 1; 1040 } 1041 1042 ooo_cmdsn = iscsit_allocate_ooo_cmdsn(); 1043 if (!ooo_cmdsn) 1044 return -ENOMEM; 1045 1046 ooo_cmdsn->cmd = cmd; 1047 ooo_cmdsn->batch_count = (batch) ? 1048 (cmdsn - sess->exp_cmd_sn) : 1; 1049 ooo_cmdsn->cid = cmd->conn->cid; 1050 ooo_cmdsn->exp_cmdsn = sess->exp_cmd_sn; 1051 ooo_cmdsn->cmdsn = cmdsn; 1052 1053 if (iscsit_attach_ooo_cmdsn(sess, ooo_cmdsn) < 0) { 1054 kmem_cache_free(lio_ooo_cache, ooo_cmdsn); 1055 return -ENOMEM; 1056 } 1057 1058 return 0; 1059 } 1060 1061 static int iscsit_set_dataout_timeout_values( 1062 struct iscsi_cmd *cmd, 1063 u32 *offset, 1064 u32 *length) 1065 { 1066 struct iscsi_conn *conn = cmd->conn; 1067 struct iscsi_r2t *r2t; 1068 1069 if (cmd->unsolicited_data) { 1070 *offset = 0; 1071 *length = (conn->sess->sess_ops->FirstBurstLength > 1072 cmd->se_cmd.data_length) ? 1073 cmd->se_cmd.data_length : 1074 conn->sess->sess_ops->FirstBurstLength; 1075 return 0; 1076 } 1077 1078 spin_lock_bh(&cmd->r2t_lock); 1079 if (list_empty(&cmd->cmd_r2t_list)) { 1080 pr_err("cmd->cmd_r2t_list is empty!\n"); 1081 spin_unlock_bh(&cmd->r2t_lock); 1082 return -1; 1083 } 1084 1085 list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) { 1086 if (r2t->sent_r2t && !r2t->recovery_r2t && !r2t->seq_complete) { 1087 *offset = r2t->offset; 1088 *length = r2t->xfer_len; 1089 spin_unlock_bh(&cmd->r2t_lock); 1090 return 0; 1091 } 1092 } 1093 spin_unlock_bh(&cmd->r2t_lock); 1094 1095 pr_err("Unable to locate any incomplete DataOUT" 1096 " sequences for ITT: 0x%08x.\n", cmd->init_task_tag); 1097 1098 return -1; 1099 } 1100 1101 /* 1102 * NOTE: Called from interrupt (timer) context. 1103 */ 1104 void iscsit_handle_dataout_timeout(struct timer_list *t) 1105 { 1106 u32 pdu_length = 0, pdu_offset = 0; 1107 u32 r2t_length = 0, r2t_offset = 0; 1108 struct iscsi_cmd *cmd = from_timer(cmd, t, dataout_timer); 1109 struct iscsi_conn *conn = cmd->conn; 1110 struct iscsi_session *sess = NULL; 1111 struct iscsi_node_attrib *na; 1112 1113 iscsit_inc_conn_usage_count(conn); 1114 1115 spin_lock_bh(&cmd->dataout_timeout_lock); 1116 if (cmd->dataout_timer_flags & ISCSI_TF_STOP) { 1117 spin_unlock_bh(&cmd->dataout_timeout_lock); 1118 iscsit_dec_conn_usage_count(conn); 1119 return; 1120 } 1121 cmd->dataout_timer_flags &= ~ISCSI_TF_RUNNING; 1122 sess = conn->sess; 1123 na = iscsit_tpg_get_node_attrib(sess); 1124 1125 if (!sess->sess_ops->ErrorRecoveryLevel) { 1126 pr_err("Unable to recover from DataOut timeout while" 1127 " in ERL=0, closing iSCSI connection for I_T Nexus" 1128 " %s,i,0x%6phN,%s,t,0x%02x\n", 1129 sess->sess_ops->InitiatorName, sess->isid, 1130 sess->tpg->tpg_tiqn->tiqn, (u32)sess->tpg->tpgt); 1131 goto failure; 1132 } 1133 1134 if (++cmd->dataout_timeout_retries == na->dataout_timeout_retries) { 1135 pr_err("Command ITT: 0x%08x exceeded max retries" 1136 " for DataOUT timeout %u, closing iSCSI connection for" 1137 " I_T Nexus %s,i,0x%6phN,%s,t,0x%02x\n", 1138 cmd->init_task_tag, na->dataout_timeout_retries, 1139 sess->sess_ops->InitiatorName, sess->isid, 1140 sess->tpg->tpg_tiqn->tiqn, (u32)sess->tpg->tpgt); 1141 goto failure; 1142 } 1143 1144 cmd->cmd_flags |= ICF_WITHIN_COMMAND_RECOVERY; 1145 1146 if (conn->sess->sess_ops->DataSequenceInOrder) { 1147 if (conn->sess->sess_ops->DataPDUInOrder) { 1148 pdu_offset = cmd->write_data_done; 1149 if ((pdu_offset + (conn->sess->sess_ops->MaxBurstLength - 1150 cmd->next_burst_len)) > cmd->se_cmd.data_length) 1151 pdu_length = (cmd->se_cmd.data_length - 1152 cmd->write_data_done); 1153 else 1154 pdu_length = (conn->sess->sess_ops->MaxBurstLength - 1155 cmd->next_burst_len); 1156 } else { 1157 pdu_offset = cmd->seq_start_offset; 1158 pdu_length = (cmd->seq_end_offset - 1159 cmd->seq_start_offset); 1160 } 1161 } else { 1162 if (iscsit_set_dataout_timeout_values(cmd, &pdu_offset, 1163 &pdu_length) < 0) 1164 goto failure; 1165 } 1166 1167 if (iscsit_recalculate_dataout_values(cmd, pdu_offset, pdu_length, 1168 &r2t_offset, &r2t_length) < 0) 1169 goto failure; 1170 1171 pr_debug("Command ITT: 0x%08x timed out waiting for" 1172 " completion of %sDataOUT Sequence Offset: %u, Length: %u\n", 1173 cmd->init_task_tag, (cmd->unsolicited_data) ? "Unsolicited " : 1174 "", r2t_offset, r2t_length); 1175 1176 if (iscsit_send_recovery_r2t(cmd, r2t_offset, r2t_length) < 0) 1177 goto failure; 1178 1179 iscsit_start_dataout_timer(cmd, conn); 1180 spin_unlock_bh(&cmd->dataout_timeout_lock); 1181 iscsit_dec_conn_usage_count(conn); 1182 1183 return; 1184 1185 failure: 1186 spin_unlock_bh(&cmd->dataout_timeout_lock); 1187 iscsit_fill_cxn_timeout_err_stats(sess); 1188 iscsit_cause_connection_reinstatement(conn, 0); 1189 iscsit_dec_conn_usage_count(conn); 1190 } 1191 1192 void iscsit_mod_dataout_timer(struct iscsi_cmd *cmd) 1193 { 1194 struct iscsi_conn *conn = cmd->conn; 1195 struct iscsi_session *sess = conn->sess; 1196 struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess); 1197 1198 spin_lock_bh(&cmd->dataout_timeout_lock); 1199 if (!(cmd->dataout_timer_flags & ISCSI_TF_RUNNING)) { 1200 spin_unlock_bh(&cmd->dataout_timeout_lock); 1201 return; 1202 } 1203 1204 mod_timer(&cmd->dataout_timer, 1205 (get_jiffies_64() + na->dataout_timeout * HZ)); 1206 pr_debug("Updated DataOUT timer for ITT: 0x%08x", 1207 cmd->init_task_tag); 1208 spin_unlock_bh(&cmd->dataout_timeout_lock); 1209 } 1210 1211 void iscsit_start_dataout_timer( 1212 struct iscsi_cmd *cmd, 1213 struct iscsi_conn *conn) 1214 { 1215 struct iscsi_session *sess = conn->sess; 1216 struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess); 1217 1218 lockdep_assert_held(&cmd->dataout_timeout_lock); 1219 1220 if (cmd->dataout_timer_flags & ISCSI_TF_RUNNING) 1221 return; 1222 1223 pr_debug("Starting DataOUT timer for ITT: 0x%08x on" 1224 " CID: %hu.\n", cmd->init_task_tag, conn->cid); 1225 1226 cmd->dataout_timer_flags &= ~ISCSI_TF_STOP; 1227 cmd->dataout_timer_flags |= ISCSI_TF_RUNNING; 1228 mod_timer(&cmd->dataout_timer, jiffies + na->dataout_timeout * HZ); 1229 } 1230 1231 void iscsit_stop_dataout_timer(struct iscsi_cmd *cmd) 1232 { 1233 spin_lock_bh(&cmd->dataout_timeout_lock); 1234 if (!(cmd->dataout_timer_flags & ISCSI_TF_RUNNING)) { 1235 spin_unlock_bh(&cmd->dataout_timeout_lock); 1236 return; 1237 } 1238 cmd->dataout_timer_flags |= ISCSI_TF_STOP; 1239 spin_unlock_bh(&cmd->dataout_timeout_lock); 1240 1241 del_timer_sync(&cmd->dataout_timer); 1242 1243 spin_lock_bh(&cmd->dataout_timeout_lock); 1244 cmd->dataout_timer_flags &= ~ISCSI_TF_RUNNING; 1245 pr_debug("Stopped DataOUT Timer for ITT: 0x%08x\n", 1246 cmd->init_task_tag); 1247 spin_unlock_bh(&cmd->dataout_timeout_lock); 1248 } 1249 EXPORT_SYMBOL(iscsit_stop_dataout_timer); 1250