1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /******************************************************************************* 3 * This file contains main functions related to the iSCSI Target Core Driver. 4 * 5 * (c) Copyright 2007-2013 Datera, Inc. 6 * 7 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org> 8 * 9 ******************************************************************************/ 10 11 #include <crypto/hash.h> 12 #include <linux/string.h> 13 #include <linux/kthread.h> 14 #include <linux/completion.h> 15 #include <linux/module.h> 16 #include <linux/vmalloc.h> 17 #include <linux/idr.h> 18 #include <linux/delay.h> 19 #include <linux/sched/signal.h> 20 #include <asm/unaligned.h> 21 #include <linux/inet.h> 22 #include <net/ipv6.h> 23 #include <scsi/scsi_proto.h> 24 #include <scsi/iscsi_proto.h> 25 #include <scsi/scsi_tcq.h> 26 #include <target/target_core_base.h> 27 #include <target/target_core_fabric.h> 28 29 #include <target/iscsi/iscsi_target_core.h> 30 #include "iscsi_target_parameters.h" 31 #include "iscsi_target_seq_pdu_list.h" 32 #include "iscsi_target_datain_values.h" 33 #include "iscsi_target_erl0.h" 34 #include "iscsi_target_erl1.h" 35 #include "iscsi_target_erl2.h" 36 #include "iscsi_target_login.h" 37 #include "iscsi_target_tmr.h" 38 #include "iscsi_target_tpg.h" 39 #include "iscsi_target_util.h" 40 #include "iscsi_target.h" 41 #include "iscsi_target_device.h" 42 #include <target/iscsi/iscsi_target_stat.h> 43 44 #include <target/iscsi/iscsi_transport.h> 45 46 static LIST_HEAD(g_tiqn_list); 47 static LIST_HEAD(g_np_list); 48 static DEFINE_SPINLOCK(tiqn_lock); 49 static DEFINE_MUTEX(np_lock); 50 51 static struct idr tiqn_idr; 52 DEFINE_IDA(sess_ida); 53 struct mutex auth_id_lock; 54 55 struct iscsit_global *iscsit_global; 56 57 struct kmem_cache *lio_qr_cache; 58 struct kmem_cache *lio_dr_cache; 59 struct kmem_cache *lio_ooo_cache; 60 struct kmem_cache *lio_r2t_cache; 61 62 static int iscsit_handle_immediate_data(struct iscsi_cmd *, 63 struct iscsi_scsi_req *, u32); 64 65 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf) 66 { 67 struct iscsi_tiqn *tiqn = NULL; 68 69 spin_lock(&tiqn_lock); 70 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) { 71 if (!strcmp(tiqn->tiqn, buf)) { 72 73 spin_lock(&tiqn->tiqn_state_lock); 74 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) { 75 tiqn->tiqn_access_count++; 76 spin_unlock(&tiqn->tiqn_state_lock); 77 spin_unlock(&tiqn_lock); 78 return tiqn; 79 } 80 spin_unlock(&tiqn->tiqn_state_lock); 81 } 82 } 83 spin_unlock(&tiqn_lock); 84 85 return NULL; 86 } 87 88 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn) 89 { 90 spin_lock(&tiqn->tiqn_state_lock); 91 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) { 92 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN; 93 spin_unlock(&tiqn->tiqn_state_lock); 94 return 0; 95 } 96 spin_unlock(&tiqn->tiqn_state_lock); 97 98 return -1; 99 } 100 101 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn) 102 { 103 spin_lock(&tiqn->tiqn_state_lock); 104 tiqn->tiqn_access_count--; 105 spin_unlock(&tiqn->tiqn_state_lock); 106 } 107 108 /* 109 * Note that IQN formatting is expected to be done in userspace, and 110 * no explict IQN format checks are done here. 111 */ 112 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf) 113 { 114 struct iscsi_tiqn *tiqn = NULL; 115 int ret; 116 117 if (strlen(buf) >= ISCSI_IQN_LEN) { 118 pr_err("Target IQN exceeds %d bytes\n", 119 ISCSI_IQN_LEN); 120 return ERR_PTR(-EINVAL); 121 } 122 123 tiqn = kzalloc(sizeof(*tiqn), GFP_KERNEL); 124 if (!tiqn) 125 return ERR_PTR(-ENOMEM); 126 127 sprintf(tiqn->tiqn, "%s", buf); 128 INIT_LIST_HEAD(&tiqn->tiqn_list); 129 INIT_LIST_HEAD(&tiqn->tiqn_tpg_list); 130 spin_lock_init(&tiqn->tiqn_state_lock); 131 spin_lock_init(&tiqn->tiqn_tpg_lock); 132 spin_lock_init(&tiqn->sess_err_stats.lock); 133 spin_lock_init(&tiqn->login_stats.lock); 134 spin_lock_init(&tiqn->logout_stats.lock); 135 136 tiqn->tiqn_state = TIQN_STATE_ACTIVE; 137 138 idr_preload(GFP_KERNEL); 139 spin_lock(&tiqn_lock); 140 141 ret = idr_alloc(&tiqn_idr, NULL, 0, 0, GFP_NOWAIT); 142 if (ret < 0) { 143 pr_err("idr_alloc() failed for tiqn->tiqn_index\n"); 144 spin_unlock(&tiqn_lock); 145 idr_preload_end(); 146 kfree(tiqn); 147 return ERR_PTR(ret); 148 } 149 tiqn->tiqn_index = ret; 150 list_add_tail(&tiqn->tiqn_list, &g_tiqn_list); 151 152 spin_unlock(&tiqn_lock); 153 idr_preload_end(); 154 155 pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn); 156 157 return tiqn; 158 159 } 160 161 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn) 162 { 163 /* 164 * Wait for accesses to said struct iscsi_tiqn to end. 165 */ 166 spin_lock(&tiqn->tiqn_state_lock); 167 while (tiqn->tiqn_access_count != 0) { 168 spin_unlock(&tiqn->tiqn_state_lock); 169 msleep(10); 170 spin_lock(&tiqn->tiqn_state_lock); 171 } 172 spin_unlock(&tiqn->tiqn_state_lock); 173 } 174 175 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn) 176 { 177 /* 178 * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN 179 * while holding tiqn->tiqn_state_lock. This means that all subsequent 180 * attempts to access this struct iscsi_tiqn will fail from both transport 181 * fabric and control code paths. 182 */ 183 if (iscsit_set_tiqn_shutdown(tiqn) < 0) { 184 pr_err("iscsit_set_tiqn_shutdown() failed\n"); 185 return; 186 } 187 188 iscsit_wait_for_tiqn(tiqn); 189 190 spin_lock(&tiqn_lock); 191 list_del(&tiqn->tiqn_list); 192 idr_remove(&tiqn_idr, tiqn->tiqn_index); 193 spin_unlock(&tiqn_lock); 194 195 pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n", 196 tiqn->tiqn); 197 kfree(tiqn); 198 } 199 200 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg) 201 { 202 int ret; 203 /* 204 * Determine if the network portal is accepting storage traffic. 205 */ 206 spin_lock_bh(&np->np_thread_lock); 207 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { 208 spin_unlock_bh(&np->np_thread_lock); 209 return -1; 210 } 211 spin_unlock_bh(&np->np_thread_lock); 212 /* 213 * Determine if the portal group is accepting storage traffic. 214 */ 215 spin_lock_bh(&tpg->tpg_state_lock); 216 if (tpg->tpg_state != TPG_STATE_ACTIVE) { 217 spin_unlock_bh(&tpg->tpg_state_lock); 218 return -1; 219 } 220 spin_unlock_bh(&tpg->tpg_state_lock); 221 222 /* 223 * Here we serialize access across the TIQN+TPG Tuple. 224 */ 225 ret = down_interruptible(&tpg->np_login_sem); 226 if (ret != 0) 227 return -1; 228 229 spin_lock_bh(&tpg->tpg_state_lock); 230 if (tpg->tpg_state != TPG_STATE_ACTIVE) { 231 spin_unlock_bh(&tpg->tpg_state_lock); 232 up(&tpg->np_login_sem); 233 return -1; 234 } 235 spin_unlock_bh(&tpg->tpg_state_lock); 236 237 return 0; 238 } 239 240 void iscsit_login_kref_put(struct kref *kref) 241 { 242 struct iscsi_tpg_np *tpg_np = container_of(kref, 243 struct iscsi_tpg_np, tpg_np_kref); 244 245 complete(&tpg_np->tpg_np_comp); 246 } 247 248 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg, 249 struct iscsi_tpg_np *tpg_np) 250 { 251 struct iscsi_tiqn *tiqn = tpg->tpg_tiqn; 252 253 up(&tpg->np_login_sem); 254 255 if (tpg_np) 256 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put); 257 258 if (tiqn) 259 iscsit_put_tiqn_for_login(tiqn); 260 261 return 0; 262 } 263 264 bool iscsit_check_np_match( 265 struct sockaddr_storage *sockaddr, 266 struct iscsi_np *np, 267 int network_transport) 268 { 269 struct sockaddr_in *sock_in, *sock_in_e; 270 struct sockaddr_in6 *sock_in6, *sock_in6_e; 271 bool ip_match = false; 272 u16 port, port_e; 273 274 if (sockaddr->ss_family == AF_INET6) { 275 sock_in6 = (struct sockaddr_in6 *)sockaddr; 276 sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr; 277 278 if (!memcmp(&sock_in6->sin6_addr.in6_u, 279 &sock_in6_e->sin6_addr.in6_u, 280 sizeof(struct in6_addr))) 281 ip_match = true; 282 283 port = ntohs(sock_in6->sin6_port); 284 port_e = ntohs(sock_in6_e->sin6_port); 285 } else { 286 sock_in = (struct sockaddr_in *)sockaddr; 287 sock_in_e = (struct sockaddr_in *)&np->np_sockaddr; 288 289 if (sock_in->sin_addr.s_addr == sock_in_e->sin_addr.s_addr) 290 ip_match = true; 291 292 port = ntohs(sock_in->sin_port); 293 port_e = ntohs(sock_in_e->sin_port); 294 } 295 296 if (ip_match && (port_e == port) && 297 (np->np_network_transport == network_transport)) 298 return true; 299 300 return false; 301 } 302 303 static struct iscsi_np *iscsit_get_np( 304 struct sockaddr_storage *sockaddr, 305 int network_transport) 306 { 307 struct iscsi_np *np; 308 bool match; 309 310 lockdep_assert_held(&np_lock); 311 312 list_for_each_entry(np, &g_np_list, np_list) { 313 spin_lock_bh(&np->np_thread_lock); 314 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { 315 spin_unlock_bh(&np->np_thread_lock); 316 continue; 317 } 318 319 match = iscsit_check_np_match(sockaddr, np, network_transport); 320 if (match) { 321 /* 322 * Increment the np_exports reference count now to 323 * prevent iscsit_del_np() below from being called 324 * while iscsi_tpg_add_network_portal() is called. 325 */ 326 np->np_exports++; 327 spin_unlock_bh(&np->np_thread_lock); 328 return np; 329 } 330 spin_unlock_bh(&np->np_thread_lock); 331 } 332 333 return NULL; 334 } 335 336 struct iscsi_np *iscsit_add_np( 337 struct sockaddr_storage *sockaddr, 338 int network_transport) 339 { 340 struct iscsi_np *np; 341 int ret; 342 343 mutex_lock(&np_lock); 344 345 /* 346 * Locate the existing struct iscsi_np if already active.. 347 */ 348 np = iscsit_get_np(sockaddr, network_transport); 349 if (np) { 350 mutex_unlock(&np_lock); 351 return np; 352 } 353 354 np = kzalloc(sizeof(*np), GFP_KERNEL); 355 if (!np) { 356 mutex_unlock(&np_lock); 357 return ERR_PTR(-ENOMEM); 358 } 359 360 np->np_flags |= NPF_IP_NETWORK; 361 np->np_network_transport = network_transport; 362 spin_lock_init(&np->np_thread_lock); 363 init_completion(&np->np_restart_comp); 364 INIT_LIST_HEAD(&np->np_list); 365 366 timer_setup(&np->np_login_timer, iscsi_handle_login_thread_timeout, 0); 367 368 ret = iscsi_target_setup_login_socket(np, sockaddr); 369 if (ret != 0) { 370 kfree(np); 371 mutex_unlock(&np_lock); 372 return ERR_PTR(ret); 373 } 374 375 np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np"); 376 if (IS_ERR(np->np_thread)) { 377 pr_err("Unable to create kthread: iscsi_np\n"); 378 ret = PTR_ERR(np->np_thread); 379 kfree(np); 380 mutex_unlock(&np_lock); 381 return ERR_PTR(ret); 382 } 383 /* 384 * Increment the np_exports reference count now to prevent 385 * iscsit_del_np() below from being run while a new call to 386 * iscsi_tpg_add_network_portal() for a matching iscsi_np is 387 * active. We don't need to hold np->np_thread_lock at this 388 * point because iscsi_np has not been added to g_np_list yet. 389 */ 390 np->np_exports = 1; 391 np->np_thread_state = ISCSI_NP_THREAD_ACTIVE; 392 393 list_add_tail(&np->np_list, &g_np_list); 394 mutex_unlock(&np_lock); 395 396 pr_debug("CORE[0] - Added Network Portal: %pISpc on %s\n", 397 &np->np_sockaddr, np->np_transport->name); 398 399 return np; 400 } 401 402 int iscsit_reset_np_thread( 403 struct iscsi_np *np, 404 struct iscsi_tpg_np *tpg_np, 405 struct iscsi_portal_group *tpg, 406 bool shutdown) 407 { 408 spin_lock_bh(&np->np_thread_lock); 409 if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) { 410 spin_unlock_bh(&np->np_thread_lock); 411 return 0; 412 } 413 np->np_thread_state = ISCSI_NP_THREAD_RESET; 414 atomic_inc(&np->np_reset_count); 415 416 if (np->np_thread) { 417 spin_unlock_bh(&np->np_thread_lock); 418 send_sig(SIGINT, np->np_thread, 1); 419 wait_for_completion(&np->np_restart_comp); 420 spin_lock_bh(&np->np_thread_lock); 421 } 422 spin_unlock_bh(&np->np_thread_lock); 423 424 if (tpg_np && shutdown) { 425 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put); 426 427 wait_for_completion(&tpg_np->tpg_np_comp); 428 } 429 430 return 0; 431 } 432 433 static void iscsit_free_np(struct iscsi_np *np) 434 { 435 if (np->np_socket) 436 sock_release(np->np_socket); 437 } 438 439 int iscsit_del_np(struct iscsi_np *np) 440 { 441 spin_lock_bh(&np->np_thread_lock); 442 np->np_exports--; 443 if (np->np_exports) { 444 np->enabled = true; 445 spin_unlock_bh(&np->np_thread_lock); 446 return 0; 447 } 448 np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN; 449 spin_unlock_bh(&np->np_thread_lock); 450 451 if (np->np_thread) { 452 /* 453 * We need to send the signal to wakeup Linux/Net 454 * which may be sleeping in sock_accept().. 455 */ 456 send_sig(SIGINT, np->np_thread, 1); 457 kthread_stop(np->np_thread); 458 np->np_thread = NULL; 459 } 460 461 np->np_transport->iscsit_free_np(np); 462 463 mutex_lock(&np_lock); 464 list_del(&np->np_list); 465 mutex_unlock(&np_lock); 466 467 pr_debug("CORE[0] - Removed Network Portal: %pISpc on %s\n", 468 &np->np_sockaddr, np->np_transport->name); 469 470 iscsit_put_transport(np->np_transport); 471 kfree(np); 472 return 0; 473 } 474 475 static void iscsit_get_rx_pdu(struct iscsi_conn *); 476 477 int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 478 { 479 return iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state); 480 } 481 EXPORT_SYMBOL(iscsit_queue_rsp); 482 483 void iscsit_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 484 { 485 spin_lock_bh(&conn->cmd_lock); 486 if (!list_empty(&cmd->i_conn_node) && 487 !(cmd->se_cmd.transport_state & CMD_T_FABRIC_STOP)) 488 list_del_init(&cmd->i_conn_node); 489 spin_unlock_bh(&conn->cmd_lock); 490 491 __iscsit_free_cmd(cmd, true); 492 } 493 EXPORT_SYMBOL(iscsit_aborted_task); 494 495 static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *, 496 u32, u32, const void *, void *); 497 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *); 498 499 static int 500 iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 501 const void *data_buf, u32 data_buf_len) 502 { 503 struct iscsi_hdr *hdr = (struct iscsi_hdr *)cmd->pdu; 504 struct kvec *iov; 505 u32 niov = 0, tx_size = ISCSI_HDR_LEN; 506 int ret; 507 508 iov = &cmd->iov_misc[0]; 509 iov[niov].iov_base = cmd->pdu; 510 iov[niov++].iov_len = ISCSI_HDR_LEN; 511 512 if (conn->conn_ops->HeaderDigest) { 513 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 514 515 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, 516 ISCSI_HDR_LEN, 0, NULL, 517 header_digest); 518 519 iov[0].iov_len += ISCSI_CRC_LEN; 520 tx_size += ISCSI_CRC_LEN; 521 pr_debug("Attaching CRC32C HeaderDigest" 522 " to opcode 0x%x 0x%08x\n", 523 hdr->opcode, *header_digest); 524 } 525 526 if (data_buf_len) { 527 u32 padding = ((-data_buf_len) & 3); 528 529 iov[niov].iov_base = (void *)data_buf; 530 iov[niov++].iov_len = data_buf_len; 531 tx_size += data_buf_len; 532 533 if (padding != 0) { 534 iov[niov].iov_base = &cmd->pad_bytes; 535 iov[niov++].iov_len = padding; 536 tx_size += padding; 537 pr_debug("Attaching %u additional" 538 " padding bytes.\n", padding); 539 } 540 541 if (conn->conn_ops->DataDigest) { 542 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, 543 data_buf, data_buf_len, 544 padding, &cmd->pad_bytes, 545 &cmd->data_crc); 546 547 iov[niov].iov_base = &cmd->data_crc; 548 iov[niov++].iov_len = ISCSI_CRC_LEN; 549 tx_size += ISCSI_CRC_LEN; 550 pr_debug("Attached DataDigest for %u" 551 " bytes opcode 0x%x, CRC 0x%08x\n", 552 data_buf_len, hdr->opcode, cmd->data_crc); 553 } 554 } 555 556 cmd->iov_misc_count = niov; 557 cmd->tx_size = tx_size; 558 559 ret = iscsit_send_tx_data(cmd, conn, 1); 560 if (ret < 0) { 561 iscsit_tx_thread_wait_for_tcp(conn); 562 return ret; 563 } 564 565 return 0; 566 } 567 568 static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec, 569 u32 data_offset, u32 data_length); 570 static void iscsit_unmap_iovec(struct iscsi_cmd *); 571 static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsi_cmd *, 572 u32, u32, u32, u8 *); 573 static int 574 iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 575 const struct iscsi_datain *datain) 576 { 577 struct kvec *iov; 578 u32 iov_count = 0, tx_size = 0; 579 int ret, iov_ret; 580 581 iov = &cmd->iov_data[0]; 582 iov[iov_count].iov_base = cmd->pdu; 583 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 584 tx_size += ISCSI_HDR_LEN; 585 586 if (conn->conn_ops->HeaderDigest) { 587 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 588 589 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu, 590 ISCSI_HDR_LEN, 0, NULL, 591 header_digest); 592 593 iov[0].iov_len += ISCSI_CRC_LEN; 594 tx_size += ISCSI_CRC_LEN; 595 596 pr_debug("Attaching CRC32 HeaderDigest for DataIN PDU 0x%08x\n", 597 *header_digest); 598 } 599 600 iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[iov_count], 601 cmd->orig_iov_data_count - (iov_count + 2), 602 datain->offset, datain->length); 603 if (iov_ret < 0) 604 return -1; 605 606 iov_count += iov_ret; 607 tx_size += datain->length; 608 609 cmd->padding = ((-datain->length) & 3); 610 if (cmd->padding) { 611 iov[iov_count].iov_base = cmd->pad_bytes; 612 iov[iov_count++].iov_len = cmd->padding; 613 tx_size += cmd->padding; 614 615 pr_debug("Attaching %u padding bytes\n", cmd->padding); 616 } 617 618 if (conn->conn_ops->DataDigest) { 619 cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash, 620 cmd, datain->offset, 621 datain->length, 622 cmd->padding, 623 cmd->pad_bytes); 624 625 iov[iov_count].iov_base = &cmd->data_crc; 626 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 627 tx_size += ISCSI_CRC_LEN; 628 629 pr_debug("Attached CRC32C DataDigest %d bytes, crc 0x%08x\n", 630 datain->length + cmd->padding, cmd->data_crc); 631 } 632 633 cmd->iov_data_count = iov_count; 634 cmd->tx_size = tx_size; 635 636 ret = iscsit_fe_sendpage_sg(cmd, conn); 637 638 iscsit_unmap_iovec(cmd); 639 640 if (ret < 0) { 641 iscsit_tx_thread_wait_for_tcp(conn); 642 return ret; 643 } 644 645 return 0; 646 } 647 648 static int iscsit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 649 struct iscsi_datain_req *dr, const void *buf, 650 u32 buf_len) 651 { 652 if (dr) 653 return iscsit_xmit_datain_pdu(conn, cmd, buf); 654 else 655 return iscsit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 656 } 657 658 static enum target_prot_op iscsit_get_sup_prot_ops(struct iscsi_conn *conn) 659 { 660 return TARGET_PROT_NORMAL; 661 } 662 663 static struct iscsit_transport iscsi_target_transport = { 664 .name = "iSCSI/TCP", 665 .transport_type = ISCSI_TCP, 666 .rdma_shutdown = false, 667 .owner = NULL, 668 .iscsit_setup_np = iscsit_setup_np, 669 .iscsit_accept_np = iscsit_accept_np, 670 .iscsit_free_np = iscsit_free_np, 671 .iscsit_get_login_rx = iscsit_get_login_rx, 672 .iscsit_put_login_tx = iscsit_put_login_tx, 673 .iscsit_get_dataout = iscsit_build_r2ts_for_cmd, 674 .iscsit_immediate_queue = iscsit_immediate_queue, 675 .iscsit_response_queue = iscsit_response_queue, 676 .iscsit_queue_data_in = iscsit_queue_rsp, 677 .iscsit_queue_status = iscsit_queue_rsp, 678 .iscsit_aborted_task = iscsit_aborted_task, 679 .iscsit_xmit_pdu = iscsit_xmit_pdu, 680 .iscsit_get_rx_pdu = iscsit_get_rx_pdu, 681 .iscsit_get_sup_prot_ops = iscsit_get_sup_prot_ops, 682 }; 683 684 static int __init iscsi_target_init_module(void) 685 { 686 int ret = 0, size; 687 688 pr_debug("iSCSI-Target "ISCSIT_VERSION"\n"); 689 iscsit_global = kzalloc(sizeof(*iscsit_global), GFP_KERNEL); 690 if (!iscsit_global) 691 return -1; 692 693 spin_lock_init(&iscsit_global->ts_bitmap_lock); 694 mutex_init(&auth_id_lock); 695 idr_init(&tiqn_idr); 696 697 ret = target_register_template(&iscsi_ops); 698 if (ret) 699 goto out; 700 701 size = BITS_TO_LONGS(ISCSIT_BITMAP_BITS) * sizeof(long); 702 iscsit_global->ts_bitmap = vzalloc(size); 703 if (!iscsit_global->ts_bitmap) 704 goto configfs_out; 705 706 lio_qr_cache = kmem_cache_create("lio_qr_cache", 707 sizeof(struct iscsi_queue_req), 708 __alignof__(struct iscsi_queue_req), 0, NULL); 709 if (!lio_qr_cache) { 710 pr_err("Unable to kmem_cache_create() for" 711 " lio_qr_cache\n"); 712 goto bitmap_out; 713 } 714 715 lio_dr_cache = kmem_cache_create("lio_dr_cache", 716 sizeof(struct iscsi_datain_req), 717 __alignof__(struct iscsi_datain_req), 0, NULL); 718 if (!lio_dr_cache) { 719 pr_err("Unable to kmem_cache_create() for" 720 " lio_dr_cache\n"); 721 goto qr_out; 722 } 723 724 lio_ooo_cache = kmem_cache_create("lio_ooo_cache", 725 sizeof(struct iscsi_ooo_cmdsn), 726 __alignof__(struct iscsi_ooo_cmdsn), 0, NULL); 727 if (!lio_ooo_cache) { 728 pr_err("Unable to kmem_cache_create() for" 729 " lio_ooo_cache\n"); 730 goto dr_out; 731 } 732 733 lio_r2t_cache = kmem_cache_create("lio_r2t_cache", 734 sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t), 735 0, NULL); 736 if (!lio_r2t_cache) { 737 pr_err("Unable to kmem_cache_create() for" 738 " lio_r2t_cache\n"); 739 goto ooo_out; 740 } 741 742 iscsit_register_transport(&iscsi_target_transport); 743 744 if (iscsit_load_discovery_tpg() < 0) 745 goto r2t_out; 746 747 return ret; 748 r2t_out: 749 iscsit_unregister_transport(&iscsi_target_transport); 750 kmem_cache_destroy(lio_r2t_cache); 751 ooo_out: 752 kmem_cache_destroy(lio_ooo_cache); 753 dr_out: 754 kmem_cache_destroy(lio_dr_cache); 755 qr_out: 756 kmem_cache_destroy(lio_qr_cache); 757 bitmap_out: 758 vfree(iscsit_global->ts_bitmap); 759 configfs_out: 760 /* XXX: this probably wants it to be it's own unwind step.. */ 761 if (iscsit_global->discovery_tpg) 762 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1); 763 target_unregister_template(&iscsi_ops); 764 out: 765 kfree(iscsit_global); 766 return -ENOMEM; 767 } 768 769 static void __exit iscsi_target_cleanup_module(void) 770 { 771 iscsit_release_discovery_tpg(); 772 iscsit_unregister_transport(&iscsi_target_transport); 773 kmem_cache_destroy(lio_qr_cache); 774 kmem_cache_destroy(lio_dr_cache); 775 kmem_cache_destroy(lio_ooo_cache); 776 kmem_cache_destroy(lio_r2t_cache); 777 778 /* 779 * Shutdown discovery sessions and disable discovery TPG 780 */ 781 if (iscsit_global->discovery_tpg) 782 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1); 783 784 target_unregister_template(&iscsi_ops); 785 786 vfree(iscsit_global->ts_bitmap); 787 kfree(iscsit_global); 788 } 789 790 int iscsit_add_reject( 791 struct iscsi_conn *conn, 792 u8 reason, 793 unsigned char *buf) 794 { 795 struct iscsi_cmd *cmd; 796 797 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 798 if (!cmd) 799 return -1; 800 801 cmd->iscsi_opcode = ISCSI_OP_REJECT; 802 cmd->reject_reason = reason; 803 804 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL); 805 if (!cmd->buf_ptr) { 806 pr_err("Unable to allocate memory for cmd->buf_ptr\n"); 807 iscsit_free_cmd(cmd, false); 808 return -1; 809 } 810 811 spin_lock_bh(&conn->cmd_lock); 812 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 813 spin_unlock_bh(&conn->cmd_lock); 814 815 cmd->i_state = ISTATE_SEND_REJECT; 816 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 817 818 return -1; 819 } 820 EXPORT_SYMBOL(iscsit_add_reject); 821 822 static int iscsit_add_reject_from_cmd( 823 struct iscsi_cmd *cmd, 824 u8 reason, 825 bool add_to_conn, 826 unsigned char *buf) 827 { 828 struct iscsi_conn *conn; 829 const bool do_put = cmd->se_cmd.se_tfo != NULL; 830 831 if (!cmd->conn) { 832 pr_err("cmd->conn is NULL for ITT: 0x%08x\n", 833 cmd->init_task_tag); 834 return -1; 835 } 836 conn = cmd->conn; 837 838 cmd->iscsi_opcode = ISCSI_OP_REJECT; 839 cmd->reject_reason = reason; 840 841 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL); 842 if (!cmd->buf_ptr) { 843 pr_err("Unable to allocate memory for cmd->buf_ptr\n"); 844 iscsit_free_cmd(cmd, false); 845 return -1; 846 } 847 848 if (add_to_conn) { 849 spin_lock_bh(&conn->cmd_lock); 850 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 851 spin_unlock_bh(&conn->cmd_lock); 852 } 853 854 cmd->i_state = ISTATE_SEND_REJECT; 855 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 856 /* 857 * Perform the kref_put now if se_cmd has already been setup by 858 * scsit_setup_scsi_cmd() 859 */ 860 if (do_put) { 861 pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n"); 862 target_put_sess_cmd(&cmd->se_cmd); 863 } 864 return -1; 865 } 866 867 static int iscsit_add_reject_cmd(struct iscsi_cmd *cmd, u8 reason, 868 unsigned char *buf) 869 { 870 return iscsit_add_reject_from_cmd(cmd, reason, true, buf); 871 } 872 873 int iscsit_reject_cmd(struct iscsi_cmd *cmd, u8 reason, unsigned char *buf) 874 { 875 return iscsit_add_reject_from_cmd(cmd, reason, false, buf); 876 } 877 EXPORT_SYMBOL(iscsit_reject_cmd); 878 879 /* 880 * Map some portion of the allocated scatterlist to an iovec, suitable for 881 * kernel sockets to copy data in/out. 882 */ 883 static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec, 884 u32 data_offset, u32 data_length) 885 { 886 u32 i = 0, orig_data_length = data_length; 887 struct scatterlist *sg; 888 unsigned int page_off; 889 890 /* 891 * We know each entry in t_data_sg contains a page. 892 */ 893 u32 ent = data_offset / PAGE_SIZE; 894 895 if (!data_length) 896 return 0; 897 898 if (ent >= cmd->se_cmd.t_data_nents) { 899 pr_err("Initial page entry out-of-bounds\n"); 900 goto overflow; 901 } 902 903 sg = &cmd->se_cmd.t_data_sg[ent]; 904 page_off = (data_offset % PAGE_SIZE); 905 906 cmd->first_data_sg = sg; 907 cmd->first_data_sg_off = page_off; 908 909 while (data_length) { 910 u32 cur_len; 911 912 if (WARN_ON_ONCE(!sg || i >= nvec)) 913 goto overflow; 914 915 cur_len = min_t(u32, data_length, sg->length - page_off); 916 917 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off; 918 iov[i].iov_len = cur_len; 919 920 data_length -= cur_len; 921 page_off = 0; 922 sg = sg_next(sg); 923 i++; 924 } 925 926 cmd->kmapped_nents = i; 927 928 return i; 929 930 overflow: 931 pr_err("offset %d + length %d overflow; %d/%d; sg-list:\n", 932 data_offset, orig_data_length, i, nvec); 933 for_each_sg(cmd->se_cmd.t_data_sg, sg, 934 cmd->se_cmd.t_data_nents, i) { 935 pr_err("[%d] off %d len %d\n", 936 i, sg->offset, sg->length); 937 } 938 return -1; 939 } 940 941 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd) 942 { 943 u32 i; 944 struct scatterlist *sg; 945 946 sg = cmd->first_data_sg; 947 948 for (i = 0; i < cmd->kmapped_nents; i++) 949 kunmap(sg_page(&sg[i])); 950 } 951 952 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn) 953 { 954 LIST_HEAD(ack_list); 955 struct iscsi_cmd *cmd, *cmd_p; 956 957 conn->exp_statsn = exp_statsn; 958 959 if (conn->sess->sess_ops->RDMAExtensions) 960 return; 961 962 spin_lock_bh(&conn->cmd_lock); 963 list_for_each_entry_safe(cmd, cmd_p, &conn->conn_cmd_list, i_conn_node) { 964 spin_lock(&cmd->istate_lock); 965 if ((cmd->i_state == ISTATE_SENT_STATUS) && 966 iscsi_sna_lt(cmd->stat_sn, exp_statsn)) { 967 cmd->i_state = ISTATE_REMOVE; 968 spin_unlock(&cmd->istate_lock); 969 list_move_tail(&cmd->i_conn_node, &ack_list); 970 continue; 971 } 972 spin_unlock(&cmd->istate_lock); 973 } 974 spin_unlock_bh(&conn->cmd_lock); 975 976 list_for_each_entry_safe(cmd, cmd_p, &ack_list, i_conn_node) { 977 list_del_init(&cmd->i_conn_node); 978 iscsit_free_cmd(cmd, false); 979 } 980 } 981 982 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd) 983 { 984 u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE)); 985 986 iov_count += ISCSI_IOV_DATA_BUFFER; 987 cmd->iov_data = kcalloc(iov_count, sizeof(*cmd->iov_data), GFP_KERNEL); 988 if (!cmd->iov_data) 989 return -ENOMEM; 990 991 cmd->orig_iov_data_count = iov_count; 992 return 0; 993 } 994 995 int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 996 unsigned char *buf) 997 { 998 int data_direction, payload_length; 999 struct iscsi_scsi_req *hdr; 1000 int iscsi_task_attr; 1001 int sam_task_attr; 1002 1003 atomic_long_inc(&conn->sess->cmd_pdus); 1004 1005 hdr = (struct iscsi_scsi_req *) buf; 1006 payload_length = ntoh24(hdr->dlength); 1007 1008 /* FIXME; Add checks for AdditionalHeaderSegment */ 1009 1010 if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) && 1011 !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) { 1012 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL" 1013 " not set. Bad iSCSI Initiator.\n"); 1014 return iscsit_add_reject_cmd(cmd, 1015 ISCSI_REASON_BOOKMARK_INVALID, buf); 1016 } 1017 1018 if (((hdr->flags & ISCSI_FLAG_CMD_READ) || 1019 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) { 1020 /* 1021 * From RFC-3720 Section 10.3.1: 1022 * 1023 * "Either or both of R and W MAY be 1 when either the 1024 * Expected Data Transfer Length and/or Bidirectional Read 1025 * Expected Data Transfer Length are 0" 1026 * 1027 * For this case, go ahead and clear the unnecssary bits 1028 * to avoid any confusion with ->data_direction. 1029 */ 1030 hdr->flags &= ~ISCSI_FLAG_CMD_READ; 1031 hdr->flags &= ~ISCSI_FLAG_CMD_WRITE; 1032 1033 pr_warn("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE" 1034 " set when Expected Data Transfer Length is 0 for" 1035 " CDB: 0x%02x, Fixing up flags\n", hdr->cdb[0]); 1036 } 1037 1038 if (!(hdr->flags & ISCSI_FLAG_CMD_READ) && 1039 !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) { 1040 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE" 1041 " MUST be set if Expected Data Transfer Length is not 0." 1042 " Bad iSCSI Initiator\n"); 1043 return iscsit_add_reject_cmd(cmd, 1044 ISCSI_REASON_BOOKMARK_INVALID, buf); 1045 } 1046 1047 if ((hdr->flags & ISCSI_FLAG_CMD_READ) && 1048 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) { 1049 pr_err("Bidirectional operations not supported!\n"); 1050 return iscsit_add_reject_cmd(cmd, 1051 ISCSI_REASON_BOOKMARK_INVALID, buf); 1052 } 1053 1054 if (hdr->opcode & ISCSI_OP_IMMEDIATE) { 1055 pr_err("Illegally set Immediate Bit in iSCSI Initiator" 1056 " Scsi Command PDU.\n"); 1057 return iscsit_add_reject_cmd(cmd, 1058 ISCSI_REASON_BOOKMARK_INVALID, buf); 1059 } 1060 1061 if (payload_length && !conn->sess->sess_ops->ImmediateData) { 1062 pr_err("ImmediateData=No but DataSegmentLength=%u," 1063 " protocol error.\n", payload_length); 1064 return iscsit_add_reject_cmd(cmd, 1065 ISCSI_REASON_PROTOCOL_ERROR, buf); 1066 } 1067 1068 if ((be32_to_cpu(hdr->data_length) == payload_length) && 1069 (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) { 1070 pr_err("Expected Data Transfer Length and Length of" 1071 " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL" 1072 " bit is not set protocol error\n"); 1073 return iscsit_add_reject_cmd(cmd, 1074 ISCSI_REASON_PROTOCOL_ERROR, buf); 1075 } 1076 1077 if (payload_length > be32_to_cpu(hdr->data_length)) { 1078 pr_err("DataSegmentLength: %u is greater than" 1079 " EDTL: %u, protocol error.\n", payload_length, 1080 hdr->data_length); 1081 return iscsit_add_reject_cmd(cmd, 1082 ISCSI_REASON_PROTOCOL_ERROR, buf); 1083 } 1084 1085 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 1086 pr_err("DataSegmentLength: %u is greater than" 1087 " MaxXmitDataSegmentLength: %u, protocol error.\n", 1088 payload_length, conn->conn_ops->MaxXmitDataSegmentLength); 1089 return iscsit_add_reject_cmd(cmd, 1090 ISCSI_REASON_PROTOCOL_ERROR, buf); 1091 } 1092 1093 if (payload_length > conn->sess->sess_ops->FirstBurstLength) { 1094 pr_err("DataSegmentLength: %u is greater than" 1095 " FirstBurstLength: %u, protocol error.\n", 1096 payload_length, conn->sess->sess_ops->FirstBurstLength); 1097 return iscsit_add_reject_cmd(cmd, 1098 ISCSI_REASON_BOOKMARK_INVALID, buf); 1099 } 1100 1101 data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE : 1102 (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE : 1103 DMA_NONE; 1104 1105 cmd->data_direction = data_direction; 1106 iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK; 1107 /* 1108 * Figure out the SAM Task Attribute for the incoming SCSI CDB 1109 */ 1110 if ((iscsi_task_attr == ISCSI_ATTR_UNTAGGED) || 1111 (iscsi_task_attr == ISCSI_ATTR_SIMPLE)) 1112 sam_task_attr = TCM_SIMPLE_TAG; 1113 else if (iscsi_task_attr == ISCSI_ATTR_ORDERED) 1114 sam_task_attr = TCM_ORDERED_TAG; 1115 else if (iscsi_task_attr == ISCSI_ATTR_HEAD_OF_QUEUE) 1116 sam_task_attr = TCM_HEAD_TAG; 1117 else if (iscsi_task_attr == ISCSI_ATTR_ACA) 1118 sam_task_attr = TCM_ACA_TAG; 1119 else { 1120 pr_debug("Unknown iSCSI Task Attribute: 0x%02x, using" 1121 " TCM_SIMPLE_TAG\n", iscsi_task_attr); 1122 sam_task_attr = TCM_SIMPLE_TAG; 1123 } 1124 1125 cmd->iscsi_opcode = ISCSI_OP_SCSI_CMD; 1126 cmd->i_state = ISTATE_NEW_CMD; 1127 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 1128 cmd->immediate_data = (payload_length) ? 1 : 0; 1129 cmd->unsolicited_data = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) && 1130 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0); 1131 if (cmd->unsolicited_data) 1132 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA; 1133 1134 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 1135 if (hdr->flags & ISCSI_FLAG_CMD_READ) 1136 cmd->targ_xfer_tag = session_get_next_ttt(conn->sess); 1137 else 1138 cmd->targ_xfer_tag = 0xFFFFFFFF; 1139 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 1140 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 1141 cmd->first_burst_len = payload_length; 1142 1143 if (!conn->sess->sess_ops->RDMAExtensions && 1144 cmd->data_direction == DMA_FROM_DEVICE) { 1145 struct iscsi_datain_req *dr; 1146 1147 dr = iscsit_allocate_datain_req(); 1148 if (!dr) 1149 return iscsit_add_reject_cmd(cmd, 1150 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1151 1152 iscsit_attach_datain_req(cmd, dr); 1153 } 1154 1155 /* 1156 * Initialize struct se_cmd descriptor from target_core_mod infrastructure 1157 */ 1158 transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops, 1159 conn->sess->se_sess, be32_to_cpu(hdr->data_length), 1160 cmd->data_direction, sam_task_attr, 1161 cmd->sense_buffer + 2); 1162 1163 pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x," 1164 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt, 1165 hdr->cmdsn, be32_to_cpu(hdr->data_length), payload_length, 1166 conn->cid); 1167 1168 target_get_sess_cmd(&cmd->se_cmd, true); 1169 1170 cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd, 1171 scsilun_to_int(&hdr->lun)); 1172 if (cmd->sense_reason) 1173 goto attach_cmd; 1174 1175 /* only used for printks or comparing with ->ref_task_tag */ 1176 cmd->se_cmd.tag = (__force u32)cmd->init_task_tag; 1177 cmd->sense_reason = target_setup_cmd_from_cdb(&cmd->se_cmd, hdr->cdb); 1178 if (cmd->sense_reason) { 1179 if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) { 1180 return iscsit_add_reject_cmd(cmd, 1181 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1182 } 1183 1184 goto attach_cmd; 1185 } 1186 1187 if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) { 1188 return iscsit_add_reject_cmd(cmd, 1189 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1190 } 1191 1192 attach_cmd: 1193 spin_lock_bh(&conn->cmd_lock); 1194 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 1195 spin_unlock_bh(&conn->cmd_lock); 1196 /* 1197 * Check if we need to delay processing because of ALUA 1198 * Active/NonOptimized primary access state.. 1199 */ 1200 core_alua_check_nonop_delay(&cmd->se_cmd); 1201 1202 return 0; 1203 } 1204 EXPORT_SYMBOL(iscsit_setup_scsi_cmd); 1205 1206 void iscsit_set_unsolicited_dataout(struct iscsi_cmd *cmd) 1207 { 1208 iscsit_set_dataout_sequence_values(cmd); 1209 1210 spin_lock_bh(&cmd->dataout_timeout_lock); 1211 iscsit_start_dataout_timer(cmd, cmd->conn); 1212 spin_unlock_bh(&cmd->dataout_timeout_lock); 1213 } 1214 EXPORT_SYMBOL(iscsit_set_unsolicited_dataout); 1215 1216 int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1217 struct iscsi_scsi_req *hdr) 1218 { 1219 int cmdsn_ret = 0; 1220 /* 1221 * Check the CmdSN against ExpCmdSN/MaxCmdSN here if 1222 * the Immediate Bit is not set, and no Immediate 1223 * Data is attached. 1224 * 1225 * A PDU/CmdSN carrying Immediate Data can only 1226 * be processed after the DataCRC has passed. 1227 * If the DataCRC fails, the CmdSN MUST NOT 1228 * be acknowledged. (See below) 1229 */ 1230 if (!cmd->immediate_data) { 1231 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 1232 (unsigned char *)hdr, hdr->cmdsn); 1233 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1234 return -1; 1235 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 1236 target_put_sess_cmd(&cmd->se_cmd); 1237 return 0; 1238 } 1239 } 1240 1241 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 1242 1243 /* 1244 * If no Immediate Data is attached, it's OK to return now. 1245 */ 1246 if (!cmd->immediate_data) { 1247 if (!cmd->sense_reason && cmd->unsolicited_data) 1248 iscsit_set_unsolicited_dataout(cmd); 1249 if (!cmd->sense_reason) 1250 return 0; 1251 1252 target_put_sess_cmd(&cmd->se_cmd); 1253 return 0; 1254 } 1255 1256 /* 1257 * Early CHECK_CONDITIONs with ImmediateData never make it to command 1258 * execution. These exceptions are processed in CmdSN order using 1259 * iscsit_check_received_cmdsn() in iscsit_get_immediate_data() below. 1260 */ 1261 if (cmd->sense_reason) 1262 return 1; 1263 /* 1264 * Call directly into transport_generic_new_cmd() to perform 1265 * the backend memory allocation. 1266 */ 1267 cmd->sense_reason = transport_generic_new_cmd(&cmd->se_cmd); 1268 if (cmd->sense_reason) 1269 return 1; 1270 1271 return 0; 1272 } 1273 EXPORT_SYMBOL(iscsit_process_scsi_cmd); 1274 1275 static int 1276 iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 1277 bool dump_payload) 1278 { 1279 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 1280 int rc; 1281 1282 /* 1283 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. 1284 */ 1285 if (dump_payload) { 1286 u32 length = min(cmd->se_cmd.data_length - cmd->write_data_done, 1287 cmd->first_burst_len); 1288 1289 pr_debug("Dumping min(%d - %d, %d) = %d bytes of immediate data\n", 1290 cmd->se_cmd.data_length, cmd->write_data_done, 1291 cmd->first_burst_len, length); 1292 rc = iscsit_dump_data_payload(cmd->conn, length, 1); 1293 pr_debug("Finished dumping immediate data\n"); 1294 if (rc < 0) 1295 immed_ret = IMMEDIATE_DATA_CANNOT_RECOVER; 1296 } else { 1297 immed_ret = iscsit_handle_immediate_data(cmd, hdr, 1298 cmd->first_burst_len); 1299 } 1300 1301 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 1302 /* 1303 * A PDU/CmdSN carrying Immediate Data passed 1304 * DataCRC, check against ExpCmdSN/MaxCmdSN if 1305 * Immediate Bit is not set. 1306 */ 1307 cmdsn_ret = iscsit_sequence_cmd(cmd->conn, cmd, 1308 (unsigned char *)hdr, hdr->cmdsn); 1309 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1310 return -1; 1311 1312 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 1313 target_put_sess_cmd(&cmd->se_cmd); 1314 1315 return 0; 1316 } else if (cmd->unsolicited_data) 1317 iscsit_set_unsolicited_dataout(cmd); 1318 1319 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 1320 /* 1321 * Immediate Data failed DataCRC and ERL>=1, 1322 * silently drop this PDU and let the initiator 1323 * plug the CmdSN gap. 1324 * 1325 * FIXME: Send Unsolicited NOPIN with reserved 1326 * TTT here to help the initiator figure out 1327 * the missing CmdSN, although they should be 1328 * intelligent enough to determine the missing 1329 * CmdSN and issue a retry to plug the sequence. 1330 */ 1331 cmd->i_state = ISTATE_REMOVE; 1332 iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, cmd->i_state); 1333 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 1334 return -1; 1335 1336 return 0; 1337 } 1338 1339 static int 1340 iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1341 unsigned char *buf) 1342 { 1343 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1344 int rc, immed_data; 1345 bool dump_payload = false; 1346 1347 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1348 if (rc < 0) 1349 return 0; 1350 /* 1351 * Allocation iovecs needed for struct socket operations for 1352 * traditional iSCSI block I/O. 1353 */ 1354 if (iscsit_allocate_iovecs(cmd) < 0) { 1355 return iscsit_reject_cmd(cmd, 1356 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1357 } 1358 immed_data = cmd->immediate_data; 1359 1360 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1361 if (rc < 0) 1362 return rc; 1363 else if (rc > 0) 1364 dump_payload = true; 1365 1366 if (!immed_data) 1367 return 0; 1368 1369 return iscsit_get_immediate_data(cmd, hdr, dump_payload); 1370 } 1371 1372 static u32 iscsit_do_crypto_hash_sg( 1373 struct ahash_request *hash, 1374 struct iscsi_cmd *cmd, 1375 u32 data_offset, 1376 u32 data_length, 1377 u32 padding, 1378 u8 *pad_bytes) 1379 { 1380 u32 data_crc; 1381 struct scatterlist *sg; 1382 unsigned int page_off; 1383 1384 crypto_ahash_init(hash); 1385 1386 sg = cmd->first_data_sg; 1387 page_off = cmd->first_data_sg_off; 1388 1389 while (data_length) { 1390 u32 cur_len = min_t(u32, data_length, (sg->length - page_off)); 1391 1392 ahash_request_set_crypt(hash, sg, NULL, cur_len); 1393 crypto_ahash_update(hash); 1394 1395 data_length -= cur_len; 1396 page_off = 0; 1397 /* iscsit_map_iovec has already checked for invalid sg pointers */ 1398 sg = sg_next(sg); 1399 } 1400 1401 if (padding) { 1402 struct scatterlist pad_sg; 1403 1404 sg_init_one(&pad_sg, pad_bytes, padding); 1405 ahash_request_set_crypt(hash, &pad_sg, (u8 *)&data_crc, 1406 padding); 1407 crypto_ahash_finup(hash); 1408 } else { 1409 ahash_request_set_crypt(hash, NULL, (u8 *)&data_crc, 0); 1410 crypto_ahash_final(hash); 1411 } 1412 1413 return data_crc; 1414 } 1415 1416 static void iscsit_do_crypto_hash_buf(struct ahash_request *hash, 1417 const void *buf, u32 payload_length, u32 padding, 1418 const void *pad_bytes, void *data_crc) 1419 { 1420 struct scatterlist sg[2]; 1421 1422 sg_init_table(sg, ARRAY_SIZE(sg)); 1423 sg_set_buf(sg, buf, payload_length); 1424 if (padding) 1425 sg_set_buf(sg + 1, pad_bytes, padding); 1426 1427 ahash_request_set_crypt(hash, sg, data_crc, payload_length + padding); 1428 1429 crypto_ahash_digest(hash); 1430 } 1431 1432 int 1433 __iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf, 1434 struct iscsi_cmd *cmd, u32 payload_length, 1435 bool *success) 1436 { 1437 struct iscsi_data *hdr = buf; 1438 struct se_cmd *se_cmd; 1439 int rc; 1440 1441 /* iSCSI write */ 1442 atomic_long_add(payload_length, &conn->sess->rx_data_octets); 1443 1444 pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x," 1445 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", 1446 hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset), 1447 payload_length, conn->cid); 1448 1449 if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) { 1450 pr_err("Command ITT: 0x%08x received DataOUT after" 1451 " last DataOUT received, dumping payload\n", 1452 cmd->init_task_tag); 1453 return iscsit_dump_data_payload(conn, payload_length, 1); 1454 } 1455 1456 if (cmd->data_direction != DMA_TO_DEVICE) { 1457 pr_err("Command ITT: 0x%08x received DataOUT for a" 1458 " NON-WRITE command.\n", cmd->init_task_tag); 1459 return iscsit_dump_data_payload(conn, payload_length, 1); 1460 } 1461 se_cmd = &cmd->se_cmd; 1462 iscsit_mod_dataout_timer(cmd); 1463 1464 if ((be32_to_cpu(hdr->offset) + payload_length) > cmd->se_cmd.data_length) { 1465 pr_err("DataOut Offset: %u, Length %u greater than iSCSI Command EDTL %u, protocol error.\n", 1466 be32_to_cpu(hdr->offset), payload_length, 1467 cmd->se_cmd.data_length); 1468 return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID, buf); 1469 } 1470 1471 if (cmd->unsolicited_data) { 1472 int dump_unsolicited_data = 0; 1473 1474 if (conn->sess->sess_ops->InitialR2T) { 1475 pr_err("Received unexpected unsolicited data" 1476 " while InitialR2T=Yes, protocol error.\n"); 1477 transport_send_check_condition_and_sense(&cmd->se_cmd, 1478 TCM_UNEXPECTED_UNSOLICITED_DATA, 0); 1479 return -1; 1480 } 1481 /* 1482 * Special case for dealing with Unsolicited DataOUT 1483 * and Unsupported SAM WRITE Opcodes and SE resource allocation 1484 * failures; 1485 */ 1486 1487 /* Something's amiss if we're not in WRITE_PENDING state... */ 1488 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING); 1489 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE)) 1490 dump_unsolicited_data = 1; 1491 1492 if (dump_unsolicited_data) { 1493 /* 1494 * Check if a delayed TASK_ABORTED status needs to 1495 * be sent now if the ISCSI_FLAG_CMD_FINAL has been 1496 * received with the unsolicited data out. 1497 */ 1498 if (hdr->flags & ISCSI_FLAG_CMD_FINAL) 1499 iscsit_stop_dataout_timer(cmd); 1500 1501 return iscsit_dump_data_payload(conn, payload_length, 1); 1502 } 1503 } else { 1504 /* 1505 * For the normal solicited data path: 1506 * 1507 * Check for a delayed TASK_ABORTED status and dump any 1508 * incoming data out payload if one exists. Also, when the 1509 * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current 1510 * data out sequence, we decrement outstanding_r2ts. Once 1511 * outstanding_r2ts reaches zero, go ahead and send the delayed 1512 * TASK_ABORTED status. 1513 */ 1514 if (se_cmd->transport_state & CMD_T_ABORTED) { 1515 if (hdr->flags & ISCSI_FLAG_CMD_FINAL && 1516 --cmd->outstanding_r2ts < 1) 1517 iscsit_stop_dataout_timer(cmd); 1518 1519 return iscsit_dump_data_payload(conn, payload_length, 1); 1520 } 1521 } 1522 /* 1523 * Perform DataSN, DataSequenceInOrder, DataPDUInOrder, and 1524 * within-command recovery checks before receiving the payload. 1525 */ 1526 rc = iscsit_check_pre_dataout(cmd, buf); 1527 if (rc == DATAOUT_WITHIN_COMMAND_RECOVERY) 1528 return 0; 1529 else if (rc == DATAOUT_CANNOT_RECOVER) 1530 return -1; 1531 *success = true; 1532 return 0; 1533 } 1534 EXPORT_SYMBOL(__iscsit_check_dataout_hdr); 1535 1536 int 1537 iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf, 1538 struct iscsi_cmd **out_cmd) 1539 { 1540 struct iscsi_data *hdr = buf; 1541 struct iscsi_cmd *cmd; 1542 u32 payload_length = ntoh24(hdr->dlength); 1543 int rc; 1544 bool success = false; 1545 1546 if (!payload_length) { 1547 pr_warn_ratelimited("DataOUT payload is ZERO, ignoring.\n"); 1548 return 0; 1549 } 1550 1551 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 1552 pr_err_ratelimited("DataSegmentLength: %u is greater than" 1553 " MaxXmitDataSegmentLength: %u\n", payload_length, 1554 conn->conn_ops->MaxXmitDataSegmentLength); 1555 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, buf); 1556 } 1557 1558 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, payload_length); 1559 if (!cmd) 1560 return 0; 1561 1562 rc = __iscsit_check_dataout_hdr(conn, buf, cmd, payload_length, &success); 1563 1564 if (success) 1565 *out_cmd = cmd; 1566 1567 return rc; 1568 } 1569 EXPORT_SYMBOL(iscsit_check_dataout_hdr); 1570 1571 static int 1572 iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1573 struct iscsi_data *hdr) 1574 { 1575 struct kvec *iov; 1576 u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0; 1577 u32 payload_length; 1578 int iov_ret, data_crc_failed = 0; 1579 1580 payload_length = min_t(u32, cmd->se_cmd.data_length, 1581 ntoh24(hdr->dlength)); 1582 rx_size += payload_length; 1583 iov = &cmd->iov_data[0]; 1584 1585 iov_ret = iscsit_map_iovec(cmd, iov, cmd->orig_iov_data_count - 2, 1586 be32_to_cpu(hdr->offset), payload_length); 1587 if (iov_ret < 0) 1588 return -1; 1589 1590 iov_count += iov_ret; 1591 1592 padding = ((-payload_length) & 3); 1593 if (padding != 0) { 1594 iov[iov_count].iov_base = cmd->pad_bytes; 1595 iov[iov_count++].iov_len = padding; 1596 rx_size += padding; 1597 pr_debug("Receiving %u padding bytes.\n", padding); 1598 } 1599 1600 if (conn->conn_ops->DataDigest) { 1601 iov[iov_count].iov_base = &checksum; 1602 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 1603 rx_size += ISCSI_CRC_LEN; 1604 } 1605 1606 WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count); 1607 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); 1608 1609 iscsit_unmap_iovec(cmd); 1610 1611 if (rx_got != rx_size) 1612 return -1; 1613 1614 if (conn->conn_ops->DataDigest) { 1615 u32 data_crc; 1616 1617 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd, 1618 be32_to_cpu(hdr->offset), 1619 payload_length, padding, 1620 cmd->pad_bytes); 1621 1622 if (checksum != data_crc) { 1623 pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 1624 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x" 1625 " does not match computed 0x%08x\n", 1626 hdr->itt, hdr->offset, payload_length, 1627 hdr->datasn, checksum, data_crc); 1628 data_crc_failed = 1; 1629 } else { 1630 pr_debug("Got CRC32C DataDigest 0x%08x for" 1631 " %u bytes of Data Out\n", checksum, 1632 payload_length); 1633 } 1634 } 1635 1636 return data_crc_failed; 1637 } 1638 1639 int 1640 iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr, 1641 bool data_crc_failed) 1642 { 1643 struct iscsi_conn *conn = cmd->conn; 1644 int rc, ooo_cmdsn; 1645 /* 1646 * Increment post receive data and CRC values or perform 1647 * within-command recovery. 1648 */ 1649 rc = iscsit_check_post_dataout(cmd, (unsigned char *)hdr, data_crc_failed); 1650 if ((rc == DATAOUT_NORMAL) || (rc == DATAOUT_WITHIN_COMMAND_RECOVERY)) 1651 return 0; 1652 else if (rc == DATAOUT_SEND_R2T) { 1653 iscsit_set_dataout_sequence_values(cmd); 1654 conn->conn_transport->iscsit_get_dataout(conn, cmd, false); 1655 } else if (rc == DATAOUT_SEND_TO_TRANSPORT) { 1656 /* 1657 * Handle extra special case for out of order 1658 * Unsolicited Data Out. 1659 */ 1660 spin_lock_bh(&cmd->istate_lock); 1661 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN); 1662 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1663 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1664 spin_unlock_bh(&cmd->istate_lock); 1665 1666 iscsit_stop_dataout_timer(cmd); 1667 if (ooo_cmdsn) 1668 return 0; 1669 target_execute_cmd(&cmd->se_cmd); 1670 return 0; 1671 } else /* DATAOUT_CANNOT_RECOVER */ 1672 return -1; 1673 1674 return 0; 1675 } 1676 EXPORT_SYMBOL(iscsit_check_dataout_payload); 1677 1678 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) 1679 { 1680 struct iscsi_cmd *cmd = NULL; 1681 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1682 int rc; 1683 bool data_crc_failed = false; 1684 1685 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1686 if (rc < 0) 1687 return 0; 1688 else if (!cmd) 1689 return 0; 1690 1691 rc = iscsit_get_dataout(conn, cmd, hdr); 1692 if (rc < 0) 1693 return rc; 1694 else if (rc > 0) 1695 data_crc_failed = true; 1696 1697 return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed); 1698 } 1699 1700 int iscsit_setup_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1701 struct iscsi_nopout *hdr) 1702 { 1703 u32 payload_length = ntoh24(hdr->dlength); 1704 1705 if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL)) { 1706 pr_err("NopOUT Flag's, Left Most Bit not set, protocol error.\n"); 1707 if (!cmd) 1708 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1709 (unsigned char *)hdr); 1710 1711 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1712 (unsigned char *)hdr); 1713 } 1714 1715 if (hdr->itt == RESERVED_ITT && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1716 pr_err("NOPOUT ITT is reserved, but Immediate Bit is" 1717 " not set, protocol error.\n"); 1718 if (!cmd) 1719 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1720 (unsigned char *)hdr); 1721 1722 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1723 (unsigned char *)hdr); 1724 } 1725 1726 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 1727 pr_err("NOPOUT Ping Data DataSegmentLength: %u is" 1728 " greater than MaxXmitDataSegmentLength: %u, protocol" 1729 " error.\n", payload_length, 1730 conn->conn_ops->MaxXmitDataSegmentLength); 1731 if (!cmd) 1732 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1733 (unsigned char *)hdr); 1734 1735 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1736 (unsigned char *)hdr); 1737 } 1738 1739 pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%08x," 1740 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n", 1741 hdr->itt == RESERVED_ITT ? "Response" : "Request", 1742 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn, 1743 payload_length); 1744 /* 1745 * This is not a response to a Unsolicited NopIN, which means 1746 * it can either be a NOPOUT ping request (with a valid ITT), 1747 * or a NOPOUT not requesting a NOPIN (with a reserved ITT). 1748 * Either way, make sure we allocate an struct iscsi_cmd, as both 1749 * can contain ping data. 1750 */ 1751 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1752 cmd->iscsi_opcode = ISCSI_OP_NOOP_OUT; 1753 cmd->i_state = ISTATE_SEND_NOPIN; 1754 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1755 1 : 0); 1756 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 1757 cmd->targ_xfer_tag = 0xFFFFFFFF; 1758 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 1759 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 1760 cmd->data_direction = DMA_NONE; 1761 } 1762 1763 return 0; 1764 } 1765 EXPORT_SYMBOL(iscsit_setup_nop_out); 1766 1767 int iscsit_process_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1768 struct iscsi_nopout *hdr) 1769 { 1770 struct iscsi_cmd *cmd_p = NULL; 1771 int cmdsn_ret = 0; 1772 /* 1773 * Initiator is expecting a NopIN ping reply.. 1774 */ 1775 if (hdr->itt != RESERVED_ITT) { 1776 if (!cmd) 1777 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1778 (unsigned char *)hdr); 1779 1780 spin_lock_bh(&conn->cmd_lock); 1781 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 1782 spin_unlock_bh(&conn->cmd_lock); 1783 1784 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 1785 1786 if (hdr->opcode & ISCSI_OP_IMMEDIATE) { 1787 iscsit_add_cmd_to_response_queue(cmd, conn, 1788 cmd->i_state); 1789 return 0; 1790 } 1791 1792 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 1793 (unsigned char *)hdr, hdr->cmdsn); 1794 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) 1795 return 0; 1796 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1797 return -1; 1798 1799 return 0; 1800 } 1801 /* 1802 * This was a response to a unsolicited NOPIN ping. 1803 */ 1804 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 1805 cmd_p = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt)); 1806 if (!cmd_p) 1807 return -EINVAL; 1808 1809 iscsit_stop_nopin_response_timer(conn); 1810 1811 cmd_p->i_state = ISTATE_REMOVE; 1812 iscsit_add_cmd_to_immediate_queue(cmd_p, conn, cmd_p->i_state); 1813 1814 iscsit_start_nopin_timer(conn); 1815 return 0; 1816 } 1817 /* 1818 * Otherwise, initiator is not expecting a NOPIN is response. 1819 * Just ignore for now. 1820 */ 1821 1822 if (cmd) 1823 iscsit_free_cmd(cmd, false); 1824 1825 return 0; 1826 } 1827 EXPORT_SYMBOL(iscsit_process_nop_out); 1828 1829 static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1830 unsigned char *buf) 1831 { 1832 unsigned char *ping_data = NULL; 1833 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1834 struct kvec *iov = NULL; 1835 u32 payload_length = ntoh24(hdr->dlength); 1836 int ret; 1837 1838 ret = iscsit_setup_nop_out(conn, cmd, hdr); 1839 if (ret < 0) 1840 return 0; 1841 /* 1842 * Handle NOP-OUT payload for traditional iSCSI sockets 1843 */ 1844 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1845 u32 checksum, data_crc, padding = 0; 1846 int niov = 0, rx_got, rx_size = payload_length; 1847 1848 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1849 if (!ping_data) { 1850 ret = -1; 1851 goto out; 1852 } 1853 1854 iov = &cmd->iov_misc[0]; 1855 iov[niov].iov_base = ping_data; 1856 iov[niov++].iov_len = payload_length; 1857 1858 padding = ((-payload_length) & 3); 1859 if (padding != 0) { 1860 pr_debug("Receiving %u additional bytes" 1861 " for padding.\n", padding); 1862 iov[niov].iov_base = &cmd->pad_bytes; 1863 iov[niov++].iov_len = padding; 1864 rx_size += padding; 1865 } 1866 if (conn->conn_ops->DataDigest) { 1867 iov[niov].iov_base = &checksum; 1868 iov[niov++].iov_len = ISCSI_CRC_LEN; 1869 rx_size += ISCSI_CRC_LEN; 1870 } 1871 1872 WARN_ON_ONCE(niov > ARRAY_SIZE(cmd->iov_misc)); 1873 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size); 1874 if (rx_got != rx_size) { 1875 ret = -1; 1876 goto out; 1877 } 1878 1879 if (conn->conn_ops->DataDigest) { 1880 iscsit_do_crypto_hash_buf(conn->conn_rx_hash, ping_data, 1881 payload_length, padding, 1882 cmd->pad_bytes, &data_crc); 1883 1884 if (checksum != data_crc) { 1885 pr_err("Ping data CRC32C DataDigest" 1886 " 0x%08x does not match computed 0x%08x\n", 1887 checksum, data_crc); 1888 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1889 pr_err("Unable to recover from" 1890 " NOPOUT Ping DataCRC failure while in" 1891 " ERL=0.\n"); 1892 ret = -1; 1893 goto out; 1894 } else { 1895 /* 1896 * Silently drop this PDU and let the 1897 * initiator plug the CmdSN gap. 1898 */ 1899 pr_debug("Dropping NOPOUT" 1900 " Command CmdSN: 0x%08x due to" 1901 " DataCRC error.\n", hdr->cmdsn); 1902 ret = 0; 1903 goto out; 1904 } 1905 } else { 1906 pr_debug("Got CRC32C DataDigest" 1907 " 0x%08x for %u bytes of ping data.\n", 1908 checksum, payload_length); 1909 } 1910 } 1911 1912 ping_data[payload_length] = '\0'; 1913 /* 1914 * Attach ping data to struct iscsi_cmd->buf_ptr. 1915 */ 1916 cmd->buf_ptr = ping_data; 1917 cmd->buf_ptr_size = payload_length; 1918 1919 pr_debug("Got %u bytes of NOPOUT ping" 1920 " data.\n", payload_length); 1921 pr_debug("Ping Data: \"%s\"\n", ping_data); 1922 } 1923 1924 return iscsit_process_nop_out(conn, cmd, hdr); 1925 out: 1926 if (cmd) 1927 iscsit_free_cmd(cmd, false); 1928 1929 kfree(ping_data); 1930 return ret; 1931 } 1932 1933 static enum tcm_tmreq_table iscsit_convert_tmf(u8 iscsi_tmf) 1934 { 1935 switch (iscsi_tmf) { 1936 case ISCSI_TM_FUNC_ABORT_TASK: 1937 return TMR_ABORT_TASK; 1938 case ISCSI_TM_FUNC_ABORT_TASK_SET: 1939 return TMR_ABORT_TASK_SET; 1940 case ISCSI_TM_FUNC_CLEAR_ACA: 1941 return TMR_CLEAR_ACA; 1942 case ISCSI_TM_FUNC_CLEAR_TASK_SET: 1943 return TMR_CLEAR_TASK_SET; 1944 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET: 1945 return TMR_LUN_RESET; 1946 case ISCSI_TM_FUNC_TARGET_WARM_RESET: 1947 return TMR_TARGET_WARM_RESET; 1948 case ISCSI_TM_FUNC_TARGET_COLD_RESET: 1949 return TMR_TARGET_COLD_RESET; 1950 default: 1951 return TMR_UNKNOWN; 1952 } 1953 } 1954 1955 int 1956 iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1957 unsigned char *buf) 1958 { 1959 struct se_tmr_req *se_tmr; 1960 struct iscsi_tmr_req *tmr_req; 1961 struct iscsi_tm *hdr; 1962 int out_of_order_cmdsn = 0, ret; 1963 u8 function, tcm_function = TMR_UNKNOWN; 1964 1965 hdr = (struct iscsi_tm *) buf; 1966 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL; 1967 function = hdr->flags; 1968 1969 pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:" 1970 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:" 1971 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function, 1972 hdr->rtt, hdr->refcmdsn, conn->cid); 1973 1974 if ((function != ISCSI_TM_FUNC_ABORT_TASK) && 1975 ((function != ISCSI_TM_FUNC_TASK_REASSIGN) && 1976 hdr->rtt != RESERVED_ITT)) { 1977 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n"); 1978 hdr->rtt = RESERVED_ITT; 1979 } 1980 1981 if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) && 1982 !(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1983 pr_err("Task Management Request TASK_REASSIGN not" 1984 " issued as immediate command, bad iSCSI Initiator" 1985 "implementation\n"); 1986 return iscsit_add_reject_cmd(cmd, 1987 ISCSI_REASON_PROTOCOL_ERROR, buf); 1988 } 1989 if ((function != ISCSI_TM_FUNC_ABORT_TASK) && 1990 be32_to_cpu(hdr->refcmdsn) != ISCSI_RESERVED_TAG) 1991 hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG); 1992 1993 cmd->data_direction = DMA_NONE; 1994 cmd->tmr_req = kzalloc(sizeof(*cmd->tmr_req), GFP_KERNEL); 1995 if (!cmd->tmr_req) { 1996 return iscsit_add_reject_cmd(cmd, 1997 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1998 buf); 1999 } 2000 2001 transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops, 2002 conn->sess->se_sess, 0, DMA_NONE, 2003 TCM_SIMPLE_TAG, cmd->sense_buffer + 2); 2004 2005 target_get_sess_cmd(&cmd->se_cmd, true); 2006 2007 /* 2008 * TASK_REASSIGN for ERL=2 / connection stays inside of 2009 * LIO-Target $FABRIC_MOD 2010 */ 2011 if (function != ISCSI_TM_FUNC_TASK_REASSIGN) { 2012 tcm_function = iscsit_convert_tmf(function); 2013 if (tcm_function == TMR_UNKNOWN) { 2014 pr_err("Unknown iSCSI TMR Function:" 2015 " 0x%02x\n", function); 2016 return iscsit_add_reject_cmd(cmd, 2017 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 2018 } 2019 } 2020 ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req, tcm_function, 2021 GFP_KERNEL); 2022 if (ret < 0) 2023 return iscsit_add_reject_cmd(cmd, 2024 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 2025 2026 cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req; 2027 2028 cmd->iscsi_opcode = ISCSI_OP_SCSI_TMFUNC; 2029 cmd->i_state = ISTATE_SEND_TASKMGTRSP; 2030 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 2031 cmd->init_task_tag = hdr->itt; 2032 cmd->targ_xfer_tag = 0xFFFFFFFF; 2033 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 2034 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 2035 se_tmr = cmd->se_cmd.se_tmr_req; 2036 tmr_req = cmd->tmr_req; 2037 /* 2038 * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN 2039 */ 2040 if (function != ISCSI_TM_FUNC_TASK_REASSIGN) { 2041 ret = transport_lookup_tmr_lun(&cmd->se_cmd, 2042 scsilun_to_int(&hdr->lun)); 2043 if (ret < 0) { 2044 se_tmr->response = ISCSI_TMF_RSP_NO_LUN; 2045 goto attach; 2046 } 2047 } 2048 2049 switch (function) { 2050 case ISCSI_TM_FUNC_ABORT_TASK: 2051 se_tmr->response = iscsit_tmr_abort_task(cmd, buf); 2052 if (se_tmr->response) 2053 goto attach; 2054 break; 2055 case ISCSI_TM_FUNC_ABORT_TASK_SET: 2056 case ISCSI_TM_FUNC_CLEAR_ACA: 2057 case ISCSI_TM_FUNC_CLEAR_TASK_SET: 2058 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET: 2059 break; 2060 case ISCSI_TM_FUNC_TARGET_WARM_RESET: 2061 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) { 2062 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED; 2063 goto attach; 2064 } 2065 break; 2066 case ISCSI_TM_FUNC_TARGET_COLD_RESET: 2067 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) { 2068 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED; 2069 goto attach; 2070 } 2071 break; 2072 case ISCSI_TM_FUNC_TASK_REASSIGN: 2073 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf); 2074 /* 2075 * Perform sanity checks on the ExpDataSN only if the 2076 * TASK_REASSIGN was successful. 2077 */ 2078 if (se_tmr->response) 2079 break; 2080 2081 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0) 2082 return iscsit_add_reject_cmd(cmd, 2083 ISCSI_REASON_BOOKMARK_INVALID, buf); 2084 break; 2085 default: 2086 pr_err("Unknown TMR function: 0x%02x, protocol" 2087 " error.\n", function); 2088 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED; 2089 goto attach; 2090 } 2091 2092 if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) && 2093 (se_tmr->response == ISCSI_TMF_RSP_COMPLETE)) 2094 se_tmr->call_transport = 1; 2095 attach: 2096 spin_lock_bh(&conn->cmd_lock); 2097 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 2098 spin_unlock_bh(&conn->cmd_lock); 2099 2100 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 2101 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 2102 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP) { 2103 out_of_order_cmdsn = 1; 2104 } else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 2105 target_put_sess_cmd(&cmd->se_cmd); 2106 return 0; 2107 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) { 2108 return -1; 2109 } 2110 } 2111 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 2112 2113 if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE)) 2114 return 0; 2115 /* 2116 * Found the referenced task, send to transport for processing. 2117 */ 2118 if (se_tmr->call_transport) 2119 return transport_generic_handle_tmr(&cmd->se_cmd); 2120 2121 /* 2122 * Could not find the referenced LUN, task, or Task Management 2123 * command not authorized or supported. Change state and 2124 * let the tx_thread send the response. 2125 * 2126 * For connection recovery, this is also the default action for 2127 * TMR TASK_REASSIGN. 2128 */ 2129 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2130 target_put_sess_cmd(&cmd->se_cmd); 2131 return 0; 2132 } 2133 EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd); 2134 2135 /* #warning FIXME: Support Text Command parameters besides SendTargets */ 2136 int 2137 iscsit_setup_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2138 struct iscsi_text *hdr) 2139 { 2140 u32 payload_length = ntoh24(hdr->dlength); 2141 2142 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 2143 pr_err("Unable to accept text parameter length: %u" 2144 "greater than MaxXmitDataSegmentLength %u.\n", 2145 payload_length, conn->conn_ops->MaxXmitDataSegmentLength); 2146 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 2147 (unsigned char *)hdr); 2148 } 2149 2150 if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL) || 2151 (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)) { 2152 pr_err("Multi sequence text commands currently not supported\n"); 2153 return iscsit_reject_cmd(cmd, ISCSI_REASON_CMD_NOT_SUPPORTED, 2154 (unsigned char *)hdr); 2155 } 2156 2157 pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x," 2158 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn, 2159 hdr->exp_statsn, payload_length); 2160 2161 cmd->iscsi_opcode = ISCSI_OP_TEXT; 2162 cmd->i_state = ISTATE_SEND_TEXTRSP; 2163 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 2164 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 2165 cmd->targ_xfer_tag = 0xFFFFFFFF; 2166 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 2167 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 2168 cmd->data_direction = DMA_NONE; 2169 kfree(cmd->text_in_ptr); 2170 cmd->text_in_ptr = NULL; 2171 2172 return 0; 2173 } 2174 EXPORT_SYMBOL(iscsit_setup_text_cmd); 2175 2176 int 2177 iscsit_process_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2178 struct iscsi_text *hdr) 2179 { 2180 unsigned char *text_in = cmd->text_in_ptr, *text_ptr; 2181 int cmdsn_ret; 2182 2183 if (!text_in) { 2184 cmd->targ_xfer_tag = be32_to_cpu(hdr->ttt); 2185 if (cmd->targ_xfer_tag == 0xFFFFFFFF) { 2186 pr_err("Unable to locate text_in buffer for sendtargets" 2187 " discovery\n"); 2188 goto reject; 2189 } 2190 goto empty_sendtargets; 2191 } 2192 if (strncmp("SendTargets", text_in, 11) != 0) { 2193 pr_err("Received Text Data that is not" 2194 " SendTargets, cannot continue.\n"); 2195 goto reject; 2196 } 2197 text_ptr = strchr(text_in, '='); 2198 if (!text_ptr) { 2199 pr_err("No \"=\" separator found in Text Data," 2200 " cannot continue.\n"); 2201 goto reject; 2202 } 2203 if (!strncmp("=All", text_ptr, 4)) { 2204 cmd->cmd_flags |= ICF_SENDTARGETS_ALL; 2205 } else if (!strncmp("=iqn.", text_ptr, 5) || 2206 !strncmp("=eui.", text_ptr, 5)) { 2207 cmd->cmd_flags |= ICF_SENDTARGETS_SINGLE; 2208 } else { 2209 pr_err("Unable to locate valid SendTargets=%s value\n", text_ptr); 2210 goto reject; 2211 } 2212 2213 spin_lock_bh(&conn->cmd_lock); 2214 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 2215 spin_unlock_bh(&conn->cmd_lock); 2216 2217 empty_sendtargets: 2218 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 2219 2220 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 2221 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 2222 (unsigned char *)hdr, hdr->cmdsn); 2223 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 2224 return -1; 2225 2226 return 0; 2227 } 2228 2229 return iscsit_execute_cmd(cmd, 0); 2230 2231 reject: 2232 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 2233 (unsigned char *)hdr); 2234 } 2235 EXPORT_SYMBOL(iscsit_process_text_cmd); 2236 2237 static int 2238 iscsit_handle_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2239 unsigned char *buf) 2240 { 2241 struct iscsi_text *hdr = (struct iscsi_text *)buf; 2242 char *text_in = NULL; 2243 u32 payload_length = ntoh24(hdr->dlength); 2244 int rx_size, rc; 2245 2246 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 2247 if (rc < 0) 2248 return 0; 2249 2250 rx_size = payload_length; 2251 if (payload_length) { 2252 u32 checksum = 0, data_crc = 0; 2253 u32 padding = 0; 2254 int niov = 0, rx_got; 2255 struct kvec iov[2]; 2256 2257 rx_size = ALIGN(payload_length, 4); 2258 text_in = kzalloc(rx_size, GFP_KERNEL); 2259 if (!text_in) 2260 goto reject; 2261 2262 cmd->text_in_ptr = text_in; 2263 2264 memset(iov, 0, sizeof(iov)); 2265 iov[niov].iov_base = text_in; 2266 iov[niov++].iov_len = rx_size; 2267 2268 padding = rx_size - payload_length; 2269 if (padding) 2270 pr_debug("Receiving %u additional bytes" 2271 " for padding.\n", padding); 2272 if (conn->conn_ops->DataDigest) { 2273 iov[niov].iov_base = &checksum; 2274 iov[niov++].iov_len = ISCSI_CRC_LEN; 2275 rx_size += ISCSI_CRC_LEN; 2276 } 2277 2278 WARN_ON_ONCE(niov > ARRAY_SIZE(iov)); 2279 rx_got = rx_data(conn, &iov[0], niov, rx_size); 2280 if (rx_got != rx_size) 2281 goto reject; 2282 2283 if (conn->conn_ops->DataDigest) { 2284 iscsit_do_crypto_hash_buf(conn->conn_rx_hash, 2285 text_in, rx_size, 0, NULL, 2286 &data_crc); 2287 2288 if (checksum != data_crc) { 2289 pr_err("Text data CRC32C DataDigest" 2290 " 0x%08x does not match computed" 2291 " 0x%08x\n", checksum, data_crc); 2292 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2293 pr_err("Unable to recover from" 2294 " Text Data digest failure while in" 2295 " ERL=0.\n"); 2296 goto reject; 2297 } else { 2298 /* 2299 * Silently drop this PDU and let the 2300 * initiator plug the CmdSN gap. 2301 */ 2302 pr_debug("Dropping Text" 2303 " Command CmdSN: 0x%08x due to" 2304 " DataCRC error.\n", hdr->cmdsn); 2305 kfree(text_in); 2306 return 0; 2307 } 2308 } else { 2309 pr_debug("Got CRC32C DataDigest" 2310 " 0x%08x for %u bytes of text data.\n", 2311 checksum, payload_length); 2312 } 2313 } 2314 text_in[payload_length - 1] = '\0'; 2315 pr_debug("Successfully read %d bytes of text" 2316 " data.\n", payload_length); 2317 } 2318 2319 return iscsit_process_text_cmd(conn, cmd, hdr); 2320 2321 reject: 2322 kfree(cmd->text_in_ptr); 2323 cmd->text_in_ptr = NULL; 2324 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf); 2325 } 2326 2327 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2328 { 2329 struct iscsi_conn *conn_p; 2330 struct iscsi_session *sess = conn->sess; 2331 2332 pr_debug("Received logout request CLOSESESSION on CID: %hu" 2333 " for SID: %u.\n", conn->cid, conn->sess->sid); 2334 2335 atomic_set(&sess->session_logout, 1); 2336 atomic_set(&conn->conn_logout_remove, 1); 2337 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION; 2338 2339 iscsit_inc_conn_usage_count(conn); 2340 iscsit_inc_session_usage_count(sess); 2341 2342 spin_lock_bh(&sess->conn_lock); 2343 list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) { 2344 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN) 2345 continue; 2346 2347 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 2348 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT; 2349 } 2350 spin_unlock_bh(&sess->conn_lock); 2351 2352 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2353 2354 return 0; 2355 } 2356 2357 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2358 { 2359 struct iscsi_conn *l_conn; 2360 struct iscsi_session *sess = conn->sess; 2361 2362 pr_debug("Received logout request CLOSECONNECTION for CID:" 2363 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); 2364 2365 /* 2366 * A Logout Request with a CLOSECONNECTION reason code for a CID 2367 * can arrive on a connection with a differing CID. 2368 */ 2369 if (conn->cid == cmd->logout_cid) { 2370 spin_lock_bh(&conn->state_lock); 2371 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 2372 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT; 2373 2374 atomic_set(&conn->conn_logout_remove, 1); 2375 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION; 2376 iscsit_inc_conn_usage_count(conn); 2377 2378 spin_unlock_bh(&conn->state_lock); 2379 } else { 2380 /* 2381 * Handle all different cid CLOSECONNECTION requests in 2382 * iscsit_logout_post_handler_diffcid() as to give enough 2383 * time for any non immediate command's CmdSN to be 2384 * acknowledged on the connection in question. 2385 * 2386 * Here we simply make sure the CID is still around. 2387 */ 2388 l_conn = iscsit_get_conn_from_cid(sess, 2389 cmd->logout_cid); 2390 if (!l_conn) { 2391 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND; 2392 iscsit_add_cmd_to_response_queue(cmd, conn, 2393 cmd->i_state); 2394 return 0; 2395 } 2396 2397 iscsit_dec_conn_usage_count(l_conn); 2398 } 2399 2400 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2401 2402 return 0; 2403 } 2404 2405 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2406 { 2407 struct iscsi_session *sess = conn->sess; 2408 2409 pr_debug("Received explicit REMOVECONNFORRECOVERY logout for" 2410 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); 2411 2412 if (sess->sess_ops->ErrorRecoveryLevel != 2) { 2413 pr_err("Received Logout Request REMOVECONNFORRECOVERY" 2414 " while ERL!=2.\n"); 2415 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED; 2416 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2417 return 0; 2418 } 2419 2420 if (conn->cid == cmd->logout_cid) { 2421 pr_err("Received Logout Request REMOVECONNFORRECOVERY" 2422 " with CID: %hu on CID: %hu, implementation error.\n", 2423 cmd->logout_cid, conn->cid); 2424 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED; 2425 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2426 return 0; 2427 } 2428 2429 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2430 2431 return 0; 2432 } 2433 2434 int 2435 iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2436 unsigned char *buf) 2437 { 2438 int cmdsn_ret, logout_remove = 0; 2439 u8 reason_code = 0; 2440 struct iscsi_logout *hdr; 2441 struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn); 2442 2443 hdr = (struct iscsi_logout *) buf; 2444 reason_code = (hdr->flags & 0x7f); 2445 2446 if (tiqn) { 2447 spin_lock(&tiqn->logout_stats.lock); 2448 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) 2449 tiqn->logout_stats.normal_logouts++; 2450 else 2451 tiqn->logout_stats.abnormal_logouts++; 2452 spin_unlock(&tiqn->logout_stats.lock); 2453 } 2454 2455 pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x" 2456 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n", 2457 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code, 2458 hdr->cid, conn->cid); 2459 2460 if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) { 2461 pr_err("Received logout request on connection that" 2462 " is not in logged in state, ignoring request.\n"); 2463 iscsit_free_cmd(cmd, false); 2464 return 0; 2465 } 2466 2467 cmd->iscsi_opcode = ISCSI_OP_LOGOUT; 2468 cmd->i_state = ISTATE_SEND_LOGOUTRSP; 2469 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 2470 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 2471 cmd->targ_xfer_tag = 0xFFFFFFFF; 2472 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 2473 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 2474 cmd->logout_cid = be16_to_cpu(hdr->cid); 2475 cmd->logout_reason = reason_code; 2476 cmd->data_direction = DMA_NONE; 2477 2478 /* 2479 * We need to sleep in these cases (by returning 1) until the Logout 2480 * Response gets sent in the tx thread. 2481 */ 2482 if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) || 2483 ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) && 2484 be16_to_cpu(hdr->cid) == conn->cid)) 2485 logout_remove = 1; 2486 2487 spin_lock_bh(&conn->cmd_lock); 2488 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 2489 spin_unlock_bh(&conn->cmd_lock); 2490 2491 if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY) 2492 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 2493 2494 /* 2495 * Immediate commands are executed, well, immediately. 2496 * Non-Immediate Logout Commands are executed in CmdSN order. 2497 */ 2498 if (cmd->immediate_cmd) { 2499 int ret = iscsit_execute_cmd(cmd, 0); 2500 2501 if (ret < 0) 2502 return ret; 2503 } else { 2504 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 2505 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) 2506 logout_remove = 0; 2507 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 2508 return -1; 2509 } 2510 2511 return logout_remove; 2512 } 2513 EXPORT_SYMBOL(iscsit_handle_logout_cmd); 2514 2515 int iscsit_handle_snack( 2516 struct iscsi_conn *conn, 2517 unsigned char *buf) 2518 { 2519 struct iscsi_snack *hdr; 2520 2521 hdr = (struct iscsi_snack *) buf; 2522 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL; 2523 2524 pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:" 2525 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x," 2526 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags, 2527 hdr->begrun, hdr->runlength, conn->cid); 2528 2529 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2530 pr_err("Initiator sent SNACK request while in" 2531 " ErrorRecoveryLevel=0.\n"); 2532 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 2533 buf); 2534 } 2535 /* 2536 * SNACK_DATA and SNACK_R2T are both 0, so check which function to 2537 * call from inside iscsi_send_recovery_datain_or_r2t(). 2538 */ 2539 switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) { 2540 case 0: 2541 return iscsit_handle_recovery_datain_or_r2t(conn, buf, 2542 hdr->itt, 2543 be32_to_cpu(hdr->ttt), 2544 be32_to_cpu(hdr->begrun), 2545 be32_to_cpu(hdr->runlength)); 2546 case ISCSI_FLAG_SNACK_TYPE_STATUS: 2547 return iscsit_handle_status_snack(conn, hdr->itt, 2548 be32_to_cpu(hdr->ttt), 2549 be32_to_cpu(hdr->begrun), be32_to_cpu(hdr->runlength)); 2550 case ISCSI_FLAG_SNACK_TYPE_DATA_ACK: 2551 return iscsit_handle_data_ack(conn, be32_to_cpu(hdr->ttt), 2552 be32_to_cpu(hdr->begrun), 2553 be32_to_cpu(hdr->runlength)); 2554 case ISCSI_FLAG_SNACK_TYPE_RDATA: 2555 /* FIXME: Support R-Data SNACK */ 2556 pr_err("R-Data SNACK Not Supported.\n"); 2557 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 2558 buf); 2559 default: 2560 pr_err("Unknown SNACK type 0x%02x, protocol" 2561 " error.\n", hdr->flags & 0x0f); 2562 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 2563 buf); 2564 } 2565 2566 return 0; 2567 } 2568 EXPORT_SYMBOL(iscsit_handle_snack); 2569 2570 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn) 2571 { 2572 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || 2573 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { 2574 wait_for_completion_interruptible_timeout( 2575 &conn->rx_half_close_comp, 2576 ISCSI_RX_THREAD_TCP_TIMEOUT * HZ); 2577 } 2578 } 2579 2580 static int iscsit_handle_immediate_data( 2581 struct iscsi_cmd *cmd, 2582 struct iscsi_scsi_req *hdr, 2583 u32 length) 2584 { 2585 int iov_ret, rx_got = 0, rx_size = 0; 2586 u32 checksum, iov_count = 0, padding = 0; 2587 struct iscsi_conn *conn = cmd->conn; 2588 struct kvec *iov; 2589 void *overflow_buf = NULL; 2590 2591 BUG_ON(cmd->write_data_done > cmd->se_cmd.data_length); 2592 rx_size = min(cmd->se_cmd.data_length - cmd->write_data_done, length); 2593 iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, 2594 cmd->orig_iov_data_count - 2, 2595 cmd->write_data_done, rx_size); 2596 if (iov_ret < 0) 2597 return IMMEDIATE_DATA_CANNOT_RECOVER; 2598 2599 iov_count = iov_ret; 2600 iov = &cmd->iov_data[0]; 2601 if (rx_size < length) { 2602 /* 2603 * Special case: length of immediate data exceeds the data 2604 * buffer size derived from the CDB. 2605 */ 2606 overflow_buf = kmalloc(length - rx_size, GFP_KERNEL); 2607 if (!overflow_buf) { 2608 iscsit_unmap_iovec(cmd); 2609 return IMMEDIATE_DATA_CANNOT_RECOVER; 2610 } 2611 cmd->overflow_buf = overflow_buf; 2612 iov[iov_count].iov_base = overflow_buf; 2613 iov[iov_count].iov_len = length - rx_size; 2614 iov_count++; 2615 rx_size = length; 2616 } 2617 2618 padding = ((-length) & 3); 2619 if (padding != 0) { 2620 iov[iov_count].iov_base = cmd->pad_bytes; 2621 iov[iov_count++].iov_len = padding; 2622 rx_size += padding; 2623 } 2624 2625 if (conn->conn_ops->DataDigest) { 2626 iov[iov_count].iov_base = &checksum; 2627 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 2628 rx_size += ISCSI_CRC_LEN; 2629 } 2630 2631 WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count); 2632 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); 2633 2634 iscsit_unmap_iovec(cmd); 2635 2636 if (rx_got != rx_size) { 2637 iscsit_rx_thread_wait_for_tcp(conn); 2638 return IMMEDIATE_DATA_CANNOT_RECOVER; 2639 } 2640 2641 if (conn->conn_ops->DataDigest) { 2642 u32 data_crc; 2643 2644 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd, 2645 cmd->write_data_done, length, padding, 2646 cmd->pad_bytes); 2647 2648 if (checksum != data_crc) { 2649 pr_err("ImmediateData CRC32C DataDigest 0x%08x" 2650 " does not match computed 0x%08x\n", checksum, 2651 data_crc); 2652 2653 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2654 pr_err("Unable to recover from" 2655 " Immediate Data digest failure while" 2656 " in ERL=0.\n"); 2657 iscsit_reject_cmd(cmd, 2658 ISCSI_REASON_DATA_DIGEST_ERROR, 2659 (unsigned char *)hdr); 2660 return IMMEDIATE_DATA_CANNOT_RECOVER; 2661 } else { 2662 iscsit_reject_cmd(cmd, 2663 ISCSI_REASON_DATA_DIGEST_ERROR, 2664 (unsigned char *)hdr); 2665 return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 2666 } 2667 } else { 2668 pr_debug("Got CRC32C DataDigest 0x%08x for" 2669 " %u bytes of Immediate Data\n", checksum, 2670 length); 2671 } 2672 } 2673 2674 cmd->write_data_done += length; 2675 2676 if (cmd->write_data_done == cmd->se_cmd.data_length) { 2677 spin_lock_bh(&cmd->istate_lock); 2678 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 2679 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 2680 spin_unlock_bh(&cmd->istate_lock); 2681 } 2682 2683 return IMMEDIATE_DATA_NORMAL_OPERATION; 2684 } 2685 2686 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections 2687 with active network interface */ 2688 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn) 2689 { 2690 struct iscsi_cmd *cmd; 2691 struct iscsi_conn *conn_p; 2692 bool found = false; 2693 2694 lockdep_assert_held(&conn->sess->conn_lock); 2695 2696 /* 2697 * Only send a Asynchronous Message on connections whos network 2698 * interface is still functional. 2699 */ 2700 list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) { 2701 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) { 2702 iscsit_inc_conn_usage_count(conn_p); 2703 found = true; 2704 break; 2705 } 2706 } 2707 2708 if (!found) 2709 return; 2710 2711 cmd = iscsit_allocate_cmd(conn_p, TASK_RUNNING); 2712 if (!cmd) { 2713 iscsit_dec_conn_usage_count(conn_p); 2714 return; 2715 } 2716 2717 cmd->logout_cid = conn->cid; 2718 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT; 2719 cmd->i_state = ISTATE_SEND_ASYNCMSG; 2720 2721 spin_lock_bh(&conn_p->cmd_lock); 2722 list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list); 2723 spin_unlock_bh(&conn_p->cmd_lock); 2724 2725 iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state); 2726 iscsit_dec_conn_usage_count(conn_p); 2727 } 2728 2729 static int iscsit_send_conn_drop_async_message( 2730 struct iscsi_cmd *cmd, 2731 struct iscsi_conn *conn) 2732 { 2733 struct iscsi_async *hdr; 2734 2735 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT; 2736 2737 hdr = (struct iscsi_async *) cmd->pdu; 2738 hdr->opcode = ISCSI_OP_ASYNC_EVENT; 2739 hdr->flags = ISCSI_FLAG_CMD_FINAL; 2740 cmd->init_task_tag = RESERVED_ITT; 2741 cmd->targ_xfer_tag = 0xFFFFFFFF; 2742 put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]); 2743 cmd->stat_sn = conn->stat_sn++; 2744 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2745 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2746 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 2747 hdr->async_event = ISCSI_ASYNC_MSG_DROPPING_CONNECTION; 2748 hdr->param1 = cpu_to_be16(cmd->logout_cid); 2749 hdr->param2 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait); 2750 hdr->param3 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain); 2751 2752 pr_debug("Sending Connection Dropped Async Message StatSN:" 2753 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn, 2754 cmd->logout_cid, conn->cid); 2755 2756 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 2757 } 2758 2759 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn) 2760 { 2761 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || 2762 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { 2763 wait_for_completion_interruptible_timeout( 2764 &conn->tx_half_close_comp, 2765 ISCSI_TX_THREAD_TCP_TIMEOUT * HZ); 2766 } 2767 } 2768 2769 void 2770 iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2771 struct iscsi_datain *datain, struct iscsi_data_rsp *hdr, 2772 bool set_statsn) 2773 { 2774 hdr->opcode = ISCSI_OP_SCSI_DATA_IN; 2775 hdr->flags = datain->flags; 2776 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) { 2777 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { 2778 hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW; 2779 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 2780 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { 2781 hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW; 2782 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 2783 } 2784 } 2785 hton24(hdr->dlength, datain->length); 2786 if (hdr->flags & ISCSI_FLAG_DATA_ACK) 2787 int_to_scsilun(cmd->se_cmd.orig_fe_lun, 2788 (struct scsi_lun *)&hdr->lun); 2789 else 2790 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); 2791 2792 hdr->itt = cmd->init_task_tag; 2793 2794 if (hdr->flags & ISCSI_FLAG_DATA_ACK) 2795 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 2796 else 2797 hdr->ttt = cpu_to_be32(0xFFFFFFFF); 2798 if (set_statsn) 2799 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2800 else 2801 hdr->statsn = cpu_to_be32(0xFFFFFFFF); 2802 2803 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2804 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 2805 hdr->datasn = cpu_to_be32(datain->data_sn); 2806 hdr->offset = cpu_to_be32(datain->offset); 2807 2808 pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x," 2809 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", 2810 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn), 2811 ntohl(hdr->offset), datain->length, conn->cid); 2812 } 2813 EXPORT_SYMBOL(iscsit_build_datain_pdu); 2814 2815 static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2816 { 2817 struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0]; 2818 struct iscsi_datain datain; 2819 struct iscsi_datain_req *dr; 2820 int eodr = 0, ret; 2821 bool set_statsn = false; 2822 2823 memset(&datain, 0, sizeof(struct iscsi_datain)); 2824 dr = iscsit_get_datain_values(cmd, &datain); 2825 if (!dr) { 2826 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n", 2827 cmd->init_task_tag); 2828 return -1; 2829 } 2830 /* 2831 * Be paranoid and double check the logic for now. 2832 */ 2833 if ((datain.offset + datain.length) > cmd->se_cmd.data_length) { 2834 pr_err("Command ITT: 0x%08x, datain.offset: %u and" 2835 " datain.length: %u exceeds cmd->data_length: %u\n", 2836 cmd->init_task_tag, datain.offset, datain.length, 2837 cmd->se_cmd.data_length); 2838 return -1; 2839 } 2840 2841 atomic_long_add(datain.length, &conn->sess->tx_data_octets); 2842 /* 2843 * Special case for successfully execution w/ both DATAIN 2844 * and Sense Data. 2845 */ 2846 if ((datain.flags & ISCSI_FLAG_DATA_STATUS) && 2847 (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)) 2848 datain.flags &= ~ISCSI_FLAG_DATA_STATUS; 2849 else { 2850 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) || 2851 (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) { 2852 iscsit_increment_maxcmdsn(cmd, conn->sess); 2853 cmd->stat_sn = conn->stat_sn++; 2854 set_statsn = true; 2855 } else if (dr->dr_complete == 2856 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY) 2857 set_statsn = true; 2858 } 2859 2860 iscsit_build_datain_pdu(cmd, conn, &datain, hdr, set_statsn); 2861 2862 ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, dr, &datain, 0); 2863 if (ret < 0) 2864 return ret; 2865 2866 if (dr->dr_complete) { 2867 eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ? 2868 2 : 1; 2869 iscsit_free_datain_req(cmd, dr); 2870 } 2871 2872 return eodr; 2873 } 2874 2875 int 2876 iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2877 struct iscsi_logout_rsp *hdr) 2878 { 2879 struct iscsi_conn *logout_conn = NULL; 2880 struct iscsi_conn_recovery *cr = NULL; 2881 struct iscsi_session *sess = conn->sess; 2882 /* 2883 * The actual shutting down of Sessions and/or Connections 2884 * for CLOSESESSION and CLOSECONNECTION Logout Requests 2885 * is done in scsi_logout_post_handler(). 2886 */ 2887 switch (cmd->logout_reason) { 2888 case ISCSI_LOGOUT_REASON_CLOSE_SESSION: 2889 pr_debug("iSCSI session logout successful, setting" 2890 " logout response to ISCSI_LOGOUT_SUCCESS.\n"); 2891 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2892 break; 2893 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: 2894 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND) 2895 break; 2896 /* 2897 * For CLOSECONNECTION logout requests carrying 2898 * a matching logout CID -> local CID, the reference 2899 * for the local CID will have been incremented in 2900 * iscsi_logout_closeconnection(). 2901 * 2902 * For CLOSECONNECTION logout requests carrying 2903 * a different CID than the connection it arrived 2904 * on, the connection responding to cmd->logout_cid 2905 * is stopped in iscsit_logout_post_handler_diffcid(). 2906 */ 2907 2908 pr_debug("iSCSI CID: %hu logout on CID: %hu" 2909 " successful.\n", cmd->logout_cid, conn->cid); 2910 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2911 break; 2912 case ISCSI_LOGOUT_REASON_RECOVERY: 2913 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) || 2914 (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED)) 2915 break; 2916 /* 2917 * If the connection is still active from our point of view 2918 * force connection recovery to occur. 2919 */ 2920 logout_conn = iscsit_get_conn_from_cid_rcfr(sess, 2921 cmd->logout_cid); 2922 if (logout_conn) { 2923 iscsit_connection_reinstatement_rcfr(logout_conn); 2924 iscsit_dec_conn_usage_count(logout_conn); 2925 } 2926 2927 cr = iscsit_get_inactive_connection_recovery_entry( 2928 conn->sess, cmd->logout_cid); 2929 if (!cr) { 2930 pr_err("Unable to locate CID: %hu for" 2931 " REMOVECONNFORRECOVERY Logout Request.\n", 2932 cmd->logout_cid); 2933 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND; 2934 break; 2935 } 2936 2937 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn); 2938 2939 pr_debug("iSCSI REMOVECONNFORRECOVERY logout" 2940 " for recovery for CID: %hu on CID: %hu successful.\n", 2941 cmd->logout_cid, conn->cid); 2942 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2943 break; 2944 default: 2945 pr_err("Unknown cmd->logout_reason: 0x%02x\n", 2946 cmd->logout_reason); 2947 return -1; 2948 } 2949 2950 hdr->opcode = ISCSI_OP_LOGOUT_RSP; 2951 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2952 hdr->response = cmd->logout_response; 2953 hdr->itt = cmd->init_task_tag; 2954 cmd->stat_sn = conn->stat_sn++; 2955 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2956 2957 iscsit_increment_maxcmdsn(cmd, conn->sess); 2958 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2959 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 2960 2961 pr_debug("Built Logout Response ITT: 0x%08x StatSN:" 2962 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n", 2963 cmd->init_task_tag, cmd->stat_sn, hdr->response, 2964 cmd->logout_cid, conn->cid); 2965 2966 return 0; 2967 } 2968 EXPORT_SYMBOL(iscsit_build_logout_rsp); 2969 2970 static int 2971 iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2972 { 2973 int rc; 2974 2975 rc = iscsit_build_logout_rsp(cmd, conn, 2976 (struct iscsi_logout_rsp *)&cmd->pdu[0]); 2977 if (rc < 0) 2978 return rc; 2979 2980 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 2981 } 2982 2983 void 2984 iscsit_build_nopin_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2985 struct iscsi_nopin *hdr, bool nopout_response) 2986 { 2987 hdr->opcode = ISCSI_OP_NOOP_IN; 2988 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2989 hton24(hdr->dlength, cmd->buf_ptr_size); 2990 if (nopout_response) 2991 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); 2992 hdr->itt = cmd->init_task_tag; 2993 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 2994 cmd->stat_sn = (nopout_response) ? conn->stat_sn++ : 2995 conn->stat_sn; 2996 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2997 2998 if (nopout_response) 2999 iscsit_increment_maxcmdsn(cmd, conn->sess); 3000 3001 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3002 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3003 3004 pr_debug("Built NOPIN %s Response ITT: 0x%08x, TTT: 0x%08x," 3005 " StatSN: 0x%08x, Length %u\n", (nopout_response) ? 3006 "Solicited" : "Unsolicited", cmd->init_task_tag, 3007 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size); 3008 } 3009 EXPORT_SYMBOL(iscsit_build_nopin_rsp); 3010 3011 /* 3012 * Unsolicited NOPIN, either requesting a response or not. 3013 */ 3014 static int iscsit_send_unsolicited_nopin( 3015 struct iscsi_cmd *cmd, 3016 struct iscsi_conn *conn, 3017 int want_response) 3018 { 3019 struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; 3020 int ret; 3021 3022 iscsit_build_nopin_rsp(cmd, conn, hdr, false); 3023 3024 pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:" 3025 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid); 3026 3027 ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 3028 if (ret < 0) 3029 return ret; 3030 3031 spin_lock_bh(&cmd->istate_lock); 3032 cmd->i_state = want_response ? 3033 ISTATE_SENT_NOPIN_WANT_RESPONSE : ISTATE_SENT_STATUS; 3034 spin_unlock_bh(&cmd->istate_lock); 3035 3036 return 0; 3037 } 3038 3039 static int 3040 iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3041 { 3042 struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; 3043 3044 iscsit_build_nopin_rsp(cmd, conn, hdr, true); 3045 3046 /* 3047 * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr. 3048 * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size. 3049 */ 3050 pr_debug("Echoing back %u bytes of ping data.\n", cmd->buf_ptr_size); 3051 3052 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 3053 cmd->buf_ptr, 3054 cmd->buf_ptr_size); 3055 } 3056 3057 static int iscsit_send_r2t( 3058 struct iscsi_cmd *cmd, 3059 struct iscsi_conn *conn) 3060 { 3061 struct iscsi_r2t *r2t; 3062 struct iscsi_r2t_rsp *hdr; 3063 int ret; 3064 3065 r2t = iscsit_get_r2t_from_list(cmd); 3066 if (!r2t) 3067 return -1; 3068 3069 hdr = (struct iscsi_r2t_rsp *) cmd->pdu; 3070 memset(hdr, 0, ISCSI_HDR_LEN); 3071 hdr->opcode = ISCSI_OP_R2T; 3072 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3073 int_to_scsilun(cmd->se_cmd.orig_fe_lun, 3074 (struct scsi_lun *)&hdr->lun); 3075 hdr->itt = cmd->init_task_tag; 3076 if (conn->conn_transport->iscsit_get_r2t_ttt) 3077 conn->conn_transport->iscsit_get_r2t_ttt(conn, cmd, r2t); 3078 else 3079 r2t->targ_xfer_tag = session_get_next_ttt(conn->sess); 3080 hdr->ttt = cpu_to_be32(r2t->targ_xfer_tag); 3081 hdr->statsn = cpu_to_be32(conn->stat_sn); 3082 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3083 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3084 hdr->r2tsn = cpu_to_be32(r2t->r2t_sn); 3085 hdr->data_offset = cpu_to_be32(r2t->offset); 3086 hdr->data_length = cpu_to_be32(r2t->xfer_len); 3087 3088 pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:" 3089 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n", 3090 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag, 3091 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn, 3092 r2t->offset, r2t->xfer_len, conn->cid); 3093 3094 spin_lock_bh(&cmd->r2t_lock); 3095 r2t->sent_r2t = 1; 3096 spin_unlock_bh(&cmd->r2t_lock); 3097 3098 ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 3099 if (ret < 0) { 3100 return ret; 3101 } 3102 3103 spin_lock_bh(&cmd->dataout_timeout_lock); 3104 iscsit_start_dataout_timer(cmd, conn); 3105 spin_unlock_bh(&cmd->dataout_timeout_lock); 3106 3107 return 0; 3108 } 3109 3110 /* 3111 * @recovery: If called from iscsi_task_reassign_complete_write() for 3112 * connection recovery. 3113 */ 3114 int iscsit_build_r2ts_for_cmd( 3115 struct iscsi_conn *conn, 3116 struct iscsi_cmd *cmd, 3117 bool recovery) 3118 { 3119 int first_r2t = 1; 3120 u32 offset = 0, xfer_len = 0; 3121 3122 spin_lock_bh(&cmd->r2t_lock); 3123 if (cmd->cmd_flags & ICF_SENT_LAST_R2T) { 3124 spin_unlock_bh(&cmd->r2t_lock); 3125 return 0; 3126 } 3127 3128 if (conn->sess->sess_ops->DataSequenceInOrder && 3129 !recovery) 3130 cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done); 3131 3132 while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) { 3133 if (conn->sess->sess_ops->DataSequenceInOrder) { 3134 offset = cmd->r2t_offset; 3135 3136 if (first_r2t && recovery) { 3137 int new_data_end = offset + 3138 conn->sess->sess_ops->MaxBurstLength - 3139 cmd->next_burst_len; 3140 3141 if (new_data_end > cmd->se_cmd.data_length) 3142 xfer_len = cmd->se_cmd.data_length - offset; 3143 else 3144 xfer_len = 3145 conn->sess->sess_ops->MaxBurstLength - 3146 cmd->next_burst_len; 3147 } else { 3148 int new_data_end = offset + 3149 conn->sess->sess_ops->MaxBurstLength; 3150 3151 if (new_data_end > cmd->se_cmd.data_length) 3152 xfer_len = cmd->se_cmd.data_length - offset; 3153 else 3154 xfer_len = conn->sess->sess_ops->MaxBurstLength; 3155 } 3156 3157 if ((s32)xfer_len < 0) { 3158 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 3159 break; 3160 } 3161 3162 cmd->r2t_offset += xfer_len; 3163 3164 if (cmd->r2t_offset == cmd->se_cmd.data_length) 3165 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 3166 } else { 3167 struct iscsi_seq *seq; 3168 3169 seq = iscsit_get_seq_holder_for_r2t(cmd); 3170 if (!seq) { 3171 spin_unlock_bh(&cmd->r2t_lock); 3172 return -1; 3173 } 3174 3175 offset = seq->offset; 3176 xfer_len = seq->xfer_len; 3177 3178 if (cmd->seq_send_order == cmd->seq_count) 3179 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 3180 } 3181 cmd->outstanding_r2ts++; 3182 first_r2t = 0; 3183 3184 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) { 3185 spin_unlock_bh(&cmd->r2t_lock); 3186 return -1; 3187 } 3188 3189 if (cmd->cmd_flags & ICF_SENT_LAST_R2T) 3190 break; 3191 } 3192 spin_unlock_bh(&cmd->r2t_lock); 3193 3194 return 0; 3195 } 3196 EXPORT_SYMBOL(iscsit_build_r2ts_for_cmd); 3197 3198 void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3199 bool inc_stat_sn, struct iscsi_scsi_rsp *hdr) 3200 { 3201 if (inc_stat_sn) 3202 cmd->stat_sn = conn->stat_sn++; 3203 3204 atomic_long_inc(&conn->sess->rsp_pdus); 3205 3206 memset(hdr, 0, ISCSI_HDR_LEN); 3207 hdr->opcode = ISCSI_OP_SCSI_CMD_RSP; 3208 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3209 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { 3210 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW; 3211 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 3212 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { 3213 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW; 3214 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 3215 } 3216 hdr->response = cmd->iscsi_response; 3217 hdr->cmd_status = cmd->se_cmd.scsi_status; 3218 hdr->itt = cmd->init_task_tag; 3219 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3220 3221 iscsit_increment_maxcmdsn(cmd, conn->sess); 3222 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3223 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3224 3225 pr_debug("Built SCSI Response, ITT: 0x%08x, StatSN: 0x%08x," 3226 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n", 3227 cmd->init_task_tag, cmd->stat_sn, cmd->se_cmd.scsi_status, 3228 cmd->se_cmd.scsi_status, conn->cid); 3229 } 3230 EXPORT_SYMBOL(iscsit_build_rsp_pdu); 3231 3232 static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3233 { 3234 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0]; 3235 bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS); 3236 void *data_buf = NULL; 3237 u32 padding = 0, data_buf_len = 0; 3238 3239 iscsit_build_rsp_pdu(cmd, conn, inc_stat_sn, hdr); 3240 3241 /* 3242 * Attach SENSE DATA payload to iSCSI Response PDU 3243 */ 3244 if (cmd->se_cmd.sense_buffer && 3245 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 3246 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 3247 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, cmd->sense_buffer); 3248 cmd->se_cmd.scsi_sense_length += sizeof (__be16); 3249 3250 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 3251 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 3252 data_buf = cmd->sense_buffer; 3253 data_buf_len = cmd->se_cmd.scsi_sense_length + padding; 3254 3255 if (padding) { 3256 memset(cmd->sense_buffer + 3257 cmd->se_cmd.scsi_sense_length, 0, padding); 3258 pr_debug("Adding %u bytes of padding to" 3259 " SENSE.\n", padding); 3260 } 3261 3262 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI" 3263 " Response PDU\n", 3264 cmd->se_cmd.scsi_sense_length); 3265 } 3266 3267 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, data_buf, 3268 data_buf_len); 3269 } 3270 3271 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr) 3272 { 3273 switch (se_tmr->response) { 3274 case TMR_FUNCTION_COMPLETE: 3275 return ISCSI_TMF_RSP_COMPLETE; 3276 case TMR_TASK_DOES_NOT_EXIST: 3277 return ISCSI_TMF_RSP_NO_TASK; 3278 case TMR_LUN_DOES_NOT_EXIST: 3279 return ISCSI_TMF_RSP_NO_LUN; 3280 case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED: 3281 return ISCSI_TMF_RSP_NOT_SUPPORTED; 3282 case TMR_FUNCTION_REJECTED: 3283 default: 3284 return ISCSI_TMF_RSP_REJECTED; 3285 } 3286 } 3287 3288 void 3289 iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3290 struct iscsi_tm_rsp *hdr) 3291 { 3292 struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req; 3293 3294 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP; 3295 hdr->flags = ISCSI_FLAG_CMD_FINAL; 3296 hdr->response = iscsit_convert_tcm_tmr_rsp(se_tmr); 3297 hdr->itt = cmd->init_task_tag; 3298 cmd->stat_sn = conn->stat_sn++; 3299 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3300 3301 iscsit_increment_maxcmdsn(cmd, conn->sess); 3302 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3303 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3304 3305 pr_debug("Built Task Management Response ITT: 0x%08x," 3306 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n", 3307 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid); 3308 } 3309 EXPORT_SYMBOL(iscsit_build_task_mgt_rsp); 3310 3311 static int 3312 iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3313 { 3314 struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0]; 3315 3316 iscsit_build_task_mgt_rsp(cmd, conn, hdr); 3317 3318 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 3319 } 3320 3321 #define SENDTARGETS_BUF_LIMIT 32768U 3322 3323 static int 3324 iscsit_build_sendtargets_response(struct iscsi_cmd *cmd, 3325 enum iscsit_transport_type network_transport, 3326 int skip_bytes, bool *completed) 3327 { 3328 char *payload = NULL; 3329 struct iscsi_conn *conn = cmd->conn; 3330 struct iscsi_portal_group *tpg; 3331 struct iscsi_tiqn *tiqn; 3332 struct iscsi_tpg_np *tpg_np; 3333 int buffer_len, end_of_buf = 0, len = 0, payload_len = 0; 3334 int target_name_printed; 3335 unsigned char buf[ISCSI_IQN_LEN+12]; /* iqn + "TargetName=" + \0 */ 3336 unsigned char *text_in = cmd->text_in_ptr, *text_ptr = NULL; 3337 bool active; 3338 3339 buffer_len = min(conn->conn_ops->MaxRecvDataSegmentLength, 3340 SENDTARGETS_BUF_LIMIT); 3341 3342 payload = kzalloc(buffer_len, GFP_KERNEL); 3343 if (!payload) 3344 return -ENOMEM; 3345 3346 /* 3347 * Locate pointer to iqn./eui. string for ICF_SENDTARGETS_SINGLE 3348 * explicit case.. 3349 */ 3350 if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) { 3351 text_ptr = strchr(text_in, '='); 3352 if (!text_ptr) { 3353 pr_err("Unable to locate '=' string in text_in:" 3354 " %s\n", text_in); 3355 kfree(payload); 3356 return -EINVAL; 3357 } 3358 /* 3359 * Skip over '=' character.. 3360 */ 3361 text_ptr += 1; 3362 } 3363 3364 spin_lock(&tiqn_lock); 3365 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) { 3366 if ((cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) && 3367 strcmp(tiqn->tiqn, text_ptr)) { 3368 continue; 3369 } 3370 3371 target_name_printed = 0; 3372 3373 spin_lock(&tiqn->tiqn_tpg_lock); 3374 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) { 3375 3376 /* If demo_mode_discovery=0 and generate_node_acls=0 3377 * (demo mode dislabed) do not return 3378 * TargetName+TargetAddress unless a NodeACL exists. 3379 */ 3380 3381 if ((tpg->tpg_attrib.generate_node_acls == 0) && 3382 (tpg->tpg_attrib.demo_mode_discovery == 0) && 3383 (!target_tpg_has_node_acl(&tpg->tpg_se_tpg, 3384 cmd->conn->sess->sess_ops->InitiatorName))) { 3385 continue; 3386 } 3387 3388 spin_lock(&tpg->tpg_state_lock); 3389 active = (tpg->tpg_state == TPG_STATE_ACTIVE); 3390 spin_unlock(&tpg->tpg_state_lock); 3391 3392 if (!active && tpg->tpg_attrib.tpg_enabled_sendtargets) 3393 continue; 3394 3395 spin_lock(&tpg->tpg_np_lock); 3396 list_for_each_entry(tpg_np, &tpg->tpg_gnp_list, 3397 tpg_np_list) { 3398 struct iscsi_np *np = tpg_np->tpg_np; 3399 struct sockaddr_storage *sockaddr; 3400 3401 if (np->np_network_transport != network_transport) 3402 continue; 3403 3404 if (!target_name_printed) { 3405 len = sprintf(buf, "TargetName=%s", 3406 tiqn->tiqn); 3407 len += 1; 3408 3409 if ((len + payload_len) > buffer_len) { 3410 spin_unlock(&tpg->tpg_np_lock); 3411 spin_unlock(&tiqn->tiqn_tpg_lock); 3412 end_of_buf = 1; 3413 goto eob; 3414 } 3415 3416 if (skip_bytes && len <= skip_bytes) { 3417 skip_bytes -= len; 3418 } else { 3419 memcpy(payload + payload_len, buf, len); 3420 payload_len += len; 3421 target_name_printed = 1; 3422 if (len > skip_bytes) 3423 skip_bytes = 0; 3424 } 3425 } 3426 3427 if (inet_addr_is_any((struct sockaddr *)&np->np_sockaddr)) 3428 sockaddr = &conn->local_sockaddr; 3429 else 3430 sockaddr = &np->np_sockaddr; 3431 3432 len = sprintf(buf, "TargetAddress=" 3433 "%pISpc,%hu", 3434 sockaddr, 3435 tpg->tpgt); 3436 len += 1; 3437 3438 if ((len + payload_len) > buffer_len) { 3439 spin_unlock(&tpg->tpg_np_lock); 3440 spin_unlock(&tiqn->tiqn_tpg_lock); 3441 end_of_buf = 1; 3442 goto eob; 3443 } 3444 3445 if (skip_bytes && len <= skip_bytes) { 3446 skip_bytes -= len; 3447 } else { 3448 memcpy(payload + payload_len, buf, len); 3449 payload_len += len; 3450 if (len > skip_bytes) 3451 skip_bytes = 0; 3452 } 3453 } 3454 spin_unlock(&tpg->tpg_np_lock); 3455 } 3456 spin_unlock(&tiqn->tiqn_tpg_lock); 3457 eob: 3458 if (end_of_buf) { 3459 *completed = false; 3460 break; 3461 } 3462 3463 if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) 3464 break; 3465 } 3466 spin_unlock(&tiqn_lock); 3467 3468 cmd->buf_ptr = payload; 3469 3470 return payload_len; 3471 } 3472 3473 int 3474 iscsit_build_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3475 struct iscsi_text_rsp *hdr, 3476 enum iscsit_transport_type network_transport) 3477 { 3478 int text_length, padding; 3479 bool completed = true; 3480 3481 text_length = iscsit_build_sendtargets_response(cmd, network_transport, 3482 cmd->read_data_done, 3483 &completed); 3484 if (text_length < 0) 3485 return text_length; 3486 3487 if (completed) { 3488 hdr->flags = ISCSI_FLAG_CMD_FINAL; 3489 } else { 3490 hdr->flags = ISCSI_FLAG_TEXT_CONTINUE; 3491 cmd->read_data_done += text_length; 3492 if (cmd->targ_xfer_tag == 0xFFFFFFFF) 3493 cmd->targ_xfer_tag = session_get_next_ttt(conn->sess); 3494 } 3495 hdr->opcode = ISCSI_OP_TEXT_RSP; 3496 padding = ((-text_length) & 3); 3497 hton24(hdr->dlength, text_length); 3498 hdr->itt = cmd->init_task_tag; 3499 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 3500 cmd->stat_sn = conn->stat_sn++; 3501 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3502 3503 iscsit_increment_maxcmdsn(cmd, conn->sess); 3504 /* 3505 * Reset maxcmdsn_inc in multi-part text payload exchanges to 3506 * correctly increment MaxCmdSN for each response answering a 3507 * non immediate text request with a valid CmdSN. 3508 */ 3509 cmd->maxcmdsn_inc = 0; 3510 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3511 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3512 3513 pr_debug("Built Text Response: ITT: 0x%08x, TTT: 0x%08x, StatSN: 0x%08x," 3514 " Length: %u, CID: %hu F: %d C: %d\n", cmd->init_task_tag, 3515 cmd->targ_xfer_tag, cmd->stat_sn, text_length, conn->cid, 3516 !!(hdr->flags & ISCSI_FLAG_CMD_FINAL), 3517 !!(hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)); 3518 3519 return text_length + padding; 3520 } 3521 EXPORT_SYMBOL(iscsit_build_text_rsp); 3522 3523 static int iscsit_send_text_rsp( 3524 struct iscsi_cmd *cmd, 3525 struct iscsi_conn *conn) 3526 { 3527 struct iscsi_text_rsp *hdr = (struct iscsi_text_rsp *)cmd->pdu; 3528 int text_length; 3529 3530 text_length = iscsit_build_text_rsp(cmd, conn, hdr, 3531 conn->conn_transport->transport_type); 3532 if (text_length < 0) 3533 return text_length; 3534 3535 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 3536 cmd->buf_ptr, 3537 text_length); 3538 } 3539 3540 void 3541 iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3542 struct iscsi_reject *hdr) 3543 { 3544 hdr->opcode = ISCSI_OP_REJECT; 3545 hdr->reason = cmd->reject_reason; 3546 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3547 hton24(hdr->dlength, ISCSI_HDR_LEN); 3548 hdr->ffffffff = cpu_to_be32(0xffffffff); 3549 cmd->stat_sn = conn->stat_sn++; 3550 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3551 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3552 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3553 3554 } 3555 EXPORT_SYMBOL(iscsit_build_reject); 3556 3557 static int iscsit_send_reject( 3558 struct iscsi_cmd *cmd, 3559 struct iscsi_conn *conn) 3560 { 3561 struct iscsi_reject *hdr = (struct iscsi_reject *)&cmd->pdu[0]; 3562 3563 iscsit_build_reject(cmd, conn, hdr); 3564 3565 pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x," 3566 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid); 3567 3568 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 3569 cmd->buf_ptr, 3570 ISCSI_HDR_LEN); 3571 } 3572 3573 void iscsit_thread_get_cpumask(struct iscsi_conn *conn) 3574 { 3575 int ord, cpu; 3576 /* 3577 * bitmap_id is assigned from iscsit_global->ts_bitmap from 3578 * within iscsit_start_kthreads() 3579 * 3580 * Here we use bitmap_id to determine which CPU that this 3581 * iSCSI connection's RX/TX threads will be scheduled to 3582 * execute upon. 3583 */ 3584 ord = conn->bitmap_id % cpumask_weight(cpu_online_mask); 3585 for_each_online_cpu(cpu) { 3586 if (ord-- == 0) { 3587 cpumask_set_cpu(cpu, conn->conn_cpumask); 3588 return; 3589 } 3590 } 3591 /* 3592 * This should never be reached.. 3593 */ 3594 dump_stack(); 3595 cpumask_setall(conn->conn_cpumask); 3596 } 3597 3598 int 3599 iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3600 { 3601 int ret; 3602 3603 switch (state) { 3604 case ISTATE_SEND_R2T: 3605 ret = iscsit_send_r2t(cmd, conn); 3606 if (ret < 0) 3607 goto err; 3608 break; 3609 case ISTATE_REMOVE: 3610 spin_lock_bh(&conn->cmd_lock); 3611 list_del_init(&cmd->i_conn_node); 3612 spin_unlock_bh(&conn->cmd_lock); 3613 3614 iscsit_free_cmd(cmd, false); 3615 break; 3616 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 3617 iscsit_mod_nopin_response_timer(conn); 3618 ret = iscsit_send_unsolicited_nopin(cmd, conn, 1); 3619 if (ret < 0) 3620 goto err; 3621 break; 3622 case ISTATE_SEND_NOPIN_NO_RESPONSE: 3623 ret = iscsit_send_unsolicited_nopin(cmd, conn, 0); 3624 if (ret < 0) 3625 goto err; 3626 break; 3627 default: 3628 pr_err("Unknown Opcode: 0x%02x ITT:" 3629 " 0x%08x, i_state: %d on CID: %hu\n", 3630 cmd->iscsi_opcode, cmd->init_task_tag, state, 3631 conn->cid); 3632 goto err; 3633 } 3634 3635 return 0; 3636 3637 err: 3638 return -1; 3639 } 3640 EXPORT_SYMBOL(iscsit_immediate_queue); 3641 3642 static int 3643 iscsit_handle_immediate_queue(struct iscsi_conn *conn) 3644 { 3645 struct iscsit_transport *t = conn->conn_transport; 3646 struct iscsi_queue_req *qr; 3647 struct iscsi_cmd *cmd; 3648 u8 state; 3649 int ret; 3650 3651 while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) { 3652 atomic_set(&conn->check_immediate_queue, 0); 3653 cmd = qr->cmd; 3654 state = qr->state; 3655 kmem_cache_free(lio_qr_cache, qr); 3656 3657 ret = t->iscsit_immediate_queue(conn, cmd, state); 3658 if (ret < 0) 3659 return ret; 3660 } 3661 3662 return 0; 3663 } 3664 3665 int 3666 iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3667 { 3668 int ret; 3669 3670 check_rsp_state: 3671 switch (state) { 3672 case ISTATE_SEND_DATAIN: 3673 ret = iscsit_send_datain(cmd, conn); 3674 if (ret < 0) 3675 goto err; 3676 else if (!ret) 3677 /* more drs */ 3678 goto check_rsp_state; 3679 else if (ret == 1) { 3680 /* all done */ 3681 spin_lock_bh(&cmd->istate_lock); 3682 cmd->i_state = ISTATE_SENT_STATUS; 3683 spin_unlock_bh(&cmd->istate_lock); 3684 3685 if (atomic_read(&conn->check_immediate_queue)) 3686 return 1; 3687 3688 return 0; 3689 } else if (ret == 2) { 3690 /* Still must send status, 3691 SCF_TRANSPORT_TASK_SENSE was set */ 3692 spin_lock_bh(&cmd->istate_lock); 3693 cmd->i_state = ISTATE_SEND_STATUS; 3694 spin_unlock_bh(&cmd->istate_lock); 3695 state = ISTATE_SEND_STATUS; 3696 goto check_rsp_state; 3697 } 3698 3699 break; 3700 case ISTATE_SEND_STATUS: 3701 case ISTATE_SEND_STATUS_RECOVERY: 3702 ret = iscsit_send_response(cmd, conn); 3703 break; 3704 case ISTATE_SEND_LOGOUTRSP: 3705 ret = iscsit_send_logout(cmd, conn); 3706 break; 3707 case ISTATE_SEND_ASYNCMSG: 3708 ret = iscsit_send_conn_drop_async_message( 3709 cmd, conn); 3710 break; 3711 case ISTATE_SEND_NOPIN: 3712 ret = iscsit_send_nopin(cmd, conn); 3713 break; 3714 case ISTATE_SEND_REJECT: 3715 ret = iscsit_send_reject(cmd, conn); 3716 break; 3717 case ISTATE_SEND_TASKMGTRSP: 3718 ret = iscsit_send_task_mgt_rsp(cmd, conn); 3719 if (ret != 0) 3720 break; 3721 ret = iscsit_tmr_post_handler(cmd, conn); 3722 if (ret != 0) 3723 iscsit_fall_back_to_erl0(conn->sess); 3724 break; 3725 case ISTATE_SEND_TEXTRSP: 3726 ret = iscsit_send_text_rsp(cmd, conn); 3727 break; 3728 default: 3729 pr_err("Unknown Opcode: 0x%02x ITT:" 3730 " 0x%08x, i_state: %d on CID: %hu\n", 3731 cmd->iscsi_opcode, cmd->init_task_tag, 3732 state, conn->cid); 3733 goto err; 3734 } 3735 if (ret < 0) 3736 goto err; 3737 3738 switch (state) { 3739 case ISTATE_SEND_LOGOUTRSP: 3740 if (!iscsit_logout_post_handler(cmd, conn)) 3741 return -ECONNRESET; 3742 /* fall through */ 3743 case ISTATE_SEND_STATUS: 3744 case ISTATE_SEND_ASYNCMSG: 3745 case ISTATE_SEND_NOPIN: 3746 case ISTATE_SEND_STATUS_RECOVERY: 3747 case ISTATE_SEND_TEXTRSP: 3748 case ISTATE_SEND_TASKMGTRSP: 3749 case ISTATE_SEND_REJECT: 3750 spin_lock_bh(&cmd->istate_lock); 3751 cmd->i_state = ISTATE_SENT_STATUS; 3752 spin_unlock_bh(&cmd->istate_lock); 3753 break; 3754 default: 3755 pr_err("Unknown Opcode: 0x%02x ITT:" 3756 " 0x%08x, i_state: %d on CID: %hu\n", 3757 cmd->iscsi_opcode, cmd->init_task_tag, 3758 cmd->i_state, conn->cid); 3759 goto err; 3760 } 3761 3762 if (atomic_read(&conn->check_immediate_queue)) 3763 return 1; 3764 3765 return 0; 3766 3767 err: 3768 return -1; 3769 } 3770 EXPORT_SYMBOL(iscsit_response_queue); 3771 3772 static int iscsit_handle_response_queue(struct iscsi_conn *conn) 3773 { 3774 struct iscsit_transport *t = conn->conn_transport; 3775 struct iscsi_queue_req *qr; 3776 struct iscsi_cmd *cmd; 3777 u8 state; 3778 int ret; 3779 3780 while ((qr = iscsit_get_cmd_from_response_queue(conn))) { 3781 cmd = qr->cmd; 3782 state = qr->state; 3783 kmem_cache_free(lio_qr_cache, qr); 3784 3785 ret = t->iscsit_response_queue(conn, cmd, state); 3786 if (ret == 1 || ret < 0) 3787 return ret; 3788 } 3789 3790 return 0; 3791 } 3792 3793 int iscsi_target_tx_thread(void *arg) 3794 { 3795 int ret = 0; 3796 struct iscsi_conn *conn = arg; 3797 bool conn_freed = false; 3798 3799 /* 3800 * Allow ourselves to be interrupted by SIGINT so that a 3801 * connection recovery / failure event can be triggered externally. 3802 */ 3803 allow_signal(SIGINT); 3804 3805 while (!kthread_should_stop()) { 3806 /* 3807 * Ensure that both TX and RX per connection kthreads 3808 * are scheduled to run on the same CPU. 3809 */ 3810 iscsit_thread_check_cpumask(conn, current, 1); 3811 3812 wait_event_interruptible(conn->queues_wq, 3813 !iscsit_conn_all_queues_empty(conn)); 3814 3815 if (signal_pending(current)) 3816 goto transport_err; 3817 3818 get_immediate: 3819 ret = iscsit_handle_immediate_queue(conn); 3820 if (ret < 0) 3821 goto transport_err; 3822 3823 ret = iscsit_handle_response_queue(conn); 3824 if (ret == 1) { 3825 goto get_immediate; 3826 } else if (ret == -ECONNRESET) { 3827 conn_freed = true; 3828 goto out; 3829 } else if (ret < 0) { 3830 goto transport_err; 3831 } 3832 } 3833 3834 transport_err: 3835 /* 3836 * Avoid the normal connection failure code-path if this connection 3837 * is still within LOGIN mode, and iscsi_np process context is 3838 * responsible for cleaning up the early connection failure. 3839 */ 3840 if (conn->conn_state != TARG_CONN_STATE_IN_LOGIN) 3841 iscsit_take_action_for_connection_exit(conn, &conn_freed); 3842 out: 3843 if (!conn_freed) { 3844 while (!kthread_should_stop()) { 3845 msleep(100); 3846 } 3847 } 3848 return 0; 3849 } 3850 3851 static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf) 3852 { 3853 struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf; 3854 struct iscsi_cmd *cmd; 3855 int ret = 0; 3856 3857 switch (hdr->opcode & ISCSI_OPCODE_MASK) { 3858 case ISCSI_OP_SCSI_CMD: 3859 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3860 if (!cmd) 3861 goto reject; 3862 3863 ret = iscsit_handle_scsi_cmd(conn, cmd, buf); 3864 break; 3865 case ISCSI_OP_SCSI_DATA_OUT: 3866 ret = iscsit_handle_data_out(conn, buf); 3867 break; 3868 case ISCSI_OP_NOOP_OUT: 3869 cmd = NULL; 3870 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 3871 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3872 if (!cmd) 3873 goto reject; 3874 } 3875 ret = iscsit_handle_nop_out(conn, cmd, buf); 3876 break; 3877 case ISCSI_OP_SCSI_TMFUNC: 3878 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3879 if (!cmd) 3880 goto reject; 3881 3882 ret = iscsit_handle_task_mgt_cmd(conn, cmd, buf); 3883 break; 3884 case ISCSI_OP_TEXT: 3885 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 3886 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 3887 if (!cmd) 3888 goto reject; 3889 } else { 3890 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3891 if (!cmd) 3892 goto reject; 3893 } 3894 3895 ret = iscsit_handle_text_cmd(conn, cmd, buf); 3896 break; 3897 case ISCSI_OP_LOGOUT: 3898 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3899 if (!cmd) 3900 goto reject; 3901 3902 ret = iscsit_handle_logout_cmd(conn, cmd, buf); 3903 if (ret > 0) 3904 wait_for_completion_timeout(&conn->conn_logout_comp, 3905 SECONDS_FOR_LOGOUT_COMP * HZ); 3906 break; 3907 case ISCSI_OP_SNACK: 3908 ret = iscsit_handle_snack(conn, buf); 3909 break; 3910 default: 3911 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", hdr->opcode); 3912 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 3913 pr_err("Cannot recover from unknown" 3914 " opcode while ERL=0, closing iSCSI connection.\n"); 3915 return -1; 3916 } 3917 pr_err("Unable to recover from unknown opcode while OFMarker=No," 3918 " closing iSCSI connection.\n"); 3919 ret = -1; 3920 break; 3921 } 3922 3923 return ret; 3924 reject: 3925 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 3926 } 3927 3928 static bool iscsi_target_check_conn_state(struct iscsi_conn *conn) 3929 { 3930 bool ret; 3931 3932 spin_lock_bh(&conn->state_lock); 3933 ret = (conn->conn_state != TARG_CONN_STATE_LOGGED_IN); 3934 spin_unlock_bh(&conn->state_lock); 3935 3936 return ret; 3937 } 3938 3939 static void iscsit_get_rx_pdu(struct iscsi_conn *conn) 3940 { 3941 int ret; 3942 u8 *buffer, opcode; 3943 u32 checksum = 0, digest = 0; 3944 struct kvec iov; 3945 3946 buffer = kcalloc(ISCSI_HDR_LEN, sizeof(*buffer), GFP_KERNEL); 3947 if (!buffer) 3948 return; 3949 3950 while (!kthread_should_stop()) { 3951 /* 3952 * Ensure that both TX and RX per connection kthreads 3953 * are scheduled to run on the same CPU. 3954 */ 3955 iscsit_thread_check_cpumask(conn, current, 0); 3956 3957 memset(&iov, 0, sizeof(struct kvec)); 3958 3959 iov.iov_base = buffer; 3960 iov.iov_len = ISCSI_HDR_LEN; 3961 3962 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN); 3963 if (ret != ISCSI_HDR_LEN) { 3964 iscsit_rx_thread_wait_for_tcp(conn); 3965 break; 3966 } 3967 3968 if (conn->conn_ops->HeaderDigest) { 3969 iov.iov_base = &digest; 3970 iov.iov_len = ISCSI_CRC_LEN; 3971 3972 ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN); 3973 if (ret != ISCSI_CRC_LEN) { 3974 iscsit_rx_thread_wait_for_tcp(conn); 3975 break; 3976 } 3977 3978 iscsit_do_crypto_hash_buf(conn->conn_rx_hash, buffer, 3979 ISCSI_HDR_LEN, 0, NULL, 3980 &checksum); 3981 3982 if (digest != checksum) { 3983 pr_err("HeaderDigest CRC32C failed," 3984 " received 0x%08x, computed 0x%08x\n", 3985 digest, checksum); 3986 /* 3987 * Set the PDU to 0xff so it will intentionally 3988 * hit default in the switch below. 3989 */ 3990 memset(buffer, 0xff, ISCSI_HDR_LEN); 3991 atomic_long_inc(&conn->sess->conn_digest_errors); 3992 } else { 3993 pr_debug("Got HeaderDigest CRC32C" 3994 " 0x%08x\n", checksum); 3995 } 3996 } 3997 3998 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 3999 break; 4000 4001 opcode = buffer[0] & ISCSI_OPCODE_MASK; 4002 4003 if (conn->sess->sess_ops->SessionType && 4004 ((!(opcode & ISCSI_OP_TEXT)) || 4005 (!(opcode & ISCSI_OP_LOGOUT)))) { 4006 pr_err("Received illegal iSCSI Opcode: 0x%02x" 4007 " while in Discovery Session, rejecting.\n", opcode); 4008 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 4009 buffer); 4010 break; 4011 } 4012 4013 ret = iscsi_target_rx_opcode(conn, buffer); 4014 if (ret < 0) 4015 break; 4016 } 4017 4018 kfree(buffer); 4019 } 4020 4021 int iscsi_target_rx_thread(void *arg) 4022 { 4023 int rc; 4024 struct iscsi_conn *conn = arg; 4025 bool conn_freed = false; 4026 4027 /* 4028 * Allow ourselves to be interrupted by SIGINT so that a 4029 * connection recovery / failure event can be triggered externally. 4030 */ 4031 allow_signal(SIGINT); 4032 /* 4033 * Wait for iscsi_post_login_handler() to complete before allowing 4034 * incoming iscsi/tcp socket I/O, and/or failing the connection. 4035 */ 4036 rc = wait_for_completion_interruptible(&conn->rx_login_comp); 4037 if (rc < 0 || iscsi_target_check_conn_state(conn)) 4038 goto out; 4039 4040 if (!conn->conn_transport->iscsit_get_rx_pdu) 4041 return 0; 4042 4043 conn->conn_transport->iscsit_get_rx_pdu(conn); 4044 4045 if (!signal_pending(current)) 4046 atomic_set(&conn->transport_failed, 1); 4047 iscsit_take_action_for_connection_exit(conn, &conn_freed); 4048 4049 out: 4050 if (!conn_freed) { 4051 while (!kthread_should_stop()) { 4052 msleep(100); 4053 } 4054 } 4055 4056 return 0; 4057 } 4058 4059 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn) 4060 { 4061 LIST_HEAD(tmp_list); 4062 struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL; 4063 struct iscsi_session *sess = conn->sess; 4064 /* 4065 * We expect this function to only ever be called from either RX or TX 4066 * thread context via iscsit_close_connection() once the other context 4067 * has been reset -> returned sleeping pre-handler state. 4068 */ 4069 spin_lock_bh(&conn->cmd_lock); 4070 list_splice_init(&conn->conn_cmd_list, &tmp_list); 4071 4072 list_for_each_entry(cmd, &tmp_list, i_conn_node) { 4073 struct se_cmd *se_cmd = &cmd->se_cmd; 4074 4075 if (se_cmd->se_tfo != NULL) { 4076 spin_lock_irq(&se_cmd->t_state_lock); 4077 se_cmd->transport_state |= CMD_T_FABRIC_STOP; 4078 spin_unlock_irq(&se_cmd->t_state_lock); 4079 } 4080 } 4081 spin_unlock_bh(&conn->cmd_lock); 4082 4083 list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) { 4084 list_del_init(&cmd->i_conn_node); 4085 4086 iscsit_increment_maxcmdsn(cmd, sess); 4087 iscsit_free_cmd(cmd, true); 4088 4089 } 4090 } 4091 4092 static void iscsit_stop_timers_for_cmds( 4093 struct iscsi_conn *conn) 4094 { 4095 struct iscsi_cmd *cmd; 4096 4097 spin_lock_bh(&conn->cmd_lock); 4098 list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) { 4099 if (cmd->data_direction == DMA_TO_DEVICE) 4100 iscsit_stop_dataout_timer(cmd); 4101 } 4102 spin_unlock_bh(&conn->cmd_lock); 4103 } 4104 4105 int iscsit_close_connection( 4106 struct iscsi_conn *conn) 4107 { 4108 int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT); 4109 struct iscsi_session *sess = conn->sess; 4110 4111 pr_debug("Closing iSCSI connection CID %hu on SID:" 4112 " %u\n", conn->cid, sess->sid); 4113 /* 4114 * Always up conn_logout_comp for the traditional TCP and HW_OFFLOAD 4115 * case just in case the RX Thread in iscsi_target_rx_opcode() is 4116 * sleeping and the logout response never got sent because the 4117 * connection failed. 4118 * 4119 * However for iser-target, isert_wait4logout() is using conn_logout_comp 4120 * to signal logout response TX interrupt completion. Go ahead and skip 4121 * this for iser since isert_rx_opcode() does not wait on logout failure, 4122 * and to avoid iscsi_conn pointer dereference in iser-target code. 4123 */ 4124 if (!conn->conn_transport->rdma_shutdown) 4125 complete(&conn->conn_logout_comp); 4126 4127 if (!strcmp(current->comm, ISCSI_RX_THREAD_NAME)) { 4128 if (conn->tx_thread && 4129 cmpxchg(&conn->tx_thread_active, true, false)) { 4130 send_sig(SIGINT, conn->tx_thread, 1); 4131 kthread_stop(conn->tx_thread); 4132 } 4133 } else if (!strcmp(current->comm, ISCSI_TX_THREAD_NAME)) { 4134 if (conn->rx_thread && 4135 cmpxchg(&conn->rx_thread_active, true, false)) { 4136 send_sig(SIGINT, conn->rx_thread, 1); 4137 kthread_stop(conn->rx_thread); 4138 } 4139 } 4140 4141 spin_lock(&iscsit_global->ts_bitmap_lock); 4142 bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id, 4143 get_order(1)); 4144 spin_unlock(&iscsit_global->ts_bitmap_lock); 4145 4146 iscsit_stop_timers_for_cmds(conn); 4147 iscsit_stop_nopin_response_timer(conn); 4148 iscsit_stop_nopin_timer(conn); 4149 4150 if (conn->conn_transport->iscsit_wait_conn) 4151 conn->conn_transport->iscsit_wait_conn(conn); 4152 4153 /* 4154 * During Connection recovery drop unacknowledged out of order 4155 * commands for this connection, and prepare the other commands 4156 * for reallegiance. 4157 * 4158 * During normal operation clear the out of order commands (but 4159 * do not free the struct iscsi_ooo_cmdsn's) and release all 4160 * struct iscsi_cmds. 4161 */ 4162 if (atomic_read(&conn->connection_recovery)) { 4163 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn); 4164 iscsit_prepare_cmds_for_reallegiance(conn); 4165 } else { 4166 iscsit_clear_ooo_cmdsns_for_conn(conn); 4167 iscsit_release_commands_from_conn(conn); 4168 } 4169 iscsit_free_queue_reqs_for_conn(conn); 4170 4171 /* 4172 * Handle decrementing session or connection usage count if 4173 * a logout response was not able to be sent because the 4174 * connection failed. Fall back to Session Recovery here. 4175 */ 4176 if (atomic_read(&conn->conn_logout_remove)) { 4177 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) { 4178 iscsit_dec_conn_usage_count(conn); 4179 iscsit_dec_session_usage_count(sess); 4180 } 4181 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) 4182 iscsit_dec_conn_usage_count(conn); 4183 4184 atomic_set(&conn->conn_logout_remove, 0); 4185 atomic_set(&sess->session_reinstatement, 0); 4186 atomic_set(&sess->session_fall_back_to_erl0, 1); 4187 } 4188 4189 spin_lock_bh(&sess->conn_lock); 4190 list_del(&conn->conn_list); 4191 4192 /* 4193 * Attempt to let the Initiator know this connection failed by 4194 * sending an Connection Dropped Async Message on another 4195 * active connection. 4196 */ 4197 if (atomic_read(&conn->connection_recovery)) 4198 iscsit_build_conn_drop_async_message(conn); 4199 4200 spin_unlock_bh(&sess->conn_lock); 4201 4202 /* 4203 * If connection reinstatement is being performed on this connection, 4204 * up the connection reinstatement semaphore that is being blocked on 4205 * in iscsit_cause_connection_reinstatement(). 4206 */ 4207 spin_lock_bh(&conn->state_lock); 4208 if (atomic_read(&conn->sleep_on_conn_wait_comp)) { 4209 spin_unlock_bh(&conn->state_lock); 4210 complete(&conn->conn_wait_comp); 4211 wait_for_completion(&conn->conn_post_wait_comp); 4212 spin_lock_bh(&conn->state_lock); 4213 } 4214 4215 /* 4216 * If connection reinstatement is being performed on this connection 4217 * by receiving a REMOVECONNFORRECOVERY logout request, up the 4218 * connection wait rcfr semaphore that is being blocked on 4219 * an iscsit_connection_reinstatement_rcfr(). 4220 */ 4221 if (atomic_read(&conn->connection_wait_rcfr)) { 4222 spin_unlock_bh(&conn->state_lock); 4223 complete(&conn->conn_wait_rcfr_comp); 4224 wait_for_completion(&conn->conn_post_wait_comp); 4225 spin_lock_bh(&conn->state_lock); 4226 } 4227 atomic_set(&conn->connection_reinstatement, 1); 4228 spin_unlock_bh(&conn->state_lock); 4229 4230 /* 4231 * If any other processes are accessing this connection pointer we 4232 * must wait until they have completed. 4233 */ 4234 iscsit_check_conn_usage_count(conn); 4235 4236 ahash_request_free(conn->conn_tx_hash); 4237 if (conn->conn_rx_hash) { 4238 struct crypto_ahash *tfm; 4239 4240 tfm = crypto_ahash_reqtfm(conn->conn_rx_hash); 4241 ahash_request_free(conn->conn_rx_hash); 4242 crypto_free_ahash(tfm); 4243 } 4244 4245 if (conn->sock) 4246 sock_release(conn->sock); 4247 4248 if (conn->conn_transport->iscsit_free_conn) 4249 conn->conn_transport->iscsit_free_conn(conn); 4250 4251 pr_debug("Moving to TARG_CONN_STATE_FREE.\n"); 4252 conn->conn_state = TARG_CONN_STATE_FREE; 4253 iscsit_free_conn(conn); 4254 4255 spin_lock_bh(&sess->conn_lock); 4256 atomic_dec(&sess->nconn); 4257 pr_debug("Decremented iSCSI connection count to %hu from node:" 4258 " %s\n", atomic_read(&sess->nconn), 4259 sess->sess_ops->InitiatorName); 4260 /* 4261 * Make sure that if one connection fails in an non ERL=2 iSCSI 4262 * Session that they all fail. 4263 */ 4264 if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout && 4265 !atomic_read(&sess->session_logout)) 4266 atomic_set(&sess->session_fall_back_to_erl0, 1); 4267 4268 /* 4269 * If this was not the last connection in the session, and we are 4270 * performing session reinstatement or falling back to ERL=0, call 4271 * iscsit_stop_session() without sleeping to shutdown the other 4272 * active connections. 4273 */ 4274 if (atomic_read(&sess->nconn)) { 4275 if (!atomic_read(&sess->session_reinstatement) && 4276 !atomic_read(&sess->session_fall_back_to_erl0)) { 4277 spin_unlock_bh(&sess->conn_lock); 4278 return 0; 4279 } 4280 if (!atomic_read(&sess->session_stop_active)) { 4281 atomic_set(&sess->session_stop_active, 1); 4282 spin_unlock_bh(&sess->conn_lock); 4283 iscsit_stop_session(sess, 0, 0); 4284 return 0; 4285 } 4286 spin_unlock_bh(&sess->conn_lock); 4287 return 0; 4288 } 4289 4290 /* 4291 * If this was the last connection in the session and one of the 4292 * following is occurring: 4293 * 4294 * Session Reinstatement is not being performed, and are falling back 4295 * to ERL=0 call iscsit_close_session(). 4296 * 4297 * Session Logout was requested. iscsit_close_session() will be called 4298 * elsewhere. 4299 * 4300 * Session Continuation is not being performed, start the Time2Retain 4301 * handler and check if sleep_on_sess_wait_sem is active. 4302 */ 4303 if (!atomic_read(&sess->session_reinstatement) && 4304 atomic_read(&sess->session_fall_back_to_erl0)) { 4305 spin_unlock_bh(&sess->conn_lock); 4306 iscsit_close_session(sess); 4307 4308 return 0; 4309 } else if (atomic_read(&sess->session_logout)) { 4310 pr_debug("Moving to TARG_SESS_STATE_FREE.\n"); 4311 sess->session_state = TARG_SESS_STATE_FREE; 4312 spin_unlock_bh(&sess->conn_lock); 4313 4314 if (atomic_read(&sess->sleep_on_sess_wait_comp)) 4315 complete(&sess->session_wait_comp); 4316 4317 return 0; 4318 } else { 4319 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); 4320 sess->session_state = TARG_SESS_STATE_FAILED; 4321 4322 if (!atomic_read(&sess->session_continuation)) { 4323 spin_unlock_bh(&sess->conn_lock); 4324 iscsit_start_time2retain_handler(sess); 4325 } else 4326 spin_unlock_bh(&sess->conn_lock); 4327 4328 if (atomic_read(&sess->sleep_on_sess_wait_comp)) 4329 complete(&sess->session_wait_comp); 4330 4331 return 0; 4332 } 4333 } 4334 4335 /* 4336 * If the iSCSI Session for the iSCSI Initiator Node exists, 4337 * forcefully shutdown the iSCSI NEXUS. 4338 */ 4339 int iscsit_close_session(struct iscsi_session *sess) 4340 { 4341 struct iscsi_portal_group *tpg = sess->tpg; 4342 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; 4343 4344 if (atomic_read(&sess->nconn)) { 4345 pr_err("%d connection(s) still exist for iSCSI session" 4346 " to %s\n", atomic_read(&sess->nconn), 4347 sess->sess_ops->InitiatorName); 4348 BUG(); 4349 } 4350 4351 spin_lock_bh(&se_tpg->session_lock); 4352 atomic_set(&sess->session_logout, 1); 4353 atomic_set(&sess->session_reinstatement, 1); 4354 iscsit_stop_time2retain_timer(sess); 4355 spin_unlock_bh(&se_tpg->session_lock); 4356 4357 /* 4358 * transport_deregister_session_configfs() will clear the 4359 * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context 4360 * can be setting it again with __transport_register_session() in 4361 * iscsi_post_login_handler() again after the iscsit_stop_session() 4362 * completes in iscsi_np context. 4363 */ 4364 transport_deregister_session_configfs(sess->se_sess); 4365 4366 /* 4367 * If any other processes are accessing this session pointer we must 4368 * wait until they have completed. If we are in an interrupt (the 4369 * time2retain handler) and contain and active session usage count we 4370 * restart the timer and exit. 4371 */ 4372 if (!in_interrupt()) { 4373 if (iscsit_check_session_usage_count(sess) == 1) 4374 iscsit_stop_session(sess, 1, 1); 4375 } else { 4376 if (iscsit_check_session_usage_count(sess) == 2) { 4377 atomic_set(&sess->session_logout, 0); 4378 iscsit_start_time2retain_handler(sess); 4379 return 0; 4380 } 4381 } 4382 4383 transport_deregister_session(sess->se_sess); 4384 4385 if (sess->sess_ops->ErrorRecoveryLevel == 2) 4386 iscsit_free_connection_recovery_entries(sess); 4387 4388 iscsit_free_all_ooo_cmdsns(sess); 4389 4390 spin_lock_bh(&se_tpg->session_lock); 4391 pr_debug("Moving to TARG_SESS_STATE_FREE.\n"); 4392 sess->session_state = TARG_SESS_STATE_FREE; 4393 pr_debug("Released iSCSI session from node: %s\n", 4394 sess->sess_ops->InitiatorName); 4395 tpg->nsessions--; 4396 if (tpg->tpg_tiqn) 4397 tpg->tpg_tiqn->tiqn_nsessions--; 4398 4399 pr_debug("Decremented number of active iSCSI Sessions on" 4400 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions); 4401 4402 ida_free(&sess_ida, sess->session_index); 4403 kfree(sess->sess_ops); 4404 sess->sess_ops = NULL; 4405 spin_unlock_bh(&se_tpg->session_lock); 4406 4407 kfree(sess); 4408 return 0; 4409 } 4410 4411 static void iscsit_logout_post_handler_closesession( 4412 struct iscsi_conn *conn) 4413 { 4414 struct iscsi_session *sess = conn->sess; 4415 int sleep = 1; 4416 /* 4417 * Traditional iscsi/tcp will invoke this logic from TX thread 4418 * context during session logout, so clear tx_thread_active and 4419 * sleep if iscsit_close_connection() has not already occured. 4420 * 4421 * Since iser-target invokes this logic from it's own workqueue, 4422 * always sleep waiting for RX/TX thread shutdown to complete 4423 * within iscsit_close_connection(). 4424 */ 4425 if (!conn->conn_transport->rdma_shutdown) { 4426 sleep = cmpxchg(&conn->tx_thread_active, true, false); 4427 if (!sleep) 4428 return; 4429 } 4430 4431 atomic_set(&conn->conn_logout_remove, 0); 4432 complete(&conn->conn_logout_comp); 4433 4434 iscsit_dec_conn_usage_count(conn); 4435 iscsit_stop_session(sess, sleep, sleep); 4436 iscsit_dec_session_usage_count(sess); 4437 iscsit_close_session(sess); 4438 } 4439 4440 static void iscsit_logout_post_handler_samecid( 4441 struct iscsi_conn *conn) 4442 { 4443 int sleep = 1; 4444 4445 if (!conn->conn_transport->rdma_shutdown) { 4446 sleep = cmpxchg(&conn->tx_thread_active, true, false); 4447 if (!sleep) 4448 return; 4449 } 4450 4451 atomic_set(&conn->conn_logout_remove, 0); 4452 complete(&conn->conn_logout_comp); 4453 4454 iscsit_cause_connection_reinstatement(conn, sleep); 4455 iscsit_dec_conn_usage_count(conn); 4456 } 4457 4458 static void iscsit_logout_post_handler_diffcid( 4459 struct iscsi_conn *conn, 4460 u16 cid) 4461 { 4462 struct iscsi_conn *l_conn; 4463 struct iscsi_session *sess = conn->sess; 4464 bool conn_found = false; 4465 4466 if (!sess) 4467 return; 4468 4469 spin_lock_bh(&sess->conn_lock); 4470 list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) { 4471 if (l_conn->cid == cid) { 4472 iscsit_inc_conn_usage_count(l_conn); 4473 conn_found = true; 4474 break; 4475 } 4476 } 4477 spin_unlock_bh(&sess->conn_lock); 4478 4479 if (!conn_found) 4480 return; 4481 4482 if (l_conn->sock) 4483 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN); 4484 4485 spin_lock_bh(&l_conn->state_lock); 4486 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 4487 l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT; 4488 spin_unlock_bh(&l_conn->state_lock); 4489 4490 iscsit_cause_connection_reinstatement(l_conn, 1); 4491 iscsit_dec_conn_usage_count(l_conn); 4492 } 4493 4494 /* 4495 * Return of 0 causes the TX thread to restart. 4496 */ 4497 int iscsit_logout_post_handler( 4498 struct iscsi_cmd *cmd, 4499 struct iscsi_conn *conn) 4500 { 4501 int ret = 0; 4502 4503 switch (cmd->logout_reason) { 4504 case ISCSI_LOGOUT_REASON_CLOSE_SESSION: 4505 switch (cmd->logout_response) { 4506 case ISCSI_LOGOUT_SUCCESS: 4507 case ISCSI_LOGOUT_CLEANUP_FAILED: 4508 default: 4509 iscsit_logout_post_handler_closesession(conn); 4510 break; 4511 } 4512 ret = 0; 4513 break; 4514 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: 4515 if (conn->cid == cmd->logout_cid) { 4516 switch (cmd->logout_response) { 4517 case ISCSI_LOGOUT_SUCCESS: 4518 case ISCSI_LOGOUT_CLEANUP_FAILED: 4519 default: 4520 iscsit_logout_post_handler_samecid(conn); 4521 break; 4522 } 4523 ret = 0; 4524 } else { 4525 switch (cmd->logout_response) { 4526 case ISCSI_LOGOUT_SUCCESS: 4527 iscsit_logout_post_handler_diffcid(conn, 4528 cmd->logout_cid); 4529 break; 4530 case ISCSI_LOGOUT_CID_NOT_FOUND: 4531 case ISCSI_LOGOUT_CLEANUP_FAILED: 4532 default: 4533 break; 4534 } 4535 ret = 1; 4536 } 4537 break; 4538 case ISCSI_LOGOUT_REASON_RECOVERY: 4539 switch (cmd->logout_response) { 4540 case ISCSI_LOGOUT_SUCCESS: 4541 case ISCSI_LOGOUT_CID_NOT_FOUND: 4542 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED: 4543 case ISCSI_LOGOUT_CLEANUP_FAILED: 4544 default: 4545 break; 4546 } 4547 ret = 1; 4548 break; 4549 default: 4550 break; 4551 4552 } 4553 return ret; 4554 } 4555 EXPORT_SYMBOL(iscsit_logout_post_handler); 4556 4557 void iscsit_fail_session(struct iscsi_session *sess) 4558 { 4559 struct iscsi_conn *conn; 4560 4561 spin_lock_bh(&sess->conn_lock); 4562 list_for_each_entry(conn, &sess->sess_conn_list, conn_list) { 4563 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n"); 4564 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT; 4565 } 4566 spin_unlock_bh(&sess->conn_lock); 4567 4568 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); 4569 sess->session_state = TARG_SESS_STATE_FAILED; 4570 } 4571 4572 int iscsit_free_session(struct iscsi_session *sess) 4573 { 4574 u16 conn_count = atomic_read(&sess->nconn); 4575 struct iscsi_conn *conn, *conn_tmp = NULL; 4576 int is_last; 4577 4578 spin_lock_bh(&sess->conn_lock); 4579 atomic_set(&sess->sleep_on_sess_wait_comp, 1); 4580 4581 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list, 4582 conn_list) { 4583 if (conn_count == 0) 4584 break; 4585 4586 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) { 4587 is_last = 1; 4588 } else { 4589 iscsit_inc_conn_usage_count(conn_tmp); 4590 is_last = 0; 4591 } 4592 iscsit_inc_conn_usage_count(conn); 4593 4594 spin_unlock_bh(&sess->conn_lock); 4595 iscsit_cause_connection_reinstatement(conn, 1); 4596 spin_lock_bh(&sess->conn_lock); 4597 4598 iscsit_dec_conn_usage_count(conn); 4599 if (is_last == 0) 4600 iscsit_dec_conn_usage_count(conn_tmp); 4601 4602 conn_count--; 4603 } 4604 4605 if (atomic_read(&sess->nconn)) { 4606 spin_unlock_bh(&sess->conn_lock); 4607 wait_for_completion(&sess->session_wait_comp); 4608 } else 4609 spin_unlock_bh(&sess->conn_lock); 4610 4611 iscsit_close_session(sess); 4612 return 0; 4613 } 4614 4615 void iscsit_stop_session( 4616 struct iscsi_session *sess, 4617 int session_sleep, 4618 int connection_sleep) 4619 { 4620 u16 conn_count = atomic_read(&sess->nconn); 4621 struct iscsi_conn *conn, *conn_tmp = NULL; 4622 int is_last; 4623 4624 spin_lock_bh(&sess->conn_lock); 4625 if (session_sleep) 4626 atomic_set(&sess->sleep_on_sess_wait_comp, 1); 4627 4628 if (connection_sleep) { 4629 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list, 4630 conn_list) { 4631 if (conn_count == 0) 4632 break; 4633 4634 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) { 4635 is_last = 1; 4636 } else { 4637 iscsit_inc_conn_usage_count(conn_tmp); 4638 is_last = 0; 4639 } 4640 iscsit_inc_conn_usage_count(conn); 4641 4642 spin_unlock_bh(&sess->conn_lock); 4643 iscsit_cause_connection_reinstatement(conn, 1); 4644 spin_lock_bh(&sess->conn_lock); 4645 4646 iscsit_dec_conn_usage_count(conn); 4647 if (is_last == 0) 4648 iscsit_dec_conn_usage_count(conn_tmp); 4649 conn_count--; 4650 } 4651 } else { 4652 list_for_each_entry(conn, &sess->sess_conn_list, conn_list) 4653 iscsit_cause_connection_reinstatement(conn, 0); 4654 } 4655 4656 if (session_sleep && atomic_read(&sess->nconn)) { 4657 spin_unlock_bh(&sess->conn_lock); 4658 wait_for_completion(&sess->session_wait_comp); 4659 } else 4660 spin_unlock_bh(&sess->conn_lock); 4661 } 4662 4663 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force) 4664 { 4665 struct iscsi_session *sess; 4666 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; 4667 struct se_session *se_sess, *se_sess_tmp; 4668 LIST_HEAD(free_list); 4669 int session_count = 0; 4670 4671 spin_lock_bh(&se_tpg->session_lock); 4672 if (tpg->nsessions && !force) { 4673 spin_unlock_bh(&se_tpg->session_lock); 4674 return -1; 4675 } 4676 4677 list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list, 4678 sess_list) { 4679 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr; 4680 4681 spin_lock(&sess->conn_lock); 4682 if (atomic_read(&sess->session_fall_back_to_erl0) || 4683 atomic_read(&sess->session_logout) || 4684 (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) { 4685 spin_unlock(&sess->conn_lock); 4686 continue; 4687 } 4688 atomic_set(&sess->session_reinstatement, 1); 4689 atomic_set(&sess->session_fall_back_to_erl0, 1); 4690 spin_unlock(&sess->conn_lock); 4691 4692 list_move_tail(&se_sess->sess_list, &free_list); 4693 } 4694 spin_unlock_bh(&se_tpg->session_lock); 4695 4696 list_for_each_entry_safe(se_sess, se_sess_tmp, &free_list, sess_list) { 4697 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr; 4698 4699 iscsit_free_session(sess); 4700 session_count++; 4701 } 4702 4703 pr_debug("Released %d iSCSI Session(s) from Target Portal" 4704 " Group: %hu\n", session_count, tpg->tpgt); 4705 return 0; 4706 } 4707 4708 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure"); 4709 MODULE_VERSION("4.1.x"); 4710 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 4711 MODULE_LICENSE("GPL"); 4712 4713 module_init(iscsi_target_init_module); 4714 module_exit(iscsi_target_cleanup_module); 4715