1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /******************************************************************************* 3 * This file contains main functions related to the iSCSI Target Core Driver. 4 * 5 * (c) Copyright 2007-2013 Datera, Inc. 6 * 7 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org> 8 * 9 ******************************************************************************/ 10 11 #include <crypto/hash.h> 12 #include <linux/string.h> 13 #include <linux/kthread.h> 14 #include <linux/completion.h> 15 #include <linux/module.h> 16 #include <linux/vmalloc.h> 17 #include <linux/idr.h> 18 #include <linux/delay.h> 19 #include <linux/sched/signal.h> 20 #include <asm/unaligned.h> 21 #include <linux/inet.h> 22 #include <net/ipv6.h> 23 #include <scsi/scsi_proto.h> 24 #include <scsi/iscsi_proto.h> 25 #include <scsi/scsi_tcq.h> 26 #include <target/target_core_base.h> 27 #include <target/target_core_fabric.h> 28 29 #include <target/iscsi/iscsi_target_core.h> 30 #include "iscsi_target_parameters.h" 31 #include "iscsi_target_seq_pdu_list.h" 32 #include "iscsi_target_datain_values.h" 33 #include "iscsi_target_erl0.h" 34 #include "iscsi_target_erl1.h" 35 #include "iscsi_target_erl2.h" 36 #include "iscsi_target_login.h" 37 #include "iscsi_target_tmr.h" 38 #include "iscsi_target_tpg.h" 39 #include "iscsi_target_util.h" 40 #include "iscsi_target.h" 41 #include "iscsi_target_device.h" 42 #include <target/iscsi/iscsi_target_stat.h> 43 44 #include <target/iscsi/iscsi_transport.h> 45 46 static LIST_HEAD(g_tiqn_list); 47 static LIST_HEAD(g_np_list); 48 static DEFINE_SPINLOCK(tiqn_lock); 49 static DEFINE_MUTEX(np_lock); 50 51 static struct idr tiqn_idr; 52 DEFINE_IDA(sess_ida); 53 struct mutex auth_id_lock; 54 55 struct iscsit_global *iscsit_global; 56 57 struct kmem_cache *lio_qr_cache; 58 struct kmem_cache *lio_dr_cache; 59 struct kmem_cache *lio_ooo_cache; 60 struct kmem_cache *lio_r2t_cache; 61 62 static int iscsit_handle_immediate_data(struct iscsi_cmd *, 63 struct iscsi_scsi_req *, u32); 64 65 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf) 66 { 67 struct iscsi_tiqn *tiqn = NULL; 68 69 spin_lock(&tiqn_lock); 70 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) { 71 if (!strcmp(tiqn->tiqn, buf)) { 72 73 spin_lock(&tiqn->tiqn_state_lock); 74 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) { 75 tiqn->tiqn_access_count++; 76 spin_unlock(&tiqn->tiqn_state_lock); 77 spin_unlock(&tiqn_lock); 78 return tiqn; 79 } 80 spin_unlock(&tiqn->tiqn_state_lock); 81 } 82 } 83 spin_unlock(&tiqn_lock); 84 85 return NULL; 86 } 87 88 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn) 89 { 90 spin_lock(&tiqn->tiqn_state_lock); 91 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) { 92 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN; 93 spin_unlock(&tiqn->tiqn_state_lock); 94 return 0; 95 } 96 spin_unlock(&tiqn->tiqn_state_lock); 97 98 return -1; 99 } 100 101 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn) 102 { 103 spin_lock(&tiqn->tiqn_state_lock); 104 tiqn->tiqn_access_count--; 105 spin_unlock(&tiqn->tiqn_state_lock); 106 } 107 108 /* 109 * Note that IQN formatting is expected to be done in userspace, and 110 * no explict IQN format checks are done here. 111 */ 112 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf) 113 { 114 struct iscsi_tiqn *tiqn = NULL; 115 int ret; 116 117 if (strlen(buf) >= ISCSI_IQN_LEN) { 118 pr_err("Target IQN exceeds %d bytes\n", 119 ISCSI_IQN_LEN); 120 return ERR_PTR(-EINVAL); 121 } 122 123 tiqn = kzalloc(sizeof(*tiqn), GFP_KERNEL); 124 if (!tiqn) 125 return ERR_PTR(-ENOMEM); 126 127 sprintf(tiqn->tiqn, "%s", buf); 128 INIT_LIST_HEAD(&tiqn->tiqn_list); 129 INIT_LIST_HEAD(&tiqn->tiqn_tpg_list); 130 spin_lock_init(&tiqn->tiqn_state_lock); 131 spin_lock_init(&tiqn->tiqn_tpg_lock); 132 spin_lock_init(&tiqn->sess_err_stats.lock); 133 spin_lock_init(&tiqn->login_stats.lock); 134 spin_lock_init(&tiqn->logout_stats.lock); 135 136 tiqn->tiqn_state = TIQN_STATE_ACTIVE; 137 138 idr_preload(GFP_KERNEL); 139 spin_lock(&tiqn_lock); 140 141 ret = idr_alloc(&tiqn_idr, NULL, 0, 0, GFP_NOWAIT); 142 if (ret < 0) { 143 pr_err("idr_alloc() failed for tiqn->tiqn_index\n"); 144 spin_unlock(&tiqn_lock); 145 idr_preload_end(); 146 kfree(tiqn); 147 return ERR_PTR(ret); 148 } 149 tiqn->tiqn_index = ret; 150 list_add_tail(&tiqn->tiqn_list, &g_tiqn_list); 151 152 spin_unlock(&tiqn_lock); 153 idr_preload_end(); 154 155 pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn); 156 157 return tiqn; 158 159 } 160 161 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn) 162 { 163 /* 164 * Wait for accesses to said struct iscsi_tiqn to end. 165 */ 166 spin_lock(&tiqn->tiqn_state_lock); 167 while (tiqn->tiqn_access_count != 0) { 168 spin_unlock(&tiqn->tiqn_state_lock); 169 msleep(10); 170 spin_lock(&tiqn->tiqn_state_lock); 171 } 172 spin_unlock(&tiqn->tiqn_state_lock); 173 } 174 175 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn) 176 { 177 /* 178 * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN 179 * while holding tiqn->tiqn_state_lock. This means that all subsequent 180 * attempts to access this struct iscsi_tiqn will fail from both transport 181 * fabric and control code paths. 182 */ 183 if (iscsit_set_tiqn_shutdown(tiqn) < 0) { 184 pr_err("iscsit_set_tiqn_shutdown() failed\n"); 185 return; 186 } 187 188 iscsit_wait_for_tiqn(tiqn); 189 190 spin_lock(&tiqn_lock); 191 list_del(&tiqn->tiqn_list); 192 idr_remove(&tiqn_idr, tiqn->tiqn_index); 193 spin_unlock(&tiqn_lock); 194 195 pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n", 196 tiqn->tiqn); 197 kfree(tiqn); 198 } 199 200 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg) 201 { 202 int ret; 203 /* 204 * Determine if the network portal is accepting storage traffic. 205 */ 206 spin_lock_bh(&np->np_thread_lock); 207 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { 208 spin_unlock_bh(&np->np_thread_lock); 209 return -1; 210 } 211 spin_unlock_bh(&np->np_thread_lock); 212 /* 213 * Determine if the portal group is accepting storage traffic. 214 */ 215 spin_lock_bh(&tpg->tpg_state_lock); 216 if (tpg->tpg_state != TPG_STATE_ACTIVE) { 217 spin_unlock_bh(&tpg->tpg_state_lock); 218 return -1; 219 } 220 spin_unlock_bh(&tpg->tpg_state_lock); 221 222 /* 223 * Here we serialize access across the TIQN+TPG Tuple. 224 */ 225 ret = down_interruptible(&tpg->np_login_sem); 226 if (ret != 0) 227 return -1; 228 229 spin_lock_bh(&tpg->tpg_state_lock); 230 if (tpg->tpg_state != TPG_STATE_ACTIVE) { 231 spin_unlock_bh(&tpg->tpg_state_lock); 232 up(&tpg->np_login_sem); 233 return -1; 234 } 235 spin_unlock_bh(&tpg->tpg_state_lock); 236 237 return 0; 238 } 239 240 void iscsit_login_kref_put(struct kref *kref) 241 { 242 struct iscsi_tpg_np *tpg_np = container_of(kref, 243 struct iscsi_tpg_np, tpg_np_kref); 244 245 complete(&tpg_np->tpg_np_comp); 246 } 247 248 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg, 249 struct iscsi_tpg_np *tpg_np) 250 { 251 struct iscsi_tiqn *tiqn = tpg->tpg_tiqn; 252 253 up(&tpg->np_login_sem); 254 255 if (tpg_np) 256 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put); 257 258 if (tiqn) 259 iscsit_put_tiqn_for_login(tiqn); 260 261 return 0; 262 } 263 264 bool iscsit_check_np_match( 265 struct sockaddr_storage *sockaddr, 266 struct iscsi_np *np, 267 int network_transport) 268 { 269 struct sockaddr_in *sock_in, *sock_in_e; 270 struct sockaddr_in6 *sock_in6, *sock_in6_e; 271 bool ip_match = false; 272 u16 port, port_e; 273 274 if (sockaddr->ss_family == AF_INET6) { 275 sock_in6 = (struct sockaddr_in6 *)sockaddr; 276 sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr; 277 278 if (!memcmp(&sock_in6->sin6_addr.in6_u, 279 &sock_in6_e->sin6_addr.in6_u, 280 sizeof(struct in6_addr))) 281 ip_match = true; 282 283 port = ntohs(sock_in6->sin6_port); 284 port_e = ntohs(sock_in6_e->sin6_port); 285 } else { 286 sock_in = (struct sockaddr_in *)sockaddr; 287 sock_in_e = (struct sockaddr_in *)&np->np_sockaddr; 288 289 if (sock_in->sin_addr.s_addr == sock_in_e->sin_addr.s_addr) 290 ip_match = true; 291 292 port = ntohs(sock_in->sin_port); 293 port_e = ntohs(sock_in_e->sin_port); 294 } 295 296 if (ip_match && (port_e == port) && 297 (np->np_network_transport == network_transport)) 298 return true; 299 300 return false; 301 } 302 303 static struct iscsi_np *iscsit_get_np( 304 struct sockaddr_storage *sockaddr, 305 int network_transport) 306 { 307 struct iscsi_np *np; 308 bool match; 309 310 lockdep_assert_held(&np_lock); 311 312 list_for_each_entry(np, &g_np_list, np_list) { 313 spin_lock_bh(&np->np_thread_lock); 314 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { 315 spin_unlock_bh(&np->np_thread_lock); 316 continue; 317 } 318 319 match = iscsit_check_np_match(sockaddr, np, network_transport); 320 if (match) { 321 /* 322 * Increment the np_exports reference count now to 323 * prevent iscsit_del_np() below from being called 324 * while iscsi_tpg_add_network_portal() is called. 325 */ 326 np->np_exports++; 327 spin_unlock_bh(&np->np_thread_lock); 328 return np; 329 } 330 spin_unlock_bh(&np->np_thread_lock); 331 } 332 333 return NULL; 334 } 335 336 struct iscsi_np *iscsit_add_np( 337 struct sockaddr_storage *sockaddr, 338 int network_transport) 339 { 340 struct iscsi_np *np; 341 int ret; 342 343 mutex_lock(&np_lock); 344 345 /* 346 * Locate the existing struct iscsi_np if already active.. 347 */ 348 np = iscsit_get_np(sockaddr, network_transport); 349 if (np) { 350 mutex_unlock(&np_lock); 351 return np; 352 } 353 354 np = kzalloc(sizeof(*np), GFP_KERNEL); 355 if (!np) { 356 mutex_unlock(&np_lock); 357 return ERR_PTR(-ENOMEM); 358 } 359 360 np->np_flags |= NPF_IP_NETWORK; 361 np->np_network_transport = network_transport; 362 spin_lock_init(&np->np_thread_lock); 363 init_completion(&np->np_restart_comp); 364 INIT_LIST_HEAD(&np->np_list); 365 366 timer_setup(&np->np_login_timer, iscsi_handle_login_thread_timeout, 0); 367 368 ret = iscsi_target_setup_login_socket(np, sockaddr); 369 if (ret != 0) { 370 kfree(np); 371 mutex_unlock(&np_lock); 372 return ERR_PTR(ret); 373 } 374 375 np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np"); 376 if (IS_ERR(np->np_thread)) { 377 pr_err("Unable to create kthread: iscsi_np\n"); 378 ret = PTR_ERR(np->np_thread); 379 kfree(np); 380 mutex_unlock(&np_lock); 381 return ERR_PTR(ret); 382 } 383 /* 384 * Increment the np_exports reference count now to prevent 385 * iscsit_del_np() below from being run while a new call to 386 * iscsi_tpg_add_network_portal() for a matching iscsi_np is 387 * active. We don't need to hold np->np_thread_lock at this 388 * point because iscsi_np has not been added to g_np_list yet. 389 */ 390 np->np_exports = 1; 391 np->np_thread_state = ISCSI_NP_THREAD_ACTIVE; 392 393 list_add_tail(&np->np_list, &g_np_list); 394 mutex_unlock(&np_lock); 395 396 pr_debug("CORE[0] - Added Network Portal: %pISpc on %s\n", 397 &np->np_sockaddr, np->np_transport->name); 398 399 return np; 400 } 401 402 int iscsit_reset_np_thread( 403 struct iscsi_np *np, 404 struct iscsi_tpg_np *tpg_np, 405 struct iscsi_portal_group *tpg, 406 bool shutdown) 407 { 408 spin_lock_bh(&np->np_thread_lock); 409 if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) { 410 spin_unlock_bh(&np->np_thread_lock); 411 return 0; 412 } 413 np->np_thread_state = ISCSI_NP_THREAD_RESET; 414 atomic_inc(&np->np_reset_count); 415 416 if (np->np_thread) { 417 spin_unlock_bh(&np->np_thread_lock); 418 send_sig(SIGINT, np->np_thread, 1); 419 wait_for_completion(&np->np_restart_comp); 420 spin_lock_bh(&np->np_thread_lock); 421 } 422 spin_unlock_bh(&np->np_thread_lock); 423 424 if (tpg_np && shutdown) { 425 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put); 426 427 wait_for_completion(&tpg_np->tpg_np_comp); 428 } 429 430 return 0; 431 } 432 433 static void iscsit_free_np(struct iscsi_np *np) 434 { 435 if (np->np_socket) 436 sock_release(np->np_socket); 437 } 438 439 int iscsit_del_np(struct iscsi_np *np) 440 { 441 spin_lock_bh(&np->np_thread_lock); 442 np->np_exports--; 443 if (np->np_exports) { 444 np->enabled = true; 445 spin_unlock_bh(&np->np_thread_lock); 446 return 0; 447 } 448 np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN; 449 spin_unlock_bh(&np->np_thread_lock); 450 451 if (np->np_thread) { 452 /* 453 * We need to send the signal to wakeup Linux/Net 454 * which may be sleeping in sock_accept().. 455 */ 456 send_sig(SIGINT, np->np_thread, 1); 457 kthread_stop(np->np_thread); 458 np->np_thread = NULL; 459 } 460 461 np->np_transport->iscsit_free_np(np); 462 463 mutex_lock(&np_lock); 464 list_del(&np->np_list); 465 mutex_unlock(&np_lock); 466 467 pr_debug("CORE[0] - Removed Network Portal: %pISpc on %s\n", 468 &np->np_sockaddr, np->np_transport->name); 469 470 iscsit_put_transport(np->np_transport); 471 kfree(np); 472 return 0; 473 } 474 475 static void iscsit_get_rx_pdu(struct iscsi_conn *); 476 477 int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 478 { 479 return iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state); 480 } 481 EXPORT_SYMBOL(iscsit_queue_rsp); 482 483 void iscsit_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 484 { 485 spin_lock_bh(&conn->cmd_lock); 486 if (!list_empty(&cmd->i_conn_node)) 487 list_del_init(&cmd->i_conn_node); 488 spin_unlock_bh(&conn->cmd_lock); 489 490 __iscsit_free_cmd(cmd, true); 491 } 492 EXPORT_SYMBOL(iscsit_aborted_task); 493 494 static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *, 495 u32, u32, const void *, void *); 496 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *); 497 498 static int 499 iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 500 const void *data_buf, u32 data_buf_len) 501 { 502 struct iscsi_hdr *hdr = (struct iscsi_hdr *)cmd->pdu; 503 struct kvec *iov; 504 u32 niov = 0, tx_size = ISCSI_HDR_LEN; 505 int ret; 506 507 iov = &cmd->iov_misc[0]; 508 iov[niov].iov_base = cmd->pdu; 509 iov[niov++].iov_len = ISCSI_HDR_LEN; 510 511 if (conn->conn_ops->HeaderDigest) { 512 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 513 514 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, 515 ISCSI_HDR_LEN, 0, NULL, 516 header_digest); 517 518 iov[0].iov_len += ISCSI_CRC_LEN; 519 tx_size += ISCSI_CRC_LEN; 520 pr_debug("Attaching CRC32C HeaderDigest" 521 " to opcode 0x%x 0x%08x\n", 522 hdr->opcode, *header_digest); 523 } 524 525 if (data_buf_len) { 526 u32 padding = ((-data_buf_len) & 3); 527 528 iov[niov].iov_base = (void *)data_buf; 529 iov[niov++].iov_len = data_buf_len; 530 tx_size += data_buf_len; 531 532 if (padding != 0) { 533 iov[niov].iov_base = &cmd->pad_bytes; 534 iov[niov++].iov_len = padding; 535 tx_size += padding; 536 pr_debug("Attaching %u additional" 537 " padding bytes.\n", padding); 538 } 539 540 if (conn->conn_ops->DataDigest) { 541 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, 542 data_buf, data_buf_len, 543 padding, &cmd->pad_bytes, 544 &cmd->data_crc); 545 546 iov[niov].iov_base = &cmd->data_crc; 547 iov[niov++].iov_len = ISCSI_CRC_LEN; 548 tx_size += ISCSI_CRC_LEN; 549 pr_debug("Attached DataDigest for %u" 550 " bytes opcode 0x%x, CRC 0x%08x\n", 551 data_buf_len, hdr->opcode, cmd->data_crc); 552 } 553 } 554 555 cmd->iov_misc_count = niov; 556 cmd->tx_size = tx_size; 557 558 ret = iscsit_send_tx_data(cmd, conn, 1); 559 if (ret < 0) { 560 iscsit_tx_thread_wait_for_tcp(conn); 561 return ret; 562 } 563 564 return 0; 565 } 566 567 static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec, 568 u32 data_offset, u32 data_length); 569 static void iscsit_unmap_iovec(struct iscsi_cmd *); 570 static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsi_cmd *, 571 u32, u32, u32, u8 *); 572 static int 573 iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 574 const struct iscsi_datain *datain) 575 { 576 struct kvec *iov; 577 u32 iov_count = 0, tx_size = 0; 578 int ret, iov_ret; 579 580 iov = &cmd->iov_data[0]; 581 iov[iov_count].iov_base = cmd->pdu; 582 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 583 tx_size += ISCSI_HDR_LEN; 584 585 if (conn->conn_ops->HeaderDigest) { 586 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 587 588 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu, 589 ISCSI_HDR_LEN, 0, NULL, 590 header_digest); 591 592 iov[0].iov_len += ISCSI_CRC_LEN; 593 tx_size += ISCSI_CRC_LEN; 594 595 pr_debug("Attaching CRC32 HeaderDigest for DataIN PDU 0x%08x\n", 596 *header_digest); 597 } 598 599 iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[iov_count], 600 cmd->orig_iov_data_count - (iov_count + 2), 601 datain->offset, datain->length); 602 if (iov_ret < 0) 603 return -1; 604 605 iov_count += iov_ret; 606 tx_size += datain->length; 607 608 cmd->padding = ((-datain->length) & 3); 609 if (cmd->padding) { 610 iov[iov_count].iov_base = cmd->pad_bytes; 611 iov[iov_count++].iov_len = cmd->padding; 612 tx_size += cmd->padding; 613 614 pr_debug("Attaching %u padding bytes\n", cmd->padding); 615 } 616 617 if (conn->conn_ops->DataDigest) { 618 cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash, 619 cmd, datain->offset, 620 datain->length, 621 cmd->padding, 622 cmd->pad_bytes); 623 624 iov[iov_count].iov_base = &cmd->data_crc; 625 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 626 tx_size += ISCSI_CRC_LEN; 627 628 pr_debug("Attached CRC32C DataDigest %d bytes, crc 0x%08x\n", 629 datain->length + cmd->padding, cmd->data_crc); 630 } 631 632 cmd->iov_data_count = iov_count; 633 cmd->tx_size = tx_size; 634 635 ret = iscsit_fe_sendpage_sg(cmd, conn); 636 637 iscsit_unmap_iovec(cmd); 638 639 if (ret < 0) { 640 iscsit_tx_thread_wait_for_tcp(conn); 641 return ret; 642 } 643 644 return 0; 645 } 646 647 static int iscsit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 648 struct iscsi_datain_req *dr, const void *buf, 649 u32 buf_len) 650 { 651 if (dr) 652 return iscsit_xmit_datain_pdu(conn, cmd, buf); 653 else 654 return iscsit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 655 } 656 657 static enum target_prot_op iscsit_get_sup_prot_ops(struct iscsi_conn *conn) 658 { 659 return TARGET_PROT_NORMAL; 660 } 661 662 static struct iscsit_transport iscsi_target_transport = { 663 .name = "iSCSI/TCP", 664 .transport_type = ISCSI_TCP, 665 .rdma_shutdown = false, 666 .owner = NULL, 667 .iscsit_setup_np = iscsit_setup_np, 668 .iscsit_accept_np = iscsit_accept_np, 669 .iscsit_free_np = iscsit_free_np, 670 .iscsit_get_login_rx = iscsit_get_login_rx, 671 .iscsit_put_login_tx = iscsit_put_login_tx, 672 .iscsit_get_dataout = iscsit_build_r2ts_for_cmd, 673 .iscsit_immediate_queue = iscsit_immediate_queue, 674 .iscsit_response_queue = iscsit_response_queue, 675 .iscsit_queue_data_in = iscsit_queue_rsp, 676 .iscsit_queue_status = iscsit_queue_rsp, 677 .iscsit_aborted_task = iscsit_aborted_task, 678 .iscsit_xmit_pdu = iscsit_xmit_pdu, 679 .iscsit_get_rx_pdu = iscsit_get_rx_pdu, 680 .iscsit_get_sup_prot_ops = iscsit_get_sup_prot_ops, 681 }; 682 683 static int __init iscsi_target_init_module(void) 684 { 685 int ret = 0, size; 686 687 pr_debug("iSCSI-Target "ISCSIT_VERSION"\n"); 688 iscsit_global = kzalloc(sizeof(*iscsit_global), GFP_KERNEL); 689 if (!iscsit_global) 690 return -1; 691 692 spin_lock_init(&iscsit_global->ts_bitmap_lock); 693 mutex_init(&auth_id_lock); 694 idr_init(&tiqn_idr); 695 696 ret = target_register_template(&iscsi_ops); 697 if (ret) 698 goto out; 699 700 size = BITS_TO_LONGS(ISCSIT_BITMAP_BITS) * sizeof(long); 701 iscsit_global->ts_bitmap = vzalloc(size); 702 if (!iscsit_global->ts_bitmap) 703 goto configfs_out; 704 705 lio_qr_cache = kmem_cache_create("lio_qr_cache", 706 sizeof(struct iscsi_queue_req), 707 __alignof__(struct iscsi_queue_req), 0, NULL); 708 if (!lio_qr_cache) { 709 pr_err("Unable to kmem_cache_create() for" 710 " lio_qr_cache\n"); 711 goto bitmap_out; 712 } 713 714 lio_dr_cache = kmem_cache_create("lio_dr_cache", 715 sizeof(struct iscsi_datain_req), 716 __alignof__(struct iscsi_datain_req), 0, NULL); 717 if (!lio_dr_cache) { 718 pr_err("Unable to kmem_cache_create() for" 719 " lio_dr_cache\n"); 720 goto qr_out; 721 } 722 723 lio_ooo_cache = kmem_cache_create("lio_ooo_cache", 724 sizeof(struct iscsi_ooo_cmdsn), 725 __alignof__(struct iscsi_ooo_cmdsn), 0, NULL); 726 if (!lio_ooo_cache) { 727 pr_err("Unable to kmem_cache_create() for" 728 " lio_ooo_cache\n"); 729 goto dr_out; 730 } 731 732 lio_r2t_cache = kmem_cache_create("lio_r2t_cache", 733 sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t), 734 0, NULL); 735 if (!lio_r2t_cache) { 736 pr_err("Unable to kmem_cache_create() for" 737 " lio_r2t_cache\n"); 738 goto ooo_out; 739 } 740 741 iscsit_register_transport(&iscsi_target_transport); 742 743 if (iscsit_load_discovery_tpg() < 0) 744 goto r2t_out; 745 746 return ret; 747 r2t_out: 748 iscsit_unregister_transport(&iscsi_target_transport); 749 kmem_cache_destroy(lio_r2t_cache); 750 ooo_out: 751 kmem_cache_destroy(lio_ooo_cache); 752 dr_out: 753 kmem_cache_destroy(lio_dr_cache); 754 qr_out: 755 kmem_cache_destroy(lio_qr_cache); 756 bitmap_out: 757 vfree(iscsit_global->ts_bitmap); 758 configfs_out: 759 /* XXX: this probably wants it to be it's own unwind step.. */ 760 if (iscsit_global->discovery_tpg) 761 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1); 762 target_unregister_template(&iscsi_ops); 763 out: 764 kfree(iscsit_global); 765 return -ENOMEM; 766 } 767 768 static void __exit iscsi_target_cleanup_module(void) 769 { 770 iscsit_release_discovery_tpg(); 771 iscsit_unregister_transport(&iscsi_target_transport); 772 kmem_cache_destroy(lio_qr_cache); 773 kmem_cache_destroy(lio_dr_cache); 774 kmem_cache_destroy(lio_ooo_cache); 775 kmem_cache_destroy(lio_r2t_cache); 776 777 /* 778 * Shutdown discovery sessions and disable discovery TPG 779 */ 780 if (iscsit_global->discovery_tpg) 781 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1); 782 783 target_unregister_template(&iscsi_ops); 784 785 vfree(iscsit_global->ts_bitmap); 786 kfree(iscsit_global); 787 } 788 789 int iscsit_add_reject( 790 struct iscsi_conn *conn, 791 u8 reason, 792 unsigned char *buf) 793 { 794 struct iscsi_cmd *cmd; 795 796 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 797 if (!cmd) 798 return -1; 799 800 cmd->iscsi_opcode = ISCSI_OP_REJECT; 801 cmd->reject_reason = reason; 802 803 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL); 804 if (!cmd->buf_ptr) { 805 pr_err("Unable to allocate memory for cmd->buf_ptr\n"); 806 iscsit_free_cmd(cmd, false); 807 return -1; 808 } 809 810 spin_lock_bh(&conn->cmd_lock); 811 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 812 spin_unlock_bh(&conn->cmd_lock); 813 814 cmd->i_state = ISTATE_SEND_REJECT; 815 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 816 817 return -1; 818 } 819 EXPORT_SYMBOL(iscsit_add_reject); 820 821 static int iscsit_add_reject_from_cmd( 822 struct iscsi_cmd *cmd, 823 u8 reason, 824 bool add_to_conn, 825 unsigned char *buf) 826 { 827 struct iscsi_conn *conn; 828 const bool do_put = cmd->se_cmd.se_tfo != NULL; 829 830 if (!cmd->conn) { 831 pr_err("cmd->conn is NULL for ITT: 0x%08x\n", 832 cmd->init_task_tag); 833 return -1; 834 } 835 conn = cmd->conn; 836 837 cmd->iscsi_opcode = ISCSI_OP_REJECT; 838 cmd->reject_reason = reason; 839 840 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL); 841 if (!cmd->buf_ptr) { 842 pr_err("Unable to allocate memory for cmd->buf_ptr\n"); 843 iscsit_free_cmd(cmd, false); 844 return -1; 845 } 846 847 if (add_to_conn) { 848 spin_lock_bh(&conn->cmd_lock); 849 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 850 spin_unlock_bh(&conn->cmd_lock); 851 } 852 853 cmd->i_state = ISTATE_SEND_REJECT; 854 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 855 /* 856 * Perform the kref_put now if se_cmd has already been setup by 857 * scsit_setup_scsi_cmd() 858 */ 859 if (do_put) { 860 pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n"); 861 target_put_sess_cmd(&cmd->se_cmd); 862 } 863 return -1; 864 } 865 866 static int iscsit_add_reject_cmd(struct iscsi_cmd *cmd, u8 reason, 867 unsigned char *buf) 868 { 869 return iscsit_add_reject_from_cmd(cmd, reason, true, buf); 870 } 871 872 int iscsit_reject_cmd(struct iscsi_cmd *cmd, u8 reason, unsigned char *buf) 873 { 874 return iscsit_add_reject_from_cmd(cmd, reason, false, buf); 875 } 876 EXPORT_SYMBOL(iscsit_reject_cmd); 877 878 /* 879 * Map some portion of the allocated scatterlist to an iovec, suitable for 880 * kernel sockets to copy data in/out. 881 */ 882 static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec, 883 u32 data_offset, u32 data_length) 884 { 885 u32 i = 0, orig_data_length = data_length; 886 struct scatterlist *sg; 887 unsigned int page_off; 888 889 /* 890 * We know each entry in t_data_sg contains a page. 891 */ 892 u32 ent = data_offset / PAGE_SIZE; 893 894 if (!data_length) 895 return 0; 896 897 if (ent >= cmd->se_cmd.t_data_nents) { 898 pr_err("Initial page entry out-of-bounds\n"); 899 goto overflow; 900 } 901 902 sg = &cmd->se_cmd.t_data_sg[ent]; 903 page_off = (data_offset % PAGE_SIZE); 904 905 cmd->first_data_sg = sg; 906 cmd->first_data_sg_off = page_off; 907 908 while (data_length) { 909 u32 cur_len; 910 911 if (WARN_ON_ONCE(!sg || i >= nvec)) 912 goto overflow; 913 914 cur_len = min_t(u32, data_length, sg->length - page_off); 915 916 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off; 917 iov[i].iov_len = cur_len; 918 919 data_length -= cur_len; 920 page_off = 0; 921 sg = sg_next(sg); 922 i++; 923 } 924 925 cmd->kmapped_nents = i; 926 927 return i; 928 929 overflow: 930 pr_err("offset %d + length %d overflow; %d/%d; sg-list:\n", 931 data_offset, orig_data_length, i, nvec); 932 for_each_sg(cmd->se_cmd.t_data_sg, sg, 933 cmd->se_cmd.t_data_nents, i) { 934 pr_err("[%d] off %d len %d\n", 935 i, sg->offset, sg->length); 936 } 937 return -1; 938 } 939 940 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd) 941 { 942 u32 i; 943 struct scatterlist *sg; 944 945 sg = cmd->first_data_sg; 946 947 for (i = 0; i < cmd->kmapped_nents; i++) 948 kunmap(sg_page(&sg[i])); 949 } 950 951 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn) 952 { 953 LIST_HEAD(ack_list); 954 struct iscsi_cmd *cmd, *cmd_p; 955 956 conn->exp_statsn = exp_statsn; 957 958 if (conn->sess->sess_ops->RDMAExtensions) 959 return; 960 961 spin_lock_bh(&conn->cmd_lock); 962 list_for_each_entry_safe(cmd, cmd_p, &conn->conn_cmd_list, i_conn_node) { 963 spin_lock(&cmd->istate_lock); 964 if ((cmd->i_state == ISTATE_SENT_STATUS) && 965 iscsi_sna_lt(cmd->stat_sn, exp_statsn)) { 966 cmd->i_state = ISTATE_REMOVE; 967 spin_unlock(&cmd->istate_lock); 968 list_move_tail(&cmd->i_conn_node, &ack_list); 969 continue; 970 } 971 spin_unlock(&cmd->istate_lock); 972 } 973 spin_unlock_bh(&conn->cmd_lock); 974 975 list_for_each_entry_safe(cmd, cmd_p, &ack_list, i_conn_node) { 976 list_del_init(&cmd->i_conn_node); 977 iscsit_free_cmd(cmd, false); 978 } 979 } 980 981 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd) 982 { 983 u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE)); 984 985 iov_count += ISCSI_IOV_DATA_BUFFER; 986 cmd->iov_data = kcalloc(iov_count, sizeof(*cmd->iov_data), GFP_KERNEL); 987 if (!cmd->iov_data) 988 return -ENOMEM; 989 990 cmd->orig_iov_data_count = iov_count; 991 return 0; 992 } 993 994 int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 995 unsigned char *buf) 996 { 997 int data_direction, payload_length; 998 struct iscsi_scsi_req *hdr; 999 int iscsi_task_attr; 1000 int sam_task_attr; 1001 1002 atomic_long_inc(&conn->sess->cmd_pdus); 1003 1004 hdr = (struct iscsi_scsi_req *) buf; 1005 payload_length = ntoh24(hdr->dlength); 1006 1007 /* FIXME; Add checks for AdditionalHeaderSegment */ 1008 1009 if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) && 1010 !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) { 1011 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL" 1012 " not set. Bad iSCSI Initiator.\n"); 1013 return iscsit_add_reject_cmd(cmd, 1014 ISCSI_REASON_BOOKMARK_INVALID, buf); 1015 } 1016 1017 if (((hdr->flags & ISCSI_FLAG_CMD_READ) || 1018 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) { 1019 /* 1020 * From RFC-3720 Section 10.3.1: 1021 * 1022 * "Either or both of R and W MAY be 1 when either the 1023 * Expected Data Transfer Length and/or Bidirectional Read 1024 * Expected Data Transfer Length are 0" 1025 * 1026 * For this case, go ahead and clear the unnecssary bits 1027 * to avoid any confusion with ->data_direction. 1028 */ 1029 hdr->flags &= ~ISCSI_FLAG_CMD_READ; 1030 hdr->flags &= ~ISCSI_FLAG_CMD_WRITE; 1031 1032 pr_warn("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE" 1033 " set when Expected Data Transfer Length is 0 for" 1034 " CDB: 0x%02x, Fixing up flags\n", hdr->cdb[0]); 1035 } 1036 1037 if (!(hdr->flags & ISCSI_FLAG_CMD_READ) && 1038 !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) { 1039 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE" 1040 " MUST be set if Expected Data Transfer Length is not 0." 1041 " Bad iSCSI Initiator\n"); 1042 return iscsit_add_reject_cmd(cmd, 1043 ISCSI_REASON_BOOKMARK_INVALID, buf); 1044 } 1045 1046 if ((hdr->flags & ISCSI_FLAG_CMD_READ) && 1047 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) { 1048 pr_err("Bidirectional operations not supported!\n"); 1049 return iscsit_add_reject_cmd(cmd, 1050 ISCSI_REASON_BOOKMARK_INVALID, buf); 1051 } 1052 1053 if (hdr->opcode & ISCSI_OP_IMMEDIATE) { 1054 pr_err("Illegally set Immediate Bit in iSCSI Initiator" 1055 " Scsi Command PDU.\n"); 1056 return iscsit_add_reject_cmd(cmd, 1057 ISCSI_REASON_BOOKMARK_INVALID, buf); 1058 } 1059 1060 if (payload_length && !conn->sess->sess_ops->ImmediateData) { 1061 pr_err("ImmediateData=No but DataSegmentLength=%u," 1062 " protocol error.\n", payload_length); 1063 return iscsit_add_reject_cmd(cmd, 1064 ISCSI_REASON_PROTOCOL_ERROR, buf); 1065 } 1066 1067 if ((be32_to_cpu(hdr->data_length) == payload_length) && 1068 (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) { 1069 pr_err("Expected Data Transfer Length and Length of" 1070 " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL" 1071 " bit is not set protocol error\n"); 1072 return iscsit_add_reject_cmd(cmd, 1073 ISCSI_REASON_PROTOCOL_ERROR, buf); 1074 } 1075 1076 if (payload_length > be32_to_cpu(hdr->data_length)) { 1077 pr_err("DataSegmentLength: %u is greater than" 1078 " EDTL: %u, protocol error.\n", payload_length, 1079 hdr->data_length); 1080 return iscsit_add_reject_cmd(cmd, 1081 ISCSI_REASON_PROTOCOL_ERROR, buf); 1082 } 1083 1084 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 1085 pr_err("DataSegmentLength: %u is greater than" 1086 " MaxXmitDataSegmentLength: %u, protocol error.\n", 1087 payload_length, conn->conn_ops->MaxXmitDataSegmentLength); 1088 return iscsit_add_reject_cmd(cmd, 1089 ISCSI_REASON_PROTOCOL_ERROR, buf); 1090 } 1091 1092 if (payload_length > conn->sess->sess_ops->FirstBurstLength) { 1093 pr_err("DataSegmentLength: %u is greater than" 1094 " FirstBurstLength: %u, protocol error.\n", 1095 payload_length, conn->sess->sess_ops->FirstBurstLength); 1096 return iscsit_add_reject_cmd(cmd, 1097 ISCSI_REASON_BOOKMARK_INVALID, buf); 1098 } 1099 1100 data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE : 1101 (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE : 1102 DMA_NONE; 1103 1104 cmd->data_direction = data_direction; 1105 iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK; 1106 /* 1107 * Figure out the SAM Task Attribute for the incoming SCSI CDB 1108 */ 1109 if ((iscsi_task_attr == ISCSI_ATTR_UNTAGGED) || 1110 (iscsi_task_attr == ISCSI_ATTR_SIMPLE)) 1111 sam_task_attr = TCM_SIMPLE_TAG; 1112 else if (iscsi_task_attr == ISCSI_ATTR_ORDERED) 1113 sam_task_attr = TCM_ORDERED_TAG; 1114 else if (iscsi_task_attr == ISCSI_ATTR_HEAD_OF_QUEUE) 1115 sam_task_attr = TCM_HEAD_TAG; 1116 else if (iscsi_task_attr == ISCSI_ATTR_ACA) 1117 sam_task_attr = TCM_ACA_TAG; 1118 else { 1119 pr_debug("Unknown iSCSI Task Attribute: 0x%02x, using" 1120 " TCM_SIMPLE_TAG\n", iscsi_task_attr); 1121 sam_task_attr = TCM_SIMPLE_TAG; 1122 } 1123 1124 cmd->iscsi_opcode = ISCSI_OP_SCSI_CMD; 1125 cmd->i_state = ISTATE_NEW_CMD; 1126 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 1127 cmd->immediate_data = (payload_length) ? 1 : 0; 1128 cmd->unsolicited_data = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) && 1129 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0); 1130 if (cmd->unsolicited_data) 1131 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA; 1132 1133 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 1134 if (hdr->flags & ISCSI_FLAG_CMD_READ) 1135 cmd->targ_xfer_tag = session_get_next_ttt(conn->sess); 1136 else 1137 cmd->targ_xfer_tag = 0xFFFFFFFF; 1138 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 1139 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 1140 cmd->first_burst_len = payload_length; 1141 1142 if (!conn->sess->sess_ops->RDMAExtensions && 1143 cmd->data_direction == DMA_FROM_DEVICE) { 1144 struct iscsi_datain_req *dr; 1145 1146 dr = iscsit_allocate_datain_req(); 1147 if (!dr) 1148 return iscsit_add_reject_cmd(cmd, 1149 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1150 1151 iscsit_attach_datain_req(cmd, dr); 1152 } 1153 1154 /* 1155 * Initialize struct se_cmd descriptor from target_core_mod infrastructure 1156 */ 1157 transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops, 1158 conn->sess->se_sess, be32_to_cpu(hdr->data_length), 1159 cmd->data_direction, sam_task_attr, 1160 cmd->sense_buffer + 2, scsilun_to_int(&hdr->lun)); 1161 1162 pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x," 1163 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt, 1164 hdr->cmdsn, be32_to_cpu(hdr->data_length), payload_length, 1165 conn->cid); 1166 1167 target_get_sess_cmd(&cmd->se_cmd, true); 1168 1169 cmd->sense_reason = target_cmd_init_cdb(&cmd->se_cmd, hdr->cdb); 1170 if (cmd->sense_reason) { 1171 if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) { 1172 return iscsit_add_reject_cmd(cmd, 1173 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1174 } 1175 1176 goto attach_cmd; 1177 } 1178 1179 cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd); 1180 if (cmd->sense_reason) 1181 goto attach_cmd; 1182 1183 /* only used for printks or comparing with ->ref_task_tag */ 1184 cmd->se_cmd.tag = (__force u32)cmd->init_task_tag; 1185 cmd->sense_reason = target_cmd_parse_cdb(&cmd->se_cmd); 1186 if (cmd->sense_reason) 1187 goto attach_cmd; 1188 1189 if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) { 1190 return iscsit_add_reject_cmd(cmd, 1191 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1192 } 1193 1194 attach_cmd: 1195 spin_lock_bh(&conn->cmd_lock); 1196 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 1197 spin_unlock_bh(&conn->cmd_lock); 1198 /* 1199 * Check if we need to delay processing because of ALUA 1200 * Active/NonOptimized primary access state.. 1201 */ 1202 core_alua_check_nonop_delay(&cmd->se_cmd); 1203 1204 return 0; 1205 } 1206 EXPORT_SYMBOL(iscsit_setup_scsi_cmd); 1207 1208 void iscsit_set_unsolicited_dataout(struct iscsi_cmd *cmd) 1209 { 1210 iscsit_set_dataout_sequence_values(cmd); 1211 1212 spin_lock_bh(&cmd->dataout_timeout_lock); 1213 iscsit_start_dataout_timer(cmd, cmd->conn); 1214 spin_unlock_bh(&cmd->dataout_timeout_lock); 1215 } 1216 EXPORT_SYMBOL(iscsit_set_unsolicited_dataout); 1217 1218 int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1219 struct iscsi_scsi_req *hdr) 1220 { 1221 int cmdsn_ret = 0; 1222 /* 1223 * Check the CmdSN against ExpCmdSN/MaxCmdSN here if 1224 * the Immediate Bit is not set, and no Immediate 1225 * Data is attached. 1226 * 1227 * A PDU/CmdSN carrying Immediate Data can only 1228 * be processed after the DataCRC has passed. 1229 * If the DataCRC fails, the CmdSN MUST NOT 1230 * be acknowledged. (See below) 1231 */ 1232 if (!cmd->immediate_data) { 1233 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 1234 (unsigned char *)hdr, hdr->cmdsn); 1235 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1236 return -1; 1237 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 1238 target_put_sess_cmd(&cmd->se_cmd); 1239 return 0; 1240 } 1241 } 1242 1243 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 1244 1245 /* 1246 * If no Immediate Data is attached, it's OK to return now. 1247 */ 1248 if (!cmd->immediate_data) { 1249 if (!cmd->sense_reason && cmd->unsolicited_data) 1250 iscsit_set_unsolicited_dataout(cmd); 1251 if (!cmd->sense_reason) 1252 return 0; 1253 1254 target_put_sess_cmd(&cmd->se_cmd); 1255 return 0; 1256 } 1257 1258 /* 1259 * Early CHECK_CONDITIONs with ImmediateData never make it to command 1260 * execution. These exceptions are processed in CmdSN order using 1261 * iscsit_check_received_cmdsn() in iscsit_get_immediate_data() below. 1262 */ 1263 if (cmd->sense_reason) 1264 return 1; 1265 /* 1266 * Call directly into transport_generic_new_cmd() to perform 1267 * the backend memory allocation. 1268 */ 1269 cmd->sense_reason = transport_generic_new_cmd(&cmd->se_cmd); 1270 if (cmd->sense_reason) 1271 return 1; 1272 1273 return 0; 1274 } 1275 EXPORT_SYMBOL(iscsit_process_scsi_cmd); 1276 1277 static int 1278 iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 1279 bool dump_payload) 1280 { 1281 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 1282 int rc; 1283 1284 /* 1285 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. 1286 */ 1287 if (dump_payload) { 1288 u32 length = min(cmd->se_cmd.data_length - cmd->write_data_done, 1289 cmd->first_burst_len); 1290 1291 pr_debug("Dumping min(%d - %d, %d) = %d bytes of immediate data\n", 1292 cmd->se_cmd.data_length, cmd->write_data_done, 1293 cmd->first_burst_len, length); 1294 rc = iscsit_dump_data_payload(cmd->conn, length, 1); 1295 pr_debug("Finished dumping immediate data\n"); 1296 if (rc < 0) 1297 immed_ret = IMMEDIATE_DATA_CANNOT_RECOVER; 1298 } else { 1299 immed_ret = iscsit_handle_immediate_data(cmd, hdr, 1300 cmd->first_burst_len); 1301 } 1302 1303 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 1304 /* 1305 * A PDU/CmdSN carrying Immediate Data passed 1306 * DataCRC, check against ExpCmdSN/MaxCmdSN if 1307 * Immediate Bit is not set. 1308 */ 1309 cmdsn_ret = iscsit_sequence_cmd(cmd->conn, cmd, 1310 (unsigned char *)hdr, hdr->cmdsn); 1311 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1312 return -1; 1313 1314 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 1315 target_put_sess_cmd(&cmd->se_cmd); 1316 1317 return 0; 1318 } else if (cmd->unsolicited_data) 1319 iscsit_set_unsolicited_dataout(cmd); 1320 1321 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 1322 /* 1323 * Immediate Data failed DataCRC and ERL>=1, 1324 * silently drop this PDU and let the initiator 1325 * plug the CmdSN gap. 1326 * 1327 * FIXME: Send Unsolicited NOPIN with reserved 1328 * TTT here to help the initiator figure out 1329 * the missing CmdSN, although they should be 1330 * intelligent enough to determine the missing 1331 * CmdSN and issue a retry to plug the sequence. 1332 */ 1333 cmd->i_state = ISTATE_REMOVE; 1334 iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, cmd->i_state); 1335 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 1336 return -1; 1337 1338 return 0; 1339 } 1340 1341 static int 1342 iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1343 unsigned char *buf) 1344 { 1345 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1346 int rc, immed_data; 1347 bool dump_payload = false; 1348 1349 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1350 if (rc < 0) 1351 return 0; 1352 /* 1353 * Allocation iovecs needed for struct socket operations for 1354 * traditional iSCSI block I/O. 1355 */ 1356 if (iscsit_allocate_iovecs(cmd) < 0) { 1357 return iscsit_reject_cmd(cmd, 1358 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1359 } 1360 immed_data = cmd->immediate_data; 1361 1362 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1363 if (rc < 0) 1364 return rc; 1365 else if (rc > 0) 1366 dump_payload = true; 1367 1368 if (!immed_data) 1369 return 0; 1370 1371 return iscsit_get_immediate_data(cmd, hdr, dump_payload); 1372 } 1373 1374 static u32 iscsit_do_crypto_hash_sg( 1375 struct ahash_request *hash, 1376 struct iscsi_cmd *cmd, 1377 u32 data_offset, 1378 u32 data_length, 1379 u32 padding, 1380 u8 *pad_bytes) 1381 { 1382 u32 data_crc; 1383 struct scatterlist *sg; 1384 unsigned int page_off; 1385 1386 crypto_ahash_init(hash); 1387 1388 sg = cmd->first_data_sg; 1389 page_off = cmd->first_data_sg_off; 1390 1391 if (data_length && page_off) { 1392 struct scatterlist first_sg; 1393 u32 len = min_t(u32, data_length, sg->length - page_off); 1394 1395 sg_init_table(&first_sg, 1); 1396 sg_set_page(&first_sg, sg_page(sg), len, sg->offset + page_off); 1397 1398 ahash_request_set_crypt(hash, &first_sg, NULL, len); 1399 crypto_ahash_update(hash); 1400 1401 data_length -= len; 1402 sg = sg_next(sg); 1403 } 1404 1405 while (data_length) { 1406 u32 cur_len = min_t(u32, data_length, sg->length); 1407 1408 ahash_request_set_crypt(hash, sg, NULL, cur_len); 1409 crypto_ahash_update(hash); 1410 1411 data_length -= cur_len; 1412 /* iscsit_map_iovec has already checked for invalid sg pointers */ 1413 sg = sg_next(sg); 1414 } 1415 1416 if (padding) { 1417 struct scatterlist pad_sg; 1418 1419 sg_init_one(&pad_sg, pad_bytes, padding); 1420 ahash_request_set_crypt(hash, &pad_sg, (u8 *)&data_crc, 1421 padding); 1422 crypto_ahash_finup(hash); 1423 } else { 1424 ahash_request_set_crypt(hash, NULL, (u8 *)&data_crc, 0); 1425 crypto_ahash_final(hash); 1426 } 1427 1428 return data_crc; 1429 } 1430 1431 static void iscsit_do_crypto_hash_buf(struct ahash_request *hash, 1432 const void *buf, u32 payload_length, u32 padding, 1433 const void *pad_bytes, void *data_crc) 1434 { 1435 struct scatterlist sg[2]; 1436 1437 sg_init_table(sg, ARRAY_SIZE(sg)); 1438 sg_set_buf(sg, buf, payload_length); 1439 if (padding) 1440 sg_set_buf(sg + 1, pad_bytes, padding); 1441 1442 ahash_request_set_crypt(hash, sg, data_crc, payload_length + padding); 1443 1444 crypto_ahash_digest(hash); 1445 } 1446 1447 int 1448 __iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf, 1449 struct iscsi_cmd *cmd, u32 payload_length, 1450 bool *success) 1451 { 1452 struct iscsi_data *hdr = buf; 1453 struct se_cmd *se_cmd; 1454 int rc; 1455 1456 /* iSCSI write */ 1457 atomic_long_add(payload_length, &conn->sess->rx_data_octets); 1458 1459 pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x," 1460 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", 1461 hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset), 1462 payload_length, conn->cid); 1463 1464 if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) { 1465 pr_err("Command ITT: 0x%08x received DataOUT after" 1466 " last DataOUT received, dumping payload\n", 1467 cmd->init_task_tag); 1468 return iscsit_dump_data_payload(conn, payload_length, 1); 1469 } 1470 1471 if (cmd->data_direction != DMA_TO_DEVICE) { 1472 pr_err("Command ITT: 0x%08x received DataOUT for a" 1473 " NON-WRITE command.\n", cmd->init_task_tag); 1474 return iscsit_dump_data_payload(conn, payload_length, 1); 1475 } 1476 se_cmd = &cmd->se_cmd; 1477 iscsit_mod_dataout_timer(cmd); 1478 1479 if ((be32_to_cpu(hdr->offset) + payload_length) > cmd->se_cmd.data_length) { 1480 pr_err("DataOut Offset: %u, Length %u greater than iSCSI Command EDTL %u, protocol error.\n", 1481 be32_to_cpu(hdr->offset), payload_length, 1482 cmd->se_cmd.data_length); 1483 return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID, buf); 1484 } 1485 1486 if (cmd->unsolicited_data) { 1487 int dump_unsolicited_data = 0; 1488 1489 if (conn->sess->sess_ops->InitialR2T) { 1490 pr_err("Received unexpected unsolicited data" 1491 " while InitialR2T=Yes, protocol error.\n"); 1492 transport_send_check_condition_and_sense(&cmd->se_cmd, 1493 TCM_UNEXPECTED_UNSOLICITED_DATA, 0); 1494 return -1; 1495 } 1496 /* 1497 * Special case for dealing with Unsolicited DataOUT 1498 * and Unsupported SAM WRITE Opcodes and SE resource allocation 1499 * failures; 1500 */ 1501 1502 /* Something's amiss if we're not in WRITE_PENDING state... */ 1503 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING); 1504 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE)) 1505 dump_unsolicited_data = 1; 1506 1507 if (dump_unsolicited_data) { 1508 /* 1509 * Check if a delayed TASK_ABORTED status needs to 1510 * be sent now if the ISCSI_FLAG_CMD_FINAL has been 1511 * received with the unsolicited data out. 1512 */ 1513 if (hdr->flags & ISCSI_FLAG_CMD_FINAL) 1514 iscsit_stop_dataout_timer(cmd); 1515 1516 return iscsit_dump_data_payload(conn, payload_length, 1); 1517 } 1518 } else { 1519 /* 1520 * For the normal solicited data path: 1521 * 1522 * Check for a delayed TASK_ABORTED status and dump any 1523 * incoming data out payload if one exists. Also, when the 1524 * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current 1525 * data out sequence, we decrement outstanding_r2ts. Once 1526 * outstanding_r2ts reaches zero, go ahead and send the delayed 1527 * TASK_ABORTED status. 1528 */ 1529 if (se_cmd->transport_state & CMD_T_ABORTED) { 1530 if (hdr->flags & ISCSI_FLAG_CMD_FINAL && 1531 --cmd->outstanding_r2ts < 1) 1532 iscsit_stop_dataout_timer(cmd); 1533 1534 return iscsit_dump_data_payload(conn, payload_length, 1); 1535 } 1536 } 1537 /* 1538 * Perform DataSN, DataSequenceInOrder, DataPDUInOrder, and 1539 * within-command recovery checks before receiving the payload. 1540 */ 1541 rc = iscsit_check_pre_dataout(cmd, buf); 1542 if (rc == DATAOUT_WITHIN_COMMAND_RECOVERY) 1543 return 0; 1544 else if (rc == DATAOUT_CANNOT_RECOVER) 1545 return -1; 1546 *success = true; 1547 return 0; 1548 } 1549 EXPORT_SYMBOL(__iscsit_check_dataout_hdr); 1550 1551 int 1552 iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf, 1553 struct iscsi_cmd **out_cmd) 1554 { 1555 struct iscsi_data *hdr = buf; 1556 struct iscsi_cmd *cmd; 1557 u32 payload_length = ntoh24(hdr->dlength); 1558 int rc; 1559 bool success = false; 1560 1561 if (!payload_length) { 1562 pr_warn_ratelimited("DataOUT payload is ZERO, ignoring.\n"); 1563 return 0; 1564 } 1565 1566 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 1567 pr_err_ratelimited("DataSegmentLength: %u is greater than" 1568 " MaxXmitDataSegmentLength: %u\n", payload_length, 1569 conn->conn_ops->MaxXmitDataSegmentLength); 1570 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, buf); 1571 } 1572 1573 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, payload_length); 1574 if (!cmd) 1575 return 0; 1576 1577 rc = __iscsit_check_dataout_hdr(conn, buf, cmd, payload_length, &success); 1578 1579 if (success) 1580 *out_cmd = cmd; 1581 1582 return rc; 1583 } 1584 EXPORT_SYMBOL(iscsit_check_dataout_hdr); 1585 1586 static int 1587 iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1588 struct iscsi_data *hdr) 1589 { 1590 struct kvec *iov; 1591 u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0; 1592 u32 payload_length; 1593 int iov_ret, data_crc_failed = 0; 1594 1595 payload_length = min_t(u32, cmd->se_cmd.data_length, 1596 ntoh24(hdr->dlength)); 1597 rx_size += payload_length; 1598 iov = &cmd->iov_data[0]; 1599 1600 iov_ret = iscsit_map_iovec(cmd, iov, cmd->orig_iov_data_count - 2, 1601 be32_to_cpu(hdr->offset), payload_length); 1602 if (iov_ret < 0) 1603 return -1; 1604 1605 iov_count += iov_ret; 1606 1607 padding = ((-payload_length) & 3); 1608 if (padding != 0) { 1609 iov[iov_count].iov_base = cmd->pad_bytes; 1610 iov[iov_count++].iov_len = padding; 1611 rx_size += padding; 1612 pr_debug("Receiving %u padding bytes.\n", padding); 1613 } 1614 1615 if (conn->conn_ops->DataDigest) { 1616 iov[iov_count].iov_base = &checksum; 1617 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 1618 rx_size += ISCSI_CRC_LEN; 1619 } 1620 1621 WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count); 1622 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); 1623 1624 iscsit_unmap_iovec(cmd); 1625 1626 if (rx_got != rx_size) 1627 return -1; 1628 1629 if (conn->conn_ops->DataDigest) { 1630 u32 data_crc; 1631 1632 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd, 1633 be32_to_cpu(hdr->offset), 1634 payload_length, padding, 1635 cmd->pad_bytes); 1636 1637 if (checksum != data_crc) { 1638 pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 1639 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x" 1640 " does not match computed 0x%08x\n", 1641 hdr->itt, hdr->offset, payload_length, 1642 hdr->datasn, checksum, data_crc); 1643 data_crc_failed = 1; 1644 } else { 1645 pr_debug("Got CRC32C DataDigest 0x%08x for" 1646 " %u bytes of Data Out\n", checksum, 1647 payload_length); 1648 } 1649 } 1650 1651 return data_crc_failed; 1652 } 1653 1654 int 1655 iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr, 1656 bool data_crc_failed) 1657 { 1658 struct iscsi_conn *conn = cmd->conn; 1659 int rc, ooo_cmdsn; 1660 /* 1661 * Increment post receive data and CRC values or perform 1662 * within-command recovery. 1663 */ 1664 rc = iscsit_check_post_dataout(cmd, (unsigned char *)hdr, data_crc_failed); 1665 if ((rc == DATAOUT_NORMAL) || (rc == DATAOUT_WITHIN_COMMAND_RECOVERY)) 1666 return 0; 1667 else if (rc == DATAOUT_SEND_R2T) { 1668 iscsit_set_dataout_sequence_values(cmd); 1669 conn->conn_transport->iscsit_get_dataout(conn, cmd, false); 1670 } else if (rc == DATAOUT_SEND_TO_TRANSPORT) { 1671 /* 1672 * Handle extra special case for out of order 1673 * Unsolicited Data Out. 1674 */ 1675 spin_lock_bh(&cmd->istate_lock); 1676 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN); 1677 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1678 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1679 spin_unlock_bh(&cmd->istate_lock); 1680 1681 iscsit_stop_dataout_timer(cmd); 1682 if (ooo_cmdsn) 1683 return 0; 1684 target_execute_cmd(&cmd->se_cmd); 1685 return 0; 1686 } else /* DATAOUT_CANNOT_RECOVER */ 1687 return -1; 1688 1689 return 0; 1690 } 1691 EXPORT_SYMBOL(iscsit_check_dataout_payload); 1692 1693 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) 1694 { 1695 struct iscsi_cmd *cmd = NULL; 1696 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1697 int rc; 1698 bool data_crc_failed = false; 1699 1700 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1701 if (rc < 0) 1702 return 0; 1703 else if (!cmd) 1704 return 0; 1705 1706 rc = iscsit_get_dataout(conn, cmd, hdr); 1707 if (rc < 0) 1708 return rc; 1709 else if (rc > 0) 1710 data_crc_failed = true; 1711 1712 return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed); 1713 } 1714 1715 int iscsit_setup_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1716 struct iscsi_nopout *hdr) 1717 { 1718 u32 payload_length = ntoh24(hdr->dlength); 1719 1720 if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL)) { 1721 pr_err("NopOUT Flag's, Left Most Bit not set, protocol error.\n"); 1722 if (!cmd) 1723 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1724 (unsigned char *)hdr); 1725 1726 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1727 (unsigned char *)hdr); 1728 } 1729 1730 if (hdr->itt == RESERVED_ITT && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1731 pr_err("NOPOUT ITT is reserved, but Immediate Bit is" 1732 " not set, protocol error.\n"); 1733 if (!cmd) 1734 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1735 (unsigned char *)hdr); 1736 1737 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1738 (unsigned char *)hdr); 1739 } 1740 1741 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 1742 pr_err("NOPOUT Ping Data DataSegmentLength: %u is" 1743 " greater than MaxXmitDataSegmentLength: %u, protocol" 1744 " error.\n", payload_length, 1745 conn->conn_ops->MaxXmitDataSegmentLength); 1746 if (!cmd) 1747 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1748 (unsigned char *)hdr); 1749 1750 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1751 (unsigned char *)hdr); 1752 } 1753 1754 pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%08x," 1755 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n", 1756 hdr->itt == RESERVED_ITT ? "Response" : "Request", 1757 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn, 1758 payload_length); 1759 /* 1760 * This is not a response to a Unsolicited NopIN, which means 1761 * it can either be a NOPOUT ping request (with a valid ITT), 1762 * or a NOPOUT not requesting a NOPIN (with a reserved ITT). 1763 * Either way, make sure we allocate an struct iscsi_cmd, as both 1764 * can contain ping data. 1765 */ 1766 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1767 cmd->iscsi_opcode = ISCSI_OP_NOOP_OUT; 1768 cmd->i_state = ISTATE_SEND_NOPIN; 1769 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1770 1 : 0); 1771 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 1772 cmd->targ_xfer_tag = 0xFFFFFFFF; 1773 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 1774 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 1775 cmd->data_direction = DMA_NONE; 1776 } 1777 1778 return 0; 1779 } 1780 EXPORT_SYMBOL(iscsit_setup_nop_out); 1781 1782 int iscsit_process_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1783 struct iscsi_nopout *hdr) 1784 { 1785 struct iscsi_cmd *cmd_p = NULL; 1786 int cmdsn_ret = 0; 1787 /* 1788 * Initiator is expecting a NopIN ping reply.. 1789 */ 1790 if (hdr->itt != RESERVED_ITT) { 1791 if (!cmd) 1792 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1793 (unsigned char *)hdr); 1794 1795 spin_lock_bh(&conn->cmd_lock); 1796 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 1797 spin_unlock_bh(&conn->cmd_lock); 1798 1799 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 1800 1801 if (hdr->opcode & ISCSI_OP_IMMEDIATE) { 1802 iscsit_add_cmd_to_response_queue(cmd, conn, 1803 cmd->i_state); 1804 return 0; 1805 } 1806 1807 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 1808 (unsigned char *)hdr, hdr->cmdsn); 1809 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) 1810 return 0; 1811 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1812 return -1; 1813 1814 return 0; 1815 } 1816 /* 1817 * This was a response to a unsolicited NOPIN ping. 1818 */ 1819 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 1820 cmd_p = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt)); 1821 if (!cmd_p) 1822 return -EINVAL; 1823 1824 iscsit_stop_nopin_response_timer(conn); 1825 1826 cmd_p->i_state = ISTATE_REMOVE; 1827 iscsit_add_cmd_to_immediate_queue(cmd_p, conn, cmd_p->i_state); 1828 1829 iscsit_start_nopin_timer(conn); 1830 return 0; 1831 } 1832 /* 1833 * Otherwise, initiator is not expecting a NOPIN is response. 1834 * Just ignore for now. 1835 */ 1836 1837 if (cmd) 1838 iscsit_free_cmd(cmd, false); 1839 1840 return 0; 1841 } 1842 EXPORT_SYMBOL(iscsit_process_nop_out); 1843 1844 static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1845 unsigned char *buf) 1846 { 1847 unsigned char *ping_data = NULL; 1848 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1849 struct kvec *iov = NULL; 1850 u32 payload_length = ntoh24(hdr->dlength); 1851 int ret; 1852 1853 ret = iscsit_setup_nop_out(conn, cmd, hdr); 1854 if (ret < 0) 1855 return 0; 1856 /* 1857 * Handle NOP-OUT payload for traditional iSCSI sockets 1858 */ 1859 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1860 u32 checksum, data_crc, padding = 0; 1861 int niov = 0, rx_got, rx_size = payload_length; 1862 1863 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1864 if (!ping_data) { 1865 ret = -1; 1866 goto out; 1867 } 1868 1869 iov = &cmd->iov_misc[0]; 1870 iov[niov].iov_base = ping_data; 1871 iov[niov++].iov_len = payload_length; 1872 1873 padding = ((-payload_length) & 3); 1874 if (padding != 0) { 1875 pr_debug("Receiving %u additional bytes" 1876 " for padding.\n", padding); 1877 iov[niov].iov_base = &cmd->pad_bytes; 1878 iov[niov++].iov_len = padding; 1879 rx_size += padding; 1880 } 1881 if (conn->conn_ops->DataDigest) { 1882 iov[niov].iov_base = &checksum; 1883 iov[niov++].iov_len = ISCSI_CRC_LEN; 1884 rx_size += ISCSI_CRC_LEN; 1885 } 1886 1887 WARN_ON_ONCE(niov > ARRAY_SIZE(cmd->iov_misc)); 1888 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size); 1889 if (rx_got != rx_size) { 1890 ret = -1; 1891 goto out; 1892 } 1893 1894 if (conn->conn_ops->DataDigest) { 1895 iscsit_do_crypto_hash_buf(conn->conn_rx_hash, ping_data, 1896 payload_length, padding, 1897 cmd->pad_bytes, &data_crc); 1898 1899 if (checksum != data_crc) { 1900 pr_err("Ping data CRC32C DataDigest" 1901 " 0x%08x does not match computed 0x%08x\n", 1902 checksum, data_crc); 1903 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1904 pr_err("Unable to recover from" 1905 " NOPOUT Ping DataCRC failure while in" 1906 " ERL=0.\n"); 1907 ret = -1; 1908 goto out; 1909 } else { 1910 /* 1911 * Silently drop this PDU and let the 1912 * initiator plug the CmdSN gap. 1913 */ 1914 pr_debug("Dropping NOPOUT" 1915 " Command CmdSN: 0x%08x due to" 1916 " DataCRC error.\n", hdr->cmdsn); 1917 ret = 0; 1918 goto out; 1919 } 1920 } else { 1921 pr_debug("Got CRC32C DataDigest" 1922 " 0x%08x for %u bytes of ping data.\n", 1923 checksum, payload_length); 1924 } 1925 } 1926 1927 ping_data[payload_length] = '\0'; 1928 /* 1929 * Attach ping data to struct iscsi_cmd->buf_ptr. 1930 */ 1931 cmd->buf_ptr = ping_data; 1932 cmd->buf_ptr_size = payload_length; 1933 1934 pr_debug("Got %u bytes of NOPOUT ping" 1935 " data.\n", payload_length); 1936 pr_debug("Ping Data: \"%s\"\n", ping_data); 1937 } 1938 1939 return iscsit_process_nop_out(conn, cmd, hdr); 1940 out: 1941 if (cmd) 1942 iscsit_free_cmd(cmd, false); 1943 1944 kfree(ping_data); 1945 return ret; 1946 } 1947 1948 static enum tcm_tmreq_table iscsit_convert_tmf(u8 iscsi_tmf) 1949 { 1950 switch (iscsi_tmf) { 1951 case ISCSI_TM_FUNC_ABORT_TASK: 1952 return TMR_ABORT_TASK; 1953 case ISCSI_TM_FUNC_ABORT_TASK_SET: 1954 return TMR_ABORT_TASK_SET; 1955 case ISCSI_TM_FUNC_CLEAR_ACA: 1956 return TMR_CLEAR_ACA; 1957 case ISCSI_TM_FUNC_CLEAR_TASK_SET: 1958 return TMR_CLEAR_TASK_SET; 1959 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET: 1960 return TMR_LUN_RESET; 1961 case ISCSI_TM_FUNC_TARGET_WARM_RESET: 1962 return TMR_TARGET_WARM_RESET; 1963 case ISCSI_TM_FUNC_TARGET_COLD_RESET: 1964 return TMR_TARGET_COLD_RESET; 1965 default: 1966 return TMR_UNKNOWN; 1967 } 1968 } 1969 1970 int 1971 iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1972 unsigned char *buf) 1973 { 1974 struct se_tmr_req *se_tmr; 1975 struct iscsi_tmr_req *tmr_req; 1976 struct iscsi_tm *hdr; 1977 int out_of_order_cmdsn = 0, ret; 1978 u8 function, tcm_function = TMR_UNKNOWN; 1979 1980 hdr = (struct iscsi_tm *) buf; 1981 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL; 1982 function = hdr->flags; 1983 1984 pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:" 1985 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:" 1986 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function, 1987 hdr->rtt, hdr->refcmdsn, conn->cid); 1988 1989 if ((function != ISCSI_TM_FUNC_ABORT_TASK) && 1990 ((function != ISCSI_TM_FUNC_TASK_REASSIGN) && 1991 hdr->rtt != RESERVED_ITT)) { 1992 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n"); 1993 hdr->rtt = RESERVED_ITT; 1994 } 1995 1996 if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) && 1997 !(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1998 pr_err("Task Management Request TASK_REASSIGN not" 1999 " issued as immediate command, bad iSCSI Initiator" 2000 "implementation\n"); 2001 return iscsit_add_reject_cmd(cmd, 2002 ISCSI_REASON_PROTOCOL_ERROR, buf); 2003 } 2004 if ((function != ISCSI_TM_FUNC_ABORT_TASK) && 2005 be32_to_cpu(hdr->refcmdsn) != ISCSI_RESERVED_TAG) 2006 hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG); 2007 2008 cmd->data_direction = DMA_NONE; 2009 cmd->tmr_req = kzalloc(sizeof(*cmd->tmr_req), GFP_KERNEL); 2010 if (!cmd->tmr_req) { 2011 return iscsit_add_reject_cmd(cmd, 2012 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 2013 buf); 2014 } 2015 2016 transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops, 2017 conn->sess->se_sess, 0, DMA_NONE, 2018 TCM_SIMPLE_TAG, cmd->sense_buffer + 2, 2019 scsilun_to_int(&hdr->lun)); 2020 2021 target_get_sess_cmd(&cmd->se_cmd, true); 2022 2023 /* 2024 * TASK_REASSIGN for ERL=2 / connection stays inside of 2025 * LIO-Target $FABRIC_MOD 2026 */ 2027 if (function != ISCSI_TM_FUNC_TASK_REASSIGN) { 2028 tcm_function = iscsit_convert_tmf(function); 2029 if (tcm_function == TMR_UNKNOWN) { 2030 pr_err("Unknown iSCSI TMR Function:" 2031 " 0x%02x\n", function); 2032 return iscsit_add_reject_cmd(cmd, 2033 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 2034 } 2035 } 2036 ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req, tcm_function, 2037 GFP_KERNEL); 2038 if (ret < 0) 2039 return iscsit_add_reject_cmd(cmd, 2040 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 2041 2042 cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req; 2043 2044 cmd->iscsi_opcode = ISCSI_OP_SCSI_TMFUNC; 2045 cmd->i_state = ISTATE_SEND_TASKMGTRSP; 2046 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 2047 cmd->init_task_tag = hdr->itt; 2048 cmd->targ_xfer_tag = 0xFFFFFFFF; 2049 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 2050 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 2051 se_tmr = cmd->se_cmd.se_tmr_req; 2052 tmr_req = cmd->tmr_req; 2053 /* 2054 * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN 2055 */ 2056 if (function != ISCSI_TM_FUNC_TASK_REASSIGN) { 2057 ret = transport_lookup_tmr_lun(&cmd->se_cmd); 2058 if (ret < 0) { 2059 se_tmr->response = ISCSI_TMF_RSP_NO_LUN; 2060 goto attach; 2061 } 2062 } 2063 2064 switch (function) { 2065 case ISCSI_TM_FUNC_ABORT_TASK: 2066 se_tmr->response = iscsit_tmr_abort_task(cmd, buf); 2067 if (se_tmr->response) 2068 goto attach; 2069 break; 2070 case ISCSI_TM_FUNC_ABORT_TASK_SET: 2071 case ISCSI_TM_FUNC_CLEAR_ACA: 2072 case ISCSI_TM_FUNC_CLEAR_TASK_SET: 2073 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET: 2074 break; 2075 case ISCSI_TM_FUNC_TARGET_WARM_RESET: 2076 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) { 2077 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED; 2078 goto attach; 2079 } 2080 break; 2081 case ISCSI_TM_FUNC_TARGET_COLD_RESET: 2082 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) { 2083 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED; 2084 goto attach; 2085 } 2086 break; 2087 case ISCSI_TM_FUNC_TASK_REASSIGN: 2088 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf); 2089 /* 2090 * Perform sanity checks on the ExpDataSN only if the 2091 * TASK_REASSIGN was successful. 2092 */ 2093 if (se_tmr->response) 2094 break; 2095 2096 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0) 2097 return iscsit_add_reject_cmd(cmd, 2098 ISCSI_REASON_BOOKMARK_INVALID, buf); 2099 break; 2100 default: 2101 pr_err("Unknown TMR function: 0x%02x, protocol" 2102 " error.\n", function); 2103 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED; 2104 goto attach; 2105 } 2106 2107 if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) && 2108 (se_tmr->response == ISCSI_TMF_RSP_COMPLETE)) 2109 se_tmr->call_transport = 1; 2110 attach: 2111 spin_lock_bh(&conn->cmd_lock); 2112 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 2113 spin_unlock_bh(&conn->cmd_lock); 2114 2115 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 2116 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 2117 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP) { 2118 out_of_order_cmdsn = 1; 2119 } else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 2120 target_put_sess_cmd(&cmd->se_cmd); 2121 return 0; 2122 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) { 2123 return -1; 2124 } 2125 } 2126 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 2127 2128 if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE)) 2129 return 0; 2130 /* 2131 * Found the referenced task, send to transport for processing. 2132 */ 2133 if (se_tmr->call_transport) 2134 return transport_generic_handle_tmr(&cmd->se_cmd); 2135 2136 /* 2137 * Could not find the referenced LUN, task, or Task Management 2138 * command not authorized or supported. Change state and 2139 * let the tx_thread send the response. 2140 * 2141 * For connection recovery, this is also the default action for 2142 * TMR TASK_REASSIGN. 2143 */ 2144 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2145 target_put_sess_cmd(&cmd->se_cmd); 2146 return 0; 2147 } 2148 EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd); 2149 2150 /* #warning FIXME: Support Text Command parameters besides SendTargets */ 2151 int 2152 iscsit_setup_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2153 struct iscsi_text *hdr) 2154 { 2155 u32 payload_length = ntoh24(hdr->dlength); 2156 2157 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 2158 pr_err("Unable to accept text parameter length: %u" 2159 "greater than MaxXmitDataSegmentLength %u.\n", 2160 payload_length, conn->conn_ops->MaxXmitDataSegmentLength); 2161 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 2162 (unsigned char *)hdr); 2163 } 2164 2165 if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL) || 2166 (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)) { 2167 pr_err("Multi sequence text commands currently not supported\n"); 2168 return iscsit_reject_cmd(cmd, ISCSI_REASON_CMD_NOT_SUPPORTED, 2169 (unsigned char *)hdr); 2170 } 2171 2172 pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x," 2173 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn, 2174 hdr->exp_statsn, payload_length); 2175 2176 cmd->iscsi_opcode = ISCSI_OP_TEXT; 2177 cmd->i_state = ISTATE_SEND_TEXTRSP; 2178 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 2179 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 2180 cmd->targ_xfer_tag = 0xFFFFFFFF; 2181 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 2182 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 2183 cmd->data_direction = DMA_NONE; 2184 kfree(cmd->text_in_ptr); 2185 cmd->text_in_ptr = NULL; 2186 2187 return 0; 2188 } 2189 EXPORT_SYMBOL(iscsit_setup_text_cmd); 2190 2191 int 2192 iscsit_process_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2193 struct iscsi_text *hdr) 2194 { 2195 unsigned char *text_in = cmd->text_in_ptr, *text_ptr; 2196 int cmdsn_ret; 2197 2198 if (!text_in) { 2199 cmd->targ_xfer_tag = be32_to_cpu(hdr->ttt); 2200 if (cmd->targ_xfer_tag == 0xFFFFFFFF) { 2201 pr_err("Unable to locate text_in buffer for sendtargets" 2202 " discovery\n"); 2203 goto reject; 2204 } 2205 goto empty_sendtargets; 2206 } 2207 if (strncmp("SendTargets=", text_in, 12) != 0) { 2208 pr_err("Received Text Data that is not" 2209 " SendTargets, cannot continue.\n"); 2210 goto reject; 2211 } 2212 /* '=' confirmed in strncmp */ 2213 text_ptr = strchr(text_in, '='); 2214 BUG_ON(!text_ptr); 2215 if (!strncmp("=All", text_ptr, 5)) { 2216 cmd->cmd_flags |= ICF_SENDTARGETS_ALL; 2217 } else if (!strncmp("=iqn.", text_ptr, 5) || 2218 !strncmp("=eui.", text_ptr, 5)) { 2219 cmd->cmd_flags |= ICF_SENDTARGETS_SINGLE; 2220 } else { 2221 pr_err("Unable to locate valid SendTargets%s value\n", 2222 text_ptr); 2223 goto reject; 2224 } 2225 2226 spin_lock_bh(&conn->cmd_lock); 2227 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 2228 spin_unlock_bh(&conn->cmd_lock); 2229 2230 empty_sendtargets: 2231 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 2232 2233 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 2234 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 2235 (unsigned char *)hdr, hdr->cmdsn); 2236 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 2237 return -1; 2238 2239 return 0; 2240 } 2241 2242 return iscsit_execute_cmd(cmd, 0); 2243 2244 reject: 2245 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 2246 (unsigned char *)hdr); 2247 } 2248 EXPORT_SYMBOL(iscsit_process_text_cmd); 2249 2250 static int 2251 iscsit_handle_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2252 unsigned char *buf) 2253 { 2254 struct iscsi_text *hdr = (struct iscsi_text *)buf; 2255 char *text_in = NULL; 2256 u32 payload_length = ntoh24(hdr->dlength); 2257 int rx_size, rc; 2258 2259 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 2260 if (rc < 0) 2261 return 0; 2262 2263 rx_size = payload_length; 2264 if (payload_length) { 2265 u32 checksum = 0, data_crc = 0; 2266 u32 padding = 0; 2267 int niov = 0, rx_got; 2268 struct kvec iov[2]; 2269 2270 rx_size = ALIGN(payload_length, 4); 2271 text_in = kzalloc(rx_size, GFP_KERNEL); 2272 if (!text_in) 2273 goto reject; 2274 2275 cmd->text_in_ptr = text_in; 2276 2277 memset(iov, 0, sizeof(iov)); 2278 iov[niov].iov_base = text_in; 2279 iov[niov++].iov_len = rx_size; 2280 2281 padding = rx_size - payload_length; 2282 if (padding) 2283 pr_debug("Receiving %u additional bytes" 2284 " for padding.\n", padding); 2285 if (conn->conn_ops->DataDigest) { 2286 iov[niov].iov_base = &checksum; 2287 iov[niov++].iov_len = ISCSI_CRC_LEN; 2288 rx_size += ISCSI_CRC_LEN; 2289 } 2290 2291 WARN_ON_ONCE(niov > ARRAY_SIZE(iov)); 2292 rx_got = rx_data(conn, &iov[0], niov, rx_size); 2293 if (rx_got != rx_size) 2294 goto reject; 2295 2296 if (conn->conn_ops->DataDigest) { 2297 iscsit_do_crypto_hash_buf(conn->conn_rx_hash, 2298 text_in, rx_size, 0, NULL, 2299 &data_crc); 2300 2301 if (checksum != data_crc) { 2302 pr_err("Text data CRC32C DataDigest" 2303 " 0x%08x does not match computed" 2304 " 0x%08x\n", checksum, data_crc); 2305 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2306 pr_err("Unable to recover from" 2307 " Text Data digest failure while in" 2308 " ERL=0.\n"); 2309 goto reject; 2310 } else { 2311 /* 2312 * Silently drop this PDU and let the 2313 * initiator plug the CmdSN gap. 2314 */ 2315 pr_debug("Dropping Text" 2316 " Command CmdSN: 0x%08x due to" 2317 " DataCRC error.\n", hdr->cmdsn); 2318 kfree(text_in); 2319 return 0; 2320 } 2321 } else { 2322 pr_debug("Got CRC32C DataDigest" 2323 " 0x%08x for %u bytes of text data.\n", 2324 checksum, payload_length); 2325 } 2326 } 2327 text_in[payload_length - 1] = '\0'; 2328 pr_debug("Successfully read %d bytes of text" 2329 " data.\n", payload_length); 2330 } 2331 2332 return iscsit_process_text_cmd(conn, cmd, hdr); 2333 2334 reject: 2335 kfree(cmd->text_in_ptr); 2336 cmd->text_in_ptr = NULL; 2337 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf); 2338 } 2339 2340 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2341 { 2342 struct iscsi_conn *conn_p; 2343 struct iscsi_session *sess = conn->sess; 2344 2345 pr_debug("Received logout request CLOSESESSION on CID: %hu" 2346 " for SID: %u.\n", conn->cid, conn->sess->sid); 2347 2348 atomic_set(&sess->session_logout, 1); 2349 atomic_set(&conn->conn_logout_remove, 1); 2350 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION; 2351 2352 iscsit_inc_conn_usage_count(conn); 2353 iscsit_inc_session_usage_count(sess); 2354 2355 spin_lock_bh(&sess->conn_lock); 2356 list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) { 2357 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN) 2358 continue; 2359 2360 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 2361 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT; 2362 } 2363 spin_unlock_bh(&sess->conn_lock); 2364 2365 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2366 2367 return 0; 2368 } 2369 2370 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2371 { 2372 struct iscsi_conn *l_conn; 2373 struct iscsi_session *sess = conn->sess; 2374 2375 pr_debug("Received logout request CLOSECONNECTION for CID:" 2376 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); 2377 2378 /* 2379 * A Logout Request with a CLOSECONNECTION reason code for a CID 2380 * can arrive on a connection with a differing CID. 2381 */ 2382 if (conn->cid == cmd->logout_cid) { 2383 spin_lock_bh(&conn->state_lock); 2384 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 2385 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT; 2386 2387 atomic_set(&conn->conn_logout_remove, 1); 2388 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION; 2389 iscsit_inc_conn_usage_count(conn); 2390 2391 spin_unlock_bh(&conn->state_lock); 2392 } else { 2393 /* 2394 * Handle all different cid CLOSECONNECTION requests in 2395 * iscsit_logout_post_handler_diffcid() as to give enough 2396 * time for any non immediate command's CmdSN to be 2397 * acknowledged on the connection in question. 2398 * 2399 * Here we simply make sure the CID is still around. 2400 */ 2401 l_conn = iscsit_get_conn_from_cid(sess, 2402 cmd->logout_cid); 2403 if (!l_conn) { 2404 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND; 2405 iscsit_add_cmd_to_response_queue(cmd, conn, 2406 cmd->i_state); 2407 return 0; 2408 } 2409 2410 iscsit_dec_conn_usage_count(l_conn); 2411 } 2412 2413 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2414 2415 return 0; 2416 } 2417 2418 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2419 { 2420 struct iscsi_session *sess = conn->sess; 2421 2422 pr_debug("Received explicit REMOVECONNFORRECOVERY logout for" 2423 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); 2424 2425 if (sess->sess_ops->ErrorRecoveryLevel != 2) { 2426 pr_err("Received Logout Request REMOVECONNFORRECOVERY" 2427 " while ERL!=2.\n"); 2428 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED; 2429 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2430 return 0; 2431 } 2432 2433 if (conn->cid == cmd->logout_cid) { 2434 pr_err("Received Logout Request REMOVECONNFORRECOVERY" 2435 " with CID: %hu on CID: %hu, implementation error.\n", 2436 cmd->logout_cid, conn->cid); 2437 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED; 2438 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2439 return 0; 2440 } 2441 2442 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2443 2444 return 0; 2445 } 2446 2447 int 2448 iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2449 unsigned char *buf) 2450 { 2451 int cmdsn_ret, logout_remove = 0; 2452 u8 reason_code = 0; 2453 struct iscsi_logout *hdr; 2454 struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn); 2455 2456 hdr = (struct iscsi_logout *) buf; 2457 reason_code = (hdr->flags & 0x7f); 2458 2459 if (tiqn) { 2460 spin_lock(&tiqn->logout_stats.lock); 2461 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) 2462 tiqn->logout_stats.normal_logouts++; 2463 else 2464 tiqn->logout_stats.abnormal_logouts++; 2465 spin_unlock(&tiqn->logout_stats.lock); 2466 } 2467 2468 pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x" 2469 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n", 2470 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code, 2471 hdr->cid, conn->cid); 2472 2473 if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) { 2474 pr_err("Received logout request on connection that" 2475 " is not in logged in state, ignoring request.\n"); 2476 iscsit_free_cmd(cmd, false); 2477 return 0; 2478 } 2479 2480 cmd->iscsi_opcode = ISCSI_OP_LOGOUT; 2481 cmd->i_state = ISTATE_SEND_LOGOUTRSP; 2482 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 2483 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 2484 cmd->targ_xfer_tag = 0xFFFFFFFF; 2485 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 2486 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 2487 cmd->logout_cid = be16_to_cpu(hdr->cid); 2488 cmd->logout_reason = reason_code; 2489 cmd->data_direction = DMA_NONE; 2490 2491 /* 2492 * We need to sleep in these cases (by returning 1) until the Logout 2493 * Response gets sent in the tx thread. 2494 */ 2495 if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) || 2496 ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) && 2497 be16_to_cpu(hdr->cid) == conn->cid)) 2498 logout_remove = 1; 2499 2500 spin_lock_bh(&conn->cmd_lock); 2501 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 2502 spin_unlock_bh(&conn->cmd_lock); 2503 2504 if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY) 2505 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 2506 2507 /* 2508 * Immediate commands are executed, well, immediately. 2509 * Non-Immediate Logout Commands are executed in CmdSN order. 2510 */ 2511 if (cmd->immediate_cmd) { 2512 int ret = iscsit_execute_cmd(cmd, 0); 2513 2514 if (ret < 0) 2515 return ret; 2516 } else { 2517 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 2518 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) 2519 logout_remove = 0; 2520 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 2521 return -1; 2522 } 2523 2524 return logout_remove; 2525 } 2526 EXPORT_SYMBOL(iscsit_handle_logout_cmd); 2527 2528 int iscsit_handle_snack( 2529 struct iscsi_conn *conn, 2530 unsigned char *buf) 2531 { 2532 struct iscsi_snack *hdr; 2533 2534 hdr = (struct iscsi_snack *) buf; 2535 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL; 2536 2537 pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:" 2538 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x," 2539 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags, 2540 hdr->begrun, hdr->runlength, conn->cid); 2541 2542 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2543 pr_err("Initiator sent SNACK request while in" 2544 " ErrorRecoveryLevel=0.\n"); 2545 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 2546 buf); 2547 } 2548 /* 2549 * SNACK_DATA and SNACK_R2T are both 0, so check which function to 2550 * call from inside iscsi_send_recovery_datain_or_r2t(). 2551 */ 2552 switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) { 2553 case 0: 2554 return iscsit_handle_recovery_datain_or_r2t(conn, buf, 2555 hdr->itt, 2556 be32_to_cpu(hdr->ttt), 2557 be32_to_cpu(hdr->begrun), 2558 be32_to_cpu(hdr->runlength)); 2559 case ISCSI_FLAG_SNACK_TYPE_STATUS: 2560 return iscsit_handle_status_snack(conn, hdr->itt, 2561 be32_to_cpu(hdr->ttt), 2562 be32_to_cpu(hdr->begrun), be32_to_cpu(hdr->runlength)); 2563 case ISCSI_FLAG_SNACK_TYPE_DATA_ACK: 2564 return iscsit_handle_data_ack(conn, be32_to_cpu(hdr->ttt), 2565 be32_to_cpu(hdr->begrun), 2566 be32_to_cpu(hdr->runlength)); 2567 case ISCSI_FLAG_SNACK_TYPE_RDATA: 2568 /* FIXME: Support R-Data SNACK */ 2569 pr_err("R-Data SNACK Not Supported.\n"); 2570 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 2571 buf); 2572 default: 2573 pr_err("Unknown SNACK type 0x%02x, protocol" 2574 " error.\n", hdr->flags & 0x0f); 2575 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 2576 buf); 2577 } 2578 2579 return 0; 2580 } 2581 EXPORT_SYMBOL(iscsit_handle_snack); 2582 2583 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn) 2584 { 2585 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || 2586 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { 2587 wait_for_completion_interruptible_timeout( 2588 &conn->rx_half_close_comp, 2589 ISCSI_RX_THREAD_TCP_TIMEOUT * HZ); 2590 } 2591 } 2592 2593 static int iscsit_handle_immediate_data( 2594 struct iscsi_cmd *cmd, 2595 struct iscsi_scsi_req *hdr, 2596 u32 length) 2597 { 2598 int iov_ret, rx_got = 0, rx_size = 0; 2599 u32 checksum, iov_count = 0, padding = 0; 2600 struct iscsi_conn *conn = cmd->conn; 2601 struct kvec *iov; 2602 void *overflow_buf = NULL; 2603 2604 BUG_ON(cmd->write_data_done > cmd->se_cmd.data_length); 2605 rx_size = min(cmd->se_cmd.data_length - cmd->write_data_done, length); 2606 iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, 2607 cmd->orig_iov_data_count - 2, 2608 cmd->write_data_done, rx_size); 2609 if (iov_ret < 0) 2610 return IMMEDIATE_DATA_CANNOT_RECOVER; 2611 2612 iov_count = iov_ret; 2613 iov = &cmd->iov_data[0]; 2614 if (rx_size < length) { 2615 /* 2616 * Special case: length of immediate data exceeds the data 2617 * buffer size derived from the CDB. 2618 */ 2619 overflow_buf = kmalloc(length - rx_size, GFP_KERNEL); 2620 if (!overflow_buf) { 2621 iscsit_unmap_iovec(cmd); 2622 return IMMEDIATE_DATA_CANNOT_RECOVER; 2623 } 2624 cmd->overflow_buf = overflow_buf; 2625 iov[iov_count].iov_base = overflow_buf; 2626 iov[iov_count].iov_len = length - rx_size; 2627 iov_count++; 2628 rx_size = length; 2629 } 2630 2631 padding = ((-length) & 3); 2632 if (padding != 0) { 2633 iov[iov_count].iov_base = cmd->pad_bytes; 2634 iov[iov_count++].iov_len = padding; 2635 rx_size += padding; 2636 } 2637 2638 if (conn->conn_ops->DataDigest) { 2639 iov[iov_count].iov_base = &checksum; 2640 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 2641 rx_size += ISCSI_CRC_LEN; 2642 } 2643 2644 WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count); 2645 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); 2646 2647 iscsit_unmap_iovec(cmd); 2648 2649 if (rx_got != rx_size) { 2650 iscsit_rx_thread_wait_for_tcp(conn); 2651 return IMMEDIATE_DATA_CANNOT_RECOVER; 2652 } 2653 2654 if (conn->conn_ops->DataDigest) { 2655 u32 data_crc; 2656 2657 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd, 2658 cmd->write_data_done, length, padding, 2659 cmd->pad_bytes); 2660 2661 if (checksum != data_crc) { 2662 pr_err("ImmediateData CRC32C DataDigest 0x%08x" 2663 " does not match computed 0x%08x\n", checksum, 2664 data_crc); 2665 2666 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2667 pr_err("Unable to recover from" 2668 " Immediate Data digest failure while" 2669 " in ERL=0.\n"); 2670 iscsit_reject_cmd(cmd, 2671 ISCSI_REASON_DATA_DIGEST_ERROR, 2672 (unsigned char *)hdr); 2673 return IMMEDIATE_DATA_CANNOT_RECOVER; 2674 } else { 2675 iscsit_reject_cmd(cmd, 2676 ISCSI_REASON_DATA_DIGEST_ERROR, 2677 (unsigned char *)hdr); 2678 return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 2679 } 2680 } else { 2681 pr_debug("Got CRC32C DataDigest 0x%08x for" 2682 " %u bytes of Immediate Data\n", checksum, 2683 length); 2684 } 2685 } 2686 2687 cmd->write_data_done += length; 2688 2689 if (cmd->write_data_done == cmd->se_cmd.data_length) { 2690 spin_lock_bh(&cmd->istate_lock); 2691 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 2692 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 2693 spin_unlock_bh(&cmd->istate_lock); 2694 } 2695 2696 return IMMEDIATE_DATA_NORMAL_OPERATION; 2697 } 2698 2699 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections 2700 with active network interface */ 2701 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn) 2702 { 2703 struct iscsi_cmd *cmd; 2704 struct iscsi_conn *conn_p; 2705 bool found = false; 2706 2707 lockdep_assert_held(&conn->sess->conn_lock); 2708 2709 /* 2710 * Only send a Asynchronous Message on connections whos network 2711 * interface is still functional. 2712 */ 2713 list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) { 2714 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) { 2715 iscsit_inc_conn_usage_count(conn_p); 2716 found = true; 2717 break; 2718 } 2719 } 2720 2721 if (!found) 2722 return; 2723 2724 cmd = iscsit_allocate_cmd(conn_p, TASK_RUNNING); 2725 if (!cmd) { 2726 iscsit_dec_conn_usage_count(conn_p); 2727 return; 2728 } 2729 2730 cmd->logout_cid = conn->cid; 2731 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT; 2732 cmd->i_state = ISTATE_SEND_ASYNCMSG; 2733 2734 spin_lock_bh(&conn_p->cmd_lock); 2735 list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list); 2736 spin_unlock_bh(&conn_p->cmd_lock); 2737 2738 iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state); 2739 iscsit_dec_conn_usage_count(conn_p); 2740 } 2741 2742 static int iscsit_send_conn_drop_async_message( 2743 struct iscsi_cmd *cmd, 2744 struct iscsi_conn *conn) 2745 { 2746 struct iscsi_async *hdr; 2747 2748 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT; 2749 2750 hdr = (struct iscsi_async *) cmd->pdu; 2751 hdr->opcode = ISCSI_OP_ASYNC_EVENT; 2752 hdr->flags = ISCSI_FLAG_CMD_FINAL; 2753 cmd->init_task_tag = RESERVED_ITT; 2754 cmd->targ_xfer_tag = 0xFFFFFFFF; 2755 put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]); 2756 cmd->stat_sn = conn->stat_sn++; 2757 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2758 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2759 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 2760 hdr->async_event = ISCSI_ASYNC_MSG_DROPPING_CONNECTION; 2761 hdr->param1 = cpu_to_be16(cmd->logout_cid); 2762 hdr->param2 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait); 2763 hdr->param3 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain); 2764 2765 pr_debug("Sending Connection Dropped Async Message StatSN:" 2766 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn, 2767 cmd->logout_cid, conn->cid); 2768 2769 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 2770 } 2771 2772 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn) 2773 { 2774 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || 2775 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { 2776 wait_for_completion_interruptible_timeout( 2777 &conn->tx_half_close_comp, 2778 ISCSI_TX_THREAD_TCP_TIMEOUT * HZ); 2779 } 2780 } 2781 2782 void 2783 iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2784 struct iscsi_datain *datain, struct iscsi_data_rsp *hdr, 2785 bool set_statsn) 2786 { 2787 hdr->opcode = ISCSI_OP_SCSI_DATA_IN; 2788 hdr->flags = datain->flags; 2789 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) { 2790 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { 2791 hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW; 2792 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 2793 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { 2794 hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW; 2795 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 2796 } 2797 } 2798 hton24(hdr->dlength, datain->length); 2799 if (hdr->flags & ISCSI_FLAG_DATA_ACK) 2800 int_to_scsilun(cmd->se_cmd.orig_fe_lun, 2801 (struct scsi_lun *)&hdr->lun); 2802 else 2803 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); 2804 2805 hdr->itt = cmd->init_task_tag; 2806 2807 if (hdr->flags & ISCSI_FLAG_DATA_ACK) 2808 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 2809 else 2810 hdr->ttt = cpu_to_be32(0xFFFFFFFF); 2811 if (set_statsn) 2812 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2813 else 2814 hdr->statsn = cpu_to_be32(0xFFFFFFFF); 2815 2816 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2817 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 2818 hdr->datasn = cpu_to_be32(datain->data_sn); 2819 hdr->offset = cpu_to_be32(datain->offset); 2820 2821 pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x," 2822 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", 2823 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn), 2824 ntohl(hdr->offset), datain->length, conn->cid); 2825 } 2826 EXPORT_SYMBOL(iscsit_build_datain_pdu); 2827 2828 static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2829 { 2830 struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0]; 2831 struct iscsi_datain datain; 2832 struct iscsi_datain_req *dr; 2833 int eodr = 0, ret; 2834 bool set_statsn = false; 2835 2836 memset(&datain, 0, sizeof(struct iscsi_datain)); 2837 dr = iscsit_get_datain_values(cmd, &datain); 2838 if (!dr) { 2839 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n", 2840 cmd->init_task_tag); 2841 return -1; 2842 } 2843 /* 2844 * Be paranoid and double check the logic for now. 2845 */ 2846 if ((datain.offset + datain.length) > cmd->se_cmd.data_length) { 2847 pr_err("Command ITT: 0x%08x, datain.offset: %u and" 2848 " datain.length: %u exceeds cmd->data_length: %u\n", 2849 cmd->init_task_tag, datain.offset, datain.length, 2850 cmd->se_cmd.data_length); 2851 return -1; 2852 } 2853 2854 atomic_long_add(datain.length, &conn->sess->tx_data_octets); 2855 /* 2856 * Special case for successfully execution w/ both DATAIN 2857 * and Sense Data. 2858 */ 2859 if ((datain.flags & ISCSI_FLAG_DATA_STATUS) && 2860 (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)) 2861 datain.flags &= ~ISCSI_FLAG_DATA_STATUS; 2862 else { 2863 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) || 2864 (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) { 2865 iscsit_increment_maxcmdsn(cmd, conn->sess); 2866 cmd->stat_sn = conn->stat_sn++; 2867 set_statsn = true; 2868 } else if (dr->dr_complete == 2869 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY) 2870 set_statsn = true; 2871 } 2872 2873 iscsit_build_datain_pdu(cmd, conn, &datain, hdr, set_statsn); 2874 2875 ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, dr, &datain, 0); 2876 if (ret < 0) 2877 return ret; 2878 2879 if (dr->dr_complete) { 2880 eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ? 2881 2 : 1; 2882 iscsit_free_datain_req(cmd, dr); 2883 } 2884 2885 return eodr; 2886 } 2887 2888 int 2889 iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2890 struct iscsi_logout_rsp *hdr) 2891 { 2892 struct iscsi_conn *logout_conn = NULL; 2893 struct iscsi_conn_recovery *cr = NULL; 2894 struct iscsi_session *sess = conn->sess; 2895 /* 2896 * The actual shutting down of Sessions and/or Connections 2897 * for CLOSESESSION and CLOSECONNECTION Logout Requests 2898 * is done in scsi_logout_post_handler(). 2899 */ 2900 switch (cmd->logout_reason) { 2901 case ISCSI_LOGOUT_REASON_CLOSE_SESSION: 2902 pr_debug("iSCSI session logout successful, setting" 2903 " logout response to ISCSI_LOGOUT_SUCCESS.\n"); 2904 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2905 break; 2906 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: 2907 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND) 2908 break; 2909 /* 2910 * For CLOSECONNECTION logout requests carrying 2911 * a matching logout CID -> local CID, the reference 2912 * for the local CID will have been incremented in 2913 * iscsi_logout_closeconnection(). 2914 * 2915 * For CLOSECONNECTION logout requests carrying 2916 * a different CID than the connection it arrived 2917 * on, the connection responding to cmd->logout_cid 2918 * is stopped in iscsit_logout_post_handler_diffcid(). 2919 */ 2920 2921 pr_debug("iSCSI CID: %hu logout on CID: %hu" 2922 " successful.\n", cmd->logout_cid, conn->cid); 2923 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2924 break; 2925 case ISCSI_LOGOUT_REASON_RECOVERY: 2926 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) || 2927 (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED)) 2928 break; 2929 /* 2930 * If the connection is still active from our point of view 2931 * force connection recovery to occur. 2932 */ 2933 logout_conn = iscsit_get_conn_from_cid_rcfr(sess, 2934 cmd->logout_cid); 2935 if (logout_conn) { 2936 iscsit_connection_reinstatement_rcfr(logout_conn); 2937 iscsit_dec_conn_usage_count(logout_conn); 2938 } 2939 2940 cr = iscsit_get_inactive_connection_recovery_entry( 2941 conn->sess, cmd->logout_cid); 2942 if (!cr) { 2943 pr_err("Unable to locate CID: %hu for" 2944 " REMOVECONNFORRECOVERY Logout Request.\n", 2945 cmd->logout_cid); 2946 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND; 2947 break; 2948 } 2949 2950 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn); 2951 2952 pr_debug("iSCSI REMOVECONNFORRECOVERY logout" 2953 " for recovery for CID: %hu on CID: %hu successful.\n", 2954 cmd->logout_cid, conn->cid); 2955 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2956 break; 2957 default: 2958 pr_err("Unknown cmd->logout_reason: 0x%02x\n", 2959 cmd->logout_reason); 2960 return -1; 2961 } 2962 2963 hdr->opcode = ISCSI_OP_LOGOUT_RSP; 2964 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2965 hdr->response = cmd->logout_response; 2966 hdr->itt = cmd->init_task_tag; 2967 cmd->stat_sn = conn->stat_sn++; 2968 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2969 2970 iscsit_increment_maxcmdsn(cmd, conn->sess); 2971 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2972 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 2973 2974 pr_debug("Built Logout Response ITT: 0x%08x StatSN:" 2975 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n", 2976 cmd->init_task_tag, cmd->stat_sn, hdr->response, 2977 cmd->logout_cid, conn->cid); 2978 2979 return 0; 2980 } 2981 EXPORT_SYMBOL(iscsit_build_logout_rsp); 2982 2983 static int 2984 iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2985 { 2986 int rc; 2987 2988 rc = iscsit_build_logout_rsp(cmd, conn, 2989 (struct iscsi_logout_rsp *)&cmd->pdu[0]); 2990 if (rc < 0) 2991 return rc; 2992 2993 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 2994 } 2995 2996 void 2997 iscsit_build_nopin_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2998 struct iscsi_nopin *hdr, bool nopout_response) 2999 { 3000 hdr->opcode = ISCSI_OP_NOOP_IN; 3001 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3002 hton24(hdr->dlength, cmd->buf_ptr_size); 3003 if (nopout_response) 3004 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); 3005 hdr->itt = cmd->init_task_tag; 3006 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 3007 cmd->stat_sn = (nopout_response) ? conn->stat_sn++ : 3008 conn->stat_sn; 3009 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3010 3011 if (nopout_response) 3012 iscsit_increment_maxcmdsn(cmd, conn->sess); 3013 3014 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3015 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3016 3017 pr_debug("Built NOPIN %s Response ITT: 0x%08x, TTT: 0x%08x," 3018 " StatSN: 0x%08x, Length %u\n", (nopout_response) ? 3019 "Solicited" : "Unsolicited", cmd->init_task_tag, 3020 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size); 3021 } 3022 EXPORT_SYMBOL(iscsit_build_nopin_rsp); 3023 3024 /* 3025 * Unsolicited NOPIN, either requesting a response or not. 3026 */ 3027 static int iscsit_send_unsolicited_nopin( 3028 struct iscsi_cmd *cmd, 3029 struct iscsi_conn *conn, 3030 int want_response) 3031 { 3032 struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; 3033 int ret; 3034 3035 iscsit_build_nopin_rsp(cmd, conn, hdr, false); 3036 3037 pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:" 3038 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid); 3039 3040 ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 3041 if (ret < 0) 3042 return ret; 3043 3044 spin_lock_bh(&cmd->istate_lock); 3045 cmd->i_state = want_response ? 3046 ISTATE_SENT_NOPIN_WANT_RESPONSE : ISTATE_SENT_STATUS; 3047 spin_unlock_bh(&cmd->istate_lock); 3048 3049 return 0; 3050 } 3051 3052 static int 3053 iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3054 { 3055 struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; 3056 3057 iscsit_build_nopin_rsp(cmd, conn, hdr, true); 3058 3059 /* 3060 * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr. 3061 * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size. 3062 */ 3063 pr_debug("Echoing back %u bytes of ping data.\n", cmd->buf_ptr_size); 3064 3065 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 3066 cmd->buf_ptr, 3067 cmd->buf_ptr_size); 3068 } 3069 3070 static int iscsit_send_r2t( 3071 struct iscsi_cmd *cmd, 3072 struct iscsi_conn *conn) 3073 { 3074 struct iscsi_r2t *r2t; 3075 struct iscsi_r2t_rsp *hdr; 3076 int ret; 3077 3078 r2t = iscsit_get_r2t_from_list(cmd); 3079 if (!r2t) 3080 return -1; 3081 3082 hdr = (struct iscsi_r2t_rsp *) cmd->pdu; 3083 memset(hdr, 0, ISCSI_HDR_LEN); 3084 hdr->opcode = ISCSI_OP_R2T; 3085 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3086 int_to_scsilun(cmd->se_cmd.orig_fe_lun, 3087 (struct scsi_lun *)&hdr->lun); 3088 hdr->itt = cmd->init_task_tag; 3089 if (conn->conn_transport->iscsit_get_r2t_ttt) 3090 conn->conn_transport->iscsit_get_r2t_ttt(conn, cmd, r2t); 3091 else 3092 r2t->targ_xfer_tag = session_get_next_ttt(conn->sess); 3093 hdr->ttt = cpu_to_be32(r2t->targ_xfer_tag); 3094 hdr->statsn = cpu_to_be32(conn->stat_sn); 3095 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3096 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3097 hdr->r2tsn = cpu_to_be32(r2t->r2t_sn); 3098 hdr->data_offset = cpu_to_be32(r2t->offset); 3099 hdr->data_length = cpu_to_be32(r2t->xfer_len); 3100 3101 pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:" 3102 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n", 3103 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag, 3104 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn, 3105 r2t->offset, r2t->xfer_len, conn->cid); 3106 3107 spin_lock_bh(&cmd->r2t_lock); 3108 r2t->sent_r2t = 1; 3109 spin_unlock_bh(&cmd->r2t_lock); 3110 3111 ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 3112 if (ret < 0) { 3113 return ret; 3114 } 3115 3116 spin_lock_bh(&cmd->dataout_timeout_lock); 3117 iscsit_start_dataout_timer(cmd, conn); 3118 spin_unlock_bh(&cmd->dataout_timeout_lock); 3119 3120 return 0; 3121 } 3122 3123 /* 3124 * @recovery: If called from iscsi_task_reassign_complete_write() for 3125 * connection recovery. 3126 */ 3127 int iscsit_build_r2ts_for_cmd( 3128 struct iscsi_conn *conn, 3129 struct iscsi_cmd *cmd, 3130 bool recovery) 3131 { 3132 int first_r2t = 1; 3133 u32 offset = 0, xfer_len = 0; 3134 3135 spin_lock_bh(&cmd->r2t_lock); 3136 if (cmd->cmd_flags & ICF_SENT_LAST_R2T) { 3137 spin_unlock_bh(&cmd->r2t_lock); 3138 return 0; 3139 } 3140 3141 if (conn->sess->sess_ops->DataSequenceInOrder && 3142 !recovery) 3143 cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done); 3144 3145 while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) { 3146 if (conn->sess->sess_ops->DataSequenceInOrder) { 3147 offset = cmd->r2t_offset; 3148 3149 if (first_r2t && recovery) { 3150 int new_data_end = offset + 3151 conn->sess->sess_ops->MaxBurstLength - 3152 cmd->next_burst_len; 3153 3154 if (new_data_end > cmd->se_cmd.data_length) 3155 xfer_len = cmd->se_cmd.data_length - offset; 3156 else 3157 xfer_len = 3158 conn->sess->sess_ops->MaxBurstLength - 3159 cmd->next_burst_len; 3160 } else { 3161 int new_data_end = offset + 3162 conn->sess->sess_ops->MaxBurstLength; 3163 3164 if (new_data_end > cmd->se_cmd.data_length) 3165 xfer_len = cmd->se_cmd.data_length - offset; 3166 else 3167 xfer_len = conn->sess->sess_ops->MaxBurstLength; 3168 } 3169 3170 if ((s32)xfer_len < 0) { 3171 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 3172 break; 3173 } 3174 3175 cmd->r2t_offset += xfer_len; 3176 3177 if (cmd->r2t_offset == cmd->se_cmd.data_length) 3178 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 3179 } else { 3180 struct iscsi_seq *seq; 3181 3182 seq = iscsit_get_seq_holder_for_r2t(cmd); 3183 if (!seq) { 3184 spin_unlock_bh(&cmd->r2t_lock); 3185 return -1; 3186 } 3187 3188 offset = seq->offset; 3189 xfer_len = seq->xfer_len; 3190 3191 if (cmd->seq_send_order == cmd->seq_count) 3192 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 3193 } 3194 cmd->outstanding_r2ts++; 3195 first_r2t = 0; 3196 3197 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) { 3198 spin_unlock_bh(&cmd->r2t_lock); 3199 return -1; 3200 } 3201 3202 if (cmd->cmd_flags & ICF_SENT_LAST_R2T) 3203 break; 3204 } 3205 spin_unlock_bh(&cmd->r2t_lock); 3206 3207 return 0; 3208 } 3209 EXPORT_SYMBOL(iscsit_build_r2ts_for_cmd); 3210 3211 void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3212 bool inc_stat_sn, struct iscsi_scsi_rsp *hdr) 3213 { 3214 if (inc_stat_sn) 3215 cmd->stat_sn = conn->stat_sn++; 3216 3217 atomic_long_inc(&conn->sess->rsp_pdus); 3218 3219 memset(hdr, 0, ISCSI_HDR_LEN); 3220 hdr->opcode = ISCSI_OP_SCSI_CMD_RSP; 3221 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3222 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { 3223 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW; 3224 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 3225 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { 3226 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW; 3227 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 3228 } 3229 hdr->response = cmd->iscsi_response; 3230 hdr->cmd_status = cmd->se_cmd.scsi_status; 3231 hdr->itt = cmd->init_task_tag; 3232 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3233 3234 iscsit_increment_maxcmdsn(cmd, conn->sess); 3235 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3236 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3237 3238 pr_debug("Built SCSI Response, ITT: 0x%08x, StatSN: 0x%08x," 3239 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n", 3240 cmd->init_task_tag, cmd->stat_sn, cmd->se_cmd.scsi_status, 3241 cmd->se_cmd.scsi_status, conn->cid); 3242 } 3243 EXPORT_SYMBOL(iscsit_build_rsp_pdu); 3244 3245 static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3246 { 3247 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0]; 3248 bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS); 3249 void *data_buf = NULL; 3250 u32 padding = 0, data_buf_len = 0; 3251 3252 iscsit_build_rsp_pdu(cmd, conn, inc_stat_sn, hdr); 3253 3254 /* 3255 * Attach SENSE DATA payload to iSCSI Response PDU 3256 */ 3257 if (cmd->se_cmd.sense_buffer && 3258 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 3259 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 3260 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, cmd->sense_buffer); 3261 cmd->se_cmd.scsi_sense_length += sizeof (__be16); 3262 3263 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 3264 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 3265 data_buf = cmd->sense_buffer; 3266 data_buf_len = cmd->se_cmd.scsi_sense_length + padding; 3267 3268 if (padding) { 3269 memset(cmd->sense_buffer + 3270 cmd->se_cmd.scsi_sense_length, 0, padding); 3271 pr_debug("Adding %u bytes of padding to" 3272 " SENSE.\n", padding); 3273 } 3274 3275 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI" 3276 " Response PDU\n", 3277 cmd->se_cmd.scsi_sense_length); 3278 } 3279 3280 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, data_buf, 3281 data_buf_len); 3282 } 3283 3284 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr) 3285 { 3286 switch (se_tmr->response) { 3287 case TMR_FUNCTION_COMPLETE: 3288 return ISCSI_TMF_RSP_COMPLETE; 3289 case TMR_TASK_DOES_NOT_EXIST: 3290 return ISCSI_TMF_RSP_NO_TASK; 3291 case TMR_LUN_DOES_NOT_EXIST: 3292 return ISCSI_TMF_RSP_NO_LUN; 3293 case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED: 3294 return ISCSI_TMF_RSP_NOT_SUPPORTED; 3295 case TMR_FUNCTION_REJECTED: 3296 default: 3297 return ISCSI_TMF_RSP_REJECTED; 3298 } 3299 } 3300 3301 void 3302 iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3303 struct iscsi_tm_rsp *hdr) 3304 { 3305 struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req; 3306 3307 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP; 3308 hdr->flags = ISCSI_FLAG_CMD_FINAL; 3309 hdr->response = iscsit_convert_tcm_tmr_rsp(se_tmr); 3310 hdr->itt = cmd->init_task_tag; 3311 cmd->stat_sn = conn->stat_sn++; 3312 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3313 3314 iscsit_increment_maxcmdsn(cmd, conn->sess); 3315 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3316 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3317 3318 pr_debug("Built Task Management Response ITT: 0x%08x," 3319 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n", 3320 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid); 3321 } 3322 EXPORT_SYMBOL(iscsit_build_task_mgt_rsp); 3323 3324 static int 3325 iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3326 { 3327 struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0]; 3328 3329 iscsit_build_task_mgt_rsp(cmd, conn, hdr); 3330 3331 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 3332 } 3333 3334 #define SENDTARGETS_BUF_LIMIT 32768U 3335 3336 static int 3337 iscsit_build_sendtargets_response(struct iscsi_cmd *cmd, 3338 enum iscsit_transport_type network_transport, 3339 int skip_bytes, bool *completed) 3340 { 3341 char *payload = NULL; 3342 struct iscsi_conn *conn = cmd->conn; 3343 struct iscsi_portal_group *tpg; 3344 struct iscsi_tiqn *tiqn; 3345 struct iscsi_tpg_np *tpg_np; 3346 int buffer_len, end_of_buf = 0, len = 0, payload_len = 0; 3347 int target_name_printed; 3348 unsigned char buf[ISCSI_IQN_LEN+12]; /* iqn + "TargetName=" + \0 */ 3349 unsigned char *text_in = cmd->text_in_ptr, *text_ptr = NULL; 3350 bool active; 3351 3352 buffer_len = min(conn->conn_ops->MaxRecvDataSegmentLength, 3353 SENDTARGETS_BUF_LIMIT); 3354 3355 payload = kzalloc(buffer_len, GFP_KERNEL); 3356 if (!payload) 3357 return -ENOMEM; 3358 3359 /* 3360 * Locate pointer to iqn./eui. string for ICF_SENDTARGETS_SINGLE 3361 * explicit case.. 3362 */ 3363 if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) { 3364 text_ptr = strchr(text_in, '='); 3365 if (!text_ptr) { 3366 pr_err("Unable to locate '=' string in text_in:" 3367 " %s\n", text_in); 3368 kfree(payload); 3369 return -EINVAL; 3370 } 3371 /* 3372 * Skip over '=' character.. 3373 */ 3374 text_ptr += 1; 3375 } 3376 3377 spin_lock(&tiqn_lock); 3378 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) { 3379 if ((cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) && 3380 strcmp(tiqn->tiqn, text_ptr)) { 3381 continue; 3382 } 3383 3384 target_name_printed = 0; 3385 3386 spin_lock(&tiqn->tiqn_tpg_lock); 3387 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) { 3388 3389 /* If demo_mode_discovery=0 and generate_node_acls=0 3390 * (demo mode dislabed) do not return 3391 * TargetName+TargetAddress unless a NodeACL exists. 3392 */ 3393 3394 if ((tpg->tpg_attrib.generate_node_acls == 0) && 3395 (tpg->tpg_attrib.demo_mode_discovery == 0) && 3396 (!target_tpg_has_node_acl(&tpg->tpg_se_tpg, 3397 cmd->conn->sess->sess_ops->InitiatorName))) { 3398 continue; 3399 } 3400 3401 spin_lock(&tpg->tpg_state_lock); 3402 active = (tpg->tpg_state == TPG_STATE_ACTIVE); 3403 spin_unlock(&tpg->tpg_state_lock); 3404 3405 if (!active && tpg->tpg_attrib.tpg_enabled_sendtargets) 3406 continue; 3407 3408 spin_lock(&tpg->tpg_np_lock); 3409 list_for_each_entry(tpg_np, &tpg->tpg_gnp_list, 3410 tpg_np_list) { 3411 struct iscsi_np *np = tpg_np->tpg_np; 3412 struct sockaddr_storage *sockaddr; 3413 3414 if (np->np_network_transport != network_transport) 3415 continue; 3416 3417 if (!target_name_printed) { 3418 len = sprintf(buf, "TargetName=%s", 3419 tiqn->tiqn); 3420 len += 1; 3421 3422 if ((len + payload_len) > buffer_len) { 3423 spin_unlock(&tpg->tpg_np_lock); 3424 spin_unlock(&tiqn->tiqn_tpg_lock); 3425 end_of_buf = 1; 3426 goto eob; 3427 } 3428 3429 if (skip_bytes && len <= skip_bytes) { 3430 skip_bytes -= len; 3431 } else { 3432 memcpy(payload + payload_len, buf, len); 3433 payload_len += len; 3434 target_name_printed = 1; 3435 if (len > skip_bytes) 3436 skip_bytes = 0; 3437 } 3438 } 3439 3440 if (inet_addr_is_any((struct sockaddr *)&np->np_sockaddr)) 3441 sockaddr = &conn->local_sockaddr; 3442 else 3443 sockaddr = &np->np_sockaddr; 3444 3445 len = sprintf(buf, "TargetAddress=" 3446 "%pISpc,%hu", 3447 sockaddr, 3448 tpg->tpgt); 3449 len += 1; 3450 3451 if ((len + payload_len) > buffer_len) { 3452 spin_unlock(&tpg->tpg_np_lock); 3453 spin_unlock(&tiqn->tiqn_tpg_lock); 3454 end_of_buf = 1; 3455 goto eob; 3456 } 3457 3458 if (skip_bytes && len <= skip_bytes) { 3459 skip_bytes -= len; 3460 } else { 3461 memcpy(payload + payload_len, buf, len); 3462 payload_len += len; 3463 if (len > skip_bytes) 3464 skip_bytes = 0; 3465 } 3466 } 3467 spin_unlock(&tpg->tpg_np_lock); 3468 } 3469 spin_unlock(&tiqn->tiqn_tpg_lock); 3470 eob: 3471 if (end_of_buf) { 3472 *completed = false; 3473 break; 3474 } 3475 3476 if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) 3477 break; 3478 } 3479 spin_unlock(&tiqn_lock); 3480 3481 cmd->buf_ptr = payload; 3482 3483 return payload_len; 3484 } 3485 3486 int 3487 iscsit_build_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3488 struct iscsi_text_rsp *hdr, 3489 enum iscsit_transport_type network_transport) 3490 { 3491 int text_length, padding; 3492 bool completed = true; 3493 3494 text_length = iscsit_build_sendtargets_response(cmd, network_transport, 3495 cmd->read_data_done, 3496 &completed); 3497 if (text_length < 0) 3498 return text_length; 3499 3500 if (completed) { 3501 hdr->flags = ISCSI_FLAG_CMD_FINAL; 3502 } else { 3503 hdr->flags = ISCSI_FLAG_TEXT_CONTINUE; 3504 cmd->read_data_done += text_length; 3505 if (cmd->targ_xfer_tag == 0xFFFFFFFF) 3506 cmd->targ_xfer_tag = session_get_next_ttt(conn->sess); 3507 } 3508 hdr->opcode = ISCSI_OP_TEXT_RSP; 3509 padding = ((-text_length) & 3); 3510 hton24(hdr->dlength, text_length); 3511 hdr->itt = cmd->init_task_tag; 3512 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 3513 cmd->stat_sn = conn->stat_sn++; 3514 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3515 3516 iscsit_increment_maxcmdsn(cmd, conn->sess); 3517 /* 3518 * Reset maxcmdsn_inc in multi-part text payload exchanges to 3519 * correctly increment MaxCmdSN for each response answering a 3520 * non immediate text request with a valid CmdSN. 3521 */ 3522 cmd->maxcmdsn_inc = 0; 3523 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3524 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3525 3526 pr_debug("Built Text Response: ITT: 0x%08x, TTT: 0x%08x, StatSN: 0x%08x," 3527 " Length: %u, CID: %hu F: %d C: %d\n", cmd->init_task_tag, 3528 cmd->targ_xfer_tag, cmd->stat_sn, text_length, conn->cid, 3529 !!(hdr->flags & ISCSI_FLAG_CMD_FINAL), 3530 !!(hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)); 3531 3532 return text_length + padding; 3533 } 3534 EXPORT_SYMBOL(iscsit_build_text_rsp); 3535 3536 static int iscsit_send_text_rsp( 3537 struct iscsi_cmd *cmd, 3538 struct iscsi_conn *conn) 3539 { 3540 struct iscsi_text_rsp *hdr = (struct iscsi_text_rsp *)cmd->pdu; 3541 int text_length; 3542 3543 text_length = iscsit_build_text_rsp(cmd, conn, hdr, 3544 conn->conn_transport->transport_type); 3545 if (text_length < 0) 3546 return text_length; 3547 3548 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 3549 cmd->buf_ptr, 3550 text_length); 3551 } 3552 3553 void 3554 iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3555 struct iscsi_reject *hdr) 3556 { 3557 hdr->opcode = ISCSI_OP_REJECT; 3558 hdr->reason = cmd->reject_reason; 3559 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3560 hton24(hdr->dlength, ISCSI_HDR_LEN); 3561 hdr->ffffffff = cpu_to_be32(0xffffffff); 3562 cmd->stat_sn = conn->stat_sn++; 3563 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3564 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3565 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3566 3567 } 3568 EXPORT_SYMBOL(iscsit_build_reject); 3569 3570 static int iscsit_send_reject( 3571 struct iscsi_cmd *cmd, 3572 struct iscsi_conn *conn) 3573 { 3574 struct iscsi_reject *hdr = (struct iscsi_reject *)&cmd->pdu[0]; 3575 3576 iscsit_build_reject(cmd, conn, hdr); 3577 3578 pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x," 3579 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid); 3580 3581 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 3582 cmd->buf_ptr, 3583 ISCSI_HDR_LEN); 3584 } 3585 3586 void iscsit_thread_get_cpumask(struct iscsi_conn *conn) 3587 { 3588 int ord, cpu; 3589 /* 3590 * bitmap_id is assigned from iscsit_global->ts_bitmap from 3591 * within iscsit_start_kthreads() 3592 * 3593 * Here we use bitmap_id to determine which CPU that this 3594 * iSCSI connection's RX/TX threads will be scheduled to 3595 * execute upon. 3596 */ 3597 ord = conn->bitmap_id % cpumask_weight(cpu_online_mask); 3598 for_each_online_cpu(cpu) { 3599 if (ord-- == 0) { 3600 cpumask_set_cpu(cpu, conn->conn_cpumask); 3601 return; 3602 } 3603 } 3604 /* 3605 * This should never be reached.. 3606 */ 3607 dump_stack(); 3608 cpumask_setall(conn->conn_cpumask); 3609 } 3610 3611 int 3612 iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3613 { 3614 int ret; 3615 3616 switch (state) { 3617 case ISTATE_SEND_R2T: 3618 ret = iscsit_send_r2t(cmd, conn); 3619 if (ret < 0) 3620 goto err; 3621 break; 3622 case ISTATE_REMOVE: 3623 spin_lock_bh(&conn->cmd_lock); 3624 list_del_init(&cmd->i_conn_node); 3625 spin_unlock_bh(&conn->cmd_lock); 3626 3627 iscsit_free_cmd(cmd, false); 3628 break; 3629 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 3630 iscsit_mod_nopin_response_timer(conn); 3631 ret = iscsit_send_unsolicited_nopin(cmd, conn, 1); 3632 if (ret < 0) 3633 goto err; 3634 break; 3635 case ISTATE_SEND_NOPIN_NO_RESPONSE: 3636 ret = iscsit_send_unsolicited_nopin(cmd, conn, 0); 3637 if (ret < 0) 3638 goto err; 3639 break; 3640 default: 3641 pr_err("Unknown Opcode: 0x%02x ITT:" 3642 " 0x%08x, i_state: %d on CID: %hu\n", 3643 cmd->iscsi_opcode, cmd->init_task_tag, state, 3644 conn->cid); 3645 goto err; 3646 } 3647 3648 return 0; 3649 3650 err: 3651 return -1; 3652 } 3653 EXPORT_SYMBOL(iscsit_immediate_queue); 3654 3655 static int 3656 iscsit_handle_immediate_queue(struct iscsi_conn *conn) 3657 { 3658 struct iscsit_transport *t = conn->conn_transport; 3659 struct iscsi_queue_req *qr; 3660 struct iscsi_cmd *cmd; 3661 u8 state; 3662 int ret; 3663 3664 while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) { 3665 atomic_set(&conn->check_immediate_queue, 0); 3666 cmd = qr->cmd; 3667 state = qr->state; 3668 kmem_cache_free(lio_qr_cache, qr); 3669 3670 ret = t->iscsit_immediate_queue(conn, cmd, state); 3671 if (ret < 0) 3672 return ret; 3673 } 3674 3675 return 0; 3676 } 3677 3678 int 3679 iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3680 { 3681 int ret; 3682 3683 check_rsp_state: 3684 switch (state) { 3685 case ISTATE_SEND_DATAIN: 3686 ret = iscsit_send_datain(cmd, conn); 3687 if (ret < 0) 3688 goto err; 3689 else if (!ret) 3690 /* more drs */ 3691 goto check_rsp_state; 3692 else if (ret == 1) { 3693 /* all done */ 3694 spin_lock_bh(&cmd->istate_lock); 3695 cmd->i_state = ISTATE_SENT_STATUS; 3696 spin_unlock_bh(&cmd->istate_lock); 3697 3698 if (atomic_read(&conn->check_immediate_queue)) 3699 return 1; 3700 3701 return 0; 3702 } else if (ret == 2) { 3703 /* Still must send status, 3704 SCF_TRANSPORT_TASK_SENSE was set */ 3705 spin_lock_bh(&cmd->istate_lock); 3706 cmd->i_state = ISTATE_SEND_STATUS; 3707 spin_unlock_bh(&cmd->istate_lock); 3708 state = ISTATE_SEND_STATUS; 3709 goto check_rsp_state; 3710 } 3711 3712 break; 3713 case ISTATE_SEND_STATUS: 3714 case ISTATE_SEND_STATUS_RECOVERY: 3715 ret = iscsit_send_response(cmd, conn); 3716 break; 3717 case ISTATE_SEND_LOGOUTRSP: 3718 ret = iscsit_send_logout(cmd, conn); 3719 break; 3720 case ISTATE_SEND_ASYNCMSG: 3721 ret = iscsit_send_conn_drop_async_message( 3722 cmd, conn); 3723 break; 3724 case ISTATE_SEND_NOPIN: 3725 ret = iscsit_send_nopin(cmd, conn); 3726 break; 3727 case ISTATE_SEND_REJECT: 3728 ret = iscsit_send_reject(cmd, conn); 3729 break; 3730 case ISTATE_SEND_TASKMGTRSP: 3731 ret = iscsit_send_task_mgt_rsp(cmd, conn); 3732 if (ret != 0) 3733 break; 3734 ret = iscsit_tmr_post_handler(cmd, conn); 3735 if (ret != 0) 3736 iscsit_fall_back_to_erl0(conn->sess); 3737 break; 3738 case ISTATE_SEND_TEXTRSP: 3739 ret = iscsit_send_text_rsp(cmd, conn); 3740 break; 3741 default: 3742 pr_err("Unknown Opcode: 0x%02x ITT:" 3743 " 0x%08x, i_state: %d on CID: %hu\n", 3744 cmd->iscsi_opcode, cmd->init_task_tag, 3745 state, conn->cid); 3746 goto err; 3747 } 3748 if (ret < 0) 3749 goto err; 3750 3751 switch (state) { 3752 case ISTATE_SEND_LOGOUTRSP: 3753 if (!iscsit_logout_post_handler(cmd, conn)) 3754 return -ECONNRESET; 3755 fallthrough; 3756 case ISTATE_SEND_STATUS: 3757 case ISTATE_SEND_ASYNCMSG: 3758 case ISTATE_SEND_NOPIN: 3759 case ISTATE_SEND_STATUS_RECOVERY: 3760 case ISTATE_SEND_TEXTRSP: 3761 case ISTATE_SEND_TASKMGTRSP: 3762 case ISTATE_SEND_REJECT: 3763 spin_lock_bh(&cmd->istate_lock); 3764 cmd->i_state = ISTATE_SENT_STATUS; 3765 spin_unlock_bh(&cmd->istate_lock); 3766 break; 3767 default: 3768 pr_err("Unknown Opcode: 0x%02x ITT:" 3769 " 0x%08x, i_state: %d on CID: %hu\n", 3770 cmd->iscsi_opcode, cmd->init_task_tag, 3771 cmd->i_state, conn->cid); 3772 goto err; 3773 } 3774 3775 if (atomic_read(&conn->check_immediate_queue)) 3776 return 1; 3777 3778 return 0; 3779 3780 err: 3781 return -1; 3782 } 3783 EXPORT_SYMBOL(iscsit_response_queue); 3784 3785 static int iscsit_handle_response_queue(struct iscsi_conn *conn) 3786 { 3787 struct iscsit_transport *t = conn->conn_transport; 3788 struct iscsi_queue_req *qr; 3789 struct iscsi_cmd *cmd; 3790 u8 state; 3791 int ret; 3792 3793 while ((qr = iscsit_get_cmd_from_response_queue(conn))) { 3794 cmd = qr->cmd; 3795 state = qr->state; 3796 kmem_cache_free(lio_qr_cache, qr); 3797 3798 ret = t->iscsit_response_queue(conn, cmd, state); 3799 if (ret == 1 || ret < 0) 3800 return ret; 3801 } 3802 3803 return 0; 3804 } 3805 3806 int iscsi_target_tx_thread(void *arg) 3807 { 3808 int ret = 0; 3809 struct iscsi_conn *conn = arg; 3810 bool conn_freed = false; 3811 3812 /* 3813 * Allow ourselves to be interrupted by SIGINT so that a 3814 * connection recovery / failure event can be triggered externally. 3815 */ 3816 allow_signal(SIGINT); 3817 3818 while (!kthread_should_stop()) { 3819 /* 3820 * Ensure that both TX and RX per connection kthreads 3821 * are scheduled to run on the same CPU. 3822 */ 3823 iscsit_thread_check_cpumask(conn, current, 1); 3824 3825 wait_event_interruptible(conn->queues_wq, 3826 !iscsit_conn_all_queues_empty(conn)); 3827 3828 if (signal_pending(current)) 3829 goto transport_err; 3830 3831 get_immediate: 3832 ret = iscsit_handle_immediate_queue(conn); 3833 if (ret < 0) 3834 goto transport_err; 3835 3836 ret = iscsit_handle_response_queue(conn); 3837 if (ret == 1) { 3838 goto get_immediate; 3839 } else if (ret == -ECONNRESET) { 3840 conn_freed = true; 3841 goto out; 3842 } else if (ret < 0) { 3843 goto transport_err; 3844 } 3845 } 3846 3847 transport_err: 3848 /* 3849 * Avoid the normal connection failure code-path if this connection 3850 * is still within LOGIN mode, and iscsi_np process context is 3851 * responsible for cleaning up the early connection failure. 3852 */ 3853 if (conn->conn_state != TARG_CONN_STATE_IN_LOGIN) 3854 iscsit_take_action_for_connection_exit(conn, &conn_freed); 3855 out: 3856 if (!conn_freed) { 3857 while (!kthread_should_stop()) { 3858 msleep(100); 3859 } 3860 } 3861 return 0; 3862 } 3863 3864 static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf) 3865 { 3866 struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf; 3867 struct iscsi_cmd *cmd; 3868 int ret = 0; 3869 3870 switch (hdr->opcode & ISCSI_OPCODE_MASK) { 3871 case ISCSI_OP_SCSI_CMD: 3872 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3873 if (!cmd) 3874 goto reject; 3875 3876 ret = iscsit_handle_scsi_cmd(conn, cmd, buf); 3877 break; 3878 case ISCSI_OP_SCSI_DATA_OUT: 3879 ret = iscsit_handle_data_out(conn, buf); 3880 break; 3881 case ISCSI_OP_NOOP_OUT: 3882 cmd = NULL; 3883 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 3884 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3885 if (!cmd) 3886 goto reject; 3887 } 3888 ret = iscsit_handle_nop_out(conn, cmd, buf); 3889 break; 3890 case ISCSI_OP_SCSI_TMFUNC: 3891 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3892 if (!cmd) 3893 goto reject; 3894 3895 ret = iscsit_handle_task_mgt_cmd(conn, cmd, buf); 3896 break; 3897 case ISCSI_OP_TEXT: 3898 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 3899 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 3900 if (!cmd) 3901 goto reject; 3902 } else { 3903 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3904 if (!cmd) 3905 goto reject; 3906 } 3907 3908 ret = iscsit_handle_text_cmd(conn, cmd, buf); 3909 break; 3910 case ISCSI_OP_LOGOUT: 3911 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3912 if (!cmd) 3913 goto reject; 3914 3915 ret = iscsit_handle_logout_cmd(conn, cmd, buf); 3916 if (ret > 0) 3917 wait_for_completion_timeout(&conn->conn_logout_comp, 3918 SECONDS_FOR_LOGOUT_COMP * HZ); 3919 break; 3920 case ISCSI_OP_SNACK: 3921 ret = iscsit_handle_snack(conn, buf); 3922 break; 3923 default: 3924 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", hdr->opcode); 3925 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 3926 pr_err("Cannot recover from unknown" 3927 " opcode while ERL=0, closing iSCSI connection.\n"); 3928 return -1; 3929 } 3930 pr_err("Unable to recover from unknown opcode while OFMarker=No," 3931 " closing iSCSI connection.\n"); 3932 ret = -1; 3933 break; 3934 } 3935 3936 return ret; 3937 reject: 3938 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 3939 } 3940 3941 static bool iscsi_target_check_conn_state(struct iscsi_conn *conn) 3942 { 3943 bool ret; 3944 3945 spin_lock_bh(&conn->state_lock); 3946 ret = (conn->conn_state != TARG_CONN_STATE_LOGGED_IN); 3947 spin_unlock_bh(&conn->state_lock); 3948 3949 return ret; 3950 } 3951 3952 static void iscsit_get_rx_pdu(struct iscsi_conn *conn) 3953 { 3954 int ret; 3955 u8 *buffer, opcode; 3956 u32 checksum = 0, digest = 0; 3957 struct kvec iov; 3958 3959 buffer = kcalloc(ISCSI_HDR_LEN, sizeof(*buffer), GFP_KERNEL); 3960 if (!buffer) 3961 return; 3962 3963 while (!kthread_should_stop()) { 3964 /* 3965 * Ensure that both TX and RX per connection kthreads 3966 * are scheduled to run on the same CPU. 3967 */ 3968 iscsit_thread_check_cpumask(conn, current, 0); 3969 3970 memset(&iov, 0, sizeof(struct kvec)); 3971 3972 iov.iov_base = buffer; 3973 iov.iov_len = ISCSI_HDR_LEN; 3974 3975 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN); 3976 if (ret != ISCSI_HDR_LEN) { 3977 iscsit_rx_thread_wait_for_tcp(conn); 3978 break; 3979 } 3980 3981 if (conn->conn_ops->HeaderDigest) { 3982 iov.iov_base = &digest; 3983 iov.iov_len = ISCSI_CRC_LEN; 3984 3985 ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN); 3986 if (ret != ISCSI_CRC_LEN) { 3987 iscsit_rx_thread_wait_for_tcp(conn); 3988 break; 3989 } 3990 3991 iscsit_do_crypto_hash_buf(conn->conn_rx_hash, buffer, 3992 ISCSI_HDR_LEN, 0, NULL, 3993 &checksum); 3994 3995 if (digest != checksum) { 3996 pr_err("HeaderDigest CRC32C failed," 3997 " received 0x%08x, computed 0x%08x\n", 3998 digest, checksum); 3999 /* 4000 * Set the PDU to 0xff so it will intentionally 4001 * hit default in the switch below. 4002 */ 4003 memset(buffer, 0xff, ISCSI_HDR_LEN); 4004 atomic_long_inc(&conn->sess->conn_digest_errors); 4005 } else { 4006 pr_debug("Got HeaderDigest CRC32C" 4007 " 0x%08x\n", checksum); 4008 } 4009 } 4010 4011 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 4012 break; 4013 4014 opcode = buffer[0] & ISCSI_OPCODE_MASK; 4015 4016 if (conn->sess->sess_ops->SessionType && 4017 ((!(opcode & ISCSI_OP_TEXT)) || 4018 (!(opcode & ISCSI_OP_LOGOUT)))) { 4019 pr_err("Received illegal iSCSI Opcode: 0x%02x" 4020 " while in Discovery Session, rejecting.\n", opcode); 4021 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 4022 buffer); 4023 break; 4024 } 4025 4026 ret = iscsi_target_rx_opcode(conn, buffer); 4027 if (ret < 0) 4028 break; 4029 } 4030 4031 kfree(buffer); 4032 } 4033 4034 int iscsi_target_rx_thread(void *arg) 4035 { 4036 int rc; 4037 struct iscsi_conn *conn = arg; 4038 bool conn_freed = false; 4039 4040 /* 4041 * Allow ourselves to be interrupted by SIGINT so that a 4042 * connection recovery / failure event can be triggered externally. 4043 */ 4044 allow_signal(SIGINT); 4045 /* 4046 * Wait for iscsi_post_login_handler() to complete before allowing 4047 * incoming iscsi/tcp socket I/O, and/or failing the connection. 4048 */ 4049 rc = wait_for_completion_interruptible(&conn->rx_login_comp); 4050 if (rc < 0 || iscsi_target_check_conn_state(conn)) 4051 goto out; 4052 4053 if (!conn->conn_transport->iscsit_get_rx_pdu) 4054 return 0; 4055 4056 conn->conn_transport->iscsit_get_rx_pdu(conn); 4057 4058 if (!signal_pending(current)) 4059 atomic_set(&conn->transport_failed, 1); 4060 iscsit_take_action_for_connection_exit(conn, &conn_freed); 4061 4062 out: 4063 if (!conn_freed) { 4064 while (!kthread_should_stop()) { 4065 msleep(100); 4066 } 4067 } 4068 4069 return 0; 4070 } 4071 4072 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn) 4073 { 4074 LIST_HEAD(tmp_list); 4075 struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL; 4076 struct iscsi_session *sess = conn->sess; 4077 /* 4078 * We expect this function to only ever be called from either RX or TX 4079 * thread context via iscsit_close_connection() once the other context 4080 * has been reset -> returned sleeping pre-handler state. 4081 */ 4082 spin_lock_bh(&conn->cmd_lock); 4083 list_splice_init(&conn->conn_cmd_list, &tmp_list); 4084 4085 list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) { 4086 struct se_cmd *se_cmd = &cmd->se_cmd; 4087 4088 if (se_cmd->se_tfo != NULL) { 4089 spin_lock_irq(&se_cmd->t_state_lock); 4090 if (se_cmd->transport_state & CMD_T_ABORTED) { 4091 /* 4092 * LIO's abort path owns the cleanup for this, 4093 * so put it back on the list and let 4094 * aborted_task handle it. 4095 */ 4096 list_move_tail(&cmd->i_conn_node, 4097 &conn->conn_cmd_list); 4098 } else { 4099 se_cmd->transport_state |= CMD_T_FABRIC_STOP; 4100 } 4101 spin_unlock_irq(&se_cmd->t_state_lock); 4102 } 4103 } 4104 spin_unlock_bh(&conn->cmd_lock); 4105 4106 list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) { 4107 list_del_init(&cmd->i_conn_node); 4108 4109 iscsit_increment_maxcmdsn(cmd, sess); 4110 iscsit_free_cmd(cmd, true); 4111 4112 } 4113 } 4114 4115 static void iscsit_stop_timers_for_cmds( 4116 struct iscsi_conn *conn) 4117 { 4118 struct iscsi_cmd *cmd; 4119 4120 spin_lock_bh(&conn->cmd_lock); 4121 list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) { 4122 if (cmd->data_direction == DMA_TO_DEVICE) 4123 iscsit_stop_dataout_timer(cmd); 4124 } 4125 spin_unlock_bh(&conn->cmd_lock); 4126 } 4127 4128 int iscsit_close_connection( 4129 struct iscsi_conn *conn) 4130 { 4131 int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT); 4132 struct iscsi_session *sess = conn->sess; 4133 4134 pr_debug("Closing iSCSI connection CID %hu on SID:" 4135 " %u\n", conn->cid, sess->sid); 4136 /* 4137 * Always up conn_logout_comp for the traditional TCP and HW_OFFLOAD 4138 * case just in case the RX Thread in iscsi_target_rx_opcode() is 4139 * sleeping and the logout response never got sent because the 4140 * connection failed. 4141 * 4142 * However for iser-target, isert_wait4logout() is using conn_logout_comp 4143 * to signal logout response TX interrupt completion. Go ahead and skip 4144 * this for iser since isert_rx_opcode() does not wait on logout failure, 4145 * and to avoid iscsi_conn pointer dereference in iser-target code. 4146 */ 4147 if (!conn->conn_transport->rdma_shutdown) 4148 complete(&conn->conn_logout_comp); 4149 4150 if (!strcmp(current->comm, ISCSI_RX_THREAD_NAME)) { 4151 if (conn->tx_thread && 4152 cmpxchg(&conn->tx_thread_active, true, false)) { 4153 send_sig(SIGINT, conn->tx_thread, 1); 4154 kthread_stop(conn->tx_thread); 4155 } 4156 } else if (!strcmp(current->comm, ISCSI_TX_THREAD_NAME)) { 4157 if (conn->rx_thread && 4158 cmpxchg(&conn->rx_thread_active, true, false)) { 4159 send_sig(SIGINT, conn->rx_thread, 1); 4160 kthread_stop(conn->rx_thread); 4161 } 4162 } 4163 4164 spin_lock(&iscsit_global->ts_bitmap_lock); 4165 bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id, 4166 get_order(1)); 4167 spin_unlock(&iscsit_global->ts_bitmap_lock); 4168 4169 iscsit_stop_timers_for_cmds(conn); 4170 iscsit_stop_nopin_response_timer(conn); 4171 iscsit_stop_nopin_timer(conn); 4172 4173 if (conn->conn_transport->iscsit_wait_conn) 4174 conn->conn_transport->iscsit_wait_conn(conn); 4175 4176 /* 4177 * During Connection recovery drop unacknowledged out of order 4178 * commands for this connection, and prepare the other commands 4179 * for reallegiance. 4180 * 4181 * During normal operation clear the out of order commands (but 4182 * do not free the struct iscsi_ooo_cmdsn's) and release all 4183 * struct iscsi_cmds. 4184 */ 4185 if (atomic_read(&conn->connection_recovery)) { 4186 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn); 4187 iscsit_prepare_cmds_for_reallegiance(conn); 4188 } else { 4189 iscsit_clear_ooo_cmdsns_for_conn(conn); 4190 iscsit_release_commands_from_conn(conn); 4191 } 4192 iscsit_free_queue_reqs_for_conn(conn); 4193 4194 /* 4195 * Handle decrementing session or connection usage count if 4196 * a logout response was not able to be sent because the 4197 * connection failed. Fall back to Session Recovery here. 4198 */ 4199 if (atomic_read(&conn->conn_logout_remove)) { 4200 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) { 4201 iscsit_dec_conn_usage_count(conn); 4202 iscsit_dec_session_usage_count(sess); 4203 } 4204 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) 4205 iscsit_dec_conn_usage_count(conn); 4206 4207 atomic_set(&conn->conn_logout_remove, 0); 4208 atomic_set(&sess->session_reinstatement, 0); 4209 atomic_set(&sess->session_fall_back_to_erl0, 1); 4210 } 4211 4212 spin_lock_bh(&sess->conn_lock); 4213 list_del(&conn->conn_list); 4214 4215 /* 4216 * Attempt to let the Initiator know this connection failed by 4217 * sending an Connection Dropped Async Message on another 4218 * active connection. 4219 */ 4220 if (atomic_read(&conn->connection_recovery)) 4221 iscsit_build_conn_drop_async_message(conn); 4222 4223 spin_unlock_bh(&sess->conn_lock); 4224 4225 /* 4226 * If connection reinstatement is being performed on this connection, 4227 * up the connection reinstatement semaphore that is being blocked on 4228 * in iscsit_cause_connection_reinstatement(). 4229 */ 4230 spin_lock_bh(&conn->state_lock); 4231 if (atomic_read(&conn->sleep_on_conn_wait_comp)) { 4232 spin_unlock_bh(&conn->state_lock); 4233 complete(&conn->conn_wait_comp); 4234 wait_for_completion(&conn->conn_post_wait_comp); 4235 spin_lock_bh(&conn->state_lock); 4236 } 4237 4238 /* 4239 * If connection reinstatement is being performed on this connection 4240 * by receiving a REMOVECONNFORRECOVERY logout request, up the 4241 * connection wait rcfr semaphore that is being blocked on 4242 * an iscsit_connection_reinstatement_rcfr(). 4243 */ 4244 if (atomic_read(&conn->connection_wait_rcfr)) { 4245 spin_unlock_bh(&conn->state_lock); 4246 complete(&conn->conn_wait_rcfr_comp); 4247 wait_for_completion(&conn->conn_post_wait_comp); 4248 spin_lock_bh(&conn->state_lock); 4249 } 4250 atomic_set(&conn->connection_reinstatement, 1); 4251 spin_unlock_bh(&conn->state_lock); 4252 4253 /* 4254 * If any other processes are accessing this connection pointer we 4255 * must wait until they have completed. 4256 */ 4257 iscsit_check_conn_usage_count(conn); 4258 4259 ahash_request_free(conn->conn_tx_hash); 4260 if (conn->conn_rx_hash) { 4261 struct crypto_ahash *tfm; 4262 4263 tfm = crypto_ahash_reqtfm(conn->conn_rx_hash); 4264 ahash_request_free(conn->conn_rx_hash); 4265 crypto_free_ahash(tfm); 4266 } 4267 4268 if (conn->sock) 4269 sock_release(conn->sock); 4270 4271 if (conn->conn_transport->iscsit_free_conn) 4272 conn->conn_transport->iscsit_free_conn(conn); 4273 4274 pr_debug("Moving to TARG_CONN_STATE_FREE.\n"); 4275 conn->conn_state = TARG_CONN_STATE_FREE; 4276 iscsit_free_conn(conn); 4277 4278 spin_lock_bh(&sess->conn_lock); 4279 atomic_dec(&sess->nconn); 4280 pr_debug("Decremented iSCSI connection count to %hu from node:" 4281 " %s\n", atomic_read(&sess->nconn), 4282 sess->sess_ops->InitiatorName); 4283 /* 4284 * Make sure that if one connection fails in an non ERL=2 iSCSI 4285 * Session that they all fail. 4286 */ 4287 if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout && 4288 !atomic_read(&sess->session_logout)) 4289 atomic_set(&sess->session_fall_back_to_erl0, 1); 4290 4291 /* 4292 * If this was not the last connection in the session, and we are 4293 * performing session reinstatement or falling back to ERL=0, call 4294 * iscsit_stop_session() without sleeping to shutdown the other 4295 * active connections. 4296 */ 4297 if (atomic_read(&sess->nconn)) { 4298 if (!atomic_read(&sess->session_reinstatement) && 4299 !atomic_read(&sess->session_fall_back_to_erl0)) { 4300 spin_unlock_bh(&sess->conn_lock); 4301 return 0; 4302 } 4303 if (!atomic_read(&sess->session_stop_active)) { 4304 atomic_set(&sess->session_stop_active, 1); 4305 spin_unlock_bh(&sess->conn_lock); 4306 iscsit_stop_session(sess, 0, 0); 4307 return 0; 4308 } 4309 spin_unlock_bh(&sess->conn_lock); 4310 return 0; 4311 } 4312 4313 /* 4314 * If this was the last connection in the session and one of the 4315 * following is occurring: 4316 * 4317 * Session Reinstatement is not being performed, and are falling back 4318 * to ERL=0 call iscsit_close_session(). 4319 * 4320 * Session Logout was requested. iscsit_close_session() will be called 4321 * elsewhere. 4322 * 4323 * Session Continuation is not being performed, start the Time2Retain 4324 * handler and check if sleep_on_sess_wait_sem is active. 4325 */ 4326 if (!atomic_read(&sess->session_reinstatement) && 4327 atomic_read(&sess->session_fall_back_to_erl0)) { 4328 spin_unlock_bh(&sess->conn_lock); 4329 complete_all(&sess->session_wait_comp); 4330 iscsit_close_session(sess, true); 4331 4332 return 0; 4333 } else if (atomic_read(&sess->session_logout)) { 4334 pr_debug("Moving to TARG_SESS_STATE_FREE.\n"); 4335 sess->session_state = TARG_SESS_STATE_FREE; 4336 4337 if (atomic_read(&sess->session_close)) { 4338 spin_unlock_bh(&sess->conn_lock); 4339 complete_all(&sess->session_wait_comp); 4340 iscsit_close_session(sess, true); 4341 } else { 4342 spin_unlock_bh(&sess->conn_lock); 4343 } 4344 4345 return 0; 4346 } else { 4347 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); 4348 sess->session_state = TARG_SESS_STATE_FAILED; 4349 4350 if (!atomic_read(&sess->session_continuation)) 4351 iscsit_start_time2retain_handler(sess); 4352 4353 if (atomic_read(&sess->session_close)) { 4354 spin_unlock_bh(&sess->conn_lock); 4355 complete_all(&sess->session_wait_comp); 4356 iscsit_close_session(sess, true); 4357 } else { 4358 spin_unlock_bh(&sess->conn_lock); 4359 } 4360 4361 return 0; 4362 } 4363 } 4364 4365 /* 4366 * If the iSCSI Session for the iSCSI Initiator Node exists, 4367 * forcefully shutdown the iSCSI NEXUS. 4368 */ 4369 int iscsit_close_session(struct iscsi_session *sess, bool can_sleep) 4370 { 4371 struct iscsi_portal_group *tpg = sess->tpg; 4372 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; 4373 4374 if (atomic_read(&sess->nconn)) { 4375 pr_err("%d connection(s) still exist for iSCSI session" 4376 " to %s\n", atomic_read(&sess->nconn), 4377 sess->sess_ops->InitiatorName); 4378 BUG(); 4379 } 4380 4381 spin_lock_bh(&se_tpg->session_lock); 4382 atomic_set(&sess->session_logout, 1); 4383 atomic_set(&sess->session_reinstatement, 1); 4384 iscsit_stop_time2retain_timer(sess); 4385 spin_unlock_bh(&se_tpg->session_lock); 4386 4387 /* 4388 * transport_deregister_session_configfs() will clear the 4389 * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context 4390 * can be setting it again with __transport_register_session() in 4391 * iscsi_post_login_handler() again after the iscsit_stop_session() 4392 * completes in iscsi_np context. 4393 */ 4394 transport_deregister_session_configfs(sess->se_sess); 4395 4396 /* 4397 * If any other processes are accessing this session pointer we must 4398 * wait until they have completed. If we are in an interrupt (the 4399 * time2retain handler) and contain and active session usage count we 4400 * restart the timer and exit. 4401 */ 4402 if (iscsit_check_session_usage_count(sess, can_sleep)) { 4403 atomic_set(&sess->session_logout, 0); 4404 iscsit_start_time2retain_handler(sess); 4405 return 0; 4406 } 4407 4408 transport_deregister_session(sess->se_sess); 4409 4410 if (sess->sess_ops->ErrorRecoveryLevel == 2) 4411 iscsit_free_connection_recovery_entries(sess); 4412 4413 iscsit_free_all_ooo_cmdsns(sess); 4414 4415 spin_lock_bh(&se_tpg->session_lock); 4416 pr_debug("Moving to TARG_SESS_STATE_FREE.\n"); 4417 sess->session_state = TARG_SESS_STATE_FREE; 4418 pr_debug("Released iSCSI session from node: %s\n", 4419 sess->sess_ops->InitiatorName); 4420 tpg->nsessions--; 4421 if (tpg->tpg_tiqn) 4422 tpg->tpg_tiqn->tiqn_nsessions--; 4423 4424 pr_debug("Decremented number of active iSCSI Sessions on" 4425 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions); 4426 4427 ida_free(&sess_ida, sess->session_index); 4428 kfree(sess->sess_ops); 4429 sess->sess_ops = NULL; 4430 spin_unlock_bh(&se_tpg->session_lock); 4431 4432 kfree(sess); 4433 return 0; 4434 } 4435 4436 static void iscsit_logout_post_handler_closesession( 4437 struct iscsi_conn *conn) 4438 { 4439 struct iscsi_session *sess = conn->sess; 4440 int sleep = 1; 4441 /* 4442 * Traditional iscsi/tcp will invoke this logic from TX thread 4443 * context during session logout, so clear tx_thread_active and 4444 * sleep if iscsit_close_connection() has not already occured. 4445 * 4446 * Since iser-target invokes this logic from it's own workqueue, 4447 * always sleep waiting for RX/TX thread shutdown to complete 4448 * within iscsit_close_connection(). 4449 */ 4450 if (!conn->conn_transport->rdma_shutdown) { 4451 sleep = cmpxchg(&conn->tx_thread_active, true, false); 4452 if (!sleep) 4453 return; 4454 } 4455 4456 atomic_set(&conn->conn_logout_remove, 0); 4457 complete(&conn->conn_logout_comp); 4458 4459 iscsit_dec_conn_usage_count(conn); 4460 atomic_set(&sess->session_close, 1); 4461 iscsit_stop_session(sess, sleep, sleep); 4462 iscsit_dec_session_usage_count(sess); 4463 } 4464 4465 static void iscsit_logout_post_handler_samecid( 4466 struct iscsi_conn *conn) 4467 { 4468 int sleep = 1; 4469 4470 if (!conn->conn_transport->rdma_shutdown) { 4471 sleep = cmpxchg(&conn->tx_thread_active, true, false); 4472 if (!sleep) 4473 return; 4474 } 4475 4476 atomic_set(&conn->conn_logout_remove, 0); 4477 complete(&conn->conn_logout_comp); 4478 4479 iscsit_cause_connection_reinstatement(conn, sleep); 4480 iscsit_dec_conn_usage_count(conn); 4481 } 4482 4483 static void iscsit_logout_post_handler_diffcid( 4484 struct iscsi_conn *conn, 4485 u16 cid) 4486 { 4487 struct iscsi_conn *l_conn; 4488 struct iscsi_session *sess = conn->sess; 4489 bool conn_found = false; 4490 4491 if (!sess) 4492 return; 4493 4494 spin_lock_bh(&sess->conn_lock); 4495 list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) { 4496 if (l_conn->cid == cid) { 4497 iscsit_inc_conn_usage_count(l_conn); 4498 conn_found = true; 4499 break; 4500 } 4501 } 4502 spin_unlock_bh(&sess->conn_lock); 4503 4504 if (!conn_found) 4505 return; 4506 4507 if (l_conn->sock) 4508 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN); 4509 4510 spin_lock_bh(&l_conn->state_lock); 4511 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 4512 l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT; 4513 spin_unlock_bh(&l_conn->state_lock); 4514 4515 iscsit_cause_connection_reinstatement(l_conn, 1); 4516 iscsit_dec_conn_usage_count(l_conn); 4517 } 4518 4519 /* 4520 * Return of 0 causes the TX thread to restart. 4521 */ 4522 int iscsit_logout_post_handler( 4523 struct iscsi_cmd *cmd, 4524 struct iscsi_conn *conn) 4525 { 4526 int ret = 0; 4527 4528 switch (cmd->logout_reason) { 4529 case ISCSI_LOGOUT_REASON_CLOSE_SESSION: 4530 switch (cmd->logout_response) { 4531 case ISCSI_LOGOUT_SUCCESS: 4532 case ISCSI_LOGOUT_CLEANUP_FAILED: 4533 default: 4534 iscsit_logout_post_handler_closesession(conn); 4535 break; 4536 } 4537 break; 4538 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: 4539 if (conn->cid == cmd->logout_cid) { 4540 switch (cmd->logout_response) { 4541 case ISCSI_LOGOUT_SUCCESS: 4542 case ISCSI_LOGOUT_CLEANUP_FAILED: 4543 default: 4544 iscsit_logout_post_handler_samecid(conn); 4545 break; 4546 } 4547 } else { 4548 switch (cmd->logout_response) { 4549 case ISCSI_LOGOUT_SUCCESS: 4550 iscsit_logout_post_handler_diffcid(conn, 4551 cmd->logout_cid); 4552 break; 4553 case ISCSI_LOGOUT_CID_NOT_FOUND: 4554 case ISCSI_LOGOUT_CLEANUP_FAILED: 4555 default: 4556 break; 4557 } 4558 ret = 1; 4559 } 4560 break; 4561 case ISCSI_LOGOUT_REASON_RECOVERY: 4562 switch (cmd->logout_response) { 4563 case ISCSI_LOGOUT_SUCCESS: 4564 case ISCSI_LOGOUT_CID_NOT_FOUND: 4565 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED: 4566 case ISCSI_LOGOUT_CLEANUP_FAILED: 4567 default: 4568 break; 4569 } 4570 ret = 1; 4571 break; 4572 default: 4573 break; 4574 4575 } 4576 return ret; 4577 } 4578 EXPORT_SYMBOL(iscsit_logout_post_handler); 4579 4580 void iscsit_fail_session(struct iscsi_session *sess) 4581 { 4582 struct iscsi_conn *conn; 4583 4584 spin_lock_bh(&sess->conn_lock); 4585 list_for_each_entry(conn, &sess->sess_conn_list, conn_list) { 4586 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n"); 4587 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT; 4588 } 4589 spin_unlock_bh(&sess->conn_lock); 4590 4591 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); 4592 sess->session_state = TARG_SESS_STATE_FAILED; 4593 } 4594 4595 void iscsit_stop_session( 4596 struct iscsi_session *sess, 4597 int session_sleep, 4598 int connection_sleep) 4599 { 4600 u16 conn_count = atomic_read(&sess->nconn); 4601 struct iscsi_conn *conn, *conn_tmp = NULL; 4602 int is_last; 4603 4604 spin_lock_bh(&sess->conn_lock); 4605 4606 if (connection_sleep) { 4607 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list, 4608 conn_list) { 4609 if (conn_count == 0) 4610 break; 4611 4612 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) { 4613 is_last = 1; 4614 } else { 4615 iscsit_inc_conn_usage_count(conn_tmp); 4616 is_last = 0; 4617 } 4618 iscsit_inc_conn_usage_count(conn); 4619 4620 spin_unlock_bh(&sess->conn_lock); 4621 iscsit_cause_connection_reinstatement(conn, 1); 4622 spin_lock_bh(&sess->conn_lock); 4623 4624 iscsit_dec_conn_usage_count(conn); 4625 if (is_last == 0) 4626 iscsit_dec_conn_usage_count(conn_tmp); 4627 conn_count--; 4628 } 4629 } else { 4630 list_for_each_entry(conn, &sess->sess_conn_list, conn_list) 4631 iscsit_cause_connection_reinstatement(conn, 0); 4632 } 4633 4634 if (session_sleep && atomic_read(&sess->nconn)) { 4635 spin_unlock_bh(&sess->conn_lock); 4636 wait_for_completion(&sess->session_wait_comp); 4637 } else 4638 spin_unlock_bh(&sess->conn_lock); 4639 } 4640 4641 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force) 4642 { 4643 struct iscsi_session *sess; 4644 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; 4645 struct se_session *se_sess, *se_sess_tmp; 4646 LIST_HEAD(free_list); 4647 int session_count = 0; 4648 4649 spin_lock_bh(&se_tpg->session_lock); 4650 if (tpg->nsessions && !force) { 4651 spin_unlock_bh(&se_tpg->session_lock); 4652 return -1; 4653 } 4654 4655 list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list, 4656 sess_list) { 4657 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr; 4658 4659 spin_lock(&sess->conn_lock); 4660 if (atomic_read(&sess->session_fall_back_to_erl0) || 4661 atomic_read(&sess->session_logout) || 4662 atomic_read(&sess->session_close) || 4663 (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) { 4664 spin_unlock(&sess->conn_lock); 4665 continue; 4666 } 4667 iscsit_inc_session_usage_count(sess); 4668 atomic_set(&sess->session_reinstatement, 1); 4669 atomic_set(&sess->session_fall_back_to_erl0, 1); 4670 atomic_set(&sess->session_close, 1); 4671 spin_unlock(&sess->conn_lock); 4672 4673 list_move_tail(&se_sess->sess_list, &free_list); 4674 } 4675 spin_unlock_bh(&se_tpg->session_lock); 4676 4677 list_for_each_entry_safe(se_sess, se_sess_tmp, &free_list, sess_list) { 4678 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr; 4679 4680 list_del_init(&se_sess->sess_list); 4681 iscsit_stop_session(sess, 1, 1); 4682 iscsit_dec_session_usage_count(sess); 4683 session_count++; 4684 } 4685 4686 pr_debug("Released %d iSCSI Session(s) from Target Portal" 4687 " Group: %hu\n", session_count, tpg->tpgt); 4688 return 0; 4689 } 4690 4691 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure"); 4692 MODULE_VERSION("4.1.x"); 4693 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 4694 MODULE_LICENSE("GPL"); 4695 4696 module_init(iscsi_target_init_module); 4697 module_exit(iscsi_target_cleanup_module); 4698