1 /******************************************************************************* 2 * This file contains main functions related to the iSCSI Target Core Driver. 3 * 4 * \u00a9 Copyright 2007-2011 RisingTide Systems LLC. 5 * 6 * Licensed to the Linux Foundation under the General Public License (GPL) version 2. 7 * 8 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License as published by 12 * the Free Software Foundation; either version 2 of the License, or 13 * (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 ******************************************************************************/ 20 21 #include <linux/string.h> 22 #include <linux/kthread.h> 23 #include <linux/crypto.h> 24 #include <linux/completion.h> 25 #include <linux/module.h> 26 #include <linux/idr.h> 27 #include <asm/unaligned.h> 28 #include <scsi/scsi_device.h> 29 #include <scsi/iscsi_proto.h> 30 #include <target/target_core_base.h> 31 #include <target/target_core_fabric.h> 32 33 #include "iscsi_target_core.h" 34 #include "iscsi_target_parameters.h" 35 #include "iscsi_target_seq_pdu_list.h" 36 #include "iscsi_target_tq.h" 37 #include "iscsi_target_configfs.h" 38 #include "iscsi_target_datain_values.h" 39 #include "iscsi_target_erl0.h" 40 #include "iscsi_target_erl1.h" 41 #include "iscsi_target_erl2.h" 42 #include "iscsi_target_login.h" 43 #include "iscsi_target_tmr.h" 44 #include "iscsi_target_tpg.h" 45 #include "iscsi_target_util.h" 46 #include "iscsi_target.h" 47 #include "iscsi_target_device.h" 48 #include "iscsi_target_stat.h" 49 50 static LIST_HEAD(g_tiqn_list); 51 static LIST_HEAD(g_np_list); 52 static DEFINE_SPINLOCK(tiqn_lock); 53 static DEFINE_SPINLOCK(np_lock); 54 55 static struct idr tiqn_idr; 56 struct idr sess_idr; 57 struct mutex auth_id_lock; 58 spinlock_t sess_idr_lock; 59 60 struct iscsit_global *iscsit_global; 61 62 struct kmem_cache *lio_cmd_cache; 63 struct kmem_cache *lio_qr_cache; 64 struct kmem_cache *lio_dr_cache; 65 struct kmem_cache *lio_ooo_cache; 66 struct kmem_cache *lio_r2t_cache; 67 68 static int iscsit_handle_immediate_data(struct iscsi_cmd *, 69 unsigned char *buf, u32); 70 static int iscsit_logout_post_handler(struct iscsi_cmd *, struct iscsi_conn *); 71 72 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf) 73 { 74 struct iscsi_tiqn *tiqn = NULL; 75 76 spin_lock(&tiqn_lock); 77 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) { 78 if (!strcmp(tiqn->tiqn, buf)) { 79 80 spin_lock(&tiqn->tiqn_state_lock); 81 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) { 82 tiqn->tiqn_access_count++; 83 spin_unlock(&tiqn->tiqn_state_lock); 84 spin_unlock(&tiqn_lock); 85 return tiqn; 86 } 87 spin_unlock(&tiqn->tiqn_state_lock); 88 } 89 } 90 spin_unlock(&tiqn_lock); 91 92 return NULL; 93 } 94 95 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn) 96 { 97 spin_lock(&tiqn->tiqn_state_lock); 98 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) { 99 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN; 100 spin_unlock(&tiqn->tiqn_state_lock); 101 return 0; 102 } 103 spin_unlock(&tiqn->tiqn_state_lock); 104 105 return -1; 106 } 107 108 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn) 109 { 110 spin_lock(&tiqn->tiqn_state_lock); 111 tiqn->tiqn_access_count--; 112 spin_unlock(&tiqn->tiqn_state_lock); 113 } 114 115 /* 116 * Note that IQN formatting is expected to be done in userspace, and 117 * no explict IQN format checks are done here. 118 */ 119 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf) 120 { 121 struct iscsi_tiqn *tiqn = NULL; 122 int ret; 123 124 if (strlen(buf) >= ISCSI_IQN_LEN) { 125 pr_err("Target IQN exceeds %d bytes\n", 126 ISCSI_IQN_LEN); 127 return ERR_PTR(-EINVAL); 128 } 129 130 tiqn = kzalloc(sizeof(struct iscsi_tiqn), GFP_KERNEL); 131 if (!tiqn) { 132 pr_err("Unable to allocate struct iscsi_tiqn\n"); 133 return ERR_PTR(-ENOMEM); 134 } 135 136 sprintf(tiqn->tiqn, "%s", buf); 137 INIT_LIST_HEAD(&tiqn->tiqn_list); 138 INIT_LIST_HEAD(&tiqn->tiqn_tpg_list); 139 spin_lock_init(&tiqn->tiqn_state_lock); 140 spin_lock_init(&tiqn->tiqn_tpg_lock); 141 spin_lock_init(&tiqn->sess_err_stats.lock); 142 spin_lock_init(&tiqn->login_stats.lock); 143 spin_lock_init(&tiqn->logout_stats.lock); 144 145 if (!idr_pre_get(&tiqn_idr, GFP_KERNEL)) { 146 pr_err("idr_pre_get() for tiqn_idr failed\n"); 147 kfree(tiqn); 148 return ERR_PTR(-ENOMEM); 149 } 150 tiqn->tiqn_state = TIQN_STATE_ACTIVE; 151 152 spin_lock(&tiqn_lock); 153 ret = idr_get_new(&tiqn_idr, NULL, &tiqn->tiqn_index); 154 if (ret < 0) { 155 pr_err("idr_get_new() failed for tiqn->tiqn_index\n"); 156 spin_unlock(&tiqn_lock); 157 kfree(tiqn); 158 return ERR_PTR(ret); 159 } 160 list_add_tail(&tiqn->tiqn_list, &g_tiqn_list); 161 spin_unlock(&tiqn_lock); 162 163 pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn); 164 165 return tiqn; 166 167 } 168 169 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn) 170 { 171 /* 172 * Wait for accesses to said struct iscsi_tiqn to end. 173 */ 174 spin_lock(&tiqn->tiqn_state_lock); 175 while (tiqn->tiqn_access_count != 0) { 176 spin_unlock(&tiqn->tiqn_state_lock); 177 msleep(10); 178 spin_lock(&tiqn->tiqn_state_lock); 179 } 180 spin_unlock(&tiqn->tiqn_state_lock); 181 } 182 183 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn) 184 { 185 /* 186 * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN 187 * while holding tiqn->tiqn_state_lock. This means that all subsequent 188 * attempts to access this struct iscsi_tiqn will fail from both transport 189 * fabric and control code paths. 190 */ 191 if (iscsit_set_tiqn_shutdown(tiqn) < 0) { 192 pr_err("iscsit_set_tiqn_shutdown() failed\n"); 193 return; 194 } 195 196 iscsit_wait_for_tiqn(tiqn); 197 198 spin_lock(&tiqn_lock); 199 list_del(&tiqn->tiqn_list); 200 idr_remove(&tiqn_idr, tiqn->tiqn_index); 201 spin_unlock(&tiqn_lock); 202 203 pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n", 204 tiqn->tiqn); 205 kfree(tiqn); 206 } 207 208 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg) 209 { 210 int ret; 211 /* 212 * Determine if the network portal is accepting storage traffic. 213 */ 214 spin_lock_bh(&np->np_thread_lock); 215 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { 216 spin_unlock_bh(&np->np_thread_lock); 217 return -1; 218 } 219 if (np->np_login_tpg) { 220 pr_err("np->np_login_tpg() is not NULL!\n"); 221 spin_unlock_bh(&np->np_thread_lock); 222 return -1; 223 } 224 spin_unlock_bh(&np->np_thread_lock); 225 /* 226 * Determine if the portal group is accepting storage traffic. 227 */ 228 spin_lock_bh(&tpg->tpg_state_lock); 229 if (tpg->tpg_state != TPG_STATE_ACTIVE) { 230 spin_unlock_bh(&tpg->tpg_state_lock); 231 return -1; 232 } 233 spin_unlock_bh(&tpg->tpg_state_lock); 234 235 /* 236 * Here we serialize access across the TIQN+TPG Tuple. 237 */ 238 ret = mutex_lock_interruptible(&tpg->np_login_lock); 239 if ((ret != 0) || signal_pending(current)) 240 return -1; 241 242 spin_lock_bh(&np->np_thread_lock); 243 np->np_login_tpg = tpg; 244 spin_unlock_bh(&np->np_thread_lock); 245 246 return 0; 247 } 248 249 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg) 250 { 251 struct iscsi_tiqn *tiqn = tpg->tpg_tiqn; 252 253 spin_lock_bh(&np->np_thread_lock); 254 np->np_login_tpg = NULL; 255 spin_unlock_bh(&np->np_thread_lock); 256 257 mutex_unlock(&tpg->np_login_lock); 258 259 if (tiqn) 260 iscsit_put_tiqn_for_login(tiqn); 261 262 return 0; 263 } 264 265 static struct iscsi_np *iscsit_get_np( 266 struct __kernel_sockaddr_storage *sockaddr, 267 int network_transport) 268 { 269 struct sockaddr_in *sock_in, *sock_in_e; 270 struct sockaddr_in6 *sock_in6, *sock_in6_e; 271 struct iscsi_np *np; 272 int ip_match = 0; 273 u16 port; 274 275 spin_lock_bh(&np_lock); 276 list_for_each_entry(np, &g_np_list, np_list) { 277 spin_lock(&np->np_thread_lock); 278 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { 279 spin_unlock(&np->np_thread_lock); 280 continue; 281 } 282 283 if (sockaddr->ss_family == AF_INET6) { 284 sock_in6 = (struct sockaddr_in6 *)sockaddr; 285 sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr; 286 287 if (!memcmp(&sock_in6->sin6_addr.in6_u, 288 &sock_in6_e->sin6_addr.in6_u, 289 sizeof(struct in6_addr))) 290 ip_match = 1; 291 292 port = ntohs(sock_in6->sin6_port); 293 } else { 294 sock_in = (struct sockaddr_in *)sockaddr; 295 sock_in_e = (struct sockaddr_in *)&np->np_sockaddr; 296 297 if (sock_in->sin_addr.s_addr == 298 sock_in_e->sin_addr.s_addr) 299 ip_match = 1; 300 301 port = ntohs(sock_in->sin_port); 302 } 303 304 if ((ip_match == 1) && (np->np_port == port) && 305 (np->np_network_transport == network_transport)) { 306 /* 307 * Increment the np_exports reference count now to 308 * prevent iscsit_del_np() below from being called 309 * while iscsi_tpg_add_network_portal() is called. 310 */ 311 np->np_exports++; 312 spin_unlock(&np->np_thread_lock); 313 spin_unlock_bh(&np_lock); 314 return np; 315 } 316 spin_unlock(&np->np_thread_lock); 317 } 318 spin_unlock_bh(&np_lock); 319 320 return NULL; 321 } 322 323 struct iscsi_np *iscsit_add_np( 324 struct __kernel_sockaddr_storage *sockaddr, 325 char *ip_str, 326 int network_transport) 327 { 328 struct sockaddr_in *sock_in; 329 struct sockaddr_in6 *sock_in6; 330 struct iscsi_np *np; 331 int ret; 332 /* 333 * Locate the existing struct iscsi_np if already active.. 334 */ 335 np = iscsit_get_np(sockaddr, network_transport); 336 if (np) 337 return np; 338 339 np = kzalloc(sizeof(struct iscsi_np), GFP_KERNEL); 340 if (!np) { 341 pr_err("Unable to allocate memory for struct iscsi_np\n"); 342 return ERR_PTR(-ENOMEM); 343 } 344 345 np->np_flags |= NPF_IP_NETWORK; 346 if (sockaddr->ss_family == AF_INET6) { 347 sock_in6 = (struct sockaddr_in6 *)sockaddr; 348 snprintf(np->np_ip, IPV6_ADDRESS_SPACE, "%s", ip_str); 349 np->np_port = ntohs(sock_in6->sin6_port); 350 } else { 351 sock_in = (struct sockaddr_in *)sockaddr; 352 sprintf(np->np_ip, "%s", ip_str); 353 np->np_port = ntohs(sock_in->sin_port); 354 } 355 356 np->np_network_transport = network_transport; 357 spin_lock_init(&np->np_thread_lock); 358 init_completion(&np->np_restart_comp); 359 INIT_LIST_HEAD(&np->np_list); 360 361 ret = iscsi_target_setup_login_socket(np, sockaddr); 362 if (ret != 0) { 363 kfree(np); 364 return ERR_PTR(ret); 365 } 366 367 np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np"); 368 if (IS_ERR(np->np_thread)) { 369 pr_err("Unable to create kthread: iscsi_np\n"); 370 ret = PTR_ERR(np->np_thread); 371 kfree(np); 372 return ERR_PTR(ret); 373 } 374 /* 375 * Increment the np_exports reference count now to prevent 376 * iscsit_del_np() below from being run while a new call to 377 * iscsi_tpg_add_network_portal() for a matching iscsi_np is 378 * active. We don't need to hold np->np_thread_lock at this 379 * point because iscsi_np has not been added to g_np_list yet. 380 */ 381 np->np_exports = 1; 382 383 spin_lock_bh(&np_lock); 384 list_add_tail(&np->np_list, &g_np_list); 385 spin_unlock_bh(&np_lock); 386 387 pr_debug("CORE[0] - Added Network Portal: %s:%hu on %s\n", 388 np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ? 389 "TCP" : "SCTP"); 390 391 return np; 392 } 393 394 int iscsit_reset_np_thread( 395 struct iscsi_np *np, 396 struct iscsi_tpg_np *tpg_np, 397 struct iscsi_portal_group *tpg) 398 { 399 spin_lock_bh(&np->np_thread_lock); 400 if (tpg && tpg_np) { 401 /* 402 * The reset operation need only be performed when the 403 * passed struct iscsi_portal_group has a login in progress 404 * to one of the network portals. 405 */ 406 if (tpg_np->tpg_np->np_login_tpg != tpg) { 407 spin_unlock_bh(&np->np_thread_lock); 408 return 0; 409 } 410 } 411 if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) { 412 spin_unlock_bh(&np->np_thread_lock); 413 return 0; 414 } 415 np->np_thread_state = ISCSI_NP_THREAD_RESET; 416 417 if (np->np_thread) { 418 spin_unlock_bh(&np->np_thread_lock); 419 send_sig(SIGINT, np->np_thread, 1); 420 wait_for_completion(&np->np_restart_comp); 421 spin_lock_bh(&np->np_thread_lock); 422 } 423 spin_unlock_bh(&np->np_thread_lock); 424 425 return 0; 426 } 427 428 int iscsit_del_np_comm(struct iscsi_np *np) 429 { 430 if (!np->np_socket) 431 return 0; 432 433 /* 434 * Some network transports allocate their own struct sock->file, 435 * see if we need to free any additional allocated resources. 436 */ 437 if (np->np_flags & NPF_SCTP_STRUCT_FILE) { 438 kfree(np->np_socket->file); 439 np->np_socket->file = NULL; 440 } 441 442 sock_release(np->np_socket); 443 return 0; 444 } 445 446 int iscsit_del_np(struct iscsi_np *np) 447 { 448 spin_lock_bh(&np->np_thread_lock); 449 np->np_exports--; 450 if (np->np_exports) { 451 spin_unlock_bh(&np->np_thread_lock); 452 return 0; 453 } 454 np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN; 455 spin_unlock_bh(&np->np_thread_lock); 456 457 if (np->np_thread) { 458 /* 459 * We need to send the signal to wakeup Linux/Net 460 * which may be sleeping in sock_accept().. 461 */ 462 send_sig(SIGINT, np->np_thread, 1); 463 kthread_stop(np->np_thread); 464 } 465 iscsit_del_np_comm(np); 466 467 spin_lock_bh(&np_lock); 468 list_del(&np->np_list); 469 spin_unlock_bh(&np_lock); 470 471 pr_debug("CORE[0] - Removed Network Portal: %s:%hu on %s\n", 472 np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ? 473 "TCP" : "SCTP"); 474 475 kfree(np); 476 return 0; 477 } 478 479 static int __init iscsi_target_init_module(void) 480 { 481 int ret = 0; 482 483 pr_debug("iSCSI-Target "ISCSIT_VERSION"\n"); 484 485 iscsit_global = kzalloc(sizeof(struct iscsit_global), GFP_KERNEL); 486 if (!iscsit_global) { 487 pr_err("Unable to allocate memory for iscsit_global\n"); 488 return -1; 489 } 490 mutex_init(&auth_id_lock); 491 spin_lock_init(&sess_idr_lock); 492 idr_init(&tiqn_idr); 493 idr_init(&sess_idr); 494 495 ret = iscsi_target_register_configfs(); 496 if (ret < 0) 497 goto out; 498 499 ret = iscsi_thread_set_init(); 500 if (ret < 0) 501 goto configfs_out; 502 503 if (iscsi_allocate_thread_sets(TARGET_THREAD_SET_COUNT) != 504 TARGET_THREAD_SET_COUNT) { 505 pr_err("iscsi_allocate_thread_sets() returned" 506 " unexpected value!\n"); 507 goto ts_out1; 508 } 509 510 lio_cmd_cache = kmem_cache_create("lio_cmd_cache", 511 sizeof(struct iscsi_cmd), __alignof__(struct iscsi_cmd), 512 0, NULL); 513 if (!lio_cmd_cache) { 514 pr_err("Unable to kmem_cache_create() for" 515 " lio_cmd_cache\n"); 516 goto ts_out2; 517 } 518 519 lio_qr_cache = kmem_cache_create("lio_qr_cache", 520 sizeof(struct iscsi_queue_req), 521 __alignof__(struct iscsi_queue_req), 0, NULL); 522 if (!lio_qr_cache) { 523 pr_err("nable to kmem_cache_create() for" 524 " lio_qr_cache\n"); 525 goto cmd_out; 526 } 527 528 lio_dr_cache = kmem_cache_create("lio_dr_cache", 529 sizeof(struct iscsi_datain_req), 530 __alignof__(struct iscsi_datain_req), 0, NULL); 531 if (!lio_dr_cache) { 532 pr_err("Unable to kmem_cache_create() for" 533 " lio_dr_cache\n"); 534 goto qr_out; 535 } 536 537 lio_ooo_cache = kmem_cache_create("lio_ooo_cache", 538 sizeof(struct iscsi_ooo_cmdsn), 539 __alignof__(struct iscsi_ooo_cmdsn), 0, NULL); 540 if (!lio_ooo_cache) { 541 pr_err("Unable to kmem_cache_create() for" 542 " lio_ooo_cache\n"); 543 goto dr_out; 544 } 545 546 lio_r2t_cache = kmem_cache_create("lio_r2t_cache", 547 sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t), 548 0, NULL); 549 if (!lio_r2t_cache) { 550 pr_err("Unable to kmem_cache_create() for" 551 " lio_r2t_cache\n"); 552 goto ooo_out; 553 } 554 555 if (iscsit_load_discovery_tpg() < 0) 556 goto r2t_out; 557 558 return ret; 559 r2t_out: 560 kmem_cache_destroy(lio_r2t_cache); 561 ooo_out: 562 kmem_cache_destroy(lio_ooo_cache); 563 dr_out: 564 kmem_cache_destroy(lio_dr_cache); 565 qr_out: 566 kmem_cache_destroy(lio_qr_cache); 567 cmd_out: 568 kmem_cache_destroy(lio_cmd_cache); 569 ts_out2: 570 iscsi_deallocate_thread_sets(); 571 ts_out1: 572 iscsi_thread_set_free(); 573 configfs_out: 574 iscsi_target_deregister_configfs(); 575 out: 576 kfree(iscsit_global); 577 return -ENOMEM; 578 } 579 580 static void __exit iscsi_target_cleanup_module(void) 581 { 582 iscsi_deallocate_thread_sets(); 583 iscsi_thread_set_free(); 584 iscsit_release_discovery_tpg(); 585 kmem_cache_destroy(lio_cmd_cache); 586 kmem_cache_destroy(lio_qr_cache); 587 kmem_cache_destroy(lio_dr_cache); 588 kmem_cache_destroy(lio_ooo_cache); 589 kmem_cache_destroy(lio_r2t_cache); 590 591 iscsi_target_deregister_configfs(); 592 593 kfree(iscsit_global); 594 } 595 596 int iscsit_add_reject( 597 u8 reason, 598 int fail_conn, 599 unsigned char *buf, 600 struct iscsi_conn *conn) 601 { 602 struct iscsi_cmd *cmd; 603 struct iscsi_reject *hdr; 604 int ret; 605 606 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 607 if (!cmd) 608 return -1; 609 610 cmd->iscsi_opcode = ISCSI_OP_REJECT; 611 if (fail_conn) 612 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN; 613 614 hdr = (struct iscsi_reject *) cmd->pdu; 615 hdr->reason = reason; 616 617 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL); 618 if (!cmd->buf_ptr) { 619 pr_err("Unable to allocate memory for cmd->buf_ptr\n"); 620 iscsit_release_cmd(cmd); 621 return -1; 622 } 623 624 spin_lock_bh(&conn->cmd_lock); 625 list_add_tail(&cmd->i_list, &conn->conn_cmd_list); 626 spin_unlock_bh(&conn->cmd_lock); 627 628 cmd->i_state = ISTATE_SEND_REJECT; 629 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 630 631 ret = wait_for_completion_interruptible(&cmd->reject_comp); 632 if (ret != 0) 633 return -1; 634 635 return (!fail_conn) ? 0 : -1; 636 } 637 638 int iscsit_add_reject_from_cmd( 639 u8 reason, 640 int fail_conn, 641 int add_to_conn, 642 unsigned char *buf, 643 struct iscsi_cmd *cmd) 644 { 645 struct iscsi_conn *conn; 646 struct iscsi_reject *hdr; 647 int ret; 648 649 if (!cmd->conn) { 650 pr_err("cmd->conn is NULL for ITT: 0x%08x\n", 651 cmd->init_task_tag); 652 return -1; 653 } 654 conn = cmd->conn; 655 656 cmd->iscsi_opcode = ISCSI_OP_REJECT; 657 if (fail_conn) 658 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN; 659 660 hdr = (struct iscsi_reject *) cmd->pdu; 661 hdr->reason = reason; 662 663 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL); 664 if (!cmd->buf_ptr) { 665 pr_err("Unable to allocate memory for cmd->buf_ptr\n"); 666 iscsit_release_cmd(cmd); 667 return -1; 668 } 669 670 if (add_to_conn) { 671 spin_lock_bh(&conn->cmd_lock); 672 list_add_tail(&cmd->i_list, &conn->conn_cmd_list); 673 spin_unlock_bh(&conn->cmd_lock); 674 } 675 676 cmd->i_state = ISTATE_SEND_REJECT; 677 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 678 679 ret = wait_for_completion_interruptible(&cmd->reject_comp); 680 if (ret != 0) 681 return -1; 682 683 return (!fail_conn) ? 0 : -1; 684 } 685 686 /* 687 * Map some portion of the allocated scatterlist to an iovec, suitable for 688 * kernel sockets to copy data in/out. This handles both pages and slab-allocated 689 * buffers, since we have been tricky and mapped t_mem_sg to the buffer in 690 * either case (see iscsit_alloc_buffs) 691 */ 692 static int iscsit_map_iovec( 693 struct iscsi_cmd *cmd, 694 struct kvec *iov, 695 u32 data_offset, 696 u32 data_length) 697 { 698 u32 i = 0; 699 struct scatterlist *sg; 700 unsigned int page_off; 701 702 /* 703 * We have a private mapping of the allocated pages in t_mem_sg. 704 * At this point, we also know each contains a page. 705 */ 706 sg = &cmd->t_mem_sg[data_offset / PAGE_SIZE]; 707 page_off = (data_offset % PAGE_SIZE); 708 709 cmd->first_data_sg = sg; 710 cmd->first_data_sg_off = page_off; 711 712 while (data_length) { 713 u32 cur_len = min_t(u32, data_length, sg->length - page_off); 714 715 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off; 716 iov[i].iov_len = cur_len; 717 718 data_length -= cur_len; 719 page_off = 0; 720 sg = sg_next(sg); 721 i++; 722 } 723 724 cmd->kmapped_nents = i; 725 726 return i; 727 } 728 729 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd) 730 { 731 u32 i; 732 struct scatterlist *sg; 733 734 sg = cmd->first_data_sg; 735 736 for (i = 0; i < cmd->kmapped_nents; i++) 737 kunmap(sg_page(&sg[i])); 738 } 739 740 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn) 741 { 742 struct iscsi_cmd *cmd; 743 744 conn->exp_statsn = exp_statsn; 745 746 spin_lock_bh(&conn->cmd_lock); 747 list_for_each_entry(cmd, &conn->conn_cmd_list, i_list) { 748 spin_lock(&cmd->istate_lock); 749 if ((cmd->i_state == ISTATE_SENT_STATUS) && 750 (cmd->stat_sn < exp_statsn)) { 751 cmd->i_state = ISTATE_REMOVE; 752 spin_unlock(&cmd->istate_lock); 753 iscsit_add_cmd_to_immediate_queue(cmd, conn, 754 cmd->i_state); 755 continue; 756 } 757 spin_unlock(&cmd->istate_lock); 758 } 759 spin_unlock_bh(&conn->cmd_lock); 760 } 761 762 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd) 763 { 764 u32 iov_count = (cmd->se_cmd.t_data_nents == 0) ? 1 : 765 cmd->se_cmd.t_data_nents; 766 767 iov_count += ISCSI_IOV_DATA_BUFFER; 768 769 cmd->iov_data = kzalloc(iov_count * sizeof(struct kvec), GFP_KERNEL); 770 if (!cmd->iov_data) { 771 pr_err("Unable to allocate cmd->iov_data\n"); 772 return -ENOMEM; 773 } 774 775 cmd->orig_iov_data_count = iov_count; 776 return 0; 777 } 778 779 static int iscsit_alloc_buffs(struct iscsi_cmd *cmd) 780 { 781 struct scatterlist *sgl; 782 u32 length = cmd->se_cmd.data_length; 783 int nents = DIV_ROUND_UP(length, PAGE_SIZE); 784 int i = 0, j = 0, ret; 785 /* 786 * If no SCSI payload is present, allocate the default iovecs used for 787 * iSCSI PDU Header 788 */ 789 if (!length) 790 return iscsit_allocate_iovecs(cmd); 791 792 sgl = kzalloc(sizeof(*sgl) * nents, GFP_KERNEL); 793 if (!sgl) 794 return -ENOMEM; 795 796 sg_init_table(sgl, nents); 797 798 while (length) { 799 int buf_size = min_t(int, length, PAGE_SIZE); 800 struct page *page; 801 802 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 803 if (!page) 804 goto page_alloc_failed; 805 806 sg_set_page(&sgl[i], page, buf_size, 0); 807 808 length -= buf_size; 809 i++; 810 } 811 812 cmd->t_mem_sg = sgl; 813 cmd->t_mem_sg_nents = nents; 814 815 /* BIDI ops not supported */ 816 817 /* Tell the core about our preallocated memory */ 818 transport_generic_map_mem_to_cmd(&cmd->se_cmd, sgl, nents, NULL, 0); 819 /* 820 * Allocate iovecs for SCSI payload after transport_generic_map_mem_to_cmd 821 * so that cmd->se_cmd.t_tasks_se_num has been set. 822 */ 823 ret = iscsit_allocate_iovecs(cmd); 824 if (ret < 0) 825 return -ENOMEM; 826 827 return 0; 828 829 page_alloc_failed: 830 while (j < i) 831 __free_page(sg_page(&sgl[j++])); 832 833 kfree(sgl); 834 return -ENOMEM; 835 } 836 837 static int iscsit_handle_scsi_cmd( 838 struct iscsi_conn *conn, 839 unsigned char *buf) 840 { 841 int data_direction, cmdsn_ret = 0, immed_ret, ret, transport_ret; 842 int dump_immediate_data = 0, send_check_condition = 0, payload_length; 843 struct iscsi_cmd *cmd = NULL; 844 struct iscsi_scsi_req *hdr; 845 846 spin_lock_bh(&conn->sess->session_stats_lock); 847 conn->sess->cmd_pdus++; 848 if (conn->sess->se_sess->se_node_acl) { 849 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock); 850 conn->sess->se_sess->se_node_acl->num_cmds++; 851 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock); 852 } 853 spin_unlock_bh(&conn->sess->session_stats_lock); 854 855 hdr = (struct iscsi_scsi_req *) buf; 856 payload_length = ntoh24(hdr->dlength); 857 hdr->itt = be32_to_cpu(hdr->itt); 858 hdr->data_length = be32_to_cpu(hdr->data_length); 859 hdr->cmdsn = be32_to_cpu(hdr->cmdsn); 860 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 861 862 /* FIXME; Add checks for AdditionalHeaderSegment */ 863 864 if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) && 865 !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) { 866 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL" 867 " not set. Bad iSCSI Initiator.\n"); 868 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 869 buf, conn); 870 } 871 872 if (((hdr->flags & ISCSI_FLAG_CMD_READ) || 873 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) { 874 /* 875 * Vmware ESX v3.0 uses a modified Cisco Initiator (v3.4.2) 876 * that adds support for RESERVE/RELEASE. There is a bug 877 * add with this new functionality that sets R/W bits when 878 * neither CDB carries any READ or WRITE datapayloads. 879 */ 880 if ((hdr->cdb[0] == 0x16) || (hdr->cdb[0] == 0x17)) { 881 hdr->flags &= ~ISCSI_FLAG_CMD_READ; 882 hdr->flags &= ~ISCSI_FLAG_CMD_WRITE; 883 goto done; 884 } 885 886 pr_err("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE" 887 " set when Expected Data Transfer Length is 0 for" 888 " CDB: 0x%02x. Bad iSCSI Initiator.\n", hdr->cdb[0]); 889 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 890 buf, conn); 891 } 892 done: 893 894 if (!(hdr->flags & ISCSI_FLAG_CMD_READ) && 895 !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) { 896 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE" 897 " MUST be set if Expected Data Transfer Length is not 0." 898 " Bad iSCSI Initiator\n"); 899 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 900 buf, conn); 901 } 902 903 if ((hdr->flags & ISCSI_FLAG_CMD_READ) && 904 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) { 905 pr_err("Bidirectional operations not supported!\n"); 906 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 907 buf, conn); 908 } 909 910 if (hdr->opcode & ISCSI_OP_IMMEDIATE) { 911 pr_err("Illegally set Immediate Bit in iSCSI Initiator" 912 " Scsi Command PDU.\n"); 913 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 914 buf, conn); 915 } 916 917 if (payload_length && !conn->sess->sess_ops->ImmediateData) { 918 pr_err("ImmediateData=No but DataSegmentLength=%u," 919 " protocol error.\n", payload_length); 920 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 921 buf, conn); 922 } 923 924 if ((hdr->data_length == payload_length) && 925 (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) { 926 pr_err("Expected Data Transfer Length and Length of" 927 " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL" 928 " bit is not set protocol error\n"); 929 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 930 buf, conn); 931 } 932 933 if (payload_length > hdr->data_length) { 934 pr_err("DataSegmentLength: %u is greater than" 935 " EDTL: %u, protocol error.\n", payload_length, 936 hdr->data_length); 937 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 938 buf, conn); 939 } 940 941 if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) { 942 pr_err("DataSegmentLength: %u is greater than" 943 " MaxRecvDataSegmentLength: %u, protocol error.\n", 944 payload_length, conn->conn_ops->MaxRecvDataSegmentLength); 945 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 946 buf, conn); 947 } 948 949 if (payload_length > conn->sess->sess_ops->FirstBurstLength) { 950 pr_err("DataSegmentLength: %u is greater than" 951 " FirstBurstLength: %u, protocol error.\n", 952 payload_length, conn->sess->sess_ops->FirstBurstLength); 953 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 954 buf, conn); 955 } 956 957 data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE : 958 (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE : 959 DMA_NONE; 960 961 cmd = iscsit_allocate_se_cmd(conn, hdr->data_length, data_direction, 962 (hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK)); 963 if (!cmd) 964 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1, 965 buf, conn); 966 967 pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x," 968 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt, 969 hdr->cmdsn, hdr->data_length, payload_length, conn->cid); 970 971 cmd->iscsi_opcode = ISCSI_OP_SCSI_CMD; 972 cmd->i_state = ISTATE_NEW_CMD; 973 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 974 cmd->immediate_data = (payload_length) ? 1 : 0; 975 cmd->unsolicited_data = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) && 976 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0); 977 if (cmd->unsolicited_data) 978 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA; 979 980 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 981 if (hdr->flags & ISCSI_FLAG_CMD_READ) { 982 spin_lock_bh(&conn->sess->ttt_lock); 983 cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++; 984 if (cmd->targ_xfer_tag == 0xFFFFFFFF) 985 cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++; 986 spin_unlock_bh(&conn->sess->ttt_lock); 987 } else if (hdr->flags & ISCSI_FLAG_CMD_WRITE) 988 cmd->targ_xfer_tag = 0xFFFFFFFF; 989 cmd->cmd_sn = hdr->cmdsn; 990 cmd->exp_stat_sn = hdr->exp_statsn; 991 cmd->first_burst_len = payload_length; 992 993 if (cmd->data_direction == DMA_FROM_DEVICE) { 994 struct iscsi_datain_req *dr; 995 996 dr = iscsit_allocate_datain_req(); 997 if (!dr) 998 return iscsit_add_reject_from_cmd( 999 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1000 1, 1, buf, cmd); 1001 1002 iscsit_attach_datain_req(cmd, dr); 1003 } 1004 1005 /* 1006 * The CDB is going to an se_device_t. 1007 */ 1008 ret = transport_lookup_cmd_lun(&cmd->se_cmd, 1009 scsilun_to_int(&hdr->lun)); 1010 if (ret < 0) { 1011 if (cmd->se_cmd.scsi_sense_reason == TCM_NON_EXISTENT_LUN) { 1012 pr_debug("Responding to non-acl'ed," 1013 " non-existent or non-exported iSCSI LUN:" 1014 " 0x%016Lx\n", get_unaligned_le64(&hdr->lun)); 1015 } 1016 send_check_condition = 1; 1017 goto attach_cmd; 1018 } 1019 /* 1020 * The Initiator Node has access to the LUN (the addressing method 1021 * is handled inside of iscsit_get_lun_for_cmd()). Now it's time to 1022 * allocate 1->N transport tasks (depending on sector count and 1023 * maximum request size the physical HBA(s) can handle. 1024 */ 1025 transport_ret = transport_generic_allocate_tasks(&cmd->se_cmd, hdr->cdb); 1026 if (transport_ret == -ENOMEM) { 1027 return iscsit_add_reject_from_cmd( 1028 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1029 1, 1, buf, cmd); 1030 } else if (transport_ret < 0) { 1031 /* 1032 * Unsupported SAM Opcode. CHECK_CONDITION will be sent 1033 * in iscsit_execute_cmd() during the CmdSN OOO Execution 1034 * Mechinism. 1035 */ 1036 send_check_condition = 1; 1037 } else { 1038 cmd->data_length = cmd->se_cmd.data_length; 1039 1040 if (iscsit_decide_list_to_build(cmd, payload_length) < 0) 1041 return iscsit_add_reject_from_cmd( 1042 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1043 1, 1, buf, cmd); 1044 } 1045 1046 attach_cmd: 1047 spin_lock_bh(&conn->cmd_lock); 1048 list_add_tail(&cmd->i_list, &conn->conn_cmd_list); 1049 spin_unlock_bh(&conn->cmd_lock); 1050 /* 1051 * Check if we need to delay processing because of ALUA 1052 * Active/NonOptimized primary access state.. 1053 */ 1054 core_alua_check_nonop_delay(&cmd->se_cmd); 1055 /* 1056 * Allocate and setup SGL used with transport_generic_map_mem_to_cmd(). 1057 * also call iscsit_allocate_iovecs() 1058 */ 1059 ret = iscsit_alloc_buffs(cmd); 1060 if (ret < 0) 1061 return iscsit_add_reject_from_cmd( 1062 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1063 1, 0, buf, cmd); 1064 /* 1065 * Check the CmdSN against ExpCmdSN/MaxCmdSN here if 1066 * the Immediate Bit is not set, and no Immediate 1067 * Data is attached. 1068 * 1069 * A PDU/CmdSN carrying Immediate Data can only 1070 * be processed after the DataCRC has passed. 1071 * If the DataCRC fails, the CmdSN MUST NOT 1072 * be acknowledged. (See below) 1073 */ 1074 if (!cmd->immediate_data) { 1075 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 1076 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) 1077 return 0; 1078 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1079 return iscsit_add_reject_from_cmd( 1080 ISCSI_REASON_PROTOCOL_ERROR, 1081 1, 0, buf, cmd); 1082 } 1083 1084 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn); 1085 1086 /* 1087 * If no Immediate Data is attached, it's OK to return now. 1088 */ 1089 if (!cmd->immediate_data) { 1090 if (send_check_condition) 1091 return 0; 1092 1093 if (cmd->unsolicited_data) { 1094 iscsit_set_dataout_sequence_values(cmd); 1095 1096 spin_lock_bh(&cmd->dataout_timeout_lock); 1097 iscsit_start_dataout_timer(cmd, cmd->conn); 1098 spin_unlock_bh(&cmd->dataout_timeout_lock); 1099 } 1100 1101 return 0; 1102 } 1103 1104 /* 1105 * Early CHECK_CONDITIONs never make it to the transport processing 1106 * thread. They are processed in CmdSN order by 1107 * iscsit_check_received_cmdsn() below. 1108 */ 1109 if (send_check_condition) { 1110 immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 1111 dump_immediate_data = 1; 1112 goto after_immediate_data; 1113 } 1114 /* 1115 * Call directly into transport_generic_new_cmd() to perform 1116 * the backend memory allocation. 1117 */ 1118 ret = transport_generic_new_cmd(&cmd->se_cmd); 1119 if (ret < 0) { 1120 immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 1121 dump_immediate_data = 1; 1122 goto after_immediate_data; 1123 } 1124 1125 immed_ret = iscsit_handle_immediate_data(cmd, buf, payload_length); 1126 after_immediate_data: 1127 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 1128 /* 1129 * A PDU/CmdSN carrying Immediate Data passed 1130 * DataCRC, check against ExpCmdSN/MaxCmdSN if 1131 * Immediate Bit is not set. 1132 */ 1133 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 1134 /* 1135 * Special case for Unsupported SAM WRITE Opcodes 1136 * and ImmediateData=Yes. 1137 */ 1138 if (dump_immediate_data) { 1139 if (iscsit_dump_data_payload(conn, payload_length, 1) < 0) 1140 return -1; 1141 } else if (cmd->unsolicited_data) { 1142 iscsit_set_dataout_sequence_values(cmd); 1143 1144 spin_lock_bh(&cmd->dataout_timeout_lock); 1145 iscsit_start_dataout_timer(cmd, cmd->conn); 1146 spin_unlock_bh(&cmd->dataout_timeout_lock); 1147 } 1148 1149 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1150 return iscsit_add_reject_from_cmd( 1151 ISCSI_REASON_PROTOCOL_ERROR, 1152 1, 0, buf, cmd); 1153 1154 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 1155 /* 1156 * Immediate Data failed DataCRC and ERL>=1, 1157 * silently drop this PDU and let the initiator 1158 * plug the CmdSN gap. 1159 * 1160 * FIXME: Send Unsolicited NOPIN with reserved 1161 * TTT here to help the initiator figure out 1162 * the missing CmdSN, although they should be 1163 * intelligent enough to determine the missing 1164 * CmdSN and issue a retry to plug the sequence. 1165 */ 1166 cmd->i_state = ISTATE_REMOVE; 1167 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 1168 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 1169 return -1; 1170 1171 return 0; 1172 } 1173 1174 static u32 iscsit_do_crypto_hash_sg( 1175 struct hash_desc *hash, 1176 struct iscsi_cmd *cmd, 1177 u32 data_offset, 1178 u32 data_length, 1179 u32 padding, 1180 u8 *pad_bytes) 1181 { 1182 u32 data_crc; 1183 u32 i; 1184 struct scatterlist *sg; 1185 unsigned int page_off; 1186 1187 crypto_hash_init(hash); 1188 1189 sg = cmd->first_data_sg; 1190 page_off = cmd->first_data_sg_off; 1191 1192 i = 0; 1193 while (data_length) { 1194 u32 cur_len = min_t(u32, data_length, (sg[i].length - page_off)); 1195 1196 crypto_hash_update(hash, &sg[i], cur_len); 1197 1198 data_length -= cur_len; 1199 page_off = 0; 1200 i++; 1201 } 1202 1203 if (padding) { 1204 struct scatterlist pad_sg; 1205 1206 sg_init_one(&pad_sg, pad_bytes, padding); 1207 crypto_hash_update(hash, &pad_sg, padding); 1208 } 1209 crypto_hash_final(hash, (u8 *) &data_crc); 1210 1211 return data_crc; 1212 } 1213 1214 static void iscsit_do_crypto_hash_buf( 1215 struct hash_desc *hash, 1216 unsigned char *buf, 1217 u32 payload_length, 1218 u32 padding, 1219 u8 *pad_bytes, 1220 u8 *data_crc) 1221 { 1222 struct scatterlist sg; 1223 1224 crypto_hash_init(hash); 1225 1226 sg_init_one(&sg, buf, payload_length); 1227 crypto_hash_update(hash, &sg, payload_length); 1228 1229 if (padding) { 1230 sg_init_one(&sg, pad_bytes, padding); 1231 crypto_hash_update(hash, &sg, padding); 1232 } 1233 crypto_hash_final(hash, data_crc); 1234 } 1235 1236 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) 1237 { 1238 int iov_ret, ooo_cmdsn = 0, ret; 1239 u8 data_crc_failed = 0; 1240 u32 checksum, iov_count = 0, padding = 0, rx_got = 0; 1241 u32 rx_size = 0, payload_length; 1242 struct iscsi_cmd *cmd = NULL; 1243 struct se_cmd *se_cmd; 1244 struct iscsi_data *hdr; 1245 struct kvec *iov; 1246 unsigned long flags; 1247 1248 hdr = (struct iscsi_data *) buf; 1249 payload_length = ntoh24(hdr->dlength); 1250 hdr->itt = be32_to_cpu(hdr->itt); 1251 hdr->ttt = be32_to_cpu(hdr->ttt); 1252 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 1253 hdr->datasn = be32_to_cpu(hdr->datasn); 1254 hdr->offset = be32_to_cpu(hdr->offset); 1255 1256 if (!payload_length) { 1257 pr_err("DataOUT payload is ZERO, protocol error.\n"); 1258 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1259 buf, conn); 1260 } 1261 1262 /* iSCSI write */ 1263 spin_lock_bh(&conn->sess->session_stats_lock); 1264 conn->sess->rx_data_octets += payload_length; 1265 if (conn->sess->se_sess->se_node_acl) { 1266 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock); 1267 conn->sess->se_sess->se_node_acl->write_bytes += payload_length; 1268 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock); 1269 } 1270 spin_unlock_bh(&conn->sess->session_stats_lock); 1271 1272 if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) { 1273 pr_err("DataSegmentLength: %u is greater than" 1274 " MaxRecvDataSegmentLength: %u\n", payload_length, 1275 conn->conn_ops->MaxRecvDataSegmentLength); 1276 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1277 buf, conn); 1278 } 1279 1280 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 1281 payload_length); 1282 if (!cmd) 1283 return 0; 1284 1285 pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x," 1286 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", 1287 hdr->itt, hdr->ttt, hdr->datasn, hdr->offset, 1288 payload_length, conn->cid); 1289 1290 if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) { 1291 pr_err("Command ITT: 0x%08x received DataOUT after" 1292 " last DataOUT received, dumping payload\n", 1293 cmd->init_task_tag); 1294 return iscsit_dump_data_payload(conn, payload_length, 1); 1295 } 1296 1297 if (cmd->data_direction != DMA_TO_DEVICE) { 1298 pr_err("Command ITT: 0x%08x received DataOUT for a" 1299 " NON-WRITE command.\n", cmd->init_task_tag); 1300 return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR, 1301 1, 0, buf, cmd); 1302 } 1303 se_cmd = &cmd->se_cmd; 1304 iscsit_mod_dataout_timer(cmd); 1305 1306 if ((hdr->offset + payload_length) > cmd->data_length) { 1307 pr_err("DataOut Offset: %u, Length %u greater than" 1308 " iSCSI Command EDTL %u, protocol error.\n", 1309 hdr->offset, payload_length, cmd->data_length); 1310 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID, 1311 1, 0, buf, cmd); 1312 } 1313 1314 if (cmd->unsolicited_data) { 1315 int dump_unsolicited_data = 0; 1316 1317 if (conn->sess->sess_ops->InitialR2T) { 1318 pr_err("Received unexpected unsolicited data" 1319 " while InitialR2T=Yes, protocol error.\n"); 1320 transport_send_check_condition_and_sense(&cmd->se_cmd, 1321 TCM_UNEXPECTED_UNSOLICITED_DATA, 0); 1322 return -1; 1323 } 1324 /* 1325 * Special case for dealing with Unsolicited DataOUT 1326 * and Unsupported SAM WRITE Opcodes and SE resource allocation 1327 * failures; 1328 */ 1329 1330 /* Something's amiss if we're not in WRITE_PENDING state... */ 1331 spin_lock_irqsave(&se_cmd->t_state_lock, flags); 1332 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING); 1333 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); 1334 1335 spin_lock_irqsave(&se_cmd->t_state_lock, flags); 1336 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) || 1337 (se_cmd->se_cmd_flags & SCF_SCSI_CDB_EXCEPTION)) 1338 dump_unsolicited_data = 1; 1339 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); 1340 1341 if (dump_unsolicited_data) { 1342 /* 1343 * Check if a delayed TASK_ABORTED status needs to 1344 * be sent now if the ISCSI_FLAG_CMD_FINAL has been 1345 * received with the unsolicitied data out. 1346 */ 1347 if (hdr->flags & ISCSI_FLAG_CMD_FINAL) 1348 iscsit_stop_dataout_timer(cmd); 1349 1350 transport_check_aborted_status(se_cmd, 1351 (hdr->flags & ISCSI_FLAG_CMD_FINAL)); 1352 return iscsit_dump_data_payload(conn, payload_length, 1); 1353 } 1354 } else { 1355 /* 1356 * For the normal solicited data path: 1357 * 1358 * Check for a delayed TASK_ABORTED status and dump any 1359 * incoming data out payload if one exists. Also, when the 1360 * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current 1361 * data out sequence, we decrement outstanding_r2ts. Once 1362 * outstanding_r2ts reaches zero, go ahead and send the delayed 1363 * TASK_ABORTED status. 1364 */ 1365 if (se_cmd->transport_state & CMD_T_ABORTED) { 1366 if (hdr->flags & ISCSI_FLAG_CMD_FINAL) 1367 if (--cmd->outstanding_r2ts < 1) { 1368 iscsit_stop_dataout_timer(cmd); 1369 transport_check_aborted_status( 1370 se_cmd, 1); 1371 } 1372 1373 return iscsit_dump_data_payload(conn, payload_length, 1); 1374 } 1375 } 1376 /* 1377 * Preform DataSN, DataSequenceInOrder, DataPDUInOrder, and 1378 * within-command recovery checks before receiving the payload. 1379 */ 1380 ret = iscsit_check_pre_dataout(cmd, buf); 1381 if (ret == DATAOUT_WITHIN_COMMAND_RECOVERY) 1382 return 0; 1383 else if (ret == DATAOUT_CANNOT_RECOVER) 1384 return -1; 1385 1386 rx_size += payload_length; 1387 iov = &cmd->iov_data[0]; 1388 1389 iov_ret = iscsit_map_iovec(cmd, iov, hdr->offset, payload_length); 1390 if (iov_ret < 0) 1391 return -1; 1392 1393 iov_count += iov_ret; 1394 1395 padding = ((-payload_length) & 3); 1396 if (padding != 0) { 1397 iov[iov_count].iov_base = cmd->pad_bytes; 1398 iov[iov_count++].iov_len = padding; 1399 rx_size += padding; 1400 pr_debug("Receiving %u padding bytes.\n", padding); 1401 } 1402 1403 if (conn->conn_ops->DataDigest) { 1404 iov[iov_count].iov_base = &checksum; 1405 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 1406 rx_size += ISCSI_CRC_LEN; 1407 } 1408 1409 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); 1410 1411 iscsit_unmap_iovec(cmd); 1412 1413 if (rx_got != rx_size) 1414 return -1; 1415 1416 if (conn->conn_ops->DataDigest) { 1417 u32 data_crc; 1418 1419 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd, 1420 hdr->offset, payload_length, padding, 1421 cmd->pad_bytes); 1422 1423 if (checksum != data_crc) { 1424 pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 1425 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x" 1426 " does not match computed 0x%08x\n", 1427 hdr->itt, hdr->offset, payload_length, 1428 hdr->datasn, checksum, data_crc); 1429 data_crc_failed = 1; 1430 } else { 1431 pr_debug("Got CRC32C DataDigest 0x%08x for" 1432 " %u bytes of Data Out\n", checksum, 1433 payload_length); 1434 } 1435 } 1436 /* 1437 * Increment post receive data and CRC values or perform 1438 * within-command recovery. 1439 */ 1440 ret = iscsit_check_post_dataout(cmd, buf, data_crc_failed); 1441 if ((ret == DATAOUT_NORMAL) || (ret == DATAOUT_WITHIN_COMMAND_RECOVERY)) 1442 return 0; 1443 else if (ret == DATAOUT_SEND_R2T) { 1444 iscsit_set_dataout_sequence_values(cmd); 1445 iscsit_build_r2ts_for_cmd(cmd, conn, 0); 1446 } else if (ret == DATAOUT_SEND_TO_TRANSPORT) { 1447 /* 1448 * Handle extra special case for out of order 1449 * Unsolicited Data Out. 1450 */ 1451 spin_lock_bh(&cmd->istate_lock); 1452 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN); 1453 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1454 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1455 spin_unlock_bh(&cmd->istate_lock); 1456 1457 iscsit_stop_dataout_timer(cmd); 1458 return (!ooo_cmdsn) ? transport_generic_handle_data( 1459 &cmd->se_cmd) : 0; 1460 } else /* DATAOUT_CANNOT_RECOVER */ 1461 return -1; 1462 1463 return 0; 1464 } 1465 1466 static int iscsit_handle_nop_out( 1467 struct iscsi_conn *conn, 1468 unsigned char *buf) 1469 { 1470 unsigned char *ping_data = NULL; 1471 int cmdsn_ret, niov = 0, ret = 0, rx_got, rx_size; 1472 u32 checksum, data_crc, padding = 0, payload_length; 1473 struct iscsi_cmd *cmd = NULL; 1474 struct kvec *iov = NULL; 1475 struct iscsi_nopout *hdr; 1476 1477 hdr = (struct iscsi_nopout *) buf; 1478 payload_length = ntoh24(hdr->dlength); 1479 hdr->itt = be32_to_cpu(hdr->itt); 1480 hdr->ttt = be32_to_cpu(hdr->ttt); 1481 hdr->cmdsn = be32_to_cpu(hdr->cmdsn); 1482 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 1483 1484 if ((hdr->itt == 0xFFFFFFFF) && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1485 pr_err("NOPOUT ITT is reserved, but Immediate Bit is" 1486 " not set, protocol error.\n"); 1487 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1488 buf, conn); 1489 } 1490 1491 if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) { 1492 pr_err("NOPOUT Ping Data DataSegmentLength: %u is" 1493 " greater than MaxRecvDataSegmentLength: %u, protocol" 1494 " error.\n", payload_length, 1495 conn->conn_ops->MaxRecvDataSegmentLength); 1496 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1497 buf, conn); 1498 } 1499 1500 pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%09x," 1501 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n", 1502 (hdr->itt == 0xFFFFFFFF) ? "Response" : "Request", 1503 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn, 1504 payload_length); 1505 /* 1506 * This is not a response to a Unsolicited NopIN, which means 1507 * it can either be a NOPOUT ping request (with a valid ITT), 1508 * or a NOPOUT not requesting a NOPIN (with a reserved ITT). 1509 * Either way, make sure we allocate an struct iscsi_cmd, as both 1510 * can contain ping data. 1511 */ 1512 if (hdr->ttt == 0xFFFFFFFF) { 1513 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 1514 if (!cmd) 1515 return iscsit_add_reject( 1516 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1517 1, buf, conn); 1518 1519 cmd->iscsi_opcode = ISCSI_OP_NOOP_OUT; 1520 cmd->i_state = ISTATE_SEND_NOPIN; 1521 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1522 1 : 0); 1523 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 1524 cmd->targ_xfer_tag = 0xFFFFFFFF; 1525 cmd->cmd_sn = hdr->cmdsn; 1526 cmd->exp_stat_sn = hdr->exp_statsn; 1527 cmd->data_direction = DMA_NONE; 1528 } 1529 1530 if (payload_length && (hdr->ttt == 0xFFFFFFFF)) { 1531 rx_size = payload_length; 1532 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1533 if (!ping_data) { 1534 pr_err("Unable to allocate memory for" 1535 " NOPOUT ping data.\n"); 1536 ret = -1; 1537 goto out; 1538 } 1539 1540 iov = &cmd->iov_misc[0]; 1541 iov[niov].iov_base = ping_data; 1542 iov[niov++].iov_len = payload_length; 1543 1544 padding = ((-payload_length) & 3); 1545 if (padding != 0) { 1546 pr_debug("Receiving %u additional bytes" 1547 " for padding.\n", padding); 1548 iov[niov].iov_base = &cmd->pad_bytes; 1549 iov[niov++].iov_len = padding; 1550 rx_size += padding; 1551 } 1552 if (conn->conn_ops->DataDigest) { 1553 iov[niov].iov_base = &checksum; 1554 iov[niov++].iov_len = ISCSI_CRC_LEN; 1555 rx_size += ISCSI_CRC_LEN; 1556 } 1557 1558 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size); 1559 if (rx_got != rx_size) { 1560 ret = -1; 1561 goto out; 1562 } 1563 1564 if (conn->conn_ops->DataDigest) { 1565 iscsit_do_crypto_hash_buf(&conn->conn_rx_hash, 1566 ping_data, payload_length, 1567 padding, cmd->pad_bytes, 1568 (u8 *)&data_crc); 1569 1570 if (checksum != data_crc) { 1571 pr_err("Ping data CRC32C DataDigest" 1572 " 0x%08x does not match computed 0x%08x\n", 1573 checksum, data_crc); 1574 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1575 pr_err("Unable to recover from" 1576 " NOPOUT Ping DataCRC failure while in" 1577 " ERL=0.\n"); 1578 ret = -1; 1579 goto out; 1580 } else { 1581 /* 1582 * Silently drop this PDU and let the 1583 * initiator plug the CmdSN gap. 1584 */ 1585 pr_debug("Dropping NOPOUT" 1586 " Command CmdSN: 0x%08x due to" 1587 " DataCRC error.\n", hdr->cmdsn); 1588 ret = 0; 1589 goto out; 1590 } 1591 } else { 1592 pr_debug("Got CRC32C DataDigest" 1593 " 0x%08x for %u bytes of ping data.\n", 1594 checksum, payload_length); 1595 } 1596 } 1597 1598 ping_data[payload_length] = '\0'; 1599 /* 1600 * Attach ping data to struct iscsi_cmd->buf_ptr. 1601 */ 1602 cmd->buf_ptr = ping_data; 1603 cmd->buf_ptr_size = payload_length; 1604 1605 pr_debug("Got %u bytes of NOPOUT ping" 1606 " data.\n", payload_length); 1607 pr_debug("Ping Data: \"%s\"\n", ping_data); 1608 } 1609 1610 if (hdr->itt != 0xFFFFFFFF) { 1611 if (!cmd) { 1612 pr_err("Checking CmdSN for NOPOUT," 1613 " but cmd is NULL!\n"); 1614 return -1; 1615 } 1616 /* 1617 * Initiator is expecting a NopIN ping reply, 1618 */ 1619 spin_lock_bh(&conn->cmd_lock); 1620 list_add_tail(&cmd->i_list, &conn->conn_cmd_list); 1621 spin_unlock_bh(&conn->cmd_lock); 1622 1623 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn); 1624 1625 if (hdr->opcode & ISCSI_OP_IMMEDIATE) { 1626 iscsit_add_cmd_to_response_queue(cmd, conn, 1627 cmd->i_state); 1628 return 0; 1629 } 1630 1631 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 1632 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 1633 ret = 0; 1634 goto ping_out; 1635 } 1636 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1637 return iscsit_add_reject_from_cmd( 1638 ISCSI_REASON_PROTOCOL_ERROR, 1639 1, 0, buf, cmd); 1640 1641 return 0; 1642 } 1643 1644 if (hdr->ttt != 0xFFFFFFFF) { 1645 /* 1646 * This was a response to a unsolicited NOPIN ping. 1647 */ 1648 cmd = iscsit_find_cmd_from_ttt(conn, hdr->ttt); 1649 if (!cmd) 1650 return -1; 1651 1652 iscsit_stop_nopin_response_timer(conn); 1653 1654 cmd->i_state = ISTATE_REMOVE; 1655 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 1656 iscsit_start_nopin_timer(conn); 1657 } else { 1658 /* 1659 * Initiator is not expecting a NOPIN is response. 1660 * Just ignore for now. 1661 * 1662 * iSCSI v19-91 10.18 1663 * "A NOP-OUT may also be used to confirm a changed 1664 * ExpStatSN if another PDU will not be available 1665 * for a long time." 1666 */ 1667 ret = 0; 1668 goto out; 1669 } 1670 1671 return 0; 1672 out: 1673 if (cmd) 1674 iscsit_release_cmd(cmd); 1675 ping_out: 1676 kfree(ping_data); 1677 return ret; 1678 } 1679 1680 static int iscsit_handle_task_mgt_cmd( 1681 struct iscsi_conn *conn, 1682 unsigned char *buf) 1683 { 1684 struct iscsi_cmd *cmd; 1685 struct se_tmr_req *se_tmr; 1686 struct iscsi_tmr_req *tmr_req; 1687 struct iscsi_tm *hdr; 1688 int out_of_order_cmdsn = 0; 1689 int ret; 1690 u8 function; 1691 1692 hdr = (struct iscsi_tm *) buf; 1693 hdr->itt = be32_to_cpu(hdr->itt); 1694 hdr->rtt = be32_to_cpu(hdr->rtt); 1695 hdr->cmdsn = be32_to_cpu(hdr->cmdsn); 1696 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 1697 hdr->refcmdsn = be32_to_cpu(hdr->refcmdsn); 1698 hdr->exp_datasn = be32_to_cpu(hdr->exp_datasn); 1699 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL; 1700 function = hdr->flags; 1701 1702 pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:" 1703 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:" 1704 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function, 1705 hdr->rtt, hdr->refcmdsn, conn->cid); 1706 1707 if ((function != ISCSI_TM_FUNC_ABORT_TASK) && 1708 ((function != ISCSI_TM_FUNC_TASK_REASSIGN) && 1709 (hdr->rtt != ISCSI_RESERVED_TAG))) { 1710 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n"); 1711 hdr->rtt = ISCSI_RESERVED_TAG; 1712 } 1713 1714 if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) && 1715 !(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1716 pr_err("Task Management Request TASK_REASSIGN not" 1717 " issued as immediate command, bad iSCSI Initiator" 1718 "implementation\n"); 1719 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1720 buf, conn); 1721 } 1722 if ((function != ISCSI_TM_FUNC_ABORT_TASK) && 1723 (hdr->refcmdsn != ISCSI_RESERVED_TAG)) 1724 hdr->refcmdsn = ISCSI_RESERVED_TAG; 1725 1726 cmd = iscsit_allocate_se_cmd_for_tmr(conn, function); 1727 if (!cmd) 1728 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1729 1, buf, conn); 1730 1731 cmd->iscsi_opcode = ISCSI_OP_SCSI_TMFUNC; 1732 cmd->i_state = ISTATE_SEND_TASKMGTRSP; 1733 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 1734 cmd->init_task_tag = hdr->itt; 1735 cmd->targ_xfer_tag = 0xFFFFFFFF; 1736 cmd->cmd_sn = hdr->cmdsn; 1737 cmd->exp_stat_sn = hdr->exp_statsn; 1738 se_tmr = cmd->se_cmd.se_tmr_req; 1739 tmr_req = cmd->tmr_req; 1740 /* 1741 * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN 1742 */ 1743 if (function != ISCSI_TM_FUNC_TASK_REASSIGN) { 1744 ret = transport_lookup_tmr_lun(&cmd->se_cmd, 1745 scsilun_to_int(&hdr->lun)); 1746 if (ret < 0) { 1747 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; 1748 se_tmr->response = ISCSI_TMF_RSP_NO_LUN; 1749 goto attach; 1750 } 1751 } 1752 1753 switch (function) { 1754 case ISCSI_TM_FUNC_ABORT_TASK: 1755 se_tmr->response = iscsit_tmr_abort_task(cmd, buf); 1756 if (se_tmr->response != ISCSI_TMF_RSP_COMPLETE) { 1757 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; 1758 goto attach; 1759 } 1760 break; 1761 case ISCSI_TM_FUNC_ABORT_TASK_SET: 1762 case ISCSI_TM_FUNC_CLEAR_ACA: 1763 case ISCSI_TM_FUNC_CLEAR_TASK_SET: 1764 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET: 1765 break; 1766 case ISCSI_TM_FUNC_TARGET_WARM_RESET: 1767 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) { 1768 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; 1769 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED; 1770 goto attach; 1771 } 1772 break; 1773 case ISCSI_TM_FUNC_TARGET_COLD_RESET: 1774 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) { 1775 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; 1776 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED; 1777 goto attach; 1778 } 1779 break; 1780 case ISCSI_TM_FUNC_TASK_REASSIGN: 1781 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf); 1782 /* 1783 * Perform sanity checks on the ExpDataSN only if the 1784 * TASK_REASSIGN was successful. 1785 */ 1786 if (se_tmr->response != ISCSI_TMF_RSP_COMPLETE) 1787 break; 1788 1789 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0) 1790 return iscsit_add_reject_from_cmd( 1791 ISCSI_REASON_BOOKMARK_INVALID, 1, 1, 1792 buf, cmd); 1793 break; 1794 default: 1795 pr_err("Unknown TMR function: 0x%02x, protocol" 1796 " error.\n", function); 1797 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; 1798 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED; 1799 goto attach; 1800 } 1801 1802 if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) && 1803 (se_tmr->response == ISCSI_TMF_RSP_COMPLETE)) 1804 se_tmr->call_transport = 1; 1805 attach: 1806 spin_lock_bh(&conn->cmd_lock); 1807 list_add_tail(&cmd->i_list, &conn->conn_cmd_list); 1808 spin_unlock_bh(&conn->cmd_lock); 1809 1810 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1811 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 1812 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP) 1813 out_of_order_cmdsn = 1; 1814 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) 1815 return 0; 1816 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1817 return iscsit_add_reject_from_cmd( 1818 ISCSI_REASON_PROTOCOL_ERROR, 1819 1, 0, buf, cmd); 1820 } 1821 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn); 1822 1823 if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE)) 1824 return 0; 1825 /* 1826 * Found the referenced task, send to transport for processing. 1827 */ 1828 if (se_tmr->call_transport) 1829 return transport_generic_handle_tmr(&cmd->se_cmd); 1830 1831 /* 1832 * Could not find the referenced LUN, task, or Task Management 1833 * command not authorized or supported. Change state and 1834 * let the tx_thread send the response. 1835 * 1836 * For connection recovery, this is also the default action for 1837 * TMR TASK_REASSIGN. 1838 */ 1839 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 1840 return 0; 1841 } 1842 1843 /* #warning FIXME: Support Text Command parameters besides SendTargets */ 1844 static int iscsit_handle_text_cmd( 1845 struct iscsi_conn *conn, 1846 unsigned char *buf) 1847 { 1848 char *text_ptr, *text_in; 1849 int cmdsn_ret, niov = 0, rx_got, rx_size; 1850 u32 checksum = 0, data_crc = 0, payload_length; 1851 u32 padding = 0, pad_bytes = 0, text_length = 0; 1852 struct iscsi_cmd *cmd; 1853 struct kvec iov[3]; 1854 struct iscsi_text *hdr; 1855 1856 hdr = (struct iscsi_text *) buf; 1857 payload_length = ntoh24(hdr->dlength); 1858 hdr->itt = be32_to_cpu(hdr->itt); 1859 hdr->ttt = be32_to_cpu(hdr->ttt); 1860 hdr->cmdsn = be32_to_cpu(hdr->cmdsn); 1861 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 1862 1863 if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) { 1864 pr_err("Unable to accept text parameter length: %u" 1865 "greater than MaxRecvDataSegmentLength %u.\n", 1866 payload_length, conn->conn_ops->MaxRecvDataSegmentLength); 1867 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1868 buf, conn); 1869 } 1870 1871 pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x," 1872 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn, 1873 hdr->exp_statsn, payload_length); 1874 1875 rx_size = text_length = payload_length; 1876 if (text_length) { 1877 text_in = kzalloc(text_length, GFP_KERNEL); 1878 if (!text_in) { 1879 pr_err("Unable to allocate memory for" 1880 " incoming text parameters\n"); 1881 return -1; 1882 } 1883 1884 memset(iov, 0, 3 * sizeof(struct kvec)); 1885 iov[niov].iov_base = text_in; 1886 iov[niov++].iov_len = text_length; 1887 1888 padding = ((-payload_length) & 3); 1889 if (padding != 0) { 1890 iov[niov].iov_base = &pad_bytes; 1891 iov[niov++].iov_len = padding; 1892 rx_size += padding; 1893 pr_debug("Receiving %u additional bytes" 1894 " for padding.\n", padding); 1895 } 1896 if (conn->conn_ops->DataDigest) { 1897 iov[niov].iov_base = &checksum; 1898 iov[niov++].iov_len = ISCSI_CRC_LEN; 1899 rx_size += ISCSI_CRC_LEN; 1900 } 1901 1902 rx_got = rx_data(conn, &iov[0], niov, rx_size); 1903 if (rx_got != rx_size) { 1904 kfree(text_in); 1905 return -1; 1906 } 1907 1908 if (conn->conn_ops->DataDigest) { 1909 iscsit_do_crypto_hash_buf(&conn->conn_rx_hash, 1910 text_in, text_length, 1911 padding, (u8 *)&pad_bytes, 1912 (u8 *)&data_crc); 1913 1914 if (checksum != data_crc) { 1915 pr_err("Text data CRC32C DataDigest" 1916 " 0x%08x does not match computed" 1917 " 0x%08x\n", checksum, data_crc); 1918 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1919 pr_err("Unable to recover from" 1920 " Text Data digest failure while in" 1921 " ERL=0.\n"); 1922 kfree(text_in); 1923 return -1; 1924 } else { 1925 /* 1926 * Silently drop this PDU and let the 1927 * initiator plug the CmdSN gap. 1928 */ 1929 pr_debug("Dropping Text" 1930 " Command CmdSN: 0x%08x due to" 1931 " DataCRC error.\n", hdr->cmdsn); 1932 kfree(text_in); 1933 return 0; 1934 } 1935 } else { 1936 pr_debug("Got CRC32C DataDigest" 1937 " 0x%08x for %u bytes of text data.\n", 1938 checksum, text_length); 1939 } 1940 } 1941 text_in[text_length - 1] = '\0'; 1942 pr_debug("Successfully read %d bytes of text" 1943 " data.\n", text_length); 1944 1945 if (strncmp("SendTargets", text_in, 11) != 0) { 1946 pr_err("Received Text Data that is not" 1947 " SendTargets, cannot continue.\n"); 1948 kfree(text_in); 1949 return -1; 1950 } 1951 text_ptr = strchr(text_in, '='); 1952 if (!text_ptr) { 1953 pr_err("No \"=\" separator found in Text Data," 1954 " cannot continue.\n"); 1955 kfree(text_in); 1956 return -1; 1957 } 1958 if (strncmp("=All", text_ptr, 4) != 0) { 1959 pr_err("Unable to locate All value for" 1960 " SendTargets key, cannot continue.\n"); 1961 kfree(text_in); 1962 return -1; 1963 } 1964 /*#warning Support SendTargets=(iSCSI Target Name/Nothing) values. */ 1965 kfree(text_in); 1966 } 1967 1968 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 1969 if (!cmd) 1970 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1971 1, buf, conn); 1972 1973 cmd->iscsi_opcode = ISCSI_OP_TEXT; 1974 cmd->i_state = ISTATE_SEND_TEXTRSP; 1975 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 1976 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 1977 cmd->targ_xfer_tag = 0xFFFFFFFF; 1978 cmd->cmd_sn = hdr->cmdsn; 1979 cmd->exp_stat_sn = hdr->exp_statsn; 1980 cmd->data_direction = DMA_NONE; 1981 1982 spin_lock_bh(&conn->cmd_lock); 1983 list_add_tail(&cmd->i_list, &conn->conn_cmd_list); 1984 spin_unlock_bh(&conn->cmd_lock); 1985 1986 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn); 1987 1988 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1989 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 1990 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1991 return iscsit_add_reject_from_cmd( 1992 ISCSI_REASON_PROTOCOL_ERROR, 1993 1, 0, buf, cmd); 1994 1995 return 0; 1996 } 1997 1998 return iscsit_execute_cmd(cmd, 0); 1999 } 2000 2001 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2002 { 2003 struct iscsi_conn *conn_p; 2004 struct iscsi_session *sess = conn->sess; 2005 2006 pr_debug("Received logout request CLOSESESSION on CID: %hu" 2007 " for SID: %u.\n", conn->cid, conn->sess->sid); 2008 2009 atomic_set(&sess->session_logout, 1); 2010 atomic_set(&conn->conn_logout_remove, 1); 2011 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION; 2012 2013 iscsit_inc_conn_usage_count(conn); 2014 iscsit_inc_session_usage_count(sess); 2015 2016 spin_lock_bh(&sess->conn_lock); 2017 list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) { 2018 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN) 2019 continue; 2020 2021 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 2022 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT; 2023 } 2024 spin_unlock_bh(&sess->conn_lock); 2025 2026 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2027 2028 return 0; 2029 } 2030 2031 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2032 { 2033 struct iscsi_conn *l_conn; 2034 struct iscsi_session *sess = conn->sess; 2035 2036 pr_debug("Received logout request CLOSECONNECTION for CID:" 2037 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); 2038 2039 /* 2040 * A Logout Request with a CLOSECONNECTION reason code for a CID 2041 * can arrive on a connection with a differing CID. 2042 */ 2043 if (conn->cid == cmd->logout_cid) { 2044 spin_lock_bh(&conn->state_lock); 2045 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 2046 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT; 2047 2048 atomic_set(&conn->conn_logout_remove, 1); 2049 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION; 2050 iscsit_inc_conn_usage_count(conn); 2051 2052 spin_unlock_bh(&conn->state_lock); 2053 } else { 2054 /* 2055 * Handle all different cid CLOSECONNECTION requests in 2056 * iscsit_logout_post_handler_diffcid() as to give enough 2057 * time for any non immediate command's CmdSN to be 2058 * acknowledged on the connection in question. 2059 * 2060 * Here we simply make sure the CID is still around. 2061 */ 2062 l_conn = iscsit_get_conn_from_cid(sess, 2063 cmd->logout_cid); 2064 if (!l_conn) { 2065 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND; 2066 iscsit_add_cmd_to_response_queue(cmd, conn, 2067 cmd->i_state); 2068 return 0; 2069 } 2070 2071 iscsit_dec_conn_usage_count(l_conn); 2072 } 2073 2074 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2075 2076 return 0; 2077 } 2078 2079 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2080 { 2081 struct iscsi_session *sess = conn->sess; 2082 2083 pr_debug("Received explicit REMOVECONNFORRECOVERY logout for" 2084 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); 2085 2086 if (sess->sess_ops->ErrorRecoveryLevel != 2) { 2087 pr_err("Received Logout Request REMOVECONNFORRECOVERY" 2088 " while ERL!=2.\n"); 2089 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED; 2090 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2091 return 0; 2092 } 2093 2094 if (conn->cid == cmd->logout_cid) { 2095 pr_err("Received Logout Request REMOVECONNFORRECOVERY" 2096 " with CID: %hu on CID: %hu, implementation error.\n", 2097 cmd->logout_cid, conn->cid); 2098 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED; 2099 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2100 return 0; 2101 } 2102 2103 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2104 2105 return 0; 2106 } 2107 2108 static int iscsit_handle_logout_cmd( 2109 struct iscsi_conn *conn, 2110 unsigned char *buf) 2111 { 2112 int cmdsn_ret, logout_remove = 0; 2113 u8 reason_code = 0; 2114 struct iscsi_cmd *cmd; 2115 struct iscsi_logout *hdr; 2116 struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn); 2117 2118 hdr = (struct iscsi_logout *) buf; 2119 reason_code = (hdr->flags & 0x7f); 2120 hdr->itt = be32_to_cpu(hdr->itt); 2121 hdr->cid = be16_to_cpu(hdr->cid); 2122 hdr->cmdsn = be32_to_cpu(hdr->cmdsn); 2123 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 2124 2125 if (tiqn) { 2126 spin_lock(&tiqn->logout_stats.lock); 2127 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) 2128 tiqn->logout_stats.normal_logouts++; 2129 else 2130 tiqn->logout_stats.abnormal_logouts++; 2131 spin_unlock(&tiqn->logout_stats.lock); 2132 } 2133 2134 pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x" 2135 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n", 2136 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code, 2137 hdr->cid, conn->cid); 2138 2139 if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) { 2140 pr_err("Received logout request on connection that" 2141 " is not in logged in state, ignoring request.\n"); 2142 return 0; 2143 } 2144 2145 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 2146 if (!cmd) 2147 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1, 2148 buf, conn); 2149 2150 cmd->iscsi_opcode = ISCSI_OP_LOGOUT; 2151 cmd->i_state = ISTATE_SEND_LOGOUTRSP; 2152 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 2153 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 2154 cmd->targ_xfer_tag = 0xFFFFFFFF; 2155 cmd->cmd_sn = hdr->cmdsn; 2156 cmd->exp_stat_sn = hdr->exp_statsn; 2157 cmd->logout_cid = hdr->cid; 2158 cmd->logout_reason = reason_code; 2159 cmd->data_direction = DMA_NONE; 2160 2161 /* 2162 * We need to sleep in these cases (by returning 1) until the Logout 2163 * Response gets sent in the tx thread. 2164 */ 2165 if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) || 2166 ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) && 2167 (hdr->cid == conn->cid))) 2168 logout_remove = 1; 2169 2170 spin_lock_bh(&conn->cmd_lock); 2171 list_add_tail(&cmd->i_list, &conn->conn_cmd_list); 2172 spin_unlock_bh(&conn->cmd_lock); 2173 2174 if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY) 2175 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn); 2176 2177 /* 2178 * Immediate commands are executed, well, immediately. 2179 * Non-Immediate Logout Commands are executed in CmdSN order. 2180 */ 2181 if (hdr->opcode & ISCSI_OP_IMMEDIATE) { 2182 int ret = iscsit_execute_cmd(cmd, 0); 2183 2184 if (ret < 0) 2185 return ret; 2186 } else { 2187 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 2188 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 2189 logout_remove = 0; 2190 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) { 2191 return iscsit_add_reject_from_cmd( 2192 ISCSI_REASON_PROTOCOL_ERROR, 2193 1, 0, buf, cmd); 2194 } 2195 } 2196 2197 return logout_remove; 2198 } 2199 2200 static int iscsit_handle_snack( 2201 struct iscsi_conn *conn, 2202 unsigned char *buf) 2203 { 2204 struct iscsi_snack *hdr; 2205 2206 hdr = (struct iscsi_snack *) buf; 2207 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL; 2208 hdr->itt = be32_to_cpu(hdr->itt); 2209 hdr->ttt = be32_to_cpu(hdr->ttt); 2210 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 2211 hdr->begrun = be32_to_cpu(hdr->begrun); 2212 hdr->runlength = be32_to_cpu(hdr->runlength); 2213 2214 pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:" 2215 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x," 2216 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags, 2217 hdr->begrun, hdr->runlength, conn->cid); 2218 2219 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2220 pr_err("Initiator sent SNACK request while in" 2221 " ErrorRecoveryLevel=0.\n"); 2222 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 2223 buf, conn); 2224 } 2225 /* 2226 * SNACK_DATA and SNACK_R2T are both 0, so check which function to 2227 * call from inside iscsi_send_recovery_datain_or_r2t(). 2228 */ 2229 switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) { 2230 case 0: 2231 return iscsit_handle_recovery_datain_or_r2t(conn, buf, 2232 hdr->itt, hdr->ttt, hdr->begrun, hdr->runlength); 2233 case ISCSI_FLAG_SNACK_TYPE_STATUS: 2234 return iscsit_handle_status_snack(conn, hdr->itt, hdr->ttt, 2235 hdr->begrun, hdr->runlength); 2236 case ISCSI_FLAG_SNACK_TYPE_DATA_ACK: 2237 return iscsit_handle_data_ack(conn, hdr->ttt, hdr->begrun, 2238 hdr->runlength); 2239 case ISCSI_FLAG_SNACK_TYPE_RDATA: 2240 /* FIXME: Support R-Data SNACK */ 2241 pr_err("R-Data SNACK Not Supported.\n"); 2242 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 2243 buf, conn); 2244 default: 2245 pr_err("Unknown SNACK type 0x%02x, protocol" 2246 " error.\n", hdr->flags & 0x0f); 2247 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 2248 buf, conn); 2249 } 2250 2251 return 0; 2252 } 2253 2254 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn) 2255 { 2256 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || 2257 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { 2258 wait_for_completion_interruptible_timeout( 2259 &conn->rx_half_close_comp, 2260 ISCSI_RX_THREAD_TCP_TIMEOUT * HZ); 2261 } 2262 } 2263 2264 static int iscsit_handle_immediate_data( 2265 struct iscsi_cmd *cmd, 2266 unsigned char *buf, 2267 u32 length) 2268 { 2269 int iov_ret, rx_got = 0, rx_size = 0; 2270 u32 checksum, iov_count = 0, padding = 0; 2271 struct iscsi_conn *conn = cmd->conn; 2272 struct kvec *iov; 2273 2274 iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, cmd->write_data_done, length); 2275 if (iov_ret < 0) 2276 return IMMEDIATE_DATA_CANNOT_RECOVER; 2277 2278 rx_size = length; 2279 iov_count = iov_ret; 2280 iov = &cmd->iov_data[0]; 2281 2282 padding = ((-length) & 3); 2283 if (padding != 0) { 2284 iov[iov_count].iov_base = cmd->pad_bytes; 2285 iov[iov_count++].iov_len = padding; 2286 rx_size += padding; 2287 } 2288 2289 if (conn->conn_ops->DataDigest) { 2290 iov[iov_count].iov_base = &checksum; 2291 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 2292 rx_size += ISCSI_CRC_LEN; 2293 } 2294 2295 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); 2296 2297 iscsit_unmap_iovec(cmd); 2298 2299 if (rx_got != rx_size) { 2300 iscsit_rx_thread_wait_for_tcp(conn); 2301 return IMMEDIATE_DATA_CANNOT_RECOVER; 2302 } 2303 2304 if (conn->conn_ops->DataDigest) { 2305 u32 data_crc; 2306 2307 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd, 2308 cmd->write_data_done, length, padding, 2309 cmd->pad_bytes); 2310 2311 if (checksum != data_crc) { 2312 pr_err("ImmediateData CRC32C DataDigest 0x%08x" 2313 " does not match computed 0x%08x\n", checksum, 2314 data_crc); 2315 2316 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2317 pr_err("Unable to recover from" 2318 " Immediate Data digest failure while" 2319 " in ERL=0.\n"); 2320 iscsit_add_reject_from_cmd( 2321 ISCSI_REASON_DATA_DIGEST_ERROR, 2322 1, 0, buf, cmd); 2323 return IMMEDIATE_DATA_CANNOT_RECOVER; 2324 } else { 2325 iscsit_add_reject_from_cmd( 2326 ISCSI_REASON_DATA_DIGEST_ERROR, 2327 0, 0, buf, cmd); 2328 return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 2329 } 2330 } else { 2331 pr_debug("Got CRC32C DataDigest 0x%08x for" 2332 " %u bytes of Immediate Data\n", checksum, 2333 length); 2334 } 2335 } 2336 2337 cmd->write_data_done += length; 2338 2339 if (cmd->write_data_done == cmd->data_length) { 2340 spin_lock_bh(&cmd->istate_lock); 2341 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 2342 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 2343 spin_unlock_bh(&cmd->istate_lock); 2344 } 2345 2346 return IMMEDIATE_DATA_NORMAL_OPERATION; 2347 } 2348 2349 /* 2350 * Called with sess->conn_lock held. 2351 */ 2352 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections 2353 with active network interface */ 2354 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn) 2355 { 2356 struct iscsi_cmd *cmd; 2357 struct iscsi_conn *conn_p; 2358 2359 /* 2360 * Only send a Asynchronous Message on connections whos network 2361 * interface is still functional. 2362 */ 2363 list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) { 2364 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) { 2365 iscsit_inc_conn_usage_count(conn_p); 2366 break; 2367 } 2368 } 2369 2370 if (!conn_p) 2371 return; 2372 2373 cmd = iscsit_allocate_cmd(conn_p, GFP_KERNEL); 2374 if (!cmd) { 2375 iscsit_dec_conn_usage_count(conn_p); 2376 return; 2377 } 2378 2379 cmd->logout_cid = conn->cid; 2380 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT; 2381 cmd->i_state = ISTATE_SEND_ASYNCMSG; 2382 2383 spin_lock_bh(&conn_p->cmd_lock); 2384 list_add_tail(&cmd->i_list, &conn_p->conn_cmd_list); 2385 spin_unlock_bh(&conn_p->cmd_lock); 2386 2387 iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state); 2388 iscsit_dec_conn_usage_count(conn_p); 2389 } 2390 2391 static int iscsit_send_conn_drop_async_message( 2392 struct iscsi_cmd *cmd, 2393 struct iscsi_conn *conn) 2394 { 2395 struct iscsi_async *hdr; 2396 2397 cmd->tx_size = ISCSI_HDR_LEN; 2398 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT; 2399 2400 hdr = (struct iscsi_async *) cmd->pdu; 2401 hdr->opcode = ISCSI_OP_ASYNC_EVENT; 2402 hdr->flags = ISCSI_FLAG_CMD_FINAL; 2403 cmd->init_task_tag = 0xFFFFFFFF; 2404 cmd->targ_xfer_tag = 0xFFFFFFFF; 2405 put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]); 2406 cmd->stat_sn = conn->stat_sn++; 2407 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2408 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2409 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2410 hdr->async_event = ISCSI_ASYNC_MSG_DROPPING_CONNECTION; 2411 hdr->param1 = cpu_to_be16(cmd->logout_cid); 2412 hdr->param2 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait); 2413 hdr->param3 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain); 2414 2415 if (conn->conn_ops->HeaderDigest) { 2416 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2417 2418 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2419 (unsigned char *)hdr, ISCSI_HDR_LEN, 2420 0, NULL, (u8 *)header_digest); 2421 2422 cmd->tx_size += ISCSI_CRC_LEN; 2423 pr_debug("Attaching CRC32C HeaderDigest to" 2424 " Async Message 0x%08x\n", *header_digest); 2425 } 2426 2427 cmd->iov_misc[0].iov_base = cmd->pdu; 2428 cmd->iov_misc[0].iov_len = cmd->tx_size; 2429 cmd->iov_misc_count = 1; 2430 2431 pr_debug("Sending Connection Dropped Async Message StatSN:" 2432 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn, 2433 cmd->logout_cid, conn->cid); 2434 return 0; 2435 } 2436 2437 static int iscsit_send_data_in( 2438 struct iscsi_cmd *cmd, 2439 struct iscsi_conn *conn, 2440 int *eodr) 2441 { 2442 int iov_ret = 0, set_statsn = 0; 2443 u32 iov_count = 0, tx_size = 0; 2444 struct iscsi_datain datain; 2445 struct iscsi_datain_req *dr; 2446 struct iscsi_data_rsp *hdr; 2447 struct kvec *iov; 2448 2449 memset(&datain, 0, sizeof(struct iscsi_datain)); 2450 dr = iscsit_get_datain_values(cmd, &datain); 2451 if (!dr) { 2452 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n", 2453 cmd->init_task_tag); 2454 return -1; 2455 } 2456 2457 /* 2458 * Be paranoid and double check the logic for now. 2459 */ 2460 if ((datain.offset + datain.length) > cmd->data_length) { 2461 pr_err("Command ITT: 0x%08x, datain.offset: %u and" 2462 " datain.length: %u exceeds cmd->data_length: %u\n", 2463 cmd->init_task_tag, datain.offset, datain.length, 2464 cmd->data_length); 2465 return -1; 2466 } 2467 2468 spin_lock_bh(&conn->sess->session_stats_lock); 2469 conn->sess->tx_data_octets += datain.length; 2470 if (conn->sess->se_sess->se_node_acl) { 2471 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock); 2472 conn->sess->se_sess->se_node_acl->read_bytes += datain.length; 2473 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock); 2474 } 2475 spin_unlock_bh(&conn->sess->session_stats_lock); 2476 /* 2477 * Special case for successfully execution w/ both DATAIN 2478 * and Sense Data. 2479 */ 2480 if ((datain.flags & ISCSI_FLAG_DATA_STATUS) && 2481 (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)) 2482 datain.flags &= ~ISCSI_FLAG_DATA_STATUS; 2483 else { 2484 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) || 2485 (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) { 2486 iscsit_increment_maxcmdsn(cmd, conn->sess); 2487 cmd->stat_sn = conn->stat_sn++; 2488 set_statsn = 1; 2489 } else if (dr->dr_complete == 2490 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY) 2491 set_statsn = 1; 2492 } 2493 2494 hdr = (struct iscsi_data_rsp *) cmd->pdu; 2495 memset(hdr, 0, ISCSI_HDR_LEN); 2496 hdr->opcode = ISCSI_OP_SCSI_DATA_IN; 2497 hdr->flags = datain.flags; 2498 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) { 2499 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { 2500 hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW; 2501 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 2502 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { 2503 hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW; 2504 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 2505 } 2506 } 2507 hton24(hdr->dlength, datain.length); 2508 if (hdr->flags & ISCSI_FLAG_DATA_ACK) 2509 int_to_scsilun(cmd->se_cmd.orig_fe_lun, 2510 (struct scsi_lun *)&hdr->lun); 2511 else 2512 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); 2513 2514 hdr->itt = cpu_to_be32(cmd->init_task_tag); 2515 hdr->ttt = (hdr->flags & ISCSI_FLAG_DATA_ACK) ? 2516 cpu_to_be32(cmd->targ_xfer_tag) : 2517 0xFFFFFFFF; 2518 hdr->statsn = (set_statsn) ? cpu_to_be32(cmd->stat_sn) : 2519 0xFFFFFFFF; 2520 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2521 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2522 hdr->datasn = cpu_to_be32(datain.data_sn); 2523 hdr->offset = cpu_to_be32(datain.offset); 2524 2525 iov = &cmd->iov_data[0]; 2526 iov[iov_count].iov_base = cmd->pdu; 2527 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 2528 tx_size += ISCSI_HDR_LEN; 2529 2530 if (conn->conn_ops->HeaderDigest) { 2531 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2532 2533 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2534 (unsigned char *)hdr, ISCSI_HDR_LEN, 2535 0, NULL, (u8 *)header_digest); 2536 2537 iov[0].iov_len += ISCSI_CRC_LEN; 2538 tx_size += ISCSI_CRC_LEN; 2539 2540 pr_debug("Attaching CRC32 HeaderDigest" 2541 " for DataIN PDU 0x%08x\n", *header_digest); 2542 } 2543 2544 iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], datain.offset, datain.length); 2545 if (iov_ret < 0) 2546 return -1; 2547 2548 iov_count += iov_ret; 2549 tx_size += datain.length; 2550 2551 cmd->padding = ((-datain.length) & 3); 2552 if (cmd->padding) { 2553 iov[iov_count].iov_base = cmd->pad_bytes; 2554 iov[iov_count++].iov_len = cmd->padding; 2555 tx_size += cmd->padding; 2556 2557 pr_debug("Attaching %u padding bytes\n", 2558 cmd->padding); 2559 } 2560 if (conn->conn_ops->DataDigest) { 2561 cmd->data_crc = iscsit_do_crypto_hash_sg(&conn->conn_tx_hash, cmd, 2562 datain.offset, datain.length, cmd->padding, cmd->pad_bytes); 2563 2564 iov[iov_count].iov_base = &cmd->data_crc; 2565 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 2566 tx_size += ISCSI_CRC_LEN; 2567 2568 pr_debug("Attached CRC32C DataDigest %d bytes, crc" 2569 " 0x%08x\n", datain.length+cmd->padding, cmd->data_crc); 2570 } 2571 2572 cmd->iov_data_count = iov_count; 2573 cmd->tx_size = tx_size; 2574 2575 pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x," 2576 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", 2577 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn), 2578 ntohl(hdr->offset), datain.length, conn->cid); 2579 2580 if (dr->dr_complete) { 2581 *eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ? 2582 2 : 1; 2583 iscsit_free_datain_req(cmd, dr); 2584 } 2585 2586 return 0; 2587 } 2588 2589 static int iscsit_send_logout_response( 2590 struct iscsi_cmd *cmd, 2591 struct iscsi_conn *conn) 2592 { 2593 int niov = 0, tx_size; 2594 struct iscsi_conn *logout_conn = NULL; 2595 struct iscsi_conn_recovery *cr = NULL; 2596 struct iscsi_session *sess = conn->sess; 2597 struct kvec *iov; 2598 struct iscsi_logout_rsp *hdr; 2599 /* 2600 * The actual shutting down of Sessions and/or Connections 2601 * for CLOSESESSION and CLOSECONNECTION Logout Requests 2602 * is done in scsi_logout_post_handler(). 2603 */ 2604 switch (cmd->logout_reason) { 2605 case ISCSI_LOGOUT_REASON_CLOSE_SESSION: 2606 pr_debug("iSCSI session logout successful, setting" 2607 " logout response to ISCSI_LOGOUT_SUCCESS.\n"); 2608 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2609 break; 2610 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: 2611 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND) 2612 break; 2613 /* 2614 * For CLOSECONNECTION logout requests carrying 2615 * a matching logout CID -> local CID, the reference 2616 * for the local CID will have been incremented in 2617 * iscsi_logout_closeconnection(). 2618 * 2619 * For CLOSECONNECTION logout requests carrying 2620 * a different CID than the connection it arrived 2621 * on, the connection responding to cmd->logout_cid 2622 * is stopped in iscsit_logout_post_handler_diffcid(). 2623 */ 2624 2625 pr_debug("iSCSI CID: %hu logout on CID: %hu" 2626 " successful.\n", cmd->logout_cid, conn->cid); 2627 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2628 break; 2629 case ISCSI_LOGOUT_REASON_RECOVERY: 2630 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) || 2631 (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED)) 2632 break; 2633 /* 2634 * If the connection is still active from our point of view 2635 * force connection recovery to occur. 2636 */ 2637 logout_conn = iscsit_get_conn_from_cid_rcfr(sess, 2638 cmd->logout_cid); 2639 if ((logout_conn)) { 2640 iscsit_connection_reinstatement_rcfr(logout_conn); 2641 iscsit_dec_conn_usage_count(logout_conn); 2642 } 2643 2644 cr = iscsit_get_inactive_connection_recovery_entry( 2645 conn->sess, cmd->logout_cid); 2646 if (!cr) { 2647 pr_err("Unable to locate CID: %hu for" 2648 " REMOVECONNFORRECOVERY Logout Request.\n", 2649 cmd->logout_cid); 2650 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND; 2651 break; 2652 } 2653 2654 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn); 2655 2656 pr_debug("iSCSI REMOVECONNFORRECOVERY logout" 2657 " for recovery for CID: %hu on CID: %hu successful.\n", 2658 cmd->logout_cid, conn->cid); 2659 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2660 break; 2661 default: 2662 pr_err("Unknown cmd->logout_reason: 0x%02x\n", 2663 cmd->logout_reason); 2664 return -1; 2665 } 2666 2667 tx_size = ISCSI_HDR_LEN; 2668 hdr = (struct iscsi_logout_rsp *)cmd->pdu; 2669 memset(hdr, 0, ISCSI_HDR_LEN); 2670 hdr->opcode = ISCSI_OP_LOGOUT_RSP; 2671 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2672 hdr->response = cmd->logout_response; 2673 hdr->itt = cpu_to_be32(cmd->init_task_tag); 2674 cmd->stat_sn = conn->stat_sn++; 2675 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2676 2677 iscsit_increment_maxcmdsn(cmd, conn->sess); 2678 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2679 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2680 2681 iov = &cmd->iov_misc[0]; 2682 iov[niov].iov_base = cmd->pdu; 2683 iov[niov++].iov_len = ISCSI_HDR_LEN; 2684 2685 if (conn->conn_ops->HeaderDigest) { 2686 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2687 2688 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2689 (unsigned char *)hdr, ISCSI_HDR_LEN, 2690 0, NULL, (u8 *)header_digest); 2691 2692 iov[0].iov_len += ISCSI_CRC_LEN; 2693 tx_size += ISCSI_CRC_LEN; 2694 pr_debug("Attaching CRC32C HeaderDigest to" 2695 " Logout Response 0x%08x\n", *header_digest); 2696 } 2697 cmd->iov_misc_count = niov; 2698 cmd->tx_size = tx_size; 2699 2700 pr_debug("Sending Logout Response ITT: 0x%08x StatSN:" 2701 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n", 2702 cmd->init_task_tag, cmd->stat_sn, hdr->response, 2703 cmd->logout_cid, conn->cid); 2704 2705 return 0; 2706 } 2707 2708 /* 2709 * Unsolicited NOPIN, either requesting a response or not. 2710 */ 2711 static int iscsit_send_unsolicited_nopin( 2712 struct iscsi_cmd *cmd, 2713 struct iscsi_conn *conn, 2714 int want_response) 2715 { 2716 int tx_size = ISCSI_HDR_LEN; 2717 struct iscsi_nopin *hdr; 2718 2719 hdr = (struct iscsi_nopin *) cmd->pdu; 2720 memset(hdr, 0, ISCSI_HDR_LEN); 2721 hdr->opcode = ISCSI_OP_NOOP_IN; 2722 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2723 hdr->itt = cpu_to_be32(cmd->init_task_tag); 2724 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 2725 cmd->stat_sn = conn->stat_sn; 2726 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2727 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2728 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2729 2730 if (conn->conn_ops->HeaderDigest) { 2731 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2732 2733 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2734 (unsigned char *)hdr, ISCSI_HDR_LEN, 2735 0, NULL, (u8 *)header_digest); 2736 2737 tx_size += ISCSI_CRC_LEN; 2738 pr_debug("Attaching CRC32C HeaderDigest to" 2739 " NopIN 0x%08x\n", *header_digest); 2740 } 2741 2742 cmd->iov_misc[0].iov_base = cmd->pdu; 2743 cmd->iov_misc[0].iov_len = tx_size; 2744 cmd->iov_misc_count = 1; 2745 cmd->tx_size = tx_size; 2746 2747 pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:" 2748 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid); 2749 2750 return 0; 2751 } 2752 2753 static int iscsit_send_nopin_response( 2754 struct iscsi_cmd *cmd, 2755 struct iscsi_conn *conn) 2756 { 2757 int niov = 0, tx_size; 2758 u32 padding = 0; 2759 struct kvec *iov; 2760 struct iscsi_nopin *hdr; 2761 2762 tx_size = ISCSI_HDR_LEN; 2763 hdr = (struct iscsi_nopin *) cmd->pdu; 2764 memset(hdr, 0, ISCSI_HDR_LEN); 2765 hdr->opcode = ISCSI_OP_NOOP_IN; 2766 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2767 hton24(hdr->dlength, cmd->buf_ptr_size); 2768 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); 2769 hdr->itt = cpu_to_be32(cmd->init_task_tag); 2770 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 2771 cmd->stat_sn = conn->stat_sn++; 2772 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2773 2774 iscsit_increment_maxcmdsn(cmd, conn->sess); 2775 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2776 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2777 2778 iov = &cmd->iov_misc[0]; 2779 iov[niov].iov_base = cmd->pdu; 2780 iov[niov++].iov_len = ISCSI_HDR_LEN; 2781 2782 if (conn->conn_ops->HeaderDigest) { 2783 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2784 2785 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2786 (unsigned char *)hdr, ISCSI_HDR_LEN, 2787 0, NULL, (u8 *)header_digest); 2788 2789 iov[0].iov_len += ISCSI_CRC_LEN; 2790 tx_size += ISCSI_CRC_LEN; 2791 pr_debug("Attaching CRC32C HeaderDigest" 2792 " to NopIn 0x%08x\n", *header_digest); 2793 } 2794 2795 /* 2796 * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr. 2797 * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size. 2798 */ 2799 if (cmd->buf_ptr_size) { 2800 iov[niov].iov_base = cmd->buf_ptr; 2801 iov[niov++].iov_len = cmd->buf_ptr_size; 2802 tx_size += cmd->buf_ptr_size; 2803 2804 pr_debug("Echoing back %u bytes of ping" 2805 " data.\n", cmd->buf_ptr_size); 2806 2807 padding = ((-cmd->buf_ptr_size) & 3); 2808 if (padding != 0) { 2809 iov[niov].iov_base = &cmd->pad_bytes; 2810 iov[niov++].iov_len = padding; 2811 tx_size += padding; 2812 pr_debug("Attaching %u additional" 2813 " padding bytes.\n", padding); 2814 } 2815 if (conn->conn_ops->DataDigest) { 2816 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2817 cmd->buf_ptr, cmd->buf_ptr_size, 2818 padding, (u8 *)&cmd->pad_bytes, 2819 (u8 *)&cmd->data_crc); 2820 2821 iov[niov].iov_base = &cmd->data_crc; 2822 iov[niov++].iov_len = ISCSI_CRC_LEN; 2823 tx_size += ISCSI_CRC_LEN; 2824 pr_debug("Attached DataDigest for %u" 2825 " bytes of ping data, CRC 0x%08x\n", 2826 cmd->buf_ptr_size, cmd->data_crc); 2827 } 2828 } 2829 2830 cmd->iov_misc_count = niov; 2831 cmd->tx_size = tx_size; 2832 2833 pr_debug("Sending NOPIN Response ITT: 0x%08x, TTT:" 2834 " 0x%08x, StatSN: 0x%08x, Length %u\n", cmd->init_task_tag, 2835 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size); 2836 2837 return 0; 2838 } 2839 2840 int iscsit_send_r2t( 2841 struct iscsi_cmd *cmd, 2842 struct iscsi_conn *conn) 2843 { 2844 int tx_size = 0; 2845 struct iscsi_r2t *r2t; 2846 struct iscsi_r2t_rsp *hdr; 2847 2848 r2t = iscsit_get_r2t_from_list(cmd); 2849 if (!r2t) 2850 return -1; 2851 2852 hdr = (struct iscsi_r2t_rsp *) cmd->pdu; 2853 memset(hdr, 0, ISCSI_HDR_LEN); 2854 hdr->opcode = ISCSI_OP_R2T; 2855 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2856 int_to_scsilun(cmd->se_cmd.orig_fe_lun, 2857 (struct scsi_lun *)&hdr->lun); 2858 hdr->itt = cpu_to_be32(cmd->init_task_tag); 2859 spin_lock_bh(&conn->sess->ttt_lock); 2860 r2t->targ_xfer_tag = conn->sess->targ_xfer_tag++; 2861 if (r2t->targ_xfer_tag == 0xFFFFFFFF) 2862 r2t->targ_xfer_tag = conn->sess->targ_xfer_tag++; 2863 spin_unlock_bh(&conn->sess->ttt_lock); 2864 hdr->ttt = cpu_to_be32(r2t->targ_xfer_tag); 2865 hdr->statsn = cpu_to_be32(conn->stat_sn); 2866 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2867 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2868 hdr->r2tsn = cpu_to_be32(r2t->r2t_sn); 2869 hdr->data_offset = cpu_to_be32(r2t->offset); 2870 hdr->data_length = cpu_to_be32(r2t->xfer_len); 2871 2872 cmd->iov_misc[0].iov_base = cmd->pdu; 2873 cmd->iov_misc[0].iov_len = ISCSI_HDR_LEN; 2874 tx_size += ISCSI_HDR_LEN; 2875 2876 if (conn->conn_ops->HeaderDigest) { 2877 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2878 2879 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2880 (unsigned char *)hdr, ISCSI_HDR_LEN, 2881 0, NULL, (u8 *)header_digest); 2882 2883 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN; 2884 tx_size += ISCSI_CRC_LEN; 2885 pr_debug("Attaching CRC32 HeaderDigest for R2T" 2886 " PDU 0x%08x\n", *header_digest); 2887 } 2888 2889 pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:" 2890 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n", 2891 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag, 2892 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn, 2893 r2t->offset, r2t->xfer_len, conn->cid); 2894 2895 cmd->iov_misc_count = 1; 2896 cmd->tx_size = tx_size; 2897 2898 spin_lock_bh(&cmd->r2t_lock); 2899 r2t->sent_r2t = 1; 2900 spin_unlock_bh(&cmd->r2t_lock); 2901 2902 return 0; 2903 } 2904 2905 /* 2906 * type 0: Normal Operation. 2907 * type 1: Called from Storage Transport. 2908 * type 2: Called from iscsi_task_reassign_complete_write() for 2909 * connection recovery. 2910 */ 2911 int iscsit_build_r2ts_for_cmd( 2912 struct iscsi_cmd *cmd, 2913 struct iscsi_conn *conn, 2914 int type) 2915 { 2916 int first_r2t = 1; 2917 u32 offset = 0, xfer_len = 0; 2918 2919 spin_lock_bh(&cmd->r2t_lock); 2920 if (cmd->cmd_flags & ICF_SENT_LAST_R2T) { 2921 spin_unlock_bh(&cmd->r2t_lock); 2922 return 0; 2923 } 2924 2925 if (conn->sess->sess_ops->DataSequenceInOrder && (type != 2)) 2926 if (cmd->r2t_offset < cmd->write_data_done) 2927 cmd->r2t_offset = cmd->write_data_done; 2928 2929 while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) { 2930 if (conn->sess->sess_ops->DataSequenceInOrder) { 2931 offset = cmd->r2t_offset; 2932 2933 if (first_r2t && (type == 2)) { 2934 xfer_len = ((offset + 2935 (conn->sess->sess_ops->MaxBurstLength - 2936 cmd->next_burst_len) > 2937 cmd->data_length) ? 2938 (cmd->data_length - offset) : 2939 (conn->sess->sess_ops->MaxBurstLength - 2940 cmd->next_burst_len)); 2941 } else { 2942 xfer_len = ((offset + 2943 conn->sess->sess_ops->MaxBurstLength) > 2944 cmd->data_length) ? 2945 (cmd->data_length - offset) : 2946 conn->sess->sess_ops->MaxBurstLength; 2947 } 2948 cmd->r2t_offset += xfer_len; 2949 2950 if (cmd->r2t_offset == cmd->data_length) 2951 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 2952 } else { 2953 struct iscsi_seq *seq; 2954 2955 seq = iscsit_get_seq_holder_for_r2t(cmd); 2956 if (!seq) { 2957 spin_unlock_bh(&cmd->r2t_lock); 2958 return -1; 2959 } 2960 2961 offset = seq->offset; 2962 xfer_len = seq->xfer_len; 2963 2964 if (cmd->seq_send_order == cmd->seq_count) 2965 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 2966 } 2967 cmd->outstanding_r2ts++; 2968 first_r2t = 0; 2969 2970 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) { 2971 spin_unlock_bh(&cmd->r2t_lock); 2972 return -1; 2973 } 2974 2975 if (cmd->cmd_flags & ICF_SENT_LAST_R2T) 2976 break; 2977 } 2978 spin_unlock_bh(&cmd->r2t_lock); 2979 2980 return 0; 2981 } 2982 2983 static int iscsit_send_status( 2984 struct iscsi_cmd *cmd, 2985 struct iscsi_conn *conn) 2986 { 2987 u8 iov_count = 0, recovery; 2988 u32 padding = 0, tx_size = 0; 2989 struct iscsi_scsi_rsp *hdr; 2990 struct kvec *iov; 2991 2992 recovery = (cmd->i_state != ISTATE_SEND_STATUS); 2993 if (!recovery) 2994 cmd->stat_sn = conn->stat_sn++; 2995 2996 spin_lock_bh(&conn->sess->session_stats_lock); 2997 conn->sess->rsp_pdus++; 2998 spin_unlock_bh(&conn->sess->session_stats_lock); 2999 3000 hdr = (struct iscsi_scsi_rsp *) cmd->pdu; 3001 memset(hdr, 0, ISCSI_HDR_LEN); 3002 hdr->opcode = ISCSI_OP_SCSI_CMD_RSP; 3003 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3004 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { 3005 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW; 3006 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 3007 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { 3008 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW; 3009 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 3010 } 3011 hdr->response = cmd->iscsi_response; 3012 hdr->cmd_status = cmd->se_cmd.scsi_status; 3013 hdr->itt = cpu_to_be32(cmd->init_task_tag); 3014 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3015 3016 iscsit_increment_maxcmdsn(cmd, conn->sess); 3017 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3018 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 3019 3020 iov = &cmd->iov_misc[0]; 3021 iov[iov_count].iov_base = cmd->pdu; 3022 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3023 tx_size += ISCSI_HDR_LEN; 3024 3025 /* 3026 * Attach SENSE DATA payload to iSCSI Response PDU 3027 */ 3028 if (cmd->se_cmd.sense_buffer && 3029 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 3030 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 3031 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 3032 hton24(hdr->dlength, cmd->se_cmd.scsi_sense_length); 3033 iov[iov_count].iov_base = cmd->se_cmd.sense_buffer; 3034 iov[iov_count++].iov_len = 3035 (cmd->se_cmd.scsi_sense_length + padding); 3036 tx_size += cmd->se_cmd.scsi_sense_length; 3037 3038 if (padding) { 3039 memset(cmd->se_cmd.sense_buffer + 3040 cmd->se_cmd.scsi_sense_length, 0, padding); 3041 tx_size += padding; 3042 pr_debug("Adding %u bytes of padding to" 3043 " SENSE.\n", padding); 3044 } 3045 3046 if (conn->conn_ops->DataDigest) { 3047 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3048 cmd->se_cmd.sense_buffer, 3049 (cmd->se_cmd.scsi_sense_length + padding), 3050 0, NULL, (u8 *)&cmd->data_crc); 3051 3052 iov[iov_count].iov_base = &cmd->data_crc; 3053 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 3054 tx_size += ISCSI_CRC_LEN; 3055 3056 pr_debug("Attaching CRC32 DataDigest for" 3057 " SENSE, %u bytes CRC 0x%08x\n", 3058 (cmd->se_cmd.scsi_sense_length + padding), 3059 cmd->data_crc); 3060 } 3061 3062 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI" 3063 " Response PDU\n", 3064 cmd->se_cmd.scsi_sense_length); 3065 } 3066 3067 if (conn->conn_ops->HeaderDigest) { 3068 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3069 3070 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3071 (unsigned char *)hdr, ISCSI_HDR_LEN, 3072 0, NULL, (u8 *)header_digest); 3073 3074 iov[0].iov_len += ISCSI_CRC_LEN; 3075 tx_size += ISCSI_CRC_LEN; 3076 pr_debug("Attaching CRC32 HeaderDigest for Response" 3077 " PDU 0x%08x\n", *header_digest); 3078 } 3079 3080 cmd->iov_misc_count = iov_count; 3081 cmd->tx_size = tx_size; 3082 3083 pr_debug("Built %sSCSI Response, ITT: 0x%08x, StatSN: 0x%08x," 3084 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n", 3085 (!recovery) ? "" : "Recovery ", cmd->init_task_tag, 3086 cmd->stat_sn, 0x00, cmd->se_cmd.scsi_status, conn->cid); 3087 3088 return 0; 3089 } 3090 3091 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr) 3092 { 3093 switch (se_tmr->response) { 3094 case TMR_FUNCTION_COMPLETE: 3095 return ISCSI_TMF_RSP_COMPLETE; 3096 case TMR_TASK_DOES_NOT_EXIST: 3097 return ISCSI_TMF_RSP_NO_TASK; 3098 case TMR_LUN_DOES_NOT_EXIST: 3099 return ISCSI_TMF_RSP_NO_LUN; 3100 case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED: 3101 return ISCSI_TMF_RSP_NOT_SUPPORTED; 3102 case TMR_FUNCTION_AUTHORIZATION_FAILED: 3103 return ISCSI_TMF_RSP_AUTH_FAILED; 3104 case TMR_FUNCTION_REJECTED: 3105 default: 3106 return ISCSI_TMF_RSP_REJECTED; 3107 } 3108 } 3109 3110 static int iscsit_send_task_mgt_rsp( 3111 struct iscsi_cmd *cmd, 3112 struct iscsi_conn *conn) 3113 { 3114 struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req; 3115 struct iscsi_tm_rsp *hdr; 3116 u32 tx_size = 0; 3117 3118 hdr = (struct iscsi_tm_rsp *) cmd->pdu; 3119 memset(hdr, 0, ISCSI_HDR_LEN); 3120 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP; 3121 hdr->flags = ISCSI_FLAG_CMD_FINAL; 3122 hdr->response = iscsit_convert_tcm_tmr_rsp(se_tmr); 3123 hdr->itt = cpu_to_be32(cmd->init_task_tag); 3124 cmd->stat_sn = conn->stat_sn++; 3125 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3126 3127 iscsit_increment_maxcmdsn(cmd, conn->sess); 3128 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3129 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 3130 3131 cmd->iov_misc[0].iov_base = cmd->pdu; 3132 cmd->iov_misc[0].iov_len = ISCSI_HDR_LEN; 3133 tx_size += ISCSI_HDR_LEN; 3134 3135 if (conn->conn_ops->HeaderDigest) { 3136 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3137 3138 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3139 (unsigned char *)hdr, ISCSI_HDR_LEN, 3140 0, NULL, (u8 *)header_digest); 3141 3142 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN; 3143 tx_size += ISCSI_CRC_LEN; 3144 pr_debug("Attaching CRC32 HeaderDigest for Task" 3145 " Mgmt Response PDU 0x%08x\n", *header_digest); 3146 } 3147 3148 cmd->iov_misc_count = 1; 3149 cmd->tx_size = tx_size; 3150 3151 pr_debug("Built Task Management Response ITT: 0x%08x," 3152 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n", 3153 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid); 3154 3155 return 0; 3156 } 3157 3158 static bool iscsit_check_inaddr_any(struct iscsi_np *np) 3159 { 3160 bool ret = false; 3161 3162 if (np->np_sockaddr.ss_family == AF_INET6) { 3163 const struct sockaddr_in6 sin6 = { 3164 .sin6_addr = IN6ADDR_ANY_INIT }; 3165 struct sockaddr_in6 *sock_in6 = 3166 (struct sockaddr_in6 *)&np->np_sockaddr; 3167 3168 if (!memcmp(sock_in6->sin6_addr.s6_addr, 3169 sin6.sin6_addr.s6_addr, 16)) 3170 ret = true; 3171 } else { 3172 struct sockaddr_in * sock_in = 3173 (struct sockaddr_in *)&np->np_sockaddr; 3174 3175 if (sock_in->sin_addr.s_addr == INADDR_ANY) 3176 ret = true; 3177 } 3178 3179 return ret; 3180 } 3181 3182 static int iscsit_build_sendtargets_response(struct iscsi_cmd *cmd) 3183 { 3184 char *payload = NULL; 3185 struct iscsi_conn *conn = cmd->conn; 3186 struct iscsi_portal_group *tpg; 3187 struct iscsi_tiqn *tiqn; 3188 struct iscsi_tpg_np *tpg_np; 3189 int buffer_len, end_of_buf = 0, len = 0, payload_len = 0; 3190 unsigned char buf[256]; 3191 3192 buffer_len = (conn->conn_ops->MaxRecvDataSegmentLength > 32768) ? 3193 32768 : conn->conn_ops->MaxRecvDataSegmentLength; 3194 3195 memset(buf, 0, 256); 3196 3197 payload = kzalloc(buffer_len, GFP_KERNEL); 3198 if (!payload) { 3199 pr_err("Unable to allocate memory for sendtargets" 3200 " response.\n"); 3201 return -ENOMEM; 3202 } 3203 3204 spin_lock(&tiqn_lock); 3205 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) { 3206 len = sprintf(buf, "TargetName=%s", tiqn->tiqn); 3207 len += 1; 3208 3209 if ((len + payload_len) > buffer_len) { 3210 spin_unlock(&tiqn->tiqn_tpg_lock); 3211 end_of_buf = 1; 3212 goto eob; 3213 } 3214 memcpy(payload + payload_len, buf, len); 3215 payload_len += len; 3216 3217 spin_lock(&tiqn->tiqn_tpg_lock); 3218 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) { 3219 3220 spin_lock(&tpg->tpg_state_lock); 3221 if ((tpg->tpg_state == TPG_STATE_FREE) || 3222 (tpg->tpg_state == TPG_STATE_INACTIVE)) { 3223 spin_unlock(&tpg->tpg_state_lock); 3224 continue; 3225 } 3226 spin_unlock(&tpg->tpg_state_lock); 3227 3228 spin_lock(&tpg->tpg_np_lock); 3229 list_for_each_entry(tpg_np, &tpg->tpg_gnp_list, 3230 tpg_np_list) { 3231 struct iscsi_np *np = tpg_np->tpg_np; 3232 bool inaddr_any = iscsit_check_inaddr_any(np); 3233 3234 len = sprintf(buf, "TargetAddress=" 3235 "%s%s%s:%hu,%hu", 3236 (np->np_sockaddr.ss_family == AF_INET6) ? 3237 "[" : "", (inaddr_any == false) ? 3238 np->np_ip : conn->local_ip, 3239 (np->np_sockaddr.ss_family == AF_INET6) ? 3240 "]" : "", (inaddr_any == false) ? 3241 np->np_port : conn->local_port, 3242 tpg->tpgt); 3243 len += 1; 3244 3245 if ((len + payload_len) > buffer_len) { 3246 spin_unlock(&tpg->tpg_np_lock); 3247 spin_unlock(&tiqn->tiqn_tpg_lock); 3248 end_of_buf = 1; 3249 goto eob; 3250 } 3251 memcpy(payload + payload_len, buf, len); 3252 payload_len += len; 3253 } 3254 spin_unlock(&tpg->tpg_np_lock); 3255 } 3256 spin_unlock(&tiqn->tiqn_tpg_lock); 3257 eob: 3258 if (end_of_buf) 3259 break; 3260 } 3261 spin_unlock(&tiqn_lock); 3262 3263 cmd->buf_ptr = payload; 3264 3265 return payload_len; 3266 } 3267 3268 /* 3269 * FIXME: Add support for F_BIT and C_BIT when the length is longer than 3270 * MaxRecvDataSegmentLength. 3271 */ 3272 static int iscsit_send_text_rsp( 3273 struct iscsi_cmd *cmd, 3274 struct iscsi_conn *conn) 3275 { 3276 struct iscsi_text_rsp *hdr; 3277 struct kvec *iov; 3278 u32 padding = 0, tx_size = 0; 3279 int text_length, iov_count = 0; 3280 3281 text_length = iscsit_build_sendtargets_response(cmd); 3282 if (text_length < 0) 3283 return text_length; 3284 3285 padding = ((-text_length) & 3); 3286 if (padding != 0) { 3287 memset(cmd->buf_ptr + text_length, 0, padding); 3288 pr_debug("Attaching %u additional bytes for" 3289 " padding.\n", padding); 3290 } 3291 3292 hdr = (struct iscsi_text_rsp *) cmd->pdu; 3293 memset(hdr, 0, ISCSI_HDR_LEN); 3294 hdr->opcode = ISCSI_OP_TEXT_RSP; 3295 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3296 hton24(hdr->dlength, text_length); 3297 hdr->itt = cpu_to_be32(cmd->init_task_tag); 3298 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 3299 cmd->stat_sn = conn->stat_sn++; 3300 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3301 3302 iscsit_increment_maxcmdsn(cmd, conn->sess); 3303 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3304 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 3305 3306 iov = &cmd->iov_misc[0]; 3307 3308 iov[iov_count].iov_base = cmd->pdu; 3309 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3310 iov[iov_count].iov_base = cmd->buf_ptr; 3311 iov[iov_count++].iov_len = text_length + padding; 3312 3313 tx_size += (ISCSI_HDR_LEN + text_length + padding); 3314 3315 if (conn->conn_ops->HeaderDigest) { 3316 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3317 3318 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3319 (unsigned char *)hdr, ISCSI_HDR_LEN, 3320 0, NULL, (u8 *)header_digest); 3321 3322 iov[0].iov_len += ISCSI_CRC_LEN; 3323 tx_size += ISCSI_CRC_LEN; 3324 pr_debug("Attaching CRC32 HeaderDigest for" 3325 " Text Response PDU 0x%08x\n", *header_digest); 3326 } 3327 3328 if (conn->conn_ops->DataDigest) { 3329 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3330 cmd->buf_ptr, (text_length + padding), 3331 0, NULL, (u8 *)&cmd->data_crc); 3332 3333 iov[iov_count].iov_base = &cmd->data_crc; 3334 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 3335 tx_size += ISCSI_CRC_LEN; 3336 3337 pr_debug("Attaching DataDigest for %u bytes of text" 3338 " data, CRC 0x%08x\n", (text_length + padding), 3339 cmd->data_crc); 3340 } 3341 3342 cmd->iov_misc_count = iov_count; 3343 cmd->tx_size = tx_size; 3344 3345 pr_debug("Built Text Response: ITT: 0x%08x, StatSN: 0x%08x," 3346 " Length: %u, CID: %hu\n", cmd->init_task_tag, cmd->stat_sn, 3347 text_length, conn->cid); 3348 return 0; 3349 } 3350 3351 static int iscsit_send_reject( 3352 struct iscsi_cmd *cmd, 3353 struct iscsi_conn *conn) 3354 { 3355 u32 iov_count = 0, tx_size = 0; 3356 struct iscsi_reject *hdr; 3357 struct kvec *iov; 3358 3359 hdr = (struct iscsi_reject *) cmd->pdu; 3360 hdr->opcode = ISCSI_OP_REJECT; 3361 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3362 hton24(hdr->dlength, ISCSI_HDR_LEN); 3363 cmd->stat_sn = conn->stat_sn++; 3364 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3365 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3366 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 3367 3368 iov = &cmd->iov_misc[0]; 3369 3370 iov[iov_count].iov_base = cmd->pdu; 3371 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3372 iov[iov_count].iov_base = cmd->buf_ptr; 3373 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3374 3375 tx_size = (ISCSI_HDR_LEN + ISCSI_HDR_LEN); 3376 3377 if (conn->conn_ops->HeaderDigest) { 3378 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3379 3380 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3381 (unsigned char *)hdr, ISCSI_HDR_LEN, 3382 0, NULL, (u8 *)header_digest); 3383 3384 iov[0].iov_len += ISCSI_CRC_LEN; 3385 tx_size += ISCSI_CRC_LEN; 3386 pr_debug("Attaching CRC32 HeaderDigest for" 3387 " REJECT PDU 0x%08x\n", *header_digest); 3388 } 3389 3390 if (conn->conn_ops->DataDigest) { 3391 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3392 (unsigned char *)cmd->buf_ptr, ISCSI_HDR_LEN, 3393 0, NULL, (u8 *)&cmd->data_crc); 3394 3395 iov[iov_count].iov_base = &cmd->data_crc; 3396 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 3397 tx_size += ISCSI_CRC_LEN; 3398 pr_debug("Attaching CRC32 DataDigest for REJECT" 3399 " PDU 0x%08x\n", cmd->data_crc); 3400 } 3401 3402 cmd->iov_misc_count = iov_count; 3403 cmd->tx_size = tx_size; 3404 3405 pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x," 3406 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid); 3407 3408 return 0; 3409 } 3410 3411 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn) 3412 { 3413 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || 3414 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { 3415 wait_for_completion_interruptible_timeout( 3416 &conn->tx_half_close_comp, 3417 ISCSI_TX_THREAD_TCP_TIMEOUT * HZ); 3418 } 3419 } 3420 3421 #ifdef CONFIG_SMP 3422 3423 void iscsit_thread_get_cpumask(struct iscsi_conn *conn) 3424 { 3425 struct iscsi_thread_set *ts = conn->thread_set; 3426 int ord, cpu; 3427 /* 3428 * thread_id is assigned from iscsit_global->ts_bitmap from 3429 * within iscsi_thread_set.c:iscsi_allocate_thread_sets() 3430 * 3431 * Here we use thread_id to determine which CPU that this 3432 * iSCSI connection's iscsi_thread_set will be scheduled to 3433 * execute upon. 3434 */ 3435 ord = ts->thread_id % cpumask_weight(cpu_online_mask); 3436 #if 0 3437 pr_debug(">>>>>>>>>>>>>>>>>>>> Generated ord: %d from" 3438 " thread_id: %d\n", ord, ts->thread_id); 3439 #endif 3440 for_each_online_cpu(cpu) { 3441 if (ord-- == 0) { 3442 cpumask_set_cpu(cpu, conn->conn_cpumask); 3443 return; 3444 } 3445 } 3446 /* 3447 * This should never be reached.. 3448 */ 3449 dump_stack(); 3450 cpumask_setall(conn->conn_cpumask); 3451 } 3452 3453 static inline void iscsit_thread_check_cpumask( 3454 struct iscsi_conn *conn, 3455 struct task_struct *p, 3456 int mode) 3457 { 3458 char buf[128]; 3459 /* 3460 * mode == 1 signals iscsi_target_tx_thread() usage. 3461 * mode == 0 signals iscsi_target_rx_thread() usage. 3462 */ 3463 if (mode == 1) { 3464 if (!conn->conn_tx_reset_cpumask) 3465 return; 3466 conn->conn_tx_reset_cpumask = 0; 3467 } else { 3468 if (!conn->conn_rx_reset_cpumask) 3469 return; 3470 conn->conn_rx_reset_cpumask = 0; 3471 } 3472 /* 3473 * Update the CPU mask for this single kthread so that 3474 * both TX and RX kthreads are scheduled to run on the 3475 * same CPU. 3476 */ 3477 memset(buf, 0, 128); 3478 cpumask_scnprintf(buf, 128, conn->conn_cpumask); 3479 #if 0 3480 pr_debug(">>>>>>>>>>>>>> Calling set_cpus_allowed_ptr():" 3481 " %s for %s\n", buf, p->comm); 3482 #endif 3483 set_cpus_allowed_ptr(p, conn->conn_cpumask); 3484 } 3485 3486 #else 3487 3488 void iscsit_thread_get_cpumask(struct iscsi_conn *conn) 3489 { 3490 return; 3491 } 3492 3493 #define iscsit_thread_check_cpumask(X, Y, Z) ({}) 3494 #endif /* CONFIG_SMP */ 3495 3496 int iscsi_target_tx_thread(void *arg) 3497 { 3498 u8 state; 3499 int eodr = 0; 3500 int ret = 0; 3501 int sent_status = 0; 3502 int use_misc = 0; 3503 int map_sg = 0; 3504 struct iscsi_cmd *cmd = NULL; 3505 struct iscsi_conn *conn; 3506 struct iscsi_queue_req *qr = NULL; 3507 struct iscsi_thread_set *ts = arg; 3508 /* 3509 * Allow ourselves to be interrupted by SIGINT so that a 3510 * connection recovery / failure event can be triggered externally. 3511 */ 3512 allow_signal(SIGINT); 3513 3514 restart: 3515 conn = iscsi_tx_thread_pre_handler(ts); 3516 if (!conn) 3517 goto out; 3518 3519 eodr = map_sg = ret = sent_status = use_misc = 0; 3520 3521 while (!kthread_should_stop()) { 3522 /* 3523 * Ensure that both TX and RX per connection kthreads 3524 * are scheduled to run on the same CPU. 3525 */ 3526 iscsit_thread_check_cpumask(conn, current, 1); 3527 3528 schedule_timeout_interruptible(MAX_SCHEDULE_TIMEOUT); 3529 3530 if ((ts->status == ISCSI_THREAD_SET_RESET) || 3531 signal_pending(current)) 3532 goto transport_err; 3533 3534 get_immediate: 3535 qr = iscsit_get_cmd_from_immediate_queue(conn); 3536 if (qr) { 3537 atomic_set(&conn->check_immediate_queue, 0); 3538 cmd = qr->cmd; 3539 state = qr->state; 3540 kmem_cache_free(lio_qr_cache, qr); 3541 3542 spin_lock_bh(&cmd->istate_lock); 3543 switch (state) { 3544 case ISTATE_SEND_R2T: 3545 spin_unlock_bh(&cmd->istate_lock); 3546 ret = iscsit_send_r2t(cmd, conn); 3547 break; 3548 case ISTATE_REMOVE: 3549 spin_unlock_bh(&cmd->istate_lock); 3550 3551 if (cmd->data_direction == DMA_TO_DEVICE) 3552 iscsit_stop_dataout_timer(cmd); 3553 3554 spin_lock_bh(&conn->cmd_lock); 3555 list_del(&cmd->i_list); 3556 spin_unlock_bh(&conn->cmd_lock); 3557 3558 iscsit_free_cmd(cmd); 3559 goto get_immediate; 3560 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 3561 spin_unlock_bh(&cmd->istate_lock); 3562 iscsit_mod_nopin_response_timer(conn); 3563 ret = iscsit_send_unsolicited_nopin(cmd, 3564 conn, 1); 3565 break; 3566 case ISTATE_SEND_NOPIN_NO_RESPONSE: 3567 spin_unlock_bh(&cmd->istate_lock); 3568 ret = iscsit_send_unsolicited_nopin(cmd, 3569 conn, 0); 3570 break; 3571 default: 3572 pr_err("Unknown Opcode: 0x%02x ITT:" 3573 " 0x%08x, i_state: %d on CID: %hu\n", 3574 cmd->iscsi_opcode, cmd->init_task_tag, state, 3575 conn->cid); 3576 spin_unlock_bh(&cmd->istate_lock); 3577 goto transport_err; 3578 } 3579 if (ret < 0) { 3580 conn->tx_immediate_queue = 0; 3581 goto transport_err; 3582 } 3583 3584 if (iscsit_send_tx_data(cmd, conn, 1) < 0) { 3585 conn->tx_immediate_queue = 0; 3586 iscsit_tx_thread_wait_for_tcp(conn); 3587 goto transport_err; 3588 } 3589 3590 spin_lock_bh(&cmd->istate_lock); 3591 switch (state) { 3592 case ISTATE_SEND_R2T: 3593 spin_unlock_bh(&cmd->istate_lock); 3594 spin_lock_bh(&cmd->dataout_timeout_lock); 3595 iscsit_start_dataout_timer(cmd, conn); 3596 spin_unlock_bh(&cmd->dataout_timeout_lock); 3597 break; 3598 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 3599 cmd->i_state = ISTATE_SENT_NOPIN_WANT_RESPONSE; 3600 spin_unlock_bh(&cmd->istate_lock); 3601 break; 3602 case ISTATE_SEND_NOPIN_NO_RESPONSE: 3603 cmd->i_state = ISTATE_SENT_STATUS; 3604 spin_unlock_bh(&cmd->istate_lock); 3605 break; 3606 default: 3607 pr_err("Unknown Opcode: 0x%02x ITT:" 3608 " 0x%08x, i_state: %d on CID: %hu\n", 3609 cmd->iscsi_opcode, cmd->init_task_tag, 3610 state, conn->cid); 3611 spin_unlock_bh(&cmd->istate_lock); 3612 goto transport_err; 3613 } 3614 goto get_immediate; 3615 } else 3616 conn->tx_immediate_queue = 0; 3617 3618 get_response: 3619 qr = iscsit_get_cmd_from_response_queue(conn); 3620 if (qr) { 3621 cmd = qr->cmd; 3622 state = qr->state; 3623 kmem_cache_free(lio_qr_cache, qr); 3624 3625 spin_lock_bh(&cmd->istate_lock); 3626 check_rsp_state: 3627 switch (state) { 3628 case ISTATE_SEND_DATAIN: 3629 spin_unlock_bh(&cmd->istate_lock); 3630 ret = iscsit_send_data_in(cmd, conn, 3631 &eodr); 3632 map_sg = 1; 3633 break; 3634 case ISTATE_SEND_STATUS: 3635 case ISTATE_SEND_STATUS_RECOVERY: 3636 spin_unlock_bh(&cmd->istate_lock); 3637 use_misc = 1; 3638 ret = iscsit_send_status(cmd, conn); 3639 break; 3640 case ISTATE_SEND_LOGOUTRSP: 3641 spin_unlock_bh(&cmd->istate_lock); 3642 use_misc = 1; 3643 ret = iscsit_send_logout_response(cmd, conn); 3644 break; 3645 case ISTATE_SEND_ASYNCMSG: 3646 spin_unlock_bh(&cmd->istate_lock); 3647 use_misc = 1; 3648 ret = iscsit_send_conn_drop_async_message( 3649 cmd, conn); 3650 break; 3651 case ISTATE_SEND_NOPIN: 3652 spin_unlock_bh(&cmd->istate_lock); 3653 use_misc = 1; 3654 ret = iscsit_send_nopin_response(cmd, conn); 3655 break; 3656 case ISTATE_SEND_REJECT: 3657 spin_unlock_bh(&cmd->istate_lock); 3658 use_misc = 1; 3659 ret = iscsit_send_reject(cmd, conn); 3660 break; 3661 case ISTATE_SEND_TASKMGTRSP: 3662 spin_unlock_bh(&cmd->istate_lock); 3663 use_misc = 1; 3664 ret = iscsit_send_task_mgt_rsp(cmd, conn); 3665 if (ret != 0) 3666 break; 3667 ret = iscsit_tmr_post_handler(cmd, conn); 3668 if (ret != 0) 3669 iscsit_fall_back_to_erl0(conn->sess); 3670 break; 3671 case ISTATE_SEND_TEXTRSP: 3672 spin_unlock_bh(&cmd->istate_lock); 3673 use_misc = 1; 3674 ret = iscsit_send_text_rsp(cmd, conn); 3675 break; 3676 default: 3677 pr_err("Unknown Opcode: 0x%02x ITT:" 3678 " 0x%08x, i_state: %d on CID: %hu\n", 3679 cmd->iscsi_opcode, cmd->init_task_tag, 3680 state, conn->cid); 3681 spin_unlock_bh(&cmd->istate_lock); 3682 goto transport_err; 3683 } 3684 if (ret < 0) { 3685 conn->tx_response_queue = 0; 3686 goto transport_err; 3687 } 3688 3689 if (map_sg && !conn->conn_ops->IFMarker) { 3690 if (iscsit_fe_sendpage_sg(cmd, conn) < 0) { 3691 conn->tx_response_queue = 0; 3692 iscsit_tx_thread_wait_for_tcp(conn); 3693 iscsit_unmap_iovec(cmd); 3694 goto transport_err; 3695 } 3696 } else { 3697 if (iscsit_send_tx_data(cmd, conn, use_misc) < 0) { 3698 conn->tx_response_queue = 0; 3699 iscsit_tx_thread_wait_for_tcp(conn); 3700 iscsit_unmap_iovec(cmd); 3701 goto transport_err; 3702 } 3703 } 3704 map_sg = 0; 3705 iscsit_unmap_iovec(cmd); 3706 3707 spin_lock_bh(&cmd->istate_lock); 3708 switch (state) { 3709 case ISTATE_SEND_DATAIN: 3710 if (!eodr) 3711 goto check_rsp_state; 3712 3713 if (eodr == 1) { 3714 cmd->i_state = ISTATE_SENT_LAST_DATAIN; 3715 sent_status = 1; 3716 eodr = use_misc = 0; 3717 } else if (eodr == 2) { 3718 cmd->i_state = state = 3719 ISTATE_SEND_STATUS; 3720 sent_status = 0; 3721 eodr = use_misc = 0; 3722 goto check_rsp_state; 3723 } 3724 break; 3725 case ISTATE_SEND_STATUS: 3726 use_misc = 0; 3727 sent_status = 1; 3728 break; 3729 case ISTATE_SEND_ASYNCMSG: 3730 case ISTATE_SEND_NOPIN: 3731 case ISTATE_SEND_STATUS_RECOVERY: 3732 case ISTATE_SEND_TEXTRSP: 3733 use_misc = 0; 3734 sent_status = 1; 3735 break; 3736 case ISTATE_SEND_REJECT: 3737 use_misc = 0; 3738 if (cmd->cmd_flags & ICF_REJECT_FAIL_CONN) { 3739 cmd->cmd_flags &= ~ICF_REJECT_FAIL_CONN; 3740 spin_unlock_bh(&cmd->istate_lock); 3741 complete(&cmd->reject_comp); 3742 goto transport_err; 3743 } 3744 complete(&cmd->reject_comp); 3745 break; 3746 case ISTATE_SEND_TASKMGTRSP: 3747 use_misc = 0; 3748 sent_status = 1; 3749 break; 3750 case ISTATE_SEND_LOGOUTRSP: 3751 spin_unlock_bh(&cmd->istate_lock); 3752 if (!iscsit_logout_post_handler(cmd, conn)) 3753 goto restart; 3754 spin_lock_bh(&cmd->istate_lock); 3755 use_misc = 0; 3756 sent_status = 1; 3757 break; 3758 default: 3759 pr_err("Unknown Opcode: 0x%02x ITT:" 3760 " 0x%08x, i_state: %d on CID: %hu\n", 3761 cmd->iscsi_opcode, cmd->init_task_tag, 3762 cmd->i_state, conn->cid); 3763 spin_unlock_bh(&cmd->istate_lock); 3764 goto transport_err; 3765 } 3766 3767 if (sent_status) { 3768 cmd->i_state = ISTATE_SENT_STATUS; 3769 sent_status = 0; 3770 } 3771 spin_unlock_bh(&cmd->istate_lock); 3772 3773 if (atomic_read(&conn->check_immediate_queue)) 3774 goto get_immediate; 3775 3776 goto get_response; 3777 } else 3778 conn->tx_response_queue = 0; 3779 } 3780 3781 transport_err: 3782 iscsit_take_action_for_connection_exit(conn); 3783 goto restart; 3784 out: 3785 return 0; 3786 } 3787 3788 int iscsi_target_rx_thread(void *arg) 3789 { 3790 int ret; 3791 u8 buffer[ISCSI_HDR_LEN], opcode; 3792 u32 checksum = 0, digest = 0; 3793 struct iscsi_conn *conn = NULL; 3794 struct iscsi_thread_set *ts = arg; 3795 struct kvec iov; 3796 /* 3797 * Allow ourselves to be interrupted by SIGINT so that a 3798 * connection recovery / failure event can be triggered externally. 3799 */ 3800 allow_signal(SIGINT); 3801 3802 restart: 3803 conn = iscsi_rx_thread_pre_handler(ts); 3804 if (!conn) 3805 goto out; 3806 3807 while (!kthread_should_stop()) { 3808 /* 3809 * Ensure that both TX and RX per connection kthreads 3810 * are scheduled to run on the same CPU. 3811 */ 3812 iscsit_thread_check_cpumask(conn, current, 0); 3813 3814 memset(buffer, 0, ISCSI_HDR_LEN); 3815 memset(&iov, 0, sizeof(struct kvec)); 3816 3817 iov.iov_base = buffer; 3818 iov.iov_len = ISCSI_HDR_LEN; 3819 3820 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN); 3821 if (ret != ISCSI_HDR_LEN) { 3822 iscsit_rx_thread_wait_for_tcp(conn); 3823 goto transport_err; 3824 } 3825 3826 /* 3827 * Set conn->bad_hdr for use with REJECT PDUs. 3828 */ 3829 memcpy(&conn->bad_hdr, &buffer, ISCSI_HDR_LEN); 3830 3831 if (conn->conn_ops->HeaderDigest) { 3832 iov.iov_base = &digest; 3833 iov.iov_len = ISCSI_CRC_LEN; 3834 3835 ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN); 3836 if (ret != ISCSI_CRC_LEN) { 3837 iscsit_rx_thread_wait_for_tcp(conn); 3838 goto transport_err; 3839 } 3840 3841 iscsit_do_crypto_hash_buf(&conn->conn_rx_hash, 3842 buffer, ISCSI_HDR_LEN, 3843 0, NULL, (u8 *)&checksum); 3844 3845 if (digest != checksum) { 3846 pr_err("HeaderDigest CRC32C failed," 3847 " received 0x%08x, computed 0x%08x\n", 3848 digest, checksum); 3849 /* 3850 * Set the PDU to 0xff so it will intentionally 3851 * hit default in the switch below. 3852 */ 3853 memset(buffer, 0xff, ISCSI_HDR_LEN); 3854 spin_lock_bh(&conn->sess->session_stats_lock); 3855 conn->sess->conn_digest_errors++; 3856 spin_unlock_bh(&conn->sess->session_stats_lock); 3857 } else { 3858 pr_debug("Got HeaderDigest CRC32C" 3859 " 0x%08x\n", checksum); 3860 } 3861 } 3862 3863 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 3864 goto transport_err; 3865 3866 opcode = buffer[0] & ISCSI_OPCODE_MASK; 3867 3868 if (conn->sess->sess_ops->SessionType && 3869 ((!(opcode & ISCSI_OP_TEXT)) || 3870 (!(opcode & ISCSI_OP_LOGOUT)))) { 3871 pr_err("Received illegal iSCSI Opcode: 0x%02x" 3872 " while in Discovery Session, rejecting.\n", opcode); 3873 iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 3874 buffer, conn); 3875 goto transport_err; 3876 } 3877 3878 switch (opcode) { 3879 case ISCSI_OP_SCSI_CMD: 3880 if (iscsit_handle_scsi_cmd(conn, buffer) < 0) 3881 goto transport_err; 3882 break; 3883 case ISCSI_OP_SCSI_DATA_OUT: 3884 if (iscsit_handle_data_out(conn, buffer) < 0) 3885 goto transport_err; 3886 break; 3887 case ISCSI_OP_NOOP_OUT: 3888 if (iscsit_handle_nop_out(conn, buffer) < 0) 3889 goto transport_err; 3890 break; 3891 case ISCSI_OP_SCSI_TMFUNC: 3892 if (iscsit_handle_task_mgt_cmd(conn, buffer) < 0) 3893 goto transport_err; 3894 break; 3895 case ISCSI_OP_TEXT: 3896 if (iscsit_handle_text_cmd(conn, buffer) < 0) 3897 goto transport_err; 3898 break; 3899 case ISCSI_OP_LOGOUT: 3900 ret = iscsit_handle_logout_cmd(conn, buffer); 3901 if (ret > 0) { 3902 wait_for_completion_timeout(&conn->conn_logout_comp, 3903 SECONDS_FOR_LOGOUT_COMP * HZ); 3904 goto transport_err; 3905 } else if (ret < 0) 3906 goto transport_err; 3907 break; 3908 case ISCSI_OP_SNACK: 3909 if (iscsit_handle_snack(conn, buffer) < 0) 3910 goto transport_err; 3911 break; 3912 default: 3913 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", 3914 opcode); 3915 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 3916 pr_err("Cannot recover from unknown" 3917 " opcode while ERL=0, closing iSCSI connection" 3918 ".\n"); 3919 goto transport_err; 3920 } 3921 if (!conn->conn_ops->OFMarker) { 3922 pr_err("Unable to recover from unknown" 3923 " opcode while OFMarker=No, closing iSCSI" 3924 " connection.\n"); 3925 goto transport_err; 3926 } 3927 if (iscsit_recover_from_unknown_opcode(conn) < 0) { 3928 pr_err("Unable to recover from unknown" 3929 " opcode, closing iSCSI connection.\n"); 3930 goto transport_err; 3931 } 3932 break; 3933 } 3934 } 3935 3936 transport_err: 3937 if (!signal_pending(current)) 3938 atomic_set(&conn->transport_failed, 1); 3939 iscsit_take_action_for_connection_exit(conn); 3940 goto restart; 3941 out: 3942 return 0; 3943 } 3944 3945 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn) 3946 { 3947 struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL; 3948 struct iscsi_session *sess = conn->sess; 3949 /* 3950 * We expect this function to only ever be called from either RX or TX 3951 * thread context via iscsit_close_connection() once the other context 3952 * has been reset -> returned sleeping pre-handler state. 3953 */ 3954 spin_lock_bh(&conn->cmd_lock); 3955 list_for_each_entry_safe(cmd, cmd_tmp, &conn->conn_cmd_list, i_list) { 3956 3957 list_del(&cmd->i_list); 3958 spin_unlock_bh(&conn->cmd_lock); 3959 3960 iscsit_increment_maxcmdsn(cmd, sess); 3961 3962 iscsit_free_cmd(cmd); 3963 3964 spin_lock_bh(&conn->cmd_lock); 3965 } 3966 spin_unlock_bh(&conn->cmd_lock); 3967 } 3968 3969 static void iscsit_stop_timers_for_cmds( 3970 struct iscsi_conn *conn) 3971 { 3972 struct iscsi_cmd *cmd; 3973 3974 spin_lock_bh(&conn->cmd_lock); 3975 list_for_each_entry(cmd, &conn->conn_cmd_list, i_list) { 3976 if (cmd->data_direction == DMA_TO_DEVICE) 3977 iscsit_stop_dataout_timer(cmd); 3978 } 3979 spin_unlock_bh(&conn->cmd_lock); 3980 } 3981 3982 int iscsit_close_connection( 3983 struct iscsi_conn *conn) 3984 { 3985 int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT); 3986 struct iscsi_session *sess = conn->sess; 3987 3988 pr_debug("Closing iSCSI connection CID %hu on SID:" 3989 " %u\n", conn->cid, sess->sid); 3990 /* 3991 * Always up conn_logout_comp just in case the RX Thread is sleeping 3992 * and the logout response never got sent because the connection 3993 * failed. 3994 */ 3995 complete(&conn->conn_logout_comp); 3996 3997 iscsi_release_thread_set(conn); 3998 3999 iscsit_stop_timers_for_cmds(conn); 4000 iscsit_stop_nopin_response_timer(conn); 4001 iscsit_stop_nopin_timer(conn); 4002 iscsit_free_queue_reqs_for_conn(conn); 4003 4004 /* 4005 * During Connection recovery drop unacknowledged out of order 4006 * commands for this connection, and prepare the other commands 4007 * for realligence. 4008 * 4009 * During normal operation clear the out of order commands (but 4010 * do not free the struct iscsi_ooo_cmdsn's) and release all 4011 * struct iscsi_cmds. 4012 */ 4013 if (atomic_read(&conn->connection_recovery)) { 4014 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn); 4015 iscsit_prepare_cmds_for_realligance(conn); 4016 } else { 4017 iscsit_clear_ooo_cmdsns_for_conn(conn); 4018 iscsit_release_commands_from_conn(conn); 4019 } 4020 4021 /* 4022 * Handle decrementing session or connection usage count if 4023 * a logout response was not able to be sent because the 4024 * connection failed. Fall back to Session Recovery here. 4025 */ 4026 if (atomic_read(&conn->conn_logout_remove)) { 4027 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) { 4028 iscsit_dec_conn_usage_count(conn); 4029 iscsit_dec_session_usage_count(sess); 4030 } 4031 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) 4032 iscsit_dec_conn_usage_count(conn); 4033 4034 atomic_set(&conn->conn_logout_remove, 0); 4035 atomic_set(&sess->session_reinstatement, 0); 4036 atomic_set(&sess->session_fall_back_to_erl0, 1); 4037 } 4038 4039 spin_lock_bh(&sess->conn_lock); 4040 list_del(&conn->conn_list); 4041 4042 /* 4043 * Attempt to let the Initiator know this connection failed by 4044 * sending an Connection Dropped Async Message on another 4045 * active connection. 4046 */ 4047 if (atomic_read(&conn->connection_recovery)) 4048 iscsit_build_conn_drop_async_message(conn); 4049 4050 spin_unlock_bh(&sess->conn_lock); 4051 4052 /* 4053 * If connection reinstatement is being performed on this connection, 4054 * up the connection reinstatement semaphore that is being blocked on 4055 * in iscsit_cause_connection_reinstatement(). 4056 */ 4057 spin_lock_bh(&conn->state_lock); 4058 if (atomic_read(&conn->sleep_on_conn_wait_comp)) { 4059 spin_unlock_bh(&conn->state_lock); 4060 complete(&conn->conn_wait_comp); 4061 wait_for_completion(&conn->conn_post_wait_comp); 4062 spin_lock_bh(&conn->state_lock); 4063 } 4064 4065 /* 4066 * If connection reinstatement is being performed on this connection 4067 * by receiving a REMOVECONNFORRECOVERY logout request, up the 4068 * connection wait rcfr semaphore that is being blocked on 4069 * an iscsit_connection_reinstatement_rcfr(). 4070 */ 4071 if (atomic_read(&conn->connection_wait_rcfr)) { 4072 spin_unlock_bh(&conn->state_lock); 4073 complete(&conn->conn_wait_rcfr_comp); 4074 wait_for_completion(&conn->conn_post_wait_comp); 4075 spin_lock_bh(&conn->state_lock); 4076 } 4077 atomic_set(&conn->connection_reinstatement, 1); 4078 spin_unlock_bh(&conn->state_lock); 4079 4080 /* 4081 * If any other processes are accessing this connection pointer we 4082 * must wait until they have completed. 4083 */ 4084 iscsit_check_conn_usage_count(conn); 4085 4086 if (conn->conn_rx_hash.tfm) 4087 crypto_free_hash(conn->conn_rx_hash.tfm); 4088 if (conn->conn_tx_hash.tfm) 4089 crypto_free_hash(conn->conn_tx_hash.tfm); 4090 4091 if (conn->conn_cpumask) 4092 free_cpumask_var(conn->conn_cpumask); 4093 4094 kfree(conn->conn_ops); 4095 conn->conn_ops = NULL; 4096 4097 if (conn->sock) { 4098 if (conn->conn_flags & CONNFLAG_SCTP_STRUCT_FILE) { 4099 kfree(conn->sock->file); 4100 conn->sock->file = NULL; 4101 } 4102 sock_release(conn->sock); 4103 } 4104 conn->thread_set = NULL; 4105 4106 pr_debug("Moving to TARG_CONN_STATE_FREE.\n"); 4107 conn->conn_state = TARG_CONN_STATE_FREE; 4108 kfree(conn); 4109 4110 spin_lock_bh(&sess->conn_lock); 4111 atomic_dec(&sess->nconn); 4112 pr_debug("Decremented iSCSI connection count to %hu from node:" 4113 " %s\n", atomic_read(&sess->nconn), 4114 sess->sess_ops->InitiatorName); 4115 /* 4116 * Make sure that if one connection fails in an non ERL=2 iSCSI 4117 * Session that they all fail. 4118 */ 4119 if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout && 4120 !atomic_read(&sess->session_logout)) 4121 atomic_set(&sess->session_fall_back_to_erl0, 1); 4122 4123 /* 4124 * If this was not the last connection in the session, and we are 4125 * performing session reinstatement or falling back to ERL=0, call 4126 * iscsit_stop_session() without sleeping to shutdown the other 4127 * active connections. 4128 */ 4129 if (atomic_read(&sess->nconn)) { 4130 if (!atomic_read(&sess->session_reinstatement) && 4131 !atomic_read(&sess->session_fall_back_to_erl0)) { 4132 spin_unlock_bh(&sess->conn_lock); 4133 return 0; 4134 } 4135 if (!atomic_read(&sess->session_stop_active)) { 4136 atomic_set(&sess->session_stop_active, 1); 4137 spin_unlock_bh(&sess->conn_lock); 4138 iscsit_stop_session(sess, 0, 0); 4139 return 0; 4140 } 4141 spin_unlock_bh(&sess->conn_lock); 4142 return 0; 4143 } 4144 4145 /* 4146 * If this was the last connection in the session and one of the 4147 * following is occurring: 4148 * 4149 * Session Reinstatement is not being performed, and are falling back 4150 * to ERL=0 call iscsit_close_session(). 4151 * 4152 * Session Logout was requested. iscsit_close_session() will be called 4153 * elsewhere. 4154 * 4155 * Session Continuation is not being performed, start the Time2Retain 4156 * handler and check if sleep_on_sess_wait_sem is active. 4157 */ 4158 if (!atomic_read(&sess->session_reinstatement) && 4159 atomic_read(&sess->session_fall_back_to_erl0)) { 4160 spin_unlock_bh(&sess->conn_lock); 4161 target_put_session(sess->se_sess); 4162 4163 return 0; 4164 } else if (atomic_read(&sess->session_logout)) { 4165 pr_debug("Moving to TARG_SESS_STATE_FREE.\n"); 4166 sess->session_state = TARG_SESS_STATE_FREE; 4167 spin_unlock_bh(&sess->conn_lock); 4168 4169 if (atomic_read(&sess->sleep_on_sess_wait_comp)) 4170 complete(&sess->session_wait_comp); 4171 4172 return 0; 4173 } else { 4174 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); 4175 sess->session_state = TARG_SESS_STATE_FAILED; 4176 4177 if (!atomic_read(&sess->session_continuation)) { 4178 spin_unlock_bh(&sess->conn_lock); 4179 iscsit_start_time2retain_handler(sess); 4180 } else 4181 spin_unlock_bh(&sess->conn_lock); 4182 4183 if (atomic_read(&sess->sleep_on_sess_wait_comp)) 4184 complete(&sess->session_wait_comp); 4185 4186 return 0; 4187 } 4188 spin_unlock_bh(&sess->conn_lock); 4189 4190 return 0; 4191 } 4192 4193 int iscsit_close_session(struct iscsi_session *sess) 4194 { 4195 struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess); 4196 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; 4197 4198 if (atomic_read(&sess->nconn)) { 4199 pr_err("%d connection(s) still exist for iSCSI session" 4200 " to %s\n", atomic_read(&sess->nconn), 4201 sess->sess_ops->InitiatorName); 4202 BUG(); 4203 } 4204 4205 spin_lock_bh(&se_tpg->session_lock); 4206 atomic_set(&sess->session_logout, 1); 4207 atomic_set(&sess->session_reinstatement, 1); 4208 iscsit_stop_time2retain_timer(sess); 4209 spin_unlock_bh(&se_tpg->session_lock); 4210 4211 /* 4212 * transport_deregister_session_configfs() will clear the 4213 * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context 4214 * can be setting it again with __transport_register_session() in 4215 * iscsi_post_login_handler() again after the iscsit_stop_session() 4216 * completes in iscsi_np context. 4217 */ 4218 transport_deregister_session_configfs(sess->se_sess); 4219 4220 /* 4221 * If any other processes are accessing this session pointer we must 4222 * wait until they have completed. If we are in an interrupt (the 4223 * time2retain handler) and contain and active session usage count we 4224 * restart the timer and exit. 4225 */ 4226 if (!in_interrupt()) { 4227 if (iscsit_check_session_usage_count(sess) == 1) 4228 iscsit_stop_session(sess, 1, 1); 4229 } else { 4230 if (iscsit_check_session_usage_count(sess) == 2) { 4231 atomic_set(&sess->session_logout, 0); 4232 iscsit_start_time2retain_handler(sess); 4233 return 0; 4234 } 4235 } 4236 4237 transport_deregister_session(sess->se_sess); 4238 4239 if (sess->sess_ops->ErrorRecoveryLevel == 2) 4240 iscsit_free_connection_recovery_entires(sess); 4241 4242 iscsit_free_all_ooo_cmdsns(sess); 4243 4244 spin_lock_bh(&se_tpg->session_lock); 4245 pr_debug("Moving to TARG_SESS_STATE_FREE.\n"); 4246 sess->session_state = TARG_SESS_STATE_FREE; 4247 pr_debug("Released iSCSI session from node: %s\n", 4248 sess->sess_ops->InitiatorName); 4249 tpg->nsessions--; 4250 if (tpg->tpg_tiqn) 4251 tpg->tpg_tiqn->tiqn_nsessions--; 4252 4253 pr_debug("Decremented number of active iSCSI Sessions on" 4254 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions); 4255 4256 spin_lock(&sess_idr_lock); 4257 idr_remove(&sess_idr, sess->session_index); 4258 spin_unlock(&sess_idr_lock); 4259 4260 kfree(sess->sess_ops); 4261 sess->sess_ops = NULL; 4262 spin_unlock_bh(&se_tpg->session_lock); 4263 4264 kfree(sess); 4265 return 0; 4266 } 4267 4268 static void iscsit_logout_post_handler_closesession( 4269 struct iscsi_conn *conn) 4270 { 4271 struct iscsi_session *sess = conn->sess; 4272 4273 iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD); 4274 iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD); 4275 4276 atomic_set(&conn->conn_logout_remove, 0); 4277 complete(&conn->conn_logout_comp); 4278 4279 iscsit_dec_conn_usage_count(conn); 4280 iscsit_stop_session(sess, 1, 1); 4281 iscsit_dec_session_usage_count(sess); 4282 target_put_session(sess->se_sess); 4283 } 4284 4285 static void iscsit_logout_post_handler_samecid( 4286 struct iscsi_conn *conn) 4287 { 4288 iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD); 4289 iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD); 4290 4291 atomic_set(&conn->conn_logout_remove, 0); 4292 complete(&conn->conn_logout_comp); 4293 4294 iscsit_cause_connection_reinstatement(conn, 1); 4295 iscsit_dec_conn_usage_count(conn); 4296 } 4297 4298 static void iscsit_logout_post_handler_diffcid( 4299 struct iscsi_conn *conn, 4300 u16 cid) 4301 { 4302 struct iscsi_conn *l_conn; 4303 struct iscsi_session *sess = conn->sess; 4304 4305 if (!sess) 4306 return; 4307 4308 spin_lock_bh(&sess->conn_lock); 4309 list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) { 4310 if (l_conn->cid == cid) { 4311 iscsit_inc_conn_usage_count(l_conn); 4312 break; 4313 } 4314 } 4315 spin_unlock_bh(&sess->conn_lock); 4316 4317 if (!l_conn) 4318 return; 4319 4320 if (l_conn->sock) 4321 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN); 4322 4323 spin_lock_bh(&l_conn->state_lock); 4324 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 4325 l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT; 4326 spin_unlock_bh(&l_conn->state_lock); 4327 4328 iscsit_cause_connection_reinstatement(l_conn, 1); 4329 iscsit_dec_conn_usage_count(l_conn); 4330 } 4331 4332 /* 4333 * Return of 0 causes the TX thread to restart. 4334 */ 4335 static int iscsit_logout_post_handler( 4336 struct iscsi_cmd *cmd, 4337 struct iscsi_conn *conn) 4338 { 4339 int ret = 0; 4340 4341 switch (cmd->logout_reason) { 4342 case ISCSI_LOGOUT_REASON_CLOSE_SESSION: 4343 switch (cmd->logout_response) { 4344 case ISCSI_LOGOUT_SUCCESS: 4345 case ISCSI_LOGOUT_CLEANUP_FAILED: 4346 default: 4347 iscsit_logout_post_handler_closesession(conn); 4348 break; 4349 } 4350 ret = 0; 4351 break; 4352 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: 4353 if (conn->cid == cmd->logout_cid) { 4354 switch (cmd->logout_response) { 4355 case ISCSI_LOGOUT_SUCCESS: 4356 case ISCSI_LOGOUT_CLEANUP_FAILED: 4357 default: 4358 iscsit_logout_post_handler_samecid(conn); 4359 break; 4360 } 4361 ret = 0; 4362 } else { 4363 switch (cmd->logout_response) { 4364 case ISCSI_LOGOUT_SUCCESS: 4365 iscsit_logout_post_handler_diffcid(conn, 4366 cmd->logout_cid); 4367 break; 4368 case ISCSI_LOGOUT_CID_NOT_FOUND: 4369 case ISCSI_LOGOUT_CLEANUP_FAILED: 4370 default: 4371 break; 4372 } 4373 ret = 1; 4374 } 4375 break; 4376 case ISCSI_LOGOUT_REASON_RECOVERY: 4377 switch (cmd->logout_response) { 4378 case ISCSI_LOGOUT_SUCCESS: 4379 case ISCSI_LOGOUT_CID_NOT_FOUND: 4380 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED: 4381 case ISCSI_LOGOUT_CLEANUP_FAILED: 4382 default: 4383 break; 4384 } 4385 ret = 1; 4386 break; 4387 default: 4388 break; 4389 4390 } 4391 return ret; 4392 } 4393 4394 void iscsit_fail_session(struct iscsi_session *sess) 4395 { 4396 struct iscsi_conn *conn; 4397 4398 spin_lock_bh(&sess->conn_lock); 4399 list_for_each_entry(conn, &sess->sess_conn_list, conn_list) { 4400 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n"); 4401 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT; 4402 } 4403 spin_unlock_bh(&sess->conn_lock); 4404 4405 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); 4406 sess->session_state = TARG_SESS_STATE_FAILED; 4407 } 4408 4409 int iscsit_free_session(struct iscsi_session *sess) 4410 { 4411 u16 conn_count = atomic_read(&sess->nconn); 4412 struct iscsi_conn *conn, *conn_tmp = NULL; 4413 int is_last; 4414 4415 spin_lock_bh(&sess->conn_lock); 4416 atomic_set(&sess->sleep_on_sess_wait_comp, 1); 4417 4418 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list, 4419 conn_list) { 4420 if (conn_count == 0) 4421 break; 4422 4423 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) { 4424 is_last = 1; 4425 } else { 4426 iscsit_inc_conn_usage_count(conn_tmp); 4427 is_last = 0; 4428 } 4429 iscsit_inc_conn_usage_count(conn); 4430 4431 spin_unlock_bh(&sess->conn_lock); 4432 iscsit_cause_connection_reinstatement(conn, 1); 4433 spin_lock_bh(&sess->conn_lock); 4434 4435 iscsit_dec_conn_usage_count(conn); 4436 if (is_last == 0) 4437 iscsit_dec_conn_usage_count(conn_tmp); 4438 4439 conn_count--; 4440 } 4441 4442 if (atomic_read(&sess->nconn)) { 4443 spin_unlock_bh(&sess->conn_lock); 4444 wait_for_completion(&sess->session_wait_comp); 4445 } else 4446 spin_unlock_bh(&sess->conn_lock); 4447 4448 target_put_session(sess->se_sess); 4449 return 0; 4450 } 4451 4452 void iscsit_stop_session( 4453 struct iscsi_session *sess, 4454 int session_sleep, 4455 int connection_sleep) 4456 { 4457 u16 conn_count = atomic_read(&sess->nconn); 4458 struct iscsi_conn *conn, *conn_tmp = NULL; 4459 int is_last; 4460 4461 spin_lock_bh(&sess->conn_lock); 4462 if (session_sleep) 4463 atomic_set(&sess->sleep_on_sess_wait_comp, 1); 4464 4465 if (connection_sleep) { 4466 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list, 4467 conn_list) { 4468 if (conn_count == 0) 4469 break; 4470 4471 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) { 4472 is_last = 1; 4473 } else { 4474 iscsit_inc_conn_usage_count(conn_tmp); 4475 is_last = 0; 4476 } 4477 iscsit_inc_conn_usage_count(conn); 4478 4479 spin_unlock_bh(&sess->conn_lock); 4480 iscsit_cause_connection_reinstatement(conn, 1); 4481 spin_lock_bh(&sess->conn_lock); 4482 4483 iscsit_dec_conn_usage_count(conn); 4484 if (is_last == 0) 4485 iscsit_dec_conn_usage_count(conn_tmp); 4486 conn_count--; 4487 } 4488 } else { 4489 list_for_each_entry(conn, &sess->sess_conn_list, conn_list) 4490 iscsit_cause_connection_reinstatement(conn, 0); 4491 } 4492 4493 if (session_sleep && atomic_read(&sess->nconn)) { 4494 spin_unlock_bh(&sess->conn_lock); 4495 wait_for_completion(&sess->session_wait_comp); 4496 } else 4497 spin_unlock_bh(&sess->conn_lock); 4498 } 4499 4500 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force) 4501 { 4502 struct iscsi_session *sess; 4503 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; 4504 struct se_session *se_sess, *se_sess_tmp; 4505 int session_count = 0; 4506 4507 spin_lock_bh(&se_tpg->session_lock); 4508 if (tpg->nsessions && !force) { 4509 spin_unlock_bh(&se_tpg->session_lock); 4510 return -1; 4511 } 4512 4513 list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list, 4514 sess_list) { 4515 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr; 4516 4517 spin_lock(&sess->conn_lock); 4518 if (atomic_read(&sess->session_fall_back_to_erl0) || 4519 atomic_read(&sess->session_logout) || 4520 (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) { 4521 spin_unlock(&sess->conn_lock); 4522 continue; 4523 } 4524 atomic_set(&sess->session_reinstatement, 1); 4525 spin_unlock(&sess->conn_lock); 4526 spin_unlock_bh(&se_tpg->session_lock); 4527 4528 iscsit_free_session(sess); 4529 spin_lock_bh(&se_tpg->session_lock); 4530 4531 session_count++; 4532 } 4533 spin_unlock_bh(&se_tpg->session_lock); 4534 4535 pr_debug("Released %d iSCSI Session(s) from Target Portal" 4536 " Group: %hu\n", session_count, tpg->tpgt); 4537 return 0; 4538 } 4539 4540 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure"); 4541 MODULE_VERSION("4.1.x"); 4542 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 4543 MODULE_LICENSE("GPL"); 4544 4545 module_init(iscsi_target_init_module); 4546 module_exit(iscsi_target_cleanup_module); 4547