1 /******************************************************************************* 2 * Filename: target_core_transport.c 3 * 4 * This file contains the Generic Target Engine Core. 5 * 6 * (c) Copyright 2002-2013 Datera, Inc. 7 * 8 * Nicholas A. Bellinger <nab@kernel.org> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License as published by 12 * the Free Software Foundation; either version 2 of the License, or 13 * (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 * 20 * You should have received a copy of the GNU General Public License 21 * along with this program; if not, write to the Free Software 22 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 23 * 24 ******************************************************************************/ 25 26 #include <linux/net.h> 27 #include <linux/delay.h> 28 #include <linux/string.h> 29 #include <linux/timer.h> 30 #include <linux/slab.h> 31 #include <linux/spinlock.h> 32 #include <linux/kthread.h> 33 #include <linux/in.h> 34 #include <linux/cdrom.h> 35 #include <linux/module.h> 36 #include <linux/ratelimit.h> 37 #include <linux/vmalloc.h> 38 #include <asm/unaligned.h> 39 #include <net/sock.h> 40 #include <net/tcp.h> 41 #include <scsi/scsi_proto.h> 42 #include <scsi/scsi_common.h> 43 44 #include <target/target_core_base.h> 45 #include <target/target_core_backend.h> 46 #include <target/target_core_fabric.h> 47 48 #include "target_core_internal.h" 49 #include "target_core_alua.h" 50 #include "target_core_pr.h" 51 #include "target_core_ua.h" 52 53 #define CREATE_TRACE_POINTS 54 #include <trace/events/target.h> 55 56 static struct workqueue_struct *target_completion_wq; 57 static struct kmem_cache *se_sess_cache; 58 struct kmem_cache *se_ua_cache; 59 struct kmem_cache *t10_pr_reg_cache; 60 struct kmem_cache *t10_alua_lu_gp_cache; 61 struct kmem_cache *t10_alua_lu_gp_mem_cache; 62 struct kmem_cache *t10_alua_tg_pt_gp_cache; 63 struct kmem_cache *t10_alua_lba_map_cache; 64 struct kmem_cache *t10_alua_lba_map_mem_cache; 65 66 static void transport_complete_task_attr(struct se_cmd *cmd); 67 static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason); 68 static void transport_handle_queue_full(struct se_cmd *cmd, 69 struct se_device *dev, int err, bool write_pending); 70 static void target_complete_ok_work(struct work_struct *work); 71 72 int init_se_kmem_caches(void) 73 { 74 se_sess_cache = kmem_cache_create("se_sess_cache", 75 sizeof(struct se_session), __alignof__(struct se_session), 76 0, NULL); 77 if (!se_sess_cache) { 78 pr_err("kmem_cache_create() for struct se_session" 79 " failed\n"); 80 goto out; 81 } 82 se_ua_cache = kmem_cache_create("se_ua_cache", 83 sizeof(struct se_ua), __alignof__(struct se_ua), 84 0, NULL); 85 if (!se_ua_cache) { 86 pr_err("kmem_cache_create() for struct se_ua failed\n"); 87 goto out_free_sess_cache; 88 } 89 t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache", 90 sizeof(struct t10_pr_registration), 91 __alignof__(struct t10_pr_registration), 0, NULL); 92 if (!t10_pr_reg_cache) { 93 pr_err("kmem_cache_create() for struct t10_pr_registration" 94 " failed\n"); 95 goto out_free_ua_cache; 96 } 97 t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache", 98 sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp), 99 0, NULL); 100 if (!t10_alua_lu_gp_cache) { 101 pr_err("kmem_cache_create() for t10_alua_lu_gp_cache" 102 " failed\n"); 103 goto out_free_pr_reg_cache; 104 } 105 t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache", 106 sizeof(struct t10_alua_lu_gp_member), 107 __alignof__(struct t10_alua_lu_gp_member), 0, NULL); 108 if (!t10_alua_lu_gp_mem_cache) { 109 pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_" 110 "cache failed\n"); 111 goto out_free_lu_gp_cache; 112 } 113 t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache", 114 sizeof(struct t10_alua_tg_pt_gp), 115 __alignof__(struct t10_alua_tg_pt_gp), 0, NULL); 116 if (!t10_alua_tg_pt_gp_cache) { 117 pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_" 118 "cache failed\n"); 119 goto out_free_lu_gp_mem_cache; 120 } 121 t10_alua_lba_map_cache = kmem_cache_create( 122 "t10_alua_lba_map_cache", 123 sizeof(struct t10_alua_lba_map), 124 __alignof__(struct t10_alua_lba_map), 0, NULL); 125 if (!t10_alua_lba_map_cache) { 126 pr_err("kmem_cache_create() for t10_alua_lba_map_" 127 "cache failed\n"); 128 goto out_free_tg_pt_gp_cache; 129 } 130 t10_alua_lba_map_mem_cache = kmem_cache_create( 131 "t10_alua_lba_map_mem_cache", 132 sizeof(struct t10_alua_lba_map_member), 133 __alignof__(struct t10_alua_lba_map_member), 0, NULL); 134 if (!t10_alua_lba_map_mem_cache) { 135 pr_err("kmem_cache_create() for t10_alua_lba_map_mem_" 136 "cache failed\n"); 137 goto out_free_lba_map_cache; 138 } 139 140 target_completion_wq = alloc_workqueue("target_completion", 141 WQ_MEM_RECLAIM, 0); 142 if (!target_completion_wq) 143 goto out_free_lba_map_mem_cache; 144 145 return 0; 146 147 out_free_lba_map_mem_cache: 148 kmem_cache_destroy(t10_alua_lba_map_mem_cache); 149 out_free_lba_map_cache: 150 kmem_cache_destroy(t10_alua_lba_map_cache); 151 out_free_tg_pt_gp_cache: 152 kmem_cache_destroy(t10_alua_tg_pt_gp_cache); 153 out_free_lu_gp_mem_cache: 154 kmem_cache_destroy(t10_alua_lu_gp_mem_cache); 155 out_free_lu_gp_cache: 156 kmem_cache_destroy(t10_alua_lu_gp_cache); 157 out_free_pr_reg_cache: 158 kmem_cache_destroy(t10_pr_reg_cache); 159 out_free_ua_cache: 160 kmem_cache_destroy(se_ua_cache); 161 out_free_sess_cache: 162 kmem_cache_destroy(se_sess_cache); 163 out: 164 return -ENOMEM; 165 } 166 167 void release_se_kmem_caches(void) 168 { 169 destroy_workqueue(target_completion_wq); 170 kmem_cache_destroy(se_sess_cache); 171 kmem_cache_destroy(se_ua_cache); 172 kmem_cache_destroy(t10_pr_reg_cache); 173 kmem_cache_destroy(t10_alua_lu_gp_cache); 174 kmem_cache_destroy(t10_alua_lu_gp_mem_cache); 175 kmem_cache_destroy(t10_alua_tg_pt_gp_cache); 176 kmem_cache_destroy(t10_alua_lba_map_cache); 177 kmem_cache_destroy(t10_alua_lba_map_mem_cache); 178 } 179 180 /* This code ensures unique mib indexes are handed out. */ 181 static DEFINE_SPINLOCK(scsi_mib_index_lock); 182 static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX]; 183 184 /* 185 * Allocate a new row index for the entry type specified 186 */ 187 u32 scsi_get_new_index(scsi_index_t type) 188 { 189 u32 new_index; 190 191 BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX)); 192 193 spin_lock(&scsi_mib_index_lock); 194 new_index = ++scsi_mib_index[type]; 195 spin_unlock(&scsi_mib_index_lock); 196 197 return new_index; 198 } 199 200 void transport_subsystem_check_init(void) 201 { 202 int ret; 203 static int sub_api_initialized; 204 205 if (sub_api_initialized) 206 return; 207 208 ret = request_module("target_core_iblock"); 209 if (ret != 0) 210 pr_err("Unable to load target_core_iblock\n"); 211 212 ret = request_module("target_core_file"); 213 if (ret != 0) 214 pr_err("Unable to load target_core_file\n"); 215 216 ret = request_module("target_core_pscsi"); 217 if (ret != 0) 218 pr_err("Unable to load target_core_pscsi\n"); 219 220 ret = request_module("target_core_user"); 221 if (ret != 0) 222 pr_err("Unable to load target_core_user\n"); 223 224 sub_api_initialized = 1; 225 } 226 227 struct se_session *transport_init_session(enum target_prot_op sup_prot_ops) 228 { 229 struct se_session *se_sess; 230 231 se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL); 232 if (!se_sess) { 233 pr_err("Unable to allocate struct se_session from" 234 " se_sess_cache\n"); 235 return ERR_PTR(-ENOMEM); 236 } 237 INIT_LIST_HEAD(&se_sess->sess_list); 238 INIT_LIST_HEAD(&se_sess->sess_acl_list); 239 INIT_LIST_HEAD(&se_sess->sess_cmd_list); 240 INIT_LIST_HEAD(&se_sess->sess_wait_list); 241 spin_lock_init(&se_sess->sess_cmd_lock); 242 se_sess->sup_prot_ops = sup_prot_ops; 243 244 return se_sess; 245 } 246 EXPORT_SYMBOL(transport_init_session); 247 248 int transport_alloc_session_tags(struct se_session *se_sess, 249 unsigned int tag_num, unsigned int tag_size) 250 { 251 int rc; 252 253 se_sess->sess_cmd_map = kcalloc(tag_size, tag_num, 254 GFP_KERNEL | __GFP_NOWARN | __GFP_RETRY_MAYFAIL); 255 if (!se_sess->sess_cmd_map) { 256 se_sess->sess_cmd_map = vzalloc(array_size(tag_size, tag_num)); 257 if (!se_sess->sess_cmd_map) { 258 pr_err("Unable to allocate se_sess->sess_cmd_map\n"); 259 return -ENOMEM; 260 } 261 } 262 263 rc = percpu_ida_init(&se_sess->sess_tag_pool, tag_num); 264 if (rc < 0) { 265 pr_err("Unable to init se_sess->sess_tag_pool," 266 " tag_num: %u\n", tag_num); 267 kvfree(se_sess->sess_cmd_map); 268 se_sess->sess_cmd_map = NULL; 269 return -ENOMEM; 270 } 271 272 return 0; 273 } 274 EXPORT_SYMBOL(transport_alloc_session_tags); 275 276 struct se_session *transport_init_session_tags(unsigned int tag_num, 277 unsigned int tag_size, 278 enum target_prot_op sup_prot_ops) 279 { 280 struct se_session *se_sess; 281 int rc; 282 283 if (tag_num != 0 && !tag_size) { 284 pr_err("init_session_tags called with percpu-ida tag_num:" 285 " %u, but zero tag_size\n", tag_num); 286 return ERR_PTR(-EINVAL); 287 } 288 if (!tag_num && tag_size) { 289 pr_err("init_session_tags called with percpu-ida tag_size:" 290 " %u, but zero tag_num\n", tag_size); 291 return ERR_PTR(-EINVAL); 292 } 293 294 se_sess = transport_init_session(sup_prot_ops); 295 if (IS_ERR(se_sess)) 296 return se_sess; 297 298 rc = transport_alloc_session_tags(se_sess, tag_num, tag_size); 299 if (rc < 0) { 300 transport_free_session(se_sess); 301 return ERR_PTR(-ENOMEM); 302 } 303 304 return se_sess; 305 } 306 EXPORT_SYMBOL(transport_init_session_tags); 307 308 /* 309 * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called. 310 */ 311 void __transport_register_session( 312 struct se_portal_group *se_tpg, 313 struct se_node_acl *se_nacl, 314 struct se_session *se_sess, 315 void *fabric_sess_ptr) 316 { 317 const struct target_core_fabric_ops *tfo = se_tpg->se_tpg_tfo; 318 unsigned char buf[PR_REG_ISID_LEN]; 319 320 se_sess->se_tpg = se_tpg; 321 se_sess->fabric_sess_ptr = fabric_sess_ptr; 322 /* 323 * Used by struct se_node_acl's under ConfigFS to locate active se_session-t 324 * 325 * Only set for struct se_session's that will actually be moving I/O. 326 * eg: *NOT* discovery sessions. 327 */ 328 if (se_nacl) { 329 /* 330 * 331 * Determine if fabric allows for T10-PI feature bits exposed to 332 * initiators for device backends with !dev->dev_attrib.pi_prot_type. 333 * 334 * If so, then always save prot_type on a per se_node_acl node 335 * basis and re-instate the previous sess_prot_type to avoid 336 * disabling PI from below any previously initiator side 337 * registered LUNs. 338 */ 339 if (se_nacl->saved_prot_type) 340 se_sess->sess_prot_type = se_nacl->saved_prot_type; 341 else if (tfo->tpg_check_prot_fabric_only) 342 se_sess->sess_prot_type = se_nacl->saved_prot_type = 343 tfo->tpg_check_prot_fabric_only(se_tpg); 344 /* 345 * If the fabric module supports an ISID based TransportID, 346 * save this value in binary from the fabric I_T Nexus now. 347 */ 348 if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) { 349 memset(&buf[0], 0, PR_REG_ISID_LEN); 350 se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess, 351 &buf[0], PR_REG_ISID_LEN); 352 se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]); 353 } 354 355 spin_lock_irq(&se_nacl->nacl_sess_lock); 356 /* 357 * The se_nacl->nacl_sess pointer will be set to the 358 * last active I_T Nexus for each struct se_node_acl. 359 */ 360 se_nacl->nacl_sess = se_sess; 361 362 list_add_tail(&se_sess->sess_acl_list, 363 &se_nacl->acl_sess_list); 364 spin_unlock_irq(&se_nacl->nacl_sess_lock); 365 } 366 list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list); 367 368 pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n", 369 se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr); 370 } 371 EXPORT_SYMBOL(__transport_register_session); 372 373 void transport_register_session( 374 struct se_portal_group *se_tpg, 375 struct se_node_acl *se_nacl, 376 struct se_session *se_sess, 377 void *fabric_sess_ptr) 378 { 379 unsigned long flags; 380 381 spin_lock_irqsave(&se_tpg->session_lock, flags); 382 __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr); 383 spin_unlock_irqrestore(&se_tpg->session_lock, flags); 384 } 385 EXPORT_SYMBOL(transport_register_session); 386 387 struct se_session * 388 target_alloc_session(struct se_portal_group *tpg, 389 unsigned int tag_num, unsigned int tag_size, 390 enum target_prot_op prot_op, 391 const char *initiatorname, void *private, 392 int (*callback)(struct se_portal_group *, 393 struct se_session *, void *)) 394 { 395 struct se_session *sess; 396 397 /* 398 * If the fabric driver is using percpu-ida based pre allocation 399 * of I/O descriptor tags, go ahead and perform that setup now.. 400 */ 401 if (tag_num != 0) 402 sess = transport_init_session_tags(tag_num, tag_size, prot_op); 403 else 404 sess = transport_init_session(prot_op); 405 406 if (IS_ERR(sess)) 407 return sess; 408 409 sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg, 410 (unsigned char *)initiatorname); 411 if (!sess->se_node_acl) { 412 transport_free_session(sess); 413 return ERR_PTR(-EACCES); 414 } 415 /* 416 * Go ahead and perform any remaining fabric setup that is 417 * required before transport_register_session(). 418 */ 419 if (callback != NULL) { 420 int rc = callback(tpg, sess, private); 421 if (rc) { 422 transport_free_session(sess); 423 return ERR_PTR(rc); 424 } 425 } 426 427 transport_register_session(tpg, sess->se_node_acl, sess, private); 428 return sess; 429 } 430 EXPORT_SYMBOL(target_alloc_session); 431 432 ssize_t target_show_dynamic_sessions(struct se_portal_group *se_tpg, char *page) 433 { 434 struct se_session *se_sess; 435 ssize_t len = 0; 436 437 spin_lock_bh(&se_tpg->session_lock); 438 list_for_each_entry(se_sess, &se_tpg->tpg_sess_list, sess_list) { 439 if (!se_sess->se_node_acl) 440 continue; 441 if (!se_sess->se_node_acl->dynamic_node_acl) 442 continue; 443 if (strlen(se_sess->se_node_acl->initiatorname) + 1 + len > PAGE_SIZE) 444 break; 445 446 len += snprintf(page + len, PAGE_SIZE - len, "%s\n", 447 se_sess->se_node_acl->initiatorname); 448 len += 1; /* Include NULL terminator */ 449 } 450 spin_unlock_bh(&se_tpg->session_lock); 451 452 return len; 453 } 454 EXPORT_SYMBOL(target_show_dynamic_sessions); 455 456 static void target_complete_nacl(struct kref *kref) 457 { 458 struct se_node_acl *nacl = container_of(kref, 459 struct se_node_acl, acl_kref); 460 struct se_portal_group *se_tpg = nacl->se_tpg; 461 462 if (!nacl->dynamic_stop) { 463 complete(&nacl->acl_free_comp); 464 return; 465 } 466 467 mutex_lock(&se_tpg->acl_node_mutex); 468 list_del_init(&nacl->acl_list); 469 mutex_unlock(&se_tpg->acl_node_mutex); 470 471 core_tpg_wait_for_nacl_pr_ref(nacl); 472 core_free_device_list_for_node(nacl, se_tpg); 473 kfree(nacl); 474 } 475 476 void target_put_nacl(struct se_node_acl *nacl) 477 { 478 kref_put(&nacl->acl_kref, target_complete_nacl); 479 } 480 EXPORT_SYMBOL(target_put_nacl); 481 482 void transport_deregister_session_configfs(struct se_session *se_sess) 483 { 484 struct se_node_acl *se_nacl; 485 unsigned long flags; 486 /* 487 * Used by struct se_node_acl's under ConfigFS to locate active struct se_session 488 */ 489 se_nacl = se_sess->se_node_acl; 490 if (se_nacl) { 491 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags); 492 if (!list_empty(&se_sess->sess_acl_list)) 493 list_del_init(&se_sess->sess_acl_list); 494 /* 495 * If the session list is empty, then clear the pointer. 496 * Otherwise, set the struct se_session pointer from the tail 497 * element of the per struct se_node_acl active session list. 498 */ 499 if (list_empty(&se_nacl->acl_sess_list)) 500 se_nacl->nacl_sess = NULL; 501 else { 502 se_nacl->nacl_sess = container_of( 503 se_nacl->acl_sess_list.prev, 504 struct se_session, sess_acl_list); 505 } 506 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags); 507 } 508 } 509 EXPORT_SYMBOL(transport_deregister_session_configfs); 510 511 void transport_free_session(struct se_session *se_sess) 512 { 513 struct se_node_acl *se_nacl = se_sess->se_node_acl; 514 515 /* 516 * Drop the se_node_acl->nacl_kref obtained from within 517 * core_tpg_get_initiator_node_acl(). 518 */ 519 if (se_nacl) { 520 struct se_portal_group *se_tpg = se_nacl->se_tpg; 521 const struct target_core_fabric_ops *se_tfo = se_tpg->se_tpg_tfo; 522 unsigned long flags; 523 524 se_sess->se_node_acl = NULL; 525 526 /* 527 * Also determine if we need to drop the extra ->cmd_kref if 528 * it had been previously dynamically generated, and 529 * the endpoint is not caching dynamic ACLs. 530 */ 531 mutex_lock(&se_tpg->acl_node_mutex); 532 if (se_nacl->dynamic_node_acl && 533 !se_tfo->tpg_check_demo_mode_cache(se_tpg)) { 534 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags); 535 if (list_empty(&se_nacl->acl_sess_list)) 536 se_nacl->dynamic_stop = true; 537 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags); 538 539 if (se_nacl->dynamic_stop) 540 list_del_init(&se_nacl->acl_list); 541 } 542 mutex_unlock(&se_tpg->acl_node_mutex); 543 544 if (se_nacl->dynamic_stop) 545 target_put_nacl(se_nacl); 546 547 target_put_nacl(se_nacl); 548 } 549 if (se_sess->sess_cmd_map) { 550 percpu_ida_destroy(&se_sess->sess_tag_pool); 551 kvfree(se_sess->sess_cmd_map); 552 } 553 kmem_cache_free(se_sess_cache, se_sess); 554 } 555 EXPORT_SYMBOL(transport_free_session); 556 557 void transport_deregister_session(struct se_session *se_sess) 558 { 559 struct se_portal_group *se_tpg = se_sess->se_tpg; 560 unsigned long flags; 561 562 if (!se_tpg) { 563 transport_free_session(se_sess); 564 return; 565 } 566 567 spin_lock_irqsave(&se_tpg->session_lock, flags); 568 list_del(&se_sess->sess_list); 569 se_sess->se_tpg = NULL; 570 se_sess->fabric_sess_ptr = NULL; 571 spin_unlock_irqrestore(&se_tpg->session_lock, flags); 572 573 pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n", 574 se_tpg->se_tpg_tfo->get_fabric_name()); 575 /* 576 * If last kref is dropping now for an explicit NodeACL, awake sleeping 577 * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group 578 * removal context from within transport_free_session() code. 579 * 580 * For dynamic ACL, target_put_nacl() uses target_complete_nacl() 581 * to release all remaining generate_node_acl=1 created ACL resources. 582 */ 583 584 transport_free_session(se_sess); 585 } 586 EXPORT_SYMBOL(transport_deregister_session); 587 588 static void target_remove_from_state_list(struct se_cmd *cmd) 589 { 590 struct se_device *dev = cmd->se_dev; 591 unsigned long flags; 592 593 if (!dev) 594 return; 595 596 spin_lock_irqsave(&dev->execute_task_lock, flags); 597 if (cmd->state_active) { 598 list_del(&cmd->state_list); 599 cmd->state_active = false; 600 } 601 spin_unlock_irqrestore(&dev->execute_task_lock, flags); 602 } 603 604 static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd) 605 { 606 unsigned long flags; 607 608 target_remove_from_state_list(cmd); 609 610 /* 611 * Clear struct se_cmd->se_lun before the handoff to FE. 612 */ 613 cmd->se_lun = NULL; 614 615 spin_lock_irqsave(&cmd->t_state_lock, flags); 616 /* 617 * Determine if frontend context caller is requesting the stopping of 618 * this command for frontend exceptions. 619 */ 620 if (cmd->transport_state & CMD_T_STOP) { 621 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 622 __func__, __LINE__, cmd->tag); 623 624 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 625 626 complete_all(&cmd->t_transport_stop_comp); 627 return 1; 628 } 629 cmd->transport_state &= ~CMD_T_ACTIVE; 630 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 631 632 /* 633 * Some fabric modules like tcm_loop can release their internally 634 * allocated I/O reference and struct se_cmd now. 635 * 636 * Fabric modules are expected to return '1' here if the se_cmd being 637 * passed is released at this point, or zero if not being released. 638 */ 639 return cmd->se_tfo->check_stop_free(cmd); 640 } 641 642 static void transport_lun_remove_cmd(struct se_cmd *cmd) 643 { 644 struct se_lun *lun = cmd->se_lun; 645 646 if (!lun) 647 return; 648 649 if (cmpxchg(&cmd->lun_ref_active, true, false)) 650 percpu_ref_put(&lun->lun_ref); 651 } 652 653 int transport_cmd_finish_abort(struct se_cmd *cmd, int remove) 654 { 655 bool ack_kref = (cmd->se_cmd_flags & SCF_ACK_KREF); 656 int ret = 0; 657 658 if (cmd->se_cmd_flags & SCF_SE_LUN_CMD) 659 transport_lun_remove_cmd(cmd); 660 /* 661 * Allow the fabric driver to unmap any resources before 662 * releasing the descriptor via TFO->release_cmd() 663 */ 664 if (remove) 665 cmd->se_tfo->aborted_task(cmd); 666 667 if (transport_cmd_check_stop_to_fabric(cmd)) 668 return 1; 669 if (remove && ack_kref) 670 ret = target_put_sess_cmd(cmd); 671 672 return ret; 673 } 674 675 static void target_complete_failure_work(struct work_struct *work) 676 { 677 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 678 679 transport_generic_request_failure(cmd, 680 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); 681 } 682 683 /* 684 * Used when asking transport to copy Sense Data from the underlying 685 * Linux/SCSI struct scsi_cmnd 686 */ 687 static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd) 688 { 689 struct se_device *dev = cmd->se_dev; 690 691 WARN_ON(!cmd->se_lun); 692 693 if (!dev) 694 return NULL; 695 696 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) 697 return NULL; 698 699 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 700 701 pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n", 702 dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status); 703 return cmd->sense_buffer; 704 } 705 706 void transport_copy_sense_to_cmd(struct se_cmd *cmd, unsigned char *sense) 707 { 708 unsigned char *cmd_sense_buf; 709 unsigned long flags; 710 711 spin_lock_irqsave(&cmd->t_state_lock, flags); 712 cmd_sense_buf = transport_get_sense_buffer(cmd); 713 if (!cmd_sense_buf) { 714 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 715 return; 716 } 717 718 cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE; 719 memcpy(cmd_sense_buf, sense, cmd->scsi_sense_length); 720 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 721 } 722 EXPORT_SYMBOL(transport_copy_sense_to_cmd); 723 724 void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) 725 { 726 struct se_device *dev = cmd->se_dev; 727 int success; 728 unsigned long flags; 729 730 cmd->scsi_status = scsi_status; 731 732 spin_lock_irqsave(&cmd->t_state_lock, flags); 733 switch (cmd->scsi_status) { 734 case SAM_STAT_CHECK_CONDITION: 735 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) 736 success = 1; 737 else 738 success = 0; 739 break; 740 default: 741 success = 1; 742 break; 743 } 744 745 /* 746 * Check for case where an explicit ABORT_TASK has been received 747 * and transport_wait_for_tasks() will be waiting for completion.. 748 */ 749 if (cmd->transport_state & CMD_T_ABORTED || 750 cmd->transport_state & CMD_T_STOP) { 751 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 752 /* 753 * If COMPARE_AND_WRITE was stopped by __transport_wait_for_tasks(), 754 * release se_device->caw_sem obtained by sbc_compare_and_write() 755 * since target_complete_ok_work() or target_complete_failure_work() 756 * won't be called to invoke the normal CAW completion callbacks. 757 */ 758 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) { 759 up(&dev->caw_sem); 760 } 761 complete_all(&cmd->t_transport_stop_comp); 762 return; 763 } else if (!success) { 764 INIT_WORK(&cmd->work, target_complete_failure_work); 765 } else { 766 INIT_WORK(&cmd->work, target_complete_ok_work); 767 } 768 769 cmd->t_state = TRANSPORT_COMPLETE; 770 cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE); 771 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 772 773 if (cmd->se_cmd_flags & SCF_USE_CPUID) 774 queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work); 775 else 776 queue_work(target_completion_wq, &cmd->work); 777 } 778 EXPORT_SYMBOL(target_complete_cmd); 779 780 void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length) 781 { 782 if ((scsi_status == SAM_STAT_GOOD || 783 cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) && 784 length < cmd->data_length) { 785 if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { 786 cmd->residual_count += cmd->data_length - length; 787 } else { 788 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 789 cmd->residual_count = cmd->data_length - length; 790 } 791 792 cmd->data_length = length; 793 } 794 795 target_complete_cmd(cmd, scsi_status); 796 } 797 EXPORT_SYMBOL(target_complete_cmd_with_length); 798 799 static void target_add_to_state_list(struct se_cmd *cmd) 800 { 801 struct se_device *dev = cmd->se_dev; 802 unsigned long flags; 803 804 spin_lock_irqsave(&dev->execute_task_lock, flags); 805 if (!cmd->state_active) { 806 list_add_tail(&cmd->state_list, &dev->state_list); 807 cmd->state_active = true; 808 } 809 spin_unlock_irqrestore(&dev->execute_task_lock, flags); 810 } 811 812 /* 813 * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status 814 */ 815 static void transport_write_pending_qf(struct se_cmd *cmd); 816 static void transport_complete_qf(struct se_cmd *cmd); 817 818 void target_qf_do_work(struct work_struct *work) 819 { 820 struct se_device *dev = container_of(work, struct se_device, 821 qf_work_queue); 822 LIST_HEAD(qf_cmd_list); 823 struct se_cmd *cmd, *cmd_tmp; 824 825 spin_lock_irq(&dev->qf_cmd_lock); 826 list_splice_init(&dev->qf_cmd_list, &qf_cmd_list); 827 spin_unlock_irq(&dev->qf_cmd_lock); 828 829 list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) { 830 list_del(&cmd->se_qf_node); 831 atomic_dec_mb(&dev->dev_qf_count); 832 833 pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue" 834 " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd, 835 (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" : 836 (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING" 837 : "UNKNOWN"); 838 839 if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) 840 transport_write_pending_qf(cmd); 841 else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK || 842 cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) 843 transport_complete_qf(cmd); 844 } 845 } 846 847 unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd) 848 { 849 switch (cmd->data_direction) { 850 case DMA_NONE: 851 return "NONE"; 852 case DMA_FROM_DEVICE: 853 return "READ"; 854 case DMA_TO_DEVICE: 855 return "WRITE"; 856 case DMA_BIDIRECTIONAL: 857 return "BIDI"; 858 default: 859 break; 860 } 861 862 return "UNKNOWN"; 863 } 864 865 void transport_dump_dev_state( 866 struct se_device *dev, 867 char *b, 868 int *bl) 869 { 870 *bl += sprintf(b + *bl, "Status: "); 871 if (dev->export_count) 872 *bl += sprintf(b + *bl, "ACTIVATED"); 873 else 874 *bl += sprintf(b + *bl, "DEACTIVATED"); 875 876 *bl += sprintf(b + *bl, " Max Queue Depth: %d", dev->queue_depth); 877 *bl += sprintf(b + *bl, " SectorSize: %u HwMaxSectors: %u\n", 878 dev->dev_attrib.block_size, 879 dev->dev_attrib.hw_max_sectors); 880 *bl += sprintf(b + *bl, " "); 881 } 882 883 void transport_dump_vpd_proto_id( 884 struct t10_vpd *vpd, 885 unsigned char *p_buf, 886 int p_buf_len) 887 { 888 unsigned char buf[VPD_TMP_BUF_SIZE]; 889 int len; 890 891 memset(buf, 0, VPD_TMP_BUF_SIZE); 892 len = sprintf(buf, "T10 VPD Protocol Identifier: "); 893 894 switch (vpd->protocol_identifier) { 895 case 0x00: 896 sprintf(buf+len, "Fibre Channel\n"); 897 break; 898 case 0x10: 899 sprintf(buf+len, "Parallel SCSI\n"); 900 break; 901 case 0x20: 902 sprintf(buf+len, "SSA\n"); 903 break; 904 case 0x30: 905 sprintf(buf+len, "IEEE 1394\n"); 906 break; 907 case 0x40: 908 sprintf(buf+len, "SCSI Remote Direct Memory Access" 909 " Protocol\n"); 910 break; 911 case 0x50: 912 sprintf(buf+len, "Internet SCSI (iSCSI)\n"); 913 break; 914 case 0x60: 915 sprintf(buf+len, "SAS Serial SCSI Protocol\n"); 916 break; 917 case 0x70: 918 sprintf(buf+len, "Automation/Drive Interface Transport" 919 " Protocol\n"); 920 break; 921 case 0x80: 922 sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n"); 923 break; 924 default: 925 sprintf(buf+len, "Unknown 0x%02x\n", 926 vpd->protocol_identifier); 927 break; 928 } 929 930 if (p_buf) 931 strncpy(p_buf, buf, p_buf_len); 932 else 933 pr_debug("%s", buf); 934 } 935 936 void 937 transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83) 938 { 939 /* 940 * Check if the Protocol Identifier Valid (PIV) bit is set.. 941 * 942 * from spc3r23.pdf section 7.5.1 943 */ 944 if (page_83[1] & 0x80) { 945 vpd->protocol_identifier = (page_83[0] & 0xf0); 946 vpd->protocol_identifier_set = 1; 947 transport_dump_vpd_proto_id(vpd, NULL, 0); 948 } 949 } 950 EXPORT_SYMBOL(transport_set_vpd_proto_id); 951 952 int transport_dump_vpd_assoc( 953 struct t10_vpd *vpd, 954 unsigned char *p_buf, 955 int p_buf_len) 956 { 957 unsigned char buf[VPD_TMP_BUF_SIZE]; 958 int ret = 0; 959 int len; 960 961 memset(buf, 0, VPD_TMP_BUF_SIZE); 962 len = sprintf(buf, "T10 VPD Identifier Association: "); 963 964 switch (vpd->association) { 965 case 0x00: 966 sprintf(buf+len, "addressed logical unit\n"); 967 break; 968 case 0x10: 969 sprintf(buf+len, "target port\n"); 970 break; 971 case 0x20: 972 sprintf(buf+len, "SCSI target device\n"); 973 break; 974 default: 975 sprintf(buf+len, "Unknown 0x%02x\n", vpd->association); 976 ret = -EINVAL; 977 break; 978 } 979 980 if (p_buf) 981 strncpy(p_buf, buf, p_buf_len); 982 else 983 pr_debug("%s", buf); 984 985 return ret; 986 } 987 988 int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83) 989 { 990 /* 991 * The VPD identification association.. 992 * 993 * from spc3r23.pdf Section 7.6.3.1 Table 297 994 */ 995 vpd->association = (page_83[1] & 0x30); 996 return transport_dump_vpd_assoc(vpd, NULL, 0); 997 } 998 EXPORT_SYMBOL(transport_set_vpd_assoc); 999 1000 int transport_dump_vpd_ident_type( 1001 struct t10_vpd *vpd, 1002 unsigned char *p_buf, 1003 int p_buf_len) 1004 { 1005 unsigned char buf[VPD_TMP_BUF_SIZE]; 1006 int ret = 0; 1007 int len; 1008 1009 memset(buf, 0, VPD_TMP_BUF_SIZE); 1010 len = sprintf(buf, "T10 VPD Identifier Type: "); 1011 1012 switch (vpd->device_identifier_type) { 1013 case 0x00: 1014 sprintf(buf+len, "Vendor specific\n"); 1015 break; 1016 case 0x01: 1017 sprintf(buf+len, "T10 Vendor ID based\n"); 1018 break; 1019 case 0x02: 1020 sprintf(buf+len, "EUI-64 based\n"); 1021 break; 1022 case 0x03: 1023 sprintf(buf+len, "NAA\n"); 1024 break; 1025 case 0x04: 1026 sprintf(buf+len, "Relative target port identifier\n"); 1027 break; 1028 case 0x08: 1029 sprintf(buf+len, "SCSI name string\n"); 1030 break; 1031 default: 1032 sprintf(buf+len, "Unsupported: 0x%02x\n", 1033 vpd->device_identifier_type); 1034 ret = -EINVAL; 1035 break; 1036 } 1037 1038 if (p_buf) { 1039 if (p_buf_len < strlen(buf)+1) 1040 return -EINVAL; 1041 strncpy(p_buf, buf, p_buf_len); 1042 } else { 1043 pr_debug("%s", buf); 1044 } 1045 1046 return ret; 1047 } 1048 1049 int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83) 1050 { 1051 /* 1052 * The VPD identifier type.. 1053 * 1054 * from spc3r23.pdf Section 7.6.3.1 Table 298 1055 */ 1056 vpd->device_identifier_type = (page_83[1] & 0x0f); 1057 return transport_dump_vpd_ident_type(vpd, NULL, 0); 1058 } 1059 EXPORT_SYMBOL(transport_set_vpd_ident_type); 1060 1061 int transport_dump_vpd_ident( 1062 struct t10_vpd *vpd, 1063 unsigned char *p_buf, 1064 int p_buf_len) 1065 { 1066 unsigned char buf[VPD_TMP_BUF_SIZE]; 1067 int ret = 0; 1068 1069 memset(buf, 0, VPD_TMP_BUF_SIZE); 1070 1071 switch (vpd->device_identifier_code_set) { 1072 case 0x01: /* Binary */ 1073 snprintf(buf, sizeof(buf), 1074 "T10 VPD Binary Device Identifier: %s\n", 1075 &vpd->device_identifier[0]); 1076 break; 1077 case 0x02: /* ASCII */ 1078 snprintf(buf, sizeof(buf), 1079 "T10 VPD ASCII Device Identifier: %s\n", 1080 &vpd->device_identifier[0]); 1081 break; 1082 case 0x03: /* UTF-8 */ 1083 snprintf(buf, sizeof(buf), 1084 "T10 VPD UTF-8 Device Identifier: %s\n", 1085 &vpd->device_identifier[0]); 1086 break; 1087 default: 1088 sprintf(buf, "T10 VPD Device Identifier encoding unsupported:" 1089 " 0x%02x", vpd->device_identifier_code_set); 1090 ret = -EINVAL; 1091 break; 1092 } 1093 1094 if (p_buf) 1095 strncpy(p_buf, buf, p_buf_len); 1096 else 1097 pr_debug("%s", buf); 1098 1099 return ret; 1100 } 1101 1102 int 1103 transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83) 1104 { 1105 static const char hex_str[] = "0123456789abcdef"; 1106 int j = 0, i = 4; /* offset to start of the identifier */ 1107 1108 /* 1109 * The VPD Code Set (encoding) 1110 * 1111 * from spc3r23.pdf Section 7.6.3.1 Table 296 1112 */ 1113 vpd->device_identifier_code_set = (page_83[0] & 0x0f); 1114 switch (vpd->device_identifier_code_set) { 1115 case 0x01: /* Binary */ 1116 vpd->device_identifier[j++] = 1117 hex_str[vpd->device_identifier_type]; 1118 while (i < (4 + page_83[3])) { 1119 vpd->device_identifier[j++] = 1120 hex_str[(page_83[i] & 0xf0) >> 4]; 1121 vpd->device_identifier[j++] = 1122 hex_str[page_83[i] & 0x0f]; 1123 i++; 1124 } 1125 break; 1126 case 0x02: /* ASCII */ 1127 case 0x03: /* UTF-8 */ 1128 while (i < (4 + page_83[3])) 1129 vpd->device_identifier[j++] = page_83[i++]; 1130 break; 1131 default: 1132 break; 1133 } 1134 1135 return transport_dump_vpd_ident(vpd, NULL, 0); 1136 } 1137 EXPORT_SYMBOL(transport_set_vpd_ident); 1138 1139 static sense_reason_t 1140 target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev, 1141 unsigned int size) 1142 { 1143 u32 mtl; 1144 1145 if (!cmd->se_tfo->max_data_sg_nents) 1146 return TCM_NO_SENSE; 1147 /* 1148 * Check if fabric enforced maximum SGL entries per I/O descriptor 1149 * exceeds se_cmd->data_length. If true, set SCF_UNDERFLOW_BIT + 1150 * residual_count and reduce original cmd->data_length to maximum 1151 * length based on single PAGE_SIZE entry scatter-lists. 1152 */ 1153 mtl = (cmd->se_tfo->max_data_sg_nents * PAGE_SIZE); 1154 if (cmd->data_length > mtl) { 1155 /* 1156 * If an existing CDB overflow is present, calculate new residual 1157 * based on CDB size minus fabric maximum transfer length. 1158 * 1159 * If an existing CDB underflow is present, calculate new residual 1160 * based on original cmd->data_length minus fabric maximum transfer 1161 * length. 1162 * 1163 * Otherwise, set the underflow residual based on cmd->data_length 1164 * minus fabric maximum transfer length. 1165 */ 1166 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) { 1167 cmd->residual_count = (size - mtl); 1168 } else if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { 1169 u32 orig_dl = size + cmd->residual_count; 1170 cmd->residual_count = (orig_dl - mtl); 1171 } else { 1172 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 1173 cmd->residual_count = (cmd->data_length - mtl); 1174 } 1175 cmd->data_length = mtl; 1176 /* 1177 * Reset sbc_check_prot() calculated protection payload 1178 * length based upon the new smaller MTL. 1179 */ 1180 if (cmd->prot_length) { 1181 u32 sectors = (mtl / dev->dev_attrib.block_size); 1182 cmd->prot_length = dev->prot_length * sectors; 1183 } 1184 } 1185 return TCM_NO_SENSE; 1186 } 1187 1188 sense_reason_t 1189 target_cmd_size_check(struct se_cmd *cmd, unsigned int size) 1190 { 1191 struct se_device *dev = cmd->se_dev; 1192 1193 if (cmd->unknown_data_length) { 1194 cmd->data_length = size; 1195 } else if (size != cmd->data_length) { 1196 pr_warn_ratelimited("TARGET_CORE[%s]: Expected Transfer Length:" 1197 " %u does not match SCSI CDB Length: %u for SAM Opcode:" 1198 " 0x%02x\n", cmd->se_tfo->get_fabric_name(), 1199 cmd->data_length, size, cmd->t_task_cdb[0]); 1200 1201 if (cmd->data_direction == DMA_TO_DEVICE) { 1202 if (cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) { 1203 pr_err_ratelimited("Rejecting underflow/overflow" 1204 " for WRITE data CDB\n"); 1205 return TCM_INVALID_CDB_FIELD; 1206 } 1207 /* 1208 * Some fabric drivers like iscsi-target still expect to 1209 * always reject overflow writes. Reject this case until 1210 * full fabric driver level support for overflow writes 1211 * is introduced tree-wide. 1212 */ 1213 if (size > cmd->data_length) { 1214 pr_err_ratelimited("Rejecting overflow for" 1215 " WRITE control CDB\n"); 1216 return TCM_INVALID_CDB_FIELD; 1217 } 1218 } 1219 /* 1220 * Reject READ_* or WRITE_* with overflow/underflow for 1221 * type SCF_SCSI_DATA_CDB. 1222 */ 1223 if (dev->dev_attrib.block_size != 512) { 1224 pr_err("Failing OVERFLOW/UNDERFLOW for LBA op" 1225 " CDB on non 512-byte sector setup subsystem" 1226 " plugin: %s\n", dev->transport->name); 1227 /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */ 1228 return TCM_INVALID_CDB_FIELD; 1229 } 1230 /* 1231 * For the overflow case keep the existing fabric provided 1232 * ->data_length. Otherwise for the underflow case, reset 1233 * ->data_length to the smaller SCSI expected data transfer 1234 * length. 1235 */ 1236 if (size > cmd->data_length) { 1237 cmd->se_cmd_flags |= SCF_OVERFLOW_BIT; 1238 cmd->residual_count = (size - cmd->data_length); 1239 } else { 1240 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 1241 cmd->residual_count = (cmd->data_length - size); 1242 cmd->data_length = size; 1243 } 1244 } 1245 1246 return target_check_max_data_sg_nents(cmd, dev, size); 1247 1248 } 1249 1250 /* 1251 * Used by fabric modules containing a local struct se_cmd within their 1252 * fabric dependent per I/O descriptor. 1253 * 1254 * Preserves the value of @cmd->tag. 1255 */ 1256 void transport_init_se_cmd( 1257 struct se_cmd *cmd, 1258 const struct target_core_fabric_ops *tfo, 1259 struct se_session *se_sess, 1260 u32 data_length, 1261 int data_direction, 1262 int task_attr, 1263 unsigned char *sense_buffer) 1264 { 1265 INIT_LIST_HEAD(&cmd->se_delayed_node); 1266 INIT_LIST_HEAD(&cmd->se_qf_node); 1267 INIT_LIST_HEAD(&cmd->se_cmd_list); 1268 INIT_LIST_HEAD(&cmd->state_list); 1269 init_completion(&cmd->t_transport_stop_comp); 1270 init_completion(&cmd->cmd_wait_comp); 1271 spin_lock_init(&cmd->t_state_lock); 1272 INIT_WORK(&cmd->work, NULL); 1273 kref_init(&cmd->cmd_kref); 1274 1275 cmd->se_tfo = tfo; 1276 cmd->se_sess = se_sess; 1277 cmd->data_length = data_length; 1278 cmd->data_direction = data_direction; 1279 cmd->sam_task_attr = task_attr; 1280 cmd->sense_buffer = sense_buffer; 1281 1282 cmd->state_active = false; 1283 } 1284 EXPORT_SYMBOL(transport_init_se_cmd); 1285 1286 static sense_reason_t 1287 transport_check_alloc_task_attr(struct se_cmd *cmd) 1288 { 1289 struct se_device *dev = cmd->se_dev; 1290 1291 /* 1292 * Check if SAM Task Attribute emulation is enabled for this 1293 * struct se_device storage object 1294 */ 1295 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1296 return 0; 1297 1298 if (cmd->sam_task_attr == TCM_ACA_TAG) { 1299 pr_debug("SAM Task Attribute ACA" 1300 " emulation is not supported\n"); 1301 return TCM_INVALID_CDB_FIELD; 1302 } 1303 1304 return 0; 1305 } 1306 1307 sense_reason_t 1308 target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) 1309 { 1310 struct se_device *dev = cmd->se_dev; 1311 sense_reason_t ret; 1312 1313 /* 1314 * Ensure that the received CDB is less than the max (252 + 8) bytes 1315 * for VARIABLE_LENGTH_CMD 1316 */ 1317 if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) { 1318 pr_err("Received SCSI CDB with command_size: %d that" 1319 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", 1320 scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE); 1321 return TCM_INVALID_CDB_FIELD; 1322 } 1323 /* 1324 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE, 1325 * allocate the additional extended CDB buffer now.. Otherwise 1326 * setup the pointer from __t_task_cdb to t_task_cdb. 1327 */ 1328 if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) { 1329 cmd->t_task_cdb = kzalloc(scsi_command_size(cdb), 1330 GFP_KERNEL); 1331 if (!cmd->t_task_cdb) { 1332 pr_err("Unable to allocate cmd->t_task_cdb" 1333 " %u > sizeof(cmd->__t_task_cdb): %lu ops\n", 1334 scsi_command_size(cdb), 1335 (unsigned long)sizeof(cmd->__t_task_cdb)); 1336 return TCM_OUT_OF_RESOURCES; 1337 } 1338 } else 1339 cmd->t_task_cdb = &cmd->__t_task_cdb[0]; 1340 /* 1341 * Copy the original CDB into cmd-> 1342 */ 1343 memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb)); 1344 1345 trace_target_sequencer_start(cmd); 1346 1347 ret = dev->transport->parse_cdb(cmd); 1348 if (ret == TCM_UNSUPPORTED_SCSI_OPCODE) 1349 pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n", 1350 cmd->se_tfo->get_fabric_name(), 1351 cmd->se_sess->se_node_acl->initiatorname, 1352 cmd->t_task_cdb[0]); 1353 if (ret) 1354 return ret; 1355 1356 ret = transport_check_alloc_task_attr(cmd); 1357 if (ret) 1358 return ret; 1359 1360 cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE; 1361 atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus); 1362 return 0; 1363 } 1364 EXPORT_SYMBOL(target_setup_cmd_from_cdb); 1365 1366 /* 1367 * Used by fabric module frontends to queue tasks directly. 1368 * May only be used from process context. 1369 */ 1370 int transport_handle_cdb_direct( 1371 struct se_cmd *cmd) 1372 { 1373 sense_reason_t ret; 1374 1375 if (!cmd->se_lun) { 1376 dump_stack(); 1377 pr_err("cmd->se_lun is NULL\n"); 1378 return -EINVAL; 1379 } 1380 if (in_interrupt()) { 1381 dump_stack(); 1382 pr_err("transport_generic_handle_cdb cannot be called" 1383 " from interrupt context\n"); 1384 return -EINVAL; 1385 } 1386 /* 1387 * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that 1388 * outstanding descriptors are handled correctly during shutdown via 1389 * transport_wait_for_tasks() 1390 * 1391 * Also, we don't take cmd->t_state_lock here as we only expect 1392 * this to be called for initial descriptor submission. 1393 */ 1394 cmd->t_state = TRANSPORT_NEW_CMD; 1395 cmd->transport_state |= CMD_T_ACTIVE; 1396 1397 /* 1398 * transport_generic_new_cmd() is already handling QUEUE_FULL, 1399 * so follow TRANSPORT_NEW_CMD processing thread context usage 1400 * and call transport_generic_request_failure() if necessary.. 1401 */ 1402 ret = transport_generic_new_cmd(cmd); 1403 if (ret) 1404 transport_generic_request_failure(cmd, ret); 1405 return 0; 1406 } 1407 EXPORT_SYMBOL(transport_handle_cdb_direct); 1408 1409 sense_reason_t 1410 transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl, 1411 u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count) 1412 { 1413 if (!sgl || !sgl_count) 1414 return 0; 1415 1416 /* 1417 * Reject SCSI data overflow with map_mem_to_cmd() as incoming 1418 * scatterlists already have been set to follow what the fabric 1419 * passes for the original expected data transfer length. 1420 */ 1421 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) { 1422 pr_warn("Rejecting SCSI DATA overflow for fabric using" 1423 " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n"); 1424 return TCM_INVALID_CDB_FIELD; 1425 } 1426 1427 cmd->t_data_sg = sgl; 1428 cmd->t_data_nents = sgl_count; 1429 cmd->t_bidi_data_sg = sgl_bidi; 1430 cmd->t_bidi_data_nents = sgl_bidi_count; 1431 1432 cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1433 return 0; 1434 } 1435 1436 /** 1437 * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized 1438 * se_cmd + use pre-allocated SGL memory. 1439 * 1440 * @se_cmd: command descriptor to submit 1441 * @se_sess: associated se_sess for endpoint 1442 * @cdb: pointer to SCSI CDB 1443 * @sense: pointer to SCSI sense buffer 1444 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1445 * @data_length: fabric expected data transfer length 1446 * @task_attr: SAM task attribute 1447 * @data_dir: DMA data direction 1448 * @flags: flags for command submission from target_sc_flags_tables 1449 * @sgl: struct scatterlist memory for unidirectional mapping 1450 * @sgl_count: scatterlist count for unidirectional mapping 1451 * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping 1452 * @sgl_bidi_count: scatterlist count for bidirectional READ mapping 1453 * @sgl_prot: struct scatterlist memory protection information 1454 * @sgl_prot_count: scatterlist count for protection information 1455 * 1456 * Task tags are supported if the caller has set @se_cmd->tag. 1457 * 1458 * Returns non zero to signal active I/O shutdown failure. All other 1459 * setup exceptions will be returned as a SCSI CHECK_CONDITION response, 1460 * but still return zero here. 1461 * 1462 * This may only be called from process context, and also currently 1463 * assumes internal allocation of fabric payload buffer by target-core. 1464 */ 1465 int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess, 1466 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, 1467 u32 data_length, int task_attr, int data_dir, int flags, 1468 struct scatterlist *sgl, u32 sgl_count, 1469 struct scatterlist *sgl_bidi, u32 sgl_bidi_count, 1470 struct scatterlist *sgl_prot, u32 sgl_prot_count) 1471 { 1472 struct se_portal_group *se_tpg; 1473 sense_reason_t rc; 1474 int ret; 1475 1476 se_tpg = se_sess->se_tpg; 1477 BUG_ON(!se_tpg); 1478 BUG_ON(se_cmd->se_tfo || se_cmd->se_sess); 1479 BUG_ON(in_interrupt()); 1480 /* 1481 * Initialize se_cmd for target operation. From this point 1482 * exceptions are handled by sending exception status via 1483 * target_core_fabric_ops->queue_status() callback 1484 */ 1485 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, 1486 data_length, data_dir, task_attr, sense); 1487 1488 if (flags & TARGET_SCF_USE_CPUID) 1489 se_cmd->se_cmd_flags |= SCF_USE_CPUID; 1490 else 1491 se_cmd->cpuid = WORK_CPU_UNBOUND; 1492 1493 if (flags & TARGET_SCF_UNKNOWN_SIZE) 1494 se_cmd->unknown_data_length = 1; 1495 /* 1496 * Obtain struct se_cmd->cmd_kref reference and add new cmd to 1497 * se_sess->sess_cmd_list. A second kref_get here is necessary 1498 * for fabrics using TARGET_SCF_ACK_KREF that expect a second 1499 * kref_put() to happen during fabric packet acknowledgement. 1500 */ 1501 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); 1502 if (ret) 1503 return ret; 1504 /* 1505 * Signal bidirectional data payloads to target-core 1506 */ 1507 if (flags & TARGET_SCF_BIDI_OP) 1508 se_cmd->se_cmd_flags |= SCF_BIDI; 1509 /* 1510 * Locate se_lun pointer and attach it to struct se_cmd 1511 */ 1512 rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun); 1513 if (rc) { 1514 transport_send_check_condition_and_sense(se_cmd, rc, 0); 1515 target_put_sess_cmd(se_cmd); 1516 return 0; 1517 } 1518 1519 rc = target_setup_cmd_from_cdb(se_cmd, cdb); 1520 if (rc != 0) { 1521 transport_generic_request_failure(se_cmd, rc); 1522 return 0; 1523 } 1524 1525 /* 1526 * Save pointers for SGLs containing protection information, 1527 * if present. 1528 */ 1529 if (sgl_prot_count) { 1530 se_cmd->t_prot_sg = sgl_prot; 1531 se_cmd->t_prot_nents = sgl_prot_count; 1532 se_cmd->se_cmd_flags |= SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC; 1533 } 1534 1535 /* 1536 * When a non zero sgl_count has been passed perform SGL passthrough 1537 * mapping for pre-allocated fabric memory instead of having target 1538 * core perform an internal SGL allocation.. 1539 */ 1540 if (sgl_count != 0) { 1541 BUG_ON(!sgl); 1542 1543 /* 1544 * A work-around for tcm_loop as some userspace code via 1545 * scsi-generic do not memset their associated read buffers, 1546 * so go ahead and do that here for type non-data CDBs. Also 1547 * note that this is currently guaranteed to be a single SGL 1548 * for this case by target core in target_setup_cmd_from_cdb() 1549 * -> transport_generic_cmd_sequencer(). 1550 */ 1551 if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) && 1552 se_cmd->data_direction == DMA_FROM_DEVICE) { 1553 unsigned char *buf = NULL; 1554 1555 if (sgl) 1556 buf = kmap(sg_page(sgl)) + sgl->offset; 1557 1558 if (buf) { 1559 memset(buf, 0, sgl->length); 1560 kunmap(sg_page(sgl)); 1561 } 1562 } 1563 1564 rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count, 1565 sgl_bidi, sgl_bidi_count); 1566 if (rc != 0) { 1567 transport_generic_request_failure(se_cmd, rc); 1568 return 0; 1569 } 1570 } 1571 1572 /* 1573 * Check if we need to delay processing because of ALUA 1574 * Active/NonOptimized primary access state.. 1575 */ 1576 core_alua_check_nonop_delay(se_cmd); 1577 1578 transport_handle_cdb_direct(se_cmd); 1579 return 0; 1580 } 1581 EXPORT_SYMBOL(target_submit_cmd_map_sgls); 1582 1583 /** 1584 * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd 1585 * 1586 * @se_cmd: command descriptor to submit 1587 * @se_sess: associated se_sess for endpoint 1588 * @cdb: pointer to SCSI CDB 1589 * @sense: pointer to SCSI sense buffer 1590 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1591 * @data_length: fabric expected data transfer length 1592 * @task_attr: SAM task attribute 1593 * @data_dir: DMA data direction 1594 * @flags: flags for command submission from target_sc_flags_tables 1595 * 1596 * Task tags are supported if the caller has set @se_cmd->tag. 1597 * 1598 * Returns non zero to signal active I/O shutdown failure. All other 1599 * setup exceptions will be returned as a SCSI CHECK_CONDITION response, 1600 * but still return zero here. 1601 * 1602 * This may only be called from process context, and also currently 1603 * assumes internal allocation of fabric payload buffer by target-core. 1604 * 1605 * It also assumes interal target core SGL memory allocation. 1606 */ 1607 int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, 1608 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, 1609 u32 data_length, int task_attr, int data_dir, int flags) 1610 { 1611 return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense, 1612 unpacked_lun, data_length, task_attr, data_dir, 1613 flags, NULL, 0, NULL, 0, NULL, 0); 1614 } 1615 EXPORT_SYMBOL(target_submit_cmd); 1616 1617 static void target_complete_tmr_failure(struct work_struct *work) 1618 { 1619 struct se_cmd *se_cmd = container_of(work, struct se_cmd, work); 1620 1621 se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST; 1622 se_cmd->se_tfo->queue_tm_rsp(se_cmd); 1623 1624 transport_lun_remove_cmd(se_cmd); 1625 transport_cmd_check_stop_to_fabric(se_cmd); 1626 } 1627 1628 static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag, 1629 u64 *unpacked_lun) 1630 { 1631 struct se_cmd *se_cmd; 1632 unsigned long flags; 1633 bool ret = false; 1634 1635 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 1636 list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) { 1637 if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) 1638 continue; 1639 1640 if (se_cmd->tag == tag) { 1641 *unpacked_lun = se_cmd->orig_fe_lun; 1642 ret = true; 1643 break; 1644 } 1645 } 1646 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 1647 1648 return ret; 1649 } 1650 1651 /** 1652 * target_submit_tmr - lookup unpacked lun and submit uninitialized se_cmd 1653 * for TMR CDBs 1654 * 1655 * @se_cmd: command descriptor to submit 1656 * @se_sess: associated se_sess for endpoint 1657 * @sense: pointer to SCSI sense buffer 1658 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1659 * @fabric_tmr_ptr: fabric context for TMR req 1660 * @tm_type: Type of TM request 1661 * @gfp: gfp type for caller 1662 * @tag: referenced task tag for TMR_ABORT_TASK 1663 * @flags: submit cmd flags 1664 * 1665 * Callable from all contexts. 1666 **/ 1667 1668 int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, 1669 unsigned char *sense, u64 unpacked_lun, 1670 void *fabric_tmr_ptr, unsigned char tm_type, 1671 gfp_t gfp, u64 tag, int flags) 1672 { 1673 struct se_portal_group *se_tpg; 1674 int ret; 1675 1676 se_tpg = se_sess->se_tpg; 1677 BUG_ON(!se_tpg); 1678 1679 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, 1680 0, DMA_NONE, TCM_SIMPLE_TAG, sense); 1681 /* 1682 * FIXME: Currently expect caller to handle se_cmd->se_tmr_req 1683 * allocation failure. 1684 */ 1685 ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp); 1686 if (ret < 0) 1687 return -ENOMEM; 1688 1689 if (tm_type == TMR_ABORT_TASK) 1690 se_cmd->se_tmr_req->ref_task_tag = tag; 1691 1692 /* See target_submit_cmd for commentary */ 1693 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); 1694 if (ret) { 1695 core_tmr_release_req(se_cmd->se_tmr_req); 1696 return ret; 1697 } 1698 /* 1699 * If this is ABORT_TASK with no explicit fabric provided LUN, 1700 * go ahead and search active session tags for a match to figure 1701 * out unpacked_lun for the original se_cmd. 1702 */ 1703 if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) { 1704 if (!target_lookup_lun_from_tag(se_sess, tag, &unpacked_lun)) 1705 goto failure; 1706 } 1707 1708 ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun); 1709 if (ret) 1710 goto failure; 1711 1712 transport_generic_handle_tmr(se_cmd); 1713 return 0; 1714 1715 /* 1716 * For callback during failure handling, push this work off 1717 * to process context with TMR_LUN_DOES_NOT_EXIST status. 1718 */ 1719 failure: 1720 INIT_WORK(&se_cmd->work, target_complete_tmr_failure); 1721 schedule_work(&se_cmd->work); 1722 return 0; 1723 } 1724 EXPORT_SYMBOL(target_submit_tmr); 1725 1726 /* 1727 * Handle SAM-esque emulation for generic transport request failures. 1728 */ 1729 void transport_generic_request_failure(struct se_cmd *cmd, 1730 sense_reason_t sense_reason) 1731 { 1732 int ret = 0, post_ret = 0; 1733 1734 pr_debug("-----[ Storage Engine Exception; sense_reason %d\n", 1735 sense_reason); 1736 target_show_cmd("-----[ ", cmd); 1737 1738 /* 1739 * For SAM Task Attribute emulation for failed struct se_cmd 1740 */ 1741 transport_complete_task_attr(cmd); 1742 1743 /* 1744 * Handle special case for COMPARE_AND_WRITE failure, where the 1745 * callback is expected to drop the per device ->caw_sem. 1746 */ 1747 if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) && 1748 cmd->transport_complete_callback) 1749 cmd->transport_complete_callback(cmd, false, &post_ret); 1750 1751 if (transport_check_aborted_status(cmd, 1)) 1752 return; 1753 1754 switch (sense_reason) { 1755 case TCM_NON_EXISTENT_LUN: 1756 case TCM_UNSUPPORTED_SCSI_OPCODE: 1757 case TCM_INVALID_CDB_FIELD: 1758 case TCM_INVALID_PARAMETER_LIST: 1759 case TCM_PARAMETER_LIST_LENGTH_ERROR: 1760 case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE: 1761 case TCM_UNKNOWN_MODE_PAGE: 1762 case TCM_WRITE_PROTECTED: 1763 case TCM_ADDRESS_OUT_OF_RANGE: 1764 case TCM_CHECK_CONDITION_ABORT_CMD: 1765 case TCM_CHECK_CONDITION_UNIT_ATTENTION: 1766 case TCM_CHECK_CONDITION_NOT_READY: 1767 case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED: 1768 case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED: 1769 case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED: 1770 case TCM_COPY_TARGET_DEVICE_NOT_REACHABLE: 1771 case TCM_TOO_MANY_TARGET_DESCS: 1772 case TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE: 1773 case TCM_TOO_MANY_SEGMENT_DESCS: 1774 case TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE: 1775 break; 1776 case TCM_OUT_OF_RESOURCES: 1777 cmd->scsi_status = SAM_STAT_TASK_SET_FULL; 1778 goto queue_status; 1779 case TCM_LUN_BUSY: 1780 cmd->scsi_status = SAM_STAT_BUSY; 1781 goto queue_status; 1782 case TCM_RESERVATION_CONFLICT: 1783 /* 1784 * No SENSE Data payload for this case, set SCSI Status 1785 * and queue the response to $FABRIC_MOD. 1786 * 1787 * Uses linux/include/scsi/scsi.h SAM status codes defs 1788 */ 1789 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT; 1790 /* 1791 * For UA Interlock Code 11b, a RESERVATION CONFLICT will 1792 * establish a UNIT ATTENTION with PREVIOUS RESERVATION 1793 * CONFLICT STATUS. 1794 * 1795 * See spc4r17, section 7.4.6 Control Mode Page, Table 349 1796 */ 1797 if (cmd->se_sess && 1798 cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) { 1799 target_ua_allocate_lun(cmd->se_sess->se_node_acl, 1800 cmd->orig_fe_lun, 0x2C, 1801 ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS); 1802 } 1803 1804 goto queue_status; 1805 default: 1806 pr_err("Unknown transport error for CDB 0x%02x: %d\n", 1807 cmd->t_task_cdb[0], sense_reason); 1808 sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE; 1809 break; 1810 } 1811 1812 ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0); 1813 if (ret) 1814 goto queue_full; 1815 1816 check_stop: 1817 transport_lun_remove_cmd(cmd); 1818 transport_cmd_check_stop_to_fabric(cmd); 1819 return; 1820 1821 queue_status: 1822 trace_target_cmd_complete(cmd); 1823 ret = cmd->se_tfo->queue_status(cmd); 1824 if (!ret) 1825 goto check_stop; 1826 queue_full: 1827 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 1828 } 1829 EXPORT_SYMBOL(transport_generic_request_failure); 1830 1831 void __target_execute_cmd(struct se_cmd *cmd, bool do_checks) 1832 { 1833 sense_reason_t ret; 1834 1835 if (!cmd->execute_cmd) { 1836 ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 1837 goto err; 1838 } 1839 if (do_checks) { 1840 /* 1841 * Check for an existing UNIT ATTENTION condition after 1842 * target_handle_task_attr() has done SAM task attr 1843 * checking, and possibly have already defered execution 1844 * out to target_restart_delayed_cmds() context. 1845 */ 1846 ret = target_scsi3_ua_check(cmd); 1847 if (ret) 1848 goto err; 1849 1850 ret = target_alua_state_check(cmd); 1851 if (ret) 1852 goto err; 1853 1854 ret = target_check_reservation(cmd); 1855 if (ret) { 1856 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT; 1857 goto err; 1858 } 1859 } 1860 1861 ret = cmd->execute_cmd(cmd); 1862 if (!ret) 1863 return; 1864 err: 1865 spin_lock_irq(&cmd->t_state_lock); 1866 cmd->transport_state &= ~CMD_T_SENT; 1867 spin_unlock_irq(&cmd->t_state_lock); 1868 1869 transport_generic_request_failure(cmd, ret); 1870 } 1871 1872 static int target_write_prot_action(struct se_cmd *cmd) 1873 { 1874 u32 sectors; 1875 /* 1876 * Perform WRITE_INSERT of PI using software emulation when backend 1877 * device has PI enabled, if the transport has not already generated 1878 * PI using hardware WRITE_INSERT offload. 1879 */ 1880 switch (cmd->prot_op) { 1881 case TARGET_PROT_DOUT_INSERT: 1882 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_INSERT)) 1883 sbc_dif_generate(cmd); 1884 break; 1885 case TARGET_PROT_DOUT_STRIP: 1886 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_STRIP) 1887 break; 1888 1889 sectors = cmd->data_length >> ilog2(cmd->se_dev->dev_attrib.block_size); 1890 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba, 1891 sectors, 0, cmd->t_prot_sg, 0); 1892 if (unlikely(cmd->pi_err)) { 1893 spin_lock_irq(&cmd->t_state_lock); 1894 cmd->transport_state &= ~CMD_T_SENT; 1895 spin_unlock_irq(&cmd->t_state_lock); 1896 transport_generic_request_failure(cmd, cmd->pi_err); 1897 return -1; 1898 } 1899 break; 1900 default: 1901 break; 1902 } 1903 1904 return 0; 1905 } 1906 1907 static bool target_handle_task_attr(struct se_cmd *cmd) 1908 { 1909 struct se_device *dev = cmd->se_dev; 1910 1911 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1912 return false; 1913 1914 cmd->se_cmd_flags |= SCF_TASK_ATTR_SET; 1915 1916 /* 1917 * Check for the existence of HEAD_OF_QUEUE, and if true return 1 1918 * to allow the passed struct se_cmd list of tasks to the front of the list. 1919 */ 1920 switch (cmd->sam_task_attr) { 1921 case TCM_HEAD_TAG: 1922 pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x\n", 1923 cmd->t_task_cdb[0]); 1924 return false; 1925 case TCM_ORDERED_TAG: 1926 atomic_inc_mb(&dev->dev_ordered_sync); 1927 1928 pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n", 1929 cmd->t_task_cdb[0]); 1930 1931 /* 1932 * Execute an ORDERED command if no other older commands 1933 * exist that need to be completed first. 1934 */ 1935 if (!atomic_read(&dev->simple_cmds)) 1936 return false; 1937 break; 1938 default: 1939 /* 1940 * For SIMPLE and UNTAGGED Task Attribute commands 1941 */ 1942 atomic_inc_mb(&dev->simple_cmds); 1943 break; 1944 } 1945 1946 if (atomic_read(&dev->dev_ordered_sync) == 0) 1947 return false; 1948 1949 spin_lock(&dev->delayed_cmd_lock); 1950 list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list); 1951 spin_unlock(&dev->delayed_cmd_lock); 1952 1953 pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn", 1954 cmd->t_task_cdb[0], cmd->sam_task_attr); 1955 return true; 1956 } 1957 1958 static int __transport_check_aborted_status(struct se_cmd *, int); 1959 1960 void target_execute_cmd(struct se_cmd *cmd) 1961 { 1962 /* 1963 * Determine if frontend context caller is requesting the stopping of 1964 * this command for frontend exceptions. 1965 * 1966 * If the received CDB has aleady been aborted stop processing it here. 1967 */ 1968 spin_lock_irq(&cmd->t_state_lock); 1969 if (__transport_check_aborted_status(cmd, 1)) { 1970 spin_unlock_irq(&cmd->t_state_lock); 1971 return; 1972 } 1973 if (cmd->transport_state & CMD_T_STOP) { 1974 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 1975 __func__, __LINE__, cmd->tag); 1976 1977 spin_unlock_irq(&cmd->t_state_lock); 1978 complete_all(&cmd->t_transport_stop_comp); 1979 return; 1980 } 1981 1982 cmd->t_state = TRANSPORT_PROCESSING; 1983 cmd->transport_state &= ~CMD_T_PRE_EXECUTE; 1984 cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT; 1985 spin_unlock_irq(&cmd->t_state_lock); 1986 1987 if (target_write_prot_action(cmd)) 1988 return; 1989 1990 if (target_handle_task_attr(cmd)) { 1991 spin_lock_irq(&cmd->t_state_lock); 1992 cmd->transport_state &= ~CMD_T_SENT; 1993 spin_unlock_irq(&cmd->t_state_lock); 1994 return; 1995 } 1996 1997 __target_execute_cmd(cmd, true); 1998 } 1999 EXPORT_SYMBOL(target_execute_cmd); 2000 2001 /* 2002 * Process all commands up to the last received ORDERED task attribute which 2003 * requires another blocking boundary 2004 */ 2005 static void target_restart_delayed_cmds(struct se_device *dev) 2006 { 2007 for (;;) { 2008 struct se_cmd *cmd; 2009 2010 spin_lock(&dev->delayed_cmd_lock); 2011 if (list_empty(&dev->delayed_cmd_list)) { 2012 spin_unlock(&dev->delayed_cmd_lock); 2013 break; 2014 } 2015 2016 cmd = list_entry(dev->delayed_cmd_list.next, 2017 struct se_cmd, se_delayed_node); 2018 list_del(&cmd->se_delayed_node); 2019 spin_unlock(&dev->delayed_cmd_lock); 2020 2021 cmd->transport_state |= CMD_T_SENT; 2022 2023 __target_execute_cmd(cmd, true); 2024 2025 if (cmd->sam_task_attr == TCM_ORDERED_TAG) 2026 break; 2027 } 2028 } 2029 2030 /* 2031 * Called from I/O completion to determine which dormant/delayed 2032 * and ordered cmds need to have their tasks added to the execution queue. 2033 */ 2034 static void transport_complete_task_attr(struct se_cmd *cmd) 2035 { 2036 struct se_device *dev = cmd->se_dev; 2037 2038 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 2039 return; 2040 2041 if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET)) 2042 goto restart; 2043 2044 if (cmd->sam_task_attr == TCM_SIMPLE_TAG) { 2045 atomic_dec_mb(&dev->simple_cmds); 2046 dev->dev_cur_ordered_id++; 2047 } else if (cmd->sam_task_attr == TCM_HEAD_TAG) { 2048 dev->dev_cur_ordered_id++; 2049 pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n", 2050 dev->dev_cur_ordered_id); 2051 } else if (cmd->sam_task_attr == TCM_ORDERED_TAG) { 2052 atomic_dec_mb(&dev->dev_ordered_sync); 2053 2054 dev->dev_cur_ordered_id++; 2055 pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n", 2056 dev->dev_cur_ordered_id); 2057 } 2058 cmd->se_cmd_flags &= ~SCF_TASK_ATTR_SET; 2059 2060 restart: 2061 target_restart_delayed_cmds(dev); 2062 } 2063 2064 static void transport_complete_qf(struct se_cmd *cmd) 2065 { 2066 int ret = 0; 2067 2068 transport_complete_task_attr(cmd); 2069 /* 2070 * If a fabric driver ->write_pending() or ->queue_data_in() callback 2071 * has returned neither -ENOMEM or -EAGAIN, assume it's fatal and 2072 * the same callbacks should not be retried. Return CHECK_CONDITION 2073 * if a scsi_status is not already set. 2074 * 2075 * If a fabric driver ->queue_status() has returned non zero, always 2076 * keep retrying no matter what.. 2077 */ 2078 if (cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) { 2079 if (cmd->scsi_status) 2080 goto queue_status; 2081 2082 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; 2083 cmd->scsi_status = SAM_STAT_CHECK_CONDITION; 2084 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 2085 translate_sense_reason(cmd, TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); 2086 goto queue_status; 2087 } 2088 2089 /* 2090 * Check if we need to send a sense buffer from 2091 * the struct se_cmd in question. We do NOT want 2092 * to take this path of the IO has been marked as 2093 * needing to be treated like a "normal read". This 2094 * is the case if it's a tape read, and either the 2095 * FM, EOM, or ILI bits are set, but there is no 2096 * sense data. 2097 */ 2098 if (!(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) && 2099 cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) 2100 goto queue_status; 2101 2102 switch (cmd->data_direction) { 2103 case DMA_FROM_DEVICE: 2104 /* queue status if not treating this as a normal read */ 2105 if (cmd->scsi_status && 2106 !(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL)) 2107 goto queue_status; 2108 2109 trace_target_cmd_complete(cmd); 2110 ret = cmd->se_tfo->queue_data_in(cmd); 2111 break; 2112 case DMA_TO_DEVICE: 2113 if (cmd->se_cmd_flags & SCF_BIDI) { 2114 ret = cmd->se_tfo->queue_data_in(cmd); 2115 break; 2116 } 2117 /* fall through */ 2118 case DMA_NONE: 2119 queue_status: 2120 trace_target_cmd_complete(cmd); 2121 ret = cmd->se_tfo->queue_status(cmd); 2122 break; 2123 default: 2124 break; 2125 } 2126 2127 if (ret < 0) { 2128 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 2129 return; 2130 } 2131 transport_lun_remove_cmd(cmd); 2132 transport_cmd_check_stop_to_fabric(cmd); 2133 } 2134 2135 static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev, 2136 int err, bool write_pending) 2137 { 2138 /* 2139 * -EAGAIN or -ENOMEM signals retry of ->write_pending() and/or 2140 * ->queue_data_in() callbacks from new process context. 2141 * 2142 * Otherwise for other errors, transport_complete_qf() will send 2143 * CHECK_CONDITION via ->queue_status() instead of attempting to 2144 * retry associated fabric driver data-transfer callbacks. 2145 */ 2146 if (err == -EAGAIN || err == -ENOMEM) { 2147 cmd->t_state = (write_pending) ? TRANSPORT_COMPLETE_QF_WP : 2148 TRANSPORT_COMPLETE_QF_OK; 2149 } else { 2150 pr_warn_ratelimited("Got unknown fabric queue status: %d\n", err); 2151 cmd->t_state = TRANSPORT_COMPLETE_QF_ERR; 2152 } 2153 2154 spin_lock_irq(&dev->qf_cmd_lock); 2155 list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list); 2156 atomic_inc_mb(&dev->dev_qf_count); 2157 spin_unlock_irq(&cmd->se_dev->qf_cmd_lock); 2158 2159 schedule_work(&cmd->se_dev->qf_work_queue); 2160 } 2161 2162 static bool target_read_prot_action(struct se_cmd *cmd) 2163 { 2164 switch (cmd->prot_op) { 2165 case TARGET_PROT_DIN_STRIP: 2166 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_STRIP)) { 2167 u32 sectors = cmd->data_length >> 2168 ilog2(cmd->se_dev->dev_attrib.block_size); 2169 2170 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba, 2171 sectors, 0, cmd->t_prot_sg, 2172 0); 2173 if (cmd->pi_err) 2174 return true; 2175 } 2176 break; 2177 case TARGET_PROT_DIN_INSERT: 2178 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_INSERT) 2179 break; 2180 2181 sbc_dif_generate(cmd); 2182 break; 2183 default: 2184 break; 2185 } 2186 2187 return false; 2188 } 2189 2190 static void target_complete_ok_work(struct work_struct *work) 2191 { 2192 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 2193 int ret; 2194 2195 /* 2196 * Check if we need to move delayed/dormant tasks from cmds on the 2197 * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task 2198 * Attribute. 2199 */ 2200 transport_complete_task_attr(cmd); 2201 2202 /* 2203 * Check to schedule QUEUE_FULL work, or execute an existing 2204 * cmd->transport_qf_callback() 2205 */ 2206 if (atomic_read(&cmd->se_dev->dev_qf_count) != 0) 2207 schedule_work(&cmd->se_dev->qf_work_queue); 2208 2209 /* 2210 * Check if we need to send a sense buffer from 2211 * the struct se_cmd in question. We do NOT want 2212 * to take this path of the IO has been marked as 2213 * needing to be treated like a "normal read". This 2214 * is the case if it's a tape read, and either the 2215 * FM, EOM, or ILI bits are set, but there is no 2216 * sense data. 2217 */ 2218 if (!(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) && 2219 cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) { 2220 WARN_ON(!cmd->scsi_status); 2221 ret = transport_send_check_condition_and_sense( 2222 cmd, 0, 1); 2223 if (ret) 2224 goto queue_full; 2225 2226 transport_lun_remove_cmd(cmd); 2227 transport_cmd_check_stop_to_fabric(cmd); 2228 return; 2229 } 2230 /* 2231 * Check for a callback, used by amongst other things 2232 * XDWRITE_READ_10 and COMPARE_AND_WRITE emulation. 2233 */ 2234 if (cmd->transport_complete_callback) { 2235 sense_reason_t rc; 2236 bool caw = (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE); 2237 bool zero_dl = !(cmd->data_length); 2238 int post_ret = 0; 2239 2240 rc = cmd->transport_complete_callback(cmd, true, &post_ret); 2241 if (!rc && !post_ret) { 2242 if (caw && zero_dl) 2243 goto queue_rsp; 2244 2245 return; 2246 } else if (rc) { 2247 ret = transport_send_check_condition_and_sense(cmd, 2248 rc, 0); 2249 if (ret) 2250 goto queue_full; 2251 2252 transport_lun_remove_cmd(cmd); 2253 transport_cmd_check_stop_to_fabric(cmd); 2254 return; 2255 } 2256 } 2257 2258 queue_rsp: 2259 switch (cmd->data_direction) { 2260 case DMA_FROM_DEVICE: 2261 /* 2262 * if this is a READ-type IO, but SCSI status 2263 * is set, then skip returning data and just 2264 * return the status -- unless this IO is marked 2265 * as needing to be treated as a normal read, 2266 * in which case we want to go ahead and return 2267 * the data. This happens, for example, for tape 2268 * reads with the FM, EOM, or ILI bits set, with 2269 * no sense data. 2270 */ 2271 if (cmd->scsi_status && 2272 !(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL)) 2273 goto queue_status; 2274 2275 atomic_long_add(cmd->data_length, 2276 &cmd->se_lun->lun_stats.tx_data_octets); 2277 /* 2278 * Perform READ_STRIP of PI using software emulation when 2279 * backend had PI enabled, if the transport will not be 2280 * performing hardware READ_STRIP offload. 2281 */ 2282 if (target_read_prot_action(cmd)) { 2283 ret = transport_send_check_condition_and_sense(cmd, 2284 cmd->pi_err, 0); 2285 if (ret) 2286 goto queue_full; 2287 2288 transport_lun_remove_cmd(cmd); 2289 transport_cmd_check_stop_to_fabric(cmd); 2290 return; 2291 } 2292 2293 trace_target_cmd_complete(cmd); 2294 ret = cmd->se_tfo->queue_data_in(cmd); 2295 if (ret) 2296 goto queue_full; 2297 break; 2298 case DMA_TO_DEVICE: 2299 atomic_long_add(cmd->data_length, 2300 &cmd->se_lun->lun_stats.rx_data_octets); 2301 /* 2302 * Check if we need to send READ payload for BIDI-COMMAND 2303 */ 2304 if (cmd->se_cmd_flags & SCF_BIDI) { 2305 atomic_long_add(cmd->data_length, 2306 &cmd->se_lun->lun_stats.tx_data_octets); 2307 ret = cmd->se_tfo->queue_data_in(cmd); 2308 if (ret) 2309 goto queue_full; 2310 break; 2311 } 2312 /* fall through */ 2313 case DMA_NONE: 2314 queue_status: 2315 trace_target_cmd_complete(cmd); 2316 ret = cmd->se_tfo->queue_status(cmd); 2317 if (ret) 2318 goto queue_full; 2319 break; 2320 default: 2321 break; 2322 } 2323 2324 transport_lun_remove_cmd(cmd); 2325 transport_cmd_check_stop_to_fabric(cmd); 2326 return; 2327 2328 queue_full: 2329 pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p," 2330 " data_direction: %d\n", cmd, cmd->data_direction); 2331 2332 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 2333 } 2334 2335 void target_free_sgl(struct scatterlist *sgl, int nents) 2336 { 2337 sgl_free_n_order(sgl, nents, 0); 2338 } 2339 EXPORT_SYMBOL(target_free_sgl); 2340 2341 static inline void transport_reset_sgl_orig(struct se_cmd *cmd) 2342 { 2343 /* 2344 * Check for saved t_data_sg that may be used for COMPARE_AND_WRITE 2345 * emulation, and free + reset pointers if necessary.. 2346 */ 2347 if (!cmd->t_data_sg_orig) 2348 return; 2349 2350 kfree(cmd->t_data_sg); 2351 cmd->t_data_sg = cmd->t_data_sg_orig; 2352 cmd->t_data_sg_orig = NULL; 2353 cmd->t_data_nents = cmd->t_data_nents_orig; 2354 cmd->t_data_nents_orig = 0; 2355 } 2356 2357 static inline void transport_free_pages(struct se_cmd *cmd) 2358 { 2359 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) { 2360 target_free_sgl(cmd->t_prot_sg, cmd->t_prot_nents); 2361 cmd->t_prot_sg = NULL; 2362 cmd->t_prot_nents = 0; 2363 } 2364 2365 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 2366 /* 2367 * Release special case READ buffer payload required for 2368 * SG_TO_MEM_NOALLOC to function with COMPARE_AND_WRITE 2369 */ 2370 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) { 2371 target_free_sgl(cmd->t_bidi_data_sg, 2372 cmd->t_bidi_data_nents); 2373 cmd->t_bidi_data_sg = NULL; 2374 cmd->t_bidi_data_nents = 0; 2375 } 2376 transport_reset_sgl_orig(cmd); 2377 return; 2378 } 2379 transport_reset_sgl_orig(cmd); 2380 2381 target_free_sgl(cmd->t_data_sg, cmd->t_data_nents); 2382 cmd->t_data_sg = NULL; 2383 cmd->t_data_nents = 0; 2384 2385 target_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents); 2386 cmd->t_bidi_data_sg = NULL; 2387 cmd->t_bidi_data_nents = 0; 2388 } 2389 2390 void *transport_kmap_data_sg(struct se_cmd *cmd) 2391 { 2392 struct scatterlist *sg = cmd->t_data_sg; 2393 struct page **pages; 2394 int i; 2395 2396 /* 2397 * We need to take into account a possible offset here for fabrics like 2398 * tcm_loop who may be using a contig buffer from the SCSI midlayer for 2399 * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd() 2400 */ 2401 if (!cmd->t_data_nents) 2402 return NULL; 2403 2404 BUG_ON(!sg); 2405 if (cmd->t_data_nents == 1) 2406 return kmap(sg_page(sg)) + sg->offset; 2407 2408 /* >1 page. use vmap */ 2409 pages = kmalloc_array(cmd->t_data_nents, sizeof(*pages), GFP_KERNEL); 2410 if (!pages) 2411 return NULL; 2412 2413 /* convert sg[] to pages[] */ 2414 for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) { 2415 pages[i] = sg_page(sg); 2416 } 2417 2418 cmd->t_data_vmap = vmap(pages, cmd->t_data_nents, VM_MAP, PAGE_KERNEL); 2419 kfree(pages); 2420 if (!cmd->t_data_vmap) 2421 return NULL; 2422 2423 return cmd->t_data_vmap + cmd->t_data_sg[0].offset; 2424 } 2425 EXPORT_SYMBOL(transport_kmap_data_sg); 2426 2427 void transport_kunmap_data_sg(struct se_cmd *cmd) 2428 { 2429 if (!cmd->t_data_nents) { 2430 return; 2431 } else if (cmd->t_data_nents == 1) { 2432 kunmap(sg_page(cmd->t_data_sg)); 2433 return; 2434 } 2435 2436 vunmap(cmd->t_data_vmap); 2437 cmd->t_data_vmap = NULL; 2438 } 2439 EXPORT_SYMBOL(transport_kunmap_data_sg); 2440 2441 int 2442 target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length, 2443 bool zero_page, bool chainable) 2444 { 2445 gfp_t gfp = GFP_KERNEL | (zero_page ? __GFP_ZERO : 0); 2446 2447 *sgl = sgl_alloc_order(length, 0, chainable, gfp, nents); 2448 return *sgl ? 0 : -ENOMEM; 2449 } 2450 EXPORT_SYMBOL(target_alloc_sgl); 2451 2452 /* 2453 * Allocate any required resources to execute the command. For writes we 2454 * might not have the payload yet, so notify the fabric via a call to 2455 * ->write_pending instead. Otherwise place it on the execution queue. 2456 */ 2457 sense_reason_t 2458 transport_generic_new_cmd(struct se_cmd *cmd) 2459 { 2460 unsigned long flags; 2461 int ret = 0; 2462 bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB); 2463 2464 if (cmd->prot_op != TARGET_PROT_NORMAL && 2465 !(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) { 2466 ret = target_alloc_sgl(&cmd->t_prot_sg, &cmd->t_prot_nents, 2467 cmd->prot_length, true, false); 2468 if (ret < 0) 2469 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2470 } 2471 2472 /* 2473 * Determine is the TCM fabric module has already allocated physical 2474 * memory, and is directly calling transport_generic_map_mem_to_cmd() 2475 * beforehand. 2476 */ 2477 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) && 2478 cmd->data_length) { 2479 2480 if ((cmd->se_cmd_flags & SCF_BIDI) || 2481 (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)) { 2482 u32 bidi_length; 2483 2484 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) 2485 bidi_length = cmd->t_task_nolb * 2486 cmd->se_dev->dev_attrib.block_size; 2487 else 2488 bidi_length = cmd->data_length; 2489 2490 ret = target_alloc_sgl(&cmd->t_bidi_data_sg, 2491 &cmd->t_bidi_data_nents, 2492 bidi_length, zero_flag, false); 2493 if (ret < 0) 2494 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2495 } 2496 2497 ret = target_alloc_sgl(&cmd->t_data_sg, &cmd->t_data_nents, 2498 cmd->data_length, zero_flag, false); 2499 if (ret < 0) 2500 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2501 } else if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) && 2502 cmd->data_length) { 2503 /* 2504 * Special case for COMPARE_AND_WRITE with fabrics 2505 * using SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC. 2506 */ 2507 u32 caw_length = cmd->t_task_nolb * 2508 cmd->se_dev->dev_attrib.block_size; 2509 2510 ret = target_alloc_sgl(&cmd->t_bidi_data_sg, 2511 &cmd->t_bidi_data_nents, 2512 caw_length, zero_flag, false); 2513 if (ret < 0) 2514 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2515 } 2516 /* 2517 * If this command is not a write we can execute it right here, 2518 * for write buffers we need to notify the fabric driver first 2519 * and let it call back once the write buffers are ready. 2520 */ 2521 target_add_to_state_list(cmd); 2522 if (cmd->data_direction != DMA_TO_DEVICE || cmd->data_length == 0) { 2523 target_execute_cmd(cmd); 2524 return 0; 2525 } 2526 2527 spin_lock_irqsave(&cmd->t_state_lock, flags); 2528 cmd->t_state = TRANSPORT_WRITE_PENDING; 2529 /* 2530 * Determine if frontend context caller is requesting the stopping of 2531 * this command for frontend exceptions. 2532 */ 2533 if (cmd->transport_state & CMD_T_STOP) { 2534 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 2535 __func__, __LINE__, cmd->tag); 2536 2537 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2538 2539 complete_all(&cmd->t_transport_stop_comp); 2540 return 0; 2541 } 2542 cmd->transport_state &= ~CMD_T_ACTIVE; 2543 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2544 2545 ret = cmd->se_tfo->write_pending(cmd); 2546 if (ret) 2547 goto queue_full; 2548 2549 return 0; 2550 2551 queue_full: 2552 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd); 2553 transport_handle_queue_full(cmd, cmd->se_dev, ret, true); 2554 return 0; 2555 } 2556 EXPORT_SYMBOL(transport_generic_new_cmd); 2557 2558 static void transport_write_pending_qf(struct se_cmd *cmd) 2559 { 2560 unsigned long flags; 2561 int ret; 2562 bool stop; 2563 2564 spin_lock_irqsave(&cmd->t_state_lock, flags); 2565 stop = (cmd->transport_state & (CMD_T_STOP | CMD_T_ABORTED)); 2566 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2567 2568 if (stop) { 2569 pr_debug("%s:%d CMD_T_STOP|CMD_T_ABORTED for ITT: 0x%08llx\n", 2570 __func__, __LINE__, cmd->tag); 2571 complete_all(&cmd->t_transport_stop_comp); 2572 return; 2573 } 2574 2575 ret = cmd->se_tfo->write_pending(cmd); 2576 if (ret) { 2577 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", 2578 cmd); 2579 transport_handle_queue_full(cmd, cmd->se_dev, ret, true); 2580 } 2581 } 2582 2583 static bool 2584 __transport_wait_for_tasks(struct se_cmd *, bool, bool *, bool *, 2585 unsigned long *flags); 2586 2587 static void target_wait_free_cmd(struct se_cmd *cmd, bool *aborted, bool *tas) 2588 { 2589 unsigned long flags; 2590 2591 spin_lock_irqsave(&cmd->t_state_lock, flags); 2592 __transport_wait_for_tasks(cmd, true, aborted, tas, &flags); 2593 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2594 } 2595 2596 int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks) 2597 { 2598 int ret = 0; 2599 bool aborted = false, tas = false; 2600 2601 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) { 2602 if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2603 target_wait_free_cmd(cmd, &aborted, &tas); 2604 2605 if (!aborted || tas) 2606 ret = target_put_sess_cmd(cmd); 2607 } else { 2608 if (wait_for_tasks) 2609 target_wait_free_cmd(cmd, &aborted, &tas); 2610 /* 2611 * Handle WRITE failure case where transport_generic_new_cmd() 2612 * has already added se_cmd to state_list, but fabric has 2613 * failed command before I/O submission. 2614 */ 2615 if (cmd->state_active) 2616 target_remove_from_state_list(cmd); 2617 2618 if (cmd->se_lun) 2619 transport_lun_remove_cmd(cmd); 2620 2621 if (!aborted || tas) 2622 ret = target_put_sess_cmd(cmd); 2623 } 2624 /* 2625 * If the task has been internally aborted due to TMR ABORT_TASK 2626 * or LUN_RESET, target_core_tmr.c is responsible for performing 2627 * the remaining calls to target_put_sess_cmd(), and not the 2628 * callers of this function. 2629 */ 2630 if (aborted) { 2631 pr_debug("Detected CMD_T_ABORTED for ITT: %llu\n", cmd->tag); 2632 wait_for_completion(&cmd->cmd_wait_comp); 2633 cmd->se_tfo->release_cmd(cmd); 2634 ret = 1; 2635 } 2636 return ret; 2637 } 2638 EXPORT_SYMBOL(transport_generic_free_cmd); 2639 2640 /** 2641 * target_get_sess_cmd - Add command to active ->sess_cmd_list 2642 * @se_cmd: command descriptor to add 2643 * @ack_kref: Signal that fabric will perform an ack target_put_sess_cmd() 2644 */ 2645 int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref) 2646 { 2647 struct se_session *se_sess = se_cmd->se_sess; 2648 unsigned long flags; 2649 int ret = 0; 2650 2651 /* 2652 * Add a second kref if the fabric caller is expecting to handle 2653 * fabric acknowledgement that requires two target_put_sess_cmd() 2654 * invocations before se_cmd descriptor release. 2655 */ 2656 if (ack_kref) { 2657 if (!kref_get_unless_zero(&se_cmd->cmd_kref)) 2658 return -EINVAL; 2659 2660 se_cmd->se_cmd_flags |= SCF_ACK_KREF; 2661 } 2662 2663 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2664 if (se_sess->sess_tearing_down) { 2665 ret = -ESHUTDOWN; 2666 goto out; 2667 } 2668 se_cmd->transport_state |= CMD_T_PRE_EXECUTE; 2669 list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list); 2670 out: 2671 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2672 2673 if (ret && ack_kref) 2674 target_put_sess_cmd(se_cmd); 2675 2676 return ret; 2677 } 2678 EXPORT_SYMBOL(target_get_sess_cmd); 2679 2680 static void target_free_cmd_mem(struct se_cmd *cmd) 2681 { 2682 transport_free_pages(cmd); 2683 2684 if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) 2685 core_tmr_release_req(cmd->se_tmr_req); 2686 if (cmd->t_task_cdb != cmd->__t_task_cdb) 2687 kfree(cmd->t_task_cdb); 2688 } 2689 2690 static void target_release_cmd_kref(struct kref *kref) 2691 { 2692 struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref); 2693 struct se_session *se_sess = se_cmd->se_sess; 2694 unsigned long flags; 2695 bool fabric_stop; 2696 2697 if (se_sess) { 2698 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2699 2700 spin_lock(&se_cmd->t_state_lock); 2701 fabric_stop = (se_cmd->transport_state & CMD_T_FABRIC_STOP) && 2702 (se_cmd->transport_state & CMD_T_ABORTED); 2703 spin_unlock(&se_cmd->t_state_lock); 2704 2705 if (se_cmd->cmd_wait_set || fabric_stop) { 2706 list_del_init(&se_cmd->se_cmd_list); 2707 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2708 target_free_cmd_mem(se_cmd); 2709 complete(&se_cmd->cmd_wait_comp); 2710 return; 2711 } 2712 list_del_init(&se_cmd->se_cmd_list); 2713 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2714 } 2715 2716 target_free_cmd_mem(se_cmd); 2717 se_cmd->se_tfo->release_cmd(se_cmd); 2718 } 2719 2720 /** 2721 * target_put_sess_cmd - decrease the command reference count 2722 * @se_cmd: command to drop a reference from 2723 * 2724 * Returns 1 if and only if this target_put_sess_cmd() call caused the 2725 * refcount to drop to zero. Returns zero otherwise. 2726 */ 2727 int target_put_sess_cmd(struct se_cmd *se_cmd) 2728 { 2729 return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref); 2730 } 2731 EXPORT_SYMBOL(target_put_sess_cmd); 2732 2733 static const char *data_dir_name(enum dma_data_direction d) 2734 { 2735 switch (d) { 2736 case DMA_BIDIRECTIONAL: return "BIDI"; 2737 case DMA_TO_DEVICE: return "WRITE"; 2738 case DMA_FROM_DEVICE: return "READ"; 2739 case DMA_NONE: return "NONE"; 2740 } 2741 2742 return "(?)"; 2743 } 2744 2745 static const char *cmd_state_name(enum transport_state_table t) 2746 { 2747 switch (t) { 2748 case TRANSPORT_NO_STATE: return "NO_STATE"; 2749 case TRANSPORT_NEW_CMD: return "NEW_CMD"; 2750 case TRANSPORT_WRITE_PENDING: return "WRITE_PENDING"; 2751 case TRANSPORT_PROCESSING: return "PROCESSING"; 2752 case TRANSPORT_COMPLETE: return "COMPLETE"; 2753 case TRANSPORT_ISTATE_PROCESSING: 2754 return "ISTATE_PROCESSING"; 2755 case TRANSPORT_COMPLETE_QF_WP: return "COMPLETE_QF_WP"; 2756 case TRANSPORT_COMPLETE_QF_OK: return "COMPLETE_QF_OK"; 2757 case TRANSPORT_COMPLETE_QF_ERR: return "COMPLETE_QF_ERR"; 2758 } 2759 2760 return "(?)"; 2761 } 2762 2763 static void target_append_str(char **str, const char *txt) 2764 { 2765 char *prev = *str; 2766 2767 *str = *str ? kasprintf(GFP_ATOMIC, "%s,%s", *str, txt) : 2768 kstrdup(txt, GFP_ATOMIC); 2769 kfree(prev); 2770 } 2771 2772 /* 2773 * Convert a transport state bitmask into a string. The caller is 2774 * responsible for freeing the returned pointer. 2775 */ 2776 static char *target_ts_to_str(u32 ts) 2777 { 2778 char *str = NULL; 2779 2780 if (ts & CMD_T_ABORTED) 2781 target_append_str(&str, "aborted"); 2782 if (ts & CMD_T_ACTIVE) 2783 target_append_str(&str, "active"); 2784 if (ts & CMD_T_COMPLETE) 2785 target_append_str(&str, "complete"); 2786 if (ts & CMD_T_SENT) 2787 target_append_str(&str, "sent"); 2788 if (ts & CMD_T_STOP) 2789 target_append_str(&str, "stop"); 2790 if (ts & CMD_T_FABRIC_STOP) 2791 target_append_str(&str, "fabric_stop"); 2792 2793 return str; 2794 } 2795 2796 static const char *target_tmf_name(enum tcm_tmreq_table tmf) 2797 { 2798 switch (tmf) { 2799 case TMR_ABORT_TASK: return "ABORT_TASK"; 2800 case TMR_ABORT_TASK_SET: return "ABORT_TASK_SET"; 2801 case TMR_CLEAR_ACA: return "CLEAR_ACA"; 2802 case TMR_CLEAR_TASK_SET: return "CLEAR_TASK_SET"; 2803 case TMR_LUN_RESET: return "LUN_RESET"; 2804 case TMR_TARGET_WARM_RESET: return "TARGET_WARM_RESET"; 2805 case TMR_TARGET_COLD_RESET: return "TARGET_COLD_RESET"; 2806 case TMR_UNKNOWN: break; 2807 } 2808 return "(?)"; 2809 } 2810 2811 void target_show_cmd(const char *pfx, struct se_cmd *cmd) 2812 { 2813 char *ts_str = target_ts_to_str(cmd->transport_state); 2814 const u8 *cdb = cmd->t_task_cdb; 2815 struct se_tmr_req *tmf = cmd->se_tmr_req; 2816 2817 if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) { 2818 pr_debug("%scmd %#02x:%#02x with tag %#llx dir %s i_state %d t_state %s len %d refcnt %d transport_state %s\n", 2819 pfx, cdb[0], cdb[1], cmd->tag, 2820 data_dir_name(cmd->data_direction), 2821 cmd->se_tfo->get_cmd_state(cmd), 2822 cmd_state_name(cmd->t_state), cmd->data_length, 2823 kref_read(&cmd->cmd_kref), ts_str); 2824 } else { 2825 pr_debug("%stmf %s with tag %#llx ref_task_tag %#llx i_state %d t_state %s refcnt %d transport_state %s\n", 2826 pfx, target_tmf_name(tmf->function), cmd->tag, 2827 tmf->ref_task_tag, cmd->se_tfo->get_cmd_state(cmd), 2828 cmd_state_name(cmd->t_state), 2829 kref_read(&cmd->cmd_kref), ts_str); 2830 } 2831 kfree(ts_str); 2832 } 2833 EXPORT_SYMBOL(target_show_cmd); 2834 2835 /** 2836 * target_sess_cmd_list_set_waiting - Flag all commands in 2837 * sess_cmd_list to complete cmd_wait_comp. Set 2838 * sess_tearing_down so no more commands are queued. 2839 * @se_sess: session to flag 2840 */ 2841 void target_sess_cmd_list_set_waiting(struct se_session *se_sess) 2842 { 2843 struct se_cmd *se_cmd, *tmp_cmd; 2844 unsigned long flags; 2845 int rc; 2846 2847 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2848 if (se_sess->sess_tearing_down) { 2849 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2850 return; 2851 } 2852 se_sess->sess_tearing_down = 1; 2853 list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list); 2854 2855 list_for_each_entry_safe(se_cmd, tmp_cmd, 2856 &se_sess->sess_wait_list, se_cmd_list) { 2857 rc = kref_get_unless_zero(&se_cmd->cmd_kref); 2858 if (rc) { 2859 se_cmd->cmd_wait_set = 1; 2860 spin_lock(&se_cmd->t_state_lock); 2861 se_cmd->transport_state |= CMD_T_FABRIC_STOP; 2862 spin_unlock(&se_cmd->t_state_lock); 2863 } else 2864 list_del_init(&se_cmd->se_cmd_list); 2865 } 2866 2867 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2868 } 2869 EXPORT_SYMBOL(target_sess_cmd_list_set_waiting); 2870 2871 /** 2872 * target_wait_for_sess_cmds - Wait for outstanding descriptors 2873 * @se_sess: session to wait for active I/O 2874 */ 2875 void target_wait_for_sess_cmds(struct se_session *se_sess) 2876 { 2877 struct se_cmd *se_cmd, *tmp_cmd; 2878 unsigned long flags; 2879 bool tas; 2880 2881 list_for_each_entry_safe(se_cmd, tmp_cmd, 2882 &se_sess->sess_wait_list, se_cmd_list) { 2883 pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:" 2884 " %d\n", se_cmd, se_cmd->t_state, 2885 se_cmd->se_tfo->get_cmd_state(se_cmd)); 2886 2887 spin_lock_irqsave(&se_cmd->t_state_lock, flags); 2888 tas = (se_cmd->transport_state & CMD_T_TAS); 2889 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); 2890 2891 if (!target_put_sess_cmd(se_cmd)) { 2892 if (tas) 2893 target_put_sess_cmd(se_cmd); 2894 } 2895 2896 wait_for_completion(&se_cmd->cmd_wait_comp); 2897 pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d" 2898 " fabric state: %d\n", se_cmd, se_cmd->t_state, 2899 se_cmd->se_tfo->get_cmd_state(se_cmd)); 2900 2901 se_cmd->se_tfo->release_cmd(se_cmd); 2902 } 2903 2904 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2905 WARN_ON(!list_empty(&se_sess->sess_cmd_list)); 2906 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2907 2908 } 2909 EXPORT_SYMBOL(target_wait_for_sess_cmds); 2910 2911 static void target_lun_confirm(struct percpu_ref *ref) 2912 { 2913 struct se_lun *lun = container_of(ref, struct se_lun, lun_ref); 2914 2915 complete(&lun->lun_ref_comp); 2916 } 2917 2918 void transport_clear_lun_ref(struct se_lun *lun) 2919 { 2920 /* 2921 * Mark the percpu-ref as DEAD, switch to atomic_t mode, drop 2922 * the initial reference and schedule confirm kill to be 2923 * executed after one full RCU grace period has completed. 2924 */ 2925 percpu_ref_kill_and_confirm(&lun->lun_ref, target_lun_confirm); 2926 /* 2927 * The first completion waits for percpu_ref_switch_to_atomic_rcu() 2928 * to call target_lun_confirm after lun->lun_ref has been marked 2929 * as __PERCPU_REF_DEAD on all CPUs, and switches to atomic_t 2930 * mode so that percpu_ref_tryget_live() lookup of lun->lun_ref 2931 * fails for all new incoming I/O. 2932 */ 2933 wait_for_completion(&lun->lun_ref_comp); 2934 /* 2935 * The second completion waits for percpu_ref_put_many() to 2936 * invoke ->release() after lun->lun_ref has switched to 2937 * atomic_t mode, and lun->lun_ref.count has reached zero. 2938 * 2939 * At this point all target-core lun->lun_ref references have 2940 * been dropped via transport_lun_remove_cmd(), and it's safe 2941 * to proceed with the remaining LUN shutdown. 2942 */ 2943 wait_for_completion(&lun->lun_shutdown_comp); 2944 } 2945 2946 static bool 2947 __transport_wait_for_tasks(struct se_cmd *cmd, bool fabric_stop, 2948 bool *aborted, bool *tas, unsigned long *flags) 2949 __releases(&cmd->t_state_lock) 2950 __acquires(&cmd->t_state_lock) 2951 { 2952 2953 assert_spin_locked(&cmd->t_state_lock); 2954 WARN_ON_ONCE(!irqs_disabled()); 2955 2956 if (fabric_stop) 2957 cmd->transport_state |= CMD_T_FABRIC_STOP; 2958 2959 if (cmd->transport_state & CMD_T_ABORTED) 2960 *aborted = true; 2961 2962 if (cmd->transport_state & CMD_T_TAS) 2963 *tas = true; 2964 2965 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) && 2966 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2967 return false; 2968 2969 if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) && 2970 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2971 return false; 2972 2973 if (!(cmd->transport_state & CMD_T_ACTIVE)) 2974 return false; 2975 2976 if (fabric_stop && *aborted) 2977 return false; 2978 2979 cmd->transport_state |= CMD_T_STOP; 2980 2981 target_show_cmd("wait_for_tasks: Stopping ", cmd); 2982 2983 spin_unlock_irqrestore(&cmd->t_state_lock, *flags); 2984 2985 while (!wait_for_completion_timeout(&cmd->t_transport_stop_comp, 2986 180 * HZ)) 2987 target_show_cmd("wait for tasks: ", cmd); 2988 2989 spin_lock_irqsave(&cmd->t_state_lock, *flags); 2990 cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP); 2991 2992 pr_debug("wait_for_tasks: Stopped wait_for_completion(&cmd->" 2993 "t_transport_stop_comp) for ITT: 0x%08llx\n", cmd->tag); 2994 2995 return true; 2996 } 2997 2998 /** 2999 * transport_wait_for_tasks - set CMD_T_STOP and wait for t_transport_stop_comp 3000 * @cmd: command to wait on 3001 */ 3002 bool transport_wait_for_tasks(struct se_cmd *cmd) 3003 { 3004 unsigned long flags; 3005 bool ret, aborted = false, tas = false; 3006 3007 spin_lock_irqsave(&cmd->t_state_lock, flags); 3008 ret = __transport_wait_for_tasks(cmd, false, &aborted, &tas, &flags); 3009 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3010 3011 return ret; 3012 } 3013 EXPORT_SYMBOL(transport_wait_for_tasks); 3014 3015 struct sense_info { 3016 u8 key; 3017 u8 asc; 3018 u8 ascq; 3019 bool add_sector_info; 3020 }; 3021 3022 static const struct sense_info sense_info_table[] = { 3023 [TCM_NO_SENSE] = { 3024 .key = NOT_READY 3025 }, 3026 [TCM_NON_EXISTENT_LUN] = { 3027 .key = ILLEGAL_REQUEST, 3028 .asc = 0x25 /* LOGICAL UNIT NOT SUPPORTED */ 3029 }, 3030 [TCM_UNSUPPORTED_SCSI_OPCODE] = { 3031 .key = ILLEGAL_REQUEST, 3032 .asc = 0x20, /* INVALID COMMAND OPERATION CODE */ 3033 }, 3034 [TCM_SECTOR_COUNT_TOO_MANY] = { 3035 .key = ILLEGAL_REQUEST, 3036 .asc = 0x20, /* INVALID COMMAND OPERATION CODE */ 3037 }, 3038 [TCM_UNKNOWN_MODE_PAGE] = { 3039 .key = ILLEGAL_REQUEST, 3040 .asc = 0x24, /* INVALID FIELD IN CDB */ 3041 }, 3042 [TCM_CHECK_CONDITION_ABORT_CMD] = { 3043 .key = ABORTED_COMMAND, 3044 .asc = 0x29, /* BUS DEVICE RESET FUNCTION OCCURRED */ 3045 .ascq = 0x03, 3046 }, 3047 [TCM_INCORRECT_AMOUNT_OF_DATA] = { 3048 .key = ABORTED_COMMAND, 3049 .asc = 0x0c, /* WRITE ERROR */ 3050 .ascq = 0x0d, /* NOT ENOUGH UNSOLICITED DATA */ 3051 }, 3052 [TCM_INVALID_CDB_FIELD] = { 3053 .key = ILLEGAL_REQUEST, 3054 .asc = 0x24, /* INVALID FIELD IN CDB */ 3055 }, 3056 [TCM_INVALID_PARAMETER_LIST] = { 3057 .key = ILLEGAL_REQUEST, 3058 .asc = 0x26, /* INVALID FIELD IN PARAMETER LIST */ 3059 }, 3060 [TCM_TOO_MANY_TARGET_DESCS] = { 3061 .key = ILLEGAL_REQUEST, 3062 .asc = 0x26, 3063 .ascq = 0x06, /* TOO MANY TARGET DESCRIPTORS */ 3064 }, 3065 [TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE] = { 3066 .key = ILLEGAL_REQUEST, 3067 .asc = 0x26, 3068 .ascq = 0x07, /* UNSUPPORTED TARGET DESCRIPTOR TYPE CODE */ 3069 }, 3070 [TCM_TOO_MANY_SEGMENT_DESCS] = { 3071 .key = ILLEGAL_REQUEST, 3072 .asc = 0x26, 3073 .ascq = 0x08, /* TOO MANY SEGMENT DESCRIPTORS */ 3074 }, 3075 [TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE] = { 3076 .key = ILLEGAL_REQUEST, 3077 .asc = 0x26, 3078 .ascq = 0x09, /* UNSUPPORTED SEGMENT DESCRIPTOR TYPE CODE */ 3079 }, 3080 [TCM_PARAMETER_LIST_LENGTH_ERROR] = { 3081 .key = ILLEGAL_REQUEST, 3082 .asc = 0x1a, /* PARAMETER LIST LENGTH ERROR */ 3083 }, 3084 [TCM_UNEXPECTED_UNSOLICITED_DATA] = { 3085 .key = ILLEGAL_REQUEST, 3086 .asc = 0x0c, /* WRITE ERROR */ 3087 .ascq = 0x0c, /* UNEXPECTED_UNSOLICITED_DATA */ 3088 }, 3089 [TCM_SERVICE_CRC_ERROR] = { 3090 .key = ABORTED_COMMAND, 3091 .asc = 0x47, /* PROTOCOL SERVICE CRC ERROR */ 3092 .ascq = 0x05, /* N/A */ 3093 }, 3094 [TCM_SNACK_REJECTED] = { 3095 .key = ABORTED_COMMAND, 3096 .asc = 0x11, /* READ ERROR */ 3097 .ascq = 0x13, /* FAILED RETRANSMISSION REQUEST */ 3098 }, 3099 [TCM_WRITE_PROTECTED] = { 3100 .key = DATA_PROTECT, 3101 .asc = 0x27, /* WRITE PROTECTED */ 3102 }, 3103 [TCM_ADDRESS_OUT_OF_RANGE] = { 3104 .key = ILLEGAL_REQUEST, 3105 .asc = 0x21, /* LOGICAL BLOCK ADDRESS OUT OF RANGE */ 3106 }, 3107 [TCM_CHECK_CONDITION_UNIT_ATTENTION] = { 3108 .key = UNIT_ATTENTION, 3109 }, 3110 [TCM_CHECK_CONDITION_NOT_READY] = { 3111 .key = NOT_READY, 3112 }, 3113 [TCM_MISCOMPARE_VERIFY] = { 3114 .key = MISCOMPARE, 3115 .asc = 0x1d, /* MISCOMPARE DURING VERIFY OPERATION */ 3116 .ascq = 0x00, 3117 }, 3118 [TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED] = { 3119 .key = ABORTED_COMMAND, 3120 .asc = 0x10, 3121 .ascq = 0x01, /* LOGICAL BLOCK GUARD CHECK FAILED */ 3122 .add_sector_info = true, 3123 }, 3124 [TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED] = { 3125 .key = ABORTED_COMMAND, 3126 .asc = 0x10, 3127 .ascq = 0x02, /* LOGICAL BLOCK APPLICATION TAG CHECK FAILED */ 3128 .add_sector_info = true, 3129 }, 3130 [TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED] = { 3131 .key = ABORTED_COMMAND, 3132 .asc = 0x10, 3133 .ascq = 0x03, /* LOGICAL BLOCK REFERENCE TAG CHECK FAILED */ 3134 .add_sector_info = true, 3135 }, 3136 [TCM_COPY_TARGET_DEVICE_NOT_REACHABLE] = { 3137 .key = COPY_ABORTED, 3138 .asc = 0x0d, 3139 .ascq = 0x02, /* COPY TARGET DEVICE NOT REACHABLE */ 3140 3141 }, 3142 [TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE] = { 3143 /* 3144 * Returning ILLEGAL REQUEST would cause immediate IO errors on 3145 * Solaris initiators. Returning NOT READY instead means the 3146 * operations will be retried a finite number of times and we 3147 * can survive intermittent errors. 3148 */ 3149 .key = NOT_READY, 3150 .asc = 0x08, /* LOGICAL UNIT COMMUNICATION FAILURE */ 3151 }, 3152 [TCM_INSUFFICIENT_REGISTRATION_RESOURCES] = { 3153 /* 3154 * From spc4r22 section5.7.7,5.7.8 3155 * If a PERSISTENT RESERVE OUT command with a REGISTER service action 3156 * or a REGISTER AND IGNORE EXISTING KEY service action or 3157 * REGISTER AND MOVE service actionis attempted, 3158 * but there are insufficient device server resources to complete the 3159 * operation, then the command shall be terminated with CHECK CONDITION 3160 * status, with the sense key set to ILLEGAL REQUEST,and the additonal 3161 * sense code set to INSUFFICIENT REGISTRATION RESOURCES. 3162 */ 3163 .key = ILLEGAL_REQUEST, 3164 .asc = 0x55, 3165 .ascq = 0x04, /* INSUFFICIENT REGISTRATION RESOURCES */ 3166 }, 3167 }; 3168 3169 static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason) 3170 { 3171 const struct sense_info *si; 3172 u8 *buffer = cmd->sense_buffer; 3173 int r = (__force int)reason; 3174 u8 asc, ascq; 3175 bool desc_format = target_sense_desc_format(cmd->se_dev); 3176 3177 if (r < ARRAY_SIZE(sense_info_table) && sense_info_table[r].key) 3178 si = &sense_info_table[r]; 3179 else 3180 si = &sense_info_table[(__force int) 3181 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE]; 3182 3183 if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) { 3184 core_scsi3_ua_for_check_condition(cmd, &asc, &ascq); 3185 WARN_ON_ONCE(asc == 0); 3186 } else if (si->asc == 0) { 3187 WARN_ON_ONCE(cmd->scsi_asc == 0); 3188 asc = cmd->scsi_asc; 3189 ascq = cmd->scsi_ascq; 3190 } else { 3191 asc = si->asc; 3192 ascq = si->ascq; 3193 } 3194 3195 scsi_build_sense_buffer(desc_format, buffer, si->key, asc, ascq); 3196 if (si->add_sector_info) 3197 return scsi_set_sense_information(buffer, 3198 cmd->scsi_sense_length, 3199 cmd->bad_sector); 3200 3201 return 0; 3202 } 3203 3204 int 3205 transport_send_check_condition_and_sense(struct se_cmd *cmd, 3206 sense_reason_t reason, int from_transport) 3207 { 3208 unsigned long flags; 3209 3210 spin_lock_irqsave(&cmd->t_state_lock, flags); 3211 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) { 3212 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3213 return 0; 3214 } 3215 cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION; 3216 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3217 3218 if (!from_transport) { 3219 int rc; 3220 3221 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; 3222 cmd->scsi_status = SAM_STAT_CHECK_CONDITION; 3223 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 3224 rc = translate_sense_reason(cmd, reason); 3225 if (rc) 3226 return rc; 3227 } 3228 3229 trace_target_cmd_complete(cmd); 3230 return cmd->se_tfo->queue_status(cmd); 3231 } 3232 EXPORT_SYMBOL(transport_send_check_condition_and_sense); 3233 3234 static int __transport_check_aborted_status(struct se_cmd *cmd, int send_status) 3235 __releases(&cmd->t_state_lock) 3236 __acquires(&cmd->t_state_lock) 3237 { 3238 int ret; 3239 3240 assert_spin_locked(&cmd->t_state_lock); 3241 WARN_ON_ONCE(!irqs_disabled()); 3242 3243 if (!(cmd->transport_state & CMD_T_ABORTED)) 3244 return 0; 3245 /* 3246 * If cmd has been aborted but either no status is to be sent or it has 3247 * already been sent, just return 3248 */ 3249 if (!send_status || !(cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS)) { 3250 if (send_status) 3251 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS; 3252 return 1; 3253 } 3254 3255 pr_debug("Sending delayed SAM_STAT_TASK_ABORTED status for CDB:" 3256 " 0x%02x ITT: 0x%08llx\n", cmd->t_task_cdb[0], cmd->tag); 3257 3258 cmd->se_cmd_flags &= ~SCF_SEND_DELAYED_TAS; 3259 cmd->scsi_status = SAM_STAT_TASK_ABORTED; 3260 trace_target_cmd_complete(cmd); 3261 3262 spin_unlock_irq(&cmd->t_state_lock); 3263 ret = cmd->se_tfo->queue_status(cmd); 3264 if (ret) 3265 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 3266 spin_lock_irq(&cmd->t_state_lock); 3267 3268 return 1; 3269 } 3270 3271 int transport_check_aborted_status(struct se_cmd *cmd, int send_status) 3272 { 3273 int ret; 3274 3275 spin_lock_irq(&cmd->t_state_lock); 3276 ret = __transport_check_aborted_status(cmd, send_status); 3277 spin_unlock_irq(&cmd->t_state_lock); 3278 3279 return ret; 3280 } 3281 EXPORT_SYMBOL(transport_check_aborted_status); 3282 3283 void transport_send_task_abort(struct se_cmd *cmd) 3284 { 3285 unsigned long flags; 3286 int ret; 3287 3288 spin_lock_irqsave(&cmd->t_state_lock, flags); 3289 if (cmd->se_cmd_flags & (SCF_SENT_CHECK_CONDITION)) { 3290 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3291 return; 3292 } 3293 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3294 3295 /* 3296 * If there are still expected incoming fabric WRITEs, we wait 3297 * until until they have completed before sending a TASK_ABORTED 3298 * response. This response with TASK_ABORTED status will be 3299 * queued back to fabric module by transport_check_aborted_status(). 3300 */ 3301 if (cmd->data_direction == DMA_TO_DEVICE) { 3302 if (cmd->se_tfo->write_pending_status(cmd) != 0) { 3303 spin_lock_irqsave(&cmd->t_state_lock, flags); 3304 if (cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS) { 3305 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3306 goto send_abort; 3307 } 3308 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS; 3309 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3310 return; 3311 } 3312 } 3313 send_abort: 3314 cmd->scsi_status = SAM_STAT_TASK_ABORTED; 3315 3316 transport_lun_remove_cmd(cmd); 3317 3318 pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n", 3319 cmd->t_task_cdb[0], cmd->tag); 3320 3321 trace_target_cmd_complete(cmd); 3322 ret = cmd->se_tfo->queue_status(cmd); 3323 if (ret) 3324 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 3325 } 3326 3327 static void target_tmr_work(struct work_struct *work) 3328 { 3329 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 3330 struct se_device *dev = cmd->se_dev; 3331 struct se_tmr_req *tmr = cmd->se_tmr_req; 3332 unsigned long flags; 3333 int ret; 3334 3335 spin_lock_irqsave(&cmd->t_state_lock, flags); 3336 if (cmd->transport_state & CMD_T_ABORTED) { 3337 tmr->response = TMR_FUNCTION_REJECTED; 3338 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3339 goto check_stop; 3340 } 3341 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3342 3343 switch (tmr->function) { 3344 case TMR_ABORT_TASK: 3345 core_tmr_abort_task(dev, tmr, cmd->se_sess); 3346 break; 3347 case TMR_ABORT_TASK_SET: 3348 case TMR_CLEAR_ACA: 3349 case TMR_CLEAR_TASK_SET: 3350 tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED; 3351 break; 3352 case TMR_LUN_RESET: 3353 ret = core_tmr_lun_reset(dev, tmr, NULL, NULL); 3354 tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE : 3355 TMR_FUNCTION_REJECTED; 3356 if (tmr->response == TMR_FUNCTION_COMPLETE) { 3357 target_ua_allocate_lun(cmd->se_sess->se_node_acl, 3358 cmd->orig_fe_lun, 0x29, 3359 ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED); 3360 } 3361 break; 3362 case TMR_TARGET_WARM_RESET: 3363 tmr->response = TMR_FUNCTION_REJECTED; 3364 break; 3365 case TMR_TARGET_COLD_RESET: 3366 tmr->response = TMR_FUNCTION_REJECTED; 3367 break; 3368 default: 3369 pr_err("Unknown TMR function: 0x%02x.\n", 3370 tmr->function); 3371 tmr->response = TMR_FUNCTION_REJECTED; 3372 break; 3373 } 3374 3375 spin_lock_irqsave(&cmd->t_state_lock, flags); 3376 if (cmd->transport_state & CMD_T_ABORTED) { 3377 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3378 goto check_stop; 3379 } 3380 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3381 3382 cmd->se_tfo->queue_tm_rsp(cmd); 3383 3384 check_stop: 3385 transport_lun_remove_cmd(cmd); 3386 transport_cmd_check_stop_to_fabric(cmd); 3387 } 3388 3389 int transport_generic_handle_tmr( 3390 struct se_cmd *cmd) 3391 { 3392 unsigned long flags; 3393 bool aborted = false; 3394 3395 spin_lock_irqsave(&cmd->t_state_lock, flags); 3396 if (cmd->transport_state & CMD_T_ABORTED) { 3397 aborted = true; 3398 } else { 3399 cmd->t_state = TRANSPORT_ISTATE_PROCESSING; 3400 cmd->transport_state |= CMD_T_ACTIVE; 3401 } 3402 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3403 3404 if (aborted) { 3405 pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d" 3406 "ref_tag: %llu tag: %llu\n", cmd->se_tmr_req->function, 3407 cmd->se_tmr_req->ref_task_tag, cmd->tag); 3408 transport_lun_remove_cmd(cmd); 3409 transport_cmd_check_stop_to_fabric(cmd); 3410 return 0; 3411 } 3412 3413 INIT_WORK(&cmd->work, target_tmr_work); 3414 queue_work(cmd->se_dev->tmr_wq, &cmd->work); 3415 return 0; 3416 } 3417 EXPORT_SYMBOL(transport_generic_handle_tmr); 3418 3419 bool 3420 target_check_wce(struct se_device *dev) 3421 { 3422 bool wce = false; 3423 3424 if (dev->transport->get_write_cache) 3425 wce = dev->transport->get_write_cache(dev); 3426 else if (dev->dev_attrib.emulate_write_cache > 0) 3427 wce = true; 3428 3429 return wce; 3430 } 3431 3432 bool 3433 target_check_fua(struct se_device *dev) 3434 { 3435 return target_check_wce(dev) && dev->dev_attrib.emulate_fua_write > 0; 3436 } 3437