1 /******************************************************************************* 2 * Filename: target_core_transport.c 3 * 4 * This file contains the Generic Target Engine Core. 5 * 6 * (c) Copyright 2002-2013 Datera, Inc. 7 * 8 * Nicholas A. Bellinger <nab@kernel.org> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License as published by 12 * the Free Software Foundation; either version 2 of the License, or 13 * (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 * 20 * You should have received a copy of the GNU General Public License 21 * along with this program; if not, write to the Free Software 22 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 23 * 24 ******************************************************************************/ 25 26 #include <linux/net.h> 27 #include <linux/delay.h> 28 #include <linux/string.h> 29 #include <linux/timer.h> 30 #include <linux/slab.h> 31 #include <linux/spinlock.h> 32 #include <linux/kthread.h> 33 #include <linux/in.h> 34 #include <linux/cdrom.h> 35 #include <linux/module.h> 36 #include <linux/ratelimit.h> 37 #include <linux/vmalloc.h> 38 #include <asm/unaligned.h> 39 #include <net/sock.h> 40 #include <net/tcp.h> 41 #include <scsi/scsi_proto.h> 42 #include <scsi/scsi_common.h> 43 44 #include <target/target_core_base.h> 45 #include <target/target_core_backend.h> 46 #include <target/target_core_fabric.h> 47 48 #include "target_core_internal.h" 49 #include "target_core_alua.h" 50 #include "target_core_pr.h" 51 #include "target_core_ua.h" 52 53 #define CREATE_TRACE_POINTS 54 #include <trace/events/target.h> 55 56 static struct workqueue_struct *target_completion_wq; 57 static struct kmem_cache *se_sess_cache; 58 struct kmem_cache *se_ua_cache; 59 struct kmem_cache *t10_pr_reg_cache; 60 struct kmem_cache *t10_alua_lu_gp_cache; 61 struct kmem_cache *t10_alua_lu_gp_mem_cache; 62 struct kmem_cache *t10_alua_tg_pt_gp_cache; 63 struct kmem_cache *t10_alua_lba_map_cache; 64 struct kmem_cache *t10_alua_lba_map_mem_cache; 65 66 static void transport_complete_task_attr(struct se_cmd *cmd); 67 static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason); 68 static void transport_handle_queue_full(struct se_cmd *cmd, 69 struct se_device *dev, int err, bool write_pending); 70 static void target_complete_ok_work(struct work_struct *work); 71 72 int init_se_kmem_caches(void) 73 { 74 se_sess_cache = kmem_cache_create("se_sess_cache", 75 sizeof(struct se_session), __alignof__(struct se_session), 76 0, NULL); 77 if (!se_sess_cache) { 78 pr_err("kmem_cache_create() for struct se_session" 79 " failed\n"); 80 goto out; 81 } 82 se_ua_cache = kmem_cache_create("se_ua_cache", 83 sizeof(struct se_ua), __alignof__(struct se_ua), 84 0, NULL); 85 if (!se_ua_cache) { 86 pr_err("kmem_cache_create() for struct se_ua failed\n"); 87 goto out_free_sess_cache; 88 } 89 t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache", 90 sizeof(struct t10_pr_registration), 91 __alignof__(struct t10_pr_registration), 0, NULL); 92 if (!t10_pr_reg_cache) { 93 pr_err("kmem_cache_create() for struct t10_pr_registration" 94 " failed\n"); 95 goto out_free_ua_cache; 96 } 97 t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache", 98 sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp), 99 0, NULL); 100 if (!t10_alua_lu_gp_cache) { 101 pr_err("kmem_cache_create() for t10_alua_lu_gp_cache" 102 " failed\n"); 103 goto out_free_pr_reg_cache; 104 } 105 t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache", 106 sizeof(struct t10_alua_lu_gp_member), 107 __alignof__(struct t10_alua_lu_gp_member), 0, NULL); 108 if (!t10_alua_lu_gp_mem_cache) { 109 pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_" 110 "cache failed\n"); 111 goto out_free_lu_gp_cache; 112 } 113 t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache", 114 sizeof(struct t10_alua_tg_pt_gp), 115 __alignof__(struct t10_alua_tg_pt_gp), 0, NULL); 116 if (!t10_alua_tg_pt_gp_cache) { 117 pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_" 118 "cache failed\n"); 119 goto out_free_lu_gp_mem_cache; 120 } 121 t10_alua_lba_map_cache = kmem_cache_create( 122 "t10_alua_lba_map_cache", 123 sizeof(struct t10_alua_lba_map), 124 __alignof__(struct t10_alua_lba_map), 0, NULL); 125 if (!t10_alua_lba_map_cache) { 126 pr_err("kmem_cache_create() for t10_alua_lba_map_" 127 "cache failed\n"); 128 goto out_free_tg_pt_gp_cache; 129 } 130 t10_alua_lba_map_mem_cache = kmem_cache_create( 131 "t10_alua_lba_map_mem_cache", 132 sizeof(struct t10_alua_lba_map_member), 133 __alignof__(struct t10_alua_lba_map_member), 0, NULL); 134 if (!t10_alua_lba_map_mem_cache) { 135 pr_err("kmem_cache_create() for t10_alua_lba_map_mem_" 136 "cache failed\n"); 137 goto out_free_lba_map_cache; 138 } 139 140 target_completion_wq = alloc_workqueue("target_completion", 141 WQ_MEM_RECLAIM, 0); 142 if (!target_completion_wq) 143 goto out_free_lba_map_mem_cache; 144 145 return 0; 146 147 out_free_lba_map_mem_cache: 148 kmem_cache_destroy(t10_alua_lba_map_mem_cache); 149 out_free_lba_map_cache: 150 kmem_cache_destroy(t10_alua_lba_map_cache); 151 out_free_tg_pt_gp_cache: 152 kmem_cache_destroy(t10_alua_tg_pt_gp_cache); 153 out_free_lu_gp_mem_cache: 154 kmem_cache_destroy(t10_alua_lu_gp_mem_cache); 155 out_free_lu_gp_cache: 156 kmem_cache_destroy(t10_alua_lu_gp_cache); 157 out_free_pr_reg_cache: 158 kmem_cache_destroy(t10_pr_reg_cache); 159 out_free_ua_cache: 160 kmem_cache_destroy(se_ua_cache); 161 out_free_sess_cache: 162 kmem_cache_destroy(se_sess_cache); 163 out: 164 return -ENOMEM; 165 } 166 167 void release_se_kmem_caches(void) 168 { 169 destroy_workqueue(target_completion_wq); 170 kmem_cache_destroy(se_sess_cache); 171 kmem_cache_destroy(se_ua_cache); 172 kmem_cache_destroy(t10_pr_reg_cache); 173 kmem_cache_destroy(t10_alua_lu_gp_cache); 174 kmem_cache_destroy(t10_alua_lu_gp_mem_cache); 175 kmem_cache_destroy(t10_alua_tg_pt_gp_cache); 176 kmem_cache_destroy(t10_alua_lba_map_cache); 177 kmem_cache_destroy(t10_alua_lba_map_mem_cache); 178 } 179 180 /* This code ensures unique mib indexes are handed out. */ 181 static DEFINE_SPINLOCK(scsi_mib_index_lock); 182 static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX]; 183 184 /* 185 * Allocate a new row index for the entry type specified 186 */ 187 u32 scsi_get_new_index(scsi_index_t type) 188 { 189 u32 new_index; 190 191 BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX)); 192 193 spin_lock(&scsi_mib_index_lock); 194 new_index = ++scsi_mib_index[type]; 195 spin_unlock(&scsi_mib_index_lock); 196 197 return new_index; 198 } 199 200 void transport_subsystem_check_init(void) 201 { 202 int ret; 203 static int sub_api_initialized; 204 205 if (sub_api_initialized) 206 return; 207 208 ret = request_module("target_core_iblock"); 209 if (ret != 0) 210 pr_err("Unable to load target_core_iblock\n"); 211 212 ret = request_module("target_core_file"); 213 if (ret != 0) 214 pr_err("Unable to load target_core_file\n"); 215 216 ret = request_module("target_core_pscsi"); 217 if (ret != 0) 218 pr_err("Unable to load target_core_pscsi\n"); 219 220 ret = request_module("target_core_user"); 221 if (ret != 0) 222 pr_err("Unable to load target_core_user\n"); 223 224 sub_api_initialized = 1; 225 } 226 227 struct se_session *transport_init_session(enum target_prot_op sup_prot_ops) 228 { 229 struct se_session *se_sess; 230 231 se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL); 232 if (!se_sess) { 233 pr_err("Unable to allocate struct se_session from" 234 " se_sess_cache\n"); 235 return ERR_PTR(-ENOMEM); 236 } 237 INIT_LIST_HEAD(&se_sess->sess_list); 238 INIT_LIST_HEAD(&se_sess->sess_acl_list); 239 INIT_LIST_HEAD(&se_sess->sess_cmd_list); 240 INIT_LIST_HEAD(&se_sess->sess_wait_list); 241 spin_lock_init(&se_sess->sess_cmd_lock); 242 se_sess->sup_prot_ops = sup_prot_ops; 243 244 return se_sess; 245 } 246 EXPORT_SYMBOL(transport_init_session); 247 248 int transport_alloc_session_tags(struct se_session *se_sess, 249 unsigned int tag_num, unsigned int tag_size) 250 { 251 int rc; 252 253 se_sess->sess_cmd_map = kzalloc(tag_num * tag_size, 254 GFP_KERNEL | __GFP_NOWARN | __GFP_RETRY_MAYFAIL); 255 if (!se_sess->sess_cmd_map) { 256 se_sess->sess_cmd_map = vzalloc(tag_num * tag_size); 257 if (!se_sess->sess_cmd_map) { 258 pr_err("Unable to allocate se_sess->sess_cmd_map\n"); 259 return -ENOMEM; 260 } 261 } 262 263 rc = percpu_ida_init(&se_sess->sess_tag_pool, tag_num); 264 if (rc < 0) { 265 pr_err("Unable to init se_sess->sess_tag_pool," 266 " tag_num: %u\n", tag_num); 267 kvfree(se_sess->sess_cmd_map); 268 se_sess->sess_cmd_map = NULL; 269 return -ENOMEM; 270 } 271 272 return 0; 273 } 274 EXPORT_SYMBOL(transport_alloc_session_tags); 275 276 struct se_session *transport_init_session_tags(unsigned int tag_num, 277 unsigned int tag_size, 278 enum target_prot_op sup_prot_ops) 279 { 280 struct se_session *se_sess; 281 int rc; 282 283 if (tag_num != 0 && !tag_size) { 284 pr_err("init_session_tags called with percpu-ida tag_num:" 285 " %u, but zero tag_size\n", tag_num); 286 return ERR_PTR(-EINVAL); 287 } 288 if (!tag_num && tag_size) { 289 pr_err("init_session_tags called with percpu-ida tag_size:" 290 " %u, but zero tag_num\n", tag_size); 291 return ERR_PTR(-EINVAL); 292 } 293 294 se_sess = transport_init_session(sup_prot_ops); 295 if (IS_ERR(se_sess)) 296 return se_sess; 297 298 rc = transport_alloc_session_tags(se_sess, tag_num, tag_size); 299 if (rc < 0) { 300 transport_free_session(se_sess); 301 return ERR_PTR(-ENOMEM); 302 } 303 304 return se_sess; 305 } 306 EXPORT_SYMBOL(transport_init_session_tags); 307 308 /* 309 * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called. 310 */ 311 void __transport_register_session( 312 struct se_portal_group *se_tpg, 313 struct se_node_acl *se_nacl, 314 struct se_session *se_sess, 315 void *fabric_sess_ptr) 316 { 317 const struct target_core_fabric_ops *tfo = se_tpg->se_tpg_tfo; 318 unsigned char buf[PR_REG_ISID_LEN]; 319 320 se_sess->se_tpg = se_tpg; 321 se_sess->fabric_sess_ptr = fabric_sess_ptr; 322 /* 323 * Used by struct se_node_acl's under ConfigFS to locate active se_session-t 324 * 325 * Only set for struct se_session's that will actually be moving I/O. 326 * eg: *NOT* discovery sessions. 327 */ 328 if (se_nacl) { 329 /* 330 * 331 * Determine if fabric allows for T10-PI feature bits exposed to 332 * initiators for device backends with !dev->dev_attrib.pi_prot_type. 333 * 334 * If so, then always save prot_type on a per se_node_acl node 335 * basis and re-instate the previous sess_prot_type to avoid 336 * disabling PI from below any previously initiator side 337 * registered LUNs. 338 */ 339 if (se_nacl->saved_prot_type) 340 se_sess->sess_prot_type = se_nacl->saved_prot_type; 341 else if (tfo->tpg_check_prot_fabric_only) 342 se_sess->sess_prot_type = se_nacl->saved_prot_type = 343 tfo->tpg_check_prot_fabric_only(se_tpg); 344 /* 345 * If the fabric module supports an ISID based TransportID, 346 * save this value in binary from the fabric I_T Nexus now. 347 */ 348 if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) { 349 memset(&buf[0], 0, PR_REG_ISID_LEN); 350 se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess, 351 &buf[0], PR_REG_ISID_LEN); 352 se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]); 353 } 354 355 spin_lock_irq(&se_nacl->nacl_sess_lock); 356 /* 357 * The se_nacl->nacl_sess pointer will be set to the 358 * last active I_T Nexus for each struct se_node_acl. 359 */ 360 se_nacl->nacl_sess = se_sess; 361 362 list_add_tail(&se_sess->sess_acl_list, 363 &se_nacl->acl_sess_list); 364 spin_unlock_irq(&se_nacl->nacl_sess_lock); 365 } 366 list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list); 367 368 pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n", 369 se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr); 370 } 371 EXPORT_SYMBOL(__transport_register_session); 372 373 void transport_register_session( 374 struct se_portal_group *se_tpg, 375 struct se_node_acl *se_nacl, 376 struct se_session *se_sess, 377 void *fabric_sess_ptr) 378 { 379 unsigned long flags; 380 381 spin_lock_irqsave(&se_tpg->session_lock, flags); 382 __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr); 383 spin_unlock_irqrestore(&se_tpg->session_lock, flags); 384 } 385 EXPORT_SYMBOL(transport_register_session); 386 387 struct se_session * 388 target_alloc_session(struct se_portal_group *tpg, 389 unsigned int tag_num, unsigned int tag_size, 390 enum target_prot_op prot_op, 391 const char *initiatorname, void *private, 392 int (*callback)(struct se_portal_group *, 393 struct se_session *, void *)) 394 { 395 struct se_session *sess; 396 397 /* 398 * If the fabric driver is using percpu-ida based pre allocation 399 * of I/O descriptor tags, go ahead and perform that setup now.. 400 */ 401 if (tag_num != 0) 402 sess = transport_init_session_tags(tag_num, tag_size, prot_op); 403 else 404 sess = transport_init_session(prot_op); 405 406 if (IS_ERR(sess)) 407 return sess; 408 409 sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg, 410 (unsigned char *)initiatorname); 411 if (!sess->se_node_acl) { 412 transport_free_session(sess); 413 return ERR_PTR(-EACCES); 414 } 415 /* 416 * Go ahead and perform any remaining fabric setup that is 417 * required before transport_register_session(). 418 */ 419 if (callback != NULL) { 420 int rc = callback(tpg, sess, private); 421 if (rc) { 422 transport_free_session(sess); 423 return ERR_PTR(rc); 424 } 425 } 426 427 transport_register_session(tpg, sess->se_node_acl, sess, private); 428 return sess; 429 } 430 EXPORT_SYMBOL(target_alloc_session); 431 432 ssize_t target_show_dynamic_sessions(struct se_portal_group *se_tpg, char *page) 433 { 434 struct se_session *se_sess; 435 ssize_t len = 0; 436 437 spin_lock_bh(&se_tpg->session_lock); 438 list_for_each_entry(se_sess, &se_tpg->tpg_sess_list, sess_list) { 439 if (!se_sess->se_node_acl) 440 continue; 441 if (!se_sess->se_node_acl->dynamic_node_acl) 442 continue; 443 if (strlen(se_sess->se_node_acl->initiatorname) + 1 + len > PAGE_SIZE) 444 break; 445 446 len += snprintf(page + len, PAGE_SIZE - len, "%s\n", 447 se_sess->se_node_acl->initiatorname); 448 len += 1; /* Include NULL terminator */ 449 } 450 spin_unlock_bh(&se_tpg->session_lock); 451 452 return len; 453 } 454 EXPORT_SYMBOL(target_show_dynamic_sessions); 455 456 static void target_complete_nacl(struct kref *kref) 457 { 458 struct se_node_acl *nacl = container_of(kref, 459 struct se_node_acl, acl_kref); 460 struct se_portal_group *se_tpg = nacl->se_tpg; 461 462 if (!nacl->dynamic_stop) { 463 complete(&nacl->acl_free_comp); 464 return; 465 } 466 467 mutex_lock(&se_tpg->acl_node_mutex); 468 list_del_init(&nacl->acl_list); 469 mutex_unlock(&se_tpg->acl_node_mutex); 470 471 core_tpg_wait_for_nacl_pr_ref(nacl); 472 core_free_device_list_for_node(nacl, se_tpg); 473 kfree(nacl); 474 } 475 476 void target_put_nacl(struct se_node_acl *nacl) 477 { 478 kref_put(&nacl->acl_kref, target_complete_nacl); 479 } 480 EXPORT_SYMBOL(target_put_nacl); 481 482 void transport_deregister_session_configfs(struct se_session *se_sess) 483 { 484 struct se_node_acl *se_nacl; 485 unsigned long flags; 486 /* 487 * Used by struct se_node_acl's under ConfigFS to locate active struct se_session 488 */ 489 se_nacl = se_sess->se_node_acl; 490 if (se_nacl) { 491 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags); 492 if (!list_empty(&se_sess->sess_acl_list)) 493 list_del_init(&se_sess->sess_acl_list); 494 /* 495 * If the session list is empty, then clear the pointer. 496 * Otherwise, set the struct se_session pointer from the tail 497 * element of the per struct se_node_acl active session list. 498 */ 499 if (list_empty(&se_nacl->acl_sess_list)) 500 se_nacl->nacl_sess = NULL; 501 else { 502 se_nacl->nacl_sess = container_of( 503 se_nacl->acl_sess_list.prev, 504 struct se_session, sess_acl_list); 505 } 506 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags); 507 } 508 } 509 EXPORT_SYMBOL(transport_deregister_session_configfs); 510 511 void transport_free_session(struct se_session *se_sess) 512 { 513 struct se_node_acl *se_nacl = se_sess->se_node_acl; 514 515 /* 516 * Drop the se_node_acl->nacl_kref obtained from within 517 * core_tpg_get_initiator_node_acl(). 518 */ 519 if (se_nacl) { 520 struct se_portal_group *se_tpg = se_nacl->se_tpg; 521 const struct target_core_fabric_ops *se_tfo = se_tpg->se_tpg_tfo; 522 unsigned long flags; 523 524 se_sess->se_node_acl = NULL; 525 526 /* 527 * Also determine if we need to drop the extra ->cmd_kref if 528 * it had been previously dynamically generated, and 529 * the endpoint is not caching dynamic ACLs. 530 */ 531 mutex_lock(&se_tpg->acl_node_mutex); 532 if (se_nacl->dynamic_node_acl && 533 !se_tfo->tpg_check_demo_mode_cache(se_tpg)) { 534 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags); 535 if (list_empty(&se_nacl->acl_sess_list)) 536 se_nacl->dynamic_stop = true; 537 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags); 538 539 if (se_nacl->dynamic_stop) 540 list_del_init(&se_nacl->acl_list); 541 } 542 mutex_unlock(&se_tpg->acl_node_mutex); 543 544 if (se_nacl->dynamic_stop) 545 target_put_nacl(se_nacl); 546 547 target_put_nacl(se_nacl); 548 } 549 if (se_sess->sess_cmd_map) { 550 percpu_ida_destroy(&se_sess->sess_tag_pool); 551 kvfree(se_sess->sess_cmd_map); 552 } 553 kmem_cache_free(se_sess_cache, se_sess); 554 } 555 EXPORT_SYMBOL(transport_free_session); 556 557 void transport_deregister_session(struct se_session *se_sess) 558 { 559 struct se_portal_group *se_tpg = se_sess->se_tpg; 560 unsigned long flags; 561 562 if (!se_tpg) { 563 transport_free_session(se_sess); 564 return; 565 } 566 567 spin_lock_irqsave(&se_tpg->session_lock, flags); 568 list_del(&se_sess->sess_list); 569 se_sess->se_tpg = NULL; 570 se_sess->fabric_sess_ptr = NULL; 571 spin_unlock_irqrestore(&se_tpg->session_lock, flags); 572 573 pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n", 574 se_tpg->se_tpg_tfo->get_fabric_name()); 575 /* 576 * If last kref is dropping now for an explicit NodeACL, awake sleeping 577 * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group 578 * removal context from within transport_free_session() code. 579 * 580 * For dynamic ACL, target_put_nacl() uses target_complete_nacl() 581 * to release all remaining generate_node_acl=1 created ACL resources. 582 */ 583 584 transport_free_session(se_sess); 585 } 586 EXPORT_SYMBOL(transport_deregister_session); 587 588 static void target_remove_from_state_list(struct se_cmd *cmd) 589 { 590 struct se_device *dev = cmd->se_dev; 591 unsigned long flags; 592 593 if (!dev) 594 return; 595 596 spin_lock_irqsave(&dev->execute_task_lock, flags); 597 if (cmd->state_active) { 598 list_del(&cmd->state_list); 599 cmd->state_active = false; 600 } 601 spin_unlock_irqrestore(&dev->execute_task_lock, flags); 602 } 603 604 static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd) 605 { 606 unsigned long flags; 607 608 target_remove_from_state_list(cmd); 609 610 /* 611 * Clear struct se_cmd->se_lun before the handoff to FE. 612 */ 613 cmd->se_lun = NULL; 614 615 spin_lock_irqsave(&cmd->t_state_lock, flags); 616 /* 617 * Determine if frontend context caller is requesting the stopping of 618 * this command for frontend exceptions. 619 */ 620 if (cmd->transport_state & CMD_T_STOP) { 621 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 622 __func__, __LINE__, cmd->tag); 623 624 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 625 626 complete_all(&cmd->t_transport_stop_comp); 627 return 1; 628 } 629 cmd->transport_state &= ~CMD_T_ACTIVE; 630 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 631 632 /* 633 * Some fabric modules like tcm_loop can release their internally 634 * allocated I/O reference and struct se_cmd now. 635 * 636 * Fabric modules are expected to return '1' here if the se_cmd being 637 * passed is released at this point, or zero if not being released. 638 */ 639 return cmd->se_tfo->check_stop_free(cmd); 640 } 641 642 static void transport_lun_remove_cmd(struct se_cmd *cmd) 643 { 644 struct se_lun *lun = cmd->se_lun; 645 646 if (!lun) 647 return; 648 649 if (cmpxchg(&cmd->lun_ref_active, true, false)) 650 percpu_ref_put(&lun->lun_ref); 651 } 652 653 int transport_cmd_finish_abort(struct se_cmd *cmd, int remove) 654 { 655 bool ack_kref = (cmd->se_cmd_flags & SCF_ACK_KREF); 656 int ret = 0; 657 658 if (cmd->se_cmd_flags & SCF_SE_LUN_CMD) 659 transport_lun_remove_cmd(cmd); 660 /* 661 * Allow the fabric driver to unmap any resources before 662 * releasing the descriptor via TFO->release_cmd() 663 */ 664 if (remove) 665 cmd->se_tfo->aborted_task(cmd); 666 667 if (transport_cmd_check_stop_to_fabric(cmd)) 668 return 1; 669 if (remove && ack_kref) 670 ret = target_put_sess_cmd(cmd); 671 672 return ret; 673 } 674 675 static void target_complete_failure_work(struct work_struct *work) 676 { 677 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 678 679 transport_generic_request_failure(cmd, 680 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); 681 } 682 683 /* 684 * Used when asking transport to copy Sense Data from the underlying 685 * Linux/SCSI struct scsi_cmnd 686 */ 687 static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd) 688 { 689 struct se_device *dev = cmd->se_dev; 690 691 WARN_ON(!cmd->se_lun); 692 693 if (!dev) 694 return NULL; 695 696 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) 697 return NULL; 698 699 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 700 701 pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n", 702 dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status); 703 return cmd->sense_buffer; 704 } 705 706 void transport_copy_sense_to_cmd(struct se_cmd *cmd, unsigned char *sense) 707 { 708 unsigned char *cmd_sense_buf; 709 unsigned long flags; 710 711 spin_lock_irqsave(&cmd->t_state_lock, flags); 712 cmd_sense_buf = transport_get_sense_buffer(cmd); 713 if (!cmd_sense_buf) { 714 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 715 return; 716 } 717 718 cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE; 719 memcpy(cmd_sense_buf, sense, cmd->scsi_sense_length); 720 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 721 } 722 EXPORT_SYMBOL(transport_copy_sense_to_cmd); 723 724 void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) 725 { 726 struct se_device *dev = cmd->se_dev; 727 int success; 728 unsigned long flags; 729 730 cmd->scsi_status = scsi_status; 731 732 spin_lock_irqsave(&cmd->t_state_lock, flags); 733 switch (cmd->scsi_status) { 734 case SAM_STAT_CHECK_CONDITION: 735 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) 736 success = 1; 737 else 738 success = 0; 739 break; 740 default: 741 success = 1; 742 break; 743 } 744 745 /* 746 * Check for case where an explicit ABORT_TASK has been received 747 * and transport_wait_for_tasks() will be waiting for completion.. 748 */ 749 if (cmd->transport_state & CMD_T_ABORTED || 750 cmd->transport_state & CMD_T_STOP) { 751 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 752 /* 753 * If COMPARE_AND_WRITE was stopped by __transport_wait_for_tasks(), 754 * release se_device->caw_sem obtained by sbc_compare_and_write() 755 * since target_complete_ok_work() or target_complete_failure_work() 756 * won't be called to invoke the normal CAW completion callbacks. 757 */ 758 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) { 759 up(&dev->caw_sem); 760 } 761 complete_all(&cmd->t_transport_stop_comp); 762 return; 763 } else if (!success) { 764 INIT_WORK(&cmd->work, target_complete_failure_work); 765 } else { 766 INIT_WORK(&cmd->work, target_complete_ok_work); 767 } 768 769 cmd->t_state = TRANSPORT_COMPLETE; 770 cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE); 771 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 772 773 if (cmd->se_cmd_flags & SCF_USE_CPUID) 774 queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work); 775 else 776 queue_work(target_completion_wq, &cmd->work); 777 } 778 EXPORT_SYMBOL(target_complete_cmd); 779 780 void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length) 781 { 782 if (scsi_status == SAM_STAT_GOOD && length < cmd->data_length) { 783 if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { 784 cmd->residual_count += cmd->data_length - length; 785 } else { 786 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 787 cmd->residual_count = cmd->data_length - length; 788 } 789 790 cmd->data_length = length; 791 } 792 793 target_complete_cmd(cmd, scsi_status); 794 } 795 EXPORT_SYMBOL(target_complete_cmd_with_length); 796 797 static void target_add_to_state_list(struct se_cmd *cmd) 798 { 799 struct se_device *dev = cmd->se_dev; 800 unsigned long flags; 801 802 spin_lock_irqsave(&dev->execute_task_lock, flags); 803 if (!cmd->state_active) { 804 list_add_tail(&cmd->state_list, &dev->state_list); 805 cmd->state_active = true; 806 } 807 spin_unlock_irqrestore(&dev->execute_task_lock, flags); 808 } 809 810 /* 811 * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status 812 */ 813 static void transport_write_pending_qf(struct se_cmd *cmd); 814 static void transport_complete_qf(struct se_cmd *cmd); 815 816 void target_qf_do_work(struct work_struct *work) 817 { 818 struct se_device *dev = container_of(work, struct se_device, 819 qf_work_queue); 820 LIST_HEAD(qf_cmd_list); 821 struct se_cmd *cmd, *cmd_tmp; 822 823 spin_lock_irq(&dev->qf_cmd_lock); 824 list_splice_init(&dev->qf_cmd_list, &qf_cmd_list); 825 spin_unlock_irq(&dev->qf_cmd_lock); 826 827 list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) { 828 list_del(&cmd->se_qf_node); 829 atomic_dec_mb(&dev->dev_qf_count); 830 831 pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue" 832 " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd, 833 (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" : 834 (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING" 835 : "UNKNOWN"); 836 837 if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) 838 transport_write_pending_qf(cmd); 839 else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK || 840 cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) 841 transport_complete_qf(cmd); 842 } 843 } 844 845 unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd) 846 { 847 switch (cmd->data_direction) { 848 case DMA_NONE: 849 return "NONE"; 850 case DMA_FROM_DEVICE: 851 return "READ"; 852 case DMA_TO_DEVICE: 853 return "WRITE"; 854 case DMA_BIDIRECTIONAL: 855 return "BIDI"; 856 default: 857 break; 858 } 859 860 return "UNKNOWN"; 861 } 862 863 void transport_dump_dev_state( 864 struct se_device *dev, 865 char *b, 866 int *bl) 867 { 868 *bl += sprintf(b + *bl, "Status: "); 869 if (dev->export_count) 870 *bl += sprintf(b + *bl, "ACTIVATED"); 871 else 872 *bl += sprintf(b + *bl, "DEACTIVATED"); 873 874 *bl += sprintf(b + *bl, " Max Queue Depth: %d", dev->queue_depth); 875 *bl += sprintf(b + *bl, " SectorSize: %u HwMaxSectors: %u\n", 876 dev->dev_attrib.block_size, 877 dev->dev_attrib.hw_max_sectors); 878 *bl += sprintf(b + *bl, " "); 879 } 880 881 void transport_dump_vpd_proto_id( 882 struct t10_vpd *vpd, 883 unsigned char *p_buf, 884 int p_buf_len) 885 { 886 unsigned char buf[VPD_TMP_BUF_SIZE]; 887 int len; 888 889 memset(buf, 0, VPD_TMP_BUF_SIZE); 890 len = sprintf(buf, "T10 VPD Protocol Identifier: "); 891 892 switch (vpd->protocol_identifier) { 893 case 0x00: 894 sprintf(buf+len, "Fibre Channel\n"); 895 break; 896 case 0x10: 897 sprintf(buf+len, "Parallel SCSI\n"); 898 break; 899 case 0x20: 900 sprintf(buf+len, "SSA\n"); 901 break; 902 case 0x30: 903 sprintf(buf+len, "IEEE 1394\n"); 904 break; 905 case 0x40: 906 sprintf(buf+len, "SCSI Remote Direct Memory Access" 907 " Protocol\n"); 908 break; 909 case 0x50: 910 sprintf(buf+len, "Internet SCSI (iSCSI)\n"); 911 break; 912 case 0x60: 913 sprintf(buf+len, "SAS Serial SCSI Protocol\n"); 914 break; 915 case 0x70: 916 sprintf(buf+len, "Automation/Drive Interface Transport" 917 " Protocol\n"); 918 break; 919 case 0x80: 920 sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n"); 921 break; 922 default: 923 sprintf(buf+len, "Unknown 0x%02x\n", 924 vpd->protocol_identifier); 925 break; 926 } 927 928 if (p_buf) 929 strncpy(p_buf, buf, p_buf_len); 930 else 931 pr_debug("%s", buf); 932 } 933 934 void 935 transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83) 936 { 937 /* 938 * Check if the Protocol Identifier Valid (PIV) bit is set.. 939 * 940 * from spc3r23.pdf section 7.5.1 941 */ 942 if (page_83[1] & 0x80) { 943 vpd->protocol_identifier = (page_83[0] & 0xf0); 944 vpd->protocol_identifier_set = 1; 945 transport_dump_vpd_proto_id(vpd, NULL, 0); 946 } 947 } 948 EXPORT_SYMBOL(transport_set_vpd_proto_id); 949 950 int transport_dump_vpd_assoc( 951 struct t10_vpd *vpd, 952 unsigned char *p_buf, 953 int p_buf_len) 954 { 955 unsigned char buf[VPD_TMP_BUF_SIZE]; 956 int ret = 0; 957 int len; 958 959 memset(buf, 0, VPD_TMP_BUF_SIZE); 960 len = sprintf(buf, "T10 VPD Identifier Association: "); 961 962 switch (vpd->association) { 963 case 0x00: 964 sprintf(buf+len, "addressed logical unit\n"); 965 break; 966 case 0x10: 967 sprintf(buf+len, "target port\n"); 968 break; 969 case 0x20: 970 sprintf(buf+len, "SCSI target device\n"); 971 break; 972 default: 973 sprintf(buf+len, "Unknown 0x%02x\n", vpd->association); 974 ret = -EINVAL; 975 break; 976 } 977 978 if (p_buf) 979 strncpy(p_buf, buf, p_buf_len); 980 else 981 pr_debug("%s", buf); 982 983 return ret; 984 } 985 986 int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83) 987 { 988 /* 989 * The VPD identification association.. 990 * 991 * from spc3r23.pdf Section 7.6.3.1 Table 297 992 */ 993 vpd->association = (page_83[1] & 0x30); 994 return transport_dump_vpd_assoc(vpd, NULL, 0); 995 } 996 EXPORT_SYMBOL(transport_set_vpd_assoc); 997 998 int transport_dump_vpd_ident_type( 999 struct t10_vpd *vpd, 1000 unsigned char *p_buf, 1001 int p_buf_len) 1002 { 1003 unsigned char buf[VPD_TMP_BUF_SIZE]; 1004 int ret = 0; 1005 int len; 1006 1007 memset(buf, 0, VPD_TMP_BUF_SIZE); 1008 len = sprintf(buf, "T10 VPD Identifier Type: "); 1009 1010 switch (vpd->device_identifier_type) { 1011 case 0x00: 1012 sprintf(buf+len, "Vendor specific\n"); 1013 break; 1014 case 0x01: 1015 sprintf(buf+len, "T10 Vendor ID based\n"); 1016 break; 1017 case 0x02: 1018 sprintf(buf+len, "EUI-64 based\n"); 1019 break; 1020 case 0x03: 1021 sprintf(buf+len, "NAA\n"); 1022 break; 1023 case 0x04: 1024 sprintf(buf+len, "Relative target port identifier\n"); 1025 break; 1026 case 0x08: 1027 sprintf(buf+len, "SCSI name string\n"); 1028 break; 1029 default: 1030 sprintf(buf+len, "Unsupported: 0x%02x\n", 1031 vpd->device_identifier_type); 1032 ret = -EINVAL; 1033 break; 1034 } 1035 1036 if (p_buf) { 1037 if (p_buf_len < strlen(buf)+1) 1038 return -EINVAL; 1039 strncpy(p_buf, buf, p_buf_len); 1040 } else { 1041 pr_debug("%s", buf); 1042 } 1043 1044 return ret; 1045 } 1046 1047 int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83) 1048 { 1049 /* 1050 * The VPD identifier type.. 1051 * 1052 * from spc3r23.pdf Section 7.6.3.1 Table 298 1053 */ 1054 vpd->device_identifier_type = (page_83[1] & 0x0f); 1055 return transport_dump_vpd_ident_type(vpd, NULL, 0); 1056 } 1057 EXPORT_SYMBOL(transport_set_vpd_ident_type); 1058 1059 int transport_dump_vpd_ident( 1060 struct t10_vpd *vpd, 1061 unsigned char *p_buf, 1062 int p_buf_len) 1063 { 1064 unsigned char buf[VPD_TMP_BUF_SIZE]; 1065 int ret = 0; 1066 1067 memset(buf, 0, VPD_TMP_BUF_SIZE); 1068 1069 switch (vpd->device_identifier_code_set) { 1070 case 0x01: /* Binary */ 1071 snprintf(buf, sizeof(buf), 1072 "T10 VPD Binary Device Identifier: %s\n", 1073 &vpd->device_identifier[0]); 1074 break; 1075 case 0x02: /* ASCII */ 1076 snprintf(buf, sizeof(buf), 1077 "T10 VPD ASCII Device Identifier: %s\n", 1078 &vpd->device_identifier[0]); 1079 break; 1080 case 0x03: /* UTF-8 */ 1081 snprintf(buf, sizeof(buf), 1082 "T10 VPD UTF-8 Device Identifier: %s\n", 1083 &vpd->device_identifier[0]); 1084 break; 1085 default: 1086 sprintf(buf, "T10 VPD Device Identifier encoding unsupported:" 1087 " 0x%02x", vpd->device_identifier_code_set); 1088 ret = -EINVAL; 1089 break; 1090 } 1091 1092 if (p_buf) 1093 strncpy(p_buf, buf, p_buf_len); 1094 else 1095 pr_debug("%s", buf); 1096 1097 return ret; 1098 } 1099 1100 int 1101 transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83) 1102 { 1103 static const char hex_str[] = "0123456789abcdef"; 1104 int j = 0, i = 4; /* offset to start of the identifier */ 1105 1106 /* 1107 * The VPD Code Set (encoding) 1108 * 1109 * from spc3r23.pdf Section 7.6.3.1 Table 296 1110 */ 1111 vpd->device_identifier_code_set = (page_83[0] & 0x0f); 1112 switch (vpd->device_identifier_code_set) { 1113 case 0x01: /* Binary */ 1114 vpd->device_identifier[j++] = 1115 hex_str[vpd->device_identifier_type]; 1116 while (i < (4 + page_83[3])) { 1117 vpd->device_identifier[j++] = 1118 hex_str[(page_83[i] & 0xf0) >> 4]; 1119 vpd->device_identifier[j++] = 1120 hex_str[page_83[i] & 0x0f]; 1121 i++; 1122 } 1123 break; 1124 case 0x02: /* ASCII */ 1125 case 0x03: /* UTF-8 */ 1126 while (i < (4 + page_83[3])) 1127 vpd->device_identifier[j++] = page_83[i++]; 1128 break; 1129 default: 1130 break; 1131 } 1132 1133 return transport_dump_vpd_ident(vpd, NULL, 0); 1134 } 1135 EXPORT_SYMBOL(transport_set_vpd_ident); 1136 1137 static sense_reason_t 1138 target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev, 1139 unsigned int size) 1140 { 1141 u32 mtl; 1142 1143 if (!cmd->se_tfo->max_data_sg_nents) 1144 return TCM_NO_SENSE; 1145 /* 1146 * Check if fabric enforced maximum SGL entries per I/O descriptor 1147 * exceeds se_cmd->data_length. If true, set SCF_UNDERFLOW_BIT + 1148 * residual_count and reduce original cmd->data_length to maximum 1149 * length based on single PAGE_SIZE entry scatter-lists. 1150 */ 1151 mtl = (cmd->se_tfo->max_data_sg_nents * PAGE_SIZE); 1152 if (cmd->data_length > mtl) { 1153 /* 1154 * If an existing CDB overflow is present, calculate new residual 1155 * based on CDB size minus fabric maximum transfer length. 1156 * 1157 * If an existing CDB underflow is present, calculate new residual 1158 * based on original cmd->data_length minus fabric maximum transfer 1159 * length. 1160 * 1161 * Otherwise, set the underflow residual based on cmd->data_length 1162 * minus fabric maximum transfer length. 1163 */ 1164 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) { 1165 cmd->residual_count = (size - mtl); 1166 } else if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { 1167 u32 orig_dl = size + cmd->residual_count; 1168 cmd->residual_count = (orig_dl - mtl); 1169 } else { 1170 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 1171 cmd->residual_count = (cmd->data_length - mtl); 1172 } 1173 cmd->data_length = mtl; 1174 /* 1175 * Reset sbc_check_prot() calculated protection payload 1176 * length based upon the new smaller MTL. 1177 */ 1178 if (cmd->prot_length) { 1179 u32 sectors = (mtl / dev->dev_attrib.block_size); 1180 cmd->prot_length = dev->prot_length * sectors; 1181 } 1182 } 1183 return TCM_NO_SENSE; 1184 } 1185 1186 sense_reason_t 1187 target_cmd_size_check(struct se_cmd *cmd, unsigned int size) 1188 { 1189 struct se_device *dev = cmd->se_dev; 1190 1191 if (cmd->unknown_data_length) { 1192 cmd->data_length = size; 1193 } else if (size != cmd->data_length) { 1194 pr_warn_ratelimited("TARGET_CORE[%s]: Expected Transfer Length:" 1195 " %u does not match SCSI CDB Length: %u for SAM Opcode:" 1196 " 0x%02x\n", cmd->se_tfo->get_fabric_name(), 1197 cmd->data_length, size, cmd->t_task_cdb[0]); 1198 1199 if (cmd->data_direction == DMA_TO_DEVICE) { 1200 if (cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) { 1201 pr_err_ratelimited("Rejecting underflow/overflow" 1202 " for WRITE data CDB\n"); 1203 return TCM_INVALID_CDB_FIELD; 1204 } 1205 /* 1206 * Some fabric drivers like iscsi-target still expect to 1207 * always reject overflow writes. Reject this case until 1208 * full fabric driver level support for overflow writes 1209 * is introduced tree-wide. 1210 */ 1211 if (size > cmd->data_length) { 1212 pr_err_ratelimited("Rejecting overflow for" 1213 " WRITE control CDB\n"); 1214 return TCM_INVALID_CDB_FIELD; 1215 } 1216 } 1217 /* 1218 * Reject READ_* or WRITE_* with overflow/underflow for 1219 * type SCF_SCSI_DATA_CDB. 1220 */ 1221 if (dev->dev_attrib.block_size != 512) { 1222 pr_err("Failing OVERFLOW/UNDERFLOW for LBA op" 1223 " CDB on non 512-byte sector setup subsystem" 1224 " plugin: %s\n", dev->transport->name); 1225 /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */ 1226 return TCM_INVALID_CDB_FIELD; 1227 } 1228 /* 1229 * For the overflow case keep the existing fabric provided 1230 * ->data_length. Otherwise for the underflow case, reset 1231 * ->data_length to the smaller SCSI expected data transfer 1232 * length. 1233 */ 1234 if (size > cmd->data_length) { 1235 cmd->se_cmd_flags |= SCF_OVERFLOW_BIT; 1236 cmd->residual_count = (size - cmd->data_length); 1237 } else { 1238 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 1239 cmd->residual_count = (cmd->data_length - size); 1240 cmd->data_length = size; 1241 } 1242 } 1243 1244 return target_check_max_data_sg_nents(cmd, dev, size); 1245 1246 } 1247 1248 /* 1249 * Used by fabric modules containing a local struct se_cmd within their 1250 * fabric dependent per I/O descriptor. 1251 * 1252 * Preserves the value of @cmd->tag. 1253 */ 1254 void transport_init_se_cmd( 1255 struct se_cmd *cmd, 1256 const struct target_core_fabric_ops *tfo, 1257 struct se_session *se_sess, 1258 u32 data_length, 1259 int data_direction, 1260 int task_attr, 1261 unsigned char *sense_buffer) 1262 { 1263 INIT_LIST_HEAD(&cmd->se_delayed_node); 1264 INIT_LIST_HEAD(&cmd->se_qf_node); 1265 INIT_LIST_HEAD(&cmd->se_cmd_list); 1266 INIT_LIST_HEAD(&cmd->state_list); 1267 init_completion(&cmd->t_transport_stop_comp); 1268 init_completion(&cmd->cmd_wait_comp); 1269 spin_lock_init(&cmd->t_state_lock); 1270 INIT_WORK(&cmd->work, NULL); 1271 kref_init(&cmd->cmd_kref); 1272 1273 cmd->se_tfo = tfo; 1274 cmd->se_sess = se_sess; 1275 cmd->data_length = data_length; 1276 cmd->data_direction = data_direction; 1277 cmd->sam_task_attr = task_attr; 1278 cmd->sense_buffer = sense_buffer; 1279 1280 cmd->state_active = false; 1281 } 1282 EXPORT_SYMBOL(transport_init_se_cmd); 1283 1284 static sense_reason_t 1285 transport_check_alloc_task_attr(struct se_cmd *cmd) 1286 { 1287 struct se_device *dev = cmd->se_dev; 1288 1289 /* 1290 * Check if SAM Task Attribute emulation is enabled for this 1291 * struct se_device storage object 1292 */ 1293 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1294 return 0; 1295 1296 if (cmd->sam_task_attr == TCM_ACA_TAG) { 1297 pr_debug("SAM Task Attribute ACA" 1298 " emulation is not supported\n"); 1299 return TCM_INVALID_CDB_FIELD; 1300 } 1301 1302 return 0; 1303 } 1304 1305 sense_reason_t 1306 target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) 1307 { 1308 struct se_device *dev = cmd->se_dev; 1309 sense_reason_t ret; 1310 1311 /* 1312 * Ensure that the received CDB is less than the max (252 + 8) bytes 1313 * for VARIABLE_LENGTH_CMD 1314 */ 1315 if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) { 1316 pr_err("Received SCSI CDB with command_size: %d that" 1317 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", 1318 scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE); 1319 return TCM_INVALID_CDB_FIELD; 1320 } 1321 /* 1322 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE, 1323 * allocate the additional extended CDB buffer now.. Otherwise 1324 * setup the pointer from __t_task_cdb to t_task_cdb. 1325 */ 1326 if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) { 1327 cmd->t_task_cdb = kzalloc(scsi_command_size(cdb), 1328 GFP_KERNEL); 1329 if (!cmd->t_task_cdb) { 1330 pr_err("Unable to allocate cmd->t_task_cdb" 1331 " %u > sizeof(cmd->__t_task_cdb): %lu ops\n", 1332 scsi_command_size(cdb), 1333 (unsigned long)sizeof(cmd->__t_task_cdb)); 1334 return TCM_OUT_OF_RESOURCES; 1335 } 1336 } else 1337 cmd->t_task_cdb = &cmd->__t_task_cdb[0]; 1338 /* 1339 * Copy the original CDB into cmd-> 1340 */ 1341 memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb)); 1342 1343 trace_target_sequencer_start(cmd); 1344 1345 ret = dev->transport->parse_cdb(cmd); 1346 if (ret == TCM_UNSUPPORTED_SCSI_OPCODE) 1347 pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n", 1348 cmd->se_tfo->get_fabric_name(), 1349 cmd->se_sess->se_node_acl->initiatorname, 1350 cmd->t_task_cdb[0]); 1351 if (ret) 1352 return ret; 1353 1354 ret = transport_check_alloc_task_attr(cmd); 1355 if (ret) 1356 return ret; 1357 1358 cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE; 1359 atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus); 1360 return 0; 1361 } 1362 EXPORT_SYMBOL(target_setup_cmd_from_cdb); 1363 1364 /* 1365 * Used by fabric module frontends to queue tasks directly. 1366 * May only be used from process context. 1367 */ 1368 int transport_handle_cdb_direct( 1369 struct se_cmd *cmd) 1370 { 1371 sense_reason_t ret; 1372 1373 if (!cmd->se_lun) { 1374 dump_stack(); 1375 pr_err("cmd->se_lun is NULL\n"); 1376 return -EINVAL; 1377 } 1378 if (in_interrupt()) { 1379 dump_stack(); 1380 pr_err("transport_generic_handle_cdb cannot be called" 1381 " from interrupt context\n"); 1382 return -EINVAL; 1383 } 1384 /* 1385 * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that 1386 * outstanding descriptors are handled correctly during shutdown via 1387 * transport_wait_for_tasks() 1388 * 1389 * Also, we don't take cmd->t_state_lock here as we only expect 1390 * this to be called for initial descriptor submission. 1391 */ 1392 cmd->t_state = TRANSPORT_NEW_CMD; 1393 cmd->transport_state |= CMD_T_ACTIVE; 1394 1395 /* 1396 * transport_generic_new_cmd() is already handling QUEUE_FULL, 1397 * so follow TRANSPORT_NEW_CMD processing thread context usage 1398 * and call transport_generic_request_failure() if necessary.. 1399 */ 1400 ret = transport_generic_new_cmd(cmd); 1401 if (ret) 1402 transport_generic_request_failure(cmd, ret); 1403 return 0; 1404 } 1405 EXPORT_SYMBOL(transport_handle_cdb_direct); 1406 1407 sense_reason_t 1408 transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl, 1409 u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count) 1410 { 1411 if (!sgl || !sgl_count) 1412 return 0; 1413 1414 /* 1415 * Reject SCSI data overflow with map_mem_to_cmd() as incoming 1416 * scatterlists already have been set to follow what the fabric 1417 * passes for the original expected data transfer length. 1418 */ 1419 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) { 1420 pr_warn("Rejecting SCSI DATA overflow for fabric using" 1421 " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n"); 1422 return TCM_INVALID_CDB_FIELD; 1423 } 1424 1425 cmd->t_data_sg = sgl; 1426 cmd->t_data_nents = sgl_count; 1427 cmd->t_bidi_data_sg = sgl_bidi; 1428 cmd->t_bidi_data_nents = sgl_bidi_count; 1429 1430 cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1431 return 0; 1432 } 1433 1434 /* 1435 * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized 1436 * se_cmd + use pre-allocated SGL memory. 1437 * 1438 * @se_cmd: command descriptor to submit 1439 * @se_sess: associated se_sess for endpoint 1440 * @cdb: pointer to SCSI CDB 1441 * @sense: pointer to SCSI sense buffer 1442 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1443 * @data_length: fabric expected data transfer length 1444 * @task_addr: SAM task attribute 1445 * @data_dir: DMA data direction 1446 * @flags: flags for command submission from target_sc_flags_tables 1447 * @sgl: struct scatterlist memory for unidirectional mapping 1448 * @sgl_count: scatterlist count for unidirectional mapping 1449 * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping 1450 * @sgl_bidi_count: scatterlist count for bidirectional READ mapping 1451 * @sgl_prot: struct scatterlist memory protection information 1452 * @sgl_prot_count: scatterlist count for protection information 1453 * 1454 * Task tags are supported if the caller has set @se_cmd->tag. 1455 * 1456 * Returns non zero to signal active I/O shutdown failure. All other 1457 * setup exceptions will be returned as a SCSI CHECK_CONDITION response, 1458 * but still return zero here. 1459 * 1460 * This may only be called from process context, and also currently 1461 * assumes internal allocation of fabric payload buffer by target-core. 1462 */ 1463 int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess, 1464 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, 1465 u32 data_length, int task_attr, int data_dir, int flags, 1466 struct scatterlist *sgl, u32 sgl_count, 1467 struct scatterlist *sgl_bidi, u32 sgl_bidi_count, 1468 struct scatterlist *sgl_prot, u32 sgl_prot_count) 1469 { 1470 struct se_portal_group *se_tpg; 1471 sense_reason_t rc; 1472 int ret; 1473 1474 se_tpg = se_sess->se_tpg; 1475 BUG_ON(!se_tpg); 1476 BUG_ON(se_cmd->se_tfo || se_cmd->se_sess); 1477 BUG_ON(in_interrupt()); 1478 /* 1479 * Initialize se_cmd for target operation. From this point 1480 * exceptions are handled by sending exception status via 1481 * target_core_fabric_ops->queue_status() callback 1482 */ 1483 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, 1484 data_length, data_dir, task_attr, sense); 1485 1486 if (flags & TARGET_SCF_USE_CPUID) 1487 se_cmd->se_cmd_flags |= SCF_USE_CPUID; 1488 else 1489 se_cmd->cpuid = WORK_CPU_UNBOUND; 1490 1491 if (flags & TARGET_SCF_UNKNOWN_SIZE) 1492 se_cmd->unknown_data_length = 1; 1493 /* 1494 * Obtain struct se_cmd->cmd_kref reference and add new cmd to 1495 * se_sess->sess_cmd_list. A second kref_get here is necessary 1496 * for fabrics using TARGET_SCF_ACK_KREF that expect a second 1497 * kref_put() to happen during fabric packet acknowledgement. 1498 */ 1499 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); 1500 if (ret) 1501 return ret; 1502 /* 1503 * Signal bidirectional data payloads to target-core 1504 */ 1505 if (flags & TARGET_SCF_BIDI_OP) 1506 se_cmd->se_cmd_flags |= SCF_BIDI; 1507 /* 1508 * Locate se_lun pointer and attach it to struct se_cmd 1509 */ 1510 rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun); 1511 if (rc) { 1512 transport_send_check_condition_and_sense(se_cmd, rc, 0); 1513 target_put_sess_cmd(se_cmd); 1514 return 0; 1515 } 1516 1517 rc = target_setup_cmd_from_cdb(se_cmd, cdb); 1518 if (rc != 0) { 1519 transport_generic_request_failure(se_cmd, rc); 1520 return 0; 1521 } 1522 1523 /* 1524 * Save pointers for SGLs containing protection information, 1525 * if present. 1526 */ 1527 if (sgl_prot_count) { 1528 se_cmd->t_prot_sg = sgl_prot; 1529 se_cmd->t_prot_nents = sgl_prot_count; 1530 se_cmd->se_cmd_flags |= SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC; 1531 } 1532 1533 /* 1534 * When a non zero sgl_count has been passed perform SGL passthrough 1535 * mapping for pre-allocated fabric memory instead of having target 1536 * core perform an internal SGL allocation.. 1537 */ 1538 if (sgl_count != 0) { 1539 BUG_ON(!sgl); 1540 1541 /* 1542 * A work-around for tcm_loop as some userspace code via 1543 * scsi-generic do not memset their associated read buffers, 1544 * so go ahead and do that here for type non-data CDBs. Also 1545 * note that this is currently guaranteed to be a single SGL 1546 * for this case by target core in target_setup_cmd_from_cdb() 1547 * -> transport_generic_cmd_sequencer(). 1548 */ 1549 if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) && 1550 se_cmd->data_direction == DMA_FROM_DEVICE) { 1551 unsigned char *buf = NULL; 1552 1553 if (sgl) 1554 buf = kmap(sg_page(sgl)) + sgl->offset; 1555 1556 if (buf) { 1557 memset(buf, 0, sgl->length); 1558 kunmap(sg_page(sgl)); 1559 } 1560 } 1561 1562 rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count, 1563 sgl_bidi, sgl_bidi_count); 1564 if (rc != 0) { 1565 transport_generic_request_failure(se_cmd, rc); 1566 return 0; 1567 } 1568 } 1569 1570 /* 1571 * Check if we need to delay processing because of ALUA 1572 * Active/NonOptimized primary access state.. 1573 */ 1574 core_alua_check_nonop_delay(se_cmd); 1575 1576 transport_handle_cdb_direct(se_cmd); 1577 return 0; 1578 } 1579 EXPORT_SYMBOL(target_submit_cmd_map_sgls); 1580 1581 /* 1582 * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd 1583 * 1584 * @se_cmd: command descriptor to submit 1585 * @se_sess: associated se_sess for endpoint 1586 * @cdb: pointer to SCSI CDB 1587 * @sense: pointer to SCSI sense buffer 1588 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1589 * @data_length: fabric expected data transfer length 1590 * @task_addr: SAM task attribute 1591 * @data_dir: DMA data direction 1592 * @flags: flags for command submission from target_sc_flags_tables 1593 * 1594 * Task tags are supported if the caller has set @se_cmd->tag. 1595 * 1596 * Returns non zero to signal active I/O shutdown failure. All other 1597 * setup exceptions will be returned as a SCSI CHECK_CONDITION response, 1598 * but still return zero here. 1599 * 1600 * This may only be called from process context, and also currently 1601 * assumes internal allocation of fabric payload buffer by target-core. 1602 * 1603 * It also assumes interal target core SGL memory allocation. 1604 */ 1605 int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, 1606 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, 1607 u32 data_length, int task_attr, int data_dir, int flags) 1608 { 1609 return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense, 1610 unpacked_lun, data_length, task_attr, data_dir, 1611 flags, NULL, 0, NULL, 0, NULL, 0); 1612 } 1613 EXPORT_SYMBOL(target_submit_cmd); 1614 1615 static void target_complete_tmr_failure(struct work_struct *work) 1616 { 1617 struct se_cmd *se_cmd = container_of(work, struct se_cmd, work); 1618 1619 se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST; 1620 se_cmd->se_tfo->queue_tm_rsp(se_cmd); 1621 1622 transport_lun_remove_cmd(se_cmd); 1623 transport_cmd_check_stop_to_fabric(se_cmd); 1624 } 1625 1626 static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag, 1627 u64 *unpacked_lun) 1628 { 1629 struct se_cmd *se_cmd; 1630 unsigned long flags; 1631 bool ret = false; 1632 1633 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 1634 list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) { 1635 if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) 1636 continue; 1637 1638 if (se_cmd->tag == tag) { 1639 *unpacked_lun = se_cmd->orig_fe_lun; 1640 ret = true; 1641 break; 1642 } 1643 } 1644 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 1645 1646 return ret; 1647 } 1648 1649 /** 1650 * target_submit_tmr - lookup unpacked lun and submit uninitialized se_cmd 1651 * for TMR CDBs 1652 * 1653 * @se_cmd: command descriptor to submit 1654 * @se_sess: associated se_sess for endpoint 1655 * @sense: pointer to SCSI sense buffer 1656 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1657 * @fabric_context: fabric context for TMR req 1658 * @tm_type: Type of TM request 1659 * @gfp: gfp type for caller 1660 * @tag: referenced task tag for TMR_ABORT_TASK 1661 * @flags: submit cmd flags 1662 * 1663 * Callable from all contexts. 1664 **/ 1665 1666 int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, 1667 unsigned char *sense, u64 unpacked_lun, 1668 void *fabric_tmr_ptr, unsigned char tm_type, 1669 gfp_t gfp, u64 tag, int flags) 1670 { 1671 struct se_portal_group *se_tpg; 1672 int ret; 1673 1674 se_tpg = se_sess->se_tpg; 1675 BUG_ON(!se_tpg); 1676 1677 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, 1678 0, DMA_NONE, TCM_SIMPLE_TAG, sense); 1679 /* 1680 * FIXME: Currently expect caller to handle se_cmd->se_tmr_req 1681 * allocation failure. 1682 */ 1683 ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp); 1684 if (ret < 0) 1685 return -ENOMEM; 1686 1687 if (tm_type == TMR_ABORT_TASK) 1688 se_cmd->se_tmr_req->ref_task_tag = tag; 1689 1690 /* See target_submit_cmd for commentary */ 1691 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); 1692 if (ret) { 1693 core_tmr_release_req(se_cmd->se_tmr_req); 1694 return ret; 1695 } 1696 /* 1697 * If this is ABORT_TASK with no explicit fabric provided LUN, 1698 * go ahead and search active session tags for a match to figure 1699 * out unpacked_lun for the original se_cmd. 1700 */ 1701 if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) { 1702 if (!target_lookup_lun_from_tag(se_sess, tag, &unpacked_lun)) 1703 goto failure; 1704 } 1705 1706 ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun); 1707 if (ret) 1708 goto failure; 1709 1710 transport_generic_handle_tmr(se_cmd); 1711 return 0; 1712 1713 /* 1714 * For callback during failure handling, push this work off 1715 * to process context with TMR_LUN_DOES_NOT_EXIST status. 1716 */ 1717 failure: 1718 INIT_WORK(&se_cmd->work, target_complete_tmr_failure); 1719 schedule_work(&se_cmd->work); 1720 return 0; 1721 } 1722 EXPORT_SYMBOL(target_submit_tmr); 1723 1724 /* 1725 * Handle SAM-esque emulation for generic transport request failures. 1726 */ 1727 void transport_generic_request_failure(struct se_cmd *cmd, 1728 sense_reason_t sense_reason) 1729 { 1730 int ret = 0, post_ret = 0; 1731 1732 pr_debug("-----[ Storage Engine Exception; sense_reason %d\n", 1733 sense_reason); 1734 target_show_cmd("-----[ ", cmd); 1735 1736 /* 1737 * For SAM Task Attribute emulation for failed struct se_cmd 1738 */ 1739 transport_complete_task_attr(cmd); 1740 1741 /* 1742 * Handle special case for COMPARE_AND_WRITE failure, where the 1743 * callback is expected to drop the per device ->caw_sem. 1744 */ 1745 if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) && 1746 cmd->transport_complete_callback) 1747 cmd->transport_complete_callback(cmd, false, &post_ret); 1748 1749 if (transport_check_aborted_status(cmd, 1)) 1750 return; 1751 1752 switch (sense_reason) { 1753 case TCM_NON_EXISTENT_LUN: 1754 case TCM_UNSUPPORTED_SCSI_OPCODE: 1755 case TCM_INVALID_CDB_FIELD: 1756 case TCM_INVALID_PARAMETER_LIST: 1757 case TCM_PARAMETER_LIST_LENGTH_ERROR: 1758 case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE: 1759 case TCM_UNKNOWN_MODE_PAGE: 1760 case TCM_WRITE_PROTECTED: 1761 case TCM_ADDRESS_OUT_OF_RANGE: 1762 case TCM_CHECK_CONDITION_ABORT_CMD: 1763 case TCM_CHECK_CONDITION_UNIT_ATTENTION: 1764 case TCM_CHECK_CONDITION_NOT_READY: 1765 case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED: 1766 case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED: 1767 case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED: 1768 case TCM_COPY_TARGET_DEVICE_NOT_REACHABLE: 1769 case TCM_TOO_MANY_TARGET_DESCS: 1770 case TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE: 1771 case TCM_TOO_MANY_SEGMENT_DESCS: 1772 case TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE: 1773 break; 1774 case TCM_OUT_OF_RESOURCES: 1775 cmd->scsi_status = SAM_STAT_TASK_SET_FULL; 1776 goto queue_status; 1777 case TCM_LUN_BUSY: 1778 cmd->scsi_status = SAM_STAT_BUSY; 1779 goto queue_status; 1780 case TCM_RESERVATION_CONFLICT: 1781 /* 1782 * No SENSE Data payload for this case, set SCSI Status 1783 * and queue the response to $FABRIC_MOD. 1784 * 1785 * Uses linux/include/scsi/scsi.h SAM status codes defs 1786 */ 1787 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT; 1788 /* 1789 * For UA Interlock Code 11b, a RESERVATION CONFLICT will 1790 * establish a UNIT ATTENTION with PREVIOUS RESERVATION 1791 * CONFLICT STATUS. 1792 * 1793 * See spc4r17, section 7.4.6 Control Mode Page, Table 349 1794 */ 1795 if (cmd->se_sess && 1796 cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) { 1797 target_ua_allocate_lun(cmd->se_sess->se_node_acl, 1798 cmd->orig_fe_lun, 0x2C, 1799 ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS); 1800 } 1801 1802 goto queue_status; 1803 default: 1804 pr_err("Unknown transport error for CDB 0x%02x: %d\n", 1805 cmd->t_task_cdb[0], sense_reason); 1806 sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE; 1807 break; 1808 } 1809 1810 ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0); 1811 if (ret) 1812 goto queue_full; 1813 1814 check_stop: 1815 transport_lun_remove_cmd(cmd); 1816 transport_cmd_check_stop_to_fabric(cmd); 1817 return; 1818 1819 queue_status: 1820 trace_target_cmd_complete(cmd); 1821 ret = cmd->se_tfo->queue_status(cmd); 1822 if (!ret) 1823 goto check_stop; 1824 queue_full: 1825 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 1826 } 1827 EXPORT_SYMBOL(transport_generic_request_failure); 1828 1829 void __target_execute_cmd(struct se_cmd *cmd, bool do_checks) 1830 { 1831 sense_reason_t ret; 1832 1833 if (!cmd->execute_cmd) { 1834 ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 1835 goto err; 1836 } 1837 if (do_checks) { 1838 /* 1839 * Check for an existing UNIT ATTENTION condition after 1840 * target_handle_task_attr() has done SAM task attr 1841 * checking, and possibly have already defered execution 1842 * out to target_restart_delayed_cmds() context. 1843 */ 1844 ret = target_scsi3_ua_check(cmd); 1845 if (ret) 1846 goto err; 1847 1848 ret = target_alua_state_check(cmd); 1849 if (ret) 1850 goto err; 1851 1852 ret = target_check_reservation(cmd); 1853 if (ret) { 1854 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT; 1855 goto err; 1856 } 1857 } 1858 1859 ret = cmd->execute_cmd(cmd); 1860 if (!ret) 1861 return; 1862 err: 1863 spin_lock_irq(&cmd->t_state_lock); 1864 cmd->transport_state &= ~CMD_T_SENT; 1865 spin_unlock_irq(&cmd->t_state_lock); 1866 1867 transport_generic_request_failure(cmd, ret); 1868 } 1869 1870 static int target_write_prot_action(struct se_cmd *cmd) 1871 { 1872 u32 sectors; 1873 /* 1874 * Perform WRITE_INSERT of PI using software emulation when backend 1875 * device has PI enabled, if the transport has not already generated 1876 * PI using hardware WRITE_INSERT offload. 1877 */ 1878 switch (cmd->prot_op) { 1879 case TARGET_PROT_DOUT_INSERT: 1880 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_INSERT)) 1881 sbc_dif_generate(cmd); 1882 break; 1883 case TARGET_PROT_DOUT_STRIP: 1884 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_STRIP) 1885 break; 1886 1887 sectors = cmd->data_length >> ilog2(cmd->se_dev->dev_attrib.block_size); 1888 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba, 1889 sectors, 0, cmd->t_prot_sg, 0); 1890 if (unlikely(cmd->pi_err)) { 1891 spin_lock_irq(&cmd->t_state_lock); 1892 cmd->transport_state &= ~CMD_T_SENT; 1893 spin_unlock_irq(&cmd->t_state_lock); 1894 transport_generic_request_failure(cmd, cmd->pi_err); 1895 return -1; 1896 } 1897 break; 1898 default: 1899 break; 1900 } 1901 1902 return 0; 1903 } 1904 1905 static bool target_handle_task_attr(struct se_cmd *cmd) 1906 { 1907 struct se_device *dev = cmd->se_dev; 1908 1909 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1910 return false; 1911 1912 cmd->se_cmd_flags |= SCF_TASK_ATTR_SET; 1913 1914 /* 1915 * Check for the existence of HEAD_OF_QUEUE, and if true return 1 1916 * to allow the passed struct se_cmd list of tasks to the front of the list. 1917 */ 1918 switch (cmd->sam_task_attr) { 1919 case TCM_HEAD_TAG: 1920 pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x\n", 1921 cmd->t_task_cdb[0]); 1922 return false; 1923 case TCM_ORDERED_TAG: 1924 atomic_inc_mb(&dev->dev_ordered_sync); 1925 1926 pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n", 1927 cmd->t_task_cdb[0]); 1928 1929 /* 1930 * Execute an ORDERED command if no other older commands 1931 * exist that need to be completed first. 1932 */ 1933 if (!atomic_read(&dev->simple_cmds)) 1934 return false; 1935 break; 1936 default: 1937 /* 1938 * For SIMPLE and UNTAGGED Task Attribute commands 1939 */ 1940 atomic_inc_mb(&dev->simple_cmds); 1941 break; 1942 } 1943 1944 if (atomic_read(&dev->dev_ordered_sync) == 0) 1945 return false; 1946 1947 spin_lock(&dev->delayed_cmd_lock); 1948 list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list); 1949 spin_unlock(&dev->delayed_cmd_lock); 1950 1951 pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn", 1952 cmd->t_task_cdb[0], cmd->sam_task_attr); 1953 return true; 1954 } 1955 1956 static int __transport_check_aborted_status(struct se_cmd *, int); 1957 1958 void target_execute_cmd(struct se_cmd *cmd) 1959 { 1960 /* 1961 * Determine if frontend context caller is requesting the stopping of 1962 * this command for frontend exceptions. 1963 * 1964 * If the received CDB has aleady been aborted stop processing it here. 1965 */ 1966 spin_lock_irq(&cmd->t_state_lock); 1967 if (__transport_check_aborted_status(cmd, 1)) { 1968 spin_unlock_irq(&cmd->t_state_lock); 1969 return; 1970 } 1971 if (cmd->transport_state & CMD_T_STOP) { 1972 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 1973 __func__, __LINE__, cmd->tag); 1974 1975 spin_unlock_irq(&cmd->t_state_lock); 1976 complete_all(&cmd->t_transport_stop_comp); 1977 return; 1978 } 1979 1980 cmd->t_state = TRANSPORT_PROCESSING; 1981 cmd->transport_state &= ~CMD_T_PRE_EXECUTE; 1982 cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT; 1983 spin_unlock_irq(&cmd->t_state_lock); 1984 1985 if (target_write_prot_action(cmd)) 1986 return; 1987 1988 if (target_handle_task_attr(cmd)) { 1989 spin_lock_irq(&cmd->t_state_lock); 1990 cmd->transport_state &= ~CMD_T_SENT; 1991 spin_unlock_irq(&cmd->t_state_lock); 1992 return; 1993 } 1994 1995 __target_execute_cmd(cmd, true); 1996 } 1997 EXPORT_SYMBOL(target_execute_cmd); 1998 1999 /* 2000 * Process all commands up to the last received ORDERED task attribute which 2001 * requires another blocking boundary 2002 */ 2003 static void target_restart_delayed_cmds(struct se_device *dev) 2004 { 2005 for (;;) { 2006 struct se_cmd *cmd; 2007 2008 spin_lock(&dev->delayed_cmd_lock); 2009 if (list_empty(&dev->delayed_cmd_list)) { 2010 spin_unlock(&dev->delayed_cmd_lock); 2011 break; 2012 } 2013 2014 cmd = list_entry(dev->delayed_cmd_list.next, 2015 struct se_cmd, se_delayed_node); 2016 list_del(&cmd->se_delayed_node); 2017 spin_unlock(&dev->delayed_cmd_lock); 2018 2019 cmd->transport_state |= CMD_T_SENT; 2020 2021 __target_execute_cmd(cmd, true); 2022 2023 if (cmd->sam_task_attr == TCM_ORDERED_TAG) 2024 break; 2025 } 2026 } 2027 2028 /* 2029 * Called from I/O completion to determine which dormant/delayed 2030 * and ordered cmds need to have their tasks added to the execution queue. 2031 */ 2032 static void transport_complete_task_attr(struct se_cmd *cmd) 2033 { 2034 struct se_device *dev = cmd->se_dev; 2035 2036 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 2037 return; 2038 2039 if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET)) 2040 goto restart; 2041 2042 if (cmd->sam_task_attr == TCM_SIMPLE_TAG) { 2043 atomic_dec_mb(&dev->simple_cmds); 2044 dev->dev_cur_ordered_id++; 2045 } else if (cmd->sam_task_attr == TCM_HEAD_TAG) { 2046 dev->dev_cur_ordered_id++; 2047 pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n", 2048 dev->dev_cur_ordered_id); 2049 } else if (cmd->sam_task_attr == TCM_ORDERED_TAG) { 2050 atomic_dec_mb(&dev->dev_ordered_sync); 2051 2052 dev->dev_cur_ordered_id++; 2053 pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n", 2054 dev->dev_cur_ordered_id); 2055 } 2056 cmd->se_cmd_flags &= ~SCF_TASK_ATTR_SET; 2057 2058 restart: 2059 target_restart_delayed_cmds(dev); 2060 } 2061 2062 static void transport_complete_qf(struct se_cmd *cmd) 2063 { 2064 int ret = 0; 2065 2066 transport_complete_task_attr(cmd); 2067 /* 2068 * If a fabric driver ->write_pending() or ->queue_data_in() callback 2069 * has returned neither -ENOMEM or -EAGAIN, assume it's fatal and 2070 * the same callbacks should not be retried. Return CHECK_CONDITION 2071 * if a scsi_status is not already set. 2072 * 2073 * If a fabric driver ->queue_status() has returned non zero, always 2074 * keep retrying no matter what.. 2075 */ 2076 if (cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) { 2077 if (cmd->scsi_status) 2078 goto queue_status; 2079 2080 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; 2081 cmd->scsi_status = SAM_STAT_CHECK_CONDITION; 2082 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 2083 translate_sense_reason(cmd, TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); 2084 goto queue_status; 2085 } 2086 2087 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) 2088 goto queue_status; 2089 2090 switch (cmd->data_direction) { 2091 case DMA_FROM_DEVICE: 2092 if (cmd->scsi_status) 2093 goto queue_status; 2094 2095 trace_target_cmd_complete(cmd); 2096 ret = cmd->se_tfo->queue_data_in(cmd); 2097 break; 2098 case DMA_TO_DEVICE: 2099 if (cmd->se_cmd_flags & SCF_BIDI) { 2100 ret = cmd->se_tfo->queue_data_in(cmd); 2101 break; 2102 } 2103 /* fall through */ 2104 case DMA_NONE: 2105 queue_status: 2106 trace_target_cmd_complete(cmd); 2107 ret = cmd->se_tfo->queue_status(cmd); 2108 break; 2109 default: 2110 break; 2111 } 2112 2113 if (ret < 0) { 2114 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 2115 return; 2116 } 2117 transport_lun_remove_cmd(cmd); 2118 transport_cmd_check_stop_to_fabric(cmd); 2119 } 2120 2121 static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev, 2122 int err, bool write_pending) 2123 { 2124 /* 2125 * -EAGAIN or -ENOMEM signals retry of ->write_pending() and/or 2126 * ->queue_data_in() callbacks from new process context. 2127 * 2128 * Otherwise for other errors, transport_complete_qf() will send 2129 * CHECK_CONDITION via ->queue_status() instead of attempting to 2130 * retry associated fabric driver data-transfer callbacks. 2131 */ 2132 if (err == -EAGAIN || err == -ENOMEM) { 2133 cmd->t_state = (write_pending) ? TRANSPORT_COMPLETE_QF_WP : 2134 TRANSPORT_COMPLETE_QF_OK; 2135 } else { 2136 pr_warn_ratelimited("Got unknown fabric queue status: %d\n", err); 2137 cmd->t_state = TRANSPORT_COMPLETE_QF_ERR; 2138 } 2139 2140 spin_lock_irq(&dev->qf_cmd_lock); 2141 list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list); 2142 atomic_inc_mb(&dev->dev_qf_count); 2143 spin_unlock_irq(&cmd->se_dev->qf_cmd_lock); 2144 2145 schedule_work(&cmd->se_dev->qf_work_queue); 2146 } 2147 2148 static bool target_read_prot_action(struct se_cmd *cmd) 2149 { 2150 switch (cmd->prot_op) { 2151 case TARGET_PROT_DIN_STRIP: 2152 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_STRIP)) { 2153 u32 sectors = cmd->data_length >> 2154 ilog2(cmd->se_dev->dev_attrib.block_size); 2155 2156 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba, 2157 sectors, 0, cmd->t_prot_sg, 2158 0); 2159 if (cmd->pi_err) 2160 return true; 2161 } 2162 break; 2163 case TARGET_PROT_DIN_INSERT: 2164 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_INSERT) 2165 break; 2166 2167 sbc_dif_generate(cmd); 2168 break; 2169 default: 2170 break; 2171 } 2172 2173 return false; 2174 } 2175 2176 static void target_complete_ok_work(struct work_struct *work) 2177 { 2178 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 2179 int ret; 2180 2181 /* 2182 * Check if we need to move delayed/dormant tasks from cmds on the 2183 * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task 2184 * Attribute. 2185 */ 2186 transport_complete_task_attr(cmd); 2187 2188 /* 2189 * Check to schedule QUEUE_FULL work, or execute an existing 2190 * cmd->transport_qf_callback() 2191 */ 2192 if (atomic_read(&cmd->se_dev->dev_qf_count) != 0) 2193 schedule_work(&cmd->se_dev->qf_work_queue); 2194 2195 /* 2196 * Check if we need to send a sense buffer from 2197 * the struct se_cmd in question. 2198 */ 2199 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) { 2200 WARN_ON(!cmd->scsi_status); 2201 ret = transport_send_check_condition_and_sense( 2202 cmd, 0, 1); 2203 if (ret) 2204 goto queue_full; 2205 2206 transport_lun_remove_cmd(cmd); 2207 transport_cmd_check_stop_to_fabric(cmd); 2208 return; 2209 } 2210 /* 2211 * Check for a callback, used by amongst other things 2212 * XDWRITE_READ_10 and COMPARE_AND_WRITE emulation. 2213 */ 2214 if (cmd->transport_complete_callback) { 2215 sense_reason_t rc; 2216 bool caw = (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE); 2217 bool zero_dl = !(cmd->data_length); 2218 int post_ret = 0; 2219 2220 rc = cmd->transport_complete_callback(cmd, true, &post_ret); 2221 if (!rc && !post_ret) { 2222 if (caw && zero_dl) 2223 goto queue_rsp; 2224 2225 return; 2226 } else if (rc) { 2227 ret = transport_send_check_condition_and_sense(cmd, 2228 rc, 0); 2229 if (ret) 2230 goto queue_full; 2231 2232 transport_lun_remove_cmd(cmd); 2233 transport_cmd_check_stop_to_fabric(cmd); 2234 return; 2235 } 2236 } 2237 2238 queue_rsp: 2239 switch (cmd->data_direction) { 2240 case DMA_FROM_DEVICE: 2241 if (cmd->scsi_status) 2242 goto queue_status; 2243 2244 atomic_long_add(cmd->data_length, 2245 &cmd->se_lun->lun_stats.tx_data_octets); 2246 /* 2247 * Perform READ_STRIP of PI using software emulation when 2248 * backend had PI enabled, if the transport will not be 2249 * performing hardware READ_STRIP offload. 2250 */ 2251 if (target_read_prot_action(cmd)) { 2252 ret = transport_send_check_condition_and_sense(cmd, 2253 cmd->pi_err, 0); 2254 if (ret) 2255 goto queue_full; 2256 2257 transport_lun_remove_cmd(cmd); 2258 transport_cmd_check_stop_to_fabric(cmd); 2259 return; 2260 } 2261 2262 trace_target_cmd_complete(cmd); 2263 ret = cmd->se_tfo->queue_data_in(cmd); 2264 if (ret) 2265 goto queue_full; 2266 break; 2267 case DMA_TO_DEVICE: 2268 atomic_long_add(cmd->data_length, 2269 &cmd->se_lun->lun_stats.rx_data_octets); 2270 /* 2271 * Check if we need to send READ payload for BIDI-COMMAND 2272 */ 2273 if (cmd->se_cmd_flags & SCF_BIDI) { 2274 atomic_long_add(cmd->data_length, 2275 &cmd->se_lun->lun_stats.tx_data_octets); 2276 ret = cmd->se_tfo->queue_data_in(cmd); 2277 if (ret) 2278 goto queue_full; 2279 break; 2280 } 2281 /* fall through */ 2282 case DMA_NONE: 2283 queue_status: 2284 trace_target_cmd_complete(cmd); 2285 ret = cmd->se_tfo->queue_status(cmd); 2286 if (ret) 2287 goto queue_full; 2288 break; 2289 default: 2290 break; 2291 } 2292 2293 transport_lun_remove_cmd(cmd); 2294 transport_cmd_check_stop_to_fabric(cmd); 2295 return; 2296 2297 queue_full: 2298 pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p," 2299 " data_direction: %d\n", cmd, cmd->data_direction); 2300 2301 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 2302 } 2303 2304 void target_free_sgl(struct scatterlist *sgl, int nents) 2305 { 2306 sgl_free_n_order(sgl, nents, 0); 2307 } 2308 EXPORT_SYMBOL(target_free_sgl); 2309 2310 static inline void transport_reset_sgl_orig(struct se_cmd *cmd) 2311 { 2312 /* 2313 * Check for saved t_data_sg that may be used for COMPARE_AND_WRITE 2314 * emulation, and free + reset pointers if necessary.. 2315 */ 2316 if (!cmd->t_data_sg_orig) 2317 return; 2318 2319 kfree(cmd->t_data_sg); 2320 cmd->t_data_sg = cmd->t_data_sg_orig; 2321 cmd->t_data_sg_orig = NULL; 2322 cmd->t_data_nents = cmd->t_data_nents_orig; 2323 cmd->t_data_nents_orig = 0; 2324 } 2325 2326 static inline void transport_free_pages(struct se_cmd *cmd) 2327 { 2328 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) { 2329 target_free_sgl(cmd->t_prot_sg, cmd->t_prot_nents); 2330 cmd->t_prot_sg = NULL; 2331 cmd->t_prot_nents = 0; 2332 } 2333 2334 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 2335 /* 2336 * Release special case READ buffer payload required for 2337 * SG_TO_MEM_NOALLOC to function with COMPARE_AND_WRITE 2338 */ 2339 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) { 2340 target_free_sgl(cmd->t_bidi_data_sg, 2341 cmd->t_bidi_data_nents); 2342 cmd->t_bidi_data_sg = NULL; 2343 cmd->t_bidi_data_nents = 0; 2344 } 2345 transport_reset_sgl_orig(cmd); 2346 return; 2347 } 2348 transport_reset_sgl_orig(cmd); 2349 2350 target_free_sgl(cmd->t_data_sg, cmd->t_data_nents); 2351 cmd->t_data_sg = NULL; 2352 cmd->t_data_nents = 0; 2353 2354 target_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents); 2355 cmd->t_bidi_data_sg = NULL; 2356 cmd->t_bidi_data_nents = 0; 2357 } 2358 2359 void *transport_kmap_data_sg(struct se_cmd *cmd) 2360 { 2361 struct scatterlist *sg = cmd->t_data_sg; 2362 struct page **pages; 2363 int i; 2364 2365 /* 2366 * We need to take into account a possible offset here for fabrics like 2367 * tcm_loop who may be using a contig buffer from the SCSI midlayer for 2368 * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd() 2369 */ 2370 if (!cmd->t_data_nents) 2371 return NULL; 2372 2373 BUG_ON(!sg); 2374 if (cmd->t_data_nents == 1) 2375 return kmap(sg_page(sg)) + sg->offset; 2376 2377 /* >1 page. use vmap */ 2378 pages = kmalloc_array(cmd->t_data_nents, sizeof(*pages), GFP_KERNEL); 2379 if (!pages) 2380 return NULL; 2381 2382 /* convert sg[] to pages[] */ 2383 for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) { 2384 pages[i] = sg_page(sg); 2385 } 2386 2387 cmd->t_data_vmap = vmap(pages, cmd->t_data_nents, VM_MAP, PAGE_KERNEL); 2388 kfree(pages); 2389 if (!cmd->t_data_vmap) 2390 return NULL; 2391 2392 return cmd->t_data_vmap + cmd->t_data_sg[0].offset; 2393 } 2394 EXPORT_SYMBOL(transport_kmap_data_sg); 2395 2396 void transport_kunmap_data_sg(struct se_cmd *cmd) 2397 { 2398 if (!cmd->t_data_nents) { 2399 return; 2400 } else if (cmd->t_data_nents == 1) { 2401 kunmap(sg_page(cmd->t_data_sg)); 2402 return; 2403 } 2404 2405 vunmap(cmd->t_data_vmap); 2406 cmd->t_data_vmap = NULL; 2407 } 2408 EXPORT_SYMBOL(transport_kunmap_data_sg); 2409 2410 int 2411 target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length, 2412 bool zero_page, bool chainable) 2413 { 2414 gfp_t gfp = GFP_KERNEL | (zero_page ? __GFP_ZERO : 0); 2415 2416 *sgl = sgl_alloc_order(length, 0, chainable, gfp, nents); 2417 return *sgl ? 0 : -ENOMEM; 2418 } 2419 EXPORT_SYMBOL(target_alloc_sgl); 2420 2421 /* 2422 * Allocate any required resources to execute the command. For writes we 2423 * might not have the payload yet, so notify the fabric via a call to 2424 * ->write_pending instead. Otherwise place it on the execution queue. 2425 */ 2426 sense_reason_t 2427 transport_generic_new_cmd(struct se_cmd *cmd) 2428 { 2429 unsigned long flags; 2430 int ret = 0; 2431 bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB); 2432 2433 if (cmd->prot_op != TARGET_PROT_NORMAL && 2434 !(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) { 2435 ret = target_alloc_sgl(&cmd->t_prot_sg, &cmd->t_prot_nents, 2436 cmd->prot_length, true, false); 2437 if (ret < 0) 2438 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2439 } 2440 2441 /* 2442 * Determine is the TCM fabric module has already allocated physical 2443 * memory, and is directly calling transport_generic_map_mem_to_cmd() 2444 * beforehand. 2445 */ 2446 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) && 2447 cmd->data_length) { 2448 2449 if ((cmd->se_cmd_flags & SCF_BIDI) || 2450 (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)) { 2451 u32 bidi_length; 2452 2453 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) 2454 bidi_length = cmd->t_task_nolb * 2455 cmd->se_dev->dev_attrib.block_size; 2456 else 2457 bidi_length = cmd->data_length; 2458 2459 ret = target_alloc_sgl(&cmd->t_bidi_data_sg, 2460 &cmd->t_bidi_data_nents, 2461 bidi_length, zero_flag, false); 2462 if (ret < 0) 2463 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2464 } 2465 2466 ret = target_alloc_sgl(&cmd->t_data_sg, &cmd->t_data_nents, 2467 cmd->data_length, zero_flag, false); 2468 if (ret < 0) 2469 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2470 } else if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) && 2471 cmd->data_length) { 2472 /* 2473 * Special case for COMPARE_AND_WRITE with fabrics 2474 * using SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC. 2475 */ 2476 u32 caw_length = cmd->t_task_nolb * 2477 cmd->se_dev->dev_attrib.block_size; 2478 2479 ret = target_alloc_sgl(&cmd->t_bidi_data_sg, 2480 &cmd->t_bidi_data_nents, 2481 caw_length, zero_flag, false); 2482 if (ret < 0) 2483 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2484 } 2485 /* 2486 * If this command is not a write we can execute it right here, 2487 * for write buffers we need to notify the fabric driver first 2488 * and let it call back once the write buffers are ready. 2489 */ 2490 target_add_to_state_list(cmd); 2491 if (cmd->data_direction != DMA_TO_DEVICE || cmd->data_length == 0) { 2492 target_execute_cmd(cmd); 2493 return 0; 2494 } 2495 2496 spin_lock_irqsave(&cmd->t_state_lock, flags); 2497 cmd->t_state = TRANSPORT_WRITE_PENDING; 2498 /* 2499 * Determine if frontend context caller is requesting the stopping of 2500 * this command for frontend exceptions. 2501 */ 2502 if (cmd->transport_state & CMD_T_STOP) { 2503 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 2504 __func__, __LINE__, cmd->tag); 2505 2506 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2507 2508 complete_all(&cmd->t_transport_stop_comp); 2509 return 0; 2510 } 2511 cmd->transport_state &= ~CMD_T_ACTIVE; 2512 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2513 2514 ret = cmd->se_tfo->write_pending(cmd); 2515 if (ret) 2516 goto queue_full; 2517 2518 return 0; 2519 2520 queue_full: 2521 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd); 2522 transport_handle_queue_full(cmd, cmd->se_dev, ret, true); 2523 return 0; 2524 } 2525 EXPORT_SYMBOL(transport_generic_new_cmd); 2526 2527 static void transport_write_pending_qf(struct se_cmd *cmd) 2528 { 2529 unsigned long flags; 2530 int ret; 2531 bool stop; 2532 2533 spin_lock_irqsave(&cmd->t_state_lock, flags); 2534 stop = (cmd->transport_state & (CMD_T_STOP | CMD_T_ABORTED)); 2535 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2536 2537 if (stop) { 2538 pr_debug("%s:%d CMD_T_STOP|CMD_T_ABORTED for ITT: 0x%08llx\n", 2539 __func__, __LINE__, cmd->tag); 2540 complete_all(&cmd->t_transport_stop_comp); 2541 return; 2542 } 2543 2544 ret = cmd->se_tfo->write_pending(cmd); 2545 if (ret) { 2546 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", 2547 cmd); 2548 transport_handle_queue_full(cmd, cmd->se_dev, ret, true); 2549 } 2550 } 2551 2552 static bool 2553 __transport_wait_for_tasks(struct se_cmd *, bool, bool *, bool *, 2554 unsigned long *flags); 2555 2556 static void target_wait_free_cmd(struct se_cmd *cmd, bool *aborted, bool *tas) 2557 { 2558 unsigned long flags; 2559 2560 spin_lock_irqsave(&cmd->t_state_lock, flags); 2561 __transport_wait_for_tasks(cmd, true, aborted, tas, &flags); 2562 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2563 } 2564 2565 int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks) 2566 { 2567 int ret = 0; 2568 bool aborted = false, tas = false; 2569 2570 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) { 2571 if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2572 target_wait_free_cmd(cmd, &aborted, &tas); 2573 2574 if (!aborted || tas) 2575 ret = target_put_sess_cmd(cmd); 2576 } else { 2577 if (wait_for_tasks) 2578 target_wait_free_cmd(cmd, &aborted, &tas); 2579 /* 2580 * Handle WRITE failure case where transport_generic_new_cmd() 2581 * has already added se_cmd to state_list, but fabric has 2582 * failed command before I/O submission. 2583 */ 2584 if (cmd->state_active) 2585 target_remove_from_state_list(cmd); 2586 2587 if (cmd->se_lun) 2588 transport_lun_remove_cmd(cmd); 2589 2590 if (!aborted || tas) 2591 ret = target_put_sess_cmd(cmd); 2592 } 2593 /* 2594 * If the task has been internally aborted due to TMR ABORT_TASK 2595 * or LUN_RESET, target_core_tmr.c is responsible for performing 2596 * the remaining calls to target_put_sess_cmd(), and not the 2597 * callers of this function. 2598 */ 2599 if (aborted) { 2600 pr_debug("Detected CMD_T_ABORTED for ITT: %llu\n", cmd->tag); 2601 wait_for_completion(&cmd->cmd_wait_comp); 2602 cmd->se_tfo->release_cmd(cmd); 2603 ret = 1; 2604 } 2605 return ret; 2606 } 2607 EXPORT_SYMBOL(transport_generic_free_cmd); 2608 2609 /* target_get_sess_cmd - Add command to active ->sess_cmd_list 2610 * @se_cmd: command descriptor to add 2611 * @ack_kref: Signal that fabric will perform an ack target_put_sess_cmd() 2612 */ 2613 int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref) 2614 { 2615 struct se_session *se_sess = se_cmd->se_sess; 2616 unsigned long flags; 2617 int ret = 0; 2618 2619 /* 2620 * Add a second kref if the fabric caller is expecting to handle 2621 * fabric acknowledgement that requires two target_put_sess_cmd() 2622 * invocations before se_cmd descriptor release. 2623 */ 2624 if (ack_kref) { 2625 if (!kref_get_unless_zero(&se_cmd->cmd_kref)) 2626 return -EINVAL; 2627 2628 se_cmd->se_cmd_flags |= SCF_ACK_KREF; 2629 } 2630 2631 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2632 if (se_sess->sess_tearing_down) { 2633 ret = -ESHUTDOWN; 2634 goto out; 2635 } 2636 se_cmd->transport_state |= CMD_T_PRE_EXECUTE; 2637 list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list); 2638 out: 2639 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2640 2641 if (ret && ack_kref) 2642 target_put_sess_cmd(se_cmd); 2643 2644 return ret; 2645 } 2646 EXPORT_SYMBOL(target_get_sess_cmd); 2647 2648 static void target_free_cmd_mem(struct se_cmd *cmd) 2649 { 2650 transport_free_pages(cmd); 2651 2652 if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) 2653 core_tmr_release_req(cmd->se_tmr_req); 2654 if (cmd->t_task_cdb != cmd->__t_task_cdb) 2655 kfree(cmd->t_task_cdb); 2656 } 2657 2658 static void target_release_cmd_kref(struct kref *kref) 2659 { 2660 struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref); 2661 struct se_session *se_sess = se_cmd->se_sess; 2662 unsigned long flags; 2663 bool fabric_stop; 2664 2665 if (se_sess) { 2666 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2667 2668 spin_lock(&se_cmd->t_state_lock); 2669 fabric_stop = (se_cmd->transport_state & CMD_T_FABRIC_STOP) && 2670 (se_cmd->transport_state & CMD_T_ABORTED); 2671 spin_unlock(&se_cmd->t_state_lock); 2672 2673 if (se_cmd->cmd_wait_set || fabric_stop) { 2674 list_del_init(&se_cmd->se_cmd_list); 2675 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2676 target_free_cmd_mem(se_cmd); 2677 complete(&se_cmd->cmd_wait_comp); 2678 return; 2679 } 2680 list_del_init(&se_cmd->se_cmd_list); 2681 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2682 } 2683 2684 target_free_cmd_mem(se_cmd); 2685 se_cmd->se_tfo->release_cmd(se_cmd); 2686 } 2687 2688 /** 2689 * target_put_sess_cmd - decrease the command reference count 2690 * @se_cmd: command to drop a reference from 2691 * 2692 * Returns 1 if and only if this target_put_sess_cmd() call caused the 2693 * refcount to drop to zero. Returns zero otherwise. 2694 */ 2695 int target_put_sess_cmd(struct se_cmd *se_cmd) 2696 { 2697 return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref); 2698 } 2699 EXPORT_SYMBOL(target_put_sess_cmd); 2700 2701 static const char *data_dir_name(enum dma_data_direction d) 2702 { 2703 switch (d) { 2704 case DMA_BIDIRECTIONAL: return "BIDI"; 2705 case DMA_TO_DEVICE: return "WRITE"; 2706 case DMA_FROM_DEVICE: return "READ"; 2707 case DMA_NONE: return "NONE"; 2708 } 2709 2710 return "(?)"; 2711 } 2712 2713 static const char *cmd_state_name(enum transport_state_table t) 2714 { 2715 switch (t) { 2716 case TRANSPORT_NO_STATE: return "NO_STATE"; 2717 case TRANSPORT_NEW_CMD: return "NEW_CMD"; 2718 case TRANSPORT_WRITE_PENDING: return "WRITE_PENDING"; 2719 case TRANSPORT_PROCESSING: return "PROCESSING"; 2720 case TRANSPORT_COMPLETE: return "COMPLETE"; 2721 case TRANSPORT_ISTATE_PROCESSING: 2722 return "ISTATE_PROCESSING"; 2723 case TRANSPORT_COMPLETE_QF_WP: return "COMPLETE_QF_WP"; 2724 case TRANSPORT_COMPLETE_QF_OK: return "COMPLETE_QF_OK"; 2725 case TRANSPORT_COMPLETE_QF_ERR: return "COMPLETE_QF_ERR"; 2726 } 2727 2728 return "(?)"; 2729 } 2730 2731 static void target_append_str(char **str, const char *txt) 2732 { 2733 char *prev = *str; 2734 2735 *str = *str ? kasprintf(GFP_ATOMIC, "%s,%s", *str, txt) : 2736 kstrdup(txt, GFP_ATOMIC); 2737 kfree(prev); 2738 } 2739 2740 /* 2741 * Convert a transport state bitmask into a string. The caller is 2742 * responsible for freeing the returned pointer. 2743 */ 2744 static char *target_ts_to_str(u32 ts) 2745 { 2746 char *str = NULL; 2747 2748 if (ts & CMD_T_ABORTED) 2749 target_append_str(&str, "aborted"); 2750 if (ts & CMD_T_ACTIVE) 2751 target_append_str(&str, "active"); 2752 if (ts & CMD_T_COMPLETE) 2753 target_append_str(&str, "complete"); 2754 if (ts & CMD_T_SENT) 2755 target_append_str(&str, "sent"); 2756 if (ts & CMD_T_STOP) 2757 target_append_str(&str, "stop"); 2758 if (ts & CMD_T_FABRIC_STOP) 2759 target_append_str(&str, "fabric_stop"); 2760 2761 return str; 2762 } 2763 2764 static const char *target_tmf_name(enum tcm_tmreq_table tmf) 2765 { 2766 switch (tmf) { 2767 case TMR_ABORT_TASK: return "ABORT_TASK"; 2768 case TMR_ABORT_TASK_SET: return "ABORT_TASK_SET"; 2769 case TMR_CLEAR_ACA: return "CLEAR_ACA"; 2770 case TMR_CLEAR_TASK_SET: return "CLEAR_TASK_SET"; 2771 case TMR_LUN_RESET: return "LUN_RESET"; 2772 case TMR_TARGET_WARM_RESET: return "TARGET_WARM_RESET"; 2773 case TMR_TARGET_COLD_RESET: return "TARGET_COLD_RESET"; 2774 case TMR_UNKNOWN: break; 2775 } 2776 return "(?)"; 2777 } 2778 2779 void target_show_cmd(const char *pfx, struct se_cmd *cmd) 2780 { 2781 char *ts_str = target_ts_to_str(cmd->transport_state); 2782 const u8 *cdb = cmd->t_task_cdb; 2783 struct se_tmr_req *tmf = cmd->se_tmr_req; 2784 2785 if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) { 2786 pr_debug("%scmd %#02x:%#02x with tag %#llx dir %s i_state %d t_state %s len %d refcnt %d transport_state %s\n", 2787 pfx, cdb[0], cdb[1], cmd->tag, 2788 data_dir_name(cmd->data_direction), 2789 cmd->se_tfo->get_cmd_state(cmd), 2790 cmd_state_name(cmd->t_state), cmd->data_length, 2791 kref_read(&cmd->cmd_kref), ts_str); 2792 } else { 2793 pr_debug("%stmf %s with tag %#llx ref_task_tag %#llx i_state %d t_state %s refcnt %d transport_state %s\n", 2794 pfx, target_tmf_name(tmf->function), cmd->tag, 2795 tmf->ref_task_tag, cmd->se_tfo->get_cmd_state(cmd), 2796 cmd_state_name(cmd->t_state), 2797 kref_read(&cmd->cmd_kref), ts_str); 2798 } 2799 kfree(ts_str); 2800 } 2801 EXPORT_SYMBOL(target_show_cmd); 2802 2803 /* target_sess_cmd_list_set_waiting - Flag all commands in 2804 * sess_cmd_list to complete cmd_wait_comp. Set 2805 * sess_tearing_down so no more commands are queued. 2806 * @se_sess: session to flag 2807 */ 2808 void target_sess_cmd_list_set_waiting(struct se_session *se_sess) 2809 { 2810 struct se_cmd *se_cmd, *tmp_cmd; 2811 unsigned long flags; 2812 int rc; 2813 2814 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2815 if (se_sess->sess_tearing_down) { 2816 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2817 return; 2818 } 2819 se_sess->sess_tearing_down = 1; 2820 list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list); 2821 2822 list_for_each_entry_safe(se_cmd, tmp_cmd, 2823 &se_sess->sess_wait_list, se_cmd_list) { 2824 rc = kref_get_unless_zero(&se_cmd->cmd_kref); 2825 if (rc) { 2826 se_cmd->cmd_wait_set = 1; 2827 spin_lock(&se_cmd->t_state_lock); 2828 se_cmd->transport_state |= CMD_T_FABRIC_STOP; 2829 spin_unlock(&se_cmd->t_state_lock); 2830 } else 2831 list_del_init(&se_cmd->se_cmd_list); 2832 } 2833 2834 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2835 } 2836 EXPORT_SYMBOL(target_sess_cmd_list_set_waiting); 2837 2838 /* target_wait_for_sess_cmds - Wait for outstanding descriptors 2839 * @se_sess: session to wait for active I/O 2840 */ 2841 void target_wait_for_sess_cmds(struct se_session *se_sess) 2842 { 2843 struct se_cmd *se_cmd, *tmp_cmd; 2844 unsigned long flags; 2845 bool tas; 2846 2847 list_for_each_entry_safe(se_cmd, tmp_cmd, 2848 &se_sess->sess_wait_list, se_cmd_list) { 2849 pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:" 2850 " %d\n", se_cmd, se_cmd->t_state, 2851 se_cmd->se_tfo->get_cmd_state(se_cmd)); 2852 2853 spin_lock_irqsave(&se_cmd->t_state_lock, flags); 2854 tas = (se_cmd->transport_state & CMD_T_TAS); 2855 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); 2856 2857 if (!target_put_sess_cmd(se_cmd)) { 2858 if (tas) 2859 target_put_sess_cmd(se_cmd); 2860 } 2861 2862 wait_for_completion(&se_cmd->cmd_wait_comp); 2863 pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d" 2864 " fabric state: %d\n", se_cmd, se_cmd->t_state, 2865 se_cmd->se_tfo->get_cmd_state(se_cmd)); 2866 2867 se_cmd->se_tfo->release_cmd(se_cmd); 2868 } 2869 2870 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2871 WARN_ON(!list_empty(&se_sess->sess_cmd_list)); 2872 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2873 2874 } 2875 EXPORT_SYMBOL(target_wait_for_sess_cmds); 2876 2877 static void target_lun_confirm(struct percpu_ref *ref) 2878 { 2879 struct se_lun *lun = container_of(ref, struct se_lun, lun_ref); 2880 2881 complete(&lun->lun_ref_comp); 2882 } 2883 2884 void transport_clear_lun_ref(struct se_lun *lun) 2885 { 2886 /* 2887 * Mark the percpu-ref as DEAD, switch to atomic_t mode, drop 2888 * the initial reference and schedule confirm kill to be 2889 * executed after one full RCU grace period has completed. 2890 */ 2891 percpu_ref_kill_and_confirm(&lun->lun_ref, target_lun_confirm); 2892 /* 2893 * The first completion waits for percpu_ref_switch_to_atomic_rcu() 2894 * to call target_lun_confirm after lun->lun_ref has been marked 2895 * as __PERCPU_REF_DEAD on all CPUs, and switches to atomic_t 2896 * mode so that percpu_ref_tryget_live() lookup of lun->lun_ref 2897 * fails for all new incoming I/O. 2898 */ 2899 wait_for_completion(&lun->lun_ref_comp); 2900 /* 2901 * The second completion waits for percpu_ref_put_many() to 2902 * invoke ->release() after lun->lun_ref has switched to 2903 * atomic_t mode, and lun->lun_ref.count has reached zero. 2904 * 2905 * At this point all target-core lun->lun_ref references have 2906 * been dropped via transport_lun_remove_cmd(), and it's safe 2907 * to proceed with the remaining LUN shutdown. 2908 */ 2909 wait_for_completion(&lun->lun_shutdown_comp); 2910 } 2911 2912 static bool 2913 __transport_wait_for_tasks(struct se_cmd *cmd, bool fabric_stop, 2914 bool *aborted, bool *tas, unsigned long *flags) 2915 __releases(&cmd->t_state_lock) 2916 __acquires(&cmd->t_state_lock) 2917 { 2918 2919 assert_spin_locked(&cmd->t_state_lock); 2920 WARN_ON_ONCE(!irqs_disabled()); 2921 2922 if (fabric_stop) 2923 cmd->transport_state |= CMD_T_FABRIC_STOP; 2924 2925 if (cmd->transport_state & CMD_T_ABORTED) 2926 *aborted = true; 2927 2928 if (cmd->transport_state & CMD_T_TAS) 2929 *tas = true; 2930 2931 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) && 2932 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2933 return false; 2934 2935 if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) && 2936 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2937 return false; 2938 2939 if (!(cmd->transport_state & CMD_T_ACTIVE)) 2940 return false; 2941 2942 if (fabric_stop && *aborted) 2943 return false; 2944 2945 cmd->transport_state |= CMD_T_STOP; 2946 2947 target_show_cmd("wait_for_tasks: Stopping ", cmd); 2948 2949 spin_unlock_irqrestore(&cmd->t_state_lock, *flags); 2950 2951 while (!wait_for_completion_timeout(&cmd->t_transport_stop_comp, 2952 180 * HZ)) 2953 target_show_cmd("wait for tasks: ", cmd); 2954 2955 spin_lock_irqsave(&cmd->t_state_lock, *flags); 2956 cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP); 2957 2958 pr_debug("wait_for_tasks: Stopped wait_for_completion(&cmd->" 2959 "t_transport_stop_comp) for ITT: 0x%08llx\n", cmd->tag); 2960 2961 return true; 2962 } 2963 2964 /** 2965 * transport_wait_for_tasks - set CMD_T_STOP and wait for t_transport_stop_comp 2966 * @cmd: command to wait on 2967 */ 2968 bool transport_wait_for_tasks(struct se_cmd *cmd) 2969 { 2970 unsigned long flags; 2971 bool ret, aborted = false, tas = false; 2972 2973 spin_lock_irqsave(&cmd->t_state_lock, flags); 2974 ret = __transport_wait_for_tasks(cmd, false, &aborted, &tas, &flags); 2975 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2976 2977 return ret; 2978 } 2979 EXPORT_SYMBOL(transport_wait_for_tasks); 2980 2981 struct sense_info { 2982 u8 key; 2983 u8 asc; 2984 u8 ascq; 2985 bool add_sector_info; 2986 }; 2987 2988 static const struct sense_info sense_info_table[] = { 2989 [TCM_NO_SENSE] = { 2990 .key = NOT_READY 2991 }, 2992 [TCM_NON_EXISTENT_LUN] = { 2993 .key = ILLEGAL_REQUEST, 2994 .asc = 0x25 /* LOGICAL UNIT NOT SUPPORTED */ 2995 }, 2996 [TCM_UNSUPPORTED_SCSI_OPCODE] = { 2997 .key = ILLEGAL_REQUEST, 2998 .asc = 0x20, /* INVALID COMMAND OPERATION CODE */ 2999 }, 3000 [TCM_SECTOR_COUNT_TOO_MANY] = { 3001 .key = ILLEGAL_REQUEST, 3002 .asc = 0x20, /* INVALID COMMAND OPERATION CODE */ 3003 }, 3004 [TCM_UNKNOWN_MODE_PAGE] = { 3005 .key = ILLEGAL_REQUEST, 3006 .asc = 0x24, /* INVALID FIELD IN CDB */ 3007 }, 3008 [TCM_CHECK_CONDITION_ABORT_CMD] = { 3009 .key = ABORTED_COMMAND, 3010 .asc = 0x29, /* BUS DEVICE RESET FUNCTION OCCURRED */ 3011 .ascq = 0x03, 3012 }, 3013 [TCM_INCORRECT_AMOUNT_OF_DATA] = { 3014 .key = ABORTED_COMMAND, 3015 .asc = 0x0c, /* WRITE ERROR */ 3016 .ascq = 0x0d, /* NOT ENOUGH UNSOLICITED DATA */ 3017 }, 3018 [TCM_INVALID_CDB_FIELD] = { 3019 .key = ILLEGAL_REQUEST, 3020 .asc = 0x24, /* INVALID FIELD IN CDB */ 3021 }, 3022 [TCM_INVALID_PARAMETER_LIST] = { 3023 .key = ILLEGAL_REQUEST, 3024 .asc = 0x26, /* INVALID FIELD IN PARAMETER LIST */ 3025 }, 3026 [TCM_TOO_MANY_TARGET_DESCS] = { 3027 .key = ILLEGAL_REQUEST, 3028 .asc = 0x26, 3029 .ascq = 0x06, /* TOO MANY TARGET DESCRIPTORS */ 3030 }, 3031 [TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE] = { 3032 .key = ILLEGAL_REQUEST, 3033 .asc = 0x26, 3034 .ascq = 0x07, /* UNSUPPORTED TARGET DESCRIPTOR TYPE CODE */ 3035 }, 3036 [TCM_TOO_MANY_SEGMENT_DESCS] = { 3037 .key = ILLEGAL_REQUEST, 3038 .asc = 0x26, 3039 .ascq = 0x08, /* TOO MANY SEGMENT DESCRIPTORS */ 3040 }, 3041 [TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE] = { 3042 .key = ILLEGAL_REQUEST, 3043 .asc = 0x26, 3044 .ascq = 0x09, /* UNSUPPORTED SEGMENT DESCRIPTOR TYPE CODE */ 3045 }, 3046 [TCM_PARAMETER_LIST_LENGTH_ERROR] = { 3047 .key = ILLEGAL_REQUEST, 3048 .asc = 0x1a, /* PARAMETER LIST LENGTH ERROR */ 3049 }, 3050 [TCM_UNEXPECTED_UNSOLICITED_DATA] = { 3051 .key = ILLEGAL_REQUEST, 3052 .asc = 0x0c, /* WRITE ERROR */ 3053 .ascq = 0x0c, /* UNEXPECTED_UNSOLICITED_DATA */ 3054 }, 3055 [TCM_SERVICE_CRC_ERROR] = { 3056 .key = ABORTED_COMMAND, 3057 .asc = 0x47, /* PROTOCOL SERVICE CRC ERROR */ 3058 .ascq = 0x05, /* N/A */ 3059 }, 3060 [TCM_SNACK_REJECTED] = { 3061 .key = ABORTED_COMMAND, 3062 .asc = 0x11, /* READ ERROR */ 3063 .ascq = 0x13, /* FAILED RETRANSMISSION REQUEST */ 3064 }, 3065 [TCM_WRITE_PROTECTED] = { 3066 .key = DATA_PROTECT, 3067 .asc = 0x27, /* WRITE PROTECTED */ 3068 }, 3069 [TCM_ADDRESS_OUT_OF_RANGE] = { 3070 .key = ILLEGAL_REQUEST, 3071 .asc = 0x21, /* LOGICAL BLOCK ADDRESS OUT OF RANGE */ 3072 }, 3073 [TCM_CHECK_CONDITION_UNIT_ATTENTION] = { 3074 .key = UNIT_ATTENTION, 3075 }, 3076 [TCM_CHECK_CONDITION_NOT_READY] = { 3077 .key = NOT_READY, 3078 }, 3079 [TCM_MISCOMPARE_VERIFY] = { 3080 .key = MISCOMPARE, 3081 .asc = 0x1d, /* MISCOMPARE DURING VERIFY OPERATION */ 3082 .ascq = 0x00, 3083 }, 3084 [TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED] = { 3085 .key = ABORTED_COMMAND, 3086 .asc = 0x10, 3087 .ascq = 0x01, /* LOGICAL BLOCK GUARD CHECK FAILED */ 3088 .add_sector_info = true, 3089 }, 3090 [TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED] = { 3091 .key = ABORTED_COMMAND, 3092 .asc = 0x10, 3093 .ascq = 0x02, /* LOGICAL BLOCK APPLICATION TAG CHECK FAILED */ 3094 .add_sector_info = true, 3095 }, 3096 [TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED] = { 3097 .key = ABORTED_COMMAND, 3098 .asc = 0x10, 3099 .ascq = 0x03, /* LOGICAL BLOCK REFERENCE TAG CHECK FAILED */ 3100 .add_sector_info = true, 3101 }, 3102 [TCM_COPY_TARGET_DEVICE_NOT_REACHABLE] = { 3103 .key = COPY_ABORTED, 3104 .asc = 0x0d, 3105 .ascq = 0x02, /* COPY TARGET DEVICE NOT REACHABLE */ 3106 3107 }, 3108 [TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE] = { 3109 /* 3110 * Returning ILLEGAL REQUEST would cause immediate IO errors on 3111 * Solaris initiators. Returning NOT READY instead means the 3112 * operations will be retried a finite number of times and we 3113 * can survive intermittent errors. 3114 */ 3115 .key = NOT_READY, 3116 .asc = 0x08, /* LOGICAL UNIT COMMUNICATION FAILURE */ 3117 }, 3118 [TCM_INSUFFICIENT_REGISTRATION_RESOURCES] = { 3119 /* 3120 * From spc4r22 section5.7.7,5.7.8 3121 * If a PERSISTENT RESERVE OUT command with a REGISTER service action 3122 * or a REGISTER AND IGNORE EXISTING KEY service action or 3123 * REGISTER AND MOVE service actionis attempted, 3124 * but there are insufficient device server resources to complete the 3125 * operation, then the command shall be terminated with CHECK CONDITION 3126 * status, with the sense key set to ILLEGAL REQUEST,and the additonal 3127 * sense code set to INSUFFICIENT REGISTRATION RESOURCES. 3128 */ 3129 .key = ILLEGAL_REQUEST, 3130 .asc = 0x55, 3131 .ascq = 0x04, /* INSUFFICIENT REGISTRATION RESOURCES */ 3132 }, 3133 }; 3134 3135 static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason) 3136 { 3137 const struct sense_info *si; 3138 u8 *buffer = cmd->sense_buffer; 3139 int r = (__force int)reason; 3140 u8 asc, ascq; 3141 bool desc_format = target_sense_desc_format(cmd->se_dev); 3142 3143 if (r < ARRAY_SIZE(sense_info_table) && sense_info_table[r].key) 3144 si = &sense_info_table[r]; 3145 else 3146 si = &sense_info_table[(__force int) 3147 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE]; 3148 3149 if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) { 3150 core_scsi3_ua_for_check_condition(cmd, &asc, &ascq); 3151 WARN_ON_ONCE(asc == 0); 3152 } else if (si->asc == 0) { 3153 WARN_ON_ONCE(cmd->scsi_asc == 0); 3154 asc = cmd->scsi_asc; 3155 ascq = cmd->scsi_ascq; 3156 } else { 3157 asc = si->asc; 3158 ascq = si->ascq; 3159 } 3160 3161 scsi_build_sense_buffer(desc_format, buffer, si->key, asc, ascq); 3162 if (si->add_sector_info) 3163 return scsi_set_sense_information(buffer, 3164 cmd->scsi_sense_length, 3165 cmd->bad_sector); 3166 3167 return 0; 3168 } 3169 3170 int 3171 transport_send_check_condition_and_sense(struct se_cmd *cmd, 3172 sense_reason_t reason, int from_transport) 3173 { 3174 unsigned long flags; 3175 3176 spin_lock_irqsave(&cmd->t_state_lock, flags); 3177 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) { 3178 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3179 return 0; 3180 } 3181 cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION; 3182 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3183 3184 if (!from_transport) { 3185 int rc; 3186 3187 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; 3188 cmd->scsi_status = SAM_STAT_CHECK_CONDITION; 3189 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 3190 rc = translate_sense_reason(cmd, reason); 3191 if (rc) 3192 return rc; 3193 } 3194 3195 trace_target_cmd_complete(cmd); 3196 return cmd->se_tfo->queue_status(cmd); 3197 } 3198 EXPORT_SYMBOL(transport_send_check_condition_and_sense); 3199 3200 static int __transport_check_aborted_status(struct se_cmd *cmd, int send_status) 3201 __releases(&cmd->t_state_lock) 3202 __acquires(&cmd->t_state_lock) 3203 { 3204 int ret; 3205 3206 assert_spin_locked(&cmd->t_state_lock); 3207 WARN_ON_ONCE(!irqs_disabled()); 3208 3209 if (!(cmd->transport_state & CMD_T_ABORTED)) 3210 return 0; 3211 /* 3212 * If cmd has been aborted but either no status is to be sent or it has 3213 * already been sent, just return 3214 */ 3215 if (!send_status || !(cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS)) { 3216 if (send_status) 3217 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS; 3218 return 1; 3219 } 3220 3221 pr_debug("Sending delayed SAM_STAT_TASK_ABORTED status for CDB:" 3222 " 0x%02x ITT: 0x%08llx\n", cmd->t_task_cdb[0], cmd->tag); 3223 3224 cmd->se_cmd_flags &= ~SCF_SEND_DELAYED_TAS; 3225 cmd->scsi_status = SAM_STAT_TASK_ABORTED; 3226 trace_target_cmd_complete(cmd); 3227 3228 spin_unlock_irq(&cmd->t_state_lock); 3229 ret = cmd->se_tfo->queue_status(cmd); 3230 if (ret) 3231 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 3232 spin_lock_irq(&cmd->t_state_lock); 3233 3234 return 1; 3235 } 3236 3237 int transport_check_aborted_status(struct se_cmd *cmd, int send_status) 3238 { 3239 int ret; 3240 3241 spin_lock_irq(&cmd->t_state_lock); 3242 ret = __transport_check_aborted_status(cmd, send_status); 3243 spin_unlock_irq(&cmd->t_state_lock); 3244 3245 return ret; 3246 } 3247 EXPORT_SYMBOL(transport_check_aborted_status); 3248 3249 void transport_send_task_abort(struct se_cmd *cmd) 3250 { 3251 unsigned long flags; 3252 int ret; 3253 3254 spin_lock_irqsave(&cmd->t_state_lock, flags); 3255 if (cmd->se_cmd_flags & (SCF_SENT_CHECK_CONDITION)) { 3256 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3257 return; 3258 } 3259 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3260 3261 /* 3262 * If there are still expected incoming fabric WRITEs, we wait 3263 * until until they have completed before sending a TASK_ABORTED 3264 * response. This response with TASK_ABORTED status will be 3265 * queued back to fabric module by transport_check_aborted_status(). 3266 */ 3267 if (cmd->data_direction == DMA_TO_DEVICE) { 3268 if (cmd->se_tfo->write_pending_status(cmd) != 0) { 3269 spin_lock_irqsave(&cmd->t_state_lock, flags); 3270 if (cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS) { 3271 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3272 goto send_abort; 3273 } 3274 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS; 3275 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3276 return; 3277 } 3278 } 3279 send_abort: 3280 cmd->scsi_status = SAM_STAT_TASK_ABORTED; 3281 3282 transport_lun_remove_cmd(cmd); 3283 3284 pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n", 3285 cmd->t_task_cdb[0], cmd->tag); 3286 3287 trace_target_cmd_complete(cmd); 3288 ret = cmd->se_tfo->queue_status(cmd); 3289 if (ret) 3290 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 3291 } 3292 3293 static void target_tmr_work(struct work_struct *work) 3294 { 3295 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 3296 struct se_device *dev = cmd->se_dev; 3297 struct se_tmr_req *tmr = cmd->se_tmr_req; 3298 unsigned long flags; 3299 int ret; 3300 3301 spin_lock_irqsave(&cmd->t_state_lock, flags); 3302 if (cmd->transport_state & CMD_T_ABORTED) { 3303 tmr->response = TMR_FUNCTION_REJECTED; 3304 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3305 goto check_stop; 3306 } 3307 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3308 3309 switch (tmr->function) { 3310 case TMR_ABORT_TASK: 3311 core_tmr_abort_task(dev, tmr, cmd->se_sess); 3312 break; 3313 case TMR_ABORT_TASK_SET: 3314 case TMR_CLEAR_ACA: 3315 case TMR_CLEAR_TASK_SET: 3316 tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED; 3317 break; 3318 case TMR_LUN_RESET: 3319 ret = core_tmr_lun_reset(dev, tmr, NULL, NULL); 3320 tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE : 3321 TMR_FUNCTION_REJECTED; 3322 if (tmr->response == TMR_FUNCTION_COMPLETE) { 3323 target_ua_allocate_lun(cmd->se_sess->se_node_acl, 3324 cmd->orig_fe_lun, 0x29, 3325 ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED); 3326 } 3327 break; 3328 case TMR_TARGET_WARM_RESET: 3329 tmr->response = TMR_FUNCTION_REJECTED; 3330 break; 3331 case TMR_TARGET_COLD_RESET: 3332 tmr->response = TMR_FUNCTION_REJECTED; 3333 break; 3334 default: 3335 pr_err("Uknown TMR function: 0x%02x.\n", 3336 tmr->function); 3337 tmr->response = TMR_FUNCTION_REJECTED; 3338 break; 3339 } 3340 3341 spin_lock_irqsave(&cmd->t_state_lock, flags); 3342 if (cmd->transport_state & CMD_T_ABORTED) { 3343 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3344 goto check_stop; 3345 } 3346 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3347 3348 cmd->se_tfo->queue_tm_rsp(cmd); 3349 3350 check_stop: 3351 transport_lun_remove_cmd(cmd); 3352 transport_cmd_check_stop_to_fabric(cmd); 3353 } 3354 3355 int transport_generic_handle_tmr( 3356 struct se_cmd *cmd) 3357 { 3358 unsigned long flags; 3359 bool aborted = false; 3360 3361 spin_lock_irqsave(&cmd->t_state_lock, flags); 3362 if (cmd->transport_state & CMD_T_ABORTED) { 3363 aborted = true; 3364 } else { 3365 cmd->t_state = TRANSPORT_ISTATE_PROCESSING; 3366 cmd->transport_state |= CMD_T_ACTIVE; 3367 } 3368 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3369 3370 if (aborted) { 3371 pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d" 3372 "ref_tag: %llu tag: %llu\n", cmd->se_tmr_req->function, 3373 cmd->se_tmr_req->ref_task_tag, cmd->tag); 3374 transport_lun_remove_cmd(cmd); 3375 transport_cmd_check_stop_to_fabric(cmd); 3376 return 0; 3377 } 3378 3379 INIT_WORK(&cmd->work, target_tmr_work); 3380 queue_work(cmd->se_dev->tmr_wq, &cmd->work); 3381 return 0; 3382 } 3383 EXPORT_SYMBOL(transport_generic_handle_tmr); 3384 3385 bool 3386 target_check_wce(struct se_device *dev) 3387 { 3388 bool wce = false; 3389 3390 if (dev->transport->get_write_cache) 3391 wce = dev->transport->get_write_cache(dev); 3392 else if (dev->dev_attrib.emulate_write_cache > 0) 3393 wce = true; 3394 3395 return wce; 3396 } 3397 3398 bool 3399 target_check_fua(struct se_device *dev) 3400 { 3401 return target_check_wce(dev) && dev->dev_attrib.emulate_fua_write > 0; 3402 } 3403