1 /******************************************************************************* 2 * Filename: target_core_transport.c 3 * 4 * This file contains the Generic Target Engine Core. 5 * 6 * (c) Copyright 2002-2013 Datera, Inc. 7 * 8 * Nicholas A. Bellinger <nab@kernel.org> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License as published by 12 * the Free Software Foundation; either version 2 of the License, or 13 * (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 * 20 * You should have received a copy of the GNU General Public License 21 * along with this program; if not, write to the Free Software 22 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 23 * 24 ******************************************************************************/ 25 26 #include <linux/net.h> 27 #include <linux/delay.h> 28 #include <linux/string.h> 29 #include <linux/timer.h> 30 #include <linux/slab.h> 31 #include <linux/spinlock.h> 32 #include <linux/kthread.h> 33 #include <linux/in.h> 34 #include <linux/cdrom.h> 35 #include <linux/module.h> 36 #include <linux/ratelimit.h> 37 #include <linux/vmalloc.h> 38 #include <asm/unaligned.h> 39 #include <net/sock.h> 40 #include <net/tcp.h> 41 #include <scsi/scsi_proto.h> 42 #include <scsi/scsi_common.h> 43 44 #include <target/target_core_base.h> 45 #include <target/target_core_backend.h> 46 #include <target/target_core_fabric.h> 47 48 #include "target_core_internal.h" 49 #include "target_core_alua.h" 50 #include "target_core_pr.h" 51 #include "target_core_ua.h" 52 53 #define CREATE_TRACE_POINTS 54 #include <trace/events/target.h> 55 56 static struct workqueue_struct *target_completion_wq; 57 static struct kmem_cache *se_sess_cache; 58 struct kmem_cache *se_ua_cache; 59 struct kmem_cache *t10_pr_reg_cache; 60 struct kmem_cache *t10_alua_lu_gp_cache; 61 struct kmem_cache *t10_alua_lu_gp_mem_cache; 62 struct kmem_cache *t10_alua_tg_pt_gp_cache; 63 struct kmem_cache *t10_alua_lba_map_cache; 64 struct kmem_cache *t10_alua_lba_map_mem_cache; 65 66 static void transport_complete_task_attr(struct se_cmd *cmd); 67 static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason); 68 static void transport_handle_queue_full(struct se_cmd *cmd, 69 struct se_device *dev, int err, bool write_pending); 70 static void target_complete_ok_work(struct work_struct *work); 71 72 int init_se_kmem_caches(void) 73 { 74 se_sess_cache = kmem_cache_create("se_sess_cache", 75 sizeof(struct se_session), __alignof__(struct se_session), 76 0, NULL); 77 if (!se_sess_cache) { 78 pr_err("kmem_cache_create() for struct se_session" 79 " failed\n"); 80 goto out; 81 } 82 se_ua_cache = kmem_cache_create("se_ua_cache", 83 sizeof(struct se_ua), __alignof__(struct se_ua), 84 0, NULL); 85 if (!se_ua_cache) { 86 pr_err("kmem_cache_create() for struct se_ua failed\n"); 87 goto out_free_sess_cache; 88 } 89 t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache", 90 sizeof(struct t10_pr_registration), 91 __alignof__(struct t10_pr_registration), 0, NULL); 92 if (!t10_pr_reg_cache) { 93 pr_err("kmem_cache_create() for struct t10_pr_registration" 94 " failed\n"); 95 goto out_free_ua_cache; 96 } 97 t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache", 98 sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp), 99 0, NULL); 100 if (!t10_alua_lu_gp_cache) { 101 pr_err("kmem_cache_create() for t10_alua_lu_gp_cache" 102 " failed\n"); 103 goto out_free_pr_reg_cache; 104 } 105 t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache", 106 sizeof(struct t10_alua_lu_gp_member), 107 __alignof__(struct t10_alua_lu_gp_member), 0, NULL); 108 if (!t10_alua_lu_gp_mem_cache) { 109 pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_" 110 "cache failed\n"); 111 goto out_free_lu_gp_cache; 112 } 113 t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache", 114 sizeof(struct t10_alua_tg_pt_gp), 115 __alignof__(struct t10_alua_tg_pt_gp), 0, NULL); 116 if (!t10_alua_tg_pt_gp_cache) { 117 pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_" 118 "cache failed\n"); 119 goto out_free_lu_gp_mem_cache; 120 } 121 t10_alua_lba_map_cache = kmem_cache_create( 122 "t10_alua_lba_map_cache", 123 sizeof(struct t10_alua_lba_map), 124 __alignof__(struct t10_alua_lba_map), 0, NULL); 125 if (!t10_alua_lba_map_cache) { 126 pr_err("kmem_cache_create() for t10_alua_lba_map_" 127 "cache failed\n"); 128 goto out_free_tg_pt_gp_cache; 129 } 130 t10_alua_lba_map_mem_cache = kmem_cache_create( 131 "t10_alua_lba_map_mem_cache", 132 sizeof(struct t10_alua_lba_map_member), 133 __alignof__(struct t10_alua_lba_map_member), 0, NULL); 134 if (!t10_alua_lba_map_mem_cache) { 135 pr_err("kmem_cache_create() for t10_alua_lba_map_mem_" 136 "cache failed\n"); 137 goto out_free_lba_map_cache; 138 } 139 140 target_completion_wq = alloc_workqueue("target_completion", 141 WQ_MEM_RECLAIM, 0); 142 if (!target_completion_wq) 143 goto out_free_lba_map_mem_cache; 144 145 return 0; 146 147 out_free_lba_map_mem_cache: 148 kmem_cache_destroy(t10_alua_lba_map_mem_cache); 149 out_free_lba_map_cache: 150 kmem_cache_destroy(t10_alua_lba_map_cache); 151 out_free_tg_pt_gp_cache: 152 kmem_cache_destroy(t10_alua_tg_pt_gp_cache); 153 out_free_lu_gp_mem_cache: 154 kmem_cache_destroy(t10_alua_lu_gp_mem_cache); 155 out_free_lu_gp_cache: 156 kmem_cache_destroy(t10_alua_lu_gp_cache); 157 out_free_pr_reg_cache: 158 kmem_cache_destroy(t10_pr_reg_cache); 159 out_free_ua_cache: 160 kmem_cache_destroy(se_ua_cache); 161 out_free_sess_cache: 162 kmem_cache_destroy(se_sess_cache); 163 out: 164 return -ENOMEM; 165 } 166 167 void release_se_kmem_caches(void) 168 { 169 destroy_workqueue(target_completion_wq); 170 kmem_cache_destroy(se_sess_cache); 171 kmem_cache_destroy(se_ua_cache); 172 kmem_cache_destroy(t10_pr_reg_cache); 173 kmem_cache_destroy(t10_alua_lu_gp_cache); 174 kmem_cache_destroy(t10_alua_lu_gp_mem_cache); 175 kmem_cache_destroy(t10_alua_tg_pt_gp_cache); 176 kmem_cache_destroy(t10_alua_lba_map_cache); 177 kmem_cache_destroy(t10_alua_lba_map_mem_cache); 178 } 179 180 /* This code ensures unique mib indexes are handed out. */ 181 static DEFINE_SPINLOCK(scsi_mib_index_lock); 182 static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX]; 183 184 /* 185 * Allocate a new row index for the entry type specified 186 */ 187 u32 scsi_get_new_index(scsi_index_t type) 188 { 189 u32 new_index; 190 191 BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX)); 192 193 spin_lock(&scsi_mib_index_lock); 194 new_index = ++scsi_mib_index[type]; 195 spin_unlock(&scsi_mib_index_lock); 196 197 return new_index; 198 } 199 200 void transport_subsystem_check_init(void) 201 { 202 int ret; 203 static int sub_api_initialized; 204 205 if (sub_api_initialized) 206 return; 207 208 ret = request_module("target_core_iblock"); 209 if (ret != 0) 210 pr_err("Unable to load target_core_iblock\n"); 211 212 ret = request_module("target_core_file"); 213 if (ret != 0) 214 pr_err("Unable to load target_core_file\n"); 215 216 ret = request_module("target_core_pscsi"); 217 if (ret != 0) 218 pr_err("Unable to load target_core_pscsi\n"); 219 220 ret = request_module("target_core_user"); 221 if (ret != 0) 222 pr_err("Unable to load target_core_user\n"); 223 224 sub_api_initialized = 1; 225 } 226 227 struct se_session *transport_init_session(enum target_prot_op sup_prot_ops) 228 { 229 struct se_session *se_sess; 230 231 se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL); 232 if (!se_sess) { 233 pr_err("Unable to allocate struct se_session from" 234 " se_sess_cache\n"); 235 return ERR_PTR(-ENOMEM); 236 } 237 INIT_LIST_HEAD(&se_sess->sess_list); 238 INIT_LIST_HEAD(&se_sess->sess_acl_list); 239 INIT_LIST_HEAD(&se_sess->sess_cmd_list); 240 INIT_LIST_HEAD(&se_sess->sess_wait_list); 241 spin_lock_init(&se_sess->sess_cmd_lock); 242 se_sess->sup_prot_ops = sup_prot_ops; 243 244 return se_sess; 245 } 246 EXPORT_SYMBOL(transport_init_session); 247 248 int transport_alloc_session_tags(struct se_session *se_sess, 249 unsigned int tag_num, unsigned int tag_size) 250 { 251 int rc; 252 253 se_sess->sess_cmd_map = kzalloc(tag_num * tag_size, 254 GFP_KERNEL | __GFP_NOWARN | __GFP_RETRY_MAYFAIL); 255 if (!se_sess->sess_cmd_map) { 256 se_sess->sess_cmd_map = vzalloc(tag_num * tag_size); 257 if (!se_sess->sess_cmd_map) { 258 pr_err("Unable to allocate se_sess->sess_cmd_map\n"); 259 return -ENOMEM; 260 } 261 } 262 263 rc = percpu_ida_init(&se_sess->sess_tag_pool, tag_num); 264 if (rc < 0) { 265 pr_err("Unable to init se_sess->sess_tag_pool," 266 " tag_num: %u\n", tag_num); 267 kvfree(se_sess->sess_cmd_map); 268 se_sess->sess_cmd_map = NULL; 269 return -ENOMEM; 270 } 271 272 return 0; 273 } 274 EXPORT_SYMBOL(transport_alloc_session_tags); 275 276 struct se_session *transport_init_session_tags(unsigned int tag_num, 277 unsigned int tag_size, 278 enum target_prot_op sup_prot_ops) 279 { 280 struct se_session *se_sess; 281 int rc; 282 283 if (tag_num != 0 && !tag_size) { 284 pr_err("init_session_tags called with percpu-ida tag_num:" 285 " %u, but zero tag_size\n", tag_num); 286 return ERR_PTR(-EINVAL); 287 } 288 if (!tag_num && tag_size) { 289 pr_err("init_session_tags called with percpu-ida tag_size:" 290 " %u, but zero tag_num\n", tag_size); 291 return ERR_PTR(-EINVAL); 292 } 293 294 se_sess = transport_init_session(sup_prot_ops); 295 if (IS_ERR(se_sess)) 296 return se_sess; 297 298 rc = transport_alloc_session_tags(se_sess, tag_num, tag_size); 299 if (rc < 0) { 300 transport_free_session(se_sess); 301 return ERR_PTR(-ENOMEM); 302 } 303 304 return se_sess; 305 } 306 EXPORT_SYMBOL(transport_init_session_tags); 307 308 /* 309 * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called. 310 */ 311 void __transport_register_session( 312 struct se_portal_group *se_tpg, 313 struct se_node_acl *se_nacl, 314 struct se_session *se_sess, 315 void *fabric_sess_ptr) 316 { 317 const struct target_core_fabric_ops *tfo = se_tpg->se_tpg_tfo; 318 unsigned char buf[PR_REG_ISID_LEN]; 319 320 se_sess->se_tpg = se_tpg; 321 se_sess->fabric_sess_ptr = fabric_sess_ptr; 322 /* 323 * Used by struct se_node_acl's under ConfigFS to locate active se_session-t 324 * 325 * Only set for struct se_session's that will actually be moving I/O. 326 * eg: *NOT* discovery sessions. 327 */ 328 if (se_nacl) { 329 /* 330 * 331 * Determine if fabric allows for T10-PI feature bits exposed to 332 * initiators for device backends with !dev->dev_attrib.pi_prot_type. 333 * 334 * If so, then always save prot_type on a per se_node_acl node 335 * basis and re-instate the previous sess_prot_type to avoid 336 * disabling PI from below any previously initiator side 337 * registered LUNs. 338 */ 339 if (se_nacl->saved_prot_type) 340 se_sess->sess_prot_type = se_nacl->saved_prot_type; 341 else if (tfo->tpg_check_prot_fabric_only) 342 se_sess->sess_prot_type = se_nacl->saved_prot_type = 343 tfo->tpg_check_prot_fabric_only(se_tpg); 344 /* 345 * If the fabric module supports an ISID based TransportID, 346 * save this value in binary from the fabric I_T Nexus now. 347 */ 348 if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) { 349 memset(&buf[0], 0, PR_REG_ISID_LEN); 350 se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess, 351 &buf[0], PR_REG_ISID_LEN); 352 se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]); 353 } 354 355 spin_lock_irq(&se_nacl->nacl_sess_lock); 356 /* 357 * The se_nacl->nacl_sess pointer will be set to the 358 * last active I_T Nexus for each struct se_node_acl. 359 */ 360 se_nacl->nacl_sess = se_sess; 361 362 list_add_tail(&se_sess->sess_acl_list, 363 &se_nacl->acl_sess_list); 364 spin_unlock_irq(&se_nacl->nacl_sess_lock); 365 } 366 list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list); 367 368 pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n", 369 se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr); 370 } 371 EXPORT_SYMBOL(__transport_register_session); 372 373 void transport_register_session( 374 struct se_portal_group *se_tpg, 375 struct se_node_acl *se_nacl, 376 struct se_session *se_sess, 377 void *fabric_sess_ptr) 378 { 379 unsigned long flags; 380 381 spin_lock_irqsave(&se_tpg->session_lock, flags); 382 __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr); 383 spin_unlock_irqrestore(&se_tpg->session_lock, flags); 384 } 385 EXPORT_SYMBOL(transport_register_session); 386 387 struct se_session * 388 target_alloc_session(struct se_portal_group *tpg, 389 unsigned int tag_num, unsigned int tag_size, 390 enum target_prot_op prot_op, 391 const char *initiatorname, void *private, 392 int (*callback)(struct se_portal_group *, 393 struct se_session *, void *)) 394 { 395 struct se_session *sess; 396 397 /* 398 * If the fabric driver is using percpu-ida based pre allocation 399 * of I/O descriptor tags, go ahead and perform that setup now.. 400 */ 401 if (tag_num != 0) 402 sess = transport_init_session_tags(tag_num, tag_size, prot_op); 403 else 404 sess = transport_init_session(prot_op); 405 406 if (IS_ERR(sess)) 407 return sess; 408 409 sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg, 410 (unsigned char *)initiatorname); 411 if (!sess->se_node_acl) { 412 transport_free_session(sess); 413 return ERR_PTR(-EACCES); 414 } 415 /* 416 * Go ahead and perform any remaining fabric setup that is 417 * required before transport_register_session(). 418 */ 419 if (callback != NULL) { 420 int rc = callback(tpg, sess, private); 421 if (rc) { 422 transport_free_session(sess); 423 return ERR_PTR(rc); 424 } 425 } 426 427 transport_register_session(tpg, sess->se_node_acl, sess, private); 428 return sess; 429 } 430 EXPORT_SYMBOL(target_alloc_session); 431 432 ssize_t target_show_dynamic_sessions(struct se_portal_group *se_tpg, char *page) 433 { 434 struct se_session *se_sess; 435 ssize_t len = 0; 436 437 spin_lock_bh(&se_tpg->session_lock); 438 list_for_each_entry(se_sess, &se_tpg->tpg_sess_list, sess_list) { 439 if (!se_sess->se_node_acl) 440 continue; 441 if (!se_sess->se_node_acl->dynamic_node_acl) 442 continue; 443 if (strlen(se_sess->se_node_acl->initiatorname) + 1 + len > PAGE_SIZE) 444 break; 445 446 len += snprintf(page + len, PAGE_SIZE - len, "%s\n", 447 se_sess->se_node_acl->initiatorname); 448 len += 1; /* Include NULL terminator */ 449 } 450 spin_unlock_bh(&se_tpg->session_lock); 451 452 return len; 453 } 454 EXPORT_SYMBOL(target_show_dynamic_sessions); 455 456 static void target_complete_nacl(struct kref *kref) 457 { 458 struct se_node_acl *nacl = container_of(kref, 459 struct se_node_acl, acl_kref); 460 struct se_portal_group *se_tpg = nacl->se_tpg; 461 462 if (!nacl->dynamic_stop) { 463 complete(&nacl->acl_free_comp); 464 return; 465 } 466 467 mutex_lock(&se_tpg->acl_node_mutex); 468 list_del_init(&nacl->acl_list); 469 mutex_unlock(&se_tpg->acl_node_mutex); 470 471 core_tpg_wait_for_nacl_pr_ref(nacl); 472 core_free_device_list_for_node(nacl, se_tpg); 473 kfree(nacl); 474 } 475 476 void target_put_nacl(struct se_node_acl *nacl) 477 { 478 kref_put(&nacl->acl_kref, target_complete_nacl); 479 } 480 EXPORT_SYMBOL(target_put_nacl); 481 482 void transport_deregister_session_configfs(struct se_session *se_sess) 483 { 484 struct se_node_acl *se_nacl; 485 unsigned long flags; 486 /* 487 * Used by struct se_node_acl's under ConfigFS to locate active struct se_session 488 */ 489 se_nacl = se_sess->se_node_acl; 490 if (se_nacl) { 491 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags); 492 if (!list_empty(&se_sess->sess_acl_list)) 493 list_del_init(&se_sess->sess_acl_list); 494 /* 495 * If the session list is empty, then clear the pointer. 496 * Otherwise, set the struct se_session pointer from the tail 497 * element of the per struct se_node_acl active session list. 498 */ 499 if (list_empty(&se_nacl->acl_sess_list)) 500 se_nacl->nacl_sess = NULL; 501 else { 502 se_nacl->nacl_sess = container_of( 503 se_nacl->acl_sess_list.prev, 504 struct se_session, sess_acl_list); 505 } 506 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags); 507 } 508 } 509 EXPORT_SYMBOL(transport_deregister_session_configfs); 510 511 void transport_free_session(struct se_session *se_sess) 512 { 513 struct se_node_acl *se_nacl = se_sess->se_node_acl; 514 515 /* 516 * Drop the se_node_acl->nacl_kref obtained from within 517 * core_tpg_get_initiator_node_acl(). 518 */ 519 if (se_nacl) { 520 struct se_portal_group *se_tpg = se_nacl->se_tpg; 521 const struct target_core_fabric_ops *se_tfo = se_tpg->se_tpg_tfo; 522 unsigned long flags; 523 524 se_sess->se_node_acl = NULL; 525 526 /* 527 * Also determine if we need to drop the extra ->cmd_kref if 528 * it had been previously dynamically generated, and 529 * the endpoint is not caching dynamic ACLs. 530 */ 531 mutex_lock(&se_tpg->acl_node_mutex); 532 if (se_nacl->dynamic_node_acl && 533 !se_tfo->tpg_check_demo_mode_cache(se_tpg)) { 534 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags); 535 if (list_empty(&se_nacl->acl_sess_list)) 536 se_nacl->dynamic_stop = true; 537 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags); 538 539 if (se_nacl->dynamic_stop) 540 list_del_init(&se_nacl->acl_list); 541 } 542 mutex_unlock(&se_tpg->acl_node_mutex); 543 544 if (se_nacl->dynamic_stop) 545 target_put_nacl(se_nacl); 546 547 target_put_nacl(se_nacl); 548 } 549 if (se_sess->sess_cmd_map) { 550 percpu_ida_destroy(&se_sess->sess_tag_pool); 551 kvfree(se_sess->sess_cmd_map); 552 } 553 kmem_cache_free(se_sess_cache, se_sess); 554 } 555 EXPORT_SYMBOL(transport_free_session); 556 557 void transport_deregister_session(struct se_session *se_sess) 558 { 559 struct se_portal_group *se_tpg = se_sess->se_tpg; 560 unsigned long flags; 561 562 if (!se_tpg) { 563 transport_free_session(se_sess); 564 return; 565 } 566 567 spin_lock_irqsave(&se_tpg->session_lock, flags); 568 list_del(&se_sess->sess_list); 569 se_sess->se_tpg = NULL; 570 se_sess->fabric_sess_ptr = NULL; 571 spin_unlock_irqrestore(&se_tpg->session_lock, flags); 572 573 pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n", 574 se_tpg->se_tpg_tfo->get_fabric_name()); 575 /* 576 * If last kref is dropping now for an explicit NodeACL, awake sleeping 577 * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group 578 * removal context from within transport_free_session() code. 579 * 580 * For dynamic ACL, target_put_nacl() uses target_complete_nacl() 581 * to release all remaining generate_node_acl=1 created ACL resources. 582 */ 583 584 transport_free_session(se_sess); 585 } 586 EXPORT_SYMBOL(transport_deregister_session); 587 588 static void target_remove_from_state_list(struct se_cmd *cmd) 589 { 590 struct se_device *dev = cmd->se_dev; 591 unsigned long flags; 592 593 if (!dev) 594 return; 595 596 spin_lock_irqsave(&dev->execute_task_lock, flags); 597 if (cmd->state_active) { 598 list_del(&cmd->state_list); 599 cmd->state_active = false; 600 } 601 spin_unlock_irqrestore(&dev->execute_task_lock, flags); 602 } 603 604 static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd) 605 { 606 unsigned long flags; 607 608 target_remove_from_state_list(cmd); 609 610 /* 611 * Clear struct se_cmd->se_lun before the handoff to FE. 612 */ 613 cmd->se_lun = NULL; 614 615 spin_lock_irqsave(&cmd->t_state_lock, flags); 616 /* 617 * Determine if frontend context caller is requesting the stopping of 618 * this command for frontend exceptions. 619 */ 620 if (cmd->transport_state & CMD_T_STOP) { 621 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 622 __func__, __LINE__, cmd->tag); 623 624 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 625 626 complete_all(&cmd->t_transport_stop_comp); 627 return 1; 628 } 629 cmd->transport_state &= ~CMD_T_ACTIVE; 630 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 631 632 /* 633 * Some fabric modules like tcm_loop can release their internally 634 * allocated I/O reference and struct se_cmd now. 635 * 636 * Fabric modules are expected to return '1' here if the se_cmd being 637 * passed is released at this point, or zero if not being released. 638 */ 639 return cmd->se_tfo->check_stop_free(cmd); 640 } 641 642 static void transport_lun_remove_cmd(struct se_cmd *cmd) 643 { 644 struct se_lun *lun = cmd->se_lun; 645 646 if (!lun) 647 return; 648 649 if (cmpxchg(&cmd->lun_ref_active, true, false)) 650 percpu_ref_put(&lun->lun_ref); 651 } 652 653 int transport_cmd_finish_abort(struct se_cmd *cmd, int remove) 654 { 655 bool ack_kref = (cmd->se_cmd_flags & SCF_ACK_KREF); 656 int ret = 0; 657 658 if (cmd->se_cmd_flags & SCF_SE_LUN_CMD) 659 transport_lun_remove_cmd(cmd); 660 /* 661 * Allow the fabric driver to unmap any resources before 662 * releasing the descriptor via TFO->release_cmd() 663 */ 664 if (remove) 665 cmd->se_tfo->aborted_task(cmd); 666 667 if (transport_cmd_check_stop_to_fabric(cmd)) 668 return 1; 669 if (remove && ack_kref) 670 ret = target_put_sess_cmd(cmd); 671 672 return ret; 673 } 674 675 static void target_complete_failure_work(struct work_struct *work) 676 { 677 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 678 679 transport_generic_request_failure(cmd, 680 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); 681 } 682 683 /* 684 * Used when asking transport to copy Sense Data from the underlying 685 * Linux/SCSI struct scsi_cmnd 686 */ 687 static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd) 688 { 689 struct se_device *dev = cmd->se_dev; 690 691 WARN_ON(!cmd->se_lun); 692 693 if (!dev) 694 return NULL; 695 696 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) 697 return NULL; 698 699 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 700 701 pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n", 702 dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status); 703 return cmd->sense_buffer; 704 } 705 706 void transport_copy_sense_to_cmd(struct se_cmd *cmd, unsigned char *sense) 707 { 708 unsigned char *cmd_sense_buf; 709 unsigned long flags; 710 711 spin_lock_irqsave(&cmd->t_state_lock, flags); 712 cmd_sense_buf = transport_get_sense_buffer(cmd); 713 if (!cmd_sense_buf) { 714 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 715 return; 716 } 717 718 cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE; 719 memcpy(cmd_sense_buf, sense, cmd->scsi_sense_length); 720 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 721 } 722 EXPORT_SYMBOL(transport_copy_sense_to_cmd); 723 724 void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) 725 { 726 struct se_device *dev = cmd->se_dev; 727 int success; 728 unsigned long flags; 729 730 cmd->scsi_status = scsi_status; 731 732 spin_lock_irqsave(&cmd->t_state_lock, flags); 733 switch (cmd->scsi_status) { 734 case SAM_STAT_CHECK_CONDITION: 735 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) 736 success = 1; 737 else 738 success = 0; 739 break; 740 default: 741 success = 1; 742 break; 743 } 744 745 /* 746 * Check for case where an explicit ABORT_TASK has been received 747 * and transport_wait_for_tasks() will be waiting for completion.. 748 */ 749 if (cmd->transport_state & CMD_T_ABORTED || 750 cmd->transport_state & CMD_T_STOP) { 751 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 752 /* 753 * If COMPARE_AND_WRITE was stopped by __transport_wait_for_tasks(), 754 * release se_device->caw_sem obtained by sbc_compare_and_write() 755 * since target_complete_ok_work() or target_complete_failure_work() 756 * won't be called to invoke the normal CAW completion callbacks. 757 */ 758 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) { 759 up(&dev->caw_sem); 760 } 761 complete_all(&cmd->t_transport_stop_comp); 762 return; 763 } else if (!success) { 764 INIT_WORK(&cmd->work, target_complete_failure_work); 765 } else { 766 INIT_WORK(&cmd->work, target_complete_ok_work); 767 } 768 769 cmd->t_state = TRANSPORT_COMPLETE; 770 cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE); 771 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 772 773 if (cmd->se_cmd_flags & SCF_USE_CPUID) 774 queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work); 775 else 776 queue_work(target_completion_wq, &cmd->work); 777 } 778 EXPORT_SYMBOL(target_complete_cmd); 779 780 void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length) 781 { 782 if (scsi_status == SAM_STAT_GOOD && length < cmd->data_length) { 783 if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { 784 cmd->residual_count += cmd->data_length - length; 785 } else { 786 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 787 cmd->residual_count = cmd->data_length - length; 788 } 789 790 cmd->data_length = length; 791 } 792 793 target_complete_cmd(cmd, scsi_status); 794 } 795 EXPORT_SYMBOL(target_complete_cmd_with_length); 796 797 static void target_add_to_state_list(struct se_cmd *cmd) 798 { 799 struct se_device *dev = cmd->se_dev; 800 unsigned long flags; 801 802 spin_lock_irqsave(&dev->execute_task_lock, flags); 803 if (!cmd->state_active) { 804 list_add_tail(&cmd->state_list, &dev->state_list); 805 cmd->state_active = true; 806 } 807 spin_unlock_irqrestore(&dev->execute_task_lock, flags); 808 } 809 810 /* 811 * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status 812 */ 813 static void transport_write_pending_qf(struct se_cmd *cmd); 814 static void transport_complete_qf(struct se_cmd *cmd); 815 816 void target_qf_do_work(struct work_struct *work) 817 { 818 struct se_device *dev = container_of(work, struct se_device, 819 qf_work_queue); 820 LIST_HEAD(qf_cmd_list); 821 struct se_cmd *cmd, *cmd_tmp; 822 823 spin_lock_irq(&dev->qf_cmd_lock); 824 list_splice_init(&dev->qf_cmd_list, &qf_cmd_list); 825 spin_unlock_irq(&dev->qf_cmd_lock); 826 827 list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) { 828 list_del(&cmd->se_qf_node); 829 atomic_dec_mb(&dev->dev_qf_count); 830 831 pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue" 832 " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd, 833 (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" : 834 (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING" 835 : "UNKNOWN"); 836 837 if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) 838 transport_write_pending_qf(cmd); 839 else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK || 840 cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) 841 transport_complete_qf(cmd); 842 } 843 } 844 845 unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd) 846 { 847 switch (cmd->data_direction) { 848 case DMA_NONE: 849 return "NONE"; 850 case DMA_FROM_DEVICE: 851 return "READ"; 852 case DMA_TO_DEVICE: 853 return "WRITE"; 854 case DMA_BIDIRECTIONAL: 855 return "BIDI"; 856 default: 857 break; 858 } 859 860 return "UNKNOWN"; 861 } 862 863 void transport_dump_dev_state( 864 struct se_device *dev, 865 char *b, 866 int *bl) 867 { 868 *bl += sprintf(b + *bl, "Status: "); 869 if (dev->export_count) 870 *bl += sprintf(b + *bl, "ACTIVATED"); 871 else 872 *bl += sprintf(b + *bl, "DEACTIVATED"); 873 874 *bl += sprintf(b + *bl, " Max Queue Depth: %d", dev->queue_depth); 875 *bl += sprintf(b + *bl, " SectorSize: %u HwMaxSectors: %u\n", 876 dev->dev_attrib.block_size, 877 dev->dev_attrib.hw_max_sectors); 878 *bl += sprintf(b + *bl, " "); 879 } 880 881 void transport_dump_vpd_proto_id( 882 struct t10_vpd *vpd, 883 unsigned char *p_buf, 884 int p_buf_len) 885 { 886 unsigned char buf[VPD_TMP_BUF_SIZE]; 887 int len; 888 889 memset(buf, 0, VPD_TMP_BUF_SIZE); 890 len = sprintf(buf, "T10 VPD Protocol Identifier: "); 891 892 switch (vpd->protocol_identifier) { 893 case 0x00: 894 sprintf(buf+len, "Fibre Channel\n"); 895 break; 896 case 0x10: 897 sprintf(buf+len, "Parallel SCSI\n"); 898 break; 899 case 0x20: 900 sprintf(buf+len, "SSA\n"); 901 break; 902 case 0x30: 903 sprintf(buf+len, "IEEE 1394\n"); 904 break; 905 case 0x40: 906 sprintf(buf+len, "SCSI Remote Direct Memory Access" 907 " Protocol\n"); 908 break; 909 case 0x50: 910 sprintf(buf+len, "Internet SCSI (iSCSI)\n"); 911 break; 912 case 0x60: 913 sprintf(buf+len, "SAS Serial SCSI Protocol\n"); 914 break; 915 case 0x70: 916 sprintf(buf+len, "Automation/Drive Interface Transport" 917 " Protocol\n"); 918 break; 919 case 0x80: 920 sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n"); 921 break; 922 default: 923 sprintf(buf+len, "Unknown 0x%02x\n", 924 vpd->protocol_identifier); 925 break; 926 } 927 928 if (p_buf) 929 strncpy(p_buf, buf, p_buf_len); 930 else 931 pr_debug("%s", buf); 932 } 933 934 void 935 transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83) 936 { 937 /* 938 * Check if the Protocol Identifier Valid (PIV) bit is set.. 939 * 940 * from spc3r23.pdf section 7.5.1 941 */ 942 if (page_83[1] & 0x80) { 943 vpd->protocol_identifier = (page_83[0] & 0xf0); 944 vpd->protocol_identifier_set = 1; 945 transport_dump_vpd_proto_id(vpd, NULL, 0); 946 } 947 } 948 EXPORT_SYMBOL(transport_set_vpd_proto_id); 949 950 int transport_dump_vpd_assoc( 951 struct t10_vpd *vpd, 952 unsigned char *p_buf, 953 int p_buf_len) 954 { 955 unsigned char buf[VPD_TMP_BUF_SIZE]; 956 int ret = 0; 957 int len; 958 959 memset(buf, 0, VPD_TMP_BUF_SIZE); 960 len = sprintf(buf, "T10 VPD Identifier Association: "); 961 962 switch (vpd->association) { 963 case 0x00: 964 sprintf(buf+len, "addressed logical unit\n"); 965 break; 966 case 0x10: 967 sprintf(buf+len, "target port\n"); 968 break; 969 case 0x20: 970 sprintf(buf+len, "SCSI target device\n"); 971 break; 972 default: 973 sprintf(buf+len, "Unknown 0x%02x\n", vpd->association); 974 ret = -EINVAL; 975 break; 976 } 977 978 if (p_buf) 979 strncpy(p_buf, buf, p_buf_len); 980 else 981 pr_debug("%s", buf); 982 983 return ret; 984 } 985 986 int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83) 987 { 988 /* 989 * The VPD identification association.. 990 * 991 * from spc3r23.pdf Section 7.6.3.1 Table 297 992 */ 993 vpd->association = (page_83[1] & 0x30); 994 return transport_dump_vpd_assoc(vpd, NULL, 0); 995 } 996 EXPORT_SYMBOL(transport_set_vpd_assoc); 997 998 int transport_dump_vpd_ident_type( 999 struct t10_vpd *vpd, 1000 unsigned char *p_buf, 1001 int p_buf_len) 1002 { 1003 unsigned char buf[VPD_TMP_BUF_SIZE]; 1004 int ret = 0; 1005 int len; 1006 1007 memset(buf, 0, VPD_TMP_BUF_SIZE); 1008 len = sprintf(buf, "T10 VPD Identifier Type: "); 1009 1010 switch (vpd->device_identifier_type) { 1011 case 0x00: 1012 sprintf(buf+len, "Vendor specific\n"); 1013 break; 1014 case 0x01: 1015 sprintf(buf+len, "T10 Vendor ID based\n"); 1016 break; 1017 case 0x02: 1018 sprintf(buf+len, "EUI-64 based\n"); 1019 break; 1020 case 0x03: 1021 sprintf(buf+len, "NAA\n"); 1022 break; 1023 case 0x04: 1024 sprintf(buf+len, "Relative target port identifier\n"); 1025 break; 1026 case 0x08: 1027 sprintf(buf+len, "SCSI name string\n"); 1028 break; 1029 default: 1030 sprintf(buf+len, "Unsupported: 0x%02x\n", 1031 vpd->device_identifier_type); 1032 ret = -EINVAL; 1033 break; 1034 } 1035 1036 if (p_buf) { 1037 if (p_buf_len < strlen(buf)+1) 1038 return -EINVAL; 1039 strncpy(p_buf, buf, p_buf_len); 1040 } else { 1041 pr_debug("%s", buf); 1042 } 1043 1044 return ret; 1045 } 1046 1047 int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83) 1048 { 1049 /* 1050 * The VPD identifier type.. 1051 * 1052 * from spc3r23.pdf Section 7.6.3.1 Table 298 1053 */ 1054 vpd->device_identifier_type = (page_83[1] & 0x0f); 1055 return transport_dump_vpd_ident_type(vpd, NULL, 0); 1056 } 1057 EXPORT_SYMBOL(transport_set_vpd_ident_type); 1058 1059 int transport_dump_vpd_ident( 1060 struct t10_vpd *vpd, 1061 unsigned char *p_buf, 1062 int p_buf_len) 1063 { 1064 unsigned char buf[VPD_TMP_BUF_SIZE]; 1065 int ret = 0; 1066 1067 memset(buf, 0, VPD_TMP_BUF_SIZE); 1068 1069 switch (vpd->device_identifier_code_set) { 1070 case 0x01: /* Binary */ 1071 snprintf(buf, sizeof(buf), 1072 "T10 VPD Binary Device Identifier: %s\n", 1073 &vpd->device_identifier[0]); 1074 break; 1075 case 0x02: /* ASCII */ 1076 snprintf(buf, sizeof(buf), 1077 "T10 VPD ASCII Device Identifier: %s\n", 1078 &vpd->device_identifier[0]); 1079 break; 1080 case 0x03: /* UTF-8 */ 1081 snprintf(buf, sizeof(buf), 1082 "T10 VPD UTF-8 Device Identifier: %s\n", 1083 &vpd->device_identifier[0]); 1084 break; 1085 default: 1086 sprintf(buf, "T10 VPD Device Identifier encoding unsupported:" 1087 " 0x%02x", vpd->device_identifier_code_set); 1088 ret = -EINVAL; 1089 break; 1090 } 1091 1092 if (p_buf) 1093 strncpy(p_buf, buf, p_buf_len); 1094 else 1095 pr_debug("%s", buf); 1096 1097 return ret; 1098 } 1099 1100 int 1101 transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83) 1102 { 1103 static const char hex_str[] = "0123456789abcdef"; 1104 int j = 0, i = 4; /* offset to start of the identifier */ 1105 1106 /* 1107 * The VPD Code Set (encoding) 1108 * 1109 * from spc3r23.pdf Section 7.6.3.1 Table 296 1110 */ 1111 vpd->device_identifier_code_set = (page_83[0] & 0x0f); 1112 switch (vpd->device_identifier_code_set) { 1113 case 0x01: /* Binary */ 1114 vpd->device_identifier[j++] = 1115 hex_str[vpd->device_identifier_type]; 1116 while (i < (4 + page_83[3])) { 1117 vpd->device_identifier[j++] = 1118 hex_str[(page_83[i] & 0xf0) >> 4]; 1119 vpd->device_identifier[j++] = 1120 hex_str[page_83[i] & 0x0f]; 1121 i++; 1122 } 1123 break; 1124 case 0x02: /* ASCII */ 1125 case 0x03: /* UTF-8 */ 1126 while (i < (4 + page_83[3])) 1127 vpd->device_identifier[j++] = page_83[i++]; 1128 break; 1129 default: 1130 break; 1131 } 1132 1133 return transport_dump_vpd_ident(vpd, NULL, 0); 1134 } 1135 EXPORT_SYMBOL(transport_set_vpd_ident); 1136 1137 static sense_reason_t 1138 target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev, 1139 unsigned int size) 1140 { 1141 u32 mtl; 1142 1143 if (!cmd->se_tfo->max_data_sg_nents) 1144 return TCM_NO_SENSE; 1145 /* 1146 * Check if fabric enforced maximum SGL entries per I/O descriptor 1147 * exceeds se_cmd->data_length. If true, set SCF_UNDERFLOW_BIT + 1148 * residual_count and reduce original cmd->data_length to maximum 1149 * length based on single PAGE_SIZE entry scatter-lists. 1150 */ 1151 mtl = (cmd->se_tfo->max_data_sg_nents * PAGE_SIZE); 1152 if (cmd->data_length > mtl) { 1153 /* 1154 * If an existing CDB overflow is present, calculate new residual 1155 * based on CDB size minus fabric maximum transfer length. 1156 * 1157 * If an existing CDB underflow is present, calculate new residual 1158 * based on original cmd->data_length minus fabric maximum transfer 1159 * length. 1160 * 1161 * Otherwise, set the underflow residual based on cmd->data_length 1162 * minus fabric maximum transfer length. 1163 */ 1164 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) { 1165 cmd->residual_count = (size - mtl); 1166 } else if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { 1167 u32 orig_dl = size + cmd->residual_count; 1168 cmd->residual_count = (orig_dl - mtl); 1169 } else { 1170 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 1171 cmd->residual_count = (cmd->data_length - mtl); 1172 } 1173 cmd->data_length = mtl; 1174 /* 1175 * Reset sbc_check_prot() calculated protection payload 1176 * length based upon the new smaller MTL. 1177 */ 1178 if (cmd->prot_length) { 1179 u32 sectors = (mtl / dev->dev_attrib.block_size); 1180 cmd->prot_length = dev->prot_length * sectors; 1181 } 1182 } 1183 return TCM_NO_SENSE; 1184 } 1185 1186 sense_reason_t 1187 target_cmd_size_check(struct se_cmd *cmd, unsigned int size) 1188 { 1189 struct se_device *dev = cmd->se_dev; 1190 1191 if (cmd->unknown_data_length) { 1192 cmd->data_length = size; 1193 } else if (size != cmd->data_length) { 1194 pr_warn_ratelimited("TARGET_CORE[%s]: Expected Transfer Length:" 1195 " %u does not match SCSI CDB Length: %u for SAM Opcode:" 1196 " 0x%02x\n", cmd->se_tfo->get_fabric_name(), 1197 cmd->data_length, size, cmd->t_task_cdb[0]); 1198 1199 if (cmd->data_direction == DMA_TO_DEVICE) { 1200 if (cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) { 1201 pr_err_ratelimited("Rejecting underflow/overflow" 1202 " for WRITE data CDB\n"); 1203 return TCM_INVALID_CDB_FIELD; 1204 } 1205 /* 1206 * Some fabric drivers like iscsi-target still expect to 1207 * always reject overflow writes. Reject this case until 1208 * full fabric driver level support for overflow writes 1209 * is introduced tree-wide. 1210 */ 1211 if (size > cmd->data_length) { 1212 pr_err_ratelimited("Rejecting overflow for" 1213 " WRITE control CDB\n"); 1214 return TCM_INVALID_CDB_FIELD; 1215 } 1216 } 1217 /* 1218 * Reject READ_* or WRITE_* with overflow/underflow for 1219 * type SCF_SCSI_DATA_CDB. 1220 */ 1221 if (dev->dev_attrib.block_size != 512) { 1222 pr_err("Failing OVERFLOW/UNDERFLOW for LBA op" 1223 " CDB on non 512-byte sector setup subsystem" 1224 " plugin: %s\n", dev->transport->name); 1225 /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */ 1226 return TCM_INVALID_CDB_FIELD; 1227 } 1228 /* 1229 * For the overflow case keep the existing fabric provided 1230 * ->data_length. Otherwise for the underflow case, reset 1231 * ->data_length to the smaller SCSI expected data transfer 1232 * length. 1233 */ 1234 if (size > cmd->data_length) { 1235 cmd->se_cmd_flags |= SCF_OVERFLOW_BIT; 1236 cmd->residual_count = (size - cmd->data_length); 1237 } else { 1238 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 1239 cmd->residual_count = (cmd->data_length - size); 1240 cmd->data_length = size; 1241 } 1242 } 1243 1244 return target_check_max_data_sg_nents(cmd, dev, size); 1245 1246 } 1247 1248 /* 1249 * Used by fabric modules containing a local struct se_cmd within their 1250 * fabric dependent per I/O descriptor. 1251 * 1252 * Preserves the value of @cmd->tag. 1253 */ 1254 void transport_init_se_cmd( 1255 struct se_cmd *cmd, 1256 const struct target_core_fabric_ops *tfo, 1257 struct se_session *se_sess, 1258 u32 data_length, 1259 int data_direction, 1260 int task_attr, 1261 unsigned char *sense_buffer) 1262 { 1263 INIT_LIST_HEAD(&cmd->se_delayed_node); 1264 INIT_LIST_HEAD(&cmd->se_qf_node); 1265 INIT_LIST_HEAD(&cmd->se_cmd_list); 1266 INIT_LIST_HEAD(&cmd->state_list); 1267 init_completion(&cmd->t_transport_stop_comp); 1268 init_completion(&cmd->cmd_wait_comp); 1269 spin_lock_init(&cmd->t_state_lock); 1270 INIT_WORK(&cmd->work, NULL); 1271 kref_init(&cmd->cmd_kref); 1272 1273 cmd->se_tfo = tfo; 1274 cmd->se_sess = se_sess; 1275 cmd->data_length = data_length; 1276 cmd->data_direction = data_direction; 1277 cmd->sam_task_attr = task_attr; 1278 cmd->sense_buffer = sense_buffer; 1279 1280 cmd->state_active = false; 1281 } 1282 EXPORT_SYMBOL(transport_init_se_cmd); 1283 1284 static sense_reason_t 1285 transport_check_alloc_task_attr(struct se_cmd *cmd) 1286 { 1287 struct se_device *dev = cmd->se_dev; 1288 1289 /* 1290 * Check if SAM Task Attribute emulation is enabled for this 1291 * struct se_device storage object 1292 */ 1293 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1294 return 0; 1295 1296 if (cmd->sam_task_attr == TCM_ACA_TAG) { 1297 pr_debug("SAM Task Attribute ACA" 1298 " emulation is not supported\n"); 1299 return TCM_INVALID_CDB_FIELD; 1300 } 1301 1302 return 0; 1303 } 1304 1305 sense_reason_t 1306 target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) 1307 { 1308 struct se_device *dev = cmd->se_dev; 1309 sense_reason_t ret; 1310 1311 /* 1312 * Ensure that the received CDB is less than the max (252 + 8) bytes 1313 * for VARIABLE_LENGTH_CMD 1314 */ 1315 if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) { 1316 pr_err("Received SCSI CDB with command_size: %d that" 1317 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", 1318 scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE); 1319 return TCM_INVALID_CDB_FIELD; 1320 } 1321 /* 1322 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE, 1323 * allocate the additional extended CDB buffer now.. Otherwise 1324 * setup the pointer from __t_task_cdb to t_task_cdb. 1325 */ 1326 if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) { 1327 cmd->t_task_cdb = kzalloc(scsi_command_size(cdb), 1328 GFP_KERNEL); 1329 if (!cmd->t_task_cdb) { 1330 pr_err("Unable to allocate cmd->t_task_cdb" 1331 " %u > sizeof(cmd->__t_task_cdb): %lu ops\n", 1332 scsi_command_size(cdb), 1333 (unsigned long)sizeof(cmd->__t_task_cdb)); 1334 return TCM_OUT_OF_RESOURCES; 1335 } 1336 } else 1337 cmd->t_task_cdb = &cmd->__t_task_cdb[0]; 1338 /* 1339 * Copy the original CDB into cmd-> 1340 */ 1341 memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb)); 1342 1343 trace_target_sequencer_start(cmd); 1344 1345 ret = dev->transport->parse_cdb(cmd); 1346 if (ret == TCM_UNSUPPORTED_SCSI_OPCODE) 1347 pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n", 1348 cmd->se_tfo->get_fabric_name(), 1349 cmd->se_sess->se_node_acl->initiatorname, 1350 cmd->t_task_cdb[0]); 1351 if (ret) 1352 return ret; 1353 1354 ret = transport_check_alloc_task_attr(cmd); 1355 if (ret) 1356 return ret; 1357 1358 cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE; 1359 atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus); 1360 return 0; 1361 } 1362 EXPORT_SYMBOL(target_setup_cmd_from_cdb); 1363 1364 /* 1365 * Used by fabric module frontends to queue tasks directly. 1366 * May only be used from process context. 1367 */ 1368 int transport_handle_cdb_direct( 1369 struct se_cmd *cmd) 1370 { 1371 sense_reason_t ret; 1372 1373 if (!cmd->se_lun) { 1374 dump_stack(); 1375 pr_err("cmd->se_lun is NULL\n"); 1376 return -EINVAL; 1377 } 1378 if (in_interrupt()) { 1379 dump_stack(); 1380 pr_err("transport_generic_handle_cdb cannot be called" 1381 " from interrupt context\n"); 1382 return -EINVAL; 1383 } 1384 /* 1385 * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that 1386 * outstanding descriptors are handled correctly during shutdown via 1387 * transport_wait_for_tasks() 1388 * 1389 * Also, we don't take cmd->t_state_lock here as we only expect 1390 * this to be called for initial descriptor submission. 1391 */ 1392 cmd->t_state = TRANSPORT_NEW_CMD; 1393 cmd->transport_state |= CMD_T_ACTIVE; 1394 1395 /* 1396 * transport_generic_new_cmd() is already handling QUEUE_FULL, 1397 * so follow TRANSPORT_NEW_CMD processing thread context usage 1398 * and call transport_generic_request_failure() if necessary.. 1399 */ 1400 ret = transport_generic_new_cmd(cmd); 1401 if (ret) 1402 transport_generic_request_failure(cmd, ret); 1403 return 0; 1404 } 1405 EXPORT_SYMBOL(transport_handle_cdb_direct); 1406 1407 sense_reason_t 1408 transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl, 1409 u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count) 1410 { 1411 if (!sgl || !sgl_count) 1412 return 0; 1413 1414 /* 1415 * Reject SCSI data overflow with map_mem_to_cmd() as incoming 1416 * scatterlists already have been set to follow what the fabric 1417 * passes for the original expected data transfer length. 1418 */ 1419 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) { 1420 pr_warn("Rejecting SCSI DATA overflow for fabric using" 1421 " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n"); 1422 return TCM_INVALID_CDB_FIELD; 1423 } 1424 1425 cmd->t_data_sg = sgl; 1426 cmd->t_data_nents = sgl_count; 1427 cmd->t_bidi_data_sg = sgl_bidi; 1428 cmd->t_bidi_data_nents = sgl_bidi_count; 1429 1430 cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1431 return 0; 1432 } 1433 1434 /* 1435 * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized 1436 * se_cmd + use pre-allocated SGL memory. 1437 * 1438 * @se_cmd: command descriptor to submit 1439 * @se_sess: associated se_sess for endpoint 1440 * @cdb: pointer to SCSI CDB 1441 * @sense: pointer to SCSI sense buffer 1442 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1443 * @data_length: fabric expected data transfer length 1444 * @task_addr: SAM task attribute 1445 * @data_dir: DMA data direction 1446 * @flags: flags for command submission from target_sc_flags_tables 1447 * @sgl: struct scatterlist memory for unidirectional mapping 1448 * @sgl_count: scatterlist count for unidirectional mapping 1449 * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping 1450 * @sgl_bidi_count: scatterlist count for bidirectional READ mapping 1451 * @sgl_prot: struct scatterlist memory protection information 1452 * @sgl_prot_count: scatterlist count for protection information 1453 * 1454 * Task tags are supported if the caller has set @se_cmd->tag. 1455 * 1456 * Returns non zero to signal active I/O shutdown failure. All other 1457 * setup exceptions will be returned as a SCSI CHECK_CONDITION response, 1458 * but still return zero here. 1459 * 1460 * This may only be called from process context, and also currently 1461 * assumes internal allocation of fabric payload buffer by target-core. 1462 */ 1463 int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess, 1464 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, 1465 u32 data_length, int task_attr, int data_dir, int flags, 1466 struct scatterlist *sgl, u32 sgl_count, 1467 struct scatterlist *sgl_bidi, u32 sgl_bidi_count, 1468 struct scatterlist *sgl_prot, u32 sgl_prot_count) 1469 { 1470 struct se_portal_group *se_tpg; 1471 sense_reason_t rc; 1472 int ret; 1473 1474 se_tpg = se_sess->se_tpg; 1475 BUG_ON(!se_tpg); 1476 BUG_ON(se_cmd->se_tfo || se_cmd->se_sess); 1477 BUG_ON(in_interrupt()); 1478 /* 1479 * Initialize se_cmd for target operation. From this point 1480 * exceptions are handled by sending exception status via 1481 * target_core_fabric_ops->queue_status() callback 1482 */ 1483 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, 1484 data_length, data_dir, task_attr, sense); 1485 1486 if (flags & TARGET_SCF_USE_CPUID) 1487 se_cmd->se_cmd_flags |= SCF_USE_CPUID; 1488 else 1489 se_cmd->cpuid = WORK_CPU_UNBOUND; 1490 1491 if (flags & TARGET_SCF_UNKNOWN_SIZE) 1492 se_cmd->unknown_data_length = 1; 1493 /* 1494 * Obtain struct se_cmd->cmd_kref reference and add new cmd to 1495 * se_sess->sess_cmd_list. A second kref_get here is necessary 1496 * for fabrics using TARGET_SCF_ACK_KREF that expect a second 1497 * kref_put() to happen during fabric packet acknowledgement. 1498 */ 1499 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); 1500 if (ret) 1501 return ret; 1502 /* 1503 * Signal bidirectional data payloads to target-core 1504 */ 1505 if (flags & TARGET_SCF_BIDI_OP) 1506 se_cmd->se_cmd_flags |= SCF_BIDI; 1507 /* 1508 * Locate se_lun pointer and attach it to struct se_cmd 1509 */ 1510 rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun); 1511 if (rc) { 1512 transport_send_check_condition_and_sense(se_cmd, rc, 0); 1513 target_put_sess_cmd(se_cmd); 1514 return 0; 1515 } 1516 1517 rc = target_setup_cmd_from_cdb(se_cmd, cdb); 1518 if (rc != 0) { 1519 transport_generic_request_failure(se_cmd, rc); 1520 return 0; 1521 } 1522 1523 /* 1524 * Save pointers for SGLs containing protection information, 1525 * if present. 1526 */ 1527 if (sgl_prot_count) { 1528 se_cmd->t_prot_sg = sgl_prot; 1529 se_cmd->t_prot_nents = sgl_prot_count; 1530 se_cmd->se_cmd_flags |= SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC; 1531 } 1532 1533 /* 1534 * When a non zero sgl_count has been passed perform SGL passthrough 1535 * mapping for pre-allocated fabric memory instead of having target 1536 * core perform an internal SGL allocation.. 1537 */ 1538 if (sgl_count != 0) { 1539 BUG_ON(!sgl); 1540 1541 /* 1542 * A work-around for tcm_loop as some userspace code via 1543 * scsi-generic do not memset their associated read buffers, 1544 * so go ahead and do that here for type non-data CDBs. Also 1545 * note that this is currently guaranteed to be a single SGL 1546 * for this case by target core in target_setup_cmd_from_cdb() 1547 * -> transport_generic_cmd_sequencer(). 1548 */ 1549 if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) && 1550 se_cmd->data_direction == DMA_FROM_DEVICE) { 1551 unsigned char *buf = NULL; 1552 1553 if (sgl) 1554 buf = kmap(sg_page(sgl)) + sgl->offset; 1555 1556 if (buf) { 1557 memset(buf, 0, sgl->length); 1558 kunmap(sg_page(sgl)); 1559 } 1560 } 1561 1562 rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count, 1563 sgl_bidi, sgl_bidi_count); 1564 if (rc != 0) { 1565 transport_generic_request_failure(se_cmd, rc); 1566 return 0; 1567 } 1568 } 1569 1570 /* 1571 * Check if we need to delay processing because of ALUA 1572 * Active/NonOptimized primary access state.. 1573 */ 1574 core_alua_check_nonop_delay(se_cmd); 1575 1576 transport_handle_cdb_direct(se_cmd); 1577 return 0; 1578 } 1579 EXPORT_SYMBOL(target_submit_cmd_map_sgls); 1580 1581 /* 1582 * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd 1583 * 1584 * @se_cmd: command descriptor to submit 1585 * @se_sess: associated se_sess for endpoint 1586 * @cdb: pointer to SCSI CDB 1587 * @sense: pointer to SCSI sense buffer 1588 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1589 * @data_length: fabric expected data transfer length 1590 * @task_addr: SAM task attribute 1591 * @data_dir: DMA data direction 1592 * @flags: flags for command submission from target_sc_flags_tables 1593 * 1594 * Task tags are supported if the caller has set @se_cmd->tag. 1595 * 1596 * Returns non zero to signal active I/O shutdown failure. All other 1597 * setup exceptions will be returned as a SCSI CHECK_CONDITION response, 1598 * but still return zero here. 1599 * 1600 * This may only be called from process context, and also currently 1601 * assumes internal allocation of fabric payload buffer by target-core. 1602 * 1603 * It also assumes interal target core SGL memory allocation. 1604 */ 1605 int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, 1606 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, 1607 u32 data_length, int task_attr, int data_dir, int flags) 1608 { 1609 return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense, 1610 unpacked_lun, data_length, task_attr, data_dir, 1611 flags, NULL, 0, NULL, 0, NULL, 0); 1612 } 1613 EXPORT_SYMBOL(target_submit_cmd); 1614 1615 static void target_complete_tmr_failure(struct work_struct *work) 1616 { 1617 struct se_cmd *se_cmd = container_of(work, struct se_cmd, work); 1618 1619 se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST; 1620 se_cmd->se_tfo->queue_tm_rsp(se_cmd); 1621 1622 transport_lun_remove_cmd(se_cmd); 1623 transport_cmd_check_stop_to_fabric(se_cmd); 1624 } 1625 1626 static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag, 1627 u64 *unpacked_lun) 1628 { 1629 struct se_cmd *se_cmd; 1630 unsigned long flags; 1631 bool ret = false; 1632 1633 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 1634 list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) { 1635 if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) 1636 continue; 1637 1638 if (se_cmd->tag == tag) { 1639 *unpacked_lun = se_cmd->orig_fe_lun; 1640 ret = true; 1641 break; 1642 } 1643 } 1644 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 1645 1646 return ret; 1647 } 1648 1649 /** 1650 * target_submit_tmr - lookup unpacked lun and submit uninitialized se_cmd 1651 * for TMR CDBs 1652 * 1653 * @se_cmd: command descriptor to submit 1654 * @se_sess: associated se_sess for endpoint 1655 * @sense: pointer to SCSI sense buffer 1656 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1657 * @fabric_context: fabric context for TMR req 1658 * @tm_type: Type of TM request 1659 * @gfp: gfp type for caller 1660 * @tag: referenced task tag for TMR_ABORT_TASK 1661 * @flags: submit cmd flags 1662 * 1663 * Callable from all contexts. 1664 **/ 1665 1666 int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, 1667 unsigned char *sense, u64 unpacked_lun, 1668 void *fabric_tmr_ptr, unsigned char tm_type, 1669 gfp_t gfp, u64 tag, int flags) 1670 { 1671 struct se_portal_group *se_tpg; 1672 int ret; 1673 1674 se_tpg = se_sess->se_tpg; 1675 BUG_ON(!se_tpg); 1676 1677 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, 1678 0, DMA_NONE, TCM_SIMPLE_TAG, sense); 1679 /* 1680 * FIXME: Currently expect caller to handle se_cmd->se_tmr_req 1681 * allocation failure. 1682 */ 1683 ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp); 1684 if (ret < 0) 1685 return -ENOMEM; 1686 1687 if (tm_type == TMR_ABORT_TASK) 1688 se_cmd->se_tmr_req->ref_task_tag = tag; 1689 1690 /* See target_submit_cmd for commentary */ 1691 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); 1692 if (ret) { 1693 core_tmr_release_req(se_cmd->se_tmr_req); 1694 return ret; 1695 } 1696 /* 1697 * If this is ABORT_TASK with no explicit fabric provided LUN, 1698 * go ahead and search active session tags for a match to figure 1699 * out unpacked_lun for the original se_cmd. 1700 */ 1701 if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) { 1702 if (!target_lookup_lun_from_tag(se_sess, tag, &unpacked_lun)) 1703 goto failure; 1704 } 1705 1706 ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun); 1707 if (ret) 1708 goto failure; 1709 1710 transport_generic_handle_tmr(se_cmd); 1711 return 0; 1712 1713 /* 1714 * For callback during failure handling, push this work off 1715 * to process context with TMR_LUN_DOES_NOT_EXIST status. 1716 */ 1717 failure: 1718 INIT_WORK(&se_cmd->work, target_complete_tmr_failure); 1719 schedule_work(&se_cmd->work); 1720 return 0; 1721 } 1722 EXPORT_SYMBOL(target_submit_tmr); 1723 1724 /* 1725 * Handle SAM-esque emulation for generic transport request failures. 1726 */ 1727 void transport_generic_request_failure(struct se_cmd *cmd, 1728 sense_reason_t sense_reason) 1729 { 1730 int ret = 0, post_ret = 0; 1731 1732 pr_debug("-----[ Storage Engine Exception; sense_reason %d\n", 1733 sense_reason); 1734 target_show_cmd("-----[ ", cmd); 1735 1736 /* 1737 * For SAM Task Attribute emulation for failed struct se_cmd 1738 */ 1739 transport_complete_task_attr(cmd); 1740 1741 /* 1742 * Handle special case for COMPARE_AND_WRITE failure, where the 1743 * callback is expected to drop the per device ->caw_sem. 1744 */ 1745 if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) && 1746 cmd->transport_complete_callback) 1747 cmd->transport_complete_callback(cmd, false, &post_ret); 1748 1749 if (transport_check_aborted_status(cmd, 1)) 1750 return; 1751 1752 switch (sense_reason) { 1753 case TCM_NON_EXISTENT_LUN: 1754 case TCM_UNSUPPORTED_SCSI_OPCODE: 1755 case TCM_INVALID_CDB_FIELD: 1756 case TCM_INVALID_PARAMETER_LIST: 1757 case TCM_PARAMETER_LIST_LENGTH_ERROR: 1758 case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE: 1759 case TCM_UNKNOWN_MODE_PAGE: 1760 case TCM_WRITE_PROTECTED: 1761 case TCM_ADDRESS_OUT_OF_RANGE: 1762 case TCM_CHECK_CONDITION_ABORT_CMD: 1763 case TCM_CHECK_CONDITION_UNIT_ATTENTION: 1764 case TCM_CHECK_CONDITION_NOT_READY: 1765 case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED: 1766 case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED: 1767 case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED: 1768 case TCM_COPY_TARGET_DEVICE_NOT_REACHABLE: 1769 case TCM_TOO_MANY_TARGET_DESCS: 1770 case TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE: 1771 case TCM_TOO_MANY_SEGMENT_DESCS: 1772 case TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE: 1773 break; 1774 case TCM_OUT_OF_RESOURCES: 1775 cmd->scsi_status = SAM_STAT_TASK_SET_FULL; 1776 goto queue_status; 1777 case TCM_RESERVATION_CONFLICT: 1778 /* 1779 * No SENSE Data payload for this case, set SCSI Status 1780 * and queue the response to $FABRIC_MOD. 1781 * 1782 * Uses linux/include/scsi/scsi.h SAM status codes defs 1783 */ 1784 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT; 1785 /* 1786 * For UA Interlock Code 11b, a RESERVATION CONFLICT will 1787 * establish a UNIT ATTENTION with PREVIOUS RESERVATION 1788 * CONFLICT STATUS. 1789 * 1790 * See spc4r17, section 7.4.6 Control Mode Page, Table 349 1791 */ 1792 if (cmd->se_sess && 1793 cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) { 1794 target_ua_allocate_lun(cmd->se_sess->se_node_acl, 1795 cmd->orig_fe_lun, 0x2C, 1796 ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS); 1797 } 1798 1799 goto queue_status; 1800 default: 1801 pr_err("Unknown transport error for CDB 0x%02x: %d\n", 1802 cmd->t_task_cdb[0], sense_reason); 1803 sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE; 1804 break; 1805 } 1806 1807 ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0); 1808 if (ret) 1809 goto queue_full; 1810 1811 check_stop: 1812 transport_lun_remove_cmd(cmd); 1813 transport_cmd_check_stop_to_fabric(cmd); 1814 return; 1815 1816 queue_status: 1817 trace_target_cmd_complete(cmd); 1818 ret = cmd->se_tfo->queue_status(cmd); 1819 if (!ret) 1820 goto check_stop; 1821 queue_full: 1822 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 1823 } 1824 EXPORT_SYMBOL(transport_generic_request_failure); 1825 1826 void __target_execute_cmd(struct se_cmd *cmd, bool do_checks) 1827 { 1828 sense_reason_t ret; 1829 1830 if (!cmd->execute_cmd) { 1831 ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 1832 goto err; 1833 } 1834 if (do_checks) { 1835 /* 1836 * Check for an existing UNIT ATTENTION condition after 1837 * target_handle_task_attr() has done SAM task attr 1838 * checking, and possibly have already defered execution 1839 * out to target_restart_delayed_cmds() context. 1840 */ 1841 ret = target_scsi3_ua_check(cmd); 1842 if (ret) 1843 goto err; 1844 1845 ret = target_alua_state_check(cmd); 1846 if (ret) 1847 goto err; 1848 1849 ret = target_check_reservation(cmd); 1850 if (ret) { 1851 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT; 1852 goto err; 1853 } 1854 } 1855 1856 ret = cmd->execute_cmd(cmd); 1857 if (!ret) 1858 return; 1859 err: 1860 spin_lock_irq(&cmd->t_state_lock); 1861 cmd->transport_state &= ~CMD_T_SENT; 1862 spin_unlock_irq(&cmd->t_state_lock); 1863 1864 transport_generic_request_failure(cmd, ret); 1865 } 1866 1867 static int target_write_prot_action(struct se_cmd *cmd) 1868 { 1869 u32 sectors; 1870 /* 1871 * Perform WRITE_INSERT of PI using software emulation when backend 1872 * device has PI enabled, if the transport has not already generated 1873 * PI using hardware WRITE_INSERT offload. 1874 */ 1875 switch (cmd->prot_op) { 1876 case TARGET_PROT_DOUT_INSERT: 1877 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_INSERT)) 1878 sbc_dif_generate(cmd); 1879 break; 1880 case TARGET_PROT_DOUT_STRIP: 1881 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_STRIP) 1882 break; 1883 1884 sectors = cmd->data_length >> ilog2(cmd->se_dev->dev_attrib.block_size); 1885 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba, 1886 sectors, 0, cmd->t_prot_sg, 0); 1887 if (unlikely(cmd->pi_err)) { 1888 spin_lock_irq(&cmd->t_state_lock); 1889 cmd->transport_state &= ~CMD_T_SENT; 1890 spin_unlock_irq(&cmd->t_state_lock); 1891 transport_generic_request_failure(cmd, cmd->pi_err); 1892 return -1; 1893 } 1894 break; 1895 default: 1896 break; 1897 } 1898 1899 return 0; 1900 } 1901 1902 static bool target_handle_task_attr(struct se_cmd *cmd) 1903 { 1904 struct se_device *dev = cmd->se_dev; 1905 1906 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1907 return false; 1908 1909 cmd->se_cmd_flags |= SCF_TASK_ATTR_SET; 1910 1911 /* 1912 * Check for the existence of HEAD_OF_QUEUE, and if true return 1 1913 * to allow the passed struct se_cmd list of tasks to the front of the list. 1914 */ 1915 switch (cmd->sam_task_attr) { 1916 case TCM_HEAD_TAG: 1917 pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x\n", 1918 cmd->t_task_cdb[0]); 1919 return false; 1920 case TCM_ORDERED_TAG: 1921 atomic_inc_mb(&dev->dev_ordered_sync); 1922 1923 pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n", 1924 cmd->t_task_cdb[0]); 1925 1926 /* 1927 * Execute an ORDERED command if no other older commands 1928 * exist that need to be completed first. 1929 */ 1930 if (!atomic_read(&dev->simple_cmds)) 1931 return false; 1932 break; 1933 default: 1934 /* 1935 * For SIMPLE and UNTAGGED Task Attribute commands 1936 */ 1937 atomic_inc_mb(&dev->simple_cmds); 1938 break; 1939 } 1940 1941 if (atomic_read(&dev->dev_ordered_sync) == 0) 1942 return false; 1943 1944 spin_lock(&dev->delayed_cmd_lock); 1945 list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list); 1946 spin_unlock(&dev->delayed_cmd_lock); 1947 1948 pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn", 1949 cmd->t_task_cdb[0], cmd->sam_task_attr); 1950 return true; 1951 } 1952 1953 static int __transport_check_aborted_status(struct se_cmd *, int); 1954 1955 void target_execute_cmd(struct se_cmd *cmd) 1956 { 1957 /* 1958 * Determine if frontend context caller is requesting the stopping of 1959 * this command for frontend exceptions. 1960 * 1961 * If the received CDB has aleady been aborted stop processing it here. 1962 */ 1963 spin_lock_irq(&cmd->t_state_lock); 1964 if (__transport_check_aborted_status(cmd, 1)) { 1965 spin_unlock_irq(&cmd->t_state_lock); 1966 return; 1967 } 1968 if (cmd->transport_state & CMD_T_STOP) { 1969 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 1970 __func__, __LINE__, cmd->tag); 1971 1972 spin_unlock_irq(&cmd->t_state_lock); 1973 complete_all(&cmd->t_transport_stop_comp); 1974 return; 1975 } 1976 1977 cmd->t_state = TRANSPORT_PROCESSING; 1978 cmd->transport_state &= ~CMD_T_PRE_EXECUTE; 1979 cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT; 1980 spin_unlock_irq(&cmd->t_state_lock); 1981 1982 if (target_write_prot_action(cmd)) 1983 return; 1984 1985 if (target_handle_task_attr(cmd)) { 1986 spin_lock_irq(&cmd->t_state_lock); 1987 cmd->transport_state &= ~CMD_T_SENT; 1988 spin_unlock_irq(&cmd->t_state_lock); 1989 return; 1990 } 1991 1992 __target_execute_cmd(cmd, true); 1993 } 1994 EXPORT_SYMBOL(target_execute_cmd); 1995 1996 /* 1997 * Process all commands up to the last received ORDERED task attribute which 1998 * requires another blocking boundary 1999 */ 2000 static void target_restart_delayed_cmds(struct se_device *dev) 2001 { 2002 for (;;) { 2003 struct se_cmd *cmd; 2004 2005 spin_lock(&dev->delayed_cmd_lock); 2006 if (list_empty(&dev->delayed_cmd_list)) { 2007 spin_unlock(&dev->delayed_cmd_lock); 2008 break; 2009 } 2010 2011 cmd = list_entry(dev->delayed_cmd_list.next, 2012 struct se_cmd, se_delayed_node); 2013 list_del(&cmd->se_delayed_node); 2014 spin_unlock(&dev->delayed_cmd_lock); 2015 2016 cmd->transport_state |= CMD_T_SENT; 2017 2018 __target_execute_cmd(cmd, true); 2019 2020 if (cmd->sam_task_attr == TCM_ORDERED_TAG) 2021 break; 2022 } 2023 } 2024 2025 /* 2026 * Called from I/O completion to determine which dormant/delayed 2027 * and ordered cmds need to have their tasks added to the execution queue. 2028 */ 2029 static void transport_complete_task_attr(struct se_cmd *cmd) 2030 { 2031 struct se_device *dev = cmd->se_dev; 2032 2033 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 2034 return; 2035 2036 if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET)) 2037 goto restart; 2038 2039 if (cmd->sam_task_attr == TCM_SIMPLE_TAG) { 2040 atomic_dec_mb(&dev->simple_cmds); 2041 dev->dev_cur_ordered_id++; 2042 } else if (cmd->sam_task_attr == TCM_HEAD_TAG) { 2043 dev->dev_cur_ordered_id++; 2044 pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n", 2045 dev->dev_cur_ordered_id); 2046 } else if (cmd->sam_task_attr == TCM_ORDERED_TAG) { 2047 atomic_dec_mb(&dev->dev_ordered_sync); 2048 2049 dev->dev_cur_ordered_id++; 2050 pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n", 2051 dev->dev_cur_ordered_id); 2052 } 2053 cmd->se_cmd_flags &= ~SCF_TASK_ATTR_SET; 2054 2055 restart: 2056 target_restart_delayed_cmds(dev); 2057 } 2058 2059 static void transport_complete_qf(struct se_cmd *cmd) 2060 { 2061 int ret = 0; 2062 2063 transport_complete_task_attr(cmd); 2064 /* 2065 * If a fabric driver ->write_pending() or ->queue_data_in() callback 2066 * has returned neither -ENOMEM or -EAGAIN, assume it's fatal and 2067 * the same callbacks should not be retried. Return CHECK_CONDITION 2068 * if a scsi_status is not already set. 2069 * 2070 * If a fabric driver ->queue_status() has returned non zero, always 2071 * keep retrying no matter what.. 2072 */ 2073 if (cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) { 2074 if (cmd->scsi_status) 2075 goto queue_status; 2076 2077 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; 2078 cmd->scsi_status = SAM_STAT_CHECK_CONDITION; 2079 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 2080 translate_sense_reason(cmd, TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); 2081 goto queue_status; 2082 } 2083 2084 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) 2085 goto queue_status; 2086 2087 switch (cmd->data_direction) { 2088 case DMA_FROM_DEVICE: 2089 if (cmd->scsi_status) 2090 goto queue_status; 2091 2092 trace_target_cmd_complete(cmd); 2093 ret = cmd->se_tfo->queue_data_in(cmd); 2094 break; 2095 case DMA_TO_DEVICE: 2096 if (cmd->se_cmd_flags & SCF_BIDI) { 2097 ret = cmd->se_tfo->queue_data_in(cmd); 2098 break; 2099 } 2100 /* fall through */ 2101 case DMA_NONE: 2102 queue_status: 2103 trace_target_cmd_complete(cmd); 2104 ret = cmd->se_tfo->queue_status(cmd); 2105 break; 2106 default: 2107 break; 2108 } 2109 2110 if (ret < 0) { 2111 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 2112 return; 2113 } 2114 transport_lun_remove_cmd(cmd); 2115 transport_cmd_check_stop_to_fabric(cmd); 2116 } 2117 2118 static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev, 2119 int err, bool write_pending) 2120 { 2121 /* 2122 * -EAGAIN or -ENOMEM signals retry of ->write_pending() and/or 2123 * ->queue_data_in() callbacks from new process context. 2124 * 2125 * Otherwise for other errors, transport_complete_qf() will send 2126 * CHECK_CONDITION via ->queue_status() instead of attempting to 2127 * retry associated fabric driver data-transfer callbacks. 2128 */ 2129 if (err == -EAGAIN || err == -ENOMEM) { 2130 cmd->t_state = (write_pending) ? TRANSPORT_COMPLETE_QF_WP : 2131 TRANSPORT_COMPLETE_QF_OK; 2132 } else { 2133 pr_warn_ratelimited("Got unknown fabric queue status: %d\n", err); 2134 cmd->t_state = TRANSPORT_COMPLETE_QF_ERR; 2135 } 2136 2137 spin_lock_irq(&dev->qf_cmd_lock); 2138 list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list); 2139 atomic_inc_mb(&dev->dev_qf_count); 2140 spin_unlock_irq(&cmd->se_dev->qf_cmd_lock); 2141 2142 schedule_work(&cmd->se_dev->qf_work_queue); 2143 } 2144 2145 static bool target_read_prot_action(struct se_cmd *cmd) 2146 { 2147 switch (cmd->prot_op) { 2148 case TARGET_PROT_DIN_STRIP: 2149 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_STRIP)) { 2150 u32 sectors = cmd->data_length >> 2151 ilog2(cmd->se_dev->dev_attrib.block_size); 2152 2153 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba, 2154 sectors, 0, cmd->t_prot_sg, 2155 0); 2156 if (cmd->pi_err) 2157 return true; 2158 } 2159 break; 2160 case TARGET_PROT_DIN_INSERT: 2161 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_INSERT) 2162 break; 2163 2164 sbc_dif_generate(cmd); 2165 break; 2166 default: 2167 break; 2168 } 2169 2170 return false; 2171 } 2172 2173 static void target_complete_ok_work(struct work_struct *work) 2174 { 2175 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 2176 int ret; 2177 2178 /* 2179 * Check if we need to move delayed/dormant tasks from cmds on the 2180 * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task 2181 * Attribute. 2182 */ 2183 transport_complete_task_attr(cmd); 2184 2185 /* 2186 * Check to schedule QUEUE_FULL work, or execute an existing 2187 * cmd->transport_qf_callback() 2188 */ 2189 if (atomic_read(&cmd->se_dev->dev_qf_count) != 0) 2190 schedule_work(&cmd->se_dev->qf_work_queue); 2191 2192 /* 2193 * Check if we need to send a sense buffer from 2194 * the struct se_cmd in question. 2195 */ 2196 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) { 2197 WARN_ON(!cmd->scsi_status); 2198 ret = transport_send_check_condition_and_sense( 2199 cmd, 0, 1); 2200 if (ret) 2201 goto queue_full; 2202 2203 transport_lun_remove_cmd(cmd); 2204 transport_cmd_check_stop_to_fabric(cmd); 2205 return; 2206 } 2207 /* 2208 * Check for a callback, used by amongst other things 2209 * XDWRITE_READ_10 and COMPARE_AND_WRITE emulation. 2210 */ 2211 if (cmd->transport_complete_callback) { 2212 sense_reason_t rc; 2213 bool caw = (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE); 2214 bool zero_dl = !(cmd->data_length); 2215 int post_ret = 0; 2216 2217 rc = cmd->transport_complete_callback(cmd, true, &post_ret); 2218 if (!rc && !post_ret) { 2219 if (caw && zero_dl) 2220 goto queue_rsp; 2221 2222 return; 2223 } else if (rc) { 2224 ret = transport_send_check_condition_and_sense(cmd, 2225 rc, 0); 2226 if (ret) 2227 goto queue_full; 2228 2229 transport_lun_remove_cmd(cmd); 2230 transport_cmd_check_stop_to_fabric(cmd); 2231 return; 2232 } 2233 } 2234 2235 queue_rsp: 2236 switch (cmd->data_direction) { 2237 case DMA_FROM_DEVICE: 2238 if (cmd->scsi_status) 2239 goto queue_status; 2240 2241 atomic_long_add(cmd->data_length, 2242 &cmd->se_lun->lun_stats.tx_data_octets); 2243 /* 2244 * Perform READ_STRIP of PI using software emulation when 2245 * backend had PI enabled, if the transport will not be 2246 * performing hardware READ_STRIP offload. 2247 */ 2248 if (target_read_prot_action(cmd)) { 2249 ret = transport_send_check_condition_and_sense(cmd, 2250 cmd->pi_err, 0); 2251 if (ret) 2252 goto queue_full; 2253 2254 transport_lun_remove_cmd(cmd); 2255 transport_cmd_check_stop_to_fabric(cmd); 2256 return; 2257 } 2258 2259 trace_target_cmd_complete(cmd); 2260 ret = cmd->se_tfo->queue_data_in(cmd); 2261 if (ret) 2262 goto queue_full; 2263 break; 2264 case DMA_TO_DEVICE: 2265 atomic_long_add(cmd->data_length, 2266 &cmd->se_lun->lun_stats.rx_data_octets); 2267 /* 2268 * Check if we need to send READ payload for BIDI-COMMAND 2269 */ 2270 if (cmd->se_cmd_flags & SCF_BIDI) { 2271 atomic_long_add(cmd->data_length, 2272 &cmd->se_lun->lun_stats.tx_data_octets); 2273 ret = cmd->se_tfo->queue_data_in(cmd); 2274 if (ret) 2275 goto queue_full; 2276 break; 2277 } 2278 /* fall through */ 2279 case DMA_NONE: 2280 queue_status: 2281 trace_target_cmd_complete(cmd); 2282 ret = cmd->se_tfo->queue_status(cmd); 2283 if (ret) 2284 goto queue_full; 2285 break; 2286 default: 2287 break; 2288 } 2289 2290 transport_lun_remove_cmd(cmd); 2291 transport_cmd_check_stop_to_fabric(cmd); 2292 return; 2293 2294 queue_full: 2295 pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p," 2296 " data_direction: %d\n", cmd, cmd->data_direction); 2297 2298 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 2299 } 2300 2301 void target_free_sgl(struct scatterlist *sgl, int nents) 2302 { 2303 struct scatterlist *sg; 2304 int count; 2305 2306 for_each_sg(sgl, sg, nents, count) 2307 __free_page(sg_page(sg)); 2308 2309 kfree(sgl); 2310 } 2311 EXPORT_SYMBOL(target_free_sgl); 2312 2313 static inline void transport_reset_sgl_orig(struct se_cmd *cmd) 2314 { 2315 /* 2316 * Check for saved t_data_sg that may be used for COMPARE_AND_WRITE 2317 * emulation, and free + reset pointers if necessary.. 2318 */ 2319 if (!cmd->t_data_sg_orig) 2320 return; 2321 2322 kfree(cmd->t_data_sg); 2323 cmd->t_data_sg = cmd->t_data_sg_orig; 2324 cmd->t_data_sg_orig = NULL; 2325 cmd->t_data_nents = cmd->t_data_nents_orig; 2326 cmd->t_data_nents_orig = 0; 2327 } 2328 2329 static inline void transport_free_pages(struct se_cmd *cmd) 2330 { 2331 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) { 2332 target_free_sgl(cmd->t_prot_sg, cmd->t_prot_nents); 2333 cmd->t_prot_sg = NULL; 2334 cmd->t_prot_nents = 0; 2335 } 2336 2337 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 2338 /* 2339 * Release special case READ buffer payload required for 2340 * SG_TO_MEM_NOALLOC to function with COMPARE_AND_WRITE 2341 */ 2342 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) { 2343 target_free_sgl(cmd->t_bidi_data_sg, 2344 cmd->t_bidi_data_nents); 2345 cmd->t_bidi_data_sg = NULL; 2346 cmd->t_bidi_data_nents = 0; 2347 } 2348 transport_reset_sgl_orig(cmd); 2349 return; 2350 } 2351 transport_reset_sgl_orig(cmd); 2352 2353 target_free_sgl(cmd->t_data_sg, cmd->t_data_nents); 2354 cmd->t_data_sg = NULL; 2355 cmd->t_data_nents = 0; 2356 2357 target_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents); 2358 cmd->t_bidi_data_sg = NULL; 2359 cmd->t_bidi_data_nents = 0; 2360 } 2361 2362 void *transport_kmap_data_sg(struct se_cmd *cmd) 2363 { 2364 struct scatterlist *sg = cmd->t_data_sg; 2365 struct page **pages; 2366 int i; 2367 2368 /* 2369 * We need to take into account a possible offset here for fabrics like 2370 * tcm_loop who may be using a contig buffer from the SCSI midlayer for 2371 * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd() 2372 */ 2373 if (!cmd->t_data_nents) 2374 return NULL; 2375 2376 BUG_ON(!sg); 2377 if (cmd->t_data_nents == 1) 2378 return kmap(sg_page(sg)) + sg->offset; 2379 2380 /* >1 page. use vmap */ 2381 pages = kmalloc_array(cmd->t_data_nents, sizeof(*pages), GFP_KERNEL); 2382 if (!pages) 2383 return NULL; 2384 2385 /* convert sg[] to pages[] */ 2386 for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) { 2387 pages[i] = sg_page(sg); 2388 } 2389 2390 cmd->t_data_vmap = vmap(pages, cmd->t_data_nents, VM_MAP, PAGE_KERNEL); 2391 kfree(pages); 2392 if (!cmd->t_data_vmap) 2393 return NULL; 2394 2395 return cmd->t_data_vmap + cmd->t_data_sg[0].offset; 2396 } 2397 EXPORT_SYMBOL(transport_kmap_data_sg); 2398 2399 void transport_kunmap_data_sg(struct se_cmd *cmd) 2400 { 2401 if (!cmd->t_data_nents) { 2402 return; 2403 } else if (cmd->t_data_nents == 1) { 2404 kunmap(sg_page(cmd->t_data_sg)); 2405 return; 2406 } 2407 2408 vunmap(cmd->t_data_vmap); 2409 cmd->t_data_vmap = NULL; 2410 } 2411 EXPORT_SYMBOL(transport_kunmap_data_sg); 2412 2413 int 2414 target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length, 2415 bool zero_page, bool chainable) 2416 { 2417 struct scatterlist *sg; 2418 struct page *page; 2419 gfp_t zero_flag = (zero_page) ? __GFP_ZERO : 0; 2420 unsigned int nalloc, nent; 2421 int i = 0; 2422 2423 nalloc = nent = DIV_ROUND_UP(length, PAGE_SIZE); 2424 if (chainable) 2425 nalloc++; 2426 sg = kmalloc_array(nalloc, sizeof(struct scatterlist), GFP_KERNEL); 2427 if (!sg) 2428 return -ENOMEM; 2429 2430 sg_init_table(sg, nalloc); 2431 2432 while (length) { 2433 u32 page_len = min_t(u32, length, PAGE_SIZE); 2434 page = alloc_page(GFP_KERNEL | zero_flag); 2435 if (!page) 2436 goto out; 2437 2438 sg_set_page(&sg[i], page, page_len, 0); 2439 length -= page_len; 2440 i++; 2441 } 2442 *sgl = sg; 2443 *nents = nent; 2444 return 0; 2445 2446 out: 2447 while (i > 0) { 2448 i--; 2449 __free_page(sg_page(&sg[i])); 2450 } 2451 kfree(sg); 2452 return -ENOMEM; 2453 } 2454 EXPORT_SYMBOL(target_alloc_sgl); 2455 2456 /* 2457 * Allocate any required resources to execute the command. For writes we 2458 * might not have the payload yet, so notify the fabric via a call to 2459 * ->write_pending instead. Otherwise place it on the execution queue. 2460 */ 2461 sense_reason_t 2462 transport_generic_new_cmd(struct se_cmd *cmd) 2463 { 2464 unsigned long flags; 2465 int ret = 0; 2466 bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB); 2467 2468 if (cmd->prot_op != TARGET_PROT_NORMAL && 2469 !(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) { 2470 ret = target_alloc_sgl(&cmd->t_prot_sg, &cmd->t_prot_nents, 2471 cmd->prot_length, true, false); 2472 if (ret < 0) 2473 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2474 } 2475 2476 /* 2477 * Determine is the TCM fabric module has already allocated physical 2478 * memory, and is directly calling transport_generic_map_mem_to_cmd() 2479 * beforehand. 2480 */ 2481 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) && 2482 cmd->data_length) { 2483 2484 if ((cmd->se_cmd_flags & SCF_BIDI) || 2485 (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)) { 2486 u32 bidi_length; 2487 2488 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) 2489 bidi_length = cmd->t_task_nolb * 2490 cmd->se_dev->dev_attrib.block_size; 2491 else 2492 bidi_length = cmd->data_length; 2493 2494 ret = target_alloc_sgl(&cmd->t_bidi_data_sg, 2495 &cmd->t_bidi_data_nents, 2496 bidi_length, zero_flag, false); 2497 if (ret < 0) 2498 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2499 } 2500 2501 ret = target_alloc_sgl(&cmd->t_data_sg, &cmd->t_data_nents, 2502 cmd->data_length, zero_flag, false); 2503 if (ret < 0) 2504 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2505 } else if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) && 2506 cmd->data_length) { 2507 /* 2508 * Special case for COMPARE_AND_WRITE with fabrics 2509 * using SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC. 2510 */ 2511 u32 caw_length = cmd->t_task_nolb * 2512 cmd->se_dev->dev_attrib.block_size; 2513 2514 ret = target_alloc_sgl(&cmd->t_bidi_data_sg, 2515 &cmd->t_bidi_data_nents, 2516 caw_length, zero_flag, false); 2517 if (ret < 0) 2518 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2519 } 2520 /* 2521 * If this command is not a write we can execute it right here, 2522 * for write buffers we need to notify the fabric driver first 2523 * and let it call back once the write buffers are ready. 2524 */ 2525 target_add_to_state_list(cmd); 2526 if (cmd->data_direction != DMA_TO_DEVICE || cmd->data_length == 0) { 2527 target_execute_cmd(cmd); 2528 return 0; 2529 } 2530 2531 spin_lock_irqsave(&cmd->t_state_lock, flags); 2532 cmd->t_state = TRANSPORT_WRITE_PENDING; 2533 /* 2534 * Determine if frontend context caller is requesting the stopping of 2535 * this command for frontend exceptions. 2536 */ 2537 if (cmd->transport_state & CMD_T_STOP) { 2538 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 2539 __func__, __LINE__, cmd->tag); 2540 2541 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2542 2543 complete_all(&cmd->t_transport_stop_comp); 2544 return 0; 2545 } 2546 cmd->transport_state &= ~CMD_T_ACTIVE; 2547 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2548 2549 ret = cmd->se_tfo->write_pending(cmd); 2550 if (ret) 2551 goto queue_full; 2552 2553 return 0; 2554 2555 queue_full: 2556 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd); 2557 transport_handle_queue_full(cmd, cmd->se_dev, ret, true); 2558 return 0; 2559 } 2560 EXPORT_SYMBOL(transport_generic_new_cmd); 2561 2562 static void transport_write_pending_qf(struct se_cmd *cmd) 2563 { 2564 unsigned long flags; 2565 int ret; 2566 bool stop; 2567 2568 spin_lock_irqsave(&cmd->t_state_lock, flags); 2569 stop = (cmd->transport_state & (CMD_T_STOP | CMD_T_ABORTED)); 2570 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2571 2572 if (stop) { 2573 pr_debug("%s:%d CMD_T_STOP|CMD_T_ABORTED for ITT: 0x%08llx\n", 2574 __func__, __LINE__, cmd->tag); 2575 complete_all(&cmd->t_transport_stop_comp); 2576 return; 2577 } 2578 2579 ret = cmd->se_tfo->write_pending(cmd); 2580 if (ret) { 2581 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", 2582 cmd); 2583 transport_handle_queue_full(cmd, cmd->se_dev, ret, true); 2584 } 2585 } 2586 2587 static bool 2588 __transport_wait_for_tasks(struct se_cmd *, bool, bool *, bool *, 2589 unsigned long *flags); 2590 2591 static void target_wait_free_cmd(struct se_cmd *cmd, bool *aborted, bool *tas) 2592 { 2593 unsigned long flags; 2594 2595 spin_lock_irqsave(&cmd->t_state_lock, flags); 2596 __transport_wait_for_tasks(cmd, true, aborted, tas, &flags); 2597 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2598 } 2599 2600 int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks) 2601 { 2602 int ret = 0; 2603 bool aborted = false, tas = false; 2604 2605 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) { 2606 if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2607 target_wait_free_cmd(cmd, &aborted, &tas); 2608 2609 if (!aborted || tas) 2610 ret = target_put_sess_cmd(cmd); 2611 } else { 2612 if (wait_for_tasks) 2613 target_wait_free_cmd(cmd, &aborted, &tas); 2614 /* 2615 * Handle WRITE failure case where transport_generic_new_cmd() 2616 * has already added se_cmd to state_list, but fabric has 2617 * failed command before I/O submission. 2618 */ 2619 if (cmd->state_active) 2620 target_remove_from_state_list(cmd); 2621 2622 if (cmd->se_lun) 2623 transport_lun_remove_cmd(cmd); 2624 2625 if (!aborted || tas) 2626 ret = target_put_sess_cmd(cmd); 2627 } 2628 /* 2629 * If the task has been internally aborted due to TMR ABORT_TASK 2630 * or LUN_RESET, target_core_tmr.c is responsible for performing 2631 * the remaining calls to target_put_sess_cmd(), and not the 2632 * callers of this function. 2633 */ 2634 if (aborted) { 2635 pr_debug("Detected CMD_T_ABORTED for ITT: %llu\n", cmd->tag); 2636 wait_for_completion(&cmd->cmd_wait_comp); 2637 cmd->se_tfo->release_cmd(cmd); 2638 ret = 1; 2639 } 2640 return ret; 2641 } 2642 EXPORT_SYMBOL(transport_generic_free_cmd); 2643 2644 /* target_get_sess_cmd - Add command to active ->sess_cmd_list 2645 * @se_cmd: command descriptor to add 2646 * @ack_kref: Signal that fabric will perform an ack target_put_sess_cmd() 2647 */ 2648 int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref) 2649 { 2650 struct se_session *se_sess = se_cmd->se_sess; 2651 unsigned long flags; 2652 int ret = 0; 2653 2654 /* 2655 * Add a second kref if the fabric caller is expecting to handle 2656 * fabric acknowledgement that requires two target_put_sess_cmd() 2657 * invocations before se_cmd descriptor release. 2658 */ 2659 if (ack_kref) { 2660 if (!kref_get_unless_zero(&se_cmd->cmd_kref)) 2661 return -EINVAL; 2662 2663 se_cmd->se_cmd_flags |= SCF_ACK_KREF; 2664 } 2665 2666 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2667 if (se_sess->sess_tearing_down) { 2668 ret = -ESHUTDOWN; 2669 goto out; 2670 } 2671 se_cmd->transport_state |= CMD_T_PRE_EXECUTE; 2672 list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list); 2673 out: 2674 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2675 2676 if (ret && ack_kref) 2677 target_put_sess_cmd(se_cmd); 2678 2679 return ret; 2680 } 2681 EXPORT_SYMBOL(target_get_sess_cmd); 2682 2683 static void target_free_cmd_mem(struct se_cmd *cmd) 2684 { 2685 transport_free_pages(cmd); 2686 2687 if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) 2688 core_tmr_release_req(cmd->se_tmr_req); 2689 if (cmd->t_task_cdb != cmd->__t_task_cdb) 2690 kfree(cmd->t_task_cdb); 2691 } 2692 2693 static void target_release_cmd_kref(struct kref *kref) 2694 { 2695 struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref); 2696 struct se_session *se_sess = se_cmd->se_sess; 2697 unsigned long flags; 2698 bool fabric_stop; 2699 2700 if (se_sess) { 2701 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2702 2703 spin_lock(&se_cmd->t_state_lock); 2704 fabric_stop = (se_cmd->transport_state & CMD_T_FABRIC_STOP) && 2705 (se_cmd->transport_state & CMD_T_ABORTED); 2706 spin_unlock(&se_cmd->t_state_lock); 2707 2708 if (se_cmd->cmd_wait_set || fabric_stop) { 2709 list_del_init(&se_cmd->se_cmd_list); 2710 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2711 target_free_cmd_mem(se_cmd); 2712 complete(&se_cmd->cmd_wait_comp); 2713 return; 2714 } 2715 list_del_init(&se_cmd->se_cmd_list); 2716 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2717 } 2718 2719 target_free_cmd_mem(se_cmd); 2720 se_cmd->se_tfo->release_cmd(se_cmd); 2721 } 2722 2723 /** 2724 * target_put_sess_cmd - decrease the command reference count 2725 * @se_cmd: command to drop a reference from 2726 * 2727 * Returns 1 if and only if this target_put_sess_cmd() call caused the 2728 * refcount to drop to zero. Returns zero otherwise. 2729 */ 2730 int target_put_sess_cmd(struct se_cmd *se_cmd) 2731 { 2732 return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref); 2733 } 2734 EXPORT_SYMBOL(target_put_sess_cmd); 2735 2736 static const char *data_dir_name(enum dma_data_direction d) 2737 { 2738 switch (d) { 2739 case DMA_BIDIRECTIONAL: return "BIDI"; 2740 case DMA_TO_DEVICE: return "WRITE"; 2741 case DMA_FROM_DEVICE: return "READ"; 2742 case DMA_NONE: return "NONE"; 2743 } 2744 2745 return "(?)"; 2746 } 2747 2748 static const char *cmd_state_name(enum transport_state_table t) 2749 { 2750 switch (t) { 2751 case TRANSPORT_NO_STATE: return "NO_STATE"; 2752 case TRANSPORT_NEW_CMD: return "NEW_CMD"; 2753 case TRANSPORT_WRITE_PENDING: return "WRITE_PENDING"; 2754 case TRANSPORT_PROCESSING: return "PROCESSING"; 2755 case TRANSPORT_COMPLETE: return "COMPLETE"; 2756 case TRANSPORT_ISTATE_PROCESSING: 2757 return "ISTATE_PROCESSING"; 2758 case TRANSPORT_COMPLETE_QF_WP: return "COMPLETE_QF_WP"; 2759 case TRANSPORT_COMPLETE_QF_OK: return "COMPLETE_QF_OK"; 2760 case TRANSPORT_COMPLETE_QF_ERR: return "COMPLETE_QF_ERR"; 2761 } 2762 2763 return "(?)"; 2764 } 2765 2766 static void target_append_str(char **str, const char *txt) 2767 { 2768 char *prev = *str; 2769 2770 *str = *str ? kasprintf(GFP_ATOMIC, "%s,%s", *str, txt) : 2771 kstrdup(txt, GFP_ATOMIC); 2772 kfree(prev); 2773 } 2774 2775 /* 2776 * Convert a transport state bitmask into a string. The caller is 2777 * responsible for freeing the returned pointer. 2778 */ 2779 static char *target_ts_to_str(u32 ts) 2780 { 2781 char *str = NULL; 2782 2783 if (ts & CMD_T_ABORTED) 2784 target_append_str(&str, "aborted"); 2785 if (ts & CMD_T_ACTIVE) 2786 target_append_str(&str, "active"); 2787 if (ts & CMD_T_COMPLETE) 2788 target_append_str(&str, "complete"); 2789 if (ts & CMD_T_SENT) 2790 target_append_str(&str, "sent"); 2791 if (ts & CMD_T_STOP) 2792 target_append_str(&str, "stop"); 2793 if (ts & CMD_T_FABRIC_STOP) 2794 target_append_str(&str, "fabric_stop"); 2795 2796 return str; 2797 } 2798 2799 static const char *target_tmf_name(enum tcm_tmreq_table tmf) 2800 { 2801 switch (tmf) { 2802 case TMR_ABORT_TASK: return "ABORT_TASK"; 2803 case TMR_ABORT_TASK_SET: return "ABORT_TASK_SET"; 2804 case TMR_CLEAR_ACA: return "CLEAR_ACA"; 2805 case TMR_CLEAR_TASK_SET: return "CLEAR_TASK_SET"; 2806 case TMR_LUN_RESET: return "LUN_RESET"; 2807 case TMR_TARGET_WARM_RESET: return "TARGET_WARM_RESET"; 2808 case TMR_TARGET_COLD_RESET: return "TARGET_COLD_RESET"; 2809 case TMR_UNKNOWN: break; 2810 } 2811 return "(?)"; 2812 } 2813 2814 void target_show_cmd(const char *pfx, struct se_cmd *cmd) 2815 { 2816 char *ts_str = target_ts_to_str(cmd->transport_state); 2817 const u8 *cdb = cmd->t_task_cdb; 2818 struct se_tmr_req *tmf = cmd->se_tmr_req; 2819 2820 if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) { 2821 pr_debug("%scmd %#02x:%#02x with tag %#llx dir %s i_state %d t_state %s len %d refcnt %d transport_state %s\n", 2822 pfx, cdb[0], cdb[1], cmd->tag, 2823 data_dir_name(cmd->data_direction), 2824 cmd->se_tfo->get_cmd_state(cmd), 2825 cmd_state_name(cmd->t_state), cmd->data_length, 2826 kref_read(&cmd->cmd_kref), ts_str); 2827 } else { 2828 pr_debug("%stmf %s with tag %#llx ref_task_tag %#llx i_state %d t_state %s refcnt %d transport_state %s\n", 2829 pfx, target_tmf_name(tmf->function), cmd->tag, 2830 tmf->ref_task_tag, cmd->se_tfo->get_cmd_state(cmd), 2831 cmd_state_name(cmd->t_state), 2832 kref_read(&cmd->cmd_kref), ts_str); 2833 } 2834 kfree(ts_str); 2835 } 2836 EXPORT_SYMBOL(target_show_cmd); 2837 2838 /* target_sess_cmd_list_set_waiting - Flag all commands in 2839 * sess_cmd_list to complete cmd_wait_comp. Set 2840 * sess_tearing_down so no more commands are queued. 2841 * @se_sess: session to flag 2842 */ 2843 void target_sess_cmd_list_set_waiting(struct se_session *se_sess) 2844 { 2845 struct se_cmd *se_cmd, *tmp_cmd; 2846 unsigned long flags; 2847 int rc; 2848 2849 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2850 if (se_sess->sess_tearing_down) { 2851 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2852 return; 2853 } 2854 se_sess->sess_tearing_down = 1; 2855 list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list); 2856 2857 list_for_each_entry_safe(se_cmd, tmp_cmd, 2858 &se_sess->sess_wait_list, se_cmd_list) { 2859 rc = kref_get_unless_zero(&se_cmd->cmd_kref); 2860 if (rc) { 2861 se_cmd->cmd_wait_set = 1; 2862 spin_lock(&se_cmd->t_state_lock); 2863 se_cmd->transport_state |= CMD_T_FABRIC_STOP; 2864 spin_unlock(&se_cmd->t_state_lock); 2865 } else 2866 list_del_init(&se_cmd->se_cmd_list); 2867 } 2868 2869 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2870 } 2871 EXPORT_SYMBOL(target_sess_cmd_list_set_waiting); 2872 2873 /* target_wait_for_sess_cmds - Wait for outstanding descriptors 2874 * @se_sess: session to wait for active I/O 2875 */ 2876 void target_wait_for_sess_cmds(struct se_session *se_sess) 2877 { 2878 struct se_cmd *se_cmd, *tmp_cmd; 2879 unsigned long flags; 2880 bool tas; 2881 2882 list_for_each_entry_safe(se_cmd, tmp_cmd, 2883 &se_sess->sess_wait_list, se_cmd_list) { 2884 pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:" 2885 " %d\n", se_cmd, se_cmd->t_state, 2886 se_cmd->se_tfo->get_cmd_state(se_cmd)); 2887 2888 spin_lock_irqsave(&se_cmd->t_state_lock, flags); 2889 tas = (se_cmd->transport_state & CMD_T_TAS); 2890 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); 2891 2892 if (!target_put_sess_cmd(se_cmd)) { 2893 if (tas) 2894 target_put_sess_cmd(se_cmd); 2895 } 2896 2897 wait_for_completion(&se_cmd->cmd_wait_comp); 2898 pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d" 2899 " fabric state: %d\n", se_cmd, se_cmd->t_state, 2900 se_cmd->se_tfo->get_cmd_state(se_cmd)); 2901 2902 se_cmd->se_tfo->release_cmd(se_cmd); 2903 } 2904 2905 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2906 WARN_ON(!list_empty(&se_sess->sess_cmd_list)); 2907 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2908 2909 } 2910 EXPORT_SYMBOL(target_wait_for_sess_cmds); 2911 2912 static void target_lun_confirm(struct percpu_ref *ref) 2913 { 2914 struct se_lun *lun = container_of(ref, struct se_lun, lun_ref); 2915 2916 complete(&lun->lun_ref_comp); 2917 } 2918 2919 void transport_clear_lun_ref(struct se_lun *lun) 2920 { 2921 /* 2922 * Mark the percpu-ref as DEAD, switch to atomic_t mode, drop 2923 * the initial reference and schedule confirm kill to be 2924 * executed after one full RCU grace period has completed. 2925 */ 2926 percpu_ref_kill_and_confirm(&lun->lun_ref, target_lun_confirm); 2927 /* 2928 * The first completion waits for percpu_ref_switch_to_atomic_rcu() 2929 * to call target_lun_confirm after lun->lun_ref has been marked 2930 * as __PERCPU_REF_DEAD on all CPUs, and switches to atomic_t 2931 * mode so that percpu_ref_tryget_live() lookup of lun->lun_ref 2932 * fails for all new incoming I/O. 2933 */ 2934 wait_for_completion(&lun->lun_ref_comp); 2935 /* 2936 * The second completion waits for percpu_ref_put_many() to 2937 * invoke ->release() after lun->lun_ref has switched to 2938 * atomic_t mode, and lun->lun_ref.count has reached zero. 2939 * 2940 * At this point all target-core lun->lun_ref references have 2941 * been dropped via transport_lun_remove_cmd(), and it's safe 2942 * to proceed with the remaining LUN shutdown. 2943 */ 2944 wait_for_completion(&lun->lun_shutdown_comp); 2945 } 2946 2947 static bool 2948 __transport_wait_for_tasks(struct se_cmd *cmd, bool fabric_stop, 2949 bool *aborted, bool *tas, unsigned long *flags) 2950 __releases(&cmd->t_state_lock) 2951 __acquires(&cmd->t_state_lock) 2952 { 2953 2954 assert_spin_locked(&cmd->t_state_lock); 2955 WARN_ON_ONCE(!irqs_disabled()); 2956 2957 if (fabric_stop) 2958 cmd->transport_state |= CMD_T_FABRIC_STOP; 2959 2960 if (cmd->transport_state & CMD_T_ABORTED) 2961 *aborted = true; 2962 2963 if (cmd->transport_state & CMD_T_TAS) 2964 *tas = true; 2965 2966 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) && 2967 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2968 return false; 2969 2970 if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) && 2971 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2972 return false; 2973 2974 if (!(cmd->transport_state & CMD_T_ACTIVE)) 2975 return false; 2976 2977 if (fabric_stop && *aborted) 2978 return false; 2979 2980 cmd->transport_state |= CMD_T_STOP; 2981 2982 target_show_cmd("wait_for_tasks: Stopping ", cmd); 2983 2984 spin_unlock_irqrestore(&cmd->t_state_lock, *flags); 2985 2986 while (!wait_for_completion_timeout(&cmd->t_transport_stop_comp, 2987 180 * HZ)) 2988 target_show_cmd("wait for tasks: ", cmd); 2989 2990 spin_lock_irqsave(&cmd->t_state_lock, *flags); 2991 cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP); 2992 2993 pr_debug("wait_for_tasks: Stopped wait_for_completion(&cmd->" 2994 "t_transport_stop_comp) for ITT: 0x%08llx\n", cmd->tag); 2995 2996 return true; 2997 } 2998 2999 /** 3000 * transport_wait_for_tasks - set CMD_T_STOP and wait for t_transport_stop_comp 3001 * @cmd: command to wait on 3002 */ 3003 bool transport_wait_for_tasks(struct se_cmd *cmd) 3004 { 3005 unsigned long flags; 3006 bool ret, aborted = false, tas = false; 3007 3008 spin_lock_irqsave(&cmd->t_state_lock, flags); 3009 ret = __transport_wait_for_tasks(cmd, false, &aborted, &tas, &flags); 3010 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3011 3012 return ret; 3013 } 3014 EXPORT_SYMBOL(transport_wait_for_tasks); 3015 3016 struct sense_info { 3017 u8 key; 3018 u8 asc; 3019 u8 ascq; 3020 bool add_sector_info; 3021 }; 3022 3023 static const struct sense_info sense_info_table[] = { 3024 [TCM_NO_SENSE] = { 3025 .key = NOT_READY 3026 }, 3027 [TCM_NON_EXISTENT_LUN] = { 3028 .key = ILLEGAL_REQUEST, 3029 .asc = 0x25 /* LOGICAL UNIT NOT SUPPORTED */ 3030 }, 3031 [TCM_UNSUPPORTED_SCSI_OPCODE] = { 3032 .key = ILLEGAL_REQUEST, 3033 .asc = 0x20, /* INVALID COMMAND OPERATION CODE */ 3034 }, 3035 [TCM_SECTOR_COUNT_TOO_MANY] = { 3036 .key = ILLEGAL_REQUEST, 3037 .asc = 0x20, /* INVALID COMMAND OPERATION CODE */ 3038 }, 3039 [TCM_UNKNOWN_MODE_PAGE] = { 3040 .key = ILLEGAL_REQUEST, 3041 .asc = 0x24, /* INVALID FIELD IN CDB */ 3042 }, 3043 [TCM_CHECK_CONDITION_ABORT_CMD] = { 3044 .key = ABORTED_COMMAND, 3045 .asc = 0x29, /* BUS DEVICE RESET FUNCTION OCCURRED */ 3046 .ascq = 0x03, 3047 }, 3048 [TCM_INCORRECT_AMOUNT_OF_DATA] = { 3049 .key = ABORTED_COMMAND, 3050 .asc = 0x0c, /* WRITE ERROR */ 3051 .ascq = 0x0d, /* NOT ENOUGH UNSOLICITED DATA */ 3052 }, 3053 [TCM_INVALID_CDB_FIELD] = { 3054 .key = ILLEGAL_REQUEST, 3055 .asc = 0x24, /* INVALID FIELD IN CDB */ 3056 }, 3057 [TCM_INVALID_PARAMETER_LIST] = { 3058 .key = ILLEGAL_REQUEST, 3059 .asc = 0x26, /* INVALID FIELD IN PARAMETER LIST */ 3060 }, 3061 [TCM_TOO_MANY_TARGET_DESCS] = { 3062 .key = ILLEGAL_REQUEST, 3063 .asc = 0x26, 3064 .ascq = 0x06, /* TOO MANY TARGET DESCRIPTORS */ 3065 }, 3066 [TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE] = { 3067 .key = ILLEGAL_REQUEST, 3068 .asc = 0x26, 3069 .ascq = 0x07, /* UNSUPPORTED TARGET DESCRIPTOR TYPE CODE */ 3070 }, 3071 [TCM_TOO_MANY_SEGMENT_DESCS] = { 3072 .key = ILLEGAL_REQUEST, 3073 .asc = 0x26, 3074 .ascq = 0x08, /* TOO MANY SEGMENT DESCRIPTORS */ 3075 }, 3076 [TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE] = { 3077 .key = ILLEGAL_REQUEST, 3078 .asc = 0x26, 3079 .ascq = 0x09, /* UNSUPPORTED SEGMENT DESCRIPTOR TYPE CODE */ 3080 }, 3081 [TCM_PARAMETER_LIST_LENGTH_ERROR] = { 3082 .key = ILLEGAL_REQUEST, 3083 .asc = 0x1a, /* PARAMETER LIST LENGTH ERROR */ 3084 }, 3085 [TCM_UNEXPECTED_UNSOLICITED_DATA] = { 3086 .key = ILLEGAL_REQUEST, 3087 .asc = 0x0c, /* WRITE ERROR */ 3088 .ascq = 0x0c, /* UNEXPECTED_UNSOLICITED_DATA */ 3089 }, 3090 [TCM_SERVICE_CRC_ERROR] = { 3091 .key = ABORTED_COMMAND, 3092 .asc = 0x47, /* PROTOCOL SERVICE CRC ERROR */ 3093 .ascq = 0x05, /* N/A */ 3094 }, 3095 [TCM_SNACK_REJECTED] = { 3096 .key = ABORTED_COMMAND, 3097 .asc = 0x11, /* READ ERROR */ 3098 .ascq = 0x13, /* FAILED RETRANSMISSION REQUEST */ 3099 }, 3100 [TCM_WRITE_PROTECTED] = { 3101 .key = DATA_PROTECT, 3102 .asc = 0x27, /* WRITE PROTECTED */ 3103 }, 3104 [TCM_ADDRESS_OUT_OF_RANGE] = { 3105 .key = ILLEGAL_REQUEST, 3106 .asc = 0x21, /* LOGICAL BLOCK ADDRESS OUT OF RANGE */ 3107 }, 3108 [TCM_CHECK_CONDITION_UNIT_ATTENTION] = { 3109 .key = UNIT_ATTENTION, 3110 }, 3111 [TCM_CHECK_CONDITION_NOT_READY] = { 3112 .key = NOT_READY, 3113 }, 3114 [TCM_MISCOMPARE_VERIFY] = { 3115 .key = MISCOMPARE, 3116 .asc = 0x1d, /* MISCOMPARE DURING VERIFY OPERATION */ 3117 .ascq = 0x00, 3118 }, 3119 [TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED] = { 3120 .key = ABORTED_COMMAND, 3121 .asc = 0x10, 3122 .ascq = 0x01, /* LOGICAL BLOCK GUARD CHECK FAILED */ 3123 .add_sector_info = true, 3124 }, 3125 [TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED] = { 3126 .key = ABORTED_COMMAND, 3127 .asc = 0x10, 3128 .ascq = 0x02, /* LOGICAL BLOCK APPLICATION TAG CHECK FAILED */ 3129 .add_sector_info = true, 3130 }, 3131 [TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED] = { 3132 .key = ABORTED_COMMAND, 3133 .asc = 0x10, 3134 .ascq = 0x03, /* LOGICAL BLOCK REFERENCE TAG CHECK FAILED */ 3135 .add_sector_info = true, 3136 }, 3137 [TCM_COPY_TARGET_DEVICE_NOT_REACHABLE] = { 3138 .key = COPY_ABORTED, 3139 .asc = 0x0d, 3140 .ascq = 0x02, /* COPY TARGET DEVICE NOT REACHABLE */ 3141 3142 }, 3143 [TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE] = { 3144 /* 3145 * Returning ILLEGAL REQUEST would cause immediate IO errors on 3146 * Solaris initiators. Returning NOT READY instead means the 3147 * operations will be retried a finite number of times and we 3148 * can survive intermittent errors. 3149 */ 3150 .key = NOT_READY, 3151 .asc = 0x08, /* LOGICAL UNIT COMMUNICATION FAILURE */ 3152 }, 3153 [TCM_INSUFFICIENT_REGISTRATION_RESOURCES] = { 3154 /* 3155 * From spc4r22 section5.7.7,5.7.8 3156 * If a PERSISTENT RESERVE OUT command with a REGISTER service action 3157 * or a REGISTER AND IGNORE EXISTING KEY service action or 3158 * REGISTER AND MOVE service actionis attempted, 3159 * but there are insufficient device server resources to complete the 3160 * operation, then the command shall be terminated with CHECK CONDITION 3161 * status, with the sense key set to ILLEGAL REQUEST,and the additonal 3162 * sense code set to INSUFFICIENT REGISTRATION RESOURCES. 3163 */ 3164 .key = ILLEGAL_REQUEST, 3165 .asc = 0x55, 3166 .ascq = 0x04, /* INSUFFICIENT REGISTRATION RESOURCES */ 3167 }, 3168 }; 3169 3170 static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason) 3171 { 3172 const struct sense_info *si; 3173 u8 *buffer = cmd->sense_buffer; 3174 int r = (__force int)reason; 3175 u8 asc, ascq; 3176 bool desc_format = target_sense_desc_format(cmd->se_dev); 3177 3178 if (r < ARRAY_SIZE(sense_info_table) && sense_info_table[r].key) 3179 si = &sense_info_table[r]; 3180 else 3181 si = &sense_info_table[(__force int) 3182 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE]; 3183 3184 if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) { 3185 core_scsi3_ua_for_check_condition(cmd, &asc, &ascq); 3186 WARN_ON_ONCE(asc == 0); 3187 } else if (si->asc == 0) { 3188 WARN_ON_ONCE(cmd->scsi_asc == 0); 3189 asc = cmd->scsi_asc; 3190 ascq = cmd->scsi_ascq; 3191 } else { 3192 asc = si->asc; 3193 ascq = si->ascq; 3194 } 3195 3196 scsi_build_sense_buffer(desc_format, buffer, si->key, asc, ascq); 3197 if (si->add_sector_info) 3198 return scsi_set_sense_information(buffer, 3199 cmd->scsi_sense_length, 3200 cmd->bad_sector); 3201 3202 return 0; 3203 } 3204 3205 int 3206 transport_send_check_condition_and_sense(struct se_cmd *cmd, 3207 sense_reason_t reason, int from_transport) 3208 { 3209 unsigned long flags; 3210 3211 spin_lock_irqsave(&cmd->t_state_lock, flags); 3212 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) { 3213 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3214 return 0; 3215 } 3216 cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION; 3217 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3218 3219 if (!from_transport) { 3220 int rc; 3221 3222 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; 3223 cmd->scsi_status = SAM_STAT_CHECK_CONDITION; 3224 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 3225 rc = translate_sense_reason(cmd, reason); 3226 if (rc) 3227 return rc; 3228 } 3229 3230 trace_target_cmd_complete(cmd); 3231 return cmd->se_tfo->queue_status(cmd); 3232 } 3233 EXPORT_SYMBOL(transport_send_check_condition_and_sense); 3234 3235 static int __transport_check_aborted_status(struct se_cmd *cmd, int send_status) 3236 __releases(&cmd->t_state_lock) 3237 __acquires(&cmd->t_state_lock) 3238 { 3239 int ret; 3240 3241 assert_spin_locked(&cmd->t_state_lock); 3242 WARN_ON_ONCE(!irqs_disabled()); 3243 3244 if (!(cmd->transport_state & CMD_T_ABORTED)) 3245 return 0; 3246 /* 3247 * If cmd has been aborted but either no status is to be sent or it has 3248 * already been sent, just return 3249 */ 3250 if (!send_status || !(cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS)) { 3251 if (send_status) 3252 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS; 3253 return 1; 3254 } 3255 3256 pr_debug("Sending delayed SAM_STAT_TASK_ABORTED status for CDB:" 3257 " 0x%02x ITT: 0x%08llx\n", cmd->t_task_cdb[0], cmd->tag); 3258 3259 cmd->se_cmd_flags &= ~SCF_SEND_DELAYED_TAS; 3260 cmd->scsi_status = SAM_STAT_TASK_ABORTED; 3261 trace_target_cmd_complete(cmd); 3262 3263 spin_unlock_irq(&cmd->t_state_lock); 3264 ret = cmd->se_tfo->queue_status(cmd); 3265 if (ret) 3266 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 3267 spin_lock_irq(&cmd->t_state_lock); 3268 3269 return 1; 3270 } 3271 3272 int transport_check_aborted_status(struct se_cmd *cmd, int send_status) 3273 { 3274 int ret; 3275 3276 spin_lock_irq(&cmd->t_state_lock); 3277 ret = __transport_check_aborted_status(cmd, send_status); 3278 spin_unlock_irq(&cmd->t_state_lock); 3279 3280 return ret; 3281 } 3282 EXPORT_SYMBOL(transport_check_aborted_status); 3283 3284 void transport_send_task_abort(struct se_cmd *cmd) 3285 { 3286 unsigned long flags; 3287 int ret; 3288 3289 spin_lock_irqsave(&cmd->t_state_lock, flags); 3290 if (cmd->se_cmd_flags & (SCF_SENT_CHECK_CONDITION)) { 3291 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3292 return; 3293 } 3294 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3295 3296 /* 3297 * If there are still expected incoming fabric WRITEs, we wait 3298 * until until they have completed before sending a TASK_ABORTED 3299 * response. This response with TASK_ABORTED status will be 3300 * queued back to fabric module by transport_check_aborted_status(). 3301 */ 3302 if (cmd->data_direction == DMA_TO_DEVICE) { 3303 if (cmd->se_tfo->write_pending_status(cmd) != 0) { 3304 spin_lock_irqsave(&cmd->t_state_lock, flags); 3305 if (cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS) { 3306 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3307 goto send_abort; 3308 } 3309 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS; 3310 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3311 return; 3312 } 3313 } 3314 send_abort: 3315 cmd->scsi_status = SAM_STAT_TASK_ABORTED; 3316 3317 transport_lun_remove_cmd(cmd); 3318 3319 pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n", 3320 cmd->t_task_cdb[0], cmd->tag); 3321 3322 trace_target_cmd_complete(cmd); 3323 ret = cmd->se_tfo->queue_status(cmd); 3324 if (ret) 3325 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 3326 } 3327 3328 static void target_tmr_work(struct work_struct *work) 3329 { 3330 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 3331 struct se_device *dev = cmd->se_dev; 3332 struct se_tmr_req *tmr = cmd->se_tmr_req; 3333 unsigned long flags; 3334 int ret; 3335 3336 spin_lock_irqsave(&cmd->t_state_lock, flags); 3337 if (cmd->transport_state & CMD_T_ABORTED) { 3338 tmr->response = TMR_FUNCTION_REJECTED; 3339 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3340 goto check_stop; 3341 } 3342 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3343 3344 switch (tmr->function) { 3345 case TMR_ABORT_TASK: 3346 core_tmr_abort_task(dev, tmr, cmd->se_sess); 3347 break; 3348 case TMR_ABORT_TASK_SET: 3349 case TMR_CLEAR_ACA: 3350 case TMR_CLEAR_TASK_SET: 3351 tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED; 3352 break; 3353 case TMR_LUN_RESET: 3354 ret = core_tmr_lun_reset(dev, tmr, NULL, NULL); 3355 tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE : 3356 TMR_FUNCTION_REJECTED; 3357 if (tmr->response == TMR_FUNCTION_COMPLETE) { 3358 target_ua_allocate_lun(cmd->se_sess->se_node_acl, 3359 cmd->orig_fe_lun, 0x29, 3360 ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED); 3361 } 3362 break; 3363 case TMR_TARGET_WARM_RESET: 3364 tmr->response = TMR_FUNCTION_REJECTED; 3365 break; 3366 case TMR_TARGET_COLD_RESET: 3367 tmr->response = TMR_FUNCTION_REJECTED; 3368 break; 3369 default: 3370 pr_err("Uknown TMR function: 0x%02x.\n", 3371 tmr->function); 3372 tmr->response = TMR_FUNCTION_REJECTED; 3373 break; 3374 } 3375 3376 spin_lock_irqsave(&cmd->t_state_lock, flags); 3377 if (cmd->transport_state & CMD_T_ABORTED) { 3378 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3379 goto check_stop; 3380 } 3381 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3382 3383 cmd->se_tfo->queue_tm_rsp(cmd); 3384 3385 check_stop: 3386 transport_lun_remove_cmd(cmd); 3387 transport_cmd_check_stop_to_fabric(cmd); 3388 } 3389 3390 int transport_generic_handle_tmr( 3391 struct se_cmd *cmd) 3392 { 3393 unsigned long flags; 3394 bool aborted = false; 3395 3396 spin_lock_irqsave(&cmd->t_state_lock, flags); 3397 if (cmd->transport_state & CMD_T_ABORTED) { 3398 aborted = true; 3399 } else { 3400 cmd->t_state = TRANSPORT_ISTATE_PROCESSING; 3401 cmd->transport_state |= CMD_T_ACTIVE; 3402 } 3403 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3404 3405 if (aborted) { 3406 pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d" 3407 "ref_tag: %llu tag: %llu\n", cmd->se_tmr_req->function, 3408 cmd->se_tmr_req->ref_task_tag, cmd->tag); 3409 transport_lun_remove_cmd(cmd); 3410 transport_cmd_check_stop_to_fabric(cmd); 3411 return 0; 3412 } 3413 3414 INIT_WORK(&cmd->work, target_tmr_work); 3415 queue_work(cmd->se_dev->tmr_wq, &cmd->work); 3416 return 0; 3417 } 3418 EXPORT_SYMBOL(transport_generic_handle_tmr); 3419 3420 bool 3421 target_check_wce(struct se_device *dev) 3422 { 3423 bool wce = false; 3424 3425 if (dev->transport->get_write_cache) 3426 wce = dev->transport->get_write_cache(dev); 3427 else if (dev->dev_attrib.emulate_write_cache > 0) 3428 wce = true; 3429 3430 return wce; 3431 } 3432 3433 bool 3434 target_check_fua(struct se_device *dev) 3435 { 3436 return target_check_wce(dev) && dev->dev_attrib.emulate_fua_write > 0; 3437 } 3438