1 /* 2 * Copyright (C) 1999 Eric Youngdale 3 * Copyright (C) 2014 Christoph Hellwig 4 * 5 * SCSI queueing library. 6 * Initial versions: Eric Youngdale (eric@andante.org). 7 * Based upon conversations with large numbers 8 * of people at Linux Expo. 9 */ 10 11 #include <linux/bio.h> 12 #include <linux/bitops.h> 13 #include <linux/blkdev.h> 14 #include <linux/completion.h> 15 #include <linux/kernel.h> 16 #include <linux/export.h> 17 #include <linux/init.h> 18 #include <linux/pci.h> 19 #include <linux/delay.h> 20 #include <linux/hardirq.h> 21 #include <linux/scatterlist.h> 22 #include <linux/blk-mq.h> 23 #include <linux/ratelimit.h> 24 #include <asm/unaligned.h> 25 26 #include <scsi/scsi.h> 27 #include <scsi/scsi_cmnd.h> 28 #include <scsi/scsi_dbg.h> 29 #include <scsi/scsi_device.h> 30 #include <scsi/scsi_driver.h> 31 #include <scsi/scsi_eh.h> 32 #include <scsi/scsi_host.h> 33 #include <scsi/scsi_dh.h> 34 35 #include <trace/events/scsi.h> 36 37 #include "scsi_priv.h" 38 #include "scsi_logging.h" 39 40 41 struct kmem_cache *scsi_sdb_cache; 42 43 /* 44 * When to reinvoke queueing after a resource shortage. It's 3 msecs to 45 * not change behaviour from the previous unplug mechanism, experimentation 46 * may prove this needs changing. 47 */ 48 #define SCSI_QUEUE_DELAY 3 49 50 static void 51 scsi_set_blocked(struct scsi_cmnd *cmd, int reason) 52 { 53 struct Scsi_Host *host = cmd->device->host; 54 struct scsi_device *device = cmd->device; 55 struct scsi_target *starget = scsi_target(device); 56 57 /* 58 * Set the appropriate busy bit for the device/host. 59 * 60 * If the host/device isn't busy, assume that something actually 61 * completed, and that we should be able to queue a command now. 62 * 63 * Note that the prior mid-layer assumption that any host could 64 * always queue at least one command is now broken. The mid-layer 65 * will implement a user specifiable stall (see 66 * scsi_host.max_host_blocked and scsi_device.max_device_blocked) 67 * if a command is requeued with no other commands outstanding 68 * either for the device or for the host. 69 */ 70 switch (reason) { 71 case SCSI_MLQUEUE_HOST_BUSY: 72 atomic_set(&host->host_blocked, host->max_host_blocked); 73 break; 74 case SCSI_MLQUEUE_DEVICE_BUSY: 75 case SCSI_MLQUEUE_EH_RETRY: 76 atomic_set(&device->device_blocked, 77 device->max_device_blocked); 78 break; 79 case SCSI_MLQUEUE_TARGET_BUSY: 80 atomic_set(&starget->target_blocked, 81 starget->max_target_blocked); 82 break; 83 } 84 } 85 86 static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd) 87 { 88 struct scsi_device *sdev = cmd->device; 89 struct request_queue *q = cmd->request->q; 90 91 blk_mq_requeue_request(cmd->request); 92 blk_mq_kick_requeue_list(q); 93 put_device(&sdev->sdev_gendev); 94 } 95 96 /** 97 * __scsi_queue_insert - private queue insertion 98 * @cmd: The SCSI command being requeued 99 * @reason: The reason for the requeue 100 * @unbusy: Whether the queue should be unbusied 101 * 102 * This is a private queue insertion. The public interface 103 * scsi_queue_insert() always assumes the queue should be unbusied 104 * because it's always called before the completion. This function is 105 * for a requeue after completion, which should only occur in this 106 * file. 107 */ 108 static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy) 109 { 110 struct scsi_device *device = cmd->device; 111 struct request_queue *q = device->request_queue; 112 unsigned long flags; 113 114 SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd, 115 "Inserting command %p into mlqueue\n", cmd)); 116 117 scsi_set_blocked(cmd, reason); 118 119 /* 120 * Decrement the counters, since these commands are no longer 121 * active on the host/device. 122 */ 123 if (unbusy) 124 scsi_device_unbusy(device); 125 126 /* 127 * Requeue this command. It will go before all other commands 128 * that are already in the queue. Schedule requeue work under 129 * lock such that the kblockd_schedule_work() call happens 130 * before blk_cleanup_queue() finishes. 131 */ 132 cmd->result = 0; 133 if (q->mq_ops) { 134 scsi_mq_requeue_cmd(cmd); 135 return; 136 } 137 spin_lock_irqsave(q->queue_lock, flags); 138 blk_requeue_request(q, cmd->request); 139 kblockd_schedule_work(&device->requeue_work); 140 spin_unlock_irqrestore(q->queue_lock, flags); 141 } 142 143 /* 144 * Function: scsi_queue_insert() 145 * 146 * Purpose: Insert a command in the midlevel queue. 147 * 148 * Arguments: cmd - command that we are adding to queue. 149 * reason - why we are inserting command to queue. 150 * 151 * Lock status: Assumed that lock is not held upon entry. 152 * 153 * Returns: Nothing. 154 * 155 * Notes: We do this for one of two cases. Either the host is busy 156 * and it cannot accept any more commands for the time being, 157 * or the device returned QUEUE_FULL and can accept no more 158 * commands. 159 * Notes: This could be called either from an interrupt context or a 160 * normal process context. 161 */ 162 void scsi_queue_insert(struct scsi_cmnd *cmd, int reason) 163 { 164 __scsi_queue_insert(cmd, reason, 1); 165 } 166 /** 167 * scsi_execute - insert request and wait for the result 168 * @sdev: scsi device 169 * @cmd: scsi command 170 * @data_direction: data direction 171 * @buffer: data buffer 172 * @bufflen: len of buffer 173 * @sense: optional sense buffer 174 * @timeout: request timeout in seconds 175 * @retries: number of times to retry request 176 * @flags: or into request flags; 177 * @resid: optional residual length 178 * 179 * returns the req->errors value which is the scsi_cmnd result 180 * field. 181 */ 182 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd, 183 int data_direction, void *buffer, unsigned bufflen, 184 unsigned char *sense, int timeout, int retries, u64 flags, 185 int *resid) 186 { 187 struct request *req; 188 int write = (data_direction == DMA_TO_DEVICE); 189 int ret = DRIVER_ERROR << 24; 190 191 req = blk_get_request(sdev->request_queue, write, __GFP_RECLAIM); 192 if (IS_ERR(req)) 193 return ret; 194 blk_rq_set_block_pc(req); 195 196 if (bufflen && blk_rq_map_kern(sdev->request_queue, req, 197 buffer, bufflen, __GFP_RECLAIM)) 198 goto out; 199 200 req->cmd_len = COMMAND_SIZE(cmd[0]); 201 memcpy(req->cmd, cmd, req->cmd_len); 202 req->sense = sense; 203 req->sense_len = 0; 204 req->retries = retries; 205 req->timeout = timeout; 206 req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT; 207 208 /* 209 * head injection *required* here otherwise quiesce won't work 210 */ 211 blk_execute_rq(req->q, NULL, req, 1); 212 213 /* 214 * Some devices (USB mass-storage in particular) may transfer 215 * garbage data together with a residue indicating that the data 216 * is invalid. Prevent the garbage from being misinterpreted 217 * and prevent security leaks by zeroing out the excess data. 218 */ 219 if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen)) 220 memset(buffer + (bufflen - req->resid_len), 0, req->resid_len); 221 222 if (resid) 223 *resid = req->resid_len; 224 ret = req->errors; 225 out: 226 blk_put_request(req); 227 228 return ret; 229 } 230 EXPORT_SYMBOL(scsi_execute); 231 232 int scsi_execute_req_flags(struct scsi_device *sdev, const unsigned char *cmd, 233 int data_direction, void *buffer, unsigned bufflen, 234 struct scsi_sense_hdr *sshdr, int timeout, int retries, 235 int *resid, u64 flags) 236 { 237 char *sense = NULL; 238 int result; 239 240 if (sshdr) { 241 sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO); 242 if (!sense) 243 return DRIVER_ERROR << 24; 244 } 245 result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen, 246 sense, timeout, retries, flags, resid); 247 if (sshdr) 248 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr); 249 250 kfree(sense); 251 return result; 252 } 253 EXPORT_SYMBOL(scsi_execute_req_flags); 254 255 /* 256 * Function: scsi_init_cmd_errh() 257 * 258 * Purpose: Initialize cmd fields related to error handling. 259 * 260 * Arguments: cmd - command that is ready to be queued. 261 * 262 * Notes: This function has the job of initializing a number of 263 * fields related to error handling. Typically this will 264 * be called once for each command, as required. 265 */ 266 static void scsi_init_cmd_errh(struct scsi_cmnd *cmd) 267 { 268 cmd->serial_number = 0; 269 scsi_set_resid(cmd, 0); 270 memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE); 271 if (cmd->cmd_len == 0) 272 cmd->cmd_len = scsi_command_size(cmd->cmnd); 273 } 274 275 void scsi_device_unbusy(struct scsi_device *sdev) 276 { 277 struct Scsi_Host *shost = sdev->host; 278 struct scsi_target *starget = scsi_target(sdev); 279 unsigned long flags; 280 281 atomic_dec(&shost->host_busy); 282 if (starget->can_queue > 0) 283 atomic_dec(&starget->target_busy); 284 285 if (unlikely(scsi_host_in_recovery(shost) && 286 (shost->host_failed || shost->host_eh_scheduled))) { 287 spin_lock_irqsave(shost->host_lock, flags); 288 scsi_eh_wakeup(shost); 289 spin_unlock_irqrestore(shost->host_lock, flags); 290 } 291 292 atomic_dec(&sdev->device_busy); 293 } 294 295 static void scsi_kick_queue(struct request_queue *q) 296 { 297 if (q->mq_ops) 298 blk_mq_start_hw_queues(q); 299 else 300 blk_run_queue(q); 301 } 302 303 /* 304 * Called for single_lun devices on IO completion. Clear starget_sdev_user, 305 * and call blk_run_queue for all the scsi_devices on the target - 306 * including current_sdev first. 307 * 308 * Called with *no* scsi locks held. 309 */ 310 static void scsi_single_lun_run(struct scsi_device *current_sdev) 311 { 312 struct Scsi_Host *shost = current_sdev->host; 313 struct scsi_device *sdev, *tmp; 314 struct scsi_target *starget = scsi_target(current_sdev); 315 unsigned long flags; 316 317 spin_lock_irqsave(shost->host_lock, flags); 318 starget->starget_sdev_user = NULL; 319 spin_unlock_irqrestore(shost->host_lock, flags); 320 321 /* 322 * Call blk_run_queue for all LUNs on the target, starting with 323 * current_sdev. We race with others (to set starget_sdev_user), 324 * but in most cases, we will be first. Ideally, each LU on the 325 * target would get some limited time or requests on the target. 326 */ 327 scsi_kick_queue(current_sdev->request_queue); 328 329 spin_lock_irqsave(shost->host_lock, flags); 330 if (starget->starget_sdev_user) 331 goto out; 332 list_for_each_entry_safe(sdev, tmp, &starget->devices, 333 same_target_siblings) { 334 if (sdev == current_sdev) 335 continue; 336 if (scsi_device_get(sdev)) 337 continue; 338 339 spin_unlock_irqrestore(shost->host_lock, flags); 340 scsi_kick_queue(sdev->request_queue); 341 spin_lock_irqsave(shost->host_lock, flags); 342 343 scsi_device_put(sdev); 344 } 345 out: 346 spin_unlock_irqrestore(shost->host_lock, flags); 347 } 348 349 static inline bool scsi_device_is_busy(struct scsi_device *sdev) 350 { 351 if (atomic_read(&sdev->device_busy) >= sdev->queue_depth) 352 return true; 353 if (atomic_read(&sdev->device_blocked) > 0) 354 return true; 355 return false; 356 } 357 358 static inline bool scsi_target_is_busy(struct scsi_target *starget) 359 { 360 if (starget->can_queue > 0) { 361 if (atomic_read(&starget->target_busy) >= starget->can_queue) 362 return true; 363 if (atomic_read(&starget->target_blocked) > 0) 364 return true; 365 } 366 return false; 367 } 368 369 static inline bool scsi_host_is_busy(struct Scsi_Host *shost) 370 { 371 if (shost->can_queue > 0 && 372 atomic_read(&shost->host_busy) >= shost->can_queue) 373 return true; 374 if (atomic_read(&shost->host_blocked) > 0) 375 return true; 376 if (shost->host_self_blocked) 377 return true; 378 return false; 379 } 380 381 static void scsi_starved_list_run(struct Scsi_Host *shost) 382 { 383 LIST_HEAD(starved_list); 384 struct scsi_device *sdev; 385 unsigned long flags; 386 387 spin_lock_irqsave(shost->host_lock, flags); 388 list_splice_init(&shost->starved_list, &starved_list); 389 390 while (!list_empty(&starved_list)) { 391 struct request_queue *slq; 392 393 /* 394 * As long as shost is accepting commands and we have 395 * starved queues, call blk_run_queue. scsi_request_fn 396 * drops the queue_lock and can add us back to the 397 * starved_list. 398 * 399 * host_lock protects the starved_list and starved_entry. 400 * scsi_request_fn must get the host_lock before checking 401 * or modifying starved_list or starved_entry. 402 */ 403 if (scsi_host_is_busy(shost)) 404 break; 405 406 sdev = list_entry(starved_list.next, 407 struct scsi_device, starved_entry); 408 list_del_init(&sdev->starved_entry); 409 if (scsi_target_is_busy(scsi_target(sdev))) { 410 list_move_tail(&sdev->starved_entry, 411 &shost->starved_list); 412 continue; 413 } 414 415 /* 416 * Once we drop the host lock, a racing scsi_remove_device() 417 * call may remove the sdev from the starved list and destroy 418 * it and the queue. Mitigate by taking a reference to the 419 * queue and never touching the sdev again after we drop the 420 * host lock. Note: if __scsi_remove_device() invokes 421 * blk_cleanup_queue() before the queue is run from this 422 * function then blk_run_queue() will return immediately since 423 * blk_cleanup_queue() marks the queue with QUEUE_FLAG_DYING. 424 */ 425 slq = sdev->request_queue; 426 if (!blk_get_queue(slq)) 427 continue; 428 spin_unlock_irqrestore(shost->host_lock, flags); 429 430 scsi_kick_queue(slq); 431 blk_put_queue(slq); 432 433 spin_lock_irqsave(shost->host_lock, flags); 434 } 435 /* put any unprocessed entries back */ 436 list_splice(&starved_list, &shost->starved_list); 437 spin_unlock_irqrestore(shost->host_lock, flags); 438 } 439 440 /* 441 * Function: scsi_run_queue() 442 * 443 * Purpose: Select a proper request queue to serve next 444 * 445 * Arguments: q - last request's queue 446 * 447 * Returns: Nothing 448 * 449 * Notes: The previous command was completely finished, start 450 * a new one if possible. 451 */ 452 static void scsi_run_queue(struct request_queue *q) 453 { 454 struct scsi_device *sdev = q->queuedata; 455 456 if (scsi_target(sdev)->single_lun) 457 scsi_single_lun_run(sdev); 458 if (!list_empty(&sdev->host->starved_list)) 459 scsi_starved_list_run(sdev->host); 460 461 if (q->mq_ops) 462 blk_mq_start_stopped_hw_queues(q, false); 463 else 464 blk_run_queue(q); 465 } 466 467 void scsi_requeue_run_queue(struct work_struct *work) 468 { 469 struct scsi_device *sdev; 470 struct request_queue *q; 471 472 sdev = container_of(work, struct scsi_device, requeue_work); 473 q = sdev->request_queue; 474 scsi_run_queue(q); 475 } 476 477 /* 478 * Function: scsi_requeue_command() 479 * 480 * Purpose: Handle post-processing of completed commands. 481 * 482 * Arguments: q - queue to operate on 483 * cmd - command that may need to be requeued. 484 * 485 * Returns: Nothing 486 * 487 * Notes: After command completion, there may be blocks left 488 * over which weren't finished by the previous command 489 * this can be for a number of reasons - the main one is 490 * I/O errors in the middle of the request, in which case 491 * we need to request the blocks that come after the bad 492 * sector. 493 * Notes: Upon return, cmd is a stale pointer. 494 */ 495 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd) 496 { 497 struct scsi_device *sdev = cmd->device; 498 struct request *req = cmd->request; 499 unsigned long flags; 500 501 spin_lock_irqsave(q->queue_lock, flags); 502 blk_unprep_request(req); 503 req->special = NULL; 504 scsi_put_command(cmd); 505 blk_requeue_request(q, req); 506 spin_unlock_irqrestore(q->queue_lock, flags); 507 508 scsi_run_queue(q); 509 510 put_device(&sdev->sdev_gendev); 511 } 512 513 void scsi_run_host_queues(struct Scsi_Host *shost) 514 { 515 struct scsi_device *sdev; 516 517 shost_for_each_device(sdev, shost) 518 scsi_run_queue(sdev->request_queue); 519 } 520 521 static void scsi_uninit_cmd(struct scsi_cmnd *cmd) 522 { 523 if (cmd->request->cmd_type == REQ_TYPE_FS) { 524 struct scsi_driver *drv = scsi_cmd_to_driver(cmd); 525 526 if (drv->uninit_command) 527 drv->uninit_command(cmd); 528 } 529 } 530 531 static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd) 532 { 533 struct scsi_data_buffer *sdb; 534 535 if (cmd->sdb.table.nents) 536 sg_free_table_chained(&cmd->sdb.table, true); 537 if (cmd->request->next_rq) { 538 sdb = cmd->request->next_rq->special; 539 if (sdb) 540 sg_free_table_chained(&sdb->table, true); 541 } 542 if (scsi_prot_sg_count(cmd)) 543 sg_free_table_chained(&cmd->prot_sdb->table, true); 544 } 545 546 static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd) 547 { 548 struct scsi_device *sdev = cmd->device; 549 struct Scsi_Host *shost = sdev->host; 550 unsigned long flags; 551 552 scsi_mq_free_sgtables(cmd); 553 scsi_uninit_cmd(cmd); 554 555 if (shost->use_cmd_list) { 556 BUG_ON(list_empty(&cmd->list)); 557 spin_lock_irqsave(&sdev->list_lock, flags); 558 list_del_init(&cmd->list); 559 spin_unlock_irqrestore(&sdev->list_lock, flags); 560 } 561 } 562 563 /* 564 * Function: scsi_release_buffers() 565 * 566 * Purpose: Free resources allocate for a scsi_command. 567 * 568 * Arguments: cmd - command that we are bailing. 569 * 570 * Lock status: Assumed that no lock is held upon entry. 571 * 572 * Returns: Nothing 573 * 574 * Notes: In the event that an upper level driver rejects a 575 * command, we must release resources allocated during 576 * the __init_io() function. Primarily this would involve 577 * the scatter-gather table. 578 */ 579 static void scsi_release_buffers(struct scsi_cmnd *cmd) 580 { 581 if (cmd->sdb.table.nents) 582 sg_free_table_chained(&cmd->sdb.table, false); 583 584 memset(&cmd->sdb, 0, sizeof(cmd->sdb)); 585 586 if (scsi_prot_sg_count(cmd)) 587 sg_free_table_chained(&cmd->prot_sdb->table, false); 588 } 589 590 static void scsi_release_bidi_buffers(struct scsi_cmnd *cmd) 591 { 592 struct scsi_data_buffer *bidi_sdb = cmd->request->next_rq->special; 593 594 sg_free_table_chained(&bidi_sdb->table, false); 595 kmem_cache_free(scsi_sdb_cache, bidi_sdb); 596 cmd->request->next_rq->special = NULL; 597 } 598 599 static bool scsi_end_request(struct request *req, int error, 600 unsigned int bytes, unsigned int bidi_bytes) 601 { 602 struct scsi_cmnd *cmd = req->special; 603 struct scsi_device *sdev = cmd->device; 604 struct request_queue *q = sdev->request_queue; 605 606 if (blk_update_request(req, error, bytes)) 607 return true; 608 609 /* Bidi request must be completed as a whole */ 610 if (unlikely(bidi_bytes) && 611 blk_update_request(req->next_rq, error, bidi_bytes)) 612 return true; 613 614 if (blk_queue_add_random(q)) 615 add_disk_randomness(req->rq_disk); 616 617 if (req->mq_ctx) { 618 /* 619 * In the MQ case the command gets freed by __blk_mq_end_request, 620 * so we have to do all cleanup that depends on it earlier. 621 * 622 * We also can't kick the queues from irq context, so we 623 * will have to defer it to a workqueue. 624 */ 625 scsi_mq_uninit_cmd(cmd); 626 627 __blk_mq_end_request(req, error); 628 629 if (scsi_target(sdev)->single_lun || 630 !list_empty(&sdev->host->starved_list)) 631 kblockd_schedule_work(&sdev->requeue_work); 632 else 633 blk_mq_start_stopped_hw_queues(q, true); 634 } else { 635 unsigned long flags; 636 637 if (bidi_bytes) 638 scsi_release_bidi_buffers(cmd); 639 640 spin_lock_irqsave(q->queue_lock, flags); 641 blk_finish_request(req, error); 642 spin_unlock_irqrestore(q->queue_lock, flags); 643 644 scsi_release_buffers(cmd); 645 646 scsi_put_command(cmd); 647 scsi_run_queue(q); 648 } 649 650 put_device(&sdev->sdev_gendev); 651 return false; 652 } 653 654 /** 655 * __scsi_error_from_host_byte - translate SCSI error code into errno 656 * @cmd: SCSI command (unused) 657 * @result: scsi error code 658 * 659 * Translate SCSI error code into standard UNIX errno. 660 * Return values: 661 * -ENOLINK temporary transport failure 662 * -EREMOTEIO permanent target failure, do not retry 663 * -EBADE permanent nexus failure, retry on other path 664 * -ENOSPC No write space available 665 * -ENODATA Medium error 666 * -EIO unspecified I/O error 667 */ 668 static int __scsi_error_from_host_byte(struct scsi_cmnd *cmd, int result) 669 { 670 int error = 0; 671 672 switch(host_byte(result)) { 673 case DID_TRANSPORT_FAILFAST: 674 error = -ENOLINK; 675 break; 676 case DID_TARGET_FAILURE: 677 set_host_byte(cmd, DID_OK); 678 error = -EREMOTEIO; 679 break; 680 case DID_NEXUS_FAILURE: 681 set_host_byte(cmd, DID_OK); 682 error = -EBADE; 683 break; 684 case DID_ALLOC_FAILURE: 685 set_host_byte(cmd, DID_OK); 686 error = -ENOSPC; 687 break; 688 case DID_MEDIUM_ERROR: 689 set_host_byte(cmd, DID_OK); 690 error = -ENODATA; 691 break; 692 default: 693 error = -EIO; 694 break; 695 } 696 697 return error; 698 } 699 700 /* 701 * Function: scsi_io_completion() 702 * 703 * Purpose: Completion processing for block device I/O requests. 704 * 705 * Arguments: cmd - command that is finished. 706 * 707 * Lock status: Assumed that no lock is held upon entry. 708 * 709 * Returns: Nothing 710 * 711 * Notes: We will finish off the specified number of sectors. If we 712 * are done, the command block will be released and the queue 713 * function will be goosed. If we are not done then we have to 714 * figure out what to do next: 715 * 716 * a) We can call scsi_requeue_command(). The request 717 * will be unprepared and put back on the queue. Then 718 * a new command will be created for it. This should 719 * be used if we made forward progress, or if we want 720 * to switch from READ(10) to READ(6) for example. 721 * 722 * b) We can call __scsi_queue_insert(). The request will 723 * be put back on the queue and retried using the same 724 * command as before, possibly after a delay. 725 * 726 * c) We can call scsi_end_request() with -EIO to fail 727 * the remainder of the request. 728 */ 729 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes) 730 { 731 int result = cmd->result; 732 struct request_queue *q = cmd->device->request_queue; 733 struct request *req = cmd->request; 734 int error = 0; 735 struct scsi_sense_hdr sshdr; 736 bool sense_valid = false; 737 int sense_deferred = 0, level = 0; 738 enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY, 739 ACTION_DELAYED_RETRY} action; 740 unsigned long wait_for = (cmd->allowed + 1) * req->timeout; 741 742 if (result) { 743 sense_valid = scsi_command_normalize_sense(cmd, &sshdr); 744 if (sense_valid) 745 sense_deferred = scsi_sense_is_deferred(&sshdr); 746 } 747 748 if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */ 749 if (result) { 750 if (sense_valid && req->sense) { 751 /* 752 * SG_IO wants current and deferred errors 753 */ 754 int len = 8 + cmd->sense_buffer[7]; 755 756 if (len > SCSI_SENSE_BUFFERSIZE) 757 len = SCSI_SENSE_BUFFERSIZE; 758 memcpy(req->sense, cmd->sense_buffer, len); 759 req->sense_len = len; 760 } 761 if (!sense_deferred) 762 error = __scsi_error_from_host_byte(cmd, result); 763 } 764 /* 765 * __scsi_error_from_host_byte may have reset the host_byte 766 */ 767 req->errors = cmd->result; 768 769 req->resid_len = scsi_get_resid(cmd); 770 771 if (scsi_bidi_cmnd(cmd)) { 772 /* 773 * Bidi commands Must be complete as a whole, 774 * both sides at once. 775 */ 776 req->next_rq->resid_len = scsi_in(cmd)->resid; 777 if (scsi_end_request(req, 0, blk_rq_bytes(req), 778 blk_rq_bytes(req->next_rq))) 779 BUG(); 780 return; 781 } 782 } else if (blk_rq_bytes(req) == 0 && result && !sense_deferred) { 783 /* 784 * Certain non BLOCK_PC requests are commands that don't 785 * actually transfer anything (FLUSH), so cannot use 786 * good_bytes != blk_rq_bytes(req) as the signal for an error. 787 * This sets the error explicitly for the problem case. 788 */ 789 error = __scsi_error_from_host_byte(cmd, result); 790 } 791 792 /* no bidi support for !REQ_TYPE_BLOCK_PC yet */ 793 BUG_ON(blk_bidi_rq(req)); 794 795 /* 796 * Next deal with any sectors which we were able to correctly 797 * handle. 798 */ 799 SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, cmd, 800 "%u sectors total, %d bytes done.\n", 801 blk_rq_sectors(req), good_bytes)); 802 803 /* 804 * Recovered errors need reporting, but they're always treated 805 * as success, so fiddle the result code here. For BLOCK_PC 806 * we already took a copy of the original into rq->errors which 807 * is what gets returned to the user 808 */ 809 if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) { 810 /* if ATA PASS-THROUGH INFORMATION AVAILABLE skip 811 * print since caller wants ATA registers. Only occurs on 812 * SCSI ATA PASS_THROUGH commands when CK_COND=1 813 */ 814 if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d)) 815 ; 816 else if (!(req->cmd_flags & REQ_QUIET)) 817 scsi_print_sense(cmd); 818 result = 0; 819 /* BLOCK_PC may have set error */ 820 error = 0; 821 } 822 823 /* 824 * special case: failed zero length commands always need to 825 * drop down into the retry code. Otherwise, if we finished 826 * all bytes in the request we are done now. 827 */ 828 if (!(blk_rq_bytes(req) == 0 && error) && 829 !scsi_end_request(req, error, good_bytes, 0)) 830 return; 831 832 /* 833 * Kill remainder if no retrys. 834 */ 835 if (error && scsi_noretry_cmd(cmd)) { 836 if (scsi_end_request(req, error, blk_rq_bytes(req), 0)) 837 BUG(); 838 return; 839 } 840 841 /* 842 * If there had been no error, but we have leftover bytes in the 843 * requeues just queue the command up again. 844 */ 845 if (result == 0) 846 goto requeue; 847 848 error = __scsi_error_from_host_byte(cmd, result); 849 850 if (host_byte(result) == DID_RESET) { 851 /* Third party bus reset or reset for error recovery 852 * reasons. Just retry the command and see what 853 * happens. 854 */ 855 action = ACTION_RETRY; 856 } else if (sense_valid && !sense_deferred) { 857 switch (sshdr.sense_key) { 858 case UNIT_ATTENTION: 859 if (cmd->device->removable) { 860 /* Detected disc change. Set a bit 861 * and quietly refuse further access. 862 */ 863 cmd->device->changed = 1; 864 action = ACTION_FAIL; 865 } else { 866 /* Must have been a power glitch, or a 867 * bus reset. Could not have been a 868 * media change, so we just retry the 869 * command and see what happens. 870 */ 871 action = ACTION_RETRY; 872 } 873 break; 874 case ILLEGAL_REQUEST: 875 /* If we had an ILLEGAL REQUEST returned, then 876 * we may have performed an unsupported 877 * command. The only thing this should be 878 * would be a ten byte read where only a six 879 * byte read was supported. Also, on a system 880 * where READ CAPACITY failed, we may have 881 * read past the end of the disk. 882 */ 883 if ((cmd->device->use_10_for_rw && 884 sshdr.asc == 0x20 && sshdr.ascq == 0x00) && 885 (cmd->cmnd[0] == READ_10 || 886 cmd->cmnd[0] == WRITE_10)) { 887 /* This will issue a new 6-byte command. */ 888 cmd->device->use_10_for_rw = 0; 889 action = ACTION_REPREP; 890 } else if (sshdr.asc == 0x10) /* DIX */ { 891 action = ACTION_FAIL; 892 error = -EILSEQ; 893 /* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */ 894 } else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) { 895 action = ACTION_FAIL; 896 error = -EREMOTEIO; 897 } else 898 action = ACTION_FAIL; 899 break; 900 case ABORTED_COMMAND: 901 action = ACTION_FAIL; 902 if (sshdr.asc == 0x10) /* DIF */ 903 error = -EILSEQ; 904 break; 905 case NOT_READY: 906 /* If the device is in the process of becoming 907 * ready, or has a temporary blockage, retry. 908 */ 909 if (sshdr.asc == 0x04) { 910 switch (sshdr.ascq) { 911 case 0x01: /* becoming ready */ 912 case 0x04: /* format in progress */ 913 case 0x05: /* rebuild in progress */ 914 case 0x06: /* recalculation in progress */ 915 case 0x07: /* operation in progress */ 916 case 0x08: /* Long write in progress */ 917 case 0x09: /* self test in progress */ 918 case 0x14: /* space allocation in progress */ 919 action = ACTION_DELAYED_RETRY; 920 break; 921 default: 922 action = ACTION_FAIL; 923 break; 924 } 925 } else 926 action = ACTION_FAIL; 927 break; 928 case VOLUME_OVERFLOW: 929 /* See SSC3rXX or current. */ 930 action = ACTION_FAIL; 931 break; 932 default: 933 action = ACTION_FAIL; 934 break; 935 } 936 } else 937 action = ACTION_FAIL; 938 939 if (action != ACTION_FAIL && 940 time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) 941 action = ACTION_FAIL; 942 943 switch (action) { 944 case ACTION_FAIL: 945 /* Give up and fail the remainder of the request */ 946 if (!(req->cmd_flags & REQ_QUIET)) { 947 static DEFINE_RATELIMIT_STATE(_rs, 948 DEFAULT_RATELIMIT_INTERVAL, 949 DEFAULT_RATELIMIT_BURST); 950 951 if (unlikely(scsi_logging_level)) 952 level = SCSI_LOG_LEVEL(SCSI_LOG_MLCOMPLETE_SHIFT, 953 SCSI_LOG_MLCOMPLETE_BITS); 954 955 /* 956 * if logging is enabled the failure will be printed 957 * in scsi_log_completion(), so avoid duplicate messages 958 */ 959 if (!level && __ratelimit(&_rs)) { 960 scsi_print_result(cmd, NULL, FAILED); 961 if (driver_byte(result) & DRIVER_SENSE) 962 scsi_print_sense(cmd); 963 scsi_print_command(cmd); 964 } 965 } 966 if (!scsi_end_request(req, error, blk_rq_err_bytes(req), 0)) 967 return; 968 /*FALLTHRU*/ 969 case ACTION_REPREP: 970 requeue: 971 /* Unprep the request and put it back at the head of the queue. 972 * A new command will be prepared and issued. 973 */ 974 if (q->mq_ops) { 975 cmd->request->cmd_flags &= ~REQ_DONTPREP; 976 scsi_mq_uninit_cmd(cmd); 977 scsi_mq_requeue_cmd(cmd); 978 } else { 979 scsi_release_buffers(cmd); 980 scsi_requeue_command(q, cmd); 981 } 982 break; 983 case ACTION_RETRY: 984 /* Retry the same command immediately */ 985 __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0); 986 break; 987 case ACTION_DELAYED_RETRY: 988 /* Retry the same command after a delay */ 989 __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0); 990 break; 991 } 992 } 993 994 static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb) 995 { 996 int count; 997 998 /* 999 * If sg table allocation fails, requeue request later. 1000 */ 1001 if (unlikely(sg_alloc_table_chained(&sdb->table, req->nr_phys_segments, 1002 sdb->table.sgl))) 1003 return BLKPREP_DEFER; 1004 1005 /* 1006 * Next, walk the list, and fill in the addresses and sizes of 1007 * each segment. 1008 */ 1009 count = blk_rq_map_sg(req->q, req, sdb->table.sgl); 1010 BUG_ON(count > sdb->table.nents); 1011 sdb->table.nents = count; 1012 sdb->length = blk_rq_bytes(req); 1013 return BLKPREP_OK; 1014 } 1015 1016 /* 1017 * Function: scsi_init_io() 1018 * 1019 * Purpose: SCSI I/O initialize function. 1020 * 1021 * Arguments: cmd - Command descriptor we wish to initialize 1022 * 1023 * Returns: 0 on success 1024 * BLKPREP_DEFER if the failure is retryable 1025 * BLKPREP_KILL if the failure is fatal 1026 */ 1027 int scsi_init_io(struct scsi_cmnd *cmd) 1028 { 1029 struct scsi_device *sdev = cmd->device; 1030 struct request *rq = cmd->request; 1031 bool is_mq = (rq->mq_ctx != NULL); 1032 int error; 1033 1034 BUG_ON(!rq->nr_phys_segments); 1035 1036 error = scsi_init_sgtable(rq, &cmd->sdb); 1037 if (error) 1038 goto err_exit; 1039 1040 if (blk_bidi_rq(rq)) { 1041 if (!rq->q->mq_ops) { 1042 struct scsi_data_buffer *bidi_sdb = 1043 kmem_cache_zalloc(scsi_sdb_cache, GFP_ATOMIC); 1044 if (!bidi_sdb) { 1045 error = BLKPREP_DEFER; 1046 goto err_exit; 1047 } 1048 1049 rq->next_rq->special = bidi_sdb; 1050 } 1051 1052 error = scsi_init_sgtable(rq->next_rq, rq->next_rq->special); 1053 if (error) 1054 goto err_exit; 1055 } 1056 1057 if (blk_integrity_rq(rq)) { 1058 struct scsi_data_buffer *prot_sdb = cmd->prot_sdb; 1059 int ivecs, count; 1060 1061 if (prot_sdb == NULL) { 1062 /* 1063 * This can happen if someone (e.g. multipath) 1064 * queues a command to a device on an adapter 1065 * that does not support DIX. 1066 */ 1067 WARN_ON_ONCE(1); 1068 error = BLKPREP_KILL; 1069 goto err_exit; 1070 } 1071 1072 ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio); 1073 1074 if (sg_alloc_table_chained(&prot_sdb->table, ivecs, 1075 prot_sdb->table.sgl)) { 1076 error = BLKPREP_DEFER; 1077 goto err_exit; 1078 } 1079 1080 count = blk_rq_map_integrity_sg(rq->q, rq->bio, 1081 prot_sdb->table.sgl); 1082 BUG_ON(unlikely(count > ivecs)); 1083 BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q))); 1084 1085 cmd->prot_sdb = prot_sdb; 1086 cmd->prot_sdb->table.nents = count; 1087 } 1088 1089 return BLKPREP_OK; 1090 err_exit: 1091 if (is_mq) { 1092 scsi_mq_free_sgtables(cmd); 1093 } else { 1094 scsi_release_buffers(cmd); 1095 cmd->request->special = NULL; 1096 scsi_put_command(cmd); 1097 put_device(&sdev->sdev_gendev); 1098 } 1099 return error; 1100 } 1101 EXPORT_SYMBOL(scsi_init_io); 1102 1103 static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev, 1104 struct request *req) 1105 { 1106 struct scsi_cmnd *cmd; 1107 1108 if (!req->special) { 1109 /* Bail if we can't get a reference to the device */ 1110 if (!get_device(&sdev->sdev_gendev)) 1111 return NULL; 1112 1113 cmd = scsi_get_command(sdev, GFP_ATOMIC); 1114 if (unlikely(!cmd)) { 1115 put_device(&sdev->sdev_gendev); 1116 return NULL; 1117 } 1118 req->special = cmd; 1119 } else { 1120 cmd = req->special; 1121 } 1122 1123 /* pull a tag out of the request if we have one */ 1124 cmd->tag = req->tag; 1125 cmd->request = req; 1126 1127 cmd->cmnd = req->cmd; 1128 cmd->prot_op = SCSI_PROT_NORMAL; 1129 1130 return cmd; 1131 } 1132 1133 static int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req) 1134 { 1135 struct scsi_cmnd *cmd = req->special; 1136 1137 /* 1138 * BLOCK_PC requests may transfer data, in which case they must 1139 * a bio attached to them. Or they might contain a SCSI command 1140 * that does not transfer data, in which case they may optionally 1141 * submit a request without an attached bio. 1142 */ 1143 if (req->bio) { 1144 int ret = scsi_init_io(cmd); 1145 if (unlikely(ret)) 1146 return ret; 1147 } else { 1148 BUG_ON(blk_rq_bytes(req)); 1149 1150 memset(&cmd->sdb, 0, sizeof(cmd->sdb)); 1151 } 1152 1153 cmd->cmd_len = req->cmd_len; 1154 cmd->transfersize = blk_rq_bytes(req); 1155 cmd->allowed = req->retries; 1156 return BLKPREP_OK; 1157 } 1158 1159 /* 1160 * Setup a REQ_TYPE_FS command. These are simple request from filesystems 1161 * that still need to be translated to SCSI CDBs from the ULD. 1162 */ 1163 static int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req) 1164 { 1165 struct scsi_cmnd *cmd = req->special; 1166 1167 if (unlikely(sdev->handler && sdev->handler->prep_fn)) { 1168 int ret = sdev->handler->prep_fn(sdev, req); 1169 if (ret != BLKPREP_OK) 1170 return ret; 1171 } 1172 1173 memset(cmd->cmnd, 0, BLK_MAX_CDB); 1174 return scsi_cmd_to_driver(cmd)->init_command(cmd); 1175 } 1176 1177 static int scsi_setup_cmnd(struct scsi_device *sdev, struct request *req) 1178 { 1179 struct scsi_cmnd *cmd = req->special; 1180 1181 if (!blk_rq_bytes(req)) 1182 cmd->sc_data_direction = DMA_NONE; 1183 else if (rq_data_dir(req) == WRITE) 1184 cmd->sc_data_direction = DMA_TO_DEVICE; 1185 else 1186 cmd->sc_data_direction = DMA_FROM_DEVICE; 1187 1188 switch (req->cmd_type) { 1189 case REQ_TYPE_FS: 1190 return scsi_setup_fs_cmnd(sdev, req); 1191 case REQ_TYPE_BLOCK_PC: 1192 return scsi_setup_blk_pc_cmnd(sdev, req); 1193 default: 1194 return BLKPREP_KILL; 1195 } 1196 } 1197 1198 static int 1199 scsi_prep_state_check(struct scsi_device *sdev, struct request *req) 1200 { 1201 int ret = BLKPREP_OK; 1202 1203 /* 1204 * If the device is not in running state we will reject some 1205 * or all commands. 1206 */ 1207 if (unlikely(sdev->sdev_state != SDEV_RUNNING)) { 1208 switch (sdev->sdev_state) { 1209 case SDEV_OFFLINE: 1210 case SDEV_TRANSPORT_OFFLINE: 1211 /* 1212 * If the device is offline we refuse to process any 1213 * commands. The device must be brought online 1214 * before trying any recovery commands. 1215 */ 1216 sdev_printk(KERN_ERR, sdev, 1217 "rejecting I/O to offline device\n"); 1218 ret = BLKPREP_KILL; 1219 break; 1220 case SDEV_DEL: 1221 /* 1222 * If the device is fully deleted, we refuse to 1223 * process any commands as well. 1224 */ 1225 sdev_printk(KERN_ERR, sdev, 1226 "rejecting I/O to dead device\n"); 1227 ret = BLKPREP_KILL; 1228 break; 1229 case SDEV_BLOCK: 1230 case SDEV_CREATED_BLOCK: 1231 ret = BLKPREP_DEFER; 1232 break; 1233 case SDEV_QUIESCE: 1234 /* 1235 * If the devices is blocked we defer normal commands. 1236 */ 1237 if (!(req->cmd_flags & REQ_PREEMPT)) 1238 ret = BLKPREP_DEFER; 1239 break; 1240 default: 1241 /* 1242 * For any other not fully online state we only allow 1243 * special commands. In particular any user initiated 1244 * command is not allowed. 1245 */ 1246 if (!(req->cmd_flags & REQ_PREEMPT)) 1247 ret = BLKPREP_KILL; 1248 break; 1249 } 1250 } 1251 return ret; 1252 } 1253 1254 static int 1255 scsi_prep_return(struct request_queue *q, struct request *req, int ret) 1256 { 1257 struct scsi_device *sdev = q->queuedata; 1258 1259 switch (ret) { 1260 case BLKPREP_KILL: 1261 case BLKPREP_INVALID: 1262 req->errors = DID_NO_CONNECT << 16; 1263 /* release the command and kill it */ 1264 if (req->special) { 1265 struct scsi_cmnd *cmd = req->special; 1266 scsi_release_buffers(cmd); 1267 scsi_put_command(cmd); 1268 put_device(&sdev->sdev_gendev); 1269 req->special = NULL; 1270 } 1271 break; 1272 case BLKPREP_DEFER: 1273 /* 1274 * If we defer, the blk_peek_request() returns NULL, but the 1275 * queue must be restarted, so we schedule a callback to happen 1276 * shortly. 1277 */ 1278 if (atomic_read(&sdev->device_busy) == 0) 1279 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1280 break; 1281 default: 1282 req->cmd_flags |= REQ_DONTPREP; 1283 } 1284 1285 return ret; 1286 } 1287 1288 static int scsi_prep_fn(struct request_queue *q, struct request *req) 1289 { 1290 struct scsi_device *sdev = q->queuedata; 1291 struct scsi_cmnd *cmd; 1292 int ret; 1293 1294 ret = scsi_prep_state_check(sdev, req); 1295 if (ret != BLKPREP_OK) 1296 goto out; 1297 1298 cmd = scsi_get_cmd_from_req(sdev, req); 1299 if (unlikely(!cmd)) { 1300 ret = BLKPREP_DEFER; 1301 goto out; 1302 } 1303 1304 ret = scsi_setup_cmnd(sdev, req); 1305 out: 1306 return scsi_prep_return(q, req, ret); 1307 } 1308 1309 static void scsi_unprep_fn(struct request_queue *q, struct request *req) 1310 { 1311 scsi_uninit_cmd(req->special); 1312 } 1313 1314 /* 1315 * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else 1316 * return 0. 1317 * 1318 * Called with the queue_lock held. 1319 */ 1320 static inline int scsi_dev_queue_ready(struct request_queue *q, 1321 struct scsi_device *sdev) 1322 { 1323 unsigned int busy; 1324 1325 busy = atomic_inc_return(&sdev->device_busy) - 1; 1326 if (atomic_read(&sdev->device_blocked)) { 1327 if (busy) 1328 goto out_dec; 1329 1330 /* 1331 * unblock after device_blocked iterates to zero 1332 */ 1333 if (atomic_dec_return(&sdev->device_blocked) > 0) { 1334 /* 1335 * For the MQ case we take care of this in the caller. 1336 */ 1337 if (!q->mq_ops) 1338 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1339 goto out_dec; 1340 } 1341 SCSI_LOG_MLQUEUE(3, sdev_printk(KERN_INFO, sdev, 1342 "unblocking device at zero depth\n")); 1343 } 1344 1345 if (busy >= sdev->queue_depth) 1346 goto out_dec; 1347 1348 return 1; 1349 out_dec: 1350 atomic_dec(&sdev->device_busy); 1351 return 0; 1352 } 1353 1354 /* 1355 * scsi_target_queue_ready: checks if there we can send commands to target 1356 * @sdev: scsi device on starget to check. 1357 */ 1358 static inline int scsi_target_queue_ready(struct Scsi_Host *shost, 1359 struct scsi_device *sdev) 1360 { 1361 struct scsi_target *starget = scsi_target(sdev); 1362 unsigned int busy; 1363 1364 if (starget->single_lun) { 1365 spin_lock_irq(shost->host_lock); 1366 if (starget->starget_sdev_user && 1367 starget->starget_sdev_user != sdev) { 1368 spin_unlock_irq(shost->host_lock); 1369 return 0; 1370 } 1371 starget->starget_sdev_user = sdev; 1372 spin_unlock_irq(shost->host_lock); 1373 } 1374 1375 if (starget->can_queue <= 0) 1376 return 1; 1377 1378 busy = atomic_inc_return(&starget->target_busy) - 1; 1379 if (atomic_read(&starget->target_blocked) > 0) { 1380 if (busy) 1381 goto starved; 1382 1383 /* 1384 * unblock after target_blocked iterates to zero 1385 */ 1386 if (atomic_dec_return(&starget->target_blocked) > 0) 1387 goto out_dec; 1388 1389 SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget, 1390 "unblocking target at zero depth\n")); 1391 } 1392 1393 if (busy >= starget->can_queue) 1394 goto starved; 1395 1396 return 1; 1397 1398 starved: 1399 spin_lock_irq(shost->host_lock); 1400 list_move_tail(&sdev->starved_entry, &shost->starved_list); 1401 spin_unlock_irq(shost->host_lock); 1402 out_dec: 1403 if (starget->can_queue > 0) 1404 atomic_dec(&starget->target_busy); 1405 return 0; 1406 } 1407 1408 /* 1409 * scsi_host_queue_ready: if we can send requests to shost, return 1 else 1410 * return 0. We must end up running the queue again whenever 0 is 1411 * returned, else IO can hang. 1412 */ 1413 static inline int scsi_host_queue_ready(struct request_queue *q, 1414 struct Scsi_Host *shost, 1415 struct scsi_device *sdev) 1416 { 1417 unsigned int busy; 1418 1419 if (scsi_host_in_recovery(shost)) 1420 return 0; 1421 1422 busy = atomic_inc_return(&shost->host_busy) - 1; 1423 if (atomic_read(&shost->host_blocked) > 0) { 1424 if (busy) 1425 goto starved; 1426 1427 /* 1428 * unblock after host_blocked iterates to zero 1429 */ 1430 if (atomic_dec_return(&shost->host_blocked) > 0) 1431 goto out_dec; 1432 1433 SCSI_LOG_MLQUEUE(3, 1434 shost_printk(KERN_INFO, shost, 1435 "unblocking host at zero depth\n")); 1436 } 1437 1438 if (shost->can_queue > 0 && busy >= shost->can_queue) 1439 goto starved; 1440 if (shost->host_self_blocked) 1441 goto starved; 1442 1443 /* We're OK to process the command, so we can't be starved */ 1444 if (!list_empty(&sdev->starved_entry)) { 1445 spin_lock_irq(shost->host_lock); 1446 if (!list_empty(&sdev->starved_entry)) 1447 list_del_init(&sdev->starved_entry); 1448 spin_unlock_irq(shost->host_lock); 1449 } 1450 1451 return 1; 1452 1453 starved: 1454 spin_lock_irq(shost->host_lock); 1455 if (list_empty(&sdev->starved_entry)) 1456 list_add_tail(&sdev->starved_entry, &shost->starved_list); 1457 spin_unlock_irq(shost->host_lock); 1458 out_dec: 1459 atomic_dec(&shost->host_busy); 1460 return 0; 1461 } 1462 1463 /* 1464 * Busy state exporting function for request stacking drivers. 1465 * 1466 * For efficiency, no lock is taken to check the busy state of 1467 * shost/starget/sdev, since the returned value is not guaranteed and 1468 * may be changed after request stacking drivers call the function, 1469 * regardless of taking lock or not. 1470 * 1471 * When scsi can't dispatch I/Os anymore and needs to kill I/Os scsi 1472 * needs to return 'not busy'. Otherwise, request stacking drivers 1473 * may hold requests forever. 1474 */ 1475 static int scsi_lld_busy(struct request_queue *q) 1476 { 1477 struct scsi_device *sdev = q->queuedata; 1478 struct Scsi_Host *shost; 1479 1480 if (blk_queue_dying(q)) 1481 return 0; 1482 1483 shost = sdev->host; 1484 1485 /* 1486 * Ignore host/starget busy state. 1487 * Since block layer does not have a concept of fairness across 1488 * multiple queues, congestion of host/starget needs to be handled 1489 * in SCSI layer. 1490 */ 1491 if (scsi_host_in_recovery(shost) || scsi_device_is_busy(sdev)) 1492 return 1; 1493 1494 return 0; 1495 } 1496 1497 /* 1498 * Kill a request for a dead device 1499 */ 1500 static void scsi_kill_request(struct request *req, struct request_queue *q) 1501 { 1502 struct scsi_cmnd *cmd = req->special; 1503 struct scsi_device *sdev; 1504 struct scsi_target *starget; 1505 struct Scsi_Host *shost; 1506 1507 blk_start_request(req); 1508 1509 scmd_printk(KERN_INFO, cmd, "killing request\n"); 1510 1511 sdev = cmd->device; 1512 starget = scsi_target(sdev); 1513 shost = sdev->host; 1514 scsi_init_cmd_errh(cmd); 1515 cmd->result = DID_NO_CONNECT << 16; 1516 atomic_inc(&cmd->device->iorequest_cnt); 1517 1518 /* 1519 * SCSI request completion path will do scsi_device_unbusy(), 1520 * bump busy counts. To bump the counters, we need to dance 1521 * with the locks as normal issue path does. 1522 */ 1523 atomic_inc(&sdev->device_busy); 1524 atomic_inc(&shost->host_busy); 1525 if (starget->can_queue > 0) 1526 atomic_inc(&starget->target_busy); 1527 1528 blk_complete_request(req); 1529 } 1530 1531 static void scsi_softirq_done(struct request *rq) 1532 { 1533 struct scsi_cmnd *cmd = rq->special; 1534 unsigned long wait_for = (cmd->allowed + 1) * rq->timeout; 1535 int disposition; 1536 1537 INIT_LIST_HEAD(&cmd->eh_entry); 1538 1539 atomic_inc(&cmd->device->iodone_cnt); 1540 if (cmd->result) 1541 atomic_inc(&cmd->device->ioerr_cnt); 1542 1543 disposition = scsi_decide_disposition(cmd); 1544 if (disposition != SUCCESS && 1545 time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) { 1546 sdev_printk(KERN_ERR, cmd->device, 1547 "timing out command, waited %lus\n", 1548 wait_for/HZ); 1549 disposition = SUCCESS; 1550 } 1551 1552 scsi_log_completion(cmd, disposition); 1553 1554 switch (disposition) { 1555 case SUCCESS: 1556 scsi_finish_command(cmd); 1557 break; 1558 case NEEDS_RETRY: 1559 scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY); 1560 break; 1561 case ADD_TO_MLQUEUE: 1562 scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY); 1563 break; 1564 default: 1565 if (!scsi_eh_scmd_add(cmd, 0)) 1566 scsi_finish_command(cmd); 1567 } 1568 } 1569 1570 /** 1571 * scsi_dispatch_command - Dispatch a command to the low-level driver. 1572 * @cmd: command block we are dispatching. 1573 * 1574 * Return: nonzero return request was rejected and device's queue needs to be 1575 * plugged. 1576 */ 1577 static int scsi_dispatch_cmd(struct scsi_cmnd *cmd) 1578 { 1579 struct Scsi_Host *host = cmd->device->host; 1580 int rtn = 0; 1581 1582 atomic_inc(&cmd->device->iorequest_cnt); 1583 1584 /* check if the device is still usable */ 1585 if (unlikely(cmd->device->sdev_state == SDEV_DEL)) { 1586 /* in SDEV_DEL we error all commands. DID_NO_CONNECT 1587 * returns an immediate error upwards, and signals 1588 * that the device is no longer present */ 1589 cmd->result = DID_NO_CONNECT << 16; 1590 goto done; 1591 } 1592 1593 /* Check to see if the scsi lld made this device blocked. */ 1594 if (unlikely(scsi_device_blocked(cmd->device))) { 1595 /* 1596 * in blocked state, the command is just put back on 1597 * the device queue. The suspend state has already 1598 * blocked the queue so future requests should not 1599 * occur until the device transitions out of the 1600 * suspend state. 1601 */ 1602 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd, 1603 "queuecommand : device blocked\n")); 1604 return SCSI_MLQUEUE_DEVICE_BUSY; 1605 } 1606 1607 /* Store the LUN value in cmnd, if needed. */ 1608 if (cmd->device->lun_in_cdb) 1609 cmd->cmnd[1] = (cmd->cmnd[1] & 0x1f) | 1610 (cmd->device->lun << 5 & 0xe0); 1611 1612 scsi_log_send(cmd); 1613 1614 /* 1615 * Before we queue this command, check if the command 1616 * length exceeds what the host adapter can handle. 1617 */ 1618 if (cmd->cmd_len > cmd->device->host->max_cmd_len) { 1619 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd, 1620 "queuecommand : command too long. " 1621 "cdb_size=%d host->max_cmd_len=%d\n", 1622 cmd->cmd_len, cmd->device->host->max_cmd_len)); 1623 cmd->result = (DID_ABORT << 16); 1624 goto done; 1625 } 1626 1627 if (unlikely(host->shost_state == SHOST_DEL)) { 1628 cmd->result = (DID_NO_CONNECT << 16); 1629 goto done; 1630 1631 } 1632 1633 trace_scsi_dispatch_cmd_start(cmd); 1634 rtn = host->hostt->queuecommand(host, cmd); 1635 if (rtn) { 1636 trace_scsi_dispatch_cmd_error(cmd, rtn); 1637 if (rtn != SCSI_MLQUEUE_DEVICE_BUSY && 1638 rtn != SCSI_MLQUEUE_TARGET_BUSY) 1639 rtn = SCSI_MLQUEUE_HOST_BUSY; 1640 1641 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd, 1642 "queuecommand : request rejected\n")); 1643 } 1644 1645 return rtn; 1646 done: 1647 cmd->scsi_done(cmd); 1648 return 0; 1649 } 1650 1651 /** 1652 * scsi_done - Invoke completion on finished SCSI command. 1653 * @cmd: The SCSI Command for which a low-level device driver (LLDD) gives 1654 * ownership back to SCSI Core -- i.e. the LLDD has finished with it. 1655 * 1656 * Description: This function is the mid-level's (SCSI Core) interrupt routine, 1657 * which regains ownership of the SCSI command (de facto) from a LLDD, and 1658 * calls blk_complete_request() for further processing. 1659 * 1660 * This function is interrupt context safe. 1661 */ 1662 static void scsi_done(struct scsi_cmnd *cmd) 1663 { 1664 trace_scsi_dispatch_cmd_done(cmd); 1665 blk_complete_request(cmd->request); 1666 } 1667 1668 /* 1669 * Function: scsi_request_fn() 1670 * 1671 * Purpose: Main strategy routine for SCSI. 1672 * 1673 * Arguments: q - Pointer to actual queue. 1674 * 1675 * Returns: Nothing 1676 * 1677 * Lock status: IO request lock assumed to be held when called. 1678 */ 1679 static void scsi_request_fn(struct request_queue *q) 1680 __releases(q->queue_lock) 1681 __acquires(q->queue_lock) 1682 { 1683 struct scsi_device *sdev = q->queuedata; 1684 struct Scsi_Host *shost; 1685 struct scsi_cmnd *cmd; 1686 struct request *req; 1687 1688 /* 1689 * To start with, we keep looping until the queue is empty, or until 1690 * the host is no longer able to accept any more requests. 1691 */ 1692 shost = sdev->host; 1693 for (;;) { 1694 int rtn; 1695 /* 1696 * get next queueable request. We do this early to make sure 1697 * that the request is fully prepared even if we cannot 1698 * accept it. 1699 */ 1700 req = blk_peek_request(q); 1701 if (!req) 1702 break; 1703 1704 if (unlikely(!scsi_device_online(sdev))) { 1705 sdev_printk(KERN_ERR, sdev, 1706 "rejecting I/O to offline device\n"); 1707 scsi_kill_request(req, q); 1708 continue; 1709 } 1710 1711 if (!scsi_dev_queue_ready(q, sdev)) 1712 break; 1713 1714 /* 1715 * Remove the request from the request list. 1716 */ 1717 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req))) 1718 blk_start_request(req); 1719 1720 spin_unlock_irq(q->queue_lock); 1721 cmd = req->special; 1722 if (unlikely(cmd == NULL)) { 1723 printk(KERN_CRIT "impossible request in %s.\n" 1724 "please mail a stack trace to " 1725 "linux-scsi@vger.kernel.org\n", 1726 __func__); 1727 blk_dump_rq_flags(req, "foo"); 1728 BUG(); 1729 } 1730 1731 /* 1732 * We hit this when the driver is using a host wide 1733 * tag map. For device level tag maps the queue_depth check 1734 * in the device ready fn would prevent us from trying 1735 * to allocate a tag. Since the map is a shared host resource 1736 * we add the dev to the starved list so it eventually gets 1737 * a run when a tag is freed. 1738 */ 1739 if (blk_queue_tagged(q) && !(req->cmd_flags & REQ_QUEUED)) { 1740 spin_lock_irq(shost->host_lock); 1741 if (list_empty(&sdev->starved_entry)) 1742 list_add_tail(&sdev->starved_entry, 1743 &shost->starved_list); 1744 spin_unlock_irq(shost->host_lock); 1745 goto not_ready; 1746 } 1747 1748 if (!scsi_target_queue_ready(shost, sdev)) 1749 goto not_ready; 1750 1751 if (!scsi_host_queue_ready(q, shost, sdev)) 1752 goto host_not_ready; 1753 1754 if (sdev->simple_tags) 1755 cmd->flags |= SCMD_TAGGED; 1756 else 1757 cmd->flags &= ~SCMD_TAGGED; 1758 1759 /* 1760 * Finally, initialize any error handling parameters, and set up 1761 * the timers for timeouts. 1762 */ 1763 scsi_init_cmd_errh(cmd); 1764 1765 /* 1766 * Dispatch the command to the low-level driver. 1767 */ 1768 cmd->scsi_done = scsi_done; 1769 rtn = scsi_dispatch_cmd(cmd); 1770 if (rtn) { 1771 scsi_queue_insert(cmd, rtn); 1772 spin_lock_irq(q->queue_lock); 1773 goto out_delay; 1774 } 1775 spin_lock_irq(q->queue_lock); 1776 } 1777 1778 return; 1779 1780 host_not_ready: 1781 if (scsi_target(sdev)->can_queue > 0) 1782 atomic_dec(&scsi_target(sdev)->target_busy); 1783 not_ready: 1784 /* 1785 * lock q, handle tag, requeue req, and decrement device_busy. We 1786 * must return with queue_lock held. 1787 * 1788 * Decrementing device_busy without checking it is OK, as all such 1789 * cases (host limits or settings) should run the queue at some 1790 * later time. 1791 */ 1792 spin_lock_irq(q->queue_lock); 1793 blk_requeue_request(q, req); 1794 atomic_dec(&sdev->device_busy); 1795 out_delay: 1796 if (!atomic_read(&sdev->device_busy) && !scsi_device_blocked(sdev)) 1797 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1798 } 1799 1800 static inline int prep_to_mq(int ret) 1801 { 1802 switch (ret) { 1803 case BLKPREP_OK: 1804 return 0; 1805 case BLKPREP_DEFER: 1806 return BLK_MQ_RQ_QUEUE_BUSY; 1807 default: 1808 return BLK_MQ_RQ_QUEUE_ERROR; 1809 } 1810 } 1811 1812 static int scsi_mq_prep_fn(struct request *req) 1813 { 1814 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 1815 struct scsi_device *sdev = req->q->queuedata; 1816 struct Scsi_Host *shost = sdev->host; 1817 unsigned char *sense_buf = cmd->sense_buffer; 1818 struct scatterlist *sg; 1819 1820 memset(cmd, 0, sizeof(struct scsi_cmnd)); 1821 1822 req->special = cmd; 1823 1824 cmd->request = req; 1825 cmd->device = sdev; 1826 cmd->sense_buffer = sense_buf; 1827 1828 cmd->tag = req->tag; 1829 1830 cmd->cmnd = req->cmd; 1831 cmd->prot_op = SCSI_PROT_NORMAL; 1832 1833 INIT_LIST_HEAD(&cmd->list); 1834 INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler); 1835 cmd->jiffies_at_alloc = jiffies; 1836 1837 if (shost->use_cmd_list) { 1838 spin_lock_irq(&sdev->list_lock); 1839 list_add_tail(&cmd->list, &sdev->cmd_list); 1840 spin_unlock_irq(&sdev->list_lock); 1841 } 1842 1843 sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size; 1844 cmd->sdb.table.sgl = sg; 1845 1846 if (scsi_host_get_prot(shost)) { 1847 cmd->prot_sdb = (void *)sg + 1848 min_t(unsigned int, 1849 shost->sg_tablesize, SG_CHUNK_SIZE) * 1850 sizeof(struct scatterlist); 1851 memset(cmd->prot_sdb, 0, sizeof(struct scsi_data_buffer)); 1852 1853 cmd->prot_sdb->table.sgl = 1854 (struct scatterlist *)(cmd->prot_sdb + 1); 1855 } 1856 1857 if (blk_bidi_rq(req)) { 1858 struct request *next_rq = req->next_rq; 1859 struct scsi_data_buffer *bidi_sdb = blk_mq_rq_to_pdu(next_rq); 1860 1861 memset(bidi_sdb, 0, sizeof(struct scsi_data_buffer)); 1862 bidi_sdb->table.sgl = 1863 (struct scatterlist *)(bidi_sdb + 1); 1864 1865 next_rq->special = bidi_sdb; 1866 } 1867 1868 blk_mq_start_request(req); 1869 1870 return scsi_setup_cmnd(sdev, req); 1871 } 1872 1873 static void scsi_mq_done(struct scsi_cmnd *cmd) 1874 { 1875 trace_scsi_dispatch_cmd_done(cmd); 1876 blk_mq_complete_request(cmd->request, cmd->request->errors); 1877 } 1878 1879 static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx, 1880 const struct blk_mq_queue_data *bd) 1881 { 1882 struct request *req = bd->rq; 1883 struct request_queue *q = req->q; 1884 struct scsi_device *sdev = q->queuedata; 1885 struct Scsi_Host *shost = sdev->host; 1886 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 1887 int ret; 1888 int reason; 1889 1890 ret = prep_to_mq(scsi_prep_state_check(sdev, req)); 1891 if (ret) 1892 goto out; 1893 1894 ret = BLK_MQ_RQ_QUEUE_BUSY; 1895 if (!get_device(&sdev->sdev_gendev)) 1896 goto out; 1897 1898 if (!scsi_dev_queue_ready(q, sdev)) 1899 goto out_put_device; 1900 if (!scsi_target_queue_ready(shost, sdev)) 1901 goto out_dec_device_busy; 1902 if (!scsi_host_queue_ready(q, shost, sdev)) 1903 goto out_dec_target_busy; 1904 1905 1906 if (!(req->cmd_flags & REQ_DONTPREP)) { 1907 ret = prep_to_mq(scsi_mq_prep_fn(req)); 1908 if (ret) 1909 goto out_dec_host_busy; 1910 req->cmd_flags |= REQ_DONTPREP; 1911 } else { 1912 blk_mq_start_request(req); 1913 } 1914 1915 if (sdev->simple_tags) 1916 cmd->flags |= SCMD_TAGGED; 1917 else 1918 cmd->flags &= ~SCMD_TAGGED; 1919 1920 scsi_init_cmd_errh(cmd); 1921 cmd->scsi_done = scsi_mq_done; 1922 1923 reason = scsi_dispatch_cmd(cmd); 1924 if (reason) { 1925 scsi_set_blocked(cmd, reason); 1926 ret = BLK_MQ_RQ_QUEUE_BUSY; 1927 goto out_dec_host_busy; 1928 } 1929 1930 return BLK_MQ_RQ_QUEUE_OK; 1931 1932 out_dec_host_busy: 1933 atomic_dec(&shost->host_busy); 1934 out_dec_target_busy: 1935 if (scsi_target(sdev)->can_queue > 0) 1936 atomic_dec(&scsi_target(sdev)->target_busy); 1937 out_dec_device_busy: 1938 atomic_dec(&sdev->device_busy); 1939 out_put_device: 1940 put_device(&sdev->sdev_gendev); 1941 out: 1942 switch (ret) { 1943 case BLK_MQ_RQ_QUEUE_BUSY: 1944 blk_mq_stop_hw_queue(hctx); 1945 if (atomic_read(&sdev->device_busy) == 0 && 1946 !scsi_device_blocked(sdev)) 1947 blk_mq_delay_queue(hctx, SCSI_QUEUE_DELAY); 1948 break; 1949 case BLK_MQ_RQ_QUEUE_ERROR: 1950 /* 1951 * Make sure to release all allocated ressources when 1952 * we hit an error, as we will never see this command 1953 * again. 1954 */ 1955 if (req->cmd_flags & REQ_DONTPREP) 1956 scsi_mq_uninit_cmd(cmd); 1957 break; 1958 default: 1959 break; 1960 } 1961 return ret; 1962 } 1963 1964 static enum blk_eh_timer_return scsi_timeout(struct request *req, 1965 bool reserved) 1966 { 1967 if (reserved) 1968 return BLK_EH_RESET_TIMER; 1969 return scsi_times_out(req); 1970 } 1971 1972 static int scsi_init_request(void *data, struct request *rq, 1973 unsigned int hctx_idx, unsigned int request_idx, 1974 unsigned int numa_node) 1975 { 1976 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 1977 1978 cmd->sense_buffer = kzalloc_node(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL, 1979 numa_node); 1980 if (!cmd->sense_buffer) 1981 return -ENOMEM; 1982 return 0; 1983 } 1984 1985 static void scsi_exit_request(void *data, struct request *rq, 1986 unsigned int hctx_idx, unsigned int request_idx) 1987 { 1988 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 1989 1990 kfree(cmd->sense_buffer); 1991 } 1992 1993 static u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost) 1994 { 1995 struct device *host_dev; 1996 u64 bounce_limit = 0xffffffff; 1997 1998 if (shost->unchecked_isa_dma) 1999 return BLK_BOUNCE_ISA; 2000 /* 2001 * Platforms with virtual-DMA translation 2002 * hardware have no practical limit. 2003 */ 2004 if (!PCI_DMA_BUS_IS_PHYS) 2005 return BLK_BOUNCE_ANY; 2006 2007 host_dev = scsi_get_device(shost); 2008 if (host_dev && host_dev->dma_mask) 2009 bounce_limit = (u64)dma_max_pfn(host_dev) << PAGE_SHIFT; 2010 2011 return bounce_limit; 2012 } 2013 2014 static void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q) 2015 { 2016 struct device *dev = shost->dma_dev; 2017 2018 /* 2019 * this limit is imposed by hardware restrictions 2020 */ 2021 blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize, 2022 SG_MAX_SEGMENTS)); 2023 2024 if (scsi_host_prot_dma(shost)) { 2025 shost->sg_prot_tablesize = 2026 min_not_zero(shost->sg_prot_tablesize, 2027 (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS); 2028 BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize); 2029 blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize); 2030 } 2031 2032 blk_queue_max_hw_sectors(q, shost->max_sectors); 2033 blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost)); 2034 blk_queue_segment_boundary(q, shost->dma_boundary); 2035 dma_set_seg_boundary(dev, shost->dma_boundary); 2036 2037 blk_queue_max_segment_size(q, dma_get_max_seg_size(dev)); 2038 2039 if (!shost->use_clustering) 2040 q->limits.cluster = 0; 2041 2042 /* 2043 * set a reasonable default alignment on word boundaries: the 2044 * host and device may alter it using 2045 * blk_queue_update_dma_alignment() later. 2046 */ 2047 blk_queue_dma_alignment(q, 0x03); 2048 } 2049 2050 struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost, 2051 request_fn_proc *request_fn) 2052 { 2053 struct request_queue *q; 2054 2055 q = blk_init_queue(request_fn, NULL); 2056 if (!q) 2057 return NULL; 2058 __scsi_init_queue(shost, q); 2059 return q; 2060 } 2061 EXPORT_SYMBOL(__scsi_alloc_queue); 2062 2063 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev) 2064 { 2065 struct request_queue *q; 2066 2067 q = __scsi_alloc_queue(sdev->host, scsi_request_fn); 2068 if (!q) 2069 return NULL; 2070 2071 blk_queue_prep_rq(q, scsi_prep_fn); 2072 blk_queue_unprep_rq(q, scsi_unprep_fn); 2073 blk_queue_softirq_done(q, scsi_softirq_done); 2074 blk_queue_rq_timed_out(q, scsi_times_out); 2075 blk_queue_lld_busy(q, scsi_lld_busy); 2076 return q; 2077 } 2078 2079 static struct blk_mq_ops scsi_mq_ops = { 2080 .map_queue = blk_mq_map_queue, 2081 .queue_rq = scsi_queue_rq, 2082 .complete = scsi_softirq_done, 2083 .timeout = scsi_timeout, 2084 .init_request = scsi_init_request, 2085 .exit_request = scsi_exit_request, 2086 }; 2087 2088 struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev) 2089 { 2090 sdev->request_queue = blk_mq_init_queue(&sdev->host->tag_set); 2091 if (IS_ERR(sdev->request_queue)) 2092 return NULL; 2093 2094 sdev->request_queue->queuedata = sdev; 2095 __scsi_init_queue(sdev->host, sdev->request_queue); 2096 return sdev->request_queue; 2097 } 2098 2099 int scsi_mq_setup_tags(struct Scsi_Host *shost) 2100 { 2101 unsigned int cmd_size, sgl_size, tbl_size; 2102 2103 tbl_size = shost->sg_tablesize; 2104 if (tbl_size > SG_CHUNK_SIZE) 2105 tbl_size = SG_CHUNK_SIZE; 2106 sgl_size = tbl_size * sizeof(struct scatterlist); 2107 cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size + sgl_size; 2108 if (scsi_host_get_prot(shost)) 2109 cmd_size += sizeof(struct scsi_data_buffer) + sgl_size; 2110 2111 memset(&shost->tag_set, 0, sizeof(shost->tag_set)); 2112 shost->tag_set.ops = &scsi_mq_ops; 2113 shost->tag_set.nr_hw_queues = shost->nr_hw_queues ? : 1; 2114 shost->tag_set.queue_depth = shost->can_queue; 2115 shost->tag_set.cmd_size = cmd_size; 2116 shost->tag_set.numa_node = NUMA_NO_NODE; 2117 shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; 2118 shost->tag_set.flags |= 2119 BLK_ALLOC_POLICY_TO_MQ_FLAG(shost->hostt->tag_alloc_policy); 2120 shost->tag_set.driver_data = shost; 2121 2122 return blk_mq_alloc_tag_set(&shost->tag_set); 2123 } 2124 2125 void scsi_mq_destroy_tags(struct Scsi_Host *shost) 2126 { 2127 blk_mq_free_tag_set(&shost->tag_set); 2128 } 2129 2130 /* 2131 * Function: scsi_block_requests() 2132 * 2133 * Purpose: Utility function used by low-level drivers to prevent further 2134 * commands from being queued to the device. 2135 * 2136 * Arguments: shost - Host in question 2137 * 2138 * Returns: Nothing 2139 * 2140 * Lock status: No locks are assumed held. 2141 * 2142 * Notes: There is no timer nor any other means by which the requests 2143 * get unblocked other than the low-level driver calling 2144 * scsi_unblock_requests(). 2145 */ 2146 void scsi_block_requests(struct Scsi_Host *shost) 2147 { 2148 shost->host_self_blocked = 1; 2149 } 2150 EXPORT_SYMBOL(scsi_block_requests); 2151 2152 /* 2153 * Function: scsi_unblock_requests() 2154 * 2155 * Purpose: Utility function used by low-level drivers to allow further 2156 * commands from being queued to the device. 2157 * 2158 * Arguments: shost - Host in question 2159 * 2160 * Returns: Nothing 2161 * 2162 * Lock status: No locks are assumed held. 2163 * 2164 * Notes: There is no timer nor any other means by which the requests 2165 * get unblocked other than the low-level driver calling 2166 * scsi_unblock_requests(). 2167 * 2168 * This is done as an API function so that changes to the 2169 * internals of the scsi mid-layer won't require wholesale 2170 * changes to drivers that use this feature. 2171 */ 2172 void scsi_unblock_requests(struct Scsi_Host *shost) 2173 { 2174 shost->host_self_blocked = 0; 2175 scsi_run_host_queues(shost); 2176 } 2177 EXPORT_SYMBOL(scsi_unblock_requests); 2178 2179 int __init scsi_init_queue(void) 2180 { 2181 scsi_sdb_cache = kmem_cache_create("scsi_data_buffer", 2182 sizeof(struct scsi_data_buffer), 2183 0, 0, NULL); 2184 if (!scsi_sdb_cache) { 2185 printk(KERN_ERR "SCSI: can't init scsi sdb cache\n"); 2186 return -ENOMEM; 2187 } 2188 2189 return 0; 2190 } 2191 2192 void scsi_exit_queue(void) 2193 { 2194 kmem_cache_destroy(scsi_sdb_cache); 2195 } 2196 2197 /** 2198 * scsi_mode_select - issue a mode select 2199 * @sdev: SCSI device to be queried 2200 * @pf: Page format bit (1 == standard, 0 == vendor specific) 2201 * @sp: Save page bit (0 == don't save, 1 == save) 2202 * @modepage: mode page being requested 2203 * @buffer: request buffer (may not be smaller than eight bytes) 2204 * @len: length of request buffer. 2205 * @timeout: command timeout 2206 * @retries: number of retries before failing 2207 * @data: returns a structure abstracting the mode header data 2208 * @sshdr: place to put sense data (or NULL if no sense to be collected). 2209 * must be SCSI_SENSE_BUFFERSIZE big. 2210 * 2211 * Returns zero if successful; negative error number or scsi 2212 * status on error 2213 * 2214 */ 2215 int 2216 scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage, 2217 unsigned char *buffer, int len, int timeout, int retries, 2218 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr) 2219 { 2220 unsigned char cmd[10]; 2221 unsigned char *real_buffer; 2222 int ret; 2223 2224 memset(cmd, 0, sizeof(cmd)); 2225 cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0); 2226 2227 if (sdev->use_10_for_ms) { 2228 if (len > 65535) 2229 return -EINVAL; 2230 real_buffer = kmalloc(8 + len, GFP_KERNEL); 2231 if (!real_buffer) 2232 return -ENOMEM; 2233 memcpy(real_buffer + 8, buffer, len); 2234 len += 8; 2235 real_buffer[0] = 0; 2236 real_buffer[1] = 0; 2237 real_buffer[2] = data->medium_type; 2238 real_buffer[3] = data->device_specific; 2239 real_buffer[4] = data->longlba ? 0x01 : 0; 2240 real_buffer[5] = 0; 2241 real_buffer[6] = data->block_descriptor_length >> 8; 2242 real_buffer[7] = data->block_descriptor_length; 2243 2244 cmd[0] = MODE_SELECT_10; 2245 cmd[7] = len >> 8; 2246 cmd[8] = len; 2247 } else { 2248 if (len > 255 || data->block_descriptor_length > 255 || 2249 data->longlba) 2250 return -EINVAL; 2251 2252 real_buffer = kmalloc(4 + len, GFP_KERNEL); 2253 if (!real_buffer) 2254 return -ENOMEM; 2255 memcpy(real_buffer + 4, buffer, len); 2256 len += 4; 2257 real_buffer[0] = 0; 2258 real_buffer[1] = data->medium_type; 2259 real_buffer[2] = data->device_specific; 2260 real_buffer[3] = data->block_descriptor_length; 2261 2262 2263 cmd[0] = MODE_SELECT; 2264 cmd[4] = len; 2265 } 2266 2267 ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len, 2268 sshdr, timeout, retries, NULL); 2269 kfree(real_buffer); 2270 return ret; 2271 } 2272 EXPORT_SYMBOL_GPL(scsi_mode_select); 2273 2274 /** 2275 * scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary. 2276 * @sdev: SCSI device to be queried 2277 * @dbd: set if mode sense will allow block descriptors to be returned 2278 * @modepage: mode page being requested 2279 * @buffer: request buffer (may not be smaller than eight bytes) 2280 * @len: length of request buffer. 2281 * @timeout: command timeout 2282 * @retries: number of retries before failing 2283 * @data: returns a structure abstracting the mode header data 2284 * @sshdr: place to put sense data (or NULL if no sense to be collected). 2285 * must be SCSI_SENSE_BUFFERSIZE big. 2286 * 2287 * Returns zero if unsuccessful, or the header offset (either 4 2288 * or 8 depending on whether a six or ten byte command was 2289 * issued) if successful. 2290 */ 2291 int 2292 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage, 2293 unsigned char *buffer, int len, int timeout, int retries, 2294 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr) 2295 { 2296 unsigned char cmd[12]; 2297 int use_10_for_ms; 2298 int header_length; 2299 int result, retry_count = retries; 2300 struct scsi_sense_hdr my_sshdr; 2301 2302 memset(data, 0, sizeof(*data)); 2303 memset(&cmd[0], 0, 12); 2304 cmd[1] = dbd & 0x18; /* allows DBD and LLBA bits */ 2305 cmd[2] = modepage; 2306 2307 /* caller might not be interested in sense, but we need it */ 2308 if (!sshdr) 2309 sshdr = &my_sshdr; 2310 2311 retry: 2312 use_10_for_ms = sdev->use_10_for_ms; 2313 2314 if (use_10_for_ms) { 2315 if (len < 8) 2316 len = 8; 2317 2318 cmd[0] = MODE_SENSE_10; 2319 cmd[8] = len; 2320 header_length = 8; 2321 } else { 2322 if (len < 4) 2323 len = 4; 2324 2325 cmd[0] = MODE_SENSE; 2326 cmd[4] = len; 2327 header_length = 4; 2328 } 2329 2330 memset(buffer, 0, len); 2331 2332 result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len, 2333 sshdr, timeout, retries, NULL); 2334 2335 /* This code looks awful: what it's doing is making sure an 2336 * ILLEGAL REQUEST sense return identifies the actual command 2337 * byte as the problem. MODE_SENSE commands can return 2338 * ILLEGAL REQUEST if the code page isn't supported */ 2339 2340 if (use_10_for_ms && !scsi_status_is_good(result) && 2341 (driver_byte(result) & DRIVER_SENSE)) { 2342 if (scsi_sense_valid(sshdr)) { 2343 if ((sshdr->sense_key == ILLEGAL_REQUEST) && 2344 (sshdr->asc == 0x20) && (sshdr->ascq == 0)) { 2345 /* 2346 * Invalid command operation code 2347 */ 2348 sdev->use_10_for_ms = 0; 2349 goto retry; 2350 } 2351 } 2352 } 2353 2354 if(scsi_status_is_good(result)) { 2355 if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b && 2356 (modepage == 6 || modepage == 8))) { 2357 /* Initio breakage? */ 2358 header_length = 0; 2359 data->length = 13; 2360 data->medium_type = 0; 2361 data->device_specific = 0; 2362 data->longlba = 0; 2363 data->block_descriptor_length = 0; 2364 } else if(use_10_for_ms) { 2365 data->length = buffer[0]*256 + buffer[1] + 2; 2366 data->medium_type = buffer[2]; 2367 data->device_specific = buffer[3]; 2368 data->longlba = buffer[4] & 0x01; 2369 data->block_descriptor_length = buffer[6]*256 2370 + buffer[7]; 2371 } else { 2372 data->length = buffer[0] + 1; 2373 data->medium_type = buffer[1]; 2374 data->device_specific = buffer[2]; 2375 data->block_descriptor_length = buffer[3]; 2376 } 2377 data->header_length = header_length; 2378 } else if ((status_byte(result) == CHECK_CONDITION) && 2379 scsi_sense_valid(sshdr) && 2380 sshdr->sense_key == UNIT_ATTENTION && retry_count) { 2381 retry_count--; 2382 goto retry; 2383 } 2384 2385 return result; 2386 } 2387 EXPORT_SYMBOL(scsi_mode_sense); 2388 2389 /** 2390 * scsi_test_unit_ready - test if unit is ready 2391 * @sdev: scsi device to change the state of. 2392 * @timeout: command timeout 2393 * @retries: number of retries before failing 2394 * @sshdr_external: Optional pointer to struct scsi_sense_hdr for 2395 * returning sense. Make sure that this is cleared before passing 2396 * in. 2397 * 2398 * Returns zero if unsuccessful or an error if TUR failed. For 2399 * removable media, UNIT_ATTENTION sets ->changed flag. 2400 **/ 2401 int 2402 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries, 2403 struct scsi_sense_hdr *sshdr_external) 2404 { 2405 char cmd[] = { 2406 TEST_UNIT_READY, 0, 0, 0, 0, 0, 2407 }; 2408 struct scsi_sense_hdr *sshdr; 2409 int result; 2410 2411 if (!sshdr_external) 2412 sshdr = kzalloc(sizeof(*sshdr), GFP_KERNEL); 2413 else 2414 sshdr = sshdr_external; 2415 2416 /* try to eat the UNIT_ATTENTION if there are enough retries */ 2417 do { 2418 result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr, 2419 timeout, retries, NULL); 2420 if (sdev->removable && scsi_sense_valid(sshdr) && 2421 sshdr->sense_key == UNIT_ATTENTION) 2422 sdev->changed = 1; 2423 } while (scsi_sense_valid(sshdr) && 2424 sshdr->sense_key == UNIT_ATTENTION && --retries); 2425 2426 if (!sshdr_external) 2427 kfree(sshdr); 2428 return result; 2429 } 2430 EXPORT_SYMBOL(scsi_test_unit_ready); 2431 2432 /** 2433 * scsi_device_set_state - Take the given device through the device state model. 2434 * @sdev: scsi device to change the state of. 2435 * @state: state to change to. 2436 * 2437 * Returns zero if unsuccessful or an error if the requested 2438 * transition is illegal. 2439 */ 2440 int 2441 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state) 2442 { 2443 enum scsi_device_state oldstate = sdev->sdev_state; 2444 2445 if (state == oldstate) 2446 return 0; 2447 2448 switch (state) { 2449 case SDEV_CREATED: 2450 switch (oldstate) { 2451 case SDEV_CREATED_BLOCK: 2452 break; 2453 default: 2454 goto illegal; 2455 } 2456 break; 2457 2458 case SDEV_RUNNING: 2459 switch (oldstate) { 2460 case SDEV_CREATED: 2461 case SDEV_OFFLINE: 2462 case SDEV_TRANSPORT_OFFLINE: 2463 case SDEV_QUIESCE: 2464 case SDEV_BLOCK: 2465 break; 2466 default: 2467 goto illegal; 2468 } 2469 break; 2470 2471 case SDEV_QUIESCE: 2472 switch (oldstate) { 2473 case SDEV_RUNNING: 2474 case SDEV_OFFLINE: 2475 case SDEV_TRANSPORT_OFFLINE: 2476 break; 2477 default: 2478 goto illegal; 2479 } 2480 break; 2481 2482 case SDEV_OFFLINE: 2483 case SDEV_TRANSPORT_OFFLINE: 2484 switch (oldstate) { 2485 case SDEV_CREATED: 2486 case SDEV_RUNNING: 2487 case SDEV_QUIESCE: 2488 case SDEV_BLOCK: 2489 break; 2490 default: 2491 goto illegal; 2492 } 2493 break; 2494 2495 case SDEV_BLOCK: 2496 switch (oldstate) { 2497 case SDEV_RUNNING: 2498 case SDEV_CREATED_BLOCK: 2499 break; 2500 default: 2501 goto illegal; 2502 } 2503 break; 2504 2505 case SDEV_CREATED_BLOCK: 2506 switch (oldstate) { 2507 case SDEV_CREATED: 2508 break; 2509 default: 2510 goto illegal; 2511 } 2512 break; 2513 2514 case SDEV_CANCEL: 2515 switch (oldstate) { 2516 case SDEV_CREATED: 2517 case SDEV_RUNNING: 2518 case SDEV_QUIESCE: 2519 case SDEV_OFFLINE: 2520 case SDEV_TRANSPORT_OFFLINE: 2521 case SDEV_BLOCK: 2522 break; 2523 default: 2524 goto illegal; 2525 } 2526 break; 2527 2528 case SDEV_DEL: 2529 switch (oldstate) { 2530 case SDEV_CREATED: 2531 case SDEV_RUNNING: 2532 case SDEV_OFFLINE: 2533 case SDEV_TRANSPORT_OFFLINE: 2534 case SDEV_CANCEL: 2535 case SDEV_CREATED_BLOCK: 2536 break; 2537 default: 2538 goto illegal; 2539 } 2540 break; 2541 2542 } 2543 sdev->sdev_state = state; 2544 return 0; 2545 2546 illegal: 2547 SCSI_LOG_ERROR_RECOVERY(1, 2548 sdev_printk(KERN_ERR, sdev, 2549 "Illegal state transition %s->%s", 2550 scsi_device_state_name(oldstate), 2551 scsi_device_state_name(state)) 2552 ); 2553 return -EINVAL; 2554 } 2555 EXPORT_SYMBOL(scsi_device_set_state); 2556 2557 /** 2558 * sdev_evt_emit - emit a single SCSI device uevent 2559 * @sdev: associated SCSI device 2560 * @evt: event to emit 2561 * 2562 * Send a single uevent (scsi_event) to the associated scsi_device. 2563 */ 2564 static void scsi_evt_emit(struct scsi_device *sdev, struct scsi_event *evt) 2565 { 2566 int idx = 0; 2567 char *envp[3]; 2568 2569 switch (evt->evt_type) { 2570 case SDEV_EVT_MEDIA_CHANGE: 2571 envp[idx++] = "SDEV_MEDIA_CHANGE=1"; 2572 break; 2573 case SDEV_EVT_INQUIRY_CHANGE_REPORTED: 2574 scsi_rescan_device(&sdev->sdev_gendev); 2575 envp[idx++] = "SDEV_UA=INQUIRY_DATA_HAS_CHANGED"; 2576 break; 2577 case SDEV_EVT_CAPACITY_CHANGE_REPORTED: 2578 envp[idx++] = "SDEV_UA=CAPACITY_DATA_HAS_CHANGED"; 2579 break; 2580 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED: 2581 envp[idx++] = "SDEV_UA=THIN_PROVISIONING_SOFT_THRESHOLD_REACHED"; 2582 break; 2583 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED: 2584 envp[idx++] = "SDEV_UA=MODE_PARAMETERS_CHANGED"; 2585 break; 2586 case SDEV_EVT_LUN_CHANGE_REPORTED: 2587 envp[idx++] = "SDEV_UA=REPORTED_LUNS_DATA_HAS_CHANGED"; 2588 break; 2589 case SDEV_EVT_ALUA_STATE_CHANGE_REPORTED: 2590 envp[idx++] = "SDEV_UA=ASYMMETRIC_ACCESS_STATE_CHANGED"; 2591 break; 2592 default: 2593 /* do nothing */ 2594 break; 2595 } 2596 2597 envp[idx++] = NULL; 2598 2599 kobject_uevent_env(&sdev->sdev_gendev.kobj, KOBJ_CHANGE, envp); 2600 } 2601 2602 /** 2603 * sdev_evt_thread - send a uevent for each scsi event 2604 * @work: work struct for scsi_device 2605 * 2606 * Dispatch queued events to their associated scsi_device kobjects 2607 * as uevents. 2608 */ 2609 void scsi_evt_thread(struct work_struct *work) 2610 { 2611 struct scsi_device *sdev; 2612 enum scsi_device_event evt_type; 2613 LIST_HEAD(event_list); 2614 2615 sdev = container_of(work, struct scsi_device, event_work); 2616 2617 for (evt_type = SDEV_EVT_FIRST; evt_type <= SDEV_EVT_LAST; evt_type++) 2618 if (test_and_clear_bit(evt_type, sdev->pending_events)) 2619 sdev_evt_send_simple(sdev, evt_type, GFP_KERNEL); 2620 2621 while (1) { 2622 struct scsi_event *evt; 2623 struct list_head *this, *tmp; 2624 unsigned long flags; 2625 2626 spin_lock_irqsave(&sdev->list_lock, flags); 2627 list_splice_init(&sdev->event_list, &event_list); 2628 spin_unlock_irqrestore(&sdev->list_lock, flags); 2629 2630 if (list_empty(&event_list)) 2631 break; 2632 2633 list_for_each_safe(this, tmp, &event_list) { 2634 evt = list_entry(this, struct scsi_event, node); 2635 list_del(&evt->node); 2636 scsi_evt_emit(sdev, evt); 2637 kfree(evt); 2638 } 2639 } 2640 } 2641 2642 /** 2643 * sdev_evt_send - send asserted event to uevent thread 2644 * @sdev: scsi_device event occurred on 2645 * @evt: event to send 2646 * 2647 * Assert scsi device event asynchronously. 2648 */ 2649 void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt) 2650 { 2651 unsigned long flags; 2652 2653 #if 0 2654 /* FIXME: currently this check eliminates all media change events 2655 * for polled devices. Need to update to discriminate between AN 2656 * and polled events */ 2657 if (!test_bit(evt->evt_type, sdev->supported_events)) { 2658 kfree(evt); 2659 return; 2660 } 2661 #endif 2662 2663 spin_lock_irqsave(&sdev->list_lock, flags); 2664 list_add_tail(&evt->node, &sdev->event_list); 2665 schedule_work(&sdev->event_work); 2666 spin_unlock_irqrestore(&sdev->list_lock, flags); 2667 } 2668 EXPORT_SYMBOL_GPL(sdev_evt_send); 2669 2670 /** 2671 * sdev_evt_alloc - allocate a new scsi event 2672 * @evt_type: type of event to allocate 2673 * @gfpflags: GFP flags for allocation 2674 * 2675 * Allocates and returns a new scsi_event. 2676 */ 2677 struct scsi_event *sdev_evt_alloc(enum scsi_device_event evt_type, 2678 gfp_t gfpflags) 2679 { 2680 struct scsi_event *evt = kzalloc(sizeof(struct scsi_event), gfpflags); 2681 if (!evt) 2682 return NULL; 2683 2684 evt->evt_type = evt_type; 2685 INIT_LIST_HEAD(&evt->node); 2686 2687 /* evt_type-specific initialization, if any */ 2688 switch (evt_type) { 2689 case SDEV_EVT_MEDIA_CHANGE: 2690 case SDEV_EVT_INQUIRY_CHANGE_REPORTED: 2691 case SDEV_EVT_CAPACITY_CHANGE_REPORTED: 2692 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED: 2693 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED: 2694 case SDEV_EVT_LUN_CHANGE_REPORTED: 2695 case SDEV_EVT_ALUA_STATE_CHANGE_REPORTED: 2696 default: 2697 /* do nothing */ 2698 break; 2699 } 2700 2701 return evt; 2702 } 2703 EXPORT_SYMBOL_GPL(sdev_evt_alloc); 2704 2705 /** 2706 * sdev_evt_send_simple - send asserted event to uevent thread 2707 * @sdev: scsi_device event occurred on 2708 * @evt_type: type of event to send 2709 * @gfpflags: GFP flags for allocation 2710 * 2711 * Assert scsi device event asynchronously, given an event type. 2712 */ 2713 void sdev_evt_send_simple(struct scsi_device *sdev, 2714 enum scsi_device_event evt_type, gfp_t gfpflags) 2715 { 2716 struct scsi_event *evt = sdev_evt_alloc(evt_type, gfpflags); 2717 if (!evt) { 2718 sdev_printk(KERN_ERR, sdev, "event %d eaten due to OOM\n", 2719 evt_type); 2720 return; 2721 } 2722 2723 sdev_evt_send(sdev, evt); 2724 } 2725 EXPORT_SYMBOL_GPL(sdev_evt_send_simple); 2726 2727 /** 2728 * scsi_device_quiesce - Block user issued commands. 2729 * @sdev: scsi device to quiesce. 2730 * 2731 * This works by trying to transition to the SDEV_QUIESCE state 2732 * (which must be a legal transition). When the device is in this 2733 * state, only special requests will be accepted, all others will 2734 * be deferred. Since special requests may also be requeued requests, 2735 * a successful return doesn't guarantee the device will be 2736 * totally quiescent. 2737 * 2738 * Must be called with user context, may sleep. 2739 * 2740 * Returns zero if unsuccessful or an error if not. 2741 */ 2742 int 2743 scsi_device_quiesce(struct scsi_device *sdev) 2744 { 2745 int err = scsi_device_set_state(sdev, SDEV_QUIESCE); 2746 if (err) 2747 return err; 2748 2749 scsi_run_queue(sdev->request_queue); 2750 while (atomic_read(&sdev->device_busy)) { 2751 msleep_interruptible(200); 2752 scsi_run_queue(sdev->request_queue); 2753 } 2754 return 0; 2755 } 2756 EXPORT_SYMBOL(scsi_device_quiesce); 2757 2758 /** 2759 * scsi_device_resume - Restart user issued commands to a quiesced device. 2760 * @sdev: scsi device to resume. 2761 * 2762 * Moves the device from quiesced back to running and restarts the 2763 * queues. 2764 * 2765 * Must be called with user context, may sleep. 2766 */ 2767 void scsi_device_resume(struct scsi_device *sdev) 2768 { 2769 /* check if the device state was mutated prior to resume, and if 2770 * so assume the state is being managed elsewhere (for example 2771 * device deleted during suspend) 2772 */ 2773 if (sdev->sdev_state != SDEV_QUIESCE || 2774 scsi_device_set_state(sdev, SDEV_RUNNING)) 2775 return; 2776 scsi_run_queue(sdev->request_queue); 2777 } 2778 EXPORT_SYMBOL(scsi_device_resume); 2779 2780 static void 2781 device_quiesce_fn(struct scsi_device *sdev, void *data) 2782 { 2783 scsi_device_quiesce(sdev); 2784 } 2785 2786 void 2787 scsi_target_quiesce(struct scsi_target *starget) 2788 { 2789 starget_for_each_device(starget, NULL, device_quiesce_fn); 2790 } 2791 EXPORT_SYMBOL(scsi_target_quiesce); 2792 2793 static void 2794 device_resume_fn(struct scsi_device *sdev, void *data) 2795 { 2796 scsi_device_resume(sdev); 2797 } 2798 2799 void 2800 scsi_target_resume(struct scsi_target *starget) 2801 { 2802 starget_for_each_device(starget, NULL, device_resume_fn); 2803 } 2804 EXPORT_SYMBOL(scsi_target_resume); 2805 2806 /** 2807 * scsi_internal_device_block - internal function to put a device temporarily into the SDEV_BLOCK state 2808 * @sdev: device to block 2809 * 2810 * Block request made by scsi lld's to temporarily stop all 2811 * scsi commands on the specified device. Called from interrupt 2812 * or normal process context. 2813 * 2814 * Returns zero if successful or error if not 2815 * 2816 * Notes: 2817 * This routine transitions the device to the SDEV_BLOCK state 2818 * (which must be a legal transition). When the device is in this 2819 * state, all commands are deferred until the scsi lld reenables 2820 * the device with scsi_device_unblock or device_block_tmo fires. 2821 */ 2822 int 2823 scsi_internal_device_block(struct scsi_device *sdev) 2824 { 2825 struct request_queue *q = sdev->request_queue; 2826 unsigned long flags; 2827 int err = 0; 2828 2829 err = scsi_device_set_state(sdev, SDEV_BLOCK); 2830 if (err) { 2831 err = scsi_device_set_state(sdev, SDEV_CREATED_BLOCK); 2832 2833 if (err) 2834 return err; 2835 } 2836 2837 /* 2838 * The device has transitioned to SDEV_BLOCK. Stop the 2839 * block layer from calling the midlayer with this device's 2840 * request queue. 2841 */ 2842 if (q->mq_ops) { 2843 blk_mq_stop_hw_queues(q); 2844 } else { 2845 spin_lock_irqsave(q->queue_lock, flags); 2846 blk_stop_queue(q); 2847 spin_unlock_irqrestore(q->queue_lock, flags); 2848 } 2849 2850 return 0; 2851 } 2852 EXPORT_SYMBOL_GPL(scsi_internal_device_block); 2853 2854 /** 2855 * scsi_internal_device_unblock - resume a device after a block request 2856 * @sdev: device to resume 2857 * @new_state: state to set devices to after unblocking 2858 * 2859 * Called by scsi lld's or the midlayer to restart the device queue 2860 * for the previously suspended scsi device. Called from interrupt or 2861 * normal process context. 2862 * 2863 * Returns zero if successful or error if not. 2864 * 2865 * Notes: 2866 * This routine transitions the device to the SDEV_RUNNING state 2867 * or to one of the offline states (which must be a legal transition) 2868 * allowing the midlayer to goose the queue for this device. 2869 */ 2870 int 2871 scsi_internal_device_unblock(struct scsi_device *sdev, 2872 enum scsi_device_state new_state) 2873 { 2874 struct request_queue *q = sdev->request_queue; 2875 unsigned long flags; 2876 2877 /* 2878 * Try to transition the scsi device to SDEV_RUNNING or one of the 2879 * offlined states and goose the device queue if successful. 2880 */ 2881 if ((sdev->sdev_state == SDEV_BLOCK) || 2882 (sdev->sdev_state == SDEV_TRANSPORT_OFFLINE)) 2883 sdev->sdev_state = new_state; 2884 else if (sdev->sdev_state == SDEV_CREATED_BLOCK) { 2885 if (new_state == SDEV_TRANSPORT_OFFLINE || 2886 new_state == SDEV_OFFLINE) 2887 sdev->sdev_state = new_state; 2888 else 2889 sdev->sdev_state = SDEV_CREATED; 2890 } else if (sdev->sdev_state != SDEV_CANCEL && 2891 sdev->sdev_state != SDEV_OFFLINE) 2892 return -EINVAL; 2893 2894 if (q->mq_ops) { 2895 blk_mq_start_stopped_hw_queues(q, false); 2896 } else { 2897 spin_lock_irqsave(q->queue_lock, flags); 2898 blk_start_queue(q); 2899 spin_unlock_irqrestore(q->queue_lock, flags); 2900 } 2901 2902 return 0; 2903 } 2904 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock); 2905 2906 static void 2907 device_block(struct scsi_device *sdev, void *data) 2908 { 2909 scsi_internal_device_block(sdev); 2910 } 2911 2912 static int 2913 target_block(struct device *dev, void *data) 2914 { 2915 if (scsi_is_target_device(dev)) 2916 starget_for_each_device(to_scsi_target(dev), NULL, 2917 device_block); 2918 return 0; 2919 } 2920 2921 void 2922 scsi_target_block(struct device *dev) 2923 { 2924 if (scsi_is_target_device(dev)) 2925 starget_for_each_device(to_scsi_target(dev), NULL, 2926 device_block); 2927 else 2928 device_for_each_child(dev, NULL, target_block); 2929 } 2930 EXPORT_SYMBOL_GPL(scsi_target_block); 2931 2932 static void 2933 device_unblock(struct scsi_device *sdev, void *data) 2934 { 2935 scsi_internal_device_unblock(sdev, *(enum scsi_device_state *)data); 2936 } 2937 2938 static int 2939 target_unblock(struct device *dev, void *data) 2940 { 2941 if (scsi_is_target_device(dev)) 2942 starget_for_each_device(to_scsi_target(dev), data, 2943 device_unblock); 2944 return 0; 2945 } 2946 2947 void 2948 scsi_target_unblock(struct device *dev, enum scsi_device_state new_state) 2949 { 2950 if (scsi_is_target_device(dev)) 2951 starget_for_each_device(to_scsi_target(dev), &new_state, 2952 device_unblock); 2953 else 2954 device_for_each_child(dev, &new_state, target_unblock); 2955 } 2956 EXPORT_SYMBOL_GPL(scsi_target_unblock); 2957 2958 /** 2959 * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt 2960 * @sgl: scatter-gather list 2961 * @sg_count: number of segments in sg 2962 * @offset: offset in bytes into sg, on return offset into the mapped area 2963 * @len: bytes to map, on return number of bytes mapped 2964 * 2965 * Returns virtual address of the start of the mapped page 2966 */ 2967 void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count, 2968 size_t *offset, size_t *len) 2969 { 2970 int i; 2971 size_t sg_len = 0, len_complete = 0; 2972 struct scatterlist *sg; 2973 struct page *page; 2974 2975 WARN_ON(!irqs_disabled()); 2976 2977 for_each_sg(sgl, sg, sg_count, i) { 2978 len_complete = sg_len; /* Complete sg-entries */ 2979 sg_len += sg->length; 2980 if (sg_len > *offset) 2981 break; 2982 } 2983 2984 if (unlikely(i == sg_count)) { 2985 printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, " 2986 "elements %d\n", 2987 __func__, sg_len, *offset, sg_count); 2988 WARN_ON(1); 2989 return NULL; 2990 } 2991 2992 /* Offset starting from the beginning of first page in this sg-entry */ 2993 *offset = *offset - len_complete + sg->offset; 2994 2995 /* Assumption: contiguous pages can be accessed as "page + i" */ 2996 page = nth_page(sg_page(sg), (*offset >> PAGE_SHIFT)); 2997 *offset &= ~PAGE_MASK; 2998 2999 /* Bytes in this sg-entry from *offset to the end of the page */ 3000 sg_len = PAGE_SIZE - *offset; 3001 if (*len > sg_len) 3002 *len = sg_len; 3003 3004 return kmap_atomic(page); 3005 } 3006 EXPORT_SYMBOL(scsi_kmap_atomic_sg); 3007 3008 /** 3009 * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously mapped with scsi_kmap_atomic_sg 3010 * @virt: virtual address to be unmapped 3011 */ 3012 void scsi_kunmap_atomic_sg(void *virt) 3013 { 3014 kunmap_atomic(virt); 3015 } 3016 EXPORT_SYMBOL(scsi_kunmap_atomic_sg); 3017 3018 void sdev_disable_disk_events(struct scsi_device *sdev) 3019 { 3020 atomic_inc(&sdev->disk_events_disable_depth); 3021 } 3022 EXPORT_SYMBOL(sdev_disable_disk_events); 3023 3024 void sdev_enable_disk_events(struct scsi_device *sdev) 3025 { 3026 if (WARN_ON_ONCE(atomic_read(&sdev->disk_events_disable_depth) <= 0)) 3027 return; 3028 atomic_dec(&sdev->disk_events_disable_depth); 3029 } 3030 EXPORT_SYMBOL(sdev_enable_disk_events); 3031 3032 /** 3033 * scsi_vpd_lun_id - return a unique device identification 3034 * @sdev: SCSI device 3035 * @id: buffer for the identification 3036 * @id_len: length of the buffer 3037 * 3038 * Copies a unique device identification into @id based 3039 * on the information in the VPD page 0x83 of the device. 3040 * The string will be formatted as a SCSI name string. 3041 * 3042 * Returns the length of the identification or error on failure. 3043 * If the identifier is longer than the supplied buffer the actual 3044 * identifier length is returned and the buffer is not zero-padded. 3045 */ 3046 int scsi_vpd_lun_id(struct scsi_device *sdev, char *id, size_t id_len) 3047 { 3048 u8 cur_id_type = 0xff; 3049 u8 cur_id_size = 0; 3050 unsigned char *d, *cur_id_str; 3051 unsigned char __rcu *vpd_pg83; 3052 int id_size = -EINVAL; 3053 3054 rcu_read_lock(); 3055 vpd_pg83 = rcu_dereference(sdev->vpd_pg83); 3056 if (!vpd_pg83) { 3057 rcu_read_unlock(); 3058 return -ENXIO; 3059 } 3060 3061 /* 3062 * Look for the correct descriptor. 3063 * Order of preference for lun descriptor: 3064 * - SCSI name string 3065 * - NAA IEEE Registered Extended 3066 * - EUI-64 based 16-byte 3067 * - EUI-64 based 12-byte 3068 * - NAA IEEE Registered 3069 * - NAA IEEE Extended 3070 * - T10 Vendor ID 3071 * as longer descriptors reduce the likelyhood 3072 * of identification clashes. 3073 */ 3074 3075 /* The id string must be at least 20 bytes + terminating NULL byte */ 3076 if (id_len < 21) { 3077 rcu_read_unlock(); 3078 return -EINVAL; 3079 } 3080 3081 memset(id, 0, id_len); 3082 d = vpd_pg83 + 4; 3083 while (d < vpd_pg83 + sdev->vpd_pg83_len) { 3084 /* Skip designators not referring to the LUN */ 3085 if ((d[1] & 0x30) != 0x00) 3086 goto next_desig; 3087 3088 switch (d[1] & 0xf) { 3089 case 0x1: 3090 /* T10 Vendor ID */ 3091 if (cur_id_size > d[3]) 3092 break; 3093 /* Prefer anything */ 3094 if (cur_id_type > 0x01 && cur_id_type != 0xff) 3095 break; 3096 cur_id_size = d[3]; 3097 if (cur_id_size + 4 > id_len) 3098 cur_id_size = id_len - 4; 3099 cur_id_str = d + 4; 3100 cur_id_type = d[1] & 0xf; 3101 id_size = snprintf(id, id_len, "t10.%*pE", 3102 cur_id_size, cur_id_str); 3103 break; 3104 case 0x2: 3105 /* EUI-64 */ 3106 if (cur_id_size > d[3]) 3107 break; 3108 /* Prefer NAA IEEE Registered Extended */ 3109 if (cur_id_type == 0x3 && 3110 cur_id_size == d[3]) 3111 break; 3112 cur_id_size = d[3]; 3113 cur_id_str = d + 4; 3114 cur_id_type = d[1] & 0xf; 3115 switch (cur_id_size) { 3116 case 8: 3117 id_size = snprintf(id, id_len, 3118 "eui.%8phN", 3119 cur_id_str); 3120 break; 3121 case 12: 3122 id_size = snprintf(id, id_len, 3123 "eui.%12phN", 3124 cur_id_str); 3125 break; 3126 case 16: 3127 id_size = snprintf(id, id_len, 3128 "eui.%16phN", 3129 cur_id_str); 3130 break; 3131 default: 3132 cur_id_size = 0; 3133 break; 3134 } 3135 break; 3136 case 0x3: 3137 /* NAA */ 3138 if (cur_id_size > d[3]) 3139 break; 3140 cur_id_size = d[3]; 3141 cur_id_str = d + 4; 3142 cur_id_type = d[1] & 0xf; 3143 switch (cur_id_size) { 3144 case 8: 3145 id_size = snprintf(id, id_len, 3146 "naa.%8phN", 3147 cur_id_str); 3148 break; 3149 case 16: 3150 id_size = snprintf(id, id_len, 3151 "naa.%16phN", 3152 cur_id_str); 3153 break; 3154 default: 3155 cur_id_size = 0; 3156 break; 3157 } 3158 break; 3159 case 0x8: 3160 /* SCSI name string */ 3161 if (cur_id_size + 4 > d[3]) 3162 break; 3163 /* Prefer others for truncated descriptor */ 3164 if (cur_id_size && d[3] > id_len) 3165 break; 3166 cur_id_size = id_size = d[3]; 3167 cur_id_str = d + 4; 3168 cur_id_type = d[1] & 0xf; 3169 if (cur_id_size >= id_len) 3170 cur_id_size = id_len - 1; 3171 memcpy(id, cur_id_str, cur_id_size); 3172 /* Decrease priority for truncated descriptor */ 3173 if (cur_id_size != id_size) 3174 cur_id_size = 6; 3175 break; 3176 default: 3177 break; 3178 } 3179 next_desig: 3180 d += d[3] + 4; 3181 } 3182 rcu_read_unlock(); 3183 3184 return id_size; 3185 } 3186 EXPORT_SYMBOL(scsi_vpd_lun_id); 3187 3188 /* 3189 * scsi_vpd_tpg_id - return a target port group identifier 3190 * @sdev: SCSI device 3191 * 3192 * Returns the Target Port Group identifier from the information 3193 * froom VPD page 0x83 of the device. 3194 * 3195 * Returns the identifier or error on failure. 3196 */ 3197 int scsi_vpd_tpg_id(struct scsi_device *sdev, int *rel_id) 3198 { 3199 unsigned char *d; 3200 unsigned char __rcu *vpd_pg83; 3201 int group_id = -EAGAIN, rel_port = -1; 3202 3203 rcu_read_lock(); 3204 vpd_pg83 = rcu_dereference(sdev->vpd_pg83); 3205 if (!vpd_pg83) { 3206 rcu_read_unlock(); 3207 return -ENXIO; 3208 } 3209 3210 d = sdev->vpd_pg83 + 4; 3211 while (d < sdev->vpd_pg83 + sdev->vpd_pg83_len) { 3212 switch (d[1] & 0xf) { 3213 case 0x4: 3214 /* Relative target port */ 3215 rel_port = get_unaligned_be16(&d[6]); 3216 break; 3217 case 0x5: 3218 /* Target port group */ 3219 group_id = get_unaligned_be16(&d[6]); 3220 break; 3221 default: 3222 break; 3223 } 3224 d += d[3] + 4; 3225 } 3226 rcu_read_unlock(); 3227 3228 if (group_id >= 0 && rel_id && rel_port != -1) 3229 *rel_id = rel_port; 3230 3231 return group_id; 3232 } 3233 EXPORT_SYMBOL(scsi_vpd_tpg_id); 3234