1 /* 2 * Copyright (C) 1999 Eric Youngdale 3 * Copyright (C) 2014 Christoph Hellwig 4 * 5 * SCSI queueing library. 6 * Initial versions: Eric Youngdale (eric@andante.org). 7 * Based upon conversations with large numbers 8 * of people at Linux Expo. 9 */ 10 11 #include <linux/bio.h> 12 #include <linux/bitops.h> 13 #include <linux/blkdev.h> 14 #include <linux/completion.h> 15 #include <linux/kernel.h> 16 #include <linux/export.h> 17 #include <linux/mempool.h> 18 #include <linux/slab.h> 19 #include <linux/init.h> 20 #include <linux/pci.h> 21 #include <linux/delay.h> 22 #include <linux/hardirq.h> 23 #include <linux/scatterlist.h> 24 #include <linux/blk-mq.h> 25 #include <linux/ratelimit.h> 26 27 #include <scsi/scsi.h> 28 #include <scsi/scsi_cmnd.h> 29 #include <scsi/scsi_dbg.h> 30 #include <scsi/scsi_device.h> 31 #include <scsi/scsi_driver.h> 32 #include <scsi/scsi_eh.h> 33 #include <scsi/scsi_host.h> 34 35 #include <trace/events/scsi.h> 36 37 #include "scsi_priv.h" 38 #include "scsi_logging.h" 39 40 41 #define SG_MEMPOOL_NR ARRAY_SIZE(scsi_sg_pools) 42 #define SG_MEMPOOL_SIZE 2 43 44 struct scsi_host_sg_pool { 45 size_t size; 46 char *name; 47 struct kmem_cache *slab; 48 mempool_t *pool; 49 }; 50 51 #define SP(x) { .size = x, "sgpool-" __stringify(x) } 52 #if (SCSI_MAX_SG_SEGMENTS < 32) 53 #error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater) 54 #endif 55 static struct scsi_host_sg_pool scsi_sg_pools[] = { 56 SP(8), 57 SP(16), 58 #if (SCSI_MAX_SG_SEGMENTS > 32) 59 SP(32), 60 #if (SCSI_MAX_SG_SEGMENTS > 64) 61 SP(64), 62 #if (SCSI_MAX_SG_SEGMENTS > 128) 63 SP(128), 64 #if (SCSI_MAX_SG_SEGMENTS > 256) 65 #error SCSI_MAX_SG_SEGMENTS is too large (256 MAX) 66 #endif 67 #endif 68 #endif 69 #endif 70 SP(SCSI_MAX_SG_SEGMENTS) 71 }; 72 #undef SP 73 74 struct kmem_cache *scsi_sdb_cache; 75 76 /* 77 * When to reinvoke queueing after a resource shortage. It's 3 msecs to 78 * not change behaviour from the previous unplug mechanism, experimentation 79 * may prove this needs changing. 80 */ 81 #define SCSI_QUEUE_DELAY 3 82 83 static void 84 scsi_set_blocked(struct scsi_cmnd *cmd, int reason) 85 { 86 struct Scsi_Host *host = cmd->device->host; 87 struct scsi_device *device = cmd->device; 88 struct scsi_target *starget = scsi_target(device); 89 90 /* 91 * Set the appropriate busy bit for the device/host. 92 * 93 * If the host/device isn't busy, assume that something actually 94 * completed, and that we should be able to queue a command now. 95 * 96 * Note that the prior mid-layer assumption that any host could 97 * always queue at least one command is now broken. The mid-layer 98 * will implement a user specifiable stall (see 99 * scsi_host.max_host_blocked and scsi_device.max_device_blocked) 100 * if a command is requeued with no other commands outstanding 101 * either for the device or for the host. 102 */ 103 switch (reason) { 104 case SCSI_MLQUEUE_HOST_BUSY: 105 atomic_set(&host->host_blocked, host->max_host_blocked); 106 break; 107 case SCSI_MLQUEUE_DEVICE_BUSY: 108 case SCSI_MLQUEUE_EH_RETRY: 109 atomic_set(&device->device_blocked, 110 device->max_device_blocked); 111 break; 112 case SCSI_MLQUEUE_TARGET_BUSY: 113 atomic_set(&starget->target_blocked, 114 starget->max_target_blocked); 115 break; 116 } 117 } 118 119 static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd) 120 { 121 struct scsi_device *sdev = cmd->device; 122 struct request_queue *q = cmd->request->q; 123 124 blk_mq_requeue_request(cmd->request); 125 blk_mq_kick_requeue_list(q); 126 put_device(&sdev->sdev_gendev); 127 } 128 129 /** 130 * __scsi_queue_insert - private queue insertion 131 * @cmd: The SCSI command being requeued 132 * @reason: The reason for the requeue 133 * @unbusy: Whether the queue should be unbusied 134 * 135 * This is a private queue insertion. The public interface 136 * scsi_queue_insert() always assumes the queue should be unbusied 137 * because it's always called before the completion. This function is 138 * for a requeue after completion, which should only occur in this 139 * file. 140 */ 141 static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy) 142 { 143 struct scsi_device *device = cmd->device; 144 struct request_queue *q = device->request_queue; 145 unsigned long flags; 146 147 SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd, 148 "Inserting command %p into mlqueue\n", cmd)); 149 150 scsi_set_blocked(cmd, reason); 151 152 /* 153 * Decrement the counters, since these commands are no longer 154 * active on the host/device. 155 */ 156 if (unbusy) 157 scsi_device_unbusy(device); 158 159 /* 160 * Requeue this command. It will go before all other commands 161 * that are already in the queue. Schedule requeue work under 162 * lock such that the kblockd_schedule_work() call happens 163 * before blk_cleanup_queue() finishes. 164 */ 165 cmd->result = 0; 166 if (q->mq_ops) { 167 scsi_mq_requeue_cmd(cmd); 168 return; 169 } 170 spin_lock_irqsave(q->queue_lock, flags); 171 blk_requeue_request(q, cmd->request); 172 kblockd_schedule_work(&device->requeue_work); 173 spin_unlock_irqrestore(q->queue_lock, flags); 174 } 175 176 /* 177 * Function: scsi_queue_insert() 178 * 179 * Purpose: Insert a command in the midlevel queue. 180 * 181 * Arguments: cmd - command that we are adding to queue. 182 * reason - why we are inserting command to queue. 183 * 184 * Lock status: Assumed that lock is not held upon entry. 185 * 186 * Returns: Nothing. 187 * 188 * Notes: We do this for one of two cases. Either the host is busy 189 * and it cannot accept any more commands for the time being, 190 * or the device returned QUEUE_FULL and can accept no more 191 * commands. 192 * Notes: This could be called either from an interrupt context or a 193 * normal process context. 194 */ 195 void scsi_queue_insert(struct scsi_cmnd *cmd, int reason) 196 { 197 __scsi_queue_insert(cmd, reason, 1); 198 } 199 /** 200 * scsi_execute - insert request and wait for the result 201 * @sdev: scsi device 202 * @cmd: scsi command 203 * @data_direction: data direction 204 * @buffer: data buffer 205 * @bufflen: len of buffer 206 * @sense: optional sense buffer 207 * @timeout: request timeout in seconds 208 * @retries: number of times to retry request 209 * @flags: or into request flags; 210 * @resid: optional residual length 211 * 212 * returns the req->errors value which is the scsi_cmnd result 213 * field. 214 */ 215 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd, 216 int data_direction, void *buffer, unsigned bufflen, 217 unsigned char *sense, int timeout, int retries, u64 flags, 218 int *resid) 219 { 220 struct request *req; 221 int write = (data_direction == DMA_TO_DEVICE); 222 int ret = DRIVER_ERROR << 24; 223 224 req = blk_get_request(sdev->request_queue, write, __GFP_WAIT); 225 if (IS_ERR(req)) 226 return ret; 227 blk_rq_set_block_pc(req); 228 229 if (bufflen && blk_rq_map_kern(sdev->request_queue, req, 230 buffer, bufflen, __GFP_WAIT)) 231 goto out; 232 233 req->cmd_len = COMMAND_SIZE(cmd[0]); 234 memcpy(req->cmd, cmd, req->cmd_len); 235 req->sense = sense; 236 req->sense_len = 0; 237 req->retries = retries; 238 req->timeout = timeout; 239 req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT; 240 241 /* 242 * head injection *required* here otherwise quiesce won't work 243 */ 244 blk_execute_rq(req->q, NULL, req, 1); 245 246 /* 247 * Some devices (USB mass-storage in particular) may transfer 248 * garbage data together with a residue indicating that the data 249 * is invalid. Prevent the garbage from being misinterpreted 250 * and prevent security leaks by zeroing out the excess data. 251 */ 252 if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen)) 253 memset(buffer + (bufflen - req->resid_len), 0, req->resid_len); 254 255 if (resid) 256 *resid = req->resid_len; 257 ret = req->errors; 258 out: 259 blk_put_request(req); 260 261 return ret; 262 } 263 EXPORT_SYMBOL(scsi_execute); 264 265 int scsi_execute_req_flags(struct scsi_device *sdev, const unsigned char *cmd, 266 int data_direction, void *buffer, unsigned bufflen, 267 struct scsi_sense_hdr *sshdr, int timeout, int retries, 268 int *resid, u64 flags) 269 { 270 char *sense = NULL; 271 int result; 272 273 if (sshdr) { 274 sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO); 275 if (!sense) 276 return DRIVER_ERROR << 24; 277 } 278 result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen, 279 sense, timeout, retries, flags, resid); 280 if (sshdr) 281 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr); 282 283 kfree(sense); 284 return result; 285 } 286 EXPORT_SYMBOL(scsi_execute_req_flags); 287 288 /* 289 * Function: scsi_init_cmd_errh() 290 * 291 * Purpose: Initialize cmd fields related to error handling. 292 * 293 * Arguments: cmd - command that is ready to be queued. 294 * 295 * Notes: This function has the job of initializing a number of 296 * fields related to error handling. Typically this will 297 * be called once for each command, as required. 298 */ 299 static void scsi_init_cmd_errh(struct scsi_cmnd *cmd) 300 { 301 cmd->serial_number = 0; 302 scsi_set_resid(cmd, 0); 303 memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE); 304 if (cmd->cmd_len == 0) 305 cmd->cmd_len = scsi_command_size(cmd->cmnd); 306 } 307 308 void scsi_device_unbusy(struct scsi_device *sdev) 309 { 310 struct Scsi_Host *shost = sdev->host; 311 struct scsi_target *starget = scsi_target(sdev); 312 unsigned long flags; 313 314 atomic_dec(&shost->host_busy); 315 if (starget->can_queue > 0) 316 atomic_dec(&starget->target_busy); 317 318 if (unlikely(scsi_host_in_recovery(shost) && 319 (shost->host_failed || shost->host_eh_scheduled))) { 320 spin_lock_irqsave(shost->host_lock, flags); 321 scsi_eh_wakeup(shost); 322 spin_unlock_irqrestore(shost->host_lock, flags); 323 } 324 325 atomic_dec(&sdev->device_busy); 326 } 327 328 static void scsi_kick_queue(struct request_queue *q) 329 { 330 if (q->mq_ops) 331 blk_mq_start_hw_queues(q); 332 else 333 blk_run_queue(q); 334 } 335 336 /* 337 * Called for single_lun devices on IO completion. Clear starget_sdev_user, 338 * and call blk_run_queue for all the scsi_devices on the target - 339 * including current_sdev first. 340 * 341 * Called with *no* scsi locks held. 342 */ 343 static void scsi_single_lun_run(struct scsi_device *current_sdev) 344 { 345 struct Scsi_Host *shost = current_sdev->host; 346 struct scsi_device *sdev, *tmp; 347 struct scsi_target *starget = scsi_target(current_sdev); 348 unsigned long flags; 349 350 spin_lock_irqsave(shost->host_lock, flags); 351 starget->starget_sdev_user = NULL; 352 spin_unlock_irqrestore(shost->host_lock, flags); 353 354 /* 355 * Call blk_run_queue for all LUNs on the target, starting with 356 * current_sdev. We race with others (to set starget_sdev_user), 357 * but in most cases, we will be first. Ideally, each LU on the 358 * target would get some limited time or requests on the target. 359 */ 360 scsi_kick_queue(current_sdev->request_queue); 361 362 spin_lock_irqsave(shost->host_lock, flags); 363 if (starget->starget_sdev_user) 364 goto out; 365 list_for_each_entry_safe(sdev, tmp, &starget->devices, 366 same_target_siblings) { 367 if (sdev == current_sdev) 368 continue; 369 if (scsi_device_get(sdev)) 370 continue; 371 372 spin_unlock_irqrestore(shost->host_lock, flags); 373 scsi_kick_queue(sdev->request_queue); 374 spin_lock_irqsave(shost->host_lock, flags); 375 376 scsi_device_put(sdev); 377 } 378 out: 379 spin_unlock_irqrestore(shost->host_lock, flags); 380 } 381 382 static inline bool scsi_device_is_busy(struct scsi_device *sdev) 383 { 384 if (atomic_read(&sdev->device_busy) >= sdev->queue_depth) 385 return true; 386 if (atomic_read(&sdev->device_blocked) > 0) 387 return true; 388 return false; 389 } 390 391 static inline bool scsi_target_is_busy(struct scsi_target *starget) 392 { 393 if (starget->can_queue > 0) { 394 if (atomic_read(&starget->target_busy) >= starget->can_queue) 395 return true; 396 if (atomic_read(&starget->target_blocked) > 0) 397 return true; 398 } 399 return false; 400 } 401 402 static inline bool scsi_host_is_busy(struct Scsi_Host *shost) 403 { 404 if (shost->can_queue > 0 && 405 atomic_read(&shost->host_busy) >= shost->can_queue) 406 return true; 407 if (atomic_read(&shost->host_blocked) > 0) 408 return true; 409 if (shost->host_self_blocked) 410 return true; 411 return false; 412 } 413 414 static void scsi_starved_list_run(struct Scsi_Host *shost) 415 { 416 LIST_HEAD(starved_list); 417 struct scsi_device *sdev; 418 unsigned long flags; 419 420 spin_lock_irqsave(shost->host_lock, flags); 421 list_splice_init(&shost->starved_list, &starved_list); 422 423 while (!list_empty(&starved_list)) { 424 struct request_queue *slq; 425 426 /* 427 * As long as shost is accepting commands and we have 428 * starved queues, call blk_run_queue. scsi_request_fn 429 * drops the queue_lock and can add us back to the 430 * starved_list. 431 * 432 * host_lock protects the starved_list and starved_entry. 433 * scsi_request_fn must get the host_lock before checking 434 * or modifying starved_list or starved_entry. 435 */ 436 if (scsi_host_is_busy(shost)) 437 break; 438 439 sdev = list_entry(starved_list.next, 440 struct scsi_device, starved_entry); 441 list_del_init(&sdev->starved_entry); 442 if (scsi_target_is_busy(scsi_target(sdev))) { 443 list_move_tail(&sdev->starved_entry, 444 &shost->starved_list); 445 continue; 446 } 447 448 /* 449 * Once we drop the host lock, a racing scsi_remove_device() 450 * call may remove the sdev from the starved list and destroy 451 * it and the queue. Mitigate by taking a reference to the 452 * queue and never touching the sdev again after we drop the 453 * host lock. Note: if __scsi_remove_device() invokes 454 * blk_cleanup_queue() before the queue is run from this 455 * function then blk_run_queue() will return immediately since 456 * blk_cleanup_queue() marks the queue with QUEUE_FLAG_DYING. 457 */ 458 slq = sdev->request_queue; 459 if (!blk_get_queue(slq)) 460 continue; 461 spin_unlock_irqrestore(shost->host_lock, flags); 462 463 scsi_kick_queue(slq); 464 blk_put_queue(slq); 465 466 spin_lock_irqsave(shost->host_lock, flags); 467 } 468 /* put any unprocessed entries back */ 469 list_splice(&starved_list, &shost->starved_list); 470 spin_unlock_irqrestore(shost->host_lock, flags); 471 } 472 473 /* 474 * Function: scsi_run_queue() 475 * 476 * Purpose: Select a proper request queue to serve next 477 * 478 * Arguments: q - last request's queue 479 * 480 * Returns: Nothing 481 * 482 * Notes: The previous command was completely finished, start 483 * a new one if possible. 484 */ 485 static void scsi_run_queue(struct request_queue *q) 486 { 487 struct scsi_device *sdev = q->queuedata; 488 489 if (scsi_target(sdev)->single_lun) 490 scsi_single_lun_run(sdev); 491 if (!list_empty(&sdev->host->starved_list)) 492 scsi_starved_list_run(sdev->host); 493 494 if (q->mq_ops) 495 blk_mq_start_stopped_hw_queues(q, false); 496 else 497 blk_run_queue(q); 498 } 499 500 void scsi_requeue_run_queue(struct work_struct *work) 501 { 502 struct scsi_device *sdev; 503 struct request_queue *q; 504 505 sdev = container_of(work, struct scsi_device, requeue_work); 506 q = sdev->request_queue; 507 scsi_run_queue(q); 508 } 509 510 /* 511 * Function: scsi_requeue_command() 512 * 513 * Purpose: Handle post-processing of completed commands. 514 * 515 * Arguments: q - queue to operate on 516 * cmd - command that may need to be requeued. 517 * 518 * Returns: Nothing 519 * 520 * Notes: After command completion, there may be blocks left 521 * over which weren't finished by the previous command 522 * this can be for a number of reasons - the main one is 523 * I/O errors in the middle of the request, in which case 524 * we need to request the blocks that come after the bad 525 * sector. 526 * Notes: Upon return, cmd is a stale pointer. 527 */ 528 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd) 529 { 530 struct scsi_device *sdev = cmd->device; 531 struct request *req = cmd->request; 532 unsigned long flags; 533 534 spin_lock_irqsave(q->queue_lock, flags); 535 blk_unprep_request(req); 536 req->special = NULL; 537 scsi_put_command(cmd); 538 blk_requeue_request(q, req); 539 spin_unlock_irqrestore(q->queue_lock, flags); 540 541 scsi_run_queue(q); 542 543 put_device(&sdev->sdev_gendev); 544 } 545 546 void scsi_run_host_queues(struct Scsi_Host *shost) 547 { 548 struct scsi_device *sdev; 549 550 shost_for_each_device(sdev, shost) 551 scsi_run_queue(sdev->request_queue); 552 } 553 554 static inline unsigned int scsi_sgtable_index(unsigned short nents) 555 { 556 unsigned int index; 557 558 BUG_ON(nents > SCSI_MAX_SG_SEGMENTS); 559 560 if (nents <= 8) 561 index = 0; 562 else 563 index = get_count_order(nents) - 3; 564 565 return index; 566 } 567 568 static void scsi_sg_free(struct scatterlist *sgl, unsigned int nents) 569 { 570 struct scsi_host_sg_pool *sgp; 571 572 sgp = scsi_sg_pools + scsi_sgtable_index(nents); 573 mempool_free(sgl, sgp->pool); 574 } 575 576 static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask) 577 { 578 struct scsi_host_sg_pool *sgp; 579 580 sgp = scsi_sg_pools + scsi_sgtable_index(nents); 581 return mempool_alloc(sgp->pool, gfp_mask); 582 } 583 584 static void scsi_free_sgtable(struct scsi_data_buffer *sdb, bool mq) 585 { 586 if (mq && sdb->table.nents <= SCSI_MAX_SG_SEGMENTS) 587 return; 588 __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, mq, scsi_sg_free); 589 } 590 591 static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents, bool mq) 592 { 593 struct scatterlist *first_chunk = NULL; 594 gfp_t gfp_mask = mq ? GFP_NOIO : GFP_ATOMIC; 595 int ret; 596 597 BUG_ON(!nents); 598 599 if (mq) { 600 if (nents <= SCSI_MAX_SG_SEGMENTS) { 601 sdb->table.nents = nents; 602 sg_init_table(sdb->table.sgl, sdb->table.nents); 603 return 0; 604 } 605 first_chunk = sdb->table.sgl; 606 } 607 608 ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS, 609 first_chunk, gfp_mask, scsi_sg_alloc); 610 if (unlikely(ret)) 611 scsi_free_sgtable(sdb, mq); 612 return ret; 613 } 614 615 static void scsi_uninit_cmd(struct scsi_cmnd *cmd) 616 { 617 if (cmd->request->cmd_type == REQ_TYPE_FS) { 618 struct scsi_driver *drv = scsi_cmd_to_driver(cmd); 619 620 if (drv->uninit_command) 621 drv->uninit_command(cmd); 622 } 623 } 624 625 static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd) 626 { 627 if (cmd->sdb.table.nents) 628 scsi_free_sgtable(&cmd->sdb, true); 629 if (cmd->request->next_rq && cmd->request->next_rq->special) 630 scsi_free_sgtable(cmd->request->next_rq->special, true); 631 if (scsi_prot_sg_count(cmd)) 632 scsi_free_sgtable(cmd->prot_sdb, true); 633 } 634 635 static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd) 636 { 637 struct scsi_device *sdev = cmd->device; 638 struct Scsi_Host *shost = sdev->host; 639 unsigned long flags; 640 641 scsi_mq_free_sgtables(cmd); 642 scsi_uninit_cmd(cmd); 643 644 if (shost->use_cmd_list) { 645 BUG_ON(list_empty(&cmd->list)); 646 spin_lock_irqsave(&sdev->list_lock, flags); 647 list_del_init(&cmd->list); 648 spin_unlock_irqrestore(&sdev->list_lock, flags); 649 } 650 } 651 652 /* 653 * Function: scsi_release_buffers() 654 * 655 * Purpose: Free resources allocate for a scsi_command. 656 * 657 * Arguments: cmd - command that we are bailing. 658 * 659 * Lock status: Assumed that no lock is held upon entry. 660 * 661 * Returns: Nothing 662 * 663 * Notes: In the event that an upper level driver rejects a 664 * command, we must release resources allocated during 665 * the __init_io() function. Primarily this would involve 666 * the scatter-gather table. 667 */ 668 static void scsi_release_buffers(struct scsi_cmnd *cmd) 669 { 670 if (cmd->sdb.table.nents) 671 scsi_free_sgtable(&cmd->sdb, false); 672 673 memset(&cmd->sdb, 0, sizeof(cmd->sdb)); 674 675 if (scsi_prot_sg_count(cmd)) 676 scsi_free_sgtable(cmd->prot_sdb, false); 677 } 678 679 static void scsi_release_bidi_buffers(struct scsi_cmnd *cmd) 680 { 681 struct scsi_data_buffer *bidi_sdb = cmd->request->next_rq->special; 682 683 scsi_free_sgtable(bidi_sdb, false); 684 kmem_cache_free(scsi_sdb_cache, bidi_sdb); 685 cmd->request->next_rq->special = NULL; 686 } 687 688 static bool scsi_end_request(struct request *req, int error, 689 unsigned int bytes, unsigned int bidi_bytes) 690 { 691 struct scsi_cmnd *cmd = req->special; 692 struct scsi_device *sdev = cmd->device; 693 struct request_queue *q = sdev->request_queue; 694 695 if (blk_update_request(req, error, bytes)) 696 return true; 697 698 /* Bidi request must be completed as a whole */ 699 if (unlikely(bidi_bytes) && 700 blk_update_request(req->next_rq, error, bidi_bytes)) 701 return true; 702 703 if (blk_queue_add_random(q)) 704 add_disk_randomness(req->rq_disk); 705 706 if (req->mq_ctx) { 707 /* 708 * In the MQ case the command gets freed by __blk_mq_end_request, 709 * so we have to do all cleanup that depends on it earlier. 710 * 711 * We also can't kick the queues from irq context, so we 712 * will have to defer it to a workqueue. 713 */ 714 scsi_mq_uninit_cmd(cmd); 715 716 __blk_mq_end_request(req, error); 717 718 if (scsi_target(sdev)->single_lun || 719 !list_empty(&sdev->host->starved_list)) 720 kblockd_schedule_work(&sdev->requeue_work); 721 else 722 blk_mq_start_stopped_hw_queues(q, true); 723 } else { 724 unsigned long flags; 725 726 if (bidi_bytes) 727 scsi_release_bidi_buffers(cmd); 728 729 spin_lock_irqsave(q->queue_lock, flags); 730 blk_finish_request(req, error); 731 spin_unlock_irqrestore(q->queue_lock, flags); 732 733 scsi_release_buffers(cmd); 734 735 scsi_put_command(cmd); 736 scsi_run_queue(q); 737 } 738 739 put_device(&sdev->sdev_gendev); 740 return false; 741 } 742 743 /** 744 * __scsi_error_from_host_byte - translate SCSI error code into errno 745 * @cmd: SCSI command (unused) 746 * @result: scsi error code 747 * 748 * Translate SCSI error code into standard UNIX errno. 749 * Return values: 750 * -ENOLINK temporary transport failure 751 * -EREMOTEIO permanent target failure, do not retry 752 * -EBADE permanent nexus failure, retry on other path 753 * -ENOSPC No write space available 754 * -ENODATA Medium error 755 * -EIO unspecified I/O error 756 */ 757 static int __scsi_error_from_host_byte(struct scsi_cmnd *cmd, int result) 758 { 759 int error = 0; 760 761 switch(host_byte(result)) { 762 case DID_TRANSPORT_FAILFAST: 763 error = -ENOLINK; 764 break; 765 case DID_TARGET_FAILURE: 766 set_host_byte(cmd, DID_OK); 767 error = -EREMOTEIO; 768 break; 769 case DID_NEXUS_FAILURE: 770 set_host_byte(cmd, DID_OK); 771 error = -EBADE; 772 break; 773 case DID_ALLOC_FAILURE: 774 set_host_byte(cmd, DID_OK); 775 error = -ENOSPC; 776 break; 777 case DID_MEDIUM_ERROR: 778 set_host_byte(cmd, DID_OK); 779 error = -ENODATA; 780 break; 781 default: 782 error = -EIO; 783 break; 784 } 785 786 return error; 787 } 788 789 /* 790 * Function: scsi_io_completion() 791 * 792 * Purpose: Completion processing for block device I/O requests. 793 * 794 * Arguments: cmd - command that is finished. 795 * 796 * Lock status: Assumed that no lock is held upon entry. 797 * 798 * Returns: Nothing 799 * 800 * Notes: We will finish off the specified number of sectors. If we 801 * are done, the command block will be released and the queue 802 * function will be goosed. If we are not done then we have to 803 * figure out what to do next: 804 * 805 * a) We can call scsi_requeue_command(). The request 806 * will be unprepared and put back on the queue. Then 807 * a new command will be created for it. This should 808 * be used if we made forward progress, or if we want 809 * to switch from READ(10) to READ(6) for example. 810 * 811 * b) We can call __scsi_queue_insert(). The request will 812 * be put back on the queue and retried using the same 813 * command as before, possibly after a delay. 814 * 815 * c) We can call scsi_end_request() with -EIO to fail 816 * the remainder of the request. 817 */ 818 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes) 819 { 820 int result = cmd->result; 821 struct request_queue *q = cmd->device->request_queue; 822 struct request *req = cmd->request; 823 int error = 0; 824 struct scsi_sense_hdr sshdr; 825 bool sense_valid = false; 826 int sense_deferred = 0, level = 0; 827 enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY, 828 ACTION_DELAYED_RETRY} action; 829 unsigned long wait_for = (cmd->allowed + 1) * req->timeout; 830 831 if (result) { 832 sense_valid = scsi_command_normalize_sense(cmd, &sshdr); 833 if (sense_valid) 834 sense_deferred = scsi_sense_is_deferred(&sshdr); 835 } 836 837 if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */ 838 if (result) { 839 if (sense_valid && req->sense) { 840 /* 841 * SG_IO wants current and deferred errors 842 */ 843 int len = 8 + cmd->sense_buffer[7]; 844 845 if (len > SCSI_SENSE_BUFFERSIZE) 846 len = SCSI_SENSE_BUFFERSIZE; 847 memcpy(req->sense, cmd->sense_buffer, len); 848 req->sense_len = len; 849 } 850 if (!sense_deferred) 851 error = __scsi_error_from_host_byte(cmd, result); 852 } 853 /* 854 * __scsi_error_from_host_byte may have reset the host_byte 855 */ 856 req->errors = cmd->result; 857 858 req->resid_len = scsi_get_resid(cmd); 859 860 if (scsi_bidi_cmnd(cmd)) { 861 /* 862 * Bidi commands Must be complete as a whole, 863 * both sides at once. 864 */ 865 req->next_rq->resid_len = scsi_in(cmd)->resid; 866 if (scsi_end_request(req, 0, blk_rq_bytes(req), 867 blk_rq_bytes(req->next_rq))) 868 BUG(); 869 return; 870 } 871 } else if (blk_rq_bytes(req) == 0 && result && !sense_deferred) { 872 /* 873 * Certain non BLOCK_PC requests are commands that don't 874 * actually transfer anything (FLUSH), so cannot use 875 * good_bytes != blk_rq_bytes(req) as the signal for an error. 876 * This sets the error explicitly for the problem case. 877 */ 878 error = __scsi_error_from_host_byte(cmd, result); 879 } 880 881 /* no bidi support for !REQ_TYPE_BLOCK_PC yet */ 882 BUG_ON(blk_bidi_rq(req)); 883 884 /* 885 * Next deal with any sectors which we were able to correctly 886 * handle. 887 */ 888 SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, cmd, 889 "%u sectors total, %d bytes done.\n", 890 blk_rq_sectors(req), good_bytes)); 891 892 /* 893 * Recovered errors need reporting, but they're always treated 894 * as success, so fiddle the result code here. For BLOCK_PC 895 * we already took a copy of the original into rq->errors which 896 * is what gets returned to the user 897 */ 898 if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) { 899 /* if ATA PASS-THROUGH INFORMATION AVAILABLE skip 900 * print since caller wants ATA registers. Only occurs on 901 * SCSI ATA PASS_THROUGH commands when CK_COND=1 902 */ 903 if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d)) 904 ; 905 else if (!(req->cmd_flags & REQ_QUIET)) 906 scsi_print_sense(cmd); 907 result = 0; 908 /* BLOCK_PC may have set error */ 909 error = 0; 910 } 911 912 /* 913 * If we finished all bytes in the request we are done now. 914 */ 915 if (!scsi_end_request(req, error, good_bytes, 0)) 916 return; 917 918 /* 919 * Kill remainder if no retrys. 920 */ 921 if (error && scsi_noretry_cmd(cmd)) { 922 if (scsi_end_request(req, error, blk_rq_bytes(req), 0)) 923 BUG(); 924 return; 925 } 926 927 /* 928 * If there had been no error, but we have leftover bytes in the 929 * requeues just queue the command up again. 930 */ 931 if (result == 0) 932 goto requeue; 933 934 error = __scsi_error_from_host_byte(cmd, result); 935 936 if (host_byte(result) == DID_RESET) { 937 /* Third party bus reset or reset for error recovery 938 * reasons. Just retry the command and see what 939 * happens. 940 */ 941 action = ACTION_RETRY; 942 } else if (sense_valid && !sense_deferred) { 943 switch (sshdr.sense_key) { 944 case UNIT_ATTENTION: 945 if (cmd->device->removable) { 946 /* Detected disc change. Set a bit 947 * and quietly refuse further access. 948 */ 949 cmd->device->changed = 1; 950 action = ACTION_FAIL; 951 } else { 952 /* Must have been a power glitch, or a 953 * bus reset. Could not have been a 954 * media change, so we just retry the 955 * command and see what happens. 956 */ 957 action = ACTION_RETRY; 958 } 959 break; 960 case ILLEGAL_REQUEST: 961 /* If we had an ILLEGAL REQUEST returned, then 962 * we may have performed an unsupported 963 * command. The only thing this should be 964 * would be a ten byte read where only a six 965 * byte read was supported. Also, on a system 966 * where READ CAPACITY failed, we may have 967 * read past the end of the disk. 968 */ 969 if ((cmd->device->use_10_for_rw && 970 sshdr.asc == 0x20 && sshdr.ascq == 0x00) && 971 (cmd->cmnd[0] == READ_10 || 972 cmd->cmnd[0] == WRITE_10)) { 973 /* This will issue a new 6-byte command. */ 974 cmd->device->use_10_for_rw = 0; 975 action = ACTION_REPREP; 976 } else if (sshdr.asc == 0x10) /* DIX */ { 977 action = ACTION_FAIL; 978 error = -EILSEQ; 979 /* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */ 980 } else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) { 981 action = ACTION_FAIL; 982 error = -EREMOTEIO; 983 } else 984 action = ACTION_FAIL; 985 break; 986 case ABORTED_COMMAND: 987 action = ACTION_FAIL; 988 if (sshdr.asc == 0x10) /* DIF */ 989 error = -EILSEQ; 990 break; 991 case NOT_READY: 992 /* If the device is in the process of becoming 993 * ready, or has a temporary blockage, retry. 994 */ 995 if (sshdr.asc == 0x04) { 996 switch (sshdr.ascq) { 997 case 0x01: /* becoming ready */ 998 case 0x04: /* format in progress */ 999 case 0x05: /* rebuild in progress */ 1000 case 0x06: /* recalculation in progress */ 1001 case 0x07: /* operation in progress */ 1002 case 0x08: /* Long write in progress */ 1003 case 0x09: /* self test in progress */ 1004 case 0x14: /* space allocation in progress */ 1005 action = ACTION_DELAYED_RETRY; 1006 break; 1007 default: 1008 action = ACTION_FAIL; 1009 break; 1010 } 1011 } else 1012 action = ACTION_FAIL; 1013 break; 1014 case VOLUME_OVERFLOW: 1015 /* See SSC3rXX or current. */ 1016 action = ACTION_FAIL; 1017 break; 1018 default: 1019 action = ACTION_FAIL; 1020 break; 1021 } 1022 } else 1023 action = ACTION_FAIL; 1024 1025 if (action != ACTION_FAIL && 1026 time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) 1027 action = ACTION_FAIL; 1028 1029 switch (action) { 1030 case ACTION_FAIL: 1031 /* Give up and fail the remainder of the request */ 1032 if (!(req->cmd_flags & REQ_QUIET)) { 1033 static DEFINE_RATELIMIT_STATE(_rs, 1034 DEFAULT_RATELIMIT_INTERVAL, 1035 DEFAULT_RATELIMIT_BURST); 1036 1037 if (unlikely(scsi_logging_level)) 1038 level = SCSI_LOG_LEVEL(SCSI_LOG_MLCOMPLETE_SHIFT, 1039 SCSI_LOG_MLCOMPLETE_BITS); 1040 1041 /* 1042 * if logging is enabled the failure will be printed 1043 * in scsi_log_completion(), so avoid duplicate messages 1044 */ 1045 if (!level && __ratelimit(&_rs)) { 1046 scsi_print_result(cmd, NULL, FAILED); 1047 if (driver_byte(result) & DRIVER_SENSE) 1048 scsi_print_sense(cmd); 1049 scsi_print_command(cmd); 1050 } 1051 } 1052 if (!scsi_end_request(req, error, blk_rq_err_bytes(req), 0)) 1053 return; 1054 /*FALLTHRU*/ 1055 case ACTION_REPREP: 1056 requeue: 1057 /* Unprep the request and put it back at the head of the queue. 1058 * A new command will be prepared and issued. 1059 */ 1060 if (q->mq_ops) { 1061 cmd->request->cmd_flags &= ~REQ_DONTPREP; 1062 scsi_mq_uninit_cmd(cmd); 1063 scsi_mq_requeue_cmd(cmd); 1064 } else { 1065 scsi_release_buffers(cmd); 1066 scsi_requeue_command(q, cmd); 1067 } 1068 break; 1069 case ACTION_RETRY: 1070 /* Retry the same command immediately */ 1071 __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0); 1072 break; 1073 case ACTION_DELAYED_RETRY: 1074 /* Retry the same command after a delay */ 1075 __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0); 1076 break; 1077 } 1078 } 1079 1080 static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb) 1081 { 1082 int count; 1083 1084 /* 1085 * If sg table allocation fails, requeue request later. 1086 */ 1087 if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments, 1088 req->mq_ctx != NULL))) 1089 return BLKPREP_DEFER; 1090 1091 /* 1092 * Next, walk the list, and fill in the addresses and sizes of 1093 * each segment. 1094 */ 1095 count = blk_rq_map_sg(req->q, req, sdb->table.sgl); 1096 BUG_ON(count > sdb->table.nents); 1097 sdb->table.nents = count; 1098 sdb->length = blk_rq_bytes(req); 1099 return BLKPREP_OK; 1100 } 1101 1102 /* 1103 * Function: scsi_init_io() 1104 * 1105 * Purpose: SCSI I/O initialize function. 1106 * 1107 * Arguments: cmd - Command descriptor we wish to initialize 1108 * 1109 * Returns: 0 on success 1110 * BLKPREP_DEFER if the failure is retryable 1111 * BLKPREP_KILL if the failure is fatal 1112 */ 1113 int scsi_init_io(struct scsi_cmnd *cmd) 1114 { 1115 struct scsi_device *sdev = cmd->device; 1116 struct request *rq = cmd->request; 1117 bool is_mq = (rq->mq_ctx != NULL); 1118 int error; 1119 1120 BUG_ON(!rq->nr_phys_segments); 1121 1122 error = scsi_init_sgtable(rq, &cmd->sdb); 1123 if (error) 1124 goto err_exit; 1125 1126 if (blk_bidi_rq(rq)) { 1127 if (!rq->q->mq_ops) { 1128 struct scsi_data_buffer *bidi_sdb = 1129 kmem_cache_zalloc(scsi_sdb_cache, GFP_ATOMIC); 1130 if (!bidi_sdb) { 1131 error = BLKPREP_DEFER; 1132 goto err_exit; 1133 } 1134 1135 rq->next_rq->special = bidi_sdb; 1136 } 1137 1138 error = scsi_init_sgtable(rq->next_rq, rq->next_rq->special); 1139 if (error) 1140 goto err_exit; 1141 } 1142 1143 if (blk_integrity_rq(rq)) { 1144 struct scsi_data_buffer *prot_sdb = cmd->prot_sdb; 1145 int ivecs, count; 1146 1147 BUG_ON(prot_sdb == NULL); 1148 ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio); 1149 1150 if (scsi_alloc_sgtable(prot_sdb, ivecs, is_mq)) { 1151 error = BLKPREP_DEFER; 1152 goto err_exit; 1153 } 1154 1155 count = blk_rq_map_integrity_sg(rq->q, rq->bio, 1156 prot_sdb->table.sgl); 1157 BUG_ON(unlikely(count > ivecs)); 1158 BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q))); 1159 1160 cmd->prot_sdb = prot_sdb; 1161 cmd->prot_sdb->table.nents = count; 1162 } 1163 1164 return BLKPREP_OK; 1165 err_exit: 1166 if (is_mq) { 1167 scsi_mq_free_sgtables(cmd); 1168 } else { 1169 scsi_release_buffers(cmd); 1170 cmd->request->special = NULL; 1171 scsi_put_command(cmd); 1172 put_device(&sdev->sdev_gendev); 1173 } 1174 return error; 1175 } 1176 EXPORT_SYMBOL(scsi_init_io); 1177 1178 static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev, 1179 struct request *req) 1180 { 1181 struct scsi_cmnd *cmd; 1182 1183 if (!req->special) { 1184 /* Bail if we can't get a reference to the device */ 1185 if (!get_device(&sdev->sdev_gendev)) 1186 return NULL; 1187 1188 cmd = scsi_get_command(sdev, GFP_ATOMIC); 1189 if (unlikely(!cmd)) { 1190 put_device(&sdev->sdev_gendev); 1191 return NULL; 1192 } 1193 req->special = cmd; 1194 } else { 1195 cmd = req->special; 1196 } 1197 1198 /* pull a tag out of the request if we have one */ 1199 cmd->tag = req->tag; 1200 cmd->request = req; 1201 1202 cmd->cmnd = req->cmd; 1203 cmd->prot_op = SCSI_PROT_NORMAL; 1204 1205 return cmd; 1206 } 1207 1208 static int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req) 1209 { 1210 struct scsi_cmnd *cmd = req->special; 1211 1212 /* 1213 * BLOCK_PC requests may transfer data, in which case they must 1214 * a bio attached to them. Or they might contain a SCSI command 1215 * that does not transfer data, in which case they may optionally 1216 * submit a request without an attached bio. 1217 */ 1218 if (req->bio) { 1219 int ret = scsi_init_io(cmd); 1220 if (unlikely(ret)) 1221 return ret; 1222 } else { 1223 BUG_ON(blk_rq_bytes(req)); 1224 1225 memset(&cmd->sdb, 0, sizeof(cmd->sdb)); 1226 } 1227 1228 cmd->cmd_len = req->cmd_len; 1229 cmd->transfersize = blk_rq_bytes(req); 1230 cmd->allowed = req->retries; 1231 return BLKPREP_OK; 1232 } 1233 1234 /* 1235 * Setup a REQ_TYPE_FS command. These are simple request from filesystems 1236 * that still need to be translated to SCSI CDBs from the ULD. 1237 */ 1238 static int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req) 1239 { 1240 struct scsi_cmnd *cmd = req->special; 1241 1242 if (unlikely(sdev->scsi_dh_data && sdev->scsi_dh_data->scsi_dh 1243 && sdev->scsi_dh_data->scsi_dh->prep_fn)) { 1244 int ret = sdev->scsi_dh_data->scsi_dh->prep_fn(sdev, req); 1245 if (ret != BLKPREP_OK) 1246 return ret; 1247 } 1248 1249 memset(cmd->cmnd, 0, BLK_MAX_CDB); 1250 return scsi_cmd_to_driver(cmd)->init_command(cmd); 1251 } 1252 1253 static int scsi_setup_cmnd(struct scsi_device *sdev, struct request *req) 1254 { 1255 struct scsi_cmnd *cmd = req->special; 1256 1257 if (!blk_rq_bytes(req)) 1258 cmd->sc_data_direction = DMA_NONE; 1259 else if (rq_data_dir(req) == WRITE) 1260 cmd->sc_data_direction = DMA_TO_DEVICE; 1261 else 1262 cmd->sc_data_direction = DMA_FROM_DEVICE; 1263 1264 switch (req->cmd_type) { 1265 case REQ_TYPE_FS: 1266 return scsi_setup_fs_cmnd(sdev, req); 1267 case REQ_TYPE_BLOCK_PC: 1268 return scsi_setup_blk_pc_cmnd(sdev, req); 1269 default: 1270 return BLKPREP_KILL; 1271 } 1272 } 1273 1274 static int 1275 scsi_prep_state_check(struct scsi_device *sdev, struct request *req) 1276 { 1277 int ret = BLKPREP_OK; 1278 1279 /* 1280 * If the device is not in running state we will reject some 1281 * or all commands. 1282 */ 1283 if (unlikely(sdev->sdev_state != SDEV_RUNNING)) { 1284 switch (sdev->sdev_state) { 1285 case SDEV_OFFLINE: 1286 case SDEV_TRANSPORT_OFFLINE: 1287 /* 1288 * If the device is offline we refuse to process any 1289 * commands. The device must be brought online 1290 * before trying any recovery commands. 1291 */ 1292 sdev_printk(KERN_ERR, sdev, 1293 "rejecting I/O to offline device\n"); 1294 ret = BLKPREP_KILL; 1295 break; 1296 case SDEV_DEL: 1297 /* 1298 * If the device is fully deleted, we refuse to 1299 * process any commands as well. 1300 */ 1301 sdev_printk(KERN_ERR, sdev, 1302 "rejecting I/O to dead device\n"); 1303 ret = BLKPREP_KILL; 1304 break; 1305 case SDEV_QUIESCE: 1306 case SDEV_BLOCK: 1307 case SDEV_CREATED_BLOCK: 1308 /* 1309 * If the devices is blocked we defer normal commands. 1310 */ 1311 if (!(req->cmd_flags & REQ_PREEMPT)) 1312 ret = BLKPREP_DEFER; 1313 break; 1314 default: 1315 /* 1316 * For any other not fully online state we only allow 1317 * special commands. In particular any user initiated 1318 * command is not allowed. 1319 */ 1320 if (!(req->cmd_flags & REQ_PREEMPT)) 1321 ret = BLKPREP_KILL; 1322 break; 1323 } 1324 } 1325 return ret; 1326 } 1327 1328 static int 1329 scsi_prep_return(struct request_queue *q, struct request *req, int ret) 1330 { 1331 struct scsi_device *sdev = q->queuedata; 1332 1333 switch (ret) { 1334 case BLKPREP_KILL: 1335 req->errors = DID_NO_CONNECT << 16; 1336 /* release the command and kill it */ 1337 if (req->special) { 1338 struct scsi_cmnd *cmd = req->special; 1339 scsi_release_buffers(cmd); 1340 scsi_put_command(cmd); 1341 put_device(&sdev->sdev_gendev); 1342 req->special = NULL; 1343 } 1344 break; 1345 case BLKPREP_DEFER: 1346 /* 1347 * If we defer, the blk_peek_request() returns NULL, but the 1348 * queue must be restarted, so we schedule a callback to happen 1349 * shortly. 1350 */ 1351 if (atomic_read(&sdev->device_busy) == 0) 1352 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1353 break; 1354 default: 1355 req->cmd_flags |= REQ_DONTPREP; 1356 } 1357 1358 return ret; 1359 } 1360 1361 static int scsi_prep_fn(struct request_queue *q, struct request *req) 1362 { 1363 struct scsi_device *sdev = q->queuedata; 1364 struct scsi_cmnd *cmd; 1365 int ret; 1366 1367 ret = scsi_prep_state_check(sdev, req); 1368 if (ret != BLKPREP_OK) 1369 goto out; 1370 1371 cmd = scsi_get_cmd_from_req(sdev, req); 1372 if (unlikely(!cmd)) { 1373 ret = BLKPREP_DEFER; 1374 goto out; 1375 } 1376 1377 ret = scsi_setup_cmnd(sdev, req); 1378 out: 1379 return scsi_prep_return(q, req, ret); 1380 } 1381 1382 static void scsi_unprep_fn(struct request_queue *q, struct request *req) 1383 { 1384 scsi_uninit_cmd(req->special); 1385 } 1386 1387 /* 1388 * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else 1389 * return 0. 1390 * 1391 * Called with the queue_lock held. 1392 */ 1393 static inline int scsi_dev_queue_ready(struct request_queue *q, 1394 struct scsi_device *sdev) 1395 { 1396 unsigned int busy; 1397 1398 busy = atomic_inc_return(&sdev->device_busy) - 1; 1399 if (atomic_read(&sdev->device_blocked)) { 1400 if (busy) 1401 goto out_dec; 1402 1403 /* 1404 * unblock after device_blocked iterates to zero 1405 */ 1406 if (atomic_dec_return(&sdev->device_blocked) > 0) { 1407 /* 1408 * For the MQ case we take care of this in the caller. 1409 */ 1410 if (!q->mq_ops) 1411 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1412 goto out_dec; 1413 } 1414 SCSI_LOG_MLQUEUE(3, sdev_printk(KERN_INFO, sdev, 1415 "unblocking device at zero depth\n")); 1416 } 1417 1418 if (busy >= sdev->queue_depth) 1419 goto out_dec; 1420 1421 return 1; 1422 out_dec: 1423 atomic_dec(&sdev->device_busy); 1424 return 0; 1425 } 1426 1427 /* 1428 * scsi_target_queue_ready: checks if there we can send commands to target 1429 * @sdev: scsi device on starget to check. 1430 */ 1431 static inline int scsi_target_queue_ready(struct Scsi_Host *shost, 1432 struct scsi_device *sdev) 1433 { 1434 struct scsi_target *starget = scsi_target(sdev); 1435 unsigned int busy; 1436 1437 if (starget->single_lun) { 1438 spin_lock_irq(shost->host_lock); 1439 if (starget->starget_sdev_user && 1440 starget->starget_sdev_user != sdev) { 1441 spin_unlock_irq(shost->host_lock); 1442 return 0; 1443 } 1444 starget->starget_sdev_user = sdev; 1445 spin_unlock_irq(shost->host_lock); 1446 } 1447 1448 if (starget->can_queue <= 0) 1449 return 1; 1450 1451 busy = atomic_inc_return(&starget->target_busy) - 1; 1452 if (atomic_read(&starget->target_blocked) > 0) { 1453 if (busy) 1454 goto starved; 1455 1456 /* 1457 * unblock after target_blocked iterates to zero 1458 */ 1459 if (atomic_dec_return(&starget->target_blocked) > 0) 1460 goto out_dec; 1461 1462 SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget, 1463 "unblocking target at zero depth\n")); 1464 } 1465 1466 if (busy >= starget->can_queue) 1467 goto starved; 1468 1469 return 1; 1470 1471 starved: 1472 spin_lock_irq(shost->host_lock); 1473 list_move_tail(&sdev->starved_entry, &shost->starved_list); 1474 spin_unlock_irq(shost->host_lock); 1475 out_dec: 1476 if (starget->can_queue > 0) 1477 atomic_dec(&starget->target_busy); 1478 return 0; 1479 } 1480 1481 /* 1482 * scsi_host_queue_ready: if we can send requests to shost, return 1 else 1483 * return 0. We must end up running the queue again whenever 0 is 1484 * returned, else IO can hang. 1485 */ 1486 static inline int scsi_host_queue_ready(struct request_queue *q, 1487 struct Scsi_Host *shost, 1488 struct scsi_device *sdev) 1489 { 1490 unsigned int busy; 1491 1492 if (scsi_host_in_recovery(shost)) 1493 return 0; 1494 1495 busy = atomic_inc_return(&shost->host_busy) - 1; 1496 if (atomic_read(&shost->host_blocked) > 0) { 1497 if (busy) 1498 goto starved; 1499 1500 /* 1501 * unblock after host_blocked iterates to zero 1502 */ 1503 if (atomic_dec_return(&shost->host_blocked) > 0) 1504 goto out_dec; 1505 1506 SCSI_LOG_MLQUEUE(3, 1507 shost_printk(KERN_INFO, shost, 1508 "unblocking host at zero depth\n")); 1509 } 1510 1511 if (shost->can_queue > 0 && busy >= shost->can_queue) 1512 goto starved; 1513 if (shost->host_self_blocked) 1514 goto starved; 1515 1516 /* We're OK to process the command, so we can't be starved */ 1517 if (!list_empty(&sdev->starved_entry)) { 1518 spin_lock_irq(shost->host_lock); 1519 if (!list_empty(&sdev->starved_entry)) 1520 list_del_init(&sdev->starved_entry); 1521 spin_unlock_irq(shost->host_lock); 1522 } 1523 1524 return 1; 1525 1526 starved: 1527 spin_lock_irq(shost->host_lock); 1528 if (list_empty(&sdev->starved_entry)) 1529 list_add_tail(&sdev->starved_entry, &shost->starved_list); 1530 spin_unlock_irq(shost->host_lock); 1531 out_dec: 1532 atomic_dec(&shost->host_busy); 1533 return 0; 1534 } 1535 1536 /* 1537 * Busy state exporting function for request stacking drivers. 1538 * 1539 * For efficiency, no lock is taken to check the busy state of 1540 * shost/starget/sdev, since the returned value is not guaranteed and 1541 * may be changed after request stacking drivers call the function, 1542 * regardless of taking lock or not. 1543 * 1544 * When scsi can't dispatch I/Os anymore and needs to kill I/Os scsi 1545 * needs to return 'not busy'. Otherwise, request stacking drivers 1546 * may hold requests forever. 1547 */ 1548 static int scsi_lld_busy(struct request_queue *q) 1549 { 1550 struct scsi_device *sdev = q->queuedata; 1551 struct Scsi_Host *shost; 1552 1553 if (blk_queue_dying(q)) 1554 return 0; 1555 1556 shost = sdev->host; 1557 1558 /* 1559 * Ignore host/starget busy state. 1560 * Since block layer does not have a concept of fairness across 1561 * multiple queues, congestion of host/starget needs to be handled 1562 * in SCSI layer. 1563 */ 1564 if (scsi_host_in_recovery(shost) || scsi_device_is_busy(sdev)) 1565 return 1; 1566 1567 return 0; 1568 } 1569 1570 /* 1571 * Kill a request for a dead device 1572 */ 1573 static void scsi_kill_request(struct request *req, struct request_queue *q) 1574 { 1575 struct scsi_cmnd *cmd = req->special; 1576 struct scsi_device *sdev; 1577 struct scsi_target *starget; 1578 struct Scsi_Host *shost; 1579 1580 blk_start_request(req); 1581 1582 scmd_printk(KERN_INFO, cmd, "killing request\n"); 1583 1584 sdev = cmd->device; 1585 starget = scsi_target(sdev); 1586 shost = sdev->host; 1587 scsi_init_cmd_errh(cmd); 1588 cmd->result = DID_NO_CONNECT << 16; 1589 atomic_inc(&cmd->device->iorequest_cnt); 1590 1591 /* 1592 * SCSI request completion path will do scsi_device_unbusy(), 1593 * bump busy counts. To bump the counters, we need to dance 1594 * with the locks as normal issue path does. 1595 */ 1596 atomic_inc(&sdev->device_busy); 1597 atomic_inc(&shost->host_busy); 1598 if (starget->can_queue > 0) 1599 atomic_inc(&starget->target_busy); 1600 1601 blk_complete_request(req); 1602 } 1603 1604 static void scsi_softirq_done(struct request *rq) 1605 { 1606 struct scsi_cmnd *cmd = rq->special; 1607 unsigned long wait_for = (cmd->allowed + 1) * rq->timeout; 1608 int disposition; 1609 1610 INIT_LIST_HEAD(&cmd->eh_entry); 1611 1612 atomic_inc(&cmd->device->iodone_cnt); 1613 if (cmd->result) 1614 atomic_inc(&cmd->device->ioerr_cnt); 1615 1616 disposition = scsi_decide_disposition(cmd); 1617 if (disposition != SUCCESS && 1618 time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) { 1619 sdev_printk(KERN_ERR, cmd->device, 1620 "timing out command, waited %lus\n", 1621 wait_for/HZ); 1622 disposition = SUCCESS; 1623 } 1624 1625 scsi_log_completion(cmd, disposition); 1626 1627 switch (disposition) { 1628 case SUCCESS: 1629 scsi_finish_command(cmd); 1630 break; 1631 case NEEDS_RETRY: 1632 scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY); 1633 break; 1634 case ADD_TO_MLQUEUE: 1635 scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY); 1636 break; 1637 default: 1638 if (!scsi_eh_scmd_add(cmd, 0)) 1639 scsi_finish_command(cmd); 1640 } 1641 } 1642 1643 /** 1644 * scsi_dispatch_command - Dispatch a command to the low-level driver. 1645 * @cmd: command block we are dispatching. 1646 * 1647 * Return: nonzero return request was rejected and device's queue needs to be 1648 * plugged. 1649 */ 1650 static int scsi_dispatch_cmd(struct scsi_cmnd *cmd) 1651 { 1652 struct Scsi_Host *host = cmd->device->host; 1653 int rtn = 0; 1654 1655 atomic_inc(&cmd->device->iorequest_cnt); 1656 1657 /* check if the device is still usable */ 1658 if (unlikely(cmd->device->sdev_state == SDEV_DEL)) { 1659 /* in SDEV_DEL we error all commands. DID_NO_CONNECT 1660 * returns an immediate error upwards, and signals 1661 * that the device is no longer present */ 1662 cmd->result = DID_NO_CONNECT << 16; 1663 goto done; 1664 } 1665 1666 /* Check to see if the scsi lld made this device blocked. */ 1667 if (unlikely(scsi_device_blocked(cmd->device))) { 1668 /* 1669 * in blocked state, the command is just put back on 1670 * the device queue. The suspend state has already 1671 * blocked the queue so future requests should not 1672 * occur until the device transitions out of the 1673 * suspend state. 1674 */ 1675 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd, 1676 "queuecommand : device blocked\n")); 1677 return SCSI_MLQUEUE_DEVICE_BUSY; 1678 } 1679 1680 /* Store the LUN value in cmnd, if needed. */ 1681 if (cmd->device->lun_in_cdb) 1682 cmd->cmnd[1] = (cmd->cmnd[1] & 0x1f) | 1683 (cmd->device->lun << 5 & 0xe0); 1684 1685 scsi_log_send(cmd); 1686 1687 /* 1688 * Before we queue this command, check if the command 1689 * length exceeds what the host adapter can handle. 1690 */ 1691 if (cmd->cmd_len > cmd->device->host->max_cmd_len) { 1692 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd, 1693 "queuecommand : command too long. " 1694 "cdb_size=%d host->max_cmd_len=%d\n", 1695 cmd->cmd_len, cmd->device->host->max_cmd_len)); 1696 cmd->result = (DID_ABORT << 16); 1697 goto done; 1698 } 1699 1700 if (unlikely(host->shost_state == SHOST_DEL)) { 1701 cmd->result = (DID_NO_CONNECT << 16); 1702 goto done; 1703 1704 } 1705 1706 trace_scsi_dispatch_cmd_start(cmd); 1707 rtn = host->hostt->queuecommand(host, cmd); 1708 if (rtn) { 1709 trace_scsi_dispatch_cmd_error(cmd, rtn); 1710 if (rtn != SCSI_MLQUEUE_DEVICE_BUSY && 1711 rtn != SCSI_MLQUEUE_TARGET_BUSY) 1712 rtn = SCSI_MLQUEUE_HOST_BUSY; 1713 1714 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd, 1715 "queuecommand : request rejected\n")); 1716 } 1717 1718 return rtn; 1719 done: 1720 cmd->scsi_done(cmd); 1721 return 0; 1722 } 1723 1724 /** 1725 * scsi_done - Invoke completion on finished SCSI command. 1726 * @cmd: The SCSI Command for which a low-level device driver (LLDD) gives 1727 * ownership back to SCSI Core -- i.e. the LLDD has finished with it. 1728 * 1729 * Description: This function is the mid-level's (SCSI Core) interrupt routine, 1730 * which regains ownership of the SCSI command (de facto) from a LLDD, and 1731 * calls blk_complete_request() for further processing. 1732 * 1733 * This function is interrupt context safe. 1734 */ 1735 static void scsi_done(struct scsi_cmnd *cmd) 1736 { 1737 trace_scsi_dispatch_cmd_done(cmd); 1738 blk_complete_request(cmd->request); 1739 } 1740 1741 /* 1742 * Function: scsi_request_fn() 1743 * 1744 * Purpose: Main strategy routine for SCSI. 1745 * 1746 * Arguments: q - Pointer to actual queue. 1747 * 1748 * Returns: Nothing 1749 * 1750 * Lock status: IO request lock assumed to be held when called. 1751 */ 1752 static void scsi_request_fn(struct request_queue *q) 1753 __releases(q->queue_lock) 1754 __acquires(q->queue_lock) 1755 { 1756 struct scsi_device *sdev = q->queuedata; 1757 struct Scsi_Host *shost; 1758 struct scsi_cmnd *cmd; 1759 struct request *req; 1760 1761 /* 1762 * To start with, we keep looping until the queue is empty, or until 1763 * the host is no longer able to accept any more requests. 1764 */ 1765 shost = sdev->host; 1766 for (;;) { 1767 int rtn; 1768 /* 1769 * get next queueable request. We do this early to make sure 1770 * that the request is fully prepared even if we cannot 1771 * accept it. 1772 */ 1773 req = blk_peek_request(q); 1774 if (!req) 1775 break; 1776 1777 if (unlikely(!scsi_device_online(sdev))) { 1778 sdev_printk(KERN_ERR, sdev, 1779 "rejecting I/O to offline device\n"); 1780 scsi_kill_request(req, q); 1781 continue; 1782 } 1783 1784 if (!scsi_dev_queue_ready(q, sdev)) 1785 break; 1786 1787 /* 1788 * Remove the request from the request list. 1789 */ 1790 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req))) 1791 blk_start_request(req); 1792 1793 spin_unlock_irq(q->queue_lock); 1794 cmd = req->special; 1795 if (unlikely(cmd == NULL)) { 1796 printk(KERN_CRIT "impossible request in %s.\n" 1797 "please mail a stack trace to " 1798 "linux-scsi@vger.kernel.org\n", 1799 __func__); 1800 blk_dump_rq_flags(req, "foo"); 1801 BUG(); 1802 } 1803 1804 /* 1805 * We hit this when the driver is using a host wide 1806 * tag map. For device level tag maps the queue_depth check 1807 * in the device ready fn would prevent us from trying 1808 * to allocate a tag. Since the map is a shared host resource 1809 * we add the dev to the starved list so it eventually gets 1810 * a run when a tag is freed. 1811 */ 1812 if (blk_queue_tagged(q) && !(req->cmd_flags & REQ_QUEUED)) { 1813 spin_lock_irq(shost->host_lock); 1814 if (list_empty(&sdev->starved_entry)) 1815 list_add_tail(&sdev->starved_entry, 1816 &shost->starved_list); 1817 spin_unlock_irq(shost->host_lock); 1818 goto not_ready; 1819 } 1820 1821 if (!scsi_target_queue_ready(shost, sdev)) 1822 goto not_ready; 1823 1824 if (!scsi_host_queue_ready(q, shost, sdev)) 1825 goto host_not_ready; 1826 1827 if (sdev->simple_tags) 1828 cmd->flags |= SCMD_TAGGED; 1829 else 1830 cmd->flags &= ~SCMD_TAGGED; 1831 1832 /* 1833 * Finally, initialize any error handling parameters, and set up 1834 * the timers for timeouts. 1835 */ 1836 scsi_init_cmd_errh(cmd); 1837 1838 /* 1839 * Dispatch the command to the low-level driver. 1840 */ 1841 cmd->scsi_done = scsi_done; 1842 rtn = scsi_dispatch_cmd(cmd); 1843 if (rtn) { 1844 scsi_queue_insert(cmd, rtn); 1845 spin_lock_irq(q->queue_lock); 1846 goto out_delay; 1847 } 1848 spin_lock_irq(q->queue_lock); 1849 } 1850 1851 return; 1852 1853 host_not_ready: 1854 if (scsi_target(sdev)->can_queue > 0) 1855 atomic_dec(&scsi_target(sdev)->target_busy); 1856 not_ready: 1857 /* 1858 * lock q, handle tag, requeue req, and decrement device_busy. We 1859 * must return with queue_lock held. 1860 * 1861 * Decrementing device_busy without checking it is OK, as all such 1862 * cases (host limits or settings) should run the queue at some 1863 * later time. 1864 */ 1865 spin_lock_irq(q->queue_lock); 1866 blk_requeue_request(q, req); 1867 atomic_dec(&sdev->device_busy); 1868 out_delay: 1869 if (!atomic_read(&sdev->device_busy) && !scsi_device_blocked(sdev)) 1870 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1871 } 1872 1873 static inline int prep_to_mq(int ret) 1874 { 1875 switch (ret) { 1876 case BLKPREP_OK: 1877 return 0; 1878 case BLKPREP_DEFER: 1879 return BLK_MQ_RQ_QUEUE_BUSY; 1880 default: 1881 return BLK_MQ_RQ_QUEUE_ERROR; 1882 } 1883 } 1884 1885 static int scsi_mq_prep_fn(struct request *req) 1886 { 1887 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 1888 struct scsi_device *sdev = req->q->queuedata; 1889 struct Scsi_Host *shost = sdev->host; 1890 unsigned char *sense_buf = cmd->sense_buffer; 1891 struct scatterlist *sg; 1892 1893 memset(cmd, 0, sizeof(struct scsi_cmnd)); 1894 1895 req->special = cmd; 1896 1897 cmd->request = req; 1898 cmd->device = sdev; 1899 cmd->sense_buffer = sense_buf; 1900 1901 cmd->tag = req->tag; 1902 1903 cmd->cmnd = req->cmd; 1904 cmd->prot_op = SCSI_PROT_NORMAL; 1905 1906 INIT_LIST_HEAD(&cmd->list); 1907 INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler); 1908 cmd->jiffies_at_alloc = jiffies; 1909 1910 if (shost->use_cmd_list) { 1911 spin_lock_irq(&sdev->list_lock); 1912 list_add_tail(&cmd->list, &sdev->cmd_list); 1913 spin_unlock_irq(&sdev->list_lock); 1914 } 1915 1916 sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size; 1917 cmd->sdb.table.sgl = sg; 1918 1919 if (scsi_host_get_prot(shost)) { 1920 cmd->prot_sdb = (void *)sg + 1921 min_t(unsigned int, 1922 shost->sg_tablesize, SCSI_MAX_SG_SEGMENTS) * 1923 sizeof(struct scatterlist); 1924 memset(cmd->prot_sdb, 0, sizeof(struct scsi_data_buffer)); 1925 1926 cmd->prot_sdb->table.sgl = 1927 (struct scatterlist *)(cmd->prot_sdb + 1); 1928 } 1929 1930 if (blk_bidi_rq(req)) { 1931 struct request *next_rq = req->next_rq; 1932 struct scsi_data_buffer *bidi_sdb = blk_mq_rq_to_pdu(next_rq); 1933 1934 memset(bidi_sdb, 0, sizeof(struct scsi_data_buffer)); 1935 bidi_sdb->table.sgl = 1936 (struct scatterlist *)(bidi_sdb + 1); 1937 1938 next_rq->special = bidi_sdb; 1939 } 1940 1941 blk_mq_start_request(req); 1942 1943 return scsi_setup_cmnd(sdev, req); 1944 } 1945 1946 static void scsi_mq_done(struct scsi_cmnd *cmd) 1947 { 1948 trace_scsi_dispatch_cmd_done(cmd); 1949 blk_mq_complete_request(cmd->request); 1950 } 1951 1952 static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx, 1953 const struct blk_mq_queue_data *bd) 1954 { 1955 struct request *req = bd->rq; 1956 struct request_queue *q = req->q; 1957 struct scsi_device *sdev = q->queuedata; 1958 struct Scsi_Host *shost = sdev->host; 1959 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 1960 int ret; 1961 int reason; 1962 1963 ret = prep_to_mq(scsi_prep_state_check(sdev, req)); 1964 if (ret) 1965 goto out; 1966 1967 ret = BLK_MQ_RQ_QUEUE_BUSY; 1968 if (!get_device(&sdev->sdev_gendev)) 1969 goto out; 1970 1971 if (!scsi_dev_queue_ready(q, sdev)) 1972 goto out_put_device; 1973 if (!scsi_target_queue_ready(shost, sdev)) 1974 goto out_dec_device_busy; 1975 if (!scsi_host_queue_ready(q, shost, sdev)) 1976 goto out_dec_target_busy; 1977 1978 1979 if (!(req->cmd_flags & REQ_DONTPREP)) { 1980 ret = prep_to_mq(scsi_mq_prep_fn(req)); 1981 if (ret) 1982 goto out_dec_host_busy; 1983 req->cmd_flags |= REQ_DONTPREP; 1984 } else { 1985 blk_mq_start_request(req); 1986 } 1987 1988 if (sdev->simple_tags) 1989 cmd->flags |= SCMD_TAGGED; 1990 else 1991 cmd->flags &= ~SCMD_TAGGED; 1992 1993 scsi_init_cmd_errh(cmd); 1994 cmd->scsi_done = scsi_mq_done; 1995 1996 reason = scsi_dispatch_cmd(cmd); 1997 if (reason) { 1998 scsi_set_blocked(cmd, reason); 1999 ret = BLK_MQ_RQ_QUEUE_BUSY; 2000 goto out_dec_host_busy; 2001 } 2002 2003 return BLK_MQ_RQ_QUEUE_OK; 2004 2005 out_dec_host_busy: 2006 atomic_dec(&shost->host_busy); 2007 out_dec_target_busy: 2008 if (scsi_target(sdev)->can_queue > 0) 2009 atomic_dec(&scsi_target(sdev)->target_busy); 2010 out_dec_device_busy: 2011 atomic_dec(&sdev->device_busy); 2012 out_put_device: 2013 put_device(&sdev->sdev_gendev); 2014 out: 2015 switch (ret) { 2016 case BLK_MQ_RQ_QUEUE_BUSY: 2017 blk_mq_stop_hw_queue(hctx); 2018 if (atomic_read(&sdev->device_busy) == 0 && 2019 !scsi_device_blocked(sdev)) 2020 blk_mq_delay_queue(hctx, SCSI_QUEUE_DELAY); 2021 break; 2022 case BLK_MQ_RQ_QUEUE_ERROR: 2023 /* 2024 * Make sure to release all allocated ressources when 2025 * we hit an error, as we will never see this command 2026 * again. 2027 */ 2028 if (req->cmd_flags & REQ_DONTPREP) 2029 scsi_mq_uninit_cmd(cmd); 2030 break; 2031 default: 2032 break; 2033 } 2034 return ret; 2035 } 2036 2037 static enum blk_eh_timer_return scsi_timeout(struct request *req, 2038 bool reserved) 2039 { 2040 if (reserved) 2041 return BLK_EH_RESET_TIMER; 2042 return scsi_times_out(req); 2043 } 2044 2045 static int scsi_init_request(void *data, struct request *rq, 2046 unsigned int hctx_idx, unsigned int request_idx, 2047 unsigned int numa_node) 2048 { 2049 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 2050 2051 cmd->sense_buffer = kzalloc_node(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL, 2052 numa_node); 2053 if (!cmd->sense_buffer) 2054 return -ENOMEM; 2055 return 0; 2056 } 2057 2058 static void scsi_exit_request(void *data, struct request *rq, 2059 unsigned int hctx_idx, unsigned int request_idx) 2060 { 2061 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 2062 2063 kfree(cmd->sense_buffer); 2064 } 2065 2066 static u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost) 2067 { 2068 struct device *host_dev; 2069 u64 bounce_limit = 0xffffffff; 2070 2071 if (shost->unchecked_isa_dma) 2072 return BLK_BOUNCE_ISA; 2073 /* 2074 * Platforms with virtual-DMA translation 2075 * hardware have no practical limit. 2076 */ 2077 if (!PCI_DMA_BUS_IS_PHYS) 2078 return BLK_BOUNCE_ANY; 2079 2080 host_dev = scsi_get_device(shost); 2081 if (host_dev && host_dev->dma_mask) 2082 bounce_limit = (u64)dma_max_pfn(host_dev) << PAGE_SHIFT; 2083 2084 return bounce_limit; 2085 } 2086 2087 static void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q) 2088 { 2089 struct device *dev = shost->dma_dev; 2090 2091 /* 2092 * this limit is imposed by hardware restrictions 2093 */ 2094 blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize, 2095 SCSI_MAX_SG_CHAIN_SEGMENTS)); 2096 2097 if (scsi_host_prot_dma(shost)) { 2098 shost->sg_prot_tablesize = 2099 min_not_zero(shost->sg_prot_tablesize, 2100 (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS); 2101 BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize); 2102 blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize); 2103 } 2104 2105 blk_queue_max_hw_sectors(q, shost->max_sectors); 2106 blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost)); 2107 blk_queue_segment_boundary(q, shost->dma_boundary); 2108 dma_set_seg_boundary(dev, shost->dma_boundary); 2109 2110 blk_queue_max_segment_size(q, dma_get_max_seg_size(dev)); 2111 2112 if (!shost->use_clustering) 2113 q->limits.cluster = 0; 2114 2115 /* 2116 * set a reasonable default alignment on word boundaries: the 2117 * host and device may alter it using 2118 * blk_queue_update_dma_alignment() later. 2119 */ 2120 blk_queue_dma_alignment(q, 0x03); 2121 } 2122 2123 struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost, 2124 request_fn_proc *request_fn) 2125 { 2126 struct request_queue *q; 2127 2128 q = blk_init_queue(request_fn, NULL); 2129 if (!q) 2130 return NULL; 2131 __scsi_init_queue(shost, q); 2132 return q; 2133 } 2134 EXPORT_SYMBOL(__scsi_alloc_queue); 2135 2136 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev) 2137 { 2138 struct request_queue *q; 2139 2140 q = __scsi_alloc_queue(sdev->host, scsi_request_fn); 2141 if (!q) 2142 return NULL; 2143 2144 blk_queue_prep_rq(q, scsi_prep_fn); 2145 blk_queue_unprep_rq(q, scsi_unprep_fn); 2146 blk_queue_softirq_done(q, scsi_softirq_done); 2147 blk_queue_rq_timed_out(q, scsi_times_out); 2148 blk_queue_lld_busy(q, scsi_lld_busy); 2149 return q; 2150 } 2151 2152 static struct blk_mq_ops scsi_mq_ops = { 2153 .map_queue = blk_mq_map_queue, 2154 .queue_rq = scsi_queue_rq, 2155 .complete = scsi_softirq_done, 2156 .timeout = scsi_timeout, 2157 .init_request = scsi_init_request, 2158 .exit_request = scsi_exit_request, 2159 }; 2160 2161 struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev) 2162 { 2163 sdev->request_queue = blk_mq_init_queue(&sdev->host->tag_set); 2164 if (IS_ERR(sdev->request_queue)) 2165 return NULL; 2166 2167 sdev->request_queue->queuedata = sdev; 2168 __scsi_init_queue(sdev->host, sdev->request_queue); 2169 return sdev->request_queue; 2170 } 2171 2172 int scsi_mq_setup_tags(struct Scsi_Host *shost) 2173 { 2174 unsigned int cmd_size, sgl_size, tbl_size; 2175 2176 tbl_size = shost->sg_tablesize; 2177 if (tbl_size > SCSI_MAX_SG_SEGMENTS) 2178 tbl_size = SCSI_MAX_SG_SEGMENTS; 2179 sgl_size = tbl_size * sizeof(struct scatterlist); 2180 cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size + sgl_size; 2181 if (scsi_host_get_prot(shost)) 2182 cmd_size += sizeof(struct scsi_data_buffer) + sgl_size; 2183 2184 memset(&shost->tag_set, 0, sizeof(shost->tag_set)); 2185 shost->tag_set.ops = &scsi_mq_ops; 2186 shost->tag_set.nr_hw_queues = shost->nr_hw_queues ? : 1; 2187 shost->tag_set.queue_depth = shost->can_queue; 2188 shost->tag_set.cmd_size = cmd_size; 2189 shost->tag_set.numa_node = NUMA_NO_NODE; 2190 shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; 2191 shost->tag_set.driver_data = shost; 2192 2193 return blk_mq_alloc_tag_set(&shost->tag_set); 2194 } 2195 2196 void scsi_mq_destroy_tags(struct Scsi_Host *shost) 2197 { 2198 blk_mq_free_tag_set(&shost->tag_set); 2199 } 2200 2201 /* 2202 * Function: scsi_block_requests() 2203 * 2204 * Purpose: Utility function used by low-level drivers to prevent further 2205 * commands from being queued to the device. 2206 * 2207 * Arguments: shost - Host in question 2208 * 2209 * Returns: Nothing 2210 * 2211 * Lock status: No locks are assumed held. 2212 * 2213 * Notes: There is no timer nor any other means by which the requests 2214 * get unblocked other than the low-level driver calling 2215 * scsi_unblock_requests(). 2216 */ 2217 void scsi_block_requests(struct Scsi_Host *shost) 2218 { 2219 shost->host_self_blocked = 1; 2220 } 2221 EXPORT_SYMBOL(scsi_block_requests); 2222 2223 /* 2224 * Function: scsi_unblock_requests() 2225 * 2226 * Purpose: Utility function used by low-level drivers to allow further 2227 * commands from being queued to the device. 2228 * 2229 * Arguments: shost - Host in question 2230 * 2231 * Returns: Nothing 2232 * 2233 * Lock status: No locks are assumed held. 2234 * 2235 * Notes: There is no timer nor any other means by which the requests 2236 * get unblocked other than the low-level driver calling 2237 * scsi_unblock_requests(). 2238 * 2239 * This is done as an API function so that changes to the 2240 * internals of the scsi mid-layer won't require wholesale 2241 * changes to drivers that use this feature. 2242 */ 2243 void scsi_unblock_requests(struct Scsi_Host *shost) 2244 { 2245 shost->host_self_blocked = 0; 2246 scsi_run_host_queues(shost); 2247 } 2248 EXPORT_SYMBOL(scsi_unblock_requests); 2249 2250 int __init scsi_init_queue(void) 2251 { 2252 int i; 2253 2254 scsi_sdb_cache = kmem_cache_create("scsi_data_buffer", 2255 sizeof(struct scsi_data_buffer), 2256 0, 0, NULL); 2257 if (!scsi_sdb_cache) { 2258 printk(KERN_ERR "SCSI: can't init scsi sdb cache\n"); 2259 return -ENOMEM; 2260 } 2261 2262 for (i = 0; i < SG_MEMPOOL_NR; i++) { 2263 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i; 2264 int size = sgp->size * sizeof(struct scatterlist); 2265 2266 sgp->slab = kmem_cache_create(sgp->name, size, 0, 2267 SLAB_HWCACHE_ALIGN, NULL); 2268 if (!sgp->slab) { 2269 printk(KERN_ERR "SCSI: can't init sg slab %s\n", 2270 sgp->name); 2271 goto cleanup_sdb; 2272 } 2273 2274 sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE, 2275 sgp->slab); 2276 if (!sgp->pool) { 2277 printk(KERN_ERR "SCSI: can't init sg mempool %s\n", 2278 sgp->name); 2279 goto cleanup_sdb; 2280 } 2281 } 2282 2283 return 0; 2284 2285 cleanup_sdb: 2286 for (i = 0; i < SG_MEMPOOL_NR; i++) { 2287 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i; 2288 if (sgp->pool) 2289 mempool_destroy(sgp->pool); 2290 if (sgp->slab) 2291 kmem_cache_destroy(sgp->slab); 2292 } 2293 kmem_cache_destroy(scsi_sdb_cache); 2294 2295 return -ENOMEM; 2296 } 2297 2298 void scsi_exit_queue(void) 2299 { 2300 int i; 2301 2302 kmem_cache_destroy(scsi_sdb_cache); 2303 2304 for (i = 0; i < SG_MEMPOOL_NR; i++) { 2305 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i; 2306 mempool_destroy(sgp->pool); 2307 kmem_cache_destroy(sgp->slab); 2308 } 2309 } 2310 2311 /** 2312 * scsi_mode_select - issue a mode select 2313 * @sdev: SCSI device to be queried 2314 * @pf: Page format bit (1 == standard, 0 == vendor specific) 2315 * @sp: Save page bit (0 == don't save, 1 == save) 2316 * @modepage: mode page being requested 2317 * @buffer: request buffer (may not be smaller than eight bytes) 2318 * @len: length of request buffer. 2319 * @timeout: command timeout 2320 * @retries: number of retries before failing 2321 * @data: returns a structure abstracting the mode header data 2322 * @sshdr: place to put sense data (or NULL if no sense to be collected). 2323 * must be SCSI_SENSE_BUFFERSIZE big. 2324 * 2325 * Returns zero if successful; negative error number or scsi 2326 * status on error 2327 * 2328 */ 2329 int 2330 scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage, 2331 unsigned char *buffer, int len, int timeout, int retries, 2332 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr) 2333 { 2334 unsigned char cmd[10]; 2335 unsigned char *real_buffer; 2336 int ret; 2337 2338 memset(cmd, 0, sizeof(cmd)); 2339 cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0); 2340 2341 if (sdev->use_10_for_ms) { 2342 if (len > 65535) 2343 return -EINVAL; 2344 real_buffer = kmalloc(8 + len, GFP_KERNEL); 2345 if (!real_buffer) 2346 return -ENOMEM; 2347 memcpy(real_buffer + 8, buffer, len); 2348 len += 8; 2349 real_buffer[0] = 0; 2350 real_buffer[1] = 0; 2351 real_buffer[2] = data->medium_type; 2352 real_buffer[3] = data->device_specific; 2353 real_buffer[4] = data->longlba ? 0x01 : 0; 2354 real_buffer[5] = 0; 2355 real_buffer[6] = data->block_descriptor_length >> 8; 2356 real_buffer[7] = data->block_descriptor_length; 2357 2358 cmd[0] = MODE_SELECT_10; 2359 cmd[7] = len >> 8; 2360 cmd[8] = len; 2361 } else { 2362 if (len > 255 || data->block_descriptor_length > 255 || 2363 data->longlba) 2364 return -EINVAL; 2365 2366 real_buffer = kmalloc(4 + len, GFP_KERNEL); 2367 if (!real_buffer) 2368 return -ENOMEM; 2369 memcpy(real_buffer + 4, buffer, len); 2370 len += 4; 2371 real_buffer[0] = 0; 2372 real_buffer[1] = data->medium_type; 2373 real_buffer[2] = data->device_specific; 2374 real_buffer[3] = data->block_descriptor_length; 2375 2376 2377 cmd[0] = MODE_SELECT; 2378 cmd[4] = len; 2379 } 2380 2381 ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len, 2382 sshdr, timeout, retries, NULL); 2383 kfree(real_buffer); 2384 return ret; 2385 } 2386 EXPORT_SYMBOL_GPL(scsi_mode_select); 2387 2388 /** 2389 * scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary. 2390 * @sdev: SCSI device to be queried 2391 * @dbd: set if mode sense will allow block descriptors to be returned 2392 * @modepage: mode page being requested 2393 * @buffer: request buffer (may not be smaller than eight bytes) 2394 * @len: length of request buffer. 2395 * @timeout: command timeout 2396 * @retries: number of retries before failing 2397 * @data: returns a structure abstracting the mode header data 2398 * @sshdr: place to put sense data (or NULL if no sense to be collected). 2399 * must be SCSI_SENSE_BUFFERSIZE big. 2400 * 2401 * Returns zero if unsuccessful, or the header offset (either 4 2402 * or 8 depending on whether a six or ten byte command was 2403 * issued) if successful. 2404 */ 2405 int 2406 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage, 2407 unsigned char *buffer, int len, int timeout, int retries, 2408 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr) 2409 { 2410 unsigned char cmd[12]; 2411 int use_10_for_ms; 2412 int header_length; 2413 int result; 2414 struct scsi_sense_hdr my_sshdr; 2415 2416 memset(data, 0, sizeof(*data)); 2417 memset(&cmd[0], 0, 12); 2418 cmd[1] = dbd & 0x18; /* allows DBD and LLBA bits */ 2419 cmd[2] = modepage; 2420 2421 /* caller might not be interested in sense, but we need it */ 2422 if (!sshdr) 2423 sshdr = &my_sshdr; 2424 2425 retry: 2426 use_10_for_ms = sdev->use_10_for_ms; 2427 2428 if (use_10_for_ms) { 2429 if (len < 8) 2430 len = 8; 2431 2432 cmd[0] = MODE_SENSE_10; 2433 cmd[8] = len; 2434 header_length = 8; 2435 } else { 2436 if (len < 4) 2437 len = 4; 2438 2439 cmd[0] = MODE_SENSE; 2440 cmd[4] = len; 2441 header_length = 4; 2442 } 2443 2444 memset(buffer, 0, len); 2445 2446 result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len, 2447 sshdr, timeout, retries, NULL); 2448 2449 /* This code looks awful: what it's doing is making sure an 2450 * ILLEGAL REQUEST sense return identifies the actual command 2451 * byte as the problem. MODE_SENSE commands can return 2452 * ILLEGAL REQUEST if the code page isn't supported */ 2453 2454 if (use_10_for_ms && !scsi_status_is_good(result) && 2455 (driver_byte(result) & DRIVER_SENSE)) { 2456 if (scsi_sense_valid(sshdr)) { 2457 if ((sshdr->sense_key == ILLEGAL_REQUEST) && 2458 (sshdr->asc == 0x20) && (sshdr->ascq == 0)) { 2459 /* 2460 * Invalid command operation code 2461 */ 2462 sdev->use_10_for_ms = 0; 2463 goto retry; 2464 } 2465 } 2466 } 2467 2468 if(scsi_status_is_good(result)) { 2469 if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b && 2470 (modepage == 6 || modepage == 8))) { 2471 /* Initio breakage? */ 2472 header_length = 0; 2473 data->length = 13; 2474 data->medium_type = 0; 2475 data->device_specific = 0; 2476 data->longlba = 0; 2477 data->block_descriptor_length = 0; 2478 } else if(use_10_for_ms) { 2479 data->length = buffer[0]*256 + buffer[1] + 2; 2480 data->medium_type = buffer[2]; 2481 data->device_specific = buffer[3]; 2482 data->longlba = buffer[4] & 0x01; 2483 data->block_descriptor_length = buffer[6]*256 2484 + buffer[7]; 2485 } else { 2486 data->length = buffer[0] + 1; 2487 data->medium_type = buffer[1]; 2488 data->device_specific = buffer[2]; 2489 data->block_descriptor_length = buffer[3]; 2490 } 2491 data->header_length = header_length; 2492 } 2493 2494 return result; 2495 } 2496 EXPORT_SYMBOL(scsi_mode_sense); 2497 2498 /** 2499 * scsi_test_unit_ready - test if unit is ready 2500 * @sdev: scsi device to change the state of. 2501 * @timeout: command timeout 2502 * @retries: number of retries before failing 2503 * @sshdr_external: Optional pointer to struct scsi_sense_hdr for 2504 * returning sense. Make sure that this is cleared before passing 2505 * in. 2506 * 2507 * Returns zero if unsuccessful or an error if TUR failed. For 2508 * removable media, UNIT_ATTENTION sets ->changed flag. 2509 **/ 2510 int 2511 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries, 2512 struct scsi_sense_hdr *sshdr_external) 2513 { 2514 char cmd[] = { 2515 TEST_UNIT_READY, 0, 0, 0, 0, 0, 2516 }; 2517 struct scsi_sense_hdr *sshdr; 2518 int result; 2519 2520 if (!sshdr_external) 2521 sshdr = kzalloc(sizeof(*sshdr), GFP_KERNEL); 2522 else 2523 sshdr = sshdr_external; 2524 2525 /* try to eat the UNIT_ATTENTION if there are enough retries */ 2526 do { 2527 result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr, 2528 timeout, retries, NULL); 2529 if (sdev->removable && scsi_sense_valid(sshdr) && 2530 sshdr->sense_key == UNIT_ATTENTION) 2531 sdev->changed = 1; 2532 } while (scsi_sense_valid(sshdr) && 2533 sshdr->sense_key == UNIT_ATTENTION && --retries); 2534 2535 if (!sshdr_external) 2536 kfree(sshdr); 2537 return result; 2538 } 2539 EXPORT_SYMBOL(scsi_test_unit_ready); 2540 2541 /** 2542 * scsi_device_set_state - Take the given device through the device state model. 2543 * @sdev: scsi device to change the state of. 2544 * @state: state to change to. 2545 * 2546 * Returns zero if unsuccessful or an error if the requested 2547 * transition is illegal. 2548 */ 2549 int 2550 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state) 2551 { 2552 enum scsi_device_state oldstate = sdev->sdev_state; 2553 2554 if (state == oldstate) 2555 return 0; 2556 2557 switch (state) { 2558 case SDEV_CREATED: 2559 switch (oldstate) { 2560 case SDEV_CREATED_BLOCK: 2561 break; 2562 default: 2563 goto illegal; 2564 } 2565 break; 2566 2567 case SDEV_RUNNING: 2568 switch (oldstate) { 2569 case SDEV_CREATED: 2570 case SDEV_OFFLINE: 2571 case SDEV_TRANSPORT_OFFLINE: 2572 case SDEV_QUIESCE: 2573 case SDEV_BLOCK: 2574 break; 2575 default: 2576 goto illegal; 2577 } 2578 break; 2579 2580 case SDEV_QUIESCE: 2581 switch (oldstate) { 2582 case SDEV_RUNNING: 2583 case SDEV_OFFLINE: 2584 case SDEV_TRANSPORT_OFFLINE: 2585 break; 2586 default: 2587 goto illegal; 2588 } 2589 break; 2590 2591 case SDEV_OFFLINE: 2592 case SDEV_TRANSPORT_OFFLINE: 2593 switch (oldstate) { 2594 case SDEV_CREATED: 2595 case SDEV_RUNNING: 2596 case SDEV_QUIESCE: 2597 case SDEV_BLOCK: 2598 break; 2599 default: 2600 goto illegal; 2601 } 2602 break; 2603 2604 case SDEV_BLOCK: 2605 switch (oldstate) { 2606 case SDEV_RUNNING: 2607 case SDEV_CREATED_BLOCK: 2608 break; 2609 default: 2610 goto illegal; 2611 } 2612 break; 2613 2614 case SDEV_CREATED_BLOCK: 2615 switch (oldstate) { 2616 case SDEV_CREATED: 2617 break; 2618 default: 2619 goto illegal; 2620 } 2621 break; 2622 2623 case SDEV_CANCEL: 2624 switch (oldstate) { 2625 case SDEV_CREATED: 2626 case SDEV_RUNNING: 2627 case SDEV_QUIESCE: 2628 case SDEV_OFFLINE: 2629 case SDEV_TRANSPORT_OFFLINE: 2630 case SDEV_BLOCK: 2631 break; 2632 default: 2633 goto illegal; 2634 } 2635 break; 2636 2637 case SDEV_DEL: 2638 switch (oldstate) { 2639 case SDEV_CREATED: 2640 case SDEV_RUNNING: 2641 case SDEV_OFFLINE: 2642 case SDEV_TRANSPORT_OFFLINE: 2643 case SDEV_CANCEL: 2644 case SDEV_CREATED_BLOCK: 2645 break; 2646 default: 2647 goto illegal; 2648 } 2649 break; 2650 2651 } 2652 sdev->sdev_state = state; 2653 return 0; 2654 2655 illegal: 2656 SCSI_LOG_ERROR_RECOVERY(1, 2657 sdev_printk(KERN_ERR, sdev, 2658 "Illegal state transition %s->%s", 2659 scsi_device_state_name(oldstate), 2660 scsi_device_state_name(state)) 2661 ); 2662 return -EINVAL; 2663 } 2664 EXPORT_SYMBOL(scsi_device_set_state); 2665 2666 /** 2667 * sdev_evt_emit - emit a single SCSI device uevent 2668 * @sdev: associated SCSI device 2669 * @evt: event to emit 2670 * 2671 * Send a single uevent (scsi_event) to the associated scsi_device. 2672 */ 2673 static void scsi_evt_emit(struct scsi_device *sdev, struct scsi_event *evt) 2674 { 2675 int idx = 0; 2676 char *envp[3]; 2677 2678 switch (evt->evt_type) { 2679 case SDEV_EVT_MEDIA_CHANGE: 2680 envp[idx++] = "SDEV_MEDIA_CHANGE=1"; 2681 break; 2682 case SDEV_EVT_INQUIRY_CHANGE_REPORTED: 2683 envp[idx++] = "SDEV_UA=INQUIRY_DATA_HAS_CHANGED"; 2684 break; 2685 case SDEV_EVT_CAPACITY_CHANGE_REPORTED: 2686 envp[idx++] = "SDEV_UA=CAPACITY_DATA_HAS_CHANGED"; 2687 break; 2688 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED: 2689 envp[idx++] = "SDEV_UA=THIN_PROVISIONING_SOFT_THRESHOLD_REACHED"; 2690 break; 2691 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED: 2692 envp[idx++] = "SDEV_UA=MODE_PARAMETERS_CHANGED"; 2693 break; 2694 case SDEV_EVT_LUN_CHANGE_REPORTED: 2695 envp[idx++] = "SDEV_UA=REPORTED_LUNS_DATA_HAS_CHANGED"; 2696 break; 2697 default: 2698 /* do nothing */ 2699 break; 2700 } 2701 2702 envp[idx++] = NULL; 2703 2704 kobject_uevent_env(&sdev->sdev_gendev.kobj, KOBJ_CHANGE, envp); 2705 } 2706 2707 /** 2708 * sdev_evt_thread - send a uevent for each scsi event 2709 * @work: work struct for scsi_device 2710 * 2711 * Dispatch queued events to their associated scsi_device kobjects 2712 * as uevents. 2713 */ 2714 void scsi_evt_thread(struct work_struct *work) 2715 { 2716 struct scsi_device *sdev; 2717 enum scsi_device_event evt_type; 2718 LIST_HEAD(event_list); 2719 2720 sdev = container_of(work, struct scsi_device, event_work); 2721 2722 for (evt_type = SDEV_EVT_FIRST; evt_type <= SDEV_EVT_LAST; evt_type++) 2723 if (test_and_clear_bit(evt_type, sdev->pending_events)) 2724 sdev_evt_send_simple(sdev, evt_type, GFP_KERNEL); 2725 2726 while (1) { 2727 struct scsi_event *evt; 2728 struct list_head *this, *tmp; 2729 unsigned long flags; 2730 2731 spin_lock_irqsave(&sdev->list_lock, flags); 2732 list_splice_init(&sdev->event_list, &event_list); 2733 spin_unlock_irqrestore(&sdev->list_lock, flags); 2734 2735 if (list_empty(&event_list)) 2736 break; 2737 2738 list_for_each_safe(this, tmp, &event_list) { 2739 evt = list_entry(this, struct scsi_event, node); 2740 list_del(&evt->node); 2741 scsi_evt_emit(sdev, evt); 2742 kfree(evt); 2743 } 2744 } 2745 } 2746 2747 /** 2748 * sdev_evt_send - send asserted event to uevent thread 2749 * @sdev: scsi_device event occurred on 2750 * @evt: event to send 2751 * 2752 * Assert scsi device event asynchronously. 2753 */ 2754 void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt) 2755 { 2756 unsigned long flags; 2757 2758 #if 0 2759 /* FIXME: currently this check eliminates all media change events 2760 * for polled devices. Need to update to discriminate between AN 2761 * and polled events */ 2762 if (!test_bit(evt->evt_type, sdev->supported_events)) { 2763 kfree(evt); 2764 return; 2765 } 2766 #endif 2767 2768 spin_lock_irqsave(&sdev->list_lock, flags); 2769 list_add_tail(&evt->node, &sdev->event_list); 2770 schedule_work(&sdev->event_work); 2771 spin_unlock_irqrestore(&sdev->list_lock, flags); 2772 } 2773 EXPORT_SYMBOL_GPL(sdev_evt_send); 2774 2775 /** 2776 * sdev_evt_alloc - allocate a new scsi event 2777 * @evt_type: type of event to allocate 2778 * @gfpflags: GFP flags for allocation 2779 * 2780 * Allocates and returns a new scsi_event. 2781 */ 2782 struct scsi_event *sdev_evt_alloc(enum scsi_device_event evt_type, 2783 gfp_t gfpflags) 2784 { 2785 struct scsi_event *evt = kzalloc(sizeof(struct scsi_event), gfpflags); 2786 if (!evt) 2787 return NULL; 2788 2789 evt->evt_type = evt_type; 2790 INIT_LIST_HEAD(&evt->node); 2791 2792 /* evt_type-specific initialization, if any */ 2793 switch (evt_type) { 2794 case SDEV_EVT_MEDIA_CHANGE: 2795 case SDEV_EVT_INQUIRY_CHANGE_REPORTED: 2796 case SDEV_EVT_CAPACITY_CHANGE_REPORTED: 2797 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED: 2798 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED: 2799 case SDEV_EVT_LUN_CHANGE_REPORTED: 2800 default: 2801 /* do nothing */ 2802 break; 2803 } 2804 2805 return evt; 2806 } 2807 EXPORT_SYMBOL_GPL(sdev_evt_alloc); 2808 2809 /** 2810 * sdev_evt_send_simple - send asserted event to uevent thread 2811 * @sdev: scsi_device event occurred on 2812 * @evt_type: type of event to send 2813 * @gfpflags: GFP flags for allocation 2814 * 2815 * Assert scsi device event asynchronously, given an event type. 2816 */ 2817 void sdev_evt_send_simple(struct scsi_device *sdev, 2818 enum scsi_device_event evt_type, gfp_t gfpflags) 2819 { 2820 struct scsi_event *evt = sdev_evt_alloc(evt_type, gfpflags); 2821 if (!evt) { 2822 sdev_printk(KERN_ERR, sdev, "event %d eaten due to OOM\n", 2823 evt_type); 2824 return; 2825 } 2826 2827 sdev_evt_send(sdev, evt); 2828 } 2829 EXPORT_SYMBOL_GPL(sdev_evt_send_simple); 2830 2831 /** 2832 * scsi_device_quiesce - Block user issued commands. 2833 * @sdev: scsi device to quiesce. 2834 * 2835 * This works by trying to transition to the SDEV_QUIESCE state 2836 * (which must be a legal transition). When the device is in this 2837 * state, only special requests will be accepted, all others will 2838 * be deferred. Since special requests may also be requeued requests, 2839 * a successful return doesn't guarantee the device will be 2840 * totally quiescent. 2841 * 2842 * Must be called with user context, may sleep. 2843 * 2844 * Returns zero if unsuccessful or an error if not. 2845 */ 2846 int 2847 scsi_device_quiesce(struct scsi_device *sdev) 2848 { 2849 int err = scsi_device_set_state(sdev, SDEV_QUIESCE); 2850 if (err) 2851 return err; 2852 2853 scsi_run_queue(sdev->request_queue); 2854 while (atomic_read(&sdev->device_busy)) { 2855 msleep_interruptible(200); 2856 scsi_run_queue(sdev->request_queue); 2857 } 2858 return 0; 2859 } 2860 EXPORT_SYMBOL(scsi_device_quiesce); 2861 2862 /** 2863 * scsi_device_resume - Restart user issued commands to a quiesced device. 2864 * @sdev: scsi device to resume. 2865 * 2866 * Moves the device from quiesced back to running and restarts the 2867 * queues. 2868 * 2869 * Must be called with user context, may sleep. 2870 */ 2871 void scsi_device_resume(struct scsi_device *sdev) 2872 { 2873 /* check if the device state was mutated prior to resume, and if 2874 * so assume the state is being managed elsewhere (for example 2875 * device deleted during suspend) 2876 */ 2877 if (sdev->sdev_state != SDEV_QUIESCE || 2878 scsi_device_set_state(sdev, SDEV_RUNNING)) 2879 return; 2880 scsi_run_queue(sdev->request_queue); 2881 } 2882 EXPORT_SYMBOL(scsi_device_resume); 2883 2884 static void 2885 device_quiesce_fn(struct scsi_device *sdev, void *data) 2886 { 2887 scsi_device_quiesce(sdev); 2888 } 2889 2890 void 2891 scsi_target_quiesce(struct scsi_target *starget) 2892 { 2893 starget_for_each_device(starget, NULL, device_quiesce_fn); 2894 } 2895 EXPORT_SYMBOL(scsi_target_quiesce); 2896 2897 static void 2898 device_resume_fn(struct scsi_device *sdev, void *data) 2899 { 2900 scsi_device_resume(sdev); 2901 } 2902 2903 void 2904 scsi_target_resume(struct scsi_target *starget) 2905 { 2906 starget_for_each_device(starget, NULL, device_resume_fn); 2907 } 2908 EXPORT_SYMBOL(scsi_target_resume); 2909 2910 /** 2911 * scsi_internal_device_block - internal function to put a device temporarily into the SDEV_BLOCK state 2912 * @sdev: device to block 2913 * 2914 * Block request made by scsi lld's to temporarily stop all 2915 * scsi commands on the specified device. Called from interrupt 2916 * or normal process context. 2917 * 2918 * Returns zero if successful or error if not 2919 * 2920 * Notes: 2921 * This routine transitions the device to the SDEV_BLOCK state 2922 * (which must be a legal transition). When the device is in this 2923 * state, all commands are deferred until the scsi lld reenables 2924 * the device with scsi_device_unblock or device_block_tmo fires. 2925 */ 2926 int 2927 scsi_internal_device_block(struct scsi_device *sdev) 2928 { 2929 struct request_queue *q = sdev->request_queue; 2930 unsigned long flags; 2931 int err = 0; 2932 2933 err = scsi_device_set_state(sdev, SDEV_BLOCK); 2934 if (err) { 2935 err = scsi_device_set_state(sdev, SDEV_CREATED_BLOCK); 2936 2937 if (err) 2938 return err; 2939 } 2940 2941 /* 2942 * The device has transitioned to SDEV_BLOCK. Stop the 2943 * block layer from calling the midlayer with this device's 2944 * request queue. 2945 */ 2946 if (q->mq_ops) { 2947 blk_mq_stop_hw_queues(q); 2948 } else { 2949 spin_lock_irqsave(q->queue_lock, flags); 2950 blk_stop_queue(q); 2951 spin_unlock_irqrestore(q->queue_lock, flags); 2952 } 2953 2954 return 0; 2955 } 2956 EXPORT_SYMBOL_GPL(scsi_internal_device_block); 2957 2958 /** 2959 * scsi_internal_device_unblock - resume a device after a block request 2960 * @sdev: device to resume 2961 * @new_state: state to set devices to after unblocking 2962 * 2963 * Called by scsi lld's or the midlayer to restart the device queue 2964 * for the previously suspended scsi device. Called from interrupt or 2965 * normal process context. 2966 * 2967 * Returns zero if successful or error if not. 2968 * 2969 * Notes: 2970 * This routine transitions the device to the SDEV_RUNNING state 2971 * or to one of the offline states (which must be a legal transition) 2972 * allowing the midlayer to goose the queue for this device. 2973 */ 2974 int 2975 scsi_internal_device_unblock(struct scsi_device *sdev, 2976 enum scsi_device_state new_state) 2977 { 2978 struct request_queue *q = sdev->request_queue; 2979 unsigned long flags; 2980 2981 /* 2982 * Try to transition the scsi device to SDEV_RUNNING or one of the 2983 * offlined states and goose the device queue if successful. 2984 */ 2985 if ((sdev->sdev_state == SDEV_BLOCK) || 2986 (sdev->sdev_state == SDEV_TRANSPORT_OFFLINE)) 2987 sdev->sdev_state = new_state; 2988 else if (sdev->sdev_state == SDEV_CREATED_BLOCK) { 2989 if (new_state == SDEV_TRANSPORT_OFFLINE || 2990 new_state == SDEV_OFFLINE) 2991 sdev->sdev_state = new_state; 2992 else 2993 sdev->sdev_state = SDEV_CREATED; 2994 } else if (sdev->sdev_state != SDEV_CANCEL && 2995 sdev->sdev_state != SDEV_OFFLINE) 2996 return -EINVAL; 2997 2998 if (q->mq_ops) { 2999 blk_mq_start_stopped_hw_queues(q, false); 3000 } else { 3001 spin_lock_irqsave(q->queue_lock, flags); 3002 blk_start_queue(q); 3003 spin_unlock_irqrestore(q->queue_lock, flags); 3004 } 3005 3006 return 0; 3007 } 3008 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock); 3009 3010 static void 3011 device_block(struct scsi_device *sdev, void *data) 3012 { 3013 scsi_internal_device_block(sdev); 3014 } 3015 3016 static int 3017 target_block(struct device *dev, void *data) 3018 { 3019 if (scsi_is_target_device(dev)) 3020 starget_for_each_device(to_scsi_target(dev), NULL, 3021 device_block); 3022 return 0; 3023 } 3024 3025 void 3026 scsi_target_block(struct device *dev) 3027 { 3028 if (scsi_is_target_device(dev)) 3029 starget_for_each_device(to_scsi_target(dev), NULL, 3030 device_block); 3031 else 3032 device_for_each_child(dev, NULL, target_block); 3033 } 3034 EXPORT_SYMBOL_GPL(scsi_target_block); 3035 3036 static void 3037 device_unblock(struct scsi_device *sdev, void *data) 3038 { 3039 scsi_internal_device_unblock(sdev, *(enum scsi_device_state *)data); 3040 } 3041 3042 static int 3043 target_unblock(struct device *dev, void *data) 3044 { 3045 if (scsi_is_target_device(dev)) 3046 starget_for_each_device(to_scsi_target(dev), data, 3047 device_unblock); 3048 return 0; 3049 } 3050 3051 void 3052 scsi_target_unblock(struct device *dev, enum scsi_device_state new_state) 3053 { 3054 if (scsi_is_target_device(dev)) 3055 starget_for_each_device(to_scsi_target(dev), &new_state, 3056 device_unblock); 3057 else 3058 device_for_each_child(dev, &new_state, target_unblock); 3059 } 3060 EXPORT_SYMBOL_GPL(scsi_target_unblock); 3061 3062 /** 3063 * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt 3064 * @sgl: scatter-gather list 3065 * @sg_count: number of segments in sg 3066 * @offset: offset in bytes into sg, on return offset into the mapped area 3067 * @len: bytes to map, on return number of bytes mapped 3068 * 3069 * Returns virtual address of the start of the mapped page 3070 */ 3071 void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count, 3072 size_t *offset, size_t *len) 3073 { 3074 int i; 3075 size_t sg_len = 0, len_complete = 0; 3076 struct scatterlist *sg; 3077 struct page *page; 3078 3079 WARN_ON(!irqs_disabled()); 3080 3081 for_each_sg(sgl, sg, sg_count, i) { 3082 len_complete = sg_len; /* Complete sg-entries */ 3083 sg_len += sg->length; 3084 if (sg_len > *offset) 3085 break; 3086 } 3087 3088 if (unlikely(i == sg_count)) { 3089 printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, " 3090 "elements %d\n", 3091 __func__, sg_len, *offset, sg_count); 3092 WARN_ON(1); 3093 return NULL; 3094 } 3095 3096 /* Offset starting from the beginning of first page in this sg-entry */ 3097 *offset = *offset - len_complete + sg->offset; 3098 3099 /* Assumption: contiguous pages can be accessed as "page + i" */ 3100 page = nth_page(sg_page(sg), (*offset >> PAGE_SHIFT)); 3101 *offset &= ~PAGE_MASK; 3102 3103 /* Bytes in this sg-entry from *offset to the end of the page */ 3104 sg_len = PAGE_SIZE - *offset; 3105 if (*len > sg_len) 3106 *len = sg_len; 3107 3108 return kmap_atomic(page); 3109 } 3110 EXPORT_SYMBOL(scsi_kmap_atomic_sg); 3111 3112 /** 3113 * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously mapped with scsi_kmap_atomic_sg 3114 * @virt: virtual address to be unmapped 3115 */ 3116 void scsi_kunmap_atomic_sg(void *virt) 3117 { 3118 kunmap_atomic(virt); 3119 } 3120 EXPORT_SYMBOL(scsi_kunmap_atomic_sg); 3121 3122 void sdev_disable_disk_events(struct scsi_device *sdev) 3123 { 3124 atomic_inc(&sdev->disk_events_disable_depth); 3125 } 3126 EXPORT_SYMBOL(sdev_disable_disk_events); 3127 3128 void sdev_enable_disk_events(struct scsi_device *sdev) 3129 { 3130 if (WARN_ON_ONCE(atomic_read(&sdev->disk_events_disable_depth) <= 0)) 3131 return; 3132 atomic_dec(&sdev->disk_events_disable_depth); 3133 } 3134 EXPORT_SYMBOL(sdev_enable_disk_events); 3135