1 /* 2 * Copyright (C) 1991, 1992 Linus Torvalds 3 * Copyright (C) 1994, Karl Keyte: Added support for disk statistics 4 * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE 5 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de> 6 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au> 7 * - July2000 8 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001 9 */ 10 11 /* 12 * This handles all read/write requests to block devices 13 */ 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/backing-dev.h> 17 #include <linux/bio.h> 18 #include <linux/blkdev.h> 19 #include <linux/blk-mq.h> 20 #include <linux/highmem.h> 21 #include <linux/mm.h> 22 #include <linux/kernel_stat.h> 23 #include <linux/string.h> 24 #include <linux/init.h> 25 #include <linux/completion.h> 26 #include <linux/slab.h> 27 #include <linux/swap.h> 28 #include <linux/writeback.h> 29 #include <linux/task_io_accounting_ops.h> 30 #include <linux/fault-inject.h> 31 #include <linux/list_sort.h> 32 #include <linux/delay.h> 33 #include <linux/ratelimit.h> 34 #include <linux/pm_runtime.h> 35 36 #define CREATE_TRACE_POINTS 37 #include <trace/events/block.h> 38 39 #include "blk.h" 40 #include "blk-cgroup.h" 41 #include "blk-mq.h" 42 43 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap); 44 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap); 45 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete); 46 EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug); 47 48 DEFINE_IDA(blk_queue_ida); 49 50 /* 51 * For the allocated request tables 52 */ 53 struct kmem_cache *request_cachep = NULL; 54 55 /* 56 * For queue allocation 57 */ 58 struct kmem_cache *blk_requestq_cachep; 59 60 /* 61 * Controlling structure to kblockd 62 */ 63 static struct workqueue_struct *kblockd_workqueue; 64 65 void blk_queue_congestion_threshold(struct request_queue *q) 66 { 67 int nr; 68 69 nr = q->nr_requests - (q->nr_requests / 8) + 1; 70 if (nr > q->nr_requests) 71 nr = q->nr_requests; 72 q->nr_congestion_on = nr; 73 74 nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1; 75 if (nr < 1) 76 nr = 1; 77 q->nr_congestion_off = nr; 78 } 79 80 /** 81 * blk_get_backing_dev_info - get the address of a queue's backing_dev_info 82 * @bdev: device 83 * 84 * Locates the passed device's request queue and returns the address of its 85 * backing_dev_info 86 * 87 * Will return NULL if the request queue cannot be located. 88 */ 89 struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev) 90 { 91 struct backing_dev_info *ret = NULL; 92 struct request_queue *q = bdev_get_queue(bdev); 93 94 if (q) 95 ret = &q->backing_dev_info; 96 return ret; 97 } 98 EXPORT_SYMBOL(blk_get_backing_dev_info); 99 100 void blk_rq_init(struct request_queue *q, struct request *rq) 101 { 102 memset(rq, 0, sizeof(*rq)); 103 104 INIT_LIST_HEAD(&rq->queuelist); 105 INIT_LIST_HEAD(&rq->timeout_list); 106 rq->cpu = -1; 107 rq->q = q; 108 rq->__sector = (sector_t) -1; 109 INIT_HLIST_NODE(&rq->hash); 110 RB_CLEAR_NODE(&rq->rb_node); 111 rq->cmd = rq->__cmd; 112 rq->cmd_len = BLK_MAX_CDB; 113 rq->tag = -1; 114 rq->start_time = jiffies; 115 set_start_time_ns(rq); 116 rq->part = NULL; 117 } 118 EXPORT_SYMBOL(blk_rq_init); 119 120 static void req_bio_endio(struct request *rq, struct bio *bio, 121 unsigned int nbytes, int error) 122 { 123 if (error) 124 clear_bit(BIO_UPTODATE, &bio->bi_flags); 125 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) 126 error = -EIO; 127 128 if (unlikely(rq->cmd_flags & REQ_QUIET)) 129 set_bit(BIO_QUIET, &bio->bi_flags); 130 131 bio_advance(bio, nbytes); 132 133 /* don't actually finish bio if it's part of flush sequence */ 134 if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ)) 135 bio_endio(bio, error); 136 } 137 138 void blk_dump_rq_flags(struct request *rq, char *msg) 139 { 140 int bit; 141 142 printk(KERN_INFO "%s: dev %s: type=%x, flags=%llx\n", msg, 143 rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type, 144 (unsigned long long) rq->cmd_flags); 145 146 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n", 147 (unsigned long long)blk_rq_pos(rq), 148 blk_rq_sectors(rq), blk_rq_cur_sectors(rq)); 149 printk(KERN_INFO " bio %p, biotail %p, buffer %p, len %u\n", 150 rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq)); 151 152 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { 153 printk(KERN_INFO " cdb: "); 154 for (bit = 0; bit < BLK_MAX_CDB; bit++) 155 printk("%02x ", rq->cmd[bit]); 156 printk("\n"); 157 } 158 } 159 EXPORT_SYMBOL(blk_dump_rq_flags); 160 161 static void blk_delay_work(struct work_struct *work) 162 { 163 struct request_queue *q; 164 165 q = container_of(work, struct request_queue, delay_work.work); 166 spin_lock_irq(q->queue_lock); 167 __blk_run_queue(q); 168 spin_unlock_irq(q->queue_lock); 169 } 170 171 /** 172 * blk_delay_queue - restart queueing after defined interval 173 * @q: The &struct request_queue in question 174 * @msecs: Delay in msecs 175 * 176 * Description: 177 * Sometimes queueing needs to be postponed for a little while, to allow 178 * resources to come back. This function will make sure that queueing is 179 * restarted around the specified time. Queue lock must be held. 180 */ 181 void blk_delay_queue(struct request_queue *q, unsigned long msecs) 182 { 183 if (likely(!blk_queue_dead(q))) 184 queue_delayed_work(kblockd_workqueue, &q->delay_work, 185 msecs_to_jiffies(msecs)); 186 } 187 EXPORT_SYMBOL(blk_delay_queue); 188 189 /** 190 * blk_start_queue - restart a previously stopped queue 191 * @q: The &struct request_queue in question 192 * 193 * Description: 194 * blk_start_queue() will clear the stop flag on the queue, and call 195 * the request_fn for the queue if it was in a stopped state when 196 * entered. Also see blk_stop_queue(). Queue lock must be held. 197 **/ 198 void blk_start_queue(struct request_queue *q) 199 { 200 WARN_ON(!irqs_disabled()); 201 202 queue_flag_clear(QUEUE_FLAG_STOPPED, q); 203 __blk_run_queue(q); 204 } 205 EXPORT_SYMBOL(blk_start_queue); 206 207 /** 208 * blk_stop_queue - stop a queue 209 * @q: The &struct request_queue in question 210 * 211 * Description: 212 * The Linux block layer assumes that a block driver will consume all 213 * entries on the request queue when the request_fn strategy is called. 214 * Often this will not happen, because of hardware limitations (queue 215 * depth settings). If a device driver gets a 'queue full' response, 216 * or if it simply chooses not to queue more I/O at one point, it can 217 * call this function to prevent the request_fn from being called until 218 * the driver has signalled it's ready to go again. This happens by calling 219 * blk_start_queue() to restart queue operations. Queue lock must be held. 220 **/ 221 void blk_stop_queue(struct request_queue *q) 222 { 223 cancel_delayed_work(&q->delay_work); 224 queue_flag_set(QUEUE_FLAG_STOPPED, q); 225 } 226 EXPORT_SYMBOL(blk_stop_queue); 227 228 /** 229 * blk_sync_queue - cancel any pending callbacks on a queue 230 * @q: the queue 231 * 232 * Description: 233 * The block layer may perform asynchronous callback activity 234 * on a queue, such as calling the unplug function after a timeout. 235 * A block device may call blk_sync_queue to ensure that any 236 * such activity is cancelled, thus allowing it to release resources 237 * that the callbacks might use. The caller must already have made sure 238 * that its ->make_request_fn will not re-add plugging prior to calling 239 * this function. 240 * 241 * This function does not cancel any asynchronous activity arising 242 * out of elevator or throttling code. That would require elevaotor_exit() 243 * and blkcg_exit_queue() to be called with queue lock initialized. 244 * 245 */ 246 void blk_sync_queue(struct request_queue *q) 247 { 248 del_timer_sync(&q->timeout); 249 250 if (q->mq_ops) { 251 struct blk_mq_hw_ctx *hctx; 252 int i; 253 254 queue_for_each_hw_ctx(q, hctx, i) 255 cancel_delayed_work_sync(&hctx->delayed_work); 256 } else { 257 cancel_delayed_work_sync(&q->delay_work); 258 } 259 } 260 EXPORT_SYMBOL(blk_sync_queue); 261 262 /** 263 * __blk_run_queue_uncond - run a queue whether or not it has been stopped 264 * @q: The queue to run 265 * 266 * Description: 267 * Invoke request handling on a queue if there are any pending requests. 268 * May be used to restart request handling after a request has completed. 269 * This variant runs the queue whether or not the queue has been 270 * stopped. Must be called with the queue lock held and interrupts 271 * disabled. See also @blk_run_queue. 272 */ 273 inline void __blk_run_queue_uncond(struct request_queue *q) 274 { 275 if (unlikely(blk_queue_dead(q))) 276 return; 277 278 /* 279 * Some request_fn implementations, e.g. scsi_request_fn(), unlock 280 * the queue lock internally. As a result multiple threads may be 281 * running such a request function concurrently. Keep track of the 282 * number of active request_fn invocations such that blk_drain_queue() 283 * can wait until all these request_fn calls have finished. 284 */ 285 q->request_fn_active++; 286 q->request_fn(q); 287 q->request_fn_active--; 288 } 289 290 /** 291 * __blk_run_queue - run a single device queue 292 * @q: The queue to run 293 * 294 * Description: 295 * See @blk_run_queue. This variant must be called with the queue lock 296 * held and interrupts disabled. 297 */ 298 void __blk_run_queue(struct request_queue *q) 299 { 300 if (unlikely(blk_queue_stopped(q))) 301 return; 302 303 __blk_run_queue_uncond(q); 304 } 305 EXPORT_SYMBOL(__blk_run_queue); 306 307 /** 308 * blk_run_queue_async - run a single device queue in workqueue context 309 * @q: The queue to run 310 * 311 * Description: 312 * Tells kblockd to perform the equivalent of @blk_run_queue on behalf 313 * of us. The caller must hold the queue lock. 314 */ 315 void blk_run_queue_async(struct request_queue *q) 316 { 317 if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q))) 318 mod_delayed_work(kblockd_workqueue, &q->delay_work, 0); 319 } 320 EXPORT_SYMBOL(blk_run_queue_async); 321 322 /** 323 * blk_run_queue - run a single device queue 324 * @q: The queue to run 325 * 326 * Description: 327 * Invoke request handling on this queue, if it has pending work to do. 328 * May be used to restart queueing when a request has completed. 329 */ 330 void blk_run_queue(struct request_queue *q) 331 { 332 unsigned long flags; 333 334 spin_lock_irqsave(q->queue_lock, flags); 335 __blk_run_queue(q); 336 spin_unlock_irqrestore(q->queue_lock, flags); 337 } 338 EXPORT_SYMBOL(blk_run_queue); 339 340 void blk_put_queue(struct request_queue *q) 341 { 342 kobject_put(&q->kobj); 343 } 344 EXPORT_SYMBOL(blk_put_queue); 345 346 /** 347 * __blk_drain_queue - drain requests from request_queue 348 * @q: queue to drain 349 * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV 350 * 351 * Drain requests from @q. If @drain_all is set, all requests are drained. 352 * If not, only ELVPRIV requests are drained. The caller is responsible 353 * for ensuring that no new requests which need to be drained are queued. 354 */ 355 static void __blk_drain_queue(struct request_queue *q, bool drain_all) 356 __releases(q->queue_lock) 357 __acquires(q->queue_lock) 358 { 359 int i; 360 361 lockdep_assert_held(q->queue_lock); 362 363 while (true) { 364 bool drain = false; 365 366 /* 367 * The caller might be trying to drain @q before its 368 * elevator is initialized. 369 */ 370 if (q->elevator) 371 elv_drain_elevator(q); 372 373 blkcg_drain_queue(q); 374 375 /* 376 * This function might be called on a queue which failed 377 * driver init after queue creation or is not yet fully 378 * active yet. Some drivers (e.g. fd and loop) get unhappy 379 * in such cases. Kick queue iff dispatch queue has 380 * something on it and @q has request_fn set. 381 */ 382 if (!list_empty(&q->queue_head) && q->request_fn) 383 __blk_run_queue(q); 384 385 drain |= q->nr_rqs_elvpriv; 386 drain |= q->request_fn_active; 387 388 /* 389 * Unfortunately, requests are queued at and tracked from 390 * multiple places and there's no single counter which can 391 * be drained. Check all the queues and counters. 392 */ 393 if (drain_all) { 394 drain |= !list_empty(&q->queue_head); 395 for (i = 0; i < 2; i++) { 396 drain |= q->nr_rqs[i]; 397 drain |= q->in_flight[i]; 398 drain |= !list_empty(&q->flush_queue[i]); 399 } 400 } 401 402 if (!drain) 403 break; 404 405 spin_unlock_irq(q->queue_lock); 406 407 msleep(10); 408 409 spin_lock_irq(q->queue_lock); 410 } 411 412 /* 413 * With queue marked dead, any woken up waiter will fail the 414 * allocation path, so the wakeup chaining is lost and we're 415 * left with hung waiters. We need to wake up those waiters. 416 */ 417 if (q->request_fn) { 418 struct request_list *rl; 419 420 blk_queue_for_each_rl(rl, q) 421 for (i = 0; i < ARRAY_SIZE(rl->wait); i++) 422 wake_up_all(&rl->wait[i]); 423 } 424 } 425 426 /** 427 * blk_queue_bypass_start - enter queue bypass mode 428 * @q: queue of interest 429 * 430 * In bypass mode, only the dispatch FIFO queue of @q is used. This 431 * function makes @q enter bypass mode and drains all requests which were 432 * throttled or issued before. On return, it's guaranteed that no request 433 * is being throttled or has ELVPRIV set and blk_queue_bypass() %true 434 * inside queue or RCU read lock. 435 */ 436 void blk_queue_bypass_start(struct request_queue *q) 437 { 438 bool drain; 439 440 spin_lock_irq(q->queue_lock); 441 drain = !q->bypass_depth++; 442 queue_flag_set(QUEUE_FLAG_BYPASS, q); 443 spin_unlock_irq(q->queue_lock); 444 445 if (drain) { 446 spin_lock_irq(q->queue_lock); 447 __blk_drain_queue(q, false); 448 spin_unlock_irq(q->queue_lock); 449 450 /* ensure blk_queue_bypass() is %true inside RCU read lock */ 451 synchronize_rcu(); 452 } 453 } 454 EXPORT_SYMBOL_GPL(blk_queue_bypass_start); 455 456 /** 457 * blk_queue_bypass_end - leave queue bypass mode 458 * @q: queue of interest 459 * 460 * Leave bypass mode and restore the normal queueing behavior. 461 */ 462 void blk_queue_bypass_end(struct request_queue *q) 463 { 464 spin_lock_irq(q->queue_lock); 465 if (!--q->bypass_depth) 466 queue_flag_clear(QUEUE_FLAG_BYPASS, q); 467 WARN_ON_ONCE(q->bypass_depth < 0); 468 spin_unlock_irq(q->queue_lock); 469 } 470 EXPORT_SYMBOL_GPL(blk_queue_bypass_end); 471 472 /** 473 * blk_cleanup_queue - shutdown a request queue 474 * @q: request queue to shutdown 475 * 476 * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and 477 * put it. All future requests will be failed immediately with -ENODEV. 478 */ 479 void blk_cleanup_queue(struct request_queue *q) 480 { 481 spinlock_t *lock = q->queue_lock; 482 483 /* mark @q DYING, no new request or merges will be allowed afterwards */ 484 mutex_lock(&q->sysfs_lock); 485 queue_flag_set_unlocked(QUEUE_FLAG_DYING, q); 486 spin_lock_irq(lock); 487 488 /* 489 * A dying queue is permanently in bypass mode till released. Note 490 * that, unlike blk_queue_bypass_start(), we aren't performing 491 * synchronize_rcu() after entering bypass mode to avoid the delay 492 * as some drivers create and destroy a lot of queues while 493 * probing. This is still safe because blk_release_queue() will be 494 * called only after the queue refcnt drops to zero and nothing, 495 * RCU or not, would be traversing the queue by then. 496 */ 497 q->bypass_depth++; 498 queue_flag_set(QUEUE_FLAG_BYPASS, q); 499 500 queue_flag_set(QUEUE_FLAG_NOMERGES, q); 501 queue_flag_set(QUEUE_FLAG_NOXMERGES, q); 502 queue_flag_set(QUEUE_FLAG_DYING, q); 503 spin_unlock_irq(lock); 504 mutex_unlock(&q->sysfs_lock); 505 506 /* 507 * Drain all requests queued before DYING marking. Set DEAD flag to 508 * prevent that q->request_fn() gets invoked after draining finished. 509 */ 510 if (q->mq_ops) { 511 blk_mq_drain_queue(q); 512 spin_lock_irq(lock); 513 } else { 514 spin_lock_irq(lock); 515 __blk_drain_queue(q, true); 516 } 517 queue_flag_set(QUEUE_FLAG_DEAD, q); 518 spin_unlock_irq(lock); 519 520 /* @q won't process any more request, flush async actions */ 521 del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer); 522 blk_sync_queue(q); 523 524 spin_lock_irq(lock); 525 if (q->queue_lock != &q->__queue_lock) 526 q->queue_lock = &q->__queue_lock; 527 spin_unlock_irq(lock); 528 529 /* @q is and will stay empty, shutdown and put */ 530 blk_put_queue(q); 531 } 532 EXPORT_SYMBOL(blk_cleanup_queue); 533 534 int blk_init_rl(struct request_list *rl, struct request_queue *q, 535 gfp_t gfp_mask) 536 { 537 if (unlikely(rl->rq_pool)) 538 return 0; 539 540 rl->q = q; 541 rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0; 542 rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0; 543 init_waitqueue_head(&rl->wait[BLK_RW_SYNC]); 544 init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]); 545 546 rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab, 547 mempool_free_slab, request_cachep, 548 gfp_mask, q->node); 549 if (!rl->rq_pool) 550 return -ENOMEM; 551 552 return 0; 553 } 554 555 void blk_exit_rl(struct request_list *rl) 556 { 557 if (rl->rq_pool) 558 mempool_destroy(rl->rq_pool); 559 } 560 561 struct request_queue *blk_alloc_queue(gfp_t gfp_mask) 562 { 563 return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE); 564 } 565 EXPORT_SYMBOL(blk_alloc_queue); 566 567 struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) 568 { 569 struct request_queue *q; 570 int err; 571 572 q = kmem_cache_alloc_node(blk_requestq_cachep, 573 gfp_mask | __GFP_ZERO, node_id); 574 if (!q) 575 return NULL; 576 577 if (percpu_counter_init(&q->mq_usage_counter, 0)) 578 goto fail_q; 579 580 q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask); 581 if (q->id < 0) 582 goto fail_c; 583 584 q->backing_dev_info.ra_pages = 585 (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; 586 q->backing_dev_info.state = 0; 587 q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY; 588 q->backing_dev_info.name = "block"; 589 q->node = node_id; 590 591 err = bdi_init(&q->backing_dev_info); 592 if (err) 593 goto fail_id; 594 595 setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, 596 laptop_mode_timer_fn, (unsigned long) q); 597 setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q); 598 INIT_LIST_HEAD(&q->queue_head); 599 INIT_LIST_HEAD(&q->timeout_list); 600 INIT_LIST_HEAD(&q->icq_list); 601 #ifdef CONFIG_BLK_CGROUP 602 INIT_LIST_HEAD(&q->blkg_list); 603 #endif 604 INIT_LIST_HEAD(&q->flush_queue[0]); 605 INIT_LIST_HEAD(&q->flush_queue[1]); 606 INIT_LIST_HEAD(&q->flush_data_in_flight); 607 INIT_DELAYED_WORK(&q->delay_work, blk_delay_work); 608 609 kobject_init(&q->kobj, &blk_queue_ktype); 610 611 mutex_init(&q->sysfs_lock); 612 spin_lock_init(&q->__queue_lock); 613 614 /* 615 * By default initialize queue_lock to internal lock and driver can 616 * override it later if need be. 617 */ 618 q->queue_lock = &q->__queue_lock; 619 620 /* 621 * A queue starts its life with bypass turned on to avoid 622 * unnecessary bypass on/off overhead and nasty surprises during 623 * init. The initial bypass will be finished when the queue is 624 * registered by blk_register_queue(). 625 */ 626 q->bypass_depth = 1; 627 __set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags); 628 629 init_waitqueue_head(&q->mq_freeze_wq); 630 631 if (blkcg_init_queue(q)) 632 goto fail_bdi; 633 634 return q; 635 636 fail_bdi: 637 bdi_destroy(&q->backing_dev_info); 638 fail_id: 639 ida_simple_remove(&blk_queue_ida, q->id); 640 fail_c: 641 percpu_counter_destroy(&q->mq_usage_counter); 642 fail_q: 643 kmem_cache_free(blk_requestq_cachep, q); 644 return NULL; 645 } 646 EXPORT_SYMBOL(blk_alloc_queue_node); 647 648 /** 649 * blk_init_queue - prepare a request queue for use with a block device 650 * @rfn: The function to be called to process requests that have been 651 * placed on the queue. 652 * @lock: Request queue spin lock 653 * 654 * Description: 655 * If a block device wishes to use the standard request handling procedures, 656 * which sorts requests and coalesces adjacent requests, then it must 657 * call blk_init_queue(). The function @rfn will be called when there 658 * are requests on the queue that need to be processed. If the device 659 * supports plugging, then @rfn may not be called immediately when requests 660 * are available on the queue, but may be called at some time later instead. 661 * Plugged queues are generally unplugged when a buffer belonging to one 662 * of the requests on the queue is needed, or due to memory pressure. 663 * 664 * @rfn is not required, or even expected, to remove all requests off the 665 * queue, but only as many as it can handle at a time. If it does leave 666 * requests on the queue, it is responsible for arranging that the requests 667 * get dealt with eventually. 668 * 669 * The queue spin lock must be held while manipulating the requests on the 670 * request queue; this lock will be taken also from interrupt context, so irq 671 * disabling is needed for it. 672 * 673 * Function returns a pointer to the initialized request queue, or %NULL if 674 * it didn't succeed. 675 * 676 * Note: 677 * blk_init_queue() must be paired with a blk_cleanup_queue() call 678 * when the block device is deactivated (such as at module unload). 679 **/ 680 681 struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock) 682 { 683 return blk_init_queue_node(rfn, lock, NUMA_NO_NODE); 684 } 685 EXPORT_SYMBOL(blk_init_queue); 686 687 struct request_queue * 688 blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id) 689 { 690 struct request_queue *uninit_q, *q; 691 692 uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id); 693 if (!uninit_q) 694 return NULL; 695 696 q = blk_init_allocated_queue(uninit_q, rfn, lock); 697 if (!q) 698 blk_cleanup_queue(uninit_q); 699 700 return q; 701 } 702 EXPORT_SYMBOL(blk_init_queue_node); 703 704 struct request_queue * 705 blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn, 706 spinlock_t *lock) 707 { 708 if (!q) 709 return NULL; 710 711 if (blk_init_rl(&q->root_rl, q, GFP_KERNEL)) 712 return NULL; 713 714 q->request_fn = rfn; 715 q->prep_rq_fn = NULL; 716 q->unprep_rq_fn = NULL; 717 q->queue_flags |= QUEUE_FLAG_DEFAULT; 718 719 /* Override internal queue lock with supplied lock pointer */ 720 if (lock) 721 q->queue_lock = lock; 722 723 /* 724 * This also sets hw/phys segments, boundary and size 725 */ 726 blk_queue_make_request(q, blk_queue_bio); 727 728 q->sg_reserved_size = INT_MAX; 729 730 /* Protect q->elevator from elevator_change */ 731 mutex_lock(&q->sysfs_lock); 732 733 /* init elevator */ 734 if (elevator_init(q, NULL)) { 735 mutex_unlock(&q->sysfs_lock); 736 return NULL; 737 } 738 739 mutex_unlock(&q->sysfs_lock); 740 741 return q; 742 } 743 EXPORT_SYMBOL(blk_init_allocated_queue); 744 745 bool blk_get_queue(struct request_queue *q) 746 { 747 if (likely(!blk_queue_dying(q))) { 748 __blk_get_queue(q); 749 return true; 750 } 751 752 return false; 753 } 754 EXPORT_SYMBOL(blk_get_queue); 755 756 static inline void blk_free_request(struct request_list *rl, struct request *rq) 757 { 758 if (rq->cmd_flags & REQ_ELVPRIV) { 759 elv_put_request(rl->q, rq); 760 if (rq->elv.icq) 761 put_io_context(rq->elv.icq->ioc); 762 } 763 764 mempool_free(rq, rl->rq_pool); 765 } 766 767 /* 768 * ioc_batching returns true if the ioc is a valid batching request and 769 * should be given priority access to a request. 770 */ 771 static inline int ioc_batching(struct request_queue *q, struct io_context *ioc) 772 { 773 if (!ioc) 774 return 0; 775 776 /* 777 * Make sure the process is able to allocate at least 1 request 778 * even if the batch times out, otherwise we could theoretically 779 * lose wakeups. 780 */ 781 return ioc->nr_batch_requests == q->nr_batching || 782 (ioc->nr_batch_requests > 0 783 && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME)); 784 } 785 786 /* 787 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This 788 * will cause the process to be a "batcher" on all queues in the system. This 789 * is the behaviour we want though - once it gets a wakeup it should be given 790 * a nice run. 791 */ 792 static void ioc_set_batching(struct request_queue *q, struct io_context *ioc) 793 { 794 if (!ioc || ioc_batching(q, ioc)) 795 return; 796 797 ioc->nr_batch_requests = q->nr_batching; 798 ioc->last_waited = jiffies; 799 } 800 801 static void __freed_request(struct request_list *rl, int sync) 802 { 803 struct request_queue *q = rl->q; 804 805 /* 806 * bdi isn't aware of blkcg yet. As all async IOs end up root 807 * blkcg anyway, just use root blkcg state. 808 */ 809 if (rl == &q->root_rl && 810 rl->count[sync] < queue_congestion_off_threshold(q)) 811 blk_clear_queue_congested(q, sync); 812 813 if (rl->count[sync] + 1 <= q->nr_requests) { 814 if (waitqueue_active(&rl->wait[sync])) 815 wake_up(&rl->wait[sync]); 816 817 blk_clear_rl_full(rl, sync); 818 } 819 } 820 821 /* 822 * A request has just been released. Account for it, update the full and 823 * congestion status, wake up any waiters. Called under q->queue_lock. 824 */ 825 static void freed_request(struct request_list *rl, unsigned int flags) 826 { 827 struct request_queue *q = rl->q; 828 int sync = rw_is_sync(flags); 829 830 q->nr_rqs[sync]--; 831 rl->count[sync]--; 832 if (flags & REQ_ELVPRIV) 833 q->nr_rqs_elvpriv--; 834 835 __freed_request(rl, sync); 836 837 if (unlikely(rl->starved[sync ^ 1])) 838 __freed_request(rl, sync ^ 1); 839 } 840 841 /* 842 * Determine if elevator data should be initialized when allocating the 843 * request associated with @bio. 844 */ 845 static bool blk_rq_should_init_elevator(struct bio *bio) 846 { 847 if (!bio) 848 return true; 849 850 /* 851 * Flush requests do not use the elevator so skip initialization. 852 * This allows a request to share the flush and elevator data. 853 */ 854 if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) 855 return false; 856 857 return true; 858 } 859 860 /** 861 * rq_ioc - determine io_context for request allocation 862 * @bio: request being allocated is for this bio (can be %NULL) 863 * 864 * Determine io_context to use for request allocation for @bio. May return 865 * %NULL if %current->io_context doesn't exist. 866 */ 867 static struct io_context *rq_ioc(struct bio *bio) 868 { 869 #ifdef CONFIG_BLK_CGROUP 870 if (bio && bio->bi_ioc) 871 return bio->bi_ioc; 872 #endif 873 return current->io_context; 874 } 875 876 /** 877 * __get_request - get a free request 878 * @rl: request list to allocate from 879 * @rw_flags: RW and SYNC flags 880 * @bio: bio to allocate request for (can be %NULL) 881 * @gfp_mask: allocation mask 882 * 883 * Get a free request from @q. This function may fail under memory 884 * pressure or if @q is dead. 885 * 886 * Must be callled with @q->queue_lock held and, 887 * Returns %NULL on failure, with @q->queue_lock held. 888 * Returns !%NULL on success, with @q->queue_lock *not held*. 889 */ 890 static struct request *__get_request(struct request_list *rl, int rw_flags, 891 struct bio *bio, gfp_t gfp_mask) 892 { 893 struct request_queue *q = rl->q; 894 struct request *rq; 895 struct elevator_type *et = q->elevator->type; 896 struct io_context *ioc = rq_ioc(bio); 897 struct io_cq *icq = NULL; 898 const bool is_sync = rw_is_sync(rw_flags) != 0; 899 int may_queue; 900 901 if (unlikely(blk_queue_dying(q))) 902 return NULL; 903 904 may_queue = elv_may_queue(q, rw_flags); 905 if (may_queue == ELV_MQUEUE_NO) 906 goto rq_starved; 907 908 if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) { 909 if (rl->count[is_sync]+1 >= q->nr_requests) { 910 /* 911 * The queue will fill after this allocation, so set 912 * it as full, and mark this process as "batching". 913 * This process will be allowed to complete a batch of 914 * requests, others will be blocked. 915 */ 916 if (!blk_rl_full(rl, is_sync)) { 917 ioc_set_batching(q, ioc); 918 blk_set_rl_full(rl, is_sync); 919 } else { 920 if (may_queue != ELV_MQUEUE_MUST 921 && !ioc_batching(q, ioc)) { 922 /* 923 * The queue is full and the allocating 924 * process is not a "batcher", and not 925 * exempted by the IO scheduler 926 */ 927 return NULL; 928 } 929 } 930 } 931 /* 932 * bdi isn't aware of blkcg yet. As all async IOs end up 933 * root blkcg anyway, just use root blkcg state. 934 */ 935 if (rl == &q->root_rl) 936 blk_set_queue_congested(q, is_sync); 937 } 938 939 /* 940 * Only allow batching queuers to allocate up to 50% over the defined 941 * limit of requests, otherwise we could have thousands of requests 942 * allocated with any setting of ->nr_requests 943 */ 944 if (rl->count[is_sync] >= (3 * q->nr_requests / 2)) 945 return NULL; 946 947 q->nr_rqs[is_sync]++; 948 rl->count[is_sync]++; 949 rl->starved[is_sync] = 0; 950 951 /* 952 * Decide whether the new request will be managed by elevator. If 953 * so, mark @rw_flags and increment elvpriv. Non-zero elvpriv will 954 * prevent the current elevator from being destroyed until the new 955 * request is freed. This guarantees icq's won't be destroyed and 956 * makes creating new ones safe. 957 * 958 * Also, lookup icq while holding queue_lock. If it doesn't exist, 959 * it will be created after releasing queue_lock. 960 */ 961 if (blk_rq_should_init_elevator(bio) && !blk_queue_bypass(q)) { 962 rw_flags |= REQ_ELVPRIV; 963 q->nr_rqs_elvpriv++; 964 if (et->icq_cache && ioc) 965 icq = ioc_lookup_icq(ioc, q); 966 } 967 968 if (blk_queue_io_stat(q)) 969 rw_flags |= REQ_IO_STAT; 970 spin_unlock_irq(q->queue_lock); 971 972 /* allocate and init request */ 973 rq = mempool_alloc(rl->rq_pool, gfp_mask); 974 if (!rq) 975 goto fail_alloc; 976 977 blk_rq_init(q, rq); 978 blk_rq_set_rl(rq, rl); 979 rq->cmd_flags = rw_flags | REQ_ALLOCED; 980 981 /* init elvpriv */ 982 if (rw_flags & REQ_ELVPRIV) { 983 if (unlikely(et->icq_cache && !icq)) { 984 if (ioc) 985 icq = ioc_create_icq(ioc, q, gfp_mask); 986 if (!icq) 987 goto fail_elvpriv; 988 } 989 990 rq->elv.icq = icq; 991 if (unlikely(elv_set_request(q, rq, bio, gfp_mask))) 992 goto fail_elvpriv; 993 994 /* @rq->elv.icq holds io_context until @rq is freed */ 995 if (icq) 996 get_io_context(icq->ioc); 997 } 998 out: 999 /* 1000 * ioc may be NULL here, and ioc_batching will be false. That's 1001 * OK, if the queue is under the request limit then requests need 1002 * not count toward the nr_batch_requests limit. There will always 1003 * be some limit enforced by BLK_BATCH_TIME. 1004 */ 1005 if (ioc_batching(q, ioc)) 1006 ioc->nr_batch_requests--; 1007 1008 trace_block_getrq(q, bio, rw_flags & 1); 1009 return rq; 1010 1011 fail_elvpriv: 1012 /* 1013 * elvpriv init failed. ioc, icq and elvpriv aren't mempool backed 1014 * and may fail indefinitely under memory pressure and thus 1015 * shouldn't stall IO. Treat this request as !elvpriv. This will 1016 * disturb iosched and blkcg but weird is bettern than dead. 1017 */ 1018 printk_ratelimited(KERN_WARNING "%s: request aux data allocation failed, iosched may be disturbed\n", 1019 dev_name(q->backing_dev_info.dev)); 1020 1021 rq->cmd_flags &= ~REQ_ELVPRIV; 1022 rq->elv.icq = NULL; 1023 1024 spin_lock_irq(q->queue_lock); 1025 q->nr_rqs_elvpriv--; 1026 spin_unlock_irq(q->queue_lock); 1027 goto out; 1028 1029 fail_alloc: 1030 /* 1031 * Allocation failed presumably due to memory. Undo anything we 1032 * might have messed up. 1033 * 1034 * Allocating task should really be put onto the front of the wait 1035 * queue, but this is pretty rare. 1036 */ 1037 spin_lock_irq(q->queue_lock); 1038 freed_request(rl, rw_flags); 1039 1040 /* 1041 * in the very unlikely event that allocation failed and no 1042 * requests for this direction was pending, mark us starved so that 1043 * freeing of a request in the other direction will notice 1044 * us. another possible fix would be to split the rq mempool into 1045 * READ and WRITE 1046 */ 1047 rq_starved: 1048 if (unlikely(rl->count[is_sync] == 0)) 1049 rl->starved[is_sync] = 1; 1050 return NULL; 1051 } 1052 1053 /** 1054 * get_request - get a free request 1055 * @q: request_queue to allocate request from 1056 * @rw_flags: RW and SYNC flags 1057 * @bio: bio to allocate request for (can be %NULL) 1058 * @gfp_mask: allocation mask 1059 * 1060 * Get a free request from @q. If %__GFP_WAIT is set in @gfp_mask, this 1061 * function keeps retrying under memory pressure and fails iff @q is dead. 1062 * 1063 * Must be callled with @q->queue_lock held and, 1064 * Returns %NULL on failure, with @q->queue_lock held. 1065 * Returns !%NULL on success, with @q->queue_lock *not held*. 1066 */ 1067 static struct request *get_request(struct request_queue *q, int rw_flags, 1068 struct bio *bio, gfp_t gfp_mask) 1069 { 1070 const bool is_sync = rw_is_sync(rw_flags) != 0; 1071 DEFINE_WAIT(wait); 1072 struct request_list *rl; 1073 struct request *rq; 1074 1075 rl = blk_get_rl(q, bio); /* transferred to @rq on success */ 1076 retry: 1077 rq = __get_request(rl, rw_flags, bio, gfp_mask); 1078 if (rq) 1079 return rq; 1080 1081 if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dying(q))) { 1082 blk_put_rl(rl); 1083 return NULL; 1084 } 1085 1086 /* wait on @rl and retry */ 1087 prepare_to_wait_exclusive(&rl->wait[is_sync], &wait, 1088 TASK_UNINTERRUPTIBLE); 1089 1090 trace_block_sleeprq(q, bio, rw_flags & 1); 1091 1092 spin_unlock_irq(q->queue_lock); 1093 io_schedule(); 1094 1095 /* 1096 * After sleeping, we become a "batching" process and will be able 1097 * to allocate at least one request, and up to a big batch of them 1098 * for a small period time. See ioc_batching, ioc_set_batching 1099 */ 1100 ioc_set_batching(q, current->io_context); 1101 1102 spin_lock_irq(q->queue_lock); 1103 finish_wait(&rl->wait[is_sync], &wait); 1104 1105 goto retry; 1106 } 1107 1108 static struct request *blk_old_get_request(struct request_queue *q, int rw, 1109 gfp_t gfp_mask) 1110 { 1111 struct request *rq; 1112 1113 BUG_ON(rw != READ && rw != WRITE); 1114 1115 /* create ioc upfront */ 1116 create_io_context(gfp_mask, q->node); 1117 1118 spin_lock_irq(q->queue_lock); 1119 rq = get_request(q, rw, NULL, gfp_mask); 1120 if (!rq) 1121 spin_unlock_irq(q->queue_lock); 1122 /* q->queue_lock is unlocked at this point */ 1123 1124 return rq; 1125 } 1126 1127 struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask) 1128 { 1129 if (q->mq_ops) 1130 return blk_mq_alloc_request(q, rw, gfp_mask, false); 1131 else 1132 return blk_old_get_request(q, rw, gfp_mask); 1133 } 1134 EXPORT_SYMBOL(blk_get_request); 1135 1136 /** 1137 * blk_make_request - given a bio, allocate a corresponding struct request. 1138 * @q: target request queue 1139 * @bio: The bio describing the memory mappings that will be submitted for IO. 1140 * It may be a chained-bio properly constructed by block/bio layer. 1141 * @gfp_mask: gfp flags to be used for memory allocation 1142 * 1143 * blk_make_request is the parallel of generic_make_request for BLOCK_PC 1144 * type commands. Where the struct request needs to be farther initialized by 1145 * the caller. It is passed a &struct bio, which describes the memory info of 1146 * the I/O transfer. 1147 * 1148 * The caller of blk_make_request must make sure that bi_io_vec 1149 * are set to describe the memory buffers. That bio_data_dir() will return 1150 * the needed direction of the request. (And all bio's in the passed bio-chain 1151 * are properly set accordingly) 1152 * 1153 * If called under none-sleepable conditions, mapped bio buffers must not 1154 * need bouncing, by calling the appropriate masked or flagged allocator, 1155 * suitable for the target device. Otherwise the call to blk_queue_bounce will 1156 * BUG. 1157 * 1158 * WARNING: When allocating/cloning a bio-chain, careful consideration should be 1159 * given to how you allocate bios. In particular, you cannot use __GFP_WAIT for 1160 * anything but the first bio in the chain. Otherwise you risk waiting for IO 1161 * completion of a bio that hasn't been submitted yet, thus resulting in a 1162 * deadlock. Alternatively bios should be allocated using bio_kmalloc() instead 1163 * of bio_alloc(), as that avoids the mempool deadlock. 1164 * If possible a big IO should be split into smaller parts when allocation 1165 * fails. Partial allocation should not be an error, or you risk a live-lock. 1166 */ 1167 struct request *blk_make_request(struct request_queue *q, struct bio *bio, 1168 gfp_t gfp_mask) 1169 { 1170 struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask); 1171 1172 if (unlikely(!rq)) 1173 return ERR_PTR(-ENOMEM); 1174 1175 for_each_bio(bio) { 1176 struct bio *bounce_bio = bio; 1177 int ret; 1178 1179 blk_queue_bounce(q, &bounce_bio); 1180 ret = blk_rq_append_bio(q, rq, bounce_bio); 1181 if (unlikely(ret)) { 1182 blk_put_request(rq); 1183 return ERR_PTR(ret); 1184 } 1185 } 1186 1187 return rq; 1188 } 1189 EXPORT_SYMBOL(blk_make_request); 1190 1191 /** 1192 * blk_requeue_request - put a request back on queue 1193 * @q: request queue where request should be inserted 1194 * @rq: request to be inserted 1195 * 1196 * Description: 1197 * Drivers often keep queueing requests until the hardware cannot accept 1198 * more, when that condition happens we need to put the request back 1199 * on the queue. Must be called with queue lock held. 1200 */ 1201 void blk_requeue_request(struct request_queue *q, struct request *rq) 1202 { 1203 blk_delete_timer(rq); 1204 blk_clear_rq_complete(rq); 1205 trace_block_rq_requeue(q, rq); 1206 1207 if (blk_rq_tagged(rq)) 1208 blk_queue_end_tag(q, rq); 1209 1210 BUG_ON(blk_queued_rq(rq)); 1211 1212 elv_requeue_request(q, rq); 1213 } 1214 EXPORT_SYMBOL(blk_requeue_request); 1215 1216 static void add_acct_request(struct request_queue *q, struct request *rq, 1217 int where) 1218 { 1219 blk_account_io_start(rq, true); 1220 __elv_add_request(q, rq, where); 1221 } 1222 1223 static void part_round_stats_single(int cpu, struct hd_struct *part, 1224 unsigned long now) 1225 { 1226 if (now == part->stamp) 1227 return; 1228 1229 if (part_in_flight(part)) { 1230 __part_stat_add(cpu, part, time_in_queue, 1231 part_in_flight(part) * (now - part->stamp)); 1232 __part_stat_add(cpu, part, io_ticks, (now - part->stamp)); 1233 } 1234 part->stamp = now; 1235 } 1236 1237 /** 1238 * part_round_stats() - Round off the performance stats on a struct disk_stats. 1239 * @cpu: cpu number for stats access 1240 * @part: target partition 1241 * 1242 * The average IO queue length and utilisation statistics are maintained 1243 * by observing the current state of the queue length and the amount of 1244 * time it has been in this state for. 1245 * 1246 * Normally, that accounting is done on IO completion, but that can result 1247 * in more than a second's worth of IO being accounted for within any one 1248 * second, leading to >100% utilisation. To deal with that, we call this 1249 * function to do a round-off before returning the results when reading 1250 * /proc/diskstats. This accounts immediately for all queue usage up to 1251 * the current jiffies and restarts the counters again. 1252 */ 1253 void part_round_stats(int cpu, struct hd_struct *part) 1254 { 1255 unsigned long now = jiffies; 1256 1257 if (part->partno) 1258 part_round_stats_single(cpu, &part_to_disk(part)->part0, now); 1259 part_round_stats_single(cpu, part, now); 1260 } 1261 EXPORT_SYMBOL_GPL(part_round_stats); 1262 1263 #ifdef CONFIG_PM_RUNTIME 1264 static void blk_pm_put_request(struct request *rq) 1265 { 1266 if (rq->q->dev && !(rq->cmd_flags & REQ_PM) && !--rq->q->nr_pending) 1267 pm_runtime_mark_last_busy(rq->q->dev); 1268 } 1269 #else 1270 static inline void blk_pm_put_request(struct request *rq) {} 1271 #endif 1272 1273 /* 1274 * queue lock must be held 1275 */ 1276 void __blk_put_request(struct request_queue *q, struct request *req) 1277 { 1278 if (unlikely(!q)) 1279 return; 1280 1281 blk_pm_put_request(req); 1282 1283 elv_completed_request(q, req); 1284 1285 /* this is a bio leak */ 1286 WARN_ON(req->bio != NULL); 1287 1288 /* 1289 * Request may not have originated from ll_rw_blk. if not, 1290 * it didn't come out of our reserved rq pools 1291 */ 1292 if (req->cmd_flags & REQ_ALLOCED) { 1293 unsigned int flags = req->cmd_flags; 1294 struct request_list *rl = blk_rq_rl(req); 1295 1296 BUG_ON(!list_empty(&req->queuelist)); 1297 BUG_ON(!hlist_unhashed(&req->hash)); 1298 1299 blk_free_request(rl, req); 1300 freed_request(rl, flags); 1301 blk_put_rl(rl); 1302 } 1303 } 1304 EXPORT_SYMBOL_GPL(__blk_put_request); 1305 1306 void blk_put_request(struct request *req) 1307 { 1308 struct request_queue *q = req->q; 1309 1310 if (q->mq_ops) 1311 blk_mq_free_request(req); 1312 else { 1313 unsigned long flags; 1314 1315 spin_lock_irqsave(q->queue_lock, flags); 1316 __blk_put_request(q, req); 1317 spin_unlock_irqrestore(q->queue_lock, flags); 1318 } 1319 } 1320 EXPORT_SYMBOL(blk_put_request); 1321 1322 /** 1323 * blk_add_request_payload - add a payload to a request 1324 * @rq: request to update 1325 * @page: page backing the payload 1326 * @len: length of the payload. 1327 * 1328 * This allows to later add a payload to an already submitted request by 1329 * a block driver. The driver needs to take care of freeing the payload 1330 * itself. 1331 * 1332 * Note that this is a quite horrible hack and nothing but handling of 1333 * discard requests should ever use it. 1334 */ 1335 void blk_add_request_payload(struct request *rq, struct page *page, 1336 unsigned int len) 1337 { 1338 struct bio *bio = rq->bio; 1339 1340 bio->bi_io_vec->bv_page = page; 1341 bio->bi_io_vec->bv_offset = 0; 1342 bio->bi_io_vec->bv_len = len; 1343 1344 bio->bi_iter.bi_size = len; 1345 bio->bi_vcnt = 1; 1346 bio->bi_phys_segments = 1; 1347 1348 rq->__data_len = rq->resid_len = len; 1349 rq->nr_phys_segments = 1; 1350 rq->buffer = bio_data(bio); 1351 } 1352 EXPORT_SYMBOL_GPL(blk_add_request_payload); 1353 1354 bool bio_attempt_back_merge(struct request_queue *q, struct request *req, 1355 struct bio *bio) 1356 { 1357 const int ff = bio->bi_rw & REQ_FAILFAST_MASK; 1358 1359 if (!ll_back_merge_fn(q, req, bio)) 1360 return false; 1361 1362 trace_block_bio_backmerge(q, req, bio); 1363 1364 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) 1365 blk_rq_set_mixed_merge(req); 1366 1367 req->biotail->bi_next = bio; 1368 req->biotail = bio; 1369 req->__data_len += bio->bi_iter.bi_size; 1370 req->ioprio = ioprio_best(req->ioprio, bio_prio(bio)); 1371 1372 blk_account_io_start(req, false); 1373 return true; 1374 } 1375 1376 bool bio_attempt_front_merge(struct request_queue *q, struct request *req, 1377 struct bio *bio) 1378 { 1379 const int ff = bio->bi_rw & REQ_FAILFAST_MASK; 1380 1381 if (!ll_front_merge_fn(q, req, bio)) 1382 return false; 1383 1384 trace_block_bio_frontmerge(q, req, bio); 1385 1386 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) 1387 blk_rq_set_mixed_merge(req); 1388 1389 bio->bi_next = req->bio; 1390 req->bio = bio; 1391 1392 /* 1393 * may not be valid. if the low level driver said 1394 * it didn't need a bounce buffer then it better 1395 * not touch req->buffer either... 1396 */ 1397 req->buffer = bio_data(bio); 1398 req->__sector = bio->bi_iter.bi_sector; 1399 req->__data_len += bio->bi_iter.bi_size; 1400 req->ioprio = ioprio_best(req->ioprio, bio_prio(bio)); 1401 1402 blk_account_io_start(req, false); 1403 return true; 1404 } 1405 1406 /** 1407 * blk_attempt_plug_merge - try to merge with %current's plugged list 1408 * @q: request_queue new bio is being queued at 1409 * @bio: new bio being queued 1410 * @request_count: out parameter for number of traversed plugged requests 1411 * 1412 * Determine whether @bio being queued on @q can be merged with a request 1413 * on %current's plugged list. Returns %true if merge was successful, 1414 * otherwise %false. 1415 * 1416 * Plugging coalesces IOs from the same issuer for the same purpose without 1417 * going through @q->queue_lock. As such it's more of an issuing mechanism 1418 * than scheduling, and the request, while may have elvpriv data, is not 1419 * added on the elevator at this point. In addition, we don't have 1420 * reliable access to the elevator outside queue lock. Only check basic 1421 * merging parameters without querying the elevator. 1422 */ 1423 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, 1424 unsigned int *request_count) 1425 { 1426 struct blk_plug *plug; 1427 struct request *rq; 1428 bool ret = false; 1429 struct list_head *plug_list; 1430 1431 if (blk_queue_nomerges(q)) 1432 goto out; 1433 1434 plug = current->plug; 1435 if (!plug) 1436 goto out; 1437 *request_count = 0; 1438 1439 if (q->mq_ops) 1440 plug_list = &plug->mq_list; 1441 else 1442 plug_list = &plug->list; 1443 1444 list_for_each_entry_reverse(rq, plug_list, queuelist) { 1445 int el_ret; 1446 1447 if (rq->q == q) 1448 (*request_count)++; 1449 1450 if (rq->q != q || !blk_rq_merge_ok(rq, bio)) 1451 continue; 1452 1453 el_ret = blk_try_merge(rq, bio); 1454 if (el_ret == ELEVATOR_BACK_MERGE) { 1455 ret = bio_attempt_back_merge(q, rq, bio); 1456 if (ret) 1457 break; 1458 } else if (el_ret == ELEVATOR_FRONT_MERGE) { 1459 ret = bio_attempt_front_merge(q, rq, bio); 1460 if (ret) 1461 break; 1462 } 1463 } 1464 out: 1465 return ret; 1466 } 1467 1468 void init_request_from_bio(struct request *req, struct bio *bio) 1469 { 1470 req->cmd_type = REQ_TYPE_FS; 1471 1472 req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK; 1473 if (bio->bi_rw & REQ_RAHEAD) 1474 req->cmd_flags |= REQ_FAILFAST_MASK; 1475 1476 req->errors = 0; 1477 req->__sector = bio->bi_iter.bi_sector; 1478 req->ioprio = bio_prio(bio); 1479 blk_rq_bio_prep(req->q, req, bio); 1480 } 1481 1482 void blk_queue_bio(struct request_queue *q, struct bio *bio) 1483 { 1484 const bool sync = !!(bio->bi_rw & REQ_SYNC); 1485 struct blk_plug *plug; 1486 int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT; 1487 struct request *req; 1488 unsigned int request_count = 0; 1489 1490 /* 1491 * low level driver can indicate that it wants pages above a 1492 * certain limit bounced to low memory (ie for highmem, or even 1493 * ISA dma in theory) 1494 */ 1495 blk_queue_bounce(q, &bio); 1496 1497 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 1498 bio_endio(bio, -EIO); 1499 return; 1500 } 1501 1502 if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) { 1503 spin_lock_irq(q->queue_lock); 1504 where = ELEVATOR_INSERT_FLUSH; 1505 goto get_rq; 1506 } 1507 1508 /* 1509 * Check if we can merge with the plugged list before grabbing 1510 * any locks. 1511 */ 1512 if (blk_attempt_plug_merge(q, bio, &request_count)) 1513 return; 1514 1515 spin_lock_irq(q->queue_lock); 1516 1517 el_ret = elv_merge(q, &req, bio); 1518 if (el_ret == ELEVATOR_BACK_MERGE) { 1519 if (bio_attempt_back_merge(q, req, bio)) { 1520 elv_bio_merged(q, req, bio); 1521 if (!attempt_back_merge(q, req)) 1522 elv_merged_request(q, req, el_ret); 1523 goto out_unlock; 1524 } 1525 } else if (el_ret == ELEVATOR_FRONT_MERGE) { 1526 if (bio_attempt_front_merge(q, req, bio)) { 1527 elv_bio_merged(q, req, bio); 1528 if (!attempt_front_merge(q, req)) 1529 elv_merged_request(q, req, el_ret); 1530 goto out_unlock; 1531 } 1532 } 1533 1534 get_rq: 1535 /* 1536 * This sync check and mask will be re-done in init_request_from_bio(), 1537 * but we need to set it earlier to expose the sync flag to the 1538 * rq allocator and io schedulers. 1539 */ 1540 rw_flags = bio_data_dir(bio); 1541 if (sync) 1542 rw_flags |= REQ_SYNC; 1543 1544 /* 1545 * Grab a free request. This is might sleep but can not fail. 1546 * Returns with the queue unlocked. 1547 */ 1548 req = get_request(q, rw_flags, bio, GFP_NOIO); 1549 if (unlikely(!req)) { 1550 bio_endio(bio, -ENODEV); /* @q is dead */ 1551 goto out_unlock; 1552 } 1553 1554 /* 1555 * After dropping the lock and possibly sleeping here, our request 1556 * may now be mergeable after it had proven unmergeable (above). 1557 * We don't worry about that case for efficiency. It won't happen 1558 * often, and the elevators are able to handle it. 1559 */ 1560 init_request_from_bio(req, bio); 1561 1562 if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags)) 1563 req->cpu = raw_smp_processor_id(); 1564 1565 plug = current->plug; 1566 if (plug) { 1567 /* 1568 * If this is the first request added after a plug, fire 1569 * of a plug trace. 1570 */ 1571 if (!request_count) 1572 trace_block_plug(q); 1573 else { 1574 if (request_count >= BLK_MAX_REQUEST_COUNT) { 1575 blk_flush_plug_list(plug, false); 1576 trace_block_plug(q); 1577 } 1578 } 1579 list_add_tail(&req->queuelist, &plug->list); 1580 blk_account_io_start(req, true); 1581 } else { 1582 spin_lock_irq(q->queue_lock); 1583 add_acct_request(q, req, where); 1584 __blk_run_queue(q); 1585 out_unlock: 1586 spin_unlock_irq(q->queue_lock); 1587 } 1588 } 1589 EXPORT_SYMBOL_GPL(blk_queue_bio); /* for device mapper only */ 1590 1591 /* 1592 * If bio->bi_dev is a partition, remap the location 1593 */ 1594 static inline void blk_partition_remap(struct bio *bio) 1595 { 1596 struct block_device *bdev = bio->bi_bdev; 1597 1598 if (bio_sectors(bio) && bdev != bdev->bd_contains) { 1599 struct hd_struct *p = bdev->bd_part; 1600 1601 bio->bi_iter.bi_sector += p->start_sect; 1602 bio->bi_bdev = bdev->bd_contains; 1603 1604 trace_block_bio_remap(bdev_get_queue(bio->bi_bdev), bio, 1605 bdev->bd_dev, 1606 bio->bi_iter.bi_sector - p->start_sect); 1607 } 1608 } 1609 1610 static void handle_bad_sector(struct bio *bio) 1611 { 1612 char b[BDEVNAME_SIZE]; 1613 1614 printk(KERN_INFO "attempt to access beyond end of device\n"); 1615 printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n", 1616 bdevname(bio->bi_bdev, b), 1617 bio->bi_rw, 1618 (unsigned long long)bio_end_sector(bio), 1619 (long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9)); 1620 1621 set_bit(BIO_EOF, &bio->bi_flags); 1622 } 1623 1624 #ifdef CONFIG_FAIL_MAKE_REQUEST 1625 1626 static DECLARE_FAULT_ATTR(fail_make_request); 1627 1628 static int __init setup_fail_make_request(char *str) 1629 { 1630 return setup_fault_attr(&fail_make_request, str); 1631 } 1632 __setup("fail_make_request=", setup_fail_make_request); 1633 1634 static bool should_fail_request(struct hd_struct *part, unsigned int bytes) 1635 { 1636 return part->make_it_fail && should_fail(&fail_make_request, bytes); 1637 } 1638 1639 static int __init fail_make_request_debugfs(void) 1640 { 1641 struct dentry *dir = fault_create_debugfs_attr("fail_make_request", 1642 NULL, &fail_make_request); 1643 1644 return IS_ERR(dir) ? PTR_ERR(dir) : 0; 1645 } 1646 1647 late_initcall(fail_make_request_debugfs); 1648 1649 #else /* CONFIG_FAIL_MAKE_REQUEST */ 1650 1651 static inline bool should_fail_request(struct hd_struct *part, 1652 unsigned int bytes) 1653 { 1654 return false; 1655 } 1656 1657 #endif /* CONFIG_FAIL_MAKE_REQUEST */ 1658 1659 /* 1660 * Check whether this bio extends beyond the end of the device. 1661 */ 1662 static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors) 1663 { 1664 sector_t maxsector; 1665 1666 if (!nr_sectors) 1667 return 0; 1668 1669 /* Test device or partition size, when known. */ 1670 maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9; 1671 if (maxsector) { 1672 sector_t sector = bio->bi_iter.bi_sector; 1673 1674 if (maxsector < nr_sectors || maxsector - nr_sectors < sector) { 1675 /* 1676 * This may well happen - the kernel calls bread() 1677 * without checking the size of the device, e.g., when 1678 * mounting a device. 1679 */ 1680 handle_bad_sector(bio); 1681 return 1; 1682 } 1683 } 1684 1685 return 0; 1686 } 1687 1688 static noinline_for_stack bool 1689 generic_make_request_checks(struct bio *bio) 1690 { 1691 struct request_queue *q; 1692 int nr_sectors = bio_sectors(bio); 1693 int err = -EIO; 1694 char b[BDEVNAME_SIZE]; 1695 struct hd_struct *part; 1696 1697 might_sleep(); 1698 1699 if (bio_check_eod(bio, nr_sectors)) 1700 goto end_io; 1701 1702 q = bdev_get_queue(bio->bi_bdev); 1703 if (unlikely(!q)) { 1704 printk(KERN_ERR 1705 "generic_make_request: Trying to access " 1706 "nonexistent block-device %s (%Lu)\n", 1707 bdevname(bio->bi_bdev, b), 1708 (long long) bio->bi_iter.bi_sector); 1709 goto end_io; 1710 } 1711 1712 if (likely(bio_is_rw(bio) && 1713 nr_sectors > queue_max_hw_sectors(q))) { 1714 printk(KERN_ERR "bio too big device %s (%u > %u)\n", 1715 bdevname(bio->bi_bdev, b), 1716 bio_sectors(bio), 1717 queue_max_hw_sectors(q)); 1718 goto end_io; 1719 } 1720 1721 part = bio->bi_bdev->bd_part; 1722 if (should_fail_request(part, bio->bi_iter.bi_size) || 1723 should_fail_request(&part_to_disk(part)->part0, 1724 bio->bi_iter.bi_size)) 1725 goto end_io; 1726 1727 /* 1728 * If this device has partitions, remap block n 1729 * of partition p to block n+start(p) of the disk. 1730 */ 1731 blk_partition_remap(bio); 1732 1733 if (bio_check_eod(bio, nr_sectors)) 1734 goto end_io; 1735 1736 /* 1737 * Filter flush bio's early so that make_request based 1738 * drivers without flush support don't have to worry 1739 * about them. 1740 */ 1741 if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && !q->flush_flags) { 1742 bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA); 1743 if (!nr_sectors) { 1744 err = 0; 1745 goto end_io; 1746 } 1747 } 1748 1749 if ((bio->bi_rw & REQ_DISCARD) && 1750 (!blk_queue_discard(q) || 1751 ((bio->bi_rw & REQ_SECURE) && !blk_queue_secdiscard(q)))) { 1752 err = -EOPNOTSUPP; 1753 goto end_io; 1754 } 1755 1756 if (bio->bi_rw & REQ_WRITE_SAME && !bdev_write_same(bio->bi_bdev)) { 1757 err = -EOPNOTSUPP; 1758 goto end_io; 1759 } 1760 1761 /* 1762 * Various block parts want %current->io_context and lazy ioc 1763 * allocation ends up trading a lot of pain for a small amount of 1764 * memory. Just allocate it upfront. This may fail and block 1765 * layer knows how to live with it. 1766 */ 1767 create_io_context(GFP_ATOMIC, q->node); 1768 1769 if (blk_throtl_bio(q, bio)) 1770 return false; /* throttled, will be resubmitted later */ 1771 1772 trace_block_bio_queue(q, bio); 1773 return true; 1774 1775 end_io: 1776 bio_endio(bio, err); 1777 return false; 1778 } 1779 1780 /** 1781 * generic_make_request - hand a buffer to its device driver for I/O 1782 * @bio: The bio describing the location in memory and on the device. 1783 * 1784 * generic_make_request() is used to make I/O requests of block 1785 * devices. It is passed a &struct bio, which describes the I/O that needs 1786 * to be done. 1787 * 1788 * generic_make_request() does not return any status. The 1789 * success/failure status of the request, along with notification of 1790 * completion, is delivered asynchronously through the bio->bi_end_io 1791 * function described (one day) else where. 1792 * 1793 * The caller of generic_make_request must make sure that bi_io_vec 1794 * are set to describe the memory buffer, and that bi_dev and bi_sector are 1795 * set to describe the device address, and the 1796 * bi_end_io and optionally bi_private are set to describe how 1797 * completion notification should be signaled. 1798 * 1799 * generic_make_request and the drivers it calls may use bi_next if this 1800 * bio happens to be merged with someone else, and may resubmit the bio to 1801 * a lower device by calling into generic_make_request recursively, which 1802 * means the bio should NOT be touched after the call to ->make_request_fn. 1803 */ 1804 void generic_make_request(struct bio *bio) 1805 { 1806 struct bio_list bio_list_on_stack; 1807 1808 if (!generic_make_request_checks(bio)) 1809 return; 1810 1811 /* 1812 * We only want one ->make_request_fn to be active at a time, else 1813 * stack usage with stacked devices could be a problem. So use 1814 * current->bio_list to keep a list of requests submited by a 1815 * make_request_fn function. current->bio_list is also used as a 1816 * flag to say if generic_make_request is currently active in this 1817 * task or not. If it is NULL, then no make_request is active. If 1818 * it is non-NULL, then a make_request is active, and new requests 1819 * should be added at the tail 1820 */ 1821 if (current->bio_list) { 1822 bio_list_add(current->bio_list, bio); 1823 return; 1824 } 1825 1826 /* following loop may be a bit non-obvious, and so deserves some 1827 * explanation. 1828 * Before entering the loop, bio->bi_next is NULL (as all callers 1829 * ensure that) so we have a list with a single bio. 1830 * We pretend that we have just taken it off a longer list, so 1831 * we assign bio_list to a pointer to the bio_list_on_stack, 1832 * thus initialising the bio_list of new bios to be 1833 * added. ->make_request() may indeed add some more bios 1834 * through a recursive call to generic_make_request. If it 1835 * did, we find a non-NULL value in bio_list and re-enter the loop 1836 * from the top. In this case we really did just take the bio 1837 * of the top of the list (no pretending) and so remove it from 1838 * bio_list, and call into ->make_request() again. 1839 */ 1840 BUG_ON(bio->bi_next); 1841 bio_list_init(&bio_list_on_stack); 1842 current->bio_list = &bio_list_on_stack; 1843 do { 1844 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 1845 1846 q->make_request_fn(q, bio); 1847 1848 bio = bio_list_pop(current->bio_list); 1849 } while (bio); 1850 current->bio_list = NULL; /* deactivate */ 1851 } 1852 EXPORT_SYMBOL(generic_make_request); 1853 1854 /** 1855 * submit_bio - submit a bio to the block device layer for I/O 1856 * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead) 1857 * @bio: The &struct bio which describes the I/O 1858 * 1859 * submit_bio() is very similar in purpose to generic_make_request(), and 1860 * uses that function to do most of the work. Both are fairly rough 1861 * interfaces; @bio must be presetup and ready for I/O. 1862 * 1863 */ 1864 void submit_bio(int rw, struct bio *bio) 1865 { 1866 bio->bi_rw |= rw; 1867 1868 /* 1869 * If it's a regular read/write or a barrier with data attached, 1870 * go through the normal accounting stuff before submission. 1871 */ 1872 if (bio_has_data(bio)) { 1873 unsigned int count; 1874 1875 if (unlikely(rw & REQ_WRITE_SAME)) 1876 count = bdev_logical_block_size(bio->bi_bdev) >> 9; 1877 else 1878 count = bio_sectors(bio); 1879 1880 if (rw & WRITE) { 1881 count_vm_events(PGPGOUT, count); 1882 } else { 1883 task_io_account_read(bio->bi_iter.bi_size); 1884 count_vm_events(PGPGIN, count); 1885 } 1886 1887 if (unlikely(block_dump)) { 1888 char b[BDEVNAME_SIZE]; 1889 printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", 1890 current->comm, task_pid_nr(current), 1891 (rw & WRITE) ? "WRITE" : "READ", 1892 (unsigned long long)bio->bi_iter.bi_sector, 1893 bdevname(bio->bi_bdev, b), 1894 count); 1895 } 1896 } 1897 1898 generic_make_request(bio); 1899 } 1900 EXPORT_SYMBOL(submit_bio); 1901 1902 /** 1903 * blk_rq_check_limits - Helper function to check a request for the queue limit 1904 * @q: the queue 1905 * @rq: the request being checked 1906 * 1907 * Description: 1908 * @rq may have been made based on weaker limitations of upper-level queues 1909 * in request stacking drivers, and it may violate the limitation of @q. 1910 * Since the block layer and the underlying device driver trust @rq 1911 * after it is inserted to @q, it should be checked against @q before 1912 * the insertion using this generic function. 1913 * 1914 * This function should also be useful for request stacking drivers 1915 * in some cases below, so export this function. 1916 * Request stacking drivers like request-based dm may change the queue 1917 * limits while requests are in the queue (e.g. dm's table swapping). 1918 * Such request stacking drivers should check those requests agaist 1919 * the new queue limits again when they dispatch those requests, 1920 * although such checkings are also done against the old queue limits 1921 * when submitting requests. 1922 */ 1923 int blk_rq_check_limits(struct request_queue *q, struct request *rq) 1924 { 1925 if (!rq_mergeable(rq)) 1926 return 0; 1927 1928 if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, rq->cmd_flags)) { 1929 printk(KERN_ERR "%s: over max size limit.\n", __func__); 1930 return -EIO; 1931 } 1932 1933 /* 1934 * queue's settings related to segment counting like q->bounce_pfn 1935 * may differ from that of other stacking queues. 1936 * Recalculate it to check the request correctly on this queue's 1937 * limitation. 1938 */ 1939 blk_recalc_rq_segments(rq); 1940 if (rq->nr_phys_segments > queue_max_segments(q)) { 1941 printk(KERN_ERR "%s: over max segments limit.\n", __func__); 1942 return -EIO; 1943 } 1944 1945 return 0; 1946 } 1947 EXPORT_SYMBOL_GPL(blk_rq_check_limits); 1948 1949 /** 1950 * blk_insert_cloned_request - Helper for stacking drivers to submit a request 1951 * @q: the queue to submit the request 1952 * @rq: the request being queued 1953 */ 1954 int blk_insert_cloned_request(struct request_queue *q, struct request *rq) 1955 { 1956 unsigned long flags; 1957 int where = ELEVATOR_INSERT_BACK; 1958 1959 if (blk_rq_check_limits(q, rq)) 1960 return -EIO; 1961 1962 if (rq->rq_disk && 1963 should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq))) 1964 return -EIO; 1965 1966 spin_lock_irqsave(q->queue_lock, flags); 1967 if (unlikely(blk_queue_dying(q))) { 1968 spin_unlock_irqrestore(q->queue_lock, flags); 1969 return -ENODEV; 1970 } 1971 1972 /* 1973 * Submitting request must be dequeued before calling this function 1974 * because it will be linked to another request_queue 1975 */ 1976 BUG_ON(blk_queued_rq(rq)); 1977 1978 if (rq->cmd_flags & (REQ_FLUSH|REQ_FUA)) 1979 where = ELEVATOR_INSERT_FLUSH; 1980 1981 add_acct_request(q, rq, where); 1982 if (where == ELEVATOR_INSERT_FLUSH) 1983 __blk_run_queue(q); 1984 spin_unlock_irqrestore(q->queue_lock, flags); 1985 1986 return 0; 1987 } 1988 EXPORT_SYMBOL_GPL(blk_insert_cloned_request); 1989 1990 /** 1991 * blk_rq_err_bytes - determine number of bytes till the next failure boundary 1992 * @rq: request to examine 1993 * 1994 * Description: 1995 * A request could be merge of IOs which require different failure 1996 * handling. This function determines the number of bytes which 1997 * can be failed from the beginning of the request without 1998 * crossing into area which need to be retried further. 1999 * 2000 * Return: 2001 * The number of bytes to fail. 2002 * 2003 * Context: 2004 * queue_lock must be held. 2005 */ 2006 unsigned int blk_rq_err_bytes(const struct request *rq) 2007 { 2008 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; 2009 unsigned int bytes = 0; 2010 struct bio *bio; 2011 2012 if (!(rq->cmd_flags & REQ_MIXED_MERGE)) 2013 return blk_rq_bytes(rq); 2014 2015 /* 2016 * Currently the only 'mixing' which can happen is between 2017 * different fastfail types. We can safely fail portions 2018 * which have all the failfast bits that the first one has - 2019 * the ones which are at least as eager to fail as the first 2020 * one. 2021 */ 2022 for (bio = rq->bio; bio; bio = bio->bi_next) { 2023 if ((bio->bi_rw & ff) != ff) 2024 break; 2025 bytes += bio->bi_iter.bi_size; 2026 } 2027 2028 /* this could lead to infinite loop */ 2029 BUG_ON(blk_rq_bytes(rq) && !bytes); 2030 return bytes; 2031 } 2032 EXPORT_SYMBOL_GPL(blk_rq_err_bytes); 2033 2034 void blk_account_io_completion(struct request *req, unsigned int bytes) 2035 { 2036 if (blk_do_io_stat(req)) { 2037 const int rw = rq_data_dir(req); 2038 struct hd_struct *part; 2039 int cpu; 2040 2041 cpu = part_stat_lock(); 2042 part = req->part; 2043 part_stat_add(cpu, part, sectors[rw], bytes >> 9); 2044 part_stat_unlock(); 2045 } 2046 } 2047 2048 void blk_account_io_done(struct request *req) 2049 { 2050 /* 2051 * Account IO completion. flush_rq isn't accounted as a 2052 * normal IO on queueing nor completion. Accounting the 2053 * containing request is enough. 2054 */ 2055 if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) { 2056 unsigned long duration = jiffies - req->start_time; 2057 const int rw = rq_data_dir(req); 2058 struct hd_struct *part; 2059 int cpu; 2060 2061 cpu = part_stat_lock(); 2062 part = req->part; 2063 2064 part_stat_inc(cpu, part, ios[rw]); 2065 part_stat_add(cpu, part, ticks[rw], duration); 2066 part_round_stats(cpu, part); 2067 part_dec_in_flight(part, rw); 2068 2069 hd_struct_put(part); 2070 part_stat_unlock(); 2071 } 2072 } 2073 2074 #ifdef CONFIG_PM_RUNTIME 2075 /* 2076 * Don't process normal requests when queue is suspended 2077 * or in the process of suspending/resuming 2078 */ 2079 static struct request *blk_pm_peek_request(struct request_queue *q, 2080 struct request *rq) 2081 { 2082 if (q->dev && (q->rpm_status == RPM_SUSPENDED || 2083 (q->rpm_status != RPM_ACTIVE && !(rq->cmd_flags & REQ_PM)))) 2084 return NULL; 2085 else 2086 return rq; 2087 } 2088 #else 2089 static inline struct request *blk_pm_peek_request(struct request_queue *q, 2090 struct request *rq) 2091 { 2092 return rq; 2093 } 2094 #endif 2095 2096 void blk_account_io_start(struct request *rq, bool new_io) 2097 { 2098 struct hd_struct *part; 2099 int rw = rq_data_dir(rq); 2100 int cpu; 2101 2102 if (!blk_do_io_stat(rq)) 2103 return; 2104 2105 cpu = part_stat_lock(); 2106 2107 if (!new_io) { 2108 part = rq->part; 2109 part_stat_inc(cpu, part, merges[rw]); 2110 } else { 2111 part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq)); 2112 if (!hd_struct_try_get(part)) { 2113 /* 2114 * The partition is already being removed, 2115 * the request will be accounted on the disk only 2116 * 2117 * We take a reference on disk->part0 although that 2118 * partition will never be deleted, so we can treat 2119 * it as any other partition. 2120 */ 2121 part = &rq->rq_disk->part0; 2122 hd_struct_get(part); 2123 } 2124 part_round_stats(cpu, part); 2125 part_inc_in_flight(part, rw); 2126 rq->part = part; 2127 } 2128 2129 part_stat_unlock(); 2130 } 2131 2132 /** 2133 * blk_peek_request - peek at the top of a request queue 2134 * @q: request queue to peek at 2135 * 2136 * Description: 2137 * Return the request at the top of @q. The returned request 2138 * should be started using blk_start_request() before LLD starts 2139 * processing it. 2140 * 2141 * Return: 2142 * Pointer to the request at the top of @q if available. Null 2143 * otherwise. 2144 * 2145 * Context: 2146 * queue_lock must be held. 2147 */ 2148 struct request *blk_peek_request(struct request_queue *q) 2149 { 2150 struct request *rq; 2151 int ret; 2152 2153 while ((rq = __elv_next_request(q)) != NULL) { 2154 2155 rq = blk_pm_peek_request(q, rq); 2156 if (!rq) 2157 break; 2158 2159 if (!(rq->cmd_flags & REQ_STARTED)) { 2160 /* 2161 * This is the first time the device driver 2162 * sees this request (possibly after 2163 * requeueing). Notify IO scheduler. 2164 */ 2165 if (rq->cmd_flags & REQ_SORTED) 2166 elv_activate_rq(q, rq); 2167 2168 /* 2169 * just mark as started even if we don't start 2170 * it, a request that has been delayed should 2171 * not be passed by new incoming requests 2172 */ 2173 rq->cmd_flags |= REQ_STARTED; 2174 trace_block_rq_issue(q, rq); 2175 } 2176 2177 if (!q->boundary_rq || q->boundary_rq == rq) { 2178 q->end_sector = rq_end_sector(rq); 2179 q->boundary_rq = NULL; 2180 } 2181 2182 if (rq->cmd_flags & REQ_DONTPREP) 2183 break; 2184 2185 if (q->dma_drain_size && blk_rq_bytes(rq)) { 2186 /* 2187 * make sure space for the drain appears we 2188 * know we can do this because max_hw_segments 2189 * has been adjusted to be one fewer than the 2190 * device can handle 2191 */ 2192 rq->nr_phys_segments++; 2193 } 2194 2195 if (!q->prep_rq_fn) 2196 break; 2197 2198 ret = q->prep_rq_fn(q, rq); 2199 if (ret == BLKPREP_OK) { 2200 break; 2201 } else if (ret == BLKPREP_DEFER) { 2202 /* 2203 * the request may have been (partially) prepped. 2204 * we need to keep this request in the front to 2205 * avoid resource deadlock. REQ_STARTED will 2206 * prevent other fs requests from passing this one. 2207 */ 2208 if (q->dma_drain_size && blk_rq_bytes(rq) && 2209 !(rq->cmd_flags & REQ_DONTPREP)) { 2210 /* 2211 * remove the space for the drain we added 2212 * so that we don't add it again 2213 */ 2214 --rq->nr_phys_segments; 2215 } 2216 2217 rq = NULL; 2218 break; 2219 } else if (ret == BLKPREP_KILL) { 2220 rq->cmd_flags |= REQ_QUIET; 2221 /* 2222 * Mark this request as started so we don't trigger 2223 * any debug logic in the end I/O path. 2224 */ 2225 blk_start_request(rq); 2226 __blk_end_request_all(rq, -EIO); 2227 } else { 2228 printk(KERN_ERR "%s: bad return=%d\n", __func__, ret); 2229 break; 2230 } 2231 } 2232 2233 return rq; 2234 } 2235 EXPORT_SYMBOL(blk_peek_request); 2236 2237 void blk_dequeue_request(struct request *rq) 2238 { 2239 struct request_queue *q = rq->q; 2240 2241 BUG_ON(list_empty(&rq->queuelist)); 2242 BUG_ON(ELV_ON_HASH(rq)); 2243 2244 list_del_init(&rq->queuelist); 2245 2246 /* 2247 * the time frame between a request being removed from the lists 2248 * and to it is freed is accounted as io that is in progress at 2249 * the driver side. 2250 */ 2251 if (blk_account_rq(rq)) { 2252 q->in_flight[rq_is_sync(rq)]++; 2253 set_io_start_time_ns(rq); 2254 } 2255 } 2256 2257 /** 2258 * blk_start_request - start request processing on the driver 2259 * @req: request to dequeue 2260 * 2261 * Description: 2262 * Dequeue @req and start timeout timer on it. This hands off the 2263 * request to the driver. 2264 * 2265 * Block internal functions which don't want to start timer should 2266 * call blk_dequeue_request(). 2267 * 2268 * Context: 2269 * queue_lock must be held. 2270 */ 2271 void blk_start_request(struct request *req) 2272 { 2273 blk_dequeue_request(req); 2274 2275 /* 2276 * We are now handing the request to the hardware, initialize 2277 * resid_len to full count and add the timeout handler. 2278 */ 2279 req->resid_len = blk_rq_bytes(req); 2280 if (unlikely(blk_bidi_rq(req))) 2281 req->next_rq->resid_len = blk_rq_bytes(req->next_rq); 2282 2283 BUG_ON(test_bit(REQ_ATOM_COMPLETE, &req->atomic_flags)); 2284 blk_add_timer(req); 2285 } 2286 EXPORT_SYMBOL(blk_start_request); 2287 2288 /** 2289 * blk_fetch_request - fetch a request from a request queue 2290 * @q: request queue to fetch a request from 2291 * 2292 * Description: 2293 * Return the request at the top of @q. The request is started on 2294 * return and LLD can start processing it immediately. 2295 * 2296 * Return: 2297 * Pointer to the request at the top of @q if available. Null 2298 * otherwise. 2299 * 2300 * Context: 2301 * queue_lock must be held. 2302 */ 2303 struct request *blk_fetch_request(struct request_queue *q) 2304 { 2305 struct request *rq; 2306 2307 rq = blk_peek_request(q); 2308 if (rq) 2309 blk_start_request(rq); 2310 return rq; 2311 } 2312 EXPORT_SYMBOL(blk_fetch_request); 2313 2314 /** 2315 * blk_update_request - Special helper function for request stacking drivers 2316 * @req: the request being processed 2317 * @error: %0 for success, < %0 for error 2318 * @nr_bytes: number of bytes to complete @req 2319 * 2320 * Description: 2321 * Ends I/O on a number of bytes attached to @req, but doesn't complete 2322 * the request structure even if @req doesn't have leftover. 2323 * If @req has leftover, sets it up for the next range of segments. 2324 * 2325 * This special helper function is only for request stacking drivers 2326 * (e.g. request-based dm) so that they can handle partial completion. 2327 * Actual device drivers should use blk_end_request instead. 2328 * 2329 * Passing the result of blk_rq_bytes() as @nr_bytes guarantees 2330 * %false return from this function. 2331 * 2332 * Return: 2333 * %false - this request doesn't have any more data 2334 * %true - this request has more data 2335 **/ 2336 bool blk_update_request(struct request *req, int error, unsigned int nr_bytes) 2337 { 2338 int total_bytes; 2339 2340 if (!req->bio) 2341 return false; 2342 2343 trace_block_rq_complete(req->q, req); 2344 2345 /* 2346 * For fs requests, rq is just carrier of independent bio's 2347 * and each partial completion should be handled separately. 2348 * Reset per-request error on each partial completion. 2349 * 2350 * TODO: tj: This is too subtle. It would be better to let 2351 * low level drivers do what they see fit. 2352 */ 2353 if (req->cmd_type == REQ_TYPE_FS) 2354 req->errors = 0; 2355 2356 if (error && req->cmd_type == REQ_TYPE_FS && 2357 !(req->cmd_flags & REQ_QUIET)) { 2358 char *error_type; 2359 2360 switch (error) { 2361 case -ENOLINK: 2362 error_type = "recoverable transport"; 2363 break; 2364 case -EREMOTEIO: 2365 error_type = "critical target"; 2366 break; 2367 case -EBADE: 2368 error_type = "critical nexus"; 2369 break; 2370 case -ETIMEDOUT: 2371 error_type = "timeout"; 2372 break; 2373 case -ENOSPC: 2374 error_type = "critical space allocation"; 2375 break; 2376 case -ENODATA: 2377 error_type = "critical medium"; 2378 break; 2379 case -EIO: 2380 default: 2381 error_type = "I/O"; 2382 break; 2383 } 2384 printk_ratelimited(KERN_ERR "end_request: %s error, dev %s, sector %llu\n", 2385 error_type, req->rq_disk ? 2386 req->rq_disk->disk_name : "?", 2387 (unsigned long long)blk_rq_pos(req)); 2388 2389 } 2390 2391 blk_account_io_completion(req, nr_bytes); 2392 2393 total_bytes = 0; 2394 while (req->bio) { 2395 struct bio *bio = req->bio; 2396 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes); 2397 2398 if (bio_bytes == bio->bi_iter.bi_size) 2399 req->bio = bio->bi_next; 2400 2401 req_bio_endio(req, bio, bio_bytes, error); 2402 2403 total_bytes += bio_bytes; 2404 nr_bytes -= bio_bytes; 2405 2406 if (!nr_bytes) 2407 break; 2408 } 2409 2410 /* 2411 * completely done 2412 */ 2413 if (!req->bio) { 2414 /* 2415 * Reset counters so that the request stacking driver 2416 * can find how many bytes remain in the request 2417 * later. 2418 */ 2419 req->__data_len = 0; 2420 return false; 2421 } 2422 2423 req->__data_len -= total_bytes; 2424 req->buffer = bio_data(req->bio); 2425 2426 /* update sector only for requests with clear definition of sector */ 2427 if (req->cmd_type == REQ_TYPE_FS) 2428 req->__sector += total_bytes >> 9; 2429 2430 /* mixed attributes always follow the first bio */ 2431 if (req->cmd_flags & REQ_MIXED_MERGE) { 2432 req->cmd_flags &= ~REQ_FAILFAST_MASK; 2433 req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK; 2434 } 2435 2436 /* 2437 * If total number of sectors is less than the first segment 2438 * size, something has gone terribly wrong. 2439 */ 2440 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) { 2441 blk_dump_rq_flags(req, "request botched"); 2442 req->__data_len = blk_rq_cur_bytes(req); 2443 } 2444 2445 /* recalculate the number of segments */ 2446 blk_recalc_rq_segments(req); 2447 2448 return true; 2449 } 2450 EXPORT_SYMBOL_GPL(blk_update_request); 2451 2452 static bool blk_update_bidi_request(struct request *rq, int error, 2453 unsigned int nr_bytes, 2454 unsigned int bidi_bytes) 2455 { 2456 if (blk_update_request(rq, error, nr_bytes)) 2457 return true; 2458 2459 /* Bidi request must be completed as a whole */ 2460 if (unlikely(blk_bidi_rq(rq)) && 2461 blk_update_request(rq->next_rq, error, bidi_bytes)) 2462 return true; 2463 2464 if (blk_queue_add_random(rq->q)) 2465 add_disk_randomness(rq->rq_disk); 2466 2467 return false; 2468 } 2469 2470 /** 2471 * blk_unprep_request - unprepare a request 2472 * @req: the request 2473 * 2474 * This function makes a request ready for complete resubmission (or 2475 * completion). It happens only after all error handling is complete, 2476 * so represents the appropriate moment to deallocate any resources 2477 * that were allocated to the request in the prep_rq_fn. The queue 2478 * lock is held when calling this. 2479 */ 2480 void blk_unprep_request(struct request *req) 2481 { 2482 struct request_queue *q = req->q; 2483 2484 req->cmd_flags &= ~REQ_DONTPREP; 2485 if (q->unprep_rq_fn) 2486 q->unprep_rq_fn(q, req); 2487 } 2488 EXPORT_SYMBOL_GPL(blk_unprep_request); 2489 2490 /* 2491 * queue lock must be held 2492 */ 2493 static void blk_finish_request(struct request *req, int error) 2494 { 2495 if (blk_rq_tagged(req)) 2496 blk_queue_end_tag(req->q, req); 2497 2498 BUG_ON(blk_queued_rq(req)); 2499 2500 if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS) 2501 laptop_io_completion(&req->q->backing_dev_info); 2502 2503 blk_delete_timer(req); 2504 2505 if (req->cmd_flags & REQ_DONTPREP) 2506 blk_unprep_request(req); 2507 2508 blk_account_io_done(req); 2509 2510 if (req->end_io) 2511 req->end_io(req, error); 2512 else { 2513 if (blk_bidi_rq(req)) 2514 __blk_put_request(req->next_rq->q, req->next_rq); 2515 2516 __blk_put_request(req->q, req); 2517 } 2518 } 2519 2520 /** 2521 * blk_end_bidi_request - Complete a bidi request 2522 * @rq: the request to complete 2523 * @error: %0 for success, < %0 for error 2524 * @nr_bytes: number of bytes to complete @rq 2525 * @bidi_bytes: number of bytes to complete @rq->next_rq 2526 * 2527 * Description: 2528 * Ends I/O on a number of bytes attached to @rq and @rq->next_rq. 2529 * Drivers that supports bidi can safely call this member for any 2530 * type of request, bidi or uni. In the later case @bidi_bytes is 2531 * just ignored. 2532 * 2533 * Return: 2534 * %false - we are done with this request 2535 * %true - still buffers pending for this request 2536 **/ 2537 static bool blk_end_bidi_request(struct request *rq, int error, 2538 unsigned int nr_bytes, unsigned int bidi_bytes) 2539 { 2540 struct request_queue *q = rq->q; 2541 unsigned long flags; 2542 2543 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes)) 2544 return true; 2545 2546 spin_lock_irqsave(q->queue_lock, flags); 2547 blk_finish_request(rq, error); 2548 spin_unlock_irqrestore(q->queue_lock, flags); 2549 2550 return false; 2551 } 2552 2553 /** 2554 * __blk_end_bidi_request - Complete a bidi request with queue lock held 2555 * @rq: the request to complete 2556 * @error: %0 for success, < %0 for error 2557 * @nr_bytes: number of bytes to complete @rq 2558 * @bidi_bytes: number of bytes to complete @rq->next_rq 2559 * 2560 * Description: 2561 * Identical to blk_end_bidi_request() except that queue lock is 2562 * assumed to be locked on entry and remains so on return. 2563 * 2564 * Return: 2565 * %false - we are done with this request 2566 * %true - still buffers pending for this request 2567 **/ 2568 bool __blk_end_bidi_request(struct request *rq, int error, 2569 unsigned int nr_bytes, unsigned int bidi_bytes) 2570 { 2571 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes)) 2572 return true; 2573 2574 blk_finish_request(rq, error); 2575 2576 return false; 2577 } 2578 2579 /** 2580 * blk_end_request - Helper function for drivers to complete the request. 2581 * @rq: the request being processed 2582 * @error: %0 for success, < %0 for error 2583 * @nr_bytes: number of bytes to complete 2584 * 2585 * Description: 2586 * Ends I/O on a number of bytes attached to @rq. 2587 * If @rq has leftover, sets it up for the next range of segments. 2588 * 2589 * Return: 2590 * %false - we are done with this request 2591 * %true - still buffers pending for this request 2592 **/ 2593 bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes) 2594 { 2595 return blk_end_bidi_request(rq, error, nr_bytes, 0); 2596 } 2597 EXPORT_SYMBOL(blk_end_request); 2598 2599 /** 2600 * blk_end_request_all - Helper function for drives to finish the request. 2601 * @rq: the request to finish 2602 * @error: %0 for success, < %0 for error 2603 * 2604 * Description: 2605 * Completely finish @rq. 2606 */ 2607 void blk_end_request_all(struct request *rq, int error) 2608 { 2609 bool pending; 2610 unsigned int bidi_bytes = 0; 2611 2612 if (unlikely(blk_bidi_rq(rq))) 2613 bidi_bytes = blk_rq_bytes(rq->next_rq); 2614 2615 pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes); 2616 BUG_ON(pending); 2617 } 2618 EXPORT_SYMBOL(blk_end_request_all); 2619 2620 /** 2621 * blk_end_request_cur - Helper function to finish the current request chunk. 2622 * @rq: the request to finish the current chunk for 2623 * @error: %0 for success, < %0 for error 2624 * 2625 * Description: 2626 * Complete the current consecutively mapped chunk from @rq. 2627 * 2628 * Return: 2629 * %false - we are done with this request 2630 * %true - still buffers pending for this request 2631 */ 2632 bool blk_end_request_cur(struct request *rq, int error) 2633 { 2634 return blk_end_request(rq, error, blk_rq_cur_bytes(rq)); 2635 } 2636 EXPORT_SYMBOL(blk_end_request_cur); 2637 2638 /** 2639 * blk_end_request_err - Finish a request till the next failure boundary. 2640 * @rq: the request to finish till the next failure boundary for 2641 * @error: must be negative errno 2642 * 2643 * Description: 2644 * Complete @rq till the next failure boundary. 2645 * 2646 * Return: 2647 * %false - we are done with this request 2648 * %true - still buffers pending for this request 2649 */ 2650 bool blk_end_request_err(struct request *rq, int error) 2651 { 2652 WARN_ON(error >= 0); 2653 return blk_end_request(rq, error, blk_rq_err_bytes(rq)); 2654 } 2655 EXPORT_SYMBOL_GPL(blk_end_request_err); 2656 2657 /** 2658 * __blk_end_request - Helper function for drivers to complete the request. 2659 * @rq: the request being processed 2660 * @error: %0 for success, < %0 for error 2661 * @nr_bytes: number of bytes to complete 2662 * 2663 * Description: 2664 * Must be called with queue lock held unlike blk_end_request(). 2665 * 2666 * Return: 2667 * %false - we are done with this request 2668 * %true - still buffers pending for this request 2669 **/ 2670 bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes) 2671 { 2672 return __blk_end_bidi_request(rq, error, nr_bytes, 0); 2673 } 2674 EXPORT_SYMBOL(__blk_end_request); 2675 2676 /** 2677 * __blk_end_request_all - Helper function for drives to finish the request. 2678 * @rq: the request to finish 2679 * @error: %0 for success, < %0 for error 2680 * 2681 * Description: 2682 * Completely finish @rq. Must be called with queue lock held. 2683 */ 2684 void __blk_end_request_all(struct request *rq, int error) 2685 { 2686 bool pending; 2687 unsigned int bidi_bytes = 0; 2688 2689 if (unlikely(blk_bidi_rq(rq))) 2690 bidi_bytes = blk_rq_bytes(rq->next_rq); 2691 2692 pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes); 2693 BUG_ON(pending); 2694 } 2695 EXPORT_SYMBOL(__blk_end_request_all); 2696 2697 /** 2698 * __blk_end_request_cur - Helper function to finish the current request chunk. 2699 * @rq: the request to finish the current chunk for 2700 * @error: %0 for success, < %0 for error 2701 * 2702 * Description: 2703 * Complete the current consecutively mapped chunk from @rq. Must 2704 * be called with queue lock held. 2705 * 2706 * Return: 2707 * %false - we are done with this request 2708 * %true - still buffers pending for this request 2709 */ 2710 bool __blk_end_request_cur(struct request *rq, int error) 2711 { 2712 return __blk_end_request(rq, error, blk_rq_cur_bytes(rq)); 2713 } 2714 EXPORT_SYMBOL(__blk_end_request_cur); 2715 2716 /** 2717 * __blk_end_request_err - Finish a request till the next failure boundary. 2718 * @rq: the request to finish till the next failure boundary for 2719 * @error: must be negative errno 2720 * 2721 * Description: 2722 * Complete @rq till the next failure boundary. Must be called 2723 * with queue lock held. 2724 * 2725 * Return: 2726 * %false - we are done with this request 2727 * %true - still buffers pending for this request 2728 */ 2729 bool __blk_end_request_err(struct request *rq, int error) 2730 { 2731 WARN_ON(error >= 0); 2732 return __blk_end_request(rq, error, blk_rq_err_bytes(rq)); 2733 } 2734 EXPORT_SYMBOL_GPL(__blk_end_request_err); 2735 2736 void blk_rq_bio_prep(struct request_queue *q, struct request *rq, 2737 struct bio *bio) 2738 { 2739 /* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */ 2740 rq->cmd_flags |= bio->bi_rw & REQ_WRITE; 2741 2742 if (bio_has_data(bio)) { 2743 rq->nr_phys_segments = bio_phys_segments(q, bio); 2744 rq->buffer = bio_data(bio); 2745 } 2746 rq->__data_len = bio->bi_iter.bi_size; 2747 rq->bio = rq->biotail = bio; 2748 2749 if (bio->bi_bdev) 2750 rq->rq_disk = bio->bi_bdev->bd_disk; 2751 } 2752 2753 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 2754 /** 2755 * rq_flush_dcache_pages - Helper function to flush all pages in a request 2756 * @rq: the request to be flushed 2757 * 2758 * Description: 2759 * Flush all pages in @rq. 2760 */ 2761 void rq_flush_dcache_pages(struct request *rq) 2762 { 2763 struct req_iterator iter; 2764 struct bio_vec bvec; 2765 2766 rq_for_each_segment(bvec, rq, iter) 2767 flush_dcache_page(bvec.bv_page); 2768 } 2769 EXPORT_SYMBOL_GPL(rq_flush_dcache_pages); 2770 #endif 2771 2772 /** 2773 * blk_lld_busy - Check if underlying low-level drivers of a device are busy 2774 * @q : the queue of the device being checked 2775 * 2776 * Description: 2777 * Check if underlying low-level drivers of a device are busy. 2778 * If the drivers want to export their busy state, they must set own 2779 * exporting function using blk_queue_lld_busy() first. 2780 * 2781 * Basically, this function is used only by request stacking drivers 2782 * to stop dispatching requests to underlying devices when underlying 2783 * devices are busy. This behavior helps more I/O merging on the queue 2784 * of the request stacking driver and prevents I/O throughput regression 2785 * on burst I/O load. 2786 * 2787 * Return: 2788 * 0 - Not busy (The request stacking driver should dispatch request) 2789 * 1 - Busy (The request stacking driver should stop dispatching request) 2790 */ 2791 int blk_lld_busy(struct request_queue *q) 2792 { 2793 if (q->lld_busy_fn) 2794 return q->lld_busy_fn(q); 2795 2796 return 0; 2797 } 2798 EXPORT_SYMBOL_GPL(blk_lld_busy); 2799 2800 /** 2801 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request 2802 * @rq: the clone request to be cleaned up 2803 * 2804 * Description: 2805 * Free all bios in @rq for a cloned request. 2806 */ 2807 void blk_rq_unprep_clone(struct request *rq) 2808 { 2809 struct bio *bio; 2810 2811 while ((bio = rq->bio) != NULL) { 2812 rq->bio = bio->bi_next; 2813 2814 bio_put(bio); 2815 } 2816 } 2817 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); 2818 2819 /* 2820 * Copy attributes of the original request to the clone request. 2821 * The actual data parts (e.g. ->cmd, ->buffer, ->sense) are not copied. 2822 */ 2823 static void __blk_rq_prep_clone(struct request *dst, struct request *src) 2824 { 2825 dst->cpu = src->cpu; 2826 dst->cmd_flags = (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE; 2827 dst->cmd_type = src->cmd_type; 2828 dst->__sector = blk_rq_pos(src); 2829 dst->__data_len = blk_rq_bytes(src); 2830 dst->nr_phys_segments = src->nr_phys_segments; 2831 dst->ioprio = src->ioprio; 2832 dst->extra_len = src->extra_len; 2833 } 2834 2835 /** 2836 * blk_rq_prep_clone - Helper function to setup clone request 2837 * @rq: the request to be setup 2838 * @rq_src: original request to be cloned 2839 * @bs: bio_set that bios for clone are allocated from 2840 * @gfp_mask: memory allocation mask for bio 2841 * @bio_ctr: setup function to be called for each clone bio. 2842 * Returns %0 for success, non %0 for failure. 2843 * @data: private data to be passed to @bio_ctr 2844 * 2845 * Description: 2846 * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq. 2847 * The actual data parts of @rq_src (e.g. ->cmd, ->buffer, ->sense) 2848 * are not copied, and copying such parts is the caller's responsibility. 2849 * Also, pages which the original bios are pointing to are not copied 2850 * and the cloned bios just point same pages. 2851 * So cloned bios must be completed before original bios, which means 2852 * the caller must complete @rq before @rq_src. 2853 */ 2854 int blk_rq_prep_clone(struct request *rq, struct request *rq_src, 2855 struct bio_set *bs, gfp_t gfp_mask, 2856 int (*bio_ctr)(struct bio *, struct bio *, void *), 2857 void *data) 2858 { 2859 struct bio *bio, *bio_src; 2860 2861 if (!bs) 2862 bs = fs_bio_set; 2863 2864 blk_rq_init(NULL, rq); 2865 2866 __rq_for_each_bio(bio_src, rq_src) { 2867 bio = bio_clone_bioset(bio_src, gfp_mask, bs); 2868 if (!bio) 2869 goto free_and_out; 2870 2871 if (bio_ctr && bio_ctr(bio, bio_src, data)) 2872 goto free_and_out; 2873 2874 if (rq->bio) { 2875 rq->biotail->bi_next = bio; 2876 rq->biotail = bio; 2877 } else 2878 rq->bio = rq->biotail = bio; 2879 } 2880 2881 __blk_rq_prep_clone(rq, rq_src); 2882 2883 return 0; 2884 2885 free_and_out: 2886 if (bio) 2887 bio_put(bio); 2888 blk_rq_unprep_clone(rq); 2889 2890 return -ENOMEM; 2891 } 2892 EXPORT_SYMBOL_GPL(blk_rq_prep_clone); 2893 2894 int kblockd_schedule_work(struct request_queue *q, struct work_struct *work) 2895 { 2896 return queue_work(kblockd_workqueue, work); 2897 } 2898 EXPORT_SYMBOL(kblockd_schedule_work); 2899 2900 int kblockd_schedule_delayed_work(struct request_queue *q, 2901 struct delayed_work *dwork, unsigned long delay) 2902 { 2903 return queue_delayed_work(kblockd_workqueue, dwork, delay); 2904 } 2905 EXPORT_SYMBOL(kblockd_schedule_delayed_work); 2906 2907 #define PLUG_MAGIC 0x91827364 2908 2909 /** 2910 * blk_start_plug - initialize blk_plug and track it inside the task_struct 2911 * @plug: The &struct blk_plug that needs to be initialized 2912 * 2913 * Description: 2914 * Tracking blk_plug inside the task_struct will help with auto-flushing the 2915 * pending I/O should the task end up blocking between blk_start_plug() and 2916 * blk_finish_plug(). This is important from a performance perspective, but 2917 * also ensures that we don't deadlock. For instance, if the task is blocking 2918 * for a memory allocation, memory reclaim could end up wanting to free a 2919 * page belonging to that request that is currently residing in our private 2920 * plug. By flushing the pending I/O when the process goes to sleep, we avoid 2921 * this kind of deadlock. 2922 */ 2923 void blk_start_plug(struct blk_plug *plug) 2924 { 2925 struct task_struct *tsk = current; 2926 2927 plug->magic = PLUG_MAGIC; 2928 INIT_LIST_HEAD(&plug->list); 2929 INIT_LIST_HEAD(&plug->mq_list); 2930 INIT_LIST_HEAD(&plug->cb_list); 2931 2932 /* 2933 * If this is a nested plug, don't actually assign it. It will be 2934 * flushed on its own. 2935 */ 2936 if (!tsk->plug) { 2937 /* 2938 * Store ordering should not be needed here, since a potential 2939 * preempt will imply a full memory barrier 2940 */ 2941 tsk->plug = plug; 2942 } 2943 } 2944 EXPORT_SYMBOL(blk_start_plug); 2945 2946 static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b) 2947 { 2948 struct request *rqa = container_of(a, struct request, queuelist); 2949 struct request *rqb = container_of(b, struct request, queuelist); 2950 2951 return !(rqa->q < rqb->q || 2952 (rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb))); 2953 } 2954 2955 /* 2956 * If 'from_schedule' is true, then postpone the dispatch of requests 2957 * until a safe kblockd context. We due this to avoid accidental big 2958 * additional stack usage in driver dispatch, in places where the originally 2959 * plugger did not intend it. 2960 */ 2961 static void queue_unplugged(struct request_queue *q, unsigned int depth, 2962 bool from_schedule) 2963 __releases(q->queue_lock) 2964 { 2965 trace_block_unplug(q, depth, !from_schedule); 2966 2967 if (from_schedule) 2968 blk_run_queue_async(q); 2969 else 2970 __blk_run_queue(q); 2971 spin_unlock(q->queue_lock); 2972 } 2973 2974 static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule) 2975 { 2976 LIST_HEAD(callbacks); 2977 2978 while (!list_empty(&plug->cb_list)) { 2979 list_splice_init(&plug->cb_list, &callbacks); 2980 2981 while (!list_empty(&callbacks)) { 2982 struct blk_plug_cb *cb = list_first_entry(&callbacks, 2983 struct blk_plug_cb, 2984 list); 2985 list_del(&cb->list); 2986 cb->callback(cb, from_schedule); 2987 } 2988 } 2989 } 2990 2991 struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data, 2992 int size) 2993 { 2994 struct blk_plug *plug = current->plug; 2995 struct blk_plug_cb *cb; 2996 2997 if (!plug) 2998 return NULL; 2999 3000 list_for_each_entry(cb, &plug->cb_list, list) 3001 if (cb->callback == unplug && cb->data == data) 3002 return cb; 3003 3004 /* Not currently on the callback list */ 3005 BUG_ON(size < sizeof(*cb)); 3006 cb = kzalloc(size, GFP_ATOMIC); 3007 if (cb) { 3008 cb->data = data; 3009 cb->callback = unplug; 3010 list_add(&cb->list, &plug->cb_list); 3011 } 3012 return cb; 3013 } 3014 EXPORT_SYMBOL(blk_check_plugged); 3015 3016 void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) 3017 { 3018 struct request_queue *q; 3019 unsigned long flags; 3020 struct request *rq; 3021 LIST_HEAD(list); 3022 unsigned int depth; 3023 3024 BUG_ON(plug->magic != PLUG_MAGIC); 3025 3026 flush_plug_callbacks(plug, from_schedule); 3027 3028 if (!list_empty(&plug->mq_list)) 3029 blk_mq_flush_plug_list(plug, from_schedule); 3030 3031 if (list_empty(&plug->list)) 3032 return; 3033 3034 list_splice_init(&plug->list, &list); 3035 3036 list_sort(NULL, &list, plug_rq_cmp); 3037 3038 q = NULL; 3039 depth = 0; 3040 3041 /* 3042 * Save and disable interrupts here, to avoid doing it for every 3043 * queue lock we have to take. 3044 */ 3045 local_irq_save(flags); 3046 while (!list_empty(&list)) { 3047 rq = list_entry_rq(list.next); 3048 list_del_init(&rq->queuelist); 3049 BUG_ON(!rq->q); 3050 if (rq->q != q) { 3051 /* 3052 * This drops the queue lock 3053 */ 3054 if (q) 3055 queue_unplugged(q, depth, from_schedule); 3056 q = rq->q; 3057 depth = 0; 3058 spin_lock(q->queue_lock); 3059 } 3060 3061 /* 3062 * Short-circuit if @q is dead 3063 */ 3064 if (unlikely(blk_queue_dying(q))) { 3065 __blk_end_request_all(rq, -ENODEV); 3066 continue; 3067 } 3068 3069 /* 3070 * rq is already accounted, so use raw insert 3071 */ 3072 if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA)) 3073 __elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH); 3074 else 3075 __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE); 3076 3077 depth++; 3078 } 3079 3080 /* 3081 * This drops the queue lock 3082 */ 3083 if (q) 3084 queue_unplugged(q, depth, from_schedule); 3085 3086 local_irq_restore(flags); 3087 } 3088 3089 void blk_finish_plug(struct blk_plug *plug) 3090 { 3091 blk_flush_plug_list(plug, false); 3092 3093 if (plug == current->plug) 3094 current->plug = NULL; 3095 } 3096 EXPORT_SYMBOL(blk_finish_plug); 3097 3098 #ifdef CONFIG_PM_RUNTIME 3099 /** 3100 * blk_pm_runtime_init - Block layer runtime PM initialization routine 3101 * @q: the queue of the device 3102 * @dev: the device the queue belongs to 3103 * 3104 * Description: 3105 * Initialize runtime-PM-related fields for @q and start auto suspend for 3106 * @dev. Drivers that want to take advantage of request-based runtime PM 3107 * should call this function after @dev has been initialized, and its 3108 * request queue @q has been allocated, and runtime PM for it can not happen 3109 * yet(either due to disabled/forbidden or its usage_count > 0). In most 3110 * cases, driver should call this function before any I/O has taken place. 3111 * 3112 * This function takes care of setting up using auto suspend for the device, 3113 * the autosuspend delay is set to -1 to make runtime suspend impossible 3114 * until an updated value is either set by user or by driver. Drivers do 3115 * not need to touch other autosuspend settings. 3116 * 3117 * The block layer runtime PM is request based, so only works for drivers 3118 * that use request as their IO unit instead of those directly use bio's. 3119 */ 3120 void blk_pm_runtime_init(struct request_queue *q, struct device *dev) 3121 { 3122 q->dev = dev; 3123 q->rpm_status = RPM_ACTIVE; 3124 pm_runtime_set_autosuspend_delay(q->dev, -1); 3125 pm_runtime_use_autosuspend(q->dev); 3126 } 3127 EXPORT_SYMBOL(blk_pm_runtime_init); 3128 3129 /** 3130 * blk_pre_runtime_suspend - Pre runtime suspend check 3131 * @q: the queue of the device 3132 * 3133 * Description: 3134 * This function will check if runtime suspend is allowed for the device 3135 * by examining if there are any requests pending in the queue. If there 3136 * are requests pending, the device can not be runtime suspended; otherwise, 3137 * the queue's status will be updated to SUSPENDING and the driver can 3138 * proceed to suspend the device. 3139 * 3140 * For the not allowed case, we mark last busy for the device so that 3141 * runtime PM core will try to autosuspend it some time later. 3142 * 3143 * This function should be called near the start of the device's 3144 * runtime_suspend callback. 3145 * 3146 * Return: 3147 * 0 - OK to runtime suspend the device 3148 * -EBUSY - Device should not be runtime suspended 3149 */ 3150 int blk_pre_runtime_suspend(struct request_queue *q) 3151 { 3152 int ret = 0; 3153 3154 spin_lock_irq(q->queue_lock); 3155 if (q->nr_pending) { 3156 ret = -EBUSY; 3157 pm_runtime_mark_last_busy(q->dev); 3158 } else { 3159 q->rpm_status = RPM_SUSPENDING; 3160 } 3161 spin_unlock_irq(q->queue_lock); 3162 return ret; 3163 } 3164 EXPORT_SYMBOL(blk_pre_runtime_suspend); 3165 3166 /** 3167 * blk_post_runtime_suspend - Post runtime suspend processing 3168 * @q: the queue of the device 3169 * @err: return value of the device's runtime_suspend function 3170 * 3171 * Description: 3172 * Update the queue's runtime status according to the return value of the 3173 * device's runtime suspend function and mark last busy for the device so 3174 * that PM core will try to auto suspend the device at a later time. 3175 * 3176 * This function should be called near the end of the device's 3177 * runtime_suspend callback. 3178 */ 3179 void blk_post_runtime_suspend(struct request_queue *q, int err) 3180 { 3181 spin_lock_irq(q->queue_lock); 3182 if (!err) { 3183 q->rpm_status = RPM_SUSPENDED; 3184 } else { 3185 q->rpm_status = RPM_ACTIVE; 3186 pm_runtime_mark_last_busy(q->dev); 3187 } 3188 spin_unlock_irq(q->queue_lock); 3189 } 3190 EXPORT_SYMBOL(blk_post_runtime_suspend); 3191 3192 /** 3193 * blk_pre_runtime_resume - Pre runtime resume processing 3194 * @q: the queue of the device 3195 * 3196 * Description: 3197 * Update the queue's runtime status to RESUMING in preparation for the 3198 * runtime resume of the device. 3199 * 3200 * This function should be called near the start of the device's 3201 * runtime_resume callback. 3202 */ 3203 void blk_pre_runtime_resume(struct request_queue *q) 3204 { 3205 spin_lock_irq(q->queue_lock); 3206 q->rpm_status = RPM_RESUMING; 3207 spin_unlock_irq(q->queue_lock); 3208 } 3209 EXPORT_SYMBOL(blk_pre_runtime_resume); 3210 3211 /** 3212 * blk_post_runtime_resume - Post runtime resume processing 3213 * @q: the queue of the device 3214 * @err: return value of the device's runtime_resume function 3215 * 3216 * Description: 3217 * Update the queue's runtime status according to the return value of the 3218 * device's runtime_resume function. If it is successfully resumed, process 3219 * the requests that are queued into the device's queue when it is resuming 3220 * and then mark last busy and initiate autosuspend for it. 3221 * 3222 * This function should be called near the end of the device's 3223 * runtime_resume callback. 3224 */ 3225 void blk_post_runtime_resume(struct request_queue *q, int err) 3226 { 3227 spin_lock_irq(q->queue_lock); 3228 if (!err) { 3229 q->rpm_status = RPM_ACTIVE; 3230 __blk_run_queue(q); 3231 pm_runtime_mark_last_busy(q->dev); 3232 pm_request_autosuspend(q->dev); 3233 } else { 3234 q->rpm_status = RPM_SUSPENDED; 3235 } 3236 spin_unlock_irq(q->queue_lock); 3237 } 3238 EXPORT_SYMBOL(blk_post_runtime_resume); 3239 #endif 3240 3241 int __init blk_dev_init(void) 3242 { 3243 BUILD_BUG_ON(__REQ_NR_BITS > 8 * 3244 sizeof(((struct request *)0)->cmd_flags)); 3245 3246 /* used for unplugging and affects IO latency/throughput - HIGHPRI */ 3247 kblockd_workqueue = alloc_workqueue("kblockd", 3248 WQ_MEM_RECLAIM | WQ_HIGHPRI | 3249 WQ_POWER_EFFICIENT, 0); 3250 if (!kblockd_workqueue) 3251 panic("Failed to create kblockd\n"); 3252 3253 request_cachep = kmem_cache_create("blkdev_requests", 3254 sizeof(struct request), 0, SLAB_PANIC, NULL); 3255 3256 blk_requestq_cachep = kmem_cache_create("blkdev_queue", 3257 sizeof(struct request_queue), 0, SLAB_PANIC, NULL); 3258 3259 return 0; 3260 } 3261