xref: /openbmc/linux/block/blk-core.c (revision 174cd4b1)
1 /*
2  * Copyright (C) 1991, 1992 Linus Torvalds
3  * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
4  * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
5  * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
6  * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
7  *	-  July2000
8  * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
9  */
10 
11 /*
12  * This handles all read/write requests to block devices
13  */
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/backing-dev.h>
17 #include <linux/bio.h>
18 #include <linux/blkdev.h>
19 #include <linux/blk-mq.h>
20 #include <linux/highmem.h>
21 #include <linux/mm.h>
22 #include <linux/kernel_stat.h>
23 #include <linux/string.h>
24 #include <linux/init.h>
25 #include <linux/completion.h>
26 #include <linux/slab.h>
27 #include <linux/swap.h>
28 #include <linux/writeback.h>
29 #include <linux/task_io_accounting_ops.h>
30 #include <linux/fault-inject.h>
31 #include <linux/list_sort.h>
32 #include <linux/delay.h>
33 #include <linux/ratelimit.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/blk-cgroup.h>
36 #include <linux/debugfs.h>
37 
38 #define CREATE_TRACE_POINTS
39 #include <trace/events/block.h>
40 
41 #include "blk.h"
42 #include "blk-mq.h"
43 #include "blk-mq-sched.h"
44 #include "blk-wbt.h"
45 
46 #ifdef CONFIG_DEBUG_FS
47 struct dentry *blk_debugfs_root;
48 #endif
49 
50 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
51 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
52 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
53 EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
54 EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
55 
56 DEFINE_IDA(blk_queue_ida);
57 
58 /*
59  * For the allocated request tables
60  */
61 struct kmem_cache *request_cachep;
62 
63 /*
64  * For queue allocation
65  */
66 struct kmem_cache *blk_requestq_cachep;
67 
68 /*
69  * Controlling structure to kblockd
70  */
71 static struct workqueue_struct *kblockd_workqueue;
72 
73 static void blk_clear_congested(struct request_list *rl, int sync)
74 {
75 #ifdef CONFIG_CGROUP_WRITEBACK
76 	clear_wb_congested(rl->blkg->wb_congested, sync);
77 #else
78 	/*
79 	 * If !CGROUP_WRITEBACK, all blkg's map to bdi->wb and we shouldn't
80 	 * flip its congestion state for events on other blkcgs.
81 	 */
82 	if (rl == &rl->q->root_rl)
83 		clear_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
84 #endif
85 }
86 
87 static void blk_set_congested(struct request_list *rl, int sync)
88 {
89 #ifdef CONFIG_CGROUP_WRITEBACK
90 	set_wb_congested(rl->blkg->wb_congested, sync);
91 #else
92 	/* see blk_clear_congested() */
93 	if (rl == &rl->q->root_rl)
94 		set_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
95 #endif
96 }
97 
98 void blk_queue_congestion_threshold(struct request_queue *q)
99 {
100 	int nr;
101 
102 	nr = q->nr_requests - (q->nr_requests / 8) + 1;
103 	if (nr > q->nr_requests)
104 		nr = q->nr_requests;
105 	q->nr_congestion_on = nr;
106 
107 	nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
108 	if (nr < 1)
109 		nr = 1;
110 	q->nr_congestion_off = nr;
111 }
112 
113 void blk_rq_init(struct request_queue *q, struct request *rq)
114 {
115 	memset(rq, 0, sizeof(*rq));
116 
117 	INIT_LIST_HEAD(&rq->queuelist);
118 	INIT_LIST_HEAD(&rq->timeout_list);
119 	rq->cpu = -1;
120 	rq->q = q;
121 	rq->__sector = (sector_t) -1;
122 	INIT_HLIST_NODE(&rq->hash);
123 	RB_CLEAR_NODE(&rq->rb_node);
124 	rq->tag = -1;
125 	rq->internal_tag = -1;
126 	rq->start_time = jiffies;
127 	set_start_time_ns(rq);
128 	rq->part = NULL;
129 }
130 EXPORT_SYMBOL(blk_rq_init);
131 
132 static void req_bio_endio(struct request *rq, struct bio *bio,
133 			  unsigned int nbytes, int error)
134 {
135 	if (error)
136 		bio->bi_error = error;
137 
138 	if (unlikely(rq->rq_flags & RQF_QUIET))
139 		bio_set_flag(bio, BIO_QUIET);
140 
141 	bio_advance(bio, nbytes);
142 
143 	/* don't actually finish bio if it's part of flush sequence */
144 	if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
145 		bio_endio(bio);
146 }
147 
148 void blk_dump_rq_flags(struct request *rq, char *msg)
149 {
150 	printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
151 		rq->rq_disk ? rq->rq_disk->disk_name : "?",
152 		(unsigned long long) rq->cmd_flags);
153 
154 	printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
155 	       (unsigned long long)blk_rq_pos(rq),
156 	       blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
157 	printk(KERN_INFO "  bio %p, biotail %p, len %u\n",
158 	       rq->bio, rq->biotail, blk_rq_bytes(rq));
159 }
160 EXPORT_SYMBOL(blk_dump_rq_flags);
161 
162 static void blk_delay_work(struct work_struct *work)
163 {
164 	struct request_queue *q;
165 
166 	q = container_of(work, struct request_queue, delay_work.work);
167 	spin_lock_irq(q->queue_lock);
168 	__blk_run_queue(q);
169 	spin_unlock_irq(q->queue_lock);
170 }
171 
172 /**
173  * blk_delay_queue - restart queueing after defined interval
174  * @q:		The &struct request_queue in question
175  * @msecs:	Delay in msecs
176  *
177  * Description:
178  *   Sometimes queueing needs to be postponed for a little while, to allow
179  *   resources to come back. This function will make sure that queueing is
180  *   restarted around the specified time. Queue lock must be held.
181  */
182 void blk_delay_queue(struct request_queue *q, unsigned long msecs)
183 {
184 	if (likely(!blk_queue_dead(q)))
185 		queue_delayed_work(kblockd_workqueue, &q->delay_work,
186 				   msecs_to_jiffies(msecs));
187 }
188 EXPORT_SYMBOL(blk_delay_queue);
189 
190 /**
191  * blk_start_queue_async - asynchronously restart a previously stopped queue
192  * @q:    The &struct request_queue in question
193  *
194  * Description:
195  *   blk_start_queue_async() will clear the stop flag on the queue, and
196  *   ensure that the request_fn for the queue is run from an async
197  *   context.
198  **/
199 void blk_start_queue_async(struct request_queue *q)
200 {
201 	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
202 	blk_run_queue_async(q);
203 }
204 EXPORT_SYMBOL(blk_start_queue_async);
205 
206 /**
207  * blk_start_queue - restart a previously stopped queue
208  * @q:    The &struct request_queue in question
209  *
210  * Description:
211  *   blk_start_queue() will clear the stop flag on the queue, and call
212  *   the request_fn for the queue if it was in a stopped state when
213  *   entered. Also see blk_stop_queue(). Queue lock must be held.
214  **/
215 void blk_start_queue(struct request_queue *q)
216 {
217 	WARN_ON(!irqs_disabled());
218 
219 	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
220 	__blk_run_queue(q);
221 }
222 EXPORT_SYMBOL(blk_start_queue);
223 
224 /**
225  * blk_stop_queue - stop a queue
226  * @q:    The &struct request_queue in question
227  *
228  * Description:
229  *   The Linux block layer assumes that a block driver will consume all
230  *   entries on the request queue when the request_fn strategy is called.
231  *   Often this will not happen, because of hardware limitations (queue
232  *   depth settings). If a device driver gets a 'queue full' response,
233  *   or if it simply chooses not to queue more I/O at one point, it can
234  *   call this function to prevent the request_fn from being called until
235  *   the driver has signalled it's ready to go again. This happens by calling
236  *   blk_start_queue() to restart queue operations. Queue lock must be held.
237  **/
238 void blk_stop_queue(struct request_queue *q)
239 {
240 	cancel_delayed_work(&q->delay_work);
241 	queue_flag_set(QUEUE_FLAG_STOPPED, q);
242 }
243 EXPORT_SYMBOL(blk_stop_queue);
244 
245 /**
246  * blk_sync_queue - cancel any pending callbacks on a queue
247  * @q: the queue
248  *
249  * Description:
250  *     The block layer may perform asynchronous callback activity
251  *     on a queue, such as calling the unplug function after a timeout.
252  *     A block device may call blk_sync_queue to ensure that any
253  *     such activity is cancelled, thus allowing it to release resources
254  *     that the callbacks might use. The caller must already have made sure
255  *     that its ->make_request_fn will not re-add plugging prior to calling
256  *     this function.
257  *
258  *     This function does not cancel any asynchronous activity arising
259  *     out of elevator or throttling code. That would require elevator_exit()
260  *     and blkcg_exit_queue() to be called with queue lock initialized.
261  *
262  */
263 void blk_sync_queue(struct request_queue *q)
264 {
265 	del_timer_sync(&q->timeout);
266 
267 	if (q->mq_ops) {
268 		struct blk_mq_hw_ctx *hctx;
269 		int i;
270 
271 		queue_for_each_hw_ctx(q, hctx, i) {
272 			cancel_work_sync(&hctx->run_work);
273 			cancel_delayed_work_sync(&hctx->delay_work);
274 		}
275 	} else {
276 		cancel_delayed_work_sync(&q->delay_work);
277 	}
278 }
279 EXPORT_SYMBOL(blk_sync_queue);
280 
281 /**
282  * __blk_run_queue_uncond - run a queue whether or not it has been stopped
283  * @q:	The queue to run
284  *
285  * Description:
286  *    Invoke request handling on a queue if there are any pending requests.
287  *    May be used to restart request handling after a request has completed.
288  *    This variant runs the queue whether or not the queue has been
289  *    stopped. Must be called with the queue lock held and interrupts
290  *    disabled. See also @blk_run_queue.
291  */
292 inline void __blk_run_queue_uncond(struct request_queue *q)
293 {
294 	if (unlikely(blk_queue_dead(q)))
295 		return;
296 
297 	/*
298 	 * Some request_fn implementations, e.g. scsi_request_fn(), unlock
299 	 * the queue lock internally. As a result multiple threads may be
300 	 * running such a request function concurrently. Keep track of the
301 	 * number of active request_fn invocations such that blk_drain_queue()
302 	 * can wait until all these request_fn calls have finished.
303 	 */
304 	q->request_fn_active++;
305 	q->request_fn(q);
306 	q->request_fn_active--;
307 }
308 EXPORT_SYMBOL_GPL(__blk_run_queue_uncond);
309 
310 /**
311  * __blk_run_queue - run a single device queue
312  * @q:	The queue to run
313  *
314  * Description:
315  *    See @blk_run_queue. This variant must be called with the queue lock
316  *    held and interrupts disabled.
317  */
318 void __blk_run_queue(struct request_queue *q)
319 {
320 	if (unlikely(blk_queue_stopped(q)))
321 		return;
322 
323 	__blk_run_queue_uncond(q);
324 }
325 EXPORT_SYMBOL(__blk_run_queue);
326 
327 /**
328  * blk_run_queue_async - run a single device queue in workqueue context
329  * @q:	The queue to run
330  *
331  * Description:
332  *    Tells kblockd to perform the equivalent of @blk_run_queue on behalf
333  *    of us. The caller must hold the queue lock.
334  */
335 void blk_run_queue_async(struct request_queue *q)
336 {
337 	if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q)))
338 		mod_delayed_work(kblockd_workqueue, &q->delay_work, 0);
339 }
340 EXPORT_SYMBOL(blk_run_queue_async);
341 
342 /**
343  * blk_run_queue - run a single device queue
344  * @q: The queue to run
345  *
346  * Description:
347  *    Invoke request handling on this queue, if it has pending work to do.
348  *    May be used to restart queueing when a request has completed.
349  */
350 void blk_run_queue(struct request_queue *q)
351 {
352 	unsigned long flags;
353 
354 	spin_lock_irqsave(q->queue_lock, flags);
355 	__blk_run_queue(q);
356 	spin_unlock_irqrestore(q->queue_lock, flags);
357 }
358 EXPORT_SYMBOL(blk_run_queue);
359 
360 void blk_put_queue(struct request_queue *q)
361 {
362 	kobject_put(&q->kobj);
363 }
364 EXPORT_SYMBOL(blk_put_queue);
365 
366 /**
367  * __blk_drain_queue - drain requests from request_queue
368  * @q: queue to drain
369  * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV
370  *
371  * Drain requests from @q.  If @drain_all is set, all requests are drained.
372  * If not, only ELVPRIV requests are drained.  The caller is responsible
373  * for ensuring that no new requests which need to be drained are queued.
374  */
375 static void __blk_drain_queue(struct request_queue *q, bool drain_all)
376 	__releases(q->queue_lock)
377 	__acquires(q->queue_lock)
378 {
379 	int i;
380 
381 	lockdep_assert_held(q->queue_lock);
382 
383 	while (true) {
384 		bool drain = false;
385 
386 		/*
387 		 * The caller might be trying to drain @q before its
388 		 * elevator is initialized.
389 		 */
390 		if (q->elevator)
391 			elv_drain_elevator(q);
392 
393 		blkcg_drain_queue(q);
394 
395 		/*
396 		 * This function might be called on a queue which failed
397 		 * driver init after queue creation or is not yet fully
398 		 * active yet.  Some drivers (e.g. fd and loop) get unhappy
399 		 * in such cases.  Kick queue iff dispatch queue has
400 		 * something on it and @q has request_fn set.
401 		 */
402 		if (!list_empty(&q->queue_head) && q->request_fn)
403 			__blk_run_queue(q);
404 
405 		drain |= q->nr_rqs_elvpriv;
406 		drain |= q->request_fn_active;
407 
408 		/*
409 		 * Unfortunately, requests are queued at and tracked from
410 		 * multiple places and there's no single counter which can
411 		 * be drained.  Check all the queues and counters.
412 		 */
413 		if (drain_all) {
414 			struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
415 			drain |= !list_empty(&q->queue_head);
416 			for (i = 0; i < 2; i++) {
417 				drain |= q->nr_rqs[i];
418 				drain |= q->in_flight[i];
419 				if (fq)
420 				    drain |= !list_empty(&fq->flush_queue[i]);
421 			}
422 		}
423 
424 		if (!drain)
425 			break;
426 
427 		spin_unlock_irq(q->queue_lock);
428 
429 		msleep(10);
430 
431 		spin_lock_irq(q->queue_lock);
432 	}
433 
434 	/*
435 	 * With queue marked dead, any woken up waiter will fail the
436 	 * allocation path, so the wakeup chaining is lost and we're
437 	 * left with hung waiters. We need to wake up those waiters.
438 	 */
439 	if (q->request_fn) {
440 		struct request_list *rl;
441 
442 		blk_queue_for_each_rl(rl, q)
443 			for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
444 				wake_up_all(&rl->wait[i]);
445 	}
446 }
447 
448 /**
449  * blk_queue_bypass_start - enter queue bypass mode
450  * @q: queue of interest
451  *
452  * In bypass mode, only the dispatch FIFO queue of @q is used.  This
453  * function makes @q enter bypass mode and drains all requests which were
454  * throttled or issued before.  On return, it's guaranteed that no request
455  * is being throttled or has ELVPRIV set and blk_queue_bypass() %true
456  * inside queue or RCU read lock.
457  */
458 void blk_queue_bypass_start(struct request_queue *q)
459 {
460 	spin_lock_irq(q->queue_lock);
461 	q->bypass_depth++;
462 	queue_flag_set(QUEUE_FLAG_BYPASS, q);
463 	spin_unlock_irq(q->queue_lock);
464 
465 	/*
466 	 * Queues start drained.  Skip actual draining till init is
467 	 * complete.  This avoids lenghty delays during queue init which
468 	 * can happen many times during boot.
469 	 */
470 	if (blk_queue_init_done(q)) {
471 		spin_lock_irq(q->queue_lock);
472 		__blk_drain_queue(q, false);
473 		spin_unlock_irq(q->queue_lock);
474 
475 		/* ensure blk_queue_bypass() is %true inside RCU read lock */
476 		synchronize_rcu();
477 	}
478 }
479 EXPORT_SYMBOL_GPL(blk_queue_bypass_start);
480 
481 /**
482  * blk_queue_bypass_end - leave queue bypass mode
483  * @q: queue of interest
484  *
485  * Leave bypass mode and restore the normal queueing behavior.
486  */
487 void blk_queue_bypass_end(struct request_queue *q)
488 {
489 	spin_lock_irq(q->queue_lock);
490 	if (!--q->bypass_depth)
491 		queue_flag_clear(QUEUE_FLAG_BYPASS, q);
492 	WARN_ON_ONCE(q->bypass_depth < 0);
493 	spin_unlock_irq(q->queue_lock);
494 }
495 EXPORT_SYMBOL_GPL(blk_queue_bypass_end);
496 
497 void blk_set_queue_dying(struct request_queue *q)
498 {
499 	spin_lock_irq(q->queue_lock);
500 	queue_flag_set(QUEUE_FLAG_DYING, q);
501 	spin_unlock_irq(q->queue_lock);
502 
503 	if (q->mq_ops)
504 		blk_mq_wake_waiters(q);
505 	else {
506 		struct request_list *rl;
507 
508 		spin_lock_irq(q->queue_lock);
509 		blk_queue_for_each_rl(rl, q) {
510 			if (rl->rq_pool) {
511 				wake_up(&rl->wait[BLK_RW_SYNC]);
512 				wake_up(&rl->wait[BLK_RW_ASYNC]);
513 			}
514 		}
515 		spin_unlock_irq(q->queue_lock);
516 	}
517 }
518 EXPORT_SYMBOL_GPL(blk_set_queue_dying);
519 
520 /**
521  * blk_cleanup_queue - shutdown a request queue
522  * @q: request queue to shutdown
523  *
524  * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
525  * put it.  All future requests will be failed immediately with -ENODEV.
526  */
527 void blk_cleanup_queue(struct request_queue *q)
528 {
529 	spinlock_t *lock = q->queue_lock;
530 
531 	/* mark @q DYING, no new request or merges will be allowed afterwards */
532 	mutex_lock(&q->sysfs_lock);
533 	blk_set_queue_dying(q);
534 	spin_lock_irq(lock);
535 
536 	/*
537 	 * A dying queue is permanently in bypass mode till released.  Note
538 	 * that, unlike blk_queue_bypass_start(), we aren't performing
539 	 * synchronize_rcu() after entering bypass mode to avoid the delay
540 	 * as some drivers create and destroy a lot of queues while
541 	 * probing.  This is still safe because blk_release_queue() will be
542 	 * called only after the queue refcnt drops to zero and nothing,
543 	 * RCU or not, would be traversing the queue by then.
544 	 */
545 	q->bypass_depth++;
546 	queue_flag_set(QUEUE_FLAG_BYPASS, q);
547 
548 	queue_flag_set(QUEUE_FLAG_NOMERGES, q);
549 	queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
550 	queue_flag_set(QUEUE_FLAG_DYING, q);
551 	spin_unlock_irq(lock);
552 	mutex_unlock(&q->sysfs_lock);
553 
554 	/*
555 	 * Drain all requests queued before DYING marking. Set DEAD flag to
556 	 * prevent that q->request_fn() gets invoked after draining finished.
557 	 */
558 	blk_freeze_queue(q);
559 	spin_lock_irq(lock);
560 	if (!q->mq_ops)
561 		__blk_drain_queue(q, true);
562 	queue_flag_set(QUEUE_FLAG_DEAD, q);
563 	spin_unlock_irq(lock);
564 
565 	/* for synchronous bio-based driver finish in-flight integrity i/o */
566 	blk_flush_integrity();
567 
568 	/* @q won't process any more request, flush async actions */
569 	del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer);
570 	blk_sync_queue(q);
571 
572 	if (q->mq_ops)
573 		blk_mq_free_queue(q);
574 	percpu_ref_exit(&q->q_usage_counter);
575 
576 	spin_lock_irq(lock);
577 	if (q->queue_lock != &q->__queue_lock)
578 		q->queue_lock = &q->__queue_lock;
579 	spin_unlock_irq(lock);
580 
581 	bdi_unregister(q->backing_dev_info);
582 	put_disk_devt(q->disk_devt);
583 
584 	/* @q is and will stay empty, shutdown and put */
585 	blk_put_queue(q);
586 }
587 EXPORT_SYMBOL(blk_cleanup_queue);
588 
589 /* Allocate memory local to the request queue */
590 static void *alloc_request_simple(gfp_t gfp_mask, void *data)
591 {
592 	struct request_queue *q = data;
593 
594 	return kmem_cache_alloc_node(request_cachep, gfp_mask, q->node);
595 }
596 
597 static void free_request_simple(void *element, void *data)
598 {
599 	kmem_cache_free(request_cachep, element);
600 }
601 
602 static void *alloc_request_size(gfp_t gfp_mask, void *data)
603 {
604 	struct request_queue *q = data;
605 	struct request *rq;
606 
607 	rq = kmalloc_node(sizeof(struct request) + q->cmd_size, gfp_mask,
608 			q->node);
609 	if (rq && q->init_rq_fn && q->init_rq_fn(q, rq, gfp_mask) < 0) {
610 		kfree(rq);
611 		rq = NULL;
612 	}
613 	return rq;
614 }
615 
616 static void free_request_size(void *element, void *data)
617 {
618 	struct request_queue *q = data;
619 
620 	if (q->exit_rq_fn)
621 		q->exit_rq_fn(q, element);
622 	kfree(element);
623 }
624 
625 int blk_init_rl(struct request_list *rl, struct request_queue *q,
626 		gfp_t gfp_mask)
627 {
628 	if (unlikely(rl->rq_pool))
629 		return 0;
630 
631 	rl->q = q;
632 	rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
633 	rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
634 	init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
635 	init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
636 
637 	if (q->cmd_size) {
638 		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
639 				alloc_request_size, free_request_size,
640 				q, gfp_mask, q->node);
641 	} else {
642 		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
643 				alloc_request_simple, free_request_simple,
644 				q, gfp_mask, q->node);
645 	}
646 	if (!rl->rq_pool)
647 		return -ENOMEM;
648 
649 	return 0;
650 }
651 
652 void blk_exit_rl(struct request_list *rl)
653 {
654 	if (rl->rq_pool)
655 		mempool_destroy(rl->rq_pool);
656 }
657 
658 struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
659 {
660 	return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE);
661 }
662 EXPORT_SYMBOL(blk_alloc_queue);
663 
664 int blk_queue_enter(struct request_queue *q, bool nowait)
665 {
666 	while (true) {
667 		int ret;
668 
669 		if (percpu_ref_tryget_live(&q->q_usage_counter))
670 			return 0;
671 
672 		if (nowait)
673 			return -EBUSY;
674 
675 		ret = wait_event_interruptible(q->mq_freeze_wq,
676 				!atomic_read(&q->mq_freeze_depth) ||
677 				blk_queue_dying(q));
678 		if (blk_queue_dying(q))
679 			return -ENODEV;
680 		if (ret)
681 			return ret;
682 	}
683 }
684 
685 void blk_queue_exit(struct request_queue *q)
686 {
687 	percpu_ref_put(&q->q_usage_counter);
688 }
689 
690 static void blk_queue_usage_counter_release(struct percpu_ref *ref)
691 {
692 	struct request_queue *q =
693 		container_of(ref, struct request_queue, q_usage_counter);
694 
695 	wake_up_all(&q->mq_freeze_wq);
696 }
697 
698 static void blk_rq_timed_out_timer(unsigned long data)
699 {
700 	struct request_queue *q = (struct request_queue *)data;
701 
702 	kblockd_schedule_work(&q->timeout_work);
703 }
704 
705 struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
706 {
707 	struct request_queue *q;
708 
709 	q = kmem_cache_alloc_node(blk_requestq_cachep,
710 				gfp_mask | __GFP_ZERO, node_id);
711 	if (!q)
712 		return NULL;
713 
714 	q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
715 	if (q->id < 0)
716 		goto fail_q;
717 
718 	q->bio_split = bioset_create(BIO_POOL_SIZE, 0);
719 	if (!q->bio_split)
720 		goto fail_id;
721 
722 	q->backing_dev_info = bdi_alloc_node(gfp_mask, node_id);
723 	if (!q->backing_dev_info)
724 		goto fail_split;
725 
726 	q->backing_dev_info->ra_pages =
727 			(VM_MAX_READAHEAD * 1024) / PAGE_SIZE;
728 	q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK;
729 	q->backing_dev_info->name = "block";
730 	q->node = node_id;
731 
732 	setup_timer(&q->backing_dev_info->laptop_mode_wb_timer,
733 		    laptop_mode_timer_fn, (unsigned long) q);
734 	setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
735 	INIT_LIST_HEAD(&q->queue_head);
736 	INIT_LIST_HEAD(&q->timeout_list);
737 	INIT_LIST_HEAD(&q->icq_list);
738 #ifdef CONFIG_BLK_CGROUP
739 	INIT_LIST_HEAD(&q->blkg_list);
740 #endif
741 	INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
742 
743 	kobject_init(&q->kobj, &blk_queue_ktype);
744 
745 	mutex_init(&q->sysfs_lock);
746 	spin_lock_init(&q->__queue_lock);
747 
748 	/*
749 	 * By default initialize queue_lock to internal lock and driver can
750 	 * override it later if need be.
751 	 */
752 	q->queue_lock = &q->__queue_lock;
753 
754 	/*
755 	 * A queue starts its life with bypass turned on to avoid
756 	 * unnecessary bypass on/off overhead and nasty surprises during
757 	 * init.  The initial bypass will be finished when the queue is
758 	 * registered by blk_register_queue().
759 	 */
760 	q->bypass_depth = 1;
761 	__set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
762 
763 	init_waitqueue_head(&q->mq_freeze_wq);
764 
765 	/*
766 	 * Init percpu_ref in atomic mode so that it's faster to shutdown.
767 	 * See blk_register_queue() for details.
768 	 */
769 	if (percpu_ref_init(&q->q_usage_counter,
770 				blk_queue_usage_counter_release,
771 				PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
772 		goto fail_bdi;
773 
774 	if (blkcg_init_queue(q))
775 		goto fail_ref;
776 
777 	return q;
778 
779 fail_ref:
780 	percpu_ref_exit(&q->q_usage_counter);
781 fail_bdi:
782 	bdi_put(q->backing_dev_info);
783 fail_split:
784 	bioset_free(q->bio_split);
785 fail_id:
786 	ida_simple_remove(&blk_queue_ida, q->id);
787 fail_q:
788 	kmem_cache_free(blk_requestq_cachep, q);
789 	return NULL;
790 }
791 EXPORT_SYMBOL(blk_alloc_queue_node);
792 
793 /**
794  * blk_init_queue  - prepare a request queue for use with a block device
795  * @rfn:  The function to be called to process requests that have been
796  *        placed on the queue.
797  * @lock: Request queue spin lock
798  *
799  * Description:
800  *    If a block device wishes to use the standard request handling procedures,
801  *    which sorts requests and coalesces adjacent requests, then it must
802  *    call blk_init_queue().  The function @rfn will be called when there
803  *    are requests on the queue that need to be processed.  If the device
804  *    supports plugging, then @rfn may not be called immediately when requests
805  *    are available on the queue, but may be called at some time later instead.
806  *    Plugged queues are generally unplugged when a buffer belonging to one
807  *    of the requests on the queue is needed, or due to memory pressure.
808  *
809  *    @rfn is not required, or even expected, to remove all requests off the
810  *    queue, but only as many as it can handle at a time.  If it does leave
811  *    requests on the queue, it is responsible for arranging that the requests
812  *    get dealt with eventually.
813  *
814  *    The queue spin lock must be held while manipulating the requests on the
815  *    request queue; this lock will be taken also from interrupt context, so irq
816  *    disabling is needed for it.
817  *
818  *    Function returns a pointer to the initialized request queue, or %NULL if
819  *    it didn't succeed.
820  *
821  * Note:
822  *    blk_init_queue() must be paired with a blk_cleanup_queue() call
823  *    when the block device is deactivated (such as at module unload).
824  **/
825 
826 struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
827 {
828 	return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
829 }
830 EXPORT_SYMBOL(blk_init_queue);
831 
832 struct request_queue *
833 blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
834 {
835 	struct request_queue *q;
836 
837 	q = blk_alloc_queue_node(GFP_KERNEL, node_id);
838 	if (!q)
839 		return NULL;
840 
841 	q->request_fn = rfn;
842 	if (lock)
843 		q->queue_lock = lock;
844 	if (blk_init_allocated_queue(q) < 0) {
845 		blk_cleanup_queue(q);
846 		return NULL;
847 	}
848 
849 	return q;
850 }
851 EXPORT_SYMBOL(blk_init_queue_node);
852 
853 static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio);
854 
855 
856 int blk_init_allocated_queue(struct request_queue *q)
857 {
858 	q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, q->cmd_size);
859 	if (!q->fq)
860 		return -ENOMEM;
861 
862 	if (q->init_rq_fn && q->init_rq_fn(q, q->fq->flush_rq, GFP_KERNEL))
863 		goto out_free_flush_queue;
864 
865 	if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
866 		goto out_exit_flush_rq;
867 
868 	INIT_WORK(&q->timeout_work, blk_timeout_work);
869 	q->queue_flags		|= QUEUE_FLAG_DEFAULT;
870 
871 	/*
872 	 * This also sets hw/phys segments, boundary and size
873 	 */
874 	blk_queue_make_request(q, blk_queue_bio);
875 
876 	q->sg_reserved_size = INT_MAX;
877 
878 	/* Protect q->elevator from elevator_change */
879 	mutex_lock(&q->sysfs_lock);
880 
881 	/* init elevator */
882 	if (elevator_init(q, NULL)) {
883 		mutex_unlock(&q->sysfs_lock);
884 		goto out_exit_flush_rq;
885 	}
886 
887 	mutex_unlock(&q->sysfs_lock);
888 	return 0;
889 
890 out_exit_flush_rq:
891 	if (q->exit_rq_fn)
892 		q->exit_rq_fn(q, q->fq->flush_rq);
893 out_free_flush_queue:
894 	blk_free_flush_queue(q->fq);
895 	wbt_exit(q);
896 	return -ENOMEM;
897 }
898 EXPORT_SYMBOL(blk_init_allocated_queue);
899 
900 bool blk_get_queue(struct request_queue *q)
901 {
902 	if (likely(!blk_queue_dying(q))) {
903 		__blk_get_queue(q);
904 		return true;
905 	}
906 
907 	return false;
908 }
909 EXPORT_SYMBOL(blk_get_queue);
910 
911 static inline void blk_free_request(struct request_list *rl, struct request *rq)
912 {
913 	if (rq->rq_flags & RQF_ELVPRIV) {
914 		elv_put_request(rl->q, rq);
915 		if (rq->elv.icq)
916 			put_io_context(rq->elv.icq->ioc);
917 	}
918 
919 	mempool_free(rq, rl->rq_pool);
920 }
921 
922 /*
923  * ioc_batching returns true if the ioc is a valid batching request and
924  * should be given priority access to a request.
925  */
926 static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
927 {
928 	if (!ioc)
929 		return 0;
930 
931 	/*
932 	 * Make sure the process is able to allocate at least 1 request
933 	 * even if the batch times out, otherwise we could theoretically
934 	 * lose wakeups.
935 	 */
936 	return ioc->nr_batch_requests == q->nr_batching ||
937 		(ioc->nr_batch_requests > 0
938 		&& time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
939 }
940 
941 /*
942  * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
943  * will cause the process to be a "batcher" on all queues in the system. This
944  * is the behaviour we want though - once it gets a wakeup it should be given
945  * a nice run.
946  */
947 static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
948 {
949 	if (!ioc || ioc_batching(q, ioc))
950 		return;
951 
952 	ioc->nr_batch_requests = q->nr_batching;
953 	ioc->last_waited = jiffies;
954 }
955 
956 static void __freed_request(struct request_list *rl, int sync)
957 {
958 	struct request_queue *q = rl->q;
959 
960 	if (rl->count[sync] < queue_congestion_off_threshold(q))
961 		blk_clear_congested(rl, sync);
962 
963 	if (rl->count[sync] + 1 <= q->nr_requests) {
964 		if (waitqueue_active(&rl->wait[sync]))
965 			wake_up(&rl->wait[sync]);
966 
967 		blk_clear_rl_full(rl, sync);
968 	}
969 }
970 
971 /*
972  * A request has just been released.  Account for it, update the full and
973  * congestion status, wake up any waiters.   Called under q->queue_lock.
974  */
975 static void freed_request(struct request_list *rl, bool sync,
976 		req_flags_t rq_flags)
977 {
978 	struct request_queue *q = rl->q;
979 
980 	q->nr_rqs[sync]--;
981 	rl->count[sync]--;
982 	if (rq_flags & RQF_ELVPRIV)
983 		q->nr_rqs_elvpriv--;
984 
985 	__freed_request(rl, sync);
986 
987 	if (unlikely(rl->starved[sync ^ 1]))
988 		__freed_request(rl, sync ^ 1);
989 }
990 
991 int blk_update_nr_requests(struct request_queue *q, unsigned int nr)
992 {
993 	struct request_list *rl;
994 	int on_thresh, off_thresh;
995 
996 	spin_lock_irq(q->queue_lock);
997 	q->nr_requests = nr;
998 	blk_queue_congestion_threshold(q);
999 	on_thresh = queue_congestion_on_threshold(q);
1000 	off_thresh = queue_congestion_off_threshold(q);
1001 
1002 	blk_queue_for_each_rl(rl, q) {
1003 		if (rl->count[BLK_RW_SYNC] >= on_thresh)
1004 			blk_set_congested(rl, BLK_RW_SYNC);
1005 		else if (rl->count[BLK_RW_SYNC] < off_thresh)
1006 			blk_clear_congested(rl, BLK_RW_SYNC);
1007 
1008 		if (rl->count[BLK_RW_ASYNC] >= on_thresh)
1009 			blk_set_congested(rl, BLK_RW_ASYNC);
1010 		else if (rl->count[BLK_RW_ASYNC] < off_thresh)
1011 			blk_clear_congested(rl, BLK_RW_ASYNC);
1012 
1013 		if (rl->count[BLK_RW_SYNC] >= q->nr_requests) {
1014 			blk_set_rl_full(rl, BLK_RW_SYNC);
1015 		} else {
1016 			blk_clear_rl_full(rl, BLK_RW_SYNC);
1017 			wake_up(&rl->wait[BLK_RW_SYNC]);
1018 		}
1019 
1020 		if (rl->count[BLK_RW_ASYNC] >= q->nr_requests) {
1021 			blk_set_rl_full(rl, BLK_RW_ASYNC);
1022 		} else {
1023 			blk_clear_rl_full(rl, BLK_RW_ASYNC);
1024 			wake_up(&rl->wait[BLK_RW_ASYNC]);
1025 		}
1026 	}
1027 
1028 	spin_unlock_irq(q->queue_lock);
1029 	return 0;
1030 }
1031 
1032 /**
1033  * __get_request - get a free request
1034  * @rl: request list to allocate from
1035  * @op: operation and flags
1036  * @bio: bio to allocate request for (can be %NULL)
1037  * @gfp_mask: allocation mask
1038  *
1039  * Get a free request from @q.  This function may fail under memory
1040  * pressure or if @q is dead.
1041  *
1042  * Must be called with @q->queue_lock held and,
1043  * Returns ERR_PTR on failure, with @q->queue_lock held.
1044  * Returns request pointer on success, with @q->queue_lock *not held*.
1045  */
1046 static struct request *__get_request(struct request_list *rl, unsigned int op,
1047 		struct bio *bio, gfp_t gfp_mask)
1048 {
1049 	struct request_queue *q = rl->q;
1050 	struct request *rq;
1051 	struct elevator_type *et = q->elevator->type;
1052 	struct io_context *ioc = rq_ioc(bio);
1053 	struct io_cq *icq = NULL;
1054 	const bool is_sync = op_is_sync(op);
1055 	int may_queue;
1056 	req_flags_t rq_flags = RQF_ALLOCED;
1057 
1058 	if (unlikely(blk_queue_dying(q)))
1059 		return ERR_PTR(-ENODEV);
1060 
1061 	may_queue = elv_may_queue(q, op);
1062 	if (may_queue == ELV_MQUEUE_NO)
1063 		goto rq_starved;
1064 
1065 	if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
1066 		if (rl->count[is_sync]+1 >= q->nr_requests) {
1067 			/*
1068 			 * The queue will fill after this allocation, so set
1069 			 * it as full, and mark this process as "batching".
1070 			 * This process will be allowed to complete a batch of
1071 			 * requests, others will be blocked.
1072 			 */
1073 			if (!blk_rl_full(rl, is_sync)) {
1074 				ioc_set_batching(q, ioc);
1075 				blk_set_rl_full(rl, is_sync);
1076 			} else {
1077 				if (may_queue != ELV_MQUEUE_MUST
1078 						&& !ioc_batching(q, ioc)) {
1079 					/*
1080 					 * The queue is full and the allocating
1081 					 * process is not a "batcher", and not
1082 					 * exempted by the IO scheduler
1083 					 */
1084 					return ERR_PTR(-ENOMEM);
1085 				}
1086 			}
1087 		}
1088 		blk_set_congested(rl, is_sync);
1089 	}
1090 
1091 	/*
1092 	 * Only allow batching queuers to allocate up to 50% over the defined
1093 	 * limit of requests, otherwise we could have thousands of requests
1094 	 * allocated with any setting of ->nr_requests
1095 	 */
1096 	if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
1097 		return ERR_PTR(-ENOMEM);
1098 
1099 	q->nr_rqs[is_sync]++;
1100 	rl->count[is_sync]++;
1101 	rl->starved[is_sync] = 0;
1102 
1103 	/*
1104 	 * Decide whether the new request will be managed by elevator.  If
1105 	 * so, mark @rq_flags and increment elvpriv.  Non-zero elvpriv will
1106 	 * prevent the current elevator from being destroyed until the new
1107 	 * request is freed.  This guarantees icq's won't be destroyed and
1108 	 * makes creating new ones safe.
1109 	 *
1110 	 * Flush requests do not use the elevator so skip initialization.
1111 	 * This allows a request to share the flush and elevator data.
1112 	 *
1113 	 * Also, lookup icq while holding queue_lock.  If it doesn't exist,
1114 	 * it will be created after releasing queue_lock.
1115 	 */
1116 	if (!op_is_flush(op) && !blk_queue_bypass(q)) {
1117 		rq_flags |= RQF_ELVPRIV;
1118 		q->nr_rqs_elvpriv++;
1119 		if (et->icq_cache && ioc)
1120 			icq = ioc_lookup_icq(ioc, q);
1121 	}
1122 
1123 	if (blk_queue_io_stat(q))
1124 		rq_flags |= RQF_IO_STAT;
1125 	spin_unlock_irq(q->queue_lock);
1126 
1127 	/* allocate and init request */
1128 	rq = mempool_alloc(rl->rq_pool, gfp_mask);
1129 	if (!rq)
1130 		goto fail_alloc;
1131 
1132 	blk_rq_init(q, rq);
1133 	blk_rq_set_rl(rq, rl);
1134 	blk_rq_set_prio(rq, ioc);
1135 	rq->cmd_flags = op;
1136 	rq->rq_flags = rq_flags;
1137 
1138 	/* init elvpriv */
1139 	if (rq_flags & RQF_ELVPRIV) {
1140 		if (unlikely(et->icq_cache && !icq)) {
1141 			if (ioc)
1142 				icq = ioc_create_icq(ioc, q, gfp_mask);
1143 			if (!icq)
1144 				goto fail_elvpriv;
1145 		}
1146 
1147 		rq->elv.icq = icq;
1148 		if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
1149 			goto fail_elvpriv;
1150 
1151 		/* @rq->elv.icq holds io_context until @rq is freed */
1152 		if (icq)
1153 			get_io_context(icq->ioc);
1154 	}
1155 out:
1156 	/*
1157 	 * ioc may be NULL here, and ioc_batching will be false. That's
1158 	 * OK, if the queue is under the request limit then requests need
1159 	 * not count toward the nr_batch_requests limit. There will always
1160 	 * be some limit enforced by BLK_BATCH_TIME.
1161 	 */
1162 	if (ioc_batching(q, ioc))
1163 		ioc->nr_batch_requests--;
1164 
1165 	trace_block_getrq(q, bio, op);
1166 	return rq;
1167 
1168 fail_elvpriv:
1169 	/*
1170 	 * elvpriv init failed.  ioc, icq and elvpriv aren't mempool backed
1171 	 * and may fail indefinitely under memory pressure and thus
1172 	 * shouldn't stall IO.  Treat this request as !elvpriv.  This will
1173 	 * disturb iosched and blkcg but weird is bettern than dead.
1174 	 */
1175 	printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n",
1176 			   __func__, dev_name(q->backing_dev_info->dev));
1177 
1178 	rq->rq_flags &= ~RQF_ELVPRIV;
1179 	rq->elv.icq = NULL;
1180 
1181 	spin_lock_irq(q->queue_lock);
1182 	q->nr_rqs_elvpriv--;
1183 	spin_unlock_irq(q->queue_lock);
1184 	goto out;
1185 
1186 fail_alloc:
1187 	/*
1188 	 * Allocation failed presumably due to memory. Undo anything we
1189 	 * might have messed up.
1190 	 *
1191 	 * Allocating task should really be put onto the front of the wait
1192 	 * queue, but this is pretty rare.
1193 	 */
1194 	spin_lock_irq(q->queue_lock);
1195 	freed_request(rl, is_sync, rq_flags);
1196 
1197 	/*
1198 	 * in the very unlikely event that allocation failed and no
1199 	 * requests for this direction was pending, mark us starved so that
1200 	 * freeing of a request in the other direction will notice
1201 	 * us. another possible fix would be to split the rq mempool into
1202 	 * READ and WRITE
1203 	 */
1204 rq_starved:
1205 	if (unlikely(rl->count[is_sync] == 0))
1206 		rl->starved[is_sync] = 1;
1207 	return ERR_PTR(-ENOMEM);
1208 }
1209 
1210 /**
1211  * get_request - get a free request
1212  * @q: request_queue to allocate request from
1213  * @op: operation and flags
1214  * @bio: bio to allocate request for (can be %NULL)
1215  * @gfp_mask: allocation mask
1216  *
1217  * Get a free request from @q.  If %__GFP_DIRECT_RECLAIM is set in @gfp_mask,
1218  * this function keeps retrying under memory pressure and fails iff @q is dead.
1219  *
1220  * Must be called with @q->queue_lock held and,
1221  * Returns ERR_PTR on failure, with @q->queue_lock held.
1222  * Returns request pointer on success, with @q->queue_lock *not held*.
1223  */
1224 static struct request *get_request(struct request_queue *q, unsigned int op,
1225 		struct bio *bio, gfp_t gfp_mask)
1226 {
1227 	const bool is_sync = op_is_sync(op);
1228 	DEFINE_WAIT(wait);
1229 	struct request_list *rl;
1230 	struct request *rq;
1231 
1232 	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */
1233 retry:
1234 	rq = __get_request(rl, op, bio, gfp_mask);
1235 	if (!IS_ERR(rq))
1236 		return rq;
1237 
1238 	if (!gfpflags_allow_blocking(gfp_mask) || unlikely(blk_queue_dying(q))) {
1239 		blk_put_rl(rl);
1240 		return rq;
1241 	}
1242 
1243 	/* wait on @rl and retry */
1244 	prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
1245 				  TASK_UNINTERRUPTIBLE);
1246 
1247 	trace_block_sleeprq(q, bio, op);
1248 
1249 	spin_unlock_irq(q->queue_lock);
1250 	io_schedule();
1251 
1252 	/*
1253 	 * After sleeping, we become a "batching" process and will be able
1254 	 * to allocate at least one request, and up to a big batch of them
1255 	 * for a small period time.  See ioc_batching, ioc_set_batching
1256 	 */
1257 	ioc_set_batching(q, current->io_context);
1258 
1259 	spin_lock_irq(q->queue_lock);
1260 	finish_wait(&rl->wait[is_sync], &wait);
1261 
1262 	goto retry;
1263 }
1264 
1265 static struct request *blk_old_get_request(struct request_queue *q, int rw,
1266 		gfp_t gfp_mask)
1267 {
1268 	struct request *rq;
1269 
1270 	/* create ioc upfront */
1271 	create_io_context(gfp_mask, q->node);
1272 
1273 	spin_lock_irq(q->queue_lock);
1274 	rq = get_request(q, rw, NULL, gfp_mask);
1275 	if (IS_ERR(rq)) {
1276 		spin_unlock_irq(q->queue_lock);
1277 		return rq;
1278 	}
1279 
1280 	/* q->queue_lock is unlocked at this point */
1281 	rq->__data_len = 0;
1282 	rq->__sector = (sector_t) -1;
1283 	rq->bio = rq->biotail = NULL;
1284 	return rq;
1285 }
1286 
1287 struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
1288 {
1289 	if (q->mq_ops)
1290 		return blk_mq_alloc_request(q, rw,
1291 			(gfp_mask & __GFP_DIRECT_RECLAIM) ?
1292 				0 : BLK_MQ_REQ_NOWAIT);
1293 	else
1294 		return blk_old_get_request(q, rw, gfp_mask);
1295 }
1296 EXPORT_SYMBOL(blk_get_request);
1297 
1298 /**
1299  * blk_requeue_request - put a request back on queue
1300  * @q:		request queue where request should be inserted
1301  * @rq:		request to be inserted
1302  *
1303  * Description:
1304  *    Drivers often keep queueing requests until the hardware cannot accept
1305  *    more, when that condition happens we need to put the request back
1306  *    on the queue. Must be called with queue lock held.
1307  */
1308 void blk_requeue_request(struct request_queue *q, struct request *rq)
1309 {
1310 	blk_delete_timer(rq);
1311 	blk_clear_rq_complete(rq);
1312 	trace_block_rq_requeue(q, rq);
1313 	wbt_requeue(q->rq_wb, &rq->issue_stat);
1314 
1315 	if (rq->rq_flags & RQF_QUEUED)
1316 		blk_queue_end_tag(q, rq);
1317 
1318 	BUG_ON(blk_queued_rq(rq));
1319 
1320 	elv_requeue_request(q, rq);
1321 }
1322 EXPORT_SYMBOL(blk_requeue_request);
1323 
1324 static void add_acct_request(struct request_queue *q, struct request *rq,
1325 			     int where)
1326 {
1327 	blk_account_io_start(rq, true);
1328 	__elv_add_request(q, rq, where);
1329 }
1330 
1331 static void part_round_stats_single(int cpu, struct hd_struct *part,
1332 				    unsigned long now)
1333 {
1334 	int inflight;
1335 
1336 	if (now == part->stamp)
1337 		return;
1338 
1339 	inflight = part_in_flight(part);
1340 	if (inflight) {
1341 		__part_stat_add(cpu, part, time_in_queue,
1342 				inflight * (now - part->stamp));
1343 		__part_stat_add(cpu, part, io_ticks, (now - part->stamp));
1344 	}
1345 	part->stamp = now;
1346 }
1347 
1348 /**
1349  * part_round_stats() - Round off the performance stats on a struct disk_stats.
1350  * @cpu: cpu number for stats access
1351  * @part: target partition
1352  *
1353  * The average IO queue length and utilisation statistics are maintained
1354  * by observing the current state of the queue length and the amount of
1355  * time it has been in this state for.
1356  *
1357  * Normally, that accounting is done on IO completion, but that can result
1358  * in more than a second's worth of IO being accounted for within any one
1359  * second, leading to >100% utilisation.  To deal with that, we call this
1360  * function to do a round-off before returning the results when reading
1361  * /proc/diskstats.  This accounts immediately for all queue usage up to
1362  * the current jiffies and restarts the counters again.
1363  */
1364 void part_round_stats(int cpu, struct hd_struct *part)
1365 {
1366 	unsigned long now = jiffies;
1367 
1368 	if (part->partno)
1369 		part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
1370 	part_round_stats_single(cpu, part, now);
1371 }
1372 EXPORT_SYMBOL_GPL(part_round_stats);
1373 
1374 #ifdef CONFIG_PM
1375 static void blk_pm_put_request(struct request *rq)
1376 {
1377 	if (rq->q->dev && !(rq->rq_flags & RQF_PM) && !--rq->q->nr_pending)
1378 		pm_runtime_mark_last_busy(rq->q->dev);
1379 }
1380 #else
1381 static inline void blk_pm_put_request(struct request *rq) {}
1382 #endif
1383 
1384 /*
1385  * queue lock must be held
1386  */
1387 void __blk_put_request(struct request_queue *q, struct request *req)
1388 {
1389 	req_flags_t rq_flags = req->rq_flags;
1390 
1391 	if (unlikely(!q))
1392 		return;
1393 
1394 	if (q->mq_ops) {
1395 		blk_mq_free_request(req);
1396 		return;
1397 	}
1398 
1399 	blk_pm_put_request(req);
1400 
1401 	elv_completed_request(q, req);
1402 
1403 	/* this is a bio leak */
1404 	WARN_ON(req->bio != NULL);
1405 
1406 	wbt_done(q->rq_wb, &req->issue_stat);
1407 
1408 	/*
1409 	 * Request may not have originated from ll_rw_blk. if not,
1410 	 * it didn't come out of our reserved rq pools
1411 	 */
1412 	if (rq_flags & RQF_ALLOCED) {
1413 		struct request_list *rl = blk_rq_rl(req);
1414 		bool sync = op_is_sync(req->cmd_flags);
1415 
1416 		BUG_ON(!list_empty(&req->queuelist));
1417 		BUG_ON(ELV_ON_HASH(req));
1418 
1419 		blk_free_request(rl, req);
1420 		freed_request(rl, sync, rq_flags);
1421 		blk_put_rl(rl);
1422 	}
1423 }
1424 EXPORT_SYMBOL_GPL(__blk_put_request);
1425 
1426 void blk_put_request(struct request *req)
1427 {
1428 	struct request_queue *q = req->q;
1429 
1430 	if (q->mq_ops)
1431 		blk_mq_free_request(req);
1432 	else {
1433 		unsigned long flags;
1434 
1435 		spin_lock_irqsave(q->queue_lock, flags);
1436 		__blk_put_request(q, req);
1437 		spin_unlock_irqrestore(q->queue_lock, flags);
1438 	}
1439 }
1440 EXPORT_SYMBOL(blk_put_request);
1441 
1442 bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
1443 			    struct bio *bio)
1444 {
1445 	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1446 
1447 	if (!ll_back_merge_fn(q, req, bio))
1448 		return false;
1449 
1450 	trace_block_bio_backmerge(q, req, bio);
1451 
1452 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1453 		blk_rq_set_mixed_merge(req);
1454 
1455 	req->biotail->bi_next = bio;
1456 	req->biotail = bio;
1457 	req->__data_len += bio->bi_iter.bi_size;
1458 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1459 
1460 	blk_account_io_start(req, false);
1461 	return true;
1462 }
1463 
1464 bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
1465 			     struct bio *bio)
1466 {
1467 	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1468 
1469 	if (!ll_front_merge_fn(q, req, bio))
1470 		return false;
1471 
1472 	trace_block_bio_frontmerge(q, req, bio);
1473 
1474 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1475 		blk_rq_set_mixed_merge(req);
1476 
1477 	bio->bi_next = req->bio;
1478 	req->bio = bio;
1479 
1480 	req->__sector = bio->bi_iter.bi_sector;
1481 	req->__data_len += bio->bi_iter.bi_size;
1482 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1483 
1484 	blk_account_io_start(req, false);
1485 	return true;
1486 }
1487 
1488 bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
1489 		struct bio *bio)
1490 {
1491 	unsigned short segments = blk_rq_nr_discard_segments(req);
1492 
1493 	if (segments >= queue_max_discard_segments(q))
1494 		goto no_merge;
1495 	if (blk_rq_sectors(req) + bio_sectors(bio) >
1496 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
1497 		goto no_merge;
1498 
1499 	req->biotail->bi_next = bio;
1500 	req->biotail = bio;
1501 	req->__data_len += bio->bi_iter.bi_size;
1502 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1503 	req->nr_phys_segments = segments + 1;
1504 
1505 	blk_account_io_start(req, false);
1506 	return true;
1507 no_merge:
1508 	req_set_nomerge(q, req);
1509 	return false;
1510 }
1511 
1512 /**
1513  * blk_attempt_plug_merge - try to merge with %current's plugged list
1514  * @q: request_queue new bio is being queued at
1515  * @bio: new bio being queued
1516  * @request_count: out parameter for number of traversed plugged requests
1517  * @same_queue_rq: pointer to &struct request that gets filled in when
1518  * another request associated with @q is found on the plug list
1519  * (optional, may be %NULL)
1520  *
1521  * Determine whether @bio being queued on @q can be merged with a request
1522  * on %current's plugged list.  Returns %true if merge was successful,
1523  * otherwise %false.
1524  *
1525  * Plugging coalesces IOs from the same issuer for the same purpose without
1526  * going through @q->queue_lock.  As such it's more of an issuing mechanism
1527  * than scheduling, and the request, while may have elvpriv data, is not
1528  * added on the elevator at this point.  In addition, we don't have
1529  * reliable access to the elevator outside queue lock.  Only check basic
1530  * merging parameters without querying the elevator.
1531  *
1532  * Caller must ensure !blk_queue_nomerges(q) beforehand.
1533  */
1534 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1535 			    unsigned int *request_count,
1536 			    struct request **same_queue_rq)
1537 {
1538 	struct blk_plug *plug;
1539 	struct request *rq;
1540 	struct list_head *plug_list;
1541 
1542 	plug = current->plug;
1543 	if (!plug)
1544 		return false;
1545 	*request_count = 0;
1546 
1547 	if (q->mq_ops)
1548 		plug_list = &plug->mq_list;
1549 	else
1550 		plug_list = &plug->list;
1551 
1552 	list_for_each_entry_reverse(rq, plug_list, queuelist) {
1553 		bool merged = false;
1554 
1555 		if (rq->q == q) {
1556 			(*request_count)++;
1557 			/*
1558 			 * Only blk-mq multiple hardware queues case checks the
1559 			 * rq in the same queue, there should be only one such
1560 			 * rq in a queue
1561 			 **/
1562 			if (same_queue_rq)
1563 				*same_queue_rq = rq;
1564 		}
1565 
1566 		if (rq->q != q || !blk_rq_merge_ok(rq, bio))
1567 			continue;
1568 
1569 		switch (blk_try_merge(rq, bio)) {
1570 		case ELEVATOR_BACK_MERGE:
1571 			merged = bio_attempt_back_merge(q, rq, bio);
1572 			break;
1573 		case ELEVATOR_FRONT_MERGE:
1574 			merged = bio_attempt_front_merge(q, rq, bio);
1575 			break;
1576 		case ELEVATOR_DISCARD_MERGE:
1577 			merged = bio_attempt_discard_merge(q, rq, bio);
1578 			break;
1579 		default:
1580 			break;
1581 		}
1582 
1583 		if (merged)
1584 			return true;
1585 	}
1586 
1587 	return false;
1588 }
1589 
1590 unsigned int blk_plug_queued_count(struct request_queue *q)
1591 {
1592 	struct blk_plug *plug;
1593 	struct request *rq;
1594 	struct list_head *plug_list;
1595 	unsigned int ret = 0;
1596 
1597 	plug = current->plug;
1598 	if (!plug)
1599 		goto out;
1600 
1601 	if (q->mq_ops)
1602 		plug_list = &plug->mq_list;
1603 	else
1604 		plug_list = &plug->list;
1605 
1606 	list_for_each_entry(rq, plug_list, queuelist) {
1607 		if (rq->q == q)
1608 			ret++;
1609 	}
1610 out:
1611 	return ret;
1612 }
1613 
1614 void init_request_from_bio(struct request *req, struct bio *bio)
1615 {
1616 	if (bio->bi_opf & REQ_RAHEAD)
1617 		req->cmd_flags |= REQ_FAILFAST_MASK;
1618 
1619 	req->errors = 0;
1620 	req->__sector = bio->bi_iter.bi_sector;
1621 	if (ioprio_valid(bio_prio(bio)))
1622 		req->ioprio = bio_prio(bio);
1623 	blk_rq_bio_prep(req->q, req, bio);
1624 }
1625 
1626 static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
1627 {
1628 	struct blk_plug *plug;
1629 	int where = ELEVATOR_INSERT_SORT;
1630 	struct request *req, *free;
1631 	unsigned int request_count = 0;
1632 	unsigned int wb_acct;
1633 
1634 	/*
1635 	 * low level driver can indicate that it wants pages above a
1636 	 * certain limit bounced to low memory (ie for highmem, or even
1637 	 * ISA dma in theory)
1638 	 */
1639 	blk_queue_bounce(q, &bio);
1640 
1641 	blk_queue_split(q, &bio, q->bio_split);
1642 
1643 	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1644 		bio->bi_error = -EIO;
1645 		bio_endio(bio);
1646 		return BLK_QC_T_NONE;
1647 	}
1648 
1649 	if (op_is_flush(bio->bi_opf)) {
1650 		spin_lock_irq(q->queue_lock);
1651 		where = ELEVATOR_INSERT_FLUSH;
1652 		goto get_rq;
1653 	}
1654 
1655 	/*
1656 	 * Check if we can merge with the plugged list before grabbing
1657 	 * any locks.
1658 	 */
1659 	if (!blk_queue_nomerges(q)) {
1660 		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
1661 			return BLK_QC_T_NONE;
1662 	} else
1663 		request_count = blk_plug_queued_count(q);
1664 
1665 	spin_lock_irq(q->queue_lock);
1666 
1667 	switch (elv_merge(q, &req, bio)) {
1668 	case ELEVATOR_BACK_MERGE:
1669 		if (!bio_attempt_back_merge(q, req, bio))
1670 			break;
1671 		elv_bio_merged(q, req, bio);
1672 		free = attempt_back_merge(q, req);
1673 		if (free)
1674 			__blk_put_request(q, free);
1675 		else
1676 			elv_merged_request(q, req, ELEVATOR_BACK_MERGE);
1677 		goto out_unlock;
1678 	case ELEVATOR_FRONT_MERGE:
1679 		if (!bio_attempt_front_merge(q, req, bio))
1680 			break;
1681 		elv_bio_merged(q, req, bio);
1682 		free = attempt_front_merge(q, req);
1683 		if (free)
1684 			__blk_put_request(q, free);
1685 		else
1686 			elv_merged_request(q, req, ELEVATOR_FRONT_MERGE);
1687 		goto out_unlock;
1688 	default:
1689 		break;
1690 	}
1691 
1692 get_rq:
1693 	wb_acct = wbt_wait(q->rq_wb, bio, q->queue_lock);
1694 
1695 	/*
1696 	 * Grab a free request. This is might sleep but can not fail.
1697 	 * Returns with the queue unlocked.
1698 	 */
1699 	req = get_request(q, bio->bi_opf, bio, GFP_NOIO);
1700 	if (IS_ERR(req)) {
1701 		__wbt_done(q->rq_wb, wb_acct);
1702 		bio->bi_error = PTR_ERR(req);
1703 		bio_endio(bio);
1704 		goto out_unlock;
1705 	}
1706 
1707 	wbt_track(&req->issue_stat, wb_acct);
1708 
1709 	/*
1710 	 * After dropping the lock and possibly sleeping here, our request
1711 	 * may now be mergeable after it had proven unmergeable (above).
1712 	 * We don't worry about that case for efficiency. It won't happen
1713 	 * often, and the elevators are able to handle it.
1714 	 */
1715 	init_request_from_bio(req, bio);
1716 
1717 	if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
1718 		req->cpu = raw_smp_processor_id();
1719 
1720 	plug = current->plug;
1721 	if (plug) {
1722 		/*
1723 		 * If this is the first request added after a plug, fire
1724 		 * of a plug trace.
1725 		 *
1726 		 * @request_count may become stale because of schedule
1727 		 * out, so check plug list again.
1728 		 */
1729 		if (!request_count || list_empty(&plug->list))
1730 			trace_block_plug(q);
1731 		else {
1732 			struct request *last = list_entry_rq(plug->list.prev);
1733 			if (request_count >= BLK_MAX_REQUEST_COUNT ||
1734 			    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE) {
1735 				blk_flush_plug_list(plug, false);
1736 				trace_block_plug(q);
1737 			}
1738 		}
1739 		list_add_tail(&req->queuelist, &plug->list);
1740 		blk_account_io_start(req, true);
1741 	} else {
1742 		spin_lock_irq(q->queue_lock);
1743 		add_acct_request(q, req, where);
1744 		__blk_run_queue(q);
1745 out_unlock:
1746 		spin_unlock_irq(q->queue_lock);
1747 	}
1748 
1749 	return BLK_QC_T_NONE;
1750 }
1751 
1752 /*
1753  * If bio->bi_dev is a partition, remap the location
1754  */
1755 static inline void blk_partition_remap(struct bio *bio)
1756 {
1757 	struct block_device *bdev = bio->bi_bdev;
1758 
1759 	/*
1760 	 * Zone reset does not include bi_size so bio_sectors() is always 0.
1761 	 * Include a test for the reset op code and perform the remap if needed.
1762 	 */
1763 	if (bdev != bdev->bd_contains &&
1764 	    (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET)) {
1765 		struct hd_struct *p = bdev->bd_part;
1766 
1767 		bio->bi_iter.bi_sector += p->start_sect;
1768 		bio->bi_bdev = bdev->bd_contains;
1769 
1770 		trace_block_bio_remap(bdev_get_queue(bio->bi_bdev), bio,
1771 				      bdev->bd_dev,
1772 				      bio->bi_iter.bi_sector - p->start_sect);
1773 	}
1774 }
1775 
1776 static void handle_bad_sector(struct bio *bio)
1777 {
1778 	char b[BDEVNAME_SIZE];
1779 
1780 	printk(KERN_INFO "attempt to access beyond end of device\n");
1781 	printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n",
1782 			bdevname(bio->bi_bdev, b),
1783 			bio->bi_opf,
1784 			(unsigned long long)bio_end_sector(bio),
1785 			(long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9));
1786 }
1787 
1788 #ifdef CONFIG_FAIL_MAKE_REQUEST
1789 
1790 static DECLARE_FAULT_ATTR(fail_make_request);
1791 
1792 static int __init setup_fail_make_request(char *str)
1793 {
1794 	return setup_fault_attr(&fail_make_request, str);
1795 }
1796 __setup("fail_make_request=", setup_fail_make_request);
1797 
1798 static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
1799 {
1800 	return part->make_it_fail && should_fail(&fail_make_request, bytes);
1801 }
1802 
1803 static int __init fail_make_request_debugfs(void)
1804 {
1805 	struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
1806 						NULL, &fail_make_request);
1807 
1808 	return PTR_ERR_OR_ZERO(dir);
1809 }
1810 
1811 late_initcall(fail_make_request_debugfs);
1812 
1813 #else /* CONFIG_FAIL_MAKE_REQUEST */
1814 
1815 static inline bool should_fail_request(struct hd_struct *part,
1816 					unsigned int bytes)
1817 {
1818 	return false;
1819 }
1820 
1821 #endif /* CONFIG_FAIL_MAKE_REQUEST */
1822 
1823 /*
1824  * Check whether this bio extends beyond the end of the device.
1825  */
1826 static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
1827 {
1828 	sector_t maxsector;
1829 
1830 	if (!nr_sectors)
1831 		return 0;
1832 
1833 	/* Test device or partition size, when known. */
1834 	maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
1835 	if (maxsector) {
1836 		sector_t sector = bio->bi_iter.bi_sector;
1837 
1838 		if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
1839 			/*
1840 			 * This may well happen - the kernel calls bread()
1841 			 * without checking the size of the device, e.g., when
1842 			 * mounting a device.
1843 			 */
1844 			handle_bad_sector(bio);
1845 			return 1;
1846 		}
1847 	}
1848 
1849 	return 0;
1850 }
1851 
1852 static noinline_for_stack bool
1853 generic_make_request_checks(struct bio *bio)
1854 {
1855 	struct request_queue *q;
1856 	int nr_sectors = bio_sectors(bio);
1857 	int err = -EIO;
1858 	char b[BDEVNAME_SIZE];
1859 	struct hd_struct *part;
1860 
1861 	might_sleep();
1862 
1863 	if (bio_check_eod(bio, nr_sectors))
1864 		goto end_io;
1865 
1866 	q = bdev_get_queue(bio->bi_bdev);
1867 	if (unlikely(!q)) {
1868 		printk(KERN_ERR
1869 		       "generic_make_request: Trying to access "
1870 			"nonexistent block-device %s (%Lu)\n",
1871 			bdevname(bio->bi_bdev, b),
1872 			(long long) bio->bi_iter.bi_sector);
1873 		goto end_io;
1874 	}
1875 
1876 	part = bio->bi_bdev->bd_part;
1877 	if (should_fail_request(part, bio->bi_iter.bi_size) ||
1878 	    should_fail_request(&part_to_disk(part)->part0,
1879 				bio->bi_iter.bi_size))
1880 		goto end_io;
1881 
1882 	/*
1883 	 * If this device has partitions, remap block n
1884 	 * of partition p to block n+start(p) of the disk.
1885 	 */
1886 	blk_partition_remap(bio);
1887 
1888 	if (bio_check_eod(bio, nr_sectors))
1889 		goto end_io;
1890 
1891 	/*
1892 	 * Filter flush bio's early so that make_request based
1893 	 * drivers without flush support don't have to worry
1894 	 * about them.
1895 	 */
1896 	if (op_is_flush(bio->bi_opf) &&
1897 	    !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
1898 		bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
1899 		if (!nr_sectors) {
1900 			err = 0;
1901 			goto end_io;
1902 		}
1903 	}
1904 
1905 	switch (bio_op(bio)) {
1906 	case REQ_OP_DISCARD:
1907 		if (!blk_queue_discard(q))
1908 			goto not_supported;
1909 		break;
1910 	case REQ_OP_SECURE_ERASE:
1911 		if (!blk_queue_secure_erase(q))
1912 			goto not_supported;
1913 		break;
1914 	case REQ_OP_WRITE_SAME:
1915 		if (!bdev_write_same(bio->bi_bdev))
1916 			goto not_supported;
1917 		break;
1918 	case REQ_OP_ZONE_REPORT:
1919 	case REQ_OP_ZONE_RESET:
1920 		if (!bdev_is_zoned(bio->bi_bdev))
1921 			goto not_supported;
1922 		break;
1923 	case REQ_OP_WRITE_ZEROES:
1924 		if (!bdev_write_zeroes_sectors(bio->bi_bdev))
1925 			goto not_supported;
1926 		break;
1927 	default:
1928 		break;
1929 	}
1930 
1931 	/*
1932 	 * Various block parts want %current->io_context and lazy ioc
1933 	 * allocation ends up trading a lot of pain for a small amount of
1934 	 * memory.  Just allocate it upfront.  This may fail and block
1935 	 * layer knows how to live with it.
1936 	 */
1937 	create_io_context(GFP_ATOMIC, q->node);
1938 
1939 	if (!blkcg_bio_issue_check(q, bio))
1940 		return false;
1941 
1942 	trace_block_bio_queue(q, bio);
1943 	return true;
1944 
1945 not_supported:
1946 	err = -EOPNOTSUPP;
1947 end_io:
1948 	bio->bi_error = err;
1949 	bio_endio(bio);
1950 	return false;
1951 }
1952 
1953 /**
1954  * generic_make_request - hand a buffer to its device driver for I/O
1955  * @bio:  The bio describing the location in memory and on the device.
1956  *
1957  * generic_make_request() is used to make I/O requests of block
1958  * devices. It is passed a &struct bio, which describes the I/O that needs
1959  * to be done.
1960  *
1961  * generic_make_request() does not return any status.  The
1962  * success/failure status of the request, along with notification of
1963  * completion, is delivered asynchronously through the bio->bi_end_io
1964  * function described (one day) else where.
1965  *
1966  * The caller of generic_make_request must make sure that bi_io_vec
1967  * are set to describe the memory buffer, and that bi_dev and bi_sector are
1968  * set to describe the device address, and the
1969  * bi_end_io and optionally bi_private are set to describe how
1970  * completion notification should be signaled.
1971  *
1972  * generic_make_request and the drivers it calls may use bi_next if this
1973  * bio happens to be merged with someone else, and may resubmit the bio to
1974  * a lower device by calling into generic_make_request recursively, which
1975  * means the bio should NOT be touched after the call to ->make_request_fn.
1976  */
1977 blk_qc_t generic_make_request(struct bio *bio)
1978 {
1979 	struct bio_list bio_list_on_stack;
1980 	blk_qc_t ret = BLK_QC_T_NONE;
1981 
1982 	if (!generic_make_request_checks(bio))
1983 		goto out;
1984 
1985 	/*
1986 	 * We only want one ->make_request_fn to be active at a time, else
1987 	 * stack usage with stacked devices could be a problem.  So use
1988 	 * current->bio_list to keep a list of requests submited by a
1989 	 * make_request_fn function.  current->bio_list is also used as a
1990 	 * flag to say if generic_make_request is currently active in this
1991 	 * task or not.  If it is NULL, then no make_request is active.  If
1992 	 * it is non-NULL, then a make_request is active, and new requests
1993 	 * should be added at the tail
1994 	 */
1995 	if (current->bio_list) {
1996 		bio_list_add(current->bio_list, bio);
1997 		goto out;
1998 	}
1999 
2000 	/* following loop may be a bit non-obvious, and so deserves some
2001 	 * explanation.
2002 	 * Before entering the loop, bio->bi_next is NULL (as all callers
2003 	 * ensure that) so we have a list with a single bio.
2004 	 * We pretend that we have just taken it off a longer list, so
2005 	 * we assign bio_list to a pointer to the bio_list_on_stack,
2006 	 * thus initialising the bio_list of new bios to be
2007 	 * added.  ->make_request() may indeed add some more bios
2008 	 * through a recursive call to generic_make_request.  If it
2009 	 * did, we find a non-NULL value in bio_list and re-enter the loop
2010 	 * from the top.  In this case we really did just take the bio
2011 	 * of the top of the list (no pretending) and so remove it from
2012 	 * bio_list, and call into ->make_request() again.
2013 	 */
2014 	BUG_ON(bio->bi_next);
2015 	bio_list_init(&bio_list_on_stack);
2016 	current->bio_list = &bio_list_on_stack;
2017 	do {
2018 		struct request_queue *q = bdev_get_queue(bio->bi_bdev);
2019 
2020 		if (likely(blk_queue_enter(q, false) == 0)) {
2021 			ret = q->make_request_fn(q, bio);
2022 
2023 			blk_queue_exit(q);
2024 
2025 			bio = bio_list_pop(current->bio_list);
2026 		} else {
2027 			struct bio *bio_next = bio_list_pop(current->bio_list);
2028 
2029 			bio_io_error(bio);
2030 			bio = bio_next;
2031 		}
2032 	} while (bio);
2033 	current->bio_list = NULL; /* deactivate */
2034 
2035 out:
2036 	return ret;
2037 }
2038 EXPORT_SYMBOL(generic_make_request);
2039 
2040 /**
2041  * submit_bio - submit a bio to the block device layer for I/O
2042  * @bio: The &struct bio which describes the I/O
2043  *
2044  * submit_bio() is very similar in purpose to generic_make_request(), and
2045  * uses that function to do most of the work. Both are fairly rough
2046  * interfaces; @bio must be presetup and ready for I/O.
2047  *
2048  */
2049 blk_qc_t submit_bio(struct bio *bio)
2050 {
2051 	/*
2052 	 * If it's a regular read/write or a barrier with data attached,
2053 	 * go through the normal accounting stuff before submission.
2054 	 */
2055 	if (bio_has_data(bio)) {
2056 		unsigned int count;
2057 
2058 		if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
2059 			count = bdev_logical_block_size(bio->bi_bdev) >> 9;
2060 		else
2061 			count = bio_sectors(bio);
2062 
2063 		if (op_is_write(bio_op(bio))) {
2064 			count_vm_events(PGPGOUT, count);
2065 		} else {
2066 			task_io_account_read(bio->bi_iter.bi_size);
2067 			count_vm_events(PGPGIN, count);
2068 		}
2069 
2070 		if (unlikely(block_dump)) {
2071 			char b[BDEVNAME_SIZE];
2072 			printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
2073 			current->comm, task_pid_nr(current),
2074 				op_is_write(bio_op(bio)) ? "WRITE" : "READ",
2075 				(unsigned long long)bio->bi_iter.bi_sector,
2076 				bdevname(bio->bi_bdev, b),
2077 				count);
2078 		}
2079 	}
2080 
2081 	return generic_make_request(bio);
2082 }
2083 EXPORT_SYMBOL(submit_bio);
2084 
2085 /**
2086  * blk_cloned_rq_check_limits - Helper function to check a cloned request
2087  *                              for new the queue limits
2088  * @q:  the queue
2089  * @rq: the request being checked
2090  *
2091  * Description:
2092  *    @rq may have been made based on weaker limitations of upper-level queues
2093  *    in request stacking drivers, and it may violate the limitation of @q.
2094  *    Since the block layer and the underlying device driver trust @rq
2095  *    after it is inserted to @q, it should be checked against @q before
2096  *    the insertion using this generic function.
2097  *
2098  *    Request stacking drivers like request-based dm may change the queue
2099  *    limits when retrying requests on other queues. Those requests need
2100  *    to be checked against the new queue limits again during dispatch.
2101  */
2102 static int blk_cloned_rq_check_limits(struct request_queue *q,
2103 				      struct request *rq)
2104 {
2105 	if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) {
2106 		printk(KERN_ERR "%s: over max size limit.\n", __func__);
2107 		return -EIO;
2108 	}
2109 
2110 	/*
2111 	 * queue's settings related to segment counting like q->bounce_pfn
2112 	 * may differ from that of other stacking queues.
2113 	 * Recalculate it to check the request correctly on this queue's
2114 	 * limitation.
2115 	 */
2116 	blk_recalc_rq_segments(rq);
2117 	if (rq->nr_phys_segments > queue_max_segments(q)) {
2118 		printk(KERN_ERR "%s: over max segments limit.\n", __func__);
2119 		return -EIO;
2120 	}
2121 
2122 	return 0;
2123 }
2124 
2125 /**
2126  * blk_insert_cloned_request - Helper for stacking drivers to submit a request
2127  * @q:  the queue to submit the request
2128  * @rq: the request being queued
2129  */
2130 int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
2131 {
2132 	unsigned long flags;
2133 	int where = ELEVATOR_INSERT_BACK;
2134 
2135 	if (blk_cloned_rq_check_limits(q, rq))
2136 		return -EIO;
2137 
2138 	if (rq->rq_disk &&
2139 	    should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
2140 		return -EIO;
2141 
2142 	if (q->mq_ops) {
2143 		if (blk_queue_io_stat(q))
2144 			blk_account_io_start(rq, true);
2145 		blk_mq_sched_insert_request(rq, false, true, false, false);
2146 		return 0;
2147 	}
2148 
2149 	spin_lock_irqsave(q->queue_lock, flags);
2150 	if (unlikely(blk_queue_dying(q))) {
2151 		spin_unlock_irqrestore(q->queue_lock, flags);
2152 		return -ENODEV;
2153 	}
2154 
2155 	/*
2156 	 * Submitting request must be dequeued before calling this function
2157 	 * because it will be linked to another request_queue
2158 	 */
2159 	BUG_ON(blk_queued_rq(rq));
2160 
2161 	if (op_is_flush(rq->cmd_flags))
2162 		where = ELEVATOR_INSERT_FLUSH;
2163 
2164 	add_acct_request(q, rq, where);
2165 	if (where == ELEVATOR_INSERT_FLUSH)
2166 		__blk_run_queue(q);
2167 	spin_unlock_irqrestore(q->queue_lock, flags);
2168 
2169 	return 0;
2170 }
2171 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
2172 
2173 /**
2174  * blk_rq_err_bytes - determine number of bytes till the next failure boundary
2175  * @rq: request to examine
2176  *
2177  * Description:
2178  *     A request could be merge of IOs which require different failure
2179  *     handling.  This function determines the number of bytes which
2180  *     can be failed from the beginning of the request without
2181  *     crossing into area which need to be retried further.
2182  *
2183  * Return:
2184  *     The number of bytes to fail.
2185  *
2186  * Context:
2187  *     queue_lock must be held.
2188  */
2189 unsigned int blk_rq_err_bytes(const struct request *rq)
2190 {
2191 	unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
2192 	unsigned int bytes = 0;
2193 	struct bio *bio;
2194 
2195 	if (!(rq->rq_flags & RQF_MIXED_MERGE))
2196 		return blk_rq_bytes(rq);
2197 
2198 	/*
2199 	 * Currently the only 'mixing' which can happen is between
2200 	 * different fastfail types.  We can safely fail portions
2201 	 * which have all the failfast bits that the first one has -
2202 	 * the ones which are at least as eager to fail as the first
2203 	 * one.
2204 	 */
2205 	for (bio = rq->bio; bio; bio = bio->bi_next) {
2206 		if ((bio->bi_opf & ff) != ff)
2207 			break;
2208 		bytes += bio->bi_iter.bi_size;
2209 	}
2210 
2211 	/* this could lead to infinite loop */
2212 	BUG_ON(blk_rq_bytes(rq) && !bytes);
2213 	return bytes;
2214 }
2215 EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
2216 
2217 void blk_account_io_completion(struct request *req, unsigned int bytes)
2218 {
2219 	if (blk_do_io_stat(req)) {
2220 		const int rw = rq_data_dir(req);
2221 		struct hd_struct *part;
2222 		int cpu;
2223 
2224 		cpu = part_stat_lock();
2225 		part = req->part;
2226 		part_stat_add(cpu, part, sectors[rw], bytes >> 9);
2227 		part_stat_unlock();
2228 	}
2229 }
2230 
2231 void blk_account_io_done(struct request *req)
2232 {
2233 	/*
2234 	 * Account IO completion.  flush_rq isn't accounted as a
2235 	 * normal IO on queueing nor completion.  Accounting the
2236 	 * containing request is enough.
2237 	 */
2238 	if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {
2239 		unsigned long duration = jiffies - req->start_time;
2240 		const int rw = rq_data_dir(req);
2241 		struct hd_struct *part;
2242 		int cpu;
2243 
2244 		cpu = part_stat_lock();
2245 		part = req->part;
2246 
2247 		part_stat_inc(cpu, part, ios[rw]);
2248 		part_stat_add(cpu, part, ticks[rw], duration);
2249 		part_round_stats(cpu, part);
2250 		part_dec_in_flight(part, rw);
2251 
2252 		hd_struct_put(part);
2253 		part_stat_unlock();
2254 	}
2255 }
2256 
2257 #ifdef CONFIG_PM
2258 /*
2259  * Don't process normal requests when queue is suspended
2260  * or in the process of suspending/resuming
2261  */
2262 static struct request *blk_pm_peek_request(struct request_queue *q,
2263 					   struct request *rq)
2264 {
2265 	if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
2266 	    (q->rpm_status != RPM_ACTIVE && !(rq->rq_flags & RQF_PM))))
2267 		return NULL;
2268 	else
2269 		return rq;
2270 }
2271 #else
2272 static inline struct request *blk_pm_peek_request(struct request_queue *q,
2273 						  struct request *rq)
2274 {
2275 	return rq;
2276 }
2277 #endif
2278 
2279 void blk_account_io_start(struct request *rq, bool new_io)
2280 {
2281 	struct hd_struct *part;
2282 	int rw = rq_data_dir(rq);
2283 	int cpu;
2284 
2285 	if (!blk_do_io_stat(rq))
2286 		return;
2287 
2288 	cpu = part_stat_lock();
2289 
2290 	if (!new_io) {
2291 		part = rq->part;
2292 		part_stat_inc(cpu, part, merges[rw]);
2293 	} else {
2294 		part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
2295 		if (!hd_struct_try_get(part)) {
2296 			/*
2297 			 * The partition is already being removed,
2298 			 * the request will be accounted on the disk only
2299 			 *
2300 			 * We take a reference on disk->part0 although that
2301 			 * partition will never be deleted, so we can treat
2302 			 * it as any other partition.
2303 			 */
2304 			part = &rq->rq_disk->part0;
2305 			hd_struct_get(part);
2306 		}
2307 		part_round_stats(cpu, part);
2308 		part_inc_in_flight(part, rw);
2309 		rq->part = part;
2310 	}
2311 
2312 	part_stat_unlock();
2313 }
2314 
2315 /**
2316  * blk_peek_request - peek at the top of a request queue
2317  * @q: request queue to peek at
2318  *
2319  * Description:
2320  *     Return the request at the top of @q.  The returned request
2321  *     should be started using blk_start_request() before LLD starts
2322  *     processing it.
2323  *
2324  * Return:
2325  *     Pointer to the request at the top of @q if available.  Null
2326  *     otherwise.
2327  *
2328  * Context:
2329  *     queue_lock must be held.
2330  */
2331 struct request *blk_peek_request(struct request_queue *q)
2332 {
2333 	struct request *rq;
2334 	int ret;
2335 
2336 	while ((rq = __elv_next_request(q)) != NULL) {
2337 
2338 		rq = blk_pm_peek_request(q, rq);
2339 		if (!rq)
2340 			break;
2341 
2342 		if (!(rq->rq_flags & RQF_STARTED)) {
2343 			/*
2344 			 * This is the first time the device driver
2345 			 * sees this request (possibly after
2346 			 * requeueing).  Notify IO scheduler.
2347 			 */
2348 			if (rq->rq_flags & RQF_SORTED)
2349 				elv_activate_rq(q, rq);
2350 
2351 			/*
2352 			 * just mark as started even if we don't start
2353 			 * it, a request that has been delayed should
2354 			 * not be passed by new incoming requests
2355 			 */
2356 			rq->rq_flags |= RQF_STARTED;
2357 			trace_block_rq_issue(q, rq);
2358 		}
2359 
2360 		if (!q->boundary_rq || q->boundary_rq == rq) {
2361 			q->end_sector = rq_end_sector(rq);
2362 			q->boundary_rq = NULL;
2363 		}
2364 
2365 		if (rq->rq_flags & RQF_DONTPREP)
2366 			break;
2367 
2368 		if (q->dma_drain_size && blk_rq_bytes(rq)) {
2369 			/*
2370 			 * make sure space for the drain appears we
2371 			 * know we can do this because max_hw_segments
2372 			 * has been adjusted to be one fewer than the
2373 			 * device can handle
2374 			 */
2375 			rq->nr_phys_segments++;
2376 		}
2377 
2378 		if (!q->prep_rq_fn)
2379 			break;
2380 
2381 		ret = q->prep_rq_fn(q, rq);
2382 		if (ret == BLKPREP_OK) {
2383 			break;
2384 		} else if (ret == BLKPREP_DEFER) {
2385 			/*
2386 			 * the request may have been (partially) prepped.
2387 			 * we need to keep this request in the front to
2388 			 * avoid resource deadlock.  RQF_STARTED will
2389 			 * prevent other fs requests from passing this one.
2390 			 */
2391 			if (q->dma_drain_size && blk_rq_bytes(rq) &&
2392 			    !(rq->rq_flags & RQF_DONTPREP)) {
2393 				/*
2394 				 * remove the space for the drain we added
2395 				 * so that we don't add it again
2396 				 */
2397 				--rq->nr_phys_segments;
2398 			}
2399 
2400 			rq = NULL;
2401 			break;
2402 		} else if (ret == BLKPREP_KILL || ret == BLKPREP_INVALID) {
2403 			int err = (ret == BLKPREP_INVALID) ? -EREMOTEIO : -EIO;
2404 
2405 			rq->rq_flags |= RQF_QUIET;
2406 			/*
2407 			 * Mark this request as started so we don't trigger
2408 			 * any debug logic in the end I/O path.
2409 			 */
2410 			blk_start_request(rq);
2411 			__blk_end_request_all(rq, err);
2412 		} else {
2413 			printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
2414 			break;
2415 		}
2416 	}
2417 
2418 	return rq;
2419 }
2420 EXPORT_SYMBOL(blk_peek_request);
2421 
2422 void blk_dequeue_request(struct request *rq)
2423 {
2424 	struct request_queue *q = rq->q;
2425 
2426 	BUG_ON(list_empty(&rq->queuelist));
2427 	BUG_ON(ELV_ON_HASH(rq));
2428 
2429 	list_del_init(&rq->queuelist);
2430 
2431 	/*
2432 	 * the time frame between a request being removed from the lists
2433 	 * and to it is freed is accounted as io that is in progress at
2434 	 * the driver side.
2435 	 */
2436 	if (blk_account_rq(rq)) {
2437 		q->in_flight[rq_is_sync(rq)]++;
2438 		set_io_start_time_ns(rq);
2439 	}
2440 }
2441 
2442 /**
2443  * blk_start_request - start request processing on the driver
2444  * @req: request to dequeue
2445  *
2446  * Description:
2447  *     Dequeue @req and start timeout timer on it.  This hands off the
2448  *     request to the driver.
2449  *
2450  *     Block internal functions which don't want to start timer should
2451  *     call blk_dequeue_request().
2452  *
2453  * Context:
2454  *     queue_lock must be held.
2455  */
2456 void blk_start_request(struct request *req)
2457 {
2458 	blk_dequeue_request(req);
2459 
2460 	if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
2461 		blk_stat_set_issue_time(&req->issue_stat);
2462 		req->rq_flags |= RQF_STATS;
2463 		wbt_issue(req->q->rq_wb, &req->issue_stat);
2464 	}
2465 
2466 	BUG_ON(test_bit(REQ_ATOM_COMPLETE, &req->atomic_flags));
2467 	blk_add_timer(req);
2468 }
2469 EXPORT_SYMBOL(blk_start_request);
2470 
2471 /**
2472  * blk_fetch_request - fetch a request from a request queue
2473  * @q: request queue to fetch a request from
2474  *
2475  * Description:
2476  *     Return the request at the top of @q.  The request is started on
2477  *     return and LLD can start processing it immediately.
2478  *
2479  * Return:
2480  *     Pointer to the request at the top of @q if available.  Null
2481  *     otherwise.
2482  *
2483  * Context:
2484  *     queue_lock must be held.
2485  */
2486 struct request *blk_fetch_request(struct request_queue *q)
2487 {
2488 	struct request *rq;
2489 
2490 	rq = blk_peek_request(q);
2491 	if (rq)
2492 		blk_start_request(rq);
2493 	return rq;
2494 }
2495 EXPORT_SYMBOL(blk_fetch_request);
2496 
2497 /**
2498  * blk_update_request - Special helper function for request stacking drivers
2499  * @req:      the request being processed
2500  * @error:    %0 for success, < %0 for error
2501  * @nr_bytes: number of bytes to complete @req
2502  *
2503  * Description:
2504  *     Ends I/O on a number of bytes attached to @req, but doesn't complete
2505  *     the request structure even if @req doesn't have leftover.
2506  *     If @req has leftover, sets it up for the next range of segments.
2507  *
2508  *     This special helper function is only for request stacking drivers
2509  *     (e.g. request-based dm) so that they can handle partial completion.
2510  *     Actual device drivers should use blk_end_request instead.
2511  *
2512  *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
2513  *     %false return from this function.
2514  *
2515  * Return:
2516  *     %false - this request doesn't have any more data
2517  *     %true  - this request has more data
2518  **/
2519 bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
2520 {
2521 	int total_bytes;
2522 
2523 	trace_block_rq_complete(req->q, req, nr_bytes);
2524 
2525 	if (!req->bio)
2526 		return false;
2527 
2528 	/*
2529 	 * For fs requests, rq is just carrier of independent bio's
2530 	 * and each partial completion should be handled separately.
2531 	 * Reset per-request error on each partial completion.
2532 	 *
2533 	 * TODO: tj: This is too subtle.  It would be better to let
2534 	 * low level drivers do what they see fit.
2535 	 */
2536 	if (!blk_rq_is_passthrough(req))
2537 		req->errors = 0;
2538 
2539 	if (error && !blk_rq_is_passthrough(req) &&
2540 	    !(req->rq_flags & RQF_QUIET)) {
2541 		char *error_type;
2542 
2543 		switch (error) {
2544 		case -ENOLINK:
2545 			error_type = "recoverable transport";
2546 			break;
2547 		case -EREMOTEIO:
2548 			error_type = "critical target";
2549 			break;
2550 		case -EBADE:
2551 			error_type = "critical nexus";
2552 			break;
2553 		case -ETIMEDOUT:
2554 			error_type = "timeout";
2555 			break;
2556 		case -ENOSPC:
2557 			error_type = "critical space allocation";
2558 			break;
2559 		case -ENODATA:
2560 			error_type = "critical medium";
2561 			break;
2562 		case -EIO:
2563 		default:
2564 			error_type = "I/O";
2565 			break;
2566 		}
2567 		printk_ratelimited(KERN_ERR "%s: %s error, dev %s, sector %llu\n",
2568 				   __func__, error_type, req->rq_disk ?
2569 				   req->rq_disk->disk_name : "?",
2570 				   (unsigned long long)blk_rq_pos(req));
2571 
2572 	}
2573 
2574 	blk_account_io_completion(req, nr_bytes);
2575 
2576 	total_bytes = 0;
2577 	while (req->bio) {
2578 		struct bio *bio = req->bio;
2579 		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
2580 
2581 		if (bio_bytes == bio->bi_iter.bi_size)
2582 			req->bio = bio->bi_next;
2583 
2584 		req_bio_endio(req, bio, bio_bytes, error);
2585 
2586 		total_bytes += bio_bytes;
2587 		nr_bytes -= bio_bytes;
2588 
2589 		if (!nr_bytes)
2590 			break;
2591 	}
2592 
2593 	/*
2594 	 * completely done
2595 	 */
2596 	if (!req->bio) {
2597 		/*
2598 		 * Reset counters so that the request stacking driver
2599 		 * can find how many bytes remain in the request
2600 		 * later.
2601 		 */
2602 		req->__data_len = 0;
2603 		return false;
2604 	}
2605 
2606 	WARN_ON_ONCE(req->rq_flags & RQF_SPECIAL_PAYLOAD);
2607 
2608 	req->__data_len -= total_bytes;
2609 
2610 	/* update sector only for requests with clear definition of sector */
2611 	if (!blk_rq_is_passthrough(req))
2612 		req->__sector += total_bytes >> 9;
2613 
2614 	/* mixed attributes always follow the first bio */
2615 	if (req->rq_flags & RQF_MIXED_MERGE) {
2616 		req->cmd_flags &= ~REQ_FAILFAST_MASK;
2617 		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
2618 	}
2619 
2620 	/*
2621 	 * If total number of sectors is less than the first segment
2622 	 * size, something has gone terribly wrong.
2623 	 */
2624 	if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
2625 		blk_dump_rq_flags(req, "request botched");
2626 		req->__data_len = blk_rq_cur_bytes(req);
2627 	}
2628 
2629 	/* recalculate the number of segments */
2630 	blk_recalc_rq_segments(req);
2631 
2632 	return true;
2633 }
2634 EXPORT_SYMBOL_GPL(blk_update_request);
2635 
2636 static bool blk_update_bidi_request(struct request *rq, int error,
2637 				    unsigned int nr_bytes,
2638 				    unsigned int bidi_bytes)
2639 {
2640 	if (blk_update_request(rq, error, nr_bytes))
2641 		return true;
2642 
2643 	/* Bidi request must be completed as a whole */
2644 	if (unlikely(blk_bidi_rq(rq)) &&
2645 	    blk_update_request(rq->next_rq, error, bidi_bytes))
2646 		return true;
2647 
2648 	if (blk_queue_add_random(rq->q))
2649 		add_disk_randomness(rq->rq_disk);
2650 
2651 	return false;
2652 }
2653 
2654 /**
2655  * blk_unprep_request - unprepare a request
2656  * @req:	the request
2657  *
2658  * This function makes a request ready for complete resubmission (or
2659  * completion).  It happens only after all error handling is complete,
2660  * so represents the appropriate moment to deallocate any resources
2661  * that were allocated to the request in the prep_rq_fn.  The queue
2662  * lock is held when calling this.
2663  */
2664 void blk_unprep_request(struct request *req)
2665 {
2666 	struct request_queue *q = req->q;
2667 
2668 	req->rq_flags &= ~RQF_DONTPREP;
2669 	if (q->unprep_rq_fn)
2670 		q->unprep_rq_fn(q, req);
2671 }
2672 EXPORT_SYMBOL_GPL(blk_unprep_request);
2673 
2674 /*
2675  * queue lock must be held
2676  */
2677 void blk_finish_request(struct request *req, int error)
2678 {
2679 	struct request_queue *q = req->q;
2680 
2681 	if (req->rq_flags & RQF_STATS)
2682 		blk_stat_add(&q->rq_stats[rq_data_dir(req)], req);
2683 
2684 	if (req->rq_flags & RQF_QUEUED)
2685 		blk_queue_end_tag(q, req);
2686 
2687 	BUG_ON(blk_queued_rq(req));
2688 
2689 	if (unlikely(laptop_mode) && !blk_rq_is_passthrough(req))
2690 		laptop_io_completion(req->q->backing_dev_info);
2691 
2692 	blk_delete_timer(req);
2693 
2694 	if (req->rq_flags & RQF_DONTPREP)
2695 		blk_unprep_request(req);
2696 
2697 	blk_account_io_done(req);
2698 
2699 	if (req->end_io) {
2700 		wbt_done(req->q->rq_wb, &req->issue_stat);
2701 		req->end_io(req, error);
2702 	} else {
2703 		if (blk_bidi_rq(req))
2704 			__blk_put_request(req->next_rq->q, req->next_rq);
2705 
2706 		__blk_put_request(q, req);
2707 	}
2708 }
2709 EXPORT_SYMBOL(blk_finish_request);
2710 
2711 /**
2712  * blk_end_bidi_request - Complete a bidi request
2713  * @rq:         the request to complete
2714  * @error:      %0 for success, < %0 for error
2715  * @nr_bytes:   number of bytes to complete @rq
2716  * @bidi_bytes: number of bytes to complete @rq->next_rq
2717  *
2718  * Description:
2719  *     Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
2720  *     Drivers that supports bidi can safely call this member for any
2721  *     type of request, bidi or uni.  In the later case @bidi_bytes is
2722  *     just ignored.
2723  *
2724  * Return:
2725  *     %false - we are done with this request
2726  *     %true  - still buffers pending for this request
2727  **/
2728 static bool blk_end_bidi_request(struct request *rq, int error,
2729 				 unsigned int nr_bytes, unsigned int bidi_bytes)
2730 {
2731 	struct request_queue *q = rq->q;
2732 	unsigned long flags;
2733 
2734 	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2735 		return true;
2736 
2737 	spin_lock_irqsave(q->queue_lock, flags);
2738 	blk_finish_request(rq, error);
2739 	spin_unlock_irqrestore(q->queue_lock, flags);
2740 
2741 	return false;
2742 }
2743 
2744 /**
2745  * __blk_end_bidi_request - Complete a bidi request with queue lock held
2746  * @rq:         the request to complete
2747  * @error:      %0 for success, < %0 for error
2748  * @nr_bytes:   number of bytes to complete @rq
2749  * @bidi_bytes: number of bytes to complete @rq->next_rq
2750  *
2751  * Description:
2752  *     Identical to blk_end_bidi_request() except that queue lock is
2753  *     assumed to be locked on entry and remains so on return.
2754  *
2755  * Return:
2756  *     %false - we are done with this request
2757  *     %true  - still buffers pending for this request
2758  **/
2759 bool __blk_end_bidi_request(struct request *rq, int error,
2760 				   unsigned int nr_bytes, unsigned int bidi_bytes)
2761 {
2762 	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2763 		return true;
2764 
2765 	blk_finish_request(rq, error);
2766 
2767 	return false;
2768 }
2769 
2770 /**
2771  * blk_end_request - Helper function for drivers to complete the request.
2772  * @rq:       the request being processed
2773  * @error:    %0 for success, < %0 for error
2774  * @nr_bytes: number of bytes to complete
2775  *
2776  * Description:
2777  *     Ends I/O on a number of bytes attached to @rq.
2778  *     If @rq has leftover, sets it up for the next range of segments.
2779  *
2780  * Return:
2781  *     %false - we are done with this request
2782  *     %true  - still buffers pending for this request
2783  **/
2784 bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2785 {
2786 	return blk_end_bidi_request(rq, error, nr_bytes, 0);
2787 }
2788 EXPORT_SYMBOL(blk_end_request);
2789 
2790 /**
2791  * blk_end_request_all - Helper function for drives to finish the request.
2792  * @rq: the request to finish
2793  * @error: %0 for success, < %0 for error
2794  *
2795  * Description:
2796  *     Completely finish @rq.
2797  */
2798 void blk_end_request_all(struct request *rq, int error)
2799 {
2800 	bool pending;
2801 	unsigned int bidi_bytes = 0;
2802 
2803 	if (unlikely(blk_bidi_rq(rq)))
2804 		bidi_bytes = blk_rq_bytes(rq->next_rq);
2805 
2806 	pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2807 	BUG_ON(pending);
2808 }
2809 EXPORT_SYMBOL(blk_end_request_all);
2810 
2811 /**
2812  * blk_end_request_cur - Helper function to finish the current request chunk.
2813  * @rq: the request to finish the current chunk for
2814  * @error: %0 for success, < %0 for error
2815  *
2816  * Description:
2817  *     Complete the current consecutively mapped chunk from @rq.
2818  *
2819  * Return:
2820  *     %false - we are done with this request
2821  *     %true  - still buffers pending for this request
2822  */
2823 bool blk_end_request_cur(struct request *rq, int error)
2824 {
2825 	return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2826 }
2827 EXPORT_SYMBOL(blk_end_request_cur);
2828 
2829 /**
2830  * blk_end_request_err - Finish a request till the next failure boundary.
2831  * @rq: the request to finish till the next failure boundary for
2832  * @error: must be negative errno
2833  *
2834  * Description:
2835  *     Complete @rq till the next failure boundary.
2836  *
2837  * Return:
2838  *     %false - we are done with this request
2839  *     %true  - still buffers pending for this request
2840  */
2841 bool blk_end_request_err(struct request *rq, int error)
2842 {
2843 	WARN_ON(error >= 0);
2844 	return blk_end_request(rq, error, blk_rq_err_bytes(rq));
2845 }
2846 EXPORT_SYMBOL_GPL(blk_end_request_err);
2847 
2848 /**
2849  * __blk_end_request - Helper function for drivers to complete the request.
2850  * @rq:       the request being processed
2851  * @error:    %0 for success, < %0 for error
2852  * @nr_bytes: number of bytes to complete
2853  *
2854  * Description:
2855  *     Must be called with queue lock held unlike blk_end_request().
2856  *
2857  * Return:
2858  *     %false - we are done with this request
2859  *     %true  - still buffers pending for this request
2860  **/
2861 bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2862 {
2863 	return __blk_end_bidi_request(rq, error, nr_bytes, 0);
2864 }
2865 EXPORT_SYMBOL(__blk_end_request);
2866 
2867 /**
2868  * __blk_end_request_all - Helper function for drives to finish the request.
2869  * @rq: the request to finish
2870  * @error: %0 for success, < %0 for error
2871  *
2872  * Description:
2873  *     Completely finish @rq.  Must be called with queue lock held.
2874  */
2875 void __blk_end_request_all(struct request *rq, int error)
2876 {
2877 	bool pending;
2878 	unsigned int bidi_bytes = 0;
2879 
2880 	if (unlikely(blk_bidi_rq(rq)))
2881 		bidi_bytes = blk_rq_bytes(rq->next_rq);
2882 
2883 	pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2884 	BUG_ON(pending);
2885 }
2886 EXPORT_SYMBOL(__blk_end_request_all);
2887 
2888 /**
2889  * __blk_end_request_cur - Helper function to finish the current request chunk.
2890  * @rq: the request to finish the current chunk for
2891  * @error: %0 for success, < %0 for error
2892  *
2893  * Description:
2894  *     Complete the current consecutively mapped chunk from @rq.  Must
2895  *     be called with queue lock held.
2896  *
2897  * Return:
2898  *     %false - we are done with this request
2899  *     %true  - still buffers pending for this request
2900  */
2901 bool __blk_end_request_cur(struct request *rq, int error)
2902 {
2903 	return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2904 }
2905 EXPORT_SYMBOL(__blk_end_request_cur);
2906 
2907 /**
2908  * __blk_end_request_err - Finish a request till the next failure boundary.
2909  * @rq: the request to finish till the next failure boundary for
2910  * @error: must be negative errno
2911  *
2912  * Description:
2913  *     Complete @rq till the next failure boundary.  Must be called
2914  *     with queue lock held.
2915  *
2916  * Return:
2917  *     %false - we are done with this request
2918  *     %true  - still buffers pending for this request
2919  */
2920 bool __blk_end_request_err(struct request *rq, int error)
2921 {
2922 	WARN_ON(error >= 0);
2923 	return __blk_end_request(rq, error, blk_rq_err_bytes(rq));
2924 }
2925 EXPORT_SYMBOL_GPL(__blk_end_request_err);
2926 
2927 void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
2928 		     struct bio *bio)
2929 {
2930 	if (bio_has_data(bio))
2931 		rq->nr_phys_segments = bio_phys_segments(q, bio);
2932 
2933 	rq->__data_len = bio->bi_iter.bi_size;
2934 	rq->bio = rq->biotail = bio;
2935 
2936 	if (bio->bi_bdev)
2937 		rq->rq_disk = bio->bi_bdev->bd_disk;
2938 }
2939 
2940 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
2941 /**
2942  * rq_flush_dcache_pages - Helper function to flush all pages in a request
2943  * @rq: the request to be flushed
2944  *
2945  * Description:
2946  *     Flush all pages in @rq.
2947  */
2948 void rq_flush_dcache_pages(struct request *rq)
2949 {
2950 	struct req_iterator iter;
2951 	struct bio_vec bvec;
2952 
2953 	rq_for_each_segment(bvec, rq, iter)
2954 		flush_dcache_page(bvec.bv_page);
2955 }
2956 EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
2957 #endif
2958 
2959 /**
2960  * blk_lld_busy - Check if underlying low-level drivers of a device are busy
2961  * @q : the queue of the device being checked
2962  *
2963  * Description:
2964  *    Check if underlying low-level drivers of a device are busy.
2965  *    If the drivers want to export their busy state, they must set own
2966  *    exporting function using blk_queue_lld_busy() first.
2967  *
2968  *    Basically, this function is used only by request stacking drivers
2969  *    to stop dispatching requests to underlying devices when underlying
2970  *    devices are busy.  This behavior helps more I/O merging on the queue
2971  *    of the request stacking driver and prevents I/O throughput regression
2972  *    on burst I/O load.
2973  *
2974  * Return:
2975  *    0 - Not busy (The request stacking driver should dispatch request)
2976  *    1 - Busy (The request stacking driver should stop dispatching request)
2977  */
2978 int blk_lld_busy(struct request_queue *q)
2979 {
2980 	if (q->lld_busy_fn)
2981 		return q->lld_busy_fn(q);
2982 
2983 	return 0;
2984 }
2985 EXPORT_SYMBOL_GPL(blk_lld_busy);
2986 
2987 /**
2988  * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
2989  * @rq: the clone request to be cleaned up
2990  *
2991  * Description:
2992  *     Free all bios in @rq for a cloned request.
2993  */
2994 void blk_rq_unprep_clone(struct request *rq)
2995 {
2996 	struct bio *bio;
2997 
2998 	while ((bio = rq->bio) != NULL) {
2999 		rq->bio = bio->bi_next;
3000 
3001 		bio_put(bio);
3002 	}
3003 }
3004 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
3005 
3006 /*
3007  * Copy attributes of the original request to the clone request.
3008  * The actual data parts (e.g. ->cmd, ->sense) are not copied.
3009  */
3010 static void __blk_rq_prep_clone(struct request *dst, struct request *src)
3011 {
3012 	dst->cpu = src->cpu;
3013 	dst->__sector = blk_rq_pos(src);
3014 	dst->__data_len = blk_rq_bytes(src);
3015 	dst->nr_phys_segments = src->nr_phys_segments;
3016 	dst->ioprio = src->ioprio;
3017 	dst->extra_len = src->extra_len;
3018 }
3019 
3020 /**
3021  * blk_rq_prep_clone - Helper function to setup clone request
3022  * @rq: the request to be setup
3023  * @rq_src: original request to be cloned
3024  * @bs: bio_set that bios for clone are allocated from
3025  * @gfp_mask: memory allocation mask for bio
3026  * @bio_ctr: setup function to be called for each clone bio.
3027  *           Returns %0 for success, non %0 for failure.
3028  * @data: private data to be passed to @bio_ctr
3029  *
3030  * Description:
3031  *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
3032  *     The actual data parts of @rq_src (e.g. ->cmd, ->sense)
3033  *     are not copied, and copying such parts is the caller's responsibility.
3034  *     Also, pages which the original bios are pointing to are not copied
3035  *     and the cloned bios just point same pages.
3036  *     So cloned bios must be completed before original bios, which means
3037  *     the caller must complete @rq before @rq_src.
3038  */
3039 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
3040 		      struct bio_set *bs, gfp_t gfp_mask,
3041 		      int (*bio_ctr)(struct bio *, struct bio *, void *),
3042 		      void *data)
3043 {
3044 	struct bio *bio, *bio_src;
3045 
3046 	if (!bs)
3047 		bs = fs_bio_set;
3048 
3049 	__rq_for_each_bio(bio_src, rq_src) {
3050 		bio = bio_clone_fast(bio_src, gfp_mask, bs);
3051 		if (!bio)
3052 			goto free_and_out;
3053 
3054 		if (bio_ctr && bio_ctr(bio, bio_src, data))
3055 			goto free_and_out;
3056 
3057 		if (rq->bio) {
3058 			rq->biotail->bi_next = bio;
3059 			rq->biotail = bio;
3060 		} else
3061 			rq->bio = rq->biotail = bio;
3062 	}
3063 
3064 	__blk_rq_prep_clone(rq, rq_src);
3065 
3066 	return 0;
3067 
3068 free_and_out:
3069 	if (bio)
3070 		bio_put(bio);
3071 	blk_rq_unprep_clone(rq);
3072 
3073 	return -ENOMEM;
3074 }
3075 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
3076 
3077 int kblockd_schedule_work(struct work_struct *work)
3078 {
3079 	return queue_work(kblockd_workqueue, work);
3080 }
3081 EXPORT_SYMBOL(kblockd_schedule_work);
3082 
3083 int kblockd_schedule_work_on(int cpu, struct work_struct *work)
3084 {
3085 	return queue_work_on(cpu, kblockd_workqueue, work);
3086 }
3087 EXPORT_SYMBOL(kblockd_schedule_work_on);
3088 
3089 int kblockd_schedule_delayed_work(struct delayed_work *dwork,
3090 				  unsigned long delay)
3091 {
3092 	return queue_delayed_work(kblockd_workqueue, dwork, delay);
3093 }
3094 EXPORT_SYMBOL(kblockd_schedule_delayed_work);
3095 
3096 int kblockd_schedule_delayed_work_on(int cpu, struct delayed_work *dwork,
3097 				     unsigned long delay)
3098 {
3099 	return queue_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
3100 }
3101 EXPORT_SYMBOL(kblockd_schedule_delayed_work_on);
3102 
3103 /**
3104  * blk_start_plug - initialize blk_plug and track it inside the task_struct
3105  * @plug:	The &struct blk_plug that needs to be initialized
3106  *
3107  * Description:
3108  *   Tracking blk_plug inside the task_struct will help with auto-flushing the
3109  *   pending I/O should the task end up blocking between blk_start_plug() and
3110  *   blk_finish_plug(). This is important from a performance perspective, but
3111  *   also ensures that we don't deadlock. For instance, if the task is blocking
3112  *   for a memory allocation, memory reclaim could end up wanting to free a
3113  *   page belonging to that request that is currently residing in our private
3114  *   plug. By flushing the pending I/O when the process goes to sleep, we avoid
3115  *   this kind of deadlock.
3116  */
3117 void blk_start_plug(struct blk_plug *plug)
3118 {
3119 	struct task_struct *tsk = current;
3120 
3121 	/*
3122 	 * If this is a nested plug, don't actually assign it.
3123 	 */
3124 	if (tsk->plug)
3125 		return;
3126 
3127 	INIT_LIST_HEAD(&plug->list);
3128 	INIT_LIST_HEAD(&plug->mq_list);
3129 	INIT_LIST_HEAD(&plug->cb_list);
3130 	/*
3131 	 * Store ordering should not be needed here, since a potential
3132 	 * preempt will imply a full memory barrier
3133 	 */
3134 	tsk->plug = plug;
3135 }
3136 EXPORT_SYMBOL(blk_start_plug);
3137 
3138 static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
3139 {
3140 	struct request *rqa = container_of(a, struct request, queuelist);
3141 	struct request *rqb = container_of(b, struct request, queuelist);
3142 
3143 	return !(rqa->q < rqb->q ||
3144 		(rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb)));
3145 }
3146 
3147 /*
3148  * If 'from_schedule' is true, then postpone the dispatch of requests
3149  * until a safe kblockd context. We due this to avoid accidental big
3150  * additional stack usage in driver dispatch, in places where the originally
3151  * plugger did not intend it.
3152  */
3153 static void queue_unplugged(struct request_queue *q, unsigned int depth,
3154 			    bool from_schedule)
3155 	__releases(q->queue_lock)
3156 {
3157 	trace_block_unplug(q, depth, !from_schedule);
3158 
3159 	if (from_schedule)
3160 		blk_run_queue_async(q);
3161 	else
3162 		__blk_run_queue(q);
3163 	spin_unlock(q->queue_lock);
3164 }
3165 
3166 static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
3167 {
3168 	LIST_HEAD(callbacks);
3169 
3170 	while (!list_empty(&plug->cb_list)) {
3171 		list_splice_init(&plug->cb_list, &callbacks);
3172 
3173 		while (!list_empty(&callbacks)) {
3174 			struct blk_plug_cb *cb = list_first_entry(&callbacks,
3175 							  struct blk_plug_cb,
3176 							  list);
3177 			list_del(&cb->list);
3178 			cb->callback(cb, from_schedule);
3179 		}
3180 	}
3181 }
3182 
3183 struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
3184 				      int size)
3185 {
3186 	struct blk_plug *plug = current->plug;
3187 	struct blk_plug_cb *cb;
3188 
3189 	if (!plug)
3190 		return NULL;
3191 
3192 	list_for_each_entry(cb, &plug->cb_list, list)
3193 		if (cb->callback == unplug && cb->data == data)
3194 			return cb;
3195 
3196 	/* Not currently on the callback list */
3197 	BUG_ON(size < sizeof(*cb));
3198 	cb = kzalloc(size, GFP_ATOMIC);
3199 	if (cb) {
3200 		cb->data = data;
3201 		cb->callback = unplug;
3202 		list_add(&cb->list, &plug->cb_list);
3203 	}
3204 	return cb;
3205 }
3206 EXPORT_SYMBOL(blk_check_plugged);
3207 
3208 void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
3209 {
3210 	struct request_queue *q;
3211 	unsigned long flags;
3212 	struct request *rq;
3213 	LIST_HEAD(list);
3214 	unsigned int depth;
3215 
3216 	flush_plug_callbacks(plug, from_schedule);
3217 
3218 	if (!list_empty(&plug->mq_list))
3219 		blk_mq_flush_plug_list(plug, from_schedule);
3220 
3221 	if (list_empty(&plug->list))
3222 		return;
3223 
3224 	list_splice_init(&plug->list, &list);
3225 
3226 	list_sort(NULL, &list, plug_rq_cmp);
3227 
3228 	q = NULL;
3229 	depth = 0;
3230 
3231 	/*
3232 	 * Save and disable interrupts here, to avoid doing it for every
3233 	 * queue lock we have to take.
3234 	 */
3235 	local_irq_save(flags);
3236 	while (!list_empty(&list)) {
3237 		rq = list_entry_rq(list.next);
3238 		list_del_init(&rq->queuelist);
3239 		BUG_ON(!rq->q);
3240 		if (rq->q != q) {
3241 			/*
3242 			 * This drops the queue lock
3243 			 */
3244 			if (q)
3245 				queue_unplugged(q, depth, from_schedule);
3246 			q = rq->q;
3247 			depth = 0;
3248 			spin_lock(q->queue_lock);
3249 		}
3250 
3251 		/*
3252 		 * Short-circuit if @q is dead
3253 		 */
3254 		if (unlikely(blk_queue_dying(q))) {
3255 			__blk_end_request_all(rq, -ENODEV);
3256 			continue;
3257 		}
3258 
3259 		/*
3260 		 * rq is already accounted, so use raw insert
3261 		 */
3262 		if (op_is_flush(rq->cmd_flags))
3263 			__elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
3264 		else
3265 			__elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
3266 
3267 		depth++;
3268 	}
3269 
3270 	/*
3271 	 * This drops the queue lock
3272 	 */
3273 	if (q)
3274 		queue_unplugged(q, depth, from_schedule);
3275 
3276 	local_irq_restore(flags);
3277 }
3278 
3279 void blk_finish_plug(struct blk_plug *plug)
3280 {
3281 	if (plug != current->plug)
3282 		return;
3283 	blk_flush_plug_list(plug, false);
3284 
3285 	current->plug = NULL;
3286 }
3287 EXPORT_SYMBOL(blk_finish_plug);
3288 
3289 #ifdef CONFIG_PM
3290 /**
3291  * blk_pm_runtime_init - Block layer runtime PM initialization routine
3292  * @q: the queue of the device
3293  * @dev: the device the queue belongs to
3294  *
3295  * Description:
3296  *    Initialize runtime-PM-related fields for @q and start auto suspend for
3297  *    @dev. Drivers that want to take advantage of request-based runtime PM
3298  *    should call this function after @dev has been initialized, and its
3299  *    request queue @q has been allocated, and runtime PM for it can not happen
3300  *    yet(either due to disabled/forbidden or its usage_count > 0). In most
3301  *    cases, driver should call this function before any I/O has taken place.
3302  *
3303  *    This function takes care of setting up using auto suspend for the device,
3304  *    the autosuspend delay is set to -1 to make runtime suspend impossible
3305  *    until an updated value is either set by user or by driver. Drivers do
3306  *    not need to touch other autosuspend settings.
3307  *
3308  *    The block layer runtime PM is request based, so only works for drivers
3309  *    that use request as their IO unit instead of those directly use bio's.
3310  */
3311 void blk_pm_runtime_init(struct request_queue *q, struct device *dev)
3312 {
3313 	q->dev = dev;
3314 	q->rpm_status = RPM_ACTIVE;
3315 	pm_runtime_set_autosuspend_delay(q->dev, -1);
3316 	pm_runtime_use_autosuspend(q->dev);
3317 }
3318 EXPORT_SYMBOL(blk_pm_runtime_init);
3319 
3320 /**
3321  * blk_pre_runtime_suspend - Pre runtime suspend check
3322  * @q: the queue of the device
3323  *
3324  * Description:
3325  *    This function will check if runtime suspend is allowed for the device
3326  *    by examining if there are any requests pending in the queue. If there
3327  *    are requests pending, the device can not be runtime suspended; otherwise,
3328  *    the queue's status will be updated to SUSPENDING and the driver can
3329  *    proceed to suspend the device.
3330  *
3331  *    For the not allowed case, we mark last busy for the device so that
3332  *    runtime PM core will try to autosuspend it some time later.
3333  *
3334  *    This function should be called near the start of the device's
3335  *    runtime_suspend callback.
3336  *
3337  * Return:
3338  *    0		- OK to runtime suspend the device
3339  *    -EBUSY	- Device should not be runtime suspended
3340  */
3341 int blk_pre_runtime_suspend(struct request_queue *q)
3342 {
3343 	int ret = 0;
3344 
3345 	if (!q->dev)
3346 		return ret;
3347 
3348 	spin_lock_irq(q->queue_lock);
3349 	if (q->nr_pending) {
3350 		ret = -EBUSY;
3351 		pm_runtime_mark_last_busy(q->dev);
3352 	} else {
3353 		q->rpm_status = RPM_SUSPENDING;
3354 	}
3355 	spin_unlock_irq(q->queue_lock);
3356 	return ret;
3357 }
3358 EXPORT_SYMBOL(blk_pre_runtime_suspend);
3359 
3360 /**
3361  * blk_post_runtime_suspend - Post runtime suspend processing
3362  * @q: the queue of the device
3363  * @err: return value of the device's runtime_suspend function
3364  *
3365  * Description:
3366  *    Update the queue's runtime status according to the return value of the
3367  *    device's runtime suspend function and mark last busy for the device so
3368  *    that PM core will try to auto suspend the device at a later time.
3369  *
3370  *    This function should be called near the end of the device's
3371  *    runtime_suspend callback.
3372  */
3373 void blk_post_runtime_suspend(struct request_queue *q, int err)
3374 {
3375 	if (!q->dev)
3376 		return;
3377 
3378 	spin_lock_irq(q->queue_lock);
3379 	if (!err) {
3380 		q->rpm_status = RPM_SUSPENDED;
3381 	} else {
3382 		q->rpm_status = RPM_ACTIVE;
3383 		pm_runtime_mark_last_busy(q->dev);
3384 	}
3385 	spin_unlock_irq(q->queue_lock);
3386 }
3387 EXPORT_SYMBOL(blk_post_runtime_suspend);
3388 
3389 /**
3390  * blk_pre_runtime_resume - Pre runtime resume processing
3391  * @q: the queue of the device
3392  *
3393  * Description:
3394  *    Update the queue's runtime status to RESUMING in preparation for the
3395  *    runtime resume of the device.
3396  *
3397  *    This function should be called near the start of the device's
3398  *    runtime_resume callback.
3399  */
3400 void blk_pre_runtime_resume(struct request_queue *q)
3401 {
3402 	if (!q->dev)
3403 		return;
3404 
3405 	spin_lock_irq(q->queue_lock);
3406 	q->rpm_status = RPM_RESUMING;
3407 	spin_unlock_irq(q->queue_lock);
3408 }
3409 EXPORT_SYMBOL(blk_pre_runtime_resume);
3410 
3411 /**
3412  * blk_post_runtime_resume - Post runtime resume processing
3413  * @q: the queue of the device
3414  * @err: return value of the device's runtime_resume function
3415  *
3416  * Description:
3417  *    Update the queue's runtime status according to the return value of the
3418  *    device's runtime_resume function. If it is successfully resumed, process
3419  *    the requests that are queued into the device's queue when it is resuming
3420  *    and then mark last busy and initiate autosuspend for it.
3421  *
3422  *    This function should be called near the end of the device's
3423  *    runtime_resume callback.
3424  */
3425 void blk_post_runtime_resume(struct request_queue *q, int err)
3426 {
3427 	if (!q->dev)
3428 		return;
3429 
3430 	spin_lock_irq(q->queue_lock);
3431 	if (!err) {
3432 		q->rpm_status = RPM_ACTIVE;
3433 		__blk_run_queue(q);
3434 		pm_runtime_mark_last_busy(q->dev);
3435 		pm_request_autosuspend(q->dev);
3436 	} else {
3437 		q->rpm_status = RPM_SUSPENDED;
3438 	}
3439 	spin_unlock_irq(q->queue_lock);
3440 }
3441 EXPORT_SYMBOL(blk_post_runtime_resume);
3442 
3443 /**
3444  * blk_set_runtime_active - Force runtime status of the queue to be active
3445  * @q: the queue of the device
3446  *
3447  * If the device is left runtime suspended during system suspend the resume
3448  * hook typically resumes the device and corrects runtime status
3449  * accordingly. However, that does not affect the queue runtime PM status
3450  * which is still "suspended". This prevents processing requests from the
3451  * queue.
3452  *
3453  * This function can be used in driver's resume hook to correct queue
3454  * runtime PM status and re-enable peeking requests from the queue. It
3455  * should be called before first request is added to the queue.
3456  */
3457 void blk_set_runtime_active(struct request_queue *q)
3458 {
3459 	spin_lock_irq(q->queue_lock);
3460 	q->rpm_status = RPM_ACTIVE;
3461 	pm_runtime_mark_last_busy(q->dev);
3462 	pm_request_autosuspend(q->dev);
3463 	spin_unlock_irq(q->queue_lock);
3464 }
3465 EXPORT_SYMBOL(blk_set_runtime_active);
3466 #endif
3467 
3468 int __init blk_dev_init(void)
3469 {
3470 	BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
3471 	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
3472 			FIELD_SIZEOF(struct request, cmd_flags));
3473 	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
3474 			FIELD_SIZEOF(struct bio, bi_opf));
3475 
3476 	/* used for unplugging and affects IO latency/throughput - HIGHPRI */
3477 	kblockd_workqueue = alloc_workqueue("kblockd",
3478 					    WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
3479 	if (!kblockd_workqueue)
3480 		panic("Failed to create kblockd\n");
3481 
3482 	request_cachep = kmem_cache_create("blkdev_requests",
3483 			sizeof(struct request), 0, SLAB_PANIC, NULL);
3484 
3485 	blk_requestq_cachep = kmem_cache_create("request_queue",
3486 			sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
3487 
3488 #ifdef CONFIG_DEBUG_FS
3489 	blk_debugfs_root = debugfs_create_dir("block", NULL);
3490 #endif
3491 
3492 	return 0;
3493 }
3494