xref: /openbmc/linux/block/blk-core.c (revision 5f32c314)
1 /*
2  * Copyright (C) 1991, 1992 Linus Torvalds
3  * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
4  * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
5  * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
6  * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
7  *	-  July2000
8  * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
9  */
10 
11 /*
12  * This handles all read/write requests to block devices
13  */
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/backing-dev.h>
17 #include <linux/bio.h>
18 #include <linux/blkdev.h>
19 #include <linux/blk-mq.h>
20 #include <linux/highmem.h>
21 #include <linux/mm.h>
22 #include <linux/kernel_stat.h>
23 #include <linux/string.h>
24 #include <linux/init.h>
25 #include <linux/completion.h>
26 #include <linux/slab.h>
27 #include <linux/swap.h>
28 #include <linux/writeback.h>
29 #include <linux/task_io_accounting_ops.h>
30 #include <linux/fault-inject.h>
31 #include <linux/list_sort.h>
32 #include <linux/delay.h>
33 #include <linux/ratelimit.h>
34 #include <linux/pm_runtime.h>
35 
36 #define CREATE_TRACE_POINTS
37 #include <trace/events/block.h>
38 
39 #include "blk.h"
40 #include "blk-cgroup.h"
41 #include "blk-mq.h"
42 
43 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
44 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
45 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
46 EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
47 
48 DEFINE_IDA(blk_queue_ida);
49 
50 /*
51  * For the allocated request tables
52  */
53 struct kmem_cache *request_cachep = NULL;
54 
55 /*
56  * For queue allocation
57  */
58 struct kmem_cache *blk_requestq_cachep;
59 
60 /*
61  * Controlling structure to kblockd
62  */
63 static struct workqueue_struct *kblockd_workqueue;
64 
65 void blk_queue_congestion_threshold(struct request_queue *q)
66 {
67 	int nr;
68 
69 	nr = q->nr_requests - (q->nr_requests / 8) + 1;
70 	if (nr > q->nr_requests)
71 		nr = q->nr_requests;
72 	q->nr_congestion_on = nr;
73 
74 	nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
75 	if (nr < 1)
76 		nr = 1;
77 	q->nr_congestion_off = nr;
78 }
79 
80 /**
81  * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
82  * @bdev:	device
83  *
84  * Locates the passed device's request queue and returns the address of its
85  * backing_dev_info
86  *
87  * Will return NULL if the request queue cannot be located.
88  */
89 struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
90 {
91 	struct backing_dev_info *ret = NULL;
92 	struct request_queue *q = bdev_get_queue(bdev);
93 
94 	if (q)
95 		ret = &q->backing_dev_info;
96 	return ret;
97 }
98 EXPORT_SYMBOL(blk_get_backing_dev_info);
99 
100 void blk_rq_init(struct request_queue *q, struct request *rq)
101 {
102 	memset(rq, 0, sizeof(*rq));
103 
104 	INIT_LIST_HEAD(&rq->queuelist);
105 	INIT_LIST_HEAD(&rq->timeout_list);
106 	rq->cpu = -1;
107 	rq->q = q;
108 	rq->__sector = (sector_t) -1;
109 	INIT_HLIST_NODE(&rq->hash);
110 	RB_CLEAR_NODE(&rq->rb_node);
111 	rq->cmd = rq->__cmd;
112 	rq->cmd_len = BLK_MAX_CDB;
113 	rq->tag = -1;
114 	rq->start_time = jiffies;
115 	set_start_time_ns(rq);
116 	rq->part = NULL;
117 }
118 EXPORT_SYMBOL(blk_rq_init);
119 
120 static void req_bio_endio(struct request *rq, struct bio *bio,
121 			  unsigned int nbytes, int error)
122 {
123 	if (error)
124 		clear_bit(BIO_UPTODATE, &bio->bi_flags);
125 	else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
126 		error = -EIO;
127 
128 	if (unlikely(rq->cmd_flags & REQ_QUIET))
129 		set_bit(BIO_QUIET, &bio->bi_flags);
130 
131 	bio_advance(bio, nbytes);
132 
133 	/* don't actually finish bio if it's part of flush sequence */
134 	if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ))
135 		bio_endio(bio, error);
136 }
137 
138 void blk_dump_rq_flags(struct request *rq, char *msg)
139 {
140 	int bit;
141 
142 	printk(KERN_INFO "%s: dev %s: type=%x, flags=%llx\n", msg,
143 		rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
144 		(unsigned long long) rq->cmd_flags);
145 
146 	printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
147 	       (unsigned long long)blk_rq_pos(rq),
148 	       blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
149 	printk(KERN_INFO "  bio %p, biotail %p, buffer %p, len %u\n",
150 	       rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq));
151 
152 	if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
153 		printk(KERN_INFO "  cdb: ");
154 		for (bit = 0; bit < BLK_MAX_CDB; bit++)
155 			printk("%02x ", rq->cmd[bit]);
156 		printk("\n");
157 	}
158 }
159 EXPORT_SYMBOL(blk_dump_rq_flags);
160 
161 static void blk_delay_work(struct work_struct *work)
162 {
163 	struct request_queue *q;
164 
165 	q = container_of(work, struct request_queue, delay_work.work);
166 	spin_lock_irq(q->queue_lock);
167 	__blk_run_queue(q);
168 	spin_unlock_irq(q->queue_lock);
169 }
170 
171 /**
172  * blk_delay_queue - restart queueing after defined interval
173  * @q:		The &struct request_queue in question
174  * @msecs:	Delay in msecs
175  *
176  * Description:
177  *   Sometimes queueing needs to be postponed for a little while, to allow
178  *   resources to come back. This function will make sure that queueing is
179  *   restarted around the specified time. Queue lock must be held.
180  */
181 void blk_delay_queue(struct request_queue *q, unsigned long msecs)
182 {
183 	if (likely(!blk_queue_dead(q)))
184 		queue_delayed_work(kblockd_workqueue, &q->delay_work,
185 				   msecs_to_jiffies(msecs));
186 }
187 EXPORT_SYMBOL(blk_delay_queue);
188 
189 /**
190  * blk_start_queue - restart a previously stopped queue
191  * @q:    The &struct request_queue in question
192  *
193  * Description:
194  *   blk_start_queue() will clear the stop flag on the queue, and call
195  *   the request_fn for the queue if it was in a stopped state when
196  *   entered. Also see blk_stop_queue(). Queue lock must be held.
197  **/
198 void blk_start_queue(struct request_queue *q)
199 {
200 	WARN_ON(!irqs_disabled());
201 
202 	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
203 	__blk_run_queue(q);
204 }
205 EXPORT_SYMBOL(blk_start_queue);
206 
207 /**
208  * blk_stop_queue - stop a queue
209  * @q:    The &struct request_queue in question
210  *
211  * Description:
212  *   The Linux block layer assumes that a block driver will consume all
213  *   entries on the request queue when the request_fn strategy is called.
214  *   Often this will not happen, because of hardware limitations (queue
215  *   depth settings). If a device driver gets a 'queue full' response,
216  *   or if it simply chooses not to queue more I/O at one point, it can
217  *   call this function to prevent the request_fn from being called until
218  *   the driver has signalled it's ready to go again. This happens by calling
219  *   blk_start_queue() to restart queue operations. Queue lock must be held.
220  **/
221 void blk_stop_queue(struct request_queue *q)
222 {
223 	cancel_delayed_work(&q->delay_work);
224 	queue_flag_set(QUEUE_FLAG_STOPPED, q);
225 }
226 EXPORT_SYMBOL(blk_stop_queue);
227 
228 /**
229  * blk_sync_queue - cancel any pending callbacks on a queue
230  * @q: the queue
231  *
232  * Description:
233  *     The block layer may perform asynchronous callback activity
234  *     on a queue, such as calling the unplug function after a timeout.
235  *     A block device may call blk_sync_queue to ensure that any
236  *     such activity is cancelled, thus allowing it to release resources
237  *     that the callbacks might use. The caller must already have made sure
238  *     that its ->make_request_fn will not re-add plugging prior to calling
239  *     this function.
240  *
241  *     This function does not cancel any asynchronous activity arising
242  *     out of elevator or throttling code. That would require elevaotor_exit()
243  *     and blkcg_exit_queue() to be called with queue lock initialized.
244  *
245  */
246 void blk_sync_queue(struct request_queue *q)
247 {
248 	del_timer_sync(&q->timeout);
249 
250 	if (q->mq_ops) {
251 		struct blk_mq_hw_ctx *hctx;
252 		int i;
253 
254 		queue_for_each_hw_ctx(q, hctx, i)
255 			cancel_delayed_work_sync(&hctx->delayed_work);
256 	} else {
257 		cancel_delayed_work_sync(&q->delay_work);
258 	}
259 }
260 EXPORT_SYMBOL(blk_sync_queue);
261 
262 /**
263  * __blk_run_queue_uncond - run a queue whether or not it has been stopped
264  * @q:	The queue to run
265  *
266  * Description:
267  *    Invoke request handling on a queue if there are any pending requests.
268  *    May be used to restart request handling after a request has completed.
269  *    This variant runs the queue whether or not the queue has been
270  *    stopped. Must be called with the queue lock held and interrupts
271  *    disabled. See also @blk_run_queue.
272  */
273 inline void __blk_run_queue_uncond(struct request_queue *q)
274 {
275 	if (unlikely(blk_queue_dead(q)))
276 		return;
277 
278 	/*
279 	 * Some request_fn implementations, e.g. scsi_request_fn(), unlock
280 	 * the queue lock internally. As a result multiple threads may be
281 	 * running such a request function concurrently. Keep track of the
282 	 * number of active request_fn invocations such that blk_drain_queue()
283 	 * can wait until all these request_fn calls have finished.
284 	 */
285 	q->request_fn_active++;
286 	q->request_fn(q);
287 	q->request_fn_active--;
288 }
289 
290 /**
291  * __blk_run_queue - run a single device queue
292  * @q:	The queue to run
293  *
294  * Description:
295  *    See @blk_run_queue. This variant must be called with the queue lock
296  *    held and interrupts disabled.
297  */
298 void __blk_run_queue(struct request_queue *q)
299 {
300 	if (unlikely(blk_queue_stopped(q)))
301 		return;
302 
303 	__blk_run_queue_uncond(q);
304 }
305 EXPORT_SYMBOL(__blk_run_queue);
306 
307 /**
308  * blk_run_queue_async - run a single device queue in workqueue context
309  * @q:	The queue to run
310  *
311  * Description:
312  *    Tells kblockd to perform the equivalent of @blk_run_queue on behalf
313  *    of us. The caller must hold the queue lock.
314  */
315 void blk_run_queue_async(struct request_queue *q)
316 {
317 	if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q)))
318 		mod_delayed_work(kblockd_workqueue, &q->delay_work, 0);
319 }
320 EXPORT_SYMBOL(blk_run_queue_async);
321 
322 /**
323  * blk_run_queue - run a single device queue
324  * @q: The queue to run
325  *
326  * Description:
327  *    Invoke request handling on this queue, if it has pending work to do.
328  *    May be used to restart queueing when a request has completed.
329  */
330 void blk_run_queue(struct request_queue *q)
331 {
332 	unsigned long flags;
333 
334 	spin_lock_irqsave(q->queue_lock, flags);
335 	__blk_run_queue(q);
336 	spin_unlock_irqrestore(q->queue_lock, flags);
337 }
338 EXPORT_SYMBOL(blk_run_queue);
339 
340 void blk_put_queue(struct request_queue *q)
341 {
342 	kobject_put(&q->kobj);
343 }
344 EXPORT_SYMBOL(blk_put_queue);
345 
346 /**
347  * __blk_drain_queue - drain requests from request_queue
348  * @q: queue to drain
349  * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV
350  *
351  * Drain requests from @q.  If @drain_all is set, all requests are drained.
352  * If not, only ELVPRIV requests are drained.  The caller is responsible
353  * for ensuring that no new requests which need to be drained are queued.
354  */
355 static void __blk_drain_queue(struct request_queue *q, bool drain_all)
356 	__releases(q->queue_lock)
357 	__acquires(q->queue_lock)
358 {
359 	int i;
360 
361 	lockdep_assert_held(q->queue_lock);
362 
363 	while (true) {
364 		bool drain = false;
365 
366 		/*
367 		 * The caller might be trying to drain @q before its
368 		 * elevator is initialized.
369 		 */
370 		if (q->elevator)
371 			elv_drain_elevator(q);
372 
373 		blkcg_drain_queue(q);
374 
375 		/*
376 		 * This function might be called on a queue which failed
377 		 * driver init after queue creation or is not yet fully
378 		 * active yet.  Some drivers (e.g. fd and loop) get unhappy
379 		 * in such cases.  Kick queue iff dispatch queue has
380 		 * something on it and @q has request_fn set.
381 		 */
382 		if (!list_empty(&q->queue_head) && q->request_fn)
383 			__blk_run_queue(q);
384 
385 		drain |= q->nr_rqs_elvpriv;
386 		drain |= q->request_fn_active;
387 
388 		/*
389 		 * Unfortunately, requests are queued at and tracked from
390 		 * multiple places and there's no single counter which can
391 		 * be drained.  Check all the queues and counters.
392 		 */
393 		if (drain_all) {
394 			drain |= !list_empty(&q->queue_head);
395 			for (i = 0; i < 2; i++) {
396 				drain |= q->nr_rqs[i];
397 				drain |= q->in_flight[i];
398 				drain |= !list_empty(&q->flush_queue[i]);
399 			}
400 		}
401 
402 		if (!drain)
403 			break;
404 
405 		spin_unlock_irq(q->queue_lock);
406 
407 		msleep(10);
408 
409 		spin_lock_irq(q->queue_lock);
410 	}
411 
412 	/*
413 	 * With queue marked dead, any woken up waiter will fail the
414 	 * allocation path, so the wakeup chaining is lost and we're
415 	 * left with hung waiters. We need to wake up those waiters.
416 	 */
417 	if (q->request_fn) {
418 		struct request_list *rl;
419 
420 		blk_queue_for_each_rl(rl, q)
421 			for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
422 				wake_up_all(&rl->wait[i]);
423 	}
424 }
425 
426 /**
427  * blk_queue_bypass_start - enter queue bypass mode
428  * @q: queue of interest
429  *
430  * In bypass mode, only the dispatch FIFO queue of @q is used.  This
431  * function makes @q enter bypass mode and drains all requests which were
432  * throttled or issued before.  On return, it's guaranteed that no request
433  * is being throttled or has ELVPRIV set and blk_queue_bypass() %true
434  * inside queue or RCU read lock.
435  */
436 void blk_queue_bypass_start(struct request_queue *q)
437 {
438 	bool drain;
439 
440 	spin_lock_irq(q->queue_lock);
441 	drain = !q->bypass_depth++;
442 	queue_flag_set(QUEUE_FLAG_BYPASS, q);
443 	spin_unlock_irq(q->queue_lock);
444 
445 	if (drain) {
446 		spin_lock_irq(q->queue_lock);
447 		__blk_drain_queue(q, false);
448 		spin_unlock_irq(q->queue_lock);
449 
450 		/* ensure blk_queue_bypass() is %true inside RCU read lock */
451 		synchronize_rcu();
452 	}
453 }
454 EXPORT_SYMBOL_GPL(blk_queue_bypass_start);
455 
456 /**
457  * blk_queue_bypass_end - leave queue bypass mode
458  * @q: queue of interest
459  *
460  * Leave bypass mode and restore the normal queueing behavior.
461  */
462 void blk_queue_bypass_end(struct request_queue *q)
463 {
464 	spin_lock_irq(q->queue_lock);
465 	if (!--q->bypass_depth)
466 		queue_flag_clear(QUEUE_FLAG_BYPASS, q);
467 	WARN_ON_ONCE(q->bypass_depth < 0);
468 	spin_unlock_irq(q->queue_lock);
469 }
470 EXPORT_SYMBOL_GPL(blk_queue_bypass_end);
471 
472 /**
473  * blk_cleanup_queue - shutdown a request queue
474  * @q: request queue to shutdown
475  *
476  * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
477  * put it.  All future requests will be failed immediately with -ENODEV.
478  */
479 void blk_cleanup_queue(struct request_queue *q)
480 {
481 	spinlock_t *lock = q->queue_lock;
482 
483 	/* mark @q DYING, no new request or merges will be allowed afterwards */
484 	mutex_lock(&q->sysfs_lock);
485 	queue_flag_set_unlocked(QUEUE_FLAG_DYING, q);
486 	spin_lock_irq(lock);
487 
488 	/*
489 	 * A dying queue is permanently in bypass mode till released.  Note
490 	 * that, unlike blk_queue_bypass_start(), we aren't performing
491 	 * synchronize_rcu() after entering bypass mode to avoid the delay
492 	 * as some drivers create and destroy a lot of queues while
493 	 * probing.  This is still safe because blk_release_queue() will be
494 	 * called only after the queue refcnt drops to zero and nothing,
495 	 * RCU or not, would be traversing the queue by then.
496 	 */
497 	q->bypass_depth++;
498 	queue_flag_set(QUEUE_FLAG_BYPASS, q);
499 
500 	queue_flag_set(QUEUE_FLAG_NOMERGES, q);
501 	queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
502 	queue_flag_set(QUEUE_FLAG_DYING, q);
503 	spin_unlock_irq(lock);
504 	mutex_unlock(&q->sysfs_lock);
505 
506 	/*
507 	 * Drain all requests queued before DYING marking. Set DEAD flag to
508 	 * prevent that q->request_fn() gets invoked after draining finished.
509 	 */
510 	if (q->mq_ops) {
511 		blk_mq_drain_queue(q);
512 		spin_lock_irq(lock);
513 	} else {
514 		spin_lock_irq(lock);
515 		__blk_drain_queue(q, true);
516 	}
517 	queue_flag_set(QUEUE_FLAG_DEAD, q);
518 	spin_unlock_irq(lock);
519 
520 	/* @q won't process any more request, flush async actions */
521 	del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer);
522 	blk_sync_queue(q);
523 
524 	spin_lock_irq(lock);
525 	if (q->queue_lock != &q->__queue_lock)
526 		q->queue_lock = &q->__queue_lock;
527 	spin_unlock_irq(lock);
528 
529 	/* @q is and will stay empty, shutdown and put */
530 	blk_put_queue(q);
531 }
532 EXPORT_SYMBOL(blk_cleanup_queue);
533 
534 int blk_init_rl(struct request_list *rl, struct request_queue *q,
535 		gfp_t gfp_mask)
536 {
537 	if (unlikely(rl->rq_pool))
538 		return 0;
539 
540 	rl->q = q;
541 	rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
542 	rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
543 	init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
544 	init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
545 
546 	rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
547 					  mempool_free_slab, request_cachep,
548 					  gfp_mask, q->node);
549 	if (!rl->rq_pool)
550 		return -ENOMEM;
551 
552 	return 0;
553 }
554 
555 void blk_exit_rl(struct request_list *rl)
556 {
557 	if (rl->rq_pool)
558 		mempool_destroy(rl->rq_pool);
559 }
560 
561 struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
562 {
563 	return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE);
564 }
565 EXPORT_SYMBOL(blk_alloc_queue);
566 
567 struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
568 {
569 	struct request_queue *q;
570 	int err;
571 
572 	q = kmem_cache_alloc_node(blk_requestq_cachep,
573 				gfp_mask | __GFP_ZERO, node_id);
574 	if (!q)
575 		return NULL;
576 
577 	if (percpu_counter_init(&q->mq_usage_counter, 0))
578 		goto fail_q;
579 
580 	q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
581 	if (q->id < 0)
582 		goto fail_c;
583 
584 	q->backing_dev_info.ra_pages =
585 			(VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
586 	q->backing_dev_info.state = 0;
587 	q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY;
588 	q->backing_dev_info.name = "block";
589 	q->node = node_id;
590 
591 	err = bdi_init(&q->backing_dev_info);
592 	if (err)
593 		goto fail_id;
594 
595 	setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
596 		    laptop_mode_timer_fn, (unsigned long) q);
597 	setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
598 	INIT_LIST_HEAD(&q->queue_head);
599 	INIT_LIST_HEAD(&q->timeout_list);
600 	INIT_LIST_HEAD(&q->icq_list);
601 #ifdef CONFIG_BLK_CGROUP
602 	INIT_LIST_HEAD(&q->blkg_list);
603 #endif
604 	INIT_LIST_HEAD(&q->flush_queue[0]);
605 	INIT_LIST_HEAD(&q->flush_queue[1]);
606 	INIT_LIST_HEAD(&q->flush_data_in_flight);
607 	INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
608 
609 	kobject_init(&q->kobj, &blk_queue_ktype);
610 
611 	mutex_init(&q->sysfs_lock);
612 	spin_lock_init(&q->__queue_lock);
613 
614 	/*
615 	 * By default initialize queue_lock to internal lock and driver can
616 	 * override it later if need be.
617 	 */
618 	q->queue_lock = &q->__queue_lock;
619 
620 	/*
621 	 * A queue starts its life with bypass turned on to avoid
622 	 * unnecessary bypass on/off overhead and nasty surprises during
623 	 * init.  The initial bypass will be finished when the queue is
624 	 * registered by blk_register_queue().
625 	 */
626 	q->bypass_depth = 1;
627 	__set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
628 
629 	init_waitqueue_head(&q->mq_freeze_wq);
630 
631 	if (blkcg_init_queue(q))
632 		goto fail_bdi;
633 
634 	return q;
635 
636 fail_bdi:
637 	bdi_destroy(&q->backing_dev_info);
638 fail_id:
639 	ida_simple_remove(&blk_queue_ida, q->id);
640 fail_c:
641 	percpu_counter_destroy(&q->mq_usage_counter);
642 fail_q:
643 	kmem_cache_free(blk_requestq_cachep, q);
644 	return NULL;
645 }
646 EXPORT_SYMBOL(blk_alloc_queue_node);
647 
648 /**
649  * blk_init_queue  - prepare a request queue for use with a block device
650  * @rfn:  The function to be called to process requests that have been
651  *        placed on the queue.
652  * @lock: Request queue spin lock
653  *
654  * Description:
655  *    If a block device wishes to use the standard request handling procedures,
656  *    which sorts requests and coalesces adjacent requests, then it must
657  *    call blk_init_queue().  The function @rfn will be called when there
658  *    are requests on the queue that need to be processed.  If the device
659  *    supports plugging, then @rfn may not be called immediately when requests
660  *    are available on the queue, but may be called at some time later instead.
661  *    Plugged queues are generally unplugged when a buffer belonging to one
662  *    of the requests on the queue is needed, or due to memory pressure.
663  *
664  *    @rfn is not required, or even expected, to remove all requests off the
665  *    queue, but only as many as it can handle at a time.  If it does leave
666  *    requests on the queue, it is responsible for arranging that the requests
667  *    get dealt with eventually.
668  *
669  *    The queue spin lock must be held while manipulating the requests on the
670  *    request queue; this lock will be taken also from interrupt context, so irq
671  *    disabling is needed for it.
672  *
673  *    Function returns a pointer to the initialized request queue, or %NULL if
674  *    it didn't succeed.
675  *
676  * Note:
677  *    blk_init_queue() must be paired with a blk_cleanup_queue() call
678  *    when the block device is deactivated (such as at module unload).
679  **/
680 
681 struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
682 {
683 	return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
684 }
685 EXPORT_SYMBOL(blk_init_queue);
686 
687 struct request_queue *
688 blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
689 {
690 	struct request_queue *uninit_q, *q;
691 
692 	uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id);
693 	if (!uninit_q)
694 		return NULL;
695 
696 	q = blk_init_allocated_queue(uninit_q, rfn, lock);
697 	if (!q)
698 		blk_cleanup_queue(uninit_q);
699 
700 	return q;
701 }
702 EXPORT_SYMBOL(blk_init_queue_node);
703 
704 struct request_queue *
705 blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
706 			 spinlock_t *lock)
707 {
708 	if (!q)
709 		return NULL;
710 
711 	if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
712 		return NULL;
713 
714 	q->request_fn		= rfn;
715 	q->prep_rq_fn		= NULL;
716 	q->unprep_rq_fn		= NULL;
717 	q->queue_flags		|= QUEUE_FLAG_DEFAULT;
718 
719 	/* Override internal queue lock with supplied lock pointer */
720 	if (lock)
721 		q->queue_lock		= lock;
722 
723 	/*
724 	 * This also sets hw/phys segments, boundary and size
725 	 */
726 	blk_queue_make_request(q, blk_queue_bio);
727 
728 	q->sg_reserved_size = INT_MAX;
729 
730 	/* Protect q->elevator from elevator_change */
731 	mutex_lock(&q->sysfs_lock);
732 
733 	/* init elevator */
734 	if (elevator_init(q, NULL)) {
735 		mutex_unlock(&q->sysfs_lock);
736 		return NULL;
737 	}
738 
739 	mutex_unlock(&q->sysfs_lock);
740 
741 	return q;
742 }
743 EXPORT_SYMBOL(blk_init_allocated_queue);
744 
745 bool blk_get_queue(struct request_queue *q)
746 {
747 	if (likely(!blk_queue_dying(q))) {
748 		__blk_get_queue(q);
749 		return true;
750 	}
751 
752 	return false;
753 }
754 EXPORT_SYMBOL(blk_get_queue);
755 
756 static inline void blk_free_request(struct request_list *rl, struct request *rq)
757 {
758 	if (rq->cmd_flags & REQ_ELVPRIV) {
759 		elv_put_request(rl->q, rq);
760 		if (rq->elv.icq)
761 			put_io_context(rq->elv.icq->ioc);
762 	}
763 
764 	mempool_free(rq, rl->rq_pool);
765 }
766 
767 /*
768  * ioc_batching returns true if the ioc is a valid batching request and
769  * should be given priority access to a request.
770  */
771 static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
772 {
773 	if (!ioc)
774 		return 0;
775 
776 	/*
777 	 * Make sure the process is able to allocate at least 1 request
778 	 * even if the batch times out, otherwise we could theoretically
779 	 * lose wakeups.
780 	 */
781 	return ioc->nr_batch_requests == q->nr_batching ||
782 		(ioc->nr_batch_requests > 0
783 		&& time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
784 }
785 
786 /*
787  * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
788  * will cause the process to be a "batcher" on all queues in the system. This
789  * is the behaviour we want though - once it gets a wakeup it should be given
790  * a nice run.
791  */
792 static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
793 {
794 	if (!ioc || ioc_batching(q, ioc))
795 		return;
796 
797 	ioc->nr_batch_requests = q->nr_batching;
798 	ioc->last_waited = jiffies;
799 }
800 
801 static void __freed_request(struct request_list *rl, int sync)
802 {
803 	struct request_queue *q = rl->q;
804 
805 	/*
806 	 * bdi isn't aware of blkcg yet.  As all async IOs end up root
807 	 * blkcg anyway, just use root blkcg state.
808 	 */
809 	if (rl == &q->root_rl &&
810 	    rl->count[sync] < queue_congestion_off_threshold(q))
811 		blk_clear_queue_congested(q, sync);
812 
813 	if (rl->count[sync] + 1 <= q->nr_requests) {
814 		if (waitqueue_active(&rl->wait[sync]))
815 			wake_up(&rl->wait[sync]);
816 
817 		blk_clear_rl_full(rl, sync);
818 	}
819 }
820 
821 /*
822  * A request has just been released.  Account for it, update the full and
823  * congestion status, wake up any waiters.   Called under q->queue_lock.
824  */
825 static void freed_request(struct request_list *rl, unsigned int flags)
826 {
827 	struct request_queue *q = rl->q;
828 	int sync = rw_is_sync(flags);
829 
830 	q->nr_rqs[sync]--;
831 	rl->count[sync]--;
832 	if (flags & REQ_ELVPRIV)
833 		q->nr_rqs_elvpriv--;
834 
835 	__freed_request(rl, sync);
836 
837 	if (unlikely(rl->starved[sync ^ 1]))
838 		__freed_request(rl, sync ^ 1);
839 }
840 
841 /*
842  * Determine if elevator data should be initialized when allocating the
843  * request associated with @bio.
844  */
845 static bool blk_rq_should_init_elevator(struct bio *bio)
846 {
847 	if (!bio)
848 		return true;
849 
850 	/*
851 	 * Flush requests do not use the elevator so skip initialization.
852 	 * This allows a request to share the flush and elevator data.
853 	 */
854 	if (bio->bi_rw & (REQ_FLUSH | REQ_FUA))
855 		return false;
856 
857 	return true;
858 }
859 
860 /**
861  * rq_ioc - determine io_context for request allocation
862  * @bio: request being allocated is for this bio (can be %NULL)
863  *
864  * Determine io_context to use for request allocation for @bio.  May return
865  * %NULL if %current->io_context doesn't exist.
866  */
867 static struct io_context *rq_ioc(struct bio *bio)
868 {
869 #ifdef CONFIG_BLK_CGROUP
870 	if (bio && bio->bi_ioc)
871 		return bio->bi_ioc;
872 #endif
873 	return current->io_context;
874 }
875 
876 /**
877  * __get_request - get a free request
878  * @rl: request list to allocate from
879  * @rw_flags: RW and SYNC flags
880  * @bio: bio to allocate request for (can be %NULL)
881  * @gfp_mask: allocation mask
882  *
883  * Get a free request from @q.  This function may fail under memory
884  * pressure or if @q is dead.
885  *
886  * Must be callled with @q->queue_lock held and,
887  * Returns %NULL on failure, with @q->queue_lock held.
888  * Returns !%NULL on success, with @q->queue_lock *not held*.
889  */
890 static struct request *__get_request(struct request_list *rl, int rw_flags,
891 				     struct bio *bio, gfp_t gfp_mask)
892 {
893 	struct request_queue *q = rl->q;
894 	struct request *rq;
895 	struct elevator_type *et = q->elevator->type;
896 	struct io_context *ioc = rq_ioc(bio);
897 	struct io_cq *icq = NULL;
898 	const bool is_sync = rw_is_sync(rw_flags) != 0;
899 	int may_queue;
900 
901 	if (unlikely(blk_queue_dying(q)))
902 		return NULL;
903 
904 	may_queue = elv_may_queue(q, rw_flags);
905 	if (may_queue == ELV_MQUEUE_NO)
906 		goto rq_starved;
907 
908 	if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
909 		if (rl->count[is_sync]+1 >= q->nr_requests) {
910 			/*
911 			 * The queue will fill after this allocation, so set
912 			 * it as full, and mark this process as "batching".
913 			 * This process will be allowed to complete a batch of
914 			 * requests, others will be blocked.
915 			 */
916 			if (!blk_rl_full(rl, is_sync)) {
917 				ioc_set_batching(q, ioc);
918 				blk_set_rl_full(rl, is_sync);
919 			} else {
920 				if (may_queue != ELV_MQUEUE_MUST
921 						&& !ioc_batching(q, ioc)) {
922 					/*
923 					 * The queue is full and the allocating
924 					 * process is not a "batcher", and not
925 					 * exempted by the IO scheduler
926 					 */
927 					return NULL;
928 				}
929 			}
930 		}
931 		/*
932 		 * bdi isn't aware of blkcg yet.  As all async IOs end up
933 		 * root blkcg anyway, just use root blkcg state.
934 		 */
935 		if (rl == &q->root_rl)
936 			blk_set_queue_congested(q, is_sync);
937 	}
938 
939 	/*
940 	 * Only allow batching queuers to allocate up to 50% over the defined
941 	 * limit of requests, otherwise we could have thousands of requests
942 	 * allocated with any setting of ->nr_requests
943 	 */
944 	if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
945 		return NULL;
946 
947 	q->nr_rqs[is_sync]++;
948 	rl->count[is_sync]++;
949 	rl->starved[is_sync] = 0;
950 
951 	/*
952 	 * Decide whether the new request will be managed by elevator.  If
953 	 * so, mark @rw_flags and increment elvpriv.  Non-zero elvpriv will
954 	 * prevent the current elevator from being destroyed until the new
955 	 * request is freed.  This guarantees icq's won't be destroyed and
956 	 * makes creating new ones safe.
957 	 *
958 	 * Also, lookup icq while holding queue_lock.  If it doesn't exist,
959 	 * it will be created after releasing queue_lock.
960 	 */
961 	if (blk_rq_should_init_elevator(bio) && !blk_queue_bypass(q)) {
962 		rw_flags |= REQ_ELVPRIV;
963 		q->nr_rqs_elvpriv++;
964 		if (et->icq_cache && ioc)
965 			icq = ioc_lookup_icq(ioc, q);
966 	}
967 
968 	if (blk_queue_io_stat(q))
969 		rw_flags |= REQ_IO_STAT;
970 	spin_unlock_irq(q->queue_lock);
971 
972 	/* allocate and init request */
973 	rq = mempool_alloc(rl->rq_pool, gfp_mask);
974 	if (!rq)
975 		goto fail_alloc;
976 
977 	blk_rq_init(q, rq);
978 	blk_rq_set_rl(rq, rl);
979 	rq->cmd_flags = rw_flags | REQ_ALLOCED;
980 
981 	/* init elvpriv */
982 	if (rw_flags & REQ_ELVPRIV) {
983 		if (unlikely(et->icq_cache && !icq)) {
984 			if (ioc)
985 				icq = ioc_create_icq(ioc, q, gfp_mask);
986 			if (!icq)
987 				goto fail_elvpriv;
988 		}
989 
990 		rq->elv.icq = icq;
991 		if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
992 			goto fail_elvpriv;
993 
994 		/* @rq->elv.icq holds io_context until @rq is freed */
995 		if (icq)
996 			get_io_context(icq->ioc);
997 	}
998 out:
999 	/*
1000 	 * ioc may be NULL here, and ioc_batching will be false. That's
1001 	 * OK, if the queue is under the request limit then requests need
1002 	 * not count toward the nr_batch_requests limit. There will always
1003 	 * be some limit enforced by BLK_BATCH_TIME.
1004 	 */
1005 	if (ioc_batching(q, ioc))
1006 		ioc->nr_batch_requests--;
1007 
1008 	trace_block_getrq(q, bio, rw_flags & 1);
1009 	return rq;
1010 
1011 fail_elvpriv:
1012 	/*
1013 	 * elvpriv init failed.  ioc, icq and elvpriv aren't mempool backed
1014 	 * and may fail indefinitely under memory pressure and thus
1015 	 * shouldn't stall IO.  Treat this request as !elvpriv.  This will
1016 	 * disturb iosched and blkcg but weird is bettern than dead.
1017 	 */
1018 	printk_ratelimited(KERN_WARNING "%s: request aux data allocation failed, iosched may be disturbed\n",
1019 			   dev_name(q->backing_dev_info.dev));
1020 
1021 	rq->cmd_flags &= ~REQ_ELVPRIV;
1022 	rq->elv.icq = NULL;
1023 
1024 	spin_lock_irq(q->queue_lock);
1025 	q->nr_rqs_elvpriv--;
1026 	spin_unlock_irq(q->queue_lock);
1027 	goto out;
1028 
1029 fail_alloc:
1030 	/*
1031 	 * Allocation failed presumably due to memory. Undo anything we
1032 	 * might have messed up.
1033 	 *
1034 	 * Allocating task should really be put onto the front of the wait
1035 	 * queue, but this is pretty rare.
1036 	 */
1037 	spin_lock_irq(q->queue_lock);
1038 	freed_request(rl, rw_flags);
1039 
1040 	/*
1041 	 * in the very unlikely event that allocation failed and no
1042 	 * requests for this direction was pending, mark us starved so that
1043 	 * freeing of a request in the other direction will notice
1044 	 * us. another possible fix would be to split the rq mempool into
1045 	 * READ and WRITE
1046 	 */
1047 rq_starved:
1048 	if (unlikely(rl->count[is_sync] == 0))
1049 		rl->starved[is_sync] = 1;
1050 	return NULL;
1051 }
1052 
1053 /**
1054  * get_request - get a free request
1055  * @q: request_queue to allocate request from
1056  * @rw_flags: RW and SYNC flags
1057  * @bio: bio to allocate request for (can be %NULL)
1058  * @gfp_mask: allocation mask
1059  *
1060  * Get a free request from @q.  If %__GFP_WAIT is set in @gfp_mask, this
1061  * function keeps retrying under memory pressure and fails iff @q is dead.
1062  *
1063  * Must be callled with @q->queue_lock held and,
1064  * Returns %NULL on failure, with @q->queue_lock held.
1065  * Returns !%NULL on success, with @q->queue_lock *not held*.
1066  */
1067 static struct request *get_request(struct request_queue *q, int rw_flags,
1068 				   struct bio *bio, gfp_t gfp_mask)
1069 {
1070 	const bool is_sync = rw_is_sync(rw_flags) != 0;
1071 	DEFINE_WAIT(wait);
1072 	struct request_list *rl;
1073 	struct request *rq;
1074 
1075 	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */
1076 retry:
1077 	rq = __get_request(rl, rw_flags, bio, gfp_mask);
1078 	if (rq)
1079 		return rq;
1080 
1081 	if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dying(q))) {
1082 		blk_put_rl(rl);
1083 		return NULL;
1084 	}
1085 
1086 	/* wait on @rl and retry */
1087 	prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
1088 				  TASK_UNINTERRUPTIBLE);
1089 
1090 	trace_block_sleeprq(q, bio, rw_flags & 1);
1091 
1092 	spin_unlock_irq(q->queue_lock);
1093 	io_schedule();
1094 
1095 	/*
1096 	 * After sleeping, we become a "batching" process and will be able
1097 	 * to allocate at least one request, and up to a big batch of them
1098 	 * for a small period time.  See ioc_batching, ioc_set_batching
1099 	 */
1100 	ioc_set_batching(q, current->io_context);
1101 
1102 	spin_lock_irq(q->queue_lock);
1103 	finish_wait(&rl->wait[is_sync], &wait);
1104 
1105 	goto retry;
1106 }
1107 
1108 static struct request *blk_old_get_request(struct request_queue *q, int rw,
1109 		gfp_t gfp_mask)
1110 {
1111 	struct request *rq;
1112 
1113 	BUG_ON(rw != READ && rw != WRITE);
1114 
1115 	/* create ioc upfront */
1116 	create_io_context(gfp_mask, q->node);
1117 
1118 	spin_lock_irq(q->queue_lock);
1119 	rq = get_request(q, rw, NULL, gfp_mask);
1120 	if (!rq)
1121 		spin_unlock_irq(q->queue_lock);
1122 	/* q->queue_lock is unlocked at this point */
1123 
1124 	return rq;
1125 }
1126 
1127 struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
1128 {
1129 	if (q->mq_ops)
1130 		return blk_mq_alloc_request(q, rw, gfp_mask, false);
1131 	else
1132 		return blk_old_get_request(q, rw, gfp_mask);
1133 }
1134 EXPORT_SYMBOL(blk_get_request);
1135 
1136 /**
1137  * blk_make_request - given a bio, allocate a corresponding struct request.
1138  * @q: target request queue
1139  * @bio:  The bio describing the memory mappings that will be submitted for IO.
1140  *        It may be a chained-bio properly constructed by block/bio layer.
1141  * @gfp_mask: gfp flags to be used for memory allocation
1142  *
1143  * blk_make_request is the parallel of generic_make_request for BLOCK_PC
1144  * type commands. Where the struct request needs to be farther initialized by
1145  * the caller. It is passed a &struct bio, which describes the memory info of
1146  * the I/O transfer.
1147  *
1148  * The caller of blk_make_request must make sure that bi_io_vec
1149  * are set to describe the memory buffers. That bio_data_dir() will return
1150  * the needed direction of the request. (And all bio's in the passed bio-chain
1151  * are properly set accordingly)
1152  *
1153  * If called under none-sleepable conditions, mapped bio buffers must not
1154  * need bouncing, by calling the appropriate masked or flagged allocator,
1155  * suitable for the target device. Otherwise the call to blk_queue_bounce will
1156  * BUG.
1157  *
1158  * WARNING: When allocating/cloning a bio-chain, careful consideration should be
1159  * given to how you allocate bios. In particular, you cannot use __GFP_WAIT for
1160  * anything but the first bio in the chain. Otherwise you risk waiting for IO
1161  * completion of a bio that hasn't been submitted yet, thus resulting in a
1162  * deadlock. Alternatively bios should be allocated using bio_kmalloc() instead
1163  * of bio_alloc(), as that avoids the mempool deadlock.
1164  * If possible a big IO should be split into smaller parts when allocation
1165  * fails. Partial allocation should not be an error, or you risk a live-lock.
1166  */
1167 struct request *blk_make_request(struct request_queue *q, struct bio *bio,
1168 				 gfp_t gfp_mask)
1169 {
1170 	struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask);
1171 
1172 	if (unlikely(!rq))
1173 		return ERR_PTR(-ENOMEM);
1174 
1175 	for_each_bio(bio) {
1176 		struct bio *bounce_bio = bio;
1177 		int ret;
1178 
1179 		blk_queue_bounce(q, &bounce_bio);
1180 		ret = blk_rq_append_bio(q, rq, bounce_bio);
1181 		if (unlikely(ret)) {
1182 			blk_put_request(rq);
1183 			return ERR_PTR(ret);
1184 		}
1185 	}
1186 
1187 	return rq;
1188 }
1189 EXPORT_SYMBOL(blk_make_request);
1190 
1191 /**
1192  * blk_requeue_request - put a request back on queue
1193  * @q:		request queue where request should be inserted
1194  * @rq:		request to be inserted
1195  *
1196  * Description:
1197  *    Drivers often keep queueing requests until the hardware cannot accept
1198  *    more, when that condition happens we need to put the request back
1199  *    on the queue. Must be called with queue lock held.
1200  */
1201 void blk_requeue_request(struct request_queue *q, struct request *rq)
1202 {
1203 	blk_delete_timer(rq);
1204 	blk_clear_rq_complete(rq);
1205 	trace_block_rq_requeue(q, rq);
1206 
1207 	if (blk_rq_tagged(rq))
1208 		blk_queue_end_tag(q, rq);
1209 
1210 	BUG_ON(blk_queued_rq(rq));
1211 
1212 	elv_requeue_request(q, rq);
1213 }
1214 EXPORT_SYMBOL(blk_requeue_request);
1215 
1216 static void add_acct_request(struct request_queue *q, struct request *rq,
1217 			     int where)
1218 {
1219 	blk_account_io_start(rq, true);
1220 	__elv_add_request(q, rq, where);
1221 }
1222 
1223 static void part_round_stats_single(int cpu, struct hd_struct *part,
1224 				    unsigned long now)
1225 {
1226 	if (now == part->stamp)
1227 		return;
1228 
1229 	if (part_in_flight(part)) {
1230 		__part_stat_add(cpu, part, time_in_queue,
1231 				part_in_flight(part) * (now - part->stamp));
1232 		__part_stat_add(cpu, part, io_ticks, (now - part->stamp));
1233 	}
1234 	part->stamp = now;
1235 }
1236 
1237 /**
1238  * part_round_stats() - Round off the performance stats on a struct disk_stats.
1239  * @cpu: cpu number for stats access
1240  * @part: target partition
1241  *
1242  * The average IO queue length and utilisation statistics are maintained
1243  * by observing the current state of the queue length and the amount of
1244  * time it has been in this state for.
1245  *
1246  * Normally, that accounting is done on IO completion, but that can result
1247  * in more than a second's worth of IO being accounted for within any one
1248  * second, leading to >100% utilisation.  To deal with that, we call this
1249  * function to do a round-off before returning the results when reading
1250  * /proc/diskstats.  This accounts immediately for all queue usage up to
1251  * the current jiffies and restarts the counters again.
1252  */
1253 void part_round_stats(int cpu, struct hd_struct *part)
1254 {
1255 	unsigned long now = jiffies;
1256 
1257 	if (part->partno)
1258 		part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
1259 	part_round_stats_single(cpu, part, now);
1260 }
1261 EXPORT_SYMBOL_GPL(part_round_stats);
1262 
1263 #ifdef CONFIG_PM_RUNTIME
1264 static void blk_pm_put_request(struct request *rq)
1265 {
1266 	if (rq->q->dev && !(rq->cmd_flags & REQ_PM) && !--rq->q->nr_pending)
1267 		pm_runtime_mark_last_busy(rq->q->dev);
1268 }
1269 #else
1270 static inline void blk_pm_put_request(struct request *rq) {}
1271 #endif
1272 
1273 /*
1274  * queue lock must be held
1275  */
1276 void __blk_put_request(struct request_queue *q, struct request *req)
1277 {
1278 	if (unlikely(!q))
1279 		return;
1280 
1281 	blk_pm_put_request(req);
1282 
1283 	elv_completed_request(q, req);
1284 
1285 	/* this is a bio leak */
1286 	WARN_ON(req->bio != NULL);
1287 
1288 	/*
1289 	 * Request may not have originated from ll_rw_blk. if not,
1290 	 * it didn't come out of our reserved rq pools
1291 	 */
1292 	if (req->cmd_flags & REQ_ALLOCED) {
1293 		unsigned int flags = req->cmd_flags;
1294 		struct request_list *rl = blk_rq_rl(req);
1295 
1296 		BUG_ON(!list_empty(&req->queuelist));
1297 		BUG_ON(!hlist_unhashed(&req->hash));
1298 
1299 		blk_free_request(rl, req);
1300 		freed_request(rl, flags);
1301 		blk_put_rl(rl);
1302 	}
1303 }
1304 EXPORT_SYMBOL_GPL(__blk_put_request);
1305 
1306 void blk_put_request(struct request *req)
1307 {
1308 	struct request_queue *q = req->q;
1309 
1310 	if (q->mq_ops)
1311 		blk_mq_free_request(req);
1312 	else {
1313 		unsigned long flags;
1314 
1315 		spin_lock_irqsave(q->queue_lock, flags);
1316 		__blk_put_request(q, req);
1317 		spin_unlock_irqrestore(q->queue_lock, flags);
1318 	}
1319 }
1320 EXPORT_SYMBOL(blk_put_request);
1321 
1322 /**
1323  * blk_add_request_payload - add a payload to a request
1324  * @rq: request to update
1325  * @page: page backing the payload
1326  * @len: length of the payload.
1327  *
1328  * This allows to later add a payload to an already submitted request by
1329  * a block driver.  The driver needs to take care of freeing the payload
1330  * itself.
1331  *
1332  * Note that this is a quite horrible hack and nothing but handling of
1333  * discard requests should ever use it.
1334  */
1335 void blk_add_request_payload(struct request *rq, struct page *page,
1336 		unsigned int len)
1337 {
1338 	struct bio *bio = rq->bio;
1339 
1340 	bio->bi_io_vec->bv_page = page;
1341 	bio->bi_io_vec->bv_offset = 0;
1342 	bio->bi_io_vec->bv_len = len;
1343 
1344 	bio->bi_iter.bi_size = len;
1345 	bio->bi_vcnt = 1;
1346 	bio->bi_phys_segments = 1;
1347 
1348 	rq->__data_len = rq->resid_len = len;
1349 	rq->nr_phys_segments = 1;
1350 	rq->buffer = bio_data(bio);
1351 }
1352 EXPORT_SYMBOL_GPL(blk_add_request_payload);
1353 
1354 bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
1355 			    struct bio *bio)
1356 {
1357 	const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
1358 
1359 	if (!ll_back_merge_fn(q, req, bio))
1360 		return false;
1361 
1362 	trace_block_bio_backmerge(q, req, bio);
1363 
1364 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1365 		blk_rq_set_mixed_merge(req);
1366 
1367 	req->biotail->bi_next = bio;
1368 	req->biotail = bio;
1369 	req->__data_len += bio->bi_iter.bi_size;
1370 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1371 
1372 	blk_account_io_start(req, false);
1373 	return true;
1374 }
1375 
1376 bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
1377 			     struct bio *bio)
1378 {
1379 	const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
1380 
1381 	if (!ll_front_merge_fn(q, req, bio))
1382 		return false;
1383 
1384 	trace_block_bio_frontmerge(q, req, bio);
1385 
1386 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1387 		blk_rq_set_mixed_merge(req);
1388 
1389 	bio->bi_next = req->bio;
1390 	req->bio = bio;
1391 
1392 	/*
1393 	 * may not be valid. if the low level driver said
1394 	 * it didn't need a bounce buffer then it better
1395 	 * not touch req->buffer either...
1396 	 */
1397 	req->buffer = bio_data(bio);
1398 	req->__sector = bio->bi_iter.bi_sector;
1399 	req->__data_len += bio->bi_iter.bi_size;
1400 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1401 
1402 	blk_account_io_start(req, false);
1403 	return true;
1404 }
1405 
1406 /**
1407  * blk_attempt_plug_merge - try to merge with %current's plugged list
1408  * @q: request_queue new bio is being queued at
1409  * @bio: new bio being queued
1410  * @request_count: out parameter for number of traversed plugged requests
1411  *
1412  * Determine whether @bio being queued on @q can be merged with a request
1413  * on %current's plugged list.  Returns %true if merge was successful,
1414  * otherwise %false.
1415  *
1416  * Plugging coalesces IOs from the same issuer for the same purpose without
1417  * going through @q->queue_lock.  As such it's more of an issuing mechanism
1418  * than scheduling, and the request, while may have elvpriv data, is not
1419  * added on the elevator at this point.  In addition, we don't have
1420  * reliable access to the elevator outside queue lock.  Only check basic
1421  * merging parameters without querying the elevator.
1422  */
1423 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1424 			    unsigned int *request_count)
1425 {
1426 	struct blk_plug *plug;
1427 	struct request *rq;
1428 	bool ret = false;
1429 	struct list_head *plug_list;
1430 
1431 	if (blk_queue_nomerges(q))
1432 		goto out;
1433 
1434 	plug = current->plug;
1435 	if (!plug)
1436 		goto out;
1437 	*request_count = 0;
1438 
1439 	if (q->mq_ops)
1440 		plug_list = &plug->mq_list;
1441 	else
1442 		plug_list = &plug->list;
1443 
1444 	list_for_each_entry_reverse(rq, plug_list, queuelist) {
1445 		int el_ret;
1446 
1447 		if (rq->q == q)
1448 			(*request_count)++;
1449 
1450 		if (rq->q != q || !blk_rq_merge_ok(rq, bio))
1451 			continue;
1452 
1453 		el_ret = blk_try_merge(rq, bio);
1454 		if (el_ret == ELEVATOR_BACK_MERGE) {
1455 			ret = bio_attempt_back_merge(q, rq, bio);
1456 			if (ret)
1457 				break;
1458 		} else if (el_ret == ELEVATOR_FRONT_MERGE) {
1459 			ret = bio_attempt_front_merge(q, rq, bio);
1460 			if (ret)
1461 				break;
1462 		}
1463 	}
1464 out:
1465 	return ret;
1466 }
1467 
1468 void init_request_from_bio(struct request *req, struct bio *bio)
1469 {
1470 	req->cmd_type = REQ_TYPE_FS;
1471 
1472 	req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK;
1473 	if (bio->bi_rw & REQ_RAHEAD)
1474 		req->cmd_flags |= REQ_FAILFAST_MASK;
1475 
1476 	req->errors = 0;
1477 	req->__sector = bio->bi_iter.bi_sector;
1478 	req->ioprio = bio_prio(bio);
1479 	blk_rq_bio_prep(req->q, req, bio);
1480 }
1481 
1482 void blk_queue_bio(struct request_queue *q, struct bio *bio)
1483 {
1484 	const bool sync = !!(bio->bi_rw & REQ_SYNC);
1485 	struct blk_plug *plug;
1486 	int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT;
1487 	struct request *req;
1488 	unsigned int request_count = 0;
1489 
1490 	/*
1491 	 * low level driver can indicate that it wants pages above a
1492 	 * certain limit bounced to low memory (ie for highmem, or even
1493 	 * ISA dma in theory)
1494 	 */
1495 	blk_queue_bounce(q, &bio);
1496 
1497 	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1498 		bio_endio(bio, -EIO);
1499 		return;
1500 	}
1501 
1502 	if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
1503 		spin_lock_irq(q->queue_lock);
1504 		where = ELEVATOR_INSERT_FLUSH;
1505 		goto get_rq;
1506 	}
1507 
1508 	/*
1509 	 * Check if we can merge with the plugged list before grabbing
1510 	 * any locks.
1511 	 */
1512 	if (blk_attempt_plug_merge(q, bio, &request_count))
1513 		return;
1514 
1515 	spin_lock_irq(q->queue_lock);
1516 
1517 	el_ret = elv_merge(q, &req, bio);
1518 	if (el_ret == ELEVATOR_BACK_MERGE) {
1519 		if (bio_attempt_back_merge(q, req, bio)) {
1520 			elv_bio_merged(q, req, bio);
1521 			if (!attempt_back_merge(q, req))
1522 				elv_merged_request(q, req, el_ret);
1523 			goto out_unlock;
1524 		}
1525 	} else if (el_ret == ELEVATOR_FRONT_MERGE) {
1526 		if (bio_attempt_front_merge(q, req, bio)) {
1527 			elv_bio_merged(q, req, bio);
1528 			if (!attempt_front_merge(q, req))
1529 				elv_merged_request(q, req, el_ret);
1530 			goto out_unlock;
1531 		}
1532 	}
1533 
1534 get_rq:
1535 	/*
1536 	 * This sync check and mask will be re-done in init_request_from_bio(),
1537 	 * but we need to set it earlier to expose the sync flag to the
1538 	 * rq allocator and io schedulers.
1539 	 */
1540 	rw_flags = bio_data_dir(bio);
1541 	if (sync)
1542 		rw_flags |= REQ_SYNC;
1543 
1544 	/*
1545 	 * Grab a free request. This is might sleep but can not fail.
1546 	 * Returns with the queue unlocked.
1547 	 */
1548 	req = get_request(q, rw_flags, bio, GFP_NOIO);
1549 	if (unlikely(!req)) {
1550 		bio_endio(bio, -ENODEV);	/* @q is dead */
1551 		goto out_unlock;
1552 	}
1553 
1554 	/*
1555 	 * After dropping the lock and possibly sleeping here, our request
1556 	 * may now be mergeable after it had proven unmergeable (above).
1557 	 * We don't worry about that case for efficiency. It won't happen
1558 	 * often, and the elevators are able to handle it.
1559 	 */
1560 	init_request_from_bio(req, bio);
1561 
1562 	if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
1563 		req->cpu = raw_smp_processor_id();
1564 
1565 	plug = current->plug;
1566 	if (plug) {
1567 		/*
1568 		 * If this is the first request added after a plug, fire
1569 		 * of a plug trace.
1570 		 */
1571 		if (!request_count)
1572 			trace_block_plug(q);
1573 		else {
1574 			if (request_count >= BLK_MAX_REQUEST_COUNT) {
1575 				blk_flush_plug_list(plug, false);
1576 				trace_block_plug(q);
1577 			}
1578 		}
1579 		list_add_tail(&req->queuelist, &plug->list);
1580 		blk_account_io_start(req, true);
1581 	} else {
1582 		spin_lock_irq(q->queue_lock);
1583 		add_acct_request(q, req, where);
1584 		__blk_run_queue(q);
1585 out_unlock:
1586 		spin_unlock_irq(q->queue_lock);
1587 	}
1588 }
1589 EXPORT_SYMBOL_GPL(blk_queue_bio);	/* for device mapper only */
1590 
1591 /*
1592  * If bio->bi_dev is a partition, remap the location
1593  */
1594 static inline void blk_partition_remap(struct bio *bio)
1595 {
1596 	struct block_device *bdev = bio->bi_bdev;
1597 
1598 	if (bio_sectors(bio) && bdev != bdev->bd_contains) {
1599 		struct hd_struct *p = bdev->bd_part;
1600 
1601 		bio->bi_iter.bi_sector += p->start_sect;
1602 		bio->bi_bdev = bdev->bd_contains;
1603 
1604 		trace_block_bio_remap(bdev_get_queue(bio->bi_bdev), bio,
1605 				      bdev->bd_dev,
1606 				      bio->bi_iter.bi_sector - p->start_sect);
1607 	}
1608 }
1609 
1610 static void handle_bad_sector(struct bio *bio)
1611 {
1612 	char b[BDEVNAME_SIZE];
1613 
1614 	printk(KERN_INFO "attempt to access beyond end of device\n");
1615 	printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n",
1616 			bdevname(bio->bi_bdev, b),
1617 			bio->bi_rw,
1618 			(unsigned long long)bio_end_sector(bio),
1619 			(long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9));
1620 
1621 	set_bit(BIO_EOF, &bio->bi_flags);
1622 }
1623 
1624 #ifdef CONFIG_FAIL_MAKE_REQUEST
1625 
1626 static DECLARE_FAULT_ATTR(fail_make_request);
1627 
1628 static int __init setup_fail_make_request(char *str)
1629 {
1630 	return setup_fault_attr(&fail_make_request, str);
1631 }
1632 __setup("fail_make_request=", setup_fail_make_request);
1633 
1634 static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
1635 {
1636 	return part->make_it_fail && should_fail(&fail_make_request, bytes);
1637 }
1638 
1639 static int __init fail_make_request_debugfs(void)
1640 {
1641 	struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
1642 						NULL, &fail_make_request);
1643 
1644 	return IS_ERR(dir) ? PTR_ERR(dir) : 0;
1645 }
1646 
1647 late_initcall(fail_make_request_debugfs);
1648 
1649 #else /* CONFIG_FAIL_MAKE_REQUEST */
1650 
1651 static inline bool should_fail_request(struct hd_struct *part,
1652 					unsigned int bytes)
1653 {
1654 	return false;
1655 }
1656 
1657 #endif /* CONFIG_FAIL_MAKE_REQUEST */
1658 
1659 /*
1660  * Check whether this bio extends beyond the end of the device.
1661  */
1662 static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
1663 {
1664 	sector_t maxsector;
1665 
1666 	if (!nr_sectors)
1667 		return 0;
1668 
1669 	/* Test device or partition size, when known. */
1670 	maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
1671 	if (maxsector) {
1672 		sector_t sector = bio->bi_iter.bi_sector;
1673 
1674 		if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
1675 			/*
1676 			 * This may well happen - the kernel calls bread()
1677 			 * without checking the size of the device, e.g., when
1678 			 * mounting a device.
1679 			 */
1680 			handle_bad_sector(bio);
1681 			return 1;
1682 		}
1683 	}
1684 
1685 	return 0;
1686 }
1687 
1688 static noinline_for_stack bool
1689 generic_make_request_checks(struct bio *bio)
1690 {
1691 	struct request_queue *q;
1692 	int nr_sectors = bio_sectors(bio);
1693 	int err = -EIO;
1694 	char b[BDEVNAME_SIZE];
1695 	struct hd_struct *part;
1696 
1697 	might_sleep();
1698 
1699 	if (bio_check_eod(bio, nr_sectors))
1700 		goto end_io;
1701 
1702 	q = bdev_get_queue(bio->bi_bdev);
1703 	if (unlikely(!q)) {
1704 		printk(KERN_ERR
1705 		       "generic_make_request: Trying to access "
1706 			"nonexistent block-device %s (%Lu)\n",
1707 			bdevname(bio->bi_bdev, b),
1708 			(long long) bio->bi_iter.bi_sector);
1709 		goto end_io;
1710 	}
1711 
1712 	if (likely(bio_is_rw(bio) &&
1713 		   nr_sectors > queue_max_hw_sectors(q))) {
1714 		printk(KERN_ERR "bio too big device %s (%u > %u)\n",
1715 		       bdevname(bio->bi_bdev, b),
1716 		       bio_sectors(bio),
1717 		       queue_max_hw_sectors(q));
1718 		goto end_io;
1719 	}
1720 
1721 	part = bio->bi_bdev->bd_part;
1722 	if (should_fail_request(part, bio->bi_iter.bi_size) ||
1723 	    should_fail_request(&part_to_disk(part)->part0,
1724 				bio->bi_iter.bi_size))
1725 		goto end_io;
1726 
1727 	/*
1728 	 * If this device has partitions, remap block n
1729 	 * of partition p to block n+start(p) of the disk.
1730 	 */
1731 	blk_partition_remap(bio);
1732 
1733 	if (bio_check_eod(bio, nr_sectors))
1734 		goto end_io;
1735 
1736 	/*
1737 	 * Filter flush bio's early so that make_request based
1738 	 * drivers without flush support don't have to worry
1739 	 * about them.
1740 	 */
1741 	if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && !q->flush_flags) {
1742 		bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA);
1743 		if (!nr_sectors) {
1744 			err = 0;
1745 			goto end_io;
1746 		}
1747 	}
1748 
1749 	if ((bio->bi_rw & REQ_DISCARD) &&
1750 	    (!blk_queue_discard(q) ||
1751 	     ((bio->bi_rw & REQ_SECURE) && !blk_queue_secdiscard(q)))) {
1752 		err = -EOPNOTSUPP;
1753 		goto end_io;
1754 	}
1755 
1756 	if (bio->bi_rw & REQ_WRITE_SAME && !bdev_write_same(bio->bi_bdev)) {
1757 		err = -EOPNOTSUPP;
1758 		goto end_io;
1759 	}
1760 
1761 	/*
1762 	 * Various block parts want %current->io_context and lazy ioc
1763 	 * allocation ends up trading a lot of pain for a small amount of
1764 	 * memory.  Just allocate it upfront.  This may fail and block
1765 	 * layer knows how to live with it.
1766 	 */
1767 	create_io_context(GFP_ATOMIC, q->node);
1768 
1769 	if (blk_throtl_bio(q, bio))
1770 		return false;	/* throttled, will be resubmitted later */
1771 
1772 	trace_block_bio_queue(q, bio);
1773 	return true;
1774 
1775 end_io:
1776 	bio_endio(bio, err);
1777 	return false;
1778 }
1779 
1780 /**
1781  * generic_make_request - hand a buffer to its device driver for I/O
1782  * @bio:  The bio describing the location in memory and on the device.
1783  *
1784  * generic_make_request() is used to make I/O requests of block
1785  * devices. It is passed a &struct bio, which describes the I/O that needs
1786  * to be done.
1787  *
1788  * generic_make_request() does not return any status.  The
1789  * success/failure status of the request, along with notification of
1790  * completion, is delivered asynchronously through the bio->bi_end_io
1791  * function described (one day) else where.
1792  *
1793  * The caller of generic_make_request must make sure that bi_io_vec
1794  * are set to describe the memory buffer, and that bi_dev and bi_sector are
1795  * set to describe the device address, and the
1796  * bi_end_io and optionally bi_private are set to describe how
1797  * completion notification should be signaled.
1798  *
1799  * generic_make_request and the drivers it calls may use bi_next if this
1800  * bio happens to be merged with someone else, and may resubmit the bio to
1801  * a lower device by calling into generic_make_request recursively, which
1802  * means the bio should NOT be touched after the call to ->make_request_fn.
1803  */
1804 void generic_make_request(struct bio *bio)
1805 {
1806 	struct bio_list bio_list_on_stack;
1807 
1808 	if (!generic_make_request_checks(bio))
1809 		return;
1810 
1811 	/*
1812 	 * We only want one ->make_request_fn to be active at a time, else
1813 	 * stack usage with stacked devices could be a problem.  So use
1814 	 * current->bio_list to keep a list of requests submited by a
1815 	 * make_request_fn function.  current->bio_list is also used as a
1816 	 * flag to say if generic_make_request is currently active in this
1817 	 * task or not.  If it is NULL, then no make_request is active.  If
1818 	 * it is non-NULL, then a make_request is active, and new requests
1819 	 * should be added at the tail
1820 	 */
1821 	if (current->bio_list) {
1822 		bio_list_add(current->bio_list, bio);
1823 		return;
1824 	}
1825 
1826 	/* following loop may be a bit non-obvious, and so deserves some
1827 	 * explanation.
1828 	 * Before entering the loop, bio->bi_next is NULL (as all callers
1829 	 * ensure that) so we have a list with a single bio.
1830 	 * We pretend that we have just taken it off a longer list, so
1831 	 * we assign bio_list to a pointer to the bio_list_on_stack,
1832 	 * thus initialising the bio_list of new bios to be
1833 	 * added.  ->make_request() may indeed add some more bios
1834 	 * through a recursive call to generic_make_request.  If it
1835 	 * did, we find a non-NULL value in bio_list and re-enter the loop
1836 	 * from the top.  In this case we really did just take the bio
1837 	 * of the top of the list (no pretending) and so remove it from
1838 	 * bio_list, and call into ->make_request() again.
1839 	 */
1840 	BUG_ON(bio->bi_next);
1841 	bio_list_init(&bio_list_on_stack);
1842 	current->bio_list = &bio_list_on_stack;
1843 	do {
1844 		struct request_queue *q = bdev_get_queue(bio->bi_bdev);
1845 
1846 		q->make_request_fn(q, bio);
1847 
1848 		bio = bio_list_pop(current->bio_list);
1849 	} while (bio);
1850 	current->bio_list = NULL; /* deactivate */
1851 }
1852 EXPORT_SYMBOL(generic_make_request);
1853 
1854 /**
1855  * submit_bio - submit a bio to the block device layer for I/O
1856  * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
1857  * @bio: The &struct bio which describes the I/O
1858  *
1859  * submit_bio() is very similar in purpose to generic_make_request(), and
1860  * uses that function to do most of the work. Both are fairly rough
1861  * interfaces; @bio must be presetup and ready for I/O.
1862  *
1863  */
1864 void submit_bio(int rw, struct bio *bio)
1865 {
1866 	bio->bi_rw |= rw;
1867 
1868 	/*
1869 	 * If it's a regular read/write or a barrier with data attached,
1870 	 * go through the normal accounting stuff before submission.
1871 	 */
1872 	if (bio_has_data(bio)) {
1873 		unsigned int count;
1874 
1875 		if (unlikely(rw & REQ_WRITE_SAME))
1876 			count = bdev_logical_block_size(bio->bi_bdev) >> 9;
1877 		else
1878 			count = bio_sectors(bio);
1879 
1880 		if (rw & WRITE) {
1881 			count_vm_events(PGPGOUT, count);
1882 		} else {
1883 			task_io_account_read(bio->bi_iter.bi_size);
1884 			count_vm_events(PGPGIN, count);
1885 		}
1886 
1887 		if (unlikely(block_dump)) {
1888 			char b[BDEVNAME_SIZE];
1889 			printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
1890 			current->comm, task_pid_nr(current),
1891 				(rw & WRITE) ? "WRITE" : "READ",
1892 				(unsigned long long)bio->bi_iter.bi_sector,
1893 				bdevname(bio->bi_bdev, b),
1894 				count);
1895 		}
1896 	}
1897 
1898 	generic_make_request(bio);
1899 }
1900 EXPORT_SYMBOL(submit_bio);
1901 
1902 /**
1903  * blk_rq_check_limits - Helper function to check a request for the queue limit
1904  * @q:  the queue
1905  * @rq: the request being checked
1906  *
1907  * Description:
1908  *    @rq may have been made based on weaker limitations of upper-level queues
1909  *    in request stacking drivers, and it may violate the limitation of @q.
1910  *    Since the block layer and the underlying device driver trust @rq
1911  *    after it is inserted to @q, it should be checked against @q before
1912  *    the insertion using this generic function.
1913  *
1914  *    This function should also be useful for request stacking drivers
1915  *    in some cases below, so export this function.
1916  *    Request stacking drivers like request-based dm may change the queue
1917  *    limits while requests are in the queue (e.g. dm's table swapping).
1918  *    Such request stacking drivers should check those requests agaist
1919  *    the new queue limits again when they dispatch those requests,
1920  *    although such checkings are also done against the old queue limits
1921  *    when submitting requests.
1922  */
1923 int blk_rq_check_limits(struct request_queue *q, struct request *rq)
1924 {
1925 	if (!rq_mergeable(rq))
1926 		return 0;
1927 
1928 	if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, rq->cmd_flags)) {
1929 		printk(KERN_ERR "%s: over max size limit.\n", __func__);
1930 		return -EIO;
1931 	}
1932 
1933 	/*
1934 	 * queue's settings related to segment counting like q->bounce_pfn
1935 	 * may differ from that of other stacking queues.
1936 	 * Recalculate it to check the request correctly on this queue's
1937 	 * limitation.
1938 	 */
1939 	blk_recalc_rq_segments(rq);
1940 	if (rq->nr_phys_segments > queue_max_segments(q)) {
1941 		printk(KERN_ERR "%s: over max segments limit.\n", __func__);
1942 		return -EIO;
1943 	}
1944 
1945 	return 0;
1946 }
1947 EXPORT_SYMBOL_GPL(blk_rq_check_limits);
1948 
1949 /**
1950  * blk_insert_cloned_request - Helper for stacking drivers to submit a request
1951  * @q:  the queue to submit the request
1952  * @rq: the request being queued
1953  */
1954 int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
1955 {
1956 	unsigned long flags;
1957 	int where = ELEVATOR_INSERT_BACK;
1958 
1959 	if (blk_rq_check_limits(q, rq))
1960 		return -EIO;
1961 
1962 	if (rq->rq_disk &&
1963 	    should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
1964 		return -EIO;
1965 
1966 	spin_lock_irqsave(q->queue_lock, flags);
1967 	if (unlikely(blk_queue_dying(q))) {
1968 		spin_unlock_irqrestore(q->queue_lock, flags);
1969 		return -ENODEV;
1970 	}
1971 
1972 	/*
1973 	 * Submitting request must be dequeued before calling this function
1974 	 * because it will be linked to another request_queue
1975 	 */
1976 	BUG_ON(blk_queued_rq(rq));
1977 
1978 	if (rq->cmd_flags & (REQ_FLUSH|REQ_FUA))
1979 		where = ELEVATOR_INSERT_FLUSH;
1980 
1981 	add_acct_request(q, rq, where);
1982 	if (where == ELEVATOR_INSERT_FLUSH)
1983 		__blk_run_queue(q);
1984 	spin_unlock_irqrestore(q->queue_lock, flags);
1985 
1986 	return 0;
1987 }
1988 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
1989 
1990 /**
1991  * blk_rq_err_bytes - determine number of bytes till the next failure boundary
1992  * @rq: request to examine
1993  *
1994  * Description:
1995  *     A request could be merge of IOs which require different failure
1996  *     handling.  This function determines the number of bytes which
1997  *     can be failed from the beginning of the request without
1998  *     crossing into area which need to be retried further.
1999  *
2000  * Return:
2001  *     The number of bytes to fail.
2002  *
2003  * Context:
2004  *     queue_lock must be held.
2005  */
2006 unsigned int blk_rq_err_bytes(const struct request *rq)
2007 {
2008 	unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
2009 	unsigned int bytes = 0;
2010 	struct bio *bio;
2011 
2012 	if (!(rq->cmd_flags & REQ_MIXED_MERGE))
2013 		return blk_rq_bytes(rq);
2014 
2015 	/*
2016 	 * Currently the only 'mixing' which can happen is between
2017 	 * different fastfail types.  We can safely fail portions
2018 	 * which have all the failfast bits that the first one has -
2019 	 * the ones which are at least as eager to fail as the first
2020 	 * one.
2021 	 */
2022 	for (bio = rq->bio; bio; bio = bio->bi_next) {
2023 		if ((bio->bi_rw & ff) != ff)
2024 			break;
2025 		bytes += bio->bi_iter.bi_size;
2026 	}
2027 
2028 	/* this could lead to infinite loop */
2029 	BUG_ON(blk_rq_bytes(rq) && !bytes);
2030 	return bytes;
2031 }
2032 EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
2033 
2034 void blk_account_io_completion(struct request *req, unsigned int bytes)
2035 {
2036 	if (blk_do_io_stat(req)) {
2037 		const int rw = rq_data_dir(req);
2038 		struct hd_struct *part;
2039 		int cpu;
2040 
2041 		cpu = part_stat_lock();
2042 		part = req->part;
2043 		part_stat_add(cpu, part, sectors[rw], bytes >> 9);
2044 		part_stat_unlock();
2045 	}
2046 }
2047 
2048 void blk_account_io_done(struct request *req)
2049 {
2050 	/*
2051 	 * Account IO completion.  flush_rq isn't accounted as a
2052 	 * normal IO on queueing nor completion.  Accounting the
2053 	 * containing request is enough.
2054 	 */
2055 	if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) {
2056 		unsigned long duration = jiffies - req->start_time;
2057 		const int rw = rq_data_dir(req);
2058 		struct hd_struct *part;
2059 		int cpu;
2060 
2061 		cpu = part_stat_lock();
2062 		part = req->part;
2063 
2064 		part_stat_inc(cpu, part, ios[rw]);
2065 		part_stat_add(cpu, part, ticks[rw], duration);
2066 		part_round_stats(cpu, part);
2067 		part_dec_in_flight(part, rw);
2068 
2069 		hd_struct_put(part);
2070 		part_stat_unlock();
2071 	}
2072 }
2073 
2074 #ifdef CONFIG_PM_RUNTIME
2075 /*
2076  * Don't process normal requests when queue is suspended
2077  * or in the process of suspending/resuming
2078  */
2079 static struct request *blk_pm_peek_request(struct request_queue *q,
2080 					   struct request *rq)
2081 {
2082 	if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
2083 	    (q->rpm_status != RPM_ACTIVE && !(rq->cmd_flags & REQ_PM))))
2084 		return NULL;
2085 	else
2086 		return rq;
2087 }
2088 #else
2089 static inline struct request *blk_pm_peek_request(struct request_queue *q,
2090 						  struct request *rq)
2091 {
2092 	return rq;
2093 }
2094 #endif
2095 
2096 void blk_account_io_start(struct request *rq, bool new_io)
2097 {
2098 	struct hd_struct *part;
2099 	int rw = rq_data_dir(rq);
2100 	int cpu;
2101 
2102 	if (!blk_do_io_stat(rq))
2103 		return;
2104 
2105 	cpu = part_stat_lock();
2106 
2107 	if (!new_io) {
2108 		part = rq->part;
2109 		part_stat_inc(cpu, part, merges[rw]);
2110 	} else {
2111 		part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
2112 		if (!hd_struct_try_get(part)) {
2113 			/*
2114 			 * The partition is already being removed,
2115 			 * the request will be accounted on the disk only
2116 			 *
2117 			 * We take a reference on disk->part0 although that
2118 			 * partition will never be deleted, so we can treat
2119 			 * it as any other partition.
2120 			 */
2121 			part = &rq->rq_disk->part0;
2122 			hd_struct_get(part);
2123 		}
2124 		part_round_stats(cpu, part);
2125 		part_inc_in_flight(part, rw);
2126 		rq->part = part;
2127 	}
2128 
2129 	part_stat_unlock();
2130 }
2131 
2132 /**
2133  * blk_peek_request - peek at the top of a request queue
2134  * @q: request queue to peek at
2135  *
2136  * Description:
2137  *     Return the request at the top of @q.  The returned request
2138  *     should be started using blk_start_request() before LLD starts
2139  *     processing it.
2140  *
2141  * Return:
2142  *     Pointer to the request at the top of @q if available.  Null
2143  *     otherwise.
2144  *
2145  * Context:
2146  *     queue_lock must be held.
2147  */
2148 struct request *blk_peek_request(struct request_queue *q)
2149 {
2150 	struct request *rq;
2151 	int ret;
2152 
2153 	while ((rq = __elv_next_request(q)) != NULL) {
2154 
2155 		rq = blk_pm_peek_request(q, rq);
2156 		if (!rq)
2157 			break;
2158 
2159 		if (!(rq->cmd_flags & REQ_STARTED)) {
2160 			/*
2161 			 * This is the first time the device driver
2162 			 * sees this request (possibly after
2163 			 * requeueing).  Notify IO scheduler.
2164 			 */
2165 			if (rq->cmd_flags & REQ_SORTED)
2166 				elv_activate_rq(q, rq);
2167 
2168 			/*
2169 			 * just mark as started even if we don't start
2170 			 * it, a request that has been delayed should
2171 			 * not be passed by new incoming requests
2172 			 */
2173 			rq->cmd_flags |= REQ_STARTED;
2174 			trace_block_rq_issue(q, rq);
2175 		}
2176 
2177 		if (!q->boundary_rq || q->boundary_rq == rq) {
2178 			q->end_sector = rq_end_sector(rq);
2179 			q->boundary_rq = NULL;
2180 		}
2181 
2182 		if (rq->cmd_flags & REQ_DONTPREP)
2183 			break;
2184 
2185 		if (q->dma_drain_size && blk_rq_bytes(rq)) {
2186 			/*
2187 			 * make sure space for the drain appears we
2188 			 * know we can do this because max_hw_segments
2189 			 * has been adjusted to be one fewer than the
2190 			 * device can handle
2191 			 */
2192 			rq->nr_phys_segments++;
2193 		}
2194 
2195 		if (!q->prep_rq_fn)
2196 			break;
2197 
2198 		ret = q->prep_rq_fn(q, rq);
2199 		if (ret == BLKPREP_OK) {
2200 			break;
2201 		} else if (ret == BLKPREP_DEFER) {
2202 			/*
2203 			 * the request may have been (partially) prepped.
2204 			 * we need to keep this request in the front to
2205 			 * avoid resource deadlock.  REQ_STARTED will
2206 			 * prevent other fs requests from passing this one.
2207 			 */
2208 			if (q->dma_drain_size && blk_rq_bytes(rq) &&
2209 			    !(rq->cmd_flags & REQ_DONTPREP)) {
2210 				/*
2211 				 * remove the space for the drain we added
2212 				 * so that we don't add it again
2213 				 */
2214 				--rq->nr_phys_segments;
2215 			}
2216 
2217 			rq = NULL;
2218 			break;
2219 		} else if (ret == BLKPREP_KILL) {
2220 			rq->cmd_flags |= REQ_QUIET;
2221 			/*
2222 			 * Mark this request as started so we don't trigger
2223 			 * any debug logic in the end I/O path.
2224 			 */
2225 			blk_start_request(rq);
2226 			__blk_end_request_all(rq, -EIO);
2227 		} else {
2228 			printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
2229 			break;
2230 		}
2231 	}
2232 
2233 	return rq;
2234 }
2235 EXPORT_SYMBOL(blk_peek_request);
2236 
2237 void blk_dequeue_request(struct request *rq)
2238 {
2239 	struct request_queue *q = rq->q;
2240 
2241 	BUG_ON(list_empty(&rq->queuelist));
2242 	BUG_ON(ELV_ON_HASH(rq));
2243 
2244 	list_del_init(&rq->queuelist);
2245 
2246 	/*
2247 	 * the time frame between a request being removed from the lists
2248 	 * and to it is freed is accounted as io that is in progress at
2249 	 * the driver side.
2250 	 */
2251 	if (blk_account_rq(rq)) {
2252 		q->in_flight[rq_is_sync(rq)]++;
2253 		set_io_start_time_ns(rq);
2254 	}
2255 }
2256 
2257 /**
2258  * blk_start_request - start request processing on the driver
2259  * @req: request to dequeue
2260  *
2261  * Description:
2262  *     Dequeue @req and start timeout timer on it.  This hands off the
2263  *     request to the driver.
2264  *
2265  *     Block internal functions which don't want to start timer should
2266  *     call blk_dequeue_request().
2267  *
2268  * Context:
2269  *     queue_lock must be held.
2270  */
2271 void blk_start_request(struct request *req)
2272 {
2273 	blk_dequeue_request(req);
2274 
2275 	/*
2276 	 * We are now handing the request to the hardware, initialize
2277 	 * resid_len to full count and add the timeout handler.
2278 	 */
2279 	req->resid_len = blk_rq_bytes(req);
2280 	if (unlikely(blk_bidi_rq(req)))
2281 		req->next_rq->resid_len = blk_rq_bytes(req->next_rq);
2282 
2283 	BUG_ON(test_bit(REQ_ATOM_COMPLETE, &req->atomic_flags));
2284 	blk_add_timer(req);
2285 }
2286 EXPORT_SYMBOL(blk_start_request);
2287 
2288 /**
2289  * blk_fetch_request - fetch a request from a request queue
2290  * @q: request queue to fetch a request from
2291  *
2292  * Description:
2293  *     Return the request at the top of @q.  The request is started on
2294  *     return and LLD can start processing it immediately.
2295  *
2296  * Return:
2297  *     Pointer to the request at the top of @q if available.  Null
2298  *     otherwise.
2299  *
2300  * Context:
2301  *     queue_lock must be held.
2302  */
2303 struct request *blk_fetch_request(struct request_queue *q)
2304 {
2305 	struct request *rq;
2306 
2307 	rq = blk_peek_request(q);
2308 	if (rq)
2309 		blk_start_request(rq);
2310 	return rq;
2311 }
2312 EXPORT_SYMBOL(blk_fetch_request);
2313 
2314 /**
2315  * blk_update_request - Special helper function for request stacking drivers
2316  * @req:      the request being processed
2317  * @error:    %0 for success, < %0 for error
2318  * @nr_bytes: number of bytes to complete @req
2319  *
2320  * Description:
2321  *     Ends I/O on a number of bytes attached to @req, but doesn't complete
2322  *     the request structure even if @req doesn't have leftover.
2323  *     If @req has leftover, sets it up for the next range of segments.
2324  *
2325  *     This special helper function is only for request stacking drivers
2326  *     (e.g. request-based dm) so that they can handle partial completion.
2327  *     Actual device drivers should use blk_end_request instead.
2328  *
2329  *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
2330  *     %false return from this function.
2331  *
2332  * Return:
2333  *     %false - this request doesn't have any more data
2334  *     %true  - this request has more data
2335  **/
2336 bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
2337 {
2338 	int total_bytes;
2339 
2340 	if (!req->bio)
2341 		return false;
2342 
2343 	trace_block_rq_complete(req->q, req);
2344 
2345 	/*
2346 	 * For fs requests, rq is just carrier of independent bio's
2347 	 * and each partial completion should be handled separately.
2348 	 * Reset per-request error on each partial completion.
2349 	 *
2350 	 * TODO: tj: This is too subtle.  It would be better to let
2351 	 * low level drivers do what they see fit.
2352 	 */
2353 	if (req->cmd_type == REQ_TYPE_FS)
2354 		req->errors = 0;
2355 
2356 	if (error && req->cmd_type == REQ_TYPE_FS &&
2357 	    !(req->cmd_flags & REQ_QUIET)) {
2358 		char *error_type;
2359 
2360 		switch (error) {
2361 		case -ENOLINK:
2362 			error_type = "recoverable transport";
2363 			break;
2364 		case -EREMOTEIO:
2365 			error_type = "critical target";
2366 			break;
2367 		case -EBADE:
2368 			error_type = "critical nexus";
2369 			break;
2370 		case -ETIMEDOUT:
2371 			error_type = "timeout";
2372 			break;
2373 		case -ENOSPC:
2374 			error_type = "critical space allocation";
2375 			break;
2376 		case -ENODATA:
2377 			error_type = "critical medium";
2378 			break;
2379 		case -EIO:
2380 		default:
2381 			error_type = "I/O";
2382 			break;
2383 		}
2384 		printk_ratelimited(KERN_ERR "end_request: %s error, dev %s, sector %llu\n",
2385 				   error_type, req->rq_disk ?
2386 				   req->rq_disk->disk_name : "?",
2387 				   (unsigned long long)blk_rq_pos(req));
2388 
2389 	}
2390 
2391 	blk_account_io_completion(req, nr_bytes);
2392 
2393 	total_bytes = 0;
2394 	while (req->bio) {
2395 		struct bio *bio = req->bio;
2396 		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
2397 
2398 		if (bio_bytes == bio->bi_iter.bi_size)
2399 			req->bio = bio->bi_next;
2400 
2401 		req_bio_endio(req, bio, bio_bytes, error);
2402 
2403 		total_bytes += bio_bytes;
2404 		nr_bytes -= bio_bytes;
2405 
2406 		if (!nr_bytes)
2407 			break;
2408 	}
2409 
2410 	/*
2411 	 * completely done
2412 	 */
2413 	if (!req->bio) {
2414 		/*
2415 		 * Reset counters so that the request stacking driver
2416 		 * can find how many bytes remain in the request
2417 		 * later.
2418 		 */
2419 		req->__data_len = 0;
2420 		return false;
2421 	}
2422 
2423 	req->__data_len -= total_bytes;
2424 	req->buffer = bio_data(req->bio);
2425 
2426 	/* update sector only for requests with clear definition of sector */
2427 	if (req->cmd_type == REQ_TYPE_FS)
2428 		req->__sector += total_bytes >> 9;
2429 
2430 	/* mixed attributes always follow the first bio */
2431 	if (req->cmd_flags & REQ_MIXED_MERGE) {
2432 		req->cmd_flags &= ~REQ_FAILFAST_MASK;
2433 		req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK;
2434 	}
2435 
2436 	/*
2437 	 * If total number of sectors is less than the first segment
2438 	 * size, something has gone terribly wrong.
2439 	 */
2440 	if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
2441 		blk_dump_rq_flags(req, "request botched");
2442 		req->__data_len = blk_rq_cur_bytes(req);
2443 	}
2444 
2445 	/* recalculate the number of segments */
2446 	blk_recalc_rq_segments(req);
2447 
2448 	return true;
2449 }
2450 EXPORT_SYMBOL_GPL(blk_update_request);
2451 
2452 static bool blk_update_bidi_request(struct request *rq, int error,
2453 				    unsigned int nr_bytes,
2454 				    unsigned int bidi_bytes)
2455 {
2456 	if (blk_update_request(rq, error, nr_bytes))
2457 		return true;
2458 
2459 	/* Bidi request must be completed as a whole */
2460 	if (unlikely(blk_bidi_rq(rq)) &&
2461 	    blk_update_request(rq->next_rq, error, bidi_bytes))
2462 		return true;
2463 
2464 	if (blk_queue_add_random(rq->q))
2465 		add_disk_randomness(rq->rq_disk);
2466 
2467 	return false;
2468 }
2469 
2470 /**
2471  * blk_unprep_request - unprepare a request
2472  * @req:	the request
2473  *
2474  * This function makes a request ready for complete resubmission (or
2475  * completion).  It happens only after all error handling is complete,
2476  * so represents the appropriate moment to deallocate any resources
2477  * that were allocated to the request in the prep_rq_fn.  The queue
2478  * lock is held when calling this.
2479  */
2480 void blk_unprep_request(struct request *req)
2481 {
2482 	struct request_queue *q = req->q;
2483 
2484 	req->cmd_flags &= ~REQ_DONTPREP;
2485 	if (q->unprep_rq_fn)
2486 		q->unprep_rq_fn(q, req);
2487 }
2488 EXPORT_SYMBOL_GPL(blk_unprep_request);
2489 
2490 /*
2491  * queue lock must be held
2492  */
2493 static void blk_finish_request(struct request *req, int error)
2494 {
2495 	if (blk_rq_tagged(req))
2496 		blk_queue_end_tag(req->q, req);
2497 
2498 	BUG_ON(blk_queued_rq(req));
2499 
2500 	if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS)
2501 		laptop_io_completion(&req->q->backing_dev_info);
2502 
2503 	blk_delete_timer(req);
2504 
2505 	if (req->cmd_flags & REQ_DONTPREP)
2506 		blk_unprep_request(req);
2507 
2508 	blk_account_io_done(req);
2509 
2510 	if (req->end_io)
2511 		req->end_io(req, error);
2512 	else {
2513 		if (blk_bidi_rq(req))
2514 			__blk_put_request(req->next_rq->q, req->next_rq);
2515 
2516 		__blk_put_request(req->q, req);
2517 	}
2518 }
2519 
2520 /**
2521  * blk_end_bidi_request - Complete a bidi request
2522  * @rq:         the request to complete
2523  * @error:      %0 for success, < %0 for error
2524  * @nr_bytes:   number of bytes to complete @rq
2525  * @bidi_bytes: number of bytes to complete @rq->next_rq
2526  *
2527  * Description:
2528  *     Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
2529  *     Drivers that supports bidi can safely call this member for any
2530  *     type of request, bidi or uni.  In the later case @bidi_bytes is
2531  *     just ignored.
2532  *
2533  * Return:
2534  *     %false - we are done with this request
2535  *     %true  - still buffers pending for this request
2536  **/
2537 static bool blk_end_bidi_request(struct request *rq, int error,
2538 				 unsigned int nr_bytes, unsigned int bidi_bytes)
2539 {
2540 	struct request_queue *q = rq->q;
2541 	unsigned long flags;
2542 
2543 	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2544 		return true;
2545 
2546 	spin_lock_irqsave(q->queue_lock, flags);
2547 	blk_finish_request(rq, error);
2548 	spin_unlock_irqrestore(q->queue_lock, flags);
2549 
2550 	return false;
2551 }
2552 
2553 /**
2554  * __blk_end_bidi_request - Complete a bidi request with queue lock held
2555  * @rq:         the request to complete
2556  * @error:      %0 for success, < %0 for error
2557  * @nr_bytes:   number of bytes to complete @rq
2558  * @bidi_bytes: number of bytes to complete @rq->next_rq
2559  *
2560  * Description:
2561  *     Identical to blk_end_bidi_request() except that queue lock is
2562  *     assumed to be locked on entry and remains so on return.
2563  *
2564  * Return:
2565  *     %false - we are done with this request
2566  *     %true  - still buffers pending for this request
2567  **/
2568 bool __blk_end_bidi_request(struct request *rq, int error,
2569 				   unsigned int nr_bytes, unsigned int bidi_bytes)
2570 {
2571 	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2572 		return true;
2573 
2574 	blk_finish_request(rq, error);
2575 
2576 	return false;
2577 }
2578 
2579 /**
2580  * blk_end_request - Helper function for drivers to complete the request.
2581  * @rq:       the request being processed
2582  * @error:    %0 for success, < %0 for error
2583  * @nr_bytes: number of bytes to complete
2584  *
2585  * Description:
2586  *     Ends I/O on a number of bytes attached to @rq.
2587  *     If @rq has leftover, sets it up for the next range of segments.
2588  *
2589  * Return:
2590  *     %false - we are done with this request
2591  *     %true  - still buffers pending for this request
2592  **/
2593 bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2594 {
2595 	return blk_end_bidi_request(rq, error, nr_bytes, 0);
2596 }
2597 EXPORT_SYMBOL(blk_end_request);
2598 
2599 /**
2600  * blk_end_request_all - Helper function for drives to finish the request.
2601  * @rq: the request to finish
2602  * @error: %0 for success, < %0 for error
2603  *
2604  * Description:
2605  *     Completely finish @rq.
2606  */
2607 void blk_end_request_all(struct request *rq, int error)
2608 {
2609 	bool pending;
2610 	unsigned int bidi_bytes = 0;
2611 
2612 	if (unlikely(blk_bidi_rq(rq)))
2613 		bidi_bytes = blk_rq_bytes(rq->next_rq);
2614 
2615 	pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2616 	BUG_ON(pending);
2617 }
2618 EXPORT_SYMBOL(blk_end_request_all);
2619 
2620 /**
2621  * blk_end_request_cur - Helper function to finish the current request chunk.
2622  * @rq: the request to finish the current chunk for
2623  * @error: %0 for success, < %0 for error
2624  *
2625  * Description:
2626  *     Complete the current consecutively mapped chunk from @rq.
2627  *
2628  * Return:
2629  *     %false - we are done with this request
2630  *     %true  - still buffers pending for this request
2631  */
2632 bool blk_end_request_cur(struct request *rq, int error)
2633 {
2634 	return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2635 }
2636 EXPORT_SYMBOL(blk_end_request_cur);
2637 
2638 /**
2639  * blk_end_request_err - Finish a request till the next failure boundary.
2640  * @rq: the request to finish till the next failure boundary for
2641  * @error: must be negative errno
2642  *
2643  * Description:
2644  *     Complete @rq till the next failure boundary.
2645  *
2646  * Return:
2647  *     %false - we are done with this request
2648  *     %true  - still buffers pending for this request
2649  */
2650 bool blk_end_request_err(struct request *rq, int error)
2651 {
2652 	WARN_ON(error >= 0);
2653 	return blk_end_request(rq, error, blk_rq_err_bytes(rq));
2654 }
2655 EXPORT_SYMBOL_GPL(blk_end_request_err);
2656 
2657 /**
2658  * __blk_end_request - Helper function for drivers to complete the request.
2659  * @rq:       the request being processed
2660  * @error:    %0 for success, < %0 for error
2661  * @nr_bytes: number of bytes to complete
2662  *
2663  * Description:
2664  *     Must be called with queue lock held unlike blk_end_request().
2665  *
2666  * Return:
2667  *     %false - we are done with this request
2668  *     %true  - still buffers pending for this request
2669  **/
2670 bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2671 {
2672 	return __blk_end_bidi_request(rq, error, nr_bytes, 0);
2673 }
2674 EXPORT_SYMBOL(__blk_end_request);
2675 
2676 /**
2677  * __blk_end_request_all - Helper function for drives to finish the request.
2678  * @rq: the request to finish
2679  * @error: %0 for success, < %0 for error
2680  *
2681  * Description:
2682  *     Completely finish @rq.  Must be called with queue lock held.
2683  */
2684 void __blk_end_request_all(struct request *rq, int error)
2685 {
2686 	bool pending;
2687 	unsigned int bidi_bytes = 0;
2688 
2689 	if (unlikely(blk_bidi_rq(rq)))
2690 		bidi_bytes = blk_rq_bytes(rq->next_rq);
2691 
2692 	pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2693 	BUG_ON(pending);
2694 }
2695 EXPORT_SYMBOL(__blk_end_request_all);
2696 
2697 /**
2698  * __blk_end_request_cur - Helper function to finish the current request chunk.
2699  * @rq: the request to finish the current chunk for
2700  * @error: %0 for success, < %0 for error
2701  *
2702  * Description:
2703  *     Complete the current consecutively mapped chunk from @rq.  Must
2704  *     be called with queue lock held.
2705  *
2706  * Return:
2707  *     %false - we are done with this request
2708  *     %true  - still buffers pending for this request
2709  */
2710 bool __blk_end_request_cur(struct request *rq, int error)
2711 {
2712 	return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2713 }
2714 EXPORT_SYMBOL(__blk_end_request_cur);
2715 
2716 /**
2717  * __blk_end_request_err - Finish a request till the next failure boundary.
2718  * @rq: the request to finish till the next failure boundary for
2719  * @error: must be negative errno
2720  *
2721  * Description:
2722  *     Complete @rq till the next failure boundary.  Must be called
2723  *     with queue lock held.
2724  *
2725  * Return:
2726  *     %false - we are done with this request
2727  *     %true  - still buffers pending for this request
2728  */
2729 bool __blk_end_request_err(struct request *rq, int error)
2730 {
2731 	WARN_ON(error >= 0);
2732 	return __blk_end_request(rq, error, blk_rq_err_bytes(rq));
2733 }
2734 EXPORT_SYMBOL_GPL(__blk_end_request_err);
2735 
2736 void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
2737 		     struct bio *bio)
2738 {
2739 	/* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */
2740 	rq->cmd_flags |= bio->bi_rw & REQ_WRITE;
2741 
2742 	if (bio_has_data(bio)) {
2743 		rq->nr_phys_segments = bio_phys_segments(q, bio);
2744 		rq->buffer = bio_data(bio);
2745 	}
2746 	rq->__data_len = bio->bi_iter.bi_size;
2747 	rq->bio = rq->biotail = bio;
2748 
2749 	if (bio->bi_bdev)
2750 		rq->rq_disk = bio->bi_bdev->bd_disk;
2751 }
2752 
2753 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
2754 /**
2755  * rq_flush_dcache_pages - Helper function to flush all pages in a request
2756  * @rq: the request to be flushed
2757  *
2758  * Description:
2759  *     Flush all pages in @rq.
2760  */
2761 void rq_flush_dcache_pages(struct request *rq)
2762 {
2763 	struct req_iterator iter;
2764 	struct bio_vec bvec;
2765 
2766 	rq_for_each_segment(bvec, rq, iter)
2767 		flush_dcache_page(bvec.bv_page);
2768 }
2769 EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
2770 #endif
2771 
2772 /**
2773  * blk_lld_busy - Check if underlying low-level drivers of a device are busy
2774  * @q : the queue of the device being checked
2775  *
2776  * Description:
2777  *    Check if underlying low-level drivers of a device are busy.
2778  *    If the drivers want to export their busy state, they must set own
2779  *    exporting function using blk_queue_lld_busy() first.
2780  *
2781  *    Basically, this function is used only by request stacking drivers
2782  *    to stop dispatching requests to underlying devices when underlying
2783  *    devices are busy.  This behavior helps more I/O merging on the queue
2784  *    of the request stacking driver and prevents I/O throughput regression
2785  *    on burst I/O load.
2786  *
2787  * Return:
2788  *    0 - Not busy (The request stacking driver should dispatch request)
2789  *    1 - Busy (The request stacking driver should stop dispatching request)
2790  */
2791 int blk_lld_busy(struct request_queue *q)
2792 {
2793 	if (q->lld_busy_fn)
2794 		return q->lld_busy_fn(q);
2795 
2796 	return 0;
2797 }
2798 EXPORT_SYMBOL_GPL(blk_lld_busy);
2799 
2800 /**
2801  * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
2802  * @rq: the clone request to be cleaned up
2803  *
2804  * Description:
2805  *     Free all bios in @rq for a cloned request.
2806  */
2807 void blk_rq_unprep_clone(struct request *rq)
2808 {
2809 	struct bio *bio;
2810 
2811 	while ((bio = rq->bio) != NULL) {
2812 		rq->bio = bio->bi_next;
2813 
2814 		bio_put(bio);
2815 	}
2816 }
2817 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
2818 
2819 /*
2820  * Copy attributes of the original request to the clone request.
2821  * The actual data parts (e.g. ->cmd, ->buffer, ->sense) are not copied.
2822  */
2823 static void __blk_rq_prep_clone(struct request *dst, struct request *src)
2824 {
2825 	dst->cpu = src->cpu;
2826 	dst->cmd_flags = (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE;
2827 	dst->cmd_type = src->cmd_type;
2828 	dst->__sector = blk_rq_pos(src);
2829 	dst->__data_len = blk_rq_bytes(src);
2830 	dst->nr_phys_segments = src->nr_phys_segments;
2831 	dst->ioprio = src->ioprio;
2832 	dst->extra_len = src->extra_len;
2833 }
2834 
2835 /**
2836  * blk_rq_prep_clone - Helper function to setup clone request
2837  * @rq: the request to be setup
2838  * @rq_src: original request to be cloned
2839  * @bs: bio_set that bios for clone are allocated from
2840  * @gfp_mask: memory allocation mask for bio
2841  * @bio_ctr: setup function to be called for each clone bio.
2842  *           Returns %0 for success, non %0 for failure.
2843  * @data: private data to be passed to @bio_ctr
2844  *
2845  * Description:
2846  *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
2847  *     The actual data parts of @rq_src (e.g. ->cmd, ->buffer, ->sense)
2848  *     are not copied, and copying such parts is the caller's responsibility.
2849  *     Also, pages which the original bios are pointing to are not copied
2850  *     and the cloned bios just point same pages.
2851  *     So cloned bios must be completed before original bios, which means
2852  *     the caller must complete @rq before @rq_src.
2853  */
2854 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
2855 		      struct bio_set *bs, gfp_t gfp_mask,
2856 		      int (*bio_ctr)(struct bio *, struct bio *, void *),
2857 		      void *data)
2858 {
2859 	struct bio *bio, *bio_src;
2860 
2861 	if (!bs)
2862 		bs = fs_bio_set;
2863 
2864 	blk_rq_init(NULL, rq);
2865 
2866 	__rq_for_each_bio(bio_src, rq_src) {
2867 		bio = bio_clone_bioset(bio_src, gfp_mask, bs);
2868 		if (!bio)
2869 			goto free_and_out;
2870 
2871 		if (bio_ctr && bio_ctr(bio, bio_src, data))
2872 			goto free_and_out;
2873 
2874 		if (rq->bio) {
2875 			rq->biotail->bi_next = bio;
2876 			rq->biotail = bio;
2877 		} else
2878 			rq->bio = rq->biotail = bio;
2879 	}
2880 
2881 	__blk_rq_prep_clone(rq, rq_src);
2882 
2883 	return 0;
2884 
2885 free_and_out:
2886 	if (bio)
2887 		bio_put(bio);
2888 	blk_rq_unprep_clone(rq);
2889 
2890 	return -ENOMEM;
2891 }
2892 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
2893 
2894 int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
2895 {
2896 	return queue_work(kblockd_workqueue, work);
2897 }
2898 EXPORT_SYMBOL(kblockd_schedule_work);
2899 
2900 int kblockd_schedule_delayed_work(struct request_queue *q,
2901 			struct delayed_work *dwork, unsigned long delay)
2902 {
2903 	return queue_delayed_work(kblockd_workqueue, dwork, delay);
2904 }
2905 EXPORT_SYMBOL(kblockd_schedule_delayed_work);
2906 
2907 #define PLUG_MAGIC	0x91827364
2908 
2909 /**
2910  * blk_start_plug - initialize blk_plug and track it inside the task_struct
2911  * @plug:	The &struct blk_plug that needs to be initialized
2912  *
2913  * Description:
2914  *   Tracking blk_plug inside the task_struct will help with auto-flushing the
2915  *   pending I/O should the task end up blocking between blk_start_plug() and
2916  *   blk_finish_plug(). This is important from a performance perspective, but
2917  *   also ensures that we don't deadlock. For instance, if the task is blocking
2918  *   for a memory allocation, memory reclaim could end up wanting to free a
2919  *   page belonging to that request that is currently residing in our private
2920  *   plug. By flushing the pending I/O when the process goes to sleep, we avoid
2921  *   this kind of deadlock.
2922  */
2923 void blk_start_plug(struct blk_plug *plug)
2924 {
2925 	struct task_struct *tsk = current;
2926 
2927 	plug->magic = PLUG_MAGIC;
2928 	INIT_LIST_HEAD(&plug->list);
2929 	INIT_LIST_HEAD(&plug->mq_list);
2930 	INIT_LIST_HEAD(&plug->cb_list);
2931 
2932 	/*
2933 	 * If this is a nested plug, don't actually assign it. It will be
2934 	 * flushed on its own.
2935 	 */
2936 	if (!tsk->plug) {
2937 		/*
2938 		 * Store ordering should not be needed here, since a potential
2939 		 * preempt will imply a full memory barrier
2940 		 */
2941 		tsk->plug = plug;
2942 	}
2943 }
2944 EXPORT_SYMBOL(blk_start_plug);
2945 
2946 static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
2947 {
2948 	struct request *rqa = container_of(a, struct request, queuelist);
2949 	struct request *rqb = container_of(b, struct request, queuelist);
2950 
2951 	return !(rqa->q < rqb->q ||
2952 		(rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb)));
2953 }
2954 
2955 /*
2956  * If 'from_schedule' is true, then postpone the dispatch of requests
2957  * until a safe kblockd context. We due this to avoid accidental big
2958  * additional stack usage in driver dispatch, in places where the originally
2959  * plugger did not intend it.
2960  */
2961 static void queue_unplugged(struct request_queue *q, unsigned int depth,
2962 			    bool from_schedule)
2963 	__releases(q->queue_lock)
2964 {
2965 	trace_block_unplug(q, depth, !from_schedule);
2966 
2967 	if (from_schedule)
2968 		blk_run_queue_async(q);
2969 	else
2970 		__blk_run_queue(q);
2971 	spin_unlock(q->queue_lock);
2972 }
2973 
2974 static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
2975 {
2976 	LIST_HEAD(callbacks);
2977 
2978 	while (!list_empty(&plug->cb_list)) {
2979 		list_splice_init(&plug->cb_list, &callbacks);
2980 
2981 		while (!list_empty(&callbacks)) {
2982 			struct blk_plug_cb *cb = list_first_entry(&callbacks,
2983 							  struct blk_plug_cb,
2984 							  list);
2985 			list_del(&cb->list);
2986 			cb->callback(cb, from_schedule);
2987 		}
2988 	}
2989 }
2990 
2991 struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
2992 				      int size)
2993 {
2994 	struct blk_plug *plug = current->plug;
2995 	struct blk_plug_cb *cb;
2996 
2997 	if (!plug)
2998 		return NULL;
2999 
3000 	list_for_each_entry(cb, &plug->cb_list, list)
3001 		if (cb->callback == unplug && cb->data == data)
3002 			return cb;
3003 
3004 	/* Not currently on the callback list */
3005 	BUG_ON(size < sizeof(*cb));
3006 	cb = kzalloc(size, GFP_ATOMIC);
3007 	if (cb) {
3008 		cb->data = data;
3009 		cb->callback = unplug;
3010 		list_add(&cb->list, &plug->cb_list);
3011 	}
3012 	return cb;
3013 }
3014 EXPORT_SYMBOL(blk_check_plugged);
3015 
3016 void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
3017 {
3018 	struct request_queue *q;
3019 	unsigned long flags;
3020 	struct request *rq;
3021 	LIST_HEAD(list);
3022 	unsigned int depth;
3023 
3024 	BUG_ON(plug->magic != PLUG_MAGIC);
3025 
3026 	flush_plug_callbacks(plug, from_schedule);
3027 
3028 	if (!list_empty(&plug->mq_list))
3029 		blk_mq_flush_plug_list(plug, from_schedule);
3030 
3031 	if (list_empty(&plug->list))
3032 		return;
3033 
3034 	list_splice_init(&plug->list, &list);
3035 
3036 	list_sort(NULL, &list, plug_rq_cmp);
3037 
3038 	q = NULL;
3039 	depth = 0;
3040 
3041 	/*
3042 	 * Save and disable interrupts here, to avoid doing it for every
3043 	 * queue lock we have to take.
3044 	 */
3045 	local_irq_save(flags);
3046 	while (!list_empty(&list)) {
3047 		rq = list_entry_rq(list.next);
3048 		list_del_init(&rq->queuelist);
3049 		BUG_ON(!rq->q);
3050 		if (rq->q != q) {
3051 			/*
3052 			 * This drops the queue lock
3053 			 */
3054 			if (q)
3055 				queue_unplugged(q, depth, from_schedule);
3056 			q = rq->q;
3057 			depth = 0;
3058 			spin_lock(q->queue_lock);
3059 		}
3060 
3061 		/*
3062 		 * Short-circuit if @q is dead
3063 		 */
3064 		if (unlikely(blk_queue_dying(q))) {
3065 			__blk_end_request_all(rq, -ENODEV);
3066 			continue;
3067 		}
3068 
3069 		/*
3070 		 * rq is already accounted, so use raw insert
3071 		 */
3072 		if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA))
3073 			__elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
3074 		else
3075 			__elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
3076 
3077 		depth++;
3078 	}
3079 
3080 	/*
3081 	 * This drops the queue lock
3082 	 */
3083 	if (q)
3084 		queue_unplugged(q, depth, from_schedule);
3085 
3086 	local_irq_restore(flags);
3087 }
3088 
3089 void blk_finish_plug(struct blk_plug *plug)
3090 {
3091 	blk_flush_plug_list(plug, false);
3092 
3093 	if (plug == current->plug)
3094 		current->plug = NULL;
3095 }
3096 EXPORT_SYMBOL(blk_finish_plug);
3097 
3098 #ifdef CONFIG_PM_RUNTIME
3099 /**
3100  * blk_pm_runtime_init - Block layer runtime PM initialization routine
3101  * @q: the queue of the device
3102  * @dev: the device the queue belongs to
3103  *
3104  * Description:
3105  *    Initialize runtime-PM-related fields for @q and start auto suspend for
3106  *    @dev. Drivers that want to take advantage of request-based runtime PM
3107  *    should call this function after @dev has been initialized, and its
3108  *    request queue @q has been allocated, and runtime PM for it can not happen
3109  *    yet(either due to disabled/forbidden or its usage_count > 0). In most
3110  *    cases, driver should call this function before any I/O has taken place.
3111  *
3112  *    This function takes care of setting up using auto suspend for the device,
3113  *    the autosuspend delay is set to -1 to make runtime suspend impossible
3114  *    until an updated value is either set by user or by driver. Drivers do
3115  *    not need to touch other autosuspend settings.
3116  *
3117  *    The block layer runtime PM is request based, so only works for drivers
3118  *    that use request as their IO unit instead of those directly use bio's.
3119  */
3120 void blk_pm_runtime_init(struct request_queue *q, struct device *dev)
3121 {
3122 	q->dev = dev;
3123 	q->rpm_status = RPM_ACTIVE;
3124 	pm_runtime_set_autosuspend_delay(q->dev, -1);
3125 	pm_runtime_use_autosuspend(q->dev);
3126 }
3127 EXPORT_SYMBOL(blk_pm_runtime_init);
3128 
3129 /**
3130  * blk_pre_runtime_suspend - Pre runtime suspend check
3131  * @q: the queue of the device
3132  *
3133  * Description:
3134  *    This function will check if runtime suspend is allowed for the device
3135  *    by examining if there are any requests pending in the queue. If there
3136  *    are requests pending, the device can not be runtime suspended; otherwise,
3137  *    the queue's status will be updated to SUSPENDING and the driver can
3138  *    proceed to suspend the device.
3139  *
3140  *    For the not allowed case, we mark last busy for the device so that
3141  *    runtime PM core will try to autosuspend it some time later.
3142  *
3143  *    This function should be called near the start of the device's
3144  *    runtime_suspend callback.
3145  *
3146  * Return:
3147  *    0		- OK to runtime suspend the device
3148  *    -EBUSY	- Device should not be runtime suspended
3149  */
3150 int blk_pre_runtime_suspend(struct request_queue *q)
3151 {
3152 	int ret = 0;
3153 
3154 	spin_lock_irq(q->queue_lock);
3155 	if (q->nr_pending) {
3156 		ret = -EBUSY;
3157 		pm_runtime_mark_last_busy(q->dev);
3158 	} else {
3159 		q->rpm_status = RPM_SUSPENDING;
3160 	}
3161 	spin_unlock_irq(q->queue_lock);
3162 	return ret;
3163 }
3164 EXPORT_SYMBOL(blk_pre_runtime_suspend);
3165 
3166 /**
3167  * blk_post_runtime_suspend - Post runtime suspend processing
3168  * @q: the queue of the device
3169  * @err: return value of the device's runtime_suspend function
3170  *
3171  * Description:
3172  *    Update the queue's runtime status according to the return value of the
3173  *    device's runtime suspend function and mark last busy for the device so
3174  *    that PM core will try to auto suspend the device at a later time.
3175  *
3176  *    This function should be called near the end of the device's
3177  *    runtime_suspend callback.
3178  */
3179 void blk_post_runtime_suspend(struct request_queue *q, int err)
3180 {
3181 	spin_lock_irq(q->queue_lock);
3182 	if (!err) {
3183 		q->rpm_status = RPM_SUSPENDED;
3184 	} else {
3185 		q->rpm_status = RPM_ACTIVE;
3186 		pm_runtime_mark_last_busy(q->dev);
3187 	}
3188 	spin_unlock_irq(q->queue_lock);
3189 }
3190 EXPORT_SYMBOL(blk_post_runtime_suspend);
3191 
3192 /**
3193  * blk_pre_runtime_resume - Pre runtime resume processing
3194  * @q: the queue of the device
3195  *
3196  * Description:
3197  *    Update the queue's runtime status to RESUMING in preparation for the
3198  *    runtime resume of the device.
3199  *
3200  *    This function should be called near the start of the device's
3201  *    runtime_resume callback.
3202  */
3203 void blk_pre_runtime_resume(struct request_queue *q)
3204 {
3205 	spin_lock_irq(q->queue_lock);
3206 	q->rpm_status = RPM_RESUMING;
3207 	spin_unlock_irq(q->queue_lock);
3208 }
3209 EXPORT_SYMBOL(blk_pre_runtime_resume);
3210 
3211 /**
3212  * blk_post_runtime_resume - Post runtime resume processing
3213  * @q: the queue of the device
3214  * @err: return value of the device's runtime_resume function
3215  *
3216  * Description:
3217  *    Update the queue's runtime status according to the return value of the
3218  *    device's runtime_resume function. If it is successfully resumed, process
3219  *    the requests that are queued into the device's queue when it is resuming
3220  *    and then mark last busy and initiate autosuspend for it.
3221  *
3222  *    This function should be called near the end of the device's
3223  *    runtime_resume callback.
3224  */
3225 void blk_post_runtime_resume(struct request_queue *q, int err)
3226 {
3227 	spin_lock_irq(q->queue_lock);
3228 	if (!err) {
3229 		q->rpm_status = RPM_ACTIVE;
3230 		__blk_run_queue(q);
3231 		pm_runtime_mark_last_busy(q->dev);
3232 		pm_request_autosuspend(q->dev);
3233 	} else {
3234 		q->rpm_status = RPM_SUSPENDED;
3235 	}
3236 	spin_unlock_irq(q->queue_lock);
3237 }
3238 EXPORT_SYMBOL(blk_post_runtime_resume);
3239 #endif
3240 
3241 int __init blk_dev_init(void)
3242 {
3243 	BUILD_BUG_ON(__REQ_NR_BITS > 8 *
3244 			sizeof(((struct request *)0)->cmd_flags));
3245 
3246 	/* used for unplugging and affects IO latency/throughput - HIGHPRI */
3247 	kblockd_workqueue = alloc_workqueue("kblockd",
3248 					    WQ_MEM_RECLAIM | WQ_HIGHPRI |
3249 					    WQ_POWER_EFFICIENT, 0);
3250 	if (!kblockd_workqueue)
3251 		panic("Failed to create kblockd\n");
3252 
3253 	request_cachep = kmem_cache_create("blkdev_requests",
3254 			sizeof(struct request), 0, SLAB_PANIC, NULL);
3255 
3256 	blk_requestq_cachep = kmem_cache_create("blkdev_queue",
3257 			sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
3258 
3259 	return 0;
3260 }
3261