xref: /openbmc/linux/block/blk-core.c (revision 9d749629)
1 /*
2  * Copyright (C) 1991, 1992 Linus Torvalds
3  * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
4  * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
5  * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
6  * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
7  *	-  July2000
8  * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
9  */
10 
11 /*
12  * This handles all read/write requests to block devices
13  */
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/backing-dev.h>
17 #include <linux/bio.h>
18 #include <linux/blkdev.h>
19 #include <linux/highmem.h>
20 #include <linux/mm.h>
21 #include <linux/kernel_stat.h>
22 #include <linux/string.h>
23 #include <linux/init.h>
24 #include <linux/completion.h>
25 #include <linux/slab.h>
26 #include <linux/swap.h>
27 #include <linux/writeback.h>
28 #include <linux/task_io_accounting_ops.h>
29 #include <linux/fault-inject.h>
30 #include <linux/list_sort.h>
31 #include <linux/delay.h>
32 #include <linux/ratelimit.h>
33 
34 #define CREATE_TRACE_POINTS
35 #include <trace/events/block.h>
36 
37 #include "blk.h"
38 #include "blk-cgroup.h"
39 
40 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
41 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
42 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
43 EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
44 
45 DEFINE_IDA(blk_queue_ida);
46 
47 /*
48  * For the allocated request tables
49  */
50 static struct kmem_cache *request_cachep;
51 
52 /*
53  * For queue allocation
54  */
55 struct kmem_cache *blk_requestq_cachep;
56 
57 /*
58  * Controlling structure to kblockd
59  */
60 static struct workqueue_struct *kblockd_workqueue;
61 
62 static void drive_stat_acct(struct request *rq, int new_io)
63 {
64 	struct hd_struct *part;
65 	int rw = rq_data_dir(rq);
66 	int cpu;
67 
68 	if (!blk_do_io_stat(rq))
69 		return;
70 
71 	cpu = part_stat_lock();
72 
73 	if (!new_io) {
74 		part = rq->part;
75 		part_stat_inc(cpu, part, merges[rw]);
76 	} else {
77 		part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
78 		if (!hd_struct_try_get(part)) {
79 			/*
80 			 * The partition is already being removed,
81 			 * the request will be accounted on the disk only
82 			 *
83 			 * We take a reference on disk->part0 although that
84 			 * partition will never be deleted, so we can treat
85 			 * it as any other partition.
86 			 */
87 			part = &rq->rq_disk->part0;
88 			hd_struct_get(part);
89 		}
90 		part_round_stats(cpu, part);
91 		part_inc_in_flight(part, rw);
92 		rq->part = part;
93 	}
94 
95 	part_stat_unlock();
96 }
97 
98 void blk_queue_congestion_threshold(struct request_queue *q)
99 {
100 	int nr;
101 
102 	nr = q->nr_requests - (q->nr_requests / 8) + 1;
103 	if (nr > q->nr_requests)
104 		nr = q->nr_requests;
105 	q->nr_congestion_on = nr;
106 
107 	nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
108 	if (nr < 1)
109 		nr = 1;
110 	q->nr_congestion_off = nr;
111 }
112 
113 /**
114  * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
115  * @bdev:	device
116  *
117  * Locates the passed device's request queue and returns the address of its
118  * backing_dev_info
119  *
120  * Will return NULL if the request queue cannot be located.
121  */
122 struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
123 {
124 	struct backing_dev_info *ret = NULL;
125 	struct request_queue *q = bdev_get_queue(bdev);
126 
127 	if (q)
128 		ret = &q->backing_dev_info;
129 	return ret;
130 }
131 EXPORT_SYMBOL(blk_get_backing_dev_info);
132 
133 void blk_rq_init(struct request_queue *q, struct request *rq)
134 {
135 	memset(rq, 0, sizeof(*rq));
136 
137 	INIT_LIST_HEAD(&rq->queuelist);
138 	INIT_LIST_HEAD(&rq->timeout_list);
139 	rq->cpu = -1;
140 	rq->q = q;
141 	rq->__sector = (sector_t) -1;
142 	INIT_HLIST_NODE(&rq->hash);
143 	RB_CLEAR_NODE(&rq->rb_node);
144 	rq->cmd = rq->__cmd;
145 	rq->cmd_len = BLK_MAX_CDB;
146 	rq->tag = -1;
147 	rq->ref_count = 1;
148 	rq->start_time = jiffies;
149 	set_start_time_ns(rq);
150 	rq->part = NULL;
151 }
152 EXPORT_SYMBOL(blk_rq_init);
153 
154 static void req_bio_endio(struct request *rq, struct bio *bio,
155 			  unsigned int nbytes, int error)
156 {
157 	if (error)
158 		clear_bit(BIO_UPTODATE, &bio->bi_flags);
159 	else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
160 		error = -EIO;
161 
162 	if (unlikely(nbytes > bio->bi_size)) {
163 		printk(KERN_ERR "%s: want %u bytes done, %u left\n",
164 		       __func__, nbytes, bio->bi_size);
165 		nbytes = bio->bi_size;
166 	}
167 
168 	if (unlikely(rq->cmd_flags & REQ_QUIET))
169 		set_bit(BIO_QUIET, &bio->bi_flags);
170 
171 	bio->bi_size -= nbytes;
172 	bio->bi_sector += (nbytes >> 9);
173 
174 	if (bio_integrity(bio))
175 		bio_integrity_advance(bio, nbytes);
176 
177 	/* don't actually finish bio if it's part of flush sequence */
178 	if (bio->bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ))
179 		bio_endio(bio, error);
180 }
181 
182 void blk_dump_rq_flags(struct request *rq, char *msg)
183 {
184 	int bit;
185 
186 	printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg,
187 		rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
188 		rq->cmd_flags);
189 
190 	printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
191 	       (unsigned long long)blk_rq_pos(rq),
192 	       blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
193 	printk(KERN_INFO "  bio %p, biotail %p, buffer %p, len %u\n",
194 	       rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq));
195 
196 	if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
197 		printk(KERN_INFO "  cdb: ");
198 		for (bit = 0; bit < BLK_MAX_CDB; bit++)
199 			printk("%02x ", rq->cmd[bit]);
200 		printk("\n");
201 	}
202 }
203 EXPORT_SYMBOL(blk_dump_rq_flags);
204 
205 static void blk_delay_work(struct work_struct *work)
206 {
207 	struct request_queue *q;
208 
209 	q = container_of(work, struct request_queue, delay_work.work);
210 	spin_lock_irq(q->queue_lock);
211 	__blk_run_queue(q);
212 	spin_unlock_irq(q->queue_lock);
213 }
214 
215 /**
216  * blk_delay_queue - restart queueing after defined interval
217  * @q:		The &struct request_queue in question
218  * @msecs:	Delay in msecs
219  *
220  * Description:
221  *   Sometimes queueing needs to be postponed for a little while, to allow
222  *   resources to come back. This function will make sure that queueing is
223  *   restarted around the specified time. Queue lock must be held.
224  */
225 void blk_delay_queue(struct request_queue *q, unsigned long msecs)
226 {
227 	if (likely(!blk_queue_dead(q)))
228 		queue_delayed_work(kblockd_workqueue, &q->delay_work,
229 				   msecs_to_jiffies(msecs));
230 }
231 EXPORT_SYMBOL(blk_delay_queue);
232 
233 /**
234  * blk_start_queue - restart a previously stopped queue
235  * @q:    The &struct request_queue in question
236  *
237  * Description:
238  *   blk_start_queue() will clear the stop flag on the queue, and call
239  *   the request_fn for the queue if it was in a stopped state when
240  *   entered. Also see blk_stop_queue(). Queue lock must be held.
241  **/
242 void blk_start_queue(struct request_queue *q)
243 {
244 	WARN_ON(!irqs_disabled());
245 
246 	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
247 	__blk_run_queue(q);
248 }
249 EXPORT_SYMBOL(blk_start_queue);
250 
251 /**
252  * blk_stop_queue - stop a queue
253  * @q:    The &struct request_queue in question
254  *
255  * Description:
256  *   The Linux block layer assumes that a block driver will consume all
257  *   entries on the request queue when the request_fn strategy is called.
258  *   Often this will not happen, because of hardware limitations (queue
259  *   depth settings). If a device driver gets a 'queue full' response,
260  *   or if it simply chooses not to queue more I/O at one point, it can
261  *   call this function to prevent the request_fn from being called until
262  *   the driver has signalled it's ready to go again. This happens by calling
263  *   blk_start_queue() to restart queue operations. Queue lock must be held.
264  **/
265 void blk_stop_queue(struct request_queue *q)
266 {
267 	cancel_delayed_work(&q->delay_work);
268 	queue_flag_set(QUEUE_FLAG_STOPPED, q);
269 }
270 EXPORT_SYMBOL(blk_stop_queue);
271 
272 /**
273  * blk_sync_queue - cancel any pending callbacks on a queue
274  * @q: the queue
275  *
276  * Description:
277  *     The block layer may perform asynchronous callback activity
278  *     on a queue, such as calling the unplug function after a timeout.
279  *     A block device may call blk_sync_queue to ensure that any
280  *     such activity is cancelled, thus allowing it to release resources
281  *     that the callbacks might use. The caller must already have made sure
282  *     that its ->make_request_fn will not re-add plugging prior to calling
283  *     this function.
284  *
285  *     This function does not cancel any asynchronous activity arising
286  *     out of elevator or throttling code. That would require elevaotor_exit()
287  *     and blkcg_exit_queue() to be called with queue lock initialized.
288  *
289  */
290 void blk_sync_queue(struct request_queue *q)
291 {
292 	del_timer_sync(&q->timeout);
293 	cancel_delayed_work_sync(&q->delay_work);
294 }
295 EXPORT_SYMBOL(blk_sync_queue);
296 
297 /**
298  * __blk_run_queue_uncond - run a queue whether or not it has been stopped
299  * @q:	The queue to run
300  *
301  * Description:
302  *    Invoke request handling on a queue if there are any pending requests.
303  *    May be used to restart request handling after a request has completed.
304  *    This variant runs the queue whether or not the queue has been
305  *    stopped. Must be called with the queue lock held and interrupts
306  *    disabled. See also @blk_run_queue.
307  */
308 inline void __blk_run_queue_uncond(struct request_queue *q)
309 {
310 	if (unlikely(blk_queue_dead(q)))
311 		return;
312 
313 	/*
314 	 * Some request_fn implementations, e.g. scsi_request_fn(), unlock
315 	 * the queue lock internally. As a result multiple threads may be
316 	 * running such a request function concurrently. Keep track of the
317 	 * number of active request_fn invocations such that blk_drain_queue()
318 	 * can wait until all these request_fn calls have finished.
319 	 */
320 	q->request_fn_active++;
321 	q->request_fn(q);
322 	q->request_fn_active--;
323 }
324 
325 /**
326  * __blk_run_queue - run a single device queue
327  * @q:	The queue to run
328  *
329  * Description:
330  *    See @blk_run_queue. This variant must be called with the queue lock
331  *    held and interrupts disabled.
332  */
333 void __blk_run_queue(struct request_queue *q)
334 {
335 	if (unlikely(blk_queue_stopped(q)))
336 		return;
337 
338 	__blk_run_queue_uncond(q);
339 }
340 EXPORT_SYMBOL(__blk_run_queue);
341 
342 /**
343  * blk_run_queue_async - run a single device queue in workqueue context
344  * @q:	The queue to run
345  *
346  * Description:
347  *    Tells kblockd to perform the equivalent of @blk_run_queue on behalf
348  *    of us. The caller must hold the queue lock.
349  */
350 void blk_run_queue_async(struct request_queue *q)
351 {
352 	if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q)))
353 		mod_delayed_work(kblockd_workqueue, &q->delay_work, 0);
354 }
355 EXPORT_SYMBOL(blk_run_queue_async);
356 
357 /**
358  * blk_run_queue - run a single device queue
359  * @q: The queue to run
360  *
361  * Description:
362  *    Invoke request handling on this queue, if it has pending work to do.
363  *    May be used to restart queueing when a request has completed.
364  */
365 void blk_run_queue(struct request_queue *q)
366 {
367 	unsigned long flags;
368 
369 	spin_lock_irqsave(q->queue_lock, flags);
370 	__blk_run_queue(q);
371 	spin_unlock_irqrestore(q->queue_lock, flags);
372 }
373 EXPORT_SYMBOL(blk_run_queue);
374 
375 void blk_put_queue(struct request_queue *q)
376 {
377 	kobject_put(&q->kobj);
378 }
379 EXPORT_SYMBOL(blk_put_queue);
380 
381 /**
382  * __blk_drain_queue - drain requests from request_queue
383  * @q: queue to drain
384  * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV
385  *
386  * Drain requests from @q.  If @drain_all is set, all requests are drained.
387  * If not, only ELVPRIV requests are drained.  The caller is responsible
388  * for ensuring that no new requests which need to be drained are queued.
389  */
390 static void __blk_drain_queue(struct request_queue *q, bool drain_all)
391 	__releases(q->queue_lock)
392 	__acquires(q->queue_lock)
393 {
394 	int i;
395 
396 	lockdep_assert_held(q->queue_lock);
397 
398 	while (true) {
399 		bool drain = false;
400 
401 		/*
402 		 * The caller might be trying to drain @q before its
403 		 * elevator is initialized.
404 		 */
405 		if (q->elevator)
406 			elv_drain_elevator(q);
407 
408 		blkcg_drain_queue(q);
409 
410 		/*
411 		 * This function might be called on a queue which failed
412 		 * driver init after queue creation or is not yet fully
413 		 * active yet.  Some drivers (e.g. fd and loop) get unhappy
414 		 * in such cases.  Kick queue iff dispatch queue has
415 		 * something on it and @q has request_fn set.
416 		 */
417 		if (!list_empty(&q->queue_head) && q->request_fn)
418 			__blk_run_queue(q);
419 
420 		drain |= q->nr_rqs_elvpriv;
421 		drain |= q->request_fn_active;
422 
423 		/*
424 		 * Unfortunately, requests are queued at and tracked from
425 		 * multiple places and there's no single counter which can
426 		 * be drained.  Check all the queues and counters.
427 		 */
428 		if (drain_all) {
429 			drain |= !list_empty(&q->queue_head);
430 			for (i = 0; i < 2; i++) {
431 				drain |= q->nr_rqs[i];
432 				drain |= q->in_flight[i];
433 				drain |= !list_empty(&q->flush_queue[i]);
434 			}
435 		}
436 
437 		if (!drain)
438 			break;
439 
440 		spin_unlock_irq(q->queue_lock);
441 
442 		msleep(10);
443 
444 		spin_lock_irq(q->queue_lock);
445 	}
446 
447 	/*
448 	 * With queue marked dead, any woken up waiter will fail the
449 	 * allocation path, so the wakeup chaining is lost and we're
450 	 * left with hung waiters. We need to wake up those waiters.
451 	 */
452 	if (q->request_fn) {
453 		struct request_list *rl;
454 
455 		blk_queue_for_each_rl(rl, q)
456 			for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
457 				wake_up_all(&rl->wait[i]);
458 	}
459 }
460 
461 /**
462  * blk_queue_bypass_start - enter queue bypass mode
463  * @q: queue of interest
464  *
465  * In bypass mode, only the dispatch FIFO queue of @q is used.  This
466  * function makes @q enter bypass mode and drains all requests which were
467  * throttled or issued before.  On return, it's guaranteed that no request
468  * is being throttled or has ELVPRIV set and blk_queue_bypass() %true
469  * inside queue or RCU read lock.
470  */
471 void blk_queue_bypass_start(struct request_queue *q)
472 {
473 	bool drain;
474 
475 	spin_lock_irq(q->queue_lock);
476 	drain = !q->bypass_depth++;
477 	queue_flag_set(QUEUE_FLAG_BYPASS, q);
478 	spin_unlock_irq(q->queue_lock);
479 
480 	if (drain) {
481 		spin_lock_irq(q->queue_lock);
482 		__blk_drain_queue(q, false);
483 		spin_unlock_irq(q->queue_lock);
484 
485 		/* ensure blk_queue_bypass() is %true inside RCU read lock */
486 		synchronize_rcu();
487 	}
488 }
489 EXPORT_SYMBOL_GPL(blk_queue_bypass_start);
490 
491 /**
492  * blk_queue_bypass_end - leave queue bypass mode
493  * @q: queue of interest
494  *
495  * Leave bypass mode and restore the normal queueing behavior.
496  */
497 void blk_queue_bypass_end(struct request_queue *q)
498 {
499 	spin_lock_irq(q->queue_lock);
500 	if (!--q->bypass_depth)
501 		queue_flag_clear(QUEUE_FLAG_BYPASS, q);
502 	WARN_ON_ONCE(q->bypass_depth < 0);
503 	spin_unlock_irq(q->queue_lock);
504 }
505 EXPORT_SYMBOL_GPL(blk_queue_bypass_end);
506 
507 /**
508  * blk_cleanup_queue - shutdown a request queue
509  * @q: request queue to shutdown
510  *
511  * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
512  * put it.  All future requests will be failed immediately with -ENODEV.
513  */
514 void blk_cleanup_queue(struct request_queue *q)
515 {
516 	spinlock_t *lock = q->queue_lock;
517 
518 	/* mark @q DYING, no new request or merges will be allowed afterwards */
519 	mutex_lock(&q->sysfs_lock);
520 	queue_flag_set_unlocked(QUEUE_FLAG_DYING, q);
521 	spin_lock_irq(lock);
522 
523 	/*
524 	 * A dying queue is permanently in bypass mode till released.  Note
525 	 * that, unlike blk_queue_bypass_start(), we aren't performing
526 	 * synchronize_rcu() after entering bypass mode to avoid the delay
527 	 * as some drivers create and destroy a lot of queues while
528 	 * probing.  This is still safe because blk_release_queue() will be
529 	 * called only after the queue refcnt drops to zero and nothing,
530 	 * RCU or not, would be traversing the queue by then.
531 	 */
532 	q->bypass_depth++;
533 	queue_flag_set(QUEUE_FLAG_BYPASS, q);
534 
535 	queue_flag_set(QUEUE_FLAG_NOMERGES, q);
536 	queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
537 	queue_flag_set(QUEUE_FLAG_DYING, q);
538 	spin_unlock_irq(lock);
539 	mutex_unlock(&q->sysfs_lock);
540 
541 	/*
542 	 * Drain all requests queued before DYING marking. Set DEAD flag to
543 	 * prevent that q->request_fn() gets invoked after draining finished.
544 	 */
545 	spin_lock_irq(lock);
546 	__blk_drain_queue(q, true);
547 	queue_flag_set(QUEUE_FLAG_DEAD, q);
548 	spin_unlock_irq(lock);
549 
550 	/* @q won't process any more request, flush async actions */
551 	del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer);
552 	blk_sync_queue(q);
553 
554 	spin_lock_irq(lock);
555 	if (q->queue_lock != &q->__queue_lock)
556 		q->queue_lock = &q->__queue_lock;
557 	spin_unlock_irq(lock);
558 
559 	/* @q is and will stay empty, shutdown and put */
560 	blk_put_queue(q);
561 }
562 EXPORT_SYMBOL(blk_cleanup_queue);
563 
564 int blk_init_rl(struct request_list *rl, struct request_queue *q,
565 		gfp_t gfp_mask)
566 {
567 	if (unlikely(rl->rq_pool))
568 		return 0;
569 
570 	rl->q = q;
571 	rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
572 	rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
573 	init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
574 	init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
575 
576 	rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
577 					  mempool_free_slab, request_cachep,
578 					  gfp_mask, q->node);
579 	if (!rl->rq_pool)
580 		return -ENOMEM;
581 
582 	return 0;
583 }
584 
585 void blk_exit_rl(struct request_list *rl)
586 {
587 	if (rl->rq_pool)
588 		mempool_destroy(rl->rq_pool);
589 }
590 
591 struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
592 {
593 	return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE);
594 }
595 EXPORT_SYMBOL(blk_alloc_queue);
596 
597 struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
598 {
599 	struct request_queue *q;
600 	int err;
601 
602 	q = kmem_cache_alloc_node(blk_requestq_cachep,
603 				gfp_mask | __GFP_ZERO, node_id);
604 	if (!q)
605 		return NULL;
606 
607 	q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
608 	if (q->id < 0)
609 		goto fail_q;
610 
611 	q->backing_dev_info.ra_pages =
612 			(VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
613 	q->backing_dev_info.state = 0;
614 	q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY;
615 	q->backing_dev_info.name = "block";
616 	q->node = node_id;
617 
618 	err = bdi_init(&q->backing_dev_info);
619 	if (err)
620 		goto fail_id;
621 
622 	setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
623 		    laptop_mode_timer_fn, (unsigned long) q);
624 	setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
625 	INIT_LIST_HEAD(&q->queue_head);
626 	INIT_LIST_HEAD(&q->timeout_list);
627 	INIT_LIST_HEAD(&q->icq_list);
628 #ifdef CONFIG_BLK_CGROUP
629 	INIT_LIST_HEAD(&q->blkg_list);
630 #endif
631 	INIT_LIST_HEAD(&q->flush_queue[0]);
632 	INIT_LIST_HEAD(&q->flush_queue[1]);
633 	INIT_LIST_HEAD(&q->flush_data_in_flight);
634 	INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
635 
636 	kobject_init(&q->kobj, &blk_queue_ktype);
637 
638 	mutex_init(&q->sysfs_lock);
639 	spin_lock_init(&q->__queue_lock);
640 
641 	/*
642 	 * By default initialize queue_lock to internal lock and driver can
643 	 * override it later if need be.
644 	 */
645 	q->queue_lock = &q->__queue_lock;
646 
647 	/*
648 	 * A queue starts its life with bypass turned on to avoid
649 	 * unnecessary bypass on/off overhead and nasty surprises during
650 	 * init.  The initial bypass will be finished when the queue is
651 	 * registered by blk_register_queue().
652 	 */
653 	q->bypass_depth = 1;
654 	__set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
655 
656 	if (blkcg_init_queue(q))
657 		goto fail_id;
658 
659 	return q;
660 
661 fail_id:
662 	ida_simple_remove(&blk_queue_ida, q->id);
663 fail_q:
664 	kmem_cache_free(blk_requestq_cachep, q);
665 	return NULL;
666 }
667 EXPORT_SYMBOL(blk_alloc_queue_node);
668 
669 /**
670  * blk_init_queue  - prepare a request queue for use with a block device
671  * @rfn:  The function to be called to process requests that have been
672  *        placed on the queue.
673  * @lock: Request queue spin lock
674  *
675  * Description:
676  *    If a block device wishes to use the standard request handling procedures,
677  *    which sorts requests and coalesces adjacent requests, then it must
678  *    call blk_init_queue().  The function @rfn will be called when there
679  *    are requests on the queue that need to be processed.  If the device
680  *    supports plugging, then @rfn may not be called immediately when requests
681  *    are available on the queue, but may be called at some time later instead.
682  *    Plugged queues are generally unplugged when a buffer belonging to one
683  *    of the requests on the queue is needed, or due to memory pressure.
684  *
685  *    @rfn is not required, or even expected, to remove all requests off the
686  *    queue, but only as many as it can handle at a time.  If it does leave
687  *    requests on the queue, it is responsible for arranging that the requests
688  *    get dealt with eventually.
689  *
690  *    The queue spin lock must be held while manipulating the requests on the
691  *    request queue; this lock will be taken also from interrupt context, so irq
692  *    disabling is needed for it.
693  *
694  *    Function returns a pointer to the initialized request queue, or %NULL if
695  *    it didn't succeed.
696  *
697  * Note:
698  *    blk_init_queue() must be paired with a blk_cleanup_queue() call
699  *    when the block device is deactivated (such as at module unload).
700  **/
701 
702 struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
703 {
704 	return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
705 }
706 EXPORT_SYMBOL(blk_init_queue);
707 
708 struct request_queue *
709 blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
710 {
711 	struct request_queue *uninit_q, *q;
712 
713 	uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id);
714 	if (!uninit_q)
715 		return NULL;
716 
717 	q = blk_init_allocated_queue(uninit_q, rfn, lock);
718 	if (!q)
719 		blk_cleanup_queue(uninit_q);
720 
721 	return q;
722 }
723 EXPORT_SYMBOL(blk_init_queue_node);
724 
725 struct request_queue *
726 blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
727 			 spinlock_t *lock)
728 {
729 	if (!q)
730 		return NULL;
731 
732 	if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
733 		return NULL;
734 
735 	q->request_fn		= rfn;
736 	q->prep_rq_fn		= NULL;
737 	q->unprep_rq_fn		= NULL;
738 	q->queue_flags		|= QUEUE_FLAG_DEFAULT;
739 
740 	/* Override internal queue lock with supplied lock pointer */
741 	if (lock)
742 		q->queue_lock		= lock;
743 
744 	/*
745 	 * This also sets hw/phys segments, boundary and size
746 	 */
747 	blk_queue_make_request(q, blk_queue_bio);
748 
749 	q->sg_reserved_size = INT_MAX;
750 
751 	/* init elevator */
752 	if (elevator_init(q, NULL))
753 		return NULL;
754 	return q;
755 }
756 EXPORT_SYMBOL(blk_init_allocated_queue);
757 
758 bool blk_get_queue(struct request_queue *q)
759 {
760 	if (likely(!blk_queue_dying(q))) {
761 		__blk_get_queue(q);
762 		return true;
763 	}
764 
765 	return false;
766 }
767 EXPORT_SYMBOL(blk_get_queue);
768 
769 static inline void blk_free_request(struct request_list *rl, struct request *rq)
770 {
771 	if (rq->cmd_flags & REQ_ELVPRIV) {
772 		elv_put_request(rl->q, rq);
773 		if (rq->elv.icq)
774 			put_io_context(rq->elv.icq->ioc);
775 	}
776 
777 	mempool_free(rq, rl->rq_pool);
778 }
779 
780 /*
781  * ioc_batching returns true if the ioc is a valid batching request and
782  * should be given priority access to a request.
783  */
784 static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
785 {
786 	if (!ioc)
787 		return 0;
788 
789 	/*
790 	 * Make sure the process is able to allocate at least 1 request
791 	 * even if the batch times out, otherwise we could theoretically
792 	 * lose wakeups.
793 	 */
794 	return ioc->nr_batch_requests == q->nr_batching ||
795 		(ioc->nr_batch_requests > 0
796 		&& time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
797 }
798 
799 /*
800  * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
801  * will cause the process to be a "batcher" on all queues in the system. This
802  * is the behaviour we want though - once it gets a wakeup it should be given
803  * a nice run.
804  */
805 static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
806 {
807 	if (!ioc || ioc_batching(q, ioc))
808 		return;
809 
810 	ioc->nr_batch_requests = q->nr_batching;
811 	ioc->last_waited = jiffies;
812 }
813 
814 static void __freed_request(struct request_list *rl, int sync)
815 {
816 	struct request_queue *q = rl->q;
817 
818 	/*
819 	 * bdi isn't aware of blkcg yet.  As all async IOs end up root
820 	 * blkcg anyway, just use root blkcg state.
821 	 */
822 	if (rl == &q->root_rl &&
823 	    rl->count[sync] < queue_congestion_off_threshold(q))
824 		blk_clear_queue_congested(q, sync);
825 
826 	if (rl->count[sync] + 1 <= q->nr_requests) {
827 		if (waitqueue_active(&rl->wait[sync]))
828 			wake_up(&rl->wait[sync]);
829 
830 		blk_clear_rl_full(rl, sync);
831 	}
832 }
833 
834 /*
835  * A request has just been released.  Account for it, update the full and
836  * congestion status, wake up any waiters.   Called under q->queue_lock.
837  */
838 static void freed_request(struct request_list *rl, unsigned int flags)
839 {
840 	struct request_queue *q = rl->q;
841 	int sync = rw_is_sync(flags);
842 
843 	q->nr_rqs[sync]--;
844 	rl->count[sync]--;
845 	if (flags & REQ_ELVPRIV)
846 		q->nr_rqs_elvpriv--;
847 
848 	__freed_request(rl, sync);
849 
850 	if (unlikely(rl->starved[sync ^ 1]))
851 		__freed_request(rl, sync ^ 1);
852 }
853 
854 /*
855  * Determine if elevator data should be initialized when allocating the
856  * request associated with @bio.
857  */
858 static bool blk_rq_should_init_elevator(struct bio *bio)
859 {
860 	if (!bio)
861 		return true;
862 
863 	/*
864 	 * Flush requests do not use the elevator so skip initialization.
865 	 * This allows a request to share the flush and elevator data.
866 	 */
867 	if (bio->bi_rw & (REQ_FLUSH | REQ_FUA))
868 		return false;
869 
870 	return true;
871 }
872 
873 /**
874  * rq_ioc - determine io_context for request allocation
875  * @bio: request being allocated is for this bio (can be %NULL)
876  *
877  * Determine io_context to use for request allocation for @bio.  May return
878  * %NULL if %current->io_context doesn't exist.
879  */
880 static struct io_context *rq_ioc(struct bio *bio)
881 {
882 #ifdef CONFIG_BLK_CGROUP
883 	if (bio && bio->bi_ioc)
884 		return bio->bi_ioc;
885 #endif
886 	return current->io_context;
887 }
888 
889 /**
890  * __get_request - get a free request
891  * @rl: request list to allocate from
892  * @rw_flags: RW and SYNC flags
893  * @bio: bio to allocate request for (can be %NULL)
894  * @gfp_mask: allocation mask
895  *
896  * Get a free request from @q.  This function may fail under memory
897  * pressure or if @q is dead.
898  *
899  * Must be callled with @q->queue_lock held and,
900  * Returns %NULL on failure, with @q->queue_lock held.
901  * Returns !%NULL on success, with @q->queue_lock *not held*.
902  */
903 static struct request *__get_request(struct request_list *rl, int rw_flags,
904 				     struct bio *bio, gfp_t gfp_mask)
905 {
906 	struct request_queue *q = rl->q;
907 	struct request *rq;
908 	struct elevator_type *et = q->elevator->type;
909 	struct io_context *ioc = rq_ioc(bio);
910 	struct io_cq *icq = NULL;
911 	const bool is_sync = rw_is_sync(rw_flags) != 0;
912 	int may_queue;
913 
914 	if (unlikely(blk_queue_dying(q)))
915 		return NULL;
916 
917 	may_queue = elv_may_queue(q, rw_flags);
918 	if (may_queue == ELV_MQUEUE_NO)
919 		goto rq_starved;
920 
921 	if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
922 		if (rl->count[is_sync]+1 >= q->nr_requests) {
923 			/*
924 			 * The queue will fill after this allocation, so set
925 			 * it as full, and mark this process as "batching".
926 			 * This process will be allowed to complete a batch of
927 			 * requests, others will be blocked.
928 			 */
929 			if (!blk_rl_full(rl, is_sync)) {
930 				ioc_set_batching(q, ioc);
931 				blk_set_rl_full(rl, is_sync);
932 			} else {
933 				if (may_queue != ELV_MQUEUE_MUST
934 						&& !ioc_batching(q, ioc)) {
935 					/*
936 					 * The queue is full and the allocating
937 					 * process is not a "batcher", and not
938 					 * exempted by the IO scheduler
939 					 */
940 					return NULL;
941 				}
942 			}
943 		}
944 		/*
945 		 * bdi isn't aware of blkcg yet.  As all async IOs end up
946 		 * root blkcg anyway, just use root blkcg state.
947 		 */
948 		if (rl == &q->root_rl)
949 			blk_set_queue_congested(q, is_sync);
950 	}
951 
952 	/*
953 	 * Only allow batching queuers to allocate up to 50% over the defined
954 	 * limit of requests, otherwise we could have thousands of requests
955 	 * allocated with any setting of ->nr_requests
956 	 */
957 	if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
958 		return NULL;
959 
960 	q->nr_rqs[is_sync]++;
961 	rl->count[is_sync]++;
962 	rl->starved[is_sync] = 0;
963 
964 	/*
965 	 * Decide whether the new request will be managed by elevator.  If
966 	 * so, mark @rw_flags and increment elvpriv.  Non-zero elvpriv will
967 	 * prevent the current elevator from being destroyed until the new
968 	 * request is freed.  This guarantees icq's won't be destroyed and
969 	 * makes creating new ones safe.
970 	 *
971 	 * Also, lookup icq while holding queue_lock.  If it doesn't exist,
972 	 * it will be created after releasing queue_lock.
973 	 */
974 	if (blk_rq_should_init_elevator(bio) && !blk_queue_bypass(q)) {
975 		rw_flags |= REQ_ELVPRIV;
976 		q->nr_rqs_elvpriv++;
977 		if (et->icq_cache && ioc)
978 			icq = ioc_lookup_icq(ioc, q);
979 	}
980 
981 	if (blk_queue_io_stat(q))
982 		rw_flags |= REQ_IO_STAT;
983 	spin_unlock_irq(q->queue_lock);
984 
985 	/* allocate and init request */
986 	rq = mempool_alloc(rl->rq_pool, gfp_mask);
987 	if (!rq)
988 		goto fail_alloc;
989 
990 	blk_rq_init(q, rq);
991 	blk_rq_set_rl(rq, rl);
992 	rq->cmd_flags = rw_flags | REQ_ALLOCED;
993 
994 	/* init elvpriv */
995 	if (rw_flags & REQ_ELVPRIV) {
996 		if (unlikely(et->icq_cache && !icq)) {
997 			if (ioc)
998 				icq = ioc_create_icq(ioc, q, gfp_mask);
999 			if (!icq)
1000 				goto fail_elvpriv;
1001 		}
1002 
1003 		rq->elv.icq = icq;
1004 		if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
1005 			goto fail_elvpriv;
1006 
1007 		/* @rq->elv.icq holds io_context until @rq is freed */
1008 		if (icq)
1009 			get_io_context(icq->ioc);
1010 	}
1011 out:
1012 	/*
1013 	 * ioc may be NULL here, and ioc_batching will be false. That's
1014 	 * OK, if the queue is under the request limit then requests need
1015 	 * not count toward the nr_batch_requests limit. There will always
1016 	 * be some limit enforced by BLK_BATCH_TIME.
1017 	 */
1018 	if (ioc_batching(q, ioc))
1019 		ioc->nr_batch_requests--;
1020 
1021 	trace_block_getrq(q, bio, rw_flags & 1);
1022 	return rq;
1023 
1024 fail_elvpriv:
1025 	/*
1026 	 * elvpriv init failed.  ioc, icq and elvpriv aren't mempool backed
1027 	 * and may fail indefinitely under memory pressure and thus
1028 	 * shouldn't stall IO.  Treat this request as !elvpriv.  This will
1029 	 * disturb iosched and blkcg but weird is bettern than dead.
1030 	 */
1031 	printk_ratelimited(KERN_WARNING "%s: request aux data allocation failed, iosched may be disturbed\n",
1032 			   dev_name(q->backing_dev_info.dev));
1033 
1034 	rq->cmd_flags &= ~REQ_ELVPRIV;
1035 	rq->elv.icq = NULL;
1036 
1037 	spin_lock_irq(q->queue_lock);
1038 	q->nr_rqs_elvpriv--;
1039 	spin_unlock_irq(q->queue_lock);
1040 	goto out;
1041 
1042 fail_alloc:
1043 	/*
1044 	 * Allocation failed presumably due to memory. Undo anything we
1045 	 * might have messed up.
1046 	 *
1047 	 * Allocating task should really be put onto the front of the wait
1048 	 * queue, but this is pretty rare.
1049 	 */
1050 	spin_lock_irq(q->queue_lock);
1051 	freed_request(rl, rw_flags);
1052 
1053 	/*
1054 	 * in the very unlikely event that allocation failed and no
1055 	 * requests for this direction was pending, mark us starved so that
1056 	 * freeing of a request in the other direction will notice
1057 	 * us. another possible fix would be to split the rq mempool into
1058 	 * READ and WRITE
1059 	 */
1060 rq_starved:
1061 	if (unlikely(rl->count[is_sync] == 0))
1062 		rl->starved[is_sync] = 1;
1063 	return NULL;
1064 }
1065 
1066 /**
1067  * get_request - get a free request
1068  * @q: request_queue to allocate request from
1069  * @rw_flags: RW and SYNC flags
1070  * @bio: bio to allocate request for (can be %NULL)
1071  * @gfp_mask: allocation mask
1072  *
1073  * Get a free request from @q.  If %__GFP_WAIT is set in @gfp_mask, this
1074  * function keeps retrying under memory pressure and fails iff @q is dead.
1075  *
1076  * Must be callled with @q->queue_lock held and,
1077  * Returns %NULL on failure, with @q->queue_lock held.
1078  * Returns !%NULL on success, with @q->queue_lock *not held*.
1079  */
1080 static struct request *get_request(struct request_queue *q, int rw_flags,
1081 				   struct bio *bio, gfp_t gfp_mask)
1082 {
1083 	const bool is_sync = rw_is_sync(rw_flags) != 0;
1084 	DEFINE_WAIT(wait);
1085 	struct request_list *rl;
1086 	struct request *rq;
1087 
1088 	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */
1089 retry:
1090 	rq = __get_request(rl, rw_flags, bio, gfp_mask);
1091 	if (rq)
1092 		return rq;
1093 
1094 	if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dying(q))) {
1095 		blk_put_rl(rl);
1096 		return NULL;
1097 	}
1098 
1099 	/* wait on @rl and retry */
1100 	prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
1101 				  TASK_UNINTERRUPTIBLE);
1102 
1103 	trace_block_sleeprq(q, bio, rw_flags & 1);
1104 
1105 	spin_unlock_irq(q->queue_lock);
1106 	io_schedule();
1107 
1108 	/*
1109 	 * After sleeping, we become a "batching" process and will be able
1110 	 * to allocate at least one request, and up to a big batch of them
1111 	 * for a small period time.  See ioc_batching, ioc_set_batching
1112 	 */
1113 	ioc_set_batching(q, current->io_context);
1114 
1115 	spin_lock_irq(q->queue_lock);
1116 	finish_wait(&rl->wait[is_sync], &wait);
1117 
1118 	goto retry;
1119 }
1120 
1121 struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
1122 {
1123 	struct request *rq;
1124 
1125 	BUG_ON(rw != READ && rw != WRITE);
1126 
1127 	/* create ioc upfront */
1128 	create_io_context(gfp_mask, q->node);
1129 
1130 	spin_lock_irq(q->queue_lock);
1131 	rq = get_request(q, rw, NULL, gfp_mask);
1132 	if (!rq)
1133 		spin_unlock_irq(q->queue_lock);
1134 	/* q->queue_lock is unlocked at this point */
1135 
1136 	return rq;
1137 }
1138 EXPORT_SYMBOL(blk_get_request);
1139 
1140 /**
1141  * blk_make_request - given a bio, allocate a corresponding struct request.
1142  * @q: target request queue
1143  * @bio:  The bio describing the memory mappings that will be submitted for IO.
1144  *        It may be a chained-bio properly constructed by block/bio layer.
1145  * @gfp_mask: gfp flags to be used for memory allocation
1146  *
1147  * blk_make_request is the parallel of generic_make_request for BLOCK_PC
1148  * type commands. Where the struct request needs to be farther initialized by
1149  * the caller. It is passed a &struct bio, which describes the memory info of
1150  * the I/O transfer.
1151  *
1152  * The caller of blk_make_request must make sure that bi_io_vec
1153  * are set to describe the memory buffers. That bio_data_dir() will return
1154  * the needed direction of the request. (And all bio's in the passed bio-chain
1155  * are properly set accordingly)
1156  *
1157  * If called under none-sleepable conditions, mapped bio buffers must not
1158  * need bouncing, by calling the appropriate masked or flagged allocator,
1159  * suitable for the target device. Otherwise the call to blk_queue_bounce will
1160  * BUG.
1161  *
1162  * WARNING: When allocating/cloning a bio-chain, careful consideration should be
1163  * given to how you allocate bios. In particular, you cannot use __GFP_WAIT for
1164  * anything but the first bio in the chain. Otherwise you risk waiting for IO
1165  * completion of a bio that hasn't been submitted yet, thus resulting in a
1166  * deadlock. Alternatively bios should be allocated using bio_kmalloc() instead
1167  * of bio_alloc(), as that avoids the mempool deadlock.
1168  * If possible a big IO should be split into smaller parts when allocation
1169  * fails. Partial allocation should not be an error, or you risk a live-lock.
1170  */
1171 struct request *blk_make_request(struct request_queue *q, struct bio *bio,
1172 				 gfp_t gfp_mask)
1173 {
1174 	struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask);
1175 
1176 	if (unlikely(!rq))
1177 		return ERR_PTR(-ENOMEM);
1178 
1179 	for_each_bio(bio) {
1180 		struct bio *bounce_bio = bio;
1181 		int ret;
1182 
1183 		blk_queue_bounce(q, &bounce_bio);
1184 		ret = blk_rq_append_bio(q, rq, bounce_bio);
1185 		if (unlikely(ret)) {
1186 			blk_put_request(rq);
1187 			return ERR_PTR(ret);
1188 		}
1189 	}
1190 
1191 	return rq;
1192 }
1193 EXPORT_SYMBOL(blk_make_request);
1194 
1195 /**
1196  * blk_requeue_request - put a request back on queue
1197  * @q:		request queue where request should be inserted
1198  * @rq:		request to be inserted
1199  *
1200  * Description:
1201  *    Drivers often keep queueing requests until the hardware cannot accept
1202  *    more, when that condition happens we need to put the request back
1203  *    on the queue. Must be called with queue lock held.
1204  */
1205 void blk_requeue_request(struct request_queue *q, struct request *rq)
1206 {
1207 	blk_delete_timer(rq);
1208 	blk_clear_rq_complete(rq);
1209 	trace_block_rq_requeue(q, rq);
1210 
1211 	if (blk_rq_tagged(rq))
1212 		blk_queue_end_tag(q, rq);
1213 
1214 	BUG_ON(blk_queued_rq(rq));
1215 
1216 	elv_requeue_request(q, rq);
1217 }
1218 EXPORT_SYMBOL(blk_requeue_request);
1219 
1220 static void add_acct_request(struct request_queue *q, struct request *rq,
1221 			     int where)
1222 {
1223 	drive_stat_acct(rq, 1);
1224 	__elv_add_request(q, rq, where);
1225 }
1226 
1227 static void part_round_stats_single(int cpu, struct hd_struct *part,
1228 				    unsigned long now)
1229 {
1230 	if (now == part->stamp)
1231 		return;
1232 
1233 	if (part_in_flight(part)) {
1234 		__part_stat_add(cpu, part, time_in_queue,
1235 				part_in_flight(part) * (now - part->stamp));
1236 		__part_stat_add(cpu, part, io_ticks, (now - part->stamp));
1237 	}
1238 	part->stamp = now;
1239 }
1240 
1241 /**
1242  * part_round_stats() - Round off the performance stats on a struct disk_stats.
1243  * @cpu: cpu number for stats access
1244  * @part: target partition
1245  *
1246  * The average IO queue length and utilisation statistics are maintained
1247  * by observing the current state of the queue length and the amount of
1248  * time it has been in this state for.
1249  *
1250  * Normally, that accounting is done on IO completion, but that can result
1251  * in more than a second's worth of IO being accounted for within any one
1252  * second, leading to >100% utilisation.  To deal with that, we call this
1253  * function to do a round-off before returning the results when reading
1254  * /proc/diskstats.  This accounts immediately for all queue usage up to
1255  * the current jiffies and restarts the counters again.
1256  */
1257 void part_round_stats(int cpu, struct hd_struct *part)
1258 {
1259 	unsigned long now = jiffies;
1260 
1261 	if (part->partno)
1262 		part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
1263 	part_round_stats_single(cpu, part, now);
1264 }
1265 EXPORT_SYMBOL_GPL(part_round_stats);
1266 
1267 /*
1268  * queue lock must be held
1269  */
1270 void __blk_put_request(struct request_queue *q, struct request *req)
1271 {
1272 	if (unlikely(!q))
1273 		return;
1274 	if (unlikely(--req->ref_count))
1275 		return;
1276 
1277 	elv_completed_request(q, req);
1278 
1279 	/* this is a bio leak */
1280 	WARN_ON(req->bio != NULL);
1281 
1282 	/*
1283 	 * Request may not have originated from ll_rw_blk. if not,
1284 	 * it didn't come out of our reserved rq pools
1285 	 */
1286 	if (req->cmd_flags & REQ_ALLOCED) {
1287 		unsigned int flags = req->cmd_flags;
1288 		struct request_list *rl = blk_rq_rl(req);
1289 
1290 		BUG_ON(!list_empty(&req->queuelist));
1291 		BUG_ON(!hlist_unhashed(&req->hash));
1292 
1293 		blk_free_request(rl, req);
1294 		freed_request(rl, flags);
1295 		blk_put_rl(rl);
1296 	}
1297 }
1298 EXPORT_SYMBOL_GPL(__blk_put_request);
1299 
1300 void blk_put_request(struct request *req)
1301 {
1302 	unsigned long flags;
1303 	struct request_queue *q = req->q;
1304 
1305 	spin_lock_irqsave(q->queue_lock, flags);
1306 	__blk_put_request(q, req);
1307 	spin_unlock_irqrestore(q->queue_lock, flags);
1308 }
1309 EXPORT_SYMBOL(blk_put_request);
1310 
1311 /**
1312  * blk_add_request_payload - add a payload to a request
1313  * @rq: request to update
1314  * @page: page backing the payload
1315  * @len: length of the payload.
1316  *
1317  * This allows to later add a payload to an already submitted request by
1318  * a block driver.  The driver needs to take care of freeing the payload
1319  * itself.
1320  *
1321  * Note that this is a quite horrible hack and nothing but handling of
1322  * discard requests should ever use it.
1323  */
1324 void blk_add_request_payload(struct request *rq, struct page *page,
1325 		unsigned int len)
1326 {
1327 	struct bio *bio = rq->bio;
1328 
1329 	bio->bi_io_vec->bv_page = page;
1330 	bio->bi_io_vec->bv_offset = 0;
1331 	bio->bi_io_vec->bv_len = len;
1332 
1333 	bio->bi_size = len;
1334 	bio->bi_vcnt = 1;
1335 	bio->bi_phys_segments = 1;
1336 
1337 	rq->__data_len = rq->resid_len = len;
1338 	rq->nr_phys_segments = 1;
1339 	rq->buffer = bio_data(bio);
1340 }
1341 EXPORT_SYMBOL_GPL(blk_add_request_payload);
1342 
1343 static bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
1344 				   struct bio *bio)
1345 {
1346 	const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
1347 
1348 	if (!ll_back_merge_fn(q, req, bio))
1349 		return false;
1350 
1351 	trace_block_bio_backmerge(q, bio);
1352 
1353 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1354 		blk_rq_set_mixed_merge(req);
1355 
1356 	req->biotail->bi_next = bio;
1357 	req->biotail = bio;
1358 	req->__data_len += bio->bi_size;
1359 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1360 
1361 	drive_stat_acct(req, 0);
1362 	return true;
1363 }
1364 
1365 static bool bio_attempt_front_merge(struct request_queue *q,
1366 				    struct request *req, struct bio *bio)
1367 {
1368 	const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
1369 
1370 	if (!ll_front_merge_fn(q, req, bio))
1371 		return false;
1372 
1373 	trace_block_bio_frontmerge(q, bio);
1374 
1375 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1376 		blk_rq_set_mixed_merge(req);
1377 
1378 	bio->bi_next = req->bio;
1379 	req->bio = bio;
1380 
1381 	/*
1382 	 * may not be valid. if the low level driver said
1383 	 * it didn't need a bounce buffer then it better
1384 	 * not touch req->buffer either...
1385 	 */
1386 	req->buffer = bio_data(bio);
1387 	req->__sector = bio->bi_sector;
1388 	req->__data_len += bio->bi_size;
1389 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1390 
1391 	drive_stat_acct(req, 0);
1392 	return true;
1393 }
1394 
1395 /**
1396  * attempt_plug_merge - try to merge with %current's plugged list
1397  * @q: request_queue new bio is being queued at
1398  * @bio: new bio being queued
1399  * @request_count: out parameter for number of traversed plugged requests
1400  *
1401  * Determine whether @bio being queued on @q can be merged with a request
1402  * on %current's plugged list.  Returns %true if merge was successful,
1403  * otherwise %false.
1404  *
1405  * Plugging coalesces IOs from the same issuer for the same purpose without
1406  * going through @q->queue_lock.  As such it's more of an issuing mechanism
1407  * than scheduling, and the request, while may have elvpriv data, is not
1408  * added on the elevator at this point.  In addition, we don't have
1409  * reliable access to the elevator outside queue lock.  Only check basic
1410  * merging parameters without querying the elevator.
1411  */
1412 static bool attempt_plug_merge(struct request_queue *q, struct bio *bio,
1413 			       unsigned int *request_count)
1414 {
1415 	struct blk_plug *plug;
1416 	struct request *rq;
1417 	bool ret = false;
1418 
1419 	plug = current->plug;
1420 	if (!plug)
1421 		goto out;
1422 	*request_count = 0;
1423 
1424 	list_for_each_entry_reverse(rq, &plug->list, queuelist) {
1425 		int el_ret;
1426 
1427 		if (rq->q == q)
1428 			(*request_count)++;
1429 
1430 		if (rq->q != q || !blk_rq_merge_ok(rq, bio))
1431 			continue;
1432 
1433 		el_ret = blk_try_merge(rq, bio);
1434 		if (el_ret == ELEVATOR_BACK_MERGE) {
1435 			ret = bio_attempt_back_merge(q, rq, bio);
1436 			if (ret)
1437 				break;
1438 		} else if (el_ret == ELEVATOR_FRONT_MERGE) {
1439 			ret = bio_attempt_front_merge(q, rq, bio);
1440 			if (ret)
1441 				break;
1442 		}
1443 	}
1444 out:
1445 	return ret;
1446 }
1447 
1448 void init_request_from_bio(struct request *req, struct bio *bio)
1449 {
1450 	req->cmd_type = REQ_TYPE_FS;
1451 
1452 	req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK;
1453 	if (bio->bi_rw & REQ_RAHEAD)
1454 		req->cmd_flags |= REQ_FAILFAST_MASK;
1455 
1456 	req->errors = 0;
1457 	req->__sector = bio->bi_sector;
1458 	req->ioprio = bio_prio(bio);
1459 	blk_rq_bio_prep(req->q, req, bio);
1460 }
1461 
1462 void blk_queue_bio(struct request_queue *q, struct bio *bio)
1463 {
1464 	const bool sync = !!(bio->bi_rw & REQ_SYNC);
1465 	struct blk_plug *plug;
1466 	int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT;
1467 	struct request *req;
1468 	unsigned int request_count = 0;
1469 
1470 	/*
1471 	 * low level driver can indicate that it wants pages above a
1472 	 * certain limit bounced to low memory (ie for highmem, or even
1473 	 * ISA dma in theory)
1474 	 */
1475 	blk_queue_bounce(q, &bio);
1476 
1477 	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1478 		bio_endio(bio, -EIO);
1479 		return;
1480 	}
1481 
1482 	if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
1483 		spin_lock_irq(q->queue_lock);
1484 		where = ELEVATOR_INSERT_FLUSH;
1485 		goto get_rq;
1486 	}
1487 
1488 	/*
1489 	 * Check if we can merge with the plugged list before grabbing
1490 	 * any locks.
1491 	 */
1492 	if (attempt_plug_merge(q, bio, &request_count))
1493 		return;
1494 
1495 	spin_lock_irq(q->queue_lock);
1496 
1497 	el_ret = elv_merge(q, &req, bio);
1498 	if (el_ret == ELEVATOR_BACK_MERGE) {
1499 		if (bio_attempt_back_merge(q, req, bio)) {
1500 			elv_bio_merged(q, req, bio);
1501 			if (!attempt_back_merge(q, req))
1502 				elv_merged_request(q, req, el_ret);
1503 			goto out_unlock;
1504 		}
1505 	} else if (el_ret == ELEVATOR_FRONT_MERGE) {
1506 		if (bio_attempt_front_merge(q, req, bio)) {
1507 			elv_bio_merged(q, req, bio);
1508 			if (!attempt_front_merge(q, req))
1509 				elv_merged_request(q, req, el_ret);
1510 			goto out_unlock;
1511 		}
1512 	}
1513 
1514 get_rq:
1515 	/*
1516 	 * This sync check and mask will be re-done in init_request_from_bio(),
1517 	 * but we need to set it earlier to expose the sync flag to the
1518 	 * rq allocator and io schedulers.
1519 	 */
1520 	rw_flags = bio_data_dir(bio);
1521 	if (sync)
1522 		rw_flags |= REQ_SYNC;
1523 
1524 	/*
1525 	 * Grab a free request. This is might sleep but can not fail.
1526 	 * Returns with the queue unlocked.
1527 	 */
1528 	req = get_request(q, rw_flags, bio, GFP_NOIO);
1529 	if (unlikely(!req)) {
1530 		bio_endio(bio, -ENODEV);	/* @q is dead */
1531 		goto out_unlock;
1532 	}
1533 
1534 	/*
1535 	 * After dropping the lock and possibly sleeping here, our request
1536 	 * may now be mergeable after it had proven unmergeable (above).
1537 	 * We don't worry about that case for efficiency. It won't happen
1538 	 * often, and the elevators are able to handle it.
1539 	 */
1540 	init_request_from_bio(req, bio);
1541 
1542 	if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
1543 		req->cpu = raw_smp_processor_id();
1544 
1545 	plug = current->plug;
1546 	if (plug) {
1547 		/*
1548 		 * If this is the first request added after a plug, fire
1549 		 * of a plug trace. If others have been added before, check
1550 		 * if we have multiple devices in this plug. If so, make a
1551 		 * note to sort the list before dispatch.
1552 		 */
1553 		if (list_empty(&plug->list))
1554 			trace_block_plug(q);
1555 		else {
1556 			if (!plug->should_sort) {
1557 				struct request *__rq;
1558 
1559 				__rq = list_entry_rq(plug->list.prev);
1560 				if (__rq->q != q)
1561 					plug->should_sort = 1;
1562 			}
1563 			if (request_count >= BLK_MAX_REQUEST_COUNT) {
1564 				blk_flush_plug_list(plug, false);
1565 				trace_block_plug(q);
1566 			}
1567 		}
1568 		list_add_tail(&req->queuelist, &plug->list);
1569 		drive_stat_acct(req, 1);
1570 	} else {
1571 		spin_lock_irq(q->queue_lock);
1572 		add_acct_request(q, req, where);
1573 		__blk_run_queue(q);
1574 out_unlock:
1575 		spin_unlock_irq(q->queue_lock);
1576 	}
1577 }
1578 EXPORT_SYMBOL_GPL(blk_queue_bio);	/* for device mapper only */
1579 
1580 /*
1581  * If bio->bi_dev is a partition, remap the location
1582  */
1583 static inline void blk_partition_remap(struct bio *bio)
1584 {
1585 	struct block_device *bdev = bio->bi_bdev;
1586 
1587 	if (bio_sectors(bio) && bdev != bdev->bd_contains) {
1588 		struct hd_struct *p = bdev->bd_part;
1589 
1590 		bio->bi_sector += p->start_sect;
1591 		bio->bi_bdev = bdev->bd_contains;
1592 
1593 		trace_block_bio_remap(bdev_get_queue(bio->bi_bdev), bio,
1594 				      bdev->bd_dev,
1595 				      bio->bi_sector - p->start_sect);
1596 	}
1597 }
1598 
1599 static void handle_bad_sector(struct bio *bio)
1600 {
1601 	char b[BDEVNAME_SIZE];
1602 
1603 	printk(KERN_INFO "attempt to access beyond end of device\n");
1604 	printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n",
1605 			bdevname(bio->bi_bdev, b),
1606 			bio->bi_rw,
1607 			(unsigned long long)bio->bi_sector + bio_sectors(bio),
1608 			(long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9));
1609 
1610 	set_bit(BIO_EOF, &bio->bi_flags);
1611 }
1612 
1613 #ifdef CONFIG_FAIL_MAKE_REQUEST
1614 
1615 static DECLARE_FAULT_ATTR(fail_make_request);
1616 
1617 static int __init setup_fail_make_request(char *str)
1618 {
1619 	return setup_fault_attr(&fail_make_request, str);
1620 }
1621 __setup("fail_make_request=", setup_fail_make_request);
1622 
1623 static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
1624 {
1625 	return part->make_it_fail && should_fail(&fail_make_request, bytes);
1626 }
1627 
1628 static int __init fail_make_request_debugfs(void)
1629 {
1630 	struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
1631 						NULL, &fail_make_request);
1632 
1633 	return IS_ERR(dir) ? PTR_ERR(dir) : 0;
1634 }
1635 
1636 late_initcall(fail_make_request_debugfs);
1637 
1638 #else /* CONFIG_FAIL_MAKE_REQUEST */
1639 
1640 static inline bool should_fail_request(struct hd_struct *part,
1641 					unsigned int bytes)
1642 {
1643 	return false;
1644 }
1645 
1646 #endif /* CONFIG_FAIL_MAKE_REQUEST */
1647 
1648 /*
1649  * Check whether this bio extends beyond the end of the device.
1650  */
1651 static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
1652 {
1653 	sector_t maxsector;
1654 
1655 	if (!nr_sectors)
1656 		return 0;
1657 
1658 	/* Test device or partition size, when known. */
1659 	maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
1660 	if (maxsector) {
1661 		sector_t sector = bio->bi_sector;
1662 
1663 		if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
1664 			/*
1665 			 * This may well happen - the kernel calls bread()
1666 			 * without checking the size of the device, e.g., when
1667 			 * mounting a device.
1668 			 */
1669 			handle_bad_sector(bio);
1670 			return 1;
1671 		}
1672 	}
1673 
1674 	return 0;
1675 }
1676 
1677 static noinline_for_stack bool
1678 generic_make_request_checks(struct bio *bio)
1679 {
1680 	struct request_queue *q;
1681 	int nr_sectors = bio_sectors(bio);
1682 	int err = -EIO;
1683 	char b[BDEVNAME_SIZE];
1684 	struct hd_struct *part;
1685 
1686 	might_sleep();
1687 
1688 	if (bio_check_eod(bio, nr_sectors))
1689 		goto end_io;
1690 
1691 	q = bdev_get_queue(bio->bi_bdev);
1692 	if (unlikely(!q)) {
1693 		printk(KERN_ERR
1694 		       "generic_make_request: Trying to access "
1695 			"nonexistent block-device %s (%Lu)\n",
1696 			bdevname(bio->bi_bdev, b),
1697 			(long long) bio->bi_sector);
1698 		goto end_io;
1699 	}
1700 
1701 	if (likely(bio_is_rw(bio) &&
1702 		   nr_sectors > queue_max_hw_sectors(q))) {
1703 		printk(KERN_ERR "bio too big device %s (%u > %u)\n",
1704 		       bdevname(bio->bi_bdev, b),
1705 		       bio_sectors(bio),
1706 		       queue_max_hw_sectors(q));
1707 		goto end_io;
1708 	}
1709 
1710 	part = bio->bi_bdev->bd_part;
1711 	if (should_fail_request(part, bio->bi_size) ||
1712 	    should_fail_request(&part_to_disk(part)->part0,
1713 				bio->bi_size))
1714 		goto end_io;
1715 
1716 	/*
1717 	 * If this device has partitions, remap block n
1718 	 * of partition p to block n+start(p) of the disk.
1719 	 */
1720 	blk_partition_remap(bio);
1721 
1722 	if (bio_check_eod(bio, nr_sectors))
1723 		goto end_io;
1724 
1725 	/*
1726 	 * Filter flush bio's early so that make_request based
1727 	 * drivers without flush support don't have to worry
1728 	 * about them.
1729 	 */
1730 	if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && !q->flush_flags) {
1731 		bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA);
1732 		if (!nr_sectors) {
1733 			err = 0;
1734 			goto end_io;
1735 		}
1736 	}
1737 
1738 	if ((bio->bi_rw & REQ_DISCARD) &&
1739 	    (!blk_queue_discard(q) ||
1740 	     ((bio->bi_rw & REQ_SECURE) && !blk_queue_secdiscard(q)))) {
1741 		err = -EOPNOTSUPP;
1742 		goto end_io;
1743 	}
1744 
1745 	if (bio->bi_rw & REQ_WRITE_SAME && !bdev_write_same(bio->bi_bdev)) {
1746 		err = -EOPNOTSUPP;
1747 		goto end_io;
1748 	}
1749 
1750 	/*
1751 	 * Various block parts want %current->io_context and lazy ioc
1752 	 * allocation ends up trading a lot of pain for a small amount of
1753 	 * memory.  Just allocate it upfront.  This may fail and block
1754 	 * layer knows how to live with it.
1755 	 */
1756 	create_io_context(GFP_ATOMIC, q->node);
1757 
1758 	if (blk_throtl_bio(q, bio))
1759 		return false;	/* throttled, will be resubmitted later */
1760 
1761 	trace_block_bio_queue(q, bio);
1762 	return true;
1763 
1764 end_io:
1765 	bio_endio(bio, err);
1766 	return false;
1767 }
1768 
1769 /**
1770  * generic_make_request - hand a buffer to its device driver for I/O
1771  * @bio:  The bio describing the location in memory and on the device.
1772  *
1773  * generic_make_request() is used to make I/O requests of block
1774  * devices. It is passed a &struct bio, which describes the I/O that needs
1775  * to be done.
1776  *
1777  * generic_make_request() does not return any status.  The
1778  * success/failure status of the request, along with notification of
1779  * completion, is delivered asynchronously through the bio->bi_end_io
1780  * function described (one day) else where.
1781  *
1782  * The caller of generic_make_request must make sure that bi_io_vec
1783  * are set to describe the memory buffer, and that bi_dev and bi_sector are
1784  * set to describe the device address, and the
1785  * bi_end_io and optionally bi_private are set to describe how
1786  * completion notification should be signaled.
1787  *
1788  * generic_make_request and the drivers it calls may use bi_next if this
1789  * bio happens to be merged with someone else, and may resubmit the bio to
1790  * a lower device by calling into generic_make_request recursively, which
1791  * means the bio should NOT be touched after the call to ->make_request_fn.
1792  */
1793 void generic_make_request(struct bio *bio)
1794 {
1795 	struct bio_list bio_list_on_stack;
1796 
1797 	if (!generic_make_request_checks(bio))
1798 		return;
1799 
1800 	/*
1801 	 * We only want one ->make_request_fn to be active at a time, else
1802 	 * stack usage with stacked devices could be a problem.  So use
1803 	 * current->bio_list to keep a list of requests submited by a
1804 	 * make_request_fn function.  current->bio_list is also used as a
1805 	 * flag to say if generic_make_request is currently active in this
1806 	 * task or not.  If it is NULL, then no make_request is active.  If
1807 	 * it is non-NULL, then a make_request is active, and new requests
1808 	 * should be added at the tail
1809 	 */
1810 	if (current->bio_list) {
1811 		bio_list_add(current->bio_list, bio);
1812 		return;
1813 	}
1814 
1815 	/* following loop may be a bit non-obvious, and so deserves some
1816 	 * explanation.
1817 	 * Before entering the loop, bio->bi_next is NULL (as all callers
1818 	 * ensure that) so we have a list with a single bio.
1819 	 * We pretend that we have just taken it off a longer list, so
1820 	 * we assign bio_list to a pointer to the bio_list_on_stack,
1821 	 * thus initialising the bio_list of new bios to be
1822 	 * added.  ->make_request() may indeed add some more bios
1823 	 * through a recursive call to generic_make_request.  If it
1824 	 * did, we find a non-NULL value in bio_list and re-enter the loop
1825 	 * from the top.  In this case we really did just take the bio
1826 	 * of the top of the list (no pretending) and so remove it from
1827 	 * bio_list, and call into ->make_request() again.
1828 	 */
1829 	BUG_ON(bio->bi_next);
1830 	bio_list_init(&bio_list_on_stack);
1831 	current->bio_list = &bio_list_on_stack;
1832 	do {
1833 		struct request_queue *q = bdev_get_queue(bio->bi_bdev);
1834 
1835 		q->make_request_fn(q, bio);
1836 
1837 		bio = bio_list_pop(current->bio_list);
1838 	} while (bio);
1839 	current->bio_list = NULL; /* deactivate */
1840 }
1841 EXPORT_SYMBOL(generic_make_request);
1842 
1843 /**
1844  * submit_bio - submit a bio to the block device layer for I/O
1845  * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
1846  * @bio: The &struct bio which describes the I/O
1847  *
1848  * submit_bio() is very similar in purpose to generic_make_request(), and
1849  * uses that function to do most of the work. Both are fairly rough
1850  * interfaces; @bio must be presetup and ready for I/O.
1851  *
1852  */
1853 void submit_bio(int rw, struct bio *bio)
1854 {
1855 	bio->bi_rw |= rw;
1856 
1857 	/*
1858 	 * If it's a regular read/write or a barrier with data attached,
1859 	 * go through the normal accounting stuff before submission.
1860 	 */
1861 	if (bio_has_data(bio)) {
1862 		unsigned int count;
1863 
1864 		if (unlikely(rw & REQ_WRITE_SAME))
1865 			count = bdev_logical_block_size(bio->bi_bdev) >> 9;
1866 		else
1867 			count = bio_sectors(bio);
1868 
1869 		if (rw & WRITE) {
1870 			count_vm_events(PGPGOUT, count);
1871 		} else {
1872 			task_io_account_read(bio->bi_size);
1873 			count_vm_events(PGPGIN, count);
1874 		}
1875 
1876 		if (unlikely(block_dump)) {
1877 			char b[BDEVNAME_SIZE];
1878 			printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
1879 			current->comm, task_pid_nr(current),
1880 				(rw & WRITE) ? "WRITE" : "READ",
1881 				(unsigned long long)bio->bi_sector,
1882 				bdevname(bio->bi_bdev, b),
1883 				count);
1884 		}
1885 	}
1886 
1887 	generic_make_request(bio);
1888 }
1889 EXPORT_SYMBOL(submit_bio);
1890 
1891 /**
1892  * blk_rq_check_limits - Helper function to check a request for the queue limit
1893  * @q:  the queue
1894  * @rq: the request being checked
1895  *
1896  * Description:
1897  *    @rq may have been made based on weaker limitations of upper-level queues
1898  *    in request stacking drivers, and it may violate the limitation of @q.
1899  *    Since the block layer and the underlying device driver trust @rq
1900  *    after it is inserted to @q, it should be checked against @q before
1901  *    the insertion using this generic function.
1902  *
1903  *    This function should also be useful for request stacking drivers
1904  *    in some cases below, so export this function.
1905  *    Request stacking drivers like request-based dm may change the queue
1906  *    limits while requests are in the queue (e.g. dm's table swapping).
1907  *    Such request stacking drivers should check those requests agaist
1908  *    the new queue limits again when they dispatch those requests,
1909  *    although such checkings are also done against the old queue limits
1910  *    when submitting requests.
1911  */
1912 int blk_rq_check_limits(struct request_queue *q, struct request *rq)
1913 {
1914 	if (!rq_mergeable(rq))
1915 		return 0;
1916 
1917 	if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, rq->cmd_flags)) {
1918 		printk(KERN_ERR "%s: over max size limit.\n", __func__);
1919 		return -EIO;
1920 	}
1921 
1922 	/*
1923 	 * queue's settings related to segment counting like q->bounce_pfn
1924 	 * may differ from that of other stacking queues.
1925 	 * Recalculate it to check the request correctly on this queue's
1926 	 * limitation.
1927 	 */
1928 	blk_recalc_rq_segments(rq);
1929 	if (rq->nr_phys_segments > queue_max_segments(q)) {
1930 		printk(KERN_ERR "%s: over max segments limit.\n", __func__);
1931 		return -EIO;
1932 	}
1933 
1934 	return 0;
1935 }
1936 EXPORT_SYMBOL_GPL(blk_rq_check_limits);
1937 
1938 /**
1939  * blk_insert_cloned_request - Helper for stacking drivers to submit a request
1940  * @q:  the queue to submit the request
1941  * @rq: the request being queued
1942  */
1943 int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
1944 {
1945 	unsigned long flags;
1946 	int where = ELEVATOR_INSERT_BACK;
1947 
1948 	if (blk_rq_check_limits(q, rq))
1949 		return -EIO;
1950 
1951 	if (rq->rq_disk &&
1952 	    should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
1953 		return -EIO;
1954 
1955 	spin_lock_irqsave(q->queue_lock, flags);
1956 	if (unlikely(blk_queue_dying(q))) {
1957 		spin_unlock_irqrestore(q->queue_lock, flags);
1958 		return -ENODEV;
1959 	}
1960 
1961 	/*
1962 	 * Submitting request must be dequeued before calling this function
1963 	 * because it will be linked to another request_queue
1964 	 */
1965 	BUG_ON(blk_queued_rq(rq));
1966 
1967 	if (rq->cmd_flags & (REQ_FLUSH|REQ_FUA))
1968 		where = ELEVATOR_INSERT_FLUSH;
1969 
1970 	add_acct_request(q, rq, where);
1971 	if (where == ELEVATOR_INSERT_FLUSH)
1972 		__blk_run_queue(q);
1973 	spin_unlock_irqrestore(q->queue_lock, flags);
1974 
1975 	return 0;
1976 }
1977 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
1978 
1979 /**
1980  * blk_rq_err_bytes - determine number of bytes till the next failure boundary
1981  * @rq: request to examine
1982  *
1983  * Description:
1984  *     A request could be merge of IOs which require different failure
1985  *     handling.  This function determines the number of bytes which
1986  *     can be failed from the beginning of the request without
1987  *     crossing into area which need to be retried further.
1988  *
1989  * Return:
1990  *     The number of bytes to fail.
1991  *
1992  * Context:
1993  *     queue_lock must be held.
1994  */
1995 unsigned int blk_rq_err_bytes(const struct request *rq)
1996 {
1997 	unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
1998 	unsigned int bytes = 0;
1999 	struct bio *bio;
2000 
2001 	if (!(rq->cmd_flags & REQ_MIXED_MERGE))
2002 		return blk_rq_bytes(rq);
2003 
2004 	/*
2005 	 * Currently the only 'mixing' which can happen is between
2006 	 * different fastfail types.  We can safely fail portions
2007 	 * which have all the failfast bits that the first one has -
2008 	 * the ones which are at least as eager to fail as the first
2009 	 * one.
2010 	 */
2011 	for (bio = rq->bio; bio; bio = bio->bi_next) {
2012 		if ((bio->bi_rw & ff) != ff)
2013 			break;
2014 		bytes += bio->bi_size;
2015 	}
2016 
2017 	/* this could lead to infinite loop */
2018 	BUG_ON(blk_rq_bytes(rq) && !bytes);
2019 	return bytes;
2020 }
2021 EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
2022 
2023 static void blk_account_io_completion(struct request *req, unsigned int bytes)
2024 {
2025 	if (blk_do_io_stat(req)) {
2026 		const int rw = rq_data_dir(req);
2027 		struct hd_struct *part;
2028 		int cpu;
2029 
2030 		cpu = part_stat_lock();
2031 		part = req->part;
2032 		part_stat_add(cpu, part, sectors[rw], bytes >> 9);
2033 		part_stat_unlock();
2034 	}
2035 }
2036 
2037 static void blk_account_io_done(struct request *req)
2038 {
2039 	/*
2040 	 * Account IO completion.  flush_rq isn't accounted as a
2041 	 * normal IO on queueing nor completion.  Accounting the
2042 	 * containing request is enough.
2043 	 */
2044 	if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) {
2045 		unsigned long duration = jiffies - req->start_time;
2046 		const int rw = rq_data_dir(req);
2047 		struct hd_struct *part;
2048 		int cpu;
2049 
2050 		cpu = part_stat_lock();
2051 		part = req->part;
2052 
2053 		part_stat_inc(cpu, part, ios[rw]);
2054 		part_stat_add(cpu, part, ticks[rw], duration);
2055 		part_round_stats(cpu, part);
2056 		part_dec_in_flight(part, rw);
2057 
2058 		hd_struct_put(part);
2059 		part_stat_unlock();
2060 	}
2061 }
2062 
2063 /**
2064  * blk_peek_request - peek at the top of a request queue
2065  * @q: request queue to peek at
2066  *
2067  * Description:
2068  *     Return the request at the top of @q.  The returned request
2069  *     should be started using blk_start_request() before LLD starts
2070  *     processing it.
2071  *
2072  * Return:
2073  *     Pointer to the request at the top of @q if available.  Null
2074  *     otherwise.
2075  *
2076  * Context:
2077  *     queue_lock must be held.
2078  */
2079 struct request *blk_peek_request(struct request_queue *q)
2080 {
2081 	struct request *rq;
2082 	int ret;
2083 
2084 	while ((rq = __elv_next_request(q)) != NULL) {
2085 		if (!(rq->cmd_flags & REQ_STARTED)) {
2086 			/*
2087 			 * This is the first time the device driver
2088 			 * sees this request (possibly after
2089 			 * requeueing).  Notify IO scheduler.
2090 			 */
2091 			if (rq->cmd_flags & REQ_SORTED)
2092 				elv_activate_rq(q, rq);
2093 
2094 			/*
2095 			 * just mark as started even if we don't start
2096 			 * it, a request that has been delayed should
2097 			 * not be passed by new incoming requests
2098 			 */
2099 			rq->cmd_flags |= REQ_STARTED;
2100 			trace_block_rq_issue(q, rq);
2101 		}
2102 
2103 		if (!q->boundary_rq || q->boundary_rq == rq) {
2104 			q->end_sector = rq_end_sector(rq);
2105 			q->boundary_rq = NULL;
2106 		}
2107 
2108 		if (rq->cmd_flags & REQ_DONTPREP)
2109 			break;
2110 
2111 		if (q->dma_drain_size && blk_rq_bytes(rq)) {
2112 			/*
2113 			 * make sure space for the drain appears we
2114 			 * know we can do this because max_hw_segments
2115 			 * has been adjusted to be one fewer than the
2116 			 * device can handle
2117 			 */
2118 			rq->nr_phys_segments++;
2119 		}
2120 
2121 		if (!q->prep_rq_fn)
2122 			break;
2123 
2124 		ret = q->prep_rq_fn(q, rq);
2125 		if (ret == BLKPREP_OK) {
2126 			break;
2127 		} else if (ret == BLKPREP_DEFER) {
2128 			/*
2129 			 * the request may have been (partially) prepped.
2130 			 * we need to keep this request in the front to
2131 			 * avoid resource deadlock.  REQ_STARTED will
2132 			 * prevent other fs requests from passing this one.
2133 			 */
2134 			if (q->dma_drain_size && blk_rq_bytes(rq) &&
2135 			    !(rq->cmd_flags & REQ_DONTPREP)) {
2136 				/*
2137 				 * remove the space for the drain we added
2138 				 * so that we don't add it again
2139 				 */
2140 				--rq->nr_phys_segments;
2141 			}
2142 
2143 			rq = NULL;
2144 			break;
2145 		} else if (ret == BLKPREP_KILL) {
2146 			rq->cmd_flags |= REQ_QUIET;
2147 			/*
2148 			 * Mark this request as started so we don't trigger
2149 			 * any debug logic in the end I/O path.
2150 			 */
2151 			blk_start_request(rq);
2152 			__blk_end_request_all(rq, -EIO);
2153 		} else {
2154 			printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
2155 			break;
2156 		}
2157 	}
2158 
2159 	return rq;
2160 }
2161 EXPORT_SYMBOL(blk_peek_request);
2162 
2163 void blk_dequeue_request(struct request *rq)
2164 {
2165 	struct request_queue *q = rq->q;
2166 
2167 	BUG_ON(list_empty(&rq->queuelist));
2168 	BUG_ON(ELV_ON_HASH(rq));
2169 
2170 	list_del_init(&rq->queuelist);
2171 
2172 	/*
2173 	 * the time frame between a request being removed from the lists
2174 	 * and to it is freed is accounted as io that is in progress at
2175 	 * the driver side.
2176 	 */
2177 	if (blk_account_rq(rq)) {
2178 		q->in_flight[rq_is_sync(rq)]++;
2179 		set_io_start_time_ns(rq);
2180 	}
2181 }
2182 
2183 /**
2184  * blk_start_request - start request processing on the driver
2185  * @req: request to dequeue
2186  *
2187  * Description:
2188  *     Dequeue @req and start timeout timer on it.  This hands off the
2189  *     request to the driver.
2190  *
2191  *     Block internal functions which don't want to start timer should
2192  *     call blk_dequeue_request().
2193  *
2194  * Context:
2195  *     queue_lock must be held.
2196  */
2197 void blk_start_request(struct request *req)
2198 {
2199 	blk_dequeue_request(req);
2200 
2201 	/*
2202 	 * We are now handing the request to the hardware, initialize
2203 	 * resid_len to full count and add the timeout handler.
2204 	 */
2205 	req->resid_len = blk_rq_bytes(req);
2206 	if (unlikely(blk_bidi_rq(req)))
2207 		req->next_rq->resid_len = blk_rq_bytes(req->next_rq);
2208 
2209 	blk_add_timer(req);
2210 }
2211 EXPORT_SYMBOL(blk_start_request);
2212 
2213 /**
2214  * blk_fetch_request - fetch a request from a request queue
2215  * @q: request queue to fetch a request from
2216  *
2217  * Description:
2218  *     Return the request at the top of @q.  The request is started on
2219  *     return and LLD can start processing it immediately.
2220  *
2221  * Return:
2222  *     Pointer to the request at the top of @q if available.  Null
2223  *     otherwise.
2224  *
2225  * Context:
2226  *     queue_lock must be held.
2227  */
2228 struct request *blk_fetch_request(struct request_queue *q)
2229 {
2230 	struct request *rq;
2231 
2232 	rq = blk_peek_request(q);
2233 	if (rq)
2234 		blk_start_request(rq);
2235 	return rq;
2236 }
2237 EXPORT_SYMBOL(blk_fetch_request);
2238 
2239 /**
2240  * blk_update_request - Special helper function for request stacking drivers
2241  * @req:      the request being processed
2242  * @error:    %0 for success, < %0 for error
2243  * @nr_bytes: number of bytes to complete @req
2244  *
2245  * Description:
2246  *     Ends I/O on a number of bytes attached to @req, but doesn't complete
2247  *     the request structure even if @req doesn't have leftover.
2248  *     If @req has leftover, sets it up for the next range of segments.
2249  *
2250  *     This special helper function is only for request stacking drivers
2251  *     (e.g. request-based dm) so that they can handle partial completion.
2252  *     Actual device drivers should use blk_end_request instead.
2253  *
2254  *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
2255  *     %false return from this function.
2256  *
2257  * Return:
2258  *     %false - this request doesn't have any more data
2259  *     %true  - this request has more data
2260  **/
2261 bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
2262 {
2263 	int total_bytes, bio_nbytes, next_idx = 0;
2264 	struct bio *bio;
2265 
2266 	if (!req->bio)
2267 		return false;
2268 
2269 	trace_block_rq_complete(req->q, req);
2270 
2271 	/*
2272 	 * For fs requests, rq is just carrier of independent bio's
2273 	 * and each partial completion should be handled separately.
2274 	 * Reset per-request error on each partial completion.
2275 	 *
2276 	 * TODO: tj: This is too subtle.  It would be better to let
2277 	 * low level drivers do what they see fit.
2278 	 */
2279 	if (req->cmd_type == REQ_TYPE_FS)
2280 		req->errors = 0;
2281 
2282 	if (error && req->cmd_type == REQ_TYPE_FS &&
2283 	    !(req->cmd_flags & REQ_QUIET)) {
2284 		char *error_type;
2285 
2286 		switch (error) {
2287 		case -ENOLINK:
2288 			error_type = "recoverable transport";
2289 			break;
2290 		case -EREMOTEIO:
2291 			error_type = "critical target";
2292 			break;
2293 		case -EBADE:
2294 			error_type = "critical nexus";
2295 			break;
2296 		case -EIO:
2297 		default:
2298 			error_type = "I/O";
2299 			break;
2300 		}
2301 		printk_ratelimited(KERN_ERR "end_request: %s error, dev %s, sector %llu\n",
2302 				   error_type, req->rq_disk ?
2303 				   req->rq_disk->disk_name : "?",
2304 				   (unsigned long long)blk_rq_pos(req));
2305 
2306 	}
2307 
2308 	blk_account_io_completion(req, nr_bytes);
2309 
2310 	total_bytes = bio_nbytes = 0;
2311 	while ((bio = req->bio) != NULL) {
2312 		int nbytes;
2313 
2314 		if (nr_bytes >= bio->bi_size) {
2315 			req->bio = bio->bi_next;
2316 			nbytes = bio->bi_size;
2317 			req_bio_endio(req, bio, nbytes, error);
2318 			next_idx = 0;
2319 			bio_nbytes = 0;
2320 		} else {
2321 			int idx = bio->bi_idx + next_idx;
2322 
2323 			if (unlikely(idx >= bio->bi_vcnt)) {
2324 				blk_dump_rq_flags(req, "__end_that");
2325 				printk(KERN_ERR "%s: bio idx %d >= vcnt %d\n",
2326 				       __func__, idx, bio->bi_vcnt);
2327 				break;
2328 			}
2329 
2330 			nbytes = bio_iovec_idx(bio, idx)->bv_len;
2331 			BIO_BUG_ON(nbytes > bio->bi_size);
2332 
2333 			/*
2334 			 * not a complete bvec done
2335 			 */
2336 			if (unlikely(nbytes > nr_bytes)) {
2337 				bio_nbytes += nr_bytes;
2338 				total_bytes += nr_bytes;
2339 				break;
2340 			}
2341 
2342 			/*
2343 			 * advance to the next vector
2344 			 */
2345 			next_idx++;
2346 			bio_nbytes += nbytes;
2347 		}
2348 
2349 		total_bytes += nbytes;
2350 		nr_bytes -= nbytes;
2351 
2352 		bio = req->bio;
2353 		if (bio) {
2354 			/*
2355 			 * end more in this run, or just return 'not-done'
2356 			 */
2357 			if (unlikely(nr_bytes <= 0))
2358 				break;
2359 		}
2360 	}
2361 
2362 	/*
2363 	 * completely done
2364 	 */
2365 	if (!req->bio) {
2366 		/*
2367 		 * Reset counters so that the request stacking driver
2368 		 * can find how many bytes remain in the request
2369 		 * later.
2370 		 */
2371 		req->__data_len = 0;
2372 		return false;
2373 	}
2374 
2375 	/*
2376 	 * if the request wasn't completed, update state
2377 	 */
2378 	if (bio_nbytes) {
2379 		req_bio_endio(req, bio, bio_nbytes, error);
2380 		bio->bi_idx += next_idx;
2381 		bio_iovec(bio)->bv_offset += nr_bytes;
2382 		bio_iovec(bio)->bv_len -= nr_bytes;
2383 	}
2384 
2385 	req->__data_len -= total_bytes;
2386 	req->buffer = bio_data(req->bio);
2387 
2388 	/* update sector only for requests with clear definition of sector */
2389 	if (req->cmd_type == REQ_TYPE_FS)
2390 		req->__sector += total_bytes >> 9;
2391 
2392 	/* mixed attributes always follow the first bio */
2393 	if (req->cmd_flags & REQ_MIXED_MERGE) {
2394 		req->cmd_flags &= ~REQ_FAILFAST_MASK;
2395 		req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK;
2396 	}
2397 
2398 	/*
2399 	 * If total number of sectors is less than the first segment
2400 	 * size, something has gone terribly wrong.
2401 	 */
2402 	if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
2403 		blk_dump_rq_flags(req, "request botched");
2404 		req->__data_len = blk_rq_cur_bytes(req);
2405 	}
2406 
2407 	/* recalculate the number of segments */
2408 	blk_recalc_rq_segments(req);
2409 
2410 	return true;
2411 }
2412 EXPORT_SYMBOL_GPL(blk_update_request);
2413 
2414 static bool blk_update_bidi_request(struct request *rq, int error,
2415 				    unsigned int nr_bytes,
2416 				    unsigned int bidi_bytes)
2417 {
2418 	if (blk_update_request(rq, error, nr_bytes))
2419 		return true;
2420 
2421 	/* Bidi request must be completed as a whole */
2422 	if (unlikely(blk_bidi_rq(rq)) &&
2423 	    blk_update_request(rq->next_rq, error, bidi_bytes))
2424 		return true;
2425 
2426 	if (blk_queue_add_random(rq->q))
2427 		add_disk_randomness(rq->rq_disk);
2428 
2429 	return false;
2430 }
2431 
2432 /**
2433  * blk_unprep_request - unprepare a request
2434  * @req:	the request
2435  *
2436  * This function makes a request ready for complete resubmission (or
2437  * completion).  It happens only after all error handling is complete,
2438  * so represents the appropriate moment to deallocate any resources
2439  * that were allocated to the request in the prep_rq_fn.  The queue
2440  * lock is held when calling this.
2441  */
2442 void blk_unprep_request(struct request *req)
2443 {
2444 	struct request_queue *q = req->q;
2445 
2446 	req->cmd_flags &= ~REQ_DONTPREP;
2447 	if (q->unprep_rq_fn)
2448 		q->unprep_rq_fn(q, req);
2449 }
2450 EXPORT_SYMBOL_GPL(blk_unprep_request);
2451 
2452 /*
2453  * queue lock must be held
2454  */
2455 static void blk_finish_request(struct request *req, int error)
2456 {
2457 	if (blk_rq_tagged(req))
2458 		blk_queue_end_tag(req->q, req);
2459 
2460 	BUG_ON(blk_queued_rq(req));
2461 
2462 	if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS)
2463 		laptop_io_completion(&req->q->backing_dev_info);
2464 
2465 	blk_delete_timer(req);
2466 
2467 	if (req->cmd_flags & REQ_DONTPREP)
2468 		blk_unprep_request(req);
2469 
2470 
2471 	blk_account_io_done(req);
2472 
2473 	if (req->end_io)
2474 		req->end_io(req, error);
2475 	else {
2476 		if (blk_bidi_rq(req))
2477 			__blk_put_request(req->next_rq->q, req->next_rq);
2478 
2479 		__blk_put_request(req->q, req);
2480 	}
2481 }
2482 
2483 /**
2484  * blk_end_bidi_request - Complete a bidi request
2485  * @rq:         the request to complete
2486  * @error:      %0 for success, < %0 for error
2487  * @nr_bytes:   number of bytes to complete @rq
2488  * @bidi_bytes: number of bytes to complete @rq->next_rq
2489  *
2490  * Description:
2491  *     Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
2492  *     Drivers that supports bidi can safely call this member for any
2493  *     type of request, bidi or uni.  In the later case @bidi_bytes is
2494  *     just ignored.
2495  *
2496  * Return:
2497  *     %false - we are done with this request
2498  *     %true  - still buffers pending for this request
2499  **/
2500 static bool blk_end_bidi_request(struct request *rq, int error,
2501 				 unsigned int nr_bytes, unsigned int bidi_bytes)
2502 {
2503 	struct request_queue *q = rq->q;
2504 	unsigned long flags;
2505 
2506 	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2507 		return true;
2508 
2509 	spin_lock_irqsave(q->queue_lock, flags);
2510 	blk_finish_request(rq, error);
2511 	spin_unlock_irqrestore(q->queue_lock, flags);
2512 
2513 	return false;
2514 }
2515 
2516 /**
2517  * __blk_end_bidi_request - Complete a bidi request with queue lock held
2518  * @rq:         the request to complete
2519  * @error:      %0 for success, < %0 for error
2520  * @nr_bytes:   number of bytes to complete @rq
2521  * @bidi_bytes: number of bytes to complete @rq->next_rq
2522  *
2523  * Description:
2524  *     Identical to blk_end_bidi_request() except that queue lock is
2525  *     assumed to be locked on entry and remains so on return.
2526  *
2527  * Return:
2528  *     %false - we are done with this request
2529  *     %true  - still buffers pending for this request
2530  **/
2531 bool __blk_end_bidi_request(struct request *rq, int error,
2532 				   unsigned int nr_bytes, unsigned int bidi_bytes)
2533 {
2534 	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2535 		return true;
2536 
2537 	blk_finish_request(rq, error);
2538 
2539 	return false;
2540 }
2541 
2542 /**
2543  * blk_end_request - Helper function for drivers to complete the request.
2544  * @rq:       the request being processed
2545  * @error:    %0 for success, < %0 for error
2546  * @nr_bytes: number of bytes to complete
2547  *
2548  * Description:
2549  *     Ends I/O on a number of bytes attached to @rq.
2550  *     If @rq has leftover, sets it up for the next range of segments.
2551  *
2552  * Return:
2553  *     %false - we are done with this request
2554  *     %true  - still buffers pending for this request
2555  **/
2556 bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2557 {
2558 	return blk_end_bidi_request(rq, error, nr_bytes, 0);
2559 }
2560 EXPORT_SYMBOL(blk_end_request);
2561 
2562 /**
2563  * blk_end_request_all - Helper function for drives to finish the request.
2564  * @rq: the request to finish
2565  * @error: %0 for success, < %0 for error
2566  *
2567  * Description:
2568  *     Completely finish @rq.
2569  */
2570 void blk_end_request_all(struct request *rq, int error)
2571 {
2572 	bool pending;
2573 	unsigned int bidi_bytes = 0;
2574 
2575 	if (unlikely(blk_bidi_rq(rq)))
2576 		bidi_bytes = blk_rq_bytes(rq->next_rq);
2577 
2578 	pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2579 	BUG_ON(pending);
2580 }
2581 EXPORT_SYMBOL(blk_end_request_all);
2582 
2583 /**
2584  * blk_end_request_cur - Helper function to finish the current request chunk.
2585  * @rq: the request to finish the current chunk for
2586  * @error: %0 for success, < %0 for error
2587  *
2588  * Description:
2589  *     Complete the current consecutively mapped chunk from @rq.
2590  *
2591  * Return:
2592  *     %false - we are done with this request
2593  *     %true  - still buffers pending for this request
2594  */
2595 bool blk_end_request_cur(struct request *rq, int error)
2596 {
2597 	return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2598 }
2599 EXPORT_SYMBOL(blk_end_request_cur);
2600 
2601 /**
2602  * blk_end_request_err - Finish a request till the next failure boundary.
2603  * @rq: the request to finish till the next failure boundary for
2604  * @error: must be negative errno
2605  *
2606  * Description:
2607  *     Complete @rq till the next failure boundary.
2608  *
2609  * Return:
2610  *     %false - we are done with this request
2611  *     %true  - still buffers pending for this request
2612  */
2613 bool blk_end_request_err(struct request *rq, int error)
2614 {
2615 	WARN_ON(error >= 0);
2616 	return blk_end_request(rq, error, blk_rq_err_bytes(rq));
2617 }
2618 EXPORT_SYMBOL_GPL(blk_end_request_err);
2619 
2620 /**
2621  * __blk_end_request - Helper function for drivers to complete the request.
2622  * @rq:       the request being processed
2623  * @error:    %0 for success, < %0 for error
2624  * @nr_bytes: number of bytes to complete
2625  *
2626  * Description:
2627  *     Must be called with queue lock held unlike blk_end_request().
2628  *
2629  * Return:
2630  *     %false - we are done with this request
2631  *     %true  - still buffers pending for this request
2632  **/
2633 bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2634 {
2635 	return __blk_end_bidi_request(rq, error, nr_bytes, 0);
2636 }
2637 EXPORT_SYMBOL(__blk_end_request);
2638 
2639 /**
2640  * __blk_end_request_all - Helper function for drives to finish the request.
2641  * @rq: the request to finish
2642  * @error: %0 for success, < %0 for error
2643  *
2644  * Description:
2645  *     Completely finish @rq.  Must be called with queue lock held.
2646  */
2647 void __blk_end_request_all(struct request *rq, int error)
2648 {
2649 	bool pending;
2650 	unsigned int bidi_bytes = 0;
2651 
2652 	if (unlikely(blk_bidi_rq(rq)))
2653 		bidi_bytes = blk_rq_bytes(rq->next_rq);
2654 
2655 	pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2656 	BUG_ON(pending);
2657 }
2658 EXPORT_SYMBOL(__blk_end_request_all);
2659 
2660 /**
2661  * __blk_end_request_cur - Helper function to finish the current request chunk.
2662  * @rq: the request to finish the current chunk for
2663  * @error: %0 for success, < %0 for error
2664  *
2665  * Description:
2666  *     Complete the current consecutively mapped chunk from @rq.  Must
2667  *     be called with queue lock held.
2668  *
2669  * Return:
2670  *     %false - we are done with this request
2671  *     %true  - still buffers pending for this request
2672  */
2673 bool __blk_end_request_cur(struct request *rq, int error)
2674 {
2675 	return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2676 }
2677 EXPORT_SYMBOL(__blk_end_request_cur);
2678 
2679 /**
2680  * __blk_end_request_err - Finish a request till the next failure boundary.
2681  * @rq: the request to finish till the next failure boundary for
2682  * @error: must be negative errno
2683  *
2684  * Description:
2685  *     Complete @rq till the next failure boundary.  Must be called
2686  *     with queue lock held.
2687  *
2688  * Return:
2689  *     %false - we are done with this request
2690  *     %true  - still buffers pending for this request
2691  */
2692 bool __blk_end_request_err(struct request *rq, int error)
2693 {
2694 	WARN_ON(error >= 0);
2695 	return __blk_end_request(rq, error, blk_rq_err_bytes(rq));
2696 }
2697 EXPORT_SYMBOL_GPL(__blk_end_request_err);
2698 
2699 void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
2700 		     struct bio *bio)
2701 {
2702 	/* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */
2703 	rq->cmd_flags |= bio->bi_rw & REQ_WRITE;
2704 
2705 	if (bio_has_data(bio)) {
2706 		rq->nr_phys_segments = bio_phys_segments(q, bio);
2707 		rq->buffer = bio_data(bio);
2708 	}
2709 	rq->__data_len = bio->bi_size;
2710 	rq->bio = rq->biotail = bio;
2711 
2712 	if (bio->bi_bdev)
2713 		rq->rq_disk = bio->bi_bdev->bd_disk;
2714 }
2715 
2716 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
2717 /**
2718  * rq_flush_dcache_pages - Helper function to flush all pages in a request
2719  * @rq: the request to be flushed
2720  *
2721  * Description:
2722  *     Flush all pages in @rq.
2723  */
2724 void rq_flush_dcache_pages(struct request *rq)
2725 {
2726 	struct req_iterator iter;
2727 	struct bio_vec *bvec;
2728 
2729 	rq_for_each_segment(bvec, rq, iter)
2730 		flush_dcache_page(bvec->bv_page);
2731 }
2732 EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
2733 #endif
2734 
2735 /**
2736  * blk_lld_busy - Check if underlying low-level drivers of a device are busy
2737  * @q : the queue of the device being checked
2738  *
2739  * Description:
2740  *    Check if underlying low-level drivers of a device are busy.
2741  *    If the drivers want to export their busy state, they must set own
2742  *    exporting function using blk_queue_lld_busy() first.
2743  *
2744  *    Basically, this function is used only by request stacking drivers
2745  *    to stop dispatching requests to underlying devices when underlying
2746  *    devices are busy.  This behavior helps more I/O merging on the queue
2747  *    of the request stacking driver and prevents I/O throughput regression
2748  *    on burst I/O load.
2749  *
2750  * Return:
2751  *    0 - Not busy (The request stacking driver should dispatch request)
2752  *    1 - Busy (The request stacking driver should stop dispatching request)
2753  */
2754 int blk_lld_busy(struct request_queue *q)
2755 {
2756 	if (q->lld_busy_fn)
2757 		return q->lld_busy_fn(q);
2758 
2759 	return 0;
2760 }
2761 EXPORT_SYMBOL_GPL(blk_lld_busy);
2762 
2763 /**
2764  * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
2765  * @rq: the clone request to be cleaned up
2766  *
2767  * Description:
2768  *     Free all bios in @rq for a cloned request.
2769  */
2770 void blk_rq_unprep_clone(struct request *rq)
2771 {
2772 	struct bio *bio;
2773 
2774 	while ((bio = rq->bio) != NULL) {
2775 		rq->bio = bio->bi_next;
2776 
2777 		bio_put(bio);
2778 	}
2779 }
2780 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
2781 
2782 /*
2783  * Copy attributes of the original request to the clone request.
2784  * The actual data parts (e.g. ->cmd, ->buffer, ->sense) are not copied.
2785  */
2786 static void __blk_rq_prep_clone(struct request *dst, struct request *src)
2787 {
2788 	dst->cpu = src->cpu;
2789 	dst->cmd_flags = (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE;
2790 	dst->cmd_type = src->cmd_type;
2791 	dst->__sector = blk_rq_pos(src);
2792 	dst->__data_len = blk_rq_bytes(src);
2793 	dst->nr_phys_segments = src->nr_phys_segments;
2794 	dst->ioprio = src->ioprio;
2795 	dst->extra_len = src->extra_len;
2796 }
2797 
2798 /**
2799  * blk_rq_prep_clone - Helper function to setup clone request
2800  * @rq: the request to be setup
2801  * @rq_src: original request to be cloned
2802  * @bs: bio_set that bios for clone are allocated from
2803  * @gfp_mask: memory allocation mask for bio
2804  * @bio_ctr: setup function to be called for each clone bio.
2805  *           Returns %0 for success, non %0 for failure.
2806  * @data: private data to be passed to @bio_ctr
2807  *
2808  * Description:
2809  *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
2810  *     The actual data parts of @rq_src (e.g. ->cmd, ->buffer, ->sense)
2811  *     are not copied, and copying such parts is the caller's responsibility.
2812  *     Also, pages which the original bios are pointing to are not copied
2813  *     and the cloned bios just point same pages.
2814  *     So cloned bios must be completed before original bios, which means
2815  *     the caller must complete @rq before @rq_src.
2816  */
2817 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
2818 		      struct bio_set *bs, gfp_t gfp_mask,
2819 		      int (*bio_ctr)(struct bio *, struct bio *, void *),
2820 		      void *data)
2821 {
2822 	struct bio *bio, *bio_src;
2823 
2824 	if (!bs)
2825 		bs = fs_bio_set;
2826 
2827 	blk_rq_init(NULL, rq);
2828 
2829 	__rq_for_each_bio(bio_src, rq_src) {
2830 		bio = bio_clone_bioset(bio_src, gfp_mask, bs);
2831 		if (!bio)
2832 			goto free_and_out;
2833 
2834 		if (bio_ctr && bio_ctr(bio, bio_src, data))
2835 			goto free_and_out;
2836 
2837 		if (rq->bio) {
2838 			rq->biotail->bi_next = bio;
2839 			rq->biotail = bio;
2840 		} else
2841 			rq->bio = rq->biotail = bio;
2842 	}
2843 
2844 	__blk_rq_prep_clone(rq, rq_src);
2845 
2846 	return 0;
2847 
2848 free_and_out:
2849 	if (bio)
2850 		bio_put(bio);
2851 	blk_rq_unprep_clone(rq);
2852 
2853 	return -ENOMEM;
2854 }
2855 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
2856 
2857 int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
2858 {
2859 	return queue_work(kblockd_workqueue, work);
2860 }
2861 EXPORT_SYMBOL(kblockd_schedule_work);
2862 
2863 int kblockd_schedule_delayed_work(struct request_queue *q,
2864 			struct delayed_work *dwork, unsigned long delay)
2865 {
2866 	return queue_delayed_work(kblockd_workqueue, dwork, delay);
2867 }
2868 EXPORT_SYMBOL(kblockd_schedule_delayed_work);
2869 
2870 #define PLUG_MAGIC	0x91827364
2871 
2872 /**
2873  * blk_start_plug - initialize blk_plug and track it inside the task_struct
2874  * @plug:	The &struct blk_plug that needs to be initialized
2875  *
2876  * Description:
2877  *   Tracking blk_plug inside the task_struct will help with auto-flushing the
2878  *   pending I/O should the task end up blocking between blk_start_plug() and
2879  *   blk_finish_plug(). This is important from a performance perspective, but
2880  *   also ensures that we don't deadlock. For instance, if the task is blocking
2881  *   for a memory allocation, memory reclaim could end up wanting to free a
2882  *   page belonging to that request that is currently residing in our private
2883  *   plug. By flushing the pending I/O when the process goes to sleep, we avoid
2884  *   this kind of deadlock.
2885  */
2886 void blk_start_plug(struct blk_plug *plug)
2887 {
2888 	struct task_struct *tsk = current;
2889 
2890 	plug->magic = PLUG_MAGIC;
2891 	INIT_LIST_HEAD(&plug->list);
2892 	INIT_LIST_HEAD(&plug->cb_list);
2893 	plug->should_sort = 0;
2894 
2895 	/*
2896 	 * If this is a nested plug, don't actually assign it. It will be
2897 	 * flushed on its own.
2898 	 */
2899 	if (!tsk->plug) {
2900 		/*
2901 		 * Store ordering should not be needed here, since a potential
2902 		 * preempt will imply a full memory barrier
2903 		 */
2904 		tsk->plug = plug;
2905 	}
2906 }
2907 EXPORT_SYMBOL(blk_start_plug);
2908 
2909 static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
2910 {
2911 	struct request *rqa = container_of(a, struct request, queuelist);
2912 	struct request *rqb = container_of(b, struct request, queuelist);
2913 
2914 	return !(rqa->q < rqb->q ||
2915 		(rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb)));
2916 }
2917 
2918 /*
2919  * If 'from_schedule' is true, then postpone the dispatch of requests
2920  * until a safe kblockd context. We due this to avoid accidental big
2921  * additional stack usage in driver dispatch, in places where the originally
2922  * plugger did not intend it.
2923  */
2924 static void queue_unplugged(struct request_queue *q, unsigned int depth,
2925 			    bool from_schedule)
2926 	__releases(q->queue_lock)
2927 {
2928 	trace_block_unplug(q, depth, !from_schedule);
2929 
2930 	if (from_schedule)
2931 		blk_run_queue_async(q);
2932 	else
2933 		__blk_run_queue(q);
2934 	spin_unlock(q->queue_lock);
2935 }
2936 
2937 static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
2938 {
2939 	LIST_HEAD(callbacks);
2940 
2941 	while (!list_empty(&plug->cb_list)) {
2942 		list_splice_init(&plug->cb_list, &callbacks);
2943 
2944 		while (!list_empty(&callbacks)) {
2945 			struct blk_plug_cb *cb = list_first_entry(&callbacks,
2946 							  struct blk_plug_cb,
2947 							  list);
2948 			list_del(&cb->list);
2949 			cb->callback(cb, from_schedule);
2950 		}
2951 	}
2952 }
2953 
2954 struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
2955 				      int size)
2956 {
2957 	struct blk_plug *plug = current->plug;
2958 	struct blk_plug_cb *cb;
2959 
2960 	if (!plug)
2961 		return NULL;
2962 
2963 	list_for_each_entry(cb, &plug->cb_list, list)
2964 		if (cb->callback == unplug && cb->data == data)
2965 			return cb;
2966 
2967 	/* Not currently on the callback list */
2968 	BUG_ON(size < sizeof(*cb));
2969 	cb = kzalloc(size, GFP_ATOMIC);
2970 	if (cb) {
2971 		cb->data = data;
2972 		cb->callback = unplug;
2973 		list_add(&cb->list, &plug->cb_list);
2974 	}
2975 	return cb;
2976 }
2977 EXPORT_SYMBOL(blk_check_plugged);
2978 
2979 void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
2980 {
2981 	struct request_queue *q;
2982 	unsigned long flags;
2983 	struct request *rq;
2984 	LIST_HEAD(list);
2985 	unsigned int depth;
2986 
2987 	BUG_ON(plug->magic != PLUG_MAGIC);
2988 
2989 	flush_plug_callbacks(plug, from_schedule);
2990 	if (list_empty(&plug->list))
2991 		return;
2992 
2993 	list_splice_init(&plug->list, &list);
2994 
2995 	if (plug->should_sort) {
2996 		list_sort(NULL, &list, plug_rq_cmp);
2997 		plug->should_sort = 0;
2998 	}
2999 
3000 	q = NULL;
3001 	depth = 0;
3002 
3003 	/*
3004 	 * Save and disable interrupts here, to avoid doing it for every
3005 	 * queue lock we have to take.
3006 	 */
3007 	local_irq_save(flags);
3008 	while (!list_empty(&list)) {
3009 		rq = list_entry_rq(list.next);
3010 		list_del_init(&rq->queuelist);
3011 		BUG_ON(!rq->q);
3012 		if (rq->q != q) {
3013 			/*
3014 			 * This drops the queue lock
3015 			 */
3016 			if (q)
3017 				queue_unplugged(q, depth, from_schedule);
3018 			q = rq->q;
3019 			depth = 0;
3020 			spin_lock(q->queue_lock);
3021 		}
3022 
3023 		/*
3024 		 * Short-circuit if @q is dead
3025 		 */
3026 		if (unlikely(blk_queue_dying(q))) {
3027 			__blk_end_request_all(rq, -ENODEV);
3028 			continue;
3029 		}
3030 
3031 		/*
3032 		 * rq is already accounted, so use raw insert
3033 		 */
3034 		if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA))
3035 			__elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
3036 		else
3037 			__elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
3038 
3039 		depth++;
3040 	}
3041 
3042 	/*
3043 	 * This drops the queue lock
3044 	 */
3045 	if (q)
3046 		queue_unplugged(q, depth, from_schedule);
3047 
3048 	local_irq_restore(flags);
3049 }
3050 
3051 void blk_finish_plug(struct blk_plug *plug)
3052 {
3053 	blk_flush_plug_list(plug, false);
3054 
3055 	if (plug == current->plug)
3056 		current->plug = NULL;
3057 }
3058 EXPORT_SYMBOL(blk_finish_plug);
3059 
3060 int __init blk_dev_init(void)
3061 {
3062 	BUILD_BUG_ON(__REQ_NR_BITS > 8 *
3063 			sizeof(((struct request *)0)->cmd_flags));
3064 
3065 	/* used for unplugging and affects IO latency/throughput - HIGHPRI */
3066 	kblockd_workqueue = alloc_workqueue("kblockd",
3067 					    WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
3068 	if (!kblockd_workqueue)
3069 		panic("Failed to create kblockd\n");
3070 
3071 	request_cachep = kmem_cache_create("blkdev_requests",
3072 			sizeof(struct request), 0, SLAB_PANIC, NULL);
3073 
3074 	blk_requestq_cachep = kmem_cache_create("blkdev_queue",
3075 			sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
3076 
3077 	return 0;
3078 }
3079