xref: /openbmc/linux/block/blk-core.c (revision 2209fda3)
1 /*
2  * Copyright (C) 1991, 1992 Linus Torvalds
3  * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
4  * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
5  * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
6  * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
7  *	-  July2000
8  * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
9  */
10 
11 /*
12  * This handles all read/write requests to block devices
13  */
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/backing-dev.h>
17 #include <linux/bio.h>
18 #include <linux/blkdev.h>
19 #include <linux/blk-mq.h>
20 #include <linux/highmem.h>
21 #include <linux/mm.h>
22 #include <linux/kernel_stat.h>
23 #include <linux/string.h>
24 #include <linux/init.h>
25 #include <linux/completion.h>
26 #include <linux/slab.h>
27 #include <linux/swap.h>
28 #include <linux/writeback.h>
29 #include <linux/task_io_accounting_ops.h>
30 #include <linux/fault-inject.h>
31 #include <linux/list_sort.h>
32 #include <linux/delay.h>
33 #include <linux/ratelimit.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/blk-cgroup.h>
36 #include <linux/debugfs.h>
37 #include <linux/bpf.h>
38 
39 #define CREATE_TRACE_POINTS
40 #include <trace/events/block.h>
41 
42 #include "blk.h"
43 #include "blk-mq.h"
44 #include "blk-mq-sched.h"
45 #include "blk-pm.h"
46 #include "blk-rq-qos.h"
47 
48 #ifdef CONFIG_DEBUG_FS
49 struct dentry *blk_debugfs_root;
50 #endif
51 
52 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
53 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
54 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
55 EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
56 EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
57 
58 DEFINE_IDA(blk_queue_ida);
59 
60 /*
61  * For the allocated request tables
62  */
63 struct kmem_cache *request_cachep;
64 
65 /*
66  * For queue allocation
67  */
68 struct kmem_cache *blk_requestq_cachep;
69 
70 /*
71  * Controlling structure to kblockd
72  */
73 static struct workqueue_struct *kblockd_workqueue;
74 
75 /**
76  * blk_queue_flag_set - atomically set a queue flag
77  * @flag: flag to be set
78  * @q: request queue
79  */
80 void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
81 {
82 	unsigned long flags;
83 
84 	spin_lock_irqsave(q->queue_lock, flags);
85 	queue_flag_set(flag, q);
86 	spin_unlock_irqrestore(q->queue_lock, flags);
87 }
88 EXPORT_SYMBOL(blk_queue_flag_set);
89 
90 /**
91  * blk_queue_flag_clear - atomically clear a queue flag
92  * @flag: flag to be cleared
93  * @q: request queue
94  */
95 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
96 {
97 	unsigned long flags;
98 
99 	spin_lock_irqsave(q->queue_lock, flags);
100 	queue_flag_clear(flag, q);
101 	spin_unlock_irqrestore(q->queue_lock, flags);
102 }
103 EXPORT_SYMBOL(blk_queue_flag_clear);
104 
105 /**
106  * blk_queue_flag_test_and_set - atomically test and set a queue flag
107  * @flag: flag to be set
108  * @q: request queue
109  *
110  * Returns the previous value of @flag - 0 if the flag was not set and 1 if
111  * the flag was already set.
112  */
113 bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q)
114 {
115 	unsigned long flags;
116 	bool res;
117 
118 	spin_lock_irqsave(q->queue_lock, flags);
119 	res = queue_flag_test_and_set(flag, q);
120 	spin_unlock_irqrestore(q->queue_lock, flags);
121 
122 	return res;
123 }
124 EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set);
125 
126 /**
127  * blk_queue_flag_test_and_clear - atomically test and clear a queue flag
128  * @flag: flag to be cleared
129  * @q: request queue
130  *
131  * Returns the previous value of @flag - 0 if the flag was not set and 1 if
132  * the flag was set.
133  */
134 bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q)
135 {
136 	unsigned long flags;
137 	bool res;
138 
139 	spin_lock_irqsave(q->queue_lock, flags);
140 	res = queue_flag_test_and_clear(flag, q);
141 	spin_unlock_irqrestore(q->queue_lock, flags);
142 
143 	return res;
144 }
145 EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_clear);
146 
147 static void blk_clear_congested(struct request_list *rl, int sync)
148 {
149 #ifdef CONFIG_CGROUP_WRITEBACK
150 	clear_wb_congested(rl->blkg->wb_congested, sync);
151 #else
152 	/*
153 	 * If !CGROUP_WRITEBACK, all blkg's map to bdi->wb and we shouldn't
154 	 * flip its congestion state for events on other blkcgs.
155 	 */
156 	if (rl == &rl->q->root_rl)
157 		clear_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
158 #endif
159 }
160 
161 static void blk_set_congested(struct request_list *rl, int sync)
162 {
163 #ifdef CONFIG_CGROUP_WRITEBACK
164 	set_wb_congested(rl->blkg->wb_congested, sync);
165 #else
166 	/* see blk_clear_congested() */
167 	if (rl == &rl->q->root_rl)
168 		set_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
169 #endif
170 }
171 
172 void blk_queue_congestion_threshold(struct request_queue *q)
173 {
174 	int nr;
175 
176 	nr = q->nr_requests - (q->nr_requests / 8) + 1;
177 	if (nr > q->nr_requests)
178 		nr = q->nr_requests;
179 	q->nr_congestion_on = nr;
180 
181 	nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
182 	if (nr < 1)
183 		nr = 1;
184 	q->nr_congestion_off = nr;
185 }
186 
187 void blk_rq_init(struct request_queue *q, struct request *rq)
188 {
189 	memset(rq, 0, sizeof(*rq));
190 
191 	INIT_LIST_HEAD(&rq->queuelist);
192 	INIT_LIST_HEAD(&rq->timeout_list);
193 	rq->cpu = -1;
194 	rq->q = q;
195 	rq->__sector = (sector_t) -1;
196 	INIT_HLIST_NODE(&rq->hash);
197 	RB_CLEAR_NODE(&rq->rb_node);
198 	rq->tag = -1;
199 	rq->internal_tag = -1;
200 	rq->start_time_ns = ktime_get_ns();
201 	rq->part = NULL;
202 }
203 EXPORT_SYMBOL(blk_rq_init);
204 
205 static const struct {
206 	int		errno;
207 	const char	*name;
208 } blk_errors[] = {
209 	[BLK_STS_OK]		= { 0,		"" },
210 	[BLK_STS_NOTSUPP]	= { -EOPNOTSUPP, "operation not supported" },
211 	[BLK_STS_TIMEOUT]	= { -ETIMEDOUT,	"timeout" },
212 	[BLK_STS_NOSPC]		= { -ENOSPC,	"critical space allocation" },
213 	[BLK_STS_TRANSPORT]	= { -ENOLINK,	"recoverable transport" },
214 	[BLK_STS_TARGET]	= { -EREMOTEIO,	"critical target" },
215 	[BLK_STS_NEXUS]		= { -EBADE,	"critical nexus" },
216 	[BLK_STS_MEDIUM]	= { -ENODATA,	"critical medium" },
217 	[BLK_STS_PROTECTION]	= { -EILSEQ,	"protection" },
218 	[BLK_STS_RESOURCE]	= { -ENOMEM,	"kernel resource" },
219 	[BLK_STS_DEV_RESOURCE]	= { -EBUSY,	"device resource" },
220 	[BLK_STS_AGAIN]		= { -EAGAIN,	"nonblocking retry" },
221 
222 	/* device mapper special case, should not leak out: */
223 	[BLK_STS_DM_REQUEUE]	= { -EREMCHG, "dm internal retry" },
224 
225 	/* everything else not covered above: */
226 	[BLK_STS_IOERR]		= { -EIO,	"I/O" },
227 };
228 
229 blk_status_t errno_to_blk_status(int errno)
230 {
231 	int i;
232 
233 	for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
234 		if (blk_errors[i].errno == errno)
235 			return (__force blk_status_t)i;
236 	}
237 
238 	return BLK_STS_IOERR;
239 }
240 EXPORT_SYMBOL_GPL(errno_to_blk_status);
241 
242 int blk_status_to_errno(blk_status_t status)
243 {
244 	int idx = (__force int)status;
245 
246 	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
247 		return -EIO;
248 	return blk_errors[idx].errno;
249 }
250 EXPORT_SYMBOL_GPL(blk_status_to_errno);
251 
252 static void print_req_error(struct request *req, blk_status_t status)
253 {
254 	int idx = (__force int)status;
255 
256 	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
257 		return;
258 
259 	printk_ratelimited(KERN_ERR "%s: %s error, dev %s, sector %llu\n",
260 			   __func__, blk_errors[idx].name, req->rq_disk ?
261 			   req->rq_disk->disk_name : "?",
262 			   (unsigned long long)blk_rq_pos(req));
263 }
264 
265 static void req_bio_endio(struct request *rq, struct bio *bio,
266 			  unsigned int nbytes, blk_status_t error)
267 {
268 	if (error)
269 		bio->bi_status = error;
270 
271 	if (unlikely(rq->rq_flags & RQF_QUIET))
272 		bio_set_flag(bio, BIO_QUIET);
273 
274 	bio_advance(bio, nbytes);
275 
276 	/* don't actually finish bio if it's part of flush sequence */
277 	if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
278 		bio_endio(bio);
279 }
280 
281 void blk_dump_rq_flags(struct request *rq, char *msg)
282 {
283 	printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
284 		rq->rq_disk ? rq->rq_disk->disk_name : "?",
285 		(unsigned long long) rq->cmd_flags);
286 
287 	printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
288 	       (unsigned long long)blk_rq_pos(rq),
289 	       blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
290 	printk(KERN_INFO "  bio %p, biotail %p, len %u\n",
291 	       rq->bio, rq->biotail, blk_rq_bytes(rq));
292 }
293 EXPORT_SYMBOL(blk_dump_rq_flags);
294 
295 static void blk_delay_work(struct work_struct *work)
296 {
297 	struct request_queue *q;
298 
299 	q = container_of(work, struct request_queue, delay_work.work);
300 	spin_lock_irq(q->queue_lock);
301 	__blk_run_queue(q);
302 	spin_unlock_irq(q->queue_lock);
303 }
304 
305 /**
306  * blk_delay_queue - restart queueing after defined interval
307  * @q:		The &struct request_queue in question
308  * @msecs:	Delay in msecs
309  *
310  * Description:
311  *   Sometimes queueing needs to be postponed for a little while, to allow
312  *   resources to come back. This function will make sure that queueing is
313  *   restarted around the specified time.
314  */
315 void blk_delay_queue(struct request_queue *q, unsigned long msecs)
316 {
317 	lockdep_assert_held(q->queue_lock);
318 	WARN_ON_ONCE(q->mq_ops);
319 
320 	if (likely(!blk_queue_dead(q)))
321 		queue_delayed_work(kblockd_workqueue, &q->delay_work,
322 				   msecs_to_jiffies(msecs));
323 }
324 EXPORT_SYMBOL(blk_delay_queue);
325 
326 /**
327  * blk_start_queue_async - asynchronously restart a previously stopped queue
328  * @q:    The &struct request_queue in question
329  *
330  * Description:
331  *   blk_start_queue_async() will clear the stop flag on the queue, and
332  *   ensure that the request_fn for the queue is run from an async
333  *   context.
334  **/
335 void blk_start_queue_async(struct request_queue *q)
336 {
337 	lockdep_assert_held(q->queue_lock);
338 	WARN_ON_ONCE(q->mq_ops);
339 
340 	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
341 	blk_run_queue_async(q);
342 }
343 EXPORT_SYMBOL(blk_start_queue_async);
344 
345 /**
346  * blk_start_queue - restart a previously stopped queue
347  * @q:    The &struct request_queue in question
348  *
349  * Description:
350  *   blk_start_queue() will clear the stop flag on the queue, and call
351  *   the request_fn for the queue if it was in a stopped state when
352  *   entered. Also see blk_stop_queue().
353  **/
354 void blk_start_queue(struct request_queue *q)
355 {
356 	lockdep_assert_held(q->queue_lock);
357 	WARN_ON_ONCE(q->mq_ops);
358 
359 	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
360 	__blk_run_queue(q);
361 }
362 EXPORT_SYMBOL(blk_start_queue);
363 
364 /**
365  * blk_stop_queue - stop a queue
366  * @q:    The &struct request_queue in question
367  *
368  * Description:
369  *   The Linux block layer assumes that a block driver will consume all
370  *   entries on the request queue when the request_fn strategy is called.
371  *   Often this will not happen, because of hardware limitations (queue
372  *   depth settings). If a device driver gets a 'queue full' response,
373  *   or if it simply chooses not to queue more I/O at one point, it can
374  *   call this function to prevent the request_fn from being called until
375  *   the driver has signalled it's ready to go again. This happens by calling
376  *   blk_start_queue() to restart queue operations.
377  **/
378 void blk_stop_queue(struct request_queue *q)
379 {
380 	lockdep_assert_held(q->queue_lock);
381 	WARN_ON_ONCE(q->mq_ops);
382 
383 	cancel_delayed_work(&q->delay_work);
384 	queue_flag_set(QUEUE_FLAG_STOPPED, q);
385 }
386 EXPORT_SYMBOL(blk_stop_queue);
387 
388 /**
389  * blk_sync_queue - cancel any pending callbacks on a queue
390  * @q: the queue
391  *
392  * Description:
393  *     The block layer may perform asynchronous callback activity
394  *     on a queue, such as calling the unplug function after a timeout.
395  *     A block device may call blk_sync_queue to ensure that any
396  *     such activity is cancelled, thus allowing it to release resources
397  *     that the callbacks might use. The caller must already have made sure
398  *     that its ->make_request_fn will not re-add plugging prior to calling
399  *     this function.
400  *
401  *     This function does not cancel any asynchronous activity arising
402  *     out of elevator or throttling code. That would require elevator_exit()
403  *     and blkcg_exit_queue() to be called with queue lock initialized.
404  *
405  */
406 void blk_sync_queue(struct request_queue *q)
407 {
408 	del_timer_sync(&q->timeout);
409 	cancel_work_sync(&q->timeout_work);
410 
411 	if (q->mq_ops) {
412 		struct blk_mq_hw_ctx *hctx;
413 		int i;
414 
415 		cancel_delayed_work_sync(&q->requeue_work);
416 		queue_for_each_hw_ctx(q, hctx, i)
417 			cancel_delayed_work_sync(&hctx->run_work);
418 	} else {
419 		cancel_delayed_work_sync(&q->delay_work);
420 	}
421 }
422 EXPORT_SYMBOL(blk_sync_queue);
423 
424 /**
425  * blk_set_pm_only - increment pm_only counter
426  * @q: request queue pointer
427  */
428 void blk_set_pm_only(struct request_queue *q)
429 {
430 	atomic_inc(&q->pm_only);
431 }
432 EXPORT_SYMBOL_GPL(blk_set_pm_only);
433 
434 void blk_clear_pm_only(struct request_queue *q)
435 {
436 	int pm_only;
437 
438 	pm_only = atomic_dec_return(&q->pm_only);
439 	WARN_ON_ONCE(pm_only < 0);
440 	if (pm_only == 0)
441 		wake_up_all(&q->mq_freeze_wq);
442 }
443 EXPORT_SYMBOL_GPL(blk_clear_pm_only);
444 
445 /**
446  * __blk_run_queue_uncond - run a queue whether or not it has been stopped
447  * @q:	The queue to run
448  *
449  * Description:
450  *    Invoke request handling on a queue if there are any pending requests.
451  *    May be used to restart request handling after a request has completed.
452  *    This variant runs the queue whether or not the queue has been
453  *    stopped. Must be called with the queue lock held and interrupts
454  *    disabled. See also @blk_run_queue.
455  */
456 inline void __blk_run_queue_uncond(struct request_queue *q)
457 {
458 	lockdep_assert_held(q->queue_lock);
459 	WARN_ON_ONCE(q->mq_ops);
460 
461 	if (unlikely(blk_queue_dead(q)))
462 		return;
463 
464 	/*
465 	 * Some request_fn implementations, e.g. scsi_request_fn(), unlock
466 	 * the queue lock internally. As a result multiple threads may be
467 	 * running such a request function concurrently. Keep track of the
468 	 * number of active request_fn invocations such that blk_drain_queue()
469 	 * can wait until all these request_fn calls have finished.
470 	 */
471 	q->request_fn_active++;
472 	q->request_fn(q);
473 	q->request_fn_active--;
474 }
475 EXPORT_SYMBOL_GPL(__blk_run_queue_uncond);
476 
477 /**
478  * __blk_run_queue - run a single device queue
479  * @q:	The queue to run
480  *
481  * Description:
482  *    See @blk_run_queue.
483  */
484 void __blk_run_queue(struct request_queue *q)
485 {
486 	lockdep_assert_held(q->queue_lock);
487 	WARN_ON_ONCE(q->mq_ops);
488 
489 	if (unlikely(blk_queue_stopped(q)))
490 		return;
491 
492 	__blk_run_queue_uncond(q);
493 }
494 EXPORT_SYMBOL(__blk_run_queue);
495 
496 /**
497  * blk_run_queue_async - run a single device queue in workqueue context
498  * @q:	The queue to run
499  *
500  * Description:
501  *    Tells kblockd to perform the equivalent of @blk_run_queue on behalf
502  *    of us.
503  *
504  * Note:
505  *    Since it is not allowed to run q->delay_work after blk_cleanup_queue()
506  *    has canceled q->delay_work, callers must hold the queue lock to avoid
507  *    race conditions between blk_cleanup_queue() and blk_run_queue_async().
508  */
509 void blk_run_queue_async(struct request_queue *q)
510 {
511 	lockdep_assert_held(q->queue_lock);
512 	WARN_ON_ONCE(q->mq_ops);
513 
514 	if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q)))
515 		mod_delayed_work(kblockd_workqueue, &q->delay_work, 0);
516 }
517 EXPORT_SYMBOL(blk_run_queue_async);
518 
519 /**
520  * blk_run_queue - run a single device queue
521  * @q: The queue to run
522  *
523  * Description:
524  *    Invoke request handling on this queue, if it has pending work to do.
525  *    May be used to restart queueing when a request has completed.
526  */
527 void blk_run_queue(struct request_queue *q)
528 {
529 	unsigned long flags;
530 
531 	WARN_ON_ONCE(q->mq_ops);
532 
533 	spin_lock_irqsave(q->queue_lock, flags);
534 	__blk_run_queue(q);
535 	spin_unlock_irqrestore(q->queue_lock, flags);
536 }
537 EXPORT_SYMBOL(blk_run_queue);
538 
539 void blk_put_queue(struct request_queue *q)
540 {
541 	kobject_put(&q->kobj);
542 }
543 EXPORT_SYMBOL(blk_put_queue);
544 
545 /**
546  * __blk_drain_queue - drain requests from request_queue
547  * @q: queue to drain
548  * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV
549  *
550  * Drain requests from @q.  If @drain_all is set, all requests are drained.
551  * If not, only ELVPRIV requests are drained.  The caller is responsible
552  * for ensuring that no new requests which need to be drained are queued.
553  */
554 static void __blk_drain_queue(struct request_queue *q, bool drain_all)
555 	__releases(q->queue_lock)
556 	__acquires(q->queue_lock)
557 {
558 	int i;
559 
560 	lockdep_assert_held(q->queue_lock);
561 	WARN_ON_ONCE(q->mq_ops);
562 
563 	while (true) {
564 		bool drain = false;
565 
566 		/*
567 		 * The caller might be trying to drain @q before its
568 		 * elevator is initialized.
569 		 */
570 		if (q->elevator)
571 			elv_drain_elevator(q);
572 
573 		blkcg_drain_queue(q);
574 
575 		/*
576 		 * This function might be called on a queue which failed
577 		 * driver init after queue creation or is not yet fully
578 		 * active yet.  Some drivers (e.g. fd and loop) get unhappy
579 		 * in such cases.  Kick queue iff dispatch queue has
580 		 * something on it and @q has request_fn set.
581 		 */
582 		if (!list_empty(&q->queue_head) && q->request_fn)
583 			__blk_run_queue(q);
584 
585 		drain |= q->nr_rqs_elvpriv;
586 		drain |= q->request_fn_active;
587 
588 		/*
589 		 * Unfortunately, requests are queued at and tracked from
590 		 * multiple places and there's no single counter which can
591 		 * be drained.  Check all the queues and counters.
592 		 */
593 		if (drain_all) {
594 			struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
595 			drain |= !list_empty(&q->queue_head);
596 			for (i = 0; i < 2; i++) {
597 				drain |= q->nr_rqs[i];
598 				drain |= q->in_flight[i];
599 				if (fq)
600 				    drain |= !list_empty(&fq->flush_queue[i]);
601 			}
602 		}
603 
604 		if (!drain)
605 			break;
606 
607 		spin_unlock_irq(q->queue_lock);
608 
609 		msleep(10);
610 
611 		spin_lock_irq(q->queue_lock);
612 	}
613 
614 	/*
615 	 * With queue marked dead, any woken up waiter will fail the
616 	 * allocation path, so the wakeup chaining is lost and we're
617 	 * left with hung waiters. We need to wake up those waiters.
618 	 */
619 	if (q->request_fn) {
620 		struct request_list *rl;
621 
622 		blk_queue_for_each_rl(rl, q)
623 			for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
624 				wake_up_all(&rl->wait[i]);
625 	}
626 }
627 
628 void blk_drain_queue(struct request_queue *q)
629 {
630 	spin_lock_irq(q->queue_lock);
631 	__blk_drain_queue(q, true);
632 	spin_unlock_irq(q->queue_lock);
633 }
634 
635 /**
636  * blk_queue_bypass_start - enter queue bypass mode
637  * @q: queue of interest
638  *
639  * In bypass mode, only the dispatch FIFO queue of @q is used.  This
640  * function makes @q enter bypass mode and drains all requests which were
641  * throttled or issued before.  On return, it's guaranteed that no request
642  * is being throttled or has ELVPRIV set and blk_queue_bypass() %true
643  * inside queue or RCU read lock.
644  */
645 void blk_queue_bypass_start(struct request_queue *q)
646 {
647 	WARN_ON_ONCE(q->mq_ops);
648 
649 	spin_lock_irq(q->queue_lock);
650 	q->bypass_depth++;
651 	queue_flag_set(QUEUE_FLAG_BYPASS, q);
652 	spin_unlock_irq(q->queue_lock);
653 
654 	/*
655 	 * Queues start drained.  Skip actual draining till init is
656 	 * complete.  This avoids lenghty delays during queue init which
657 	 * can happen many times during boot.
658 	 */
659 	if (blk_queue_init_done(q)) {
660 		spin_lock_irq(q->queue_lock);
661 		__blk_drain_queue(q, false);
662 		spin_unlock_irq(q->queue_lock);
663 
664 		/* ensure blk_queue_bypass() is %true inside RCU read lock */
665 		synchronize_rcu();
666 	}
667 }
668 EXPORT_SYMBOL_GPL(blk_queue_bypass_start);
669 
670 /**
671  * blk_queue_bypass_end - leave queue bypass mode
672  * @q: queue of interest
673  *
674  * Leave bypass mode and restore the normal queueing behavior.
675  *
676  * Note: although blk_queue_bypass_start() is only called for blk-sq queues,
677  * this function is called for both blk-sq and blk-mq queues.
678  */
679 void blk_queue_bypass_end(struct request_queue *q)
680 {
681 	spin_lock_irq(q->queue_lock);
682 	if (!--q->bypass_depth)
683 		queue_flag_clear(QUEUE_FLAG_BYPASS, q);
684 	WARN_ON_ONCE(q->bypass_depth < 0);
685 	spin_unlock_irq(q->queue_lock);
686 }
687 EXPORT_SYMBOL_GPL(blk_queue_bypass_end);
688 
689 void blk_set_queue_dying(struct request_queue *q)
690 {
691 	blk_queue_flag_set(QUEUE_FLAG_DYING, q);
692 
693 	/*
694 	 * When queue DYING flag is set, we need to block new req
695 	 * entering queue, so we call blk_freeze_queue_start() to
696 	 * prevent I/O from crossing blk_queue_enter().
697 	 */
698 	blk_freeze_queue_start(q);
699 
700 	if (q->mq_ops)
701 		blk_mq_wake_waiters(q);
702 	else {
703 		struct request_list *rl;
704 
705 		spin_lock_irq(q->queue_lock);
706 		blk_queue_for_each_rl(rl, q) {
707 			if (rl->rq_pool) {
708 				wake_up_all(&rl->wait[BLK_RW_SYNC]);
709 				wake_up_all(&rl->wait[BLK_RW_ASYNC]);
710 			}
711 		}
712 		spin_unlock_irq(q->queue_lock);
713 	}
714 
715 	/* Make blk_queue_enter() reexamine the DYING flag. */
716 	wake_up_all(&q->mq_freeze_wq);
717 }
718 EXPORT_SYMBOL_GPL(blk_set_queue_dying);
719 
720 /* Unconfigure the I/O scheduler and dissociate from the cgroup controller. */
721 void blk_exit_queue(struct request_queue *q)
722 {
723 	/*
724 	 * Since the I/O scheduler exit code may access cgroup information,
725 	 * perform I/O scheduler exit before disassociating from the block
726 	 * cgroup controller.
727 	 */
728 	if (q->elevator) {
729 		ioc_clear_queue(q);
730 		elevator_exit(q, q->elevator);
731 		q->elevator = NULL;
732 	}
733 
734 	/*
735 	 * Remove all references to @q from the block cgroup controller before
736 	 * restoring @q->queue_lock to avoid that restoring this pointer causes
737 	 * e.g. blkcg_print_blkgs() to crash.
738 	 */
739 	blkcg_exit_queue(q);
740 
741 	/*
742 	 * Since the cgroup code may dereference the @q->backing_dev_info
743 	 * pointer, only decrease its reference count after having removed the
744 	 * association with the block cgroup controller.
745 	 */
746 	bdi_put(q->backing_dev_info);
747 }
748 
749 /**
750  * blk_cleanup_queue - shutdown a request queue
751  * @q: request queue to shutdown
752  *
753  * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
754  * put it.  All future requests will be failed immediately with -ENODEV.
755  */
756 void blk_cleanup_queue(struct request_queue *q)
757 {
758 	spinlock_t *lock = q->queue_lock;
759 
760 	/* mark @q DYING, no new request or merges will be allowed afterwards */
761 	mutex_lock(&q->sysfs_lock);
762 	blk_set_queue_dying(q);
763 	spin_lock_irq(lock);
764 
765 	/*
766 	 * A dying queue is permanently in bypass mode till released.  Note
767 	 * that, unlike blk_queue_bypass_start(), we aren't performing
768 	 * synchronize_rcu() after entering bypass mode to avoid the delay
769 	 * as some drivers create and destroy a lot of queues while
770 	 * probing.  This is still safe because blk_release_queue() will be
771 	 * called only after the queue refcnt drops to zero and nothing,
772 	 * RCU or not, would be traversing the queue by then.
773 	 */
774 	q->bypass_depth++;
775 	queue_flag_set(QUEUE_FLAG_BYPASS, q);
776 
777 	queue_flag_set(QUEUE_FLAG_NOMERGES, q);
778 	queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
779 	queue_flag_set(QUEUE_FLAG_DYING, q);
780 	spin_unlock_irq(lock);
781 	mutex_unlock(&q->sysfs_lock);
782 
783 	/*
784 	 * Drain all requests queued before DYING marking. Set DEAD flag to
785 	 * prevent that q->request_fn() gets invoked after draining finished.
786 	 */
787 	blk_freeze_queue(q);
788 	spin_lock_irq(lock);
789 	queue_flag_set(QUEUE_FLAG_DEAD, q);
790 	spin_unlock_irq(lock);
791 
792 	/*
793 	 * make sure all in-progress dispatch are completed because
794 	 * blk_freeze_queue() can only complete all requests, and
795 	 * dispatch may still be in-progress since we dispatch requests
796 	 * from more than one contexts.
797 	 *
798 	 * No need to quiesce queue if it isn't initialized yet since
799 	 * blk_freeze_queue() should be enough for cases of passthrough
800 	 * request.
801 	 */
802 	if (q->mq_ops && blk_queue_init_done(q))
803 		blk_mq_quiesce_queue(q);
804 
805 	/* for synchronous bio-based driver finish in-flight integrity i/o */
806 	blk_flush_integrity();
807 
808 	/* @q won't process any more request, flush async actions */
809 	del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer);
810 	blk_sync_queue(q);
811 
812 	/*
813 	 * I/O scheduler exit is only safe after the sysfs scheduler attribute
814 	 * has been removed.
815 	 */
816 	WARN_ON_ONCE(q->kobj.state_in_sysfs);
817 
818 	blk_exit_queue(q);
819 
820 	if (q->mq_ops)
821 		blk_mq_free_queue(q);
822 	percpu_ref_exit(&q->q_usage_counter);
823 
824 	spin_lock_irq(lock);
825 	if (q->queue_lock != &q->__queue_lock)
826 		q->queue_lock = &q->__queue_lock;
827 	spin_unlock_irq(lock);
828 
829 	/* @q is and will stay empty, shutdown and put */
830 	blk_put_queue(q);
831 }
832 EXPORT_SYMBOL(blk_cleanup_queue);
833 
834 /* Allocate memory local to the request queue */
835 static void *alloc_request_simple(gfp_t gfp_mask, void *data)
836 {
837 	struct request_queue *q = data;
838 
839 	return kmem_cache_alloc_node(request_cachep, gfp_mask, q->node);
840 }
841 
842 static void free_request_simple(void *element, void *data)
843 {
844 	kmem_cache_free(request_cachep, element);
845 }
846 
847 static void *alloc_request_size(gfp_t gfp_mask, void *data)
848 {
849 	struct request_queue *q = data;
850 	struct request *rq;
851 
852 	rq = kmalloc_node(sizeof(struct request) + q->cmd_size, gfp_mask,
853 			q->node);
854 	if (rq && q->init_rq_fn && q->init_rq_fn(q, rq, gfp_mask) < 0) {
855 		kfree(rq);
856 		rq = NULL;
857 	}
858 	return rq;
859 }
860 
861 static void free_request_size(void *element, void *data)
862 {
863 	struct request_queue *q = data;
864 
865 	if (q->exit_rq_fn)
866 		q->exit_rq_fn(q, element);
867 	kfree(element);
868 }
869 
870 int blk_init_rl(struct request_list *rl, struct request_queue *q,
871 		gfp_t gfp_mask)
872 {
873 	if (unlikely(rl->rq_pool) || q->mq_ops)
874 		return 0;
875 
876 	rl->q = q;
877 	rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
878 	rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
879 	init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
880 	init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
881 
882 	if (q->cmd_size) {
883 		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
884 				alloc_request_size, free_request_size,
885 				q, gfp_mask, q->node);
886 	} else {
887 		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
888 				alloc_request_simple, free_request_simple,
889 				q, gfp_mask, q->node);
890 	}
891 	if (!rl->rq_pool)
892 		return -ENOMEM;
893 
894 	if (rl != &q->root_rl)
895 		WARN_ON_ONCE(!blk_get_queue(q));
896 
897 	return 0;
898 }
899 
900 void blk_exit_rl(struct request_queue *q, struct request_list *rl)
901 {
902 	if (rl->rq_pool) {
903 		mempool_destroy(rl->rq_pool);
904 		if (rl != &q->root_rl)
905 			blk_put_queue(q);
906 	}
907 }
908 
909 struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
910 {
911 	return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE, NULL);
912 }
913 EXPORT_SYMBOL(blk_alloc_queue);
914 
915 /**
916  * blk_queue_enter() - try to increase q->q_usage_counter
917  * @q: request queue pointer
918  * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PREEMPT
919  */
920 int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
921 {
922 	const bool pm = flags & BLK_MQ_REQ_PREEMPT;
923 
924 	while (true) {
925 		bool success = false;
926 
927 		rcu_read_lock();
928 		if (percpu_ref_tryget_live(&q->q_usage_counter)) {
929 			/*
930 			 * The code that increments the pm_only counter is
931 			 * responsible for ensuring that that counter is
932 			 * globally visible before the queue is unfrozen.
933 			 */
934 			if (pm || !blk_queue_pm_only(q)) {
935 				success = true;
936 			} else {
937 				percpu_ref_put(&q->q_usage_counter);
938 			}
939 		}
940 		rcu_read_unlock();
941 
942 		if (success)
943 			return 0;
944 
945 		if (flags & BLK_MQ_REQ_NOWAIT)
946 			return -EBUSY;
947 
948 		/*
949 		 * read pair of barrier in blk_freeze_queue_start(),
950 		 * we need to order reading __PERCPU_REF_DEAD flag of
951 		 * .q_usage_counter and reading .mq_freeze_depth or
952 		 * queue dying flag, otherwise the following wait may
953 		 * never return if the two reads are reordered.
954 		 */
955 		smp_rmb();
956 
957 		wait_event(q->mq_freeze_wq,
958 			   (atomic_read(&q->mq_freeze_depth) == 0 &&
959 			    (pm || (blk_pm_request_resume(q),
960 				    !blk_queue_pm_only(q)))) ||
961 			   blk_queue_dying(q));
962 		if (blk_queue_dying(q))
963 			return -ENODEV;
964 	}
965 }
966 
967 void blk_queue_exit(struct request_queue *q)
968 {
969 	percpu_ref_put(&q->q_usage_counter);
970 }
971 
972 static void blk_queue_usage_counter_release(struct percpu_ref *ref)
973 {
974 	struct request_queue *q =
975 		container_of(ref, struct request_queue, q_usage_counter);
976 
977 	wake_up_all(&q->mq_freeze_wq);
978 }
979 
980 static void blk_rq_timed_out_timer(struct timer_list *t)
981 {
982 	struct request_queue *q = from_timer(q, t, timeout);
983 
984 	kblockd_schedule_work(&q->timeout_work);
985 }
986 
987 /**
988  * blk_alloc_queue_node - allocate a request queue
989  * @gfp_mask: memory allocation flags
990  * @node_id: NUMA node to allocate memory from
991  * @lock: For legacy queues, pointer to a spinlock that will be used to e.g.
992  *        serialize calls to the legacy .request_fn() callback. Ignored for
993  *	  blk-mq request queues.
994  *
995  * Note: pass the queue lock as the third argument to this function instead of
996  * setting the queue lock pointer explicitly to avoid triggering a sporadic
997  * crash in the blkcg code. This function namely calls blkcg_init_queue() and
998  * the queue lock pointer must be set before blkcg_init_queue() is called.
999  */
1000 struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
1001 					   spinlock_t *lock)
1002 {
1003 	struct request_queue *q;
1004 	int ret;
1005 
1006 	q = kmem_cache_alloc_node(blk_requestq_cachep,
1007 				gfp_mask | __GFP_ZERO, node_id);
1008 	if (!q)
1009 		return NULL;
1010 
1011 	INIT_LIST_HEAD(&q->queue_head);
1012 	q->last_merge = NULL;
1013 	q->end_sector = 0;
1014 	q->boundary_rq = NULL;
1015 
1016 	q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
1017 	if (q->id < 0)
1018 		goto fail_q;
1019 
1020 	ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
1021 	if (ret)
1022 		goto fail_id;
1023 
1024 	q->backing_dev_info = bdi_alloc_node(gfp_mask, node_id);
1025 	if (!q->backing_dev_info)
1026 		goto fail_split;
1027 
1028 	q->stats = blk_alloc_queue_stats();
1029 	if (!q->stats)
1030 		goto fail_stats;
1031 
1032 	q->backing_dev_info->ra_pages =
1033 			(VM_MAX_READAHEAD * 1024) / PAGE_SIZE;
1034 	q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK;
1035 	q->backing_dev_info->name = "block";
1036 	q->node = node_id;
1037 
1038 	timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,
1039 		    laptop_mode_timer_fn, 0);
1040 	timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
1041 	INIT_WORK(&q->timeout_work, NULL);
1042 	INIT_LIST_HEAD(&q->timeout_list);
1043 	INIT_LIST_HEAD(&q->icq_list);
1044 #ifdef CONFIG_BLK_CGROUP
1045 	INIT_LIST_HEAD(&q->blkg_list);
1046 #endif
1047 	INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
1048 
1049 	kobject_init(&q->kobj, &blk_queue_ktype);
1050 
1051 #ifdef CONFIG_BLK_DEV_IO_TRACE
1052 	mutex_init(&q->blk_trace_mutex);
1053 #endif
1054 	mutex_init(&q->sysfs_lock);
1055 	spin_lock_init(&q->__queue_lock);
1056 
1057 	q->queue_lock = lock ? : &q->__queue_lock;
1058 
1059 	/*
1060 	 * A queue starts its life with bypass turned on to avoid
1061 	 * unnecessary bypass on/off overhead and nasty surprises during
1062 	 * init.  The initial bypass will be finished when the queue is
1063 	 * registered by blk_register_queue().
1064 	 */
1065 	q->bypass_depth = 1;
1066 	queue_flag_set_unlocked(QUEUE_FLAG_BYPASS, q);
1067 
1068 	init_waitqueue_head(&q->mq_freeze_wq);
1069 
1070 	/*
1071 	 * Init percpu_ref in atomic mode so that it's faster to shutdown.
1072 	 * See blk_register_queue() for details.
1073 	 */
1074 	if (percpu_ref_init(&q->q_usage_counter,
1075 				blk_queue_usage_counter_release,
1076 				PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
1077 		goto fail_bdi;
1078 
1079 	if (blkcg_init_queue(q))
1080 		goto fail_ref;
1081 
1082 	return q;
1083 
1084 fail_ref:
1085 	percpu_ref_exit(&q->q_usage_counter);
1086 fail_bdi:
1087 	blk_free_queue_stats(q->stats);
1088 fail_stats:
1089 	bdi_put(q->backing_dev_info);
1090 fail_split:
1091 	bioset_exit(&q->bio_split);
1092 fail_id:
1093 	ida_simple_remove(&blk_queue_ida, q->id);
1094 fail_q:
1095 	kmem_cache_free(blk_requestq_cachep, q);
1096 	return NULL;
1097 }
1098 EXPORT_SYMBOL(blk_alloc_queue_node);
1099 
1100 /**
1101  * blk_init_queue  - prepare a request queue for use with a block device
1102  * @rfn:  The function to be called to process requests that have been
1103  *        placed on the queue.
1104  * @lock: Request queue spin lock
1105  *
1106  * Description:
1107  *    If a block device wishes to use the standard request handling procedures,
1108  *    which sorts requests and coalesces adjacent requests, then it must
1109  *    call blk_init_queue().  The function @rfn will be called when there
1110  *    are requests on the queue that need to be processed.  If the device
1111  *    supports plugging, then @rfn may not be called immediately when requests
1112  *    are available on the queue, but may be called at some time later instead.
1113  *    Plugged queues are generally unplugged when a buffer belonging to one
1114  *    of the requests on the queue is needed, or due to memory pressure.
1115  *
1116  *    @rfn is not required, or even expected, to remove all requests off the
1117  *    queue, but only as many as it can handle at a time.  If it does leave
1118  *    requests on the queue, it is responsible for arranging that the requests
1119  *    get dealt with eventually.
1120  *
1121  *    The queue spin lock must be held while manipulating the requests on the
1122  *    request queue; this lock will be taken also from interrupt context, so irq
1123  *    disabling is needed for it.
1124  *
1125  *    Function returns a pointer to the initialized request queue, or %NULL if
1126  *    it didn't succeed.
1127  *
1128  * Note:
1129  *    blk_init_queue() must be paired with a blk_cleanup_queue() call
1130  *    when the block device is deactivated (such as at module unload).
1131  **/
1132 
1133 struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
1134 {
1135 	return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
1136 }
1137 EXPORT_SYMBOL(blk_init_queue);
1138 
1139 struct request_queue *
1140 blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
1141 {
1142 	struct request_queue *q;
1143 
1144 	q = blk_alloc_queue_node(GFP_KERNEL, node_id, lock);
1145 	if (!q)
1146 		return NULL;
1147 
1148 	q->request_fn = rfn;
1149 	if (blk_init_allocated_queue(q) < 0) {
1150 		blk_cleanup_queue(q);
1151 		return NULL;
1152 	}
1153 
1154 	return q;
1155 }
1156 EXPORT_SYMBOL(blk_init_queue_node);
1157 
1158 static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio);
1159 
1160 
1161 int blk_init_allocated_queue(struct request_queue *q)
1162 {
1163 	WARN_ON_ONCE(q->mq_ops);
1164 
1165 	q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, q->cmd_size, GFP_KERNEL);
1166 	if (!q->fq)
1167 		return -ENOMEM;
1168 
1169 	if (q->init_rq_fn && q->init_rq_fn(q, q->fq->flush_rq, GFP_KERNEL))
1170 		goto out_free_flush_queue;
1171 
1172 	if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
1173 		goto out_exit_flush_rq;
1174 
1175 	INIT_WORK(&q->timeout_work, blk_timeout_work);
1176 	q->queue_flags		|= QUEUE_FLAG_DEFAULT;
1177 
1178 	/*
1179 	 * This also sets hw/phys segments, boundary and size
1180 	 */
1181 	blk_queue_make_request(q, blk_queue_bio);
1182 
1183 	q->sg_reserved_size = INT_MAX;
1184 
1185 	if (elevator_init(q))
1186 		goto out_exit_flush_rq;
1187 	return 0;
1188 
1189 out_exit_flush_rq:
1190 	if (q->exit_rq_fn)
1191 		q->exit_rq_fn(q, q->fq->flush_rq);
1192 out_free_flush_queue:
1193 	blk_free_flush_queue(q->fq);
1194 	q->fq = NULL;
1195 	return -ENOMEM;
1196 }
1197 EXPORT_SYMBOL(blk_init_allocated_queue);
1198 
1199 bool blk_get_queue(struct request_queue *q)
1200 {
1201 	if (likely(!blk_queue_dying(q))) {
1202 		__blk_get_queue(q);
1203 		return true;
1204 	}
1205 
1206 	return false;
1207 }
1208 EXPORT_SYMBOL(blk_get_queue);
1209 
1210 static inline void blk_free_request(struct request_list *rl, struct request *rq)
1211 {
1212 	if (rq->rq_flags & RQF_ELVPRIV) {
1213 		elv_put_request(rl->q, rq);
1214 		if (rq->elv.icq)
1215 			put_io_context(rq->elv.icq->ioc);
1216 	}
1217 
1218 	mempool_free(rq, rl->rq_pool);
1219 }
1220 
1221 /*
1222  * ioc_batching returns true if the ioc is a valid batching request and
1223  * should be given priority access to a request.
1224  */
1225 static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
1226 {
1227 	if (!ioc)
1228 		return 0;
1229 
1230 	/*
1231 	 * Make sure the process is able to allocate at least 1 request
1232 	 * even if the batch times out, otherwise we could theoretically
1233 	 * lose wakeups.
1234 	 */
1235 	return ioc->nr_batch_requests == q->nr_batching ||
1236 		(ioc->nr_batch_requests > 0
1237 		&& time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
1238 }
1239 
1240 /*
1241  * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
1242  * will cause the process to be a "batcher" on all queues in the system. This
1243  * is the behaviour we want though - once it gets a wakeup it should be given
1244  * a nice run.
1245  */
1246 static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
1247 {
1248 	if (!ioc || ioc_batching(q, ioc))
1249 		return;
1250 
1251 	ioc->nr_batch_requests = q->nr_batching;
1252 	ioc->last_waited = jiffies;
1253 }
1254 
1255 static void __freed_request(struct request_list *rl, int sync)
1256 {
1257 	struct request_queue *q = rl->q;
1258 
1259 	if (rl->count[sync] < queue_congestion_off_threshold(q))
1260 		blk_clear_congested(rl, sync);
1261 
1262 	if (rl->count[sync] + 1 <= q->nr_requests) {
1263 		if (waitqueue_active(&rl->wait[sync]))
1264 			wake_up(&rl->wait[sync]);
1265 
1266 		blk_clear_rl_full(rl, sync);
1267 	}
1268 }
1269 
1270 /*
1271  * A request has just been released.  Account for it, update the full and
1272  * congestion status, wake up any waiters.   Called under q->queue_lock.
1273  */
1274 static void freed_request(struct request_list *rl, bool sync,
1275 		req_flags_t rq_flags)
1276 {
1277 	struct request_queue *q = rl->q;
1278 
1279 	q->nr_rqs[sync]--;
1280 	rl->count[sync]--;
1281 	if (rq_flags & RQF_ELVPRIV)
1282 		q->nr_rqs_elvpriv--;
1283 
1284 	__freed_request(rl, sync);
1285 
1286 	if (unlikely(rl->starved[sync ^ 1]))
1287 		__freed_request(rl, sync ^ 1);
1288 }
1289 
1290 int blk_update_nr_requests(struct request_queue *q, unsigned int nr)
1291 {
1292 	struct request_list *rl;
1293 	int on_thresh, off_thresh;
1294 
1295 	WARN_ON_ONCE(q->mq_ops);
1296 
1297 	spin_lock_irq(q->queue_lock);
1298 	q->nr_requests = nr;
1299 	blk_queue_congestion_threshold(q);
1300 	on_thresh = queue_congestion_on_threshold(q);
1301 	off_thresh = queue_congestion_off_threshold(q);
1302 
1303 	blk_queue_for_each_rl(rl, q) {
1304 		if (rl->count[BLK_RW_SYNC] >= on_thresh)
1305 			blk_set_congested(rl, BLK_RW_SYNC);
1306 		else if (rl->count[BLK_RW_SYNC] < off_thresh)
1307 			blk_clear_congested(rl, BLK_RW_SYNC);
1308 
1309 		if (rl->count[BLK_RW_ASYNC] >= on_thresh)
1310 			blk_set_congested(rl, BLK_RW_ASYNC);
1311 		else if (rl->count[BLK_RW_ASYNC] < off_thresh)
1312 			blk_clear_congested(rl, BLK_RW_ASYNC);
1313 
1314 		if (rl->count[BLK_RW_SYNC] >= q->nr_requests) {
1315 			blk_set_rl_full(rl, BLK_RW_SYNC);
1316 		} else {
1317 			blk_clear_rl_full(rl, BLK_RW_SYNC);
1318 			wake_up(&rl->wait[BLK_RW_SYNC]);
1319 		}
1320 
1321 		if (rl->count[BLK_RW_ASYNC] >= q->nr_requests) {
1322 			blk_set_rl_full(rl, BLK_RW_ASYNC);
1323 		} else {
1324 			blk_clear_rl_full(rl, BLK_RW_ASYNC);
1325 			wake_up(&rl->wait[BLK_RW_ASYNC]);
1326 		}
1327 	}
1328 
1329 	spin_unlock_irq(q->queue_lock);
1330 	return 0;
1331 }
1332 
1333 /**
1334  * __get_request - get a free request
1335  * @rl: request list to allocate from
1336  * @op: operation and flags
1337  * @bio: bio to allocate request for (can be %NULL)
1338  * @flags: BLQ_MQ_REQ_* flags
1339  * @gfp_mask: allocator flags
1340  *
1341  * Get a free request from @q.  This function may fail under memory
1342  * pressure or if @q is dead.
1343  *
1344  * Must be called with @q->queue_lock held and,
1345  * Returns ERR_PTR on failure, with @q->queue_lock held.
1346  * Returns request pointer on success, with @q->queue_lock *not held*.
1347  */
1348 static struct request *__get_request(struct request_list *rl, unsigned int op,
1349 		struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp_mask)
1350 {
1351 	struct request_queue *q = rl->q;
1352 	struct request *rq;
1353 	struct elevator_type *et = q->elevator->type;
1354 	struct io_context *ioc = rq_ioc(bio);
1355 	struct io_cq *icq = NULL;
1356 	const bool is_sync = op_is_sync(op);
1357 	int may_queue;
1358 	req_flags_t rq_flags = RQF_ALLOCED;
1359 
1360 	lockdep_assert_held(q->queue_lock);
1361 
1362 	if (unlikely(blk_queue_dying(q)))
1363 		return ERR_PTR(-ENODEV);
1364 
1365 	may_queue = elv_may_queue(q, op);
1366 	if (may_queue == ELV_MQUEUE_NO)
1367 		goto rq_starved;
1368 
1369 	if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
1370 		if (rl->count[is_sync]+1 >= q->nr_requests) {
1371 			/*
1372 			 * The queue will fill after this allocation, so set
1373 			 * it as full, and mark this process as "batching".
1374 			 * This process will be allowed to complete a batch of
1375 			 * requests, others will be blocked.
1376 			 */
1377 			if (!blk_rl_full(rl, is_sync)) {
1378 				ioc_set_batching(q, ioc);
1379 				blk_set_rl_full(rl, is_sync);
1380 			} else {
1381 				if (may_queue != ELV_MQUEUE_MUST
1382 						&& !ioc_batching(q, ioc)) {
1383 					/*
1384 					 * The queue is full and the allocating
1385 					 * process is not a "batcher", and not
1386 					 * exempted by the IO scheduler
1387 					 */
1388 					return ERR_PTR(-ENOMEM);
1389 				}
1390 			}
1391 		}
1392 		blk_set_congested(rl, is_sync);
1393 	}
1394 
1395 	/*
1396 	 * Only allow batching queuers to allocate up to 50% over the defined
1397 	 * limit of requests, otherwise we could have thousands of requests
1398 	 * allocated with any setting of ->nr_requests
1399 	 */
1400 	if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
1401 		return ERR_PTR(-ENOMEM);
1402 
1403 	q->nr_rqs[is_sync]++;
1404 	rl->count[is_sync]++;
1405 	rl->starved[is_sync] = 0;
1406 
1407 	/*
1408 	 * Decide whether the new request will be managed by elevator.  If
1409 	 * so, mark @rq_flags and increment elvpriv.  Non-zero elvpriv will
1410 	 * prevent the current elevator from being destroyed until the new
1411 	 * request is freed.  This guarantees icq's won't be destroyed and
1412 	 * makes creating new ones safe.
1413 	 *
1414 	 * Flush requests do not use the elevator so skip initialization.
1415 	 * This allows a request to share the flush and elevator data.
1416 	 *
1417 	 * Also, lookup icq while holding queue_lock.  If it doesn't exist,
1418 	 * it will be created after releasing queue_lock.
1419 	 */
1420 	if (!op_is_flush(op) && !blk_queue_bypass(q)) {
1421 		rq_flags |= RQF_ELVPRIV;
1422 		q->nr_rqs_elvpriv++;
1423 		if (et->icq_cache && ioc)
1424 			icq = ioc_lookup_icq(ioc, q);
1425 	}
1426 
1427 	if (blk_queue_io_stat(q))
1428 		rq_flags |= RQF_IO_STAT;
1429 	spin_unlock_irq(q->queue_lock);
1430 
1431 	/* allocate and init request */
1432 	rq = mempool_alloc(rl->rq_pool, gfp_mask);
1433 	if (!rq)
1434 		goto fail_alloc;
1435 
1436 	blk_rq_init(q, rq);
1437 	blk_rq_set_rl(rq, rl);
1438 	rq->cmd_flags = op;
1439 	rq->rq_flags = rq_flags;
1440 	if (flags & BLK_MQ_REQ_PREEMPT)
1441 		rq->rq_flags |= RQF_PREEMPT;
1442 
1443 	/* init elvpriv */
1444 	if (rq_flags & RQF_ELVPRIV) {
1445 		if (unlikely(et->icq_cache && !icq)) {
1446 			if (ioc)
1447 				icq = ioc_create_icq(ioc, q, gfp_mask);
1448 			if (!icq)
1449 				goto fail_elvpriv;
1450 		}
1451 
1452 		rq->elv.icq = icq;
1453 		if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
1454 			goto fail_elvpriv;
1455 
1456 		/* @rq->elv.icq holds io_context until @rq is freed */
1457 		if (icq)
1458 			get_io_context(icq->ioc);
1459 	}
1460 out:
1461 	/*
1462 	 * ioc may be NULL here, and ioc_batching will be false. That's
1463 	 * OK, if the queue is under the request limit then requests need
1464 	 * not count toward the nr_batch_requests limit. There will always
1465 	 * be some limit enforced by BLK_BATCH_TIME.
1466 	 */
1467 	if (ioc_batching(q, ioc))
1468 		ioc->nr_batch_requests--;
1469 
1470 	trace_block_getrq(q, bio, op);
1471 	return rq;
1472 
1473 fail_elvpriv:
1474 	/*
1475 	 * elvpriv init failed.  ioc, icq and elvpriv aren't mempool backed
1476 	 * and may fail indefinitely under memory pressure and thus
1477 	 * shouldn't stall IO.  Treat this request as !elvpriv.  This will
1478 	 * disturb iosched and blkcg but weird is bettern than dead.
1479 	 */
1480 	printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n",
1481 			   __func__, dev_name(q->backing_dev_info->dev));
1482 
1483 	rq->rq_flags &= ~RQF_ELVPRIV;
1484 	rq->elv.icq = NULL;
1485 
1486 	spin_lock_irq(q->queue_lock);
1487 	q->nr_rqs_elvpriv--;
1488 	spin_unlock_irq(q->queue_lock);
1489 	goto out;
1490 
1491 fail_alloc:
1492 	/*
1493 	 * Allocation failed presumably due to memory. Undo anything we
1494 	 * might have messed up.
1495 	 *
1496 	 * Allocating task should really be put onto the front of the wait
1497 	 * queue, but this is pretty rare.
1498 	 */
1499 	spin_lock_irq(q->queue_lock);
1500 	freed_request(rl, is_sync, rq_flags);
1501 
1502 	/*
1503 	 * in the very unlikely event that allocation failed and no
1504 	 * requests for this direction was pending, mark us starved so that
1505 	 * freeing of a request in the other direction will notice
1506 	 * us. another possible fix would be to split the rq mempool into
1507 	 * READ and WRITE
1508 	 */
1509 rq_starved:
1510 	if (unlikely(rl->count[is_sync] == 0))
1511 		rl->starved[is_sync] = 1;
1512 	return ERR_PTR(-ENOMEM);
1513 }
1514 
1515 /**
1516  * get_request - get a free request
1517  * @q: request_queue to allocate request from
1518  * @op: operation and flags
1519  * @bio: bio to allocate request for (can be %NULL)
1520  * @flags: BLK_MQ_REQ_* flags.
1521  * @gfp: allocator flags
1522  *
1523  * Get a free request from @q.  If %BLK_MQ_REQ_NOWAIT is set in @flags,
1524  * this function keeps retrying under memory pressure and fails iff @q is dead.
1525  *
1526  * Must be called with @q->queue_lock held and,
1527  * Returns ERR_PTR on failure, with @q->queue_lock held.
1528  * Returns request pointer on success, with @q->queue_lock *not held*.
1529  */
1530 static struct request *get_request(struct request_queue *q, unsigned int op,
1531 		struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp)
1532 {
1533 	const bool is_sync = op_is_sync(op);
1534 	DEFINE_WAIT(wait);
1535 	struct request_list *rl;
1536 	struct request *rq;
1537 
1538 	lockdep_assert_held(q->queue_lock);
1539 	WARN_ON_ONCE(q->mq_ops);
1540 
1541 	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */
1542 retry:
1543 	rq = __get_request(rl, op, bio, flags, gfp);
1544 	if (!IS_ERR(rq))
1545 		return rq;
1546 
1547 	if (op & REQ_NOWAIT) {
1548 		blk_put_rl(rl);
1549 		return ERR_PTR(-EAGAIN);
1550 	}
1551 
1552 	if ((flags & BLK_MQ_REQ_NOWAIT) || unlikely(blk_queue_dying(q))) {
1553 		blk_put_rl(rl);
1554 		return rq;
1555 	}
1556 
1557 	/* wait on @rl and retry */
1558 	prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
1559 				  TASK_UNINTERRUPTIBLE);
1560 
1561 	trace_block_sleeprq(q, bio, op);
1562 
1563 	spin_unlock_irq(q->queue_lock);
1564 	io_schedule();
1565 
1566 	/*
1567 	 * After sleeping, we become a "batching" process and will be able
1568 	 * to allocate at least one request, and up to a big batch of them
1569 	 * for a small period time.  See ioc_batching, ioc_set_batching
1570 	 */
1571 	ioc_set_batching(q, current->io_context);
1572 
1573 	spin_lock_irq(q->queue_lock);
1574 	finish_wait(&rl->wait[is_sync], &wait);
1575 
1576 	goto retry;
1577 }
1578 
1579 /* flags: BLK_MQ_REQ_PREEMPT and/or BLK_MQ_REQ_NOWAIT. */
1580 static struct request *blk_old_get_request(struct request_queue *q,
1581 				unsigned int op, blk_mq_req_flags_t flags)
1582 {
1583 	struct request *rq;
1584 	gfp_t gfp_mask = flags & BLK_MQ_REQ_NOWAIT ? GFP_ATOMIC : GFP_NOIO;
1585 	int ret = 0;
1586 
1587 	WARN_ON_ONCE(q->mq_ops);
1588 
1589 	/* create ioc upfront */
1590 	create_io_context(gfp_mask, q->node);
1591 
1592 	ret = blk_queue_enter(q, flags);
1593 	if (ret)
1594 		return ERR_PTR(ret);
1595 	spin_lock_irq(q->queue_lock);
1596 	rq = get_request(q, op, NULL, flags, gfp_mask);
1597 	if (IS_ERR(rq)) {
1598 		spin_unlock_irq(q->queue_lock);
1599 		blk_queue_exit(q);
1600 		return rq;
1601 	}
1602 
1603 	/* q->queue_lock is unlocked at this point */
1604 	rq->__data_len = 0;
1605 	rq->__sector = (sector_t) -1;
1606 	rq->bio = rq->biotail = NULL;
1607 	return rq;
1608 }
1609 
1610 /**
1611  * blk_get_request - allocate a request
1612  * @q: request queue to allocate a request for
1613  * @op: operation (REQ_OP_*) and REQ_* flags, e.g. REQ_SYNC.
1614  * @flags: BLK_MQ_REQ_* flags, e.g. BLK_MQ_REQ_NOWAIT.
1615  */
1616 struct request *blk_get_request(struct request_queue *q, unsigned int op,
1617 				blk_mq_req_flags_t flags)
1618 {
1619 	struct request *req;
1620 
1621 	WARN_ON_ONCE(op & REQ_NOWAIT);
1622 	WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PREEMPT));
1623 
1624 	if (q->mq_ops) {
1625 		req = blk_mq_alloc_request(q, op, flags);
1626 		if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)
1627 			q->mq_ops->initialize_rq_fn(req);
1628 	} else {
1629 		req = blk_old_get_request(q, op, flags);
1630 		if (!IS_ERR(req) && q->initialize_rq_fn)
1631 			q->initialize_rq_fn(req);
1632 	}
1633 
1634 	return req;
1635 }
1636 EXPORT_SYMBOL(blk_get_request);
1637 
1638 /**
1639  * blk_requeue_request - put a request back on queue
1640  * @q:		request queue where request should be inserted
1641  * @rq:		request to be inserted
1642  *
1643  * Description:
1644  *    Drivers often keep queueing requests until the hardware cannot accept
1645  *    more, when that condition happens we need to put the request back
1646  *    on the queue. Must be called with queue lock held.
1647  */
1648 void blk_requeue_request(struct request_queue *q, struct request *rq)
1649 {
1650 	lockdep_assert_held(q->queue_lock);
1651 	WARN_ON_ONCE(q->mq_ops);
1652 
1653 	blk_delete_timer(rq);
1654 	blk_clear_rq_complete(rq);
1655 	trace_block_rq_requeue(q, rq);
1656 	rq_qos_requeue(q, rq);
1657 
1658 	if (rq->rq_flags & RQF_QUEUED)
1659 		blk_queue_end_tag(q, rq);
1660 
1661 	BUG_ON(blk_queued_rq(rq));
1662 
1663 	elv_requeue_request(q, rq);
1664 }
1665 EXPORT_SYMBOL(blk_requeue_request);
1666 
1667 static void add_acct_request(struct request_queue *q, struct request *rq,
1668 			     int where)
1669 {
1670 	blk_account_io_start(rq, true);
1671 	__elv_add_request(q, rq, where);
1672 }
1673 
1674 static void part_round_stats_single(struct request_queue *q, int cpu,
1675 				    struct hd_struct *part, unsigned long now,
1676 				    unsigned int inflight)
1677 {
1678 	if (inflight) {
1679 		__part_stat_add(cpu, part, time_in_queue,
1680 				inflight * (now - part->stamp));
1681 		__part_stat_add(cpu, part, io_ticks, (now - part->stamp));
1682 	}
1683 	part->stamp = now;
1684 }
1685 
1686 /**
1687  * part_round_stats() - Round off the performance stats on a struct disk_stats.
1688  * @q: target block queue
1689  * @cpu: cpu number for stats access
1690  * @part: target partition
1691  *
1692  * The average IO queue length and utilisation statistics are maintained
1693  * by observing the current state of the queue length and the amount of
1694  * time it has been in this state for.
1695  *
1696  * Normally, that accounting is done on IO completion, but that can result
1697  * in more than a second's worth of IO being accounted for within any one
1698  * second, leading to >100% utilisation.  To deal with that, we call this
1699  * function to do a round-off before returning the results when reading
1700  * /proc/diskstats.  This accounts immediately for all queue usage up to
1701  * the current jiffies and restarts the counters again.
1702  */
1703 void part_round_stats(struct request_queue *q, int cpu, struct hd_struct *part)
1704 {
1705 	struct hd_struct *part2 = NULL;
1706 	unsigned long now = jiffies;
1707 	unsigned int inflight[2];
1708 	int stats = 0;
1709 
1710 	if (part->stamp != now)
1711 		stats |= 1;
1712 
1713 	if (part->partno) {
1714 		part2 = &part_to_disk(part)->part0;
1715 		if (part2->stamp != now)
1716 			stats |= 2;
1717 	}
1718 
1719 	if (!stats)
1720 		return;
1721 
1722 	part_in_flight(q, part, inflight);
1723 
1724 	if (stats & 2)
1725 		part_round_stats_single(q, cpu, part2, now, inflight[1]);
1726 	if (stats & 1)
1727 		part_round_stats_single(q, cpu, part, now, inflight[0]);
1728 }
1729 EXPORT_SYMBOL_GPL(part_round_stats);
1730 
1731 void __blk_put_request(struct request_queue *q, struct request *req)
1732 {
1733 	req_flags_t rq_flags = req->rq_flags;
1734 
1735 	if (unlikely(!q))
1736 		return;
1737 
1738 	if (q->mq_ops) {
1739 		blk_mq_free_request(req);
1740 		return;
1741 	}
1742 
1743 	lockdep_assert_held(q->queue_lock);
1744 
1745 	blk_req_zone_write_unlock(req);
1746 	blk_pm_put_request(req);
1747 	blk_pm_mark_last_busy(req);
1748 
1749 	elv_completed_request(q, req);
1750 
1751 	/* this is a bio leak */
1752 	WARN_ON(req->bio != NULL);
1753 
1754 	rq_qos_done(q, req);
1755 
1756 	/*
1757 	 * Request may not have originated from ll_rw_blk. if not,
1758 	 * it didn't come out of our reserved rq pools
1759 	 */
1760 	if (rq_flags & RQF_ALLOCED) {
1761 		struct request_list *rl = blk_rq_rl(req);
1762 		bool sync = op_is_sync(req->cmd_flags);
1763 
1764 		BUG_ON(!list_empty(&req->queuelist));
1765 		BUG_ON(ELV_ON_HASH(req));
1766 
1767 		blk_free_request(rl, req);
1768 		freed_request(rl, sync, rq_flags);
1769 		blk_put_rl(rl);
1770 		blk_queue_exit(q);
1771 	}
1772 }
1773 EXPORT_SYMBOL_GPL(__blk_put_request);
1774 
1775 void blk_put_request(struct request *req)
1776 {
1777 	struct request_queue *q = req->q;
1778 
1779 	if (q->mq_ops)
1780 		blk_mq_free_request(req);
1781 	else {
1782 		unsigned long flags;
1783 
1784 		spin_lock_irqsave(q->queue_lock, flags);
1785 		__blk_put_request(q, req);
1786 		spin_unlock_irqrestore(q->queue_lock, flags);
1787 	}
1788 }
1789 EXPORT_SYMBOL(blk_put_request);
1790 
1791 bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
1792 			    struct bio *bio)
1793 {
1794 	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1795 
1796 	if (!ll_back_merge_fn(q, req, bio))
1797 		return false;
1798 
1799 	trace_block_bio_backmerge(q, req, bio);
1800 
1801 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1802 		blk_rq_set_mixed_merge(req);
1803 
1804 	req->biotail->bi_next = bio;
1805 	req->biotail = bio;
1806 	req->__data_len += bio->bi_iter.bi_size;
1807 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1808 
1809 	blk_account_io_start(req, false);
1810 	return true;
1811 }
1812 
1813 bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
1814 			     struct bio *bio)
1815 {
1816 	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1817 
1818 	if (!ll_front_merge_fn(q, req, bio))
1819 		return false;
1820 
1821 	trace_block_bio_frontmerge(q, req, bio);
1822 
1823 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1824 		blk_rq_set_mixed_merge(req);
1825 
1826 	bio->bi_next = req->bio;
1827 	req->bio = bio;
1828 
1829 	req->__sector = bio->bi_iter.bi_sector;
1830 	req->__data_len += bio->bi_iter.bi_size;
1831 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1832 
1833 	blk_account_io_start(req, false);
1834 	return true;
1835 }
1836 
1837 bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
1838 		struct bio *bio)
1839 {
1840 	unsigned short segments = blk_rq_nr_discard_segments(req);
1841 
1842 	if (segments >= queue_max_discard_segments(q))
1843 		goto no_merge;
1844 	if (blk_rq_sectors(req) + bio_sectors(bio) >
1845 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
1846 		goto no_merge;
1847 
1848 	req->biotail->bi_next = bio;
1849 	req->biotail = bio;
1850 	req->__data_len += bio->bi_iter.bi_size;
1851 	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1852 	req->nr_phys_segments = segments + 1;
1853 
1854 	blk_account_io_start(req, false);
1855 	return true;
1856 no_merge:
1857 	req_set_nomerge(q, req);
1858 	return false;
1859 }
1860 
1861 /**
1862  * blk_attempt_plug_merge - try to merge with %current's plugged list
1863  * @q: request_queue new bio is being queued at
1864  * @bio: new bio being queued
1865  * @request_count: out parameter for number of traversed plugged requests
1866  * @same_queue_rq: pointer to &struct request that gets filled in when
1867  * another request associated with @q is found on the plug list
1868  * (optional, may be %NULL)
1869  *
1870  * Determine whether @bio being queued on @q can be merged with a request
1871  * on %current's plugged list.  Returns %true if merge was successful,
1872  * otherwise %false.
1873  *
1874  * Plugging coalesces IOs from the same issuer for the same purpose without
1875  * going through @q->queue_lock.  As such it's more of an issuing mechanism
1876  * than scheduling, and the request, while may have elvpriv data, is not
1877  * added on the elevator at this point.  In addition, we don't have
1878  * reliable access to the elevator outside queue lock.  Only check basic
1879  * merging parameters without querying the elevator.
1880  *
1881  * Caller must ensure !blk_queue_nomerges(q) beforehand.
1882  */
1883 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1884 			    unsigned int *request_count,
1885 			    struct request **same_queue_rq)
1886 {
1887 	struct blk_plug *plug;
1888 	struct request *rq;
1889 	struct list_head *plug_list;
1890 
1891 	plug = current->plug;
1892 	if (!plug)
1893 		return false;
1894 	*request_count = 0;
1895 
1896 	if (q->mq_ops)
1897 		plug_list = &plug->mq_list;
1898 	else
1899 		plug_list = &plug->list;
1900 
1901 	list_for_each_entry_reverse(rq, plug_list, queuelist) {
1902 		bool merged = false;
1903 
1904 		if (rq->q == q) {
1905 			(*request_count)++;
1906 			/*
1907 			 * Only blk-mq multiple hardware queues case checks the
1908 			 * rq in the same queue, there should be only one such
1909 			 * rq in a queue
1910 			 **/
1911 			if (same_queue_rq)
1912 				*same_queue_rq = rq;
1913 		}
1914 
1915 		if (rq->q != q || !blk_rq_merge_ok(rq, bio))
1916 			continue;
1917 
1918 		switch (blk_try_merge(rq, bio)) {
1919 		case ELEVATOR_BACK_MERGE:
1920 			merged = bio_attempt_back_merge(q, rq, bio);
1921 			break;
1922 		case ELEVATOR_FRONT_MERGE:
1923 			merged = bio_attempt_front_merge(q, rq, bio);
1924 			break;
1925 		case ELEVATOR_DISCARD_MERGE:
1926 			merged = bio_attempt_discard_merge(q, rq, bio);
1927 			break;
1928 		default:
1929 			break;
1930 		}
1931 
1932 		if (merged)
1933 			return true;
1934 	}
1935 
1936 	return false;
1937 }
1938 
1939 unsigned int blk_plug_queued_count(struct request_queue *q)
1940 {
1941 	struct blk_plug *plug;
1942 	struct request *rq;
1943 	struct list_head *plug_list;
1944 	unsigned int ret = 0;
1945 
1946 	plug = current->plug;
1947 	if (!plug)
1948 		goto out;
1949 
1950 	if (q->mq_ops)
1951 		plug_list = &plug->mq_list;
1952 	else
1953 		plug_list = &plug->list;
1954 
1955 	list_for_each_entry(rq, plug_list, queuelist) {
1956 		if (rq->q == q)
1957 			ret++;
1958 	}
1959 out:
1960 	return ret;
1961 }
1962 
1963 void blk_init_request_from_bio(struct request *req, struct bio *bio)
1964 {
1965 	struct io_context *ioc = rq_ioc(bio);
1966 
1967 	if (bio->bi_opf & REQ_RAHEAD)
1968 		req->cmd_flags |= REQ_FAILFAST_MASK;
1969 
1970 	req->__sector = bio->bi_iter.bi_sector;
1971 	if (ioprio_valid(bio_prio(bio)))
1972 		req->ioprio = bio_prio(bio);
1973 	else if (ioc)
1974 		req->ioprio = ioc->ioprio;
1975 	else
1976 		req->ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
1977 	req->write_hint = bio->bi_write_hint;
1978 	blk_rq_bio_prep(req->q, req, bio);
1979 }
1980 EXPORT_SYMBOL_GPL(blk_init_request_from_bio);
1981 
1982 static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
1983 {
1984 	struct blk_plug *plug;
1985 	int where = ELEVATOR_INSERT_SORT;
1986 	struct request *req, *free;
1987 	unsigned int request_count = 0;
1988 
1989 	/*
1990 	 * low level driver can indicate that it wants pages above a
1991 	 * certain limit bounced to low memory (ie for highmem, or even
1992 	 * ISA dma in theory)
1993 	 */
1994 	blk_queue_bounce(q, &bio);
1995 
1996 	blk_queue_split(q, &bio);
1997 
1998 	if (!bio_integrity_prep(bio))
1999 		return BLK_QC_T_NONE;
2000 
2001 	if (op_is_flush(bio->bi_opf)) {
2002 		spin_lock_irq(q->queue_lock);
2003 		where = ELEVATOR_INSERT_FLUSH;
2004 		goto get_rq;
2005 	}
2006 
2007 	/*
2008 	 * Check if we can merge with the plugged list before grabbing
2009 	 * any locks.
2010 	 */
2011 	if (!blk_queue_nomerges(q)) {
2012 		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
2013 			return BLK_QC_T_NONE;
2014 	} else
2015 		request_count = blk_plug_queued_count(q);
2016 
2017 	spin_lock_irq(q->queue_lock);
2018 
2019 	switch (elv_merge(q, &req, bio)) {
2020 	case ELEVATOR_BACK_MERGE:
2021 		if (!bio_attempt_back_merge(q, req, bio))
2022 			break;
2023 		elv_bio_merged(q, req, bio);
2024 		free = attempt_back_merge(q, req);
2025 		if (free)
2026 			__blk_put_request(q, free);
2027 		else
2028 			elv_merged_request(q, req, ELEVATOR_BACK_MERGE);
2029 		goto out_unlock;
2030 	case ELEVATOR_FRONT_MERGE:
2031 		if (!bio_attempt_front_merge(q, req, bio))
2032 			break;
2033 		elv_bio_merged(q, req, bio);
2034 		free = attempt_front_merge(q, req);
2035 		if (free)
2036 			__blk_put_request(q, free);
2037 		else
2038 			elv_merged_request(q, req, ELEVATOR_FRONT_MERGE);
2039 		goto out_unlock;
2040 	default:
2041 		break;
2042 	}
2043 
2044 get_rq:
2045 	rq_qos_throttle(q, bio, q->queue_lock);
2046 
2047 	/*
2048 	 * Grab a free request. This is might sleep but can not fail.
2049 	 * Returns with the queue unlocked.
2050 	 */
2051 	blk_queue_enter_live(q);
2052 	req = get_request(q, bio->bi_opf, bio, 0, GFP_NOIO);
2053 	if (IS_ERR(req)) {
2054 		blk_queue_exit(q);
2055 		rq_qos_cleanup(q, bio);
2056 		if (PTR_ERR(req) == -ENOMEM)
2057 			bio->bi_status = BLK_STS_RESOURCE;
2058 		else
2059 			bio->bi_status = BLK_STS_IOERR;
2060 		bio_endio(bio);
2061 		goto out_unlock;
2062 	}
2063 
2064 	rq_qos_track(q, req, bio);
2065 
2066 	/*
2067 	 * After dropping the lock and possibly sleeping here, our request
2068 	 * may now be mergeable after it had proven unmergeable (above).
2069 	 * We don't worry about that case for efficiency. It won't happen
2070 	 * often, and the elevators are able to handle it.
2071 	 */
2072 	blk_init_request_from_bio(req, bio);
2073 
2074 	if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
2075 		req->cpu = raw_smp_processor_id();
2076 
2077 	plug = current->plug;
2078 	if (plug) {
2079 		/*
2080 		 * If this is the first request added after a plug, fire
2081 		 * of a plug trace.
2082 		 *
2083 		 * @request_count may become stale because of schedule
2084 		 * out, so check plug list again.
2085 		 */
2086 		if (!request_count || list_empty(&plug->list))
2087 			trace_block_plug(q);
2088 		else {
2089 			struct request *last = list_entry_rq(plug->list.prev);
2090 			if (request_count >= BLK_MAX_REQUEST_COUNT ||
2091 			    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE) {
2092 				blk_flush_plug_list(plug, false);
2093 				trace_block_plug(q);
2094 			}
2095 		}
2096 		list_add_tail(&req->queuelist, &plug->list);
2097 		blk_account_io_start(req, true);
2098 	} else {
2099 		spin_lock_irq(q->queue_lock);
2100 		add_acct_request(q, req, where);
2101 		__blk_run_queue(q);
2102 out_unlock:
2103 		spin_unlock_irq(q->queue_lock);
2104 	}
2105 
2106 	return BLK_QC_T_NONE;
2107 }
2108 
2109 static void handle_bad_sector(struct bio *bio, sector_t maxsector)
2110 {
2111 	char b[BDEVNAME_SIZE];
2112 
2113 	printk(KERN_INFO "attempt to access beyond end of device\n");
2114 	printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n",
2115 			bio_devname(bio, b), bio->bi_opf,
2116 			(unsigned long long)bio_end_sector(bio),
2117 			(long long)maxsector);
2118 }
2119 
2120 #ifdef CONFIG_FAIL_MAKE_REQUEST
2121 
2122 static DECLARE_FAULT_ATTR(fail_make_request);
2123 
2124 static int __init setup_fail_make_request(char *str)
2125 {
2126 	return setup_fault_attr(&fail_make_request, str);
2127 }
2128 __setup("fail_make_request=", setup_fail_make_request);
2129 
2130 static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
2131 {
2132 	return part->make_it_fail && should_fail(&fail_make_request, bytes);
2133 }
2134 
2135 static int __init fail_make_request_debugfs(void)
2136 {
2137 	struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
2138 						NULL, &fail_make_request);
2139 
2140 	return PTR_ERR_OR_ZERO(dir);
2141 }
2142 
2143 late_initcall(fail_make_request_debugfs);
2144 
2145 #else /* CONFIG_FAIL_MAKE_REQUEST */
2146 
2147 static inline bool should_fail_request(struct hd_struct *part,
2148 					unsigned int bytes)
2149 {
2150 	return false;
2151 }
2152 
2153 #endif /* CONFIG_FAIL_MAKE_REQUEST */
2154 
2155 static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part)
2156 {
2157 	const int op = bio_op(bio);
2158 
2159 	if (part->policy && op_is_write(op)) {
2160 		char b[BDEVNAME_SIZE];
2161 
2162 		if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
2163 			return false;
2164 
2165 		WARN_ONCE(1,
2166 		       "generic_make_request: Trying to write "
2167 			"to read-only block-device %s (partno %d)\n",
2168 			bio_devname(bio, b), part->partno);
2169 		/* Older lvm-tools actually trigger this */
2170 		return false;
2171 	}
2172 
2173 	return false;
2174 }
2175 
2176 static noinline int should_fail_bio(struct bio *bio)
2177 {
2178 	if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size))
2179 		return -EIO;
2180 	return 0;
2181 }
2182 ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);
2183 
2184 /*
2185  * Check whether this bio extends beyond the end of the device or partition.
2186  * This may well happen - the kernel calls bread() without checking the size of
2187  * the device, e.g., when mounting a file system.
2188  */
2189 static inline int bio_check_eod(struct bio *bio, sector_t maxsector)
2190 {
2191 	unsigned int nr_sectors = bio_sectors(bio);
2192 
2193 	if (nr_sectors && maxsector &&
2194 	    (nr_sectors > maxsector ||
2195 	     bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
2196 		handle_bad_sector(bio, maxsector);
2197 		return -EIO;
2198 	}
2199 	return 0;
2200 }
2201 
2202 /*
2203  * Remap block n of partition p to block n+start(p) of the disk.
2204  */
2205 static inline int blk_partition_remap(struct bio *bio)
2206 {
2207 	struct hd_struct *p;
2208 	int ret = -EIO;
2209 
2210 	rcu_read_lock();
2211 	p = __disk_get_part(bio->bi_disk, bio->bi_partno);
2212 	if (unlikely(!p))
2213 		goto out;
2214 	if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
2215 		goto out;
2216 	if (unlikely(bio_check_ro(bio, p)))
2217 		goto out;
2218 
2219 	/*
2220 	 * Zone reset does not include bi_size so bio_sectors() is always 0.
2221 	 * Include a test for the reset op code and perform the remap if needed.
2222 	 */
2223 	if (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET) {
2224 		if (bio_check_eod(bio, part_nr_sects_read(p)))
2225 			goto out;
2226 		bio->bi_iter.bi_sector += p->start_sect;
2227 		trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p),
2228 				      bio->bi_iter.bi_sector - p->start_sect);
2229 	}
2230 	bio->bi_partno = 0;
2231 	ret = 0;
2232 out:
2233 	rcu_read_unlock();
2234 	return ret;
2235 }
2236 
2237 static noinline_for_stack bool
2238 generic_make_request_checks(struct bio *bio)
2239 {
2240 	struct request_queue *q;
2241 	int nr_sectors = bio_sectors(bio);
2242 	blk_status_t status = BLK_STS_IOERR;
2243 	char b[BDEVNAME_SIZE];
2244 
2245 	might_sleep();
2246 
2247 	q = bio->bi_disk->queue;
2248 	if (unlikely(!q)) {
2249 		printk(KERN_ERR
2250 		       "generic_make_request: Trying to access "
2251 			"nonexistent block-device %s (%Lu)\n",
2252 			bio_devname(bio, b), (long long)bio->bi_iter.bi_sector);
2253 		goto end_io;
2254 	}
2255 
2256 	/*
2257 	 * For a REQ_NOWAIT based request, return -EOPNOTSUPP
2258 	 * if queue is not a request based queue.
2259 	 */
2260 	if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_rq_based(q))
2261 		goto not_supported;
2262 
2263 	if (should_fail_bio(bio))
2264 		goto end_io;
2265 
2266 	if (bio->bi_partno) {
2267 		if (unlikely(blk_partition_remap(bio)))
2268 			goto end_io;
2269 	} else {
2270 		if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0)))
2271 			goto end_io;
2272 		if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))
2273 			goto end_io;
2274 	}
2275 
2276 	/*
2277 	 * Filter flush bio's early so that make_request based
2278 	 * drivers without flush support don't have to worry
2279 	 * about them.
2280 	 */
2281 	if (op_is_flush(bio->bi_opf) &&
2282 	    !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
2283 		bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
2284 		if (!nr_sectors) {
2285 			status = BLK_STS_OK;
2286 			goto end_io;
2287 		}
2288 	}
2289 
2290 	switch (bio_op(bio)) {
2291 	case REQ_OP_DISCARD:
2292 		if (!blk_queue_discard(q))
2293 			goto not_supported;
2294 		break;
2295 	case REQ_OP_SECURE_ERASE:
2296 		if (!blk_queue_secure_erase(q))
2297 			goto not_supported;
2298 		break;
2299 	case REQ_OP_WRITE_SAME:
2300 		if (!q->limits.max_write_same_sectors)
2301 			goto not_supported;
2302 		break;
2303 	case REQ_OP_ZONE_RESET:
2304 		if (!blk_queue_is_zoned(q))
2305 			goto not_supported;
2306 		break;
2307 	case REQ_OP_WRITE_ZEROES:
2308 		if (!q->limits.max_write_zeroes_sectors)
2309 			goto not_supported;
2310 		break;
2311 	default:
2312 		break;
2313 	}
2314 
2315 	/*
2316 	 * Various block parts want %current->io_context and lazy ioc
2317 	 * allocation ends up trading a lot of pain for a small amount of
2318 	 * memory.  Just allocate it upfront.  This may fail and block
2319 	 * layer knows how to live with it.
2320 	 */
2321 	create_io_context(GFP_ATOMIC, q->node);
2322 
2323 	if (!blkcg_bio_issue_check(q, bio))
2324 		return false;
2325 
2326 	if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
2327 		trace_block_bio_queue(q, bio);
2328 		/* Now that enqueuing has been traced, we need to trace
2329 		 * completion as well.
2330 		 */
2331 		bio_set_flag(bio, BIO_TRACE_COMPLETION);
2332 	}
2333 	return true;
2334 
2335 not_supported:
2336 	status = BLK_STS_NOTSUPP;
2337 end_io:
2338 	bio->bi_status = status;
2339 	bio_endio(bio);
2340 	return false;
2341 }
2342 
2343 /**
2344  * generic_make_request - hand a buffer to its device driver for I/O
2345  * @bio:  The bio describing the location in memory and on the device.
2346  *
2347  * generic_make_request() is used to make I/O requests of block
2348  * devices. It is passed a &struct bio, which describes the I/O that needs
2349  * to be done.
2350  *
2351  * generic_make_request() does not return any status.  The
2352  * success/failure status of the request, along with notification of
2353  * completion, is delivered asynchronously through the bio->bi_end_io
2354  * function described (one day) else where.
2355  *
2356  * The caller of generic_make_request must make sure that bi_io_vec
2357  * are set to describe the memory buffer, and that bi_dev and bi_sector are
2358  * set to describe the device address, and the
2359  * bi_end_io and optionally bi_private are set to describe how
2360  * completion notification should be signaled.
2361  *
2362  * generic_make_request and the drivers it calls may use bi_next if this
2363  * bio happens to be merged with someone else, and may resubmit the bio to
2364  * a lower device by calling into generic_make_request recursively, which
2365  * means the bio should NOT be touched after the call to ->make_request_fn.
2366  */
2367 blk_qc_t generic_make_request(struct bio *bio)
2368 {
2369 	/*
2370 	 * bio_list_on_stack[0] contains bios submitted by the current
2371 	 * make_request_fn.
2372 	 * bio_list_on_stack[1] contains bios that were submitted before
2373 	 * the current make_request_fn, but that haven't been processed
2374 	 * yet.
2375 	 */
2376 	struct bio_list bio_list_on_stack[2];
2377 	blk_mq_req_flags_t flags = 0;
2378 	struct request_queue *q = bio->bi_disk->queue;
2379 	blk_qc_t ret = BLK_QC_T_NONE;
2380 
2381 	if (bio->bi_opf & REQ_NOWAIT)
2382 		flags = BLK_MQ_REQ_NOWAIT;
2383 	if (bio_flagged(bio, BIO_QUEUE_ENTERED))
2384 		blk_queue_enter_live(q);
2385 	else if (blk_queue_enter(q, flags) < 0) {
2386 		if (!blk_queue_dying(q) && (bio->bi_opf & REQ_NOWAIT))
2387 			bio_wouldblock_error(bio);
2388 		else
2389 			bio_io_error(bio);
2390 		return ret;
2391 	}
2392 
2393 	if (!generic_make_request_checks(bio))
2394 		goto out;
2395 
2396 	/*
2397 	 * We only want one ->make_request_fn to be active at a time, else
2398 	 * stack usage with stacked devices could be a problem.  So use
2399 	 * current->bio_list to keep a list of requests submited by a
2400 	 * make_request_fn function.  current->bio_list is also used as a
2401 	 * flag to say if generic_make_request is currently active in this
2402 	 * task or not.  If it is NULL, then no make_request is active.  If
2403 	 * it is non-NULL, then a make_request is active, and new requests
2404 	 * should be added at the tail
2405 	 */
2406 	if (current->bio_list) {
2407 		bio_list_add(&current->bio_list[0], bio);
2408 		goto out;
2409 	}
2410 
2411 	/* following loop may be a bit non-obvious, and so deserves some
2412 	 * explanation.
2413 	 * Before entering the loop, bio->bi_next is NULL (as all callers
2414 	 * ensure that) so we have a list with a single bio.
2415 	 * We pretend that we have just taken it off a longer list, so
2416 	 * we assign bio_list to a pointer to the bio_list_on_stack,
2417 	 * thus initialising the bio_list of new bios to be
2418 	 * added.  ->make_request() may indeed add some more bios
2419 	 * through a recursive call to generic_make_request.  If it
2420 	 * did, we find a non-NULL value in bio_list and re-enter the loop
2421 	 * from the top.  In this case we really did just take the bio
2422 	 * of the top of the list (no pretending) and so remove it from
2423 	 * bio_list, and call into ->make_request() again.
2424 	 */
2425 	BUG_ON(bio->bi_next);
2426 	bio_list_init(&bio_list_on_stack[0]);
2427 	current->bio_list = bio_list_on_stack;
2428 	do {
2429 		bool enter_succeeded = true;
2430 
2431 		if (unlikely(q != bio->bi_disk->queue)) {
2432 			if (q)
2433 				blk_queue_exit(q);
2434 			q = bio->bi_disk->queue;
2435 			bio_reassociate_blkg(q, bio);
2436 			flags = 0;
2437 			if (bio->bi_opf & REQ_NOWAIT)
2438 				flags = BLK_MQ_REQ_NOWAIT;
2439 			if (blk_queue_enter(q, flags) < 0) {
2440 				enter_succeeded = false;
2441 				q = NULL;
2442 			}
2443 		}
2444 
2445 		if (enter_succeeded) {
2446 			struct bio_list lower, same;
2447 
2448 			/* Create a fresh bio_list for all subordinate requests */
2449 			bio_list_on_stack[1] = bio_list_on_stack[0];
2450 			bio_list_init(&bio_list_on_stack[0]);
2451 			ret = q->make_request_fn(q, bio);
2452 
2453 			/* sort new bios into those for a lower level
2454 			 * and those for the same level
2455 			 */
2456 			bio_list_init(&lower);
2457 			bio_list_init(&same);
2458 			while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
2459 				if (q == bio->bi_disk->queue)
2460 					bio_list_add(&same, bio);
2461 				else
2462 					bio_list_add(&lower, bio);
2463 			/* now assemble so we handle the lowest level first */
2464 			bio_list_merge(&bio_list_on_stack[0], &lower);
2465 			bio_list_merge(&bio_list_on_stack[0], &same);
2466 			bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
2467 		} else {
2468 			if (unlikely(!blk_queue_dying(q) &&
2469 					(bio->bi_opf & REQ_NOWAIT)))
2470 				bio_wouldblock_error(bio);
2471 			else
2472 				bio_io_error(bio);
2473 		}
2474 		bio = bio_list_pop(&bio_list_on_stack[0]);
2475 	} while (bio);
2476 	current->bio_list = NULL; /* deactivate */
2477 
2478 out:
2479 	if (q)
2480 		blk_queue_exit(q);
2481 	return ret;
2482 }
2483 EXPORT_SYMBOL(generic_make_request);
2484 
2485 /**
2486  * direct_make_request - hand a buffer directly to its device driver for I/O
2487  * @bio:  The bio describing the location in memory and on the device.
2488  *
2489  * This function behaves like generic_make_request(), but does not protect
2490  * against recursion.  Must only be used if the called driver is known
2491  * to not call generic_make_request (or direct_make_request) again from
2492  * its make_request function.  (Calling direct_make_request again from
2493  * a workqueue is perfectly fine as that doesn't recurse).
2494  */
2495 blk_qc_t direct_make_request(struct bio *bio)
2496 {
2497 	struct request_queue *q = bio->bi_disk->queue;
2498 	bool nowait = bio->bi_opf & REQ_NOWAIT;
2499 	blk_qc_t ret;
2500 
2501 	if (!generic_make_request_checks(bio))
2502 		return BLK_QC_T_NONE;
2503 
2504 	if (unlikely(blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0))) {
2505 		if (nowait && !blk_queue_dying(q))
2506 			bio->bi_status = BLK_STS_AGAIN;
2507 		else
2508 			bio->bi_status = BLK_STS_IOERR;
2509 		bio_endio(bio);
2510 		return BLK_QC_T_NONE;
2511 	}
2512 
2513 	ret = q->make_request_fn(q, bio);
2514 	blk_queue_exit(q);
2515 	return ret;
2516 }
2517 EXPORT_SYMBOL_GPL(direct_make_request);
2518 
2519 /**
2520  * submit_bio - submit a bio to the block device layer for I/O
2521  * @bio: The &struct bio which describes the I/O
2522  *
2523  * submit_bio() is very similar in purpose to generic_make_request(), and
2524  * uses that function to do most of the work. Both are fairly rough
2525  * interfaces; @bio must be presetup and ready for I/O.
2526  *
2527  */
2528 blk_qc_t submit_bio(struct bio *bio)
2529 {
2530 	/*
2531 	 * If it's a regular read/write or a barrier with data attached,
2532 	 * go through the normal accounting stuff before submission.
2533 	 */
2534 	if (bio_has_data(bio)) {
2535 		unsigned int count;
2536 
2537 		if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
2538 			count = queue_logical_block_size(bio->bi_disk->queue) >> 9;
2539 		else
2540 			count = bio_sectors(bio);
2541 
2542 		if (op_is_write(bio_op(bio))) {
2543 			count_vm_events(PGPGOUT, count);
2544 		} else {
2545 			task_io_account_read(bio->bi_iter.bi_size);
2546 			count_vm_events(PGPGIN, count);
2547 		}
2548 
2549 		if (unlikely(block_dump)) {
2550 			char b[BDEVNAME_SIZE];
2551 			printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
2552 			current->comm, task_pid_nr(current),
2553 				op_is_write(bio_op(bio)) ? "WRITE" : "READ",
2554 				(unsigned long long)bio->bi_iter.bi_sector,
2555 				bio_devname(bio, b), count);
2556 		}
2557 	}
2558 
2559 	return generic_make_request(bio);
2560 }
2561 EXPORT_SYMBOL(submit_bio);
2562 
2563 bool blk_poll(struct request_queue *q, blk_qc_t cookie)
2564 {
2565 	if (!q->poll_fn || !blk_qc_t_valid(cookie))
2566 		return false;
2567 
2568 	if (current->plug)
2569 		blk_flush_plug_list(current->plug, false);
2570 	return q->poll_fn(q, cookie);
2571 }
2572 EXPORT_SYMBOL_GPL(blk_poll);
2573 
2574 /**
2575  * blk_cloned_rq_check_limits - Helper function to check a cloned request
2576  *                              for new the queue limits
2577  * @q:  the queue
2578  * @rq: the request being checked
2579  *
2580  * Description:
2581  *    @rq may have been made based on weaker limitations of upper-level queues
2582  *    in request stacking drivers, and it may violate the limitation of @q.
2583  *    Since the block layer and the underlying device driver trust @rq
2584  *    after it is inserted to @q, it should be checked against @q before
2585  *    the insertion using this generic function.
2586  *
2587  *    Request stacking drivers like request-based dm may change the queue
2588  *    limits when retrying requests on other queues. Those requests need
2589  *    to be checked against the new queue limits again during dispatch.
2590  */
2591 static int blk_cloned_rq_check_limits(struct request_queue *q,
2592 				      struct request *rq)
2593 {
2594 	if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) {
2595 		printk(KERN_ERR "%s: over max size limit.\n", __func__);
2596 		return -EIO;
2597 	}
2598 
2599 	/*
2600 	 * queue's settings related to segment counting like q->bounce_pfn
2601 	 * may differ from that of other stacking queues.
2602 	 * Recalculate it to check the request correctly on this queue's
2603 	 * limitation.
2604 	 */
2605 	blk_recalc_rq_segments(rq);
2606 	if (rq->nr_phys_segments > queue_max_segments(q)) {
2607 		printk(KERN_ERR "%s: over max segments limit.\n", __func__);
2608 		return -EIO;
2609 	}
2610 
2611 	return 0;
2612 }
2613 
2614 /**
2615  * blk_insert_cloned_request - Helper for stacking drivers to submit a request
2616  * @q:  the queue to submit the request
2617  * @rq: the request being queued
2618  */
2619 blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
2620 {
2621 	unsigned long flags;
2622 	int where = ELEVATOR_INSERT_BACK;
2623 
2624 	if (blk_cloned_rq_check_limits(q, rq))
2625 		return BLK_STS_IOERR;
2626 
2627 	if (rq->rq_disk &&
2628 	    should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
2629 		return BLK_STS_IOERR;
2630 
2631 	if (q->mq_ops) {
2632 		if (blk_queue_io_stat(q))
2633 			blk_account_io_start(rq, true);
2634 		/*
2635 		 * Since we have a scheduler attached on the top device,
2636 		 * bypass a potential scheduler on the bottom device for
2637 		 * insert.
2638 		 */
2639 		return blk_mq_request_issue_directly(rq);
2640 	}
2641 
2642 	spin_lock_irqsave(q->queue_lock, flags);
2643 	if (unlikely(blk_queue_dying(q))) {
2644 		spin_unlock_irqrestore(q->queue_lock, flags);
2645 		return BLK_STS_IOERR;
2646 	}
2647 
2648 	/*
2649 	 * Submitting request must be dequeued before calling this function
2650 	 * because it will be linked to another request_queue
2651 	 */
2652 	BUG_ON(blk_queued_rq(rq));
2653 
2654 	if (op_is_flush(rq->cmd_flags))
2655 		where = ELEVATOR_INSERT_FLUSH;
2656 
2657 	add_acct_request(q, rq, where);
2658 	if (where == ELEVATOR_INSERT_FLUSH)
2659 		__blk_run_queue(q);
2660 	spin_unlock_irqrestore(q->queue_lock, flags);
2661 
2662 	return BLK_STS_OK;
2663 }
2664 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
2665 
2666 /**
2667  * blk_rq_err_bytes - determine number of bytes till the next failure boundary
2668  * @rq: request to examine
2669  *
2670  * Description:
2671  *     A request could be merge of IOs which require different failure
2672  *     handling.  This function determines the number of bytes which
2673  *     can be failed from the beginning of the request without
2674  *     crossing into area which need to be retried further.
2675  *
2676  * Return:
2677  *     The number of bytes to fail.
2678  */
2679 unsigned int blk_rq_err_bytes(const struct request *rq)
2680 {
2681 	unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
2682 	unsigned int bytes = 0;
2683 	struct bio *bio;
2684 
2685 	if (!(rq->rq_flags & RQF_MIXED_MERGE))
2686 		return blk_rq_bytes(rq);
2687 
2688 	/*
2689 	 * Currently the only 'mixing' which can happen is between
2690 	 * different fastfail types.  We can safely fail portions
2691 	 * which have all the failfast bits that the first one has -
2692 	 * the ones which are at least as eager to fail as the first
2693 	 * one.
2694 	 */
2695 	for (bio = rq->bio; bio; bio = bio->bi_next) {
2696 		if ((bio->bi_opf & ff) != ff)
2697 			break;
2698 		bytes += bio->bi_iter.bi_size;
2699 	}
2700 
2701 	/* this could lead to infinite loop */
2702 	BUG_ON(blk_rq_bytes(rq) && !bytes);
2703 	return bytes;
2704 }
2705 EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
2706 
2707 void blk_account_io_completion(struct request *req, unsigned int bytes)
2708 {
2709 	if (blk_do_io_stat(req)) {
2710 		const int sgrp = op_stat_group(req_op(req));
2711 		struct hd_struct *part;
2712 		int cpu;
2713 
2714 		cpu = part_stat_lock();
2715 		part = req->part;
2716 		part_stat_add(cpu, part, sectors[sgrp], bytes >> 9);
2717 		part_stat_unlock();
2718 	}
2719 }
2720 
2721 void blk_account_io_done(struct request *req, u64 now)
2722 {
2723 	/*
2724 	 * Account IO completion.  flush_rq isn't accounted as a
2725 	 * normal IO on queueing nor completion.  Accounting the
2726 	 * containing request is enough.
2727 	 */
2728 	if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {
2729 		const int sgrp = op_stat_group(req_op(req));
2730 		struct hd_struct *part;
2731 		int cpu;
2732 
2733 		cpu = part_stat_lock();
2734 		part = req->part;
2735 
2736 		part_stat_inc(cpu, part, ios[sgrp]);
2737 		part_stat_add(cpu, part, nsecs[sgrp], now - req->start_time_ns);
2738 		part_round_stats(req->q, cpu, part);
2739 		part_dec_in_flight(req->q, part, rq_data_dir(req));
2740 
2741 		hd_struct_put(part);
2742 		part_stat_unlock();
2743 	}
2744 }
2745 
2746 void blk_account_io_start(struct request *rq, bool new_io)
2747 {
2748 	struct hd_struct *part;
2749 	int rw = rq_data_dir(rq);
2750 	int cpu;
2751 
2752 	if (!blk_do_io_stat(rq))
2753 		return;
2754 
2755 	cpu = part_stat_lock();
2756 
2757 	if (!new_io) {
2758 		part = rq->part;
2759 		part_stat_inc(cpu, part, merges[rw]);
2760 	} else {
2761 		part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
2762 		if (!hd_struct_try_get(part)) {
2763 			/*
2764 			 * The partition is already being removed,
2765 			 * the request will be accounted on the disk only
2766 			 *
2767 			 * We take a reference on disk->part0 although that
2768 			 * partition will never be deleted, so we can treat
2769 			 * it as any other partition.
2770 			 */
2771 			part = &rq->rq_disk->part0;
2772 			hd_struct_get(part);
2773 		}
2774 		part_round_stats(rq->q, cpu, part);
2775 		part_inc_in_flight(rq->q, part, rw);
2776 		rq->part = part;
2777 	}
2778 
2779 	part_stat_unlock();
2780 }
2781 
2782 static struct request *elv_next_request(struct request_queue *q)
2783 {
2784 	struct request *rq;
2785 	struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
2786 
2787 	WARN_ON_ONCE(q->mq_ops);
2788 
2789 	while (1) {
2790 		list_for_each_entry(rq, &q->queue_head, queuelist) {
2791 #ifdef CONFIG_PM
2792 			/*
2793 			 * If a request gets queued in state RPM_SUSPENDED
2794 			 * then that's a kernel bug.
2795 			 */
2796 			WARN_ON_ONCE(q->rpm_status == RPM_SUSPENDED);
2797 #endif
2798 			return rq;
2799 		}
2800 
2801 		/*
2802 		 * Flush request is running and flush request isn't queueable
2803 		 * in the drive, we can hold the queue till flush request is
2804 		 * finished. Even we don't do this, driver can't dispatch next
2805 		 * requests and will requeue them. And this can improve
2806 		 * throughput too. For example, we have request flush1, write1,
2807 		 * flush 2. flush1 is dispatched, then queue is hold, write1
2808 		 * isn't inserted to queue. After flush1 is finished, flush2
2809 		 * will be dispatched. Since disk cache is already clean,
2810 		 * flush2 will be finished very soon, so looks like flush2 is
2811 		 * folded to flush1.
2812 		 * Since the queue is hold, a flag is set to indicate the queue
2813 		 * should be restarted later. Please see flush_end_io() for
2814 		 * details.
2815 		 */
2816 		if (fq->flush_pending_idx != fq->flush_running_idx &&
2817 				!queue_flush_queueable(q)) {
2818 			fq->flush_queue_delayed = 1;
2819 			return NULL;
2820 		}
2821 		if (unlikely(blk_queue_bypass(q)) ||
2822 		    !q->elevator->type->ops.sq.elevator_dispatch_fn(q, 0))
2823 			return NULL;
2824 	}
2825 }
2826 
2827 /**
2828  * blk_peek_request - peek at the top of a request queue
2829  * @q: request queue to peek at
2830  *
2831  * Description:
2832  *     Return the request at the top of @q.  The returned request
2833  *     should be started using blk_start_request() before LLD starts
2834  *     processing it.
2835  *
2836  * Return:
2837  *     Pointer to the request at the top of @q if available.  Null
2838  *     otherwise.
2839  */
2840 struct request *blk_peek_request(struct request_queue *q)
2841 {
2842 	struct request *rq;
2843 	int ret;
2844 
2845 	lockdep_assert_held(q->queue_lock);
2846 	WARN_ON_ONCE(q->mq_ops);
2847 
2848 	while ((rq = elv_next_request(q)) != NULL) {
2849 		if (!(rq->rq_flags & RQF_STARTED)) {
2850 			/*
2851 			 * This is the first time the device driver
2852 			 * sees this request (possibly after
2853 			 * requeueing).  Notify IO scheduler.
2854 			 */
2855 			if (rq->rq_flags & RQF_SORTED)
2856 				elv_activate_rq(q, rq);
2857 
2858 			/*
2859 			 * just mark as started even if we don't start
2860 			 * it, a request that has been delayed should
2861 			 * not be passed by new incoming requests
2862 			 */
2863 			rq->rq_flags |= RQF_STARTED;
2864 			trace_block_rq_issue(q, rq);
2865 		}
2866 
2867 		if (!q->boundary_rq || q->boundary_rq == rq) {
2868 			q->end_sector = rq_end_sector(rq);
2869 			q->boundary_rq = NULL;
2870 		}
2871 
2872 		if (rq->rq_flags & RQF_DONTPREP)
2873 			break;
2874 
2875 		if (q->dma_drain_size && blk_rq_bytes(rq)) {
2876 			/*
2877 			 * make sure space for the drain appears we
2878 			 * know we can do this because max_hw_segments
2879 			 * has been adjusted to be one fewer than the
2880 			 * device can handle
2881 			 */
2882 			rq->nr_phys_segments++;
2883 		}
2884 
2885 		if (!q->prep_rq_fn)
2886 			break;
2887 
2888 		ret = q->prep_rq_fn(q, rq);
2889 		if (ret == BLKPREP_OK) {
2890 			break;
2891 		} else if (ret == BLKPREP_DEFER) {
2892 			/*
2893 			 * the request may have been (partially) prepped.
2894 			 * we need to keep this request in the front to
2895 			 * avoid resource deadlock.  RQF_STARTED will
2896 			 * prevent other fs requests from passing this one.
2897 			 */
2898 			if (q->dma_drain_size && blk_rq_bytes(rq) &&
2899 			    !(rq->rq_flags & RQF_DONTPREP)) {
2900 				/*
2901 				 * remove the space for the drain we added
2902 				 * so that we don't add it again
2903 				 */
2904 				--rq->nr_phys_segments;
2905 			}
2906 
2907 			rq = NULL;
2908 			break;
2909 		} else if (ret == BLKPREP_KILL || ret == BLKPREP_INVALID) {
2910 			rq->rq_flags |= RQF_QUIET;
2911 			/*
2912 			 * Mark this request as started so we don't trigger
2913 			 * any debug logic in the end I/O path.
2914 			 */
2915 			blk_start_request(rq);
2916 			__blk_end_request_all(rq, ret == BLKPREP_INVALID ?
2917 					BLK_STS_TARGET : BLK_STS_IOERR);
2918 		} else {
2919 			printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
2920 			break;
2921 		}
2922 	}
2923 
2924 	return rq;
2925 }
2926 EXPORT_SYMBOL(blk_peek_request);
2927 
2928 static void blk_dequeue_request(struct request *rq)
2929 {
2930 	struct request_queue *q = rq->q;
2931 
2932 	BUG_ON(list_empty(&rq->queuelist));
2933 	BUG_ON(ELV_ON_HASH(rq));
2934 
2935 	list_del_init(&rq->queuelist);
2936 
2937 	/*
2938 	 * the time frame between a request being removed from the lists
2939 	 * and to it is freed is accounted as io that is in progress at
2940 	 * the driver side.
2941 	 */
2942 	if (blk_account_rq(rq))
2943 		q->in_flight[rq_is_sync(rq)]++;
2944 }
2945 
2946 /**
2947  * blk_start_request - start request processing on the driver
2948  * @req: request to dequeue
2949  *
2950  * Description:
2951  *     Dequeue @req and start timeout timer on it.  This hands off the
2952  *     request to the driver.
2953  */
2954 void blk_start_request(struct request *req)
2955 {
2956 	lockdep_assert_held(req->q->queue_lock);
2957 	WARN_ON_ONCE(req->q->mq_ops);
2958 
2959 	blk_dequeue_request(req);
2960 
2961 	if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
2962 		req->io_start_time_ns = ktime_get_ns();
2963 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
2964 		req->throtl_size = blk_rq_sectors(req);
2965 #endif
2966 		req->rq_flags |= RQF_STATS;
2967 		rq_qos_issue(req->q, req);
2968 	}
2969 
2970 	BUG_ON(blk_rq_is_complete(req));
2971 	blk_add_timer(req);
2972 }
2973 EXPORT_SYMBOL(blk_start_request);
2974 
2975 /**
2976  * blk_fetch_request - fetch a request from a request queue
2977  * @q: request queue to fetch a request from
2978  *
2979  * Description:
2980  *     Return the request at the top of @q.  The request is started on
2981  *     return and LLD can start processing it immediately.
2982  *
2983  * Return:
2984  *     Pointer to the request at the top of @q if available.  Null
2985  *     otherwise.
2986  */
2987 struct request *blk_fetch_request(struct request_queue *q)
2988 {
2989 	struct request *rq;
2990 
2991 	lockdep_assert_held(q->queue_lock);
2992 	WARN_ON_ONCE(q->mq_ops);
2993 
2994 	rq = blk_peek_request(q);
2995 	if (rq)
2996 		blk_start_request(rq);
2997 	return rq;
2998 }
2999 EXPORT_SYMBOL(blk_fetch_request);
3000 
3001 /*
3002  * Steal bios from a request and add them to a bio list.
3003  * The request must not have been partially completed before.
3004  */
3005 void blk_steal_bios(struct bio_list *list, struct request *rq)
3006 {
3007 	if (rq->bio) {
3008 		if (list->tail)
3009 			list->tail->bi_next = rq->bio;
3010 		else
3011 			list->head = rq->bio;
3012 		list->tail = rq->biotail;
3013 
3014 		rq->bio = NULL;
3015 		rq->biotail = NULL;
3016 	}
3017 
3018 	rq->__data_len = 0;
3019 }
3020 EXPORT_SYMBOL_GPL(blk_steal_bios);
3021 
3022 /**
3023  * blk_update_request - Special helper function for request stacking drivers
3024  * @req:      the request being processed
3025  * @error:    block status code
3026  * @nr_bytes: number of bytes to complete @req
3027  *
3028  * Description:
3029  *     Ends I/O on a number of bytes attached to @req, but doesn't complete
3030  *     the request structure even if @req doesn't have leftover.
3031  *     If @req has leftover, sets it up for the next range of segments.
3032  *
3033  *     This special helper function is only for request stacking drivers
3034  *     (e.g. request-based dm) so that they can handle partial completion.
3035  *     Actual device drivers should use blk_end_request instead.
3036  *
3037  *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
3038  *     %false return from this function.
3039  *
3040  * Note:
3041  *	The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in both
3042  *	blk_rq_bytes() and in blk_update_request().
3043  *
3044  * Return:
3045  *     %false - this request doesn't have any more data
3046  *     %true  - this request has more data
3047  **/
3048 bool blk_update_request(struct request *req, blk_status_t error,
3049 		unsigned int nr_bytes)
3050 {
3051 	int total_bytes;
3052 
3053 	trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
3054 
3055 	if (!req->bio)
3056 		return false;
3057 
3058 	if (unlikely(error && !blk_rq_is_passthrough(req) &&
3059 		     !(req->rq_flags & RQF_QUIET)))
3060 		print_req_error(req, error);
3061 
3062 	blk_account_io_completion(req, nr_bytes);
3063 
3064 	total_bytes = 0;
3065 	while (req->bio) {
3066 		struct bio *bio = req->bio;
3067 		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
3068 
3069 		if (bio_bytes == bio->bi_iter.bi_size)
3070 			req->bio = bio->bi_next;
3071 
3072 		/* Completion has already been traced */
3073 		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
3074 		req_bio_endio(req, bio, bio_bytes, error);
3075 
3076 		total_bytes += bio_bytes;
3077 		nr_bytes -= bio_bytes;
3078 
3079 		if (!nr_bytes)
3080 			break;
3081 	}
3082 
3083 	/*
3084 	 * completely done
3085 	 */
3086 	if (!req->bio) {
3087 		/*
3088 		 * Reset counters so that the request stacking driver
3089 		 * can find how many bytes remain in the request
3090 		 * later.
3091 		 */
3092 		req->__data_len = 0;
3093 		return false;
3094 	}
3095 
3096 	req->__data_len -= total_bytes;
3097 
3098 	/* update sector only for requests with clear definition of sector */
3099 	if (!blk_rq_is_passthrough(req))
3100 		req->__sector += total_bytes >> 9;
3101 
3102 	/* mixed attributes always follow the first bio */
3103 	if (req->rq_flags & RQF_MIXED_MERGE) {
3104 		req->cmd_flags &= ~REQ_FAILFAST_MASK;
3105 		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
3106 	}
3107 
3108 	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
3109 		/*
3110 		 * If total number of sectors is less than the first segment
3111 		 * size, something has gone terribly wrong.
3112 		 */
3113 		if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
3114 			blk_dump_rq_flags(req, "request botched");
3115 			req->__data_len = blk_rq_cur_bytes(req);
3116 		}
3117 
3118 		/* recalculate the number of segments */
3119 		blk_recalc_rq_segments(req);
3120 	}
3121 
3122 	return true;
3123 }
3124 EXPORT_SYMBOL_GPL(blk_update_request);
3125 
3126 static bool blk_update_bidi_request(struct request *rq, blk_status_t error,
3127 				    unsigned int nr_bytes,
3128 				    unsigned int bidi_bytes)
3129 {
3130 	if (blk_update_request(rq, error, nr_bytes))
3131 		return true;
3132 
3133 	/* Bidi request must be completed as a whole */
3134 	if (unlikely(blk_bidi_rq(rq)) &&
3135 	    blk_update_request(rq->next_rq, error, bidi_bytes))
3136 		return true;
3137 
3138 	if (blk_queue_add_random(rq->q))
3139 		add_disk_randomness(rq->rq_disk);
3140 
3141 	return false;
3142 }
3143 
3144 /**
3145  * blk_unprep_request - unprepare a request
3146  * @req:	the request
3147  *
3148  * This function makes a request ready for complete resubmission (or
3149  * completion).  It happens only after all error handling is complete,
3150  * so represents the appropriate moment to deallocate any resources
3151  * that were allocated to the request in the prep_rq_fn.  The queue
3152  * lock is held when calling this.
3153  */
3154 void blk_unprep_request(struct request *req)
3155 {
3156 	struct request_queue *q = req->q;
3157 
3158 	req->rq_flags &= ~RQF_DONTPREP;
3159 	if (q->unprep_rq_fn)
3160 		q->unprep_rq_fn(q, req);
3161 }
3162 EXPORT_SYMBOL_GPL(blk_unprep_request);
3163 
3164 void blk_finish_request(struct request *req, blk_status_t error)
3165 {
3166 	struct request_queue *q = req->q;
3167 	u64 now = ktime_get_ns();
3168 
3169 	lockdep_assert_held(req->q->queue_lock);
3170 	WARN_ON_ONCE(q->mq_ops);
3171 
3172 	if (req->rq_flags & RQF_STATS)
3173 		blk_stat_add(req, now);
3174 
3175 	if (req->rq_flags & RQF_QUEUED)
3176 		blk_queue_end_tag(q, req);
3177 
3178 	BUG_ON(blk_queued_rq(req));
3179 
3180 	if (unlikely(laptop_mode) && !blk_rq_is_passthrough(req))
3181 		laptop_io_completion(req->q->backing_dev_info);
3182 
3183 	blk_delete_timer(req);
3184 
3185 	if (req->rq_flags & RQF_DONTPREP)
3186 		blk_unprep_request(req);
3187 
3188 	blk_account_io_done(req, now);
3189 
3190 	if (req->end_io) {
3191 		rq_qos_done(q, req);
3192 		req->end_io(req, error);
3193 	} else {
3194 		if (blk_bidi_rq(req))
3195 			__blk_put_request(req->next_rq->q, req->next_rq);
3196 
3197 		__blk_put_request(q, req);
3198 	}
3199 }
3200 EXPORT_SYMBOL(blk_finish_request);
3201 
3202 /**
3203  * blk_end_bidi_request - Complete a bidi request
3204  * @rq:         the request to complete
3205  * @error:      block status code
3206  * @nr_bytes:   number of bytes to complete @rq
3207  * @bidi_bytes: number of bytes to complete @rq->next_rq
3208  *
3209  * Description:
3210  *     Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
3211  *     Drivers that supports bidi can safely call this member for any
3212  *     type of request, bidi or uni.  In the later case @bidi_bytes is
3213  *     just ignored.
3214  *
3215  * Return:
3216  *     %false - we are done with this request
3217  *     %true  - still buffers pending for this request
3218  **/
3219 static bool blk_end_bidi_request(struct request *rq, blk_status_t error,
3220 				 unsigned int nr_bytes, unsigned int bidi_bytes)
3221 {
3222 	struct request_queue *q = rq->q;
3223 	unsigned long flags;
3224 
3225 	WARN_ON_ONCE(q->mq_ops);
3226 
3227 	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
3228 		return true;
3229 
3230 	spin_lock_irqsave(q->queue_lock, flags);
3231 	blk_finish_request(rq, error);
3232 	spin_unlock_irqrestore(q->queue_lock, flags);
3233 
3234 	return false;
3235 }
3236 
3237 /**
3238  * __blk_end_bidi_request - Complete a bidi request with queue lock held
3239  * @rq:         the request to complete
3240  * @error:      block status code
3241  * @nr_bytes:   number of bytes to complete @rq
3242  * @bidi_bytes: number of bytes to complete @rq->next_rq
3243  *
3244  * Description:
3245  *     Identical to blk_end_bidi_request() except that queue lock is
3246  *     assumed to be locked on entry and remains so on return.
3247  *
3248  * Return:
3249  *     %false - we are done with this request
3250  *     %true  - still buffers pending for this request
3251  **/
3252 static bool __blk_end_bidi_request(struct request *rq, blk_status_t error,
3253 				   unsigned int nr_bytes, unsigned int bidi_bytes)
3254 {
3255 	lockdep_assert_held(rq->q->queue_lock);
3256 	WARN_ON_ONCE(rq->q->mq_ops);
3257 
3258 	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
3259 		return true;
3260 
3261 	blk_finish_request(rq, error);
3262 
3263 	return false;
3264 }
3265 
3266 /**
3267  * blk_end_request - Helper function for drivers to complete the request.
3268  * @rq:       the request being processed
3269  * @error:    block status code
3270  * @nr_bytes: number of bytes to complete
3271  *
3272  * Description:
3273  *     Ends I/O on a number of bytes attached to @rq.
3274  *     If @rq has leftover, sets it up for the next range of segments.
3275  *
3276  * Return:
3277  *     %false - we are done with this request
3278  *     %true  - still buffers pending for this request
3279  **/
3280 bool blk_end_request(struct request *rq, blk_status_t error,
3281 		unsigned int nr_bytes)
3282 {
3283 	WARN_ON_ONCE(rq->q->mq_ops);
3284 	return blk_end_bidi_request(rq, error, nr_bytes, 0);
3285 }
3286 EXPORT_SYMBOL(blk_end_request);
3287 
3288 /**
3289  * blk_end_request_all - Helper function for drives to finish the request.
3290  * @rq: the request to finish
3291  * @error: block status code
3292  *
3293  * Description:
3294  *     Completely finish @rq.
3295  */
3296 void blk_end_request_all(struct request *rq, blk_status_t error)
3297 {
3298 	bool pending;
3299 	unsigned int bidi_bytes = 0;
3300 
3301 	if (unlikely(blk_bidi_rq(rq)))
3302 		bidi_bytes = blk_rq_bytes(rq->next_rq);
3303 
3304 	pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
3305 	BUG_ON(pending);
3306 }
3307 EXPORT_SYMBOL(blk_end_request_all);
3308 
3309 /**
3310  * __blk_end_request - Helper function for drivers to complete the request.
3311  * @rq:       the request being processed
3312  * @error:    block status code
3313  * @nr_bytes: number of bytes to complete
3314  *
3315  * Description:
3316  *     Must be called with queue lock held unlike blk_end_request().
3317  *
3318  * Return:
3319  *     %false - we are done with this request
3320  *     %true  - still buffers pending for this request
3321  **/
3322 bool __blk_end_request(struct request *rq, blk_status_t error,
3323 		unsigned int nr_bytes)
3324 {
3325 	lockdep_assert_held(rq->q->queue_lock);
3326 	WARN_ON_ONCE(rq->q->mq_ops);
3327 
3328 	return __blk_end_bidi_request(rq, error, nr_bytes, 0);
3329 }
3330 EXPORT_SYMBOL(__blk_end_request);
3331 
3332 /**
3333  * __blk_end_request_all - Helper function for drives to finish the request.
3334  * @rq: the request to finish
3335  * @error:    block status code
3336  *
3337  * Description:
3338  *     Completely finish @rq.  Must be called with queue lock held.
3339  */
3340 void __blk_end_request_all(struct request *rq, blk_status_t error)
3341 {
3342 	bool pending;
3343 	unsigned int bidi_bytes = 0;
3344 
3345 	lockdep_assert_held(rq->q->queue_lock);
3346 	WARN_ON_ONCE(rq->q->mq_ops);
3347 
3348 	if (unlikely(blk_bidi_rq(rq)))
3349 		bidi_bytes = blk_rq_bytes(rq->next_rq);
3350 
3351 	pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
3352 	BUG_ON(pending);
3353 }
3354 EXPORT_SYMBOL(__blk_end_request_all);
3355 
3356 /**
3357  * __blk_end_request_cur - Helper function to finish the current request chunk.
3358  * @rq: the request to finish the current chunk for
3359  * @error:    block status code
3360  *
3361  * Description:
3362  *     Complete the current consecutively mapped chunk from @rq.  Must
3363  *     be called with queue lock held.
3364  *
3365  * Return:
3366  *     %false - we are done with this request
3367  *     %true  - still buffers pending for this request
3368  */
3369 bool __blk_end_request_cur(struct request *rq, blk_status_t error)
3370 {
3371 	return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
3372 }
3373 EXPORT_SYMBOL(__blk_end_request_cur);
3374 
3375 void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
3376 		     struct bio *bio)
3377 {
3378 	if (bio_has_data(bio))
3379 		rq->nr_phys_segments = bio_phys_segments(q, bio);
3380 	else if (bio_op(bio) == REQ_OP_DISCARD)
3381 		rq->nr_phys_segments = 1;
3382 
3383 	rq->__data_len = bio->bi_iter.bi_size;
3384 	rq->bio = rq->biotail = bio;
3385 
3386 	if (bio->bi_disk)
3387 		rq->rq_disk = bio->bi_disk;
3388 }
3389 
3390 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
3391 /**
3392  * rq_flush_dcache_pages - Helper function to flush all pages in a request
3393  * @rq: the request to be flushed
3394  *
3395  * Description:
3396  *     Flush all pages in @rq.
3397  */
3398 void rq_flush_dcache_pages(struct request *rq)
3399 {
3400 	struct req_iterator iter;
3401 	struct bio_vec bvec;
3402 
3403 	rq_for_each_segment(bvec, rq, iter)
3404 		flush_dcache_page(bvec.bv_page);
3405 }
3406 EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
3407 #endif
3408 
3409 /**
3410  * blk_lld_busy - Check if underlying low-level drivers of a device are busy
3411  * @q : the queue of the device being checked
3412  *
3413  * Description:
3414  *    Check if underlying low-level drivers of a device are busy.
3415  *    If the drivers want to export their busy state, they must set own
3416  *    exporting function using blk_queue_lld_busy() first.
3417  *
3418  *    Basically, this function is used only by request stacking drivers
3419  *    to stop dispatching requests to underlying devices when underlying
3420  *    devices are busy.  This behavior helps more I/O merging on the queue
3421  *    of the request stacking driver and prevents I/O throughput regression
3422  *    on burst I/O load.
3423  *
3424  * Return:
3425  *    0 - Not busy (The request stacking driver should dispatch request)
3426  *    1 - Busy (The request stacking driver should stop dispatching request)
3427  */
3428 int blk_lld_busy(struct request_queue *q)
3429 {
3430 	if (q->lld_busy_fn)
3431 		return q->lld_busy_fn(q);
3432 
3433 	return 0;
3434 }
3435 EXPORT_SYMBOL_GPL(blk_lld_busy);
3436 
3437 /**
3438  * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
3439  * @rq: the clone request to be cleaned up
3440  *
3441  * Description:
3442  *     Free all bios in @rq for a cloned request.
3443  */
3444 void blk_rq_unprep_clone(struct request *rq)
3445 {
3446 	struct bio *bio;
3447 
3448 	while ((bio = rq->bio) != NULL) {
3449 		rq->bio = bio->bi_next;
3450 
3451 		bio_put(bio);
3452 	}
3453 }
3454 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
3455 
3456 /*
3457  * Copy attributes of the original request to the clone request.
3458  * The actual data parts (e.g. ->cmd, ->sense) are not copied.
3459  */
3460 static void __blk_rq_prep_clone(struct request *dst, struct request *src)
3461 {
3462 	dst->cpu = src->cpu;
3463 	dst->__sector = blk_rq_pos(src);
3464 	dst->__data_len = blk_rq_bytes(src);
3465 	if (src->rq_flags & RQF_SPECIAL_PAYLOAD) {
3466 		dst->rq_flags |= RQF_SPECIAL_PAYLOAD;
3467 		dst->special_vec = src->special_vec;
3468 	}
3469 	dst->nr_phys_segments = src->nr_phys_segments;
3470 	dst->ioprio = src->ioprio;
3471 	dst->extra_len = src->extra_len;
3472 }
3473 
3474 /**
3475  * blk_rq_prep_clone - Helper function to setup clone request
3476  * @rq: the request to be setup
3477  * @rq_src: original request to be cloned
3478  * @bs: bio_set that bios for clone are allocated from
3479  * @gfp_mask: memory allocation mask for bio
3480  * @bio_ctr: setup function to be called for each clone bio.
3481  *           Returns %0 for success, non %0 for failure.
3482  * @data: private data to be passed to @bio_ctr
3483  *
3484  * Description:
3485  *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
3486  *     The actual data parts of @rq_src (e.g. ->cmd, ->sense)
3487  *     are not copied, and copying such parts is the caller's responsibility.
3488  *     Also, pages which the original bios are pointing to are not copied
3489  *     and the cloned bios just point same pages.
3490  *     So cloned bios must be completed before original bios, which means
3491  *     the caller must complete @rq before @rq_src.
3492  */
3493 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
3494 		      struct bio_set *bs, gfp_t gfp_mask,
3495 		      int (*bio_ctr)(struct bio *, struct bio *, void *),
3496 		      void *data)
3497 {
3498 	struct bio *bio, *bio_src;
3499 
3500 	if (!bs)
3501 		bs = &fs_bio_set;
3502 
3503 	__rq_for_each_bio(bio_src, rq_src) {
3504 		bio = bio_clone_fast(bio_src, gfp_mask, bs);
3505 		if (!bio)
3506 			goto free_and_out;
3507 
3508 		if (bio_ctr && bio_ctr(bio, bio_src, data))
3509 			goto free_and_out;
3510 
3511 		if (rq->bio) {
3512 			rq->biotail->bi_next = bio;
3513 			rq->biotail = bio;
3514 		} else
3515 			rq->bio = rq->biotail = bio;
3516 	}
3517 
3518 	__blk_rq_prep_clone(rq, rq_src);
3519 
3520 	return 0;
3521 
3522 free_and_out:
3523 	if (bio)
3524 		bio_put(bio);
3525 	blk_rq_unprep_clone(rq);
3526 
3527 	return -ENOMEM;
3528 }
3529 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
3530 
3531 int kblockd_schedule_work(struct work_struct *work)
3532 {
3533 	return queue_work(kblockd_workqueue, work);
3534 }
3535 EXPORT_SYMBOL(kblockd_schedule_work);
3536 
3537 int kblockd_schedule_work_on(int cpu, struct work_struct *work)
3538 {
3539 	return queue_work_on(cpu, kblockd_workqueue, work);
3540 }
3541 EXPORT_SYMBOL(kblockd_schedule_work_on);
3542 
3543 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
3544 				unsigned long delay)
3545 {
3546 	return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
3547 }
3548 EXPORT_SYMBOL(kblockd_mod_delayed_work_on);
3549 
3550 /**
3551  * blk_start_plug - initialize blk_plug and track it inside the task_struct
3552  * @plug:	The &struct blk_plug that needs to be initialized
3553  *
3554  * Description:
3555  *   Tracking blk_plug inside the task_struct will help with auto-flushing the
3556  *   pending I/O should the task end up blocking between blk_start_plug() and
3557  *   blk_finish_plug(). This is important from a performance perspective, but
3558  *   also ensures that we don't deadlock. For instance, if the task is blocking
3559  *   for a memory allocation, memory reclaim could end up wanting to free a
3560  *   page belonging to that request that is currently residing in our private
3561  *   plug. By flushing the pending I/O when the process goes to sleep, we avoid
3562  *   this kind of deadlock.
3563  */
3564 void blk_start_plug(struct blk_plug *plug)
3565 {
3566 	struct task_struct *tsk = current;
3567 
3568 	/*
3569 	 * If this is a nested plug, don't actually assign it.
3570 	 */
3571 	if (tsk->plug)
3572 		return;
3573 
3574 	INIT_LIST_HEAD(&plug->list);
3575 	INIT_LIST_HEAD(&plug->mq_list);
3576 	INIT_LIST_HEAD(&plug->cb_list);
3577 	/*
3578 	 * Store ordering should not be needed here, since a potential
3579 	 * preempt will imply a full memory barrier
3580 	 */
3581 	tsk->plug = plug;
3582 }
3583 EXPORT_SYMBOL(blk_start_plug);
3584 
3585 static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
3586 {
3587 	struct request *rqa = container_of(a, struct request, queuelist);
3588 	struct request *rqb = container_of(b, struct request, queuelist);
3589 
3590 	return !(rqa->q < rqb->q ||
3591 		(rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb)));
3592 }
3593 
3594 /*
3595  * If 'from_schedule' is true, then postpone the dispatch of requests
3596  * until a safe kblockd context. We due this to avoid accidental big
3597  * additional stack usage in driver dispatch, in places where the originally
3598  * plugger did not intend it.
3599  */
3600 static void queue_unplugged(struct request_queue *q, unsigned int depth,
3601 			    bool from_schedule)
3602 	__releases(q->queue_lock)
3603 {
3604 	lockdep_assert_held(q->queue_lock);
3605 
3606 	trace_block_unplug(q, depth, !from_schedule);
3607 
3608 	if (from_schedule)
3609 		blk_run_queue_async(q);
3610 	else
3611 		__blk_run_queue(q);
3612 	spin_unlock_irq(q->queue_lock);
3613 }
3614 
3615 static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
3616 {
3617 	LIST_HEAD(callbacks);
3618 
3619 	while (!list_empty(&plug->cb_list)) {
3620 		list_splice_init(&plug->cb_list, &callbacks);
3621 
3622 		while (!list_empty(&callbacks)) {
3623 			struct blk_plug_cb *cb = list_first_entry(&callbacks,
3624 							  struct blk_plug_cb,
3625 							  list);
3626 			list_del(&cb->list);
3627 			cb->callback(cb, from_schedule);
3628 		}
3629 	}
3630 }
3631 
3632 struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
3633 				      int size)
3634 {
3635 	struct blk_plug *plug = current->plug;
3636 	struct blk_plug_cb *cb;
3637 
3638 	if (!plug)
3639 		return NULL;
3640 
3641 	list_for_each_entry(cb, &plug->cb_list, list)
3642 		if (cb->callback == unplug && cb->data == data)
3643 			return cb;
3644 
3645 	/* Not currently on the callback list */
3646 	BUG_ON(size < sizeof(*cb));
3647 	cb = kzalloc(size, GFP_ATOMIC);
3648 	if (cb) {
3649 		cb->data = data;
3650 		cb->callback = unplug;
3651 		list_add(&cb->list, &plug->cb_list);
3652 	}
3653 	return cb;
3654 }
3655 EXPORT_SYMBOL(blk_check_plugged);
3656 
3657 void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
3658 {
3659 	struct request_queue *q;
3660 	struct request *rq;
3661 	LIST_HEAD(list);
3662 	unsigned int depth;
3663 
3664 	flush_plug_callbacks(plug, from_schedule);
3665 
3666 	if (!list_empty(&plug->mq_list))
3667 		blk_mq_flush_plug_list(plug, from_schedule);
3668 
3669 	if (list_empty(&plug->list))
3670 		return;
3671 
3672 	list_splice_init(&plug->list, &list);
3673 
3674 	list_sort(NULL, &list, plug_rq_cmp);
3675 
3676 	q = NULL;
3677 	depth = 0;
3678 
3679 	while (!list_empty(&list)) {
3680 		rq = list_entry_rq(list.next);
3681 		list_del_init(&rq->queuelist);
3682 		BUG_ON(!rq->q);
3683 		if (rq->q != q) {
3684 			/*
3685 			 * This drops the queue lock
3686 			 */
3687 			if (q)
3688 				queue_unplugged(q, depth, from_schedule);
3689 			q = rq->q;
3690 			depth = 0;
3691 			spin_lock_irq(q->queue_lock);
3692 		}
3693 
3694 		/*
3695 		 * Short-circuit if @q is dead
3696 		 */
3697 		if (unlikely(blk_queue_dying(q))) {
3698 			__blk_end_request_all(rq, BLK_STS_IOERR);
3699 			continue;
3700 		}
3701 
3702 		/*
3703 		 * rq is already accounted, so use raw insert
3704 		 */
3705 		if (op_is_flush(rq->cmd_flags))
3706 			__elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
3707 		else
3708 			__elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
3709 
3710 		depth++;
3711 	}
3712 
3713 	/*
3714 	 * This drops the queue lock
3715 	 */
3716 	if (q)
3717 		queue_unplugged(q, depth, from_schedule);
3718 }
3719 
3720 void blk_finish_plug(struct blk_plug *plug)
3721 {
3722 	if (plug != current->plug)
3723 		return;
3724 	blk_flush_plug_list(plug, false);
3725 
3726 	current->plug = NULL;
3727 }
3728 EXPORT_SYMBOL(blk_finish_plug);
3729 
3730 int __init blk_dev_init(void)
3731 {
3732 	BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
3733 	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
3734 			FIELD_SIZEOF(struct request, cmd_flags));
3735 	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
3736 			FIELD_SIZEOF(struct bio, bi_opf));
3737 
3738 	/* used for unplugging and affects IO latency/throughput - HIGHPRI */
3739 	kblockd_workqueue = alloc_workqueue("kblockd",
3740 					    WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
3741 	if (!kblockd_workqueue)
3742 		panic("Failed to create kblockd\n");
3743 
3744 	request_cachep = kmem_cache_create("blkdev_requests",
3745 			sizeof(struct request), 0, SLAB_PANIC, NULL);
3746 
3747 	blk_requestq_cachep = kmem_cache_create("request_queue",
3748 			sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
3749 
3750 #ifdef CONFIG_DEBUG_FS
3751 	blk_debugfs_root = debugfs_create_dir("block", NULL);
3752 #endif
3753 
3754 	return 0;
3755 }
3756