xref: /openbmc/linux/block/blk-mq.c (revision 174cd4b1)
1 /*
2  * Block multiqueue core code
3  *
4  * Copyright (C) 2013-2014 Jens Axboe
5  * Copyright (C) 2013-2014 Christoph Hellwig
6  */
7 #include <linux/kernel.h>
8 #include <linux/module.h>
9 #include <linux/backing-dev.h>
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/kmemleak.h>
13 #include <linux/mm.h>
14 #include <linux/init.h>
15 #include <linux/slab.h>
16 #include <linux/workqueue.h>
17 #include <linux/smp.h>
18 #include <linux/llist.h>
19 #include <linux/list_sort.h>
20 #include <linux/cpu.h>
21 #include <linux/cache.h>
22 #include <linux/sched/sysctl.h>
23 #include <linux/sched/topology.h>
24 #include <linux/sched/signal.h>
25 #include <linux/delay.h>
26 #include <linux/crash_dump.h>
27 #include <linux/prefetch.h>
28 
29 #include <trace/events/block.h>
30 
31 #include <linux/blk-mq.h>
32 #include "blk.h"
33 #include "blk-mq.h"
34 #include "blk-mq-tag.h"
35 #include "blk-stat.h"
36 #include "blk-wbt.h"
37 #include "blk-mq-sched.h"
38 
39 static DEFINE_MUTEX(all_q_mutex);
40 static LIST_HEAD(all_q_list);
41 
42 /*
43  * Check if any of the ctx's have pending work in this hardware queue
44  */
45 bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
46 {
47 	return sbitmap_any_bit_set(&hctx->ctx_map) ||
48 			!list_empty_careful(&hctx->dispatch) ||
49 			blk_mq_sched_has_work(hctx);
50 }
51 
52 /*
53  * Mark this ctx as having pending work in this hardware queue
54  */
55 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
56 				     struct blk_mq_ctx *ctx)
57 {
58 	if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw))
59 		sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw);
60 }
61 
62 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
63 				      struct blk_mq_ctx *ctx)
64 {
65 	sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw);
66 }
67 
68 void blk_mq_freeze_queue_start(struct request_queue *q)
69 {
70 	int freeze_depth;
71 
72 	freeze_depth = atomic_inc_return(&q->mq_freeze_depth);
73 	if (freeze_depth == 1) {
74 		percpu_ref_kill(&q->q_usage_counter);
75 		blk_mq_run_hw_queues(q, false);
76 	}
77 }
78 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_start);
79 
80 static void blk_mq_freeze_queue_wait(struct request_queue *q)
81 {
82 	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
83 }
84 
85 /*
86  * Guarantee no request is in use, so we can change any data structure of
87  * the queue afterward.
88  */
89 void blk_freeze_queue(struct request_queue *q)
90 {
91 	/*
92 	 * In the !blk_mq case we are only calling this to kill the
93 	 * q_usage_counter, otherwise this increases the freeze depth
94 	 * and waits for it to return to zero.  For this reason there is
95 	 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
96 	 * exported to drivers as the only user for unfreeze is blk_mq.
97 	 */
98 	blk_mq_freeze_queue_start(q);
99 	blk_mq_freeze_queue_wait(q);
100 }
101 
102 void blk_mq_freeze_queue(struct request_queue *q)
103 {
104 	/*
105 	 * ...just an alias to keep freeze and unfreeze actions balanced
106 	 * in the blk_mq_* namespace
107 	 */
108 	blk_freeze_queue(q);
109 }
110 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
111 
112 void blk_mq_unfreeze_queue(struct request_queue *q)
113 {
114 	int freeze_depth;
115 
116 	freeze_depth = atomic_dec_return(&q->mq_freeze_depth);
117 	WARN_ON_ONCE(freeze_depth < 0);
118 	if (!freeze_depth) {
119 		percpu_ref_reinit(&q->q_usage_counter);
120 		wake_up_all(&q->mq_freeze_wq);
121 	}
122 }
123 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
124 
125 /**
126  * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished
127  * @q: request queue.
128  *
129  * Note: this function does not prevent that the struct request end_io()
130  * callback function is invoked. Additionally, it is not prevented that
131  * new queue_rq() calls occur unless the queue has been stopped first.
132  */
133 void blk_mq_quiesce_queue(struct request_queue *q)
134 {
135 	struct blk_mq_hw_ctx *hctx;
136 	unsigned int i;
137 	bool rcu = false;
138 
139 	blk_mq_stop_hw_queues(q);
140 
141 	queue_for_each_hw_ctx(q, hctx, i) {
142 		if (hctx->flags & BLK_MQ_F_BLOCKING)
143 			synchronize_srcu(&hctx->queue_rq_srcu);
144 		else
145 			rcu = true;
146 	}
147 	if (rcu)
148 		synchronize_rcu();
149 }
150 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
151 
152 void blk_mq_wake_waiters(struct request_queue *q)
153 {
154 	struct blk_mq_hw_ctx *hctx;
155 	unsigned int i;
156 
157 	queue_for_each_hw_ctx(q, hctx, i)
158 		if (blk_mq_hw_queue_mapped(hctx))
159 			blk_mq_tag_wakeup_all(hctx->tags, true);
160 
161 	/*
162 	 * If we are called because the queue has now been marked as
163 	 * dying, we need to ensure that processes currently waiting on
164 	 * the queue are notified as well.
165 	 */
166 	wake_up_all(&q->mq_freeze_wq);
167 }
168 
169 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
170 {
171 	return blk_mq_has_free_tags(hctx->tags);
172 }
173 EXPORT_SYMBOL(blk_mq_can_queue);
174 
175 void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
176 			struct request *rq, unsigned int op)
177 {
178 	INIT_LIST_HEAD(&rq->queuelist);
179 	/* csd/requeue_work/fifo_time is initialized before use */
180 	rq->q = q;
181 	rq->mq_ctx = ctx;
182 	rq->cmd_flags = op;
183 	if (blk_queue_io_stat(q))
184 		rq->rq_flags |= RQF_IO_STAT;
185 	/* do not touch atomic flags, it needs atomic ops against the timer */
186 	rq->cpu = -1;
187 	INIT_HLIST_NODE(&rq->hash);
188 	RB_CLEAR_NODE(&rq->rb_node);
189 	rq->rq_disk = NULL;
190 	rq->part = NULL;
191 	rq->start_time = jiffies;
192 #ifdef CONFIG_BLK_CGROUP
193 	rq->rl = NULL;
194 	set_start_time_ns(rq);
195 	rq->io_start_time_ns = 0;
196 #endif
197 	rq->nr_phys_segments = 0;
198 #if defined(CONFIG_BLK_DEV_INTEGRITY)
199 	rq->nr_integrity_segments = 0;
200 #endif
201 	rq->special = NULL;
202 	/* tag was already set */
203 	rq->errors = 0;
204 	rq->extra_len = 0;
205 
206 	INIT_LIST_HEAD(&rq->timeout_list);
207 	rq->timeout = 0;
208 
209 	rq->end_io = NULL;
210 	rq->end_io_data = NULL;
211 	rq->next_rq = NULL;
212 
213 	ctx->rq_dispatched[op_is_sync(op)]++;
214 }
215 EXPORT_SYMBOL_GPL(blk_mq_rq_ctx_init);
216 
217 struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
218 				       unsigned int op)
219 {
220 	struct request *rq;
221 	unsigned int tag;
222 
223 	tag = blk_mq_get_tag(data);
224 	if (tag != BLK_MQ_TAG_FAIL) {
225 		struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
226 
227 		rq = tags->static_rqs[tag];
228 
229 		if (data->flags & BLK_MQ_REQ_INTERNAL) {
230 			rq->tag = -1;
231 			rq->internal_tag = tag;
232 		} else {
233 			if (blk_mq_tag_busy(data->hctx)) {
234 				rq->rq_flags = RQF_MQ_INFLIGHT;
235 				atomic_inc(&data->hctx->nr_active);
236 			}
237 			rq->tag = tag;
238 			rq->internal_tag = -1;
239 		}
240 
241 		blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
242 		return rq;
243 	}
244 
245 	return NULL;
246 }
247 EXPORT_SYMBOL_GPL(__blk_mq_alloc_request);
248 
249 struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
250 		unsigned int flags)
251 {
252 	struct blk_mq_alloc_data alloc_data = { .flags = flags };
253 	struct request *rq;
254 	int ret;
255 
256 	ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT);
257 	if (ret)
258 		return ERR_PTR(ret);
259 
260 	rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
261 
262 	blk_mq_put_ctx(alloc_data.ctx);
263 	blk_queue_exit(q);
264 
265 	if (!rq)
266 		return ERR_PTR(-EWOULDBLOCK);
267 
268 	rq->__data_len = 0;
269 	rq->__sector = (sector_t) -1;
270 	rq->bio = rq->biotail = NULL;
271 	return rq;
272 }
273 EXPORT_SYMBOL(blk_mq_alloc_request);
274 
275 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
276 		unsigned int flags, unsigned int hctx_idx)
277 {
278 	struct blk_mq_hw_ctx *hctx;
279 	struct blk_mq_ctx *ctx;
280 	struct request *rq;
281 	struct blk_mq_alloc_data alloc_data;
282 	int ret;
283 
284 	/*
285 	 * If the tag allocator sleeps we could get an allocation for a
286 	 * different hardware context.  No need to complicate the low level
287 	 * allocator for this for the rare use case of a command tied to
288 	 * a specific queue.
289 	 */
290 	if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)))
291 		return ERR_PTR(-EINVAL);
292 
293 	if (hctx_idx >= q->nr_hw_queues)
294 		return ERR_PTR(-EIO);
295 
296 	ret = blk_queue_enter(q, true);
297 	if (ret)
298 		return ERR_PTR(ret);
299 
300 	/*
301 	 * Check if the hardware context is actually mapped to anything.
302 	 * If not tell the caller that it should skip this queue.
303 	 */
304 	hctx = q->queue_hw_ctx[hctx_idx];
305 	if (!blk_mq_hw_queue_mapped(hctx)) {
306 		ret = -EXDEV;
307 		goto out_queue_exit;
308 	}
309 	ctx = __blk_mq_get_ctx(q, cpumask_first(hctx->cpumask));
310 
311 	blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx);
312 	rq = __blk_mq_alloc_request(&alloc_data, rw);
313 	if (!rq) {
314 		ret = -EWOULDBLOCK;
315 		goto out_queue_exit;
316 	}
317 
318 	return rq;
319 
320 out_queue_exit:
321 	blk_queue_exit(q);
322 	return ERR_PTR(ret);
323 }
324 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
325 
326 void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
327 			     struct request *rq)
328 {
329 	const int sched_tag = rq->internal_tag;
330 	struct request_queue *q = rq->q;
331 
332 	if (rq->rq_flags & RQF_MQ_INFLIGHT)
333 		atomic_dec(&hctx->nr_active);
334 
335 	wbt_done(q->rq_wb, &rq->issue_stat);
336 	rq->rq_flags = 0;
337 
338 	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
339 	clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
340 	if (rq->tag != -1)
341 		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
342 	if (sched_tag != -1)
343 		blk_mq_sched_completed_request(hctx, rq);
344 	blk_mq_sched_restart_queues(hctx);
345 	blk_queue_exit(q);
346 }
347 
348 static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx,
349 				     struct request *rq)
350 {
351 	struct blk_mq_ctx *ctx = rq->mq_ctx;
352 
353 	ctx->rq_completed[rq_is_sync(rq)]++;
354 	__blk_mq_finish_request(hctx, ctx, rq);
355 }
356 
357 void blk_mq_finish_request(struct request *rq)
358 {
359 	blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq);
360 }
361 
362 void blk_mq_free_request(struct request *rq)
363 {
364 	blk_mq_sched_put_request(rq);
365 }
366 EXPORT_SYMBOL_GPL(blk_mq_free_request);
367 
368 inline void __blk_mq_end_request(struct request *rq, int error)
369 {
370 	blk_account_io_done(rq);
371 
372 	if (rq->end_io) {
373 		wbt_done(rq->q->rq_wb, &rq->issue_stat);
374 		rq->end_io(rq, error);
375 	} else {
376 		if (unlikely(blk_bidi_rq(rq)))
377 			blk_mq_free_request(rq->next_rq);
378 		blk_mq_free_request(rq);
379 	}
380 }
381 EXPORT_SYMBOL(__blk_mq_end_request);
382 
383 void blk_mq_end_request(struct request *rq, int error)
384 {
385 	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
386 		BUG();
387 	__blk_mq_end_request(rq, error);
388 }
389 EXPORT_SYMBOL(blk_mq_end_request);
390 
391 static void __blk_mq_complete_request_remote(void *data)
392 {
393 	struct request *rq = data;
394 
395 	rq->q->softirq_done_fn(rq);
396 }
397 
398 static void blk_mq_ipi_complete_request(struct request *rq)
399 {
400 	struct blk_mq_ctx *ctx = rq->mq_ctx;
401 	bool shared = false;
402 	int cpu;
403 
404 	if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
405 		rq->q->softirq_done_fn(rq);
406 		return;
407 	}
408 
409 	cpu = get_cpu();
410 	if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
411 		shared = cpus_share_cache(cpu, ctx->cpu);
412 
413 	if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
414 		rq->csd.func = __blk_mq_complete_request_remote;
415 		rq->csd.info = rq;
416 		rq->csd.flags = 0;
417 		smp_call_function_single_async(ctx->cpu, &rq->csd);
418 	} else {
419 		rq->q->softirq_done_fn(rq);
420 	}
421 	put_cpu();
422 }
423 
424 static void blk_mq_stat_add(struct request *rq)
425 {
426 	if (rq->rq_flags & RQF_STATS) {
427 		/*
428 		 * We could rq->mq_ctx here, but there's less of a risk
429 		 * of races if we have the completion event add the stats
430 		 * to the local software queue.
431 		 */
432 		struct blk_mq_ctx *ctx;
433 
434 		ctx = __blk_mq_get_ctx(rq->q, raw_smp_processor_id());
435 		blk_stat_add(&ctx->stat[rq_data_dir(rq)], rq);
436 	}
437 }
438 
439 static void __blk_mq_complete_request(struct request *rq)
440 {
441 	struct request_queue *q = rq->q;
442 
443 	blk_mq_stat_add(rq);
444 
445 	if (!q->softirq_done_fn)
446 		blk_mq_end_request(rq, rq->errors);
447 	else
448 		blk_mq_ipi_complete_request(rq);
449 }
450 
451 /**
452  * blk_mq_complete_request - end I/O on a request
453  * @rq:		the request being processed
454  *
455  * Description:
456  *	Ends all I/O on a request. It does not handle partial completions.
457  *	The actual completion happens out-of-order, through a IPI handler.
458  **/
459 void blk_mq_complete_request(struct request *rq, int error)
460 {
461 	struct request_queue *q = rq->q;
462 
463 	if (unlikely(blk_should_fake_timeout(q)))
464 		return;
465 	if (!blk_mark_rq_complete(rq)) {
466 		rq->errors = error;
467 		__blk_mq_complete_request(rq);
468 	}
469 }
470 EXPORT_SYMBOL(blk_mq_complete_request);
471 
472 int blk_mq_request_started(struct request *rq)
473 {
474 	return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
475 }
476 EXPORT_SYMBOL_GPL(blk_mq_request_started);
477 
478 void blk_mq_start_request(struct request *rq)
479 {
480 	struct request_queue *q = rq->q;
481 
482 	blk_mq_sched_started_request(rq);
483 
484 	trace_block_rq_issue(q, rq);
485 
486 	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
487 		blk_stat_set_issue_time(&rq->issue_stat);
488 		rq->rq_flags |= RQF_STATS;
489 		wbt_issue(q->rq_wb, &rq->issue_stat);
490 	}
491 
492 	blk_add_timer(rq);
493 
494 	/*
495 	 * Ensure that ->deadline is visible before set the started
496 	 * flag and clear the completed flag.
497 	 */
498 	smp_mb__before_atomic();
499 
500 	/*
501 	 * Mark us as started and clear complete. Complete might have been
502 	 * set if requeue raced with timeout, which then marked it as
503 	 * complete. So be sure to clear complete again when we start
504 	 * the request, otherwise we'll ignore the completion event.
505 	 */
506 	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
507 		set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
508 	if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
509 		clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
510 
511 	if (q->dma_drain_size && blk_rq_bytes(rq)) {
512 		/*
513 		 * Make sure space for the drain appears.  We know we can do
514 		 * this because max_hw_segments has been adjusted to be one
515 		 * fewer than the device can handle.
516 		 */
517 		rq->nr_phys_segments++;
518 	}
519 }
520 EXPORT_SYMBOL(blk_mq_start_request);
521 
522 static void __blk_mq_requeue_request(struct request *rq)
523 {
524 	struct request_queue *q = rq->q;
525 
526 	trace_block_rq_requeue(q, rq);
527 	wbt_requeue(q->rq_wb, &rq->issue_stat);
528 	blk_mq_sched_requeue_request(rq);
529 
530 	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
531 		if (q->dma_drain_size && blk_rq_bytes(rq))
532 			rq->nr_phys_segments--;
533 	}
534 }
535 
536 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
537 {
538 	__blk_mq_requeue_request(rq);
539 
540 	BUG_ON(blk_queued_rq(rq));
541 	blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
542 }
543 EXPORT_SYMBOL(blk_mq_requeue_request);
544 
545 static void blk_mq_requeue_work(struct work_struct *work)
546 {
547 	struct request_queue *q =
548 		container_of(work, struct request_queue, requeue_work.work);
549 	LIST_HEAD(rq_list);
550 	struct request *rq, *next;
551 	unsigned long flags;
552 
553 	spin_lock_irqsave(&q->requeue_lock, flags);
554 	list_splice_init(&q->requeue_list, &rq_list);
555 	spin_unlock_irqrestore(&q->requeue_lock, flags);
556 
557 	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
558 		if (!(rq->rq_flags & RQF_SOFTBARRIER))
559 			continue;
560 
561 		rq->rq_flags &= ~RQF_SOFTBARRIER;
562 		list_del_init(&rq->queuelist);
563 		blk_mq_sched_insert_request(rq, true, false, false, true);
564 	}
565 
566 	while (!list_empty(&rq_list)) {
567 		rq = list_entry(rq_list.next, struct request, queuelist);
568 		list_del_init(&rq->queuelist);
569 		blk_mq_sched_insert_request(rq, false, false, false, true);
570 	}
571 
572 	blk_mq_run_hw_queues(q, false);
573 }
574 
575 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
576 				bool kick_requeue_list)
577 {
578 	struct request_queue *q = rq->q;
579 	unsigned long flags;
580 
581 	/*
582 	 * We abuse this flag that is otherwise used by the I/O scheduler to
583 	 * request head insertation from the workqueue.
584 	 */
585 	BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
586 
587 	spin_lock_irqsave(&q->requeue_lock, flags);
588 	if (at_head) {
589 		rq->rq_flags |= RQF_SOFTBARRIER;
590 		list_add(&rq->queuelist, &q->requeue_list);
591 	} else {
592 		list_add_tail(&rq->queuelist, &q->requeue_list);
593 	}
594 	spin_unlock_irqrestore(&q->requeue_lock, flags);
595 
596 	if (kick_requeue_list)
597 		blk_mq_kick_requeue_list(q);
598 }
599 EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
600 
601 void blk_mq_kick_requeue_list(struct request_queue *q)
602 {
603 	kblockd_schedule_delayed_work(&q->requeue_work, 0);
604 }
605 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
606 
607 void blk_mq_delay_kick_requeue_list(struct request_queue *q,
608 				    unsigned long msecs)
609 {
610 	kblockd_schedule_delayed_work(&q->requeue_work,
611 				      msecs_to_jiffies(msecs));
612 }
613 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
614 
615 void blk_mq_abort_requeue_list(struct request_queue *q)
616 {
617 	unsigned long flags;
618 	LIST_HEAD(rq_list);
619 
620 	spin_lock_irqsave(&q->requeue_lock, flags);
621 	list_splice_init(&q->requeue_list, &rq_list);
622 	spin_unlock_irqrestore(&q->requeue_lock, flags);
623 
624 	while (!list_empty(&rq_list)) {
625 		struct request *rq;
626 
627 		rq = list_first_entry(&rq_list, struct request, queuelist);
628 		list_del_init(&rq->queuelist);
629 		rq->errors = -EIO;
630 		blk_mq_end_request(rq, rq->errors);
631 	}
632 }
633 EXPORT_SYMBOL(blk_mq_abort_requeue_list);
634 
635 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
636 {
637 	if (tag < tags->nr_tags) {
638 		prefetch(tags->rqs[tag]);
639 		return tags->rqs[tag];
640 	}
641 
642 	return NULL;
643 }
644 EXPORT_SYMBOL(blk_mq_tag_to_rq);
645 
646 struct blk_mq_timeout_data {
647 	unsigned long next;
648 	unsigned int next_set;
649 };
650 
651 void blk_mq_rq_timed_out(struct request *req, bool reserved)
652 {
653 	const struct blk_mq_ops *ops = req->q->mq_ops;
654 	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
655 
656 	/*
657 	 * We know that complete is set at this point. If STARTED isn't set
658 	 * anymore, then the request isn't active and the "timeout" should
659 	 * just be ignored. This can happen due to the bitflag ordering.
660 	 * Timeout first checks if STARTED is set, and if it is, assumes
661 	 * the request is active. But if we race with completion, then
662 	 * we both flags will get cleared. So check here again, and ignore
663 	 * a timeout event with a request that isn't active.
664 	 */
665 	if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
666 		return;
667 
668 	if (ops->timeout)
669 		ret = ops->timeout(req, reserved);
670 
671 	switch (ret) {
672 	case BLK_EH_HANDLED:
673 		__blk_mq_complete_request(req);
674 		break;
675 	case BLK_EH_RESET_TIMER:
676 		blk_add_timer(req);
677 		blk_clear_rq_complete(req);
678 		break;
679 	case BLK_EH_NOT_HANDLED:
680 		break;
681 	default:
682 		printk(KERN_ERR "block: bad eh return: %d\n", ret);
683 		break;
684 	}
685 }
686 
687 static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
688 		struct request *rq, void *priv, bool reserved)
689 {
690 	struct blk_mq_timeout_data *data = priv;
691 
692 	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
693 		/*
694 		 * If a request wasn't started before the queue was
695 		 * marked dying, kill it here or it'll go unnoticed.
696 		 */
697 		if (unlikely(blk_queue_dying(rq->q))) {
698 			rq->errors = -EIO;
699 			blk_mq_end_request(rq, rq->errors);
700 		}
701 		return;
702 	}
703 
704 	if (time_after_eq(jiffies, rq->deadline)) {
705 		if (!blk_mark_rq_complete(rq))
706 			blk_mq_rq_timed_out(rq, reserved);
707 	} else if (!data->next_set || time_after(data->next, rq->deadline)) {
708 		data->next = rq->deadline;
709 		data->next_set = 1;
710 	}
711 }
712 
713 static void blk_mq_timeout_work(struct work_struct *work)
714 {
715 	struct request_queue *q =
716 		container_of(work, struct request_queue, timeout_work);
717 	struct blk_mq_timeout_data data = {
718 		.next		= 0,
719 		.next_set	= 0,
720 	};
721 	int i;
722 
723 	/* A deadlock might occur if a request is stuck requiring a
724 	 * timeout at the same time a queue freeze is waiting
725 	 * completion, since the timeout code would not be able to
726 	 * acquire the queue reference here.
727 	 *
728 	 * That's why we don't use blk_queue_enter here; instead, we use
729 	 * percpu_ref_tryget directly, because we need to be able to
730 	 * obtain a reference even in the short window between the queue
731 	 * starting to freeze, by dropping the first reference in
732 	 * blk_mq_freeze_queue_start, and the moment the last request is
733 	 * consumed, marked by the instant q_usage_counter reaches
734 	 * zero.
735 	 */
736 	if (!percpu_ref_tryget(&q->q_usage_counter))
737 		return;
738 
739 	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
740 
741 	if (data.next_set) {
742 		data.next = blk_rq_timeout(round_jiffies_up(data.next));
743 		mod_timer(&q->timeout, data.next);
744 	} else {
745 		struct blk_mq_hw_ctx *hctx;
746 
747 		queue_for_each_hw_ctx(q, hctx, i) {
748 			/* the hctx may be unmapped, so check it here */
749 			if (blk_mq_hw_queue_mapped(hctx))
750 				blk_mq_tag_idle(hctx);
751 		}
752 	}
753 	blk_queue_exit(q);
754 }
755 
756 /*
757  * Reverse check our software queue for entries that we could potentially
758  * merge with. Currently includes a hand-wavy stop count of 8, to not spend
759  * too much time checking for merges.
760  */
761 static bool blk_mq_attempt_merge(struct request_queue *q,
762 				 struct blk_mq_ctx *ctx, struct bio *bio)
763 {
764 	struct request *rq;
765 	int checked = 8;
766 
767 	list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
768 		bool merged = false;
769 
770 		if (!checked--)
771 			break;
772 
773 		if (!blk_rq_merge_ok(rq, bio))
774 			continue;
775 
776 		switch (blk_try_merge(rq, bio)) {
777 		case ELEVATOR_BACK_MERGE:
778 			if (blk_mq_sched_allow_merge(q, rq, bio))
779 				merged = bio_attempt_back_merge(q, rq, bio);
780 			break;
781 		case ELEVATOR_FRONT_MERGE:
782 			if (blk_mq_sched_allow_merge(q, rq, bio))
783 				merged = bio_attempt_front_merge(q, rq, bio);
784 			break;
785 		case ELEVATOR_DISCARD_MERGE:
786 			merged = bio_attempt_discard_merge(q, rq, bio);
787 			break;
788 		default:
789 			continue;
790 		}
791 
792 		if (merged)
793 			ctx->rq_merged++;
794 		return merged;
795 	}
796 
797 	return false;
798 }
799 
800 struct flush_busy_ctx_data {
801 	struct blk_mq_hw_ctx *hctx;
802 	struct list_head *list;
803 };
804 
805 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
806 {
807 	struct flush_busy_ctx_data *flush_data = data;
808 	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
809 	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
810 
811 	sbitmap_clear_bit(sb, bitnr);
812 	spin_lock(&ctx->lock);
813 	list_splice_tail_init(&ctx->rq_list, flush_data->list);
814 	spin_unlock(&ctx->lock);
815 	return true;
816 }
817 
818 /*
819  * Process software queues that have been marked busy, splicing them
820  * to the for-dispatch
821  */
822 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
823 {
824 	struct flush_busy_ctx_data data = {
825 		.hctx = hctx,
826 		.list = list,
827 	};
828 
829 	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
830 }
831 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
832 
833 static inline unsigned int queued_to_index(unsigned int queued)
834 {
835 	if (!queued)
836 		return 0;
837 
838 	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
839 }
840 
841 bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx,
842 			   bool wait)
843 {
844 	struct blk_mq_alloc_data data = {
845 		.q = rq->q,
846 		.hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu),
847 		.flags = wait ? 0 : BLK_MQ_REQ_NOWAIT,
848 	};
849 
850 	if (rq->tag != -1) {
851 done:
852 		if (hctx)
853 			*hctx = data.hctx;
854 		return true;
855 	}
856 
857 	rq->tag = blk_mq_get_tag(&data);
858 	if (rq->tag >= 0) {
859 		if (blk_mq_tag_busy(data.hctx)) {
860 			rq->rq_flags |= RQF_MQ_INFLIGHT;
861 			atomic_inc(&data.hctx->nr_active);
862 		}
863 		data.hctx->tags->rqs[rq->tag] = rq;
864 		goto done;
865 	}
866 
867 	return false;
868 }
869 
870 static void blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
871 				  struct request *rq)
872 {
873 	if (rq->tag == -1 || rq->internal_tag == -1)
874 		return;
875 
876 	blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag);
877 	rq->tag = -1;
878 
879 	if (rq->rq_flags & RQF_MQ_INFLIGHT) {
880 		rq->rq_flags &= ~RQF_MQ_INFLIGHT;
881 		atomic_dec(&hctx->nr_active);
882 	}
883 }
884 
885 /*
886  * If we fail getting a driver tag because all the driver tags are already
887  * assigned and on the dispatch list, BUT the first entry does not have a
888  * tag, then we could deadlock. For that case, move entries with assigned
889  * driver tags to the front, leaving the set of tagged requests in the
890  * same order, and the untagged set in the same order.
891  */
892 static bool reorder_tags_to_front(struct list_head *list)
893 {
894 	struct request *rq, *tmp, *first = NULL;
895 
896 	list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) {
897 		if (rq == first)
898 			break;
899 		if (rq->tag != -1) {
900 			list_move(&rq->queuelist, list);
901 			if (!first)
902 				first = rq;
903 		}
904 	}
905 
906 	return first != NULL;
907 }
908 
909 static int blk_mq_dispatch_wake(wait_queue_t *wait, unsigned mode, int flags,
910 				void *key)
911 {
912 	struct blk_mq_hw_ctx *hctx;
913 
914 	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
915 
916 	list_del(&wait->task_list);
917 	clear_bit_unlock(BLK_MQ_S_TAG_WAITING, &hctx->state);
918 	blk_mq_run_hw_queue(hctx, true);
919 	return 1;
920 }
921 
922 static bool blk_mq_dispatch_wait_add(struct blk_mq_hw_ctx *hctx)
923 {
924 	struct sbq_wait_state *ws;
925 
926 	/*
927 	 * The TAG_WAITING bit serves as a lock protecting hctx->dispatch_wait.
928 	 * The thread which wins the race to grab this bit adds the hardware
929 	 * queue to the wait queue.
930 	 */
931 	if (test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state) ||
932 	    test_and_set_bit_lock(BLK_MQ_S_TAG_WAITING, &hctx->state))
933 		return false;
934 
935 	init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
936 	ws = bt_wait_ptr(&hctx->tags->bitmap_tags, hctx);
937 
938 	/*
939 	 * As soon as this returns, it's no longer safe to fiddle with
940 	 * hctx->dispatch_wait, since a completion can wake up the wait queue
941 	 * and unlock the bit.
942 	 */
943 	add_wait_queue(&ws->wait, &hctx->dispatch_wait);
944 	return true;
945 }
946 
947 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list)
948 {
949 	struct request_queue *q = hctx->queue;
950 	struct request *rq;
951 	LIST_HEAD(driver_list);
952 	struct list_head *dptr;
953 	int queued, ret = BLK_MQ_RQ_QUEUE_OK;
954 
955 	/*
956 	 * Start off with dptr being NULL, so we start the first request
957 	 * immediately, even if we have more pending.
958 	 */
959 	dptr = NULL;
960 
961 	/*
962 	 * Now process all the entries, sending them to the driver.
963 	 */
964 	queued = 0;
965 	while (!list_empty(list)) {
966 		struct blk_mq_queue_data bd;
967 
968 		rq = list_first_entry(list, struct request, queuelist);
969 		if (!blk_mq_get_driver_tag(rq, &hctx, false)) {
970 			if (!queued && reorder_tags_to_front(list))
971 				continue;
972 
973 			/*
974 			 * The initial allocation attempt failed, so we need to
975 			 * rerun the hardware queue when a tag is freed.
976 			 */
977 			if (blk_mq_dispatch_wait_add(hctx)) {
978 				/*
979 				 * It's possible that a tag was freed in the
980 				 * window between the allocation failure and
981 				 * adding the hardware queue to the wait queue.
982 				 */
983 				if (!blk_mq_get_driver_tag(rq, &hctx, false))
984 					break;
985 			} else {
986 				break;
987 			}
988 		}
989 
990 		list_del_init(&rq->queuelist);
991 
992 		bd.rq = rq;
993 		bd.list = dptr;
994 		bd.last = list_empty(list);
995 
996 		ret = q->mq_ops->queue_rq(hctx, &bd);
997 		switch (ret) {
998 		case BLK_MQ_RQ_QUEUE_OK:
999 			queued++;
1000 			break;
1001 		case BLK_MQ_RQ_QUEUE_BUSY:
1002 			blk_mq_put_driver_tag(hctx, rq);
1003 			list_add(&rq->queuelist, list);
1004 			__blk_mq_requeue_request(rq);
1005 			break;
1006 		default:
1007 			pr_err("blk-mq: bad return on queue: %d\n", ret);
1008 		case BLK_MQ_RQ_QUEUE_ERROR:
1009 			rq->errors = -EIO;
1010 			blk_mq_end_request(rq, rq->errors);
1011 			break;
1012 		}
1013 
1014 		if (ret == BLK_MQ_RQ_QUEUE_BUSY)
1015 			break;
1016 
1017 		/*
1018 		 * We've done the first request. If we have more than 1
1019 		 * left in the list, set dptr to defer issue.
1020 		 */
1021 		if (!dptr && list->next != list->prev)
1022 			dptr = &driver_list;
1023 	}
1024 
1025 	hctx->dispatched[queued_to_index(queued)]++;
1026 
1027 	/*
1028 	 * Any items that need requeuing? Stuff them into hctx->dispatch,
1029 	 * that is where we will continue on next queue run.
1030 	 */
1031 	if (!list_empty(list)) {
1032 		spin_lock(&hctx->lock);
1033 		list_splice_init(list, &hctx->dispatch);
1034 		spin_unlock(&hctx->lock);
1035 
1036 		/*
1037 		 * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but
1038 		 * it's possible the queue is stopped and restarted again
1039 		 * before this. Queue restart will dispatch requests. And since
1040 		 * requests in rq_list aren't added into hctx->dispatch yet,
1041 		 * the requests in rq_list might get lost.
1042 		 *
1043 		 * blk_mq_run_hw_queue() already checks the STOPPED bit
1044 		 *
1045 		 * If RESTART or TAG_WAITING is set, then let completion restart
1046 		 * the queue instead of potentially looping here.
1047 		 */
1048 		if (!blk_mq_sched_needs_restart(hctx) &&
1049 		    !test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state))
1050 			blk_mq_run_hw_queue(hctx, true);
1051 	}
1052 
1053 	return queued != 0;
1054 }
1055 
1056 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
1057 {
1058 	int srcu_idx;
1059 
1060 	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
1061 		cpu_online(hctx->next_cpu));
1062 
1063 	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
1064 		rcu_read_lock();
1065 		blk_mq_sched_dispatch_requests(hctx);
1066 		rcu_read_unlock();
1067 	} else {
1068 		srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
1069 		blk_mq_sched_dispatch_requests(hctx);
1070 		srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
1071 	}
1072 }
1073 
1074 /*
1075  * It'd be great if the workqueue API had a way to pass
1076  * in a mask and had some smarts for more clever placement.
1077  * For now we just round-robin here, switching for every
1078  * BLK_MQ_CPU_WORK_BATCH queued items.
1079  */
1080 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
1081 {
1082 	if (hctx->queue->nr_hw_queues == 1)
1083 		return WORK_CPU_UNBOUND;
1084 
1085 	if (--hctx->next_cpu_batch <= 0) {
1086 		int next_cpu;
1087 
1088 		next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
1089 		if (next_cpu >= nr_cpu_ids)
1090 			next_cpu = cpumask_first(hctx->cpumask);
1091 
1092 		hctx->next_cpu = next_cpu;
1093 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1094 	}
1095 
1096 	return hctx->next_cpu;
1097 }
1098 
1099 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1100 {
1101 	if (unlikely(blk_mq_hctx_stopped(hctx) ||
1102 		     !blk_mq_hw_queue_mapped(hctx)))
1103 		return;
1104 
1105 	if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
1106 		int cpu = get_cpu();
1107 		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
1108 			__blk_mq_run_hw_queue(hctx);
1109 			put_cpu();
1110 			return;
1111 		}
1112 
1113 		put_cpu();
1114 	}
1115 
1116 	kblockd_schedule_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work);
1117 }
1118 
1119 void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1120 {
1121 	struct blk_mq_hw_ctx *hctx;
1122 	int i;
1123 
1124 	queue_for_each_hw_ctx(q, hctx, i) {
1125 		if (!blk_mq_hctx_has_pending(hctx) ||
1126 		    blk_mq_hctx_stopped(hctx))
1127 			continue;
1128 
1129 		blk_mq_run_hw_queue(hctx, async);
1130 	}
1131 }
1132 EXPORT_SYMBOL(blk_mq_run_hw_queues);
1133 
1134 /**
1135  * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
1136  * @q: request queue.
1137  *
1138  * The caller is responsible for serializing this function against
1139  * blk_mq_{start,stop}_hw_queue().
1140  */
1141 bool blk_mq_queue_stopped(struct request_queue *q)
1142 {
1143 	struct blk_mq_hw_ctx *hctx;
1144 	int i;
1145 
1146 	queue_for_each_hw_ctx(q, hctx, i)
1147 		if (blk_mq_hctx_stopped(hctx))
1148 			return true;
1149 
1150 	return false;
1151 }
1152 EXPORT_SYMBOL(blk_mq_queue_stopped);
1153 
1154 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
1155 {
1156 	cancel_work(&hctx->run_work);
1157 	cancel_delayed_work(&hctx->delay_work);
1158 	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
1159 }
1160 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
1161 
1162 void blk_mq_stop_hw_queues(struct request_queue *q)
1163 {
1164 	struct blk_mq_hw_ctx *hctx;
1165 	int i;
1166 
1167 	queue_for_each_hw_ctx(q, hctx, i)
1168 		blk_mq_stop_hw_queue(hctx);
1169 }
1170 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
1171 
1172 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
1173 {
1174 	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1175 
1176 	blk_mq_run_hw_queue(hctx, false);
1177 }
1178 EXPORT_SYMBOL(blk_mq_start_hw_queue);
1179 
1180 void blk_mq_start_hw_queues(struct request_queue *q)
1181 {
1182 	struct blk_mq_hw_ctx *hctx;
1183 	int i;
1184 
1185 	queue_for_each_hw_ctx(q, hctx, i)
1186 		blk_mq_start_hw_queue(hctx);
1187 }
1188 EXPORT_SYMBOL(blk_mq_start_hw_queues);
1189 
1190 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1191 {
1192 	if (!blk_mq_hctx_stopped(hctx))
1193 		return;
1194 
1195 	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1196 	blk_mq_run_hw_queue(hctx, async);
1197 }
1198 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
1199 
1200 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
1201 {
1202 	struct blk_mq_hw_ctx *hctx;
1203 	int i;
1204 
1205 	queue_for_each_hw_ctx(q, hctx, i)
1206 		blk_mq_start_stopped_hw_queue(hctx, async);
1207 }
1208 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
1209 
1210 static void blk_mq_run_work_fn(struct work_struct *work)
1211 {
1212 	struct blk_mq_hw_ctx *hctx;
1213 
1214 	hctx = container_of(work, struct blk_mq_hw_ctx, run_work);
1215 
1216 	__blk_mq_run_hw_queue(hctx);
1217 }
1218 
1219 static void blk_mq_delay_work_fn(struct work_struct *work)
1220 {
1221 	struct blk_mq_hw_ctx *hctx;
1222 
1223 	hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);
1224 
1225 	if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
1226 		__blk_mq_run_hw_queue(hctx);
1227 }
1228 
1229 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
1230 {
1231 	if (unlikely(!blk_mq_hw_queue_mapped(hctx)))
1232 		return;
1233 
1234 	blk_mq_stop_hw_queue(hctx);
1235 	kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
1236 			&hctx->delay_work, msecs_to_jiffies(msecs));
1237 }
1238 EXPORT_SYMBOL(blk_mq_delay_queue);
1239 
1240 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
1241 					    struct request *rq,
1242 					    bool at_head)
1243 {
1244 	struct blk_mq_ctx *ctx = rq->mq_ctx;
1245 
1246 	trace_block_rq_insert(hctx->queue, rq);
1247 
1248 	if (at_head)
1249 		list_add(&rq->queuelist, &ctx->rq_list);
1250 	else
1251 		list_add_tail(&rq->queuelist, &ctx->rq_list);
1252 }
1253 
1254 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
1255 			     bool at_head)
1256 {
1257 	struct blk_mq_ctx *ctx = rq->mq_ctx;
1258 
1259 	__blk_mq_insert_req_list(hctx, rq, at_head);
1260 	blk_mq_hctx_mark_pending(hctx, ctx);
1261 }
1262 
1263 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
1264 			    struct list_head *list)
1265 
1266 {
1267 	/*
1268 	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
1269 	 * offline now
1270 	 */
1271 	spin_lock(&ctx->lock);
1272 	while (!list_empty(list)) {
1273 		struct request *rq;
1274 
1275 		rq = list_first_entry(list, struct request, queuelist);
1276 		BUG_ON(rq->mq_ctx != ctx);
1277 		list_del_init(&rq->queuelist);
1278 		__blk_mq_insert_req_list(hctx, rq, false);
1279 	}
1280 	blk_mq_hctx_mark_pending(hctx, ctx);
1281 	spin_unlock(&ctx->lock);
1282 }
1283 
1284 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1285 {
1286 	struct request *rqa = container_of(a, struct request, queuelist);
1287 	struct request *rqb = container_of(b, struct request, queuelist);
1288 
1289 	return !(rqa->mq_ctx < rqb->mq_ctx ||
1290 		 (rqa->mq_ctx == rqb->mq_ctx &&
1291 		  blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1292 }
1293 
1294 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1295 {
1296 	struct blk_mq_ctx *this_ctx;
1297 	struct request_queue *this_q;
1298 	struct request *rq;
1299 	LIST_HEAD(list);
1300 	LIST_HEAD(ctx_list);
1301 	unsigned int depth;
1302 
1303 	list_splice_init(&plug->mq_list, &list);
1304 
1305 	list_sort(NULL, &list, plug_ctx_cmp);
1306 
1307 	this_q = NULL;
1308 	this_ctx = NULL;
1309 	depth = 0;
1310 
1311 	while (!list_empty(&list)) {
1312 		rq = list_entry_rq(list.next);
1313 		list_del_init(&rq->queuelist);
1314 		BUG_ON(!rq->q);
1315 		if (rq->mq_ctx != this_ctx) {
1316 			if (this_ctx) {
1317 				trace_block_unplug(this_q, depth, from_schedule);
1318 				blk_mq_sched_insert_requests(this_q, this_ctx,
1319 								&ctx_list,
1320 								from_schedule);
1321 			}
1322 
1323 			this_ctx = rq->mq_ctx;
1324 			this_q = rq->q;
1325 			depth = 0;
1326 		}
1327 
1328 		depth++;
1329 		list_add_tail(&rq->queuelist, &ctx_list);
1330 	}
1331 
1332 	/*
1333 	 * If 'this_ctx' is set, we know we have entries to complete
1334 	 * on 'ctx_list'. Do those.
1335 	 */
1336 	if (this_ctx) {
1337 		trace_block_unplug(this_q, depth, from_schedule);
1338 		blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list,
1339 						from_schedule);
1340 	}
1341 }
1342 
1343 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1344 {
1345 	init_request_from_bio(rq, bio);
1346 
1347 	blk_account_io_start(rq, true);
1348 }
1349 
1350 static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
1351 {
1352 	return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
1353 		!blk_queue_nomerges(hctx->queue);
1354 }
1355 
1356 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1357 					 struct blk_mq_ctx *ctx,
1358 					 struct request *rq, struct bio *bio)
1359 {
1360 	if (!hctx_allow_merges(hctx) || !bio_mergeable(bio)) {
1361 		blk_mq_bio_to_request(rq, bio);
1362 		spin_lock(&ctx->lock);
1363 insert_rq:
1364 		__blk_mq_insert_request(hctx, rq, false);
1365 		spin_unlock(&ctx->lock);
1366 		return false;
1367 	} else {
1368 		struct request_queue *q = hctx->queue;
1369 
1370 		spin_lock(&ctx->lock);
1371 		if (!blk_mq_attempt_merge(q, ctx, bio)) {
1372 			blk_mq_bio_to_request(rq, bio);
1373 			goto insert_rq;
1374 		}
1375 
1376 		spin_unlock(&ctx->lock);
1377 		__blk_mq_finish_request(hctx, ctx, rq);
1378 		return true;
1379 	}
1380 }
1381 
1382 static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
1383 {
1384 	if (rq->tag != -1)
1385 		return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false);
1386 
1387 	return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true);
1388 }
1389 
1390 static void blk_mq_try_issue_directly(struct request *rq, blk_qc_t *cookie)
1391 {
1392 	struct request_queue *q = rq->q;
1393 	struct blk_mq_queue_data bd = {
1394 		.rq = rq,
1395 		.list = NULL,
1396 		.last = 1
1397 	};
1398 	struct blk_mq_hw_ctx *hctx;
1399 	blk_qc_t new_cookie;
1400 	int ret;
1401 
1402 	if (q->elevator)
1403 		goto insert;
1404 
1405 	if (!blk_mq_get_driver_tag(rq, &hctx, false))
1406 		goto insert;
1407 
1408 	new_cookie = request_to_qc_t(hctx, rq);
1409 
1410 	/*
1411 	 * For OK queue, we are done. For error, kill it. Any other
1412 	 * error (busy), just add it to our list as we previously
1413 	 * would have done
1414 	 */
1415 	ret = q->mq_ops->queue_rq(hctx, &bd);
1416 	if (ret == BLK_MQ_RQ_QUEUE_OK) {
1417 		*cookie = new_cookie;
1418 		return;
1419 	}
1420 
1421 	__blk_mq_requeue_request(rq);
1422 
1423 	if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1424 		*cookie = BLK_QC_T_NONE;
1425 		rq->errors = -EIO;
1426 		blk_mq_end_request(rq, rq->errors);
1427 		return;
1428 	}
1429 
1430 insert:
1431 	blk_mq_sched_insert_request(rq, false, true, true, false);
1432 }
1433 
1434 /*
1435  * Multiple hardware queue variant. This will not use per-process plugs,
1436  * but will attempt to bypass the hctx queueing if we can go straight to
1437  * hardware for SYNC IO.
1438  */
1439 static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
1440 {
1441 	const int is_sync = op_is_sync(bio->bi_opf);
1442 	const int is_flush_fua = op_is_flush(bio->bi_opf);
1443 	struct blk_mq_alloc_data data = { .flags = 0 };
1444 	struct request *rq;
1445 	unsigned int request_count = 0, srcu_idx;
1446 	struct blk_plug *plug;
1447 	struct request *same_queue_rq = NULL;
1448 	blk_qc_t cookie;
1449 	unsigned int wb_acct;
1450 
1451 	blk_queue_bounce(q, &bio);
1452 
1453 	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1454 		bio_io_error(bio);
1455 		return BLK_QC_T_NONE;
1456 	}
1457 
1458 	blk_queue_split(q, &bio, q->bio_split);
1459 
1460 	if (!is_flush_fua && !blk_queue_nomerges(q) &&
1461 	    blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
1462 		return BLK_QC_T_NONE;
1463 
1464 	if (blk_mq_sched_bio_merge(q, bio))
1465 		return BLK_QC_T_NONE;
1466 
1467 	wb_acct = wbt_wait(q->rq_wb, bio, NULL);
1468 
1469 	trace_block_getrq(q, bio, bio->bi_opf);
1470 
1471 	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
1472 	if (unlikely(!rq)) {
1473 		__wbt_done(q->rq_wb, wb_acct);
1474 		return BLK_QC_T_NONE;
1475 	}
1476 
1477 	wbt_track(&rq->issue_stat, wb_acct);
1478 
1479 	cookie = request_to_qc_t(data.hctx, rq);
1480 
1481 	if (unlikely(is_flush_fua)) {
1482 		if (q->elevator)
1483 			goto elv_insert;
1484 		blk_mq_bio_to_request(rq, bio);
1485 		blk_insert_flush(rq);
1486 		goto run_queue;
1487 	}
1488 
1489 	plug = current->plug;
1490 	/*
1491 	 * If the driver supports defer issued based on 'last', then
1492 	 * queue it up like normal since we can potentially save some
1493 	 * CPU this way.
1494 	 */
1495 	if (((plug && !blk_queue_nomerges(q)) || is_sync) &&
1496 	    !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {
1497 		struct request *old_rq = NULL;
1498 
1499 		blk_mq_bio_to_request(rq, bio);
1500 
1501 		/*
1502 		 * We do limited plugging. If the bio can be merged, do that.
1503 		 * Otherwise the existing request in the plug list will be
1504 		 * issued. So the plug list will have one request at most
1505 		 */
1506 		if (plug) {
1507 			/*
1508 			 * The plug list might get flushed before this. If that
1509 			 * happens, same_queue_rq is invalid and plug list is
1510 			 * empty
1511 			 */
1512 			if (same_queue_rq && !list_empty(&plug->mq_list)) {
1513 				old_rq = same_queue_rq;
1514 				list_del_init(&old_rq->queuelist);
1515 			}
1516 			list_add_tail(&rq->queuelist, &plug->mq_list);
1517 		} else /* is_sync */
1518 			old_rq = rq;
1519 		blk_mq_put_ctx(data.ctx);
1520 		if (!old_rq)
1521 			goto done;
1522 
1523 		if (!(data.hctx->flags & BLK_MQ_F_BLOCKING)) {
1524 			rcu_read_lock();
1525 			blk_mq_try_issue_directly(old_rq, &cookie);
1526 			rcu_read_unlock();
1527 		} else {
1528 			srcu_idx = srcu_read_lock(&data.hctx->queue_rq_srcu);
1529 			blk_mq_try_issue_directly(old_rq, &cookie);
1530 			srcu_read_unlock(&data.hctx->queue_rq_srcu, srcu_idx);
1531 		}
1532 		goto done;
1533 	}
1534 
1535 	if (q->elevator) {
1536 elv_insert:
1537 		blk_mq_put_ctx(data.ctx);
1538 		blk_mq_bio_to_request(rq, bio);
1539 		blk_mq_sched_insert_request(rq, false, true,
1540 						!is_sync || is_flush_fua, true);
1541 		goto done;
1542 	}
1543 	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1544 		/*
1545 		 * For a SYNC request, send it to the hardware immediately. For
1546 		 * an ASYNC request, just ensure that we run it later on. The
1547 		 * latter allows for merging opportunities and more efficient
1548 		 * dispatching.
1549 		 */
1550 run_queue:
1551 		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1552 	}
1553 	blk_mq_put_ctx(data.ctx);
1554 done:
1555 	return cookie;
1556 }
1557 
1558 /*
1559  * Single hardware queue variant. This will attempt to use any per-process
1560  * plug for merging and IO deferral.
1561  */
1562 static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio)
1563 {
1564 	const int is_sync = op_is_sync(bio->bi_opf);
1565 	const int is_flush_fua = op_is_flush(bio->bi_opf);
1566 	struct blk_plug *plug;
1567 	unsigned int request_count = 0;
1568 	struct blk_mq_alloc_data data = { .flags = 0 };
1569 	struct request *rq;
1570 	blk_qc_t cookie;
1571 	unsigned int wb_acct;
1572 
1573 	blk_queue_bounce(q, &bio);
1574 
1575 	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1576 		bio_io_error(bio);
1577 		return BLK_QC_T_NONE;
1578 	}
1579 
1580 	blk_queue_split(q, &bio, q->bio_split);
1581 
1582 	if (!is_flush_fua && !blk_queue_nomerges(q)) {
1583 		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
1584 			return BLK_QC_T_NONE;
1585 	} else
1586 		request_count = blk_plug_queued_count(q);
1587 
1588 	if (blk_mq_sched_bio_merge(q, bio))
1589 		return BLK_QC_T_NONE;
1590 
1591 	wb_acct = wbt_wait(q->rq_wb, bio, NULL);
1592 
1593 	trace_block_getrq(q, bio, bio->bi_opf);
1594 
1595 	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
1596 	if (unlikely(!rq)) {
1597 		__wbt_done(q->rq_wb, wb_acct);
1598 		return BLK_QC_T_NONE;
1599 	}
1600 
1601 	wbt_track(&rq->issue_stat, wb_acct);
1602 
1603 	cookie = request_to_qc_t(data.hctx, rq);
1604 
1605 	if (unlikely(is_flush_fua)) {
1606 		if (q->elevator)
1607 			goto elv_insert;
1608 		blk_mq_bio_to_request(rq, bio);
1609 		blk_insert_flush(rq);
1610 		goto run_queue;
1611 	}
1612 
1613 	/*
1614 	 * A task plug currently exists. Since this is completely lockless,
1615 	 * utilize that to temporarily store requests until the task is
1616 	 * either done or scheduled away.
1617 	 */
1618 	plug = current->plug;
1619 	if (plug) {
1620 		struct request *last = NULL;
1621 
1622 		blk_mq_bio_to_request(rq, bio);
1623 
1624 		/*
1625 		 * @request_count may become stale because of schedule
1626 		 * out, so check the list again.
1627 		 */
1628 		if (list_empty(&plug->mq_list))
1629 			request_count = 0;
1630 		if (!request_count)
1631 			trace_block_plug(q);
1632 		else
1633 			last = list_entry_rq(plug->mq_list.prev);
1634 
1635 		blk_mq_put_ctx(data.ctx);
1636 
1637 		if (request_count >= BLK_MAX_REQUEST_COUNT || (last &&
1638 		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1639 			blk_flush_plug_list(plug, false);
1640 			trace_block_plug(q);
1641 		}
1642 
1643 		list_add_tail(&rq->queuelist, &plug->mq_list);
1644 		return cookie;
1645 	}
1646 
1647 	if (q->elevator) {
1648 elv_insert:
1649 		blk_mq_put_ctx(data.ctx);
1650 		blk_mq_bio_to_request(rq, bio);
1651 		blk_mq_sched_insert_request(rq, false, true,
1652 						!is_sync || is_flush_fua, true);
1653 		goto done;
1654 	}
1655 	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1656 		/*
1657 		 * For a SYNC request, send it to the hardware immediately. For
1658 		 * an ASYNC request, just ensure that we run it later on. The
1659 		 * latter allows for merging opportunities and more efficient
1660 		 * dispatching.
1661 		 */
1662 run_queue:
1663 		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1664 	}
1665 
1666 	blk_mq_put_ctx(data.ctx);
1667 done:
1668 	return cookie;
1669 }
1670 
1671 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
1672 		     unsigned int hctx_idx)
1673 {
1674 	struct page *page;
1675 
1676 	if (tags->rqs && set->ops->exit_request) {
1677 		int i;
1678 
1679 		for (i = 0; i < tags->nr_tags; i++) {
1680 			struct request *rq = tags->static_rqs[i];
1681 
1682 			if (!rq)
1683 				continue;
1684 			set->ops->exit_request(set->driver_data, rq,
1685 						hctx_idx, i);
1686 			tags->static_rqs[i] = NULL;
1687 		}
1688 	}
1689 
1690 	while (!list_empty(&tags->page_list)) {
1691 		page = list_first_entry(&tags->page_list, struct page, lru);
1692 		list_del_init(&page->lru);
1693 		/*
1694 		 * Remove kmemleak object previously allocated in
1695 		 * blk_mq_init_rq_map().
1696 		 */
1697 		kmemleak_free(page_address(page));
1698 		__free_pages(page, page->private);
1699 	}
1700 }
1701 
1702 void blk_mq_free_rq_map(struct blk_mq_tags *tags)
1703 {
1704 	kfree(tags->rqs);
1705 	tags->rqs = NULL;
1706 	kfree(tags->static_rqs);
1707 	tags->static_rqs = NULL;
1708 
1709 	blk_mq_free_tags(tags);
1710 }
1711 
1712 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
1713 					unsigned int hctx_idx,
1714 					unsigned int nr_tags,
1715 					unsigned int reserved_tags)
1716 {
1717 	struct blk_mq_tags *tags;
1718 
1719 	tags = blk_mq_init_tags(nr_tags, reserved_tags,
1720 				set->numa_node,
1721 				BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
1722 	if (!tags)
1723 		return NULL;
1724 
1725 	tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1726 				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1727 				 set->numa_node);
1728 	if (!tags->rqs) {
1729 		blk_mq_free_tags(tags);
1730 		return NULL;
1731 	}
1732 
1733 	tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1734 				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1735 				 set->numa_node);
1736 	if (!tags->static_rqs) {
1737 		kfree(tags->rqs);
1738 		blk_mq_free_tags(tags);
1739 		return NULL;
1740 	}
1741 
1742 	return tags;
1743 }
1744 
1745 static size_t order_to_size(unsigned int order)
1746 {
1747 	return (size_t)PAGE_SIZE << order;
1748 }
1749 
1750 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
1751 		     unsigned int hctx_idx, unsigned int depth)
1752 {
1753 	unsigned int i, j, entries_per_page, max_order = 4;
1754 	size_t rq_size, left;
1755 
1756 	INIT_LIST_HEAD(&tags->page_list);
1757 
1758 	/*
1759 	 * rq_size is the size of the request plus driver payload, rounded
1760 	 * to the cacheline size
1761 	 */
1762 	rq_size = round_up(sizeof(struct request) + set->cmd_size,
1763 				cache_line_size());
1764 	left = rq_size * depth;
1765 
1766 	for (i = 0; i < depth; ) {
1767 		int this_order = max_order;
1768 		struct page *page;
1769 		int to_do;
1770 		void *p;
1771 
1772 		while (this_order && left < order_to_size(this_order - 1))
1773 			this_order--;
1774 
1775 		do {
1776 			page = alloc_pages_node(set->numa_node,
1777 				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
1778 				this_order);
1779 			if (page)
1780 				break;
1781 			if (!this_order--)
1782 				break;
1783 			if (order_to_size(this_order) < rq_size)
1784 				break;
1785 		} while (1);
1786 
1787 		if (!page)
1788 			goto fail;
1789 
1790 		page->private = this_order;
1791 		list_add_tail(&page->lru, &tags->page_list);
1792 
1793 		p = page_address(page);
1794 		/*
1795 		 * Allow kmemleak to scan these pages as they contain pointers
1796 		 * to additional allocations like via ops->init_request().
1797 		 */
1798 		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
1799 		entries_per_page = order_to_size(this_order) / rq_size;
1800 		to_do = min(entries_per_page, depth - i);
1801 		left -= to_do * rq_size;
1802 		for (j = 0; j < to_do; j++) {
1803 			struct request *rq = p;
1804 
1805 			tags->static_rqs[i] = rq;
1806 			if (set->ops->init_request) {
1807 				if (set->ops->init_request(set->driver_data,
1808 						rq, hctx_idx, i,
1809 						set->numa_node)) {
1810 					tags->static_rqs[i] = NULL;
1811 					goto fail;
1812 				}
1813 			}
1814 
1815 			p += rq_size;
1816 			i++;
1817 		}
1818 	}
1819 	return 0;
1820 
1821 fail:
1822 	blk_mq_free_rqs(set, tags, hctx_idx);
1823 	return -ENOMEM;
1824 }
1825 
1826 /*
1827  * 'cpu' is going away. splice any existing rq_list entries from this
1828  * software queue to the hw queue dispatch list, and ensure that it
1829  * gets run.
1830  */
1831 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
1832 {
1833 	struct blk_mq_hw_ctx *hctx;
1834 	struct blk_mq_ctx *ctx;
1835 	LIST_HEAD(tmp);
1836 
1837 	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
1838 	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
1839 
1840 	spin_lock(&ctx->lock);
1841 	if (!list_empty(&ctx->rq_list)) {
1842 		list_splice_init(&ctx->rq_list, &tmp);
1843 		blk_mq_hctx_clear_pending(hctx, ctx);
1844 	}
1845 	spin_unlock(&ctx->lock);
1846 
1847 	if (list_empty(&tmp))
1848 		return 0;
1849 
1850 	spin_lock(&hctx->lock);
1851 	list_splice_tail_init(&tmp, &hctx->dispatch);
1852 	spin_unlock(&hctx->lock);
1853 
1854 	blk_mq_run_hw_queue(hctx, true);
1855 	return 0;
1856 }
1857 
1858 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
1859 {
1860 	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
1861 					    &hctx->cpuhp_dead);
1862 }
1863 
1864 /* hctx->ctxs will be freed in queue's release handler */
1865 static void blk_mq_exit_hctx(struct request_queue *q,
1866 		struct blk_mq_tag_set *set,
1867 		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
1868 {
1869 	unsigned flush_start_tag = set->queue_depth;
1870 
1871 	blk_mq_tag_idle(hctx);
1872 
1873 	if (set->ops->exit_request)
1874 		set->ops->exit_request(set->driver_data,
1875 				       hctx->fq->flush_rq, hctx_idx,
1876 				       flush_start_tag + hctx_idx);
1877 
1878 	if (set->ops->exit_hctx)
1879 		set->ops->exit_hctx(hctx, hctx_idx);
1880 
1881 	if (hctx->flags & BLK_MQ_F_BLOCKING)
1882 		cleanup_srcu_struct(&hctx->queue_rq_srcu);
1883 
1884 	blk_mq_remove_cpuhp(hctx);
1885 	blk_free_flush_queue(hctx->fq);
1886 	sbitmap_free(&hctx->ctx_map);
1887 }
1888 
1889 static void blk_mq_exit_hw_queues(struct request_queue *q,
1890 		struct blk_mq_tag_set *set, int nr_queue)
1891 {
1892 	struct blk_mq_hw_ctx *hctx;
1893 	unsigned int i;
1894 
1895 	queue_for_each_hw_ctx(q, hctx, i) {
1896 		if (i == nr_queue)
1897 			break;
1898 		blk_mq_exit_hctx(q, set, hctx, i);
1899 	}
1900 }
1901 
1902 static void blk_mq_free_hw_queues(struct request_queue *q,
1903 		struct blk_mq_tag_set *set)
1904 {
1905 	struct blk_mq_hw_ctx *hctx;
1906 	unsigned int i;
1907 
1908 	queue_for_each_hw_ctx(q, hctx, i)
1909 		free_cpumask_var(hctx->cpumask);
1910 }
1911 
1912 static int blk_mq_init_hctx(struct request_queue *q,
1913 		struct blk_mq_tag_set *set,
1914 		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1915 {
1916 	int node;
1917 	unsigned flush_start_tag = set->queue_depth;
1918 
1919 	node = hctx->numa_node;
1920 	if (node == NUMA_NO_NODE)
1921 		node = hctx->numa_node = set->numa_node;
1922 
1923 	INIT_WORK(&hctx->run_work, blk_mq_run_work_fn);
1924 	INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
1925 	spin_lock_init(&hctx->lock);
1926 	INIT_LIST_HEAD(&hctx->dispatch);
1927 	hctx->queue = q;
1928 	hctx->queue_num = hctx_idx;
1929 	hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
1930 
1931 	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
1932 
1933 	hctx->tags = set->tags[hctx_idx];
1934 
1935 	/*
1936 	 * Allocate space for all possible cpus to avoid allocation at
1937 	 * runtime
1938 	 */
1939 	hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1940 					GFP_KERNEL, node);
1941 	if (!hctx->ctxs)
1942 		goto unregister_cpu_notifier;
1943 
1944 	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL,
1945 			      node))
1946 		goto free_ctxs;
1947 
1948 	hctx->nr_ctx = 0;
1949 
1950 	if (set->ops->init_hctx &&
1951 	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
1952 		goto free_bitmap;
1953 
1954 	hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
1955 	if (!hctx->fq)
1956 		goto exit_hctx;
1957 
1958 	if (set->ops->init_request &&
1959 	    set->ops->init_request(set->driver_data,
1960 				   hctx->fq->flush_rq, hctx_idx,
1961 				   flush_start_tag + hctx_idx, node))
1962 		goto free_fq;
1963 
1964 	if (hctx->flags & BLK_MQ_F_BLOCKING)
1965 		init_srcu_struct(&hctx->queue_rq_srcu);
1966 
1967 	return 0;
1968 
1969  free_fq:
1970 	kfree(hctx->fq);
1971  exit_hctx:
1972 	if (set->ops->exit_hctx)
1973 		set->ops->exit_hctx(hctx, hctx_idx);
1974  free_bitmap:
1975 	sbitmap_free(&hctx->ctx_map);
1976  free_ctxs:
1977 	kfree(hctx->ctxs);
1978  unregister_cpu_notifier:
1979 	blk_mq_remove_cpuhp(hctx);
1980 	return -1;
1981 }
1982 
1983 static void blk_mq_init_cpu_queues(struct request_queue *q,
1984 				   unsigned int nr_hw_queues)
1985 {
1986 	unsigned int i;
1987 
1988 	for_each_possible_cpu(i) {
1989 		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1990 		struct blk_mq_hw_ctx *hctx;
1991 
1992 		memset(__ctx, 0, sizeof(*__ctx));
1993 		__ctx->cpu = i;
1994 		spin_lock_init(&__ctx->lock);
1995 		INIT_LIST_HEAD(&__ctx->rq_list);
1996 		__ctx->queue = q;
1997 		blk_stat_init(&__ctx->stat[BLK_STAT_READ]);
1998 		blk_stat_init(&__ctx->stat[BLK_STAT_WRITE]);
1999 
2000 		/* If the cpu isn't online, the cpu is mapped to first hctx */
2001 		if (!cpu_online(i))
2002 			continue;
2003 
2004 		hctx = blk_mq_map_queue(q, i);
2005 
2006 		/*
2007 		 * Set local node, IFF we have more than one hw queue. If
2008 		 * not, we remain on the home node of the device
2009 		 */
2010 		if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
2011 			hctx->numa_node = local_memory_node(cpu_to_node(i));
2012 	}
2013 }
2014 
2015 static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx)
2016 {
2017 	int ret = 0;
2018 
2019 	set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
2020 					set->queue_depth, set->reserved_tags);
2021 	if (!set->tags[hctx_idx])
2022 		return false;
2023 
2024 	ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
2025 				set->queue_depth);
2026 	if (!ret)
2027 		return true;
2028 
2029 	blk_mq_free_rq_map(set->tags[hctx_idx]);
2030 	set->tags[hctx_idx] = NULL;
2031 	return false;
2032 }
2033 
2034 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
2035 					 unsigned int hctx_idx)
2036 {
2037 	if (set->tags[hctx_idx]) {
2038 		blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
2039 		blk_mq_free_rq_map(set->tags[hctx_idx]);
2040 		set->tags[hctx_idx] = NULL;
2041 	}
2042 }
2043 
2044 static void blk_mq_map_swqueue(struct request_queue *q,
2045 			       const struct cpumask *online_mask)
2046 {
2047 	unsigned int i, hctx_idx;
2048 	struct blk_mq_hw_ctx *hctx;
2049 	struct blk_mq_ctx *ctx;
2050 	struct blk_mq_tag_set *set = q->tag_set;
2051 
2052 	/*
2053 	 * Avoid others reading imcomplete hctx->cpumask through sysfs
2054 	 */
2055 	mutex_lock(&q->sysfs_lock);
2056 
2057 	queue_for_each_hw_ctx(q, hctx, i) {
2058 		cpumask_clear(hctx->cpumask);
2059 		hctx->nr_ctx = 0;
2060 	}
2061 
2062 	/*
2063 	 * Map software to hardware queues
2064 	 */
2065 	for_each_possible_cpu(i) {
2066 		/* If the cpu isn't online, the cpu is mapped to first hctx */
2067 		if (!cpumask_test_cpu(i, online_mask))
2068 			continue;
2069 
2070 		hctx_idx = q->mq_map[i];
2071 		/* unmapped hw queue can be remapped after CPU topo changed */
2072 		if (!set->tags[hctx_idx] &&
2073 		    !__blk_mq_alloc_rq_map(set, hctx_idx)) {
2074 			/*
2075 			 * If tags initialization fail for some hctx,
2076 			 * that hctx won't be brought online.  In this
2077 			 * case, remap the current ctx to hctx[0] which
2078 			 * is guaranteed to always have tags allocated
2079 			 */
2080 			q->mq_map[i] = 0;
2081 		}
2082 
2083 		ctx = per_cpu_ptr(q->queue_ctx, i);
2084 		hctx = blk_mq_map_queue(q, i);
2085 
2086 		cpumask_set_cpu(i, hctx->cpumask);
2087 		ctx->index_hw = hctx->nr_ctx;
2088 		hctx->ctxs[hctx->nr_ctx++] = ctx;
2089 	}
2090 
2091 	mutex_unlock(&q->sysfs_lock);
2092 
2093 	queue_for_each_hw_ctx(q, hctx, i) {
2094 		/*
2095 		 * If no software queues are mapped to this hardware queue,
2096 		 * disable it and free the request entries.
2097 		 */
2098 		if (!hctx->nr_ctx) {
2099 			/* Never unmap queue 0.  We need it as a
2100 			 * fallback in case of a new remap fails
2101 			 * allocation
2102 			 */
2103 			if (i && set->tags[i])
2104 				blk_mq_free_map_and_requests(set, i);
2105 
2106 			hctx->tags = NULL;
2107 			continue;
2108 		}
2109 
2110 		hctx->tags = set->tags[i];
2111 		WARN_ON(!hctx->tags);
2112 
2113 		/*
2114 		 * Set the map size to the number of mapped software queues.
2115 		 * This is more accurate and more efficient than looping
2116 		 * over all possibly mapped software queues.
2117 		 */
2118 		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
2119 
2120 		/*
2121 		 * Initialize batch roundrobin counts
2122 		 */
2123 		hctx->next_cpu = cpumask_first(hctx->cpumask);
2124 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
2125 	}
2126 }
2127 
2128 static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2129 {
2130 	struct blk_mq_hw_ctx *hctx;
2131 	int i;
2132 
2133 	queue_for_each_hw_ctx(q, hctx, i) {
2134 		if (shared)
2135 			hctx->flags |= BLK_MQ_F_TAG_SHARED;
2136 		else
2137 			hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
2138 	}
2139 }
2140 
2141 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared)
2142 {
2143 	struct request_queue *q;
2144 
2145 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
2146 		blk_mq_freeze_queue(q);
2147 		queue_set_hctx_shared(q, shared);
2148 		blk_mq_unfreeze_queue(q);
2149 	}
2150 }
2151 
2152 static void blk_mq_del_queue_tag_set(struct request_queue *q)
2153 {
2154 	struct blk_mq_tag_set *set = q->tag_set;
2155 
2156 	mutex_lock(&set->tag_list_lock);
2157 	list_del_init(&q->tag_set_list);
2158 	if (list_is_singular(&set->tag_list)) {
2159 		/* just transitioned to unshared */
2160 		set->flags &= ~BLK_MQ_F_TAG_SHARED;
2161 		/* update existing queue */
2162 		blk_mq_update_tag_set_depth(set, false);
2163 	}
2164 	mutex_unlock(&set->tag_list_lock);
2165 }
2166 
2167 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
2168 				     struct request_queue *q)
2169 {
2170 	q->tag_set = set;
2171 
2172 	mutex_lock(&set->tag_list_lock);
2173 
2174 	/* Check to see if we're transitioning to shared (from 1 to 2 queues). */
2175 	if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) {
2176 		set->flags |= BLK_MQ_F_TAG_SHARED;
2177 		/* update existing queue */
2178 		blk_mq_update_tag_set_depth(set, true);
2179 	}
2180 	if (set->flags & BLK_MQ_F_TAG_SHARED)
2181 		queue_set_hctx_shared(q, true);
2182 	list_add_tail(&q->tag_set_list, &set->tag_list);
2183 
2184 	mutex_unlock(&set->tag_list_lock);
2185 }
2186 
2187 /*
2188  * It is the actual release handler for mq, but we do it from
2189  * request queue's release handler for avoiding use-after-free
2190  * and headache because q->mq_kobj shouldn't have been introduced,
2191  * but we can't group ctx/kctx kobj without it.
2192  */
2193 void blk_mq_release(struct request_queue *q)
2194 {
2195 	struct blk_mq_hw_ctx *hctx;
2196 	unsigned int i;
2197 
2198 	blk_mq_sched_teardown(q);
2199 
2200 	/* hctx kobj stays in hctx */
2201 	queue_for_each_hw_ctx(q, hctx, i) {
2202 		if (!hctx)
2203 			continue;
2204 		kfree(hctx->ctxs);
2205 		kfree(hctx);
2206 	}
2207 
2208 	q->mq_map = NULL;
2209 
2210 	kfree(q->queue_hw_ctx);
2211 
2212 	/* ctx kobj stays in queue_ctx */
2213 	free_percpu(q->queue_ctx);
2214 }
2215 
2216 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
2217 {
2218 	struct request_queue *uninit_q, *q;
2219 
2220 	uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
2221 	if (!uninit_q)
2222 		return ERR_PTR(-ENOMEM);
2223 
2224 	q = blk_mq_init_allocated_queue(set, uninit_q);
2225 	if (IS_ERR(q))
2226 		blk_cleanup_queue(uninit_q);
2227 
2228 	return q;
2229 }
2230 EXPORT_SYMBOL(blk_mq_init_queue);
2231 
2232 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
2233 						struct request_queue *q)
2234 {
2235 	int i, j;
2236 	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
2237 
2238 	blk_mq_sysfs_unregister(q);
2239 	for (i = 0; i < set->nr_hw_queues; i++) {
2240 		int node;
2241 
2242 		if (hctxs[i])
2243 			continue;
2244 
2245 		node = blk_mq_hw_queue_to_node(q->mq_map, i);
2246 		hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
2247 					GFP_KERNEL, node);
2248 		if (!hctxs[i])
2249 			break;
2250 
2251 		if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
2252 						node)) {
2253 			kfree(hctxs[i]);
2254 			hctxs[i] = NULL;
2255 			break;
2256 		}
2257 
2258 		atomic_set(&hctxs[i]->nr_active, 0);
2259 		hctxs[i]->numa_node = node;
2260 		hctxs[i]->queue_num = i;
2261 
2262 		if (blk_mq_init_hctx(q, set, hctxs[i], i)) {
2263 			free_cpumask_var(hctxs[i]->cpumask);
2264 			kfree(hctxs[i]);
2265 			hctxs[i] = NULL;
2266 			break;
2267 		}
2268 		blk_mq_hctx_kobj_init(hctxs[i]);
2269 	}
2270 	for (j = i; j < q->nr_hw_queues; j++) {
2271 		struct blk_mq_hw_ctx *hctx = hctxs[j];
2272 
2273 		if (hctx) {
2274 			if (hctx->tags)
2275 				blk_mq_free_map_and_requests(set, j);
2276 			blk_mq_exit_hctx(q, set, hctx, j);
2277 			free_cpumask_var(hctx->cpumask);
2278 			kobject_put(&hctx->kobj);
2279 			kfree(hctx->ctxs);
2280 			kfree(hctx);
2281 			hctxs[j] = NULL;
2282 
2283 		}
2284 	}
2285 	q->nr_hw_queues = i;
2286 	blk_mq_sysfs_register(q);
2287 }
2288 
2289 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
2290 						  struct request_queue *q)
2291 {
2292 	/* mark the queue as mq asap */
2293 	q->mq_ops = set->ops;
2294 
2295 	q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
2296 	if (!q->queue_ctx)
2297 		goto err_exit;
2298 
2299 	q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
2300 						GFP_KERNEL, set->numa_node);
2301 	if (!q->queue_hw_ctx)
2302 		goto err_percpu;
2303 
2304 	q->mq_map = set->mq_map;
2305 
2306 	blk_mq_realloc_hw_ctxs(set, q);
2307 	if (!q->nr_hw_queues)
2308 		goto err_hctxs;
2309 
2310 	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
2311 	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
2312 
2313 	q->nr_queues = nr_cpu_ids;
2314 
2315 	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
2316 
2317 	if (!(set->flags & BLK_MQ_F_SG_MERGE))
2318 		q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
2319 
2320 	q->sg_reserved_size = INT_MAX;
2321 
2322 	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
2323 	INIT_LIST_HEAD(&q->requeue_list);
2324 	spin_lock_init(&q->requeue_lock);
2325 
2326 	if (q->nr_hw_queues > 1)
2327 		blk_queue_make_request(q, blk_mq_make_request);
2328 	else
2329 		blk_queue_make_request(q, blk_sq_make_request);
2330 
2331 	/*
2332 	 * Do this after blk_queue_make_request() overrides it...
2333 	 */
2334 	q->nr_requests = set->queue_depth;
2335 
2336 	/*
2337 	 * Default to classic polling
2338 	 */
2339 	q->poll_nsec = -1;
2340 
2341 	if (set->ops->complete)
2342 		blk_queue_softirq_done(q, set->ops->complete);
2343 
2344 	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
2345 
2346 	get_online_cpus();
2347 	mutex_lock(&all_q_mutex);
2348 
2349 	list_add_tail(&q->all_q_node, &all_q_list);
2350 	blk_mq_add_queue_tag_set(set, q);
2351 	blk_mq_map_swqueue(q, cpu_online_mask);
2352 
2353 	mutex_unlock(&all_q_mutex);
2354 	put_online_cpus();
2355 
2356 	if (!(set->flags & BLK_MQ_F_NO_SCHED)) {
2357 		int ret;
2358 
2359 		ret = blk_mq_sched_init(q);
2360 		if (ret)
2361 			return ERR_PTR(ret);
2362 	}
2363 
2364 	return q;
2365 
2366 err_hctxs:
2367 	kfree(q->queue_hw_ctx);
2368 err_percpu:
2369 	free_percpu(q->queue_ctx);
2370 err_exit:
2371 	q->mq_ops = NULL;
2372 	return ERR_PTR(-ENOMEM);
2373 }
2374 EXPORT_SYMBOL(blk_mq_init_allocated_queue);
2375 
2376 void blk_mq_free_queue(struct request_queue *q)
2377 {
2378 	struct blk_mq_tag_set	*set = q->tag_set;
2379 
2380 	mutex_lock(&all_q_mutex);
2381 	list_del_init(&q->all_q_node);
2382 	mutex_unlock(&all_q_mutex);
2383 
2384 	wbt_exit(q);
2385 
2386 	blk_mq_del_queue_tag_set(q);
2387 
2388 	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
2389 	blk_mq_free_hw_queues(q, set);
2390 }
2391 
2392 /* Basically redo blk_mq_init_queue with queue frozen */
2393 static void blk_mq_queue_reinit(struct request_queue *q,
2394 				const struct cpumask *online_mask)
2395 {
2396 	WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
2397 
2398 	blk_mq_sysfs_unregister(q);
2399 
2400 	/*
2401 	 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
2402 	 * we should change hctx numa_node according to new topology (this
2403 	 * involves free and re-allocate memory, worthy doing?)
2404 	 */
2405 
2406 	blk_mq_map_swqueue(q, online_mask);
2407 
2408 	blk_mq_sysfs_register(q);
2409 }
2410 
2411 /*
2412  * New online cpumask which is going to be set in this hotplug event.
2413  * Declare this cpumasks as global as cpu-hotplug operation is invoked
2414  * one-by-one and dynamically allocating this could result in a failure.
2415  */
2416 static struct cpumask cpuhp_online_new;
2417 
2418 static void blk_mq_queue_reinit_work(void)
2419 {
2420 	struct request_queue *q;
2421 
2422 	mutex_lock(&all_q_mutex);
2423 	/*
2424 	 * We need to freeze and reinit all existing queues.  Freezing
2425 	 * involves synchronous wait for an RCU grace period and doing it
2426 	 * one by one may take a long time.  Start freezing all queues in
2427 	 * one swoop and then wait for the completions so that freezing can
2428 	 * take place in parallel.
2429 	 */
2430 	list_for_each_entry(q, &all_q_list, all_q_node)
2431 		blk_mq_freeze_queue_start(q);
2432 	list_for_each_entry(q, &all_q_list, all_q_node)
2433 		blk_mq_freeze_queue_wait(q);
2434 
2435 	list_for_each_entry(q, &all_q_list, all_q_node)
2436 		blk_mq_queue_reinit(q, &cpuhp_online_new);
2437 
2438 	list_for_each_entry(q, &all_q_list, all_q_node)
2439 		blk_mq_unfreeze_queue(q);
2440 
2441 	mutex_unlock(&all_q_mutex);
2442 }
2443 
2444 static int blk_mq_queue_reinit_dead(unsigned int cpu)
2445 {
2446 	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2447 	blk_mq_queue_reinit_work();
2448 	return 0;
2449 }
2450 
2451 /*
2452  * Before hotadded cpu starts handling requests, new mappings must be
2453  * established.  Otherwise, these requests in hw queue might never be
2454  * dispatched.
2455  *
2456  * For example, there is a single hw queue (hctx) and two CPU queues (ctx0
2457  * for CPU0, and ctx1 for CPU1).
2458  *
2459  * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list
2460  * and set bit0 in pending bitmap as ctx1->index_hw is still zero.
2461  *
2462  * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set
2463  * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list.
2464  * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is
2465  * ignored.
2466  */
2467 static int blk_mq_queue_reinit_prepare(unsigned int cpu)
2468 {
2469 	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2470 	cpumask_set_cpu(cpu, &cpuhp_online_new);
2471 	blk_mq_queue_reinit_work();
2472 	return 0;
2473 }
2474 
2475 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2476 {
2477 	int i;
2478 
2479 	for (i = 0; i < set->nr_hw_queues; i++)
2480 		if (!__blk_mq_alloc_rq_map(set, i))
2481 			goto out_unwind;
2482 
2483 	return 0;
2484 
2485 out_unwind:
2486 	while (--i >= 0)
2487 		blk_mq_free_rq_map(set->tags[i]);
2488 
2489 	return -ENOMEM;
2490 }
2491 
2492 /*
2493  * Allocate the request maps associated with this tag_set. Note that this
2494  * may reduce the depth asked for, if memory is tight. set->queue_depth
2495  * will be updated to reflect the allocated depth.
2496  */
2497 static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2498 {
2499 	unsigned int depth;
2500 	int err;
2501 
2502 	depth = set->queue_depth;
2503 	do {
2504 		err = __blk_mq_alloc_rq_maps(set);
2505 		if (!err)
2506 			break;
2507 
2508 		set->queue_depth >>= 1;
2509 		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
2510 			err = -ENOMEM;
2511 			break;
2512 		}
2513 	} while (set->queue_depth);
2514 
2515 	if (!set->queue_depth || err) {
2516 		pr_err("blk-mq: failed to allocate request map\n");
2517 		return -ENOMEM;
2518 	}
2519 
2520 	if (depth != set->queue_depth)
2521 		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
2522 						depth, set->queue_depth);
2523 
2524 	return 0;
2525 }
2526 
2527 /*
2528  * Alloc a tag set to be associated with one or more request queues.
2529  * May fail with EINVAL for various error conditions. May adjust the
2530  * requested depth down, if if it too large. In that case, the set
2531  * value will be stored in set->queue_depth.
2532  */
2533 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
2534 {
2535 	int ret;
2536 
2537 	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
2538 
2539 	if (!set->nr_hw_queues)
2540 		return -EINVAL;
2541 	if (!set->queue_depth)
2542 		return -EINVAL;
2543 	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
2544 		return -EINVAL;
2545 
2546 	if (!set->ops->queue_rq)
2547 		return -EINVAL;
2548 
2549 	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
2550 		pr_info("blk-mq: reduced tag depth to %u\n",
2551 			BLK_MQ_MAX_DEPTH);
2552 		set->queue_depth = BLK_MQ_MAX_DEPTH;
2553 	}
2554 
2555 	/*
2556 	 * If a crashdump is active, then we are potentially in a very
2557 	 * memory constrained environment. Limit us to 1 queue and
2558 	 * 64 tags to prevent using too much memory.
2559 	 */
2560 	if (is_kdump_kernel()) {
2561 		set->nr_hw_queues = 1;
2562 		set->queue_depth = min(64U, set->queue_depth);
2563 	}
2564 	/*
2565 	 * There is no use for more h/w queues than cpus.
2566 	 */
2567 	if (set->nr_hw_queues > nr_cpu_ids)
2568 		set->nr_hw_queues = nr_cpu_ids;
2569 
2570 	set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *),
2571 				 GFP_KERNEL, set->numa_node);
2572 	if (!set->tags)
2573 		return -ENOMEM;
2574 
2575 	ret = -ENOMEM;
2576 	set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids,
2577 			GFP_KERNEL, set->numa_node);
2578 	if (!set->mq_map)
2579 		goto out_free_tags;
2580 
2581 	if (set->ops->map_queues)
2582 		ret = set->ops->map_queues(set);
2583 	else
2584 		ret = blk_mq_map_queues(set);
2585 	if (ret)
2586 		goto out_free_mq_map;
2587 
2588 	ret = blk_mq_alloc_rq_maps(set);
2589 	if (ret)
2590 		goto out_free_mq_map;
2591 
2592 	mutex_init(&set->tag_list_lock);
2593 	INIT_LIST_HEAD(&set->tag_list);
2594 
2595 	return 0;
2596 
2597 out_free_mq_map:
2598 	kfree(set->mq_map);
2599 	set->mq_map = NULL;
2600 out_free_tags:
2601 	kfree(set->tags);
2602 	set->tags = NULL;
2603 	return ret;
2604 }
2605 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
2606 
2607 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
2608 {
2609 	int i;
2610 
2611 	for (i = 0; i < nr_cpu_ids; i++)
2612 		blk_mq_free_map_and_requests(set, i);
2613 
2614 	kfree(set->mq_map);
2615 	set->mq_map = NULL;
2616 
2617 	kfree(set->tags);
2618 	set->tags = NULL;
2619 }
2620 EXPORT_SYMBOL(blk_mq_free_tag_set);
2621 
2622 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2623 {
2624 	struct blk_mq_tag_set *set = q->tag_set;
2625 	struct blk_mq_hw_ctx *hctx;
2626 	int i, ret;
2627 
2628 	if (!set)
2629 		return -EINVAL;
2630 
2631 	blk_mq_freeze_queue(q);
2632 	blk_mq_quiesce_queue(q);
2633 
2634 	ret = 0;
2635 	queue_for_each_hw_ctx(q, hctx, i) {
2636 		if (!hctx->tags)
2637 			continue;
2638 		/*
2639 		 * If we're using an MQ scheduler, just update the scheduler
2640 		 * queue depth. This is similar to what the old code would do.
2641 		 */
2642 		if (!hctx->sched_tags) {
2643 			ret = blk_mq_tag_update_depth(hctx, &hctx->tags,
2644 							min(nr, set->queue_depth),
2645 							false);
2646 		} else {
2647 			ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
2648 							nr, true);
2649 		}
2650 		if (ret)
2651 			break;
2652 	}
2653 
2654 	if (!ret)
2655 		q->nr_requests = nr;
2656 
2657 	blk_mq_unfreeze_queue(q);
2658 	blk_mq_start_stopped_hw_queues(q, true);
2659 
2660 	return ret;
2661 }
2662 
2663 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
2664 {
2665 	struct request_queue *q;
2666 
2667 	if (nr_hw_queues > nr_cpu_ids)
2668 		nr_hw_queues = nr_cpu_ids;
2669 	if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues)
2670 		return;
2671 
2672 	list_for_each_entry(q, &set->tag_list, tag_set_list)
2673 		blk_mq_freeze_queue(q);
2674 
2675 	set->nr_hw_queues = nr_hw_queues;
2676 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
2677 		blk_mq_realloc_hw_ctxs(set, q);
2678 
2679 		/*
2680 		 * Manually set the make_request_fn as blk_queue_make_request
2681 		 * resets a lot of the queue settings.
2682 		 */
2683 		if (q->nr_hw_queues > 1)
2684 			q->make_request_fn = blk_mq_make_request;
2685 		else
2686 			q->make_request_fn = blk_sq_make_request;
2687 
2688 		blk_mq_queue_reinit(q, cpu_online_mask);
2689 	}
2690 
2691 	list_for_each_entry(q, &set->tag_list, tag_set_list)
2692 		blk_mq_unfreeze_queue(q);
2693 }
2694 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
2695 
2696 static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
2697 				       struct blk_mq_hw_ctx *hctx,
2698 				       struct request *rq)
2699 {
2700 	struct blk_rq_stat stat[2];
2701 	unsigned long ret = 0;
2702 
2703 	/*
2704 	 * If stats collection isn't on, don't sleep but turn it on for
2705 	 * future users
2706 	 */
2707 	if (!blk_stat_enable(q))
2708 		return 0;
2709 
2710 	/*
2711 	 * We don't have to do this once per IO, should optimize this
2712 	 * to just use the current window of stats until it changes
2713 	 */
2714 	memset(&stat, 0, sizeof(stat));
2715 	blk_hctx_stat_get(hctx, stat);
2716 
2717 	/*
2718 	 * As an optimistic guess, use half of the mean service time
2719 	 * for this type of request. We can (and should) make this smarter.
2720 	 * For instance, if the completion latencies are tight, we can
2721 	 * get closer than just half the mean. This is especially
2722 	 * important on devices where the completion latencies are longer
2723 	 * than ~10 usec.
2724 	 */
2725 	if (req_op(rq) == REQ_OP_READ && stat[BLK_STAT_READ].nr_samples)
2726 		ret = (stat[BLK_STAT_READ].mean + 1) / 2;
2727 	else if (req_op(rq) == REQ_OP_WRITE && stat[BLK_STAT_WRITE].nr_samples)
2728 		ret = (stat[BLK_STAT_WRITE].mean + 1) / 2;
2729 
2730 	return ret;
2731 }
2732 
2733 static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
2734 				     struct blk_mq_hw_ctx *hctx,
2735 				     struct request *rq)
2736 {
2737 	struct hrtimer_sleeper hs;
2738 	enum hrtimer_mode mode;
2739 	unsigned int nsecs;
2740 	ktime_t kt;
2741 
2742 	if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags))
2743 		return false;
2744 
2745 	/*
2746 	 * poll_nsec can be:
2747 	 *
2748 	 * -1:	don't ever hybrid sleep
2749 	 *  0:	use half of prev avg
2750 	 * >0:	use this specific value
2751 	 */
2752 	if (q->poll_nsec == -1)
2753 		return false;
2754 	else if (q->poll_nsec > 0)
2755 		nsecs = q->poll_nsec;
2756 	else
2757 		nsecs = blk_mq_poll_nsecs(q, hctx, rq);
2758 
2759 	if (!nsecs)
2760 		return false;
2761 
2762 	set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
2763 
2764 	/*
2765 	 * This will be replaced with the stats tracking code, using
2766 	 * 'avg_completion_time / 2' as the pre-sleep target.
2767 	 */
2768 	kt = nsecs;
2769 
2770 	mode = HRTIMER_MODE_REL;
2771 	hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
2772 	hrtimer_set_expires(&hs.timer, kt);
2773 
2774 	hrtimer_init_sleeper(&hs, current);
2775 	do {
2776 		if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
2777 			break;
2778 		set_current_state(TASK_UNINTERRUPTIBLE);
2779 		hrtimer_start_expires(&hs.timer, mode);
2780 		if (hs.task)
2781 			io_schedule();
2782 		hrtimer_cancel(&hs.timer);
2783 		mode = HRTIMER_MODE_ABS;
2784 	} while (hs.task && !signal_pending(current));
2785 
2786 	__set_current_state(TASK_RUNNING);
2787 	destroy_hrtimer_on_stack(&hs.timer);
2788 	return true;
2789 }
2790 
2791 static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
2792 {
2793 	struct request_queue *q = hctx->queue;
2794 	long state;
2795 
2796 	/*
2797 	 * If we sleep, have the caller restart the poll loop to reset
2798 	 * the state. Like for the other success return cases, the
2799 	 * caller is responsible for checking if the IO completed. If
2800 	 * the IO isn't complete, we'll get called again and will go
2801 	 * straight to the busy poll loop.
2802 	 */
2803 	if (blk_mq_poll_hybrid_sleep(q, hctx, rq))
2804 		return true;
2805 
2806 	hctx->poll_considered++;
2807 
2808 	state = current->state;
2809 	while (!need_resched()) {
2810 		int ret;
2811 
2812 		hctx->poll_invoked++;
2813 
2814 		ret = q->mq_ops->poll(hctx, rq->tag);
2815 		if (ret > 0) {
2816 			hctx->poll_success++;
2817 			set_current_state(TASK_RUNNING);
2818 			return true;
2819 		}
2820 
2821 		if (signal_pending_state(state, current))
2822 			set_current_state(TASK_RUNNING);
2823 
2824 		if (current->state == TASK_RUNNING)
2825 			return true;
2826 		if (ret < 0)
2827 			break;
2828 		cpu_relax();
2829 	}
2830 
2831 	return false;
2832 }
2833 
2834 bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
2835 {
2836 	struct blk_mq_hw_ctx *hctx;
2837 	struct blk_plug *plug;
2838 	struct request *rq;
2839 
2840 	if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
2841 	    !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
2842 		return false;
2843 
2844 	plug = current->plug;
2845 	if (plug)
2846 		blk_flush_plug_list(plug, false);
2847 
2848 	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
2849 	if (!blk_qc_t_is_internal(cookie))
2850 		rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
2851 	else
2852 		rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
2853 
2854 	return __blk_mq_poll(hctx, rq);
2855 }
2856 EXPORT_SYMBOL_GPL(blk_mq_poll);
2857 
2858 void blk_mq_disable_hotplug(void)
2859 {
2860 	mutex_lock(&all_q_mutex);
2861 }
2862 
2863 void blk_mq_enable_hotplug(void)
2864 {
2865 	mutex_unlock(&all_q_mutex);
2866 }
2867 
2868 static int __init blk_mq_init(void)
2869 {
2870 	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
2871 				blk_mq_hctx_notify_dead);
2872 
2873 	cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare",
2874 				  blk_mq_queue_reinit_prepare,
2875 				  blk_mq_queue_reinit_dead);
2876 	return 0;
2877 }
2878 subsys_initcall(blk_mq_init);
2879