xref: /openbmc/linux/block/blk-mq.c (revision 232b0b08)
1 /*
2  * Block multiqueue core code
3  *
4  * Copyright (C) 2013-2014 Jens Axboe
5  * Copyright (C) 2013-2014 Christoph Hellwig
6  */
7 #include <linux/kernel.h>
8 #include <linux/module.h>
9 #include <linux/backing-dev.h>
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/kmemleak.h>
13 #include <linux/mm.h>
14 #include <linux/init.h>
15 #include <linux/slab.h>
16 #include <linux/workqueue.h>
17 #include <linux/smp.h>
18 #include <linux/llist.h>
19 #include <linux/list_sort.h>
20 #include <linux/cpu.h>
21 #include <linux/cache.h>
22 #include <linux/sched/sysctl.h>
23 #include <linux/sched/topology.h>
24 #include <linux/sched/signal.h>
25 #include <linux/delay.h>
26 #include <linux/crash_dump.h>
27 #include <linux/prefetch.h>
28 
29 #include <trace/events/block.h>
30 
31 #include <linux/blk-mq.h>
32 #include "blk.h"
33 #include "blk-mq.h"
34 #include "blk-mq-tag.h"
35 #include "blk-stat.h"
36 #include "blk-wbt.h"
37 #include "blk-mq-sched.h"
38 
39 static DEFINE_MUTEX(all_q_mutex);
40 static LIST_HEAD(all_q_list);
41 
42 /*
43  * Check if any of the ctx's have pending work in this hardware queue
44  */
45 bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
46 {
47 	return sbitmap_any_bit_set(&hctx->ctx_map) ||
48 			!list_empty_careful(&hctx->dispatch) ||
49 			blk_mq_sched_has_work(hctx);
50 }
51 
52 /*
53  * Mark this ctx as having pending work in this hardware queue
54  */
55 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
56 				     struct blk_mq_ctx *ctx)
57 {
58 	if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw))
59 		sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw);
60 }
61 
62 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
63 				      struct blk_mq_ctx *ctx)
64 {
65 	sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw);
66 }
67 
68 void blk_mq_freeze_queue_start(struct request_queue *q)
69 {
70 	int freeze_depth;
71 
72 	freeze_depth = atomic_inc_return(&q->mq_freeze_depth);
73 	if (freeze_depth == 1) {
74 		percpu_ref_kill(&q->q_usage_counter);
75 		blk_mq_run_hw_queues(q, false);
76 	}
77 }
78 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_start);
79 
80 void blk_mq_freeze_queue_wait(struct request_queue *q)
81 {
82 	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
83 }
84 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
85 
86 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
87 				     unsigned long timeout)
88 {
89 	return wait_event_timeout(q->mq_freeze_wq,
90 					percpu_ref_is_zero(&q->q_usage_counter),
91 					timeout);
92 }
93 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
94 
95 /*
96  * Guarantee no request is in use, so we can change any data structure of
97  * the queue afterward.
98  */
99 void blk_freeze_queue(struct request_queue *q)
100 {
101 	/*
102 	 * In the !blk_mq case we are only calling this to kill the
103 	 * q_usage_counter, otherwise this increases the freeze depth
104 	 * and waits for it to return to zero.  For this reason there is
105 	 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
106 	 * exported to drivers as the only user for unfreeze is blk_mq.
107 	 */
108 	blk_mq_freeze_queue_start(q);
109 	blk_mq_freeze_queue_wait(q);
110 }
111 
112 void blk_mq_freeze_queue(struct request_queue *q)
113 {
114 	/*
115 	 * ...just an alias to keep freeze and unfreeze actions balanced
116 	 * in the blk_mq_* namespace
117 	 */
118 	blk_freeze_queue(q);
119 }
120 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
121 
122 void blk_mq_unfreeze_queue(struct request_queue *q)
123 {
124 	int freeze_depth;
125 
126 	freeze_depth = atomic_dec_return(&q->mq_freeze_depth);
127 	WARN_ON_ONCE(freeze_depth < 0);
128 	if (!freeze_depth) {
129 		percpu_ref_reinit(&q->q_usage_counter);
130 		wake_up_all(&q->mq_freeze_wq);
131 	}
132 }
133 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
134 
135 /**
136  * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished
137  * @q: request queue.
138  *
139  * Note: this function does not prevent that the struct request end_io()
140  * callback function is invoked. Additionally, it is not prevented that
141  * new queue_rq() calls occur unless the queue has been stopped first.
142  */
143 void blk_mq_quiesce_queue(struct request_queue *q)
144 {
145 	struct blk_mq_hw_ctx *hctx;
146 	unsigned int i;
147 	bool rcu = false;
148 
149 	blk_mq_stop_hw_queues(q);
150 
151 	queue_for_each_hw_ctx(q, hctx, i) {
152 		if (hctx->flags & BLK_MQ_F_BLOCKING)
153 			synchronize_srcu(&hctx->queue_rq_srcu);
154 		else
155 			rcu = true;
156 	}
157 	if (rcu)
158 		synchronize_rcu();
159 }
160 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
161 
162 void blk_mq_wake_waiters(struct request_queue *q)
163 {
164 	struct blk_mq_hw_ctx *hctx;
165 	unsigned int i;
166 
167 	queue_for_each_hw_ctx(q, hctx, i)
168 		if (blk_mq_hw_queue_mapped(hctx))
169 			blk_mq_tag_wakeup_all(hctx->tags, true);
170 
171 	/*
172 	 * If we are called because the queue has now been marked as
173 	 * dying, we need to ensure that processes currently waiting on
174 	 * the queue are notified as well.
175 	 */
176 	wake_up_all(&q->mq_freeze_wq);
177 }
178 
179 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
180 {
181 	return blk_mq_has_free_tags(hctx->tags);
182 }
183 EXPORT_SYMBOL(blk_mq_can_queue);
184 
185 void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
186 			struct request *rq, unsigned int op)
187 {
188 	INIT_LIST_HEAD(&rq->queuelist);
189 	/* csd/requeue_work/fifo_time is initialized before use */
190 	rq->q = q;
191 	rq->mq_ctx = ctx;
192 	rq->cmd_flags = op;
193 	if (blk_queue_io_stat(q))
194 		rq->rq_flags |= RQF_IO_STAT;
195 	/* do not touch atomic flags, it needs atomic ops against the timer */
196 	rq->cpu = -1;
197 	INIT_HLIST_NODE(&rq->hash);
198 	RB_CLEAR_NODE(&rq->rb_node);
199 	rq->rq_disk = NULL;
200 	rq->part = NULL;
201 	rq->start_time = jiffies;
202 #ifdef CONFIG_BLK_CGROUP
203 	rq->rl = NULL;
204 	set_start_time_ns(rq);
205 	rq->io_start_time_ns = 0;
206 #endif
207 	rq->nr_phys_segments = 0;
208 #if defined(CONFIG_BLK_DEV_INTEGRITY)
209 	rq->nr_integrity_segments = 0;
210 #endif
211 	rq->special = NULL;
212 	/* tag was already set */
213 	rq->errors = 0;
214 	rq->extra_len = 0;
215 
216 	INIT_LIST_HEAD(&rq->timeout_list);
217 	rq->timeout = 0;
218 
219 	rq->end_io = NULL;
220 	rq->end_io_data = NULL;
221 	rq->next_rq = NULL;
222 
223 	ctx->rq_dispatched[op_is_sync(op)]++;
224 }
225 EXPORT_SYMBOL_GPL(blk_mq_rq_ctx_init);
226 
227 struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
228 				       unsigned int op)
229 {
230 	struct request *rq;
231 	unsigned int tag;
232 
233 	tag = blk_mq_get_tag(data);
234 	if (tag != BLK_MQ_TAG_FAIL) {
235 		struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
236 
237 		rq = tags->static_rqs[tag];
238 
239 		if (data->flags & BLK_MQ_REQ_INTERNAL) {
240 			rq->tag = -1;
241 			rq->internal_tag = tag;
242 		} else {
243 			if (blk_mq_tag_busy(data->hctx)) {
244 				rq->rq_flags = RQF_MQ_INFLIGHT;
245 				atomic_inc(&data->hctx->nr_active);
246 			}
247 			rq->tag = tag;
248 			rq->internal_tag = -1;
249 			data->hctx->tags->rqs[rq->tag] = rq;
250 		}
251 
252 		blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
253 		return rq;
254 	}
255 
256 	return NULL;
257 }
258 EXPORT_SYMBOL_GPL(__blk_mq_alloc_request);
259 
260 struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
261 		unsigned int flags)
262 {
263 	struct blk_mq_alloc_data alloc_data = { .flags = flags };
264 	struct request *rq;
265 	int ret;
266 
267 	ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT);
268 	if (ret)
269 		return ERR_PTR(ret);
270 
271 	rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
272 
273 	blk_mq_put_ctx(alloc_data.ctx);
274 	blk_queue_exit(q);
275 
276 	if (!rq)
277 		return ERR_PTR(-EWOULDBLOCK);
278 
279 	rq->__data_len = 0;
280 	rq->__sector = (sector_t) -1;
281 	rq->bio = rq->biotail = NULL;
282 	return rq;
283 }
284 EXPORT_SYMBOL(blk_mq_alloc_request);
285 
286 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
287 		unsigned int flags, unsigned int hctx_idx)
288 {
289 	struct blk_mq_alloc_data alloc_data = { .flags = flags };
290 	struct request *rq;
291 	unsigned int cpu;
292 	int ret;
293 
294 	/*
295 	 * If the tag allocator sleeps we could get an allocation for a
296 	 * different hardware context.  No need to complicate the low level
297 	 * allocator for this for the rare use case of a command tied to
298 	 * a specific queue.
299 	 */
300 	if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)))
301 		return ERR_PTR(-EINVAL);
302 
303 	if (hctx_idx >= q->nr_hw_queues)
304 		return ERR_PTR(-EIO);
305 
306 	ret = blk_queue_enter(q, true);
307 	if (ret)
308 		return ERR_PTR(ret);
309 
310 	/*
311 	 * Check if the hardware context is actually mapped to anything.
312 	 * If not tell the caller that it should skip this queue.
313 	 */
314 	alloc_data.hctx = q->queue_hw_ctx[hctx_idx];
315 	if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) {
316 		blk_queue_exit(q);
317 		return ERR_PTR(-EXDEV);
318 	}
319 	cpu = cpumask_first(alloc_data.hctx->cpumask);
320 	alloc_data.ctx = __blk_mq_get_ctx(q, cpu);
321 
322 	rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
323 
324 	blk_mq_put_ctx(alloc_data.ctx);
325 	blk_queue_exit(q);
326 
327 	if (!rq)
328 		return ERR_PTR(-EWOULDBLOCK);
329 
330 	return rq;
331 }
332 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
333 
334 void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
335 			     struct request *rq)
336 {
337 	const int sched_tag = rq->internal_tag;
338 	struct request_queue *q = rq->q;
339 
340 	if (rq->rq_flags & RQF_MQ_INFLIGHT)
341 		atomic_dec(&hctx->nr_active);
342 
343 	wbt_done(q->rq_wb, &rq->issue_stat);
344 	rq->rq_flags = 0;
345 
346 	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
347 	clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
348 	if (rq->tag != -1)
349 		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
350 	if (sched_tag != -1)
351 		blk_mq_sched_completed_request(hctx, rq);
352 	blk_mq_sched_restart_queues(hctx);
353 	blk_queue_exit(q);
354 }
355 
356 static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx,
357 				     struct request *rq)
358 {
359 	struct blk_mq_ctx *ctx = rq->mq_ctx;
360 
361 	ctx->rq_completed[rq_is_sync(rq)]++;
362 	__blk_mq_finish_request(hctx, ctx, rq);
363 }
364 
365 void blk_mq_finish_request(struct request *rq)
366 {
367 	blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq);
368 }
369 
370 void blk_mq_free_request(struct request *rq)
371 {
372 	blk_mq_sched_put_request(rq);
373 }
374 EXPORT_SYMBOL_GPL(blk_mq_free_request);
375 
376 inline void __blk_mq_end_request(struct request *rq, int error)
377 {
378 	blk_account_io_done(rq);
379 
380 	if (rq->end_io) {
381 		wbt_done(rq->q->rq_wb, &rq->issue_stat);
382 		rq->end_io(rq, error);
383 	} else {
384 		if (unlikely(blk_bidi_rq(rq)))
385 			blk_mq_free_request(rq->next_rq);
386 		blk_mq_free_request(rq);
387 	}
388 }
389 EXPORT_SYMBOL(__blk_mq_end_request);
390 
391 void blk_mq_end_request(struct request *rq, int error)
392 {
393 	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
394 		BUG();
395 	__blk_mq_end_request(rq, error);
396 }
397 EXPORT_SYMBOL(blk_mq_end_request);
398 
399 static void __blk_mq_complete_request_remote(void *data)
400 {
401 	struct request *rq = data;
402 
403 	rq->q->softirq_done_fn(rq);
404 }
405 
406 static void blk_mq_ipi_complete_request(struct request *rq)
407 {
408 	struct blk_mq_ctx *ctx = rq->mq_ctx;
409 	bool shared = false;
410 	int cpu;
411 
412 	if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
413 		rq->q->softirq_done_fn(rq);
414 		return;
415 	}
416 
417 	cpu = get_cpu();
418 	if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
419 		shared = cpus_share_cache(cpu, ctx->cpu);
420 
421 	if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
422 		rq->csd.func = __blk_mq_complete_request_remote;
423 		rq->csd.info = rq;
424 		rq->csd.flags = 0;
425 		smp_call_function_single_async(ctx->cpu, &rq->csd);
426 	} else {
427 		rq->q->softirq_done_fn(rq);
428 	}
429 	put_cpu();
430 }
431 
432 static void blk_mq_stat_add(struct request *rq)
433 {
434 	if (rq->rq_flags & RQF_STATS) {
435 		/*
436 		 * We could rq->mq_ctx here, but there's less of a risk
437 		 * of races if we have the completion event add the stats
438 		 * to the local software queue.
439 		 */
440 		struct blk_mq_ctx *ctx;
441 
442 		ctx = __blk_mq_get_ctx(rq->q, raw_smp_processor_id());
443 		blk_stat_add(&ctx->stat[rq_data_dir(rq)], rq);
444 	}
445 }
446 
447 static void __blk_mq_complete_request(struct request *rq)
448 {
449 	struct request_queue *q = rq->q;
450 
451 	blk_mq_stat_add(rq);
452 
453 	if (!q->softirq_done_fn)
454 		blk_mq_end_request(rq, rq->errors);
455 	else
456 		blk_mq_ipi_complete_request(rq);
457 }
458 
459 /**
460  * blk_mq_complete_request - end I/O on a request
461  * @rq:		the request being processed
462  *
463  * Description:
464  *	Ends all I/O on a request. It does not handle partial completions.
465  *	The actual completion happens out-of-order, through a IPI handler.
466  **/
467 void blk_mq_complete_request(struct request *rq, int error)
468 {
469 	struct request_queue *q = rq->q;
470 
471 	if (unlikely(blk_should_fake_timeout(q)))
472 		return;
473 	if (!blk_mark_rq_complete(rq)) {
474 		rq->errors = error;
475 		__blk_mq_complete_request(rq);
476 	}
477 }
478 EXPORT_SYMBOL(blk_mq_complete_request);
479 
480 int blk_mq_request_started(struct request *rq)
481 {
482 	return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
483 }
484 EXPORT_SYMBOL_GPL(blk_mq_request_started);
485 
486 void blk_mq_start_request(struct request *rq)
487 {
488 	struct request_queue *q = rq->q;
489 
490 	blk_mq_sched_started_request(rq);
491 
492 	trace_block_rq_issue(q, rq);
493 
494 	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
495 		blk_stat_set_issue_time(&rq->issue_stat);
496 		rq->rq_flags |= RQF_STATS;
497 		wbt_issue(q->rq_wb, &rq->issue_stat);
498 	}
499 
500 	blk_add_timer(rq);
501 
502 	/*
503 	 * Ensure that ->deadline is visible before set the started
504 	 * flag and clear the completed flag.
505 	 */
506 	smp_mb__before_atomic();
507 
508 	/*
509 	 * Mark us as started and clear complete. Complete might have been
510 	 * set if requeue raced with timeout, which then marked it as
511 	 * complete. So be sure to clear complete again when we start
512 	 * the request, otherwise we'll ignore the completion event.
513 	 */
514 	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
515 		set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
516 	if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
517 		clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
518 
519 	if (q->dma_drain_size && blk_rq_bytes(rq)) {
520 		/*
521 		 * Make sure space for the drain appears.  We know we can do
522 		 * this because max_hw_segments has been adjusted to be one
523 		 * fewer than the device can handle.
524 		 */
525 		rq->nr_phys_segments++;
526 	}
527 }
528 EXPORT_SYMBOL(blk_mq_start_request);
529 
530 static void __blk_mq_requeue_request(struct request *rq)
531 {
532 	struct request_queue *q = rq->q;
533 
534 	trace_block_rq_requeue(q, rq);
535 	wbt_requeue(q->rq_wb, &rq->issue_stat);
536 	blk_mq_sched_requeue_request(rq);
537 
538 	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
539 		if (q->dma_drain_size && blk_rq_bytes(rq))
540 			rq->nr_phys_segments--;
541 	}
542 }
543 
544 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
545 {
546 	__blk_mq_requeue_request(rq);
547 
548 	BUG_ON(blk_queued_rq(rq));
549 	blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
550 }
551 EXPORT_SYMBOL(blk_mq_requeue_request);
552 
553 static void blk_mq_requeue_work(struct work_struct *work)
554 {
555 	struct request_queue *q =
556 		container_of(work, struct request_queue, requeue_work.work);
557 	LIST_HEAD(rq_list);
558 	struct request *rq, *next;
559 	unsigned long flags;
560 
561 	spin_lock_irqsave(&q->requeue_lock, flags);
562 	list_splice_init(&q->requeue_list, &rq_list);
563 	spin_unlock_irqrestore(&q->requeue_lock, flags);
564 
565 	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
566 		if (!(rq->rq_flags & RQF_SOFTBARRIER))
567 			continue;
568 
569 		rq->rq_flags &= ~RQF_SOFTBARRIER;
570 		list_del_init(&rq->queuelist);
571 		blk_mq_sched_insert_request(rq, true, false, false, true);
572 	}
573 
574 	while (!list_empty(&rq_list)) {
575 		rq = list_entry(rq_list.next, struct request, queuelist);
576 		list_del_init(&rq->queuelist);
577 		blk_mq_sched_insert_request(rq, false, false, false, true);
578 	}
579 
580 	blk_mq_run_hw_queues(q, false);
581 }
582 
583 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
584 				bool kick_requeue_list)
585 {
586 	struct request_queue *q = rq->q;
587 	unsigned long flags;
588 
589 	/*
590 	 * We abuse this flag that is otherwise used by the I/O scheduler to
591 	 * request head insertation from the workqueue.
592 	 */
593 	BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
594 
595 	spin_lock_irqsave(&q->requeue_lock, flags);
596 	if (at_head) {
597 		rq->rq_flags |= RQF_SOFTBARRIER;
598 		list_add(&rq->queuelist, &q->requeue_list);
599 	} else {
600 		list_add_tail(&rq->queuelist, &q->requeue_list);
601 	}
602 	spin_unlock_irqrestore(&q->requeue_lock, flags);
603 
604 	if (kick_requeue_list)
605 		blk_mq_kick_requeue_list(q);
606 }
607 EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
608 
609 void blk_mq_kick_requeue_list(struct request_queue *q)
610 {
611 	kblockd_schedule_delayed_work(&q->requeue_work, 0);
612 }
613 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
614 
615 void blk_mq_delay_kick_requeue_list(struct request_queue *q,
616 				    unsigned long msecs)
617 {
618 	kblockd_schedule_delayed_work(&q->requeue_work,
619 				      msecs_to_jiffies(msecs));
620 }
621 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
622 
623 void blk_mq_abort_requeue_list(struct request_queue *q)
624 {
625 	unsigned long flags;
626 	LIST_HEAD(rq_list);
627 
628 	spin_lock_irqsave(&q->requeue_lock, flags);
629 	list_splice_init(&q->requeue_list, &rq_list);
630 	spin_unlock_irqrestore(&q->requeue_lock, flags);
631 
632 	while (!list_empty(&rq_list)) {
633 		struct request *rq;
634 
635 		rq = list_first_entry(&rq_list, struct request, queuelist);
636 		list_del_init(&rq->queuelist);
637 		rq->errors = -EIO;
638 		blk_mq_end_request(rq, rq->errors);
639 	}
640 }
641 EXPORT_SYMBOL(blk_mq_abort_requeue_list);
642 
643 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
644 {
645 	if (tag < tags->nr_tags) {
646 		prefetch(tags->rqs[tag]);
647 		return tags->rqs[tag];
648 	}
649 
650 	return NULL;
651 }
652 EXPORT_SYMBOL(blk_mq_tag_to_rq);
653 
654 struct blk_mq_timeout_data {
655 	unsigned long next;
656 	unsigned int next_set;
657 };
658 
659 void blk_mq_rq_timed_out(struct request *req, bool reserved)
660 {
661 	const struct blk_mq_ops *ops = req->q->mq_ops;
662 	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
663 
664 	/*
665 	 * We know that complete is set at this point. If STARTED isn't set
666 	 * anymore, then the request isn't active and the "timeout" should
667 	 * just be ignored. This can happen due to the bitflag ordering.
668 	 * Timeout first checks if STARTED is set, and if it is, assumes
669 	 * the request is active. But if we race with completion, then
670 	 * we both flags will get cleared. So check here again, and ignore
671 	 * a timeout event with a request that isn't active.
672 	 */
673 	if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
674 		return;
675 
676 	if (ops->timeout)
677 		ret = ops->timeout(req, reserved);
678 
679 	switch (ret) {
680 	case BLK_EH_HANDLED:
681 		__blk_mq_complete_request(req);
682 		break;
683 	case BLK_EH_RESET_TIMER:
684 		blk_add_timer(req);
685 		blk_clear_rq_complete(req);
686 		break;
687 	case BLK_EH_NOT_HANDLED:
688 		break;
689 	default:
690 		printk(KERN_ERR "block: bad eh return: %d\n", ret);
691 		break;
692 	}
693 }
694 
695 static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
696 		struct request *rq, void *priv, bool reserved)
697 {
698 	struct blk_mq_timeout_data *data = priv;
699 
700 	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
701 		return;
702 
703 	if (time_after_eq(jiffies, rq->deadline)) {
704 		if (!blk_mark_rq_complete(rq))
705 			blk_mq_rq_timed_out(rq, reserved);
706 	} else if (!data->next_set || time_after(data->next, rq->deadline)) {
707 		data->next = rq->deadline;
708 		data->next_set = 1;
709 	}
710 }
711 
712 static void blk_mq_timeout_work(struct work_struct *work)
713 {
714 	struct request_queue *q =
715 		container_of(work, struct request_queue, timeout_work);
716 	struct blk_mq_timeout_data data = {
717 		.next		= 0,
718 		.next_set	= 0,
719 	};
720 	int i;
721 
722 	/* A deadlock might occur if a request is stuck requiring a
723 	 * timeout at the same time a queue freeze is waiting
724 	 * completion, since the timeout code would not be able to
725 	 * acquire the queue reference here.
726 	 *
727 	 * That's why we don't use blk_queue_enter here; instead, we use
728 	 * percpu_ref_tryget directly, because we need to be able to
729 	 * obtain a reference even in the short window between the queue
730 	 * starting to freeze, by dropping the first reference in
731 	 * blk_mq_freeze_queue_start, and the moment the last request is
732 	 * consumed, marked by the instant q_usage_counter reaches
733 	 * zero.
734 	 */
735 	if (!percpu_ref_tryget(&q->q_usage_counter))
736 		return;
737 
738 	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
739 
740 	if (data.next_set) {
741 		data.next = blk_rq_timeout(round_jiffies_up(data.next));
742 		mod_timer(&q->timeout, data.next);
743 	} else {
744 		struct blk_mq_hw_ctx *hctx;
745 
746 		queue_for_each_hw_ctx(q, hctx, i) {
747 			/* the hctx may be unmapped, so check it here */
748 			if (blk_mq_hw_queue_mapped(hctx))
749 				blk_mq_tag_idle(hctx);
750 		}
751 	}
752 	blk_queue_exit(q);
753 }
754 
755 /*
756  * Reverse check our software queue for entries that we could potentially
757  * merge with. Currently includes a hand-wavy stop count of 8, to not spend
758  * too much time checking for merges.
759  */
760 static bool blk_mq_attempt_merge(struct request_queue *q,
761 				 struct blk_mq_ctx *ctx, struct bio *bio)
762 {
763 	struct request *rq;
764 	int checked = 8;
765 
766 	list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
767 		bool merged = false;
768 
769 		if (!checked--)
770 			break;
771 
772 		if (!blk_rq_merge_ok(rq, bio))
773 			continue;
774 
775 		switch (blk_try_merge(rq, bio)) {
776 		case ELEVATOR_BACK_MERGE:
777 			if (blk_mq_sched_allow_merge(q, rq, bio))
778 				merged = bio_attempt_back_merge(q, rq, bio);
779 			break;
780 		case ELEVATOR_FRONT_MERGE:
781 			if (blk_mq_sched_allow_merge(q, rq, bio))
782 				merged = bio_attempt_front_merge(q, rq, bio);
783 			break;
784 		case ELEVATOR_DISCARD_MERGE:
785 			merged = bio_attempt_discard_merge(q, rq, bio);
786 			break;
787 		default:
788 			continue;
789 		}
790 
791 		if (merged)
792 			ctx->rq_merged++;
793 		return merged;
794 	}
795 
796 	return false;
797 }
798 
799 struct flush_busy_ctx_data {
800 	struct blk_mq_hw_ctx *hctx;
801 	struct list_head *list;
802 };
803 
804 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
805 {
806 	struct flush_busy_ctx_data *flush_data = data;
807 	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
808 	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
809 
810 	sbitmap_clear_bit(sb, bitnr);
811 	spin_lock(&ctx->lock);
812 	list_splice_tail_init(&ctx->rq_list, flush_data->list);
813 	spin_unlock(&ctx->lock);
814 	return true;
815 }
816 
817 /*
818  * Process software queues that have been marked busy, splicing them
819  * to the for-dispatch
820  */
821 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
822 {
823 	struct flush_busy_ctx_data data = {
824 		.hctx = hctx,
825 		.list = list,
826 	};
827 
828 	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
829 }
830 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
831 
832 static inline unsigned int queued_to_index(unsigned int queued)
833 {
834 	if (!queued)
835 		return 0;
836 
837 	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
838 }
839 
840 bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx,
841 			   bool wait)
842 {
843 	struct blk_mq_alloc_data data = {
844 		.q = rq->q,
845 		.hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu),
846 		.flags = wait ? 0 : BLK_MQ_REQ_NOWAIT,
847 	};
848 
849 	if (rq->tag != -1) {
850 done:
851 		if (hctx)
852 			*hctx = data.hctx;
853 		return true;
854 	}
855 
856 	if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag))
857 		data.flags |= BLK_MQ_REQ_RESERVED;
858 
859 	rq->tag = blk_mq_get_tag(&data);
860 	if (rq->tag >= 0) {
861 		if (blk_mq_tag_busy(data.hctx)) {
862 			rq->rq_flags |= RQF_MQ_INFLIGHT;
863 			atomic_inc(&data.hctx->nr_active);
864 		}
865 		data.hctx->tags->rqs[rq->tag] = rq;
866 		goto done;
867 	}
868 
869 	return false;
870 }
871 
872 static void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
873 				    struct request *rq)
874 {
875 	blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag);
876 	rq->tag = -1;
877 
878 	if (rq->rq_flags & RQF_MQ_INFLIGHT) {
879 		rq->rq_flags &= ~RQF_MQ_INFLIGHT;
880 		atomic_dec(&hctx->nr_active);
881 	}
882 }
883 
884 static void blk_mq_put_driver_tag_hctx(struct blk_mq_hw_ctx *hctx,
885 				       struct request *rq)
886 {
887 	if (rq->tag == -1 || rq->internal_tag == -1)
888 		return;
889 
890 	__blk_mq_put_driver_tag(hctx, rq);
891 }
892 
893 static void blk_mq_put_driver_tag(struct request *rq)
894 {
895 	struct blk_mq_hw_ctx *hctx;
896 
897 	if (rq->tag == -1 || rq->internal_tag == -1)
898 		return;
899 
900 	hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu);
901 	__blk_mq_put_driver_tag(hctx, rq);
902 }
903 
904 /*
905  * If we fail getting a driver tag because all the driver tags are already
906  * assigned and on the dispatch list, BUT the first entry does not have a
907  * tag, then we could deadlock. For that case, move entries with assigned
908  * driver tags to the front, leaving the set of tagged requests in the
909  * same order, and the untagged set in the same order.
910  */
911 static bool reorder_tags_to_front(struct list_head *list)
912 {
913 	struct request *rq, *tmp, *first = NULL;
914 
915 	list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) {
916 		if (rq == first)
917 			break;
918 		if (rq->tag != -1) {
919 			list_move(&rq->queuelist, list);
920 			if (!first)
921 				first = rq;
922 		}
923 	}
924 
925 	return first != NULL;
926 }
927 
928 static int blk_mq_dispatch_wake(wait_queue_t *wait, unsigned mode, int flags,
929 				void *key)
930 {
931 	struct blk_mq_hw_ctx *hctx;
932 
933 	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
934 
935 	list_del(&wait->task_list);
936 	clear_bit_unlock(BLK_MQ_S_TAG_WAITING, &hctx->state);
937 	blk_mq_run_hw_queue(hctx, true);
938 	return 1;
939 }
940 
941 static bool blk_mq_dispatch_wait_add(struct blk_mq_hw_ctx *hctx)
942 {
943 	struct sbq_wait_state *ws;
944 
945 	/*
946 	 * The TAG_WAITING bit serves as a lock protecting hctx->dispatch_wait.
947 	 * The thread which wins the race to grab this bit adds the hardware
948 	 * queue to the wait queue.
949 	 */
950 	if (test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state) ||
951 	    test_and_set_bit_lock(BLK_MQ_S_TAG_WAITING, &hctx->state))
952 		return false;
953 
954 	init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
955 	ws = bt_wait_ptr(&hctx->tags->bitmap_tags, hctx);
956 
957 	/*
958 	 * As soon as this returns, it's no longer safe to fiddle with
959 	 * hctx->dispatch_wait, since a completion can wake up the wait queue
960 	 * and unlock the bit.
961 	 */
962 	add_wait_queue(&ws->wait, &hctx->dispatch_wait);
963 	return true;
964 }
965 
966 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list)
967 {
968 	struct request_queue *q = hctx->queue;
969 	struct request *rq;
970 	LIST_HEAD(driver_list);
971 	struct list_head *dptr;
972 	int queued, ret = BLK_MQ_RQ_QUEUE_OK;
973 
974 	/*
975 	 * Start off with dptr being NULL, so we start the first request
976 	 * immediately, even if we have more pending.
977 	 */
978 	dptr = NULL;
979 
980 	/*
981 	 * Now process all the entries, sending them to the driver.
982 	 */
983 	queued = 0;
984 	while (!list_empty(list)) {
985 		struct blk_mq_queue_data bd;
986 
987 		rq = list_first_entry(list, struct request, queuelist);
988 		if (!blk_mq_get_driver_tag(rq, &hctx, false)) {
989 			if (!queued && reorder_tags_to_front(list))
990 				continue;
991 
992 			/*
993 			 * The initial allocation attempt failed, so we need to
994 			 * rerun the hardware queue when a tag is freed.
995 			 */
996 			if (blk_mq_dispatch_wait_add(hctx)) {
997 				/*
998 				 * It's possible that a tag was freed in the
999 				 * window between the allocation failure and
1000 				 * adding the hardware queue to the wait queue.
1001 				 */
1002 				if (!blk_mq_get_driver_tag(rq, &hctx, false))
1003 					break;
1004 			} else {
1005 				break;
1006 			}
1007 		}
1008 
1009 		list_del_init(&rq->queuelist);
1010 
1011 		bd.rq = rq;
1012 		bd.list = dptr;
1013 
1014 		/*
1015 		 * Flag last if we have no more requests, or if we have more
1016 		 * but can't assign a driver tag to it.
1017 		 */
1018 		if (list_empty(list))
1019 			bd.last = true;
1020 		else {
1021 			struct request *nxt;
1022 
1023 			nxt = list_first_entry(list, struct request, queuelist);
1024 			bd.last = !blk_mq_get_driver_tag(nxt, NULL, false);
1025 		}
1026 
1027 		ret = q->mq_ops->queue_rq(hctx, &bd);
1028 		switch (ret) {
1029 		case BLK_MQ_RQ_QUEUE_OK:
1030 			queued++;
1031 			break;
1032 		case BLK_MQ_RQ_QUEUE_BUSY:
1033 			blk_mq_put_driver_tag_hctx(hctx, rq);
1034 			list_add(&rq->queuelist, list);
1035 			__blk_mq_requeue_request(rq);
1036 			break;
1037 		default:
1038 			pr_err("blk-mq: bad return on queue: %d\n", ret);
1039 		case BLK_MQ_RQ_QUEUE_ERROR:
1040 			rq->errors = -EIO;
1041 			blk_mq_end_request(rq, rq->errors);
1042 			break;
1043 		}
1044 
1045 		if (ret == BLK_MQ_RQ_QUEUE_BUSY)
1046 			break;
1047 
1048 		/*
1049 		 * We've done the first request. If we have more than 1
1050 		 * left in the list, set dptr to defer issue.
1051 		 */
1052 		if (!dptr && list->next != list->prev)
1053 			dptr = &driver_list;
1054 	}
1055 
1056 	hctx->dispatched[queued_to_index(queued)]++;
1057 
1058 	/*
1059 	 * Any items that need requeuing? Stuff them into hctx->dispatch,
1060 	 * that is where we will continue on next queue run.
1061 	 */
1062 	if (!list_empty(list)) {
1063 		/*
1064 		 * If we got a driver tag for the next request already,
1065 		 * free it again.
1066 		 */
1067 		rq = list_first_entry(list, struct request, queuelist);
1068 		blk_mq_put_driver_tag(rq);
1069 
1070 		spin_lock(&hctx->lock);
1071 		list_splice_init(list, &hctx->dispatch);
1072 		spin_unlock(&hctx->lock);
1073 
1074 		/*
1075 		 * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but
1076 		 * it's possible the queue is stopped and restarted again
1077 		 * before this. Queue restart will dispatch requests. And since
1078 		 * requests in rq_list aren't added into hctx->dispatch yet,
1079 		 * the requests in rq_list might get lost.
1080 		 *
1081 		 * blk_mq_run_hw_queue() already checks the STOPPED bit
1082 		 *
1083 		 * If RESTART or TAG_WAITING is set, then let completion restart
1084 		 * the queue instead of potentially looping here.
1085 		 */
1086 		if (!blk_mq_sched_needs_restart(hctx) &&
1087 		    !test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state))
1088 			blk_mq_run_hw_queue(hctx, true);
1089 	}
1090 
1091 	return queued != 0;
1092 }
1093 
1094 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
1095 {
1096 	int srcu_idx;
1097 
1098 	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
1099 		cpu_online(hctx->next_cpu));
1100 
1101 	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
1102 		rcu_read_lock();
1103 		blk_mq_sched_dispatch_requests(hctx);
1104 		rcu_read_unlock();
1105 	} else {
1106 		srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
1107 		blk_mq_sched_dispatch_requests(hctx);
1108 		srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
1109 	}
1110 }
1111 
1112 /*
1113  * It'd be great if the workqueue API had a way to pass
1114  * in a mask and had some smarts for more clever placement.
1115  * For now we just round-robin here, switching for every
1116  * BLK_MQ_CPU_WORK_BATCH queued items.
1117  */
1118 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
1119 {
1120 	if (hctx->queue->nr_hw_queues == 1)
1121 		return WORK_CPU_UNBOUND;
1122 
1123 	if (--hctx->next_cpu_batch <= 0) {
1124 		int next_cpu;
1125 
1126 		next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
1127 		if (next_cpu >= nr_cpu_ids)
1128 			next_cpu = cpumask_first(hctx->cpumask);
1129 
1130 		hctx->next_cpu = next_cpu;
1131 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1132 	}
1133 
1134 	return hctx->next_cpu;
1135 }
1136 
1137 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1138 {
1139 	if (unlikely(blk_mq_hctx_stopped(hctx) ||
1140 		     !blk_mq_hw_queue_mapped(hctx)))
1141 		return;
1142 
1143 	if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
1144 		int cpu = get_cpu();
1145 		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
1146 			__blk_mq_run_hw_queue(hctx);
1147 			put_cpu();
1148 			return;
1149 		}
1150 
1151 		put_cpu();
1152 	}
1153 
1154 	kblockd_schedule_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work);
1155 }
1156 
1157 void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1158 {
1159 	struct blk_mq_hw_ctx *hctx;
1160 	int i;
1161 
1162 	queue_for_each_hw_ctx(q, hctx, i) {
1163 		if (!blk_mq_hctx_has_pending(hctx) ||
1164 		    blk_mq_hctx_stopped(hctx))
1165 			continue;
1166 
1167 		blk_mq_run_hw_queue(hctx, async);
1168 	}
1169 }
1170 EXPORT_SYMBOL(blk_mq_run_hw_queues);
1171 
1172 /**
1173  * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
1174  * @q: request queue.
1175  *
1176  * The caller is responsible for serializing this function against
1177  * blk_mq_{start,stop}_hw_queue().
1178  */
1179 bool blk_mq_queue_stopped(struct request_queue *q)
1180 {
1181 	struct blk_mq_hw_ctx *hctx;
1182 	int i;
1183 
1184 	queue_for_each_hw_ctx(q, hctx, i)
1185 		if (blk_mq_hctx_stopped(hctx))
1186 			return true;
1187 
1188 	return false;
1189 }
1190 EXPORT_SYMBOL(blk_mq_queue_stopped);
1191 
1192 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
1193 {
1194 	cancel_work(&hctx->run_work);
1195 	cancel_delayed_work(&hctx->delay_work);
1196 	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
1197 }
1198 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
1199 
1200 void blk_mq_stop_hw_queues(struct request_queue *q)
1201 {
1202 	struct blk_mq_hw_ctx *hctx;
1203 	int i;
1204 
1205 	queue_for_each_hw_ctx(q, hctx, i)
1206 		blk_mq_stop_hw_queue(hctx);
1207 }
1208 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
1209 
1210 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
1211 {
1212 	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1213 
1214 	blk_mq_run_hw_queue(hctx, false);
1215 }
1216 EXPORT_SYMBOL(blk_mq_start_hw_queue);
1217 
1218 void blk_mq_start_hw_queues(struct request_queue *q)
1219 {
1220 	struct blk_mq_hw_ctx *hctx;
1221 	int i;
1222 
1223 	queue_for_each_hw_ctx(q, hctx, i)
1224 		blk_mq_start_hw_queue(hctx);
1225 }
1226 EXPORT_SYMBOL(blk_mq_start_hw_queues);
1227 
1228 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1229 {
1230 	if (!blk_mq_hctx_stopped(hctx))
1231 		return;
1232 
1233 	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1234 	blk_mq_run_hw_queue(hctx, async);
1235 }
1236 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
1237 
1238 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
1239 {
1240 	struct blk_mq_hw_ctx *hctx;
1241 	int i;
1242 
1243 	queue_for_each_hw_ctx(q, hctx, i)
1244 		blk_mq_start_stopped_hw_queue(hctx, async);
1245 }
1246 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
1247 
1248 static void blk_mq_run_work_fn(struct work_struct *work)
1249 {
1250 	struct blk_mq_hw_ctx *hctx;
1251 
1252 	hctx = container_of(work, struct blk_mq_hw_ctx, run_work);
1253 
1254 	__blk_mq_run_hw_queue(hctx);
1255 }
1256 
1257 static void blk_mq_delay_work_fn(struct work_struct *work)
1258 {
1259 	struct blk_mq_hw_ctx *hctx;
1260 
1261 	hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);
1262 
1263 	if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
1264 		__blk_mq_run_hw_queue(hctx);
1265 }
1266 
1267 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
1268 {
1269 	if (unlikely(!blk_mq_hw_queue_mapped(hctx)))
1270 		return;
1271 
1272 	blk_mq_stop_hw_queue(hctx);
1273 	kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
1274 			&hctx->delay_work, msecs_to_jiffies(msecs));
1275 }
1276 EXPORT_SYMBOL(blk_mq_delay_queue);
1277 
1278 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
1279 					    struct request *rq,
1280 					    bool at_head)
1281 {
1282 	struct blk_mq_ctx *ctx = rq->mq_ctx;
1283 
1284 	trace_block_rq_insert(hctx->queue, rq);
1285 
1286 	if (at_head)
1287 		list_add(&rq->queuelist, &ctx->rq_list);
1288 	else
1289 		list_add_tail(&rq->queuelist, &ctx->rq_list);
1290 }
1291 
1292 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
1293 			     bool at_head)
1294 {
1295 	struct blk_mq_ctx *ctx = rq->mq_ctx;
1296 
1297 	__blk_mq_insert_req_list(hctx, rq, at_head);
1298 	blk_mq_hctx_mark_pending(hctx, ctx);
1299 }
1300 
1301 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
1302 			    struct list_head *list)
1303 
1304 {
1305 	/*
1306 	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
1307 	 * offline now
1308 	 */
1309 	spin_lock(&ctx->lock);
1310 	while (!list_empty(list)) {
1311 		struct request *rq;
1312 
1313 		rq = list_first_entry(list, struct request, queuelist);
1314 		BUG_ON(rq->mq_ctx != ctx);
1315 		list_del_init(&rq->queuelist);
1316 		__blk_mq_insert_req_list(hctx, rq, false);
1317 	}
1318 	blk_mq_hctx_mark_pending(hctx, ctx);
1319 	spin_unlock(&ctx->lock);
1320 }
1321 
1322 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1323 {
1324 	struct request *rqa = container_of(a, struct request, queuelist);
1325 	struct request *rqb = container_of(b, struct request, queuelist);
1326 
1327 	return !(rqa->mq_ctx < rqb->mq_ctx ||
1328 		 (rqa->mq_ctx == rqb->mq_ctx &&
1329 		  blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1330 }
1331 
1332 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1333 {
1334 	struct blk_mq_ctx *this_ctx;
1335 	struct request_queue *this_q;
1336 	struct request *rq;
1337 	LIST_HEAD(list);
1338 	LIST_HEAD(ctx_list);
1339 	unsigned int depth;
1340 
1341 	list_splice_init(&plug->mq_list, &list);
1342 
1343 	list_sort(NULL, &list, plug_ctx_cmp);
1344 
1345 	this_q = NULL;
1346 	this_ctx = NULL;
1347 	depth = 0;
1348 
1349 	while (!list_empty(&list)) {
1350 		rq = list_entry_rq(list.next);
1351 		list_del_init(&rq->queuelist);
1352 		BUG_ON(!rq->q);
1353 		if (rq->mq_ctx != this_ctx) {
1354 			if (this_ctx) {
1355 				trace_block_unplug(this_q, depth, from_schedule);
1356 				blk_mq_sched_insert_requests(this_q, this_ctx,
1357 								&ctx_list,
1358 								from_schedule);
1359 			}
1360 
1361 			this_ctx = rq->mq_ctx;
1362 			this_q = rq->q;
1363 			depth = 0;
1364 		}
1365 
1366 		depth++;
1367 		list_add_tail(&rq->queuelist, &ctx_list);
1368 	}
1369 
1370 	/*
1371 	 * If 'this_ctx' is set, we know we have entries to complete
1372 	 * on 'ctx_list'. Do those.
1373 	 */
1374 	if (this_ctx) {
1375 		trace_block_unplug(this_q, depth, from_schedule);
1376 		blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list,
1377 						from_schedule);
1378 	}
1379 }
1380 
1381 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1382 {
1383 	init_request_from_bio(rq, bio);
1384 
1385 	blk_account_io_start(rq, true);
1386 }
1387 
1388 static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
1389 {
1390 	return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
1391 		!blk_queue_nomerges(hctx->queue);
1392 }
1393 
1394 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1395 					 struct blk_mq_ctx *ctx,
1396 					 struct request *rq, struct bio *bio)
1397 {
1398 	if (!hctx_allow_merges(hctx) || !bio_mergeable(bio)) {
1399 		blk_mq_bio_to_request(rq, bio);
1400 		spin_lock(&ctx->lock);
1401 insert_rq:
1402 		__blk_mq_insert_request(hctx, rq, false);
1403 		spin_unlock(&ctx->lock);
1404 		return false;
1405 	} else {
1406 		struct request_queue *q = hctx->queue;
1407 
1408 		spin_lock(&ctx->lock);
1409 		if (!blk_mq_attempt_merge(q, ctx, bio)) {
1410 			blk_mq_bio_to_request(rq, bio);
1411 			goto insert_rq;
1412 		}
1413 
1414 		spin_unlock(&ctx->lock);
1415 		__blk_mq_finish_request(hctx, ctx, rq);
1416 		return true;
1417 	}
1418 }
1419 
1420 static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
1421 {
1422 	if (rq->tag != -1)
1423 		return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false);
1424 
1425 	return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true);
1426 }
1427 
1428 static void blk_mq_try_issue_directly(struct request *rq, blk_qc_t *cookie,
1429 				      bool may_sleep)
1430 {
1431 	struct request_queue *q = rq->q;
1432 	struct blk_mq_queue_data bd = {
1433 		.rq = rq,
1434 		.list = NULL,
1435 		.last = 1
1436 	};
1437 	struct blk_mq_hw_ctx *hctx;
1438 	blk_qc_t new_cookie;
1439 	int ret;
1440 
1441 	if (q->elevator)
1442 		goto insert;
1443 
1444 	if (!blk_mq_get_driver_tag(rq, &hctx, false))
1445 		goto insert;
1446 
1447 	new_cookie = request_to_qc_t(hctx, rq);
1448 
1449 	/*
1450 	 * For OK queue, we are done. For error, kill it. Any other
1451 	 * error (busy), just add it to our list as we previously
1452 	 * would have done
1453 	 */
1454 	ret = q->mq_ops->queue_rq(hctx, &bd);
1455 	if (ret == BLK_MQ_RQ_QUEUE_OK) {
1456 		*cookie = new_cookie;
1457 		return;
1458 	}
1459 
1460 	__blk_mq_requeue_request(rq);
1461 
1462 	if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1463 		*cookie = BLK_QC_T_NONE;
1464 		rq->errors = -EIO;
1465 		blk_mq_end_request(rq, rq->errors);
1466 		return;
1467 	}
1468 
1469 insert:
1470 	blk_mq_sched_insert_request(rq, false, true, false, may_sleep);
1471 }
1472 
1473 /*
1474  * Multiple hardware queue variant. This will not use per-process plugs,
1475  * but will attempt to bypass the hctx queueing if we can go straight to
1476  * hardware for SYNC IO.
1477  */
1478 static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
1479 {
1480 	const int is_sync = op_is_sync(bio->bi_opf);
1481 	const int is_flush_fua = op_is_flush(bio->bi_opf);
1482 	struct blk_mq_alloc_data data = { .flags = 0 };
1483 	struct request *rq;
1484 	unsigned int request_count = 0, srcu_idx;
1485 	struct blk_plug *plug;
1486 	struct request *same_queue_rq = NULL;
1487 	blk_qc_t cookie;
1488 	unsigned int wb_acct;
1489 
1490 	blk_queue_bounce(q, &bio);
1491 
1492 	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1493 		bio_io_error(bio);
1494 		return BLK_QC_T_NONE;
1495 	}
1496 
1497 	blk_queue_split(q, &bio, q->bio_split);
1498 
1499 	if (!is_flush_fua && !blk_queue_nomerges(q) &&
1500 	    blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
1501 		return BLK_QC_T_NONE;
1502 
1503 	if (blk_mq_sched_bio_merge(q, bio))
1504 		return BLK_QC_T_NONE;
1505 
1506 	wb_acct = wbt_wait(q->rq_wb, bio, NULL);
1507 
1508 	trace_block_getrq(q, bio, bio->bi_opf);
1509 
1510 	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
1511 	if (unlikely(!rq)) {
1512 		__wbt_done(q->rq_wb, wb_acct);
1513 		return BLK_QC_T_NONE;
1514 	}
1515 
1516 	wbt_track(&rq->issue_stat, wb_acct);
1517 
1518 	cookie = request_to_qc_t(data.hctx, rq);
1519 
1520 	if (unlikely(is_flush_fua)) {
1521 		if (q->elevator)
1522 			goto elv_insert;
1523 		blk_mq_bio_to_request(rq, bio);
1524 		blk_insert_flush(rq);
1525 		goto run_queue;
1526 	}
1527 
1528 	plug = current->plug;
1529 	/*
1530 	 * If the driver supports defer issued based on 'last', then
1531 	 * queue it up like normal since we can potentially save some
1532 	 * CPU this way.
1533 	 */
1534 	if (((plug && !blk_queue_nomerges(q)) || is_sync) &&
1535 	    !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {
1536 		struct request *old_rq = NULL;
1537 
1538 		blk_mq_bio_to_request(rq, bio);
1539 
1540 		/*
1541 		 * We do limited plugging. If the bio can be merged, do that.
1542 		 * Otherwise the existing request in the plug list will be
1543 		 * issued. So the plug list will have one request at most
1544 		 */
1545 		if (plug) {
1546 			/*
1547 			 * The plug list might get flushed before this. If that
1548 			 * happens, same_queue_rq is invalid and plug list is
1549 			 * empty
1550 			 */
1551 			if (same_queue_rq && !list_empty(&plug->mq_list)) {
1552 				old_rq = same_queue_rq;
1553 				list_del_init(&old_rq->queuelist);
1554 			}
1555 			list_add_tail(&rq->queuelist, &plug->mq_list);
1556 		} else /* is_sync */
1557 			old_rq = rq;
1558 		blk_mq_put_ctx(data.ctx);
1559 		if (!old_rq)
1560 			goto done;
1561 
1562 		if (!(data.hctx->flags & BLK_MQ_F_BLOCKING)) {
1563 			rcu_read_lock();
1564 			blk_mq_try_issue_directly(old_rq, &cookie, false);
1565 			rcu_read_unlock();
1566 		} else {
1567 			srcu_idx = srcu_read_lock(&data.hctx->queue_rq_srcu);
1568 			blk_mq_try_issue_directly(old_rq, &cookie, true);
1569 			srcu_read_unlock(&data.hctx->queue_rq_srcu, srcu_idx);
1570 		}
1571 		goto done;
1572 	}
1573 
1574 	if (q->elevator) {
1575 elv_insert:
1576 		blk_mq_put_ctx(data.ctx);
1577 		blk_mq_bio_to_request(rq, bio);
1578 		blk_mq_sched_insert_request(rq, false, true,
1579 						!is_sync || is_flush_fua, true);
1580 		goto done;
1581 	}
1582 	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1583 		/*
1584 		 * For a SYNC request, send it to the hardware immediately. For
1585 		 * an ASYNC request, just ensure that we run it later on. The
1586 		 * latter allows for merging opportunities and more efficient
1587 		 * dispatching.
1588 		 */
1589 run_queue:
1590 		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1591 	}
1592 	blk_mq_put_ctx(data.ctx);
1593 done:
1594 	return cookie;
1595 }
1596 
1597 /*
1598  * Single hardware queue variant. This will attempt to use any per-process
1599  * plug for merging and IO deferral.
1600  */
1601 static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio)
1602 {
1603 	const int is_sync = op_is_sync(bio->bi_opf);
1604 	const int is_flush_fua = op_is_flush(bio->bi_opf);
1605 	struct blk_plug *plug;
1606 	unsigned int request_count = 0;
1607 	struct blk_mq_alloc_data data = { .flags = 0 };
1608 	struct request *rq;
1609 	blk_qc_t cookie;
1610 	unsigned int wb_acct;
1611 
1612 	blk_queue_bounce(q, &bio);
1613 
1614 	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1615 		bio_io_error(bio);
1616 		return BLK_QC_T_NONE;
1617 	}
1618 
1619 	blk_queue_split(q, &bio, q->bio_split);
1620 
1621 	if (!is_flush_fua && !blk_queue_nomerges(q)) {
1622 		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
1623 			return BLK_QC_T_NONE;
1624 	} else
1625 		request_count = blk_plug_queued_count(q);
1626 
1627 	if (blk_mq_sched_bio_merge(q, bio))
1628 		return BLK_QC_T_NONE;
1629 
1630 	wb_acct = wbt_wait(q->rq_wb, bio, NULL);
1631 
1632 	trace_block_getrq(q, bio, bio->bi_opf);
1633 
1634 	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
1635 	if (unlikely(!rq)) {
1636 		__wbt_done(q->rq_wb, wb_acct);
1637 		return BLK_QC_T_NONE;
1638 	}
1639 
1640 	wbt_track(&rq->issue_stat, wb_acct);
1641 
1642 	cookie = request_to_qc_t(data.hctx, rq);
1643 
1644 	if (unlikely(is_flush_fua)) {
1645 		if (q->elevator)
1646 			goto elv_insert;
1647 		blk_mq_bio_to_request(rq, bio);
1648 		blk_insert_flush(rq);
1649 		goto run_queue;
1650 	}
1651 
1652 	/*
1653 	 * A task plug currently exists. Since this is completely lockless,
1654 	 * utilize that to temporarily store requests until the task is
1655 	 * either done or scheduled away.
1656 	 */
1657 	plug = current->plug;
1658 	if (plug) {
1659 		struct request *last = NULL;
1660 
1661 		blk_mq_bio_to_request(rq, bio);
1662 
1663 		/*
1664 		 * @request_count may become stale because of schedule
1665 		 * out, so check the list again.
1666 		 */
1667 		if (list_empty(&plug->mq_list))
1668 			request_count = 0;
1669 		if (!request_count)
1670 			trace_block_plug(q);
1671 		else
1672 			last = list_entry_rq(plug->mq_list.prev);
1673 
1674 		blk_mq_put_ctx(data.ctx);
1675 
1676 		if (request_count >= BLK_MAX_REQUEST_COUNT || (last &&
1677 		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1678 			blk_flush_plug_list(plug, false);
1679 			trace_block_plug(q);
1680 		}
1681 
1682 		list_add_tail(&rq->queuelist, &plug->mq_list);
1683 		return cookie;
1684 	}
1685 
1686 	if (q->elevator) {
1687 elv_insert:
1688 		blk_mq_put_ctx(data.ctx);
1689 		blk_mq_bio_to_request(rq, bio);
1690 		blk_mq_sched_insert_request(rq, false, true,
1691 						!is_sync || is_flush_fua, true);
1692 		goto done;
1693 	}
1694 	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1695 		/*
1696 		 * For a SYNC request, send it to the hardware immediately. For
1697 		 * an ASYNC request, just ensure that we run it later on. The
1698 		 * latter allows for merging opportunities and more efficient
1699 		 * dispatching.
1700 		 */
1701 run_queue:
1702 		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1703 	}
1704 
1705 	blk_mq_put_ctx(data.ctx);
1706 done:
1707 	return cookie;
1708 }
1709 
1710 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
1711 		     unsigned int hctx_idx)
1712 {
1713 	struct page *page;
1714 
1715 	if (tags->rqs && set->ops->exit_request) {
1716 		int i;
1717 
1718 		for (i = 0; i < tags->nr_tags; i++) {
1719 			struct request *rq = tags->static_rqs[i];
1720 
1721 			if (!rq)
1722 				continue;
1723 			set->ops->exit_request(set->driver_data, rq,
1724 						hctx_idx, i);
1725 			tags->static_rqs[i] = NULL;
1726 		}
1727 	}
1728 
1729 	while (!list_empty(&tags->page_list)) {
1730 		page = list_first_entry(&tags->page_list, struct page, lru);
1731 		list_del_init(&page->lru);
1732 		/*
1733 		 * Remove kmemleak object previously allocated in
1734 		 * blk_mq_init_rq_map().
1735 		 */
1736 		kmemleak_free(page_address(page));
1737 		__free_pages(page, page->private);
1738 	}
1739 }
1740 
1741 void blk_mq_free_rq_map(struct blk_mq_tags *tags)
1742 {
1743 	kfree(tags->rqs);
1744 	tags->rqs = NULL;
1745 	kfree(tags->static_rqs);
1746 	tags->static_rqs = NULL;
1747 
1748 	blk_mq_free_tags(tags);
1749 }
1750 
1751 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
1752 					unsigned int hctx_idx,
1753 					unsigned int nr_tags,
1754 					unsigned int reserved_tags)
1755 {
1756 	struct blk_mq_tags *tags;
1757 	int node;
1758 
1759 	node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
1760 	if (node == NUMA_NO_NODE)
1761 		node = set->numa_node;
1762 
1763 	tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
1764 				BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
1765 	if (!tags)
1766 		return NULL;
1767 
1768 	tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1769 				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1770 				 node);
1771 	if (!tags->rqs) {
1772 		blk_mq_free_tags(tags);
1773 		return NULL;
1774 	}
1775 
1776 	tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1777 				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1778 				 node);
1779 	if (!tags->static_rqs) {
1780 		kfree(tags->rqs);
1781 		blk_mq_free_tags(tags);
1782 		return NULL;
1783 	}
1784 
1785 	return tags;
1786 }
1787 
1788 static size_t order_to_size(unsigned int order)
1789 {
1790 	return (size_t)PAGE_SIZE << order;
1791 }
1792 
1793 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
1794 		     unsigned int hctx_idx, unsigned int depth)
1795 {
1796 	unsigned int i, j, entries_per_page, max_order = 4;
1797 	size_t rq_size, left;
1798 	int node;
1799 
1800 	node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
1801 	if (node == NUMA_NO_NODE)
1802 		node = set->numa_node;
1803 
1804 	INIT_LIST_HEAD(&tags->page_list);
1805 
1806 	/*
1807 	 * rq_size is the size of the request plus driver payload, rounded
1808 	 * to the cacheline size
1809 	 */
1810 	rq_size = round_up(sizeof(struct request) + set->cmd_size,
1811 				cache_line_size());
1812 	left = rq_size * depth;
1813 
1814 	for (i = 0; i < depth; ) {
1815 		int this_order = max_order;
1816 		struct page *page;
1817 		int to_do;
1818 		void *p;
1819 
1820 		while (this_order && left < order_to_size(this_order - 1))
1821 			this_order--;
1822 
1823 		do {
1824 			page = alloc_pages_node(node,
1825 				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
1826 				this_order);
1827 			if (page)
1828 				break;
1829 			if (!this_order--)
1830 				break;
1831 			if (order_to_size(this_order) < rq_size)
1832 				break;
1833 		} while (1);
1834 
1835 		if (!page)
1836 			goto fail;
1837 
1838 		page->private = this_order;
1839 		list_add_tail(&page->lru, &tags->page_list);
1840 
1841 		p = page_address(page);
1842 		/*
1843 		 * Allow kmemleak to scan these pages as they contain pointers
1844 		 * to additional allocations like via ops->init_request().
1845 		 */
1846 		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
1847 		entries_per_page = order_to_size(this_order) / rq_size;
1848 		to_do = min(entries_per_page, depth - i);
1849 		left -= to_do * rq_size;
1850 		for (j = 0; j < to_do; j++) {
1851 			struct request *rq = p;
1852 
1853 			tags->static_rqs[i] = rq;
1854 			if (set->ops->init_request) {
1855 				if (set->ops->init_request(set->driver_data,
1856 						rq, hctx_idx, i,
1857 						node)) {
1858 					tags->static_rqs[i] = NULL;
1859 					goto fail;
1860 				}
1861 			}
1862 
1863 			p += rq_size;
1864 			i++;
1865 		}
1866 	}
1867 	return 0;
1868 
1869 fail:
1870 	blk_mq_free_rqs(set, tags, hctx_idx);
1871 	return -ENOMEM;
1872 }
1873 
1874 /*
1875  * 'cpu' is going away. splice any existing rq_list entries from this
1876  * software queue to the hw queue dispatch list, and ensure that it
1877  * gets run.
1878  */
1879 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
1880 {
1881 	struct blk_mq_hw_ctx *hctx;
1882 	struct blk_mq_ctx *ctx;
1883 	LIST_HEAD(tmp);
1884 
1885 	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
1886 	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
1887 
1888 	spin_lock(&ctx->lock);
1889 	if (!list_empty(&ctx->rq_list)) {
1890 		list_splice_init(&ctx->rq_list, &tmp);
1891 		blk_mq_hctx_clear_pending(hctx, ctx);
1892 	}
1893 	spin_unlock(&ctx->lock);
1894 
1895 	if (list_empty(&tmp))
1896 		return 0;
1897 
1898 	spin_lock(&hctx->lock);
1899 	list_splice_tail_init(&tmp, &hctx->dispatch);
1900 	spin_unlock(&hctx->lock);
1901 
1902 	blk_mq_run_hw_queue(hctx, true);
1903 	return 0;
1904 }
1905 
1906 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
1907 {
1908 	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
1909 					    &hctx->cpuhp_dead);
1910 }
1911 
1912 /* hctx->ctxs will be freed in queue's release handler */
1913 static void blk_mq_exit_hctx(struct request_queue *q,
1914 		struct blk_mq_tag_set *set,
1915 		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
1916 {
1917 	unsigned flush_start_tag = set->queue_depth;
1918 
1919 	blk_mq_tag_idle(hctx);
1920 
1921 	if (set->ops->exit_request)
1922 		set->ops->exit_request(set->driver_data,
1923 				       hctx->fq->flush_rq, hctx_idx,
1924 				       flush_start_tag + hctx_idx);
1925 
1926 	if (set->ops->exit_hctx)
1927 		set->ops->exit_hctx(hctx, hctx_idx);
1928 
1929 	if (hctx->flags & BLK_MQ_F_BLOCKING)
1930 		cleanup_srcu_struct(&hctx->queue_rq_srcu);
1931 
1932 	blk_mq_remove_cpuhp(hctx);
1933 	blk_free_flush_queue(hctx->fq);
1934 	sbitmap_free(&hctx->ctx_map);
1935 }
1936 
1937 static void blk_mq_exit_hw_queues(struct request_queue *q,
1938 		struct blk_mq_tag_set *set, int nr_queue)
1939 {
1940 	struct blk_mq_hw_ctx *hctx;
1941 	unsigned int i;
1942 
1943 	queue_for_each_hw_ctx(q, hctx, i) {
1944 		if (i == nr_queue)
1945 			break;
1946 		blk_mq_exit_hctx(q, set, hctx, i);
1947 	}
1948 }
1949 
1950 static int blk_mq_init_hctx(struct request_queue *q,
1951 		struct blk_mq_tag_set *set,
1952 		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1953 {
1954 	int node;
1955 	unsigned flush_start_tag = set->queue_depth;
1956 
1957 	node = hctx->numa_node;
1958 	if (node == NUMA_NO_NODE)
1959 		node = hctx->numa_node = set->numa_node;
1960 
1961 	INIT_WORK(&hctx->run_work, blk_mq_run_work_fn);
1962 	INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
1963 	spin_lock_init(&hctx->lock);
1964 	INIT_LIST_HEAD(&hctx->dispatch);
1965 	hctx->queue = q;
1966 	hctx->queue_num = hctx_idx;
1967 	hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
1968 
1969 	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
1970 
1971 	hctx->tags = set->tags[hctx_idx];
1972 
1973 	/*
1974 	 * Allocate space for all possible cpus to avoid allocation at
1975 	 * runtime
1976 	 */
1977 	hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1978 					GFP_KERNEL, node);
1979 	if (!hctx->ctxs)
1980 		goto unregister_cpu_notifier;
1981 
1982 	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL,
1983 			      node))
1984 		goto free_ctxs;
1985 
1986 	hctx->nr_ctx = 0;
1987 
1988 	if (set->ops->init_hctx &&
1989 	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
1990 		goto free_bitmap;
1991 
1992 	hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
1993 	if (!hctx->fq)
1994 		goto exit_hctx;
1995 
1996 	if (set->ops->init_request &&
1997 	    set->ops->init_request(set->driver_data,
1998 				   hctx->fq->flush_rq, hctx_idx,
1999 				   flush_start_tag + hctx_idx, node))
2000 		goto free_fq;
2001 
2002 	if (hctx->flags & BLK_MQ_F_BLOCKING)
2003 		init_srcu_struct(&hctx->queue_rq_srcu);
2004 
2005 	return 0;
2006 
2007  free_fq:
2008 	kfree(hctx->fq);
2009  exit_hctx:
2010 	if (set->ops->exit_hctx)
2011 		set->ops->exit_hctx(hctx, hctx_idx);
2012  free_bitmap:
2013 	sbitmap_free(&hctx->ctx_map);
2014  free_ctxs:
2015 	kfree(hctx->ctxs);
2016  unregister_cpu_notifier:
2017 	blk_mq_remove_cpuhp(hctx);
2018 	return -1;
2019 }
2020 
2021 static void blk_mq_init_cpu_queues(struct request_queue *q,
2022 				   unsigned int nr_hw_queues)
2023 {
2024 	unsigned int i;
2025 
2026 	for_each_possible_cpu(i) {
2027 		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
2028 		struct blk_mq_hw_ctx *hctx;
2029 
2030 		__ctx->cpu = i;
2031 		spin_lock_init(&__ctx->lock);
2032 		INIT_LIST_HEAD(&__ctx->rq_list);
2033 		__ctx->queue = q;
2034 		blk_stat_init(&__ctx->stat[BLK_STAT_READ]);
2035 		blk_stat_init(&__ctx->stat[BLK_STAT_WRITE]);
2036 
2037 		/* If the cpu isn't online, the cpu is mapped to first hctx */
2038 		if (!cpu_online(i))
2039 			continue;
2040 
2041 		hctx = blk_mq_map_queue(q, i);
2042 
2043 		/*
2044 		 * Set local node, IFF we have more than one hw queue. If
2045 		 * not, we remain on the home node of the device
2046 		 */
2047 		if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
2048 			hctx->numa_node = local_memory_node(cpu_to_node(i));
2049 	}
2050 }
2051 
2052 static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx)
2053 {
2054 	int ret = 0;
2055 
2056 	set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
2057 					set->queue_depth, set->reserved_tags);
2058 	if (!set->tags[hctx_idx])
2059 		return false;
2060 
2061 	ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
2062 				set->queue_depth);
2063 	if (!ret)
2064 		return true;
2065 
2066 	blk_mq_free_rq_map(set->tags[hctx_idx]);
2067 	set->tags[hctx_idx] = NULL;
2068 	return false;
2069 }
2070 
2071 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
2072 					 unsigned int hctx_idx)
2073 {
2074 	if (set->tags[hctx_idx]) {
2075 		blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
2076 		blk_mq_free_rq_map(set->tags[hctx_idx]);
2077 		set->tags[hctx_idx] = NULL;
2078 	}
2079 }
2080 
2081 static void blk_mq_map_swqueue(struct request_queue *q,
2082 			       const struct cpumask *online_mask)
2083 {
2084 	unsigned int i, hctx_idx;
2085 	struct blk_mq_hw_ctx *hctx;
2086 	struct blk_mq_ctx *ctx;
2087 	struct blk_mq_tag_set *set = q->tag_set;
2088 
2089 	/*
2090 	 * Avoid others reading imcomplete hctx->cpumask through sysfs
2091 	 */
2092 	mutex_lock(&q->sysfs_lock);
2093 
2094 	queue_for_each_hw_ctx(q, hctx, i) {
2095 		cpumask_clear(hctx->cpumask);
2096 		hctx->nr_ctx = 0;
2097 	}
2098 
2099 	/*
2100 	 * Map software to hardware queues
2101 	 */
2102 	for_each_possible_cpu(i) {
2103 		/* If the cpu isn't online, the cpu is mapped to first hctx */
2104 		if (!cpumask_test_cpu(i, online_mask))
2105 			continue;
2106 
2107 		hctx_idx = q->mq_map[i];
2108 		/* unmapped hw queue can be remapped after CPU topo changed */
2109 		if (!set->tags[hctx_idx] &&
2110 		    !__blk_mq_alloc_rq_map(set, hctx_idx)) {
2111 			/*
2112 			 * If tags initialization fail for some hctx,
2113 			 * that hctx won't be brought online.  In this
2114 			 * case, remap the current ctx to hctx[0] which
2115 			 * is guaranteed to always have tags allocated
2116 			 */
2117 			q->mq_map[i] = 0;
2118 		}
2119 
2120 		ctx = per_cpu_ptr(q->queue_ctx, i);
2121 		hctx = blk_mq_map_queue(q, i);
2122 
2123 		cpumask_set_cpu(i, hctx->cpumask);
2124 		ctx->index_hw = hctx->nr_ctx;
2125 		hctx->ctxs[hctx->nr_ctx++] = ctx;
2126 	}
2127 
2128 	mutex_unlock(&q->sysfs_lock);
2129 
2130 	queue_for_each_hw_ctx(q, hctx, i) {
2131 		/*
2132 		 * If no software queues are mapped to this hardware queue,
2133 		 * disable it and free the request entries.
2134 		 */
2135 		if (!hctx->nr_ctx) {
2136 			/* Never unmap queue 0.  We need it as a
2137 			 * fallback in case of a new remap fails
2138 			 * allocation
2139 			 */
2140 			if (i && set->tags[i])
2141 				blk_mq_free_map_and_requests(set, i);
2142 
2143 			hctx->tags = NULL;
2144 			continue;
2145 		}
2146 
2147 		hctx->tags = set->tags[i];
2148 		WARN_ON(!hctx->tags);
2149 
2150 		/*
2151 		 * Set the map size to the number of mapped software queues.
2152 		 * This is more accurate and more efficient than looping
2153 		 * over all possibly mapped software queues.
2154 		 */
2155 		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
2156 
2157 		/*
2158 		 * Initialize batch roundrobin counts
2159 		 */
2160 		hctx->next_cpu = cpumask_first(hctx->cpumask);
2161 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
2162 	}
2163 }
2164 
2165 static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2166 {
2167 	struct blk_mq_hw_ctx *hctx;
2168 	int i;
2169 
2170 	queue_for_each_hw_ctx(q, hctx, i) {
2171 		if (shared)
2172 			hctx->flags |= BLK_MQ_F_TAG_SHARED;
2173 		else
2174 			hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
2175 	}
2176 }
2177 
2178 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared)
2179 {
2180 	struct request_queue *q;
2181 
2182 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
2183 		blk_mq_freeze_queue(q);
2184 		queue_set_hctx_shared(q, shared);
2185 		blk_mq_unfreeze_queue(q);
2186 	}
2187 }
2188 
2189 static void blk_mq_del_queue_tag_set(struct request_queue *q)
2190 {
2191 	struct blk_mq_tag_set *set = q->tag_set;
2192 
2193 	mutex_lock(&set->tag_list_lock);
2194 	list_del_init(&q->tag_set_list);
2195 	if (list_is_singular(&set->tag_list)) {
2196 		/* just transitioned to unshared */
2197 		set->flags &= ~BLK_MQ_F_TAG_SHARED;
2198 		/* update existing queue */
2199 		blk_mq_update_tag_set_depth(set, false);
2200 	}
2201 	mutex_unlock(&set->tag_list_lock);
2202 }
2203 
2204 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
2205 				     struct request_queue *q)
2206 {
2207 	q->tag_set = set;
2208 
2209 	mutex_lock(&set->tag_list_lock);
2210 
2211 	/* Check to see if we're transitioning to shared (from 1 to 2 queues). */
2212 	if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) {
2213 		set->flags |= BLK_MQ_F_TAG_SHARED;
2214 		/* update existing queue */
2215 		blk_mq_update_tag_set_depth(set, true);
2216 	}
2217 	if (set->flags & BLK_MQ_F_TAG_SHARED)
2218 		queue_set_hctx_shared(q, true);
2219 	list_add_tail(&q->tag_set_list, &set->tag_list);
2220 
2221 	mutex_unlock(&set->tag_list_lock);
2222 }
2223 
2224 /*
2225  * It is the actual release handler for mq, but we do it from
2226  * request queue's release handler for avoiding use-after-free
2227  * and headache because q->mq_kobj shouldn't have been introduced,
2228  * but we can't group ctx/kctx kobj without it.
2229  */
2230 void blk_mq_release(struct request_queue *q)
2231 {
2232 	struct blk_mq_hw_ctx *hctx;
2233 	unsigned int i;
2234 
2235 	blk_mq_sched_teardown(q);
2236 
2237 	/* hctx kobj stays in hctx */
2238 	queue_for_each_hw_ctx(q, hctx, i) {
2239 		if (!hctx)
2240 			continue;
2241 		kobject_put(&hctx->kobj);
2242 	}
2243 
2244 	q->mq_map = NULL;
2245 
2246 	kfree(q->queue_hw_ctx);
2247 
2248 	/*
2249 	 * release .mq_kobj and sw queue's kobject now because
2250 	 * both share lifetime with request queue.
2251 	 */
2252 	blk_mq_sysfs_deinit(q);
2253 
2254 	free_percpu(q->queue_ctx);
2255 }
2256 
2257 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
2258 {
2259 	struct request_queue *uninit_q, *q;
2260 
2261 	uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
2262 	if (!uninit_q)
2263 		return ERR_PTR(-ENOMEM);
2264 
2265 	q = blk_mq_init_allocated_queue(set, uninit_q);
2266 	if (IS_ERR(q))
2267 		blk_cleanup_queue(uninit_q);
2268 
2269 	return q;
2270 }
2271 EXPORT_SYMBOL(blk_mq_init_queue);
2272 
2273 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
2274 						struct request_queue *q)
2275 {
2276 	int i, j;
2277 	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
2278 
2279 	blk_mq_sysfs_unregister(q);
2280 	for (i = 0; i < set->nr_hw_queues; i++) {
2281 		int node;
2282 
2283 		if (hctxs[i])
2284 			continue;
2285 
2286 		node = blk_mq_hw_queue_to_node(q->mq_map, i);
2287 		hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
2288 					GFP_KERNEL, node);
2289 		if (!hctxs[i])
2290 			break;
2291 
2292 		if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
2293 						node)) {
2294 			kfree(hctxs[i]);
2295 			hctxs[i] = NULL;
2296 			break;
2297 		}
2298 
2299 		atomic_set(&hctxs[i]->nr_active, 0);
2300 		hctxs[i]->numa_node = node;
2301 		hctxs[i]->queue_num = i;
2302 
2303 		if (blk_mq_init_hctx(q, set, hctxs[i], i)) {
2304 			free_cpumask_var(hctxs[i]->cpumask);
2305 			kfree(hctxs[i]);
2306 			hctxs[i] = NULL;
2307 			break;
2308 		}
2309 		blk_mq_hctx_kobj_init(hctxs[i]);
2310 	}
2311 	for (j = i; j < q->nr_hw_queues; j++) {
2312 		struct blk_mq_hw_ctx *hctx = hctxs[j];
2313 
2314 		if (hctx) {
2315 			if (hctx->tags)
2316 				blk_mq_free_map_and_requests(set, j);
2317 			blk_mq_exit_hctx(q, set, hctx, j);
2318 			kobject_put(&hctx->kobj);
2319 			hctxs[j] = NULL;
2320 
2321 		}
2322 	}
2323 	q->nr_hw_queues = i;
2324 	blk_mq_sysfs_register(q);
2325 }
2326 
2327 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
2328 						  struct request_queue *q)
2329 {
2330 	/* mark the queue as mq asap */
2331 	q->mq_ops = set->ops;
2332 
2333 	q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
2334 	if (!q->queue_ctx)
2335 		goto err_exit;
2336 
2337 	/* init q->mq_kobj and sw queues' kobjects */
2338 	blk_mq_sysfs_init(q);
2339 
2340 	q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
2341 						GFP_KERNEL, set->numa_node);
2342 	if (!q->queue_hw_ctx)
2343 		goto err_percpu;
2344 
2345 	q->mq_map = set->mq_map;
2346 
2347 	blk_mq_realloc_hw_ctxs(set, q);
2348 	if (!q->nr_hw_queues)
2349 		goto err_hctxs;
2350 
2351 	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
2352 	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
2353 
2354 	q->nr_queues = nr_cpu_ids;
2355 
2356 	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
2357 
2358 	if (!(set->flags & BLK_MQ_F_SG_MERGE))
2359 		q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
2360 
2361 	q->sg_reserved_size = INT_MAX;
2362 
2363 	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
2364 	INIT_LIST_HEAD(&q->requeue_list);
2365 	spin_lock_init(&q->requeue_lock);
2366 
2367 	if (q->nr_hw_queues > 1)
2368 		blk_queue_make_request(q, blk_mq_make_request);
2369 	else
2370 		blk_queue_make_request(q, blk_sq_make_request);
2371 
2372 	/*
2373 	 * Do this after blk_queue_make_request() overrides it...
2374 	 */
2375 	q->nr_requests = set->queue_depth;
2376 
2377 	/*
2378 	 * Default to classic polling
2379 	 */
2380 	q->poll_nsec = -1;
2381 
2382 	if (set->ops->complete)
2383 		blk_queue_softirq_done(q, set->ops->complete);
2384 
2385 	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
2386 
2387 	get_online_cpus();
2388 	mutex_lock(&all_q_mutex);
2389 
2390 	list_add_tail(&q->all_q_node, &all_q_list);
2391 	blk_mq_add_queue_tag_set(set, q);
2392 	blk_mq_map_swqueue(q, cpu_online_mask);
2393 
2394 	mutex_unlock(&all_q_mutex);
2395 	put_online_cpus();
2396 
2397 	if (!(set->flags & BLK_MQ_F_NO_SCHED)) {
2398 		int ret;
2399 
2400 		ret = blk_mq_sched_init(q);
2401 		if (ret)
2402 			return ERR_PTR(ret);
2403 	}
2404 
2405 	return q;
2406 
2407 err_hctxs:
2408 	kfree(q->queue_hw_ctx);
2409 err_percpu:
2410 	free_percpu(q->queue_ctx);
2411 err_exit:
2412 	q->mq_ops = NULL;
2413 	return ERR_PTR(-ENOMEM);
2414 }
2415 EXPORT_SYMBOL(blk_mq_init_allocated_queue);
2416 
2417 void blk_mq_free_queue(struct request_queue *q)
2418 {
2419 	struct blk_mq_tag_set	*set = q->tag_set;
2420 
2421 	mutex_lock(&all_q_mutex);
2422 	list_del_init(&q->all_q_node);
2423 	mutex_unlock(&all_q_mutex);
2424 
2425 	wbt_exit(q);
2426 
2427 	blk_mq_del_queue_tag_set(q);
2428 
2429 	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
2430 }
2431 
2432 /* Basically redo blk_mq_init_queue with queue frozen */
2433 static void blk_mq_queue_reinit(struct request_queue *q,
2434 				const struct cpumask *online_mask)
2435 {
2436 	WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
2437 
2438 	blk_mq_sysfs_unregister(q);
2439 
2440 	/*
2441 	 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
2442 	 * we should change hctx numa_node according to new topology (this
2443 	 * involves free and re-allocate memory, worthy doing?)
2444 	 */
2445 
2446 	blk_mq_map_swqueue(q, online_mask);
2447 
2448 	blk_mq_sysfs_register(q);
2449 }
2450 
2451 /*
2452  * New online cpumask which is going to be set in this hotplug event.
2453  * Declare this cpumasks as global as cpu-hotplug operation is invoked
2454  * one-by-one and dynamically allocating this could result in a failure.
2455  */
2456 static struct cpumask cpuhp_online_new;
2457 
2458 static void blk_mq_queue_reinit_work(void)
2459 {
2460 	struct request_queue *q;
2461 
2462 	mutex_lock(&all_q_mutex);
2463 	/*
2464 	 * We need to freeze and reinit all existing queues.  Freezing
2465 	 * involves synchronous wait for an RCU grace period and doing it
2466 	 * one by one may take a long time.  Start freezing all queues in
2467 	 * one swoop and then wait for the completions so that freezing can
2468 	 * take place in parallel.
2469 	 */
2470 	list_for_each_entry(q, &all_q_list, all_q_node)
2471 		blk_mq_freeze_queue_start(q);
2472 	list_for_each_entry(q, &all_q_list, all_q_node)
2473 		blk_mq_freeze_queue_wait(q);
2474 
2475 	list_for_each_entry(q, &all_q_list, all_q_node)
2476 		blk_mq_queue_reinit(q, &cpuhp_online_new);
2477 
2478 	list_for_each_entry(q, &all_q_list, all_q_node)
2479 		blk_mq_unfreeze_queue(q);
2480 
2481 	mutex_unlock(&all_q_mutex);
2482 }
2483 
2484 static int blk_mq_queue_reinit_dead(unsigned int cpu)
2485 {
2486 	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2487 	blk_mq_queue_reinit_work();
2488 	return 0;
2489 }
2490 
2491 /*
2492  * Before hotadded cpu starts handling requests, new mappings must be
2493  * established.  Otherwise, these requests in hw queue might never be
2494  * dispatched.
2495  *
2496  * For example, there is a single hw queue (hctx) and two CPU queues (ctx0
2497  * for CPU0, and ctx1 for CPU1).
2498  *
2499  * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list
2500  * and set bit0 in pending bitmap as ctx1->index_hw is still zero.
2501  *
2502  * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set
2503  * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list.
2504  * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is
2505  * ignored.
2506  */
2507 static int blk_mq_queue_reinit_prepare(unsigned int cpu)
2508 {
2509 	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2510 	cpumask_set_cpu(cpu, &cpuhp_online_new);
2511 	blk_mq_queue_reinit_work();
2512 	return 0;
2513 }
2514 
2515 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2516 {
2517 	int i;
2518 
2519 	for (i = 0; i < set->nr_hw_queues; i++)
2520 		if (!__blk_mq_alloc_rq_map(set, i))
2521 			goto out_unwind;
2522 
2523 	return 0;
2524 
2525 out_unwind:
2526 	while (--i >= 0)
2527 		blk_mq_free_rq_map(set->tags[i]);
2528 
2529 	return -ENOMEM;
2530 }
2531 
2532 /*
2533  * Allocate the request maps associated with this tag_set. Note that this
2534  * may reduce the depth asked for, if memory is tight. set->queue_depth
2535  * will be updated to reflect the allocated depth.
2536  */
2537 static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2538 {
2539 	unsigned int depth;
2540 	int err;
2541 
2542 	depth = set->queue_depth;
2543 	do {
2544 		err = __blk_mq_alloc_rq_maps(set);
2545 		if (!err)
2546 			break;
2547 
2548 		set->queue_depth >>= 1;
2549 		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
2550 			err = -ENOMEM;
2551 			break;
2552 		}
2553 	} while (set->queue_depth);
2554 
2555 	if (!set->queue_depth || err) {
2556 		pr_err("blk-mq: failed to allocate request map\n");
2557 		return -ENOMEM;
2558 	}
2559 
2560 	if (depth != set->queue_depth)
2561 		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
2562 						depth, set->queue_depth);
2563 
2564 	return 0;
2565 }
2566 
2567 /*
2568  * Alloc a tag set to be associated with one or more request queues.
2569  * May fail with EINVAL for various error conditions. May adjust the
2570  * requested depth down, if if it too large. In that case, the set
2571  * value will be stored in set->queue_depth.
2572  */
2573 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
2574 {
2575 	int ret;
2576 
2577 	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
2578 
2579 	if (!set->nr_hw_queues)
2580 		return -EINVAL;
2581 	if (!set->queue_depth)
2582 		return -EINVAL;
2583 	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
2584 		return -EINVAL;
2585 
2586 	if (!set->ops->queue_rq)
2587 		return -EINVAL;
2588 
2589 	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
2590 		pr_info("blk-mq: reduced tag depth to %u\n",
2591 			BLK_MQ_MAX_DEPTH);
2592 		set->queue_depth = BLK_MQ_MAX_DEPTH;
2593 	}
2594 
2595 	/*
2596 	 * If a crashdump is active, then we are potentially in a very
2597 	 * memory constrained environment. Limit us to 1 queue and
2598 	 * 64 tags to prevent using too much memory.
2599 	 */
2600 	if (is_kdump_kernel()) {
2601 		set->nr_hw_queues = 1;
2602 		set->queue_depth = min(64U, set->queue_depth);
2603 	}
2604 	/*
2605 	 * There is no use for more h/w queues than cpus.
2606 	 */
2607 	if (set->nr_hw_queues > nr_cpu_ids)
2608 		set->nr_hw_queues = nr_cpu_ids;
2609 
2610 	set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *),
2611 				 GFP_KERNEL, set->numa_node);
2612 	if (!set->tags)
2613 		return -ENOMEM;
2614 
2615 	ret = -ENOMEM;
2616 	set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids,
2617 			GFP_KERNEL, set->numa_node);
2618 	if (!set->mq_map)
2619 		goto out_free_tags;
2620 
2621 	if (set->ops->map_queues)
2622 		ret = set->ops->map_queues(set);
2623 	else
2624 		ret = blk_mq_map_queues(set);
2625 	if (ret)
2626 		goto out_free_mq_map;
2627 
2628 	ret = blk_mq_alloc_rq_maps(set);
2629 	if (ret)
2630 		goto out_free_mq_map;
2631 
2632 	mutex_init(&set->tag_list_lock);
2633 	INIT_LIST_HEAD(&set->tag_list);
2634 
2635 	return 0;
2636 
2637 out_free_mq_map:
2638 	kfree(set->mq_map);
2639 	set->mq_map = NULL;
2640 out_free_tags:
2641 	kfree(set->tags);
2642 	set->tags = NULL;
2643 	return ret;
2644 }
2645 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
2646 
2647 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
2648 {
2649 	int i;
2650 
2651 	for (i = 0; i < nr_cpu_ids; i++)
2652 		blk_mq_free_map_and_requests(set, i);
2653 
2654 	kfree(set->mq_map);
2655 	set->mq_map = NULL;
2656 
2657 	kfree(set->tags);
2658 	set->tags = NULL;
2659 }
2660 EXPORT_SYMBOL(blk_mq_free_tag_set);
2661 
2662 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2663 {
2664 	struct blk_mq_tag_set *set = q->tag_set;
2665 	struct blk_mq_hw_ctx *hctx;
2666 	int i, ret;
2667 
2668 	if (!set)
2669 		return -EINVAL;
2670 
2671 	blk_mq_freeze_queue(q);
2672 	blk_mq_quiesce_queue(q);
2673 
2674 	ret = 0;
2675 	queue_for_each_hw_ctx(q, hctx, i) {
2676 		if (!hctx->tags)
2677 			continue;
2678 		/*
2679 		 * If we're using an MQ scheduler, just update the scheduler
2680 		 * queue depth. This is similar to what the old code would do.
2681 		 */
2682 		if (!hctx->sched_tags) {
2683 			ret = blk_mq_tag_update_depth(hctx, &hctx->tags,
2684 							min(nr, set->queue_depth),
2685 							false);
2686 		} else {
2687 			ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
2688 							nr, true);
2689 		}
2690 		if (ret)
2691 			break;
2692 	}
2693 
2694 	if (!ret)
2695 		q->nr_requests = nr;
2696 
2697 	blk_mq_unfreeze_queue(q);
2698 	blk_mq_start_stopped_hw_queues(q, true);
2699 
2700 	return ret;
2701 }
2702 
2703 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
2704 {
2705 	struct request_queue *q;
2706 
2707 	if (nr_hw_queues > nr_cpu_ids)
2708 		nr_hw_queues = nr_cpu_ids;
2709 	if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues)
2710 		return;
2711 
2712 	list_for_each_entry(q, &set->tag_list, tag_set_list)
2713 		blk_mq_freeze_queue(q);
2714 
2715 	set->nr_hw_queues = nr_hw_queues;
2716 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
2717 		blk_mq_realloc_hw_ctxs(set, q);
2718 
2719 		/*
2720 		 * Manually set the make_request_fn as blk_queue_make_request
2721 		 * resets a lot of the queue settings.
2722 		 */
2723 		if (q->nr_hw_queues > 1)
2724 			q->make_request_fn = blk_mq_make_request;
2725 		else
2726 			q->make_request_fn = blk_sq_make_request;
2727 
2728 		blk_mq_queue_reinit(q, cpu_online_mask);
2729 	}
2730 
2731 	list_for_each_entry(q, &set->tag_list, tag_set_list)
2732 		blk_mq_unfreeze_queue(q);
2733 }
2734 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
2735 
2736 static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
2737 				       struct blk_mq_hw_ctx *hctx,
2738 				       struct request *rq)
2739 {
2740 	struct blk_rq_stat stat[2];
2741 	unsigned long ret = 0;
2742 
2743 	/*
2744 	 * If stats collection isn't on, don't sleep but turn it on for
2745 	 * future users
2746 	 */
2747 	if (!blk_stat_enable(q))
2748 		return 0;
2749 
2750 	/*
2751 	 * We don't have to do this once per IO, should optimize this
2752 	 * to just use the current window of stats until it changes
2753 	 */
2754 	memset(&stat, 0, sizeof(stat));
2755 	blk_hctx_stat_get(hctx, stat);
2756 
2757 	/*
2758 	 * As an optimistic guess, use half of the mean service time
2759 	 * for this type of request. We can (and should) make this smarter.
2760 	 * For instance, if the completion latencies are tight, we can
2761 	 * get closer than just half the mean. This is especially
2762 	 * important on devices where the completion latencies are longer
2763 	 * than ~10 usec.
2764 	 */
2765 	if (req_op(rq) == REQ_OP_READ && stat[BLK_STAT_READ].nr_samples)
2766 		ret = (stat[BLK_STAT_READ].mean + 1) / 2;
2767 	else if (req_op(rq) == REQ_OP_WRITE && stat[BLK_STAT_WRITE].nr_samples)
2768 		ret = (stat[BLK_STAT_WRITE].mean + 1) / 2;
2769 
2770 	return ret;
2771 }
2772 
2773 static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
2774 				     struct blk_mq_hw_ctx *hctx,
2775 				     struct request *rq)
2776 {
2777 	struct hrtimer_sleeper hs;
2778 	enum hrtimer_mode mode;
2779 	unsigned int nsecs;
2780 	ktime_t kt;
2781 
2782 	if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags))
2783 		return false;
2784 
2785 	/*
2786 	 * poll_nsec can be:
2787 	 *
2788 	 * -1:	don't ever hybrid sleep
2789 	 *  0:	use half of prev avg
2790 	 * >0:	use this specific value
2791 	 */
2792 	if (q->poll_nsec == -1)
2793 		return false;
2794 	else if (q->poll_nsec > 0)
2795 		nsecs = q->poll_nsec;
2796 	else
2797 		nsecs = blk_mq_poll_nsecs(q, hctx, rq);
2798 
2799 	if (!nsecs)
2800 		return false;
2801 
2802 	set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
2803 
2804 	/*
2805 	 * This will be replaced with the stats tracking code, using
2806 	 * 'avg_completion_time / 2' as the pre-sleep target.
2807 	 */
2808 	kt = nsecs;
2809 
2810 	mode = HRTIMER_MODE_REL;
2811 	hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
2812 	hrtimer_set_expires(&hs.timer, kt);
2813 
2814 	hrtimer_init_sleeper(&hs, current);
2815 	do {
2816 		if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
2817 			break;
2818 		set_current_state(TASK_UNINTERRUPTIBLE);
2819 		hrtimer_start_expires(&hs.timer, mode);
2820 		if (hs.task)
2821 			io_schedule();
2822 		hrtimer_cancel(&hs.timer);
2823 		mode = HRTIMER_MODE_ABS;
2824 	} while (hs.task && !signal_pending(current));
2825 
2826 	__set_current_state(TASK_RUNNING);
2827 	destroy_hrtimer_on_stack(&hs.timer);
2828 	return true;
2829 }
2830 
2831 static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
2832 {
2833 	struct request_queue *q = hctx->queue;
2834 	long state;
2835 
2836 	/*
2837 	 * If we sleep, have the caller restart the poll loop to reset
2838 	 * the state. Like for the other success return cases, the
2839 	 * caller is responsible for checking if the IO completed. If
2840 	 * the IO isn't complete, we'll get called again and will go
2841 	 * straight to the busy poll loop.
2842 	 */
2843 	if (blk_mq_poll_hybrid_sleep(q, hctx, rq))
2844 		return true;
2845 
2846 	hctx->poll_considered++;
2847 
2848 	state = current->state;
2849 	while (!need_resched()) {
2850 		int ret;
2851 
2852 		hctx->poll_invoked++;
2853 
2854 		ret = q->mq_ops->poll(hctx, rq->tag);
2855 		if (ret > 0) {
2856 			hctx->poll_success++;
2857 			set_current_state(TASK_RUNNING);
2858 			return true;
2859 		}
2860 
2861 		if (signal_pending_state(state, current))
2862 			set_current_state(TASK_RUNNING);
2863 
2864 		if (current->state == TASK_RUNNING)
2865 			return true;
2866 		if (ret < 0)
2867 			break;
2868 		cpu_relax();
2869 	}
2870 
2871 	return false;
2872 }
2873 
2874 bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
2875 {
2876 	struct blk_mq_hw_ctx *hctx;
2877 	struct blk_plug *plug;
2878 	struct request *rq;
2879 
2880 	if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
2881 	    !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
2882 		return false;
2883 
2884 	plug = current->plug;
2885 	if (plug)
2886 		blk_flush_plug_list(plug, false);
2887 
2888 	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
2889 	if (!blk_qc_t_is_internal(cookie))
2890 		rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
2891 	else
2892 		rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
2893 
2894 	return __blk_mq_poll(hctx, rq);
2895 }
2896 EXPORT_SYMBOL_GPL(blk_mq_poll);
2897 
2898 void blk_mq_disable_hotplug(void)
2899 {
2900 	mutex_lock(&all_q_mutex);
2901 }
2902 
2903 void blk_mq_enable_hotplug(void)
2904 {
2905 	mutex_unlock(&all_q_mutex);
2906 }
2907 
2908 static int __init blk_mq_init(void)
2909 {
2910 	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
2911 				blk_mq_hctx_notify_dead);
2912 
2913 	cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare",
2914 				  blk_mq_queue_reinit_prepare,
2915 				  blk_mq_queue_reinit_dead);
2916 	return 0;
2917 }
2918 subsys_initcall(blk_mq_init);
2919