1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Block multiqueue core code
4 *
5 * Copyright (C) 2013-2014 Jens Axboe
6 * Copyright (C) 2013-2014 Christoph Hellwig
7 */
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/backing-dev.h>
11 #include <linux/bio.h>
12 #include <linux/blkdev.h>
13 #include <linux/blk-integrity.h>
14 #include <linux/kmemleak.h>
15 #include <linux/mm.h>
16 #include <linux/init.h>
17 #include <linux/slab.h>
18 #include <linux/workqueue.h>
19 #include <linux/smp.h>
20 #include <linux/interrupt.h>
21 #include <linux/llist.h>
22 #include <linux/cpu.h>
23 #include <linux/cache.h>
24 #include <linux/sched/sysctl.h>
25 #include <linux/sched/topology.h>
26 #include <linux/sched/signal.h>
27 #include <linux/delay.h>
28 #include <linux/crash_dump.h>
29 #include <linux/prefetch.h>
30 #include <linux/blk-crypto.h>
31 #include <linux/part_stat.h>
32
33 #include <trace/events/block.h>
34
35 #include <linux/t10-pi.h>
36 #include "blk.h"
37 #include "blk-mq.h"
38 #include "blk-mq-debugfs.h"
39 #include "blk-pm.h"
40 #include "blk-stat.h"
41 #include "blk-mq-sched.h"
42 #include "blk-rq-qos.h"
43 #include "blk-ioprio.h"
44
45 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
46 static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd);
47
48 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags);
49 static void blk_mq_request_bypass_insert(struct request *rq,
50 blk_insert_t flags);
51 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
52 struct list_head *list);
53 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
54 struct io_comp_batch *iob, unsigned int flags);
55
56 /*
57 * Check if any of the ctx, dispatch list or elevator
58 * have pending work in this hardware queue.
59 */
blk_mq_hctx_has_pending(struct blk_mq_hw_ctx * hctx)60 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
61 {
62 return !list_empty_careful(&hctx->dispatch) ||
63 sbitmap_any_bit_set(&hctx->ctx_map) ||
64 blk_mq_sched_has_work(hctx);
65 }
66
67 /*
68 * Mark this ctx as having pending work in this hardware queue
69 */
blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx * hctx,struct blk_mq_ctx * ctx)70 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
71 struct blk_mq_ctx *ctx)
72 {
73 const int bit = ctx->index_hw[hctx->type];
74
75 if (!sbitmap_test_bit(&hctx->ctx_map, bit))
76 sbitmap_set_bit(&hctx->ctx_map, bit);
77 }
78
blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx * hctx,struct blk_mq_ctx * ctx)79 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
80 struct blk_mq_ctx *ctx)
81 {
82 const int bit = ctx->index_hw[hctx->type];
83
84 sbitmap_clear_bit(&hctx->ctx_map, bit);
85 }
86
87 struct mq_inflight {
88 struct block_device *part;
89 unsigned int inflight[2];
90 };
91
blk_mq_check_inflight(struct request * rq,void * priv)92 static bool blk_mq_check_inflight(struct request *rq, void *priv)
93 {
94 struct mq_inflight *mi = priv;
95
96 if (rq->part && blk_do_io_stat(rq) &&
97 (!mi->part->bd_partno || rq->part == mi->part) &&
98 blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
99 mi->inflight[rq_data_dir(rq)]++;
100
101 return true;
102 }
103
blk_mq_in_flight(struct request_queue * q,struct block_device * part)104 unsigned int blk_mq_in_flight(struct request_queue *q,
105 struct block_device *part)
106 {
107 struct mq_inflight mi = { .part = part };
108
109 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
110
111 return mi.inflight[0] + mi.inflight[1];
112 }
113
blk_mq_in_flight_rw(struct request_queue * q,struct block_device * part,unsigned int inflight[2])114 void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part,
115 unsigned int inflight[2])
116 {
117 struct mq_inflight mi = { .part = part };
118
119 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
120 inflight[0] = mi.inflight[0];
121 inflight[1] = mi.inflight[1];
122 }
123
blk_freeze_queue_start(struct request_queue * q)124 void blk_freeze_queue_start(struct request_queue *q)
125 {
126 mutex_lock(&q->mq_freeze_lock);
127 if (++q->mq_freeze_depth == 1) {
128 percpu_ref_kill(&q->q_usage_counter);
129 mutex_unlock(&q->mq_freeze_lock);
130 if (queue_is_mq(q))
131 blk_mq_run_hw_queues(q, false);
132 } else {
133 mutex_unlock(&q->mq_freeze_lock);
134 }
135 }
136 EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
137
blk_mq_freeze_queue_wait(struct request_queue * q)138 void blk_mq_freeze_queue_wait(struct request_queue *q)
139 {
140 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
141 }
142 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
143
blk_mq_freeze_queue_wait_timeout(struct request_queue * q,unsigned long timeout)144 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
145 unsigned long timeout)
146 {
147 return wait_event_timeout(q->mq_freeze_wq,
148 percpu_ref_is_zero(&q->q_usage_counter),
149 timeout);
150 }
151 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
152
153 /*
154 * Guarantee no request is in use, so we can change any data structure of
155 * the queue afterward.
156 */
blk_freeze_queue(struct request_queue * q)157 void blk_freeze_queue(struct request_queue *q)
158 {
159 /*
160 * In the !blk_mq case we are only calling this to kill the
161 * q_usage_counter, otherwise this increases the freeze depth
162 * and waits for it to return to zero. For this reason there is
163 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
164 * exported to drivers as the only user for unfreeze is blk_mq.
165 */
166 blk_freeze_queue_start(q);
167 blk_mq_freeze_queue_wait(q);
168 }
169
blk_mq_freeze_queue(struct request_queue * q)170 void blk_mq_freeze_queue(struct request_queue *q)
171 {
172 /*
173 * ...just an alias to keep freeze and unfreeze actions balanced
174 * in the blk_mq_* namespace
175 */
176 blk_freeze_queue(q);
177 }
178 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
179
__blk_mq_unfreeze_queue(struct request_queue * q,bool force_atomic)180 void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic)
181 {
182 mutex_lock(&q->mq_freeze_lock);
183 if (force_atomic)
184 q->q_usage_counter.data->force_atomic = true;
185 q->mq_freeze_depth--;
186 WARN_ON_ONCE(q->mq_freeze_depth < 0);
187 if (!q->mq_freeze_depth) {
188 percpu_ref_resurrect(&q->q_usage_counter);
189 wake_up_all(&q->mq_freeze_wq);
190 }
191 mutex_unlock(&q->mq_freeze_lock);
192 }
193
blk_mq_unfreeze_queue(struct request_queue * q)194 void blk_mq_unfreeze_queue(struct request_queue *q)
195 {
196 __blk_mq_unfreeze_queue(q, false);
197 }
198 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
199
200 /*
201 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the
202 * mpt3sas driver such that this function can be removed.
203 */
blk_mq_quiesce_queue_nowait(struct request_queue * q)204 void blk_mq_quiesce_queue_nowait(struct request_queue *q)
205 {
206 unsigned long flags;
207
208 spin_lock_irqsave(&q->queue_lock, flags);
209 if (!q->quiesce_depth++)
210 blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
211 spin_unlock_irqrestore(&q->queue_lock, flags);
212 }
213 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
214
215 /**
216 * blk_mq_wait_quiesce_done() - wait until in-progress quiesce is done
217 * @set: tag_set to wait on
218 *
219 * Note: it is driver's responsibility for making sure that quiesce has
220 * been started on or more of the request_queues of the tag_set. This
221 * function only waits for the quiesce on those request_queues that had
222 * the quiesce flag set using blk_mq_quiesce_queue_nowait.
223 */
blk_mq_wait_quiesce_done(struct blk_mq_tag_set * set)224 void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set)
225 {
226 if (set->flags & BLK_MQ_F_BLOCKING)
227 synchronize_srcu(set->srcu);
228 else
229 synchronize_rcu();
230 }
231 EXPORT_SYMBOL_GPL(blk_mq_wait_quiesce_done);
232
233 /**
234 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
235 * @q: request queue.
236 *
237 * Note: this function does not prevent that the struct request end_io()
238 * callback function is invoked. Once this function is returned, we make
239 * sure no dispatch can happen until the queue is unquiesced via
240 * blk_mq_unquiesce_queue().
241 */
blk_mq_quiesce_queue(struct request_queue * q)242 void blk_mq_quiesce_queue(struct request_queue *q)
243 {
244 blk_mq_quiesce_queue_nowait(q);
245 /* nothing to wait for non-mq queues */
246 if (queue_is_mq(q))
247 blk_mq_wait_quiesce_done(q->tag_set);
248 }
249 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
250
251 /*
252 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
253 * @q: request queue.
254 *
255 * This function recovers queue into the state before quiescing
256 * which is done by blk_mq_quiesce_queue.
257 */
blk_mq_unquiesce_queue(struct request_queue * q)258 void blk_mq_unquiesce_queue(struct request_queue *q)
259 {
260 unsigned long flags;
261 bool run_queue = false;
262
263 spin_lock_irqsave(&q->queue_lock, flags);
264 if (WARN_ON_ONCE(q->quiesce_depth <= 0)) {
265 ;
266 } else if (!--q->quiesce_depth) {
267 blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
268 run_queue = true;
269 }
270 spin_unlock_irqrestore(&q->queue_lock, flags);
271
272 /* dispatch requests which are inserted during quiescing */
273 if (run_queue)
274 blk_mq_run_hw_queues(q, true);
275 }
276 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);
277
blk_mq_quiesce_tagset(struct blk_mq_tag_set * set)278 void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
279 {
280 struct request_queue *q;
281
282 mutex_lock(&set->tag_list_lock);
283 list_for_each_entry(q, &set->tag_list, tag_set_list) {
284 if (!blk_queue_skip_tagset_quiesce(q))
285 blk_mq_quiesce_queue_nowait(q);
286 }
287 blk_mq_wait_quiesce_done(set);
288 mutex_unlock(&set->tag_list_lock);
289 }
290 EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset);
291
blk_mq_unquiesce_tagset(struct blk_mq_tag_set * set)292 void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
293 {
294 struct request_queue *q;
295
296 mutex_lock(&set->tag_list_lock);
297 list_for_each_entry(q, &set->tag_list, tag_set_list) {
298 if (!blk_queue_skip_tagset_quiesce(q))
299 blk_mq_unquiesce_queue(q);
300 }
301 mutex_unlock(&set->tag_list_lock);
302 }
303 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
304
blk_mq_wake_waiters(struct request_queue * q)305 void blk_mq_wake_waiters(struct request_queue *q)
306 {
307 struct blk_mq_hw_ctx *hctx;
308 unsigned long i;
309
310 queue_for_each_hw_ctx(q, hctx, i)
311 if (blk_mq_hw_queue_mapped(hctx))
312 blk_mq_tag_wakeup_all(hctx->tags, true);
313 }
314
blk_rq_init(struct request_queue * q,struct request * rq)315 void blk_rq_init(struct request_queue *q, struct request *rq)
316 {
317 memset(rq, 0, sizeof(*rq));
318
319 INIT_LIST_HEAD(&rq->queuelist);
320 rq->q = q;
321 rq->__sector = (sector_t) -1;
322 INIT_HLIST_NODE(&rq->hash);
323 RB_CLEAR_NODE(&rq->rb_node);
324 rq->tag = BLK_MQ_NO_TAG;
325 rq->internal_tag = BLK_MQ_NO_TAG;
326 rq->start_time_ns = ktime_get_ns();
327 rq->part = NULL;
328 blk_crypto_rq_set_defaults(rq);
329 }
330 EXPORT_SYMBOL(blk_rq_init);
331
332 /* Set start and alloc time when the allocated request is actually used */
blk_mq_rq_time_init(struct request * rq,u64 alloc_time_ns)333 static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns)
334 {
335 if (blk_mq_need_time_stamp(rq))
336 rq->start_time_ns = ktime_get_ns();
337 else
338 rq->start_time_ns = 0;
339
340 #ifdef CONFIG_BLK_RQ_ALLOC_TIME
341 if (blk_queue_rq_alloc_time(rq->q))
342 rq->alloc_time_ns = alloc_time_ns ?: rq->start_time_ns;
343 else
344 rq->alloc_time_ns = 0;
345 #endif
346 }
347
blk_mq_rq_ctx_init(struct blk_mq_alloc_data * data,struct blk_mq_tags * tags,unsigned int tag)348 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
349 struct blk_mq_tags *tags, unsigned int tag)
350 {
351 struct blk_mq_ctx *ctx = data->ctx;
352 struct blk_mq_hw_ctx *hctx = data->hctx;
353 struct request_queue *q = data->q;
354 struct request *rq = tags->static_rqs[tag];
355
356 rq->q = q;
357 rq->mq_ctx = ctx;
358 rq->mq_hctx = hctx;
359 rq->cmd_flags = data->cmd_flags;
360
361 if (data->flags & BLK_MQ_REQ_PM)
362 data->rq_flags |= RQF_PM;
363 if (blk_queue_io_stat(q))
364 data->rq_flags |= RQF_IO_STAT;
365 rq->rq_flags = data->rq_flags;
366
367 if (data->rq_flags & RQF_SCHED_TAGS) {
368 rq->tag = BLK_MQ_NO_TAG;
369 rq->internal_tag = tag;
370 } else {
371 rq->tag = tag;
372 rq->internal_tag = BLK_MQ_NO_TAG;
373 }
374 rq->timeout = 0;
375
376 rq->part = NULL;
377 rq->io_start_time_ns = 0;
378 rq->stats_sectors = 0;
379 rq->nr_phys_segments = 0;
380 #if defined(CONFIG_BLK_DEV_INTEGRITY)
381 rq->nr_integrity_segments = 0;
382 #endif
383 rq->end_io = NULL;
384 rq->end_io_data = NULL;
385
386 blk_crypto_rq_set_defaults(rq);
387 INIT_LIST_HEAD(&rq->queuelist);
388 /* tag was already set */
389 WRITE_ONCE(rq->deadline, 0);
390 req_ref_set(rq, 1);
391
392 if (rq->rq_flags & RQF_USE_SCHED) {
393 struct elevator_queue *e = data->q->elevator;
394
395 INIT_HLIST_NODE(&rq->hash);
396 RB_CLEAR_NODE(&rq->rb_node);
397
398 if (e->type->ops.prepare_request)
399 e->type->ops.prepare_request(rq);
400 }
401
402 return rq;
403 }
404
405 static inline struct request *
__blk_mq_alloc_requests_batch(struct blk_mq_alloc_data * data)406 __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data)
407 {
408 unsigned int tag, tag_offset;
409 struct blk_mq_tags *tags;
410 struct request *rq;
411 unsigned long tag_mask;
412 int i, nr = 0;
413
414 tag_mask = blk_mq_get_tags(data, data->nr_tags, &tag_offset);
415 if (unlikely(!tag_mask))
416 return NULL;
417
418 tags = blk_mq_tags_from_data(data);
419 for (i = 0; tag_mask; i++) {
420 if (!(tag_mask & (1UL << i)))
421 continue;
422 tag = tag_offset + i;
423 prefetch(tags->static_rqs[tag]);
424 tag_mask &= ~(1UL << i);
425 rq = blk_mq_rq_ctx_init(data, tags, tag);
426 rq_list_add(data->cached_rq, rq);
427 nr++;
428 }
429 /* caller already holds a reference, add for remainder */
430 percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);
431 data->nr_tags -= nr;
432
433 return rq_list_pop(data->cached_rq);
434 }
435
__blk_mq_alloc_requests(struct blk_mq_alloc_data * data)436 static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
437 {
438 struct request_queue *q = data->q;
439 u64 alloc_time_ns = 0;
440 struct request *rq;
441 unsigned int tag;
442
443 /* alloc_time includes depth and tag waits */
444 if (blk_queue_rq_alloc_time(q))
445 alloc_time_ns = ktime_get_ns();
446
447 if (data->cmd_flags & REQ_NOWAIT)
448 data->flags |= BLK_MQ_REQ_NOWAIT;
449
450 retry:
451 data->ctx = blk_mq_get_ctx(q);
452 data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
453
454 if (q->elevator) {
455 /*
456 * All requests use scheduler tags when an I/O scheduler is
457 * enabled for the queue.
458 */
459 data->rq_flags |= RQF_SCHED_TAGS;
460
461 /*
462 * Flush/passthrough requests are special and go directly to the
463 * dispatch list.
464 */
465 if ((data->cmd_flags & REQ_OP_MASK) != REQ_OP_FLUSH &&
466 !blk_op_is_passthrough(data->cmd_flags)) {
467 struct elevator_mq_ops *ops = &q->elevator->type->ops;
468
469 WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED);
470
471 data->rq_flags |= RQF_USE_SCHED;
472 if (ops->limit_depth)
473 ops->limit_depth(data->cmd_flags, data);
474 }
475 } else {
476 blk_mq_tag_busy(data->hctx);
477 }
478
479 if (data->flags & BLK_MQ_REQ_RESERVED)
480 data->rq_flags |= RQF_RESV;
481
482 /*
483 * Try batched alloc if we want more than 1 tag.
484 */
485 if (data->nr_tags > 1) {
486 rq = __blk_mq_alloc_requests_batch(data);
487 if (rq) {
488 blk_mq_rq_time_init(rq, alloc_time_ns);
489 return rq;
490 }
491 data->nr_tags = 1;
492 }
493
494 /*
495 * Waiting allocations only fail because of an inactive hctx. In that
496 * case just retry the hctx assignment and tag allocation as CPU hotplug
497 * should have migrated us to an online CPU by now.
498 */
499 tag = blk_mq_get_tag(data);
500 if (tag == BLK_MQ_NO_TAG) {
501 if (data->flags & BLK_MQ_REQ_NOWAIT)
502 return NULL;
503 /*
504 * Give up the CPU and sleep for a random short time to
505 * ensure that thread using a realtime scheduling class
506 * are migrated off the CPU, and thus off the hctx that
507 * is going away.
508 */
509 msleep(3);
510 goto retry;
511 }
512
513 rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag);
514 blk_mq_rq_time_init(rq, alloc_time_ns);
515 return rq;
516 }
517
blk_mq_rq_cache_fill(struct request_queue * q,struct blk_plug * plug,blk_opf_t opf,blk_mq_req_flags_t flags)518 static struct request *blk_mq_rq_cache_fill(struct request_queue *q,
519 struct blk_plug *plug,
520 blk_opf_t opf,
521 blk_mq_req_flags_t flags)
522 {
523 struct blk_mq_alloc_data data = {
524 .q = q,
525 .flags = flags,
526 .cmd_flags = opf,
527 .nr_tags = plug->nr_ios,
528 .cached_rq = &plug->cached_rq,
529 };
530 struct request *rq;
531
532 if (blk_queue_enter(q, flags))
533 return NULL;
534
535 plug->nr_ios = 1;
536
537 rq = __blk_mq_alloc_requests(&data);
538 if (unlikely(!rq))
539 blk_queue_exit(q);
540 return rq;
541 }
542
blk_mq_alloc_cached_request(struct request_queue * q,blk_opf_t opf,blk_mq_req_flags_t flags)543 static struct request *blk_mq_alloc_cached_request(struct request_queue *q,
544 blk_opf_t opf,
545 blk_mq_req_flags_t flags)
546 {
547 struct blk_plug *plug = current->plug;
548 struct request *rq;
549
550 if (!plug)
551 return NULL;
552
553 if (rq_list_empty(plug->cached_rq)) {
554 if (plug->nr_ios == 1)
555 return NULL;
556 rq = blk_mq_rq_cache_fill(q, plug, opf, flags);
557 if (!rq)
558 return NULL;
559 } else {
560 rq = rq_list_peek(&plug->cached_rq);
561 if (!rq || rq->q != q)
562 return NULL;
563
564 if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type)
565 return NULL;
566 if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
567 return NULL;
568
569 plug->cached_rq = rq_list_next(rq);
570 blk_mq_rq_time_init(rq, 0);
571 }
572
573 rq->cmd_flags = opf;
574 INIT_LIST_HEAD(&rq->queuelist);
575 return rq;
576 }
577
blk_mq_alloc_request(struct request_queue * q,blk_opf_t opf,blk_mq_req_flags_t flags)578 struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf,
579 blk_mq_req_flags_t flags)
580 {
581 struct request *rq;
582
583 rq = blk_mq_alloc_cached_request(q, opf, flags);
584 if (!rq) {
585 struct blk_mq_alloc_data data = {
586 .q = q,
587 .flags = flags,
588 .cmd_flags = opf,
589 .nr_tags = 1,
590 };
591 int ret;
592
593 ret = blk_queue_enter(q, flags);
594 if (ret)
595 return ERR_PTR(ret);
596
597 rq = __blk_mq_alloc_requests(&data);
598 if (!rq)
599 goto out_queue_exit;
600 }
601 rq->__data_len = 0;
602 rq->__sector = (sector_t) -1;
603 rq->bio = rq->biotail = NULL;
604 return rq;
605 out_queue_exit:
606 blk_queue_exit(q);
607 return ERR_PTR(-EWOULDBLOCK);
608 }
609 EXPORT_SYMBOL(blk_mq_alloc_request);
610
blk_mq_alloc_request_hctx(struct request_queue * q,blk_opf_t opf,blk_mq_req_flags_t flags,unsigned int hctx_idx)611 struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
612 blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx)
613 {
614 struct blk_mq_alloc_data data = {
615 .q = q,
616 .flags = flags,
617 .cmd_flags = opf,
618 .nr_tags = 1,
619 };
620 u64 alloc_time_ns = 0;
621 struct request *rq;
622 unsigned int cpu;
623 unsigned int tag;
624 int ret;
625
626 /* alloc_time includes depth and tag waits */
627 if (blk_queue_rq_alloc_time(q))
628 alloc_time_ns = ktime_get_ns();
629
630 /*
631 * If the tag allocator sleeps we could get an allocation for a
632 * different hardware context. No need to complicate the low level
633 * allocator for this for the rare use case of a command tied to
634 * a specific queue.
635 */
636 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)) ||
637 WARN_ON_ONCE(!(flags & BLK_MQ_REQ_RESERVED)))
638 return ERR_PTR(-EINVAL);
639
640 if (hctx_idx >= q->nr_hw_queues)
641 return ERR_PTR(-EIO);
642
643 ret = blk_queue_enter(q, flags);
644 if (ret)
645 return ERR_PTR(ret);
646
647 /*
648 * Check if the hardware context is actually mapped to anything.
649 * If not tell the caller that it should skip this queue.
650 */
651 ret = -EXDEV;
652 data.hctx = xa_load(&q->hctx_table, hctx_idx);
653 if (!blk_mq_hw_queue_mapped(data.hctx))
654 goto out_queue_exit;
655 cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
656 if (cpu >= nr_cpu_ids)
657 goto out_queue_exit;
658 data.ctx = __blk_mq_get_ctx(q, cpu);
659
660 if (q->elevator)
661 data.rq_flags |= RQF_SCHED_TAGS;
662 else
663 blk_mq_tag_busy(data.hctx);
664
665 if (flags & BLK_MQ_REQ_RESERVED)
666 data.rq_flags |= RQF_RESV;
667
668 ret = -EWOULDBLOCK;
669 tag = blk_mq_get_tag(&data);
670 if (tag == BLK_MQ_NO_TAG)
671 goto out_queue_exit;
672 rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag);
673 blk_mq_rq_time_init(rq, alloc_time_ns);
674 rq->__data_len = 0;
675 rq->__sector = (sector_t) -1;
676 rq->bio = rq->biotail = NULL;
677 return rq;
678
679 out_queue_exit:
680 blk_queue_exit(q);
681 return ERR_PTR(ret);
682 }
683 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
684
blk_mq_finish_request(struct request * rq)685 static void blk_mq_finish_request(struct request *rq)
686 {
687 struct request_queue *q = rq->q;
688
689 if (rq->rq_flags & RQF_USE_SCHED) {
690 q->elevator->type->ops.finish_request(rq);
691 /*
692 * For postflush request that may need to be
693 * completed twice, we should clear this flag
694 * to avoid double finish_request() on the rq.
695 */
696 rq->rq_flags &= ~RQF_USE_SCHED;
697 }
698 }
699
__blk_mq_free_request(struct request * rq)700 static void __blk_mq_free_request(struct request *rq)
701 {
702 struct request_queue *q = rq->q;
703 struct blk_mq_ctx *ctx = rq->mq_ctx;
704 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
705 const int sched_tag = rq->internal_tag;
706
707 blk_crypto_free_request(rq);
708 blk_pm_mark_last_busy(rq);
709 rq->mq_hctx = NULL;
710
711 if (rq->rq_flags & RQF_MQ_INFLIGHT)
712 __blk_mq_dec_active_requests(hctx);
713
714 if (rq->tag != BLK_MQ_NO_TAG)
715 blk_mq_put_tag(hctx->tags, ctx, rq->tag);
716 if (sched_tag != BLK_MQ_NO_TAG)
717 blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
718 blk_mq_sched_restart(hctx);
719 blk_queue_exit(q);
720 }
721
blk_mq_free_request(struct request * rq)722 void blk_mq_free_request(struct request *rq)
723 {
724 struct request_queue *q = rq->q;
725
726 blk_mq_finish_request(rq);
727
728 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
729 laptop_io_completion(q->disk->bdi);
730
731 rq_qos_done(q, rq);
732
733 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
734 if (req_ref_put_and_test(rq))
735 __blk_mq_free_request(rq);
736 }
737 EXPORT_SYMBOL_GPL(blk_mq_free_request);
738
blk_mq_free_plug_rqs(struct blk_plug * plug)739 void blk_mq_free_plug_rqs(struct blk_plug *plug)
740 {
741 struct request *rq;
742
743 while ((rq = rq_list_pop(&plug->cached_rq)) != NULL)
744 blk_mq_free_request(rq);
745 }
746
blk_dump_rq_flags(struct request * rq,char * msg)747 void blk_dump_rq_flags(struct request *rq, char *msg)
748 {
749 printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
750 rq->q->disk ? rq->q->disk->disk_name : "?",
751 (__force unsigned long long) rq->cmd_flags);
752
753 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
754 (unsigned long long)blk_rq_pos(rq),
755 blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
756 printk(KERN_INFO " bio %p, biotail %p, len %u\n",
757 rq->bio, rq->biotail, blk_rq_bytes(rq));
758 }
759 EXPORT_SYMBOL(blk_dump_rq_flags);
760
req_bio_endio(struct request * rq,struct bio * bio,unsigned int nbytes,blk_status_t error)761 static void req_bio_endio(struct request *rq, struct bio *bio,
762 unsigned int nbytes, blk_status_t error)
763 {
764 if (unlikely(error)) {
765 bio->bi_status = error;
766 } else if (req_op(rq) == REQ_OP_ZONE_APPEND) {
767 /*
768 * Partial zone append completions cannot be supported as the
769 * BIO fragments may end up not being written sequentially.
770 */
771 if (bio->bi_iter.bi_size != nbytes)
772 bio->bi_status = BLK_STS_IOERR;
773 else
774 bio->bi_iter.bi_sector = rq->__sector;
775 }
776
777 bio_advance(bio, nbytes);
778
779 if (unlikely(rq->rq_flags & RQF_QUIET))
780 bio_set_flag(bio, BIO_QUIET);
781 /* don't actually finish bio if it's part of flush sequence */
782 if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
783 bio_endio(bio);
784 }
785
blk_account_io_completion(struct request * req,unsigned int bytes)786 static void blk_account_io_completion(struct request *req, unsigned int bytes)
787 {
788 if (req->part && blk_do_io_stat(req)) {
789 const int sgrp = op_stat_group(req_op(req));
790
791 part_stat_lock();
792 part_stat_add(req->part, sectors[sgrp], bytes >> 9);
793 part_stat_unlock();
794 }
795 }
796
blk_print_req_error(struct request * req,blk_status_t status)797 static void blk_print_req_error(struct request *req, blk_status_t status)
798 {
799 printk_ratelimited(KERN_ERR
800 "%s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
801 "phys_seg %u prio class %u\n",
802 blk_status_to_str(status),
803 req->q->disk ? req->q->disk->disk_name : "?",
804 blk_rq_pos(req), (__force u32)req_op(req),
805 blk_op_str(req_op(req)),
806 (__force u32)(req->cmd_flags & ~REQ_OP_MASK),
807 req->nr_phys_segments,
808 IOPRIO_PRIO_CLASS(req->ioprio));
809 }
810
811 /*
812 * Fully end IO on a request. Does not support partial completions, or
813 * errors.
814 */
blk_complete_request(struct request * req)815 static void blk_complete_request(struct request *req)
816 {
817 const bool is_flush = (req->rq_flags & RQF_FLUSH_SEQ) != 0;
818 int total_bytes = blk_rq_bytes(req);
819 struct bio *bio = req->bio;
820
821 trace_block_rq_complete(req, BLK_STS_OK, total_bytes);
822
823 if (!bio)
824 return;
825
826 #ifdef CONFIG_BLK_DEV_INTEGRITY
827 if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ)
828 req->q->integrity.profile->complete_fn(req, total_bytes);
829 #endif
830
831 /*
832 * Upper layers may call blk_crypto_evict_key() anytime after the last
833 * bio_endio(). Therefore, the keyslot must be released before that.
834 */
835 blk_crypto_rq_put_keyslot(req);
836
837 blk_account_io_completion(req, total_bytes);
838
839 do {
840 struct bio *next = bio->bi_next;
841
842 /* Completion has already been traced */
843 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
844
845 if (req_op(req) == REQ_OP_ZONE_APPEND)
846 bio->bi_iter.bi_sector = req->__sector;
847
848 if (!is_flush)
849 bio_endio(bio);
850 bio = next;
851 } while (bio);
852
853 /*
854 * Reset counters so that the request stacking driver
855 * can find how many bytes remain in the request
856 * later.
857 */
858 if (!req->end_io) {
859 req->bio = NULL;
860 req->__data_len = 0;
861 }
862 }
863
864 /**
865 * blk_update_request - Complete multiple bytes without completing the request
866 * @req: the request being processed
867 * @error: block status code
868 * @nr_bytes: number of bytes to complete for @req
869 *
870 * Description:
871 * Ends I/O on a number of bytes attached to @req, but doesn't complete
872 * the request structure even if @req doesn't have leftover.
873 * If @req has leftover, sets it up for the next range of segments.
874 *
875 * Passing the result of blk_rq_bytes() as @nr_bytes guarantees
876 * %false return from this function.
877 *
878 * Note:
879 * The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
880 * except in the consistency check at the end of this function.
881 *
882 * Return:
883 * %false - this request doesn't have any more data
884 * %true - this request has more data
885 **/
blk_update_request(struct request * req,blk_status_t error,unsigned int nr_bytes)886 bool blk_update_request(struct request *req, blk_status_t error,
887 unsigned int nr_bytes)
888 {
889 int total_bytes;
890
891 trace_block_rq_complete(req, error, nr_bytes);
892
893 if (!req->bio)
894 return false;
895
896 #ifdef CONFIG_BLK_DEV_INTEGRITY
897 if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
898 error == BLK_STS_OK)
899 req->q->integrity.profile->complete_fn(req, nr_bytes);
900 #endif
901
902 /*
903 * Upper layers may call blk_crypto_evict_key() anytime after the last
904 * bio_endio(). Therefore, the keyslot must be released before that.
905 */
906 if (blk_crypto_rq_has_keyslot(req) && nr_bytes >= blk_rq_bytes(req))
907 __blk_crypto_rq_put_keyslot(req);
908
909 if (unlikely(error && !blk_rq_is_passthrough(req) &&
910 !(req->rq_flags & RQF_QUIET)) &&
911 !test_bit(GD_DEAD, &req->q->disk->state)) {
912 blk_print_req_error(req, error);
913 trace_block_rq_error(req, error, nr_bytes);
914 }
915
916 blk_account_io_completion(req, nr_bytes);
917
918 total_bytes = 0;
919 while (req->bio) {
920 struct bio *bio = req->bio;
921 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
922
923 if (bio_bytes == bio->bi_iter.bi_size)
924 req->bio = bio->bi_next;
925
926 /* Completion has already been traced */
927 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
928 req_bio_endio(req, bio, bio_bytes, error);
929
930 total_bytes += bio_bytes;
931 nr_bytes -= bio_bytes;
932
933 if (!nr_bytes)
934 break;
935 }
936
937 /*
938 * completely done
939 */
940 if (!req->bio) {
941 /*
942 * Reset counters so that the request stacking driver
943 * can find how many bytes remain in the request
944 * later.
945 */
946 req->__data_len = 0;
947 return false;
948 }
949
950 req->__data_len -= total_bytes;
951
952 /* update sector only for requests with clear definition of sector */
953 if (!blk_rq_is_passthrough(req))
954 req->__sector += total_bytes >> 9;
955
956 /* mixed attributes always follow the first bio */
957 if (req->rq_flags & RQF_MIXED_MERGE) {
958 req->cmd_flags &= ~REQ_FAILFAST_MASK;
959 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
960 }
961
962 if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
963 /*
964 * If total number of sectors is less than the first segment
965 * size, something has gone terribly wrong.
966 */
967 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
968 blk_dump_rq_flags(req, "request botched");
969 req->__data_len = blk_rq_cur_bytes(req);
970 }
971
972 /* recalculate the number of segments */
973 req->nr_phys_segments = blk_recalc_rq_segments(req);
974 }
975
976 return true;
977 }
978 EXPORT_SYMBOL_GPL(blk_update_request);
979
blk_account_io_done(struct request * req,u64 now)980 static inline void blk_account_io_done(struct request *req, u64 now)
981 {
982 trace_block_io_done(req);
983
984 /*
985 * Account IO completion. flush_rq isn't accounted as a
986 * normal IO on queueing nor completion. Accounting the
987 * containing request is enough.
988 */
989 if (blk_do_io_stat(req) && req->part &&
990 !(req->rq_flags & RQF_FLUSH_SEQ)) {
991 const int sgrp = op_stat_group(req_op(req));
992
993 part_stat_lock();
994 update_io_ticks(req->part, jiffies, true);
995 part_stat_inc(req->part, ios[sgrp]);
996 part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns);
997 part_stat_local_dec(req->part,
998 in_flight[op_is_write(req_op(req))]);
999 part_stat_unlock();
1000 }
1001 }
1002
blk_account_io_start(struct request * req)1003 static inline void blk_account_io_start(struct request *req)
1004 {
1005 trace_block_io_start(req);
1006
1007 if (blk_do_io_stat(req)) {
1008 /*
1009 * All non-passthrough requests are created from a bio with one
1010 * exception: when a flush command that is part of a flush sequence
1011 * generated by the state machine in blk-flush.c is cloned onto the
1012 * lower device by dm-multipath we can get here without a bio.
1013 */
1014 if (req->bio)
1015 req->part = req->bio->bi_bdev;
1016 else
1017 req->part = req->q->disk->part0;
1018
1019 part_stat_lock();
1020 update_io_ticks(req->part, jiffies, false);
1021 part_stat_local_inc(req->part,
1022 in_flight[op_is_write(req_op(req))]);
1023 part_stat_unlock();
1024 }
1025 }
1026
__blk_mq_end_request_acct(struct request * rq,u64 now)1027 static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)
1028 {
1029 if (rq->rq_flags & RQF_STATS)
1030 blk_stat_add(rq, now);
1031
1032 blk_mq_sched_completed_request(rq, now);
1033 blk_account_io_done(rq, now);
1034 }
1035
__blk_mq_end_request(struct request * rq,blk_status_t error)1036 inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
1037 {
1038 if (blk_mq_need_time_stamp(rq))
1039 __blk_mq_end_request_acct(rq, ktime_get_ns());
1040
1041 blk_mq_finish_request(rq);
1042
1043 if (rq->end_io) {
1044 rq_qos_done(rq->q, rq);
1045 if (rq->end_io(rq, error) == RQ_END_IO_FREE)
1046 blk_mq_free_request(rq);
1047 } else {
1048 blk_mq_free_request(rq);
1049 }
1050 }
1051 EXPORT_SYMBOL(__blk_mq_end_request);
1052
blk_mq_end_request(struct request * rq,blk_status_t error)1053 void blk_mq_end_request(struct request *rq, blk_status_t error)
1054 {
1055 if (blk_update_request(rq, error, blk_rq_bytes(rq)))
1056 BUG();
1057 __blk_mq_end_request(rq, error);
1058 }
1059 EXPORT_SYMBOL(blk_mq_end_request);
1060
1061 #define TAG_COMP_BATCH 32
1062
blk_mq_flush_tag_batch(struct blk_mq_hw_ctx * hctx,int * tag_array,int nr_tags)1063 static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,
1064 int *tag_array, int nr_tags)
1065 {
1066 struct request_queue *q = hctx->queue;
1067
1068 /*
1069 * All requests should have been marked as RQF_MQ_INFLIGHT, so
1070 * update hctx->nr_active in batch
1071 */
1072 if (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
1073 __blk_mq_sub_active_requests(hctx, nr_tags);
1074
1075 blk_mq_put_tags(hctx->tags, tag_array, nr_tags);
1076 percpu_ref_put_many(&q->q_usage_counter, nr_tags);
1077 }
1078
blk_mq_end_request_batch(struct io_comp_batch * iob)1079 void blk_mq_end_request_batch(struct io_comp_batch *iob)
1080 {
1081 int tags[TAG_COMP_BATCH], nr_tags = 0;
1082 struct blk_mq_hw_ctx *cur_hctx = NULL;
1083 struct request *rq;
1084 u64 now = 0;
1085
1086 if (iob->need_ts)
1087 now = ktime_get_ns();
1088
1089 while ((rq = rq_list_pop(&iob->req_list)) != NULL) {
1090 prefetch(rq->bio);
1091 prefetch(rq->rq_next);
1092
1093 blk_complete_request(rq);
1094 if (iob->need_ts)
1095 __blk_mq_end_request_acct(rq, now);
1096
1097 blk_mq_finish_request(rq);
1098
1099 rq_qos_done(rq->q, rq);
1100
1101 /*
1102 * If end_io handler returns NONE, then it still has
1103 * ownership of the request.
1104 */
1105 if (rq->end_io && rq->end_io(rq, 0) == RQ_END_IO_NONE)
1106 continue;
1107
1108 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
1109 if (!req_ref_put_and_test(rq))
1110 continue;
1111
1112 blk_crypto_free_request(rq);
1113 blk_pm_mark_last_busy(rq);
1114
1115 if (nr_tags == TAG_COMP_BATCH || cur_hctx != rq->mq_hctx) {
1116 if (cur_hctx)
1117 blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
1118 nr_tags = 0;
1119 cur_hctx = rq->mq_hctx;
1120 }
1121 tags[nr_tags++] = rq->tag;
1122 }
1123
1124 if (nr_tags)
1125 blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
1126 }
1127 EXPORT_SYMBOL_GPL(blk_mq_end_request_batch);
1128
blk_complete_reqs(struct llist_head * list)1129 static void blk_complete_reqs(struct llist_head *list)
1130 {
1131 struct llist_node *entry = llist_reverse_order(llist_del_all(list));
1132 struct request *rq, *next;
1133
1134 llist_for_each_entry_safe(rq, next, entry, ipi_list)
1135 rq->q->mq_ops->complete(rq);
1136 }
1137
blk_done_softirq(struct softirq_action * h)1138 static __latent_entropy void blk_done_softirq(struct softirq_action *h)
1139 {
1140 blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
1141 }
1142
blk_softirq_cpu_dead(unsigned int cpu)1143 static int blk_softirq_cpu_dead(unsigned int cpu)
1144 {
1145 blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
1146 return 0;
1147 }
1148
__blk_mq_complete_request_remote(void * data)1149 static void __blk_mq_complete_request_remote(void *data)
1150 {
1151 __raise_softirq_irqoff(BLOCK_SOFTIRQ);
1152 }
1153
blk_mq_complete_need_ipi(struct request * rq)1154 static inline bool blk_mq_complete_need_ipi(struct request *rq)
1155 {
1156 int cpu = raw_smp_processor_id();
1157
1158 if (!IS_ENABLED(CONFIG_SMP) ||
1159 !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
1160 return false;
1161 /*
1162 * With force threaded interrupts enabled, raising softirq from an SMP
1163 * function call will always result in waking the ksoftirqd thread.
1164 * This is probably worse than completing the request on a different
1165 * cache domain.
1166 */
1167 if (force_irqthreads())
1168 return false;
1169
1170 /* same CPU or cache domain? Complete locally */
1171 if (cpu == rq->mq_ctx->cpu ||
1172 (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
1173 cpus_share_cache(cpu, rq->mq_ctx->cpu)))
1174 return false;
1175
1176 /* don't try to IPI to an offline CPU */
1177 return cpu_online(rq->mq_ctx->cpu);
1178 }
1179
blk_mq_complete_send_ipi(struct request * rq)1180 static void blk_mq_complete_send_ipi(struct request *rq)
1181 {
1182 unsigned int cpu;
1183
1184 cpu = rq->mq_ctx->cpu;
1185 if (llist_add(&rq->ipi_list, &per_cpu(blk_cpu_done, cpu)))
1186 smp_call_function_single_async(cpu, &per_cpu(blk_cpu_csd, cpu));
1187 }
1188
blk_mq_raise_softirq(struct request * rq)1189 static void blk_mq_raise_softirq(struct request *rq)
1190 {
1191 struct llist_head *list;
1192
1193 preempt_disable();
1194 list = this_cpu_ptr(&blk_cpu_done);
1195 if (llist_add(&rq->ipi_list, list))
1196 raise_softirq(BLOCK_SOFTIRQ);
1197 preempt_enable();
1198 }
1199
blk_mq_complete_request_remote(struct request * rq)1200 bool blk_mq_complete_request_remote(struct request *rq)
1201 {
1202 WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
1203
1204 /*
1205 * For request which hctx has only one ctx mapping,
1206 * or a polled request, always complete locally,
1207 * it's pointless to redirect the completion.
1208 */
1209 if ((rq->mq_hctx->nr_ctx == 1 &&
1210 rq->mq_ctx->cpu == raw_smp_processor_id()) ||
1211 rq->cmd_flags & REQ_POLLED)
1212 return false;
1213
1214 if (blk_mq_complete_need_ipi(rq)) {
1215 blk_mq_complete_send_ipi(rq);
1216 return true;
1217 }
1218
1219 if (rq->q->nr_hw_queues == 1) {
1220 blk_mq_raise_softirq(rq);
1221 return true;
1222 }
1223 return false;
1224 }
1225 EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);
1226
1227 /**
1228 * blk_mq_complete_request - end I/O on a request
1229 * @rq: the request being processed
1230 *
1231 * Description:
1232 * Complete a request by scheduling the ->complete_rq operation.
1233 **/
blk_mq_complete_request(struct request * rq)1234 void blk_mq_complete_request(struct request *rq)
1235 {
1236 if (!blk_mq_complete_request_remote(rq))
1237 rq->q->mq_ops->complete(rq);
1238 }
1239 EXPORT_SYMBOL(blk_mq_complete_request);
1240
1241 /**
1242 * blk_mq_start_request - Start processing a request
1243 * @rq: Pointer to request to be started
1244 *
1245 * Function used by device drivers to notify the block layer that a request
1246 * is going to be processed now, so blk layer can do proper initializations
1247 * such as starting the timeout timer.
1248 */
blk_mq_start_request(struct request * rq)1249 void blk_mq_start_request(struct request *rq)
1250 {
1251 struct request_queue *q = rq->q;
1252
1253 trace_block_rq_issue(rq);
1254
1255 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
1256 rq->io_start_time_ns = ktime_get_ns();
1257 rq->stats_sectors = blk_rq_sectors(rq);
1258 rq->rq_flags |= RQF_STATS;
1259 rq_qos_issue(q, rq);
1260 }
1261
1262 WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
1263
1264 blk_add_timer(rq);
1265 WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
1266
1267 #ifdef CONFIG_BLK_DEV_INTEGRITY
1268 if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE)
1269 q->integrity.profile->prepare_fn(rq);
1270 #endif
1271 if (rq->bio && rq->bio->bi_opf & REQ_POLLED)
1272 WRITE_ONCE(rq->bio->bi_cookie, rq->mq_hctx->queue_num);
1273 }
1274 EXPORT_SYMBOL(blk_mq_start_request);
1275
1276 /*
1277 * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
1278 * queues. This is important for md arrays to benefit from merging
1279 * requests.
1280 */
blk_plug_max_rq_count(struct blk_plug * plug)1281 static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug)
1282 {
1283 if (plug->multiple_queues)
1284 return BLK_MAX_REQUEST_COUNT * 2;
1285 return BLK_MAX_REQUEST_COUNT;
1286 }
1287
blk_add_rq_to_plug(struct blk_plug * plug,struct request * rq)1288 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
1289 {
1290 struct request *last = rq_list_peek(&plug->mq_list);
1291
1292 if (!plug->rq_count) {
1293 trace_block_plug(rq->q);
1294 } else if (plug->rq_count >= blk_plug_max_rq_count(plug) ||
1295 (!blk_queue_nomerges(rq->q) &&
1296 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1297 blk_mq_flush_plug_list(plug, false);
1298 last = NULL;
1299 trace_block_plug(rq->q);
1300 }
1301
1302 if (!plug->multiple_queues && last && last->q != rq->q)
1303 plug->multiple_queues = true;
1304 /*
1305 * Any request allocated from sched tags can't be issued to
1306 * ->queue_rqs() directly
1307 */
1308 if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS))
1309 plug->has_elevator = true;
1310 rq->rq_next = NULL;
1311 rq_list_add(&plug->mq_list, rq);
1312 plug->rq_count++;
1313 }
1314
1315 /**
1316 * blk_execute_rq_nowait - insert a request to I/O scheduler for execution
1317 * @rq: request to insert
1318 * @at_head: insert request at head or tail of queue
1319 *
1320 * Description:
1321 * Insert a fully prepared request at the back of the I/O scheduler queue
1322 * for execution. Don't wait for completion.
1323 *
1324 * Note:
1325 * This function will invoke @done directly if the queue is dead.
1326 */
blk_execute_rq_nowait(struct request * rq,bool at_head)1327 void blk_execute_rq_nowait(struct request *rq, bool at_head)
1328 {
1329 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1330
1331 WARN_ON(irqs_disabled());
1332 WARN_ON(!blk_rq_is_passthrough(rq));
1333
1334 blk_account_io_start(rq);
1335
1336 /*
1337 * As plugging can be enabled for passthrough requests on a zoned
1338 * device, directly accessing the plug instead of using blk_mq_plug()
1339 * should not have any consequences.
1340 */
1341 if (current->plug && !at_head) {
1342 blk_add_rq_to_plug(current->plug, rq);
1343 return;
1344 }
1345
1346 blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
1347 blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
1348 }
1349 EXPORT_SYMBOL_GPL(blk_execute_rq_nowait);
1350
1351 struct blk_rq_wait {
1352 struct completion done;
1353 blk_status_t ret;
1354 };
1355
blk_end_sync_rq(struct request * rq,blk_status_t ret)1356 static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret)
1357 {
1358 struct blk_rq_wait *wait = rq->end_io_data;
1359
1360 wait->ret = ret;
1361 complete(&wait->done);
1362 return RQ_END_IO_NONE;
1363 }
1364
blk_rq_is_poll(struct request * rq)1365 bool blk_rq_is_poll(struct request *rq)
1366 {
1367 if (!rq->mq_hctx)
1368 return false;
1369 if (rq->mq_hctx->type != HCTX_TYPE_POLL)
1370 return false;
1371 return true;
1372 }
1373 EXPORT_SYMBOL_GPL(blk_rq_is_poll);
1374
blk_rq_poll_completion(struct request * rq,struct completion * wait)1375 static void blk_rq_poll_completion(struct request *rq, struct completion *wait)
1376 {
1377 do {
1378 blk_hctx_poll(rq->q, rq->mq_hctx, NULL, 0);
1379 cond_resched();
1380 } while (!completion_done(wait));
1381 }
1382
1383 /**
1384 * blk_execute_rq - insert a request into queue for execution
1385 * @rq: request to insert
1386 * @at_head: insert request at head or tail of queue
1387 *
1388 * Description:
1389 * Insert a fully prepared request at the back of the I/O scheduler queue
1390 * for execution and wait for completion.
1391 * Return: The blk_status_t result provided to blk_mq_end_request().
1392 */
blk_execute_rq(struct request * rq,bool at_head)1393 blk_status_t blk_execute_rq(struct request *rq, bool at_head)
1394 {
1395 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1396 struct blk_rq_wait wait = {
1397 .done = COMPLETION_INITIALIZER_ONSTACK(wait.done),
1398 };
1399
1400 WARN_ON(irqs_disabled());
1401 WARN_ON(!blk_rq_is_passthrough(rq));
1402
1403 rq->end_io_data = &wait;
1404 rq->end_io = blk_end_sync_rq;
1405
1406 blk_account_io_start(rq);
1407 blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
1408 blk_mq_run_hw_queue(hctx, false);
1409
1410 if (blk_rq_is_poll(rq)) {
1411 blk_rq_poll_completion(rq, &wait.done);
1412 } else {
1413 /*
1414 * Prevent hang_check timer from firing at us during very long
1415 * I/O
1416 */
1417 unsigned long hang_check = sysctl_hung_task_timeout_secs;
1418
1419 if (hang_check)
1420 while (!wait_for_completion_io_timeout(&wait.done,
1421 hang_check * (HZ/2)))
1422 ;
1423 else
1424 wait_for_completion_io(&wait.done);
1425 }
1426
1427 return wait.ret;
1428 }
1429 EXPORT_SYMBOL(blk_execute_rq);
1430
__blk_mq_requeue_request(struct request * rq)1431 static void __blk_mq_requeue_request(struct request *rq)
1432 {
1433 struct request_queue *q = rq->q;
1434
1435 blk_mq_put_driver_tag(rq);
1436
1437 trace_block_rq_requeue(rq);
1438 rq_qos_requeue(q, rq);
1439
1440 if (blk_mq_request_started(rq)) {
1441 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
1442 rq->rq_flags &= ~RQF_TIMED_OUT;
1443 }
1444 }
1445
blk_mq_requeue_request(struct request * rq,bool kick_requeue_list)1446 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
1447 {
1448 struct request_queue *q = rq->q;
1449 unsigned long flags;
1450
1451 __blk_mq_requeue_request(rq);
1452
1453 /* this request will be re-inserted to io scheduler queue */
1454 blk_mq_sched_requeue_request(rq);
1455
1456 spin_lock_irqsave(&q->requeue_lock, flags);
1457 list_add_tail(&rq->queuelist, &q->requeue_list);
1458 spin_unlock_irqrestore(&q->requeue_lock, flags);
1459
1460 if (kick_requeue_list)
1461 blk_mq_kick_requeue_list(q);
1462 }
1463 EXPORT_SYMBOL(blk_mq_requeue_request);
1464
blk_mq_requeue_work(struct work_struct * work)1465 static void blk_mq_requeue_work(struct work_struct *work)
1466 {
1467 struct request_queue *q =
1468 container_of(work, struct request_queue, requeue_work.work);
1469 LIST_HEAD(rq_list);
1470 LIST_HEAD(flush_list);
1471 struct request *rq;
1472
1473 spin_lock_irq(&q->requeue_lock);
1474 list_splice_init(&q->requeue_list, &rq_list);
1475 list_splice_init(&q->flush_list, &flush_list);
1476 spin_unlock_irq(&q->requeue_lock);
1477
1478 while (!list_empty(&rq_list)) {
1479 rq = list_entry(rq_list.next, struct request, queuelist);
1480 /*
1481 * If RQF_DONTPREP ist set, the request has been started by the
1482 * driver already and might have driver-specific data allocated
1483 * already. Insert it into the hctx dispatch list to avoid
1484 * block layer merges for the request.
1485 */
1486 if (rq->rq_flags & RQF_DONTPREP) {
1487 list_del_init(&rq->queuelist);
1488 blk_mq_request_bypass_insert(rq, 0);
1489 } else {
1490 list_del_init(&rq->queuelist);
1491 blk_mq_insert_request(rq, BLK_MQ_INSERT_AT_HEAD);
1492 }
1493 }
1494
1495 while (!list_empty(&flush_list)) {
1496 rq = list_entry(flush_list.next, struct request, queuelist);
1497 list_del_init(&rq->queuelist);
1498 blk_mq_insert_request(rq, 0);
1499 }
1500
1501 blk_mq_run_hw_queues(q, false);
1502 }
1503
blk_mq_kick_requeue_list(struct request_queue * q)1504 void blk_mq_kick_requeue_list(struct request_queue *q)
1505 {
1506 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
1507 }
1508 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
1509
blk_mq_delay_kick_requeue_list(struct request_queue * q,unsigned long msecs)1510 void blk_mq_delay_kick_requeue_list(struct request_queue *q,
1511 unsigned long msecs)
1512 {
1513 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
1514 msecs_to_jiffies(msecs));
1515 }
1516 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
1517
blk_is_flush_data_rq(struct request * rq)1518 static bool blk_is_flush_data_rq(struct request *rq)
1519 {
1520 return (rq->rq_flags & RQF_FLUSH_SEQ) && !is_flush_rq(rq);
1521 }
1522
blk_mq_rq_inflight(struct request * rq,void * priv)1523 static bool blk_mq_rq_inflight(struct request *rq, void *priv)
1524 {
1525 /*
1526 * If we find a request that isn't idle we know the queue is busy
1527 * as it's checked in the iter.
1528 * Return false to stop the iteration.
1529 *
1530 * In case of queue quiesce, if one flush data request is completed,
1531 * don't count it as inflight given the flush sequence is suspended,
1532 * and the original flush data request is invisible to driver, just
1533 * like other pending requests because of quiesce
1534 */
1535 if (blk_mq_request_started(rq) && !(blk_queue_quiesced(rq->q) &&
1536 blk_is_flush_data_rq(rq) &&
1537 blk_mq_request_completed(rq))) {
1538 bool *busy = priv;
1539
1540 *busy = true;
1541 return false;
1542 }
1543
1544 return true;
1545 }
1546
blk_mq_queue_inflight(struct request_queue * q)1547 bool blk_mq_queue_inflight(struct request_queue *q)
1548 {
1549 bool busy = false;
1550
1551 blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
1552 return busy;
1553 }
1554 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
1555
blk_mq_rq_timed_out(struct request * req)1556 static void blk_mq_rq_timed_out(struct request *req)
1557 {
1558 req->rq_flags |= RQF_TIMED_OUT;
1559 if (req->q->mq_ops->timeout) {
1560 enum blk_eh_timer_return ret;
1561
1562 ret = req->q->mq_ops->timeout(req);
1563 if (ret == BLK_EH_DONE)
1564 return;
1565 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
1566 }
1567
1568 blk_add_timer(req);
1569 }
1570
1571 struct blk_expired_data {
1572 bool has_timedout_rq;
1573 unsigned long next;
1574 unsigned long timeout_start;
1575 };
1576
blk_mq_req_expired(struct request * rq,struct blk_expired_data * expired)1577 static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expired)
1578 {
1579 unsigned long deadline;
1580
1581 if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
1582 return false;
1583 if (rq->rq_flags & RQF_TIMED_OUT)
1584 return false;
1585
1586 deadline = READ_ONCE(rq->deadline);
1587 if (time_after_eq(expired->timeout_start, deadline))
1588 return true;
1589
1590 if (expired->next == 0)
1591 expired->next = deadline;
1592 else if (time_after(expired->next, deadline))
1593 expired->next = deadline;
1594 return false;
1595 }
1596
blk_mq_put_rq_ref(struct request * rq)1597 void blk_mq_put_rq_ref(struct request *rq)
1598 {
1599 if (is_flush_rq(rq)) {
1600 if (rq->end_io(rq, 0) == RQ_END_IO_FREE)
1601 blk_mq_free_request(rq);
1602 } else if (req_ref_put_and_test(rq)) {
1603 __blk_mq_free_request(rq);
1604 }
1605 }
1606
blk_mq_check_expired(struct request * rq,void * priv)1607 static bool blk_mq_check_expired(struct request *rq, void *priv)
1608 {
1609 struct blk_expired_data *expired = priv;
1610
1611 /*
1612 * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot
1613 * be reallocated underneath the timeout handler's processing, then
1614 * the expire check is reliable. If the request is not expired, then
1615 * it was completed and reallocated as a new request after returning
1616 * from blk_mq_check_expired().
1617 */
1618 if (blk_mq_req_expired(rq, expired)) {
1619 expired->has_timedout_rq = true;
1620 return false;
1621 }
1622 return true;
1623 }
1624
blk_mq_handle_expired(struct request * rq,void * priv)1625 static bool blk_mq_handle_expired(struct request *rq, void *priv)
1626 {
1627 struct blk_expired_data *expired = priv;
1628
1629 if (blk_mq_req_expired(rq, expired))
1630 blk_mq_rq_timed_out(rq);
1631 return true;
1632 }
1633
blk_mq_timeout_work(struct work_struct * work)1634 static void blk_mq_timeout_work(struct work_struct *work)
1635 {
1636 struct request_queue *q =
1637 container_of(work, struct request_queue, timeout_work);
1638 struct blk_expired_data expired = {
1639 .timeout_start = jiffies,
1640 };
1641 struct blk_mq_hw_ctx *hctx;
1642 unsigned long i;
1643
1644 /* A deadlock might occur if a request is stuck requiring a
1645 * timeout at the same time a queue freeze is waiting
1646 * completion, since the timeout code would not be able to
1647 * acquire the queue reference here.
1648 *
1649 * That's why we don't use blk_queue_enter here; instead, we use
1650 * percpu_ref_tryget directly, because we need to be able to
1651 * obtain a reference even in the short window between the queue
1652 * starting to freeze, by dropping the first reference in
1653 * blk_freeze_queue_start, and the moment the last request is
1654 * consumed, marked by the instant q_usage_counter reaches
1655 * zero.
1656 */
1657 if (!percpu_ref_tryget(&q->q_usage_counter))
1658 return;
1659
1660 /* check if there is any timed-out request */
1661 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);
1662 if (expired.has_timedout_rq) {
1663 /*
1664 * Before walking tags, we must ensure any submit started
1665 * before the current time has finished. Since the submit
1666 * uses srcu or rcu, wait for a synchronization point to
1667 * ensure all running submits have finished
1668 */
1669 blk_mq_wait_quiesce_done(q->tag_set);
1670
1671 expired.next = 0;
1672 blk_mq_queue_tag_busy_iter(q, blk_mq_handle_expired, &expired);
1673 }
1674
1675 if (expired.next != 0) {
1676 mod_timer(&q->timeout, expired.next);
1677 } else {
1678 /*
1679 * Request timeouts are handled as a forward rolling timer. If
1680 * we end up here it means that no requests are pending and
1681 * also that no request has been pending for a while. Mark
1682 * each hctx as idle.
1683 */
1684 queue_for_each_hw_ctx(q, hctx, i) {
1685 /* the hctx may be unmapped, so check it here */
1686 if (blk_mq_hw_queue_mapped(hctx))
1687 blk_mq_tag_idle(hctx);
1688 }
1689 }
1690 blk_queue_exit(q);
1691 }
1692
1693 struct flush_busy_ctx_data {
1694 struct blk_mq_hw_ctx *hctx;
1695 struct list_head *list;
1696 };
1697
flush_busy_ctx(struct sbitmap * sb,unsigned int bitnr,void * data)1698 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
1699 {
1700 struct flush_busy_ctx_data *flush_data = data;
1701 struct blk_mq_hw_ctx *hctx = flush_data->hctx;
1702 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
1703 enum hctx_type type = hctx->type;
1704
1705 spin_lock(&ctx->lock);
1706 list_splice_tail_init(&ctx->rq_lists[type], flush_data->list);
1707 sbitmap_clear_bit(sb, bitnr);
1708 spin_unlock(&ctx->lock);
1709 return true;
1710 }
1711
1712 /*
1713 * Process software queues that have been marked busy, splicing them
1714 * to the for-dispatch
1715 */
blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx * hctx,struct list_head * list)1716 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
1717 {
1718 struct flush_busy_ctx_data data = {
1719 .hctx = hctx,
1720 .list = list,
1721 };
1722
1723 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
1724 }
1725 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
1726
1727 struct dispatch_rq_data {
1728 struct blk_mq_hw_ctx *hctx;
1729 struct request *rq;
1730 };
1731
dispatch_rq_from_ctx(struct sbitmap * sb,unsigned int bitnr,void * data)1732 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
1733 void *data)
1734 {
1735 struct dispatch_rq_data *dispatch_data = data;
1736 struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
1737 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
1738 enum hctx_type type = hctx->type;
1739
1740 spin_lock(&ctx->lock);
1741 if (!list_empty(&ctx->rq_lists[type])) {
1742 dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next);
1743 list_del_init(&dispatch_data->rq->queuelist);
1744 if (list_empty(&ctx->rq_lists[type]))
1745 sbitmap_clear_bit(sb, bitnr);
1746 }
1747 spin_unlock(&ctx->lock);
1748
1749 return !dispatch_data->rq;
1750 }
1751
blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx * hctx,struct blk_mq_ctx * start)1752 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
1753 struct blk_mq_ctx *start)
1754 {
1755 unsigned off = start ? start->index_hw[hctx->type] : 0;
1756 struct dispatch_rq_data data = {
1757 .hctx = hctx,
1758 .rq = NULL,
1759 };
1760
1761 __sbitmap_for_each_set(&hctx->ctx_map, off,
1762 dispatch_rq_from_ctx, &data);
1763
1764 return data.rq;
1765 }
1766
__blk_mq_alloc_driver_tag(struct request * rq)1767 static bool __blk_mq_alloc_driver_tag(struct request *rq)
1768 {
1769 struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
1770 unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
1771 int tag;
1772
1773 blk_mq_tag_busy(rq->mq_hctx);
1774
1775 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
1776 bt = &rq->mq_hctx->tags->breserved_tags;
1777 tag_offset = 0;
1778 } else {
1779 if (!hctx_may_queue(rq->mq_hctx, bt))
1780 return false;
1781 }
1782
1783 tag = __sbitmap_queue_get(bt);
1784 if (tag == BLK_MQ_NO_TAG)
1785 return false;
1786
1787 rq->tag = tag + tag_offset;
1788 return true;
1789 }
1790
__blk_mq_get_driver_tag(struct blk_mq_hw_ctx * hctx,struct request * rq)1791 bool __blk_mq_get_driver_tag(struct blk_mq_hw_ctx *hctx, struct request *rq)
1792 {
1793 if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_alloc_driver_tag(rq))
1794 return false;
1795
1796 if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
1797 !(rq->rq_flags & RQF_MQ_INFLIGHT)) {
1798 rq->rq_flags |= RQF_MQ_INFLIGHT;
1799 __blk_mq_inc_active_requests(hctx);
1800 }
1801 hctx->tags->rqs[rq->tag] = rq;
1802 return true;
1803 }
1804
blk_mq_dispatch_wake(wait_queue_entry_t * wait,unsigned mode,int flags,void * key)1805 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
1806 int flags, void *key)
1807 {
1808 struct blk_mq_hw_ctx *hctx;
1809
1810 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
1811
1812 spin_lock(&hctx->dispatch_wait_lock);
1813 if (!list_empty(&wait->entry)) {
1814 struct sbitmap_queue *sbq;
1815
1816 list_del_init(&wait->entry);
1817 sbq = &hctx->tags->bitmap_tags;
1818 atomic_dec(&sbq->ws_active);
1819 }
1820 spin_unlock(&hctx->dispatch_wait_lock);
1821
1822 blk_mq_run_hw_queue(hctx, true);
1823 return 1;
1824 }
1825
1826 /*
1827 * Mark us waiting for a tag. For shared tags, this involves hooking us into
1828 * the tag wakeups. For non-shared tags, we can simply mark us needing a
1829 * restart. For both cases, take care to check the condition again after
1830 * marking us as waiting.
1831 */
blk_mq_mark_tag_wait(struct blk_mq_hw_ctx * hctx,struct request * rq)1832 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
1833 struct request *rq)
1834 {
1835 struct sbitmap_queue *sbq;
1836 struct wait_queue_head *wq;
1837 wait_queue_entry_t *wait;
1838 bool ret;
1839
1840 if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
1841 !(blk_mq_is_shared_tags(hctx->flags))) {
1842 blk_mq_sched_mark_restart_hctx(hctx);
1843
1844 /*
1845 * It's possible that a tag was freed in the window between the
1846 * allocation failure and adding the hardware queue to the wait
1847 * queue.
1848 *
1849 * Don't clear RESTART here, someone else could have set it.
1850 * At most this will cost an extra queue run.
1851 */
1852 return blk_mq_get_driver_tag(rq);
1853 }
1854
1855 wait = &hctx->dispatch_wait;
1856 if (!list_empty_careful(&wait->entry))
1857 return false;
1858
1859 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag))
1860 sbq = &hctx->tags->breserved_tags;
1861 else
1862 sbq = &hctx->tags->bitmap_tags;
1863 wq = &bt_wait_ptr(sbq, hctx)->wait;
1864
1865 spin_lock_irq(&wq->lock);
1866 spin_lock(&hctx->dispatch_wait_lock);
1867 if (!list_empty(&wait->entry)) {
1868 spin_unlock(&hctx->dispatch_wait_lock);
1869 spin_unlock_irq(&wq->lock);
1870 return false;
1871 }
1872
1873 atomic_inc(&sbq->ws_active);
1874 wait->flags &= ~WQ_FLAG_EXCLUSIVE;
1875 __add_wait_queue(wq, wait);
1876
1877 /*
1878 * Add one explicit barrier since blk_mq_get_driver_tag() may
1879 * not imply barrier in case of failure.
1880 *
1881 * Order adding us to wait queue and allocating driver tag.
1882 *
1883 * The pair is the one implied in sbitmap_queue_wake_up() which
1884 * orders clearing sbitmap tag bits and waitqueue_active() in
1885 * __sbitmap_queue_wake_up(), since waitqueue_active() is lockless
1886 *
1887 * Otherwise, re-order of adding wait queue and getting driver tag
1888 * may cause __sbitmap_queue_wake_up() to wake up nothing because
1889 * the waitqueue_active() may not observe us in wait queue.
1890 */
1891 smp_mb();
1892
1893 /*
1894 * It's possible that a tag was freed in the window between the
1895 * allocation failure and adding the hardware queue to the wait
1896 * queue.
1897 */
1898 ret = blk_mq_get_driver_tag(rq);
1899 if (!ret) {
1900 spin_unlock(&hctx->dispatch_wait_lock);
1901 spin_unlock_irq(&wq->lock);
1902 return false;
1903 }
1904
1905 /*
1906 * We got a tag, remove ourselves from the wait queue to ensure
1907 * someone else gets the wakeup.
1908 */
1909 list_del_init(&wait->entry);
1910 atomic_dec(&sbq->ws_active);
1911 spin_unlock(&hctx->dispatch_wait_lock);
1912 spin_unlock_irq(&wq->lock);
1913
1914 return true;
1915 }
1916
1917 #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT 8
1918 #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR 4
1919 /*
1920 * Update dispatch busy with the Exponential Weighted Moving Average(EWMA):
1921 * - EWMA is one simple way to compute running average value
1922 * - weight(7/8 and 1/8) is applied so that it can decrease exponentially
1923 * - take 4 as factor for avoiding to get too small(0) result, and this
1924 * factor doesn't matter because EWMA decreases exponentially
1925 */
blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx * hctx,bool busy)1926 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)
1927 {
1928 unsigned int ewma;
1929
1930 ewma = hctx->dispatch_busy;
1931
1932 if (!ewma && !busy)
1933 return;
1934
1935 ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1;
1936 if (busy)
1937 ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR;
1938 ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT;
1939
1940 hctx->dispatch_busy = ewma;
1941 }
1942
1943 #define BLK_MQ_RESOURCE_DELAY 3 /* ms units */
1944
blk_mq_handle_dev_resource(struct request * rq,struct list_head * list)1945 static void blk_mq_handle_dev_resource(struct request *rq,
1946 struct list_head *list)
1947 {
1948 list_add(&rq->queuelist, list);
1949 __blk_mq_requeue_request(rq);
1950 }
1951
blk_mq_handle_zone_resource(struct request * rq,struct list_head * zone_list)1952 static void blk_mq_handle_zone_resource(struct request *rq,
1953 struct list_head *zone_list)
1954 {
1955 /*
1956 * If we end up here it is because we cannot dispatch a request to a
1957 * specific zone due to LLD level zone-write locking or other zone
1958 * related resource not being available. In this case, set the request
1959 * aside in zone_list for retrying it later.
1960 */
1961 list_add(&rq->queuelist, zone_list);
1962 __blk_mq_requeue_request(rq);
1963 }
1964
1965 enum prep_dispatch {
1966 PREP_DISPATCH_OK,
1967 PREP_DISPATCH_NO_TAG,
1968 PREP_DISPATCH_NO_BUDGET,
1969 };
1970
blk_mq_prep_dispatch_rq(struct request * rq,bool need_budget)1971 static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
1972 bool need_budget)
1973 {
1974 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1975 int budget_token = -1;
1976
1977 if (need_budget) {
1978 budget_token = blk_mq_get_dispatch_budget(rq->q);
1979 if (budget_token < 0) {
1980 blk_mq_put_driver_tag(rq);
1981 return PREP_DISPATCH_NO_BUDGET;
1982 }
1983 blk_mq_set_rq_budget_token(rq, budget_token);
1984 }
1985
1986 if (!blk_mq_get_driver_tag(rq)) {
1987 /*
1988 * The initial allocation attempt failed, so we need to
1989 * rerun the hardware queue when a tag is freed. The
1990 * waitqueue takes care of that. If the queue is run
1991 * before we add this entry back on the dispatch list,
1992 * we'll re-run it below.
1993 */
1994 if (!blk_mq_mark_tag_wait(hctx, rq)) {
1995 /*
1996 * All budgets not got from this function will be put
1997 * together during handling partial dispatch
1998 */
1999 if (need_budget)
2000 blk_mq_put_dispatch_budget(rq->q, budget_token);
2001 return PREP_DISPATCH_NO_TAG;
2002 }
2003 }
2004
2005 return PREP_DISPATCH_OK;
2006 }
2007
2008 /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
blk_mq_release_budgets(struct request_queue * q,struct list_head * list)2009 static void blk_mq_release_budgets(struct request_queue *q,
2010 struct list_head *list)
2011 {
2012 struct request *rq;
2013
2014 list_for_each_entry(rq, list, queuelist) {
2015 int budget_token = blk_mq_get_rq_budget_token(rq);
2016
2017 if (budget_token >= 0)
2018 blk_mq_put_dispatch_budget(q, budget_token);
2019 }
2020 }
2021
2022 /*
2023 * blk_mq_commit_rqs will notify driver using bd->last that there is no
2024 * more requests. (See comment in struct blk_mq_ops for commit_rqs for
2025 * details)
2026 * Attention, we should explicitly call this in unusual cases:
2027 * 1) did not queue everything initially scheduled to queue
2028 * 2) the last attempt to queue a request failed
2029 */
blk_mq_commit_rqs(struct blk_mq_hw_ctx * hctx,int queued,bool from_schedule)2030 static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued,
2031 bool from_schedule)
2032 {
2033 if (hctx->queue->mq_ops->commit_rqs && queued) {
2034 trace_block_unplug(hctx->queue, queued, !from_schedule);
2035 hctx->queue->mq_ops->commit_rqs(hctx);
2036 }
2037 }
2038
2039 /*
2040 * Returns true if we did some work AND can potentially do more.
2041 */
blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx * hctx,struct list_head * list,unsigned int nr_budgets)2042 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
2043 unsigned int nr_budgets)
2044 {
2045 enum prep_dispatch prep;
2046 struct request_queue *q = hctx->queue;
2047 struct request *rq;
2048 int queued;
2049 blk_status_t ret = BLK_STS_OK;
2050 LIST_HEAD(zone_list);
2051 bool needs_resource = false;
2052
2053 if (list_empty(list))
2054 return false;
2055
2056 /*
2057 * Now process all the entries, sending them to the driver.
2058 */
2059 queued = 0;
2060 do {
2061 struct blk_mq_queue_data bd;
2062
2063 rq = list_first_entry(list, struct request, queuelist);
2064
2065 WARN_ON_ONCE(hctx != rq->mq_hctx);
2066 prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets);
2067 if (prep != PREP_DISPATCH_OK)
2068 break;
2069
2070 list_del_init(&rq->queuelist);
2071
2072 bd.rq = rq;
2073 bd.last = list_empty(list);
2074
2075 /*
2076 * once the request is queued to lld, no need to cover the
2077 * budget any more
2078 */
2079 if (nr_budgets)
2080 nr_budgets--;
2081 ret = q->mq_ops->queue_rq(hctx, &bd);
2082 switch (ret) {
2083 case BLK_STS_OK:
2084 queued++;
2085 break;
2086 case BLK_STS_RESOURCE:
2087 needs_resource = true;
2088 fallthrough;
2089 case BLK_STS_DEV_RESOURCE:
2090 blk_mq_handle_dev_resource(rq, list);
2091 goto out;
2092 case BLK_STS_ZONE_RESOURCE:
2093 /*
2094 * Move the request to zone_list and keep going through
2095 * the dispatch list to find more requests the drive can
2096 * accept.
2097 */
2098 blk_mq_handle_zone_resource(rq, &zone_list);
2099 needs_resource = true;
2100 break;
2101 default:
2102 blk_mq_end_request(rq, ret);
2103 }
2104 } while (!list_empty(list));
2105 out:
2106 if (!list_empty(&zone_list))
2107 list_splice_tail_init(&zone_list, list);
2108
2109 /* If we didn't flush the entire list, we could have told the driver
2110 * there was more coming, but that turned out to be a lie.
2111 */
2112 if (!list_empty(list) || ret != BLK_STS_OK)
2113 blk_mq_commit_rqs(hctx, queued, false);
2114
2115 /*
2116 * Any items that need requeuing? Stuff them into hctx->dispatch,
2117 * that is where we will continue on next queue run.
2118 */
2119 if (!list_empty(list)) {
2120 bool needs_restart;
2121 /* For non-shared tags, the RESTART check will suffice */
2122 bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
2123 ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) ||
2124 blk_mq_is_shared_tags(hctx->flags));
2125
2126 if (nr_budgets)
2127 blk_mq_release_budgets(q, list);
2128
2129 spin_lock(&hctx->lock);
2130 list_splice_tail_init(list, &hctx->dispatch);
2131 spin_unlock(&hctx->lock);
2132
2133 /*
2134 * Order adding requests to hctx->dispatch and checking
2135 * SCHED_RESTART flag. The pair of this smp_mb() is the one
2136 * in blk_mq_sched_restart(). Avoid restart code path to
2137 * miss the new added requests to hctx->dispatch, meantime
2138 * SCHED_RESTART is observed here.
2139 */
2140 smp_mb();
2141
2142 /*
2143 * If SCHED_RESTART was set by the caller of this function and
2144 * it is no longer set that means that it was cleared by another
2145 * thread and hence that a queue rerun is needed.
2146 *
2147 * If 'no_tag' is set, that means that we failed getting
2148 * a driver tag with an I/O scheduler attached. If our dispatch
2149 * waitqueue is no longer active, ensure that we run the queue
2150 * AFTER adding our entries back to the list.
2151 *
2152 * If no I/O scheduler has been configured it is possible that
2153 * the hardware queue got stopped and restarted before requests
2154 * were pushed back onto the dispatch list. Rerun the queue to
2155 * avoid starvation. Notes:
2156 * - blk_mq_run_hw_queue() checks whether or not a queue has
2157 * been stopped before rerunning a queue.
2158 * - Some but not all block drivers stop a queue before
2159 * returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
2160 * and dm-rq.
2161 *
2162 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART
2163 * bit is set, run queue after a delay to avoid IO stalls
2164 * that could otherwise occur if the queue is idle. We'll do
2165 * similar if we couldn't get budget or couldn't lock a zone
2166 * and SCHED_RESTART is set.
2167 */
2168 needs_restart = blk_mq_sched_needs_restart(hctx);
2169 if (prep == PREP_DISPATCH_NO_BUDGET)
2170 needs_resource = true;
2171 if (!needs_restart ||
2172 (no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
2173 blk_mq_run_hw_queue(hctx, true);
2174 else if (needs_resource)
2175 blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
2176
2177 blk_mq_update_dispatch_busy(hctx, true);
2178 return false;
2179 }
2180
2181 blk_mq_update_dispatch_busy(hctx, false);
2182 return true;
2183 }
2184
blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx * hctx)2185 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
2186 {
2187 int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);
2188
2189 if (cpu >= nr_cpu_ids)
2190 cpu = cpumask_first(hctx->cpumask);
2191 return cpu;
2192 }
2193
2194 /*
2195 * It'd be great if the workqueue API had a way to pass
2196 * in a mask and had some smarts for more clever placement.
2197 * For now we just round-robin here, switching for every
2198 * BLK_MQ_CPU_WORK_BATCH queued items.
2199 */
blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx * hctx)2200 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
2201 {
2202 bool tried = false;
2203 int next_cpu = hctx->next_cpu;
2204
2205 if (hctx->queue->nr_hw_queues == 1)
2206 return WORK_CPU_UNBOUND;
2207
2208 if (--hctx->next_cpu_batch <= 0) {
2209 select_cpu:
2210 next_cpu = cpumask_next_and(next_cpu, hctx->cpumask,
2211 cpu_online_mask);
2212 if (next_cpu >= nr_cpu_ids)
2213 next_cpu = blk_mq_first_mapped_cpu(hctx);
2214 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
2215 }
2216
2217 /*
2218 * Do unbound schedule if we can't find a online CPU for this hctx,
2219 * and it should only happen in the path of handling CPU DEAD.
2220 */
2221 if (!cpu_online(next_cpu)) {
2222 if (!tried) {
2223 tried = true;
2224 goto select_cpu;
2225 }
2226
2227 /*
2228 * Make sure to re-select CPU next time once after CPUs
2229 * in hctx->cpumask become online again.
2230 */
2231 hctx->next_cpu = next_cpu;
2232 hctx->next_cpu_batch = 1;
2233 return WORK_CPU_UNBOUND;
2234 }
2235
2236 hctx->next_cpu = next_cpu;
2237 return next_cpu;
2238 }
2239
2240 /**
2241 * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
2242 * @hctx: Pointer to the hardware queue to run.
2243 * @msecs: Milliseconds of delay to wait before running the queue.
2244 *
2245 * Run a hardware queue asynchronously with a delay of @msecs.
2246 */
blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx * hctx,unsigned long msecs)2247 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
2248 {
2249 if (unlikely(blk_mq_hctx_stopped(hctx)))
2250 return;
2251 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
2252 msecs_to_jiffies(msecs));
2253 }
2254 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
2255
2256 /**
2257 * blk_mq_run_hw_queue - Start to run a hardware queue.
2258 * @hctx: Pointer to the hardware queue to run.
2259 * @async: If we want to run the queue asynchronously.
2260 *
2261 * Check if the request queue is not in a quiesced state and if there are
2262 * pending requests to be sent. If this is true, run the queue to send requests
2263 * to hardware.
2264 */
blk_mq_run_hw_queue(struct blk_mq_hw_ctx * hctx,bool async)2265 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
2266 {
2267 bool need_run;
2268
2269 /*
2270 * We can't run the queue inline with interrupts disabled.
2271 */
2272 WARN_ON_ONCE(!async && in_interrupt());
2273
2274 might_sleep_if(!async && hctx->flags & BLK_MQ_F_BLOCKING);
2275
2276 /*
2277 * When queue is quiesced, we may be switching io scheduler, or
2278 * updating nr_hw_queues, or other things, and we can't run queue
2279 * any more, even __blk_mq_hctx_has_pending() can't be called safely.
2280 *
2281 * And queue will be rerun in blk_mq_unquiesce_queue() if it is
2282 * quiesced.
2283 */
2284 __blk_mq_run_dispatch_ops(hctx->queue, false,
2285 need_run = !blk_queue_quiesced(hctx->queue) &&
2286 blk_mq_hctx_has_pending(hctx));
2287
2288 if (!need_run)
2289 return;
2290
2291 if (async || !cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) {
2292 blk_mq_delay_run_hw_queue(hctx, 0);
2293 return;
2294 }
2295
2296 blk_mq_run_dispatch_ops(hctx->queue,
2297 blk_mq_sched_dispatch_requests(hctx));
2298 }
2299 EXPORT_SYMBOL(blk_mq_run_hw_queue);
2300
2301 /*
2302 * Return prefered queue to dispatch from (if any) for non-mq aware IO
2303 * scheduler.
2304 */
blk_mq_get_sq_hctx(struct request_queue * q)2305 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
2306 {
2307 struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
2308 /*
2309 * If the IO scheduler does not respect hardware queues when
2310 * dispatching, we just don't bother with multiple HW queues and
2311 * dispatch from hctx for the current CPU since running multiple queues
2312 * just causes lock contention inside the scheduler and pointless cache
2313 * bouncing.
2314 */
2315 struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT];
2316
2317 if (!blk_mq_hctx_stopped(hctx))
2318 return hctx;
2319 return NULL;
2320 }
2321
2322 /**
2323 * blk_mq_run_hw_queues - Run all hardware queues in a request queue.
2324 * @q: Pointer to the request queue to run.
2325 * @async: If we want to run the queue asynchronously.
2326 */
blk_mq_run_hw_queues(struct request_queue * q,bool async)2327 void blk_mq_run_hw_queues(struct request_queue *q, bool async)
2328 {
2329 struct blk_mq_hw_ctx *hctx, *sq_hctx;
2330 unsigned long i;
2331
2332 sq_hctx = NULL;
2333 if (blk_queue_sq_sched(q))
2334 sq_hctx = blk_mq_get_sq_hctx(q);
2335 queue_for_each_hw_ctx(q, hctx, i) {
2336 if (blk_mq_hctx_stopped(hctx))
2337 continue;
2338 /*
2339 * Dispatch from this hctx either if there's no hctx preferred
2340 * by IO scheduler or if it has requests that bypass the
2341 * scheduler.
2342 */
2343 if (!sq_hctx || sq_hctx == hctx ||
2344 !list_empty_careful(&hctx->dispatch))
2345 blk_mq_run_hw_queue(hctx, async);
2346 }
2347 }
2348 EXPORT_SYMBOL(blk_mq_run_hw_queues);
2349
2350 /**
2351 * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
2352 * @q: Pointer to the request queue to run.
2353 * @msecs: Milliseconds of delay to wait before running the queues.
2354 */
blk_mq_delay_run_hw_queues(struct request_queue * q,unsigned long msecs)2355 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
2356 {
2357 struct blk_mq_hw_ctx *hctx, *sq_hctx;
2358 unsigned long i;
2359
2360 sq_hctx = NULL;
2361 if (blk_queue_sq_sched(q))
2362 sq_hctx = blk_mq_get_sq_hctx(q);
2363 queue_for_each_hw_ctx(q, hctx, i) {
2364 if (blk_mq_hctx_stopped(hctx))
2365 continue;
2366 /*
2367 * If there is already a run_work pending, leave the
2368 * pending delay untouched. Otherwise, a hctx can stall
2369 * if another hctx is re-delaying the other's work
2370 * before the work executes.
2371 */
2372 if (delayed_work_pending(&hctx->run_work))
2373 continue;
2374 /*
2375 * Dispatch from this hctx either if there's no hctx preferred
2376 * by IO scheduler or if it has requests that bypass the
2377 * scheduler.
2378 */
2379 if (!sq_hctx || sq_hctx == hctx ||
2380 !list_empty_careful(&hctx->dispatch))
2381 blk_mq_delay_run_hw_queue(hctx, msecs);
2382 }
2383 }
2384 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
2385
2386 /*
2387 * This function is often used for pausing .queue_rq() by driver when
2388 * there isn't enough resource or some conditions aren't satisfied, and
2389 * BLK_STS_RESOURCE is usually returned.
2390 *
2391 * We do not guarantee that dispatch can be drained or blocked
2392 * after blk_mq_stop_hw_queue() returns. Please use
2393 * blk_mq_quiesce_queue() for that requirement.
2394 */
blk_mq_stop_hw_queue(struct blk_mq_hw_ctx * hctx)2395 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
2396 {
2397 cancel_delayed_work(&hctx->run_work);
2398
2399 set_bit(BLK_MQ_S_STOPPED, &hctx->state);
2400 }
2401 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
2402
2403 /*
2404 * This function is often used for pausing .queue_rq() by driver when
2405 * there isn't enough resource or some conditions aren't satisfied, and
2406 * BLK_STS_RESOURCE is usually returned.
2407 *
2408 * We do not guarantee that dispatch can be drained or blocked
2409 * after blk_mq_stop_hw_queues() returns. Please use
2410 * blk_mq_quiesce_queue() for that requirement.
2411 */
blk_mq_stop_hw_queues(struct request_queue * q)2412 void blk_mq_stop_hw_queues(struct request_queue *q)
2413 {
2414 struct blk_mq_hw_ctx *hctx;
2415 unsigned long i;
2416
2417 queue_for_each_hw_ctx(q, hctx, i)
2418 blk_mq_stop_hw_queue(hctx);
2419 }
2420 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
2421
blk_mq_start_hw_queue(struct blk_mq_hw_ctx * hctx)2422 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
2423 {
2424 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
2425
2426 blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
2427 }
2428 EXPORT_SYMBOL(blk_mq_start_hw_queue);
2429
blk_mq_start_hw_queues(struct request_queue * q)2430 void blk_mq_start_hw_queues(struct request_queue *q)
2431 {
2432 struct blk_mq_hw_ctx *hctx;
2433 unsigned long i;
2434
2435 queue_for_each_hw_ctx(q, hctx, i)
2436 blk_mq_start_hw_queue(hctx);
2437 }
2438 EXPORT_SYMBOL(blk_mq_start_hw_queues);
2439
blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx * hctx,bool async)2440 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
2441 {
2442 if (!blk_mq_hctx_stopped(hctx))
2443 return;
2444
2445 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
2446 blk_mq_run_hw_queue(hctx, async);
2447 }
2448 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
2449
blk_mq_start_stopped_hw_queues(struct request_queue * q,bool async)2450 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
2451 {
2452 struct blk_mq_hw_ctx *hctx;
2453 unsigned long i;
2454
2455 queue_for_each_hw_ctx(q, hctx, i)
2456 blk_mq_start_stopped_hw_queue(hctx, async ||
2457 (hctx->flags & BLK_MQ_F_BLOCKING));
2458 }
2459 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
2460
blk_mq_run_work_fn(struct work_struct * work)2461 static void blk_mq_run_work_fn(struct work_struct *work)
2462 {
2463 struct blk_mq_hw_ctx *hctx =
2464 container_of(work, struct blk_mq_hw_ctx, run_work.work);
2465
2466 blk_mq_run_dispatch_ops(hctx->queue,
2467 blk_mq_sched_dispatch_requests(hctx));
2468 }
2469
2470 /**
2471 * blk_mq_request_bypass_insert - Insert a request at dispatch list.
2472 * @rq: Pointer to request to be inserted.
2473 * @flags: BLK_MQ_INSERT_*
2474 *
2475 * Should only be used carefully, when the caller knows we want to
2476 * bypass a potential IO scheduler on the target device.
2477 */
blk_mq_request_bypass_insert(struct request * rq,blk_insert_t flags)2478 static void blk_mq_request_bypass_insert(struct request *rq, blk_insert_t flags)
2479 {
2480 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2481
2482 spin_lock(&hctx->lock);
2483 if (flags & BLK_MQ_INSERT_AT_HEAD)
2484 list_add(&rq->queuelist, &hctx->dispatch);
2485 else
2486 list_add_tail(&rq->queuelist, &hctx->dispatch);
2487 spin_unlock(&hctx->lock);
2488 }
2489
blk_mq_insert_requests(struct blk_mq_hw_ctx * hctx,struct blk_mq_ctx * ctx,struct list_head * list,bool run_queue_async)2490 static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,
2491 struct blk_mq_ctx *ctx, struct list_head *list,
2492 bool run_queue_async)
2493 {
2494 struct request *rq;
2495 enum hctx_type type = hctx->type;
2496
2497 /*
2498 * Try to issue requests directly if the hw queue isn't busy to save an
2499 * extra enqueue & dequeue to the sw queue.
2500 */
2501 if (!hctx->dispatch_busy && !run_queue_async) {
2502 blk_mq_run_dispatch_ops(hctx->queue,
2503 blk_mq_try_issue_list_directly(hctx, list));
2504 if (list_empty(list))
2505 goto out;
2506 }
2507
2508 /*
2509 * preemption doesn't flush plug list, so it's possible ctx->cpu is
2510 * offline now
2511 */
2512 list_for_each_entry(rq, list, queuelist) {
2513 BUG_ON(rq->mq_ctx != ctx);
2514 trace_block_rq_insert(rq);
2515 if (rq->cmd_flags & REQ_NOWAIT)
2516 run_queue_async = true;
2517 }
2518
2519 spin_lock(&ctx->lock);
2520 list_splice_tail_init(list, &ctx->rq_lists[type]);
2521 blk_mq_hctx_mark_pending(hctx, ctx);
2522 spin_unlock(&ctx->lock);
2523 out:
2524 blk_mq_run_hw_queue(hctx, run_queue_async);
2525 }
2526
blk_mq_insert_request(struct request * rq,blk_insert_t flags)2527 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags)
2528 {
2529 struct request_queue *q = rq->q;
2530 struct blk_mq_ctx *ctx = rq->mq_ctx;
2531 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2532
2533 if (blk_rq_is_passthrough(rq)) {
2534 /*
2535 * Passthrough request have to be added to hctx->dispatch
2536 * directly. The device may be in a situation where it can't
2537 * handle FS request, and always returns BLK_STS_RESOURCE for
2538 * them, which gets them added to hctx->dispatch.
2539 *
2540 * If a passthrough request is required to unblock the queues,
2541 * and it is added to the scheduler queue, there is no chance to
2542 * dispatch it given we prioritize requests in hctx->dispatch.
2543 */
2544 blk_mq_request_bypass_insert(rq, flags);
2545 } else if (req_op(rq) == REQ_OP_FLUSH) {
2546 /*
2547 * Firstly normal IO request is inserted to scheduler queue or
2548 * sw queue, meantime we add flush request to dispatch queue(
2549 * hctx->dispatch) directly and there is at most one in-flight
2550 * flush request for each hw queue, so it doesn't matter to add
2551 * flush request to tail or front of the dispatch queue.
2552 *
2553 * Secondly in case of NCQ, flush request belongs to non-NCQ
2554 * command, and queueing it will fail when there is any
2555 * in-flight normal IO request(NCQ command). When adding flush
2556 * rq to the front of hctx->dispatch, it is easier to introduce
2557 * extra time to flush rq's latency because of S_SCHED_RESTART
2558 * compared with adding to the tail of dispatch queue, then
2559 * chance of flush merge is increased, and less flush requests
2560 * will be issued to controller. It is observed that ~10% time
2561 * is saved in blktests block/004 on disk attached to AHCI/NCQ
2562 * drive when adding flush rq to the front of hctx->dispatch.
2563 *
2564 * Simply queue flush rq to the front of hctx->dispatch so that
2565 * intensive flush workloads can benefit in case of NCQ HW.
2566 */
2567 blk_mq_request_bypass_insert(rq, BLK_MQ_INSERT_AT_HEAD);
2568 } else if (q->elevator) {
2569 LIST_HEAD(list);
2570
2571 WARN_ON_ONCE(rq->tag != BLK_MQ_NO_TAG);
2572
2573 list_add(&rq->queuelist, &list);
2574 q->elevator->type->ops.insert_requests(hctx, &list, flags);
2575 } else {
2576 trace_block_rq_insert(rq);
2577
2578 spin_lock(&ctx->lock);
2579 if (flags & BLK_MQ_INSERT_AT_HEAD)
2580 list_add(&rq->queuelist, &ctx->rq_lists[hctx->type]);
2581 else
2582 list_add_tail(&rq->queuelist,
2583 &ctx->rq_lists[hctx->type]);
2584 blk_mq_hctx_mark_pending(hctx, ctx);
2585 spin_unlock(&ctx->lock);
2586 }
2587 }
2588
blk_mq_bio_to_request(struct request * rq,struct bio * bio,unsigned int nr_segs)2589 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
2590 unsigned int nr_segs)
2591 {
2592 int err;
2593
2594 if (bio->bi_opf & REQ_RAHEAD)
2595 rq->cmd_flags |= REQ_FAILFAST_MASK;
2596
2597 rq->__sector = bio->bi_iter.bi_sector;
2598 blk_rq_bio_prep(rq, bio, nr_segs);
2599
2600 /* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
2601 err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
2602 WARN_ON_ONCE(err);
2603
2604 blk_account_io_start(rq);
2605 }
2606
__blk_mq_issue_directly(struct blk_mq_hw_ctx * hctx,struct request * rq,bool last)2607 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
2608 struct request *rq, bool last)
2609 {
2610 struct request_queue *q = rq->q;
2611 struct blk_mq_queue_data bd = {
2612 .rq = rq,
2613 .last = last,
2614 };
2615 blk_status_t ret;
2616
2617 /*
2618 * For OK queue, we are done. For error, caller may kill it.
2619 * Any other error (busy), just add it to our list as we
2620 * previously would have done.
2621 */
2622 ret = q->mq_ops->queue_rq(hctx, &bd);
2623 switch (ret) {
2624 case BLK_STS_OK:
2625 blk_mq_update_dispatch_busy(hctx, false);
2626 break;
2627 case BLK_STS_RESOURCE:
2628 case BLK_STS_DEV_RESOURCE:
2629 blk_mq_update_dispatch_busy(hctx, true);
2630 __blk_mq_requeue_request(rq);
2631 break;
2632 default:
2633 blk_mq_update_dispatch_busy(hctx, false);
2634 break;
2635 }
2636
2637 return ret;
2638 }
2639
blk_mq_get_budget_and_tag(struct request * rq)2640 static bool blk_mq_get_budget_and_tag(struct request *rq)
2641 {
2642 int budget_token;
2643
2644 budget_token = blk_mq_get_dispatch_budget(rq->q);
2645 if (budget_token < 0)
2646 return false;
2647 blk_mq_set_rq_budget_token(rq, budget_token);
2648 if (!blk_mq_get_driver_tag(rq)) {
2649 blk_mq_put_dispatch_budget(rq->q, budget_token);
2650 return false;
2651 }
2652 return true;
2653 }
2654
2655 /**
2656 * blk_mq_try_issue_directly - Try to send a request directly to device driver.
2657 * @hctx: Pointer of the associated hardware queue.
2658 * @rq: Pointer to request to be sent.
2659 *
2660 * If the device has enough resources to accept a new request now, send the
2661 * request directly to device driver. Else, insert at hctx->dispatch queue, so
2662 * we can try send it another time in the future. Requests inserted at this
2663 * queue have higher priority.
2664 */
blk_mq_try_issue_directly(struct blk_mq_hw_ctx * hctx,struct request * rq)2665 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
2666 struct request *rq)
2667 {
2668 blk_status_t ret;
2669
2670 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
2671 blk_mq_insert_request(rq, 0);
2672 return;
2673 }
2674
2675 if ((rq->rq_flags & RQF_USE_SCHED) || !blk_mq_get_budget_and_tag(rq)) {
2676 blk_mq_insert_request(rq, 0);
2677 blk_mq_run_hw_queue(hctx, rq->cmd_flags & REQ_NOWAIT);
2678 return;
2679 }
2680
2681 ret = __blk_mq_issue_directly(hctx, rq, true);
2682 switch (ret) {
2683 case BLK_STS_OK:
2684 break;
2685 case BLK_STS_RESOURCE:
2686 case BLK_STS_DEV_RESOURCE:
2687 blk_mq_request_bypass_insert(rq, 0);
2688 blk_mq_run_hw_queue(hctx, false);
2689 break;
2690 default:
2691 blk_mq_end_request(rq, ret);
2692 break;
2693 }
2694 }
2695
blk_mq_request_issue_directly(struct request * rq,bool last)2696 static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
2697 {
2698 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2699
2700 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
2701 blk_mq_insert_request(rq, 0);
2702 return BLK_STS_OK;
2703 }
2704
2705 if (!blk_mq_get_budget_and_tag(rq))
2706 return BLK_STS_RESOURCE;
2707 return __blk_mq_issue_directly(hctx, rq, last);
2708 }
2709
blk_mq_plug_issue_direct(struct blk_plug * plug)2710 static void blk_mq_plug_issue_direct(struct blk_plug *plug)
2711 {
2712 struct blk_mq_hw_ctx *hctx = NULL;
2713 struct request *rq;
2714 int queued = 0;
2715 blk_status_t ret = BLK_STS_OK;
2716
2717 while ((rq = rq_list_pop(&plug->mq_list))) {
2718 bool last = rq_list_empty(plug->mq_list);
2719
2720 if (hctx != rq->mq_hctx) {
2721 if (hctx) {
2722 blk_mq_commit_rqs(hctx, queued, false);
2723 queued = 0;
2724 }
2725 hctx = rq->mq_hctx;
2726 }
2727
2728 ret = blk_mq_request_issue_directly(rq, last);
2729 switch (ret) {
2730 case BLK_STS_OK:
2731 queued++;
2732 break;
2733 case BLK_STS_RESOURCE:
2734 case BLK_STS_DEV_RESOURCE:
2735 blk_mq_request_bypass_insert(rq, 0);
2736 blk_mq_run_hw_queue(hctx, false);
2737 goto out;
2738 default:
2739 blk_mq_end_request(rq, ret);
2740 break;
2741 }
2742 }
2743
2744 out:
2745 if (ret != BLK_STS_OK)
2746 blk_mq_commit_rqs(hctx, queued, false);
2747 }
2748
__blk_mq_flush_plug_list(struct request_queue * q,struct blk_plug * plug)2749 static void __blk_mq_flush_plug_list(struct request_queue *q,
2750 struct blk_plug *plug)
2751 {
2752 if (blk_queue_quiesced(q))
2753 return;
2754 q->mq_ops->queue_rqs(&plug->mq_list);
2755 }
2756
blk_mq_dispatch_plug_list(struct blk_plug * plug,bool from_sched)2757 static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched)
2758 {
2759 struct blk_mq_hw_ctx *this_hctx = NULL;
2760 struct blk_mq_ctx *this_ctx = NULL;
2761 struct request *requeue_list = NULL;
2762 struct request **requeue_lastp = &requeue_list;
2763 unsigned int depth = 0;
2764 bool is_passthrough = false;
2765 LIST_HEAD(list);
2766
2767 do {
2768 struct request *rq = rq_list_pop(&plug->mq_list);
2769
2770 if (!this_hctx) {
2771 this_hctx = rq->mq_hctx;
2772 this_ctx = rq->mq_ctx;
2773 is_passthrough = blk_rq_is_passthrough(rq);
2774 } else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx ||
2775 is_passthrough != blk_rq_is_passthrough(rq)) {
2776 rq_list_add_tail(&requeue_lastp, rq);
2777 continue;
2778 }
2779 list_add(&rq->queuelist, &list);
2780 depth++;
2781 } while (!rq_list_empty(plug->mq_list));
2782
2783 plug->mq_list = requeue_list;
2784 trace_block_unplug(this_hctx->queue, depth, !from_sched);
2785
2786 percpu_ref_get(&this_hctx->queue->q_usage_counter);
2787 /* passthrough requests should never be issued to the I/O scheduler */
2788 if (is_passthrough) {
2789 spin_lock(&this_hctx->lock);
2790 list_splice_tail_init(&list, &this_hctx->dispatch);
2791 spin_unlock(&this_hctx->lock);
2792 blk_mq_run_hw_queue(this_hctx, from_sched);
2793 } else if (this_hctx->queue->elevator) {
2794 this_hctx->queue->elevator->type->ops.insert_requests(this_hctx,
2795 &list, 0);
2796 blk_mq_run_hw_queue(this_hctx, from_sched);
2797 } else {
2798 blk_mq_insert_requests(this_hctx, this_ctx, &list, from_sched);
2799 }
2800 percpu_ref_put(&this_hctx->queue->q_usage_counter);
2801 }
2802
blk_mq_flush_plug_list(struct blk_plug * plug,bool from_schedule)2803 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
2804 {
2805 struct request *rq;
2806
2807 /*
2808 * We may have been called recursively midway through handling
2809 * plug->mq_list via a schedule() in the driver's queue_rq() callback.
2810 * To avoid mq_list changing under our feet, clear rq_count early and
2811 * bail out specifically if rq_count is 0 rather than checking
2812 * whether the mq_list is empty.
2813 */
2814 if (plug->rq_count == 0)
2815 return;
2816 plug->rq_count = 0;
2817
2818 if (!plug->multiple_queues && !plug->has_elevator && !from_schedule) {
2819 struct request_queue *q;
2820
2821 rq = rq_list_peek(&plug->mq_list);
2822 q = rq->q;
2823
2824 /*
2825 * Peek first request and see if we have a ->queue_rqs() hook.
2826 * If we do, we can dispatch the whole plug list in one go. We
2827 * already know at this point that all requests belong to the
2828 * same queue, caller must ensure that's the case.
2829 *
2830 * Since we pass off the full list to the driver at this point,
2831 * we do not increment the active request count for the queue.
2832 * Bypass shared tags for now because of that.
2833 */
2834 if (q->mq_ops->queue_rqs &&
2835 !(rq->mq_hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
2836 blk_mq_run_dispatch_ops(q,
2837 __blk_mq_flush_plug_list(q, plug));
2838 if (rq_list_empty(plug->mq_list))
2839 return;
2840 }
2841
2842 blk_mq_run_dispatch_ops(q,
2843 blk_mq_plug_issue_direct(plug));
2844 if (rq_list_empty(plug->mq_list))
2845 return;
2846 }
2847
2848 do {
2849 blk_mq_dispatch_plug_list(plug, from_schedule);
2850 } while (!rq_list_empty(plug->mq_list));
2851 }
2852
blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx * hctx,struct list_head * list)2853 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
2854 struct list_head *list)
2855 {
2856 int queued = 0;
2857 blk_status_t ret = BLK_STS_OK;
2858
2859 while (!list_empty(list)) {
2860 struct request *rq = list_first_entry(list, struct request,
2861 queuelist);
2862
2863 list_del_init(&rq->queuelist);
2864 ret = blk_mq_request_issue_directly(rq, list_empty(list));
2865 switch (ret) {
2866 case BLK_STS_OK:
2867 queued++;
2868 break;
2869 case BLK_STS_RESOURCE:
2870 case BLK_STS_DEV_RESOURCE:
2871 blk_mq_request_bypass_insert(rq, 0);
2872 if (list_empty(list))
2873 blk_mq_run_hw_queue(hctx, false);
2874 goto out;
2875 default:
2876 blk_mq_end_request(rq, ret);
2877 break;
2878 }
2879 }
2880
2881 out:
2882 if (ret != BLK_STS_OK)
2883 blk_mq_commit_rqs(hctx, queued, false);
2884 }
2885
blk_mq_attempt_bio_merge(struct request_queue * q,struct bio * bio,unsigned int nr_segs)2886 static bool blk_mq_attempt_bio_merge(struct request_queue *q,
2887 struct bio *bio, unsigned int nr_segs)
2888 {
2889 if (!blk_queue_nomerges(q) && bio_mergeable(bio)) {
2890 if (blk_attempt_plug_merge(q, bio, nr_segs))
2891 return true;
2892 if (blk_mq_sched_bio_merge(q, bio, nr_segs))
2893 return true;
2894 }
2895 return false;
2896 }
2897
blk_mq_get_new_requests(struct request_queue * q,struct blk_plug * plug,struct bio * bio,unsigned int nsegs)2898 static struct request *blk_mq_get_new_requests(struct request_queue *q,
2899 struct blk_plug *plug,
2900 struct bio *bio,
2901 unsigned int nsegs)
2902 {
2903 struct blk_mq_alloc_data data = {
2904 .q = q,
2905 .nr_tags = 1,
2906 .cmd_flags = bio->bi_opf,
2907 };
2908 struct request *rq;
2909
2910 if (blk_mq_attempt_bio_merge(q, bio, nsegs))
2911 return NULL;
2912
2913 rq_qos_throttle(q, bio);
2914
2915 if (plug) {
2916 data.nr_tags = plug->nr_ios;
2917 plug->nr_ios = 1;
2918 data.cached_rq = &plug->cached_rq;
2919 }
2920
2921 rq = __blk_mq_alloc_requests(&data);
2922 if (rq)
2923 return rq;
2924 rq_qos_cleanup(q, bio);
2925 if (bio->bi_opf & REQ_NOWAIT)
2926 bio_wouldblock_error(bio);
2927 return NULL;
2928 }
2929
2930 /* return true if this @rq can be used for @bio */
blk_mq_can_use_cached_rq(struct request * rq,struct blk_plug * plug,struct bio * bio)2931 static bool blk_mq_can_use_cached_rq(struct request *rq, struct blk_plug *plug,
2932 struct bio *bio)
2933 {
2934 enum hctx_type type = blk_mq_get_hctx_type(bio->bi_opf);
2935 enum hctx_type hctx_type = rq->mq_hctx->type;
2936
2937 WARN_ON_ONCE(rq_list_peek(&plug->cached_rq) != rq);
2938
2939 if (type != hctx_type &&
2940 !(type == HCTX_TYPE_READ && hctx_type == HCTX_TYPE_DEFAULT))
2941 return false;
2942 if (op_is_flush(rq->cmd_flags) != op_is_flush(bio->bi_opf))
2943 return false;
2944
2945 /*
2946 * If any qos ->throttle() end up blocking, we will have flushed the
2947 * plug and hence killed the cached_rq list as well. Pop this entry
2948 * before we throttle.
2949 */
2950 plug->cached_rq = rq_list_next(rq);
2951 rq_qos_throttle(rq->q, bio);
2952
2953 blk_mq_rq_time_init(rq, 0);
2954 rq->cmd_flags = bio->bi_opf;
2955 INIT_LIST_HEAD(&rq->queuelist);
2956 return true;
2957 }
2958
bio_set_ioprio(struct bio * bio)2959 static void bio_set_ioprio(struct bio *bio)
2960 {
2961 /* Nobody set ioprio so far? Initialize it based on task's nice value */
2962 if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) == IOPRIO_CLASS_NONE)
2963 bio->bi_ioprio = get_current_ioprio();
2964 blkcg_set_ioprio(bio);
2965 }
2966
2967 /**
2968 * blk_mq_submit_bio - Create and send a request to block device.
2969 * @bio: Bio pointer.
2970 *
2971 * Builds up a request structure from @q and @bio and send to the device. The
2972 * request may not be queued directly to hardware if:
2973 * * This request can be merged with another one
2974 * * We want to place request at plug queue for possible future merging
2975 * * There is an IO scheduler active at this queue
2976 *
2977 * It will not queue the request if there is an error with the bio, or at the
2978 * request creation.
2979 */
blk_mq_submit_bio(struct bio * bio)2980 void blk_mq_submit_bio(struct bio *bio)
2981 {
2982 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
2983 struct blk_plug *plug = blk_mq_plug(bio);
2984 const int is_sync = op_is_sync(bio->bi_opf);
2985 struct blk_mq_hw_ctx *hctx;
2986 struct request *rq = NULL;
2987 unsigned int nr_segs = 1;
2988 blk_status_t ret;
2989
2990 bio = blk_queue_bounce(bio, q);
2991 bio_set_ioprio(bio);
2992
2993 if (plug) {
2994 rq = rq_list_peek(&plug->cached_rq);
2995 if (rq && rq->q != q)
2996 rq = NULL;
2997 }
2998 if (rq) {
2999 if (unlikely(bio_may_exceed_limits(bio, &q->limits))) {
3000 bio = __bio_split_to_limits(bio, &q->limits, &nr_segs);
3001 if (!bio)
3002 return;
3003 }
3004 if (!bio_integrity_prep(bio))
3005 return;
3006 if (blk_mq_attempt_bio_merge(q, bio, nr_segs))
3007 return;
3008 if (blk_mq_can_use_cached_rq(rq, plug, bio))
3009 goto done;
3010 percpu_ref_get(&q->q_usage_counter);
3011 } else {
3012 if (unlikely(bio_queue_enter(bio)))
3013 return;
3014 if (unlikely(bio_may_exceed_limits(bio, &q->limits))) {
3015 bio = __bio_split_to_limits(bio, &q->limits, &nr_segs);
3016 if (!bio)
3017 goto fail;
3018 }
3019 if (!bio_integrity_prep(bio))
3020 goto fail;
3021 }
3022
3023 rq = blk_mq_get_new_requests(q, plug, bio, nr_segs);
3024 if (unlikely(!rq)) {
3025 fail:
3026 blk_queue_exit(q);
3027 return;
3028 }
3029
3030 done:
3031 trace_block_getrq(bio);
3032
3033 rq_qos_track(q, rq, bio);
3034
3035 blk_mq_bio_to_request(rq, bio, nr_segs);
3036
3037 ret = blk_crypto_rq_get_keyslot(rq);
3038 if (ret != BLK_STS_OK) {
3039 bio->bi_status = ret;
3040 bio_endio(bio);
3041 blk_mq_free_request(rq);
3042 return;
3043 }
3044
3045 if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq))
3046 return;
3047
3048 if (plug) {
3049 blk_add_rq_to_plug(plug, rq);
3050 return;
3051 }
3052
3053 hctx = rq->mq_hctx;
3054 if ((rq->rq_flags & RQF_USE_SCHED) ||
3055 (hctx->dispatch_busy && (q->nr_hw_queues == 1 || !is_sync))) {
3056 blk_mq_insert_request(rq, 0);
3057 blk_mq_run_hw_queue(hctx, true);
3058 } else {
3059 blk_mq_run_dispatch_ops(q, blk_mq_try_issue_directly(hctx, rq));
3060 }
3061 }
3062
3063 #ifdef CONFIG_BLK_MQ_STACKING
3064 /**
3065 * blk_insert_cloned_request - Helper for stacking drivers to submit a request
3066 * @rq: the request being queued
3067 */
blk_insert_cloned_request(struct request * rq)3068 blk_status_t blk_insert_cloned_request(struct request *rq)
3069 {
3070 struct request_queue *q = rq->q;
3071 unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq));
3072 unsigned int max_segments = blk_rq_get_max_segments(rq);
3073 blk_status_t ret;
3074
3075 if (blk_rq_sectors(rq) > max_sectors) {
3076 /*
3077 * SCSI device does not have a good way to return if
3078 * Write Same/Zero is actually supported. If a device rejects
3079 * a non-read/write command (discard, write same,etc.) the
3080 * low-level device driver will set the relevant queue limit to
3081 * 0 to prevent blk-lib from issuing more of the offending
3082 * operations. Commands queued prior to the queue limit being
3083 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O
3084 * errors being propagated to upper layers.
3085 */
3086 if (max_sectors == 0)
3087 return BLK_STS_NOTSUPP;
3088
3089 printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
3090 __func__, blk_rq_sectors(rq), max_sectors);
3091 return BLK_STS_IOERR;
3092 }
3093
3094 /*
3095 * The queue settings related to segment counting may differ from the
3096 * original queue.
3097 */
3098 rq->nr_phys_segments = blk_recalc_rq_segments(rq);
3099 if (rq->nr_phys_segments > max_segments) {
3100 printk(KERN_ERR "%s: over max segments limit. (%u > %u)\n",
3101 __func__, rq->nr_phys_segments, max_segments);
3102 return BLK_STS_IOERR;
3103 }
3104
3105 if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq)))
3106 return BLK_STS_IOERR;
3107
3108 ret = blk_crypto_rq_get_keyslot(rq);
3109 if (ret != BLK_STS_OK)
3110 return ret;
3111
3112 blk_account_io_start(rq);
3113
3114 /*
3115 * Since we have a scheduler attached on the top device,
3116 * bypass a potential scheduler on the bottom device for
3117 * insert.
3118 */
3119 blk_mq_run_dispatch_ops(q,
3120 ret = blk_mq_request_issue_directly(rq, true));
3121 if (ret)
3122 blk_account_io_done(rq, ktime_get_ns());
3123 return ret;
3124 }
3125 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
3126
3127 /**
3128 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
3129 * @rq: the clone request to be cleaned up
3130 *
3131 * Description:
3132 * Free all bios in @rq for a cloned request.
3133 */
blk_rq_unprep_clone(struct request * rq)3134 void blk_rq_unprep_clone(struct request *rq)
3135 {
3136 struct bio *bio;
3137
3138 while ((bio = rq->bio) != NULL) {
3139 rq->bio = bio->bi_next;
3140
3141 bio_put(bio);
3142 }
3143 }
3144 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
3145
3146 /**
3147 * blk_rq_prep_clone - Helper function to setup clone request
3148 * @rq: the request to be setup
3149 * @rq_src: original request to be cloned
3150 * @bs: bio_set that bios for clone are allocated from
3151 * @gfp_mask: memory allocation mask for bio
3152 * @bio_ctr: setup function to be called for each clone bio.
3153 * Returns %0 for success, non %0 for failure.
3154 * @data: private data to be passed to @bio_ctr
3155 *
3156 * Description:
3157 * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
3158 * Also, pages which the original bios are pointing to are not copied
3159 * and the cloned bios just point same pages.
3160 * So cloned bios must be completed before original bios, which means
3161 * the caller must complete @rq before @rq_src.
3162 */
blk_rq_prep_clone(struct request * rq,struct request * rq_src,struct bio_set * bs,gfp_t gfp_mask,int (* bio_ctr)(struct bio *,struct bio *,void *),void * data)3163 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
3164 struct bio_set *bs, gfp_t gfp_mask,
3165 int (*bio_ctr)(struct bio *, struct bio *, void *),
3166 void *data)
3167 {
3168 struct bio *bio, *bio_src;
3169
3170 if (!bs)
3171 bs = &fs_bio_set;
3172
3173 __rq_for_each_bio(bio_src, rq_src) {
3174 bio = bio_alloc_clone(rq->q->disk->part0, bio_src, gfp_mask,
3175 bs);
3176 if (!bio)
3177 goto free_and_out;
3178
3179 if (bio_ctr && bio_ctr(bio, bio_src, data))
3180 goto free_and_out;
3181
3182 if (rq->bio) {
3183 rq->biotail->bi_next = bio;
3184 rq->biotail = bio;
3185 } else {
3186 rq->bio = rq->biotail = bio;
3187 }
3188 bio = NULL;
3189 }
3190
3191 /* Copy attributes of the original request to the clone request. */
3192 rq->__sector = blk_rq_pos(rq_src);
3193 rq->__data_len = blk_rq_bytes(rq_src);
3194 if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) {
3195 rq->rq_flags |= RQF_SPECIAL_PAYLOAD;
3196 rq->special_vec = rq_src->special_vec;
3197 }
3198 rq->nr_phys_segments = rq_src->nr_phys_segments;
3199 rq->ioprio = rq_src->ioprio;
3200
3201 if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
3202 goto free_and_out;
3203
3204 return 0;
3205
3206 free_and_out:
3207 if (bio)
3208 bio_put(bio);
3209 blk_rq_unprep_clone(rq);
3210
3211 return -ENOMEM;
3212 }
3213 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
3214 #endif /* CONFIG_BLK_MQ_STACKING */
3215
3216 /*
3217 * Steal bios from a request and add them to a bio list.
3218 * The request must not have been partially completed before.
3219 */
blk_steal_bios(struct bio_list * list,struct request * rq)3220 void blk_steal_bios(struct bio_list *list, struct request *rq)
3221 {
3222 if (rq->bio) {
3223 if (list->tail)
3224 list->tail->bi_next = rq->bio;
3225 else
3226 list->head = rq->bio;
3227 list->tail = rq->biotail;
3228
3229 rq->bio = NULL;
3230 rq->biotail = NULL;
3231 }
3232
3233 rq->__data_len = 0;
3234 }
3235 EXPORT_SYMBOL_GPL(blk_steal_bios);
3236
order_to_size(unsigned int order)3237 static size_t order_to_size(unsigned int order)
3238 {
3239 return (size_t)PAGE_SIZE << order;
3240 }
3241
3242 /* called before freeing request pool in @tags */
blk_mq_clear_rq_mapping(struct blk_mq_tags * drv_tags,struct blk_mq_tags * tags)3243 static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags,
3244 struct blk_mq_tags *tags)
3245 {
3246 struct page *page;
3247 unsigned long flags;
3248
3249 /*
3250 * There is no need to clear mapping if driver tags is not initialized
3251 * or the mapping belongs to the driver tags.
3252 */
3253 if (!drv_tags || drv_tags == tags)
3254 return;
3255
3256 list_for_each_entry(page, &tags->page_list, lru) {
3257 unsigned long start = (unsigned long)page_address(page);
3258 unsigned long end = start + order_to_size(page->private);
3259 int i;
3260
3261 for (i = 0; i < drv_tags->nr_tags; i++) {
3262 struct request *rq = drv_tags->rqs[i];
3263 unsigned long rq_addr = (unsigned long)rq;
3264
3265 if (rq_addr >= start && rq_addr < end) {
3266 WARN_ON_ONCE(req_ref_read(rq) != 0);
3267 cmpxchg(&drv_tags->rqs[i], rq, NULL);
3268 }
3269 }
3270 }
3271
3272 /*
3273 * Wait until all pending iteration is done.
3274 *
3275 * Request reference is cleared and it is guaranteed to be observed
3276 * after the ->lock is released.
3277 */
3278 spin_lock_irqsave(&drv_tags->lock, flags);
3279 spin_unlock_irqrestore(&drv_tags->lock, flags);
3280 }
3281
blk_mq_free_rqs(struct blk_mq_tag_set * set,struct blk_mq_tags * tags,unsigned int hctx_idx)3282 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
3283 unsigned int hctx_idx)
3284 {
3285 struct blk_mq_tags *drv_tags;
3286 struct page *page;
3287
3288 if (list_empty(&tags->page_list))
3289 return;
3290
3291 if (blk_mq_is_shared_tags(set->flags))
3292 drv_tags = set->shared_tags;
3293 else
3294 drv_tags = set->tags[hctx_idx];
3295
3296 if (tags->static_rqs && set->ops->exit_request) {
3297 int i;
3298
3299 for (i = 0; i < tags->nr_tags; i++) {
3300 struct request *rq = tags->static_rqs[i];
3301
3302 if (!rq)
3303 continue;
3304 set->ops->exit_request(set, rq, hctx_idx);
3305 tags->static_rqs[i] = NULL;
3306 }
3307 }
3308
3309 blk_mq_clear_rq_mapping(drv_tags, tags);
3310
3311 while (!list_empty(&tags->page_list)) {
3312 page = list_first_entry(&tags->page_list, struct page, lru);
3313 list_del_init(&page->lru);
3314 /*
3315 * Remove kmemleak object previously allocated in
3316 * blk_mq_alloc_rqs().
3317 */
3318 kmemleak_free(page_address(page));
3319 __free_pages(page, page->private);
3320 }
3321 }
3322
blk_mq_free_rq_map(struct blk_mq_tags * tags)3323 void blk_mq_free_rq_map(struct blk_mq_tags *tags)
3324 {
3325 kfree(tags->rqs);
3326 tags->rqs = NULL;
3327 kfree(tags->static_rqs);
3328 tags->static_rqs = NULL;
3329
3330 blk_mq_free_tags(tags);
3331 }
3332
hctx_idx_to_type(struct blk_mq_tag_set * set,unsigned int hctx_idx)3333 static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set,
3334 unsigned int hctx_idx)
3335 {
3336 int i;
3337
3338 for (i = 0; i < set->nr_maps; i++) {
3339 unsigned int start = set->map[i].queue_offset;
3340 unsigned int end = start + set->map[i].nr_queues;
3341
3342 if (hctx_idx >= start && hctx_idx < end)
3343 break;
3344 }
3345
3346 if (i >= set->nr_maps)
3347 i = HCTX_TYPE_DEFAULT;
3348
3349 return i;
3350 }
3351
blk_mq_get_hctx_node(struct blk_mq_tag_set * set,unsigned int hctx_idx)3352 static int blk_mq_get_hctx_node(struct blk_mq_tag_set *set,
3353 unsigned int hctx_idx)
3354 {
3355 enum hctx_type type = hctx_idx_to_type(set, hctx_idx);
3356
3357 return blk_mq_hw_queue_to_node(&set->map[type], hctx_idx);
3358 }
3359
blk_mq_alloc_rq_map(struct blk_mq_tag_set * set,unsigned int hctx_idx,unsigned int nr_tags,unsigned int reserved_tags)3360 static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
3361 unsigned int hctx_idx,
3362 unsigned int nr_tags,
3363 unsigned int reserved_tags)
3364 {
3365 int node = blk_mq_get_hctx_node(set, hctx_idx);
3366 struct blk_mq_tags *tags;
3367
3368 if (node == NUMA_NO_NODE)
3369 node = set->numa_node;
3370
3371 tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
3372 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
3373 if (!tags)
3374 return NULL;
3375
3376 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
3377 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
3378 node);
3379 if (!tags->rqs)
3380 goto err_free_tags;
3381
3382 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
3383 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
3384 node);
3385 if (!tags->static_rqs)
3386 goto err_free_rqs;
3387
3388 return tags;
3389
3390 err_free_rqs:
3391 kfree(tags->rqs);
3392 err_free_tags:
3393 blk_mq_free_tags(tags);
3394 return NULL;
3395 }
3396
blk_mq_init_request(struct blk_mq_tag_set * set,struct request * rq,unsigned int hctx_idx,int node)3397 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
3398 unsigned int hctx_idx, int node)
3399 {
3400 int ret;
3401
3402 if (set->ops->init_request) {
3403 ret = set->ops->init_request(set, rq, hctx_idx, node);
3404 if (ret)
3405 return ret;
3406 }
3407
3408 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
3409 return 0;
3410 }
3411
blk_mq_alloc_rqs(struct blk_mq_tag_set * set,struct blk_mq_tags * tags,unsigned int hctx_idx,unsigned int depth)3412 static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set,
3413 struct blk_mq_tags *tags,
3414 unsigned int hctx_idx, unsigned int depth)
3415 {
3416 unsigned int i, j, entries_per_page, max_order = 4;
3417 int node = blk_mq_get_hctx_node(set, hctx_idx);
3418 size_t rq_size, left;
3419
3420 if (node == NUMA_NO_NODE)
3421 node = set->numa_node;
3422
3423 INIT_LIST_HEAD(&tags->page_list);
3424
3425 /*
3426 * rq_size is the size of the request plus driver payload, rounded
3427 * to the cacheline size
3428 */
3429 rq_size = round_up(sizeof(struct request) + set->cmd_size,
3430 cache_line_size());
3431 left = rq_size * depth;
3432
3433 for (i = 0; i < depth; ) {
3434 int this_order = max_order;
3435 struct page *page;
3436 int to_do;
3437 void *p;
3438
3439 while (this_order && left < order_to_size(this_order - 1))
3440 this_order--;
3441
3442 do {
3443 page = alloc_pages_node(node,
3444 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
3445 this_order);
3446 if (page)
3447 break;
3448 if (!this_order--)
3449 break;
3450 if (order_to_size(this_order) < rq_size)
3451 break;
3452 } while (1);
3453
3454 if (!page)
3455 goto fail;
3456
3457 page->private = this_order;
3458 list_add_tail(&page->lru, &tags->page_list);
3459
3460 p = page_address(page);
3461 /*
3462 * Allow kmemleak to scan these pages as they contain pointers
3463 * to additional allocations like via ops->init_request().
3464 */
3465 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
3466 entries_per_page = order_to_size(this_order) / rq_size;
3467 to_do = min(entries_per_page, depth - i);
3468 left -= to_do * rq_size;
3469 for (j = 0; j < to_do; j++) {
3470 struct request *rq = p;
3471
3472 tags->static_rqs[i] = rq;
3473 if (blk_mq_init_request(set, rq, hctx_idx, node)) {
3474 tags->static_rqs[i] = NULL;
3475 goto fail;
3476 }
3477
3478 p += rq_size;
3479 i++;
3480 }
3481 }
3482 return 0;
3483
3484 fail:
3485 blk_mq_free_rqs(set, tags, hctx_idx);
3486 return -ENOMEM;
3487 }
3488
3489 struct rq_iter_data {
3490 struct blk_mq_hw_ctx *hctx;
3491 bool has_rq;
3492 };
3493
blk_mq_has_request(struct request * rq,void * data)3494 static bool blk_mq_has_request(struct request *rq, void *data)
3495 {
3496 struct rq_iter_data *iter_data = data;
3497
3498 if (rq->mq_hctx != iter_data->hctx)
3499 return true;
3500 iter_data->has_rq = true;
3501 return false;
3502 }
3503
blk_mq_hctx_has_requests(struct blk_mq_hw_ctx * hctx)3504 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
3505 {
3506 struct blk_mq_tags *tags = hctx->sched_tags ?
3507 hctx->sched_tags : hctx->tags;
3508 struct rq_iter_data data = {
3509 .hctx = hctx,
3510 };
3511
3512 blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
3513 return data.has_rq;
3514 }
3515
blk_mq_last_cpu_in_hctx(unsigned int cpu,struct blk_mq_hw_ctx * hctx)3516 static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu,
3517 struct blk_mq_hw_ctx *hctx)
3518 {
3519 if (cpumask_first_and(hctx->cpumask, cpu_online_mask) != cpu)
3520 return false;
3521 if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids)
3522 return false;
3523 return true;
3524 }
3525
blk_mq_hctx_notify_offline(unsigned int cpu,struct hlist_node * node)3526 static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
3527 {
3528 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
3529 struct blk_mq_hw_ctx, cpuhp_online);
3530
3531 if (!cpumask_test_cpu(cpu, hctx->cpumask) ||
3532 !blk_mq_last_cpu_in_hctx(cpu, hctx))
3533 return 0;
3534
3535 /*
3536 * Prevent new request from being allocated on the current hctx.
3537 *
3538 * The smp_mb__after_atomic() Pairs with the implied barrier in
3539 * test_and_set_bit_lock in sbitmap_get(). Ensures the inactive flag is
3540 * seen once we return from the tag allocator.
3541 */
3542 set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3543 smp_mb__after_atomic();
3544
3545 /*
3546 * Try to grab a reference to the queue and wait for any outstanding
3547 * requests. If we could not grab a reference the queue has been
3548 * frozen and there are no requests.
3549 */
3550 if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
3551 while (blk_mq_hctx_has_requests(hctx))
3552 msleep(5);
3553 percpu_ref_put(&hctx->queue->q_usage_counter);
3554 }
3555
3556 return 0;
3557 }
3558
blk_mq_hctx_notify_online(unsigned int cpu,struct hlist_node * node)3559 static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
3560 {
3561 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
3562 struct blk_mq_hw_ctx, cpuhp_online);
3563
3564 if (cpumask_test_cpu(cpu, hctx->cpumask))
3565 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3566 return 0;
3567 }
3568
3569 /*
3570 * 'cpu' is going away. splice any existing rq_list entries from this
3571 * software queue to the hw queue dispatch list, and ensure that it
3572 * gets run.
3573 */
blk_mq_hctx_notify_dead(unsigned int cpu,struct hlist_node * node)3574 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
3575 {
3576 struct blk_mq_hw_ctx *hctx;
3577 struct blk_mq_ctx *ctx;
3578 LIST_HEAD(tmp);
3579 enum hctx_type type;
3580
3581 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
3582 if (!cpumask_test_cpu(cpu, hctx->cpumask))
3583 return 0;
3584
3585 ctx = __blk_mq_get_ctx(hctx->queue, cpu);
3586 type = hctx->type;
3587
3588 spin_lock(&ctx->lock);
3589 if (!list_empty(&ctx->rq_lists[type])) {
3590 list_splice_init(&ctx->rq_lists[type], &tmp);
3591 blk_mq_hctx_clear_pending(hctx, ctx);
3592 }
3593 spin_unlock(&ctx->lock);
3594
3595 if (list_empty(&tmp))
3596 return 0;
3597
3598 spin_lock(&hctx->lock);
3599 list_splice_tail_init(&tmp, &hctx->dispatch);
3600 spin_unlock(&hctx->lock);
3601
3602 blk_mq_run_hw_queue(hctx, true);
3603 return 0;
3604 }
3605
blk_mq_remove_cpuhp(struct blk_mq_hw_ctx * hctx)3606 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
3607 {
3608 if (!(hctx->flags & BLK_MQ_F_STACKING))
3609 cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
3610 &hctx->cpuhp_online);
3611 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
3612 &hctx->cpuhp_dead);
3613 }
3614
3615 /*
3616 * Before freeing hw queue, clearing the flush request reference in
3617 * tags->rqs[] for avoiding potential UAF.
3618 */
blk_mq_clear_flush_rq_mapping(struct blk_mq_tags * tags,unsigned int queue_depth,struct request * flush_rq)3619 static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags,
3620 unsigned int queue_depth, struct request *flush_rq)
3621 {
3622 int i;
3623 unsigned long flags;
3624
3625 /* The hw queue may not be mapped yet */
3626 if (!tags)
3627 return;
3628
3629 WARN_ON_ONCE(req_ref_read(flush_rq) != 0);
3630
3631 for (i = 0; i < queue_depth; i++)
3632 cmpxchg(&tags->rqs[i], flush_rq, NULL);
3633
3634 /*
3635 * Wait until all pending iteration is done.
3636 *
3637 * Request reference is cleared and it is guaranteed to be observed
3638 * after the ->lock is released.
3639 */
3640 spin_lock_irqsave(&tags->lock, flags);
3641 spin_unlock_irqrestore(&tags->lock, flags);
3642 }
3643
3644 /* hctx->ctxs will be freed in queue's release handler */
blk_mq_exit_hctx(struct request_queue * q,struct blk_mq_tag_set * set,struct blk_mq_hw_ctx * hctx,unsigned int hctx_idx)3645 static void blk_mq_exit_hctx(struct request_queue *q,
3646 struct blk_mq_tag_set *set,
3647 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
3648 {
3649 struct request *flush_rq = hctx->fq->flush_rq;
3650
3651 if (blk_mq_hw_queue_mapped(hctx))
3652 blk_mq_tag_idle(hctx);
3653
3654 if (blk_queue_init_done(q))
3655 blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx],
3656 set->queue_depth, flush_rq);
3657 if (set->ops->exit_request)
3658 set->ops->exit_request(set, flush_rq, hctx_idx);
3659
3660 if (set->ops->exit_hctx)
3661 set->ops->exit_hctx(hctx, hctx_idx);
3662
3663 blk_mq_remove_cpuhp(hctx);
3664
3665 xa_erase(&q->hctx_table, hctx_idx);
3666
3667 spin_lock(&q->unused_hctx_lock);
3668 list_add(&hctx->hctx_list, &q->unused_hctx_list);
3669 spin_unlock(&q->unused_hctx_lock);
3670 }
3671
blk_mq_exit_hw_queues(struct request_queue * q,struct blk_mq_tag_set * set,int nr_queue)3672 static void blk_mq_exit_hw_queues(struct request_queue *q,
3673 struct blk_mq_tag_set *set, int nr_queue)
3674 {
3675 struct blk_mq_hw_ctx *hctx;
3676 unsigned long i;
3677
3678 queue_for_each_hw_ctx(q, hctx, i) {
3679 if (i == nr_queue)
3680 break;
3681 blk_mq_exit_hctx(q, set, hctx, i);
3682 }
3683 }
3684
blk_mq_init_hctx(struct request_queue * q,struct blk_mq_tag_set * set,struct blk_mq_hw_ctx * hctx,unsigned hctx_idx)3685 static int blk_mq_init_hctx(struct request_queue *q,
3686 struct blk_mq_tag_set *set,
3687 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
3688 {
3689 hctx->queue_num = hctx_idx;
3690
3691 if (!(hctx->flags & BLK_MQ_F_STACKING))
3692 cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
3693 &hctx->cpuhp_online);
3694 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
3695
3696 hctx->tags = set->tags[hctx_idx];
3697
3698 if (set->ops->init_hctx &&
3699 set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
3700 goto unregister_cpu_notifier;
3701
3702 if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
3703 hctx->numa_node))
3704 goto exit_hctx;
3705
3706 if (xa_insert(&q->hctx_table, hctx_idx, hctx, GFP_KERNEL))
3707 goto exit_flush_rq;
3708
3709 return 0;
3710
3711 exit_flush_rq:
3712 if (set->ops->exit_request)
3713 set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx);
3714 exit_hctx:
3715 if (set->ops->exit_hctx)
3716 set->ops->exit_hctx(hctx, hctx_idx);
3717 unregister_cpu_notifier:
3718 blk_mq_remove_cpuhp(hctx);
3719 return -1;
3720 }
3721
3722 static struct blk_mq_hw_ctx *
blk_mq_alloc_hctx(struct request_queue * q,struct blk_mq_tag_set * set,int node)3723 blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
3724 int node)
3725 {
3726 struct blk_mq_hw_ctx *hctx;
3727 gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
3728
3729 hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node);
3730 if (!hctx)
3731 goto fail_alloc_hctx;
3732
3733 if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
3734 goto free_hctx;
3735
3736 atomic_set(&hctx->nr_active, 0);
3737 if (node == NUMA_NO_NODE)
3738 node = set->numa_node;
3739 hctx->numa_node = node;
3740
3741 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
3742 spin_lock_init(&hctx->lock);
3743 INIT_LIST_HEAD(&hctx->dispatch);
3744 hctx->queue = q;
3745 hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
3746
3747 INIT_LIST_HEAD(&hctx->hctx_list);
3748
3749 /*
3750 * Allocate space for all possible cpus to avoid allocation at
3751 * runtime
3752 */
3753 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),
3754 gfp, node);
3755 if (!hctx->ctxs)
3756 goto free_cpumask;
3757
3758 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
3759 gfp, node, false, false))
3760 goto free_ctxs;
3761 hctx->nr_ctx = 0;
3762
3763 spin_lock_init(&hctx->dispatch_wait_lock);
3764 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
3765 INIT_LIST_HEAD(&hctx->dispatch_wait.entry);
3766
3767 hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp);
3768 if (!hctx->fq)
3769 goto free_bitmap;
3770
3771 blk_mq_hctx_kobj_init(hctx);
3772
3773 return hctx;
3774
3775 free_bitmap:
3776 sbitmap_free(&hctx->ctx_map);
3777 free_ctxs:
3778 kfree(hctx->ctxs);
3779 free_cpumask:
3780 free_cpumask_var(hctx->cpumask);
3781 free_hctx:
3782 kfree(hctx);
3783 fail_alloc_hctx:
3784 return NULL;
3785 }
3786
blk_mq_init_cpu_queues(struct request_queue * q,unsigned int nr_hw_queues)3787 static void blk_mq_init_cpu_queues(struct request_queue *q,
3788 unsigned int nr_hw_queues)
3789 {
3790 struct blk_mq_tag_set *set = q->tag_set;
3791 unsigned int i, j;
3792
3793 for_each_possible_cpu(i) {
3794 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
3795 struct blk_mq_hw_ctx *hctx;
3796 int k;
3797
3798 __ctx->cpu = i;
3799 spin_lock_init(&__ctx->lock);
3800 for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++)
3801 INIT_LIST_HEAD(&__ctx->rq_lists[k]);
3802
3803 __ctx->queue = q;
3804
3805 /*
3806 * Set local node, IFF we have more than one hw queue. If
3807 * not, we remain on the home node of the device
3808 */
3809 for (j = 0; j < set->nr_maps; j++) {
3810 hctx = blk_mq_map_queue_type(q, j, i);
3811 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
3812 hctx->numa_node = cpu_to_node(i);
3813 }
3814 }
3815 }
3816
blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set * set,unsigned int hctx_idx,unsigned int depth)3817 struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
3818 unsigned int hctx_idx,
3819 unsigned int depth)
3820 {
3821 struct blk_mq_tags *tags;
3822 int ret;
3823
3824 tags = blk_mq_alloc_rq_map(set, hctx_idx, depth, set->reserved_tags);
3825 if (!tags)
3826 return NULL;
3827
3828 ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth);
3829 if (ret) {
3830 blk_mq_free_rq_map(tags);
3831 return NULL;
3832 }
3833
3834 return tags;
3835 }
3836
__blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set * set,int hctx_idx)3837 static bool __blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
3838 int hctx_idx)
3839 {
3840 if (blk_mq_is_shared_tags(set->flags)) {
3841 set->tags[hctx_idx] = set->shared_tags;
3842
3843 return true;
3844 }
3845
3846 set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs(set, hctx_idx,
3847 set->queue_depth);
3848
3849 return set->tags[hctx_idx];
3850 }
3851
blk_mq_free_map_and_rqs(struct blk_mq_tag_set * set,struct blk_mq_tags * tags,unsigned int hctx_idx)3852 void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
3853 struct blk_mq_tags *tags,
3854 unsigned int hctx_idx)
3855 {
3856 if (tags) {
3857 blk_mq_free_rqs(set, tags, hctx_idx);
3858 blk_mq_free_rq_map(tags);
3859 }
3860 }
3861
__blk_mq_free_map_and_rqs(struct blk_mq_tag_set * set,unsigned int hctx_idx)3862 static void __blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
3863 unsigned int hctx_idx)
3864 {
3865 if (!blk_mq_is_shared_tags(set->flags))
3866 blk_mq_free_map_and_rqs(set, set->tags[hctx_idx], hctx_idx);
3867
3868 set->tags[hctx_idx] = NULL;
3869 }
3870
blk_mq_map_swqueue(struct request_queue * q)3871 static void blk_mq_map_swqueue(struct request_queue *q)
3872 {
3873 unsigned int j, hctx_idx;
3874 unsigned long i;
3875 struct blk_mq_hw_ctx *hctx;
3876 struct blk_mq_ctx *ctx;
3877 struct blk_mq_tag_set *set = q->tag_set;
3878
3879 queue_for_each_hw_ctx(q, hctx, i) {
3880 cpumask_clear(hctx->cpumask);
3881 hctx->nr_ctx = 0;
3882 hctx->dispatch_from = NULL;
3883 }
3884
3885 /*
3886 * Map software to hardware queues.
3887 *
3888 * If the cpu isn't present, the cpu is mapped to first hctx.
3889 */
3890 for_each_possible_cpu(i) {
3891
3892 ctx = per_cpu_ptr(q->queue_ctx, i);
3893 for (j = 0; j < set->nr_maps; j++) {
3894 if (!set->map[j].nr_queues) {
3895 ctx->hctxs[j] = blk_mq_map_queue_type(q,
3896 HCTX_TYPE_DEFAULT, i);
3897 continue;
3898 }
3899 hctx_idx = set->map[j].mq_map[i];
3900 /* unmapped hw queue can be remapped after CPU topo changed */
3901 if (!set->tags[hctx_idx] &&
3902 !__blk_mq_alloc_map_and_rqs(set, hctx_idx)) {
3903 /*
3904 * If tags initialization fail for some hctx,
3905 * that hctx won't be brought online. In this
3906 * case, remap the current ctx to hctx[0] which
3907 * is guaranteed to always have tags allocated
3908 */
3909 set->map[j].mq_map[i] = 0;
3910 }
3911
3912 hctx = blk_mq_map_queue_type(q, j, i);
3913 ctx->hctxs[j] = hctx;
3914 /*
3915 * If the CPU is already set in the mask, then we've
3916 * mapped this one already. This can happen if
3917 * devices share queues across queue maps.
3918 */
3919 if (cpumask_test_cpu(i, hctx->cpumask))
3920 continue;
3921
3922 cpumask_set_cpu(i, hctx->cpumask);
3923 hctx->type = j;
3924 ctx->index_hw[hctx->type] = hctx->nr_ctx;
3925 hctx->ctxs[hctx->nr_ctx++] = ctx;
3926
3927 /*
3928 * If the nr_ctx type overflows, we have exceeded the
3929 * amount of sw queues we can support.
3930 */
3931 BUG_ON(!hctx->nr_ctx);
3932 }
3933
3934 for (; j < HCTX_MAX_TYPES; j++)
3935 ctx->hctxs[j] = blk_mq_map_queue_type(q,
3936 HCTX_TYPE_DEFAULT, i);
3937 }
3938
3939 queue_for_each_hw_ctx(q, hctx, i) {
3940 /*
3941 * If no software queues are mapped to this hardware queue,
3942 * disable it and free the request entries.
3943 */
3944 if (!hctx->nr_ctx) {
3945 /* Never unmap queue 0. We need it as a
3946 * fallback in case of a new remap fails
3947 * allocation
3948 */
3949 if (i)
3950 __blk_mq_free_map_and_rqs(set, i);
3951
3952 hctx->tags = NULL;
3953 continue;
3954 }
3955
3956 hctx->tags = set->tags[i];
3957 WARN_ON(!hctx->tags);
3958
3959 /*
3960 * Set the map size to the number of mapped software queues.
3961 * This is more accurate and more efficient than looping
3962 * over all possibly mapped software queues.
3963 */
3964 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
3965
3966 /*
3967 * Initialize batch roundrobin counts
3968 */
3969 hctx->next_cpu = blk_mq_first_mapped_cpu(hctx);
3970 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
3971 }
3972 }
3973
3974 /*
3975 * Caller needs to ensure that we're either frozen/quiesced, or that
3976 * the queue isn't live yet.
3977 */
queue_set_hctx_shared(struct request_queue * q,bool shared)3978 static void queue_set_hctx_shared(struct request_queue *q, bool shared)
3979 {
3980 struct blk_mq_hw_ctx *hctx;
3981 unsigned long i;
3982
3983 queue_for_each_hw_ctx(q, hctx, i) {
3984 if (shared) {
3985 hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
3986 } else {
3987 blk_mq_tag_idle(hctx);
3988 hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
3989 }
3990 }
3991 }
3992
blk_mq_update_tag_set_shared(struct blk_mq_tag_set * set,bool shared)3993 static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
3994 bool shared)
3995 {
3996 struct request_queue *q;
3997
3998 lockdep_assert_held(&set->tag_list_lock);
3999
4000 list_for_each_entry(q, &set->tag_list, tag_set_list) {
4001 blk_mq_freeze_queue(q);
4002 queue_set_hctx_shared(q, shared);
4003 blk_mq_unfreeze_queue(q);
4004 }
4005 }
4006
blk_mq_del_queue_tag_set(struct request_queue * q)4007 static void blk_mq_del_queue_tag_set(struct request_queue *q)
4008 {
4009 struct blk_mq_tag_set *set = q->tag_set;
4010
4011 mutex_lock(&set->tag_list_lock);
4012 list_del(&q->tag_set_list);
4013 if (list_is_singular(&set->tag_list)) {
4014 /* just transitioned to unshared */
4015 set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
4016 /* update existing queue */
4017 blk_mq_update_tag_set_shared(set, false);
4018 }
4019 mutex_unlock(&set->tag_list_lock);
4020 INIT_LIST_HEAD(&q->tag_set_list);
4021 }
4022
blk_mq_add_queue_tag_set(struct blk_mq_tag_set * set,struct request_queue * q)4023 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
4024 struct request_queue *q)
4025 {
4026 mutex_lock(&set->tag_list_lock);
4027
4028 /*
4029 * Check to see if we're transitioning to shared (from 1 to 2 queues).
4030 */
4031 if (!list_empty(&set->tag_list) &&
4032 !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
4033 set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
4034 /* update existing queue */
4035 blk_mq_update_tag_set_shared(set, true);
4036 }
4037 if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
4038 queue_set_hctx_shared(q, true);
4039 list_add_tail(&q->tag_set_list, &set->tag_list);
4040
4041 mutex_unlock(&set->tag_list_lock);
4042 }
4043
4044 /* All allocations will be freed in release handler of q->mq_kobj */
blk_mq_alloc_ctxs(struct request_queue * q)4045 static int blk_mq_alloc_ctxs(struct request_queue *q)
4046 {
4047 struct blk_mq_ctxs *ctxs;
4048 int cpu;
4049
4050 ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL);
4051 if (!ctxs)
4052 return -ENOMEM;
4053
4054 ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx);
4055 if (!ctxs->queue_ctx)
4056 goto fail;
4057
4058 for_each_possible_cpu(cpu) {
4059 struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu);
4060 ctx->ctxs = ctxs;
4061 }
4062
4063 q->mq_kobj = &ctxs->kobj;
4064 q->queue_ctx = ctxs->queue_ctx;
4065
4066 return 0;
4067 fail:
4068 kfree(ctxs);
4069 return -ENOMEM;
4070 }
4071
4072 /*
4073 * It is the actual release handler for mq, but we do it from
4074 * request queue's release handler for avoiding use-after-free
4075 * and headache because q->mq_kobj shouldn't have been introduced,
4076 * but we can't group ctx/kctx kobj without it.
4077 */
blk_mq_release(struct request_queue * q)4078 void blk_mq_release(struct request_queue *q)
4079 {
4080 struct blk_mq_hw_ctx *hctx, *next;
4081 unsigned long i;
4082
4083 queue_for_each_hw_ctx(q, hctx, i)
4084 WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
4085
4086 /* all hctx are in .unused_hctx_list now */
4087 list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
4088 list_del_init(&hctx->hctx_list);
4089 kobject_put(&hctx->kobj);
4090 }
4091
4092 xa_destroy(&q->hctx_table);
4093
4094 /*
4095 * release .mq_kobj and sw queue's kobject now because
4096 * both share lifetime with request queue.
4097 */
4098 blk_mq_sysfs_deinit(q);
4099 }
4100
blk_mq_init_queue_data(struct blk_mq_tag_set * set,void * queuedata)4101 static struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set,
4102 void *queuedata)
4103 {
4104 struct request_queue *q;
4105 int ret;
4106
4107 q = blk_alloc_queue(set->numa_node);
4108 if (!q)
4109 return ERR_PTR(-ENOMEM);
4110 q->queuedata = queuedata;
4111 ret = blk_mq_init_allocated_queue(set, q);
4112 if (ret) {
4113 blk_put_queue(q);
4114 return ERR_PTR(ret);
4115 }
4116 return q;
4117 }
4118
blk_mq_init_queue(struct blk_mq_tag_set * set)4119 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
4120 {
4121 return blk_mq_init_queue_data(set, NULL);
4122 }
4123 EXPORT_SYMBOL(blk_mq_init_queue);
4124
4125 /**
4126 * blk_mq_destroy_queue - shutdown a request queue
4127 * @q: request queue to shutdown
4128 *
4129 * This shuts down a request queue allocated by blk_mq_init_queue(). All future
4130 * requests will be failed with -ENODEV. The caller is responsible for dropping
4131 * the reference from blk_mq_init_queue() by calling blk_put_queue().
4132 *
4133 * Context: can sleep
4134 */
blk_mq_destroy_queue(struct request_queue * q)4135 void blk_mq_destroy_queue(struct request_queue *q)
4136 {
4137 WARN_ON_ONCE(!queue_is_mq(q));
4138 WARN_ON_ONCE(blk_queue_registered(q));
4139
4140 might_sleep();
4141
4142 blk_queue_flag_set(QUEUE_FLAG_DYING, q);
4143 blk_queue_start_drain(q);
4144 blk_mq_freeze_queue_wait(q);
4145
4146 blk_sync_queue(q);
4147 blk_mq_cancel_work_sync(q);
4148 blk_mq_exit_queue(q);
4149 }
4150 EXPORT_SYMBOL(blk_mq_destroy_queue);
4151
__blk_mq_alloc_disk(struct blk_mq_tag_set * set,void * queuedata,struct lock_class_key * lkclass)4152 struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata,
4153 struct lock_class_key *lkclass)
4154 {
4155 struct request_queue *q;
4156 struct gendisk *disk;
4157
4158 q = blk_mq_init_queue_data(set, queuedata);
4159 if (IS_ERR(q))
4160 return ERR_CAST(q);
4161
4162 disk = __alloc_disk_node(q, set->numa_node, lkclass);
4163 if (!disk) {
4164 blk_mq_destroy_queue(q);
4165 blk_put_queue(q);
4166 return ERR_PTR(-ENOMEM);
4167 }
4168 set_bit(GD_OWNS_QUEUE, &disk->state);
4169 return disk;
4170 }
4171 EXPORT_SYMBOL(__blk_mq_alloc_disk);
4172
blk_mq_alloc_disk_for_queue(struct request_queue * q,struct lock_class_key * lkclass)4173 struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q,
4174 struct lock_class_key *lkclass)
4175 {
4176 struct gendisk *disk;
4177
4178 if (!blk_get_queue(q))
4179 return NULL;
4180 disk = __alloc_disk_node(q, NUMA_NO_NODE, lkclass);
4181 if (!disk)
4182 blk_put_queue(q);
4183 return disk;
4184 }
4185 EXPORT_SYMBOL(blk_mq_alloc_disk_for_queue);
4186
blk_mq_alloc_and_init_hctx(struct blk_mq_tag_set * set,struct request_queue * q,int hctx_idx,int node)4187 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
4188 struct blk_mq_tag_set *set, struct request_queue *q,
4189 int hctx_idx, int node)
4190 {
4191 struct blk_mq_hw_ctx *hctx = NULL, *tmp;
4192
4193 /* reuse dead hctx first */
4194 spin_lock(&q->unused_hctx_lock);
4195 list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
4196 if (tmp->numa_node == node) {
4197 hctx = tmp;
4198 break;
4199 }
4200 }
4201 if (hctx)
4202 list_del_init(&hctx->hctx_list);
4203 spin_unlock(&q->unused_hctx_lock);
4204
4205 if (!hctx)
4206 hctx = blk_mq_alloc_hctx(q, set, node);
4207 if (!hctx)
4208 goto fail;
4209
4210 if (blk_mq_init_hctx(q, set, hctx, hctx_idx))
4211 goto free_hctx;
4212
4213 return hctx;
4214
4215 free_hctx:
4216 kobject_put(&hctx->kobj);
4217 fail:
4218 return NULL;
4219 }
4220
blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set * set,struct request_queue * q)4221 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
4222 struct request_queue *q)
4223 {
4224 struct blk_mq_hw_ctx *hctx;
4225 unsigned long i, j;
4226
4227 /* protect against switching io scheduler */
4228 mutex_lock(&q->sysfs_lock);
4229 for (i = 0; i < set->nr_hw_queues; i++) {
4230 int old_node;
4231 int node = blk_mq_get_hctx_node(set, i);
4232 struct blk_mq_hw_ctx *old_hctx = xa_load(&q->hctx_table, i);
4233
4234 if (old_hctx) {
4235 old_node = old_hctx->numa_node;
4236 blk_mq_exit_hctx(q, set, old_hctx, i);
4237 }
4238
4239 if (!blk_mq_alloc_and_init_hctx(set, q, i, node)) {
4240 if (!old_hctx)
4241 break;
4242 pr_warn("Allocate new hctx on node %d fails, fallback to previous one on node %d\n",
4243 node, old_node);
4244 hctx = blk_mq_alloc_and_init_hctx(set, q, i, old_node);
4245 WARN_ON_ONCE(!hctx);
4246 }
4247 }
4248 /*
4249 * Increasing nr_hw_queues fails. Free the newly allocated
4250 * hctxs and keep the previous q->nr_hw_queues.
4251 */
4252 if (i != set->nr_hw_queues) {
4253 j = q->nr_hw_queues;
4254 } else {
4255 j = i;
4256 q->nr_hw_queues = set->nr_hw_queues;
4257 }
4258
4259 xa_for_each_start(&q->hctx_table, j, hctx, j)
4260 blk_mq_exit_hctx(q, set, hctx, j);
4261 mutex_unlock(&q->sysfs_lock);
4262 }
4263
blk_mq_update_poll_flag(struct request_queue * q)4264 static void blk_mq_update_poll_flag(struct request_queue *q)
4265 {
4266 struct blk_mq_tag_set *set = q->tag_set;
4267
4268 if (set->nr_maps > HCTX_TYPE_POLL &&
4269 set->map[HCTX_TYPE_POLL].nr_queues)
4270 blk_queue_flag_set(QUEUE_FLAG_POLL, q);
4271 else
4272 blk_queue_flag_clear(QUEUE_FLAG_POLL, q);
4273 }
4274
blk_mq_init_allocated_queue(struct blk_mq_tag_set * set,struct request_queue * q)4275 int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
4276 struct request_queue *q)
4277 {
4278 /* mark the queue as mq asap */
4279 q->mq_ops = set->ops;
4280
4281 if (blk_mq_alloc_ctxs(q))
4282 goto err_exit;
4283
4284 /* init q->mq_kobj and sw queues' kobjects */
4285 blk_mq_sysfs_init(q);
4286
4287 INIT_LIST_HEAD(&q->unused_hctx_list);
4288 spin_lock_init(&q->unused_hctx_lock);
4289
4290 xa_init(&q->hctx_table);
4291
4292 blk_mq_realloc_hw_ctxs(set, q);
4293 if (!q->nr_hw_queues)
4294 goto err_hctxs;
4295
4296 INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
4297 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
4298
4299 q->tag_set = set;
4300
4301 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
4302 blk_mq_update_poll_flag(q);
4303
4304 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
4305 INIT_LIST_HEAD(&q->flush_list);
4306 INIT_LIST_HEAD(&q->requeue_list);
4307 spin_lock_init(&q->requeue_lock);
4308
4309 q->nr_requests = set->queue_depth;
4310
4311 blk_mq_init_cpu_queues(q, set->nr_hw_queues);
4312 blk_mq_add_queue_tag_set(set, q);
4313 blk_mq_map_swqueue(q);
4314 return 0;
4315
4316 err_hctxs:
4317 blk_mq_release(q);
4318 err_exit:
4319 q->mq_ops = NULL;
4320 return -ENOMEM;
4321 }
4322 EXPORT_SYMBOL(blk_mq_init_allocated_queue);
4323
4324 /* tags can _not_ be used after returning from blk_mq_exit_queue */
blk_mq_exit_queue(struct request_queue * q)4325 void blk_mq_exit_queue(struct request_queue *q)
4326 {
4327 struct blk_mq_tag_set *set = q->tag_set;
4328
4329 /* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
4330 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
4331 /* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
4332 blk_mq_del_queue_tag_set(q);
4333 }
4334
__blk_mq_alloc_rq_maps(struct blk_mq_tag_set * set)4335 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
4336 {
4337 int i;
4338
4339 if (blk_mq_is_shared_tags(set->flags)) {
4340 set->shared_tags = blk_mq_alloc_map_and_rqs(set,
4341 BLK_MQ_NO_HCTX_IDX,
4342 set->queue_depth);
4343 if (!set->shared_tags)
4344 return -ENOMEM;
4345 }
4346
4347 for (i = 0; i < set->nr_hw_queues; i++) {
4348 if (!__blk_mq_alloc_map_and_rqs(set, i))
4349 goto out_unwind;
4350 cond_resched();
4351 }
4352
4353 return 0;
4354
4355 out_unwind:
4356 while (--i >= 0)
4357 __blk_mq_free_map_and_rqs(set, i);
4358
4359 if (blk_mq_is_shared_tags(set->flags)) {
4360 blk_mq_free_map_and_rqs(set, set->shared_tags,
4361 BLK_MQ_NO_HCTX_IDX);
4362 }
4363
4364 return -ENOMEM;
4365 }
4366
4367 /*
4368 * Allocate the request maps associated with this tag_set. Note that this
4369 * may reduce the depth asked for, if memory is tight. set->queue_depth
4370 * will be updated to reflect the allocated depth.
4371 */
blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set * set)4372 static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set)
4373 {
4374 unsigned int depth;
4375 int err;
4376
4377 depth = set->queue_depth;
4378 do {
4379 err = __blk_mq_alloc_rq_maps(set);
4380 if (!err)
4381 break;
4382
4383 set->queue_depth >>= 1;
4384 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
4385 err = -ENOMEM;
4386 break;
4387 }
4388 } while (set->queue_depth);
4389
4390 if (!set->queue_depth || err) {
4391 pr_err("blk-mq: failed to allocate request map\n");
4392 return -ENOMEM;
4393 }
4394
4395 if (depth != set->queue_depth)
4396 pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
4397 depth, set->queue_depth);
4398
4399 return 0;
4400 }
4401
blk_mq_update_queue_map(struct blk_mq_tag_set * set)4402 static void blk_mq_update_queue_map(struct blk_mq_tag_set *set)
4403 {
4404 /*
4405 * blk_mq_map_queues() and multiple .map_queues() implementations
4406 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the
4407 * number of hardware queues.
4408 */
4409 if (set->nr_maps == 1)
4410 set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues;
4411
4412 if (set->ops->map_queues && !is_kdump_kernel()) {
4413 int i;
4414
4415 /*
4416 * transport .map_queues is usually done in the following
4417 * way:
4418 *
4419 * for (queue = 0; queue < set->nr_hw_queues; queue++) {
4420 * mask = get_cpu_mask(queue)
4421 * for_each_cpu(cpu, mask)
4422 * set->map[x].mq_map[cpu] = queue;
4423 * }
4424 *
4425 * When we need to remap, the table has to be cleared for
4426 * killing stale mapping since one CPU may not be mapped
4427 * to any hw queue.
4428 */
4429 for (i = 0; i < set->nr_maps; i++)
4430 blk_mq_clear_mq_map(&set->map[i]);
4431
4432 set->ops->map_queues(set);
4433 } else {
4434 BUG_ON(set->nr_maps > 1);
4435 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
4436 }
4437 }
4438
blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set * set,int new_nr_hw_queues)4439 static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set,
4440 int new_nr_hw_queues)
4441 {
4442 struct blk_mq_tags **new_tags;
4443 int i;
4444
4445 if (set->nr_hw_queues >= new_nr_hw_queues)
4446 goto done;
4447
4448 new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
4449 GFP_KERNEL, set->numa_node);
4450 if (!new_tags)
4451 return -ENOMEM;
4452
4453 if (set->tags)
4454 memcpy(new_tags, set->tags, set->nr_hw_queues *
4455 sizeof(*set->tags));
4456 kfree(set->tags);
4457 set->tags = new_tags;
4458
4459 for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) {
4460 if (!__blk_mq_alloc_map_and_rqs(set, i)) {
4461 while (--i >= set->nr_hw_queues)
4462 __blk_mq_free_map_and_rqs(set, i);
4463 return -ENOMEM;
4464 }
4465 cond_resched();
4466 }
4467
4468 done:
4469 set->nr_hw_queues = new_nr_hw_queues;
4470 return 0;
4471 }
4472
4473 /*
4474 * Alloc a tag set to be associated with one or more request queues.
4475 * May fail with EINVAL for various error conditions. May adjust the
4476 * requested depth down, if it's too large. In that case, the set
4477 * value will be stored in set->queue_depth.
4478 */
blk_mq_alloc_tag_set(struct blk_mq_tag_set * set)4479 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
4480 {
4481 int i, ret;
4482
4483 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
4484
4485 if (!set->nr_hw_queues)
4486 return -EINVAL;
4487 if (!set->queue_depth)
4488 return -EINVAL;
4489 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
4490 return -EINVAL;
4491
4492 if (!set->ops->queue_rq)
4493 return -EINVAL;
4494
4495 if (!set->ops->get_budget ^ !set->ops->put_budget)
4496 return -EINVAL;
4497
4498 if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
4499 pr_info("blk-mq: reduced tag depth to %u\n",
4500 BLK_MQ_MAX_DEPTH);
4501 set->queue_depth = BLK_MQ_MAX_DEPTH;
4502 }
4503
4504 if (!set->nr_maps)
4505 set->nr_maps = 1;
4506 else if (set->nr_maps > HCTX_MAX_TYPES)
4507 return -EINVAL;
4508
4509 /*
4510 * If a crashdump is active, then we are potentially in a very
4511 * memory constrained environment. Limit us to 1 queue and
4512 * 64 tags to prevent using too much memory.
4513 */
4514 if (is_kdump_kernel()) {
4515 set->nr_hw_queues = 1;
4516 set->nr_maps = 1;
4517 set->queue_depth = min(64U, set->queue_depth);
4518 }
4519 /*
4520 * There is no use for more h/w queues than cpus if we just have
4521 * a single map
4522 */
4523 if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
4524 set->nr_hw_queues = nr_cpu_ids;
4525
4526 if (set->flags & BLK_MQ_F_BLOCKING) {
4527 set->srcu = kmalloc(sizeof(*set->srcu), GFP_KERNEL);
4528 if (!set->srcu)
4529 return -ENOMEM;
4530 ret = init_srcu_struct(set->srcu);
4531 if (ret)
4532 goto out_free_srcu;
4533 }
4534
4535 ret = -ENOMEM;
4536 set->tags = kcalloc_node(set->nr_hw_queues,
4537 sizeof(struct blk_mq_tags *), GFP_KERNEL,
4538 set->numa_node);
4539 if (!set->tags)
4540 goto out_cleanup_srcu;
4541
4542 for (i = 0; i < set->nr_maps; i++) {
4543 set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
4544 sizeof(set->map[i].mq_map[0]),
4545 GFP_KERNEL, set->numa_node);
4546 if (!set->map[i].mq_map)
4547 goto out_free_mq_map;
4548 set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues;
4549 }
4550
4551 blk_mq_update_queue_map(set);
4552
4553 ret = blk_mq_alloc_set_map_and_rqs(set);
4554 if (ret)
4555 goto out_free_mq_map;
4556
4557 mutex_init(&set->tag_list_lock);
4558 INIT_LIST_HEAD(&set->tag_list);
4559
4560 return 0;
4561
4562 out_free_mq_map:
4563 for (i = 0; i < set->nr_maps; i++) {
4564 kfree(set->map[i].mq_map);
4565 set->map[i].mq_map = NULL;
4566 }
4567 kfree(set->tags);
4568 set->tags = NULL;
4569 out_cleanup_srcu:
4570 if (set->flags & BLK_MQ_F_BLOCKING)
4571 cleanup_srcu_struct(set->srcu);
4572 out_free_srcu:
4573 if (set->flags & BLK_MQ_F_BLOCKING)
4574 kfree(set->srcu);
4575 return ret;
4576 }
4577 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
4578
4579 /* allocate and initialize a tagset for a simple single-queue device */
blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set * set,const struct blk_mq_ops * ops,unsigned int queue_depth,unsigned int set_flags)4580 int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
4581 const struct blk_mq_ops *ops, unsigned int queue_depth,
4582 unsigned int set_flags)
4583 {
4584 memset(set, 0, sizeof(*set));
4585 set->ops = ops;
4586 set->nr_hw_queues = 1;
4587 set->nr_maps = 1;
4588 set->queue_depth = queue_depth;
4589 set->numa_node = NUMA_NO_NODE;
4590 set->flags = set_flags;
4591 return blk_mq_alloc_tag_set(set);
4592 }
4593 EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set);
4594
blk_mq_free_tag_set(struct blk_mq_tag_set * set)4595 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
4596 {
4597 int i, j;
4598
4599 for (i = 0; i < set->nr_hw_queues; i++)
4600 __blk_mq_free_map_and_rqs(set, i);
4601
4602 if (blk_mq_is_shared_tags(set->flags)) {
4603 blk_mq_free_map_and_rqs(set, set->shared_tags,
4604 BLK_MQ_NO_HCTX_IDX);
4605 }
4606
4607 for (j = 0; j < set->nr_maps; j++) {
4608 kfree(set->map[j].mq_map);
4609 set->map[j].mq_map = NULL;
4610 }
4611
4612 kfree(set->tags);
4613 set->tags = NULL;
4614 if (set->flags & BLK_MQ_F_BLOCKING) {
4615 cleanup_srcu_struct(set->srcu);
4616 kfree(set->srcu);
4617 }
4618 }
4619 EXPORT_SYMBOL(blk_mq_free_tag_set);
4620
blk_mq_update_nr_requests(struct request_queue * q,unsigned int nr)4621 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
4622 {
4623 struct blk_mq_tag_set *set = q->tag_set;
4624 struct blk_mq_hw_ctx *hctx;
4625 int ret;
4626 unsigned long i;
4627
4628 if (!set)
4629 return -EINVAL;
4630
4631 if (q->nr_requests == nr)
4632 return 0;
4633
4634 blk_mq_freeze_queue(q);
4635 blk_mq_quiesce_queue(q);
4636
4637 ret = 0;
4638 queue_for_each_hw_ctx(q, hctx, i) {
4639 if (!hctx->tags)
4640 continue;
4641 /*
4642 * If we're using an MQ scheduler, just update the scheduler
4643 * queue depth. This is similar to what the old code would do.
4644 */
4645 if (hctx->sched_tags) {
4646 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
4647 nr, true);
4648 } else {
4649 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
4650 false);
4651 }
4652 if (ret)
4653 break;
4654 if (q->elevator && q->elevator->type->ops.depth_updated)
4655 q->elevator->type->ops.depth_updated(hctx);
4656 }
4657 if (!ret) {
4658 q->nr_requests = nr;
4659 if (blk_mq_is_shared_tags(set->flags)) {
4660 if (q->elevator)
4661 blk_mq_tag_update_sched_shared_tags(q);
4662 else
4663 blk_mq_tag_resize_shared_tags(set, nr);
4664 }
4665 }
4666
4667 blk_mq_unquiesce_queue(q);
4668 blk_mq_unfreeze_queue(q);
4669
4670 return ret;
4671 }
4672
4673 /*
4674 * request_queue and elevator_type pair.
4675 * It is just used by __blk_mq_update_nr_hw_queues to cache
4676 * the elevator_type associated with a request_queue.
4677 */
4678 struct blk_mq_qe_pair {
4679 struct list_head node;
4680 struct request_queue *q;
4681 struct elevator_type *type;
4682 };
4683
4684 /*
4685 * Cache the elevator_type in qe pair list and switch the
4686 * io scheduler to 'none'
4687 */
blk_mq_elv_switch_none(struct list_head * head,struct request_queue * q)4688 static bool blk_mq_elv_switch_none(struct list_head *head,
4689 struct request_queue *q)
4690 {
4691 struct blk_mq_qe_pair *qe;
4692
4693 qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY);
4694 if (!qe)
4695 return false;
4696
4697 /* q->elevator needs protection from ->sysfs_lock */
4698 mutex_lock(&q->sysfs_lock);
4699
4700 /* the check has to be done with holding sysfs_lock */
4701 if (!q->elevator) {
4702 kfree(qe);
4703 goto unlock;
4704 }
4705
4706 INIT_LIST_HEAD(&qe->node);
4707 qe->q = q;
4708 qe->type = q->elevator->type;
4709 /* keep a reference to the elevator module as we'll switch back */
4710 __elevator_get(qe->type);
4711 list_add(&qe->node, head);
4712 elevator_disable(q);
4713 unlock:
4714 mutex_unlock(&q->sysfs_lock);
4715
4716 return true;
4717 }
4718
blk_lookup_qe_pair(struct list_head * head,struct request_queue * q)4719 static struct blk_mq_qe_pair *blk_lookup_qe_pair(struct list_head *head,
4720 struct request_queue *q)
4721 {
4722 struct blk_mq_qe_pair *qe;
4723
4724 list_for_each_entry(qe, head, node)
4725 if (qe->q == q)
4726 return qe;
4727
4728 return NULL;
4729 }
4730
blk_mq_elv_switch_back(struct list_head * head,struct request_queue * q)4731 static void blk_mq_elv_switch_back(struct list_head *head,
4732 struct request_queue *q)
4733 {
4734 struct blk_mq_qe_pair *qe;
4735 struct elevator_type *t;
4736
4737 qe = blk_lookup_qe_pair(head, q);
4738 if (!qe)
4739 return;
4740 t = qe->type;
4741 list_del(&qe->node);
4742 kfree(qe);
4743
4744 mutex_lock(&q->sysfs_lock);
4745 elevator_switch(q, t);
4746 /* drop the reference acquired in blk_mq_elv_switch_none */
4747 elevator_put(t);
4748 mutex_unlock(&q->sysfs_lock);
4749 }
4750
__blk_mq_update_nr_hw_queues(struct blk_mq_tag_set * set,int nr_hw_queues)4751 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
4752 int nr_hw_queues)
4753 {
4754 struct request_queue *q;
4755 LIST_HEAD(head);
4756 int prev_nr_hw_queues = set->nr_hw_queues;
4757 int i;
4758
4759 lockdep_assert_held(&set->tag_list_lock);
4760
4761 if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids)
4762 nr_hw_queues = nr_cpu_ids;
4763 if (nr_hw_queues < 1)
4764 return;
4765 if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues)
4766 return;
4767
4768 list_for_each_entry(q, &set->tag_list, tag_set_list)
4769 blk_mq_freeze_queue(q);
4770 /*
4771 * Switch IO scheduler to 'none', cleaning up the data associated
4772 * with the previous scheduler. We will switch back once we are done
4773 * updating the new sw to hw queue mappings.
4774 */
4775 list_for_each_entry(q, &set->tag_list, tag_set_list)
4776 if (!blk_mq_elv_switch_none(&head, q))
4777 goto switch_back;
4778
4779 list_for_each_entry(q, &set->tag_list, tag_set_list) {
4780 blk_mq_debugfs_unregister_hctxs(q);
4781 blk_mq_sysfs_unregister_hctxs(q);
4782 }
4783
4784 if (blk_mq_realloc_tag_set_tags(set, nr_hw_queues) < 0)
4785 goto reregister;
4786
4787 fallback:
4788 blk_mq_update_queue_map(set);
4789 list_for_each_entry(q, &set->tag_list, tag_set_list) {
4790 blk_mq_realloc_hw_ctxs(set, q);
4791 blk_mq_update_poll_flag(q);
4792 if (q->nr_hw_queues != set->nr_hw_queues) {
4793 int i = prev_nr_hw_queues;
4794
4795 pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n",
4796 nr_hw_queues, prev_nr_hw_queues);
4797 for (; i < set->nr_hw_queues; i++)
4798 __blk_mq_free_map_and_rqs(set, i);
4799
4800 set->nr_hw_queues = prev_nr_hw_queues;
4801 goto fallback;
4802 }
4803 blk_mq_map_swqueue(q);
4804 }
4805
4806 reregister:
4807 list_for_each_entry(q, &set->tag_list, tag_set_list) {
4808 blk_mq_sysfs_register_hctxs(q);
4809 blk_mq_debugfs_register_hctxs(q);
4810 }
4811
4812 switch_back:
4813 list_for_each_entry(q, &set->tag_list, tag_set_list)
4814 blk_mq_elv_switch_back(&head, q);
4815
4816 list_for_each_entry(q, &set->tag_list, tag_set_list)
4817 blk_mq_unfreeze_queue(q);
4818
4819 /* Free the excess tags when nr_hw_queues shrink. */
4820 for (i = set->nr_hw_queues; i < prev_nr_hw_queues; i++)
4821 __blk_mq_free_map_and_rqs(set, i);
4822 }
4823
blk_mq_update_nr_hw_queues(struct blk_mq_tag_set * set,int nr_hw_queues)4824 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
4825 {
4826 mutex_lock(&set->tag_list_lock);
4827 __blk_mq_update_nr_hw_queues(set, nr_hw_queues);
4828 mutex_unlock(&set->tag_list_lock);
4829 }
4830 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
4831
blk_hctx_poll(struct request_queue * q,struct blk_mq_hw_ctx * hctx,struct io_comp_batch * iob,unsigned int flags)4832 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
4833 struct io_comp_batch *iob, unsigned int flags)
4834 {
4835 long state = get_current_state();
4836 int ret;
4837
4838 do {
4839 ret = q->mq_ops->poll(hctx, iob);
4840 if (ret > 0) {
4841 __set_current_state(TASK_RUNNING);
4842 return ret;
4843 }
4844
4845 if (signal_pending_state(state, current))
4846 __set_current_state(TASK_RUNNING);
4847 if (task_is_running(current))
4848 return 1;
4849
4850 if (ret < 0 || (flags & BLK_POLL_ONESHOT))
4851 break;
4852 cpu_relax();
4853 } while (!need_resched());
4854
4855 __set_current_state(TASK_RUNNING);
4856 return 0;
4857 }
4858
blk_mq_poll(struct request_queue * q,blk_qc_t cookie,struct io_comp_batch * iob,unsigned int flags)4859 int blk_mq_poll(struct request_queue *q, blk_qc_t cookie,
4860 struct io_comp_batch *iob, unsigned int flags)
4861 {
4862 struct blk_mq_hw_ctx *hctx = xa_load(&q->hctx_table, cookie);
4863
4864 return blk_hctx_poll(q, hctx, iob, flags);
4865 }
4866
blk_rq_poll(struct request * rq,struct io_comp_batch * iob,unsigned int poll_flags)4867 int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,
4868 unsigned int poll_flags)
4869 {
4870 struct request_queue *q = rq->q;
4871 int ret;
4872
4873 if (!blk_rq_is_poll(rq))
4874 return 0;
4875 if (!percpu_ref_tryget(&q->q_usage_counter))
4876 return 0;
4877
4878 ret = blk_hctx_poll(q, rq->mq_hctx, iob, poll_flags);
4879 blk_queue_exit(q);
4880
4881 return ret;
4882 }
4883 EXPORT_SYMBOL_GPL(blk_rq_poll);
4884
blk_mq_rq_cpu(struct request * rq)4885 unsigned int blk_mq_rq_cpu(struct request *rq)
4886 {
4887 return rq->mq_ctx->cpu;
4888 }
4889 EXPORT_SYMBOL(blk_mq_rq_cpu);
4890
blk_mq_cancel_work_sync(struct request_queue * q)4891 void blk_mq_cancel_work_sync(struct request_queue *q)
4892 {
4893 struct blk_mq_hw_ctx *hctx;
4894 unsigned long i;
4895
4896 cancel_delayed_work_sync(&q->requeue_work);
4897
4898 queue_for_each_hw_ctx(q, hctx, i)
4899 cancel_delayed_work_sync(&hctx->run_work);
4900 }
4901
blk_mq_init(void)4902 static int __init blk_mq_init(void)
4903 {
4904 int i;
4905
4906 for_each_possible_cpu(i)
4907 init_llist_head(&per_cpu(blk_cpu_done, i));
4908 for_each_possible_cpu(i)
4909 INIT_CSD(&per_cpu(blk_cpu_csd, i),
4910 __blk_mq_complete_request_remote, NULL);
4911 open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
4912
4913 cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
4914 "block/softirq:dead", NULL,
4915 blk_softirq_cpu_dead);
4916 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
4917 blk_mq_hctx_notify_dead);
4918 cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
4919 blk_mq_hctx_notify_online,
4920 blk_mq_hctx_notify_offline);
4921 return 0;
4922 }
4923 subsys_initcall(blk_mq_init);
4924