1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Block multiqueue core code 4 * 5 * Copyright (C) 2013-2014 Jens Axboe 6 * Copyright (C) 2013-2014 Christoph Hellwig 7 */ 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/backing-dev.h> 11 #include <linux/bio.h> 12 #include <linux/blkdev.h> 13 #include <linux/kmemleak.h> 14 #include <linux/mm.h> 15 #include <linux/init.h> 16 #include <linux/slab.h> 17 #include <linux/workqueue.h> 18 #include <linux/smp.h> 19 #include <linux/llist.h> 20 #include <linux/list_sort.h> 21 #include <linux/cpu.h> 22 #include <linux/cache.h> 23 #include <linux/sched/sysctl.h> 24 #include <linux/sched/topology.h> 25 #include <linux/sched/signal.h> 26 #include <linux/delay.h> 27 #include <linux/crash_dump.h> 28 #include <linux/prefetch.h> 29 #include <linux/blk-crypto.h> 30 31 #include <trace/events/block.h> 32 33 #include <linux/blk-mq.h> 34 #include <linux/t10-pi.h> 35 #include "blk.h" 36 #include "blk-mq.h" 37 #include "blk-mq-debugfs.h" 38 #include "blk-mq-tag.h" 39 #include "blk-pm.h" 40 #include "blk-stat.h" 41 #include "blk-mq-sched.h" 42 #include "blk-rq-qos.h" 43 44 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); 45 46 static void blk_mq_poll_stats_start(struct request_queue *q); 47 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); 48 49 static int blk_mq_poll_stats_bkt(const struct request *rq) 50 { 51 int ddir, sectors, bucket; 52 53 ddir = rq_data_dir(rq); 54 sectors = blk_rq_stats_sectors(rq); 55 56 bucket = ddir + 2 * ilog2(sectors); 57 58 if (bucket < 0) 59 return -1; 60 else if (bucket >= BLK_MQ_POLL_STATS_BKTS) 61 return ddir + BLK_MQ_POLL_STATS_BKTS - 2; 62 63 return bucket; 64 } 65 66 /* 67 * Check if any of the ctx, dispatch list or elevator 68 * have pending work in this hardware queue. 69 */ 70 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) 71 { 72 return !list_empty_careful(&hctx->dispatch) || 73 sbitmap_any_bit_set(&hctx->ctx_map) || 74 blk_mq_sched_has_work(hctx); 75 } 76 77 /* 78 * Mark this ctx as having pending work in this hardware queue 79 */ 80 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, 81 struct blk_mq_ctx *ctx) 82 { 83 const int bit = ctx->index_hw[hctx->type]; 84 85 if (!sbitmap_test_bit(&hctx->ctx_map, bit)) 86 sbitmap_set_bit(&hctx->ctx_map, bit); 87 } 88 89 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, 90 struct blk_mq_ctx *ctx) 91 { 92 const int bit = ctx->index_hw[hctx->type]; 93 94 sbitmap_clear_bit(&hctx->ctx_map, bit); 95 } 96 97 struct mq_inflight { 98 struct block_device *part; 99 unsigned int inflight[2]; 100 }; 101 102 static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx, 103 struct request *rq, void *priv, 104 bool reserved) 105 { 106 struct mq_inflight *mi = priv; 107 108 if ((!mi->part->bd_partno || rq->part == mi->part) && 109 blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT) 110 mi->inflight[rq_data_dir(rq)]++; 111 112 return true; 113 } 114 115 unsigned int blk_mq_in_flight(struct request_queue *q, 116 struct block_device *part) 117 { 118 struct mq_inflight mi = { .part = part }; 119 120 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); 121 122 return mi.inflight[0] + mi.inflight[1]; 123 } 124 125 void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part, 126 unsigned int inflight[2]) 127 { 128 struct mq_inflight mi = { .part = part }; 129 130 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); 131 inflight[0] = mi.inflight[0]; 132 inflight[1] = mi.inflight[1]; 133 } 134 135 void blk_freeze_queue_start(struct request_queue *q) 136 { 137 mutex_lock(&q->mq_freeze_lock); 138 if (++q->mq_freeze_depth == 1) { 139 percpu_ref_kill(&q->q_usage_counter); 140 mutex_unlock(&q->mq_freeze_lock); 141 if (queue_is_mq(q)) 142 blk_mq_run_hw_queues(q, false); 143 } else { 144 mutex_unlock(&q->mq_freeze_lock); 145 } 146 } 147 EXPORT_SYMBOL_GPL(blk_freeze_queue_start); 148 149 void blk_mq_freeze_queue_wait(struct request_queue *q) 150 { 151 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter)); 152 } 153 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait); 154 155 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, 156 unsigned long timeout) 157 { 158 return wait_event_timeout(q->mq_freeze_wq, 159 percpu_ref_is_zero(&q->q_usage_counter), 160 timeout); 161 } 162 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout); 163 164 /* 165 * Guarantee no request is in use, so we can change any data structure of 166 * the queue afterward. 167 */ 168 void blk_freeze_queue(struct request_queue *q) 169 { 170 /* 171 * In the !blk_mq case we are only calling this to kill the 172 * q_usage_counter, otherwise this increases the freeze depth 173 * and waits for it to return to zero. For this reason there is 174 * no blk_unfreeze_queue(), and blk_freeze_queue() is not 175 * exported to drivers as the only user for unfreeze is blk_mq. 176 */ 177 blk_freeze_queue_start(q); 178 blk_mq_freeze_queue_wait(q); 179 } 180 181 void blk_mq_freeze_queue(struct request_queue *q) 182 { 183 /* 184 * ...just an alias to keep freeze and unfreeze actions balanced 185 * in the blk_mq_* namespace 186 */ 187 blk_freeze_queue(q); 188 } 189 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue); 190 191 void blk_mq_unfreeze_queue(struct request_queue *q) 192 { 193 mutex_lock(&q->mq_freeze_lock); 194 q->mq_freeze_depth--; 195 WARN_ON_ONCE(q->mq_freeze_depth < 0); 196 if (!q->mq_freeze_depth) { 197 percpu_ref_resurrect(&q->q_usage_counter); 198 wake_up_all(&q->mq_freeze_wq); 199 } 200 mutex_unlock(&q->mq_freeze_lock); 201 } 202 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); 203 204 /* 205 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the 206 * mpt3sas driver such that this function can be removed. 207 */ 208 void blk_mq_quiesce_queue_nowait(struct request_queue *q) 209 { 210 blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q); 211 } 212 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); 213 214 /** 215 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished 216 * @q: request queue. 217 * 218 * Note: this function does not prevent that the struct request end_io() 219 * callback function is invoked. Once this function is returned, we make 220 * sure no dispatch can happen until the queue is unquiesced via 221 * blk_mq_unquiesce_queue(). 222 */ 223 void blk_mq_quiesce_queue(struct request_queue *q) 224 { 225 struct blk_mq_hw_ctx *hctx; 226 unsigned int i; 227 bool rcu = false; 228 229 blk_mq_quiesce_queue_nowait(q); 230 231 queue_for_each_hw_ctx(q, hctx, i) { 232 if (hctx->flags & BLK_MQ_F_BLOCKING) 233 synchronize_srcu(hctx->srcu); 234 else 235 rcu = true; 236 } 237 if (rcu) 238 synchronize_rcu(); 239 } 240 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue); 241 242 /* 243 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue() 244 * @q: request queue. 245 * 246 * This function recovers queue into the state before quiescing 247 * which is done by blk_mq_quiesce_queue. 248 */ 249 void blk_mq_unquiesce_queue(struct request_queue *q) 250 { 251 blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q); 252 253 /* dispatch requests which are inserted during quiescing */ 254 blk_mq_run_hw_queues(q, true); 255 } 256 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue); 257 258 void blk_mq_wake_waiters(struct request_queue *q) 259 { 260 struct blk_mq_hw_ctx *hctx; 261 unsigned int i; 262 263 queue_for_each_hw_ctx(q, hctx, i) 264 if (blk_mq_hw_queue_mapped(hctx)) 265 blk_mq_tag_wakeup_all(hctx->tags, true); 266 } 267 268 /* 269 * Only need start/end time stamping if we have iostat or 270 * blk stats enabled, or using an IO scheduler. 271 */ 272 static inline bool blk_mq_need_time_stamp(struct request *rq) 273 { 274 return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS)) || rq->q->elevator; 275 } 276 277 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, 278 unsigned int tag, u64 alloc_time_ns) 279 { 280 struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 281 struct request *rq = tags->static_rqs[tag]; 282 283 if (data->q->elevator) { 284 rq->tag = BLK_MQ_NO_TAG; 285 rq->internal_tag = tag; 286 } else { 287 rq->tag = tag; 288 rq->internal_tag = BLK_MQ_NO_TAG; 289 } 290 291 /* csd/requeue_work/fifo_time is initialized before use */ 292 rq->q = data->q; 293 rq->mq_ctx = data->ctx; 294 rq->mq_hctx = data->hctx; 295 rq->rq_flags = 0; 296 rq->cmd_flags = data->cmd_flags; 297 if (data->flags & BLK_MQ_REQ_PM) 298 rq->rq_flags |= RQF_PM; 299 if (blk_queue_io_stat(data->q)) 300 rq->rq_flags |= RQF_IO_STAT; 301 INIT_LIST_HEAD(&rq->queuelist); 302 INIT_HLIST_NODE(&rq->hash); 303 RB_CLEAR_NODE(&rq->rb_node); 304 rq->rq_disk = NULL; 305 rq->part = NULL; 306 #ifdef CONFIG_BLK_RQ_ALLOC_TIME 307 rq->alloc_time_ns = alloc_time_ns; 308 #endif 309 if (blk_mq_need_time_stamp(rq)) 310 rq->start_time_ns = ktime_get_ns(); 311 else 312 rq->start_time_ns = 0; 313 rq->io_start_time_ns = 0; 314 rq->stats_sectors = 0; 315 rq->nr_phys_segments = 0; 316 #if defined(CONFIG_BLK_DEV_INTEGRITY) 317 rq->nr_integrity_segments = 0; 318 #endif 319 blk_crypto_rq_set_defaults(rq); 320 /* tag was already set */ 321 WRITE_ONCE(rq->deadline, 0); 322 323 rq->timeout = 0; 324 325 rq->end_io = NULL; 326 rq->end_io_data = NULL; 327 328 data->ctx->rq_dispatched[op_is_sync(data->cmd_flags)]++; 329 refcount_set(&rq->ref, 1); 330 331 if (!op_is_flush(data->cmd_flags)) { 332 struct elevator_queue *e = data->q->elevator; 333 334 rq->elv.icq = NULL; 335 if (e && e->type->ops.prepare_request) { 336 if (e->type->icq_cache) 337 blk_mq_sched_assign_ioc(rq); 338 339 e->type->ops.prepare_request(rq); 340 rq->rq_flags |= RQF_ELVPRIV; 341 } 342 } 343 344 data->hctx->queued++; 345 return rq; 346 } 347 348 static struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data) 349 { 350 struct request_queue *q = data->q; 351 struct elevator_queue *e = q->elevator; 352 u64 alloc_time_ns = 0; 353 unsigned int tag; 354 355 /* alloc_time includes depth and tag waits */ 356 if (blk_queue_rq_alloc_time(q)) 357 alloc_time_ns = ktime_get_ns(); 358 359 if (data->cmd_flags & REQ_NOWAIT) 360 data->flags |= BLK_MQ_REQ_NOWAIT; 361 362 if (e) { 363 /* 364 * Flush requests are special and go directly to the 365 * dispatch list. Don't include reserved tags in the 366 * limiting, as it isn't useful. 367 */ 368 if (!op_is_flush(data->cmd_flags) && 369 e->type->ops.limit_depth && 370 !(data->flags & BLK_MQ_REQ_RESERVED)) 371 e->type->ops.limit_depth(data->cmd_flags, data); 372 } 373 374 retry: 375 data->ctx = blk_mq_get_ctx(q); 376 data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx); 377 if (!e) 378 blk_mq_tag_busy(data->hctx); 379 380 /* 381 * Waiting allocations only fail because of an inactive hctx. In that 382 * case just retry the hctx assignment and tag allocation as CPU hotplug 383 * should have migrated us to an online CPU by now. 384 */ 385 tag = blk_mq_get_tag(data); 386 if (tag == BLK_MQ_NO_TAG) { 387 if (data->flags & BLK_MQ_REQ_NOWAIT) 388 return NULL; 389 390 /* 391 * Give up the CPU and sleep for a random short time to ensure 392 * that thread using a realtime scheduling class are migrated 393 * off the CPU, and thus off the hctx that is going away. 394 */ 395 msleep(3); 396 goto retry; 397 } 398 return blk_mq_rq_ctx_init(data, tag, alloc_time_ns); 399 } 400 401 struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, 402 blk_mq_req_flags_t flags) 403 { 404 struct blk_mq_alloc_data data = { 405 .q = q, 406 .flags = flags, 407 .cmd_flags = op, 408 }; 409 struct request *rq; 410 int ret; 411 412 ret = blk_queue_enter(q, flags); 413 if (ret) 414 return ERR_PTR(ret); 415 416 rq = __blk_mq_alloc_request(&data); 417 if (!rq) 418 goto out_queue_exit; 419 rq->__data_len = 0; 420 rq->__sector = (sector_t) -1; 421 rq->bio = rq->biotail = NULL; 422 return rq; 423 out_queue_exit: 424 blk_queue_exit(q); 425 return ERR_PTR(-EWOULDBLOCK); 426 } 427 EXPORT_SYMBOL(blk_mq_alloc_request); 428 429 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, 430 unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx) 431 { 432 struct blk_mq_alloc_data data = { 433 .q = q, 434 .flags = flags, 435 .cmd_flags = op, 436 }; 437 u64 alloc_time_ns = 0; 438 unsigned int cpu; 439 unsigned int tag; 440 int ret; 441 442 /* alloc_time includes depth and tag waits */ 443 if (blk_queue_rq_alloc_time(q)) 444 alloc_time_ns = ktime_get_ns(); 445 446 /* 447 * If the tag allocator sleeps we could get an allocation for a 448 * different hardware context. No need to complicate the low level 449 * allocator for this for the rare use case of a command tied to 450 * a specific queue. 451 */ 452 if (WARN_ON_ONCE(!(flags & (BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED)))) 453 return ERR_PTR(-EINVAL); 454 455 if (hctx_idx >= q->nr_hw_queues) 456 return ERR_PTR(-EIO); 457 458 ret = blk_queue_enter(q, flags); 459 if (ret) 460 return ERR_PTR(ret); 461 462 /* 463 * Check if the hardware context is actually mapped to anything. 464 * If not tell the caller that it should skip this queue. 465 */ 466 ret = -EXDEV; 467 data.hctx = q->queue_hw_ctx[hctx_idx]; 468 if (!blk_mq_hw_queue_mapped(data.hctx)) 469 goto out_queue_exit; 470 cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask); 471 data.ctx = __blk_mq_get_ctx(q, cpu); 472 473 if (!q->elevator) 474 blk_mq_tag_busy(data.hctx); 475 476 ret = -EWOULDBLOCK; 477 tag = blk_mq_get_tag(&data); 478 if (tag == BLK_MQ_NO_TAG) 479 goto out_queue_exit; 480 return blk_mq_rq_ctx_init(&data, tag, alloc_time_ns); 481 482 out_queue_exit: 483 blk_queue_exit(q); 484 return ERR_PTR(ret); 485 } 486 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); 487 488 static void __blk_mq_free_request(struct request *rq) 489 { 490 struct request_queue *q = rq->q; 491 struct blk_mq_ctx *ctx = rq->mq_ctx; 492 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 493 const int sched_tag = rq->internal_tag; 494 495 blk_crypto_free_request(rq); 496 blk_pm_mark_last_busy(rq); 497 rq->mq_hctx = NULL; 498 if (rq->tag != BLK_MQ_NO_TAG) 499 blk_mq_put_tag(hctx->tags, ctx, rq->tag); 500 if (sched_tag != BLK_MQ_NO_TAG) 501 blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag); 502 blk_mq_sched_restart(hctx); 503 blk_queue_exit(q); 504 } 505 506 void blk_mq_free_request(struct request *rq) 507 { 508 struct request_queue *q = rq->q; 509 struct elevator_queue *e = q->elevator; 510 struct blk_mq_ctx *ctx = rq->mq_ctx; 511 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 512 513 if (rq->rq_flags & RQF_ELVPRIV) { 514 if (e && e->type->ops.finish_request) 515 e->type->ops.finish_request(rq); 516 if (rq->elv.icq) { 517 put_io_context(rq->elv.icq->ioc); 518 rq->elv.icq = NULL; 519 } 520 } 521 522 ctx->rq_completed[rq_is_sync(rq)]++; 523 if (rq->rq_flags & RQF_MQ_INFLIGHT) 524 __blk_mq_dec_active_requests(hctx); 525 526 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq))) 527 laptop_io_completion(q->backing_dev_info); 528 529 rq_qos_done(q, rq); 530 531 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 532 if (refcount_dec_and_test(&rq->ref)) 533 __blk_mq_free_request(rq); 534 } 535 EXPORT_SYMBOL_GPL(blk_mq_free_request); 536 537 inline void __blk_mq_end_request(struct request *rq, blk_status_t error) 538 { 539 u64 now = 0; 540 541 if (blk_mq_need_time_stamp(rq)) 542 now = ktime_get_ns(); 543 544 if (rq->rq_flags & RQF_STATS) { 545 blk_mq_poll_stats_start(rq->q); 546 blk_stat_add(rq, now); 547 } 548 549 blk_mq_sched_completed_request(rq, now); 550 551 blk_account_io_done(rq, now); 552 553 if (rq->end_io) { 554 rq_qos_done(rq->q, rq); 555 rq->end_io(rq, error); 556 } else { 557 blk_mq_free_request(rq); 558 } 559 } 560 EXPORT_SYMBOL(__blk_mq_end_request); 561 562 void blk_mq_end_request(struct request *rq, blk_status_t error) 563 { 564 if (blk_update_request(rq, error, blk_rq_bytes(rq))) 565 BUG(); 566 __blk_mq_end_request(rq, error); 567 } 568 EXPORT_SYMBOL(blk_mq_end_request); 569 570 static void blk_complete_reqs(struct llist_head *list) 571 { 572 struct llist_node *entry = llist_reverse_order(llist_del_all(list)); 573 struct request *rq, *next; 574 575 llist_for_each_entry_safe(rq, next, entry, ipi_list) 576 rq->q->mq_ops->complete(rq); 577 } 578 579 static __latent_entropy void blk_done_softirq(struct softirq_action *h) 580 { 581 blk_complete_reqs(this_cpu_ptr(&blk_cpu_done)); 582 } 583 584 static int blk_softirq_cpu_dead(unsigned int cpu) 585 { 586 blk_complete_reqs(&per_cpu(blk_cpu_done, cpu)); 587 return 0; 588 } 589 590 static void __blk_mq_complete_request_remote(void *data) 591 { 592 __raise_softirq_irqoff(BLOCK_SOFTIRQ); 593 } 594 595 static inline bool blk_mq_complete_need_ipi(struct request *rq) 596 { 597 int cpu = raw_smp_processor_id(); 598 599 if (!IS_ENABLED(CONFIG_SMP) || 600 !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) 601 return false; 602 /* 603 * With force threaded interrupts enabled, raising softirq from an SMP 604 * function call will always result in waking the ksoftirqd thread. 605 * This is probably worse than completing the request on a different 606 * cache domain. 607 */ 608 if (force_irqthreads) 609 return false; 610 611 /* same CPU or cache domain? Complete locally */ 612 if (cpu == rq->mq_ctx->cpu || 613 (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) && 614 cpus_share_cache(cpu, rq->mq_ctx->cpu))) 615 return false; 616 617 /* don't try to IPI to an offline CPU */ 618 return cpu_online(rq->mq_ctx->cpu); 619 } 620 621 static void blk_mq_complete_send_ipi(struct request *rq) 622 { 623 struct llist_head *list; 624 unsigned int cpu; 625 626 cpu = rq->mq_ctx->cpu; 627 list = &per_cpu(blk_cpu_done, cpu); 628 if (llist_add(&rq->ipi_list, list)) { 629 INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq); 630 smp_call_function_single_async(cpu, &rq->csd); 631 } 632 } 633 634 static void blk_mq_raise_softirq(struct request *rq) 635 { 636 struct llist_head *list; 637 638 preempt_disable(); 639 list = this_cpu_ptr(&blk_cpu_done); 640 if (llist_add(&rq->ipi_list, list)) 641 raise_softirq(BLOCK_SOFTIRQ); 642 preempt_enable(); 643 } 644 645 bool blk_mq_complete_request_remote(struct request *rq) 646 { 647 WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); 648 649 /* 650 * For a polled request, always complete locallly, it's pointless 651 * to redirect the completion. 652 */ 653 if (rq->cmd_flags & REQ_HIPRI) 654 return false; 655 656 if (blk_mq_complete_need_ipi(rq)) { 657 blk_mq_complete_send_ipi(rq); 658 return true; 659 } 660 661 if (rq->q->nr_hw_queues == 1) { 662 blk_mq_raise_softirq(rq); 663 return true; 664 } 665 return false; 666 } 667 EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote); 668 669 /** 670 * blk_mq_complete_request - end I/O on a request 671 * @rq: the request being processed 672 * 673 * Description: 674 * Complete a request by scheduling the ->complete_rq operation. 675 **/ 676 void blk_mq_complete_request(struct request *rq) 677 { 678 if (!blk_mq_complete_request_remote(rq)) 679 rq->q->mq_ops->complete(rq); 680 } 681 EXPORT_SYMBOL(blk_mq_complete_request); 682 683 static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx) 684 __releases(hctx->srcu) 685 { 686 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) 687 rcu_read_unlock(); 688 else 689 srcu_read_unlock(hctx->srcu, srcu_idx); 690 } 691 692 static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx) 693 __acquires(hctx->srcu) 694 { 695 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { 696 /* shut up gcc false positive */ 697 *srcu_idx = 0; 698 rcu_read_lock(); 699 } else 700 *srcu_idx = srcu_read_lock(hctx->srcu); 701 } 702 703 /** 704 * blk_mq_start_request - Start processing a request 705 * @rq: Pointer to request to be started 706 * 707 * Function used by device drivers to notify the block layer that a request 708 * is going to be processed now, so blk layer can do proper initializations 709 * such as starting the timeout timer. 710 */ 711 void blk_mq_start_request(struct request *rq) 712 { 713 struct request_queue *q = rq->q; 714 715 trace_block_rq_issue(rq); 716 717 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 718 rq->io_start_time_ns = ktime_get_ns(); 719 rq->stats_sectors = blk_rq_sectors(rq); 720 rq->rq_flags |= RQF_STATS; 721 rq_qos_issue(q, rq); 722 } 723 724 WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE); 725 726 blk_add_timer(rq); 727 WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT); 728 729 #ifdef CONFIG_BLK_DEV_INTEGRITY 730 if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE) 731 q->integrity.profile->prepare_fn(rq); 732 #endif 733 } 734 EXPORT_SYMBOL(blk_mq_start_request); 735 736 static void __blk_mq_requeue_request(struct request *rq) 737 { 738 struct request_queue *q = rq->q; 739 740 blk_mq_put_driver_tag(rq); 741 742 trace_block_rq_requeue(rq); 743 rq_qos_requeue(q, rq); 744 745 if (blk_mq_request_started(rq)) { 746 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 747 rq->rq_flags &= ~RQF_TIMED_OUT; 748 } 749 } 750 751 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) 752 { 753 __blk_mq_requeue_request(rq); 754 755 /* this request will be re-inserted to io scheduler queue */ 756 blk_mq_sched_requeue_request(rq); 757 758 BUG_ON(!list_empty(&rq->queuelist)); 759 blk_mq_add_to_requeue_list(rq, true, kick_requeue_list); 760 } 761 EXPORT_SYMBOL(blk_mq_requeue_request); 762 763 static void blk_mq_requeue_work(struct work_struct *work) 764 { 765 struct request_queue *q = 766 container_of(work, struct request_queue, requeue_work.work); 767 LIST_HEAD(rq_list); 768 struct request *rq, *next; 769 770 spin_lock_irq(&q->requeue_lock); 771 list_splice_init(&q->requeue_list, &rq_list); 772 spin_unlock_irq(&q->requeue_lock); 773 774 list_for_each_entry_safe(rq, next, &rq_list, queuelist) { 775 if (!(rq->rq_flags & (RQF_SOFTBARRIER | RQF_DONTPREP))) 776 continue; 777 778 rq->rq_flags &= ~RQF_SOFTBARRIER; 779 list_del_init(&rq->queuelist); 780 /* 781 * If RQF_DONTPREP, rq has contained some driver specific 782 * data, so insert it to hctx dispatch list to avoid any 783 * merge. 784 */ 785 if (rq->rq_flags & RQF_DONTPREP) 786 blk_mq_request_bypass_insert(rq, false, false); 787 else 788 blk_mq_sched_insert_request(rq, true, false, false); 789 } 790 791 while (!list_empty(&rq_list)) { 792 rq = list_entry(rq_list.next, struct request, queuelist); 793 list_del_init(&rq->queuelist); 794 blk_mq_sched_insert_request(rq, false, false, false); 795 } 796 797 blk_mq_run_hw_queues(q, false); 798 } 799 800 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, 801 bool kick_requeue_list) 802 { 803 struct request_queue *q = rq->q; 804 unsigned long flags; 805 806 /* 807 * We abuse this flag that is otherwise used by the I/O scheduler to 808 * request head insertion from the workqueue. 809 */ 810 BUG_ON(rq->rq_flags & RQF_SOFTBARRIER); 811 812 spin_lock_irqsave(&q->requeue_lock, flags); 813 if (at_head) { 814 rq->rq_flags |= RQF_SOFTBARRIER; 815 list_add(&rq->queuelist, &q->requeue_list); 816 } else { 817 list_add_tail(&rq->queuelist, &q->requeue_list); 818 } 819 spin_unlock_irqrestore(&q->requeue_lock, flags); 820 821 if (kick_requeue_list) 822 blk_mq_kick_requeue_list(q); 823 } 824 825 void blk_mq_kick_requeue_list(struct request_queue *q) 826 { 827 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0); 828 } 829 EXPORT_SYMBOL(blk_mq_kick_requeue_list); 830 831 void blk_mq_delay_kick_requeue_list(struct request_queue *q, 832 unsigned long msecs) 833 { 834 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 835 msecs_to_jiffies(msecs)); 836 } 837 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); 838 839 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) 840 { 841 if (tag < tags->nr_tags) { 842 prefetch(tags->rqs[tag]); 843 return tags->rqs[tag]; 844 } 845 846 return NULL; 847 } 848 EXPORT_SYMBOL(blk_mq_tag_to_rq); 849 850 static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq, 851 void *priv, bool reserved) 852 { 853 /* 854 * If we find a request that isn't idle and the queue matches, 855 * we know the queue is busy. Return false to stop the iteration. 856 */ 857 if (blk_mq_request_started(rq) && rq->q == hctx->queue) { 858 bool *busy = priv; 859 860 *busy = true; 861 return false; 862 } 863 864 return true; 865 } 866 867 bool blk_mq_queue_inflight(struct request_queue *q) 868 { 869 bool busy = false; 870 871 blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy); 872 return busy; 873 } 874 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight); 875 876 static void blk_mq_rq_timed_out(struct request *req, bool reserved) 877 { 878 req->rq_flags |= RQF_TIMED_OUT; 879 if (req->q->mq_ops->timeout) { 880 enum blk_eh_timer_return ret; 881 882 ret = req->q->mq_ops->timeout(req, reserved); 883 if (ret == BLK_EH_DONE) 884 return; 885 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER); 886 } 887 888 blk_add_timer(req); 889 } 890 891 static bool blk_mq_req_expired(struct request *rq, unsigned long *next) 892 { 893 unsigned long deadline; 894 895 if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT) 896 return false; 897 if (rq->rq_flags & RQF_TIMED_OUT) 898 return false; 899 900 deadline = READ_ONCE(rq->deadline); 901 if (time_after_eq(jiffies, deadline)) 902 return true; 903 904 if (*next == 0) 905 *next = deadline; 906 else if (time_after(*next, deadline)) 907 *next = deadline; 908 return false; 909 } 910 911 static bool blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, 912 struct request *rq, void *priv, bool reserved) 913 { 914 unsigned long *next = priv; 915 916 /* 917 * Just do a quick check if it is expired before locking the request in 918 * so we're not unnecessarilly synchronizing across CPUs. 919 */ 920 if (!blk_mq_req_expired(rq, next)) 921 return true; 922 923 /* 924 * We have reason to believe the request may be expired. Take a 925 * reference on the request to lock this request lifetime into its 926 * currently allocated context to prevent it from being reallocated in 927 * the event the completion by-passes this timeout handler. 928 * 929 * If the reference was already released, then the driver beat the 930 * timeout handler to posting a natural completion. 931 */ 932 if (!refcount_inc_not_zero(&rq->ref)) 933 return true; 934 935 /* 936 * The request is now locked and cannot be reallocated underneath the 937 * timeout handler's processing. Re-verify this exact request is truly 938 * expired; if it is not expired, then the request was completed and 939 * reallocated as a new request. 940 */ 941 if (blk_mq_req_expired(rq, next)) 942 blk_mq_rq_timed_out(rq, reserved); 943 944 if (is_flush_rq(rq, hctx)) 945 rq->end_io(rq, 0); 946 else if (refcount_dec_and_test(&rq->ref)) 947 __blk_mq_free_request(rq); 948 949 return true; 950 } 951 952 static void blk_mq_timeout_work(struct work_struct *work) 953 { 954 struct request_queue *q = 955 container_of(work, struct request_queue, timeout_work); 956 unsigned long next = 0; 957 struct blk_mq_hw_ctx *hctx; 958 int i; 959 960 /* A deadlock might occur if a request is stuck requiring a 961 * timeout at the same time a queue freeze is waiting 962 * completion, since the timeout code would not be able to 963 * acquire the queue reference here. 964 * 965 * That's why we don't use blk_queue_enter here; instead, we use 966 * percpu_ref_tryget directly, because we need to be able to 967 * obtain a reference even in the short window between the queue 968 * starting to freeze, by dropping the first reference in 969 * blk_freeze_queue_start, and the moment the last request is 970 * consumed, marked by the instant q_usage_counter reaches 971 * zero. 972 */ 973 if (!percpu_ref_tryget(&q->q_usage_counter)) 974 return; 975 976 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next); 977 978 if (next != 0) { 979 mod_timer(&q->timeout, next); 980 } else { 981 /* 982 * Request timeouts are handled as a forward rolling timer. If 983 * we end up here it means that no requests are pending and 984 * also that no request has been pending for a while. Mark 985 * each hctx as idle. 986 */ 987 queue_for_each_hw_ctx(q, hctx, i) { 988 /* the hctx may be unmapped, so check it here */ 989 if (blk_mq_hw_queue_mapped(hctx)) 990 blk_mq_tag_idle(hctx); 991 } 992 } 993 blk_queue_exit(q); 994 } 995 996 struct flush_busy_ctx_data { 997 struct blk_mq_hw_ctx *hctx; 998 struct list_head *list; 999 }; 1000 1001 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) 1002 { 1003 struct flush_busy_ctx_data *flush_data = data; 1004 struct blk_mq_hw_ctx *hctx = flush_data->hctx; 1005 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 1006 enum hctx_type type = hctx->type; 1007 1008 spin_lock(&ctx->lock); 1009 list_splice_tail_init(&ctx->rq_lists[type], flush_data->list); 1010 sbitmap_clear_bit(sb, bitnr); 1011 spin_unlock(&ctx->lock); 1012 return true; 1013 } 1014 1015 /* 1016 * Process software queues that have been marked busy, splicing them 1017 * to the for-dispatch 1018 */ 1019 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) 1020 { 1021 struct flush_busy_ctx_data data = { 1022 .hctx = hctx, 1023 .list = list, 1024 }; 1025 1026 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data); 1027 } 1028 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs); 1029 1030 struct dispatch_rq_data { 1031 struct blk_mq_hw_ctx *hctx; 1032 struct request *rq; 1033 }; 1034 1035 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr, 1036 void *data) 1037 { 1038 struct dispatch_rq_data *dispatch_data = data; 1039 struct blk_mq_hw_ctx *hctx = dispatch_data->hctx; 1040 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 1041 enum hctx_type type = hctx->type; 1042 1043 spin_lock(&ctx->lock); 1044 if (!list_empty(&ctx->rq_lists[type])) { 1045 dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next); 1046 list_del_init(&dispatch_data->rq->queuelist); 1047 if (list_empty(&ctx->rq_lists[type])) 1048 sbitmap_clear_bit(sb, bitnr); 1049 } 1050 spin_unlock(&ctx->lock); 1051 1052 return !dispatch_data->rq; 1053 } 1054 1055 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, 1056 struct blk_mq_ctx *start) 1057 { 1058 unsigned off = start ? start->index_hw[hctx->type] : 0; 1059 struct dispatch_rq_data data = { 1060 .hctx = hctx, 1061 .rq = NULL, 1062 }; 1063 1064 __sbitmap_for_each_set(&hctx->ctx_map, off, 1065 dispatch_rq_from_ctx, &data); 1066 1067 return data.rq; 1068 } 1069 1070 static inline unsigned int queued_to_index(unsigned int queued) 1071 { 1072 if (!queued) 1073 return 0; 1074 1075 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); 1076 } 1077 1078 static bool __blk_mq_get_driver_tag(struct request *rq) 1079 { 1080 struct sbitmap_queue *bt = rq->mq_hctx->tags->bitmap_tags; 1081 unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags; 1082 int tag; 1083 1084 blk_mq_tag_busy(rq->mq_hctx); 1085 1086 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) { 1087 bt = rq->mq_hctx->tags->breserved_tags; 1088 tag_offset = 0; 1089 } else { 1090 if (!hctx_may_queue(rq->mq_hctx, bt)) 1091 return false; 1092 } 1093 1094 tag = __sbitmap_queue_get(bt); 1095 if (tag == BLK_MQ_NO_TAG) 1096 return false; 1097 1098 rq->tag = tag + tag_offset; 1099 return true; 1100 } 1101 1102 static bool blk_mq_get_driver_tag(struct request *rq) 1103 { 1104 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1105 1106 if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag(rq)) 1107 return false; 1108 1109 if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) && 1110 !(rq->rq_flags & RQF_MQ_INFLIGHT)) { 1111 rq->rq_flags |= RQF_MQ_INFLIGHT; 1112 __blk_mq_inc_active_requests(hctx); 1113 } 1114 hctx->tags->rqs[rq->tag] = rq; 1115 return true; 1116 } 1117 1118 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, 1119 int flags, void *key) 1120 { 1121 struct blk_mq_hw_ctx *hctx; 1122 1123 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); 1124 1125 spin_lock(&hctx->dispatch_wait_lock); 1126 if (!list_empty(&wait->entry)) { 1127 struct sbitmap_queue *sbq; 1128 1129 list_del_init(&wait->entry); 1130 sbq = hctx->tags->bitmap_tags; 1131 atomic_dec(&sbq->ws_active); 1132 } 1133 spin_unlock(&hctx->dispatch_wait_lock); 1134 1135 blk_mq_run_hw_queue(hctx, true); 1136 return 1; 1137 } 1138 1139 /* 1140 * Mark us waiting for a tag. For shared tags, this involves hooking us into 1141 * the tag wakeups. For non-shared tags, we can simply mark us needing a 1142 * restart. For both cases, take care to check the condition again after 1143 * marking us as waiting. 1144 */ 1145 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx, 1146 struct request *rq) 1147 { 1148 struct sbitmap_queue *sbq = hctx->tags->bitmap_tags; 1149 struct wait_queue_head *wq; 1150 wait_queue_entry_t *wait; 1151 bool ret; 1152 1153 if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) { 1154 blk_mq_sched_mark_restart_hctx(hctx); 1155 1156 /* 1157 * It's possible that a tag was freed in the window between the 1158 * allocation failure and adding the hardware queue to the wait 1159 * queue. 1160 * 1161 * Don't clear RESTART here, someone else could have set it. 1162 * At most this will cost an extra queue run. 1163 */ 1164 return blk_mq_get_driver_tag(rq); 1165 } 1166 1167 wait = &hctx->dispatch_wait; 1168 if (!list_empty_careful(&wait->entry)) 1169 return false; 1170 1171 wq = &bt_wait_ptr(sbq, hctx)->wait; 1172 1173 spin_lock_irq(&wq->lock); 1174 spin_lock(&hctx->dispatch_wait_lock); 1175 if (!list_empty(&wait->entry)) { 1176 spin_unlock(&hctx->dispatch_wait_lock); 1177 spin_unlock_irq(&wq->lock); 1178 return false; 1179 } 1180 1181 atomic_inc(&sbq->ws_active); 1182 wait->flags &= ~WQ_FLAG_EXCLUSIVE; 1183 __add_wait_queue(wq, wait); 1184 1185 /* 1186 * It's possible that a tag was freed in the window between the 1187 * allocation failure and adding the hardware queue to the wait 1188 * queue. 1189 */ 1190 ret = blk_mq_get_driver_tag(rq); 1191 if (!ret) { 1192 spin_unlock(&hctx->dispatch_wait_lock); 1193 spin_unlock_irq(&wq->lock); 1194 return false; 1195 } 1196 1197 /* 1198 * We got a tag, remove ourselves from the wait queue to ensure 1199 * someone else gets the wakeup. 1200 */ 1201 list_del_init(&wait->entry); 1202 atomic_dec(&sbq->ws_active); 1203 spin_unlock(&hctx->dispatch_wait_lock); 1204 spin_unlock_irq(&wq->lock); 1205 1206 return true; 1207 } 1208 1209 #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT 8 1210 #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR 4 1211 /* 1212 * Update dispatch busy with the Exponential Weighted Moving Average(EWMA): 1213 * - EWMA is one simple way to compute running average value 1214 * - weight(7/8 and 1/8) is applied so that it can decrease exponentially 1215 * - take 4 as factor for avoiding to get too small(0) result, and this 1216 * factor doesn't matter because EWMA decreases exponentially 1217 */ 1218 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy) 1219 { 1220 unsigned int ewma; 1221 1222 if (hctx->queue->elevator) 1223 return; 1224 1225 ewma = hctx->dispatch_busy; 1226 1227 if (!ewma && !busy) 1228 return; 1229 1230 ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1; 1231 if (busy) 1232 ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR; 1233 ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT; 1234 1235 hctx->dispatch_busy = ewma; 1236 } 1237 1238 #define BLK_MQ_RESOURCE_DELAY 3 /* ms units */ 1239 1240 static void blk_mq_handle_dev_resource(struct request *rq, 1241 struct list_head *list) 1242 { 1243 struct request *next = 1244 list_first_entry_or_null(list, struct request, queuelist); 1245 1246 /* 1247 * If an I/O scheduler has been configured and we got a driver tag for 1248 * the next request already, free it. 1249 */ 1250 if (next) 1251 blk_mq_put_driver_tag(next); 1252 1253 list_add(&rq->queuelist, list); 1254 __blk_mq_requeue_request(rq); 1255 } 1256 1257 static void blk_mq_handle_zone_resource(struct request *rq, 1258 struct list_head *zone_list) 1259 { 1260 /* 1261 * If we end up here it is because we cannot dispatch a request to a 1262 * specific zone due to LLD level zone-write locking or other zone 1263 * related resource not being available. In this case, set the request 1264 * aside in zone_list for retrying it later. 1265 */ 1266 list_add(&rq->queuelist, zone_list); 1267 __blk_mq_requeue_request(rq); 1268 } 1269 1270 enum prep_dispatch { 1271 PREP_DISPATCH_OK, 1272 PREP_DISPATCH_NO_TAG, 1273 PREP_DISPATCH_NO_BUDGET, 1274 }; 1275 1276 static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq, 1277 bool need_budget) 1278 { 1279 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1280 int budget_token = -1; 1281 1282 if (need_budget) { 1283 budget_token = blk_mq_get_dispatch_budget(rq->q); 1284 if (budget_token < 0) { 1285 blk_mq_put_driver_tag(rq); 1286 return PREP_DISPATCH_NO_BUDGET; 1287 } 1288 blk_mq_set_rq_budget_token(rq, budget_token); 1289 } 1290 1291 if (!blk_mq_get_driver_tag(rq)) { 1292 /* 1293 * The initial allocation attempt failed, so we need to 1294 * rerun the hardware queue when a tag is freed. The 1295 * waitqueue takes care of that. If the queue is run 1296 * before we add this entry back on the dispatch list, 1297 * we'll re-run it below. 1298 */ 1299 if (!blk_mq_mark_tag_wait(hctx, rq)) { 1300 /* 1301 * All budgets not got from this function will be put 1302 * together during handling partial dispatch 1303 */ 1304 if (need_budget) 1305 blk_mq_put_dispatch_budget(rq->q, budget_token); 1306 return PREP_DISPATCH_NO_TAG; 1307 } 1308 } 1309 1310 return PREP_DISPATCH_OK; 1311 } 1312 1313 /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */ 1314 static void blk_mq_release_budgets(struct request_queue *q, 1315 struct list_head *list) 1316 { 1317 struct request *rq; 1318 1319 list_for_each_entry(rq, list, queuelist) { 1320 int budget_token = blk_mq_get_rq_budget_token(rq); 1321 1322 if (budget_token >= 0) 1323 blk_mq_put_dispatch_budget(q, budget_token); 1324 } 1325 } 1326 1327 /* 1328 * Returns true if we did some work AND can potentially do more. 1329 */ 1330 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, 1331 unsigned int nr_budgets) 1332 { 1333 enum prep_dispatch prep; 1334 struct request_queue *q = hctx->queue; 1335 struct request *rq, *nxt; 1336 int errors, queued; 1337 blk_status_t ret = BLK_STS_OK; 1338 LIST_HEAD(zone_list); 1339 1340 if (list_empty(list)) 1341 return false; 1342 1343 /* 1344 * Now process all the entries, sending them to the driver. 1345 */ 1346 errors = queued = 0; 1347 do { 1348 struct blk_mq_queue_data bd; 1349 1350 rq = list_first_entry(list, struct request, queuelist); 1351 1352 WARN_ON_ONCE(hctx != rq->mq_hctx); 1353 prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets); 1354 if (prep != PREP_DISPATCH_OK) 1355 break; 1356 1357 list_del_init(&rq->queuelist); 1358 1359 bd.rq = rq; 1360 1361 /* 1362 * Flag last if we have no more requests, or if we have more 1363 * but can't assign a driver tag to it. 1364 */ 1365 if (list_empty(list)) 1366 bd.last = true; 1367 else { 1368 nxt = list_first_entry(list, struct request, queuelist); 1369 bd.last = !blk_mq_get_driver_tag(nxt); 1370 } 1371 1372 /* 1373 * once the request is queued to lld, no need to cover the 1374 * budget any more 1375 */ 1376 if (nr_budgets) 1377 nr_budgets--; 1378 ret = q->mq_ops->queue_rq(hctx, &bd); 1379 switch (ret) { 1380 case BLK_STS_OK: 1381 queued++; 1382 break; 1383 case BLK_STS_RESOURCE: 1384 case BLK_STS_DEV_RESOURCE: 1385 blk_mq_handle_dev_resource(rq, list); 1386 goto out; 1387 case BLK_STS_ZONE_RESOURCE: 1388 /* 1389 * Move the request to zone_list and keep going through 1390 * the dispatch list to find more requests the drive can 1391 * accept. 1392 */ 1393 blk_mq_handle_zone_resource(rq, &zone_list); 1394 break; 1395 default: 1396 errors++; 1397 blk_mq_end_request(rq, ret); 1398 } 1399 } while (!list_empty(list)); 1400 out: 1401 if (!list_empty(&zone_list)) 1402 list_splice_tail_init(&zone_list, list); 1403 1404 hctx->dispatched[queued_to_index(queued)]++; 1405 1406 /* If we didn't flush the entire list, we could have told the driver 1407 * there was more coming, but that turned out to be a lie. 1408 */ 1409 if ((!list_empty(list) || errors) && q->mq_ops->commit_rqs && queued) 1410 q->mq_ops->commit_rqs(hctx); 1411 /* 1412 * Any items that need requeuing? Stuff them into hctx->dispatch, 1413 * that is where we will continue on next queue run. 1414 */ 1415 if (!list_empty(list)) { 1416 bool needs_restart; 1417 /* For non-shared tags, the RESTART check will suffice */ 1418 bool no_tag = prep == PREP_DISPATCH_NO_TAG && 1419 (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED); 1420 bool no_budget_avail = prep == PREP_DISPATCH_NO_BUDGET; 1421 1422 if (nr_budgets) 1423 blk_mq_release_budgets(q, list); 1424 1425 spin_lock(&hctx->lock); 1426 list_splice_tail_init(list, &hctx->dispatch); 1427 spin_unlock(&hctx->lock); 1428 1429 /* 1430 * Order adding requests to hctx->dispatch and checking 1431 * SCHED_RESTART flag. The pair of this smp_mb() is the one 1432 * in blk_mq_sched_restart(). Avoid restart code path to 1433 * miss the new added requests to hctx->dispatch, meantime 1434 * SCHED_RESTART is observed here. 1435 */ 1436 smp_mb(); 1437 1438 /* 1439 * If SCHED_RESTART was set by the caller of this function and 1440 * it is no longer set that means that it was cleared by another 1441 * thread and hence that a queue rerun is needed. 1442 * 1443 * If 'no_tag' is set, that means that we failed getting 1444 * a driver tag with an I/O scheduler attached. If our dispatch 1445 * waitqueue is no longer active, ensure that we run the queue 1446 * AFTER adding our entries back to the list. 1447 * 1448 * If no I/O scheduler has been configured it is possible that 1449 * the hardware queue got stopped and restarted before requests 1450 * were pushed back onto the dispatch list. Rerun the queue to 1451 * avoid starvation. Notes: 1452 * - blk_mq_run_hw_queue() checks whether or not a queue has 1453 * been stopped before rerunning a queue. 1454 * - Some but not all block drivers stop a queue before 1455 * returning BLK_STS_RESOURCE. Two exceptions are scsi-mq 1456 * and dm-rq. 1457 * 1458 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART 1459 * bit is set, run queue after a delay to avoid IO stalls 1460 * that could otherwise occur if the queue is idle. We'll do 1461 * similar if we couldn't get budget and SCHED_RESTART is set. 1462 */ 1463 needs_restart = blk_mq_sched_needs_restart(hctx); 1464 if (!needs_restart || 1465 (no_tag && list_empty_careful(&hctx->dispatch_wait.entry))) 1466 blk_mq_run_hw_queue(hctx, true); 1467 else if (needs_restart && (ret == BLK_STS_RESOURCE || 1468 no_budget_avail)) 1469 blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY); 1470 1471 blk_mq_update_dispatch_busy(hctx, true); 1472 return false; 1473 } else 1474 blk_mq_update_dispatch_busy(hctx, false); 1475 1476 return (queued + errors) != 0; 1477 } 1478 1479 /** 1480 * __blk_mq_run_hw_queue - Run a hardware queue. 1481 * @hctx: Pointer to the hardware queue to run. 1482 * 1483 * Send pending requests to the hardware. 1484 */ 1485 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) 1486 { 1487 int srcu_idx; 1488 1489 /* 1490 * We can't run the queue inline with ints disabled. Ensure that 1491 * we catch bad users of this early. 1492 */ 1493 WARN_ON_ONCE(in_interrupt()); 1494 1495 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); 1496 1497 hctx_lock(hctx, &srcu_idx); 1498 blk_mq_sched_dispatch_requests(hctx); 1499 hctx_unlock(hctx, srcu_idx); 1500 } 1501 1502 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx) 1503 { 1504 int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask); 1505 1506 if (cpu >= nr_cpu_ids) 1507 cpu = cpumask_first(hctx->cpumask); 1508 return cpu; 1509 } 1510 1511 /* 1512 * It'd be great if the workqueue API had a way to pass 1513 * in a mask and had some smarts for more clever placement. 1514 * For now we just round-robin here, switching for every 1515 * BLK_MQ_CPU_WORK_BATCH queued items. 1516 */ 1517 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) 1518 { 1519 bool tried = false; 1520 int next_cpu = hctx->next_cpu; 1521 1522 if (hctx->queue->nr_hw_queues == 1) 1523 return WORK_CPU_UNBOUND; 1524 1525 if (--hctx->next_cpu_batch <= 0) { 1526 select_cpu: 1527 next_cpu = cpumask_next_and(next_cpu, hctx->cpumask, 1528 cpu_online_mask); 1529 if (next_cpu >= nr_cpu_ids) 1530 next_cpu = blk_mq_first_mapped_cpu(hctx); 1531 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 1532 } 1533 1534 /* 1535 * Do unbound schedule if we can't find a online CPU for this hctx, 1536 * and it should only happen in the path of handling CPU DEAD. 1537 */ 1538 if (!cpu_online(next_cpu)) { 1539 if (!tried) { 1540 tried = true; 1541 goto select_cpu; 1542 } 1543 1544 /* 1545 * Make sure to re-select CPU next time once after CPUs 1546 * in hctx->cpumask become online again. 1547 */ 1548 hctx->next_cpu = next_cpu; 1549 hctx->next_cpu_batch = 1; 1550 return WORK_CPU_UNBOUND; 1551 } 1552 1553 hctx->next_cpu = next_cpu; 1554 return next_cpu; 1555 } 1556 1557 /** 1558 * __blk_mq_delay_run_hw_queue - Run (or schedule to run) a hardware queue. 1559 * @hctx: Pointer to the hardware queue to run. 1560 * @async: If we want to run the queue asynchronously. 1561 * @msecs: Milliseconds of delay to wait before running the queue. 1562 * 1563 * If !@async, try to run the queue now. Else, run the queue asynchronously and 1564 * with a delay of @msecs. 1565 */ 1566 static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, 1567 unsigned long msecs) 1568 { 1569 if (unlikely(blk_mq_hctx_stopped(hctx))) 1570 return; 1571 1572 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) { 1573 int cpu = get_cpu(); 1574 if (cpumask_test_cpu(cpu, hctx->cpumask)) { 1575 __blk_mq_run_hw_queue(hctx); 1576 put_cpu(); 1577 return; 1578 } 1579 1580 put_cpu(); 1581 } 1582 1583 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, 1584 msecs_to_jiffies(msecs)); 1585 } 1586 1587 /** 1588 * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously. 1589 * @hctx: Pointer to the hardware queue to run. 1590 * @msecs: Milliseconds of delay to wait before running the queue. 1591 * 1592 * Run a hardware queue asynchronously with a delay of @msecs. 1593 */ 1594 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) 1595 { 1596 __blk_mq_delay_run_hw_queue(hctx, true, msecs); 1597 } 1598 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); 1599 1600 /** 1601 * blk_mq_run_hw_queue - Start to run a hardware queue. 1602 * @hctx: Pointer to the hardware queue to run. 1603 * @async: If we want to run the queue asynchronously. 1604 * 1605 * Check if the request queue is not in a quiesced state and if there are 1606 * pending requests to be sent. If this is true, run the queue to send requests 1607 * to hardware. 1608 */ 1609 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1610 { 1611 int srcu_idx; 1612 bool need_run; 1613 1614 /* 1615 * When queue is quiesced, we may be switching io scheduler, or 1616 * updating nr_hw_queues, or other things, and we can't run queue 1617 * any more, even __blk_mq_hctx_has_pending() can't be called safely. 1618 * 1619 * And queue will be rerun in blk_mq_unquiesce_queue() if it is 1620 * quiesced. 1621 */ 1622 hctx_lock(hctx, &srcu_idx); 1623 need_run = !blk_queue_quiesced(hctx->queue) && 1624 blk_mq_hctx_has_pending(hctx); 1625 hctx_unlock(hctx, srcu_idx); 1626 1627 if (need_run) 1628 __blk_mq_delay_run_hw_queue(hctx, async, 0); 1629 } 1630 EXPORT_SYMBOL(blk_mq_run_hw_queue); 1631 1632 /* 1633 * Is the request queue handled by an IO scheduler that does not respect 1634 * hardware queues when dispatching? 1635 */ 1636 static bool blk_mq_has_sqsched(struct request_queue *q) 1637 { 1638 struct elevator_queue *e = q->elevator; 1639 1640 if (e && e->type->ops.dispatch_request && 1641 !(e->type->elevator_features & ELEVATOR_F_MQ_AWARE)) 1642 return true; 1643 return false; 1644 } 1645 1646 /* 1647 * Return prefered queue to dispatch from (if any) for non-mq aware IO 1648 * scheduler. 1649 */ 1650 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q) 1651 { 1652 struct blk_mq_hw_ctx *hctx; 1653 1654 /* 1655 * If the IO scheduler does not respect hardware queues when 1656 * dispatching, we just don't bother with multiple HW queues and 1657 * dispatch from hctx for the current CPU since running multiple queues 1658 * just causes lock contention inside the scheduler and pointless cache 1659 * bouncing. 1660 */ 1661 hctx = blk_mq_map_queue_type(q, HCTX_TYPE_DEFAULT, 1662 raw_smp_processor_id()); 1663 if (!blk_mq_hctx_stopped(hctx)) 1664 return hctx; 1665 return NULL; 1666 } 1667 1668 /** 1669 * blk_mq_run_hw_queues - Run all hardware queues in a request queue. 1670 * @q: Pointer to the request queue to run. 1671 * @async: If we want to run the queue asynchronously. 1672 */ 1673 void blk_mq_run_hw_queues(struct request_queue *q, bool async) 1674 { 1675 struct blk_mq_hw_ctx *hctx, *sq_hctx; 1676 int i; 1677 1678 sq_hctx = NULL; 1679 if (blk_mq_has_sqsched(q)) 1680 sq_hctx = blk_mq_get_sq_hctx(q); 1681 queue_for_each_hw_ctx(q, hctx, i) { 1682 if (blk_mq_hctx_stopped(hctx)) 1683 continue; 1684 /* 1685 * Dispatch from this hctx either if there's no hctx preferred 1686 * by IO scheduler or if it has requests that bypass the 1687 * scheduler. 1688 */ 1689 if (!sq_hctx || sq_hctx == hctx || 1690 !list_empty_careful(&hctx->dispatch)) 1691 blk_mq_run_hw_queue(hctx, async); 1692 } 1693 } 1694 EXPORT_SYMBOL(blk_mq_run_hw_queues); 1695 1696 /** 1697 * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously. 1698 * @q: Pointer to the request queue to run. 1699 * @msecs: Milliseconds of delay to wait before running the queues. 1700 */ 1701 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs) 1702 { 1703 struct blk_mq_hw_ctx *hctx, *sq_hctx; 1704 int i; 1705 1706 sq_hctx = NULL; 1707 if (blk_mq_has_sqsched(q)) 1708 sq_hctx = blk_mq_get_sq_hctx(q); 1709 queue_for_each_hw_ctx(q, hctx, i) { 1710 if (blk_mq_hctx_stopped(hctx)) 1711 continue; 1712 /* 1713 * Dispatch from this hctx either if there's no hctx preferred 1714 * by IO scheduler or if it has requests that bypass the 1715 * scheduler. 1716 */ 1717 if (!sq_hctx || sq_hctx == hctx || 1718 !list_empty_careful(&hctx->dispatch)) 1719 blk_mq_delay_run_hw_queue(hctx, msecs); 1720 } 1721 } 1722 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues); 1723 1724 /** 1725 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped 1726 * @q: request queue. 1727 * 1728 * The caller is responsible for serializing this function against 1729 * blk_mq_{start,stop}_hw_queue(). 1730 */ 1731 bool blk_mq_queue_stopped(struct request_queue *q) 1732 { 1733 struct blk_mq_hw_ctx *hctx; 1734 int i; 1735 1736 queue_for_each_hw_ctx(q, hctx, i) 1737 if (blk_mq_hctx_stopped(hctx)) 1738 return true; 1739 1740 return false; 1741 } 1742 EXPORT_SYMBOL(blk_mq_queue_stopped); 1743 1744 /* 1745 * This function is often used for pausing .queue_rq() by driver when 1746 * there isn't enough resource or some conditions aren't satisfied, and 1747 * BLK_STS_RESOURCE is usually returned. 1748 * 1749 * We do not guarantee that dispatch can be drained or blocked 1750 * after blk_mq_stop_hw_queue() returns. Please use 1751 * blk_mq_quiesce_queue() for that requirement. 1752 */ 1753 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) 1754 { 1755 cancel_delayed_work(&hctx->run_work); 1756 1757 set_bit(BLK_MQ_S_STOPPED, &hctx->state); 1758 } 1759 EXPORT_SYMBOL(blk_mq_stop_hw_queue); 1760 1761 /* 1762 * This function is often used for pausing .queue_rq() by driver when 1763 * there isn't enough resource or some conditions aren't satisfied, and 1764 * BLK_STS_RESOURCE is usually returned. 1765 * 1766 * We do not guarantee that dispatch can be drained or blocked 1767 * after blk_mq_stop_hw_queues() returns. Please use 1768 * blk_mq_quiesce_queue() for that requirement. 1769 */ 1770 void blk_mq_stop_hw_queues(struct request_queue *q) 1771 { 1772 struct blk_mq_hw_ctx *hctx; 1773 int i; 1774 1775 queue_for_each_hw_ctx(q, hctx, i) 1776 blk_mq_stop_hw_queue(hctx); 1777 } 1778 EXPORT_SYMBOL(blk_mq_stop_hw_queues); 1779 1780 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) 1781 { 1782 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1783 1784 blk_mq_run_hw_queue(hctx, false); 1785 } 1786 EXPORT_SYMBOL(blk_mq_start_hw_queue); 1787 1788 void blk_mq_start_hw_queues(struct request_queue *q) 1789 { 1790 struct blk_mq_hw_ctx *hctx; 1791 int i; 1792 1793 queue_for_each_hw_ctx(q, hctx, i) 1794 blk_mq_start_hw_queue(hctx); 1795 } 1796 EXPORT_SYMBOL(blk_mq_start_hw_queues); 1797 1798 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1799 { 1800 if (!blk_mq_hctx_stopped(hctx)) 1801 return; 1802 1803 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1804 blk_mq_run_hw_queue(hctx, async); 1805 } 1806 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue); 1807 1808 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) 1809 { 1810 struct blk_mq_hw_ctx *hctx; 1811 int i; 1812 1813 queue_for_each_hw_ctx(q, hctx, i) 1814 blk_mq_start_stopped_hw_queue(hctx, async); 1815 } 1816 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); 1817 1818 static void blk_mq_run_work_fn(struct work_struct *work) 1819 { 1820 struct blk_mq_hw_ctx *hctx; 1821 1822 hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work); 1823 1824 /* 1825 * If we are stopped, don't run the queue. 1826 */ 1827 if (blk_mq_hctx_stopped(hctx)) 1828 return; 1829 1830 __blk_mq_run_hw_queue(hctx); 1831 } 1832 1833 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx, 1834 struct request *rq, 1835 bool at_head) 1836 { 1837 struct blk_mq_ctx *ctx = rq->mq_ctx; 1838 enum hctx_type type = hctx->type; 1839 1840 lockdep_assert_held(&ctx->lock); 1841 1842 trace_block_rq_insert(rq); 1843 1844 if (at_head) 1845 list_add(&rq->queuelist, &ctx->rq_lists[type]); 1846 else 1847 list_add_tail(&rq->queuelist, &ctx->rq_lists[type]); 1848 } 1849 1850 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, 1851 bool at_head) 1852 { 1853 struct blk_mq_ctx *ctx = rq->mq_ctx; 1854 1855 lockdep_assert_held(&ctx->lock); 1856 1857 __blk_mq_insert_req_list(hctx, rq, at_head); 1858 blk_mq_hctx_mark_pending(hctx, ctx); 1859 } 1860 1861 /** 1862 * blk_mq_request_bypass_insert - Insert a request at dispatch list. 1863 * @rq: Pointer to request to be inserted. 1864 * @at_head: true if the request should be inserted at the head of the list. 1865 * @run_queue: If we should run the hardware queue after inserting the request. 1866 * 1867 * Should only be used carefully, when the caller knows we want to 1868 * bypass a potential IO scheduler on the target device. 1869 */ 1870 void blk_mq_request_bypass_insert(struct request *rq, bool at_head, 1871 bool run_queue) 1872 { 1873 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1874 1875 spin_lock(&hctx->lock); 1876 if (at_head) 1877 list_add(&rq->queuelist, &hctx->dispatch); 1878 else 1879 list_add_tail(&rq->queuelist, &hctx->dispatch); 1880 spin_unlock(&hctx->lock); 1881 1882 if (run_queue) 1883 blk_mq_run_hw_queue(hctx, false); 1884 } 1885 1886 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 1887 struct list_head *list) 1888 1889 { 1890 struct request *rq; 1891 enum hctx_type type = hctx->type; 1892 1893 /* 1894 * preemption doesn't flush plug list, so it's possible ctx->cpu is 1895 * offline now 1896 */ 1897 list_for_each_entry(rq, list, queuelist) { 1898 BUG_ON(rq->mq_ctx != ctx); 1899 trace_block_rq_insert(rq); 1900 } 1901 1902 spin_lock(&ctx->lock); 1903 list_splice_tail_init(list, &ctx->rq_lists[type]); 1904 blk_mq_hctx_mark_pending(hctx, ctx); 1905 spin_unlock(&ctx->lock); 1906 } 1907 1908 static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b) 1909 { 1910 struct request *rqa = container_of(a, struct request, queuelist); 1911 struct request *rqb = container_of(b, struct request, queuelist); 1912 1913 if (rqa->mq_ctx != rqb->mq_ctx) 1914 return rqa->mq_ctx > rqb->mq_ctx; 1915 if (rqa->mq_hctx != rqb->mq_hctx) 1916 return rqa->mq_hctx > rqb->mq_hctx; 1917 1918 return blk_rq_pos(rqa) > blk_rq_pos(rqb); 1919 } 1920 1921 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 1922 { 1923 LIST_HEAD(list); 1924 1925 if (list_empty(&plug->mq_list)) 1926 return; 1927 list_splice_init(&plug->mq_list, &list); 1928 1929 if (plug->rq_count > 2 && plug->multiple_queues) 1930 list_sort(NULL, &list, plug_rq_cmp); 1931 1932 plug->rq_count = 0; 1933 1934 do { 1935 struct list_head rq_list; 1936 struct request *rq, *head_rq = list_entry_rq(list.next); 1937 struct list_head *pos = &head_rq->queuelist; /* skip first */ 1938 struct blk_mq_hw_ctx *this_hctx = head_rq->mq_hctx; 1939 struct blk_mq_ctx *this_ctx = head_rq->mq_ctx; 1940 unsigned int depth = 1; 1941 1942 list_for_each_continue(pos, &list) { 1943 rq = list_entry_rq(pos); 1944 BUG_ON(!rq->q); 1945 if (rq->mq_hctx != this_hctx || rq->mq_ctx != this_ctx) 1946 break; 1947 depth++; 1948 } 1949 1950 list_cut_before(&rq_list, &list, pos); 1951 trace_block_unplug(head_rq->q, depth, !from_schedule); 1952 blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list, 1953 from_schedule); 1954 } while(!list_empty(&list)); 1955 } 1956 1957 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio, 1958 unsigned int nr_segs) 1959 { 1960 int err; 1961 1962 if (bio->bi_opf & REQ_RAHEAD) 1963 rq->cmd_flags |= REQ_FAILFAST_MASK; 1964 1965 rq->__sector = bio->bi_iter.bi_sector; 1966 rq->write_hint = bio->bi_write_hint; 1967 blk_rq_bio_prep(rq, bio, nr_segs); 1968 1969 /* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */ 1970 err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO); 1971 WARN_ON_ONCE(err); 1972 1973 blk_account_io_start(rq); 1974 } 1975 1976 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx, 1977 struct request *rq, 1978 blk_qc_t *cookie, bool last) 1979 { 1980 struct request_queue *q = rq->q; 1981 struct blk_mq_queue_data bd = { 1982 .rq = rq, 1983 .last = last, 1984 }; 1985 blk_qc_t new_cookie; 1986 blk_status_t ret; 1987 1988 new_cookie = request_to_qc_t(hctx, rq); 1989 1990 /* 1991 * For OK queue, we are done. For error, caller may kill it. 1992 * Any other error (busy), just add it to our list as we 1993 * previously would have done. 1994 */ 1995 ret = q->mq_ops->queue_rq(hctx, &bd); 1996 switch (ret) { 1997 case BLK_STS_OK: 1998 blk_mq_update_dispatch_busy(hctx, false); 1999 *cookie = new_cookie; 2000 break; 2001 case BLK_STS_RESOURCE: 2002 case BLK_STS_DEV_RESOURCE: 2003 blk_mq_update_dispatch_busy(hctx, true); 2004 __blk_mq_requeue_request(rq); 2005 break; 2006 default: 2007 blk_mq_update_dispatch_busy(hctx, false); 2008 *cookie = BLK_QC_T_NONE; 2009 break; 2010 } 2011 2012 return ret; 2013 } 2014 2015 static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 2016 struct request *rq, 2017 blk_qc_t *cookie, 2018 bool bypass_insert, bool last) 2019 { 2020 struct request_queue *q = rq->q; 2021 bool run_queue = true; 2022 int budget_token; 2023 2024 /* 2025 * RCU or SRCU read lock is needed before checking quiesced flag. 2026 * 2027 * When queue is stopped or quiesced, ignore 'bypass_insert' from 2028 * blk_mq_request_issue_directly(), and return BLK_STS_OK to caller, 2029 * and avoid driver to try to dispatch again. 2030 */ 2031 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) { 2032 run_queue = false; 2033 bypass_insert = false; 2034 goto insert; 2035 } 2036 2037 if (q->elevator && !bypass_insert) 2038 goto insert; 2039 2040 budget_token = blk_mq_get_dispatch_budget(q); 2041 if (budget_token < 0) 2042 goto insert; 2043 2044 blk_mq_set_rq_budget_token(rq, budget_token); 2045 2046 if (!blk_mq_get_driver_tag(rq)) { 2047 blk_mq_put_dispatch_budget(q, budget_token); 2048 goto insert; 2049 } 2050 2051 return __blk_mq_issue_directly(hctx, rq, cookie, last); 2052 insert: 2053 if (bypass_insert) 2054 return BLK_STS_RESOURCE; 2055 2056 blk_mq_sched_insert_request(rq, false, run_queue, false); 2057 2058 return BLK_STS_OK; 2059 } 2060 2061 /** 2062 * blk_mq_try_issue_directly - Try to send a request directly to device driver. 2063 * @hctx: Pointer of the associated hardware queue. 2064 * @rq: Pointer to request to be sent. 2065 * @cookie: Request queue cookie. 2066 * 2067 * If the device has enough resources to accept a new request now, send the 2068 * request directly to device driver. Else, insert at hctx->dispatch queue, so 2069 * we can try send it another time in the future. Requests inserted at this 2070 * queue have higher priority. 2071 */ 2072 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 2073 struct request *rq, blk_qc_t *cookie) 2074 { 2075 blk_status_t ret; 2076 int srcu_idx; 2077 2078 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); 2079 2080 hctx_lock(hctx, &srcu_idx); 2081 2082 ret = __blk_mq_try_issue_directly(hctx, rq, cookie, false, true); 2083 if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) 2084 blk_mq_request_bypass_insert(rq, false, true); 2085 else if (ret != BLK_STS_OK) 2086 blk_mq_end_request(rq, ret); 2087 2088 hctx_unlock(hctx, srcu_idx); 2089 } 2090 2091 blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last) 2092 { 2093 blk_status_t ret; 2094 int srcu_idx; 2095 blk_qc_t unused_cookie; 2096 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 2097 2098 hctx_lock(hctx, &srcu_idx); 2099 ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true, last); 2100 hctx_unlock(hctx, srcu_idx); 2101 2102 return ret; 2103 } 2104 2105 void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, 2106 struct list_head *list) 2107 { 2108 int queued = 0; 2109 int errors = 0; 2110 2111 while (!list_empty(list)) { 2112 blk_status_t ret; 2113 struct request *rq = list_first_entry(list, struct request, 2114 queuelist); 2115 2116 list_del_init(&rq->queuelist); 2117 ret = blk_mq_request_issue_directly(rq, list_empty(list)); 2118 if (ret != BLK_STS_OK) { 2119 if (ret == BLK_STS_RESOURCE || 2120 ret == BLK_STS_DEV_RESOURCE) { 2121 blk_mq_request_bypass_insert(rq, false, 2122 list_empty(list)); 2123 break; 2124 } 2125 blk_mq_end_request(rq, ret); 2126 errors++; 2127 } else 2128 queued++; 2129 } 2130 2131 /* 2132 * If we didn't flush the entire list, we could have told 2133 * the driver there was more coming, but that turned out to 2134 * be a lie. 2135 */ 2136 if ((!list_empty(list) || errors) && 2137 hctx->queue->mq_ops->commit_rqs && queued) 2138 hctx->queue->mq_ops->commit_rqs(hctx); 2139 } 2140 2141 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) 2142 { 2143 list_add_tail(&rq->queuelist, &plug->mq_list); 2144 plug->rq_count++; 2145 if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) { 2146 struct request *tmp; 2147 2148 tmp = list_first_entry(&plug->mq_list, struct request, 2149 queuelist); 2150 if (tmp->q != rq->q) 2151 plug->multiple_queues = true; 2152 } 2153 } 2154 2155 /** 2156 * blk_mq_submit_bio - Create and send a request to block device. 2157 * @bio: Bio pointer. 2158 * 2159 * Builds up a request structure from @q and @bio and send to the device. The 2160 * request may not be queued directly to hardware if: 2161 * * This request can be merged with another one 2162 * * We want to place request at plug queue for possible future merging 2163 * * There is an IO scheduler active at this queue 2164 * 2165 * It will not queue the request if there is an error with the bio, or at the 2166 * request creation. 2167 * 2168 * Returns: Request queue cookie. 2169 */ 2170 blk_qc_t blk_mq_submit_bio(struct bio *bio) 2171 { 2172 struct request_queue *q = bio->bi_bdev->bd_disk->queue; 2173 const int is_sync = op_is_sync(bio->bi_opf); 2174 const int is_flush_fua = op_is_flush(bio->bi_opf); 2175 struct blk_mq_alloc_data data = { 2176 .q = q, 2177 }; 2178 struct request *rq; 2179 struct blk_plug *plug; 2180 struct request *same_queue_rq = NULL; 2181 unsigned int nr_segs; 2182 blk_qc_t cookie; 2183 blk_status_t ret; 2184 bool hipri; 2185 2186 blk_queue_bounce(q, &bio); 2187 __blk_queue_split(&bio, &nr_segs); 2188 2189 if (!bio_integrity_prep(bio)) 2190 goto queue_exit; 2191 2192 if (!is_flush_fua && !blk_queue_nomerges(q) && 2193 blk_attempt_plug_merge(q, bio, nr_segs, &same_queue_rq)) 2194 goto queue_exit; 2195 2196 if (blk_mq_sched_bio_merge(q, bio, nr_segs)) 2197 goto queue_exit; 2198 2199 rq_qos_throttle(q, bio); 2200 2201 hipri = bio->bi_opf & REQ_HIPRI; 2202 2203 data.cmd_flags = bio->bi_opf; 2204 rq = __blk_mq_alloc_request(&data); 2205 if (unlikely(!rq)) { 2206 rq_qos_cleanup(q, bio); 2207 if (bio->bi_opf & REQ_NOWAIT) 2208 bio_wouldblock_error(bio); 2209 goto queue_exit; 2210 } 2211 2212 trace_block_getrq(bio); 2213 2214 rq_qos_track(q, rq, bio); 2215 2216 cookie = request_to_qc_t(data.hctx, rq); 2217 2218 blk_mq_bio_to_request(rq, bio, nr_segs); 2219 2220 ret = blk_crypto_init_request(rq); 2221 if (ret != BLK_STS_OK) { 2222 bio->bi_status = ret; 2223 bio_endio(bio); 2224 blk_mq_free_request(rq); 2225 return BLK_QC_T_NONE; 2226 } 2227 2228 plug = blk_mq_plug(q, bio); 2229 if (unlikely(is_flush_fua)) { 2230 /* Bypass scheduler for flush requests */ 2231 blk_insert_flush(rq); 2232 blk_mq_run_hw_queue(data.hctx, true); 2233 } else if (plug && (q->nr_hw_queues == 1 || q->mq_ops->commit_rqs || 2234 !blk_queue_nonrot(q))) { 2235 /* 2236 * Use plugging if we have a ->commit_rqs() hook as well, as 2237 * we know the driver uses bd->last in a smart fashion. 2238 * 2239 * Use normal plugging if this disk is slow HDD, as sequential 2240 * IO may benefit a lot from plug merging. 2241 */ 2242 unsigned int request_count = plug->rq_count; 2243 struct request *last = NULL; 2244 2245 if (!request_count) 2246 trace_block_plug(q); 2247 else 2248 last = list_entry_rq(plug->mq_list.prev); 2249 2250 if (request_count >= BLK_MAX_REQUEST_COUNT || (last && 2251 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { 2252 blk_flush_plug_list(plug, false); 2253 trace_block_plug(q); 2254 } 2255 2256 blk_add_rq_to_plug(plug, rq); 2257 } else if (q->elevator) { 2258 /* Insert the request at the IO scheduler queue */ 2259 blk_mq_sched_insert_request(rq, false, true, true); 2260 } else if (plug && !blk_queue_nomerges(q)) { 2261 /* 2262 * We do limited plugging. If the bio can be merged, do that. 2263 * Otherwise the existing request in the plug list will be 2264 * issued. So the plug list will have one request at most 2265 * The plug list might get flushed before this. If that happens, 2266 * the plug list is empty, and same_queue_rq is invalid. 2267 */ 2268 if (list_empty(&plug->mq_list)) 2269 same_queue_rq = NULL; 2270 if (same_queue_rq) { 2271 list_del_init(&same_queue_rq->queuelist); 2272 plug->rq_count--; 2273 } 2274 blk_add_rq_to_plug(plug, rq); 2275 trace_block_plug(q); 2276 2277 if (same_queue_rq) { 2278 data.hctx = same_queue_rq->mq_hctx; 2279 trace_block_unplug(q, 1, true); 2280 blk_mq_try_issue_directly(data.hctx, same_queue_rq, 2281 &cookie); 2282 } 2283 } else if ((q->nr_hw_queues > 1 && is_sync) || 2284 !data.hctx->dispatch_busy) { 2285 /* 2286 * There is no scheduler and we can try to send directly 2287 * to the hardware. 2288 */ 2289 blk_mq_try_issue_directly(data.hctx, rq, &cookie); 2290 } else { 2291 /* Default case. */ 2292 blk_mq_sched_insert_request(rq, false, true, true); 2293 } 2294 2295 if (!hipri) 2296 return BLK_QC_T_NONE; 2297 return cookie; 2298 queue_exit: 2299 blk_queue_exit(q); 2300 return BLK_QC_T_NONE; 2301 } 2302 2303 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 2304 unsigned int hctx_idx) 2305 { 2306 struct page *page; 2307 2308 if (tags->rqs && set->ops->exit_request) { 2309 int i; 2310 2311 for (i = 0; i < tags->nr_tags; i++) { 2312 struct request *rq = tags->static_rqs[i]; 2313 2314 if (!rq) 2315 continue; 2316 set->ops->exit_request(set, rq, hctx_idx); 2317 tags->static_rqs[i] = NULL; 2318 } 2319 } 2320 2321 while (!list_empty(&tags->page_list)) { 2322 page = list_first_entry(&tags->page_list, struct page, lru); 2323 list_del_init(&page->lru); 2324 /* 2325 * Remove kmemleak object previously allocated in 2326 * blk_mq_alloc_rqs(). 2327 */ 2328 kmemleak_free(page_address(page)); 2329 __free_pages(page, page->private); 2330 } 2331 } 2332 2333 void blk_mq_free_rq_map(struct blk_mq_tags *tags, unsigned int flags) 2334 { 2335 kfree(tags->rqs); 2336 tags->rqs = NULL; 2337 kfree(tags->static_rqs); 2338 tags->static_rqs = NULL; 2339 2340 blk_mq_free_tags(tags, flags); 2341 } 2342 2343 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, 2344 unsigned int hctx_idx, 2345 unsigned int nr_tags, 2346 unsigned int reserved_tags, 2347 unsigned int flags) 2348 { 2349 struct blk_mq_tags *tags; 2350 int node; 2351 2352 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx); 2353 if (node == NUMA_NO_NODE) 2354 node = set->numa_node; 2355 2356 tags = blk_mq_init_tags(nr_tags, reserved_tags, node, flags); 2357 if (!tags) 2358 return NULL; 2359 2360 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *), 2361 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 2362 node); 2363 if (!tags->rqs) { 2364 blk_mq_free_tags(tags, flags); 2365 return NULL; 2366 } 2367 2368 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *), 2369 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 2370 node); 2371 if (!tags->static_rqs) { 2372 kfree(tags->rqs); 2373 blk_mq_free_tags(tags, flags); 2374 return NULL; 2375 } 2376 2377 return tags; 2378 } 2379 2380 static size_t order_to_size(unsigned int order) 2381 { 2382 return (size_t)PAGE_SIZE << order; 2383 } 2384 2385 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, 2386 unsigned int hctx_idx, int node) 2387 { 2388 int ret; 2389 2390 if (set->ops->init_request) { 2391 ret = set->ops->init_request(set, rq, hctx_idx, node); 2392 if (ret) 2393 return ret; 2394 } 2395 2396 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 2397 return 0; 2398 } 2399 2400 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 2401 unsigned int hctx_idx, unsigned int depth) 2402 { 2403 unsigned int i, j, entries_per_page, max_order = 4; 2404 size_t rq_size, left; 2405 int node; 2406 2407 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx); 2408 if (node == NUMA_NO_NODE) 2409 node = set->numa_node; 2410 2411 INIT_LIST_HEAD(&tags->page_list); 2412 2413 /* 2414 * rq_size is the size of the request plus driver payload, rounded 2415 * to the cacheline size 2416 */ 2417 rq_size = round_up(sizeof(struct request) + set->cmd_size, 2418 cache_line_size()); 2419 left = rq_size * depth; 2420 2421 for (i = 0; i < depth; ) { 2422 int this_order = max_order; 2423 struct page *page; 2424 int to_do; 2425 void *p; 2426 2427 while (this_order && left < order_to_size(this_order - 1)) 2428 this_order--; 2429 2430 do { 2431 page = alloc_pages_node(node, 2432 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, 2433 this_order); 2434 if (page) 2435 break; 2436 if (!this_order--) 2437 break; 2438 if (order_to_size(this_order) < rq_size) 2439 break; 2440 } while (1); 2441 2442 if (!page) 2443 goto fail; 2444 2445 page->private = this_order; 2446 list_add_tail(&page->lru, &tags->page_list); 2447 2448 p = page_address(page); 2449 /* 2450 * Allow kmemleak to scan these pages as they contain pointers 2451 * to additional allocations like via ops->init_request(). 2452 */ 2453 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); 2454 entries_per_page = order_to_size(this_order) / rq_size; 2455 to_do = min(entries_per_page, depth - i); 2456 left -= to_do * rq_size; 2457 for (j = 0; j < to_do; j++) { 2458 struct request *rq = p; 2459 2460 tags->static_rqs[i] = rq; 2461 if (blk_mq_init_request(set, rq, hctx_idx, node)) { 2462 tags->static_rqs[i] = NULL; 2463 goto fail; 2464 } 2465 2466 p += rq_size; 2467 i++; 2468 } 2469 } 2470 return 0; 2471 2472 fail: 2473 blk_mq_free_rqs(set, tags, hctx_idx); 2474 return -ENOMEM; 2475 } 2476 2477 struct rq_iter_data { 2478 struct blk_mq_hw_ctx *hctx; 2479 bool has_rq; 2480 }; 2481 2482 static bool blk_mq_has_request(struct request *rq, void *data, bool reserved) 2483 { 2484 struct rq_iter_data *iter_data = data; 2485 2486 if (rq->mq_hctx != iter_data->hctx) 2487 return true; 2488 iter_data->has_rq = true; 2489 return false; 2490 } 2491 2492 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx) 2493 { 2494 struct blk_mq_tags *tags = hctx->sched_tags ? 2495 hctx->sched_tags : hctx->tags; 2496 struct rq_iter_data data = { 2497 .hctx = hctx, 2498 }; 2499 2500 blk_mq_all_tag_iter(tags, blk_mq_has_request, &data); 2501 return data.has_rq; 2502 } 2503 2504 static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu, 2505 struct blk_mq_hw_ctx *hctx) 2506 { 2507 if (cpumask_next_and(-1, hctx->cpumask, cpu_online_mask) != cpu) 2508 return false; 2509 if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids) 2510 return false; 2511 return true; 2512 } 2513 2514 static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node) 2515 { 2516 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node, 2517 struct blk_mq_hw_ctx, cpuhp_online); 2518 2519 if (!cpumask_test_cpu(cpu, hctx->cpumask) || 2520 !blk_mq_last_cpu_in_hctx(cpu, hctx)) 2521 return 0; 2522 2523 /* 2524 * Prevent new request from being allocated on the current hctx. 2525 * 2526 * The smp_mb__after_atomic() Pairs with the implied barrier in 2527 * test_and_set_bit_lock in sbitmap_get(). Ensures the inactive flag is 2528 * seen once we return from the tag allocator. 2529 */ 2530 set_bit(BLK_MQ_S_INACTIVE, &hctx->state); 2531 smp_mb__after_atomic(); 2532 2533 /* 2534 * Try to grab a reference to the queue and wait for any outstanding 2535 * requests. If we could not grab a reference the queue has been 2536 * frozen and there are no requests. 2537 */ 2538 if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) { 2539 while (blk_mq_hctx_has_requests(hctx)) 2540 msleep(5); 2541 percpu_ref_put(&hctx->queue->q_usage_counter); 2542 } 2543 2544 return 0; 2545 } 2546 2547 static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node) 2548 { 2549 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node, 2550 struct blk_mq_hw_ctx, cpuhp_online); 2551 2552 if (cpumask_test_cpu(cpu, hctx->cpumask)) 2553 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state); 2554 return 0; 2555 } 2556 2557 /* 2558 * 'cpu' is going away. splice any existing rq_list entries from this 2559 * software queue to the hw queue dispatch list, and ensure that it 2560 * gets run. 2561 */ 2562 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) 2563 { 2564 struct blk_mq_hw_ctx *hctx; 2565 struct blk_mq_ctx *ctx; 2566 LIST_HEAD(tmp); 2567 enum hctx_type type; 2568 2569 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead); 2570 if (!cpumask_test_cpu(cpu, hctx->cpumask)) 2571 return 0; 2572 2573 ctx = __blk_mq_get_ctx(hctx->queue, cpu); 2574 type = hctx->type; 2575 2576 spin_lock(&ctx->lock); 2577 if (!list_empty(&ctx->rq_lists[type])) { 2578 list_splice_init(&ctx->rq_lists[type], &tmp); 2579 blk_mq_hctx_clear_pending(hctx, ctx); 2580 } 2581 spin_unlock(&ctx->lock); 2582 2583 if (list_empty(&tmp)) 2584 return 0; 2585 2586 spin_lock(&hctx->lock); 2587 list_splice_tail_init(&tmp, &hctx->dispatch); 2588 spin_unlock(&hctx->lock); 2589 2590 blk_mq_run_hw_queue(hctx, true); 2591 return 0; 2592 } 2593 2594 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) 2595 { 2596 if (!(hctx->flags & BLK_MQ_F_STACKING)) 2597 cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE, 2598 &hctx->cpuhp_online); 2599 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD, 2600 &hctx->cpuhp_dead); 2601 } 2602 2603 /* hctx->ctxs will be freed in queue's release handler */ 2604 static void blk_mq_exit_hctx(struct request_queue *q, 2605 struct blk_mq_tag_set *set, 2606 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 2607 { 2608 if (blk_mq_hw_queue_mapped(hctx)) 2609 blk_mq_tag_idle(hctx); 2610 2611 if (set->ops->exit_request) 2612 set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx); 2613 2614 if (set->ops->exit_hctx) 2615 set->ops->exit_hctx(hctx, hctx_idx); 2616 2617 blk_mq_remove_cpuhp(hctx); 2618 2619 spin_lock(&q->unused_hctx_lock); 2620 list_add(&hctx->hctx_list, &q->unused_hctx_list); 2621 spin_unlock(&q->unused_hctx_lock); 2622 } 2623 2624 static void blk_mq_exit_hw_queues(struct request_queue *q, 2625 struct blk_mq_tag_set *set, int nr_queue) 2626 { 2627 struct blk_mq_hw_ctx *hctx; 2628 unsigned int i; 2629 2630 queue_for_each_hw_ctx(q, hctx, i) { 2631 if (i == nr_queue) 2632 break; 2633 blk_mq_debugfs_unregister_hctx(hctx); 2634 blk_mq_exit_hctx(q, set, hctx, i); 2635 } 2636 } 2637 2638 static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set) 2639 { 2640 int hw_ctx_size = sizeof(struct blk_mq_hw_ctx); 2641 2642 BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu), 2643 __alignof__(struct blk_mq_hw_ctx)) != 2644 sizeof(struct blk_mq_hw_ctx)); 2645 2646 if (tag_set->flags & BLK_MQ_F_BLOCKING) 2647 hw_ctx_size += sizeof(struct srcu_struct); 2648 2649 return hw_ctx_size; 2650 } 2651 2652 static int blk_mq_init_hctx(struct request_queue *q, 2653 struct blk_mq_tag_set *set, 2654 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) 2655 { 2656 hctx->queue_num = hctx_idx; 2657 2658 if (!(hctx->flags & BLK_MQ_F_STACKING)) 2659 cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE, 2660 &hctx->cpuhp_online); 2661 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead); 2662 2663 hctx->tags = set->tags[hctx_idx]; 2664 2665 if (set->ops->init_hctx && 2666 set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) 2667 goto unregister_cpu_notifier; 2668 2669 if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx, 2670 hctx->numa_node)) 2671 goto exit_hctx; 2672 return 0; 2673 2674 exit_hctx: 2675 if (set->ops->exit_hctx) 2676 set->ops->exit_hctx(hctx, hctx_idx); 2677 unregister_cpu_notifier: 2678 blk_mq_remove_cpuhp(hctx); 2679 return -1; 2680 } 2681 2682 static struct blk_mq_hw_ctx * 2683 blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, 2684 int node) 2685 { 2686 struct blk_mq_hw_ctx *hctx; 2687 gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY; 2688 2689 hctx = kzalloc_node(blk_mq_hw_ctx_size(set), gfp, node); 2690 if (!hctx) 2691 goto fail_alloc_hctx; 2692 2693 if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node)) 2694 goto free_hctx; 2695 2696 atomic_set(&hctx->nr_active, 0); 2697 if (node == NUMA_NO_NODE) 2698 node = set->numa_node; 2699 hctx->numa_node = node; 2700 2701 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); 2702 spin_lock_init(&hctx->lock); 2703 INIT_LIST_HEAD(&hctx->dispatch); 2704 hctx->queue = q; 2705 hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED; 2706 2707 INIT_LIST_HEAD(&hctx->hctx_list); 2708 2709 /* 2710 * Allocate space for all possible cpus to avoid allocation at 2711 * runtime 2712 */ 2713 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *), 2714 gfp, node); 2715 if (!hctx->ctxs) 2716 goto free_cpumask; 2717 2718 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), 2719 gfp, node, false, false)) 2720 goto free_ctxs; 2721 hctx->nr_ctx = 0; 2722 2723 spin_lock_init(&hctx->dispatch_wait_lock); 2724 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); 2725 INIT_LIST_HEAD(&hctx->dispatch_wait.entry); 2726 2727 hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp); 2728 if (!hctx->fq) 2729 goto free_bitmap; 2730 2731 if (hctx->flags & BLK_MQ_F_BLOCKING) 2732 init_srcu_struct(hctx->srcu); 2733 blk_mq_hctx_kobj_init(hctx); 2734 2735 return hctx; 2736 2737 free_bitmap: 2738 sbitmap_free(&hctx->ctx_map); 2739 free_ctxs: 2740 kfree(hctx->ctxs); 2741 free_cpumask: 2742 free_cpumask_var(hctx->cpumask); 2743 free_hctx: 2744 kfree(hctx); 2745 fail_alloc_hctx: 2746 return NULL; 2747 } 2748 2749 static void blk_mq_init_cpu_queues(struct request_queue *q, 2750 unsigned int nr_hw_queues) 2751 { 2752 struct blk_mq_tag_set *set = q->tag_set; 2753 unsigned int i, j; 2754 2755 for_each_possible_cpu(i) { 2756 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i); 2757 struct blk_mq_hw_ctx *hctx; 2758 int k; 2759 2760 __ctx->cpu = i; 2761 spin_lock_init(&__ctx->lock); 2762 for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++) 2763 INIT_LIST_HEAD(&__ctx->rq_lists[k]); 2764 2765 __ctx->queue = q; 2766 2767 /* 2768 * Set local node, IFF we have more than one hw queue. If 2769 * not, we remain on the home node of the device 2770 */ 2771 for (j = 0; j < set->nr_maps; j++) { 2772 hctx = blk_mq_map_queue_type(q, j, i); 2773 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) 2774 hctx->numa_node = cpu_to_node(i); 2775 } 2776 } 2777 } 2778 2779 static bool __blk_mq_alloc_map_and_request(struct blk_mq_tag_set *set, 2780 int hctx_idx) 2781 { 2782 unsigned int flags = set->flags; 2783 int ret = 0; 2784 2785 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx, 2786 set->queue_depth, set->reserved_tags, flags); 2787 if (!set->tags[hctx_idx]) 2788 return false; 2789 2790 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx, 2791 set->queue_depth); 2792 if (!ret) 2793 return true; 2794 2795 blk_mq_free_rq_map(set->tags[hctx_idx], flags); 2796 set->tags[hctx_idx] = NULL; 2797 return false; 2798 } 2799 2800 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set, 2801 unsigned int hctx_idx) 2802 { 2803 unsigned int flags = set->flags; 2804 2805 if (set->tags && set->tags[hctx_idx]) { 2806 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx); 2807 blk_mq_free_rq_map(set->tags[hctx_idx], flags); 2808 set->tags[hctx_idx] = NULL; 2809 } 2810 } 2811 2812 static void blk_mq_map_swqueue(struct request_queue *q) 2813 { 2814 unsigned int i, j, hctx_idx; 2815 struct blk_mq_hw_ctx *hctx; 2816 struct blk_mq_ctx *ctx; 2817 struct blk_mq_tag_set *set = q->tag_set; 2818 2819 queue_for_each_hw_ctx(q, hctx, i) { 2820 cpumask_clear(hctx->cpumask); 2821 hctx->nr_ctx = 0; 2822 hctx->dispatch_from = NULL; 2823 } 2824 2825 /* 2826 * Map software to hardware queues. 2827 * 2828 * If the cpu isn't present, the cpu is mapped to first hctx. 2829 */ 2830 for_each_possible_cpu(i) { 2831 2832 ctx = per_cpu_ptr(q->queue_ctx, i); 2833 for (j = 0; j < set->nr_maps; j++) { 2834 if (!set->map[j].nr_queues) { 2835 ctx->hctxs[j] = blk_mq_map_queue_type(q, 2836 HCTX_TYPE_DEFAULT, i); 2837 continue; 2838 } 2839 hctx_idx = set->map[j].mq_map[i]; 2840 /* unmapped hw queue can be remapped after CPU topo changed */ 2841 if (!set->tags[hctx_idx] && 2842 !__blk_mq_alloc_map_and_request(set, hctx_idx)) { 2843 /* 2844 * If tags initialization fail for some hctx, 2845 * that hctx won't be brought online. In this 2846 * case, remap the current ctx to hctx[0] which 2847 * is guaranteed to always have tags allocated 2848 */ 2849 set->map[j].mq_map[i] = 0; 2850 } 2851 2852 hctx = blk_mq_map_queue_type(q, j, i); 2853 ctx->hctxs[j] = hctx; 2854 /* 2855 * If the CPU is already set in the mask, then we've 2856 * mapped this one already. This can happen if 2857 * devices share queues across queue maps. 2858 */ 2859 if (cpumask_test_cpu(i, hctx->cpumask)) 2860 continue; 2861 2862 cpumask_set_cpu(i, hctx->cpumask); 2863 hctx->type = j; 2864 ctx->index_hw[hctx->type] = hctx->nr_ctx; 2865 hctx->ctxs[hctx->nr_ctx++] = ctx; 2866 2867 /* 2868 * If the nr_ctx type overflows, we have exceeded the 2869 * amount of sw queues we can support. 2870 */ 2871 BUG_ON(!hctx->nr_ctx); 2872 } 2873 2874 for (; j < HCTX_MAX_TYPES; j++) 2875 ctx->hctxs[j] = blk_mq_map_queue_type(q, 2876 HCTX_TYPE_DEFAULT, i); 2877 } 2878 2879 queue_for_each_hw_ctx(q, hctx, i) { 2880 /* 2881 * If no software queues are mapped to this hardware queue, 2882 * disable it and free the request entries. 2883 */ 2884 if (!hctx->nr_ctx) { 2885 /* Never unmap queue 0. We need it as a 2886 * fallback in case of a new remap fails 2887 * allocation 2888 */ 2889 if (i && set->tags[i]) 2890 blk_mq_free_map_and_requests(set, i); 2891 2892 hctx->tags = NULL; 2893 continue; 2894 } 2895 2896 hctx->tags = set->tags[i]; 2897 WARN_ON(!hctx->tags); 2898 2899 /* 2900 * Set the map size to the number of mapped software queues. 2901 * This is more accurate and more efficient than looping 2902 * over all possibly mapped software queues. 2903 */ 2904 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx); 2905 2906 /* 2907 * Initialize batch roundrobin counts 2908 */ 2909 hctx->next_cpu = blk_mq_first_mapped_cpu(hctx); 2910 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 2911 } 2912 } 2913 2914 /* 2915 * Caller needs to ensure that we're either frozen/quiesced, or that 2916 * the queue isn't live yet. 2917 */ 2918 static void queue_set_hctx_shared(struct request_queue *q, bool shared) 2919 { 2920 struct blk_mq_hw_ctx *hctx; 2921 int i; 2922 2923 queue_for_each_hw_ctx(q, hctx, i) { 2924 if (shared) 2925 hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED; 2926 else 2927 hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED; 2928 } 2929 } 2930 2931 static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set, 2932 bool shared) 2933 { 2934 struct request_queue *q; 2935 2936 lockdep_assert_held(&set->tag_list_lock); 2937 2938 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2939 blk_mq_freeze_queue(q); 2940 queue_set_hctx_shared(q, shared); 2941 blk_mq_unfreeze_queue(q); 2942 } 2943 } 2944 2945 static void blk_mq_del_queue_tag_set(struct request_queue *q) 2946 { 2947 struct blk_mq_tag_set *set = q->tag_set; 2948 2949 mutex_lock(&set->tag_list_lock); 2950 list_del(&q->tag_set_list); 2951 if (list_is_singular(&set->tag_list)) { 2952 /* just transitioned to unshared */ 2953 set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED; 2954 /* update existing queue */ 2955 blk_mq_update_tag_set_shared(set, false); 2956 } 2957 mutex_unlock(&set->tag_list_lock); 2958 INIT_LIST_HEAD(&q->tag_set_list); 2959 } 2960 2961 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, 2962 struct request_queue *q) 2963 { 2964 mutex_lock(&set->tag_list_lock); 2965 2966 /* 2967 * Check to see if we're transitioning to shared (from 1 to 2 queues). 2968 */ 2969 if (!list_empty(&set->tag_list) && 2970 !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) { 2971 set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED; 2972 /* update existing queue */ 2973 blk_mq_update_tag_set_shared(set, true); 2974 } 2975 if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED) 2976 queue_set_hctx_shared(q, true); 2977 list_add_tail(&q->tag_set_list, &set->tag_list); 2978 2979 mutex_unlock(&set->tag_list_lock); 2980 } 2981 2982 /* All allocations will be freed in release handler of q->mq_kobj */ 2983 static int blk_mq_alloc_ctxs(struct request_queue *q) 2984 { 2985 struct blk_mq_ctxs *ctxs; 2986 int cpu; 2987 2988 ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL); 2989 if (!ctxs) 2990 return -ENOMEM; 2991 2992 ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx); 2993 if (!ctxs->queue_ctx) 2994 goto fail; 2995 2996 for_each_possible_cpu(cpu) { 2997 struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu); 2998 ctx->ctxs = ctxs; 2999 } 3000 3001 q->mq_kobj = &ctxs->kobj; 3002 q->queue_ctx = ctxs->queue_ctx; 3003 3004 return 0; 3005 fail: 3006 kfree(ctxs); 3007 return -ENOMEM; 3008 } 3009 3010 /* 3011 * It is the actual release handler for mq, but we do it from 3012 * request queue's release handler for avoiding use-after-free 3013 * and headache because q->mq_kobj shouldn't have been introduced, 3014 * but we can't group ctx/kctx kobj without it. 3015 */ 3016 void blk_mq_release(struct request_queue *q) 3017 { 3018 struct blk_mq_hw_ctx *hctx, *next; 3019 int i; 3020 3021 queue_for_each_hw_ctx(q, hctx, i) 3022 WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list)); 3023 3024 /* all hctx are in .unused_hctx_list now */ 3025 list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) { 3026 list_del_init(&hctx->hctx_list); 3027 kobject_put(&hctx->kobj); 3028 } 3029 3030 kfree(q->queue_hw_ctx); 3031 3032 /* 3033 * release .mq_kobj and sw queue's kobject now because 3034 * both share lifetime with request queue. 3035 */ 3036 blk_mq_sysfs_deinit(q); 3037 } 3038 3039 struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set, 3040 void *queuedata) 3041 { 3042 struct request_queue *uninit_q, *q; 3043 3044 uninit_q = blk_alloc_queue(set->numa_node); 3045 if (!uninit_q) 3046 return ERR_PTR(-ENOMEM); 3047 uninit_q->queuedata = queuedata; 3048 3049 /* 3050 * Initialize the queue without an elevator. device_add_disk() will do 3051 * the initialization. 3052 */ 3053 q = blk_mq_init_allocated_queue(set, uninit_q, false); 3054 if (IS_ERR(q)) 3055 blk_cleanup_queue(uninit_q); 3056 3057 return q; 3058 } 3059 EXPORT_SYMBOL_GPL(blk_mq_init_queue_data); 3060 3061 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) 3062 { 3063 return blk_mq_init_queue_data(set, NULL); 3064 } 3065 EXPORT_SYMBOL(blk_mq_init_queue); 3066 3067 /* 3068 * Helper for setting up a queue with mq ops, given queue depth, and 3069 * the passed in mq ops flags. 3070 */ 3071 struct request_queue *blk_mq_init_sq_queue(struct blk_mq_tag_set *set, 3072 const struct blk_mq_ops *ops, 3073 unsigned int queue_depth, 3074 unsigned int set_flags) 3075 { 3076 struct request_queue *q; 3077 int ret; 3078 3079 memset(set, 0, sizeof(*set)); 3080 set->ops = ops; 3081 set->nr_hw_queues = 1; 3082 set->nr_maps = 1; 3083 set->queue_depth = queue_depth; 3084 set->numa_node = NUMA_NO_NODE; 3085 set->flags = set_flags; 3086 3087 ret = blk_mq_alloc_tag_set(set); 3088 if (ret) 3089 return ERR_PTR(ret); 3090 3091 q = blk_mq_init_queue(set); 3092 if (IS_ERR(q)) { 3093 blk_mq_free_tag_set(set); 3094 return q; 3095 } 3096 3097 return q; 3098 } 3099 EXPORT_SYMBOL(blk_mq_init_sq_queue); 3100 3101 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx( 3102 struct blk_mq_tag_set *set, struct request_queue *q, 3103 int hctx_idx, int node) 3104 { 3105 struct blk_mq_hw_ctx *hctx = NULL, *tmp; 3106 3107 /* reuse dead hctx first */ 3108 spin_lock(&q->unused_hctx_lock); 3109 list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) { 3110 if (tmp->numa_node == node) { 3111 hctx = tmp; 3112 break; 3113 } 3114 } 3115 if (hctx) 3116 list_del_init(&hctx->hctx_list); 3117 spin_unlock(&q->unused_hctx_lock); 3118 3119 if (!hctx) 3120 hctx = blk_mq_alloc_hctx(q, set, node); 3121 if (!hctx) 3122 goto fail; 3123 3124 if (blk_mq_init_hctx(q, set, hctx, hctx_idx)) 3125 goto free_hctx; 3126 3127 return hctx; 3128 3129 free_hctx: 3130 kobject_put(&hctx->kobj); 3131 fail: 3132 return NULL; 3133 } 3134 3135 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, 3136 struct request_queue *q) 3137 { 3138 int i, j, end; 3139 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; 3140 3141 if (q->nr_hw_queues < set->nr_hw_queues) { 3142 struct blk_mq_hw_ctx **new_hctxs; 3143 3144 new_hctxs = kcalloc_node(set->nr_hw_queues, 3145 sizeof(*new_hctxs), GFP_KERNEL, 3146 set->numa_node); 3147 if (!new_hctxs) 3148 return; 3149 if (hctxs) 3150 memcpy(new_hctxs, hctxs, q->nr_hw_queues * 3151 sizeof(*hctxs)); 3152 q->queue_hw_ctx = new_hctxs; 3153 kfree(hctxs); 3154 hctxs = new_hctxs; 3155 } 3156 3157 /* protect against switching io scheduler */ 3158 mutex_lock(&q->sysfs_lock); 3159 for (i = 0; i < set->nr_hw_queues; i++) { 3160 int node; 3161 struct blk_mq_hw_ctx *hctx; 3162 3163 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], i); 3164 /* 3165 * If the hw queue has been mapped to another numa node, 3166 * we need to realloc the hctx. If allocation fails, fallback 3167 * to use the previous one. 3168 */ 3169 if (hctxs[i] && (hctxs[i]->numa_node == node)) 3170 continue; 3171 3172 hctx = blk_mq_alloc_and_init_hctx(set, q, i, node); 3173 if (hctx) { 3174 if (hctxs[i]) 3175 blk_mq_exit_hctx(q, set, hctxs[i], i); 3176 hctxs[i] = hctx; 3177 } else { 3178 if (hctxs[i]) 3179 pr_warn("Allocate new hctx on node %d fails,\ 3180 fallback to previous one on node %d\n", 3181 node, hctxs[i]->numa_node); 3182 else 3183 break; 3184 } 3185 } 3186 /* 3187 * Increasing nr_hw_queues fails. Free the newly allocated 3188 * hctxs and keep the previous q->nr_hw_queues. 3189 */ 3190 if (i != set->nr_hw_queues) { 3191 j = q->nr_hw_queues; 3192 end = i; 3193 } else { 3194 j = i; 3195 end = q->nr_hw_queues; 3196 q->nr_hw_queues = set->nr_hw_queues; 3197 } 3198 3199 for (; j < end; j++) { 3200 struct blk_mq_hw_ctx *hctx = hctxs[j]; 3201 3202 if (hctx) { 3203 if (hctx->tags) 3204 blk_mq_free_map_and_requests(set, j); 3205 blk_mq_exit_hctx(q, set, hctx, j); 3206 hctxs[j] = NULL; 3207 } 3208 } 3209 mutex_unlock(&q->sysfs_lock); 3210 } 3211 3212 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 3213 struct request_queue *q, 3214 bool elevator_init) 3215 { 3216 /* mark the queue as mq asap */ 3217 q->mq_ops = set->ops; 3218 3219 q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn, 3220 blk_mq_poll_stats_bkt, 3221 BLK_MQ_POLL_STATS_BKTS, q); 3222 if (!q->poll_cb) 3223 goto err_exit; 3224 3225 if (blk_mq_alloc_ctxs(q)) 3226 goto err_poll; 3227 3228 /* init q->mq_kobj and sw queues' kobjects */ 3229 blk_mq_sysfs_init(q); 3230 3231 INIT_LIST_HEAD(&q->unused_hctx_list); 3232 spin_lock_init(&q->unused_hctx_lock); 3233 3234 blk_mq_realloc_hw_ctxs(set, q); 3235 if (!q->nr_hw_queues) 3236 goto err_hctxs; 3237 3238 INIT_WORK(&q->timeout_work, blk_mq_timeout_work); 3239 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ); 3240 3241 q->tag_set = set; 3242 3243 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; 3244 if (set->nr_maps > HCTX_TYPE_POLL && 3245 set->map[HCTX_TYPE_POLL].nr_queues) 3246 blk_queue_flag_set(QUEUE_FLAG_POLL, q); 3247 3248 q->sg_reserved_size = INT_MAX; 3249 3250 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); 3251 INIT_LIST_HEAD(&q->requeue_list); 3252 spin_lock_init(&q->requeue_lock); 3253 3254 q->nr_requests = set->queue_depth; 3255 3256 /* 3257 * Default to classic polling 3258 */ 3259 q->poll_nsec = BLK_MQ_POLL_CLASSIC; 3260 3261 blk_mq_init_cpu_queues(q, set->nr_hw_queues); 3262 blk_mq_add_queue_tag_set(set, q); 3263 blk_mq_map_swqueue(q); 3264 3265 if (elevator_init) 3266 elevator_init_mq(q); 3267 3268 return q; 3269 3270 err_hctxs: 3271 kfree(q->queue_hw_ctx); 3272 q->nr_hw_queues = 0; 3273 blk_mq_sysfs_deinit(q); 3274 err_poll: 3275 blk_stat_free_callback(q->poll_cb); 3276 q->poll_cb = NULL; 3277 err_exit: 3278 q->mq_ops = NULL; 3279 return ERR_PTR(-ENOMEM); 3280 } 3281 EXPORT_SYMBOL(blk_mq_init_allocated_queue); 3282 3283 /* tags can _not_ be used after returning from blk_mq_exit_queue */ 3284 void blk_mq_exit_queue(struct request_queue *q) 3285 { 3286 struct blk_mq_tag_set *set = q->tag_set; 3287 3288 blk_mq_del_queue_tag_set(q); 3289 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); 3290 } 3291 3292 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 3293 { 3294 int i; 3295 3296 for (i = 0; i < set->nr_hw_queues; i++) { 3297 if (!__blk_mq_alloc_map_and_request(set, i)) 3298 goto out_unwind; 3299 cond_resched(); 3300 } 3301 3302 return 0; 3303 3304 out_unwind: 3305 while (--i >= 0) 3306 blk_mq_free_map_and_requests(set, i); 3307 3308 return -ENOMEM; 3309 } 3310 3311 /* 3312 * Allocate the request maps associated with this tag_set. Note that this 3313 * may reduce the depth asked for, if memory is tight. set->queue_depth 3314 * will be updated to reflect the allocated depth. 3315 */ 3316 static int blk_mq_alloc_map_and_requests(struct blk_mq_tag_set *set) 3317 { 3318 unsigned int depth; 3319 int err; 3320 3321 depth = set->queue_depth; 3322 do { 3323 err = __blk_mq_alloc_rq_maps(set); 3324 if (!err) 3325 break; 3326 3327 set->queue_depth >>= 1; 3328 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) { 3329 err = -ENOMEM; 3330 break; 3331 } 3332 } while (set->queue_depth); 3333 3334 if (!set->queue_depth || err) { 3335 pr_err("blk-mq: failed to allocate request map\n"); 3336 return -ENOMEM; 3337 } 3338 3339 if (depth != set->queue_depth) 3340 pr_info("blk-mq: reduced tag depth (%u -> %u)\n", 3341 depth, set->queue_depth); 3342 3343 return 0; 3344 } 3345 3346 static int blk_mq_update_queue_map(struct blk_mq_tag_set *set) 3347 { 3348 /* 3349 * blk_mq_map_queues() and multiple .map_queues() implementations 3350 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the 3351 * number of hardware queues. 3352 */ 3353 if (set->nr_maps == 1) 3354 set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues; 3355 3356 if (set->ops->map_queues && !is_kdump_kernel()) { 3357 int i; 3358 3359 /* 3360 * transport .map_queues is usually done in the following 3361 * way: 3362 * 3363 * for (queue = 0; queue < set->nr_hw_queues; queue++) { 3364 * mask = get_cpu_mask(queue) 3365 * for_each_cpu(cpu, mask) 3366 * set->map[x].mq_map[cpu] = queue; 3367 * } 3368 * 3369 * When we need to remap, the table has to be cleared for 3370 * killing stale mapping since one CPU may not be mapped 3371 * to any hw queue. 3372 */ 3373 for (i = 0; i < set->nr_maps; i++) 3374 blk_mq_clear_mq_map(&set->map[i]); 3375 3376 return set->ops->map_queues(set); 3377 } else { 3378 BUG_ON(set->nr_maps > 1); 3379 return blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 3380 } 3381 } 3382 3383 static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set, 3384 int cur_nr_hw_queues, int new_nr_hw_queues) 3385 { 3386 struct blk_mq_tags **new_tags; 3387 3388 if (cur_nr_hw_queues >= new_nr_hw_queues) 3389 return 0; 3390 3391 new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *), 3392 GFP_KERNEL, set->numa_node); 3393 if (!new_tags) 3394 return -ENOMEM; 3395 3396 if (set->tags) 3397 memcpy(new_tags, set->tags, cur_nr_hw_queues * 3398 sizeof(*set->tags)); 3399 kfree(set->tags); 3400 set->tags = new_tags; 3401 set->nr_hw_queues = new_nr_hw_queues; 3402 3403 return 0; 3404 } 3405 3406 static int blk_mq_alloc_tag_set_tags(struct blk_mq_tag_set *set, 3407 int new_nr_hw_queues) 3408 { 3409 return blk_mq_realloc_tag_set_tags(set, 0, new_nr_hw_queues); 3410 } 3411 3412 /* 3413 * Alloc a tag set to be associated with one or more request queues. 3414 * May fail with EINVAL for various error conditions. May adjust the 3415 * requested depth down, if it's too large. In that case, the set 3416 * value will be stored in set->queue_depth. 3417 */ 3418 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) 3419 { 3420 int i, ret; 3421 3422 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS); 3423 3424 if (!set->nr_hw_queues) 3425 return -EINVAL; 3426 if (!set->queue_depth) 3427 return -EINVAL; 3428 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) 3429 return -EINVAL; 3430 3431 if (!set->ops->queue_rq) 3432 return -EINVAL; 3433 3434 if (!set->ops->get_budget ^ !set->ops->put_budget) 3435 return -EINVAL; 3436 3437 if (set->queue_depth > BLK_MQ_MAX_DEPTH) { 3438 pr_info("blk-mq: reduced tag depth to %u\n", 3439 BLK_MQ_MAX_DEPTH); 3440 set->queue_depth = BLK_MQ_MAX_DEPTH; 3441 } 3442 3443 if (!set->nr_maps) 3444 set->nr_maps = 1; 3445 else if (set->nr_maps > HCTX_MAX_TYPES) 3446 return -EINVAL; 3447 3448 /* 3449 * If a crashdump is active, then we are potentially in a very 3450 * memory constrained environment. Limit us to 1 queue and 3451 * 64 tags to prevent using too much memory. 3452 */ 3453 if (is_kdump_kernel()) { 3454 set->nr_hw_queues = 1; 3455 set->nr_maps = 1; 3456 set->queue_depth = min(64U, set->queue_depth); 3457 } 3458 /* 3459 * There is no use for more h/w queues than cpus if we just have 3460 * a single map 3461 */ 3462 if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids) 3463 set->nr_hw_queues = nr_cpu_ids; 3464 3465 if (blk_mq_alloc_tag_set_tags(set, set->nr_hw_queues) < 0) 3466 return -ENOMEM; 3467 3468 ret = -ENOMEM; 3469 for (i = 0; i < set->nr_maps; i++) { 3470 set->map[i].mq_map = kcalloc_node(nr_cpu_ids, 3471 sizeof(set->map[i].mq_map[0]), 3472 GFP_KERNEL, set->numa_node); 3473 if (!set->map[i].mq_map) 3474 goto out_free_mq_map; 3475 set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues; 3476 } 3477 3478 ret = blk_mq_update_queue_map(set); 3479 if (ret) 3480 goto out_free_mq_map; 3481 3482 ret = blk_mq_alloc_map_and_requests(set); 3483 if (ret) 3484 goto out_free_mq_map; 3485 3486 if (blk_mq_is_sbitmap_shared(set->flags)) { 3487 atomic_set(&set->active_queues_shared_sbitmap, 0); 3488 3489 if (blk_mq_init_shared_sbitmap(set, set->flags)) { 3490 ret = -ENOMEM; 3491 goto out_free_mq_rq_maps; 3492 } 3493 } 3494 3495 mutex_init(&set->tag_list_lock); 3496 INIT_LIST_HEAD(&set->tag_list); 3497 3498 return 0; 3499 3500 out_free_mq_rq_maps: 3501 for (i = 0; i < set->nr_hw_queues; i++) 3502 blk_mq_free_map_and_requests(set, i); 3503 out_free_mq_map: 3504 for (i = 0; i < set->nr_maps; i++) { 3505 kfree(set->map[i].mq_map); 3506 set->map[i].mq_map = NULL; 3507 } 3508 kfree(set->tags); 3509 set->tags = NULL; 3510 return ret; 3511 } 3512 EXPORT_SYMBOL(blk_mq_alloc_tag_set); 3513 3514 void blk_mq_free_tag_set(struct blk_mq_tag_set *set) 3515 { 3516 int i, j; 3517 3518 for (i = 0; i < set->nr_hw_queues; i++) 3519 blk_mq_free_map_and_requests(set, i); 3520 3521 if (blk_mq_is_sbitmap_shared(set->flags)) 3522 blk_mq_exit_shared_sbitmap(set); 3523 3524 for (j = 0; j < set->nr_maps; j++) { 3525 kfree(set->map[j].mq_map); 3526 set->map[j].mq_map = NULL; 3527 } 3528 3529 kfree(set->tags); 3530 set->tags = NULL; 3531 } 3532 EXPORT_SYMBOL(blk_mq_free_tag_set); 3533 3534 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) 3535 { 3536 struct blk_mq_tag_set *set = q->tag_set; 3537 struct blk_mq_hw_ctx *hctx; 3538 int i, ret; 3539 3540 if (!set) 3541 return -EINVAL; 3542 3543 if (q->nr_requests == nr) 3544 return 0; 3545 3546 blk_mq_freeze_queue(q); 3547 blk_mq_quiesce_queue(q); 3548 3549 ret = 0; 3550 queue_for_each_hw_ctx(q, hctx, i) { 3551 if (!hctx->tags) 3552 continue; 3553 /* 3554 * If we're using an MQ scheduler, just update the scheduler 3555 * queue depth. This is similar to what the old code would do. 3556 */ 3557 if (!hctx->sched_tags) { 3558 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr, 3559 false); 3560 if (!ret && blk_mq_is_sbitmap_shared(set->flags)) 3561 blk_mq_tag_resize_shared_sbitmap(set, nr); 3562 } else { 3563 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags, 3564 nr, true); 3565 } 3566 if (ret) 3567 break; 3568 if (q->elevator && q->elevator->type->ops.depth_updated) 3569 q->elevator->type->ops.depth_updated(hctx); 3570 } 3571 3572 if (!ret) 3573 q->nr_requests = nr; 3574 3575 blk_mq_unquiesce_queue(q); 3576 blk_mq_unfreeze_queue(q); 3577 3578 return ret; 3579 } 3580 3581 /* 3582 * request_queue and elevator_type pair. 3583 * It is just used by __blk_mq_update_nr_hw_queues to cache 3584 * the elevator_type associated with a request_queue. 3585 */ 3586 struct blk_mq_qe_pair { 3587 struct list_head node; 3588 struct request_queue *q; 3589 struct elevator_type *type; 3590 }; 3591 3592 /* 3593 * Cache the elevator_type in qe pair list and switch the 3594 * io scheduler to 'none' 3595 */ 3596 static bool blk_mq_elv_switch_none(struct list_head *head, 3597 struct request_queue *q) 3598 { 3599 struct blk_mq_qe_pair *qe; 3600 3601 if (!q->elevator) 3602 return true; 3603 3604 qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY); 3605 if (!qe) 3606 return false; 3607 3608 INIT_LIST_HEAD(&qe->node); 3609 qe->q = q; 3610 qe->type = q->elevator->type; 3611 list_add(&qe->node, head); 3612 3613 mutex_lock(&q->sysfs_lock); 3614 /* 3615 * After elevator_switch_mq, the previous elevator_queue will be 3616 * released by elevator_release. The reference of the io scheduler 3617 * module get by elevator_get will also be put. So we need to get 3618 * a reference of the io scheduler module here to prevent it to be 3619 * removed. 3620 */ 3621 __module_get(qe->type->elevator_owner); 3622 elevator_switch_mq(q, NULL); 3623 mutex_unlock(&q->sysfs_lock); 3624 3625 return true; 3626 } 3627 3628 static void blk_mq_elv_switch_back(struct list_head *head, 3629 struct request_queue *q) 3630 { 3631 struct blk_mq_qe_pair *qe; 3632 struct elevator_type *t = NULL; 3633 3634 list_for_each_entry(qe, head, node) 3635 if (qe->q == q) { 3636 t = qe->type; 3637 break; 3638 } 3639 3640 if (!t) 3641 return; 3642 3643 list_del(&qe->node); 3644 kfree(qe); 3645 3646 mutex_lock(&q->sysfs_lock); 3647 elevator_switch_mq(q, t); 3648 mutex_unlock(&q->sysfs_lock); 3649 } 3650 3651 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, 3652 int nr_hw_queues) 3653 { 3654 struct request_queue *q; 3655 LIST_HEAD(head); 3656 int prev_nr_hw_queues; 3657 3658 lockdep_assert_held(&set->tag_list_lock); 3659 3660 if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids) 3661 nr_hw_queues = nr_cpu_ids; 3662 if (nr_hw_queues < 1) 3663 return; 3664 if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues) 3665 return; 3666 3667 list_for_each_entry(q, &set->tag_list, tag_set_list) 3668 blk_mq_freeze_queue(q); 3669 /* 3670 * Switch IO scheduler to 'none', cleaning up the data associated 3671 * with the previous scheduler. We will switch back once we are done 3672 * updating the new sw to hw queue mappings. 3673 */ 3674 list_for_each_entry(q, &set->tag_list, tag_set_list) 3675 if (!blk_mq_elv_switch_none(&head, q)) 3676 goto switch_back; 3677 3678 list_for_each_entry(q, &set->tag_list, tag_set_list) { 3679 blk_mq_debugfs_unregister_hctxs(q); 3680 blk_mq_sysfs_unregister(q); 3681 } 3682 3683 prev_nr_hw_queues = set->nr_hw_queues; 3684 if (blk_mq_realloc_tag_set_tags(set, set->nr_hw_queues, nr_hw_queues) < 3685 0) 3686 goto reregister; 3687 3688 set->nr_hw_queues = nr_hw_queues; 3689 fallback: 3690 blk_mq_update_queue_map(set); 3691 list_for_each_entry(q, &set->tag_list, tag_set_list) { 3692 blk_mq_realloc_hw_ctxs(set, q); 3693 if (q->nr_hw_queues != set->nr_hw_queues) { 3694 pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n", 3695 nr_hw_queues, prev_nr_hw_queues); 3696 set->nr_hw_queues = prev_nr_hw_queues; 3697 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 3698 goto fallback; 3699 } 3700 blk_mq_map_swqueue(q); 3701 } 3702 3703 reregister: 3704 list_for_each_entry(q, &set->tag_list, tag_set_list) { 3705 blk_mq_sysfs_register(q); 3706 blk_mq_debugfs_register_hctxs(q); 3707 } 3708 3709 switch_back: 3710 list_for_each_entry(q, &set->tag_list, tag_set_list) 3711 blk_mq_elv_switch_back(&head, q); 3712 3713 list_for_each_entry(q, &set->tag_list, tag_set_list) 3714 blk_mq_unfreeze_queue(q); 3715 } 3716 3717 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) 3718 { 3719 mutex_lock(&set->tag_list_lock); 3720 __blk_mq_update_nr_hw_queues(set, nr_hw_queues); 3721 mutex_unlock(&set->tag_list_lock); 3722 } 3723 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); 3724 3725 /* Enable polling stats and return whether they were already enabled. */ 3726 static bool blk_poll_stats_enable(struct request_queue *q) 3727 { 3728 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 3729 blk_queue_flag_test_and_set(QUEUE_FLAG_POLL_STATS, q)) 3730 return true; 3731 blk_stat_add_callback(q, q->poll_cb); 3732 return false; 3733 } 3734 3735 static void blk_mq_poll_stats_start(struct request_queue *q) 3736 { 3737 /* 3738 * We don't arm the callback if polling stats are not enabled or the 3739 * callback is already active. 3740 */ 3741 if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 3742 blk_stat_is_active(q->poll_cb)) 3743 return; 3744 3745 blk_stat_activate_msecs(q->poll_cb, 100); 3746 } 3747 3748 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb) 3749 { 3750 struct request_queue *q = cb->data; 3751 int bucket; 3752 3753 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) { 3754 if (cb->stat[bucket].nr_samples) 3755 q->poll_stat[bucket] = cb->stat[bucket]; 3756 } 3757 } 3758 3759 static unsigned long blk_mq_poll_nsecs(struct request_queue *q, 3760 struct request *rq) 3761 { 3762 unsigned long ret = 0; 3763 int bucket; 3764 3765 /* 3766 * If stats collection isn't on, don't sleep but turn it on for 3767 * future users 3768 */ 3769 if (!blk_poll_stats_enable(q)) 3770 return 0; 3771 3772 /* 3773 * As an optimistic guess, use half of the mean service time 3774 * for this type of request. We can (and should) make this smarter. 3775 * For instance, if the completion latencies are tight, we can 3776 * get closer than just half the mean. This is especially 3777 * important on devices where the completion latencies are longer 3778 * than ~10 usec. We do use the stats for the relevant IO size 3779 * if available which does lead to better estimates. 3780 */ 3781 bucket = blk_mq_poll_stats_bkt(rq); 3782 if (bucket < 0) 3783 return ret; 3784 3785 if (q->poll_stat[bucket].nr_samples) 3786 ret = (q->poll_stat[bucket].mean + 1) / 2; 3787 3788 return ret; 3789 } 3790 3791 static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, 3792 struct request *rq) 3793 { 3794 struct hrtimer_sleeper hs; 3795 enum hrtimer_mode mode; 3796 unsigned int nsecs; 3797 ktime_t kt; 3798 3799 if (rq->rq_flags & RQF_MQ_POLL_SLEPT) 3800 return false; 3801 3802 /* 3803 * If we get here, hybrid polling is enabled. Hence poll_nsec can be: 3804 * 3805 * 0: use half of prev avg 3806 * >0: use this specific value 3807 */ 3808 if (q->poll_nsec > 0) 3809 nsecs = q->poll_nsec; 3810 else 3811 nsecs = blk_mq_poll_nsecs(q, rq); 3812 3813 if (!nsecs) 3814 return false; 3815 3816 rq->rq_flags |= RQF_MQ_POLL_SLEPT; 3817 3818 /* 3819 * This will be replaced with the stats tracking code, using 3820 * 'avg_completion_time / 2' as the pre-sleep target. 3821 */ 3822 kt = nsecs; 3823 3824 mode = HRTIMER_MODE_REL; 3825 hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode); 3826 hrtimer_set_expires(&hs.timer, kt); 3827 3828 do { 3829 if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE) 3830 break; 3831 set_current_state(TASK_UNINTERRUPTIBLE); 3832 hrtimer_sleeper_start_expires(&hs, mode); 3833 if (hs.task) 3834 io_schedule(); 3835 hrtimer_cancel(&hs.timer); 3836 mode = HRTIMER_MODE_ABS; 3837 } while (hs.task && !signal_pending(current)); 3838 3839 __set_current_state(TASK_RUNNING); 3840 destroy_hrtimer_on_stack(&hs.timer); 3841 return true; 3842 } 3843 3844 static bool blk_mq_poll_hybrid(struct request_queue *q, 3845 struct blk_mq_hw_ctx *hctx, blk_qc_t cookie) 3846 { 3847 struct request *rq; 3848 3849 if (q->poll_nsec == BLK_MQ_POLL_CLASSIC) 3850 return false; 3851 3852 if (!blk_qc_t_is_internal(cookie)) 3853 rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); 3854 else { 3855 rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie)); 3856 /* 3857 * With scheduling, if the request has completed, we'll 3858 * get a NULL return here, as we clear the sched tag when 3859 * that happens. The request still remains valid, like always, 3860 * so we should be safe with just the NULL check. 3861 */ 3862 if (!rq) 3863 return false; 3864 } 3865 3866 return blk_mq_poll_hybrid_sleep(q, rq); 3867 } 3868 3869 /** 3870 * blk_poll - poll for IO completions 3871 * @q: the queue 3872 * @cookie: cookie passed back at IO submission time 3873 * @spin: whether to spin for completions 3874 * 3875 * Description: 3876 * Poll for completions on the passed in queue. Returns number of 3877 * completed entries found. If @spin is true, then blk_poll will continue 3878 * looping until at least one completion is found, unless the task is 3879 * otherwise marked running (or we need to reschedule). 3880 */ 3881 int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin) 3882 { 3883 struct blk_mq_hw_ctx *hctx; 3884 long state; 3885 3886 if (!blk_qc_t_valid(cookie) || 3887 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 3888 return 0; 3889 3890 if (current->plug) 3891 blk_flush_plug_list(current->plug, false); 3892 3893 hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; 3894 3895 /* 3896 * If we sleep, have the caller restart the poll loop to reset 3897 * the state. Like for the other success return cases, the 3898 * caller is responsible for checking if the IO completed. If 3899 * the IO isn't complete, we'll get called again and will go 3900 * straight to the busy poll loop. If specified not to spin, 3901 * we also should not sleep. 3902 */ 3903 if (spin && blk_mq_poll_hybrid(q, hctx, cookie)) 3904 return 1; 3905 3906 hctx->poll_considered++; 3907 3908 state = current->state; 3909 do { 3910 int ret; 3911 3912 hctx->poll_invoked++; 3913 3914 ret = q->mq_ops->poll(hctx); 3915 if (ret > 0) { 3916 hctx->poll_success++; 3917 __set_current_state(TASK_RUNNING); 3918 return ret; 3919 } 3920 3921 if (signal_pending_state(state, current)) 3922 __set_current_state(TASK_RUNNING); 3923 3924 if (current->state == TASK_RUNNING) 3925 return 1; 3926 if (ret < 0 || !spin) 3927 break; 3928 cpu_relax(); 3929 } while (!need_resched()); 3930 3931 __set_current_state(TASK_RUNNING); 3932 return 0; 3933 } 3934 EXPORT_SYMBOL_GPL(blk_poll); 3935 3936 unsigned int blk_mq_rq_cpu(struct request *rq) 3937 { 3938 return rq->mq_ctx->cpu; 3939 } 3940 EXPORT_SYMBOL(blk_mq_rq_cpu); 3941 3942 static int __init blk_mq_init(void) 3943 { 3944 int i; 3945 3946 for_each_possible_cpu(i) 3947 init_llist_head(&per_cpu(blk_cpu_done, i)); 3948 open_softirq(BLOCK_SOFTIRQ, blk_done_softirq); 3949 3950 cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD, 3951 "block/softirq:dead", NULL, 3952 blk_softirq_cpu_dead); 3953 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, 3954 blk_mq_hctx_notify_dead); 3955 cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online", 3956 blk_mq_hctx_notify_online, 3957 blk_mq_hctx_notify_offline); 3958 return 0; 3959 } 3960 subsys_initcall(blk_mq_init); 3961