1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Block multiqueue core code 4 * 5 * Copyright (C) 2013-2014 Jens Axboe 6 * Copyright (C) 2013-2014 Christoph Hellwig 7 */ 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/backing-dev.h> 11 #include <linux/bio.h> 12 #include <linux/blkdev.h> 13 #include <linux/kmemleak.h> 14 #include <linux/mm.h> 15 #include <linux/init.h> 16 #include <linux/slab.h> 17 #include <linux/workqueue.h> 18 #include <linux/smp.h> 19 #include <linux/llist.h> 20 #include <linux/list_sort.h> 21 #include <linux/cpu.h> 22 #include <linux/cache.h> 23 #include <linux/sched/sysctl.h> 24 #include <linux/sched/topology.h> 25 #include <linux/sched/signal.h> 26 #include <linux/delay.h> 27 #include <linux/crash_dump.h> 28 #include <linux/prefetch.h> 29 #include <linux/blk-crypto.h> 30 31 #include <trace/events/block.h> 32 33 #include <linux/blk-mq.h> 34 #include <linux/t10-pi.h> 35 #include "blk.h" 36 #include "blk-mq.h" 37 #include "blk-mq-debugfs.h" 38 #include "blk-mq-tag.h" 39 #include "blk-pm.h" 40 #include "blk-stat.h" 41 #include "blk-mq-sched.h" 42 #include "blk-rq-qos.h" 43 44 static DEFINE_PER_CPU(struct list_head, blk_cpu_done); 45 46 static void blk_mq_poll_stats_start(struct request_queue *q); 47 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); 48 49 static int blk_mq_poll_stats_bkt(const struct request *rq) 50 { 51 int ddir, sectors, bucket; 52 53 ddir = rq_data_dir(rq); 54 sectors = blk_rq_stats_sectors(rq); 55 56 bucket = ddir + 2 * ilog2(sectors); 57 58 if (bucket < 0) 59 return -1; 60 else if (bucket >= BLK_MQ_POLL_STATS_BKTS) 61 return ddir + BLK_MQ_POLL_STATS_BKTS - 2; 62 63 return bucket; 64 } 65 66 /* 67 * Check if any of the ctx, dispatch list or elevator 68 * have pending work in this hardware queue. 69 */ 70 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) 71 { 72 return !list_empty_careful(&hctx->dispatch) || 73 sbitmap_any_bit_set(&hctx->ctx_map) || 74 blk_mq_sched_has_work(hctx); 75 } 76 77 /* 78 * Mark this ctx as having pending work in this hardware queue 79 */ 80 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, 81 struct blk_mq_ctx *ctx) 82 { 83 const int bit = ctx->index_hw[hctx->type]; 84 85 if (!sbitmap_test_bit(&hctx->ctx_map, bit)) 86 sbitmap_set_bit(&hctx->ctx_map, bit); 87 } 88 89 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, 90 struct blk_mq_ctx *ctx) 91 { 92 const int bit = ctx->index_hw[hctx->type]; 93 94 sbitmap_clear_bit(&hctx->ctx_map, bit); 95 } 96 97 struct mq_inflight { 98 struct hd_struct *part; 99 unsigned int inflight[2]; 100 }; 101 102 static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx, 103 struct request *rq, void *priv, 104 bool reserved) 105 { 106 struct mq_inflight *mi = priv; 107 108 if (rq->part == mi->part) 109 mi->inflight[rq_data_dir(rq)]++; 110 111 return true; 112 } 113 114 unsigned int blk_mq_in_flight(struct request_queue *q, struct hd_struct *part) 115 { 116 struct mq_inflight mi = { .part = part }; 117 118 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); 119 120 return mi.inflight[0] + mi.inflight[1]; 121 } 122 123 void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part, 124 unsigned int inflight[2]) 125 { 126 struct mq_inflight mi = { .part = part }; 127 128 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); 129 inflight[0] = mi.inflight[0]; 130 inflight[1] = mi.inflight[1]; 131 } 132 133 void blk_freeze_queue_start(struct request_queue *q) 134 { 135 mutex_lock(&q->mq_freeze_lock); 136 if (++q->mq_freeze_depth == 1) { 137 percpu_ref_kill(&q->q_usage_counter); 138 mutex_unlock(&q->mq_freeze_lock); 139 if (queue_is_mq(q)) 140 blk_mq_run_hw_queues(q, false); 141 } else { 142 mutex_unlock(&q->mq_freeze_lock); 143 } 144 } 145 EXPORT_SYMBOL_GPL(blk_freeze_queue_start); 146 147 void blk_mq_freeze_queue_wait(struct request_queue *q) 148 { 149 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter)); 150 } 151 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait); 152 153 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, 154 unsigned long timeout) 155 { 156 return wait_event_timeout(q->mq_freeze_wq, 157 percpu_ref_is_zero(&q->q_usage_counter), 158 timeout); 159 } 160 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout); 161 162 /* 163 * Guarantee no request is in use, so we can change any data structure of 164 * the queue afterward. 165 */ 166 void blk_freeze_queue(struct request_queue *q) 167 { 168 /* 169 * In the !blk_mq case we are only calling this to kill the 170 * q_usage_counter, otherwise this increases the freeze depth 171 * and waits for it to return to zero. For this reason there is 172 * no blk_unfreeze_queue(), and blk_freeze_queue() is not 173 * exported to drivers as the only user for unfreeze is blk_mq. 174 */ 175 blk_freeze_queue_start(q); 176 blk_mq_freeze_queue_wait(q); 177 } 178 179 void blk_mq_freeze_queue(struct request_queue *q) 180 { 181 /* 182 * ...just an alias to keep freeze and unfreeze actions balanced 183 * in the blk_mq_* namespace 184 */ 185 blk_freeze_queue(q); 186 } 187 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue); 188 189 void blk_mq_unfreeze_queue(struct request_queue *q) 190 { 191 mutex_lock(&q->mq_freeze_lock); 192 q->mq_freeze_depth--; 193 WARN_ON_ONCE(q->mq_freeze_depth < 0); 194 if (!q->mq_freeze_depth) { 195 percpu_ref_resurrect(&q->q_usage_counter); 196 wake_up_all(&q->mq_freeze_wq); 197 } 198 mutex_unlock(&q->mq_freeze_lock); 199 } 200 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); 201 202 /* 203 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the 204 * mpt3sas driver such that this function can be removed. 205 */ 206 void blk_mq_quiesce_queue_nowait(struct request_queue *q) 207 { 208 blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q); 209 } 210 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); 211 212 /** 213 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished 214 * @q: request queue. 215 * 216 * Note: this function does not prevent that the struct request end_io() 217 * callback function is invoked. Once this function is returned, we make 218 * sure no dispatch can happen until the queue is unquiesced via 219 * blk_mq_unquiesce_queue(). 220 */ 221 void blk_mq_quiesce_queue(struct request_queue *q) 222 { 223 struct blk_mq_hw_ctx *hctx; 224 unsigned int i; 225 bool rcu = false; 226 227 blk_mq_quiesce_queue_nowait(q); 228 229 queue_for_each_hw_ctx(q, hctx, i) { 230 if (hctx->flags & BLK_MQ_F_BLOCKING) 231 synchronize_srcu(hctx->srcu); 232 else 233 rcu = true; 234 } 235 if (rcu) 236 synchronize_rcu(); 237 } 238 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue); 239 240 /* 241 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue() 242 * @q: request queue. 243 * 244 * This function recovers queue into the state before quiescing 245 * which is done by blk_mq_quiesce_queue. 246 */ 247 void blk_mq_unquiesce_queue(struct request_queue *q) 248 { 249 blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q); 250 251 /* dispatch requests which are inserted during quiescing */ 252 blk_mq_run_hw_queues(q, true); 253 } 254 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue); 255 256 void blk_mq_wake_waiters(struct request_queue *q) 257 { 258 struct blk_mq_hw_ctx *hctx; 259 unsigned int i; 260 261 queue_for_each_hw_ctx(q, hctx, i) 262 if (blk_mq_hw_queue_mapped(hctx)) 263 blk_mq_tag_wakeup_all(hctx->tags, true); 264 } 265 266 /* 267 * Only need start/end time stamping if we have iostat or 268 * blk stats enabled, or using an IO scheduler. 269 */ 270 static inline bool blk_mq_need_time_stamp(struct request *rq) 271 { 272 return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS)) || rq->q->elevator; 273 } 274 275 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, 276 unsigned int tag, u64 alloc_time_ns) 277 { 278 struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 279 struct request *rq = tags->static_rqs[tag]; 280 281 if (data->q->elevator) { 282 rq->tag = BLK_MQ_NO_TAG; 283 rq->internal_tag = tag; 284 } else { 285 rq->tag = tag; 286 rq->internal_tag = BLK_MQ_NO_TAG; 287 } 288 289 /* csd/requeue_work/fifo_time is initialized before use */ 290 rq->q = data->q; 291 rq->mq_ctx = data->ctx; 292 rq->mq_hctx = data->hctx; 293 rq->rq_flags = 0; 294 rq->cmd_flags = data->cmd_flags; 295 if (data->flags & BLK_MQ_REQ_PREEMPT) 296 rq->rq_flags |= RQF_PREEMPT; 297 if (blk_queue_io_stat(data->q)) 298 rq->rq_flags |= RQF_IO_STAT; 299 INIT_LIST_HEAD(&rq->queuelist); 300 INIT_HLIST_NODE(&rq->hash); 301 RB_CLEAR_NODE(&rq->rb_node); 302 rq->rq_disk = NULL; 303 rq->part = NULL; 304 #ifdef CONFIG_BLK_RQ_ALLOC_TIME 305 rq->alloc_time_ns = alloc_time_ns; 306 #endif 307 if (blk_mq_need_time_stamp(rq)) 308 rq->start_time_ns = ktime_get_ns(); 309 else 310 rq->start_time_ns = 0; 311 rq->io_start_time_ns = 0; 312 rq->stats_sectors = 0; 313 rq->nr_phys_segments = 0; 314 #if defined(CONFIG_BLK_DEV_INTEGRITY) 315 rq->nr_integrity_segments = 0; 316 #endif 317 blk_crypto_rq_set_defaults(rq); 318 /* tag was already set */ 319 WRITE_ONCE(rq->deadline, 0); 320 321 rq->timeout = 0; 322 323 rq->end_io = NULL; 324 rq->end_io_data = NULL; 325 326 data->ctx->rq_dispatched[op_is_sync(data->cmd_flags)]++; 327 refcount_set(&rq->ref, 1); 328 329 if (!op_is_flush(data->cmd_flags)) { 330 struct elevator_queue *e = data->q->elevator; 331 332 rq->elv.icq = NULL; 333 if (e && e->type->ops.prepare_request) { 334 if (e->type->icq_cache) 335 blk_mq_sched_assign_ioc(rq); 336 337 e->type->ops.prepare_request(rq); 338 rq->rq_flags |= RQF_ELVPRIV; 339 } 340 } 341 342 data->hctx->queued++; 343 return rq; 344 } 345 346 static struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data) 347 { 348 struct request_queue *q = data->q; 349 struct elevator_queue *e = q->elevator; 350 u64 alloc_time_ns = 0; 351 unsigned int tag; 352 353 /* alloc_time includes depth and tag waits */ 354 if (blk_queue_rq_alloc_time(q)) 355 alloc_time_ns = ktime_get_ns(); 356 357 if (data->cmd_flags & REQ_NOWAIT) 358 data->flags |= BLK_MQ_REQ_NOWAIT; 359 360 if (e) { 361 /* 362 * Flush requests are special and go directly to the 363 * dispatch list. Don't include reserved tags in the 364 * limiting, as it isn't useful. 365 */ 366 if (!op_is_flush(data->cmd_flags) && 367 e->type->ops.limit_depth && 368 !(data->flags & BLK_MQ_REQ_RESERVED)) 369 e->type->ops.limit_depth(data->cmd_flags, data); 370 } 371 372 retry: 373 data->ctx = blk_mq_get_ctx(q); 374 data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx); 375 if (!e) 376 blk_mq_tag_busy(data->hctx); 377 378 /* 379 * Waiting allocations only fail because of an inactive hctx. In that 380 * case just retry the hctx assignment and tag allocation as CPU hotplug 381 * should have migrated us to an online CPU by now. 382 */ 383 tag = blk_mq_get_tag(data); 384 if (tag == BLK_MQ_NO_TAG) { 385 if (data->flags & BLK_MQ_REQ_NOWAIT) 386 return NULL; 387 388 /* 389 * Give up the CPU and sleep for a random short time to ensure 390 * that thread using a realtime scheduling class are migrated 391 * off the the CPU, and thus off the hctx that is going away. 392 */ 393 msleep(3); 394 goto retry; 395 } 396 return blk_mq_rq_ctx_init(data, tag, alloc_time_ns); 397 } 398 399 struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, 400 blk_mq_req_flags_t flags) 401 { 402 struct blk_mq_alloc_data data = { 403 .q = q, 404 .flags = flags, 405 .cmd_flags = op, 406 }; 407 struct request *rq; 408 int ret; 409 410 ret = blk_queue_enter(q, flags); 411 if (ret) 412 return ERR_PTR(ret); 413 414 rq = __blk_mq_alloc_request(&data); 415 if (!rq) 416 goto out_queue_exit; 417 rq->__data_len = 0; 418 rq->__sector = (sector_t) -1; 419 rq->bio = rq->biotail = NULL; 420 return rq; 421 out_queue_exit: 422 blk_queue_exit(q); 423 return ERR_PTR(-EWOULDBLOCK); 424 } 425 EXPORT_SYMBOL(blk_mq_alloc_request); 426 427 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, 428 unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx) 429 { 430 struct blk_mq_alloc_data data = { 431 .q = q, 432 .flags = flags, 433 .cmd_flags = op, 434 }; 435 u64 alloc_time_ns = 0; 436 unsigned int cpu; 437 unsigned int tag; 438 int ret; 439 440 /* alloc_time includes depth and tag waits */ 441 if (blk_queue_rq_alloc_time(q)) 442 alloc_time_ns = ktime_get_ns(); 443 444 /* 445 * If the tag allocator sleeps we could get an allocation for a 446 * different hardware context. No need to complicate the low level 447 * allocator for this for the rare use case of a command tied to 448 * a specific queue. 449 */ 450 if (WARN_ON_ONCE(!(flags & (BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED)))) 451 return ERR_PTR(-EINVAL); 452 453 if (hctx_idx >= q->nr_hw_queues) 454 return ERR_PTR(-EIO); 455 456 ret = blk_queue_enter(q, flags); 457 if (ret) 458 return ERR_PTR(ret); 459 460 /* 461 * Check if the hardware context is actually mapped to anything. 462 * If not tell the caller that it should skip this queue. 463 */ 464 ret = -EXDEV; 465 data.hctx = q->queue_hw_ctx[hctx_idx]; 466 if (!blk_mq_hw_queue_mapped(data.hctx)) 467 goto out_queue_exit; 468 cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask); 469 data.ctx = __blk_mq_get_ctx(q, cpu); 470 471 if (!q->elevator) 472 blk_mq_tag_busy(data.hctx); 473 474 ret = -EWOULDBLOCK; 475 tag = blk_mq_get_tag(&data); 476 if (tag == BLK_MQ_NO_TAG) 477 goto out_queue_exit; 478 return blk_mq_rq_ctx_init(&data, tag, alloc_time_ns); 479 480 out_queue_exit: 481 blk_queue_exit(q); 482 return ERR_PTR(ret); 483 } 484 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); 485 486 static void __blk_mq_free_request(struct request *rq) 487 { 488 struct request_queue *q = rq->q; 489 struct blk_mq_ctx *ctx = rq->mq_ctx; 490 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 491 const int sched_tag = rq->internal_tag; 492 493 blk_crypto_free_request(rq); 494 blk_pm_mark_last_busy(rq); 495 rq->mq_hctx = NULL; 496 if (rq->tag != BLK_MQ_NO_TAG) 497 blk_mq_put_tag(hctx->tags, ctx, rq->tag); 498 if (sched_tag != BLK_MQ_NO_TAG) 499 blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag); 500 blk_mq_sched_restart(hctx); 501 blk_queue_exit(q); 502 } 503 504 void blk_mq_free_request(struct request *rq) 505 { 506 struct request_queue *q = rq->q; 507 struct elevator_queue *e = q->elevator; 508 struct blk_mq_ctx *ctx = rq->mq_ctx; 509 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 510 511 if (rq->rq_flags & RQF_ELVPRIV) { 512 if (e && e->type->ops.finish_request) 513 e->type->ops.finish_request(rq); 514 if (rq->elv.icq) { 515 put_io_context(rq->elv.icq->ioc); 516 rq->elv.icq = NULL; 517 } 518 } 519 520 ctx->rq_completed[rq_is_sync(rq)]++; 521 if (rq->rq_flags & RQF_MQ_INFLIGHT) 522 atomic_dec(&hctx->nr_active); 523 524 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq))) 525 laptop_io_completion(q->backing_dev_info); 526 527 rq_qos_done(q, rq); 528 529 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 530 if (refcount_dec_and_test(&rq->ref)) 531 __blk_mq_free_request(rq); 532 } 533 EXPORT_SYMBOL_GPL(blk_mq_free_request); 534 535 inline void __blk_mq_end_request(struct request *rq, blk_status_t error) 536 { 537 u64 now = 0; 538 539 if (blk_mq_need_time_stamp(rq)) 540 now = ktime_get_ns(); 541 542 if (rq->rq_flags & RQF_STATS) { 543 blk_mq_poll_stats_start(rq->q); 544 blk_stat_add(rq, now); 545 } 546 547 if (rq->internal_tag != BLK_MQ_NO_TAG) 548 blk_mq_sched_completed_request(rq, now); 549 550 blk_account_io_done(rq, now); 551 552 if (rq->end_io) { 553 rq_qos_done(rq->q, rq); 554 rq->end_io(rq, error); 555 } else { 556 blk_mq_free_request(rq); 557 } 558 } 559 EXPORT_SYMBOL(__blk_mq_end_request); 560 561 void blk_mq_end_request(struct request *rq, blk_status_t error) 562 { 563 if (blk_update_request(rq, error, blk_rq_bytes(rq))) 564 BUG(); 565 __blk_mq_end_request(rq, error); 566 } 567 EXPORT_SYMBOL(blk_mq_end_request); 568 569 /* 570 * Softirq action handler - move entries to local list and loop over them 571 * while passing them to the queue registered handler. 572 */ 573 static __latent_entropy void blk_done_softirq(struct softirq_action *h) 574 { 575 struct list_head *cpu_list, local_list; 576 577 local_irq_disable(); 578 cpu_list = this_cpu_ptr(&blk_cpu_done); 579 list_replace_init(cpu_list, &local_list); 580 local_irq_enable(); 581 582 while (!list_empty(&local_list)) { 583 struct request *rq; 584 585 rq = list_entry(local_list.next, struct request, ipi_list); 586 list_del_init(&rq->ipi_list); 587 rq->q->mq_ops->complete(rq); 588 } 589 } 590 591 static void blk_mq_trigger_softirq(struct request *rq) 592 { 593 struct list_head *list; 594 unsigned long flags; 595 596 local_irq_save(flags); 597 list = this_cpu_ptr(&blk_cpu_done); 598 list_add_tail(&rq->ipi_list, list); 599 600 /* 601 * If the list only contains our just added request, signal a raise of 602 * the softirq. If there are already entries there, someone already 603 * raised the irq but it hasn't run yet. 604 */ 605 if (list->next == &rq->ipi_list) 606 raise_softirq_irqoff(BLOCK_SOFTIRQ); 607 local_irq_restore(flags); 608 } 609 610 static int blk_softirq_cpu_dead(unsigned int cpu) 611 { 612 /* 613 * If a CPU goes away, splice its entries to the current CPU 614 * and trigger a run of the softirq 615 */ 616 local_irq_disable(); 617 list_splice_init(&per_cpu(blk_cpu_done, cpu), 618 this_cpu_ptr(&blk_cpu_done)); 619 raise_softirq_irqoff(BLOCK_SOFTIRQ); 620 local_irq_enable(); 621 622 return 0; 623 } 624 625 626 static void __blk_mq_complete_request_remote(void *data) 627 { 628 struct request *rq = data; 629 630 /* 631 * For most of single queue controllers, there is only one irq vector 632 * for handling I/O completion, and the only irq's affinity is set 633 * to all possible CPUs. On most of ARCHs, this affinity means the irq 634 * is handled on one specific CPU. 635 * 636 * So complete I/O requests in softirq context in case of single queue 637 * devices to avoid degrading I/O performance due to irqsoff latency. 638 */ 639 if (rq->q->nr_hw_queues == 1) 640 blk_mq_trigger_softirq(rq); 641 else 642 rq->q->mq_ops->complete(rq); 643 } 644 645 static inline bool blk_mq_complete_need_ipi(struct request *rq) 646 { 647 int cpu = raw_smp_processor_id(); 648 649 if (!IS_ENABLED(CONFIG_SMP) || 650 !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) 651 return false; 652 653 /* same CPU or cache domain? Complete locally */ 654 if (cpu == rq->mq_ctx->cpu || 655 (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) && 656 cpus_share_cache(cpu, rq->mq_ctx->cpu))) 657 return false; 658 659 /* don't try to IPI to an offline CPU */ 660 return cpu_online(rq->mq_ctx->cpu); 661 } 662 663 static void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx, 664 struct request *rq) 665 { 666 blk_mq_put_tag(hctx->tags, rq->mq_ctx, rq->tag); 667 rq->tag = BLK_MQ_NO_TAG; 668 669 if (rq->rq_flags & RQF_MQ_INFLIGHT) { 670 rq->rq_flags &= ~RQF_MQ_INFLIGHT; 671 atomic_dec(&hctx->nr_active); 672 } 673 } 674 675 static inline void blk_mq_put_driver_tag(struct request *rq) 676 { 677 if (rq->tag == BLK_MQ_NO_TAG || rq->internal_tag == BLK_MQ_NO_TAG) 678 return; 679 680 __blk_mq_put_driver_tag(rq->mq_hctx, rq); 681 } 682 683 bool blk_mq_complete_request_remote(struct request *rq) 684 { 685 WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); 686 687 blk_mq_put_driver_tag(rq); 688 689 /* 690 * For a polled request, always complete locallly, it's pointless 691 * to redirect the completion. 692 */ 693 if (rq->cmd_flags & REQ_HIPRI) 694 return false; 695 696 if (blk_mq_complete_need_ipi(rq)) { 697 rq->csd.func = __blk_mq_complete_request_remote; 698 rq->csd.info = rq; 699 rq->csd.flags = 0; 700 smp_call_function_single_async(rq->mq_ctx->cpu, &rq->csd); 701 } else { 702 if (rq->q->nr_hw_queues > 1) 703 return false; 704 blk_mq_trigger_softirq(rq); 705 } 706 707 return true; 708 } 709 EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote); 710 711 /** 712 * blk_mq_complete_request - end I/O on a request 713 * @rq: the request being processed 714 * 715 * Description: 716 * Complete a request by scheduling the ->complete_rq operation. 717 **/ 718 void blk_mq_complete_request(struct request *rq) 719 { 720 if (!blk_mq_complete_request_remote(rq)) 721 rq->q->mq_ops->complete(rq); 722 } 723 EXPORT_SYMBOL(blk_mq_complete_request); 724 725 static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx) 726 __releases(hctx->srcu) 727 { 728 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) 729 rcu_read_unlock(); 730 else 731 srcu_read_unlock(hctx->srcu, srcu_idx); 732 } 733 734 static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx) 735 __acquires(hctx->srcu) 736 { 737 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { 738 /* shut up gcc false positive */ 739 *srcu_idx = 0; 740 rcu_read_lock(); 741 } else 742 *srcu_idx = srcu_read_lock(hctx->srcu); 743 } 744 745 /** 746 * blk_mq_start_request - Start processing a request 747 * @rq: Pointer to request to be started 748 * 749 * Function used by device drivers to notify the block layer that a request 750 * is going to be processed now, so blk layer can do proper initializations 751 * such as starting the timeout timer. 752 */ 753 void blk_mq_start_request(struct request *rq) 754 { 755 struct request_queue *q = rq->q; 756 757 trace_block_rq_issue(q, rq); 758 759 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 760 rq->io_start_time_ns = ktime_get_ns(); 761 rq->stats_sectors = blk_rq_sectors(rq); 762 rq->rq_flags |= RQF_STATS; 763 rq_qos_issue(q, rq); 764 } 765 766 WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE); 767 768 blk_add_timer(rq); 769 WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT); 770 771 #ifdef CONFIG_BLK_DEV_INTEGRITY 772 if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE) 773 q->integrity.profile->prepare_fn(rq); 774 #endif 775 } 776 EXPORT_SYMBOL(blk_mq_start_request); 777 778 static void __blk_mq_requeue_request(struct request *rq) 779 { 780 struct request_queue *q = rq->q; 781 782 blk_mq_put_driver_tag(rq); 783 784 trace_block_rq_requeue(q, rq); 785 rq_qos_requeue(q, rq); 786 787 if (blk_mq_request_started(rq)) { 788 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 789 rq->rq_flags &= ~RQF_TIMED_OUT; 790 } 791 } 792 793 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) 794 { 795 __blk_mq_requeue_request(rq); 796 797 /* this request will be re-inserted to io scheduler queue */ 798 blk_mq_sched_requeue_request(rq); 799 800 BUG_ON(!list_empty(&rq->queuelist)); 801 blk_mq_add_to_requeue_list(rq, true, kick_requeue_list); 802 } 803 EXPORT_SYMBOL(blk_mq_requeue_request); 804 805 static void blk_mq_requeue_work(struct work_struct *work) 806 { 807 struct request_queue *q = 808 container_of(work, struct request_queue, requeue_work.work); 809 LIST_HEAD(rq_list); 810 struct request *rq, *next; 811 812 spin_lock_irq(&q->requeue_lock); 813 list_splice_init(&q->requeue_list, &rq_list); 814 spin_unlock_irq(&q->requeue_lock); 815 816 list_for_each_entry_safe(rq, next, &rq_list, queuelist) { 817 if (!(rq->rq_flags & (RQF_SOFTBARRIER | RQF_DONTPREP))) 818 continue; 819 820 rq->rq_flags &= ~RQF_SOFTBARRIER; 821 list_del_init(&rq->queuelist); 822 /* 823 * If RQF_DONTPREP, rq has contained some driver specific 824 * data, so insert it to hctx dispatch list to avoid any 825 * merge. 826 */ 827 if (rq->rq_flags & RQF_DONTPREP) 828 blk_mq_request_bypass_insert(rq, false, false); 829 else 830 blk_mq_sched_insert_request(rq, true, false, false); 831 } 832 833 while (!list_empty(&rq_list)) { 834 rq = list_entry(rq_list.next, struct request, queuelist); 835 list_del_init(&rq->queuelist); 836 blk_mq_sched_insert_request(rq, false, false, false); 837 } 838 839 blk_mq_run_hw_queues(q, false); 840 } 841 842 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, 843 bool kick_requeue_list) 844 { 845 struct request_queue *q = rq->q; 846 unsigned long flags; 847 848 /* 849 * We abuse this flag that is otherwise used by the I/O scheduler to 850 * request head insertion from the workqueue. 851 */ 852 BUG_ON(rq->rq_flags & RQF_SOFTBARRIER); 853 854 spin_lock_irqsave(&q->requeue_lock, flags); 855 if (at_head) { 856 rq->rq_flags |= RQF_SOFTBARRIER; 857 list_add(&rq->queuelist, &q->requeue_list); 858 } else { 859 list_add_tail(&rq->queuelist, &q->requeue_list); 860 } 861 spin_unlock_irqrestore(&q->requeue_lock, flags); 862 863 if (kick_requeue_list) 864 blk_mq_kick_requeue_list(q); 865 } 866 867 void blk_mq_kick_requeue_list(struct request_queue *q) 868 { 869 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0); 870 } 871 EXPORT_SYMBOL(blk_mq_kick_requeue_list); 872 873 void blk_mq_delay_kick_requeue_list(struct request_queue *q, 874 unsigned long msecs) 875 { 876 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 877 msecs_to_jiffies(msecs)); 878 } 879 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); 880 881 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) 882 { 883 if (tag < tags->nr_tags) { 884 prefetch(tags->rqs[tag]); 885 return tags->rqs[tag]; 886 } 887 888 return NULL; 889 } 890 EXPORT_SYMBOL(blk_mq_tag_to_rq); 891 892 static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq, 893 void *priv, bool reserved) 894 { 895 /* 896 * If we find a request that is inflight and the queue matches, 897 * we know the queue is busy. Return false to stop the iteration. 898 */ 899 if (rq->state == MQ_RQ_IN_FLIGHT && rq->q == hctx->queue) { 900 bool *busy = priv; 901 902 *busy = true; 903 return false; 904 } 905 906 return true; 907 } 908 909 bool blk_mq_queue_inflight(struct request_queue *q) 910 { 911 bool busy = false; 912 913 blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy); 914 return busy; 915 } 916 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight); 917 918 static void blk_mq_rq_timed_out(struct request *req, bool reserved) 919 { 920 req->rq_flags |= RQF_TIMED_OUT; 921 if (req->q->mq_ops->timeout) { 922 enum blk_eh_timer_return ret; 923 924 ret = req->q->mq_ops->timeout(req, reserved); 925 if (ret == BLK_EH_DONE) 926 return; 927 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER); 928 } 929 930 blk_add_timer(req); 931 } 932 933 static bool blk_mq_req_expired(struct request *rq, unsigned long *next) 934 { 935 unsigned long deadline; 936 937 if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT) 938 return false; 939 if (rq->rq_flags & RQF_TIMED_OUT) 940 return false; 941 942 deadline = READ_ONCE(rq->deadline); 943 if (time_after_eq(jiffies, deadline)) 944 return true; 945 946 if (*next == 0) 947 *next = deadline; 948 else if (time_after(*next, deadline)) 949 *next = deadline; 950 return false; 951 } 952 953 static bool blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, 954 struct request *rq, void *priv, bool reserved) 955 { 956 unsigned long *next = priv; 957 958 /* 959 * Just do a quick check if it is expired before locking the request in 960 * so we're not unnecessarilly synchronizing across CPUs. 961 */ 962 if (!blk_mq_req_expired(rq, next)) 963 return true; 964 965 /* 966 * We have reason to believe the request may be expired. Take a 967 * reference on the request to lock this request lifetime into its 968 * currently allocated context to prevent it from being reallocated in 969 * the event the completion by-passes this timeout handler. 970 * 971 * If the reference was already released, then the driver beat the 972 * timeout handler to posting a natural completion. 973 */ 974 if (!refcount_inc_not_zero(&rq->ref)) 975 return true; 976 977 /* 978 * The request is now locked and cannot be reallocated underneath the 979 * timeout handler's processing. Re-verify this exact request is truly 980 * expired; if it is not expired, then the request was completed and 981 * reallocated as a new request. 982 */ 983 if (blk_mq_req_expired(rq, next)) 984 blk_mq_rq_timed_out(rq, reserved); 985 986 if (is_flush_rq(rq, hctx)) 987 rq->end_io(rq, 0); 988 else if (refcount_dec_and_test(&rq->ref)) 989 __blk_mq_free_request(rq); 990 991 return true; 992 } 993 994 static void blk_mq_timeout_work(struct work_struct *work) 995 { 996 struct request_queue *q = 997 container_of(work, struct request_queue, timeout_work); 998 unsigned long next = 0; 999 struct blk_mq_hw_ctx *hctx; 1000 int i; 1001 1002 /* A deadlock might occur if a request is stuck requiring a 1003 * timeout at the same time a queue freeze is waiting 1004 * completion, since the timeout code would not be able to 1005 * acquire the queue reference here. 1006 * 1007 * That's why we don't use blk_queue_enter here; instead, we use 1008 * percpu_ref_tryget directly, because we need to be able to 1009 * obtain a reference even in the short window between the queue 1010 * starting to freeze, by dropping the first reference in 1011 * blk_freeze_queue_start, and the moment the last request is 1012 * consumed, marked by the instant q_usage_counter reaches 1013 * zero. 1014 */ 1015 if (!percpu_ref_tryget(&q->q_usage_counter)) 1016 return; 1017 1018 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next); 1019 1020 if (next != 0) { 1021 mod_timer(&q->timeout, next); 1022 } else { 1023 /* 1024 * Request timeouts are handled as a forward rolling timer. If 1025 * we end up here it means that no requests are pending and 1026 * also that no request has been pending for a while. Mark 1027 * each hctx as idle. 1028 */ 1029 queue_for_each_hw_ctx(q, hctx, i) { 1030 /* the hctx may be unmapped, so check it here */ 1031 if (blk_mq_hw_queue_mapped(hctx)) 1032 blk_mq_tag_idle(hctx); 1033 } 1034 } 1035 blk_queue_exit(q); 1036 } 1037 1038 struct flush_busy_ctx_data { 1039 struct blk_mq_hw_ctx *hctx; 1040 struct list_head *list; 1041 }; 1042 1043 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) 1044 { 1045 struct flush_busy_ctx_data *flush_data = data; 1046 struct blk_mq_hw_ctx *hctx = flush_data->hctx; 1047 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 1048 enum hctx_type type = hctx->type; 1049 1050 spin_lock(&ctx->lock); 1051 list_splice_tail_init(&ctx->rq_lists[type], flush_data->list); 1052 sbitmap_clear_bit(sb, bitnr); 1053 spin_unlock(&ctx->lock); 1054 return true; 1055 } 1056 1057 /* 1058 * Process software queues that have been marked busy, splicing them 1059 * to the for-dispatch 1060 */ 1061 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) 1062 { 1063 struct flush_busy_ctx_data data = { 1064 .hctx = hctx, 1065 .list = list, 1066 }; 1067 1068 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data); 1069 } 1070 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs); 1071 1072 struct dispatch_rq_data { 1073 struct blk_mq_hw_ctx *hctx; 1074 struct request *rq; 1075 }; 1076 1077 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr, 1078 void *data) 1079 { 1080 struct dispatch_rq_data *dispatch_data = data; 1081 struct blk_mq_hw_ctx *hctx = dispatch_data->hctx; 1082 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 1083 enum hctx_type type = hctx->type; 1084 1085 spin_lock(&ctx->lock); 1086 if (!list_empty(&ctx->rq_lists[type])) { 1087 dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next); 1088 list_del_init(&dispatch_data->rq->queuelist); 1089 if (list_empty(&ctx->rq_lists[type])) 1090 sbitmap_clear_bit(sb, bitnr); 1091 } 1092 spin_unlock(&ctx->lock); 1093 1094 return !dispatch_data->rq; 1095 } 1096 1097 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, 1098 struct blk_mq_ctx *start) 1099 { 1100 unsigned off = start ? start->index_hw[hctx->type] : 0; 1101 struct dispatch_rq_data data = { 1102 .hctx = hctx, 1103 .rq = NULL, 1104 }; 1105 1106 __sbitmap_for_each_set(&hctx->ctx_map, off, 1107 dispatch_rq_from_ctx, &data); 1108 1109 return data.rq; 1110 } 1111 1112 static inline unsigned int queued_to_index(unsigned int queued) 1113 { 1114 if (!queued) 1115 return 0; 1116 1117 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); 1118 } 1119 1120 static bool __blk_mq_get_driver_tag(struct request *rq) 1121 { 1122 struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags; 1123 unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags; 1124 int tag; 1125 1126 blk_mq_tag_busy(rq->mq_hctx); 1127 1128 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) { 1129 bt = &rq->mq_hctx->tags->breserved_tags; 1130 tag_offset = 0; 1131 } 1132 1133 if (!hctx_may_queue(rq->mq_hctx, bt)) 1134 return false; 1135 tag = __sbitmap_queue_get(bt); 1136 if (tag == BLK_MQ_NO_TAG) 1137 return false; 1138 1139 rq->tag = tag + tag_offset; 1140 return true; 1141 } 1142 1143 static bool blk_mq_get_driver_tag(struct request *rq) 1144 { 1145 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1146 1147 if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag(rq)) 1148 return false; 1149 1150 if (hctx->flags & BLK_MQ_F_TAG_SHARED) { 1151 rq->rq_flags |= RQF_MQ_INFLIGHT; 1152 atomic_inc(&hctx->nr_active); 1153 } 1154 hctx->tags->rqs[rq->tag] = rq; 1155 return true; 1156 } 1157 1158 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, 1159 int flags, void *key) 1160 { 1161 struct blk_mq_hw_ctx *hctx; 1162 1163 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); 1164 1165 spin_lock(&hctx->dispatch_wait_lock); 1166 if (!list_empty(&wait->entry)) { 1167 struct sbitmap_queue *sbq; 1168 1169 list_del_init(&wait->entry); 1170 sbq = &hctx->tags->bitmap_tags; 1171 atomic_dec(&sbq->ws_active); 1172 } 1173 spin_unlock(&hctx->dispatch_wait_lock); 1174 1175 blk_mq_run_hw_queue(hctx, true); 1176 return 1; 1177 } 1178 1179 /* 1180 * Mark us waiting for a tag. For shared tags, this involves hooking us into 1181 * the tag wakeups. For non-shared tags, we can simply mark us needing a 1182 * restart. For both cases, take care to check the condition again after 1183 * marking us as waiting. 1184 */ 1185 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx, 1186 struct request *rq) 1187 { 1188 struct sbitmap_queue *sbq = &hctx->tags->bitmap_tags; 1189 struct wait_queue_head *wq; 1190 wait_queue_entry_t *wait; 1191 bool ret; 1192 1193 if (!(hctx->flags & BLK_MQ_F_TAG_SHARED)) { 1194 blk_mq_sched_mark_restart_hctx(hctx); 1195 1196 /* 1197 * It's possible that a tag was freed in the window between the 1198 * allocation failure and adding the hardware queue to the wait 1199 * queue. 1200 * 1201 * Don't clear RESTART here, someone else could have set it. 1202 * At most this will cost an extra queue run. 1203 */ 1204 return blk_mq_get_driver_tag(rq); 1205 } 1206 1207 wait = &hctx->dispatch_wait; 1208 if (!list_empty_careful(&wait->entry)) 1209 return false; 1210 1211 wq = &bt_wait_ptr(sbq, hctx)->wait; 1212 1213 spin_lock_irq(&wq->lock); 1214 spin_lock(&hctx->dispatch_wait_lock); 1215 if (!list_empty(&wait->entry)) { 1216 spin_unlock(&hctx->dispatch_wait_lock); 1217 spin_unlock_irq(&wq->lock); 1218 return false; 1219 } 1220 1221 atomic_inc(&sbq->ws_active); 1222 wait->flags &= ~WQ_FLAG_EXCLUSIVE; 1223 __add_wait_queue(wq, wait); 1224 1225 /* 1226 * It's possible that a tag was freed in the window between the 1227 * allocation failure and adding the hardware queue to the wait 1228 * queue. 1229 */ 1230 ret = blk_mq_get_driver_tag(rq); 1231 if (!ret) { 1232 spin_unlock(&hctx->dispatch_wait_lock); 1233 spin_unlock_irq(&wq->lock); 1234 return false; 1235 } 1236 1237 /* 1238 * We got a tag, remove ourselves from the wait queue to ensure 1239 * someone else gets the wakeup. 1240 */ 1241 list_del_init(&wait->entry); 1242 atomic_dec(&sbq->ws_active); 1243 spin_unlock(&hctx->dispatch_wait_lock); 1244 spin_unlock_irq(&wq->lock); 1245 1246 return true; 1247 } 1248 1249 #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT 8 1250 #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR 4 1251 /* 1252 * Update dispatch busy with the Exponential Weighted Moving Average(EWMA): 1253 * - EWMA is one simple way to compute running average value 1254 * - weight(7/8 and 1/8) is applied so that it can decrease exponentially 1255 * - take 4 as factor for avoiding to get too small(0) result, and this 1256 * factor doesn't matter because EWMA decreases exponentially 1257 */ 1258 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy) 1259 { 1260 unsigned int ewma; 1261 1262 if (hctx->queue->elevator) 1263 return; 1264 1265 ewma = hctx->dispatch_busy; 1266 1267 if (!ewma && !busy) 1268 return; 1269 1270 ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1; 1271 if (busy) 1272 ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR; 1273 ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT; 1274 1275 hctx->dispatch_busy = ewma; 1276 } 1277 1278 #define BLK_MQ_RESOURCE_DELAY 3 /* ms units */ 1279 1280 static void blk_mq_handle_dev_resource(struct request *rq, 1281 struct list_head *list) 1282 { 1283 struct request *next = 1284 list_first_entry_or_null(list, struct request, queuelist); 1285 1286 /* 1287 * If an I/O scheduler has been configured and we got a driver tag for 1288 * the next request already, free it. 1289 */ 1290 if (next) 1291 blk_mq_put_driver_tag(next); 1292 1293 list_add(&rq->queuelist, list); 1294 __blk_mq_requeue_request(rq); 1295 } 1296 1297 static void blk_mq_handle_zone_resource(struct request *rq, 1298 struct list_head *zone_list) 1299 { 1300 /* 1301 * If we end up here it is because we cannot dispatch a request to a 1302 * specific zone due to LLD level zone-write locking or other zone 1303 * related resource not being available. In this case, set the request 1304 * aside in zone_list for retrying it later. 1305 */ 1306 list_add(&rq->queuelist, zone_list); 1307 __blk_mq_requeue_request(rq); 1308 } 1309 1310 enum prep_dispatch { 1311 PREP_DISPATCH_OK, 1312 PREP_DISPATCH_NO_TAG, 1313 PREP_DISPATCH_NO_BUDGET, 1314 }; 1315 1316 static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq, 1317 bool need_budget) 1318 { 1319 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1320 1321 if (need_budget && !blk_mq_get_dispatch_budget(rq->q)) { 1322 blk_mq_put_driver_tag(rq); 1323 return PREP_DISPATCH_NO_BUDGET; 1324 } 1325 1326 if (!blk_mq_get_driver_tag(rq)) { 1327 /* 1328 * The initial allocation attempt failed, so we need to 1329 * rerun the hardware queue when a tag is freed. The 1330 * waitqueue takes care of that. If the queue is run 1331 * before we add this entry back on the dispatch list, 1332 * we'll re-run it below. 1333 */ 1334 if (!blk_mq_mark_tag_wait(hctx, rq)) { 1335 /* 1336 * All budgets not got from this function will be put 1337 * together during handling partial dispatch 1338 */ 1339 if (need_budget) 1340 blk_mq_put_dispatch_budget(rq->q); 1341 return PREP_DISPATCH_NO_TAG; 1342 } 1343 } 1344 1345 return PREP_DISPATCH_OK; 1346 } 1347 1348 /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */ 1349 static void blk_mq_release_budgets(struct request_queue *q, 1350 unsigned int nr_budgets) 1351 { 1352 int i; 1353 1354 for (i = 0; i < nr_budgets; i++) 1355 blk_mq_put_dispatch_budget(q); 1356 } 1357 1358 /* 1359 * Returns true if we did some work AND can potentially do more. 1360 */ 1361 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, 1362 unsigned int nr_budgets) 1363 { 1364 enum prep_dispatch prep; 1365 struct request_queue *q = hctx->queue; 1366 struct request *rq, *nxt; 1367 int errors, queued; 1368 blk_status_t ret = BLK_STS_OK; 1369 LIST_HEAD(zone_list); 1370 1371 if (list_empty(list)) 1372 return false; 1373 1374 /* 1375 * Now process all the entries, sending them to the driver. 1376 */ 1377 errors = queued = 0; 1378 do { 1379 struct blk_mq_queue_data bd; 1380 1381 rq = list_first_entry(list, struct request, queuelist); 1382 1383 WARN_ON_ONCE(hctx != rq->mq_hctx); 1384 prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets); 1385 if (prep != PREP_DISPATCH_OK) 1386 break; 1387 1388 list_del_init(&rq->queuelist); 1389 1390 bd.rq = rq; 1391 1392 /* 1393 * Flag last if we have no more requests, or if we have more 1394 * but can't assign a driver tag to it. 1395 */ 1396 if (list_empty(list)) 1397 bd.last = true; 1398 else { 1399 nxt = list_first_entry(list, struct request, queuelist); 1400 bd.last = !blk_mq_get_driver_tag(nxt); 1401 } 1402 1403 /* 1404 * once the request is queued to lld, no need to cover the 1405 * budget any more 1406 */ 1407 if (nr_budgets) 1408 nr_budgets--; 1409 ret = q->mq_ops->queue_rq(hctx, &bd); 1410 if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) { 1411 blk_mq_handle_dev_resource(rq, list); 1412 break; 1413 } else if (ret == BLK_STS_ZONE_RESOURCE) { 1414 /* 1415 * Move the request to zone_list and keep going through 1416 * the dispatch list to find more requests the drive can 1417 * accept. 1418 */ 1419 blk_mq_handle_zone_resource(rq, &zone_list); 1420 if (list_empty(list)) 1421 break; 1422 continue; 1423 } 1424 1425 if (unlikely(ret != BLK_STS_OK)) { 1426 errors++; 1427 blk_mq_end_request(rq, BLK_STS_IOERR); 1428 continue; 1429 } 1430 1431 queued++; 1432 } while (!list_empty(list)); 1433 1434 if (!list_empty(&zone_list)) 1435 list_splice_tail_init(&zone_list, list); 1436 1437 hctx->dispatched[queued_to_index(queued)]++; 1438 1439 /* 1440 * Any items that need requeuing? Stuff them into hctx->dispatch, 1441 * that is where we will continue on next queue run. 1442 */ 1443 if (!list_empty(list)) { 1444 bool needs_restart; 1445 /* For non-shared tags, the RESTART check will suffice */ 1446 bool no_tag = prep == PREP_DISPATCH_NO_TAG && 1447 (hctx->flags & BLK_MQ_F_TAG_SHARED); 1448 bool no_budget_avail = prep == PREP_DISPATCH_NO_BUDGET; 1449 1450 blk_mq_release_budgets(q, nr_budgets); 1451 1452 /* 1453 * If we didn't flush the entire list, we could have told 1454 * the driver there was more coming, but that turned out to 1455 * be a lie. 1456 */ 1457 if (q->mq_ops->commit_rqs && queued) 1458 q->mq_ops->commit_rqs(hctx); 1459 1460 spin_lock(&hctx->lock); 1461 list_splice_tail_init(list, &hctx->dispatch); 1462 spin_unlock(&hctx->lock); 1463 1464 /* 1465 * If SCHED_RESTART was set by the caller of this function and 1466 * it is no longer set that means that it was cleared by another 1467 * thread and hence that a queue rerun is needed. 1468 * 1469 * If 'no_tag' is set, that means that we failed getting 1470 * a driver tag with an I/O scheduler attached. If our dispatch 1471 * waitqueue is no longer active, ensure that we run the queue 1472 * AFTER adding our entries back to the list. 1473 * 1474 * If no I/O scheduler has been configured it is possible that 1475 * the hardware queue got stopped and restarted before requests 1476 * were pushed back onto the dispatch list. Rerun the queue to 1477 * avoid starvation. Notes: 1478 * - blk_mq_run_hw_queue() checks whether or not a queue has 1479 * been stopped before rerunning a queue. 1480 * - Some but not all block drivers stop a queue before 1481 * returning BLK_STS_RESOURCE. Two exceptions are scsi-mq 1482 * and dm-rq. 1483 * 1484 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART 1485 * bit is set, run queue after a delay to avoid IO stalls 1486 * that could otherwise occur if the queue is idle. We'll do 1487 * similar if we couldn't get budget and SCHED_RESTART is set. 1488 */ 1489 needs_restart = blk_mq_sched_needs_restart(hctx); 1490 if (!needs_restart || 1491 (no_tag && list_empty_careful(&hctx->dispatch_wait.entry))) 1492 blk_mq_run_hw_queue(hctx, true); 1493 else if (needs_restart && (ret == BLK_STS_RESOURCE || 1494 no_budget_avail)) 1495 blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY); 1496 1497 blk_mq_update_dispatch_busy(hctx, true); 1498 return false; 1499 } else 1500 blk_mq_update_dispatch_busy(hctx, false); 1501 1502 return (queued + errors) != 0; 1503 } 1504 1505 /** 1506 * __blk_mq_run_hw_queue - Run a hardware queue. 1507 * @hctx: Pointer to the hardware queue to run. 1508 * 1509 * Send pending requests to the hardware. 1510 */ 1511 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) 1512 { 1513 int srcu_idx; 1514 1515 /* 1516 * We should be running this queue from one of the CPUs that 1517 * are mapped to it. 1518 * 1519 * There are at least two related races now between setting 1520 * hctx->next_cpu from blk_mq_hctx_next_cpu() and running 1521 * __blk_mq_run_hw_queue(): 1522 * 1523 * - hctx->next_cpu is found offline in blk_mq_hctx_next_cpu(), 1524 * but later it becomes online, then this warning is harmless 1525 * at all 1526 * 1527 * - hctx->next_cpu is found online in blk_mq_hctx_next_cpu(), 1528 * but later it becomes offline, then the warning can't be 1529 * triggered, and we depend on blk-mq timeout handler to 1530 * handle dispatched requests to this hctx 1531 */ 1532 if (!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) && 1533 cpu_online(hctx->next_cpu)) { 1534 printk(KERN_WARNING "run queue from wrong CPU %d, hctx %s\n", 1535 raw_smp_processor_id(), 1536 cpumask_empty(hctx->cpumask) ? "inactive": "active"); 1537 dump_stack(); 1538 } 1539 1540 /* 1541 * We can't run the queue inline with ints disabled. Ensure that 1542 * we catch bad users of this early. 1543 */ 1544 WARN_ON_ONCE(in_interrupt()); 1545 1546 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); 1547 1548 hctx_lock(hctx, &srcu_idx); 1549 blk_mq_sched_dispatch_requests(hctx); 1550 hctx_unlock(hctx, srcu_idx); 1551 } 1552 1553 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx) 1554 { 1555 int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask); 1556 1557 if (cpu >= nr_cpu_ids) 1558 cpu = cpumask_first(hctx->cpumask); 1559 return cpu; 1560 } 1561 1562 /* 1563 * It'd be great if the workqueue API had a way to pass 1564 * in a mask and had some smarts for more clever placement. 1565 * For now we just round-robin here, switching for every 1566 * BLK_MQ_CPU_WORK_BATCH queued items. 1567 */ 1568 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) 1569 { 1570 bool tried = false; 1571 int next_cpu = hctx->next_cpu; 1572 1573 if (hctx->queue->nr_hw_queues == 1) 1574 return WORK_CPU_UNBOUND; 1575 1576 if (--hctx->next_cpu_batch <= 0) { 1577 select_cpu: 1578 next_cpu = cpumask_next_and(next_cpu, hctx->cpumask, 1579 cpu_online_mask); 1580 if (next_cpu >= nr_cpu_ids) 1581 next_cpu = blk_mq_first_mapped_cpu(hctx); 1582 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 1583 } 1584 1585 /* 1586 * Do unbound schedule if we can't find a online CPU for this hctx, 1587 * and it should only happen in the path of handling CPU DEAD. 1588 */ 1589 if (!cpu_online(next_cpu)) { 1590 if (!tried) { 1591 tried = true; 1592 goto select_cpu; 1593 } 1594 1595 /* 1596 * Make sure to re-select CPU next time once after CPUs 1597 * in hctx->cpumask become online again. 1598 */ 1599 hctx->next_cpu = next_cpu; 1600 hctx->next_cpu_batch = 1; 1601 return WORK_CPU_UNBOUND; 1602 } 1603 1604 hctx->next_cpu = next_cpu; 1605 return next_cpu; 1606 } 1607 1608 /** 1609 * __blk_mq_delay_run_hw_queue - Run (or schedule to run) a hardware queue. 1610 * @hctx: Pointer to the hardware queue to run. 1611 * @async: If we want to run the queue asynchronously. 1612 * @msecs: Microseconds of delay to wait before running the queue. 1613 * 1614 * If !@async, try to run the queue now. Else, run the queue asynchronously and 1615 * with a delay of @msecs. 1616 */ 1617 static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, 1618 unsigned long msecs) 1619 { 1620 if (unlikely(blk_mq_hctx_stopped(hctx))) 1621 return; 1622 1623 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) { 1624 int cpu = get_cpu(); 1625 if (cpumask_test_cpu(cpu, hctx->cpumask)) { 1626 __blk_mq_run_hw_queue(hctx); 1627 put_cpu(); 1628 return; 1629 } 1630 1631 put_cpu(); 1632 } 1633 1634 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, 1635 msecs_to_jiffies(msecs)); 1636 } 1637 1638 /** 1639 * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously. 1640 * @hctx: Pointer to the hardware queue to run. 1641 * @msecs: Microseconds of delay to wait before running the queue. 1642 * 1643 * Run a hardware queue asynchronously with a delay of @msecs. 1644 */ 1645 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) 1646 { 1647 __blk_mq_delay_run_hw_queue(hctx, true, msecs); 1648 } 1649 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); 1650 1651 /** 1652 * blk_mq_run_hw_queue - Start to run a hardware queue. 1653 * @hctx: Pointer to the hardware queue to run. 1654 * @async: If we want to run the queue asynchronously. 1655 * 1656 * Check if the request queue is not in a quiesced state and if there are 1657 * pending requests to be sent. If this is true, run the queue to send requests 1658 * to hardware. 1659 */ 1660 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1661 { 1662 int srcu_idx; 1663 bool need_run; 1664 1665 /* 1666 * When queue is quiesced, we may be switching io scheduler, or 1667 * updating nr_hw_queues, or other things, and we can't run queue 1668 * any more, even __blk_mq_hctx_has_pending() can't be called safely. 1669 * 1670 * And queue will be rerun in blk_mq_unquiesce_queue() if it is 1671 * quiesced. 1672 */ 1673 hctx_lock(hctx, &srcu_idx); 1674 need_run = !blk_queue_quiesced(hctx->queue) && 1675 blk_mq_hctx_has_pending(hctx); 1676 hctx_unlock(hctx, srcu_idx); 1677 1678 if (need_run) 1679 __blk_mq_delay_run_hw_queue(hctx, async, 0); 1680 } 1681 EXPORT_SYMBOL(blk_mq_run_hw_queue); 1682 1683 /** 1684 * blk_mq_run_hw_queue - Run all hardware queues in a request queue. 1685 * @q: Pointer to the request queue to run. 1686 * @async: If we want to run the queue asynchronously. 1687 */ 1688 void blk_mq_run_hw_queues(struct request_queue *q, bool async) 1689 { 1690 struct blk_mq_hw_ctx *hctx; 1691 int i; 1692 1693 queue_for_each_hw_ctx(q, hctx, i) { 1694 if (blk_mq_hctx_stopped(hctx)) 1695 continue; 1696 1697 blk_mq_run_hw_queue(hctx, async); 1698 } 1699 } 1700 EXPORT_SYMBOL(blk_mq_run_hw_queues); 1701 1702 /** 1703 * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously. 1704 * @q: Pointer to the request queue to run. 1705 * @msecs: Microseconds of delay to wait before running the queues. 1706 */ 1707 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs) 1708 { 1709 struct blk_mq_hw_ctx *hctx; 1710 int i; 1711 1712 queue_for_each_hw_ctx(q, hctx, i) { 1713 if (blk_mq_hctx_stopped(hctx)) 1714 continue; 1715 1716 blk_mq_delay_run_hw_queue(hctx, msecs); 1717 } 1718 } 1719 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues); 1720 1721 /** 1722 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped 1723 * @q: request queue. 1724 * 1725 * The caller is responsible for serializing this function against 1726 * blk_mq_{start,stop}_hw_queue(). 1727 */ 1728 bool blk_mq_queue_stopped(struct request_queue *q) 1729 { 1730 struct blk_mq_hw_ctx *hctx; 1731 int i; 1732 1733 queue_for_each_hw_ctx(q, hctx, i) 1734 if (blk_mq_hctx_stopped(hctx)) 1735 return true; 1736 1737 return false; 1738 } 1739 EXPORT_SYMBOL(blk_mq_queue_stopped); 1740 1741 /* 1742 * This function is often used for pausing .queue_rq() by driver when 1743 * there isn't enough resource or some conditions aren't satisfied, and 1744 * BLK_STS_RESOURCE is usually returned. 1745 * 1746 * We do not guarantee that dispatch can be drained or blocked 1747 * after blk_mq_stop_hw_queue() returns. Please use 1748 * blk_mq_quiesce_queue() for that requirement. 1749 */ 1750 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) 1751 { 1752 cancel_delayed_work(&hctx->run_work); 1753 1754 set_bit(BLK_MQ_S_STOPPED, &hctx->state); 1755 } 1756 EXPORT_SYMBOL(blk_mq_stop_hw_queue); 1757 1758 /* 1759 * This function is often used for pausing .queue_rq() by driver when 1760 * there isn't enough resource or some conditions aren't satisfied, and 1761 * BLK_STS_RESOURCE is usually returned. 1762 * 1763 * We do not guarantee that dispatch can be drained or blocked 1764 * after blk_mq_stop_hw_queues() returns. Please use 1765 * blk_mq_quiesce_queue() for that requirement. 1766 */ 1767 void blk_mq_stop_hw_queues(struct request_queue *q) 1768 { 1769 struct blk_mq_hw_ctx *hctx; 1770 int i; 1771 1772 queue_for_each_hw_ctx(q, hctx, i) 1773 blk_mq_stop_hw_queue(hctx); 1774 } 1775 EXPORT_SYMBOL(blk_mq_stop_hw_queues); 1776 1777 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) 1778 { 1779 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1780 1781 blk_mq_run_hw_queue(hctx, false); 1782 } 1783 EXPORT_SYMBOL(blk_mq_start_hw_queue); 1784 1785 void blk_mq_start_hw_queues(struct request_queue *q) 1786 { 1787 struct blk_mq_hw_ctx *hctx; 1788 int i; 1789 1790 queue_for_each_hw_ctx(q, hctx, i) 1791 blk_mq_start_hw_queue(hctx); 1792 } 1793 EXPORT_SYMBOL(blk_mq_start_hw_queues); 1794 1795 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1796 { 1797 if (!blk_mq_hctx_stopped(hctx)) 1798 return; 1799 1800 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1801 blk_mq_run_hw_queue(hctx, async); 1802 } 1803 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue); 1804 1805 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) 1806 { 1807 struct blk_mq_hw_ctx *hctx; 1808 int i; 1809 1810 queue_for_each_hw_ctx(q, hctx, i) 1811 blk_mq_start_stopped_hw_queue(hctx, async); 1812 } 1813 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); 1814 1815 static void blk_mq_run_work_fn(struct work_struct *work) 1816 { 1817 struct blk_mq_hw_ctx *hctx; 1818 1819 hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work); 1820 1821 /* 1822 * If we are stopped, don't run the queue. 1823 */ 1824 if (test_bit(BLK_MQ_S_STOPPED, &hctx->state)) 1825 return; 1826 1827 __blk_mq_run_hw_queue(hctx); 1828 } 1829 1830 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx, 1831 struct request *rq, 1832 bool at_head) 1833 { 1834 struct blk_mq_ctx *ctx = rq->mq_ctx; 1835 enum hctx_type type = hctx->type; 1836 1837 lockdep_assert_held(&ctx->lock); 1838 1839 trace_block_rq_insert(hctx->queue, rq); 1840 1841 if (at_head) 1842 list_add(&rq->queuelist, &ctx->rq_lists[type]); 1843 else 1844 list_add_tail(&rq->queuelist, &ctx->rq_lists[type]); 1845 } 1846 1847 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, 1848 bool at_head) 1849 { 1850 struct blk_mq_ctx *ctx = rq->mq_ctx; 1851 1852 lockdep_assert_held(&ctx->lock); 1853 1854 __blk_mq_insert_req_list(hctx, rq, at_head); 1855 blk_mq_hctx_mark_pending(hctx, ctx); 1856 } 1857 1858 /** 1859 * blk_mq_request_bypass_insert - Insert a request at dispatch list. 1860 * @rq: Pointer to request to be inserted. 1861 * @run_queue: If we should run the hardware queue after inserting the request. 1862 * 1863 * Should only be used carefully, when the caller knows we want to 1864 * bypass a potential IO scheduler on the target device. 1865 */ 1866 void blk_mq_request_bypass_insert(struct request *rq, bool at_head, 1867 bool run_queue) 1868 { 1869 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1870 1871 spin_lock(&hctx->lock); 1872 if (at_head) 1873 list_add(&rq->queuelist, &hctx->dispatch); 1874 else 1875 list_add_tail(&rq->queuelist, &hctx->dispatch); 1876 spin_unlock(&hctx->lock); 1877 1878 if (run_queue) 1879 blk_mq_run_hw_queue(hctx, false); 1880 } 1881 1882 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 1883 struct list_head *list) 1884 1885 { 1886 struct request *rq; 1887 enum hctx_type type = hctx->type; 1888 1889 /* 1890 * preemption doesn't flush plug list, so it's possible ctx->cpu is 1891 * offline now 1892 */ 1893 list_for_each_entry(rq, list, queuelist) { 1894 BUG_ON(rq->mq_ctx != ctx); 1895 trace_block_rq_insert(hctx->queue, rq); 1896 } 1897 1898 spin_lock(&ctx->lock); 1899 list_splice_tail_init(list, &ctx->rq_lists[type]); 1900 blk_mq_hctx_mark_pending(hctx, ctx); 1901 spin_unlock(&ctx->lock); 1902 } 1903 1904 static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b) 1905 { 1906 struct request *rqa = container_of(a, struct request, queuelist); 1907 struct request *rqb = container_of(b, struct request, queuelist); 1908 1909 if (rqa->mq_ctx != rqb->mq_ctx) 1910 return rqa->mq_ctx > rqb->mq_ctx; 1911 if (rqa->mq_hctx != rqb->mq_hctx) 1912 return rqa->mq_hctx > rqb->mq_hctx; 1913 1914 return blk_rq_pos(rqa) > blk_rq_pos(rqb); 1915 } 1916 1917 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 1918 { 1919 LIST_HEAD(list); 1920 1921 if (list_empty(&plug->mq_list)) 1922 return; 1923 list_splice_init(&plug->mq_list, &list); 1924 1925 if (plug->rq_count > 2 && plug->multiple_queues) 1926 list_sort(NULL, &list, plug_rq_cmp); 1927 1928 plug->rq_count = 0; 1929 1930 do { 1931 struct list_head rq_list; 1932 struct request *rq, *head_rq = list_entry_rq(list.next); 1933 struct list_head *pos = &head_rq->queuelist; /* skip first */ 1934 struct blk_mq_hw_ctx *this_hctx = head_rq->mq_hctx; 1935 struct blk_mq_ctx *this_ctx = head_rq->mq_ctx; 1936 unsigned int depth = 1; 1937 1938 list_for_each_continue(pos, &list) { 1939 rq = list_entry_rq(pos); 1940 BUG_ON(!rq->q); 1941 if (rq->mq_hctx != this_hctx || rq->mq_ctx != this_ctx) 1942 break; 1943 depth++; 1944 } 1945 1946 list_cut_before(&rq_list, &list, pos); 1947 trace_block_unplug(head_rq->q, depth, !from_schedule); 1948 blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list, 1949 from_schedule); 1950 } while(!list_empty(&list)); 1951 } 1952 1953 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio, 1954 unsigned int nr_segs) 1955 { 1956 if (bio->bi_opf & REQ_RAHEAD) 1957 rq->cmd_flags |= REQ_FAILFAST_MASK; 1958 1959 rq->__sector = bio->bi_iter.bi_sector; 1960 rq->write_hint = bio->bi_write_hint; 1961 blk_rq_bio_prep(rq, bio, nr_segs); 1962 blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO); 1963 1964 blk_account_io_start(rq); 1965 } 1966 1967 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx, 1968 struct request *rq, 1969 blk_qc_t *cookie, bool last) 1970 { 1971 struct request_queue *q = rq->q; 1972 struct blk_mq_queue_data bd = { 1973 .rq = rq, 1974 .last = last, 1975 }; 1976 blk_qc_t new_cookie; 1977 blk_status_t ret; 1978 1979 new_cookie = request_to_qc_t(hctx, rq); 1980 1981 /* 1982 * For OK queue, we are done. For error, caller may kill it. 1983 * Any other error (busy), just add it to our list as we 1984 * previously would have done. 1985 */ 1986 ret = q->mq_ops->queue_rq(hctx, &bd); 1987 switch (ret) { 1988 case BLK_STS_OK: 1989 blk_mq_update_dispatch_busy(hctx, false); 1990 *cookie = new_cookie; 1991 break; 1992 case BLK_STS_RESOURCE: 1993 case BLK_STS_DEV_RESOURCE: 1994 blk_mq_update_dispatch_busy(hctx, true); 1995 __blk_mq_requeue_request(rq); 1996 break; 1997 default: 1998 blk_mq_update_dispatch_busy(hctx, false); 1999 *cookie = BLK_QC_T_NONE; 2000 break; 2001 } 2002 2003 return ret; 2004 } 2005 2006 static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 2007 struct request *rq, 2008 blk_qc_t *cookie, 2009 bool bypass_insert, bool last) 2010 { 2011 struct request_queue *q = rq->q; 2012 bool run_queue = true; 2013 2014 /* 2015 * RCU or SRCU read lock is needed before checking quiesced flag. 2016 * 2017 * When queue is stopped or quiesced, ignore 'bypass_insert' from 2018 * blk_mq_request_issue_directly(), and return BLK_STS_OK to caller, 2019 * and avoid driver to try to dispatch again. 2020 */ 2021 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) { 2022 run_queue = false; 2023 bypass_insert = false; 2024 goto insert; 2025 } 2026 2027 if (q->elevator && !bypass_insert) 2028 goto insert; 2029 2030 if (!blk_mq_get_dispatch_budget(q)) 2031 goto insert; 2032 2033 if (!blk_mq_get_driver_tag(rq)) { 2034 blk_mq_put_dispatch_budget(q); 2035 goto insert; 2036 } 2037 2038 return __blk_mq_issue_directly(hctx, rq, cookie, last); 2039 insert: 2040 if (bypass_insert) 2041 return BLK_STS_RESOURCE; 2042 2043 blk_mq_request_bypass_insert(rq, false, run_queue); 2044 return BLK_STS_OK; 2045 } 2046 2047 /** 2048 * blk_mq_try_issue_directly - Try to send a request directly to device driver. 2049 * @hctx: Pointer of the associated hardware queue. 2050 * @rq: Pointer to request to be sent. 2051 * @cookie: Request queue cookie. 2052 * 2053 * If the device has enough resources to accept a new request now, send the 2054 * request directly to device driver. Else, insert at hctx->dispatch queue, so 2055 * we can try send it another time in the future. Requests inserted at this 2056 * queue have higher priority. 2057 */ 2058 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 2059 struct request *rq, blk_qc_t *cookie) 2060 { 2061 blk_status_t ret; 2062 int srcu_idx; 2063 2064 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); 2065 2066 hctx_lock(hctx, &srcu_idx); 2067 2068 ret = __blk_mq_try_issue_directly(hctx, rq, cookie, false, true); 2069 if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) 2070 blk_mq_request_bypass_insert(rq, false, true); 2071 else if (ret != BLK_STS_OK) 2072 blk_mq_end_request(rq, ret); 2073 2074 hctx_unlock(hctx, srcu_idx); 2075 } 2076 2077 blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last) 2078 { 2079 blk_status_t ret; 2080 int srcu_idx; 2081 blk_qc_t unused_cookie; 2082 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 2083 2084 hctx_lock(hctx, &srcu_idx); 2085 ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true, last); 2086 hctx_unlock(hctx, srcu_idx); 2087 2088 return ret; 2089 } 2090 2091 void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, 2092 struct list_head *list) 2093 { 2094 int queued = 0; 2095 2096 while (!list_empty(list)) { 2097 blk_status_t ret; 2098 struct request *rq = list_first_entry(list, struct request, 2099 queuelist); 2100 2101 list_del_init(&rq->queuelist); 2102 ret = blk_mq_request_issue_directly(rq, list_empty(list)); 2103 if (ret != BLK_STS_OK) { 2104 if (ret == BLK_STS_RESOURCE || 2105 ret == BLK_STS_DEV_RESOURCE) { 2106 blk_mq_request_bypass_insert(rq, false, 2107 list_empty(list)); 2108 break; 2109 } 2110 blk_mq_end_request(rq, ret); 2111 } else 2112 queued++; 2113 } 2114 2115 /* 2116 * If we didn't flush the entire list, we could have told 2117 * the driver there was more coming, but that turned out to 2118 * be a lie. 2119 */ 2120 if (!list_empty(list) && hctx->queue->mq_ops->commit_rqs && queued) 2121 hctx->queue->mq_ops->commit_rqs(hctx); 2122 } 2123 2124 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) 2125 { 2126 list_add_tail(&rq->queuelist, &plug->mq_list); 2127 plug->rq_count++; 2128 if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) { 2129 struct request *tmp; 2130 2131 tmp = list_first_entry(&plug->mq_list, struct request, 2132 queuelist); 2133 if (tmp->q != rq->q) 2134 plug->multiple_queues = true; 2135 } 2136 } 2137 2138 /** 2139 * blk_mq_submit_bio - Create and send a request to block device. 2140 * @bio: Bio pointer. 2141 * 2142 * Builds up a request structure from @q and @bio and send to the device. The 2143 * request may not be queued directly to hardware if: 2144 * * This request can be merged with another one 2145 * * We want to place request at plug queue for possible future merging 2146 * * There is an IO scheduler active at this queue 2147 * 2148 * It will not queue the request if there is an error with the bio, or at the 2149 * request creation. 2150 * 2151 * Returns: Request queue cookie. 2152 */ 2153 blk_qc_t blk_mq_submit_bio(struct bio *bio) 2154 { 2155 struct request_queue *q = bio->bi_disk->queue; 2156 const int is_sync = op_is_sync(bio->bi_opf); 2157 const int is_flush_fua = op_is_flush(bio->bi_opf); 2158 struct blk_mq_alloc_data data = { 2159 .q = q, 2160 }; 2161 struct request *rq; 2162 struct blk_plug *plug; 2163 struct request *same_queue_rq = NULL; 2164 unsigned int nr_segs; 2165 blk_qc_t cookie; 2166 blk_status_t ret; 2167 2168 blk_queue_bounce(q, &bio); 2169 __blk_queue_split(&bio, &nr_segs); 2170 2171 if (!bio_integrity_prep(bio)) 2172 goto queue_exit; 2173 2174 if (!is_flush_fua && !blk_queue_nomerges(q) && 2175 blk_attempt_plug_merge(q, bio, nr_segs, &same_queue_rq)) 2176 goto queue_exit; 2177 2178 if (blk_mq_sched_bio_merge(q, bio, nr_segs)) 2179 goto queue_exit; 2180 2181 rq_qos_throttle(q, bio); 2182 2183 data.cmd_flags = bio->bi_opf; 2184 rq = __blk_mq_alloc_request(&data); 2185 if (unlikely(!rq)) { 2186 rq_qos_cleanup(q, bio); 2187 if (bio->bi_opf & REQ_NOWAIT) 2188 bio_wouldblock_error(bio); 2189 goto queue_exit; 2190 } 2191 2192 trace_block_getrq(q, bio, bio->bi_opf); 2193 2194 rq_qos_track(q, rq, bio); 2195 2196 cookie = request_to_qc_t(data.hctx, rq); 2197 2198 blk_mq_bio_to_request(rq, bio, nr_segs); 2199 2200 ret = blk_crypto_init_request(rq); 2201 if (ret != BLK_STS_OK) { 2202 bio->bi_status = ret; 2203 bio_endio(bio); 2204 blk_mq_free_request(rq); 2205 return BLK_QC_T_NONE; 2206 } 2207 2208 plug = blk_mq_plug(q, bio); 2209 if (unlikely(is_flush_fua)) { 2210 /* Bypass scheduler for flush requests */ 2211 blk_insert_flush(rq); 2212 blk_mq_run_hw_queue(data.hctx, true); 2213 } else if (plug && (q->nr_hw_queues == 1 || q->mq_ops->commit_rqs || 2214 !blk_queue_nonrot(q))) { 2215 /* 2216 * Use plugging if we have a ->commit_rqs() hook as well, as 2217 * we know the driver uses bd->last in a smart fashion. 2218 * 2219 * Use normal plugging if this disk is slow HDD, as sequential 2220 * IO may benefit a lot from plug merging. 2221 */ 2222 unsigned int request_count = plug->rq_count; 2223 struct request *last = NULL; 2224 2225 if (!request_count) 2226 trace_block_plug(q); 2227 else 2228 last = list_entry_rq(plug->mq_list.prev); 2229 2230 if (request_count >= BLK_MAX_REQUEST_COUNT || (last && 2231 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { 2232 blk_flush_plug_list(plug, false); 2233 trace_block_plug(q); 2234 } 2235 2236 blk_add_rq_to_plug(plug, rq); 2237 } else if (q->elevator) { 2238 /* Insert the request at the IO scheduler queue */ 2239 blk_mq_sched_insert_request(rq, false, true, true); 2240 } else if (plug && !blk_queue_nomerges(q)) { 2241 /* 2242 * We do limited plugging. If the bio can be merged, do that. 2243 * Otherwise the existing request in the plug list will be 2244 * issued. So the plug list will have one request at most 2245 * The plug list might get flushed before this. If that happens, 2246 * the plug list is empty, and same_queue_rq is invalid. 2247 */ 2248 if (list_empty(&plug->mq_list)) 2249 same_queue_rq = NULL; 2250 if (same_queue_rq) { 2251 list_del_init(&same_queue_rq->queuelist); 2252 plug->rq_count--; 2253 } 2254 blk_add_rq_to_plug(plug, rq); 2255 trace_block_plug(q); 2256 2257 if (same_queue_rq) { 2258 data.hctx = same_queue_rq->mq_hctx; 2259 trace_block_unplug(q, 1, true); 2260 blk_mq_try_issue_directly(data.hctx, same_queue_rq, 2261 &cookie); 2262 } 2263 } else if ((q->nr_hw_queues > 1 && is_sync) || 2264 !data.hctx->dispatch_busy) { 2265 /* 2266 * There is no scheduler and we can try to send directly 2267 * to the hardware. 2268 */ 2269 blk_mq_try_issue_directly(data.hctx, rq, &cookie); 2270 } else { 2271 /* Default case. */ 2272 blk_mq_sched_insert_request(rq, false, true, true); 2273 } 2274 2275 return cookie; 2276 queue_exit: 2277 blk_queue_exit(q); 2278 return BLK_QC_T_NONE; 2279 } 2280 EXPORT_SYMBOL_GPL(blk_mq_submit_bio); /* only for request based dm */ 2281 2282 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 2283 unsigned int hctx_idx) 2284 { 2285 struct page *page; 2286 2287 if (tags->rqs && set->ops->exit_request) { 2288 int i; 2289 2290 for (i = 0; i < tags->nr_tags; i++) { 2291 struct request *rq = tags->static_rqs[i]; 2292 2293 if (!rq) 2294 continue; 2295 set->ops->exit_request(set, rq, hctx_idx); 2296 tags->static_rqs[i] = NULL; 2297 } 2298 } 2299 2300 while (!list_empty(&tags->page_list)) { 2301 page = list_first_entry(&tags->page_list, struct page, lru); 2302 list_del_init(&page->lru); 2303 /* 2304 * Remove kmemleak object previously allocated in 2305 * blk_mq_alloc_rqs(). 2306 */ 2307 kmemleak_free(page_address(page)); 2308 __free_pages(page, page->private); 2309 } 2310 } 2311 2312 void blk_mq_free_rq_map(struct blk_mq_tags *tags) 2313 { 2314 kfree(tags->rqs); 2315 tags->rqs = NULL; 2316 kfree(tags->static_rqs); 2317 tags->static_rqs = NULL; 2318 2319 blk_mq_free_tags(tags); 2320 } 2321 2322 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, 2323 unsigned int hctx_idx, 2324 unsigned int nr_tags, 2325 unsigned int reserved_tags) 2326 { 2327 struct blk_mq_tags *tags; 2328 int node; 2329 2330 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx); 2331 if (node == NUMA_NO_NODE) 2332 node = set->numa_node; 2333 2334 tags = blk_mq_init_tags(nr_tags, reserved_tags, node, 2335 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags)); 2336 if (!tags) 2337 return NULL; 2338 2339 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *), 2340 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 2341 node); 2342 if (!tags->rqs) { 2343 blk_mq_free_tags(tags); 2344 return NULL; 2345 } 2346 2347 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *), 2348 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 2349 node); 2350 if (!tags->static_rqs) { 2351 kfree(tags->rqs); 2352 blk_mq_free_tags(tags); 2353 return NULL; 2354 } 2355 2356 return tags; 2357 } 2358 2359 static size_t order_to_size(unsigned int order) 2360 { 2361 return (size_t)PAGE_SIZE << order; 2362 } 2363 2364 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, 2365 unsigned int hctx_idx, int node) 2366 { 2367 int ret; 2368 2369 if (set->ops->init_request) { 2370 ret = set->ops->init_request(set, rq, hctx_idx, node); 2371 if (ret) 2372 return ret; 2373 } 2374 2375 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 2376 return 0; 2377 } 2378 2379 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 2380 unsigned int hctx_idx, unsigned int depth) 2381 { 2382 unsigned int i, j, entries_per_page, max_order = 4; 2383 size_t rq_size, left; 2384 int node; 2385 2386 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx); 2387 if (node == NUMA_NO_NODE) 2388 node = set->numa_node; 2389 2390 INIT_LIST_HEAD(&tags->page_list); 2391 2392 /* 2393 * rq_size is the size of the request plus driver payload, rounded 2394 * to the cacheline size 2395 */ 2396 rq_size = round_up(sizeof(struct request) + set->cmd_size, 2397 cache_line_size()); 2398 left = rq_size * depth; 2399 2400 for (i = 0; i < depth; ) { 2401 int this_order = max_order; 2402 struct page *page; 2403 int to_do; 2404 void *p; 2405 2406 while (this_order && left < order_to_size(this_order - 1)) 2407 this_order--; 2408 2409 do { 2410 page = alloc_pages_node(node, 2411 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, 2412 this_order); 2413 if (page) 2414 break; 2415 if (!this_order--) 2416 break; 2417 if (order_to_size(this_order) < rq_size) 2418 break; 2419 } while (1); 2420 2421 if (!page) 2422 goto fail; 2423 2424 page->private = this_order; 2425 list_add_tail(&page->lru, &tags->page_list); 2426 2427 p = page_address(page); 2428 /* 2429 * Allow kmemleak to scan these pages as they contain pointers 2430 * to additional allocations like via ops->init_request(). 2431 */ 2432 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); 2433 entries_per_page = order_to_size(this_order) / rq_size; 2434 to_do = min(entries_per_page, depth - i); 2435 left -= to_do * rq_size; 2436 for (j = 0; j < to_do; j++) { 2437 struct request *rq = p; 2438 2439 tags->static_rqs[i] = rq; 2440 if (blk_mq_init_request(set, rq, hctx_idx, node)) { 2441 tags->static_rqs[i] = NULL; 2442 goto fail; 2443 } 2444 2445 p += rq_size; 2446 i++; 2447 } 2448 } 2449 return 0; 2450 2451 fail: 2452 blk_mq_free_rqs(set, tags, hctx_idx); 2453 return -ENOMEM; 2454 } 2455 2456 struct rq_iter_data { 2457 struct blk_mq_hw_ctx *hctx; 2458 bool has_rq; 2459 }; 2460 2461 static bool blk_mq_has_request(struct request *rq, void *data, bool reserved) 2462 { 2463 struct rq_iter_data *iter_data = data; 2464 2465 if (rq->mq_hctx != iter_data->hctx) 2466 return true; 2467 iter_data->has_rq = true; 2468 return false; 2469 } 2470 2471 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx) 2472 { 2473 struct blk_mq_tags *tags = hctx->sched_tags ? 2474 hctx->sched_tags : hctx->tags; 2475 struct rq_iter_data data = { 2476 .hctx = hctx, 2477 }; 2478 2479 blk_mq_all_tag_iter(tags, blk_mq_has_request, &data); 2480 return data.has_rq; 2481 } 2482 2483 static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu, 2484 struct blk_mq_hw_ctx *hctx) 2485 { 2486 if (cpumask_next_and(-1, hctx->cpumask, cpu_online_mask) != cpu) 2487 return false; 2488 if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids) 2489 return false; 2490 return true; 2491 } 2492 2493 static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node) 2494 { 2495 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node, 2496 struct blk_mq_hw_ctx, cpuhp_online); 2497 2498 if (!cpumask_test_cpu(cpu, hctx->cpumask) || 2499 !blk_mq_last_cpu_in_hctx(cpu, hctx)) 2500 return 0; 2501 2502 /* 2503 * Prevent new request from being allocated on the current hctx. 2504 * 2505 * The smp_mb__after_atomic() Pairs with the implied barrier in 2506 * test_and_set_bit_lock in sbitmap_get(). Ensures the inactive flag is 2507 * seen once we return from the tag allocator. 2508 */ 2509 set_bit(BLK_MQ_S_INACTIVE, &hctx->state); 2510 smp_mb__after_atomic(); 2511 2512 /* 2513 * Try to grab a reference to the queue and wait for any outstanding 2514 * requests. If we could not grab a reference the queue has been 2515 * frozen and there are no requests. 2516 */ 2517 if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) { 2518 while (blk_mq_hctx_has_requests(hctx)) 2519 msleep(5); 2520 percpu_ref_put(&hctx->queue->q_usage_counter); 2521 } 2522 2523 return 0; 2524 } 2525 2526 static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node) 2527 { 2528 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node, 2529 struct blk_mq_hw_ctx, cpuhp_online); 2530 2531 if (cpumask_test_cpu(cpu, hctx->cpumask)) 2532 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state); 2533 return 0; 2534 } 2535 2536 /* 2537 * 'cpu' is going away. splice any existing rq_list entries from this 2538 * software queue to the hw queue dispatch list, and ensure that it 2539 * gets run. 2540 */ 2541 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) 2542 { 2543 struct blk_mq_hw_ctx *hctx; 2544 struct blk_mq_ctx *ctx; 2545 LIST_HEAD(tmp); 2546 enum hctx_type type; 2547 2548 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead); 2549 if (!cpumask_test_cpu(cpu, hctx->cpumask)) 2550 return 0; 2551 2552 ctx = __blk_mq_get_ctx(hctx->queue, cpu); 2553 type = hctx->type; 2554 2555 spin_lock(&ctx->lock); 2556 if (!list_empty(&ctx->rq_lists[type])) { 2557 list_splice_init(&ctx->rq_lists[type], &tmp); 2558 blk_mq_hctx_clear_pending(hctx, ctx); 2559 } 2560 spin_unlock(&ctx->lock); 2561 2562 if (list_empty(&tmp)) 2563 return 0; 2564 2565 spin_lock(&hctx->lock); 2566 list_splice_tail_init(&tmp, &hctx->dispatch); 2567 spin_unlock(&hctx->lock); 2568 2569 blk_mq_run_hw_queue(hctx, true); 2570 return 0; 2571 } 2572 2573 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) 2574 { 2575 if (!(hctx->flags & BLK_MQ_F_STACKING)) 2576 cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE, 2577 &hctx->cpuhp_online); 2578 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD, 2579 &hctx->cpuhp_dead); 2580 } 2581 2582 /* hctx->ctxs will be freed in queue's release handler */ 2583 static void blk_mq_exit_hctx(struct request_queue *q, 2584 struct blk_mq_tag_set *set, 2585 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 2586 { 2587 if (blk_mq_hw_queue_mapped(hctx)) 2588 blk_mq_tag_idle(hctx); 2589 2590 if (set->ops->exit_request) 2591 set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx); 2592 2593 if (set->ops->exit_hctx) 2594 set->ops->exit_hctx(hctx, hctx_idx); 2595 2596 blk_mq_remove_cpuhp(hctx); 2597 2598 spin_lock(&q->unused_hctx_lock); 2599 list_add(&hctx->hctx_list, &q->unused_hctx_list); 2600 spin_unlock(&q->unused_hctx_lock); 2601 } 2602 2603 static void blk_mq_exit_hw_queues(struct request_queue *q, 2604 struct blk_mq_tag_set *set, int nr_queue) 2605 { 2606 struct blk_mq_hw_ctx *hctx; 2607 unsigned int i; 2608 2609 queue_for_each_hw_ctx(q, hctx, i) { 2610 if (i == nr_queue) 2611 break; 2612 blk_mq_debugfs_unregister_hctx(hctx); 2613 blk_mq_exit_hctx(q, set, hctx, i); 2614 } 2615 } 2616 2617 static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set) 2618 { 2619 int hw_ctx_size = sizeof(struct blk_mq_hw_ctx); 2620 2621 BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu), 2622 __alignof__(struct blk_mq_hw_ctx)) != 2623 sizeof(struct blk_mq_hw_ctx)); 2624 2625 if (tag_set->flags & BLK_MQ_F_BLOCKING) 2626 hw_ctx_size += sizeof(struct srcu_struct); 2627 2628 return hw_ctx_size; 2629 } 2630 2631 static int blk_mq_init_hctx(struct request_queue *q, 2632 struct blk_mq_tag_set *set, 2633 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) 2634 { 2635 hctx->queue_num = hctx_idx; 2636 2637 if (!(hctx->flags & BLK_MQ_F_STACKING)) 2638 cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE, 2639 &hctx->cpuhp_online); 2640 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead); 2641 2642 hctx->tags = set->tags[hctx_idx]; 2643 2644 if (set->ops->init_hctx && 2645 set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) 2646 goto unregister_cpu_notifier; 2647 2648 if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx, 2649 hctx->numa_node)) 2650 goto exit_hctx; 2651 return 0; 2652 2653 exit_hctx: 2654 if (set->ops->exit_hctx) 2655 set->ops->exit_hctx(hctx, hctx_idx); 2656 unregister_cpu_notifier: 2657 blk_mq_remove_cpuhp(hctx); 2658 return -1; 2659 } 2660 2661 static struct blk_mq_hw_ctx * 2662 blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, 2663 int node) 2664 { 2665 struct blk_mq_hw_ctx *hctx; 2666 gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY; 2667 2668 hctx = kzalloc_node(blk_mq_hw_ctx_size(set), gfp, node); 2669 if (!hctx) 2670 goto fail_alloc_hctx; 2671 2672 if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node)) 2673 goto free_hctx; 2674 2675 atomic_set(&hctx->nr_active, 0); 2676 if (node == NUMA_NO_NODE) 2677 node = set->numa_node; 2678 hctx->numa_node = node; 2679 2680 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); 2681 spin_lock_init(&hctx->lock); 2682 INIT_LIST_HEAD(&hctx->dispatch); 2683 hctx->queue = q; 2684 hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED; 2685 2686 INIT_LIST_HEAD(&hctx->hctx_list); 2687 2688 /* 2689 * Allocate space for all possible cpus to avoid allocation at 2690 * runtime 2691 */ 2692 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *), 2693 gfp, node); 2694 if (!hctx->ctxs) 2695 goto free_cpumask; 2696 2697 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), 2698 gfp, node)) 2699 goto free_ctxs; 2700 hctx->nr_ctx = 0; 2701 2702 spin_lock_init(&hctx->dispatch_wait_lock); 2703 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); 2704 INIT_LIST_HEAD(&hctx->dispatch_wait.entry); 2705 2706 hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp); 2707 if (!hctx->fq) 2708 goto free_bitmap; 2709 2710 if (hctx->flags & BLK_MQ_F_BLOCKING) 2711 init_srcu_struct(hctx->srcu); 2712 blk_mq_hctx_kobj_init(hctx); 2713 2714 return hctx; 2715 2716 free_bitmap: 2717 sbitmap_free(&hctx->ctx_map); 2718 free_ctxs: 2719 kfree(hctx->ctxs); 2720 free_cpumask: 2721 free_cpumask_var(hctx->cpumask); 2722 free_hctx: 2723 kfree(hctx); 2724 fail_alloc_hctx: 2725 return NULL; 2726 } 2727 2728 static void blk_mq_init_cpu_queues(struct request_queue *q, 2729 unsigned int nr_hw_queues) 2730 { 2731 struct blk_mq_tag_set *set = q->tag_set; 2732 unsigned int i, j; 2733 2734 for_each_possible_cpu(i) { 2735 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i); 2736 struct blk_mq_hw_ctx *hctx; 2737 int k; 2738 2739 __ctx->cpu = i; 2740 spin_lock_init(&__ctx->lock); 2741 for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++) 2742 INIT_LIST_HEAD(&__ctx->rq_lists[k]); 2743 2744 __ctx->queue = q; 2745 2746 /* 2747 * Set local node, IFF we have more than one hw queue. If 2748 * not, we remain on the home node of the device 2749 */ 2750 for (j = 0; j < set->nr_maps; j++) { 2751 hctx = blk_mq_map_queue_type(q, j, i); 2752 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) 2753 hctx->numa_node = local_memory_node(cpu_to_node(i)); 2754 } 2755 } 2756 } 2757 2758 static bool __blk_mq_alloc_map_and_request(struct blk_mq_tag_set *set, 2759 int hctx_idx) 2760 { 2761 int ret = 0; 2762 2763 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx, 2764 set->queue_depth, set->reserved_tags); 2765 if (!set->tags[hctx_idx]) 2766 return false; 2767 2768 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx, 2769 set->queue_depth); 2770 if (!ret) 2771 return true; 2772 2773 blk_mq_free_rq_map(set->tags[hctx_idx]); 2774 set->tags[hctx_idx] = NULL; 2775 return false; 2776 } 2777 2778 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set, 2779 unsigned int hctx_idx) 2780 { 2781 if (set->tags && set->tags[hctx_idx]) { 2782 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx); 2783 blk_mq_free_rq_map(set->tags[hctx_idx]); 2784 set->tags[hctx_idx] = NULL; 2785 } 2786 } 2787 2788 static void blk_mq_map_swqueue(struct request_queue *q) 2789 { 2790 unsigned int i, j, hctx_idx; 2791 struct blk_mq_hw_ctx *hctx; 2792 struct blk_mq_ctx *ctx; 2793 struct blk_mq_tag_set *set = q->tag_set; 2794 2795 queue_for_each_hw_ctx(q, hctx, i) { 2796 cpumask_clear(hctx->cpumask); 2797 hctx->nr_ctx = 0; 2798 hctx->dispatch_from = NULL; 2799 } 2800 2801 /* 2802 * Map software to hardware queues. 2803 * 2804 * If the cpu isn't present, the cpu is mapped to first hctx. 2805 */ 2806 for_each_possible_cpu(i) { 2807 2808 ctx = per_cpu_ptr(q->queue_ctx, i); 2809 for (j = 0; j < set->nr_maps; j++) { 2810 if (!set->map[j].nr_queues) { 2811 ctx->hctxs[j] = blk_mq_map_queue_type(q, 2812 HCTX_TYPE_DEFAULT, i); 2813 continue; 2814 } 2815 hctx_idx = set->map[j].mq_map[i]; 2816 /* unmapped hw queue can be remapped after CPU topo changed */ 2817 if (!set->tags[hctx_idx] && 2818 !__blk_mq_alloc_map_and_request(set, hctx_idx)) { 2819 /* 2820 * If tags initialization fail for some hctx, 2821 * that hctx won't be brought online. In this 2822 * case, remap the current ctx to hctx[0] which 2823 * is guaranteed to always have tags allocated 2824 */ 2825 set->map[j].mq_map[i] = 0; 2826 } 2827 2828 hctx = blk_mq_map_queue_type(q, j, i); 2829 ctx->hctxs[j] = hctx; 2830 /* 2831 * If the CPU is already set in the mask, then we've 2832 * mapped this one already. This can happen if 2833 * devices share queues across queue maps. 2834 */ 2835 if (cpumask_test_cpu(i, hctx->cpumask)) 2836 continue; 2837 2838 cpumask_set_cpu(i, hctx->cpumask); 2839 hctx->type = j; 2840 ctx->index_hw[hctx->type] = hctx->nr_ctx; 2841 hctx->ctxs[hctx->nr_ctx++] = ctx; 2842 2843 /* 2844 * If the nr_ctx type overflows, we have exceeded the 2845 * amount of sw queues we can support. 2846 */ 2847 BUG_ON(!hctx->nr_ctx); 2848 } 2849 2850 for (; j < HCTX_MAX_TYPES; j++) 2851 ctx->hctxs[j] = blk_mq_map_queue_type(q, 2852 HCTX_TYPE_DEFAULT, i); 2853 } 2854 2855 queue_for_each_hw_ctx(q, hctx, i) { 2856 /* 2857 * If no software queues are mapped to this hardware queue, 2858 * disable it and free the request entries. 2859 */ 2860 if (!hctx->nr_ctx) { 2861 /* Never unmap queue 0. We need it as a 2862 * fallback in case of a new remap fails 2863 * allocation 2864 */ 2865 if (i && set->tags[i]) 2866 blk_mq_free_map_and_requests(set, i); 2867 2868 hctx->tags = NULL; 2869 continue; 2870 } 2871 2872 hctx->tags = set->tags[i]; 2873 WARN_ON(!hctx->tags); 2874 2875 /* 2876 * Set the map size to the number of mapped software queues. 2877 * This is more accurate and more efficient than looping 2878 * over all possibly mapped software queues. 2879 */ 2880 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx); 2881 2882 /* 2883 * Initialize batch roundrobin counts 2884 */ 2885 hctx->next_cpu = blk_mq_first_mapped_cpu(hctx); 2886 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 2887 } 2888 } 2889 2890 /* 2891 * Caller needs to ensure that we're either frozen/quiesced, or that 2892 * the queue isn't live yet. 2893 */ 2894 static void queue_set_hctx_shared(struct request_queue *q, bool shared) 2895 { 2896 struct blk_mq_hw_ctx *hctx; 2897 int i; 2898 2899 queue_for_each_hw_ctx(q, hctx, i) { 2900 if (shared) 2901 hctx->flags |= BLK_MQ_F_TAG_SHARED; 2902 else 2903 hctx->flags &= ~BLK_MQ_F_TAG_SHARED; 2904 } 2905 } 2906 2907 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, 2908 bool shared) 2909 { 2910 struct request_queue *q; 2911 2912 lockdep_assert_held(&set->tag_list_lock); 2913 2914 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2915 blk_mq_freeze_queue(q); 2916 queue_set_hctx_shared(q, shared); 2917 blk_mq_unfreeze_queue(q); 2918 } 2919 } 2920 2921 static void blk_mq_del_queue_tag_set(struct request_queue *q) 2922 { 2923 struct blk_mq_tag_set *set = q->tag_set; 2924 2925 mutex_lock(&set->tag_list_lock); 2926 list_del_rcu(&q->tag_set_list); 2927 if (list_is_singular(&set->tag_list)) { 2928 /* just transitioned to unshared */ 2929 set->flags &= ~BLK_MQ_F_TAG_SHARED; 2930 /* update existing queue */ 2931 blk_mq_update_tag_set_depth(set, false); 2932 } 2933 mutex_unlock(&set->tag_list_lock); 2934 INIT_LIST_HEAD(&q->tag_set_list); 2935 } 2936 2937 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, 2938 struct request_queue *q) 2939 { 2940 mutex_lock(&set->tag_list_lock); 2941 2942 /* 2943 * Check to see if we're transitioning to shared (from 1 to 2 queues). 2944 */ 2945 if (!list_empty(&set->tag_list) && 2946 !(set->flags & BLK_MQ_F_TAG_SHARED)) { 2947 set->flags |= BLK_MQ_F_TAG_SHARED; 2948 /* update existing queue */ 2949 blk_mq_update_tag_set_depth(set, true); 2950 } 2951 if (set->flags & BLK_MQ_F_TAG_SHARED) 2952 queue_set_hctx_shared(q, true); 2953 list_add_tail_rcu(&q->tag_set_list, &set->tag_list); 2954 2955 mutex_unlock(&set->tag_list_lock); 2956 } 2957 2958 /* All allocations will be freed in release handler of q->mq_kobj */ 2959 static int blk_mq_alloc_ctxs(struct request_queue *q) 2960 { 2961 struct blk_mq_ctxs *ctxs; 2962 int cpu; 2963 2964 ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL); 2965 if (!ctxs) 2966 return -ENOMEM; 2967 2968 ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx); 2969 if (!ctxs->queue_ctx) 2970 goto fail; 2971 2972 for_each_possible_cpu(cpu) { 2973 struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu); 2974 ctx->ctxs = ctxs; 2975 } 2976 2977 q->mq_kobj = &ctxs->kobj; 2978 q->queue_ctx = ctxs->queue_ctx; 2979 2980 return 0; 2981 fail: 2982 kfree(ctxs); 2983 return -ENOMEM; 2984 } 2985 2986 /* 2987 * It is the actual release handler for mq, but we do it from 2988 * request queue's release handler for avoiding use-after-free 2989 * and headache because q->mq_kobj shouldn't have been introduced, 2990 * but we can't group ctx/kctx kobj without it. 2991 */ 2992 void blk_mq_release(struct request_queue *q) 2993 { 2994 struct blk_mq_hw_ctx *hctx, *next; 2995 int i; 2996 2997 queue_for_each_hw_ctx(q, hctx, i) 2998 WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list)); 2999 3000 /* all hctx are in .unused_hctx_list now */ 3001 list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) { 3002 list_del_init(&hctx->hctx_list); 3003 kobject_put(&hctx->kobj); 3004 } 3005 3006 kfree(q->queue_hw_ctx); 3007 3008 /* 3009 * release .mq_kobj and sw queue's kobject now because 3010 * both share lifetime with request queue. 3011 */ 3012 blk_mq_sysfs_deinit(q); 3013 } 3014 3015 struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set, 3016 void *queuedata) 3017 { 3018 struct request_queue *uninit_q, *q; 3019 3020 uninit_q = blk_alloc_queue(set->numa_node); 3021 if (!uninit_q) 3022 return ERR_PTR(-ENOMEM); 3023 uninit_q->queuedata = queuedata; 3024 3025 /* 3026 * Initialize the queue without an elevator. device_add_disk() will do 3027 * the initialization. 3028 */ 3029 q = blk_mq_init_allocated_queue(set, uninit_q, false); 3030 if (IS_ERR(q)) 3031 blk_cleanup_queue(uninit_q); 3032 3033 return q; 3034 } 3035 EXPORT_SYMBOL_GPL(blk_mq_init_queue_data); 3036 3037 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) 3038 { 3039 return blk_mq_init_queue_data(set, NULL); 3040 } 3041 EXPORT_SYMBOL(blk_mq_init_queue); 3042 3043 /* 3044 * Helper for setting up a queue with mq ops, given queue depth, and 3045 * the passed in mq ops flags. 3046 */ 3047 struct request_queue *blk_mq_init_sq_queue(struct blk_mq_tag_set *set, 3048 const struct blk_mq_ops *ops, 3049 unsigned int queue_depth, 3050 unsigned int set_flags) 3051 { 3052 struct request_queue *q; 3053 int ret; 3054 3055 memset(set, 0, sizeof(*set)); 3056 set->ops = ops; 3057 set->nr_hw_queues = 1; 3058 set->nr_maps = 1; 3059 set->queue_depth = queue_depth; 3060 set->numa_node = NUMA_NO_NODE; 3061 set->flags = set_flags; 3062 3063 ret = blk_mq_alloc_tag_set(set); 3064 if (ret) 3065 return ERR_PTR(ret); 3066 3067 q = blk_mq_init_queue(set); 3068 if (IS_ERR(q)) { 3069 blk_mq_free_tag_set(set); 3070 return q; 3071 } 3072 3073 return q; 3074 } 3075 EXPORT_SYMBOL(blk_mq_init_sq_queue); 3076 3077 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx( 3078 struct blk_mq_tag_set *set, struct request_queue *q, 3079 int hctx_idx, int node) 3080 { 3081 struct blk_mq_hw_ctx *hctx = NULL, *tmp; 3082 3083 /* reuse dead hctx first */ 3084 spin_lock(&q->unused_hctx_lock); 3085 list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) { 3086 if (tmp->numa_node == node) { 3087 hctx = tmp; 3088 break; 3089 } 3090 } 3091 if (hctx) 3092 list_del_init(&hctx->hctx_list); 3093 spin_unlock(&q->unused_hctx_lock); 3094 3095 if (!hctx) 3096 hctx = blk_mq_alloc_hctx(q, set, node); 3097 if (!hctx) 3098 goto fail; 3099 3100 if (blk_mq_init_hctx(q, set, hctx, hctx_idx)) 3101 goto free_hctx; 3102 3103 return hctx; 3104 3105 free_hctx: 3106 kobject_put(&hctx->kobj); 3107 fail: 3108 return NULL; 3109 } 3110 3111 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, 3112 struct request_queue *q) 3113 { 3114 int i, j, end; 3115 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; 3116 3117 if (q->nr_hw_queues < set->nr_hw_queues) { 3118 struct blk_mq_hw_ctx **new_hctxs; 3119 3120 new_hctxs = kcalloc_node(set->nr_hw_queues, 3121 sizeof(*new_hctxs), GFP_KERNEL, 3122 set->numa_node); 3123 if (!new_hctxs) 3124 return; 3125 if (hctxs) 3126 memcpy(new_hctxs, hctxs, q->nr_hw_queues * 3127 sizeof(*hctxs)); 3128 q->queue_hw_ctx = new_hctxs; 3129 kfree(hctxs); 3130 hctxs = new_hctxs; 3131 } 3132 3133 /* protect against switching io scheduler */ 3134 mutex_lock(&q->sysfs_lock); 3135 for (i = 0; i < set->nr_hw_queues; i++) { 3136 int node; 3137 struct blk_mq_hw_ctx *hctx; 3138 3139 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], i); 3140 /* 3141 * If the hw queue has been mapped to another numa node, 3142 * we need to realloc the hctx. If allocation fails, fallback 3143 * to use the previous one. 3144 */ 3145 if (hctxs[i] && (hctxs[i]->numa_node == node)) 3146 continue; 3147 3148 hctx = blk_mq_alloc_and_init_hctx(set, q, i, node); 3149 if (hctx) { 3150 if (hctxs[i]) 3151 blk_mq_exit_hctx(q, set, hctxs[i], i); 3152 hctxs[i] = hctx; 3153 } else { 3154 if (hctxs[i]) 3155 pr_warn("Allocate new hctx on node %d fails,\ 3156 fallback to previous one on node %d\n", 3157 node, hctxs[i]->numa_node); 3158 else 3159 break; 3160 } 3161 } 3162 /* 3163 * Increasing nr_hw_queues fails. Free the newly allocated 3164 * hctxs and keep the previous q->nr_hw_queues. 3165 */ 3166 if (i != set->nr_hw_queues) { 3167 j = q->nr_hw_queues; 3168 end = i; 3169 } else { 3170 j = i; 3171 end = q->nr_hw_queues; 3172 q->nr_hw_queues = set->nr_hw_queues; 3173 } 3174 3175 for (; j < end; j++) { 3176 struct blk_mq_hw_ctx *hctx = hctxs[j]; 3177 3178 if (hctx) { 3179 if (hctx->tags) 3180 blk_mq_free_map_and_requests(set, j); 3181 blk_mq_exit_hctx(q, set, hctx, j); 3182 hctxs[j] = NULL; 3183 } 3184 } 3185 mutex_unlock(&q->sysfs_lock); 3186 } 3187 3188 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 3189 struct request_queue *q, 3190 bool elevator_init) 3191 { 3192 /* mark the queue as mq asap */ 3193 q->mq_ops = set->ops; 3194 3195 q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn, 3196 blk_mq_poll_stats_bkt, 3197 BLK_MQ_POLL_STATS_BKTS, q); 3198 if (!q->poll_cb) 3199 goto err_exit; 3200 3201 if (blk_mq_alloc_ctxs(q)) 3202 goto err_poll; 3203 3204 /* init q->mq_kobj and sw queues' kobjects */ 3205 blk_mq_sysfs_init(q); 3206 3207 INIT_LIST_HEAD(&q->unused_hctx_list); 3208 spin_lock_init(&q->unused_hctx_lock); 3209 3210 blk_mq_realloc_hw_ctxs(set, q); 3211 if (!q->nr_hw_queues) 3212 goto err_hctxs; 3213 3214 INIT_WORK(&q->timeout_work, blk_mq_timeout_work); 3215 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ); 3216 3217 q->tag_set = set; 3218 3219 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; 3220 if (set->nr_maps > HCTX_TYPE_POLL && 3221 set->map[HCTX_TYPE_POLL].nr_queues) 3222 blk_queue_flag_set(QUEUE_FLAG_POLL, q); 3223 3224 q->sg_reserved_size = INT_MAX; 3225 3226 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); 3227 INIT_LIST_HEAD(&q->requeue_list); 3228 spin_lock_init(&q->requeue_lock); 3229 3230 q->nr_requests = set->queue_depth; 3231 3232 /* 3233 * Default to classic polling 3234 */ 3235 q->poll_nsec = BLK_MQ_POLL_CLASSIC; 3236 3237 blk_mq_init_cpu_queues(q, set->nr_hw_queues); 3238 blk_mq_add_queue_tag_set(set, q); 3239 blk_mq_map_swqueue(q); 3240 3241 if (elevator_init) 3242 elevator_init_mq(q); 3243 3244 return q; 3245 3246 err_hctxs: 3247 kfree(q->queue_hw_ctx); 3248 q->nr_hw_queues = 0; 3249 blk_mq_sysfs_deinit(q); 3250 err_poll: 3251 blk_stat_free_callback(q->poll_cb); 3252 q->poll_cb = NULL; 3253 err_exit: 3254 q->mq_ops = NULL; 3255 return ERR_PTR(-ENOMEM); 3256 } 3257 EXPORT_SYMBOL(blk_mq_init_allocated_queue); 3258 3259 /* tags can _not_ be used after returning from blk_mq_exit_queue */ 3260 void blk_mq_exit_queue(struct request_queue *q) 3261 { 3262 struct blk_mq_tag_set *set = q->tag_set; 3263 3264 blk_mq_del_queue_tag_set(q); 3265 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); 3266 } 3267 3268 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 3269 { 3270 int i; 3271 3272 for (i = 0; i < set->nr_hw_queues; i++) 3273 if (!__blk_mq_alloc_map_and_request(set, i)) 3274 goto out_unwind; 3275 3276 return 0; 3277 3278 out_unwind: 3279 while (--i >= 0) 3280 blk_mq_free_map_and_requests(set, i); 3281 3282 return -ENOMEM; 3283 } 3284 3285 /* 3286 * Allocate the request maps associated with this tag_set. Note that this 3287 * may reduce the depth asked for, if memory is tight. set->queue_depth 3288 * will be updated to reflect the allocated depth. 3289 */ 3290 static int blk_mq_alloc_map_and_requests(struct blk_mq_tag_set *set) 3291 { 3292 unsigned int depth; 3293 int err; 3294 3295 depth = set->queue_depth; 3296 do { 3297 err = __blk_mq_alloc_rq_maps(set); 3298 if (!err) 3299 break; 3300 3301 set->queue_depth >>= 1; 3302 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) { 3303 err = -ENOMEM; 3304 break; 3305 } 3306 } while (set->queue_depth); 3307 3308 if (!set->queue_depth || err) { 3309 pr_err("blk-mq: failed to allocate request map\n"); 3310 return -ENOMEM; 3311 } 3312 3313 if (depth != set->queue_depth) 3314 pr_info("blk-mq: reduced tag depth (%u -> %u)\n", 3315 depth, set->queue_depth); 3316 3317 return 0; 3318 } 3319 3320 static int blk_mq_update_queue_map(struct blk_mq_tag_set *set) 3321 { 3322 /* 3323 * blk_mq_map_queues() and multiple .map_queues() implementations 3324 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the 3325 * number of hardware queues. 3326 */ 3327 if (set->nr_maps == 1) 3328 set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues; 3329 3330 if (set->ops->map_queues && !is_kdump_kernel()) { 3331 int i; 3332 3333 /* 3334 * transport .map_queues is usually done in the following 3335 * way: 3336 * 3337 * for (queue = 0; queue < set->nr_hw_queues; queue++) { 3338 * mask = get_cpu_mask(queue) 3339 * for_each_cpu(cpu, mask) 3340 * set->map[x].mq_map[cpu] = queue; 3341 * } 3342 * 3343 * When we need to remap, the table has to be cleared for 3344 * killing stale mapping since one CPU may not be mapped 3345 * to any hw queue. 3346 */ 3347 for (i = 0; i < set->nr_maps; i++) 3348 blk_mq_clear_mq_map(&set->map[i]); 3349 3350 return set->ops->map_queues(set); 3351 } else { 3352 BUG_ON(set->nr_maps > 1); 3353 return blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 3354 } 3355 } 3356 3357 static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set, 3358 int cur_nr_hw_queues, int new_nr_hw_queues) 3359 { 3360 struct blk_mq_tags **new_tags; 3361 3362 if (cur_nr_hw_queues >= new_nr_hw_queues) 3363 return 0; 3364 3365 new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *), 3366 GFP_KERNEL, set->numa_node); 3367 if (!new_tags) 3368 return -ENOMEM; 3369 3370 if (set->tags) 3371 memcpy(new_tags, set->tags, cur_nr_hw_queues * 3372 sizeof(*set->tags)); 3373 kfree(set->tags); 3374 set->tags = new_tags; 3375 set->nr_hw_queues = new_nr_hw_queues; 3376 3377 return 0; 3378 } 3379 3380 /* 3381 * Alloc a tag set to be associated with one or more request queues. 3382 * May fail with EINVAL for various error conditions. May adjust the 3383 * requested depth down, if it's too large. In that case, the set 3384 * value will be stored in set->queue_depth. 3385 */ 3386 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) 3387 { 3388 int i, ret; 3389 3390 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS); 3391 3392 if (!set->nr_hw_queues) 3393 return -EINVAL; 3394 if (!set->queue_depth) 3395 return -EINVAL; 3396 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) 3397 return -EINVAL; 3398 3399 if (!set->ops->queue_rq) 3400 return -EINVAL; 3401 3402 if (!set->ops->get_budget ^ !set->ops->put_budget) 3403 return -EINVAL; 3404 3405 if (set->queue_depth > BLK_MQ_MAX_DEPTH) { 3406 pr_info("blk-mq: reduced tag depth to %u\n", 3407 BLK_MQ_MAX_DEPTH); 3408 set->queue_depth = BLK_MQ_MAX_DEPTH; 3409 } 3410 3411 if (!set->nr_maps) 3412 set->nr_maps = 1; 3413 else if (set->nr_maps > HCTX_MAX_TYPES) 3414 return -EINVAL; 3415 3416 /* 3417 * If a crashdump is active, then we are potentially in a very 3418 * memory constrained environment. Limit us to 1 queue and 3419 * 64 tags to prevent using too much memory. 3420 */ 3421 if (is_kdump_kernel()) { 3422 set->nr_hw_queues = 1; 3423 set->nr_maps = 1; 3424 set->queue_depth = min(64U, set->queue_depth); 3425 } 3426 /* 3427 * There is no use for more h/w queues than cpus if we just have 3428 * a single map 3429 */ 3430 if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids) 3431 set->nr_hw_queues = nr_cpu_ids; 3432 3433 if (blk_mq_realloc_tag_set_tags(set, 0, set->nr_hw_queues) < 0) 3434 return -ENOMEM; 3435 3436 ret = -ENOMEM; 3437 for (i = 0; i < set->nr_maps; i++) { 3438 set->map[i].mq_map = kcalloc_node(nr_cpu_ids, 3439 sizeof(set->map[i].mq_map[0]), 3440 GFP_KERNEL, set->numa_node); 3441 if (!set->map[i].mq_map) 3442 goto out_free_mq_map; 3443 set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues; 3444 } 3445 3446 ret = blk_mq_update_queue_map(set); 3447 if (ret) 3448 goto out_free_mq_map; 3449 3450 ret = blk_mq_alloc_map_and_requests(set); 3451 if (ret) 3452 goto out_free_mq_map; 3453 3454 mutex_init(&set->tag_list_lock); 3455 INIT_LIST_HEAD(&set->tag_list); 3456 3457 return 0; 3458 3459 out_free_mq_map: 3460 for (i = 0; i < set->nr_maps; i++) { 3461 kfree(set->map[i].mq_map); 3462 set->map[i].mq_map = NULL; 3463 } 3464 kfree(set->tags); 3465 set->tags = NULL; 3466 return ret; 3467 } 3468 EXPORT_SYMBOL(blk_mq_alloc_tag_set); 3469 3470 void blk_mq_free_tag_set(struct blk_mq_tag_set *set) 3471 { 3472 int i, j; 3473 3474 for (i = 0; i < set->nr_hw_queues; i++) 3475 blk_mq_free_map_and_requests(set, i); 3476 3477 for (j = 0; j < set->nr_maps; j++) { 3478 kfree(set->map[j].mq_map); 3479 set->map[j].mq_map = NULL; 3480 } 3481 3482 kfree(set->tags); 3483 set->tags = NULL; 3484 } 3485 EXPORT_SYMBOL(blk_mq_free_tag_set); 3486 3487 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) 3488 { 3489 struct blk_mq_tag_set *set = q->tag_set; 3490 struct blk_mq_hw_ctx *hctx; 3491 int i, ret; 3492 3493 if (!set) 3494 return -EINVAL; 3495 3496 if (q->nr_requests == nr) 3497 return 0; 3498 3499 blk_mq_freeze_queue(q); 3500 blk_mq_quiesce_queue(q); 3501 3502 ret = 0; 3503 queue_for_each_hw_ctx(q, hctx, i) { 3504 if (!hctx->tags) 3505 continue; 3506 /* 3507 * If we're using an MQ scheduler, just update the scheduler 3508 * queue depth. This is similar to what the old code would do. 3509 */ 3510 if (!hctx->sched_tags) { 3511 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr, 3512 false); 3513 } else { 3514 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags, 3515 nr, true); 3516 } 3517 if (ret) 3518 break; 3519 if (q->elevator && q->elevator->type->ops.depth_updated) 3520 q->elevator->type->ops.depth_updated(hctx); 3521 } 3522 3523 if (!ret) 3524 q->nr_requests = nr; 3525 3526 blk_mq_unquiesce_queue(q); 3527 blk_mq_unfreeze_queue(q); 3528 3529 return ret; 3530 } 3531 3532 /* 3533 * request_queue and elevator_type pair. 3534 * It is just used by __blk_mq_update_nr_hw_queues to cache 3535 * the elevator_type associated with a request_queue. 3536 */ 3537 struct blk_mq_qe_pair { 3538 struct list_head node; 3539 struct request_queue *q; 3540 struct elevator_type *type; 3541 }; 3542 3543 /* 3544 * Cache the elevator_type in qe pair list and switch the 3545 * io scheduler to 'none' 3546 */ 3547 static bool blk_mq_elv_switch_none(struct list_head *head, 3548 struct request_queue *q) 3549 { 3550 struct blk_mq_qe_pair *qe; 3551 3552 if (!q->elevator) 3553 return true; 3554 3555 qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY); 3556 if (!qe) 3557 return false; 3558 3559 INIT_LIST_HEAD(&qe->node); 3560 qe->q = q; 3561 qe->type = q->elevator->type; 3562 list_add(&qe->node, head); 3563 3564 mutex_lock(&q->sysfs_lock); 3565 /* 3566 * After elevator_switch_mq, the previous elevator_queue will be 3567 * released by elevator_release. The reference of the io scheduler 3568 * module get by elevator_get will also be put. So we need to get 3569 * a reference of the io scheduler module here to prevent it to be 3570 * removed. 3571 */ 3572 __module_get(qe->type->elevator_owner); 3573 elevator_switch_mq(q, NULL); 3574 mutex_unlock(&q->sysfs_lock); 3575 3576 return true; 3577 } 3578 3579 static void blk_mq_elv_switch_back(struct list_head *head, 3580 struct request_queue *q) 3581 { 3582 struct blk_mq_qe_pair *qe; 3583 struct elevator_type *t = NULL; 3584 3585 list_for_each_entry(qe, head, node) 3586 if (qe->q == q) { 3587 t = qe->type; 3588 break; 3589 } 3590 3591 if (!t) 3592 return; 3593 3594 list_del(&qe->node); 3595 kfree(qe); 3596 3597 mutex_lock(&q->sysfs_lock); 3598 elevator_switch_mq(q, t); 3599 mutex_unlock(&q->sysfs_lock); 3600 } 3601 3602 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, 3603 int nr_hw_queues) 3604 { 3605 struct request_queue *q; 3606 LIST_HEAD(head); 3607 int prev_nr_hw_queues; 3608 3609 lockdep_assert_held(&set->tag_list_lock); 3610 3611 if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids) 3612 nr_hw_queues = nr_cpu_ids; 3613 if (nr_hw_queues < 1) 3614 return; 3615 if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues) 3616 return; 3617 3618 list_for_each_entry(q, &set->tag_list, tag_set_list) 3619 blk_mq_freeze_queue(q); 3620 /* 3621 * Switch IO scheduler to 'none', cleaning up the data associated 3622 * with the previous scheduler. We will switch back once we are done 3623 * updating the new sw to hw queue mappings. 3624 */ 3625 list_for_each_entry(q, &set->tag_list, tag_set_list) 3626 if (!blk_mq_elv_switch_none(&head, q)) 3627 goto switch_back; 3628 3629 list_for_each_entry(q, &set->tag_list, tag_set_list) { 3630 blk_mq_debugfs_unregister_hctxs(q); 3631 blk_mq_sysfs_unregister(q); 3632 } 3633 3634 prev_nr_hw_queues = set->nr_hw_queues; 3635 if (blk_mq_realloc_tag_set_tags(set, set->nr_hw_queues, nr_hw_queues) < 3636 0) 3637 goto reregister; 3638 3639 set->nr_hw_queues = nr_hw_queues; 3640 fallback: 3641 blk_mq_update_queue_map(set); 3642 list_for_each_entry(q, &set->tag_list, tag_set_list) { 3643 blk_mq_realloc_hw_ctxs(set, q); 3644 if (q->nr_hw_queues != set->nr_hw_queues) { 3645 pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n", 3646 nr_hw_queues, prev_nr_hw_queues); 3647 set->nr_hw_queues = prev_nr_hw_queues; 3648 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 3649 goto fallback; 3650 } 3651 blk_mq_map_swqueue(q); 3652 } 3653 3654 reregister: 3655 list_for_each_entry(q, &set->tag_list, tag_set_list) { 3656 blk_mq_sysfs_register(q); 3657 blk_mq_debugfs_register_hctxs(q); 3658 } 3659 3660 switch_back: 3661 list_for_each_entry(q, &set->tag_list, tag_set_list) 3662 blk_mq_elv_switch_back(&head, q); 3663 3664 list_for_each_entry(q, &set->tag_list, tag_set_list) 3665 blk_mq_unfreeze_queue(q); 3666 } 3667 3668 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) 3669 { 3670 mutex_lock(&set->tag_list_lock); 3671 __blk_mq_update_nr_hw_queues(set, nr_hw_queues); 3672 mutex_unlock(&set->tag_list_lock); 3673 } 3674 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); 3675 3676 /* Enable polling stats and return whether they were already enabled. */ 3677 static bool blk_poll_stats_enable(struct request_queue *q) 3678 { 3679 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 3680 blk_queue_flag_test_and_set(QUEUE_FLAG_POLL_STATS, q)) 3681 return true; 3682 blk_stat_add_callback(q, q->poll_cb); 3683 return false; 3684 } 3685 3686 static void blk_mq_poll_stats_start(struct request_queue *q) 3687 { 3688 /* 3689 * We don't arm the callback if polling stats are not enabled or the 3690 * callback is already active. 3691 */ 3692 if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 3693 blk_stat_is_active(q->poll_cb)) 3694 return; 3695 3696 blk_stat_activate_msecs(q->poll_cb, 100); 3697 } 3698 3699 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb) 3700 { 3701 struct request_queue *q = cb->data; 3702 int bucket; 3703 3704 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) { 3705 if (cb->stat[bucket].nr_samples) 3706 q->poll_stat[bucket] = cb->stat[bucket]; 3707 } 3708 } 3709 3710 static unsigned long blk_mq_poll_nsecs(struct request_queue *q, 3711 struct request *rq) 3712 { 3713 unsigned long ret = 0; 3714 int bucket; 3715 3716 /* 3717 * If stats collection isn't on, don't sleep but turn it on for 3718 * future users 3719 */ 3720 if (!blk_poll_stats_enable(q)) 3721 return 0; 3722 3723 /* 3724 * As an optimistic guess, use half of the mean service time 3725 * for this type of request. We can (and should) make this smarter. 3726 * For instance, if the completion latencies are tight, we can 3727 * get closer than just half the mean. This is especially 3728 * important on devices where the completion latencies are longer 3729 * than ~10 usec. We do use the stats for the relevant IO size 3730 * if available which does lead to better estimates. 3731 */ 3732 bucket = blk_mq_poll_stats_bkt(rq); 3733 if (bucket < 0) 3734 return ret; 3735 3736 if (q->poll_stat[bucket].nr_samples) 3737 ret = (q->poll_stat[bucket].mean + 1) / 2; 3738 3739 return ret; 3740 } 3741 3742 static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, 3743 struct request *rq) 3744 { 3745 struct hrtimer_sleeper hs; 3746 enum hrtimer_mode mode; 3747 unsigned int nsecs; 3748 ktime_t kt; 3749 3750 if (rq->rq_flags & RQF_MQ_POLL_SLEPT) 3751 return false; 3752 3753 /* 3754 * If we get here, hybrid polling is enabled. Hence poll_nsec can be: 3755 * 3756 * 0: use half of prev avg 3757 * >0: use this specific value 3758 */ 3759 if (q->poll_nsec > 0) 3760 nsecs = q->poll_nsec; 3761 else 3762 nsecs = blk_mq_poll_nsecs(q, rq); 3763 3764 if (!nsecs) 3765 return false; 3766 3767 rq->rq_flags |= RQF_MQ_POLL_SLEPT; 3768 3769 /* 3770 * This will be replaced with the stats tracking code, using 3771 * 'avg_completion_time / 2' as the pre-sleep target. 3772 */ 3773 kt = nsecs; 3774 3775 mode = HRTIMER_MODE_REL; 3776 hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode); 3777 hrtimer_set_expires(&hs.timer, kt); 3778 3779 do { 3780 if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE) 3781 break; 3782 set_current_state(TASK_UNINTERRUPTIBLE); 3783 hrtimer_sleeper_start_expires(&hs, mode); 3784 if (hs.task) 3785 io_schedule(); 3786 hrtimer_cancel(&hs.timer); 3787 mode = HRTIMER_MODE_ABS; 3788 } while (hs.task && !signal_pending(current)); 3789 3790 __set_current_state(TASK_RUNNING); 3791 destroy_hrtimer_on_stack(&hs.timer); 3792 return true; 3793 } 3794 3795 static bool blk_mq_poll_hybrid(struct request_queue *q, 3796 struct blk_mq_hw_ctx *hctx, blk_qc_t cookie) 3797 { 3798 struct request *rq; 3799 3800 if (q->poll_nsec == BLK_MQ_POLL_CLASSIC) 3801 return false; 3802 3803 if (!blk_qc_t_is_internal(cookie)) 3804 rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); 3805 else { 3806 rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie)); 3807 /* 3808 * With scheduling, if the request has completed, we'll 3809 * get a NULL return here, as we clear the sched tag when 3810 * that happens. The request still remains valid, like always, 3811 * so we should be safe with just the NULL check. 3812 */ 3813 if (!rq) 3814 return false; 3815 } 3816 3817 return blk_mq_poll_hybrid_sleep(q, rq); 3818 } 3819 3820 /** 3821 * blk_poll - poll for IO completions 3822 * @q: the queue 3823 * @cookie: cookie passed back at IO submission time 3824 * @spin: whether to spin for completions 3825 * 3826 * Description: 3827 * Poll for completions on the passed in queue. Returns number of 3828 * completed entries found. If @spin is true, then blk_poll will continue 3829 * looping until at least one completion is found, unless the task is 3830 * otherwise marked running (or we need to reschedule). 3831 */ 3832 int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin) 3833 { 3834 struct blk_mq_hw_ctx *hctx; 3835 long state; 3836 3837 if (!blk_qc_t_valid(cookie) || 3838 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 3839 return 0; 3840 3841 if (current->plug) 3842 blk_flush_plug_list(current->plug, false); 3843 3844 hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; 3845 3846 /* 3847 * If we sleep, have the caller restart the poll loop to reset 3848 * the state. Like for the other success return cases, the 3849 * caller is responsible for checking if the IO completed. If 3850 * the IO isn't complete, we'll get called again and will go 3851 * straight to the busy poll loop. 3852 */ 3853 if (blk_mq_poll_hybrid(q, hctx, cookie)) 3854 return 1; 3855 3856 hctx->poll_considered++; 3857 3858 state = current->state; 3859 do { 3860 int ret; 3861 3862 hctx->poll_invoked++; 3863 3864 ret = q->mq_ops->poll(hctx); 3865 if (ret > 0) { 3866 hctx->poll_success++; 3867 __set_current_state(TASK_RUNNING); 3868 return ret; 3869 } 3870 3871 if (signal_pending_state(state, current)) 3872 __set_current_state(TASK_RUNNING); 3873 3874 if (current->state == TASK_RUNNING) 3875 return 1; 3876 if (ret < 0 || !spin) 3877 break; 3878 cpu_relax(); 3879 } while (!need_resched()); 3880 3881 __set_current_state(TASK_RUNNING); 3882 return 0; 3883 } 3884 EXPORT_SYMBOL_GPL(blk_poll); 3885 3886 unsigned int blk_mq_rq_cpu(struct request *rq) 3887 { 3888 return rq->mq_ctx->cpu; 3889 } 3890 EXPORT_SYMBOL(blk_mq_rq_cpu); 3891 3892 static int __init blk_mq_init(void) 3893 { 3894 int i; 3895 3896 for_each_possible_cpu(i) 3897 INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i)); 3898 open_softirq(BLOCK_SOFTIRQ, blk_done_softirq); 3899 3900 cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD, 3901 "block/softirq:dead", NULL, 3902 blk_softirq_cpu_dead); 3903 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, 3904 blk_mq_hctx_notify_dead); 3905 cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online", 3906 blk_mq_hctx_notify_online, 3907 blk_mq_hctx_notify_offline); 3908 return 0; 3909 } 3910 subsys_initcall(blk_mq_init); 3911