1 /* 2 * Block multiqueue core code 3 * 4 * Copyright (C) 2013-2014 Jens Axboe 5 * Copyright (C) 2013-2014 Christoph Hellwig 6 */ 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/backing-dev.h> 10 #include <linux/bio.h> 11 #include <linux/blkdev.h> 12 #include <linux/kmemleak.h> 13 #include <linux/mm.h> 14 #include <linux/init.h> 15 #include <linux/slab.h> 16 #include <linux/workqueue.h> 17 #include <linux/smp.h> 18 #include <linux/llist.h> 19 #include <linux/list_sort.h> 20 #include <linux/cpu.h> 21 #include <linux/cache.h> 22 #include <linux/sched/sysctl.h> 23 #include <linux/sched/topology.h> 24 #include <linux/sched/signal.h> 25 #include <linux/delay.h> 26 #include <linux/crash_dump.h> 27 #include <linux/prefetch.h> 28 29 #include <trace/events/block.h> 30 31 #include <linux/blk-mq.h> 32 #include "blk.h" 33 #include "blk-mq.h" 34 #include "blk-mq-tag.h" 35 #include "blk-stat.h" 36 #include "blk-wbt.h" 37 #include "blk-mq-sched.h" 38 39 static DEFINE_MUTEX(all_q_mutex); 40 static LIST_HEAD(all_q_list); 41 42 /* 43 * Check if any of the ctx's have pending work in this hardware queue 44 */ 45 bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) 46 { 47 return sbitmap_any_bit_set(&hctx->ctx_map) || 48 !list_empty_careful(&hctx->dispatch) || 49 blk_mq_sched_has_work(hctx); 50 } 51 52 /* 53 * Mark this ctx as having pending work in this hardware queue 54 */ 55 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, 56 struct blk_mq_ctx *ctx) 57 { 58 if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw)) 59 sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw); 60 } 61 62 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, 63 struct blk_mq_ctx *ctx) 64 { 65 sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw); 66 } 67 68 void blk_mq_freeze_queue_start(struct request_queue *q) 69 { 70 int freeze_depth; 71 72 freeze_depth = atomic_inc_return(&q->mq_freeze_depth); 73 if (freeze_depth == 1) { 74 percpu_ref_kill(&q->q_usage_counter); 75 blk_mq_run_hw_queues(q, false); 76 } 77 } 78 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_start); 79 80 void blk_mq_freeze_queue_wait(struct request_queue *q) 81 { 82 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter)); 83 } 84 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait); 85 86 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, 87 unsigned long timeout) 88 { 89 return wait_event_timeout(q->mq_freeze_wq, 90 percpu_ref_is_zero(&q->q_usage_counter), 91 timeout); 92 } 93 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout); 94 95 /* 96 * Guarantee no request is in use, so we can change any data structure of 97 * the queue afterward. 98 */ 99 void blk_freeze_queue(struct request_queue *q) 100 { 101 /* 102 * In the !blk_mq case we are only calling this to kill the 103 * q_usage_counter, otherwise this increases the freeze depth 104 * and waits for it to return to zero. For this reason there is 105 * no blk_unfreeze_queue(), and blk_freeze_queue() is not 106 * exported to drivers as the only user for unfreeze is blk_mq. 107 */ 108 blk_mq_freeze_queue_start(q); 109 blk_mq_freeze_queue_wait(q); 110 } 111 112 void blk_mq_freeze_queue(struct request_queue *q) 113 { 114 /* 115 * ...just an alias to keep freeze and unfreeze actions balanced 116 * in the blk_mq_* namespace 117 */ 118 blk_freeze_queue(q); 119 } 120 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue); 121 122 void blk_mq_unfreeze_queue(struct request_queue *q) 123 { 124 int freeze_depth; 125 126 freeze_depth = atomic_dec_return(&q->mq_freeze_depth); 127 WARN_ON_ONCE(freeze_depth < 0); 128 if (!freeze_depth) { 129 percpu_ref_reinit(&q->q_usage_counter); 130 wake_up_all(&q->mq_freeze_wq); 131 } 132 } 133 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); 134 135 /** 136 * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished 137 * @q: request queue. 138 * 139 * Note: this function does not prevent that the struct request end_io() 140 * callback function is invoked. Additionally, it is not prevented that 141 * new queue_rq() calls occur unless the queue has been stopped first. 142 */ 143 void blk_mq_quiesce_queue(struct request_queue *q) 144 { 145 struct blk_mq_hw_ctx *hctx; 146 unsigned int i; 147 bool rcu = false; 148 149 blk_mq_stop_hw_queues(q); 150 151 queue_for_each_hw_ctx(q, hctx, i) { 152 if (hctx->flags & BLK_MQ_F_BLOCKING) 153 synchronize_srcu(&hctx->queue_rq_srcu); 154 else 155 rcu = true; 156 } 157 if (rcu) 158 synchronize_rcu(); 159 } 160 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue); 161 162 void blk_mq_wake_waiters(struct request_queue *q) 163 { 164 struct blk_mq_hw_ctx *hctx; 165 unsigned int i; 166 167 queue_for_each_hw_ctx(q, hctx, i) 168 if (blk_mq_hw_queue_mapped(hctx)) 169 blk_mq_tag_wakeup_all(hctx->tags, true); 170 171 /* 172 * If we are called because the queue has now been marked as 173 * dying, we need to ensure that processes currently waiting on 174 * the queue are notified as well. 175 */ 176 wake_up_all(&q->mq_freeze_wq); 177 } 178 179 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) 180 { 181 return blk_mq_has_free_tags(hctx->tags); 182 } 183 EXPORT_SYMBOL(blk_mq_can_queue); 184 185 void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, 186 struct request *rq, unsigned int op) 187 { 188 INIT_LIST_HEAD(&rq->queuelist); 189 /* csd/requeue_work/fifo_time is initialized before use */ 190 rq->q = q; 191 rq->mq_ctx = ctx; 192 rq->cmd_flags = op; 193 if (blk_queue_io_stat(q)) 194 rq->rq_flags |= RQF_IO_STAT; 195 /* do not touch atomic flags, it needs atomic ops against the timer */ 196 rq->cpu = -1; 197 INIT_HLIST_NODE(&rq->hash); 198 RB_CLEAR_NODE(&rq->rb_node); 199 rq->rq_disk = NULL; 200 rq->part = NULL; 201 rq->start_time = jiffies; 202 #ifdef CONFIG_BLK_CGROUP 203 rq->rl = NULL; 204 set_start_time_ns(rq); 205 rq->io_start_time_ns = 0; 206 #endif 207 rq->nr_phys_segments = 0; 208 #if defined(CONFIG_BLK_DEV_INTEGRITY) 209 rq->nr_integrity_segments = 0; 210 #endif 211 rq->special = NULL; 212 /* tag was already set */ 213 rq->errors = 0; 214 rq->extra_len = 0; 215 216 INIT_LIST_HEAD(&rq->timeout_list); 217 rq->timeout = 0; 218 219 rq->end_io = NULL; 220 rq->end_io_data = NULL; 221 rq->next_rq = NULL; 222 223 ctx->rq_dispatched[op_is_sync(op)]++; 224 } 225 EXPORT_SYMBOL_GPL(blk_mq_rq_ctx_init); 226 227 struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data, 228 unsigned int op) 229 { 230 struct request *rq; 231 unsigned int tag; 232 233 tag = blk_mq_get_tag(data); 234 if (tag != BLK_MQ_TAG_FAIL) { 235 struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 236 237 rq = tags->static_rqs[tag]; 238 239 if (data->flags & BLK_MQ_REQ_INTERNAL) { 240 rq->tag = -1; 241 rq->internal_tag = tag; 242 } else { 243 if (blk_mq_tag_busy(data->hctx)) { 244 rq->rq_flags = RQF_MQ_INFLIGHT; 245 atomic_inc(&data->hctx->nr_active); 246 } 247 rq->tag = tag; 248 rq->internal_tag = -1; 249 data->hctx->tags->rqs[rq->tag] = rq; 250 } 251 252 blk_mq_rq_ctx_init(data->q, data->ctx, rq, op); 253 return rq; 254 } 255 256 return NULL; 257 } 258 EXPORT_SYMBOL_GPL(__blk_mq_alloc_request); 259 260 struct request *blk_mq_alloc_request(struct request_queue *q, int rw, 261 unsigned int flags) 262 { 263 struct blk_mq_alloc_data alloc_data = { .flags = flags }; 264 struct request *rq; 265 int ret; 266 267 ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT); 268 if (ret) 269 return ERR_PTR(ret); 270 271 rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data); 272 273 blk_mq_put_ctx(alloc_data.ctx); 274 blk_queue_exit(q); 275 276 if (!rq) 277 return ERR_PTR(-EWOULDBLOCK); 278 279 rq->__data_len = 0; 280 rq->__sector = (sector_t) -1; 281 rq->bio = rq->biotail = NULL; 282 return rq; 283 } 284 EXPORT_SYMBOL(blk_mq_alloc_request); 285 286 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw, 287 unsigned int flags, unsigned int hctx_idx) 288 { 289 struct blk_mq_alloc_data alloc_data = { .flags = flags }; 290 struct request *rq; 291 unsigned int cpu; 292 int ret; 293 294 /* 295 * If the tag allocator sleeps we could get an allocation for a 296 * different hardware context. No need to complicate the low level 297 * allocator for this for the rare use case of a command tied to 298 * a specific queue. 299 */ 300 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT))) 301 return ERR_PTR(-EINVAL); 302 303 if (hctx_idx >= q->nr_hw_queues) 304 return ERR_PTR(-EIO); 305 306 ret = blk_queue_enter(q, true); 307 if (ret) 308 return ERR_PTR(ret); 309 310 /* 311 * Check if the hardware context is actually mapped to anything. 312 * If not tell the caller that it should skip this queue. 313 */ 314 alloc_data.hctx = q->queue_hw_ctx[hctx_idx]; 315 if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) { 316 blk_queue_exit(q); 317 return ERR_PTR(-EXDEV); 318 } 319 cpu = cpumask_first(alloc_data.hctx->cpumask); 320 alloc_data.ctx = __blk_mq_get_ctx(q, cpu); 321 322 rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data); 323 324 blk_mq_put_ctx(alloc_data.ctx); 325 blk_queue_exit(q); 326 327 if (!rq) 328 return ERR_PTR(-EWOULDBLOCK); 329 330 return rq; 331 } 332 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); 333 334 void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 335 struct request *rq) 336 { 337 const int sched_tag = rq->internal_tag; 338 struct request_queue *q = rq->q; 339 340 if (rq->rq_flags & RQF_MQ_INFLIGHT) 341 atomic_dec(&hctx->nr_active); 342 343 wbt_done(q->rq_wb, &rq->issue_stat); 344 rq->rq_flags = 0; 345 346 clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags); 347 clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags); 348 if (rq->tag != -1) 349 blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag); 350 if (sched_tag != -1) 351 blk_mq_sched_completed_request(hctx, rq); 352 blk_mq_sched_restart_queues(hctx); 353 blk_queue_exit(q); 354 } 355 356 static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx, 357 struct request *rq) 358 { 359 struct blk_mq_ctx *ctx = rq->mq_ctx; 360 361 ctx->rq_completed[rq_is_sync(rq)]++; 362 __blk_mq_finish_request(hctx, ctx, rq); 363 } 364 365 void blk_mq_finish_request(struct request *rq) 366 { 367 blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq); 368 } 369 370 void blk_mq_free_request(struct request *rq) 371 { 372 blk_mq_sched_put_request(rq); 373 } 374 EXPORT_SYMBOL_GPL(blk_mq_free_request); 375 376 inline void __blk_mq_end_request(struct request *rq, int error) 377 { 378 blk_account_io_done(rq); 379 380 if (rq->end_io) { 381 wbt_done(rq->q->rq_wb, &rq->issue_stat); 382 rq->end_io(rq, error); 383 } else { 384 if (unlikely(blk_bidi_rq(rq))) 385 blk_mq_free_request(rq->next_rq); 386 blk_mq_free_request(rq); 387 } 388 } 389 EXPORT_SYMBOL(__blk_mq_end_request); 390 391 void blk_mq_end_request(struct request *rq, int error) 392 { 393 if (blk_update_request(rq, error, blk_rq_bytes(rq))) 394 BUG(); 395 __blk_mq_end_request(rq, error); 396 } 397 EXPORT_SYMBOL(blk_mq_end_request); 398 399 static void __blk_mq_complete_request_remote(void *data) 400 { 401 struct request *rq = data; 402 403 rq->q->softirq_done_fn(rq); 404 } 405 406 static void blk_mq_ipi_complete_request(struct request *rq) 407 { 408 struct blk_mq_ctx *ctx = rq->mq_ctx; 409 bool shared = false; 410 int cpu; 411 412 if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) { 413 rq->q->softirq_done_fn(rq); 414 return; 415 } 416 417 cpu = get_cpu(); 418 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags)) 419 shared = cpus_share_cache(cpu, ctx->cpu); 420 421 if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { 422 rq->csd.func = __blk_mq_complete_request_remote; 423 rq->csd.info = rq; 424 rq->csd.flags = 0; 425 smp_call_function_single_async(ctx->cpu, &rq->csd); 426 } else { 427 rq->q->softirq_done_fn(rq); 428 } 429 put_cpu(); 430 } 431 432 static void blk_mq_stat_add(struct request *rq) 433 { 434 if (rq->rq_flags & RQF_STATS) { 435 /* 436 * We could rq->mq_ctx here, but there's less of a risk 437 * of races if we have the completion event add the stats 438 * to the local software queue. 439 */ 440 struct blk_mq_ctx *ctx; 441 442 ctx = __blk_mq_get_ctx(rq->q, raw_smp_processor_id()); 443 blk_stat_add(&ctx->stat[rq_data_dir(rq)], rq); 444 } 445 } 446 447 static void __blk_mq_complete_request(struct request *rq) 448 { 449 struct request_queue *q = rq->q; 450 451 blk_mq_stat_add(rq); 452 453 if (!q->softirq_done_fn) 454 blk_mq_end_request(rq, rq->errors); 455 else 456 blk_mq_ipi_complete_request(rq); 457 } 458 459 /** 460 * blk_mq_complete_request - end I/O on a request 461 * @rq: the request being processed 462 * 463 * Description: 464 * Ends all I/O on a request. It does not handle partial completions. 465 * The actual completion happens out-of-order, through a IPI handler. 466 **/ 467 void blk_mq_complete_request(struct request *rq, int error) 468 { 469 struct request_queue *q = rq->q; 470 471 if (unlikely(blk_should_fake_timeout(q))) 472 return; 473 if (!blk_mark_rq_complete(rq)) { 474 rq->errors = error; 475 __blk_mq_complete_request(rq); 476 } 477 } 478 EXPORT_SYMBOL(blk_mq_complete_request); 479 480 int blk_mq_request_started(struct request *rq) 481 { 482 return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags); 483 } 484 EXPORT_SYMBOL_GPL(blk_mq_request_started); 485 486 void blk_mq_start_request(struct request *rq) 487 { 488 struct request_queue *q = rq->q; 489 490 blk_mq_sched_started_request(rq); 491 492 trace_block_rq_issue(q, rq); 493 494 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 495 blk_stat_set_issue_time(&rq->issue_stat); 496 rq->rq_flags |= RQF_STATS; 497 wbt_issue(q->rq_wb, &rq->issue_stat); 498 } 499 500 blk_add_timer(rq); 501 502 /* 503 * Ensure that ->deadline is visible before set the started 504 * flag and clear the completed flag. 505 */ 506 smp_mb__before_atomic(); 507 508 /* 509 * Mark us as started and clear complete. Complete might have been 510 * set if requeue raced with timeout, which then marked it as 511 * complete. So be sure to clear complete again when we start 512 * the request, otherwise we'll ignore the completion event. 513 */ 514 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) 515 set_bit(REQ_ATOM_STARTED, &rq->atomic_flags); 516 if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) 517 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags); 518 519 if (q->dma_drain_size && blk_rq_bytes(rq)) { 520 /* 521 * Make sure space for the drain appears. We know we can do 522 * this because max_hw_segments has been adjusted to be one 523 * fewer than the device can handle. 524 */ 525 rq->nr_phys_segments++; 526 } 527 } 528 EXPORT_SYMBOL(blk_mq_start_request); 529 530 static void __blk_mq_requeue_request(struct request *rq) 531 { 532 struct request_queue *q = rq->q; 533 534 trace_block_rq_requeue(q, rq); 535 wbt_requeue(q->rq_wb, &rq->issue_stat); 536 blk_mq_sched_requeue_request(rq); 537 538 if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) { 539 if (q->dma_drain_size && blk_rq_bytes(rq)) 540 rq->nr_phys_segments--; 541 } 542 } 543 544 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) 545 { 546 __blk_mq_requeue_request(rq); 547 548 BUG_ON(blk_queued_rq(rq)); 549 blk_mq_add_to_requeue_list(rq, true, kick_requeue_list); 550 } 551 EXPORT_SYMBOL(blk_mq_requeue_request); 552 553 static void blk_mq_requeue_work(struct work_struct *work) 554 { 555 struct request_queue *q = 556 container_of(work, struct request_queue, requeue_work.work); 557 LIST_HEAD(rq_list); 558 struct request *rq, *next; 559 unsigned long flags; 560 561 spin_lock_irqsave(&q->requeue_lock, flags); 562 list_splice_init(&q->requeue_list, &rq_list); 563 spin_unlock_irqrestore(&q->requeue_lock, flags); 564 565 list_for_each_entry_safe(rq, next, &rq_list, queuelist) { 566 if (!(rq->rq_flags & RQF_SOFTBARRIER)) 567 continue; 568 569 rq->rq_flags &= ~RQF_SOFTBARRIER; 570 list_del_init(&rq->queuelist); 571 blk_mq_sched_insert_request(rq, true, false, false, true); 572 } 573 574 while (!list_empty(&rq_list)) { 575 rq = list_entry(rq_list.next, struct request, queuelist); 576 list_del_init(&rq->queuelist); 577 blk_mq_sched_insert_request(rq, false, false, false, true); 578 } 579 580 blk_mq_run_hw_queues(q, false); 581 } 582 583 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, 584 bool kick_requeue_list) 585 { 586 struct request_queue *q = rq->q; 587 unsigned long flags; 588 589 /* 590 * We abuse this flag that is otherwise used by the I/O scheduler to 591 * request head insertation from the workqueue. 592 */ 593 BUG_ON(rq->rq_flags & RQF_SOFTBARRIER); 594 595 spin_lock_irqsave(&q->requeue_lock, flags); 596 if (at_head) { 597 rq->rq_flags |= RQF_SOFTBARRIER; 598 list_add(&rq->queuelist, &q->requeue_list); 599 } else { 600 list_add_tail(&rq->queuelist, &q->requeue_list); 601 } 602 spin_unlock_irqrestore(&q->requeue_lock, flags); 603 604 if (kick_requeue_list) 605 blk_mq_kick_requeue_list(q); 606 } 607 EXPORT_SYMBOL(blk_mq_add_to_requeue_list); 608 609 void blk_mq_kick_requeue_list(struct request_queue *q) 610 { 611 kblockd_schedule_delayed_work(&q->requeue_work, 0); 612 } 613 EXPORT_SYMBOL(blk_mq_kick_requeue_list); 614 615 void blk_mq_delay_kick_requeue_list(struct request_queue *q, 616 unsigned long msecs) 617 { 618 kblockd_schedule_delayed_work(&q->requeue_work, 619 msecs_to_jiffies(msecs)); 620 } 621 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); 622 623 void blk_mq_abort_requeue_list(struct request_queue *q) 624 { 625 unsigned long flags; 626 LIST_HEAD(rq_list); 627 628 spin_lock_irqsave(&q->requeue_lock, flags); 629 list_splice_init(&q->requeue_list, &rq_list); 630 spin_unlock_irqrestore(&q->requeue_lock, flags); 631 632 while (!list_empty(&rq_list)) { 633 struct request *rq; 634 635 rq = list_first_entry(&rq_list, struct request, queuelist); 636 list_del_init(&rq->queuelist); 637 rq->errors = -EIO; 638 blk_mq_end_request(rq, rq->errors); 639 } 640 } 641 EXPORT_SYMBOL(blk_mq_abort_requeue_list); 642 643 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) 644 { 645 if (tag < tags->nr_tags) { 646 prefetch(tags->rqs[tag]); 647 return tags->rqs[tag]; 648 } 649 650 return NULL; 651 } 652 EXPORT_SYMBOL(blk_mq_tag_to_rq); 653 654 struct blk_mq_timeout_data { 655 unsigned long next; 656 unsigned int next_set; 657 }; 658 659 void blk_mq_rq_timed_out(struct request *req, bool reserved) 660 { 661 const struct blk_mq_ops *ops = req->q->mq_ops; 662 enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER; 663 664 /* 665 * We know that complete is set at this point. If STARTED isn't set 666 * anymore, then the request isn't active and the "timeout" should 667 * just be ignored. This can happen due to the bitflag ordering. 668 * Timeout first checks if STARTED is set, and if it is, assumes 669 * the request is active. But if we race with completion, then 670 * we both flags will get cleared. So check here again, and ignore 671 * a timeout event with a request that isn't active. 672 */ 673 if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags)) 674 return; 675 676 if (ops->timeout) 677 ret = ops->timeout(req, reserved); 678 679 switch (ret) { 680 case BLK_EH_HANDLED: 681 __blk_mq_complete_request(req); 682 break; 683 case BLK_EH_RESET_TIMER: 684 blk_add_timer(req); 685 blk_clear_rq_complete(req); 686 break; 687 case BLK_EH_NOT_HANDLED: 688 break; 689 default: 690 printk(KERN_ERR "block: bad eh return: %d\n", ret); 691 break; 692 } 693 } 694 695 static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, 696 struct request *rq, void *priv, bool reserved) 697 { 698 struct blk_mq_timeout_data *data = priv; 699 700 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) 701 return; 702 703 if (time_after_eq(jiffies, rq->deadline)) { 704 if (!blk_mark_rq_complete(rq)) 705 blk_mq_rq_timed_out(rq, reserved); 706 } else if (!data->next_set || time_after(data->next, rq->deadline)) { 707 data->next = rq->deadline; 708 data->next_set = 1; 709 } 710 } 711 712 static void blk_mq_timeout_work(struct work_struct *work) 713 { 714 struct request_queue *q = 715 container_of(work, struct request_queue, timeout_work); 716 struct blk_mq_timeout_data data = { 717 .next = 0, 718 .next_set = 0, 719 }; 720 int i; 721 722 /* A deadlock might occur if a request is stuck requiring a 723 * timeout at the same time a queue freeze is waiting 724 * completion, since the timeout code would not be able to 725 * acquire the queue reference here. 726 * 727 * That's why we don't use blk_queue_enter here; instead, we use 728 * percpu_ref_tryget directly, because we need to be able to 729 * obtain a reference even in the short window between the queue 730 * starting to freeze, by dropping the first reference in 731 * blk_mq_freeze_queue_start, and the moment the last request is 732 * consumed, marked by the instant q_usage_counter reaches 733 * zero. 734 */ 735 if (!percpu_ref_tryget(&q->q_usage_counter)) 736 return; 737 738 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data); 739 740 if (data.next_set) { 741 data.next = blk_rq_timeout(round_jiffies_up(data.next)); 742 mod_timer(&q->timeout, data.next); 743 } else { 744 struct blk_mq_hw_ctx *hctx; 745 746 queue_for_each_hw_ctx(q, hctx, i) { 747 /* the hctx may be unmapped, so check it here */ 748 if (blk_mq_hw_queue_mapped(hctx)) 749 blk_mq_tag_idle(hctx); 750 } 751 } 752 blk_queue_exit(q); 753 } 754 755 /* 756 * Reverse check our software queue for entries that we could potentially 757 * merge with. Currently includes a hand-wavy stop count of 8, to not spend 758 * too much time checking for merges. 759 */ 760 static bool blk_mq_attempt_merge(struct request_queue *q, 761 struct blk_mq_ctx *ctx, struct bio *bio) 762 { 763 struct request *rq; 764 int checked = 8; 765 766 list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) { 767 bool merged = false; 768 769 if (!checked--) 770 break; 771 772 if (!blk_rq_merge_ok(rq, bio)) 773 continue; 774 775 switch (blk_try_merge(rq, bio)) { 776 case ELEVATOR_BACK_MERGE: 777 if (blk_mq_sched_allow_merge(q, rq, bio)) 778 merged = bio_attempt_back_merge(q, rq, bio); 779 break; 780 case ELEVATOR_FRONT_MERGE: 781 if (blk_mq_sched_allow_merge(q, rq, bio)) 782 merged = bio_attempt_front_merge(q, rq, bio); 783 break; 784 case ELEVATOR_DISCARD_MERGE: 785 merged = bio_attempt_discard_merge(q, rq, bio); 786 break; 787 default: 788 continue; 789 } 790 791 if (merged) 792 ctx->rq_merged++; 793 return merged; 794 } 795 796 return false; 797 } 798 799 struct flush_busy_ctx_data { 800 struct blk_mq_hw_ctx *hctx; 801 struct list_head *list; 802 }; 803 804 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) 805 { 806 struct flush_busy_ctx_data *flush_data = data; 807 struct blk_mq_hw_ctx *hctx = flush_data->hctx; 808 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 809 810 sbitmap_clear_bit(sb, bitnr); 811 spin_lock(&ctx->lock); 812 list_splice_tail_init(&ctx->rq_list, flush_data->list); 813 spin_unlock(&ctx->lock); 814 return true; 815 } 816 817 /* 818 * Process software queues that have been marked busy, splicing them 819 * to the for-dispatch 820 */ 821 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) 822 { 823 struct flush_busy_ctx_data data = { 824 .hctx = hctx, 825 .list = list, 826 }; 827 828 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data); 829 } 830 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs); 831 832 static inline unsigned int queued_to_index(unsigned int queued) 833 { 834 if (!queued) 835 return 0; 836 837 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); 838 } 839 840 bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx, 841 bool wait) 842 { 843 struct blk_mq_alloc_data data = { 844 .q = rq->q, 845 .hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), 846 .flags = wait ? 0 : BLK_MQ_REQ_NOWAIT, 847 }; 848 849 if (rq->tag != -1) { 850 done: 851 if (hctx) 852 *hctx = data.hctx; 853 return true; 854 } 855 856 if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag)) 857 data.flags |= BLK_MQ_REQ_RESERVED; 858 859 rq->tag = blk_mq_get_tag(&data); 860 if (rq->tag >= 0) { 861 if (blk_mq_tag_busy(data.hctx)) { 862 rq->rq_flags |= RQF_MQ_INFLIGHT; 863 atomic_inc(&data.hctx->nr_active); 864 } 865 data.hctx->tags->rqs[rq->tag] = rq; 866 goto done; 867 } 868 869 return false; 870 } 871 872 static void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx, 873 struct request *rq) 874 { 875 blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag); 876 rq->tag = -1; 877 878 if (rq->rq_flags & RQF_MQ_INFLIGHT) { 879 rq->rq_flags &= ~RQF_MQ_INFLIGHT; 880 atomic_dec(&hctx->nr_active); 881 } 882 } 883 884 static void blk_mq_put_driver_tag_hctx(struct blk_mq_hw_ctx *hctx, 885 struct request *rq) 886 { 887 if (rq->tag == -1 || rq->internal_tag == -1) 888 return; 889 890 __blk_mq_put_driver_tag(hctx, rq); 891 } 892 893 static void blk_mq_put_driver_tag(struct request *rq) 894 { 895 struct blk_mq_hw_ctx *hctx; 896 897 if (rq->tag == -1 || rq->internal_tag == -1) 898 return; 899 900 hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); 901 __blk_mq_put_driver_tag(hctx, rq); 902 } 903 904 /* 905 * If we fail getting a driver tag because all the driver tags are already 906 * assigned and on the dispatch list, BUT the first entry does not have a 907 * tag, then we could deadlock. For that case, move entries with assigned 908 * driver tags to the front, leaving the set of tagged requests in the 909 * same order, and the untagged set in the same order. 910 */ 911 static bool reorder_tags_to_front(struct list_head *list) 912 { 913 struct request *rq, *tmp, *first = NULL; 914 915 list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) { 916 if (rq == first) 917 break; 918 if (rq->tag != -1) { 919 list_move(&rq->queuelist, list); 920 if (!first) 921 first = rq; 922 } 923 } 924 925 return first != NULL; 926 } 927 928 static int blk_mq_dispatch_wake(wait_queue_t *wait, unsigned mode, int flags, 929 void *key) 930 { 931 struct blk_mq_hw_ctx *hctx; 932 933 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); 934 935 list_del(&wait->task_list); 936 clear_bit_unlock(BLK_MQ_S_TAG_WAITING, &hctx->state); 937 blk_mq_run_hw_queue(hctx, true); 938 return 1; 939 } 940 941 static bool blk_mq_dispatch_wait_add(struct blk_mq_hw_ctx *hctx) 942 { 943 struct sbq_wait_state *ws; 944 945 /* 946 * The TAG_WAITING bit serves as a lock protecting hctx->dispatch_wait. 947 * The thread which wins the race to grab this bit adds the hardware 948 * queue to the wait queue. 949 */ 950 if (test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state) || 951 test_and_set_bit_lock(BLK_MQ_S_TAG_WAITING, &hctx->state)) 952 return false; 953 954 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); 955 ws = bt_wait_ptr(&hctx->tags->bitmap_tags, hctx); 956 957 /* 958 * As soon as this returns, it's no longer safe to fiddle with 959 * hctx->dispatch_wait, since a completion can wake up the wait queue 960 * and unlock the bit. 961 */ 962 add_wait_queue(&ws->wait, &hctx->dispatch_wait); 963 return true; 964 } 965 966 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list) 967 { 968 struct request_queue *q = hctx->queue; 969 struct request *rq; 970 LIST_HEAD(driver_list); 971 struct list_head *dptr; 972 int queued, ret = BLK_MQ_RQ_QUEUE_OK; 973 974 /* 975 * Start off with dptr being NULL, so we start the first request 976 * immediately, even if we have more pending. 977 */ 978 dptr = NULL; 979 980 /* 981 * Now process all the entries, sending them to the driver. 982 */ 983 queued = 0; 984 while (!list_empty(list)) { 985 struct blk_mq_queue_data bd; 986 987 rq = list_first_entry(list, struct request, queuelist); 988 if (!blk_mq_get_driver_tag(rq, &hctx, false)) { 989 if (!queued && reorder_tags_to_front(list)) 990 continue; 991 992 /* 993 * The initial allocation attempt failed, so we need to 994 * rerun the hardware queue when a tag is freed. 995 */ 996 if (blk_mq_dispatch_wait_add(hctx)) { 997 /* 998 * It's possible that a tag was freed in the 999 * window between the allocation failure and 1000 * adding the hardware queue to the wait queue. 1001 */ 1002 if (!blk_mq_get_driver_tag(rq, &hctx, false)) 1003 break; 1004 } else { 1005 break; 1006 } 1007 } 1008 1009 list_del_init(&rq->queuelist); 1010 1011 bd.rq = rq; 1012 bd.list = dptr; 1013 1014 /* 1015 * Flag last if we have no more requests, or if we have more 1016 * but can't assign a driver tag to it. 1017 */ 1018 if (list_empty(list)) 1019 bd.last = true; 1020 else { 1021 struct request *nxt; 1022 1023 nxt = list_first_entry(list, struct request, queuelist); 1024 bd.last = !blk_mq_get_driver_tag(nxt, NULL, false); 1025 } 1026 1027 ret = q->mq_ops->queue_rq(hctx, &bd); 1028 switch (ret) { 1029 case BLK_MQ_RQ_QUEUE_OK: 1030 queued++; 1031 break; 1032 case BLK_MQ_RQ_QUEUE_BUSY: 1033 blk_mq_put_driver_tag_hctx(hctx, rq); 1034 list_add(&rq->queuelist, list); 1035 __blk_mq_requeue_request(rq); 1036 break; 1037 default: 1038 pr_err("blk-mq: bad return on queue: %d\n", ret); 1039 case BLK_MQ_RQ_QUEUE_ERROR: 1040 rq->errors = -EIO; 1041 blk_mq_end_request(rq, rq->errors); 1042 break; 1043 } 1044 1045 if (ret == BLK_MQ_RQ_QUEUE_BUSY) 1046 break; 1047 1048 /* 1049 * We've done the first request. If we have more than 1 1050 * left in the list, set dptr to defer issue. 1051 */ 1052 if (!dptr && list->next != list->prev) 1053 dptr = &driver_list; 1054 } 1055 1056 hctx->dispatched[queued_to_index(queued)]++; 1057 1058 /* 1059 * Any items that need requeuing? Stuff them into hctx->dispatch, 1060 * that is where we will continue on next queue run. 1061 */ 1062 if (!list_empty(list)) { 1063 /* 1064 * If we got a driver tag for the next request already, 1065 * free it again. 1066 */ 1067 rq = list_first_entry(list, struct request, queuelist); 1068 blk_mq_put_driver_tag(rq); 1069 1070 spin_lock(&hctx->lock); 1071 list_splice_init(list, &hctx->dispatch); 1072 spin_unlock(&hctx->lock); 1073 1074 /* 1075 * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but 1076 * it's possible the queue is stopped and restarted again 1077 * before this. Queue restart will dispatch requests. And since 1078 * requests in rq_list aren't added into hctx->dispatch yet, 1079 * the requests in rq_list might get lost. 1080 * 1081 * blk_mq_run_hw_queue() already checks the STOPPED bit 1082 * 1083 * If RESTART or TAG_WAITING is set, then let completion restart 1084 * the queue instead of potentially looping here. 1085 */ 1086 if (!blk_mq_sched_needs_restart(hctx) && 1087 !test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state)) 1088 blk_mq_run_hw_queue(hctx, true); 1089 } 1090 1091 return queued != 0; 1092 } 1093 1094 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) 1095 { 1096 int srcu_idx; 1097 1098 WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) && 1099 cpu_online(hctx->next_cpu)); 1100 1101 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { 1102 rcu_read_lock(); 1103 blk_mq_sched_dispatch_requests(hctx); 1104 rcu_read_unlock(); 1105 } else { 1106 srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu); 1107 blk_mq_sched_dispatch_requests(hctx); 1108 srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx); 1109 } 1110 } 1111 1112 /* 1113 * It'd be great if the workqueue API had a way to pass 1114 * in a mask and had some smarts for more clever placement. 1115 * For now we just round-robin here, switching for every 1116 * BLK_MQ_CPU_WORK_BATCH queued items. 1117 */ 1118 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) 1119 { 1120 if (hctx->queue->nr_hw_queues == 1) 1121 return WORK_CPU_UNBOUND; 1122 1123 if (--hctx->next_cpu_batch <= 0) { 1124 int next_cpu; 1125 1126 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask); 1127 if (next_cpu >= nr_cpu_ids) 1128 next_cpu = cpumask_first(hctx->cpumask); 1129 1130 hctx->next_cpu = next_cpu; 1131 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 1132 } 1133 1134 return hctx->next_cpu; 1135 } 1136 1137 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1138 { 1139 if (unlikely(blk_mq_hctx_stopped(hctx) || 1140 !blk_mq_hw_queue_mapped(hctx))) 1141 return; 1142 1143 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) { 1144 int cpu = get_cpu(); 1145 if (cpumask_test_cpu(cpu, hctx->cpumask)) { 1146 __blk_mq_run_hw_queue(hctx); 1147 put_cpu(); 1148 return; 1149 } 1150 1151 put_cpu(); 1152 } 1153 1154 kblockd_schedule_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work); 1155 } 1156 1157 void blk_mq_run_hw_queues(struct request_queue *q, bool async) 1158 { 1159 struct blk_mq_hw_ctx *hctx; 1160 int i; 1161 1162 queue_for_each_hw_ctx(q, hctx, i) { 1163 if (!blk_mq_hctx_has_pending(hctx) || 1164 blk_mq_hctx_stopped(hctx)) 1165 continue; 1166 1167 blk_mq_run_hw_queue(hctx, async); 1168 } 1169 } 1170 EXPORT_SYMBOL(blk_mq_run_hw_queues); 1171 1172 /** 1173 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped 1174 * @q: request queue. 1175 * 1176 * The caller is responsible for serializing this function against 1177 * blk_mq_{start,stop}_hw_queue(). 1178 */ 1179 bool blk_mq_queue_stopped(struct request_queue *q) 1180 { 1181 struct blk_mq_hw_ctx *hctx; 1182 int i; 1183 1184 queue_for_each_hw_ctx(q, hctx, i) 1185 if (blk_mq_hctx_stopped(hctx)) 1186 return true; 1187 1188 return false; 1189 } 1190 EXPORT_SYMBOL(blk_mq_queue_stopped); 1191 1192 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) 1193 { 1194 cancel_work(&hctx->run_work); 1195 cancel_delayed_work(&hctx->delay_work); 1196 set_bit(BLK_MQ_S_STOPPED, &hctx->state); 1197 } 1198 EXPORT_SYMBOL(blk_mq_stop_hw_queue); 1199 1200 void blk_mq_stop_hw_queues(struct request_queue *q) 1201 { 1202 struct blk_mq_hw_ctx *hctx; 1203 int i; 1204 1205 queue_for_each_hw_ctx(q, hctx, i) 1206 blk_mq_stop_hw_queue(hctx); 1207 } 1208 EXPORT_SYMBOL(blk_mq_stop_hw_queues); 1209 1210 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) 1211 { 1212 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1213 1214 blk_mq_run_hw_queue(hctx, false); 1215 } 1216 EXPORT_SYMBOL(blk_mq_start_hw_queue); 1217 1218 void blk_mq_start_hw_queues(struct request_queue *q) 1219 { 1220 struct blk_mq_hw_ctx *hctx; 1221 int i; 1222 1223 queue_for_each_hw_ctx(q, hctx, i) 1224 blk_mq_start_hw_queue(hctx); 1225 } 1226 EXPORT_SYMBOL(blk_mq_start_hw_queues); 1227 1228 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1229 { 1230 if (!blk_mq_hctx_stopped(hctx)) 1231 return; 1232 1233 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1234 blk_mq_run_hw_queue(hctx, async); 1235 } 1236 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue); 1237 1238 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) 1239 { 1240 struct blk_mq_hw_ctx *hctx; 1241 int i; 1242 1243 queue_for_each_hw_ctx(q, hctx, i) 1244 blk_mq_start_stopped_hw_queue(hctx, async); 1245 } 1246 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); 1247 1248 static void blk_mq_run_work_fn(struct work_struct *work) 1249 { 1250 struct blk_mq_hw_ctx *hctx; 1251 1252 hctx = container_of(work, struct blk_mq_hw_ctx, run_work); 1253 1254 __blk_mq_run_hw_queue(hctx); 1255 } 1256 1257 static void blk_mq_delay_work_fn(struct work_struct *work) 1258 { 1259 struct blk_mq_hw_ctx *hctx; 1260 1261 hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work); 1262 1263 if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state)) 1264 __blk_mq_run_hw_queue(hctx); 1265 } 1266 1267 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) 1268 { 1269 if (unlikely(!blk_mq_hw_queue_mapped(hctx))) 1270 return; 1271 1272 blk_mq_stop_hw_queue(hctx); 1273 kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx), 1274 &hctx->delay_work, msecs_to_jiffies(msecs)); 1275 } 1276 EXPORT_SYMBOL(blk_mq_delay_queue); 1277 1278 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx, 1279 struct request *rq, 1280 bool at_head) 1281 { 1282 struct blk_mq_ctx *ctx = rq->mq_ctx; 1283 1284 trace_block_rq_insert(hctx->queue, rq); 1285 1286 if (at_head) 1287 list_add(&rq->queuelist, &ctx->rq_list); 1288 else 1289 list_add_tail(&rq->queuelist, &ctx->rq_list); 1290 } 1291 1292 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, 1293 bool at_head) 1294 { 1295 struct blk_mq_ctx *ctx = rq->mq_ctx; 1296 1297 __blk_mq_insert_req_list(hctx, rq, at_head); 1298 blk_mq_hctx_mark_pending(hctx, ctx); 1299 } 1300 1301 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 1302 struct list_head *list) 1303 1304 { 1305 /* 1306 * preemption doesn't flush plug list, so it's possible ctx->cpu is 1307 * offline now 1308 */ 1309 spin_lock(&ctx->lock); 1310 while (!list_empty(list)) { 1311 struct request *rq; 1312 1313 rq = list_first_entry(list, struct request, queuelist); 1314 BUG_ON(rq->mq_ctx != ctx); 1315 list_del_init(&rq->queuelist); 1316 __blk_mq_insert_req_list(hctx, rq, false); 1317 } 1318 blk_mq_hctx_mark_pending(hctx, ctx); 1319 spin_unlock(&ctx->lock); 1320 } 1321 1322 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b) 1323 { 1324 struct request *rqa = container_of(a, struct request, queuelist); 1325 struct request *rqb = container_of(b, struct request, queuelist); 1326 1327 return !(rqa->mq_ctx < rqb->mq_ctx || 1328 (rqa->mq_ctx == rqb->mq_ctx && 1329 blk_rq_pos(rqa) < blk_rq_pos(rqb))); 1330 } 1331 1332 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 1333 { 1334 struct blk_mq_ctx *this_ctx; 1335 struct request_queue *this_q; 1336 struct request *rq; 1337 LIST_HEAD(list); 1338 LIST_HEAD(ctx_list); 1339 unsigned int depth; 1340 1341 list_splice_init(&plug->mq_list, &list); 1342 1343 list_sort(NULL, &list, plug_ctx_cmp); 1344 1345 this_q = NULL; 1346 this_ctx = NULL; 1347 depth = 0; 1348 1349 while (!list_empty(&list)) { 1350 rq = list_entry_rq(list.next); 1351 list_del_init(&rq->queuelist); 1352 BUG_ON(!rq->q); 1353 if (rq->mq_ctx != this_ctx) { 1354 if (this_ctx) { 1355 trace_block_unplug(this_q, depth, from_schedule); 1356 blk_mq_sched_insert_requests(this_q, this_ctx, 1357 &ctx_list, 1358 from_schedule); 1359 } 1360 1361 this_ctx = rq->mq_ctx; 1362 this_q = rq->q; 1363 depth = 0; 1364 } 1365 1366 depth++; 1367 list_add_tail(&rq->queuelist, &ctx_list); 1368 } 1369 1370 /* 1371 * If 'this_ctx' is set, we know we have entries to complete 1372 * on 'ctx_list'. Do those. 1373 */ 1374 if (this_ctx) { 1375 trace_block_unplug(this_q, depth, from_schedule); 1376 blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list, 1377 from_schedule); 1378 } 1379 } 1380 1381 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio) 1382 { 1383 init_request_from_bio(rq, bio); 1384 1385 blk_account_io_start(rq, true); 1386 } 1387 1388 static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx) 1389 { 1390 return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) && 1391 !blk_queue_nomerges(hctx->queue); 1392 } 1393 1394 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx, 1395 struct blk_mq_ctx *ctx, 1396 struct request *rq, struct bio *bio) 1397 { 1398 if (!hctx_allow_merges(hctx) || !bio_mergeable(bio)) { 1399 blk_mq_bio_to_request(rq, bio); 1400 spin_lock(&ctx->lock); 1401 insert_rq: 1402 __blk_mq_insert_request(hctx, rq, false); 1403 spin_unlock(&ctx->lock); 1404 return false; 1405 } else { 1406 struct request_queue *q = hctx->queue; 1407 1408 spin_lock(&ctx->lock); 1409 if (!blk_mq_attempt_merge(q, ctx, bio)) { 1410 blk_mq_bio_to_request(rq, bio); 1411 goto insert_rq; 1412 } 1413 1414 spin_unlock(&ctx->lock); 1415 __blk_mq_finish_request(hctx, ctx, rq); 1416 return true; 1417 } 1418 } 1419 1420 static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq) 1421 { 1422 if (rq->tag != -1) 1423 return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false); 1424 1425 return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true); 1426 } 1427 1428 static void blk_mq_try_issue_directly(struct request *rq, blk_qc_t *cookie, 1429 bool may_sleep) 1430 { 1431 struct request_queue *q = rq->q; 1432 struct blk_mq_queue_data bd = { 1433 .rq = rq, 1434 .list = NULL, 1435 .last = 1 1436 }; 1437 struct blk_mq_hw_ctx *hctx; 1438 blk_qc_t new_cookie; 1439 int ret; 1440 1441 if (q->elevator) 1442 goto insert; 1443 1444 if (!blk_mq_get_driver_tag(rq, &hctx, false)) 1445 goto insert; 1446 1447 new_cookie = request_to_qc_t(hctx, rq); 1448 1449 /* 1450 * For OK queue, we are done. For error, kill it. Any other 1451 * error (busy), just add it to our list as we previously 1452 * would have done 1453 */ 1454 ret = q->mq_ops->queue_rq(hctx, &bd); 1455 if (ret == BLK_MQ_RQ_QUEUE_OK) { 1456 *cookie = new_cookie; 1457 return; 1458 } 1459 1460 __blk_mq_requeue_request(rq); 1461 1462 if (ret == BLK_MQ_RQ_QUEUE_ERROR) { 1463 *cookie = BLK_QC_T_NONE; 1464 rq->errors = -EIO; 1465 blk_mq_end_request(rq, rq->errors); 1466 return; 1467 } 1468 1469 insert: 1470 blk_mq_sched_insert_request(rq, false, true, false, may_sleep); 1471 } 1472 1473 /* 1474 * Multiple hardware queue variant. This will not use per-process plugs, 1475 * but will attempt to bypass the hctx queueing if we can go straight to 1476 * hardware for SYNC IO. 1477 */ 1478 static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) 1479 { 1480 const int is_sync = op_is_sync(bio->bi_opf); 1481 const int is_flush_fua = op_is_flush(bio->bi_opf); 1482 struct blk_mq_alloc_data data = { .flags = 0 }; 1483 struct request *rq; 1484 unsigned int request_count = 0, srcu_idx; 1485 struct blk_plug *plug; 1486 struct request *same_queue_rq = NULL; 1487 blk_qc_t cookie; 1488 unsigned int wb_acct; 1489 1490 blk_queue_bounce(q, &bio); 1491 1492 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 1493 bio_io_error(bio); 1494 return BLK_QC_T_NONE; 1495 } 1496 1497 blk_queue_split(q, &bio, q->bio_split); 1498 1499 if (!is_flush_fua && !blk_queue_nomerges(q) && 1500 blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq)) 1501 return BLK_QC_T_NONE; 1502 1503 if (blk_mq_sched_bio_merge(q, bio)) 1504 return BLK_QC_T_NONE; 1505 1506 wb_acct = wbt_wait(q->rq_wb, bio, NULL); 1507 1508 trace_block_getrq(q, bio, bio->bi_opf); 1509 1510 rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data); 1511 if (unlikely(!rq)) { 1512 __wbt_done(q->rq_wb, wb_acct); 1513 return BLK_QC_T_NONE; 1514 } 1515 1516 wbt_track(&rq->issue_stat, wb_acct); 1517 1518 cookie = request_to_qc_t(data.hctx, rq); 1519 1520 if (unlikely(is_flush_fua)) { 1521 if (q->elevator) 1522 goto elv_insert; 1523 blk_mq_bio_to_request(rq, bio); 1524 blk_insert_flush(rq); 1525 goto run_queue; 1526 } 1527 1528 plug = current->plug; 1529 /* 1530 * If the driver supports defer issued based on 'last', then 1531 * queue it up like normal since we can potentially save some 1532 * CPU this way. 1533 */ 1534 if (((plug && !blk_queue_nomerges(q)) || is_sync) && 1535 !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) { 1536 struct request *old_rq = NULL; 1537 1538 blk_mq_bio_to_request(rq, bio); 1539 1540 /* 1541 * We do limited plugging. If the bio can be merged, do that. 1542 * Otherwise the existing request in the plug list will be 1543 * issued. So the plug list will have one request at most 1544 */ 1545 if (plug) { 1546 /* 1547 * The plug list might get flushed before this. If that 1548 * happens, same_queue_rq is invalid and plug list is 1549 * empty 1550 */ 1551 if (same_queue_rq && !list_empty(&plug->mq_list)) { 1552 old_rq = same_queue_rq; 1553 list_del_init(&old_rq->queuelist); 1554 } 1555 list_add_tail(&rq->queuelist, &plug->mq_list); 1556 } else /* is_sync */ 1557 old_rq = rq; 1558 blk_mq_put_ctx(data.ctx); 1559 if (!old_rq) 1560 goto done; 1561 1562 if (!(data.hctx->flags & BLK_MQ_F_BLOCKING)) { 1563 rcu_read_lock(); 1564 blk_mq_try_issue_directly(old_rq, &cookie, false); 1565 rcu_read_unlock(); 1566 } else { 1567 srcu_idx = srcu_read_lock(&data.hctx->queue_rq_srcu); 1568 blk_mq_try_issue_directly(old_rq, &cookie, true); 1569 srcu_read_unlock(&data.hctx->queue_rq_srcu, srcu_idx); 1570 } 1571 goto done; 1572 } 1573 1574 if (q->elevator) { 1575 elv_insert: 1576 blk_mq_put_ctx(data.ctx); 1577 blk_mq_bio_to_request(rq, bio); 1578 blk_mq_sched_insert_request(rq, false, true, 1579 !is_sync || is_flush_fua, true); 1580 goto done; 1581 } 1582 if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) { 1583 /* 1584 * For a SYNC request, send it to the hardware immediately. For 1585 * an ASYNC request, just ensure that we run it later on. The 1586 * latter allows for merging opportunities and more efficient 1587 * dispatching. 1588 */ 1589 run_queue: 1590 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua); 1591 } 1592 blk_mq_put_ctx(data.ctx); 1593 done: 1594 return cookie; 1595 } 1596 1597 /* 1598 * Single hardware queue variant. This will attempt to use any per-process 1599 * plug for merging and IO deferral. 1600 */ 1601 static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio) 1602 { 1603 const int is_sync = op_is_sync(bio->bi_opf); 1604 const int is_flush_fua = op_is_flush(bio->bi_opf); 1605 struct blk_plug *plug; 1606 unsigned int request_count = 0; 1607 struct blk_mq_alloc_data data = { .flags = 0 }; 1608 struct request *rq; 1609 blk_qc_t cookie; 1610 unsigned int wb_acct; 1611 1612 blk_queue_bounce(q, &bio); 1613 1614 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 1615 bio_io_error(bio); 1616 return BLK_QC_T_NONE; 1617 } 1618 1619 blk_queue_split(q, &bio, q->bio_split); 1620 1621 if (!is_flush_fua && !blk_queue_nomerges(q)) { 1622 if (blk_attempt_plug_merge(q, bio, &request_count, NULL)) 1623 return BLK_QC_T_NONE; 1624 } else 1625 request_count = blk_plug_queued_count(q); 1626 1627 if (blk_mq_sched_bio_merge(q, bio)) 1628 return BLK_QC_T_NONE; 1629 1630 wb_acct = wbt_wait(q->rq_wb, bio, NULL); 1631 1632 trace_block_getrq(q, bio, bio->bi_opf); 1633 1634 rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data); 1635 if (unlikely(!rq)) { 1636 __wbt_done(q->rq_wb, wb_acct); 1637 return BLK_QC_T_NONE; 1638 } 1639 1640 wbt_track(&rq->issue_stat, wb_acct); 1641 1642 cookie = request_to_qc_t(data.hctx, rq); 1643 1644 if (unlikely(is_flush_fua)) { 1645 if (q->elevator) 1646 goto elv_insert; 1647 blk_mq_bio_to_request(rq, bio); 1648 blk_insert_flush(rq); 1649 goto run_queue; 1650 } 1651 1652 /* 1653 * A task plug currently exists. Since this is completely lockless, 1654 * utilize that to temporarily store requests until the task is 1655 * either done or scheduled away. 1656 */ 1657 plug = current->plug; 1658 if (plug) { 1659 struct request *last = NULL; 1660 1661 blk_mq_bio_to_request(rq, bio); 1662 1663 /* 1664 * @request_count may become stale because of schedule 1665 * out, so check the list again. 1666 */ 1667 if (list_empty(&plug->mq_list)) 1668 request_count = 0; 1669 if (!request_count) 1670 trace_block_plug(q); 1671 else 1672 last = list_entry_rq(plug->mq_list.prev); 1673 1674 blk_mq_put_ctx(data.ctx); 1675 1676 if (request_count >= BLK_MAX_REQUEST_COUNT || (last && 1677 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { 1678 blk_flush_plug_list(plug, false); 1679 trace_block_plug(q); 1680 } 1681 1682 list_add_tail(&rq->queuelist, &plug->mq_list); 1683 return cookie; 1684 } 1685 1686 if (q->elevator) { 1687 elv_insert: 1688 blk_mq_put_ctx(data.ctx); 1689 blk_mq_bio_to_request(rq, bio); 1690 blk_mq_sched_insert_request(rq, false, true, 1691 !is_sync || is_flush_fua, true); 1692 goto done; 1693 } 1694 if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) { 1695 /* 1696 * For a SYNC request, send it to the hardware immediately. For 1697 * an ASYNC request, just ensure that we run it later on. The 1698 * latter allows for merging opportunities and more efficient 1699 * dispatching. 1700 */ 1701 run_queue: 1702 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua); 1703 } 1704 1705 blk_mq_put_ctx(data.ctx); 1706 done: 1707 return cookie; 1708 } 1709 1710 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 1711 unsigned int hctx_idx) 1712 { 1713 struct page *page; 1714 1715 if (tags->rqs && set->ops->exit_request) { 1716 int i; 1717 1718 for (i = 0; i < tags->nr_tags; i++) { 1719 struct request *rq = tags->static_rqs[i]; 1720 1721 if (!rq) 1722 continue; 1723 set->ops->exit_request(set->driver_data, rq, 1724 hctx_idx, i); 1725 tags->static_rqs[i] = NULL; 1726 } 1727 } 1728 1729 while (!list_empty(&tags->page_list)) { 1730 page = list_first_entry(&tags->page_list, struct page, lru); 1731 list_del_init(&page->lru); 1732 /* 1733 * Remove kmemleak object previously allocated in 1734 * blk_mq_init_rq_map(). 1735 */ 1736 kmemleak_free(page_address(page)); 1737 __free_pages(page, page->private); 1738 } 1739 } 1740 1741 void blk_mq_free_rq_map(struct blk_mq_tags *tags) 1742 { 1743 kfree(tags->rqs); 1744 tags->rqs = NULL; 1745 kfree(tags->static_rqs); 1746 tags->static_rqs = NULL; 1747 1748 blk_mq_free_tags(tags); 1749 } 1750 1751 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, 1752 unsigned int hctx_idx, 1753 unsigned int nr_tags, 1754 unsigned int reserved_tags) 1755 { 1756 struct blk_mq_tags *tags; 1757 int node; 1758 1759 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx); 1760 if (node == NUMA_NO_NODE) 1761 node = set->numa_node; 1762 1763 tags = blk_mq_init_tags(nr_tags, reserved_tags, node, 1764 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags)); 1765 if (!tags) 1766 return NULL; 1767 1768 tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *), 1769 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 1770 node); 1771 if (!tags->rqs) { 1772 blk_mq_free_tags(tags); 1773 return NULL; 1774 } 1775 1776 tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *), 1777 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 1778 node); 1779 if (!tags->static_rqs) { 1780 kfree(tags->rqs); 1781 blk_mq_free_tags(tags); 1782 return NULL; 1783 } 1784 1785 return tags; 1786 } 1787 1788 static size_t order_to_size(unsigned int order) 1789 { 1790 return (size_t)PAGE_SIZE << order; 1791 } 1792 1793 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 1794 unsigned int hctx_idx, unsigned int depth) 1795 { 1796 unsigned int i, j, entries_per_page, max_order = 4; 1797 size_t rq_size, left; 1798 int node; 1799 1800 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx); 1801 if (node == NUMA_NO_NODE) 1802 node = set->numa_node; 1803 1804 INIT_LIST_HEAD(&tags->page_list); 1805 1806 /* 1807 * rq_size is the size of the request plus driver payload, rounded 1808 * to the cacheline size 1809 */ 1810 rq_size = round_up(sizeof(struct request) + set->cmd_size, 1811 cache_line_size()); 1812 left = rq_size * depth; 1813 1814 for (i = 0; i < depth; ) { 1815 int this_order = max_order; 1816 struct page *page; 1817 int to_do; 1818 void *p; 1819 1820 while (this_order && left < order_to_size(this_order - 1)) 1821 this_order--; 1822 1823 do { 1824 page = alloc_pages_node(node, 1825 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, 1826 this_order); 1827 if (page) 1828 break; 1829 if (!this_order--) 1830 break; 1831 if (order_to_size(this_order) < rq_size) 1832 break; 1833 } while (1); 1834 1835 if (!page) 1836 goto fail; 1837 1838 page->private = this_order; 1839 list_add_tail(&page->lru, &tags->page_list); 1840 1841 p = page_address(page); 1842 /* 1843 * Allow kmemleak to scan these pages as they contain pointers 1844 * to additional allocations like via ops->init_request(). 1845 */ 1846 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); 1847 entries_per_page = order_to_size(this_order) / rq_size; 1848 to_do = min(entries_per_page, depth - i); 1849 left -= to_do * rq_size; 1850 for (j = 0; j < to_do; j++) { 1851 struct request *rq = p; 1852 1853 tags->static_rqs[i] = rq; 1854 if (set->ops->init_request) { 1855 if (set->ops->init_request(set->driver_data, 1856 rq, hctx_idx, i, 1857 node)) { 1858 tags->static_rqs[i] = NULL; 1859 goto fail; 1860 } 1861 } 1862 1863 p += rq_size; 1864 i++; 1865 } 1866 } 1867 return 0; 1868 1869 fail: 1870 blk_mq_free_rqs(set, tags, hctx_idx); 1871 return -ENOMEM; 1872 } 1873 1874 /* 1875 * 'cpu' is going away. splice any existing rq_list entries from this 1876 * software queue to the hw queue dispatch list, and ensure that it 1877 * gets run. 1878 */ 1879 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) 1880 { 1881 struct blk_mq_hw_ctx *hctx; 1882 struct blk_mq_ctx *ctx; 1883 LIST_HEAD(tmp); 1884 1885 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead); 1886 ctx = __blk_mq_get_ctx(hctx->queue, cpu); 1887 1888 spin_lock(&ctx->lock); 1889 if (!list_empty(&ctx->rq_list)) { 1890 list_splice_init(&ctx->rq_list, &tmp); 1891 blk_mq_hctx_clear_pending(hctx, ctx); 1892 } 1893 spin_unlock(&ctx->lock); 1894 1895 if (list_empty(&tmp)) 1896 return 0; 1897 1898 spin_lock(&hctx->lock); 1899 list_splice_tail_init(&tmp, &hctx->dispatch); 1900 spin_unlock(&hctx->lock); 1901 1902 blk_mq_run_hw_queue(hctx, true); 1903 return 0; 1904 } 1905 1906 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) 1907 { 1908 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD, 1909 &hctx->cpuhp_dead); 1910 } 1911 1912 /* hctx->ctxs will be freed in queue's release handler */ 1913 static void blk_mq_exit_hctx(struct request_queue *q, 1914 struct blk_mq_tag_set *set, 1915 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 1916 { 1917 unsigned flush_start_tag = set->queue_depth; 1918 1919 blk_mq_tag_idle(hctx); 1920 1921 if (set->ops->exit_request) 1922 set->ops->exit_request(set->driver_data, 1923 hctx->fq->flush_rq, hctx_idx, 1924 flush_start_tag + hctx_idx); 1925 1926 if (set->ops->exit_hctx) 1927 set->ops->exit_hctx(hctx, hctx_idx); 1928 1929 if (hctx->flags & BLK_MQ_F_BLOCKING) 1930 cleanup_srcu_struct(&hctx->queue_rq_srcu); 1931 1932 blk_mq_remove_cpuhp(hctx); 1933 blk_free_flush_queue(hctx->fq); 1934 sbitmap_free(&hctx->ctx_map); 1935 } 1936 1937 static void blk_mq_exit_hw_queues(struct request_queue *q, 1938 struct blk_mq_tag_set *set, int nr_queue) 1939 { 1940 struct blk_mq_hw_ctx *hctx; 1941 unsigned int i; 1942 1943 queue_for_each_hw_ctx(q, hctx, i) { 1944 if (i == nr_queue) 1945 break; 1946 blk_mq_exit_hctx(q, set, hctx, i); 1947 } 1948 } 1949 1950 static int blk_mq_init_hctx(struct request_queue *q, 1951 struct blk_mq_tag_set *set, 1952 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) 1953 { 1954 int node; 1955 unsigned flush_start_tag = set->queue_depth; 1956 1957 node = hctx->numa_node; 1958 if (node == NUMA_NO_NODE) 1959 node = hctx->numa_node = set->numa_node; 1960 1961 INIT_WORK(&hctx->run_work, blk_mq_run_work_fn); 1962 INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn); 1963 spin_lock_init(&hctx->lock); 1964 INIT_LIST_HEAD(&hctx->dispatch); 1965 hctx->queue = q; 1966 hctx->queue_num = hctx_idx; 1967 hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED; 1968 1969 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead); 1970 1971 hctx->tags = set->tags[hctx_idx]; 1972 1973 /* 1974 * Allocate space for all possible cpus to avoid allocation at 1975 * runtime 1976 */ 1977 hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *), 1978 GFP_KERNEL, node); 1979 if (!hctx->ctxs) 1980 goto unregister_cpu_notifier; 1981 1982 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL, 1983 node)) 1984 goto free_ctxs; 1985 1986 hctx->nr_ctx = 0; 1987 1988 if (set->ops->init_hctx && 1989 set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) 1990 goto free_bitmap; 1991 1992 hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size); 1993 if (!hctx->fq) 1994 goto exit_hctx; 1995 1996 if (set->ops->init_request && 1997 set->ops->init_request(set->driver_data, 1998 hctx->fq->flush_rq, hctx_idx, 1999 flush_start_tag + hctx_idx, node)) 2000 goto free_fq; 2001 2002 if (hctx->flags & BLK_MQ_F_BLOCKING) 2003 init_srcu_struct(&hctx->queue_rq_srcu); 2004 2005 return 0; 2006 2007 free_fq: 2008 kfree(hctx->fq); 2009 exit_hctx: 2010 if (set->ops->exit_hctx) 2011 set->ops->exit_hctx(hctx, hctx_idx); 2012 free_bitmap: 2013 sbitmap_free(&hctx->ctx_map); 2014 free_ctxs: 2015 kfree(hctx->ctxs); 2016 unregister_cpu_notifier: 2017 blk_mq_remove_cpuhp(hctx); 2018 return -1; 2019 } 2020 2021 static void blk_mq_init_cpu_queues(struct request_queue *q, 2022 unsigned int nr_hw_queues) 2023 { 2024 unsigned int i; 2025 2026 for_each_possible_cpu(i) { 2027 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i); 2028 struct blk_mq_hw_ctx *hctx; 2029 2030 __ctx->cpu = i; 2031 spin_lock_init(&__ctx->lock); 2032 INIT_LIST_HEAD(&__ctx->rq_list); 2033 __ctx->queue = q; 2034 blk_stat_init(&__ctx->stat[BLK_STAT_READ]); 2035 blk_stat_init(&__ctx->stat[BLK_STAT_WRITE]); 2036 2037 /* If the cpu isn't online, the cpu is mapped to first hctx */ 2038 if (!cpu_online(i)) 2039 continue; 2040 2041 hctx = blk_mq_map_queue(q, i); 2042 2043 /* 2044 * Set local node, IFF we have more than one hw queue. If 2045 * not, we remain on the home node of the device 2046 */ 2047 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) 2048 hctx->numa_node = local_memory_node(cpu_to_node(i)); 2049 } 2050 } 2051 2052 static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx) 2053 { 2054 int ret = 0; 2055 2056 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx, 2057 set->queue_depth, set->reserved_tags); 2058 if (!set->tags[hctx_idx]) 2059 return false; 2060 2061 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx, 2062 set->queue_depth); 2063 if (!ret) 2064 return true; 2065 2066 blk_mq_free_rq_map(set->tags[hctx_idx]); 2067 set->tags[hctx_idx] = NULL; 2068 return false; 2069 } 2070 2071 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set, 2072 unsigned int hctx_idx) 2073 { 2074 if (set->tags[hctx_idx]) { 2075 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx); 2076 blk_mq_free_rq_map(set->tags[hctx_idx]); 2077 set->tags[hctx_idx] = NULL; 2078 } 2079 } 2080 2081 static void blk_mq_map_swqueue(struct request_queue *q, 2082 const struct cpumask *online_mask) 2083 { 2084 unsigned int i, hctx_idx; 2085 struct blk_mq_hw_ctx *hctx; 2086 struct blk_mq_ctx *ctx; 2087 struct blk_mq_tag_set *set = q->tag_set; 2088 2089 /* 2090 * Avoid others reading imcomplete hctx->cpumask through sysfs 2091 */ 2092 mutex_lock(&q->sysfs_lock); 2093 2094 queue_for_each_hw_ctx(q, hctx, i) { 2095 cpumask_clear(hctx->cpumask); 2096 hctx->nr_ctx = 0; 2097 } 2098 2099 /* 2100 * Map software to hardware queues 2101 */ 2102 for_each_possible_cpu(i) { 2103 /* If the cpu isn't online, the cpu is mapped to first hctx */ 2104 if (!cpumask_test_cpu(i, online_mask)) 2105 continue; 2106 2107 hctx_idx = q->mq_map[i]; 2108 /* unmapped hw queue can be remapped after CPU topo changed */ 2109 if (!set->tags[hctx_idx] && 2110 !__blk_mq_alloc_rq_map(set, hctx_idx)) { 2111 /* 2112 * If tags initialization fail for some hctx, 2113 * that hctx won't be brought online. In this 2114 * case, remap the current ctx to hctx[0] which 2115 * is guaranteed to always have tags allocated 2116 */ 2117 q->mq_map[i] = 0; 2118 } 2119 2120 ctx = per_cpu_ptr(q->queue_ctx, i); 2121 hctx = blk_mq_map_queue(q, i); 2122 2123 cpumask_set_cpu(i, hctx->cpumask); 2124 ctx->index_hw = hctx->nr_ctx; 2125 hctx->ctxs[hctx->nr_ctx++] = ctx; 2126 } 2127 2128 mutex_unlock(&q->sysfs_lock); 2129 2130 queue_for_each_hw_ctx(q, hctx, i) { 2131 /* 2132 * If no software queues are mapped to this hardware queue, 2133 * disable it and free the request entries. 2134 */ 2135 if (!hctx->nr_ctx) { 2136 /* Never unmap queue 0. We need it as a 2137 * fallback in case of a new remap fails 2138 * allocation 2139 */ 2140 if (i && set->tags[i]) 2141 blk_mq_free_map_and_requests(set, i); 2142 2143 hctx->tags = NULL; 2144 continue; 2145 } 2146 2147 hctx->tags = set->tags[i]; 2148 WARN_ON(!hctx->tags); 2149 2150 /* 2151 * Set the map size to the number of mapped software queues. 2152 * This is more accurate and more efficient than looping 2153 * over all possibly mapped software queues. 2154 */ 2155 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx); 2156 2157 /* 2158 * Initialize batch roundrobin counts 2159 */ 2160 hctx->next_cpu = cpumask_first(hctx->cpumask); 2161 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 2162 } 2163 } 2164 2165 static void queue_set_hctx_shared(struct request_queue *q, bool shared) 2166 { 2167 struct blk_mq_hw_ctx *hctx; 2168 int i; 2169 2170 queue_for_each_hw_ctx(q, hctx, i) { 2171 if (shared) 2172 hctx->flags |= BLK_MQ_F_TAG_SHARED; 2173 else 2174 hctx->flags &= ~BLK_MQ_F_TAG_SHARED; 2175 } 2176 } 2177 2178 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared) 2179 { 2180 struct request_queue *q; 2181 2182 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2183 blk_mq_freeze_queue(q); 2184 queue_set_hctx_shared(q, shared); 2185 blk_mq_unfreeze_queue(q); 2186 } 2187 } 2188 2189 static void blk_mq_del_queue_tag_set(struct request_queue *q) 2190 { 2191 struct blk_mq_tag_set *set = q->tag_set; 2192 2193 mutex_lock(&set->tag_list_lock); 2194 list_del_init(&q->tag_set_list); 2195 if (list_is_singular(&set->tag_list)) { 2196 /* just transitioned to unshared */ 2197 set->flags &= ~BLK_MQ_F_TAG_SHARED; 2198 /* update existing queue */ 2199 blk_mq_update_tag_set_depth(set, false); 2200 } 2201 mutex_unlock(&set->tag_list_lock); 2202 } 2203 2204 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, 2205 struct request_queue *q) 2206 { 2207 q->tag_set = set; 2208 2209 mutex_lock(&set->tag_list_lock); 2210 2211 /* Check to see if we're transitioning to shared (from 1 to 2 queues). */ 2212 if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) { 2213 set->flags |= BLK_MQ_F_TAG_SHARED; 2214 /* update existing queue */ 2215 blk_mq_update_tag_set_depth(set, true); 2216 } 2217 if (set->flags & BLK_MQ_F_TAG_SHARED) 2218 queue_set_hctx_shared(q, true); 2219 list_add_tail(&q->tag_set_list, &set->tag_list); 2220 2221 mutex_unlock(&set->tag_list_lock); 2222 } 2223 2224 /* 2225 * It is the actual release handler for mq, but we do it from 2226 * request queue's release handler for avoiding use-after-free 2227 * and headache because q->mq_kobj shouldn't have been introduced, 2228 * but we can't group ctx/kctx kobj without it. 2229 */ 2230 void blk_mq_release(struct request_queue *q) 2231 { 2232 struct blk_mq_hw_ctx *hctx; 2233 unsigned int i; 2234 2235 blk_mq_sched_teardown(q); 2236 2237 /* hctx kobj stays in hctx */ 2238 queue_for_each_hw_ctx(q, hctx, i) { 2239 if (!hctx) 2240 continue; 2241 kobject_put(&hctx->kobj); 2242 } 2243 2244 q->mq_map = NULL; 2245 2246 kfree(q->queue_hw_ctx); 2247 2248 /* 2249 * release .mq_kobj and sw queue's kobject now because 2250 * both share lifetime with request queue. 2251 */ 2252 blk_mq_sysfs_deinit(q); 2253 2254 free_percpu(q->queue_ctx); 2255 } 2256 2257 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) 2258 { 2259 struct request_queue *uninit_q, *q; 2260 2261 uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node); 2262 if (!uninit_q) 2263 return ERR_PTR(-ENOMEM); 2264 2265 q = blk_mq_init_allocated_queue(set, uninit_q); 2266 if (IS_ERR(q)) 2267 blk_cleanup_queue(uninit_q); 2268 2269 return q; 2270 } 2271 EXPORT_SYMBOL(blk_mq_init_queue); 2272 2273 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, 2274 struct request_queue *q) 2275 { 2276 int i, j; 2277 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; 2278 2279 blk_mq_sysfs_unregister(q); 2280 for (i = 0; i < set->nr_hw_queues; i++) { 2281 int node; 2282 2283 if (hctxs[i]) 2284 continue; 2285 2286 node = blk_mq_hw_queue_to_node(q->mq_map, i); 2287 hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx), 2288 GFP_KERNEL, node); 2289 if (!hctxs[i]) 2290 break; 2291 2292 if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL, 2293 node)) { 2294 kfree(hctxs[i]); 2295 hctxs[i] = NULL; 2296 break; 2297 } 2298 2299 atomic_set(&hctxs[i]->nr_active, 0); 2300 hctxs[i]->numa_node = node; 2301 hctxs[i]->queue_num = i; 2302 2303 if (blk_mq_init_hctx(q, set, hctxs[i], i)) { 2304 free_cpumask_var(hctxs[i]->cpumask); 2305 kfree(hctxs[i]); 2306 hctxs[i] = NULL; 2307 break; 2308 } 2309 blk_mq_hctx_kobj_init(hctxs[i]); 2310 } 2311 for (j = i; j < q->nr_hw_queues; j++) { 2312 struct blk_mq_hw_ctx *hctx = hctxs[j]; 2313 2314 if (hctx) { 2315 if (hctx->tags) 2316 blk_mq_free_map_and_requests(set, j); 2317 blk_mq_exit_hctx(q, set, hctx, j); 2318 kobject_put(&hctx->kobj); 2319 hctxs[j] = NULL; 2320 2321 } 2322 } 2323 q->nr_hw_queues = i; 2324 blk_mq_sysfs_register(q); 2325 } 2326 2327 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 2328 struct request_queue *q) 2329 { 2330 /* mark the queue as mq asap */ 2331 q->mq_ops = set->ops; 2332 2333 q->queue_ctx = alloc_percpu(struct blk_mq_ctx); 2334 if (!q->queue_ctx) 2335 goto err_exit; 2336 2337 /* init q->mq_kobj and sw queues' kobjects */ 2338 blk_mq_sysfs_init(q); 2339 2340 q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)), 2341 GFP_KERNEL, set->numa_node); 2342 if (!q->queue_hw_ctx) 2343 goto err_percpu; 2344 2345 q->mq_map = set->mq_map; 2346 2347 blk_mq_realloc_hw_ctxs(set, q); 2348 if (!q->nr_hw_queues) 2349 goto err_hctxs; 2350 2351 INIT_WORK(&q->timeout_work, blk_mq_timeout_work); 2352 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ); 2353 2354 q->nr_queues = nr_cpu_ids; 2355 2356 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; 2357 2358 if (!(set->flags & BLK_MQ_F_SG_MERGE)) 2359 q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE; 2360 2361 q->sg_reserved_size = INT_MAX; 2362 2363 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); 2364 INIT_LIST_HEAD(&q->requeue_list); 2365 spin_lock_init(&q->requeue_lock); 2366 2367 if (q->nr_hw_queues > 1) 2368 blk_queue_make_request(q, blk_mq_make_request); 2369 else 2370 blk_queue_make_request(q, blk_sq_make_request); 2371 2372 /* 2373 * Do this after blk_queue_make_request() overrides it... 2374 */ 2375 q->nr_requests = set->queue_depth; 2376 2377 /* 2378 * Default to classic polling 2379 */ 2380 q->poll_nsec = -1; 2381 2382 if (set->ops->complete) 2383 blk_queue_softirq_done(q, set->ops->complete); 2384 2385 blk_mq_init_cpu_queues(q, set->nr_hw_queues); 2386 2387 get_online_cpus(); 2388 mutex_lock(&all_q_mutex); 2389 2390 list_add_tail(&q->all_q_node, &all_q_list); 2391 blk_mq_add_queue_tag_set(set, q); 2392 blk_mq_map_swqueue(q, cpu_online_mask); 2393 2394 mutex_unlock(&all_q_mutex); 2395 put_online_cpus(); 2396 2397 if (!(set->flags & BLK_MQ_F_NO_SCHED)) { 2398 int ret; 2399 2400 ret = blk_mq_sched_init(q); 2401 if (ret) 2402 return ERR_PTR(ret); 2403 } 2404 2405 return q; 2406 2407 err_hctxs: 2408 kfree(q->queue_hw_ctx); 2409 err_percpu: 2410 free_percpu(q->queue_ctx); 2411 err_exit: 2412 q->mq_ops = NULL; 2413 return ERR_PTR(-ENOMEM); 2414 } 2415 EXPORT_SYMBOL(blk_mq_init_allocated_queue); 2416 2417 void blk_mq_free_queue(struct request_queue *q) 2418 { 2419 struct blk_mq_tag_set *set = q->tag_set; 2420 2421 mutex_lock(&all_q_mutex); 2422 list_del_init(&q->all_q_node); 2423 mutex_unlock(&all_q_mutex); 2424 2425 wbt_exit(q); 2426 2427 blk_mq_del_queue_tag_set(q); 2428 2429 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); 2430 } 2431 2432 /* Basically redo blk_mq_init_queue with queue frozen */ 2433 static void blk_mq_queue_reinit(struct request_queue *q, 2434 const struct cpumask *online_mask) 2435 { 2436 WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth)); 2437 2438 blk_mq_sysfs_unregister(q); 2439 2440 /* 2441 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe 2442 * we should change hctx numa_node according to new topology (this 2443 * involves free and re-allocate memory, worthy doing?) 2444 */ 2445 2446 blk_mq_map_swqueue(q, online_mask); 2447 2448 blk_mq_sysfs_register(q); 2449 } 2450 2451 /* 2452 * New online cpumask which is going to be set in this hotplug event. 2453 * Declare this cpumasks as global as cpu-hotplug operation is invoked 2454 * one-by-one and dynamically allocating this could result in a failure. 2455 */ 2456 static struct cpumask cpuhp_online_new; 2457 2458 static void blk_mq_queue_reinit_work(void) 2459 { 2460 struct request_queue *q; 2461 2462 mutex_lock(&all_q_mutex); 2463 /* 2464 * We need to freeze and reinit all existing queues. Freezing 2465 * involves synchronous wait for an RCU grace period and doing it 2466 * one by one may take a long time. Start freezing all queues in 2467 * one swoop and then wait for the completions so that freezing can 2468 * take place in parallel. 2469 */ 2470 list_for_each_entry(q, &all_q_list, all_q_node) 2471 blk_mq_freeze_queue_start(q); 2472 list_for_each_entry(q, &all_q_list, all_q_node) 2473 blk_mq_freeze_queue_wait(q); 2474 2475 list_for_each_entry(q, &all_q_list, all_q_node) 2476 blk_mq_queue_reinit(q, &cpuhp_online_new); 2477 2478 list_for_each_entry(q, &all_q_list, all_q_node) 2479 blk_mq_unfreeze_queue(q); 2480 2481 mutex_unlock(&all_q_mutex); 2482 } 2483 2484 static int blk_mq_queue_reinit_dead(unsigned int cpu) 2485 { 2486 cpumask_copy(&cpuhp_online_new, cpu_online_mask); 2487 blk_mq_queue_reinit_work(); 2488 return 0; 2489 } 2490 2491 /* 2492 * Before hotadded cpu starts handling requests, new mappings must be 2493 * established. Otherwise, these requests in hw queue might never be 2494 * dispatched. 2495 * 2496 * For example, there is a single hw queue (hctx) and two CPU queues (ctx0 2497 * for CPU0, and ctx1 for CPU1). 2498 * 2499 * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list 2500 * and set bit0 in pending bitmap as ctx1->index_hw is still zero. 2501 * 2502 * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set 2503 * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list. 2504 * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is 2505 * ignored. 2506 */ 2507 static int blk_mq_queue_reinit_prepare(unsigned int cpu) 2508 { 2509 cpumask_copy(&cpuhp_online_new, cpu_online_mask); 2510 cpumask_set_cpu(cpu, &cpuhp_online_new); 2511 blk_mq_queue_reinit_work(); 2512 return 0; 2513 } 2514 2515 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2516 { 2517 int i; 2518 2519 for (i = 0; i < set->nr_hw_queues; i++) 2520 if (!__blk_mq_alloc_rq_map(set, i)) 2521 goto out_unwind; 2522 2523 return 0; 2524 2525 out_unwind: 2526 while (--i >= 0) 2527 blk_mq_free_rq_map(set->tags[i]); 2528 2529 return -ENOMEM; 2530 } 2531 2532 /* 2533 * Allocate the request maps associated with this tag_set. Note that this 2534 * may reduce the depth asked for, if memory is tight. set->queue_depth 2535 * will be updated to reflect the allocated depth. 2536 */ 2537 static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2538 { 2539 unsigned int depth; 2540 int err; 2541 2542 depth = set->queue_depth; 2543 do { 2544 err = __blk_mq_alloc_rq_maps(set); 2545 if (!err) 2546 break; 2547 2548 set->queue_depth >>= 1; 2549 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) { 2550 err = -ENOMEM; 2551 break; 2552 } 2553 } while (set->queue_depth); 2554 2555 if (!set->queue_depth || err) { 2556 pr_err("blk-mq: failed to allocate request map\n"); 2557 return -ENOMEM; 2558 } 2559 2560 if (depth != set->queue_depth) 2561 pr_info("blk-mq: reduced tag depth (%u -> %u)\n", 2562 depth, set->queue_depth); 2563 2564 return 0; 2565 } 2566 2567 /* 2568 * Alloc a tag set to be associated with one or more request queues. 2569 * May fail with EINVAL for various error conditions. May adjust the 2570 * requested depth down, if if it too large. In that case, the set 2571 * value will be stored in set->queue_depth. 2572 */ 2573 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) 2574 { 2575 int ret; 2576 2577 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS); 2578 2579 if (!set->nr_hw_queues) 2580 return -EINVAL; 2581 if (!set->queue_depth) 2582 return -EINVAL; 2583 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) 2584 return -EINVAL; 2585 2586 if (!set->ops->queue_rq) 2587 return -EINVAL; 2588 2589 if (set->queue_depth > BLK_MQ_MAX_DEPTH) { 2590 pr_info("blk-mq: reduced tag depth to %u\n", 2591 BLK_MQ_MAX_DEPTH); 2592 set->queue_depth = BLK_MQ_MAX_DEPTH; 2593 } 2594 2595 /* 2596 * If a crashdump is active, then we are potentially in a very 2597 * memory constrained environment. Limit us to 1 queue and 2598 * 64 tags to prevent using too much memory. 2599 */ 2600 if (is_kdump_kernel()) { 2601 set->nr_hw_queues = 1; 2602 set->queue_depth = min(64U, set->queue_depth); 2603 } 2604 /* 2605 * There is no use for more h/w queues than cpus. 2606 */ 2607 if (set->nr_hw_queues > nr_cpu_ids) 2608 set->nr_hw_queues = nr_cpu_ids; 2609 2610 set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *), 2611 GFP_KERNEL, set->numa_node); 2612 if (!set->tags) 2613 return -ENOMEM; 2614 2615 ret = -ENOMEM; 2616 set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids, 2617 GFP_KERNEL, set->numa_node); 2618 if (!set->mq_map) 2619 goto out_free_tags; 2620 2621 if (set->ops->map_queues) 2622 ret = set->ops->map_queues(set); 2623 else 2624 ret = blk_mq_map_queues(set); 2625 if (ret) 2626 goto out_free_mq_map; 2627 2628 ret = blk_mq_alloc_rq_maps(set); 2629 if (ret) 2630 goto out_free_mq_map; 2631 2632 mutex_init(&set->tag_list_lock); 2633 INIT_LIST_HEAD(&set->tag_list); 2634 2635 return 0; 2636 2637 out_free_mq_map: 2638 kfree(set->mq_map); 2639 set->mq_map = NULL; 2640 out_free_tags: 2641 kfree(set->tags); 2642 set->tags = NULL; 2643 return ret; 2644 } 2645 EXPORT_SYMBOL(blk_mq_alloc_tag_set); 2646 2647 void blk_mq_free_tag_set(struct blk_mq_tag_set *set) 2648 { 2649 int i; 2650 2651 for (i = 0; i < nr_cpu_ids; i++) 2652 blk_mq_free_map_and_requests(set, i); 2653 2654 kfree(set->mq_map); 2655 set->mq_map = NULL; 2656 2657 kfree(set->tags); 2658 set->tags = NULL; 2659 } 2660 EXPORT_SYMBOL(blk_mq_free_tag_set); 2661 2662 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) 2663 { 2664 struct blk_mq_tag_set *set = q->tag_set; 2665 struct blk_mq_hw_ctx *hctx; 2666 int i, ret; 2667 2668 if (!set) 2669 return -EINVAL; 2670 2671 blk_mq_freeze_queue(q); 2672 blk_mq_quiesce_queue(q); 2673 2674 ret = 0; 2675 queue_for_each_hw_ctx(q, hctx, i) { 2676 if (!hctx->tags) 2677 continue; 2678 /* 2679 * If we're using an MQ scheduler, just update the scheduler 2680 * queue depth. This is similar to what the old code would do. 2681 */ 2682 if (!hctx->sched_tags) { 2683 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, 2684 min(nr, set->queue_depth), 2685 false); 2686 } else { 2687 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags, 2688 nr, true); 2689 } 2690 if (ret) 2691 break; 2692 } 2693 2694 if (!ret) 2695 q->nr_requests = nr; 2696 2697 blk_mq_unfreeze_queue(q); 2698 blk_mq_start_stopped_hw_queues(q, true); 2699 2700 return ret; 2701 } 2702 2703 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) 2704 { 2705 struct request_queue *q; 2706 2707 if (nr_hw_queues > nr_cpu_ids) 2708 nr_hw_queues = nr_cpu_ids; 2709 if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues) 2710 return; 2711 2712 list_for_each_entry(q, &set->tag_list, tag_set_list) 2713 blk_mq_freeze_queue(q); 2714 2715 set->nr_hw_queues = nr_hw_queues; 2716 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2717 blk_mq_realloc_hw_ctxs(set, q); 2718 2719 /* 2720 * Manually set the make_request_fn as blk_queue_make_request 2721 * resets a lot of the queue settings. 2722 */ 2723 if (q->nr_hw_queues > 1) 2724 q->make_request_fn = blk_mq_make_request; 2725 else 2726 q->make_request_fn = blk_sq_make_request; 2727 2728 blk_mq_queue_reinit(q, cpu_online_mask); 2729 } 2730 2731 list_for_each_entry(q, &set->tag_list, tag_set_list) 2732 blk_mq_unfreeze_queue(q); 2733 } 2734 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); 2735 2736 static unsigned long blk_mq_poll_nsecs(struct request_queue *q, 2737 struct blk_mq_hw_ctx *hctx, 2738 struct request *rq) 2739 { 2740 struct blk_rq_stat stat[2]; 2741 unsigned long ret = 0; 2742 2743 /* 2744 * If stats collection isn't on, don't sleep but turn it on for 2745 * future users 2746 */ 2747 if (!blk_stat_enable(q)) 2748 return 0; 2749 2750 /* 2751 * We don't have to do this once per IO, should optimize this 2752 * to just use the current window of stats until it changes 2753 */ 2754 memset(&stat, 0, sizeof(stat)); 2755 blk_hctx_stat_get(hctx, stat); 2756 2757 /* 2758 * As an optimistic guess, use half of the mean service time 2759 * for this type of request. We can (and should) make this smarter. 2760 * For instance, if the completion latencies are tight, we can 2761 * get closer than just half the mean. This is especially 2762 * important on devices where the completion latencies are longer 2763 * than ~10 usec. 2764 */ 2765 if (req_op(rq) == REQ_OP_READ && stat[BLK_STAT_READ].nr_samples) 2766 ret = (stat[BLK_STAT_READ].mean + 1) / 2; 2767 else if (req_op(rq) == REQ_OP_WRITE && stat[BLK_STAT_WRITE].nr_samples) 2768 ret = (stat[BLK_STAT_WRITE].mean + 1) / 2; 2769 2770 return ret; 2771 } 2772 2773 static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, 2774 struct blk_mq_hw_ctx *hctx, 2775 struct request *rq) 2776 { 2777 struct hrtimer_sleeper hs; 2778 enum hrtimer_mode mode; 2779 unsigned int nsecs; 2780 ktime_t kt; 2781 2782 if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags)) 2783 return false; 2784 2785 /* 2786 * poll_nsec can be: 2787 * 2788 * -1: don't ever hybrid sleep 2789 * 0: use half of prev avg 2790 * >0: use this specific value 2791 */ 2792 if (q->poll_nsec == -1) 2793 return false; 2794 else if (q->poll_nsec > 0) 2795 nsecs = q->poll_nsec; 2796 else 2797 nsecs = blk_mq_poll_nsecs(q, hctx, rq); 2798 2799 if (!nsecs) 2800 return false; 2801 2802 set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags); 2803 2804 /* 2805 * This will be replaced with the stats tracking code, using 2806 * 'avg_completion_time / 2' as the pre-sleep target. 2807 */ 2808 kt = nsecs; 2809 2810 mode = HRTIMER_MODE_REL; 2811 hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode); 2812 hrtimer_set_expires(&hs.timer, kt); 2813 2814 hrtimer_init_sleeper(&hs, current); 2815 do { 2816 if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) 2817 break; 2818 set_current_state(TASK_UNINTERRUPTIBLE); 2819 hrtimer_start_expires(&hs.timer, mode); 2820 if (hs.task) 2821 io_schedule(); 2822 hrtimer_cancel(&hs.timer); 2823 mode = HRTIMER_MODE_ABS; 2824 } while (hs.task && !signal_pending(current)); 2825 2826 __set_current_state(TASK_RUNNING); 2827 destroy_hrtimer_on_stack(&hs.timer); 2828 return true; 2829 } 2830 2831 static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq) 2832 { 2833 struct request_queue *q = hctx->queue; 2834 long state; 2835 2836 /* 2837 * If we sleep, have the caller restart the poll loop to reset 2838 * the state. Like for the other success return cases, the 2839 * caller is responsible for checking if the IO completed. If 2840 * the IO isn't complete, we'll get called again and will go 2841 * straight to the busy poll loop. 2842 */ 2843 if (blk_mq_poll_hybrid_sleep(q, hctx, rq)) 2844 return true; 2845 2846 hctx->poll_considered++; 2847 2848 state = current->state; 2849 while (!need_resched()) { 2850 int ret; 2851 2852 hctx->poll_invoked++; 2853 2854 ret = q->mq_ops->poll(hctx, rq->tag); 2855 if (ret > 0) { 2856 hctx->poll_success++; 2857 set_current_state(TASK_RUNNING); 2858 return true; 2859 } 2860 2861 if (signal_pending_state(state, current)) 2862 set_current_state(TASK_RUNNING); 2863 2864 if (current->state == TASK_RUNNING) 2865 return true; 2866 if (ret < 0) 2867 break; 2868 cpu_relax(); 2869 } 2870 2871 return false; 2872 } 2873 2874 bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie) 2875 { 2876 struct blk_mq_hw_ctx *hctx; 2877 struct blk_plug *plug; 2878 struct request *rq; 2879 2880 if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) || 2881 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 2882 return false; 2883 2884 plug = current->plug; 2885 if (plug) 2886 blk_flush_plug_list(plug, false); 2887 2888 hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; 2889 if (!blk_qc_t_is_internal(cookie)) 2890 rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); 2891 else 2892 rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie)); 2893 2894 return __blk_mq_poll(hctx, rq); 2895 } 2896 EXPORT_SYMBOL_GPL(blk_mq_poll); 2897 2898 void blk_mq_disable_hotplug(void) 2899 { 2900 mutex_lock(&all_q_mutex); 2901 } 2902 2903 void blk_mq_enable_hotplug(void) 2904 { 2905 mutex_unlock(&all_q_mutex); 2906 } 2907 2908 static int __init blk_mq_init(void) 2909 { 2910 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, 2911 blk_mq_hctx_notify_dead); 2912 2913 cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare", 2914 blk_mq_queue_reinit_prepare, 2915 blk_mq_queue_reinit_dead); 2916 return 0; 2917 } 2918 subsys_initcall(blk_mq_init); 2919