1 /* 2 * Block multiqueue core code 3 * 4 * Copyright (C) 2013-2014 Jens Axboe 5 * Copyright (C) 2013-2014 Christoph Hellwig 6 */ 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/backing-dev.h> 10 #include <linux/bio.h> 11 #include <linux/blkdev.h> 12 #include <linux/kmemleak.h> 13 #include <linux/mm.h> 14 #include <linux/init.h> 15 #include <linux/slab.h> 16 #include <linux/workqueue.h> 17 #include <linux/smp.h> 18 #include <linux/llist.h> 19 #include <linux/list_sort.h> 20 #include <linux/cpu.h> 21 #include <linux/cache.h> 22 #include <linux/sched/sysctl.h> 23 #include <linux/sched/topology.h> 24 #include <linux/sched/signal.h> 25 #include <linux/delay.h> 26 #include <linux/crash_dump.h> 27 #include <linux/prefetch.h> 28 29 #include <trace/events/block.h> 30 31 #include <linux/blk-mq.h> 32 #include "blk.h" 33 #include "blk-mq.h" 34 #include "blk-mq-debugfs.h" 35 #include "blk-mq-tag.h" 36 #include "blk-stat.h" 37 #include "blk-wbt.h" 38 #include "blk-mq-sched.h" 39 40 static DEFINE_MUTEX(all_q_mutex); 41 static LIST_HEAD(all_q_list); 42 43 static void blk_mq_poll_stats_start(struct request_queue *q); 44 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); 45 static void __blk_mq_stop_hw_queues(struct request_queue *q, bool sync); 46 47 static int blk_mq_poll_stats_bkt(const struct request *rq) 48 { 49 int ddir, bytes, bucket; 50 51 ddir = rq_data_dir(rq); 52 bytes = blk_rq_bytes(rq); 53 54 bucket = ddir + 2*(ilog2(bytes) - 9); 55 56 if (bucket < 0) 57 return -1; 58 else if (bucket >= BLK_MQ_POLL_STATS_BKTS) 59 return ddir + BLK_MQ_POLL_STATS_BKTS - 2; 60 61 return bucket; 62 } 63 64 /* 65 * Check if any of the ctx's have pending work in this hardware queue 66 */ 67 bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) 68 { 69 return sbitmap_any_bit_set(&hctx->ctx_map) || 70 !list_empty_careful(&hctx->dispatch) || 71 blk_mq_sched_has_work(hctx); 72 } 73 74 /* 75 * Mark this ctx as having pending work in this hardware queue 76 */ 77 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, 78 struct blk_mq_ctx *ctx) 79 { 80 if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw)) 81 sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw); 82 } 83 84 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, 85 struct blk_mq_ctx *ctx) 86 { 87 sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw); 88 } 89 90 void blk_freeze_queue_start(struct request_queue *q) 91 { 92 int freeze_depth; 93 94 freeze_depth = atomic_inc_return(&q->mq_freeze_depth); 95 if (freeze_depth == 1) { 96 percpu_ref_kill(&q->q_usage_counter); 97 blk_mq_run_hw_queues(q, false); 98 } 99 } 100 EXPORT_SYMBOL_GPL(blk_freeze_queue_start); 101 102 void blk_mq_freeze_queue_wait(struct request_queue *q) 103 { 104 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter)); 105 } 106 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait); 107 108 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, 109 unsigned long timeout) 110 { 111 return wait_event_timeout(q->mq_freeze_wq, 112 percpu_ref_is_zero(&q->q_usage_counter), 113 timeout); 114 } 115 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout); 116 117 /* 118 * Guarantee no request is in use, so we can change any data structure of 119 * the queue afterward. 120 */ 121 void blk_freeze_queue(struct request_queue *q) 122 { 123 /* 124 * In the !blk_mq case we are only calling this to kill the 125 * q_usage_counter, otherwise this increases the freeze depth 126 * and waits for it to return to zero. For this reason there is 127 * no blk_unfreeze_queue(), and blk_freeze_queue() is not 128 * exported to drivers as the only user for unfreeze is blk_mq. 129 */ 130 blk_freeze_queue_start(q); 131 blk_mq_freeze_queue_wait(q); 132 } 133 134 void blk_mq_freeze_queue(struct request_queue *q) 135 { 136 /* 137 * ...just an alias to keep freeze and unfreeze actions balanced 138 * in the blk_mq_* namespace 139 */ 140 blk_freeze_queue(q); 141 } 142 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue); 143 144 void blk_mq_unfreeze_queue(struct request_queue *q) 145 { 146 int freeze_depth; 147 148 freeze_depth = atomic_dec_return(&q->mq_freeze_depth); 149 WARN_ON_ONCE(freeze_depth < 0); 150 if (!freeze_depth) { 151 percpu_ref_reinit(&q->q_usage_counter); 152 wake_up_all(&q->mq_freeze_wq); 153 } 154 } 155 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); 156 157 /** 158 * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished 159 * @q: request queue. 160 * 161 * Note: this function does not prevent that the struct request end_io() 162 * callback function is invoked. Additionally, it is not prevented that 163 * new queue_rq() calls occur unless the queue has been stopped first. 164 */ 165 void blk_mq_quiesce_queue(struct request_queue *q) 166 { 167 struct blk_mq_hw_ctx *hctx; 168 unsigned int i; 169 bool rcu = false; 170 171 __blk_mq_stop_hw_queues(q, true); 172 173 queue_for_each_hw_ctx(q, hctx, i) { 174 if (hctx->flags & BLK_MQ_F_BLOCKING) 175 synchronize_srcu(&hctx->queue_rq_srcu); 176 else 177 rcu = true; 178 } 179 if (rcu) 180 synchronize_rcu(); 181 } 182 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue); 183 184 void blk_mq_wake_waiters(struct request_queue *q) 185 { 186 struct blk_mq_hw_ctx *hctx; 187 unsigned int i; 188 189 queue_for_each_hw_ctx(q, hctx, i) 190 if (blk_mq_hw_queue_mapped(hctx)) 191 blk_mq_tag_wakeup_all(hctx->tags, true); 192 193 /* 194 * If we are called because the queue has now been marked as 195 * dying, we need to ensure that processes currently waiting on 196 * the queue are notified as well. 197 */ 198 wake_up_all(&q->mq_freeze_wq); 199 } 200 201 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) 202 { 203 return blk_mq_has_free_tags(hctx->tags); 204 } 205 EXPORT_SYMBOL(blk_mq_can_queue); 206 207 void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, 208 struct request *rq, unsigned int op) 209 { 210 INIT_LIST_HEAD(&rq->queuelist); 211 /* csd/requeue_work/fifo_time is initialized before use */ 212 rq->q = q; 213 rq->mq_ctx = ctx; 214 rq->cmd_flags = op; 215 if (blk_queue_io_stat(q)) 216 rq->rq_flags |= RQF_IO_STAT; 217 /* do not touch atomic flags, it needs atomic ops against the timer */ 218 rq->cpu = -1; 219 INIT_HLIST_NODE(&rq->hash); 220 RB_CLEAR_NODE(&rq->rb_node); 221 rq->rq_disk = NULL; 222 rq->part = NULL; 223 rq->start_time = jiffies; 224 #ifdef CONFIG_BLK_CGROUP 225 rq->rl = NULL; 226 set_start_time_ns(rq); 227 rq->io_start_time_ns = 0; 228 #endif 229 rq->nr_phys_segments = 0; 230 #if defined(CONFIG_BLK_DEV_INTEGRITY) 231 rq->nr_integrity_segments = 0; 232 #endif 233 rq->special = NULL; 234 /* tag was already set */ 235 rq->extra_len = 0; 236 237 INIT_LIST_HEAD(&rq->timeout_list); 238 rq->timeout = 0; 239 240 rq->end_io = NULL; 241 rq->end_io_data = NULL; 242 rq->next_rq = NULL; 243 244 ctx->rq_dispatched[op_is_sync(op)]++; 245 } 246 EXPORT_SYMBOL_GPL(blk_mq_rq_ctx_init); 247 248 struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data, 249 unsigned int op) 250 { 251 struct request *rq; 252 unsigned int tag; 253 254 tag = blk_mq_get_tag(data); 255 if (tag != BLK_MQ_TAG_FAIL) { 256 struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 257 258 rq = tags->static_rqs[tag]; 259 260 if (data->flags & BLK_MQ_REQ_INTERNAL) { 261 rq->tag = -1; 262 rq->internal_tag = tag; 263 } else { 264 if (blk_mq_tag_busy(data->hctx)) { 265 rq->rq_flags = RQF_MQ_INFLIGHT; 266 atomic_inc(&data->hctx->nr_active); 267 } 268 rq->tag = tag; 269 rq->internal_tag = -1; 270 data->hctx->tags->rqs[rq->tag] = rq; 271 } 272 273 blk_mq_rq_ctx_init(data->q, data->ctx, rq, op); 274 return rq; 275 } 276 277 return NULL; 278 } 279 EXPORT_SYMBOL_GPL(__blk_mq_alloc_request); 280 281 struct request *blk_mq_alloc_request(struct request_queue *q, int rw, 282 unsigned int flags) 283 { 284 struct blk_mq_alloc_data alloc_data = { .flags = flags }; 285 struct request *rq; 286 int ret; 287 288 ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT); 289 if (ret) 290 return ERR_PTR(ret); 291 292 rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data); 293 294 blk_mq_put_ctx(alloc_data.ctx); 295 blk_queue_exit(q); 296 297 if (!rq) 298 return ERR_PTR(-EWOULDBLOCK); 299 300 rq->__data_len = 0; 301 rq->__sector = (sector_t) -1; 302 rq->bio = rq->biotail = NULL; 303 return rq; 304 } 305 EXPORT_SYMBOL(blk_mq_alloc_request); 306 307 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw, 308 unsigned int flags, unsigned int hctx_idx) 309 { 310 struct blk_mq_alloc_data alloc_data = { .flags = flags }; 311 struct request *rq; 312 unsigned int cpu; 313 int ret; 314 315 /* 316 * If the tag allocator sleeps we could get an allocation for a 317 * different hardware context. No need to complicate the low level 318 * allocator for this for the rare use case of a command tied to 319 * a specific queue. 320 */ 321 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT))) 322 return ERR_PTR(-EINVAL); 323 324 if (hctx_idx >= q->nr_hw_queues) 325 return ERR_PTR(-EIO); 326 327 ret = blk_queue_enter(q, true); 328 if (ret) 329 return ERR_PTR(ret); 330 331 /* 332 * Check if the hardware context is actually mapped to anything. 333 * If not tell the caller that it should skip this queue. 334 */ 335 alloc_data.hctx = q->queue_hw_ctx[hctx_idx]; 336 if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) { 337 blk_queue_exit(q); 338 return ERR_PTR(-EXDEV); 339 } 340 cpu = cpumask_first(alloc_data.hctx->cpumask); 341 alloc_data.ctx = __blk_mq_get_ctx(q, cpu); 342 343 rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data); 344 345 blk_queue_exit(q); 346 347 if (!rq) 348 return ERR_PTR(-EWOULDBLOCK); 349 350 return rq; 351 } 352 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); 353 354 void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 355 struct request *rq) 356 { 357 const int sched_tag = rq->internal_tag; 358 struct request_queue *q = rq->q; 359 360 if (rq->rq_flags & RQF_MQ_INFLIGHT) 361 atomic_dec(&hctx->nr_active); 362 363 wbt_done(q->rq_wb, &rq->issue_stat); 364 rq->rq_flags = 0; 365 366 clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags); 367 clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags); 368 if (rq->tag != -1) 369 blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag); 370 if (sched_tag != -1) 371 blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag); 372 blk_mq_sched_restart(hctx); 373 blk_queue_exit(q); 374 } 375 376 static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx, 377 struct request *rq) 378 { 379 struct blk_mq_ctx *ctx = rq->mq_ctx; 380 381 ctx->rq_completed[rq_is_sync(rq)]++; 382 __blk_mq_finish_request(hctx, ctx, rq); 383 } 384 385 void blk_mq_finish_request(struct request *rq) 386 { 387 blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq); 388 } 389 EXPORT_SYMBOL_GPL(blk_mq_finish_request); 390 391 void blk_mq_free_request(struct request *rq) 392 { 393 blk_mq_sched_put_request(rq); 394 } 395 EXPORT_SYMBOL_GPL(blk_mq_free_request); 396 397 inline void __blk_mq_end_request(struct request *rq, int error) 398 { 399 blk_account_io_done(rq); 400 401 if (rq->end_io) { 402 wbt_done(rq->q->rq_wb, &rq->issue_stat); 403 rq->end_io(rq, error); 404 } else { 405 if (unlikely(blk_bidi_rq(rq))) 406 blk_mq_free_request(rq->next_rq); 407 blk_mq_free_request(rq); 408 } 409 } 410 EXPORT_SYMBOL(__blk_mq_end_request); 411 412 void blk_mq_end_request(struct request *rq, int error) 413 { 414 if (blk_update_request(rq, error, blk_rq_bytes(rq))) 415 BUG(); 416 __blk_mq_end_request(rq, error); 417 } 418 EXPORT_SYMBOL(blk_mq_end_request); 419 420 static void __blk_mq_complete_request_remote(void *data) 421 { 422 struct request *rq = data; 423 424 rq->q->softirq_done_fn(rq); 425 } 426 427 static void __blk_mq_complete_request(struct request *rq) 428 { 429 struct blk_mq_ctx *ctx = rq->mq_ctx; 430 bool shared = false; 431 int cpu; 432 433 if (rq->internal_tag != -1) 434 blk_mq_sched_completed_request(rq); 435 if (rq->rq_flags & RQF_STATS) { 436 blk_mq_poll_stats_start(rq->q); 437 blk_stat_add(rq); 438 } 439 440 if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) { 441 rq->q->softirq_done_fn(rq); 442 return; 443 } 444 445 cpu = get_cpu(); 446 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags)) 447 shared = cpus_share_cache(cpu, ctx->cpu); 448 449 if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { 450 rq->csd.func = __blk_mq_complete_request_remote; 451 rq->csd.info = rq; 452 rq->csd.flags = 0; 453 smp_call_function_single_async(ctx->cpu, &rq->csd); 454 } else { 455 rq->q->softirq_done_fn(rq); 456 } 457 put_cpu(); 458 } 459 460 /** 461 * blk_mq_complete_request - end I/O on a request 462 * @rq: the request being processed 463 * 464 * Description: 465 * Ends all I/O on a request. It does not handle partial completions. 466 * The actual completion happens out-of-order, through a IPI handler. 467 **/ 468 void blk_mq_complete_request(struct request *rq) 469 { 470 struct request_queue *q = rq->q; 471 472 if (unlikely(blk_should_fake_timeout(q))) 473 return; 474 if (!blk_mark_rq_complete(rq)) 475 __blk_mq_complete_request(rq); 476 } 477 EXPORT_SYMBOL(blk_mq_complete_request); 478 479 int blk_mq_request_started(struct request *rq) 480 { 481 return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags); 482 } 483 EXPORT_SYMBOL_GPL(blk_mq_request_started); 484 485 void blk_mq_start_request(struct request *rq) 486 { 487 struct request_queue *q = rq->q; 488 489 blk_mq_sched_started_request(rq); 490 491 trace_block_rq_issue(q, rq); 492 493 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 494 blk_stat_set_issue(&rq->issue_stat, blk_rq_sectors(rq)); 495 rq->rq_flags |= RQF_STATS; 496 wbt_issue(q->rq_wb, &rq->issue_stat); 497 } 498 499 blk_add_timer(rq); 500 501 /* 502 * Ensure that ->deadline is visible before set the started 503 * flag and clear the completed flag. 504 */ 505 smp_mb__before_atomic(); 506 507 /* 508 * Mark us as started and clear complete. Complete might have been 509 * set if requeue raced with timeout, which then marked it as 510 * complete. So be sure to clear complete again when we start 511 * the request, otherwise we'll ignore the completion event. 512 */ 513 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) 514 set_bit(REQ_ATOM_STARTED, &rq->atomic_flags); 515 if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) 516 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags); 517 518 if (q->dma_drain_size && blk_rq_bytes(rq)) { 519 /* 520 * Make sure space for the drain appears. We know we can do 521 * this because max_hw_segments has been adjusted to be one 522 * fewer than the device can handle. 523 */ 524 rq->nr_phys_segments++; 525 } 526 } 527 EXPORT_SYMBOL(blk_mq_start_request); 528 529 /* 530 * When we reach here because queue is busy, REQ_ATOM_COMPLETE 531 * flag isn't set yet, so there may be race with timeout handler, 532 * but given rq->deadline is just set in .queue_rq() under 533 * this situation, the race won't be possible in reality because 534 * rq->timeout should be set as big enough to cover the window 535 * between blk_mq_start_request() called from .queue_rq() and 536 * clearing REQ_ATOM_STARTED here. 537 */ 538 static void __blk_mq_requeue_request(struct request *rq) 539 { 540 struct request_queue *q = rq->q; 541 542 trace_block_rq_requeue(q, rq); 543 wbt_requeue(q->rq_wb, &rq->issue_stat); 544 blk_mq_sched_requeue_request(rq); 545 546 if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) { 547 if (q->dma_drain_size && blk_rq_bytes(rq)) 548 rq->nr_phys_segments--; 549 } 550 } 551 552 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) 553 { 554 __blk_mq_requeue_request(rq); 555 556 BUG_ON(blk_queued_rq(rq)); 557 blk_mq_add_to_requeue_list(rq, true, kick_requeue_list); 558 } 559 EXPORT_SYMBOL(blk_mq_requeue_request); 560 561 static void blk_mq_requeue_work(struct work_struct *work) 562 { 563 struct request_queue *q = 564 container_of(work, struct request_queue, requeue_work.work); 565 LIST_HEAD(rq_list); 566 struct request *rq, *next; 567 unsigned long flags; 568 569 spin_lock_irqsave(&q->requeue_lock, flags); 570 list_splice_init(&q->requeue_list, &rq_list); 571 spin_unlock_irqrestore(&q->requeue_lock, flags); 572 573 list_for_each_entry_safe(rq, next, &rq_list, queuelist) { 574 if (!(rq->rq_flags & RQF_SOFTBARRIER)) 575 continue; 576 577 rq->rq_flags &= ~RQF_SOFTBARRIER; 578 list_del_init(&rq->queuelist); 579 blk_mq_sched_insert_request(rq, true, false, false, true); 580 } 581 582 while (!list_empty(&rq_list)) { 583 rq = list_entry(rq_list.next, struct request, queuelist); 584 list_del_init(&rq->queuelist); 585 blk_mq_sched_insert_request(rq, false, false, false, true); 586 } 587 588 blk_mq_run_hw_queues(q, false); 589 } 590 591 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, 592 bool kick_requeue_list) 593 { 594 struct request_queue *q = rq->q; 595 unsigned long flags; 596 597 /* 598 * We abuse this flag that is otherwise used by the I/O scheduler to 599 * request head insertation from the workqueue. 600 */ 601 BUG_ON(rq->rq_flags & RQF_SOFTBARRIER); 602 603 spin_lock_irqsave(&q->requeue_lock, flags); 604 if (at_head) { 605 rq->rq_flags |= RQF_SOFTBARRIER; 606 list_add(&rq->queuelist, &q->requeue_list); 607 } else { 608 list_add_tail(&rq->queuelist, &q->requeue_list); 609 } 610 spin_unlock_irqrestore(&q->requeue_lock, flags); 611 612 if (kick_requeue_list) 613 blk_mq_kick_requeue_list(q); 614 } 615 EXPORT_SYMBOL(blk_mq_add_to_requeue_list); 616 617 void blk_mq_kick_requeue_list(struct request_queue *q) 618 { 619 kblockd_schedule_delayed_work(&q->requeue_work, 0); 620 } 621 EXPORT_SYMBOL(blk_mq_kick_requeue_list); 622 623 void blk_mq_delay_kick_requeue_list(struct request_queue *q, 624 unsigned long msecs) 625 { 626 kblockd_schedule_delayed_work(&q->requeue_work, 627 msecs_to_jiffies(msecs)); 628 } 629 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); 630 631 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) 632 { 633 if (tag < tags->nr_tags) { 634 prefetch(tags->rqs[tag]); 635 return tags->rqs[tag]; 636 } 637 638 return NULL; 639 } 640 EXPORT_SYMBOL(blk_mq_tag_to_rq); 641 642 struct blk_mq_timeout_data { 643 unsigned long next; 644 unsigned int next_set; 645 }; 646 647 void blk_mq_rq_timed_out(struct request *req, bool reserved) 648 { 649 const struct blk_mq_ops *ops = req->q->mq_ops; 650 enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER; 651 652 /* 653 * We know that complete is set at this point. If STARTED isn't set 654 * anymore, then the request isn't active and the "timeout" should 655 * just be ignored. This can happen due to the bitflag ordering. 656 * Timeout first checks if STARTED is set, and if it is, assumes 657 * the request is active. But if we race with completion, then 658 * both flags will get cleared. So check here again, and ignore 659 * a timeout event with a request that isn't active. 660 */ 661 if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags)) 662 return; 663 664 if (ops->timeout) 665 ret = ops->timeout(req, reserved); 666 667 switch (ret) { 668 case BLK_EH_HANDLED: 669 __blk_mq_complete_request(req); 670 break; 671 case BLK_EH_RESET_TIMER: 672 blk_add_timer(req); 673 blk_clear_rq_complete(req); 674 break; 675 case BLK_EH_NOT_HANDLED: 676 break; 677 default: 678 printk(KERN_ERR "block: bad eh return: %d\n", ret); 679 break; 680 } 681 } 682 683 static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, 684 struct request *rq, void *priv, bool reserved) 685 { 686 struct blk_mq_timeout_data *data = priv; 687 688 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) 689 return; 690 691 /* 692 * The rq being checked may have been freed and reallocated 693 * out already here, we avoid this race by checking rq->deadline 694 * and REQ_ATOM_COMPLETE flag together: 695 * 696 * - if rq->deadline is observed as new value because of 697 * reusing, the rq won't be timed out because of timing. 698 * - if rq->deadline is observed as previous value, 699 * REQ_ATOM_COMPLETE flag won't be cleared in reuse path 700 * because we put a barrier between setting rq->deadline 701 * and clearing the flag in blk_mq_start_request(), so 702 * this rq won't be timed out too. 703 */ 704 if (time_after_eq(jiffies, rq->deadline)) { 705 if (!blk_mark_rq_complete(rq)) 706 blk_mq_rq_timed_out(rq, reserved); 707 } else if (!data->next_set || time_after(data->next, rq->deadline)) { 708 data->next = rq->deadline; 709 data->next_set = 1; 710 } 711 } 712 713 static void blk_mq_timeout_work(struct work_struct *work) 714 { 715 struct request_queue *q = 716 container_of(work, struct request_queue, timeout_work); 717 struct blk_mq_timeout_data data = { 718 .next = 0, 719 .next_set = 0, 720 }; 721 int i; 722 723 /* A deadlock might occur if a request is stuck requiring a 724 * timeout at the same time a queue freeze is waiting 725 * completion, since the timeout code would not be able to 726 * acquire the queue reference here. 727 * 728 * That's why we don't use blk_queue_enter here; instead, we use 729 * percpu_ref_tryget directly, because we need to be able to 730 * obtain a reference even in the short window between the queue 731 * starting to freeze, by dropping the first reference in 732 * blk_freeze_queue_start, and the moment the last request is 733 * consumed, marked by the instant q_usage_counter reaches 734 * zero. 735 */ 736 if (!percpu_ref_tryget(&q->q_usage_counter)) 737 return; 738 739 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data); 740 741 if (data.next_set) { 742 data.next = blk_rq_timeout(round_jiffies_up(data.next)); 743 mod_timer(&q->timeout, data.next); 744 } else { 745 struct blk_mq_hw_ctx *hctx; 746 747 queue_for_each_hw_ctx(q, hctx, i) { 748 /* the hctx may be unmapped, so check it here */ 749 if (blk_mq_hw_queue_mapped(hctx)) 750 blk_mq_tag_idle(hctx); 751 } 752 } 753 blk_queue_exit(q); 754 } 755 756 /* 757 * Reverse check our software queue for entries that we could potentially 758 * merge with. Currently includes a hand-wavy stop count of 8, to not spend 759 * too much time checking for merges. 760 */ 761 static bool blk_mq_attempt_merge(struct request_queue *q, 762 struct blk_mq_ctx *ctx, struct bio *bio) 763 { 764 struct request *rq; 765 int checked = 8; 766 767 list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) { 768 bool merged = false; 769 770 if (!checked--) 771 break; 772 773 if (!blk_rq_merge_ok(rq, bio)) 774 continue; 775 776 switch (blk_try_merge(rq, bio)) { 777 case ELEVATOR_BACK_MERGE: 778 if (blk_mq_sched_allow_merge(q, rq, bio)) 779 merged = bio_attempt_back_merge(q, rq, bio); 780 break; 781 case ELEVATOR_FRONT_MERGE: 782 if (blk_mq_sched_allow_merge(q, rq, bio)) 783 merged = bio_attempt_front_merge(q, rq, bio); 784 break; 785 case ELEVATOR_DISCARD_MERGE: 786 merged = bio_attempt_discard_merge(q, rq, bio); 787 break; 788 default: 789 continue; 790 } 791 792 if (merged) 793 ctx->rq_merged++; 794 return merged; 795 } 796 797 return false; 798 } 799 800 struct flush_busy_ctx_data { 801 struct blk_mq_hw_ctx *hctx; 802 struct list_head *list; 803 }; 804 805 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) 806 { 807 struct flush_busy_ctx_data *flush_data = data; 808 struct blk_mq_hw_ctx *hctx = flush_data->hctx; 809 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 810 811 sbitmap_clear_bit(sb, bitnr); 812 spin_lock(&ctx->lock); 813 list_splice_tail_init(&ctx->rq_list, flush_data->list); 814 spin_unlock(&ctx->lock); 815 return true; 816 } 817 818 /* 819 * Process software queues that have been marked busy, splicing them 820 * to the for-dispatch 821 */ 822 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) 823 { 824 struct flush_busy_ctx_data data = { 825 .hctx = hctx, 826 .list = list, 827 }; 828 829 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data); 830 } 831 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs); 832 833 static inline unsigned int queued_to_index(unsigned int queued) 834 { 835 if (!queued) 836 return 0; 837 838 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); 839 } 840 841 bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx, 842 bool wait) 843 { 844 struct blk_mq_alloc_data data = { 845 .q = rq->q, 846 .hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), 847 .flags = wait ? 0 : BLK_MQ_REQ_NOWAIT, 848 }; 849 850 might_sleep_if(wait); 851 852 if (rq->tag != -1) 853 goto done; 854 855 if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag)) 856 data.flags |= BLK_MQ_REQ_RESERVED; 857 858 rq->tag = blk_mq_get_tag(&data); 859 if (rq->tag >= 0) { 860 if (blk_mq_tag_busy(data.hctx)) { 861 rq->rq_flags |= RQF_MQ_INFLIGHT; 862 atomic_inc(&data.hctx->nr_active); 863 } 864 data.hctx->tags->rqs[rq->tag] = rq; 865 } 866 867 done: 868 if (hctx) 869 *hctx = data.hctx; 870 return rq->tag != -1; 871 } 872 873 static void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx, 874 struct request *rq) 875 { 876 blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag); 877 rq->tag = -1; 878 879 if (rq->rq_flags & RQF_MQ_INFLIGHT) { 880 rq->rq_flags &= ~RQF_MQ_INFLIGHT; 881 atomic_dec(&hctx->nr_active); 882 } 883 } 884 885 static void blk_mq_put_driver_tag_hctx(struct blk_mq_hw_ctx *hctx, 886 struct request *rq) 887 { 888 if (rq->tag == -1 || rq->internal_tag == -1) 889 return; 890 891 __blk_mq_put_driver_tag(hctx, rq); 892 } 893 894 static void blk_mq_put_driver_tag(struct request *rq) 895 { 896 struct blk_mq_hw_ctx *hctx; 897 898 if (rq->tag == -1 || rq->internal_tag == -1) 899 return; 900 901 hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); 902 __blk_mq_put_driver_tag(hctx, rq); 903 } 904 905 /* 906 * If we fail getting a driver tag because all the driver tags are already 907 * assigned and on the dispatch list, BUT the first entry does not have a 908 * tag, then we could deadlock. For that case, move entries with assigned 909 * driver tags to the front, leaving the set of tagged requests in the 910 * same order, and the untagged set in the same order. 911 */ 912 static bool reorder_tags_to_front(struct list_head *list) 913 { 914 struct request *rq, *tmp, *first = NULL; 915 916 list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) { 917 if (rq == first) 918 break; 919 if (rq->tag != -1) { 920 list_move(&rq->queuelist, list); 921 if (!first) 922 first = rq; 923 } 924 } 925 926 return first != NULL; 927 } 928 929 static int blk_mq_dispatch_wake(wait_queue_t *wait, unsigned mode, int flags, 930 void *key) 931 { 932 struct blk_mq_hw_ctx *hctx; 933 934 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); 935 936 list_del(&wait->task_list); 937 clear_bit_unlock(BLK_MQ_S_TAG_WAITING, &hctx->state); 938 blk_mq_run_hw_queue(hctx, true); 939 return 1; 940 } 941 942 static bool blk_mq_dispatch_wait_add(struct blk_mq_hw_ctx *hctx) 943 { 944 struct sbq_wait_state *ws; 945 946 /* 947 * The TAG_WAITING bit serves as a lock protecting hctx->dispatch_wait. 948 * The thread which wins the race to grab this bit adds the hardware 949 * queue to the wait queue. 950 */ 951 if (test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state) || 952 test_and_set_bit_lock(BLK_MQ_S_TAG_WAITING, &hctx->state)) 953 return false; 954 955 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); 956 ws = bt_wait_ptr(&hctx->tags->bitmap_tags, hctx); 957 958 /* 959 * As soon as this returns, it's no longer safe to fiddle with 960 * hctx->dispatch_wait, since a completion can wake up the wait queue 961 * and unlock the bit. 962 */ 963 add_wait_queue(&ws->wait, &hctx->dispatch_wait); 964 return true; 965 } 966 967 bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list) 968 { 969 struct blk_mq_hw_ctx *hctx; 970 struct request *rq; 971 int errors, queued, ret = BLK_MQ_RQ_QUEUE_OK; 972 973 if (list_empty(list)) 974 return false; 975 976 /* 977 * Now process all the entries, sending them to the driver. 978 */ 979 errors = queued = 0; 980 do { 981 struct blk_mq_queue_data bd; 982 983 rq = list_first_entry(list, struct request, queuelist); 984 if (!blk_mq_get_driver_tag(rq, &hctx, false)) { 985 if (!queued && reorder_tags_to_front(list)) 986 continue; 987 988 /* 989 * The initial allocation attempt failed, so we need to 990 * rerun the hardware queue when a tag is freed. 991 */ 992 if (!blk_mq_dispatch_wait_add(hctx)) 993 break; 994 995 /* 996 * It's possible that a tag was freed in the window 997 * between the allocation failure and adding the 998 * hardware queue to the wait queue. 999 */ 1000 if (!blk_mq_get_driver_tag(rq, &hctx, false)) 1001 break; 1002 } 1003 1004 list_del_init(&rq->queuelist); 1005 1006 bd.rq = rq; 1007 1008 /* 1009 * Flag last if we have no more requests, or if we have more 1010 * but can't assign a driver tag to it. 1011 */ 1012 if (list_empty(list)) 1013 bd.last = true; 1014 else { 1015 struct request *nxt; 1016 1017 nxt = list_first_entry(list, struct request, queuelist); 1018 bd.last = !blk_mq_get_driver_tag(nxt, NULL, false); 1019 } 1020 1021 ret = q->mq_ops->queue_rq(hctx, &bd); 1022 switch (ret) { 1023 case BLK_MQ_RQ_QUEUE_OK: 1024 queued++; 1025 break; 1026 case BLK_MQ_RQ_QUEUE_BUSY: 1027 blk_mq_put_driver_tag_hctx(hctx, rq); 1028 list_add(&rq->queuelist, list); 1029 __blk_mq_requeue_request(rq); 1030 break; 1031 default: 1032 pr_err("blk-mq: bad return on queue: %d\n", ret); 1033 case BLK_MQ_RQ_QUEUE_ERROR: 1034 errors++; 1035 blk_mq_end_request(rq, -EIO); 1036 break; 1037 } 1038 1039 if (ret == BLK_MQ_RQ_QUEUE_BUSY) 1040 break; 1041 } while (!list_empty(list)); 1042 1043 hctx->dispatched[queued_to_index(queued)]++; 1044 1045 /* 1046 * Any items that need requeuing? Stuff them into hctx->dispatch, 1047 * that is where we will continue on next queue run. 1048 */ 1049 if (!list_empty(list)) { 1050 /* 1051 * If an I/O scheduler has been configured and we got a driver 1052 * tag for the next request already, free it again. 1053 */ 1054 rq = list_first_entry(list, struct request, queuelist); 1055 blk_mq_put_driver_tag(rq); 1056 1057 spin_lock(&hctx->lock); 1058 list_splice_init(list, &hctx->dispatch); 1059 spin_unlock(&hctx->lock); 1060 1061 /* 1062 * If SCHED_RESTART was set by the caller of this function and 1063 * it is no longer set that means that it was cleared by another 1064 * thread and hence that a queue rerun is needed. 1065 * 1066 * If TAG_WAITING is set that means that an I/O scheduler has 1067 * been configured and another thread is waiting for a driver 1068 * tag. To guarantee fairness, do not rerun this hardware queue 1069 * but let the other thread grab the driver tag. 1070 * 1071 * If no I/O scheduler has been configured it is possible that 1072 * the hardware queue got stopped and restarted before requests 1073 * were pushed back onto the dispatch list. Rerun the queue to 1074 * avoid starvation. Notes: 1075 * - blk_mq_run_hw_queue() checks whether or not a queue has 1076 * been stopped before rerunning a queue. 1077 * - Some but not all block drivers stop a queue before 1078 * returning BLK_MQ_RQ_QUEUE_BUSY. Two exceptions are scsi-mq 1079 * and dm-rq. 1080 */ 1081 if (!blk_mq_sched_needs_restart(hctx) && 1082 !test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state)) 1083 blk_mq_run_hw_queue(hctx, true); 1084 } 1085 1086 return (queued + errors) != 0; 1087 } 1088 1089 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) 1090 { 1091 int srcu_idx; 1092 1093 WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) && 1094 cpu_online(hctx->next_cpu)); 1095 1096 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { 1097 rcu_read_lock(); 1098 blk_mq_sched_dispatch_requests(hctx); 1099 rcu_read_unlock(); 1100 } else { 1101 might_sleep(); 1102 1103 srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu); 1104 blk_mq_sched_dispatch_requests(hctx); 1105 srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx); 1106 } 1107 } 1108 1109 /* 1110 * It'd be great if the workqueue API had a way to pass 1111 * in a mask and had some smarts for more clever placement. 1112 * For now we just round-robin here, switching for every 1113 * BLK_MQ_CPU_WORK_BATCH queued items. 1114 */ 1115 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) 1116 { 1117 if (hctx->queue->nr_hw_queues == 1) 1118 return WORK_CPU_UNBOUND; 1119 1120 if (--hctx->next_cpu_batch <= 0) { 1121 int next_cpu; 1122 1123 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask); 1124 if (next_cpu >= nr_cpu_ids) 1125 next_cpu = cpumask_first(hctx->cpumask); 1126 1127 hctx->next_cpu = next_cpu; 1128 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 1129 } 1130 1131 return hctx->next_cpu; 1132 } 1133 1134 static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, 1135 unsigned long msecs) 1136 { 1137 if (unlikely(blk_mq_hctx_stopped(hctx) || 1138 !blk_mq_hw_queue_mapped(hctx))) 1139 return; 1140 1141 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) { 1142 int cpu = get_cpu(); 1143 if (cpumask_test_cpu(cpu, hctx->cpumask)) { 1144 __blk_mq_run_hw_queue(hctx); 1145 put_cpu(); 1146 return; 1147 } 1148 1149 put_cpu(); 1150 } 1151 1152 kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx), 1153 &hctx->run_work, 1154 msecs_to_jiffies(msecs)); 1155 } 1156 1157 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) 1158 { 1159 __blk_mq_delay_run_hw_queue(hctx, true, msecs); 1160 } 1161 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); 1162 1163 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1164 { 1165 __blk_mq_delay_run_hw_queue(hctx, async, 0); 1166 } 1167 EXPORT_SYMBOL(blk_mq_run_hw_queue); 1168 1169 void blk_mq_run_hw_queues(struct request_queue *q, bool async) 1170 { 1171 struct blk_mq_hw_ctx *hctx; 1172 int i; 1173 1174 queue_for_each_hw_ctx(q, hctx, i) { 1175 if (!blk_mq_hctx_has_pending(hctx) || 1176 blk_mq_hctx_stopped(hctx)) 1177 continue; 1178 1179 blk_mq_run_hw_queue(hctx, async); 1180 } 1181 } 1182 EXPORT_SYMBOL(blk_mq_run_hw_queues); 1183 1184 /** 1185 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped 1186 * @q: request queue. 1187 * 1188 * The caller is responsible for serializing this function against 1189 * blk_mq_{start,stop}_hw_queue(). 1190 */ 1191 bool blk_mq_queue_stopped(struct request_queue *q) 1192 { 1193 struct blk_mq_hw_ctx *hctx; 1194 int i; 1195 1196 queue_for_each_hw_ctx(q, hctx, i) 1197 if (blk_mq_hctx_stopped(hctx)) 1198 return true; 1199 1200 return false; 1201 } 1202 EXPORT_SYMBOL(blk_mq_queue_stopped); 1203 1204 static void __blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx, bool sync) 1205 { 1206 if (sync) 1207 cancel_delayed_work_sync(&hctx->run_work); 1208 else 1209 cancel_delayed_work(&hctx->run_work); 1210 1211 set_bit(BLK_MQ_S_STOPPED, &hctx->state); 1212 } 1213 1214 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) 1215 { 1216 __blk_mq_stop_hw_queue(hctx, false); 1217 } 1218 EXPORT_SYMBOL(blk_mq_stop_hw_queue); 1219 1220 static void __blk_mq_stop_hw_queues(struct request_queue *q, bool sync) 1221 { 1222 struct blk_mq_hw_ctx *hctx; 1223 int i; 1224 1225 queue_for_each_hw_ctx(q, hctx, i) 1226 __blk_mq_stop_hw_queue(hctx, sync); 1227 } 1228 1229 void blk_mq_stop_hw_queues(struct request_queue *q) 1230 { 1231 __blk_mq_stop_hw_queues(q, false); 1232 } 1233 EXPORT_SYMBOL(blk_mq_stop_hw_queues); 1234 1235 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) 1236 { 1237 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1238 1239 blk_mq_run_hw_queue(hctx, false); 1240 } 1241 EXPORT_SYMBOL(blk_mq_start_hw_queue); 1242 1243 void blk_mq_start_hw_queues(struct request_queue *q) 1244 { 1245 struct blk_mq_hw_ctx *hctx; 1246 int i; 1247 1248 queue_for_each_hw_ctx(q, hctx, i) 1249 blk_mq_start_hw_queue(hctx); 1250 } 1251 EXPORT_SYMBOL(blk_mq_start_hw_queues); 1252 1253 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1254 { 1255 if (!blk_mq_hctx_stopped(hctx)) 1256 return; 1257 1258 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1259 blk_mq_run_hw_queue(hctx, async); 1260 } 1261 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue); 1262 1263 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) 1264 { 1265 struct blk_mq_hw_ctx *hctx; 1266 int i; 1267 1268 queue_for_each_hw_ctx(q, hctx, i) 1269 blk_mq_start_stopped_hw_queue(hctx, async); 1270 } 1271 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); 1272 1273 static void blk_mq_run_work_fn(struct work_struct *work) 1274 { 1275 struct blk_mq_hw_ctx *hctx; 1276 1277 hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work); 1278 1279 /* 1280 * If we are stopped, don't run the queue. The exception is if 1281 * BLK_MQ_S_START_ON_RUN is set. For that case, we auto-clear 1282 * the STOPPED bit and run it. 1283 */ 1284 if (test_bit(BLK_MQ_S_STOPPED, &hctx->state)) { 1285 if (!test_bit(BLK_MQ_S_START_ON_RUN, &hctx->state)) 1286 return; 1287 1288 clear_bit(BLK_MQ_S_START_ON_RUN, &hctx->state); 1289 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1290 } 1291 1292 __blk_mq_run_hw_queue(hctx); 1293 } 1294 1295 1296 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) 1297 { 1298 if (unlikely(!blk_mq_hw_queue_mapped(hctx))) 1299 return; 1300 1301 /* 1302 * Stop the hw queue, then modify currently delayed work. 1303 * This should prevent us from running the queue prematurely. 1304 * Mark the queue as auto-clearing STOPPED when it runs. 1305 */ 1306 blk_mq_stop_hw_queue(hctx); 1307 set_bit(BLK_MQ_S_START_ON_RUN, &hctx->state); 1308 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), 1309 &hctx->run_work, 1310 msecs_to_jiffies(msecs)); 1311 } 1312 EXPORT_SYMBOL(blk_mq_delay_queue); 1313 1314 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx, 1315 struct request *rq, 1316 bool at_head) 1317 { 1318 struct blk_mq_ctx *ctx = rq->mq_ctx; 1319 1320 trace_block_rq_insert(hctx->queue, rq); 1321 1322 if (at_head) 1323 list_add(&rq->queuelist, &ctx->rq_list); 1324 else 1325 list_add_tail(&rq->queuelist, &ctx->rq_list); 1326 } 1327 1328 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, 1329 bool at_head) 1330 { 1331 struct blk_mq_ctx *ctx = rq->mq_ctx; 1332 1333 __blk_mq_insert_req_list(hctx, rq, at_head); 1334 blk_mq_hctx_mark_pending(hctx, ctx); 1335 } 1336 1337 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 1338 struct list_head *list) 1339 1340 { 1341 /* 1342 * preemption doesn't flush plug list, so it's possible ctx->cpu is 1343 * offline now 1344 */ 1345 spin_lock(&ctx->lock); 1346 while (!list_empty(list)) { 1347 struct request *rq; 1348 1349 rq = list_first_entry(list, struct request, queuelist); 1350 BUG_ON(rq->mq_ctx != ctx); 1351 list_del_init(&rq->queuelist); 1352 __blk_mq_insert_req_list(hctx, rq, false); 1353 } 1354 blk_mq_hctx_mark_pending(hctx, ctx); 1355 spin_unlock(&ctx->lock); 1356 } 1357 1358 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b) 1359 { 1360 struct request *rqa = container_of(a, struct request, queuelist); 1361 struct request *rqb = container_of(b, struct request, queuelist); 1362 1363 return !(rqa->mq_ctx < rqb->mq_ctx || 1364 (rqa->mq_ctx == rqb->mq_ctx && 1365 blk_rq_pos(rqa) < blk_rq_pos(rqb))); 1366 } 1367 1368 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 1369 { 1370 struct blk_mq_ctx *this_ctx; 1371 struct request_queue *this_q; 1372 struct request *rq; 1373 LIST_HEAD(list); 1374 LIST_HEAD(ctx_list); 1375 unsigned int depth; 1376 1377 list_splice_init(&plug->mq_list, &list); 1378 1379 list_sort(NULL, &list, plug_ctx_cmp); 1380 1381 this_q = NULL; 1382 this_ctx = NULL; 1383 depth = 0; 1384 1385 while (!list_empty(&list)) { 1386 rq = list_entry_rq(list.next); 1387 list_del_init(&rq->queuelist); 1388 BUG_ON(!rq->q); 1389 if (rq->mq_ctx != this_ctx) { 1390 if (this_ctx) { 1391 trace_block_unplug(this_q, depth, from_schedule); 1392 blk_mq_sched_insert_requests(this_q, this_ctx, 1393 &ctx_list, 1394 from_schedule); 1395 } 1396 1397 this_ctx = rq->mq_ctx; 1398 this_q = rq->q; 1399 depth = 0; 1400 } 1401 1402 depth++; 1403 list_add_tail(&rq->queuelist, &ctx_list); 1404 } 1405 1406 /* 1407 * If 'this_ctx' is set, we know we have entries to complete 1408 * on 'ctx_list'. Do those. 1409 */ 1410 if (this_ctx) { 1411 trace_block_unplug(this_q, depth, from_schedule); 1412 blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list, 1413 from_schedule); 1414 } 1415 } 1416 1417 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio) 1418 { 1419 blk_init_request_from_bio(rq, bio); 1420 1421 blk_account_io_start(rq, true); 1422 } 1423 1424 static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx) 1425 { 1426 return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) && 1427 !blk_queue_nomerges(hctx->queue); 1428 } 1429 1430 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx, 1431 struct blk_mq_ctx *ctx, 1432 struct request *rq, struct bio *bio) 1433 { 1434 if (!hctx_allow_merges(hctx) || !bio_mergeable(bio)) { 1435 blk_mq_bio_to_request(rq, bio); 1436 spin_lock(&ctx->lock); 1437 insert_rq: 1438 __blk_mq_insert_request(hctx, rq, false); 1439 spin_unlock(&ctx->lock); 1440 return false; 1441 } else { 1442 struct request_queue *q = hctx->queue; 1443 1444 spin_lock(&ctx->lock); 1445 if (!blk_mq_attempt_merge(q, ctx, bio)) { 1446 blk_mq_bio_to_request(rq, bio); 1447 goto insert_rq; 1448 } 1449 1450 spin_unlock(&ctx->lock); 1451 __blk_mq_finish_request(hctx, ctx, rq); 1452 return true; 1453 } 1454 } 1455 1456 static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq) 1457 { 1458 if (rq->tag != -1) 1459 return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false); 1460 1461 return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true); 1462 } 1463 1464 static void __blk_mq_try_issue_directly(struct request *rq, blk_qc_t *cookie, 1465 bool may_sleep) 1466 { 1467 struct request_queue *q = rq->q; 1468 struct blk_mq_queue_data bd = { 1469 .rq = rq, 1470 .last = true, 1471 }; 1472 struct blk_mq_hw_ctx *hctx; 1473 blk_qc_t new_cookie; 1474 int ret; 1475 1476 if (q->elevator) 1477 goto insert; 1478 1479 if (!blk_mq_get_driver_tag(rq, &hctx, false)) 1480 goto insert; 1481 1482 new_cookie = request_to_qc_t(hctx, rq); 1483 1484 /* 1485 * For OK queue, we are done. For error, kill it. Any other 1486 * error (busy), just add it to our list as we previously 1487 * would have done 1488 */ 1489 ret = q->mq_ops->queue_rq(hctx, &bd); 1490 if (ret == BLK_MQ_RQ_QUEUE_OK) { 1491 *cookie = new_cookie; 1492 return; 1493 } 1494 1495 if (ret == BLK_MQ_RQ_QUEUE_ERROR) { 1496 *cookie = BLK_QC_T_NONE; 1497 blk_mq_end_request(rq, -EIO); 1498 return; 1499 } 1500 1501 __blk_mq_requeue_request(rq); 1502 insert: 1503 blk_mq_sched_insert_request(rq, false, true, false, may_sleep); 1504 } 1505 1506 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 1507 struct request *rq, blk_qc_t *cookie) 1508 { 1509 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { 1510 rcu_read_lock(); 1511 __blk_mq_try_issue_directly(rq, cookie, false); 1512 rcu_read_unlock(); 1513 } else { 1514 unsigned int srcu_idx; 1515 1516 might_sleep(); 1517 1518 srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu); 1519 __blk_mq_try_issue_directly(rq, cookie, true); 1520 srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx); 1521 } 1522 } 1523 1524 static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) 1525 { 1526 const int is_sync = op_is_sync(bio->bi_opf); 1527 const int is_flush_fua = op_is_flush(bio->bi_opf); 1528 struct blk_mq_alloc_data data = { .flags = 0 }; 1529 struct request *rq; 1530 unsigned int request_count = 0; 1531 struct blk_plug *plug; 1532 struct request *same_queue_rq = NULL; 1533 blk_qc_t cookie; 1534 unsigned int wb_acct; 1535 1536 blk_queue_bounce(q, &bio); 1537 1538 blk_queue_split(q, &bio, q->bio_split); 1539 1540 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 1541 bio_io_error(bio); 1542 return BLK_QC_T_NONE; 1543 } 1544 1545 if (!is_flush_fua && !blk_queue_nomerges(q) && 1546 blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq)) 1547 return BLK_QC_T_NONE; 1548 1549 if (blk_mq_sched_bio_merge(q, bio)) 1550 return BLK_QC_T_NONE; 1551 1552 wb_acct = wbt_wait(q->rq_wb, bio, NULL); 1553 1554 trace_block_getrq(q, bio, bio->bi_opf); 1555 1556 rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data); 1557 if (unlikely(!rq)) { 1558 __wbt_done(q->rq_wb, wb_acct); 1559 return BLK_QC_T_NONE; 1560 } 1561 1562 wbt_track(&rq->issue_stat, wb_acct); 1563 1564 cookie = request_to_qc_t(data.hctx, rq); 1565 1566 plug = current->plug; 1567 if (unlikely(is_flush_fua)) { 1568 blk_mq_put_ctx(data.ctx); 1569 blk_mq_bio_to_request(rq, bio); 1570 if (q->elevator) { 1571 blk_mq_sched_insert_request(rq, false, true, true, 1572 true); 1573 } else { 1574 blk_insert_flush(rq); 1575 blk_mq_run_hw_queue(data.hctx, true); 1576 } 1577 } else if (plug && q->nr_hw_queues == 1) { 1578 struct request *last = NULL; 1579 1580 blk_mq_put_ctx(data.ctx); 1581 blk_mq_bio_to_request(rq, bio); 1582 1583 /* 1584 * @request_count may become stale because of schedule 1585 * out, so check the list again. 1586 */ 1587 if (list_empty(&plug->mq_list)) 1588 request_count = 0; 1589 else if (blk_queue_nomerges(q)) 1590 request_count = blk_plug_queued_count(q); 1591 1592 if (!request_count) 1593 trace_block_plug(q); 1594 else 1595 last = list_entry_rq(plug->mq_list.prev); 1596 1597 if (request_count >= BLK_MAX_REQUEST_COUNT || (last && 1598 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { 1599 blk_flush_plug_list(plug, false); 1600 trace_block_plug(q); 1601 } 1602 1603 list_add_tail(&rq->queuelist, &plug->mq_list); 1604 } else if (plug && !blk_queue_nomerges(q)) { 1605 blk_mq_bio_to_request(rq, bio); 1606 1607 /* 1608 * We do limited plugging. If the bio can be merged, do that. 1609 * Otherwise the existing request in the plug list will be 1610 * issued. So the plug list will have one request at most 1611 * The plug list might get flushed before this. If that happens, 1612 * the plug list is empty, and same_queue_rq is invalid. 1613 */ 1614 if (list_empty(&plug->mq_list)) 1615 same_queue_rq = NULL; 1616 if (same_queue_rq) 1617 list_del_init(&same_queue_rq->queuelist); 1618 list_add_tail(&rq->queuelist, &plug->mq_list); 1619 1620 blk_mq_put_ctx(data.ctx); 1621 1622 if (same_queue_rq) 1623 blk_mq_try_issue_directly(data.hctx, same_queue_rq, 1624 &cookie); 1625 } else if (q->nr_hw_queues > 1 && is_sync) { 1626 blk_mq_put_ctx(data.ctx); 1627 blk_mq_bio_to_request(rq, bio); 1628 blk_mq_try_issue_directly(data.hctx, rq, &cookie); 1629 } else if (q->elevator) { 1630 blk_mq_put_ctx(data.ctx); 1631 blk_mq_bio_to_request(rq, bio); 1632 blk_mq_sched_insert_request(rq, false, true, true, true); 1633 } else if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) { 1634 blk_mq_put_ctx(data.ctx); 1635 blk_mq_run_hw_queue(data.hctx, true); 1636 } else 1637 blk_mq_put_ctx(data.ctx); 1638 1639 return cookie; 1640 } 1641 1642 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 1643 unsigned int hctx_idx) 1644 { 1645 struct page *page; 1646 1647 if (tags->rqs && set->ops->exit_request) { 1648 int i; 1649 1650 for (i = 0; i < tags->nr_tags; i++) { 1651 struct request *rq = tags->static_rqs[i]; 1652 1653 if (!rq) 1654 continue; 1655 set->ops->exit_request(set, rq, hctx_idx); 1656 tags->static_rqs[i] = NULL; 1657 } 1658 } 1659 1660 while (!list_empty(&tags->page_list)) { 1661 page = list_first_entry(&tags->page_list, struct page, lru); 1662 list_del_init(&page->lru); 1663 /* 1664 * Remove kmemleak object previously allocated in 1665 * blk_mq_init_rq_map(). 1666 */ 1667 kmemleak_free(page_address(page)); 1668 __free_pages(page, page->private); 1669 } 1670 } 1671 1672 void blk_mq_free_rq_map(struct blk_mq_tags *tags) 1673 { 1674 kfree(tags->rqs); 1675 tags->rqs = NULL; 1676 kfree(tags->static_rqs); 1677 tags->static_rqs = NULL; 1678 1679 blk_mq_free_tags(tags); 1680 } 1681 1682 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, 1683 unsigned int hctx_idx, 1684 unsigned int nr_tags, 1685 unsigned int reserved_tags) 1686 { 1687 struct blk_mq_tags *tags; 1688 int node; 1689 1690 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx); 1691 if (node == NUMA_NO_NODE) 1692 node = set->numa_node; 1693 1694 tags = blk_mq_init_tags(nr_tags, reserved_tags, node, 1695 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags)); 1696 if (!tags) 1697 return NULL; 1698 1699 tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *), 1700 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 1701 node); 1702 if (!tags->rqs) { 1703 blk_mq_free_tags(tags); 1704 return NULL; 1705 } 1706 1707 tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *), 1708 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 1709 node); 1710 if (!tags->static_rqs) { 1711 kfree(tags->rqs); 1712 blk_mq_free_tags(tags); 1713 return NULL; 1714 } 1715 1716 return tags; 1717 } 1718 1719 static size_t order_to_size(unsigned int order) 1720 { 1721 return (size_t)PAGE_SIZE << order; 1722 } 1723 1724 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 1725 unsigned int hctx_idx, unsigned int depth) 1726 { 1727 unsigned int i, j, entries_per_page, max_order = 4; 1728 size_t rq_size, left; 1729 int node; 1730 1731 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx); 1732 if (node == NUMA_NO_NODE) 1733 node = set->numa_node; 1734 1735 INIT_LIST_HEAD(&tags->page_list); 1736 1737 /* 1738 * rq_size is the size of the request plus driver payload, rounded 1739 * to the cacheline size 1740 */ 1741 rq_size = round_up(sizeof(struct request) + set->cmd_size, 1742 cache_line_size()); 1743 left = rq_size * depth; 1744 1745 for (i = 0; i < depth; ) { 1746 int this_order = max_order; 1747 struct page *page; 1748 int to_do; 1749 void *p; 1750 1751 while (this_order && left < order_to_size(this_order - 1)) 1752 this_order--; 1753 1754 do { 1755 page = alloc_pages_node(node, 1756 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, 1757 this_order); 1758 if (page) 1759 break; 1760 if (!this_order--) 1761 break; 1762 if (order_to_size(this_order) < rq_size) 1763 break; 1764 } while (1); 1765 1766 if (!page) 1767 goto fail; 1768 1769 page->private = this_order; 1770 list_add_tail(&page->lru, &tags->page_list); 1771 1772 p = page_address(page); 1773 /* 1774 * Allow kmemleak to scan these pages as they contain pointers 1775 * to additional allocations like via ops->init_request(). 1776 */ 1777 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); 1778 entries_per_page = order_to_size(this_order) / rq_size; 1779 to_do = min(entries_per_page, depth - i); 1780 left -= to_do * rq_size; 1781 for (j = 0; j < to_do; j++) { 1782 struct request *rq = p; 1783 1784 tags->static_rqs[i] = rq; 1785 if (set->ops->init_request) { 1786 if (set->ops->init_request(set, rq, hctx_idx, 1787 node)) { 1788 tags->static_rqs[i] = NULL; 1789 goto fail; 1790 } 1791 } 1792 1793 p += rq_size; 1794 i++; 1795 } 1796 } 1797 return 0; 1798 1799 fail: 1800 blk_mq_free_rqs(set, tags, hctx_idx); 1801 return -ENOMEM; 1802 } 1803 1804 /* 1805 * 'cpu' is going away. splice any existing rq_list entries from this 1806 * software queue to the hw queue dispatch list, and ensure that it 1807 * gets run. 1808 */ 1809 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) 1810 { 1811 struct blk_mq_hw_ctx *hctx; 1812 struct blk_mq_ctx *ctx; 1813 LIST_HEAD(tmp); 1814 1815 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead); 1816 ctx = __blk_mq_get_ctx(hctx->queue, cpu); 1817 1818 spin_lock(&ctx->lock); 1819 if (!list_empty(&ctx->rq_list)) { 1820 list_splice_init(&ctx->rq_list, &tmp); 1821 blk_mq_hctx_clear_pending(hctx, ctx); 1822 } 1823 spin_unlock(&ctx->lock); 1824 1825 if (list_empty(&tmp)) 1826 return 0; 1827 1828 spin_lock(&hctx->lock); 1829 list_splice_tail_init(&tmp, &hctx->dispatch); 1830 spin_unlock(&hctx->lock); 1831 1832 blk_mq_run_hw_queue(hctx, true); 1833 return 0; 1834 } 1835 1836 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) 1837 { 1838 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD, 1839 &hctx->cpuhp_dead); 1840 } 1841 1842 /* hctx->ctxs will be freed in queue's release handler */ 1843 static void blk_mq_exit_hctx(struct request_queue *q, 1844 struct blk_mq_tag_set *set, 1845 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 1846 { 1847 blk_mq_debugfs_unregister_hctx(hctx); 1848 1849 blk_mq_tag_idle(hctx); 1850 1851 if (set->ops->exit_request) 1852 set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx); 1853 1854 blk_mq_sched_exit_hctx(q, hctx, hctx_idx); 1855 1856 if (set->ops->exit_hctx) 1857 set->ops->exit_hctx(hctx, hctx_idx); 1858 1859 if (hctx->flags & BLK_MQ_F_BLOCKING) 1860 cleanup_srcu_struct(&hctx->queue_rq_srcu); 1861 1862 blk_mq_remove_cpuhp(hctx); 1863 blk_free_flush_queue(hctx->fq); 1864 sbitmap_free(&hctx->ctx_map); 1865 } 1866 1867 static void blk_mq_exit_hw_queues(struct request_queue *q, 1868 struct blk_mq_tag_set *set, int nr_queue) 1869 { 1870 struct blk_mq_hw_ctx *hctx; 1871 unsigned int i; 1872 1873 queue_for_each_hw_ctx(q, hctx, i) { 1874 if (i == nr_queue) 1875 break; 1876 blk_mq_exit_hctx(q, set, hctx, i); 1877 } 1878 } 1879 1880 static int blk_mq_init_hctx(struct request_queue *q, 1881 struct blk_mq_tag_set *set, 1882 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) 1883 { 1884 int node; 1885 1886 node = hctx->numa_node; 1887 if (node == NUMA_NO_NODE) 1888 node = hctx->numa_node = set->numa_node; 1889 1890 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); 1891 spin_lock_init(&hctx->lock); 1892 INIT_LIST_HEAD(&hctx->dispatch); 1893 hctx->queue = q; 1894 hctx->queue_num = hctx_idx; 1895 hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED; 1896 1897 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead); 1898 1899 hctx->tags = set->tags[hctx_idx]; 1900 1901 /* 1902 * Allocate space for all possible cpus to avoid allocation at 1903 * runtime 1904 */ 1905 hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *), 1906 GFP_KERNEL, node); 1907 if (!hctx->ctxs) 1908 goto unregister_cpu_notifier; 1909 1910 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL, 1911 node)) 1912 goto free_ctxs; 1913 1914 hctx->nr_ctx = 0; 1915 1916 if (set->ops->init_hctx && 1917 set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) 1918 goto free_bitmap; 1919 1920 if (blk_mq_sched_init_hctx(q, hctx, hctx_idx)) 1921 goto exit_hctx; 1922 1923 hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size); 1924 if (!hctx->fq) 1925 goto sched_exit_hctx; 1926 1927 if (set->ops->init_request && 1928 set->ops->init_request(set, hctx->fq->flush_rq, hctx_idx, 1929 node)) 1930 goto free_fq; 1931 1932 if (hctx->flags & BLK_MQ_F_BLOCKING) 1933 init_srcu_struct(&hctx->queue_rq_srcu); 1934 1935 blk_mq_debugfs_register_hctx(q, hctx); 1936 1937 return 0; 1938 1939 free_fq: 1940 kfree(hctx->fq); 1941 sched_exit_hctx: 1942 blk_mq_sched_exit_hctx(q, hctx, hctx_idx); 1943 exit_hctx: 1944 if (set->ops->exit_hctx) 1945 set->ops->exit_hctx(hctx, hctx_idx); 1946 free_bitmap: 1947 sbitmap_free(&hctx->ctx_map); 1948 free_ctxs: 1949 kfree(hctx->ctxs); 1950 unregister_cpu_notifier: 1951 blk_mq_remove_cpuhp(hctx); 1952 return -1; 1953 } 1954 1955 static void blk_mq_init_cpu_queues(struct request_queue *q, 1956 unsigned int nr_hw_queues) 1957 { 1958 unsigned int i; 1959 1960 for_each_possible_cpu(i) { 1961 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i); 1962 struct blk_mq_hw_ctx *hctx; 1963 1964 __ctx->cpu = i; 1965 spin_lock_init(&__ctx->lock); 1966 INIT_LIST_HEAD(&__ctx->rq_list); 1967 __ctx->queue = q; 1968 1969 /* If the cpu isn't online, the cpu is mapped to first hctx */ 1970 if (!cpu_online(i)) 1971 continue; 1972 1973 hctx = blk_mq_map_queue(q, i); 1974 1975 /* 1976 * Set local node, IFF we have more than one hw queue. If 1977 * not, we remain on the home node of the device 1978 */ 1979 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) 1980 hctx->numa_node = local_memory_node(cpu_to_node(i)); 1981 } 1982 } 1983 1984 static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx) 1985 { 1986 int ret = 0; 1987 1988 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx, 1989 set->queue_depth, set->reserved_tags); 1990 if (!set->tags[hctx_idx]) 1991 return false; 1992 1993 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx, 1994 set->queue_depth); 1995 if (!ret) 1996 return true; 1997 1998 blk_mq_free_rq_map(set->tags[hctx_idx]); 1999 set->tags[hctx_idx] = NULL; 2000 return false; 2001 } 2002 2003 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set, 2004 unsigned int hctx_idx) 2005 { 2006 if (set->tags[hctx_idx]) { 2007 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx); 2008 blk_mq_free_rq_map(set->tags[hctx_idx]); 2009 set->tags[hctx_idx] = NULL; 2010 } 2011 } 2012 2013 static void blk_mq_map_swqueue(struct request_queue *q, 2014 const struct cpumask *online_mask) 2015 { 2016 unsigned int i, hctx_idx; 2017 struct blk_mq_hw_ctx *hctx; 2018 struct blk_mq_ctx *ctx; 2019 struct blk_mq_tag_set *set = q->tag_set; 2020 2021 /* 2022 * Avoid others reading imcomplete hctx->cpumask through sysfs 2023 */ 2024 mutex_lock(&q->sysfs_lock); 2025 2026 queue_for_each_hw_ctx(q, hctx, i) { 2027 cpumask_clear(hctx->cpumask); 2028 hctx->nr_ctx = 0; 2029 } 2030 2031 /* 2032 * Map software to hardware queues 2033 */ 2034 for_each_possible_cpu(i) { 2035 /* If the cpu isn't online, the cpu is mapped to first hctx */ 2036 if (!cpumask_test_cpu(i, online_mask)) 2037 continue; 2038 2039 hctx_idx = q->mq_map[i]; 2040 /* unmapped hw queue can be remapped after CPU topo changed */ 2041 if (!set->tags[hctx_idx] && 2042 !__blk_mq_alloc_rq_map(set, hctx_idx)) { 2043 /* 2044 * If tags initialization fail for some hctx, 2045 * that hctx won't be brought online. In this 2046 * case, remap the current ctx to hctx[0] which 2047 * is guaranteed to always have tags allocated 2048 */ 2049 q->mq_map[i] = 0; 2050 } 2051 2052 ctx = per_cpu_ptr(q->queue_ctx, i); 2053 hctx = blk_mq_map_queue(q, i); 2054 2055 cpumask_set_cpu(i, hctx->cpumask); 2056 ctx->index_hw = hctx->nr_ctx; 2057 hctx->ctxs[hctx->nr_ctx++] = ctx; 2058 } 2059 2060 mutex_unlock(&q->sysfs_lock); 2061 2062 queue_for_each_hw_ctx(q, hctx, i) { 2063 /* 2064 * If no software queues are mapped to this hardware queue, 2065 * disable it and free the request entries. 2066 */ 2067 if (!hctx->nr_ctx) { 2068 /* Never unmap queue 0. We need it as a 2069 * fallback in case of a new remap fails 2070 * allocation 2071 */ 2072 if (i && set->tags[i]) 2073 blk_mq_free_map_and_requests(set, i); 2074 2075 hctx->tags = NULL; 2076 continue; 2077 } 2078 2079 hctx->tags = set->tags[i]; 2080 WARN_ON(!hctx->tags); 2081 2082 /* 2083 * Set the map size to the number of mapped software queues. 2084 * This is more accurate and more efficient than looping 2085 * over all possibly mapped software queues. 2086 */ 2087 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx); 2088 2089 /* 2090 * Initialize batch roundrobin counts 2091 */ 2092 hctx->next_cpu = cpumask_first(hctx->cpumask); 2093 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 2094 } 2095 } 2096 2097 static void queue_set_hctx_shared(struct request_queue *q, bool shared) 2098 { 2099 struct blk_mq_hw_ctx *hctx; 2100 int i; 2101 2102 queue_for_each_hw_ctx(q, hctx, i) { 2103 if (shared) 2104 hctx->flags |= BLK_MQ_F_TAG_SHARED; 2105 else 2106 hctx->flags &= ~BLK_MQ_F_TAG_SHARED; 2107 } 2108 } 2109 2110 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared) 2111 { 2112 struct request_queue *q; 2113 2114 lockdep_assert_held(&set->tag_list_lock); 2115 2116 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2117 blk_mq_freeze_queue(q); 2118 queue_set_hctx_shared(q, shared); 2119 blk_mq_unfreeze_queue(q); 2120 } 2121 } 2122 2123 static void blk_mq_del_queue_tag_set(struct request_queue *q) 2124 { 2125 struct blk_mq_tag_set *set = q->tag_set; 2126 2127 mutex_lock(&set->tag_list_lock); 2128 list_del_rcu(&q->tag_set_list); 2129 INIT_LIST_HEAD(&q->tag_set_list); 2130 if (list_is_singular(&set->tag_list)) { 2131 /* just transitioned to unshared */ 2132 set->flags &= ~BLK_MQ_F_TAG_SHARED; 2133 /* update existing queue */ 2134 blk_mq_update_tag_set_depth(set, false); 2135 } 2136 mutex_unlock(&set->tag_list_lock); 2137 2138 synchronize_rcu(); 2139 } 2140 2141 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, 2142 struct request_queue *q) 2143 { 2144 q->tag_set = set; 2145 2146 mutex_lock(&set->tag_list_lock); 2147 2148 /* Check to see if we're transitioning to shared (from 1 to 2 queues). */ 2149 if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) { 2150 set->flags |= BLK_MQ_F_TAG_SHARED; 2151 /* update existing queue */ 2152 blk_mq_update_tag_set_depth(set, true); 2153 } 2154 if (set->flags & BLK_MQ_F_TAG_SHARED) 2155 queue_set_hctx_shared(q, true); 2156 list_add_tail_rcu(&q->tag_set_list, &set->tag_list); 2157 2158 mutex_unlock(&set->tag_list_lock); 2159 } 2160 2161 /* 2162 * It is the actual release handler for mq, but we do it from 2163 * request queue's release handler for avoiding use-after-free 2164 * and headache because q->mq_kobj shouldn't have been introduced, 2165 * but we can't group ctx/kctx kobj without it. 2166 */ 2167 void blk_mq_release(struct request_queue *q) 2168 { 2169 struct blk_mq_hw_ctx *hctx; 2170 unsigned int i; 2171 2172 /* hctx kobj stays in hctx */ 2173 queue_for_each_hw_ctx(q, hctx, i) { 2174 if (!hctx) 2175 continue; 2176 kobject_put(&hctx->kobj); 2177 } 2178 2179 q->mq_map = NULL; 2180 2181 kfree(q->queue_hw_ctx); 2182 2183 /* 2184 * release .mq_kobj and sw queue's kobject now because 2185 * both share lifetime with request queue. 2186 */ 2187 blk_mq_sysfs_deinit(q); 2188 2189 free_percpu(q->queue_ctx); 2190 } 2191 2192 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) 2193 { 2194 struct request_queue *uninit_q, *q; 2195 2196 uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node); 2197 if (!uninit_q) 2198 return ERR_PTR(-ENOMEM); 2199 2200 q = blk_mq_init_allocated_queue(set, uninit_q); 2201 if (IS_ERR(q)) 2202 blk_cleanup_queue(uninit_q); 2203 2204 return q; 2205 } 2206 EXPORT_SYMBOL(blk_mq_init_queue); 2207 2208 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, 2209 struct request_queue *q) 2210 { 2211 int i, j; 2212 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; 2213 2214 blk_mq_sysfs_unregister(q); 2215 for (i = 0; i < set->nr_hw_queues; i++) { 2216 int node; 2217 2218 if (hctxs[i]) 2219 continue; 2220 2221 node = blk_mq_hw_queue_to_node(q->mq_map, i); 2222 hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx), 2223 GFP_KERNEL, node); 2224 if (!hctxs[i]) 2225 break; 2226 2227 if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL, 2228 node)) { 2229 kfree(hctxs[i]); 2230 hctxs[i] = NULL; 2231 break; 2232 } 2233 2234 atomic_set(&hctxs[i]->nr_active, 0); 2235 hctxs[i]->numa_node = node; 2236 hctxs[i]->queue_num = i; 2237 2238 if (blk_mq_init_hctx(q, set, hctxs[i], i)) { 2239 free_cpumask_var(hctxs[i]->cpumask); 2240 kfree(hctxs[i]); 2241 hctxs[i] = NULL; 2242 break; 2243 } 2244 blk_mq_hctx_kobj_init(hctxs[i]); 2245 } 2246 for (j = i; j < q->nr_hw_queues; j++) { 2247 struct blk_mq_hw_ctx *hctx = hctxs[j]; 2248 2249 if (hctx) { 2250 if (hctx->tags) 2251 blk_mq_free_map_and_requests(set, j); 2252 blk_mq_exit_hctx(q, set, hctx, j); 2253 kobject_put(&hctx->kobj); 2254 hctxs[j] = NULL; 2255 2256 } 2257 } 2258 q->nr_hw_queues = i; 2259 blk_mq_sysfs_register(q); 2260 } 2261 2262 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 2263 struct request_queue *q) 2264 { 2265 /* mark the queue as mq asap */ 2266 q->mq_ops = set->ops; 2267 2268 q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn, 2269 blk_mq_poll_stats_bkt, 2270 BLK_MQ_POLL_STATS_BKTS, q); 2271 if (!q->poll_cb) 2272 goto err_exit; 2273 2274 q->queue_ctx = alloc_percpu(struct blk_mq_ctx); 2275 if (!q->queue_ctx) 2276 goto err_exit; 2277 2278 /* init q->mq_kobj and sw queues' kobjects */ 2279 blk_mq_sysfs_init(q); 2280 2281 q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)), 2282 GFP_KERNEL, set->numa_node); 2283 if (!q->queue_hw_ctx) 2284 goto err_percpu; 2285 2286 q->mq_map = set->mq_map; 2287 2288 blk_mq_realloc_hw_ctxs(set, q); 2289 if (!q->nr_hw_queues) 2290 goto err_hctxs; 2291 2292 INIT_WORK(&q->timeout_work, blk_mq_timeout_work); 2293 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ); 2294 2295 q->nr_queues = nr_cpu_ids; 2296 2297 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; 2298 2299 if (!(set->flags & BLK_MQ_F_SG_MERGE)) 2300 q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE; 2301 2302 q->sg_reserved_size = INT_MAX; 2303 2304 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); 2305 INIT_LIST_HEAD(&q->requeue_list); 2306 spin_lock_init(&q->requeue_lock); 2307 2308 blk_queue_make_request(q, blk_mq_make_request); 2309 2310 /* 2311 * Do this after blk_queue_make_request() overrides it... 2312 */ 2313 q->nr_requests = set->queue_depth; 2314 2315 /* 2316 * Default to classic polling 2317 */ 2318 q->poll_nsec = -1; 2319 2320 if (set->ops->complete) 2321 blk_queue_softirq_done(q, set->ops->complete); 2322 2323 blk_mq_init_cpu_queues(q, set->nr_hw_queues); 2324 2325 get_online_cpus(); 2326 mutex_lock(&all_q_mutex); 2327 2328 list_add_tail(&q->all_q_node, &all_q_list); 2329 blk_mq_add_queue_tag_set(set, q); 2330 blk_mq_map_swqueue(q, cpu_online_mask); 2331 2332 mutex_unlock(&all_q_mutex); 2333 put_online_cpus(); 2334 2335 if (!(set->flags & BLK_MQ_F_NO_SCHED)) { 2336 int ret; 2337 2338 ret = blk_mq_sched_init(q); 2339 if (ret) 2340 return ERR_PTR(ret); 2341 } 2342 2343 return q; 2344 2345 err_hctxs: 2346 kfree(q->queue_hw_ctx); 2347 err_percpu: 2348 free_percpu(q->queue_ctx); 2349 err_exit: 2350 q->mq_ops = NULL; 2351 return ERR_PTR(-ENOMEM); 2352 } 2353 EXPORT_SYMBOL(blk_mq_init_allocated_queue); 2354 2355 void blk_mq_free_queue(struct request_queue *q) 2356 { 2357 struct blk_mq_tag_set *set = q->tag_set; 2358 2359 mutex_lock(&all_q_mutex); 2360 list_del_init(&q->all_q_node); 2361 mutex_unlock(&all_q_mutex); 2362 2363 blk_mq_del_queue_tag_set(q); 2364 2365 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); 2366 } 2367 2368 /* Basically redo blk_mq_init_queue with queue frozen */ 2369 static void blk_mq_queue_reinit(struct request_queue *q, 2370 const struct cpumask *online_mask) 2371 { 2372 WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth)); 2373 2374 blk_mq_debugfs_unregister_hctxs(q); 2375 blk_mq_sysfs_unregister(q); 2376 2377 /* 2378 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe 2379 * we should change hctx numa_node according to new topology (this 2380 * involves free and re-allocate memory, worthy doing?) 2381 */ 2382 2383 blk_mq_map_swqueue(q, online_mask); 2384 2385 blk_mq_sysfs_register(q); 2386 blk_mq_debugfs_register_hctxs(q); 2387 } 2388 2389 /* 2390 * New online cpumask which is going to be set in this hotplug event. 2391 * Declare this cpumasks as global as cpu-hotplug operation is invoked 2392 * one-by-one and dynamically allocating this could result in a failure. 2393 */ 2394 static struct cpumask cpuhp_online_new; 2395 2396 static void blk_mq_queue_reinit_work(void) 2397 { 2398 struct request_queue *q; 2399 2400 mutex_lock(&all_q_mutex); 2401 /* 2402 * We need to freeze and reinit all existing queues. Freezing 2403 * involves synchronous wait for an RCU grace period and doing it 2404 * one by one may take a long time. Start freezing all queues in 2405 * one swoop and then wait for the completions so that freezing can 2406 * take place in parallel. 2407 */ 2408 list_for_each_entry(q, &all_q_list, all_q_node) 2409 blk_freeze_queue_start(q); 2410 list_for_each_entry(q, &all_q_list, all_q_node) 2411 blk_mq_freeze_queue_wait(q); 2412 2413 list_for_each_entry(q, &all_q_list, all_q_node) 2414 blk_mq_queue_reinit(q, &cpuhp_online_new); 2415 2416 list_for_each_entry(q, &all_q_list, all_q_node) 2417 blk_mq_unfreeze_queue(q); 2418 2419 mutex_unlock(&all_q_mutex); 2420 } 2421 2422 static int blk_mq_queue_reinit_dead(unsigned int cpu) 2423 { 2424 cpumask_copy(&cpuhp_online_new, cpu_online_mask); 2425 blk_mq_queue_reinit_work(); 2426 return 0; 2427 } 2428 2429 /* 2430 * Before hotadded cpu starts handling requests, new mappings must be 2431 * established. Otherwise, these requests in hw queue might never be 2432 * dispatched. 2433 * 2434 * For example, there is a single hw queue (hctx) and two CPU queues (ctx0 2435 * for CPU0, and ctx1 for CPU1). 2436 * 2437 * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list 2438 * and set bit0 in pending bitmap as ctx1->index_hw is still zero. 2439 * 2440 * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set 2441 * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list. 2442 * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is 2443 * ignored. 2444 */ 2445 static int blk_mq_queue_reinit_prepare(unsigned int cpu) 2446 { 2447 cpumask_copy(&cpuhp_online_new, cpu_online_mask); 2448 cpumask_set_cpu(cpu, &cpuhp_online_new); 2449 blk_mq_queue_reinit_work(); 2450 return 0; 2451 } 2452 2453 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2454 { 2455 int i; 2456 2457 for (i = 0; i < set->nr_hw_queues; i++) 2458 if (!__blk_mq_alloc_rq_map(set, i)) 2459 goto out_unwind; 2460 2461 return 0; 2462 2463 out_unwind: 2464 while (--i >= 0) 2465 blk_mq_free_rq_map(set->tags[i]); 2466 2467 return -ENOMEM; 2468 } 2469 2470 /* 2471 * Allocate the request maps associated with this tag_set. Note that this 2472 * may reduce the depth asked for, if memory is tight. set->queue_depth 2473 * will be updated to reflect the allocated depth. 2474 */ 2475 static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2476 { 2477 unsigned int depth; 2478 int err; 2479 2480 depth = set->queue_depth; 2481 do { 2482 err = __blk_mq_alloc_rq_maps(set); 2483 if (!err) 2484 break; 2485 2486 set->queue_depth >>= 1; 2487 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) { 2488 err = -ENOMEM; 2489 break; 2490 } 2491 } while (set->queue_depth); 2492 2493 if (!set->queue_depth || err) { 2494 pr_err("blk-mq: failed to allocate request map\n"); 2495 return -ENOMEM; 2496 } 2497 2498 if (depth != set->queue_depth) 2499 pr_info("blk-mq: reduced tag depth (%u -> %u)\n", 2500 depth, set->queue_depth); 2501 2502 return 0; 2503 } 2504 2505 static int blk_mq_update_queue_map(struct blk_mq_tag_set *set) 2506 { 2507 if (set->ops->map_queues) 2508 return set->ops->map_queues(set); 2509 else 2510 return blk_mq_map_queues(set); 2511 } 2512 2513 /* 2514 * Alloc a tag set to be associated with one or more request queues. 2515 * May fail with EINVAL for various error conditions. May adjust the 2516 * requested depth down, if if it too large. In that case, the set 2517 * value will be stored in set->queue_depth. 2518 */ 2519 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) 2520 { 2521 int ret; 2522 2523 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS); 2524 2525 if (!set->nr_hw_queues) 2526 return -EINVAL; 2527 if (!set->queue_depth) 2528 return -EINVAL; 2529 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) 2530 return -EINVAL; 2531 2532 if (!set->ops->queue_rq) 2533 return -EINVAL; 2534 2535 if (set->queue_depth > BLK_MQ_MAX_DEPTH) { 2536 pr_info("blk-mq: reduced tag depth to %u\n", 2537 BLK_MQ_MAX_DEPTH); 2538 set->queue_depth = BLK_MQ_MAX_DEPTH; 2539 } 2540 2541 /* 2542 * If a crashdump is active, then we are potentially in a very 2543 * memory constrained environment. Limit us to 1 queue and 2544 * 64 tags to prevent using too much memory. 2545 */ 2546 if (is_kdump_kernel()) { 2547 set->nr_hw_queues = 1; 2548 set->queue_depth = min(64U, set->queue_depth); 2549 } 2550 /* 2551 * There is no use for more h/w queues than cpus. 2552 */ 2553 if (set->nr_hw_queues > nr_cpu_ids) 2554 set->nr_hw_queues = nr_cpu_ids; 2555 2556 set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *), 2557 GFP_KERNEL, set->numa_node); 2558 if (!set->tags) 2559 return -ENOMEM; 2560 2561 ret = -ENOMEM; 2562 set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids, 2563 GFP_KERNEL, set->numa_node); 2564 if (!set->mq_map) 2565 goto out_free_tags; 2566 2567 ret = blk_mq_update_queue_map(set); 2568 if (ret) 2569 goto out_free_mq_map; 2570 2571 ret = blk_mq_alloc_rq_maps(set); 2572 if (ret) 2573 goto out_free_mq_map; 2574 2575 mutex_init(&set->tag_list_lock); 2576 INIT_LIST_HEAD(&set->tag_list); 2577 2578 return 0; 2579 2580 out_free_mq_map: 2581 kfree(set->mq_map); 2582 set->mq_map = NULL; 2583 out_free_tags: 2584 kfree(set->tags); 2585 set->tags = NULL; 2586 return ret; 2587 } 2588 EXPORT_SYMBOL(blk_mq_alloc_tag_set); 2589 2590 void blk_mq_free_tag_set(struct blk_mq_tag_set *set) 2591 { 2592 int i; 2593 2594 for (i = 0; i < nr_cpu_ids; i++) 2595 blk_mq_free_map_and_requests(set, i); 2596 2597 kfree(set->mq_map); 2598 set->mq_map = NULL; 2599 2600 kfree(set->tags); 2601 set->tags = NULL; 2602 } 2603 EXPORT_SYMBOL(blk_mq_free_tag_set); 2604 2605 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) 2606 { 2607 struct blk_mq_tag_set *set = q->tag_set; 2608 struct blk_mq_hw_ctx *hctx; 2609 int i, ret; 2610 2611 if (!set) 2612 return -EINVAL; 2613 2614 blk_mq_freeze_queue(q); 2615 2616 ret = 0; 2617 queue_for_each_hw_ctx(q, hctx, i) { 2618 if (!hctx->tags) 2619 continue; 2620 /* 2621 * If we're using an MQ scheduler, just update the scheduler 2622 * queue depth. This is similar to what the old code would do. 2623 */ 2624 if (!hctx->sched_tags) { 2625 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, 2626 min(nr, set->queue_depth), 2627 false); 2628 } else { 2629 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags, 2630 nr, true); 2631 } 2632 if (ret) 2633 break; 2634 } 2635 2636 if (!ret) 2637 q->nr_requests = nr; 2638 2639 blk_mq_unfreeze_queue(q); 2640 2641 return ret; 2642 } 2643 2644 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, 2645 int nr_hw_queues) 2646 { 2647 struct request_queue *q; 2648 2649 lockdep_assert_held(&set->tag_list_lock); 2650 2651 if (nr_hw_queues > nr_cpu_ids) 2652 nr_hw_queues = nr_cpu_ids; 2653 if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues) 2654 return; 2655 2656 list_for_each_entry(q, &set->tag_list, tag_set_list) 2657 blk_mq_freeze_queue(q); 2658 2659 set->nr_hw_queues = nr_hw_queues; 2660 blk_mq_update_queue_map(set); 2661 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2662 blk_mq_realloc_hw_ctxs(set, q); 2663 blk_mq_queue_reinit(q, cpu_online_mask); 2664 } 2665 2666 list_for_each_entry(q, &set->tag_list, tag_set_list) 2667 blk_mq_unfreeze_queue(q); 2668 } 2669 2670 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) 2671 { 2672 mutex_lock(&set->tag_list_lock); 2673 __blk_mq_update_nr_hw_queues(set, nr_hw_queues); 2674 mutex_unlock(&set->tag_list_lock); 2675 } 2676 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); 2677 2678 /* Enable polling stats and return whether they were already enabled. */ 2679 static bool blk_poll_stats_enable(struct request_queue *q) 2680 { 2681 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 2682 test_and_set_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags)) 2683 return true; 2684 blk_stat_add_callback(q, q->poll_cb); 2685 return false; 2686 } 2687 2688 static void blk_mq_poll_stats_start(struct request_queue *q) 2689 { 2690 /* 2691 * We don't arm the callback if polling stats are not enabled or the 2692 * callback is already active. 2693 */ 2694 if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 2695 blk_stat_is_active(q->poll_cb)) 2696 return; 2697 2698 blk_stat_activate_msecs(q->poll_cb, 100); 2699 } 2700 2701 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb) 2702 { 2703 struct request_queue *q = cb->data; 2704 int bucket; 2705 2706 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) { 2707 if (cb->stat[bucket].nr_samples) 2708 q->poll_stat[bucket] = cb->stat[bucket]; 2709 } 2710 } 2711 2712 static unsigned long blk_mq_poll_nsecs(struct request_queue *q, 2713 struct blk_mq_hw_ctx *hctx, 2714 struct request *rq) 2715 { 2716 unsigned long ret = 0; 2717 int bucket; 2718 2719 /* 2720 * If stats collection isn't on, don't sleep but turn it on for 2721 * future users 2722 */ 2723 if (!blk_poll_stats_enable(q)) 2724 return 0; 2725 2726 /* 2727 * As an optimistic guess, use half of the mean service time 2728 * for this type of request. We can (and should) make this smarter. 2729 * For instance, if the completion latencies are tight, we can 2730 * get closer than just half the mean. This is especially 2731 * important on devices where the completion latencies are longer 2732 * than ~10 usec. We do use the stats for the relevant IO size 2733 * if available which does lead to better estimates. 2734 */ 2735 bucket = blk_mq_poll_stats_bkt(rq); 2736 if (bucket < 0) 2737 return ret; 2738 2739 if (q->poll_stat[bucket].nr_samples) 2740 ret = (q->poll_stat[bucket].mean + 1) / 2; 2741 2742 return ret; 2743 } 2744 2745 static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, 2746 struct blk_mq_hw_ctx *hctx, 2747 struct request *rq) 2748 { 2749 struct hrtimer_sleeper hs; 2750 enum hrtimer_mode mode; 2751 unsigned int nsecs; 2752 ktime_t kt; 2753 2754 if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags)) 2755 return false; 2756 2757 /* 2758 * poll_nsec can be: 2759 * 2760 * -1: don't ever hybrid sleep 2761 * 0: use half of prev avg 2762 * >0: use this specific value 2763 */ 2764 if (q->poll_nsec == -1) 2765 return false; 2766 else if (q->poll_nsec > 0) 2767 nsecs = q->poll_nsec; 2768 else 2769 nsecs = blk_mq_poll_nsecs(q, hctx, rq); 2770 2771 if (!nsecs) 2772 return false; 2773 2774 set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags); 2775 2776 /* 2777 * This will be replaced with the stats tracking code, using 2778 * 'avg_completion_time / 2' as the pre-sleep target. 2779 */ 2780 kt = nsecs; 2781 2782 mode = HRTIMER_MODE_REL; 2783 hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode); 2784 hrtimer_set_expires(&hs.timer, kt); 2785 2786 hrtimer_init_sleeper(&hs, current); 2787 do { 2788 if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) 2789 break; 2790 set_current_state(TASK_UNINTERRUPTIBLE); 2791 hrtimer_start_expires(&hs.timer, mode); 2792 if (hs.task) 2793 io_schedule(); 2794 hrtimer_cancel(&hs.timer); 2795 mode = HRTIMER_MODE_ABS; 2796 } while (hs.task && !signal_pending(current)); 2797 2798 __set_current_state(TASK_RUNNING); 2799 destroy_hrtimer_on_stack(&hs.timer); 2800 return true; 2801 } 2802 2803 static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq) 2804 { 2805 struct request_queue *q = hctx->queue; 2806 long state; 2807 2808 /* 2809 * If we sleep, have the caller restart the poll loop to reset 2810 * the state. Like for the other success return cases, the 2811 * caller is responsible for checking if the IO completed. If 2812 * the IO isn't complete, we'll get called again and will go 2813 * straight to the busy poll loop. 2814 */ 2815 if (blk_mq_poll_hybrid_sleep(q, hctx, rq)) 2816 return true; 2817 2818 hctx->poll_considered++; 2819 2820 state = current->state; 2821 while (!need_resched()) { 2822 int ret; 2823 2824 hctx->poll_invoked++; 2825 2826 ret = q->mq_ops->poll(hctx, rq->tag); 2827 if (ret > 0) { 2828 hctx->poll_success++; 2829 set_current_state(TASK_RUNNING); 2830 return true; 2831 } 2832 2833 if (signal_pending_state(state, current)) 2834 set_current_state(TASK_RUNNING); 2835 2836 if (current->state == TASK_RUNNING) 2837 return true; 2838 if (ret < 0) 2839 break; 2840 cpu_relax(); 2841 } 2842 2843 return false; 2844 } 2845 2846 bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie) 2847 { 2848 struct blk_mq_hw_ctx *hctx; 2849 struct blk_plug *plug; 2850 struct request *rq; 2851 2852 if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) || 2853 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 2854 return false; 2855 2856 plug = current->plug; 2857 if (plug) 2858 blk_flush_plug_list(plug, false); 2859 2860 hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; 2861 if (!blk_qc_t_is_internal(cookie)) 2862 rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); 2863 else { 2864 rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie)); 2865 /* 2866 * With scheduling, if the request has completed, we'll 2867 * get a NULL return here, as we clear the sched tag when 2868 * that happens. The request still remains valid, like always, 2869 * so we should be safe with just the NULL check. 2870 */ 2871 if (!rq) 2872 return false; 2873 } 2874 2875 return __blk_mq_poll(hctx, rq); 2876 } 2877 EXPORT_SYMBOL_GPL(blk_mq_poll); 2878 2879 void blk_mq_disable_hotplug(void) 2880 { 2881 mutex_lock(&all_q_mutex); 2882 } 2883 2884 void blk_mq_enable_hotplug(void) 2885 { 2886 mutex_unlock(&all_q_mutex); 2887 } 2888 2889 static int __init blk_mq_init(void) 2890 { 2891 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, 2892 blk_mq_hctx_notify_dead); 2893 2894 cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare", 2895 blk_mq_queue_reinit_prepare, 2896 blk_mq_queue_reinit_dead); 2897 return 0; 2898 } 2899 subsys_initcall(blk_mq_init); 2900