1 /* 2 * Block multiqueue core code 3 * 4 * Copyright (C) 2013-2014 Jens Axboe 5 * Copyright (C) 2013-2014 Christoph Hellwig 6 */ 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/backing-dev.h> 10 #include <linux/bio.h> 11 #include <linux/blkdev.h> 12 #include <linux/kmemleak.h> 13 #include <linux/mm.h> 14 #include <linux/init.h> 15 #include <linux/slab.h> 16 #include <linux/workqueue.h> 17 #include <linux/smp.h> 18 #include <linux/llist.h> 19 #include <linux/list_sort.h> 20 #include <linux/cpu.h> 21 #include <linux/cache.h> 22 #include <linux/sched/sysctl.h> 23 #include <linux/sched/topology.h> 24 #include <linux/sched/signal.h> 25 #include <linux/delay.h> 26 #include <linux/crash_dump.h> 27 #include <linux/prefetch.h> 28 29 #include <trace/events/block.h> 30 31 #include <linux/blk-mq.h> 32 #include "blk.h" 33 #include "blk-mq.h" 34 #include "blk-mq-tag.h" 35 #include "blk-stat.h" 36 #include "blk-wbt.h" 37 #include "blk-mq-sched.h" 38 39 static DEFINE_MUTEX(all_q_mutex); 40 static LIST_HEAD(all_q_list); 41 42 /* 43 * Check if any of the ctx's have pending work in this hardware queue 44 */ 45 bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) 46 { 47 return sbitmap_any_bit_set(&hctx->ctx_map) || 48 !list_empty_careful(&hctx->dispatch) || 49 blk_mq_sched_has_work(hctx); 50 } 51 52 /* 53 * Mark this ctx as having pending work in this hardware queue 54 */ 55 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, 56 struct blk_mq_ctx *ctx) 57 { 58 if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw)) 59 sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw); 60 } 61 62 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, 63 struct blk_mq_ctx *ctx) 64 { 65 sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw); 66 } 67 68 void blk_mq_freeze_queue_start(struct request_queue *q) 69 { 70 int freeze_depth; 71 72 freeze_depth = atomic_inc_return(&q->mq_freeze_depth); 73 if (freeze_depth == 1) { 74 percpu_ref_kill(&q->q_usage_counter); 75 blk_mq_run_hw_queues(q, false); 76 } 77 } 78 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_start); 79 80 static void blk_mq_freeze_queue_wait(struct request_queue *q) 81 { 82 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter)); 83 } 84 85 /* 86 * Guarantee no request is in use, so we can change any data structure of 87 * the queue afterward. 88 */ 89 void blk_freeze_queue(struct request_queue *q) 90 { 91 /* 92 * In the !blk_mq case we are only calling this to kill the 93 * q_usage_counter, otherwise this increases the freeze depth 94 * and waits for it to return to zero. For this reason there is 95 * no blk_unfreeze_queue(), and blk_freeze_queue() is not 96 * exported to drivers as the only user for unfreeze is blk_mq. 97 */ 98 blk_mq_freeze_queue_start(q); 99 blk_mq_freeze_queue_wait(q); 100 } 101 102 void blk_mq_freeze_queue(struct request_queue *q) 103 { 104 /* 105 * ...just an alias to keep freeze and unfreeze actions balanced 106 * in the blk_mq_* namespace 107 */ 108 blk_freeze_queue(q); 109 } 110 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue); 111 112 void blk_mq_unfreeze_queue(struct request_queue *q) 113 { 114 int freeze_depth; 115 116 freeze_depth = atomic_dec_return(&q->mq_freeze_depth); 117 WARN_ON_ONCE(freeze_depth < 0); 118 if (!freeze_depth) { 119 percpu_ref_reinit(&q->q_usage_counter); 120 wake_up_all(&q->mq_freeze_wq); 121 } 122 } 123 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); 124 125 /** 126 * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished 127 * @q: request queue. 128 * 129 * Note: this function does not prevent that the struct request end_io() 130 * callback function is invoked. Additionally, it is not prevented that 131 * new queue_rq() calls occur unless the queue has been stopped first. 132 */ 133 void blk_mq_quiesce_queue(struct request_queue *q) 134 { 135 struct blk_mq_hw_ctx *hctx; 136 unsigned int i; 137 bool rcu = false; 138 139 blk_mq_stop_hw_queues(q); 140 141 queue_for_each_hw_ctx(q, hctx, i) { 142 if (hctx->flags & BLK_MQ_F_BLOCKING) 143 synchronize_srcu(&hctx->queue_rq_srcu); 144 else 145 rcu = true; 146 } 147 if (rcu) 148 synchronize_rcu(); 149 } 150 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue); 151 152 void blk_mq_wake_waiters(struct request_queue *q) 153 { 154 struct blk_mq_hw_ctx *hctx; 155 unsigned int i; 156 157 queue_for_each_hw_ctx(q, hctx, i) 158 if (blk_mq_hw_queue_mapped(hctx)) 159 blk_mq_tag_wakeup_all(hctx->tags, true); 160 161 /* 162 * If we are called because the queue has now been marked as 163 * dying, we need to ensure that processes currently waiting on 164 * the queue are notified as well. 165 */ 166 wake_up_all(&q->mq_freeze_wq); 167 } 168 169 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) 170 { 171 return blk_mq_has_free_tags(hctx->tags); 172 } 173 EXPORT_SYMBOL(blk_mq_can_queue); 174 175 void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, 176 struct request *rq, unsigned int op) 177 { 178 INIT_LIST_HEAD(&rq->queuelist); 179 /* csd/requeue_work/fifo_time is initialized before use */ 180 rq->q = q; 181 rq->mq_ctx = ctx; 182 rq->cmd_flags = op; 183 if (blk_queue_io_stat(q)) 184 rq->rq_flags |= RQF_IO_STAT; 185 /* do not touch atomic flags, it needs atomic ops against the timer */ 186 rq->cpu = -1; 187 INIT_HLIST_NODE(&rq->hash); 188 RB_CLEAR_NODE(&rq->rb_node); 189 rq->rq_disk = NULL; 190 rq->part = NULL; 191 rq->start_time = jiffies; 192 #ifdef CONFIG_BLK_CGROUP 193 rq->rl = NULL; 194 set_start_time_ns(rq); 195 rq->io_start_time_ns = 0; 196 #endif 197 rq->nr_phys_segments = 0; 198 #if defined(CONFIG_BLK_DEV_INTEGRITY) 199 rq->nr_integrity_segments = 0; 200 #endif 201 rq->special = NULL; 202 /* tag was already set */ 203 rq->errors = 0; 204 rq->extra_len = 0; 205 206 INIT_LIST_HEAD(&rq->timeout_list); 207 rq->timeout = 0; 208 209 rq->end_io = NULL; 210 rq->end_io_data = NULL; 211 rq->next_rq = NULL; 212 213 ctx->rq_dispatched[op_is_sync(op)]++; 214 } 215 EXPORT_SYMBOL_GPL(blk_mq_rq_ctx_init); 216 217 struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data, 218 unsigned int op) 219 { 220 struct request *rq; 221 unsigned int tag; 222 223 tag = blk_mq_get_tag(data); 224 if (tag != BLK_MQ_TAG_FAIL) { 225 struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 226 227 rq = tags->static_rqs[tag]; 228 229 if (data->flags & BLK_MQ_REQ_INTERNAL) { 230 rq->tag = -1; 231 rq->internal_tag = tag; 232 } else { 233 if (blk_mq_tag_busy(data->hctx)) { 234 rq->rq_flags = RQF_MQ_INFLIGHT; 235 atomic_inc(&data->hctx->nr_active); 236 } 237 rq->tag = tag; 238 rq->internal_tag = -1; 239 } 240 241 blk_mq_rq_ctx_init(data->q, data->ctx, rq, op); 242 return rq; 243 } 244 245 return NULL; 246 } 247 EXPORT_SYMBOL_GPL(__blk_mq_alloc_request); 248 249 struct request *blk_mq_alloc_request(struct request_queue *q, int rw, 250 unsigned int flags) 251 { 252 struct blk_mq_alloc_data alloc_data = { .flags = flags }; 253 struct request *rq; 254 int ret; 255 256 ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT); 257 if (ret) 258 return ERR_PTR(ret); 259 260 rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data); 261 262 blk_mq_put_ctx(alloc_data.ctx); 263 blk_queue_exit(q); 264 265 if (!rq) 266 return ERR_PTR(-EWOULDBLOCK); 267 268 rq->__data_len = 0; 269 rq->__sector = (sector_t) -1; 270 rq->bio = rq->biotail = NULL; 271 return rq; 272 } 273 EXPORT_SYMBOL(blk_mq_alloc_request); 274 275 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw, 276 unsigned int flags, unsigned int hctx_idx) 277 { 278 struct blk_mq_hw_ctx *hctx; 279 struct blk_mq_ctx *ctx; 280 struct request *rq; 281 struct blk_mq_alloc_data alloc_data; 282 int ret; 283 284 /* 285 * If the tag allocator sleeps we could get an allocation for a 286 * different hardware context. No need to complicate the low level 287 * allocator for this for the rare use case of a command tied to 288 * a specific queue. 289 */ 290 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT))) 291 return ERR_PTR(-EINVAL); 292 293 if (hctx_idx >= q->nr_hw_queues) 294 return ERR_PTR(-EIO); 295 296 ret = blk_queue_enter(q, true); 297 if (ret) 298 return ERR_PTR(ret); 299 300 /* 301 * Check if the hardware context is actually mapped to anything. 302 * If not tell the caller that it should skip this queue. 303 */ 304 hctx = q->queue_hw_ctx[hctx_idx]; 305 if (!blk_mq_hw_queue_mapped(hctx)) { 306 ret = -EXDEV; 307 goto out_queue_exit; 308 } 309 ctx = __blk_mq_get_ctx(q, cpumask_first(hctx->cpumask)); 310 311 blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx); 312 rq = __blk_mq_alloc_request(&alloc_data, rw); 313 if (!rq) { 314 ret = -EWOULDBLOCK; 315 goto out_queue_exit; 316 } 317 318 return rq; 319 320 out_queue_exit: 321 blk_queue_exit(q); 322 return ERR_PTR(ret); 323 } 324 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); 325 326 void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 327 struct request *rq) 328 { 329 const int sched_tag = rq->internal_tag; 330 struct request_queue *q = rq->q; 331 332 if (rq->rq_flags & RQF_MQ_INFLIGHT) 333 atomic_dec(&hctx->nr_active); 334 335 wbt_done(q->rq_wb, &rq->issue_stat); 336 rq->rq_flags = 0; 337 338 clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags); 339 clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags); 340 if (rq->tag != -1) 341 blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag); 342 if (sched_tag != -1) 343 blk_mq_sched_completed_request(hctx, rq); 344 blk_mq_sched_restart_queues(hctx); 345 blk_queue_exit(q); 346 } 347 348 static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx, 349 struct request *rq) 350 { 351 struct blk_mq_ctx *ctx = rq->mq_ctx; 352 353 ctx->rq_completed[rq_is_sync(rq)]++; 354 __blk_mq_finish_request(hctx, ctx, rq); 355 } 356 357 void blk_mq_finish_request(struct request *rq) 358 { 359 blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq); 360 } 361 362 void blk_mq_free_request(struct request *rq) 363 { 364 blk_mq_sched_put_request(rq); 365 } 366 EXPORT_SYMBOL_GPL(blk_mq_free_request); 367 368 inline void __blk_mq_end_request(struct request *rq, int error) 369 { 370 blk_account_io_done(rq); 371 372 if (rq->end_io) { 373 wbt_done(rq->q->rq_wb, &rq->issue_stat); 374 rq->end_io(rq, error); 375 } else { 376 if (unlikely(blk_bidi_rq(rq))) 377 blk_mq_free_request(rq->next_rq); 378 blk_mq_free_request(rq); 379 } 380 } 381 EXPORT_SYMBOL(__blk_mq_end_request); 382 383 void blk_mq_end_request(struct request *rq, int error) 384 { 385 if (blk_update_request(rq, error, blk_rq_bytes(rq))) 386 BUG(); 387 __blk_mq_end_request(rq, error); 388 } 389 EXPORT_SYMBOL(blk_mq_end_request); 390 391 static void __blk_mq_complete_request_remote(void *data) 392 { 393 struct request *rq = data; 394 395 rq->q->softirq_done_fn(rq); 396 } 397 398 static void blk_mq_ipi_complete_request(struct request *rq) 399 { 400 struct blk_mq_ctx *ctx = rq->mq_ctx; 401 bool shared = false; 402 int cpu; 403 404 if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) { 405 rq->q->softirq_done_fn(rq); 406 return; 407 } 408 409 cpu = get_cpu(); 410 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags)) 411 shared = cpus_share_cache(cpu, ctx->cpu); 412 413 if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { 414 rq->csd.func = __blk_mq_complete_request_remote; 415 rq->csd.info = rq; 416 rq->csd.flags = 0; 417 smp_call_function_single_async(ctx->cpu, &rq->csd); 418 } else { 419 rq->q->softirq_done_fn(rq); 420 } 421 put_cpu(); 422 } 423 424 static void blk_mq_stat_add(struct request *rq) 425 { 426 if (rq->rq_flags & RQF_STATS) { 427 /* 428 * We could rq->mq_ctx here, but there's less of a risk 429 * of races if we have the completion event add the stats 430 * to the local software queue. 431 */ 432 struct blk_mq_ctx *ctx; 433 434 ctx = __blk_mq_get_ctx(rq->q, raw_smp_processor_id()); 435 blk_stat_add(&ctx->stat[rq_data_dir(rq)], rq); 436 } 437 } 438 439 static void __blk_mq_complete_request(struct request *rq) 440 { 441 struct request_queue *q = rq->q; 442 443 blk_mq_stat_add(rq); 444 445 if (!q->softirq_done_fn) 446 blk_mq_end_request(rq, rq->errors); 447 else 448 blk_mq_ipi_complete_request(rq); 449 } 450 451 /** 452 * blk_mq_complete_request - end I/O on a request 453 * @rq: the request being processed 454 * 455 * Description: 456 * Ends all I/O on a request. It does not handle partial completions. 457 * The actual completion happens out-of-order, through a IPI handler. 458 **/ 459 void blk_mq_complete_request(struct request *rq, int error) 460 { 461 struct request_queue *q = rq->q; 462 463 if (unlikely(blk_should_fake_timeout(q))) 464 return; 465 if (!blk_mark_rq_complete(rq)) { 466 rq->errors = error; 467 __blk_mq_complete_request(rq); 468 } 469 } 470 EXPORT_SYMBOL(blk_mq_complete_request); 471 472 int blk_mq_request_started(struct request *rq) 473 { 474 return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags); 475 } 476 EXPORT_SYMBOL_GPL(blk_mq_request_started); 477 478 void blk_mq_start_request(struct request *rq) 479 { 480 struct request_queue *q = rq->q; 481 482 blk_mq_sched_started_request(rq); 483 484 trace_block_rq_issue(q, rq); 485 486 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 487 blk_stat_set_issue_time(&rq->issue_stat); 488 rq->rq_flags |= RQF_STATS; 489 wbt_issue(q->rq_wb, &rq->issue_stat); 490 } 491 492 blk_add_timer(rq); 493 494 /* 495 * Ensure that ->deadline is visible before set the started 496 * flag and clear the completed flag. 497 */ 498 smp_mb__before_atomic(); 499 500 /* 501 * Mark us as started and clear complete. Complete might have been 502 * set if requeue raced with timeout, which then marked it as 503 * complete. So be sure to clear complete again when we start 504 * the request, otherwise we'll ignore the completion event. 505 */ 506 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) 507 set_bit(REQ_ATOM_STARTED, &rq->atomic_flags); 508 if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) 509 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags); 510 511 if (q->dma_drain_size && blk_rq_bytes(rq)) { 512 /* 513 * Make sure space for the drain appears. We know we can do 514 * this because max_hw_segments has been adjusted to be one 515 * fewer than the device can handle. 516 */ 517 rq->nr_phys_segments++; 518 } 519 } 520 EXPORT_SYMBOL(blk_mq_start_request); 521 522 static void __blk_mq_requeue_request(struct request *rq) 523 { 524 struct request_queue *q = rq->q; 525 526 trace_block_rq_requeue(q, rq); 527 wbt_requeue(q->rq_wb, &rq->issue_stat); 528 blk_mq_sched_requeue_request(rq); 529 530 if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) { 531 if (q->dma_drain_size && blk_rq_bytes(rq)) 532 rq->nr_phys_segments--; 533 } 534 } 535 536 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) 537 { 538 __blk_mq_requeue_request(rq); 539 540 BUG_ON(blk_queued_rq(rq)); 541 blk_mq_add_to_requeue_list(rq, true, kick_requeue_list); 542 } 543 EXPORT_SYMBOL(blk_mq_requeue_request); 544 545 static void blk_mq_requeue_work(struct work_struct *work) 546 { 547 struct request_queue *q = 548 container_of(work, struct request_queue, requeue_work.work); 549 LIST_HEAD(rq_list); 550 struct request *rq, *next; 551 unsigned long flags; 552 553 spin_lock_irqsave(&q->requeue_lock, flags); 554 list_splice_init(&q->requeue_list, &rq_list); 555 spin_unlock_irqrestore(&q->requeue_lock, flags); 556 557 list_for_each_entry_safe(rq, next, &rq_list, queuelist) { 558 if (!(rq->rq_flags & RQF_SOFTBARRIER)) 559 continue; 560 561 rq->rq_flags &= ~RQF_SOFTBARRIER; 562 list_del_init(&rq->queuelist); 563 blk_mq_sched_insert_request(rq, true, false, false, true); 564 } 565 566 while (!list_empty(&rq_list)) { 567 rq = list_entry(rq_list.next, struct request, queuelist); 568 list_del_init(&rq->queuelist); 569 blk_mq_sched_insert_request(rq, false, false, false, true); 570 } 571 572 blk_mq_run_hw_queues(q, false); 573 } 574 575 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, 576 bool kick_requeue_list) 577 { 578 struct request_queue *q = rq->q; 579 unsigned long flags; 580 581 /* 582 * We abuse this flag that is otherwise used by the I/O scheduler to 583 * request head insertation from the workqueue. 584 */ 585 BUG_ON(rq->rq_flags & RQF_SOFTBARRIER); 586 587 spin_lock_irqsave(&q->requeue_lock, flags); 588 if (at_head) { 589 rq->rq_flags |= RQF_SOFTBARRIER; 590 list_add(&rq->queuelist, &q->requeue_list); 591 } else { 592 list_add_tail(&rq->queuelist, &q->requeue_list); 593 } 594 spin_unlock_irqrestore(&q->requeue_lock, flags); 595 596 if (kick_requeue_list) 597 blk_mq_kick_requeue_list(q); 598 } 599 EXPORT_SYMBOL(blk_mq_add_to_requeue_list); 600 601 void blk_mq_kick_requeue_list(struct request_queue *q) 602 { 603 kblockd_schedule_delayed_work(&q->requeue_work, 0); 604 } 605 EXPORT_SYMBOL(blk_mq_kick_requeue_list); 606 607 void blk_mq_delay_kick_requeue_list(struct request_queue *q, 608 unsigned long msecs) 609 { 610 kblockd_schedule_delayed_work(&q->requeue_work, 611 msecs_to_jiffies(msecs)); 612 } 613 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); 614 615 void blk_mq_abort_requeue_list(struct request_queue *q) 616 { 617 unsigned long flags; 618 LIST_HEAD(rq_list); 619 620 spin_lock_irqsave(&q->requeue_lock, flags); 621 list_splice_init(&q->requeue_list, &rq_list); 622 spin_unlock_irqrestore(&q->requeue_lock, flags); 623 624 while (!list_empty(&rq_list)) { 625 struct request *rq; 626 627 rq = list_first_entry(&rq_list, struct request, queuelist); 628 list_del_init(&rq->queuelist); 629 rq->errors = -EIO; 630 blk_mq_end_request(rq, rq->errors); 631 } 632 } 633 EXPORT_SYMBOL(blk_mq_abort_requeue_list); 634 635 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) 636 { 637 if (tag < tags->nr_tags) { 638 prefetch(tags->rqs[tag]); 639 return tags->rqs[tag]; 640 } 641 642 return NULL; 643 } 644 EXPORT_SYMBOL(blk_mq_tag_to_rq); 645 646 struct blk_mq_timeout_data { 647 unsigned long next; 648 unsigned int next_set; 649 }; 650 651 void blk_mq_rq_timed_out(struct request *req, bool reserved) 652 { 653 const struct blk_mq_ops *ops = req->q->mq_ops; 654 enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER; 655 656 /* 657 * We know that complete is set at this point. If STARTED isn't set 658 * anymore, then the request isn't active and the "timeout" should 659 * just be ignored. This can happen due to the bitflag ordering. 660 * Timeout first checks if STARTED is set, and if it is, assumes 661 * the request is active. But if we race with completion, then 662 * we both flags will get cleared. So check here again, and ignore 663 * a timeout event with a request that isn't active. 664 */ 665 if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags)) 666 return; 667 668 if (ops->timeout) 669 ret = ops->timeout(req, reserved); 670 671 switch (ret) { 672 case BLK_EH_HANDLED: 673 __blk_mq_complete_request(req); 674 break; 675 case BLK_EH_RESET_TIMER: 676 blk_add_timer(req); 677 blk_clear_rq_complete(req); 678 break; 679 case BLK_EH_NOT_HANDLED: 680 break; 681 default: 682 printk(KERN_ERR "block: bad eh return: %d\n", ret); 683 break; 684 } 685 } 686 687 static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, 688 struct request *rq, void *priv, bool reserved) 689 { 690 struct blk_mq_timeout_data *data = priv; 691 692 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) { 693 /* 694 * If a request wasn't started before the queue was 695 * marked dying, kill it here or it'll go unnoticed. 696 */ 697 if (unlikely(blk_queue_dying(rq->q))) { 698 rq->errors = -EIO; 699 blk_mq_end_request(rq, rq->errors); 700 } 701 return; 702 } 703 704 if (time_after_eq(jiffies, rq->deadline)) { 705 if (!blk_mark_rq_complete(rq)) 706 blk_mq_rq_timed_out(rq, reserved); 707 } else if (!data->next_set || time_after(data->next, rq->deadline)) { 708 data->next = rq->deadline; 709 data->next_set = 1; 710 } 711 } 712 713 static void blk_mq_timeout_work(struct work_struct *work) 714 { 715 struct request_queue *q = 716 container_of(work, struct request_queue, timeout_work); 717 struct blk_mq_timeout_data data = { 718 .next = 0, 719 .next_set = 0, 720 }; 721 int i; 722 723 /* A deadlock might occur if a request is stuck requiring a 724 * timeout at the same time a queue freeze is waiting 725 * completion, since the timeout code would not be able to 726 * acquire the queue reference here. 727 * 728 * That's why we don't use blk_queue_enter here; instead, we use 729 * percpu_ref_tryget directly, because we need to be able to 730 * obtain a reference even in the short window between the queue 731 * starting to freeze, by dropping the first reference in 732 * blk_mq_freeze_queue_start, and the moment the last request is 733 * consumed, marked by the instant q_usage_counter reaches 734 * zero. 735 */ 736 if (!percpu_ref_tryget(&q->q_usage_counter)) 737 return; 738 739 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data); 740 741 if (data.next_set) { 742 data.next = blk_rq_timeout(round_jiffies_up(data.next)); 743 mod_timer(&q->timeout, data.next); 744 } else { 745 struct blk_mq_hw_ctx *hctx; 746 747 queue_for_each_hw_ctx(q, hctx, i) { 748 /* the hctx may be unmapped, so check it here */ 749 if (blk_mq_hw_queue_mapped(hctx)) 750 blk_mq_tag_idle(hctx); 751 } 752 } 753 blk_queue_exit(q); 754 } 755 756 /* 757 * Reverse check our software queue for entries that we could potentially 758 * merge with. Currently includes a hand-wavy stop count of 8, to not spend 759 * too much time checking for merges. 760 */ 761 static bool blk_mq_attempt_merge(struct request_queue *q, 762 struct blk_mq_ctx *ctx, struct bio *bio) 763 { 764 struct request *rq; 765 int checked = 8; 766 767 list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) { 768 bool merged = false; 769 770 if (!checked--) 771 break; 772 773 if (!blk_rq_merge_ok(rq, bio)) 774 continue; 775 776 switch (blk_try_merge(rq, bio)) { 777 case ELEVATOR_BACK_MERGE: 778 if (blk_mq_sched_allow_merge(q, rq, bio)) 779 merged = bio_attempt_back_merge(q, rq, bio); 780 break; 781 case ELEVATOR_FRONT_MERGE: 782 if (blk_mq_sched_allow_merge(q, rq, bio)) 783 merged = bio_attempt_front_merge(q, rq, bio); 784 break; 785 case ELEVATOR_DISCARD_MERGE: 786 merged = bio_attempt_discard_merge(q, rq, bio); 787 break; 788 default: 789 continue; 790 } 791 792 if (merged) 793 ctx->rq_merged++; 794 return merged; 795 } 796 797 return false; 798 } 799 800 struct flush_busy_ctx_data { 801 struct blk_mq_hw_ctx *hctx; 802 struct list_head *list; 803 }; 804 805 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) 806 { 807 struct flush_busy_ctx_data *flush_data = data; 808 struct blk_mq_hw_ctx *hctx = flush_data->hctx; 809 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 810 811 sbitmap_clear_bit(sb, bitnr); 812 spin_lock(&ctx->lock); 813 list_splice_tail_init(&ctx->rq_list, flush_data->list); 814 spin_unlock(&ctx->lock); 815 return true; 816 } 817 818 /* 819 * Process software queues that have been marked busy, splicing them 820 * to the for-dispatch 821 */ 822 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) 823 { 824 struct flush_busy_ctx_data data = { 825 .hctx = hctx, 826 .list = list, 827 }; 828 829 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data); 830 } 831 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs); 832 833 static inline unsigned int queued_to_index(unsigned int queued) 834 { 835 if (!queued) 836 return 0; 837 838 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); 839 } 840 841 bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx, 842 bool wait) 843 { 844 struct blk_mq_alloc_data data = { 845 .q = rq->q, 846 .hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), 847 .flags = wait ? 0 : BLK_MQ_REQ_NOWAIT, 848 }; 849 850 if (rq->tag != -1) { 851 done: 852 if (hctx) 853 *hctx = data.hctx; 854 return true; 855 } 856 857 rq->tag = blk_mq_get_tag(&data); 858 if (rq->tag >= 0) { 859 if (blk_mq_tag_busy(data.hctx)) { 860 rq->rq_flags |= RQF_MQ_INFLIGHT; 861 atomic_inc(&data.hctx->nr_active); 862 } 863 data.hctx->tags->rqs[rq->tag] = rq; 864 goto done; 865 } 866 867 return false; 868 } 869 870 static void blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx, 871 struct request *rq) 872 { 873 if (rq->tag == -1 || rq->internal_tag == -1) 874 return; 875 876 blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag); 877 rq->tag = -1; 878 879 if (rq->rq_flags & RQF_MQ_INFLIGHT) { 880 rq->rq_flags &= ~RQF_MQ_INFLIGHT; 881 atomic_dec(&hctx->nr_active); 882 } 883 } 884 885 /* 886 * If we fail getting a driver tag because all the driver tags are already 887 * assigned and on the dispatch list, BUT the first entry does not have a 888 * tag, then we could deadlock. For that case, move entries with assigned 889 * driver tags to the front, leaving the set of tagged requests in the 890 * same order, and the untagged set in the same order. 891 */ 892 static bool reorder_tags_to_front(struct list_head *list) 893 { 894 struct request *rq, *tmp, *first = NULL; 895 896 list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) { 897 if (rq == first) 898 break; 899 if (rq->tag != -1) { 900 list_move(&rq->queuelist, list); 901 if (!first) 902 first = rq; 903 } 904 } 905 906 return first != NULL; 907 } 908 909 static int blk_mq_dispatch_wake(wait_queue_t *wait, unsigned mode, int flags, 910 void *key) 911 { 912 struct blk_mq_hw_ctx *hctx; 913 914 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); 915 916 list_del(&wait->task_list); 917 clear_bit_unlock(BLK_MQ_S_TAG_WAITING, &hctx->state); 918 blk_mq_run_hw_queue(hctx, true); 919 return 1; 920 } 921 922 static bool blk_mq_dispatch_wait_add(struct blk_mq_hw_ctx *hctx) 923 { 924 struct sbq_wait_state *ws; 925 926 /* 927 * The TAG_WAITING bit serves as a lock protecting hctx->dispatch_wait. 928 * The thread which wins the race to grab this bit adds the hardware 929 * queue to the wait queue. 930 */ 931 if (test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state) || 932 test_and_set_bit_lock(BLK_MQ_S_TAG_WAITING, &hctx->state)) 933 return false; 934 935 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); 936 ws = bt_wait_ptr(&hctx->tags->bitmap_tags, hctx); 937 938 /* 939 * As soon as this returns, it's no longer safe to fiddle with 940 * hctx->dispatch_wait, since a completion can wake up the wait queue 941 * and unlock the bit. 942 */ 943 add_wait_queue(&ws->wait, &hctx->dispatch_wait); 944 return true; 945 } 946 947 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list) 948 { 949 struct request_queue *q = hctx->queue; 950 struct request *rq; 951 LIST_HEAD(driver_list); 952 struct list_head *dptr; 953 int queued, ret = BLK_MQ_RQ_QUEUE_OK; 954 955 /* 956 * Start off with dptr being NULL, so we start the first request 957 * immediately, even if we have more pending. 958 */ 959 dptr = NULL; 960 961 /* 962 * Now process all the entries, sending them to the driver. 963 */ 964 queued = 0; 965 while (!list_empty(list)) { 966 struct blk_mq_queue_data bd; 967 968 rq = list_first_entry(list, struct request, queuelist); 969 if (!blk_mq_get_driver_tag(rq, &hctx, false)) { 970 if (!queued && reorder_tags_to_front(list)) 971 continue; 972 973 /* 974 * The initial allocation attempt failed, so we need to 975 * rerun the hardware queue when a tag is freed. 976 */ 977 if (blk_mq_dispatch_wait_add(hctx)) { 978 /* 979 * It's possible that a tag was freed in the 980 * window between the allocation failure and 981 * adding the hardware queue to the wait queue. 982 */ 983 if (!blk_mq_get_driver_tag(rq, &hctx, false)) 984 break; 985 } else { 986 break; 987 } 988 } 989 990 list_del_init(&rq->queuelist); 991 992 bd.rq = rq; 993 bd.list = dptr; 994 bd.last = list_empty(list); 995 996 ret = q->mq_ops->queue_rq(hctx, &bd); 997 switch (ret) { 998 case BLK_MQ_RQ_QUEUE_OK: 999 queued++; 1000 break; 1001 case BLK_MQ_RQ_QUEUE_BUSY: 1002 blk_mq_put_driver_tag(hctx, rq); 1003 list_add(&rq->queuelist, list); 1004 __blk_mq_requeue_request(rq); 1005 break; 1006 default: 1007 pr_err("blk-mq: bad return on queue: %d\n", ret); 1008 case BLK_MQ_RQ_QUEUE_ERROR: 1009 rq->errors = -EIO; 1010 blk_mq_end_request(rq, rq->errors); 1011 break; 1012 } 1013 1014 if (ret == BLK_MQ_RQ_QUEUE_BUSY) 1015 break; 1016 1017 /* 1018 * We've done the first request. If we have more than 1 1019 * left in the list, set dptr to defer issue. 1020 */ 1021 if (!dptr && list->next != list->prev) 1022 dptr = &driver_list; 1023 } 1024 1025 hctx->dispatched[queued_to_index(queued)]++; 1026 1027 /* 1028 * Any items that need requeuing? Stuff them into hctx->dispatch, 1029 * that is where we will continue on next queue run. 1030 */ 1031 if (!list_empty(list)) { 1032 spin_lock(&hctx->lock); 1033 list_splice_init(list, &hctx->dispatch); 1034 spin_unlock(&hctx->lock); 1035 1036 /* 1037 * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but 1038 * it's possible the queue is stopped and restarted again 1039 * before this. Queue restart will dispatch requests. And since 1040 * requests in rq_list aren't added into hctx->dispatch yet, 1041 * the requests in rq_list might get lost. 1042 * 1043 * blk_mq_run_hw_queue() already checks the STOPPED bit 1044 * 1045 * If RESTART or TAG_WAITING is set, then let completion restart 1046 * the queue instead of potentially looping here. 1047 */ 1048 if (!blk_mq_sched_needs_restart(hctx) && 1049 !test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state)) 1050 blk_mq_run_hw_queue(hctx, true); 1051 } 1052 1053 return queued != 0; 1054 } 1055 1056 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) 1057 { 1058 int srcu_idx; 1059 1060 WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) && 1061 cpu_online(hctx->next_cpu)); 1062 1063 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { 1064 rcu_read_lock(); 1065 blk_mq_sched_dispatch_requests(hctx); 1066 rcu_read_unlock(); 1067 } else { 1068 srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu); 1069 blk_mq_sched_dispatch_requests(hctx); 1070 srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx); 1071 } 1072 } 1073 1074 /* 1075 * It'd be great if the workqueue API had a way to pass 1076 * in a mask and had some smarts for more clever placement. 1077 * For now we just round-robin here, switching for every 1078 * BLK_MQ_CPU_WORK_BATCH queued items. 1079 */ 1080 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) 1081 { 1082 if (hctx->queue->nr_hw_queues == 1) 1083 return WORK_CPU_UNBOUND; 1084 1085 if (--hctx->next_cpu_batch <= 0) { 1086 int next_cpu; 1087 1088 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask); 1089 if (next_cpu >= nr_cpu_ids) 1090 next_cpu = cpumask_first(hctx->cpumask); 1091 1092 hctx->next_cpu = next_cpu; 1093 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 1094 } 1095 1096 return hctx->next_cpu; 1097 } 1098 1099 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1100 { 1101 if (unlikely(blk_mq_hctx_stopped(hctx) || 1102 !blk_mq_hw_queue_mapped(hctx))) 1103 return; 1104 1105 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) { 1106 int cpu = get_cpu(); 1107 if (cpumask_test_cpu(cpu, hctx->cpumask)) { 1108 __blk_mq_run_hw_queue(hctx); 1109 put_cpu(); 1110 return; 1111 } 1112 1113 put_cpu(); 1114 } 1115 1116 kblockd_schedule_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work); 1117 } 1118 1119 void blk_mq_run_hw_queues(struct request_queue *q, bool async) 1120 { 1121 struct blk_mq_hw_ctx *hctx; 1122 int i; 1123 1124 queue_for_each_hw_ctx(q, hctx, i) { 1125 if (!blk_mq_hctx_has_pending(hctx) || 1126 blk_mq_hctx_stopped(hctx)) 1127 continue; 1128 1129 blk_mq_run_hw_queue(hctx, async); 1130 } 1131 } 1132 EXPORT_SYMBOL(blk_mq_run_hw_queues); 1133 1134 /** 1135 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped 1136 * @q: request queue. 1137 * 1138 * The caller is responsible for serializing this function against 1139 * blk_mq_{start,stop}_hw_queue(). 1140 */ 1141 bool blk_mq_queue_stopped(struct request_queue *q) 1142 { 1143 struct blk_mq_hw_ctx *hctx; 1144 int i; 1145 1146 queue_for_each_hw_ctx(q, hctx, i) 1147 if (blk_mq_hctx_stopped(hctx)) 1148 return true; 1149 1150 return false; 1151 } 1152 EXPORT_SYMBOL(blk_mq_queue_stopped); 1153 1154 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) 1155 { 1156 cancel_work(&hctx->run_work); 1157 cancel_delayed_work(&hctx->delay_work); 1158 set_bit(BLK_MQ_S_STOPPED, &hctx->state); 1159 } 1160 EXPORT_SYMBOL(blk_mq_stop_hw_queue); 1161 1162 void blk_mq_stop_hw_queues(struct request_queue *q) 1163 { 1164 struct blk_mq_hw_ctx *hctx; 1165 int i; 1166 1167 queue_for_each_hw_ctx(q, hctx, i) 1168 blk_mq_stop_hw_queue(hctx); 1169 } 1170 EXPORT_SYMBOL(blk_mq_stop_hw_queues); 1171 1172 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) 1173 { 1174 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1175 1176 blk_mq_run_hw_queue(hctx, false); 1177 } 1178 EXPORT_SYMBOL(blk_mq_start_hw_queue); 1179 1180 void blk_mq_start_hw_queues(struct request_queue *q) 1181 { 1182 struct blk_mq_hw_ctx *hctx; 1183 int i; 1184 1185 queue_for_each_hw_ctx(q, hctx, i) 1186 blk_mq_start_hw_queue(hctx); 1187 } 1188 EXPORT_SYMBOL(blk_mq_start_hw_queues); 1189 1190 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1191 { 1192 if (!blk_mq_hctx_stopped(hctx)) 1193 return; 1194 1195 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1196 blk_mq_run_hw_queue(hctx, async); 1197 } 1198 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue); 1199 1200 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) 1201 { 1202 struct blk_mq_hw_ctx *hctx; 1203 int i; 1204 1205 queue_for_each_hw_ctx(q, hctx, i) 1206 blk_mq_start_stopped_hw_queue(hctx, async); 1207 } 1208 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); 1209 1210 static void blk_mq_run_work_fn(struct work_struct *work) 1211 { 1212 struct blk_mq_hw_ctx *hctx; 1213 1214 hctx = container_of(work, struct blk_mq_hw_ctx, run_work); 1215 1216 __blk_mq_run_hw_queue(hctx); 1217 } 1218 1219 static void blk_mq_delay_work_fn(struct work_struct *work) 1220 { 1221 struct blk_mq_hw_ctx *hctx; 1222 1223 hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work); 1224 1225 if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state)) 1226 __blk_mq_run_hw_queue(hctx); 1227 } 1228 1229 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) 1230 { 1231 if (unlikely(!blk_mq_hw_queue_mapped(hctx))) 1232 return; 1233 1234 blk_mq_stop_hw_queue(hctx); 1235 kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx), 1236 &hctx->delay_work, msecs_to_jiffies(msecs)); 1237 } 1238 EXPORT_SYMBOL(blk_mq_delay_queue); 1239 1240 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx, 1241 struct request *rq, 1242 bool at_head) 1243 { 1244 struct blk_mq_ctx *ctx = rq->mq_ctx; 1245 1246 trace_block_rq_insert(hctx->queue, rq); 1247 1248 if (at_head) 1249 list_add(&rq->queuelist, &ctx->rq_list); 1250 else 1251 list_add_tail(&rq->queuelist, &ctx->rq_list); 1252 } 1253 1254 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, 1255 bool at_head) 1256 { 1257 struct blk_mq_ctx *ctx = rq->mq_ctx; 1258 1259 __blk_mq_insert_req_list(hctx, rq, at_head); 1260 blk_mq_hctx_mark_pending(hctx, ctx); 1261 } 1262 1263 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 1264 struct list_head *list) 1265 1266 { 1267 /* 1268 * preemption doesn't flush plug list, so it's possible ctx->cpu is 1269 * offline now 1270 */ 1271 spin_lock(&ctx->lock); 1272 while (!list_empty(list)) { 1273 struct request *rq; 1274 1275 rq = list_first_entry(list, struct request, queuelist); 1276 BUG_ON(rq->mq_ctx != ctx); 1277 list_del_init(&rq->queuelist); 1278 __blk_mq_insert_req_list(hctx, rq, false); 1279 } 1280 blk_mq_hctx_mark_pending(hctx, ctx); 1281 spin_unlock(&ctx->lock); 1282 } 1283 1284 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b) 1285 { 1286 struct request *rqa = container_of(a, struct request, queuelist); 1287 struct request *rqb = container_of(b, struct request, queuelist); 1288 1289 return !(rqa->mq_ctx < rqb->mq_ctx || 1290 (rqa->mq_ctx == rqb->mq_ctx && 1291 blk_rq_pos(rqa) < blk_rq_pos(rqb))); 1292 } 1293 1294 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 1295 { 1296 struct blk_mq_ctx *this_ctx; 1297 struct request_queue *this_q; 1298 struct request *rq; 1299 LIST_HEAD(list); 1300 LIST_HEAD(ctx_list); 1301 unsigned int depth; 1302 1303 list_splice_init(&plug->mq_list, &list); 1304 1305 list_sort(NULL, &list, plug_ctx_cmp); 1306 1307 this_q = NULL; 1308 this_ctx = NULL; 1309 depth = 0; 1310 1311 while (!list_empty(&list)) { 1312 rq = list_entry_rq(list.next); 1313 list_del_init(&rq->queuelist); 1314 BUG_ON(!rq->q); 1315 if (rq->mq_ctx != this_ctx) { 1316 if (this_ctx) { 1317 trace_block_unplug(this_q, depth, from_schedule); 1318 blk_mq_sched_insert_requests(this_q, this_ctx, 1319 &ctx_list, 1320 from_schedule); 1321 } 1322 1323 this_ctx = rq->mq_ctx; 1324 this_q = rq->q; 1325 depth = 0; 1326 } 1327 1328 depth++; 1329 list_add_tail(&rq->queuelist, &ctx_list); 1330 } 1331 1332 /* 1333 * If 'this_ctx' is set, we know we have entries to complete 1334 * on 'ctx_list'. Do those. 1335 */ 1336 if (this_ctx) { 1337 trace_block_unplug(this_q, depth, from_schedule); 1338 blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list, 1339 from_schedule); 1340 } 1341 } 1342 1343 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio) 1344 { 1345 init_request_from_bio(rq, bio); 1346 1347 blk_account_io_start(rq, true); 1348 } 1349 1350 static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx) 1351 { 1352 return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) && 1353 !blk_queue_nomerges(hctx->queue); 1354 } 1355 1356 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx, 1357 struct blk_mq_ctx *ctx, 1358 struct request *rq, struct bio *bio) 1359 { 1360 if (!hctx_allow_merges(hctx) || !bio_mergeable(bio)) { 1361 blk_mq_bio_to_request(rq, bio); 1362 spin_lock(&ctx->lock); 1363 insert_rq: 1364 __blk_mq_insert_request(hctx, rq, false); 1365 spin_unlock(&ctx->lock); 1366 return false; 1367 } else { 1368 struct request_queue *q = hctx->queue; 1369 1370 spin_lock(&ctx->lock); 1371 if (!blk_mq_attempt_merge(q, ctx, bio)) { 1372 blk_mq_bio_to_request(rq, bio); 1373 goto insert_rq; 1374 } 1375 1376 spin_unlock(&ctx->lock); 1377 __blk_mq_finish_request(hctx, ctx, rq); 1378 return true; 1379 } 1380 } 1381 1382 static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq) 1383 { 1384 if (rq->tag != -1) 1385 return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false); 1386 1387 return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true); 1388 } 1389 1390 static void blk_mq_try_issue_directly(struct request *rq, blk_qc_t *cookie) 1391 { 1392 struct request_queue *q = rq->q; 1393 struct blk_mq_queue_data bd = { 1394 .rq = rq, 1395 .list = NULL, 1396 .last = 1 1397 }; 1398 struct blk_mq_hw_ctx *hctx; 1399 blk_qc_t new_cookie; 1400 int ret; 1401 1402 if (q->elevator) 1403 goto insert; 1404 1405 if (!blk_mq_get_driver_tag(rq, &hctx, false)) 1406 goto insert; 1407 1408 new_cookie = request_to_qc_t(hctx, rq); 1409 1410 /* 1411 * For OK queue, we are done. For error, kill it. Any other 1412 * error (busy), just add it to our list as we previously 1413 * would have done 1414 */ 1415 ret = q->mq_ops->queue_rq(hctx, &bd); 1416 if (ret == BLK_MQ_RQ_QUEUE_OK) { 1417 *cookie = new_cookie; 1418 return; 1419 } 1420 1421 __blk_mq_requeue_request(rq); 1422 1423 if (ret == BLK_MQ_RQ_QUEUE_ERROR) { 1424 *cookie = BLK_QC_T_NONE; 1425 rq->errors = -EIO; 1426 blk_mq_end_request(rq, rq->errors); 1427 return; 1428 } 1429 1430 insert: 1431 blk_mq_sched_insert_request(rq, false, true, true, false); 1432 } 1433 1434 /* 1435 * Multiple hardware queue variant. This will not use per-process plugs, 1436 * but will attempt to bypass the hctx queueing if we can go straight to 1437 * hardware for SYNC IO. 1438 */ 1439 static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) 1440 { 1441 const int is_sync = op_is_sync(bio->bi_opf); 1442 const int is_flush_fua = op_is_flush(bio->bi_opf); 1443 struct blk_mq_alloc_data data = { .flags = 0 }; 1444 struct request *rq; 1445 unsigned int request_count = 0, srcu_idx; 1446 struct blk_plug *plug; 1447 struct request *same_queue_rq = NULL; 1448 blk_qc_t cookie; 1449 unsigned int wb_acct; 1450 1451 blk_queue_bounce(q, &bio); 1452 1453 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 1454 bio_io_error(bio); 1455 return BLK_QC_T_NONE; 1456 } 1457 1458 blk_queue_split(q, &bio, q->bio_split); 1459 1460 if (!is_flush_fua && !blk_queue_nomerges(q) && 1461 blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq)) 1462 return BLK_QC_T_NONE; 1463 1464 if (blk_mq_sched_bio_merge(q, bio)) 1465 return BLK_QC_T_NONE; 1466 1467 wb_acct = wbt_wait(q->rq_wb, bio, NULL); 1468 1469 trace_block_getrq(q, bio, bio->bi_opf); 1470 1471 rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data); 1472 if (unlikely(!rq)) { 1473 __wbt_done(q->rq_wb, wb_acct); 1474 return BLK_QC_T_NONE; 1475 } 1476 1477 wbt_track(&rq->issue_stat, wb_acct); 1478 1479 cookie = request_to_qc_t(data.hctx, rq); 1480 1481 if (unlikely(is_flush_fua)) { 1482 if (q->elevator) 1483 goto elv_insert; 1484 blk_mq_bio_to_request(rq, bio); 1485 blk_insert_flush(rq); 1486 goto run_queue; 1487 } 1488 1489 plug = current->plug; 1490 /* 1491 * If the driver supports defer issued based on 'last', then 1492 * queue it up like normal since we can potentially save some 1493 * CPU this way. 1494 */ 1495 if (((plug && !blk_queue_nomerges(q)) || is_sync) && 1496 !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) { 1497 struct request *old_rq = NULL; 1498 1499 blk_mq_bio_to_request(rq, bio); 1500 1501 /* 1502 * We do limited plugging. If the bio can be merged, do that. 1503 * Otherwise the existing request in the plug list will be 1504 * issued. So the plug list will have one request at most 1505 */ 1506 if (plug) { 1507 /* 1508 * The plug list might get flushed before this. If that 1509 * happens, same_queue_rq is invalid and plug list is 1510 * empty 1511 */ 1512 if (same_queue_rq && !list_empty(&plug->mq_list)) { 1513 old_rq = same_queue_rq; 1514 list_del_init(&old_rq->queuelist); 1515 } 1516 list_add_tail(&rq->queuelist, &plug->mq_list); 1517 } else /* is_sync */ 1518 old_rq = rq; 1519 blk_mq_put_ctx(data.ctx); 1520 if (!old_rq) 1521 goto done; 1522 1523 if (!(data.hctx->flags & BLK_MQ_F_BLOCKING)) { 1524 rcu_read_lock(); 1525 blk_mq_try_issue_directly(old_rq, &cookie); 1526 rcu_read_unlock(); 1527 } else { 1528 srcu_idx = srcu_read_lock(&data.hctx->queue_rq_srcu); 1529 blk_mq_try_issue_directly(old_rq, &cookie); 1530 srcu_read_unlock(&data.hctx->queue_rq_srcu, srcu_idx); 1531 } 1532 goto done; 1533 } 1534 1535 if (q->elevator) { 1536 elv_insert: 1537 blk_mq_put_ctx(data.ctx); 1538 blk_mq_bio_to_request(rq, bio); 1539 blk_mq_sched_insert_request(rq, false, true, 1540 !is_sync || is_flush_fua, true); 1541 goto done; 1542 } 1543 if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) { 1544 /* 1545 * For a SYNC request, send it to the hardware immediately. For 1546 * an ASYNC request, just ensure that we run it later on. The 1547 * latter allows for merging opportunities and more efficient 1548 * dispatching. 1549 */ 1550 run_queue: 1551 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua); 1552 } 1553 blk_mq_put_ctx(data.ctx); 1554 done: 1555 return cookie; 1556 } 1557 1558 /* 1559 * Single hardware queue variant. This will attempt to use any per-process 1560 * plug for merging and IO deferral. 1561 */ 1562 static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio) 1563 { 1564 const int is_sync = op_is_sync(bio->bi_opf); 1565 const int is_flush_fua = op_is_flush(bio->bi_opf); 1566 struct blk_plug *plug; 1567 unsigned int request_count = 0; 1568 struct blk_mq_alloc_data data = { .flags = 0 }; 1569 struct request *rq; 1570 blk_qc_t cookie; 1571 unsigned int wb_acct; 1572 1573 blk_queue_bounce(q, &bio); 1574 1575 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 1576 bio_io_error(bio); 1577 return BLK_QC_T_NONE; 1578 } 1579 1580 blk_queue_split(q, &bio, q->bio_split); 1581 1582 if (!is_flush_fua && !blk_queue_nomerges(q)) { 1583 if (blk_attempt_plug_merge(q, bio, &request_count, NULL)) 1584 return BLK_QC_T_NONE; 1585 } else 1586 request_count = blk_plug_queued_count(q); 1587 1588 if (blk_mq_sched_bio_merge(q, bio)) 1589 return BLK_QC_T_NONE; 1590 1591 wb_acct = wbt_wait(q->rq_wb, bio, NULL); 1592 1593 trace_block_getrq(q, bio, bio->bi_opf); 1594 1595 rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data); 1596 if (unlikely(!rq)) { 1597 __wbt_done(q->rq_wb, wb_acct); 1598 return BLK_QC_T_NONE; 1599 } 1600 1601 wbt_track(&rq->issue_stat, wb_acct); 1602 1603 cookie = request_to_qc_t(data.hctx, rq); 1604 1605 if (unlikely(is_flush_fua)) { 1606 if (q->elevator) 1607 goto elv_insert; 1608 blk_mq_bio_to_request(rq, bio); 1609 blk_insert_flush(rq); 1610 goto run_queue; 1611 } 1612 1613 /* 1614 * A task plug currently exists. Since this is completely lockless, 1615 * utilize that to temporarily store requests until the task is 1616 * either done or scheduled away. 1617 */ 1618 plug = current->plug; 1619 if (plug) { 1620 struct request *last = NULL; 1621 1622 blk_mq_bio_to_request(rq, bio); 1623 1624 /* 1625 * @request_count may become stale because of schedule 1626 * out, so check the list again. 1627 */ 1628 if (list_empty(&plug->mq_list)) 1629 request_count = 0; 1630 if (!request_count) 1631 trace_block_plug(q); 1632 else 1633 last = list_entry_rq(plug->mq_list.prev); 1634 1635 blk_mq_put_ctx(data.ctx); 1636 1637 if (request_count >= BLK_MAX_REQUEST_COUNT || (last && 1638 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { 1639 blk_flush_plug_list(plug, false); 1640 trace_block_plug(q); 1641 } 1642 1643 list_add_tail(&rq->queuelist, &plug->mq_list); 1644 return cookie; 1645 } 1646 1647 if (q->elevator) { 1648 elv_insert: 1649 blk_mq_put_ctx(data.ctx); 1650 blk_mq_bio_to_request(rq, bio); 1651 blk_mq_sched_insert_request(rq, false, true, 1652 !is_sync || is_flush_fua, true); 1653 goto done; 1654 } 1655 if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) { 1656 /* 1657 * For a SYNC request, send it to the hardware immediately. For 1658 * an ASYNC request, just ensure that we run it later on. The 1659 * latter allows for merging opportunities and more efficient 1660 * dispatching. 1661 */ 1662 run_queue: 1663 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua); 1664 } 1665 1666 blk_mq_put_ctx(data.ctx); 1667 done: 1668 return cookie; 1669 } 1670 1671 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 1672 unsigned int hctx_idx) 1673 { 1674 struct page *page; 1675 1676 if (tags->rqs && set->ops->exit_request) { 1677 int i; 1678 1679 for (i = 0; i < tags->nr_tags; i++) { 1680 struct request *rq = tags->static_rqs[i]; 1681 1682 if (!rq) 1683 continue; 1684 set->ops->exit_request(set->driver_data, rq, 1685 hctx_idx, i); 1686 tags->static_rqs[i] = NULL; 1687 } 1688 } 1689 1690 while (!list_empty(&tags->page_list)) { 1691 page = list_first_entry(&tags->page_list, struct page, lru); 1692 list_del_init(&page->lru); 1693 /* 1694 * Remove kmemleak object previously allocated in 1695 * blk_mq_init_rq_map(). 1696 */ 1697 kmemleak_free(page_address(page)); 1698 __free_pages(page, page->private); 1699 } 1700 } 1701 1702 void blk_mq_free_rq_map(struct blk_mq_tags *tags) 1703 { 1704 kfree(tags->rqs); 1705 tags->rqs = NULL; 1706 kfree(tags->static_rqs); 1707 tags->static_rqs = NULL; 1708 1709 blk_mq_free_tags(tags); 1710 } 1711 1712 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, 1713 unsigned int hctx_idx, 1714 unsigned int nr_tags, 1715 unsigned int reserved_tags) 1716 { 1717 struct blk_mq_tags *tags; 1718 1719 tags = blk_mq_init_tags(nr_tags, reserved_tags, 1720 set->numa_node, 1721 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags)); 1722 if (!tags) 1723 return NULL; 1724 1725 tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *), 1726 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 1727 set->numa_node); 1728 if (!tags->rqs) { 1729 blk_mq_free_tags(tags); 1730 return NULL; 1731 } 1732 1733 tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *), 1734 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 1735 set->numa_node); 1736 if (!tags->static_rqs) { 1737 kfree(tags->rqs); 1738 blk_mq_free_tags(tags); 1739 return NULL; 1740 } 1741 1742 return tags; 1743 } 1744 1745 static size_t order_to_size(unsigned int order) 1746 { 1747 return (size_t)PAGE_SIZE << order; 1748 } 1749 1750 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 1751 unsigned int hctx_idx, unsigned int depth) 1752 { 1753 unsigned int i, j, entries_per_page, max_order = 4; 1754 size_t rq_size, left; 1755 1756 INIT_LIST_HEAD(&tags->page_list); 1757 1758 /* 1759 * rq_size is the size of the request plus driver payload, rounded 1760 * to the cacheline size 1761 */ 1762 rq_size = round_up(sizeof(struct request) + set->cmd_size, 1763 cache_line_size()); 1764 left = rq_size * depth; 1765 1766 for (i = 0; i < depth; ) { 1767 int this_order = max_order; 1768 struct page *page; 1769 int to_do; 1770 void *p; 1771 1772 while (this_order && left < order_to_size(this_order - 1)) 1773 this_order--; 1774 1775 do { 1776 page = alloc_pages_node(set->numa_node, 1777 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, 1778 this_order); 1779 if (page) 1780 break; 1781 if (!this_order--) 1782 break; 1783 if (order_to_size(this_order) < rq_size) 1784 break; 1785 } while (1); 1786 1787 if (!page) 1788 goto fail; 1789 1790 page->private = this_order; 1791 list_add_tail(&page->lru, &tags->page_list); 1792 1793 p = page_address(page); 1794 /* 1795 * Allow kmemleak to scan these pages as they contain pointers 1796 * to additional allocations like via ops->init_request(). 1797 */ 1798 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); 1799 entries_per_page = order_to_size(this_order) / rq_size; 1800 to_do = min(entries_per_page, depth - i); 1801 left -= to_do * rq_size; 1802 for (j = 0; j < to_do; j++) { 1803 struct request *rq = p; 1804 1805 tags->static_rqs[i] = rq; 1806 if (set->ops->init_request) { 1807 if (set->ops->init_request(set->driver_data, 1808 rq, hctx_idx, i, 1809 set->numa_node)) { 1810 tags->static_rqs[i] = NULL; 1811 goto fail; 1812 } 1813 } 1814 1815 p += rq_size; 1816 i++; 1817 } 1818 } 1819 return 0; 1820 1821 fail: 1822 blk_mq_free_rqs(set, tags, hctx_idx); 1823 return -ENOMEM; 1824 } 1825 1826 /* 1827 * 'cpu' is going away. splice any existing rq_list entries from this 1828 * software queue to the hw queue dispatch list, and ensure that it 1829 * gets run. 1830 */ 1831 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) 1832 { 1833 struct blk_mq_hw_ctx *hctx; 1834 struct blk_mq_ctx *ctx; 1835 LIST_HEAD(tmp); 1836 1837 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead); 1838 ctx = __blk_mq_get_ctx(hctx->queue, cpu); 1839 1840 spin_lock(&ctx->lock); 1841 if (!list_empty(&ctx->rq_list)) { 1842 list_splice_init(&ctx->rq_list, &tmp); 1843 blk_mq_hctx_clear_pending(hctx, ctx); 1844 } 1845 spin_unlock(&ctx->lock); 1846 1847 if (list_empty(&tmp)) 1848 return 0; 1849 1850 spin_lock(&hctx->lock); 1851 list_splice_tail_init(&tmp, &hctx->dispatch); 1852 spin_unlock(&hctx->lock); 1853 1854 blk_mq_run_hw_queue(hctx, true); 1855 return 0; 1856 } 1857 1858 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) 1859 { 1860 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD, 1861 &hctx->cpuhp_dead); 1862 } 1863 1864 /* hctx->ctxs will be freed in queue's release handler */ 1865 static void blk_mq_exit_hctx(struct request_queue *q, 1866 struct blk_mq_tag_set *set, 1867 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 1868 { 1869 unsigned flush_start_tag = set->queue_depth; 1870 1871 blk_mq_tag_idle(hctx); 1872 1873 if (set->ops->exit_request) 1874 set->ops->exit_request(set->driver_data, 1875 hctx->fq->flush_rq, hctx_idx, 1876 flush_start_tag + hctx_idx); 1877 1878 if (set->ops->exit_hctx) 1879 set->ops->exit_hctx(hctx, hctx_idx); 1880 1881 if (hctx->flags & BLK_MQ_F_BLOCKING) 1882 cleanup_srcu_struct(&hctx->queue_rq_srcu); 1883 1884 blk_mq_remove_cpuhp(hctx); 1885 blk_free_flush_queue(hctx->fq); 1886 sbitmap_free(&hctx->ctx_map); 1887 } 1888 1889 static void blk_mq_exit_hw_queues(struct request_queue *q, 1890 struct blk_mq_tag_set *set, int nr_queue) 1891 { 1892 struct blk_mq_hw_ctx *hctx; 1893 unsigned int i; 1894 1895 queue_for_each_hw_ctx(q, hctx, i) { 1896 if (i == nr_queue) 1897 break; 1898 blk_mq_exit_hctx(q, set, hctx, i); 1899 } 1900 } 1901 1902 static void blk_mq_free_hw_queues(struct request_queue *q, 1903 struct blk_mq_tag_set *set) 1904 { 1905 struct blk_mq_hw_ctx *hctx; 1906 unsigned int i; 1907 1908 queue_for_each_hw_ctx(q, hctx, i) 1909 free_cpumask_var(hctx->cpumask); 1910 } 1911 1912 static int blk_mq_init_hctx(struct request_queue *q, 1913 struct blk_mq_tag_set *set, 1914 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) 1915 { 1916 int node; 1917 unsigned flush_start_tag = set->queue_depth; 1918 1919 node = hctx->numa_node; 1920 if (node == NUMA_NO_NODE) 1921 node = hctx->numa_node = set->numa_node; 1922 1923 INIT_WORK(&hctx->run_work, blk_mq_run_work_fn); 1924 INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn); 1925 spin_lock_init(&hctx->lock); 1926 INIT_LIST_HEAD(&hctx->dispatch); 1927 hctx->queue = q; 1928 hctx->queue_num = hctx_idx; 1929 hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED; 1930 1931 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead); 1932 1933 hctx->tags = set->tags[hctx_idx]; 1934 1935 /* 1936 * Allocate space for all possible cpus to avoid allocation at 1937 * runtime 1938 */ 1939 hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *), 1940 GFP_KERNEL, node); 1941 if (!hctx->ctxs) 1942 goto unregister_cpu_notifier; 1943 1944 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL, 1945 node)) 1946 goto free_ctxs; 1947 1948 hctx->nr_ctx = 0; 1949 1950 if (set->ops->init_hctx && 1951 set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) 1952 goto free_bitmap; 1953 1954 hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size); 1955 if (!hctx->fq) 1956 goto exit_hctx; 1957 1958 if (set->ops->init_request && 1959 set->ops->init_request(set->driver_data, 1960 hctx->fq->flush_rq, hctx_idx, 1961 flush_start_tag + hctx_idx, node)) 1962 goto free_fq; 1963 1964 if (hctx->flags & BLK_MQ_F_BLOCKING) 1965 init_srcu_struct(&hctx->queue_rq_srcu); 1966 1967 return 0; 1968 1969 free_fq: 1970 kfree(hctx->fq); 1971 exit_hctx: 1972 if (set->ops->exit_hctx) 1973 set->ops->exit_hctx(hctx, hctx_idx); 1974 free_bitmap: 1975 sbitmap_free(&hctx->ctx_map); 1976 free_ctxs: 1977 kfree(hctx->ctxs); 1978 unregister_cpu_notifier: 1979 blk_mq_remove_cpuhp(hctx); 1980 return -1; 1981 } 1982 1983 static void blk_mq_init_cpu_queues(struct request_queue *q, 1984 unsigned int nr_hw_queues) 1985 { 1986 unsigned int i; 1987 1988 for_each_possible_cpu(i) { 1989 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i); 1990 struct blk_mq_hw_ctx *hctx; 1991 1992 memset(__ctx, 0, sizeof(*__ctx)); 1993 __ctx->cpu = i; 1994 spin_lock_init(&__ctx->lock); 1995 INIT_LIST_HEAD(&__ctx->rq_list); 1996 __ctx->queue = q; 1997 blk_stat_init(&__ctx->stat[BLK_STAT_READ]); 1998 blk_stat_init(&__ctx->stat[BLK_STAT_WRITE]); 1999 2000 /* If the cpu isn't online, the cpu is mapped to first hctx */ 2001 if (!cpu_online(i)) 2002 continue; 2003 2004 hctx = blk_mq_map_queue(q, i); 2005 2006 /* 2007 * Set local node, IFF we have more than one hw queue. If 2008 * not, we remain on the home node of the device 2009 */ 2010 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) 2011 hctx->numa_node = local_memory_node(cpu_to_node(i)); 2012 } 2013 } 2014 2015 static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx) 2016 { 2017 int ret = 0; 2018 2019 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx, 2020 set->queue_depth, set->reserved_tags); 2021 if (!set->tags[hctx_idx]) 2022 return false; 2023 2024 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx, 2025 set->queue_depth); 2026 if (!ret) 2027 return true; 2028 2029 blk_mq_free_rq_map(set->tags[hctx_idx]); 2030 set->tags[hctx_idx] = NULL; 2031 return false; 2032 } 2033 2034 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set, 2035 unsigned int hctx_idx) 2036 { 2037 if (set->tags[hctx_idx]) { 2038 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx); 2039 blk_mq_free_rq_map(set->tags[hctx_idx]); 2040 set->tags[hctx_idx] = NULL; 2041 } 2042 } 2043 2044 static void blk_mq_map_swqueue(struct request_queue *q, 2045 const struct cpumask *online_mask) 2046 { 2047 unsigned int i, hctx_idx; 2048 struct blk_mq_hw_ctx *hctx; 2049 struct blk_mq_ctx *ctx; 2050 struct blk_mq_tag_set *set = q->tag_set; 2051 2052 /* 2053 * Avoid others reading imcomplete hctx->cpumask through sysfs 2054 */ 2055 mutex_lock(&q->sysfs_lock); 2056 2057 queue_for_each_hw_ctx(q, hctx, i) { 2058 cpumask_clear(hctx->cpumask); 2059 hctx->nr_ctx = 0; 2060 } 2061 2062 /* 2063 * Map software to hardware queues 2064 */ 2065 for_each_possible_cpu(i) { 2066 /* If the cpu isn't online, the cpu is mapped to first hctx */ 2067 if (!cpumask_test_cpu(i, online_mask)) 2068 continue; 2069 2070 hctx_idx = q->mq_map[i]; 2071 /* unmapped hw queue can be remapped after CPU topo changed */ 2072 if (!set->tags[hctx_idx] && 2073 !__blk_mq_alloc_rq_map(set, hctx_idx)) { 2074 /* 2075 * If tags initialization fail for some hctx, 2076 * that hctx won't be brought online. In this 2077 * case, remap the current ctx to hctx[0] which 2078 * is guaranteed to always have tags allocated 2079 */ 2080 q->mq_map[i] = 0; 2081 } 2082 2083 ctx = per_cpu_ptr(q->queue_ctx, i); 2084 hctx = blk_mq_map_queue(q, i); 2085 2086 cpumask_set_cpu(i, hctx->cpumask); 2087 ctx->index_hw = hctx->nr_ctx; 2088 hctx->ctxs[hctx->nr_ctx++] = ctx; 2089 } 2090 2091 mutex_unlock(&q->sysfs_lock); 2092 2093 queue_for_each_hw_ctx(q, hctx, i) { 2094 /* 2095 * If no software queues are mapped to this hardware queue, 2096 * disable it and free the request entries. 2097 */ 2098 if (!hctx->nr_ctx) { 2099 /* Never unmap queue 0. We need it as a 2100 * fallback in case of a new remap fails 2101 * allocation 2102 */ 2103 if (i && set->tags[i]) 2104 blk_mq_free_map_and_requests(set, i); 2105 2106 hctx->tags = NULL; 2107 continue; 2108 } 2109 2110 hctx->tags = set->tags[i]; 2111 WARN_ON(!hctx->tags); 2112 2113 /* 2114 * Set the map size to the number of mapped software queues. 2115 * This is more accurate and more efficient than looping 2116 * over all possibly mapped software queues. 2117 */ 2118 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx); 2119 2120 /* 2121 * Initialize batch roundrobin counts 2122 */ 2123 hctx->next_cpu = cpumask_first(hctx->cpumask); 2124 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 2125 } 2126 } 2127 2128 static void queue_set_hctx_shared(struct request_queue *q, bool shared) 2129 { 2130 struct blk_mq_hw_ctx *hctx; 2131 int i; 2132 2133 queue_for_each_hw_ctx(q, hctx, i) { 2134 if (shared) 2135 hctx->flags |= BLK_MQ_F_TAG_SHARED; 2136 else 2137 hctx->flags &= ~BLK_MQ_F_TAG_SHARED; 2138 } 2139 } 2140 2141 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared) 2142 { 2143 struct request_queue *q; 2144 2145 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2146 blk_mq_freeze_queue(q); 2147 queue_set_hctx_shared(q, shared); 2148 blk_mq_unfreeze_queue(q); 2149 } 2150 } 2151 2152 static void blk_mq_del_queue_tag_set(struct request_queue *q) 2153 { 2154 struct blk_mq_tag_set *set = q->tag_set; 2155 2156 mutex_lock(&set->tag_list_lock); 2157 list_del_init(&q->tag_set_list); 2158 if (list_is_singular(&set->tag_list)) { 2159 /* just transitioned to unshared */ 2160 set->flags &= ~BLK_MQ_F_TAG_SHARED; 2161 /* update existing queue */ 2162 blk_mq_update_tag_set_depth(set, false); 2163 } 2164 mutex_unlock(&set->tag_list_lock); 2165 } 2166 2167 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, 2168 struct request_queue *q) 2169 { 2170 q->tag_set = set; 2171 2172 mutex_lock(&set->tag_list_lock); 2173 2174 /* Check to see if we're transitioning to shared (from 1 to 2 queues). */ 2175 if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) { 2176 set->flags |= BLK_MQ_F_TAG_SHARED; 2177 /* update existing queue */ 2178 blk_mq_update_tag_set_depth(set, true); 2179 } 2180 if (set->flags & BLK_MQ_F_TAG_SHARED) 2181 queue_set_hctx_shared(q, true); 2182 list_add_tail(&q->tag_set_list, &set->tag_list); 2183 2184 mutex_unlock(&set->tag_list_lock); 2185 } 2186 2187 /* 2188 * It is the actual release handler for mq, but we do it from 2189 * request queue's release handler for avoiding use-after-free 2190 * and headache because q->mq_kobj shouldn't have been introduced, 2191 * but we can't group ctx/kctx kobj without it. 2192 */ 2193 void blk_mq_release(struct request_queue *q) 2194 { 2195 struct blk_mq_hw_ctx *hctx; 2196 unsigned int i; 2197 2198 blk_mq_sched_teardown(q); 2199 2200 /* hctx kobj stays in hctx */ 2201 queue_for_each_hw_ctx(q, hctx, i) { 2202 if (!hctx) 2203 continue; 2204 kfree(hctx->ctxs); 2205 kfree(hctx); 2206 } 2207 2208 q->mq_map = NULL; 2209 2210 kfree(q->queue_hw_ctx); 2211 2212 /* ctx kobj stays in queue_ctx */ 2213 free_percpu(q->queue_ctx); 2214 } 2215 2216 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) 2217 { 2218 struct request_queue *uninit_q, *q; 2219 2220 uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node); 2221 if (!uninit_q) 2222 return ERR_PTR(-ENOMEM); 2223 2224 q = blk_mq_init_allocated_queue(set, uninit_q); 2225 if (IS_ERR(q)) 2226 blk_cleanup_queue(uninit_q); 2227 2228 return q; 2229 } 2230 EXPORT_SYMBOL(blk_mq_init_queue); 2231 2232 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, 2233 struct request_queue *q) 2234 { 2235 int i, j; 2236 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; 2237 2238 blk_mq_sysfs_unregister(q); 2239 for (i = 0; i < set->nr_hw_queues; i++) { 2240 int node; 2241 2242 if (hctxs[i]) 2243 continue; 2244 2245 node = blk_mq_hw_queue_to_node(q->mq_map, i); 2246 hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx), 2247 GFP_KERNEL, node); 2248 if (!hctxs[i]) 2249 break; 2250 2251 if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL, 2252 node)) { 2253 kfree(hctxs[i]); 2254 hctxs[i] = NULL; 2255 break; 2256 } 2257 2258 atomic_set(&hctxs[i]->nr_active, 0); 2259 hctxs[i]->numa_node = node; 2260 hctxs[i]->queue_num = i; 2261 2262 if (blk_mq_init_hctx(q, set, hctxs[i], i)) { 2263 free_cpumask_var(hctxs[i]->cpumask); 2264 kfree(hctxs[i]); 2265 hctxs[i] = NULL; 2266 break; 2267 } 2268 blk_mq_hctx_kobj_init(hctxs[i]); 2269 } 2270 for (j = i; j < q->nr_hw_queues; j++) { 2271 struct blk_mq_hw_ctx *hctx = hctxs[j]; 2272 2273 if (hctx) { 2274 if (hctx->tags) 2275 blk_mq_free_map_and_requests(set, j); 2276 blk_mq_exit_hctx(q, set, hctx, j); 2277 free_cpumask_var(hctx->cpumask); 2278 kobject_put(&hctx->kobj); 2279 kfree(hctx->ctxs); 2280 kfree(hctx); 2281 hctxs[j] = NULL; 2282 2283 } 2284 } 2285 q->nr_hw_queues = i; 2286 blk_mq_sysfs_register(q); 2287 } 2288 2289 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 2290 struct request_queue *q) 2291 { 2292 /* mark the queue as mq asap */ 2293 q->mq_ops = set->ops; 2294 2295 q->queue_ctx = alloc_percpu(struct blk_mq_ctx); 2296 if (!q->queue_ctx) 2297 goto err_exit; 2298 2299 q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)), 2300 GFP_KERNEL, set->numa_node); 2301 if (!q->queue_hw_ctx) 2302 goto err_percpu; 2303 2304 q->mq_map = set->mq_map; 2305 2306 blk_mq_realloc_hw_ctxs(set, q); 2307 if (!q->nr_hw_queues) 2308 goto err_hctxs; 2309 2310 INIT_WORK(&q->timeout_work, blk_mq_timeout_work); 2311 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ); 2312 2313 q->nr_queues = nr_cpu_ids; 2314 2315 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; 2316 2317 if (!(set->flags & BLK_MQ_F_SG_MERGE)) 2318 q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE; 2319 2320 q->sg_reserved_size = INT_MAX; 2321 2322 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); 2323 INIT_LIST_HEAD(&q->requeue_list); 2324 spin_lock_init(&q->requeue_lock); 2325 2326 if (q->nr_hw_queues > 1) 2327 blk_queue_make_request(q, blk_mq_make_request); 2328 else 2329 blk_queue_make_request(q, blk_sq_make_request); 2330 2331 /* 2332 * Do this after blk_queue_make_request() overrides it... 2333 */ 2334 q->nr_requests = set->queue_depth; 2335 2336 /* 2337 * Default to classic polling 2338 */ 2339 q->poll_nsec = -1; 2340 2341 if (set->ops->complete) 2342 blk_queue_softirq_done(q, set->ops->complete); 2343 2344 blk_mq_init_cpu_queues(q, set->nr_hw_queues); 2345 2346 get_online_cpus(); 2347 mutex_lock(&all_q_mutex); 2348 2349 list_add_tail(&q->all_q_node, &all_q_list); 2350 blk_mq_add_queue_tag_set(set, q); 2351 blk_mq_map_swqueue(q, cpu_online_mask); 2352 2353 mutex_unlock(&all_q_mutex); 2354 put_online_cpus(); 2355 2356 if (!(set->flags & BLK_MQ_F_NO_SCHED)) { 2357 int ret; 2358 2359 ret = blk_mq_sched_init(q); 2360 if (ret) 2361 return ERR_PTR(ret); 2362 } 2363 2364 return q; 2365 2366 err_hctxs: 2367 kfree(q->queue_hw_ctx); 2368 err_percpu: 2369 free_percpu(q->queue_ctx); 2370 err_exit: 2371 q->mq_ops = NULL; 2372 return ERR_PTR(-ENOMEM); 2373 } 2374 EXPORT_SYMBOL(blk_mq_init_allocated_queue); 2375 2376 void blk_mq_free_queue(struct request_queue *q) 2377 { 2378 struct blk_mq_tag_set *set = q->tag_set; 2379 2380 mutex_lock(&all_q_mutex); 2381 list_del_init(&q->all_q_node); 2382 mutex_unlock(&all_q_mutex); 2383 2384 wbt_exit(q); 2385 2386 blk_mq_del_queue_tag_set(q); 2387 2388 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); 2389 blk_mq_free_hw_queues(q, set); 2390 } 2391 2392 /* Basically redo blk_mq_init_queue with queue frozen */ 2393 static void blk_mq_queue_reinit(struct request_queue *q, 2394 const struct cpumask *online_mask) 2395 { 2396 WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth)); 2397 2398 blk_mq_sysfs_unregister(q); 2399 2400 /* 2401 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe 2402 * we should change hctx numa_node according to new topology (this 2403 * involves free and re-allocate memory, worthy doing?) 2404 */ 2405 2406 blk_mq_map_swqueue(q, online_mask); 2407 2408 blk_mq_sysfs_register(q); 2409 } 2410 2411 /* 2412 * New online cpumask which is going to be set in this hotplug event. 2413 * Declare this cpumasks as global as cpu-hotplug operation is invoked 2414 * one-by-one and dynamically allocating this could result in a failure. 2415 */ 2416 static struct cpumask cpuhp_online_new; 2417 2418 static void blk_mq_queue_reinit_work(void) 2419 { 2420 struct request_queue *q; 2421 2422 mutex_lock(&all_q_mutex); 2423 /* 2424 * We need to freeze and reinit all existing queues. Freezing 2425 * involves synchronous wait for an RCU grace period and doing it 2426 * one by one may take a long time. Start freezing all queues in 2427 * one swoop and then wait for the completions so that freezing can 2428 * take place in parallel. 2429 */ 2430 list_for_each_entry(q, &all_q_list, all_q_node) 2431 blk_mq_freeze_queue_start(q); 2432 list_for_each_entry(q, &all_q_list, all_q_node) 2433 blk_mq_freeze_queue_wait(q); 2434 2435 list_for_each_entry(q, &all_q_list, all_q_node) 2436 blk_mq_queue_reinit(q, &cpuhp_online_new); 2437 2438 list_for_each_entry(q, &all_q_list, all_q_node) 2439 blk_mq_unfreeze_queue(q); 2440 2441 mutex_unlock(&all_q_mutex); 2442 } 2443 2444 static int blk_mq_queue_reinit_dead(unsigned int cpu) 2445 { 2446 cpumask_copy(&cpuhp_online_new, cpu_online_mask); 2447 blk_mq_queue_reinit_work(); 2448 return 0; 2449 } 2450 2451 /* 2452 * Before hotadded cpu starts handling requests, new mappings must be 2453 * established. Otherwise, these requests in hw queue might never be 2454 * dispatched. 2455 * 2456 * For example, there is a single hw queue (hctx) and two CPU queues (ctx0 2457 * for CPU0, and ctx1 for CPU1). 2458 * 2459 * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list 2460 * and set bit0 in pending bitmap as ctx1->index_hw is still zero. 2461 * 2462 * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set 2463 * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list. 2464 * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is 2465 * ignored. 2466 */ 2467 static int blk_mq_queue_reinit_prepare(unsigned int cpu) 2468 { 2469 cpumask_copy(&cpuhp_online_new, cpu_online_mask); 2470 cpumask_set_cpu(cpu, &cpuhp_online_new); 2471 blk_mq_queue_reinit_work(); 2472 return 0; 2473 } 2474 2475 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2476 { 2477 int i; 2478 2479 for (i = 0; i < set->nr_hw_queues; i++) 2480 if (!__blk_mq_alloc_rq_map(set, i)) 2481 goto out_unwind; 2482 2483 return 0; 2484 2485 out_unwind: 2486 while (--i >= 0) 2487 blk_mq_free_rq_map(set->tags[i]); 2488 2489 return -ENOMEM; 2490 } 2491 2492 /* 2493 * Allocate the request maps associated with this tag_set. Note that this 2494 * may reduce the depth asked for, if memory is tight. set->queue_depth 2495 * will be updated to reflect the allocated depth. 2496 */ 2497 static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2498 { 2499 unsigned int depth; 2500 int err; 2501 2502 depth = set->queue_depth; 2503 do { 2504 err = __blk_mq_alloc_rq_maps(set); 2505 if (!err) 2506 break; 2507 2508 set->queue_depth >>= 1; 2509 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) { 2510 err = -ENOMEM; 2511 break; 2512 } 2513 } while (set->queue_depth); 2514 2515 if (!set->queue_depth || err) { 2516 pr_err("blk-mq: failed to allocate request map\n"); 2517 return -ENOMEM; 2518 } 2519 2520 if (depth != set->queue_depth) 2521 pr_info("blk-mq: reduced tag depth (%u -> %u)\n", 2522 depth, set->queue_depth); 2523 2524 return 0; 2525 } 2526 2527 /* 2528 * Alloc a tag set to be associated with one or more request queues. 2529 * May fail with EINVAL for various error conditions. May adjust the 2530 * requested depth down, if if it too large. In that case, the set 2531 * value will be stored in set->queue_depth. 2532 */ 2533 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) 2534 { 2535 int ret; 2536 2537 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS); 2538 2539 if (!set->nr_hw_queues) 2540 return -EINVAL; 2541 if (!set->queue_depth) 2542 return -EINVAL; 2543 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) 2544 return -EINVAL; 2545 2546 if (!set->ops->queue_rq) 2547 return -EINVAL; 2548 2549 if (set->queue_depth > BLK_MQ_MAX_DEPTH) { 2550 pr_info("blk-mq: reduced tag depth to %u\n", 2551 BLK_MQ_MAX_DEPTH); 2552 set->queue_depth = BLK_MQ_MAX_DEPTH; 2553 } 2554 2555 /* 2556 * If a crashdump is active, then we are potentially in a very 2557 * memory constrained environment. Limit us to 1 queue and 2558 * 64 tags to prevent using too much memory. 2559 */ 2560 if (is_kdump_kernel()) { 2561 set->nr_hw_queues = 1; 2562 set->queue_depth = min(64U, set->queue_depth); 2563 } 2564 /* 2565 * There is no use for more h/w queues than cpus. 2566 */ 2567 if (set->nr_hw_queues > nr_cpu_ids) 2568 set->nr_hw_queues = nr_cpu_ids; 2569 2570 set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *), 2571 GFP_KERNEL, set->numa_node); 2572 if (!set->tags) 2573 return -ENOMEM; 2574 2575 ret = -ENOMEM; 2576 set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids, 2577 GFP_KERNEL, set->numa_node); 2578 if (!set->mq_map) 2579 goto out_free_tags; 2580 2581 if (set->ops->map_queues) 2582 ret = set->ops->map_queues(set); 2583 else 2584 ret = blk_mq_map_queues(set); 2585 if (ret) 2586 goto out_free_mq_map; 2587 2588 ret = blk_mq_alloc_rq_maps(set); 2589 if (ret) 2590 goto out_free_mq_map; 2591 2592 mutex_init(&set->tag_list_lock); 2593 INIT_LIST_HEAD(&set->tag_list); 2594 2595 return 0; 2596 2597 out_free_mq_map: 2598 kfree(set->mq_map); 2599 set->mq_map = NULL; 2600 out_free_tags: 2601 kfree(set->tags); 2602 set->tags = NULL; 2603 return ret; 2604 } 2605 EXPORT_SYMBOL(blk_mq_alloc_tag_set); 2606 2607 void blk_mq_free_tag_set(struct blk_mq_tag_set *set) 2608 { 2609 int i; 2610 2611 for (i = 0; i < nr_cpu_ids; i++) 2612 blk_mq_free_map_and_requests(set, i); 2613 2614 kfree(set->mq_map); 2615 set->mq_map = NULL; 2616 2617 kfree(set->tags); 2618 set->tags = NULL; 2619 } 2620 EXPORT_SYMBOL(blk_mq_free_tag_set); 2621 2622 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) 2623 { 2624 struct blk_mq_tag_set *set = q->tag_set; 2625 struct blk_mq_hw_ctx *hctx; 2626 int i, ret; 2627 2628 if (!set) 2629 return -EINVAL; 2630 2631 blk_mq_freeze_queue(q); 2632 blk_mq_quiesce_queue(q); 2633 2634 ret = 0; 2635 queue_for_each_hw_ctx(q, hctx, i) { 2636 if (!hctx->tags) 2637 continue; 2638 /* 2639 * If we're using an MQ scheduler, just update the scheduler 2640 * queue depth. This is similar to what the old code would do. 2641 */ 2642 if (!hctx->sched_tags) { 2643 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, 2644 min(nr, set->queue_depth), 2645 false); 2646 } else { 2647 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags, 2648 nr, true); 2649 } 2650 if (ret) 2651 break; 2652 } 2653 2654 if (!ret) 2655 q->nr_requests = nr; 2656 2657 blk_mq_unfreeze_queue(q); 2658 blk_mq_start_stopped_hw_queues(q, true); 2659 2660 return ret; 2661 } 2662 2663 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) 2664 { 2665 struct request_queue *q; 2666 2667 if (nr_hw_queues > nr_cpu_ids) 2668 nr_hw_queues = nr_cpu_ids; 2669 if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues) 2670 return; 2671 2672 list_for_each_entry(q, &set->tag_list, tag_set_list) 2673 blk_mq_freeze_queue(q); 2674 2675 set->nr_hw_queues = nr_hw_queues; 2676 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2677 blk_mq_realloc_hw_ctxs(set, q); 2678 2679 /* 2680 * Manually set the make_request_fn as blk_queue_make_request 2681 * resets a lot of the queue settings. 2682 */ 2683 if (q->nr_hw_queues > 1) 2684 q->make_request_fn = blk_mq_make_request; 2685 else 2686 q->make_request_fn = blk_sq_make_request; 2687 2688 blk_mq_queue_reinit(q, cpu_online_mask); 2689 } 2690 2691 list_for_each_entry(q, &set->tag_list, tag_set_list) 2692 blk_mq_unfreeze_queue(q); 2693 } 2694 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); 2695 2696 static unsigned long blk_mq_poll_nsecs(struct request_queue *q, 2697 struct blk_mq_hw_ctx *hctx, 2698 struct request *rq) 2699 { 2700 struct blk_rq_stat stat[2]; 2701 unsigned long ret = 0; 2702 2703 /* 2704 * If stats collection isn't on, don't sleep but turn it on for 2705 * future users 2706 */ 2707 if (!blk_stat_enable(q)) 2708 return 0; 2709 2710 /* 2711 * We don't have to do this once per IO, should optimize this 2712 * to just use the current window of stats until it changes 2713 */ 2714 memset(&stat, 0, sizeof(stat)); 2715 blk_hctx_stat_get(hctx, stat); 2716 2717 /* 2718 * As an optimistic guess, use half of the mean service time 2719 * for this type of request. We can (and should) make this smarter. 2720 * For instance, if the completion latencies are tight, we can 2721 * get closer than just half the mean. This is especially 2722 * important on devices where the completion latencies are longer 2723 * than ~10 usec. 2724 */ 2725 if (req_op(rq) == REQ_OP_READ && stat[BLK_STAT_READ].nr_samples) 2726 ret = (stat[BLK_STAT_READ].mean + 1) / 2; 2727 else if (req_op(rq) == REQ_OP_WRITE && stat[BLK_STAT_WRITE].nr_samples) 2728 ret = (stat[BLK_STAT_WRITE].mean + 1) / 2; 2729 2730 return ret; 2731 } 2732 2733 static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, 2734 struct blk_mq_hw_ctx *hctx, 2735 struct request *rq) 2736 { 2737 struct hrtimer_sleeper hs; 2738 enum hrtimer_mode mode; 2739 unsigned int nsecs; 2740 ktime_t kt; 2741 2742 if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags)) 2743 return false; 2744 2745 /* 2746 * poll_nsec can be: 2747 * 2748 * -1: don't ever hybrid sleep 2749 * 0: use half of prev avg 2750 * >0: use this specific value 2751 */ 2752 if (q->poll_nsec == -1) 2753 return false; 2754 else if (q->poll_nsec > 0) 2755 nsecs = q->poll_nsec; 2756 else 2757 nsecs = blk_mq_poll_nsecs(q, hctx, rq); 2758 2759 if (!nsecs) 2760 return false; 2761 2762 set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags); 2763 2764 /* 2765 * This will be replaced with the stats tracking code, using 2766 * 'avg_completion_time / 2' as the pre-sleep target. 2767 */ 2768 kt = nsecs; 2769 2770 mode = HRTIMER_MODE_REL; 2771 hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode); 2772 hrtimer_set_expires(&hs.timer, kt); 2773 2774 hrtimer_init_sleeper(&hs, current); 2775 do { 2776 if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) 2777 break; 2778 set_current_state(TASK_UNINTERRUPTIBLE); 2779 hrtimer_start_expires(&hs.timer, mode); 2780 if (hs.task) 2781 io_schedule(); 2782 hrtimer_cancel(&hs.timer); 2783 mode = HRTIMER_MODE_ABS; 2784 } while (hs.task && !signal_pending(current)); 2785 2786 __set_current_state(TASK_RUNNING); 2787 destroy_hrtimer_on_stack(&hs.timer); 2788 return true; 2789 } 2790 2791 static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq) 2792 { 2793 struct request_queue *q = hctx->queue; 2794 long state; 2795 2796 /* 2797 * If we sleep, have the caller restart the poll loop to reset 2798 * the state. Like for the other success return cases, the 2799 * caller is responsible for checking if the IO completed. If 2800 * the IO isn't complete, we'll get called again and will go 2801 * straight to the busy poll loop. 2802 */ 2803 if (blk_mq_poll_hybrid_sleep(q, hctx, rq)) 2804 return true; 2805 2806 hctx->poll_considered++; 2807 2808 state = current->state; 2809 while (!need_resched()) { 2810 int ret; 2811 2812 hctx->poll_invoked++; 2813 2814 ret = q->mq_ops->poll(hctx, rq->tag); 2815 if (ret > 0) { 2816 hctx->poll_success++; 2817 set_current_state(TASK_RUNNING); 2818 return true; 2819 } 2820 2821 if (signal_pending_state(state, current)) 2822 set_current_state(TASK_RUNNING); 2823 2824 if (current->state == TASK_RUNNING) 2825 return true; 2826 if (ret < 0) 2827 break; 2828 cpu_relax(); 2829 } 2830 2831 return false; 2832 } 2833 2834 bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie) 2835 { 2836 struct blk_mq_hw_ctx *hctx; 2837 struct blk_plug *plug; 2838 struct request *rq; 2839 2840 if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) || 2841 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 2842 return false; 2843 2844 plug = current->plug; 2845 if (plug) 2846 blk_flush_plug_list(plug, false); 2847 2848 hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; 2849 if (!blk_qc_t_is_internal(cookie)) 2850 rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); 2851 else 2852 rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie)); 2853 2854 return __blk_mq_poll(hctx, rq); 2855 } 2856 EXPORT_SYMBOL_GPL(blk_mq_poll); 2857 2858 void blk_mq_disable_hotplug(void) 2859 { 2860 mutex_lock(&all_q_mutex); 2861 } 2862 2863 void blk_mq_enable_hotplug(void) 2864 { 2865 mutex_unlock(&all_q_mutex); 2866 } 2867 2868 static int __init blk_mq_init(void) 2869 { 2870 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, 2871 blk_mq_hctx_notify_dead); 2872 2873 cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare", 2874 blk_mq_queue_reinit_prepare, 2875 blk_mq_queue_reinit_dead); 2876 return 0; 2877 } 2878 subsys_initcall(blk_mq_init); 2879