1 /* 2 * Block multiqueue core code 3 * 4 * Copyright (C) 2013-2014 Jens Axboe 5 * Copyright (C) 2013-2014 Christoph Hellwig 6 */ 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/backing-dev.h> 10 #include <linux/bio.h> 11 #include <linux/blkdev.h> 12 #include <linux/kmemleak.h> 13 #include <linux/mm.h> 14 #include <linux/init.h> 15 #include <linux/slab.h> 16 #include <linux/workqueue.h> 17 #include <linux/smp.h> 18 #include <linux/llist.h> 19 #include <linux/list_sort.h> 20 #include <linux/cpu.h> 21 #include <linux/cache.h> 22 #include <linux/sched/sysctl.h> 23 #include <linux/sched/topology.h> 24 #include <linux/sched/signal.h> 25 #include <linux/delay.h> 26 #include <linux/crash_dump.h> 27 #include <linux/prefetch.h> 28 29 #include <trace/events/block.h> 30 31 #include <linux/blk-mq.h> 32 #include "blk.h" 33 #include "blk-mq.h" 34 #include "blk-mq-debugfs.h" 35 #include "blk-mq-tag.h" 36 #include "blk-stat.h" 37 #include "blk-wbt.h" 38 #include "blk-mq-sched.h" 39 40 static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie); 41 static void blk_mq_poll_stats_start(struct request_queue *q); 42 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); 43 44 static int blk_mq_poll_stats_bkt(const struct request *rq) 45 { 46 int ddir, bytes, bucket; 47 48 ddir = rq_data_dir(rq); 49 bytes = blk_rq_bytes(rq); 50 51 bucket = ddir + 2*(ilog2(bytes) - 9); 52 53 if (bucket < 0) 54 return -1; 55 else if (bucket >= BLK_MQ_POLL_STATS_BKTS) 56 return ddir + BLK_MQ_POLL_STATS_BKTS - 2; 57 58 return bucket; 59 } 60 61 /* 62 * Check if any of the ctx's have pending work in this hardware queue 63 */ 64 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) 65 { 66 return !list_empty_careful(&hctx->dispatch) || 67 sbitmap_any_bit_set(&hctx->ctx_map) || 68 blk_mq_sched_has_work(hctx); 69 } 70 71 /* 72 * Mark this ctx as having pending work in this hardware queue 73 */ 74 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, 75 struct blk_mq_ctx *ctx) 76 { 77 if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw)) 78 sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw); 79 } 80 81 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, 82 struct blk_mq_ctx *ctx) 83 { 84 sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw); 85 } 86 87 struct mq_inflight { 88 struct hd_struct *part; 89 unsigned int *inflight; 90 }; 91 92 static void blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx, 93 struct request *rq, void *priv, 94 bool reserved) 95 { 96 struct mq_inflight *mi = priv; 97 98 /* 99 * index[0] counts the specific partition that was asked for. index[1] 100 * counts the ones that are active on the whole device, so increment 101 * that if mi->part is indeed a partition, and not a whole device. 102 */ 103 if (rq->part == mi->part) 104 mi->inflight[0]++; 105 if (mi->part->partno) 106 mi->inflight[1]++; 107 } 108 109 void blk_mq_in_flight(struct request_queue *q, struct hd_struct *part, 110 unsigned int inflight[2]) 111 { 112 struct mq_inflight mi = { .part = part, .inflight = inflight, }; 113 114 inflight[0] = inflight[1] = 0; 115 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); 116 } 117 118 static void blk_mq_check_inflight_rw(struct blk_mq_hw_ctx *hctx, 119 struct request *rq, void *priv, 120 bool reserved) 121 { 122 struct mq_inflight *mi = priv; 123 124 if (rq->part == mi->part) 125 mi->inflight[rq_data_dir(rq)]++; 126 } 127 128 void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part, 129 unsigned int inflight[2]) 130 { 131 struct mq_inflight mi = { .part = part, .inflight = inflight, }; 132 133 inflight[0] = inflight[1] = 0; 134 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight_rw, &mi); 135 } 136 137 void blk_freeze_queue_start(struct request_queue *q) 138 { 139 int freeze_depth; 140 141 freeze_depth = atomic_inc_return(&q->mq_freeze_depth); 142 if (freeze_depth == 1) { 143 percpu_ref_kill(&q->q_usage_counter); 144 if (q->mq_ops) 145 blk_mq_run_hw_queues(q, false); 146 } 147 } 148 EXPORT_SYMBOL_GPL(blk_freeze_queue_start); 149 150 void blk_mq_freeze_queue_wait(struct request_queue *q) 151 { 152 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter)); 153 } 154 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait); 155 156 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, 157 unsigned long timeout) 158 { 159 return wait_event_timeout(q->mq_freeze_wq, 160 percpu_ref_is_zero(&q->q_usage_counter), 161 timeout); 162 } 163 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout); 164 165 /* 166 * Guarantee no request is in use, so we can change any data structure of 167 * the queue afterward. 168 */ 169 void blk_freeze_queue(struct request_queue *q) 170 { 171 /* 172 * In the !blk_mq case we are only calling this to kill the 173 * q_usage_counter, otherwise this increases the freeze depth 174 * and waits for it to return to zero. For this reason there is 175 * no blk_unfreeze_queue(), and blk_freeze_queue() is not 176 * exported to drivers as the only user for unfreeze is blk_mq. 177 */ 178 blk_freeze_queue_start(q); 179 if (!q->mq_ops) 180 blk_drain_queue(q); 181 blk_mq_freeze_queue_wait(q); 182 } 183 184 void blk_mq_freeze_queue(struct request_queue *q) 185 { 186 /* 187 * ...just an alias to keep freeze and unfreeze actions balanced 188 * in the blk_mq_* namespace 189 */ 190 blk_freeze_queue(q); 191 } 192 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue); 193 194 void blk_mq_unfreeze_queue(struct request_queue *q) 195 { 196 int freeze_depth; 197 198 freeze_depth = atomic_dec_return(&q->mq_freeze_depth); 199 WARN_ON_ONCE(freeze_depth < 0); 200 if (!freeze_depth) { 201 percpu_ref_reinit(&q->q_usage_counter); 202 wake_up_all(&q->mq_freeze_wq); 203 } 204 } 205 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); 206 207 /* 208 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the 209 * mpt3sas driver such that this function can be removed. 210 */ 211 void blk_mq_quiesce_queue_nowait(struct request_queue *q) 212 { 213 blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q); 214 } 215 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); 216 217 /** 218 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished 219 * @q: request queue. 220 * 221 * Note: this function does not prevent that the struct request end_io() 222 * callback function is invoked. Once this function is returned, we make 223 * sure no dispatch can happen until the queue is unquiesced via 224 * blk_mq_unquiesce_queue(). 225 */ 226 void blk_mq_quiesce_queue(struct request_queue *q) 227 { 228 struct blk_mq_hw_ctx *hctx; 229 unsigned int i; 230 bool rcu = false; 231 232 blk_mq_quiesce_queue_nowait(q); 233 234 queue_for_each_hw_ctx(q, hctx, i) { 235 if (hctx->flags & BLK_MQ_F_BLOCKING) 236 synchronize_srcu(hctx->srcu); 237 else 238 rcu = true; 239 } 240 if (rcu) 241 synchronize_rcu(); 242 } 243 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue); 244 245 /* 246 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue() 247 * @q: request queue. 248 * 249 * This function recovers queue into the state before quiescing 250 * which is done by blk_mq_quiesce_queue. 251 */ 252 void blk_mq_unquiesce_queue(struct request_queue *q) 253 { 254 blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q); 255 256 /* dispatch requests which are inserted during quiescing */ 257 blk_mq_run_hw_queues(q, true); 258 } 259 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue); 260 261 void blk_mq_wake_waiters(struct request_queue *q) 262 { 263 struct blk_mq_hw_ctx *hctx; 264 unsigned int i; 265 266 queue_for_each_hw_ctx(q, hctx, i) 267 if (blk_mq_hw_queue_mapped(hctx)) 268 blk_mq_tag_wakeup_all(hctx->tags, true); 269 } 270 271 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) 272 { 273 return blk_mq_has_free_tags(hctx->tags); 274 } 275 EXPORT_SYMBOL(blk_mq_can_queue); 276 277 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, 278 unsigned int tag, unsigned int op) 279 { 280 struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 281 struct request *rq = tags->static_rqs[tag]; 282 req_flags_t rq_flags = 0; 283 284 if (data->flags & BLK_MQ_REQ_INTERNAL) { 285 rq->tag = -1; 286 rq->internal_tag = tag; 287 } else { 288 if (blk_mq_tag_busy(data->hctx)) { 289 rq_flags = RQF_MQ_INFLIGHT; 290 atomic_inc(&data->hctx->nr_active); 291 } 292 rq->tag = tag; 293 rq->internal_tag = -1; 294 data->hctx->tags->rqs[rq->tag] = rq; 295 } 296 297 /* csd/requeue_work/fifo_time is initialized before use */ 298 rq->q = data->q; 299 rq->mq_ctx = data->ctx; 300 rq->rq_flags = rq_flags; 301 rq->cpu = -1; 302 rq->cmd_flags = op; 303 if (data->flags & BLK_MQ_REQ_PREEMPT) 304 rq->rq_flags |= RQF_PREEMPT; 305 if (blk_queue_io_stat(data->q)) 306 rq->rq_flags |= RQF_IO_STAT; 307 INIT_LIST_HEAD(&rq->queuelist); 308 INIT_HLIST_NODE(&rq->hash); 309 RB_CLEAR_NODE(&rq->rb_node); 310 rq->rq_disk = NULL; 311 rq->part = NULL; 312 rq->start_time_ns = ktime_get_ns(); 313 rq->io_start_time_ns = 0; 314 rq->nr_phys_segments = 0; 315 #if defined(CONFIG_BLK_DEV_INTEGRITY) 316 rq->nr_integrity_segments = 0; 317 #endif 318 rq->special = NULL; 319 /* tag was already set */ 320 rq->extra_len = 0; 321 rq->__deadline = 0; 322 323 INIT_LIST_HEAD(&rq->timeout_list); 324 rq->timeout = 0; 325 326 rq->end_io = NULL; 327 rq->end_io_data = NULL; 328 rq->next_rq = NULL; 329 330 #ifdef CONFIG_BLK_CGROUP 331 rq->rl = NULL; 332 #endif 333 334 data->ctx->rq_dispatched[op_is_sync(op)]++; 335 refcount_set(&rq->ref, 1); 336 return rq; 337 } 338 339 static struct request *blk_mq_get_request(struct request_queue *q, 340 struct bio *bio, unsigned int op, 341 struct blk_mq_alloc_data *data) 342 { 343 struct elevator_queue *e = q->elevator; 344 struct request *rq; 345 unsigned int tag; 346 bool put_ctx_on_error = false; 347 348 blk_queue_enter_live(q); 349 data->q = q; 350 if (likely(!data->ctx)) { 351 data->ctx = blk_mq_get_ctx(q); 352 put_ctx_on_error = true; 353 } 354 if (likely(!data->hctx)) 355 data->hctx = blk_mq_map_queue(q, data->ctx->cpu); 356 if (op & REQ_NOWAIT) 357 data->flags |= BLK_MQ_REQ_NOWAIT; 358 359 if (e) { 360 data->flags |= BLK_MQ_REQ_INTERNAL; 361 362 /* 363 * Flush requests are special and go directly to the 364 * dispatch list. Don't include reserved tags in the 365 * limiting, as it isn't useful. 366 */ 367 if (!op_is_flush(op) && e->type->ops.mq.limit_depth && 368 !(data->flags & BLK_MQ_REQ_RESERVED)) 369 e->type->ops.mq.limit_depth(op, data); 370 } 371 372 tag = blk_mq_get_tag(data); 373 if (tag == BLK_MQ_TAG_FAIL) { 374 if (put_ctx_on_error) { 375 blk_mq_put_ctx(data->ctx); 376 data->ctx = NULL; 377 } 378 blk_queue_exit(q); 379 return NULL; 380 } 381 382 rq = blk_mq_rq_ctx_init(data, tag, op); 383 if (!op_is_flush(op)) { 384 rq->elv.icq = NULL; 385 if (e && e->type->ops.mq.prepare_request) { 386 if (e->type->icq_cache && rq_ioc(bio)) 387 blk_mq_sched_assign_ioc(rq, bio); 388 389 e->type->ops.mq.prepare_request(rq, bio); 390 rq->rq_flags |= RQF_ELVPRIV; 391 } 392 } 393 data->hctx->queued++; 394 return rq; 395 } 396 397 struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, 398 blk_mq_req_flags_t flags) 399 { 400 struct blk_mq_alloc_data alloc_data = { .flags = flags }; 401 struct request *rq; 402 int ret; 403 404 ret = blk_queue_enter(q, flags); 405 if (ret) 406 return ERR_PTR(ret); 407 408 rq = blk_mq_get_request(q, NULL, op, &alloc_data); 409 blk_queue_exit(q); 410 411 if (!rq) 412 return ERR_PTR(-EWOULDBLOCK); 413 414 blk_mq_put_ctx(alloc_data.ctx); 415 416 rq->__data_len = 0; 417 rq->__sector = (sector_t) -1; 418 rq->bio = rq->biotail = NULL; 419 return rq; 420 } 421 EXPORT_SYMBOL(blk_mq_alloc_request); 422 423 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, 424 unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx) 425 { 426 struct blk_mq_alloc_data alloc_data = { .flags = flags }; 427 struct request *rq; 428 unsigned int cpu; 429 int ret; 430 431 /* 432 * If the tag allocator sleeps we could get an allocation for a 433 * different hardware context. No need to complicate the low level 434 * allocator for this for the rare use case of a command tied to 435 * a specific queue. 436 */ 437 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT))) 438 return ERR_PTR(-EINVAL); 439 440 if (hctx_idx >= q->nr_hw_queues) 441 return ERR_PTR(-EIO); 442 443 ret = blk_queue_enter(q, flags); 444 if (ret) 445 return ERR_PTR(ret); 446 447 /* 448 * Check if the hardware context is actually mapped to anything. 449 * If not tell the caller that it should skip this queue. 450 */ 451 alloc_data.hctx = q->queue_hw_ctx[hctx_idx]; 452 if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) { 453 blk_queue_exit(q); 454 return ERR_PTR(-EXDEV); 455 } 456 cpu = cpumask_first_and(alloc_data.hctx->cpumask, cpu_online_mask); 457 alloc_data.ctx = __blk_mq_get_ctx(q, cpu); 458 459 rq = blk_mq_get_request(q, NULL, op, &alloc_data); 460 blk_queue_exit(q); 461 462 if (!rq) 463 return ERR_PTR(-EWOULDBLOCK); 464 465 return rq; 466 } 467 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); 468 469 static void __blk_mq_free_request(struct request *rq) 470 { 471 struct request_queue *q = rq->q; 472 struct blk_mq_ctx *ctx = rq->mq_ctx; 473 struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu); 474 const int sched_tag = rq->internal_tag; 475 476 if (rq->tag != -1) 477 blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag); 478 if (sched_tag != -1) 479 blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag); 480 blk_mq_sched_restart(hctx); 481 blk_queue_exit(q); 482 } 483 484 void blk_mq_free_request(struct request *rq) 485 { 486 struct request_queue *q = rq->q; 487 struct elevator_queue *e = q->elevator; 488 struct blk_mq_ctx *ctx = rq->mq_ctx; 489 struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu); 490 491 if (rq->rq_flags & RQF_ELVPRIV) { 492 if (e && e->type->ops.mq.finish_request) 493 e->type->ops.mq.finish_request(rq); 494 if (rq->elv.icq) { 495 put_io_context(rq->elv.icq->ioc); 496 rq->elv.icq = NULL; 497 } 498 } 499 500 ctx->rq_completed[rq_is_sync(rq)]++; 501 if (rq->rq_flags & RQF_MQ_INFLIGHT) 502 atomic_dec(&hctx->nr_active); 503 504 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq))) 505 laptop_io_completion(q->backing_dev_info); 506 507 wbt_done(q->rq_wb, rq); 508 509 if (blk_rq_rl(rq)) 510 blk_put_rl(blk_rq_rl(rq)); 511 512 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 513 if (refcount_dec_and_test(&rq->ref)) 514 __blk_mq_free_request(rq); 515 } 516 EXPORT_SYMBOL_GPL(blk_mq_free_request); 517 518 inline void __blk_mq_end_request(struct request *rq, blk_status_t error) 519 { 520 u64 now = ktime_get_ns(); 521 522 if (rq->rq_flags & RQF_STATS) { 523 blk_mq_poll_stats_start(rq->q); 524 blk_stat_add(rq, now); 525 } 526 527 blk_account_io_done(rq, now); 528 529 if (rq->end_io) { 530 wbt_done(rq->q->rq_wb, rq); 531 rq->end_io(rq, error); 532 } else { 533 if (unlikely(blk_bidi_rq(rq))) 534 blk_mq_free_request(rq->next_rq); 535 blk_mq_free_request(rq); 536 } 537 } 538 EXPORT_SYMBOL(__blk_mq_end_request); 539 540 void blk_mq_end_request(struct request *rq, blk_status_t error) 541 { 542 if (blk_update_request(rq, error, blk_rq_bytes(rq))) 543 BUG(); 544 __blk_mq_end_request(rq, error); 545 } 546 EXPORT_SYMBOL(blk_mq_end_request); 547 548 static void __blk_mq_complete_request_remote(void *data) 549 { 550 struct request *rq = data; 551 552 rq->q->softirq_done_fn(rq); 553 } 554 555 static void __blk_mq_complete_request(struct request *rq) 556 { 557 struct blk_mq_ctx *ctx = rq->mq_ctx; 558 bool shared = false; 559 int cpu; 560 561 if (cmpxchg(&rq->state, MQ_RQ_IN_FLIGHT, MQ_RQ_COMPLETE) != 562 MQ_RQ_IN_FLIGHT) 563 return; 564 565 if (rq->internal_tag != -1) 566 blk_mq_sched_completed_request(rq); 567 568 if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) { 569 rq->q->softirq_done_fn(rq); 570 return; 571 } 572 573 cpu = get_cpu(); 574 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags)) 575 shared = cpus_share_cache(cpu, ctx->cpu); 576 577 if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { 578 rq->csd.func = __blk_mq_complete_request_remote; 579 rq->csd.info = rq; 580 rq->csd.flags = 0; 581 smp_call_function_single_async(ctx->cpu, &rq->csd); 582 } else { 583 rq->q->softirq_done_fn(rq); 584 } 585 put_cpu(); 586 } 587 588 static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx) 589 __releases(hctx->srcu) 590 { 591 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) 592 rcu_read_unlock(); 593 else 594 srcu_read_unlock(hctx->srcu, srcu_idx); 595 } 596 597 static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx) 598 __acquires(hctx->srcu) 599 { 600 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { 601 /* shut up gcc false positive */ 602 *srcu_idx = 0; 603 rcu_read_lock(); 604 } else 605 *srcu_idx = srcu_read_lock(hctx->srcu); 606 } 607 608 /** 609 * blk_mq_complete_request - end I/O on a request 610 * @rq: the request being processed 611 * 612 * Description: 613 * Ends all I/O on a request. It does not handle partial completions. 614 * The actual completion happens out-of-order, through a IPI handler. 615 **/ 616 void blk_mq_complete_request(struct request *rq) 617 { 618 if (unlikely(blk_should_fake_timeout(rq->q))) 619 return; 620 __blk_mq_complete_request(rq); 621 } 622 EXPORT_SYMBOL(blk_mq_complete_request); 623 624 int blk_mq_request_started(struct request *rq) 625 { 626 return blk_mq_rq_state(rq) != MQ_RQ_IDLE; 627 } 628 EXPORT_SYMBOL_GPL(blk_mq_request_started); 629 630 void blk_mq_start_request(struct request *rq) 631 { 632 struct request_queue *q = rq->q; 633 634 blk_mq_sched_started_request(rq); 635 636 trace_block_rq_issue(q, rq); 637 638 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 639 rq->io_start_time_ns = ktime_get_ns(); 640 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW 641 rq->throtl_size = blk_rq_sectors(rq); 642 #endif 643 rq->rq_flags |= RQF_STATS; 644 wbt_issue(q->rq_wb, rq); 645 } 646 647 WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE); 648 649 blk_add_timer(rq); 650 WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT); 651 652 if (q->dma_drain_size && blk_rq_bytes(rq)) { 653 /* 654 * Make sure space for the drain appears. We know we can do 655 * this because max_hw_segments has been adjusted to be one 656 * fewer than the device can handle. 657 */ 658 rq->nr_phys_segments++; 659 } 660 } 661 EXPORT_SYMBOL(blk_mq_start_request); 662 663 static void __blk_mq_requeue_request(struct request *rq) 664 { 665 struct request_queue *q = rq->q; 666 667 blk_mq_put_driver_tag(rq); 668 669 trace_block_rq_requeue(q, rq); 670 wbt_requeue(q->rq_wb, rq); 671 672 if (blk_mq_request_started(rq)) { 673 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 674 rq->rq_flags &= ~RQF_TIMED_OUT; 675 if (q->dma_drain_size && blk_rq_bytes(rq)) 676 rq->nr_phys_segments--; 677 } 678 } 679 680 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) 681 { 682 __blk_mq_requeue_request(rq); 683 684 /* this request will be re-inserted to io scheduler queue */ 685 blk_mq_sched_requeue_request(rq); 686 687 BUG_ON(blk_queued_rq(rq)); 688 blk_mq_add_to_requeue_list(rq, true, kick_requeue_list); 689 } 690 EXPORT_SYMBOL(blk_mq_requeue_request); 691 692 static void blk_mq_requeue_work(struct work_struct *work) 693 { 694 struct request_queue *q = 695 container_of(work, struct request_queue, requeue_work.work); 696 LIST_HEAD(rq_list); 697 struct request *rq, *next; 698 699 spin_lock_irq(&q->requeue_lock); 700 list_splice_init(&q->requeue_list, &rq_list); 701 spin_unlock_irq(&q->requeue_lock); 702 703 list_for_each_entry_safe(rq, next, &rq_list, queuelist) { 704 if (!(rq->rq_flags & RQF_SOFTBARRIER)) 705 continue; 706 707 rq->rq_flags &= ~RQF_SOFTBARRIER; 708 list_del_init(&rq->queuelist); 709 blk_mq_sched_insert_request(rq, true, false, false); 710 } 711 712 while (!list_empty(&rq_list)) { 713 rq = list_entry(rq_list.next, struct request, queuelist); 714 list_del_init(&rq->queuelist); 715 blk_mq_sched_insert_request(rq, false, false, false); 716 } 717 718 blk_mq_run_hw_queues(q, false); 719 } 720 721 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, 722 bool kick_requeue_list) 723 { 724 struct request_queue *q = rq->q; 725 unsigned long flags; 726 727 /* 728 * We abuse this flag that is otherwise used by the I/O scheduler to 729 * request head insertion from the workqueue. 730 */ 731 BUG_ON(rq->rq_flags & RQF_SOFTBARRIER); 732 733 spin_lock_irqsave(&q->requeue_lock, flags); 734 if (at_head) { 735 rq->rq_flags |= RQF_SOFTBARRIER; 736 list_add(&rq->queuelist, &q->requeue_list); 737 } else { 738 list_add_tail(&rq->queuelist, &q->requeue_list); 739 } 740 spin_unlock_irqrestore(&q->requeue_lock, flags); 741 742 if (kick_requeue_list) 743 blk_mq_kick_requeue_list(q); 744 } 745 EXPORT_SYMBOL(blk_mq_add_to_requeue_list); 746 747 void blk_mq_kick_requeue_list(struct request_queue *q) 748 { 749 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0); 750 } 751 EXPORT_SYMBOL(blk_mq_kick_requeue_list); 752 753 void blk_mq_delay_kick_requeue_list(struct request_queue *q, 754 unsigned long msecs) 755 { 756 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 757 msecs_to_jiffies(msecs)); 758 } 759 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); 760 761 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) 762 { 763 if (tag < tags->nr_tags) { 764 prefetch(tags->rqs[tag]); 765 return tags->rqs[tag]; 766 } 767 768 return NULL; 769 } 770 EXPORT_SYMBOL(blk_mq_tag_to_rq); 771 772 static void blk_mq_rq_timed_out(struct request *req, bool reserved) 773 { 774 req->rq_flags |= RQF_TIMED_OUT; 775 if (req->q->mq_ops->timeout) { 776 enum blk_eh_timer_return ret; 777 778 ret = req->q->mq_ops->timeout(req, reserved); 779 if (ret == BLK_EH_DONE) 780 return; 781 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER); 782 } 783 784 blk_add_timer(req); 785 } 786 787 static bool blk_mq_req_expired(struct request *rq, unsigned long *next) 788 { 789 unsigned long deadline; 790 791 if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT) 792 return false; 793 if (rq->rq_flags & RQF_TIMED_OUT) 794 return false; 795 796 deadline = blk_rq_deadline(rq); 797 if (time_after_eq(jiffies, deadline)) 798 return true; 799 800 if (*next == 0) 801 *next = deadline; 802 else if (time_after(*next, deadline)) 803 *next = deadline; 804 return false; 805 } 806 807 static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, 808 struct request *rq, void *priv, bool reserved) 809 { 810 unsigned long *next = priv; 811 812 /* 813 * Just do a quick check if it is expired before locking the request in 814 * so we're not unnecessarilly synchronizing across CPUs. 815 */ 816 if (!blk_mq_req_expired(rq, next)) 817 return; 818 819 /* 820 * We have reason to believe the request may be expired. Take a 821 * reference on the request to lock this request lifetime into its 822 * currently allocated context to prevent it from being reallocated in 823 * the event the completion by-passes this timeout handler. 824 * 825 * If the reference was already released, then the driver beat the 826 * timeout handler to posting a natural completion. 827 */ 828 if (!refcount_inc_not_zero(&rq->ref)) 829 return; 830 831 /* 832 * The request is now locked and cannot be reallocated underneath the 833 * timeout handler's processing. Re-verify this exact request is truly 834 * expired; if it is not expired, then the request was completed and 835 * reallocated as a new request. 836 */ 837 if (blk_mq_req_expired(rq, next)) 838 blk_mq_rq_timed_out(rq, reserved); 839 if (refcount_dec_and_test(&rq->ref)) 840 __blk_mq_free_request(rq); 841 } 842 843 static void blk_mq_timeout_work(struct work_struct *work) 844 { 845 struct request_queue *q = 846 container_of(work, struct request_queue, timeout_work); 847 unsigned long next = 0; 848 struct blk_mq_hw_ctx *hctx; 849 int i; 850 851 /* A deadlock might occur if a request is stuck requiring a 852 * timeout at the same time a queue freeze is waiting 853 * completion, since the timeout code would not be able to 854 * acquire the queue reference here. 855 * 856 * That's why we don't use blk_queue_enter here; instead, we use 857 * percpu_ref_tryget directly, because we need to be able to 858 * obtain a reference even in the short window between the queue 859 * starting to freeze, by dropping the first reference in 860 * blk_freeze_queue_start, and the moment the last request is 861 * consumed, marked by the instant q_usage_counter reaches 862 * zero. 863 */ 864 if (!percpu_ref_tryget(&q->q_usage_counter)) 865 return; 866 867 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next); 868 869 if (next != 0) { 870 mod_timer(&q->timeout, next); 871 } else { 872 /* 873 * Request timeouts are handled as a forward rolling timer. If 874 * we end up here it means that no requests are pending and 875 * also that no request has been pending for a while. Mark 876 * each hctx as idle. 877 */ 878 queue_for_each_hw_ctx(q, hctx, i) { 879 /* the hctx may be unmapped, so check it here */ 880 if (blk_mq_hw_queue_mapped(hctx)) 881 blk_mq_tag_idle(hctx); 882 } 883 } 884 blk_queue_exit(q); 885 } 886 887 struct flush_busy_ctx_data { 888 struct blk_mq_hw_ctx *hctx; 889 struct list_head *list; 890 }; 891 892 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) 893 { 894 struct flush_busy_ctx_data *flush_data = data; 895 struct blk_mq_hw_ctx *hctx = flush_data->hctx; 896 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 897 898 spin_lock(&ctx->lock); 899 list_splice_tail_init(&ctx->rq_list, flush_data->list); 900 sbitmap_clear_bit(sb, bitnr); 901 spin_unlock(&ctx->lock); 902 return true; 903 } 904 905 /* 906 * Process software queues that have been marked busy, splicing them 907 * to the for-dispatch 908 */ 909 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) 910 { 911 struct flush_busy_ctx_data data = { 912 .hctx = hctx, 913 .list = list, 914 }; 915 916 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data); 917 } 918 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs); 919 920 struct dispatch_rq_data { 921 struct blk_mq_hw_ctx *hctx; 922 struct request *rq; 923 }; 924 925 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr, 926 void *data) 927 { 928 struct dispatch_rq_data *dispatch_data = data; 929 struct blk_mq_hw_ctx *hctx = dispatch_data->hctx; 930 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 931 932 spin_lock(&ctx->lock); 933 if (!list_empty(&ctx->rq_list)) { 934 dispatch_data->rq = list_entry_rq(ctx->rq_list.next); 935 list_del_init(&dispatch_data->rq->queuelist); 936 if (list_empty(&ctx->rq_list)) 937 sbitmap_clear_bit(sb, bitnr); 938 } 939 spin_unlock(&ctx->lock); 940 941 return !dispatch_data->rq; 942 } 943 944 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, 945 struct blk_mq_ctx *start) 946 { 947 unsigned off = start ? start->index_hw : 0; 948 struct dispatch_rq_data data = { 949 .hctx = hctx, 950 .rq = NULL, 951 }; 952 953 __sbitmap_for_each_set(&hctx->ctx_map, off, 954 dispatch_rq_from_ctx, &data); 955 956 return data.rq; 957 } 958 959 static inline unsigned int queued_to_index(unsigned int queued) 960 { 961 if (!queued) 962 return 0; 963 964 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); 965 } 966 967 bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx, 968 bool wait) 969 { 970 struct blk_mq_alloc_data data = { 971 .q = rq->q, 972 .hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), 973 .flags = wait ? 0 : BLK_MQ_REQ_NOWAIT, 974 }; 975 976 might_sleep_if(wait); 977 978 if (rq->tag != -1) 979 goto done; 980 981 if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag)) 982 data.flags |= BLK_MQ_REQ_RESERVED; 983 984 rq->tag = blk_mq_get_tag(&data); 985 if (rq->tag >= 0) { 986 if (blk_mq_tag_busy(data.hctx)) { 987 rq->rq_flags |= RQF_MQ_INFLIGHT; 988 atomic_inc(&data.hctx->nr_active); 989 } 990 data.hctx->tags->rqs[rq->tag] = rq; 991 } 992 993 done: 994 if (hctx) 995 *hctx = data.hctx; 996 return rq->tag != -1; 997 } 998 999 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, 1000 int flags, void *key) 1001 { 1002 struct blk_mq_hw_ctx *hctx; 1003 1004 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); 1005 1006 list_del_init(&wait->entry); 1007 blk_mq_run_hw_queue(hctx, true); 1008 return 1; 1009 } 1010 1011 /* 1012 * Mark us waiting for a tag. For shared tags, this involves hooking us into 1013 * the tag wakeups. For non-shared tags, we can simply mark us needing a 1014 * restart. For both cases, take care to check the condition again after 1015 * marking us as waiting. 1016 */ 1017 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx **hctx, 1018 struct request *rq) 1019 { 1020 struct blk_mq_hw_ctx *this_hctx = *hctx; 1021 struct sbq_wait_state *ws; 1022 wait_queue_entry_t *wait; 1023 bool ret; 1024 1025 if (!(this_hctx->flags & BLK_MQ_F_TAG_SHARED)) { 1026 if (!test_bit(BLK_MQ_S_SCHED_RESTART, &this_hctx->state)) 1027 set_bit(BLK_MQ_S_SCHED_RESTART, &this_hctx->state); 1028 1029 /* 1030 * It's possible that a tag was freed in the window between the 1031 * allocation failure and adding the hardware queue to the wait 1032 * queue. 1033 * 1034 * Don't clear RESTART here, someone else could have set it. 1035 * At most this will cost an extra queue run. 1036 */ 1037 return blk_mq_get_driver_tag(rq, hctx, false); 1038 } 1039 1040 wait = &this_hctx->dispatch_wait; 1041 if (!list_empty_careful(&wait->entry)) 1042 return false; 1043 1044 spin_lock(&this_hctx->lock); 1045 if (!list_empty(&wait->entry)) { 1046 spin_unlock(&this_hctx->lock); 1047 return false; 1048 } 1049 1050 ws = bt_wait_ptr(&this_hctx->tags->bitmap_tags, this_hctx); 1051 add_wait_queue(&ws->wait, wait); 1052 1053 /* 1054 * It's possible that a tag was freed in the window between the 1055 * allocation failure and adding the hardware queue to the wait 1056 * queue. 1057 */ 1058 ret = blk_mq_get_driver_tag(rq, hctx, false); 1059 if (!ret) { 1060 spin_unlock(&this_hctx->lock); 1061 return false; 1062 } 1063 1064 /* 1065 * We got a tag, remove ourselves from the wait queue to ensure 1066 * someone else gets the wakeup. 1067 */ 1068 spin_lock_irq(&ws->wait.lock); 1069 list_del_init(&wait->entry); 1070 spin_unlock_irq(&ws->wait.lock); 1071 spin_unlock(&this_hctx->lock); 1072 1073 return true; 1074 } 1075 1076 #define BLK_MQ_RESOURCE_DELAY 3 /* ms units */ 1077 1078 bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, 1079 bool got_budget) 1080 { 1081 struct blk_mq_hw_ctx *hctx; 1082 struct request *rq, *nxt; 1083 bool no_tag = false; 1084 int errors, queued; 1085 blk_status_t ret = BLK_STS_OK; 1086 1087 if (list_empty(list)) 1088 return false; 1089 1090 WARN_ON(!list_is_singular(list) && got_budget); 1091 1092 /* 1093 * Now process all the entries, sending them to the driver. 1094 */ 1095 errors = queued = 0; 1096 do { 1097 struct blk_mq_queue_data bd; 1098 1099 rq = list_first_entry(list, struct request, queuelist); 1100 1101 hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); 1102 if (!got_budget && !blk_mq_get_dispatch_budget(hctx)) 1103 break; 1104 1105 if (!blk_mq_get_driver_tag(rq, NULL, false)) { 1106 /* 1107 * The initial allocation attempt failed, so we need to 1108 * rerun the hardware queue when a tag is freed. The 1109 * waitqueue takes care of that. If the queue is run 1110 * before we add this entry back on the dispatch list, 1111 * we'll re-run it below. 1112 */ 1113 if (!blk_mq_mark_tag_wait(&hctx, rq)) { 1114 blk_mq_put_dispatch_budget(hctx); 1115 /* 1116 * For non-shared tags, the RESTART check 1117 * will suffice. 1118 */ 1119 if (hctx->flags & BLK_MQ_F_TAG_SHARED) 1120 no_tag = true; 1121 break; 1122 } 1123 } 1124 1125 list_del_init(&rq->queuelist); 1126 1127 bd.rq = rq; 1128 1129 /* 1130 * Flag last if we have no more requests, or if we have more 1131 * but can't assign a driver tag to it. 1132 */ 1133 if (list_empty(list)) 1134 bd.last = true; 1135 else { 1136 nxt = list_first_entry(list, struct request, queuelist); 1137 bd.last = !blk_mq_get_driver_tag(nxt, NULL, false); 1138 } 1139 1140 ret = q->mq_ops->queue_rq(hctx, &bd); 1141 if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) { 1142 /* 1143 * If an I/O scheduler has been configured and we got a 1144 * driver tag for the next request already, free it 1145 * again. 1146 */ 1147 if (!list_empty(list)) { 1148 nxt = list_first_entry(list, struct request, queuelist); 1149 blk_mq_put_driver_tag(nxt); 1150 } 1151 list_add(&rq->queuelist, list); 1152 __blk_mq_requeue_request(rq); 1153 break; 1154 } 1155 1156 if (unlikely(ret != BLK_STS_OK)) { 1157 errors++; 1158 blk_mq_end_request(rq, BLK_STS_IOERR); 1159 continue; 1160 } 1161 1162 queued++; 1163 } while (!list_empty(list)); 1164 1165 hctx->dispatched[queued_to_index(queued)]++; 1166 1167 /* 1168 * Any items that need requeuing? Stuff them into hctx->dispatch, 1169 * that is where we will continue on next queue run. 1170 */ 1171 if (!list_empty(list)) { 1172 bool needs_restart; 1173 1174 spin_lock(&hctx->lock); 1175 list_splice_init(list, &hctx->dispatch); 1176 spin_unlock(&hctx->lock); 1177 1178 /* 1179 * If SCHED_RESTART was set by the caller of this function and 1180 * it is no longer set that means that it was cleared by another 1181 * thread and hence that a queue rerun is needed. 1182 * 1183 * If 'no_tag' is set, that means that we failed getting 1184 * a driver tag with an I/O scheduler attached. If our dispatch 1185 * waitqueue is no longer active, ensure that we run the queue 1186 * AFTER adding our entries back to the list. 1187 * 1188 * If no I/O scheduler has been configured it is possible that 1189 * the hardware queue got stopped and restarted before requests 1190 * were pushed back onto the dispatch list. Rerun the queue to 1191 * avoid starvation. Notes: 1192 * - blk_mq_run_hw_queue() checks whether or not a queue has 1193 * been stopped before rerunning a queue. 1194 * - Some but not all block drivers stop a queue before 1195 * returning BLK_STS_RESOURCE. Two exceptions are scsi-mq 1196 * and dm-rq. 1197 * 1198 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART 1199 * bit is set, run queue after a delay to avoid IO stalls 1200 * that could otherwise occur if the queue is idle. 1201 */ 1202 needs_restart = blk_mq_sched_needs_restart(hctx); 1203 if (!needs_restart || 1204 (no_tag && list_empty_careful(&hctx->dispatch_wait.entry))) 1205 blk_mq_run_hw_queue(hctx, true); 1206 else if (needs_restart && (ret == BLK_STS_RESOURCE)) 1207 blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY); 1208 } 1209 1210 return (queued + errors) != 0; 1211 } 1212 1213 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) 1214 { 1215 int srcu_idx; 1216 1217 /* 1218 * We should be running this queue from one of the CPUs that 1219 * are mapped to it. 1220 * 1221 * There are at least two related races now between setting 1222 * hctx->next_cpu from blk_mq_hctx_next_cpu() and running 1223 * __blk_mq_run_hw_queue(): 1224 * 1225 * - hctx->next_cpu is found offline in blk_mq_hctx_next_cpu(), 1226 * but later it becomes online, then this warning is harmless 1227 * at all 1228 * 1229 * - hctx->next_cpu is found online in blk_mq_hctx_next_cpu(), 1230 * but later it becomes offline, then the warning can't be 1231 * triggered, and we depend on blk-mq timeout handler to 1232 * handle dispatched requests to this hctx 1233 */ 1234 if (!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) && 1235 cpu_online(hctx->next_cpu)) { 1236 printk(KERN_WARNING "run queue from wrong CPU %d, hctx %s\n", 1237 raw_smp_processor_id(), 1238 cpumask_empty(hctx->cpumask) ? "inactive": "active"); 1239 dump_stack(); 1240 } 1241 1242 /* 1243 * We can't run the queue inline with ints disabled. Ensure that 1244 * we catch bad users of this early. 1245 */ 1246 WARN_ON_ONCE(in_interrupt()); 1247 1248 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); 1249 1250 hctx_lock(hctx, &srcu_idx); 1251 blk_mq_sched_dispatch_requests(hctx); 1252 hctx_unlock(hctx, srcu_idx); 1253 } 1254 1255 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx) 1256 { 1257 int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask); 1258 1259 if (cpu >= nr_cpu_ids) 1260 cpu = cpumask_first(hctx->cpumask); 1261 return cpu; 1262 } 1263 1264 /* 1265 * It'd be great if the workqueue API had a way to pass 1266 * in a mask and had some smarts for more clever placement. 1267 * For now we just round-robin here, switching for every 1268 * BLK_MQ_CPU_WORK_BATCH queued items. 1269 */ 1270 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) 1271 { 1272 bool tried = false; 1273 int next_cpu = hctx->next_cpu; 1274 1275 if (hctx->queue->nr_hw_queues == 1) 1276 return WORK_CPU_UNBOUND; 1277 1278 if (--hctx->next_cpu_batch <= 0) { 1279 select_cpu: 1280 next_cpu = cpumask_next_and(next_cpu, hctx->cpumask, 1281 cpu_online_mask); 1282 if (next_cpu >= nr_cpu_ids) 1283 next_cpu = blk_mq_first_mapped_cpu(hctx); 1284 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 1285 } 1286 1287 /* 1288 * Do unbound schedule if we can't find a online CPU for this hctx, 1289 * and it should only happen in the path of handling CPU DEAD. 1290 */ 1291 if (!cpu_online(next_cpu)) { 1292 if (!tried) { 1293 tried = true; 1294 goto select_cpu; 1295 } 1296 1297 /* 1298 * Make sure to re-select CPU next time once after CPUs 1299 * in hctx->cpumask become online again. 1300 */ 1301 hctx->next_cpu = next_cpu; 1302 hctx->next_cpu_batch = 1; 1303 return WORK_CPU_UNBOUND; 1304 } 1305 1306 hctx->next_cpu = next_cpu; 1307 return next_cpu; 1308 } 1309 1310 static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, 1311 unsigned long msecs) 1312 { 1313 if (unlikely(blk_mq_hctx_stopped(hctx))) 1314 return; 1315 1316 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) { 1317 int cpu = get_cpu(); 1318 if (cpumask_test_cpu(cpu, hctx->cpumask)) { 1319 __blk_mq_run_hw_queue(hctx); 1320 put_cpu(); 1321 return; 1322 } 1323 1324 put_cpu(); 1325 } 1326 1327 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, 1328 msecs_to_jiffies(msecs)); 1329 } 1330 1331 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) 1332 { 1333 __blk_mq_delay_run_hw_queue(hctx, true, msecs); 1334 } 1335 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); 1336 1337 bool blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1338 { 1339 int srcu_idx; 1340 bool need_run; 1341 1342 /* 1343 * When queue is quiesced, we may be switching io scheduler, or 1344 * updating nr_hw_queues, or other things, and we can't run queue 1345 * any more, even __blk_mq_hctx_has_pending() can't be called safely. 1346 * 1347 * And queue will be rerun in blk_mq_unquiesce_queue() if it is 1348 * quiesced. 1349 */ 1350 hctx_lock(hctx, &srcu_idx); 1351 need_run = !blk_queue_quiesced(hctx->queue) && 1352 blk_mq_hctx_has_pending(hctx); 1353 hctx_unlock(hctx, srcu_idx); 1354 1355 if (need_run) { 1356 __blk_mq_delay_run_hw_queue(hctx, async, 0); 1357 return true; 1358 } 1359 1360 return false; 1361 } 1362 EXPORT_SYMBOL(blk_mq_run_hw_queue); 1363 1364 void blk_mq_run_hw_queues(struct request_queue *q, bool async) 1365 { 1366 struct blk_mq_hw_ctx *hctx; 1367 int i; 1368 1369 queue_for_each_hw_ctx(q, hctx, i) { 1370 if (blk_mq_hctx_stopped(hctx)) 1371 continue; 1372 1373 blk_mq_run_hw_queue(hctx, async); 1374 } 1375 } 1376 EXPORT_SYMBOL(blk_mq_run_hw_queues); 1377 1378 /** 1379 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped 1380 * @q: request queue. 1381 * 1382 * The caller is responsible for serializing this function against 1383 * blk_mq_{start,stop}_hw_queue(). 1384 */ 1385 bool blk_mq_queue_stopped(struct request_queue *q) 1386 { 1387 struct blk_mq_hw_ctx *hctx; 1388 int i; 1389 1390 queue_for_each_hw_ctx(q, hctx, i) 1391 if (blk_mq_hctx_stopped(hctx)) 1392 return true; 1393 1394 return false; 1395 } 1396 EXPORT_SYMBOL(blk_mq_queue_stopped); 1397 1398 /* 1399 * This function is often used for pausing .queue_rq() by driver when 1400 * there isn't enough resource or some conditions aren't satisfied, and 1401 * BLK_STS_RESOURCE is usually returned. 1402 * 1403 * We do not guarantee that dispatch can be drained or blocked 1404 * after blk_mq_stop_hw_queue() returns. Please use 1405 * blk_mq_quiesce_queue() for that requirement. 1406 */ 1407 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) 1408 { 1409 cancel_delayed_work(&hctx->run_work); 1410 1411 set_bit(BLK_MQ_S_STOPPED, &hctx->state); 1412 } 1413 EXPORT_SYMBOL(blk_mq_stop_hw_queue); 1414 1415 /* 1416 * This function is often used for pausing .queue_rq() by driver when 1417 * there isn't enough resource or some conditions aren't satisfied, and 1418 * BLK_STS_RESOURCE is usually returned. 1419 * 1420 * We do not guarantee that dispatch can be drained or blocked 1421 * after blk_mq_stop_hw_queues() returns. Please use 1422 * blk_mq_quiesce_queue() for that requirement. 1423 */ 1424 void blk_mq_stop_hw_queues(struct request_queue *q) 1425 { 1426 struct blk_mq_hw_ctx *hctx; 1427 int i; 1428 1429 queue_for_each_hw_ctx(q, hctx, i) 1430 blk_mq_stop_hw_queue(hctx); 1431 } 1432 EXPORT_SYMBOL(blk_mq_stop_hw_queues); 1433 1434 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) 1435 { 1436 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1437 1438 blk_mq_run_hw_queue(hctx, false); 1439 } 1440 EXPORT_SYMBOL(blk_mq_start_hw_queue); 1441 1442 void blk_mq_start_hw_queues(struct request_queue *q) 1443 { 1444 struct blk_mq_hw_ctx *hctx; 1445 int i; 1446 1447 queue_for_each_hw_ctx(q, hctx, i) 1448 blk_mq_start_hw_queue(hctx); 1449 } 1450 EXPORT_SYMBOL(blk_mq_start_hw_queues); 1451 1452 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1453 { 1454 if (!blk_mq_hctx_stopped(hctx)) 1455 return; 1456 1457 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1458 blk_mq_run_hw_queue(hctx, async); 1459 } 1460 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue); 1461 1462 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) 1463 { 1464 struct blk_mq_hw_ctx *hctx; 1465 int i; 1466 1467 queue_for_each_hw_ctx(q, hctx, i) 1468 blk_mq_start_stopped_hw_queue(hctx, async); 1469 } 1470 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); 1471 1472 static void blk_mq_run_work_fn(struct work_struct *work) 1473 { 1474 struct blk_mq_hw_ctx *hctx; 1475 1476 hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work); 1477 1478 /* 1479 * If we are stopped, don't run the queue. 1480 */ 1481 if (test_bit(BLK_MQ_S_STOPPED, &hctx->state)) 1482 return; 1483 1484 __blk_mq_run_hw_queue(hctx); 1485 } 1486 1487 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx, 1488 struct request *rq, 1489 bool at_head) 1490 { 1491 struct blk_mq_ctx *ctx = rq->mq_ctx; 1492 1493 lockdep_assert_held(&ctx->lock); 1494 1495 trace_block_rq_insert(hctx->queue, rq); 1496 1497 if (at_head) 1498 list_add(&rq->queuelist, &ctx->rq_list); 1499 else 1500 list_add_tail(&rq->queuelist, &ctx->rq_list); 1501 } 1502 1503 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, 1504 bool at_head) 1505 { 1506 struct blk_mq_ctx *ctx = rq->mq_ctx; 1507 1508 lockdep_assert_held(&ctx->lock); 1509 1510 __blk_mq_insert_req_list(hctx, rq, at_head); 1511 blk_mq_hctx_mark_pending(hctx, ctx); 1512 } 1513 1514 /* 1515 * Should only be used carefully, when the caller knows we want to 1516 * bypass a potential IO scheduler on the target device. 1517 */ 1518 void blk_mq_request_bypass_insert(struct request *rq, bool run_queue) 1519 { 1520 struct blk_mq_ctx *ctx = rq->mq_ctx; 1521 struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(rq->q, ctx->cpu); 1522 1523 spin_lock(&hctx->lock); 1524 list_add_tail(&rq->queuelist, &hctx->dispatch); 1525 spin_unlock(&hctx->lock); 1526 1527 if (run_queue) 1528 blk_mq_run_hw_queue(hctx, false); 1529 } 1530 1531 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 1532 struct list_head *list) 1533 1534 { 1535 /* 1536 * preemption doesn't flush plug list, so it's possible ctx->cpu is 1537 * offline now 1538 */ 1539 spin_lock(&ctx->lock); 1540 while (!list_empty(list)) { 1541 struct request *rq; 1542 1543 rq = list_first_entry(list, struct request, queuelist); 1544 BUG_ON(rq->mq_ctx != ctx); 1545 list_del_init(&rq->queuelist); 1546 __blk_mq_insert_req_list(hctx, rq, false); 1547 } 1548 blk_mq_hctx_mark_pending(hctx, ctx); 1549 spin_unlock(&ctx->lock); 1550 } 1551 1552 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b) 1553 { 1554 struct request *rqa = container_of(a, struct request, queuelist); 1555 struct request *rqb = container_of(b, struct request, queuelist); 1556 1557 return !(rqa->mq_ctx < rqb->mq_ctx || 1558 (rqa->mq_ctx == rqb->mq_ctx && 1559 blk_rq_pos(rqa) < blk_rq_pos(rqb))); 1560 } 1561 1562 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 1563 { 1564 struct blk_mq_ctx *this_ctx; 1565 struct request_queue *this_q; 1566 struct request *rq; 1567 LIST_HEAD(list); 1568 LIST_HEAD(ctx_list); 1569 unsigned int depth; 1570 1571 list_splice_init(&plug->mq_list, &list); 1572 1573 list_sort(NULL, &list, plug_ctx_cmp); 1574 1575 this_q = NULL; 1576 this_ctx = NULL; 1577 depth = 0; 1578 1579 while (!list_empty(&list)) { 1580 rq = list_entry_rq(list.next); 1581 list_del_init(&rq->queuelist); 1582 BUG_ON(!rq->q); 1583 if (rq->mq_ctx != this_ctx) { 1584 if (this_ctx) { 1585 trace_block_unplug(this_q, depth, from_schedule); 1586 blk_mq_sched_insert_requests(this_q, this_ctx, 1587 &ctx_list, 1588 from_schedule); 1589 } 1590 1591 this_ctx = rq->mq_ctx; 1592 this_q = rq->q; 1593 depth = 0; 1594 } 1595 1596 depth++; 1597 list_add_tail(&rq->queuelist, &ctx_list); 1598 } 1599 1600 /* 1601 * If 'this_ctx' is set, we know we have entries to complete 1602 * on 'ctx_list'. Do those. 1603 */ 1604 if (this_ctx) { 1605 trace_block_unplug(this_q, depth, from_schedule); 1606 blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list, 1607 from_schedule); 1608 } 1609 } 1610 1611 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio) 1612 { 1613 blk_init_request_from_bio(rq, bio); 1614 1615 blk_rq_set_rl(rq, blk_get_rl(rq->q, bio)); 1616 1617 blk_account_io_start(rq, true); 1618 } 1619 1620 static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq) 1621 { 1622 if (rq->tag != -1) 1623 return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false); 1624 1625 return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true); 1626 } 1627 1628 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx, 1629 struct request *rq, 1630 blk_qc_t *cookie) 1631 { 1632 struct request_queue *q = rq->q; 1633 struct blk_mq_queue_data bd = { 1634 .rq = rq, 1635 .last = true, 1636 }; 1637 blk_qc_t new_cookie; 1638 blk_status_t ret; 1639 1640 new_cookie = request_to_qc_t(hctx, rq); 1641 1642 /* 1643 * For OK queue, we are done. For error, caller may kill it. 1644 * Any other error (busy), just add it to our list as we 1645 * previously would have done. 1646 */ 1647 ret = q->mq_ops->queue_rq(hctx, &bd); 1648 switch (ret) { 1649 case BLK_STS_OK: 1650 *cookie = new_cookie; 1651 break; 1652 case BLK_STS_RESOURCE: 1653 case BLK_STS_DEV_RESOURCE: 1654 __blk_mq_requeue_request(rq); 1655 break; 1656 default: 1657 *cookie = BLK_QC_T_NONE; 1658 break; 1659 } 1660 1661 return ret; 1662 } 1663 1664 static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 1665 struct request *rq, 1666 blk_qc_t *cookie, 1667 bool bypass_insert) 1668 { 1669 struct request_queue *q = rq->q; 1670 bool run_queue = true; 1671 1672 /* 1673 * RCU or SRCU read lock is needed before checking quiesced flag. 1674 * 1675 * When queue is stopped or quiesced, ignore 'bypass_insert' from 1676 * blk_mq_request_issue_directly(), and return BLK_STS_OK to caller, 1677 * and avoid driver to try to dispatch again. 1678 */ 1679 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) { 1680 run_queue = false; 1681 bypass_insert = false; 1682 goto insert; 1683 } 1684 1685 if (q->elevator && !bypass_insert) 1686 goto insert; 1687 1688 if (!blk_mq_get_dispatch_budget(hctx)) 1689 goto insert; 1690 1691 if (!blk_mq_get_driver_tag(rq, NULL, false)) { 1692 blk_mq_put_dispatch_budget(hctx); 1693 goto insert; 1694 } 1695 1696 return __blk_mq_issue_directly(hctx, rq, cookie); 1697 insert: 1698 if (bypass_insert) 1699 return BLK_STS_RESOURCE; 1700 1701 blk_mq_sched_insert_request(rq, false, run_queue, false); 1702 return BLK_STS_OK; 1703 } 1704 1705 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 1706 struct request *rq, blk_qc_t *cookie) 1707 { 1708 blk_status_t ret; 1709 int srcu_idx; 1710 1711 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); 1712 1713 hctx_lock(hctx, &srcu_idx); 1714 1715 ret = __blk_mq_try_issue_directly(hctx, rq, cookie, false); 1716 if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) 1717 blk_mq_sched_insert_request(rq, false, true, false); 1718 else if (ret != BLK_STS_OK) 1719 blk_mq_end_request(rq, ret); 1720 1721 hctx_unlock(hctx, srcu_idx); 1722 } 1723 1724 blk_status_t blk_mq_request_issue_directly(struct request *rq) 1725 { 1726 blk_status_t ret; 1727 int srcu_idx; 1728 blk_qc_t unused_cookie; 1729 struct blk_mq_ctx *ctx = rq->mq_ctx; 1730 struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(rq->q, ctx->cpu); 1731 1732 hctx_lock(hctx, &srcu_idx); 1733 ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true); 1734 hctx_unlock(hctx, srcu_idx); 1735 1736 return ret; 1737 } 1738 1739 static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) 1740 { 1741 const int is_sync = op_is_sync(bio->bi_opf); 1742 const int is_flush_fua = op_is_flush(bio->bi_opf); 1743 struct blk_mq_alloc_data data = { .flags = 0 }; 1744 struct request *rq; 1745 unsigned int request_count = 0; 1746 struct blk_plug *plug; 1747 struct request *same_queue_rq = NULL; 1748 blk_qc_t cookie; 1749 unsigned int wb_acct; 1750 1751 blk_queue_bounce(q, &bio); 1752 1753 blk_queue_split(q, &bio); 1754 1755 if (!bio_integrity_prep(bio)) 1756 return BLK_QC_T_NONE; 1757 1758 if (!is_flush_fua && !blk_queue_nomerges(q) && 1759 blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq)) 1760 return BLK_QC_T_NONE; 1761 1762 if (blk_mq_sched_bio_merge(q, bio)) 1763 return BLK_QC_T_NONE; 1764 1765 wb_acct = wbt_wait(q->rq_wb, bio, NULL); 1766 1767 trace_block_getrq(q, bio, bio->bi_opf); 1768 1769 rq = blk_mq_get_request(q, bio, bio->bi_opf, &data); 1770 if (unlikely(!rq)) { 1771 __wbt_done(q->rq_wb, wb_acct); 1772 if (bio->bi_opf & REQ_NOWAIT) 1773 bio_wouldblock_error(bio); 1774 return BLK_QC_T_NONE; 1775 } 1776 1777 wbt_track(rq, wb_acct); 1778 1779 cookie = request_to_qc_t(data.hctx, rq); 1780 1781 plug = current->plug; 1782 if (unlikely(is_flush_fua)) { 1783 blk_mq_put_ctx(data.ctx); 1784 blk_mq_bio_to_request(rq, bio); 1785 1786 /* bypass scheduler for flush rq */ 1787 blk_insert_flush(rq); 1788 blk_mq_run_hw_queue(data.hctx, true); 1789 } else if (plug && q->nr_hw_queues == 1) { 1790 struct request *last = NULL; 1791 1792 blk_mq_put_ctx(data.ctx); 1793 blk_mq_bio_to_request(rq, bio); 1794 1795 /* 1796 * @request_count may become stale because of schedule 1797 * out, so check the list again. 1798 */ 1799 if (list_empty(&plug->mq_list)) 1800 request_count = 0; 1801 else if (blk_queue_nomerges(q)) 1802 request_count = blk_plug_queued_count(q); 1803 1804 if (!request_count) 1805 trace_block_plug(q); 1806 else 1807 last = list_entry_rq(plug->mq_list.prev); 1808 1809 if (request_count >= BLK_MAX_REQUEST_COUNT || (last && 1810 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { 1811 blk_flush_plug_list(plug, false); 1812 trace_block_plug(q); 1813 } 1814 1815 list_add_tail(&rq->queuelist, &plug->mq_list); 1816 } else if (plug && !blk_queue_nomerges(q)) { 1817 blk_mq_bio_to_request(rq, bio); 1818 1819 /* 1820 * We do limited plugging. If the bio can be merged, do that. 1821 * Otherwise the existing request in the plug list will be 1822 * issued. So the plug list will have one request at most 1823 * The plug list might get flushed before this. If that happens, 1824 * the plug list is empty, and same_queue_rq is invalid. 1825 */ 1826 if (list_empty(&plug->mq_list)) 1827 same_queue_rq = NULL; 1828 if (same_queue_rq) 1829 list_del_init(&same_queue_rq->queuelist); 1830 list_add_tail(&rq->queuelist, &plug->mq_list); 1831 1832 blk_mq_put_ctx(data.ctx); 1833 1834 if (same_queue_rq) { 1835 data.hctx = blk_mq_map_queue(q, 1836 same_queue_rq->mq_ctx->cpu); 1837 blk_mq_try_issue_directly(data.hctx, same_queue_rq, 1838 &cookie); 1839 } 1840 } else if (q->nr_hw_queues > 1 && is_sync) { 1841 blk_mq_put_ctx(data.ctx); 1842 blk_mq_bio_to_request(rq, bio); 1843 blk_mq_try_issue_directly(data.hctx, rq, &cookie); 1844 } else { 1845 blk_mq_put_ctx(data.ctx); 1846 blk_mq_bio_to_request(rq, bio); 1847 blk_mq_sched_insert_request(rq, false, true, true); 1848 } 1849 1850 return cookie; 1851 } 1852 1853 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 1854 unsigned int hctx_idx) 1855 { 1856 struct page *page; 1857 1858 if (tags->rqs && set->ops->exit_request) { 1859 int i; 1860 1861 for (i = 0; i < tags->nr_tags; i++) { 1862 struct request *rq = tags->static_rqs[i]; 1863 1864 if (!rq) 1865 continue; 1866 set->ops->exit_request(set, rq, hctx_idx); 1867 tags->static_rqs[i] = NULL; 1868 } 1869 } 1870 1871 while (!list_empty(&tags->page_list)) { 1872 page = list_first_entry(&tags->page_list, struct page, lru); 1873 list_del_init(&page->lru); 1874 /* 1875 * Remove kmemleak object previously allocated in 1876 * blk_mq_init_rq_map(). 1877 */ 1878 kmemleak_free(page_address(page)); 1879 __free_pages(page, page->private); 1880 } 1881 } 1882 1883 void blk_mq_free_rq_map(struct blk_mq_tags *tags) 1884 { 1885 kfree(tags->rqs); 1886 tags->rqs = NULL; 1887 kfree(tags->static_rqs); 1888 tags->static_rqs = NULL; 1889 1890 blk_mq_free_tags(tags); 1891 } 1892 1893 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, 1894 unsigned int hctx_idx, 1895 unsigned int nr_tags, 1896 unsigned int reserved_tags) 1897 { 1898 struct blk_mq_tags *tags; 1899 int node; 1900 1901 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx); 1902 if (node == NUMA_NO_NODE) 1903 node = set->numa_node; 1904 1905 tags = blk_mq_init_tags(nr_tags, reserved_tags, node, 1906 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags)); 1907 if (!tags) 1908 return NULL; 1909 1910 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *), 1911 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 1912 node); 1913 if (!tags->rqs) { 1914 blk_mq_free_tags(tags); 1915 return NULL; 1916 } 1917 1918 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *), 1919 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 1920 node); 1921 if (!tags->static_rqs) { 1922 kfree(tags->rqs); 1923 blk_mq_free_tags(tags); 1924 return NULL; 1925 } 1926 1927 return tags; 1928 } 1929 1930 static size_t order_to_size(unsigned int order) 1931 { 1932 return (size_t)PAGE_SIZE << order; 1933 } 1934 1935 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, 1936 unsigned int hctx_idx, int node) 1937 { 1938 int ret; 1939 1940 if (set->ops->init_request) { 1941 ret = set->ops->init_request(set, rq, hctx_idx, node); 1942 if (ret) 1943 return ret; 1944 } 1945 1946 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 1947 return 0; 1948 } 1949 1950 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 1951 unsigned int hctx_idx, unsigned int depth) 1952 { 1953 unsigned int i, j, entries_per_page, max_order = 4; 1954 size_t rq_size, left; 1955 int node; 1956 1957 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx); 1958 if (node == NUMA_NO_NODE) 1959 node = set->numa_node; 1960 1961 INIT_LIST_HEAD(&tags->page_list); 1962 1963 /* 1964 * rq_size is the size of the request plus driver payload, rounded 1965 * to the cacheline size 1966 */ 1967 rq_size = round_up(sizeof(struct request) + set->cmd_size, 1968 cache_line_size()); 1969 left = rq_size * depth; 1970 1971 for (i = 0; i < depth; ) { 1972 int this_order = max_order; 1973 struct page *page; 1974 int to_do; 1975 void *p; 1976 1977 while (this_order && left < order_to_size(this_order - 1)) 1978 this_order--; 1979 1980 do { 1981 page = alloc_pages_node(node, 1982 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, 1983 this_order); 1984 if (page) 1985 break; 1986 if (!this_order--) 1987 break; 1988 if (order_to_size(this_order) < rq_size) 1989 break; 1990 } while (1); 1991 1992 if (!page) 1993 goto fail; 1994 1995 page->private = this_order; 1996 list_add_tail(&page->lru, &tags->page_list); 1997 1998 p = page_address(page); 1999 /* 2000 * Allow kmemleak to scan these pages as they contain pointers 2001 * to additional allocations like via ops->init_request(). 2002 */ 2003 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); 2004 entries_per_page = order_to_size(this_order) / rq_size; 2005 to_do = min(entries_per_page, depth - i); 2006 left -= to_do * rq_size; 2007 for (j = 0; j < to_do; j++) { 2008 struct request *rq = p; 2009 2010 tags->static_rqs[i] = rq; 2011 if (blk_mq_init_request(set, rq, hctx_idx, node)) { 2012 tags->static_rqs[i] = NULL; 2013 goto fail; 2014 } 2015 2016 p += rq_size; 2017 i++; 2018 } 2019 } 2020 return 0; 2021 2022 fail: 2023 blk_mq_free_rqs(set, tags, hctx_idx); 2024 return -ENOMEM; 2025 } 2026 2027 /* 2028 * 'cpu' is going away. splice any existing rq_list entries from this 2029 * software queue to the hw queue dispatch list, and ensure that it 2030 * gets run. 2031 */ 2032 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) 2033 { 2034 struct blk_mq_hw_ctx *hctx; 2035 struct blk_mq_ctx *ctx; 2036 LIST_HEAD(tmp); 2037 2038 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead); 2039 ctx = __blk_mq_get_ctx(hctx->queue, cpu); 2040 2041 spin_lock(&ctx->lock); 2042 if (!list_empty(&ctx->rq_list)) { 2043 list_splice_init(&ctx->rq_list, &tmp); 2044 blk_mq_hctx_clear_pending(hctx, ctx); 2045 } 2046 spin_unlock(&ctx->lock); 2047 2048 if (list_empty(&tmp)) 2049 return 0; 2050 2051 spin_lock(&hctx->lock); 2052 list_splice_tail_init(&tmp, &hctx->dispatch); 2053 spin_unlock(&hctx->lock); 2054 2055 blk_mq_run_hw_queue(hctx, true); 2056 return 0; 2057 } 2058 2059 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) 2060 { 2061 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD, 2062 &hctx->cpuhp_dead); 2063 } 2064 2065 /* hctx->ctxs will be freed in queue's release handler */ 2066 static void blk_mq_exit_hctx(struct request_queue *q, 2067 struct blk_mq_tag_set *set, 2068 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 2069 { 2070 blk_mq_debugfs_unregister_hctx(hctx); 2071 2072 if (blk_mq_hw_queue_mapped(hctx)) 2073 blk_mq_tag_idle(hctx); 2074 2075 if (set->ops->exit_request) 2076 set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx); 2077 2078 blk_mq_sched_exit_hctx(q, hctx, hctx_idx); 2079 2080 if (set->ops->exit_hctx) 2081 set->ops->exit_hctx(hctx, hctx_idx); 2082 2083 if (hctx->flags & BLK_MQ_F_BLOCKING) 2084 cleanup_srcu_struct(hctx->srcu); 2085 2086 blk_mq_remove_cpuhp(hctx); 2087 blk_free_flush_queue(hctx->fq); 2088 sbitmap_free(&hctx->ctx_map); 2089 } 2090 2091 static void blk_mq_exit_hw_queues(struct request_queue *q, 2092 struct blk_mq_tag_set *set, int nr_queue) 2093 { 2094 struct blk_mq_hw_ctx *hctx; 2095 unsigned int i; 2096 2097 queue_for_each_hw_ctx(q, hctx, i) { 2098 if (i == nr_queue) 2099 break; 2100 blk_mq_exit_hctx(q, set, hctx, i); 2101 } 2102 } 2103 2104 static int blk_mq_init_hctx(struct request_queue *q, 2105 struct blk_mq_tag_set *set, 2106 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) 2107 { 2108 int node; 2109 2110 node = hctx->numa_node; 2111 if (node == NUMA_NO_NODE) 2112 node = hctx->numa_node = set->numa_node; 2113 2114 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); 2115 spin_lock_init(&hctx->lock); 2116 INIT_LIST_HEAD(&hctx->dispatch); 2117 hctx->queue = q; 2118 hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED; 2119 2120 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead); 2121 2122 hctx->tags = set->tags[hctx_idx]; 2123 2124 /* 2125 * Allocate space for all possible cpus to avoid allocation at 2126 * runtime 2127 */ 2128 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *), 2129 GFP_KERNEL, node); 2130 if (!hctx->ctxs) 2131 goto unregister_cpu_notifier; 2132 2133 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL, 2134 node)) 2135 goto free_ctxs; 2136 2137 hctx->nr_ctx = 0; 2138 2139 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); 2140 INIT_LIST_HEAD(&hctx->dispatch_wait.entry); 2141 2142 if (set->ops->init_hctx && 2143 set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) 2144 goto free_bitmap; 2145 2146 if (blk_mq_sched_init_hctx(q, hctx, hctx_idx)) 2147 goto exit_hctx; 2148 2149 hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size); 2150 if (!hctx->fq) 2151 goto sched_exit_hctx; 2152 2153 if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx, node)) 2154 goto free_fq; 2155 2156 if (hctx->flags & BLK_MQ_F_BLOCKING) 2157 init_srcu_struct(hctx->srcu); 2158 2159 blk_mq_debugfs_register_hctx(q, hctx); 2160 2161 return 0; 2162 2163 free_fq: 2164 kfree(hctx->fq); 2165 sched_exit_hctx: 2166 blk_mq_sched_exit_hctx(q, hctx, hctx_idx); 2167 exit_hctx: 2168 if (set->ops->exit_hctx) 2169 set->ops->exit_hctx(hctx, hctx_idx); 2170 free_bitmap: 2171 sbitmap_free(&hctx->ctx_map); 2172 free_ctxs: 2173 kfree(hctx->ctxs); 2174 unregister_cpu_notifier: 2175 blk_mq_remove_cpuhp(hctx); 2176 return -1; 2177 } 2178 2179 static void blk_mq_init_cpu_queues(struct request_queue *q, 2180 unsigned int nr_hw_queues) 2181 { 2182 unsigned int i; 2183 2184 for_each_possible_cpu(i) { 2185 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i); 2186 struct blk_mq_hw_ctx *hctx; 2187 2188 __ctx->cpu = i; 2189 spin_lock_init(&__ctx->lock); 2190 INIT_LIST_HEAD(&__ctx->rq_list); 2191 __ctx->queue = q; 2192 2193 /* 2194 * Set local node, IFF we have more than one hw queue. If 2195 * not, we remain on the home node of the device 2196 */ 2197 hctx = blk_mq_map_queue(q, i); 2198 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) 2199 hctx->numa_node = local_memory_node(cpu_to_node(i)); 2200 } 2201 } 2202 2203 static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx) 2204 { 2205 int ret = 0; 2206 2207 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx, 2208 set->queue_depth, set->reserved_tags); 2209 if (!set->tags[hctx_idx]) 2210 return false; 2211 2212 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx, 2213 set->queue_depth); 2214 if (!ret) 2215 return true; 2216 2217 blk_mq_free_rq_map(set->tags[hctx_idx]); 2218 set->tags[hctx_idx] = NULL; 2219 return false; 2220 } 2221 2222 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set, 2223 unsigned int hctx_idx) 2224 { 2225 if (set->tags[hctx_idx]) { 2226 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx); 2227 blk_mq_free_rq_map(set->tags[hctx_idx]); 2228 set->tags[hctx_idx] = NULL; 2229 } 2230 } 2231 2232 static void blk_mq_map_swqueue(struct request_queue *q) 2233 { 2234 unsigned int i, hctx_idx; 2235 struct blk_mq_hw_ctx *hctx; 2236 struct blk_mq_ctx *ctx; 2237 struct blk_mq_tag_set *set = q->tag_set; 2238 2239 /* 2240 * Avoid others reading imcomplete hctx->cpumask through sysfs 2241 */ 2242 mutex_lock(&q->sysfs_lock); 2243 2244 queue_for_each_hw_ctx(q, hctx, i) { 2245 cpumask_clear(hctx->cpumask); 2246 hctx->nr_ctx = 0; 2247 hctx->dispatch_from = NULL; 2248 } 2249 2250 /* 2251 * Map software to hardware queues. 2252 * 2253 * If the cpu isn't present, the cpu is mapped to first hctx. 2254 */ 2255 for_each_possible_cpu(i) { 2256 hctx_idx = q->mq_map[i]; 2257 /* unmapped hw queue can be remapped after CPU topo changed */ 2258 if (!set->tags[hctx_idx] && 2259 !__blk_mq_alloc_rq_map(set, hctx_idx)) { 2260 /* 2261 * If tags initialization fail for some hctx, 2262 * that hctx won't be brought online. In this 2263 * case, remap the current ctx to hctx[0] which 2264 * is guaranteed to always have tags allocated 2265 */ 2266 q->mq_map[i] = 0; 2267 } 2268 2269 ctx = per_cpu_ptr(q->queue_ctx, i); 2270 hctx = blk_mq_map_queue(q, i); 2271 2272 cpumask_set_cpu(i, hctx->cpumask); 2273 ctx->index_hw = hctx->nr_ctx; 2274 hctx->ctxs[hctx->nr_ctx++] = ctx; 2275 } 2276 2277 mutex_unlock(&q->sysfs_lock); 2278 2279 queue_for_each_hw_ctx(q, hctx, i) { 2280 /* 2281 * If no software queues are mapped to this hardware queue, 2282 * disable it and free the request entries. 2283 */ 2284 if (!hctx->nr_ctx) { 2285 /* Never unmap queue 0. We need it as a 2286 * fallback in case of a new remap fails 2287 * allocation 2288 */ 2289 if (i && set->tags[i]) 2290 blk_mq_free_map_and_requests(set, i); 2291 2292 hctx->tags = NULL; 2293 continue; 2294 } 2295 2296 hctx->tags = set->tags[i]; 2297 WARN_ON(!hctx->tags); 2298 2299 /* 2300 * Set the map size to the number of mapped software queues. 2301 * This is more accurate and more efficient than looping 2302 * over all possibly mapped software queues. 2303 */ 2304 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx); 2305 2306 /* 2307 * Initialize batch roundrobin counts 2308 */ 2309 hctx->next_cpu = blk_mq_first_mapped_cpu(hctx); 2310 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 2311 } 2312 } 2313 2314 /* 2315 * Caller needs to ensure that we're either frozen/quiesced, or that 2316 * the queue isn't live yet. 2317 */ 2318 static void queue_set_hctx_shared(struct request_queue *q, bool shared) 2319 { 2320 struct blk_mq_hw_ctx *hctx; 2321 int i; 2322 2323 queue_for_each_hw_ctx(q, hctx, i) { 2324 if (shared) { 2325 if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) 2326 atomic_inc(&q->shared_hctx_restart); 2327 hctx->flags |= BLK_MQ_F_TAG_SHARED; 2328 } else { 2329 if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) 2330 atomic_dec(&q->shared_hctx_restart); 2331 hctx->flags &= ~BLK_MQ_F_TAG_SHARED; 2332 } 2333 } 2334 } 2335 2336 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, 2337 bool shared) 2338 { 2339 struct request_queue *q; 2340 2341 lockdep_assert_held(&set->tag_list_lock); 2342 2343 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2344 blk_mq_freeze_queue(q); 2345 queue_set_hctx_shared(q, shared); 2346 blk_mq_unfreeze_queue(q); 2347 } 2348 } 2349 2350 static void blk_mq_del_queue_tag_set(struct request_queue *q) 2351 { 2352 struct blk_mq_tag_set *set = q->tag_set; 2353 2354 mutex_lock(&set->tag_list_lock); 2355 list_del_rcu(&q->tag_set_list); 2356 if (list_is_singular(&set->tag_list)) { 2357 /* just transitioned to unshared */ 2358 set->flags &= ~BLK_MQ_F_TAG_SHARED; 2359 /* update existing queue */ 2360 blk_mq_update_tag_set_depth(set, false); 2361 } 2362 mutex_unlock(&set->tag_list_lock); 2363 synchronize_rcu(); 2364 INIT_LIST_HEAD(&q->tag_set_list); 2365 } 2366 2367 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, 2368 struct request_queue *q) 2369 { 2370 q->tag_set = set; 2371 2372 mutex_lock(&set->tag_list_lock); 2373 2374 /* 2375 * Check to see if we're transitioning to shared (from 1 to 2 queues). 2376 */ 2377 if (!list_empty(&set->tag_list) && 2378 !(set->flags & BLK_MQ_F_TAG_SHARED)) { 2379 set->flags |= BLK_MQ_F_TAG_SHARED; 2380 /* update existing queue */ 2381 blk_mq_update_tag_set_depth(set, true); 2382 } 2383 if (set->flags & BLK_MQ_F_TAG_SHARED) 2384 queue_set_hctx_shared(q, true); 2385 list_add_tail_rcu(&q->tag_set_list, &set->tag_list); 2386 2387 mutex_unlock(&set->tag_list_lock); 2388 } 2389 2390 /* 2391 * It is the actual release handler for mq, but we do it from 2392 * request queue's release handler for avoiding use-after-free 2393 * and headache because q->mq_kobj shouldn't have been introduced, 2394 * but we can't group ctx/kctx kobj without it. 2395 */ 2396 void blk_mq_release(struct request_queue *q) 2397 { 2398 struct blk_mq_hw_ctx *hctx; 2399 unsigned int i; 2400 2401 /* hctx kobj stays in hctx */ 2402 queue_for_each_hw_ctx(q, hctx, i) { 2403 if (!hctx) 2404 continue; 2405 kobject_put(&hctx->kobj); 2406 } 2407 2408 q->mq_map = NULL; 2409 2410 kfree(q->queue_hw_ctx); 2411 2412 /* 2413 * release .mq_kobj and sw queue's kobject now because 2414 * both share lifetime with request queue. 2415 */ 2416 blk_mq_sysfs_deinit(q); 2417 2418 free_percpu(q->queue_ctx); 2419 } 2420 2421 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) 2422 { 2423 struct request_queue *uninit_q, *q; 2424 2425 uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node, NULL); 2426 if (!uninit_q) 2427 return ERR_PTR(-ENOMEM); 2428 2429 q = blk_mq_init_allocated_queue(set, uninit_q); 2430 if (IS_ERR(q)) 2431 blk_cleanup_queue(uninit_q); 2432 2433 return q; 2434 } 2435 EXPORT_SYMBOL(blk_mq_init_queue); 2436 2437 static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set) 2438 { 2439 int hw_ctx_size = sizeof(struct blk_mq_hw_ctx); 2440 2441 BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu), 2442 __alignof__(struct blk_mq_hw_ctx)) != 2443 sizeof(struct blk_mq_hw_ctx)); 2444 2445 if (tag_set->flags & BLK_MQ_F_BLOCKING) 2446 hw_ctx_size += sizeof(struct srcu_struct); 2447 2448 return hw_ctx_size; 2449 } 2450 2451 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, 2452 struct request_queue *q) 2453 { 2454 int i, j; 2455 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; 2456 2457 blk_mq_sysfs_unregister(q); 2458 2459 /* protect against switching io scheduler */ 2460 mutex_lock(&q->sysfs_lock); 2461 for (i = 0; i < set->nr_hw_queues; i++) { 2462 int node; 2463 2464 if (hctxs[i]) 2465 continue; 2466 2467 node = blk_mq_hw_queue_to_node(q->mq_map, i); 2468 hctxs[i] = kzalloc_node(blk_mq_hw_ctx_size(set), 2469 GFP_KERNEL, node); 2470 if (!hctxs[i]) 2471 break; 2472 2473 if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL, 2474 node)) { 2475 kfree(hctxs[i]); 2476 hctxs[i] = NULL; 2477 break; 2478 } 2479 2480 atomic_set(&hctxs[i]->nr_active, 0); 2481 hctxs[i]->numa_node = node; 2482 hctxs[i]->queue_num = i; 2483 2484 if (blk_mq_init_hctx(q, set, hctxs[i], i)) { 2485 free_cpumask_var(hctxs[i]->cpumask); 2486 kfree(hctxs[i]); 2487 hctxs[i] = NULL; 2488 break; 2489 } 2490 blk_mq_hctx_kobj_init(hctxs[i]); 2491 } 2492 for (j = i; j < q->nr_hw_queues; j++) { 2493 struct blk_mq_hw_ctx *hctx = hctxs[j]; 2494 2495 if (hctx) { 2496 if (hctx->tags) 2497 blk_mq_free_map_and_requests(set, j); 2498 blk_mq_exit_hctx(q, set, hctx, j); 2499 kobject_put(&hctx->kobj); 2500 hctxs[j] = NULL; 2501 2502 } 2503 } 2504 q->nr_hw_queues = i; 2505 mutex_unlock(&q->sysfs_lock); 2506 blk_mq_sysfs_register(q); 2507 } 2508 2509 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 2510 struct request_queue *q) 2511 { 2512 /* mark the queue as mq asap */ 2513 q->mq_ops = set->ops; 2514 2515 q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn, 2516 blk_mq_poll_stats_bkt, 2517 BLK_MQ_POLL_STATS_BKTS, q); 2518 if (!q->poll_cb) 2519 goto err_exit; 2520 2521 q->queue_ctx = alloc_percpu(struct blk_mq_ctx); 2522 if (!q->queue_ctx) 2523 goto err_exit; 2524 2525 /* init q->mq_kobj and sw queues' kobjects */ 2526 blk_mq_sysfs_init(q); 2527 2528 q->queue_hw_ctx = kcalloc_node(nr_cpu_ids, sizeof(*(q->queue_hw_ctx)), 2529 GFP_KERNEL, set->numa_node); 2530 if (!q->queue_hw_ctx) 2531 goto err_percpu; 2532 2533 q->mq_map = set->mq_map; 2534 2535 blk_mq_realloc_hw_ctxs(set, q); 2536 if (!q->nr_hw_queues) 2537 goto err_hctxs; 2538 2539 INIT_WORK(&q->timeout_work, blk_mq_timeout_work); 2540 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ); 2541 2542 q->nr_queues = nr_cpu_ids; 2543 2544 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; 2545 2546 if (!(set->flags & BLK_MQ_F_SG_MERGE)) 2547 queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q); 2548 2549 q->sg_reserved_size = INT_MAX; 2550 2551 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); 2552 INIT_LIST_HEAD(&q->requeue_list); 2553 spin_lock_init(&q->requeue_lock); 2554 2555 blk_queue_make_request(q, blk_mq_make_request); 2556 if (q->mq_ops->poll) 2557 q->poll_fn = blk_mq_poll; 2558 2559 /* 2560 * Do this after blk_queue_make_request() overrides it... 2561 */ 2562 q->nr_requests = set->queue_depth; 2563 2564 /* 2565 * Default to classic polling 2566 */ 2567 q->poll_nsec = -1; 2568 2569 if (set->ops->complete) 2570 blk_queue_softirq_done(q, set->ops->complete); 2571 2572 blk_mq_init_cpu_queues(q, set->nr_hw_queues); 2573 blk_mq_add_queue_tag_set(set, q); 2574 blk_mq_map_swqueue(q); 2575 2576 if (!(set->flags & BLK_MQ_F_NO_SCHED)) { 2577 int ret; 2578 2579 ret = elevator_init_mq(q); 2580 if (ret) 2581 return ERR_PTR(ret); 2582 } 2583 2584 return q; 2585 2586 err_hctxs: 2587 kfree(q->queue_hw_ctx); 2588 err_percpu: 2589 free_percpu(q->queue_ctx); 2590 err_exit: 2591 q->mq_ops = NULL; 2592 return ERR_PTR(-ENOMEM); 2593 } 2594 EXPORT_SYMBOL(blk_mq_init_allocated_queue); 2595 2596 void blk_mq_free_queue(struct request_queue *q) 2597 { 2598 struct blk_mq_tag_set *set = q->tag_set; 2599 2600 blk_mq_del_queue_tag_set(q); 2601 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); 2602 } 2603 2604 /* Basically redo blk_mq_init_queue with queue frozen */ 2605 static void blk_mq_queue_reinit(struct request_queue *q) 2606 { 2607 WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth)); 2608 2609 blk_mq_debugfs_unregister_hctxs(q); 2610 blk_mq_sysfs_unregister(q); 2611 2612 /* 2613 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe 2614 * we should change hctx numa_node according to the new topology (this 2615 * involves freeing and re-allocating memory, worth doing?) 2616 */ 2617 blk_mq_map_swqueue(q); 2618 2619 blk_mq_sysfs_register(q); 2620 blk_mq_debugfs_register_hctxs(q); 2621 } 2622 2623 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2624 { 2625 int i; 2626 2627 for (i = 0; i < set->nr_hw_queues; i++) 2628 if (!__blk_mq_alloc_rq_map(set, i)) 2629 goto out_unwind; 2630 2631 return 0; 2632 2633 out_unwind: 2634 while (--i >= 0) 2635 blk_mq_free_rq_map(set->tags[i]); 2636 2637 return -ENOMEM; 2638 } 2639 2640 /* 2641 * Allocate the request maps associated with this tag_set. Note that this 2642 * may reduce the depth asked for, if memory is tight. set->queue_depth 2643 * will be updated to reflect the allocated depth. 2644 */ 2645 static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2646 { 2647 unsigned int depth; 2648 int err; 2649 2650 depth = set->queue_depth; 2651 do { 2652 err = __blk_mq_alloc_rq_maps(set); 2653 if (!err) 2654 break; 2655 2656 set->queue_depth >>= 1; 2657 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) { 2658 err = -ENOMEM; 2659 break; 2660 } 2661 } while (set->queue_depth); 2662 2663 if (!set->queue_depth || err) { 2664 pr_err("blk-mq: failed to allocate request map\n"); 2665 return -ENOMEM; 2666 } 2667 2668 if (depth != set->queue_depth) 2669 pr_info("blk-mq: reduced tag depth (%u -> %u)\n", 2670 depth, set->queue_depth); 2671 2672 return 0; 2673 } 2674 2675 static int blk_mq_update_queue_map(struct blk_mq_tag_set *set) 2676 { 2677 if (set->ops->map_queues) { 2678 int cpu; 2679 /* 2680 * transport .map_queues is usually done in the following 2681 * way: 2682 * 2683 * for (queue = 0; queue < set->nr_hw_queues; queue++) { 2684 * mask = get_cpu_mask(queue) 2685 * for_each_cpu(cpu, mask) 2686 * set->mq_map[cpu] = queue; 2687 * } 2688 * 2689 * When we need to remap, the table has to be cleared for 2690 * killing stale mapping since one CPU may not be mapped 2691 * to any hw queue. 2692 */ 2693 for_each_possible_cpu(cpu) 2694 set->mq_map[cpu] = 0; 2695 2696 return set->ops->map_queues(set); 2697 } else 2698 return blk_mq_map_queues(set); 2699 } 2700 2701 /* 2702 * Alloc a tag set to be associated with one or more request queues. 2703 * May fail with EINVAL for various error conditions. May adjust the 2704 * requested depth down, if if it too large. In that case, the set 2705 * value will be stored in set->queue_depth. 2706 */ 2707 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) 2708 { 2709 int ret; 2710 2711 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS); 2712 2713 if (!set->nr_hw_queues) 2714 return -EINVAL; 2715 if (!set->queue_depth) 2716 return -EINVAL; 2717 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) 2718 return -EINVAL; 2719 2720 if (!set->ops->queue_rq) 2721 return -EINVAL; 2722 2723 if (!set->ops->get_budget ^ !set->ops->put_budget) 2724 return -EINVAL; 2725 2726 if (set->queue_depth > BLK_MQ_MAX_DEPTH) { 2727 pr_info("blk-mq: reduced tag depth to %u\n", 2728 BLK_MQ_MAX_DEPTH); 2729 set->queue_depth = BLK_MQ_MAX_DEPTH; 2730 } 2731 2732 /* 2733 * If a crashdump is active, then we are potentially in a very 2734 * memory constrained environment. Limit us to 1 queue and 2735 * 64 tags to prevent using too much memory. 2736 */ 2737 if (is_kdump_kernel()) { 2738 set->nr_hw_queues = 1; 2739 set->queue_depth = min(64U, set->queue_depth); 2740 } 2741 /* 2742 * There is no use for more h/w queues than cpus. 2743 */ 2744 if (set->nr_hw_queues > nr_cpu_ids) 2745 set->nr_hw_queues = nr_cpu_ids; 2746 2747 set->tags = kcalloc_node(nr_cpu_ids, sizeof(struct blk_mq_tags *), 2748 GFP_KERNEL, set->numa_node); 2749 if (!set->tags) 2750 return -ENOMEM; 2751 2752 ret = -ENOMEM; 2753 set->mq_map = kcalloc_node(nr_cpu_ids, sizeof(*set->mq_map), 2754 GFP_KERNEL, set->numa_node); 2755 if (!set->mq_map) 2756 goto out_free_tags; 2757 2758 ret = blk_mq_update_queue_map(set); 2759 if (ret) 2760 goto out_free_mq_map; 2761 2762 ret = blk_mq_alloc_rq_maps(set); 2763 if (ret) 2764 goto out_free_mq_map; 2765 2766 mutex_init(&set->tag_list_lock); 2767 INIT_LIST_HEAD(&set->tag_list); 2768 2769 return 0; 2770 2771 out_free_mq_map: 2772 kfree(set->mq_map); 2773 set->mq_map = NULL; 2774 out_free_tags: 2775 kfree(set->tags); 2776 set->tags = NULL; 2777 return ret; 2778 } 2779 EXPORT_SYMBOL(blk_mq_alloc_tag_set); 2780 2781 void blk_mq_free_tag_set(struct blk_mq_tag_set *set) 2782 { 2783 int i; 2784 2785 for (i = 0; i < nr_cpu_ids; i++) 2786 blk_mq_free_map_and_requests(set, i); 2787 2788 kfree(set->mq_map); 2789 set->mq_map = NULL; 2790 2791 kfree(set->tags); 2792 set->tags = NULL; 2793 } 2794 EXPORT_SYMBOL(blk_mq_free_tag_set); 2795 2796 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) 2797 { 2798 struct blk_mq_tag_set *set = q->tag_set; 2799 struct blk_mq_hw_ctx *hctx; 2800 int i, ret; 2801 2802 if (!set) 2803 return -EINVAL; 2804 2805 blk_mq_freeze_queue(q); 2806 blk_mq_quiesce_queue(q); 2807 2808 ret = 0; 2809 queue_for_each_hw_ctx(q, hctx, i) { 2810 if (!hctx->tags) 2811 continue; 2812 /* 2813 * If we're using an MQ scheduler, just update the scheduler 2814 * queue depth. This is similar to what the old code would do. 2815 */ 2816 if (!hctx->sched_tags) { 2817 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr, 2818 false); 2819 } else { 2820 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags, 2821 nr, true); 2822 } 2823 if (ret) 2824 break; 2825 } 2826 2827 if (!ret) 2828 q->nr_requests = nr; 2829 2830 blk_mq_unquiesce_queue(q); 2831 blk_mq_unfreeze_queue(q); 2832 2833 return ret; 2834 } 2835 2836 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, 2837 int nr_hw_queues) 2838 { 2839 struct request_queue *q; 2840 2841 lockdep_assert_held(&set->tag_list_lock); 2842 2843 if (nr_hw_queues > nr_cpu_ids) 2844 nr_hw_queues = nr_cpu_ids; 2845 if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues) 2846 return; 2847 2848 list_for_each_entry(q, &set->tag_list, tag_set_list) 2849 blk_mq_freeze_queue(q); 2850 2851 set->nr_hw_queues = nr_hw_queues; 2852 blk_mq_update_queue_map(set); 2853 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2854 blk_mq_realloc_hw_ctxs(set, q); 2855 blk_mq_queue_reinit(q); 2856 } 2857 2858 list_for_each_entry(q, &set->tag_list, tag_set_list) 2859 blk_mq_unfreeze_queue(q); 2860 } 2861 2862 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) 2863 { 2864 mutex_lock(&set->tag_list_lock); 2865 __blk_mq_update_nr_hw_queues(set, nr_hw_queues); 2866 mutex_unlock(&set->tag_list_lock); 2867 } 2868 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); 2869 2870 /* Enable polling stats and return whether they were already enabled. */ 2871 static bool blk_poll_stats_enable(struct request_queue *q) 2872 { 2873 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 2874 blk_queue_flag_test_and_set(QUEUE_FLAG_POLL_STATS, q)) 2875 return true; 2876 blk_stat_add_callback(q, q->poll_cb); 2877 return false; 2878 } 2879 2880 static void blk_mq_poll_stats_start(struct request_queue *q) 2881 { 2882 /* 2883 * We don't arm the callback if polling stats are not enabled or the 2884 * callback is already active. 2885 */ 2886 if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 2887 blk_stat_is_active(q->poll_cb)) 2888 return; 2889 2890 blk_stat_activate_msecs(q->poll_cb, 100); 2891 } 2892 2893 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb) 2894 { 2895 struct request_queue *q = cb->data; 2896 int bucket; 2897 2898 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) { 2899 if (cb->stat[bucket].nr_samples) 2900 q->poll_stat[bucket] = cb->stat[bucket]; 2901 } 2902 } 2903 2904 static unsigned long blk_mq_poll_nsecs(struct request_queue *q, 2905 struct blk_mq_hw_ctx *hctx, 2906 struct request *rq) 2907 { 2908 unsigned long ret = 0; 2909 int bucket; 2910 2911 /* 2912 * If stats collection isn't on, don't sleep but turn it on for 2913 * future users 2914 */ 2915 if (!blk_poll_stats_enable(q)) 2916 return 0; 2917 2918 /* 2919 * As an optimistic guess, use half of the mean service time 2920 * for this type of request. We can (and should) make this smarter. 2921 * For instance, if the completion latencies are tight, we can 2922 * get closer than just half the mean. This is especially 2923 * important on devices where the completion latencies are longer 2924 * than ~10 usec. We do use the stats for the relevant IO size 2925 * if available which does lead to better estimates. 2926 */ 2927 bucket = blk_mq_poll_stats_bkt(rq); 2928 if (bucket < 0) 2929 return ret; 2930 2931 if (q->poll_stat[bucket].nr_samples) 2932 ret = (q->poll_stat[bucket].mean + 1) / 2; 2933 2934 return ret; 2935 } 2936 2937 static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, 2938 struct blk_mq_hw_ctx *hctx, 2939 struct request *rq) 2940 { 2941 struct hrtimer_sleeper hs; 2942 enum hrtimer_mode mode; 2943 unsigned int nsecs; 2944 ktime_t kt; 2945 2946 if (rq->rq_flags & RQF_MQ_POLL_SLEPT) 2947 return false; 2948 2949 /* 2950 * poll_nsec can be: 2951 * 2952 * -1: don't ever hybrid sleep 2953 * 0: use half of prev avg 2954 * >0: use this specific value 2955 */ 2956 if (q->poll_nsec == -1) 2957 return false; 2958 else if (q->poll_nsec > 0) 2959 nsecs = q->poll_nsec; 2960 else 2961 nsecs = blk_mq_poll_nsecs(q, hctx, rq); 2962 2963 if (!nsecs) 2964 return false; 2965 2966 rq->rq_flags |= RQF_MQ_POLL_SLEPT; 2967 2968 /* 2969 * This will be replaced with the stats tracking code, using 2970 * 'avg_completion_time / 2' as the pre-sleep target. 2971 */ 2972 kt = nsecs; 2973 2974 mode = HRTIMER_MODE_REL; 2975 hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode); 2976 hrtimer_set_expires(&hs.timer, kt); 2977 2978 hrtimer_init_sleeper(&hs, current); 2979 do { 2980 if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE) 2981 break; 2982 set_current_state(TASK_UNINTERRUPTIBLE); 2983 hrtimer_start_expires(&hs.timer, mode); 2984 if (hs.task) 2985 io_schedule(); 2986 hrtimer_cancel(&hs.timer); 2987 mode = HRTIMER_MODE_ABS; 2988 } while (hs.task && !signal_pending(current)); 2989 2990 __set_current_state(TASK_RUNNING); 2991 destroy_hrtimer_on_stack(&hs.timer); 2992 return true; 2993 } 2994 2995 static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq) 2996 { 2997 struct request_queue *q = hctx->queue; 2998 long state; 2999 3000 /* 3001 * If we sleep, have the caller restart the poll loop to reset 3002 * the state. Like for the other success return cases, the 3003 * caller is responsible for checking if the IO completed. If 3004 * the IO isn't complete, we'll get called again and will go 3005 * straight to the busy poll loop. 3006 */ 3007 if (blk_mq_poll_hybrid_sleep(q, hctx, rq)) 3008 return true; 3009 3010 hctx->poll_considered++; 3011 3012 state = current->state; 3013 while (!need_resched()) { 3014 int ret; 3015 3016 hctx->poll_invoked++; 3017 3018 ret = q->mq_ops->poll(hctx, rq->tag); 3019 if (ret > 0) { 3020 hctx->poll_success++; 3021 set_current_state(TASK_RUNNING); 3022 return true; 3023 } 3024 3025 if (signal_pending_state(state, current)) 3026 set_current_state(TASK_RUNNING); 3027 3028 if (current->state == TASK_RUNNING) 3029 return true; 3030 if (ret < 0) 3031 break; 3032 cpu_relax(); 3033 } 3034 3035 __set_current_state(TASK_RUNNING); 3036 return false; 3037 } 3038 3039 static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie) 3040 { 3041 struct blk_mq_hw_ctx *hctx; 3042 struct request *rq; 3043 3044 if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 3045 return false; 3046 3047 hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; 3048 if (!blk_qc_t_is_internal(cookie)) 3049 rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); 3050 else { 3051 rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie)); 3052 /* 3053 * With scheduling, if the request has completed, we'll 3054 * get a NULL return here, as we clear the sched tag when 3055 * that happens. The request still remains valid, like always, 3056 * so we should be safe with just the NULL check. 3057 */ 3058 if (!rq) 3059 return false; 3060 } 3061 3062 return __blk_mq_poll(hctx, rq); 3063 } 3064 3065 static int __init blk_mq_init(void) 3066 { 3067 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, 3068 blk_mq_hctx_notify_dead); 3069 return 0; 3070 } 3071 subsys_initcall(blk_mq_init); 3072