1 /* 2 * Block multiqueue core code 3 * 4 * Copyright (C) 2013-2014 Jens Axboe 5 * Copyright (C) 2013-2014 Christoph Hellwig 6 */ 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/backing-dev.h> 10 #include <linux/bio.h> 11 #include <linux/blkdev.h> 12 #include <linux/kmemleak.h> 13 #include <linux/mm.h> 14 #include <linux/init.h> 15 #include <linux/slab.h> 16 #include <linux/workqueue.h> 17 #include <linux/smp.h> 18 #include <linux/llist.h> 19 #include <linux/list_sort.h> 20 #include <linux/cpu.h> 21 #include <linux/cache.h> 22 #include <linux/sched/sysctl.h> 23 #include <linux/sched/topology.h> 24 #include <linux/sched/signal.h> 25 #include <linux/delay.h> 26 #include <linux/crash_dump.h> 27 #include <linux/prefetch.h> 28 29 #include <trace/events/block.h> 30 31 #include <linux/blk-mq.h> 32 #include "blk.h" 33 #include "blk-mq.h" 34 #include "blk-mq-debugfs.h" 35 #include "blk-mq-tag.h" 36 #include "blk-stat.h" 37 #include "blk-wbt.h" 38 #include "blk-mq-sched.h" 39 40 static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie); 41 static void blk_mq_poll_stats_start(struct request_queue *q); 42 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); 43 44 static int blk_mq_poll_stats_bkt(const struct request *rq) 45 { 46 int ddir, bytes, bucket; 47 48 ddir = rq_data_dir(rq); 49 bytes = blk_rq_bytes(rq); 50 51 bucket = ddir + 2*(ilog2(bytes) - 9); 52 53 if (bucket < 0) 54 return -1; 55 else if (bucket >= BLK_MQ_POLL_STATS_BKTS) 56 return ddir + BLK_MQ_POLL_STATS_BKTS - 2; 57 58 return bucket; 59 } 60 61 /* 62 * Check if any of the ctx's have pending work in this hardware queue 63 */ 64 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) 65 { 66 return !list_empty_careful(&hctx->dispatch) || 67 sbitmap_any_bit_set(&hctx->ctx_map) || 68 blk_mq_sched_has_work(hctx); 69 } 70 71 /* 72 * Mark this ctx as having pending work in this hardware queue 73 */ 74 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, 75 struct blk_mq_ctx *ctx) 76 { 77 if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw)) 78 sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw); 79 } 80 81 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, 82 struct blk_mq_ctx *ctx) 83 { 84 sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw); 85 } 86 87 struct mq_inflight { 88 struct hd_struct *part; 89 unsigned int *inflight; 90 }; 91 92 static void blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx, 93 struct request *rq, void *priv, 94 bool reserved) 95 { 96 struct mq_inflight *mi = priv; 97 98 /* 99 * index[0] counts the specific partition that was asked for. index[1] 100 * counts the ones that are active on the whole device, so increment 101 * that if mi->part is indeed a partition, and not a whole device. 102 */ 103 if (rq->part == mi->part) 104 mi->inflight[0]++; 105 if (mi->part->partno) 106 mi->inflight[1]++; 107 } 108 109 void blk_mq_in_flight(struct request_queue *q, struct hd_struct *part, 110 unsigned int inflight[2]) 111 { 112 struct mq_inflight mi = { .part = part, .inflight = inflight, }; 113 114 inflight[0] = inflight[1] = 0; 115 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); 116 } 117 118 static void blk_mq_check_inflight_rw(struct blk_mq_hw_ctx *hctx, 119 struct request *rq, void *priv, 120 bool reserved) 121 { 122 struct mq_inflight *mi = priv; 123 124 if (rq->part == mi->part) 125 mi->inflight[rq_data_dir(rq)]++; 126 } 127 128 void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part, 129 unsigned int inflight[2]) 130 { 131 struct mq_inflight mi = { .part = part, .inflight = inflight, }; 132 133 inflight[0] = inflight[1] = 0; 134 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight_rw, &mi); 135 } 136 137 void blk_freeze_queue_start(struct request_queue *q) 138 { 139 int freeze_depth; 140 141 freeze_depth = atomic_inc_return(&q->mq_freeze_depth); 142 if (freeze_depth == 1) { 143 percpu_ref_kill(&q->q_usage_counter); 144 if (q->mq_ops) 145 blk_mq_run_hw_queues(q, false); 146 } 147 } 148 EXPORT_SYMBOL_GPL(blk_freeze_queue_start); 149 150 void blk_mq_freeze_queue_wait(struct request_queue *q) 151 { 152 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter)); 153 } 154 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait); 155 156 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, 157 unsigned long timeout) 158 { 159 return wait_event_timeout(q->mq_freeze_wq, 160 percpu_ref_is_zero(&q->q_usage_counter), 161 timeout); 162 } 163 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout); 164 165 /* 166 * Guarantee no request is in use, so we can change any data structure of 167 * the queue afterward. 168 */ 169 void blk_freeze_queue(struct request_queue *q) 170 { 171 /* 172 * In the !blk_mq case we are only calling this to kill the 173 * q_usage_counter, otherwise this increases the freeze depth 174 * and waits for it to return to zero. For this reason there is 175 * no blk_unfreeze_queue(), and blk_freeze_queue() is not 176 * exported to drivers as the only user for unfreeze is blk_mq. 177 */ 178 blk_freeze_queue_start(q); 179 if (!q->mq_ops) 180 blk_drain_queue(q); 181 blk_mq_freeze_queue_wait(q); 182 } 183 184 void blk_mq_freeze_queue(struct request_queue *q) 185 { 186 /* 187 * ...just an alias to keep freeze and unfreeze actions balanced 188 * in the blk_mq_* namespace 189 */ 190 blk_freeze_queue(q); 191 } 192 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue); 193 194 void blk_mq_unfreeze_queue(struct request_queue *q) 195 { 196 int freeze_depth; 197 198 freeze_depth = atomic_dec_return(&q->mq_freeze_depth); 199 WARN_ON_ONCE(freeze_depth < 0); 200 if (!freeze_depth) { 201 percpu_ref_reinit(&q->q_usage_counter); 202 wake_up_all(&q->mq_freeze_wq); 203 } 204 } 205 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); 206 207 /* 208 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the 209 * mpt3sas driver such that this function can be removed. 210 */ 211 void blk_mq_quiesce_queue_nowait(struct request_queue *q) 212 { 213 blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q); 214 } 215 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); 216 217 /** 218 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished 219 * @q: request queue. 220 * 221 * Note: this function does not prevent that the struct request end_io() 222 * callback function is invoked. Once this function is returned, we make 223 * sure no dispatch can happen until the queue is unquiesced via 224 * blk_mq_unquiesce_queue(). 225 */ 226 void blk_mq_quiesce_queue(struct request_queue *q) 227 { 228 struct blk_mq_hw_ctx *hctx; 229 unsigned int i; 230 bool rcu = false; 231 232 blk_mq_quiesce_queue_nowait(q); 233 234 queue_for_each_hw_ctx(q, hctx, i) { 235 if (hctx->flags & BLK_MQ_F_BLOCKING) 236 synchronize_srcu(hctx->srcu); 237 else 238 rcu = true; 239 } 240 if (rcu) 241 synchronize_rcu(); 242 } 243 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue); 244 245 /* 246 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue() 247 * @q: request queue. 248 * 249 * This function recovers queue into the state before quiescing 250 * which is done by blk_mq_quiesce_queue. 251 */ 252 void blk_mq_unquiesce_queue(struct request_queue *q) 253 { 254 blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q); 255 256 /* dispatch requests which are inserted during quiescing */ 257 blk_mq_run_hw_queues(q, true); 258 } 259 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue); 260 261 void blk_mq_wake_waiters(struct request_queue *q) 262 { 263 struct blk_mq_hw_ctx *hctx; 264 unsigned int i; 265 266 queue_for_each_hw_ctx(q, hctx, i) 267 if (blk_mq_hw_queue_mapped(hctx)) 268 blk_mq_tag_wakeup_all(hctx->tags, true); 269 } 270 271 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) 272 { 273 return blk_mq_has_free_tags(hctx->tags); 274 } 275 EXPORT_SYMBOL(blk_mq_can_queue); 276 277 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, 278 unsigned int tag, unsigned int op) 279 { 280 struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 281 struct request *rq = tags->static_rqs[tag]; 282 req_flags_t rq_flags = 0; 283 284 if (data->flags & BLK_MQ_REQ_INTERNAL) { 285 rq->tag = -1; 286 rq->internal_tag = tag; 287 } else { 288 if (blk_mq_tag_busy(data->hctx)) { 289 rq_flags = RQF_MQ_INFLIGHT; 290 atomic_inc(&data->hctx->nr_active); 291 } 292 rq->tag = tag; 293 rq->internal_tag = -1; 294 data->hctx->tags->rqs[rq->tag] = rq; 295 } 296 297 /* csd/requeue_work/fifo_time is initialized before use */ 298 rq->q = data->q; 299 rq->mq_ctx = data->ctx; 300 rq->rq_flags = rq_flags; 301 rq->cpu = -1; 302 rq->cmd_flags = op; 303 if (data->flags & BLK_MQ_REQ_PREEMPT) 304 rq->rq_flags |= RQF_PREEMPT; 305 if (blk_queue_io_stat(data->q)) 306 rq->rq_flags |= RQF_IO_STAT; 307 INIT_LIST_HEAD(&rq->queuelist); 308 INIT_HLIST_NODE(&rq->hash); 309 RB_CLEAR_NODE(&rq->rb_node); 310 rq->rq_disk = NULL; 311 rq->part = NULL; 312 rq->start_time_ns = ktime_get_ns(); 313 rq->io_start_time_ns = 0; 314 rq->nr_phys_segments = 0; 315 #if defined(CONFIG_BLK_DEV_INTEGRITY) 316 rq->nr_integrity_segments = 0; 317 #endif 318 rq->special = NULL; 319 /* tag was already set */ 320 rq->extra_len = 0; 321 rq->__deadline = 0; 322 323 INIT_LIST_HEAD(&rq->timeout_list); 324 rq->timeout = 0; 325 326 rq->end_io = NULL; 327 rq->end_io_data = NULL; 328 rq->next_rq = NULL; 329 330 #ifdef CONFIG_BLK_CGROUP 331 rq->rl = NULL; 332 #endif 333 334 data->ctx->rq_dispatched[op_is_sync(op)]++; 335 refcount_set(&rq->ref, 1); 336 return rq; 337 } 338 339 static struct request *blk_mq_get_request(struct request_queue *q, 340 struct bio *bio, unsigned int op, 341 struct blk_mq_alloc_data *data) 342 { 343 struct elevator_queue *e = q->elevator; 344 struct request *rq; 345 unsigned int tag; 346 bool put_ctx_on_error = false; 347 348 blk_queue_enter_live(q); 349 data->q = q; 350 if (likely(!data->ctx)) { 351 data->ctx = blk_mq_get_ctx(q); 352 put_ctx_on_error = true; 353 } 354 if (likely(!data->hctx)) 355 data->hctx = blk_mq_map_queue(q, data->ctx->cpu); 356 if (op & REQ_NOWAIT) 357 data->flags |= BLK_MQ_REQ_NOWAIT; 358 359 if (e) { 360 data->flags |= BLK_MQ_REQ_INTERNAL; 361 362 /* 363 * Flush requests are special and go directly to the 364 * dispatch list. Don't include reserved tags in the 365 * limiting, as it isn't useful. 366 */ 367 if (!op_is_flush(op) && e->type->ops.mq.limit_depth && 368 !(data->flags & BLK_MQ_REQ_RESERVED)) 369 e->type->ops.mq.limit_depth(op, data); 370 } 371 372 tag = blk_mq_get_tag(data); 373 if (tag == BLK_MQ_TAG_FAIL) { 374 if (put_ctx_on_error) { 375 blk_mq_put_ctx(data->ctx); 376 data->ctx = NULL; 377 } 378 blk_queue_exit(q); 379 return NULL; 380 } 381 382 rq = blk_mq_rq_ctx_init(data, tag, op); 383 if (!op_is_flush(op)) { 384 rq->elv.icq = NULL; 385 if (e && e->type->ops.mq.prepare_request) { 386 if (e->type->icq_cache && rq_ioc(bio)) 387 blk_mq_sched_assign_ioc(rq, bio); 388 389 e->type->ops.mq.prepare_request(rq, bio); 390 rq->rq_flags |= RQF_ELVPRIV; 391 } 392 } 393 data->hctx->queued++; 394 return rq; 395 } 396 397 struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, 398 blk_mq_req_flags_t flags) 399 { 400 struct blk_mq_alloc_data alloc_data = { .flags = flags }; 401 struct request *rq; 402 int ret; 403 404 ret = blk_queue_enter(q, flags); 405 if (ret) 406 return ERR_PTR(ret); 407 408 rq = blk_mq_get_request(q, NULL, op, &alloc_data); 409 blk_queue_exit(q); 410 411 if (!rq) 412 return ERR_PTR(-EWOULDBLOCK); 413 414 blk_mq_put_ctx(alloc_data.ctx); 415 416 rq->__data_len = 0; 417 rq->__sector = (sector_t) -1; 418 rq->bio = rq->biotail = NULL; 419 return rq; 420 } 421 EXPORT_SYMBOL(blk_mq_alloc_request); 422 423 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, 424 unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx) 425 { 426 struct blk_mq_alloc_data alloc_data = { .flags = flags }; 427 struct request *rq; 428 unsigned int cpu; 429 int ret; 430 431 /* 432 * If the tag allocator sleeps we could get an allocation for a 433 * different hardware context. No need to complicate the low level 434 * allocator for this for the rare use case of a command tied to 435 * a specific queue. 436 */ 437 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT))) 438 return ERR_PTR(-EINVAL); 439 440 if (hctx_idx >= q->nr_hw_queues) 441 return ERR_PTR(-EIO); 442 443 ret = blk_queue_enter(q, flags); 444 if (ret) 445 return ERR_PTR(ret); 446 447 /* 448 * Check if the hardware context is actually mapped to anything. 449 * If not tell the caller that it should skip this queue. 450 */ 451 alloc_data.hctx = q->queue_hw_ctx[hctx_idx]; 452 if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) { 453 blk_queue_exit(q); 454 return ERR_PTR(-EXDEV); 455 } 456 cpu = cpumask_first_and(alloc_data.hctx->cpumask, cpu_online_mask); 457 alloc_data.ctx = __blk_mq_get_ctx(q, cpu); 458 459 rq = blk_mq_get_request(q, NULL, op, &alloc_data); 460 blk_queue_exit(q); 461 462 if (!rq) 463 return ERR_PTR(-EWOULDBLOCK); 464 465 return rq; 466 } 467 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); 468 469 static void __blk_mq_free_request(struct request *rq) 470 { 471 struct request_queue *q = rq->q; 472 struct blk_mq_ctx *ctx = rq->mq_ctx; 473 struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu); 474 const int sched_tag = rq->internal_tag; 475 476 if (rq->tag != -1) 477 blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag); 478 if (sched_tag != -1) 479 blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag); 480 blk_mq_sched_restart(hctx); 481 blk_queue_exit(q); 482 } 483 484 void blk_mq_free_request(struct request *rq) 485 { 486 struct request_queue *q = rq->q; 487 struct elevator_queue *e = q->elevator; 488 struct blk_mq_ctx *ctx = rq->mq_ctx; 489 struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu); 490 491 if (rq->rq_flags & RQF_ELVPRIV) { 492 if (e && e->type->ops.mq.finish_request) 493 e->type->ops.mq.finish_request(rq); 494 if (rq->elv.icq) { 495 put_io_context(rq->elv.icq->ioc); 496 rq->elv.icq = NULL; 497 } 498 } 499 500 ctx->rq_completed[rq_is_sync(rq)]++; 501 if (rq->rq_flags & RQF_MQ_INFLIGHT) 502 atomic_dec(&hctx->nr_active); 503 504 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq))) 505 laptop_io_completion(q->backing_dev_info); 506 507 wbt_done(q->rq_wb, rq); 508 509 if (blk_rq_rl(rq)) 510 blk_put_rl(blk_rq_rl(rq)); 511 512 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 513 if (refcount_dec_and_test(&rq->ref)) 514 __blk_mq_free_request(rq); 515 } 516 EXPORT_SYMBOL_GPL(blk_mq_free_request); 517 518 inline void __blk_mq_end_request(struct request *rq, blk_status_t error) 519 { 520 u64 now = ktime_get_ns(); 521 522 if (rq->rq_flags & RQF_STATS) { 523 blk_mq_poll_stats_start(rq->q); 524 blk_stat_add(rq, now); 525 } 526 527 blk_account_io_done(rq, now); 528 529 if (rq->end_io) { 530 wbt_done(rq->q->rq_wb, rq); 531 rq->end_io(rq, error); 532 } else { 533 if (unlikely(blk_bidi_rq(rq))) 534 blk_mq_free_request(rq->next_rq); 535 blk_mq_free_request(rq); 536 } 537 } 538 EXPORT_SYMBOL(__blk_mq_end_request); 539 540 void blk_mq_end_request(struct request *rq, blk_status_t error) 541 { 542 if (blk_update_request(rq, error, blk_rq_bytes(rq))) 543 BUG(); 544 __blk_mq_end_request(rq, error); 545 } 546 EXPORT_SYMBOL(blk_mq_end_request); 547 548 static void __blk_mq_complete_request_remote(void *data) 549 { 550 struct request *rq = data; 551 552 rq->q->softirq_done_fn(rq); 553 } 554 555 static void __blk_mq_complete_request(struct request *rq) 556 { 557 struct blk_mq_ctx *ctx = rq->mq_ctx; 558 bool shared = false; 559 int cpu; 560 561 if (cmpxchg(&rq->state, MQ_RQ_IN_FLIGHT, MQ_RQ_COMPLETE) != 562 MQ_RQ_IN_FLIGHT) 563 return; 564 565 if (rq->internal_tag != -1) 566 blk_mq_sched_completed_request(rq); 567 568 if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) { 569 rq->q->softirq_done_fn(rq); 570 return; 571 } 572 573 cpu = get_cpu(); 574 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags)) 575 shared = cpus_share_cache(cpu, ctx->cpu); 576 577 if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { 578 rq->csd.func = __blk_mq_complete_request_remote; 579 rq->csd.info = rq; 580 rq->csd.flags = 0; 581 smp_call_function_single_async(ctx->cpu, &rq->csd); 582 } else { 583 rq->q->softirq_done_fn(rq); 584 } 585 put_cpu(); 586 } 587 588 static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx) 589 __releases(hctx->srcu) 590 { 591 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) 592 rcu_read_unlock(); 593 else 594 srcu_read_unlock(hctx->srcu, srcu_idx); 595 } 596 597 static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx) 598 __acquires(hctx->srcu) 599 { 600 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { 601 /* shut up gcc false positive */ 602 *srcu_idx = 0; 603 rcu_read_lock(); 604 } else 605 *srcu_idx = srcu_read_lock(hctx->srcu); 606 } 607 608 /** 609 * blk_mq_complete_request - end I/O on a request 610 * @rq: the request being processed 611 * 612 * Description: 613 * Ends all I/O on a request. It does not handle partial completions. 614 * The actual completion happens out-of-order, through a IPI handler. 615 **/ 616 void blk_mq_complete_request(struct request *rq) 617 { 618 if (unlikely(blk_should_fake_timeout(rq->q))) 619 return; 620 __blk_mq_complete_request(rq); 621 } 622 EXPORT_SYMBOL(blk_mq_complete_request); 623 624 int blk_mq_request_started(struct request *rq) 625 { 626 return blk_mq_rq_state(rq) != MQ_RQ_IDLE; 627 } 628 EXPORT_SYMBOL_GPL(blk_mq_request_started); 629 630 void blk_mq_start_request(struct request *rq) 631 { 632 struct request_queue *q = rq->q; 633 634 blk_mq_sched_started_request(rq); 635 636 trace_block_rq_issue(q, rq); 637 638 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 639 rq->io_start_time_ns = ktime_get_ns(); 640 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW 641 rq->throtl_size = blk_rq_sectors(rq); 642 #endif 643 rq->rq_flags |= RQF_STATS; 644 wbt_issue(q->rq_wb, rq); 645 } 646 647 WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE); 648 649 blk_add_timer(rq); 650 WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT); 651 652 if (q->dma_drain_size && blk_rq_bytes(rq)) { 653 /* 654 * Make sure space for the drain appears. We know we can do 655 * this because max_hw_segments has been adjusted to be one 656 * fewer than the device can handle. 657 */ 658 rq->nr_phys_segments++; 659 } 660 } 661 EXPORT_SYMBOL(blk_mq_start_request); 662 663 static void __blk_mq_requeue_request(struct request *rq) 664 { 665 struct request_queue *q = rq->q; 666 667 blk_mq_put_driver_tag(rq); 668 669 trace_block_rq_requeue(q, rq); 670 wbt_requeue(q->rq_wb, rq); 671 672 if (blk_mq_request_started(rq)) { 673 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 674 rq->rq_flags &= ~RQF_TIMED_OUT; 675 if (q->dma_drain_size && blk_rq_bytes(rq)) 676 rq->nr_phys_segments--; 677 } 678 } 679 680 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) 681 { 682 __blk_mq_requeue_request(rq); 683 684 /* this request will be re-inserted to io scheduler queue */ 685 blk_mq_sched_requeue_request(rq); 686 687 BUG_ON(blk_queued_rq(rq)); 688 blk_mq_add_to_requeue_list(rq, true, kick_requeue_list); 689 } 690 EXPORT_SYMBOL(blk_mq_requeue_request); 691 692 static void blk_mq_requeue_work(struct work_struct *work) 693 { 694 struct request_queue *q = 695 container_of(work, struct request_queue, requeue_work.work); 696 LIST_HEAD(rq_list); 697 struct request *rq, *next; 698 699 spin_lock_irq(&q->requeue_lock); 700 list_splice_init(&q->requeue_list, &rq_list); 701 spin_unlock_irq(&q->requeue_lock); 702 703 list_for_each_entry_safe(rq, next, &rq_list, queuelist) { 704 if (!(rq->rq_flags & RQF_SOFTBARRIER)) 705 continue; 706 707 rq->rq_flags &= ~RQF_SOFTBARRIER; 708 list_del_init(&rq->queuelist); 709 blk_mq_sched_insert_request(rq, true, false, false); 710 } 711 712 while (!list_empty(&rq_list)) { 713 rq = list_entry(rq_list.next, struct request, queuelist); 714 list_del_init(&rq->queuelist); 715 blk_mq_sched_insert_request(rq, false, false, false); 716 } 717 718 blk_mq_run_hw_queues(q, false); 719 } 720 721 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, 722 bool kick_requeue_list) 723 { 724 struct request_queue *q = rq->q; 725 unsigned long flags; 726 727 /* 728 * We abuse this flag that is otherwise used by the I/O scheduler to 729 * request head insertion from the workqueue. 730 */ 731 BUG_ON(rq->rq_flags & RQF_SOFTBARRIER); 732 733 spin_lock_irqsave(&q->requeue_lock, flags); 734 if (at_head) { 735 rq->rq_flags |= RQF_SOFTBARRIER; 736 list_add(&rq->queuelist, &q->requeue_list); 737 } else { 738 list_add_tail(&rq->queuelist, &q->requeue_list); 739 } 740 spin_unlock_irqrestore(&q->requeue_lock, flags); 741 742 if (kick_requeue_list) 743 blk_mq_kick_requeue_list(q); 744 } 745 EXPORT_SYMBOL(blk_mq_add_to_requeue_list); 746 747 void blk_mq_kick_requeue_list(struct request_queue *q) 748 { 749 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0); 750 } 751 EXPORT_SYMBOL(blk_mq_kick_requeue_list); 752 753 void blk_mq_delay_kick_requeue_list(struct request_queue *q, 754 unsigned long msecs) 755 { 756 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 757 msecs_to_jiffies(msecs)); 758 } 759 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); 760 761 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) 762 { 763 if (tag < tags->nr_tags) { 764 prefetch(tags->rqs[tag]); 765 return tags->rqs[tag]; 766 } 767 768 return NULL; 769 } 770 EXPORT_SYMBOL(blk_mq_tag_to_rq); 771 772 static void blk_mq_rq_timed_out(struct request *req, bool reserved) 773 { 774 req->rq_flags |= RQF_TIMED_OUT; 775 if (req->q->mq_ops->timeout) { 776 enum blk_eh_timer_return ret; 777 778 ret = req->q->mq_ops->timeout(req, reserved); 779 if (ret == BLK_EH_DONE) 780 return; 781 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER); 782 } 783 784 req->rq_flags &= ~RQF_TIMED_OUT; 785 blk_add_timer(req); 786 } 787 788 static bool blk_mq_req_expired(struct request *rq, unsigned long *next) 789 { 790 unsigned long deadline; 791 792 if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT) 793 return false; 794 if (rq->rq_flags & RQF_TIMED_OUT) 795 return false; 796 797 deadline = blk_rq_deadline(rq); 798 if (time_after_eq(jiffies, deadline)) 799 return true; 800 801 if (*next == 0) 802 *next = deadline; 803 else if (time_after(*next, deadline)) 804 *next = deadline; 805 return false; 806 } 807 808 static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, 809 struct request *rq, void *priv, bool reserved) 810 { 811 unsigned long *next = priv; 812 813 /* 814 * Just do a quick check if it is expired before locking the request in 815 * so we're not unnecessarilly synchronizing across CPUs. 816 */ 817 if (!blk_mq_req_expired(rq, next)) 818 return; 819 820 /* 821 * We have reason to believe the request may be expired. Take a 822 * reference on the request to lock this request lifetime into its 823 * currently allocated context to prevent it from being reallocated in 824 * the event the completion by-passes this timeout handler. 825 * 826 * If the reference was already released, then the driver beat the 827 * timeout handler to posting a natural completion. 828 */ 829 if (!refcount_inc_not_zero(&rq->ref)) 830 return; 831 832 /* 833 * The request is now locked and cannot be reallocated underneath the 834 * timeout handler's processing. Re-verify this exact request is truly 835 * expired; if it is not expired, then the request was completed and 836 * reallocated as a new request. 837 */ 838 if (blk_mq_req_expired(rq, next)) 839 blk_mq_rq_timed_out(rq, reserved); 840 if (refcount_dec_and_test(&rq->ref)) 841 __blk_mq_free_request(rq); 842 } 843 844 static void blk_mq_timeout_work(struct work_struct *work) 845 { 846 struct request_queue *q = 847 container_of(work, struct request_queue, timeout_work); 848 unsigned long next = 0; 849 struct blk_mq_hw_ctx *hctx; 850 int i; 851 852 /* A deadlock might occur if a request is stuck requiring a 853 * timeout at the same time a queue freeze is waiting 854 * completion, since the timeout code would not be able to 855 * acquire the queue reference here. 856 * 857 * That's why we don't use blk_queue_enter here; instead, we use 858 * percpu_ref_tryget directly, because we need to be able to 859 * obtain a reference even in the short window between the queue 860 * starting to freeze, by dropping the first reference in 861 * blk_freeze_queue_start, and the moment the last request is 862 * consumed, marked by the instant q_usage_counter reaches 863 * zero. 864 */ 865 if (!percpu_ref_tryget(&q->q_usage_counter)) 866 return; 867 868 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next); 869 870 if (next != 0) { 871 mod_timer(&q->timeout, next); 872 } else { 873 /* 874 * Request timeouts are handled as a forward rolling timer. If 875 * we end up here it means that no requests are pending and 876 * also that no request has been pending for a while. Mark 877 * each hctx as idle. 878 */ 879 queue_for_each_hw_ctx(q, hctx, i) { 880 /* the hctx may be unmapped, so check it here */ 881 if (blk_mq_hw_queue_mapped(hctx)) 882 blk_mq_tag_idle(hctx); 883 } 884 } 885 blk_queue_exit(q); 886 } 887 888 struct flush_busy_ctx_data { 889 struct blk_mq_hw_ctx *hctx; 890 struct list_head *list; 891 }; 892 893 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) 894 { 895 struct flush_busy_ctx_data *flush_data = data; 896 struct blk_mq_hw_ctx *hctx = flush_data->hctx; 897 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 898 899 spin_lock(&ctx->lock); 900 list_splice_tail_init(&ctx->rq_list, flush_data->list); 901 sbitmap_clear_bit(sb, bitnr); 902 spin_unlock(&ctx->lock); 903 return true; 904 } 905 906 /* 907 * Process software queues that have been marked busy, splicing them 908 * to the for-dispatch 909 */ 910 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) 911 { 912 struct flush_busy_ctx_data data = { 913 .hctx = hctx, 914 .list = list, 915 }; 916 917 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data); 918 } 919 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs); 920 921 struct dispatch_rq_data { 922 struct blk_mq_hw_ctx *hctx; 923 struct request *rq; 924 }; 925 926 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr, 927 void *data) 928 { 929 struct dispatch_rq_data *dispatch_data = data; 930 struct blk_mq_hw_ctx *hctx = dispatch_data->hctx; 931 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 932 933 spin_lock(&ctx->lock); 934 if (!list_empty(&ctx->rq_list)) { 935 dispatch_data->rq = list_entry_rq(ctx->rq_list.next); 936 list_del_init(&dispatch_data->rq->queuelist); 937 if (list_empty(&ctx->rq_list)) 938 sbitmap_clear_bit(sb, bitnr); 939 } 940 spin_unlock(&ctx->lock); 941 942 return !dispatch_data->rq; 943 } 944 945 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, 946 struct blk_mq_ctx *start) 947 { 948 unsigned off = start ? start->index_hw : 0; 949 struct dispatch_rq_data data = { 950 .hctx = hctx, 951 .rq = NULL, 952 }; 953 954 __sbitmap_for_each_set(&hctx->ctx_map, off, 955 dispatch_rq_from_ctx, &data); 956 957 return data.rq; 958 } 959 960 static inline unsigned int queued_to_index(unsigned int queued) 961 { 962 if (!queued) 963 return 0; 964 965 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); 966 } 967 968 bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx, 969 bool wait) 970 { 971 struct blk_mq_alloc_data data = { 972 .q = rq->q, 973 .hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), 974 .flags = wait ? 0 : BLK_MQ_REQ_NOWAIT, 975 }; 976 977 might_sleep_if(wait); 978 979 if (rq->tag != -1) 980 goto done; 981 982 if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag)) 983 data.flags |= BLK_MQ_REQ_RESERVED; 984 985 rq->tag = blk_mq_get_tag(&data); 986 if (rq->tag >= 0) { 987 if (blk_mq_tag_busy(data.hctx)) { 988 rq->rq_flags |= RQF_MQ_INFLIGHT; 989 atomic_inc(&data.hctx->nr_active); 990 } 991 data.hctx->tags->rqs[rq->tag] = rq; 992 } 993 994 done: 995 if (hctx) 996 *hctx = data.hctx; 997 return rq->tag != -1; 998 } 999 1000 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, 1001 int flags, void *key) 1002 { 1003 struct blk_mq_hw_ctx *hctx; 1004 1005 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); 1006 1007 list_del_init(&wait->entry); 1008 blk_mq_run_hw_queue(hctx, true); 1009 return 1; 1010 } 1011 1012 /* 1013 * Mark us waiting for a tag. For shared tags, this involves hooking us into 1014 * the tag wakeups. For non-shared tags, we can simply mark us needing a 1015 * restart. For both cases, take care to check the condition again after 1016 * marking us as waiting. 1017 */ 1018 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx **hctx, 1019 struct request *rq) 1020 { 1021 struct blk_mq_hw_ctx *this_hctx = *hctx; 1022 struct sbq_wait_state *ws; 1023 wait_queue_entry_t *wait; 1024 bool ret; 1025 1026 if (!(this_hctx->flags & BLK_MQ_F_TAG_SHARED)) { 1027 if (!test_bit(BLK_MQ_S_SCHED_RESTART, &this_hctx->state)) 1028 set_bit(BLK_MQ_S_SCHED_RESTART, &this_hctx->state); 1029 1030 /* 1031 * It's possible that a tag was freed in the window between the 1032 * allocation failure and adding the hardware queue to the wait 1033 * queue. 1034 * 1035 * Don't clear RESTART here, someone else could have set it. 1036 * At most this will cost an extra queue run. 1037 */ 1038 return blk_mq_get_driver_tag(rq, hctx, false); 1039 } 1040 1041 wait = &this_hctx->dispatch_wait; 1042 if (!list_empty_careful(&wait->entry)) 1043 return false; 1044 1045 spin_lock(&this_hctx->lock); 1046 if (!list_empty(&wait->entry)) { 1047 spin_unlock(&this_hctx->lock); 1048 return false; 1049 } 1050 1051 ws = bt_wait_ptr(&this_hctx->tags->bitmap_tags, this_hctx); 1052 add_wait_queue(&ws->wait, wait); 1053 1054 /* 1055 * It's possible that a tag was freed in the window between the 1056 * allocation failure and adding the hardware queue to the wait 1057 * queue. 1058 */ 1059 ret = blk_mq_get_driver_tag(rq, hctx, false); 1060 if (!ret) { 1061 spin_unlock(&this_hctx->lock); 1062 return false; 1063 } 1064 1065 /* 1066 * We got a tag, remove ourselves from the wait queue to ensure 1067 * someone else gets the wakeup. 1068 */ 1069 spin_lock_irq(&ws->wait.lock); 1070 list_del_init(&wait->entry); 1071 spin_unlock_irq(&ws->wait.lock); 1072 spin_unlock(&this_hctx->lock); 1073 1074 return true; 1075 } 1076 1077 #define BLK_MQ_RESOURCE_DELAY 3 /* ms units */ 1078 1079 bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, 1080 bool got_budget) 1081 { 1082 struct blk_mq_hw_ctx *hctx; 1083 struct request *rq, *nxt; 1084 bool no_tag = false; 1085 int errors, queued; 1086 blk_status_t ret = BLK_STS_OK; 1087 1088 if (list_empty(list)) 1089 return false; 1090 1091 WARN_ON(!list_is_singular(list) && got_budget); 1092 1093 /* 1094 * Now process all the entries, sending them to the driver. 1095 */ 1096 errors = queued = 0; 1097 do { 1098 struct blk_mq_queue_data bd; 1099 1100 rq = list_first_entry(list, struct request, queuelist); 1101 1102 hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); 1103 if (!got_budget && !blk_mq_get_dispatch_budget(hctx)) 1104 break; 1105 1106 if (!blk_mq_get_driver_tag(rq, NULL, false)) { 1107 /* 1108 * The initial allocation attempt failed, so we need to 1109 * rerun the hardware queue when a tag is freed. The 1110 * waitqueue takes care of that. If the queue is run 1111 * before we add this entry back on the dispatch list, 1112 * we'll re-run it below. 1113 */ 1114 if (!blk_mq_mark_tag_wait(&hctx, rq)) { 1115 blk_mq_put_dispatch_budget(hctx); 1116 /* 1117 * For non-shared tags, the RESTART check 1118 * will suffice. 1119 */ 1120 if (hctx->flags & BLK_MQ_F_TAG_SHARED) 1121 no_tag = true; 1122 break; 1123 } 1124 } 1125 1126 list_del_init(&rq->queuelist); 1127 1128 bd.rq = rq; 1129 1130 /* 1131 * Flag last if we have no more requests, or if we have more 1132 * but can't assign a driver tag to it. 1133 */ 1134 if (list_empty(list)) 1135 bd.last = true; 1136 else { 1137 nxt = list_first_entry(list, struct request, queuelist); 1138 bd.last = !blk_mq_get_driver_tag(nxt, NULL, false); 1139 } 1140 1141 ret = q->mq_ops->queue_rq(hctx, &bd); 1142 if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) { 1143 /* 1144 * If an I/O scheduler has been configured and we got a 1145 * driver tag for the next request already, free it 1146 * again. 1147 */ 1148 if (!list_empty(list)) { 1149 nxt = list_first_entry(list, struct request, queuelist); 1150 blk_mq_put_driver_tag(nxt); 1151 } 1152 list_add(&rq->queuelist, list); 1153 __blk_mq_requeue_request(rq); 1154 break; 1155 } 1156 1157 if (unlikely(ret != BLK_STS_OK)) { 1158 errors++; 1159 blk_mq_end_request(rq, BLK_STS_IOERR); 1160 continue; 1161 } 1162 1163 queued++; 1164 } while (!list_empty(list)); 1165 1166 hctx->dispatched[queued_to_index(queued)]++; 1167 1168 /* 1169 * Any items that need requeuing? Stuff them into hctx->dispatch, 1170 * that is where we will continue on next queue run. 1171 */ 1172 if (!list_empty(list)) { 1173 bool needs_restart; 1174 1175 spin_lock(&hctx->lock); 1176 list_splice_init(list, &hctx->dispatch); 1177 spin_unlock(&hctx->lock); 1178 1179 /* 1180 * If SCHED_RESTART was set by the caller of this function and 1181 * it is no longer set that means that it was cleared by another 1182 * thread and hence that a queue rerun is needed. 1183 * 1184 * If 'no_tag' is set, that means that we failed getting 1185 * a driver tag with an I/O scheduler attached. If our dispatch 1186 * waitqueue is no longer active, ensure that we run the queue 1187 * AFTER adding our entries back to the list. 1188 * 1189 * If no I/O scheduler has been configured it is possible that 1190 * the hardware queue got stopped and restarted before requests 1191 * were pushed back onto the dispatch list. Rerun the queue to 1192 * avoid starvation. Notes: 1193 * - blk_mq_run_hw_queue() checks whether or not a queue has 1194 * been stopped before rerunning a queue. 1195 * - Some but not all block drivers stop a queue before 1196 * returning BLK_STS_RESOURCE. Two exceptions are scsi-mq 1197 * and dm-rq. 1198 * 1199 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART 1200 * bit is set, run queue after a delay to avoid IO stalls 1201 * that could otherwise occur if the queue is idle. 1202 */ 1203 needs_restart = blk_mq_sched_needs_restart(hctx); 1204 if (!needs_restart || 1205 (no_tag && list_empty_careful(&hctx->dispatch_wait.entry))) 1206 blk_mq_run_hw_queue(hctx, true); 1207 else if (needs_restart && (ret == BLK_STS_RESOURCE)) 1208 blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY); 1209 } 1210 1211 return (queued + errors) != 0; 1212 } 1213 1214 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) 1215 { 1216 int srcu_idx; 1217 1218 /* 1219 * We should be running this queue from one of the CPUs that 1220 * are mapped to it. 1221 * 1222 * There are at least two related races now between setting 1223 * hctx->next_cpu from blk_mq_hctx_next_cpu() and running 1224 * __blk_mq_run_hw_queue(): 1225 * 1226 * - hctx->next_cpu is found offline in blk_mq_hctx_next_cpu(), 1227 * but later it becomes online, then this warning is harmless 1228 * at all 1229 * 1230 * - hctx->next_cpu is found online in blk_mq_hctx_next_cpu(), 1231 * but later it becomes offline, then the warning can't be 1232 * triggered, and we depend on blk-mq timeout handler to 1233 * handle dispatched requests to this hctx 1234 */ 1235 if (!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) && 1236 cpu_online(hctx->next_cpu)) { 1237 printk(KERN_WARNING "run queue from wrong CPU %d, hctx %s\n", 1238 raw_smp_processor_id(), 1239 cpumask_empty(hctx->cpumask) ? "inactive": "active"); 1240 dump_stack(); 1241 } 1242 1243 /* 1244 * We can't run the queue inline with ints disabled. Ensure that 1245 * we catch bad users of this early. 1246 */ 1247 WARN_ON_ONCE(in_interrupt()); 1248 1249 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); 1250 1251 hctx_lock(hctx, &srcu_idx); 1252 blk_mq_sched_dispatch_requests(hctx); 1253 hctx_unlock(hctx, srcu_idx); 1254 } 1255 1256 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx) 1257 { 1258 int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask); 1259 1260 if (cpu >= nr_cpu_ids) 1261 cpu = cpumask_first(hctx->cpumask); 1262 return cpu; 1263 } 1264 1265 /* 1266 * It'd be great if the workqueue API had a way to pass 1267 * in a mask and had some smarts for more clever placement. 1268 * For now we just round-robin here, switching for every 1269 * BLK_MQ_CPU_WORK_BATCH queued items. 1270 */ 1271 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) 1272 { 1273 bool tried = false; 1274 int next_cpu = hctx->next_cpu; 1275 1276 if (hctx->queue->nr_hw_queues == 1) 1277 return WORK_CPU_UNBOUND; 1278 1279 if (--hctx->next_cpu_batch <= 0) { 1280 select_cpu: 1281 next_cpu = cpumask_next_and(next_cpu, hctx->cpumask, 1282 cpu_online_mask); 1283 if (next_cpu >= nr_cpu_ids) 1284 next_cpu = blk_mq_first_mapped_cpu(hctx); 1285 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 1286 } 1287 1288 /* 1289 * Do unbound schedule if we can't find a online CPU for this hctx, 1290 * and it should only happen in the path of handling CPU DEAD. 1291 */ 1292 if (!cpu_online(next_cpu)) { 1293 if (!tried) { 1294 tried = true; 1295 goto select_cpu; 1296 } 1297 1298 /* 1299 * Make sure to re-select CPU next time once after CPUs 1300 * in hctx->cpumask become online again. 1301 */ 1302 hctx->next_cpu = next_cpu; 1303 hctx->next_cpu_batch = 1; 1304 return WORK_CPU_UNBOUND; 1305 } 1306 1307 hctx->next_cpu = next_cpu; 1308 return next_cpu; 1309 } 1310 1311 static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, 1312 unsigned long msecs) 1313 { 1314 if (unlikely(blk_mq_hctx_stopped(hctx))) 1315 return; 1316 1317 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) { 1318 int cpu = get_cpu(); 1319 if (cpumask_test_cpu(cpu, hctx->cpumask)) { 1320 __blk_mq_run_hw_queue(hctx); 1321 put_cpu(); 1322 return; 1323 } 1324 1325 put_cpu(); 1326 } 1327 1328 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, 1329 msecs_to_jiffies(msecs)); 1330 } 1331 1332 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) 1333 { 1334 __blk_mq_delay_run_hw_queue(hctx, true, msecs); 1335 } 1336 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); 1337 1338 bool blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1339 { 1340 int srcu_idx; 1341 bool need_run; 1342 1343 /* 1344 * When queue is quiesced, we may be switching io scheduler, or 1345 * updating nr_hw_queues, or other things, and we can't run queue 1346 * any more, even __blk_mq_hctx_has_pending() can't be called safely. 1347 * 1348 * And queue will be rerun in blk_mq_unquiesce_queue() if it is 1349 * quiesced. 1350 */ 1351 hctx_lock(hctx, &srcu_idx); 1352 need_run = !blk_queue_quiesced(hctx->queue) && 1353 blk_mq_hctx_has_pending(hctx); 1354 hctx_unlock(hctx, srcu_idx); 1355 1356 if (need_run) { 1357 __blk_mq_delay_run_hw_queue(hctx, async, 0); 1358 return true; 1359 } 1360 1361 return false; 1362 } 1363 EXPORT_SYMBOL(blk_mq_run_hw_queue); 1364 1365 void blk_mq_run_hw_queues(struct request_queue *q, bool async) 1366 { 1367 struct blk_mq_hw_ctx *hctx; 1368 int i; 1369 1370 queue_for_each_hw_ctx(q, hctx, i) { 1371 if (blk_mq_hctx_stopped(hctx)) 1372 continue; 1373 1374 blk_mq_run_hw_queue(hctx, async); 1375 } 1376 } 1377 EXPORT_SYMBOL(blk_mq_run_hw_queues); 1378 1379 /** 1380 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped 1381 * @q: request queue. 1382 * 1383 * The caller is responsible for serializing this function against 1384 * blk_mq_{start,stop}_hw_queue(). 1385 */ 1386 bool blk_mq_queue_stopped(struct request_queue *q) 1387 { 1388 struct blk_mq_hw_ctx *hctx; 1389 int i; 1390 1391 queue_for_each_hw_ctx(q, hctx, i) 1392 if (blk_mq_hctx_stopped(hctx)) 1393 return true; 1394 1395 return false; 1396 } 1397 EXPORT_SYMBOL(blk_mq_queue_stopped); 1398 1399 /* 1400 * This function is often used for pausing .queue_rq() by driver when 1401 * there isn't enough resource or some conditions aren't satisfied, and 1402 * BLK_STS_RESOURCE is usually returned. 1403 * 1404 * We do not guarantee that dispatch can be drained or blocked 1405 * after blk_mq_stop_hw_queue() returns. Please use 1406 * blk_mq_quiesce_queue() for that requirement. 1407 */ 1408 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) 1409 { 1410 cancel_delayed_work(&hctx->run_work); 1411 1412 set_bit(BLK_MQ_S_STOPPED, &hctx->state); 1413 } 1414 EXPORT_SYMBOL(blk_mq_stop_hw_queue); 1415 1416 /* 1417 * This function is often used for pausing .queue_rq() by driver when 1418 * there isn't enough resource or some conditions aren't satisfied, and 1419 * BLK_STS_RESOURCE is usually returned. 1420 * 1421 * We do not guarantee that dispatch can be drained or blocked 1422 * after blk_mq_stop_hw_queues() returns. Please use 1423 * blk_mq_quiesce_queue() for that requirement. 1424 */ 1425 void blk_mq_stop_hw_queues(struct request_queue *q) 1426 { 1427 struct blk_mq_hw_ctx *hctx; 1428 int i; 1429 1430 queue_for_each_hw_ctx(q, hctx, i) 1431 blk_mq_stop_hw_queue(hctx); 1432 } 1433 EXPORT_SYMBOL(blk_mq_stop_hw_queues); 1434 1435 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) 1436 { 1437 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1438 1439 blk_mq_run_hw_queue(hctx, false); 1440 } 1441 EXPORT_SYMBOL(blk_mq_start_hw_queue); 1442 1443 void blk_mq_start_hw_queues(struct request_queue *q) 1444 { 1445 struct blk_mq_hw_ctx *hctx; 1446 int i; 1447 1448 queue_for_each_hw_ctx(q, hctx, i) 1449 blk_mq_start_hw_queue(hctx); 1450 } 1451 EXPORT_SYMBOL(blk_mq_start_hw_queues); 1452 1453 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1454 { 1455 if (!blk_mq_hctx_stopped(hctx)) 1456 return; 1457 1458 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1459 blk_mq_run_hw_queue(hctx, async); 1460 } 1461 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue); 1462 1463 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) 1464 { 1465 struct blk_mq_hw_ctx *hctx; 1466 int i; 1467 1468 queue_for_each_hw_ctx(q, hctx, i) 1469 blk_mq_start_stopped_hw_queue(hctx, async); 1470 } 1471 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); 1472 1473 static void blk_mq_run_work_fn(struct work_struct *work) 1474 { 1475 struct blk_mq_hw_ctx *hctx; 1476 1477 hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work); 1478 1479 /* 1480 * If we are stopped, don't run the queue. 1481 */ 1482 if (test_bit(BLK_MQ_S_STOPPED, &hctx->state)) 1483 return; 1484 1485 __blk_mq_run_hw_queue(hctx); 1486 } 1487 1488 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx, 1489 struct request *rq, 1490 bool at_head) 1491 { 1492 struct blk_mq_ctx *ctx = rq->mq_ctx; 1493 1494 lockdep_assert_held(&ctx->lock); 1495 1496 trace_block_rq_insert(hctx->queue, rq); 1497 1498 if (at_head) 1499 list_add(&rq->queuelist, &ctx->rq_list); 1500 else 1501 list_add_tail(&rq->queuelist, &ctx->rq_list); 1502 } 1503 1504 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, 1505 bool at_head) 1506 { 1507 struct blk_mq_ctx *ctx = rq->mq_ctx; 1508 1509 lockdep_assert_held(&ctx->lock); 1510 1511 __blk_mq_insert_req_list(hctx, rq, at_head); 1512 blk_mq_hctx_mark_pending(hctx, ctx); 1513 } 1514 1515 /* 1516 * Should only be used carefully, when the caller knows we want to 1517 * bypass a potential IO scheduler on the target device. 1518 */ 1519 void blk_mq_request_bypass_insert(struct request *rq, bool run_queue) 1520 { 1521 struct blk_mq_ctx *ctx = rq->mq_ctx; 1522 struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(rq->q, ctx->cpu); 1523 1524 spin_lock(&hctx->lock); 1525 list_add_tail(&rq->queuelist, &hctx->dispatch); 1526 spin_unlock(&hctx->lock); 1527 1528 if (run_queue) 1529 blk_mq_run_hw_queue(hctx, false); 1530 } 1531 1532 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 1533 struct list_head *list) 1534 1535 { 1536 /* 1537 * preemption doesn't flush plug list, so it's possible ctx->cpu is 1538 * offline now 1539 */ 1540 spin_lock(&ctx->lock); 1541 while (!list_empty(list)) { 1542 struct request *rq; 1543 1544 rq = list_first_entry(list, struct request, queuelist); 1545 BUG_ON(rq->mq_ctx != ctx); 1546 list_del_init(&rq->queuelist); 1547 __blk_mq_insert_req_list(hctx, rq, false); 1548 } 1549 blk_mq_hctx_mark_pending(hctx, ctx); 1550 spin_unlock(&ctx->lock); 1551 } 1552 1553 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b) 1554 { 1555 struct request *rqa = container_of(a, struct request, queuelist); 1556 struct request *rqb = container_of(b, struct request, queuelist); 1557 1558 return !(rqa->mq_ctx < rqb->mq_ctx || 1559 (rqa->mq_ctx == rqb->mq_ctx && 1560 blk_rq_pos(rqa) < blk_rq_pos(rqb))); 1561 } 1562 1563 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 1564 { 1565 struct blk_mq_ctx *this_ctx; 1566 struct request_queue *this_q; 1567 struct request *rq; 1568 LIST_HEAD(list); 1569 LIST_HEAD(ctx_list); 1570 unsigned int depth; 1571 1572 list_splice_init(&plug->mq_list, &list); 1573 1574 list_sort(NULL, &list, plug_ctx_cmp); 1575 1576 this_q = NULL; 1577 this_ctx = NULL; 1578 depth = 0; 1579 1580 while (!list_empty(&list)) { 1581 rq = list_entry_rq(list.next); 1582 list_del_init(&rq->queuelist); 1583 BUG_ON(!rq->q); 1584 if (rq->mq_ctx != this_ctx) { 1585 if (this_ctx) { 1586 trace_block_unplug(this_q, depth, from_schedule); 1587 blk_mq_sched_insert_requests(this_q, this_ctx, 1588 &ctx_list, 1589 from_schedule); 1590 } 1591 1592 this_ctx = rq->mq_ctx; 1593 this_q = rq->q; 1594 depth = 0; 1595 } 1596 1597 depth++; 1598 list_add_tail(&rq->queuelist, &ctx_list); 1599 } 1600 1601 /* 1602 * If 'this_ctx' is set, we know we have entries to complete 1603 * on 'ctx_list'. Do those. 1604 */ 1605 if (this_ctx) { 1606 trace_block_unplug(this_q, depth, from_schedule); 1607 blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list, 1608 from_schedule); 1609 } 1610 } 1611 1612 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio) 1613 { 1614 blk_init_request_from_bio(rq, bio); 1615 1616 blk_rq_set_rl(rq, blk_get_rl(rq->q, bio)); 1617 1618 blk_account_io_start(rq, true); 1619 } 1620 1621 static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq) 1622 { 1623 if (rq->tag != -1) 1624 return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false); 1625 1626 return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true); 1627 } 1628 1629 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx, 1630 struct request *rq, 1631 blk_qc_t *cookie) 1632 { 1633 struct request_queue *q = rq->q; 1634 struct blk_mq_queue_data bd = { 1635 .rq = rq, 1636 .last = true, 1637 }; 1638 blk_qc_t new_cookie; 1639 blk_status_t ret; 1640 1641 new_cookie = request_to_qc_t(hctx, rq); 1642 1643 /* 1644 * For OK queue, we are done. For error, caller may kill it. 1645 * Any other error (busy), just add it to our list as we 1646 * previously would have done. 1647 */ 1648 ret = q->mq_ops->queue_rq(hctx, &bd); 1649 switch (ret) { 1650 case BLK_STS_OK: 1651 *cookie = new_cookie; 1652 break; 1653 case BLK_STS_RESOURCE: 1654 case BLK_STS_DEV_RESOURCE: 1655 __blk_mq_requeue_request(rq); 1656 break; 1657 default: 1658 *cookie = BLK_QC_T_NONE; 1659 break; 1660 } 1661 1662 return ret; 1663 } 1664 1665 static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 1666 struct request *rq, 1667 blk_qc_t *cookie, 1668 bool bypass_insert) 1669 { 1670 struct request_queue *q = rq->q; 1671 bool run_queue = true; 1672 1673 /* 1674 * RCU or SRCU read lock is needed before checking quiesced flag. 1675 * 1676 * When queue is stopped or quiesced, ignore 'bypass_insert' from 1677 * blk_mq_request_issue_directly(), and return BLK_STS_OK to caller, 1678 * and avoid driver to try to dispatch again. 1679 */ 1680 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) { 1681 run_queue = false; 1682 bypass_insert = false; 1683 goto insert; 1684 } 1685 1686 if (q->elevator && !bypass_insert) 1687 goto insert; 1688 1689 if (!blk_mq_get_dispatch_budget(hctx)) 1690 goto insert; 1691 1692 if (!blk_mq_get_driver_tag(rq, NULL, false)) { 1693 blk_mq_put_dispatch_budget(hctx); 1694 goto insert; 1695 } 1696 1697 return __blk_mq_issue_directly(hctx, rq, cookie); 1698 insert: 1699 if (bypass_insert) 1700 return BLK_STS_RESOURCE; 1701 1702 blk_mq_sched_insert_request(rq, false, run_queue, false); 1703 return BLK_STS_OK; 1704 } 1705 1706 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 1707 struct request *rq, blk_qc_t *cookie) 1708 { 1709 blk_status_t ret; 1710 int srcu_idx; 1711 1712 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); 1713 1714 hctx_lock(hctx, &srcu_idx); 1715 1716 ret = __blk_mq_try_issue_directly(hctx, rq, cookie, false); 1717 if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) 1718 blk_mq_sched_insert_request(rq, false, true, false); 1719 else if (ret != BLK_STS_OK) 1720 blk_mq_end_request(rq, ret); 1721 1722 hctx_unlock(hctx, srcu_idx); 1723 } 1724 1725 blk_status_t blk_mq_request_issue_directly(struct request *rq) 1726 { 1727 blk_status_t ret; 1728 int srcu_idx; 1729 blk_qc_t unused_cookie; 1730 struct blk_mq_ctx *ctx = rq->mq_ctx; 1731 struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(rq->q, ctx->cpu); 1732 1733 hctx_lock(hctx, &srcu_idx); 1734 ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true); 1735 hctx_unlock(hctx, srcu_idx); 1736 1737 return ret; 1738 } 1739 1740 static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) 1741 { 1742 const int is_sync = op_is_sync(bio->bi_opf); 1743 const int is_flush_fua = op_is_flush(bio->bi_opf); 1744 struct blk_mq_alloc_data data = { .flags = 0 }; 1745 struct request *rq; 1746 unsigned int request_count = 0; 1747 struct blk_plug *plug; 1748 struct request *same_queue_rq = NULL; 1749 blk_qc_t cookie; 1750 unsigned int wb_acct; 1751 1752 blk_queue_bounce(q, &bio); 1753 1754 blk_queue_split(q, &bio); 1755 1756 if (!bio_integrity_prep(bio)) 1757 return BLK_QC_T_NONE; 1758 1759 if (!is_flush_fua && !blk_queue_nomerges(q) && 1760 blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq)) 1761 return BLK_QC_T_NONE; 1762 1763 if (blk_mq_sched_bio_merge(q, bio)) 1764 return BLK_QC_T_NONE; 1765 1766 wb_acct = wbt_wait(q->rq_wb, bio, NULL); 1767 1768 trace_block_getrq(q, bio, bio->bi_opf); 1769 1770 rq = blk_mq_get_request(q, bio, bio->bi_opf, &data); 1771 if (unlikely(!rq)) { 1772 __wbt_done(q->rq_wb, wb_acct); 1773 if (bio->bi_opf & REQ_NOWAIT) 1774 bio_wouldblock_error(bio); 1775 return BLK_QC_T_NONE; 1776 } 1777 1778 wbt_track(rq, wb_acct); 1779 1780 cookie = request_to_qc_t(data.hctx, rq); 1781 1782 plug = current->plug; 1783 if (unlikely(is_flush_fua)) { 1784 blk_mq_put_ctx(data.ctx); 1785 blk_mq_bio_to_request(rq, bio); 1786 1787 /* bypass scheduler for flush rq */ 1788 blk_insert_flush(rq); 1789 blk_mq_run_hw_queue(data.hctx, true); 1790 } else if (plug && q->nr_hw_queues == 1) { 1791 struct request *last = NULL; 1792 1793 blk_mq_put_ctx(data.ctx); 1794 blk_mq_bio_to_request(rq, bio); 1795 1796 /* 1797 * @request_count may become stale because of schedule 1798 * out, so check the list again. 1799 */ 1800 if (list_empty(&plug->mq_list)) 1801 request_count = 0; 1802 else if (blk_queue_nomerges(q)) 1803 request_count = blk_plug_queued_count(q); 1804 1805 if (!request_count) 1806 trace_block_plug(q); 1807 else 1808 last = list_entry_rq(plug->mq_list.prev); 1809 1810 if (request_count >= BLK_MAX_REQUEST_COUNT || (last && 1811 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { 1812 blk_flush_plug_list(plug, false); 1813 trace_block_plug(q); 1814 } 1815 1816 list_add_tail(&rq->queuelist, &plug->mq_list); 1817 } else if (plug && !blk_queue_nomerges(q)) { 1818 blk_mq_bio_to_request(rq, bio); 1819 1820 /* 1821 * We do limited plugging. If the bio can be merged, do that. 1822 * Otherwise the existing request in the plug list will be 1823 * issued. So the plug list will have one request at most 1824 * The plug list might get flushed before this. If that happens, 1825 * the plug list is empty, and same_queue_rq is invalid. 1826 */ 1827 if (list_empty(&plug->mq_list)) 1828 same_queue_rq = NULL; 1829 if (same_queue_rq) 1830 list_del_init(&same_queue_rq->queuelist); 1831 list_add_tail(&rq->queuelist, &plug->mq_list); 1832 1833 blk_mq_put_ctx(data.ctx); 1834 1835 if (same_queue_rq) { 1836 data.hctx = blk_mq_map_queue(q, 1837 same_queue_rq->mq_ctx->cpu); 1838 blk_mq_try_issue_directly(data.hctx, same_queue_rq, 1839 &cookie); 1840 } 1841 } else if (q->nr_hw_queues > 1 && is_sync) { 1842 blk_mq_put_ctx(data.ctx); 1843 blk_mq_bio_to_request(rq, bio); 1844 blk_mq_try_issue_directly(data.hctx, rq, &cookie); 1845 } else { 1846 blk_mq_put_ctx(data.ctx); 1847 blk_mq_bio_to_request(rq, bio); 1848 blk_mq_sched_insert_request(rq, false, true, true); 1849 } 1850 1851 return cookie; 1852 } 1853 1854 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 1855 unsigned int hctx_idx) 1856 { 1857 struct page *page; 1858 1859 if (tags->rqs && set->ops->exit_request) { 1860 int i; 1861 1862 for (i = 0; i < tags->nr_tags; i++) { 1863 struct request *rq = tags->static_rqs[i]; 1864 1865 if (!rq) 1866 continue; 1867 set->ops->exit_request(set, rq, hctx_idx); 1868 tags->static_rqs[i] = NULL; 1869 } 1870 } 1871 1872 while (!list_empty(&tags->page_list)) { 1873 page = list_first_entry(&tags->page_list, struct page, lru); 1874 list_del_init(&page->lru); 1875 /* 1876 * Remove kmemleak object previously allocated in 1877 * blk_mq_init_rq_map(). 1878 */ 1879 kmemleak_free(page_address(page)); 1880 __free_pages(page, page->private); 1881 } 1882 } 1883 1884 void blk_mq_free_rq_map(struct blk_mq_tags *tags) 1885 { 1886 kfree(tags->rqs); 1887 tags->rqs = NULL; 1888 kfree(tags->static_rqs); 1889 tags->static_rqs = NULL; 1890 1891 blk_mq_free_tags(tags); 1892 } 1893 1894 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, 1895 unsigned int hctx_idx, 1896 unsigned int nr_tags, 1897 unsigned int reserved_tags) 1898 { 1899 struct blk_mq_tags *tags; 1900 int node; 1901 1902 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx); 1903 if (node == NUMA_NO_NODE) 1904 node = set->numa_node; 1905 1906 tags = blk_mq_init_tags(nr_tags, reserved_tags, node, 1907 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags)); 1908 if (!tags) 1909 return NULL; 1910 1911 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *), 1912 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 1913 node); 1914 if (!tags->rqs) { 1915 blk_mq_free_tags(tags); 1916 return NULL; 1917 } 1918 1919 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *), 1920 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 1921 node); 1922 if (!tags->static_rqs) { 1923 kfree(tags->rqs); 1924 blk_mq_free_tags(tags); 1925 return NULL; 1926 } 1927 1928 return tags; 1929 } 1930 1931 static size_t order_to_size(unsigned int order) 1932 { 1933 return (size_t)PAGE_SIZE << order; 1934 } 1935 1936 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, 1937 unsigned int hctx_idx, int node) 1938 { 1939 int ret; 1940 1941 if (set->ops->init_request) { 1942 ret = set->ops->init_request(set, rq, hctx_idx, node); 1943 if (ret) 1944 return ret; 1945 } 1946 1947 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 1948 return 0; 1949 } 1950 1951 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 1952 unsigned int hctx_idx, unsigned int depth) 1953 { 1954 unsigned int i, j, entries_per_page, max_order = 4; 1955 size_t rq_size, left; 1956 int node; 1957 1958 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx); 1959 if (node == NUMA_NO_NODE) 1960 node = set->numa_node; 1961 1962 INIT_LIST_HEAD(&tags->page_list); 1963 1964 /* 1965 * rq_size is the size of the request plus driver payload, rounded 1966 * to the cacheline size 1967 */ 1968 rq_size = round_up(sizeof(struct request) + set->cmd_size, 1969 cache_line_size()); 1970 left = rq_size * depth; 1971 1972 for (i = 0; i < depth; ) { 1973 int this_order = max_order; 1974 struct page *page; 1975 int to_do; 1976 void *p; 1977 1978 while (this_order && left < order_to_size(this_order - 1)) 1979 this_order--; 1980 1981 do { 1982 page = alloc_pages_node(node, 1983 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, 1984 this_order); 1985 if (page) 1986 break; 1987 if (!this_order--) 1988 break; 1989 if (order_to_size(this_order) < rq_size) 1990 break; 1991 } while (1); 1992 1993 if (!page) 1994 goto fail; 1995 1996 page->private = this_order; 1997 list_add_tail(&page->lru, &tags->page_list); 1998 1999 p = page_address(page); 2000 /* 2001 * Allow kmemleak to scan these pages as they contain pointers 2002 * to additional allocations like via ops->init_request(). 2003 */ 2004 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); 2005 entries_per_page = order_to_size(this_order) / rq_size; 2006 to_do = min(entries_per_page, depth - i); 2007 left -= to_do * rq_size; 2008 for (j = 0; j < to_do; j++) { 2009 struct request *rq = p; 2010 2011 tags->static_rqs[i] = rq; 2012 if (blk_mq_init_request(set, rq, hctx_idx, node)) { 2013 tags->static_rqs[i] = NULL; 2014 goto fail; 2015 } 2016 2017 p += rq_size; 2018 i++; 2019 } 2020 } 2021 return 0; 2022 2023 fail: 2024 blk_mq_free_rqs(set, tags, hctx_idx); 2025 return -ENOMEM; 2026 } 2027 2028 /* 2029 * 'cpu' is going away. splice any existing rq_list entries from this 2030 * software queue to the hw queue dispatch list, and ensure that it 2031 * gets run. 2032 */ 2033 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) 2034 { 2035 struct blk_mq_hw_ctx *hctx; 2036 struct blk_mq_ctx *ctx; 2037 LIST_HEAD(tmp); 2038 2039 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead); 2040 ctx = __blk_mq_get_ctx(hctx->queue, cpu); 2041 2042 spin_lock(&ctx->lock); 2043 if (!list_empty(&ctx->rq_list)) { 2044 list_splice_init(&ctx->rq_list, &tmp); 2045 blk_mq_hctx_clear_pending(hctx, ctx); 2046 } 2047 spin_unlock(&ctx->lock); 2048 2049 if (list_empty(&tmp)) 2050 return 0; 2051 2052 spin_lock(&hctx->lock); 2053 list_splice_tail_init(&tmp, &hctx->dispatch); 2054 spin_unlock(&hctx->lock); 2055 2056 blk_mq_run_hw_queue(hctx, true); 2057 return 0; 2058 } 2059 2060 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) 2061 { 2062 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD, 2063 &hctx->cpuhp_dead); 2064 } 2065 2066 /* hctx->ctxs will be freed in queue's release handler */ 2067 static void blk_mq_exit_hctx(struct request_queue *q, 2068 struct blk_mq_tag_set *set, 2069 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 2070 { 2071 blk_mq_debugfs_unregister_hctx(hctx); 2072 2073 if (blk_mq_hw_queue_mapped(hctx)) 2074 blk_mq_tag_idle(hctx); 2075 2076 if (set->ops->exit_request) 2077 set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx); 2078 2079 blk_mq_sched_exit_hctx(q, hctx, hctx_idx); 2080 2081 if (set->ops->exit_hctx) 2082 set->ops->exit_hctx(hctx, hctx_idx); 2083 2084 if (hctx->flags & BLK_MQ_F_BLOCKING) 2085 cleanup_srcu_struct(hctx->srcu); 2086 2087 blk_mq_remove_cpuhp(hctx); 2088 blk_free_flush_queue(hctx->fq); 2089 sbitmap_free(&hctx->ctx_map); 2090 } 2091 2092 static void blk_mq_exit_hw_queues(struct request_queue *q, 2093 struct blk_mq_tag_set *set, int nr_queue) 2094 { 2095 struct blk_mq_hw_ctx *hctx; 2096 unsigned int i; 2097 2098 queue_for_each_hw_ctx(q, hctx, i) { 2099 if (i == nr_queue) 2100 break; 2101 blk_mq_exit_hctx(q, set, hctx, i); 2102 } 2103 } 2104 2105 static int blk_mq_init_hctx(struct request_queue *q, 2106 struct blk_mq_tag_set *set, 2107 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) 2108 { 2109 int node; 2110 2111 node = hctx->numa_node; 2112 if (node == NUMA_NO_NODE) 2113 node = hctx->numa_node = set->numa_node; 2114 2115 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); 2116 spin_lock_init(&hctx->lock); 2117 INIT_LIST_HEAD(&hctx->dispatch); 2118 hctx->queue = q; 2119 hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED; 2120 2121 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead); 2122 2123 hctx->tags = set->tags[hctx_idx]; 2124 2125 /* 2126 * Allocate space for all possible cpus to avoid allocation at 2127 * runtime 2128 */ 2129 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *), 2130 GFP_KERNEL, node); 2131 if (!hctx->ctxs) 2132 goto unregister_cpu_notifier; 2133 2134 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL, 2135 node)) 2136 goto free_ctxs; 2137 2138 hctx->nr_ctx = 0; 2139 2140 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); 2141 INIT_LIST_HEAD(&hctx->dispatch_wait.entry); 2142 2143 if (set->ops->init_hctx && 2144 set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) 2145 goto free_bitmap; 2146 2147 if (blk_mq_sched_init_hctx(q, hctx, hctx_idx)) 2148 goto exit_hctx; 2149 2150 hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size); 2151 if (!hctx->fq) 2152 goto sched_exit_hctx; 2153 2154 if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx, node)) 2155 goto free_fq; 2156 2157 if (hctx->flags & BLK_MQ_F_BLOCKING) 2158 init_srcu_struct(hctx->srcu); 2159 2160 blk_mq_debugfs_register_hctx(q, hctx); 2161 2162 return 0; 2163 2164 free_fq: 2165 kfree(hctx->fq); 2166 sched_exit_hctx: 2167 blk_mq_sched_exit_hctx(q, hctx, hctx_idx); 2168 exit_hctx: 2169 if (set->ops->exit_hctx) 2170 set->ops->exit_hctx(hctx, hctx_idx); 2171 free_bitmap: 2172 sbitmap_free(&hctx->ctx_map); 2173 free_ctxs: 2174 kfree(hctx->ctxs); 2175 unregister_cpu_notifier: 2176 blk_mq_remove_cpuhp(hctx); 2177 return -1; 2178 } 2179 2180 static void blk_mq_init_cpu_queues(struct request_queue *q, 2181 unsigned int nr_hw_queues) 2182 { 2183 unsigned int i; 2184 2185 for_each_possible_cpu(i) { 2186 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i); 2187 struct blk_mq_hw_ctx *hctx; 2188 2189 __ctx->cpu = i; 2190 spin_lock_init(&__ctx->lock); 2191 INIT_LIST_HEAD(&__ctx->rq_list); 2192 __ctx->queue = q; 2193 2194 /* 2195 * Set local node, IFF we have more than one hw queue. If 2196 * not, we remain on the home node of the device 2197 */ 2198 hctx = blk_mq_map_queue(q, i); 2199 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) 2200 hctx->numa_node = local_memory_node(cpu_to_node(i)); 2201 } 2202 } 2203 2204 static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx) 2205 { 2206 int ret = 0; 2207 2208 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx, 2209 set->queue_depth, set->reserved_tags); 2210 if (!set->tags[hctx_idx]) 2211 return false; 2212 2213 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx, 2214 set->queue_depth); 2215 if (!ret) 2216 return true; 2217 2218 blk_mq_free_rq_map(set->tags[hctx_idx]); 2219 set->tags[hctx_idx] = NULL; 2220 return false; 2221 } 2222 2223 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set, 2224 unsigned int hctx_idx) 2225 { 2226 if (set->tags[hctx_idx]) { 2227 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx); 2228 blk_mq_free_rq_map(set->tags[hctx_idx]); 2229 set->tags[hctx_idx] = NULL; 2230 } 2231 } 2232 2233 static void blk_mq_map_swqueue(struct request_queue *q) 2234 { 2235 unsigned int i, hctx_idx; 2236 struct blk_mq_hw_ctx *hctx; 2237 struct blk_mq_ctx *ctx; 2238 struct blk_mq_tag_set *set = q->tag_set; 2239 2240 /* 2241 * Avoid others reading imcomplete hctx->cpumask through sysfs 2242 */ 2243 mutex_lock(&q->sysfs_lock); 2244 2245 queue_for_each_hw_ctx(q, hctx, i) { 2246 cpumask_clear(hctx->cpumask); 2247 hctx->nr_ctx = 0; 2248 hctx->dispatch_from = NULL; 2249 } 2250 2251 /* 2252 * Map software to hardware queues. 2253 * 2254 * If the cpu isn't present, the cpu is mapped to first hctx. 2255 */ 2256 for_each_possible_cpu(i) { 2257 hctx_idx = q->mq_map[i]; 2258 /* unmapped hw queue can be remapped after CPU topo changed */ 2259 if (!set->tags[hctx_idx] && 2260 !__blk_mq_alloc_rq_map(set, hctx_idx)) { 2261 /* 2262 * If tags initialization fail for some hctx, 2263 * that hctx won't be brought online. In this 2264 * case, remap the current ctx to hctx[0] which 2265 * is guaranteed to always have tags allocated 2266 */ 2267 q->mq_map[i] = 0; 2268 } 2269 2270 ctx = per_cpu_ptr(q->queue_ctx, i); 2271 hctx = blk_mq_map_queue(q, i); 2272 2273 cpumask_set_cpu(i, hctx->cpumask); 2274 ctx->index_hw = hctx->nr_ctx; 2275 hctx->ctxs[hctx->nr_ctx++] = ctx; 2276 } 2277 2278 mutex_unlock(&q->sysfs_lock); 2279 2280 queue_for_each_hw_ctx(q, hctx, i) { 2281 /* 2282 * If no software queues are mapped to this hardware queue, 2283 * disable it and free the request entries. 2284 */ 2285 if (!hctx->nr_ctx) { 2286 /* Never unmap queue 0. We need it as a 2287 * fallback in case of a new remap fails 2288 * allocation 2289 */ 2290 if (i && set->tags[i]) 2291 blk_mq_free_map_and_requests(set, i); 2292 2293 hctx->tags = NULL; 2294 continue; 2295 } 2296 2297 hctx->tags = set->tags[i]; 2298 WARN_ON(!hctx->tags); 2299 2300 /* 2301 * Set the map size to the number of mapped software queues. 2302 * This is more accurate and more efficient than looping 2303 * over all possibly mapped software queues. 2304 */ 2305 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx); 2306 2307 /* 2308 * Initialize batch roundrobin counts 2309 */ 2310 hctx->next_cpu = blk_mq_first_mapped_cpu(hctx); 2311 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 2312 } 2313 } 2314 2315 /* 2316 * Caller needs to ensure that we're either frozen/quiesced, or that 2317 * the queue isn't live yet. 2318 */ 2319 static void queue_set_hctx_shared(struct request_queue *q, bool shared) 2320 { 2321 struct blk_mq_hw_ctx *hctx; 2322 int i; 2323 2324 queue_for_each_hw_ctx(q, hctx, i) { 2325 if (shared) { 2326 if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) 2327 atomic_inc(&q->shared_hctx_restart); 2328 hctx->flags |= BLK_MQ_F_TAG_SHARED; 2329 } else { 2330 if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) 2331 atomic_dec(&q->shared_hctx_restart); 2332 hctx->flags &= ~BLK_MQ_F_TAG_SHARED; 2333 } 2334 } 2335 } 2336 2337 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, 2338 bool shared) 2339 { 2340 struct request_queue *q; 2341 2342 lockdep_assert_held(&set->tag_list_lock); 2343 2344 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2345 blk_mq_freeze_queue(q); 2346 queue_set_hctx_shared(q, shared); 2347 blk_mq_unfreeze_queue(q); 2348 } 2349 } 2350 2351 static void blk_mq_del_queue_tag_set(struct request_queue *q) 2352 { 2353 struct blk_mq_tag_set *set = q->tag_set; 2354 2355 mutex_lock(&set->tag_list_lock); 2356 list_del_rcu(&q->tag_set_list); 2357 if (list_is_singular(&set->tag_list)) { 2358 /* just transitioned to unshared */ 2359 set->flags &= ~BLK_MQ_F_TAG_SHARED; 2360 /* update existing queue */ 2361 blk_mq_update_tag_set_depth(set, false); 2362 } 2363 mutex_unlock(&set->tag_list_lock); 2364 synchronize_rcu(); 2365 INIT_LIST_HEAD(&q->tag_set_list); 2366 } 2367 2368 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, 2369 struct request_queue *q) 2370 { 2371 q->tag_set = set; 2372 2373 mutex_lock(&set->tag_list_lock); 2374 2375 /* 2376 * Check to see if we're transitioning to shared (from 1 to 2 queues). 2377 */ 2378 if (!list_empty(&set->tag_list) && 2379 !(set->flags & BLK_MQ_F_TAG_SHARED)) { 2380 set->flags |= BLK_MQ_F_TAG_SHARED; 2381 /* update existing queue */ 2382 blk_mq_update_tag_set_depth(set, true); 2383 } 2384 if (set->flags & BLK_MQ_F_TAG_SHARED) 2385 queue_set_hctx_shared(q, true); 2386 list_add_tail_rcu(&q->tag_set_list, &set->tag_list); 2387 2388 mutex_unlock(&set->tag_list_lock); 2389 } 2390 2391 /* 2392 * It is the actual release handler for mq, but we do it from 2393 * request queue's release handler for avoiding use-after-free 2394 * and headache because q->mq_kobj shouldn't have been introduced, 2395 * but we can't group ctx/kctx kobj without it. 2396 */ 2397 void blk_mq_release(struct request_queue *q) 2398 { 2399 struct blk_mq_hw_ctx *hctx; 2400 unsigned int i; 2401 2402 /* hctx kobj stays in hctx */ 2403 queue_for_each_hw_ctx(q, hctx, i) { 2404 if (!hctx) 2405 continue; 2406 kobject_put(&hctx->kobj); 2407 } 2408 2409 q->mq_map = NULL; 2410 2411 kfree(q->queue_hw_ctx); 2412 2413 /* 2414 * release .mq_kobj and sw queue's kobject now because 2415 * both share lifetime with request queue. 2416 */ 2417 blk_mq_sysfs_deinit(q); 2418 2419 free_percpu(q->queue_ctx); 2420 } 2421 2422 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) 2423 { 2424 struct request_queue *uninit_q, *q; 2425 2426 uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node, NULL); 2427 if (!uninit_q) 2428 return ERR_PTR(-ENOMEM); 2429 2430 q = blk_mq_init_allocated_queue(set, uninit_q); 2431 if (IS_ERR(q)) 2432 blk_cleanup_queue(uninit_q); 2433 2434 return q; 2435 } 2436 EXPORT_SYMBOL(blk_mq_init_queue); 2437 2438 static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set) 2439 { 2440 int hw_ctx_size = sizeof(struct blk_mq_hw_ctx); 2441 2442 BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu), 2443 __alignof__(struct blk_mq_hw_ctx)) != 2444 sizeof(struct blk_mq_hw_ctx)); 2445 2446 if (tag_set->flags & BLK_MQ_F_BLOCKING) 2447 hw_ctx_size += sizeof(struct srcu_struct); 2448 2449 return hw_ctx_size; 2450 } 2451 2452 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, 2453 struct request_queue *q) 2454 { 2455 int i, j; 2456 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; 2457 2458 blk_mq_sysfs_unregister(q); 2459 2460 /* protect against switching io scheduler */ 2461 mutex_lock(&q->sysfs_lock); 2462 for (i = 0; i < set->nr_hw_queues; i++) { 2463 int node; 2464 2465 if (hctxs[i]) 2466 continue; 2467 2468 node = blk_mq_hw_queue_to_node(q->mq_map, i); 2469 hctxs[i] = kzalloc_node(blk_mq_hw_ctx_size(set), 2470 GFP_KERNEL, node); 2471 if (!hctxs[i]) 2472 break; 2473 2474 if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL, 2475 node)) { 2476 kfree(hctxs[i]); 2477 hctxs[i] = NULL; 2478 break; 2479 } 2480 2481 atomic_set(&hctxs[i]->nr_active, 0); 2482 hctxs[i]->numa_node = node; 2483 hctxs[i]->queue_num = i; 2484 2485 if (blk_mq_init_hctx(q, set, hctxs[i], i)) { 2486 free_cpumask_var(hctxs[i]->cpumask); 2487 kfree(hctxs[i]); 2488 hctxs[i] = NULL; 2489 break; 2490 } 2491 blk_mq_hctx_kobj_init(hctxs[i]); 2492 } 2493 for (j = i; j < q->nr_hw_queues; j++) { 2494 struct blk_mq_hw_ctx *hctx = hctxs[j]; 2495 2496 if (hctx) { 2497 if (hctx->tags) 2498 blk_mq_free_map_and_requests(set, j); 2499 blk_mq_exit_hctx(q, set, hctx, j); 2500 kobject_put(&hctx->kobj); 2501 hctxs[j] = NULL; 2502 2503 } 2504 } 2505 q->nr_hw_queues = i; 2506 mutex_unlock(&q->sysfs_lock); 2507 blk_mq_sysfs_register(q); 2508 } 2509 2510 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 2511 struct request_queue *q) 2512 { 2513 /* mark the queue as mq asap */ 2514 q->mq_ops = set->ops; 2515 2516 q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn, 2517 blk_mq_poll_stats_bkt, 2518 BLK_MQ_POLL_STATS_BKTS, q); 2519 if (!q->poll_cb) 2520 goto err_exit; 2521 2522 q->queue_ctx = alloc_percpu(struct blk_mq_ctx); 2523 if (!q->queue_ctx) 2524 goto err_exit; 2525 2526 /* init q->mq_kobj and sw queues' kobjects */ 2527 blk_mq_sysfs_init(q); 2528 2529 q->queue_hw_ctx = kcalloc_node(nr_cpu_ids, sizeof(*(q->queue_hw_ctx)), 2530 GFP_KERNEL, set->numa_node); 2531 if (!q->queue_hw_ctx) 2532 goto err_percpu; 2533 2534 q->mq_map = set->mq_map; 2535 2536 blk_mq_realloc_hw_ctxs(set, q); 2537 if (!q->nr_hw_queues) 2538 goto err_hctxs; 2539 2540 INIT_WORK(&q->timeout_work, blk_mq_timeout_work); 2541 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ); 2542 2543 q->nr_queues = nr_cpu_ids; 2544 2545 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; 2546 2547 if (!(set->flags & BLK_MQ_F_SG_MERGE)) 2548 queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q); 2549 2550 q->sg_reserved_size = INT_MAX; 2551 2552 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); 2553 INIT_LIST_HEAD(&q->requeue_list); 2554 spin_lock_init(&q->requeue_lock); 2555 2556 blk_queue_make_request(q, blk_mq_make_request); 2557 if (q->mq_ops->poll) 2558 q->poll_fn = blk_mq_poll; 2559 2560 /* 2561 * Do this after blk_queue_make_request() overrides it... 2562 */ 2563 q->nr_requests = set->queue_depth; 2564 2565 /* 2566 * Default to classic polling 2567 */ 2568 q->poll_nsec = -1; 2569 2570 if (set->ops->complete) 2571 blk_queue_softirq_done(q, set->ops->complete); 2572 2573 blk_mq_init_cpu_queues(q, set->nr_hw_queues); 2574 blk_mq_add_queue_tag_set(set, q); 2575 blk_mq_map_swqueue(q); 2576 2577 if (!(set->flags & BLK_MQ_F_NO_SCHED)) { 2578 int ret; 2579 2580 ret = elevator_init_mq(q); 2581 if (ret) 2582 return ERR_PTR(ret); 2583 } 2584 2585 return q; 2586 2587 err_hctxs: 2588 kfree(q->queue_hw_ctx); 2589 err_percpu: 2590 free_percpu(q->queue_ctx); 2591 err_exit: 2592 q->mq_ops = NULL; 2593 return ERR_PTR(-ENOMEM); 2594 } 2595 EXPORT_SYMBOL(blk_mq_init_allocated_queue); 2596 2597 void blk_mq_free_queue(struct request_queue *q) 2598 { 2599 struct blk_mq_tag_set *set = q->tag_set; 2600 2601 blk_mq_del_queue_tag_set(q); 2602 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); 2603 } 2604 2605 /* Basically redo blk_mq_init_queue with queue frozen */ 2606 static void blk_mq_queue_reinit(struct request_queue *q) 2607 { 2608 WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth)); 2609 2610 blk_mq_debugfs_unregister_hctxs(q); 2611 blk_mq_sysfs_unregister(q); 2612 2613 /* 2614 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe 2615 * we should change hctx numa_node according to the new topology (this 2616 * involves freeing and re-allocating memory, worth doing?) 2617 */ 2618 blk_mq_map_swqueue(q); 2619 2620 blk_mq_sysfs_register(q); 2621 blk_mq_debugfs_register_hctxs(q); 2622 } 2623 2624 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2625 { 2626 int i; 2627 2628 for (i = 0; i < set->nr_hw_queues; i++) 2629 if (!__blk_mq_alloc_rq_map(set, i)) 2630 goto out_unwind; 2631 2632 return 0; 2633 2634 out_unwind: 2635 while (--i >= 0) 2636 blk_mq_free_rq_map(set->tags[i]); 2637 2638 return -ENOMEM; 2639 } 2640 2641 /* 2642 * Allocate the request maps associated with this tag_set. Note that this 2643 * may reduce the depth asked for, if memory is tight. set->queue_depth 2644 * will be updated to reflect the allocated depth. 2645 */ 2646 static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2647 { 2648 unsigned int depth; 2649 int err; 2650 2651 depth = set->queue_depth; 2652 do { 2653 err = __blk_mq_alloc_rq_maps(set); 2654 if (!err) 2655 break; 2656 2657 set->queue_depth >>= 1; 2658 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) { 2659 err = -ENOMEM; 2660 break; 2661 } 2662 } while (set->queue_depth); 2663 2664 if (!set->queue_depth || err) { 2665 pr_err("blk-mq: failed to allocate request map\n"); 2666 return -ENOMEM; 2667 } 2668 2669 if (depth != set->queue_depth) 2670 pr_info("blk-mq: reduced tag depth (%u -> %u)\n", 2671 depth, set->queue_depth); 2672 2673 return 0; 2674 } 2675 2676 static int blk_mq_update_queue_map(struct blk_mq_tag_set *set) 2677 { 2678 if (set->ops->map_queues) { 2679 int cpu; 2680 /* 2681 * transport .map_queues is usually done in the following 2682 * way: 2683 * 2684 * for (queue = 0; queue < set->nr_hw_queues; queue++) { 2685 * mask = get_cpu_mask(queue) 2686 * for_each_cpu(cpu, mask) 2687 * set->mq_map[cpu] = queue; 2688 * } 2689 * 2690 * When we need to remap, the table has to be cleared for 2691 * killing stale mapping since one CPU may not be mapped 2692 * to any hw queue. 2693 */ 2694 for_each_possible_cpu(cpu) 2695 set->mq_map[cpu] = 0; 2696 2697 return set->ops->map_queues(set); 2698 } else 2699 return blk_mq_map_queues(set); 2700 } 2701 2702 /* 2703 * Alloc a tag set to be associated with one or more request queues. 2704 * May fail with EINVAL for various error conditions. May adjust the 2705 * requested depth down, if if it too large. In that case, the set 2706 * value will be stored in set->queue_depth. 2707 */ 2708 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) 2709 { 2710 int ret; 2711 2712 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS); 2713 2714 if (!set->nr_hw_queues) 2715 return -EINVAL; 2716 if (!set->queue_depth) 2717 return -EINVAL; 2718 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) 2719 return -EINVAL; 2720 2721 if (!set->ops->queue_rq) 2722 return -EINVAL; 2723 2724 if (!set->ops->get_budget ^ !set->ops->put_budget) 2725 return -EINVAL; 2726 2727 if (set->queue_depth > BLK_MQ_MAX_DEPTH) { 2728 pr_info("blk-mq: reduced tag depth to %u\n", 2729 BLK_MQ_MAX_DEPTH); 2730 set->queue_depth = BLK_MQ_MAX_DEPTH; 2731 } 2732 2733 /* 2734 * If a crashdump is active, then we are potentially in a very 2735 * memory constrained environment. Limit us to 1 queue and 2736 * 64 tags to prevent using too much memory. 2737 */ 2738 if (is_kdump_kernel()) { 2739 set->nr_hw_queues = 1; 2740 set->queue_depth = min(64U, set->queue_depth); 2741 } 2742 /* 2743 * There is no use for more h/w queues than cpus. 2744 */ 2745 if (set->nr_hw_queues > nr_cpu_ids) 2746 set->nr_hw_queues = nr_cpu_ids; 2747 2748 set->tags = kcalloc_node(nr_cpu_ids, sizeof(struct blk_mq_tags *), 2749 GFP_KERNEL, set->numa_node); 2750 if (!set->tags) 2751 return -ENOMEM; 2752 2753 ret = -ENOMEM; 2754 set->mq_map = kcalloc_node(nr_cpu_ids, sizeof(*set->mq_map), 2755 GFP_KERNEL, set->numa_node); 2756 if (!set->mq_map) 2757 goto out_free_tags; 2758 2759 ret = blk_mq_update_queue_map(set); 2760 if (ret) 2761 goto out_free_mq_map; 2762 2763 ret = blk_mq_alloc_rq_maps(set); 2764 if (ret) 2765 goto out_free_mq_map; 2766 2767 mutex_init(&set->tag_list_lock); 2768 INIT_LIST_HEAD(&set->tag_list); 2769 2770 return 0; 2771 2772 out_free_mq_map: 2773 kfree(set->mq_map); 2774 set->mq_map = NULL; 2775 out_free_tags: 2776 kfree(set->tags); 2777 set->tags = NULL; 2778 return ret; 2779 } 2780 EXPORT_SYMBOL(blk_mq_alloc_tag_set); 2781 2782 void blk_mq_free_tag_set(struct blk_mq_tag_set *set) 2783 { 2784 int i; 2785 2786 for (i = 0; i < nr_cpu_ids; i++) 2787 blk_mq_free_map_and_requests(set, i); 2788 2789 kfree(set->mq_map); 2790 set->mq_map = NULL; 2791 2792 kfree(set->tags); 2793 set->tags = NULL; 2794 } 2795 EXPORT_SYMBOL(blk_mq_free_tag_set); 2796 2797 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) 2798 { 2799 struct blk_mq_tag_set *set = q->tag_set; 2800 struct blk_mq_hw_ctx *hctx; 2801 int i, ret; 2802 2803 if (!set) 2804 return -EINVAL; 2805 2806 blk_mq_freeze_queue(q); 2807 blk_mq_quiesce_queue(q); 2808 2809 ret = 0; 2810 queue_for_each_hw_ctx(q, hctx, i) { 2811 if (!hctx->tags) 2812 continue; 2813 /* 2814 * If we're using an MQ scheduler, just update the scheduler 2815 * queue depth. This is similar to what the old code would do. 2816 */ 2817 if (!hctx->sched_tags) { 2818 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr, 2819 false); 2820 } else { 2821 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags, 2822 nr, true); 2823 } 2824 if (ret) 2825 break; 2826 } 2827 2828 if (!ret) 2829 q->nr_requests = nr; 2830 2831 blk_mq_unquiesce_queue(q); 2832 blk_mq_unfreeze_queue(q); 2833 2834 return ret; 2835 } 2836 2837 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, 2838 int nr_hw_queues) 2839 { 2840 struct request_queue *q; 2841 2842 lockdep_assert_held(&set->tag_list_lock); 2843 2844 if (nr_hw_queues > nr_cpu_ids) 2845 nr_hw_queues = nr_cpu_ids; 2846 if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues) 2847 return; 2848 2849 list_for_each_entry(q, &set->tag_list, tag_set_list) 2850 blk_mq_freeze_queue(q); 2851 2852 set->nr_hw_queues = nr_hw_queues; 2853 blk_mq_update_queue_map(set); 2854 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2855 blk_mq_realloc_hw_ctxs(set, q); 2856 blk_mq_queue_reinit(q); 2857 } 2858 2859 list_for_each_entry(q, &set->tag_list, tag_set_list) 2860 blk_mq_unfreeze_queue(q); 2861 } 2862 2863 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) 2864 { 2865 mutex_lock(&set->tag_list_lock); 2866 __blk_mq_update_nr_hw_queues(set, nr_hw_queues); 2867 mutex_unlock(&set->tag_list_lock); 2868 } 2869 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); 2870 2871 /* Enable polling stats and return whether they were already enabled. */ 2872 static bool blk_poll_stats_enable(struct request_queue *q) 2873 { 2874 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 2875 blk_queue_flag_test_and_set(QUEUE_FLAG_POLL_STATS, q)) 2876 return true; 2877 blk_stat_add_callback(q, q->poll_cb); 2878 return false; 2879 } 2880 2881 static void blk_mq_poll_stats_start(struct request_queue *q) 2882 { 2883 /* 2884 * We don't arm the callback if polling stats are not enabled or the 2885 * callback is already active. 2886 */ 2887 if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 2888 blk_stat_is_active(q->poll_cb)) 2889 return; 2890 2891 blk_stat_activate_msecs(q->poll_cb, 100); 2892 } 2893 2894 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb) 2895 { 2896 struct request_queue *q = cb->data; 2897 int bucket; 2898 2899 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) { 2900 if (cb->stat[bucket].nr_samples) 2901 q->poll_stat[bucket] = cb->stat[bucket]; 2902 } 2903 } 2904 2905 static unsigned long blk_mq_poll_nsecs(struct request_queue *q, 2906 struct blk_mq_hw_ctx *hctx, 2907 struct request *rq) 2908 { 2909 unsigned long ret = 0; 2910 int bucket; 2911 2912 /* 2913 * If stats collection isn't on, don't sleep but turn it on for 2914 * future users 2915 */ 2916 if (!blk_poll_stats_enable(q)) 2917 return 0; 2918 2919 /* 2920 * As an optimistic guess, use half of the mean service time 2921 * for this type of request. We can (and should) make this smarter. 2922 * For instance, if the completion latencies are tight, we can 2923 * get closer than just half the mean. This is especially 2924 * important on devices where the completion latencies are longer 2925 * than ~10 usec. We do use the stats for the relevant IO size 2926 * if available which does lead to better estimates. 2927 */ 2928 bucket = blk_mq_poll_stats_bkt(rq); 2929 if (bucket < 0) 2930 return ret; 2931 2932 if (q->poll_stat[bucket].nr_samples) 2933 ret = (q->poll_stat[bucket].mean + 1) / 2; 2934 2935 return ret; 2936 } 2937 2938 static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, 2939 struct blk_mq_hw_ctx *hctx, 2940 struct request *rq) 2941 { 2942 struct hrtimer_sleeper hs; 2943 enum hrtimer_mode mode; 2944 unsigned int nsecs; 2945 ktime_t kt; 2946 2947 if (rq->rq_flags & RQF_MQ_POLL_SLEPT) 2948 return false; 2949 2950 /* 2951 * poll_nsec can be: 2952 * 2953 * -1: don't ever hybrid sleep 2954 * 0: use half of prev avg 2955 * >0: use this specific value 2956 */ 2957 if (q->poll_nsec == -1) 2958 return false; 2959 else if (q->poll_nsec > 0) 2960 nsecs = q->poll_nsec; 2961 else 2962 nsecs = blk_mq_poll_nsecs(q, hctx, rq); 2963 2964 if (!nsecs) 2965 return false; 2966 2967 rq->rq_flags |= RQF_MQ_POLL_SLEPT; 2968 2969 /* 2970 * This will be replaced with the stats tracking code, using 2971 * 'avg_completion_time / 2' as the pre-sleep target. 2972 */ 2973 kt = nsecs; 2974 2975 mode = HRTIMER_MODE_REL; 2976 hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode); 2977 hrtimer_set_expires(&hs.timer, kt); 2978 2979 hrtimer_init_sleeper(&hs, current); 2980 do { 2981 if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE) 2982 break; 2983 set_current_state(TASK_UNINTERRUPTIBLE); 2984 hrtimer_start_expires(&hs.timer, mode); 2985 if (hs.task) 2986 io_schedule(); 2987 hrtimer_cancel(&hs.timer); 2988 mode = HRTIMER_MODE_ABS; 2989 } while (hs.task && !signal_pending(current)); 2990 2991 __set_current_state(TASK_RUNNING); 2992 destroy_hrtimer_on_stack(&hs.timer); 2993 return true; 2994 } 2995 2996 static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq) 2997 { 2998 struct request_queue *q = hctx->queue; 2999 long state; 3000 3001 /* 3002 * If we sleep, have the caller restart the poll loop to reset 3003 * the state. Like for the other success return cases, the 3004 * caller is responsible for checking if the IO completed. If 3005 * the IO isn't complete, we'll get called again and will go 3006 * straight to the busy poll loop. 3007 */ 3008 if (blk_mq_poll_hybrid_sleep(q, hctx, rq)) 3009 return true; 3010 3011 hctx->poll_considered++; 3012 3013 state = current->state; 3014 while (!need_resched()) { 3015 int ret; 3016 3017 hctx->poll_invoked++; 3018 3019 ret = q->mq_ops->poll(hctx, rq->tag); 3020 if (ret > 0) { 3021 hctx->poll_success++; 3022 set_current_state(TASK_RUNNING); 3023 return true; 3024 } 3025 3026 if (signal_pending_state(state, current)) 3027 set_current_state(TASK_RUNNING); 3028 3029 if (current->state == TASK_RUNNING) 3030 return true; 3031 if (ret < 0) 3032 break; 3033 cpu_relax(); 3034 } 3035 3036 __set_current_state(TASK_RUNNING); 3037 return false; 3038 } 3039 3040 static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie) 3041 { 3042 struct blk_mq_hw_ctx *hctx; 3043 struct request *rq; 3044 3045 if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 3046 return false; 3047 3048 hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; 3049 if (!blk_qc_t_is_internal(cookie)) 3050 rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); 3051 else { 3052 rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie)); 3053 /* 3054 * With scheduling, if the request has completed, we'll 3055 * get a NULL return here, as we clear the sched tag when 3056 * that happens. The request still remains valid, like always, 3057 * so we should be safe with just the NULL check. 3058 */ 3059 if (!rq) 3060 return false; 3061 } 3062 3063 return __blk_mq_poll(hctx, rq); 3064 } 3065 3066 static int __init blk_mq_init(void) 3067 { 3068 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, 3069 blk_mq_hctx_notify_dead); 3070 return 0; 3071 } 3072 subsys_initcall(blk_mq_init); 3073