1 /* 2 * Block multiqueue core code 3 * 4 * Copyright (C) 2013-2014 Jens Axboe 5 * Copyright (C) 2013-2014 Christoph Hellwig 6 */ 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/backing-dev.h> 10 #include <linux/bio.h> 11 #include <linux/blkdev.h> 12 #include <linux/kmemleak.h> 13 #include <linux/mm.h> 14 #include <linux/init.h> 15 #include <linux/slab.h> 16 #include <linux/workqueue.h> 17 #include <linux/smp.h> 18 #include <linux/llist.h> 19 #include <linux/list_sort.h> 20 #include <linux/cpu.h> 21 #include <linux/cache.h> 22 #include <linux/sched/sysctl.h> 23 #include <linux/sched/topology.h> 24 #include <linux/sched/signal.h> 25 #include <linux/delay.h> 26 #include <linux/crash_dump.h> 27 #include <linux/prefetch.h> 28 29 #include <trace/events/block.h> 30 31 #include <linux/blk-mq.h> 32 #include "blk.h" 33 #include "blk-mq.h" 34 #include "blk-mq-debugfs.h" 35 #include "blk-mq-tag.h" 36 #include "blk-pm.h" 37 #include "blk-stat.h" 38 #include "blk-mq-sched.h" 39 #include "blk-rq-qos.h" 40 41 static void blk_mq_poll_stats_start(struct request_queue *q); 42 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); 43 44 static int blk_mq_poll_stats_bkt(const struct request *rq) 45 { 46 int ddir, bytes, bucket; 47 48 ddir = rq_data_dir(rq); 49 bytes = blk_rq_bytes(rq); 50 51 bucket = ddir + 2*(ilog2(bytes) - 9); 52 53 if (bucket < 0) 54 return -1; 55 else if (bucket >= BLK_MQ_POLL_STATS_BKTS) 56 return ddir + BLK_MQ_POLL_STATS_BKTS - 2; 57 58 return bucket; 59 } 60 61 /* 62 * Check if any of the ctx's have pending work in this hardware queue 63 */ 64 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) 65 { 66 return !list_empty_careful(&hctx->dispatch) || 67 sbitmap_any_bit_set(&hctx->ctx_map) || 68 blk_mq_sched_has_work(hctx); 69 } 70 71 /* 72 * Mark this ctx as having pending work in this hardware queue 73 */ 74 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, 75 struct blk_mq_ctx *ctx) 76 { 77 const int bit = ctx->index_hw[hctx->type]; 78 79 if (!sbitmap_test_bit(&hctx->ctx_map, bit)) 80 sbitmap_set_bit(&hctx->ctx_map, bit); 81 } 82 83 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, 84 struct blk_mq_ctx *ctx) 85 { 86 const int bit = ctx->index_hw[hctx->type]; 87 88 sbitmap_clear_bit(&hctx->ctx_map, bit); 89 } 90 91 struct mq_inflight { 92 struct hd_struct *part; 93 unsigned int *inflight; 94 }; 95 96 static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx, 97 struct request *rq, void *priv, 98 bool reserved) 99 { 100 struct mq_inflight *mi = priv; 101 102 /* 103 * index[0] counts the specific partition that was asked for. 104 */ 105 if (rq->part == mi->part) 106 mi->inflight[0]++; 107 108 return true; 109 } 110 111 unsigned int blk_mq_in_flight(struct request_queue *q, struct hd_struct *part) 112 { 113 unsigned inflight[2]; 114 struct mq_inflight mi = { .part = part, .inflight = inflight, }; 115 116 inflight[0] = inflight[1] = 0; 117 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); 118 119 return inflight[0]; 120 } 121 122 static bool blk_mq_check_inflight_rw(struct blk_mq_hw_ctx *hctx, 123 struct request *rq, void *priv, 124 bool reserved) 125 { 126 struct mq_inflight *mi = priv; 127 128 if (rq->part == mi->part) 129 mi->inflight[rq_data_dir(rq)]++; 130 131 return true; 132 } 133 134 void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part, 135 unsigned int inflight[2]) 136 { 137 struct mq_inflight mi = { .part = part, .inflight = inflight, }; 138 139 inflight[0] = inflight[1] = 0; 140 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight_rw, &mi); 141 } 142 143 void blk_freeze_queue_start(struct request_queue *q) 144 { 145 int freeze_depth; 146 147 freeze_depth = atomic_inc_return(&q->mq_freeze_depth); 148 if (freeze_depth == 1) { 149 percpu_ref_kill(&q->q_usage_counter); 150 if (queue_is_mq(q)) 151 blk_mq_run_hw_queues(q, false); 152 } 153 } 154 EXPORT_SYMBOL_GPL(blk_freeze_queue_start); 155 156 void blk_mq_freeze_queue_wait(struct request_queue *q) 157 { 158 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter)); 159 } 160 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait); 161 162 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, 163 unsigned long timeout) 164 { 165 return wait_event_timeout(q->mq_freeze_wq, 166 percpu_ref_is_zero(&q->q_usage_counter), 167 timeout); 168 } 169 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout); 170 171 /* 172 * Guarantee no request is in use, so we can change any data structure of 173 * the queue afterward. 174 */ 175 void blk_freeze_queue(struct request_queue *q) 176 { 177 /* 178 * In the !blk_mq case we are only calling this to kill the 179 * q_usage_counter, otherwise this increases the freeze depth 180 * and waits for it to return to zero. For this reason there is 181 * no blk_unfreeze_queue(), and blk_freeze_queue() is not 182 * exported to drivers as the only user for unfreeze is blk_mq. 183 */ 184 blk_freeze_queue_start(q); 185 blk_mq_freeze_queue_wait(q); 186 } 187 188 void blk_mq_freeze_queue(struct request_queue *q) 189 { 190 /* 191 * ...just an alias to keep freeze and unfreeze actions balanced 192 * in the blk_mq_* namespace 193 */ 194 blk_freeze_queue(q); 195 } 196 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue); 197 198 void blk_mq_unfreeze_queue(struct request_queue *q) 199 { 200 int freeze_depth; 201 202 freeze_depth = atomic_dec_return(&q->mq_freeze_depth); 203 WARN_ON_ONCE(freeze_depth < 0); 204 if (!freeze_depth) { 205 percpu_ref_resurrect(&q->q_usage_counter); 206 wake_up_all(&q->mq_freeze_wq); 207 } 208 } 209 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); 210 211 /* 212 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the 213 * mpt3sas driver such that this function can be removed. 214 */ 215 void blk_mq_quiesce_queue_nowait(struct request_queue *q) 216 { 217 blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q); 218 } 219 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); 220 221 /** 222 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished 223 * @q: request queue. 224 * 225 * Note: this function does not prevent that the struct request end_io() 226 * callback function is invoked. Once this function is returned, we make 227 * sure no dispatch can happen until the queue is unquiesced via 228 * blk_mq_unquiesce_queue(). 229 */ 230 void blk_mq_quiesce_queue(struct request_queue *q) 231 { 232 struct blk_mq_hw_ctx *hctx; 233 unsigned int i; 234 bool rcu = false; 235 236 blk_mq_quiesce_queue_nowait(q); 237 238 queue_for_each_hw_ctx(q, hctx, i) { 239 if (hctx->flags & BLK_MQ_F_BLOCKING) 240 synchronize_srcu(hctx->srcu); 241 else 242 rcu = true; 243 } 244 if (rcu) 245 synchronize_rcu(); 246 } 247 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue); 248 249 /* 250 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue() 251 * @q: request queue. 252 * 253 * This function recovers queue into the state before quiescing 254 * which is done by blk_mq_quiesce_queue. 255 */ 256 void blk_mq_unquiesce_queue(struct request_queue *q) 257 { 258 blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q); 259 260 /* dispatch requests which are inserted during quiescing */ 261 blk_mq_run_hw_queues(q, true); 262 } 263 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue); 264 265 void blk_mq_wake_waiters(struct request_queue *q) 266 { 267 struct blk_mq_hw_ctx *hctx; 268 unsigned int i; 269 270 queue_for_each_hw_ctx(q, hctx, i) 271 if (blk_mq_hw_queue_mapped(hctx)) 272 blk_mq_tag_wakeup_all(hctx->tags, true); 273 } 274 275 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) 276 { 277 return blk_mq_has_free_tags(hctx->tags); 278 } 279 EXPORT_SYMBOL(blk_mq_can_queue); 280 281 /* 282 * Only need start/end time stamping if we have stats enabled, or using 283 * an IO scheduler. 284 */ 285 static inline bool blk_mq_need_time_stamp(struct request *rq) 286 { 287 return (rq->rq_flags & RQF_IO_STAT) || rq->q->elevator; 288 } 289 290 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, 291 unsigned int tag, unsigned int op) 292 { 293 struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 294 struct request *rq = tags->static_rqs[tag]; 295 req_flags_t rq_flags = 0; 296 297 if (data->flags & BLK_MQ_REQ_INTERNAL) { 298 rq->tag = -1; 299 rq->internal_tag = tag; 300 } else { 301 if (data->hctx->flags & BLK_MQ_F_TAG_SHARED) { 302 rq_flags = RQF_MQ_INFLIGHT; 303 atomic_inc(&data->hctx->nr_active); 304 } 305 rq->tag = tag; 306 rq->internal_tag = -1; 307 data->hctx->tags->rqs[rq->tag] = rq; 308 } 309 310 /* csd/requeue_work/fifo_time is initialized before use */ 311 rq->q = data->q; 312 rq->mq_ctx = data->ctx; 313 rq->mq_hctx = data->hctx; 314 rq->rq_flags = rq_flags; 315 rq->cmd_flags = op; 316 if (data->flags & BLK_MQ_REQ_PREEMPT) 317 rq->rq_flags |= RQF_PREEMPT; 318 if (blk_queue_io_stat(data->q)) 319 rq->rq_flags |= RQF_IO_STAT; 320 INIT_LIST_HEAD(&rq->queuelist); 321 INIT_HLIST_NODE(&rq->hash); 322 RB_CLEAR_NODE(&rq->rb_node); 323 rq->rq_disk = NULL; 324 rq->part = NULL; 325 if (blk_mq_need_time_stamp(rq)) 326 rq->start_time_ns = ktime_get_ns(); 327 else 328 rq->start_time_ns = 0; 329 rq->io_start_time_ns = 0; 330 rq->nr_phys_segments = 0; 331 #if defined(CONFIG_BLK_DEV_INTEGRITY) 332 rq->nr_integrity_segments = 0; 333 #endif 334 rq->special = NULL; 335 /* tag was already set */ 336 rq->extra_len = 0; 337 WRITE_ONCE(rq->deadline, 0); 338 339 rq->timeout = 0; 340 341 rq->end_io = NULL; 342 rq->end_io_data = NULL; 343 rq->next_rq = NULL; 344 345 data->ctx->rq_dispatched[op_is_sync(op)]++; 346 refcount_set(&rq->ref, 1); 347 return rq; 348 } 349 350 static struct request *blk_mq_get_request(struct request_queue *q, 351 struct bio *bio, 352 struct blk_mq_alloc_data *data) 353 { 354 struct elevator_queue *e = q->elevator; 355 struct request *rq; 356 unsigned int tag; 357 bool put_ctx_on_error = false; 358 359 blk_queue_enter_live(q); 360 data->q = q; 361 if (likely(!data->ctx)) { 362 data->ctx = blk_mq_get_ctx(q); 363 put_ctx_on_error = true; 364 } 365 if (likely(!data->hctx)) 366 data->hctx = blk_mq_map_queue(q, data->cmd_flags, 367 data->ctx->cpu); 368 if (data->cmd_flags & REQ_NOWAIT) 369 data->flags |= BLK_MQ_REQ_NOWAIT; 370 371 if (e) { 372 data->flags |= BLK_MQ_REQ_INTERNAL; 373 374 /* 375 * Flush requests are special and go directly to the 376 * dispatch list. Don't include reserved tags in the 377 * limiting, as it isn't useful. 378 */ 379 if (!op_is_flush(data->cmd_flags) && 380 e->type->ops.limit_depth && 381 !(data->flags & BLK_MQ_REQ_RESERVED)) 382 e->type->ops.limit_depth(data->cmd_flags, data); 383 } else { 384 blk_mq_tag_busy(data->hctx); 385 } 386 387 tag = blk_mq_get_tag(data); 388 if (tag == BLK_MQ_TAG_FAIL) { 389 if (put_ctx_on_error) { 390 blk_mq_put_ctx(data->ctx); 391 data->ctx = NULL; 392 } 393 blk_queue_exit(q); 394 return NULL; 395 } 396 397 rq = blk_mq_rq_ctx_init(data, tag, data->cmd_flags); 398 if (!op_is_flush(data->cmd_flags)) { 399 rq->elv.icq = NULL; 400 if (e && e->type->ops.prepare_request) { 401 if (e->type->icq_cache) 402 blk_mq_sched_assign_ioc(rq); 403 404 e->type->ops.prepare_request(rq, bio); 405 rq->rq_flags |= RQF_ELVPRIV; 406 } 407 } 408 data->hctx->queued++; 409 return rq; 410 } 411 412 struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, 413 blk_mq_req_flags_t flags) 414 { 415 struct blk_mq_alloc_data alloc_data = { .flags = flags, .cmd_flags = op }; 416 struct request *rq; 417 int ret; 418 419 ret = blk_queue_enter(q, flags); 420 if (ret) 421 return ERR_PTR(ret); 422 423 rq = blk_mq_get_request(q, NULL, &alloc_data); 424 blk_queue_exit(q); 425 426 if (!rq) 427 return ERR_PTR(-EWOULDBLOCK); 428 429 blk_mq_put_ctx(alloc_data.ctx); 430 431 rq->__data_len = 0; 432 rq->__sector = (sector_t) -1; 433 rq->bio = rq->biotail = NULL; 434 return rq; 435 } 436 EXPORT_SYMBOL(blk_mq_alloc_request); 437 438 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, 439 unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx) 440 { 441 struct blk_mq_alloc_data alloc_data = { .flags = flags, .cmd_flags = op }; 442 struct request *rq; 443 unsigned int cpu; 444 int ret; 445 446 /* 447 * If the tag allocator sleeps we could get an allocation for a 448 * different hardware context. No need to complicate the low level 449 * allocator for this for the rare use case of a command tied to 450 * a specific queue. 451 */ 452 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT))) 453 return ERR_PTR(-EINVAL); 454 455 if (hctx_idx >= q->nr_hw_queues) 456 return ERR_PTR(-EIO); 457 458 ret = blk_queue_enter(q, flags); 459 if (ret) 460 return ERR_PTR(ret); 461 462 /* 463 * Check if the hardware context is actually mapped to anything. 464 * If not tell the caller that it should skip this queue. 465 */ 466 alloc_data.hctx = q->queue_hw_ctx[hctx_idx]; 467 if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) { 468 blk_queue_exit(q); 469 return ERR_PTR(-EXDEV); 470 } 471 cpu = cpumask_first_and(alloc_data.hctx->cpumask, cpu_online_mask); 472 alloc_data.ctx = __blk_mq_get_ctx(q, cpu); 473 474 rq = blk_mq_get_request(q, NULL, &alloc_data); 475 blk_queue_exit(q); 476 477 if (!rq) 478 return ERR_PTR(-EWOULDBLOCK); 479 480 return rq; 481 } 482 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); 483 484 static void __blk_mq_free_request(struct request *rq) 485 { 486 struct request_queue *q = rq->q; 487 struct blk_mq_ctx *ctx = rq->mq_ctx; 488 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 489 const int sched_tag = rq->internal_tag; 490 491 blk_pm_mark_last_busy(rq); 492 rq->mq_hctx = NULL; 493 if (rq->tag != -1) 494 blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag); 495 if (sched_tag != -1) 496 blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag); 497 blk_mq_sched_restart(hctx); 498 blk_queue_exit(q); 499 } 500 501 void blk_mq_free_request(struct request *rq) 502 { 503 struct request_queue *q = rq->q; 504 struct elevator_queue *e = q->elevator; 505 struct blk_mq_ctx *ctx = rq->mq_ctx; 506 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 507 508 if (rq->rq_flags & RQF_ELVPRIV) { 509 if (e && e->type->ops.finish_request) 510 e->type->ops.finish_request(rq); 511 if (rq->elv.icq) { 512 put_io_context(rq->elv.icq->ioc); 513 rq->elv.icq = NULL; 514 } 515 } 516 517 ctx->rq_completed[rq_is_sync(rq)]++; 518 if (rq->rq_flags & RQF_MQ_INFLIGHT) 519 atomic_dec(&hctx->nr_active); 520 521 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq))) 522 laptop_io_completion(q->backing_dev_info); 523 524 rq_qos_done(q, rq); 525 526 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 527 if (refcount_dec_and_test(&rq->ref)) 528 __blk_mq_free_request(rq); 529 } 530 EXPORT_SYMBOL_GPL(blk_mq_free_request); 531 532 inline void __blk_mq_end_request(struct request *rq, blk_status_t error) 533 { 534 u64 now = 0; 535 536 if (blk_mq_need_time_stamp(rq)) 537 now = ktime_get_ns(); 538 539 if (rq->rq_flags & RQF_STATS) { 540 blk_mq_poll_stats_start(rq->q); 541 blk_stat_add(rq, now); 542 } 543 544 if (rq->internal_tag != -1) 545 blk_mq_sched_completed_request(rq, now); 546 547 blk_account_io_done(rq, now); 548 549 if (rq->end_io) { 550 rq_qos_done(rq->q, rq); 551 rq->end_io(rq, error); 552 } else { 553 if (unlikely(blk_bidi_rq(rq))) 554 blk_mq_free_request(rq->next_rq); 555 blk_mq_free_request(rq); 556 } 557 } 558 EXPORT_SYMBOL(__blk_mq_end_request); 559 560 void blk_mq_end_request(struct request *rq, blk_status_t error) 561 { 562 if (blk_update_request(rq, error, blk_rq_bytes(rq))) 563 BUG(); 564 __blk_mq_end_request(rq, error); 565 } 566 EXPORT_SYMBOL(blk_mq_end_request); 567 568 static void __blk_mq_complete_request_remote(void *data) 569 { 570 struct request *rq = data; 571 struct request_queue *q = rq->q; 572 573 q->mq_ops->complete(rq); 574 } 575 576 static void __blk_mq_complete_request(struct request *rq) 577 { 578 struct blk_mq_ctx *ctx = rq->mq_ctx; 579 struct request_queue *q = rq->q; 580 bool shared = false; 581 int cpu; 582 583 WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); 584 /* 585 * Most of single queue controllers, there is only one irq vector 586 * for handling IO completion, and the only irq's affinity is set 587 * as all possible CPUs. On most of ARCHs, this affinity means the 588 * irq is handled on one specific CPU. 589 * 590 * So complete IO reqeust in softirq context in case of single queue 591 * for not degrading IO performance by irqsoff latency. 592 */ 593 if (q->nr_hw_queues == 1) { 594 __blk_complete_request(rq); 595 return; 596 } 597 598 /* 599 * For a polled request, always complete locallly, it's pointless 600 * to redirect the completion. 601 */ 602 if ((rq->cmd_flags & REQ_HIPRI) || 603 !test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags)) { 604 q->mq_ops->complete(rq); 605 return; 606 } 607 608 cpu = get_cpu(); 609 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags)) 610 shared = cpus_share_cache(cpu, ctx->cpu); 611 612 if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { 613 rq->csd.func = __blk_mq_complete_request_remote; 614 rq->csd.info = rq; 615 rq->csd.flags = 0; 616 smp_call_function_single_async(ctx->cpu, &rq->csd); 617 } else { 618 q->mq_ops->complete(rq); 619 } 620 put_cpu(); 621 } 622 623 static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx) 624 __releases(hctx->srcu) 625 { 626 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) 627 rcu_read_unlock(); 628 else 629 srcu_read_unlock(hctx->srcu, srcu_idx); 630 } 631 632 static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx) 633 __acquires(hctx->srcu) 634 { 635 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { 636 /* shut up gcc false positive */ 637 *srcu_idx = 0; 638 rcu_read_lock(); 639 } else 640 *srcu_idx = srcu_read_lock(hctx->srcu); 641 } 642 643 /** 644 * blk_mq_complete_request - end I/O on a request 645 * @rq: the request being processed 646 * 647 * Description: 648 * Ends all I/O on a request. It does not handle partial completions. 649 * The actual completion happens out-of-order, through a IPI handler. 650 **/ 651 bool blk_mq_complete_request(struct request *rq) 652 { 653 if (unlikely(blk_should_fake_timeout(rq->q))) 654 return false; 655 __blk_mq_complete_request(rq); 656 return true; 657 } 658 EXPORT_SYMBOL(blk_mq_complete_request); 659 660 int blk_mq_request_started(struct request *rq) 661 { 662 return blk_mq_rq_state(rq) != MQ_RQ_IDLE; 663 } 664 EXPORT_SYMBOL_GPL(blk_mq_request_started); 665 666 void blk_mq_start_request(struct request *rq) 667 { 668 struct request_queue *q = rq->q; 669 670 blk_mq_sched_started_request(rq); 671 672 trace_block_rq_issue(q, rq); 673 674 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 675 rq->io_start_time_ns = ktime_get_ns(); 676 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW 677 rq->throtl_size = blk_rq_sectors(rq); 678 #endif 679 rq->rq_flags |= RQF_STATS; 680 rq_qos_issue(q, rq); 681 } 682 683 WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE); 684 685 blk_add_timer(rq); 686 WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT); 687 688 if (q->dma_drain_size && blk_rq_bytes(rq)) { 689 /* 690 * Make sure space for the drain appears. We know we can do 691 * this because max_hw_segments has been adjusted to be one 692 * fewer than the device can handle. 693 */ 694 rq->nr_phys_segments++; 695 } 696 } 697 EXPORT_SYMBOL(blk_mq_start_request); 698 699 static void __blk_mq_requeue_request(struct request *rq) 700 { 701 struct request_queue *q = rq->q; 702 703 blk_mq_put_driver_tag(rq); 704 705 trace_block_rq_requeue(q, rq); 706 rq_qos_requeue(q, rq); 707 708 if (blk_mq_request_started(rq)) { 709 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 710 rq->rq_flags &= ~RQF_TIMED_OUT; 711 if (q->dma_drain_size && blk_rq_bytes(rq)) 712 rq->nr_phys_segments--; 713 } 714 } 715 716 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) 717 { 718 __blk_mq_requeue_request(rq); 719 720 /* this request will be re-inserted to io scheduler queue */ 721 blk_mq_sched_requeue_request(rq); 722 723 BUG_ON(!list_empty(&rq->queuelist)); 724 blk_mq_add_to_requeue_list(rq, true, kick_requeue_list); 725 } 726 EXPORT_SYMBOL(blk_mq_requeue_request); 727 728 static void blk_mq_requeue_work(struct work_struct *work) 729 { 730 struct request_queue *q = 731 container_of(work, struct request_queue, requeue_work.work); 732 LIST_HEAD(rq_list); 733 struct request *rq, *next; 734 735 spin_lock_irq(&q->requeue_lock); 736 list_splice_init(&q->requeue_list, &rq_list); 737 spin_unlock_irq(&q->requeue_lock); 738 739 list_for_each_entry_safe(rq, next, &rq_list, queuelist) { 740 if (!(rq->rq_flags & RQF_SOFTBARRIER)) 741 continue; 742 743 rq->rq_flags &= ~RQF_SOFTBARRIER; 744 list_del_init(&rq->queuelist); 745 blk_mq_sched_insert_request(rq, true, false, false); 746 } 747 748 while (!list_empty(&rq_list)) { 749 rq = list_entry(rq_list.next, struct request, queuelist); 750 list_del_init(&rq->queuelist); 751 blk_mq_sched_insert_request(rq, false, false, false); 752 } 753 754 blk_mq_run_hw_queues(q, false); 755 } 756 757 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, 758 bool kick_requeue_list) 759 { 760 struct request_queue *q = rq->q; 761 unsigned long flags; 762 763 /* 764 * We abuse this flag that is otherwise used by the I/O scheduler to 765 * request head insertion from the workqueue. 766 */ 767 BUG_ON(rq->rq_flags & RQF_SOFTBARRIER); 768 769 spin_lock_irqsave(&q->requeue_lock, flags); 770 if (at_head) { 771 rq->rq_flags |= RQF_SOFTBARRIER; 772 list_add(&rq->queuelist, &q->requeue_list); 773 } else { 774 list_add_tail(&rq->queuelist, &q->requeue_list); 775 } 776 spin_unlock_irqrestore(&q->requeue_lock, flags); 777 778 if (kick_requeue_list) 779 blk_mq_kick_requeue_list(q); 780 } 781 EXPORT_SYMBOL(blk_mq_add_to_requeue_list); 782 783 void blk_mq_kick_requeue_list(struct request_queue *q) 784 { 785 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0); 786 } 787 EXPORT_SYMBOL(blk_mq_kick_requeue_list); 788 789 void blk_mq_delay_kick_requeue_list(struct request_queue *q, 790 unsigned long msecs) 791 { 792 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 793 msecs_to_jiffies(msecs)); 794 } 795 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); 796 797 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) 798 { 799 if (tag < tags->nr_tags) { 800 prefetch(tags->rqs[tag]); 801 return tags->rqs[tag]; 802 } 803 804 return NULL; 805 } 806 EXPORT_SYMBOL(blk_mq_tag_to_rq); 807 808 static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq, 809 void *priv, bool reserved) 810 { 811 /* 812 * If we find a request that is inflight and the queue matches, 813 * we know the queue is busy. Return false to stop the iteration. 814 */ 815 if (rq->state == MQ_RQ_IN_FLIGHT && rq->q == hctx->queue) { 816 bool *busy = priv; 817 818 *busy = true; 819 return false; 820 } 821 822 return true; 823 } 824 825 bool blk_mq_queue_inflight(struct request_queue *q) 826 { 827 bool busy = false; 828 829 blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy); 830 return busy; 831 } 832 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight); 833 834 static void blk_mq_rq_timed_out(struct request *req, bool reserved) 835 { 836 req->rq_flags |= RQF_TIMED_OUT; 837 if (req->q->mq_ops->timeout) { 838 enum blk_eh_timer_return ret; 839 840 ret = req->q->mq_ops->timeout(req, reserved); 841 if (ret == BLK_EH_DONE) 842 return; 843 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER); 844 } 845 846 blk_add_timer(req); 847 } 848 849 static bool blk_mq_req_expired(struct request *rq, unsigned long *next) 850 { 851 unsigned long deadline; 852 853 if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT) 854 return false; 855 if (rq->rq_flags & RQF_TIMED_OUT) 856 return false; 857 858 deadline = READ_ONCE(rq->deadline); 859 if (time_after_eq(jiffies, deadline)) 860 return true; 861 862 if (*next == 0) 863 *next = deadline; 864 else if (time_after(*next, deadline)) 865 *next = deadline; 866 return false; 867 } 868 869 static bool blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, 870 struct request *rq, void *priv, bool reserved) 871 { 872 unsigned long *next = priv; 873 874 /* 875 * Just do a quick check if it is expired before locking the request in 876 * so we're not unnecessarilly synchronizing across CPUs. 877 */ 878 if (!blk_mq_req_expired(rq, next)) 879 return true; 880 881 /* 882 * We have reason to believe the request may be expired. Take a 883 * reference on the request to lock this request lifetime into its 884 * currently allocated context to prevent it from being reallocated in 885 * the event the completion by-passes this timeout handler. 886 * 887 * If the reference was already released, then the driver beat the 888 * timeout handler to posting a natural completion. 889 */ 890 if (!refcount_inc_not_zero(&rq->ref)) 891 return true; 892 893 /* 894 * The request is now locked and cannot be reallocated underneath the 895 * timeout handler's processing. Re-verify this exact request is truly 896 * expired; if it is not expired, then the request was completed and 897 * reallocated as a new request. 898 */ 899 if (blk_mq_req_expired(rq, next)) 900 blk_mq_rq_timed_out(rq, reserved); 901 if (refcount_dec_and_test(&rq->ref)) 902 __blk_mq_free_request(rq); 903 904 return true; 905 } 906 907 static void blk_mq_timeout_work(struct work_struct *work) 908 { 909 struct request_queue *q = 910 container_of(work, struct request_queue, timeout_work); 911 unsigned long next = 0; 912 struct blk_mq_hw_ctx *hctx; 913 int i; 914 915 /* A deadlock might occur if a request is stuck requiring a 916 * timeout at the same time a queue freeze is waiting 917 * completion, since the timeout code would not be able to 918 * acquire the queue reference here. 919 * 920 * That's why we don't use blk_queue_enter here; instead, we use 921 * percpu_ref_tryget directly, because we need to be able to 922 * obtain a reference even in the short window between the queue 923 * starting to freeze, by dropping the first reference in 924 * blk_freeze_queue_start, and the moment the last request is 925 * consumed, marked by the instant q_usage_counter reaches 926 * zero. 927 */ 928 if (!percpu_ref_tryget(&q->q_usage_counter)) 929 return; 930 931 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next); 932 933 if (next != 0) { 934 mod_timer(&q->timeout, next); 935 } else { 936 /* 937 * Request timeouts are handled as a forward rolling timer. If 938 * we end up here it means that no requests are pending and 939 * also that no request has been pending for a while. Mark 940 * each hctx as idle. 941 */ 942 queue_for_each_hw_ctx(q, hctx, i) { 943 /* the hctx may be unmapped, so check it here */ 944 if (blk_mq_hw_queue_mapped(hctx)) 945 blk_mq_tag_idle(hctx); 946 } 947 } 948 blk_queue_exit(q); 949 } 950 951 struct flush_busy_ctx_data { 952 struct blk_mq_hw_ctx *hctx; 953 struct list_head *list; 954 }; 955 956 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) 957 { 958 struct flush_busy_ctx_data *flush_data = data; 959 struct blk_mq_hw_ctx *hctx = flush_data->hctx; 960 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 961 enum hctx_type type = hctx->type; 962 963 spin_lock(&ctx->lock); 964 list_splice_tail_init(&ctx->rq_lists[type], flush_data->list); 965 sbitmap_clear_bit(sb, bitnr); 966 spin_unlock(&ctx->lock); 967 return true; 968 } 969 970 /* 971 * Process software queues that have been marked busy, splicing them 972 * to the for-dispatch 973 */ 974 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) 975 { 976 struct flush_busy_ctx_data data = { 977 .hctx = hctx, 978 .list = list, 979 }; 980 981 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data); 982 } 983 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs); 984 985 struct dispatch_rq_data { 986 struct blk_mq_hw_ctx *hctx; 987 struct request *rq; 988 }; 989 990 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr, 991 void *data) 992 { 993 struct dispatch_rq_data *dispatch_data = data; 994 struct blk_mq_hw_ctx *hctx = dispatch_data->hctx; 995 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 996 enum hctx_type type = hctx->type; 997 998 spin_lock(&ctx->lock); 999 if (!list_empty(&ctx->rq_lists[type])) { 1000 dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next); 1001 list_del_init(&dispatch_data->rq->queuelist); 1002 if (list_empty(&ctx->rq_lists[type])) 1003 sbitmap_clear_bit(sb, bitnr); 1004 } 1005 spin_unlock(&ctx->lock); 1006 1007 return !dispatch_data->rq; 1008 } 1009 1010 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, 1011 struct blk_mq_ctx *start) 1012 { 1013 unsigned off = start ? start->index_hw[hctx->type] : 0; 1014 struct dispatch_rq_data data = { 1015 .hctx = hctx, 1016 .rq = NULL, 1017 }; 1018 1019 __sbitmap_for_each_set(&hctx->ctx_map, off, 1020 dispatch_rq_from_ctx, &data); 1021 1022 return data.rq; 1023 } 1024 1025 static inline unsigned int queued_to_index(unsigned int queued) 1026 { 1027 if (!queued) 1028 return 0; 1029 1030 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); 1031 } 1032 1033 bool blk_mq_get_driver_tag(struct request *rq) 1034 { 1035 struct blk_mq_alloc_data data = { 1036 .q = rq->q, 1037 .hctx = rq->mq_hctx, 1038 .flags = BLK_MQ_REQ_NOWAIT, 1039 .cmd_flags = rq->cmd_flags, 1040 }; 1041 bool shared; 1042 1043 if (rq->tag != -1) 1044 goto done; 1045 1046 if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag)) 1047 data.flags |= BLK_MQ_REQ_RESERVED; 1048 1049 shared = blk_mq_tag_busy(data.hctx); 1050 rq->tag = blk_mq_get_tag(&data); 1051 if (rq->tag >= 0) { 1052 if (shared) { 1053 rq->rq_flags |= RQF_MQ_INFLIGHT; 1054 atomic_inc(&data.hctx->nr_active); 1055 } 1056 data.hctx->tags->rqs[rq->tag] = rq; 1057 } 1058 1059 done: 1060 return rq->tag != -1; 1061 } 1062 1063 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, 1064 int flags, void *key) 1065 { 1066 struct blk_mq_hw_ctx *hctx; 1067 1068 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); 1069 1070 spin_lock(&hctx->dispatch_wait_lock); 1071 list_del_init(&wait->entry); 1072 spin_unlock(&hctx->dispatch_wait_lock); 1073 1074 blk_mq_run_hw_queue(hctx, true); 1075 return 1; 1076 } 1077 1078 /* 1079 * Mark us waiting for a tag. For shared tags, this involves hooking us into 1080 * the tag wakeups. For non-shared tags, we can simply mark us needing a 1081 * restart. For both cases, take care to check the condition again after 1082 * marking us as waiting. 1083 */ 1084 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx, 1085 struct request *rq) 1086 { 1087 struct wait_queue_head *wq; 1088 wait_queue_entry_t *wait; 1089 bool ret; 1090 1091 if (!(hctx->flags & BLK_MQ_F_TAG_SHARED)) { 1092 if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) 1093 set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); 1094 1095 /* 1096 * It's possible that a tag was freed in the window between the 1097 * allocation failure and adding the hardware queue to the wait 1098 * queue. 1099 * 1100 * Don't clear RESTART here, someone else could have set it. 1101 * At most this will cost an extra queue run. 1102 */ 1103 return blk_mq_get_driver_tag(rq); 1104 } 1105 1106 wait = &hctx->dispatch_wait; 1107 if (!list_empty_careful(&wait->entry)) 1108 return false; 1109 1110 wq = &bt_wait_ptr(&hctx->tags->bitmap_tags, hctx)->wait; 1111 1112 spin_lock_irq(&wq->lock); 1113 spin_lock(&hctx->dispatch_wait_lock); 1114 if (!list_empty(&wait->entry)) { 1115 spin_unlock(&hctx->dispatch_wait_lock); 1116 spin_unlock_irq(&wq->lock); 1117 return false; 1118 } 1119 1120 wait->flags &= ~WQ_FLAG_EXCLUSIVE; 1121 __add_wait_queue(wq, wait); 1122 1123 /* 1124 * It's possible that a tag was freed in the window between the 1125 * allocation failure and adding the hardware queue to the wait 1126 * queue. 1127 */ 1128 ret = blk_mq_get_driver_tag(rq); 1129 if (!ret) { 1130 spin_unlock(&hctx->dispatch_wait_lock); 1131 spin_unlock_irq(&wq->lock); 1132 return false; 1133 } 1134 1135 /* 1136 * We got a tag, remove ourselves from the wait queue to ensure 1137 * someone else gets the wakeup. 1138 */ 1139 list_del_init(&wait->entry); 1140 spin_unlock(&hctx->dispatch_wait_lock); 1141 spin_unlock_irq(&wq->lock); 1142 1143 return true; 1144 } 1145 1146 #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT 8 1147 #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR 4 1148 /* 1149 * Update dispatch busy with the Exponential Weighted Moving Average(EWMA): 1150 * - EWMA is one simple way to compute running average value 1151 * - weight(7/8 and 1/8) is applied so that it can decrease exponentially 1152 * - take 4 as factor for avoiding to get too small(0) result, and this 1153 * factor doesn't matter because EWMA decreases exponentially 1154 */ 1155 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy) 1156 { 1157 unsigned int ewma; 1158 1159 if (hctx->queue->elevator) 1160 return; 1161 1162 ewma = hctx->dispatch_busy; 1163 1164 if (!ewma && !busy) 1165 return; 1166 1167 ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1; 1168 if (busy) 1169 ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR; 1170 ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT; 1171 1172 hctx->dispatch_busy = ewma; 1173 } 1174 1175 #define BLK_MQ_RESOURCE_DELAY 3 /* ms units */ 1176 1177 /* 1178 * Returns true if we did some work AND can potentially do more. 1179 */ 1180 bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, 1181 bool got_budget) 1182 { 1183 struct blk_mq_hw_ctx *hctx; 1184 struct request *rq, *nxt; 1185 bool no_tag = false; 1186 int errors, queued; 1187 blk_status_t ret = BLK_STS_OK; 1188 1189 if (list_empty(list)) 1190 return false; 1191 1192 WARN_ON(!list_is_singular(list) && got_budget); 1193 1194 /* 1195 * Now process all the entries, sending them to the driver. 1196 */ 1197 errors = queued = 0; 1198 do { 1199 struct blk_mq_queue_data bd; 1200 1201 rq = list_first_entry(list, struct request, queuelist); 1202 1203 hctx = rq->mq_hctx; 1204 if (!got_budget && !blk_mq_get_dispatch_budget(hctx)) 1205 break; 1206 1207 if (!blk_mq_get_driver_tag(rq)) { 1208 /* 1209 * The initial allocation attempt failed, so we need to 1210 * rerun the hardware queue when a tag is freed. The 1211 * waitqueue takes care of that. If the queue is run 1212 * before we add this entry back on the dispatch list, 1213 * we'll re-run it below. 1214 */ 1215 if (!blk_mq_mark_tag_wait(hctx, rq)) { 1216 blk_mq_put_dispatch_budget(hctx); 1217 /* 1218 * For non-shared tags, the RESTART check 1219 * will suffice. 1220 */ 1221 if (hctx->flags & BLK_MQ_F_TAG_SHARED) 1222 no_tag = true; 1223 break; 1224 } 1225 } 1226 1227 list_del_init(&rq->queuelist); 1228 1229 bd.rq = rq; 1230 1231 /* 1232 * Flag last if we have no more requests, or if we have more 1233 * but can't assign a driver tag to it. 1234 */ 1235 if (list_empty(list)) 1236 bd.last = true; 1237 else { 1238 nxt = list_first_entry(list, struct request, queuelist); 1239 bd.last = !blk_mq_get_driver_tag(nxt); 1240 } 1241 1242 ret = q->mq_ops->queue_rq(hctx, &bd); 1243 if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) { 1244 /* 1245 * If an I/O scheduler has been configured and we got a 1246 * driver tag for the next request already, free it 1247 * again. 1248 */ 1249 if (!list_empty(list)) { 1250 nxt = list_first_entry(list, struct request, queuelist); 1251 blk_mq_put_driver_tag(nxt); 1252 } 1253 list_add(&rq->queuelist, list); 1254 __blk_mq_requeue_request(rq); 1255 break; 1256 } 1257 1258 if (unlikely(ret != BLK_STS_OK)) { 1259 errors++; 1260 blk_mq_end_request(rq, BLK_STS_IOERR); 1261 continue; 1262 } 1263 1264 queued++; 1265 } while (!list_empty(list)); 1266 1267 hctx->dispatched[queued_to_index(queued)]++; 1268 1269 /* 1270 * Any items that need requeuing? Stuff them into hctx->dispatch, 1271 * that is where we will continue on next queue run. 1272 */ 1273 if (!list_empty(list)) { 1274 bool needs_restart; 1275 1276 /* 1277 * If we didn't flush the entire list, we could have told 1278 * the driver there was more coming, but that turned out to 1279 * be a lie. 1280 */ 1281 if (q->mq_ops->commit_rqs) 1282 q->mq_ops->commit_rqs(hctx); 1283 1284 spin_lock(&hctx->lock); 1285 list_splice_init(list, &hctx->dispatch); 1286 spin_unlock(&hctx->lock); 1287 1288 /* 1289 * If SCHED_RESTART was set by the caller of this function and 1290 * it is no longer set that means that it was cleared by another 1291 * thread and hence that a queue rerun is needed. 1292 * 1293 * If 'no_tag' is set, that means that we failed getting 1294 * a driver tag with an I/O scheduler attached. If our dispatch 1295 * waitqueue is no longer active, ensure that we run the queue 1296 * AFTER adding our entries back to the list. 1297 * 1298 * If no I/O scheduler has been configured it is possible that 1299 * the hardware queue got stopped and restarted before requests 1300 * were pushed back onto the dispatch list. Rerun the queue to 1301 * avoid starvation. Notes: 1302 * - blk_mq_run_hw_queue() checks whether or not a queue has 1303 * been stopped before rerunning a queue. 1304 * - Some but not all block drivers stop a queue before 1305 * returning BLK_STS_RESOURCE. Two exceptions are scsi-mq 1306 * and dm-rq. 1307 * 1308 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART 1309 * bit is set, run queue after a delay to avoid IO stalls 1310 * that could otherwise occur if the queue is idle. 1311 */ 1312 needs_restart = blk_mq_sched_needs_restart(hctx); 1313 if (!needs_restart || 1314 (no_tag && list_empty_careful(&hctx->dispatch_wait.entry))) 1315 blk_mq_run_hw_queue(hctx, true); 1316 else if (needs_restart && (ret == BLK_STS_RESOURCE)) 1317 blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY); 1318 1319 blk_mq_update_dispatch_busy(hctx, true); 1320 return false; 1321 } else 1322 blk_mq_update_dispatch_busy(hctx, false); 1323 1324 /* 1325 * If the host/device is unable to accept more work, inform the 1326 * caller of that. 1327 */ 1328 if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) 1329 return false; 1330 1331 return (queued + errors) != 0; 1332 } 1333 1334 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) 1335 { 1336 int srcu_idx; 1337 1338 /* 1339 * We should be running this queue from one of the CPUs that 1340 * are mapped to it. 1341 * 1342 * There are at least two related races now between setting 1343 * hctx->next_cpu from blk_mq_hctx_next_cpu() and running 1344 * __blk_mq_run_hw_queue(): 1345 * 1346 * - hctx->next_cpu is found offline in blk_mq_hctx_next_cpu(), 1347 * but later it becomes online, then this warning is harmless 1348 * at all 1349 * 1350 * - hctx->next_cpu is found online in blk_mq_hctx_next_cpu(), 1351 * but later it becomes offline, then the warning can't be 1352 * triggered, and we depend on blk-mq timeout handler to 1353 * handle dispatched requests to this hctx 1354 */ 1355 if (!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) && 1356 cpu_online(hctx->next_cpu)) { 1357 printk(KERN_WARNING "run queue from wrong CPU %d, hctx %s\n", 1358 raw_smp_processor_id(), 1359 cpumask_empty(hctx->cpumask) ? "inactive": "active"); 1360 dump_stack(); 1361 } 1362 1363 /* 1364 * We can't run the queue inline with ints disabled. Ensure that 1365 * we catch bad users of this early. 1366 */ 1367 WARN_ON_ONCE(in_interrupt()); 1368 1369 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); 1370 1371 hctx_lock(hctx, &srcu_idx); 1372 blk_mq_sched_dispatch_requests(hctx); 1373 hctx_unlock(hctx, srcu_idx); 1374 } 1375 1376 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx) 1377 { 1378 int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask); 1379 1380 if (cpu >= nr_cpu_ids) 1381 cpu = cpumask_first(hctx->cpumask); 1382 return cpu; 1383 } 1384 1385 /* 1386 * It'd be great if the workqueue API had a way to pass 1387 * in a mask and had some smarts for more clever placement. 1388 * For now we just round-robin here, switching for every 1389 * BLK_MQ_CPU_WORK_BATCH queued items. 1390 */ 1391 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) 1392 { 1393 bool tried = false; 1394 int next_cpu = hctx->next_cpu; 1395 1396 if (hctx->queue->nr_hw_queues == 1) 1397 return WORK_CPU_UNBOUND; 1398 1399 if (--hctx->next_cpu_batch <= 0) { 1400 select_cpu: 1401 next_cpu = cpumask_next_and(next_cpu, hctx->cpumask, 1402 cpu_online_mask); 1403 if (next_cpu >= nr_cpu_ids) 1404 next_cpu = blk_mq_first_mapped_cpu(hctx); 1405 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 1406 } 1407 1408 /* 1409 * Do unbound schedule if we can't find a online CPU for this hctx, 1410 * and it should only happen in the path of handling CPU DEAD. 1411 */ 1412 if (!cpu_online(next_cpu)) { 1413 if (!tried) { 1414 tried = true; 1415 goto select_cpu; 1416 } 1417 1418 /* 1419 * Make sure to re-select CPU next time once after CPUs 1420 * in hctx->cpumask become online again. 1421 */ 1422 hctx->next_cpu = next_cpu; 1423 hctx->next_cpu_batch = 1; 1424 return WORK_CPU_UNBOUND; 1425 } 1426 1427 hctx->next_cpu = next_cpu; 1428 return next_cpu; 1429 } 1430 1431 static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, 1432 unsigned long msecs) 1433 { 1434 if (unlikely(blk_mq_hctx_stopped(hctx))) 1435 return; 1436 1437 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) { 1438 int cpu = get_cpu(); 1439 if (cpumask_test_cpu(cpu, hctx->cpumask)) { 1440 __blk_mq_run_hw_queue(hctx); 1441 put_cpu(); 1442 return; 1443 } 1444 1445 put_cpu(); 1446 } 1447 1448 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, 1449 msecs_to_jiffies(msecs)); 1450 } 1451 1452 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) 1453 { 1454 __blk_mq_delay_run_hw_queue(hctx, true, msecs); 1455 } 1456 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); 1457 1458 bool blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1459 { 1460 int srcu_idx; 1461 bool need_run; 1462 1463 /* 1464 * When queue is quiesced, we may be switching io scheduler, or 1465 * updating nr_hw_queues, or other things, and we can't run queue 1466 * any more, even __blk_mq_hctx_has_pending() can't be called safely. 1467 * 1468 * And queue will be rerun in blk_mq_unquiesce_queue() if it is 1469 * quiesced. 1470 */ 1471 hctx_lock(hctx, &srcu_idx); 1472 need_run = !blk_queue_quiesced(hctx->queue) && 1473 blk_mq_hctx_has_pending(hctx); 1474 hctx_unlock(hctx, srcu_idx); 1475 1476 if (need_run) { 1477 __blk_mq_delay_run_hw_queue(hctx, async, 0); 1478 return true; 1479 } 1480 1481 return false; 1482 } 1483 EXPORT_SYMBOL(blk_mq_run_hw_queue); 1484 1485 void blk_mq_run_hw_queues(struct request_queue *q, bool async) 1486 { 1487 struct blk_mq_hw_ctx *hctx; 1488 int i; 1489 1490 queue_for_each_hw_ctx(q, hctx, i) { 1491 if (blk_mq_hctx_stopped(hctx)) 1492 continue; 1493 1494 blk_mq_run_hw_queue(hctx, async); 1495 } 1496 } 1497 EXPORT_SYMBOL(blk_mq_run_hw_queues); 1498 1499 /** 1500 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped 1501 * @q: request queue. 1502 * 1503 * The caller is responsible for serializing this function against 1504 * blk_mq_{start,stop}_hw_queue(). 1505 */ 1506 bool blk_mq_queue_stopped(struct request_queue *q) 1507 { 1508 struct blk_mq_hw_ctx *hctx; 1509 int i; 1510 1511 queue_for_each_hw_ctx(q, hctx, i) 1512 if (blk_mq_hctx_stopped(hctx)) 1513 return true; 1514 1515 return false; 1516 } 1517 EXPORT_SYMBOL(blk_mq_queue_stopped); 1518 1519 /* 1520 * This function is often used for pausing .queue_rq() by driver when 1521 * there isn't enough resource or some conditions aren't satisfied, and 1522 * BLK_STS_RESOURCE is usually returned. 1523 * 1524 * We do not guarantee that dispatch can be drained or blocked 1525 * after blk_mq_stop_hw_queue() returns. Please use 1526 * blk_mq_quiesce_queue() for that requirement. 1527 */ 1528 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) 1529 { 1530 cancel_delayed_work(&hctx->run_work); 1531 1532 set_bit(BLK_MQ_S_STOPPED, &hctx->state); 1533 } 1534 EXPORT_SYMBOL(blk_mq_stop_hw_queue); 1535 1536 /* 1537 * This function is often used for pausing .queue_rq() by driver when 1538 * there isn't enough resource or some conditions aren't satisfied, and 1539 * BLK_STS_RESOURCE is usually returned. 1540 * 1541 * We do not guarantee that dispatch can be drained or blocked 1542 * after blk_mq_stop_hw_queues() returns. Please use 1543 * blk_mq_quiesce_queue() for that requirement. 1544 */ 1545 void blk_mq_stop_hw_queues(struct request_queue *q) 1546 { 1547 struct blk_mq_hw_ctx *hctx; 1548 int i; 1549 1550 queue_for_each_hw_ctx(q, hctx, i) 1551 blk_mq_stop_hw_queue(hctx); 1552 } 1553 EXPORT_SYMBOL(blk_mq_stop_hw_queues); 1554 1555 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) 1556 { 1557 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1558 1559 blk_mq_run_hw_queue(hctx, false); 1560 } 1561 EXPORT_SYMBOL(blk_mq_start_hw_queue); 1562 1563 void blk_mq_start_hw_queues(struct request_queue *q) 1564 { 1565 struct blk_mq_hw_ctx *hctx; 1566 int i; 1567 1568 queue_for_each_hw_ctx(q, hctx, i) 1569 blk_mq_start_hw_queue(hctx); 1570 } 1571 EXPORT_SYMBOL(blk_mq_start_hw_queues); 1572 1573 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1574 { 1575 if (!blk_mq_hctx_stopped(hctx)) 1576 return; 1577 1578 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 1579 blk_mq_run_hw_queue(hctx, async); 1580 } 1581 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue); 1582 1583 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) 1584 { 1585 struct blk_mq_hw_ctx *hctx; 1586 int i; 1587 1588 queue_for_each_hw_ctx(q, hctx, i) 1589 blk_mq_start_stopped_hw_queue(hctx, async); 1590 } 1591 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); 1592 1593 static void blk_mq_run_work_fn(struct work_struct *work) 1594 { 1595 struct blk_mq_hw_ctx *hctx; 1596 1597 hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work); 1598 1599 /* 1600 * If we are stopped, don't run the queue. 1601 */ 1602 if (test_bit(BLK_MQ_S_STOPPED, &hctx->state)) 1603 return; 1604 1605 __blk_mq_run_hw_queue(hctx); 1606 } 1607 1608 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx, 1609 struct request *rq, 1610 bool at_head) 1611 { 1612 struct blk_mq_ctx *ctx = rq->mq_ctx; 1613 enum hctx_type type = hctx->type; 1614 1615 lockdep_assert_held(&ctx->lock); 1616 1617 trace_block_rq_insert(hctx->queue, rq); 1618 1619 if (at_head) 1620 list_add(&rq->queuelist, &ctx->rq_lists[type]); 1621 else 1622 list_add_tail(&rq->queuelist, &ctx->rq_lists[type]); 1623 } 1624 1625 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, 1626 bool at_head) 1627 { 1628 struct blk_mq_ctx *ctx = rq->mq_ctx; 1629 1630 lockdep_assert_held(&ctx->lock); 1631 1632 __blk_mq_insert_req_list(hctx, rq, at_head); 1633 blk_mq_hctx_mark_pending(hctx, ctx); 1634 } 1635 1636 /* 1637 * Should only be used carefully, when the caller knows we want to 1638 * bypass a potential IO scheduler on the target device. 1639 */ 1640 void blk_mq_request_bypass_insert(struct request *rq, bool run_queue) 1641 { 1642 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1643 1644 spin_lock(&hctx->lock); 1645 list_add_tail(&rq->queuelist, &hctx->dispatch); 1646 spin_unlock(&hctx->lock); 1647 1648 if (run_queue) 1649 blk_mq_run_hw_queue(hctx, false); 1650 } 1651 1652 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 1653 struct list_head *list) 1654 1655 { 1656 struct request *rq; 1657 enum hctx_type type = hctx->type; 1658 1659 /* 1660 * preemption doesn't flush plug list, so it's possible ctx->cpu is 1661 * offline now 1662 */ 1663 list_for_each_entry(rq, list, queuelist) { 1664 BUG_ON(rq->mq_ctx != ctx); 1665 trace_block_rq_insert(hctx->queue, rq); 1666 } 1667 1668 spin_lock(&ctx->lock); 1669 list_splice_tail_init(list, &ctx->rq_lists[type]); 1670 blk_mq_hctx_mark_pending(hctx, ctx); 1671 spin_unlock(&ctx->lock); 1672 } 1673 1674 static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b) 1675 { 1676 struct request *rqa = container_of(a, struct request, queuelist); 1677 struct request *rqb = container_of(b, struct request, queuelist); 1678 1679 if (rqa->mq_ctx < rqb->mq_ctx) 1680 return -1; 1681 else if (rqa->mq_ctx > rqb->mq_ctx) 1682 return 1; 1683 else if (rqa->mq_hctx < rqb->mq_hctx) 1684 return -1; 1685 else if (rqa->mq_hctx > rqb->mq_hctx) 1686 return 1; 1687 1688 return blk_rq_pos(rqa) > blk_rq_pos(rqb); 1689 } 1690 1691 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 1692 { 1693 struct blk_mq_hw_ctx *this_hctx; 1694 struct blk_mq_ctx *this_ctx; 1695 struct request_queue *this_q; 1696 struct request *rq; 1697 LIST_HEAD(list); 1698 LIST_HEAD(rq_list); 1699 unsigned int depth; 1700 1701 list_splice_init(&plug->mq_list, &list); 1702 plug->rq_count = 0; 1703 1704 if (plug->rq_count > 2 && plug->multiple_queues) 1705 list_sort(NULL, &list, plug_rq_cmp); 1706 1707 this_q = NULL; 1708 this_hctx = NULL; 1709 this_ctx = NULL; 1710 depth = 0; 1711 1712 while (!list_empty(&list)) { 1713 rq = list_entry_rq(list.next); 1714 list_del_init(&rq->queuelist); 1715 BUG_ON(!rq->q); 1716 if (rq->mq_hctx != this_hctx || rq->mq_ctx != this_ctx) { 1717 if (this_hctx) { 1718 trace_block_unplug(this_q, depth, !from_schedule); 1719 blk_mq_sched_insert_requests(this_hctx, this_ctx, 1720 &rq_list, 1721 from_schedule); 1722 } 1723 1724 this_q = rq->q; 1725 this_ctx = rq->mq_ctx; 1726 this_hctx = rq->mq_hctx; 1727 depth = 0; 1728 } 1729 1730 depth++; 1731 list_add_tail(&rq->queuelist, &rq_list); 1732 } 1733 1734 /* 1735 * If 'this_hctx' is set, we know we have entries to complete 1736 * on 'rq_list'. Do those. 1737 */ 1738 if (this_hctx) { 1739 trace_block_unplug(this_q, depth, !from_schedule); 1740 blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list, 1741 from_schedule); 1742 } 1743 } 1744 1745 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio) 1746 { 1747 blk_init_request_from_bio(rq, bio); 1748 1749 blk_account_io_start(rq, true); 1750 } 1751 1752 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx, 1753 struct request *rq, 1754 blk_qc_t *cookie, bool last) 1755 { 1756 struct request_queue *q = rq->q; 1757 struct blk_mq_queue_data bd = { 1758 .rq = rq, 1759 .last = last, 1760 }; 1761 blk_qc_t new_cookie; 1762 blk_status_t ret; 1763 1764 new_cookie = request_to_qc_t(hctx, rq); 1765 1766 /* 1767 * For OK queue, we are done. For error, caller may kill it. 1768 * Any other error (busy), just add it to our list as we 1769 * previously would have done. 1770 */ 1771 ret = q->mq_ops->queue_rq(hctx, &bd); 1772 switch (ret) { 1773 case BLK_STS_OK: 1774 blk_mq_update_dispatch_busy(hctx, false); 1775 *cookie = new_cookie; 1776 break; 1777 case BLK_STS_RESOURCE: 1778 case BLK_STS_DEV_RESOURCE: 1779 blk_mq_update_dispatch_busy(hctx, true); 1780 __blk_mq_requeue_request(rq); 1781 break; 1782 default: 1783 blk_mq_update_dispatch_busy(hctx, false); 1784 *cookie = BLK_QC_T_NONE; 1785 break; 1786 } 1787 1788 return ret; 1789 } 1790 1791 blk_status_t blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 1792 struct request *rq, 1793 blk_qc_t *cookie, 1794 bool bypass, bool last) 1795 { 1796 struct request_queue *q = rq->q; 1797 bool run_queue = true; 1798 blk_status_t ret = BLK_STS_RESOURCE; 1799 int srcu_idx; 1800 bool force = false; 1801 1802 hctx_lock(hctx, &srcu_idx); 1803 /* 1804 * hctx_lock is needed before checking quiesced flag. 1805 * 1806 * When queue is stopped or quiesced, ignore 'bypass', insert 1807 * and return BLK_STS_OK to caller, and avoid driver to try to 1808 * dispatch again. 1809 */ 1810 if (unlikely(blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q))) { 1811 run_queue = false; 1812 bypass = false; 1813 goto out_unlock; 1814 } 1815 1816 if (unlikely(q->elevator && !bypass)) 1817 goto out_unlock; 1818 1819 if (!blk_mq_get_dispatch_budget(hctx)) 1820 goto out_unlock; 1821 1822 if (!blk_mq_get_driver_tag(rq)) { 1823 blk_mq_put_dispatch_budget(hctx); 1824 goto out_unlock; 1825 } 1826 1827 /* 1828 * Always add a request that has been through 1829 *.queue_rq() to the hardware dispatch list. 1830 */ 1831 force = true; 1832 ret = __blk_mq_issue_directly(hctx, rq, cookie, last); 1833 out_unlock: 1834 hctx_unlock(hctx, srcu_idx); 1835 switch (ret) { 1836 case BLK_STS_OK: 1837 break; 1838 case BLK_STS_DEV_RESOURCE: 1839 case BLK_STS_RESOURCE: 1840 if (force) { 1841 blk_mq_request_bypass_insert(rq, run_queue); 1842 /* 1843 * We have to return BLK_STS_OK for the DM 1844 * to avoid livelock. Otherwise, we return 1845 * the real result to indicate whether the 1846 * request is direct-issued successfully. 1847 */ 1848 ret = bypass ? BLK_STS_OK : ret; 1849 } else if (!bypass) { 1850 blk_mq_sched_insert_request(rq, false, 1851 run_queue, false); 1852 } 1853 break; 1854 default: 1855 if (!bypass) 1856 blk_mq_end_request(rq, ret); 1857 break; 1858 } 1859 1860 return ret; 1861 } 1862 1863 void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, 1864 struct list_head *list) 1865 { 1866 blk_qc_t unused; 1867 blk_status_t ret = BLK_STS_OK; 1868 1869 while (!list_empty(list)) { 1870 struct request *rq = list_first_entry(list, struct request, 1871 queuelist); 1872 1873 list_del_init(&rq->queuelist); 1874 if (ret == BLK_STS_OK) 1875 ret = blk_mq_try_issue_directly(hctx, rq, &unused, 1876 false, 1877 list_empty(list)); 1878 else 1879 blk_mq_sched_insert_request(rq, false, true, false); 1880 } 1881 1882 /* 1883 * If we didn't flush the entire list, we could have told 1884 * the driver there was more coming, but that turned out to 1885 * be a lie. 1886 */ 1887 if (ret != BLK_STS_OK && hctx->queue->mq_ops->commit_rqs) 1888 hctx->queue->mq_ops->commit_rqs(hctx); 1889 } 1890 1891 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) 1892 { 1893 list_add_tail(&rq->queuelist, &plug->mq_list); 1894 plug->rq_count++; 1895 if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) { 1896 struct request *tmp; 1897 1898 tmp = list_first_entry(&plug->mq_list, struct request, 1899 queuelist); 1900 if (tmp->q != rq->q) 1901 plug->multiple_queues = true; 1902 } 1903 } 1904 1905 static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) 1906 { 1907 const int is_sync = op_is_sync(bio->bi_opf); 1908 const int is_flush_fua = op_is_flush(bio->bi_opf); 1909 struct blk_mq_alloc_data data = { .flags = 0}; 1910 struct request *rq; 1911 struct blk_plug *plug; 1912 struct request *same_queue_rq = NULL; 1913 blk_qc_t cookie; 1914 1915 blk_queue_bounce(q, &bio); 1916 1917 blk_queue_split(q, &bio); 1918 1919 if (!bio_integrity_prep(bio)) 1920 return BLK_QC_T_NONE; 1921 1922 if (!is_flush_fua && !blk_queue_nomerges(q) && 1923 blk_attempt_plug_merge(q, bio, &same_queue_rq)) 1924 return BLK_QC_T_NONE; 1925 1926 if (blk_mq_sched_bio_merge(q, bio)) 1927 return BLK_QC_T_NONE; 1928 1929 rq_qos_throttle(q, bio); 1930 1931 data.cmd_flags = bio->bi_opf; 1932 rq = blk_mq_get_request(q, bio, &data); 1933 if (unlikely(!rq)) { 1934 rq_qos_cleanup(q, bio); 1935 if (bio->bi_opf & REQ_NOWAIT) 1936 bio_wouldblock_error(bio); 1937 return BLK_QC_T_NONE; 1938 } 1939 1940 trace_block_getrq(q, bio, bio->bi_opf); 1941 1942 rq_qos_track(q, rq, bio); 1943 1944 cookie = request_to_qc_t(data.hctx, rq); 1945 1946 plug = current->plug; 1947 if (unlikely(is_flush_fua)) { 1948 blk_mq_put_ctx(data.ctx); 1949 blk_mq_bio_to_request(rq, bio); 1950 1951 /* bypass scheduler for flush rq */ 1952 blk_insert_flush(rq); 1953 blk_mq_run_hw_queue(data.hctx, true); 1954 } else if (plug && (q->nr_hw_queues == 1 || q->mq_ops->commit_rqs)) { 1955 /* 1956 * Use plugging if we have a ->commit_rqs() hook as well, as 1957 * we know the driver uses bd->last in a smart fashion. 1958 */ 1959 unsigned int request_count = plug->rq_count; 1960 struct request *last = NULL; 1961 1962 blk_mq_put_ctx(data.ctx); 1963 blk_mq_bio_to_request(rq, bio); 1964 1965 if (!request_count) 1966 trace_block_plug(q); 1967 else 1968 last = list_entry_rq(plug->mq_list.prev); 1969 1970 if (request_count >= BLK_MAX_REQUEST_COUNT || (last && 1971 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { 1972 blk_flush_plug_list(plug, false); 1973 trace_block_plug(q); 1974 } 1975 1976 blk_add_rq_to_plug(plug, rq); 1977 } else if (plug && !blk_queue_nomerges(q)) { 1978 blk_mq_bio_to_request(rq, bio); 1979 1980 /* 1981 * We do limited plugging. If the bio can be merged, do that. 1982 * Otherwise the existing request in the plug list will be 1983 * issued. So the plug list will have one request at most 1984 * The plug list might get flushed before this. If that happens, 1985 * the plug list is empty, and same_queue_rq is invalid. 1986 */ 1987 if (list_empty(&plug->mq_list)) 1988 same_queue_rq = NULL; 1989 if (same_queue_rq) { 1990 list_del_init(&same_queue_rq->queuelist); 1991 plug->rq_count--; 1992 } 1993 blk_add_rq_to_plug(plug, rq); 1994 1995 blk_mq_put_ctx(data.ctx); 1996 1997 if (same_queue_rq) { 1998 data.hctx = same_queue_rq->mq_hctx; 1999 blk_mq_try_issue_directly(data.hctx, same_queue_rq, 2000 &cookie, false, true); 2001 } 2002 } else if ((q->nr_hw_queues > 1 && is_sync) || (!q->elevator && 2003 !data.hctx->dispatch_busy)) { 2004 blk_mq_put_ctx(data.ctx); 2005 blk_mq_bio_to_request(rq, bio); 2006 blk_mq_try_issue_directly(data.hctx, rq, &cookie, false, true); 2007 } else { 2008 blk_mq_put_ctx(data.ctx); 2009 blk_mq_bio_to_request(rq, bio); 2010 blk_mq_sched_insert_request(rq, false, true, true); 2011 } 2012 2013 return cookie; 2014 } 2015 2016 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 2017 unsigned int hctx_idx) 2018 { 2019 struct page *page; 2020 2021 if (tags->rqs && set->ops->exit_request) { 2022 int i; 2023 2024 for (i = 0; i < tags->nr_tags; i++) { 2025 struct request *rq = tags->static_rqs[i]; 2026 2027 if (!rq) 2028 continue; 2029 set->ops->exit_request(set, rq, hctx_idx); 2030 tags->static_rqs[i] = NULL; 2031 } 2032 } 2033 2034 while (!list_empty(&tags->page_list)) { 2035 page = list_first_entry(&tags->page_list, struct page, lru); 2036 list_del_init(&page->lru); 2037 /* 2038 * Remove kmemleak object previously allocated in 2039 * blk_mq_init_rq_map(). 2040 */ 2041 kmemleak_free(page_address(page)); 2042 __free_pages(page, page->private); 2043 } 2044 } 2045 2046 void blk_mq_free_rq_map(struct blk_mq_tags *tags) 2047 { 2048 kfree(tags->rqs); 2049 tags->rqs = NULL; 2050 kfree(tags->static_rqs); 2051 tags->static_rqs = NULL; 2052 2053 blk_mq_free_tags(tags); 2054 } 2055 2056 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, 2057 unsigned int hctx_idx, 2058 unsigned int nr_tags, 2059 unsigned int reserved_tags) 2060 { 2061 struct blk_mq_tags *tags; 2062 int node; 2063 2064 node = blk_mq_hw_queue_to_node(&set->map[0], hctx_idx); 2065 if (node == NUMA_NO_NODE) 2066 node = set->numa_node; 2067 2068 tags = blk_mq_init_tags(nr_tags, reserved_tags, node, 2069 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags)); 2070 if (!tags) 2071 return NULL; 2072 2073 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *), 2074 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 2075 node); 2076 if (!tags->rqs) { 2077 blk_mq_free_tags(tags); 2078 return NULL; 2079 } 2080 2081 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *), 2082 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 2083 node); 2084 if (!tags->static_rqs) { 2085 kfree(tags->rqs); 2086 blk_mq_free_tags(tags); 2087 return NULL; 2088 } 2089 2090 return tags; 2091 } 2092 2093 static size_t order_to_size(unsigned int order) 2094 { 2095 return (size_t)PAGE_SIZE << order; 2096 } 2097 2098 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, 2099 unsigned int hctx_idx, int node) 2100 { 2101 int ret; 2102 2103 if (set->ops->init_request) { 2104 ret = set->ops->init_request(set, rq, hctx_idx, node); 2105 if (ret) 2106 return ret; 2107 } 2108 2109 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 2110 return 0; 2111 } 2112 2113 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 2114 unsigned int hctx_idx, unsigned int depth) 2115 { 2116 unsigned int i, j, entries_per_page, max_order = 4; 2117 size_t rq_size, left; 2118 int node; 2119 2120 node = blk_mq_hw_queue_to_node(&set->map[0], hctx_idx); 2121 if (node == NUMA_NO_NODE) 2122 node = set->numa_node; 2123 2124 INIT_LIST_HEAD(&tags->page_list); 2125 2126 /* 2127 * rq_size is the size of the request plus driver payload, rounded 2128 * to the cacheline size 2129 */ 2130 rq_size = round_up(sizeof(struct request) + set->cmd_size, 2131 cache_line_size()); 2132 left = rq_size * depth; 2133 2134 for (i = 0; i < depth; ) { 2135 int this_order = max_order; 2136 struct page *page; 2137 int to_do; 2138 void *p; 2139 2140 while (this_order && left < order_to_size(this_order - 1)) 2141 this_order--; 2142 2143 do { 2144 page = alloc_pages_node(node, 2145 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, 2146 this_order); 2147 if (page) 2148 break; 2149 if (!this_order--) 2150 break; 2151 if (order_to_size(this_order) < rq_size) 2152 break; 2153 } while (1); 2154 2155 if (!page) 2156 goto fail; 2157 2158 page->private = this_order; 2159 list_add_tail(&page->lru, &tags->page_list); 2160 2161 p = page_address(page); 2162 /* 2163 * Allow kmemleak to scan these pages as they contain pointers 2164 * to additional allocations like via ops->init_request(). 2165 */ 2166 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); 2167 entries_per_page = order_to_size(this_order) / rq_size; 2168 to_do = min(entries_per_page, depth - i); 2169 left -= to_do * rq_size; 2170 for (j = 0; j < to_do; j++) { 2171 struct request *rq = p; 2172 2173 tags->static_rqs[i] = rq; 2174 if (blk_mq_init_request(set, rq, hctx_idx, node)) { 2175 tags->static_rqs[i] = NULL; 2176 goto fail; 2177 } 2178 2179 p += rq_size; 2180 i++; 2181 } 2182 } 2183 return 0; 2184 2185 fail: 2186 blk_mq_free_rqs(set, tags, hctx_idx); 2187 return -ENOMEM; 2188 } 2189 2190 /* 2191 * 'cpu' is going away. splice any existing rq_list entries from this 2192 * software queue to the hw queue dispatch list, and ensure that it 2193 * gets run. 2194 */ 2195 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) 2196 { 2197 struct blk_mq_hw_ctx *hctx; 2198 struct blk_mq_ctx *ctx; 2199 LIST_HEAD(tmp); 2200 enum hctx_type type; 2201 2202 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead); 2203 ctx = __blk_mq_get_ctx(hctx->queue, cpu); 2204 type = hctx->type; 2205 2206 spin_lock(&ctx->lock); 2207 if (!list_empty(&ctx->rq_lists[type])) { 2208 list_splice_init(&ctx->rq_lists[type], &tmp); 2209 blk_mq_hctx_clear_pending(hctx, ctx); 2210 } 2211 spin_unlock(&ctx->lock); 2212 2213 if (list_empty(&tmp)) 2214 return 0; 2215 2216 spin_lock(&hctx->lock); 2217 list_splice_tail_init(&tmp, &hctx->dispatch); 2218 spin_unlock(&hctx->lock); 2219 2220 blk_mq_run_hw_queue(hctx, true); 2221 return 0; 2222 } 2223 2224 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) 2225 { 2226 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD, 2227 &hctx->cpuhp_dead); 2228 } 2229 2230 /* hctx->ctxs will be freed in queue's release handler */ 2231 static void blk_mq_exit_hctx(struct request_queue *q, 2232 struct blk_mq_tag_set *set, 2233 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 2234 { 2235 if (blk_mq_hw_queue_mapped(hctx)) 2236 blk_mq_tag_idle(hctx); 2237 2238 if (set->ops->exit_request) 2239 set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx); 2240 2241 if (set->ops->exit_hctx) 2242 set->ops->exit_hctx(hctx, hctx_idx); 2243 2244 if (hctx->flags & BLK_MQ_F_BLOCKING) 2245 cleanup_srcu_struct(hctx->srcu); 2246 2247 blk_mq_remove_cpuhp(hctx); 2248 blk_free_flush_queue(hctx->fq); 2249 sbitmap_free(&hctx->ctx_map); 2250 } 2251 2252 static void blk_mq_exit_hw_queues(struct request_queue *q, 2253 struct blk_mq_tag_set *set, int nr_queue) 2254 { 2255 struct blk_mq_hw_ctx *hctx; 2256 unsigned int i; 2257 2258 queue_for_each_hw_ctx(q, hctx, i) { 2259 if (i == nr_queue) 2260 break; 2261 blk_mq_debugfs_unregister_hctx(hctx); 2262 blk_mq_exit_hctx(q, set, hctx, i); 2263 } 2264 } 2265 2266 static int blk_mq_init_hctx(struct request_queue *q, 2267 struct blk_mq_tag_set *set, 2268 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) 2269 { 2270 int node; 2271 2272 node = hctx->numa_node; 2273 if (node == NUMA_NO_NODE) 2274 node = hctx->numa_node = set->numa_node; 2275 2276 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); 2277 spin_lock_init(&hctx->lock); 2278 INIT_LIST_HEAD(&hctx->dispatch); 2279 hctx->queue = q; 2280 hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED; 2281 2282 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead); 2283 2284 hctx->tags = set->tags[hctx_idx]; 2285 2286 /* 2287 * Allocate space for all possible cpus to avoid allocation at 2288 * runtime 2289 */ 2290 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *), 2291 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, node); 2292 if (!hctx->ctxs) 2293 goto unregister_cpu_notifier; 2294 2295 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), 2296 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, node)) 2297 goto free_ctxs; 2298 2299 hctx->nr_ctx = 0; 2300 2301 spin_lock_init(&hctx->dispatch_wait_lock); 2302 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); 2303 INIT_LIST_HEAD(&hctx->dispatch_wait.entry); 2304 2305 if (set->ops->init_hctx && 2306 set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) 2307 goto free_bitmap; 2308 2309 hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size, 2310 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY); 2311 if (!hctx->fq) 2312 goto exit_hctx; 2313 2314 if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx, node)) 2315 goto free_fq; 2316 2317 if (hctx->flags & BLK_MQ_F_BLOCKING) 2318 init_srcu_struct(hctx->srcu); 2319 2320 return 0; 2321 2322 free_fq: 2323 kfree(hctx->fq); 2324 exit_hctx: 2325 if (set->ops->exit_hctx) 2326 set->ops->exit_hctx(hctx, hctx_idx); 2327 free_bitmap: 2328 sbitmap_free(&hctx->ctx_map); 2329 free_ctxs: 2330 kfree(hctx->ctxs); 2331 unregister_cpu_notifier: 2332 blk_mq_remove_cpuhp(hctx); 2333 return -1; 2334 } 2335 2336 static void blk_mq_init_cpu_queues(struct request_queue *q, 2337 unsigned int nr_hw_queues) 2338 { 2339 struct blk_mq_tag_set *set = q->tag_set; 2340 unsigned int i, j; 2341 2342 for_each_possible_cpu(i) { 2343 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i); 2344 struct blk_mq_hw_ctx *hctx; 2345 int k; 2346 2347 __ctx->cpu = i; 2348 spin_lock_init(&__ctx->lock); 2349 for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++) 2350 INIT_LIST_HEAD(&__ctx->rq_lists[k]); 2351 2352 __ctx->queue = q; 2353 2354 /* 2355 * Set local node, IFF we have more than one hw queue. If 2356 * not, we remain on the home node of the device 2357 */ 2358 for (j = 0; j < set->nr_maps; j++) { 2359 hctx = blk_mq_map_queue_type(q, j, i); 2360 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) 2361 hctx->numa_node = local_memory_node(cpu_to_node(i)); 2362 } 2363 } 2364 } 2365 2366 static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx) 2367 { 2368 int ret = 0; 2369 2370 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx, 2371 set->queue_depth, set->reserved_tags); 2372 if (!set->tags[hctx_idx]) 2373 return false; 2374 2375 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx, 2376 set->queue_depth); 2377 if (!ret) 2378 return true; 2379 2380 blk_mq_free_rq_map(set->tags[hctx_idx]); 2381 set->tags[hctx_idx] = NULL; 2382 return false; 2383 } 2384 2385 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set, 2386 unsigned int hctx_idx) 2387 { 2388 if (set->tags && set->tags[hctx_idx]) { 2389 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx); 2390 blk_mq_free_rq_map(set->tags[hctx_idx]); 2391 set->tags[hctx_idx] = NULL; 2392 } 2393 } 2394 2395 static void blk_mq_map_swqueue(struct request_queue *q) 2396 { 2397 unsigned int i, j, hctx_idx; 2398 struct blk_mq_hw_ctx *hctx; 2399 struct blk_mq_ctx *ctx; 2400 struct blk_mq_tag_set *set = q->tag_set; 2401 2402 /* 2403 * Avoid others reading imcomplete hctx->cpumask through sysfs 2404 */ 2405 mutex_lock(&q->sysfs_lock); 2406 2407 queue_for_each_hw_ctx(q, hctx, i) { 2408 cpumask_clear(hctx->cpumask); 2409 hctx->nr_ctx = 0; 2410 hctx->dispatch_from = NULL; 2411 } 2412 2413 /* 2414 * Map software to hardware queues. 2415 * 2416 * If the cpu isn't present, the cpu is mapped to first hctx. 2417 */ 2418 for_each_possible_cpu(i) { 2419 hctx_idx = set->map[0].mq_map[i]; 2420 /* unmapped hw queue can be remapped after CPU topo changed */ 2421 if (!set->tags[hctx_idx] && 2422 !__blk_mq_alloc_rq_map(set, hctx_idx)) { 2423 /* 2424 * If tags initialization fail for some hctx, 2425 * that hctx won't be brought online. In this 2426 * case, remap the current ctx to hctx[0] which 2427 * is guaranteed to always have tags allocated 2428 */ 2429 set->map[0].mq_map[i] = 0; 2430 } 2431 2432 ctx = per_cpu_ptr(q->queue_ctx, i); 2433 for (j = 0; j < set->nr_maps; j++) { 2434 if (!set->map[j].nr_queues) 2435 continue; 2436 2437 hctx = blk_mq_map_queue_type(q, j, i); 2438 2439 /* 2440 * If the CPU is already set in the mask, then we've 2441 * mapped this one already. This can happen if 2442 * devices share queues across queue maps. 2443 */ 2444 if (cpumask_test_cpu(i, hctx->cpumask)) 2445 continue; 2446 2447 cpumask_set_cpu(i, hctx->cpumask); 2448 hctx->type = j; 2449 ctx->index_hw[hctx->type] = hctx->nr_ctx; 2450 hctx->ctxs[hctx->nr_ctx++] = ctx; 2451 2452 /* 2453 * If the nr_ctx type overflows, we have exceeded the 2454 * amount of sw queues we can support. 2455 */ 2456 BUG_ON(!hctx->nr_ctx); 2457 } 2458 } 2459 2460 mutex_unlock(&q->sysfs_lock); 2461 2462 queue_for_each_hw_ctx(q, hctx, i) { 2463 /* 2464 * If no software queues are mapped to this hardware queue, 2465 * disable it and free the request entries. 2466 */ 2467 if (!hctx->nr_ctx) { 2468 /* Never unmap queue 0. We need it as a 2469 * fallback in case of a new remap fails 2470 * allocation 2471 */ 2472 if (i && set->tags[i]) 2473 blk_mq_free_map_and_requests(set, i); 2474 2475 hctx->tags = NULL; 2476 continue; 2477 } 2478 2479 hctx->tags = set->tags[i]; 2480 WARN_ON(!hctx->tags); 2481 2482 /* 2483 * Set the map size to the number of mapped software queues. 2484 * This is more accurate and more efficient than looping 2485 * over all possibly mapped software queues. 2486 */ 2487 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx); 2488 2489 /* 2490 * Initialize batch roundrobin counts 2491 */ 2492 hctx->next_cpu = blk_mq_first_mapped_cpu(hctx); 2493 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 2494 } 2495 } 2496 2497 /* 2498 * Caller needs to ensure that we're either frozen/quiesced, or that 2499 * the queue isn't live yet. 2500 */ 2501 static void queue_set_hctx_shared(struct request_queue *q, bool shared) 2502 { 2503 struct blk_mq_hw_ctx *hctx; 2504 int i; 2505 2506 queue_for_each_hw_ctx(q, hctx, i) { 2507 if (shared) 2508 hctx->flags |= BLK_MQ_F_TAG_SHARED; 2509 else 2510 hctx->flags &= ~BLK_MQ_F_TAG_SHARED; 2511 } 2512 } 2513 2514 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, 2515 bool shared) 2516 { 2517 struct request_queue *q; 2518 2519 lockdep_assert_held(&set->tag_list_lock); 2520 2521 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2522 blk_mq_freeze_queue(q); 2523 queue_set_hctx_shared(q, shared); 2524 blk_mq_unfreeze_queue(q); 2525 } 2526 } 2527 2528 static void blk_mq_del_queue_tag_set(struct request_queue *q) 2529 { 2530 struct blk_mq_tag_set *set = q->tag_set; 2531 2532 mutex_lock(&set->tag_list_lock); 2533 list_del_rcu(&q->tag_set_list); 2534 if (list_is_singular(&set->tag_list)) { 2535 /* just transitioned to unshared */ 2536 set->flags &= ~BLK_MQ_F_TAG_SHARED; 2537 /* update existing queue */ 2538 blk_mq_update_tag_set_depth(set, false); 2539 } 2540 mutex_unlock(&set->tag_list_lock); 2541 INIT_LIST_HEAD(&q->tag_set_list); 2542 } 2543 2544 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, 2545 struct request_queue *q) 2546 { 2547 mutex_lock(&set->tag_list_lock); 2548 2549 /* 2550 * Check to see if we're transitioning to shared (from 1 to 2 queues). 2551 */ 2552 if (!list_empty(&set->tag_list) && 2553 !(set->flags & BLK_MQ_F_TAG_SHARED)) { 2554 set->flags |= BLK_MQ_F_TAG_SHARED; 2555 /* update existing queue */ 2556 blk_mq_update_tag_set_depth(set, true); 2557 } 2558 if (set->flags & BLK_MQ_F_TAG_SHARED) 2559 queue_set_hctx_shared(q, true); 2560 list_add_tail_rcu(&q->tag_set_list, &set->tag_list); 2561 2562 mutex_unlock(&set->tag_list_lock); 2563 } 2564 2565 /* All allocations will be freed in release handler of q->mq_kobj */ 2566 static int blk_mq_alloc_ctxs(struct request_queue *q) 2567 { 2568 struct blk_mq_ctxs *ctxs; 2569 int cpu; 2570 2571 ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL); 2572 if (!ctxs) 2573 return -ENOMEM; 2574 2575 ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx); 2576 if (!ctxs->queue_ctx) 2577 goto fail; 2578 2579 for_each_possible_cpu(cpu) { 2580 struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu); 2581 ctx->ctxs = ctxs; 2582 } 2583 2584 q->mq_kobj = &ctxs->kobj; 2585 q->queue_ctx = ctxs->queue_ctx; 2586 2587 return 0; 2588 fail: 2589 kfree(ctxs); 2590 return -ENOMEM; 2591 } 2592 2593 /* 2594 * It is the actual release handler for mq, but we do it from 2595 * request queue's release handler for avoiding use-after-free 2596 * and headache because q->mq_kobj shouldn't have been introduced, 2597 * but we can't group ctx/kctx kobj without it. 2598 */ 2599 void blk_mq_release(struct request_queue *q) 2600 { 2601 struct blk_mq_hw_ctx *hctx; 2602 unsigned int i; 2603 2604 /* hctx kobj stays in hctx */ 2605 queue_for_each_hw_ctx(q, hctx, i) { 2606 if (!hctx) 2607 continue; 2608 kobject_put(&hctx->kobj); 2609 } 2610 2611 kfree(q->queue_hw_ctx); 2612 2613 /* 2614 * release .mq_kobj and sw queue's kobject now because 2615 * both share lifetime with request queue. 2616 */ 2617 blk_mq_sysfs_deinit(q); 2618 } 2619 2620 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) 2621 { 2622 struct request_queue *uninit_q, *q; 2623 2624 uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node); 2625 if (!uninit_q) 2626 return ERR_PTR(-ENOMEM); 2627 2628 q = blk_mq_init_allocated_queue(set, uninit_q); 2629 if (IS_ERR(q)) 2630 blk_cleanup_queue(uninit_q); 2631 2632 return q; 2633 } 2634 EXPORT_SYMBOL(blk_mq_init_queue); 2635 2636 /* 2637 * Helper for setting up a queue with mq ops, given queue depth, and 2638 * the passed in mq ops flags. 2639 */ 2640 struct request_queue *blk_mq_init_sq_queue(struct blk_mq_tag_set *set, 2641 const struct blk_mq_ops *ops, 2642 unsigned int queue_depth, 2643 unsigned int set_flags) 2644 { 2645 struct request_queue *q; 2646 int ret; 2647 2648 memset(set, 0, sizeof(*set)); 2649 set->ops = ops; 2650 set->nr_hw_queues = 1; 2651 set->nr_maps = 1; 2652 set->queue_depth = queue_depth; 2653 set->numa_node = NUMA_NO_NODE; 2654 set->flags = set_flags; 2655 2656 ret = blk_mq_alloc_tag_set(set); 2657 if (ret) 2658 return ERR_PTR(ret); 2659 2660 q = blk_mq_init_queue(set); 2661 if (IS_ERR(q)) { 2662 blk_mq_free_tag_set(set); 2663 return q; 2664 } 2665 2666 return q; 2667 } 2668 EXPORT_SYMBOL(blk_mq_init_sq_queue); 2669 2670 static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set) 2671 { 2672 int hw_ctx_size = sizeof(struct blk_mq_hw_ctx); 2673 2674 BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu), 2675 __alignof__(struct blk_mq_hw_ctx)) != 2676 sizeof(struct blk_mq_hw_ctx)); 2677 2678 if (tag_set->flags & BLK_MQ_F_BLOCKING) 2679 hw_ctx_size += sizeof(struct srcu_struct); 2680 2681 return hw_ctx_size; 2682 } 2683 2684 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx( 2685 struct blk_mq_tag_set *set, struct request_queue *q, 2686 int hctx_idx, int node) 2687 { 2688 struct blk_mq_hw_ctx *hctx; 2689 2690 hctx = kzalloc_node(blk_mq_hw_ctx_size(set), 2691 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 2692 node); 2693 if (!hctx) 2694 return NULL; 2695 2696 if (!zalloc_cpumask_var_node(&hctx->cpumask, 2697 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 2698 node)) { 2699 kfree(hctx); 2700 return NULL; 2701 } 2702 2703 atomic_set(&hctx->nr_active, 0); 2704 hctx->numa_node = node; 2705 hctx->queue_num = hctx_idx; 2706 2707 if (blk_mq_init_hctx(q, set, hctx, hctx_idx)) { 2708 free_cpumask_var(hctx->cpumask); 2709 kfree(hctx); 2710 return NULL; 2711 } 2712 blk_mq_hctx_kobj_init(hctx); 2713 2714 return hctx; 2715 } 2716 2717 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, 2718 struct request_queue *q) 2719 { 2720 int i, j, end; 2721 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; 2722 2723 /* protect against switching io scheduler */ 2724 mutex_lock(&q->sysfs_lock); 2725 for (i = 0; i < set->nr_hw_queues; i++) { 2726 int node; 2727 struct blk_mq_hw_ctx *hctx; 2728 2729 node = blk_mq_hw_queue_to_node(&set->map[0], i); 2730 /* 2731 * If the hw queue has been mapped to another numa node, 2732 * we need to realloc the hctx. If allocation fails, fallback 2733 * to use the previous one. 2734 */ 2735 if (hctxs[i] && (hctxs[i]->numa_node == node)) 2736 continue; 2737 2738 hctx = blk_mq_alloc_and_init_hctx(set, q, i, node); 2739 if (hctx) { 2740 if (hctxs[i]) { 2741 blk_mq_exit_hctx(q, set, hctxs[i], i); 2742 kobject_put(&hctxs[i]->kobj); 2743 } 2744 hctxs[i] = hctx; 2745 } else { 2746 if (hctxs[i]) 2747 pr_warn("Allocate new hctx on node %d fails,\ 2748 fallback to previous one on node %d\n", 2749 node, hctxs[i]->numa_node); 2750 else 2751 break; 2752 } 2753 } 2754 /* 2755 * Increasing nr_hw_queues fails. Free the newly allocated 2756 * hctxs and keep the previous q->nr_hw_queues. 2757 */ 2758 if (i != set->nr_hw_queues) { 2759 j = q->nr_hw_queues; 2760 end = i; 2761 } else { 2762 j = i; 2763 end = q->nr_hw_queues; 2764 q->nr_hw_queues = set->nr_hw_queues; 2765 } 2766 2767 for (; j < end; j++) { 2768 struct blk_mq_hw_ctx *hctx = hctxs[j]; 2769 2770 if (hctx) { 2771 if (hctx->tags) 2772 blk_mq_free_map_and_requests(set, j); 2773 blk_mq_exit_hctx(q, set, hctx, j); 2774 kobject_put(&hctx->kobj); 2775 hctxs[j] = NULL; 2776 2777 } 2778 } 2779 mutex_unlock(&q->sysfs_lock); 2780 } 2781 2782 /* 2783 * Maximum number of hardware queues we support. For single sets, we'll never 2784 * have more than the CPUs (software queues). For multiple sets, the tag_set 2785 * user may have set ->nr_hw_queues larger. 2786 */ 2787 static unsigned int nr_hw_queues(struct blk_mq_tag_set *set) 2788 { 2789 if (set->nr_maps == 1) 2790 return nr_cpu_ids; 2791 2792 return max(set->nr_hw_queues, nr_cpu_ids); 2793 } 2794 2795 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 2796 struct request_queue *q) 2797 { 2798 /* mark the queue as mq asap */ 2799 q->mq_ops = set->ops; 2800 2801 q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn, 2802 blk_mq_poll_stats_bkt, 2803 BLK_MQ_POLL_STATS_BKTS, q); 2804 if (!q->poll_cb) 2805 goto err_exit; 2806 2807 if (blk_mq_alloc_ctxs(q)) 2808 goto err_exit; 2809 2810 /* init q->mq_kobj and sw queues' kobjects */ 2811 blk_mq_sysfs_init(q); 2812 2813 q->nr_queues = nr_hw_queues(set); 2814 q->queue_hw_ctx = kcalloc_node(q->nr_queues, sizeof(*(q->queue_hw_ctx)), 2815 GFP_KERNEL, set->numa_node); 2816 if (!q->queue_hw_ctx) 2817 goto err_sys_init; 2818 2819 blk_mq_realloc_hw_ctxs(set, q); 2820 if (!q->nr_hw_queues) 2821 goto err_hctxs; 2822 2823 INIT_WORK(&q->timeout_work, blk_mq_timeout_work); 2824 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ); 2825 2826 q->tag_set = set; 2827 2828 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; 2829 if (set->nr_maps > HCTX_TYPE_POLL && 2830 set->map[HCTX_TYPE_POLL].nr_queues) 2831 blk_queue_flag_set(QUEUE_FLAG_POLL, q); 2832 2833 if (!(set->flags & BLK_MQ_F_SG_MERGE)) 2834 blk_queue_flag_set(QUEUE_FLAG_NO_SG_MERGE, q); 2835 2836 q->sg_reserved_size = INT_MAX; 2837 2838 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); 2839 INIT_LIST_HEAD(&q->requeue_list); 2840 spin_lock_init(&q->requeue_lock); 2841 2842 blk_queue_make_request(q, blk_mq_make_request); 2843 2844 /* 2845 * Do this after blk_queue_make_request() overrides it... 2846 */ 2847 q->nr_requests = set->queue_depth; 2848 2849 /* 2850 * Default to classic polling 2851 */ 2852 q->poll_nsec = -1; 2853 2854 blk_mq_init_cpu_queues(q, set->nr_hw_queues); 2855 blk_mq_add_queue_tag_set(set, q); 2856 blk_mq_map_swqueue(q); 2857 2858 if (!(set->flags & BLK_MQ_F_NO_SCHED)) { 2859 int ret; 2860 2861 ret = elevator_init_mq(q); 2862 if (ret) 2863 return ERR_PTR(ret); 2864 } 2865 2866 return q; 2867 2868 err_hctxs: 2869 kfree(q->queue_hw_ctx); 2870 err_sys_init: 2871 blk_mq_sysfs_deinit(q); 2872 err_exit: 2873 q->mq_ops = NULL; 2874 return ERR_PTR(-ENOMEM); 2875 } 2876 EXPORT_SYMBOL(blk_mq_init_allocated_queue); 2877 2878 void blk_mq_free_queue(struct request_queue *q) 2879 { 2880 struct blk_mq_tag_set *set = q->tag_set; 2881 2882 blk_mq_del_queue_tag_set(q); 2883 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); 2884 } 2885 2886 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2887 { 2888 int i; 2889 2890 for (i = 0; i < set->nr_hw_queues; i++) 2891 if (!__blk_mq_alloc_rq_map(set, i)) 2892 goto out_unwind; 2893 2894 return 0; 2895 2896 out_unwind: 2897 while (--i >= 0) 2898 blk_mq_free_rq_map(set->tags[i]); 2899 2900 return -ENOMEM; 2901 } 2902 2903 /* 2904 * Allocate the request maps associated with this tag_set. Note that this 2905 * may reduce the depth asked for, if memory is tight. set->queue_depth 2906 * will be updated to reflect the allocated depth. 2907 */ 2908 static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2909 { 2910 unsigned int depth; 2911 int err; 2912 2913 depth = set->queue_depth; 2914 do { 2915 err = __blk_mq_alloc_rq_maps(set); 2916 if (!err) 2917 break; 2918 2919 set->queue_depth >>= 1; 2920 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) { 2921 err = -ENOMEM; 2922 break; 2923 } 2924 } while (set->queue_depth); 2925 2926 if (!set->queue_depth || err) { 2927 pr_err("blk-mq: failed to allocate request map\n"); 2928 return -ENOMEM; 2929 } 2930 2931 if (depth != set->queue_depth) 2932 pr_info("blk-mq: reduced tag depth (%u -> %u)\n", 2933 depth, set->queue_depth); 2934 2935 return 0; 2936 } 2937 2938 static int blk_mq_update_queue_map(struct blk_mq_tag_set *set) 2939 { 2940 if (set->ops->map_queues && !is_kdump_kernel()) { 2941 int i; 2942 2943 /* 2944 * transport .map_queues is usually done in the following 2945 * way: 2946 * 2947 * for (queue = 0; queue < set->nr_hw_queues; queue++) { 2948 * mask = get_cpu_mask(queue) 2949 * for_each_cpu(cpu, mask) 2950 * set->map[x].mq_map[cpu] = queue; 2951 * } 2952 * 2953 * When we need to remap, the table has to be cleared for 2954 * killing stale mapping since one CPU may not be mapped 2955 * to any hw queue. 2956 */ 2957 for (i = 0; i < set->nr_maps; i++) 2958 blk_mq_clear_mq_map(&set->map[i]); 2959 2960 return set->ops->map_queues(set); 2961 } else { 2962 BUG_ON(set->nr_maps > 1); 2963 return blk_mq_map_queues(&set->map[0]); 2964 } 2965 } 2966 2967 /* 2968 * Alloc a tag set to be associated with one or more request queues. 2969 * May fail with EINVAL for various error conditions. May adjust the 2970 * requested depth down, if it's too large. In that case, the set 2971 * value will be stored in set->queue_depth. 2972 */ 2973 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) 2974 { 2975 int i, ret; 2976 2977 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS); 2978 2979 if (!set->nr_hw_queues) 2980 return -EINVAL; 2981 if (!set->queue_depth) 2982 return -EINVAL; 2983 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) 2984 return -EINVAL; 2985 2986 if (!set->ops->queue_rq) 2987 return -EINVAL; 2988 2989 if (!set->ops->get_budget ^ !set->ops->put_budget) 2990 return -EINVAL; 2991 2992 if (set->queue_depth > BLK_MQ_MAX_DEPTH) { 2993 pr_info("blk-mq: reduced tag depth to %u\n", 2994 BLK_MQ_MAX_DEPTH); 2995 set->queue_depth = BLK_MQ_MAX_DEPTH; 2996 } 2997 2998 if (!set->nr_maps) 2999 set->nr_maps = 1; 3000 else if (set->nr_maps > HCTX_MAX_TYPES) 3001 return -EINVAL; 3002 3003 /* 3004 * If a crashdump is active, then we are potentially in a very 3005 * memory constrained environment. Limit us to 1 queue and 3006 * 64 tags to prevent using too much memory. 3007 */ 3008 if (is_kdump_kernel()) { 3009 set->nr_hw_queues = 1; 3010 set->nr_maps = 1; 3011 set->queue_depth = min(64U, set->queue_depth); 3012 } 3013 /* 3014 * There is no use for more h/w queues than cpus if we just have 3015 * a single map 3016 */ 3017 if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids) 3018 set->nr_hw_queues = nr_cpu_ids; 3019 3020 set->tags = kcalloc_node(nr_hw_queues(set), sizeof(struct blk_mq_tags *), 3021 GFP_KERNEL, set->numa_node); 3022 if (!set->tags) 3023 return -ENOMEM; 3024 3025 ret = -ENOMEM; 3026 for (i = 0; i < set->nr_maps; i++) { 3027 set->map[i].mq_map = kcalloc_node(nr_cpu_ids, 3028 sizeof(set->map[i].mq_map[0]), 3029 GFP_KERNEL, set->numa_node); 3030 if (!set->map[i].mq_map) 3031 goto out_free_mq_map; 3032 set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues; 3033 } 3034 3035 ret = blk_mq_update_queue_map(set); 3036 if (ret) 3037 goto out_free_mq_map; 3038 3039 ret = blk_mq_alloc_rq_maps(set); 3040 if (ret) 3041 goto out_free_mq_map; 3042 3043 mutex_init(&set->tag_list_lock); 3044 INIT_LIST_HEAD(&set->tag_list); 3045 3046 return 0; 3047 3048 out_free_mq_map: 3049 for (i = 0; i < set->nr_maps; i++) { 3050 kfree(set->map[i].mq_map); 3051 set->map[i].mq_map = NULL; 3052 } 3053 kfree(set->tags); 3054 set->tags = NULL; 3055 return ret; 3056 } 3057 EXPORT_SYMBOL(blk_mq_alloc_tag_set); 3058 3059 void blk_mq_free_tag_set(struct blk_mq_tag_set *set) 3060 { 3061 int i, j; 3062 3063 for (i = 0; i < nr_hw_queues(set); i++) 3064 blk_mq_free_map_and_requests(set, i); 3065 3066 for (j = 0; j < set->nr_maps; j++) { 3067 kfree(set->map[j].mq_map); 3068 set->map[j].mq_map = NULL; 3069 } 3070 3071 kfree(set->tags); 3072 set->tags = NULL; 3073 } 3074 EXPORT_SYMBOL(blk_mq_free_tag_set); 3075 3076 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) 3077 { 3078 struct blk_mq_tag_set *set = q->tag_set; 3079 struct blk_mq_hw_ctx *hctx; 3080 int i, ret; 3081 3082 if (!set) 3083 return -EINVAL; 3084 3085 blk_mq_freeze_queue(q); 3086 blk_mq_quiesce_queue(q); 3087 3088 ret = 0; 3089 queue_for_each_hw_ctx(q, hctx, i) { 3090 if (!hctx->tags) 3091 continue; 3092 /* 3093 * If we're using an MQ scheduler, just update the scheduler 3094 * queue depth. This is similar to what the old code would do. 3095 */ 3096 if (!hctx->sched_tags) { 3097 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr, 3098 false); 3099 } else { 3100 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags, 3101 nr, true); 3102 } 3103 if (ret) 3104 break; 3105 } 3106 3107 if (!ret) 3108 q->nr_requests = nr; 3109 3110 blk_mq_unquiesce_queue(q); 3111 blk_mq_unfreeze_queue(q); 3112 3113 return ret; 3114 } 3115 3116 /* 3117 * request_queue and elevator_type pair. 3118 * It is just used by __blk_mq_update_nr_hw_queues to cache 3119 * the elevator_type associated with a request_queue. 3120 */ 3121 struct blk_mq_qe_pair { 3122 struct list_head node; 3123 struct request_queue *q; 3124 struct elevator_type *type; 3125 }; 3126 3127 /* 3128 * Cache the elevator_type in qe pair list and switch the 3129 * io scheduler to 'none' 3130 */ 3131 static bool blk_mq_elv_switch_none(struct list_head *head, 3132 struct request_queue *q) 3133 { 3134 struct blk_mq_qe_pair *qe; 3135 3136 if (!q->elevator) 3137 return true; 3138 3139 qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY); 3140 if (!qe) 3141 return false; 3142 3143 INIT_LIST_HEAD(&qe->node); 3144 qe->q = q; 3145 qe->type = q->elevator->type; 3146 list_add(&qe->node, head); 3147 3148 mutex_lock(&q->sysfs_lock); 3149 /* 3150 * After elevator_switch_mq, the previous elevator_queue will be 3151 * released by elevator_release. The reference of the io scheduler 3152 * module get by elevator_get will also be put. So we need to get 3153 * a reference of the io scheduler module here to prevent it to be 3154 * removed. 3155 */ 3156 __module_get(qe->type->elevator_owner); 3157 elevator_switch_mq(q, NULL); 3158 mutex_unlock(&q->sysfs_lock); 3159 3160 return true; 3161 } 3162 3163 static void blk_mq_elv_switch_back(struct list_head *head, 3164 struct request_queue *q) 3165 { 3166 struct blk_mq_qe_pair *qe; 3167 struct elevator_type *t = NULL; 3168 3169 list_for_each_entry(qe, head, node) 3170 if (qe->q == q) { 3171 t = qe->type; 3172 break; 3173 } 3174 3175 if (!t) 3176 return; 3177 3178 list_del(&qe->node); 3179 kfree(qe); 3180 3181 mutex_lock(&q->sysfs_lock); 3182 elevator_switch_mq(q, t); 3183 mutex_unlock(&q->sysfs_lock); 3184 } 3185 3186 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, 3187 int nr_hw_queues) 3188 { 3189 struct request_queue *q; 3190 LIST_HEAD(head); 3191 int prev_nr_hw_queues; 3192 3193 lockdep_assert_held(&set->tag_list_lock); 3194 3195 if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids) 3196 nr_hw_queues = nr_cpu_ids; 3197 if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues) 3198 return; 3199 3200 list_for_each_entry(q, &set->tag_list, tag_set_list) 3201 blk_mq_freeze_queue(q); 3202 /* 3203 * Sync with blk_mq_queue_tag_busy_iter. 3204 */ 3205 synchronize_rcu(); 3206 /* 3207 * Switch IO scheduler to 'none', cleaning up the data associated 3208 * with the previous scheduler. We will switch back once we are done 3209 * updating the new sw to hw queue mappings. 3210 */ 3211 list_for_each_entry(q, &set->tag_list, tag_set_list) 3212 if (!blk_mq_elv_switch_none(&head, q)) 3213 goto switch_back; 3214 3215 list_for_each_entry(q, &set->tag_list, tag_set_list) { 3216 blk_mq_debugfs_unregister_hctxs(q); 3217 blk_mq_sysfs_unregister(q); 3218 } 3219 3220 prev_nr_hw_queues = set->nr_hw_queues; 3221 set->nr_hw_queues = nr_hw_queues; 3222 blk_mq_update_queue_map(set); 3223 fallback: 3224 list_for_each_entry(q, &set->tag_list, tag_set_list) { 3225 blk_mq_realloc_hw_ctxs(set, q); 3226 if (q->nr_hw_queues != set->nr_hw_queues) { 3227 pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n", 3228 nr_hw_queues, prev_nr_hw_queues); 3229 set->nr_hw_queues = prev_nr_hw_queues; 3230 blk_mq_map_queues(&set->map[0]); 3231 goto fallback; 3232 } 3233 blk_mq_map_swqueue(q); 3234 } 3235 3236 list_for_each_entry(q, &set->tag_list, tag_set_list) { 3237 blk_mq_sysfs_register(q); 3238 blk_mq_debugfs_register_hctxs(q); 3239 } 3240 3241 switch_back: 3242 list_for_each_entry(q, &set->tag_list, tag_set_list) 3243 blk_mq_elv_switch_back(&head, q); 3244 3245 list_for_each_entry(q, &set->tag_list, tag_set_list) 3246 blk_mq_unfreeze_queue(q); 3247 } 3248 3249 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) 3250 { 3251 mutex_lock(&set->tag_list_lock); 3252 __blk_mq_update_nr_hw_queues(set, nr_hw_queues); 3253 mutex_unlock(&set->tag_list_lock); 3254 } 3255 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); 3256 3257 /* Enable polling stats and return whether they were already enabled. */ 3258 static bool blk_poll_stats_enable(struct request_queue *q) 3259 { 3260 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 3261 blk_queue_flag_test_and_set(QUEUE_FLAG_POLL_STATS, q)) 3262 return true; 3263 blk_stat_add_callback(q, q->poll_cb); 3264 return false; 3265 } 3266 3267 static void blk_mq_poll_stats_start(struct request_queue *q) 3268 { 3269 /* 3270 * We don't arm the callback if polling stats are not enabled or the 3271 * callback is already active. 3272 */ 3273 if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 3274 blk_stat_is_active(q->poll_cb)) 3275 return; 3276 3277 blk_stat_activate_msecs(q->poll_cb, 100); 3278 } 3279 3280 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb) 3281 { 3282 struct request_queue *q = cb->data; 3283 int bucket; 3284 3285 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) { 3286 if (cb->stat[bucket].nr_samples) 3287 q->poll_stat[bucket] = cb->stat[bucket]; 3288 } 3289 } 3290 3291 static unsigned long blk_mq_poll_nsecs(struct request_queue *q, 3292 struct blk_mq_hw_ctx *hctx, 3293 struct request *rq) 3294 { 3295 unsigned long ret = 0; 3296 int bucket; 3297 3298 /* 3299 * If stats collection isn't on, don't sleep but turn it on for 3300 * future users 3301 */ 3302 if (!blk_poll_stats_enable(q)) 3303 return 0; 3304 3305 /* 3306 * As an optimistic guess, use half of the mean service time 3307 * for this type of request. We can (and should) make this smarter. 3308 * For instance, if the completion latencies are tight, we can 3309 * get closer than just half the mean. This is especially 3310 * important on devices where the completion latencies are longer 3311 * than ~10 usec. We do use the stats for the relevant IO size 3312 * if available which does lead to better estimates. 3313 */ 3314 bucket = blk_mq_poll_stats_bkt(rq); 3315 if (bucket < 0) 3316 return ret; 3317 3318 if (q->poll_stat[bucket].nr_samples) 3319 ret = (q->poll_stat[bucket].mean + 1) / 2; 3320 3321 return ret; 3322 } 3323 3324 static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, 3325 struct blk_mq_hw_ctx *hctx, 3326 struct request *rq) 3327 { 3328 struct hrtimer_sleeper hs; 3329 enum hrtimer_mode mode; 3330 unsigned int nsecs; 3331 ktime_t kt; 3332 3333 if (rq->rq_flags & RQF_MQ_POLL_SLEPT) 3334 return false; 3335 3336 /* 3337 * If we get here, hybrid polling is enabled. Hence poll_nsec can be: 3338 * 3339 * 0: use half of prev avg 3340 * >0: use this specific value 3341 */ 3342 if (q->poll_nsec > 0) 3343 nsecs = q->poll_nsec; 3344 else 3345 nsecs = blk_mq_poll_nsecs(q, hctx, rq); 3346 3347 if (!nsecs) 3348 return false; 3349 3350 rq->rq_flags |= RQF_MQ_POLL_SLEPT; 3351 3352 /* 3353 * This will be replaced with the stats tracking code, using 3354 * 'avg_completion_time / 2' as the pre-sleep target. 3355 */ 3356 kt = nsecs; 3357 3358 mode = HRTIMER_MODE_REL; 3359 hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode); 3360 hrtimer_set_expires(&hs.timer, kt); 3361 3362 hrtimer_init_sleeper(&hs, current); 3363 do { 3364 if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE) 3365 break; 3366 set_current_state(TASK_UNINTERRUPTIBLE); 3367 hrtimer_start_expires(&hs.timer, mode); 3368 if (hs.task) 3369 io_schedule(); 3370 hrtimer_cancel(&hs.timer); 3371 mode = HRTIMER_MODE_ABS; 3372 } while (hs.task && !signal_pending(current)); 3373 3374 __set_current_state(TASK_RUNNING); 3375 destroy_hrtimer_on_stack(&hs.timer); 3376 return true; 3377 } 3378 3379 static bool blk_mq_poll_hybrid(struct request_queue *q, 3380 struct blk_mq_hw_ctx *hctx, blk_qc_t cookie) 3381 { 3382 struct request *rq; 3383 3384 if (q->poll_nsec == -1) 3385 return false; 3386 3387 if (!blk_qc_t_is_internal(cookie)) 3388 rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); 3389 else { 3390 rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie)); 3391 /* 3392 * With scheduling, if the request has completed, we'll 3393 * get a NULL return here, as we clear the sched tag when 3394 * that happens. The request still remains valid, like always, 3395 * so we should be safe with just the NULL check. 3396 */ 3397 if (!rq) 3398 return false; 3399 } 3400 3401 return blk_mq_poll_hybrid_sleep(q, hctx, rq); 3402 } 3403 3404 /** 3405 * blk_poll - poll for IO completions 3406 * @q: the queue 3407 * @cookie: cookie passed back at IO submission time 3408 * @spin: whether to spin for completions 3409 * 3410 * Description: 3411 * Poll for completions on the passed in queue. Returns number of 3412 * completed entries found. If @spin is true, then blk_poll will continue 3413 * looping until at least one completion is found, unless the task is 3414 * otherwise marked running (or we need to reschedule). 3415 */ 3416 int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin) 3417 { 3418 struct blk_mq_hw_ctx *hctx; 3419 long state; 3420 3421 if (!blk_qc_t_valid(cookie) || 3422 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 3423 return 0; 3424 3425 if (current->plug) 3426 blk_flush_plug_list(current->plug, false); 3427 3428 hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; 3429 3430 /* 3431 * If we sleep, have the caller restart the poll loop to reset 3432 * the state. Like for the other success return cases, the 3433 * caller is responsible for checking if the IO completed. If 3434 * the IO isn't complete, we'll get called again and will go 3435 * straight to the busy poll loop. 3436 */ 3437 if (blk_mq_poll_hybrid(q, hctx, cookie)) 3438 return 1; 3439 3440 hctx->poll_considered++; 3441 3442 state = current->state; 3443 do { 3444 int ret; 3445 3446 hctx->poll_invoked++; 3447 3448 ret = q->mq_ops->poll(hctx); 3449 if (ret > 0) { 3450 hctx->poll_success++; 3451 __set_current_state(TASK_RUNNING); 3452 return ret; 3453 } 3454 3455 if (signal_pending_state(state, current)) 3456 __set_current_state(TASK_RUNNING); 3457 3458 if (current->state == TASK_RUNNING) 3459 return 1; 3460 if (ret < 0 || !spin) 3461 break; 3462 cpu_relax(); 3463 } while (!need_resched()); 3464 3465 __set_current_state(TASK_RUNNING); 3466 return 0; 3467 } 3468 EXPORT_SYMBOL_GPL(blk_poll); 3469 3470 unsigned int blk_mq_rq_cpu(struct request *rq) 3471 { 3472 return rq->mq_ctx->cpu; 3473 } 3474 EXPORT_SYMBOL(blk_mq_rq_cpu); 3475 3476 static int __init blk_mq_init(void) 3477 { 3478 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, 3479 blk_mq_hctx_notify_dead); 3480 return 0; 3481 } 3482 subsys_initcall(blk_mq_init); 3483