1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 * Copyright (C) 1994, Karl Keyte: Added support for disk statistics 5 * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE 6 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de> 7 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au> 8 * - July2000 9 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001 10 */ 11 12 /* 13 * This handles all read/write requests to block devices 14 */ 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/bio.h> 18 #include <linux/blkdev.h> 19 #include <linux/blk-mq.h> 20 #include <linux/blk-pm.h> 21 #include <linux/highmem.h> 22 #include <linux/mm.h> 23 #include <linux/pagemap.h> 24 #include <linux/kernel_stat.h> 25 #include <linux/string.h> 26 #include <linux/init.h> 27 #include <linux/completion.h> 28 #include <linux/slab.h> 29 #include <linux/swap.h> 30 #include <linux/writeback.h> 31 #include <linux/task_io_accounting_ops.h> 32 #include <linux/fault-inject.h> 33 #include <linux/list_sort.h> 34 #include <linux/delay.h> 35 #include <linux/ratelimit.h> 36 #include <linux/pm_runtime.h> 37 #include <linux/blk-cgroup.h> 38 #include <linux/t10-pi.h> 39 #include <linux/debugfs.h> 40 #include <linux/bpf.h> 41 #include <linux/psi.h> 42 #include <linux/sched/sysctl.h> 43 #include <linux/blk-crypto.h> 44 45 #define CREATE_TRACE_POINTS 46 #include <trace/events/block.h> 47 48 #include "blk.h" 49 #include "blk-mq.h" 50 #include "blk-mq-sched.h" 51 #include "blk-pm.h" 52 53 struct dentry *blk_debugfs_root; 54 55 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap); 56 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap); 57 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete); 58 EXPORT_TRACEPOINT_SYMBOL_GPL(block_split); 59 EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug); 60 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_insert); 61 62 DEFINE_IDA(blk_queue_ida); 63 64 /* 65 * For queue allocation 66 */ 67 struct kmem_cache *blk_requestq_cachep; 68 69 /* 70 * Controlling structure to kblockd 71 */ 72 static struct workqueue_struct *kblockd_workqueue; 73 74 /** 75 * blk_queue_flag_set - atomically set a queue flag 76 * @flag: flag to be set 77 * @q: request queue 78 */ 79 void blk_queue_flag_set(unsigned int flag, struct request_queue *q) 80 { 81 set_bit(flag, &q->queue_flags); 82 } 83 EXPORT_SYMBOL(blk_queue_flag_set); 84 85 /** 86 * blk_queue_flag_clear - atomically clear a queue flag 87 * @flag: flag to be cleared 88 * @q: request queue 89 */ 90 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q) 91 { 92 clear_bit(flag, &q->queue_flags); 93 } 94 EXPORT_SYMBOL(blk_queue_flag_clear); 95 96 /** 97 * blk_queue_flag_test_and_set - atomically test and set a queue flag 98 * @flag: flag to be set 99 * @q: request queue 100 * 101 * Returns the previous value of @flag - 0 if the flag was not set and 1 if 102 * the flag was already set. 103 */ 104 bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q) 105 { 106 return test_and_set_bit(flag, &q->queue_flags); 107 } 108 EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set); 109 110 void blk_rq_init(struct request_queue *q, struct request *rq) 111 { 112 memset(rq, 0, sizeof(*rq)); 113 114 INIT_LIST_HEAD(&rq->queuelist); 115 rq->q = q; 116 rq->__sector = (sector_t) -1; 117 INIT_HLIST_NODE(&rq->hash); 118 RB_CLEAR_NODE(&rq->rb_node); 119 rq->tag = BLK_MQ_NO_TAG; 120 rq->internal_tag = BLK_MQ_NO_TAG; 121 rq->start_time_ns = ktime_get_ns(); 122 rq->part = NULL; 123 blk_crypto_rq_set_defaults(rq); 124 } 125 EXPORT_SYMBOL(blk_rq_init); 126 127 #define REQ_OP_NAME(name) [REQ_OP_##name] = #name 128 static const char *const blk_op_name[] = { 129 REQ_OP_NAME(READ), 130 REQ_OP_NAME(WRITE), 131 REQ_OP_NAME(FLUSH), 132 REQ_OP_NAME(DISCARD), 133 REQ_OP_NAME(SECURE_ERASE), 134 REQ_OP_NAME(ZONE_RESET), 135 REQ_OP_NAME(ZONE_RESET_ALL), 136 REQ_OP_NAME(ZONE_OPEN), 137 REQ_OP_NAME(ZONE_CLOSE), 138 REQ_OP_NAME(ZONE_FINISH), 139 REQ_OP_NAME(ZONE_APPEND), 140 REQ_OP_NAME(WRITE_SAME), 141 REQ_OP_NAME(WRITE_ZEROES), 142 REQ_OP_NAME(DRV_IN), 143 REQ_OP_NAME(DRV_OUT), 144 }; 145 #undef REQ_OP_NAME 146 147 /** 148 * blk_op_str - Return string XXX in the REQ_OP_XXX. 149 * @op: REQ_OP_XXX. 150 * 151 * Description: Centralize block layer function to convert REQ_OP_XXX into 152 * string format. Useful in the debugging and tracing bio or request. For 153 * invalid REQ_OP_XXX it returns string "UNKNOWN". 154 */ 155 inline const char *blk_op_str(unsigned int op) 156 { 157 const char *op_str = "UNKNOWN"; 158 159 if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op]) 160 op_str = blk_op_name[op]; 161 162 return op_str; 163 } 164 EXPORT_SYMBOL_GPL(blk_op_str); 165 166 static const struct { 167 int errno; 168 const char *name; 169 } blk_errors[] = { 170 [BLK_STS_OK] = { 0, "" }, 171 [BLK_STS_NOTSUPP] = { -EOPNOTSUPP, "operation not supported" }, 172 [BLK_STS_TIMEOUT] = { -ETIMEDOUT, "timeout" }, 173 [BLK_STS_NOSPC] = { -ENOSPC, "critical space allocation" }, 174 [BLK_STS_TRANSPORT] = { -ENOLINK, "recoverable transport" }, 175 [BLK_STS_TARGET] = { -EREMOTEIO, "critical target" }, 176 [BLK_STS_NEXUS] = { -EBADE, "critical nexus" }, 177 [BLK_STS_MEDIUM] = { -ENODATA, "critical medium" }, 178 [BLK_STS_PROTECTION] = { -EILSEQ, "protection" }, 179 [BLK_STS_RESOURCE] = { -ENOMEM, "kernel resource" }, 180 [BLK_STS_DEV_RESOURCE] = { -EBUSY, "device resource" }, 181 [BLK_STS_AGAIN] = { -EAGAIN, "nonblocking retry" }, 182 183 /* device mapper special case, should not leak out: */ 184 [BLK_STS_DM_REQUEUE] = { -EREMCHG, "dm internal retry" }, 185 186 /* zone device specific errors */ 187 [BLK_STS_ZONE_OPEN_RESOURCE] = { -ETOOMANYREFS, "open zones exceeded" }, 188 [BLK_STS_ZONE_ACTIVE_RESOURCE] = { -EOVERFLOW, "active zones exceeded" }, 189 190 /* everything else not covered above: */ 191 [BLK_STS_IOERR] = { -EIO, "I/O" }, 192 }; 193 194 blk_status_t errno_to_blk_status(int errno) 195 { 196 int i; 197 198 for (i = 0; i < ARRAY_SIZE(blk_errors); i++) { 199 if (blk_errors[i].errno == errno) 200 return (__force blk_status_t)i; 201 } 202 203 return BLK_STS_IOERR; 204 } 205 EXPORT_SYMBOL_GPL(errno_to_blk_status); 206 207 int blk_status_to_errno(blk_status_t status) 208 { 209 int idx = (__force int)status; 210 211 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors))) 212 return -EIO; 213 return blk_errors[idx].errno; 214 } 215 EXPORT_SYMBOL_GPL(blk_status_to_errno); 216 217 static void print_req_error(struct request *req, blk_status_t status, 218 const char *caller) 219 { 220 int idx = (__force int)status; 221 222 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors))) 223 return; 224 225 printk_ratelimited(KERN_ERR 226 "%s: %s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x " 227 "phys_seg %u prio class %u\n", 228 caller, blk_errors[idx].name, 229 req->rq_disk ? req->rq_disk->disk_name : "?", 230 blk_rq_pos(req), req_op(req), blk_op_str(req_op(req)), 231 req->cmd_flags & ~REQ_OP_MASK, 232 req->nr_phys_segments, 233 IOPRIO_PRIO_CLASS(req->ioprio)); 234 } 235 236 static void req_bio_endio(struct request *rq, struct bio *bio, 237 unsigned int nbytes, blk_status_t error) 238 { 239 if (error) 240 bio->bi_status = error; 241 242 if (unlikely(rq->rq_flags & RQF_QUIET)) 243 bio_set_flag(bio, BIO_QUIET); 244 245 bio_advance(bio, nbytes); 246 247 if (req_op(rq) == REQ_OP_ZONE_APPEND && error == BLK_STS_OK) { 248 /* 249 * Partial zone append completions cannot be supported as the 250 * BIO fragments may end up not being written sequentially. 251 */ 252 if (bio->bi_iter.bi_size) 253 bio->bi_status = BLK_STS_IOERR; 254 else 255 bio->bi_iter.bi_sector = rq->__sector; 256 } 257 258 /* don't actually finish bio if it's part of flush sequence */ 259 if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ)) 260 bio_endio(bio); 261 } 262 263 void blk_dump_rq_flags(struct request *rq, char *msg) 264 { 265 printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg, 266 rq->rq_disk ? rq->rq_disk->disk_name : "?", 267 (unsigned long long) rq->cmd_flags); 268 269 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n", 270 (unsigned long long)blk_rq_pos(rq), 271 blk_rq_sectors(rq), blk_rq_cur_sectors(rq)); 272 printk(KERN_INFO " bio %p, biotail %p, len %u\n", 273 rq->bio, rq->biotail, blk_rq_bytes(rq)); 274 } 275 EXPORT_SYMBOL(blk_dump_rq_flags); 276 277 /** 278 * blk_sync_queue - cancel any pending callbacks on a queue 279 * @q: the queue 280 * 281 * Description: 282 * The block layer may perform asynchronous callback activity 283 * on a queue, such as calling the unplug function after a timeout. 284 * A block device may call blk_sync_queue to ensure that any 285 * such activity is cancelled, thus allowing it to release resources 286 * that the callbacks might use. The caller must already have made sure 287 * that its ->submit_bio will not re-add plugging prior to calling 288 * this function. 289 * 290 * This function does not cancel any asynchronous activity arising 291 * out of elevator or throttling code. That would require elevator_exit() 292 * and blkcg_exit_queue() to be called with queue lock initialized. 293 * 294 */ 295 void blk_sync_queue(struct request_queue *q) 296 { 297 del_timer_sync(&q->timeout); 298 cancel_work_sync(&q->timeout_work); 299 } 300 EXPORT_SYMBOL(blk_sync_queue); 301 302 /** 303 * blk_set_pm_only - increment pm_only counter 304 * @q: request queue pointer 305 */ 306 void blk_set_pm_only(struct request_queue *q) 307 { 308 atomic_inc(&q->pm_only); 309 } 310 EXPORT_SYMBOL_GPL(blk_set_pm_only); 311 312 void blk_clear_pm_only(struct request_queue *q) 313 { 314 int pm_only; 315 316 pm_only = atomic_dec_return(&q->pm_only); 317 WARN_ON_ONCE(pm_only < 0); 318 if (pm_only == 0) 319 wake_up_all(&q->mq_freeze_wq); 320 } 321 EXPORT_SYMBOL_GPL(blk_clear_pm_only); 322 323 /** 324 * blk_put_queue - decrement the request_queue refcount 325 * @q: the request_queue structure to decrement the refcount for 326 * 327 * Decrements the refcount of the request_queue kobject. When this reaches 0 328 * we'll have blk_release_queue() called. 329 * 330 * Context: Any context, but the last reference must not be dropped from 331 * atomic context. 332 */ 333 void blk_put_queue(struct request_queue *q) 334 { 335 kobject_put(&q->kobj); 336 } 337 EXPORT_SYMBOL(blk_put_queue); 338 339 void blk_queue_start_drain(struct request_queue *q) 340 { 341 /* 342 * When queue DYING flag is set, we need to block new req 343 * entering queue, so we call blk_freeze_queue_start() to 344 * prevent I/O from crossing blk_queue_enter(). 345 */ 346 blk_freeze_queue_start(q); 347 if (queue_is_mq(q)) 348 blk_mq_wake_waiters(q); 349 /* Make blk_queue_enter() reexamine the DYING flag. */ 350 wake_up_all(&q->mq_freeze_wq); 351 } 352 353 void blk_set_queue_dying(struct request_queue *q) 354 { 355 blk_queue_flag_set(QUEUE_FLAG_DYING, q); 356 blk_queue_start_drain(q); 357 } 358 EXPORT_SYMBOL_GPL(blk_set_queue_dying); 359 360 /** 361 * blk_cleanup_queue - shutdown a request queue 362 * @q: request queue to shutdown 363 * 364 * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and 365 * put it. All future requests will be failed immediately with -ENODEV. 366 * 367 * Context: can sleep 368 */ 369 void blk_cleanup_queue(struct request_queue *q) 370 { 371 /* cannot be called from atomic context */ 372 might_sleep(); 373 374 WARN_ON_ONCE(blk_queue_registered(q)); 375 376 /* mark @q DYING, no new request or merges will be allowed afterwards */ 377 blk_set_queue_dying(q); 378 379 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q); 380 blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q); 381 382 /* 383 * Drain all requests queued before DYING marking. Set DEAD flag to 384 * prevent that blk_mq_run_hw_queues() accesses the hardware queues 385 * after draining finished. 386 */ 387 blk_freeze_queue(q); 388 389 blk_queue_flag_set(QUEUE_FLAG_DEAD, q); 390 391 blk_sync_queue(q); 392 if (queue_is_mq(q)) 393 blk_mq_exit_queue(q); 394 395 /* 396 * In theory, request pool of sched_tags belongs to request queue. 397 * However, the current implementation requires tag_set for freeing 398 * requests, so free the pool now. 399 * 400 * Queue has become frozen, there can't be any in-queue requests, so 401 * it is safe to free requests now. 402 */ 403 mutex_lock(&q->sysfs_lock); 404 if (q->elevator) 405 blk_mq_sched_free_requests(q); 406 mutex_unlock(&q->sysfs_lock); 407 408 percpu_ref_exit(&q->q_usage_counter); 409 410 /* @q is and will stay empty, shutdown and put */ 411 blk_put_queue(q); 412 } 413 EXPORT_SYMBOL(blk_cleanup_queue); 414 415 static bool blk_try_enter_queue(struct request_queue *q, bool pm) 416 { 417 rcu_read_lock(); 418 if (!percpu_ref_tryget_live(&q->q_usage_counter)) 419 goto fail; 420 421 /* 422 * The code that increments the pm_only counter must ensure that the 423 * counter is globally visible before the queue is unfrozen. 424 */ 425 if (blk_queue_pm_only(q) && 426 (!pm || queue_rpm_status(q) == RPM_SUSPENDED)) 427 goto fail_put; 428 429 rcu_read_unlock(); 430 return true; 431 432 fail_put: 433 percpu_ref_put(&q->q_usage_counter); 434 fail: 435 rcu_read_unlock(); 436 return false; 437 } 438 439 /** 440 * blk_queue_enter() - try to increase q->q_usage_counter 441 * @q: request queue pointer 442 * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PM 443 */ 444 int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) 445 { 446 const bool pm = flags & BLK_MQ_REQ_PM; 447 448 while (!blk_try_enter_queue(q, pm)) { 449 if (flags & BLK_MQ_REQ_NOWAIT) 450 return -EBUSY; 451 452 /* 453 * read pair of barrier in blk_freeze_queue_start(), we need to 454 * order reading __PERCPU_REF_DEAD flag of .q_usage_counter and 455 * reading .mq_freeze_depth or queue dying flag, otherwise the 456 * following wait may never return if the two reads are 457 * reordered. 458 */ 459 smp_rmb(); 460 wait_event(q->mq_freeze_wq, 461 (!q->mq_freeze_depth && 462 blk_pm_resume_queue(pm, q)) || 463 blk_queue_dying(q)); 464 if (blk_queue_dying(q)) 465 return -ENODEV; 466 } 467 468 return 0; 469 } 470 471 static inline int bio_queue_enter(struct bio *bio) 472 { 473 struct gendisk *disk = bio->bi_bdev->bd_disk; 474 struct request_queue *q = disk->queue; 475 476 while (!blk_try_enter_queue(q, false)) { 477 if (bio->bi_opf & REQ_NOWAIT) { 478 if (test_bit(GD_DEAD, &disk->state)) 479 goto dead; 480 bio_wouldblock_error(bio); 481 return -EBUSY; 482 } 483 484 /* 485 * read pair of barrier in blk_freeze_queue_start(), we need to 486 * order reading __PERCPU_REF_DEAD flag of .q_usage_counter and 487 * reading .mq_freeze_depth or queue dying flag, otherwise the 488 * following wait may never return if the two reads are 489 * reordered. 490 */ 491 smp_rmb(); 492 wait_event(q->mq_freeze_wq, 493 (!q->mq_freeze_depth && 494 blk_pm_resume_queue(false, q)) || 495 test_bit(GD_DEAD, &disk->state)); 496 if (test_bit(GD_DEAD, &disk->state)) 497 goto dead; 498 } 499 500 return 0; 501 dead: 502 bio_io_error(bio); 503 return -ENODEV; 504 } 505 506 void blk_queue_exit(struct request_queue *q) 507 { 508 percpu_ref_put(&q->q_usage_counter); 509 } 510 511 static void blk_queue_usage_counter_release(struct percpu_ref *ref) 512 { 513 struct request_queue *q = 514 container_of(ref, struct request_queue, q_usage_counter); 515 516 wake_up_all(&q->mq_freeze_wq); 517 } 518 519 static void blk_rq_timed_out_timer(struct timer_list *t) 520 { 521 struct request_queue *q = from_timer(q, t, timeout); 522 523 kblockd_schedule_work(&q->timeout_work); 524 } 525 526 static void blk_timeout_work(struct work_struct *work) 527 { 528 } 529 530 struct request_queue *blk_alloc_queue(int node_id) 531 { 532 struct request_queue *q; 533 int ret; 534 535 q = kmem_cache_alloc_node(blk_requestq_cachep, 536 GFP_KERNEL | __GFP_ZERO, node_id); 537 if (!q) 538 return NULL; 539 540 q->last_merge = NULL; 541 542 q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL); 543 if (q->id < 0) 544 goto fail_q; 545 546 ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0); 547 if (ret) 548 goto fail_id; 549 550 q->stats = blk_alloc_queue_stats(); 551 if (!q->stats) 552 goto fail_split; 553 554 q->node = node_id; 555 556 atomic_set(&q->nr_active_requests_shared_sbitmap, 0); 557 558 timer_setup(&q->timeout, blk_rq_timed_out_timer, 0); 559 INIT_WORK(&q->timeout_work, blk_timeout_work); 560 INIT_LIST_HEAD(&q->icq_list); 561 #ifdef CONFIG_BLK_CGROUP 562 INIT_LIST_HEAD(&q->blkg_list); 563 #endif 564 565 kobject_init(&q->kobj, &blk_queue_ktype); 566 567 mutex_init(&q->debugfs_mutex); 568 mutex_init(&q->sysfs_lock); 569 mutex_init(&q->sysfs_dir_lock); 570 spin_lock_init(&q->queue_lock); 571 572 init_waitqueue_head(&q->mq_freeze_wq); 573 mutex_init(&q->mq_freeze_lock); 574 575 /* 576 * Init percpu_ref in atomic mode so that it's faster to shutdown. 577 * See blk_register_queue() for details. 578 */ 579 if (percpu_ref_init(&q->q_usage_counter, 580 blk_queue_usage_counter_release, 581 PERCPU_REF_INIT_ATOMIC, GFP_KERNEL)) 582 goto fail_stats; 583 584 if (blkcg_init_queue(q)) 585 goto fail_ref; 586 587 blk_queue_dma_alignment(q, 511); 588 blk_set_default_limits(&q->limits); 589 q->nr_requests = BLKDEV_MAX_RQ; 590 591 return q; 592 593 fail_ref: 594 percpu_ref_exit(&q->q_usage_counter); 595 fail_stats: 596 blk_free_queue_stats(q->stats); 597 fail_split: 598 bioset_exit(&q->bio_split); 599 fail_id: 600 ida_simple_remove(&blk_queue_ida, q->id); 601 fail_q: 602 kmem_cache_free(blk_requestq_cachep, q); 603 return NULL; 604 } 605 606 /** 607 * blk_get_queue - increment the request_queue refcount 608 * @q: the request_queue structure to increment the refcount for 609 * 610 * Increment the refcount of the request_queue kobject. 611 * 612 * Context: Any context. 613 */ 614 bool blk_get_queue(struct request_queue *q) 615 { 616 if (likely(!blk_queue_dying(q))) { 617 __blk_get_queue(q); 618 return true; 619 } 620 621 return false; 622 } 623 EXPORT_SYMBOL(blk_get_queue); 624 625 /** 626 * blk_get_request - allocate a request 627 * @q: request queue to allocate a request for 628 * @op: operation (REQ_OP_*) and REQ_* flags, e.g. REQ_SYNC. 629 * @flags: BLK_MQ_REQ_* flags, e.g. BLK_MQ_REQ_NOWAIT. 630 */ 631 struct request *blk_get_request(struct request_queue *q, unsigned int op, 632 blk_mq_req_flags_t flags) 633 { 634 struct request *req; 635 636 WARN_ON_ONCE(op & REQ_NOWAIT); 637 WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PM)); 638 639 req = blk_mq_alloc_request(q, op, flags); 640 if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn) 641 q->mq_ops->initialize_rq_fn(req); 642 643 return req; 644 } 645 EXPORT_SYMBOL(blk_get_request); 646 647 void blk_put_request(struct request *req) 648 { 649 blk_mq_free_request(req); 650 } 651 EXPORT_SYMBOL(blk_put_request); 652 653 static void handle_bad_sector(struct bio *bio, sector_t maxsector) 654 { 655 char b[BDEVNAME_SIZE]; 656 657 pr_info_ratelimited("attempt to access beyond end of device\n" 658 "%s: rw=%d, want=%llu, limit=%llu\n", 659 bio_devname(bio, b), bio->bi_opf, 660 bio_end_sector(bio), maxsector); 661 } 662 663 #ifdef CONFIG_FAIL_MAKE_REQUEST 664 665 static DECLARE_FAULT_ATTR(fail_make_request); 666 667 static int __init setup_fail_make_request(char *str) 668 { 669 return setup_fault_attr(&fail_make_request, str); 670 } 671 __setup("fail_make_request=", setup_fail_make_request); 672 673 static bool should_fail_request(struct block_device *part, unsigned int bytes) 674 { 675 return part->bd_make_it_fail && should_fail(&fail_make_request, bytes); 676 } 677 678 static int __init fail_make_request_debugfs(void) 679 { 680 struct dentry *dir = fault_create_debugfs_attr("fail_make_request", 681 NULL, &fail_make_request); 682 683 return PTR_ERR_OR_ZERO(dir); 684 } 685 686 late_initcall(fail_make_request_debugfs); 687 688 #else /* CONFIG_FAIL_MAKE_REQUEST */ 689 690 static inline bool should_fail_request(struct block_device *part, 691 unsigned int bytes) 692 { 693 return false; 694 } 695 696 #endif /* CONFIG_FAIL_MAKE_REQUEST */ 697 698 static inline bool bio_check_ro(struct bio *bio) 699 { 700 if (op_is_write(bio_op(bio)) && bdev_read_only(bio->bi_bdev)) { 701 char b[BDEVNAME_SIZE]; 702 703 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) 704 return false; 705 706 WARN_ONCE(1, 707 "Trying to write to read-only block-device %s (partno %d)\n", 708 bio_devname(bio, b), bio->bi_bdev->bd_partno); 709 /* Older lvm-tools actually trigger this */ 710 return false; 711 } 712 713 return false; 714 } 715 716 static noinline int should_fail_bio(struct bio *bio) 717 { 718 if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size)) 719 return -EIO; 720 return 0; 721 } 722 ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO); 723 724 /* 725 * Check whether this bio extends beyond the end of the device or partition. 726 * This may well happen - the kernel calls bread() without checking the size of 727 * the device, e.g., when mounting a file system. 728 */ 729 static inline int bio_check_eod(struct bio *bio) 730 { 731 sector_t maxsector = bdev_nr_sectors(bio->bi_bdev); 732 unsigned int nr_sectors = bio_sectors(bio); 733 734 if (nr_sectors && maxsector && 735 (nr_sectors > maxsector || 736 bio->bi_iter.bi_sector > maxsector - nr_sectors)) { 737 handle_bad_sector(bio, maxsector); 738 return -EIO; 739 } 740 return 0; 741 } 742 743 /* 744 * Remap block n of partition p to block n+start(p) of the disk. 745 */ 746 static int blk_partition_remap(struct bio *bio) 747 { 748 struct block_device *p = bio->bi_bdev; 749 750 if (unlikely(should_fail_request(p, bio->bi_iter.bi_size))) 751 return -EIO; 752 if (bio_sectors(bio)) { 753 bio->bi_iter.bi_sector += p->bd_start_sect; 754 trace_block_bio_remap(bio, p->bd_dev, 755 bio->bi_iter.bi_sector - 756 p->bd_start_sect); 757 } 758 bio_set_flag(bio, BIO_REMAPPED); 759 return 0; 760 } 761 762 /* 763 * Check write append to a zoned block device. 764 */ 765 static inline blk_status_t blk_check_zone_append(struct request_queue *q, 766 struct bio *bio) 767 { 768 sector_t pos = bio->bi_iter.bi_sector; 769 int nr_sectors = bio_sectors(bio); 770 771 /* Only applicable to zoned block devices */ 772 if (!blk_queue_is_zoned(q)) 773 return BLK_STS_NOTSUPP; 774 775 /* The bio sector must point to the start of a sequential zone */ 776 if (pos & (blk_queue_zone_sectors(q) - 1) || 777 !blk_queue_zone_is_seq(q, pos)) 778 return BLK_STS_IOERR; 779 780 /* 781 * Not allowed to cross zone boundaries. Otherwise, the BIO will be 782 * split and could result in non-contiguous sectors being written in 783 * different zones. 784 */ 785 if (nr_sectors > q->limits.chunk_sectors) 786 return BLK_STS_IOERR; 787 788 /* Make sure the BIO is small enough and will not get split */ 789 if (nr_sectors > q->limits.max_zone_append_sectors) 790 return BLK_STS_IOERR; 791 792 bio->bi_opf |= REQ_NOMERGE; 793 794 return BLK_STS_OK; 795 } 796 797 static noinline_for_stack bool submit_bio_checks(struct bio *bio) 798 { 799 struct block_device *bdev = bio->bi_bdev; 800 struct request_queue *q = bdev->bd_disk->queue; 801 blk_status_t status = BLK_STS_IOERR; 802 struct blk_plug *plug; 803 804 might_sleep(); 805 806 plug = blk_mq_plug(q, bio); 807 if (plug && plug->nowait) 808 bio->bi_opf |= REQ_NOWAIT; 809 810 /* 811 * For a REQ_NOWAIT based request, return -EOPNOTSUPP 812 * if queue does not support NOWAIT. 813 */ 814 if ((bio->bi_opf & REQ_NOWAIT) && !blk_queue_nowait(q)) 815 goto not_supported; 816 817 if (should_fail_bio(bio)) 818 goto end_io; 819 if (unlikely(bio_check_ro(bio))) 820 goto end_io; 821 if (!bio_flagged(bio, BIO_REMAPPED)) { 822 if (unlikely(bio_check_eod(bio))) 823 goto end_io; 824 if (bdev->bd_partno && unlikely(blk_partition_remap(bio))) 825 goto end_io; 826 } 827 828 /* 829 * Filter flush bio's early so that bio based drivers without flush 830 * support don't have to worry about them. 831 */ 832 if (op_is_flush(bio->bi_opf) && 833 !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) { 834 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA); 835 if (!bio_sectors(bio)) { 836 status = BLK_STS_OK; 837 goto end_io; 838 } 839 } 840 841 if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 842 bio_clear_hipri(bio); 843 844 switch (bio_op(bio)) { 845 case REQ_OP_DISCARD: 846 if (!blk_queue_discard(q)) 847 goto not_supported; 848 break; 849 case REQ_OP_SECURE_ERASE: 850 if (!blk_queue_secure_erase(q)) 851 goto not_supported; 852 break; 853 case REQ_OP_WRITE_SAME: 854 if (!q->limits.max_write_same_sectors) 855 goto not_supported; 856 break; 857 case REQ_OP_ZONE_APPEND: 858 status = blk_check_zone_append(q, bio); 859 if (status != BLK_STS_OK) 860 goto end_io; 861 break; 862 case REQ_OP_ZONE_RESET: 863 case REQ_OP_ZONE_OPEN: 864 case REQ_OP_ZONE_CLOSE: 865 case REQ_OP_ZONE_FINISH: 866 if (!blk_queue_is_zoned(q)) 867 goto not_supported; 868 break; 869 case REQ_OP_ZONE_RESET_ALL: 870 if (!blk_queue_is_zoned(q) || !blk_queue_zone_resetall(q)) 871 goto not_supported; 872 break; 873 case REQ_OP_WRITE_ZEROES: 874 if (!q->limits.max_write_zeroes_sectors) 875 goto not_supported; 876 break; 877 default: 878 break; 879 } 880 881 /* 882 * Various block parts want %current->io_context, so allocate it up 883 * front rather than dealing with lots of pain to allocate it only 884 * where needed. This may fail and the block layer knows how to live 885 * with it. 886 */ 887 if (unlikely(!current->io_context)) 888 create_task_io_context(current, GFP_ATOMIC, q->node); 889 890 if (blk_throtl_bio(bio)) { 891 blkcg_bio_issue_init(bio); 892 return false; 893 } 894 895 blk_cgroup_bio_start(bio); 896 blkcg_bio_issue_init(bio); 897 898 if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) { 899 trace_block_bio_queue(bio); 900 /* Now that enqueuing has been traced, we need to trace 901 * completion as well. 902 */ 903 bio_set_flag(bio, BIO_TRACE_COMPLETION); 904 } 905 return true; 906 907 not_supported: 908 status = BLK_STS_NOTSUPP; 909 end_io: 910 bio->bi_status = status; 911 bio_endio(bio); 912 return false; 913 } 914 915 static blk_qc_t __submit_bio(struct bio *bio) 916 { 917 struct gendisk *disk = bio->bi_bdev->bd_disk; 918 blk_qc_t ret = BLK_QC_T_NONE; 919 920 if (unlikely(bio_queue_enter(bio) != 0)) 921 return BLK_QC_T_NONE; 922 923 if (!submit_bio_checks(bio) || !blk_crypto_bio_prep(&bio)) 924 goto queue_exit; 925 if (disk->fops->submit_bio) { 926 ret = disk->fops->submit_bio(bio); 927 goto queue_exit; 928 } 929 return blk_mq_submit_bio(bio); 930 931 queue_exit: 932 blk_queue_exit(disk->queue); 933 return ret; 934 } 935 936 /* 937 * The loop in this function may be a bit non-obvious, and so deserves some 938 * explanation: 939 * 940 * - Before entering the loop, bio->bi_next is NULL (as all callers ensure 941 * that), so we have a list with a single bio. 942 * - We pretend that we have just taken it off a longer list, so we assign 943 * bio_list to a pointer to the bio_list_on_stack, thus initialising the 944 * bio_list of new bios to be added. ->submit_bio() may indeed add some more 945 * bios through a recursive call to submit_bio_noacct. If it did, we find a 946 * non-NULL value in bio_list and re-enter the loop from the top. 947 * - In this case we really did just take the bio of the top of the list (no 948 * pretending) and so remove it from bio_list, and call into ->submit_bio() 949 * again. 950 * 951 * bio_list_on_stack[0] contains bios submitted by the current ->submit_bio. 952 * bio_list_on_stack[1] contains bios that were submitted before the current 953 * ->submit_bio_bio, but that haven't been processed yet. 954 */ 955 static blk_qc_t __submit_bio_noacct(struct bio *bio) 956 { 957 struct bio_list bio_list_on_stack[2]; 958 blk_qc_t ret = BLK_QC_T_NONE; 959 960 BUG_ON(bio->bi_next); 961 962 bio_list_init(&bio_list_on_stack[0]); 963 current->bio_list = bio_list_on_stack; 964 965 do { 966 struct request_queue *q = bio->bi_bdev->bd_disk->queue; 967 struct bio_list lower, same; 968 969 /* 970 * Create a fresh bio_list for all subordinate requests. 971 */ 972 bio_list_on_stack[1] = bio_list_on_stack[0]; 973 bio_list_init(&bio_list_on_stack[0]); 974 975 ret = __submit_bio(bio); 976 977 /* 978 * Sort new bios into those for a lower level and those for the 979 * same level. 980 */ 981 bio_list_init(&lower); 982 bio_list_init(&same); 983 while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL) 984 if (q == bio->bi_bdev->bd_disk->queue) 985 bio_list_add(&same, bio); 986 else 987 bio_list_add(&lower, bio); 988 989 /* 990 * Now assemble so we handle the lowest level first. 991 */ 992 bio_list_merge(&bio_list_on_stack[0], &lower); 993 bio_list_merge(&bio_list_on_stack[0], &same); 994 bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]); 995 } while ((bio = bio_list_pop(&bio_list_on_stack[0]))); 996 997 current->bio_list = NULL; 998 return ret; 999 } 1000 1001 static blk_qc_t __submit_bio_noacct_mq(struct bio *bio) 1002 { 1003 struct bio_list bio_list[2] = { }; 1004 blk_qc_t ret; 1005 1006 current->bio_list = bio_list; 1007 1008 do { 1009 ret = __submit_bio(bio); 1010 } while ((bio = bio_list_pop(&bio_list[0]))); 1011 1012 current->bio_list = NULL; 1013 return ret; 1014 } 1015 1016 /** 1017 * submit_bio_noacct - re-submit a bio to the block device layer for I/O 1018 * @bio: The bio describing the location in memory and on the device. 1019 * 1020 * This is a version of submit_bio() that shall only be used for I/O that is 1021 * resubmitted to lower level drivers by stacking block drivers. All file 1022 * systems and other upper level users of the block layer should use 1023 * submit_bio() instead. 1024 */ 1025 blk_qc_t submit_bio_noacct(struct bio *bio) 1026 { 1027 /* 1028 * We only want one ->submit_bio to be active at a time, else stack 1029 * usage with stacked devices could be a problem. Use current->bio_list 1030 * to collect a list of requests submited by a ->submit_bio method while 1031 * it is active, and then process them after it returned. 1032 */ 1033 if (current->bio_list) { 1034 bio_list_add(¤t->bio_list[0], bio); 1035 return BLK_QC_T_NONE; 1036 } 1037 1038 if (!bio->bi_bdev->bd_disk->fops->submit_bio) 1039 return __submit_bio_noacct_mq(bio); 1040 return __submit_bio_noacct(bio); 1041 } 1042 EXPORT_SYMBOL(submit_bio_noacct); 1043 1044 /** 1045 * submit_bio - submit a bio to the block device layer for I/O 1046 * @bio: The &struct bio which describes the I/O 1047 * 1048 * submit_bio() is used to submit I/O requests to block devices. It is passed a 1049 * fully set up &struct bio that describes the I/O that needs to be done. The 1050 * bio will be send to the device described by the bi_bdev field. 1051 * 1052 * The success/failure status of the request, along with notification of 1053 * completion, is delivered asynchronously through the ->bi_end_io() callback 1054 * in @bio. The bio must NOT be touched by thecaller until ->bi_end_io() has 1055 * been called. 1056 */ 1057 blk_qc_t submit_bio(struct bio *bio) 1058 { 1059 if (blkcg_punt_bio_submit(bio)) 1060 return BLK_QC_T_NONE; 1061 1062 /* 1063 * If it's a regular read/write or a barrier with data attached, 1064 * go through the normal accounting stuff before submission. 1065 */ 1066 if (bio_has_data(bio)) { 1067 unsigned int count; 1068 1069 if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME)) 1070 count = queue_logical_block_size( 1071 bio->bi_bdev->bd_disk->queue) >> 9; 1072 else 1073 count = bio_sectors(bio); 1074 1075 if (op_is_write(bio_op(bio))) { 1076 count_vm_events(PGPGOUT, count); 1077 } else { 1078 task_io_account_read(bio->bi_iter.bi_size); 1079 count_vm_events(PGPGIN, count); 1080 } 1081 } 1082 1083 /* 1084 * If we're reading data that is part of the userspace workingset, count 1085 * submission time as memory stall. When the device is congested, or 1086 * the submitting cgroup IO-throttled, submission can be a significant 1087 * part of overall IO time. 1088 */ 1089 if (unlikely(bio_op(bio) == REQ_OP_READ && 1090 bio_flagged(bio, BIO_WORKINGSET))) { 1091 unsigned long pflags; 1092 blk_qc_t ret; 1093 1094 psi_memstall_enter(&pflags); 1095 ret = submit_bio_noacct(bio); 1096 psi_memstall_leave(&pflags); 1097 1098 return ret; 1099 } 1100 1101 return submit_bio_noacct(bio); 1102 } 1103 EXPORT_SYMBOL(submit_bio); 1104 1105 /** 1106 * blk_cloned_rq_check_limits - Helper function to check a cloned request 1107 * for the new queue limits 1108 * @q: the queue 1109 * @rq: the request being checked 1110 * 1111 * Description: 1112 * @rq may have been made based on weaker limitations of upper-level queues 1113 * in request stacking drivers, and it may violate the limitation of @q. 1114 * Since the block layer and the underlying device driver trust @rq 1115 * after it is inserted to @q, it should be checked against @q before 1116 * the insertion using this generic function. 1117 * 1118 * Request stacking drivers like request-based dm may change the queue 1119 * limits when retrying requests on other queues. Those requests need 1120 * to be checked against the new queue limits again during dispatch. 1121 */ 1122 static blk_status_t blk_cloned_rq_check_limits(struct request_queue *q, 1123 struct request *rq) 1124 { 1125 unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq)); 1126 1127 if (blk_rq_sectors(rq) > max_sectors) { 1128 /* 1129 * SCSI device does not have a good way to return if 1130 * Write Same/Zero is actually supported. If a device rejects 1131 * a non-read/write command (discard, write same,etc.) the 1132 * low-level device driver will set the relevant queue limit to 1133 * 0 to prevent blk-lib from issuing more of the offending 1134 * operations. Commands queued prior to the queue limit being 1135 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O 1136 * errors being propagated to upper layers. 1137 */ 1138 if (max_sectors == 0) 1139 return BLK_STS_NOTSUPP; 1140 1141 printk(KERN_ERR "%s: over max size limit. (%u > %u)\n", 1142 __func__, blk_rq_sectors(rq), max_sectors); 1143 return BLK_STS_IOERR; 1144 } 1145 1146 /* 1147 * The queue settings related to segment counting may differ from the 1148 * original queue. 1149 */ 1150 rq->nr_phys_segments = blk_recalc_rq_segments(rq); 1151 if (rq->nr_phys_segments > queue_max_segments(q)) { 1152 printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n", 1153 __func__, rq->nr_phys_segments, queue_max_segments(q)); 1154 return BLK_STS_IOERR; 1155 } 1156 1157 return BLK_STS_OK; 1158 } 1159 1160 /** 1161 * blk_insert_cloned_request - Helper for stacking drivers to submit a request 1162 * @q: the queue to submit the request 1163 * @rq: the request being queued 1164 */ 1165 blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq) 1166 { 1167 blk_status_t ret; 1168 1169 ret = blk_cloned_rq_check_limits(q, rq); 1170 if (ret != BLK_STS_OK) 1171 return ret; 1172 1173 if (rq->rq_disk && 1174 should_fail_request(rq->rq_disk->part0, blk_rq_bytes(rq))) 1175 return BLK_STS_IOERR; 1176 1177 if (blk_crypto_insert_cloned_request(rq)) 1178 return BLK_STS_IOERR; 1179 1180 if (blk_queue_io_stat(q)) 1181 blk_account_io_start(rq); 1182 1183 /* 1184 * Since we have a scheduler attached on the top device, 1185 * bypass a potential scheduler on the bottom device for 1186 * insert. 1187 */ 1188 return blk_mq_request_issue_directly(rq, true); 1189 } 1190 EXPORT_SYMBOL_GPL(blk_insert_cloned_request); 1191 1192 /** 1193 * blk_rq_err_bytes - determine number of bytes till the next failure boundary 1194 * @rq: request to examine 1195 * 1196 * Description: 1197 * A request could be merge of IOs which require different failure 1198 * handling. This function determines the number of bytes which 1199 * can be failed from the beginning of the request without 1200 * crossing into area which need to be retried further. 1201 * 1202 * Return: 1203 * The number of bytes to fail. 1204 */ 1205 unsigned int blk_rq_err_bytes(const struct request *rq) 1206 { 1207 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; 1208 unsigned int bytes = 0; 1209 struct bio *bio; 1210 1211 if (!(rq->rq_flags & RQF_MIXED_MERGE)) 1212 return blk_rq_bytes(rq); 1213 1214 /* 1215 * Currently the only 'mixing' which can happen is between 1216 * different fastfail types. We can safely fail portions 1217 * which have all the failfast bits that the first one has - 1218 * the ones which are at least as eager to fail as the first 1219 * one. 1220 */ 1221 for (bio = rq->bio; bio; bio = bio->bi_next) { 1222 if ((bio->bi_opf & ff) != ff) 1223 break; 1224 bytes += bio->bi_iter.bi_size; 1225 } 1226 1227 /* this could lead to infinite loop */ 1228 BUG_ON(blk_rq_bytes(rq) && !bytes); 1229 return bytes; 1230 } 1231 EXPORT_SYMBOL_GPL(blk_rq_err_bytes); 1232 1233 static void update_io_ticks(struct block_device *part, unsigned long now, 1234 bool end) 1235 { 1236 unsigned long stamp; 1237 again: 1238 stamp = READ_ONCE(part->bd_stamp); 1239 if (unlikely(time_after(now, stamp))) { 1240 if (likely(cmpxchg(&part->bd_stamp, stamp, now) == stamp)) 1241 __part_stat_add(part, io_ticks, end ? now - stamp : 1); 1242 } 1243 if (part->bd_partno) { 1244 part = bdev_whole(part); 1245 goto again; 1246 } 1247 } 1248 1249 static void blk_account_io_completion(struct request *req, unsigned int bytes) 1250 { 1251 if (req->part && blk_do_io_stat(req)) { 1252 const int sgrp = op_stat_group(req_op(req)); 1253 1254 part_stat_lock(); 1255 part_stat_add(req->part, sectors[sgrp], bytes >> 9); 1256 part_stat_unlock(); 1257 } 1258 } 1259 1260 void blk_account_io_done(struct request *req, u64 now) 1261 { 1262 /* 1263 * Account IO completion. flush_rq isn't accounted as a 1264 * normal IO on queueing nor completion. Accounting the 1265 * containing request is enough. 1266 */ 1267 if (req->part && blk_do_io_stat(req) && 1268 !(req->rq_flags & RQF_FLUSH_SEQ)) { 1269 const int sgrp = op_stat_group(req_op(req)); 1270 1271 part_stat_lock(); 1272 update_io_ticks(req->part, jiffies, true); 1273 part_stat_inc(req->part, ios[sgrp]); 1274 part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns); 1275 part_stat_unlock(); 1276 } 1277 } 1278 1279 void blk_account_io_start(struct request *rq) 1280 { 1281 if (!blk_do_io_stat(rq)) 1282 return; 1283 1284 /* passthrough requests can hold bios that do not have ->bi_bdev set */ 1285 if (rq->bio && rq->bio->bi_bdev) 1286 rq->part = rq->bio->bi_bdev; 1287 else 1288 rq->part = rq->rq_disk->part0; 1289 1290 part_stat_lock(); 1291 update_io_ticks(rq->part, jiffies, false); 1292 part_stat_unlock(); 1293 } 1294 1295 static unsigned long __part_start_io_acct(struct block_device *part, 1296 unsigned int sectors, unsigned int op) 1297 { 1298 const int sgrp = op_stat_group(op); 1299 unsigned long now = READ_ONCE(jiffies); 1300 1301 part_stat_lock(); 1302 update_io_ticks(part, now, false); 1303 part_stat_inc(part, ios[sgrp]); 1304 part_stat_add(part, sectors[sgrp], sectors); 1305 part_stat_local_inc(part, in_flight[op_is_write(op)]); 1306 part_stat_unlock(); 1307 1308 return now; 1309 } 1310 1311 /** 1312 * bio_start_io_acct - start I/O accounting for bio based drivers 1313 * @bio: bio to start account for 1314 * 1315 * Returns the start time that should be passed back to bio_end_io_acct(). 1316 */ 1317 unsigned long bio_start_io_acct(struct bio *bio) 1318 { 1319 return __part_start_io_acct(bio->bi_bdev, bio_sectors(bio), bio_op(bio)); 1320 } 1321 EXPORT_SYMBOL_GPL(bio_start_io_acct); 1322 1323 unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, 1324 unsigned int op) 1325 { 1326 return __part_start_io_acct(disk->part0, sectors, op); 1327 } 1328 EXPORT_SYMBOL(disk_start_io_acct); 1329 1330 static void __part_end_io_acct(struct block_device *part, unsigned int op, 1331 unsigned long start_time) 1332 { 1333 const int sgrp = op_stat_group(op); 1334 unsigned long now = READ_ONCE(jiffies); 1335 unsigned long duration = now - start_time; 1336 1337 part_stat_lock(); 1338 update_io_ticks(part, now, true); 1339 part_stat_add(part, nsecs[sgrp], jiffies_to_nsecs(duration)); 1340 part_stat_local_dec(part, in_flight[op_is_write(op)]); 1341 part_stat_unlock(); 1342 } 1343 1344 void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time, 1345 struct block_device *orig_bdev) 1346 { 1347 __part_end_io_acct(orig_bdev, bio_op(bio), start_time); 1348 } 1349 EXPORT_SYMBOL_GPL(bio_end_io_acct_remapped); 1350 1351 void disk_end_io_acct(struct gendisk *disk, unsigned int op, 1352 unsigned long start_time) 1353 { 1354 __part_end_io_acct(disk->part0, op, start_time); 1355 } 1356 EXPORT_SYMBOL(disk_end_io_acct); 1357 1358 /* 1359 * Steal bios from a request and add them to a bio list. 1360 * The request must not have been partially completed before. 1361 */ 1362 void blk_steal_bios(struct bio_list *list, struct request *rq) 1363 { 1364 if (rq->bio) { 1365 if (list->tail) 1366 list->tail->bi_next = rq->bio; 1367 else 1368 list->head = rq->bio; 1369 list->tail = rq->biotail; 1370 1371 rq->bio = NULL; 1372 rq->biotail = NULL; 1373 } 1374 1375 rq->__data_len = 0; 1376 } 1377 EXPORT_SYMBOL_GPL(blk_steal_bios); 1378 1379 /** 1380 * blk_update_request - Complete multiple bytes without completing the request 1381 * @req: the request being processed 1382 * @error: block status code 1383 * @nr_bytes: number of bytes to complete for @req 1384 * 1385 * Description: 1386 * Ends I/O on a number of bytes attached to @req, but doesn't complete 1387 * the request structure even if @req doesn't have leftover. 1388 * If @req has leftover, sets it up for the next range of segments. 1389 * 1390 * Passing the result of blk_rq_bytes() as @nr_bytes guarantees 1391 * %false return from this function. 1392 * 1393 * Note: 1394 * The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function 1395 * except in the consistency check at the end of this function. 1396 * 1397 * Return: 1398 * %false - this request doesn't have any more data 1399 * %true - this request has more data 1400 **/ 1401 bool blk_update_request(struct request *req, blk_status_t error, 1402 unsigned int nr_bytes) 1403 { 1404 int total_bytes; 1405 1406 trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes); 1407 1408 if (!req->bio) 1409 return false; 1410 1411 #ifdef CONFIG_BLK_DEV_INTEGRITY 1412 if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ && 1413 error == BLK_STS_OK) 1414 req->q->integrity.profile->complete_fn(req, nr_bytes); 1415 #endif 1416 1417 if (unlikely(error && !blk_rq_is_passthrough(req) && 1418 !(req->rq_flags & RQF_QUIET))) 1419 print_req_error(req, error, __func__); 1420 1421 blk_account_io_completion(req, nr_bytes); 1422 1423 total_bytes = 0; 1424 while (req->bio) { 1425 struct bio *bio = req->bio; 1426 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes); 1427 1428 if (bio_bytes == bio->bi_iter.bi_size) 1429 req->bio = bio->bi_next; 1430 1431 /* Completion has already been traced */ 1432 bio_clear_flag(bio, BIO_TRACE_COMPLETION); 1433 req_bio_endio(req, bio, bio_bytes, error); 1434 1435 total_bytes += bio_bytes; 1436 nr_bytes -= bio_bytes; 1437 1438 if (!nr_bytes) 1439 break; 1440 } 1441 1442 /* 1443 * completely done 1444 */ 1445 if (!req->bio) { 1446 /* 1447 * Reset counters so that the request stacking driver 1448 * can find how many bytes remain in the request 1449 * later. 1450 */ 1451 req->__data_len = 0; 1452 return false; 1453 } 1454 1455 req->__data_len -= total_bytes; 1456 1457 /* update sector only for requests with clear definition of sector */ 1458 if (!blk_rq_is_passthrough(req)) 1459 req->__sector += total_bytes >> 9; 1460 1461 /* mixed attributes always follow the first bio */ 1462 if (req->rq_flags & RQF_MIXED_MERGE) { 1463 req->cmd_flags &= ~REQ_FAILFAST_MASK; 1464 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK; 1465 } 1466 1467 if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) { 1468 /* 1469 * If total number of sectors is less than the first segment 1470 * size, something has gone terribly wrong. 1471 */ 1472 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) { 1473 blk_dump_rq_flags(req, "request botched"); 1474 req->__data_len = blk_rq_cur_bytes(req); 1475 } 1476 1477 /* recalculate the number of segments */ 1478 req->nr_phys_segments = blk_recalc_rq_segments(req); 1479 } 1480 1481 return true; 1482 } 1483 EXPORT_SYMBOL_GPL(blk_update_request); 1484 1485 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1486 /** 1487 * rq_flush_dcache_pages - Helper function to flush all pages in a request 1488 * @rq: the request to be flushed 1489 * 1490 * Description: 1491 * Flush all pages in @rq. 1492 */ 1493 void rq_flush_dcache_pages(struct request *rq) 1494 { 1495 struct req_iterator iter; 1496 struct bio_vec bvec; 1497 1498 rq_for_each_segment(bvec, rq, iter) 1499 flush_dcache_page(bvec.bv_page); 1500 } 1501 EXPORT_SYMBOL_GPL(rq_flush_dcache_pages); 1502 #endif 1503 1504 /** 1505 * blk_lld_busy - Check if underlying low-level drivers of a device are busy 1506 * @q : the queue of the device being checked 1507 * 1508 * Description: 1509 * Check if underlying low-level drivers of a device are busy. 1510 * If the drivers want to export their busy state, they must set own 1511 * exporting function using blk_queue_lld_busy() first. 1512 * 1513 * Basically, this function is used only by request stacking drivers 1514 * to stop dispatching requests to underlying devices when underlying 1515 * devices are busy. This behavior helps more I/O merging on the queue 1516 * of the request stacking driver and prevents I/O throughput regression 1517 * on burst I/O load. 1518 * 1519 * Return: 1520 * 0 - Not busy (The request stacking driver should dispatch request) 1521 * 1 - Busy (The request stacking driver should stop dispatching request) 1522 */ 1523 int blk_lld_busy(struct request_queue *q) 1524 { 1525 if (queue_is_mq(q) && q->mq_ops->busy) 1526 return q->mq_ops->busy(q); 1527 1528 return 0; 1529 } 1530 EXPORT_SYMBOL_GPL(blk_lld_busy); 1531 1532 /** 1533 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request 1534 * @rq: the clone request to be cleaned up 1535 * 1536 * Description: 1537 * Free all bios in @rq for a cloned request. 1538 */ 1539 void blk_rq_unprep_clone(struct request *rq) 1540 { 1541 struct bio *bio; 1542 1543 while ((bio = rq->bio) != NULL) { 1544 rq->bio = bio->bi_next; 1545 1546 bio_put(bio); 1547 } 1548 } 1549 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); 1550 1551 /** 1552 * blk_rq_prep_clone - Helper function to setup clone request 1553 * @rq: the request to be setup 1554 * @rq_src: original request to be cloned 1555 * @bs: bio_set that bios for clone are allocated from 1556 * @gfp_mask: memory allocation mask for bio 1557 * @bio_ctr: setup function to be called for each clone bio. 1558 * Returns %0 for success, non %0 for failure. 1559 * @data: private data to be passed to @bio_ctr 1560 * 1561 * Description: 1562 * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq. 1563 * Also, pages which the original bios are pointing to are not copied 1564 * and the cloned bios just point same pages. 1565 * So cloned bios must be completed before original bios, which means 1566 * the caller must complete @rq before @rq_src. 1567 */ 1568 int blk_rq_prep_clone(struct request *rq, struct request *rq_src, 1569 struct bio_set *bs, gfp_t gfp_mask, 1570 int (*bio_ctr)(struct bio *, struct bio *, void *), 1571 void *data) 1572 { 1573 struct bio *bio, *bio_src; 1574 1575 if (!bs) 1576 bs = &fs_bio_set; 1577 1578 __rq_for_each_bio(bio_src, rq_src) { 1579 bio = bio_clone_fast(bio_src, gfp_mask, bs); 1580 if (!bio) 1581 goto free_and_out; 1582 1583 if (bio_ctr && bio_ctr(bio, bio_src, data)) 1584 goto free_and_out; 1585 1586 if (rq->bio) { 1587 rq->biotail->bi_next = bio; 1588 rq->biotail = bio; 1589 } else { 1590 rq->bio = rq->biotail = bio; 1591 } 1592 bio = NULL; 1593 } 1594 1595 /* Copy attributes of the original request to the clone request. */ 1596 rq->__sector = blk_rq_pos(rq_src); 1597 rq->__data_len = blk_rq_bytes(rq_src); 1598 if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) { 1599 rq->rq_flags |= RQF_SPECIAL_PAYLOAD; 1600 rq->special_vec = rq_src->special_vec; 1601 } 1602 rq->nr_phys_segments = rq_src->nr_phys_segments; 1603 rq->ioprio = rq_src->ioprio; 1604 1605 if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0) 1606 goto free_and_out; 1607 1608 return 0; 1609 1610 free_and_out: 1611 if (bio) 1612 bio_put(bio); 1613 blk_rq_unprep_clone(rq); 1614 1615 return -ENOMEM; 1616 } 1617 EXPORT_SYMBOL_GPL(blk_rq_prep_clone); 1618 1619 int kblockd_schedule_work(struct work_struct *work) 1620 { 1621 return queue_work(kblockd_workqueue, work); 1622 } 1623 EXPORT_SYMBOL(kblockd_schedule_work); 1624 1625 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, 1626 unsigned long delay) 1627 { 1628 return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay); 1629 } 1630 EXPORT_SYMBOL(kblockd_mod_delayed_work_on); 1631 1632 /** 1633 * blk_start_plug - initialize blk_plug and track it inside the task_struct 1634 * @plug: The &struct blk_plug that needs to be initialized 1635 * 1636 * Description: 1637 * blk_start_plug() indicates to the block layer an intent by the caller 1638 * to submit multiple I/O requests in a batch. The block layer may use 1639 * this hint to defer submitting I/Os from the caller until blk_finish_plug() 1640 * is called. However, the block layer may choose to submit requests 1641 * before a call to blk_finish_plug() if the number of queued I/Os 1642 * exceeds %BLK_MAX_REQUEST_COUNT, or if the size of the I/O is larger than 1643 * %BLK_PLUG_FLUSH_SIZE. The queued I/Os may also be submitted early if 1644 * the task schedules (see below). 1645 * 1646 * Tracking blk_plug inside the task_struct will help with auto-flushing the 1647 * pending I/O should the task end up blocking between blk_start_plug() and 1648 * blk_finish_plug(). This is important from a performance perspective, but 1649 * also ensures that we don't deadlock. For instance, if the task is blocking 1650 * for a memory allocation, memory reclaim could end up wanting to free a 1651 * page belonging to that request that is currently residing in our private 1652 * plug. By flushing the pending I/O when the process goes to sleep, we avoid 1653 * this kind of deadlock. 1654 */ 1655 void blk_start_plug(struct blk_plug *plug) 1656 { 1657 struct task_struct *tsk = current; 1658 1659 /* 1660 * If this is a nested plug, don't actually assign it. 1661 */ 1662 if (tsk->plug) 1663 return; 1664 1665 INIT_LIST_HEAD(&plug->mq_list); 1666 INIT_LIST_HEAD(&plug->cb_list); 1667 plug->rq_count = 0; 1668 plug->multiple_queues = false; 1669 plug->nowait = false; 1670 1671 /* 1672 * Store ordering should not be needed here, since a potential 1673 * preempt will imply a full memory barrier 1674 */ 1675 tsk->plug = plug; 1676 } 1677 EXPORT_SYMBOL(blk_start_plug); 1678 1679 static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule) 1680 { 1681 LIST_HEAD(callbacks); 1682 1683 while (!list_empty(&plug->cb_list)) { 1684 list_splice_init(&plug->cb_list, &callbacks); 1685 1686 while (!list_empty(&callbacks)) { 1687 struct blk_plug_cb *cb = list_first_entry(&callbacks, 1688 struct blk_plug_cb, 1689 list); 1690 list_del(&cb->list); 1691 cb->callback(cb, from_schedule); 1692 } 1693 } 1694 } 1695 1696 struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data, 1697 int size) 1698 { 1699 struct blk_plug *plug = current->plug; 1700 struct blk_plug_cb *cb; 1701 1702 if (!plug) 1703 return NULL; 1704 1705 list_for_each_entry(cb, &plug->cb_list, list) 1706 if (cb->callback == unplug && cb->data == data) 1707 return cb; 1708 1709 /* Not currently on the callback list */ 1710 BUG_ON(size < sizeof(*cb)); 1711 cb = kzalloc(size, GFP_ATOMIC); 1712 if (cb) { 1713 cb->data = data; 1714 cb->callback = unplug; 1715 list_add(&cb->list, &plug->cb_list); 1716 } 1717 return cb; 1718 } 1719 EXPORT_SYMBOL(blk_check_plugged); 1720 1721 void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) 1722 { 1723 flush_plug_callbacks(plug, from_schedule); 1724 1725 if (!list_empty(&plug->mq_list)) 1726 blk_mq_flush_plug_list(plug, from_schedule); 1727 } 1728 1729 /** 1730 * blk_finish_plug - mark the end of a batch of submitted I/O 1731 * @plug: The &struct blk_plug passed to blk_start_plug() 1732 * 1733 * Description: 1734 * Indicate that a batch of I/O submissions is complete. This function 1735 * must be paired with an initial call to blk_start_plug(). The intent 1736 * is to allow the block layer to optimize I/O submission. See the 1737 * documentation for blk_start_plug() for more information. 1738 */ 1739 void blk_finish_plug(struct blk_plug *plug) 1740 { 1741 if (plug != current->plug) 1742 return; 1743 blk_flush_plug_list(plug, false); 1744 1745 current->plug = NULL; 1746 } 1747 EXPORT_SYMBOL(blk_finish_plug); 1748 1749 void blk_io_schedule(void) 1750 { 1751 /* Prevent hang_check timer from firing at us during very long I/O */ 1752 unsigned long timeout = sysctl_hung_task_timeout_secs * HZ / 2; 1753 1754 if (timeout) 1755 io_schedule_timeout(timeout); 1756 else 1757 io_schedule(); 1758 } 1759 EXPORT_SYMBOL_GPL(blk_io_schedule); 1760 1761 int __init blk_dev_init(void) 1762 { 1763 BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS)); 1764 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 * 1765 sizeof_field(struct request, cmd_flags)); 1766 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 * 1767 sizeof_field(struct bio, bi_opf)); 1768 1769 /* used for unplugging and affects IO latency/throughput - HIGHPRI */ 1770 kblockd_workqueue = alloc_workqueue("kblockd", 1771 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 1772 if (!kblockd_workqueue) 1773 panic("Failed to create kblockd\n"); 1774 1775 blk_requestq_cachep = kmem_cache_create("request_queue", 1776 sizeof(struct request_queue), 0, SLAB_PANIC, NULL); 1777 1778 blk_debugfs_root = debugfs_create_dir("block", NULL); 1779 1780 return 0; 1781 } 1782