1 /* 2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm.h" 9 #include "dm-uevent.h" 10 11 #include <linux/init.h> 12 #include <linux/module.h> 13 #include <linux/mutex.h> 14 #include <linux/moduleparam.h> 15 #include <linux/blkpg.h> 16 #include <linux/bio.h> 17 #include <linux/mempool.h> 18 #include <linux/slab.h> 19 #include <linux/idr.h> 20 #include <linux/hdreg.h> 21 #include <linux/delay.h> 22 #include <linux/wait.h> 23 #include <linux/kthread.h> 24 #include <linux/ktime.h> 25 #include <linux/elevator.h> /* for rq_end_sector() */ 26 #include <linux/blk-mq.h> 27 28 #include <trace/events/block.h> 29 30 #define DM_MSG_PREFIX "core" 31 32 #ifdef CONFIG_PRINTK 33 /* 34 * ratelimit state to be used in DMXXX_LIMIT(). 35 */ 36 DEFINE_RATELIMIT_STATE(dm_ratelimit_state, 37 DEFAULT_RATELIMIT_INTERVAL, 38 DEFAULT_RATELIMIT_BURST); 39 EXPORT_SYMBOL(dm_ratelimit_state); 40 #endif 41 42 /* 43 * Cookies are numeric values sent with CHANGE and REMOVE 44 * uevents while resuming, removing or renaming the device. 45 */ 46 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" 47 #define DM_COOKIE_LENGTH 24 48 49 static const char *_name = DM_NAME; 50 51 static unsigned int major = 0; 52 static unsigned int _major = 0; 53 54 static DEFINE_IDR(_minor_idr); 55 56 static DEFINE_SPINLOCK(_minor_lock); 57 58 static void do_deferred_remove(struct work_struct *w); 59 60 static DECLARE_WORK(deferred_remove_work, do_deferred_remove); 61 62 static struct workqueue_struct *deferred_remove_workqueue; 63 64 /* 65 * For bio-based dm. 66 * One of these is allocated per bio. 67 */ 68 struct dm_io { 69 struct mapped_device *md; 70 int error; 71 atomic_t io_count; 72 struct bio *bio; 73 unsigned long start_time; 74 spinlock_t endio_lock; 75 struct dm_stats_aux stats_aux; 76 }; 77 78 /* 79 * For request-based dm. 80 * One of these is allocated per request. 81 */ 82 struct dm_rq_target_io { 83 struct mapped_device *md; 84 struct dm_target *ti; 85 struct request *orig, *clone; 86 struct kthread_work work; 87 int error; 88 union map_info info; 89 struct dm_stats_aux stats_aux; 90 unsigned long duration_jiffies; 91 unsigned n_sectors; 92 }; 93 94 /* 95 * For request-based dm - the bio clones we allocate are embedded in these 96 * structs. 97 * 98 * We allocate these with bio_alloc_bioset, using the front_pad parameter when 99 * the bioset is created - this means the bio has to come at the end of the 100 * struct. 101 */ 102 struct dm_rq_clone_bio_info { 103 struct bio *orig; 104 struct dm_rq_target_io *tio; 105 struct bio clone; 106 }; 107 108 union map_info *dm_get_rq_mapinfo(struct request *rq) 109 { 110 if (rq && rq->end_io_data) 111 return &((struct dm_rq_target_io *)rq->end_io_data)->info; 112 return NULL; 113 } 114 EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo); 115 116 #define MINOR_ALLOCED ((void *)-1) 117 118 /* 119 * Bits for the md->flags field. 120 */ 121 #define DMF_BLOCK_IO_FOR_SUSPEND 0 122 #define DMF_SUSPENDED 1 123 #define DMF_FROZEN 2 124 #define DMF_FREEING 3 125 #define DMF_DELETING 4 126 #define DMF_NOFLUSH_SUSPENDING 5 127 #define DMF_MERGE_IS_OPTIONAL 6 128 #define DMF_DEFERRED_REMOVE 7 129 #define DMF_SUSPENDED_INTERNALLY 8 130 131 /* 132 * A dummy definition to make RCU happy. 133 * struct dm_table should never be dereferenced in this file. 134 */ 135 struct dm_table { 136 int undefined__; 137 }; 138 139 /* 140 * Work processed by per-device workqueue. 141 */ 142 struct mapped_device { 143 struct srcu_struct io_barrier; 144 struct mutex suspend_lock; 145 atomic_t holders; 146 atomic_t open_count; 147 148 /* 149 * The current mapping. 150 * Use dm_get_live_table{_fast} or take suspend_lock for 151 * dereference. 152 */ 153 struct dm_table __rcu *map; 154 155 struct list_head table_devices; 156 struct mutex table_devices_lock; 157 158 unsigned long flags; 159 160 struct request_queue *queue; 161 unsigned type; 162 /* Protect queue and type against concurrent access. */ 163 struct mutex type_lock; 164 165 struct target_type *immutable_target_type; 166 167 struct gendisk *disk; 168 char name[16]; 169 170 void *interface_ptr; 171 172 /* 173 * A list of ios that arrived while we were suspended. 174 */ 175 atomic_t pending[2]; 176 wait_queue_head_t wait; 177 struct work_struct work; 178 struct bio_list deferred; 179 spinlock_t deferred_lock; 180 181 /* 182 * Processing queue (flush) 183 */ 184 struct workqueue_struct *wq; 185 186 /* 187 * io objects are allocated from here. 188 */ 189 mempool_t *io_pool; 190 mempool_t *rq_pool; 191 192 struct bio_set *bs; 193 194 /* 195 * Event handling. 196 */ 197 atomic_t event_nr; 198 wait_queue_head_t eventq; 199 atomic_t uevent_seq; 200 struct list_head uevent_list; 201 spinlock_t uevent_lock; /* Protect access to uevent_list */ 202 203 /* 204 * freeze/thaw support require holding onto a super block 205 */ 206 struct super_block *frozen_sb; 207 struct block_device *bdev; 208 209 /* forced geometry settings */ 210 struct hd_geometry geometry; 211 212 /* kobject and completion */ 213 struct dm_kobject_holder kobj_holder; 214 215 /* zero-length flush that will be cloned and submitted to targets */ 216 struct bio flush_bio; 217 218 /* the number of internal suspends */ 219 unsigned internal_suspend_count; 220 221 struct dm_stats stats; 222 223 struct kthread_worker kworker; 224 struct task_struct *kworker_task; 225 226 /* for request-based merge heuristic in dm_request_fn() */ 227 unsigned seq_rq_merge_deadline_usecs; 228 int last_rq_rw; 229 sector_t last_rq_pos; 230 ktime_t last_rq_start_time; 231 232 /* for blk-mq request-based DM support */ 233 struct blk_mq_tag_set tag_set; 234 bool use_blk_mq; 235 }; 236 237 #ifdef CONFIG_DM_MQ_DEFAULT 238 static bool use_blk_mq = true; 239 #else 240 static bool use_blk_mq = false; 241 #endif 242 243 bool dm_use_blk_mq(struct mapped_device *md) 244 { 245 return md->use_blk_mq; 246 } 247 248 /* 249 * For mempools pre-allocation at the table loading time. 250 */ 251 struct dm_md_mempools { 252 mempool_t *io_pool; 253 mempool_t *rq_pool; 254 struct bio_set *bs; 255 }; 256 257 struct table_device { 258 struct list_head list; 259 atomic_t count; 260 struct dm_dev dm_dev; 261 }; 262 263 #define RESERVED_BIO_BASED_IOS 16 264 #define RESERVED_REQUEST_BASED_IOS 256 265 #define RESERVED_MAX_IOS 1024 266 static struct kmem_cache *_io_cache; 267 static struct kmem_cache *_rq_tio_cache; 268 static struct kmem_cache *_rq_cache; 269 270 /* 271 * Bio-based DM's mempools' reserved IOs set by the user. 272 */ 273 static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; 274 275 /* 276 * Request-based DM's mempools' reserved IOs set by the user. 277 */ 278 static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS; 279 280 static unsigned __dm_get_module_param(unsigned *module_param, 281 unsigned def, unsigned max) 282 { 283 unsigned param = ACCESS_ONCE(*module_param); 284 unsigned modified_param = 0; 285 286 if (!param) 287 modified_param = def; 288 else if (param > max) 289 modified_param = max; 290 291 if (modified_param) { 292 (void)cmpxchg(module_param, param, modified_param); 293 param = modified_param; 294 } 295 296 return param; 297 } 298 299 unsigned dm_get_reserved_bio_based_ios(void) 300 { 301 return __dm_get_module_param(&reserved_bio_based_ios, 302 RESERVED_BIO_BASED_IOS, RESERVED_MAX_IOS); 303 } 304 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); 305 306 unsigned dm_get_reserved_rq_based_ios(void) 307 { 308 return __dm_get_module_param(&reserved_rq_based_ios, 309 RESERVED_REQUEST_BASED_IOS, RESERVED_MAX_IOS); 310 } 311 EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios); 312 313 static int __init local_init(void) 314 { 315 int r = -ENOMEM; 316 317 /* allocate a slab for the dm_ios */ 318 _io_cache = KMEM_CACHE(dm_io, 0); 319 if (!_io_cache) 320 return r; 321 322 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0); 323 if (!_rq_tio_cache) 324 goto out_free_io_cache; 325 326 _rq_cache = kmem_cache_create("dm_clone_request", sizeof(struct request), 327 __alignof__(struct request), 0, NULL); 328 if (!_rq_cache) 329 goto out_free_rq_tio_cache; 330 331 r = dm_uevent_init(); 332 if (r) 333 goto out_free_rq_cache; 334 335 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1); 336 if (!deferred_remove_workqueue) { 337 r = -ENOMEM; 338 goto out_uevent_exit; 339 } 340 341 _major = major; 342 r = register_blkdev(_major, _name); 343 if (r < 0) 344 goto out_free_workqueue; 345 346 if (!_major) 347 _major = r; 348 349 return 0; 350 351 out_free_workqueue: 352 destroy_workqueue(deferred_remove_workqueue); 353 out_uevent_exit: 354 dm_uevent_exit(); 355 out_free_rq_cache: 356 kmem_cache_destroy(_rq_cache); 357 out_free_rq_tio_cache: 358 kmem_cache_destroy(_rq_tio_cache); 359 out_free_io_cache: 360 kmem_cache_destroy(_io_cache); 361 362 return r; 363 } 364 365 static void local_exit(void) 366 { 367 flush_scheduled_work(); 368 destroy_workqueue(deferred_remove_workqueue); 369 370 kmem_cache_destroy(_rq_cache); 371 kmem_cache_destroy(_rq_tio_cache); 372 kmem_cache_destroy(_io_cache); 373 unregister_blkdev(_major, _name); 374 dm_uevent_exit(); 375 376 _major = 0; 377 378 DMINFO("cleaned up"); 379 } 380 381 static int (*_inits[])(void) __initdata = { 382 local_init, 383 dm_target_init, 384 dm_linear_init, 385 dm_stripe_init, 386 dm_io_init, 387 dm_kcopyd_init, 388 dm_interface_init, 389 dm_statistics_init, 390 }; 391 392 static void (*_exits[])(void) = { 393 local_exit, 394 dm_target_exit, 395 dm_linear_exit, 396 dm_stripe_exit, 397 dm_io_exit, 398 dm_kcopyd_exit, 399 dm_interface_exit, 400 dm_statistics_exit, 401 }; 402 403 static int __init dm_init(void) 404 { 405 const int count = ARRAY_SIZE(_inits); 406 407 int r, i; 408 409 for (i = 0; i < count; i++) { 410 r = _inits[i](); 411 if (r) 412 goto bad; 413 } 414 415 return 0; 416 417 bad: 418 while (i--) 419 _exits[i](); 420 421 return r; 422 } 423 424 static void __exit dm_exit(void) 425 { 426 int i = ARRAY_SIZE(_exits); 427 428 while (i--) 429 _exits[i](); 430 431 /* 432 * Should be empty by this point. 433 */ 434 idr_destroy(&_minor_idr); 435 } 436 437 /* 438 * Block device functions 439 */ 440 int dm_deleting_md(struct mapped_device *md) 441 { 442 return test_bit(DMF_DELETING, &md->flags); 443 } 444 445 static int dm_blk_open(struct block_device *bdev, fmode_t mode) 446 { 447 struct mapped_device *md; 448 449 spin_lock(&_minor_lock); 450 451 md = bdev->bd_disk->private_data; 452 if (!md) 453 goto out; 454 455 if (test_bit(DMF_FREEING, &md->flags) || 456 dm_deleting_md(md)) { 457 md = NULL; 458 goto out; 459 } 460 461 dm_get(md); 462 atomic_inc(&md->open_count); 463 out: 464 spin_unlock(&_minor_lock); 465 466 return md ? 0 : -ENXIO; 467 } 468 469 static void dm_blk_close(struct gendisk *disk, fmode_t mode) 470 { 471 struct mapped_device *md; 472 473 spin_lock(&_minor_lock); 474 475 md = disk->private_data; 476 if (WARN_ON(!md)) 477 goto out; 478 479 if (atomic_dec_and_test(&md->open_count) && 480 (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) 481 queue_work(deferred_remove_workqueue, &deferred_remove_work); 482 483 dm_put(md); 484 out: 485 spin_unlock(&_minor_lock); 486 } 487 488 int dm_open_count(struct mapped_device *md) 489 { 490 return atomic_read(&md->open_count); 491 } 492 493 /* 494 * Guarantees nothing is using the device before it's deleted. 495 */ 496 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) 497 { 498 int r = 0; 499 500 spin_lock(&_minor_lock); 501 502 if (dm_open_count(md)) { 503 r = -EBUSY; 504 if (mark_deferred) 505 set_bit(DMF_DEFERRED_REMOVE, &md->flags); 506 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) 507 r = -EEXIST; 508 else 509 set_bit(DMF_DELETING, &md->flags); 510 511 spin_unlock(&_minor_lock); 512 513 return r; 514 } 515 516 int dm_cancel_deferred_remove(struct mapped_device *md) 517 { 518 int r = 0; 519 520 spin_lock(&_minor_lock); 521 522 if (test_bit(DMF_DELETING, &md->flags)) 523 r = -EBUSY; 524 else 525 clear_bit(DMF_DEFERRED_REMOVE, &md->flags); 526 527 spin_unlock(&_minor_lock); 528 529 return r; 530 } 531 532 static void do_deferred_remove(struct work_struct *w) 533 { 534 dm_deferred_remove(); 535 } 536 537 sector_t dm_get_size(struct mapped_device *md) 538 { 539 return get_capacity(md->disk); 540 } 541 542 struct request_queue *dm_get_md_queue(struct mapped_device *md) 543 { 544 return md->queue; 545 } 546 547 struct dm_stats *dm_get_stats(struct mapped_device *md) 548 { 549 return &md->stats; 550 } 551 552 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 553 { 554 struct mapped_device *md = bdev->bd_disk->private_data; 555 556 return dm_get_geometry(md, geo); 557 } 558 559 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 560 unsigned int cmd, unsigned long arg) 561 { 562 struct mapped_device *md = bdev->bd_disk->private_data; 563 int srcu_idx; 564 struct dm_table *map; 565 struct dm_target *tgt; 566 int r = -ENOTTY; 567 568 retry: 569 map = dm_get_live_table(md, &srcu_idx); 570 571 if (!map || !dm_table_get_size(map)) 572 goto out; 573 574 /* We only support devices that have a single target */ 575 if (dm_table_get_num_targets(map) != 1) 576 goto out; 577 578 tgt = dm_table_get_target(map, 0); 579 if (!tgt->type->ioctl) 580 goto out; 581 582 if (dm_suspended_md(md)) { 583 r = -EAGAIN; 584 goto out; 585 } 586 587 r = tgt->type->ioctl(tgt, cmd, arg); 588 589 out: 590 dm_put_live_table(md, srcu_idx); 591 592 if (r == -ENOTCONN) { 593 msleep(10); 594 goto retry; 595 } 596 597 return r; 598 } 599 600 static struct dm_io *alloc_io(struct mapped_device *md) 601 { 602 return mempool_alloc(md->io_pool, GFP_NOIO); 603 } 604 605 static void free_io(struct mapped_device *md, struct dm_io *io) 606 { 607 mempool_free(io, md->io_pool); 608 } 609 610 static void free_tio(struct mapped_device *md, struct dm_target_io *tio) 611 { 612 bio_put(&tio->clone); 613 } 614 615 static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md, 616 gfp_t gfp_mask) 617 { 618 return mempool_alloc(md->io_pool, gfp_mask); 619 } 620 621 static void free_rq_tio(struct dm_rq_target_io *tio) 622 { 623 mempool_free(tio, tio->md->io_pool); 624 } 625 626 static struct request *alloc_clone_request(struct mapped_device *md, 627 gfp_t gfp_mask) 628 { 629 return mempool_alloc(md->rq_pool, gfp_mask); 630 } 631 632 static void free_clone_request(struct mapped_device *md, struct request *rq) 633 { 634 mempool_free(rq, md->rq_pool); 635 } 636 637 static int md_in_flight(struct mapped_device *md) 638 { 639 return atomic_read(&md->pending[READ]) + 640 atomic_read(&md->pending[WRITE]); 641 } 642 643 static void start_io_acct(struct dm_io *io) 644 { 645 struct mapped_device *md = io->md; 646 struct bio *bio = io->bio; 647 int cpu; 648 int rw = bio_data_dir(bio); 649 650 io->start_time = jiffies; 651 652 cpu = part_stat_lock(); 653 part_round_stats(cpu, &dm_disk(md)->part0); 654 part_stat_unlock(); 655 atomic_set(&dm_disk(md)->part0.in_flight[rw], 656 atomic_inc_return(&md->pending[rw])); 657 658 if (unlikely(dm_stats_used(&md->stats))) 659 dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector, 660 bio_sectors(bio), false, 0, &io->stats_aux); 661 } 662 663 static void end_io_acct(struct dm_io *io) 664 { 665 struct mapped_device *md = io->md; 666 struct bio *bio = io->bio; 667 unsigned long duration = jiffies - io->start_time; 668 int pending; 669 int rw = bio_data_dir(bio); 670 671 generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time); 672 673 if (unlikely(dm_stats_used(&md->stats))) 674 dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector, 675 bio_sectors(bio), true, duration, &io->stats_aux); 676 677 /* 678 * After this is decremented the bio must not be touched if it is 679 * a flush. 680 */ 681 pending = atomic_dec_return(&md->pending[rw]); 682 atomic_set(&dm_disk(md)->part0.in_flight[rw], pending); 683 pending += atomic_read(&md->pending[rw^0x1]); 684 685 /* nudge anyone waiting on suspend queue */ 686 if (!pending) 687 wake_up(&md->wait); 688 } 689 690 /* 691 * Add the bio to the list of deferred io. 692 */ 693 static void queue_io(struct mapped_device *md, struct bio *bio) 694 { 695 unsigned long flags; 696 697 spin_lock_irqsave(&md->deferred_lock, flags); 698 bio_list_add(&md->deferred, bio); 699 spin_unlock_irqrestore(&md->deferred_lock, flags); 700 queue_work(md->wq, &md->work); 701 } 702 703 /* 704 * Everyone (including functions in this file), should use this 705 * function to access the md->map field, and make sure they call 706 * dm_put_live_table() when finished. 707 */ 708 struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier) 709 { 710 *srcu_idx = srcu_read_lock(&md->io_barrier); 711 712 return srcu_dereference(md->map, &md->io_barrier); 713 } 714 715 void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier) 716 { 717 srcu_read_unlock(&md->io_barrier, srcu_idx); 718 } 719 720 void dm_sync_table(struct mapped_device *md) 721 { 722 synchronize_srcu(&md->io_barrier); 723 synchronize_rcu_expedited(); 724 } 725 726 /* 727 * A fast alternative to dm_get_live_table/dm_put_live_table. 728 * The caller must not block between these two functions. 729 */ 730 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) 731 { 732 rcu_read_lock(); 733 return rcu_dereference(md->map); 734 } 735 736 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) 737 { 738 rcu_read_unlock(); 739 } 740 741 /* 742 * Open a table device so we can use it as a map destination. 743 */ 744 static int open_table_device(struct table_device *td, dev_t dev, 745 struct mapped_device *md) 746 { 747 static char *_claim_ptr = "I belong to device-mapper"; 748 struct block_device *bdev; 749 750 int r; 751 752 BUG_ON(td->dm_dev.bdev); 753 754 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr); 755 if (IS_ERR(bdev)) 756 return PTR_ERR(bdev); 757 758 r = bd_link_disk_holder(bdev, dm_disk(md)); 759 if (r) { 760 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL); 761 return r; 762 } 763 764 td->dm_dev.bdev = bdev; 765 return 0; 766 } 767 768 /* 769 * Close a table device that we've been using. 770 */ 771 static void close_table_device(struct table_device *td, struct mapped_device *md) 772 { 773 if (!td->dm_dev.bdev) 774 return; 775 776 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md)); 777 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL); 778 td->dm_dev.bdev = NULL; 779 } 780 781 static struct table_device *find_table_device(struct list_head *l, dev_t dev, 782 fmode_t mode) { 783 struct table_device *td; 784 785 list_for_each_entry(td, l, list) 786 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) 787 return td; 788 789 return NULL; 790 } 791 792 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode, 793 struct dm_dev **result) { 794 int r; 795 struct table_device *td; 796 797 mutex_lock(&md->table_devices_lock); 798 td = find_table_device(&md->table_devices, dev, mode); 799 if (!td) { 800 td = kmalloc(sizeof(*td), GFP_KERNEL); 801 if (!td) { 802 mutex_unlock(&md->table_devices_lock); 803 return -ENOMEM; 804 } 805 806 td->dm_dev.mode = mode; 807 td->dm_dev.bdev = NULL; 808 809 if ((r = open_table_device(td, dev, md))) { 810 mutex_unlock(&md->table_devices_lock); 811 kfree(td); 812 return r; 813 } 814 815 format_dev_t(td->dm_dev.name, dev); 816 817 atomic_set(&td->count, 0); 818 list_add(&td->list, &md->table_devices); 819 } 820 atomic_inc(&td->count); 821 mutex_unlock(&md->table_devices_lock); 822 823 *result = &td->dm_dev; 824 return 0; 825 } 826 EXPORT_SYMBOL_GPL(dm_get_table_device); 827 828 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) 829 { 830 struct table_device *td = container_of(d, struct table_device, dm_dev); 831 832 mutex_lock(&md->table_devices_lock); 833 if (atomic_dec_and_test(&td->count)) { 834 close_table_device(td, md); 835 list_del(&td->list); 836 kfree(td); 837 } 838 mutex_unlock(&md->table_devices_lock); 839 } 840 EXPORT_SYMBOL(dm_put_table_device); 841 842 static void free_table_devices(struct list_head *devices) 843 { 844 struct list_head *tmp, *next; 845 846 list_for_each_safe(tmp, next, devices) { 847 struct table_device *td = list_entry(tmp, struct table_device, list); 848 849 DMWARN("dm_destroy: %s still exists with %d references", 850 td->dm_dev.name, atomic_read(&td->count)); 851 kfree(td); 852 } 853 } 854 855 /* 856 * Get the geometry associated with a dm device 857 */ 858 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 859 { 860 *geo = md->geometry; 861 862 return 0; 863 } 864 865 /* 866 * Set the geometry of a device. 867 */ 868 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 869 { 870 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 871 872 if (geo->start > sz) { 873 DMWARN("Start sector is beyond the geometry limits."); 874 return -EINVAL; 875 } 876 877 md->geometry = *geo; 878 879 return 0; 880 } 881 882 /*----------------------------------------------------------------- 883 * CRUD START: 884 * A more elegant soln is in the works that uses the queue 885 * merge fn, unfortunately there are a couple of changes to 886 * the block layer that I want to make for this. So in the 887 * interests of getting something for people to use I give 888 * you this clearly demarcated crap. 889 *---------------------------------------------------------------*/ 890 891 static int __noflush_suspending(struct mapped_device *md) 892 { 893 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 894 } 895 896 /* 897 * Decrements the number of outstanding ios that a bio has been 898 * cloned into, completing the original io if necc. 899 */ 900 static void dec_pending(struct dm_io *io, int error) 901 { 902 unsigned long flags; 903 int io_error; 904 struct bio *bio; 905 struct mapped_device *md = io->md; 906 907 /* Push-back supersedes any I/O errors */ 908 if (unlikely(error)) { 909 spin_lock_irqsave(&io->endio_lock, flags); 910 if (!(io->error > 0 && __noflush_suspending(md))) 911 io->error = error; 912 spin_unlock_irqrestore(&io->endio_lock, flags); 913 } 914 915 if (atomic_dec_and_test(&io->io_count)) { 916 if (io->error == DM_ENDIO_REQUEUE) { 917 /* 918 * Target requested pushing back the I/O. 919 */ 920 spin_lock_irqsave(&md->deferred_lock, flags); 921 if (__noflush_suspending(md)) 922 bio_list_add_head(&md->deferred, io->bio); 923 else 924 /* noflush suspend was interrupted. */ 925 io->error = -EIO; 926 spin_unlock_irqrestore(&md->deferred_lock, flags); 927 } 928 929 io_error = io->error; 930 bio = io->bio; 931 end_io_acct(io); 932 free_io(md, io); 933 934 if (io_error == DM_ENDIO_REQUEUE) 935 return; 936 937 if ((bio->bi_rw & REQ_FLUSH) && bio->bi_iter.bi_size) { 938 /* 939 * Preflush done for flush with data, reissue 940 * without REQ_FLUSH. 941 */ 942 bio->bi_rw &= ~REQ_FLUSH; 943 queue_io(md, bio); 944 } else { 945 /* done with normal IO or empty flush */ 946 trace_block_bio_complete(md->queue, bio, io_error); 947 bio_endio(bio, io_error); 948 } 949 } 950 } 951 952 static void disable_write_same(struct mapped_device *md) 953 { 954 struct queue_limits *limits = dm_get_queue_limits(md); 955 956 /* device doesn't really support WRITE SAME, disable it */ 957 limits->max_write_same_sectors = 0; 958 } 959 960 static void clone_endio(struct bio *bio, int error) 961 { 962 int r = error; 963 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 964 struct dm_io *io = tio->io; 965 struct mapped_device *md = tio->io->md; 966 dm_endio_fn endio = tio->ti->type->end_io; 967 968 if (!bio_flagged(bio, BIO_UPTODATE) && !error) 969 error = -EIO; 970 971 if (endio) { 972 r = endio(tio->ti, bio, error); 973 if (r < 0 || r == DM_ENDIO_REQUEUE) 974 /* 975 * error and requeue request are handled 976 * in dec_pending(). 977 */ 978 error = r; 979 else if (r == DM_ENDIO_INCOMPLETE) 980 /* The target will handle the io */ 981 return; 982 else if (r) { 983 DMWARN("unimplemented target endio return value: %d", r); 984 BUG(); 985 } 986 } 987 988 if (unlikely(r == -EREMOTEIO && (bio->bi_rw & REQ_WRITE_SAME) && 989 !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors)) 990 disable_write_same(md); 991 992 free_tio(md, tio); 993 dec_pending(io, error); 994 } 995 996 /* 997 * Partial completion handling for request-based dm 998 */ 999 static void end_clone_bio(struct bio *clone, int error) 1000 { 1001 struct dm_rq_clone_bio_info *info = 1002 container_of(clone, struct dm_rq_clone_bio_info, clone); 1003 struct dm_rq_target_io *tio = info->tio; 1004 struct bio *bio = info->orig; 1005 unsigned int nr_bytes = info->orig->bi_iter.bi_size; 1006 1007 bio_put(clone); 1008 1009 if (tio->error) 1010 /* 1011 * An error has already been detected on the request. 1012 * Once error occurred, just let clone->end_io() handle 1013 * the remainder. 1014 */ 1015 return; 1016 else if (error) { 1017 /* 1018 * Don't notice the error to the upper layer yet. 1019 * The error handling decision is made by the target driver, 1020 * when the request is completed. 1021 */ 1022 tio->error = error; 1023 return; 1024 } 1025 1026 /* 1027 * I/O for the bio successfully completed. 1028 * Notice the data completion to the upper layer. 1029 */ 1030 1031 /* 1032 * bios are processed from the head of the list. 1033 * So the completing bio should always be rq->bio. 1034 * If it's not, something wrong is happening. 1035 */ 1036 if (tio->orig->bio != bio) 1037 DMERR("bio completion is going in the middle of the request"); 1038 1039 /* 1040 * Update the original request. 1041 * Do not use blk_end_request() here, because it may complete 1042 * the original request before the clone, and break the ordering. 1043 */ 1044 blk_update_request(tio->orig, 0, nr_bytes); 1045 } 1046 1047 static struct dm_rq_target_io *tio_from_request(struct request *rq) 1048 { 1049 return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special); 1050 } 1051 1052 static void rq_end_stats(struct mapped_device *md, struct request *orig) 1053 { 1054 if (unlikely(dm_stats_used(&md->stats))) { 1055 struct dm_rq_target_io *tio = tio_from_request(orig); 1056 tio->duration_jiffies = jiffies - tio->duration_jiffies; 1057 dm_stats_account_io(&md->stats, orig->cmd_flags, blk_rq_pos(orig), 1058 tio->n_sectors, true, tio->duration_jiffies, 1059 &tio->stats_aux); 1060 } 1061 } 1062 1063 /* 1064 * Don't touch any member of the md after calling this function because 1065 * the md may be freed in dm_put() at the end of this function. 1066 * Or do dm_get() before calling this function and dm_put() later. 1067 */ 1068 static void rq_completed(struct mapped_device *md, int rw, bool run_queue) 1069 { 1070 atomic_dec(&md->pending[rw]); 1071 1072 /* nudge anyone waiting on suspend queue */ 1073 if (!md_in_flight(md)) 1074 wake_up(&md->wait); 1075 1076 /* 1077 * Run this off this callpath, as drivers could invoke end_io while 1078 * inside their request_fn (and holding the queue lock). Calling 1079 * back into ->request_fn() could deadlock attempting to grab the 1080 * queue lock again. 1081 */ 1082 if (run_queue) { 1083 if (md->queue->mq_ops) 1084 blk_mq_run_hw_queues(md->queue, true); 1085 else 1086 blk_run_queue_async(md->queue); 1087 } 1088 1089 /* 1090 * dm_put() must be at the end of this function. See the comment above 1091 */ 1092 dm_put(md); 1093 } 1094 1095 static void free_rq_clone(struct request *clone) 1096 { 1097 struct dm_rq_target_io *tio = clone->end_io_data; 1098 struct mapped_device *md = tio->md; 1099 1100 blk_rq_unprep_clone(clone); 1101 1102 if (md->type == DM_TYPE_MQ_REQUEST_BASED) 1103 /* stacked on blk-mq queue(s) */ 1104 tio->ti->type->release_clone_rq(clone); 1105 else if (!md->queue->mq_ops) 1106 /* request_fn queue stacked on request_fn queue(s) */ 1107 free_clone_request(md, clone); 1108 /* 1109 * NOTE: for the blk-mq queue stacked on request_fn queue(s) case: 1110 * no need to call free_clone_request() because we leverage blk-mq by 1111 * allocating the clone at the end of the blk-mq pdu (see: clone_rq) 1112 */ 1113 1114 if (!md->queue->mq_ops) 1115 free_rq_tio(tio); 1116 } 1117 1118 /* 1119 * Complete the clone and the original request. 1120 * Must be called without clone's queue lock held, 1121 * see end_clone_request() for more details. 1122 */ 1123 static void dm_end_request(struct request *clone, int error) 1124 { 1125 int rw = rq_data_dir(clone); 1126 struct dm_rq_target_io *tio = clone->end_io_data; 1127 struct mapped_device *md = tio->md; 1128 struct request *rq = tio->orig; 1129 1130 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { 1131 rq->errors = clone->errors; 1132 rq->resid_len = clone->resid_len; 1133 1134 if (rq->sense) 1135 /* 1136 * We are using the sense buffer of the original 1137 * request. 1138 * So setting the length of the sense data is enough. 1139 */ 1140 rq->sense_len = clone->sense_len; 1141 } 1142 1143 free_rq_clone(clone); 1144 rq_end_stats(md, rq); 1145 if (!rq->q->mq_ops) 1146 blk_end_request_all(rq, error); 1147 else 1148 blk_mq_end_request(rq, error); 1149 rq_completed(md, rw, true); 1150 } 1151 1152 static void dm_unprep_request(struct request *rq) 1153 { 1154 struct dm_rq_target_io *tio = tio_from_request(rq); 1155 struct request *clone = tio->clone; 1156 1157 if (!rq->q->mq_ops) { 1158 rq->special = NULL; 1159 rq->cmd_flags &= ~REQ_DONTPREP; 1160 } 1161 1162 if (clone) 1163 free_rq_clone(clone); 1164 } 1165 1166 /* 1167 * Requeue the original request of a clone. 1168 */ 1169 static void old_requeue_request(struct request *rq) 1170 { 1171 struct request_queue *q = rq->q; 1172 unsigned long flags; 1173 1174 spin_lock_irqsave(q->queue_lock, flags); 1175 blk_requeue_request(q, rq); 1176 blk_run_queue_async(q); 1177 spin_unlock_irqrestore(q->queue_lock, flags); 1178 } 1179 1180 static void dm_requeue_original_request(struct mapped_device *md, 1181 struct request *rq) 1182 { 1183 int rw = rq_data_dir(rq); 1184 1185 dm_unprep_request(rq); 1186 1187 rq_end_stats(md, rq); 1188 if (!rq->q->mq_ops) 1189 old_requeue_request(rq); 1190 else { 1191 blk_mq_requeue_request(rq); 1192 blk_mq_kick_requeue_list(rq->q); 1193 } 1194 1195 rq_completed(md, rw, false); 1196 } 1197 1198 static void old_stop_queue(struct request_queue *q) 1199 { 1200 unsigned long flags; 1201 1202 if (blk_queue_stopped(q)) 1203 return; 1204 1205 spin_lock_irqsave(q->queue_lock, flags); 1206 blk_stop_queue(q); 1207 spin_unlock_irqrestore(q->queue_lock, flags); 1208 } 1209 1210 static void stop_queue(struct request_queue *q) 1211 { 1212 if (!q->mq_ops) 1213 old_stop_queue(q); 1214 else 1215 blk_mq_stop_hw_queues(q); 1216 } 1217 1218 static void old_start_queue(struct request_queue *q) 1219 { 1220 unsigned long flags; 1221 1222 spin_lock_irqsave(q->queue_lock, flags); 1223 if (blk_queue_stopped(q)) 1224 blk_start_queue(q); 1225 spin_unlock_irqrestore(q->queue_lock, flags); 1226 } 1227 1228 static void start_queue(struct request_queue *q) 1229 { 1230 if (!q->mq_ops) 1231 old_start_queue(q); 1232 else 1233 blk_mq_start_stopped_hw_queues(q, true); 1234 } 1235 1236 static void dm_done(struct request *clone, int error, bool mapped) 1237 { 1238 int r = error; 1239 struct dm_rq_target_io *tio = clone->end_io_data; 1240 dm_request_endio_fn rq_end_io = NULL; 1241 1242 if (tio->ti) { 1243 rq_end_io = tio->ti->type->rq_end_io; 1244 1245 if (mapped && rq_end_io) 1246 r = rq_end_io(tio->ti, clone, error, &tio->info); 1247 } 1248 1249 if (unlikely(r == -EREMOTEIO && (clone->cmd_flags & REQ_WRITE_SAME) && 1250 !clone->q->limits.max_write_same_sectors)) 1251 disable_write_same(tio->md); 1252 1253 if (r <= 0) 1254 /* The target wants to complete the I/O */ 1255 dm_end_request(clone, r); 1256 else if (r == DM_ENDIO_INCOMPLETE) 1257 /* The target will handle the I/O */ 1258 return; 1259 else if (r == DM_ENDIO_REQUEUE) 1260 /* The target wants to requeue the I/O */ 1261 dm_requeue_original_request(tio->md, tio->orig); 1262 else { 1263 DMWARN("unimplemented target endio return value: %d", r); 1264 BUG(); 1265 } 1266 } 1267 1268 /* 1269 * Request completion handler for request-based dm 1270 */ 1271 static void dm_softirq_done(struct request *rq) 1272 { 1273 bool mapped = true; 1274 struct dm_rq_target_io *tio = tio_from_request(rq); 1275 struct request *clone = tio->clone; 1276 int rw; 1277 1278 if (!clone) { 1279 rq_end_stats(tio->md, rq); 1280 rw = rq_data_dir(rq); 1281 if (!rq->q->mq_ops) { 1282 blk_end_request_all(rq, tio->error); 1283 rq_completed(tio->md, rw, false); 1284 free_rq_tio(tio); 1285 } else { 1286 blk_mq_end_request(rq, tio->error); 1287 rq_completed(tio->md, rw, false); 1288 } 1289 return; 1290 } 1291 1292 if (rq->cmd_flags & REQ_FAILED) 1293 mapped = false; 1294 1295 dm_done(clone, tio->error, mapped); 1296 } 1297 1298 /* 1299 * Complete the clone and the original request with the error status 1300 * through softirq context. 1301 */ 1302 static void dm_complete_request(struct request *rq, int error) 1303 { 1304 struct dm_rq_target_io *tio = tio_from_request(rq); 1305 1306 tio->error = error; 1307 blk_complete_request(rq); 1308 } 1309 1310 /* 1311 * Complete the not-mapped clone and the original request with the error status 1312 * through softirq context. 1313 * Target's rq_end_io() function isn't called. 1314 * This may be used when the target's map_rq() or clone_and_map_rq() functions fail. 1315 */ 1316 static void dm_kill_unmapped_request(struct request *rq, int error) 1317 { 1318 rq->cmd_flags |= REQ_FAILED; 1319 dm_complete_request(rq, error); 1320 } 1321 1322 /* 1323 * Called with the clone's queue lock held (for non-blk-mq) 1324 */ 1325 static void end_clone_request(struct request *clone, int error) 1326 { 1327 struct dm_rq_target_io *tio = clone->end_io_data; 1328 1329 if (!clone->q->mq_ops) { 1330 /* 1331 * For just cleaning up the information of the queue in which 1332 * the clone was dispatched. 1333 * The clone is *NOT* freed actually here because it is alloced 1334 * from dm own mempool (REQ_ALLOCED isn't set). 1335 */ 1336 __blk_put_request(clone->q, clone); 1337 } 1338 1339 /* 1340 * Actual request completion is done in a softirq context which doesn't 1341 * hold the clone's queue lock. Otherwise, deadlock could occur because: 1342 * - another request may be submitted by the upper level driver 1343 * of the stacking during the completion 1344 * - the submission which requires queue lock may be done 1345 * against this clone's queue 1346 */ 1347 dm_complete_request(tio->orig, error); 1348 } 1349 1350 /* 1351 * Return maximum size of I/O possible at the supplied sector up to the current 1352 * target boundary. 1353 */ 1354 static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti) 1355 { 1356 sector_t target_offset = dm_target_offset(ti, sector); 1357 1358 return ti->len - target_offset; 1359 } 1360 1361 static sector_t max_io_len(sector_t sector, struct dm_target *ti) 1362 { 1363 sector_t len = max_io_len_target_boundary(sector, ti); 1364 sector_t offset, max_len; 1365 1366 /* 1367 * Does the target need to split even further? 1368 */ 1369 if (ti->max_io_len) { 1370 offset = dm_target_offset(ti, sector); 1371 if (unlikely(ti->max_io_len & (ti->max_io_len - 1))) 1372 max_len = sector_div(offset, ti->max_io_len); 1373 else 1374 max_len = offset & (ti->max_io_len - 1); 1375 max_len = ti->max_io_len - max_len; 1376 1377 if (len > max_len) 1378 len = max_len; 1379 } 1380 1381 return len; 1382 } 1383 1384 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) 1385 { 1386 if (len > UINT_MAX) { 1387 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", 1388 (unsigned long long)len, UINT_MAX); 1389 ti->error = "Maximum size of target IO is too large"; 1390 return -EINVAL; 1391 } 1392 1393 ti->max_io_len = (uint32_t) len; 1394 1395 return 0; 1396 } 1397 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); 1398 1399 /* 1400 * A target may call dm_accept_partial_bio only from the map routine. It is 1401 * allowed for all bio types except REQ_FLUSH. 1402 * 1403 * dm_accept_partial_bio informs the dm that the target only wants to process 1404 * additional n_sectors sectors of the bio and the rest of the data should be 1405 * sent in a next bio. 1406 * 1407 * A diagram that explains the arithmetics: 1408 * +--------------------+---------------+-------+ 1409 * | 1 | 2 | 3 | 1410 * +--------------------+---------------+-------+ 1411 * 1412 * <-------------- *tio->len_ptr ---------------> 1413 * <------- bi_size -------> 1414 * <-- n_sectors --> 1415 * 1416 * Region 1 was already iterated over with bio_advance or similar function. 1417 * (it may be empty if the target doesn't use bio_advance) 1418 * Region 2 is the remaining bio size that the target wants to process. 1419 * (it may be empty if region 1 is non-empty, although there is no reason 1420 * to make it empty) 1421 * The target requires that region 3 is to be sent in the next bio. 1422 * 1423 * If the target wants to receive multiple copies of the bio (via num_*bios, etc), 1424 * the partially processed part (the sum of regions 1+2) must be the same for all 1425 * copies of the bio. 1426 */ 1427 void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) 1428 { 1429 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 1430 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT; 1431 BUG_ON(bio->bi_rw & REQ_FLUSH); 1432 BUG_ON(bi_size > *tio->len_ptr); 1433 BUG_ON(n_sectors > bi_size); 1434 *tio->len_ptr -= bi_size - n_sectors; 1435 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; 1436 } 1437 EXPORT_SYMBOL_GPL(dm_accept_partial_bio); 1438 1439 static void __map_bio(struct dm_target_io *tio) 1440 { 1441 int r; 1442 sector_t sector; 1443 struct mapped_device *md; 1444 struct bio *clone = &tio->clone; 1445 struct dm_target *ti = tio->ti; 1446 1447 clone->bi_end_io = clone_endio; 1448 1449 /* 1450 * Map the clone. If r == 0 we don't need to do 1451 * anything, the target has assumed ownership of 1452 * this io. 1453 */ 1454 atomic_inc(&tio->io->io_count); 1455 sector = clone->bi_iter.bi_sector; 1456 r = ti->type->map(ti, clone); 1457 if (r == DM_MAPIO_REMAPPED) { 1458 /* the bio has been remapped so dispatch it */ 1459 1460 trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone, 1461 tio->io->bio->bi_bdev->bd_dev, sector); 1462 1463 generic_make_request(clone); 1464 } else if (r < 0 || r == DM_MAPIO_REQUEUE) { 1465 /* error the io and bail out, or requeue it if needed */ 1466 md = tio->io->md; 1467 dec_pending(tio->io, r); 1468 free_tio(md, tio); 1469 } else if (r) { 1470 DMWARN("unimplemented target map return value: %d", r); 1471 BUG(); 1472 } 1473 } 1474 1475 struct clone_info { 1476 struct mapped_device *md; 1477 struct dm_table *map; 1478 struct bio *bio; 1479 struct dm_io *io; 1480 sector_t sector; 1481 unsigned sector_count; 1482 }; 1483 1484 static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len) 1485 { 1486 bio->bi_iter.bi_sector = sector; 1487 bio->bi_iter.bi_size = to_bytes(len); 1488 } 1489 1490 /* 1491 * Creates a bio that consists of range of complete bvecs. 1492 */ 1493 static void clone_bio(struct dm_target_io *tio, struct bio *bio, 1494 sector_t sector, unsigned len) 1495 { 1496 struct bio *clone = &tio->clone; 1497 1498 __bio_clone_fast(clone, bio); 1499 1500 if (bio_integrity(bio)) 1501 bio_integrity_clone(clone, bio, GFP_NOIO); 1502 1503 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); 1504 clone->bi_iter.bi_size = to_bytes(len); 1505 1506 if (bio_integrity(bio)) 1507 bio_integrity_trim(clone, 0, len); 1508 } 1509 1510 static struct dm_target_io *alloc_tio(struct clone_info *ci, 1511 struct dm_target *ti, 1512 unsigned target_bio_nr) 1513 { 1514 struct dm_target_io *tio; 1515 struct bio *clone; 1516 1517 clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs); 1518 tio = container_of(clone, struct dm_target_io, clone); 1519 1520 tio->io = ci->io; 1521 tio->ti = ti; 1522 tio->target_bio_nr = target_bio_nr; 1523 1524 return tio; 1525 } 1526 1527 static void __clone_and_map_simple_bio(struct clone_info *ci, 1528 struct dm_target *ti, 1529 unsigned target_bio_nr, unsigned *len) 1530 { 1531 struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr); 1532 struct bio *clone = &tio->clone; 1533 1534 tio->len_ptr = len; 1535 1536 __bio_clone_fast(clone, ci->bio); 1537 if (len) 1538 bio_setup_sector(clone, ci->sector, *len); 1539 1540 __map_bio(tio); 1541 } 1542 1543 static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, 1544 unsigned num_bios, unsigned *len) 1545 { 1546 unsigned target_bio_nr; 1547 1548 for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++) 1549 __clone_and_map_simple_bio(ci, ti, target_bio_nr, len); 1550 } 1551 1552 static int __send_empty_flush(struct clone_info *ci) 1553 { 1554 unsigned target_nr = 0; 1555 struct dm_target *ti; 1556 1557 BUG_ON(bio_has_data(ci->bio)); 1558 while ((ti = dm_table_get_target(ci->map, target_nr++))) 1559 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); 1560 1561 return 0; 1562 } 1563 1564 static void __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti, 1565 sector_t sector, unsigned *len) 1566 { 1567 struct bio *bio = ci->bio; 1568 struct dm_target_io *tio; 1569 unsigned target_bio_nr; 1570 unsigned num_target_bios = 1; 1571 1572 /* 1573 * Does the target want to receive duplicate copies of the bio? 1574 */ 1575 if (bio_data_dir(bio) == WRITE && ti->num_write_bios) 1576 num_target_bios = ti->num_write_bios(ti, bio); 1577 1578 for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) { 1579 tio = alloc_tio(ci, ti, target_bio_nr); 1580 tio->len_ptr = len; 1581 clone_bio(tio, bio, sector, *len); 1582 __map_bio(tio); 1583 } 1584 } 1585 1586 typedef unsigned (*get_num_bios_fn)(struct dm_target *ti); 1587 1588 static unsigned get_num_discard_bios(struct dm_target *ti) 1589 { 1590 return ti->num_discard_bios; 1591 } 1592 1593 static unsigned get_num_write_same_bios(struct dm_target *ti) 1594 { 1595 return ti->num_write_same_bios; 1596 } 1597 1598 typedef bool (*is_split_required_fn)(struct dm_target *ti); 1599 1600 static bool is_split_required_for_discard(struct dm_target *ti) 1601 { 1602 return ti->split_discard_bios; 1603 } 1604 1605 static int __send_changing_extent_only(struct clone_info *ci, 1606 get_num_bios_fn get_num_bios, 1607 is_split_required_fn is_split_required) 1608 { 1609 struct dm_target *ti; 1610 unsigned len; 1611 unsigned num_bios; 1612 1613 do { 1614 ti = dm_table_find_target(ci->map, ci->sector); 1615 if (!dm_target_is_valid(ti)) 1616 return -EIO; 1617 1618 /* 1619 * Even though the device advertised support for this type of 1620 * request, that does not mean every target supports it, and 1621 * reconfiguration might also have changed that since the 1622 * check was performed. 1623 */ 1624 num_bios = get_num_bios ? get_num_bios(ti) : 0; 1625 if (!num_bios) 1626 return -EOPNOTSUPP; 1627 1628 if (is_split_required && !is_split_required(ti)) 1629 len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti)); 1630 else 1631 len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti)); 1632 1633 __send_duplicate_bios(ci, ti, num_bios, &len); 1634 1635 ci->sector += len; 1636 } while (ci->sector_count -= len); 1637 1638 return 0; 1639 } 1640 1641 static int __send_discard(struct clone_info *ci) 1642 { 1643 return __send_changing_extent_only(ci, get_num_discard_bios, 1644 is_split_required_for_discard); 1645 } 1646 1647 static int __send_write_same(struct clone_info *ci) 1648 { 1649 return __send_changing_extent_only(ci, get_num_write_same_bios, NULL); 1650 } 1651 1652 /* 1653 * Select the correct strategy for processing a non-flush bio. 1654 */ 1655 static int __split_and_process_non_flush(struct clone_info *ci) 1656 { 1657 struct bio *bio = ci->bio; 1658 struct dm_target *ti; 1659 unsigned len; 1660 1661 if (unlikely(bio->bi_rw & REQ_DISCARD)) 1662 return __send_discard(ci); 1663 else if (unlikely(bio->bi_rw & REQ_WRITE_SAME)) 1664 return __send_write_same(ci); 1665 1666 ti = dm_table_find_target(ci->map, ci->sector); 1667 if (!dm_target_is_valid(ti)) 1668 return -EIO; 1669 1670 len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count); 1671 1672 __clone_and_map_data_bio(ci, ti, ci->sector, &len); 1673 1674 ci->sector += len; 1675 ci->sector_count -= len; 1676 1677 return 0; 1678 } 1679 1680 /* 1681 * Entry point to split a bio into clones and submit them to the targets. 1682 */ 1683 static void __split_and_process_bio(struct mapped_device *md, 1684 struct dm_table *map, struct bio *bio) 1685 { 1686 struct clone_info ci; 1687 int error = 0; 1688 1689 if (unlikely(!map)) { 1690 bio_io_error(bio); 1691 return; 1692 } 1693 1694 ci.map = map; 1695 ci.md = md; 1696 ci.io = alloc_io(md); 1697 ci.io->error = 0; 1698 atomic_set(&ci.io->io_count, 1); 1699 ci.io->bio = bio; 1700 ci.io->md = md; 1701 spin_lock_init(&ci.io->endio_lock); 1702 ci.sector = bio->bi_iter.bi_sector; 1703 1704 start_io_acct(ci.io); 1705 1706 if (bio->bi_rw & REQ_FLUSH) { 1707 ci.bio = &ci.md->flush_bio; 1708 ci.sector_count = 0; 1709 error = __send_empty_flush(&ci); 1710 /* dec_pending submits any data associated with flush */ 1711 } else { 1712 ci.bio = bio; 1713 ci.sector_count = bio_sectors(bio); 1714 while (ci.sector_count && !error) 1715 error = __split_and_process_non_flush(&ci); 1716 } 1717 1718 /* drop the extra reference count */ 1719 dec_pending(ci.io, error); 1720 } 1721 /*----------------------------------------------------------------- 1722 * CRUD END 1723 *---------------------------------------------------------------*/ 1724 1725 static int dm_merge_bvec(struct request_queue *q, 1726 struct bvec_merge_data *bvm, 1727 struct bio_vec *biovec) 1728 { 1729 struct mapped_device *md = q->queuedata; 1730 struct dm_table *map = dm_get_live_table_fast(md); 1731 struct dm_target *ti; 1732 sector_t max_sectors; 1733 int max_size = 0; 1734 1735 if (unlikely(!map)) 1736 goto out; 1737 1738 ti = dm_table_find_target(map, bvm->bi_sector); 1739 if (!dm_target_is_valid(ti)) 1740 goto out; 1741 1742 /* 1743 * Find maximum amount of I/O that won't need splitting 1744 */ 1745 max_sectors = min(max_io_len(bvm->bi_sector, ti), 1746 (sector_t) BIO_MAX_SECTORS); 1747 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size; 1748 if (max_size < 0) 1749 max_size = 0; 1750 1751 /* 1752 * merge_bvec_fn() returns number of bytes 1753 * it can accept at this offset 1754 * max is precomputed maximal io size 1755 */ 1756 if (max_size && ti->type->merge) 1757 max_size = ti->type->merge(ti, bvm, biovec, max_size); 1758 /* 1759 * If the target doesn't support merge method and some of the devices 1760 * provided their merge_bvec method (we know this by looking at 1761 * queue_max_hw_sectors), then we can't allow bios with multiple vector 1762 * entries. So always set max_size to 0, and the code below allows 1763 * just one page. 1764 */ 1765 else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9) 1766 max_size = 0; 1767 1768 out: 1769 dm_put_live_table_fast(md); 1770 /* 1771 * Always allow an entire first page 1772 */ 1773 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT)) 1774 max_size = biovec->bv_len; 1775 1776 return max_size; 1777 } 1778 1779 /* 1780 * The request function that just remaps the bio built up by 1781 * dm_merge_bvec. 1782 */ 1783 static void dm_make_request(struct request_queue *q, struct bio *bio) 1784 { 1785 int rw = bio_data_dir(bio); 1786 struct mapped_device *md = q->queuedata; 1787 int srcu_idx; 1788 struct dm_table *map; 1789 1790 map = dm_get_live_table(md, &srcu_idx); 1791 1792 generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0); 1793 1794 /* if we're suspended, we have to queue this io for later */ 1795 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { 1796 dm_put_live_table(md, srcu_idx); 1797 1798 if (bio_rw(bio) != READA) 1799 queue_io(md, bio); 1800 else 1801 bio_io_error(bio); 1802 return; 1803 } 1804 1805 __split_and_process_bio(md, map, bio); 1806 dm_put_live_table(md, srcu_idx); 1807 return; 1808 } 1809 1810 int dm_request_based(struct mapped_device *md) 1811 { 1812 return blk_queue_stackable(md->queue); 1813 } 1814 1815 static void dm_dispatch_clone_request(struct request *clone, struct request *rq) 1816 { 1817 int r; 1818 1819 if (blk_queue_io_stat(clone->q)) 1820 clone->cmd_flags |= REQ_IO_STAT; 1821 1822 clone->start_time = jiffies; 1823 r = blk_insert_cloned_request(clone->q, clone); 1824 if (r) 1825 /* must complete clone in terms of original request */ 1826 dm_complete_request(rq, r); 1827 } 1828 1829 static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig, 1830 void *data) 1831 { 1832 struct dm_rq_target_io *tio = data; 1833 struct dm_rq_clone_bio_info *info = 1834 container_of(bio, struct dm_rq_clone_bio_info, clone); 1835 1836 info->orig = bio_orig; 1837 info->tio = tio; 1838 bio->bi_end_io = end_clone_bio; 1839 1840 return 0; 1841 } 1842 1843 static int setup_clone(struct request *clone, struct request *rq, 1844 struct dm_rq_target_io *tio, gfp_t gfp_mask) 1845 { 1846 int r; 1847 1848 r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask, 1849 dm_rq_bio_constructor, tio); 1850 if (r) 1851 return r; 1852 1853 clone->cmd = rq->cmd; 1854 clone->cmd_len = rq->cmd_len; 1855 clone->sense = rq->sense; 1856 clone->end_io = end_clone_request; 1857 clone->end_io_data = tio; 1858 1859 tio->clone = clone; 1860 1861 return 0; 1862 } 1863 1864 static struct request *clone_rq(struct request *rq, struct mapped_device *md, 1865 struct dm_rq_target_io *tio, gfp_t gfp_mask) 1866 { 1867 /* 1868 * Do not allocate a clone if tio->clone was already set 1869 * (see: dm_mq_queue_rq). 1870 */ 1871 bool alloc_clone = !tio->clone; 1872 struct request *clone; 1873 1874 if (alloc_clone) { 1875 clone = alloc_clone_request(md, gfp_mask); 1876 if (!clone) 1877 return NULL; 1878 } else 1879 clone = tio->clone; 1880 1881 blk_rq_init(NULL, clone); 1882 if (setup_clone(clone, rq, tio, gfp_mask)) { 1883 /* -ENOMEM */ 1884 if (alloc_clone) 1885 free_clone_request(md, clone); 1886 return NULL; 1887 } 1888 1889 return clone; 1890 } 1891 1892 static void map_tio_request(struct kthread_work *work); 1893 1894 static void init_tio(struct dm_rq_target_io *tio, struct request *rq, 1895 struct mapped_device *md) 1896 { 1897 tio->md = md; 1898 tio->ti = NULL; 1899 tio->clone = NULL; 1900 tio->orig = rq; 1901 tio->error = 0; 1902 memset(&tio->info, 0, sizeof(tio->info)); 1903 if (md->kworker_task) 1904 init_kthread_work(&tio->work, map_tio_request); 1905 } 1906 1907 static struct dm_rq_target_io *prep_tio(struct request *rq, 1908 struct mapped_device *md, gfp_t gfp_mask) 1909 { 1910 struct dm_rq_target_io *tio; 1911 int srcu_idx; 1912 struct dm_table *table; 1913 1914 tio = alloc_rq_tio(md, gfp_mask); 1915 if (!tio) 1916 return NULL; 1917 1918 init_tio(tio, rq, md); 1919 1920 table = dm_get_live_table(md, &srcu_idx); 1921 if (!dm_table_mq_request_based(table)) { 1922 if (!clone_rq(rq, md, tio, gfp_mask)) { 1923 dm_put_live_table(md, srcu_idx); 1924 free_rq_tio(tio); 1925 return NULL; 1926 } 1927 } 1928 dm_put_live_table(md, srcu_idx); 1929 1930 return tio; 1931 } 1932 1933 /* 1934 * Called with the queue lock held. 1935 */ 1936 static int dm_prep_fn(struct request_queue *q, struct request *rq) 1937 { 1938 struct mapped_device *md = q->queuedata; 1939 struct dm_rq_target_io *tio; 1940 1941 if (unlikely(rq->special)) { 1942 DMWARN("Already has something in rq->special."); 1943 return BLKPREP_KILL; 1944 } 1945 1946 tio = prep_tio(rq, md, GFP_ATOMIC); 1947 if (!tio) 1948 return BLKPREP_DEFER; 1949 1950 rq->special = tio; 1951 rq->cmd_flags |= REQ_DONTPREP; 1952 1953 return BLKPREP_OK; 1954 } 1955 1956 /* 1957 * Returns: 1958 * 0 : the request has been processed 1959 * DM_MAPIO_REQUEUE : the original request needs to be requeued 1960 * < 0 : the request was completed due to failure 1961 */ 1962 static int map_request(struct dm_rq_target_io *tio, struct request *rq, 1963 struct mapped_device *md) 1964 { 1965 int r; 1966 struct dm_target *ti = tio->ti; 1967 struct request *clone = NULL; 1968 1969 if (tio->clone) { 1970 clone = tio->clone; 1971 r = ti->type->map_rq(ti, clone, &tio->info); 1972 } else { 1973 r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone); 1974 if (r < 0) { 1975 /* The target wants to complete the I/O */ 1976 dm_kill_unmapped_request(rq, r); 1977 return r; 1978 } 1979 if (r != DM_MAPIO_REMAPPED) 1980 return r; 1981 if (setup_clone(clone, rq, tio, GFP_ATOMIC)) { 1982 /* -ENOMEM */ 1983 ti->type->release_clone_rq(clone); 1984 return DM_MAPIO_REQUEUE; 1985 } 1986 } 1987 1988 switch (r) { 1989 case DM_MAPIO_SUBMITTED: 1990 /* The target has taken the I/O to submit by itself later */ 1991 break; 1992 case DM_MAPIO_REMAPPED: 1993 /* The target has remapped the I/O so dispatch it */ 1994 trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)), 1995 blk_rq_pos(rq)); 1996 dm_dispatch_clone_request(clone, rq); 1997 break; 1998 case DM_MAPIO_REQUEUE: 1999 /* The target wants to requeue the I/O */ 2000 dm_requeue_original_request(md, tio->orig); 2001 break; 2002 default: 2003 if (r > 0) { 2004 DMWARN("unimplemented target map return value: %d", r); 2005 BUG(); 2006 } 2007 2008 /* The target wants to complete the I/O */ 2009 dm_kill_unmapped_request(rq, r); 2010 return r; 2011 } 2012 2013 return 0; 2014 } 2015 2016 static void map_tio_request(struct kthread_work *work) 2017 { 2018 struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work); 2019 struct request *rq = tio->orig; 2020 struct mapped_device *md = tio->md; 2021 2022 if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) 2023 dm_requeue_original_request(md, rq); 2024 } 2025 2026 static void dm_start_request(struct mapped_device *md, struct request *orig) 2027 { 2028 if (!orig->q->mq_ops) 2029 blk_start_request(orig); 2030 else 2031 blk_mq_start_request(orig); 2032 atomic_inc(&md->pending[rq_data_dir(orig)]); 2033 2034 if (md->seq_rq_merge_deadline_usecs) { 2035 md->last_rq_pos = rq_end_sector(orig); 2036 md->last_rq_rw = rq_data_dir(orig); 2037 md->last_rq_start_time = ktime_get(); 2038 } 2039 2040 if (unlikely(dm_stats_used(&md->stats))) { 2041 struct dm_rq_target_io *tio = tio_from_request(orig); 2042 tio->duration_jiffies = jiffies; 2043 tio->n_sectors = blk_rq_sectors(orig); 2044 dm_stats_account_io(&md->stats, orig->cmd_flags, blk_rq_pos(orig), 2045 tio->n_sectors, false, 0, &tio->stats_aux); 2046 } 2047 2048 /* 2049 * Hold the md reference here for the in-flight I/O. 2050 * We can't rely on the reference count by device opener, 2051 * because the device may be closed during the request completion 2052 * when all bios are completed. 2053 * See the comment in rq_completed() too. 2054 */ 2055 dm_get(md); 2056 } 2057 2058 #define MAX_SEQ_RQ_MERGE_DEADLINE_USECS 100000 2059 2060 ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf) 2061 { 2062 return sprintf(buf, "%u\n", md->seq_rq_merge_deadline_usecs); 2063 } 2064 2065 ssize_t dm_attr_rq_based_seq_io_merge_deadline_store(struct mapped_device *md, 2066 const char *buf, size_t count) 2067 { 2068 unsigned deadline; 2069 2070 if (!dm_request_based(md) || md->use_blk_mq) 2071 return count; 2072 2073 if (kstrtouint(buf, 10, &deadline)) 2074 return -EINVAL; 2075 2076 if (deadline > MAX_SEQ_RQ_MERGE_DEADLINE_USECS) 2077 deadline = MAX_SEQ_RQ_MERGE_DEADLINE_USECS; 2078 2079 md->seq_rq_merge_deadline_usecs = deadline; 2080 2081 return count; 2082 } 2083 2084 static bool dm_request_peeked_before_merge_deadline(struct mapped_device *md) 2085 { 2086 ktime_t kt_deadline; 2087 2088 if (!md->seq_rq_merge_deadline_usecs) 2089 return false; 2090 2091 kt_deadline = ns_to_ktime((u64)md->seq_rq_merge_deadline_usecs * NSEC_PER_USEC); 2092 kt_deadline = ktime_add_safe(md->last_rq_start_time, kt_deadline); 2093 2094 return !ktime_after(ktime_get(), kt_deadline); 2095 } 2096 2097 /* 2098 * q->request_fn for request-based dm. 2099 * Called with the queue lock held. 2100 */ 2101 static void dm_request_fn(struct request_queue *q) 2102 { 2103 struct mapped_device *md = q->queuedata; 2104 int srcu_idx; 2105 struct dm_table *map = dm_get_live_table(md, &srcu_idx); 2106 struct dm_target *ti; 2107 struct request *rq; 2108 struct dm_rq_target_io *tio; 2109 sector_t pos; 2110 2111 /* 2112 * For suspend, check blk_queue_stopped() and increment 2113 * ->pending within a single queue_lock not to increment the 2114 * number of in-flight I/Os after the queue is stopped in 2115 * dm_suspend(). 2116 */ 2117 while (!blk_queue_stopped(q)) { 2118 rq = blk_peek_request(q); 2119 if (!rq) 2120 goto out; 2121 2122 /* always use block 0 to find the target for flushes for now */ 2123 pos = 0; 2124 if (!(rq->cmd_flags & REQ_FLUSH)) 2125 pos = blk_rq_pos(rq); 2126 2127 ti = dm_table_find_target(map, pos); 2128 if (!dm_target_is_valid(ti)) { 2129 /* 2130 * Must perform setup, that rq_completed() requires, 2131 * before calling dm_kill_unmapped_request 2132 */ 2133 DMERR_LIMIT("request attempted access beyond the end of device"); 2134 dm_start_request(md, rq); 2135 dm_kill_unmapped_request(rq, -EIO); 2136 continue; 2137 } 2138 2139 if (dm_request_peeked_before_merge_deadline(md) && 2140 md_in_flight(md) && rq->bio && rq->bio->bi_vcnt == 1 && 2141 md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq)) 2142 goto delay_and_out; 2143 2144 if (ti->type->busy && ti->type->busy(ti)) 2145 goto delay_and_out; 2146 2147 dm_start_request(md, rq); 2148 2149 tio = tio_from_request(rq); 2150 /* Establish tio->ti before queuing work (map_tio_request) */ 2151 tio->ti = ti; 2152 queue_kthread_work(&md->kworker, &tio->work); 2153 BUG_ON(!irqs_disabled()); 2154 } 2155 2156 goto out; 2157 2158 delay_and_out: 2159 blk_delay_queue(q, HZ / 100); 2160 out: 2161 dm_put_live_table(md, srcu_idx); 2162 } 2163 2164 static int dm_any_congested(void *congested_data, int bdi_bits) 2165 { 2166 int r = bdi_bits; 2167 struct mapped_device *md = congested_data; 2168 struct dm_table *map; 2169 2170 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2171 map = dm_get_live_table_fast(md); 2172 if (map) { 2173 /* 2174 * Request-based dm cares about only own queue for 2175 * the query about congestion status of request_queue 2176 */ 2177 if (dm_request_based(md)) 2178 r = md->queue->backing_dev_info.wb.state & 2179 bdi_bits; 2180 else 2181 r = dm_table_any_congested(map, bdi_bits); 2182 } 2183 dm_put_live_table_fast(md); 2184 } 2185 2186 return r; 2187 } 2188 2189 /*----------------------------------------------------------------- 2190 * An IDR is used to keep track of allocated minor numbers. 2191 *---------------------------------------------------------------*/ 2192 static void free_minor(int minor) 2193 { 2194 spin_lock(&_minor_lock); 2195 idr_remove(&_minor_idr, minor); 2196 spin_unlock(&_minor_lock); 2197 } 2198 2199 /* 2200 * See if the device with a specific minor # is free. 2201 */ 2202 static int specific_minor(int minor) 2203 { 2204 int r; 2205 2206 if (minor >= (1 << MINORBITS)) 2207 return -EINVAL; 2208 2209 idr_preload(GFP_KERNEL); 2210 spin_lock(&_minor_lock); 2211 2212 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); 2213 2214 spin_unlock(&_minor_lock); 2215 idr_preload_end(); 2216 if (r < 0) 2217 return r == -ENOSPC ? -EBUSY : r; 2218 return 0; 2219 } 2220 2221 static int next_free_minor(int *minor) 2222 { 2223 int r; 2224 2225 idr_preload(GFP_KERNEL); 2226 spin_lock(&_minor_lock); 2227 2228 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); 2229 2230 spin_unlock(&_minor_lock); 2231 idr_preload_end(); 2232 if (r < 0) 2233 return r; 2234 *minor = r; 2235 return 0; 2236 } 2237 2238 static const struct block_device_operations dm_blk_dops; 2239 2240 static void dm_wq_work(struct work_struct *work); 2241 2242 static void dm_init_md_queue(struct mapped_device *md) 2243 { 2244 /* 2245 * Request-based dm devices cannot be stacked on top of bio-based dm 2246 * devices. The type of this dm device may not have been decided yet. 2247 * The type is decided at the first table loading time. 2248 * To prevent problematic device stacking, clear the queue flag 2249 * for request stacking support until then. 2250 * 2251 * This queue is new, so no concurrency on the queue_flags. 2252 */ 2253 queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue); 2254 } 2255 2256 static void dm_init_old_md_queue(struct mapped_device *md) 2257 { 2258 md->use_blk_mq = false; 2259 dm_init_md_queue(md); 2260 2261 /* 2262 * Initialize aspects of queue that aren't relevant for blk-mq 2263 */ 2264 md->queue->queuedata = md; 2265 md->queue->backing_dev_info.congested_fn = dm_any_congested; 2266 md->queue->backing_dev_info.congested_data = md; 2267 2268 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY); 2269 } 2270 2271 static void cleanup_mapped_device(struct mapped_device *md) 2272 { 2273 if (md->wq) 2274 destroy_workqueue(md->wq); 2275 if (md->kworker_task) 2276 kthread_stop(md->kworker_task); 2277 if (md->io_pool) 2278 mempool_destroy(md->io_pool); 2279 if (md->rq_pool) 2280 mempool_destroy(md->rq_pool); 2281 if (md->bs) 2282 bioset_free(md->bs); 2283 2284 cleanup_srcu_struct(&md->io_barrier); 2285 2286 if (md->disk) { 2287 spin_lock(&_minor_lock); 2288 md->disk->private_data = NULL; 2289 spin_unlock(&_minor_lock); 2290 if (blk_get_integrity(md->disk)) 2291 blk_integrity_unregister(md->disk); 2292 del_gendisk(md->disk); 2293 put_disk(md->disk); 2294 } 2295 2296 if (md->queue) 2297 blk_cleanup_queue(md->queue); 2298 2299 if (md->bdev) { 2300 bdput(md->bdev); 2301 md->bdev = NULL; 2302 } 2303 } 2304 2305 /* 2306 * Allocate and initialise a blank device with a given minor. 2307 */ 2308 static struct mapped_device *alloc_dev(int minor) 2309 { 2310 int r; 2311 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL); 2312 void *old_md; 2313 2314 if (!md) { 2315 DMWARN("unable to allocate device, out of memory."); 2316 return NULL; 2317 } 2318 2319 if (!try_module_get(THIS_MODULE)) 2320 goto bad_module_get; 2321 2322 /* get a minor number for the dev */ 2323 if (minor == DM_ANY_MINOR) 2324 r = next_free_minor(&minor); 2325 else 2326 r = specific_minor(minor); 2327 if (r < 0) 2328 goto bad_minor; 2329 2330 r = init_srcu_struct(&md->io_barrier); 2331 if (r < 0) 2332 goto bad_io_barrier; 2333 2334 md->use_blk_mq = use_blk_mq; 2335 md->type = DM_TYPE_NONE; 2336 mutex_init(&md->suspend_lock); 2337 mutex_init(&md->type_lock); 2338 mutex_init(&md->table_devices_lock); 2339 spin_lock_init(&md->deferred_lock); 2340 atomic_set(&md->holders, 1); 2341 atomic_set(&md->open_count, 0); 2342 atomic_set(&md->event_nr, 0); 2343 atomic_set(&md->uevent_seq, 0); 2344 INIT_LIST_HEAD(&md->uevent_list); 2345 INIT_LIST_HEAD(&md->table_devices); 2346 spin_lock_init(&md->uevent_lock); 2347 2348 md->queue = blk_alloc_queue(GFP_KERNEL); 2349 if (!md->queue) 2350 goto bad; 2351 2352 dm_init_md_queue(md); 2353 2354 md->disk = alloc_disk(1); 2355 if (!md->disk) 2356 goto bad; 2357 2358 atomic_set(&md->pending[0], 0); 2359 atomic_set(&md->pending[1], 0); 2360 init_waitqueue_head(&md->wait); 2361 INIT_WORK(&md->work, dm_wq_work); 2362 init_waitqueue_head(&md->eventq); 2363 init_completion(&md->kobj_holder.completion); 2364 md->kworker_task = NULL; 2365 2366 md->disk->major = _major; 2367 md->disk->first_minor = minor; 2368 md->disk->fops = &dm_blk_dops; 2369 md->disk->queue = md->queue; 2370 md->disk->private_data = md; 2371 sprintf(md->disk->disk_name, "dm-%d", minor); 2372 add_disk(md->disk); 2373 format_dev_t(md->name, MKDEV(_major, minor)); 2374 2375 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0); 2376 if (!md->wq) 2377 goto bad; 2378 2379 md->bdev = bdget_disk(md->disk, 0); 2380 if (!md->bdev) 2381 goto bad; 2382 2383 bio_init(&md->flush_bio); 2384 md->flush_bio.bi_bdev = md->bdev; 2385 md->flush_bio.bi_rw = WRITE_FLUSH; 2386 2387 dm_stats_init(&md->stats); 2388 2389 /* Populate the mapping, nobody knows we exist yet */ 2390 spin_lock(&_minor_lock); 2391 old_md = idr_replace(&_minor_idr, md, minor); 2392 spin_unlock(&_minor_lock); 2393 2394 BUG_ON(old_md != MINOR_ALLOCED); 2395 2396 return md; 2397 2398 bad: 2399 cleanup_mapped_device(md); 2400 bad_io_barrier: 2401 free_minor(minor); 2402 bad_minor: 2403 module_put(THIS_MODULE); 2404 bad_module_get: 2405 kfree(md); 2406 return NULL; 2407 } 2408 2409 static void unlock_fs(struct mapped_device *md); 2410 2411 static void free_dev(struct mapped_device *md) 2412 { 2413 int minor = MINOR(disk_devt(md->disk)); 2414 2415 unlock_fs(md); 2416 2417 cleanup_mapped_device(md); 2418 if (md->use_blk_mq) 2419 blk_mq_free_tag_set(&md->tag_set); 2420 2421 free_table_devices(&md->table_devices); 2422 dm_stats_cleanup(&md->stats); 2423 free_minor(minor); 2424 2425 module_put(THIS_MODULE); 2426 kfree(md); 2427 } 2428 2429 static void __bind_mempools(struct mapped_device *md, struct dm_table *t) 2430 { 2431 struct dm_md_mempools *p = dm_table_get_md_mempools(t); 2432 2433 if (md->bs) { 2434 /* The md already has necessary mempools. */ 2435 if (dm_table_get_type(t) == DM_TYPE_BIO_BASED) { 2436 /* 2437 * Reload bioset because front_pad may have changed 2438 * because a different table was loaded. 2439 */ 2440 bioset_free(md->bs); 2441 md->bs = p->bs; 2442 p->bs = NULL; 2443 } 2444 /* 2445 * There's no need to reload with request-based dm 2446 * because the size of front_pad doesn't change. 2447 * Note for future: If you are to reload bioset, 2448 * prep-ed requests in the queue may refer 2449 * to bio from the old bioset, so you must walk 2450 * through the queue to unprep. 2451 */ 2452 goto out; 2453 } 2454 2455 BUG_ON(!p || md->io_pool || md->rq_pool || md->bs); 2456 2457 md->io_pool = p->io_pool; 2458 p->io_pool = NULL; 2459 md->rq_pool = p->rq_pool; 2460 p->rq_pool = NULL; 2461 md->bs = p->bs; 2462 p->bs = NULL; 2463 2464 out: 2465 /* mempool bind completed, no longer need any mempools in the table */ 2466 dm_table_free_md_mempools(t); 2467 } 2468 2469 /* 2470 * Bind a table to the device. 2471 */ 2472 static void event_callback(void *context) 2473 { 2474 unsigned long flags; 2475 LIST_HEAD(uevents); 2476 struct mapped_device *md = (struct mapped_device *) context; 2477 2478 spin_lock_irqsave(&md->uevent_lock, flags); 2479 list_splice_init(&md->uevent_list, &uevents); 2480 spin_unlock_irqrestore(&md->uevent_lock, flags); 2481 2482 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 2483 2484 atomic_inc(&md->event_nr); 2485 wake_up(&md->eventq); 2486 } 2487 2488 /* 2489 * Protected by md->suspend_lock obtained by dm_swap_table(). 2490 */ 2491 static void __set_size(struct mapped_device *md, sector_t size) 2492 { 2493 set_capacity(md->disk, size); 2494 2495 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT); 2496 } 2497 2498 /* 2499 * Return 1 if the queue has a compulsory merge_bvec_fn function. 2500 * 2501 * If this function returns 0, then the device is either a non-dm 2502 * device without a merge_bvec_fn, or it is a dm device that is 2503 * able to split any bios it receives that are too big. 2504 */ 2505 int dm_queue_merge_is_compulsory(struct request_queue *q) 2506 { 2507 struct mapped_device *dev_md; 2508 2509 if (!q->merge_bvec_fn) 2510 return 0; 2511 2512 if (q->make_request_fn == dm_make_request) { 2513 dev_md = q->queuedata; 2514 if (test_bit(DMF_MERGE_IS_OPTIONAL, &dev_md->flags)) 2515 return 0; 2516 } 2517 2518 return 1; 2519 } 2520 2521 static int dm_device_merge_is_compulsory(struct dm_target *ti, 2522 struct dm_dev *dev, sector_t start, 2523 sector_t len, void *data) 2524 { 2525 struct block_device *bdev = dev->bdev; 2526 struct request_queue *q = bdev_get_queue(bdev); 2527 2528 return dm_queue_merge_is_compulsory(q); 2529 } 2530 2531 /* 2532 * Return 1 if it is acceptable to ignore merge_bvec_fn based 2533 * on the properties of the underlying devices. 2534 */ 2535 static int dm_table_merge_is_optional(struct dm_table *table) 2536 { 2537 unsigned i = 0; 2538 struct dm_target *ti; 2539 2540 while (i < dm_table_get_num_targets(table)) { 2541 ti = dm_table_get_target(table, i++); 2542 2543 if (ti->type->iterate_devices && 2544 ti->type->iterate_devices(ti, dm_device_merge_is_compulsory, NULL)) 2545 return 0; 2546 } 2547 2548 return 1; 2549 } 2550 2551 /* 2552 * Returns old map, which caller must destroy. 2553 */ 2554 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 2555 struct queue_limits *limits) 2556 { 2557 struct dm_table *old_map; 2558 struct request_queue *q = md->queue; 2559 sector_t size; 2560 int merge_is_optional; 2561 2562 size = dm_table_get_size(t); 2563 2564 /* 2565 * Wipe any geometry if the size of the table changed. 2566 */ 2567 if (size != dm_get_size(md)) 2568 memset(&md->geometry, 0, sizeof(md->geometry)); 2569 2570 __set_size(md, size); 2571 2572 dm_table_event_callback(t, event_callback, md); 2573 2574 /* 2575 * The queue hasn't been stopped yet, if the old table type wasn't 2576 * for request-based during suspension. So stop it to prevent 2577 * I/O mapping before resume. 2578 * This must be done before setting the queue restrictions, 2579 * because request-based dm may be run just after the setting. 2580 */ 2581 if (dm_table_request_based(t)) 2582 stop_queue(q); 2583 2584 __bind_mempools(md, t); 2585 2586 merge_is_optional = dm_table_merge_is_optional(t); 2587 2588 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2589 rcu_assign_pointer(md->map, t); 2590 md->immutable_target_type = dm_table_get_immutable_target_type(t); 2591 2592 dm_table_set_restrictions(t, q, limits); 2593 if (merge_is_optional) 2594 set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags); 2595 else 2596 clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags); 2597 if (old_map) 2598 dm_sync_table(md); 2599 2600 return old_map; 2601 } 2602 2603 /* 2604 * Returns unbound table for the caller to free. 2605 */ 2606 static struct dm_table *__unbind(struct mapped_device *md) 2607 { 2608 struct dm_table *map = rcu_dereference_protected(md->map, 1); 2609 2610 if (!map) 2611 return NULL; 2612 2613 dm_table_event_callback(map, NULL, NULL); 2614 RCU_INIT_POINTER(md->map, NULL); 2615 dm_sync_table(md); 2616 2617 return map; 2618 } 2619 2620 /* 2621 * Constructor for a new device. 2622 */ 2623 int dm_create(int minor, struct mapped_device **result) 2624 { 2625 struct mapped_device *md; 2626 2627 md = alloc_dev(minor); 2628 if (!md) 2629 return -ENXIO; 2630 2631 dm_sysfs_init(md); 2632 2633 *result = md; 2634 return 0; 2635 } 2636 2637 /* 2638 * Functions to manage md->type. 2639 * All are required to hold md->type_lock. 2640 */ 2641 void dm_lock_md_type(struct mapped_device *md) 2642 { 2643 mutex_lock(&md->type_lock); 2644 } 2645 2646 void dm_unlock_md_type(struct mapped_device *md) 2647 { 2648 mutex_unlock(&md->type_lock); 2649 } 2650 2651 void dm_set_md_type(struct mapped_device *md, unsigned type) 2652 { 2653 BUG_ON(!mutex_is_locked(&md->type_lock)); 2654 md->type = type; 2655 } 2656 2657 unsigned dm_get_md_type(struct mapped_device *md) 2658 { 2659 BUG_ON(!mutex_is_locked(&md->type_lock)); 2660 return md->type; 2661 } 2662 2663 struct target_type *dm_get_immutable_target_type(struct mapped_device *md) 2664 { 2665 return md->immutable_target_type; 2666 } 2667 2668 /* 2669 * The queue_limits are only valid as long as you have a reference 2670 * count on 'md'. 2671 */ 2672 struct queue_limits *dm_get_queue_limits(struct mapped_device *md) 2673 { 2674 BUG_ON(!atomic_read(&md->holders)); 2675 return &md->queue->limits; 2676 } 2677 EXPORT_SYMBOL_GPL(dm_get_queue_limits); 2678 2679 static void init_rq_based_worker_thread(struct mapped_device *md) 2680 { 2681 /* Initialize the request-based DM worker thread */ 2682 init_kthread_worker(&md->kworker); 2683 md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker, 2684 "kdmwork-%s", dm_device_name(md)); 2685 } 2686 2687 /* 2688 * Fully initialize a request-based queue (->elevator, ->request_fn, etc). 2689 */ 2690 static int dm_init_request_based_queue(struct mapped_device *md) 2691 { 2692 struct request_queue *q = NULL; 2693 2694 /* Fully initialize the queue */ 2695 q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL); 2696 if (!q) 2697 return -EINVAL; 2698 2699 /* disable dm_request_fn's merge heuristic by default */ 2700 md->seq_rq_merge_deadline_usecs = 0; 2701 2702 md->queue = q; 2703 dm_init_old_md_queue(md); 2704 blk_queue_softirq_done(md->queue, dm_softirq_done); 2705 blk_queue_prep_rq(md->queue, dm_prep_fn); 2706 2707 init_rq_based_worker_thread(md); 2708 2709 elv_register_queue(md->queue); 2710 2711 return 0; 2712 } 2713 2714 static int dm_mq_init_request(void *data, struct request *rq, 2715 unsigned int hctx_idx, unsigned int request_idx, 2716 unsigned int numa_node) 2717 { 2718 struct mapped_device *md = data; 2719 struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); 2720 2721 /* 2722 * Must initialize md member of tio, otherwise it won't 2723 * be available in dm_mq_queue_rq. 2724 */ 2725 tio->md = md; 2726 2727 return 0; 2728 } 2729 2730 static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, 2731 const struct blk_mq_queue_data *bd) 2732 { 2733 struct request *rq = bd->rq; 2734 struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); 2735 struct mapped_device *md = tio->md; 2736 int srcu_idx; 2737 struct dm_table *map = dm_get_live_table(md, &srcu_idx); 2738 struct dm_target *ti; 2739 sector_t pos; 2740 2741 /* always use block 0 to find the target for flushes for now */ 2742 pos = 0; 2743 if (!(rq->cmd_flags & REQ_FLUSH)) 2744 pos = blk_rq_pos(rq); 2745 2746 ti = dm_table_find_target(map, pos); 2747 if (!dm_target_is_valid(ti)) { 2748 dm_put_live_table(md, srcu_idx); 2749 DMERR_LIMIT("request attempted access beyond the end of device"); 2750 /* 2751 * Must perform setup, that rq_completed() requires, 2752 * before returning BLK_MQ_RQ_QUEUE_ERROR 2753 */ 2754 dm_start_request(md, rq); 2755 return BLK_MQ_RQ_QUEUE_ERROR; 2756 } 2757 dm_put_live_table(md, srcu_idx); 2758 2759 if (ti->type->busy && ti->type->busy(ti)) 2760 return BLK_MQ_RQ_QUEUE_BUSY; 2761 2762 dm_start_request(md, rq); 2763 2764 /* Init tio using md established in .init_request */ 2765 init_tio(tio, rq, md); 2766 2767 /* 2768 * Establish tio->ti before queuing work (map_tio_request) 2769 * or making direct call to map_request(). 2770 */ 2771 tio->ti = ti; 2772 2773 /* Clone the request if underlying devices aren't blk-mq */ 2774 if (dm_table_get_type(map) == DM_TYPE_REQUEST_BASED) { 2775 /* clone request is allocated at the end of the pdu */ 2776 tio->clone = (void *)blk_mq_rq_to_pdu(rq) + sizeof(struct dm_rq_target_io); 2777 (void) clone_rq(rq, md, tio, GFP_ATOMIC); 2778 queue_kthread_work(&md->kworker, &tio->work); 2779 } else { 2780 /* Direct call is fine since .queue_rq allows allocations */ 2781 if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) { 2782 /* Undo dm_start_request() before requeuing */ 2783 rq_end_stats(md, rq); 2784 rq_completed(md, rq_data_dir(rq), false); 2785 return BLK_MQ_RQ_QUEUE_BUSY; 2786 } 2787 } 2788 2789 return BLK_MQ_RQ_QUEUE_OK; 2790 } 2791 2792 static struct blk_mq_ops dm_mq_ops = { 2793 .queue_rq = dm_mq_queue_rq, 2794 .map_queue = blk_mq_map_queue, 2795 .complete = dm_softirq_done, 2796 .init_request = dm_mq_init_request, 2797 }; 2798 2799 static int dm_init_request_based_blk_mq_queue(struct mapped_device *md) 2800 { 2801 unsigned md_type = dm_get_md_type(md); 2802 struct request_queue *q; 2803 int err; 2804 2805 memset(&md->tag_set, 0, sizeof(md->tag_set)); 2806 md->tag_set.ops = &dm_mq_ops; 2807 md->tag_set.queue_depth = BLKDEV_MAX_RQ; 2808 md->tag_set.numa_node = NUMA_NO_NODE; 2809 md->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; 2810 md->tag_set.nr_hw_queues = 1; 2811 if (md_type == DM_TYPE_REQUEST_BASED) { 2812 /* make the memory for non-blk-mq clone part of the pdu */ 2813 md->tag_set.cmd_size = sizeof(struct dm_rq_target_io) + sizeof(struct request); 2814 } else 2815 md->tag_set.cmd_size = sizeof(struct dm_rq_target_io); 2816 md->tag_set.driver_data = md; 2817 2818 err = blk_mq_alloc_tag_set(&md->tag_set); 2819 if (err) 2820 return err; 2821 2822 q = blk_mq_init_allocated_queue(&md->tag_set, md->queue); 2823 if (IS_ERR(q)) { 2824 err = PTR_ERR(q); 2825 goto out_tag_set; 2826 } 2827 md->queue = q; 2828 dm_init_md_queue(md); 2829 2830 /* backfill 'mq' sysfs registration normally done in blk_register_queue */ 2831 blk_mq_register_disk(md->disk); 2832 2833 if (md_type == DM_TYPE_REQUEST_BASED) 2834 init_rq_based_worker_thread(md); 2835 2836 return 0; 2837 2838 out_tag_set: 2839 blk_mq_free_tag_set(&md->tag_set); 2840 return err; 2841 } 2842 2843 static unsigned filter_md_type(unsigned type, struct mapped_device *md) 2844 { 2845 if (type == DM_TYPE_BIO_BASED) 2846 return type; 2847 2848 return !md->use_blk_mq ? DM_TYPE_REQUEST_BASED : DM_TYPE_MQ_REQUEST_BASED; 2849 } 2850 2851 /* 2852 * Setup the DM device's queue based on md's type 2853 */ 2854 int dm_setup_md_queue(struct mapped_device *md) 2855 { 2856 int r; 2857 unsigned md_type = filter_md_type(dm_get_md_type(md), md); 2858 2859 switch (md_type) { 2860 case DM_TYPE_REQUEST_BASED: 2861 r = dm_init_request_based_queue(md); 2862 if (r) { 2863 DMWARN("Cannot initialize queue for request-based mapped device"); 2864 return r; 2865 } 2866 break; 2867 case DM_TYPE_MQ_REQUEST_BASED: 2868 r = dm_init_request_based_blk_mq_queue(md); 2869 if (r) { 2870 DMWARN("Cannot initialize queue for request-based blk-mq mapped device"); 2871 return r; 2872 } 2873 break; 2874 case DM_TYPE_BIO_BASED: 2875 dm_init_old_md_queue(md); 2876 blk_queue_make_request(md->queue, dm_make_request); 2877 blk_queue_merge_bvec(md->queue, dm_merge_bvec); 2878 break; 2879 } 2880 2881 return 0; 2882 } 2883 2884 struct mapped_device *dm_get_md(dev_t dev) 2885 { 2886 struct mapped_device *md; 2887 unsigned minor = MINOR(dev); 2888 2889 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 2890 return NULL; 2891 2892 spin_lock(&_minor_lock); 2893 2894 md = idr_find(&_minor_idr, minor); 2895 if (md) { 2896 if ((md == MINOR_ALLOCED || 2897 (MINOR(disk_devt(dm_disk(md))) != minor) || 2898 dm_deleting_md(md) || 2899 test_bit(DMF_FREEING, &md->flags))) { 2900 md = NULL; 2901 goto out; 2902 } 2903 dm_get(md); 2904 } 2905 2906 out: 2907 spin_unlock(&_minor_lock); 2908 2909 return md; 2910 } 2911 EXPORT_SYMBOL_GPL(dm_get_md); 2912 2913 void *dm_get_mdptr(struct mapped_device *md) 2914 { 2915 return md->interface_ptr; 2916 } 2917 2918 void dm_set_mdptr(struct mapped_device *md, void *ptr) 2919 { 2920 md->interface_ptr = ptr; 2921 } 2922 2923 void dm_get(struct mapped_device *md) 2924 { 2925 atomic_inc(&md->holders); 2926 BUG_ON(test_bit(DMF_FREEING, &md->flags)); 2927 } 2928 2929 int dm_hold(struct mapped_device *md) 2930 { 2931 spin_lock(&_minor_lock); 2932 if (test_bit(DMF_FREEING, &md->flags)) { 2933 spin_unlock(&_minor_lock); 2934 return -EBUSY; 2935 } 2936 dm_get(md); 2937 spin_unlock(&_minor_lock); 2938 return 0; 2939 } 2940 EXPORT_SYMBOL_GPL(dm_hold); 2941 2942 const char *dm_device_name(struct mapped_device *md) 2943 { 2944 return md->name; 2945 } 2946 EXPORT_SYMBOL_GPL(dm_device_name); 2947 2948 static void __dm_destroy(struct mapped_device *md, bool wait) 2949 { 2950 struct dm_table *map; 2951 int srcu_idx; 2952 2953 might_sleep(); 2954 2955 map = dm_get_live_table(md, &srcu_idx); 2956 2957 spin_lock(&_minor_lock); 2958 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); 2959 set_bit(DMF_FREEING, &md->flags); 2960 spin_unlock(&_minor_lock); 2961 2962 if (dm_request_based(md) && md->kworker_task) 2963 flush_kthread_worker(&md->kworker); 2964 2965 /* 2966 * Take suspend_lock so that presuspend and postsuspend methods 2967 * do not race with internal suspend. 2968 */ 2969 mutex_lock(&md->suspend_lock); 2970 if (!dm_suspended_md(md)) { 2971 dm_table_presuspend_targets(map); 2972 dm_table_postsuspend_targets(map); 2973 } 2974 mutex_unlock(&md->suspend_lock); 2975 2976 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */ 2977 dm_put_live_table(md, srcu_idx); 2978 2979 /* 2980 * Rare, but there may be I/O requests still going to complete, 2981 * for example. Wait for all references to disappear. 2982 * No one should increment the reference count of the mapped_device, 2983 * after the mapped_device state becomes DMF_FREEING. 2984 */ 2985 if (wait) 2986 while (atomic_read(&md->holders)) 2987 msleep(1); 2988 else if (atomic_read(&md->holders)) 2989 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", 2990 dm_device_name(md), atomic_read(&md->holders)); 2991 2992 dm_sysfs_exit(md); 2993 dm_table_destroy(__unbind(md)); 2994 free_dev(md); 2995 } 2996 2997 void dm_destroy(struct mapped_device *md) 2998 { 2999 __dm_destroy(md, true); 3000 } 3001 3002 void dm_destroy_immediate(struct mapped_device *md) 3003 { 3004 __dm_destroy(md, false); 3005 } 3006 3007 void dm_put(struct mapped_device *md) 3008 { 3009 atomic_dec(&md->holders); 3010 } 3011 EXPORT_SYMBOL_GPL(dm_put); 3012 3013 static int dm_wait_for_completion(struct mapped_device *md, int interruptible) 3014 { 3015 int r = 0; 3016 DECLARE_WAITQUEUE(wait, current); 3017 3018 add_wait_queue(&md->wait, &wait); 3019 3020 while (1) { 3021 set_current_state(interruptible); 3022 3023 if (!md_in_flight(md)) 3024 break; 3025 3026 if (interruptible == TASK_INTERRUPTIBLE && 3027 signal_pending(current)) { 3028 r = -EINTR; 3029 break; 3030 } 3031 3032 io_schedule(); 3033 } 3034 set_current_state(TASK_RUNNING); 3035 3036 remove_wait_queue(&md->wait, &wait); 3037 3038 return r; 3039 } 3040 3041 /* 3042 * Process the deferred bios 3043 */ 3044 static void dm_wq_work(struct work_struct *work) 3045 { 3046 struct mapped_device *md = container_of(work, struct mapped_device, 3047 work); 3048 struct bio *c; 3049 int srcu_idx; 3050 struct dm_table *map; 3051 3052 map = dm_get_live_table(md, &srcu_idx); 3053 3054 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 3055 spin_lock_irq(&md->deferred_lock); 3056 c = bio_list_pop(&md->deferred); 3057 spin_unlock_irq(&md->deferred_lock); 3058 3059 if (!c) 3060 break; 3061 3062 if (dm_request_based(md)) 3063 generic_make_request(c); 3064 else 3065 __split_and_process_bio(md, map, c); 3066 } 3067 3068 dm_put_live_table(md, srcu_idx); 3069 } 3070 3071 static void dm_queue_flush(struct mapped_device *md) 3072 { 3073 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 3074 smp_mb__after_atomic(); 3075 queue_work(md->wq, &md->work); 3076 } 3077 3078 /* 3079 * Swap in a new table, returning the old one for the caller to destroy. 3080 */ 3081 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) 3082 { 3083 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); 3084 struct queue_limits limits; 3085 int r; 3086 3087 mutex_lock(&md->suspend_lock); 3088 3089 /* device must be suspended */ 3090 if (!dm_suspended_md(md)) 3091 goto out; 3092 3093 /* 3094 * If the new table has no data devices, retain the existing limits. 3095 * This helps multipath with queue_if_no_path if all paths disappear, 3096 * then new I/O is queued based on these limits, and then some paths 3097 * reappear. 3098 */ 3099 if (dm_table_has_no_data_devices(table)) { 3100 live_map = dm_get_live_table_fast(md); 3101 if (live_map) 3102 limits = md->queue->limits; 3103 dm_put_live_table_fast(md); 3104 } 3105 3106 if (!live_map) { 3107 r = dm_calculate_queue_limits(table, &limits); 3108 if (r) { 3109 map = ERR_PTR(r); 3110 goto out; 3111 } 3112 } 3113 3114 map = __bind(md, table, &limits); 3115 3116 out: 3117 mutex_unlock(&md->suspend_lock); 3118 return map; 3119 } 3120 3121 /* 3122 * Functions to lock and unlock any filesystem running on the 3123 * device. 3124 */ 3125 static int lock_fs(struct mapped_device *md) 3126 { 3127 int r; 3128 3129 WARN_ON(md->frozen_sb); 3130 3131 md->frozen_sb = freeze_bdev(md->bdev); 3132 if (IS_ERR(md->frozen_sb)) { 3133 r = PTR_ERR(md->frozen_sb); 3134 md->frozen_sb = NULL; 3135 return r; 3136 } 3137 3138 set_bit(DMF_FROZEN, &md->flags); 3139 3140 return 0; 3141 } 3142 3143 static void unlock_fs(struct mapped_device *md) 3144 { 3145 if (!test_bit(DMF_FROZEN, &md->flags)) 3146 return; 3147 3148 thaw_bdev(md->bdev, md->frozen_sb); 3149 md->frozen_sb = NULL; 3150 clear_bit(DMF_FROZEN, &md->flags); 3151 } 3152 3153 /* 3154 * If __dm_suspend returns 0, the device is completely quiescent 3155 * now. There is no request-processing activity. All new requests 3156 * are being added to md->deferred list. 3157 * 3158 * Caller must hold md->suspend_lock 3159 */ 3160 static int __dm_suspend(struct mapped_device *md, struct dm_table *map, 3161 unsigned suspend_flags, int interruptible) 3162 { 3163 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; 3164 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; 3165 int r; 3166 3167 /* 3168 * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 3169 * This flag is cleared before dm_suspend returns. 3170 */ 3171 if (noflush) 3172 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 3173 3174 /* 3175 * This gets reverted if there's an error later and the targets 3176 * provide the .presuspend_undo hook. 3177 */ 3178 dm_table_presuspend_targets(map); 3179 3180 /* 3181 * Flush I/O to the device. 3182 * Any I/O submitted after lock_fs() may not be flushed. 3183 * noflush takes precedence over do_lockfs. 3184 * (lock_fs() flushes I/Os and waits for them to complete.) 3185 */ 3186 if (!noflush && do_lockfs) { 3187 r = lock_fs(md); 3188 if (r) { 3189 dm_table_presuspend_undo_targets(map); 3190 return r; 3191 } 3192 } 3193 3194 /* 3195 * Here we must make sure that no processes are submitting requests 3196 * to target drivers i.e. no one may be executing 3197 * __split_and_process_bio. This is called from dm_request and 3198 * dm_wq_work. 3199 * 3200 * To get all processes out of __split_and_process_bio in dm_request, 3201 * we take the write lock. To prevent any process from reentering 3202 * __split_and_process_bio from dm_request and quiesce the thread 3203 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call 3204 * flush_workqueue(md->wq). 3205 */ 3206 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 3207 if (map) 3208 synchronize_srcu(&md->io_barrier); 3209 3210 /* 3211 * Stop md->queue before flushing md->wq in case request-based 3212 * dm defers requests to md->wq from md->queue. 3213 */ 3214 if (dm_request_based(md)) { 3215 stop_queue(md->queue); 3216 if (md->kworker_task) 3217 flush_kthread_worker(&md->kworker); 3218 } 3219 3220 flush_workqueue(md->wq); 3221 3222 /* 3223 * At this point no more requests are entering target request routines. 3224 * We call dm_wait_for_completion to wait for all existing requests 3225 * to finish. 3226 */ 3227 r = dm_wait_for_completion(md, interruptible); 3228 3229 if (noflush) 3230 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 3231 if (map) 3232 synchronize_srcu(&md->io_barrier); 3233 3234 /* were we interrupted ? */ 3235 if (r < 0) { 3236 dm_queue_flush(md); 3237 3238 if (dm_request_based(md)) 3239 start_queue(md->queue); 3240 3241 unlock_fs(md); 3242 dm_table_presuspend_undo_targets(map); 3243 /* pushback list is already flushed, so skip flush */ 3244 } 3245 3246 return r; 3247 } 3248 3249 /* 3250 * We need to be able to change a mapping table under a mounted 3251 * filesystem. For example we might want to move some data in 3252 * the background. Before the table can be swapped with 3253 * dm_bind_table, dm_suspend must be called to flush any in 3254 * flight bios and ensure that any further io gets deferred. 3255 */ 3256 /* 3257 * Suspend mechanism in request-based dm. 3258 * 3259 * 1. Flush all I/Os by lock_fs() if needed. 3260 * 2. Stop dispatching any I/O by stopping the request_queue. 3261 * 3. Wait for all in-flight I/Os to be completed or requeued. 3262 * 3263 * To abort suspend, start the request_queue. 3264 */ 3265 int dm_suspend(struct mapped_device *md, unsigned suspend_flags) 3266 { 3267 struct dm_table *map = NULL; 3268 int r = 0; 3269 3270 retry: 3271 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 3272 3273 if (dm_suspended_md(md)) { 3274 r = -EINVAL; 3275 goto out_unlock; 3276 } 3277 3278 if (dm_suspended_internally_md(md)) { 3279 /* already internally suspended, wait for internal resume */ 3280 mutex_unlock(&md->suspend_lock); 3281 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 3282 if (r) 3283 return r; 3284 goto retry; 3285 } 3286 3287 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 3288 3289 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE); 3290 if (r) 3291 goto out_unlock; 3292 3293 set_bit(DMF_SUSPENDED, &md->flags); 3294 3295 dm_table_postsuspend_targets(map); 3296 3297 out_unlock: 3298 mutex_unlock(&md->suspend_lock); 3299 return r; 3300 } 3301 3302 static int __dm_resume(struct mapped_device *md, struct dm_table *map) 3303 { 3304 if (map) { 3305 int r = dm_table_resume_targets(map); 3306 if (r) 3307 return r; 3308 } 3309 3310 dm_queue_flush(md); 3311 3312 /* 3313 * Flushing deferred I/Os must be done after targets are resumed 3314 * so that mapping of targets can work correctly. 3315 * Request-based dm is queueing the deferred I/Os in its request_queue. 3316 */ 3317 if (dm_request_based(md)) 3318 start_queue(md->queue); 3319 3320 unlock_fs(md); 3321 3322 return 0; 3323 } 3324 3325 int dm_resume(struct mapped_device *md) 3326 { 3327 int r = -EINVAL; 3328 struct dm_table *map = NULL; 3329 3330 retry: 3331 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 3332 3333 if (!dm_suspended_md(md)) 3334 goto out; 3335 3336 if (dm_suspended_internally_md(md)) { 3337 /* already internally suspended, wait for internal resume */ 3338 mutex_unlock(&md->suspend_lock); 3339 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 3340 if (r) 3341 return r; 3342 goto retry; 3343 } 3344 3345 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 3346 if (!map || !dm_table_get_size(map)) 3347 goto out; 3348 3349 r = __dm_resume(md, map); 3350 if (r) 3351 goto out; 3352 3353 clear_bit(DMF_SUSPENDED, &md->flags); 3354 3355 r = 0; 3356 out: 3357 mutex_unlock(&md->suspend_lock); 3358 3359 return r; 3360 } 3361 3362 /* 3363 * Internal suspend/resume works like userspace-driven suspend. It waits 3364 * until all bios finish and prevents issuing new bios to the target drivers. 3365 * It may be used only from the kernel. 3366 */ 3367 3368 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags) 3369 { 3370 struct dm_table *map = NULL; 3371 3372 if (md->internal_suspend_count++) 3373 return; /* nested internal suspend */ 3374 3375 if (dm_suspended_md(md)) { 3376 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3377 return; /* nest suspend */ 3378 } 3379 3380 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 3381 3382 /* 3383 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is 3384 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend 3385 * would require changing .presuspend to return an error -- avoid this 3386 * until there is a need for more elaborate variants of internal suspend. 3387 */ 3388 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE); 3389 3390 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3391 3392 dm_table_postsuspend_targets(map); 3393 } 3394 3395 static void __dm_internal_resume(struct mapped_device *md) 3396 { 3397 BUG_ON(!md->internal_suspend_count); 3398 3399 if (--md->internal_suspend_count) 3400 return; /* resume from nested internal suspend */ 3401 3402 if (dm_suspended_md(md)) 3403 goto done; /* resume from nested suspend */ 3404 3405 /* 3406 * NOTE: existing callers don't need to call dm_table_resume_targets 3407 * (which may fail -- so best to avoid it for now by passing NULL map) 3408 */ 3409 (void) __dm_resume(md, NULL); 3410 3411 done: 3412 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3413 smp_mb__after_atomic(); 3414 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); 3415 } 3416 3417 void dm_internal_suspend_noflush(struct mapped_device *md) 3418 { 3419 mutex_lock(&md->suspend_lock); 3420 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); 3421 mutex_unlock(&md->suspend_lock); 3422 } 3423 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); 3424 3425 void dm_internal_resume(struct mapped_device *md) 3426 { 3427 mutex_lock(&md->suspend_lock); 3428 __dm_internal_resume(md); 3429 mutex_unlock(&md->suspend_lock); 3430 } 3431 EXPORT_SYMBOL_GPL(dm_internal_resume); 3432 3433 /* 3434 * Fast variants of internal suspend/resume hold md->suspend_lock, 3435 * which prevents interaction with userspace-driven suspend. 3436 */ 3437 3438 void dm_internal_suspend_fast(struct mapped_device *md) 3439 { 3440 mutex_lock(&md->suspend_lock); 3441 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 3442 return; 3443 3444 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 3445 synchronize_srcu(&md->io_barrier); 3446 flush_workqueue(md->wq); 3447 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); 3448 } 3449 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); 3450 3451 void dm_internal_resume_fast(struct mapped_device *md) 3452 { 3453 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 3454 goto done; 3455 3456 dm_queue_flush(md); 3457 3458 done: 3459 mutex_unlock(&md->suspend_lock); 3460 } 3461 EXPORT_SYMBOL_GPL(dm_internal_resume_fast); 3462 3463 /*----------------------------------------------------------------- 3464 * Event notification. 3465 *---------------------------------------------------------------*/ 3466 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, 3467 unsigned cookie) 3468 { 3469 char udev_cookie[DM_COOKIE_LENGTH]; 3470 char *envp[] = { udev_cookie, NULL }; 3471 3472 if (!cookie) 3473 return kobject_uevent(&disk_to_dev(md->disk)->kobj, action); 3474 else { 3475 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", 3476 DM_COOKIE_ENV_VAR_NAME, cookie); 3477 return kobject_uevent_env(&disk_to_dev(md->disk)->kobj, 3478 action, envp); 3479 } 3480 } 3481 3482 uint32_t dm_next_uevent_seq(struct mapped_device *md) 3483 { 3484 return atomic_add_return(1, &md->uevent_seq); 3485 } 3486 3487 uint32_t dm_get_event_nr(struct mapped_device *md) 3488 { 3489 return atomic_read(&md->event_nr); 3490 } 3491 3492 int dm_wait_event(struct mapped_device *md, int event_nr) 3493 { 3494 return wait_event_interruptible(md->eventq, 3495 (event_nr != atomic_read(&md->event_nr))); 3496 } 3497 3498 void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 3499 { 3500 unsigned long flags; 3501 3502 spin_lock_irqsave(&md->uevent_lock, flags); 3503 list_add(elist, &md->uevent_list); 3504 spin_unlock_irqrestore(&md->uevent_lock, flags); 3505 } 3506 3507 /* 3508 * The gendisk is only valid as long as you have a reference 3509 * count on 'md'. 3510 */ 3511 struct gendisk *dm_disk(struct mapped_device *md) 3512 { 3513 return md->disk; 3514 } 3515 EXPORT_SYMBOL_GPL(dm_disk); 3516 3517 struct kobject *dm_kobject(struct mapped_device *md) 3518 { 3519 return &md->kobj_holder.kobj; 3520 } 3521 3522 struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 3523 { 3524 struct mapped_device *md; 3525 3526 md = container_of(kobj, struct mapped_device, kobj_holder.kobj); 3527 3528 if (test_bit(DMF_FREEING, &md->flags) || 3529 dm_deleting_md(md)) 3530 return NULL; 3531 3532 dm_get(md); 3533 return md; 3534 } 3535 3536 int dm_suspended_md(struct mapped_device *md) 3537 { 3538 return test_bit(DMF_SUSPENDED, &md->flags); 3539 } 3540 3541 int dm_suspended_internally_md(struct mapped_device *md) 3542 { 3543 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3544 } 3545 3546 int dm_test_deferred_remove_flag(struct mapped_device *md) 3547 { 3548 return test_bit(DMF_DEFERRED_REMOVE, &md->flags); 3549 } 3550 3551 int dm_suspended(struct dm_target *ti) 3552 { 3553 return dm_suspended_md(dm_table_get_md(ti->table)); 3554 } 3555 EXPORT_SYMBOL_GPL(dm_suspended); 3556 3557 int dm_noflush_suspending(struct dm_target *ti) 3558 { 3559 return __noflush_suspending(dm_table_get_md(ti->table)); 3560 } 3561 EXPORT_SYMBOL_GPL(dm_noflush_suspending); 3562 3563 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned type, 3564 unsigned integrity, unsigned per_bio_data_size) 3565 { 3566 struct dm_md_mempools *pools = kzalloc(sizeof(*pools), GFP_KERNEL); 3567 struct kmem_cache *cachep = NULL; 3568 unsigned int pool_size = 0; 3569 unsigned int front_pad; 3570 3571 if (!pools) 3572 return NULL; 3573 3574 type = filter_md_type(type, md); 3575 3576 switch (type) { 3577 case DM_TYPE_BIO_BASED: 3578 cachep = _io_cache; 3579 pool_size = dm_get_reserved_bio_based_ios(); 3580 front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone); 3581 break; 3582 case DM_TYPE_REQUEST_BASED: 3583 cachep = _rq_tio_cache; 3584 pool_size = dm_get_reserved_rq_based_ios(); 3585 pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache); 3586 if (!pools->rq_pool) 3587 goto out; 3588 /* fall through to setup remaining rq-based pools */ 3589 case DM_TYPE_MQ_REQUEST_BASED: 3590 if (!pool_size) 3591 pool_size = dm_get_reserved_rq_based_ios(); 3592 front_pad = offsetof(struct dm_rq_clone_bio_info, clone); 3593 /* per_bio_data_size is not used. See __bind_mempools(). */ 3594 WARN_ON(per_bio_data_size != 0); 3595 break; 3596 default: 3597 BUG(); 3598 } 3599 3600 if (cachep) { 3601 pools->io_pool = mempool_create_slab_pool(pool_size, cachep); 3602 if (!pools->io_pool) 3603 goto out; 3604 } 3605 3606 pools->bs = bioset_create_nobvec(pool_size, front_pad); 3607 if (!pools->bs) 3608 goto out; 3609 3610 if (integrity && bioset_integrity_create(pools->bs, pool_size)) 3611 goto out; 3612 3613 return pools; 3614 3615 out: 3616 dm_free_md_mempools(pools); 3617 3618 return NULL; 3619 } 3620 3621 void dm_free_md_mempools(struct dm_md_mempools *pools) 3622 { 3623 if (!pools) 3624 return; 3625 3626 if (pools->io_pool) 3627 mempool_destroy(pools->io_pool); 3628 3629 if (pools->rq_pool) 3630 mempool_destroy(pools->rq_pool); 3631 3632 if (pools->bs) 3633 bioset_free(pools->bs); 3634 3635 kfree(pools); 3636 } 3637 3638 static const struct block_device_operations dm_blk_dops = { 3639 .open = dm_blk_open, 3640 .release = dm_blk_close, 3641 .ioctl = dm_blk_ioctl, 3642 .getgeo = dm_blk_getgeo, 3643 .owner = THIS_MODULE 3644 }; 3645 3646 /* 3647 * module hooks 3648 */ 3649 module_init(dm_init); 3650 module_exit(dm_exit); 3651 3652 module_param(major, uint, 0); 3653 MODULE_PARM_DESC(major, "The major number of the device mapper"); 3654 3655 module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR); 3656 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); 3657 3658 module_param(reserved_rq_based_ios, uint, S_IRUGO | S_IWUSR); 3659 MODULE_PARM_DESC(reserved_rq_based_ios, "Reserved IOs in request-based mempools"); 3660 3661 module_param(use_blk_mq, bool, S_IRUGO | S_IWUSR); 3662 MODULE_PARM_DESC(use_blk_mq, "Use block multiqueue for request-based DM devices"); 3663 3664 MODULE_DESCRIPTION(DM_NAME " driver"); 3665 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 3666 MODULE_LICENSE("GPL"); 3667